Merge branch 'develop' into get-single-value-oof

.editorconfig

@@ -9,6 +9,6 @@ trim_trailing_whitespace = true
 charset = utf-8
 
 # python, js indentation settings
-[{*.py,*.js}]
+[{*.py,*.js,*.vue}]
 indent_style = tab
 indent_size = 4

.github/helper/install_dependencies.sh

@@ -2,11 +2,6 @@
 
 set -e
 
-# python "${GITHUB_WORKSPACE}/.github/helper/roulette.py"
-# if [[ $? != 2 ]];then
-#   exit;
-# fi
-
  # install wkhtmltopdf
 wget -O /tmp/wkhtmltox.tar.xz https://github.com/frappe/wkhtmltopdf/raw/master/wkhtmltox-0.12.3_linux-generic-amd64.tar.xz
 tar -xf /tmp/wkhtmltox.tar.xz -C /tmp

.github/helper/roulette.py

@@ -1,56 +1,68 @@
-# if the script ends with exit code 0, then no tests are run further, else all tests are run
+import json
 import os
 import re
 import shlex
 import subprocess
 import sys
+import urllib.request
 
 
+def get_files_list(pr_number, repo="frappe/frappe"):
+	req = urllib.request.Request(f"https://api.github.com/repos/{repo}/pulls/{pr_number}/files")
+	res = urllib.request.urlopen(req)
+	dump = json.loads(res.read().decode('utf8'))
+	return [change["filename"] for change in dump]
+
 def get_output(command, shell=True):
-    print(command)
-    command = shlex.split(command)
-    return subprocess.check_output(command, shell=shell, encoding="utf8").strip()
+	print(command)
+	command = shlex.split(command)
+	return subprocess.check_output(command, shell=shell, encoding="utf8").strip()
 
 def is_py(file):
-    return file.endswith("py")
+	return file.endswith("py")
 
-def is_js(file):
-    return file.endswith("js")
+def is_ci(file):
+	return ".github" in file
+
+def is_frontend_code(file):
+	return file.lower().endswith((".css", ".scss", ".less", ".sass", ".styl", ".js", ".ts", ".vue"))
 
 def is_docs(file):
-    regex = re.compile(r'\.(md|png|jpg|jpeg)$|^.github|LICENSE')
-    return bool(regex.search(file))
+	regex = re.compile(r'\.(md|png|jpg|jpeg|csv)$|^.github|LICENSE')
+	return bool(regex.search(file))
 
 
 if __name__ == "__main__":
-    build_type = os.environ.get("TYPE")
-    before = os.environ.get("BEFORE")
-    after = os.environ.get("AFTER")
-    commit_range = before + '...' + after
-    print("Build Type: {}".format(build_type))
-    print("Commit Range: {}".format(commit_range))
+	files_list = sys.argv[1:]
+	build_type = os.environ.get("TYPE")
+	pr_number = os.environ.get("PR_NUMBER")
+	repo = os.environ.get("REPO_NAME")
 
-    try:
-        files_changed = get_output("git diff --name-only {}".format(commit_range), shell=False)
-    except Exception:
-        sys.exit(2)
+	if not files_list and pr_number:
+		files_list = get_files_list(pr_number=pr_number, repo=repo)
 
-    if "fatal" not in files_changed:
-        files_list = files_changed.split()
-        only_docs_changed = len(list(filter(is_docs, files_list))) == len(files_list)
-        only_js_changed = len(list(filter(is_js, files_list))) == len(files_list)
-        only_py_changed = len(list(filter(is_py, files_list))) == len(files_list)
+	if not files_list:
+		print("No files' changes detected. Build is shutting")
+		sys.exit(0)
 
-        if only_docs_changed:
-            print("Only docs were updated, stopping build process.")
-            sys.exit(0)
+	ci_files_changed = any(f for f in files_list if is_ci(f))
+	only_docs_changed = len(list(filter(is_docs, files_list))) == len(files_list)
+	only_frontend_code_changed = len(list(filter(is_frontend_code, files_list))) == len(files_list)
+	only_py_changed = len(list(filter(is_py, files_list))) == len(files_list)
 
-        if only_js_changed and build_type == "server":
-            print("Only JavaScript code was updated; Stopping Python build process.")
-            sys.exit(0)
+	if ci_files_changed:
+		print("CI related files were updated, running all build processes.")
 
-        if only_py_changed and build_type == "ui":
-            print("Only Python code was updated, stopping Cypress build process.")
-            sys.exit(0)
+	elif only_docs_changed:
+		print("Only docs were updated, stopping build process.")
+		sys.exit(0)
 
-    sys.exit(2)
+	elif only_frontend_code_changed and build_type == "server":
+		print("Only Frontend code was updated; Stopping Python build process.")
+		sys.exit(0)
+
+	elif only_py_changed and build_type == "ui":
+		print("Only Python code was updated, stopping Cypress build process.")
+		sys.exit(0)
+
+	os.system('echo "::set-output name=build::strawberry"')

.github/workflows/patch-mariadb-tests.yml

@@ -1,11 +1,11 @@
 name: Patch
 
-on:
-  pull_request:
-    paths-ignore:
-      - '**.js'
-      - '**.md'
-  workflow_dispatch:
+on: [pull_request, workflow_dispatch]
+
+
+concurrency:
+  group: patch-mariadb-develop-${{ github.event.number }}
+  cancel-in-progress: true
 
 jobs:
   test:
@@ -31,10 +31,21 @@ jobs:
         with:
           python-version: 3.7
 
+      - name: Check if build should be run
+        id: check-build
+        run: |
+          python "${GITHUB_WORKSPACE}/.github/helper/roulette.py"
+        env:
+          TYPE: "server"
+          PR_NUMBER: ${{ github.event.number }}
+          REPO_NAME: ${{ github.repository }}
+
       - name: Add to Hosts
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         run: echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts
 
       - name: Cache pip
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         uses: actions/cache@v2
         with:
           path: ~/.cache/pip
@@ -44,6 +55,7 @@ jobs:
             ${{ runner.os }}-
 
       - name: Cache node modules
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         uses: actions/cache@v2
         env:
           cache-name: cache-node-modules
@@ -56,10 +68,12 @@ jobs:
             ${{ runner.os }}-
 
       - name: Get yarn cache directory path
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         id: yarn-cache-dir-path
         run: echo "::set-output name=dir::$(yarn cache dir)"
 
       - uses: actions/cache@v2
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         id: yarn-cache
         with:
           path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
@@ -68,6 +82,7 @@ jobs:
             ${{ runner.os }}-yarn-
 
       - name: Install Dependencies
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh
         env:
           BEFORE: ${{ env.GITHUB_EVENT_PATH.before }}
@@ -75,12 +90,14 @@ jobs:
           TYPE: server
 
       - name: Install
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh
         env:
           DB: mariadb
           TYPE: server
 
       - name: Run Patch Tests
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         run: |
           cd ~/frappe-bench/
           wget https://frappeframework.com/files/v10-frappe.sql.gz

.github/workflows/server-mariadb-tests.yml

@@ -2,15 +2,14 @@ name: Server
 
 on:
   pull_request:
-    paths-ignore:
-      - '**.js'
-      - '**.md'
   workflow_dispatch:
   push:
     branches: [ develop ]
-    paths-ignore:
-      - '**.js'
-      - '**.md'
+
+concurrency:
+  group: server-mariadb-develop-${{ github.event.number }}
+  cancel-in-progress: true
+
 
 jobs:
   test:
@@ -41,17 +40,29 @@ jobs:
         with:
           python-version: 3.7
 
+      - name: Check if build should be run
+        id: check-build
+        run: |
+          python "${GITHUB_WORKSPACE}/.github/helper/roulette.py"
+        env:
+          TYPE: "server"
+          PR_NUMBER: ${{ github.event.number }}
+          REPO_NAME: ${{ github.repository }}
+
       - uses: actions/setup-node@v2
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         with:
           node-version: 14
           check-latest: true
 
       - name: Add to Hosts
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         run: |
           echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts
           echo "127.0.0.1 test_site_producer" | sudo tee -a /etc/hosts
 
       - name: Cache pip
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         uses: actions/cache@v2
         with:
           path: ~/.cache/pip
@@ -61,6 +72,7 @@ jobs:
             ${{ runner.os }}-
 
       - name: Cache node modules
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         uses: actions/cache@v2
         env:
           cache-name: cache-node-modules
@@ -73,10 +85,12 @@ jobs:
             ${{ runner.os }}-
 
       - name: Get yarn cache directory path
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         id: yarn-cache-dir-path
         run: echo "::set-output name=dir::$(yarn cache dir)"
 
       - uses: actions/cache@v2
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         id: yarn-cache
         with:
           path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
@@ -85,6 +99,7 @@ jobs:
             ${{ runner.os }}-yarn-
 
       - name: Install Dependencies
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh
         env:
           BEFORE: ${{ env.GITHUB_EVENT_PATH.before }}
@@ -92,18 +107,22 @@ jobs:
           TYPE: server
 
       - name: Install
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh
         env:
           DB: mariadb
           TYPE: server
 
       - name: Run Tests
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         run: cd ~/frappe-bench/ && bench --site test_site run-parallel-tests --use-orchestrator --with-coverage
         env:
           CI_BUILD_ID: ${{ github.run_id }}
           ORCHESTRATOR_URL: http://test-orchestrator.frappe.io
 
       - name: Upload Coverage Data
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
+        id: upload-coverage-data
         run: |
           cp ~/frappe-bench/sites/.coverage ${GITHUB_WORKSPACE}
           cd ${GITHUB_WORKSPACE}
@@ -117,16 +136,29 @@ jobs:
           COVERALLS_SERVICE_NAME: ${{ github.event_name == 'pull_request' && 'github' || 'github-actions' }}
           COVERALLS_PARALLEL: true
 
+      - run: echo ${{ steps.check-build.outputs.build }} > guess-the-fruit.txt
+      - uses: actions/upload-artifact@v1
+        with:
+          name: fruit
+          path: guess-the-fruit.txt
+
   coveralls:
     name: Coverage Wrap Up
     needs: test
     container: python:3-slim
     runs-on: ubuntu-18.04
     steps:
+      - uses: actions/download-artifact@v1
+        with:
+          name: fruit
+      - run: echo "WILDCARD=$(cat fruit/guess-the-fruit.txt)" >> $GITHUB_ENV
+
       - name: Clone
+        if: ${{ env.WILDCARD == 'strawberry' }}
         uses: actions/checkout@v2
 
       - name: Coveralls Finished
+        if: ${{ env.WILDCARD == 'strawberry' }}
         run: |
           cd ${GITHUB_WORKSPACE}
           pip3 install coverage==5.5

.github/workflows/server-postgres-tests.yml

@@ -2,11 +2,12 @@ name: Server
 
 on:
   pull_request:
-    paths-ignore:
-      - '**.js'
-      - '**.md'
   workflow_dispatch:
 
+concurrency:
+  group: server-postgres-develop-${{ github.event.number }}
+  cancel-in-progress: true
+
 jobs:
   test:
     runs-on: ubuntu-18.04
@@ -40,17 +41,29 @@ jobs:
         with:
           python-version: 3.7
 
+      - name: Check if build should be run
+        id: check-build
+        run: |
+          python "${GITHUB_WORKSPACE}/.github/helper/roulette.py"
+        env:
+          TYPE: "server"
+          PR_NUMBER: ${{ github.event.number }}
+          REPO_NAME: ${{ github.repository }}
+
       - uses: actions/setup-node@v2
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         with:
           node-version: '14'
           check-latest: true
 
       - name: Add to Hosts
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         run: |
           echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts
           echo "127.0.0.1 test_site_producer" | sudo tee -a /etc/hosts
 
       - name: Cache pip
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         uses: actions/cache@v2
         with:
           path: ~/.cache/pip
@@ -60,6 +73,7 @@ jobs:
             ${{ runner.os }}-
 
       - name: Cache node modules
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         uses: actions/cache@v2
         env:
           cache-name: cache-node-modules
@@ -72,10 +86,12 @@ jobs:
             ${{ runner.os }}-
 
       - name: Get yarn cache directory path
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         id: yarn-cache-dir-path
         run: echo "::set-output name=dir::$(yarn cache dir)"
 
       - uses: actions/cache@v2
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         id: yarn-cache
         with:
           path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
@@ -84,6 +100,7 @@ jobs:
             ${{ runner.os }}-yarn-
 
       - name: Install Dependencies
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh
         env:
           BEFORE: ${{ env.GITHUB_EVENT_PATH.before }}
@@ -91,12 +108,14 @@ jobs:
           TYPE: server
 
       - name: Install
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh
         env:
           DB: postgres
           TYPE: server
 
       - name: Run Tests
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         run: cd ~/frappe-bench/ && bench --site test_site run-parallel-tests --use-orchestrator
         env:
           CI_BUILD_ID: ${{ github.run_id }}

.github/workflows/ui-tests.yml

@@ -2,12 +2,14 @@ name: UI
 
 on:
   pull_request:
-    paths-ignore:
-      - '**.md'
   workflow_dispatch:
   push:
     branches: [ develop ]
 
+concurrency:
+  group: ui-develop-${{ github.event.number }}
+  cancel-in-progress: true
+
 jobs:
   test:
     runs-on: ubuntu-18.04
@@ -37,17 +39,29 @@ jobs:
         with:
           python-version: 3.7
 
+      - name: Check if build should be run
+        id: check-build
+        run: |
+          python "${GITHUB_WORKSPACE}/.github/helper/roulette.py"
+        env:
+          TYPE: "ui"
+          PR_NUMBER: ${{ github.event.number }}
+          REPO_NAME: ${{ github.repository }}
+
       - uses: actions/setup-node@v2
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         with:
           node-version: 14
           check-latest: true
 
       - name: Add to Hosts
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         run: |
           echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts
           echo "127.0.0.1 test_site_producer" | sudo tee -a /etc/hosts
 
       - name: Cache pip
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         uses: actions/cache@v2
         with:
           path: ~/.cache/pip
@@ -57,6 +71,7 @@ jobs:
             ${{ runner.os }}-
 
       - name: Cache node modules
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         uses: actions/cache@v2
         env:
           cache-name: cache-node-modules
@@ -69,10 +84,12 @@ jobs:
             ${{ runner.os }}-
 
       - name: Get yarn cache directory path
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         id: yarn-cache-dir-path
         run: echo "::set-output name=dir::$(yarn cache dir)"
 
       - uses: actions/cache@v2
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         id: yarn-cache
         with:
           path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
@@ -81,6 +98,7 @@ jobs:
             ${{ runner.os }}-yarn-
 
       - name: Cache cypress binary
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         uses: actions/cache@v2
         with:
           path: ~/.cache
@@ -90,6 +108,7 @@ jobs:
             ${{ runner.os }}-
 
       - name: Install Dependencies
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh
         env:
           BEFORE: ${{ env.GITHUB_EVENT_PATH.before }}
@@ -97,15 +116,18 @@ jobs:
           TYPE: ui
 
       - name: Install
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh
         env:
           DB: mariadb
           TYPE: ui
 
       - name: Site Setup
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         run: cd ~/frappe-bench/ && bench --site test_site execute frappe.utils.install.complete_setup_wizard
 
       - name: UI Tests
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         run: cd ~/frappe-bench/ && bench --site test_site run-ui-tests frappe --headless --parallel --ci-build-id $GITHUB_RUN_ID
         env:
           CYPRESS_RECORD_KEY: 4a48f41c-11b3-425b-aa88-c58048fa69eb

.mergify.yml

@@ -1,9 +1,11 @@
 pull_request_rules:
   - name: Auto-close PRs on stable branch
     conditions:
-      - or:
-        - base=version-13
-        - base=version-12
+      - and:
+        - author!=surajshetty3416
+        - or:
+          - base=version-13
+          - base=version-12
     actions:
       close:
       comment:

CODEOWNERS

@@ -7,10 +7,13 @@
 templates/				@surajshetty3416
 www/							@surajshetty3416
 integrations/			@leela
-patches/					@surajshetty3416
+patches/					@surajshetty3416 @gavindsouza
 email/						@leela
 event_streaming/	@ruchamahabal
 data_import*			@netchampfaris
 core/							@surajshetty3416
+database          @gavindsouza
+model             @gavindsouza
 requirements.txt	@gavindsouza
 commands/					@gavindsouza
+workspace         @shariquerik

cypress/integration/dashboard_links.js

@@ -0,0 +1,63 @@
+context('Dashboard links', () => {
+	before(() => {
+		cy.visit('/login');
+		cy.login();
+	});
+
+	it('Adding a new contact, checking for the counter on the dashboard and deleting the created contact', () => {
+		cy.visit('/app/contact');
+		cy.clear_filters();
+
+		cy.visit('/app/user');
+		cy.get('.list-row-col > .level-item > .ellipsis').eq(0).click();
+
+		//To check if initially the dashboard contains only the "Contact" link and there is no counter
+		cy.get('[data-doctype="Contact"]').should('contain', 'Contact');
+
+		//Adding a new contact
+		cy.get('.btn[data-doctype="Contact"]').click();
+		cy.get('[data-doctype="Contact"][data-fieldname="first_name"]').type('Admin');
+		cy.findByRole('button', {name: 'Save'}).click();
+		cy.visit('/app/user');
+		cy.get('.list-row-col > .level-item > .ellipsis').eq(0).click();
+
+		//To check if the counter for contact doc is "1" after adding the contact
+		cy.get('[data-doctype="Contact"] > .count').should('contain', '1');
+		cy.get('[data-doctype="Contact"]').contains('Contact').click();
+
+		//Deleting the newly created contact
+		cy.visit('/app/contact');
+		cy.get('.list-subject > .select-like > .list-row-checkbox').eq(0).click();
+		cy.findByRole('button', {name: 'Actions'}).click();
+		cy.get('.actions-btn-group [data-label="Delete"]').click();
+		cy.findByRole('button', {name: 'Yes'}).click({delay: 700});
+
+
+		//To check if the counter from the "Contact" doc link is removed
+		cy.wait(700);
+		cy.visit('/app/user');
+		cy.get('.list-row-col > .level-item > .ellipsis').eq(0).click();
+		cy.get('[data-doctype="Contact"]').should('contain', 'Contact');
+	});
+
+	it('Report link in dashboard', () => {
+		cy.visit('/app/user');
+		cy.visit('/app/user/Administrator');
+		cy.get('[data-doctype="Contact"]').should('contain', 'Contact');
+		cy.findByText('Connections');
+		cy.window()
+			.its('cur_frm')
+			.then(cur_frm => {
+				cur_frm.dashboard.data.reports = [
+					{
+						'label': 'Reports',
+						'items': ['Permitted Documents For User']
+					}
+				];
+				cur_frm.dashboard.render_report_links();
+				cy.get('[data-report="Permitted Documents For User"]').contains('Permitted Documents For User').click();
+				cy.findByText('Permitted Documents For User');
+				cy.findByPlaceholderText('User').should("have.value", "Administrator");
+			});
+	});
+});

cypress/integration/datetime_field_form_validation.js

@@ -0,0 +1,19 @@
+context('Datetime Field Validation', () => {
+	before(() => {
+		cy.login();
+		cy.visit('/app/communication');
+		cy.window().its('frappe').then(frappe => {
+			frappe.call("frappe.tests.ui_test_helpers.create_communication_records");
+		});
+	});
+
+	// validating datetime field value when value is set from backend and get validated on form load.
+	it('datetime field form validation', () => {
+		cy.visit('/app/communication');
+		cy.get('a[title="Test Form Communication 1"]').invoke('attr', 'data-name')
+			.then((name) => {
+				cy.visit(`/app/communication/${name}`);
+				cy.get('.indicator-pill').should('contain', 'Open').should('have.class', 'red');
+			});
+	});
+});

cypress/integration/folder_navigation.js

@@ -0,0 +1,79 @@
+context('Folder Navigation', () => {
+	before(() => {
+		cy.visit('/login');
+		cy.login();
+		cy.visit('/app/file');
+	});
+
+	it('Adding Folders', () => {
+		//Adding filter to go into the home folder
+		cy.get('.filter-selector > .btn').findByText('1 filter').click();
+		cy.findByRole('button', {name: 'Clear Filters'}).click();
+		cy.get('.filter-action-buttons > .text-muted').findByText('+ Add a Filter').click();
+		cy.get('.fieldname-select-area > .awesomplete > .form-control').type('Fol{enter}');
+		cy.get('.filter-field > .form-group > .link-field > .awesomplete > .input-with-feedback').type('Home{enter}');
+		cy.get('.filter-action-buttons > div > .btn-primary').findByText('Apply Filters').click();
+
+		//Adding folder (Test Folder)
+		cy.get('.menu-btn-group > .btn').click();
+		cy.get('.menu-btn-group [data-label="New Folder"]').click();
+		cy.get('form > [data-fieldname="value"]').type('Test Folder');
+		cy.findByRole('button', {name: 'Create'}).click();   
+	});
+
+	it('Navigating the nested folders, checking if the URL formed is correct, checking if the added content in the child folder is correct', () => {
+		//Navigating inside the Attachments folder
+		cy.get('[title="Attachments"] > span').click();
+
+		//To check if the URL formed after visiting the attachments folder is correct
+		cy.location('pathname').should('eq', '/app/file/view/home/Attachments');
+		cy.visit('/app/file/view/home/Attachments');
+
+		//Adding folder inside the attachments folder
+		cy.get('.menu-btn-group > .btn').click();
+		cy.get('.menu-btn-group [data-label="New Folder"]').click();
+		cy.get('form > [data-fieldname="value"]').type('Test Folder');
+		cy.findByRole('button', {name: 'Create'}).click();
+
+		//Navigating inside the added folder in the Attachments folder
+		cy.get('[title="Test Folder"] > span').click();
+
+		//To check if the URL is correct after visiting the Test Folder
+		cy.location('pathname').should('eq', '/app/file/view/home/Attachments/Test%20Folder');
+		cy.visit('/app/file/view/home/Attachments/Test%20Folder');
+
+		//Adding a file inside the Test Folder
+		cy.findByRole('button', {name: 'Add File'}).eq(0).click({force: true});
+		cy.get('.file-uploader').findByText('Link').click();
+		cy.get('.input-group > .form-control').type('https://wallpaperplay.com/walls/full/8/2/b/72402.jpg');
+		cy.findByRole('button', {name: 'Upload'}).click();
+
+		//To check if the added file is present in the Test Folder
+		cy.get('span.level-item > span').should('contain', 'Test Folder');
+		cy.get('.list-row-container').eq(0).should('contain.text', '72402.jpg');
+		cy.get('.list-row-checkbox').eq(0).click();
+
+		//Deleting the added file from the Test folder
+		cy.findByRole('button', {name: 'Actions'}).click();
+		cy.get('.actions-btn-group [data-label="Delete"]').click();
+		cy.wait(700);
+		cy.findByRole('button', {name: 'Yes'}).click();
+		cy.wait(700);
+
+		//Deleting the Test Folder
+		cy.visit('/app/file/view/home/Attachments');
+		cy.get('.list-row-checkbox').eq(0).click();
+		cy.findByRole('button', {name: 'Actions'}).click();
+		cy.get('.actions-btn-group [data-label="Delete"]').click();  
+		cy.findByRole('button', {name: 'Yes'}).click();
+	});
+
+	it('Deleting Test Folder from the home', () => {
+	//Deleting the Test Folder added in the home directory
+		cy.visit('/app/file/view/home');
+		cy.get('.level-left > .list-subject > .list-row-checkbox').eq(0).click({force: true, delay: 500});
+		cy.findByRole('button', {name: 'Actions'}).click();
+		cy.get('.actions-btn-group [data-label="Delete"]').click();
+		cy.findByRole('button', {name: 'Yes'}).click();
+	});
+}); 

cypress/integration/form_tour.js

@@ -20,10 +20,10 @@ context('Form Tour', () => {
 	it('navigates a form tour', () => {
 		open_test_form_tour();
 
-		cy.get('#driver-popover-item').should('be.visible');
+		cy.get('.frappe-driver').should('be.visible');
 		cy.get('.frappe-control[data-fieldname="first_name"]').as('first_name');
 		cy.get('@first_name').should('have.class', 'driver-highlighted-element');
-		cy.get('#driver-popover-item').findByRole('button', {name: 'Next'}).as('next_btn');
+		cy.get('.frappe-driver').findByRole('button', {name: 'Next'}).as('next_btn');
 
 		// next btn shouldn't move to next step, if first name is not entered
 		cy.get('@next_btn').click();
@@ -68,13 +68,13 @@ context('Form Tour', () => {
 		cy.get('.grid-row-open .frappe-control[data-fieldname="phone"]').as('phone');
 		cy.get('@phone').should('have.class', 'driver-highlighted-element');
 		// enter value in a table field
-		cy.fill_table_field('phone_nos', '1', 'phone', '1234567890');
+		let field = cy.fill_table_field('phone_nos', '1', 'phone', '1234567890');
+		field.blur();
 
 		// move to collapse row step
 		cy.wait(500);
-		cy.get('@next_btn').click();
+		cy.get('.driver-popover-title').contains('Test Title 4').siblings().get('@next_btn').click();
 		cy.wait(500);
-
 		// collapse row
 		cy.get('.grid-row-open .grid-collapse-row').click();
 		cy.wait(500);
@@ -82,7 +82,7 @@ context('Form Tour', () => {
 		// assert save btn is highlighted
 		cy.get('.primary-action').should('have.class', 'driver-highlighted-element');
 		cy.wait(500);
-		cy.get('#driver-popover-item').findByRole('button', {name: 'Save'}).should('be.visible');
+		cy.get('.frappe-driver').findByRole('button', {name: 'Save'}).should('be.visible');
 
 	});
 });

cypress/integration/timeline_email.js

@@ -8,14 +8,13 @@ context('Timeline Email', () => {
 	it('Adding new ToDo, adding email and verifying timeline content for email attachment, deleting attachment and ToDo', () => {
 		//Adding new ToDo
 		cy.click_listview_primary_button('Add ToDo');
-		cy.get('.custom-actions > .btn').trigger('click', {delay: 500});
-		cy.get('.row > .section-body > .form-column > form > .frappe-control > .form-group > .control-input-wrapper > .control-input > .ql-container > .ql-editor').eq(0).type('Test ToDo', {force: true});
+		cy.get('.custom-actions:visible > .btn').contains("Edit in full page").click({delay: 500});
+		cy.fill_field("description", "Test ToDo",  "Text Editor");
 		cy.wait(500);
-		//cy.click_listview_primary_button('Save');
 		cy.get('.primary-action').contains('Save').click({force: true});
 		cy.wait(700);
 		cy.visit('/app/todo');
-		cy.get('.list-row > .level-left > .list-subject > .level-item.ellipsis > .ellipsis').eq(0).click();
+		cy.get('.list-row > .level-left > .list-subject').eq(0).click();
 
 		//Creating a new email
 		cy.get('.timeline-actions > .btn').click();
@@ -47,7 +46,7 @@ context('Timeline Email', () => {
 
 		//Removing the added attachment
 		cy.get('.attachment-row > .data-pill > .remove-btn > .icon').click();
-		cy.get('.modal-dialog > .modal-content > .modal-footer > .standard-actions > .btn-primary').contains('Yes').click();
+		cy.get('.modal-footer:visible > .standard-actions > .btn-primary').contains('Yes').click();
 
 		//To check if the removed attachment is shown in the timeline content
 		cy.get('.timeline-content').should('contain', 'Removed 72402.jpg');
@@ -55,17 +54,17 @@ context('Timeline Email', () => {
 
 		//To check if the discard button functionality in email is working correctly
 		cy.get('.timeline-actions > .btn').click();
-		cy.fill_field('recipients', 'test@example.com', 'MultiSelect'); 
+		cy.fill_field('recipients', 'test@example.com', 'MultiSelect');
 		cy.get('.modal-footer > .standard-actions > .btn-secondary').contains('Discard').click();
 		cy.wait(500);
 		cy.get('.timeline-actions > .btn').click();
 		cy.wait(500);
 		cy.get_field('recipients', 'MultiSelect').should('have.text', '');
-		cy.get('.modal.show > .modal-dialog > .modal-content > .modal-header > .modal-actions > .btn-modal-close > .icon').click();  
+		cy.get('.modal-header:visible > .modal-actions > .btn-modal-close > .icon').click();
 
 		//Deleting the added ToDo
-		cy.get('#page-ToDo > .page-head > .container > .row > .col > .standard-actions > .menu-btn-group > .btn').click();
-		cy.get('.menu-btn-group > .dropdown-menu > li > .grey-link').eq(17).click();
-		cy.get('.modal.show > .modal-dialog > .modal-content > .modal-footer > .standard-actions > .btn-primary').click();
+		cy.get('.menu-btn-group:visible > .btn').click();
+		cy.get('.menu-btn-group:visible > .dropdown-menu > li > .dropdown-item').contains('Delete').click();
+		cy.get('.modal-footer:visible > .standard-actions > .btn-primary').click();
 	});
 });

cypress/integration/workspace.js

@@ -36,12 +36,12 @@ context('Workspace 2.0', () => {
 		cy.get('.codex-editor__redactor .ce-block');
 		cy.get('.custom-actions .inner-group-button[data-label="Add%20Block"]').click();
 		cy.get('.custom-actions .inner-group-button .dropdown-menu .block-menu-item-label').contains('Heading').click();
-		cy.get(".ce-block:last").find('h2').click({force: true}).type('Header');
+		cy.get(":focus").type('Header');
 		cy.get(".ce-block:last").find('.ce-header').should('exist');
 
 		cy.get('.custom-actions .inner-group-button[data-label="Add%20Block"]').click();
 		cy.get('.custom-actions .inner-group-button .dropdown-menu .block-menu-item-label').contains('Text').click();
-		cy.get(".ce-block:last").find('.ce-paragraph').click({force: true}).type('Paragraph text');
+		cy.get(":focus").type('Paragraph text');
 		cy.get(".ce-block:last").find('.ce-paragraph').should('exist');
 	});
 

frappe/automation/doctype/assignment_rule/test_assignment_rule.py

@@ -76,7 +76,7 @@ class TestAutoAssign(unittest.TestCase):
 		# clear 5 assignments for first user
 		# can't do a limit in "delete" since postgres does not support it
 		for d in frappe.get_all('ToDo', dict(reference_type = 'Note', owner = 'test@example.com'), limit=5):
-			frappe.db.sql("delete from tabToDo where name = %s", d.name)
+			frappe.db.delete("ToDo", {"name": d.name})
 
 		# add 5 more assignments
 		for i in range(5):
@@ -177,7 +177,7 @@ class TestAutoAssign(unittest.TestCase):
 		), 'owner'), 'test@example.com')
 
 	def check_assignment_rule_scheduling(self):
-		frappe.db.sql("DELETE FROM `tabAssignment Rule`")
+		frappe.db.delete("Assignment Rule")
 
 		days_1 = [dict(day = 'Sunday'), dict(day = 'Monday'), dict(day = 'Tuesday')]
 
@@ -204,7 +204,7 @@ class TestAutoAssign(unittest.TestCase):
 		), 'owner'), ['test3@example.com'])
 
 	def test_assignment_rule_condition(self):
-		frappe.db.sql("DELETE FROM `tabAssignment Rule`")
+		frappe.db.delete("Assignment Rule")
 
 		# Add expiry_date custom field
 		from frappe.custom.doctype.custom_field.custom_field import create_custom_field
@@ -253,7 +253,7 @@ class TestAutoAssign(unittest.TestCase):
 		assignment_rule.delete()
 
 def clear_assignments():
-	frappe.db.sql("delete from tabToDo where reference_type = 'Note'")
+	frappe.db.delete("ToDo", {"reference_type": "Note"})
 
 def get_assignment_rule(days, assign=None):
 	frappe.delete_doc_if_exists('Assignment Rule', 'For Note 1')

frappe/automation/doctype/milestone_tracker/test_milestone_tracker.py

@@ -7,7 +7,7 @@ import unittest
 
 class TestMilestoneTracker(unittest.TestCase):
 	def test_milestone(self):
-		frappe.db.sql('delete from `tabMilestone Tracker`')
+		frappe.db.delete("Milestone Tracker")
 
 		frappe.cache().delete_key('milestone_tracker_map')
 
@@ -44,5 +44,5 @@ class TestMilestoneTracker(unittest.TestCase):
 		self.assertEqual(milestones[0].value, 'Closed')
 
 		# cleanup
-		frappe.db.sql('delete from tabMilestone')
+		frappe.db.delete("Milestone")
 		milestone_tracker.delete()

frappe/commands/utils.py

@@ -486,15 +486,26 @@ frappe.db.connect()
 
 
 @click.command('console')
+@click.option(
+	'--autoreload',
+	is_flag=True,
+	help="Reload changes to code automatically"
+)
 @pass_context
-def console(context):
+def console(context, autoreload=False):
 	"Start ipython console for a site"
 	site = get_site(context)
 	frappe.init(site=site)
 	frappe.connect()
 	frappe.local.lang = frappe.db.get_default("lang")
 
-	import IPython
+	from IPython.terminal.embed import InteractiveShellEmbed
+
+	terminal = InteractiveShellEmbed()
+	if autoreload:
+		terminal.extension_manager.load_extension("autoreload")
+		terminal.run_line_magic("autoreload", "2")
+
 	all_apps = frappe.get_installed_apps()
 	failed_to_import = []
 
@@ -509,7 +520,9 @@ def console(context):
 	if failed_to_import:
 		print("\nFailed to import:\n{}".format(", ".join(failed_to_import)))
 
-	IPython.embed(display_banner="", header="", colors="neutral")
+	terminal.colors = "neutral"
+	terminal.display_banner = False
+	terminal()
 
 
 @click.command('run-tests')
@@ -524,7 +537,7 @@ def console(context):
 @click.option('--skip-test-records', is_flag=True, default=False, help="Don't create test records")
 @click.option('--skip-before-tests', is_flag=True, default=False, help="Don't run before tests hook")
 @click.option('--junit-xml-output', help="Destination file path for junit xml report")
-@click.option('--failfast', is_flag=True, default=False)
+@click.option('--failfast', is_flag=True, default=False, help="Stop the test run on the first error or failure")
 @pass_context
 def run_tests(context, app=None, module=None, doctype=None, test=(), profile=False,
 		coverage=False, junit_xml_output=False, ui_tests = False, doctype_list_path=None,

frappe/core/doctype/access_log/access_log.py

@@ -29,4 +29,5 @@ def make_access_log(doctype=None, document=None, method=None, file_type=None,
 	doc.insert(ignore_permissions=True)
 
 	# `frappe.db.commit` added because insert doesnt `commit` when called in GET requests like `printview`
-	frappe.db.commit()
+	if frappe.request and frappe.request.method == 'GET':
+		frappe.db.commit()

frappe/core/doctype/comment/test_comment.py

@@ -30,7 +30,7 @@ class TestComment(unittest.TestCase):
 		from frappe.website.doctype.blog_post.test_blog_post import make_test_blog
 		test_blog = make_test_blog()
 
-		frappe.db.sql("delete from `tabComment` where reference_doctype = 'Blog Post'")
+		frappe.db.delete("Comment", {"reference_doctype": "Blog Post"})
 
 		from frappe.templates.includes.comments.comments import add_comment
 		add_comment('Good comment with 10 chars', 'test@test.com', 'Good Tester',
@@ -41,7 +41,7 @@ class TestComment(unittest.TestCase):
 			reference_name = test_blog.name
 		))[0].published, 1)
 
-		frappe.db.sql("delete from `tabComment` where reference_doctype = 'Blog Post'")
+		frappe.db.delete("Comment", {"reference_doctype": "Blog Post"})
 
 		add_comment('pleez vizits my site http://mysite.com', 'test@test.com', 'bad commentor',
 			'Blog Post', test_blog.name, test_blog.route)

frappe/core/doctype/feedback/test_feedback.py

@@ -9,7 +9,7 @@ class TestFeedback(unittest.TestCase):
 		from frappe.website.doctype.blog_post.test_blog_post import make_test_blog
 		test_blog = make_test_blog()
 
-		frappe.db.sql("delete from `tabFeedback` where reference_doctype = 'Blog Post'")
+		frappe.db.delete("Feedback", {"reference_doctype": "Blog Post"})
 
 		from frappe.templates.includes.feedback.feedback import add_feedback, update_feedback
 		feedback = add_feedback('Blog Post', test_blog.name, 5, 'New feedback')
@@ -22,6 +22,6 @@ class TestFeedback(unittest.TestCase):
 		self.assertEqual(updated_feedback.feedback, 'Updated feedback')
 		self.assertEqual(updated_feedback.rating, 6)
 
-		frappe.db.sql("delete from `tabFeedback` where reference_doctype = 'Blog Post'")
+		frappe.db.delete("Feedback", {"reference_doctype": "Blog Post"})
 
 		test_blog.delete()

frappe/core/doctype/file/file.js

@@ -29,15 +29,8 @@ frappe.ui.form.on("File", "refresh", function(frm) {
 	if (is_optimizable) {
 		frm.add_custom_button(__("Optimize"), function() {
 			frappe.show_alert(__("Optimizing image..."));
-			frappe.call({
-				method: "frappe.core.doctype.file.file.optimize_saved_image",
-				args: {
-					doc_name: frm.doc.name,
-				},
-				callback: function() {
-					frappe.show_alert(__("Image optimized"));
-					frappe.set_route("List", "File");
-				}
+			frm.call("optimize_file").then(() => {
+				frappe.show_alert(__("Image optimized"));
 			});
 		});
 	}

frappe/core/doctype/file/file.py

@@ -21,11 +21,11 @@ import zipfile
 import requests
 import requests.exceptions
 from PIL import Image, ImageFile, ImageOps
-from io import StringIO
+from io import BytesIO
 from urllib.parse import quote, unquote
 
 import frappe
-from frappe import _, conf
+from frappe import _, conf, safe_decode
 from frappe.model.document import Document
 from frappe.utils import call_hook_method, cint, cstr, encode, get_files_path, get_hook_method, random_string, strip
 from frappe.utils.image import strip_exif_data, optimize_image
@@ -257,8 +257,7 @@ class File(Document):
 			with open(get_files_path(file_name, is_private=self.is_private), "rb") as f:
 				self.content_hash = get_content_hash(f.read())
 		except IOError:
-			frappe.msgprint(_("File {0} does not exist").format(self.file_url))
-			raise
+			frappe.throw(_("File {0} does not exist").format(self.file_url))
 
 	def on_trash(self):
 		if self.is_home_folder or self.is_attachments_folder:
@@ -270,16 +269,12 @@ class File(Document):
 
 	def make_thumbnail(self, set_as_thumbnail=True, width=300, height=300, suffix="small", crop=False):
 		if self.file_url:
-			if self.file_url.startswith("/files"):
-				try:
+			try:
+				if self.file_url.startswith(("/files", "/private/files")):
 					image, filename, extn = get_local_image(self.file_url)
-				except IOError:
-					return
-
-			else:
-				try:
+				else:
 					image, filename, extn = get_web_image(self.file_url)
-				except (requests.exceptions.HTTPError, requests.exceptions.SSLError, IOError, TypeError):
+			except (requests.exceptions.HTTPError, requests.exceptions.SSLError, IOError, TypeError):
 					return
 
 			size = width, height
@@ -289,16 +284,13 @@ class File(Document):
 				image.thumbnail(size, Image.ANTIALIAS)
 
 			thumbnail_url = filename + "_" + suffix + "." + extn
-
 			path = os.path.abspath(frappe.get_site_path("public", thumbnail_url.lstrip("/")))
 
 			try:
 				image.save(path)
-
 				if set_as_thumbnail:
 					self.db_set("thumbnail_url", thumbnail_url)
 
-				self.db_set("thumbnail_url", thumbnail_url)
 			except IOError:
 				frappe.msgprint(_("Unable to write file format for {0}").format(path))
 				return
@@ -321,17 +313,23 @@ class File(Document):
 			self.delete_file_data_content(only_thumbnail=True)
 
 	def on_rollback(self):
-		self.flags.on_rollback = True
-		self.on_trash()
+		# if original_content flag is set, this rollback should revert the file to its original state
+		if self.flags.original_content:
+			file_path = self.get_full_path()
+			with open(file_path, "wb+") as f:
+				f.write(self.flags.original_content)
+
+		# following condition is only executed when an insert has been rolledback
+		else:
+			self.flags.on_rollback = True
+			self.on_trash()
 
 	def unzip(self):
 		'''Unzip current file and replace it by its children'''
-		if not ".zip" in self.file_name:
-			frappe.msgprint(_("Not a zip file"))
-			return
+		if not self.file_url.endswith(".zip"):
+			frappe.throw(_("{0} is not a zip file").format(self.file_name))
 
-		zip_path = frappe.get_site_path(self.file_url.strip('/'))
-		base_url = os.path.dirname(self.file_url)
+		zip_path = self.get_full_path()
 
 		files = []
 		with zipfile.ZipFile(zip_path) as z:
@@ -359,10 +357,6 @@ class File(Document):
 		return files
 
 
-	def get_file_url(self):
-		data = frappe.db.get_value("File", self.file_data_name, ["file_name", "file_url"], as_dict=True)
-		return data.file_url or data.file_name
-
 	def exists_on_disk(self):
 		exists = os.path.exists(self.get_full_path())
 		return exists
@@ -431,47 +425,6 @@ class File(Document):
 
 		return get_files_path(self.file_name, is_private=self.is_private)
 
-	def get_file_doc(self):
-		'''returns File object (Document) from given parameters or form_dict'''
-		r = frappe.form_dict
-
-		if self.file_url is None: self.file_url = r.file_url
-		if self.file_name is None: self.file_name = r.file_name
-		if self.attached_to_doctype is None: self.attached_to_doctype = r.doctype
-		if self.attached_to_name is None: self.attached_to_name = r.docname
-		if self.attached_to_field is None: self.attached_to_field = r.docfield
-		if self.folder is None: self.folder = r.folder
-		if self.is_private is None: self.is_private = r.is_private
-
-		if r.filedata:
-			file_doc = self.save_uploaded()
-
-		elif r.file_url:
-			file_doc = self.save()
-
-		return file_doc
-
-
-	def save_uploaded(self):
-		self.content = self.get_uploaded_content()
-		if self.content:
-			return self.save()
-		else:
-			raise Exception
-
-	def get_uploaded_content(self):
-		# should not be unicode when reading a file, hence using frappe.form
-		if 'filedata' in frappe.form_dict:
-			if "," in frappe.form_dict.filedata:
-				frappe.form_dict.filedata = frappe.form_dict.filedata.rsplit(",", 1)[1]
-			frappe.uploaded_content = base64.b64decode(frappe.form_dict.filedata)
-			return frappe.uploaded_content
-		elif self.content:
-			return self.content
-		frappe.msgprint(_('No file attached'))
-		return None
-
-
 	def save_file(self, content=None, decode=False, ignore_existing_file_check=False):
 		file_exists = False
 		self.content = content
@@ -539,14 +492,6 @@ class File(Document):
 			'file_url': self.file_url
 		}
 
-	def get_file_data_from_hash(self):
-		for name in frappe.db.sql_list("select name from `tabFile` where content_hash=%s and is_private=%s",
-			(self.content_hash, self.is_private)):
-			b = frappe.get_doc('File', name)
-			return {k: b.get(k) for k in frappe.get_hooks()['write_file_keys']}
-		return False
-
-
 	def check_max_file_size(self):
 		max_file_size = get_max_file_size()
 		file_size = len(self.content)
@@ -594,6 +539,35 @@ class File(Document):
 		if self.file_url:
 			self.is_private = cint(self.file_url.startswith('/private'))
 
+	@frappe.whitelist()
+	def optimize_file(self):
+		if self.is_folder:
+			raise TypeError('Folders cannot be optimized')
+
+		content_type = mimetypes.guess_type(self.file_name)[0]
+		is_local_image = content_type.startswith('image/') and self.file_size > 0
+		is_svg = content_type == 'image/svg+xml'
+
+		if not is_local_image:
+			raise NotImplementedError('Only local image files can be optimized')
+
+		if is_svg:
+			raise TypeError('Optimization of SVG images is not supported')
+
+		content = self.get_content()
+		file_path = self.get_full_path()
+		optimized_content = optimize_image(content, content_type)
+
+		with open(file_path, 'wb+') as f:
+			f.write(optimized_content)
+
+		self.file_size = len(optimized_content)
+		self.content_hash = get_content_hash(optimized_content)
+		# if rolledback, revert back to original
+		self.flags.original_content = content
+		frappe.local.rollback_observers.append(self)
+		self.save()
+
 def on_doctype_update():
 	frappe.db.add_index("File", ["attached_to_doctype", "attached_to_name"])
 
@@ -621,7 +595,8 @@ def create_new_folder(file_name, folder):
 	file.file_name = file_name
 	file.is_folder = 1
 	file.folder = folder
-	file.insert()
+	file.insert(ignore_if_duplicate=True)
+	return file
 
 @frappe.whitelist()
 def move_file(file_list, new_parent, old_parent):
@@ -672,7 +647,7 @@ def get_local_image(file_url):
 	try:
 		image = Image.open(file_path)
 	except IOError:
-		frappe.msgprint(_("Unable to read file format for {0}").format(file_url), raise_exception=True)
+		frappe.throw(_("Unable to read file format for {0}").format(file_url))
 
 	content = None
 
@@ -704,7 +679,7 @@ def get_web_image(file_url):
 		raise
 
 	try:
-		image = Image.open(StringIO(frappe.safe_decode(r.content)))
+		image = Image.open(BytesIO(r.content))
 	except Exception as e:
 		frappe.msgprint(_("Image link '{0}' is not valid").format(file_url), raise_exception=e)
 
@@ -740,48 +715,12 @@ def delete_file(path):
 			os.remove(path)
 
 
-def remove_file(fid=None, attached_to_doctype=None, attached_to_name=None, from_delete=False, delete_permanently=False):
-	"""Remove file and File entry"""
-	file_name = None
-	if not (attached_to_doctype and attached_to_name):
-		attached = frappe.db.get_value("File", fid,
-			["attached_to_doctype", "attached_to_name", "file_name"])
-		if attached:
-			attached_to_doctype, attached_to_name, file_name = attached
-
-	ignore_permissions, comment = False, None
-	if attached_to_doctype and attached_to_name and not from_delete:
-		doc = frappe.get_doc(attached_to_doctype, attached_to_name)
-		ignore_permissions = doc.has_permission("write") or False
-		if frappe.flags.in_web_form:
-			ignore_permissions = True
-		if not file_name:
-			file_name = frappe.db.get_value("File", fid, "file_name")
-		comment = doc.add_comment("Attachment Removed", _("Removed {0}").format(file_name))
-		frappe.delete_doc("File", fid, ignore_permissions=ignore_permissions, delete_permanently=delete_permanently)
-
-	return comment
 
 
 def get_max_file_size():
 	return cint(conf.get('max_file_size')) or 10485760
 
 
-def remove_all(dt, dn, from_delete=False, delete_permanently=False):
-	"""remove all files in a transaction"""
-	try:
-		for fid in frappe.db.sql_list("""select name from `tabFile` where
-			attached_to_doctype=%s and attached_to_name=%s""", (dt, dn)):
-			if from_delete:
-				# If deleting a doc, directly delete files
-				frappe.delete_doc("File", fid, ignore_permissions=True, delete_permanently=delete_permanently)
-			else:
-				# Removes file and adds a comment in the document it is attached to
-				remove_file(fid=fid, attached_to_doctype=dt, attached_to_name=dn,
-					from_delete=from_delete, delete_permanently=delete_permanently)
-	except Exception as e:
-		if e.args[0]!=1054: raise # (temp till for patched)
-
 
 def has_permission(doc, ptype=None, user=None):
 	has_access = False
@@ -827,6 +766,7 @@ def remove_file_by_url(file_url, doctype=None, name=None):
 		fid = frappe.db.get_value("File", {"file_url": file_url})
 
 	if fid:
+		from frappe.utils.file_manager import remove_file
 		return remove_file(fid=fid)
 
 
@@ -886,15 +826,13 @@ def extract_images_from_html(doc, content):
 		if b"," in content:
 			content = content.split(b",")[1]
 		content = base64.b64decode(content)
-		
+
 		content = optimize_image(content, mtype)
 
 		if "filename=" in headers:
 			filename = headers.split("filename=")[-1]
+			filename = safe_decode(filename).split(";")[0]
 
-			# decode filename
-			if not isinstance(filename, str):
-				filename = str(filename, 'utf-8')
 		else:
 			filename = get_random_filename(content_type=mtype)
 
@@ -922,12 +860,9 @@ def extract_images_from_html(doc, content):
 	return content
 
 
-def get_random_filename(extn=None, content_type=None):
-	if extn:
-		if not extn.startswith("."):
-			extn = "." + extn
-
-	elif content_type:
+def get_random_filename(content_type=None):
+	extn = None
+	if content_type:
 		extn = mimetypes.guess_extension(content_type)
 
 	return random_string(7) + (extn or "")
@@ -938,24 +873,8 @@ def unzip_file(name):
 	'''Unzip the given file and make file records for each of the extracted files'''
 	file_obj = frappe.get_doc('File', name)
 	files = file_obj.unzip()
-	return len(files)
+	return files
 
-@frappe.whitelist()
-def optimize_saved_image(doc_name):
-	file_doc = frappe.get_doc('File', doc_name)
-	content = file_doc.get_content()
-	content_type = mimetypes.guess_type(file_doc.file_name)[0]
-
-	optimized_content = optimize_image(content, content_type)
-
-	file_path = get_files_path(is_private=file_doc.is_private)
-	file_path = os.path.join(file_path.encode('utf-8'), file_doc.file_name.encode('utf-8'))
-	with open(file_path, 'wb+') as f:
-		f.write(optimized_content)
-
-	file_doc.file_size = len(optimized_content)
-	file_doc.content_hash = get_content_hash(optimized_content)
-	file_doc.save()
 
 @frappe.whitelist()
 def get_attached_images(doctype, names):
@@ -979,13 +898,6 @@ def get_attached_images(doctype, names):
 	return out
 
 
-@frappe.whitelist()
-def validate_filename(filename):
-	from frappe.utils import now_datetime
-	timestamp = now_datetime().strftime(" %Y-%m-%d %H:%M:%S")
-	fname = get_file_name(filename, timestamp)
-	return fname
-
 @frappe.whitelist()
 def get_files_in_folder(folder, start=0, page_length=20):
 	start = cint(start)

frappe/core/doctype/file/test_file.py

@@ -2,11 +2,12 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # See license.txt
 import base64
+import json
 import frappe
 import os
 import unittest
 from frappe import _
-from frappe.core.doctype.file.file import move_file, get_files_in_folder
+from frappe.core.doctype.file.file import get_attached_images, move_file, get_files_in_folder, unzip_file
 from frappe.utils import get_files_path
 # test_records = frappe.get_test_records('File')
 
@@ -365,6 +366,81 @@ class TestFile(unittest.TestCase):
 		file1.file_url = '/private/files/parent_dir2.txt'
 		file1.save()
 
+	def test_file_url_validation(self):
+		test_file = frappe.get_doc({
+			"doctype": "File",
+			"file_name": 'logo',
+			"file_url": 'https://frappe.io/files/frappe.png'
+		})
+
+		self.assertIsNone(test_file.validate())
+
+		# bad path
+		test_file.file_url = "/usr/bin/man"
+		self.assertRaisesRegex(frappe.exceptions.ValidationError, "URL must start with http:// or https://", test_file.validate)
+
+		test_file.file_url = None
+		test_file.file_name = "/usr/bin/man"
+		self.assertRaisesRegex(frappe.exceptions.ValidationError, "There is some problem with the file url", test_file.validate)
+
+		test_file.file_url = None
+		test_file.file_name = "_file"
+		self.assertRaisesRegex(IOError, "does not exist", test_file.validate)
+
+		test_file.file_url = None
+		test_file.file_name = "/private/files/_file"
+		self.assertRaisesRegex(IOError, "does not exist", test_file.validate)
+
+	def test_make_thumbnail(self):
+		# test web image
+		test_file = frappe.get_doc({
+			"doctype": "File",
+			"file_name": 'logo',
+			"file_url": frappe.utils.get_url('/_test/assets/image.jpg'),
+		}).insert(ignore_permissions=True)
+
+		test_file.make_thumbnail()
+		self.assertEquals(test_file.thumbnail_url, '/files/image_small.jpg')
+
+		# test local image
+		test_file.db_set('thumbnail_url', None)
+		test_file.reload()
+		test_file.file_url = "/files/image_small.jpg"
+		test_file.make_thumbnail(suffix="xs", crop=True)
+		self.assertEquals(test_file.thumbnail_url, '/files/image_small_xs.jpg')
+
+		frappe.clear_messages()
+		test_file.db_set('thumbnail_url', None)
+		test_file.reload()
+		test_file.file_url = frappe.utils.get_url('unknown.jpg')
+		test_file.make_thumbnail(suffix="xs")
+		self.assertEqual(json.loads(frappe.message_log[0]), {"message": f"File '{frappe.utils.get_url('unknown.jpg')}' not found"})
+		self.assertEquals(test_file.thumbnail_url, None)
+
+	def test_file_unzip(self):
+		file_path = frappe.get_app_path('frappe', 'www/_test/assets/file.zip')
+		public_file_path = frappe.get_site_path('public', 'files')
+		try:
+			import shutil
+			shutil.copy(file_path, public_file_path)
+		except Exception:
+			pass
+
+		test_file = frappe.get_doc({
+			"doctype": "File",
+			"file_url": '/files/file.zip',
+		}).insert(ignore_permissions=True)
+
+		self.assertListEqual([file.file_name for file in unzip_file(test_file.name)],
+			['css_asset.css', 'image.jpg', 'js_asset.min.js'])
+
+		test_file = frappe.get_doc({
+			"doctype": "File",
+			"file_url": frappe.utils.get_url('/_test/assets/image.jpg'),
+		}).insert(ignore_permissions=True)
+		self.assertRaisesRegex(frappe.exceptions.ValidationError, 'not a zip file', test_file.unzip)
+
+
 class TestAttachment(unittest.TestCase):
 	test_doctype = 'Test For Attachment'
 
@@ -469,3 +545,93 @@ class TestAttachmentsAccess(unittest.TestCase):
 
 		frappe.set_user('Administrator')
 		frappe.db.rollback()
+
+
+class TestFileUtils(unittest.TestCase):
+	def test_extract_images_from_doc(self):
+		# with filename in data URI
+		todo = frappe.get_doc({
+			"doctype": "ToDo",
+			"description": 'Test <img src="data:image/png;filename=pix.png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=">'
+		}).insert()
+		self.assertTrue(frappe.db.exists("File", {"attached_to_name": todo.name}))
+		self.assertIn('<img src="/files/pix.png">', todo.description)
+		self.assertListEqual(get_attached_images('ToDo', [todo.name])[todo.name], ['/files/pix.png'])
+
+		# without filename in data URI
+		todo = frappe.get_doc({
+			"doctype": "ToDo",
+			"description": 'Test <img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=">'
+		}).insert()
+		filename = frappe.db.exists("File", {"attached_to_name": todo.name})
+		self.assertIn(f'<img src="{frappe.get_doc("File", filename).file_url}', todo.description)
+
+	def test_create_new_folder(self):
+		from frappe.core.doctype.file.file import create_new_folder
+		folder = create_new_folder('test_folder', 'Home')
+		self.assertTrue(folder.is_folder)
+
+
+class TestFileOptimization(unittest.TestCase):
+	def test_optimize_file(self):
+		file_path = frappe.get_app_path("frappe", "tests/data/sample_image_for_optimization.jpg")
+		with open(file_path, "rb") as f:
+			file_content = f.read()
+		test_file = frappe.get_doc({
+			"doctype": "File",
+			"file_name": "sample_image_for_optimization.jpg",
+			"content": file_content
+		}).insert()
+		original_size = test_file.file_size
+		original_content_hash = test_file.content_hash
+
+		test_file.optimize_file()
+		optimized_size = test_file.file_size
+		updated_content_hash = test_file.content_hash
+
+		self.assertLess(optimized_size, original_size)
+		self.assertNotEqual(original_content_hash, updated_content_hash)
+		test_file.delete()
+
+	def test_optimize_svg(self):
+		file_path = frappe.get_app_path("frappe", "tests/data/sample_svg.svg")
+		with open(file_path, "rb") as f:
+			file_content = f.read()
+		test_file = frappe.get_doc({
+			"doctype": "File",
+			"file_name": "sample_svg.svg",
+			"content": file_content
+		}).insert()
+		self.assertRaises(TypeError, test_file.optimize_file)
+		test_file.delete()
+
+	def test_optimize_textfile(self):
+		test_file = frappe.get_doc({
+			"doctype": "File",
+			"file_name": "sample_text.txt",
+			"content": "Text files cannot be optimized"
+		}).insert()
+		self.assertRaises(NotImplementedError, test_file.optimize_file)
+		test_file.delete()
+
+	def test_optimize_folder(self):
+		test_folder = frappe.get_doc("File", "Home/Attachments")
+		self.assertRaises(TypeError, test_folder.optimize_file)
+
+	def test_revert_optimized_file_on_rollback(self):
+		file_path = frappe.get_app_path("frappe", "tests/data/sample_image_for_optimization.jpg")
+		with open(file_path, "rb") as f:
+			file_content = f.read()
+		test_file = frappe.get_doc({
+			"doctype": "File",
+			"file_name": "sample_image_for_optimization.jpg",
+			"content": file_content
+		}).insert()
+		image_path = test_file.get_full_path()
+		size_before_optimization = os.stat(image_path).st_size
+
+		test_file.optimize_file()
+		frappe.db.rollback()
+		size_after_rollback = os.stat(image_path).st_size
+		self.assertEqual(size_before_optimization, size_after_rollback)
+		test_file.delete()

frappe/core/doctype/prepared_report/prepared_report.py

@@ -11,8 +11,6 @@ from frappe.desk.query_report import generate_report_result
 from frappe.model.document import Document
 from frappe.utils import gzip_compress, gzip_decompress
 from frappe.utils.background_jobs import enqueue
-from frappe.core.doctype.file.file import remove_all
-
 
 class PreparedReport(Document):
 	def before_insert(self):

frappe/core/doctype/role_profile/role_profile.py

@@ -3,6 +3,7 @@
 # For license information, please see license.txt
 
 from frappe.model.document import Document
+import frappe
 
 class RoleProfile(Document):
 	def autoname(self):
@@ -11,5 +12,9 @@ class RoleProfile(Document):
 
 	def on_update(self):
 		""" Changes in role_profile reflected across all its user """
-		from frappe.core.doctype.user.user import update_roles
-		update_roles(self.name)
+		users = frappe.get_all('User', filters={'role_profile_name': self.name})
+		roles = [role.role for role in self.roles]
+		for d in users:
+			user = frappe.get_doc('User', d)
+			user.set('roles', [])
+			user.add_roles(*roles)

frappe/core/doctype/role_profile/test_role_profile.py

@@ -8,6 +8,7 @@ test_dependencies = ['Role']
 
 class TestRoleProfile(unittest.TestCase):
 	def test_make_new_role_profile(self):
+		frappe.delete_doc_if_exists('Role Profile', 'Test 1', force=1)
 		new_role_profile = frappe.get_doc(dict(doctype='Role Profile', role_profile='Test 1')).insert()
 
 		self.assertEqual(new_role_profile.role_profile, 'Test 1')
@@ -19,7 +20,25 @@ class TestRoleProfile(unittest.TestCase):
 		new_role_profile.save()
 		self.assertEqual(new_role_profile.roles[0].role, '_Test Role 2')
 
+		# user with a role profile
+		random_user = frappe.mock("email")
+		random_user_name = frappe.mock("name")
+
+		random_user = frappe.get_doc({
+			"doctype": "User",
+			"email": random_user,
+			"enabled": 1,
+			"first_name": random_user_name,
+			"new_password": "Eastern_43A1W",
+			"role_profile_name": 'Test 1'
+		}).insert(ignore_permissions=True, ignore_if_duplicate=True)
+		self.assertListEqual([role.role for role in random_user.roles], [role.role for role in new_role_profile.roles])
+
 		# clear roles
 		new_role_profile.roles = []
 		new_role_profile.save()
 		self.assertEqual(new_role_profile.roles, [])
+
+		# user roles with the role profile should also be updated
+		random_user.reload()
+		self.assertListEqual(random_user.roles, [])

frappe/core/doctype/translation/test_translation.py

@@ -8,7 +8,7 @@ from frappe import _
 
 class TestTranslation(unittest.TestCase):
 	def setUp(self):
-		frappe.db.sql('delete from tabTranslation')
+		frappe.db.delete("Translation")
 
 	def tearDown(self):
 		frappe.local.lang = 'en'

frappe/core/doctype/user/test_records.json

@@ -70,5 +70,19 @@
     "role": "System Manager"
    }
   ]
- }
+ },
+ {
+    "doctype": "User",
+    "email": "testpassword@example.com",
+    "enabled": 1,
+    "first_name": "_Test",
+    "new_password": "Eastern_43A1W",
+    "roles": [
+     {
+      "doctype": "Has Role",
+      "parentfield": "roles",
+      "role": "System Manager"
+     }
+    ]
+   }
 ]

frappe/core/doctype/user/test_user.py

@@ -1,16 +1,18 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # MIT License. See license.txt
-import frappe, unittest, uuid
+import json
+import unittest
+from unittest.mock import patch
 
-from frappe.model.delete_doc import delete_doc
-from frappe.utils.data import today, add_to_date
-from frappe import _dict
-from frappe.utils import get_url
-from frappe.core.doctype.user.user import get_total_users
-from frappe.core.doctype.user.user import MaxUsersReachedError, test_password_strength
-from frappe.core.doctype.user.user import extract_mentions
+import frappe
+import frappe.exceptions
+from frappe.core.doctype.user.user import (extract_mentions, reset_password,
+	sign_up, test_password_strength, update_password, verify_password)
 from frappe.frappeclient import FrappeClient
+from frappe.model.delete_doc import delete_doc
+from frappe.utils import get_url
 
+user_module = frappe.core.doctype.user.user
 test_records = frappe.get_test_records('User')
 
 class TestUser(unittest.TestCase):
@@ -23,7 +25,7 @@ class TestUser(unittest.TestCase):
 
 	def test_user_type(self):
 		new_user = frappe.get_doc(dict(doctype='User', email='test-for-type@example.com',
-			first_name='Tester')).insert()
+			first_name='Tester')).insert(ignore_if_duplicate=True)
 		self.assertEqual(new_user.user_type, 'Website User')
 
 		# social login userid for frappe
@@ -52,7 +54,7 @@ class TestUser(unittest.TestCase):
 	def test_delete(self):
 		frappe.get_doc("User", "test@example.com").add_roles("_Test Role 2")
 		self.assertRaises(frappe.LinkExistsError, delete_doc, "Role", "_Test Role 2")
-		frappe.db.sql("""delete from `tabHas Role` where role='_Test Role 2'""")
+		frappe.db.delete("Has Role", {"role": "_Test Role 2"})
 		delete_doc("Role","_Test Role 2")
 
 		if frappe.db.exists("User", "_test@example.com"):
@@ -119,40 +121,9 @@ class TestUser(unittest.TestCase):
 		# system manager now added by Administrator
 		self.assertTrue("System Manager" in [d.role for d in me.get("roles")])
 
-	# def test_deny_multiple_sessions(self):
-	#	from frappe.installer import update_site_config
-	# 	clear_limit('users')
-	#
-	# 	# allow one session
-	# 	user = frappe.get_doc('User', 'test@example.com')
-	# 	user.simultaneous_sessions = 1
-	# 	user.new_password = 'Eastern_43A1W'
-	# 	user.save()
-	#
-	# 	def test_request(conn):
-	# 		value = conn.get_value('User', 'first_name', {'name': 'test@example.com'})
-	# 		self.assertTrue('first_name' in value)
-	#
-	# 	from frappe.frappeclient import FrappeClient
-	# 	update_site_config('deny_multiple_sessions', 0)
-	#
-	# 	conn1 = FrappeClient(get_url(), "test@example.com", "Eastern_43A1W", verify=False)
-	# 	test_request(conn1)
-	#
-	# 	conn2 = FrappeClient(get_url(), "test@example.com", "Eastern_43A1W", verify=False)
-	# 	test_request(conn2)
-	#
-	# 	update_site_config('deny_multiple_sessions', 1)
-	# 	conn3 = FrappeClient(get_url(), "test@example.com", "Eastern_43A1W", verify=False)
-	# 	test_request(conn3)
-	#
-	# 	# first connection should fail
-	# 	test_request(conn1)
-
-
 	def test_delete_user(self):
 		new_user = frappe.get_doc(dict(doctype='User', email='test-for-delete@example.com',
-			first_name='Tester Delete User')).insert()
+			first_name='Tester Delete User')).insert(ignore_if_duplicate=True)
 		self.assertEqual(new_user.user_type, 'Website User')
 
 		# role with desk access
@@ -174,7 +145,7 @@ class TestUser(unittest.TestCase):
 		self.assertFalse(frappe.db.exists('User', new_user.name))
 
 	def test_password_strength(self):
-		# Test Password without Password Strenth Policy
+		# Test Password without Password Strength Policy
 		frappe.db.set_value("System Settings", "System Settings", "enable_password_policy", 0)
 
 		# password policy is disabled, test_password_strength should be ignored
@@ -193,6 +164,17 @@ class TestUser(unittest.TestCase):
 		result = test_password_strength("Eastern_43A1W")
 		self.assertEqual(result['feedback']['password_policy_validation_passed'], True)
 
+
+		# test password strength while saving user with new password
+		user = frappe.get_doc("User", "test@example.com")
+		frappe.flags.in_test = False
+		user.new_password = "password"
+		self.assertRaisesRegex(frappe.exceptions.ValidationError, "Invalid Password", user.save)
+		user.reload()
+		user.new_password = "Eastern_43A1W"
+		user.save()
+		frappe.flags.in_test = True
+
 	def test_comment_mentions(self):
 		comment = '''
 			<span class="mention" data-id="test.comment@example.com" data-value="Test" data-denotation-char="@">
@@ -227,6 +209,7 @@ class TestUser(unittest.TestCase):
 		self.assertEqual(extract_mentions(comment)[0], "test_user@example.com")
 		self.assertEqual(extract_mentions(comment)[1], "test.again@example1.com")
 
+		frappe.delete_doc("User Group", "Team")
 		doc = frappe.get_doc({
 			'doctype': 'User Group',
 			'name': 'Team',
@@ -236,14 +219,18 @@ class TestUser(unittest.TestCase):
 				'user': 'test1@example.com'
 			}]
 		})
-		doc.insert(ignore_if_duplicate=True)
+
+		doc.insert()
 
 		comment = '''
 			<div>
 				Testing comment for
 				<span class="mention" data-id="Team" data-value="Team" data-is-group="true" data-denotation-char="@">
 					<span><span class="ql-mention-denotation-char">@</span>Team</span>
-				</span>
+				</span> and
+				<span class="mention" data-id="Unknown Team" data-value="Unknown Team" data-is-group="true" data-denotation-char="@">
+					<span><span class="ql-mention-denotation-char">@</span>Unknown Team</span>
+				</span><!-- this should be ignored-->
 				please check
 			</div>
 		'''
@@ -267,32 +254,125 @@ class TestUser(unittest.TestCase):
 		self.assertEqual(res1.status_code, 200)
 		self.assertEqual(res2.status_code, 417)
 
-	# def test_user_rollback(self):
-	# 	"""
-	#	FIXME: This is failing with PR #12693 as Rollback can't happen if notifications sent on user creation.
-	#	Make sure that notifications disabled.
-	# 	"""
-	# 	frappe.db.commit()
-	# 	frappe.db.begin()
-	# 	user_id = str(uuid.uuid4())
-	# 	email = f'{user_id}@example.com'
-	# 	try:
-	# 		frappe.flags.in_import = True  # disable throttling
-	# 		frappe.get_doc(dict(
-	# 			doctype='User',
-	# 			email=email,
-	# 			first_name=user_id,
-	# 		)).insert()
-	# 	finally:
-	# 		frappe.flags.in_import = False
+	def test_user_rename(self):
+		old_name = "test_user_rename@example.com"
+		new_name = "test_user_rename_new@example.com"
+		user = frappe.get_doc({
+			"doctype": "User",
+			"email": old_name,
+			"enabled": 1,
+			"first_name": "_Test",
+			"new_password": "Eastern_43A1W",
+			"roles": [
+				{
+					"doctype": "Has Role",
+					"parentfield": "roles",
+					"role": "System Manager"
+				}]
+		}).insert(ignore_permissions=True, ignore_if_duplicate=True)
 
-	# 	# Check user has been added
-	# 	self.assertIsNotNone(frappe.db.get("User", {"email": email}))
+		frappe.rename_doc('User', user.name, new_name)
+		self.assertTrue(frappe.db.exists("Notification Settings", new_name))
+
+		frappe.delete_doc("User", new_name)
+
+	def test_signup(self):
+		import frappe.website.utils
+		random_user = frappe.mock('email')
+		random_user_name = frappe.mock('name')
+		# disabled signup
+		with patch.object(user_module, "is_signup_disabled", return_value=True):
+			self.assertRaisesRegex(frappe.exceptions.ValidationError, "Sign Up is disabled",
+				sign_up, random_user, random_user_name, "/signup")
+
+		self.assertTupleEqual(sign_up(random_user, random_user_name, "/welcome"), (1, "Please check your email for verification"))
+		self.assertEqual(frappe.cache().hget('redirect_after_login', random_user), "/welcome")
+
+		# re-register
+		self.assertTupleEqual(sign_up(random_user, random_user_name, "/welcome"), (0, "Already Registered"))
+
+		# disabled user
+		user = frappe.get_doc("User", random_user)
+		user.enabled = 0
+		user.save()
+
+		self.assertTupleEqual(sign_up(random_user, random_user_name, "/welcome"), (0, "Registered but disabled"))
+
+		# throttle user creation
+		with patch.object(user_module.frappe.db, "get_creation_count", return_value=301):
+			self.assertRaisesRegex(frappe.exceptions.ValidationError, "Throttled",
+				sign_up, frappe.mock('email'), random_user_name, "/signup")
+
+
+	def test_reset_password(self):
+		from frappe.auth import CookieManager, LoginManager
+		from frappe.utils import set_request
+		old_password = "Eastern_43A1W"
+		new_password = "easy_password"
+
+		set_request(path="/random")
+		frappe.local.cookie_manager = CookieManager()
+		frappe.local.login_manager = LoginManager()
+
+		frappe.set_user("testpassword@example.com")
+		test_user = frappe.get_doc("User", "testpassword@example.com")
+		test_user.reset_password()
+		self.assertEqual(update_password(new_password, key=test_user.reset_password_key), "/app")
+		self.assertEqual(update_password(new_password, key="wrong_key"), "The Link specified has either been used before or Invalid")
+
+		# password verification should fail with old password
+		self.assertRaises(frappe.exceptions.AuthenticationError, verify_password, old_password)
+		verify_password(new_password)
+
+		# reset password
+		update_password(old_password, old_password=new_password)
+
+		self.assertRaisesRegex(frappe.exceptions.ValidationError, "Invalid key type", update_password, "test", 1, ['like', '%'])
+
+		password_strength_response = {
+			"feedback": {
+				"password_policy_validation_passed": False,
+				"suggestions": ["Fix password"]
+			}
+		}
+
+		# password strength failure test
+		with patch.object(user_module, "test_password_strength", return_value=password_strength_response):
+			self.assertRaisesRegex(frappe.exceptions.ValidationError, "Fix password", update_password, new_password, 0, test_user.reset_password_key)
+
+
+		# test redirect URL for website users
+		frappe.set_user("test2@example.com")
+		self.assertEqual(update_password(new_password, old_password=old_password), "/")
+		# reset password
+		update_password(old_password, old_password=new_password)
+
+		# test API endpoint
+		with patch.object(user_module.frappe, 'sendmail') as sendmail:
+			frappe.clear_messages()
+			test_user = frappe.get_doc("User", "test2@example.com")
+			self.assertEqual(reset_password(user="test2@example.com"), None)
+			test_user.reload()
+			self.assertEqual(update_password(new_password, key=test_user.reset_password_key), "/")
+			update_password(old_password, old_password=new_password)
+			self.assertEqual(json.loads(frappe.message_log[0]), {"message": "Password reset instructions have been sent to your email"})
+		sendmail.assert_called_once()
+		self.assertEqual(sendmail.call_args[1]["recipients"], "test2@example.com")
+
+		self.assertEqual(reset_password(user="test2@example.com"), None)
+		self.assertEqual(reset_password(user="Administrator"), "not allowed")
+		self.assertEqual(reset_password(user="random"), "not found")
+
+	def test_user_onload_modules(self):
+		from frappe.config import get_modules_from_all_apps
+		from frappe.desk.form.load import getdoc
+		frappe.response.docs = []
+		getdoc("User", "Administrator")
+		doc = frappe.response.docs[0]
+		self.assertListEqual(doc.get("__onload").get('all_modules', []),
+			[m.get("module_name") for m in get_modules_from_all_apps()])
 
-	# 	# Check that rollback works
-	# 	frappe.db.rollback()
-	# 	self.assertIsNone(frappe.db.get("User", {"email": email}))
 
 def delete_contact(user):
-	frappe.db.sql("DELETE FROM `tabContact` WHERE `email_id`= %s", user)
-	frappe.db.sql("DELETE FROM `tabContact Email` WHERE `email_id`= %s", user)
+	frappe.db.delete("Contact", {"email_id": user})
+	frappe.db.delete("Contact Email", {"email_id": user})

frappe/core/doctype/user/user.js

@@ -166,7 +166,7 @@ frappe.ui.form.on('User', {
 
 			frm.add_custom_button(__("Reset OTP Secret"), function() {
 				frappe.call({
-					method: "frappe.core.doctype.user.user.reset_otp_secret",
+					method: "frappe.twofactor.reset_otp_secret",
 					args: {
 						"user": frm.doc.name
 					}

frappe/core/doctype/user/user.py

@@ -15,17 +15,11 @@ from frappe.desk.doctype.notification_settings.notification_settings import crea
 from frappe.utils.user import get_system_managers
 from frappe.website.utils import is_signup_disabled
 from frappe.rate_limiter import rate_limit
-from frappe.utils.background_jobs import enqueue
 from frappe.core.doctype.user_type.user_type import user_linked_with_permission_on_doctype
 
 
 STANDARD_USERS = ("Guest", "Administrator")
 
-
-class MaxUsersReachedError(frappe.ValidationError):
-	pass
-
-
 class User(Document):
 	__new_password = None
 
@@ -56,8 +50,6 @@ class User(Document):
 		frappe.cache().delete_key('enabled_users')
 
 	def validate(self):
-		self.check_demo()
-
 		# clear new password
 		self.__new_password = self.new_password
 		self.new_password = ""
@@ -137,10 +129,6 @@ class User(Document):
 		"""Returns true if current user is the session user"""
 		return self.name == frappe.session.user
 
-	def check_demo(self):
-		if frappe.session.user == 'demo@erpnext.com':
-			frappe.throw(_('Cannot change user details in demo. Please signup for a new account at https://erpnext.com'), title=_('Not Allowed'))
-
 	def set_full_name(self):
 		self.full_name = " ".join(filter(None, [self.first_name, self.last_name]))
 
@@ -398,7 +386,6 @@ class User(Document):
 
 
 	def before_rename(self, old_name, new_name, merge=False):
-		self.check_demo()
 		frappe.clear_cache(user=old_name)
 		self.validate_rename(old_name, new_name)
 
@@ -718,85 +705,6 @@ def get_email_awaiting(user):
 				where parent = %(user)s""",{"user":user})
 		return False
 
-@frappe.whitelist(allow_guest=False)
-def set_email_password(email_account, user, password):
-	account = frappe.get_doc("Email Account", email_account)
-	if account.awaiting_password:
-		account.awaiting_password = 0
-		account.password = password
-		try:
-			account.save(ignore_permissions=True)
-		except Exception:
-			frappe.db.rollback()
-			return False
-
-	return True
-
-def setup_user_email_inbox(email_account, awaiting_password, email_id, enable_outgoing):
-	""" setup email inbox for user """
-	def add_user_email(user):
-		user = frappe.get_doc("User", user)
-		row = user.append("user_emails", {})
-
-		row.email_id = email_id
-		row.email_account = email_account
-		row.awaiting_password = awaiting_password or 0
-		row.enable_outgoing = enable_outgoing or 0
-
-		user.save(ignore_permissions=True)
-
-	udpate_user_email_settings = False
-	if not all([email_account, email_id]):
-		return
-
-	user_names = frappe.db.get_values("User", { "email": email_id }, as_dict=True)
-	if not user_names:
-		return
-
-	for user in user_names:
-		user_name = user.get("name")
-
-		# check if inbox is alreay configured
-		user_inbox = frappe.db.get_value("User Email", {
-			"email_account": email_account,
-			"parent": user_name
-		}, ["name"]) or None
-
-		if not user_inbox:
-			add_user_email(user_name)
-		else:
-			# update awaiting password for email account
-			udpate_user_email_settings = True
-
-	if udpate_user_email_settings:
-		frappe.db.sql("""UPDATE `tabUser Email` SET awaiting_password = %(awaiting_password)s,
-			enable_outgoing = %(enable_outgoing)s WHERE email_account = %(email_account)s""", {
-				"email_account": email_account,
-				"enable_outgoing": enable_outgoing,
-				"awaiting_password": awaiting_password or 0
-			})
-	else:
-		users = " and ".join([frappe.bold(user.get("name")) for user in user_names])
-		frappe.msgprint(_("Enabled email inbox for user {0}").format(users))
-
-	ask_pass_update()
-
-def remove_user_email_inbox(email_account):
-	""" remove user email inbox settings if email account is deleted """
-	if not email_account:
-		return
-
-	users = frappe.get_all("User Email", filters={
-		"email_account": email_account
-	}, fields=["parent as name"])
-
-	for user in users:
-		doc = frappe.get_doc("User", user.get("name"))
-		to_remove = [ row for row in doc.user_emails if row.email_account == email_account ]
-		[ doc.remove(row) for row in to_remove ]
-
-		doc.save(ignore_permissions=True)
-
 def ask_pass_update():
 	# update the sys defaults as to awaiting users
 	from frappe.utils import set_default
@@ -809,24 +717,19 @@ def ask_pass_update():
 
 def _get_user_for_update_password(key, old_password):
 	# verify old password
+	result = frappe._dict()
 	if key:
-		user = frappe.db.get_value("User", {"reset_password_key": key})
-		if not user:
-			return {
-				'message': _("The Link specified has either been used before or Invalid")
-			}
+		result.user = frappe.db.get_value("User", {"reset_password_key": key})
+		if not result.user:
+			result.message = _("The Link specified has either been used before or Invalid")
 
 	elif old_password:
 		# verify old password
 		frappe.local.login_manager.check_password(frappe.session.user, old_password)
 		user = frappe.session.user
+		result.user = user
 
-	else:
-		return
-
-	return {
-		'user': user
-	}
+	return result
 
 def reset_user_data(user):
 	user_doc = frappe.get_doc("User", user)
@@ -848,14 +751,12 @@ def sign_up(email, full_name, redirect_to):
 
 	user = frappe.db.get("User", {"email": email})
 	if user:
-		if user.disabled:
-			return 0, _("Registered but disabled")
-		else:
+		if user.enabled:
 			return 0, _("Already Registered")
+		else:
+			return 0, _("Registered but disabled")
 	else:
-		if frappe.db.sql("""select count(*) from tabUser where
-			HOUR(TIMEDIFF(CURRENT_TIMESTAMP, TIMESTAMP(modified)))=1""")[0][0] > 300:
-
+		if frappe.db.get_creation_count('User', 60) > 300:
 			frappe.respond_as_web_page(_('Temporarily Disabled'),
 				_('Too many users signed up recently, so the registration is disabled. Please try back in an hour'),
 				http_status_code=429)
@@ -1048,91 +949,6 @@ def update_gravatar(name):
 	if gravatar:
 		frappe.db.set_value('User', name, 'user_image', gravatar)
 
-@frappe.whitelist(allow_guest=True)
-def send_token_via_sms(tmp_id,phone_no=None,user=None):
-	try:
-		from frappe.core.doctype.sms_settings.sms_settings import send_request
-	except:
-		return False
-
-	if not frappe.cache().ttl(tmp_id + '_token'):
-		return False
-	ss = frappe.get_doc('SMS Settings', 'SMS Settings')
-	if not ss.sms_gateway_url:
-		return False
-
-	token = frappe.cache().get(tmp_id + '_token')
-	args = {ss.message_parameter: 'verification code is {}'.format(token)}
-
-	for d in ss.get("parameters"):
-		args[d.parameter] = d.value
-
-	if user:
-		user_phone = frappe.db.get_value('User', user, ['phone','mobile_no'], as_dict=1)
-		usr_phone = user_phone.mobile_no or user_phone.phone
-		if not usr_phone:
-			return False
-	else:
-		if phone_no:
-			usr_phone = phone_no
-		else:
-			return False
-
-	args[ss.receiver_parameter] = usr_phone
-	status = send_request(ss.sms_gateway_url, args, use_post=ss.use_post)
-
-	if 200 <= status < 300:
-		frappe.cache().delete(tmp_id + '_token')
-		return True
-	else:
-		return False
-
-@frappe.whitelist(allow_guest=True)
-def send_token_via_email(tmp_id,token=None):
-	import pyotp
-
-	user = frappe.cache().get(tmp_id + '_user')
-	count = token or frappe.cache().get(tmp_id + '_token')
-
-	if ((not user) or (user == 'None') or (not count)):
-		return False
-	user_email = frappe.db.get_value('User',user, 'email')
-	if not user_email:
-		return False
-
-	otpsecret = frappe.cache().get(tmp_id + '_otp_secret')
-	hotp = pyotp.HOTP(otpsecret)
-
-	frappe.sendmail(
-		recipients=user_email,
-		sender=None,
-		subject="Verification Code",
-		template="verification_code",
-		args=dict(code=hotp.at(int(count))),
-		delayed=False,
-		retry=3
-	)
-
-	return True
-
-@frappe.whitelist(allow_guest=True)
-def reset_otp_secret(user):
-	otp_issuer = frappe.db.get_value('System Settings', 'System Settings', 'otp_issuer_name')
-	user_email = frappe.db.get_value('User',user, 'email')
-	if frappe.session.user in ["Administrator", user] :
-		frappe.defaults.clear_default(user + '_otplogin')
-		frappe.defaults.clear_default(user + '_otpsecret')
-		email_args = {
-			'recipients':user_email, 'sender':None, 'subject':'OTP Secret Reset - {}'.format(otp_issuer or "Frappe Framework"),
-			'message':'<p>Your OTP secret on {} has been reset. If you did not perform this reset and did not request it, please contact your System Administrator immediately.</p>'.format(otp_issuer or "Frappe Framework"),
-			'delayed':False,
-			'retry':3
-		}
-		enqueue(method=frappe.sendmail, queue='short', timeout=300, event=None, is_async=True, job_name=None, now=False, **email_args)
-		return frappe.msgprint(_("OTP Secret has been reset. Re-registration will be required on next login."))
-	else:
-		return frappe.throw(_("OTP secret can only be reset by the Administrator."))
-
 def throttle_user_creation():
 	if frappe.flags.in_import:
 		return
@@ -1150,15 +966,6 @@ def get_module_profile(module_profile):
 	module_profile = frappe.get_doc('Module Profile', {'module_profile_name': module_profile})
 	return module_profile.get('block_modules')
 
-def update_roles(role_profile):
-	users = frappe.get_all('User', filters={'role_profile_name': role_profile})
-	role_profile = frappe.get_doc('Role Profile', role_profile)
-	roles = [role.role for role in role_profile.roles]
-	for d in users:
-		user = frappe.get_doc('User', d)
-		user.set('roles', [])
-		user.add_roles(*roles)
-
 def create_contact(user, ignore_links=False, ignore_mandatory=False):
 	from frappe.contacts.doctype.contact.contact import get_contact_name
 	if user.name in ["Administrator", "Guest"]: return
@@ -1217,18 +1024,18 @@ def generate_keys(user):
 
 	:param user: str
 	"""
-	if "System Manager" in frappe.get_roles():
-		user_details = frappe.get_doc("User", user)
-		api_secret = frappe.generate_hash(length=15)
-		# if api key is not set generate api key
-		if not user_details.api_key:
-			api_key = frappe.generate_hash(length=15)
-			user_details.api_key = api_key
-		user_details.api_secret = api_secret
-		user_details.save()
+	frappe.only_for("System Manager")
+	user_details = frappe.get_doc("User", user)
+	api_secret = frappe.generate_hash(length=15)
+	# if api key is not set generate api key
+	if not user_details.api_key:
+		api_key = frappe.generate_hash(length=15)
+		user_details.api_key = api_key
+	user_details.api_secret = api_secret
+	user_details.save()
+
+	return {"api_secret": api_secret}
 
-		return {"api_secret": api_secret}
-	frappe.throw(frappe._("Not Permitted"), frappe.PermissionError)
 
 @frappe.whitelist()
 def switch_theme(theme):

frappe/desk/doctype/event/test_event.py

@@ -14,7 +14,7 @@ test_records = frappe.get_test_records('Event')
 
 class TestEvent(unittest.TestCase):
 	def setUp(self):
-		frappe.db.sql('delete from tabEvent')
+		frappe.db.delete("Event")
 		make_test_objects('Event', reset=True)
 
 		self.test_records = frappe.get_test_records('Event')

frappe/desk/doctype/note/test_note.py

@@ -8,9 +8,9 @@ test_records = frappe.get_test_records('Note')
 
 class TestNote(unittest.TestCase):
 	def insert_note(self):
-		frappe.db.sql('delete from tabVersion')
-		frappe.db.sql('delete from tabNote')
-		frappe.db.sql('delete from `tabNote Seen By`')
+		frappe.db.delete("Version")
+		frappe.db.delete("Note")
+		frappe.db.delete("Note Seen By")
 
 		return frappe.get_doc(dict(doctype='Note', title='test note',
 			content='test note content')).insert()

frappe/desk/doctype/notification_log/test_notification_log.py

@@ -2,6 +2,7 @@
 # Copyright (c) 2019, Frappe Technologies and Contributors
 # See license.txt
 import frappe
+from frappe.core.doctype.user.user import get_system_users
 from frappe.desk.form.assign_to import add as assign_task
 import unittest
 
@@ -54,7 +55,4 @@ def get_todo():
 	return frappe.get_cached_doc('ToDo', res[0].name)
 
 def get_user():
-	users = frappe.db.get_all('User',
-		filters={'name': ('not in', ['Administrator', 'Guest'])},
-		fields='name', limit=1)
-	return users[0].name
+	return get_system_users(limit=1)[0]

frappe/desk/doctype/system_console/system_console.js

@@ -5,7 +5,7 @@ frappe.ui.form.on('System Console', {
 	onload: function(frm) {
 		frappe.ui.keys.add_shortcut({
 			shortcut: 'shift+enter',
-			action: () => frm.execute_action('Execute'),
+			action: () => frm.page.btn_primary.trigger('click'),
 			page: frm.page,
 			description: __('Execute Console script'),
 			ignore_inputs: true,
@@ -14,8 +14,11 @@ frappe.ui.form.on('System Console', {
 
 	refresh: function(frm) {
 		frm.disable_save();
-		frm.page.set_primary_action(__("Execute"), () => {
-			frm.execute_action('Execute');
+		frm.page.set_primary_action(__("Execute"), $btn => {
+			$btn.text(__('Executing...'));
+			return frm.execute_action("Execute").then(() => {
+				$btn.text(__('Execute'));
+			});
 		});
 	}
 });

frappe/desk/doctype/tag/test_tag.py

@@ -6,7 +6,7 @@ from frappe.desk.doctype.tag.tag import add_tag
 
 class TestTag(unittest.TestCase):
 	def setUp(self) -> None:
-		frappe.db.sql("DELETE from `tabTag`")
+		frappe.db.delete("Tag")
 		frappe.db.sql("UPDATE `tabDocType` set _user_tags=''")
 
 	def test_tag_count_query(self):

frappe/desk/doctype/todo/test_todo.py

@@ -14,7 +14,7 @@ class TestToDo(unittest.TestCase):
 		todo = frappe.get_doc(dict(doctype='ToDo', description='test todo',
 			assigned_by='Administrator')).insert()
 
-		frappe.db.sql('delete from `tabDeleted Document`')
+		frappe.db.delete("Deleted Document")
 		todo.delete()
 
 		deleted = frappe.get_doc('Deleted Document', dict(deleted_doctype=todo.doctype, deleted_name=todo.name))
@@ -27,7 +27,7 @@ class TestToDo(unittest.TestCase):
 			frappe.db.get_value('User', todo.assigned_by, 'full_name'))
 
 	def test_fetch_setup(self):
-		frappe.db.sql('delete from tabToDo')
+		frappe.db.delete("ToDo")
 
 		todo_meta = frappe.get_doc('DocType', 'ToDo')
 		todo_meta.get('fields', dict(fieldname='assigned_by_full_name'))[0].fetch_from = ''
@@ -104,8 +104,8 @@ class TestToDo(unittest.TestCase):
 		clear_permissions_cache('ToDo')
 		frappe.db.rollback()
 
-def test_fetch_if_empty(self):
-		frappe.db.sql('delete from tabToDo')
+	def test_fetch_if_empty(self):
+		frappe.db.delete("ToDo")
 
 		# Allow user changes
 		todo_meta = frappe.get_doc('DocType', 'ToDo')
@@ -122,9 +122,8 @@ def test_fetch_if_empty(self):
 		self.assertEqual(todo.assigned_by_full_name, 'Admin')
 
 		# Overwrite user changes
-		todo_meta = frappe.get_doc('DocType', 'ToDo')
-		todo_meta.get('fields', dict(fieldname='assigned_by_full_name'))[0].fetch_if_empty = 0
-		todo_meta.save()
+		todo.meta.get('fields', dict(fieldname='assigned_by_full_name'))[0].fetch_if_empty = 0
+		todo.meta.save()
 
 		todo.reload()
 		todo.save()

frappe/desk/doctype/todo/todo.py

@@ -29,8 +29,15 @@ class ToDo(Document):
 		else:
 			# NOTE the previous value is only available in validate method
 			if self.get_db_value("status") != self.status:
+				if self.owner == frappe.session.user:
+					removal_message = frappe._("{0} removed their assignment.").format(
+						get_fullname(frappe.session.user))
+				else:
+					removal_message = frappe._("Assignment of {0} removed by {1}").format(
+						get_fullname(self.owner), get_fullname(frappe.session.user))
+
 				self._assignment = {
-					"text": frappe._("Assignment closed by {0}").format(get_fullname(frappe.session.user)),
+					"text": removal_message,
 					"comment_type": "Assignment Completed"
 				}
 

frappe/desk/doctype/workspace/test_workspace.py

@@ -1,8 +1,95 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2020, Frappe Technologies and Contributors
 # See license.txt
-# import frappe
+import frappe
 import unittest
-
 class TestWorkspace(unittest.TestCase):
-	pass
+	def setUp(self):
+		create_module("Test Module")
+
+	def tearDown(self):
+		frappe.db.delete("Workspace", {"module": "Test Module"})
+		frappe.db.delete("DocType", {"module": "Test Module"})
+		frappe.delete_doc("Module Def", "Test Module")
+
+	# TODO: FIX ME - flaky test!!!
+	# def test_workspace_with_cards_specific_to_a_country(self):
+	# 	workspace = create_workspace()
+	# 	insert_card(workspace, "Card Label 1", "DocType 1", "DocType 2", "France")
+	# 	insert_card(workspace, "Card Label 2", "DocType A", "DocType B")
+
+	# 	workspace.insert(ignore_if_duplicate = True)
+
+	# 	cards = workspace.get_link_groups()
+
+	# 	if frappe.get_system_settings('country') == "France":
+	# 		self.assertEqual(len(cards), 2)
+	# 	else:
+	# 		self.assertEqual(len(cards), 1)
+
+def create_module(module_name):
+	module = frappe.get_doc({
+		"doctype": "Module Def",
+		"module_name": module_name,
+		"app_name": "frappe"
+	})
+	module.insert(ignore_if_duplicate = True)
+
+	return module
+
+def create_workspace(**args):
+	workspace = frappe.new_doc("Workspace")
+	args = frappe._dict(args)
+
+	workspace.name = args.name or "Test Workspace"
+	workspace.label = args.label or "Test Workspace"
+	workspace.category = args.category or "Modules"
+	workspace.is_standard = args.is_standard or 1
+	workspace.module = "Test Module"
+
+	return workspace
+
+def insert_card(workspace, card_label, doctype1, doctype2, country=None):
+	workspace.append("links", {
+		"type": "Card Break",
+		"label": card_label,
+		"only_for": country
+	})
+
+	create_doctype(doctype1, "Test Module")
+	workspace.append("links", {
+		"type": "Link",
+		"label": doctype1,
+		"only_for": country,
+		"link_type": "DocType",
+		"link_to": doctype1
+	})
+
+	create_doctype(doctype2, "Test Module")
+	workspace.append("links", {
+		"type": "Link",
+		"label": doctype2,
+		"only_for": country,
+		"link_type": "DocType",
+		"link_to": doctype2
+	})
+
+def create_doctype(doctype_name, module):
+	frappe.get_doc({
+		'doctype': 'DocType',
+		'name': doctype_name,
+		'module': module,
+		'custom': 1,
+		'autoname': 'field:title',
+		'fields': [
+			{'label': 'Title', 'fieldname': 'title', 'reqd': 1, 'fieldtype': 'Data'},
+			{'label': 'Description', 'fieldname': 'description', 'fieldtype': 'Small Text'},
+			{'label': 'Date', 'fieldname': 'date', 'fieldtype': 'Date'},
+			{'label': 'Duration', 'fieldname': 'duration', 'fieldtype': 'Duration'},
+			{'label': 'Number', 'fieldname': 'number', 'fieldtype': 'Int'},
+			{'label': 'Number', 'fieldname': 'another_number', 'fieldtype': 'Int'}
+		],
+		'permissions': [
+			{'role': 'System Manager'}
+		]
+	}).insert(ignore_if_duplicate = True)

frappe/desk/doctype/workspace/workspace.json

@@ -28,7 +28,6 @@
   "pin_to_bottom",
   "hide_custom",
   "public",
-  "content_section",
   "content",
   "section_break_2",
   "charts_label",
@@ -39,6 +38,7 @@
   "section_break_18",
   "cards_label",
   "links",
+  "roles_section",
   "roles"
  ],
  "fields": [
@@ -46,6 +46,7 @@
    "fieldname": "label",
    "fieldtype": "Data",
    "label": "Name",
+   "reqd": 1,
    "unique": 1
   },
   {
@@ -232,21 +233,18 @@
   {
    "fieldname": "title",
    "fieldtype": "Data",
-   "label": "Title"
+   "label": "Title",
+   "reqd": 1
   },
   {
    "fieldname": "parent_page",
    "fieldtype": "Data",
    "label": "Parent Page"
   },
-  {
-   "fieldname": "content_section",
-   "fieldtype": "Section Break",
-   "label": "Content"
-  },
   {
    "fieldname": "content",
    "fieldtype": "Long Text",
+   "hidden": 1,
    "label": "Content"
   },
   {
@@ -259,10 +257,15 @@
    "fieldtype": "Table",
    "label": "Roles",
    "options": "Has Role"
+  },
+  {
+   "fieldname": "roles_section",
+   "fieldtype": "Section Break",
+   "label": "Roles"
   }
  ],
  "links": [],
- "modified": "2021-08-05 11:49:09.028243",
+ "modified": "2021-08-19 12:51:00.233017",
  "modified_by": "Administrator",
  "module": "Desk",
  "name": "Workspace",

frappe/desk/doctype/workspace/workspace.py

@@ -17,6 +17,12 @@ class Workspace(Document):
 			frappe.throw(_("You need to be in developer mode to edit this document"))
 		validate_route_conflict(self.doctype, self.name)
 
+		try:
+			if not isinstance(loads(self.content), list):
+				raise
+		except Exception:
+			frappe.throw(_("Content data shoud be a list"))
+
 		duplicate_exists = frappe.db.exists("Workspace", {
 			"name": ["!=", self.name], 'is_default': 1, 'extends': self.extends
 		})
@@ -56,7 +62,7 @@ class Workspace(Document):
 		for link in self.links:
 			link = link.as_dict()
 			if link.type == "Card Break":
-				if card_links and (not current_card.only_for or current_card.only_for == frappe.get_system_settings('country')):
+				if card_links and (not current_card['only_for'] or current_card['only_for'] == frappe.get_system_settings('country')): 
 					current_card['links'] = card_links
 					cards.append(current_card)
 

frappe/desk/form/utils.py

@@ -5,7 +5,7 @@ import frappe, json
 import frappe.desk.form.meta
 import frappe.desk.form.load
 from frappe.desk.form.document_follow import follow_document
-from frappe.utils.file_manager import extract_images_from_html
+from frappe.core.doctype.file.file import extract_images_from_html
 
 from frappe import _
 

frappe/desk/query_report.py

@@ -391,7 +391,7 @@ def handle_duration_fieldtype_values(result, columns):
 	return result
 
 
-def build_xlsx_data(columns, data, visible_idx, include_indentation):
+def build_xlsx_data(columns, data, visible_idx, include_indentation, ignore_visible_idx=False):
 	result = [[]]
 	column_widths = []
 
@@ -407,7 +407,7 @@ def build_xlsx_data(columns, data, visible_idx, include_indentation):
 	# build table from result
 	for row_idx, row in enumerate(data.result):
 		# only pick up rows that are visible in the report
-		if row_idx in visible_idx:
+		if ignore_visible_idx or row_idx in visible_idx:
 			row_data = []
 			if isinstance(row, dict):
 				for col_idx, column in enumerate(data.columns):

frappe/desk/search.py

@@ -265,6 +265,7 @@ def get_users_for_mentions():
 			'name': ['not in', ('Administrator', 'Guest')],
 			'allowed_in_mentions': True,
 			'user_type': 'System User',
+			'enabled': True,
 		})
 
 def get_user_groups():

frappe/email/doctype/auto_email_report/auto_email_report.py

@@ -13,6 +13,7 @@ from frappe.utils import (format_time, get_link_to_form, get_url_to_report,
 from frappe.model.naming import append_number_if_name_exists
 from frappe.utils.csvutils import to_csv
 from frappe.utils.xlsxutils import make_xlsx
+from frappe.desk.query_report import build_xlsx_data
 
 max_reports_per_user = frappe.local.conf.max_reports_per_user or 3
 
@@ -99,13 +100,21 @@ class AutoEmailReport(Document):
 			return self.get_html_table(columns, data)
 
 		elif self.format == 'XLSX':
-			spreadsheet_data = self.get_spreadsheet_data(columns, data)
-			xlsx_file = make_xlsx(spreadsheet_data, "Auto Email Report")
+			report_data = frappe._dict()
+			report_data['columns'] = columns
+			report_data['result'] = data
+
+			xlsx_data, column_widths = build_xlsx_data(columns, report_data, [], 1, ignore_visible_idx=True)
+			xlsx_file = make_xlsx(xlsx_data, "Auto Email Report", column_widths=column_widths)
 			return xlsx_file.getvalue()
 
 		elif self.format == 'CSV':
-			spreadsheet_data = self.get_spreadsheet_data(columns, data)
-			return to_csv(spreadsheet_data)
+			report_data = frappe._dict()
+			report_data['columns'] = columns
+			report_data['result'] = data
+
+			xlsx_data, column_widths = build_xlsx_data(columns, report_data, [], 1, ignore_visible_idx=True)
+			return to_csv(xlsx_data)
 
 		else:
 			frappe.throw(_('Invalid Output Format'))
@@ -126,18 +135,6 @@ class AutoEmailReport(Document):
 			'edit_report_settings': get_link_to_form('Auto Email Report', self.name)
 		})
 
-	@staticmethod
-	def get_spreadsheet_data(columns, data):
-		out = [[_(df.label) for df in columns], ]
-		for row in data:
-			new_row = []
-			out.append(new_row)
-			for df in columns:
-				if df.fieldname not in row: continue
-				new_row.append(frappe.format(row[df.fieldname], df, row))
-
-		return out
-
 	def get_file_name(self):
 		return "{0}.{1}".format(self.report.replace(" ", "-").replace("/", "-"), self.format.lower())
 

frappe/email/doctype/email_account/email_account.py

@@ -137,8 +137,6 @@ class EmailAccount(Document):
 
 	def on_update(self):
 		"""Check there is only one default of each type."""
-		from frappe.core.doctype.user.user import setup_user_email_inbox
-
 		self.check_automatic_linking_email_account()
 		self.there_must_be_only_one_default()
 		setup_user_email_inbox(email_account=self.name, awaiting_password=self.awaiting_password,
@@ -532,8 +530,6 @@ class EmailAccount(Document):
 
 	def on_trash(self):
 		"""Clear communications where email account is linked"""
-		from frappe.core.doctype.user.user import remove_user_email_inbox
-
 		frappe.db.sql("update `tabCommunication` set email_account='' where email_account=%s", self.name)
 		remove_user_email_inbox(email_account=self.name)
 
@@ -724,3 +720,84 @@ def get_max_email_uid(email_account):
 	else:
 		max_uid = cint(result[0].get("uid", 0)) + 1
 		return max_uid
+
+
+def setup_user_email_inbox(email_account, awaiting_password, email_id, enable_outgoing):
+	""" setup email inbox for user """
+	from frappe.core.doctype.user.user import ask_pass_update
+
+	def add_user_email(user):
+		user = frappe.get_doc("User", user)
+		row = user.append("user_emails", {})
+
+		row.email_id = email_id
+		row.email_account = email_account
+		row.awaiting_password = awaiting_password or 0
+		row.enable_outgoing = enable_outgoing or 0
+
+		user.save(ignore_permissions=True)
+
+	update_user_email_settings = False
+	if not all([email_account, email_id]):
+		return
+
+	user_names = frappe.db.get_values("User", {"email": email_id}, as_dict=True)
+	if not user_names:
+		return
+
+	for user in user_names:
+		user_name = user.get("name")
+
+		# check if inbox is alreay configured
+		user_inbox = frappe.db.get_value("User Email", {
+			"email_account": email_account,
+			"parent": user_name
+		}, ["name"]) or None
+
+		if not user_inbox:
+			add_user_email(user_name)
+		else:
+			# update awaiting password for email account
+			update_user_email_settings = True
+
+	if update_user_email_settings:
+		frappe.db.sql("""UPDATE `tabUser Email` SET awaiting_password = %(awaiting_password)s,
+			enable_outgoing = %(enable_outgoing)s WHERE email_account = %(email_account)s""", {
+				"email_account": email_account,
+				"enable_outgoing": enable_outgoing,
+				"awaiting_password": awaiting_password or 0
+			})
+	else:
+		users = " and ".join([frappe.bold(user.get("name")) for user in user_names])
+		frappe.msgprint(_("Enabled email inbox for user {0}").format(users))
+	ask_pass_update()
+
+def remove_user_email_inbox(email_account):
+	""" remove user email inbox settings if email account is deleted """
+	if not email_account:
+		return
+
+	users = frappe.get_all("User Email", filters={
+		"email_account": email_account
+	}, fields=["parent as name"])
+
+	for user in users:
+		doc = frappe.get_doc("User", user.get("name"))
+		to_remove = [row for row in doc.user_emails if row.email_account == email_account]
+		[doc.remove(row) for row in to_remove]
+
+		doc.save(ignore_permissions=True)
+
+@frappe.whitelist(allow_guest=False)
+def set_email_password(email_account, user, password):
+	account = frappe.get_doc("Email Account", email_account)
+	if account.awaiting_password:
+		account.awaiting_password = 0
+		account.password = password
+		try:
+			account.save(ignore_permissions=True)
+		except Exception:
+			frappe.db.rollback()
+			return False
+
+	return True

frappe/email/doctype/email_account/test_email_account.py

@@ -34,8 +34,8 @@ class TestEmailAccount(unittest.TestCase):
 	def setUp(self):
 		frappe.flags.mute_emails = False
 		frappe.flags.sent_mail = None
-		frappe.db.sql('delete from `tabEmail Queue`')
-		frappe.db.sql('delete from `tabUnhandled Email`')
+		frappe.db.delete("Email Queue")
+		frappe.db.delete("Unhandled Email")
 
 	def get_test_mail(self, fname):
 		with open(os.path.join(os.path.dirname(__file__), "test_mails", fname), "r") as f:
@@ -60,7 +60,7 @@ class TestEmailAccount(unittest.TestCase):
 		comm = frappe.get_doc("Communication", {"sender": "test_sender@example.com"})
 		comm.db_set("creation", datetime.now() - timedelta(seconds = 30 * 60))
 
-		frappe.db.sql("DELETE FROM `tabEmail Queue`")
+		frappe.db.delete("Email Queue")
 		notify_unreplied()
 		self.assertTrue(frappe.db.get_value("Email Queue", {"reference_doctype": comm.reference_doctype,
 			"reference_name": comm.reference_name, "status":"Not Sent"}))
@@ -183,7 +183,7 @@ class TestEmailAccount(unittest.TestCase):
 
 	def test_threading_by_message_id(self):
 		cleanup()
-		frappe.db.sql("""delete from `tabEmail Queue`""")
+		frappe.db.delete("Email Queue")
 
 		# reference document for testing
 		event = frappe.get_doc(dict(doctype='Event', subject='test-message')).insert()
@@ -242,8 +242,8 @@ class TestInboundMail(unittest.TestCase):
 
 	def setUp(self):
 		cleanup()
-		frappe.db.sql('delete from `tabEmail Queue`')
-		frappe.db.sql('delete from `tabToDo`')
+		frappe.db.delete("Email Queue")
+		frappe.db.delete("ToDo")
 
 	def get_test_mail(self, fname):
 		with open(os.path.join(os.path.dirname(__file__), "test_mails", fname), "r") as f:

frappe/email/doctype/newsletter/newsletter.py

@@ -165,6 +165,7 @@ class Newsletter(WebsiteGenerator):
 		sender = self.send_from or frappe.utils.get_formatted_email(self.owner)
 		args = {"message": message, "name": self.name}
 
+		is_auto_commit_set = bool(frappe.db.auto_commit_on_many_writes)
 		frappe.db.auto_commit_on_many_writes = not frappe.flags.in_test
 
 		frappe.sendmail(
@@ -184,7 +185,7 @@ class Newsletter(WebsiteGenerator):
 			args=args,
 		)
 
-		frappe.db.auto_commit_on_many_writes = not frappe.flags.in_test
+		frappe.db.auto_commit_on_many_writes = is_auto_commit_set
 
 	def get_message(self) -> str:
 		if self.content_type == "HTML":

frappe/email/doctype/notification/test_notification.py

@@ -9,7 +9,7 @@ test_dependencies = ["User", "Notification"]
 
 class TestNotification(unittest.TestCase):
 	def setUp(self):
-		frappe.db.sql("""delete from `tabEmail Queue`""")
+		frappe.db.delete("Email Queue")
 		frappe.set_user("test@example.com")
 
 		if not frappe.db.exists('Notification', {'name': 'ToDo Status Update'}, 'name'):
@@ -50,7 +50,7 @@ class TestNotification(unittest.TestCase):
 
 		self.assertTrue(frappe.db.get_value("Email Queue", {"reference_doctype": "Communication",
 			"reference_name": communication.name, "status":"Not Sent"}))
-		frappe.db.sql("""delete from `tabEmail Queue`""")
+		frappe.db.delete("Email Queue")
 
 		communication.reload()
 		communication.content = "test 2"
@@ -189,9 +189,9 @@ class TestNotification(unittest.TestCase):
 
 	def test_cc_jinja(self):
 
-		frappe.db.sql("""delete from `tabUser` where email='test_jinja@example.com'""")
-		frappe.db.sql("""delete from `tabEmail Queue`""")
-		frappe.db.sql("""delete from `tabEmail Queue Recipient`""")
+		frappe.db.delete("User", {"email": "test_jinja@example.com"})
+		frappe.db.delete("Email Queue")
+		frappe.db.delete("Email Queue Recipient")
 
 		test_user = frappe.new_doc("User")
 		test_user.name = 'test_jinja'
@@ -205,9 +205,9 @@ class TestNotification(unittest.TestCase):
 
 		self.assertTrue(frappe.db.get_value("Email Queue Recipient", {"recipient": "test_jinja@example.com"}))
 
-		frappe.db.sql("""delete from `tabUser` where email='test_jinja@example.com'""")
-		frappe.db.sql("""delete from `tabEmail Queue`""")
-		frappe.db.sql("""delete from `tabEmail Queue Recipient`""")
+		frappe.db.delete("User", {"email": "test_jinja@example.com"})
+		frappe.db.delete("Email Queue")
+		frappe.db.delete("Email Queue Recipient")
 
 	def test_notification_to_assignee(self):
 		todo = frappe.new_doc('ToDo')

frappe/integrations/doctype/webhook/test_webhook.py

@@ -11,9 +11,9 @@ class TestWebhook(unittest.TestCase):
 	@classmethod
 	def setUpClass(cls):
 		# delete any existing webhooks
-		frappe.db.sql("DELETE FROM tabWebhook")
+		frappe.db.delete("Webhook")
 		# Delete existing logs if any
-		frappe.db.sql("DELETE FROM `tabWebhook Request Log`")
+		frappe.db.delete("Webhook Request Log")
 		# create test webhooks
 		cls.create_sample_webhooks()
 
@@ -46,7 +46,7 @@ class TestWebhook(unittest.TestCase):
 	@classmethod
 	def tearDownClass(cls):
 		# delete any existing webhooks
-		frappe.db.sql("DELETE FROM tabWebhook")
+		frappe.db.delete("Webhook")
 
 	def setUp(self):
 		# retrieve or create a User webhook for `after_insert`
@@ -168,7 +168,7 @@ class TestWebhook(unittest.TestCase):
 	def test_webhook_req_log_creation(self):
 		if not frappe.db.get_value('User', 'user2@integration.webhooks.test.com'):
 			user = frappe.get_doc({
-				'doctype': 'User', 
+				'doctype': 'User',
 				'email': 'user2@integration.webhooks.test.com',
 				'first_name': 'user2'
 			}).insert()

frappe/model/base_document.py

@@ -874,7 +874,7 @@ class BaseDocument(object):
 		return self._precision[cache_key][fieldname]
 
 
-	def get_formatted(self, fieldname, doc=None, currency=None, absolute_value=False, translated=False):
+	def get_formatted(self, fieldname, doc=None, currency=None, absolute_value=False, translated=False, format=None):
 		from frappe.utils.formatters import format_value
 
 		df = self.meta.get_field(fieldname)
@@ -898,7 +898,7 @@ class BaseDocument(object):
 		if (absolute_value or doc.get('absolute_value')) and isinstance(val, (int, float)):
 			val = abs(self.get(fieldname))
 
-		return format_value(val, df=df, doc=doc, currency=currency)
+		return format_value(val, df=df, doc=doc, currency=currency, format=format)
 
 	def is_print_hide(self, fieldname, df=None, for_print=True):
 		"""Returns true if fieldname is to be hidden for print.

frappe/model/delete_doc.py

@@ -10,7 +10,7 @@ import frappe.model.meta
 from frappe import _
 from frappe import get_module_path
 from frappe.model.dynamic_links import get_dynamic_link_map
-from frappe.core.doctype.file.file import remove_all
+from frappe.utils.file_manager import remove_all
 from frappe.utils.password import delete_all_passwords_for
 from frappe.model.naming import revert_series_if_last
 from frappe.utils.global_search import delete_for_document
@@ -190,7 +190,7 @@ def delete_from_table(doctype, name, ignore_doctypes, doc):
 	# delete from child tables
 	for t in list(set(tables)):
 		if t not in ignore_doctypes:
-			frappe.db.sql("delete from `tab%s` where parenttype=%s and parent = %s" % (t, '%s', '%s'), (doctype, name))
+			frappe.db.delete(t, {"parenttype": doctype, "parent": name})
 
 def update_flags(doc, flags=None, ignore_permissions=False):
 	if ignore_permissions:
@@ -323,9 +323,10 @@ def delete_dynamic_links(doctype, name):
 
 def delete_references(doctype, reference_doctype, reference_name,
 		reference_doctype_field = 'reference_doctype', reference_name_field = 'reference_name'):
-	frappe.db.sql('''delete from `tab{0}`
-		where {1}=%s and {2}=%s'''.format(doctype, reference_doctype_field, reference_name_field), # nosec
-		(reference_doctype, reference_name))
+	frappe.db.delete(doctype, {
+		reference_doctype_field: reference_doctype,
+		reference_name_field: reference_name
+	})
 
 def clear_references(doctype, reference_doctype, reference_name,
 		reference_doctype_field = 'reference_doctype', reference_name_field = 'reference_name'):

frappe/model/document.py

@@ -385,8 +385,7 @@ class Document(BaseDocument):
 					[self.name, self.doctype, fieldname] + rows)
 			if len(deleted_rows) > 0:
 				# delete rows that do not match the ones in the document
-				frappe.db.sql("""delete from `tab{0}` where name in ({1})""".format(df.options,
-					','.join(['%s'] * len(deleted_rows))), tuple(row[0] for row in deleted_rows))
+				frappe.db.delete(df.options, {"name": ("in", tuple(row[0] for row in deleted_rows))})
 
 		else:
 			# no rows found, delete all rows

frappe/modules/utils.py

@@ -114,8 +114,7 @@ def sync_customizations_for_doctype(data, folder):
 					doc.db_insert()
 
 			if custom_doctype != 'Custom Field':
-				frappe.db.sql('delete from `tab{0}` where `{1}` =%s'.format(
-					custom_doctype, doctype_fieldname), doc_type)
+				frappe.db.delete(custom_doctype, {doctype_fieldname: doc_type})
 
 				for d in data[key]:
 					_insert(d)

frappe/patches.txt

@@ -182,4 +182,4 @@ frappe.patches.v13_0.jinja_hook
 frappe.patches.v13_0.update_notification_channel_if_empty
 frappe.patches.v14_0.drop_data_import_legacy
 frappe.patches.v14_0.rename_cancelled_documents
-frappe.patches.v14_0.update_workspace2
+frappe.patches.v14_0.update_workspace2 # 25.08.2021

frappe/patches/v14_0/update_workspace2.py

@@ -50,11 +50,11 @@ def create_content(doc):
 	return content
 
 def update_wspace(doc, seq, content):
-	if not doc.is_standard and not doc.public:
+	if not doc.title and not doc.content and not doc.is_standard and not doc.public:
 		doc.sequence_id = seq + 1
 		doc.content = json.dumps(content)
 		doc.public = 0
-		doc.title = doc.extends
+		doc.title = doc.extends or doc.label
 		doc.extends = ''
 		doc.category = ''
 		doc.onboarding = ''

frappe/public/icons/timeless/symbol-defs.svg

@@ -567,7 +567,7 @@
 		<path d="M6.45466 8.81824L4.47873 10.7942C3.85205 11.4211 3.5 12.2713 3.5 13.1577C3.5 14.0442 3.85205 14.8943 4.47873 15.5213V15.5213C5.10568 16.148 5.95584 16.5 6.84229 16.5C7.72874 16.5 8.5789 16.148 9.20584 15.5213L11.1818 13.5453" stroke="var(--icon-stroke)" stroke-linecap="round" stroke-linejoin="round"/>
 	</symbol>
 	<symbol viewBox="0 0 24 24" fill="none" id="icon-scan" xmlns="http://www.w3.org/2000/svg">
-		<path d="M8 3H5a2 2 0 0 0-2 2v3m18 0V5a2 2 0 0 0-2-2h-3m0 18h3a2 2 0 0 0 2-2v-3M3 16v3a2 2 0 0 0 2 2h3" 
+		<path d="M8 3H5a2 2 0 0 0-2 2v3m18 0V5a2 2 0 0 0-2-2h-3m0 18h3a2 2 0 0 0 2-2v-3M3 16v3a2 2 0 0 0 2 2h3"
 			stroke="var(--icon-stroke)" stroke-linecap="round" stroke-linejoin="round" stroke-width="1.5"/>
 	</symbol>
 	<symbol viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg" id="icon-dashboard">

frappe/public/js/frappe/desk.js

@@ -230,7 +230,7 @@ frappe.Application = class Application {
 			s.fields_dict.checking.$wrapper.html('<i class="fa fa-spinner fa-spin fa-4x"></i>');
 			s.show();
 			frappe.call({
-				method: 'frappe.core.doctype.user.user.set_email_password',
+				method: 'frappe.email.doctype.email_account.email_account.set_email_password',
 				args: {
 					"email_account": email_account[i]["email_account"],
 					"user": user,

frappe/public/js/frappe/file_uploader/index.js

@@ -28,7 +28,7 @@ export default class FileUploader {
 		}
 
 		if (attach_doc_image) {
-			restrictions.allowed_file_types = ['.jpg', '.jpeg', '.png'];
+			restrictions.allowed_file_types = ['image/jpeg', 'image/png'];
 		}
 
 		this.$fileuploader = new Vue({
@@ -70,8 +70,10 @@ export default class FileUploader {
 		this.uploader.$watch('hide_dialog_footer', (hide_dialog_footer) => {
 			if (hide_dialog_footer) {
 				this.dialog && this.dialog.footer.addClass('hide');
+				this.dialog.$wrapper.data('bs.modal')._config.backdrop = 'static';
 			} else {
 				this.dialog && this.dialog.footer.removeClass('hide');
+				this.dialog.$wrapper.data('bs.modal')._config.backdrop = true;
 			}
 		});
 

frappe/public/js/frappe/form/controls/code.js

@@ -9,12 +9,7 @@ frappe.ui.form.ControlCode = class ControlCode extends frappe.ui.form.ControlTex
 		this.ace_editor_target = $('<div class="ace-editor-target"></div>')
 			.appendTo(this.input_area);
 
-		this.expanded = false;
-		this.$expand_button = $(`<button class="btn btn-xs btn-default">${this.get_button_label()}</button>`).click(() => {
-			this.expanded = !this.expanded;
-			this.refresh_height();
-			this.toggle_label();
-		}).appendTo(this.$input_wrapper);
+
 		// styling
 		this.ace_editor_target.addClass('border rounded');
 		this.ace_editor_target.css('height', 300);
@@ -22,6 +17,21 @@ frappe.ui.form.ControlCode = class ControlCode extends frappe.ui.form.ControlTex
 		// initialize
 		const ace = window.ace;
 		this.editor = ace.edit(this.ace_editor_target.get(0));
+
+		if (this.df.max_lines || this.df.min_lines) {
+			if (this.df.max_lines)
+				this.editor.setOption("maxLines", this.df.max_lines);
+			if (this.df.min_lines)
+				this.editor.setOption("minLines", this.df.min_lines);
+		} else {
+			this.expanded = false;
+			this.$expand_button = $(`<button class="btn btn-xs btn-default">${this.get_button_label()}</button>`).click(() => {
+				this.expanded = !this.expanded;
+				this.refresh_height();
+				this.toggle_label();
+			}).appendTo(this.$input_wrapper);
+		}
+
 		this.editor.setTheme('ace/theme/tomorrow');
 		this.editor.setOption("showPrintMargin", false);
 		this.editor.setOption("wrap", this.df.wrap);

frappe/public/js/frappe/form/controls/data.js

@@ -163,7 +163,7 @@ frappe.ui.form.ControlData = class ControlData extends frappe.ui.form.ControlInp
 			}
 		};
 		this.$input.on("change", change_handler);
-		if (this.constructor.trigger_change_on_input_event) {
+		if (this.constructor.trigger_change_on_input_event && !this.in_grid()) {
 			// debounce to avoid repeated validations on value change
 			this.$input.on("input", frappe.utils.debounce(change_handler, 500));
 		}
@@ -267,4 +267,7 @@ frappe.ui.form.ControlData = class ControlData extends frappe.ui.form.ControlInp
 		let el = this.$input.parents(el_class)[0];
 		if (el) $(el).toggleClass(scroll_class, add);
 	}
+	in_grid() {
+		return this.grid || this.layout && this.layout.grid;
+	}
 };

frappe/public/js/frappe/form/controls/datetime.js

@@ -36,4 +36,9 @@ frappe.ui.form.ControlDatetime = class ControlDatetime extends frappe.ui.form.Co
 			$tp.$secondsText.prev().css('display', 'none');
 		}
 	}
+
+	get_model_value() {
+		let value = super.get_model_value();
+		return frappe.datetime.get_datetime_as_string(value);
+	}
 };

frappe/public/js/frappe/form/dashboard.js

@@ -283,21 +283,13 @@ frappe.ui.form.Dashboard = class FormDashboard {
 		$(frappe.render_template('form_links', this.data))
 			.appendTo(transactions_area_body);
 
-		if (this.data.reports && this.data.reports.length) {
-			$(frappe.render_template('report_links', this.data))
-				.appendTo(transactions_area_body);
-		}
+		this.render_report_links();
 
 		// bind links
 		transactions_area_body.find(".badge-link").on('click', function() {
 			me.open_document_list($(this).closest('.document-link'));
 		});
 
-		// bind reports
-		transactions_area_body.find(".report-link").on('click', function() {
-			me.open_report($(this).parent());
-		});
-
 		// bind open notifications
 		transactions_area_body.find('.open-notification').on('click', function() {
 			me.open_document_list($(this).parent(), true);
@@ -311,6 +303,18 @@ frappe.ui.form.Dashboard = class FormDashboard {
 		this.data_rendered = true;
 	}
 
+	render_report_links() {
+		let parent = this.transactions_area;
+		if (this.data.reports && this.data.reports.length) {
+			$(frappe.render_template('report_links', this.data))
+				.appendTo(parent);
+			// bind reports
+			parent.find(".report-link").on('click', (e) => {
+				this.open_report($(e.target).parent());
+			});
+		}
+	}
+
 	open_report($link) {
 		let report = $link.attr('data-report');
 
@@ -318,6 +322,7 @@ frappe.ui.form.Dashboard = class FormDashboard {
 			? (this.data.non_standard_fieldnames[report] || this.data.fieldname)
 			: this.data.fieldname;
 
+		frappe.provide('frappe.route_options');
 		frappe.route_options[fieldname] = this.frm.doc.name;
 		frappe.set_route("query-report", report);
 	}

frappe/public/js/frappe/form/form.js

@@ -339,7 +339,7 @@ frappe.ui.form.Form = class FrappeForm {
 			}
 		}
 		if (action.action_type==='Server Action') {
-			frappe.xcall(action.action, {'doc': this.doc}).then((doc) => {
+			return frappe.xcall(action.action, {'doc': this.doc}).then((doc) => {
 				if (doc.doctype) {
 					// document is returned by the method,
 					// apply the changes locally and refresh
@@ -354,7 +354,7 @@ frappe.ui.form.Form = class FrappeForm {
 				});
 			});
 		} else if (action.action_type==='Route') {
-			frappe.set_route(action.action);
+			return frappe.set_route(action.action);
 		}
 	}
 

frappe/public/js/frappe/form/form_tour.js

@@ -82,10 +82,16 @@ frappe.ui.form.FormTour = class FormTour {
 
 	get_step(step_info, on_next) {
 		const { name, fieldname, title, description, position, is_table_field } = step_info;
+		let element = `.frappe-control[data-fieldname='${fieldname}']`;
+
 		const field = this.frm.get_field(fieldname);
-		let element = field ? field.wrapper : `.frappe-control[data-fieldname='${fieldname}']`;
+		if (field) {
+			// wrapper for section breaks returns in a list
+			element = field.wrapper[0] ? field.wrapper[0] : field.wrapper;
+		}
 
 		if (is_table_field) {
+			// TODO: fix wrapper for grid sections
 			element = `.grid-row-open .frappe-control[data-fieldname='${fieldname}']`;
 		}
 

frappe/public/js/frappe/form/grid.js

@@ -38,7 +38,7 @@ export default class Grid {
 
 		this.is_grid = true;
 		this.debounced_refresh = this.refresh.bind(this);
-		this.debounced_refresh = frappe.utils.debounce(this.debounced_refresh, 500);
+		this.debounced_refresh = frappe.utils.debounce(this.debounced_refresh, 100);
 	}
 
 	allow_on_grid_editing() {

frappe/public/js/frappe/form/layout.js

@@ -252,14 +252,18 @@ frappe.ui.form.Layout = class Layout {
 		}
 
 		if (document.activeElement) {
-			document.activeElement.focus();
-	
-			if (document.activeElement.tagName == 'INPUT') {
+			if (document.activeElement.tagName == 'INPUT' && this.is_numeric_field_active()) {
 				document.activeElement.select();
 			}
 		}
 	}
 
+	is_numeric_field_active() {
+		const control = $(document.activeElement).closest(".frappe-control");
+		const fieldtype = (control.data() || {}).fieldtype;
+		return frappe.model.numeric_fieldtypes.includes(fieldtype);
+	}
+
 	refresh_sections() {
 		// hide invisible sections
 		this.wrapper.find(".form-section:not(.hide-control)").each(function() {

frappe/public/js/frappe/list/list_view.js

@@ -367,6 +367,7 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList {
 
 		if (
 			!this.settings.hide_name_column &&
+			this.meta.title_field &&
 			this.meta.title_field !== 'name'
 		) {
 			this.columns.push({
@@ -867,8 +868,9 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList {
 			filters: this.get_filters_for_args()
 		}).then(total_count => {
 			this.total_count = total_count || current_count;
+			this.count_without_children = count_without_children !== current_count ? count_without_children : undefined;
 			let str = __('{0} of {1}', [current_count, this.total_count]);
-			if (count_without_children !== current_count) {
+			if (this.count_without_children) {
 				str = __('{0} of {1} ({2} rows with children)', [count_without_children, this.total_count, current_count]);
 			}
 			return str;

frappe/public/js/frappe/recorder/RecorderDetail.vue

@@ -1,5 +1,5 @@
 <template>
-	<div>
+	<div v-cloak @drop.prevent="import_data" @dragover.prevent>
 		<div class="page-form">
 			<div class="filter-list">
 				<div class="tag-filters-area">
@@ -46,7 +46,7 @@
 
 				</div>
 				<div class="result-list">
-					<div class="list-row-container" v-for="(request, index) in paginated(sorted(filtered(requests)))" :key="index" @click="route_to_request_detail(request.uuid)">
+					<div class="list-row-container" v-for="(request, index) in paginated(sorted(filtered(requests)))" :key="index" @click="route_to_request_detail(request)">
 						<div class="level list-row small">
 							<div class="level-left ellipsis">
 								<div class="list-row-col ellipsis list-subject level ">
@@ -71,15 +71,16 @@
 			</div>
 			<div v-if="requests.length == 0" class="no-result text-muted flex justify-center align-center" style="">
 				<div class="msg-box no-border" v-if="status.status == 'Inactive'" >
-					<p>{{ __("Recorder is Inactive") }}</p>
 					<p><button class="btn btn-primary btn-sm btn-new-doc" @click="start()">{{ __("Start Recording") }}</button></p>
+					<p>{{ __("Recorder is Inactive.") }}</p>
+					<p>{{ __("Start recording or drag & drop a previously exported data file to view it.") }}</p>
 				</div>
 				<div class="msg-box no-border" v-if="status.status == 'Active'" >
 					<p>{{ __("No Requests found") }}</p>
 					<p>{{ __("Go make some noise") }}</p>
 				</div>
 			</div>
-			<div v-if="requests.length != 0" class="list-paging-area">
+			<div v-else class="list-paging-area">
 				<div class="row">
 					<div class="col-xs-6">
 						<div class="btn-group btn-group-paging">
@@ -144,7 +145,7 @@ export default {
 			frappe.set_route("recorder");
 			this.clear();
 		});
-
+		this.$root.page.add_menu_item("Export data", () => this.export_data());
 	},
 	computed: {
 		pages: function() {
@@ -239,8 +240,36 @@ export default {
 				});
 			}
 		},
-		route_to_request_detail(id) {
-			this.$router.push({name: 'request-detail', params: {id}});
+		route_to_request_detail(request) {
+			this.$router.push({name: 'request-detail', params: {request, id: request.uuid}});
+		},
+		export_data: function() {
+			if (!this.requests) {
+				return;
+			}
+			frappe.call("frappe.recorder.export_data")
+				.then((r) => {
+					const data = r.message;
+					const filename = `${data[0]['uuid']}..${data[data.length -1]['uuid']}.json`
+
+					const el = document.createElement('a');
+					el.setAttribute('href', 'data:application/json,' + encodeURIComponent(JSON.stringify(data)));
+					el.setAttribute('download', filename);
+					el.click();
+				});
+		},
+		import_data: function(e) {
+			if (this.requests.length > 0) {
+				// don't replace existing capture
+				return;
+			}
+			const request_file = e.dataTransfer.files[0];
+
+			const file_reader = new FileReader();
+			file_reader.readAsText(request_file, 'UTF-8');
+			file_reader.onload = ({target: {result}}) => {
+				 this.requests = JSON.parse(result);
+			}
 		}
 	}
 };

frappe/public/js/frappe/recorder/RequestDetail.vue

@@ -283,14 +283,21 @@ export default {
 			label: __('Recorder'),
 			route: '/app/recorder'
 		});
-		frappe.call({
-			method: "frappe.recorder.get",
-			args: {
-				uuid: this.$route.params.id
-			}
-		}).then( r => {
-			this.request = r.message
-		});
-	}
+
+		const request = this.$route.params.request;
+		if (request.headers || request.form_dict || request.calls) {
+			// complete request data passed as parameter.
+			this.request = request;
+		} else {
+			frappe.call({
+				method: "frappe.recorder.get",
+				args: {
+					uuid: request.uuid
+				}
+			}).then( r => {
+				this.request = r.message
+			});
+		}
+	},
 };
 </script>

frappe/public/js/frappe/router.js

@@ -129,7 +129,7 @@ frappe.router = {
 		if (frappe.workspaces[route[0]]) {
 			// public workspace
 			route = ['Workspaces', frappe.workspaces[route[0]].title];
-		} else if (frappe.workspaces[route[1]]) {
+		} else if (route[0] == 'private' && frappe.workspaces[route[1]]) {
 			// private workspace
 			route = ['Workspaces', 'private', frappe.workspaces[route[1]].title];
 		} else if (this.routes[route[0]]) {
@@ -354,8 +354,8 @@ frappe.router = {
 				return a;
 			}
 		}).join('/');
-
-		return '/app/' + (path_string || 'home');
+		let default_page = frappe.workspaces['home'] ? 'home' : Object.keys(frappe.workspaces)[0];
+		return '/app/' + (path_string || default_page);
 	},
 
 	push_state(url) {

frappe/public/js/frappe/ui/page.js

@@ -251,7 +251,7 @@ frappe.ui.Page = class Page {
 			.prop("disabled", false)
 			.html(opts.label)
 			.on("click", function() {
-				let response = opts.click.apply(this);
+				let response = opts.click.apply(this, [btn]);
 				me.btn_disable_enable(btn, response);
 			});
 

frappe/public/js/frappe/ui/toolbar/awesome_bar.js

@@ -194,9 +194,14 @@ frappe.search.AwesomeBar = class AwesomeBar {
 		var out = [], routes = [];
 		options.forEach(function(option) {
 			if(option.route) {
-				if(option.route[0] === "List" && option.route[2] !== 'Report') {
+				if (
+					option.route[0] === "List" &&
+					option.route[2] !== 'Report' &&
+					option.route[2] !== 'Inbox'
+				) {
 					option.route.splice(2);
 				}
+
 				var str_route = (typeof option.route==='string') ?
 					option.route : option.route.join('/');
 				if(routes.indexOf(str_route)===-1) {

frappe/public/js/frappe/views/inbox/inbox_view.js

@@ -94,19 +94,20 @@ frappe.views.InboxView = class InboxView extends frappe.views.ListView {
 		this.render_list();
 		this.on_row_checked();
 		this.render_count();
-		this.render_tags();
 	}
 
 	get_meta_html(email) {
 		const attachment = email.has_attachment ?
 			`<span class="fa fa-paperclip fa-large" title="${__('Has Attachments')}"></span>` : '';
 
-		const form_link = frappe.utils.get_form_link(email.reference_doctype, email.reference_name);
-		const link = email.reference_doctype && email.reference_doctype !== this.doctype ?
-			`<a class="text-muted grey" href="${form_link}"
+		let link = "";
+		if (email.reference_doctype && email.reference_doctype !== this.doctype) {
+			link = `<a class="text-muted grey"
+				href="${frappe.utils.get_form_link(email.reference_doctype, email.reference_name)}"
 				title="${__('Linked with {0}', [email.reference_doctype])}">
 				<i class="fa fa-link fa-large"></i>
-			</a>` : '';
+			</a>`;
+		}
 
 		const communication_date = comment_when(email.communication_date, true);
 		const status =

frappe/public/js/frappe/views/reports/query_report.js

@@ -854,6 +854,10 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
 				}
 			};
 
+			if (this.raw_data.add_total_row) {
+				this.$page.find('.layout-main-section').css('--report-total-height', '310px');
+			}
+
 			if (this.report_settings.get_datatable_options) {
 				datatable_options = this.report_settings.get_datatable_options(datatable_options);
 			}

frappe/public/js/frappe/views/reports/report_view.js

@@ -1401,7 +1401,7 @@ frappe.views.ReportView = class ReportView extends frappe.views.ListView {
 						}
 					];
 
-					if (this.total_count > args.page_length) {
+					if (this.total_count > this.count_without_children || args.page_length) {
 						fields.push({
 							fieldtype: 'Check',
 							fieldname: 'export_all_rows',

frappe/public/js/frappe/views/workspace/workspace.js

@@ -174,10 +174,6 @@ frappe.views.Workspace = class Workspace {
 			$(e.target).parent().find('.sidebar-item-container').toggleClass('hidden');
 		});
 
-		if (!this.current_page.name) {
-			$title.trigger("click");
-		}
-
 		if (Object.keys(root_pages).length === 0) {
 			sidebar_section.addClass('hidden');
 		}

frappe/public/scss/desk/form.scss

@@ -115,7 +115,7 @@
 		margin-bottom: var(--margin-sm);
 		font-weight: var(--text-bold);
 	}
-	.row:first-child {
+	.form-documents:first-of-type .row:first-child {
 		.form-link-title {
 			margin-top: 0;
 		}

frappe/public/scss/desk/report.scss

@@ -84,8 +84,9 @@
 	margin-bottom: 10px;
 }
 
-.layout-main-section .frappe-card {
+.layout-main-section {
 	--report-filter-height: 0px;
+	--report-total-height: 275px;
 }
 
 .report-wrapper {
@@ -95,7 +96,7 @@
 		height: calc(100vh - var(--report-filter-height) - 205px);
 
 		.dt-scrollable {
-			height: calc(100vh - var(--report-filter-height) - 275px);
+			height: calc(100vh - var(--report-filter-height) - var(--report-total-height));
 		}
 	}
 }

frappe/rate_limiter.py

@@ -103,7 +103,7 @@ def rate_limit(key: str, limit: Union[int, Callable] = 5, seconds: int= 24*60*60
 		def wrapper(*args, **kwargs):
 			# Do not apply rate limits if method is not opted to check
 			if methods != 'ALL' and frappe.request.method.upper() not in methods:
-				return frappe.call(fun, **frappe.form_dict)
+				return frappe.call(fun, **frappe.form_dict or kwargs)
 
 			_limit = limit() if callable(limit) else limit
 
@@ -118,6 +118,6 @@ def rate_limit(key: str, limit: Union[int, Callable] = 5, seconds: int= 24*60*60
 			if value > _limit:
 				frappe.throw(_("You hit the rate limit because of too many requests. Please try after sometime."))
 
-			return frappe.call(fun, **frappe.form_dict)
+			return frappe.call(fun, **frappe.form_dict or kwargs)
 		return wrapper
 	return ratelimit_decorator

frappe/recorder.py

@@ -181,6 +181,13 @@ def get(uuid=None, *args, **kwargs):
 	return result
 
 
+@frappe.whitelist()
+@do_not_record
+@administrator_only
+def export_data(*args, **kwargs):
+	return list(frappe.cache().hgetall(RECORDER_REQUEST_HASH).values())
+
+
 @frappe.whitelist()
 @do_not_record
 @administrator_only

frappe/social/doctype/energy_point_log/test_energy_point_log.py

@@ -13,8 +13,8 @@ class TestEnergyPointLog(unittest.TestCase):
 
 	def tearDown(self):
 		frappe.set_user('Administrator')
-		frappe.db.sql('DELETE FROM `tabEnergy Point Log`')
-		frappe.db.sql('DELETE FROM `tabEnergy Point Rule`')
+		frappe.db.delete("Energy Point Log")
+		frappe.db.delete("Energy Point Rule")
 		frappe.cache().delete_value('energy_point_rule_map')
 
 	def test_user_energy_point(self):

frappe/test_runner.py

@@ -50,7 +50,7 @@ def main(app=None, module=None, doctype=None, verbose=False, tests=(),
 			frappe.connect()
 
 		# if not frappe.conf.get("db_name").startswith("test_"):
-		# 	raise Exception, 'db_name must start with "test_"'
+		#	raise Exception, 'db_name must start with "test_"'
 
 		# workaround! since there is no separate test db
 		frappe.clear_cache()
@@ -65,9 +65,9 @@ def main(app=None, module=None, doctype=None, verbose=False, tests=(),
 				frappe.get_attr(fn)()
 
 		if doctype:
-			ret = run_tests_for_doctype(doctype, verbose, tests, force, profile, junit_xml_output=junit_xml_output)
+			ret = run_tests_for_doctype(doctype, verbose, tests, force, profile, failfast=failfast, junit_xml_output=junit_xml_output)
 		elif module:
-			ret = run_tests_for_module(module, verbose, tests, profile, junit_xml_output=junit_xml_output)
+			ret = run_tests_for_module(module, verbose, tests, profile, failfast=failfast, junit_xml_output=junit_xml_output)
 		else:
 			ret = run_all_tests(app, verbose, profile, ui_tests, failfast=failfast, junit_xml_output=junit_xml_output)
 
@@ -150,7 +150,7 @@ def run_all_tests(app=None, verbose=False, profile=False, ui_tests=False, failfa
 
 	return out
 
-def run_tests_for_doctype(doctypes, verbose=False, tests=(), force=False, profile=False, junit_xml_output=False):
+def run_tests_for_doctype(doctypes, verbose=False, tests=(), force=False, profile=False, failfast=False, junit_xml_output=False):
 	modules = []
 	if not isinstance(doctypes, (list, tuple)):
 		doctypes = [doctypes]
@@ -168,18 +168,18 @@ def run_tests_for_doctype(doctypes, verbose=False, tests=(), force=False, profil
 		make_test_records(doctype, verbose=verbose, force=force)
 		modules.append(importlib.import_module(test_module))
 
-	return _run_unittest(modules, verbose=verbose, tests=tests, profile=profile, junit_xml_output=junit_xml_output)
+	return _run_unittest(modules, verbose=verbose, tests=tests, profile=profile, failfast=failfast, junit_xml_output=junit_xml_output)
 
-def run_tests_for_module(module, verbose=False, tests=(), profile=False, junit_xml_output=False):
+def run_tests_for_module(module, verbose=False, tests=(), profile=False, failfast=False, junit_xml_output=False):
 	module = importlib.import_module(module)
 	if hasattr(module, "test_dependencies"):
 		for doctype in module.test_dependencies:
 			make_test_records(doctype, verbose=verbose)
 
 	frappe.db.commit()
-	return _run_unittest(module, verbose=verbose, tests=tests, profile=profile, junit_xml_output=junit_xml_output)
+	return _run_unittest(module, verbose=verbose, tests=tests, profile=profile, failfast=failfast, junit_xml_output=junit_xml_output)
 
-def _run_unittest(modules, verbose=False, tests=(), profile=False, junit_xml_output=False):
+def _run_unittest(modules, verbose=False, tests=(), profile=False, failfast=False, junit_xml_output=False):
 	frappe.db.begin()
 
 	test_suite = unittest.TestSuite()
@@ -198,9 +198,9 @@ def _run_unittest(modules, verbose=False, tests=(), profile=False, junit_xml_out
 			test_suite.addTest(module_test_cases)
 
 	if junit_xml_output:
-		runner = unittest_runner(verbosity=1+(verbose and 1 or 0))
+		runner = unittest_runner(verbosity=1+(verbose and 1 or 0), failfast=failfast)
 	else:
-		runner = unittest_runner(resultclass=TimeLoggingTestResult, verbosity=1+(verbose and 1 or 0))
+		runner = unittest_runner(resultclass=TimeLoggingTestResult, verbosity=1+(verbose and 1 or 0), failfast=failfast)
 
 	if profile:
 		pr = cProfile.Profile()

frappe/tests/data/sample_svg.svg

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg width="16px" height="16px" viewBox="0 0 16 16" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+    <!-- Generator: Sketch 61.2 (89653) - https://sketch.com -->
+    <title>Artboard</title>
+    <desc>Created with Sketch.</desc>
+    <g id="Artboard" stroke="none" stroke-width="1" fill="none" fill-rule="evenodd">
+        <g id="frappe" transform="translate(3.000000, 1.000000)" fill="#0089FF" fill-rule="nonzero">
+            <polygon id="Path" points="9.360932 0 0 0 0 2.46232 9.360932 2.46232"></polygon>
+            <polygon id="Path" points="0 6.281996 0 14 2.98788 14 2.98788 8.74846 8.740172 8.74846 8.740172 6.281996"></polygon>
+        </g>
+    </g>
+</svg>

frappe/tests/test_assign.py

@@ -22,7 +22,7 @@ class TestAssign(unittest.TestCase):
 		self.assertEqual(len(assignments), 0)
 
 	def test_assignment_count(self):
-		frappe.db.sql('delete from tabToDo')
+		frappe.db.delete("ToDo")
 
 		if not frappe.db.exists("User", "test_assign1@example.com"):
 			frappe.get_doc({"doctype":"User", "email":"test_assign1@example.com", "first_name":"Test", "roles": [{"role": "System Manager"}]}).insert()

frappe/tests/test_db_query.py

@@ -105,7 +105,7 @@ class TestReportview(unittest.TestCase):
 
 	def test_between_filters(self):
 		""" test case to check between filter for date fields """
-		frappe.db.sql("delete from tabEvent")
+		frappe.db.delete("Event")
 
 		# create events to test the between operator filter
 		todays_event = create_event()

frappe/tests/test_document.py

@@ -227,3 +227,28 @@ class TestDocument(unittest.TestCase):
 		self.assertEqual(frappe.db.get_value("Currency", d.name), d.name)
 
 		frappe.delete_doc_if_exists("Currency", "Frappe Coin", 1)
+
+	def test_get_formatted(self):
+		frappe.get_doc({
+			'doctype': 'DocType',
+			'name': 'Test Formatted',
+			'module': 'Custom',
+			'custom': 1,
+			'fields': [
+				{'label': 'Currency', 'fieldname': 'currency', 'reqd': 1, 'fieldtype': 'Currency'},
+			]
+		}).insert()
+
+		frappe.delete_doc_if_exists("Currency", "INR", 1)
+
+		d = frappe.get_doc({
+			'doctype': 'Currency',
+			'currency_name': 'INR',
+			'symbol': '₹',
+		}).insert()
+
+		d = frappe.get_doc({
+			'doctype': 'Test Formatted',
+			'currency': 100000
+		})
+		self.assertEquals(d.get_formatted('currency', currency='INR', format="#,###.##"), '₹ 100,000.00')

frappe/tests/test_domainification.py

@@ -17,9 +17,9 @@ class TestDomainification(unittest.TestCase):
 		self.add_active_domain("_Test Domain 1")
 
 	def tearDown(self):
-		frappe.db.sql("delete from tabRole where name='_Test Role'")
-		frappe.db.sql("delete from `tabHas Role` where role='_Test Role'")
-		frappe.db.sql("delete from tabDomain where name in ('_Test Domain 1', '_Test Domain 2')")
+		frappe.db.delete("Role", {"name": "_Test Role"})
+		frappe.db.delete("Has Role", {"role": "_Test Role"})
+		frappe.db.delete("Domain", {"name": ("in", ("_Test Domain 1", "_Test Domain 2"))})
 		frappe.delete_doc('DocType', 'Test Domainification')
 		self.remove_from_active_domains(remove_all=True)
 

frappe/tests/test_dynamic_links.py

@@ -4,7 +4,7 @@ import frappe, unittest
 
 class TestDynamicLinks(unittest.TestCase):
 	def setUp(self):
-		frappe.db.sql('delete from `tabEmail Unsubscribe`')
+		frappe.db.delete("Email Unsubscribe")
 
 	def test_delete_normal(self):
 		event = frappe.get_doc({

frappe/tests/test_email.py

@@ -7,9 +7,9 @@ test_dependencies = ['Email Account']
 
 class TestEmail(unittest.TestCase):
 	def setUp(self):
-		frappe.db.sql("""delete from `tabEmail Unsubscribe`""")
-		frappe.db.sql("""delete from `tabEmail Queue`""")
-		frappe.db.sql("""delete from `tabEmail Queue Recipient`""")
+		frappe.db.delete("Email Unsubscribe")
+		frappe.db.delete("Email Queue")
+		frappe.db.delete("Email Queue Recipient")
 
 	def test_email_queue(self, send_after=None):
 		frappe.sendmail(recipients=['test@example.com', 'test1@example.com'],
@@ -170,7 +170,7 @@ class TestEmail(unittest.TestCase):
 		import re
 		email_account = frappe.get_doc('Email Account', '_Test Email Account 1')
 
-		frappe.db.sql('''delete from `tabCommunication` where sender = 'sukh@yyy.com' ''')
+		frappe.db.delete("Communication", {"sender": "sukh@yyy.com"})
 
 		with open(frappe.get_app_path('frappe', 'tests', 'data', 'email_with_image.txt'), 'r') as raw:
 			mails = email_account.get_inbound_mails(test_mails=[raw.read()])

frappe/tests/test_fmt_money.py

@@ -92,6 +92,9 @@ class TestFmtMoney(unittest.TestCase):
 		self.assertEqual(fmt_money(1000.456), "1.000,456")
 		frappe.db.set_default("currency_precision", "")
 
+	def test_custom_fmt_money_format(self):
+		self.assertEqual(fmt_money(100000, format="#,###.##"), '100,000.00')
+
 if __name__=="__main__":
 	frappe.connect()
 	unittest.main()

frappe/tests/test_formatter.py

@@ -17,9 +17,9 @@ class TestFormatter(unittest.TestCase):
 		frappe.db.set_default("currency", 'INR')
 
 		# if currency field is not passed then default currency should be used.
-		self.assertEqual(format(100, df, doc), '₹ 100.00')
+		self.assertEqual(format(100000, df, doc, format="#,###.##"), '₹ 100,000.00')
 
 		doc.currency = 'USD'
-		self.assertEqual(format(100, df, doc), "$ 100.00")
+		self.assertEqual(format(100000, df, doc, format="#,###.##"), "$ 100,000.00")
 
 		frappe.db.set_default("currency", None)

frappe/tests/test_frappe_client.py

@@ -12,7 +12,7 @@ import base64
 class TestFrappeClient(unittest.TestCase):
 	def test_insert_many(self):
 		server = FrappeClient(get_url(), "Administrator", "admin", verify=False)
-		frappe.db.sql("delete from `tabNote` where title in ('Sing','a','song','of','sixpence')")
+		frappe.db.delete("Note", {"title": ("in", ('Sing','a','song','of','sixpence'))})
 		frappe.db.commit()
 
 		server.insert_many([
@@ -31,7 +31,7 @@ class TestFrappeClient(unittest.TestCase):
 
 	def test_create_doc(self):
 		server = FrappeClient(get_url(), "Administrator", "admin", verify=False)
-		frappe.db.sql("delete from `tabNote` where title = 'test_create'")
+		frappe.db.delete("Note", {"title": "test_create"})
 		frappe.db.commit()
 
 		server.insert({"doctype": "Note", "public": True, "title": "test_create"})
@@ -46,7 +46,7 @@ class TestFrappeClient(unittest.TestCase):
 
 	def test_get_doc(self):
 		server = FrappeClient(get_url(), "Administrator", "admin", verify=False)
-		frappe.db.sql("delete from `tabNote` where title = 'get_this'")
+		frappe.db.delete("Note", {"title": "get_this"})
 		frappe.db.commit()
 
 		server.insert_many([
@@ -57,7 +57,7 @@ class TestFrappeClient(unittest.TestCase):
 
 	def test_get_value(self):
 		server = FrappeClient(get_url(), "Administrator", "admin", verify=False)
-		frappe.db.sql("delete from `tabNote` where title = 'get_value'")
+		frappe.db.delete("Note", {"title": "get_value"})
 		frappe.db.commit()
 
 		test_content = "test get value"
@@ -82,7 +82,7 @@ class TestFrappeClient(unittest.TestCase):
 
 	def test_update_doc(self):
 		server = FrappeClient(get_url(), "Administrator", "admin", verify=False)
-		frappe.db.sql("delete from `tabNote` where title in ('Sing','sing')")
+		frappe.db.delete("Note", {"title": ("in", ("Sing", "sing"))})
 		frappe.db.commit()
 
 		server.insert({"doctype":"Note", "public": True, "title": "Sing"})
@@ -94,12 +94,12 @@ class TestFrappeClient(unittest.TestCase):
 
 	def test_update_child_doc(self):
 		server = FrappeClient(get_url(), "Administrator", "admin", verify=False)
-		frappe.db.sql("delete from `tabContact` where first_name = 'George' and last_name = 'Steevens'")
-		frappe.db.sql("delete from `tabContact` where first_name = 'William' and last_name = 'Shakespeare'")
-		frappe.db.sql("delete from `tabCommunication` where reference_doctype = 'Event'")
-		frappe.db.sql("delete from `tabCommunication Link` where link_doctype = 'Contact'")
-		frappe.db.sql("delete from `tabEvent` where subject = 'Sing a song of sixpence'")
-		frappe.db.sql("delete from `tabEvent Participants` where reference_doctype = 'Contact'")
+		frappe.db.delete("Contact", {"first_name": "George", "last_name": "Steevens"})
+		frappe.db.delete("Contact", {"first_name": "William", "last_name": "Shakespeare"})
+		frappe.db.delete("Communication", {"reference_doctype": "Event"})
+		frappe.db.delete("Communication Link", {"link_doctype": "Contact"})
+		frappe.db.delete("Event", {"subject": "Sing a song of sixpence"})
+		frappe.db.delete("Event Participants", {"reference_doctype": "Contact"})
 		frappe.db.commit()
 
 		# create multiple contacts
@@ -131,7 +131,7 @@ class TestFrappeClient(unittest.TestCase):
 
 	def test_delete_doc(self):
 		server = FrappeClient(get_url(), "Administrator", "admin", verify=False)
-		frappe.db.sql("delete from `tabNote` where title = 'delete'")
+		frappe.db.delete("Note", {"title": "delete"})
 		frappe.db.commit()
 
 		server.insert_many([

frappe/tests/test_global_search.py

@@ -24,15 +24,15 @@ class TestGlobalSearch(unittest.TestCase):
 		make_property_setter(doctype, "repeat_on", "in_global_search", 0, "Int")
 
 	def tearDown(self):
-		frappe.db.sql("DELETE FROM `tabProperty Setter` WHERE `doc_type`='Event'")
+		frappe.db.delete("Property Setter", {"doc_type": "Event"})
 		frappe.clear_cache(doctype='Event')
-		frappe.db.sql('DELETE FROM `tabEvent`')
-		frappe.db.sql('DELETE FROM `__global_search`')
+		frappe.db.delete("Event")
+		frappe.db.delete("__global_search")
 		make_test_objects('Event')
 		frappe.db.commit()
 
 	def insert_test_events(self):
-		frappe.db.sql('DELETE FROM `tabEvent`')
+		frappe.db.delete("Event")
 		phrases = ['"The Sixth Extinction II: Amor Fati" is the second episode of the seventh season of the American science fiction.',
 		'After Mulder awakens from his coma, he realizes his duty to prevent alien colonization. ',
 		'Carter explored themes of extraterrestrial involvement in ancient mass extinctions in this episode, the third in a trilogy.']
@@ -97,7 +97,7 @@ class TestGlobalSearch(unittest.TestCase):
 		self.assertEqual(len(results), 0)
 
 	def test_insert_child_table(self):
-		frappe.db.sql('delete from tabEvent')
+		frappe.db.delete("Event")
 		phrases = ['Hydrus is a small constellation in the deep southern sky. ',
 		'It was first depicted on a celestial atlas by Johann Bayer in his 1603 Uranometria. ',
 		'The French explorer and astronomer Nicolas Louis de Lacaille charted the brighter stars and gave their Bayer designations in 1756. ',

frappe/tests/test_goal.py

@@ -13,7 +13,7 @@ class TestGoal(unittest.TestCase):
 		make_test_objects('Event', reset=True)
 
 	def tearDown(self):
-		frappe.db.sql('delete from `tabEvent`')
+		frappe.db.delete("Event")
 		# make_test_objects('Event', reset=True)
 		frappe.db.commit()
 

frappe/tests/test_naming.py

@@ -72,7 +72,7 @@ class TestNaming(unittest.TestCase):
 		current_index = frappe.db.sql("""SELECT current from `tabSeries` where name = %s""", series, as_dict=True)[0]
 
 		self.assertEqual(current_index.get('current'), 0)
-		frappe.db.sql("""delete from `tabSeries` where name = %s""", series)
+		frappe.db.delete("Series", {"name": series})
 
 		series = 'TEST-{}-'.format(year)
 		key = 'TEST-.YYYY.-.#####'
@@ -82,40 +82,40 @@ class TestNaming(unittest.TestCase):
 		current_index = frappe.db.sql("""SELECT current from `tabSeries` where name = %s""", series, as_dict=True)[0]
 
 		self.assertEqual(current_index.get('current'), 1)
-		frappe.db.sql("""delete from `tabSeries` where name = %s""", series)
+		frappe.db.delete("Series", {"name": series})
 
 		series = 'TEST-'
 		key = 'TEST-'
 		name = 'TEST-00003'
-		frappe.db.sql("DELETE FROM `tabSeries` WHERE `name`=%s", series)
+		frappe.db.delete("Series", {"name": series})
 		frappe.db.sql("""INSERT INTO `tabSeries` (name, current) values (%s, 3)""", (series,))
 		revert_series_if_last(key, name)
 		current_index = frappe.db.sql("""SELECT current from `tabSeries` where name = %s""", series, as_dict=True)[0]
 
 		self.assertEqual(current_index.get('current'), 2)
-		frappe.db.sql("""delete from `tabSeries` where name = %s""", series)
+		frappe.db.delete("Series", {"name": series})
 
 		series = 'TEST1-'
 		key = 'TEST1-.#####.-2021-22'
 		name = 'TEST1-00003-2021-22'
-		frappe.db.sql("DELETE FROM `tabSeries` WHERE `name`=%s", series)
+		frappe.db.delete("Series", {"name": series})
 		frappe.db.sql("""INSERT INTO `tabSeries` (name, current) values (%s, 3)""", (series,))
 		revert_series_if_last(key, name)
 		current_index = frappe.db.sql("""SELECT current from `tabSeries` where name = %s""", series, as_dict=True)[0]
 
 		self.assertEqual(current_index.get('current'), 2)
-		frappe.db.sql("""delete from `tabSeries` where name = %s""", series)
+		frappe.db.delete("Series", {"name": series})
 
 		series = ''
 		key = '.#####.-2021-22'
 		name = '00003-2021-22'
-		frappe.db.sql("DELETE FROM `tabSeries` WHERE `name`=%s", series)
+		frappe.db.delete("Series", {"name": series})
 		frappe.db.sql("""INSERT INTO `tabSeries` (name, current) values (%s, 3)""", (series,))
 		revert_series_if_last(key, name)
 		current_index = frappe.db.sql("""SELECT current from `tabSeries` where name = %s""", series, as_dict=True)[0]
 
 		self.assertEqual(current_index.get('current'), 2)
-		frappe.db.sql("""delete from `tabSeries` where name = %s""", series)
+		frappe.db.delete("Series", {"name": series})
 
 	def test_naming_for_cancelled_and_amended_doc(self):
 		submittable_doctype = frappe.get_doc({

frappe/tests/test_permissions.py

@@ -38,7 +38,7 @@ class TestPermissions(unittest.TestCase):
 		reset('Blogger')
 		reset('Blog Post')
 
-		frappe.db.sql('delete from `tabUser Permission`')
+		frappe.db.delete("User Permission")
 
 		frappe.set_user("test1@example.com")
 
@@ -334,9 +334,9 @@ class TestPermissions(unittest.TestCase):
 			doctype"""
 
 		frappe.set_user('Administrator')
-		frappe.db.sql('DELETE FROM `tabContact`')
-		frappe.db.sql('DELETE FROM `tabContact Email`')
-		frappe.db.sql('DELETE FROM `tabContact Phone`')
+		frappe.db.delete("Contact")
+		frappe.db.delete("Contact Email")
+		frappe.db.delete("Contact Phone")
 
 		reset('Salutation')
 		reset('Contact')

frappe/tests/test_scheduler.py

@@ -45,7 +45,7 @@ class TestScheduler(TestCase):
 
 		# 1st job is in the queue (or running), don't enqueue it again
 		self.assertFalse(job.enqueue())
-		frappe.db.sql('DELETE FROM `tabScheduled Job Log` WHERE `scheduled_job_type`=%s', job.name)
+		frappe.db.delete("Scheduled Job Log", {"scheduled_job_type": job.name})
 
 	def test_is_dormant(self):
 		self.assertTrue(is_dormant(check_time= get_datetime('2100-01-01 00:00:00')))

frappe/tests/test_search.py

@@ -3,8 +3,7 @@
 
 import unittest
 import frappe
-from frappe.desk.search import search_link
-from frappe.desk.search import search_widget
+from frappe.desk.search import search_link, search_widget, get_names_for_mentions
 
 
 class TestSearch(unittest.TestCase):
@@ -47,6 +46,23 @@ class TestSearch(unittest.TestCase):
 			search_link, 'DocType', 'Customer', query=None, filters=None,
 			page_length=20, searchfield=';')
 
+	def test_only_enabled_in_mention(self):
+		email = 'test_disabled_user_in_mentions@example.com'
+		frappe.delete_doc('User', email)
+		if not frappe.db.exists('User', email):
+			user = frappe.new_doc('User')
+			user.update({
+				'email' : email,
+				'first_name' : email.split("@")[0],
+				'enabled' : False,
+				'allowed_in_mentions' : True,
+			})
+			# saved when roles are added
+			user.add_roles('System Manager',)
+
+		names_for_mention = [user.get('id') for user in get_names_for_mentions('')]
+		self.assertNotIn(email, names_for_mention)
+
 	def test_link_field_order(self):
 		# Making a request to the search_link with the tree doctype
 		search_link(doctype=self.tree_doctype_name, txt='all', query=None,