Merge branch 'develop' into 12928-OTPLoginFix

.eslintrc

@@ -80,6 +80,7 @@
 		"validate_email": true,
 		"validate_name": true,
 		"validate_phone": true,
+		"validate_url": true,
 		"get_number_format": true,
 		"format_number": true,
 		"format_currency": true,
@@ -148,6 +149,7 @@
 		"before": true,
 		"beforeEach": true,
 		"qz": true,
-		"localforage": true
+		"localforage": true,
+		"extend_cscript": true
 	}
 }

.flake8

@@ -29,4 +29,5 @@ ignore =
     B950,
     W191,
 
-max-line-length = 200
+max-line-length = 200
+exclude=.github/helper/semgrep_rules

.git-blame-ignore-revs

@@ -0,0 +1,12 @@
+# Since version 2.23 (released in August 2019), git-blame has a feature
+# to ignore or bypass certain commits.
+#
+# This file contains a list of commits that are not likely what you
+# are looking for in a blame, such as mass reformatting or renaming.
+# You can set this file as a default ignore file for blame by running
+# the following command.
+#
+# $ git config blame.ignoreRevsFile .git-blame-ignore-revs
+
+# Replace use of Class.extend with native JS class
+fe20515c23a3ac41f1092bf0eaf0a0a452ec2e85

.github/helper/roulette.py

@@ -18,7 +18,7 @@ def is_js(file):
     return file.endswith("js")
 
 def is_docs(file):
-    regex = re.compile('\.(md|png|jpg|jpeg)$|^.github|LICENSE')
+    regex = re.compile(r'\.(md|png|jpg|jpeg)$|^.github|LICENSE')
     return bool(regex.search(file))
 
 

.github/helper/semgrep_rules/frappe_correctness.py

@@ -4,25 +4,61 @@ from frappe import _, flt
 from frappe.model.document import Document
 
 
+# ruleid: frappe-modifying-but-not-comitting
 def on_submit(self):
 	if self.value_of_goods == 0:
 		frappe.throw(_('Value of goods cannot be 0'))
-	# ruleid: frappe-modifying-after-submit
 	self.status = 'Submitted'
 
-def on_submit(self):   # noqa
-	if flt(self.per_billed) < 100:
-		self.update_billing_status()
-	else:
-		# todook: frappe-modifying-after-submit
-		self.status = "Completed"
-		self.db_set("status", "Completed")
 
-class TestDoc(Document):
-	pass
+# ok: frappe-modifying-but-not-comitting
+def on_submit(self):
+	if self.value_of_goods == 0:
+		frappe.throw(_('Value of goods cannot be 0'))
+	self.status = 'Submitted'
+	self.db_set('status', 'Submitted')
 
-	def validate(self):
-		#ruleid: frappe-modifying-child-tables-while-iterating
-		for item in self.child_table:
-			if item.value < 0:
-				self.remove(item)
+# ok: frappe-modifying-but-not-comitting
+def on_submit(self):
+	if self.value_of_goods == 0:
+		frappe.throw(_('Value of goods cannot be 0'))
+	x = "y"
+	self.status = x
+	self.db_set('status', x)
+
+
+# ok: frappe-modifying-but-not-comitting
+def on_submit(self):
+	x = "y"
+	self.status = x
+	self.save()
+
+# ruleid: frappe-modifying-but-not-comitting-other-method
+class DoctypeClass(Document):
+	def on_submit(self):
+		self.good_method()
+		self.tainted_method()
+
+	def tainted_method(self):
+		self.status = "uptate"
+
+
+# ok: frappe-modifying-but-not-comitting-other-method
+class DoctypeClass(Document):
+	def on_submit(self):
+		self.good_method()
+		self.tainted_method()
+
+	def tainted_method(self):
+		self.status = "update"
+		self.db_set("status", "update")
+
+# ok: frappe-modifying-but-not-comitting-other-method
+class DoctypeClass(Document):
+	def on_submit(self):
+		self.good_method()
+		self.tainted_method()
+		self.save()
+
+	def tainted_method(self):
+		self.status = "uptate"

.github/helper/semgrep_rules/translate.js

@@ -35,3 +35,10 @@ __('You have' + 'subscribers in your mailing list.')
 // ruleid: frappe-translation-js-splitting
 __('You have {0} subscribers' +
     'in your mailing list', [subscribers.length])
+
+// ok: frappe-translation-js-splitting
+__("Ctrl+Enter to add comment")
+
+// ruleid: frappe-translation-js-splitting
+__('You have {0} subscribers \
+    in your mailing list', [subscribers.length])

.github/helper/semgrep_rules/translate.py

@@ -51,3 +51,11 @@ _(f"what" + f"this is also not cool")
 _("")
 # ruleid: frappe-translation-empty-string
 _('')
+
+
+class Test:
+	# ok: frappe-translation-python-splitting
+	def __init__(
+			args
+			):
+		pass

.github/helper/semgrep_rules/translate.yml

@@ -42,10 +42,10 @@ rules:
 
 - id: frappe-translation-python-splitting
   pattern-either:
-  - pattern: _(...) + ... + _(...)
+  - pattern: _(...) + _(...)
   - pattern: _("..." + "...")
-  - pattern-regex: '_\([^\)]*\\\s*'    # lines broken by `\`
-  - pattern-regex: '_\(\s*\n'          # line breaks allowed by python for using ( )
+  - pattern-regex: '[\s\.]_\([^\)]*\\\s*'    # lines broken by `\`
+  - pattern-regex: '[\s\.]_\(\s*\n'          # line breaks allowed by python for using ( )
   message: |
     Do not split strings inside translate function. Do not concatenate using translate functions.
     Please refer: https://frappeframework.com/docs/user/en/translations
@@ -54,8 +54,8 @@ rules:
 
 - id: frappe-translation-js-splitting
   pattern-either:
-  - pattern-regex: '__\([^\)]*[\+\\]\s*'
-  - pattern: __('...' + '...')
+  - pattern-regex: '__\([^\)]*[\\]\s+'
+  - pattern: __('...' + '...', ...)
   - pattern: __('...') + __('...')
   message: |
     Do not split strings inside translate function. Do not concatenate using translate functions.

.github/helper/semgrep_rules/ux.js

@@ -0,0 +1,9 @@
+
+// ok: frappe-missing-translate-function-js
+frappe.msgprint('{{ _("Both login and password required") }}');
+
+// ruleid: frappe-missing-translate-function-js
+frappe.msgprint('What');
+
+// ok: frappe-missing-translate-function-js
+frappe.throw('  {{ _("Both login and password required") }}.  ');

.github/helper/semgrep_rules/ux.py

@@ -2,30 +2,30 @@ import frappe
 from frappe import msgprint, throw, _
 
 
-# ruleid: frappe-missing-translate-function
+# ruleid: frappe-missing-translate-function-python
 throw("Error Occured")
 
-# ruleid: frappe-missing-translate-function
+# ruleid: frappe-missing-translate-function-python
 frappe.throw("Error Occured")
 
-# ruleid: frappe-missing-translate-function
+# ruleid: frappe-missing-translate-function-python
 frappe.msgprint("Useful message")
 
-# ruleid: frappe-missing-translate-function
+# ruleid: frappe-missing-translate-function-python
 msgprint("Useful message")
 
 
-# ok: frappe-missing-translate-function
+# ok: frappe-missing-translate-function-python
 translatedmessage = _("Hello")
 
-# ok: frappe-missing-translate-function
+# ok: frappe-missing-translate-function-python
 throw(translatedmessage)
 
-# ok: frappe-missing-translate-function
+# ok: frappe-missing-translate-function-python
 msgprint(translatedmessage)
 
-# ok: frappe-missing-translate-function
+# ok: frappe-missing-translate-function-python
 msgprint(_("Helpful message"))
 
-# ok: frappe-missing-translate-function
+# ok: frappe-missing-translate-function-python
 frappe.throw(_("Error occured"))

.github/helper/semgrep_rules/ux.yml

@@ -1,15 +1,30 @@
 rules:
-- id: frappe-missing-translate-function
+- id: frappe-missing-translate-function-python
   pattern-either:
   - patterns:
       - pattern: frappe.msgprint("...", ...)
       - pattern-not: frappe.msgprint(_("..."), ...)
-      - pattern-not: frappe.msgprint(__("..."), ...)
   - patterns:
       - pattern: frappe.throw("...", ...)
       - pattern-not: frappe.throw(_("..."), ...)
-      - pattern-not: frappe.throw(__("..."), ...)
   message: |
       All user facing text must be wrapped in translate function. Please refer to translation documentation. https://frappeframework.com/docs/user/en/guides/basics/translations
-  languages: [python, javascript, json]
+  languages: [python]
+  severity: ERROR
+
+- id: frappe-missing-translate-function-js
+  pattern-either:
+  - patterns:
+      - pattern: frappe.msgprint("...", ...)
+      - pattern-not: frappe.msgprint(__("..."), ...)
+      # ignore microtemplating e.g. msgprint("{{ _("server side translation") }}")
+      - pattern-not: frappe.msgprint("=~/\{\{.*\_.*\}\}/i", ...)
+  - patterns:
+      - pattern: frappe.throw("...", ...)
+      - pattern-not: frappe.throw(__("..."), ...)
+      # ignore microtemplating
+      - pattern-not: frappe.throw("=~/\{\{.*\_.*\}\}/i", ...)
+  message: |
+      All user facing text must be wrapped in translate function. Please refer to translation documentation. https://frappeframework.com/docs/user/en/guides/basics/translations
+  languages: [javascript]
   severity: ERROR

.github/workflows/patch-mariadb-tests.yml

@@ -0,0 +1,83 @@
+name: Patch
+
+on: [pull_request, workflow_dispatch]
+
+jobs:
+  test:
+    runs-on: ubuntu-18.04
+
+    name: Patch Test
+
+    services:
+      mysql:
+        image: mariadb:10.3
+        env:
+          MYSQL_ALLOW_EMPTY_PASSWORD: YES
+        ports:
+          - 3306:3306
+        options: --health-cmd="mysqladmin ping" --health-interval=5s --health-timeout=2s --health-retries=3
+
+    steps:
+      - name: Clone
+        uses: actions/checkout@v2
+
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+
+      - name: Add to Hosts
+        run: echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts
+
+      - name: Cache pip
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+            ${{ runner.os }}-
+
+      - name: Cache node modules
+        uses: actions/cache@v2
+        env:
+          cache-name: cache-node-modules
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-build-${{ env.cache-name }}-
+            ${{ runner.os }}-build-
+            ${{ runner.os }}-
+
+      - name: Get yarn cache directory path
+        id: yarn-cache-dir-path
+        run: echo "::set-output name=dir::$(yarn cache dir)"
+
+      - uses: actions/cache@v2
+        id: yarn-cache
+        with:
+          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
+          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-yarn-
+
+      - name: Install Dependencies
+        run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh
+        env:
+          BEFORE: ${{ env.GITHUB_EVENT_PATH.before }}
+          AFTER: ${{ env.GITHUB_EVENT_PATH.after }}
+          TYPE: server
+
+      - name: Install
+        run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh
+        env:
+          DB: mariadb
+          TYPE: server
+
+      - name: Run Patch Tests
+        run: |
+          cd ~/frappe-bench/
+          wget https://frappeframework.com/files/v10-frappe.sql.gz
+          bench --site test_site --force restore ~/frappe-bench/v10-frappe.sql.gz
+          bench --site test_site migrate

.github/workflows/publish-assets-develop.yml

@@ -15,11 +15,11 @@ jobs:
           path: 'frappe'
       - uses: actions/setup-node@v1
         with:
-          python-version: '12.x'
+          node-version: 14
       - uses: actions/setup-python@v2
         with:
           python-version: '3.6'
-      - name: Set up bench for current push
+      - name: Set up bench and build assets
         run: |
           npm install -g yarn
           pip3 install -U frappe-bench
@@ -29,7 +29,7 @@ jobs:
       - name: Package assets
         run: |
           mkdir -p $GITHUB_WORKSPACE/build
-          tar -cvpzf $GITHUB_WORKSPACE/build/$GITHUB_SHA.tar.gz ./frappe-bench/sites/assets/js ./frappe-bench/sites/assets/css
+          tar -cvpzf $GITHUB_WORKSPACE/build/$GITHUB_SHA.tar.gz ./frappe-bench/sites/assets/frappe/dist
 
       - name: Publish assets to S3
         uses: jakejarvis/s3-sync-action@master

.github/workflows/publish-assets-releases.yml

@@ -22,7 +22,7 @@ jobs:
       - uses: actions/setup-python@v2
         with:
           python-version: '3.6'
-      - name: Set up bench for current push
+      - name: Set up bench and build assets
         run: |
           npm install -g yarn
           pip3 install -U frappe-bench
@@ -32,7 +32,7 @@ jobs:
       - name: Package assets
         run: |
           mkdir -p $GITHUB_WORKSPACE/build
-          tar -cvpzf $GITHUB_WORKSPACE/build/assets.tar.gz ./frappe-bench/sites/assets/js ./frappe-bench/sites/assets/css
+          tar -cvpzf $GITHUB_WORKSPACE/build/assets.tar.gz ./frappe-bench/sites/assets/frappe/dist
 
       - name: Get release
         id: get_release

.github/workflows/semgrep.yml

@@ -4,6 +4,8 @@ on:
   pull_request:
     branches:
       - develop
+      - version-13-hotfix
+      - version-13-pre-release
 jobs:
   semgrep:
     name: Frappe Linter

.github/workflows/server-mariadb-tests.yml

@@ -0,0 +1,130 @@
+name: Server
+
+on:
+  pull_request:
+  workflow_dispatch:
+  push:
+    branches: [ develop ]
+
+jobs:
+  test:
+    runs-on: ubuntu-18.04
+
+    strategy:
+      fail-fast: false
+      matrix:
+       container: [1, 2]
+
+    name: Python Unit Tests (MariaDB)
+
+    services:
+      mysql:
+        image: mariadb:10.3
+        env:
+          MYSQL_ALLOW_EMPTY_PASSWORD: YES
+        ports:
+          - 3306:3306
+        options: --health-cmd="mysqladmin ping" --health-interval=5s --health-timeout=2s --health-retries=3
+
+    steps:
+      - name: Clone
+        uses: actions/checkout@v2
+
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+
+      - uses: actions/setup-node@v2
+        with:
+          node-version: 14
+          check-latest: true
+
+      - name: Add to Hosts
+        run: |
+          echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts
+          echo "127.0.0.1 test_site_producer" | sudo tee -a /etc/hosts
+
+      - name: Cache pip
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+            ${{ runner.os }}-
+
+      - name: Cache node modules
+        uses: actions/cache@v2
+        env:
+          cache-name: cache-node-modules
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-build-${{ env.cache-name }}-
+            ${{ runner.os }}-build-
+            ${{ runner.os }}-
+
+      - name: Get yarn cache directory path
+        id: yarn-cache-dir-path
+        run: echo "::set-output name=dir::$(yarn cache dir)"
+
+      - uses: actions/cache@v2
+        id: yarn-cache
+        with:
+          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
+          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-yarn-
+
+      - name: Install Dependencies
+        run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh
+        env:
+          BEFORE: ${{ env.GITHUB_EVENT_PATH.before }}
+          AFTER: ${{ env.GITHUB_EVENT_PATH.after }}
+          TYPE: server
+
+      - name: Install
+        run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh
+        env:
+          DB: mariadb
+          TYPE: server
+
+      - name: Run Tests
+        run: cd ~/frappe-bench/ && bench --site test_site run-parallel-tests --use-orchestrator --with-coverage
+        env:
+          CI_BUILD_ID: ${{ github.run_id }}
+          ORCHESTRATOR_URL: http://test-orchestrator.frappe.io
+
+      - name: Upload Coverage Data
+        run: |
+          cp ~/frappe-bench/sites/.coverage ${GITHUB_WORKSPACE}
+          cd ${GITHUB_WORKSPACE}
+          pip3 install coverage==5.5
+          pip3 install coveralls==3.0.1
+          coveralls
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_TOKEN }}
+          COVERALLS_FLAG_NAME: run-${{ matrix.container }}
+          COVERALLS_SERVICE_NAME: ${{ github.event_name == 'pull_request' && 'github' || 'github-actions' }}
+          COVERALLS_PARALLEL: true
+
+  coveralls:
+    name: Coverage Wrap Up
+    needs: test
+    container: python:3-slim
+    runs-on: ubuntu-18.04
+    steps:
+      - name: Clone
+        uses: actions/checkout@v2
+
+      - name: Coveralls Finished
+        run: |
+          cd ${GITHUB_WORKSPACE}
+          pip3 install coverage==5.5
+          pip3 install coveralls==3.0.1
+          coveralls --finish
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/server-postgres-tests.yml

@@ -0,0 +1,100 @@
+name: Server
+
+on:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ubuntu-18.04
+
+    strategy:
+      fail-fast: false
+      matrix:
+       container: [1, 2]
+
+    name: Python Unit Tests (Postgres)
+
+    services:
+      postgres:
+        image: postgres:12.4
+        env:
+          POSTGRES_PASSWORD: travis
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
+    steps:
+      - name: Clone
+        uses: actions/checkout@v2
+
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+
+      - uses: actions/setup-node@v2
+        with:
+          node-version: '14'
+          check-latest: true
+
+      - name: Add to Hosts
+        run: |
+          echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts
+          echo "127.0.0.1 test_site_producer" | sudo tee -a /etc/hosts
+
+      - name: Cache pip
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+            ${{ runner.os }}-
+
+      - name: Cache node modules
+        uses: actions/cache@v2
+        env:
+          cache-name: cache-node-modules
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-build-${{ env.cache-name }}-
+            ${{ runner.os }}-build-
+            ${{ runner.os }}-
+
+      - name: Get yarn cache directory path
+        id: yarn-cache-dir-path
+        run: echo "::set-output name=dir::$(yarn cache dir)"
+
+      - uses: actions/cache@v2
+        id: yarn-cache
+        with:
+          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
+          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-yarn-
+
+      - name: Install Dependencies
+        run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh
+        env:
+          BEFORE: ${{ env.GITHUB_EVENT_PATH.before }}
+          AFTER: ${{ env.GITHUB_EVENT_PATH.after }}
+          TYPE: server
+
+      - name: Install
+        run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh
+        env:
+          DB: postgres
+          TYPE: server
+
+      - name: Run Tests
+        run: cd ~/frappe-bench/ && bench --site test_site run-parallel-tests --use-orchestrator
+        env:
+          CI_BUILD_ID: ${{ github.run_id }}
+          ORCHESTRATOR_URL: http://test-orchestrator.frappe.io

.github/workflows/ui-tests.yml

@@ -1,10 +1,10 @@
-name: CI
+name: UI
 
 on:
   pull_request:
-    types: [opened, synchronize, reopened, labeled, unlabeled]
   workflow_dispatch:
   push:
+    branches: [ develop ]
 
 jobs:
   test:
@@ -13,23 +13,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-       include:
-        - DB: "mariadb"
-          TYPE: "server"
-          JOB_NAME: "Python MariaDB"
-          RUN_COMMAND: bench --site test_site run-tests --coverage
+       containers: [1, 2]
 
-        - DB: "postgres"
-          TYPE: "server"
-          JOB_NAME: "Python PostgreSQL"
-          RUN_COMMAND: bench --site test_site run-tests --coverage
-
-        - DB: "mariadb"
-          TYPE: "ui"
-          JOB_NAME: "UI MariaDB"
-          RUN_COMMAND:  bench --site test_site run-ui-tests frappe --headless
-
-    name: ${{ matrix.JOB_NAME }}
+    name: UI Tests (Cypress)
 
     services:
       mysql:
@@ -40,18 +26,6 @@ jobs:
           - 3306:3306
         options: --health-cmd="mysqladmin ping" --health-interval=5s --health-timeout=2s --health-retries=3
 
-      postgres:
-        image: postgres:12.4
-        env:
-          POSTGRES_PASSWORD: travis
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          - 5432:5432
-
     steps:
       - name: Clone
         uses: actions/checkout@v2
@@ -63,7 +37,7 @@ jobs:
 
       - uses: actions/setup-node@v2
         with:
-          node-version: '12'
+          node-version: 14
           check-latest: true
 
       - name: Add to Hosts
@@ -105,7 +79,6 @@ jobs:
             ${{ runner.os }}-yarn-
 
       - name: Cache cypress binary
-        if: matrix.TYPE == 'ui'
         uses: actions/cache@v2
         with:
           path: ~/.cache
@@ -119,40 +92,16 @@ jobs:
         env:
           BEFORE: ${{ env.GITHUB_EVENT_PATH.before }}
           AFTER: ${{ env.GITHUB_EVENT_PATH.after }}
-          TYPE: ${{ matrix.TYPE }}
+          TYPE: ui
 
       - name: Install
         run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh
         env:
-          DB: ${{ matrix.DB }}
-          TYPE: ${{ matrix.TYPE }}
+          DB: mariadb
+          TYPE: ui
 
-      - name: Run Set-Up
-        if: matrix.TYPE == 'ui'
+      - name: Site Setup
         run: cd ~/frappe-bench/ && bench --site test_site execute frappe.utils.install.complete_setup_wizard
-        env:
-          DB: ${{ matrix.DB }}
-          TYPE: ${{ matrix.TYPE }}
 
-      - name: Setup tmate session
-        if: contains(github.event.pull_request.labels.*.name, 'debug-gha')
-        uses: mxschmitt/action-tmate@v3
-
-      - name: Run Tests
-        run: cd ~/frappe-bench/ && ${{ matrix.RUN_COMMAND }}
-        env:
-          DB: ${{ matrix.DB }}
-          TYPE: ${{ matrix.TYPE }}
-
-      - name: Coverage
-        if: matrix.TYPE == 'server'
-        run: |
-          cp ~/frappe-bench/sites/.coverage ${GITHUB_WORKSPACE}
-          cd ${GITHUB_WORKSPACE}
-          pip install coveralls==2.2.0
-          pip install coverage==4.5.4
-          coveralls --service=github
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_TOKEN }}
-          COVERALLS_SERVICE_NAME: github
+      - name: UI Tests
+        run: cd ~/frappe-bench/ && bench --site test_site run-ui-tests frappe --headless --parallel --ci-build-id $GITHUB_RUN_ID

.gitignore

@@ -9,6 +9,7 @@ locale
 dist/
 # build/
 frappe/docs/current
+frappe/public/dist
 .vscode
 node_modules
 .kdev4/

.mergify.yml

@@ -3,9 +3,12 @@ pull_request_rules:
     conditions:
       - status-success=Sider
       - status-success=Semantic Pull Request
-      - status-success=Python MariaDB
-      - status-success=Python PostgreSQL
-      - status-success=UI MariaDB
+      - status-success=Python Unit Tests (MariaDB) (1)
+      - status-success=Python Unit Tests (MariaDB) (2)
+      - status-success=Python Unit Tests (Postgres) (1)
+      - status-success=Python Unit Tests (Postgres) (2)
+      - status-success=UI Tests (Cypress) (1)
+      - status-success=UI Tests (Cypress) (2)
       - status-success=security/snyk (frappe)
       - label!=dont-merge
       - label!=squash
@@ -16,9 +19,12 @@ pull_request_rules:
   - name: Automatic squash on CI success and review
     conditions:
       - status-success=Sider
-      - status-success=Python MariaDB
-      - status-success=Python PostgreSQL
-      - status-success=UI MariaDB
+      - status-success=Python Unit Tests (MariaDB) (1)
+      - status-success=Python Unit Tests (MariaDB) (2)
+      - status-success=Python Unit Tests (Postgres) (1)
+      - status-success=Python Unit Tests (Postgres) (2)
+      - status-success=UI Tests (Cypress) (1)
+      - status-success=UI Tests (Cypress) (2)
       - status-success=security/snyk (frappe)
       - label!=dont-merge
       - label=squash

README.md

@@ -14,18 +14,21 @@
 </div>
 
 <div align="center">
-    <a href="https://github.com/frappe/frappe/actions/workflows/ci-tests.yml">
-        <img src="https://github.com/frappe/frappe/actions/workflows/ci-tests.yml/badge.svg?branch=develop">
-    </a>
-    <a href='https://frappeframework.com/docs'>
-        <img src='https://img.shields.io/badge/docs-📖-7575FF.svg?style=flat-square'/>
-    </a>
+	<a href="https://github.com/frappe/frappe/actions/workflows/server-mariadb-tests.yml">
+		<img src="https://github.com/frappe/frappe/actions/workflows/server-mariadb-tests.yml/badge.svg">
+	</a>
+	<a href="https://github.com/frappe/frappe/actions/workflows/ui-tests.yml">
+		<img src="https://github.com/frappe/frappe/actions/workflows/ui-tests.yml/badge.svg?branch=develop">
+	</a>
+	<a href='https://frappeframework.com/docs'>
+		<img src='https://img.shields.io/badge/docs-📖-7575FF.svg?style=flat-square'/>
+	</a>
 	<a href='https://www.codetriage.com/frappe/frappe'>
 		<img src='https://www.codetriage.com/frappe/frappe/badges/users.svg'>
 	</a>
-    <a href='https://coveralls.io/github/frappe/frappe?branch=develop'>
-        <img src='https://coveralls.io/repos/github/frappe/frappe/badge.svg?branch=develop'>
-    </a>
+	<a href='https://coveralls.io/github/frappe/frappe?branch=develop'>
+		<img src='https://coveralls.io/repos/github/frappe/frappe/badge.svg?branch=develop'>
+	</a>
 </div>
 
 

cypress/fixtures/data_field_validation_doctype.js

@@ -0,0 +1,65 @@
+export default {
+	name: 'Validation Test',
+	custom: 1,
+	actions: [],
+	creation: '2019-03-15 06:29:07.215072',
+	doctype: 'DocType',
+	editable_grid: 1,
+	engine: 'InnoDB',
+	fields: [
+		{
+			fieldname: 'email',
+			fieldtype: 'Data',
+			label: 'Email',
+			options: 'Email'
+		},
+		{
+			fieldname: 'URL',
+			fieldtype: 'Data',
+			label: 'URL',
+			options: 'URL'
+		},
+		{
+			fieldname: 'Phone',
+			fieldtype: 'Data',
+			label: 'Phone',
+			options: 'Phone'
+		},
+		{
+			fieldname: 'person_name',
+			fieldtype: 'Data',
+			label: 'Person Name',
+			options: 'Name'
+		},
+		{
+			fieldname: 'read_only_url',
+			fieldtype: 'Data',
+			label: 'Read Only URL',
+			options: 'URL',
+			read_only: '1',
+			default: 'https://frappe.io'
+		}
+	],
+	issingle: 1,
+	links: [],
+	modified: '2021-04-19 14:40:53.127615',
+	modified_by: 'Administrator',
+	module: 'Custom',
+	owner: 'Administrator',
+	permissions: [
+		{
+			create: 1,
+			delete: 1,
+			email: 1,
+			print: 1,
+			read: 1,
+			role: 'System Manager',
+			share: 1,
+			write: 1
+		}
+	],
+	quick_entry: 1,
+	sort_field: 'modified',
+	sort_order: 'ASC',
+	track_changes: 1
+};

cypress/integration/data_field_form_validation.js

@@ -0,0 +1,43 @@
+import data_field_validation_doctype from '../fixtures/data_field_validation_doctype';
+const doctype_name = data_field_validation_doctype.name;
+
+
+context('Data Field Input Validation in New Form', () => {
+	before(() => {
+		cy.login();
+		cy.visit('/app/website');
+		return cy.insert_doc('DocType', data_field_validation_doctype, true);
+	});
+
+	function validateField(fieldname, invalid_value, valid_value) {
+		// Invalid, should have has-error class
+		cy.get_field(fieldname).clear().type(invalid_value).blur();
+		cy.get(`.frappe-control[data-fieldname="${fieldname}"]`).should('have.class', 'has-error');
+		// Valid value, should not have has-error class
+		cy.get_field(fieldname).clear().type(valid_value);
+		cy.get(`.frappe-control[data-fieldname="${fieldname}"]`).should('not.have.class', 'has-error');
+	}
+
+	describe('Data Field Options', () => {
+		it('should validate email address', () => {
+			cy.new_form(doctype_name);
+			validateField('email', 'captian', 'hello@test.com');
+		});
+
+		it('should validate URL', () => {
+			validateField('url', 'jkl', 'https://frappe.io');
+			validateField('url', 'abcd.com', 'http://google.com/home');
+			validateField('url', '&&http://google.uae', 'gopher://frappe.io');
+			validateField('url', 'ftt2:://google.in?q=news', 'ftps2://frappe.io/__/#home');
+			validateField('url', 'ftt2://', 'ntps://localhost'); // For intranet URLs
+		});
+
+		it('should validate phone number', () => {
+			validateField('phone', 'america', '89787878');
+		});
+
+		it('should validate name', () => {
+			validateField('person_name', ' 777Hello', 'James Bond');
+		});
+	});
+});

cypress/integration/recorder.js

@@ -50,7 +50,7 @@ context('Recorder', () => {
 		cy.get('.result-list').should('contain', '/api/method/frappe.desk.reportview.get');
 	});
 
-	it.only('Recorder View Request', () => {
+	it('Recorder View Request', () => {
 		cy.get('.primary-action').should('contain', 'Start').click();
 
 		cy.visit('/app/List/DocType/List');

cypress/integration/url_data_field.js

@@ -0,0 +1,43 @@
+import data_field_validation_doctype from '../fixtures/data_field_validation_doctype';
+
+const doctype_name = data_field_validation_doctype.name;
+
+context('URL Data Field Input', () => {
+	before(() => {
+		cy.login();
+		cy.visit('/app/website');
+		return cy.insert_doc('DocType', data_field_validation_doctype, true);
+	});
+
+
+	describe('URL Data Field Input ', () => {
+		it('should not show URL link button without focus', () => {
+			cy.new_form(doctype_name);
+			cy.get_field('url').clear().type('https://frappe.io');
+			cy.get_field('url').blur().wait(500);
+			cy.get('.link-btn').should('not.be.visible');
+		});
+
+		it('should show URL link button on focus', () => {
+			cy.get_field('url').focus().wait(500);
+			cy.get('.link-btn').should('be.visible');
+		});
+
+		it('should not show URL link button for invalid URL', () => {
+			cy.get_field('url').clear().type('fuzzbuzz');
+			cy.get('.link-btn').should('not.be.visible');
+		});
+
+		it('should have valid URL link with target _blank', () => {
+			cy.get_field('url').clear().type('https://frappe.io');
+			cy.get('.link-btn .btn-open').should('have.attr', 'href', 'https://frappe.io');
+			cy.get('.link-btn .btn-open').should('have.attr', 'target', '_blank');
+		});
+
+		it('should inject anchor tag in read-only URL data field', () => {
+			cy.get('[data-fieldname="read_only_url"]')
+				.find('a')
+				.should('have.attr', 'target', '_blank');
+		});
+	});
+});

esbuild/esbuild.js

@@ -0,0 +1,481 @@
+/* eslint-disable no-console */
+let path = require("path");
+let fs = require("fs");
+let glob = require("fast-glob");
+let esbuild = require("esbuild");
+let vue = require("esbuild-vue");
+let yargs = require("yargs");
+let cliui = require("cliui")();
+let chalk = require("chalk");
+let html_plugin = require("./frappe-html");
+let postCssPlugin = require("esbuild-plugin-postcss2").default;
+let ignore_assets = require("./ignore-assets");
+let sass_options = require("./sass_options");
+let {
+	app_list,
+	assets_path,
+	apps_path,
+	sites_path,
+	get_app_path,
+	get_public_path,
+	log,
+	log_warn,
+	log_error,
+	bench_path,
+	get_redis_subscriber
+} = require("./utils");
+
+let argv = yargs
+	.usage("Usage: node esbuild [options]")
+	.option("apps", {
+		type: "string",
+		description: "Run build for specific apps"
+	})
+	.option("skip_frappe", {
+		type: "boolean",
+		description: "Skip building frappe assets"
+	})
+	.option("files", {
+		type: "string",
+		description: "Run build for specified bundles"
+	})
+	.option("watch", {
+		type: "boolean",
+		description: "Run in watch mode and rebuild on file changes"
+	})
+	.option("production", {
+		type: "boolean",
+		description: "Run build in production mode"
+	})
+	.option("run-build-command", {
+		type: "boolean",
+		description: "Run build command for apps"
+	})
+	.example(
+		"node esbuild --apps frappe,erpnext",
+		"Run build only for frappe and erpnext"
+	)
+	.example(
+		"node esbuild --files frappe/website.bundle.js,frappe/desk.bundle.js",
+		"Run build only for specified bundles"
+	)
+	.version(false).argv;
+
+const APPS = (!argv.apps ? app_list : argv.apps.split(",")).filter(
+	app => !(argv.skip_frappe && app == "frappe")
+);
+const FILES_TO_BUILD = argv.files ? argv.files.split(",") : [];
+const WATCH_MODE = Boolean(argv.watch);
+const PRODUCTION = Boolean(argv.production);
+const RUN_BUILD_COMMAND = !WATCH_MODE && Boolean(argv["run-build-command"]);
+
+const TOTAL_BUILD_TIME = `${chalk.black.bgGreen(" DONE ")} Total Build Time`;
+const NODE_PATHS = [].concat(
+	// node_modules of apps directly importable
+	app_list
+		.map(app => path.resolve(get_app_path(app), "../node_modules"))
+		.filter(fs.existsSync),
+	// import js file of any app if you provide the full path
+	app_list
+		.map(app => path.resolve(get_app_path(app), ".."))
+		.filter(fs.existsSync)
+);
+
+execute()
+	.then(() => RUN_BUILD_COMMAND && run_build_command_for_apps(APPS))
+	.catch(e => console.error(e));
+
+if (WATCH_MODE) {
+	// listen for open files in editor event
+	open_in_editor();
+}
+
+async function execute() {
+	console.time(TOTAL_BUILD_TIME);
+	if (!FILES_TO_BUILD.length) {
+		await clean_dist_folders(APPS);
+	}
+
+	let result;
+	try {
+		result = await build_assets_for_apps(APPS, FILES_TO_BUILD);
+	} catch (e) {
+		log_error("There were some problems during build");
+		log();
+		log(chalk.dim(e.stack));
+		return;
+	}
+
+	if (!WATCH_MODE) {
+		log_built_assets(result.metafile);
+		console.timeEnd(TOTAL_BUILD_TIME);
+		log();
+	} else {
+		log("Watching for changes...");
+	}
+	return await write_assets_json(result.metafile);
+}
+
+function build_assets_for_apps(apps, files) {
+	let { include_patterns, ignore_patterns } = files.length
+		? get_files_to_build(files)
+		: get_all_files_to_build(apps);
+
+	return glob(include_patterns, { ignore: ignore_patterns }).then(files => {
+		let output_path = assets_path;
+
+		let file_map = {};
+		for (let file of files) {
+			let relative_app_path = path.relative(apps_path, file);
+			let app = relative_app_path.split(path.sep)[0];
+
+			let extension = path.extname(file);
+			let output_name = path.basename(file, extension);
+			if (
+				[".css", ".scss", ".less", ".sass", ".styl"].includes(extension)
+			) {
+				output_name = path.join("css", output_name);
+			} else if ([".js", ".ts"].includes(extension)) {
+				output_name = path.join("js", output_name);
+			}
+			output_name = path.join(app, "dist", output_name);
+
+			if (Object.keys(file_map).includes(output_name)) {
+				log_warn(
+					`Duplicate output file ${output_name} generated from ${file}`
+				);
+			}
+
+			file_map[output_name] = file;
+		}
+
+		return build_files({
+			files: file_map,
+			outdir: output_path
+		});
+	});
+}
+
+function get_all_files_to_build(apps) {
+	let include_patterns = [];
+	let ignore_patterns = [];
+
+	for (let app of apps) {
+		let public_path = get_public_path(app);
+		include_patterns.push(
+			path.resolve(
+				public_path,
+				"**",
+				"*.bundle.{js,ts,css,sass,scss,less,styl}"
+			)
+		);
+		ignore_patterns.push(
+			path.resolve(public_path, "node_modules"),
+			path.resolve(public_path, "dist")
+		);
+	}
+
+	return {
+		include_patterns,
+		ignore_patterns
+	};
+}
+
+function get_files_to_build(files) {
+	// files: ['frappe/website.bundle.js', 'erpnext/main.bundle.js']
+	let include_patterns = [];
+	let ignore_patterns = [];
+
+	for (let file of files) {
+		let [app, bundle] = file.split("/");
+		let public_path = get_public_path(app);
+		include_patterns.push(path.resolve(public_path, "**", bundle));
+		ignore_patterns.push(
+			path.resolve(public_path, "node_modules"),
+			path.resolve(public_path, "dist")
+		);
+	}
+
+	return {
+		include_patterns,
+		ignore_patterns
+	};
+}
+
+function build_files({ files, outdir }) {
+	return esbuild.build({
+		entryPoints: files,
+		entryNames: "[dir]/[name].[hash]",
+		outdir,
+		sourcemap: true,
+		bundle: true,
+		metafile: true,
+		minify: PRODUCTION,
+		nodePaths: NODE_PATHS,
+		define: {
+			"process.env.NODE_ENV": JSON.stringify(
+				PRODUCTION ? "production" : "development"
+			)
+		},
+		plugins: [
+			html_plugin,
+			ignore_assets,
+			vue(),
+			postCssPlugin({
+				plugins: [require("autoprefixer")],
+				sassOptions: sass_options
+			})
+		],
+		watch: get_watch_config()
+	});
+}
+
+function get_watch_config() {
+	if (WATCH_MODE) {
+		return {
+			async onRebuild(error, result) {
+				if (error) {
+					log_error("There was an error during rebuilding changes.");
+					log();
+					log(chalk.dim(error.stack));
+					notify_redis({ error });
+				} else {
+					let {
+						assets_json,
+						prev_assets_json
+					} = await write_assets_json(result.metafile);
+					if (prev_assets_json) {
+						log_rebuilt_assets(prev_assets_json, assets_json);
+					}
+					notify_redis({ success: true });
+				}
+			}
+		};
+	}
+	return null;
+}
+
+async function clean_dist_folders(apps) {
+	for (let app of apps) {
+		let public_path = get_public_path(app);
+		await fs.promises.rmdir(path.resolve(public_path, "dist", "js"), {
+			recursive: true
+		});
+		await fs.promises.rmdir(path.resolve(public_path, "dist", "css"), {
+			recursive: true
+		});
+	}
+}
+
+function log_built_assets(metafile) {
+	let column_widths = [60, 20];
+	cliui.div(
+		{
+			text: chalk.cyan.bold("File"),
+			width: column_widths[0]
+		},
+		{
+			text: chalk.cyan.bold("Size"),
+			width: column_widths[1]
+		}
+	);
+	cliui.div("");
+
+	let output_by_dist_path = {};
+	for (let outfile in metafile.outputs) {
+		if (outfile.endsWith(".map")) continue;
+		let data = metafile.outputs[outfile];
+		outfile = path.resolve(outfile);
+		outfile = path.relative(assets_path, outfile);
+		let filename = path.basename(outfile);
+		let dist_path = outfile.replace(filename, "");
+		output_by_dist_path[dist_path] = output_by_dist_path[dist_path] || [];
+		output_by_dist_path[dist_path].push({
+			name: filename,
+			size: (data.bytes / 1000).toFixed(2) + " Kb"
+		});
+	}
+
+	for (let dist_path in output_by_dist_path) {
+		let files = output_by_dist_path[dist_path];
+		cliui.div({
+			text: dist_path,
+			width: column_widths[0]
+		});
+
+		for (let i in files) {
+			let file = files[i];
+			let branch = "";
+			if (i < files.length - 1) {
+				branch = "├─ ";
+			} else {
+				branch = "└─ ";
+			}
+			let color = file.name.endsWith(".js") ? "green" : "blue";
+			cliui.div(
+				{
+					text: branch + chalk[color]("" + file.name),
+					width: column_widths[0]
+				},
+				{
+					text: file.size,
+					width: column_widths[1]
+				}
+			);
+		}
+		cliui.div("");
+	}
+	log(cliui.toString());
+}
+
+// to store previous build's assets.json for comparison
+let prev_assets_json;
+let curr_assets_json;
+
+async function write_assets_json(metafile) {
+	prev_assets_json = curr_assets_json;
+	let out = {};
+	for (let output in metafile.outputs) {
+		let info = metafile.outputs[output];
+		let asset_path = "/" + path.relative(sites_path, output);
+		if (info.entryPoint) {
+			out[path.basename(info.entryPoint)] = asset_path;
+		}
+	}
+
+	let assets_json_path = path.resolve(assets_path, "assets.json");
+	let assets_json;
+	try {
+		assets_json = await fs.promises.readFile(assets_json_path, "utf-8");
+	} catch (error) {
+		assets_json = "{}";
+	}
+	assets_json = JSON.parse(assets_json);
+	// update with new values
+	assets_json = Object.assign({}, assets_json, out);
+	curr_assets_json = assets_json;
+
+	await fs.promises.writeFile(
+		assets_json_path,
+		JSON.stringify(assets_json, null, 4)
+	);
+	await update_assets_json_in_cache(assets_json);
+	return {
+		assets_json,
+		prev_assets_json
+	};
+}
+
+function update_assets_json_in_cache(assets_json) {
+	// update assets_json cache in redis, so that it can be read directly by python
+	return new Promise(resolve => {
+		let client = get_redis_subscriber("redis_cache");
+		// handle error event to avoid printing stack traces
+		client.on("error", _ => {
+			log_warn("Cannot connect to redis_cache to update assets_json");
+		});
+		client.set("assets_json", JSON.stringify(assets_json), err => {
+			client.unref();
+			resolve();
+		});
+	});
+}
+
+function run_build_command_for_apps(apps) {
+	let cwd = process.cwd();
+	let { execSync } = require("child_process");
+
+	for (let app of apps) {
+		if (app === "frappe") continue;
+
+		let root_app_path = path.resolve(get_app_path(app), "..");
+		let package_json = path.resolve(root_app_path, "package.json");
+		if (fs.existsSync(package_json)) {
+			let { scripts } = require(package_json);
+			if (scripts && scripts.build) {
+				log("\nRunning build command for", chalk.bold(app));
+				process.chdir(root_app_path);
+				execSync("yarn build", { encoding: "utf8", stdio: "inherit" });
+			}
+		}
+	}
+
+	process.chdir(cwd);
+}
+
+async function notify_redis({ error, success }) {
+	// notify redis which in turns tells socketio to publish this to browser
+	let subscriber = get_redis_subscriber("redis_socketio");
+	subscriber.on("error", _ => {
+		log_warn("Cannot connect to redis_socketio for browser events");
+	});
+
+	let payload = null;
+	if (error) {
+		let formatted = await esbuild.formatMessages(error.errors, {
+			kind: "error",
+			terminalWidth: 100
+		});
+		let stack = error.stack.replace(new RegExp(bench_path, "g"), "");
+		payload = {
+			error,
+			formatted,
+			stack
+		};
+	}
+	if (success) {
+		payload = {
+			success: true
+		};
+	}
+
+	subscriber.publish(
+		"events",
+		JSON.stringify({
+			event: "build_event",
+			message: payload
+		})
+	);
+}
+
+function open_in_editor() {
+	let subscriber = get_redis_subscriber("redis_socketio");
+	subscriber.on("error", _ => {
+		log_warn("Cannot connect to redis_socketio for open_in_editor events");
+	});
+	subscriber.on("message", (event, file) => {
+		if (event === "open_in_editor") {
+			file = JSON.parse(file);
+			let file_path = path.resolve(file.file);
+			log("Opening file in editor:", file_path);
+			let launch = require("launch-editor");
+			launch(`${file_path}:${file.line}:${file.column}`);
+		}
+	});
+	subscriber.subscribe("open_in_editor");
+}
+
+function log_rebuilt_assets(prev_assets, new_assets) {
+	let added_files = [];
+	let old_files = Object.values(prev_assets);
+	let new_files = Object.values(new_assets);
+
+	for (let filepath of new_files) {
+		if (!old_files.includes(filepath)) {
+			added_files.push(filepath);
+		}
+	}
+
+	log(
+		chalk.yellow(
+			`${new Date().toLocaleTimeString()}: Compiled ${
+				added_files.length
+			} files...`
+		)
+	);
+	for (let filepath of added_files) {
+		let filename = path.basename(filepath);
+		log("    " + filename);
+	}
+	log();
+}

esbuild/frappe-html.js

@@ -0,0 +1,43 @@
+module.exports = {
+	name: "frappe-html",
+	setup(build) {
+		let path = require("path");
+		let fs = require("fs/promises");
+
+		build.onResolve({ filter: /\.html$/ }, args => {
+			return {
+				path: path.join(args.resolveDir, args.path),
+				namespace: "frappe-html"
+			};
+		});
+
+		build.onLoad({ filter: /.*/, namespace: "frappe-html" }, args => {
+			let filepath = args.path;
+			let filename = path.basename(filepath).split(".")[0];
+
+			return fs
+				.readFile(filepath, "utf-8")
+				.then(content => {
+					content = scrub_html_template(content);
+					return {
+						contents: `\n\tfrappe.templates['${filename}'] = \`${content}\`;\n`
+					};
+				})
+				.catch(() => {
+					return {
+						contents: "",
+						warnings: [
+							{
+								text: `There was an error importing ${filepath}`
+							}
+						]
+					};
+				});
+		});
+	}
+};
+
+function scrub_html_template(content) {
+	content = content.replace(/`/g, "\\`");
+	return content;
+}

esbuild/ignore-assets.js

@@ -0,0 +1,11 @@
+module.exports = {
+	name: "frappe-ignore-asset",
+	setup(build) {
+		build.onResolve({ filter: /^\/assets\// }, args => {
+			return {
+				path: args.path,
+				external: true
+			};
+		});
+	}
+};

esbuild/index.js

@@ -0,0 +1 @@
+require("./esbuild");

esbuild/sass_options.js

@@ -0,0 +1,29 @@
+let path = require("path");
+let { get_app_path, app_list } = require("./utils");
+
+let node_modules_path = path.resolve(
+	get_app_path("frappe"),
+	"..",
+	"node_modules"
+);
+let app_paths = app_list
+	.map(get_app_path)
+	.map(app_path => path.resolve(app_path, ".."));
+
+module.exports = {
+	includePaths: [node_modules_path, ...app_paths],
+	importer: function(url) {
+		if (url.startsWith("~")) {
+			// strip ~ so that it can resolve from node_modules
+			url = url.slice(1);
+		}
+		if (url.endsWith(".css")) {
+			// strip .css from end of path
+			url = url.slice(0, -4);
+		}
+		// normal file, let it go
+		return {
+			file: url
+		};
+	}
+};

esbuild/utils.js

@@ -0,0 +1,145 @@
+const path = require("path");
+const fs = require("fs");
+const chalk = require("chalk");
+
+const frappe_path = path.resolve(__dirname, "..");
+const bench_path = path.resolve(frappe_path, "..", "..");
+const sites_path = path.resolve(bench_path, "sites");
+const apps_path = path.resolve(bench_path, "apps");
+const assets_path = path.resolve(sites_path, "assets");
+const app_list = get_apps_list();
+
+const app_paths = app_list.reduce((out, app) => {
+	out[app] = path.resolve(apps_path, app, app);
+	return out;
+}, {});
+const public_paths = app_list.reduce((out, app) => {
+	out[app] = path.resolve(app_paths[app], "public");
+	return out;
+}, {});
+const public_js_paths = app_list.reduce((out, app) => {
+	out[app] = path.resolve(app_paths[app], "public/js");
+	return out;
+}, {});
+
+const bundle_map = app_list.reduce((out, app) => {
+	const public_js_path = public_js_paths[app];
+	if (fs.existsSync(public_js_path)) {
+		const all_files = fs.readdirSync(public_js_path);
+		const js_files = all_files.filter(file => file.endsWith(".js"));
+
+		for (let js_file of js_files) {
+			const filename = path.basename(js_file).split(".")[0];
+			out[path.join(app, "js", filename)] = path.resolve(
+				public_js_path,
+				js_file
+			);
+		}
+	}
+
+	return out;
+}, {});
+
+const get_public_path = app => public_paths[app];
+
+const get_build_json_path = app =>
+	path.resolve(get_public_path(app), "build.json");
+
+function get_build_json(app) {
+	try {
+		return require(get_build_json_path(app));
+	} catch (e) {
+		// build.json does not exist
+		return null;
+	}
+}
+
+function delete_file(path) {
+	if (fs.existsSync(path)) {
+		fs.unlinkSync(path);
+	}
+}
+
+function run_serially(tasks) {
+	let result = Promise.resolve();
+	tasks.forEach(task => {
+		if (task) {
+			result = result.then ? result.then(task) : Promise.resolve();
+		}
+	});
+	return result;
+}
+
+const get_app_path = app => app_paths[app];
+
+function get_apps_list() {
+	return fs
+		.readFileSync(path.resolve(sites_path, "apps.txt"), {
+			encoding: "utf-8"
+		})
+		.split("\n")
+		.filter(Boolean);
+}
+
+function get_cli_arg(name) {
+	let args = process.argv.slice(2);
+	let arg = `--${name}`;
+	let index = args.indexOf(arg);
+
+	let value = null;
+	if (index != -1) {
+		value = true;
+	}
+	if (value && args[index + 1]) {
+		value = args[index + 1];
+	}
+	return value;
+}
+
+function log_error(message, badge = "ERROR") {
+	badge = chalk.white.bgRed(` ${badge} `);
+	console.error(`${badge} ${message}`); // eslint-disable-line no-console
+}
+
+function log_warn(message, badge = "WARN") {
+	badge = chalk.black.bgYellowBright(` ${badge} `);
+	console.warn(`${badge} ${message}`); // eslint-disable-line no-console
+}
+
+function log(...args) {
+	console.log(...args); // eslint-disable-line no-console
+}
+
+function get_redis_subscriber(kind) {
+	// get redis subscriber that aborts after 10 connection attempts
+	let { get_redis_subscriber: get_redis } = require("../node_utils");
+	return get_redis(kind, {
+		retry_strategy: function(options) {
+			// abort after 10 connection attempts
+			if (options.attempt > 10) {
+				return undefined;
+			}
+			return Math.min(options.attempt * 100, 2000);
+		}
+	});
+}
+
+module.exports = {
+	app_list,
+	bench_path,
+	assets_path,
+	sites_path,
+	apps_path,
+	bundle_map,
+	get_public_path,
+	get_build_json_path,
+	get_build_json,
+	get_app_path,
+	delete_file,
+	run_serially,
+	get_cli_arg,
+	log,
+	log_warn,
+	log_error,
+	get_redis_subscriber
+};

frappe/__init__.py

@@ -10,13 +10,17 @@ be used to build database driven apps.
 
 Read the documentation: https://frappeframework.com/docs
 """
-from __future__ import unicode_literals, print_function
+import os, warnings
+
+_dev_server = os.environ.get('DEV_SERVER', False)
+
+if _dev_server:
+	warnings.simplefilter('always', DeprecationWarning)
+	warnings.simplefilter('always', PendingDeprecationWarning)
 
-from six import iteritems, binary_type, text_type, string_types, PY2
 from werkzeug.local import Local, release_local
-import os, sys, importlib, inspect, json
+import sys, importlib, inspect, json
 import typing
-from past.builtins import cmp
 import click
 
 # Local application imports
@@ -27,14 +31,7 @@ from .utils.lazy_loader import lazy_import
 # Lazy imports
 faker = lazy_import('faker')
 
-
-# Harmless for Python 3
-# For Python 2 set default encoding to utf-8
-if PY2:
-	reload(sys)
-	sys.setdefaultencoding("utf-8")
-
-__version__ = '13.2.2'
+__version__ = '14.0.0-dev'
 
 __title__ = "Frappe Framework"
 
@@ -97,14 +94,14 @@ def _(msg, lang=None, context=None):
 
 def as_unicode(text, encoding='utf-8'):
 	'''Convert to unicode if required'''
-	if isinstance(text, text_type):
+	if isinstance(text, str):
 		return text
 	elif text==None:
 		return ''
-	elif isinstance(text, binary_type):
-		return text_type(text, encoding)
+	elif isinstance(text, bytes):
+		return str(text, encoding)
 	else:
-		return text_type(text)
+		return str(text)
 
 def get_lang_dict(fortype, name=None):
 	"""Returns the translated language dict for the given type and name.
@@ -204,7 +201,7 @@ def init(site, sites_path=None, new_site=False):
 	local.meta_cache = {}
 	local.form_dict = _dict()
 	local.session = _dict()
-	local.dev_server = os.environ.get('DEV_SERVER', False)
+	local.dev_server = _dev_server
 
 	setup_module_map()
 
@@ -530,16 +527,20 @@ def sendmail(recipients=[], sender="", subject="No Subject", message="No Message
 	if not delayed:
 		now = True
 
-	from frappe.email import queue
-	queue.send(recipients=recipients, sender=sender,
+	from frappe.email.doctype.email_queue.email_queue import QueueBuilder
+	builder = QueueBuilder(recipients=recipients, sender=sender,
 		subject=subject, message=message, text_content=text_content,
 		reference_doctype = doctype or reference_doctype, reference_name = name or reference_name, add_unsubscribe_link=add_unsubscribe_link,
 		unsubscribe_method=unsubscribe_method, unsubscribe_params=unsubscribe_params, unsubscribe_message=unsubscribe_message,
 		attachments=attachments, reply_to=reply_to, cc=cc, bcc=bcc, message_id=message_id, in_reply_to=in_reply_to,
 		send_after=send_after, expose_recipients=expose_recipients, send_priority=send_priority, queue_separately=queue_separately,
-		communication=communication, now=now, read_receipt=read_receipt, is_notification=is_notification,
+		communication=communication, read_receipt=read_receipt, is_notification=is_notification,
 		inline_images=inline_images, header=header, print_letterhead=print_letterhead, with_container=with_container)
 
+	# build email queue and send the email if send_now is True.
+	builder.process(send_now=now)
+
+
 whitelisted = []
 guest_methods = []
 xss_safe_methods = []
@@ -597,7 +598,7 @@ def is_whitelisted(method):
 		# strictly sanitize form_dict
 		# escapes html characters like <> except for predefined tags like a, b, ul etc.
 		for key, value in form_dict.items():
-			if isinstance(value, string_types):
+			if isinstance(value, str):
 				form_dict[key] = sanitize_html(value)
 
 def read_only():
@@ -721,7 +722,7 @@ def has_website_permission(doc=None, ptype='read', user=None, verbose=False, doc
 		user = session.user
 
 	if doc:
-		if isinstance(doc, string_types):
+		if isinstance(doc, str):
 			doc = get_doc(doctype, doc)
 
 		doctype = doc.doctype
@@ -790,7 +791,7 @@ def set_value(doctype, docname, fieldname, value=None):
 	return frappe.client.set_value(doctype, docname, fieldname, value)
 
 def get_cached_doc(*args, **kwargs):
-	if args and len(args) > 1 and isinstance(args[1], text_type):
+	if args and len(args) > 1 and isinstance(args[1], str):
 		key = get_document_cache_key(args[0], args[1])
 		# local cache
 		doc = local.document_cache.get(key)
@@ -821,7 +822,7 @@ def clear_document_cache(doctype, name):
 
 def get_cached_value(doctype, name, fieldname, as_dict=False):
 	doc = get_cached_doc(doctype, name)
-	if isinstance(fieldname, string_types):
+	if isinstance(fieldname, str):
 		if as_dict:
 			throw('Cannot make dict for single fieldname')
 		return doc.get(fieldname)
@@ -1027,7 +1028,7 @@ def get_doc_hooks():
 	if not hasattr(local, 'doc_events_hooks'):
 		hooks = get_hooks('doc_events', {})
 		out = {}
-		for key, value in iteritems(hooks):
+		for key, value in hooks.items():
 			if isinstance(key, tuple):
 				for doctype in key:
 					append_hook(out, doctype, value)
@@ -1109,9 +1110,7 @@ def setup_module_map():
 
 	if not (local.app_modules and local.module_app):
 		local.module_app, local.app_modules = {}, {}
-		for app in get_all_apps(True):
-			if app == "webnotes":
-				app = "frappe"
+		for app in get_all_apps(with_internal_apps=True):
 			local.app_modules.setdefault(app, [])
 			for module in get_module_list(app):
 				module = scrub(module)
@@ -1144,7 +1143,7 @@ def get_file_json(path):
 
 def read_file(path, raise_not_found=False):
 	"""Open a file and return its content as Unicode."""
-	if isinstance(path, text_type):
+	if isinstance(path, str):
 		path = path.encode("utf-8")
 
 	if os.path.exists(path):
@@ -1167,7 +1166,7 @@ def get_attr(method_string):
 
 def call(fn, *args, **kwargs):
 	"""Call a function and match arguments."""
-	if isinstance(fn, string_types):
+	if isinstance(fn, str):
 		fn = get_attr(fn)
 
 	newargs = get_newargs(fn, kwargs)
@@ -1178,13 +1177,9 @@ def get_newargs(fn, kwargs):
 	if hasattr(fn, 'fnargs'):
 		fnargs = fn.fnargs
 	else:
-		try:
-			fnargs, varargs, varkw, defaults = inspect.getargspec(fn)
-		except ValueError:
-			fnargs = inspect.getfullargspec(fn).args
-			varargs = inspect.getfullargspec(fn).varargs
-			varkw = inspect.getfullargspec(fn).varkw
-			defaults = inspect.getfullargspec(fn).defaults
+		fnargs = inspect.getfullargspec(fn).args
+		fnargs.extend(inspect.getfullargspec(fn).kwonlyargs)
+		varkw = inspect.getfullargspec(fn).varkw
 
 	newargs = {}
 	for a in kwargs:
@@ -1626,6 +1621,12 @@ def enqueue(*args, **kwargs):
 	import frappe.utils.background_jobs
 	return frappe.utils.background_jobs.enqueue(*args, **kwargs)
 
+def task(**task_kwargs):
+	def decorator_task(f):
+		f.enqueue = lambda **fun_kwargs: enqueue(f, **task_kwargs, **fun_kwargs)
+		return f
+	return decorator_task
+
 def enqueue_doc(*args, **kwargs):
 	'''
 		Enqueue method to be executed using a background worker
@@ -1693,6 +1694,23 @@ def safe_eval(code, eval_globals=None, eval_locals=None):
 		"round": round
 	}
 
+	UNSAFE_ATTRIBUTES = {
+		# Generator Attributes
+		"gi_frame", "gi_code",
+		# Coroutine Attributes
+		"cr_frame", "cr_code", "cr_origin",
+		# Async Generator Attributes
+		"ag_code", "ag_frame",
+		# Traceback Attributes
+		"tb_frame", "tb_next",
+		# Format Attributes
+		"format", "format_map",
+	}
+
+	for attribute in UNSAFE_ATTRIBUTES:
+		if attribute in code:
+			throw('Illegal rule {0}. Cannot use "{1}"'.format(bold(code), attribute))
+
 	if '__' in code:
 		throw('Illegal rule {0}. Cannot use "__"'.format(bold(code)))
 

frappe/api.py

@@ -1,6 +1,5 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # MIT License. See license.txt
-
 import base64
 import binascii
 import json
@@ -11,6 +10,7 @@ import frappe.client
 import frappe.handler
 from frappe import _
 from frappe.utils.response import build_response
+from frappe.utils.data import sbool
 
 
 def handle():
@@ -108,25 +108,40 @@ def handle():
 
 			elif doctype:
 				if frappe.local.request.method == "GET":
-					if frappe.local.form_dict.get('fields'):
-						frappe.local.form_dict['fields'] = json.loads(frappe.local.form_dict['fields'])
-					frappe.local.form_dict.setdefault('limit_page_length', 20)
-					frappe.local.response.update({
-						"data": frappe.call(
-							frappe.client.get_list,
-							doctype,
-							**frappe.local.form_dict
-						)
-					})
+					# set fields for frappe.get_list
+					if frappe.local.form_dict.get("fields"):
+						frappe.local.form_dict["fields"] = json.loads(frappe.local.form_dict["fields"])
+
+					# set limit of records for frappe.get_list
+					frappe.local.form_dict.setdefault(
+						"limit_page_length",
+						frappe.local.form_dict.limit or frappe.local.form_dict.limit_page_length or 20,
+					)
+
+					# convert strings to native types - only as_dict and debug accept bool
+					for param in ["as_dict", "debug"]:
+						param_val = frappe.local.form_dict.get(param)
+						if param_val is not None:
+							frappe.local.form_dict[param] = sbool(param_val)
+
+					# evaluate frappe.get_list
+					data = frappe.call(frappe.client.get_list, doctype, **frappe.local.form_dict)
+
+					# set frappe.get_list result to response
+					frappe.local.response.update({"data": data})
 
 				if frappe.local.request.method == "POST":
+					# fetch data from from dict
 					data = get_request_form_data()
-					data.update({
-						"doctype": doctype
-					})
-					frappe.local.response.update({
-						"data": frappe.get_doc(data).insert().as_dict()
-					})
+					data.update({"doctype": doctype})
+
+					# insert document from request data
+					doc = frappe.get_doc(data).insert()
+
+					# set response data
+					frappe.local.response.update({"data": doc.as_dict()})
+
+					# commit for POST requests
 					frappe.db.commit()
 			else:
 				raise frappe.DoesNotExistError

frappe/app.py

@@ -1,10 +1,8 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # MIT License. See license.txt
-from __future__ import unicode_literals
 
 import os
-from six import iteritems
 import logging
 
 from werkzeug.local import LocalManager
@@ -99,17 +97,7 @@ def application(request):
 		frappe.monitor.stop(response)
 		frappe.recorder.dump()
 
-		if hasattr(frappe.local, 'conf') and frappe.local.conf.enable_frappe_logger:
-			frappe.logger("frappe.web", allow_site=frappe.local.site).info({
-				"site": get_site_name(request.host),
-				"remote_addr": getattr(request, "remote_addr", "NOTFOUND"),
-				"base_url": getattr(request, "base_url", "NOTFOUND"),
-				"full_path": getattr(request, "full_path", "NOTFOUND"),
-				"method": getattr(request, "method", "NOTFOUND"),
-				"scheme": getattr(request, "scheme", "NOTFOUND"),
-				"http_status_code": getattr(response, "status_code", "NOTFOUND")
-			})
-
+		log_request(request, response)
 		process_response(response)
 		frappe.destroy()
 
@@ -137,6 +125,19 @@ def init_request(request):
 	if request.method != "OPTIONS":
 		frappe.local.http_request = frappe.auth.HTTPRequest()
 
+def log_request(request, response):
+	if hasattr(frappe.local, 'conf') and frappe.local.conf.enable_frappe_logger:
+		frappe.logger("frappe.web", allow_site=frappe.local.site).info({
+			"site": get_site_name(request.host),
+			"remote_addr": getattr(request, "remote_addr", "NOTFOUND"),
+			"base_url": getattr(request, "base_url", "NOTFOUND"),
+			"full_path": getattr(request, "full_path", "NOTFOUND"),
+			"method": getattr(request, "method", "NOTFOUND"),
+			"scheme": getattr(request, "scheme", "NOTFOUND"),
+			"http_status_code": getattr(response, "status_code", "NOTFOUND")
+		})
+
+
 def process_response(response):
 	if not response:
 		return
@@ -185,11 +186,12 @@ def make_form_dict(request):
 		args = request.form or request.args
 
 	if not isinstance(args, dict):
-		frappe.throw("Invalid request arguments")
+		frappe.throw(_("Invalid request arguments"))
 
 	try:
-		frappe.local.form_dict = frappe._dict({ k:v[0] if isinstance(v, (list, tuple)) else v \
-			for k, v in iteritems(args) })
+		frappe.local.form_dict = frappe._dict({
+			k: v[0] if isinstance(v, (list, tuple)) else v for k, v in args.items()
+		})
 	except IndexError:
 		frappe.local.form_dict = frappe._dict(args)
 
@@ -294,8 +296,9 @@ def serve(port=8000, profile=False, no_reload=False, no_threading=False, site=No
 	_sites_path = sites_path
 
 	from werkzeug.serving import run_simple
+	patch_werkzeug_reloader()
 
-	if profile:
+	if profile or os.environ.get('USE_PROFILER'):
 		application = ProfilerMiddleware(application, sort_by=('cumtime', 'calls'))
 
 	if not os.environ.get('NO_STATICS'):
@@ -324,3 +327,23 @@ def serve(port=8000, profile=False, no_reload=False, no_threading=False, site=No
 		use_debugger=not in_test_env,
 		use_evalex=not in_test_env,
 		threaded=not no_threading)
+
+def patch_werkzeug_reloader():
+	"""
+	This function monkey patches Werkzeug reloader to ignore reloading files in
+	the __pycache__ directory.
+
+	To be deprecated when upgrading to Werkzeug 2.
+	"""
+
+	from werkzeug._reloader import WatchdogReloaderLoop
+
+	trigger_reload = WatchdogReloaderLoop.trigger_reload
+
+	def custom_trigger_reload(self, filename):
+		if os.path.basename(os.path.dirname(filename)) == "__pycache__":
+			return
+
+		return trigger_reload(self, filename)
+
+	WatchdogReloaderLoop.trigger_reload = custom_trigger_reload

frappe/auth.py

@@ -1,9 +1,6 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # MIT License. See license.txt
-
-from __future__ import unicode_literals
 import datetime
-
 from frappe import _
 import frappe
 import frappe.database
@@ -19,8 +16,7 @@ from frappe.core.doctype.activity_log.activity_log import add_authentication_log
 from frappe.twofactor import (should_run_2fa, authenticate_for_2factor,
 	confirm_otp_token, get_cached_user_pass)
 from frappe.website.utils import get_home_page
-
-from six.moves.urllib.parse import quote
+from urllib.parse import quote
 
 
 class HTTPRequest:

frappe/automation/doctype/assignment_rule/assignment_rule.py

@@ -2,8 +2,6 @@
 # Copyright (c) 2019, Frappe Technologies and contributors
 # For license information, please see license.txt
 
-from __future__ import unicode_literals
-
 import frappe
 from frappe.model.document import Document
 from frappe.desk.form import assign_to

frappe/automation/doctype/assignment_rule/test_assignment_rule.py

@@ -1,8 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2019, Frappe Technologies and Contributors
 # See license.txt
-from __future__ import unicode_literals
-
 import frappe
 import unittest
 from frappe.utils import random_string

frappe/automation/doctype/assignment_rule_day/assignment_rule_day.py

@@ -2,7 +2,6 @@
 # Copyright (c) 2019, Frappe Technologies and contributors
 # For license information, please see license.txt
 
-from __future__ import unicode_literals
 # import frappe
 from frappe.model.document import Document
 

frappe/automation/doctype/assignment_rule_user/assignment_rule_user.py

@@ -2,7 +2,6 @@
 # Copyright (c) 2019, Frappe Technologies and contributors
 # For license information, please see license.txt
 
-from __future__ import unicode_literals
 # import frappe
 from frappe.model.document import Document
 

frappe/automation/doctype/auto_repeat/auto_repeat.js

@@ -103,7 +103,7 @@ frappe.ui.form.on('Auto Repeat', {
 frappe.auto_repeat.render_schedule = function(frm) {
 	if (!frm.is_dirty() && frm.doc.status !== 'Disabled') {
 		frm.call("get_auto_repeat_schedule").then(r => {
-			frm.dashboard.wrapper.empty();
+			frm.dashboard.reset();
 			frm.dashboard.add_section(
 				frappe.render_template("auto_repeat_schedule", {
 					schedule_details: r.message || []

frappe/automation/doctype/auto_repeat/auto_repeat.py

@@ -2,7 +2,6 @@
 # Copyright (c) 2018, Frappe Technologies Pvt. Ltd. and contributors
 # For license information, please see license.txt
 
-from __future__ import unicode_literals
 import frappe
 from frappe import _
 from datetime import timedelta

frappe/automation/doctype/auto_repeat/test_auto_repeat.py

@@ -1,8 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2018, Frappe Technologies and Contributors
 # See license.txt
-from __future__ import unicode_literals
-
 import unittest
 
 import frappe
@@ -173,7 +171,7 @@ class TestAutoRepeat(unittest.TestCase):
 			fields=['docstatus'],
 			limit=1
 		)
-		self.assertEquals(docnames[0].docstatus, 1)
+		self.assertEqual(docnames[0].docstatus, 1)
 
 
 def make_auto_repeat(**args):

frappe/automation/doctype/auto_repeat_day/auto_repeat_day.py

@@ -2,7 +2,6 @@
 # Copyright (c) 2020, Frappe Technologies and contributors
 # For license information, please see license.txt
 
-from __future__ import unicode_literals
 # import frappe
 from frappe.model.document import Document
 

frappe/automation/doctype/milestone/milestone.py

@@ -2,8 +2,6 @@
 # Copyright (c) 2019, Frappe Technologies and contributors
 # For license information, please see license.txt
 
-from __future__ import unicode_literals
-
 import frappe
 from frappe.model.document import Document
 

frappe/automation/doctype/milestone/test_milestone.py

@@ -1,8 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2019, Frappe Technologies and Contributors
 # See license.txt
-from __future__ import unicode_literals
-
 #import frappe
 import unittest
 

frappe/automation/doctype/milestone_tracker/milestone_tracker.py

@@ -2,8 +2,6 @@
 # Copyright (c) 2019, Frappe Technologies and contributors
 # For license information, please see license.txt
 
-from __future__ import unicode_literals
-
 import frappe
 from frappe.model.document import Document
 import frappe.cache_manager

frappe/automation/doctype/milestone_tracker/test_milestone_tracker.py

@@ -1,8 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2019, Frappe Technologies and Contributors
 # See license.txt
-from __future__ import unicode_literals
-
 import frappe
 import frappe.cache_manager
 import unittest

frappe/boot.py

@@ -1,10 +1,5 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # MIT License. See license.txt
-
-from __future__ import unicode_literals
-
-from six import iteritems, text_type
-
 """
 bootstrap client session
 """
@@ -42,8 +37,6 @@ def get_bootinfo():
 		bootinfo.user_info = get_user_info()
 		bootinfo.sid = frappe.session['sid']
 
-	bootinfo.user_groups = frappe.get_all('User Group', pluck="name")
-
 	bootinfo.modules = {}
 	bootinfo.module_list = []
 	load_desktop_data(bootinfo)
@@ -77,7 +70,7 @@ def get_bootinfo():
 		frappe.get_attr(method)(bootinfo)
 
 	if bootinfo.lang:
-		bootinfo.lang = text_type(bootinfo.lang)
+		bootinfo.lang = str(bootinfo.lang)
 	bootinfo.versions = {k: v['version'] for k, v in get_versions().items()}
 
 	bootinfo.error_report_email = frappe.conf.error_report_email
@@ -222,7 +215,7 @@ def load_translations(bootinfo):
 		messages[name] = frappe._(name)
 
 	# only untranslated
-	messages = {k:v for k, v in iteritems(messages) if k!=v}
+	messages = {k: v for k, v in messages.items() if k!=v}
 
 	bootinfo["__messages"] = messages
 

frappe/build.py

@@ -1,14 +1,12 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # MIT License. See license.txt
-
-from __future__ import print_function, unicode_literals
-
 import os
 import re
 import json
 import shutil
-import warnings
-import tempfile
+import subprocess
+from io import StringIO
+from tempfile import mkdtemp, mktemp
 from distutils.spawn import find_executable
 
 import frappe
@@ -16,8 +14,9 @@ from frappe.utils.minify import JavascriptMinify
 
 import click
 import psutil
-from six import iteritems, text_type
-from six.moves.urllib.parse import urlparse
+from urllib.parse import urlparse
+from simple_chalk import green
+from semantic_version import Version
 
 
 timestamps = {}
@@ -39,35 +38,36 @@ def download_file(url, prefix):
 
 
 def build_missing_files():
-	# check which files dont exist yet from the build.json and tell build.js to build only those!
+	'''Check which files dont exist yet from the assets.json and run build for those files'''
+
 	missing_assets = []
 	current_asset_files = []
-	frappe_build = os.path.join("..", "apps", "frappe", "frappe", "public", "build.json")
 
 	for type in ["css", "js"]:
-		current_asset_files.extend(
-			[
-				"{0}/{1}".format(type, name)
-				for name in os.listdir(os.path.join(sites_path, "assets", type))
-			]
-		)
+		folder = os.path.join(sites_path, "assets", "frappe", "dist", type)
+		current_asset_files.extend(os.listdir(folder))
 
-	with open(frappe_build) as f:
-		all_asset_files = json.load(f).keys()
+	development = frappe.local.conf.developer_mode or frappe.local.dev_server
+	build_mode = "development" if development else "production"
 
-	for asset in all_asset_files:
-		if asset.replace("concat:", "") not in current_asset_files:
-			missing_assets.append(asset)
+	assets_json = frappe.read_file("assets/assets.json")
+	if assets_json:
+		assets_json = frappe.parse_json(assets_json)
 
-	if missing_assets:
-		from subprocess import check_call
-		from shlex import split
+		for bundle_file, output_file in assets_json.items():
+			if not output_file.startswith('/assets/frappe'):
+				continue
 
-		click.secho("\nBuilding missing assets...\n", fg="yellow")
-		command = split(
-			"node rollup/build.js --files {0} --no-concat".format(",".join(missing_assets))
-		)
-		check_call(command, cwd=os.path.join("..", "apps", "frappe"))
+			if os.path.basename(output_file) not in current_asset_files:
+				missing_assets.append(bundle_file)
+
+		if missing_assets:
+			click.secho("\nBuilding missing assets...\n", fg="yellow")
+			files_to_build = ["frappe/" + name for name in missing_assets]
+			bundle(build_mode, files=files_to_build)
+	else:
+		# no assets.json, run full build
+		bundle(build_mode, apps="frappe")
 
 
 def get_assets_link(frappe_head):
@@ -75,8 +75,8 @@ def get_assets_link(frappe_head):
 	from requests import head
 
 	tag = getoutput(
-			"cd ../apps/frappe && git show-ref --tags -d | grep %s | sed -e 's,.*"
-			" refs/tags/,,' -e 's/\^{}//'"
+			r"cd ../apps/frappe && git show-ref --tags -d | grep %s | sed -e 's,.*"
+			r" refs/tags/,,' -e 's/\^{}//'"
 			% frappe_head
 		)
 
@@ -97,9 +97,7 @@ def download_frappe_assets(verbose=True):
 	commit HEAD.
 	Returns True if correctly setup else returns False.
 	"""
-	from simple_chalk import green
 	from subprocess import getoutput
-	from tempfile import mkdtemp
 
 	assets_setup = False
 	frappe_head = getoutput("cd ../apps/frappe && git rev-parse HEAD")
@@ -166,7 +164,7 @@ def symlink(target, link_name, overwrite=False):
 
 	# Create link to target with temporary filename
 	while True:
-		temp_link_name = tempfile.mktemp(dir=link_dir)
+		temp_link_name = mktemp(dir=link_dir)
 
 		# os.* functions mimic as closely as possible system functions
 		# The POSIX symlink() returns EEXIST if link_name already exists
@@ -193,7 +191,8 @@ def symlink(target, link_name, overwrite=False):
 
 
 def setup():
-	global app_paths
+	global app_paths, assets_path
+
 	pymodules = []
 	for app in frappe.get_all_apps(True):
 		try:
@@ -201,51 +200,54 @@ def setup():
 		except ImportError:
 			pass
 	app_paths = [os.path.dirname(pymodule.__file__) for pymodule in pymodules]
+	assets_path = os.path.join(frappe.local.sites_path, "assets")
 
 
-def get_node_pacman():
-	exec_ = find_executable("yarn")
-	if exec_:
-		return exec_
-	raise ValueError("Yarn not found")
-
-
-def bundle(no_compress, app=None, make_copy=False, restore=False, verbose=False, skip_frappe=False):
+def bundle(mode, apps=None, hard_link=False, make_copy=False, restore=False, verbose=False, skip_frappe=False, files=None):
 	"""concat / minify js files"""
 	setup()
-	make_asset_dirs(make_copy=make_copy, restore=restore)
+	make_asset_dirs(hard_link=hard_link)
 
-	pacman = get_node_pacman()
-	mode = "build" if no_compress else "production"
-	command = "{pacman} run {mode}".format(pacman=pacman, mode=mode)
+	mode = "production" if mode == "production" else "build"
+	command = "yarn run {mode}".format(mode=mode)
 
-	if app:
-		command += " --app {app}".format(app=app)
+	if apps:
+		command += " --apps {apps}".format(apps=apps)
 
 	if skip_frappe:
 		command += " --skip_frappe"
 
-	frappe_app_path = os.path.abspath(os.path.join(app_paths[0], ".."))
-	check_yarn()
+	if files:
+		command += " --files {files}".format(files=','.join(files))
+
+	command += " --run-build-command"
+
+	check_node_executable()
+	frappe_app_path = frappe.get_app_path("frappe", "..")
 	frappe.commands.popen(command, cwd=frappe_app_path, env=get_node_env())
 
 
-def watch(no_compress):
+def watch(apps=None):
 	"""watch and rebuild if necessary"""
 	setup()
 
-	pacman = get_node_pacman()
+	command = "yarn run watch"
+	if apps:
+		command += " --apps {apps}".format(apps=apps)
 
-	frappe_app_path = os.path.abspath(os.path.join(app_paths[0], ".."))
-	check_yarn()
+	check_node_executable()
 	frappe_app_path = frappe.get_app_path("frappe", "..")
-	frappe.commands.popen("{pacman} run watch".format(pacman=pacman),
-		cwd=frappe_app_path, env=get_node_env())
+	frappe.commands.popen(command, cwd=frappe_app_path, env=get_node_env())
 
 
-def check_yarn():
+def check_node_executable():
+	node_version = Version(subprocess.getoutput('node -v')[1:])
+	warn = '⚠️ '
+	if node_version.major < 14:
+		click.echo(f"{warn} Please update your node version to 14")
 	if not find_executable("yarn"):
-		print("Please install yarn using below command and try again.\nnpm install -g yarn")
+		click.echo(f"{warn} Please install yarn using below command and try again.\nnpm install -g yarn")
+	click.echo()
 
 def get_node_env():
 	node_env = {
@@ -266,75 +268,109 @@ def get_safe_max_old_space_size():
 
 	return safe_max_old_space_size
 
-def make_asset_dirs(make_copy=False, restore=False):
-	# don't even think of making assets_path absolute - rm -rf ahead.
-	assets_path = os.path.join(frappe.local.sites_path, "assets")
+def generate_assets_map():
+	symlinks = {}
 
-	for dir_path in [os.path.join(assets_path, "js"), os.path.join(assets_path, "css")]:
-		if not os.path.exists(dir_path):
-			os.makedirs(dir_path)
+	for app_name in frappe.get_all_apps():
+		app_doc_path = None
 
-	for app_name in frappe.get_all_apps(True):
 		pymodule = frappe.get_module(app_name)
 		app_base_path = os.path.abspath(os.path.dirname(pymodule.__file__))
-
-		symlinks = []
 		app_public_path = os.path.join(app_base_path, "public")
-		# app/public > assets/app
-		symlinks.append([app_public_path, os.path.join(assets_path, app_name)])
-		# app/node_modules > assets/app/node_modules
-		if os.path.exists(os.path.abspath(app_public_path)):
-			symlinks.append(
-				[
-					os.path.join(app_base_path, "..", "node_modules"),
-					os.path.join(assets_path, app_name, "node_modules"),
-				]
-			)
+		app_node_modules_path = os.path.join(app_base_path, "..", "node_modules")
+		app_docs_path = os.path.join(app_base_path, "docs")
+		app_www_docs_path = os.path.join(app_base_path, "www", "docs")
 
-		app_doc_path = None
-		if os.path.isdir(os.path.join(app_base_path, "docs")):
+		app_assets = os.path.abspath(app_public_path)
+		app_node_modules = os.path.abspath(app_node_modules_path)
+
+		# {app}/public > assets/{app}
+		if os.path.isdir(app_assets):
+			symlinks[app_assets] = os.path.join(assets_path, app_name)
+
+		# {app}/node_modules > assets/{app}/node_modules
+		if os.path.isdir(app_node_modules):
+			symlinks[app_node_modules] = os.path.join(assets_path, app_name, "node_modules")
+
+		# {app}/docs > assets/{app}_docs
+		if os.path.isdir(app_docs_path):
 			app_doc_path = os.path.join(app_base_path, "docs")
-
-		elif os.path.isdir(os.path.join(app_base_path, "www", "docs")):
+		elif os.path.isdir(app_www_docs_path):
 			app_doc_path = os.path.join(app_base_path, "www", "docs")
-
 		if app_doc_path:
-			symlinks.append([app_doc_path, os.path.join(assets_path, app_name + "_docs")])
+			app_docs = os.path.abspath(app_doc_path)
+			symlinks[app_docs] = os.path.join(assets_path, app_name + "_docs")
 
-		for source, target in symlinks:
-			source = os.path.abspath(source)
-			if os.path.exists(source):
-				if restore:
-					if os.path.exists(target):
-						if os.path.islink(target):
-							os.unlink(target)
-						else:
-							shutil.rmtree(target)
-						shutil.copytree(source, target)
-				elif make_copy:
-					if os.path.exists(target):
-						warnings.warn("Target {target} already exists.".format(target=target))
-					else:
-						shutil.copytree(source, target)
-				else:
-					if os.path.exists(target):
-						if os.path.islink(target):
-							os.unlink(target)
-						else:
-							shutil.rmtree(target)
-					try:
-						symlink(source, target, overwrite=True)
-					except OSError:
-						print("Cannot link {} to {}".format(source, target))
-			else:
-				# warnings.warn('Source {source} does not exist.'.format(source = source))
-				pass
+	return symlinks
+
+
+def setup_assets_dirs():
+	for dir_path in (os.path.join(assets_path, x) for x in ("js", "css")):
+		os.makedirs(dir_path, exist_ok=True)
+
+
+def clear_broken_symlinks():
+	for path in os.listdir(assets_path):
+		path = os.path.join(assets_path, path)
+		if os.path.islink(path) and not os.path.exists(path):
+			os.remove(path)
+
+
+
+def unstrip(message: str) -> str:
+	"""Pads input string on the right side until the last available column in the terminal
+	"""
+	_len = len(message)
+	try:
+		max_str = os.get_terminal_size().columns
+	except Exception:
+		max_str = 80
+
+	if _len < max_str:
+		_rem = max_str - _len
+	else:
+		_rem = max_str % _len
+
+	return f"{message}{' ' * _rem}"
+
+
+def make_asset_dirs(hard_link=False):
+	setup_assets_dirs()
+	clear_broken_symlinks()
+	symlinks = generate_assets_map()
+
+	for source, target in symlinks.items():
+		start_message = unstrip(f"{'Copying assets from' if hard_link else 'Linking'} {source} to {target}")
+		fail_message = unstrip(f"Cannot {'copy' if hard_link else 'link'} {source} to {target}")
+
+		# Used '\r' instead of '\x1b[1K\r' to print entire lines in smaller terminal sizes
+		try:
+			print(start_message, end="\r")
+			link_assets_dir(source, target, hard_link=hard_link)
+		except Exception:
+			print(fail_message, end="\r")
+
+	print(unstrip(f"{green('✔')} Application Assets Linked") + "\n")
+
+
+def link_assets_dir(source, target, hard_link=False):
+	if not os.path.exists(source):
+		return
+
+	if os.path.exists(target):
+		if os.path.islink(target):
+			os.unlink(target)
+		else:
+			shutil.rmtree(target)
+
+	if hard_link:
+		shutil.copytree(source, target, dirs_exist_ok=True)
+	else:
+		symlink(source, target, overwrite=True)
 
 
 def build(no_compress=False, verbose=False):
-	assets_path = os.path.join(frappe.local.sites_path, "assets")
-
-	for target, sources in iteritems(get_build_maps()):
+	for target, sources in get_build_maps().items():
 		pack(os.path.join(assets_path, target), sources, no_compress, verbose)
 
 
@@ -348,7 +384,7 @@ def get_build_maps():
 		if os.path.exists(path):
 			with open(path) as f:
 				try:
-					for target, sources in iteritems(json.loads(f.read())):
+					for target, sources in (json.loads(f.read() or "{}")).items():
 						# update app path
 						source_paths = []
 						for source in sources:
@@ -366,8 +402,6 @@ def get_build_maps():
 
 
 def pack(target, sources, no_compress, verbose):
-	from six import StringIO
-
 	outtype, outtxt = target.split(".")[-1], ""
 	jsm = JavascriptMinify()
 
@@ -381,7 +415,7 @@ def pack(target, sources, no_compress, verbose):
 		timestamps[f] = os.path.getmtime(f)
 		try:
 			with open(f, "r") as sourcefile:
-				data = text_type(sourcefile.read(), "utf-8", errors="ignore")
+				data = str(sourcefile.read(), "utf-8", errors="ignore")
 
 			extn = f.rsplit(".", 1)[1]
 
@@ -396,7 +430,7 @@ def pack(target, sources, no_compress, verbose):
 				jsm.minify(tmpin, tmpout)
 				minified = tmpout.getvalue()
 				if minified:
-					outtxt += text_type(minified or "", "utf-8").strip("\n") + ";"
+					outtxt += str(minified or "", "utf-8").strip("\n") + ";"
 
 				if verbose:
 					print("{0}: {1}k".format(f, int(len(minified) / 1024)))
@@ -426,16 +460,16 @@ def html_to_js_template(path, content):
 def scrub_html_template(content):
 	"""Returns HTML content with removed whitespace and comments"""
 	# remove whitespace to a single space
-	content = re.sub("\s+", " ", content)
+	content = re.sub(r"\s+", " ", content)
 
 	# strip comments
-	content = re.sub("(<!--.*?-->)", "", content)
+	content = re.sub(r"(<!--.*?-->)", "", content)
 
 	return content.replace("'", "\'")
 
 
 def files_dirty():
-	for target, sources in iteritems(get_build_maps()):
+	for target, sources in get_build_maps().items():
 		for f in sources:
 			if ":" in f:
 				f, suffix = f.split(":")

frappe/cache_manager.py

@@ -1,8 +1,6 @@
 # Copyright (c) 2018, Frappe Technologies Pvt. Ltd. and Contributors
 # MIT License. See license.txt
 
-from __future__ import unicode_literals
-
 import frappe, json
 from frappe.model.document import Document
 from frappe.desk.notifications import (delete_notification_count_for,
@@ -13,6 +11,8 @@ common_default_keys = ["__default", "__global"]
 doctype_map_keys = ('energy_point_rule_map', 'assignment_rule_map',
 	'milestone_tracker_map', 'event_consumer_document_type_map')
 
+bench_cache_keys = ('assets_json',)
+
 global_cache_keys = ("app_hooks", "installed_apps", 'all_apps',
 		"app_modules", "module_app", "system_settings",
 		'scheduler_events', 'time_zone', 'webhooks', 'active_domains',
@@ -58,6 +58,7 @@ def clear_global_cache():
 	clear_doctype_cache()
 	clear_website_cache()
 	frappe.cache().delete_value(global_cache_keys)
+	frappe.cache().delete_value(bench_cache_keys)
 	frappe.setup_module_map()
 
 def clear_defaults_cache(user=None):

frappe/change_log/v13/v13_3_0.md

@@ -0,0 +1,49 @@
+# Version 13.3.0 Release Notes
+
+### Features & Enhancements
+
+- Deletion Steps in Data Deletion Tool ([#13124](https://github.com/frappe/frappe/pull/13124))
+- Format Option for list-apps in bench CLI ([#13125](https://github.com/frappe/frappe/pull/13125))
+- Add password fieldtype option for Web Form ([#13093](https://github.com/frappe/frappe/pull/13093))
+- Add simple __repr__ for DocTypes ([#13151](https://github.com/frappe/frappe/pull/13151))
+- Switch theme with left/right keys ([#13077](https://github.com/frappe/frappe/pull/13077))
+- sourceURL for injected javascript ([#13022](https://github.com/frappe/frappe/pull/13022))
+
+### Fixes
+
+- Decode uri before importing file via weblink ([#13026](https://github.com/frappe/frappe/pull/13026))
+- Respond to /api requests as JSON by default ([#13053](https://github.com/frappe/frappe/pull/13053))
+- Disabled checkbox should be disabled ([#13021](https://github.com/frappe/frappe/pull/13021))
+- Moving Site folder across different FileSystems failed ([#13038](https://github.com/frappe/frappe/pull/13038))
+- Freeze screen till the background request is complete ([#13078](https://github.com/frappe/frappe/pull/13078))
+- Added conditional rendering for content field in split section w… ([#13075](https://github.com/frappe/frappe/pull/13075))
+- Show delete button on portal if user has permission to delete document ([#13149](https://github.com/frappe/frappe/pull/13149))
+- Dont disable dialog scroll on focusing a Link/Autocomplete field ([#13119](https://github.com/frappe/frappe/pull/13119))
+- Typo in RecorderDetail.vue ([#13086](https://github.com/frappe/frappe/pull/13086))
+- Error for bench drop-site. Added missing import. ([#13064](https://github.com/frappe/frappe/pull/13064))
+- Report column context ([#13090](https://github.com/frappe/frappe/pull/13090))
+- Different service name for push and pull request events ([#13094](https://github.com/frappe/frappe/pull/13094))
+- Moving Site folder across different FileSystems failed ([#13033](https://github.com/frappe/frappe/pull/13033))
+- Consistent checkboxes on all browsers ([#13042](https://github.com/frappe/frappe/pull/13042))
+- Changed shorcut widgets color picker to dropdown ([#13073](https://github.com/frappe/frappe/pull/13073))
+- Error while exporting reports with duration field ([#13118](https://github.com/frappe/frappe/pull/13118))
+- Add margin to download backup card ([#13079](https://github.com/frappe/frappe/pull/13079))
+- Move mention list generation logic to server-side ([#13074](https://github.com/frappe/frappe/pull/13074))
+- Make strings translatable ([#13046](https://github.com/frappe/frappe/pull/13046))
+- Don't evaluate dynamic properties to check if conflicts exist ([#13186](https://github.com/frappe/frappe/pull/13186))
+- Add __ function in vue global for translation in recorder ([#13089](https://github.com/frappe/frappe/pull/13089))
+- Make strings translatable ([#13076](https://github.com/frappe/frappe/pull/13076))
+- Show config in bench CLI ([#13128](https://github.com/frappe/frappe/pull/13128))
+- Add breadcrumbs for list view ([#13091](https://github.com/frappe/frappe/pull/13091))
+- Do not skip data in save while using shortcut ([#13182](https://github.com/frappe/frappe/pull/13182))
+- Use docfields from options if no docfields are returned from meta ([#13188](https://github.com/frappe/frappe/pull/13188))
+- Disable reloading files in `__pycache__` directory ([#13109](https://github.com/frappe/frappe/pull/13109))
+- RTL stylesheet route to load RTL style on demand. ([#13007](https://github.com/frappe/frappe/pull/13007))
+- Do not show messsage when exception is handled ([#13111](https://github.com/frappe/frappe/pull/13111))
+- Replace parseFloat by Number ([#13082](https://github.com/frappe/frappe/pull/13082))
+- Add margin to download backup card ([#13050](https://github.com/frappe/frappe/pull/13050))
+- Translate report column labels ([#13083](https://github.com/frappe/frappe/pull/13083))
+- Grid row color picker field not working ([#13040](https://github.com/frappe/frappe/pull/13040))
+- Improve oauthlib implementation ([#13045](https://github.com/frappe/frappe/pull/13045))
+- Replace filter_by like with full text filter ([#13126](https://github.com/frappe/frappe/pull/13126))
+- Focus jumps to first field ([#13067](https://github.com/frappe/frappe/pull/13067))

frappe/chat/__init__.py

@@ -1,4 +1,4 @@
-from __future__ import unicode_literals
+
 import frappe
 from   frappe import _
 

frappe/chat/doctype/chat_message/chat_message.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 # imports - standard imports
 import json
 

frappe/chat/doctype/chat_profile/chat_profile.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 # imports - module imports
 from   frappe.model.document import Document
 from   frappe import _

frappe/chat/doctype/chat_room/chat_room.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 # imports - module imports
 from frappe.model.document import Document
 from frappe import _

frappe/chat/doctype/chat_room_user/chat_room_user.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 # imports - module imports
 from   frappe.model.document import Document
 import frappe

frappe/chat/doctype/chat_token/chat_token.py

@@ -2,7 +2,6 @@
 # Copyright (c) 2018, Frappe Technologies and contributors
 # For license information, please see license.txt
 
-from __future__ import unicode_literals
 import frappe
 from frappe.model.document import Document
 

frappe/chat/util/__init__.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 # imports - module imports
 from frappe.chat.util.util import (
 	get_user_doc,

frappe/chat/util/test_util.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 # imports - standard imports
 import unittest
 
@@ -9,7 +7,6 @@ from frappe.chat.util import (
 	safe_json_loads
 )
 import frappe
-import six
 
 class TestChatUtil(unittest.TestCase):
 	def test_safe_json_loads(self):
@@ -20,7 +17,7 @@ class TestChatUtil(unittest.TestCase):
 		self.assertEqual(type(number), float)
 
 		string = safe_json_loads("foobar")
-		self.assertEqual(type(string), six.text_type)
+		self.assertEqual(type(string), str)
 
 		array  = safe_json_loads('[{ "foo": "bar" }]')
 		self.assertEqual(type(array), list)

frappe/chat/util/util.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 # imports - standard imports
 import json
 from collections.abc import MutableMapping, MutableSequence, Sequence

frappe/chat/website/__init__.py

@@ -1,4 +1,4 @@
-from __future__ import unicode_literals
+
 import frappe
 from   frappe.chat.util import filter_dict, safe_json_loads
 

frappe/client.py

@@ -1,7 +1,5 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # MIT License. See license.txt
-
-from __future__ import unicode_literals
 import frappe
 from frappe import _
 import frappe.model
@@ -11,7 +9,6 @@ from frappe.utils import get_safe_filters
 from frappe.desk.reportview import validate_args
 from frappe.model.db_query import check_parent_permission
 
-from six import iteritems, string_types, integer_types
 
 '''
 Handle RESTful requests that are mapped to the `/api/resource` route.
@@ -86,7 +83,7 @@ def get_value(doctype, fieldname, filters=None, as_dict=True, debug=False, paren
 		frappe.throw(_("No permission for {0}").format(doctype), frappe.PermissionError)
 
 	filters = get_safe_filters(filters)
-	if isinstance(filters, string_types):
+	if isinstance(filters, str):
 		filters = {"name": filters}
 
 	try:
@@ -135,7 +132,7 @@ def set_value(doctype, name, fieldname, value=None):
 
 	if not value:
 		values = fieldname
-		if isinstance(fieldname, string_types):
+		if isinstance(fieldname, str):
 			try:
 				values = json.loads(fieldname)
 			except ValueError:
@@ -161,7 +158,7 @@ def insert(doc=None):
 	'''Insert a document
 
 	:param doc: JSON or dict object to be inserted'''
-	if isinstance(doc, string_types):
+	if isinstance(doc, str):
 		doc = json.loads(doc)
 
 	if doc.get("parent") and doc.get("parenttype"):
@@ -179,7 +176,7 @@ def insert_many(docs=None):
 	'''Insert multiple documents
 
 	:param docs: JSON or list of dict objects to be inserted in one request'''
-	if isinstance(docs, string_types):
+	if isinstance(docs, str):
 		docs = json.loads(docs)
 
 	out = []
@@ -205,7 +202,7 @@ def save(doc):
 	'''Update (save) an existing document
 
 	:param doc: JSON or dict object with the properties of the document to be updated'''
-	if isinstance(doc, string_types):
+	if isinstance(doc, str):
 		doc = json.loads(doc)
 
 	doc = frappe.get_doc(doc)
@@ -228,7 +225,7 @@ def submit(doc):
 	'''Submit a document
 
 	:param doc: JSON or dict object to be submitted remotely'''
-	if isinstance(doc, string_types):
+	if isinstance(doc, str):
 		doc = json.loads(doc)
 
 	doc = frappe.get_doc(doc)
@@ -266,7 +263,7 @@ def make_width_property_setter(doc):
 	'''Set width Property Setter
 
 	:param doc: Property Setter document with `width` property'''
-	if isinstance(doc, string_types):
+	if isinstance(doc, str):
 		doc = json.loads(doc)
 	if doc["doctype"]=="Property Setter" and doc["property"]=="width":
 		frappe.get_doc(doc).insert(ignore_permissions = True)
@@ -280,7 +277,7 @@ def bulk_update(docs):
 	failed_docs = []
 	for doc in docs:
 		try:
-			ddoc = {key: val for key, val in iteritems(doc) if key not in ['doctype', 'docname']}
+			ddoc = {key: val for key, val in doc.items() if key not in ['doctype', 'docname']}
 			doctype = doc['doctype']
 			docname = doc['docname']
 			doc = frappe.get_doc(doctype, docname)

frappe/commands/__init__.py

@@ -1,7 +1,6 @@
 # Copyright (c) 2015, Web Notes Technologies Pvt. Ltd. and Contributors
 # MIT License. See license.txt
 
-from __future__ import unicode_literals, absolute_import, print_function
 import sys
 import click
 import cProfile
@@ -10,7 +9,7 @@ import frappe
 import frappe.utils
 import subprocess # nosec
 from functools import wraps
-from six import StringIO
+from io import StringIO
 from os import environ
 
 click.disable_unicode_literals_warning = True
@@ -28,6 +27,10 @@ def pass_context(f):
 		except frappe.exceptions.SiteNotSpecifiedError as e:
 			click.secho(str(e), fg='yellow')
 			sys.exit(1)
+		except frappe.exceptions.IncorrectSitePath:
+			site = ctx.obj.get("sites", "")[0]
+			click.secho(f'Site {site} does not exist!', fg='yellow')
+			sys.exit(1)
 
 		if profile:
 			pr.disable()

frappe/commands/scheduler.py

@@ -1,4 +1,3 @@
-from __future__ import unicode_literals, absolute_import, print_function
 import click
 import sys
 import frappe

frappe/commands/site.py

@@ -203,10 +203,13 @@ def install_app(context, apps):
 
 
 @click.command("list-apps")
+@click.option("--format", "-f", type=click.Choice(["text", "json"]), default="text")
 @pass_context
-def list_apps(context):
+def list_apps(context, format):
 	"List apps in site"
 
+	summary_dict = {}
+
 	def fix_whitespaces(text):
 		if site == context.sites[-1]:
 			text = text.rstrip()
@@ -235,18 +238,23 @@ def list_apps(context):
 			]
 			applications_summary = "\n".join(installed_applications)
 			summary = f"{site_title}\n{applications_summary}\n"
+			summary_dict[site] = [app.app_name for app in apps]
 
 		else:
-			applications_summary = "\n".join(frappe.get_installed_apps())
+			installed_applications = frappe.get_installed_apps()
+			applications_summary = "\n".join(installed_applications)
 			summary = f"{site_title}\n{applications_summary}\n"
+			summary_dict[site] = installed_applications
 
 		summary = fix_whitespaces(summary)
 
-		if applications_summary and summary:
+		if format == "text" and applications_summary and summary:
 			print(summary)
 
 		frappe.destroy()
 
+	if format == "json":
+		click.echo(frappe.as_json(summary_dict))
 
 @click.command('add-system-manager')
 @click.argument('email')
@@ -548,7 +556,7 @@ def move(dest_dir, site):
 		site_dump_exists = os.path.exists(final_new_path)
 		count = int(count or 0) + 1
 
-	os.rename(old_path, final_new_path)
+	shutil.move(old_path, final_new_path)
 	frappe.destroy()
 	return final_new_path
 

frappe/commands/translate.py

@@ -1,4 +1,3 @@
-from __future__ import unicode_literals, absolute_import, print_function
 import click
 from frappe.commands import pass_context, get_site
 from frappe.exceptions import SiteNotSpecifiedError

frappe/commands/utils.py

@@ -16,33 +16,52 @@ from frappe.utils import get_bench_path, update_progress_bar, cint
 
 @click.command('build')
 @click.option('--app', help='Build assets for app')
-@click.option('--make-copy', is_flag=True, default=False, help='Copy the files instead of symlinking')
-@click.option('--restore', is_flag=True, default=False, help='Copy the files instead of symlinking with force')
+@click.option('--apps', help='Build assets for specific apps')
+@click.option('--hard-link', is_flag=True, default=False, help='Copy the files instead of symlinking')
+@click.option('--make-copy', is_flag=True, default=False, help='[DEPRECATED] Copy the files instead of symlinking')
+@click.option('--restore', is_flag=True, default=False, help='[DEPRECATED] Copy the files instead of symlinking with force')
+@click.option('--production', is_flag=True, default=False, help='Build assets in production mode')
 @click.option('--verbose', is_flag=True, default=False, help='Verbose')
 @click.option('--force', is_flag=True, default=False, help='Force build assets instead of downloading available')
-def build(app=None, make_copy=False, restore=False, verbose=False, force=False):
-	"Minify + concatenate JS and CSS files, build translations"
-	import frappe.build
+def build(app=None, apps=None, hard_link=False, make_copy=False, restore=False, production=False, verbose=False, force=False):
+	"Compile JS and CSS source files"
+	from frappe.build import bundle, download_frappe_assets
 	frappe.init('')
-	# don't minify in developer_mode for faster builds
-	no_compress = frappe.local.conf.developer_mode or False
+
+	if not apps and app:
+		apps = app
 
 	# dont try downloading assets if force used, app specified or running via CI
-	if not (force or app or os.environ.get('CI')):
+	if not (force or apps or os.environ.get('CI')):
 		# skip building frappe if assets exist remotely
-		skip_frappe = frappe.build.download_frappe_assets(verbose=verbose)
+		skip_frappe = download_frappe_assets(verbose=verbose)
 	else:
 		skip_frappe = False
 
-	frappe.build.bundle(no_compress, app=app, make_copy=make_copy, restore=restore, verbose=verbose, skip_frappe=skip_frappe)
+	# don't minify in developer_mode for faster builds
+	development = frappe.local.conf.developer_mode or frappe.local.dev_server
+	mode = "development" if development else "production"
+	if production:
+		mode = "production"
+
+	if make_copy or restore:
+		hard_link = make_copy or restore
+		click.secho(
+			"bench build: --make-copy and --restore options are deprecated in favour of --hard-link",
+			fg="yellow",
+		)
+
+	bundle(mode, apps=apps, hard_link=hard_link, verbose=verbose, skip_frappe=skip_frappe)
+
 
 
 @click.command('watch')
-def watch():
-	"Watch and concatenate JS and CSS files as and when they change"
-	import frappe.build
+@click.option('--apps', help='Watch assets for specific apps')
+def watch(apps=None):
+	"Watch and compile JS and CSS files as and when they change"
+	from frappe.build import watch
 	frappe.init('')
-	frappe.build.watch(True)
+	watch(apps)
 
 
 @click.command('clear-cache')
@@ -96,22 +115,54 @@ def destroy_all_sessions(context, reason=None):
 		raise SiteNotSpecifiedError
 
 @click.command('show-config')
+@click.option("--format", "-f", type=click.Choice(["text", "json"]), default="text")
 @pass_context
-def show_config(context):
-	"print configuration file"
-	print("\t\033[92m{:<50}\033[0m \033[92m{:<15}\033[0m".format('Config','Value'))
-	sites_path = os.path.join(frappe.utils.get_bench_path(), 'sites')
-	site_path = context.sites[0]
-	configuration = frappe.get_site_config(sites_path=sites_path, site_path=site_path)
-	print_config(configuration)
+def show_config(context, format):
+	"Print configuration file to STDOUT in speified format"
 
+	if not context.sites:
+		raise SiteNotSpecifiedError
 
-def print_config(config):
-	for conf, value in config.items():
-		if isinstance(value, dict):
-			print_config(value)
-		else:
-			print("\t{:<50} {:<15}".format(conf, value))
+	sites_config = {}
+	sites_path = os.getcwd()
+
+	from frappe.utils.commands import render_table
+
+	def transform_config(config, prefix=None):
+		prefix = f"{prefix}." if prefix else ""
+		site_config = []
+
+		for conf, value in config.items():
+			if isinstance(value, dict):
+				site_config += transform_config(value, prefix=f"{prefix}{conf}")
+			else:
+				log_value = json.dumps(value) if isinstance(value, list) else value
+				site_config += [[f"{prefix}{conf}", log_value]]
+
+		return site_config
+
+	for site in context.sites:
+		frappe.init(site)
+
+		if len(context.sites) != 1 and format == "text":
+			if context.sites.index(site) != 0:
+				click.echo()
+			click.secho(f"Site {site}", fg="yellow")
+
+		configuration = frappe.get_site_config(sites_path=sites_path, site_path=site)
+
+		if format == "text":
+			data = transform_config(configuration)
+			data.insert(0, ['Config','Value'])
+			render_table(data)
+
+		if format == "json":
+			sites_config[site] = configuration
+
+		frappe.destroy()
+
+	if format == "json":
+		click.echo(frappe.as_json(sites_config))
 
 
 @click.command('reset-perms')
@@ -171,7 +222,7 @@ def execute(context, method, args=None, kwargs=None, profile=False):
 
 			if profile:
 				import pstats
-				from six import StringIO
+				from io import StringIO
 
 				pr.disable()
 				s = StringIO()
@@ -470,6 +521,7 @@ def console(context):
 			locals()[app] = __import__(app)
 		except ModuleNotFoundError:
 			failed_to_import.append(app)
+			all_apps.remove(app)
 
 	print("Apps in this namespace:\n{}".format(", ".join(all_apps)))
 	if failed_to_import:
@@ -520,17 +572,29 @@ def run_tests(context, app=None, module=None, doctype=None, test=(), profile=Fal
 
 		# Generate coverage report only for app that is being tested
 		source_path = os.path.join(get_bench_path(), 'apps', app or 'frappe')
-		cov = Coverage(source=[source_path], omit=[
-			'*.html',
+		incl = [
+			'*.py',
+		]
+		omit = [
 			'*.js',
 			'*.xml',
+			'*.pyc',
 			'*.css',
 			'*.less',
 			'*.scss',
 			'*.vue',
+			'*.html',
+			'*/test_*',
+			'*/node_modules/*',
 			'*/doctype/*/*_dashboard.py',
-			'*/patches/*'
-		])
+			'*/patches/*',
+		]
+
+		if not app or app == 'frappe':
+			omit.append('*/tests/*')
+			omit.append('*/commands/*')
+
+		cov = Coverage(source=[source_path], omit=omit, include=incl)
 		cov.start()
 
 	ret = frappe.test_runner.main(app, module, doctype, context.verbose, tests=tests,
@@ -547,12 +611,29 @@ def run_tests(context, app=None, module=None, doctype=None, test=(), profile=Fal
 	if os.environ.get('CI'):
 		sys.exit(ret)
 
+@click.command('run-parallel-tests')
+@click.option('--app', help="For App", default='frappe')
+@click.option('--build-number', help="Build number", default=1)
+@click.option('--total-builds', help="Total number of builds", default=1)
+@click.option('--with-coverage', is_flag=True, help="Build coverage file")
+@click.option('--use-orchestrator', is_flag=True, help="Use orchestrator to run parallel tests")
+@pass_context
+def run_parallel_tests(context, app, build_number, total_builds, with_coverage=False, use_orchestrator=False):
+	site = get_site(context)
+	if use_orchestrator:
+		from frappe.parallel_test_runner import ParallelTestWithOrchestrator
+		ParallelTestWithOrchestrator(app, site=site, with_coverage=with_coverage)
+	else:
+		from frappe.parallel_test_runner import ParallelTestRunner
+		ParallelTestRunner(app, site=site, build_number=build_number, total_builds=total_builds, with_coverage=with_coverage)
 
 @click.command('run-ui-tests')
 @click.argument('app')
 @click.option('--headless', is_flag=True, help="Run UI Test in headless mode")
+@click.option('--parallel', is_flag=True, help="Run UI Test in parallel mode")
+@click.option('--ci-build-id')
 @pass_context
-def run_ui_tests(context, app, headless=False):
+def run_ui_tests(context, app, headless=False, parallel=True, ci_build_id=None):
 	"Run UI tests"
 	site = get_site(context)
 	app_base_path = os.path.abspath(os.path.join(frappe.get_app_path(app), '..'))
@@ -584,6 +665,12 @@ def run_ui_tests(context, app, headless=False):
 	command = '{site_env} {password_env} {cypress} {run_or_open}'
 	formatted_command = command.format(site_env=site_env, password_env=password_env, cypress=cypress_path, run_or_open=run_or_open)
 
+	if parallel:
+		formatted_command += ' --parallel'
+
+	if ci_build_id:
+		formatted_command += ' --ci-build-id {}'.format(ci_build_id)
+
 	click.secho("Running Cypress...", fg="yellow")
 	frappe.commands.popen(formatted_command, cwd=app_base_path, raise_err=True)
 
@@ -652,20 +739,27 @@ def make_app(destination, app_name):
 @click.command('set-config')
 @click.argument('key')
 @click.argument('value')
-@click.option('-g', '--global', 'global_', is_flag = True, default = False, help = 'Set Global Site Config')
-@click.option('--as-dict', is_flag=True, default=False)
+@click.option('-g', '--global', 'global_', is_flag=True, default=False, help='Set value in bench config')
+@click.option('-p', '--parse', is_flag=True, default=False, help='Evaluate as Python Object')
+@click.option('--as-dict', is_flag=True, default=False, help='Legacy: Evaluate as Python Object')
 @pass_context
-def set_config(context, key, value, global_ = False, as_dict=False):
+def set_config(context, key, value, global_=False, parse=False, as_dict=False):
 	"Insert/Update a value in site_config.json"
 	from frappe.installer import update_site_config
-	import ast
+
 	if as_dict:
+		from frappe.utils.commands import warn
+		warn("--as-dict will be deprecated in v14. Use --parse instead", category=PendingDeprecationWarning)
+		parse = as_dict
+
+	if parse:
+		import ast
 		value = ast.literal_eval(value)
 
 	if global_:
-		sites_path = os.getcwd() # big assumption.
+		sites_path = os.getcwd()
 		common_site_config_path = os.path.join(sites_path, 'common_site_config.json')
-		update_site_config(key, value, validate = False, site_config_path = common_site_config_path)
+		update_site_config(key, value, validate=False, site_config_path=common_site_config_path)
 	else:
 		for site in context.sites:
 			frappe.init(site=site)
@@ -722,50 +816,6 @@ def rebuild_global_search(context, static_pages=False):
 	if not context.sites:
 		raise SiteNotSpecifiedError
 
-@click.command('auto-deploy')
-@click.argument('app')
-@click.option('--migrate', is_flag=True, default=False, help='Migrate after pulling')
-@click.option('--restart', is_flag=True, default=False, help='Restart after migration')
-@click.option('--remote', default='upstream', help='Remote, default is "upstream"')
-@pass_context
-def auto_deploy(context, app, migrate=False, restart=False, remote='upstream'):
-	'''Pull and migrate sites that have new version'''
-	from frappe.utils.gitutils import get_app_branch
-	from frappe.utils import get_sites
-
-	branch = get_app_branch(app)
-	app_path = frappe.get_app_path(app)
-
-	# fetch
-	subprocess.check_output(['git', 'fetch', remote, branch], cwd = app_path)
-
-	# get diff
-	if subprocess.check_output(['git', 'diff', '{0}..{1}/{0}'.format(branch, remote)], cwd = app_path):
-		print('Updates found for {0}'.format(app))
-		if app=='frappe':
-			# run bench update
-			import shlex
-			subprocess.check_output(shlex.split('bench update --no-backup'), cwd = '..')
-		else:
-			updated = False
-			subprocess.check_output(['git', 'pull', '--rebase', remote, branch],
-				cwd = app_path)
-			# find all sites with that app
-			for site in get_sites():
-				frappe.init(site)
-				if app in frappe.get_installed_apps():
-					print('Updating {0}'.format(site))
-					updated = True
-					subprocess.check_output(['bench', '--site', site, 'clear-cache'], cwd = '..')
-					if migrate:
-						subprocess.check_output(['bench', '--site', site, 'migrate'], cwd = '..')
-				frappe.destroy()
-
-			if updated or restart:
-				subprocess.check_output(['bench', 'restart'], cwd = '..')
-	else:
-		print('No Updates')
-
 
 commands = [
 	build,
@@ -796,5 +846,6 @@ commands = [
 	watch,
 	bulk_rename,
 	add_to_email_queue,
-	rebuild_global_search
+	rebuild_global_search,
+	run_parallel_tests
 ]

frappe/config/__init__.py

@@ -1,6 +1,3 @@
-from __future__ import unicode_literals
-import json
-from six import iteritems
 import frappe
 from frappe import _
 from frappe.desk.moduleview import (get_data, get_onboard_items, config_exists, get_module_link_items_from_list)

frappe/contacts/address_and_contact.py

@@ -1,7 +1,6 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # License: GNU General Public License v3. See license.txt
 
-from __future__ import unicode_literals
 import frappe
 
 from frappe import _

frappe/contacts/doctype/address/address.py

@@ -2,7 +2,6 @@
 # Copyright (c) 2015, Frappe Technologies and contributors
 # For license information, please see license.txt
 
-from __future__ import unicode_literals
 import frappe
 
 from frappe import throw, _
@@ -10,15 +9,10 @@ from frappe.utils import cstr
 
 from frappe.model.document import Document
 from jinja2 import TemplateSyntaxError
-from frappe.utils.user import is_website_user
 from frappe.model.naming import make_autoname
 from frappe.core.doctype.dynamic_link.dynamic_link import deduplicate_dynamic_links
-from six import iteritems, string_types
-from past.builtins import cmp
 from frappe.contacts.address_and_contact import set_link_title
 
-import functools
-
 
 class Address(Document):
 	def __setup__(self):
@@ -112,10 +106,13 @@ def get_default_address(doctype, name, sort_key='is_primary_address'):
 		WHERE
 			dl.parent = addr.name and dl.link_doctype = %s and
 			dl.link_name = %s and ifnull(addr.disabled, 0) = 0
-		""" %(sort_key, '%s', '%s'), (doctype, name))
+		""" %(sort_key, '%s', '%s'), (doctype, name), as_dict=True)
 
 	if out:
-		return sorted(out, key = functools.cmp_to_key(lambda x,y: cmp(y[1], x[1])))[0][0]
+		for contact in out:
+			if contact.get(sort_key):
+				return contact.name
+		return out[0].name
 	else:
 		return None
 
@@ -141,7 +138,7 @@ def get_territory_from_address(address):
 	if not address:
 		return
 
-	if isinstance(address, string_types):
+	if isinstance(address, str):
 		address = frappe.get_cached_doc("Address", address)
 
 	territory = None
@@ -174,14 +171,11 @@ def get_address_list(doctype, txt, filters, limit_start, limit_page_length = 20,
 def has_website_permission(doc, ptype, user, verbose=False):
 	"""Returns true if there is a related lead or contact related to this document"""
 	contact_name = frappe.db.get_value("Contact", {"email_id": frappe.session.user})
+
 	if contact_name:
 		contact = frappe.get_doc('Contact', contact_name)
 		return contact.has_common_link(doc)
 
-		lead_name = frappe.db.get_value("Lead", {"email_id": frappe.session.user})
-		if lead_name:
-			return doc.has_link('Lead', lead_name)
-
 	return False
 
 def get_address_templates(address):
@@ -214,7 +208,7 @@ def address_query(doctype, txt, searchfield, start, page_len, filters):
 
 	condition = ""
 	meta = frappe.get_meta("Address")
-	for fieldname, value in iteritems(filters):
+	for fieldname, value in filters.items():
 		if meta.get_field(fieldname) or fieldname in frappe.db.DEFAULT_COLUMNS:
 			condition += " and {field}={value}".format(
 				field=fieldname,

frappe/contacts/doctype/address/test_address.py

@@ -1,8 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2015, Frappe Technologies and Contributors
 # See license.txt
-from __future__ import unicode_literals
-
 import frappe, unittest
 from frappe.contacts.doctype.address.address import get_address_display
 

frappe/contacts/doctype/address_template/address_template.py

@@ -2,7 +2,6 @@
 # Copyright (c) 2015, Frappe Technologies and contributors
 # For license information, please see license.txt
 
-from __future__ import unicode_literals
 import frappe
 from frappe.model.document import Document
 from frappe.utils import cint

frappe/contacts/doctype/address_template/test_address_template.py

@@ -1,8 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2015, Frappe Technologies and Contributors
 # See license.txt
-from __future__ import unicode_literals
-
 import frappe, unittest
 
 class TestAddressTemplate(unittest.TestCase):
@@ -42,4 +40,4 @@ class TestAddressTemplate(unittest.TestCase):
 				"doctype": "Address Template",
 				"country": 'Brazil',
 				"template": template
-			}).insert()		
+			}).insert()

frappe/contacts/doctype/contact/contact.py

@@ -1,18 +1,13 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # License: GNU General Public License v3. See license.txt
-
-from __future__ import unicode_literals
 import frappe
-from frappe.utils import cstr, has_gravatar, cint
+from frappe.utils import cstr, has_gravatar
 from frappe import _
 from frappe.model.document import Document
 from frappe.core.doctype.dynamic_link.dynamic_link import deduplicate_dynamic_links
-from six import iteritems
-from past.builtins import cmp
 from frappe.model.naming import append_number_if_name_exists
 from frappe.contacts.address_and_contact import set_link_title
 
-import functools
 
 class Contact(Document):
 	def autoname(self):
@@ -120,7 +115,7 @@ class Contact(Document):
 		if len(is_primary) > 1:
 			frappe.throw(_("Only one {0} can be set as primary.").format(frappe.bold(frappe.unscrub(fieldname))))
 
-		primary_number_exists = False 
+		primary_number_exists = False
 		for d in self.phone_nos:
 			if d.get(field_name) == 1:
 				primary_number_exists = True
@@ -140,10 +135,13 @@ def get_default_contact(doctype, name):
 		where
 			dl.link_doctype=%s and
 			dl.link_name=%s and
-			dl.parenttype = "Contact"''', (doctype, name))
+			dl.parenttype = "Contact"''', (doctype, name), as_dict=True)
 
 	if out:
-		return sorted(out, key = functools.cmp_to_key(lambda x,y: cmp(cint(y[1]), cint(x[1]))))[0][0]
+		for contact in out:
+			if contact.is_primary_contact:
+				return contact.parent
+		return out[0].parent
 	else:
 		return None
 

frappe/contacts/doctype/contact/test_contact.py

@@ -1,11 +1,10 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2017, Frappe Technologies and Contributors
 # See license.txt
-from __future__ import unicode_literals
-
 import frappe
 import unittest
-from frappe.exceptions import ValidationError
+
+test_dependencies = ['Contact', 'Salutation']
 
 class TestContact(unittest.TestCase):
 
@@ -52,4 +51,4 @@ def create_contact(name, salutation, emails=None, phones=None, save=True):
 	if save:
 		doc.insert()
 
-	return doc
+	return doc

frappe/contacts/doctype/contact_email/contact_email.py

@@ -2,7 +2,6 @@
 # Copyright (c) 2019, Frappe Technologies and contributors
 # For license information, please see license.txt
 
-from __future__ import unicode_literals
 # import frappe
 from frappe.model.document import Document
 

frappe/contacts/doctype/contact_phone/contact_phone.py

@@ -2,7 +2,6 @@
 # Copyright (c) 2019, Frappe Technologies and contributors
 # For license information, please see license.txt
 
-from __future__ import unicode_literals
 # import frappe
 from frappe.model.document import Document
 

frappe/contacts/doctype/gender/gender.py

@@ -2,7 +2,6 @@
 # Copyright (c) 2017, Frappe Technologies and contributors
 # For license information, please see license.txt
 
-from __future__ import unicode_literals
 from frappe.model.document import Document
 
 class Gender(Document):

frappe/contacts/doctype/gender/test_gender.py

@@ -1,8 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2017, Frappe Technologies and Contributors
 # See license.txt
-from __future__ import unicode_literals
-
 import unittest
 
 class TestGender(unittest.TestCase):

frappe/contacts/doctype/salutation/salutation.py

@@ -2,7 +2,6 @@
 # Copyright (c) 2017, Frappe Technologies and contributors
 # For license information, please see license.txt
 
-from __future__ import unicode_literals
 from frappe.model.document import Document
 
 class Salutation(Document):

frappe/contacts/doctype/salutation/test_salutation.py

@@ -1,8 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2017, Frappe Technologies and Contributors
 # See license.txt
-from __future__ import unicode_literals
-
 import unittest
 
 class TestSalutation(unittest.TestCase):

frappe/contacts/report/addresses_and_contacts/addresses_and_contacts.py

@@ -1,8 +1,5 @@
 # Copyright (c) 2013, Frappe Technologies Pvt. Ltd. and contributors
 # For license information, please see license.txt
-
-from __future__ import unicode_literals
-from six import iteritems
 import frappe
 from frappe import _
 
@@ -58,7 +55,7 @@ def get_reference_addresses_and_contact(reference_doctype, reference_name):
 	reference_details = get_reference_details(reference_doctype, "Address", reference_list, reference_details)
 	reference_details = get_reference_details(reference_doctype, "Contact", reference_list, reference_details)
 
-	for reference_name, details in iteritems(reference_details):
+	for reference_name, details in reference_details.items():
 		addresses = details.get("address", [])
 		contacts  = details.get("contact", [])
 		if not any([addresses, contacts]):

frappe/contacts/report/addresses_and_contacts/test_addresses_and_contacts.py

@@ -1,4 +1,4 @@
-from __future__ import unicode_literals
+
 import frappe
 import frappe.defaults
 import unittest

frappe/core/__init__.py

@@ -1,4 +1,2 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # MIT License. See license.txt
-
-from __future__ import unicode_literals

frappe/core/doctype/__init__.py

@@ -1,4 +1,3 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # MIT License. See license.txt
 
-from __future__ import unicode_literals

frappe/core/doctype/access_log/access_log.py

@@ -3,8 +3,6 @@
 # For license information, please see license.txt
 
 # imports - standard imports
-from __future__ import unicode_literals
-
 # imports - module imports
 import frappe
 from frappe.model.document import Document

frappe/core/doctype/activity_log/activity_log.py

@@ -2,7 +2,6 @@
 # Copyright (c) 2017, Frappe Technologies and contributors
 # For license information, please see license.txt
 
-from __future__ import unicode_literals
 from frappe import _
 from frappe.utils import get_fullname, now
 from frappe.model.document import Document

frappe/core/doctype/activity_log/feed.py

@@ -1,13 +1,11 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # License: See license.txt
 
-from __future__ import unicode_literals
 import frappe
 import frappe.permissions
 from frappe.utils import get_fullname
 from frappe import _
 from frappe.core.doctype.activity_log.activity_log import add_authentication_log
-from six import string_types
 
 def update_feed(doc, method=None):
 	if frappe.flags.in_patch or frappe.flags.in_install or frappe.flags.in_import:
@@ -23,7 +21,7 @@ def update_feed(doc, method=None):
 		feed = doc.get_feed()
 
 		if feed:
-			if isinstance(feed, string_types):
+			if isinstance(feed, str):
 				feed = {"subject": feed}
 
 			feed = frappe._dict(feed)

frappe/core/doctype/activity_log/test_activity_log.py

@@ -1,8 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2015, Frappe Technologies and Contributors
 # See license.txt
-from __future__ import unicode_literals
-
 import frappe
 import unittest
 import time
@@ -65,12 +63,12 @@ class TestActivityLog(unittest.TestCase):
 		frappe.local.login_manager = LoginManager()
 
 		auth_log = self.get_auth_log()
-		self.assertEquals(auth_log.status, 'Success')
+		self.assertEqual(auth_log.status, 'Success')
 
 		# test user logout log
 		frappe.local.login_manager.logout()
 		auth_log = self.get_auth_log(operation='Logout')
-		self.assertEquals(auth_log.status, 'Success')
+		self.assertEqual(auth_log.status, 'Success')
 
 		# test invalid login
 		frappe.form_dict.update({ 'pwd': 'password' })
@@ -90,4 +88,5 @@ class TestActivityLog(unittest.TestCase):
 def update_system_settings(args):
 	doc = frappe.get_doc('System Settings')
 	doc.update(args)
+	doc.flags.ignore_mandatory = 1
 	doc.save()

frappe/core/doctype/block_module/block_module.py

@@ -2,7 +2,6 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and contributors
 # For license information, please see license.txt
 
-from __future__ import unicode_literals
 import frappe
 from frappe.model.document import Document
 

frappe/core/doctype/comment/comment.py

@@ -1,8 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2019, Frappe Technologies and contributors
 # For license information, please see license.txt
-
-from __future__ import unicode_literals, absolute_import
 import frappe
 from frappe import _
 import json

frappe/core/doctype/comment/test_comment.py

@@ -1,8 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2019, Frappe Technologies and Contributors
 # See license.txt
-from __future__ import unicode_literals
-
 import frappe, json
 import unittest
 

frappe/core/doctype/communication/communication.py

@@ -1,29 +1,31 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # MIT License. See license.txt
 
-from __future__ import unicode_literals, absolute_import
 from collections import Counter
 import frappe
 from frappe import _
 from frappe.model.document import Document
 from frappe.utils import validate_email_address, strip_html, cstr, time_diff_in_seconds
-from frappe.core.doctype.communication.email import validate_email, notify, _notify
+from frappe.core.doctype.communication.email import validate_email
+from frappe.core.doctype.communication.mixins import CommunicationEmailMixin
 from frappe.core.utils import get_parent_doc
 from frappe.utils.bot import BotReply
-from frappe.utils import parse_addr
+from frappe.utils import parse_addr, split_emails
 from frappe.core.doctype.comment.comment import update_comment_in_doc
 from email.utils import parseaddr
-from six.moves.urllib.parse import unquote
+from urllib.parse import unquote
 from frappe.utils.user import is_system_user
 from frappe.contacts.doctype.contact.contact import get_contact_name
 from frappe.automation.doctype.assignment_rule.assignment_rule import apply as apply_assignment_rule
 
 exclude_from_linked_with = True
 
-class Communication(Document):
+class Communication(Document, CommunicationEmailMixin):
+	"""Communication represents an external communication like Email.
+	"""
 	no_feed_on_delete = True
+	DOCTYPE = 'Communication'
 
-	"""Communication represents an external communication like Email."""
 	def onload(self):
 		"""create email flag queue"""
 		if self.communication_type == "Communication" and self.communication_medium == "Email" \
@@ -124,6 +126,45 @@ class Communication(Document):
 		if self.communication_type == "Communication":
 			self.notify_change('delete')
 
+	@property
+	def sender_mailid(self):
+		return parse_addr(self.sender)[1] if self.sender else ""
+
+	@staticmethod
+	def _get_emails_list(emails=None, exclude_displayname = False):
+		"""Returns list of emails from given email string.
+
+		* Removes duplicate mailids
+		* Removes display name from email address if exclude_displayname is True
+		"""
+		emails = split_emails(emails) if isinstance(emails, str) else (emails or [])
+		if exclude_displayname:
+			return [email.lower() for email in set([parse_addr(email)[1] for email in emails]) if email]
+		return [email.lower() for email in set(emails) if email]
+
+	def to_list(self, exclude_displayname = True):
+		"""Returns to list.
+		"""
+		return self._get_emails_list(self.recipients, exclude_displayname=exclude_displayname)
+
+	def cc_list(self, exclude_displayname = True):
+		"""Returns cc list.
+		"""
+		return self._get_emails_list(self.cc, exclude_displayname=exclude_displayname)
+
+	def bcc_list(self, exclude_displayname = True):
+		"""Returns bcc list.
+		"""
+		return self._get_emails_list(self.bcc, exclude_displayname=exclude_displayname)
+
+	def get_attachments(self):
+		attachments = frappe.get_all(
+			"File",
+			fields=["name", "file_name", "file_url", "is_private"],
+			filters = {"attached_to_name": self.name, "attached_to_doctype": self.DOCTYPE}
+		)
+		return attachments
+
 	def notify_change(self, action):
 		frappe.publish_realtime('update_docinfo_for_{}_{}'.format(self.reference_doctype, self.reference_name), {
 			'doc': self.as_dict(),
@@ -149,6 +190,23 @@ class Communication(Document):
 
 			self.email_status = "Spam"
 
+	@classmethod
+	def find(cls, name, ignore_error=False):
+		try:
+			return frappe.get_doc(cls.DOCTYPE, name)
+		except frappe.DoesNotExistError:
+			if ignore_error:
+				return
+			raise
+
+	@classmethod
+	def find_one_by_filters(cls, *, order_by=None, **kwargs):
+		name = frappe.db.get_value(cls.DOCTYPE, kwargs, order_by=order_by)
+		return cls.find(name) if name else None
+
+	def update_db(self, **kwargs):
+		frappe.db.set_value(self.DOCTYPE, self.name, kwargs)
+
 	def set_sender_full_name(self):
 		if not self.sender_full_name and self.sender:
 			if self.sender == "Administrator":
@@ -180,36 +238,6 @@ class Communication(Document):
 				if not self.sender_full_name:
 					self.sender_full_name = sender_email
 
-	def send(self, print_html=None, print_format=None, attachments=None,
-		send_me_a_copy=False, recipients=None):
-		"""Send communication via Email.
-
-		:param print_html: Send given value as HTML attachment.
-		:param print_format: Attach print format of parent document."""
-
-		self.send_me_a_copy = send_me_a_copy
-		self.notify(print_html, print_format, attachments, recipients)
-
-	def notify(self, print_html=None, print_format=None, attachments=None,
-		recipients=None, cc=None, bcc=None,fetched_from_email_account=False):
-		"""Calls a delayed task 'sendmail' that enqueus email in Email Queue queue
-
-		:param print_html: Send given value as HTML attachment
-		:param print_format: Attach print format of parent document
-		:param attachments: A list of filenames that should be attached when sending this email
-		:param recipients: Email recipients
-		:param cc: Send email as CC to
-		:param fetched_from_email_account: True when pulling email, the notification shouldn't go to the main recipient
-
-		"""
-		notify(self, print_html, print_format, attachments, recipients, cc, bcc,
-			fetched_from_email_account)
-
-	def _notify(self, print_html=None, print_format=None, attachments=None,
-		recipients=None, cc=None, bcc=None):
-
-		_notify(self, print_html, print_format, attachments, recipients, cc, bcc)
-
 	def bot_reply(self):
 		if self.comment_type == 'Bot' and self.communication_type == 'Chat':
 			reply = BotReply().get_reply(self.content)
@@ -485,4 +513,5 @@ def set_avg_response_time(parent, communication):
 						response_times.append(response_time)
 			if response_times:
 				avg_response_time = sum(response_times) / len(response_times)
-				parent.db_set("avg_response_time", avg_response_time)
+				parent.db_set("avg_response_time", avg_response_time)
+

frappe/core/doctype/communication/email.py

@@ -1,9 +1,6 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # MIT License. See license.txt
 
-from __future__ import unicode_literals, absolute_import
-from six.moves import range
-from six import string_types
 import frappe
 import json
 from email.utils import formataddr
@@ -16,6 +13,11 @@ import time
 from frappe import _
 from frappe.utils.background_jobs import enqueue
 
+OUTGOING_EMAIL_ACCOUNT_MISSING = _("""
+	Unable to send mail because of a missing email account.
+	Please setup default Email Account from Setup > Email > Email Account
+""")
+
 @frappe.whitelist()
 def make(doctype=None, name=None, content=None, subject=None, sent_or_received = "Sent",
 	sender=None, sender_full_name=None, recipients=None, communication_medium="Email", send_email=False,
@@ -39,7 +41,6 @@ def make(doctype=None, name=None, content=None, subject=None, sent_or_received =
 	:param send_me_a_copy: Send a copy to the sender (default **False**).
 	:param email_template: Template which is used to compose mail .
 	"""
-
 	is_error_report = (doctype=="User" and name==frappe.session.user and subject=="Error Report")
 	send_me_a_copy = cint(send_me_a_copy)
 
@@ -77,7 +78,7 @@ def make(doctype=None, name=None, content=None, subject=None, sent_or_received =
 
 	comm.save(ignore_permissions=True)
 
-	if isinstance(attachments, string_types):
+	if isinstance(attachments, str):
 		attachments = json.loads(attachments)
 
 	# if not committed, delayed task doesn't find the communication
@@ -87,12 +88,16 @@ def make(doctype=None, name=None, content=None, subject=None, sent_or_received =
 	frappe.db.commit()
 
 	if cint(send_email):
-		frappe.flags.print_letterhead = cint(print_letterhead)
-		comm.send(print_html, print_format, attachments, send_me_a_copy=send_me_a_copy)
+		if not comm.get_outgoing_email_account():
+			frappe.throw(msg=OUTGOING_EMAIL_ACCOUNT_MISSING, exc=frappe.OutgoingEmailError)
 
+		comm.send_email(print_html=print_html, print_format=print_format,
+			send_me_a_copy=send_me_a_copy, print_letterhead=print_letterhead)
+
+	emails_not_sent_to = comm.exclude_emails_list(include_sender=send_me_a_copy)
 	return {
 		"name": comm.name,
-		"emails_not_sent_to": ", ".join(comm.emails_not_sent_to) if hasattr(comm, "emails_not_sent_to") else None
+		"emails_not_sent_to": ", ".join(emails_not_sent_to or [])
 	}
 
 def validate_email(doc):
@@ -113,261 +118,25 @@ def validate_email(doc):
 
 	# validate sender
 
-def notify(doc, print_html=None, print_format=None, attachments=None,
-	recipients=None, cc=None, bcc=None, fetched_from_email_account=False):
-	"""Calls a delayed task 'sendmail' that enqueus email in Email Queue queue
-
-	:param print_html: Send given value as HTML attachment
-	:param print_format: Attach print format of parent document
-	:param attachments: A list of filenames that should be attached when sending this email
-	:param recipients: Email recipients
-	:param cc: Send email as CC to
-	:param bcc: Send email as BCC to
-	:param fetched_from_email_account: True when pulling email, the notification shouldn't go to the main recipient
-
-	"""
-	recipients, cc, bcc = get_recipients_cc_and_bcc(doc, recipients, cc, bcc,
-		fetched_from_email_account=fetched_from_email_account)
-
-	if not recipients and not cc:
-		return
-
-	doc.emails_not_sent_to = set(doc.all_email_addresses) - set(doc.sent_email_addresses)
-
-	if frappe.flags.in_test:
-		# for test cases, run synchronously
-		doc._notify(print_html=print_html, print_format=print_format, attachments=attachments,
-			recipients=recipients, cc=cc, bcc=None)
-	else:
-		enqueue(sendmail, queue="default", timeout=300, event="sendmail",
-			communication_name=doc.name,
-			print_html=print_html, print_format=print_format, attachments=attachments,
-			recipients=recipients, cc=cc, bcc=bcc, lang=frappe.local.lang,
-			session=frappe.local.session, print_letterhead=frappe.flags.print_letterhead)
-
-def _notify(doc, print_html=None, print_format=None, attachments=None,
-	recipients=None, cc=None, bcc=None):
-
-	prepare_to_notify(doc, print_html, print_format, attachments)
-
-	if doc.outgoing_email_account.send_unsubscribe_message:
-		unsubscribe_message = _("Leave this conversation")
-	else:
-		unsubscribe_message = ""
-
-	frappe.sendmail(
-		recipients=(recipients or []),
-		cc=(cc or []),
-		bcc=(bcc or []),
-		expose_recipients="header",
-		sender=doc.sender,
-		reply_to=doc.incoming_email_account,
-		subject=doc.subject,
-		content=doc.content,
-		reference_doctype=doc.reference_doctype,
-		reference_name=doc.reference_name,
-		attachments=doc.attachments,
-		message_id=doc.message_id,
-		unsubscribe_message=unsubscribe_message,
-		delayed=True,
-		communication=doc.name,
-		read_receipt=doc.read_receipt,
-		is_notification=True if doc.sent_or_received =="Received" else False,
-		print_letterhead=frappe.flags.print_letterhead
-	)
-
-def get_recipients_cc_and_bcc(doc, recipients, cc, bcc, fetched_from_email_account=False):
-	doc.all_email_addresses = []
-	doc.sent_email_addresses = []
-	doc.previous_email_sender = None
-
-	if not recipients:
-		recipients = get_recipients(doc, fetched_from_email_account=fetched_from_email_account)
-
-	if not cc:
-		cc = get_cc(doc, recipients, fetched_from_email_account=fetched_from_email_account)
-
-	if not bcc:
-		bcc = get_bcc(doc, recipients, fetched_from_email_account=fetched_from_email_account)
-
-	if fetched_from_email_account:
-		# email was already sent to the original recipient by the sender's email service
-		original_recipients, recipients = recipients, []
-
-		# send email to the sender of the previous email in the thread which this email is a reply to
-		#provides erratic results and can send external
-		#if doc.previous_email_sender:
-		#	recipients.append(doc.previous_email_sender)
-
-		# cc that was received in the email
-		original_cc = split_emails(doc.cc)
-
-		# don't cc to people who already received the mail from sender's email service
-		cc = list(set(cc) - set(original_cc) - set(original_recipients))
-		remove_administrator_from_email_list(cc)
-
-		original_bcc = split_emails(doc.bcc)
-		bcc = list(set(bcc) - set(original_bcc) - set(original_recipients))
-		remove_administrator_from_email_list(bcc)
-
-	remove_administrator_from_email_list(recipients)
-
-	return recipients, cc, bcc
-
-def remove_administrator_from_email_list(email_list):
-	administrator_email = list(filter(lambda emails: "Administrator" in emails, email_list))
-	if administrator_email:
-		email_list.remove(administrator_email[0])
-
-def prepare_to_notify(doc, print_html=None, print_format=None, attachments=None):
-	"""Prepare to make multipart MIME Email
-
-	:param print_html: Send given value as HTML attachment.
-	:param print_format: Attach print format of parent document."""
-
-	view_link = frappe.utils.cint(frappe.db.get_value("System Settings", "System Settings", "attach_view_link"))
-
-	if print_format and view_link:
-		doc.content += get_attach_link(doc, print_format)
-
-	set_incoming_outgoing_accounts(doc)
-
-	if not doc.sender:
-		doc.sender = doc.outgoing_email_account.email_id
-
-	if not doc.sender_full_name:
-		doc.sender_full_name = doc.outgoing_email_account.name or _("Notification")
-
-	if doc.sender:
-		# combine for sending to get the format 'Jane <jane@example.com>'
-		doc.sender = get_formatted_email(doc.sender_full_name, mail=doc.sender)
-
-	doc.attachments = []
-
-	if print_html or print_format:
-		doc.attachments.append({"print_format_attachment":1, "doctype":doc.reference_doctype,
-			"name":doc.reference_name, "print_format":print_format, "html":print_html})
-
-	if attachments:
-		if isinstance(attachments, string_types):
-			attachments = json.loads(attachments)
-
-		for a in attachments:
-			if isinstance(a, string_types):
-				# is it a filename?
-				try:
-					# check for both filename and file id
-					file_id = frappe.db.get_list('File', or_filters={'file_name': a, 'name': a}, limit=1)
-					if not file_id:
-						frappe.throw(_("Unable to find attachment {0}").format(a))
-					file_id = file_id[0]['name']
-					_file = frappe.get_doc("File", file_id)
-					_file.get_content()
-					# these attachments will be attached on-demand
-					# and won't be stored in the message
-					doc.attachments.append({"fid": file_id})
-				except IOError:
-					frappe.throw(_("Unable to find attachment {0}").format(a))
-			else:
-				doc.attachments.append(a)
-
 def set_incoming_outgoing_accounts(doc):
-	doc.incoming_email_account = doc.outgoing_email_account = None
+	from frappe.email.doctype.email_account.email_account import EmailAccount
+	incoming_email_account = EmailAccount.find_incoming(
+		match_by_email=doc.sender, match_by_doctype=doc.reference_doctype)
+	doc.incoming_email_account = incoming_email_account.email_id if incoming_email_account else None
 
-	if not doc.incoming_email_account and doc.sender:
-		doc.incoming_email_account = frappe.db.get_value("Email Account",
-			{"email_id": doc.sender, "enable_incoming": 1}, "email_id")
-
-	if not doc.incoming_email_account and doc.reference_doctype:
-		doc.incoming_email_account = frappe.db.get_value("Email Account",
-			{"append_to": doc.reference_doctype, }, "email_id")
-
-	if not doc.incoming_email_account:
-		doc.incoming_email_account = frappe.db.get_value("Email Account",
-			{"default_incoming": 1, "enable_incoming": 1},  "email_id")
-
-	doc.outgoing_email_account = frappe.email.smtp.get_outgoing_email_account(raise_exception_not_set=False,
-		append_to=doc.doctype, sender=doc.sender)
+	doc.outgoing_email_account = EmailAccount.find_outgoing(
+		match_by_email=doc.sender, match_by_doctype=doc.reference_doctype)
 
 	if doc.sent_or_received == "Sent":
 		doc.db_set("email_account", doc.outgoing_email_account.name)
 
-def get_recipients(doc, fetched_from_email_account=False):
-	"""Build a list of email addresses for To"""
-	# [EDGE CASE] doc.recipients can be None when an email is sent as BCC
-	recipients = split_emails(doc.recipients)
-
-	#if fetched_from_email_account and doc.in_reply_to:
-		# add sender of previous reply
-		#doc.previous_email_sender = frappe.db.get_value("Communication", doc.in_reply_to, "sender")
-		#recipients.append(doc.previous_email_sender)
-
-	if recipients:
-		recipients = filter_email_list(doc, recipients, [])
-
-	return recipients
-
-def get_cc(doc, recipients=None, fetched_from_email_account=False):
-	"""Build a list of email addresses for CC"""
-	# get a copy of CC list
-	cc = split_emails(doc.cc)
-
-	if doc.reference_doctype and doc.reference_name:
-		if fetched_from_email_account:
-			# if it is a fetched email, add follows to CC
-			cc.append(get_owner_email(doc))
-			cc += get_assignees(doc)
-
-	if getattr(doc, "send_me_a_copy", False) and doc.sender not in cc:
-		cc.append(doc.sender)
-
-	if cc:
-		# exclude unfollows, recipients and unsubscribes
-		exclude = [] #added to remove account check
-		exclude += [d[0] for d in frappe.db.get_all("User", ["email"], {"thread_notify": 0}, as_list=True)]
-		exclude += [(parse_addr(email)[1] or "").lower() for email in recipients]
-
-		if fetched_from_email_account:
-			# exclude sender when pulling email
-			exclude += [parse_addr(doc.sender)[1]]
-
-		if doc.reference_doctype and doc.reference_name:
-			exclude += [d[0] for d in frappe.db.get_all("Email Unsubscribe", ["email"],
-				{"reference_doctype": doc.reference_doctype, "reference_name": doc.reference_name}, as_list=True)]
-
-		cc = filter_email_list(doc, cc, exclude, is_cc=True)
-
-	return cc
-
-def get_bcc(doc, recipients=None, fetched_from_email_account=False):
-	"""Build a list of email addresses for BCC"""
-	bcc = split_emails(doc.bcc)
-
-	if bcc:
-		exclude = []
-		exclude += [d[0] for d in frappe.db.get_all("User", ["email"], {"thread_notify": 0}, as_list=True)]
-		exclude += [(parse_addr(email)[1] or "").lower() for email in recipients]
-
-		if fetched_from_email_account:
-			# exclude sender when pulling email
-			exclude += [parse_addr(doc.sender)[1]]
-
-		if doc.reference_doctype and doc.reference_name:
-			exclude += [d[0] for d in frappe.db.get_all("Email Unsubscribe", ["email"],
-				{"reference_doctype": doc.reference_doctype, "reference_name": doc.reference_name}, as_list=True)]
-
-		bcc = filter_email_list(doc, bcc, exclude, is_bcc=True)
-
-	return bcc
-
 def add_attachments(name, attachments):
 	'''Add attachments to the given Communication'''
 	# loop through attachments
 	for a in attachments:
-		if isinstance(a, string_types):
+		if isinstance(a, str):
 			attach = frappe.db.get_value("File", {"name":a},
 				["file_name", "file_url", "is_private"], as_dict=1)
-
 			# save attachments to new doc
 			_file = frappe.get_doc({
 				"doctype": "File",
@@ -379,103 +148,6 @@ def add_attachments(name, attachments):
 			})
 			_file.save(ignore_permissions=True)
 
-def filter_email_list(doc, email_list, exclude, is_cc=False, is_bcc=False):
-	# temp variables
-	filtered = []
-	email_address_list = []
-
-	for email in list(set(email_list)):
-		email_address = (parse_addr(email)[1] or "").lower()
-		if not email_address:
-			continue
-
-		# this will be used to eventually find email addresses that aren't sent to
-		doc.all_email_addresses.append(email_address)
-
-		if (email in exclude) or (email_address in exclude):
-			continue
-
-		if is_cc:
-			is_user_enabled = frappe.db.get_value("User", email_address, "enabled")
-			if is_user_enabled==0:
-				# don't send to disabled users
-				continue
-
-		if is_bcc:
-			is_user_enabled = frappe.db.get_value("User", email_address, "enabled")
-			if is_user_enabled==0:
-				continue
-
-		# make sure of case-insensitive uniqueness of email address
-		if email_address not in email_address_list:
-			# append the full email i.e. "Human <human@example.com>"
-			filtered.append(email)
-			email_address_list.append(email_address)
-
-	doc.sent_email_addresses.extend(email_address_list)
-
-	return filtered
-
-def get_owner_email(doc):
-	owner = get_parent_doc(doc).owner
-	return get_formatted_email(owner) or owner
-
-def get_assignees(doc):
-	return [( get_formatted_email(d.owner) or d.owner ) for d in
-		frappe.db.get_all("ToDo", filters={
-			"reference_type": doc.reference_doctype,
-			"reference_name": doc.reference_name,
-			"status": "Open"
-		}, fields=["owner"])
-	]
-
-def get_attach_link(doc, print_format):
-	"""Returns public link for the attachment via `templates/emails/print_link.html`."""
-	return frappe.get_template("templates/emails/print_link.html").render({
-		"url": get_url(),
-		"doctype": doc.reference_doctype,
-		"name": doc.reference_name,
-		"print_format": print_format,
-		"key": get_parent_doc(doc).get_signature()
-	})
-
-def sendmail(communication_name, print_html=None, print_format=None, attachments=None,
-	recipients=None, cc=None, bcc=None, lang=None, session=None, print_letterhead=None):
-	try:
-
-		if lang:
-			frappe.local.lang = lang
-
-		if session:
-			# hack to enable access to private files in PDF
-			session['data'] = frappe._dict(session['data'])
-			frappe.local.session.update(session)
-
-		if print_letterhead:
-			frappe.flags.print_letterhead = print_letterhead
-
-		# upto 3 retries
-		for i in range(3):
-			try:
-				communication = frappe.get_doc("Communication", communication_name)
-				communication._notify(print_html=print_html, print_format=print_format, attachments=attachments,
-					recipients=recipients, cc=cc, bcc=bcc)
-
-			except frappe.db.InternalError as e:
-				# deadlock, try again
-				if frappe.db.is_deadlocked(e):
-					frappe.db.rollback()
-					time.sleep(1)
-					continue
-				else:
-					raise
-			else:
-				break
-
-	except:
-		traceback = frappe.log_error("frappe.core.doctype.communication.email.sendmail")
-		raise
-
 @frappe.whitelist(allow_guest=True)
 def mark_email_as_seen(name=None):
 	try:

frappe/core/doctype/communication/mixins.py

@@ -0,0 +1,297 @@
+import frappe
+from frappe import _
+from frappe.core.utils import get_parent_doc
+from frappe.utils import parse_addr, get_formatted_email, get_url
+from frappe.email.doctype.email_account.email_account import EmailAccount
+
+class CommunicationEmailMixin:
+	"""Mixin class to handle communication mails.
+	"""
+	def is_email_communication(self):
+		return self.communication_type=="Communication" and self.communication_medium == "Email"
+
+	def get_owner(self):
+		"""Get owner of the communication docs parent.
+		"""
+		parent_doc = get_parent_doc(self)
+		return parent_doc.owner if parent_doc else None
+
+	def get_all_email_addresses(self, exclude_displayname=False):
+		"""Get all Email addresses mentioned in the doc along with display name.
+		"""
+		return self.to_list(exclude_displayname=exclude_displayname) + \
+			self.cc_list(exclude_displayname=exclude_displayname) + \
+			self.bcc_list(exclude_displayname=exclude_displayname)
+
+	def get_email_with_displayname(self, email_address):
+		"""Returns email address after adding displayname.
+		"""
+		display_name, email = parse_addr(email_address)
+		if display_name and display_name != email:
+			return email_address
+
+		# emailid to emailid with display name map.
+		email_map = {parse_addr(email)[1]: email for email in self.get_all_email_addresses()}
+		return email_map.get(email, email)
+
+	def mail_recipients(self, is_inbound_mail_communcation=False):
+		"""Build to(recipient) list to send an email.
+		"""
+		# Incase of inbound mail, recipients already received the mail, no need to send again.
+		if is_inbound_mail_communcation:
+			return []
+
+		if hasattr(self, '_final_recipients'):
+			return self._final_recipients
+
+		to = self.to_list()
+		self._final_recipients = list(filter(lambda id: id != 'Administrator', to))
+		return self._final_recipients
+
+	def get_mail_recipients_with_displayname(self, is_inbound_mail_communcation=False):
+		"""Build to(recipient) list to send an email including displayname in email.
+		"""
+		to_list = self.mail_recipients(is_inbound_mail_communcation=is_inbound_mail_communcation)
+		return [self.get_email_with_displayname(email) for email in to_list]
+
+	def mail_cc(self, is_inbound_mail_communcation=False, include_sender = False):
+		"""Build cc list to send an email.
+
+		* if email copy is requested by sender, then add sender to CC.
+		* If this doc is created through inbound mail, then add doc owner to cc list
+		* remove all the thread_notify disabled users.
+		* Make sure that all users enabled in the system
+		* Remove admin from email list
+
+		* FixMe: Removed adding TODO owners to cc list. Check if that is needed.
+		"""
+		if hasattr(self, '_final_cc'):
+			return self._final_cc
+
+		cc = self.cc_list()
+
+		# Need to inform parent document owner incase communication is created through inbound mail
+		if include_sender:
+			cc.append(self.sender_mailid)
+		if is_inbound_mail_communcation:
+			cc.append(self.get_owner())
+			cc = set(cc) - {self.sender_mailid}
+
+		cc = set(cc) - set(self.filter_thread_notification_disbled_users(cc))
+		cc = cc - set(self.mail_recipients(is_inbound_mail_communcation=is_inbound_mail_communcation))
+		cc = cc - set(self.filter_disabled_users(cc))
+
+		# # Incase of inbound mail, to and cc already received the mail, no need to send again.
+		if is_inbound_mail_communcation:
+			cc = cc - set(self.cc_list() + self.to_list())
+
+		self._final_cc = list(filter(lambda id: id != 'Administrator', cc))
+		return self._final_cc
+
+	def get_mail_cc_with_displayname(self, is_inbound_mail_communcation=False, include_sender = False):
+		cc_list = self.mail_cc(is_inbound_mail_communcation=False, include_sender = False)
+		return [self.get_email_with_displayname(email) for email in cc_list]
+
+	def mail_bcc(self, is_inbound_mail_communcation=False):
+		"""
+		* Thread_notify check
+		* Email unsubscribe list
+		* User must be enabled in the system
+		* remove_administrator_from_email_list
+		"""
+		if hasattr(self, '_final_bcc'):
+			return self._final_bcc
+
+		bcc = set(self.bcc_list())
+		if is_inbound_mail_communcation:
+			bcc = bcc - {self.sender_mailid}
+		bcc = bcc - set(self.filter_thread_notification_disbled_users(bcc))
+		bcc = bcc - set(self.mail_recipients(is_inbound_mail_communcation=is_inbound_mail_communcation))
+		bcc = bcc - set(self.filter_disabled_users(bcc))
+
+		# Incase of inbound mail, to and cc & bcc already received the mail, no need to send again.
+		if is_inbound_mail_communcation:
+			bcc = bcc - set(self.bcc_list() + self.to_list())
+
+		self._final_bcc = list(filter(lambda id: id != 'Administrator', bcc))
+		return self._final_bcc
+
+	def get_mail_bcc_with_displayname(self, is_inbound_mail_communcation=False):
+		bcc_list = self.mail_bcc(is_inbound_mail_communcation=is_inbound_mail_communcation)
+		return [self.get_email_with_displayname(email) for email in bcc_list]
+
+	def mail_sender(self):
+		email_account = self.get_outgoing_email_account()
+		if not self.sender_mailid and email_account:
+			return email_account.email_id
+		return self.sender_mailid
+
+	def mail_sender_fullname(self):
+		email_account = self.get_outgoing_email_account()
+		if not self.sender_full_name:
+			return (email_account and email_account.name) or _("Notification")
+		return self.sender_full_name
+
+	def get_mail_sender_with_displayname(self):
+		return get_formatted_email(self.mail_sender_fullname(), mail=self.mail_sender())
+
+	def get_content(self, print_format=None):
+		if print_format:
+			return self.content + self.get_attach_link(print_format)
+		return self.content
+
+	def get_attach_link(self, print_format):
+		"""Returns public link for the attachment via `templates/emails/print_link.html`."""
+		return frappe.get_template("templates/emails/print_link.html").render({
+			"url": get_url(),
+			"doctype": self.reference_doctype,
+			"name": self.reference_name,
+			"print_format": print_format,
+			"key": get_parent_doc(self).get_signature()
+		})
+
+	def get_outgoing_email_account(self):
+		if not hasattr(self, '_outgoing_email_account'):
+			if self.email_account:
+				self._outgoing_email_account = EmailAccount.find(self.email_account)
+			else:
+				self._outgoing_email_account = EmailAccount.find_outgoing(
+					match_by_email=self.sender_mailid,
+					match_by_doctype=self.reference_doctype
+				)
+
+				if self.sent_or_received == "Sent" and self._outgoing_email_account:
+					self.db_set("email_account", self._outgoing_email_account.name)
+
+		return self._outgoing_email_account
+
+	def get_incoming_email_account(self):
+		if not hasattr(self, '_incoming_email_account'):
+			self._incoming_email_account = EmailAccount.find_incoming(
+				match_by_email=self.sender_mailid,
+				match_by_doctype=self.reference_doctype
+			)
+		return self._incoming_email_account
+
+	def mail_attachments(self, print_format=None, print_html=None):
+		final_attachments = []
+
+		if print_format and print_html:
+			d = {'print_format': print_format, 'print_html': print_html, 'print_format_attachment': 1,
+					'doctype': self.reference_doctype, 'name': self.reference_name}
+			final_attachments.append(d)
+
+		for a in self.get_attachments() or []:
+			final_attachments.append({"fid": a['name']})
+
+		return final_attachments
+
+	def get_unsubscribe_message(self):
+		email_account = self.get_outgoing_email_account()
+		if email_account and email_account.send_unsubscribe_message:
+			return _("Leave this conversation")
+		return ''
+
+	def exclude_emails_list(self, is_inbound_mail_communcation=False, include_sender=False):
+		"""List of mail id's excluded while sending mail.
+		"""
+		all_ids = self.get_all_email_addresses(exclude_displayname=True)
+		final_ids = self.mail_recipients(is_inbound_mail_communcation = is_inbound_mail_communcation) + \
+			self.mail_bcc(is_inbound_mail_communcation = is_inbound_mail_communcation) + \
+			self.mail_cc(is_inbound_mail_communcation = is_inbound_mail_communcation, include_sender=include_sender)
+		return set(all_ids) - set(final_ids)
+
+	@staticmethod
+	def filter_thread_notification_disbled_users(emails):
+		"""Filter users based on notifications for email threads setting is disabled.
+		"""
+		if not emails:
+			return []
+
+		disabled_users = frappe.db.sql_list("""
+			SELECT
+				email
+			FROM
+				`tabUser`
+			where
+				email in %(emails)s
+				and
+				thread_notify=0
+		""", {'emails': tuple(emails)})
+		return disabled_users
+
+	@staticmethod
+	def filter_disabled_users(emails):
+		"""
+		"""
+		if not emails:
+			return []
+
+		disabled_users = frappe.db.sql_list("""
+			SELECT
+				email
+			FROM
+				`tabUser`
+			where
+				email in %(emails)s
+				and
+				enabled=0
+		""", {'emails': tuple(emails)})
+		return disabled_users
+
+	def sendmail_input_dict(self, print_html=None, print_format=None,
+			send_me_a_copy=None, print_letterhead=None, is_inbound_mail_communcation=None):
+
+		outgoing_email_account = self.get_outgoing_email_account()
+		if not outgoing_email_account:
+			return {}
+
+		recipients = self.get_mail_recipients_with_displayname(
+			is_inbound_mail_communcation=is_inbound_mail_communcation
+		)
+		cc = self.get_mail_cc_with_displayname(
+			is_inbound_mail_communcation=is_inbound_mail_communcation,
+			include_sender = send_me_a_copy
+		)
+		bcc = self.get_mail_bcc_with_displayname(
+			is_inbound_mail_communcation=is_inbound_mail_communcation
+		)
+
+		if not (recipients or cc):
+			return {}
+
+		final_attachments = self.mail_attachments(print_format=print_format, print_html=print_html)
+		incoming_email_account = self.get_incoming_email_account()
+		return {
+			"recipients": recipients,
+			"cc": cc,
+			"bcc": bcc,
+			"expose_recipients": "header",
+			"sender": self.get_mail_sender_with_displayname(),
+			"reply_to": incoming_email_account and incoming_email_account.email_id,
+			"subject": self.subject,
+			"content": self.get_content(print_format=print_format),
+			"reference_doctype": self.reference_doctype,
+			"reference_name": self.reference_name,
+			"attachments": final_attachments,
+			"message_id": self.message_id,
+			"unsubscribe_message": self.get_unsubscribe_message(),
+			"delayed": True,
+			"communication": self.name,
+			"read_receipt": self.read_receipt,
+			"is_notification": (self.sent_or_received =="Received" and True) or False,
+			"print_letterhead": print_letterhead
+		}
+
+	def send_email(self, print_html=None, print_format=None,
+			send_me_a_copy=None, print_letterhead=None, is_inbound_mail_communcation=None):
+		input_dict = self.sendmail_input_dict(
+			print_html=print_html,
+			print_format=print_format,
+			send_me_a_copy=send_me_a_copy,
+			print_letterhead=print_letterhead,
+			is_inbound_mail_communcation=is_inbound_mail_communcation
+		)
+
+		if input_dict:
+			frappe.sendmail(**input_dict)

frappe/core/doctype/communication/test_communication.py

@@ -1,12 +1,12 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # See license.txt
-from __future__ import unicode_literals
+import unittest
+from urllib.parse import quote
 
 import frappe
-import unittest
-from six.moves.urllib.parse import quote
-test_records = frappe.get_test_records('Communication')
+from frappe.email.doctype.email_queue.email_queue import EmailQueue
 
+test_records = frappe.get_test_records('Communication')
 
 class TestCommunication(unittest.TestCase):
 
@@ -201,6 +201,70 @@ class TestCommunication(unittest.TestCase):
 
 		self.assertIn(("Note", note.name), doc_links)
 
+class TestCommunicationEmailMixin(unittest.TestCase):
+	def new_communication(self, recipients=None, cc=None, bcc=None):
+		recipients = ', '.join(recipients or [])
+		cc = ', '.join(cc or [])
+		bcc = ', '.join(bcc or [])
+
+		comm = frappe.get_doc({
+			"doctype": "Communication",
+			"communication_type": "Communication",
+			"communication_medium": "Email",
+			"content": "Test content",
+			"recipients": recipients,
+			"cc": cc,
+			"bcc": bcc
+		}).insert(ignore_permissions=True)
+		return comm
+
+	def new_user(self, email, **user_data):
+		user_data.setdefault('first_name', 'first_name')
+		user = frappe.new_doc('User')
+		user.email = email
+		user.update(user_data)
+		user.insert(ignore_permissions=True, ignore_if_duplicate=True)
+		return user
+
+	def test_recipients(self):
+		to_list = ['to@test.com', 'receiver <to+1@test.com>', 'to@test.com']
+		comm = self.new_communication(recipients = to_list)
+		res = comm.get_mail_recipients_with_displayname()
+		self.assertCountEqual(res, ['to@test.com', 'receiver <to+1@test.com>'])
+		comm.delete()
+
+	def test_cc(self):
+		to_list = ['to@test.com']
+		cc_list = ['cc+1@test.com', 'cc <cc+2@test.com>', 'to@test.com']
+		user = self.new_user(email='cc+1@test.com', thread_notify=0)
+		comm = self.new_communication(recipients=to_list, cc=cc_list)
+		res = comm.get_mail_cc_with_displayname()
+		self.assertCountEqual(res, ['cc <cc+2@test.com>'])
+		user.delete()
+		comm.delete()
+
+	def test_bcc(self):
+		bcc_list = ['bcc+1@test.com', 'cc <bcc+2@test.com>', ]
+		user = self.new_user(email='bcc+2@test.com', enabled=0)
+		comm = self.new_communication(bcc=bcc_list)
+		res = comm.get_mail_bcc_with_displayname()
+		self.assertCountEqual(res, ['bcc+1@test.com'])
+		user.delete()
+		comm.delete()
+
+	def test_sendmail(self):
+		to_list = ['to <to@test.com>']
+		cc_list = ['cc <cc+1@test.com>', 'cc <cc+2@test.com>']
+
+		comm = self.new_communication(recipients=to_list, cc=cc_list)
+		comm.send_email()
+		doc = EmailQueue.find_one_by_filters(communication=comm.name)
+		mail_receivers = [each.recipient for each in doc.recipients]
+		self.assertIsNotNone(doc)
+		self.assertCountEqual(to_list+cc_list, mail_receivers)
+		doc.delete()
+		comm.delete()
+
 def create_email_account():
 	frappe.delete_doc_if_exists("Email Account", "_Test Comm Account 1")
 
@@ -231,4 +295,4 @@ def create_email_account():
 		"enable_automatic_linking": 1
 	}).insert(ignore_permissions=True)
 
-	return email_account
+	return email_account

frappe/core/doctype/communication_link/communication_link.py

@@ -2,7 +2,6 @@
 # Copyright (c) 2019, Frappe Technologies and contributors
 # For license information, please see license.txt
 
-from __future__ import unicode_literals
 import frappe
 from frappe.model.document import Document
 

frappe/core/doctype/custom_docperm/custom_docperm.py

@@ -2,7 +2,6 @@
 # Copyright (c) 2015, Frappe Technologies and contributors
 # For license information, please see license.txt
 
-from __future__ import unicode_literals
 import frappe
 from frappe.model.document import Document
 

frappe/core/doctype/custom_docperm/test_custom_docperm.py

@@ -1,8 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2015, Frappe Technologies and Contributors
 # See license.txt
-from __future__ import unicode_literals
-
 import frappe
 import unittest
 

frappe/core/doctype/custom_role/custom_role.py

@@ -2,7 +2,6 @@
 # Copyright (c) 2015, Frappe Technologies and contributors
 # For license information, please see license.txt
 
-from __future__ import unicode_literals
 import frappe
 from frappe.model.document import Document