Merge branch 'develop' into grid-row-deletion-fix

.eslintrc

@@ -149,6 +149,7 @@
 		"before": true,
 		"beforeEach": true,
 		"qz": true,
-		"localforage": true
+		"localforage": true,
+		"extend_cscript": true
 	}
 }

.flake8

@@ -29,4 +29,5 @@ ignore =
     B950,
     W191,
 
-max-line-length = 200
+max-line-length = 200
+exclude=.github/helper/semgrep_rules

.git-blame-ignore-revs

@@ -0,0 +1,12 @@
+# Since version 2.23 (released in August 2019), git-blame has a feature
+# to ignore or bypass certain commits.
+#
+# This file contains a list of commits that are not likely what you
+# are looking for in a blame, such as mass reformatting or renaming.
+# You can set this file as a default ignore file for blame by running
+# the following command.
+#
+# $ git config blame.ignoreRevsFile .git-blame-ignore-revs
+
+# Replace use of Class.extend with native JS class
+fe20515c23a3ac41f1092bf0eaf0a0a452ec2e85

.github/helper/semgrep_rules/frappe_correctness.py

@@ -4,25 +4,61 @@ from frappe import _, flt
 from frappe.model.document import Document
 
 
+# ruleid: frappe-modifying-but-not-comitting
 def on_submit(self):
 	if self.value_of_goods == 0:
 		frappe.throw(_('Value of goods cannot be 0'))
-	# ruleid: frappe-modifying-after-submit
 	self.status = 'Submitted'
 
-def on_submit(self):   # noqa
-	if flt(self.per_billed) < 100:
-		self.update_billing_status()
-	else:
-		# todook: frappe-modifying-after-submit
-		self.status = "Completed"
-		self.db_set("status", "Completed")
 
-class TestDoc(Document):
-	pass
+# ok: frappe-modifying-but-not-comitting
+def on_submit(self):
+	if self.value_of_goods == 0:
+		frappe.throw(_('Value of goods cannot be 0'))
+	self.status = 'Submitted'
+	self.db_set('status', 'Submitted')
 
-	def validate(self):
-		#ruleid: frappe-modifying-child-tables-while-iterating
-		for item in self.child_table:
-			if item.value < 0:
-				self.remove(item)
+# ok: frappe-modifying-but-not-comitting
+def on_submit(self):
+	if self.value_of_goods == 0:
+		frappe.throw(_('Value of goods cannot be 0'))
+	x = "y"
+	self.status = x
+	self.db_set('status', x)
+
+
+# ok: frappe-modifying-but-not-comitting
+def on_submit(self):
+	x = "y"
+	self.status = x
+	self.save()
+
+# ruleid: frappe-modifying-but-not-comitting-other-method
+class DoctypeClass(Document):
+	def on_submit(self):
+		self.good_method()
+		self.tainted_method()
+
+	def tainted_method(self):
+		self.status = "uptate"
+
+
+# ok: frappe-modifying-but-not-comitting-other-method
+class DoctypeClass(Document):
+	def on_submit(self):
+		self.good_method()
+		self.tainted_method()
+
+	def tainted_method(self):
+		self.status = "update"
+		self.db_set("status", "update")
+
+# ok: frappe-modifying-but-not-comitting-other-method
+class DoctypeClass(Document):
+	def on_submit(self):
+		self.good_method()
+		self.tainted_method()
+		self.save()
+
+	def tainted_method(self):
+		self.status = "uptate"

.github/helper/semgrep_rules/translate.js

@@ -35,3 +35,10 @@ __('You have' + 'subscribers in your mailing list.')
 // ruleid: frappe-translation-js-splitting
 __('You have {0} subscribers' +
     'in your mailing list', [subscribers.length])
+
+// ok: frappe-translation-js-splitting
+__("Ctrl+Enter to add comment")
+
+// ruleid: frappe-translation-js-splitting
+__('You have {0} subscribers \
+    in your mailing list', [subscribers.length])

.github/helper/semgrep_rules/translate.py

@@ -51,3 +51,11 @@ _(f"what" + f"this is also not cool")
 _("")
 # ruleid: frappe-translation-empty-string
 _('')
+
+
+class Test:
+	# ok: frappe-translation-python-splitting
+	def __init__(
+			args
+			):
+		pass

.github/helper/semgrep_rules/translate.yml

@@ -44,8 +44,8 @@ rules:
   pattern-either:
   - pattern: _(...) + _(...)
   - pattern: _("..." + "...")
-  - pattern-regex: '_\([^\)]*\\\s*'    # lines broken by `\`
-  - pattern-regex: '_\(\s*\n'          # line breaks allowed by python for using ( )
+  - pattern-regex: '[\s\.]_\([^\)]*\\\s*'    # lines broken by `\`
+  - pattern-regex: '[\s\.]_\(\s*\n'          # line breaks allowed by python for using ( )
   message: |
     Do not split strings inside translate function. Do not concatenate using translate functions.
     Please refer: https://frappeframework.com/docs/user/en/translations
@@ -54,8 +54,8 @@ rules:
 
 - id: frappe-translation-js-splitting
   pattern-either:
-  - pattern-regex: '__\([^\)]*[\+\\]\s*'
-  - pattern: __('...' + '...')
+  - pattern-regex: '__\([^\)]*[\\]\s+'
+  - pattern: __('...' + '...', ...)
   - pattern: __('...') + __('...')
   message: |
     Do not split strings inside translate function. Do not concatenate using translate functions.

.github/workflows/publish-assets-develop.yml

@@ -15,11 +15,11 @@ jobs:
           path: 'frappe'
       - uses: actions/setup-node@v1
         with:
-          python-version: '12.x'
+          node-version: 14
       - uses: actions/setup-python@v2
         with:
           python-version: '3.6'
-      - name: Set up bench for current push
+      - name: Set up bench and build assets
         run: |
           npm install -g yarn
           pip3 install -U frappe-bench
@@ -29,7 +29,7 @@ jobs:
       - name: Package assets
         run: |
           mkdir -p $GITHUB_WORKSPACE/build
-          tar -cvpzf $GITHUB_WORKSPACE/build/$GITHUB_SHA.tar.gz ./frappe-bench/sites/assets/js ./frappe-bench/sites/assets/css
+          tar -cvpzf $GITHUB_WORKSPACE/build/$GITHUB_SHA.tar.gz ./frappe-bench/sites/assets/frappe/dist
 
       - name: Publish assets to S3
         uses: jakejarvis/s3-sync-action@master

.github/workflows/publish-assets-releases.yml

@@ -22,7 +22,7 @@ jobs:
       - uses: actions/setup-python@v2
         with:
           python-version: '3.6'
-      - name: Set up bench for current push
+      - name: Set up bench and build assets
         run: |
           npm install -g yarn
           pip3 install -U frappe-bench
@@ -32,7 +32,7 @@ jobs:
       - name: Package assets
         run: |
           mkdir -p $GITHUB_WORKSPACE/build
-          tar -cvpzf $GITHUB_WORKSPACE/build/assets.tar.gz ./frappe-bench/sites/assets/js ./frappe-bench/sites/assets/css
+          tar -cvpzf $GITHUB_WORKSPACE/build/assets.tar.gz ./frappe-bench/sites/assets/frappe/dist
 
       - name: Get release
         id: get_release

.github/workflows/semgrep.yml

@@ -4,6 +4,8 @@ on:
   pull_request:
     branches:
       - develop
+      - version-13-hotfix
+      - version-13-pre-release
 jobs:
   semgrep:
     name: Frappe Linter

.github/workflows/server-mariadb-tests.yml

@@ -1,10 +1,10 @@
-name: CI
+name: Server
 
 on:
   pull_request:
-    types: [opened, synchronize, reopened, labeled, unlabeled]
   workflow_dispatch:
   push:
+    branches: [ develop ]
 
 jobs:
   test:
@@ -13,23 +13,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-       include:
-        - DB: "mariadb"
-          TYPE: "server"
-          JOB_NAME: "Python MariaDB"
-          RUN_COMMAND: bench --site test_site run-tests --coverage
+       container: [1, 2]
 
-        - DB: "postgres"
-          TYPE: "server"
-          JOB_NAME: "Python PostgreSQL"
-          RUN_COMMAND: bench --site test_site run-tests --coverage
-
-        - DB: "mariadb"
-          TYPE: "ui"
-          JOB_NAME: "UI MariaDB"
-          RUN_COMMAND:  bench --site test_site run-ui-tests frappe --headless
-
-    name: ${{ matrix.JOB_NAME }}
+    name: Python Unit Tests (MariaDB)
 
     services:
       mysql:
@@ -40,18 +26,6 @@ jobs:
           - 3306:3306
         options: --health-cmd="mysqladmin ping" --health-interval=5s --health-timeout=2s --health-retries=3
 
-      postgres:
-        image: postgres:12.4
-        env:
-          POSTGRES_PASSWORD: travis
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          - 5432:5432
-
     steps:
       - name: Clone
         uses: actions/checkout@v2
@@ -63,7 +37,7 @@ jobs:
 
       - uses: actions/setup-node@v2
         with:
-          node-version: '12'
+          node-version: 14
           check-latest: true
 
       - name: Add to Hosts
@@ -104,68 +78,54 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-yarn-
 
-      - name: Cache cypress binary
-        if: matrix.TYPE == 'ui'
-        uses: actions/cache@v2
-        with:
-          path: ~/.cache
-          key: ${{ runner.os }}-cypress-
-          restore-keys: |
-            ${{ runner.os }}-cypress-
-            ${{ runner.os }}-
-
       - name: Install Dependencies
         run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh
         env:
           BEFORE: ${{ env.GITHUB_EVENT_PATH.before }}
           AFTER: ${{ env.GITHUB_EVENT_PATH.after }}
-          TYPE: ${{ matrix.TYPE }}
+          TYPE: server
 
       - name: Install
         run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh
         env:
-          DB: ${{ matrix.DB }}
-          TYPE: ${{ matrix.TYPE }}
+          DB: mariadb
+          TYPE: server
 
-      - name: Run Set-Up
-        if: matrix.TYPE == 'ui'
-        run: cd ~/frappe-bench/ && bench --site test_site execute frappe.utils.install.complete_setup_wizard
-        env:
-          DB: ${{ matrix.DB }}
-          TYPE: ${{ matrix.TYPE }}
-
-      - name: Setup tmate session
-        if: contains(github.event.pull_request.labels.*.name, 'debug-gha')
-        uses: mxschmitt/action-tmate@v3
 
       - name: Run Tests
-        run: cd ~/frappe-bench/ && ${{ matrix.RUN_COMMAND }}
+        run: cd ~/frappe-bench/ && bench --site test_site run-parallel-tests --use-orchestrator --with-coverage
         env:
-          DB: ${{ matrix.DB }}
-          TYPE: ${{ matrix.TYPE }}
+          CI_BUILD_ID: ${{ github.run_id }}
+          ORCHESTRATOR_URL: http://test-orchestrator.frappe.io
 
-      - name: Coverage - Pull Request
-        if: matrix.TYPE == 'server' && github.event_name == 'pull_request'
+      - name: Upload Coverage Data
         run: |
           cp ~/frappe-bench/sites/.coverage ${GITHUB_WORKSPACE}
           cd ${GITHUB_WORKSPACE}
-          pip install coveralls==2.2.0
-          pip install coverage==4.5.4
-          coveralls --service=github
+          pip3 install coverage==5.5
+          pip3 install coveralls==3.0.1
+          coveralls
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_TOKEN }}
-          COVERALLS_SERVICE_NAME: github
-          
-      - name: Coverage - Push
-        if: matrix.TYPE == 'server' && github.event_name == 'push'
+          COVERALLS_FLAG_NAME: run-${{ matrix.container }}
+          COVERALLS_SERVICE_NAME: ${{ github.event_name == 'pull_request' && 'github' || 'github-actions' }}
+          COVERALLS_PARALLEL: true
+
+  coveralls:
+    name: Coverage Wrap Up
+    needs: test
+    container: python:3-slim
+    runs-on: ubuntu-18.04
+    steps:
+      - name: Clone
+        uses: actions/checkout@v2
+
+      - name: Coveralls Finished
         run: |
-          cp ~/frappe-bench/sites/.coverage ${GITHUB_WORKSPACE}
           cd ${GITHUB_WORKSPACE}
-          pip install coveralls==2.2.0
-          pip install coverage==4.5.4
-          coveralls --service=github-actions
+          pip3 install coverage==5.5
+          pip3 install coveralls==3.0.1
+          coveralls --finish
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_TOKEN }}
-          COVERALLS_SERVICE_NAME: github-actions

.github/workflows/server-postgres-tests.yml

@@ -0,0 +1,100 @@
+name: Server
+
+on:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ubuntu-18.04
+
+    strategy:
+      fail-fast: false
+      matrix:
+       container: [1, 2]
+
+    name: Python Unit Tests (Postgres)
+
+    services:
+      postgres:
+        image: postgres:12.4
+        env:
+          POSTGRES_PASSWORD: travis
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
+    steps:
+      - name: Clone
+        uses: actions/checkout@v2
+
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+
+      - uses: actions/setup-node@v2
+        with:
+          node-version: '14'
+          check-latest: true
+
+      - name: Add to Hosts
+        run: |
+          echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts
+          echo "127.0.0.1 test_site_producer" | sudo tee -a /etc/hosts
+
+      - name: Cache pip
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+            ${{ runner.os }}-
+
+      - name: Cache node modules
+        uses: actions/cache@v2
+        env:
+          cache-name: cache-node-modules
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-build-${{ env.cache-name }}-
+            ${{ runner.os }}-build-
+            ${{ runner.os }}-
+
+      - name: Get yarn cache directory path
+        id: yarn-cache-dir-path
+        run: echo "::set-output name=dir::$(yarn cache dir)"
+
+      - uses: actions/cache@v2
+        id: yarn-cache
+        with:
+          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
+          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-yarn-
+
+      - name: Install Dependencies
+        run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh
+        env:
+          BEFORE: ${{ env.GITHUB_EVENT_PATH.before }}
+          AFTER: ${{ env.GITHUB_EVENT_PATH.after }}
+          TYPE: server
+
+      - name: Install
+        run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh
+        env:
+          DB: postgres
+          TYPE: server
+
+      - name: Run Tests
+        run: cd ~/frappe-bench/ && bench --site test_site run-parallel-tests --use-orchestrator
+        env:
+          CI_BUILD_ID: ${{ github.run_id }}
+          ORCHESTRATOR_URL: http://test-orchestrator.frappe.io

.github/workflows/ui-tests.yml

@@ -0,0 +1,107 @@
+name: UI
+
+on:
+  pull_request:
+  workflow_dispatch:
+  push:
+    branches: [ develop ]
+
+jobs:
+  test:
+    runs-on: ubuntu-18.04
+
+    strategy:
+      fail-fast: false
+      matrix:
+       containers: [1, 2]
+
+    name: UI Tests (Cypress)
+
+    services:
+      mysql:
+        image: mariadb:10.3
+        env:
+          MYSQL_ALLOW_EMPTY_PASSWORD: YES
+        ports:
+          - 3306:3306
+        options: --health-cmd="mysqladmin ping" --health-interval=5s --health-timeout=2s --health-retries=3
+
+    steps:
+      - name: Clone
+        uses: actions/checkout@v2
+
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+
+      - uses: actions/setup-node@v2
+        with:
+          node-version: 14
+          check-latest: true
+
+      - name: Add to Hosts
+        run: |
+          echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts
+          echo "127.0.0.1 test_site_producer" | sudo tee -a /etc/hosts
+
+      - name: Cache pip
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+            ${{ runner.os }}-
+
+      - name: Cache node modules
+        uses: actions/cache@v2
+        env:
+          cache-name: cache-node-modules
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-build-${{ env.cache-name }}-
+            ${{ runner.os }}-build-
+            ${{ runner.os }}-
+
+      - name: Get yarn cache directory path
+        id: yarn-cache-dir-path
+        run: echo "::set-output name=dir::$(yarn cache dir)"
+
+      - uses: actions/cache@v2
+        id: yarn-cache
+        with:
+          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
+          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-yarn-
+
+      - name: Cache cypress binary
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache
+          key: ${{ runner.os }}-cypress-
+          restore-keys: |
+            ${{ runner.os }}-cypress-
+            ${{ runner.os }}-
+
+      - name: Install Dependencies
+        run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh
+        env:
+          BEFORE: ${{ env.GITHUB_EVENT_PATH.before }}
+          AFTER: ${{ env.GITHUB_EVENT_PATH.after }}
+          TYPE: ui
+
+      - name: Install
+        run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh
+        env:
+          DB: mariadb
+          TYPE: ui
+
+      - name: Site Setup
+        run: cd ~/frappe-bench/ && bench --site test_site execute frappe.utils.install.complete_setup_wizard
+
+      - name: UI Tests
+        run: cd ~/frappe-bench/ && bench --site test_site run-ui-tests frappe --headless --parallel --ci-build-id $GITHUB_RUN_ID

.gitignore

@@ -9,6 +9,7 @@ locale
 dist/
 # build/
 frappe/docs/current
+frappe/public/dist
 .vscode
 node_modules
 .kdev4/

.mergify.yml

@@ -3,9 +3,12 @@ pull_request_rules:
     conditions:
       - status-success=Sider
       - status-success=Semantic Pull Request
-      - status-success=Python MariaDB
-      - status-success=Python PostgreSQL
-      - status-success=UI MariaDB
+      - status-success=Python Unit Tests (MariaDB) (1)
+      - status-success=Python Unit Tests (MariaDB) (2)
+      - status-success=Python Unit Tests (Postgres) (1)
+      - status-success=Python Unit Tests (Postgres) (2)
+      - status-success=UI Tests (Cypress) (1)
+      - status-success=UI Tests (Cypress) (2)
       - status-success=security/snyk (frappe)
       - label!=dont-merge
       - label!=squash
@@ -16,9 +19,12 @@ pull_request_rules:
   - name: Automatic squash on CI success and review
     conditions:
       - status-success=Sider
-      - status-success=Python MariaDB
-      - status-success=Python PostgreSQL
-      - status-success=UI MariaDB
+      - status-success=Python Unit Tests (MariaDB) (1)
+      - status-success=Python Unit Tests (MariaDB) (2)
+      - status-success=Python Unit Tests (Postgres) (1)
+      - status-success=Python Unit Tests (Postgres) (2)
+      - status-success=UI Tests (Cypress) (1)
+      - status-success=UI Tests (Cypress) (2)
       - status-success=security/snyk (frappe)
       - label!=dont-merge
       - label=squash

README.md

@@ -14,18 +14,21 @@
 </div>
 
 <div align="center">
-    <a href="https://github.com/frappe/frappe/actions/workflows/ci-tests.yml">
-        <img src="https://github.com/frappe/frappe/actions/workflows/ci-tests.yml/badge.svg?branch=develop">
-    </a>
-    <a href='https://frappeframework.com/docs'>
-        <img src='https://img.shields.io/badge/docs-📖-7575FF.svg?style=flat-square'/>
-    </a>
+	<a href="https://github.com/frappe/frappe/actions/workflows/server-mariadb-tests.yml">
+		<img src="https://github.com/frappe/frappe/actions/workflows/server-mariadb-tests.yml/badge.svg">
+	</a>
+	<a href="https://github.com/frappe/frappe/actions/workflows/ui-tests.yml">
+		<img src="https://github.com/frappe/frappe/actions/workflows/ui-tests.yml/badge.svg?branch=develop">
+	</a>
+	<a href='https://frappeframework.com/docs'>
+		<img src='https://img.shields.io/badge/docs-📖-7575FF.svg?style=flat-square'/>
+	</a>
 	<a href='https://www.codetriage.com/frappe/frappe'>
 		<img src='https://www.codetriage.com/frappe/frappe/badges/users.svg'>
 	</a>
-    <a href='https://coveralls.io/github/frappe/frappe?branch=develop'>
-        <img src='https://coveralls.io/repos/github/frappe/frappe/badge.svg?branch=develop'>
-    </a>
+	<a href='https://coveralls.io/github/frappe/frappe?branch=develop'>
+		<img src='https://coveralls.io/repos/github/frappe/frappe/badge.svg?branch=develop'>
+	</a>
 </div>
 
 

cypress/integration/recorder.js

@@ -50,7 +50,7 @@ context('Recorder', () => {
 		cy.get('.result-list').should('contain', '/api/method/frappe.desk.reportview.get');
 	});
 
-	it.only('Recorder View Request', () => {
+	it('Recorder View Request', () => {
 		cy.get('.primary-action').should('contain', 'Start').click();
 
 		cy.visit('/app/List/DocType/List');

esbuild/esbuild.js

@@ -0,0 +1,481 @@
+/* eslint-disable no-console */
+let path = require("path");
+let fs = require("fs");
+let glob = require("fast-glob");
+let esbuild = require("esbuild");
+let vue = require("esbuild-vue");
+let yargs = require("yargs");
+let cliui = require("cliui")();
+let chalk = require("chalk");
+let html_plugin = require("./frappe-html");
+let postCssPlugin = require("esbuild-plugin-postcss2").default;
+let ignore_assets = require("./ignore-assets");
+let sass_options = require("./sass_options");
+let {
+	app_list,
+	assets_path,
+	apps_path,
+	sites_path,
+	get_app_path,
+	get_public_path,
+	log,
+	log_warn,
+	log_error,
+	bench_path,
+	get_redis_subscriber
+} = require("./utils");
+
+let argv = yargs
+	.usage("Usage: node esbuild [options]")
+	.option("apps", {
+		type: "string",
+		description: "Run build for specific apps"
+	})
+	.option("skip_frappe", {
+		type: "boolean",
+		description: "Skip building frappe assets"
+	})
+	.option("files", {
+		type: "string",
+		description: "Run build for specified bundles"
+	})
+	.option("watch", {
+		type: "boolean",
+		description: "Run in watch mode and rebuild on file changes"
+	})
+	.option("production", {
+		type: "boolean",
+		description: "Run build in production mode"
+	})
+	.option("run-build-command", {
+		type: "boolean",
+		description: "Run build command for apps"
+	})
+	.example(
+		"node esbuild --apps frappe,erpnext",
+		"Run build only for frappe and erpnext"
+	)
+	.example(
+		"node esbuild --files frappe/website.bundle.js,frappe/desk.bundle.js",
+		"Run build only for specified bundles"
+	)
+	.version(false).argv;
+
+const APPS = (!argv.apps ? app_list : argv.apps.split(",")).filter(
+	app => !(argv.skip_frappe && app == "frappe")
+);
+const FILES_TO_BUILD = argv.files ? argv.files.split(",") : [];
+const WATCH_MODE = Boolean(argv.watch);
+const PRODUCTION = Boolean(argv.production);
+const RUN_BUILD_COMMAND = !WATCH_MODE && Boolean(argv["run-build-command"]);
+
+const TOTAL_BUILD_TIME = `${chalk.black.bgGreen(" DONE ")} Total Build Time`;
+const NODE_PATHS = [].concat(
+	// node_modules of apps directly importable
+	app_list
+		.map(app => path.resolve(get_app_path(app), "../node_modules"))
+		.filter(fs.existsSync),
+	// import js file of any app if you provide the full path
+	app_list
+		.map(app => path.resolve(get_app_path(app), ".."))
+		.filter(fs.existsSync)
+);
+
+execute()
+	.then(() => RUN_BUILD_COMMAND && run_build_command_for_apps(APPS))
+	.catch(e => console.error(e));
+
+if (WATCH_MODE) {
+	// listen for open files in editor event
+	open_in_editor();
+}
+
+async function execute() {
+	console.time(TOTAL_BUILD_TIME);
+	if (!FILES_TO_BUILD.length) {
+		await clean_dist_folders(APPS);
+	}
+
+	let result;
+	try {
+		result = await build_assets_for_apps(APPS, FILES_TO_BUILD);
+	} catch (e) {
+		log_error("There were some problems during build");
+		log();
+		log(chalk.dim(e.stack));
+		return;
+	}
+
+	if (!WATCH_MODE) {
+		log_built_assets(result.metafile);
+		console.timeEnd(TOTAL_BUILD_TIME);
+		log();
+	} else {
+		log("Watching for changes...");
+	}
+	return await write_assets_json(result.metafile);
+}
+
+function build_assets_for_apps(apps, files) {
+	let { include_patterns, ignore_patterns } = files.length
+		? get_files_to_build(files)
+		: get_all_files_to_build(apps);
+
+	return glob(include_patterns, { ignore: ignore_patterns }).then(files => {
+		let output_path = assets_path;
+
+		let file_map = {};
+		for (let file of files) {
+			let relative_app_path = path.relative(apps_path, file);
+			let app = relative_app_path.split(path.sep)[0];
+
+			let extension = path.extname(file);
+			let output_name = path.basename(file, extension);
+			if (
+				[".css", ".scss", ".less", ".sass", ".styl"].includes(extension)
+			) {
+				output_name = path.join("css", output_name);
+			} else if ([".js", ".ts"].includes(extension)) {
+				output_name = path.join("js", output_name);
+			}
+			output_name = path.join(app, "dist", output_name);
+
+			if (Object.keys(file_map).includes(output_name)) {
+				log_warn(
+					`Duplicate output file ${output_name} generated from ${file}`
+				);
+			}
+
+			file_map[output_name] = file;
+		}
+
+		return build_files({
+			files: file_map,
+			outdir: output_path
+		});
+	});
+}
+
+function get_all_files_to_build(apps) {
+	let include_patterns = [];
+	let ignore_patterns = [];
+
+	for (let app of apps) {
+		let public_path = get_public_path(app);
+		include_patterns.push(
+			path.resolve(
+				public_path,
+				"**",
+				"*.bundle.{js,ts,css,sass,scss,less,styl}"
+			)
+		);
+		ignore_patterns.push(
+			path.resolve(public_path, "node_modules"),
+			path.resolve(public_path, "dist")
+		);
+	}
+
+	return {
+		include_patterns,
+		ignore_patterns
+	};
+}
+
+function get_files_to_build(files) {
+	// files: ['frappe/website.bundle.js', 'erpnext/main.bundle.js']
+	let include_patterns = [];
+	let ignore_patterns = [];
+
+	for (let file of files) {
+		let [app, bundle] = file.split("/");
+		let public_path = get_public_path(app);
+		include_patterns.push(path.resolve(public_path, "**", bundle));
+		ignore_patterns.push(
+			path.resolve(public_path, "node_modules"),
+			path.resolve(public_path, "dist")
+		);
+	}
+
+	return {
+		include_patterns,
+		ignore_patterns
+	};
+}
+
+function build_files({ files, outdir }) {
+	return esbuild.build({
+		entryPoints: files,
+		entryNames: "[dir]/[name].[hash]",
+		outdir,
+		sourcemap: true,
+		bundle: true,
+		metafile: true,
+		minify: PRODUCTION,
+		nodePaths: NODE_PATHS,
+		define: {
+			"process.env.NODE_ENV": JSON.stringify(
+				PRODUCTION ? "production" : "development"
+			)
+		},
+		plugins: [
+			html_plugin,
+			ignore_assets,
+			vue(),
+			postCssPlugin({
+				plugins: [require("autoprefixer")],
+				sassOptions: sass_options
+			})
+		],
+		watch: get_watch_config()
+	});
+}
+
+function get_watch_config() {
+	if (WATCH_MODE) {
+		return {
+			async onRebuild(error, result) {
+				if (error) {
+					log_error("There was an error during rebuilding changes.");
+					log();
+					log(chalk.dim(error.stack));
+					notify_redis({ error });
+				} else {
+					let {
+						assets_json,
+						prev_assets_json
+					} = await write_assets_json(result.metafile);
+					if (prev_assets_json) {
+						log_rebuilt_assets(prev_assets_json, assets_json);
+					}
+					notify_redis({ success: true });
+				}
+			}
+		};
+	}
+	return null;
+}
+
+async function clean_dist_folders(apps) {
+	for (let app of apps) {
+		let public_path = get_public_path(app);
+		await fs.promises.rmdir(path.resolve(public_path, "dist", "js"), {
+			recursive: true
+		});
+		await fs.promises.rmdir(path.resolve(public_path, "dist", "css"), {
+			recursive: true
+		});
+	}
+}
+
+function log_built_assets(metafile) {
+	let column_widths = [60, 20];
+	cliui.div(
+		{
+			text: chalk.cyan.bold("File"),
+			width: column_widths[0]
+		},
+		{
+			text: chalk.cyan.bold("Size"),
+			width: column_widths[1]
+		}
+	);
+	cliui.div("");
+
+	let output_by_dist_path = {};
+	for (let outfile in metafile.outputs) {
+		if (outfile.endsWith(".map")) continue;
+		let data = metafile.outputs[outfile];
+		outfile = path.resolve(outfile);
+		outfile = path.relative(assets_path, outfile);
+		let filename = path.basename(outfile);
+		let dist_path = outfile.replace(filename, "");
+		output_by_dist_path[dist_path] = output_by_dist_path[dist_path] || [];
+		output_by_dist_path[dist_path].push({
+			name: filename,
+			size: (data.bytes / 1000).toFixed(2) + " Kb"
+		});
+	}
+
+	for (let dist_path in output_by_dist_path) {
+		let files = output_by_dist_path[dist_path];
+		cliui.div({
+			text: dist_path,
+			width: column_widths[0]
+		});
+
+		for (let i in files) {
+			let file = files[i];
+			let branch = "";
+			if (i < files.length - 1) {
+				branch = "├─ ";
+			} else {
+				branch = "└─ ";
+			}
+			let color = file.name.endsWith(".js") ? "green" : "blue";
+			cliui.div(
+				{
+					text: branch + chalk[color]("" + file.name),
+					width: column_widths[0]
+				},
+				{
+					text: file.size,
+					width: column_widths[1]
+				}
+			);
+		}
+		cliui.div("");
+	}
+	log(cliui.toString());
+}
+
+// to store previous build's assets.json for comparison
+let prev_assets_json;
+let curr_assets_json;
+
+async function write_assets_json(metafile) {
+	prev_assets_json = curr_assets_json;
+	let out = {};
+	for (let output in metafile.outputs) {
+		let info = metafile.outputs[output];
+		let asset_path = "/" + path.relative(sites_path, output);
+		if (info.entryPoint) {
+			out[path.basename(info.entryPoint)] = asset_path;
+		}
+	}
+
+	let assets_json_path = path.resolve(assets_path, "assets.json");
+	let assets_json;
+	try {
+		assets_json = await fs.promises.readFile(assets_json_path, "utf-8");
+	} catch (error) {
+		assets_json = "{}";
+	}
+	assets_json = JSON.parse(assets_json);
+	// update with new values
+	assets_json = Object.assign({}, assets_json, out);
+	curr_assets_json = assets_json;
+
+	await fs.promises.writeFile(
+		assets_json_path,
+		JSON.stringify(assets_json, null, 4)
+	);
+	await update_assets_json_in_cache(assets_json);
+	return {
+		assets_json,
+		prev_assets_json
+	};
+}
+
+function update_assets_json_in_cache(assets_json) {
+	// update assets_json cache in redis, so that it can be read directly by python
+	return new Promise(resolve => {
+		let client = get_redis_subscriber("redis_cache");
+		// handle error event to avoid printing stack traces
+		client.on("error", _ => {
+			log_warn("Cannot connect to redis_cache to update assets_json");
+		});
+		client.set("assets_json", JSON.stringify(assets_json), err => {
+			client.unref();
+			resolve();
+		});
+	});
+}
+
+function run_build_command_for_apps(apps) {
+	let cwd = process.cwd();
+	let { execSync } = require("child_process");
+
+	for (let app of apps) {
+		if (app === "frappe") continue;
+
+		let root_app_path = path.resolve(get_app_path(app), "..");
+		let package_json = path.resolve(root_app_path, "package.json");
+		if (fs.existsSync(package_json)) {
+			let { scripts } = require(package_json);
+			if (scripts && scripts.build) {
+				log("\nRunning build command for", chalk.bold(app));
+				process.chdir(root_app_path);
+				execSync("yarn build", { encoding: "utf8", stdio: "inherit" });
+			}
+		}
+	}
+
+	process.chdir(cwd);
+}
+
+async function notify_redis({ error, success }) {
+	// notify redis which in turns tells socketio to publish this to browser
+	let subscriber = get_redis_subscriber("redis_socketio");
+	subscriber.on("error", _ => {
+		log_warn("Cannot connect to redis_socketio for browser events");
+	});
+
+	let payload = null;
+	if (error) {
+		let formatted = await esbuild.formatMessages(error.errors, {
+			kind: "error",
+			terminalWidth: 100
+		});
+		let stack = error.stack.replace(new RegExp(bench_path, "g"), "");
+		payload = {
+			error,
+			formatted,
+			stack
+		};
+	}
+	if (success) {
+		payload = {
+			success: true
+		};
+	}
+
+	subscriber.publish(
+		"events",
+		JSON.stringify({
+			event: "build_event",
+			message: payload
+		})
+	);
+}
+
+function open_in_editor() {
+	let subscriber = get_redis_subscriber("redis_socketio");
+	subscriber.on("error", _ => {
+		log_warn("Cannot connect to redis_socketio for open_in_editor events");
+	});
+	subscriber.on("message", (event, file) => {
+		if (event === "open_in_editor") {
+			file = JSON.parse(file);
+			let file_path = path.resolve(file.file);
+			log("Opening file in editor:", file_path);
+			let launch = require("launch-editor");
+			launch(`${file_path}:${file.line}:${file.column}`);
+		}
+	});
+	subscriber.subscribe("open_in_editor");
+}
+
+function log_rebuilt_assets(prev_assets, new_assets) {
+	let added_files = [];
+	let old_files = Object.values(prev_assets);
+	let new_files = Object.values(new_assets);
+
+	for (let filepath of new_files) {
+		if (!old_files.includes(filepath)) {
+			added_files.push(filepath);
+		}
+	}
+
+	log(
+		chalk.yellow(
+			`${new Date().toLocaleTimeString()}: Compiled ${
+				added_files.length
+			} files...`
+		)
+	);
+	for (let filepath of added_files) {
+		let filename = path.basename(filepath);
+		log("    " + filename);
+	}
+	log();
+}

esbuild/frappe-html.js

@@ -0,0 +1,43 @@
+module.exports = {
+	name: "frappe-html",
+	setup(build) {
+		let path = require("path");
+		let fs = require("fs/promises");
+
+		build.onResolve({ filter: /\.html$/ }, args => {
+			return {
+				path: path.join(args.resolveDir, args.path),
+				namespace: "frappe-html"
+			};
+		});
+
+		build.onLoad({ filter: /.*/, namespace: "frappe-html" }, args => {
+			let filepath = args.path;
+			let filename = path.basename(filepath).split(".")[0];
+
+			return fs
+				.readFile(filepath, "utf-8")
+				.then(content => {
+					content = scrub_html_template(content);
+					return {
+						contents: `\n\tfrappe.templates['${filename}'] = \`${content}\`;\n`
+					};
+				})
+				.catch(() => {
+					return {
+						contents: "",
+						warnings: [
+							{
+								text: `There was an error importing ${filepath}`
+							}
+						]
+					};
+				});
+		});
+	}
+};
+
+function scrub_html_template(content) {
+	content = content.replace(/`/g, "\\`");
+	return content;
+}

esbuild/ignore-assets.js

@@ -0,0 +1,11 @@
+module.exports = {
+	name: "frappe-ignore-asset",
+	setup(build) {
+		build.onResolve({ filter: /^\/assets\// }, args => {
+			return {
+				path: args.path,
+				external: true
+			};
+		});
+	}
+};

esbuild/index.js

@@ -0,0 +1 @@
+require("./esbuild");

esbuild/sass_options.js

@@ -0,0 +1,29 @@
+let path = require("path");
+let { get_app_path, app_list } = require("./utils");
+
+let node_modules_path = path.resolve(
+	get_app_path("frappe"),
+	"..",
+	"node_modules"
+);
+let app_paths = app_list
+	.map(get_app_path)
+	.map(app_path => path.resolve(app_path, ".."));
+
+module.exports = {
+	includePaths: [node_modules_path, ...app_paths],
+	importer: function(url) {
+		if (url.startsWith("~")) {
+			// strip ~ so that it can resolve from node_modules
+			url = url.slice(1);
+		}
+		if (url.endsWith(".css")) {
+			// strip .css from end of path
+			url = url.slice(0, -4);
+		}
+		// normal file, let it go
+		return {
+			file: url
+		};
+	}
+};

esbuild/utils.js

@@ -0,0 +1,145 @@
+const path = require("path");
+const fs = require("fs");
+const chalk = require("chalk");
+
+const frappe_path = path.resolve(__dirname, "..");
+const bench_path = path.resolve(frappe_path, "..", "..");
+const sites_path = path.resolve(bench_path, "sites");
+const apps_path = path.resolve(bench_path, "apps");
+const assets_path = path.resolve(sites_path, "assets");
+const app_list = get_apps_list();
+
+const app_paths = app_list.reduce((out, app) => {
+	out[app] = path.resolve(apps_path, app, app);
+	return out;
+}, {});
+const public_paths = app_list.reduce((out, app) => {
+	out[app] = path.resolve(app_paths[app], "public");
+	return out;
+}, {});
+const public_js_paths = app_list.reduce((out, app) => {
+	out[app] = path.resolve(app_paths[app], "public/js");
+	return out;
+}, {});
+
+const bundle_map = app_list.reduce((out, app) => {
+	const public_js_path = public_js_paths[app];
+	if (fs.existsSync(public_js_path)) {
+		const all_files = fs.readdirSync(public_js_path);
+		const js_files = all_files.filter(file => file.endsWith(".js"));
+
+		for (let js_file of js_files) {
+			const filename = path.basename(js_file).split(".")[0];
+			out[path.join(app, "js", filename)] = path.resolve(
+				public_js_path,
+				js_file
+			);
+		}
+	}
+
+	return out;
+}, {});
+
+const get_public_path = app => public_paths[app];
+
+const get_build_json_path = app =>
+	path.resolve(get_public_path(app), "build.json");
+
+function get_build_json(app) {
+	try {
+		return require(get_build_json_path(app));
+	} catch (e) {
+		// build.json does not exist
+		return null;
+	}
+}
+
+function delete_file(path) {
+	if (fs.existsSync(path)) {
+		fs.unlinkSync(path);
+	}
+}
+
+function run_serially(tasks) {
+	let result = Promise.resolve();
+	tasks.forEach(task => {
+		if (task) {
+			result = result.then ? result.then(task) : Promise.resolve();
+		}
+	});
+	return result;
+}
+
+const get_app_path = app => app_paths[app];
+
+function get_apps_list() {
+	return fs
+		.readFileSync(path.resolve(sites_path, "apps.txt"), {
+			encoding: "utf-8"
+		})
+		.split("\n")
+		.filter(Boolean);
+}
+
+function get_cli_arg(name) {
+	let args = process.argv.slice(2);
+	let arg = `--${name}`;
+	let index = args.indexOf(arg);
+
+	let value = null;
+	if (index != -1) {
+		value = true;
+	}
+	if (value && args[index + 1]) {
+		value = args[index + 1];
+	}
+	return value;
+}
+
+function log_error(message, badge = "ERROR") {
+	badge = chalk.white.bgRed(` ${badge} `);
+	console.error(`${badge} ${message}`); // eslint-disable-line no-console
+}
+
+function log_warn(message, badge = "WARN") {
+	badge = chalk.black.bgYellowBright(` ${badge} `);
+	console.warn(`${badge} ${message}`); // eslint-disable-line no-console
+}
+
+function log(...args) {
+	console.log(...args); // eslint-disable-line no-console
+}
+
+function get_redis_subscriber(kind) {
+	// get redis subscriber that aborts after 10 connection attempts
+	let { get_redis_subscriber: get_redis } = require("../node_utils");
+	return get_redis(kind, {
+		retry_strategy: function(options) {
+			// abort after 10 connection attempts
+			if (options.attempt > 10) {
+				return undefined;
+			}
+			return Math.min(options.attempt * 100, 2000);
+		}
+	});
+}
+
+module.exports = {
+	app_list,
+	bench_path,
+	assets_path,
+	sites_path,
+	apps_path,
+	bundle_map,
+	get_public_path,
+	get_build_json_path,
+	get_build_json,
+	get_app_path,
+	delete_file,
+	run_serially,
+	get_cli_arg,
+	log,
+	log_warn,
+	log_error,
+	get_redis_subscriber
+};

frappe/__init__.py

@@ -10,9 +10,16 @@ be used to build database driven apps.
 
 Read the documentation: https://frappeframework.com/docs
 """
+import os, warnings
+
+_dev_server = os.environ.get('DEV_SERVER', False)
+
+if _dev_server:
+	warnings.simplefilter('always', DeprecationWarning)
+	warnings.simplefilter('always', PendingDeprecationWarning)
 
 from werkzeug.local import Local, release_local
-import os, sys, importlib, inspect, json, warnings
+import sys, importlib, inspect, json
 import typing
 from past.builtins import cmp
 import click
@@ -31,8 +38,6 @@ __title__ = "Frappe Framework"
 
 local = Local()
 controllers = {}
-warnings.simplefilter('always', DeprecationWarning)
-warnings.simplefilter('always', PendingDeprecationWarning)
 
 class _dict(dict):
 	"""dict like object that exposes keys as attributes"""
@@ -197,7 +202,7 @@ def init(site, sites_path=None, new_site=False):
 	local.meta_cache = {}
 	local.form_dict = _dict()
 	local.session = _dict()
-	local.dev_server = os.environ.get('DEV_SERVER', False)
+	local.dev_server = _dev_server
 
 	setup_module_map()
 

frappe/api.py

@@ -11,6 +11,7 @@ import frappe.client
 import frappe.handler
 from frappe import _
 from frappe.utils.response import build_response
+from frappe.utils.data import sbool
 
 
 def handle():
@@ -108,25 +109,40 @@ def handle():
 
 			elif doctype:
 				if frappe.local.request.method == "GET":
-					if frappe.local.form_dict.get('fields'):
-						frappe.local.form_dict['fields'] = json.loads(frappe.local.form_dict['fields'])
-					frappe.local.form_dict.setdefault('limit_page_length', 20)
-					frappe.local.response.update({
-						"data": frappe.call(
-							frappe.client.get_list,
-							doctype,
-							**frappe.local.form_dict
-						)
-					})
+					# set fields for frappe.get_list
+					if frappe.local.form_dict.get("fields"):
+						frappe.local.form_dict["fields"] = json.loads(frappe.local.form_dict["fields"])
+
+					# set limit of records for frappe.get_list
+					frappe.local.form_dict.setdefault(
+						"limit_page_length",
+						frappe.local.form_dict.limit or frappe.local.form_dict.limit_page_length or 20,
+					)
+
+					# convert strings to native types - only as_dict and debug accept bool
+					for param in ["as_dict", "debug"]:
+						param_val = frappe.local.form_dict.get(param)
+						if param_val is not None:
+							frappe.local.form_dict[param] = sbool(param_val)
+
+					# evaluate frappe.get_list
+					data = frappe.call(frappe.client.get_list, doctype, **frappe.local.form_dict)
+
+					# set frappe.get_list result to response
+					frappe.local.response.update({"data": data})
 
 				if frappe.local.request.method == "POST":
+					# fetch data from from dict
 					data = get_request_form_data()
-					data.update({
-						"doctype": doctype
-					})
-					frappe.local.response.update({
-						"data": frappe.get_doc(data).insert().as_dict()
-					})
+					data.update({"doctype": doctype})
+
+					# insert document from request data
+					doc = frappe.get_doc(data).insert()
+
+					# set response data
+					frappe.local.response.update({"data": doc.as_dict()})
+
+					# commit for POST requests
 					frappe.db.commit()
 			else:
 				raise frappe.DoesNotExistError

frappe/app.py

@@ -99,17 +99,7 @@ def application(request):
 		frappe.monitor.stop(response)
 		frappe.recorder.dump()
 
-		if hasattr(frappe.local, 'conf') and frappe.local.conf.enable_frappe_logger:
-			frappe.logger("frappe.web", allow_site=frappe.local.site).info({
-				"site": get_site_name(request.host),
-				"remote_addr": getattr(request, "remote_addr", "NOTFOUND"),
-				"base_url": getattr(request, "base_url", "NOTFOUND"),
-				"full_path": getattr(request, "full_path", "NOTFOUND"),
-				"method": getattr(request, "method", "NOTFOUND"),
-				"scheme": getattr(request, "scheme", "NOTFOUND"),
-				"http_status_code": getattr(response, "status_code", "NOTFOUND")
-			})
-
+		log_request(request, response)
 		process_response(response)
 		frappe.destroy()
 
@@ -137,6 +127,19 @@ def init_request(request):
 	if request.method != "OPTIONS":
 		frappe.local.http_request = frappe.auth.HTTPRequest()
 
+def log_request(request, response):
+	if hasattr(frappe.local, 'conf') and frappe.local.conf.enable_frappe_logger:
+		frappe.logger("frappe.web", allow_site=frappe.local.site).info({
+			"site": get_site_name(request.host),
+			"remote_addr": getattr(request, "remote_addr", "NOTFOUND"),
+			"base_url": getattr(request, "base_url", "NOTFOUND"),
+			"full_path": getattr(request, "full_path", "NOTFOUND"),
+			"method": getattr(request, "method", "NOTFOUND"),
+			"scheme": getattr(request, "scheme", "NOTFOUND"),
+			"http_status_code": getattr(response, "status_code", "NOTFOUND")
+		})
+
+
 def process_response(response):
 	if not response:
 		return

frappe/automation/doctype/auto_repeat/auto_repeat.js

@@ -103,7 +103,7 @@ frappe.ui.form.on('Auto Repeat', {
 frappe.auto_repeat.render_schedule = function(frm) {
 	if (!frm.is_dirty() && frm.doc.status !== 'Disabled') {
 		frm.call("get_auto_repeat_schedule").then(r => {
-			frm.dashboard.wrapper.empty();
+			frm.dashboard.reset();
 			frm.dashboard.add_section(
 				frappe.render_template("auto_repeat_schedule", {
 					schedule_details: r.message || []

frappe/build.py

@@ -1,14 +1,12 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # MIT License. See license.txt
 
-from __future__ import print_function, unicode_literals
-
 import os
 import re
 import json
 import shutil
-import warnings
-import tempfile
+import subprocess
+from tempfile import mkdtemp, mktemp
 from distutils.spawn import find_executable
 
 import frappe
@@ -16,8 +14,9 @@ from frappe.utils.minify import JavascriptMinify
 
 import click
 import psutil
-from six import iteritems, text_type
-from six.moves.urllib.parse import urlparse
+from urllib.parse import urlparse
+from simple_chalk import green
+from semantic_version import Version
 
 
 timestamps = {}
@@ -39,35 +38,36 @@ def download_file(url, prefix):
 
 
 def build_missing_files():
-	# check which files dont exist yet from the build.json and tell build.js to build only those!
+	'''Check which files dont exist yet from the assets.json and run build for those files'''
+
 	missing_assets = []
 	current_asset_files = []
-	frappe_build = os.path.join("..", "apps", "frappe", "frappe", "public", "build.json")
 
 	for type in ["css", "js"]:
-		current_asset_files.extend(
-			[
-				"{0}/{1}".format(type, name)
-				for name in os.listdir(os.path.join(sites_path, "assets", type))
-			]
-		)
+		folder = os.path.join(sites_path, "assets", "frappe", "dist", type)
+		current_asset_files.extend(os.listdir(folder))
 
-	with open(frappe_build) as f:
-		all_asset_files = json.load(f).keys()
+	development = frappe.local.conf.developer_mode or frappe.local.dev_server
+	build_mode = "development" if development else "production"
 
-	for asset in all_asset_files:
-		if asset.replace("concat:", "") not in current_asset_files:
-			missing_assets.append(asset)
+	assets_json = frappe.read_file("assets/assets.json")
+	if assets_json:
+		assets_json = frappe.parse_json(assets_json)
 
-	if missing_assets:
-		from subprocess import check_call
-		from shlex import split
+		for bundle_file, output_file in assets_json.items():
+			if not output_file.startswith('/assets/frappe'):
+				continue
 
-		click.secho("\nBuilding missing assets...\n", fg="yellow")
-		command = split(
-			"node rollup/build.js --files {0} --no-concat".format(",".join(missing_assets))
-		)
-		check_call(command, cwd=os.path.join("..", "apps", "frappe"))
+			if os.path.basename(output_file) not in current_asset_files:
+				missing_assets.append(bundle_file)
+
+		if missing_assets:
+			click.secho("\nBuilding missing assets...\n", fg="yellow")
+			files_to_build = ["frappe/" + name for name in missing_assets]
+			bundle(build_mode, files=files_to_build)
+	else:
+		# no assets.json, run full build
+		bundle(build_mode, apps="frappe")
 
 
 def get_assets_link(frappe_head):
@@ -75,8 +75,8 @@ def get_assets_link(frappe_head):
 	from requests import head
 
 	tag = getoutput(
-			"cd ../apps/frappe && git show-ref --tags -d | grep %s | sed -e 's,.*"
-			" refs/tags/,,' -e 's/\^{}//'"
+			r"cd ../apps/frappe && git show-ref --tags -d | grep %s | sed -e 's,.*"
+			r" refs/tags/,,' -e 's/\^{}//'"
 			% frappe_head
 		)
 
@@ -97,9 +97,7 @@ def download_frappe_assets(verbose=True):
 	commit HEAD.
 	Returns True if correctly setup else returns False.
 	"""
-	from simple_chalk import green
 	from subprocess import getoutput
-	from tempfile import mkdtemp
 
 	assets_setup = False
 	frappe_head = getoutput("cd ../apps/frappe && git rev-parse HEAD")
@@ -166,7 +164,7 @@ def symlink(target, link_name, overwrite=False):
 
 	# Create link to target with temporary filename
 	while True:
-		temp_link_name = tempfile.mktemp(dir=link_dir)
+		temp_link_name = mktemp(dir=link_dir)
 
 		# os.* functions mimic as closely as possible system functions
 		# The POSIX symlink() returns EEXIST if link_name already exists
@@ -193,7 +191,8 @@ def symlink(target, link_name, overwrite=False):
 
 
 def setup():
-	global app_paths
+	global app_paths, assets_path
+
 	pymodules = []
 	for app in frappe.get_all_apps(True):
 		try:
@@ -201,51 +200,54 @@ def setup():
 		except ImportError:
 			pass
 	app_paths = [os.path.dirname(pymodule.__file__) for pymodule in pymodules]
+	assets_path = os.path.join(frappe.local.sites_path, "assets")
 
 
-def get_node_pacman():
-	exec_ = find_executable("yarn")
-	if exec_:
-		return exec_
-	raise ValueError("Yarn not found")
-
-
-def bundle(no_compress, app=None, make_copy=False, restore=False, verbose=False, skip_frappe=False):
+def bundle(mode, apps=None, hard_link=False, make_copy=False, restore=False, verbose=False, skip_frappe=False, files=None):
 	"""concat / minify js files"""
 	setup()
-	make_asset_dirs(make_copy=make_copy, restore=restore)
+	make_asset_dirs(hard_link=hard_link)
 
-	pacman = get_node_pacman()
-	mode = "build" if no_compress else "production"
-	command = "{pacman} run {mode}".format(pacman=pacman, mode=mode)
+	mode = "production" if mode == "production" else "build"
+	command = "yarn run {mode}".format(mode=mode)
 
-	if app:
-		command += " --app {app}".format(app=app)
+	if apps:
+		command += " --apps {apps}".format(apps=apps)
 
 	if skip_frappe:
 		command += " --skip_frappe"
 
-	frappe_app_path = os.path.abspath(os.path.join(app_paths[0], ".."))
-	check_yarn()
+	if files:
+		command += " --files {files}".format(files=','.join(files))
+
+	command += " --run-build-command"
+
+	check_node_executable()
+	frappe_app_path = frappe.get_app_path("frappe", "..")
 	frappe.commands.popen(command, cwd=frappe_app_path, env=get_node_env())
 
 
-def watch(no_compress):
+def watch(apps=None):
 	"""watch and rebuild if necessary"""
 	setup()
 
-	pacman = get_node_pacman()
+	command = "yarn run watch"
+	if apps:
+		command += " --apps {apps}".format(apps=apps)
 
-	frappe_app_path = os.path.abspath(os.path.join(app_paths[0], ".."))
-	check_yarn()
+	check_node_executable()
 	frappe_app_path = frappe.get_app_path("frappe", "..")
-	frappe.commands.popen("{pacman} run watch".format(pacman=pacman),
-		cwd=frappe_app_path, env=get_node_env())
+	frappe.commands.popen(command, cwd=frappe_app_path, env=get_node_env())
 
 
-def check_yarn():
+def check_node_executable():
+	node_version = Version(subprocess.getoutput('node -v')[1:])
+	warn = '⚠️ '
+	if node_version.major < 14:
+		click.echo(f"{warn} Please update your node version to 14")
 	if not find_executable("yarn"):
-		print("Please install yarn using below command and try again.\nnpm install -g yarn")
+		click.echo(f"{warn} Please install yarn using below command and try again.\nnpm install -g yarn")
+	click.echo()
 
 def get_node_env():
 	node_env = {
@@ -266,75 +268,109 @@ def get_safe_max_old_space_size():
 
 	return safe_max_old_space_size
 
-def make_asset_dirs(make_copy=False, restore=False):
-	# don't even think of making assets_path absolute - rm -rf ahead.
-	assets_path = os.path.join(frappe.local.sites_path, "assets")
+def generate_assets_map():
+	symlinks = {}
 
-	for dir_path in [os.path.join(assets_path, "js"), os.path.join(assets_path, "css")]:
-		if not os.path.exists(dir_path):
-			os.makedirs(dir_path)
+	for app_name in frappe.get_all_apps():
+		app_doc_path = None
 
-	for app_name in frappe.get_all_apps(True):
 		pymodule = frappe.get_module(app_name)
 		app_base_path = os.path.abspath(os.path.dirname(pymodule.__file__))
-
-		symlinks = []
 		app_public_path = os.path.join(app_base_path, "public")
-		# app/public > assets/app
-		symlinks.append([app_public_path, os.path.join(assets_path, app_name)])
-		# app/node_modules > assets/app/node_modules
-		if os.path.exists(os.path.abspath(app_public_path)):
-			symlinks.append(
-				[
-					os.path.join(app_base_path, "..", "node_modules"),
-					os.path.join(assets_path, app_name, "node_modules"),
-				]
-			)
+		app_node_modules_path = os.path.join(app_base_path, "..", "node_modules")
+		app_docs_path = os.path.join(app_base_path, "docs")
+		app_www_docs_path = os.path.join(app_base_path, "www", "docs")
 
-		app_doc_path = None
-		if os.path.isdir(os.path.join(app_base_path, "docs")):
+		app_assets = os.path.abspath(app_public_path)
+		app_node_modules = os.path.abspath(app_node_modules_path)
+
+		# {app}/public > assets/{app}
+		if os.path.isdir(app_assets):
+			symlinks[app_assets] = os.path.join(assets_path, app_name)
+
+		# {app}/node_modules > assets/{app}/node_modules
+		if os.path.isdir(app_node_modules):
+			symlinks[app_node_modules] = os.path.join(assets_path, app_name, "node_modules")
+
+		# {app}/docs > assets/{app}_docs
+		if os.path.isdir(app_docs_path):
 			app_doc_path = os.path.join(app_base_path, "docs")
-
-		elif os.path.isdir(os.path.join(app_base_path, "www", "docs")):
+		elif os.path.isdir(app_www_docs_path):
 			app_doc_path = os.path.join(app_base_path, "www", "docs")
-
 		if app_doc_path:
-			symlinks.append([app_doc_path, os.path.join(assets_path, app_name + "_docs")])
+			app_docs = os.path.abspath(app_doc_path)
+			symlinks[app_docs] = os.path.join(assets_path, app_name + "_docs")
 
-		for source, target in symlinks:
-			source = os.path.abspath(source)
-			if os.path.exists(source):
-				if restore:
-					if os.path.exists(target):
-						if os.path.islink(target):
-							os.unlink(target)
-						else:
-							shutil.rmtree(target)
-						shutil.copytree(source, target)
-				elif make_copy:
-					if os.path.exists(target):
-						warnings.warn("Target {target} already exists.".format(target=target))
-					else:
-						shutil.copytree(source, target)
-				else:
-					if os.path.exists(target):
-						if os.path.islink(target):
-							os.unlink(target)
-						else:
-							shutil.rmtree(target)
-					try:
-						symlink(source, target, overwrite=True)
-					except OSError:
-						print("Cannot link {} to {}".format(source, target))
-			else:
-				warnings.warn('Source {source} does not exist.'.format(source = source))
-				pass
+	return symlinks
+
+
+def setup_assets_dirs():
+	for dir_path in (os.path.join(assets_path, x) for x in ("js", "css")):
+		os.makedirs(dir_path, exist_ok=True)
+
+
+def clear_broken_symlinks():
+	for path in os.listdir(assets_path):
+		path = os.path.join(assets_path, path)
+		if os.path.islink(path) and not os.path.exists(path):
+			os.remove(path)
+
+
+
+def unstrip(message: str) -> str:
+	"""Pads input string on the right side until the last available column in the terminal
+	"""
+	_len = len(message)
+	try:
+		max_str = os.get_terminal_size().columns
+	except Exception:
+		max_str = 80
+
+	if _len < max_str:
+		_rem = max_str - _len
+	else:
+		_rem = max_str % _len
+
+	return f"{message}{' ' * _rem}"
+
+
+def make_asset_dirs(hard_link=False):
+	setup_assets_dirs()
+	clear_broken_symlinks()
+	symlinks = generate_assets_map()
+
+	for source, target in symlinks.items():
+		start_message = unstrip(f"{'Copying assets from' if hard_link else 'Linking'} {source} to {target}")
+		fail_message = unstrip(f"Cannot {'copy' if hard_link else 'link'} {source} to {target}")
+
+		# Used '\r' instead of '\x1b[1K\r' to print entire lines in smaller terminal sizes
+		try:
+			print(start_message, end="\r")
+			link_assets_dir(source, target, hard_link=hard_link)
+		except Exception:
+			print(fail_message, end="\r")
+
+	print(unstrip(f"{green('✔')} Application Assets Linked") + "\n")
+
+
+def link_assets_dir(source, target, hard_link=False):
+	if not os.path.exists(source):
+		return
+
+	if os.path.exists(target):
+		if os.path.islink(target):
+			os.unlink(target)
+		else:
+			shutil.rmtree(target)
+
+	if hard_link:
+		shutil.copytree(source, target, dirs_exist_ok=True)
+	else:
+		symlink(source, target, overwrite=True)
 
 
 def build(no_compress=False, verbose=False):
-	assets_path = os.path.join(frappe.local.sites_path, "assets")
-
-	for target, sources in iteritems(get_build_maps()):
+	for target, sources in get_build_maps().items():
 		pack(os.path.join(assets_path, target), sources, no_compress, verbose)
 
 
@@ -348,7 +384,7 @@ def get_build_maps():
 		if os.path.exists(path):
 			with open(path) as f:
 				try:
-					for target, sources in iteritems(json.loads(f.read())):
+					for target, sources in (json.loads(f.read() or "{}")).items():
 						# update app path
 						source_paths = []
 						for source in sources:
@@ -381,7 +417,7 @@ def pack(target, sources, no_compress, verbose):
 		timestamps[f] = os.path.getmtime(f)
 		try:
 			with open(f, "r") as sourcefile:
-				data = text_type(sourcefile.read(), "utf-8", errors="ignore")
+				data = str(sourcefile.read(), "utf-8", errors="ignore")
 
 			extn = f.rsplit(".", 1)[1]
 
@@ -396,7 +432,7 @@ def pack(target, sources, no_compress, verbose):
 				jsm.minify(tmpin, tmpout)
 				minified = tmpout.getvalue()
 				if minified:
-					outtxt += text_type(minified or "", "utf-8").strip("\n") + ";"
+					outtxt += str(minified or "", "utf-8").strip("\n") + ";"
 
 				if verbose:
 					print("{0}: {1}k".format(f, int(len(minified) / 1024)))
@@ -426,16 +462,16 @@ def html_to_js_template(path, content):
 def scrub_html_template(content):
 	"""Returns HTML content with removed whitespace and comments"""
 	# remove whitespace to a single space
-	content = re.sub("\s+", " ", content)
+	content = re.sub(r"\s+", " ", content)
 
 	# strip comments
-	content = re.sub("(<!--.*?-->)", "", content)
+	content = re.sub(r"(<!--.*?-->)", "", content)
 
 	return content.replace("'", "\'")
 
 
 def files_dirty():
-	for target, sources in iteritems(get_build_maps()):
+	for target, sources in get_build_maps().items():
 		for f in sources:
 			if ":" in f:
 				f, suffix = f.split(":")

frappe/cache_manager.py

@@ -13,6 +13,8 @@ common_default_keys = ["__default", "__global"]
 doctype_map_keys = ('energy_point_rule_map', 'assignment_rule_map',
 	'milestone_tracker_map', 'event_consumer_document_type_map')
 
+bench_cache_keys = ('assets_json',)
+
 global_cache_keys = ("app_hooks", "installed_apps", 'all_apps',
 		"app_modules", "module_app", "system_settings",
 		'scheduler_events', 'time_zone', 'webhooks', 'active_domains',
@@ -58,6 +60,7 @@ def clear_global_cache():
 	clear_doctype_cache()
 	clear_website_cache()
 	frappe.cache().delete_value(global_cache_keys)
+	frappe.cache().delete_value(bench_cache_keys)
 	frappe.setup_module_map()
 
 def clear_defaults_cache(user=None):

frappe/change_log/v13/v13_3_0.md

@@ -0,0 +1,49 @@
+# Version 13.3.0 Release Notes
+
+### Features & Enhancements
+
+- Deletion Steps in Data Deletion Tool ([#13124](https://github.com/frappe/frappe/pull/13124))
+- Format Option for list-apps in bench CLI ([#13125](https://github.com/frappe/frappe/pull/13125))
+- Add password fieldtype option for Web Form ([#13093](https://github.com/frappe/frappe/pull/13093))
+- Add simple __repr__ for DocTypes ([#13151](https://github.com/frappe/frappe/pull/13151))
+- Switch theme with left/right keys ([#13077](https://github.com/frappe/frappe/pull/13077))
+- sourceURL for injected javascript ([#13022](https://github.com/frappe/frappe/pull/13022))
+
+### Fixes
+
+- Decode uri before importing file via weblink ([#13026](https://github.com/frappe/frappe/pull/13026))
+- Respond to /api requests as JSON by default ([#13053](https://github.com/frappe/frappe/pull/13053))
+- Disabled checkbox should be disabled ([#13021](https://github.com/frappe/frappe/pull/13021))
+- Moving Site folder across different FileSystems failed ([#13038](https://github.com/frappe/frappe/pull/13038))
+- Freeze screen till the background request is complete ([#13078](https://github.com/frappe/frappe/pull/13078))
+- Added conditional rendering for content field in split section w… ([#13075](https://github.com/frappe/frappe/pull/13075))
+- Show delete button on portal if user has permission to delete document ([#13149](https://github.com/frappe/frappe/pull/13149))
+- Dont disable dialog scroll on focusing a Link/Autocomplete field ([#13119](https://github.com/frappe/frappe/pull/13119))
+- Typo in RecorderDetail.vue ([#13086](https://github.com/frappe/frappe/pull/13086))
+- Error for bench drop-site. Added missing import. ([#13064](https://github.com/frappe/frappe/pull/13064))
+- Report column context ([#13090](https://github.com/frappe/frappe/pull/13090))
+- Different service name for push and pull request events ([#13094](https://github.com/frappe/frappe/pull/13094))
+- Moving Site folder across different FileSystems failed ([#13033](https://github.com/frappe/frappe/pull/13033))
+- Consistent checkboxes on all browsers ([#13042](https://github.com/frappe/frappe/pull/13042))
+- Changed shorcut widgets color picker to dropdown ([#13073](https://github.com/frappe/frappe/pull/13073))
+- Error while exporting reports with duration field ([#13118](https://github.com/frappe/frappe/pull/13118))
+- Add margin to download backup card ([#13079](https://github.com/frappe/frappe/pull/13079))
+- Move mention list generation logic to server-side ([#13074](https://github.com/frappe/frappe/pull/13074))
+- Make strings translatable ([#13046](https://github.com/frappe/frappe/pull/13046))
+- Don't evaluate dynamic properties to check if conflicts exist ([#13186](https://github.com/frappe/frappe/pull/13186))
+- Add __ function in vue global for translation in recorder ([#13089](https://github.com/frappe/frappe/pull/13089))
+- Make strings translatable ([#13076](https://github.com/frappe/frappe/pull/13076))
+- Show config in bench CLI ([#13128](https://github.com/frappe/frappe/pull/13128))
+- Add breadcrumbs for list view ([#13091](https://github.com/frappe/frappe/pull/13091))
+- Do not skip data in save while using shortcut ([#13182](https://github.com/frappe/frappe/pull/13182))
+- Use docfields from options if no docfields are returned from meta ([#13188](https://github.com/frappe/frappe/pull/13188))
+- Disable reloading files in `__pycache__` directory ([#13109](https://github.com/frappe/frappe/pull/13109))
+- RTL stylesheet route to load RTL style on demand. ([#13007](https://github.com/frappe/frappe/pull/13007))
+- Do not show messsage when exception is handled ([#13111](https://github.com/frappe/frappe/pull/13111))
+- Replace parseFloat by Number ([#13082](https://github.com/frappe/frappe/pull/13082))
+- Add margin to download backup card ([#13050](https://github.com/frappe/frappe/pull/13050))
+- Translate report column labels ([#13083](https://github.com/frappe/frappe/pull/13083))
+- Grid row color picker field not working ([#13040](https://github.com/frappe/frappe/pull/13040))
+- Improve oauthlib implementation ([#13045](https://github.com/frappe/frappe/pull/13045))
+- Replace filter_by like with full text filter ([#13126](https://github.com/frappe/frappe/pull/13126))
+- Focus jumps to first field ([#13067](https://github.com/frappe/frappe/pull/13067))

frappe/commands/__init__.py

@@ -28,6 +28,10 @@ def pass_context(f):
 		except frappe.exceptions.SiteNotSpecifiedError as e:
 			click.secho(str(e), fg='yellow')
 			sys.exit(1)
+		except frappe.exceptions.IncorrectSitePath:
+			site = ctx.obj.get("sites", "")[0]
+			click.secho(f'Site {site} does not exist!', fg='yellow')
+			sys.exit(1)
 
 		if profile:
 			pr.disable()

frappe/commands/utils.py

@@ -16,33 +16,52 @@ from frappe.utils import get_bench_path, update_progress_bar, cint
 
 @click.command('build')
 @click.option('--app', help='Build assets for app')
-@click.option('--make-copy', is_flag=True, default=False, help='Copy the files instead of symlinking')
-@click.option('--restore', is_flag=True, default=False, help='Copy the files instead of symlinking with force')
+@click.option('--apps', help='Build assets for specific apps')
+@click.option('--hard-link', is_flag=True, default=False, help='Copy the files instead of symlinking')
+@click.option('--make-copy', is_flag=True, default=False, help='[DEPRECATED] Copy the files instead of symlinking')
+@click.option('--restore', is_flag=True, default=False, help='[DEPRECATED] Copy the files instead of symlinking with force')
+@click.option('--production', is_flag=True, default=False, help='Build assets in production mode')
 @click.option('--verbose', is_flag=True, default=False, help='Verbose')
 @click.option('--force', is_flag=True, default=False, help='Force build assets instead of downloading available')
-def build(app=None, make_copy=False, restore=False, verbose=False, force=False):
-	"Minify + concatenate JS and CSS files, build translations"
-	import frappe.build
+def build(app=None, apps=None, hard_link=False, make_copy=False, restore=False, production=False, verbose=False, force=False):
+	"Compile JS and CSS source files"
+	from frappe.build import bundle, download_frappe_assets
 	frappe.init('')
-	# don't minify in developer_mode for faster builds
-	no_compress = frappe.local.conf.developer_mode or False
+
+	if not apps and app:
+		apps = app
 
 	# dont try downloading assets if force used, app specified or running via CI
-	if not (force or app or os.environ.get('CI')):
+	if not (force or apps or os.environ.get('CI')):
 		# skip building frappe if assets exist remotely
-		skip_frappe = frappe.build.download_frappe_assets(verbose=verbose)
+		skip_frappe = download_frappe_assets(verbose=verbose)
 	else:
 		skip_frappe = False
 
-	frappe.build.bundle(no_compress, app=app, make_copy=make_copy, restore=restore, verbose=verbose, skip_frappe=skip_frappe)
+	# don't minify in developer_mode for faster builds
+	development = frappe.local.conf.developer_mode or frappe.local.dev_server
+	mode = "development" if development else "production"
+	if production:
+		mode = "production"
+
+	if make_copy or restore:
+		hard_link = make_copy or restore
+		click.secho(
+			"bench build: --make-copy and --restore options are deprecated in favour of --hard-link",
+			fg="yellow",
+		)
+
+	bundle(mode, apps=apps, hard_link=hard_link, verbose=verbose, skip_frappe=skip_frappe)
+
 
 
 @click.command('watch')
-def watch():
-	"Watch and concatenate JS and CSS files as and when they change"
-	import frappe.build
+@click.option('--apps', help='Watch assets for specific apps')
+def watch(apps=None):
+	"Watch and compile JS and CSS files as and when they change"
+	from frappe.build import watch
 	frappe.init('')
-	frappe.build.watch(True)
+	watch(apps)
 
 
 @click.command('clear-cache')
@@ -585,12 +604,29 @@ def run_tests(context, app=None, module=None, doctype=None, test=(), profile=Fal
 	if os.environ.get('CI'):
 		sys.exit(ret)
 
+@click.command('run-parallel-tests')
+@click.option('--app', help="For App", default='frappe')
+@click.option('--build-number', help="Build number", default=1)
+@click.option('--total-builds', help="Total number of builds", default=1)
+@click.option('--with-coverage', is_flag=True, help="Build coverage file")
+@click.option('--use-orchestrator', is_flag=True, help="Use orchestrator to run parallel tests")
+@pass_context
+def run_parallel_tests(context, app, build_number, total_builds, with_coverage=False, use_orchestrator=False):
+	site = get_site(context)
+	if use_orchestrator:
+		from frappe.parallel_test_runner import ParallelTestWithOrchestrator
+		ParallelTestWithOrchestrator(app, site=site, with_coverage=with_coverage)
+	else:
+		from frappe.parallel_test_runner import ParallelTestRunner
+		ParallelTestRunner(app, site=site, build_number=build_number, total_builds=total_builds, with_coverage=with_coverage)
 
 @click.command('run-ui-tests')
 @click.argument('app')
 @click.option('--headless', is_flag=True, help="Run UI Test in headless mode")
+@click.option('--parallel', is_flag=True, help="Run UI Test in parallel mode")
+@click.option('--ci-build-id')
 @pass_context
-def run_ui_tests(context, app, headless=False):
+def run_ui_tests(context, app, headless=False, parallel=True, ci_build_id=None):
 	"Run UI tests"
 	site = get_site(context)
 	app_base_path = os.path.abspath(os.path.join(frappe.get_app_path(app), '..'))
@@ -622,6 +658,12 @@ def run_ui_tests(context, app, headless=False):
 	command = '{site_env} {password_env} {cypress} {run_or_open}'
 	formatted_command = command.format(site_env=site_env, password_env=password_env, cypress=cypress_path, run_or_open=run_or_open)
 
+	if parallel:
+		formatted_command += ' --parallel'
+
+	if ci_build_id:
+		formatted_command += ' --ci-build-id {}'.format(ci_build_id)
+
 	click.secho("Running Cypress...", fg="yellow")
 	frappe.commands.popen(formatted_command, cwd=app_base_path, raise_err=True)
 
@@ -797,5 +839,6 @@ commands = [
 	watch,
 	bulk_rename,
 	add_to_email_queue,
-	rebuild_global_search
+	rebuild_global_search,
+	run_parallel_tests
 ]

frappe/contacts/doctype/contact/test_contact.py

@@ -5,7 +5,8 @@ from __future__ import unicode_literals
 
 import frappe
 import unittest
-from frappe.exceptions import ValidationError
+
+test_dependencies = ['Contact', 'Salutation']
 
 class TestContact(unittest.TestCase):
 
@@ -52,4 +53,4 @@ def create_contact(name, salutation, emails=None, phones=None, save=True):
 	if save:
 		doc.insert()
 
-	return doc
+	return doc

frappe/core/doctype/activity_log/test_activity_log.py

@@ -90,4 +90,5 @@ class TestActivityLog(unittest.TestCase):
 def update_system_settings(args):
 	doc = frappe.get_doc('System Settings')
 	doc.update(args)
+	doc.flags.ignore_mandatory = 1
 	doc.save()

frappe/core/doctype/communication/communication.py

@@ -21,9 +21,11 @@ from frappe.automation.doctype.assignment_rule.assignment_rule import apply as a
 exclude_from_linked_with = True
 
 class Communication(Document):
+	"""Communication represents an external communication like Email.
+	"""
 	no_feed_on_delete = True
+	DOCTYPE = 'Communication'
 
-	"""Communication represents an external communication like Email."""
 	def onload(self):
 		"""create email flag queue"""
 		if self.communication_type == "Communication" and self.communication_medium == "Email" \
@@ -149,6 +151,23 @@ class Communication(Document):
 
 			self.email_status = "Spam"
 
+	@classmethod
+	def find(cls, name, ignore_error=False):
+		try:
+			return frappe.get_doc(cls.DOCTYPE, name)
+		except frappe.DoesNotExistError:
+			if ignore_error:
+				return
+			raise
+
+	@classmethod
+	def find_one_by_filters(cls, *, order_by=None, **kwargs):
+		name = frappe.db.get_value(cls.DOCTYPE, kwargs, order_by=order_by)
+		return cls.find(name) if name else None
+
+	def update_db(self, **kwargs):
+		frappe.db.set_value(self.DOCTYPE, self.name, kwargs)
+
 	def set_sender_full_name(self):
 		if not self.sender_full_name and self.sender:
 			if self.sender == "Administrator":
@@ -485,4 +504,4 @@ def set_avg_response_time(parent, communication):
 						response_times.append(response_time)
 			if response_times:
 				avg_response_time = sum(response_times) / len(response_times)
-				parent.db_set("avg_response_time", avg_response_time)
+				parent.db_set("avg_response_time", avg_response_time)

frappe/core/doctype/data_export/exporter.py

@@ -282,7 +282,7 @@ class DataExporter:
 				try:
 					sflags = self.docs_to_export.get("flags", "I,U").upper()
 					flags = 0
-					for a in re.split('\W+',sflags):
+					for a in re.split(r'\W+', sflags):
 						flags = flags | reflags.get(a,0)
 
 					c = re.compile(names, flags)

frappe/core/doctype/data_import/data_import.js

@@ -91,7 +91,7 @@ frappe.ui.form.on('Data Import', {
 
 		if (frm.doc.status.includes('Success')) {
 			frm.add_custom_button(
-				__('Go to {0} List', [frm.doc.reference_doctype]),
+				__('Go to {0} List', [__(frm.doc.reference_doctype)]),
 				() => frappe.set_route('List', frm.doc.reference_doctype)
 			);
 		}
@@ -203,7 +203,7 @@ frappe.ui.form.on('Data Import', {
 	},
 
 	download_template(frm) {
-		frappe.require('/assets/js/data_import_tools.min.js', () => {
+		frappe.require('data_import_tools.bundle.js', () => {
 			frm.data_exporter = new frappe.data_import.DataExporter(
 				frm.doc.reference_doctype,
 				frm.doc.import_type
@@ -287,7 +287,7 @@ frappe.ui.form.on('Data Import', {
 			return;
 		}
 
-		frappe.require('/assets/js/data_import_tools.min.js', () => {
+		frappe.require('data_import_tools.bundle.js', () => {
 			frm.import_preview = new frappe.data_import.ImportPreview({
 				wrapper: frm.get_field('import_preview').$wrapper,
 				doctype: frm.doc.reference_doctype,

frappe/core/doctype/data_import/data_import.py

@@ -211,7 +211,12 @@ def export_json(
 	doctype, path, filters=None, or_filters=None, name=None, order_by="creation asc"
 ):
 	def post_process(out):
-		del_keys = ("modified_by", "creation", "owner", "idx")
+		# Note on Tree DocTypes:
+		# The tree structure is maintained in the database via the fields "lft"
+		# and "rgt". They are automatically set and kept up-to-date. Importing
+		# them would destroy any existing tree structure. For this reason they
+		# are not exported as well.
+		del_keys = ("modified_by", "creation", "owner", "idx", "lft", "rgt")
 		for doc in out:
 			for key in del_keys:
 				if key in doc:

frappe/core/doctype/data_import/importer.py

@@ -641,7 +641,7 @@ class Row:
 				return
 		elif df.fieldtype == "Duration":
 			import re
-			is_valid_duration = re.match("^(?:(\d+d)?((^|\s)\d+h)?((^|\s)\d+m)?((^|\s)\d+s)?)$", value)
+			is_valid_duration = re.match(r"^(?:(\d+d)?((^|\s)\d+h)?((^|\s)\d+m)?((^|\s)\d+s)?)$", value)
 			if not is_valid_duration:
 				self.warnings.append(
 					{
@@ -929,10 +929,7 @@ class Column:
 				self.warnings.append(
 					{
 						"col": self.column_number,
-						"message": _(
-							"Date format could not be determined from the values in"
-							" this column. Defaulting to yyyy-mm-dd."
-						),
+						"message": _("Date format could not be determined from the values in this column. Defaulting to yyyy-mm-dd."),
 						"type": "info",
 					}
 				)

frappe/core/doctype/docshare/test_docshare.py

@@ -7,6 +7,8 @@ import frappe.share
 import unittest
 from frappe.automation.doctype.auto_repeat.test_auto_repeat import create_submittable_doctype
 
+test_dependencies = ['User']
+
 class TestDocShare(unittest.TestCase):
 	def setUp(self):
 		self.user = "test@example.com"
@@ -112,4 +114,4 @@ class TestDocShare(unittest.TestCase):
 		self.assertTrue(frappe.has_permission(doctype, "read", doc=submittable_doc.name, user=self.user))
 		self.assertTrue(frappe.has_permission(doctype, "write", doc=submittable_doc.name, user=self.user))
 
-		frappe.share.remove(doctype, submittable_doc.name, self.user)
+		frappe.share.remove(doctype, submittable_doc.name, self.user)

frappe/core/doctype/doctype/doctype.js

@@ -33,11 +33,11 @@ frappe.ui.form.on('DocType', {
 
 		if (!frm.is_new() && !frm.doc.istable) {
 			if (frm.doc.issingle) {
-				frm.add_custom_button(__('Go to {0}', [frm.doc.name]), () => {
+				frm.add_custom_button(__('Go to {0}', [__(frm.doc.name)]), () => {
 					window.open(`/app/${frappe.router.slug(frm.doc.name)}`);
 				});
 			} else {
-				frm.add_custom_button(__('Go to {0} List', [frm.doc.name]), () => {
+				frm.add_custom_button(__('Go to {0} List', [__(frm.doc.name)]), () => {
 					window.open(`/app/${frappe.router.slug(frm.doc.name)}`);
 				});
 			}

frappe/core/doctype/doctype/doctype.py

@@ -18,6 +18,7 @@ from frappe import _
 from frappe.utils import now, cint
 from frappe.model import no_value_fields, default_fields, data_fieldtypes, table_fields, data_field_options
 from frappe.model.document import Document
+from frappe.model.base_document import get_controller
 from frappe.custom.doctype.property_setter.property_setter import make_property_setter
 from frappe.custom.doctype.custom_field.custom_field import create_custom_field
 from frappe.desk.notifications import delete_notification_count_for
@@ -83,12 +84,62 @@ class DocType(Document):
 		if not self.is_new():
 			self.before_update = frappe.get_doc('DocType', self.name)
 			self.setup_fields_to_fetch()
+			self.validate_field_name_conflicts()
 
 		check_email_append_to(self)
 
 		if self.default_print_format and not self.custom:
 			frappe.throw(_('Standard DocType cannot have default print format, use Customize Form'))
 
+		if frappe.conf.get('developer_mode'):
+			self.owner = 'Administrator'
+			self.modified_by = 'Administrator'
+
+	def validate_field_name_conflicts(self):
+		"""Check if field names dont conflict with controller properties and methods"""
+		core_doctypes = [
+			"Custom DocPerm",
+			"DocPerm",
+			"Custom Field",
+			"Customize Form Field",
+			"DocField",
+		]
+
+		if self.name in core_doctypes:
+			return
+
+		try:
+			controller = get_controller(self.name)
+		except ImportError:
+			controller = Document
+
+		available_objects = {x for x in dir(controller) if isinstance(x, str)}
+		property_set = {
+			x for x in available_objects if isinstance(getattr(controller, x, None), property)
+		}
+		method_set = {
+			x for x in available_objects if x not in property_set and callable(getattr(controller, x, None))
+		}
+
+		for docfield in self.get("fields") or []:
+			if docfield.fieldtype in no_value_fields:
+				continue
+
+			conflict_type = None
+			field = docfield.fieldname
+			field_label = docfield.label or docfield.fieldname
+
+			if docfield.fieldname in method_set:
+				conflict_type = "controller method"
+			if docfield.fieldname in property_set:
+				conflict_type = "class property"
+
+			if conflict_type:
+				frappe.throw(
+					_("Fieldname '{0}' conflicting with a {1} of the name {2} in {3}")
+						.format(field_label, conflict_type, field, self.name)
+				)
+
 	def after_insert(self):
 		# clear user cache so that on the next reload this doctype is included in boot
 		clear_user_cache(frappe.session.user)
@@ -622,12 +673,12 @@ class DocType(Document):
 		flags = {"flags": re.ASCII} if six.PY3 else {}
 
 		# a DocType name should not start or end with an empty space
-		if re.search("^[ \t\n\r]+|[ \t\n\r]+$", name, **flags):
+		if re.search(r"^[ \t\n\r]+|[ \t\n\r]+$", name, **flags):
 			frappe.throw(_("DocType's name should not start or end with whitespace"), frappe.NameError)
 
 		# a DocType's name should not start with a number or underscore
 		# and should only contain letters, numbers and underscore
-		if not re.match("^(?![\W])[^\d_\s][\w ]+$", name, **flags):
+		if not re.match(r"^(?![\W])[^\d_\s][\w ]+$", name, **flags):
 			frappe.throw(_("DocType's name should start with a letter and it can only consist of letters, numbers, spaces and underscores"), frappe.NameError)
 
 		validate_route_conflict(self.doctype, self.name)
@@ -915,7 +966,7 @@ def validate_fields(meta):
 		for field in depends_on_fields:
 			depends_on = docfield.get(field, None)
 			if depends_on and ("=" in depends_on) and \
-				re.match("""[\w\.:_]+\s*={1}\s*[\w\.@'"]+""", depends_on):
+				re.match(r'[\w\.:_]+\s*={1}\s*[\w\.@\'"]+', depends_on):
 				frappe.throw(_("Invalid {0} condition").format(frappe.unscrub(field)), frappe.ValidationError)
 
 	def check_table_multiselect_option(docfield):
@@ -1174,11 +1225,19 @@ def make_module_and_roles(doc, perm_fieldname="permissions"):
 		else:
 			raise
 
-def check_if_fieldname_conflicts_with_methods(doctype, fieldname):
-	doc = frappe.get_doc({"doctype": doctype})
-	method_list = [method for method in dir(doc) if isinstance(method, str) and callable(getattr(doc, method))]
+def check_fieldname_conflicts(doctype, fieldname):
+	"""Checks if fieldname conflicts with methods or properties"""
 
-	if fieldname in method_list:
+	doc = frappe.get_doc({"doctype": doctype})
+	available_objects = [x for x in dir(doc) if isinstance(x, str)]
+	property_list = [
+		x for x in available_objects if isinstance(getattr(type(doc), x, None), property)
+	]
+	method_list = [
+		x for x in available_objects if x not in property_list and callable(getattr(doc, x))
+	]
+
+	if fieldname in method_list + property_list:
 		frappe.throw(_("Fieldname {0} conflicting with meta object").format(fieldname))
 
 def clear_linked_doctype_cache():

frappe/core/doctype/doctype/test_doctype.py

@@ -92,7 +92,7 @@ class TestDocType(unittest.TestCase):
 			fields=["parent", "depends_on", "collapsible_depends_on", "mandatory_depends_on",\
 				"read_only_depends_on", "fieldname", "fieldtype"])
 
-		pattern = """[\w\.:_]+\s*={1}\s*[\w\.@'"]+"""
+		pattern = r'[\w\.:_]+\s*={1}\s*[\w\.@\'"]+'
 		for field in docfields:
 			for depends_on in ["depends_on", "collapsible_depends_on", "mandatory_depends_on", "read_only_depends_on"]:
 				condition = field.get(depends_on)
@@ -517,4 +517,4 @@ def new_doctype(name, unique=0, depends_on='', fields=None):
 		for f in fields:
 			doc.append('fields', f)
 
-	return doc
+	return doc

frappe/core/doctype/document_naming_rule/document_naming_rule.js

@@ -4,6 +4,7 @@
 frappe.ui.form.on('Document Naming Rule', {
 	refresh: function(frm) {
 		frm.trigger('document_type');
+		if (!frm.doc.__islocal) frm.trigger("add_update_counter_button");
 	},
 	document_type: (frm) => {
 		// update the select field options with fieldnames
@@ -20,5 +21,44 @@ frappe.ui.form.on('Document Naming Rule', {
 				);
 			});
 		}
+	},
+	add_update_counter_button: (frm) => {
+		frm.add_custom_button(__('Update Counter'), function() {
+
+			const fields = [{
+				fieldtype: 'Data',
+				fieldname: 'new_counter',
+				label: __('New Counter'),
+				default: frm.doc.counter,
+				reqd: 1,
+				description: __('Warning: Updating counter may lead to document name conflicts if not done properly')
+			}];
+
+			let primary_action_label = __('Save');
+
+			let primary_action = (fields) => {
+				frappe.call({
+					method: 'frappe.core.doctype.document_naming_rule.document_naming_rule.update_current',
+					args: {
+						name: frm.doc.name,
+						new_counter: fields.new_counter
+					},
+					callback: function() {
+						frm.set_value("counter", fields.new_counter);
+						dialog.hide();
+					}
+				});
+			};
+
+			const dialog = new frappe.ui.Dialog({
+				title: __('Update Counter Value for Prefix: {0}', [frm.doc.prefix]),
+				fields,
+				primary_action_label,
+				primary_action
+			});
+
+			dialog.show();
+
+		});
 	}
 });

frappe/core/doctype/document_naming_rule/document_naming_rule.py

@@ -30,3 +30,8 @@ class DocumentNamingRule(Document):
 		counter = frappe.db.get_value(self.doctype, self.name, 'counter', for_update=True) or 0
 		doc.name = self.prefix + ('%0'+str(self.prefix_digits)+'d') % (counter + 1)
 		frappe.db.set_value(self.doctype, self.name, 'counter', counter + 1)
+
+@frappe.whitelist()
+def update_current(name, new_counter):
+	frappe.only_for('System Manager')
+	frappe.db.set_value('Document Naming Rule', name, 'counter', new_counter)

frappe/core/doctype/file/file.py

@@ -498,7 +498,7 @@ class File(Document):
 		self.file_size = self.check_max_file_size()
 
 		if (
-			self.content_type and "image" in self.content_type
+			self.content_type and self.content_type == "image/jpeg"
 			and frappe.get_system_settings("strip_exif_metadata_from_uploaded_images")
 		):
 			self.content = strip_exif_data(self.content, self.content_type)
@@ -912,7 +912,7 @@ def extract_images_from_html(doc, content):
 		return '<img src="{file_url}"'.format(file_url=file_url)
 
 	if content and isinstance(content, string_types):
-		content = re.sub('<img[^>]*src\s*=\s*["\'](?=data:)(.*?)["\']', _save_file, content)
+		content = re.sub(r'<img[^>]*src\s*=\s*["\'](?=data:)(.*?)["\']', _save_file, content)
 
 	return content
 

frappe/core/doctype/file/test_file.py

@@ -193,6 +193,7 @@ class TestSameContent(unittest.TestCase):
 
 class TestFile(unittest.TestCase):
 	def setUp(self):
+		frappe.set_user('Administrator')
 		self.delete_test_data()
 		self.upload_file()
 

frappe/core/doctype/role_profile/test_role_profile.py

@@ -5,6 +5,8 @@ from __future__ import unicode_literals
 import frappe
 import unittest
 
+test_dependencies = ['Role']
+
 class TestRoleProfile(unittest.TestCase):
 	def test_make_new_role_profile(self):
 		new_role_profile = frappe.get_doc(dict(doctype='Role Profile', role_profile='Test 1')).insert()
@@ -21,4 +23,4 @@ class TestRoleProfile(unittest.TestCase):
 		# clear roles
 		new_role_profile.roles = []
 		new_role_profile.save()
-		self.assertEqual(new_role_profile.roles, [])
+		self.assertEqual(new_role_profile.roles, [])

frappe/core/doctype/system_settings/system_settings.py

@@ -42,7 +42,7 @@ class SystemSettings(Document):
 
 	def on_update(self):
 		for df in self.meta.get("fields"):
-			if df.fieldtype not in no_value_fields:
+			if df.fieldtype not in no_value_fields and self.has_value_changed(df.fieldname):
 				frappe.db.set_default(df.fieldname, self.get(df.fieldname))
 
 		if self.language:

frappe/core/page/recorder/recorder.js

@@ -11,7 +11,7 @@ frappe.pages['recorder'].on_page_load = function(wrapper) {
 		frappe.recorder.show();
 	});
 
-	frappe.require('/assets/js/frappe-recorder.min.js');
+	frappe.require('recorder.bundle.js');
 };
 
 class Recorder {

frappe/custom/doctype/custom_field/custom_field.py

@@ -64,8 +64,8 @@ class CustomField(Document):
 			self.translatable = 0
 
 		if not self.flags.ignore_validate:
-			from frappe.core.doctype.doctype.doctype import check_if_fieldname_conflicts_with_methods
-			check_if_fieldname_conflicts_with_methods(self.dt, self.fieldname)
+			from frappe.core.doctype.doctype.doctype import check_fieldname_conflicts
+			check_fieldname_conflicts(self.dt, self.fieldname)
 
 	def on_update(self):
 		if not frappe.flags.in_setup_wizard:

frappe/custom/doctype/customize_form/customize_form.js

@@ -117,7 +117,7 @@ frappe.ui.form.on("Customize Form", {
 			frappe.customize_form.set_primary_action(frm);
 
 			frm.add_custom_button(
-				__("Go to {0} List", [frm.doc.doc_type]),
+				__("Go to {0} List", [__(frm.doc.doc_type)]),
 				function() {
 					frappe.set_route("List", frm.doc.doc_type);
 				},

frappe/desk/desktop.py

@@ -359,15 +359,18 @@ def get_desktop_page(page):
 	Returns:
 		dict: dictionary of cards, charts and shortcuts to be displayed on website
 	"""
-	wspace = Workspace(page)
-	wspace.build_workspace()
-	return {
-		'charts': wspace.charts,
-		'shortcuts': wspace.shortcuts,
-		'cards': wspace.cards,
-		'onboarding': wspace.onboarding,
-		'allow_customization': not wspace.doc.disable_user_customization
-	}
+	try:
+		wspace = Workspace(page)
+		wspace.build_workspace()
+		return {
+			'charts': wspace.charts,
+			'shortcuts': wspace.shortcuts,
+			'cards': wspace.cards,
+			'onboarding': wspace.onboarding,
+			'allow_customization': not wspace.doc.disable_user_customization
+		}
+	except DoesNotExistError:
+		return {}
 
 @frappe.whitelist()
 def get_desk_sidebar_items():
@@ -608,3 +611,4 @@ def merge_cards_based_on_label(cards):
 			cards_dict[label] = card
 
 	return list(cards_dict.values())
+

frappe/desk/doctype/todo/test_todo.py

@@ -9,8 +9,7 @@ from frappe.model.db_query import DatabaseQuery
 from frappe.permissions import add_permission, reset_perms
 from frappe.core.doctype.doctype.doctype import clear_permissions_cache
 
-# test_records = frappe.get_test_records('ToDo')
-test_user_records = frappe.get_test_records('User')
+test_dependencies = ['User']
 
 class TestToDo(unittest.TestCase):
 	def test_delete(self):
@@ -77,7 +76,7 @@ class TestToDo(unittest.TestCase):
 		frappe.set_user('test4@example.com')
 		#owner and assigned_by is test4
 		todo3 = create_new_todo('Test3', 'test4@example.com')
-		
+
 		# user without any role to read or write todo document
 		self.assertFalse(todo1.has_permission("read"))
 		self.assertFalse(todo1.has_permission("write"))

frappe/desk/doctype/workspace_link/workspace_link.json

@@ -8,13 +8,13 @@
   "type",
   "label",
   "icon",
+  "only_for",
   "hidden",
   "link_details_section",
   "link_type",
   "link_to",
   "column_break_7",
   "dependencies",
-  "only_for",
   "onboard",
   "is_query_report"
  ],
@@ -84,7 +84,7 @@
   {
    "fieldname": "only_for",
    "fieldtype": "Link",
-   "label": "Only for ",
+   "label": "Only for",
    "options": "Country"
   },
   {
@@ -104,7 +104,7 @@
  "index_web_pages_for_search": 1,
  "istable": 1,
  "links": [],
- "modified": "2021-01-12 13:13:12.379443",
+ "modified": "2021-05-13 13:10:18.128512",
  "modified_by": "Administrator",
  "module": "Desk",
  "name": "Workspace Link",

frappe/desk/page/activity/activity.js

@@ -67,8 +67,8 @@ frappe.pages['activity'].on_page_show = function () {
 }
 
 frappe.activity.last_feed_date = false;
-frappe.activity.Feed = Class.extend({
-	init: function (row, data) {
+frappe.activity.Feed = class Feed {
+	constructor(row, data) {
 		this.scrub_data(data);
 		this.add_date_separator(row, data);
 		if (!data.add_class)
@@ -97,8 +97,9 @@ frappe.activity.Feed = Class.extend({
 		$(row)
 			.append(frappe.render_template("activity_row", data))
 			.find("a").addClass("grey");
-	},
-	scrub_data: function (data) {
+	}
+
+	scrub_data(data) {
 		data.by = frappe.user.full_name(data.owner);
 		data.avatar = frappe.avatar(data.owner);
 
@@ -113,9 +114,9 @@ frappe.activity.Feed = Class.extend({
 
 		data.when = comment_when(data.creation);
 		data.feed_type = data.comment_type || data.communication_medium;
-	},
+	}
 
-	add_date_separator: function (row, data) {
+	add_date_separator(row, data) {
 		var date = frappe.datetime.str_to_obj(data.creation);
 		var last = frappe.activity.last_feed_date;
 
@@ -137,7 +138,7 @@ frappe.activity.Feed = Class.extend({
 		}
 		frappe.activity.last_feed_date = date;
 	}
-});
+};
 
 frappe.activity.render_heatmap = function (page) {
 	$('<div class="heatmap-container" style="text-align:center">\

frappe/desk/page/user_profile/user_profile.js

@@ -1,6 +1,6 @@
 frappe.pages['user-profile'].on_page_load = function (wrapper) {
-	frappe.require('assets/js/user_profile_controller.min.js', () => {
+	frappe.require('user_profile_controller.bundle.js', () => {
 		let user_profile = new frappe.ui.UserProfile(wrapper);
 		user_profile.show();
 	});
-};
+};

frappe/email/doctype/auto_email_report/auto_email_report.py

@@ -245,6 +245,7 @@ def send_monthly():
 
 def make_links(columns, data):
 	for row in data:
+		doc_name = row.get('name')
 		for col in columns:
 			if col.fieldtype == "Link" and col.options != "Currency":
 				if col.options and row.get(col.fieldname):
@@ -253,8 +254,9 @@ def make_links(columns, data):
 				if col.options and row.get(col.fieldname) and row.get(col.options):
 					row[col.fieldname] = get_link_to_form(row[col.options], row[col.fieldname])
 			elif col.fieldtype == "Currency" and row.get(col.fieldname):
-				row[col.fieldname] = frappe.format_value(row[col.fieldname], col)
-
+				doc = frappe.get_doc(col.parent, doc_name) if doc_name else None
+				# Pass the Document to get the currency based on docfield option
+				row[col.fieldname] = frappe.format_value(row[col.fieldname], col, doc=doc)
 	return columns, data
 
 def update_field_types(columns):
@@ -262,4 +264,4 @@ def update_field_types(columns):
 		if col.fieldtype in  ("Link", "Dynamic Link", "Currency")  and col.options != "Currency":
 			col.fieldtype = "Data"
 			col.options = ""
-	return columns
+	return columns

frappe/email/doctype/email_account/email_account.py

@@ -19,7 +19,7 @@ from frappe.utils import (validate_email_address, cint, cstr, get_datetime,
 from frappe.utils.user import is_system_user
 from frappe.utils.jinja import render_template
 from frappe.email.smtp import SMTPServer
-from frappe.email.receive import EmailServer, Email
+from frappe.email.receive import EmailServer, InboundMail, SentEmailInInboxError
 from poplib import error_proto
 from dateutil.relativedelta import relativedelta
 from datetime import datetime, timedelta
@@ -430,89 +430,76 @@ class EmailAccount(Document):
 
 	def receive(self, test_mails=None):
 		"""Called by scheduler to receive emails from this EMail account using POP3/IMAP."""
-		def get_seen(status):
-			if not status:
-				return None
-			seen = 1 if status == "SEEN" else 0
-			return seen
+		exceptions = []
+		inbound_mails = self.get_inbound_mails(test_mails=test_mails)
+		for mail in inbound_mails:
+			try:
+				communication = mail.process()
+				frappe.db.commit()
+				# If email already exists in the system
+				# then do not send notifications for the same email.
+				if communication and mail.flags.is_new_communication:
+					# notify all participants of this thread
+					if self.enable_auto_reply:
+						self.send_auto_reply(communication, mail)
 
-		if self.enable_incoming:
-			uid_list = []
-			exceptions = []
-			seen_status = []
-			uid_reindexed = False
-			email_server = None
+					attachments = []
+					if hasattr(communication, '_attachments'):
+						attachments = [d.file_name for d in communication._attachments]
+					communication.notify(attachments=attachments, fetched_from_email_account=True)
+			except SentEmailInInboxError:
+				frappe.db.rollback()
+			except Exception:
+				frappe.db.rollback()
+				frappe.log_error('email_account.receive')
+				if self.use_imap:
+					self.handle_bad_emails(mail.uid, mail.raw_message, frappe.get_traceback())
+				exceptions.append(frappe.get_traceback())
 
-			if frappe.local.flags.in_test:
-				incoming_mails = test_mails or []
-			else:
-				email_sync_rule = self.build_email_sync_rule()
+		#notify if user is linked to account
+		if len(inbound_mails)>0 and not frappe.local.flags.in_test:
+			frappe.publish_realtime('new_email',
+				{"account":self.email_account_name, "number":len(inbound_mails)}
+			)
 
-				try:
-					email_server = self.get_incoming_server(in_receive=True, email_sync_rule=email_sync_rule)
-				except Exception:
-					frappe.log_error(title=_("Error while connecting to email account {0}").format(self.name))
+		if exceptions:
+			raise Exception(frappe.as_json(exceptions))
 
-				if not email_server:
-					return
+	def get_inbound_mails(self, test_mails=None):
+		"""retrive and return inbound mails.
 
-				emails = email_server.get_messages()
-				if not emails:
-					return
+		"""
+		if frappe.local.flags.in_test:
+			return [InboundMail(msg, self) for msg in test_mails or []]
 
-				incoming_mails = emails.get("latest_messages", [])
-				uid_list = emails.get("uid_list", [])
-				seen_status = emails.get("seen_status", [])
-				uid_reindexed = emails.get("uid_reindexed", False)
+		if not self.enable_incoming:
+			return []
 
-			for idx, msg in enumerate(incoming_mails):
-				uid = None if not uid_list else uid_list[idx]
-				self.flags.notify = True
+		email_sync_rule = self.build_email_sync_rule()
+		try:
+			email_server = self.get_incoming_server(in_receive=True, email_sync_rule=email_sync_rule)
+			messages = email_server.get_messages() or {}
+		except Exception:
+			raise
+			frappe.log_error(title=_("Error while connecting to email account {0}").format(self.name))
+			return []
 
-				try:
-					args = {
-						"uid": uid,
-						"seen": None if not seen_status else get_seen(seen_status.get(uid, None)),
-						"uid_reindexed": uid_reindexed
-					}
-					communication = self.insert_communication(msg, args=args)
+		mails = []
+		for index, message in enumerate(messages.get("latest_messages", [])):
+			uid = messages['uid_list'][index]
+			seen_status = 1 if messages['seen_status'][uid]=='SEEN' else 0
+			mails.append(InboundMail(message, self, uid, seen_status))
 
-				except SentEmailInInbox:
-					frappe.db.rollback()
+		return mails
 
-				except Exception:
-					frappe.db.rollback()
-					frappe.log_error('email_account.receive')
-					if self.use_imap:
-						self.handle_bad_emails(email_server, uid, msg, frappe.get_traceback())
-					exceptions.append(frappe.get_traceback())
-
-				else:
-					frappe.db.commit()
-					if communication and self.flags.notify:
-
-						# If email already exists in the system
-						# then do not send notifications for the same email.
-
-						attachments = []
-
-						if hasattr(communication, '_attachments'):
-							attachments = [d.file_name for d in communication._attachments]
-
-						communication.notify(attachments=attachments, fetched_from_email_account=True)
-
-			#notify if user is linked to account
-			if len(incoming_mails)>0 and not frappe.local.flags.in_test:
-				frappe.publish_realtime('new_email', {"account":self.email_account_name, "number":len(incoming_mails)})
-
-			if exceptions:
-				raise Exception(frappe.as_json(exceptions))
-
-	def handle_bad_emails(self, email_server, uid, raw, reason):
-		if email_server and cint(email_server.settings.use_imap):
+	def handle_bad_emails(self, uid, raw, reason):
+		if cint(self.use_imap):
 			import email
 			try:
-				mail = email.message_from_string(raw)
+				if isinstance(raw, bytes):
+					mail = email.message_from_bytes(raw)
+				else:
+					mail = email.message_from_string(raw)
 
 				message_id = mail.get('Message-ID')
 			except Exception:
@@ -524,275 +511,18 @@ class EmailAccount(Document):
 				"reason":reason,
 				"message_id": message_id,
 				"doctype": "Unhandled Email",
-				"email_account": email_server.settings.email_account
+				"email_account": self.name
 			})
 			unhandled_email.insert(ignore_permissions=True)
 			frappe.db.commit()
 
-	def insert_communication(self, msg, args=None):
-		if isinstance(msg, list):
-			raw, uid, seen = msg
-		else:
-			raw = msg
-			uid = -1
-			seen = 0
-		if isinstance(args, dict):
-			if args.get("uid", -1): uid = args.get("uid", -1)
-			if args.get("seen", 0): seen = args.get("seen", 0)
-
-		email = Email(raw)
-
-		if email.from_email == self.email_id and not email.mail.get("Reply-To"):
-			# gmail shows sent emails in inbox
-			# and we don't want emails sent by us to be pulled back into the system again
-			# dont count emails sent by the system get those
-			if frappe.flags.in_test:
-				print('WARN: Cannot pull email. Sender sames as recipient inbox')
-			raise SentEmailInInbox
-
-		if email.message_id:
-			# https://stackoverflow.com/a/18367248
-			names = frappe.db.sql("""SELECT DISTINCT `name`, `creation` FROM `tabCommunication`
-				WHERE `message_id`='{message_id}'
-				ORDER BY `creation` DESC LIMIT 1""".format(
-					message_id=email.message_id
-				), as_dict=True)
-
-			if names:
-				name = names[0].get("name")
-				# email is already available update communication uid instead
-				frappe.db.set_value("Communication", name, "uid", uid, update_modified=False)
-
-				self.flags.notify = False
-
-				return frappe.get_doc("Communication", name)
-
-		if email.content_type == 'text/html':
-			email.content = clean_email_html(email.content)
-
-		communication = frappe.get_doc({
-			"doctype": "Communication",
-			"subject": email.subject,
-			"content": email.content,
-			'text_content': email.text_content,
-			"sent_or_received": "Received",
-			"sender_full_name": email.from_real_name,
-			"sender": email.from_email,
-			"recipients": email.mail.get("To"),
-			"cc": email.mail.get("CC"),
-			"email_account": self.name,
-			"communication_medium": "Email",
-			"uid": int(uid or -1),
-			"message_id": email.message_id,
-			"communication_date": email.date,
-			"has_attachment": 1 if email.attachments else 0,
-			"seen": seen or 0
-		})
-
-		self.set_thread(communication, email)
-		if communication.seen:
-			# get email account user and set communication as seen
-			users = frappe.get_all("User Email", filters={ "email_account": self.name },
-				fields=["parent"])
-			users = list(set([ user.get("parent") for user in users ]))
-			communication._seen = json.dumps(users)
-
-		communication.flags.in_receive = True
-		communication.insert(ignore_permissions=True)
-
-		# save attachments
-		communication._attachments = email.save_attachments_in_doc(communication)
-
-		# replace inline images
-		dirty = False
-		for file in communication._attachments:
-			if file.name in email.cid_map and email.cid_map[file.name]:
-				dirty = True
-
-				email.content = email.content.replace("cid:{0}".format(email.cid_map[file.name]),
-					file.file_url)
-
-		if dirty:
-			# not sure if using save() will trigger anything
-			communication.db_set("content", sanitize_html(email.content))
-
-		# notify all participants of this thread
-		if self.enable_auto_reply and getattr(communication, "is_first", False):
-			self.send_auto_reply(communication, email)
-
-		return communication
-
-	def set_thread(self, communication, email):
-		"""Appends communication to parent based on thread ID. Will extract
-		parent communication and will link the communication to the reference of that
-		communication. Also set the status of parent transaction to Open or Replied.
-
-		If no thread id is found and `append_to` is set for the email account,
-		it will create a new parent transaction (e.g. Issue)"""
-		parent = None
-
-		parent = self.find_parent_from_in_reply_to(communication, email)
-
-		if not parent and self.append_to:
-			self.set_sender_field_and_subject_field()
-
-		if not parent and self.append_to:
-			parent = self.find_parent_based_on_subject_and_sender(communication, email)
-
-		if not parent and self.append_to and self.append_to!="Communication":
-			parent = self.create_new_parent(communication, email)
-
-		if parent:
-			communication.reference_doctype = parent.doctype
-			communication.reference_name = parent.name
-
-		# check if message is notification and disable notifications for this message
-		isnotification = email.mail.get("isnotification")
-		if isnotification:
-			if "notification" in isnotification:
-				communication.unread_notification_sent = 1
-
-	def set_sender_field_and_subject_field(self):
-		'''Identify the sender and subject fields from the `append_to` DocType'''
-		# set subject_field and sender_field
-		meta = frappe.get_meta(self.append_to)
-		self.subject_field = None
-		self.sender_field = None
-
-		if hasattr(meta, "subject_field"):
-			self.subject_field = meta.subject_field
-
-		if hasattr(meta, "sender_field"):
-			self.sender_field = meta.sender_field
-
-	def find_parent_based_on_subject_and_sender(self, communication, email):
-		'''Find parent document based on subject and sender match'''
-		parent = None
-
-		if self.append_to and self.sender_field:
-			if self.subject_field:
-				if '#' in email.subject:
-					# try and match if ID is found
-					# document ID is appended to subject
-					# example "Re: Your email (#OPP-2020-2334343)"
-					parent_id = email.subject.rsplit('#', 1)[-1].strip(' ()')
-					if parent_id:
-						parent = frappe.db.get_all(self.append_to, filters = dict(name = parent_id),
-							fields = 'name')
-
-				if not parent:
-					# try and match by subject and sender
-					# if sent by same sender with same subject,
-					# append it to old coversation
-					subject = frappe.as_unicode(strip(re.sub(r"(^\s*(fw|fwd|wg)[^:]*:|\s*(re|aw)[^:]*:\s*)*",
-						"", email.subject, 0, flags=re.IGNORECASE)))
-
-					parent = frappe.db.get_all(self.append_to, filters={
-						self.sender_field: email.from_email,
-						self.subject_field: ("like", "%{0}%".format(subject)),
-						"creation": (">", (get_datetime() - relativedelta(days=60)).strftime(DATE_FORMAT))
-					}, fields = "name", limit = 1)
-
-				if not parent and len(subject) > 10 and is_system_user(email.from_email):
-					# match only subject field
-					# when the from_email is of a user in the system
-					# and subject is atleast 10 chars long
-					parent = frappe.db.get_all(self.append_to, filters={
-						self.subject_field: ("like", "%{0}%".format(subject)),
-						"creation": (">", (get_datetime() - relativedelta(days=60)).strftime(DATE_FORMAT))
-					}, fields = "name", limit = 1)
-
-
-
-			if parent:
-				parent = frappe._dict(doctype=self.append_to, name=parent[0].name)
-				return parent
-
-	def create_new_parent(self, communication, email):
-		'''If no parent found, create a new reference document'''
-
-		# no parent found, but must be tagged
-		# insert parent type doc
-		parent = frappe.new_doc(self.append_to)
-
-		if self.subject_field:
-			parent.set(self.subject_field, frappe.as_unicode(email.subject)[:140])
-
-		if self.sender_field:
-			parent.set(self.sender_field, frappe.as_unicode(email.from_email))
-
-		if parent.meta.has_field("email_account"):
-			parent.email_account = self.name
-
-		parent.flags.ignore_mandatory = True
-
-		try:
-			parent.insert(ignore_permissions=True)
-		except frappe.DuplicateEntryError:
-			# try and find matching parent
-			parent_name = frappe.db.get_value(self.append_to, {self.sender_field: email.from_email})
-			if parent_name:
-				parent.name = parent_name
-			else:
-				parent = None
-
-		# NOTE if parent isn't found and there's no subject match, it is likely that it is a new conversation thread and hence is_first = True
-		communication.is_first = True
-
-		return parent
-
-	def find_parent_from_in_reply_to(self, communication, email):
-		'''Returns parent reference if embedded in In-Reply-To header
-
-		Message-ID is formatted as `{message_id}@{site}`'''
-		parent = None
-		in_reply_to = (email.mail.get("In-Reply-To") or "").strip(" <>")
-
-		if in_reply_to:
-			if "@{0}".format(frappe.local.site) in in_reply_to:
-				# reply to a communication sent from the system
-				email_queue = frappe.db.get_value('Email Queue', dict(message_id=in_reply_to), ['communication','reference_doctype', 'reference_name'])
-				if email_queue:
-					parent_communication, parent_doctype, parent_name = email_queue
-					if parent_communication:
-						communication.in_reply_to = parent_communication
-				else:
-					reference, domain = in_reply_to.split("@", 1)
-					parent_doctype, parent_name = 'Communication', reference
-
-				if frappe.db.exists(parent_doctype, parent_name):
-					parent = frappe._dict(doctype=parent_doctype, name=parent_name)
-
-					# set in_reply_to of current communication
-					if parent_doctype=='Communication':
-						# communication.in_reply_to = email_queue.communication
-
-						if parent.reference_name:
-							# the true parent is the communication parent
-							parent = frappe.get_doc(parent.reference_doctype,
-								parent.reference_name)
-			else:
-				comm = frappe.db.get_value('Communication',
-						dict(
-							message_id=in_reply_to,
-							creation=['>=', add_days(get_datetime(), -30)]),
-						['reference_doctype', 'reference_name'], as_dict=1)
-				if comm:
-					parent = frappe._dict(doctype=comm.reference_doctype, name=comm.reference_name)
-
-		return parent
-
 	def send_auto_reply(self, communication, email):
 		"""Send auto reply if set."""
 		from frappe.core.doctype.communication.email import set_incoming_outgoing_accounts
-
 		if self.enable_auto_reply:
 			set_incoming_outgoing_accounts(communication)
 
-			if self.send_unsubscribe_message:
-				unsubscribe_message = _("Leave this conversation")
-			else:
-				unsubscribe_message = ""
+			unsubscribe_message = (self.send_unsubscribe_message and _("Leave this conversation")) or ""
 
 			frappe.sendmail(recipients = [email.from_email],
 				sender = self.email_id,

frappe/email/doctype/email_account/test_email_account.py

@@ -1,45 +1,56 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # See license.txt
 
-from __future__ import unicode_literals
-import frappe, os
-import unittest, email
+import os
+import email
+import unittest
+from datetime import datetime, timedelta
 
+from frappe.email.receive import InboundMail, SentEmailInInboxError, Email
+from frappe.email.email_body import get_message_id
+import frappe
 from frappe.test_runner import make_test_records
+from frappe.core.doctype.communication.email import make
+from frappe.desk.form.load import get_attachments
+from frappe.email.doctype.email_account.email_account import notify_unreplied
 
 make_test_records("User")
 make_test_records("Email Account")
 
-from frappe.core.doctype.communication.email import make
-from frappe.desk.form.load import get_attachments
-from frappe.email.doctype.email_account.email_account import notify_unreplied
-from datetime import datetime, timedelta
+
 
 class TestEmailAccount(unittest.TestCase):
+	@classmethod
+	def setUpClass(cls):
+		email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
+		email_account.db_set("enable_incoming", 1)
+		email_account.db_set("enable_auto_reply", 1)
+
+	@classmethod
+	def tearDownClass(cls):
+		email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
+		email_account.db_set("enable_incoming", 0)
+
 	def setUp(self):
 		frappe.flags.mute_emails = False
 		frappe.flags.sent_mail = None
-
-		email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
-		email_account.db_set("enable_incoming", 1)
 		frappe.db.sql('delete from `tabEmail Queue`')
+		frappe.db.sql('delete from `tabUnhandled Email`')
 
-	def tearDown(self):
-		email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
-		email_account.db_set("enable_incoming", 0)
+	def get_test_mail(self, fname):
+		with open(os.path.join(os.path.dirname(__file__), "test_mails", fname), "r") as f:
+			return f.read()
 
 	def test_incoming(self):
 		cleanup("test_sender@example.com")
 
-		with open(os.path.join(os.path.dirname(__file__), "test_mails", "incoming-1.raw"), "r") as f:
-			test_mails = [f.read()]
+		test_mails = [self.get_test_mail('incoming-1.raw')]
 
 		email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
 		email_account.receive(test_mails=test_mails)
 
 		comm = frappe.get_doc("Communication", {"sender": "test_sender@example.com"})
 		self.assertTrue("test_receiver@example.com" in comm.recipients)
-
 		# check if todo is created
 		self.assertTrue(frappe.db.get_value(comm.reference_doctype, comm.reference_name, "name"))
 
@@ -88,7 +99,7 @@ class TestEmailAccount(unittest.TestCase):
 		email_account.receive(test_mails=test_mails)
 
 		comm = frappe.get_doc("Communication", {"sender": "test_sender@example.com"})
-		self.assertTrue("From: \"Microsoft Outlook\" <test_sender@example.com>" in comm.content)
+		self.assertTrue("From: "Microsoft Outlook" <test_sender@example.com>" in comm.content)
 		self.assertTrue("This is an e-mail message sent automatically by Microsoft Outlook while" in comm.content)
 
 	def test_incoming_attached_email_from_outlook_layers(self):
@@ -101,7 +112,7 @@ class TestEmailAccount(unittest.TestCase):
 		email_account.receive(test_mails=test_mails)
 
 		comm = frappe.get_doc("Communication", {"sender": "test_sender@example.com"})
-		self.assertTrue("From: \"Microsoft Outlook\" <test_sender@example.com>" in comm.content)
+		self.assertTrue("From: "Microsoft Outlook" <test_sender@example.com>" in comm.content)
 		self.assertTrue("This is an e-mail message sent automatically by Microsoft Outlook while" in comm.content)
 
 	def test_outgoing(self):
@@ -166,7 +177,6 @@ class TestEmailAccount(unittest.TestCase):
 
 		comm_list = frappe.get_all("Communication", filters={"sender":"test_sender@example.com"},
 			fields=["name", "reference_doctype", "reference_name"])
-
 		# both communications attached to the same reference
 		self.assertEqual(comm_list[0].reference_doctype, comm_list[1].reference_doctype)
 		self.assertEqual(comm_list[0].reference_name, comm_list[1].reference_name)
@@ -199,6 +209,215 @@ class TestEmailAccount(unittest.TestCase):
 		self.assertEqual(comm_list[0].reference_doctype, event.doctype)
 		self.assertEqual(comm_list[0].reference_name, event.name)
 
+	def test_auto_reply(self):
+		cleanup("test_sender@example.com")
+
+		test_mails = [self.get_test_mail('incoming-1.raw')]
+
+		email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
+		email_account.receive(test_mails=test_mails)
+
+		comm = frappe.get_doc("Communication", {"sender": "test_sender@example.com"})
+		self.assertTrue(frappe.db.get_value("Email Queue", {"reference_doctype": comm.reference_doctype,
+			"reference_name": comm.reference_name}))
+
+	def test_handle_bad_emails(self):
+		mail_content = self.get_test_mail(fname="incoming-1.raw")
+		message_id = Email(mail_content).mail.get('Message-ID')
+
+		email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
+		email_account.handle_bad_emails(uid=-1, raw=mail_content, reason="Testing")
+		self.assertTrue(frappe.db.get_value("Unhandled Email", {'message_id': message_id}))
+
+class TestInboundMail(unittest.TestCase):
+	@classmethod
+	def setUpClass(cls):
+		email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
+		email_account.db_set("enable_incoming", 1)
+
+	@classmethod
+	def tearDownClass(cls):
+		email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
+		email_account.db_set("enable_incoming", 0)
+
+	def setUp(self):
+		cleanup()
+		frappe.db.sql('delete from `tabEmail Queue`')
+		frappe.db.sql('delete from `tabToDo`')
+
+	def get_test_mail(self, fname):
+		with open(os.path.join(os.path.dirname(__file__), "test_mails", fname), "r") as f:
+			return f.read()
+
+	def new_doc(self, doctype, **data):
+		doc = frappe.new_doc(doctype)
+		for field, value in data.items():
+			setattr(doc, field, value)
+		doc.insert()
+		return doc
+
+	def new_communication(self, **kwargs):
+		defaults = {
+			'subject': "Test Subject"
+		}
+		d = {**defaults, **kwargs}
+		return self.new_doc('Communication', **d)
+
+	def new_email_queue(self, **kwargs):
+		defaults = {
+			'message_id': get_message_id().strip(" <>")
+		}
+		d = {**defaults, **kwargs}
+		return self.new_doc('Email Queue', **d)
+
+	def new_todo(self, **kwargs):
+		defaults = {
+			'description': "Description"
+		}
+		d = {**defaults, **kwargs}
+		return self.new_doc('ToDo', **d)
+
+	def test_self_sent_mail(self):
+		"""Check that we raise SentEmailInInboxError if the inbound mail is self sent mail.
+		"""
+		mail_content = self.get_test_mail(fname="incoming-self-sent.raw")
+		email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
+		inbound_mail = InboundMail(mail_content, email_account, 1, 1)
+		with self.assertRaises(SentEmailInInboxError):
+			inbound_mail.process()
+
+	def test_mail_exist_validation(self):
+		"""Do not create communication record if the mail is already downloaded into the system.
+		"""
+		mail_content = self.get_test_mail(fname="incoming-1.raw")
+		message_id = Email(mail_content).message_id
+		# Create new communication record in DB
+		communication = self.new_communication(message_id=message_id)
+
+		email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
+		inbound_mail = InboundMail(mail_content, email_account, 12345, 1)
+		new_communiction = inbound_mail.process()
+
+		# Make sure that uid is changed to new uid
+		self.assertEqual(new_communiction.uid, 12345)
+		self.assertEqual(communication.name, new_communiction.name)
+
+	def test_find_parent_email_queue(self):
+		"""If the mail is reply to the already sent mail, there will be a email queue record.
+		"""
+		# Create email queue record
+		queue_record = self.new_email_queue()
+
+		mail_content = self.get_test_mail(fname="reply-4.raw").replace(
+			"{{ message_id }}", queue_record.message_id
+		)
+
+		email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
+		inbound_mail = InboundMail(mail_content, email_account, 12345, 1)
+		parent_queue = inbound_mail.parent_email_queue()
+		self.assertEqual(queue_record.name, parent_queue.name)
+
+	def test_find_parent_communication_through_queue(self):
+		"""Find parent communication of an inbound mail.
+		Cases where parent communication does exist:
+		1. No parent communication is the mail is not a reply.
+
+		Cases where parent communication does not exist:
+		2. If mail is not a reply to system sent mail, then there can exist co
+		"""
+		# Create email queue record
+		communication = self.new_communication()
+		queue_record = self.new_email_queue(communication=communication.name)
+		mail_content = self.get_test_mail(fname="reply-4.raw").replace(
+			"{{ message_id }}", queue_record.message_id
+		)
+
+		email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
+		inbound_mail = InboundMail(mail_content, email_account, 12345, 1)
+		parent_communication = inbound_mail.parent_communication()
+		self.assertEqual(parent_communication.name, communication.name)
+
+	def test_find_parent_communication_for_self_reply(self):
+		"""If the inbound email is a reply but not reply to system sent mail.
+
+		Ex: User replied to his/her mail.
+		"""
+		message_id = "new-message-id"
+		mail_content = self.get_test_mail(fname="reply-4.raw").replace(
+			"{{ message_id }}", message_id
+		)
+
+		email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
+		inbound_mail = InboundMail(mail_content, email_account, 12345, 1)
+		parent_communication = inbound_mail.parent_communication()
+		self.assertFalse(parent_communication)
+
+		communication = self.new_communication(message_id=message_id)
+		inbound_mail = InboundMail(mail_content, email_account, 12345, 1)
+		parent_communication = inbound_mail.parent_communication()
+		self.assertEqual(parent_communication.name, communication.name)
+
+	def test_find_parent_communication_from_header(self):
+		"""Incase of header contains parent communication name
+		"""
+		communication = self.new_communication()
+		mail_content = self.get_test_mail(fname="reply-4.raw").replace(
+			"{{ message_id }}", f"<{communication.name}@{frappe.local.site}>"
+		)
+
+		email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
+		inbound_mail = InboundMail(mail_content, email_account, 12345, 1)
+		parent_communication = inbound_mail.parent_communication()
+		self.assertEqual(parent_communication.name, communication.name)
+
+	def test_reference_document(self):
+		# Create email queue record
+		todo = self.new_todo()
+		# communication = self.new_communication(reference_doctype='ToDo', reference_name=todo.name)
+		queue_record = self.new_email_queue(reference_doctype='ToDo', reference_name=todo.name)
+		mail_content = self.get_test_mail(fname="reply-4.raw").replace(
+			"{{ message_id }}", queue_record.message_id
+		)
+
+		email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
+		inbound_mail = InboundMail(mail_content, email_account, 12345, 1)
+		reference_doc = inbound_mail.reference_document()
+		self.assertEqual(todo.name, reference_doc.name)
+
+	def test_reference_document_by_record_name_in_subject(self):
+		# Create email queue record
+		todo = self.new_todo()
+
+		mail_content = self.get_test_mail(fname="incoming-subject-placeholder.raw").replace(
+			"{{ subject }}", f"RE: (#{todo.name})"
+		)
+
+		email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
+		inbound_mail = InboundMail(mail_content, email_account, 12345, 1)
+		reference_doc = inbound_mail.reference_document()
+		self.assertEqual(todo.name, reference_doc.name)
+
+	def test_reference_document_by_subject_match(self):
+		subject = "New todo"
+		todo = self.new_todo(sender='test_sender@example.com', description=subject)
+
+		mail_content = self.get_test_mail(fname="incoming-subject-placeholder.raw").replace(
+			"{{ subject }}", f"RE: {subject}"
+		)
+		email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
+		inbound_mail = InboundMail(mail_content, email_account, 12345, 1)
+		reference_doc = inbound_mail.reference_document()
+		self.assertEqual(todo.name, reference_doc.name)
+
+	def test_create_communication_from_mail(self):
+		# Create email queue record
+		mail_content = self.get_test_mail(fname="incoming-2.raw")
+		email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
+		inbound_mail = InboundMail(mail_content, email_account, 12345, 1)
+		communication = inbound_mail.process()
+		self.assertTrue(communication.is_first)
+		self.assertTrue(communication._attachments)
+
 def cleanup(sender=None):
 	filters = {}
 	if sender:
@@ -207,4 +426,4 @@ def cleanup(sender=None):
 	names = frappe.get_list("Communication", filters=filters, fields=["name"])
 	for name in names:
 		frappe.delete_doc_if_exists("Communication", name.name)
-		frappe.delete_doc_if_exists("Communication Link", {"parent": name.name})
+		frappe.delete_doc_if_exists("Communication Link", {"parent": name.name})

frappe/email/doctype/email_account/test_mails/incoming-self-sent.raw

@@ -0,0 +1,91 @@
+Delivered-To: test_receiver@example.com
+Received: by 10.96.153.227 with SMTP id vj3csp416144qdb;
+        Mon, 15 Sep 2014 03:35:07 -0700 (PDT)
+X-Received: by 10.66.119.103 with SMTP id kt7mr36981968pab.95.1410777306321;
+        Mon, 15 Sep 2014 03:35:06 -0700 (PDT)
+Return-Path: <test@example.com>
+Received: from mail-pa0-x230.google.com (mail-pa0-x230.google.com [2607:f8b0:400e:c03::230])
+        by mx.google.com with ESMTPS id dg10si22178346pdb.115.2014.09.15.03.35.06
+        for <test_receiver@example.com>
+        (version=TLSv1 cipher=ECDHE-RSA-RC4-SHA bits=128/128);
+        Mon, 15 Sep 2014 03:35:06 -0700 (PDT)
+Received-SPF: pass (google.com: domain of test@example.com designates 2607:f8b0:400e:c03::230 as permitted sender) client-ip=2607:f8b0:400e:c03::230;
+Authentication-Results: mx.google.com;
+       spf=pass (google.com: domain of test@example.com designates 2607:f8b0:400e:c03::230 as permitted sender) smtp.mail=test@example.com;
+       dkim=pass header.i=@gmail.com;
+       dmarc=pass (p=NONE dis=NONE) header.from=gmail.com
+Received: by mail-pa0-f48.google.com with SMTP id hz1so6118714pad.21
+        for <test_receiver@example.com>; Mon, 15 Sep 2014 03:35:06 -0700 (PDT)
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;
+        d=gmail.com; s=20120113;
+        h=from:content-type:subject:message-id:date:to:mime-version;
+        bh=rwiLijtF3lfy9M6cP/7dv2Hm7NJuBwFZn1OFsN8Tlvs=;
+        b=x7U4Ny3Kz2ULRJ7a04NDBrBTVhP2ImIB9n3LVNGQDnDonPUM5Ro/wZcxPTVnBWZ2L1
+         o1bGfP+lhBrvYUlHsd5r4FYC0Uvpad6hbzLr0DGUQgPTxW4cGKbtDEAq+BR2JWd9f803
+         vdjSWdGk8w2dt2qbngTqIZkm5U2XWjICDOAYuPIseLUgCFwi9lLyOSARFB7mjAa2YL7Q
+         Nswk7mbWU1hbnHP6jaBb0m8QanTc7Up944HpNDRxIrB1ZHgKzYhXtx8nhnOx588ZGIAe
+         E6tyG8IwogR11vLkkrBhtMaOme9PohYx4F1CSTiwspmDCadEzJFGRe//lEXKmZHAYH6g
+         90Zg==
+X-Received: by 10.70.38.135 with SMTP id g7mr22078275pdk.100.1410777305744;
+        Mon, 15 Sep 2014 03:35:05 -0700 (PDT)
+Return-Path: <test@example.com>
+Received: from [192.168.0.100] ([27.106.4.70])
+        by mx.google.com with ESMTPSA id zr6sm11025126pbc.50.2014.09.15.03.35.02
+        for <test_receiver@example.com>
+        (version=TLSv1 cipher=ECDHE-RSA-RC4-SHA bits=128/128);
+        Mon, 15 Sep 2014 03:35:04 -0700 (PDT)
+From: Rushabh Mehta <test@example.com>
+Content-Type: multipart/alternative; boundary="Apple-Mail=_57F71261-5C3A-43F6-918B-4438B96F61AA"
+Subject: test mail 🦄🌈😎
+Message-Id: <9143999C-8456-4399-9CF1-4A2DA9DD7711@gmail.com>
+Date: Mon, 15 Sep 2014 16:04:57 +0530
+To: Rushabh Mehta <test_receiver@example.com>
+Mime-Version: 1.0 (Mac OS X Mail 7.3 \(1878.6\))
+X-Mailer: Apple Mail (2.1878.6)
+
+
+--Apple-Mail=_57F71261-5C3A-43F6-918B-4438B96F61AA
+Content-Transfer-Encoding: 7bit
+Content-Type: text/plain;
+	charset=us-ascii
+
+test mail
+
+
+
+@rushabh_mehta
+https://erpnext.org
+
+
+--Apple-Mail=_57F71261-5C3A-43F6-918B-4438B96F61AA
+Content-Transfer-Encoding: quoted-printable
+Content-Type: text/html;
+	charset=us-ascii
+
+<html><head><meta http-equiv=3D"Content-Type" content=3D"text/html =
+charset=3Dus-ascii"></head><body style=3D"word-wrap: break-word; =
+-webkit-nbsp-mode: space; -webkit-line-break: after-white-space;">test =
+mail<br><div apple-content-edited=3D"true">
+<div style=3D"color: rgb(0, 0, 0); letter-spacing: normal; orphans: =
+auto; text-align: start; text-indent: 0px; text-transform: none; =
+white-space: normal; widows: auto; word-spacing: 0px; =
+-webkit-text-stroke-width: 0px; word-wrap: break-word; =
+-webkit-nbsp-mode: space; -webkit-line-break: after-white-space;"><div =
+style=3D"color: rgb(0, 0, 0); font-family: Helvetica; font-style: =
+normal; font-variant: normal; font-weight: normal; letter-spacing: =
+normal; line-height: normal; orphans: 2; text-align: -webkit-auto; =
+text-indent: 0px; text-transform: none; white-space: normal; widows: 2; =
+word-spacing: 0px; -webkit-text-stroke-width: 0px; word-wrap: =
+break-word; -webkit-nbsp-mode: space; -webkit-line-break: =
+after-white-space;"><br><br><br>@rushabh_mehta</div><div style=3D"color: =
+rgb(0, 0, 0); font-family: Helvetica; font-style: normal; font-variant: =
+normal; font-weight: normal; letter-spacing: normal; line-height: =
+normal; orphans: 2; text-align: -webkit-auto; text-indent: 0px; =
+text-transform: none; white-space: normal; widows: 2; word-spacing: 0px; =
+-webkit-text-stroke-width: 0px; word-wrap: break-word; =
+-webkit-nbsp-mode: space; -webkit-line-break: after-white-space;"><a =
+href=3D"https://erpnext.org">https://erpnext.org</a><br></div></div>
+</div>
+<br></body></html>=
+
+--Apple-Mail=_57F71261-5C3A-43F6-918B-4438B96F61AA--

frappe/email/doctype/email_account/test_mails/incoming-subject-placeholder.raw

@@ -0,0 +1,183 @@
+Return-path: <test_sender@example.com>
+Envelope-to: test_receiver@example.com
+Delivery-date: Wed, 27 Jan 2016 16:24:20 +0800
+Received: from 23-59-23-10.perm.iinet.net.au ([23.59.23.10]:62191 helo=DESKTOP7C66I2M)
+	by webcloud85.au.syrahost.com with esmtp (Exim 4.86)
+	(envelope-from <test_sender@example.com>)
+	id 1aOLOj-002xFL-CP
+	for test_receiver@example.com; Wed, 27 Jan 2016 16:24:20 +0800
+From: <test_sender@example.com>
+To: <test_receiver@example.com>
+References: <COMM-02154@site1.local>
+In-Reply-To: <COMM-02154@site1.local>
+Subject: RE: {{ subject }}
+Date: Wed, 27 Jan 2016 16:24:09 +0800
+Message-ID: <000001d158dc$1b8363a0$528a2ae0$@example.com>
+MIME-Version: 1.0
+Content-Type: multipart/mixed;
+	boundary="----=_NextPart_000_0001_01D1591F.29A7DC20"
+X-Mailer: Microsoft Outlook 14.0
+Thread-Index: AQJZfZxrgcB9KnMqoZ+S4Qq9hcoSeZ3+vGiQ
+Content-Language: en-au
+
+This is a multipart message in MIME format.
+
+------=_NextPart_000_0001_01D1591F.29A7DC20
+Content-Type: multipart/alternative;
+	boundary="----=_NextPart_001_0002_01D1591F.29A7DC20"
+
+
+------=_NextPart_001_0002_01D1591F.29A7DC20
+Content-Type: text/plain;
+	charset="utf-8"
+Content-Transfer-Encoding: quoted-printable
+
+Test purely for testing with the debugger has email attached
+
+=20
+
+From: Notification [mailto:test_receiver@example.com]=20
+Sent: Wednesday, 27 January 2016 9:30 AM
+To: test_receiver@example.com
+Subject: Sales Invoice: SINV-12276
+
+=20
+
+test no 6 sent from bench to outlook to be replied to with messaging
+
+
+
+
+------=_NextPart_001_0002_01D1591F.29A7DC20
+Content-Type: text/html;
+	charset="utf-8"
+Content-Transfer-Encoding: quoted-printable
+
+<html xmlns:v=3D"urn:schemas-microsoft-com:vml" =
+xmlns:o=3D"urn:schemas-microsoft-com:office:office" =
+xmlns:w=3D"urn:schemas-microsoft-com:office:word" =
+xmlns:m=3D"http://schemas.microsoft.com/office/2004/12/omml" =
+xmlns=3D"http://www.w3.org/TR/REC-html40"><head><meta =
+http-equiv=3DContent-Type content=3D"text/html; charset=3Dutf-8"><meta =
+name=3DGenerator content=3D"Microsoft Word 14 (filtered =
+medium)"><title>hi there</title><style><!--
+/* Font Definitions */
+@font-face
+	{font-family:Helvetica;
+	panose-1:2 11 6 4 2 2 2 2 2 4;}
+@font-face
+	{font-family:"Cambria Math";
+	panose-1:0 0 0 0 0 0 0 0 0 0;}
+@font-face
+	{font-family:Calibri;
+	panose-1:2 15 5 2 2 2 4 3 2 4;}
+@font-face
+	{font-family:Tahoma;
+	panose-1:2 11 6 4 3 5 4 4 2 4;}
+/* Style Definitions */
+p.MsoNormal, li.MsoNormal, div.MsoNormal
+	{margin:0cm;
+	margin-bottom:.0001pt;
+	font-size:12.0pt;
+	font-family:"Times New Roman","serif";}
+a:link, span.MsoHyperlink
+	{mso-style-priority:99;
+	color:blue;
+	text-decoration:underline;}
+a:visited, span.MsoHyperlinkFollowed
+	{mso-style-priority:99;
+	color:purple;
+	text-decoration:underline;}
+p
+	{mso-style-priority:99;
+	mso-margin-top-alt:auto;
+	margin-right:0cm;
+	mso-margin-bottom-alt:auto;
+	margin-left:0cm;
+	font-size:12.0pt;
+	font-family:"Times New Roman","serif";}
+span.EmailStyle18
+	{mso-style-type:personal-reply;
+	font-family:"Calibri","sans-serif";
+	color:#1F497D;}
+.MsoChpDefault
+	{mso-style-type:export-only;
+	font-size:10.0pt;}
+@page WordSection1
+	{size:612.0pt 792.0pt;
+	margin:72.0pt 72.0pt 72.0pt 72.0pt;}
+div.WordSection1
+	{page:WordSection1;}
+--></style><!--[if gte mso 9]><xml>
+<o:shapedefaults v:ext=3D"edit" spidmax=3D"1026" />
+</xml><![endif]--><!--[if gte mso 9]><xml>
+<o:shapelayout v:ext=3D"edit">
+<o:idmap v:ext=3D"edit" data=3D"1" />
+</o:shapelayout></xml><![endif]--></head><body lang=3DEN-AU link=3Dblue =
+vlink=3Dpurple><div class=3DWordSection1><p class=3DMsoNormal><span =
+style=3D'font-size:11.0pt;font-family:"Calibri","sans-serif";color:#1F497=
+D'>Test purely for testing with the debugger has email =
+attached<o:p></o:p></span></p><p class=3DMsoNormal><a =
+name=3D"_MailEndCompose"><span =
+style=3D'font-size:11.0pt;font-family:"Calibri","sans-serif";color:#1F497=
+D'><o:p>&nbsp;</o:p></span></a></p><div><div =
+style=3D'border:none;border-top:solid #B5C4DF 1.0pt;padding:3.0pt 0cm =
+0cm 0cm'><p class=3DMsoNormal><b><span lang=3DEN-US =
+style=3D'font-size:10.0pt;font-family:"Tahoma","sans-serif"'>From:</span>=
+</b><span lang=3DEN-US =
+style=3D'font-size:10.0pt;font-family:"Tahoma","sans-serif"'> =
+Notification [mailto:test_receiver@example.com] <br><b>Sent:</b> Wednesday, 27 =
+January 2016 9:30 AM<br><b>To:</b> =
+test_receiver@example.com<br><b>Subject:</b> Sales Invoice: =
+SINV-12276<o:p></o:p></span></p></div></div><p =
+class=3DMsoNormal><o:p>&nbsp;</o:p></p><div><p><span =
+style=3D'font-size:10.5pt;font-family:"Helvetica","sans-serif";color:#364=
+14C'>test no 3 sent from bench to outlook to be replied to with =
+messaging<o:p></o:p></span></p><p><span =
+style=3D'font-size:10.5pt;font-family:"Helvetica","sans-serif";color:#364=
+14C'>fizz buzz <o:p></o:p></span></p></div><div =
+style=3D'border:none;border-top:solid #D1D8DD 1.0pt;padding:0cm 0cm 0cm =
+0cm;margin-top:22.5pt;margin-bottom:11.25pt'><div =
+style=3D'margin-top:11.25pt;margin-bottom:11.25pt'><p class=3DMsoNormal =
+align=3Dcenter style=3D'text-align:center'><span =
+style=3D'font-size:8.5pt;font-family:"Helvetica","sans-serif";color:#8D99=
+A6'>This email was sent to <a =
+href=3D"mailto:test_receiver@example.com">test_receiver@example.=
+com</a> and copied to SuperUser <o:p></o:p></span></p><p =
+align=3Dcenter =
+style=3D'mso-margin-top-alt:11.25pt;margin-right:0cm;margin-bottom:11.25p=
+t;margin-left:0cm;text-align:center'><span =
+style=3D'font-size:8.5pt;font-family:"Helvetica","sans-serif";color:#8D99=
+A6'><span =
+style=3D'color:#8D99A6'>Leave this conversation =
+</span></a><o:p></o:p></span></p></div><div =
+style=3D'margin-top:11.25pt;margin-bottom:11.25pt'><p class=3DMsoNormal =
+align=3Dcenter style=3D'text-align:center'><span =
+style=3D'font-size:8.5pt;font-family:"Helvetica","sans-serif";color:#8D99=
+A6'>hi<o:p></o:p></span></p></div></div></div></body></html>
+------=_NextPart_001_0002_01D1591F.29A7DC20--
+
+------=_NextPart_000_0001_01D1591F.29A7DC20
+Content-Type: message/rfc822
+Content-Transfer-Encoding: 7bit
+Content-Disposition: attachment
+
+Received: from 203-59-223-10.perm.iinet.net.au ([23.59.23.10]:49772 helo=DESKTOP7C66I2M)
+	by webcloud85.au.syrahost.com with esmtpsa (TLSv1.2:DHE-RSA-AES256-GCM-SHA384:256)
+	(Exim 4.86)
+	(envelope-from <test_sender@example.com>)
+	id 1aOEtO-003tI4-Kv
+	for test_receiver@example.com; Wed, 27 Jan 2016 09:27:30 +0800
+Return-Path: <test_sender@example.com>
+From: "Microsoft Outlook" <test_sender@example.com>
+To: <test_receiver@example.com>
+Subject: Microsoft Outlook Test Message
+MIME-Version: 1.0
+Content-Type: text/plain;
+	charset="utf-8"
+Content-Transfer-Encoding: quoted-printable
+X-Mailer: Microsoft Outlook 14.0
+Thread-Index: AdFYoeN8x8wUI/+QSoCJkp33NKPVmw==
+
+This is an e-mail message sent automatically by Microsoft Outlook while =
+testing the settings for your account.

frappe/email/doctype/email_account/test_records.json

@@ -19,7 +19,8 @@
 		"unreplied_for_mins": 20,
 		"send_notification_to": "test_unreplied@example.com",
 		"pop3_server": "pop.test.example.com",
-		"no_remaining":"0"
+		"no_remaining":"0",
+		"track_email_status": 1
 	},
 	{
 		"doctype": "ToDo",

frappe/email/doctype/email_group/email_group.py

@@ -105,6 +105,6 @@ def send_welcome_email(welcome_email, email, email_group):
 		email=email,
 		email_group=email_group
 	)
-
-	message = frappe.render_template(welcome_email.response, args)
+	email_message = welcome_email.response or welcome_email.response_html
+	message = frappe.render_template(email_message, args)
 	frappe.sendmail(email, subject=welcome_email.subject, message=message)

frappe/email/doctype/email_queue/email_queue.py

@@ -45,6 +45,11 @@ class EmailQueue(Document):
 	def find(cls, name):
 		return frappe.get_doc(cls.DOCTYPE, name)
 
+	@classmethod
+	def find_one_by_filters(cls, **kwargs):
+		name = frappe.db.get_value(cls.DOCTYPE, kwargs)
+		return cls.find(name) if name else None
+
 	def update_db(self, commit=False, **kwargs):
 		frappe.db.set_value(self.DOCTYPE, self.name, kwargs)
 		if commit:
@@ -102,7 +107,7 @@ class EmailQueue(Document):
 
 				message = ctx.build_message(recipient.recipient)
 				if not frappe.flags.in_test:
-					ctx.smtp_session.sendmail(recipient.recipient, self.sender, message)
+					ctx.smtp_session.sendmail(from_addr=self.sender, to_addrs=recipient.recipient, msg=message)
 				ctx.add_to_sent_list(recipient)
 
 			if frappe.flags.in_test:
@@ -218,7 +223,7 @@ class SendMailContext:
 			'<img src="https://{}/api/method/frappe.core.doctype.communication.email.mark_email_as_seen?name={}"/>'
 
 		message = ''
-		if frappe.conf.use_ssl and self.queue_doc.track_email_status:
+		if frappe.conf.use_ssl and self.email_account_doc.track_email_status:
 			message = quopri.encodestring(
 				tracker_url_html.format(frappe.local.site, self.queue_doc.communication).encode()
 			).decode()

frappe/email/doctype/notification/test_notification.py

@@ -7,9 +7,7 @@ import frappe, frappe.utils, frappe.utils.scheduler
 from frappe.desk.form import assign_to
 import unittest
 
-test_records = frappe.get_test_records('Notification')
-
-test_dependencies = ["User"]
+test_dependencies = ["User", "Notification"]
 
 class TestNotification(unittest.TestCase):
 	def setUp(self):

frappe/email/email_body.py

@@ -292,18 +292,12 @@ def inline_style_in_html(html):
 	''' Convert email.css and html to inline-styled html
 	'''
 	from premailer import Premailer
+	from frappe.utils.jinja_globals import bundled_asset
 
-	apps = frappe.get_installed_apps()
-
-	# add frappe email css file
-	css_files = ['assets/css/email.css']
-	if 'frappe' in apps:
-		apps.remove('frappe')
-
-	for app in apps:
-		path = 'assets/{0}/css/email.css'.format(app)
-		css_files.append(path)
-
+	# get email css files from hooks
+	css_files = frappe.get_hooks('email_css')
+	css_files = [bundled_asset(path) for path in css_files]
+	css_files = [path.lstrip('/') for path in css_files]
 	css_files = [css_file for css_file in css_files if os.path.exists(os.path.abspath(css_file))]
 
 	p = Premailer(html=html, external_styles=css_files, strip_important=False)
@@ -359,9 +353,7 @@ def add_attachment(fname, fcontent, content_type=None,
 
 def get_message_id():
 	'''Returns Message ID created from doctype and name'''
-	return "<{unique}@{site}>".format(
-			site=frappe.local.site,
-			unique=email.utils.make_msgid(random_string(10)).split('@')[0].split('<')[1])
+	return email.utils.make_msgid(domain=frappe.local.site)
 
 def get_signature(email_account):
 	if email_account and email_account.add_signature and email_account.signature:

frappe/email/receive.py

@@ -8,6 +8,7 @@ import imaplib
 import poplib
 import re
 import time
+import json
 from email.header import decode_header
 
 import _socket
@@ -20,13 +21,26 @@ from frappe import _, safe_decode, safe_encode
 from frappe.core.doctype.file.file import (MaxFileSizeReachedError,
 	get_random_filename)
 from frappe.utils import (cint, convert_utc_to_user_timezone, cstr,
-	extract_email_id, markdown, now, parse_addr, strip)
+	extract_email_id, markdown, now, parse_addr, strip, get_datetime,
+	add_days, sanitize_html)
+from frappe.utils.user import is_system_user
+from frappe.utils.html_utils import clean_email_html
+
+# fix due to a python bug in poplib that limits it to 2048
+poplib._MAXLINE = 20480
+imaplib._MAXLINE = 20480
+
+# fix due to a python bug in poplib that limits it to 2048
+poplib._MAXLINE = 20480
+imaplib._MAXLINE = 20480
 
 
 class EmailSizeExceededError(frappe.ValidationError): pass
 class EmailTimeoutError(frappe.ValidationError): pass
 class TotalSizeExceededError(frappe.ValidationError): pass
 class LoginLimitExceeded(frappe.ValidationError): pass
+class SentEmailInInboxError(Exception):
+	pass
 
 class EmailServer:
 	"""Wrapper for POP server to pull emails."""
@@ -100,14 +114,11 @@ class EmailServer:
 
 	def get_messages(self):
 		"""Returns new email messages in a list."""
-		if not self.check_mails():
-			return # nothing to do
+		if not (self.check_mails() or self.connect()):
+			return []
 
 		frappe.db.commit()
 
-		if not self.connect():
-			return
-
 		uid_list = []
 
 		try:
@@ -116,7 +127,6 @@ class EmailServer:
 			self.latest_messages = []
 			self.seen_status = {}
 			self.uid_reindexed = False
-
 			uid_list = email_list = self.get_new_mails()
 
 			if not email_list:
@@ -132,11 +142,7 @@ class EmailServer:
 			self.max_email_size = cint(frappe.local.conf.get("max_email_size"))
 			self.max_total_size = 5 * self.max_email_size
 
-			for i, message_meta in enumerate(email_list):
-				# do not pull more than NUM emails
-				if (i+1) > num:
-					break
-
+			for i, message_meta in enumerate(email_list[:num]):
 				try:
 					self.retrieve_message(message_meta, i+1)
 				except (TotalSizeExceededError, EmailTimeoutError, LoginLimitExceeded):
@@ -152,7 +158,6 @@ class EmailServer:
 		except Exception as e:
 			if self.has_login_limit_exceeded(e):
 				pass
-
 			else:
 				raise
 
@@ -284,7 +289,7 @@ class EmailServer:
 
 		flags = []
 		for flag in imaplib.ParseFlags(flag_string) or []:
-			pattern = re.compile("\w+")
+			pattern = re.compile(r"\w+")
 			match = re.search(pattern, frappe.as_unicode(flag))
 			flags.append(match.group(0))
 
@@ -369,6 +374,7 @@ class Email:
 			else:
 				self.mail = email.message_from_string(content)
 
+		self.raw_message = content
 		self.text_content = ''
 		self.html_content = ''
 		self.attachments = []
@@ -391,6 +397,10 @@ class Email:
 		if self.date > now():
 			self.date = now()
 
+	@property
+	def in_reply_to(self):
+		return (self.mail.get("In-Reply-To") or "").strip(" <>")
+
 	def parse(self):
 		"""Walk and process multi-part email."""
 		for part in self.mail.walk():
@@ -555,13 +565,333 @@ class Email:
 
 	def get_thread_id(self):
 		"""Extract thread ID from `[]`"""
-		l = re.findall('(?<=\[)[\w/-]+', self.subject)
+		l = re.findall(r'(?<=\[)[\w/-]+', self.subject)
 		return l and l[0] or None
 
+	def is_reply(self):
+		return bool(self.in_reply_to)
 
-# fix due to a python bug in poplib that limits it to 2048
-poplib._MAXLINE = 20480
-imaplib._MAXLINE = 20480
+class InboundMail(Email):
+	"""Class representation of incoming mail along with mail handlers.
+	"""
+	def __init__(self, content, email_account, uid=None, seen_status=None):
+		super().__init__(content)
+		self.email_account = email_account
+		self.uid = uid or -1
+		self.seen_status = seen_status or 0
+
+		# System documents related to this mail
+		self._parent_email_queue = None
+		self._parent_communication = None
+		self._reference_document = None
+
+		self.flags = frappe._dict()
+
+	def get_content(self):
+		if self.content_type == 'text/html':
+			return clean_email_html(self.content)
+
+	def process(self):
+		"""Create communication record from email.
+		"""
+		if self.is_sender_same_as_receiver() and not self.is_reply():
+			if frappe.flags.in_test:
+				print('WARN: Cannot pull email. Sender same as recipient inbox')
+			raise SentEmailInInboxError
+
+		communication = self.is_exist_in_system()
+		if communication:
+			communication.update_db(uid=self.uid)
+			communication.reload()
+			return communication
+
+		self.flags.is_new_communication = True
+		return self._build_communication_doc()
+
+	def _build_communication_doc(self):
+		data = self.as_dict()
+		data['doctype'] = "Communication"
+
+		if self.parent_communication():
+			data['in_reply_to'] = self.parent_communication().name
+
+		if self.reference_document():
+			data['reference_doctype'] = self.reference_document().doctype
+			data['reference_name'] = self.reference_document().name
+		elif self.email_account.append_to and self.email_account.append_to != 'Communication':
+			reference_doc = self._create_reference_document(self.email_account.append_to)
+			if reference_doc:
+				data['reference_doctype'] = reference_doc.doctype
+				data['reference_name'] = reference_doc.name
+				data['is_first'] = True
+
+		if self.is_notification():
+			# Disable notifications for notification.
+			data['unread_notification_sent'] = 1
+
+		if self.seen_status:
+			data['_seen'] = json.dumps(self.get_users_linked_to_account(self.email_account))
+
+		communication = frappe.get_doc(data)
+		communication.flags.in_receive = True
+		communication.insert(ignore_permissions=True)
+
+		# save attachments
+		communication._attachments = self.save_attachments_in_doc(communication)
+		communication.content = sanitize_html(self.replace_inline_images(communication._attachments))
+		communication.save()
+		return communication
+
+	def replace_inline_images(self, attachments):
+		# replace inline images
+		content = self.content
+		for file in attachments:
+			if file.name in self.cid_map and self.cid_map[file.name]:
+				content = content.replace("cid:{0}".format(self.cid_map[file.name]),
+					file.file_url)
+		return content
+
+	def is_notification(self):
+		isnotification = self.mail.get("isnotification")
+		return isnotification and ("notification" in isnotification)
+
+	def is_exist_in_system(self):
+		"""Check if this email already exists in the system(as communication document).
+		"""
+		from frappe.core.doctype.communication.communication import Communication
+		if not self.message_id:
+			return
+
+		return Communication.find_one_by_filters(message_id = self.message_id,
+			order_by = 'creation DESC')
+
+	def is_sender_same_as_receiver(self):
+		return self.from_email == self.email_account.email_id
+
+	def is_reply_to_system_sent_mail(self):
+		"""Is it a reply to already sent mail.
+		"""
+		return self.is_reply() and frappe.local.site in self.in_reply_to
+
+	def parent_email_queue(self):
+		"""Get parent record from `Email Queue`.
+
+		If it is a reply to already sent mail, then there will be a parent record in EMail Queue.
+		"""
+		from frappe.email.doctype.email_queue.email_queue import EmailQueue
+
+		if self._parent_email_queue is not None:
+			return self._parent_email_queue
+
+		parent_email_queue = ''
+		if self.is_reply_to_system_sent_mail():
+			parent_email_queue = EmailQueue.find_one_by_filters(message_id=self.in_reply_to)
+
+		self._parent_email_queue = parent_email_queue or ''
+		return self._parent_email_queue
+
+	def parent_communication(self):
+		"""Find a related communication so that we can prepare a mail thread.
+
+		The way it happens is by using in-reply-to header, and we can't make thread if it does not exist.
+
+		Here are the cases to handle:
+		1. If mail is a reply to already sent mail, then we can get parent communicaion from
+			Email Queue record.
+		2. Sometimes we send communication name in message-ID directly, use that to get parent communication.
+		3. Sender sent a reply but reply is on top of what (s)he sent before,
+			then parent record exists directly in communication.
+		"""
+		from frappe.core.doctype.communication.communication import Communication
+		if self._parent_communication is not None:
+			return self._parent_communication
+
+		if not self.is_reply():
+			return ''
+
+		if not self.is_reply_to_system_sent_mail():
+			communication = Communication.find_one_by_filters(message_id=self.in_reply_to,
+				creation = ['>=', self.get_relative_dt(-30)])
+		elif self.parent_email_queue() and self.parent_email_queue().communication:
+			communication = Communication.find(self.parent_email_queue().communication, ignore_error=True)
+		else:
+			reference = self.in_reply_to
+			if '@' in self.in_reply_to:
+				reference, _ = self.in_reply_to.split("@", 1)
+			communication = Communication.find(reference, ignore_error=True)
+
+		self._parent_communication = communication or ''
+		return self._parent_communication
+
+	def reference_document(self):
+		"""Reference document is a document to which mail relate to.
+
+		We can get reference document from Parent record(EmailQueue | Communication) if exists.
+		Otherwise we do subject match to find reference document if we know the reference(append_to) doctype.
+		"""
+		if self._reference_document is not None:
+			return self._reference_document
+
+		reference_document = ""
+		parent = self.parent_email_queue() or self.parent_communication()
+
+		if parent and parent.reference_doctype:
+			reference_doctype, reference_name = parent.reference_doctype, parent.reference_name
+			reference_document = self.get_doc(reference_doctype, reference_name, ignore_error=True)
+
+		if not reference_document and self.email_account.append_to:
+			reference_document = self.match_record_by_subject_and_sender(self.email_account.append_to)
+
+			# if not reference_document:
+			# 	reference_document = Create_reference_document(self.email_account.append_to)
+
+		self._reference_document = reference_document or ''
+		return self._reference_document
+
+	def get_reference_name_from_subject(self):
+		"""
+		Ex: "Re: Your email (#OPP-2020-2334343)"
+		"""
+		return self.subject.rsplit('#', 1)[-1].strip(' ()')
+
+	def match_record_by_subject_and_sender(self, doctype):
+		"""Find a record in the given doctype that matches with email subject and sender.
+
+		Cases:
+		1. Sometimes record name is part of subject. We can get document by parsing name from subject
+		2. Find by matching sender and subject
+		3. Find by matching subject alone (Special case)
+			Ex: when a System User is using Outlook and replies to an email from their own client,
+			it reaches the Email Account with the threading info lost and the (sender + subject match)
+			doesn't work because the sender in the first communication was someone different to whom
+			the system user is replying to via the common email account in Frappe. This fix bypasses
+			the sender match when the sender is a system user and subject is atleast 10 chars long
+			(for additional safety)
+
+		NOTE: We consider not to match by subject if match record is very old.
+		"""
+		name = self.get_reference_name_from_subject()
+		email_fields = self.get_email_fields(doctype)
+
+		record = self.get_doc(doctype, name, ignore_error=True) if name else None
+
+		if not record:
+			subject = self.clean_subject(self.subject)
+			filters = {
+				email_fields.subject_field: ("like", f"%{subject}%"),
+				"creation": (">", self.get_relative_dt(days=-60))
+			}
+
+			# Sender check is not needed incase mail is from system user.
+			if not (len(subject) > 10 and is_system_user(self.from_email)):
+				filters[email_fields.sender_field] = self.from_email
+
+			name = frappe.db.get_value(self.email_account.append_to, filters = filters)
+			record = self.get_doc(doctype, name, ignore_error=True) if name else None
+		return record
+
+	def _create_reference_document(self, doctype):
+		""" Create reference document if it does not exist in the system.
+		"""
+		parent = frappe.new_doc(doctype)
+		email_fileds = self.get_email_fields(doctype)
+
+		if email_fileds.subject_field:
+			parent.set(email_fileds.subject_field, frappe.as_unicode(self.subject)[:140])
+
+		if email_fileds.sender_field:
+			parent.set(email_fileds.sender_field, frappe.as_unicode(self.from_email))
+
+		parent.flags.ignore_mandatory = True
+
+		try:
+			parent.insert(ignore_permissions=True)
+		except frappe.DuplicateEntryError:
+			# try and find matching parent
+			parent_name = frappe.db.get_value(self.email_account.append_to,
+				{email_fileds.sender_field: email.from_email}
+			)
+			if parent_name:
+				parent.name = parent_name
+			else:
+				parent = None
+		return parent
+
+
+	@staticmethod
+	def get_doc(doctype, docname, ignore_error=False):
+		try:
+			return frappe.get_doc(doctype, docname)
+		except frappe.DoesNotExistError:
+			if ignore_error:
+				return
+			raise
+
+	@staticmethod
+	def get_relative_dt(days):
+		"""Get relative to current datetime. Only relative days are supported.
+		"""
+		return add_days(get_datetime(), days)
+
+	@staticmethod
+	def get_users_linked_to_account(email_account):
+		"""Get list of users who linked to Email account.
+		"""
+		users = frappe.get_all("User Email", filters={"email_account": email_account.name},
+			fields=["parent"])
+		return list(set([user.get("parent") for user in users]))
+
+	@staticmethod
+	def clean_subject(subject):
+		"""Remove Prefixes like 'fw', FWD', 're' etc from subject.
+		"""
+		# Match strings like "fw:", "re	:" etc.
+		regex = r"(^\s*(fw|fwd|wg)[^:]*:|\s*(re|aw)[^:]*:\s*)*"
+		return frappe.as_unicode(strip(re.sub(regex, "", subject, 0, flags=re.IGNORECASE)))
+
+	@staticmethod
+	def get_email_fields(doctype):
+		"""Returns Email related fields of a doctype.
+		"""
+		fields = frappe._dict()
+
+		email_fields = ['subject_field', 'sender_field']
+		meta = frappe.get_meta(doctype)
+
+		for field in email_fields:
+			if hasattr(meta, field):
+				fields[field] = getattr(meta, field)
+		return fields
+
+	@staticmethod
+	def get_document(self, doctype, name):
+		"""Is same as frappe.get_doc but suppresses the DoesNotExist error.
+		"""
+		try:
+			return frappe.get_doc(doctype, name)
+		except frappe.DoesNotExistError:
+			return None
+
+	def as_dict(self):
+		"""
+		"""
+		return {
+			"subject": self.subject,
+			"content": self.get_content(),
+			'text_content': self.text_content,
+			"sent_or_received": "Received",
+			"sender_full_name": self.from_real_name,
+			"sender": self.from_email,
+			"recipients": self.mail.get("To"),
+			"cc": self.mail.get("CC"),
+			"email_account": self.email_account.name,
+			"communication_medium": "Email",
+			"uid": self.uid,
+			"message_id": self.message_id,
+			"communication_date": self.date,
+			"has_attachment": 1 if self.attachments else 0,
+			"seen": self.seen_status or 0
+		}
 
 class TimerMixin(object):
 	def __init__(self, *args, **kwargs):

frappe/geo/country_info.json

@@ -953,7 +953,7 @@
   "currency_fraction_units": 100,
   "smallest_currency_fraction_value": 0.01,
   "currency_symbol": "\u20ac",
-  "number_format": "#,###.##",
+  "number_format": "#.###,##",
   "timezones": [
    "Europe/Berlin"
   ]

frappe/hooks.py

@@ -29,16 +29,16 @@ page_js = {
 
 # website
 app_include_js = [
-	"/assets/js/libs.min.js",
-	"/assets/js/desk.min.js",
-	"/assets/js/list.min.js",
-	"/assets/js/form.min.js",
-	"/assets/js/control.min.js",
-	"/assets/js/report.min.js",
+	"libs.bundle.js",
+	"desk.bundle.js",
+	"list.bundle.js",
+	"form.bundle.js",
+	"controls.bundle.js",
+	"report.bundle.js",
 ]
 app_include_css = [
-	"/assets/css/desk.min.css",
-	"/assets/css/report.min.css",
+	"desk.bundle.css",
+	"report.bundle.css",
 ]
 
 doctype_js = {
@@ -52,6 +52,8 @@ web_include_js = [
 
 web_include_css = []
 
+email_css = ['email.bundle.css']
+
 website_route_rules = [
 	{"from_route": "/blog/<category>", "to_route": "Blog Post"},
 	{"from_route": "/kb/<category>", "to_route": "Help Article"},
@@ -226,7 +228,6 @@ scheduler_events = {
 		"frappe.desk.doctype.event.event.send_event_digest",
 		"frappe.sessions.clear_expired_sessions",
 		"frappe.email.doctype.notification.notification.trigger_daily_alerts",
-		"frappe.realtime.remove_old_task_logs",
 		"frappe.utils.scheduler.restrict_scheduler_events_if_dormant",
 		"frappe.email.doctype.auto_email_report.auto_email_report.send_daily",
 		"frappe.website.doctype.personal_data_deletion_request.personal_data_deletion_request.remove_unverified_record",

frappe/installer.py

@@ -390,19 +390,16 @@ def get_conf_params(db_name=None, db_password=None):
 
 
 def make_site_dirs():
-	site_public_path = os.path.join(frappe.local.site_path, 'public')
-	site_private_path = os.path.join(frappe.local.site_path, 'private')
-	for dir_path in (
-			os.path.join(site_private_path, 'backups'),
-			os.path.join(site_public_path, 'files'),
-			os.path.join(site_private_path, 'files'),
-			os.path.join(frappe.local.site_path, 'logs'),
-			os.path.join(frappe.local.site_path, 'task-logs')):
-		if not os.path.exists(dir_path):
-			os.makedirs(dir_path)
-	locks_dir = frappe.get_site_path('locks')
-	if not os.path.exists(locks_dir):
-			os.makedirs(locks_dir)
+	for dir_path in [
+		os.path.join("public", "files"),
+		os.path.join("private", "backups"),
+		os.path.join("private", "files"),
+		"error-snapshots",
+		"locks",
+		"logs",
+	]:
+		path = frappe.get_site_path(dir_path)
+		os.makedirs(path, exist_ok=True)
 
 
 def add_module_defs(app):

frappe/integrations/doctype/connected_app/connected_app.json

@@ -54,7 +54,8 @@
    "fieldname": "client_id",
    "fieldtype": "Data",
    "in_list_view": 1,
-   "label": "Client Id"
+   "label": "Client Id",
+   "mandatory_depends_on": "eval:doc.redirect_uri"
   },
   {
    "fieldname": "redirect_uri",
@@ -96,12 +97,14 @@
   {
    "fieldname": "authorization_uri",
    "fieldtype": "Data",
-   "label": "Authorization URI"
+   "label": "Authorization URI",
+   "mandatory_depends_on": "eval:doc.redirect_uri"
   },
   {
    "fieldname": "token_uri",
    "fieldtype": "Data",
-   "label": "Token URI"
+   "label": "Token URI",
+   "mandatory_depends_on": "eval:doc.redirect_uri"
   },
   {
    "fieldname": "revocation_uri",
@@ -136,7 +139,7 @@
    "link_fieldname": "connected_app"
   }
  ],
- "modified": "2020-11-16 16:29:50.277405",
+ "modified": "2021-05-10 05:03:06.296863",
  "modified_by": "Administrator",
  "module": "Integrations",
  "name": "Connected App",

frappe/integrations/doctype/connected_app/connected_app.py

@@ -26,20 +26,27 @@ class ConnectedApp(Document):
 		self.redirect_uri = urljoin(base_url, callback_path)
 
 	def get_oauth2_session(self, user=None, init=False):
+		"""Return an auto-refreshing OAuth2 session which is an extension of a requests.Session()"""
 		token = None
 		token_updater = None
+		auto_refresh_kwargs = None
 
 		if not init:
 			user = user or frappe.session.user
 			token_cache = self.get_user_token(user)
 			token = token_cache.get_json()
 			token_updater = token_cache.update_data
+			auto_refresh_kwargs = {'client_id': self.client_id}
+			client_secret = self.get_password('client_secret')
+			if client_secret:
+				auto_refresh_kwargs['client_secret'] = client_secret
 
 		return OAuth2Session(
 			client_id=self.client_id,
 			token=token,
 			token_updater=token_updater,
 			auto_refresh_url=self.token_uri,
+			auto_refresh_kwargs=auto_refresh_kwargs,
 			redirect_uri=self.redirect_uri,
 			scope=self.get_scopes()
 		)

frappe/integrations/doctype/slack_webhook_url/slack_webhook_url.json

@@ -1,126 +1,61 @@
 {
- "allow_copy": 0, 
- "allow_guest_to_view": 0, 
- "allow_import": 0, 
- "allow_rename": 0, 
- "autoname": "field:webhook_name", 
- "beta": 0, 
- "creation": "2018-05-22 13:20:51.450815", 
- "custom": 0, 
- "docstatus": 0, 
- "doctype": "DocType", 
- "document_type": "", 
- "editable_grid": 1, 
- "engine": "InnoDB", 
+ "actions": [],
+ "autoname": "field:webhook_name",
+ "creation": "2018-05-22 13:20:51.450815",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "webhook_name",
+  "webhook_url",
+  "show_document_link"
+ ],
  "fields": [
   {
-   "allow_bulk_edit": 0, 
-   "allow_in_quick_entry": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "webhook_name", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Name", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "translatable": 0, 
-   "unique": 0
-  }, 
+   "fieldname": "webhook_name",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Name",
+   "reqd": 1,
+   "unique": 1
+  },
   {
-   "allow_bulk_edit": 0, 
-   "allow_in_quick_entry": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "webhook_url", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Webhook URL", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "translatable": 0, 
-   "unique": 0
+   "fieldname": "webhook_url",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Webhook URL",
+   "reqd": 1
+  },
+  {
+   "allow_in_quick_entry": 1,
+   "default": "1",
+   "fieldname": "show_document_link",
+   "fieldtype": "Check",
+   "label": "Show link to document"
   }
- ], 
- "has_web_view": 0, 
- "hide_heading": 0, 
- "hide_toolbar": 0, 
- "idx": 0, 
- "image_view": 0, 
- "in_create": 0, 
- "is_submittable": 0, 
- "issingle": 0, 
- "istable": 0, 
- "max_attachments": 0, 
- "modified": "2018-05-22 13:25:24.621129", 
- "modified_by": "Administrator", 
- "module": "Integrations", 
- "name": "Slack Webhook URL", 
- "name_case": "", 
- "owner": "Administrator", 
+ ],
+ "links": [],
+ "modified": "2021-05-12 18:24:37.810235",
+ "modified_by": "Administrator",
+ "module": "Integrations",
+ "name": "Slack Webhook URL",
+ "owner": "Administrator",
  "permissions": [
   {
-   "amend": 0, 
-   "cancel": 0, 
-   "create": 1, 
-   "delete": 1, 
-   "email": 1, 
-   "export": 1, 
-   "if_owner": 0, 
-   "import": 0, 
-   "permlevel": 0, 
-   "print": 1, 
-   "read": 1, 
-   "report": 1, 
-   "role": "System Manager", 
-   "set_user_permissions": 0, 
-   "share": 1, 
-   "submit": 0, 
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "export": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "System Manager",
+   "share": 1,
    "write": 1
   }
- ], 
- "quick_entry": 1, 
- "read_only": 0, 
- "read_only_onload": 0, 
- "show_name_in_global_search": 0, 
- "sort_field": "modified", 
- "sort_order": "DESC", 
- "track_changes": 1, 
- "track_seen": 0
+ ],
+ "quick_entry": 1,
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "track_changes": 1
 }

frappe/integrations/doctype/slack_webhook_url/slack_webhook_url.py

@@ -25,22 +25,27 @@ class SlackWebhookURL(Document):
 
 
 def send_slack_message(webhook_url, message, reference_doctype, reference_name):
-	slack_url = frappe.db.get_value("Slack Webhook URL", webhook_url, "webhook_url")
-	doc_url = get_url_to_form(reference_doctype, reference_name)
-	attachments = [
-		{
+	data = {"text": message, "attachments": []}
+
+	slack_url, show_link = frappe.db.get_value(
+		"Slack Webhook URL", webhook_url, ["webhook_url", "show_document_link"]
+	)
+
+	if show_link:
+		doc_url = get_url_to_form(reference_doctype, reference_name)
+		link_to_doc = {
 			"fallback": _("See the document at {0}").format(doc_url),
 			"actions": [
 				{
 					"type": "button",
 					"text": _("Go to the document"),
 					"url": doc_url,
-					"style": "primary"
+					"style": "primary",
 				}
-			]
+			],
 		}
-	]
-	data = {"text": message, "attachments": attachments}
+		data["attachments"].append(link_to_doc)
+
 	r = requests.post(slack_url, data=json.dumps(data))
 
 	if not r.ok:

frappe/integrations/oauth2.py

@@ -1,6 +1,5 @@
 import json
 from urllib.parse import quote, urlencode
-
 from oauthlib.oauth2 import FatalClientError, OAuth2Error
 from oauthlib.openid.connect.core.endpoints.pre_configured import (
 	Server as WebApplicationServer,

frappe/model/base_document.py

@@ -34,8 +34,9 @@ def get_controller(doctype):
 		from frappe.model.document import Document
 		from frappe.utils.nestedset import NestedSet
 
-		module_name, custom = frappe.db.get_value("DocType", doctype, ("module", "custom"), cache=True) \
-			or ["Core", False]
+		module_name, custom = frappe.db.get_value(
+			"DocType", doctype, ("module", "custom"), cache=True
+		) or ["Core", False]
 
 		if custom:
 			if frappe.db.field_exists("DocType", "is_tree"):
@@ -869,7 +870,7 @@ class BaseDocument(object):
 			from frappe.model.meta import get_default_df
 			df = get_default_df(fieldname)
 
-		if not currency:
+		if not currency and df:
 			currency = self.get(df.get("options"))
 			if not frappe.db.exists('Currency', currency, cache=True):
 				currency = None

frappe/model/document.py

@@ -17,6 +17,7 @@ from frappe.model.workflow import set_workflow_state_on_action
 from frappe.utils.global_search import update_global_search
 from frappe.integrations.doctype.webhook import run_webhooks
 from frappe.desk.form.document_follow import follow_document
+from frappe.desk.utils import slug
 from frappe.core.doctype.server_script.server_script_utils import run_server_script_for_doc_event
 
 # once_only validation
@@ -1202,8 +1203,8 @@ class Document(BaseDocument):
 			doc.set(fieldname, flt(doc.get(fieldname), self.precision(fieldname, doc.parentfield)))
 
 	def get_url(self):
-		"""Returns Desk URL for this document. `/app/Form/{doctype}/{name}`"""
-		return "/app/Form/{doctype}/{name}".format(doctype=self.doctype, name=self.name)
+		"""Returns Desk URL for this document. `/app/{doctype}/{name}`"""
+		return f"/app/{slug(self.doctype)}/{self.name}"
 
 	def add_comment(self, comment_type='Comment', text=None, comment_email=None, link_doctype=None, link_name=None, comment_by=None):
 		"""Add a comment to this document.

frappe/model/naming.py

@@ -199,10 +199,39 @@ def getseries(key, digits):
 
 
 def revert_series_if_last(key, name, doc=None):
-	if ".#" in key:
+	"""
+	Reverts the series for particular naming series:
+	* key is naming series		- SINV-.YYYY-.####
+	* name is actual name		- SINV-2021-0001
+		
+	1. This function split the key into two parts prefix (SINV-YYYY) & hashes (####).
+	2. Use prefix to get the current index of that naming series from Series table
+	3. Then revert the current index.
+
+	*For custom naming series:*
+	1. hash can exist anywhere, if it exist in hashes then it take normal flow.
+	2. If hash doesn't exit in hashes, we get the hash from prefix, then update name and prefix accordingly.
+
+	*Example:*
+		1. key = SINV-.YYYY.- 
+			* If key doesn't have hash it will add hash at the end
+			* prefix will be SINV-YYYY based on this will get current index from Series table.
+		2. key = SINV-.####.-2021
+			* now prefix = SINV-#### and hashes = 2021 (hash doesn't exist)
+			* will search hash in key then accordingly get prefix = SINV-
+		3. key = ####.-2021
+			* prefix = #### and hashes = 2021 (hash doesn't exist)
+			* will search hash in key then accordingly get prefix = "" 
+	"""
+	if ".#" in key: 
 		prefix, hashes = key.rsplit(".", 1)
 		if "#" not in hashes:
-			return
+			# get the hash part from the key
+			hash = re.search("#+", key)
+			if not hash:
+				return
+			name = name.replace(hashes, "")
+			prefix = prefix.replace(hash.group(), "")
 	else:
 		prefix = key
 
@@ -254,7 +283,7 @@ def append_number_if_name_exists(doctype, value, fieldname="name", separator="-"
 	filters.update({fieldname: value})
 	exists = frappe.db.exists(doctype, filters)
 
-	regex = "^{value}{separator}\d+$".format(value=re.escape(value), separator=separator)
+	regex = "^{value}{separator}\\d+$".format(value=re.escape(value), separator=separator)
 
 	if exists:
 		last = frappe.db.sql("""SELECT `{fieldname}` FROM `tab{doctype}`

frappe/modules/import_file.py

@@ -107,6 +107,15 @@ def import_doc(docdict, force=False, data_import=False, pre_process=None,
 
 	doc = frappe.get_doc(docdict)
 
+	# Note on Tree DocTypes:
+	# The tree structure is maintained in the database via the fields "lft" and
+	# "rgt". They are automatically set and kept up-to-date. Importing them
+	# would destroy any existing tree structure.
+	if getattr(doc.meta, 'is_tree', None) and any([doc.lft, doc.rgt]):
+		print('Ignoring values of `lft` and `rgt` for {} "{}"'.format(doc.doctype, doc.name))
+		doc.lft = None
+		doc.rgt = None
+
 	doc.run_method("before_import")
 
 	doc.flags.ignore_version = ignore_version

frappe/oauth.py

@@ -4,11 +4,9 @@ import hashlib
 import re
 from http import cookies
 from urllib.parse import unquote, urlparse
-
 import jwt
 import pytz
 from oauthlib.openid import RequestValidator
-
 import frappe
 from frappe.auth import LoginManager
 

frappe/parallel_test_runner.py

@@ -0,0 +1,282 @@
+import json
+import os
+import re
+import sys
+import time
+import unittest
+import click
+import frappe
+import requests
+
+from .test_runner import (SLOW_TEST_THRESHOLD, make_test_records, set_test_email_config)
+
+click_ctx = click.get_current_context(True)
+if click_ctx:
+	click_ctx.color = True
+
+class ParallelTestRunner():
+	def __init__(self, app, site, build_number=1, total_builds=1, with_coverage=False):
+		self.app = app
+		self.site = site
+		self.with_coverage = with_coverage
+		self.build_number = frappe.utils.cint(build_number) or 1
+		self.total_builds = frappe.utils.cint(total_builds)
+		self.setup_test_site()
+		self.run_tests()
+
+	def setup_test_site(self):
+		frappe.init(site=self.site)
+		if not frappe.db:
+			frappe.connect()
+
+		frappe.flags.in_test = True
+		frappe.clear_cache()
+		frappe.utils.scheduler.disable_scheduler()
+		set_test_email_config()
+		self.before_test_setup()
+
+	def before_test_setup(self):
+		start_time = time.time()
+		for fn in frappe.get_hooks("before_tests", app_name=self.app):
+			frappe.get_attr(fn)()
+
+		test_module = frappe.get_module(f'{self.app}.tests')
+
+		if hasattr(test_module, "global_test_dependencies"):
+			for doctype in test_module.global_test_dependencies:
+				make_test_records(doctype)
+
+		elapsed = time.time() - start_time
+		elapsed = click.style(f' ({elapsed:.03}s)', fg='red')
+		click.echo(f'Before Test {elapsed}')
+
+	def run_tests(self):
+		self.test_result = ParallelTestResult(stream=sys.stderr, descriptions=True, verbosity=2)
+
+		self.start_coverage()
+
+		for test_file_info in self.get_test_file_list():
+			self.run_tests_for_file(test_file_info)
+
+		self.save_coverage()
+		self.print_result()
+
+	def run_tests_for_file(self, file_info):
+		if not file_info: return
+
+		frappe.set_user('Administrator')
+		path, filename = file_info
+		module = self.get_module(path, filename)
+		self.create_test_dependency_records(module, path, filename)
+		test_suite = unittest.TestSuite()
+		module_test_cases = unittest.TestLoader().loadTestsFromModule(module)
+		test_suite.addTest(module_test_cases)
+		test_suite(self.test_result)
+
+	def create_test_dependency_records(self, module, path, filename):
+		if hasattr(module, "test_dependencies"):
+			for doctype in module.test_dependencies:
+				make_test_records(doctype)
+
+		if os.path.basename(os.path.dirname(path)) == "doctype":
+			# test_data_migration_connector.py > data_migration_connector.json
+			test_record_filename = re.sub('^test_', '', filename).replace(".py", ".json")
+			test_record_file_path = os.path.join(path, test_record_filename)
+			if os.path.exists(test_record_file_path):
+				with open(test_record_file_path, 'r') as f:
+					doc = json.loads(f.read())
+					doctype = doc["name"]
+					make_test_records(doctype)
+
+	def get_module(self, path, filename):
+		app_path = frappe.get_pymodule_path(self.app)
+		relative_path = os.path.relpath(path, app_path)
+		if relative_path == '.':
+			module_name = self.app
+		else:
+			relative_path = relative_path.replace('/', '.')
+			module_name = os.path.splitext(filename)[0]
+			module_name = f'{self.app}.{relative_path}.{module_name}'
+
+		return frappe.get_module(module_name)
+
+	def print_result(self):
+		self.test_result.printErrors()
+		click.echo(self.test_result)
+		if self.test_result.failures or self.test_result.errors:
+			if os.environ.get('CI'):
+				sys.exit(1)
+
+	def start_coverage(self):
+		if self.with_coverage:
+			from coverage import Coverage
+			from frappe.utils import get_bench_path
+
+			# Generate coverage report only for app that is being tested
+			source_path = os.path.join(get_bench_path(), 'apps', self.app)
+			omit=['*.html', '*.js', '*.xml', '*.css', '*.less', '*.scss',
+				'*.vue', '*/doctype/*/*_dashboard.py', '*/patches/*']
+
+			if self.app == 'frappe':
+				omit.append('*/commands/*')
+
+			self.coverage = Coverage(source=[source_path], omit=omit)
+			self.coverage.start()
+
+	def save_coverage(self):
+		if not self.with_coverage:
+			return
+		self.coverage.stop()
+		self.coverage.save()
+
+	def get_test_file_list(self):
+		test_list = get_all_tests(self.app)
+		split_size = frappe.utils.ceil(len(test_list) / self.total_builds)
+		# [1,2,3,4,5,6] to [[1,2], [3,4], [4,6]] if split_size is 2
+		test_chunks = [test_list[x:x+split_size] for x in range(0, len(test_list), split_size)]
+		return test_chunks[self.build_number - 1]
+
+
+class ParallelTestResult(unittest.TextTestResult):
+	def startTest(self, test):
+		self._started_at = time.time()
+		super(unittest.TextTestResult, self).startTest(test)
+		test_class = unittest.util.strclass(test.__class__)
+		if not hasattr(self, 'current_test_class') or self.current_test_class != test_class:
+			click.echo(f"\n{unittest.util.strclass(test.__class__)}")
+			self.current_test_class = test_class
+
+	def getTestMethodName(self, test):
+		return test._testMethodName if hasattr(test, '_testMethodName') else str(test)
+
+	def addSuccess(self, test):
+		super(unittest.TextTestResult, self).addSuccess(test)
+		elapsed = time.time() - self._started_at
+		threshold_passed = elapsed >= SLOW_TEST_THRESHOLD
+		elapsed = click.style(f' ({elapsed:.03}s)', fg='red') if threshold_passed else ''
+		click.echo(f"  {click.style(' ✔ ', fg='green')} {self.getTestMethodName(test)}{elapsed}")
+
+	def addError(self, test, err):
+		super(unittest.TextTestResult, self).addError(test, err)
+		click.echo(f"  {click.style(' ✖ ', fg='red')} {self.getTestMethodName(test)}")
+
+	def addFailure(self, test, err):
+		super(unittest.TextTestResult, self).addFailure(test, err)
+		click.echo(f"  {click.style(' ✖ ', fg='red')} {self.getTestMethodName(test)}")
+
+	def addSkip(self, test, reason):
+		super(unittest.TextTestResult, self).addSkip(test, reason)
+		click.echo(f"  {click.style(' = ', fg='white')} {self.getTestMethodName(test)}")
+
+	def addExpectedFailure(self, test, err):
+		super(unittest.TextTestResult, self).addExpectedFailure(test, err)
+		click.echo(f"  {click.style(' ✖ ', fg='red')} {self.getTestMethodName(test)}")
+
+	def addUnexpectedSuccess(self, test):
+		super(unittest.TextTestResult, self).addUnexpectedSuccess(test)
+		click.echo(f"  {click.style(' ✔ ', fg='green')} {self.getTestMethodName(test)}")
+
+	def printErrors(self):
+		click.echo('\n')
+		self.printErrorList(' ERROR ', self.errors, 'red')
+		self.printErrorList(' FAIL ', self.failures, 'red')
+
+	def printErrorList(self, flavour, errors, color):
+		for test, err in errors:
+			click.echo(self.separator1)
+			click.echo(f"{click.style(flavour, bg=color)} {self.getDescription(test)}")
+			click.echo(self.separator2)
+			click.echo(err)
+
+	def __str__(self):
+		return f"Tests: {self.testsRun}, Failing: {len(self.failures)}, Errors: {len(self.errors)}"
+
+def get_all_tests(app):
+	test_file_list = []
+	for path, folders, files in os.walk(frappe.get_pymodule_path(app)):
+		for dontwalk in ('locals', '.git', 'public', '__pycache__'):
+			if dontwalk in folders:
+				folders.remove(dontwalk)
+
+		# for predictability
+		folders.sort()
+		files.sort()
+
+		if os.path.sep.join(["doctype", "doctype", "boilerplate"]) in path:
+			# in /doctype/doctype/boilerplate/
+			continue
+
+		for filename in files:
+			if filename.startswith("test_") and filename.endswith(".py") \
+				and filename != 'test_runner.py':
+				test_file_list.append([path, filename])
+
+	return test_file_list
+
+
+class ParallelTestWithOrchestrator(ParallelTestRunner):
+	'''
+		This can be used to balance-out test time across multiple instances
+		This is dependent on external orchestrator which returns next test to run
+
+		orchestrator endpoints
+		- register-instance (<build_id>, <instance_id>, test_spec_list)
+		- get-next-test-spec (<build_id>, <instance_id>)
+		- test-completed (<build_id>, <instance_id>)
+	'''
+	def __init__(self, app, site, with_coverage=False):
+		self.orchestrator_url = os.environ.get('ORCHESTRATOR_URL')
+		if not self.orchestrator_url:
+			click.echo('ORCHESTRATOR_URL environment variable not found!')
+			click.echo('Pass public URL after hosting https://github.com/frappe/test-orchestrator')
+			sys.exit(1)
+
+		self.ci_build_id = os.environ.get('CI_BUILD_ID')
+		self.ci_instance_id = os.environ.get('CI_INSTANCE_ID') or frappe.generate_hash(length=10)
+		if not self.ci_build_id:
+			click.echo('CI_BUILD_ID environment variable not found!')
+			sys.exit(1)
+
+		ParallelTestRunner.__init__(self, app, site, with_coverage=with_coverage)
+
+	def run_tests(self):
+		self.test_status = 'ongoing'
+		self.register_instance()
+		super().run_tests()
+
+	def get_test_file_list(self):
+		while self.test_status == 'ongoing':
+			yield self.get_next_test()
+
+	def register_instance(self):
+		test_spec_list = get_all_tests(self.app)
+		response_data = self.call_orchestrator('register-instance', data={
+			'test_spec_list': test_spec_list
+		})
+		self.is_master = response_data.get('is_master')
+
+	def get_next_test(self):
+		response_data = self.call_orchestrator('get-next-test-spec')
+		self.test_status = response_data.get('status')
+		return response_data.get('next_test')
+
+	def print_result(self):
+		self.call_orchestrator('test-completed')
+		return super().print_result()
+
+	def call_orchestrator(self, endpoint, data={}):
+		# add repo token header
+		# build id in header
+		headers = {
+			'CI-BUILD-ID': self.ci_build_id,
+			'CI-INSTANCE-ID': self.ci_instance_id,
+			'REPO-TOKEN': '2948288382838DE'
+		}
+		url = f'{self.orchestrator_url}/{endpoint}'
+		res = requests.get(url, json=data, headers=headers)
+		res.raise_for_status()
+		response_data = {}
+		if 'application/json' in res.headers.get('content-type'):
+			response_data = res.json()
+
+		return response_data

frappe/patches/v5_0/fix_text_editor_file_urls.py

@@ -33,8 +33,7 @@ def execute():
 def scrub_relative_urls(html):
 	"""prepend a slash before a relative url"""
 	try:
-		return re.sub("""src[\s]*=[\s]*['"]files/([^'"]*)['"]""", 'src="/files/\g<1>"', html)
-		# return re.sub("""(src|href)[^\w'"]*['"](?!http|ftp|mailto|/|#|%|{|cid:|\.com/www\.)([^'" >]+)['"]""", '\g<1>="/\g<2>"', html)
+		return re.sub(r'src[\s]*=[\s]*[\'"]files/([^\'"]*)[\'"]', r'src="/files/\g<1>"', html)
 	except:
 		print("Error", html)
 		raise

frappe/printing/doctype/print_format/test_print_format.py

@@ -12,13 +12,13 @@ class TestPrintFormat(unittest.TestCase):
 	def test_print_user(self, style=None):
 		print_html = frappe.get_print("User", "Administrator", style=style)
 		self.assertTrue("<label>First Name: </label>" in print_html)
-		self.assertTrue(re.findall('<div class="col-xs-[^"]*">[\s]*administrator[\s]*</div>', print_html))
+		self.assertTrue(re.findall(r'<div class="col-xs-[^"]*">[\s]*administrator[\s]*</div>', print_html))
 		return print_html
 
 	def test_print_user_standard(self):
 		print_html = self.test_print_user("Standard")
-		self.assertTrue(re.findall('\.print-format {[\s]*font-size: 9pt;', print_html))
-		self.assertFalse(re.findall('th {[\s]*background-color: #eee;[\s]*}', print_html))
+		self.assertTrue(re.findall(r'\.print-format {[\s]*font-size: 9pt;', print_html))
+		self.assertFalse(re.findall(r'th {[\s]*background-color: #eee;[\s]*}', print_html))
 		self.assertFalse("font-family: serif;" in print_html)
 
 	def test_print_user_modern(self):

frappe/printing/page/print/print.js

@@ -408,14 +408,17 @@ frappe.ui.form.PrintView = class {
 
 	setup_print_format_dom(out, $print_format) {
 		this.print_wrapper.find('.print-format-skeleton').remove();
+		let base_url = frappe.urllib.get_base_url();
+		let print_css = frappe.assets.bundled_asset('print.bundle.css');
 		this.$print_format_body.find('head').html(
 			`<style type="text/css">${out.style}</style>
-			<link href="${frappe.urllib.get_base_url()}/assets/css/printview.css" rel="stylesheet">`
+			<link href="${base_url}${print_css}" rel="stylesheet">`
 		);
 
 		if (frappe.utils.is_rtl(this.lang_code)) {
+			let rtl_css = frappe.assets.bundled_asset('frappe-rtl.bundle.css');
 			this.$print_format_body.find('head').append(
-				`<link type="text/css" rel="stylesheet" href="${frappe.urllib.get_base_url()}/assets/css/frappe-rtl.css"></link>`
+				`<link type="text/css" rel="stylesheet" href="${base_url}${rtl_css}"></link>`
 			);
 		}
 

frappe/printing/page/print_format_builder/print_format_builder.js

@@ -23,13 +23,13 @@ frappe.pages['print-format-builder'].on_page_show = function(wrapper) {
 	}
 }
 
-frappe.PrintFormatBuilder = Class.extend({
-	init: function(parent) {
+frappe.PrintFormatBuilder = class PrintFormatBuilder {
+	constructor(parent) {
 		this.parent = parent;
 		this.make();
 		this.refresh();
-	},
-	refresh: function() {
+	}
+	refresh() {
 		this.custom_html_count = 0;
 		if(!this.print_format) {
 			this.show_start();
@@ -37,8 +37,8 @@ frappe.PrintFormatBuilder = Class.extend({
 			this.page.set_title(this.print_format.name);
 			this.setup_print_format();
 		}
-	},
-	make: function() {
+	}
+	make() {
 		this.page = frappe.ui.make_app_page({
 			parent: this.parent,
 			title: __("Print Format Builder"),
@@ -56,15 +56,15 @@ frappe.PrintFormatBuilder = Class.extend({
 		this.setup_edit_custom_html();
 		// $(this.page.sidebar).css({"position": 'fixed'});
 		// $(this.page.main).parent().css({"margin-left": '16.67%'});
-	},
-	show_start: function() {
+	}
+	show_start() {
 		this.page.main.html(frappe.render_template("print_format_builder_start", {}));
 		this.page.clear_actions();
 		this.page.set_title(__("Print Format Builder"));
 		this.start_edit_print_format();
 		this.start_new_print_format();
-	},
-	start_edit_print_format: function() {
+	}
+	start_edit_print_format() {
 		// print format control
 		var me = this;
 		this.print_format_input = frappe.ui.form.make_control({
@@ -89,8 +89,8 @@ frappe.PrintFormatBuilder = Class.extend({
 				frappe.set_route('print-format-builder', name);
 			});
 		});
-	},
-	start_new_print_format: function() {
+	}
+	start_new_print_format() {
 		var me = this;
 		this.doctype_input = frappe.ui.form.make_control({
 			parent: this.page.main.find(".doctype-selector"),
@@ -125,8 +125,8 @@ frappe.PrintFormatBuilder = Class.extend({
 			me.setup_new_print_format(doctype, name);
 
 		});
-	},
-	setup_new_print_format: function(doctype, name, based_on) {
+	}
+	setup_new_print_format(doctype, name, based_on) {
 		frappe.call({
 			method: 'frappe.printing.page.print_format_builder.print_format_builder.create_custom_format',
 			args: {
@@ -143,8 +143,8 @@ frappe.PrintFormatBuilder = Class.extend({
 				}
 			},
 		});
-	},
-	setup_print_format: function() {
+	}
+	setup_print_format() {
 		var me = this;
 		frappe.model.with_doctype(this.print_format.doc_type, function(doctype) {
 			me.meta = frappe.get_meta(me.print_format.doc_type);
@@ -163,23 +163,23 @@ frappe.PrintFormatBuilder = Class.extend({
 				frappe.set_route("Form", "Print Format", me.print_format.name);
 			});
 		});
-	},
-	setup_sidebar: function() {
+	}
+	setup_sidebar() {
 		// prepend custom HTML field
 		var fields = [this.get_custom_html_field()].concat(this.meta.fields);
 		this.page.sidebar.html(
 			$(frappe.render_template("print_format_builder_sidebar", {fields: fields}))
 		);
 		this.setup_field_filter();
-	},
-	get_custom_html_field: function() {
+	}
+	get_custom_html_field() {
 		return {
 			fieldtype: "Custom HTML",
 			fieldname: "_custom_html",
 			label: __("Custom HTML")
-		}
-	},
-	render_layout: function() {
+		};
+	}
+	render_layout() {
 		this.page.main.empty();
 		this.prepare_data();
 		$(frappe.render_template("print_format_builder_layout", {
@@ -190,8 +190,8 @@ frappe.PrintFormatBuilder = Class.extend({
 		this.setup_edit_heading();
 		this.setup_field_settings();
 		this.setup_html_data();
-	},
-	prepare_data: function() {
+	}
+	prepare_data() {
 		this.print_heading_template = null;
 		this.data = JSON.parse(this.print_format.format_data || "[]");
 		if(!this.data.length) {
@@ -280,22 +280,22 @@ frappe.PrintFormatBuilder = Class.extend({
 		this.layout_data = $.map(this.layout_data, function(s) {
 			return s.has_fields ? s : null
 		});
-	},
-	get_new_section: function() {
+	}
+	get_new_section() {
 		return {columns: [], no_of_columns: 0, label:''};
-	},
-	get_new_column: function() {
+	}
+	get_new_column() {
 		return {fields: []}
-	},
-	add_table_properties: function(f) {
+	}
+	add_table_properties(f) {
 		// build table columns and widths in a dict
 		// visible_columns
 		var me = this;
 		if(!f.visible_columns) {
 			me.init_visible_columns(f);
 		}
-	},
-	init_visible_columns: function(f) {
+	}
+	init_visible_columns(f) {
 		f.visible_columns = []
 		$.each(frappe.get_meta(f.options).fields, function(i, _f) {
 			if(!in_list(["Section Break", "Column Break"], _f.fieldtype) &&
@@ -306,8 +306,8 @@ frappe.PrintFormatBuilder = Class.extend({
 					print_width: (_f.width || ""), print_hide:0});
 			}
 		});
-	},
-	setup_sortable: function() {
+	}
+	setup_sortable() {
 		var me = this;
 
 		// drag from fields library
@@ -332,8 +332,8 @@ frappe.PrintFormatBuilder = Class.extend({
 		Sortable.create(this.page.main.find(".print-format-builder-layout").get(0),
 			{ handle: ".print-format-builder-section-head" }
 		);
-	},
-	setup_sortable_for_column: function(col) {
+	}
+	setup_sortable_for_column(col) {
 		var me = this;
 		Sortable.create(col, {
 			group: {
@@ -363,8 +363,8 @@ frappe.PrintFormatBuilder = Class.extend({
 			}
 		});
 
-	},
-	setup_field_filter: function() {
+	}
+	setup_field_filter() {
 		var me = this;
 		this.page.sidebar.find(".filter-fields").on("keyup", function() {
 			var text = $(this).val();
@@ -373,8 +373,8 @@ frappe.PrintFormatBuilder = Class.extend({
 				$(this).parent().toggle(show);
 			})
 		});
-	},
-	setup_section_settings: function() {
+	}
+	setup_section_settings() {
 		var me = this;
 		this.page.main.on("click", ".section-settings", function() {
 			var section = $(this).parent().parent();
@@ -431,8 +431,8 @@ frappe.PrintFormatBuilder = Class.extend({
 
 			return false;
 		});
-	},
-	setup_field_settings: function() {
+	}
+	setup_field_settings() {
 		this.page.main.find(".field-settings").on("click", e => {
 			const field = $(e.currentTarget).parent();
 			// new dialog
@@ -482,8 +482,8 @@ frappe.PrintFormatBuilder = Class.extend({
 
 			return false;
 		});
-	},
-	setup_html_data: function() {
+	}
+	setup_html_data() {
 		// set JQuery `data` for Custom HTML fields, since editing the HTML
 		// directly causes problem becuase of HTML reformatting
 		//
@@ -496,8 +496,8 @@ frappe.PrintFormatBuilder = Class.extend({
 			var html = me.custom_html_dict[parseInt(content.attr('data-custom-html-id'))].options;
 			content.data('content', html);
 		})
-	},
-	update_columns_in_section: function(section, no_of_columns, new_no_of_columns) {
+	}
+	update_columns_in_section(section, no_of_columns, new_no_of_columns) {
 		var col_size = 12 / new_no_of_columns,
 			me = this,
 			resize = function() {
@@ -539,8 +539,8 @@ frappe.PrintFormatBuilder = Class.extend({
 			resize();
 		}
 
-	},
-	setup_add_section: function() {
+	}
+	setup_add_section() {
 		var me = this;
 		this.page.main.find(".print-format-builder-add-section").on("click", function() {
 			// boostrap new section info
@@ -554,8 +554,8 @@ frappe.PrintFormatBuilder = Class.extend({
 
 			me.setup_sortable_for_column($section.find(".print-format-builder-column").get(0));
 		});
-	},
-	setup_edit_heading: function() {
+	}
+	setup_edit_heading() {
 		var me = this;
 		var $heading = this.page.main.find(".print-format-builder-print-heading");
 
@@ -565,8 +565,8 @@ frappe.PrintFormatBuilder = Class.extend({
 		this.page.main.find(".edit-heading").on("click", function() {
 			var d = me.get_edit_html_dialog(__("Edit Heading"), __("Heading"), $heading);
 		})
-	},
-	setup_column_selector: function() {
+	}
+	setup_column_selector() {
 		var me = this;
 		this.page.main.on("click", ".select-columns", function() {
 			var parent = $(this).parents(".print-format-builder-field:first"),
@@ -657,24 +657,24 @@ frappe.PrintFormatBuilder = Class.extend({
 
 			return false;
 		});
-	},
-	get_visible_columns_string: function(f) {
+	}
+	get_visible_columns_string(f) {
 		if(!f.visible_columns) {
 			this.init_visible_columns(f);
 		}
 		return $.map(f.visible_columns, function(v) { return v.fieldname + "|" + (v.print_width || "") }).join(",");
-	},
-	get_no_content: function() {
+	}
+	get_no_content() {
 		return __("Edit to add content")
-	},
-	setup_edit_custom_html: function() {
+	}
+	setup_edit_custom_html() {
 		var me = this;
 		this.page.main.on("click", ".edit-html", function() {
 			me.get_edit_html_dialog(__("Edit Custom HTML"), __("Custom HTML"),
 				$(this).parents(".print-format-builder-field:first").find(".html-content"));
 		});
-	},
-	get_edit_html_dialog: function(title, label, $content) {
+	}
+	get_edit_html_dialog(title, label, $content) {
 		var me = this;
 		var d = new frappe.ui.Dialog({
 			title: title,
@@ -710,8 +710,8 @@ frappe.PrintFormatBuilder = Class.extend({
 		d.show();
 
 		return d;
-	},
-	save_print_format: function() {
+	}
+	save_print_format() {
 		var data = [],
 			me = this;
 
@@ -789,4 +789,4 @@ frappe.PrintFormatBuilder = Class.extend({
 			}
 		});
 	}
-});
+};

frappe/public/html/print_template.html

@@ -7,7 +7,7 @@
 	<meta name="description" content="">
 	<meta name="author" content="">
 	<title>{{ title }}</title>
-	<link href="{{ base_url }}/assets/css/printview.css" rel="stylesheet">
+	<link href="{{ base_url }}{{ frappe.assets.bundled_asset('print.bundle.css') }}" rel="stylesheet">
 	<style>
 		{{ print_css }}
 	</style>

frappe/public/js/barcode_scanner.bundle.js

@@ -0,0 +1 @@
+import "./frappe/barcode_scanner/quagga";

frappe/public/js/bootstrap-4-web.bundle.js

@@ -0,0 +1,64 @@
+
+// multilevel dropdown
+$('.dropdown-menu a.dropdown-toggle').on('click', function (e) {
+	e.preventDefault();
+	e.stopImmediatePropagation();
+	if (!$(this).next().hasClass('show')) {
+		$(this).parents('.dropdown-menu').first().find('.show').removeClass("show");
+	}
+	var $subMenu = $(this).next(".dropdown-menu");
+	$subMenu.toggleClass('show');
+
+
+	$(this).parents('li.nav-item.dropdown.show').on('hidden.bs.dropdown', function () {
+		$('.dropdown-submenu .show').removeClass("show");
+	});
+
+	return false;
+});
+
+frappe.get_modal = function (title, content) {
+	return $(
+		`<div class="modal" tabindex="-1" role="dialog">
+			<div class="modal-dialog modal-dialog-scrollable" role="document">
+				<div class="modal-content">
+					<div class="modal-header">
+						<h5 class="modal-title">${title}</h5>
+						<button type="button" class="close" data-dismiss="modal" aria-label="Close">
+						${frappe.utils.icon('close-alt', 'sm', 'close-alt')}
+						</button>
+					</div>
+					<div class="modal-body">
+						${content}
+					</div>
+					<div class="modal-footer hidden">
+						<button type="button" class="btn btn-default btn-sm btn-modal-close" data-dismiss="modal">
+							<i class="octicon octicon-x visible-xs" style="padding: 1px 0px;"></i>
+							<span class="hidden-xs">${__("Close")}</span>
+						</button>
+						<button type="button" class="btn btn-sm btn-primary hidden"></button>
+					</div>
+				</div>
+			</div>
+		</div>`
+	);
+};
+
+frappe.ui.Dialog = class Dialog extends frappe.ui.Dialog {
+	get_primary_btn() {
+		return this.$wrapper.find(".modal-footer .btn-primary");
+	}
+
+	set_primary_action(label, click) {
+		this.$wrapper.find('.modal-footer').removeClass('hidden');
+		return super.set_primary_action(label, click)
+			.removeClass('hidden');
+	}
+
+	make() {
+		super.make();
+		if (this.fields) {
+			this.$wrapper.find('.section-body').addClass('w-100');
+		}
+	}
+};

frappe/public/js/chat.bundle.js

@@ -0,0 +1 @@
+import "./frappe/chat";

frappe/public/js/checkout.bundle.js

@@ -0,0 +1 @@
+import "./integrations/razorpay";

frappe/public/js/controls.bundle.js

@@ -0,0 +1,18 @@
+import "air-datepicker/dist/js/datepicker.min.js";
+import "air-datepicker/dist/js/i18n/datepicker.cs.js";
+import "air-datepicker/dist/js/i18n/datepicker.da.js";
+import "air-datepicker/dist/js/i18n/datepicker.de.js";
+import "air-datepicker/dist/js/i18n/datepicker.en.js";
+import "air-datepicker/dist/js/i18n/datepicker.es.js";
+import "air-datepicker/dist/js/i18n/datepicker.fi.js";
+import "air-datepicker/dist/js/i18n/datepicker.fr.js";
+import "air-datepicker/dist/js/i18n/datepicker.hu.js";
+import "air-datepicker/dist/js/i18n/datepicker.nl.js";
+import "air-datepicker/dist/js/i18n/datepicker.pl.js";
+import "air-datepicker/dist/js/i18n/datepicker.pt-BR.js";
+import "air-datepicker/dist/js/i18n/datepicker.pt.js";
+import "air-datepicker/dist/js/i18n/datepicker.ro.js";
+import "air-datepicker/dist/js/i18n/datepicker.sk.js";
+import "air-datepicker/dist/js/i18n/datepicker.zh.js";
+import "./frappe/ui/capture.js";
+import "./frappe/form/controls/control.js";

frappe/public/js/data_import_tools.bundle.js

@@ -0,0 +1 @@
+import "./frappe/data_import";

frappe/public/js/desk.bundle.js

@@ -0,0 +1,105 @@
+import "./frappe/translate.js";
+import "./frappe/class.js";
+import "./frappe/polyfill.js";
+import "./frappe/provide.js";
+import "./frappe/assets.js";
+import "./frappe/format.js";
+import "./frappe/form/formatters.js";
+import "./frappe/dom.js";
+import "./frappe/ui/messages.js";
+import "./frappe/ui/keyboard.js";
+import "./frappe/ui/colors.js";
+import "./frappe/ui/sidebar.js";
+import "./frappe/ui/link_preview.js";
+
+import "./frappe/request.js";
+import "./frappe/socketio_client.js";
+import "./frappe/utils/utils.js";
+import "./frappe/event_emitter.js";
+import "./frappe/router.js";
+import "./frappe/router_history.js";
+import "./frappe/defaults.js";
+import "./frappe/roles_editor.js";
+import "./frappe/module_editor.js";
+import "./frappe/microtemplate.js";
+
+import "./frappe/ui/page.html";
+import "./frappe/ui/page.js";
+import "./frappe/ui/slides.js";
+// import "./frappe/ui/onboarding_dialog.js";
+import "./frappe/ui/find.js";
+import "./frappe/ui/iconbar.js";
+import "./frappe/form/layout.js";
+import "./frappe/ui/field_group.js";
+import "./frappe/form/link_selector.js";
+import "./frappe/form/multi_select_dialog.js";
+import "./frappe/ui/dialog.js";
+import "./frappe/ui/capture.js";
+import "./frappe/ui/app_icon.js";
+import "./frappe/ui/theme_switcher.js";
+
+import "./frappe/model/model.js";
+import "./frappe/db.js";
+import "./frappe/model/meta.js";
+import "./frappe/model/sync.js";
+import "./frappe/model/create_new.js";
+import "./frappe/model/perm.js";
+import "./frappe/model/workflow.js";
+import "./frappe/model/user_settings.js";
+
+import "./frappe/utils/user.js";
+import "./frappe/utils/common.js";
+import "./frappe/utils/urllib.js";
+import "./frappe/utils/pretty_date.js";
+import "./frappe/utils/tools.js";
+import "./frappe/utils/datetime.js";
+import "./frappe/utils/number_format.js";
+import "./frappe/utils/help.js";
+import "./frappe/utils/help_links.js";
+import "./frappe/utils/address_and_contact.js";
+import "./frappe/utils/preview_email.js";
+import "./frappe/utils/file_manager.js";
+
+import "./frappe/upload.js";
+import "./frappe/ui/tree.js";
+
+import "./frappe/views/container.js";
+import "./frappe/views/breadcrumbs.js";
+import "./frappe/views/factory.js";
+import "./frappe/views/pageview.js";
+
+import "./frappe/ui/toolbar/awesome_bar.js";
+// import "./frappe/ui/toolbar/energy_points_notifications.js";
+import "./frappe/ui/notifications/notifications.js";
+import "./frappe/ui/toolbar/search.js";
+import "./frappe/ui/toolbar/tag_utils.js";
+import "./frappe/ui/toolbar/search.html";
+import "./frappe/ui/toolbar/search_utils.js";
+import "./frappe/ui/toolbar/about.js";
+import "./frappe/ui/toolbar/navbar.html";
+import "./frappe/ui/toolbar/toolbar.js";
+// import "./frappe/ui/toolbar/notifications.js";
+import "./frappe/views/communication.js";
+import "./frappe/views/translation_manager.js";
+import "./frappe/views/workspace/workspace.js";
+
+import "./frappe/widgets/widget_group.js";
+
+import "./frappe/ui/sort_selector.html";
+import "./frappe/ui/sort_selector.js";
+
+import "./frappe/change_log.html";
+import "./frappe/ui/workspace_loading_skeleton.html";
+import "./frappe/desk.js";
+import "./frappe/query_string.js";
+
+// import "./frappe/ui/comment.js";
+
+import "./frappe/chat.js";
+import "./frappe/utils/energy_point_utils.js";
+import "./frappe/utils/dashboard_utils.js";
+import "./frappe/ui/chart.js";
+import "./frappe/ui/datatable.js";
+import "./frappe/ui/driver.js";
+import "./frappe/ui/plyr.js";
+import "./frappe/barcode_scanner/index.js";

frappe/public/js/dialog.bundle.js

@@ -0,0 +1,7 @@
+import "./frappe/dom.js";
+import "./frappe/form/formatters.js";
+import "./frappe/form/layout.js";
+import "./frappe/ui/field_group.js";
+import "./frappe/form/link_selector.js";
+import "./frappe/form/multi_select_dialog.js";
+import "./frappe/ui/dialog.js";

frappe/public/js/form.bundle.js

@@ -0,0 +1,17 @@
+import "./frappe/form/templates/address_list.html";
+import "./frappe/form/templates/contact_list.html";
+import "./frappe/form/templates/form_dashboard.html";
+import "./frappe/form/templates/form_footer.html";
+import "./frappe/form/templates/form_links.html";
+import "./frappe/form/templates/form_sidebar.html";
+import "./frappe/form/templates/print_layout.html";
+import "./frappe/form/templates/report_links.html";
+import "./frappe/form/templates/set_sharing.html";
+import "./frappe/form/templates/timeline_message_box.html";
+import "./frappe/form/templates/users_in_sidebar.html";
+
+import "./frappe/form/controls/control.js";
+import "./frappe/views/formview.js";
+import "./frappe/form/form.js";
+import "./frappe/meta_tag.js";
+

frappe/public/js/frappe-web.bundle.js

@@ -0,0 +1,26 @@
+import "./jquery-bootstrap";
+import "./frappe/class.js";
+import "./frappe/polyfill.js";
+import "./lib/md5.min.js";
+import "./frappe/provide.js";
+import "./frappe/format.js";
+import "./frappe/utils/number_format.js";
+import "./frappe/utils/utils.js";
+import "./frappe/utils/common.js";
+import "./frappe/ui/messages.js";
+import "./frappe/translate.js";
+import "./frappe/utils/pretty_date.js";
+import "./frappe/microtemplate.js";
+import "./frappe/query_string.js";
+
+import "./frappe/upload.js";
+
+import "./frappe/model/meta.js";
+import "./frappe/model/model.js";
+import "./frappe/model/perm.js";
+
+import "./bootstrap-4-web.bundle";
+
+
+import "../../website/js/website.js";
+import "./frappe/socketio_client.js";

frappe/public/js/frappe/assets.js

@@ -9,7 +9,14 @@ frappe.require = function(items, callback) {
 	if(typeof items === "string") {
 		items = [items];
 	}
-	frappe.assets.execute(items, callback);
+	items = items.map(item => frappe.assets.bundled_asset(item));
+
+	return new Promise(resolve => {
+		frappe.assets.execute(items, () => {
+			resolve();
+			callback && callback();
+		});
+	});
 };
 
 frappe.assets = {
@@ -160,4 +167,11 @@ frappe.assets = {
 			frappe.dom.set_style(txt);
 		}
 	},
+
+	bundled_asset(path) {
+		if (!path.startsWith('/assets') && path.includes('.bundle.')) {
+			return frappe.boot.assets_json[path] || path;
+		}
+		return path;
+	}
 };

frappe/public/js/frappe/barcode_scanner/index.js

@@ -13,7 +13,7 @@ frappe.barcode.scan_barcode = function() {
 				}
 			}, reject);
 		} else {
-			frappe.require('/assets/js/barcode_scanner.min.js', () => {
+			frappe.require('barcode_scanner.bundle.js', () => {
 				frappe.barcode.get_barcode().then(barcode => {
 					resolve(barcode);
 				});