Merge branch 'develop' into virtual_doctype

.codacy.yml

@@ -1,2 +0,0 @@
-exclude_paths:
-  - '**.sql'

.deepsource.toml

@@ -1,17 +0,0 @@
-version = 1
-
-test_patterns = [
-  "**/test_*.py"
-]
-
-exclude_patterns = [
-  "frappe/patches/**",
-  "*.min.js"
-]
-
-[[analyzers]]
-name = "python"
-enabled = true
-
-  [analyzers.meta]
-  runtime_version = "3.x.x"

.eslintignore

@@ -5,5 +5,4 @@ frappe/core/doctype/doctype/boilerplate/*
 frappe/core/doctype/report/boilerplate/*
 frappe/public/js/frappe/class.js
 frappe/templates/includes/*
-frappe/tests/testcafe/*
-frappe/www/website_script.js
+frappe/www/website_script.js

.eslintrc

@@ -147,6 +147,7 @@
 		"context": true,
 		"before": true,
 		"beforeEach": true,
-		"qz": true
+		"qz": true,
+		"localforage": true
 	}
 }

.flake8

@@ -0,0 +1,32 @@
+[flake8]
+ignore =
+    E121,
+    E126,
+    E127,
+    E128,
+    E203,
+    E225,
+    E226,
+    E231,
+    E241,
+    E251,
+    E261,
+    E265,
+    E302,
+    E303,
+    E305,
+    E402,
+    E501,
+    E741,
+    W291,
+    W292,
+    W293,
+    W391,
+    W503,
+    W504,
+    F403,
+    B007,
+    B950,
+    W191,
+
+max-line-length = 200

.github/helper/consumer_db/mariadb.json

@@ -1,5 +1,6 @@
 {
-    "db_host": "localhost",
+    "db_host": "127.0.0.1",
+    "db_port": 3306,
     "db_name": "test_frappe_consumer",
     "db_password": "test_frappe",
     "allow_tests": true,

.github/helper/consumer_db/postgres.json

@@ -1,5 +1,6 @@
 {
-    "db_host": "localhost",
+    "db_host": "127.0.0.1",
+    "db_port": 5432,
     "db_name": "test_frappe_consumer",
     "db_password": "test_frappe",
     "db_type": "postgres",

.github/helper/install.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+
+set -e
+
+cd ~ || exit
+
+pip install frappe-bench
+
+bench init frappe-bench --skip-assets --python "$(which python)" --frappe-path "${GITHUB_WORKSPACE}"
+
+mkdir ~/frappe-bench/sites/test_site
+cp "${GITHUB_WORKSPACE}/.github/helper/consumer_db/$DB.json" ~/frappe-bench/sites/test_site/site_config.json
+
+if [ "$TYPE" == "server" ]; then
+      mkdir ~/frappe-bench/sites/test_site_producer;
+      cp "${GITHUB_WORKSPACE}/.github/helper/producer_db/$DB.json" ~/frappe-bench/sites/test_site_producer/site_config.json;
+fi
+
+if [ "$DB" == "mariadb" ];then
+      mysql --host 127.0.0.1 --port 3306 -u root -e "SET GLOBAL character_set_server = 'utf8mb4'";
+      mysql --host 127.0.0.1 --port 3306 -u root -e "SET GLOBAL collation_server = 'utf8mb4_unicode_ci'";
+
+      mysql --host 127.0.0.1 --port 3306 -u root -e "CREATE DATABASE test_frappe_consumer";
+      mysql --host 127.0.0.1 --port 3306 -u root -e "CREATE USER 'test_frappe_consumer'@'localhost' IDENTIFIED BY 'test_frappe_consumer'";
+      mysql --host 127.0.0.1 --port 3306 -u root -e "GRANT ALL PRIVILEGES ON \`test_frappe_consumer\`.* TO 'test_frappe_consumer'@'localhost'";
+
+      mysql --host 127.0.0.1 --port 3306 -u root -e "CREATE DATABASE test_frappe_producer";
+      mysql --host 127.0.0.1 --port 3306 -u root -e "CREATE USER 'test_frappe_producer'@'localhost' IDENTIFIED BY 'test_frappe_producer'";
+      mysql --host 127.0.0.1 --port 3306 -u root -e "GRANT ALL PRIVILEGES ON \`test_frappe_producer\`.* TO 'test_frappe_producer'@'localhost'";
+
+      mysql --host 127.0.0.1 --port 3306 -u root -e "UPDATE mysql.user SET Password=PASSWORD('travis') WHERE User='root'";
+      mysql --host 127.0.0.1 --port 3306 -u root -e "FLUSH PRIVILEGES";
+    fi
+
+if [ "$DB" == "postgres" ];then
+    echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE DATABASE test_frappe_consumer" -U postgres;
+    echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE USER test_frappe_consumer WITH PASSWORD 'test_frappe'" -U postgres;
+
+    echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE DATABASE test_frappe_producer" -U postgres;
+    echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE USER test_frappe_producer WITH PASSWORD 'test_frappe'" -U postgres;
+fi
+
+cd ./frappe-bench || exit
+
+sed -i 's/^watch:/# watch:/g' Procfile
+sed -i 's/^schedule:/# schedule:/g' Procfile
+
+if [ "$TYPE" == "server" ]; then sed -i 's/^socketio:/# socketio:/g' Procfile; fi
+if [ "$TYPE" == "server" ]; then sed -i 's/^redis_socketio:/# redis_socketio:/g' Procfile; fi
+
+if [ "$TYPE" == "ui" ]; then bench setup requirements --node; fi
+
+# install node-sass which is required for website theme test
+cd ./apps/frappe || exit
+yarn add node-sass@4.13.1
+cd ../..
+
+bench start &
+bench --site test_site reinstall --yes
+if [ "$TYPE" == "server" ]; then bench --site test_site_producer reinstall --yes; fi
+bench build --app frappe

.github/helper/install_dependencies.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -e
+
+# python "${GITHUB_WORKSPACE}/.github/helper/roulette.py"
+# if [[ $? != 2 ]];then
+#   exit;
+# fi
+
+ # install wkhtmltopdf
+wget -O /tmp/wkhtmltox.tar.xz https://github.com/frappe/wkhtmltopdf/raw/master/wkhtmltox-0.12.3_linux-generic-amd64.tar.xz
+tar -xf /tmp/wkhtmltox.tar.xz -C /tmp
+sudo mv /tmp/wkhtmltox/bin/wkhtmltopdf /usr/local/bin/wkhtmltopdf
+sudo chmod o+x /usr/local/bin/wkhtmltopdf
+
+# install cups
+sudo apt-get install libcups2-dev
+
+# install redis
+sudo apt-get install redis-server
+

.github/helper/producer_db/mariadb.json

@@ -1,5 +1,6 @@
 {
-    "db_host": "localhost",
+    "db_host": "127.0.0.1",
+    "db_port": 3306,
     "db_name": "test_frappe_producer",
     "db_password": "test_frappe",
     "allow_tests": true,

.github/helper/producer_db/postgres.json

@@ -1,5 +1,6 @@
 {
-    "db_host": "localhost",
+    "db_host": "127.0.0.1",
+    "db_port": 5432,
     "db_name": "test_frappe_producer",
     "db_password": "test_frappe",
     "db_type": "postgres",

.github/helper/roulette.py

@@ -24,10 +24,12 @@ def is_docs(file):
 
 if __name__ == "__main__":
     build_type = os.environ.get("TYPE")
-    commit_range = os.environ.get("TRAVIS_COMMIT_RANGE")
+    before = os.environ.get("BEFORE")
+    after = os.environ.get("AFTER")
+    commit_range = before + '...' + after
     print("Build Type: {}".format(build_type))
     print("Commit Range: {}".format(commit_range))
-    
+
     try:
         files_changed = get_output("git diff --name-only {}".format(commit_range), shell=False)
     except Exception:

.github/workflows/ci-tests.yml

@@ -0,0 +1,148 @@
+name: CI
+
+on: [pull_request, workflow_dispatch, push]
+
+jobs:
+  test:
+    runs-on: ubuntu-18.04
+
+    strategy:
+      fail-fast: false
+      matrix:
+       include:
+        - DB: "mariadb"
+          TYPE: "server"
+          JOB_NAME: "Python MariaDB"
+          RUN_COMMAND: bench --site test_site run-tests --coverage
+
+        - DB: "postgres"
+          TYPE: "server"
+          JOB_NAME: "Python PostgreSQL"
+          RUN_COMMAND: bench --site test_site run-tests --coverage
+
+        - DB: "mariadb"
+          TYPE: "ui"
+          JOB_NAME: "UI MariaDB"
+          RUN_COMMAND:  bench --site test_site run-ui-tests frappe --headless
+
+    name: ${{ matrix.JOB_NAME }}
+
+    services:
+      mysql:
+        image: mariadb:10.3
+        env:
+          MYSQL_ALLOW_EMPTY_PASSWORD: YES
+        ports:
+          - 3306:3306
+        options: --health-cmd="mysqladmin ping" --health-interval=5s --health-timeout=2s --health-retries=3
+
+      postgres:
+        image: postgres:12.4
+        env:
+          POSTGRES_PASSWORD: travis
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
+    steps:
+      - name: Clone
+        uses: actions/checkout@v2
+
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+
+      - uses: actions/setup-node@v2
+        with:
+          node-version: '12'
+          check-latest: true
+
+      - name: Add to Hosts
+        run: |
+          echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts
+          echo "127.0.0.1 test_site_producer" | sudo tee -a /etc/hosts
+
+      - name: Cache pip
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+            ${{ runner.os }}-
+
+      - name: Cache node modules
+        uses: actions/cache@v2
+        env:
+          cache-name: cache-node-modules
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-build-${{ env.cache-name }}-
+            ${{ runner.os }}-build-
+            ${{ runner.os }}-
+
+      - name: Get yarn cache directory path
+        id: yarn-cache-dir-path
+        run: echo "::set-output name=dir::$(yarn cache dir)"
+
+      - uses: actions/cache@v2
+        id: yarn-cache
+        with:
+          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
+          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-yarn-
+
+      - name: Cache cypress binary
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache
+          key: ${{ runner.os }}-cypress-
+          restore-keys: |
+            ${{ runner.os }}-cypress-
+            ${{ runner.os }}-
+
+      - name: Install Dependencies
+        run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh
+        env:
+          BEFORE: ${{ env.GITHUB_EVENT_PATH.before }}
+          AFTER: ${{ env.GITHUB_EVENT_PATH.after }}
+          TYPE: ${{ matrix.TYPE }}
+
+      - name: Install
+        run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh
+        env:
+          DB: ${{ matrix.DB }}
+          TYPE: ${{ matrix.TYPE }}
+
+      - name: Run Set-Up
+        if: matrix.TYPE == 'ui'
+        run: cd ~/frappe-bench/ && bench --site test_site execute frappe.utils.install.complete_setup_wizard
+        env:
+          DB: ${{ matrix.DB }}
+          TYPE: ${{ matrix.TYPE }}
+
+      - name: Run Tests
+        run: cd ~/frappe-bench/ && ${{ matrix.RUN_COMMAND }}
+        env:
+          DB: ${{ matrix.DB }}
+          TYPE: ${{ matrix.TYPE }}
+
+      - name: Coverage
+        if: matrix.TYPE == 'server'
+        run: |
+          cp ~/frappe-bench/sites/.coverage ${GITHUB_WORKSPACE}
+          cd ${GITHUB_WORKSPACE}
+          pip install coveralls==2.2.0
+          pip install coverage==4.5.4
+          coveralls
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_TOKEN }}

.github/workflows/semgrep.yml

@@ -0,0 +1,22 @@
+name: Semgrep
+
+on:
+  pull_request:
+    branches:
+      - develop
+jobs:
+  semgrep:
+    name: Frappe Linter
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Setup python3
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+    - name: Run semgrep
+      run: |
+        python -m pip install -q semgrep
+        git fetch origin $GITHUB_BASE_REF:$GITHUB_BASE_REF -q
+        files=$(git diff --name-only --diff-filter=d $GITHUB_BASE_REF)
+        if [ -f .semgrep.yml ]; then semgrep --config=.semgrep.yml --quiet --error $files; fi

.mergify.yml

@@ -3,7 +3,9 @@ pull_request_rules:
     conditions:
       - status-success=Sider
       - status-success=Semantic Pull Request
-      - status-success=Travis CI - Pull Request
+      - status-success=Python MariaDB
+      - status-success=Python PostgreSQL
+      - status-success=UI MariaDB
       - status-success=security/snyk (frappe)
       - label!=dont-merge
       - label!=squash
@@ -14,8 +16,9 @@ pull_request_rules:
   - name: Automatic squash on CI success and review
     conditions:
       - status-success=Sider
-      - status-success=Semantic Pull Request
-      - status-success=Travis CI - Pull Request
+      - status-success=Python MariaDB
+      - status-success=Python PostgreSQL
+      - status-success=UI MariaDB
       - status-success=security/snyk (frappe)
       - label!=dont-merge
       - label=squash

.semgrep.yml

@@ -0,0 +1,29 @@
+#Reference: https://semgrep.dev/docs/writing-rules/rule-syntax/
+
+rules:
+- id: eval
+  patterns:
+  - pattern-not: eval("...")
+  - pattern: eval(...)
+  message: |
+    Detected the use of eval(). eval() can be dangerous if used to evaluate
+    dynamic content. Avoid it or use safe_eval().
+  languages:
+  - python
+  severity: ERROR
+
+# translations
+- id: frappe-translation-syntax-python
+  pattern-either:
+  - pattern: _(f"...")  # f-strings not allowed
+  - pattern: _("..." + "...")  # concatenation not allowed
+  - pattern: _("")  # empty string is meaningless
+  - pattern: _("..." % ...)  # Only positional formatters are allowed.
+  - pattern: _("...".format(...))  # format should not be used before translating
+  - pattern: _("...") + ... + _("...")  # don't split strings
+  message: |
+    Incorrect use of translation function detected.
+    Please refer: https://frappeframework.com/docs/user/en/translations
+  languages:
+  - python
+  severity: ERROR

.travis.yml

@@ -1,129 +0,0 @@
-language: python
-dist: bionic
-
-addons:
-  hosts:
-    - test_site
-    - test_site_producer
-  mariadb: 10.3
-  postgresql: 9.5
-  chrome: stable
-
-services:
-  - xvfb
-  - mysql
-
-git:
-  depth: 1
-
-cache:
-  pip: true
-  npm: true
-  yarn: true
-  directories:
-    # we also need to cache folder with Cypress binary
-    # https://docs.cypress.io/guides/guides/continuous-integration.html#Caching
-    - ~/.cache
-
-
-matrix:
-  include:
-  - name: "Python 3.7 MariaDB"
-    python: 3.7
-    env: DB=mariadb TYPE=server
-    script: bench --verbose --site test_site run-tests --coverage
-
-  - name: "Python 3.7 PostgreSQL"
-    python: 3.7
-    env: DB=postgres TYPE=server
-    script: bench --verbose --site test_site run-tests --coverage
-
-  - name: "Cypress"
-    python: 3.7
-    env: DB=mariadb TYPE=ui
-    before_script:
-    - bench --site test_site execute frappe.utils.install.complete_setup_wizard
-    script: bench --site test_site run-ui-tests frappe --headless
-
-before_install:
- # do we really want to run travis?
-  - |
-    python ./.travis/roulette.py
-    if [[ $? != 2 ]];then
-      exit;
-    fi
-
- # install wkhtmltopdf
-  - wget -O /tmp/wkhtmltox.tar.xz https://github.com/frappe/wkhtmltopdf/raw/master/wkhtmltox-0.12.3_linux-generic-amd64.tar.xz
-  - tar -xf /tmp/wkhtmltox.tar.xz -C /tmp
-  - sudo mv /tmp/wkhtmltox/bin/wkhtmltopdf /usr/local/bin/wkhtmltopdf
-  - sudo chmod o+x /usr/local/bin/wkhtmltopdf
-
-  # install cups
-  - sudo apt-get install libcups2-dev
-
-install:
-  - cd ~
-  - source ./.nvm/nvm.sh
-  - nvm install 12
-
-  - pip install frappe-bench
-
-  - bench init frappe-bench --skip-assets --python $(which python) --frappe-path $TRAVIS_BUILD_DIR
-
-  - mkdir ~/frappe-bench/sites/test_site
-  - cp $TRAVIS_BUILD_DIR/.travis/consumer_db/$DB.json ~/frappe-bench/sites/test_site/site_config.json
-
-  - if [ $TYPE == "server" ]; then
-      mkdir ~/frappe-bench/sites/test_site_producer;
-      cp $TRAVIS_BUILD_DIR/.travis/producer_db/$DB.json ~/frappe-bench/sites/test_site_producer/site_config.json;
-    fi
-
-  - if [ $DB == "mariadb" ];then
-      mysql -u root -e "SET GLOBAL character_set_server = 'utf8mb4'";
-      mysql -u root -e "SET GLOBAL collation_server = 'utf8mb4_unicode_ci'";
-
-      mysql -u root -e "CREATE DATABASE test_frappe_consumer";
-      mysql -u root -e "CREATE USER 'test_frappe_consumer'@'localhost' IDENTIFIED BY 'test_frappe_consumer'";
-      mysql -u root -e "GRANT ALL PRIVILEGES ON \`test_frappe_consumer\`.* TO 'test_frappe_consumer'@'localhost'";
-
-      mysql -u root -e "CREATE DATABASE test_frappe_producer";
-      mysql -u root -e "CREATE USER 'test_frappe_producer'@'localhost' IDENTIFIED BY 'test_frappe_producer'";
-      mysql -u root -e "GRANT ALL PRIVILEGES ON \`test_frappe_producer\`.* TO 'test_frappe_producer'@'localhost'";
-
-      mysql -u root -e "UPDATE mysql.user SET Password=PASSWORD('travis') WHERE User='root'";
-      mysql -u root -e "FLUSH PRIVILEGES";
-    fi
-
-  - if [ $DB == "postgres" ];then
-      psql -c "CREATE DATABASE test_frappe_consumer" -U postgres;
-      psql -c "CREATE USER test_frappe_consumer WITH PASSWORD 'test_frappe'" -U postgres;
-
-      psql -c "CREATE DATABASE test_frappe_producer" -U postgres;
-      psql -c "CREATE USER test_frappe_producer WITH PASSWORD 'test_frappe'" -U postgres;
-    fi
-
-  - cd ./frappe-bench
-
-  - sed -i 's/^watch:/# watch:/g' Procfile
-  - sed -i 's/^schedule:/# schedule:/g' Procfile
-
-  - if [ $TYPE == "server" ]; then sed -i 's/^socketio:/# socketio:/g' Procfile; fi
-  - if [ $TYPE == "server" ]; then sed -i 's/^redis_socketio:/# redis_socketio:/g' Procfile; fi
-
-  - if [ $TYPE == "ui" ]; then bench setup requirements --node; fi
-
-  # install node-sass which is required for website theme test
-  - cd ./apps/frappe
-  - yarn add node-sass@4.13.1
-  - cd ../..
-
-  - bench start &
-  - bench --site test_site reinstall --yes
-  - if [ $TYPE == "server" ]; then bench --site test_site_producer reinstall --yes; fi
-  - bench build --app frappe
-
-after_script:
-  - pip install coverage==4.5.4
-  - pip install python-coveralls
-  - coveralls -b apps/frappe -d ../../sites/.coverage

CODEOWNERS

@@ -8,10 +8,10 @@ website/					@prssanna
 web_form/					@prssanna
 templates/				@surajshetty3416
 www/							@surajshetty3416
-integrations/			@nextchamp-saqib
+integrations/			@leela
 patches/					@surajshetty3416
 dashboard/				@prssanna
-email/						@saurabh6790
+email/						@leela
 event_streaming/	@ruchamahabal
 data_import*			@netchampfaris
 core/							@surajshetty3416

README.md

@@ -14,8 +14,8 @@
 </div>
 
 <div align="center">
-    <a href="https://travis-ci.com/frappe/frappe">
-        <img src="https://travis-ci.com/frappe/frappe.svg?branch=develop">
+    <a href="https://github.com/frappe/frappe/actions/workflows/ci-tests.yml">
+        <img src="https://github.com/frappe/frappe/actions/workflows/ci-tests.yml/badge.svg?branch=develop">
     </a>
     <a href='https://frappeframework.com/docs'>
         <img src='https://img.shields.io/badge/docs-📖-7575FF.svg?style=flat-square'/>

ci/fix-mariadb.sh

@@ -1,11 +0,0 @@
-#!/bin/bash
-
-# stolen from http://cgit.drupalcode.org/octopus/commit/?id=db4f837
-includedir=`mysql_config --variable=pkgincludedir`
-thiscwd=`pwd`
-_THIS_DB_VERSION=`mysql -V 2>&1 | tr -d "\n" | cut -d" " -f6 | awk '{ print $1}' | cut -d"-" -f1 | awk '{ print $1}' | sed "s/[\,']//g"`
-if [ "$_THIS_DB_VERSION" = "5.5.40" ] && [ ! -e "$includedir-$_THIS_DB_VERSION-fixed.log" ] ; then
-  cd $includedir
-  sudo patch -p1 < $thiscwd/ci/my_config.h.patch &> /dev/null
-  sudo touch $includedir-$_THIS_DB_VERSION-fixed.log
-fi

ci/my_config.h.patch

@@ -1,22 +0,0 @@
-diff -burp a/my_config.h b/my_config.h
---- a/my_config.h	2014-10-09 19:32:46.000000000 -0400
-+++ b/my_config.h	2014-10-09 19:35:12.000000000 -0400
-@@ -641,17 +641,4 @@
- #define SIZEOF_TIME_T 8
- /* #undef TIME_T_UNSIGNED */
-
--/*
--  stat structure (from <sys/stat.h>) is conditionally defined
--  to have different layout and size depending on the defined macros.
--  The correct macro is defined in my_config.h, which means it MUST be
--  included first (or at least before <features.h> - so, practically,
--  before including any system headers).
--
--  __GLIBC__ is defined in <features.h>
--*/
--#ifdef __GLIBC__
--#error <my_config.h> MUST be included first!
--#endif
--
- #endif
-

cypress/integration/control_rating.js

@@ -18,7 +18,7 @@ context('Control Rating', () => {
 		get_dialog_with_rating().as('dialog');
 
 		cy.get('div.rating')
-			.children('i.fa')
+			.children('svg')
 			.first()
 			.click()
 			.should('have.class', 'star-click');
@@ -33,11 +33,11 @@ context('Control Rating', () => {
 		get_dialog_with_rating();
 
 		cy.get('div.rating')
-			.children('i.fa')
+			.children('svg')
 			.first()
 			.invoke('trigger', 'mouseenter')
 			.should('have.class', 'star-hover')
 			.invoke('trigger', 'mouseleave')
 			.should('not.have.class', 'star-hover');
 	});
-});
+});

cypress/integration/control_select.js

@@ -0,0 +1,36 @@
+context('Control Select', () => {
+	before(() => {
+		cy.login();
+		cy.visit('/app/website');
+	});
+
+	function get_dialog_with_select() {
+		return cy.dialog({
+			title: 'Select',
+			fields: [{
+				'fieldname': 'select_control',
+				'fieldtype': 'Select',
+				'placeholder': 'Select an Option',
+				'options': ['', 'Option 1', 'Option 2', 'Option 2'],
+			}]
+		});
+	}
+
+	it('toggles placholder on clicking an option', () => {
+		get_dialog_with_select().as('dialog');
+
+		cy.get('.frappe-control[data-fieldname=select_control] .control-input').as('control');
+		cy.get('.frappe-control[data-fieldname=select_control] .control-input select').as('select');
+		cy.get('@control').get('.select-icon').should('exist');
+		cy.get('@control').get('.placeholder').should('have.css', 'display', 'block');
+		cy.get('@select').select('Option 1');
+		cy.get('@control').get('.placeholder').should('have.css', 'display', 'none');
+		cy.get('@select').invoke('val', '');
+		cy.get('@control').get('.placeholder').should('have.css', 'display', 'block');
+
+
+		cy.get('@dialog').then(dialog => {
+			dialog.hide();
+		});
+	});
+});

cypress/integration/recorder.js

@@ -3,6 +3,16 @@ context('Recorder', () => {
 		cy.login();
 	});
 
+	beforeEach(() => {
+		cy.visit('/app/recorder');
+		return cy.window().its('frappe').then(frappe => {
+			// reset recorder
+			return frappe.xcall("frappe.recorder.stop").then(() => {
+				return frappe.xcall("frappe.recorder.delete");
+			});
+		});
+	});
+
 	it('Navigate to Recorder', () => {
 		cy.visit('/app');
 		cy.awesomebar('recorder');
@@ -11,7 +21,6 @@ context('Recorder', () => {
 	});
 
 	it('Recorder Empty State', () => {
-		cy.visit('/app/recorder');
 		cy.get('.title-text').should('contain', 'Recorder');
 
 		cy.get('.indicator-pill').should('contain', 'Inactive').should('have.class', 'red');
@@ -24,7 +33,6 @@ context('Recorder', () => {
 	});
 
 	it('Recorder Start', () => {
-		cy.visit('/app/recorder');
 		cy.get('.primary-action').should('contain', 'Start').click();
 		cy.get('.indicator-pill').should('contain', 'Active').should('have.class', 'green');
 
@@ -40,15 +48,9 @@ context('Recorder', () => {
 		cy.visit('/app/recorder');
 		cy.get('.title-text').should('contain', 'Recorder');
 		cy.get('.result-list').should('contain', '/api/method/frappe.desk.reportview.get');
-
-		cy.get('#page-recorder .primary-action').should('contain', 'Stop').click();
-		cy.wait(500);
-		cy.get('#page-recorder .btn-secondary').should('contain', 'Clear').click();
-		cy.get('.msg-box').should('contain', 'Inactive');
 	});
 
-	it('Recorder View Request', () => {
-		cy.visit('/app/recorder');
+	it.only('Recorder View Request', () => {
 		cy.get('.primary-action').should('contain', 'Start').click();
 
 		cy.visit('/app/List/DocType/List');
@@ -64,9 +66,5 @@ context('Recorder', () => {
 
 		cy.url().should('include', '/recorder/request');
 		cy.get('form').should('contain', '/api/method/frappe');
-
-		cy.get('#page-recorder .primary-action').should('contain', 'Stop').click();
-		cy.wait(200);
-		cy.get('#page-recorder .btn-secondary').should('contain', 'Clear').click();
 	});
-});
+});

cypress/integration/table_multiselect.js

@@ -8,7 +8,7 @@ context('Table MultiSelect', () => {
 	it('select value from multiselect dropdown', () => {
 		cy.new_form('Assignment Rule');
 		cy.fill_field('__newname', name);
-		cy.fill_field('document_type', 'ToDo');
+		cy.fill_field('document_type', 'Blog Post');
 		cy.fill_field('assign_condition', 'status=="Open"', 'Code');
 		cy.get('input[data-fieldname="users"]').focus().as('input');
 		cy.get('input[data-fieldname="users"] + ul').should('be.visible');
@@ -45,6 +45,6 @@ context('Table MultiSelect', () => {
 		cy.get(`.list-subject:contains("table multiselect")`).last().find('a').click();
 		cy.get('.frappe-control[data-fieldname="users"] .form-control .tb-selected-value').as('existing_value');
 		cy.get('@existing_value').find('.btn-link-to-form').click();
-		cy.location('pathname').should('contain', '/user/test@erpnext.com');
+		cy.location('pathname').should('contain', '/user/test%40erpnext.com');
 	});
 });

cypress/support/commands.js

@@ -275,7 +275,7 @@ Cypress.Commands.add('get_open_dialog', () => {
 });
 
 Cypress.Commands.add('hide_dialog', () => {
-	cy.wait(200);
+	cy.wait(300);
 	cy.get_open_dialog().find('.btn-modal-close').click();
 	cy.get('.modal:visible').should('not.exist');
 });
@@ -312,7 +312,6 @@ Cypress.Commands.add('add_filter', () => {
 	cy.get('.filter-section .filter-button').click();
 	cy.wait(300);
 	cy.get('.filter-popover').should('exist');
-	cy.get('.filter-popover').find('.add-filter').click();
 });
 
 Cypress.Commands.add('clear_filters', () => {

frappe/__init__.py

@@ -1,8 +1,14 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # MIT License. See license.txt
 """
-globals attached to frappe module
-+ some utility functions that should probably be moved
+Frappe - Low Code Open Source Framework in Python and JS
+
+Frappe, pronounced fra-pay, is a full stack, batteries-included, web
+framework written in Python and Javascript with MariaDB as the database.
+It is the framework which powers ERPNext. It is pretty generic and can
+be used to build database driven apps.
+
+Read the documentation: https://frappeframework.com/docs
 """
 from __future__ import unicode_literals, print_function
 
@@ -11,11 +17,15 @@ from werkzeug.local import Local, release_local
 import os, sys, importlib, inspect, json
 from past.builtins import cmp
 import click
-from faker import Faker
 
-# public
+# Local application imports
 from .exceptions import *
 from .utils.jinja import (get_jenv, get_template, render_template, get_email_from_template, get_jloader)
+from .utils.lazy_loader import lazy_import
+
+# Lazy imports
+faker = lazy_import('faker')
+
 
 # Harmless for Python 3
 # For Python 2 set default encoding to utf-8
@@ -190,17 +200,20 @@ def init(site, sites_path=None, new_site=False):
 
 	local.initialised = True
 
-def connect(site=None, db_name=None):
+def connect(site=None, db_name=None, set_admin_as_user=True):
 	"""Connect to site database instance.
 
 	:param site: If site is given, calls `frappe.init`.
-	:param db_name: Optional. Will use from `site_config.json`."""
+	:param db_name: Optional. Will use from `site_config.json`.
+	:param set_admin_as_user: Set Administrator as current user.
+	"""
 	from frappe.database import get_db
 	if site:
 		init(site)
 
 	local.db = get_db(user=db_name or local.conf.db_name)
-	set_user("Administrator")
+	if set_admin_as_user:
+		set_user("Administrator")
 
 def connect_replica():
 	from frappe.database import get_db
@@ -462,8 +475,8 @@ def get_request_header(key, default=None):
 
 def sendmail(recipients=[], sender="", subject="No Subject", message="No Message",
 		as_markdown=False, delayed=True, reference_doctype=None, reference_name=None,
-		unsubscribe_method=None, unsubscribe_params=None, unsubscribe_message=None,
-		attachments=None, content=None, doctype=None, name=None, reply_to=None,
+		unsubscribe_method=None, unsubscribe_params=None, unsubscribe_message=None, add_unsubscribe_link=1,
+		attachments=None, content=None, doctype=None, name=None, reply_to=None, queue_separately=False,
 		cc=[], bcc=[], message_id=None, in_reply_to=None, send_after=None, expose_recipients=None,
 		send_priority=1, communication=None, retry=1, now=None, read_receipt=None, is_notification=False,
 		inline_images=None, template=None, args=None, header=None, print_letterhead=False, with_container=False):
@@ -510,10 +523,10 @@ def sendmail(recipients=[], sender="", subject="No Subject", message="No Message
 	from frappe.email import queue
 	queue.send(recipients=recipients, sender=sender,
 		subject=subject, message=message, text_content=text_content,
-		reference_doctype = doctype or reference_doctype, reference_name = name or reference_name,
+		reference_doctype = doctype or reference_doctype, reference_name = name or reference_name, add_unsubscribe_link=add_unsubscribe_link,
 		unsubscribe_method=unsubscribe_method, unsubscribe_params=unsubscribe_params, unsubscribe_message=unsubscribe_message,
 		attachments=attachments, reply_to=reply_to, cc=cc, bcc=bcc, message_id=message_id, in_reply_to=in_reply_to,
-		send_after=send_after, expose_recipients=expose_recipients, send_priority=send_priority,
+		send_after=send_after, expose_recipients=expose_recipients, send_priority=send_priority, queue_separately=queue_separately,
 		communication=communication, now=now, read_receipt=read_receipt, is_notification=is_notification,
 		inline_images=inline_images, header=header, print_letterhead=print_letterhead, with_container=with_container)
 
@@ -1189,10 +1202,10 @@ def make_property_setter(args, ignore_validate=False, validate_fields_for_doctyp
 		ps.validate_fieldtype_change()
 		ps.insert()
 
-def import_doc(path, ignore_links=False, ignore_insert=False, insert=False):
+def import_doc(path):
 	"""Import a file using Data Import."""
 	from frappe.core.doctype.data_import.data_import import import_doc
-	import_doc(path, ignore_links=ignore_links, ignore_insert=ignore_insert, insert=insert)
+	import_doc(path)
 
 def copy_doc(doc, ignore_no_copy=True):
 	""" No_copy fields also get copied."""
@@ -1742,12 +1755,12 @@ def parse_json(val):
 
 def mock(type, size=1, locale='en'):
 	results = []
-	faker = Faker(locale)
-	if not type in dir(faker):
+	fake = faker.Faker(locale)
+	if type not in dir(fake):
 		raise ValueError('Not a valid mock type.')
 	else:
 		for i in range(size):
-			data = getattr(faker, type)()
+			data = getattr(fake, type)()
 			results.append(data)
 
 	from frappe.chat.util import squashify

frappe/app.py

@@ -128,6 +128,8 @@ def init_request(request):
 	if frappe.local.conf.get('maintenance_mode'):
 		frappe.connect()
 		raise frappe.SessionStopped('Session Stopped')
+	else:
+		frappe.connect(set_admin_as_user=False)
 
 	make_form_dict(request)
 
@@ -152,10 +154,10 @@ def process_response(response):
 
 def set_cors_headers(response):
 	origin = frappe.request.headers.get('Origin')
-	if not origin:
+	allow_cors = frappe.conf.allow_cors
+	if not (origin and allow_cors):
 		return
 
-	allow_cors = frappe.conf.allow_cors
 	if allow_cors != "*":
 		if not isinstance(allow_cors, list):
 			allow_cors = [allow_cors]
@@ -181,6 +183,9 @@ def make_form_dict(request):
 	else:
 		args = request.form or request.args
 
+	if not isinstance(args, dict):
+		frappe.throw("Invalid request arguments")
+
 	try:
 		frappe.local.form_dict = frappe._dict({ k:v[0] if isinstance(v, (list, tuple)) else v \
 			for k, v in iteritems(args) })

frappe/auth.py

@@ -207,23 +207,44 @@ class LoginManager:
 		if frappe.session.user != "Guest":
 			clear_sessions(frappe.session.user, keep_current=True)
 
-	def authenticate(self, user=None, pwd=None):
+	def authenticate(self, user: str = None, pwd: str = None):
+		from frappe.core.doctype.user.user import User
+
 		if not (user and pwd):
 			user, pwd = frappe.form_dict.get('usr'), frappe.form_dict.get('pwd')
 		if not (user and pwd):
 			self.fail(_('Incomplete login details'), user=user)
 
-		if cint(frappe.db.get_value("System Settings", "System Settings", "allow_login_using_mobile_number")):
-			user = frappe.db.get_value("User", filters={"mobile_no": user}, fieldname="name") or user
+		# Ignore password check if tmp_id is set, 2FA takes care of authentication.
+		validate_password = not bool(frappe.form_dict.get('tmp_id'))
+		user = User.find_by_credentials(user, pwd, validate_password=validate_password)
 
-		if cint(frappe.db.get_value("System Settings", "System Settings", "allow_login_using_user_name")):
-			user = frappe.db.get_value("User", filters={"username": user}, fieldname="name") or user
+		if not user:
+			self.fail('Invalid login credentials')
 
-		self.check_if_enabled(user)
-		if not frappe.form_dict.get('tmp_id'):
-			self.user = self.check_password(user, pwd)
+		sys_settings = frappe.get_doc("System Settings")
+		track_login_attempts = (sys_settings.allow_consecutive_login_attempts >0)
+
+		tracker_kwargs = {}
+		if track_login_attempts:
+			tracker_kwargs['lock_interval'] = sys_settings.allow_login_after_fail
+			tracker_kwargs['max_consecutive_login_attempts'] = sys_settings.allow_consecutive_login_attempts
+
+		tracker = LoginAttemptTracker(user.name, **tracker_kwargs)
+
+		if track_login_attempts and not tracker.is_user_allowed():
+			frappe.throw(_("Your account has been locked and will resume after {0} seconds")
+				.format(sys_settings.allow_login_after_fail), frappe.SecurityException)
+
+		if not user.is_authenticated:
+			tracker.add_failure_attempt()
+			self.fail('Invalid login credentials', user=user.name)
+		elif not (user.name == 'Administrator' or user.enabled):
+			tracker.add_failure_attempt()
+			self.fail('User disabled or missing', user=user.name)
 		else:
-			self.user = user
+			tracker.add_success_attempt()
+		self.user = user.name
 
 	def force_user_to_reset_password(self):
 		if not self.user:
@@ -245,23 +266,12 @@ class LoginManager:
 			if last_pwd_reset_days > reset_pwd_after_days:
 				return True
 
-	def check_if_enabled(self, user):
-		"""raise exception if user not enabled"""
-		doc = frappe.get_doc("System Settings")
-		if cint(doc.allow_consecutive_login_attempts) > 0:
-			check_consecutive_login_attempts(user, doc)
-
-		if user=='Administrator': return
-		if not cint(frappe.db.get_value('User', user, 'enabled')):
-			self.fail('User disabled or missing', user=user)
-
 	def check_password(self, user, pwd):
 		"""check password"""
 		try:
 			# returns user in correct case
 			return check_password(user, pwd)
 		except frappe.AuthenticationError:
-			self.update_invalid_login(user)
 			self.fail('Incorrect password', user=user)
 
 	def fail(self, message, user=None):
@@ -272,15 +282,6 @@ class LoginManager:
 		frappe.db.commit()
 		raise frappe.AuthenticationError
 
-	def update_invalid_login(self, user):
-		last_login_tried = get_last_tried_login_data(user)
-
-		failed_count = 0
-		if last_login_tried > get_datetime():
-			failed_count = get_login_failed_count(user)
-
-		frappe.cache().hset('login_failed_count', user, failed_count + 1)
-
 	def run_trigger(self, event='on_login'):
 		for method in frappe.get_hooks().get(event, []):
 			frappe.call(frappe.get_attr(method), login_manager=self)
@@ -383,38 +384,6 @@ def clear_cookies():
 		frappe.session.sid = ""
 	frappe.local.cookie_manager.delete_cookie(["full_name", "user_id", "sid", "user_image", "system_user"])
 
-def get_last_tried_login_data(user, get_last_login=False):
-	locked_account_time = frappe.cache().hget('locked_account_time', user)
-	if get_last_login and locked_account_time:
-		return locked_account_time
-
-	last_login_tried = frappe.cache().hget('last_login_tried', user)
-	if not last_login_tried or last_login_tried < get_datetime():
-		last_login_tried = get_datetime() + datetime.timedelta(seconds=60)
-
-	frappe.cache().hset('last_login_tried', user, last_login_tried)
-
-	return last_login_tried
-
-def get_login_failed_count(user):
-	return cint(frappe.cache().hget('login_failed_count', user)) or 0
-
-def check_consecutive_login_attempts(user, doc):
-	login_failed_count = get_login_failed_count(user)
-	last_login_tried = (get_last_tried_login_data(user, True)
-		+ datetime.timedelta(seconds=doc.allow_login_after_fail))
-
-	if login_failed_count >= cint(doc.allow_consecutive_login_attempts):
-		locked_account_time = frappe.cache().hget('locked_account_time', user)
-		if not locked_account_time:
-			frappe.cache().hset('locked_account_time', user, get_datetime())
-
-		if last_login_tried > get_datetime():
-			frappe.throw(_("Your account has been locked and will resume after {0} seconds")
-				.format(doc.allow_login_after_fail), frappe.SecurityException)
-		else:
-			delete_login_failed_cache(user)
-
 def validate_ip_address(user):
 	"""check if IP Address is valid"""
 	user = frappe.get_cached_doc("User", user) if not frappe.flags.in_test else frappe.get_doc("User", user)
@@ -436,3 +405,87 @@ def validate_ip_address(user):
 			return
 
 	frappe.throw(_("Access not allowed from this IP Address"), frappe.AuthenticationError)
+
+
+class LoginAttemptTracker(object):
+	"""Track login attemts of a user.
+
+	Lock the account for s number of seconds if there have been n consecutive unsuccessful attempts to log in.
+	"""
+	def __init__(self, user_name: str, max_consecutive_login_attempts: int=3, lock_interval:int = 5*60):
+		""" Initialize the tracker.
+
+		:param user_name: Name of the loggedin user
+		:param max_consecutive_login_attempts: Maximum allowed consecutive failed login attempts
+		:param lock_interval: Locking interval incase of maximum failed attempts
+		"""
+		self.user_name = user_name
+		self.lock_interval = datetime.timedelta(seconds=lock_interval)
+		self.max_failed_logins = max_consecutive_login_attempts
+
+	@property
+	def login_failed_count(self):
+		return frappe.cache().hget('login_failed_count', self.user_name)
+
+	@login_failed_count.setter
+	def login_failed_count(self, count):
+		frappe.cache().hset('login_failed_count', self.user_name, count)
+
+	@login_failed_count.deleter
+	def login_failed_count(self):
+		frappe.cache().hdel('login_failed_count', self.user_name)
+
+	@property
+	def login_failed_time(self):
+		"""First failed login attempt time within lock interval.
+
+		For every user we track only First failed login attempt time within lock interval of time.
+		"""
+		return frappe.cache().hget('login_failed_time', self.user_name)
+
+	@login_failed_time.setter
+	def login_failed_time(self, timestamp):
+		frappe.cache().hset('login_failed_time', self.user_name, timestamp)
+
+	@login_failed_time.deleter
+	def login_failed_time(self):
+		frappe.cache().hdel('login_failed_time', self.user_name)
+
+	def add_failure_attempt(self):
+		""" Log user failure attempts into the system.
+
+		Increase the failure count if new failure is with in current lock interval time period, if not reset the login failure count.
+		"""
+		login_failed_time = self.login_failed_time
+		login_failed_count = self.login_failed_count # Consecutive login failure count
+		current_time = get_datetime()
+
+		if not (login_failed_time and login_failed_count):
+			login_failed_time, login_failed_count = current_time, 0
+
+		if login_failed_time + self.lock_interval > current_time:
+			login_failed_count += 1
+		else:
+			login_failed_time, login_failed_count = current_time, 1
+
+		self.login_failed_time = login_failed_time
+		self.login_failed_count = login_failed_count
+
+	def add_success_attempt(self):
+		"""Reset login failures.
+		"""
+		del self.login_failed_count
+		del self.login_failed_time
+
+	def is_user_allowed(self) -> bool:
+		"""Is user allowed to login
+
+		User is not allowed to login if login failures are greater than threshold within in lock interval from first login failure.
+		"""
+		login_failed_time = self.login_failed_time
+		login_failed_count = self.login_failed_count or 0
+		current_time = get_datetime()
+
+		if login_failed_time and login_failed_time + self.lock_interval > current_time and login_failed_count > self.max_failed_logins:
+			return False
+		return True

frappe/automation/doctype/assignment_rule/assignment_rule.js

@@ -9,6 +9,16 @@ frappe.ui.form.on('Assignment Rule', {
 		frm.events.rule(frm);
 	},
 
+	setup: function(frm) {
+		frm.set_query("document_type", () => {
+			return {
+				filters: {
+					name: ["!=", "ToDo"]
+				}
+			};
+		});
+	},
+
 	document_type: function(frm) {
 		frm.trigger('set_options');
 	},

frappe/automation/doctype/assignment_rule/assignment_rule.py

@@ -18,6 +18,8 @@ class AssignmentRule(Document):
 		if not len(set(assignment_days)) == len(assignment_days):
 			repeated_days = get_repeated(assignment_days)
 			frappe.throw(_("Assignment Day {0} has been repeated.").format(frappe.bold(repeated_days)))
+		if self.document_type == 'ToDo':
+			frappe.throw(_('Assignment Rule is not allowed on {0} document type').format(frappe.bold("ToDo")))
 
 	def on_update(self):
 		clear_assignment_rule_cache(self)
@@ -298,4 +300,4 @@ def get_repeated(values):
 
 def clear_assignment_rule_cache(rule):
 	frappe.cache_manager.clear_doctype_map('Assignment Rule', rule.document_type)
-	frappe.cache_manager.clear_doctype_map('Assignment Rule', 'due_date_rules_for_' + rule.document_type)
+	frappe.cache_manager.clear_doctype_map('Assignment Rule', 'due_date_rules_for_' + rule.document_type)

frappe/automation/doctype/auto_repeat/auto_repeat.py

@@ -15,6 +15,8 @@ from frappe.model.document import Document
 from frappe.core.doctype.communication.email import make
 from frappe.utils.background_jobs import get_jobs
 from frappe.automation.doctype.assignment_rule.assignment_rule import get_repeated
+from frappe.contacts.doctype.contact.contact import get_contacts_linked_from
+from frappe.contacts.doctype.contact.contact import get_contacts_linking_to
 
 month_map = {'Monthly': 1, 'Quarterly': 3, 'Half-yearly': 6, 'Yearly': 12}
 week_map = {'Monday': 0, 'Tuesday': 1, 'Wednesday': 2, 'Thursday': 3, 'Friday': 4, 'Saturday': 5, 'Sunday': 6}
@@ -328,13 +330,8 @@ class AutoRepeat(Document):
 
 	def fetch_linked_contacts(self):
 		if self.reference_doctype and self.reference_document:
-			res = frappe.db.get_all('Contact',
-				fields=['email_id'],
-				filters=[
-					['Dynamic Link', 'link_doctype', '=', self.reference_doctype],
-					['Dynamic Link', 'link_name', '=', self.reference_document]
-				])
-
+			res = get_contacts_linking_to(self.reference_doctype, self.reference_document, fields=['email_id'])
+			res += get_contacts_linked_from(self.reference_doctype, self.reference_document, fields=['email_id'])
 			email_ids = list(set([d.email_id for d in res]))
 			if not email_ids:
 				frappe.msgprint(_('No contacts linked to document'), alert=True)

frappe/boot.py

@@ -21,7 +21,7 @@ from frappe.website.doctype.web_page_view.web_page_view import is_tracking_enabl
 from frappe.social.doctype.energy_point_log.energy_point_log import get_energy_points
 from frappe.model.base_document import get_controller
 from frappe.social.doctype.post.post import frequently_visited_links
-from frappe.core.doctype.navbar_settings.navbar_settings import get_navbar_settings
+from frappe.core.doctype.navbar_settings.navbar_settings import get_navbar_settings, get_app_logo
 
 def get_bootinfo():
 	"""build and return boot info"""
@@ -62,6 +62,7 @@ def get_bootinfo():
 	doclist.extend(get_meta_bundle("Page"))
 	bootinfo.home_folder = frappe.db.get_value("File", {"is_home_folder": 1})
 	bootinfo.navbar_settings = get_navbar_settings()
+	bootinfo.notification_settings = get_notification_settings()
 
 	# ipinfo
 	if frappe.session.data.get('ipinfo'):
@@ -90,6 +91,7 @@ def get_bootinfo():
 	bootinfo.link_preview_doctypes = get_link_preview_doctypes()
 	bootinfo.additional_filters_config = get_additional_filters_from_hooks()
 	bootinfo.desk_settings = get_desk_settings()
+	bootinfo.app_logo_url = get_app_logo()
 
 	return bootinfo
 
@@ -323,4 +325,7 @@ def get_desk_settings():
 		for key in desk_properties:
 			desk_settings[key] = desk_settings.get(key) or role.get(key)
 
-	return desk_settings
+	return desk_settings
+
+def get_notification_settings():
+	return frappe.get_cached_doc('Notification Settings', frappe.session.user)

frappe/build.py

@@ -15,7 +15,7 @@ import frappe
 from frappe.utils.minify import JavascriptMinify
 
 import click
-from requests import get
+import psutil
 from six import iteritems, text_type
 from six.moves.urllib.parse import urlparse
 
@@ -26,6 +26,8 @@ sites_path = os.path.abspath(os.getcwd())
 
 
 def download_file(url, prefix):
+	from requests import get
+
 	filename = urlparse(url).path.split("/")[-1]
 	local_filename = os.path.join(prefix, filename)
 	with get(url, stream=True, allow_redirects=True) as r:
@@ -225,7 +227,7 @@ def bundle(no_compress, app=None, make_copy=False, restore=False, verbose=False,
 
 	frappe_app_path = os.path.abspath(os.path.join(app_paths[0], ".."))
 	check_yarn()
-	frappe.commands.popen(command, cwd=frappe_app_path)
+	frappe.commands.popen(command, cwd=frappe_app_path, env=get_node_env())
 
 
 def watch(no_compress):
@@ -237,13 +239,32 @@ def watch(no_compress):
 	frappe_app_path = os.path.abspath(os.path.join(app_paths[0], ".."))
 	check_yarn()
 	frappe_app_path = frappe.get_app_path("frappe", "..")
-	frappe.commands.popen("{pacman} run watch".format(pacman=pacman), cwd=frappe_app_path)
+	frappe.commands.popen("{pacman} run watch".format(pacman=pacman),
+		cwd=frappe_app_path, env=get_node_env())
 
 
 def check_yarn():
 	if not find_executable("yarn"):
 		print("Please install yarn using below command and try again.\nnpm install -g yarn")
 
+def get_node_env():
+	node_env = {
+		"NODE_OPTIONS": f"--max_old_space_size={get_safe_max_old_space_size()}"
+	}
+	return node_env
+
+def get_safe_max_old_space_size():
+	safe_max_old_space_size = 0
+	try:
+		total_memory = psutil.virtual_memory().total / (1024 * 1024)
+		# reference for the safe limit assumption
+		# https://nodejs.org/api/cli.html#cli_max_old_space_size_size_in_megabytes
+		# set minimum value 1GB
+		safe_max_old_space_size = max(1024, int(total_memory * 0.75))
+	except Exception:
+		pass
+
+	return safe_max_old_space_size
 
 def make_asset_dirs(make_copy=False, restore=False):
 	# don't even think of making assets_path absolute - rm -rf ahead.

frappe/commands/__init__.py

@@ -11,6 +11,7 @@ import frappe.utils
 import subprocess # nosec
 from functools import wraps
 from six import StringIO
+from os import environ
 
 click.disable_unicode_literals_warning = True
 
@@ -53,16 +54,20 @@ def get_site(context, raise_err=True):
 		return None
 
 def popen(command, *args, **kwargs):
-	output    = kwargs.get('output', True)
-	cwd       = kwargs.get('cwd')
-	shell     = kwargs.get('shell', True)
+	output = kwargs.get('output', True)
+	cwd = kwargs.get('cwd')
+	shell = kwargs.get('shell', True)
 	raise_err = kwargs.get('raise_err')
+	env = kwargs.get('env')
+	if env:
+		env = dict(environ, **env)
 
 	proc = subprocess.Popen(command,
-		stdout = None if output else subprocess.PIPE,
-		stderr = None if output else subprocess.PIPE,
-		shell  = shell,
-		cwd    = cwd
+		stdout=None if output else subprocess.PIPE,
+		stderr=None if output else subprocess.PIPE,
+		shell=shell,
+		cwd=cwd,
+		env=env
 	)
 
 	return_ = proc.wait()

frappe/commands/site.py

@@ -9,7 +9,6 @@ import click
 import frappe
 from frappe.commands import get_site, pass_context
 from frappe.exceptions import SiteNotSpecifiedError
-from frappe.installer import _new_site
 
 
 @click.command('new-site')
@@ -31,6 +30,8 @@ def new_site(site, mariadb_root_username=None, mariadb_root_password=None, admin
 			 verbose=False, install_apps=None, source_sql=None, force=None, no_mariadb_socket=False,
 			 install_app=None, db_name=None, db_password=None, db_type=None, db_host=None, db_port=None):
 	"Create a new site"
+	from frappe.installer import _new_site
+
 	frappe.init(site=site, new_site=True)
 
 	_new_site(db_name, site, mariadb_root_username=mariadb_root_username,
@@ -57,6 +58,7 @@ def new_site(site, mariadb_root_username=None, mariadb_root_password=None, admin
 def restore(context, sql_file_path, mariadb_root_username=None, mariadb_root_password=None, db_name=None, verbose=None, install_app=None, admin_password=None, force=None, with_public_files=None, with_private_files=None):
 	"Restore site database from an sql file"
 	from frappe.installer import (
+		_new_site,
 		extract_sql_from_archive,
 		extract_files,
 		is_downgrade,
@@ -145,6 +147,8 @@ def reinstall(context, admin_password=None, mariadb_root_username=None, mariadb_
 	_reinstall(site, admin_password, mariadb_root_username, mariadb_root_password, yes, verbose=context.verbose)
 
 def _reinstall(site, admin_password=None, mariadb_root_username=None, mariadb_root_password=None, yes=False, verbose=False):
+	from frappe.installer import _new_site
+
 	if not yes:
 		click.confirm('This will wipe your database. Are you sure you want to reinstall?', abort=True)
 	try:

frappe/commands/utils.py

@@ -293,7 +293,7 @@ def import_doc(context, path, force=False):
 		try:
 			frappe.init(site=site)
 			frappe.connect()
-			import_doc(path, overwrite=context.force)
+			import_doc(path)
 		finally:
 			frappe.destroy()
 	if not context.sites:
@@ -483,7 +483,6 @@ def console(context):
 @click.option('--doctype', help="For DocType")
 @click.option('--doctype-list-path', help="Path to .txt file for list of doctypes. Example erpnext/tests/server/agriculture.txt")
 @click.option('--test', multiple=True, help="Specific test")
-@click.option('--driver', help="For Travis")
 @click.option('--ui-tests', is_flag=True, default=False, help="Run UI Tests")
 @click.option('--module', help="Run tests in a module")
 @click.option('--profile', is_flag=True, default=False)
@@ -493,9 +492,9 @@ def console(context):
 @click.option('--junit-xml-output', help="Destination file path for junit xml report")
 @click.option('--failfast', is_flag=True, default=False)
 @pass_context
-def run_tests(context, app=None, module=None, doctype=None, test=(),
-	driver=None, profile=False, coverage=False, junit_xml_output=False, ui_tests = False,
-	doctype_list_path=None, skip_test_records=False, skip_before_tests=False, failfast=False):
+def run_tests(context, app=None, module=None, doctype=None, test=(), profile=False,
+		coverage=False, junit_xml_output=False, ui_tests = False, doctype_list_path=None,
+		skip_test_records=False, skip_before_tests=False, failfast=False):
 
 	"Run tests"
 	import frappe.test_runner
@@ -535,8 +534,8 @@ def run_tests(context, app=None, module=None, doctype=None, test=(),
 		cov.start()
 
 	ret = frappe.test_runner.main(app, module, doctype, context.verbose, tests=tests,
-								force=context.force, profile=profile, junit_xml_output=junit_xml_output,
-								ui_tests=ui_tests, doctype_list_path=doctype_list_path, failfast=failfast)
+		force=context.force, profile=profile, junit_xml_output=junit_xml_output,
+		ui_tests=ui_tests, doctype_list_path=doctype_list_path, failfast=failfast)
 
 	if coverage:
 		cov.stop()
@@ -578,7 +577,7 @@ def run_ui_tests(context, app, headless=False):
 		frappe.commands.popen("yarn add cypress@^6 cypress-file-upload@^5 --no-lockfile")
 
 	# run for headless mode
-	run_or_open = 'run --browser chrome --record --key 4a48f41c-11b3-425b-aa88-c58048fa69eb' if headless else 'open'
+	run_or_open = 'run --browser firefox --record --key 4a48f41c-11b3-425b-aa88-c58048fa69eb' if headless else 'open'
 	command = '{site_env} {password_env} {cypress} {run_or_open}'
 	formatted_command = command.format(site_env=site_env, password_env=password_env, cypress=cypress_path, run_or_open=run_or_open)
 

frappe/contacts/doctype/contact/contact.py

@@ -97,11 +97,16 @@ class Contact(Document):
 		if len([email.email_id for email in self.email_ids if email.is_primary]) > 1:
 			frappe.throw(_("Only one {0} can be set as primary.").format(frappe.bold("Email ID")))
 
+		primary_email_exists = False
 		for d in self.email_ids:
 			if d.is_primary == 1:
+				primary_email_exists = True
 				self.email_id = d.email_id.strip()
 				break
 
+		if not primary_email_exists:
+			self.email_id = ""
+
 	def set_primary(self, fieldname):
 		# Used to set primary mobile and phone no.
 		if len(self.phone_nos) == 0:
@@ -115,11 +120,16 @@ class Contact(Document):
 		if len(is_primary) > 1:
 			frappe.throw(_("Only one {0} can be set as primary.").format(frappe.bold(frappe.unscrub(fieldname))))
 
+		primary_number_exists = False 
 		for d in self.phone_nos:
 			if d.get(field_name) == 1:
+				primary_number_exists = True
 				setattr(self, fieldname, d.phone)
 				break
 
+		if not primary_number_exists:
+			setattr(self, fieldname, "")
+
 def get_default_contact(doctype, name):
 	'''Returns default contact for the given doctype, name'''
 	out = frappe.db.sql('''select parent,
@@ -256,3 +266,27 @@ def get_contact_with_phone_number(number):
 def get_contact_name(email_id):
 	contact = frappe.get_list("Contact Email", filters={"email_id": email_id}, fields=["parent"], limit=1)
 	return contact[0].parent if contact else None
+
+def get_contacts_linking_to(doctype, docname, fields=None):
+	"""Return a list of contacts containing a link to the given document."""
+	return frappe.get_list('Contact', fields=fields, filters=[
+		['Dynamic Link', 'link_doctype', '=', doctype],
+		['Dynamic Link', 'link_name', '=', docname]
+	])
+
+def get_contacts_linked_from(doctype, docname, fields=None):
+	"""Return a list of contacts that are contained in (linked from) the given document."""
+	link_fields = frappe.get_meta(doctype).get('fields', {
+		'fieldtype': 'Link',
+		'options': 'Contact'
+	})
+	if not link_fields:
+		return []
+
+	contact_names = frappe.get_value(doctype, docname, fieldname=[f.fieldname for f in link_fields])
+	if not contact_names:
+		return []
+
+	return frappe.get_list('Contact', fields=fields, filters={
+		'name': ('in', contact_names)
+	})

frappe/contacts/report/addresses_and_contacts/test_addresses_and_contacts.py

@@ -103,7 +103,7 @@ class TestAddressesAndContacts(unittest.TestCase):
 		create_linked_contact(links_list, d)
 		report_data = get_data({"reference_doctype": "Test Custom Doctype"})
 		for idx, link in enumerate(links_list):
-			test_item = [link, 'test address line 1', 'test address line 2', 'Milan', None, None, 'Italy', 0, '_Test First Name', '_Test Last Name', '_Test Address-Billing', '+91 0000000000', None, 'test_contact@example.com', 1]
+			test_item = [link, 'test address line 1', 'test address line 2', 'Milan', None, None, 'Italy', 0, '_Test First Name', '_Test Last Name', '_Test Address-Billing', '+91 0000000000', '', 'test_contact@example.com', 1]
 			self.assertListEqual(test_item, report_data[idx])
 
 	def tearDown(self):

frappe/core/doctype/activity_log/test_activity_log.py

@@ -77,6 +77,10 @@ class TestActivityLog(unittest.TestCase):
 		self.assertRaises(frappe.AuthenticationError, LoginManager)
 		self.assertRaises(frappe.AuthenticationError, LoginManager)
 		self.assertRaises(frappe.AuthenticationError, LoginManager)
+
+		# REMOVE ME: current logic allows allow_consecutive_login_attempts+1 attempts
+		# before raising security exception, remove below line when that is fixed.
+		self.assertRaises(frappe.AuthenticationError, LoginManager)
 		self.assertRaises(frappe.SecurityException, LoginManager)
 		time.sleep(5)
 		self.assertRaises(frappe.AuthenticationError, LoginManager)

frappe/core/doctype/data_import/data_import.py

@@ -2,16 +2,16 @@
 # Copyright (c) 2019, Frappe Technologies and contributors
 # For license information, please see license.txt
 
-from __future__ import unicode_literals
 import os
-import frappe
-from frappe.model.document import Document
 
-from frappe.core.doctype.data_import.importer import Importer
+import frappe
+from frappe import _
 from frappe.core.doctype.data_import.exporter import Exporter
+from frappe.core.doctype.data_import.importer import Importer
+from frappe.model.document import Document
+from frappe.modules.import_file import import_file_by_path
 from frappe.utils.background_jobs import enqueue
 from frappe.utils.csvutils import validate_google_sheets_url
-from frappe import _
 
 
 class DataImport(Document):
@@ -173,15 +173,7 @@ def import_file(
 ##############
 
 
-def import_doc(
-	path,
-	overwrite=False,
-	ignore_links=False,
-	ignore_insert=False,
-	insert=False,
-	submit=False,
-	pre_process=None,
-):
+def import_doc(path, pre_process=None):
 	if os.path.isdir(path):
 		files = [os.path.join(path, f) for f in os.listdir(path)]
 	else:
@@ -190,30 +182,21 @@ def import_doc(
 	for f in files:
 		if f.endswith(".json"):
 			frappe.flags.mute_emails = True
-			frappe.modules.import_file.import_file_by_path(
-				f, data_import=True, force=True, pre_process=pre_process, reset_permissions=True
+			import_file_by_path(
+				f,
+				data_import=True,
+				force=True,
+				pre_process=pre_process,
+				reset_permissions=True
 			)
 			frappe.flags.mute_emails = False
 			frappe.db.commit()
 		elif f.endswith(".csv"):
-			import_file_by_path(
-				f,
-				ignore_links=ignore_links,
-				overwrite=overwrite,
-				submit=submit,
-				pre_process=pre_process,
-			)
+			validate_csv_import_file(f)
 			frappe.db.commit()
 
 
-def import_file_by_path(
-	path,
-	ignore_links=False,
-	overwrite=False,
-	submit=False,
-	pre_process=None,
-	no_email=True,
-):
+def validate_csv_import_file(path):
 	if path.endswith(".csv"):
 		print()
 		print("This method is deprecated.")

frappe/core/doctype/data_import/exporter.py

@@ -2,13 +2,15 @@
 # Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and Contributors
 # MIT License. See license.txt
 
+import typing
+
 import frappe
 from frappe.model import (
 	display_fieldtypes,
 	no_value_fields,
 	table_fields as table_fieldtypes,
 )
-from frappe.utils import flt, format_duration
+from frappe.utils import flt, format_duration, groupby_metric
 from frappe.utils.csvutils import build_csv_response
 from frappe.utils.xlsxutils import build_xlsx_response
 
@@ -116,7 +118,6 @@ class Exporter:
 
 	def get_data_to_export(self):
 		frappe.permissions.can_export(self.doctype, raise_exception=True)
-		data_to_export = []
 
 		table_fields = [f for f in self.exportable_fields if f != self.doctype]
 		data = self.get_data_as_docs()
@@ -128,14 +129,13 @@ class Exporter:
 			if table_fields:
 				# add child table data
 				for f in table_fields:
-					for i, child_row in enumerate(doc[f]):
+					for i, child_row in enumerate(doc.get(f, [])):
 						table_df = self.meta.get_field(f)
 						child_doctype = table_df.options
 						rows = self.add_data_row(child_doctype, child_row.parentfield, child_row, rows, i)
 
-			data_to_export += rows
-
-		return data_to_export
+			for row in rows:
+				yield row
 
 	def add_data_row(self, doctype, parentfield, doc, rows, row_idx):
 		if len(rows) < row_idx + 1:
@@ -204,17 +204,13 @@ class Exporter:
 			)
 			child_data[key] = data
 
-		return self.merge_data(parent_data, child_data)
-
-	def merge_data(self, parent_data, child_data):
+		# Group children data by parent name
+		grouped_children_data = self.group_children_data_by_parent(child_data)
 		for doc in parent_data:
-			for table_field, table_rows in child_data.items():
-				doc[table_field] = [row for row in table_rows if row.parent == doc.name]
-
-		return parent_data
+			related_children_docs = grouped_children_data.get(doc.name, {})
+			yield {**doc, **related_children_docs}
 
 	def add_header(self):
-
 		header = []
 		for df in self.fields:
 			is_parent = not df.is_child_table_field
@@ -261,3 +257,6 @@ class Exporter:
 
 	def build_xlsx_response(self):
 		build_xlsx_response(self.get_csv_array_for_export(), self.doctype)
+
+	def group_children_data_by_parent(self, children_data: typing.Dict[str, list]):
+		return groupby_metric(children_data, key='parent')

frappe/core/doctype/data_import/importer.py

@@ -472,32 +472,6 @@ class ImportFile:
 
 		doc = parent_doc
 
-		if self.import_type == INSERT:
-			# check if there is atleast one row for mandatory table fields
-			meta = frappe.get_meta(self.doctype)
-			mandatory_table_fields = [
-				df
-				for df in meta.fields
-				if df.fieldtype in table_fieldtypes
-				and df.reqd
-				and len(doc.get(df.fieldname, [])) == 0
-			]
-			if len(mandatory_table_fields) == 1:
-				self.warnings.append(
-					{
-						"row": first_row.row_number,
-						"message": _("There should be atleast one row for {0} table").format(
-							frappe.bold(mandatory_table_fields[0].label)
-						),
-					}
-				)
-			elif mandatory_table_fields:
-				fields_string = ", ".join([df.label for df in mandatory_table_fields])
-				message = _("There should be atleast one row for the following tables: {0}").format(
-					fields_string
-				)
-				self.warnings.append({"row": first_row.row_number, "message": message})
-
 		return doc, rows, data[len(rows) :]
 
 	def get_warnings(self):
@@ -626,7 +600,6 @@ class Row:
 				new_doc.update(doc)
 				doc = new_doc
 
-		self.check_mandatory_fields(doctype, doc, table_df)
 		return doc
 
 	def validate_value(self, value, col):
@@ -727,66 +700,6 @@ class Row:
 				pass
 		return value
 
-	def check_mandatory_fields(self, doctype, doc, table_df=None):
-		"""If import type is Insert:
-			Check for mandatory fields (except table fields) in doc
-		if import type is Update:
-			Check for name field or autoname field in doc
-		"""
-		meta = frappe.get_meta(doctype)
-		if self.import_type == UPDATE:
-			if meta.istable:
-				# when updating records with table rows,
-				# there are two scenarios:
-				# 1. if row 'name' is provided in the template
-				# the table row will be updated
-				# 2. if row 'name' is not provided
-				# then a new row will be added
-				# so we dont need to check for mandatory
-				return
-
-			# for update, only ID (name) field is mandatory
-			id_field = get_id_field(doctype)
-			if doc.get(id_field.fieldname) in INVALID_VALUES:
-				self.warnings.append(
-					{
-						"row": self.row_number,
-						"message": _("{0} is a mandatory field").format(id_field.label),
-					}
-				)
-			return
-
-		fields = [
-			df
-			for df in meta.fields
-			if df.fieldtype not in table_fieldtypes
-			and df.reqd
-			and doc.get(df.fieldname) in INVALID_VALUES
-		]
-
-		if not fields:
-			return
-
-		def get_field_label(df):
-			return "{0}{1}".format(df.label, " ({})".format(table_df.label) if table_df else "")
-
-		if len(fields) == 1:
-			field_label = get_field_label(fields[0])
-			self.warnings.append(
-				{
-					"row": self.row_number,
-					"message": _("{0} is a mandatory field").format(frappe.bold(field_label)),
-				}
-			)
-		else:
-			fields_string = ", ".join([frappe.bold(get_field_label(df)) for df in fields])
-			self.warnings.append(
-				{
-					"row": self.row_number,
-					"message": _("{0} are mandatory fields").format(fields_string),
-				}
-			)
-
 	def get_values(self, indexes):
 		return [self.data[i] for i in indexes]
 

frappe/core/doctype/data_import/test_importer.py

@@ -13,7 +13,7 @@ doctype_name = 'DocType for Import'
 class TestImporter(unittest.TestCase):
 	@classmethod
 	def setUpClass(cls):
-		create_doctype_if_not_exists(doctype_name)
+		create_doctype_if_not_exists(doctype_name,)
 
 	def test_data_import_from_file(self):
 		import_file = get_import_file('sample_import_file')
@@ -59,18 +59,18 @@ class TestImporter(unittest.TestCase):
 	def test_data_import_without_mandatory_values(self):
 		import_file = get_import_file('sample_import_file_without_mandatory')
 		data_import = self.get_importer(doctype_name, import_file)
+		frappe.local.message_log = []
 		data_import.start_import()
 		data_import.reload()
-		warnings = frappe.parse_json(data_import.template_warnings)
+		import_log = frappe.parse_json(data_import.import_log)
+		self.assertEqual(import_log[0]['row_indexes'], [2,3])
+		expected_error = "Error: <b>Child 1 of DocType for Import</b> Row #1: Value missing for: Child Title"
+		self.assertEqual(frappe.parse_json(import_log[0]['messages'][0])['message'], expected_error)
+		expected_error = "Error: <b>Child 1 of DocType for Import</b> Row #2: Value missing for: Child Title"
+		self.assertEqual(frappe.parse_json(import_log[0]['messages'][1])['message'], expected_error)
 
-		self.assertEqual(warnings[0]['row'], 2)
-		self.assertEqual(warnings[0]['message'], "<b>Child Title (Table Field 1)</b> is a mandatory field")
-
-		self.assertEqual(warnings[1]['row'], 3)
-		self.assertEqual(warnings[1]['message'], "<b>Child Title (Table Field 1 Again)</b> is a mandatory field")
-
-		self.assertEqual(warnings[2]['row'], 4)
-		self.assertEqual(warnings[2]['message'], "<b>Title</b> is a mandatory field")
+		self.assertEqual(import_log[1]['row_indexes'], [4])
+		self.assertEqual(frappe.parse_json(import_log[1]['messages'][0])['message'], "Title is required")
 
 	def test_data_import_update(self):
 		existing_doc = frappe.get_doc(
@@ -104,6 +104,8 @@ class TestImporter(unittest.TestCase):
 		data_import.reference_doctype = doctype
 		data_import.import_file = import_file.file_url
 		data_import.insert()
+		# Commit so that the first import failure does not rollback the Data Import insert.
+		frappe.db.commit()
 
 		return data_import
 

frappe/core/doctype/data_import_legacy/data_import_legacy.py

@@ -2,20 +2,22 @@
 # Copyright (c) 2017, Frappe Technologies and contributors
 # For license information, please see license.txt
 
-from __future__ import unicode_literals
-import frappe, os
-from frappe import _
+import os
+
+import frappe
 import frappe.modules.import_file
-from frappe.model.document import Document
-from frappe.utils.data import format_datetime
+from frappe import _
 from frappe.core.doctype.data_import_legacy.importer import upload
+from frappe.model.document import Document
+from frappe.modules.import_file import import_file_by_path as _import_file_by_path
 from frappe.utils.background_jobs import enqueue
+from frappe.utils.data import format_datetime
 
 
 class DataImportLegacy(Document):
 	def autoname(self):
 		if not self.name:
-			self.name = "Import on " +format_datetime(self.creation)
+			self.name = "Import on " + format_datetime(self.creation)
 
 	def validate(self):
 		if not self.import_file:
@@ -33,6 +35,7 @@ class DataImportLegacy(Document):
 def get_importable_doctypes():
 	return frappe.cache().hget("can_import", frappe.session.user)
 
+
 @frappe.whitelist()
 def import_data(data_import):
 	frappe.db.set_value("Data Import Legacy", data_import, "import_status", "In Progress", update_modified=False)
@@ -57,7 +60,7 @@ def import_doc(path, overwrite=False, ignore_links=False, ignore_insert=False,
 	for f in files:
 		if f.endswith(".json"):
 			frappe.flags.mute_emails = True
-			frappe.modules.import_file.import_file_by_path(f, data_import=True, force=True, pre_process=pre_process, reset_permissions=True)
+			_import_file_by_path(f, data_import=True, force=True, pre_process=pre_process, reset_permissions=True)
 			frappe.flags.mute_emails = False
 			frappe.db.commit()
 		elif f.endswith(".csv"):
@@ -69,7 +72,7 @@ def import_file_by_path(path, ignore_links=False, overwrite=False, submit=False,
 	from frappe.utils.csvutils import read_csv_content
 	print("Importing " + path)
 	with open(path, "r") as infile:
-		upload(rows = read_csv_content(infile.read()), ignore_links=ignore_links, no_email=no_email, overwrite=overwrite,
+		upload(rows=read_csv_content(infile.read()), ignore_links=ignore_links, no_email=no_email, overwrite=overwrite,
 			submit_after_import=submit, pre_process=pre_process)
 
 

frappe/core/doctype/doctype/doctype.json

@@ -562,7 +562,7 @@
   },
   {
    "group": "Customization",
-   "link_doctype": "Custom Script",
+   "link_doctype": "Client Script",
    "link_fieldname": "dt"
   },
   {
@@ -616,7 +616,7 @@
    "link_fieldname": "reference_doctype"
   }
  ],
- "modified": "2020-12-23 23:48:33.752219",
+ "modified": "2021-02-17 20:18:06.212232",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "DocType",

frappe/core/doctype/doctype/doctype.py

@@ -659,7 +659,7 @@ class DocType(Document):
 		flags = {"flags": re.ASCII} if six.PY3 else {}
 
 		# a DocType name should not start or end with an empty space
-		if re.match("^[ \t\n\r]+|[ \t\n\r]+$", name, **flags):
+		if re.search("^[ \t\n\r]+|[ \t\n\r]+$", name, **flags):
 			frappe.throw(_("DocType's name should not start or end with whitespace"), frappe.NameError)
 
 		# a DocType's name should not start with a number or underscore

frappe/core/doctype/file/file.py

@@ -75,7 +75,7 @@ class File(Document):
 			self.add_comment_in_reference_doc('Attachment',
 				_('Added {0}').format("<a href='{file_url}' target='_blank'>{file_name}</a>{icon}".format(**{
 					"icon": ' <i class="fa fa-lock text-warning"></i>' if self.is_private else "",
-					"file_url": quote(self.file_url) if self.file_url else self.file_name,
+					"file_url": quote(frappe.safe_encode(self.file_url)) if self.file_url else self.file_name,
 					"file_name": self.file_name or self.file_url
 				})))
 

frappe/core/doctype/navbar_settings/navbar_settings.py

@@ -25,7 +25,7 @@ class NavbarSettings(Document):
 
 @frappe.whitelist(allow_guest=True)
 def get_app_logo():
-	app_logo = frappe.db.get_single_value('Navbar Settings', 'app_logo')
+	app_logo = frappe.db.get_single_value('Navbar Settings', 'app_logo', cache=True)
 	if not app_logo:
 		app_logo = frappe.get_hooks('app_logo_url')[-1]
 

frappe/core/doctype/scheduled_job_type/scheduled_job_type.py

@@ -3,6 +3,7 @@
 # For license information, please see license.txt
 
 from __future__ import unicode_literals
+from typing import Dict, List
 
 import frappe, json
 from frappe.model.document import Document
@@ -11,12 +12,13 @@ from datetime import datetime
 from croniter import croniter
 from frappe.utils.background_jobs import enqueue, get_jobs
 
+
 class ScheduledJobType(Document):
 	def autoname(self):
-		self.name = '.'.join(self.method.split('.')[-2:])
+		self.name = ".".join(self.method.split(".")[-2:])
 
 	def validate(self):
-		if self.frequency != 'All':
+		if self.frequency != "All":
 			# force logging for all events other than continuous ones (ALL)
 			self.create_log = 1
 
@@ -84,7 +86,7 @@ class ScheduledJobType(Document):
 
 	def log_status(self, status):
 		# log file
-		frappe.logger("scheduler").info('Scheduled Job {0}: {1} for {2}'.format(status, self.method, frappe.local.site))
+		frappe.logger("scheduler").info(f"Scheduled Job {status}: {self.method} for {frappe.local.site}")
 		self.update_scheduler_log(status)
 
 	def update_scheduler_log(self, status):
@@ -111,28 +113,29 @@ class ScheduledJobType(Document):
 
 
 @frappe.whitelist()
-def execute_event(doc):
-	frappe.only_for('System Manager')
+def execute_event(doc: str):
+	frappe.only_for("System Manager")
 	doc = json.loads(doc)
-	frappe.get_doc('Scheduled Job Type', doc.get('name')).enqueue(force=True)
+	frappe.get_doc("Scheduled Job Type", doc.get("name")).enqueue(force=True)
+	return doc
 
 
-def run_scheduled_job(job_type):
-	'''This is a wrapper function that runs a hooks.scheduler_events method'''
+def run_scheduled_job(job_type: str):
+	"""This is a wrapper function that runs a hooks.scheduler_events method"""
 	try:
-		frappe.get_doc('Scheduled Job Type', dict(method=job_type)).execute()
+		frappe.get_doc("Scheduled Job Type", dict(method=job_type)).execute()
 	except Exception:
 		print(frappe.get_traceback())
 
 
-def sync_jobs(hooks=None):
+def sync_jobs(hooks: Dict = None):
 	frappe.reload_doc("core", "doctype", "scheduled_job_type")
 	scheduler_events = hooks or frappe.get_hooks("scheduler_events")
 	all_events = insert_events(scheduler_events)
 	clear_events(all_events)
 
 
-def insert_events(scheduler_events):
+def insert_events(scheduler_events: Dict) -> List:
 	cron_jobs, event_jobs = [], []
 	for event_type in scheduler_events:
 		events = scheduler_events.get(event_type)
@@ -144,7 +147,7 @@ def insert_events(scheduler_events):
 	return cron_jobs + event_jobs
 
 
-def insert_cron_jobs(events):
+def insert_cron_jobs(events: Dict) -> List:
 	cron_jobs = []
 	for cron_format in events:
 		for event in events.get(cron_format):
@@ -153,25 +156,29 @@ def insert_cron_jobs(events):
 	return cron_jobs
 
 
-def insert_event_jobs(events, event_type):
+def insert_event_jobs(events: List, event_type: str) -> List:
 	event_jobs = []
 	for event in events:
 		event_jobs.append(event)
-		frequency = event_type.replace('_', ' ').title()
+		frequency = event_type.replace("_", " ").title()
 		insert_single_event(frequency, event)
 	return event_jobs
 
 
-def insert_single_event(frequency, event, cron_format=None):
+def insert_single_event(frequency: str, event: str, cron_format: str = None):
 	cron_expr = {"cron_format": cron_format} if cron_format else {}
-	doc = frappe.get_doc({
-		"doctype": "Scheduled Job Type",
-		"method": event,
-		"cron_format": cron_format,
-		"frequency": frequency
-	})
+	doc = frappe.get_doc(
+		{
+			"doctype": "Scheduled Job Type",
+			"method": event,
+			"cron_format": cron_format,
+			"frequency": frequency,
+		}
+	)
 
-	if not frappe.db.exists("Scheduled Job Type", {"method": event, "frequency": frequency, **cron_expr }):
+	if not frappe.db.exists(
+		"Scheduled Job Type", {"method": event, "frequency": frequency, **cron_expr}
+	):
 		try:
 			doc.insert()
 		except frappe.DuplicateEntryError:
@@ -179,7 +186,12 @@ def insert_single_event(frequency, event, cron_format=None):
 			doc.insert()
 
 
-def clear_events(all_events):
-	for event in frappe.get_all("Scheduled Job Type", ("name", "method")):
-		if event.method not in all_events:
+def clear_events(all_events: List):
+	for event in frappe.get_all(
+		"Scheduled Job Type", fields=["name", "method", "server_script"]
+	):
+		is_server_script = event.server_script
+		is_defined_in_hooks = event.method in all_events
+
+		if not (is_defined_in_hooks or is_server_script):
 			frappe.delete_doc("Scheduled Job Type", event.name)

frappe/core/doctype/server_script/server_script.js

@@ -6,46 +6,11 @@ frappe.ui.form.on('Server Script', {
 		frm.trigger('setup_help');
 	},
 	refresh: function(frm) {
-		if (frm.doc.script_type === 'Scheduler Event' && !frm.doc.disabled) {
-			frm.add_custom_button('Schedule Script', function() {
-				var d = new frappe.ui.Dialog({
-					title: "Schedule Script Execution",
-					fields: [
-						{
-							fieldname: "event_type",
-							label: __('Select Event Type'),
-							fieldtype: "Select",
-							options: "All\nHourly\nDaily\nWeekly\nMonthly\nYearly\nHourly Long\nDaily Long\nWeekly Long\nMonthly Long"
-						},
-					],
-					primary_action_label: __('Schedule Script'),
-					primary_action: () => {
-						d.get_primary_btn().attr('disabled', true);
-						var data = d.get_values();
-						d.hide();
-						if(data) {
-							frm.events.schedule_script(frm, data);
-						}
-
-					}
-				});
-
-				d.show();
-
-			});
+		if (frm.doc.script_type != 'Scheduler Event') {
+			frm.dashboard.hide();
 		}
 	},
 
-	schedule_script(frm, data) {
-		frm.call({
-			method: "frappe.core.doctype.server_script.server_script.setup_scheduler_events",
-			args: {
-				'script_name': frm.doc.name,
-				'frequency': data.event_type
-			}
-		});
-	},
-
 	setup_help(frm) {
 		frm.get_field('help_html').html(`
 <h4>DocType Event</h4>

frappe/core/doctype/server_script/server_script.json

@@ -8,6 +8,7 @@
  "field_order": [
   "script_type",
   "reference_doctype",
+  "event_frequency",
   "doctype_event",
   "api_method",
   "allow_guest",
@@ -84,11 +85,24 @@
   {
    "fieldname": "help_html",
    "fieldtype": "HTML"
+  },
+  {
+   "depends_on": "eval:doc.script_type == \"Scheduler Event\"",
+   "fieldname": "event_frequency",
+   "fieldtype": "Select",
+   "label": "Event Frequency",
+   "mandatory_depends_on": "eval:doc.script_type == \"Scheduler Event\"",
+   "options": "All\nHourly\nDaily\nWeekly\nMonthly\nYearly\nHourly Long\nDaily Long\nWeekly Long\nMonthly Long"
   }
  ],
  "index_web_pages_for_search": 1,
- "links": [],
- "modified": "2021-01-03 18:50:14.767595",
+ "links": [
+  {
+   "link_doctype": "Scheduled Job Type",
+   "link_fieldname": "server_script"
+  }
+ ],
+ "modified": "2021-02-18 12:36:19.803425",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "Server Script",

frappe/core/doctype/server_script/server_script.py

@@ -5,6 +5,7 @@
 from __future__ import unicode_literals
 
 import ast
+from typing import Dict, List
 
 import frappe
 from frappe.model.document import Document
@@ -14,67 +15,146 @@ from frappe import _
 
 class ServerScript(Document):
 	def validate(self):
-		frappe.only_for('Script Manager', True)
+		frappe.only_for("Script Manager", True)
+		self.validate_script()
+		self.sync_scheduled_jobs()
+		self.clear_scheduled_events()
+
+	def on_update(self):
+		frappe.cache().delete_value("server_script_map")
+		self.sync_scheduler_events()
+
+	def on_trash(self):
+		if self.script_type == "Scheduler Event":
+			for job in self.scheduled_jobs:
+				frappe.delete_doc("Scheduled Job Type", job.name)
+
+	@property
+	def scheduled_jobs(self) -> List[Dict[str, str]]:
+		return frappe.get_all(
+			"Scheduled Job Type",
+			filters={"server_script": self.name},
+			fields=["name", "stopped"],
+		)
+
+	def validate_script(self):
+		"""Utilizes the ast module to check for syntax errors
+		"""
 		ast.parse(self.script)
 
-	@staticmethod
-	def on_update():
-		frappe.cache().delete_value('server_script_map')
+	def sync_scheduled_jobs(self):
+		"""Sync Scheduled Job Type statuses if Server Script's disabled status is changed
+		"""
+		if self.script_type != "Scheduler Event" or not self.has_value_changed("disabled"):
+			return
 
-	def execute_method(self):
-		if self.script_type == 'API':
-			# validate if guest is allowed
-			if frappe.session.user == 'Guest' and not self.allow_guest:
-				raise frappe.PermissionError
-			_globals, _locals = safe_exec(self.script)
-			return _globals.frappe.flags # output can be stored in flags
-		else:
-			# wrong report type!
+		for scheduled_job in self.scheduled_jobs:
+			if bool(scheduled_job.stopped) != bool(self.disabled):
+				job = frappe.get_doc("Scheduled Job Type", scheduled_job.name)
+				job.stopped = self.disabled
+				job.save()
+
+	def sync_scheduler_events(self):
+		"""Create or update Scheduled Job Type documents for Scheduler Event Server Scripts
+		"""
+		if not self.disabled and self.event_frequency and self.script_type == "Scheduler Event":
+			setup_scheduler_events(script_name=self.name, frequency=self.event_frequency)
+
+	def clear_scheduled_events(self):
+		"""Deletes existing scheduled jobs by Server Script if self.event_frequency has changed
+		"""
+		if self.script_type == "Scheduler Event" and self.has_value_changed("event_frequency"):
+			for scheduled_job in self.scheduled_jobs:
+				frappe.delete_doc("Scheduled Job Type", scheduled_job.name)
+
+	def execute_method(self) -> Dict:
+		"""Specific to API endpoint Server Scripts
+
+		Raises:
+			frappe.DoesNotExistError: If self.script_type is not API
+			frappe.PermissionError: If self.allow_guest is unset for API accessed by Guest user
+
+		Returns:
+			dict: Evaluates self.script with frappe.utils.safe_exec.safe_exec and returns the flags set in it's safe globals
+		"""
+		# wrong report type!
+		if self.script_type != "API":
 			raise frappe.DoesNotExistError
 
-	def execute_doc(self, doc):
-		# execute event
-		safe_exec(self.script, None, dict(doc = doc))
+		# validate if guest is allowed
+		if frappe.session.user == "Guest" and not self.allow_guest:
+			raise frappe.PermissionError
+
+		# output can be stored in flags
+		_globals, _locals = safe_exec(self.script)
+		return _globals.frappe.flags
+
+	def execute_doc(self, doc: Document):
+		"""Specific to Document Event triggered Server Scripts
+
+		Args:
+			doc (Document): Executes script with for a certain document's events
+		"""
+		safe_exec(self.script, _locals={"doc": doc})
 
 	def execute_scheduled_method(self):
-		if self.script_type == 'Scheduler Event':
-			safe_exec(self.script)
-		else:
-			# wrong report type!
+		"""Specific to Scheduled Jobs via Server Scripts
+
+		Raises:
+			frappe.DoesNotExistError: If script type is not a scheduler event
+		"""
+		if self.script_type != "Scheduler Event":
 			raise frappe.DoesNotExistError
 
-	def get_permission_query_conditions(self, user):
+		safe_exec(self.script)
+
+	def get_permission_query_conditions(self, user: str) -> List[str]:
+		"""Specific to Permission Query Server Scripts
+
+		Args:
+			user (str): Takes user email to execute script and return list of conditions
+
+		Returns:
+			list: Returns list of conditions defined by rules in self.script
+		"""
 		locals = {"user": user, "conditions": ""}
 		safe_exec(self.script, None, locals)
 		if locals["conditions"]:
 			return locals["conditions"]
 
+
 @frappe.whitelist()
 def setup_scheduler_events(script_name, frequency):
-	method = frappe.scrub('{0}-{1}'.format(script_name, frequency))
-	scheduled_script = frappe.db.get_value('Scheduled Job Type',
-		dict(method=method))
+	"""Creates or Updates Scheduled Job Type documents based on the specified script name and frequency
+
+	Args:
+		script_name (str): Name of the Server Script document
+		frequency (str): Event label compatible with the Frappe scheduler
+	"""
+	method = frappe.scrub(f"{script_name}-{frequency}")
+	scheduled_script = frappe.db.get_value("Scheduled Job Type", {"method": method})
 
 	if not scheduled_script:
-		doc = frappe.get_doc(dict(
-			doctype = 'Scheduled Job Type',
-			method = method,
-			frequency = frequency,
-			server_script = script_name
-		))
+		frappe.get_doc(
+			{
+				"doctype": "Scheduled Job Type",
+				"method": method,
+				"frequency": frequency,
+				"server_script": script_name,
+			}
+		).insert()
 
-		doc.insert()
-
-		frappe.msgprint(_('Enabled scheduled execution for script {0}').format(script_name))
+		frappe.msgprint(_("Enabled scheduled execution for script {0}").format(script_name))
 
 	else:
-		doc = frappe.get_doc('Scheduled Job Type', scheduled_script)
-		doc.update(dict(
-			doctype = 'Scheduled Job Type',
-			method = method,
-			frequency = frequency,
-			server_script = script_name
-		))
+		doc = frappe.get_doc("Scheduled Job Type", scheduled_script)
+
+		if doc.frequency == frequency:
+			return
+
+		doc.frequency = frequency
 		doc.save()
 
-		frappe.msgprint(_('Scheduled execution for script {0} has updated').format(script_name))
+		frappe.msgprint(
+			_("Scheduled execution for script {0} has updated").format(script_name)
+		)

frappe/core/doctype/sms_settings/sms_settings.json

@@ -202,7 +202,7 @@
  "issingle": 1, 
  "istable": 0, 
  "max_attachments": 0, 
- "modified": "2017-11-01 12:57:20.943845", 
+ "modified": "2021-03-02 18:06:00.868688", 
  "modified_by": "Administrator", 
  "module": "Core", 
  "name": "SMS Settings", 
@@ -233,6 +233,6 @@
  "read_only": 0, 
  "read_only_onload": 0, 
  "show_name_in_global_search": 0, 
- "track_changes": 0, 
+ "track_changes": 1, 
  "track_seen": 0
-}
+}

frappe/core/doctype/user/test_user.py

@@ -2,7 +2,7 @@
 # MIT License. See license.txt
 from __future__ import unicode_literals
 
-import frappe, unittest
+import frappe, unittest, uuid
 
 from frappe.model.delete_doc import delete_doc
 from frappe.utils.data import today, add_to_date
@@ -11,6 +11,7 @@ from frappe.utils import get_url
 from frappe.core.doctype.user.user import get_total_users
 from frappe.core.doctype.user.user import MaxUsersReachedError, test_password_strength
 from frappe.core.doctype.user.user import extract_mentions
+from frappe.frappeclient import FrappeClient
 
 test_records = frappe.get_test_records('User')
 
@@ -229,16 +230,45 @@ class TestUser(unittest.TestCase):
 		self.assertEqual(extract_mentions(comment)[1], "test.again@example1.com")
 
 	def test_rate_limiting_for_reset_password(self):
-		from frappe.utils.password import delete_password_reset_cache
-		delete_password_reset_cache()
-
+		# Allow only one reset request for a day
 		frappe.db.set_value("System Settings", "System Settings", "password_reset_limit", 1)
+		frappe.db.commit()
 
-		user = frappe.get_doc("User", "testperm@example.com")
-		link = user.reset_password()
-		self.assertRegex(link, "\/update-password\?key=[A-Za-z0-9]*")
+		url = get_url()
+		data={'cmd': 'frappe.core.doctype.user.user.reset_password', 'user': 'test@test.com'}
 
-		self.assertRaises(frappe.ValidationError, user.reset_password, False)
+		# Clear rate limit tracker to start fresh
+		key = f"rl:{data['cmd']}:{data['user']}"
+		frappe.cache().delete(key)
+
+		c = FrappeClient(url)
+		res1 = c.session.post(url, data=data, verify=c.verify, headers=c.headers)
+		res2 = c.session.post(url, data=data, verify=c.verify, headers=c.headers)
+		self.assertEqual(res1.status_code, 200)
+		self.assertEqual(res2.status_code, 417)
+
+	def test_user_rollback(self):
+		""" """
+		frappe.db.commit()
+		frappe.db.begin()
+		user_id = str(uuid.uuid4())
+		email = f'{user_id}@example.com'
+		try:
+			frappe.flags.in_import = True  # disable throttling
+			frappe.get_doc(dict(
+				doctype='User',
+				email=email,
+				first_name=user_id,
+			)).insert()
+		finally:
+			frappe.flags.in_import = False
+
+		# Check user has been added
+		self.assertIsNotNone(frappe.db.get("User", {"email": email}))
+
+		# Check that rollback works
+		frappe.db.rollback()
+		self.assertIsNone(frappe.db.get("User", {"email": email}))
 
 
 def delete_contact(user):

frappe/core/doctype/user/user.json

@@ -302,7 +302,7 @@
    "no_copy": 1
   },
   {
-   "default": "0",
+   "default": "1",
    "fieldname": "logout_all_sessions",
    "fieldtype": "Check",
    "label": "Logout From All Devices After Changing Password"
@@ -669,7 +669,7 @@
   }
  ],
  "max_attachments": 5,
- "modified": "2021-01-02 11:21:50.507786",
+ "modified": "2021-02-01 16:11:06.037543",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "User",

frappe/core/doctype/user/user.py

@@ -2,21 +2,25 @@
 # MIT License. See license.txt
 
 from __future__ import unicode_literals, print_function
+
+from bs4 import BeautifulSoup
+
 import frappe
+import frappe.share
+import frappe.defaults
+import frappe.permissions
 from frappe.model.document import Document
 from frappe.utils import cint, flt, has_gravatar, escape_html, format_datetime, now_datetime, get_formatted_email, today
 from frappe import throw, msgprint, _
-from frappe.utils.password import update_password as _update_password
+from frappe.utils.password import update_password as _update_password, check_password, get_password_reset_limit
 from frappe.desk.notifications import clear_notifications
-from frappe.desk.doctype.notification_settings.notification_settings import create_notification_settings
+from frappe.desk.doctype.notification_settings.notification_settings import create_notification_settings, toggle_notifications
 from frappe.utils.user import get_system_managers
-from bs4 import BeautifulSoup
-import frappe.permissions
-import frappe.share
-import frappe.defaults
 from frappe.website.utils import is_signup_enabled
+from frappe.rate_limiter import rate_limit
 from frappe.utils.background_jobs import enqueue
 
+
 STANDARD_USERS = ("Guest", "Administrator")
 
 
@@ -146,6 +150,9 @@ class User(Document):
 		if not cint(self.enabled) and getattr(frappe.local, "login_manager", None):
 			frappe.local.login_manager.logout(user=self.name)
 
+		# toggle notifications based on the user's status
+		toggle_notifications(self.name, enable=cint(self.enabled))
+
 	def add_system_manager_role(self):
 		# if adding system manager, do nothing
 		if not cint(self.enabled) or ("System Manager" in [user_role.role for user_role in
@@ -238,11 +245,6 @@ class User(Document):
 	def reset_password(self, send_email=False, password_expired=False):
 		from frappe.utils import random_string, get_url
 
-		rate_limit = frappe.db.get_single_value("System Settings", "password_reset_limit")
-
-		if rate_limit:
-			check_password_reset_limit(self.name, rate_limit)
-
 		key = random_string(32)
 		self.db_set("reset_password_key", key)
 
@@ -254,7 +256,6 @@ class User(Document):
 		if send_email:
 			self.password_reset_mail(link)
 
-		update_password_reset_limit(self.name)
 		return link
 
 	def get_other_system_managers(self):
@@ -358,6 +359,9 @@ class User(Document):
 			set `user`=null
 			where `user`=%s""", (self.name))
 
+		# delete notification settings
+		frappe.delete_doc("Notification Settings", self.name, ignore_permissions=True)
+
 
 	def before_rename(self, old_name, new_name, merge=False):
 		self.check_demo()
@@ -527,6 +531,27 @@ class User(Document):
 
 		return [i.strip() for i in self.restrict_ip.split(",")]
 
+	@classmethod
+	def find_by_credentials(cls, user_name: str, password: str, validate_password: bool = True):
+		"""Find the user by credentials.
+		"""
+		login_with_mobile = cint(frappe.db.get_value("System Settings", "System Settings", "allow_login_using_mobile_number"))
+		filter = {"mobile_no": user_name} if login_with_mobile else {"name": user_name}
+
+		user = frappe.db.get_value("User", filters=filter, fieldname=['name', 'enabled'], as_dict=True) or {}
+		if not user:
+			return
+
+		user['is_authenticated'] = True
+		if validate_password:
+			try:
+				check_password(user_name, password)
+			except frappe.AuthenticationError:
+				user['is_authenticated'] = False
+
+		return user
+
+
 @frappe.whitelist()
 def get_timezones():
 	import pytz
@@ -562,6 +587,10 @@ def get_perm_info(role):
 
 @frappe.whitelist(allow_guest=True)
 def update_password(new_password, logout_all_sessions=0, key=None, old_password=None):
+	#validate key to avoid key input like ['like', '%'], '', ['in', ['']]
+	if key and not isinstance(key, str):
+		frappe.throw(_('Invalid key type'))
+
 	result = test_password_strength(new_password, key, old_password)
 	feedback = result.get("feedback", None)
 
@@ -812,6 +841,7 @@ def sign_up(email, full_name, redirect_to):
 			return 2, _("Please ask your administrator to verify your sign-up")
 
 @frappe.whitelist(allow_guest=True)
+@rate_limit(key='user', limit=get_password_reset_limit, seconds = 24*60*60, methods=['POST'])
 def reset_password(user):
 	if user=="Administrator":
 		return 'not allowed'
@@ -1143,16 +1173,3 @@ def generate_keys(user):
 def switch_theme(theme):
 	if theme in ["Dark", "Light"]:
 		frappe.db.set_value("User", frappe.session.user, "desk_theme", theme)
-
-def update_password_reset_limit(user):
-	generated_link_count = get_generated_link_count(user)
-	generated_link_count += 1
-	frappe.cache().hset("password_reset_link_count", user, generated_link_count)
-
-def check_password_reset_limit(user, rate_limit):
-	generated_link_count = get_generated_link_count(user)
-	if generated_link_count >= rate_limit:
-		frappe.throw(_("You have reached the hourly limit for generating password reset links. Please try again later."))
-
-def get_generated_link_count(user):
-	return cint(frappe.cache().hget("password_reset_link_count", user)) or 0

frappe/core/page/background_jobs/background_jobs.js

@@ -28,6 +28,16 @@ class BackgroundJobs {
 			}
 		});
 
+		// add a "Remove Failed Jobs button"
+		this.remove_failed_button = this.page.add_inner_button(__("Remove Failed Jobs"), () => {
+			frappe.call({
+				method: 'frappe.core.page.background_jobs.background_jobs.remove_failed_jobs',
+				callback: () => {
+					this.refresh_jobs();
+				}
+			});
+		});
+
 		$(frappe.render_template('background_jobs_outer')).appendTo(this.page.body);
 		this.content = $(this.page.body).find('.table-area');
 	}
@@ -62,4 +72,4 @@ class BackgroundJobs {
 			}
 		});
 	}
-}
+}

frappe/core/page/background_jobs/background_jobs.py

@@ -1,58 +1,88 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # MIT License. See license.txt
 
-from __future__ import unicode_literals
-import frappe
+import json
+from typing import TYPE_CHECKING, Dict, List
 
 from rq import Queue, Worker
-from frappe.utils.background_jobs import get_redis_conn
-from frappe.utils import format_datetime, cint, convert_utc_to_user_timezone
-from frappe.utils.scheduler import is_scheduler_inactive
-from frappe import _
 
-colors = {
+import frappe
+from frappe import _
+from frappe.utils import convert_utc_to_user_timezone, format_datetime
+from frappe.utils.background_jobs import get_redis_conn
+from frappe.utils.scheduler import is_scheduler_inactive
+
+if TYPE_CHECKING:
+	from rq.job import Job
+
+JOB_COLORS = {
 	'queued': 'orange',
 	'failed': 'red',
 	'started': 'blue',
 	'finished': 'green'
 }
 
+
 @frappe.whitelist()
-def get_info(show_failed=False):
+def get_info(show_failed=False) -> List[Dict]:
+	if isinstance(show_failed, str):
+		show_failed = json.loads(show_failed)
+
 	conn = get_redis_conn()
 	queues = Queue.all(conn)
 	workers = Worker.all(conn)
 	jobs = []
 
-	def add_job(j, name):
-		if j.kwargs.get('site')==frappe.local.site:
-			jobs.append({
-				'job_name': j.kwargs.get('kwargs', {}).get('playbook_method') \
-					or j.kwargs.get('kwargs', {}).get('job_type') \
-					or str(j.kwargs.get('job_name')),
-				'status': j.get_status(), 'queue': name,
-				'creation': format_datetime(convert_utc_to_user_timezone(j.created_at)),
-				'color': colors[j.get_status()]
-			})
-			if j.exc_info:
-				jobs[-1]['exc_info'] = j.exc_info
+	def add_job(job: 'Job', name: str) -> None:
+		if job.kwargs.get('site') == frappe.local.site:
+			job_info = {
+				'job_name': job.kwargs.get('kwargs', {}).get('playbook_method')
+					or job.kwargs.get('kwargs', {}).get('job_type')
+					or str(job.kwargs.get('job_name')),
+				'status': job.get_status(),
+				'queue': name,
+				'creation': format_datetime(convert_utc_to_user_timezone(job.created_at)),
+				'color': JOB_COLORS[job.get_status()]
+			}
 
-	for w in workers:
-		j = w.get_current_job()
-		if j:
-			add_job(j, w.name)
+			if job.exc_info:
+				job_info['exc_info'] = job.exc_info
 
-	for q in queues:
-		if q.name != 'failed':
-			for j in q.get_jobs(): add_job(j, q.name)
+			jobs.append(job_info)
 
-	if cint(show_failed):
-		for q in queues:
-			if q.name == 'failed':
-				for j in q.get_jobs()[:10]: add_job(j, q.name)
+	# show worker jobs
+	for worker in workers:
+		job = worker.get_current_job()
+		if job:
+			add_job(job, worker.name)
+
+	for queue in queues:
+		# show active queued jobs
+		if queue.name != 'failed':
+			for job in queue.jobs:
+				add_job(job, queue.name)
+
+		# show failed jobs, if requested
+		if show_failed:
+			fail_registry = queue.failed_job_registry
+			for job_id in fail_registry.get_job_ids():
+				job = queue.fetch_job(job_id)
+				add_job(job, queue.name)
 
 	return jobs
 
+
+@frappe.whitelist()
+def remove_failed_jobs():
+	conn = get_redis_conn()
+	queues = Queue.all(conn)
+	for queue in queues:
+		fail_registry = queue.failed_job_registry
+		for job_id in fail_registry.get_job_ids():
+			job = queue.fetch_job(job_id)
+			fail_registry.remove(job, delete_job=True)
+
+
 @frappe.whitelist()
 def get_scheduler_status():
 	if is_scheduler_inactive():

frappe/core/page/dashboard_view/dashboard_view.js

@@ -36,17 +36,17 @@ class Dashboard {
 		} else {
 			// last opened
 			if (frappe.last_dashboard) {
-				frappe.set_route('dashboard-view', frappe.last_dashboard);
+				frappe.set_re_route('dashboard-view', frappe.last_dashboard);
 			} else {
 				// default dashboard
 				frappe.db.get_list('Dashboard', {filters: {is_default: 1}}).then(data => {
 					if (data && data.length) {
-						frappe.set_route('dashboard-view', data[0].name);
+						frappe.set_re_route('dashboard-view', data[0].name);
 					} else {
 						// no default, get the latest one
 						frappe.db.get_list('Dashboard', {limit: 1}).then(data => {
 							if (data && data.length) {
-								frappe.set_route('dashboard-view', data[0].name);
+								frappe.set_re_route('dashboard-view', data[0].name);
 							} else {
 								// create a new dashboard!
 								frappe.new_doc('Dashboard');

frappe/core/page/recorder/recorder.js

@@ -22,6 +22,7 @@ class Recorder {
 	}
 
 	show() {
-
+		if (!this.view || this.view.$route.name == "recorder-detail") return;
+		this.view.$router.replace({name: "recorder-detail"});
 	}
 }

frappe/core/workspace/build/build.json

@@ -11,6 +11,7 @@
  "hide_custom": 0,
  "icon": "tool",
  "idx": 0,
+ "is_default": 0,
  "is_standard": 1,
  "label": "Build",
  "links": [
@@ -163,8 +164,8 @@
   {
    "hidden": 0,
    "is_query_report": 0,
-   "label": "Custom Script",
-   "link_to": "Custom Script",
+   "label": "Client Script",
+   "link_to": "Client Script",
    "link_type": "DocType",
    "onboard": 0,
    "only_for": "",
@@ -181,7 +182,7 @@
    "type": "Link"
   }
  ],
- "modified": "2021-01-02 14:03:15.029699",
+ "modified": "2021-02-04 13:48:48.493146",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "Build",

frappe/custom/doctype/client_script/client_script.js

@@ -0,0 +1,93 @@
+// Copyright (c) 2016, Frappe Technologies and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on('Client Script', {
+	refresh(frm) {
+		if (frm.doc.dt && frm.doc.script) {
+			frm.add_custom_button(__('Go to {0}', [frm.doc.dt]),
+				() => frappe.set_route('List', frm.doc.dt, 'List'));
+		}
+
+		if (frm.doc.view == 'Form') {
+			frm.add_custom_button(__('Add script for Child Table'), () => {
+				frappe.model.with_doctype(frm.doc.dt, () => {
+					const child_tables = frappe.meta.get_docfields(frm.doc.dt, null, {
+						fieldtype: 'Table'
+					}).map(df => df.options);
+
+					const d = new frappe.ui.Dialog({
+						title: __('Select Child Table'),
+						fields: [
+							{
+								label: __('Select Child Table'),
+								fieldtype: 'Link',
+								fieldname: 'cdt',
+								options: 'DocType',
+								get_query: () => {
+									return {
+										filters: {
+											istable: 1,
+											name: ['in', child_tables]
+										}
+									};
+								}
+							}
+						],
+						primary_action: ({ cdt }) => {
+							cdt = d.get_field('cdt').value;
+							frm.events.add_script_for_doctype(frm, cdt);
+							d.hide();
+						}
+					});
+
+					d.show();
+				});
+			});
+		}
+
+		frm.set_query('dt', {
+			filters: {
+				istable: 0
+			}
+		});
+	},
+
+	dt(frm) {
+		frm.toggle_display('view', !frappe.boot.single_types.includes(frm.doc.dt));
+
+		if (!frm.doc.script) {
+			frm.events.add_script_for_doctype(frm, frm.doc.dt);
+		}
+
+		if (frm.doc.script && !frm.doc.script.includes(frm.doc.dt)) {
+			frm.doc.script = '';
+			frm.events.add_script_for_doctype(frm, frm.doc.dt);
+		}
+	},
+
+	view(frm) {
+		let has_form_boilerplate = frm.doc.script.includes('frappe.ui.form.on')
+		if (frm.doc.view === 'List' && has_form_boilerplate) {
+			frm.set_value('script', '');
+		}
+		if (frm.doc.view === 'Form' && !has_form_boilerplate) {
+			frm.trigger('dt');
+		}
+	},
+
+	add_script_for_doctype(frm, doctype) {
+		if (!doctype) return;
+		let boilerplate = `
+frappe.ui.form.on('${doctype}', {
+	refresh(frm) {
+		// your code here
+	}
+})
+		`.trim();
+		let script = (frm.doc.script || '');
+		if (script) {
+			script += '\n\n';
+		}
+		frm.set_value('script', script + boilerplate);
+	}
+});

frappe/custom/doctype/client_script/client_script.json

@@ -2,12 +2,13 @@
  "actions": [],
  "allow_import": 1,
  "creation": "2013-01-10 16:34:01",
- "description": "Adds a client custom script to a DocType",
+ "description": "Adds a custom client script to a DocType",
  "doctype": "DocType",
  "document_type": "Document",
  "engine": "InnoDB",
  "field_order": [
   "dt",
+  "view",
   "enabled",
   "script",
   "sample"
@@ -23,8 +24,7 @@
    "oldfieldtype": "Link",
    "options": "DocType",
    "reqd": 1,
-   "show_days": 1,
-   "show_seconds": 1
+   "set_only_once": 1
   },
   {
    "fieldname": "script",
@@ -32,35 +32,37 @@
    "label": "Script",
    "oldfieldname": "script",
    "oldfieldtype": "Code",
-   "options": "JS",
-   "show_days": 1,
-   "show_seconds": 1
+   "options": "JS"
   },
   {
    "fieldname": "sample",
    "fieldtype": "HTML",
    "label": "Sample",
-   "options": "<h3>Custom Script Help</h3>\n<p>Custom Scripts are executed only on the client-side (i.e. in Forms). Here are some examples to get you started</p>\n<pre><code>\n\n// fetch local_tax_no on selection of customer \n// cur_frm.add_fetch(link_field,  source_fieldname,  target_fieldname); \ncur_frm.add_fetch('customer',  'local_tax_no',  'local_tax_no');\n\n// additional validation on dates \nfrappe.ui.form.on('Task',  'validate',  function(frm) {\n    if (frm.doc.from_date &lt; get_today()) {\n        msgprint('You can not select past date in From Date');\n        validated = false;\n    } \n});\n\n// make a field read-only after saving \nfrappe.ui.form.on('Task',  {\n    refresh: function(frm) {\n        // use the __islocal value of doc,  to check if the doc is saved or not\n        frm.set_df_property('myfield',  'read_only',  frm.doc.__islocal ? 0 : 1);\n    } \n});\n\n// additional permission check\nfrappe.ui.form.on('Task',  {\n    validate: function(frm) {\n        if(user=='user1@example.com' &amp;&amp; frm.doc.purpose!='Material Receipt') {\n            msgprint('You are only allowed Material Receipt');\n            validated = false;\n        }\n    } \n});\n\n// calculate sales incentive\nfrappe.ui.form.on('Sales Invoice',  {\n    validate: function(frm) {\n        // calculate incentives for each person on the deal\n        total_incentive = 0\n        $.each(frm.doc.sales_team,  function(i,  d) {\n            // calculate incentive\n            var incentive_percent = 2;\n            if(frm.doc.base_grand_total &gt; 400) incentive_percent = 4;\n            // actual incentive\n            d.incentives = flt(frm.doc.base_grand_total) * incentive_percent / 100;\n            total_incentive += flt(d.incentives)\n        });\n        frm.doc.total_incentive = total_incentive;\n    } \n})\n\n</code></pre>",
-   "show_days": 1,
-   "show_seconds": 1
+   "options": "<h3>Client Script Help</h3>\n<p>Client Scripts are executed only on the client-side (i.e. in Forms). Here are some examples to get you started</p>\n<pre><code>\n\n// fetch local_tax_no on selection of customer \n// cur_frm.add_fetch(link_field,  source_fieldname,  target_fieldname); \ncur_frm.add_fetch('customer',  'local_tax_no',  'local_tax_no');\n\n// additional validation on dates \nfrappe.ui.form.on('Task',  'validate',  function(frm) {\n    if (frm.doc.from_date &lt; get_today()) {\n        msgprint('You can not select past date in From Date');\n        validated = false;\n    } \n});\n\n// make a field read-only after saving \nfrappe.ui.form.on('Task',  {\n    refresh: function(frm) {\n        // use the __islocal value of doc,  to check if the doc is saved or not\n        frm.set_df_property('myfield',  'read_only',  frm.doc.__islocal ? 0 : 1);\n    } \n});\n\n// additional permission check\nfrappe.ui.form.on('Task',  {\n    validate: function(frm) {\n        if(user=='user1@example.com' &amp;&amp; frm.doc.purpose!='Material Receipt') {\n            msgprint('You are only allowed Material Receipt');\n            validated = false;\n        }\n    } \n});\n\n// calculate sales incentive\nfrappe.ui.form.on('Sales Invoice',  {\n    validate: function(frm) {\n        // calculate incentives for each person on the deal\n        total_incentive = 0\n        $.each(frm.doc.sales_team,  function(i,  d) {\n            // calculate incentive\n            var incentive_percent = 2;\n            if(frm.doc.base_grand_total &gt; 400) incentive_percent = 4;\n            // actual incentive\n            d.incentives = flt(frm.doc.base_grand_total) * incentive_percent / 100;\n            total_incentive += flt(d.incentives)\n        });\n        frm.doc.total_incentive = total_incentive;\n    } \n})\n\n</code></pre>"
   },
   {
    "default": "0",
    "fieldname": "enabled",
    "fieldtype": "Check",
-   "label": "Enabled",
-   "show_days": 1,
-   "show_seconds": 1
+   "label": "Enabled"
+  },
+  {
+   "default": "Form",
+   "fieldname": "view",
+   "fieldtype": "Select",
+   "label": "Apply To",
+   "options": "List\nForm",
+   "set_only_once": 1
   }
  ],
  "icon": "fa fa-glass",
  "idx": 1,
  "index_web_pages_for_search": 1,
  "links": [],
- "modified": "2020-08-24 21:56:07.719579",
+ "modified": "2021-03-16 20:33:51.400191",
  "modified_by": "Administrator",
  "module": "Custom",
- "name": "Custom Script",
+ "name": "Client Script",
  "owner": "Administrator",
  "permissions": [
   {
@@ -86,6 +88,7 @@
    "write": 1
   }
  ],
+ "sort_field": "modified",
  "sort_order": "ASC",
  "track_changes": 1
 }

frappe/custom/doctype/client_script/client_script.py

@@ -0,0 +1,31 @@
+# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
+# MIT License. See license.txt
+from __future__ import unicode_literals
+import frappe
+
+from frappe import _
+from frappe.model.document import Document
+
+
+class ClientScript(Document):
+	def autoname(self):
+		self.name = f"{self.dt}-{self.view}"
+
+	def validate(self):
+		if not self.is_new():
+			return
+
+		exists = frappe.db.exists(
+			"Client Script", {"dt": self.dt, "view": self.view}
+		)
+		if exists:
+			frappe.throw(
+				_("Client Script for {0} {1} already exists").format(frappe.bold(self.dt), self.view),
+				frappe.DuplicateEntryError,
+			)
+
+	def on_update(self):
+		frappe.clear_cache(doctype=self.dt)
+
+	def on_trash(self):
+		frappe.clear_cache(doctype=self.dt)

frappe/custom/doctype/client_script/test_client_script.py

@@ -6,7 +6,7 @@ from __future__ import unicode_literals
 import frappe
 import unittest
 
-# test_records = frappe.get_test_records('Custom Script')
+# test_records = frappe.get_test_records('Client Script')
 
-class TestCustomScript(unittest.TestCase):
+class TestClientScript(unittest.TestCase):
 	pass

frappe/custom/doctype/custom_script/custom_script.js

@@ -1,78 +0,0 @@
-// Copyright (c) 2016, Frappe Technologies and contributors
-// For license information, please see license.txt
-
-frappe.ui.form.on('Custom Script', {
-	refresh(frm) {
-		if (frm.doc.dt && frm.doc.script) {
-			frm.add_custom_button(__('Go to {0}', [frm.doc.dt]),
-				() => frappe.set_route('List', frm.doc.dt, 'List'));
-		}
-
-		frm.add_custom_button(__('Add script for Child Table'), () => {
-			frappe.model.with_doctype(frm.doc.dt, () => {
-				const child_tables = frappe.meta.get_docfields(frm.doc.dt, null, {
-					fieldtype: 'Table'
-				}).map(df => df.options);
-
-				const d = new frappe.ui.Dialog({
-					title: __('Select Child Table'),
-					fields: [
-						{
-							label: __('Select Child Table'),
-							fieldtype: 'Link',
-							fieldname: 'cdt',
-							options: 'DocType',
-							get_query: () => {
-								return {
-									filters: {
-										istable: 1,
-										name: ['in', child_tables]
-									}
-								};
-							}
-						}
-					],
-					primary_action: ({ cdt }) => {
-						cdt = d.get_field('cdt').value;
-						frm.events.add_script_for_doctype(frm, cdt);
-						d.hide();
-					}
-				});
-
-				d.show();
-			});
-		});
-
-		frm.set_query('dt', {
-			filters: {
-				istable: 0
-			}
-		});
-	},
-
-	dt(frm) {
-		if (!frm.doc.script) {
-			frm.events.add_script_for_doctype(frm, frm.doc.dt);
-		}
-
-		if (frm.doc.script && !frm.doc.script.includes(frm.doc.dt)) {
-			frm.doc.script = '';
-			frm.events.add_script_for_doctype(frm, frm.doc.dt);
-		}
-	},
-
-	add_script_for_doctype(frm, doctype) {
-		let boilerplate = `
-frappe.ui.form.on('${doctype}', {
-	refresh(frm) {
-		// your code here
-	}
-})
-		`.trim();
-		let script = (frm.doc.script || '');
-		if (script) {
-			script += '\n\n';
-		}
-		frm.set_value('script', script + boilerplate);
-	}
-});

frappe/custom/doctype/custom_script/custom_script.py

@@ -1,17 +0,0 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
-from __future__ import unicode_literals
-import frappe
-
-from frappe.model.document import Document
-
-class CustomScript(Document):
-	def autoname(self):
-		self.name = self.dt + "-Client"
-
-	def on_update(self):
-		frappe.clear_cache(doctype=self.dt)
-
-	def on_trash(self):
-		frappe.clear_cache(doctype=self.dt)
-

frappe/custom/doctype/customize_form/customize_form.js

@@ -76,6 +76,7 @@ frappe.ui.form.on("Customize Form", {
 							frm.trigger("setup_sortable");
 						}
 					}
+					localStorage["customize_doctype"] = frm.doc.doc_type;
 				}
 			});
 		} else {

frappe/custom/workspace/customization/customization.json

@@ -10,6 +10,7 @@
  "hide_custom": 0,
  "icon": "customization",
  "idx": 0,
+ "is_default": 0,
  "is_standard": 1,
  "label": "Customization",
  "links": [
@@ -81,8 +82,8 @@
    "dependencies": "",
    "hidden": 0,
    "is_query_report": 0,
-   "label": "Custom Script",
-   "link_to": "Custom Script",
+   "label": "Client Script",
+   "link_to": "Client Script",
    "link_type": "DocType",
    "onboard": 0,
    "type": "Link"
@@ -115,7 +116,7 @@
    "type": "Link"
   }
  ],
- "modified": "2020-12-01 13:38:39.843773",
+ "modified": "2021-02-04 13:50:35.750463",
  "modified_by": "Administrator",
  "module": "Custom",
  "name": "Customization",
@@ -134,8 +135,14 @@
    "type": "DocType"
   },
   {
-   "label": "Custom Script",
-   "link_to": "Custom Script",
+   "label": "Client Script",
+   "link_to": "Client Script",
+   "type": "DocType"
+  },
+  {
+   "doc_view": "",
+   "label": "Server Script",
+   "link_to": "Server Script",
    "type": "DocType"
   }
  ]

frappe/database/database.py

@@ -16,7 +16,6 @@ import frappe.model.meta
 from frappe import _
 from time import time
 from frappe.utils import now, getdate, cast_fieldtype, get_datetime
-from frappe.utils.background_jobs import execute_job, get_queue
 from frappe.model.utils.link_count import flush_local_link_count
 from frappe.utils import cint
 
@@ -1032,6 +1031,8 @@ class Database(object):
 				insert_list = []
 
 def enqueue_jobs_after_commit():
+	from frappe.utils.background_jobs import execute_job, get_queue
+
 	if frappe.flags.enqueue_after_commit and len(frappe.flags.enqueue_after_commit) > 0:
 		for job in frappe.flags.enqueue_after_commit:
 			q = get_queue(job.get("queue"), is_async=job.get("is_async"))

frappe/database/mariadb/database.py

@@ -8,8 +8,7 @@ from pymysql.times import TimeDelta
 from pymysql.constants 	import ER, FIELD_TYPE
 from pymysql.converters import conversions
 
-from frappe.utils import get_datetime, cstr
-from markdown2 import UnicodeWithAttrs
+from frappe.utils import get_datetime, cstr, UnicodeWithAttrs
 from frappe.database.database import Database
 from six import PY2, binary_type, text_type, string_types
 from frappe.database.mariadb.schema import MariaDBTable

frappe/desk/desktop.py

@@ -601,8 +601,8 @@ def merge_cards_based_on_label(cards):
 	for card in cards:
 		label = card.get('label')
 		if label in cards_dict:
-			links = loads(cards_dict[label].links) + loads(card.links)
-			cards_dict[label].update(dict(links=dumps(links)))
+			links = cards_dict[label].links + card.links
+			cards_dict[label].update(dict(links=links))
 			cards_dict[label] = cards_dict.pop(label)
 		else:
 			cards_dict[label] = card

frappe/desk/doctype/dashboard_chart/dashboard_chart.py

@@ -171,7 +171,6 @@ def get_chart_config(chart, filters, timespan, timegrain, from_date, to_date):
 
 	doctype = chart.document_type
 	datefield = chart.based_on
-	aggregate_function = get_aggregate_function(chart.chart_type)
 	value_field = chart.value_based_on or '1'
 	from_date = from_date.strftime('%Y-%m-%d')
 	to_date = to_date
@@ -183,7 +182,8 @@ def get_chart_config(chart, filters, timespan, timegrain, from_date, to_date):
 		doctype,
 		fields = [
 			'{} as _unit'.format(datefield),
-			'{aggregate_function}({value_field})'.format(aggregate_function=aggregate_function, value_field=value_field),
+			'SUM({})'.format(value_field),
+			'COUNT(*)'
 		],
 		filters = filters,
 		group_by = '_unit',
@@ -192,7 +192,7 @@ def get_chart_config(chart, filters, timespan, timegrain, from_date, to_date):
 		ignore_ifnull = True
 	)
 
-	result = get_result(data, timegrain, from_date, to_date)
+	result = get_result(data, timegrain, from_date, to_date, chart.chart_type)
 
 	chart_config = {
 		"labels": [get_period(r[0], timegrain) for r in result],
@@ -288,15 +288,21 @@ def get_aggregate_function(chart_type):
 	}[chart_type]
 
 
-def get_result(data, timegrain, from_date, to_date):
+def get_result(data, timegrain, from_date, to_date, chart_type):
 	dates = get_dates_from_timegrain(from_date, to_date, timegrain)
 	result = [[date, 0] for date in dates]
 	data_index = 0
 	if data:
 		for i, d in enumerate(result):
+			count = 0
 			while data_index < len(data) and getdate(data[data_index][0]) <= d[0]:
 				d[1] += data[data_index][1]
+				count += data[data_index][2]
 				data_index += 1
+			if chart_type == 'Average' and not count == 0:
+				d[1] = d[1]/count
+			if chart_type == 'Count':
+				d[1] = count
 
 	return result
 

frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py

@@ -212,19 +212,52 @@ class TestDashboardChart(unittest.TestCase):
 
 		frappe.db.rollback()
 
-def insert_test_records():
-	create_new_communication(datetime(2018, 12, 30), 50)
-	create_new_communication(datetime(2019, 1, 4), 100)
-	create_new_communication(datetime(2019, 1, 6), 200)
-	create_new_communication(datetime(2019, 1, 7), 400)
-	create_new_communication(datetime(2019, 1, 8), 300)
-	create_new_communication(datetime(2019, 1, 10), 100)
+	def test_avg_dashboard_chart(self):
+		insert_test_records()
 
-def create_new_communication(date, rating):
+		if frappe.db.exists('Dashboard Chart', 'Test Average Dashboard Chart'):
+			frappe.delete_doc('Dashboard Chart', 'Test Average Dashboard Chart')
+
+		frappe.get_doc(dict(
+			doctype = 'Dashboard Chart',
+			chart_name = 'Test Average Dashboard Chart',
+			chart_type = 'Average',
+			document_type = 'Communication',
+			based_on = 'communication_date',
+			value_based_on = 'rating',
+			timespan = 'Select Date Range',
+			time_interval = 'Weekly',
+			from_date = datetime(2018, 12, 30),
+			to_date = datetime(2019, 1, 15),
+			filters_json = '[]',
+			timeseries = 1
+		)).insert()
+
+		result = get(chart_name='Test Average Dashboard Chart', refresh = 1)
+
+		self.assertEqual(result.get('datasets')[0].get('values'), [50.0, 150.0, 266.6666666666667, 0.0])
+		self.assertEqual(
+			result.get('labels'),
+			['30-12-18', '06-01-19', '13-01-19', '20-01-19']
+		)
+
+		frappe.db.rollback()
+
+def insert_test_records():
+	create_new_communication('Communication 1', datetime(2018, 12, 30), 50)
+	create_new_communication('Communication 2', datetime(2019, 1, 4), 100)
+	create_new_communication('Communication 3', datetime(2019, 1, 6), 200)
+	create_new_communication('Communication 4', datetime(2019, 1, 7), 400)
+	create_new_communication('Communication 5', datetime(2019, 1, 8), 300)
+	create_new_communication('Communication 6', datetime(2019, 1, 10), 100)
+
+def create_new_communication(subject, date, rating):
 	communication = {
 		'doctype': 'Communication',
-		'subject': 'Test Communication',
+		'subject': subject,
 		'rating': rating,
 		'communication_date': date
 	}
-	frappe.get_doc(communication).insert()
+	comm = frappe.get_doc(communication)
+	if not frappe.db.exists("Communication", {'subject' : comm.subject}):
+		comm.insert()

frappe/desk/doctype/kanban_board/kanban_board.py

@@ -17,6 +17,10 @@ class KanbanBoard(Document):
 	def on_update(self):
 		frappe.clear_cache(doctype=self.reference_doctype)
 
+	def before_insert(self):
+		for column in self.columns:
+			column.order = get_order_for_column(self, column.column_name)
+
 	def validate_column_name(self):
 		for column in self.columns:
 			if not column.column_name:
@@ -125,6 +129,53 @@ def update_order(board_name, order):
 	board.save()
 	return board, updated_cards
 
+@frappe.whitelist()
+def update_order_for_single_card(board_name, docname, from_colname, to_colname, old_index, new_index):
+	'''Save the order of cards in columns'''
+	board = frappe.get_doc('Kanban Board', board_name)
+	doctype = board.reference_doctype
+	fieldname = board.field_name
+	old_index = frappe.parse_json(old_index)
+	new_index = frappe.parse_json(new_index)
+
+	# save current order and index of columns to be updated
+	from_col_order, from_col_idx = get_kanban_column_order_and_index(board, from_colname)
+	to_col_order, to_col_idx = get_kanban_column_order_and_index(board, to_colname)
+
+	if from_colname == to_colname:
+		from_col_order = to_col_order
+
+	to_col_order.insert(new_index, from_col_order.pop((old_index)))
+
+	# save updated order
+	board.columns[from_col_idx].order = frappe.as_json(from_col_order)
+	board.columns[to_col_idx].order = frappe.as_json(to_col_order)
+	board.save()
+
+	# update changed value in doc
+	frappe.set_value(doctype, docname, fieldname, to_colname)
+
+	return board
+
+def get_kanban_column_order_and_index(board, colname):
+	for i, col in enumerate(board.columns):
+		if col.column_name == colname:
+			col_order = frappe.parse_json(col.order)
+			col_idx = i
+
+	return col_order, col_idx
+
+@frappe.whitelist()
+def add_card(board_name, docname, colname):
+	board = frappe.get_doc('Kanban Board', board_name)
+
+	col_order, col_idx = get_kanban_column_order_and_index(board, colname)
+	col_order.insert(0, docname)
+
+	board.columns[col_idx].order = frappe.as_json(col_order)
+
+	board.save()
+	return board
 
 @frappe.whitelist()
 def quick_kanban_board(doctype, board_name, field_name, project=None):
@@ -133,6 +184,13 @@ def quick_kanban_board(doctype, board_name, field_name, project=None):
 	doc = frappe.new_doc('Kanban Board')
 	meta = frappe.get_meta(doctype)
 
+	doc.kanban_board_name = board_name
+	doc.reference_doctype = doctype
+	doc.field_name = field_name
+
+	if project:
+		doc.filters = '[["Task","project","=","{0}"]]'.format(project)
+
 	options = ''
 	for field in meta.fields:
 		if field.fieldname == field_name:
@@ -149,12 +207,6 @@ def quick_kanban_board(doctype, board_name, field_name, project=None):
 			column_name=column
 		))
 
-	doc.kanban_board_name = board_name
-	doc.reference_doctype = doctype
-	doc.field_name = field_name
-
-	if project:
-		doc.filters = '[["Task","project","=","{0}"]]'.format(project)
 
 	if doctype in ['Note', 'ToDo']:
 		doc.private = 1
@@ -162,6 +214,12 @@ def quick_kanban_board(doctype, board_name, field_name, project=None):
 	doc.save()
 	return doc
 
+def get_order_for_column(board, colname):
+	filters = [[board.reference_doctype, board.field_name, '=', colname]]
+	if board.filters:
+		filters.append(frappe.parse_json(board.filters)[0])
+
+	return frappe.as_json(frappe.get_list(board.reference_doctype, filters=filters, pluck='name'))
 
 @frappe.whitelist()
 def update_column_order(board_name, order):

frappe/desk/doctype/notification_settings/notification_settings.py

@@ -42,7 +42,11 @@ def create_notification_settings(user):
 		_doc = frappe.new_doc('Notification Settings')
 		_doc.name = user
 		_doc.insert(ignore_permissions=True)
-		frappe.db.commit()
+
+
+def toggle_notifications(user, enable=False):
+	if frappe.db.exists("Notification Settings", user):
+		frappe.db.set_value("Notification Settings", user, 'enabled', enable)
 
 
 @frappe.whitelist()
@@ -76,4 +80,4 @@ def get_permission_query_conditions(user):
 
 @frappe.whitelist()
 def set_seen_value(value, user):
-	frappe.db.set_value('Notification Settings', user, 'seen', value, update_modified=False)
+	frappe.db.set_value('Notification Settings', user, 'seen', value, update_modified=False)

frappe/desk/doctype/number_card/number_card.py

@@ -86,7 +86,7 @@ def get_result(doc, filters, to_date=None):
 	filters = frappe.parse_json(filters)
 
 	if not filters:
-			filters = []
+		filters = []
 
 	if to_date:
 		filters.append([doc.document_type, 'creation', '<', to_date])
@@ -107,9 +107,13 @@ def get_percentage_difference(doc, filters, result):
 		return
 
 	previous_result = calculate_previous_result(doc, filters)
-	difference = (result - previous_result)/100.0
-
-	return difference
+	if previous_result == 0:
+		return None
+	else:
+		if result == previous_result:
+			return 0
+		else:
+			return ((result/previous_result)-1)*100.0
 
 
 def calculate_previous_result(doc, filters):
@@ -197,4 +201,4 @@ def add_card_to_dashboard(args):
 		card.save()
 
 	dashboard.append('cards', dashboard_link)
-	dashboard.save()
+	dashboard.save()

frappe/desk/form/document_follow.py

@@ -147,6 +147,8 @@ def get_version(doctype, doc_name, frequency, user):
 	return timeline
 
 def get_comments(doctype, doc_name, frequency, user):
+	from html2text import html2text
+
 	timeline = []
 	filters = get_filters("reference_name", doc_name, frequency, user)
 	comments = frappe.get_all("Comment",
@@ -166,7 +168,7 @@ def get_comments(doctype, doc_name, frequency, user):
 			"time": comment.modified,
 			"data": {
 				"time": time,
-				"comment": frappe.utils.html2text(str(comment.content)),
+				"comment": html2text(str(comment.content)),
 				"by": by
 			},
 			"doctype": doctype,
@@ -197,6 +199,8 @@ def get_follow_users(doctype, doc_name):
 	)
 
 def get_row_changed(row_changed, time, doctype, doc_name, v):
+	from html2text import html2text
+
 	items = []
 	for d in row_changed:
 		d[2] = d[2] if d[2] else ' '
@@ -209,8 +213,8 @@ def get_row_changed(row_changed, time, doctype, doc_name, v):
 					"table_field": d[0],
 					"row": str(d[1]),
 					"field": d[3][0][0],
-					"from": frappe.utils.html2text(str(d[3][0][1])),
-					"to": frappe.utils.html2text(str(d[3][0][2]))
+					"from": html2text(str(d[3][0][1])),
+					"to": html2text(str(d[3][0][2]))
 				},
 			"doctype": doctype,
 			"doc_name": doc_name,
@@ -236,6 +240,8 @@ def get_added_row(added, time, doctype, doc_name, v):
 	return items
 
 def get_field_changed(changed, time, doctype, doc_name, v):
+	from html2text import html2text
+
 	items = []
 	for d in changed:
 		d[1] = d[1] if d[1] else ' '
@@ -246,8 +252,8 @@ def get_field_changed(changed, time, doctype, doc_name, v):
 			"data": {
 					"time": time,
 					"field": d[0],
-					"from": frappe.utils.html2text(str(d[1])),
-					"to": frappe.utils.html2text(str(d[2]))
+					"from": html2text(str(d[1])),
+					"to": html2text(str(d[2]))
 				},
 			"doctype": doctype,
 			"doc_name": doc_name,

frappe/desk/form/linked_with.py

@@ -79,28 +79,30 @@ def get_submitted_linked_docs(doctype, name, docs=None, visited=None):
 @frappe.whitelist()
 def cancel_all_linked_docs(docs, ignore_doctypes_on_cancel_all=[]):
 	"""
-	Cancel all linked doctype
+	Cancel all linked doctype, optionally ignore doctypes specified in a list.
 
 	Arguments:
-		docs (str) - It contains all list of dictionaries of a linked documents.
+		docs (json str) - It contains list of dictionaries of a linked documents.
+		ignore_doctypes_on_cancel_all (list) - List of doctypes to ignore while cancelling.
 	"""
 
 	docs = json.loads(docs)
 	if isinstance(ignore_doctypes_on_cancel_all, string_types):
 		ignore_doctypes_on_cancel_all = json.loads(ignore_doctypes_on_cancel_all)
 	for i, doc in enumerate(docs, 1):
-		if validate_linked_doc(doc, ignore_doctypes_on_cancel_all) is True:
-			frappe.publish_progress(percent=i * 100 / ((len(docs) - len(ignore_doctypes_on_cancel_all))), title=_("Cancelling documents"))
+		if validate_linked_doc(doc, ignore_doctypes_on_cancel_all):
 			linked_doc = frappe.get_doc(doc.get("doctype"), doc.get("name"))
 			linked_doc.cancel()
+		frappe.publish_progress(percent=i/len(docs) * 100, title=_("Cancelling documents"))
 
 
 def validate_linked_doc(docinfo, ignore_doctypes_on_cancel_all=[]):
 	"""
 	Validate a document to be submitted and non-exempted from auto-cancel.
 
-	Args:
-		docs (dict): The document to check for submitted and non-exempt from auto-cancel
+	Arguments:
+		docinfo (dict): The document to check for submitted and non-exempt from auto-cancel
+		ignore_doctypes_on_cancel_all (list) - List of doctypes to ignore while cancelling.
 
 	Returns:
 		bool: True if linked document passes all validations, else False

frappe/desk/form/meta.py

@@ -63,7 +63,7 @@ class FormMeta(Meta):
 			"__linked_with", "__messages", "__print_formats", "__workflow_docs",
 			"__form_grid_templates", "__listview_template", "__tree_js",
 			"__dashboard", "__kanban_column_fields", '__templates',
-			'__custom_js'):
+			'__custom_js', '__custom_list_js'):
 			d[k] = self.get(k)
 
 		# d['fields'] = d.get('fields', [])
@@ -130,9 +130,23 @@ class FormMeta(Meta):
 	def add_custom_script(self):
 		"""embed all require files"""
 		# custom script
-		custom = frappe.db.get_value("Custom Script", {"dt": self.name, "enabled": 1}, "script") or ""
+		client_scripts = frappe.db.get_all("Client Script",
+			filters={"dt": self.name, "enabled": 1},
+			fields=["script", "view"],
+			order_by="creation asc"
+		) or ""
 
-		self.set("__custom_js", custom)
+		list_script = ''
+		form_script = ''
+		for script in client_scripts:
+			if script.view == 'List':
+				list_script += script.script
+
+			if script.view == 'Form':
+				form_script += script.script
+
+		self.set("__custom_js", form_script)
+		self.set("__custom_list_js", list_script)
 
 	def add_search_fields(self):
 		"""add search fields found in the doctypes indicated by link fields' options"""

frappe/desk/form/utils.py

@@ -47,7 +47,7 @@ def validate_link():
 			except Exception as e:
 				error_message = str(e).split("Unknown column '")
 				fieldname = None if len(error_message)<=1 else error_message[1].split("'")[0]
-				frappe.msgprint(_("Wrong fieldname <b>{0}</b> in add_fetch configuration of custom script").format(fieldname))
+				frappe.msgprint(_("Wrong fieldname <b>{0}</b> in add_fetch configuration of custom client script").format(fieldname))
 				frappe.errprint(frappe.get_traceback())
 
 			if fetch_value:

frappe/desk/leaderboard.py

@@ -16,8 +16,18 @@ def get_leaderboards():
 
 @frappe.whitelist()
 def get_energy_point_leaderboard(date_range, company = None, field = None, limit = None):
+	all_users = frappe.db.get_all('User',
+		filters = {
+			'name': ['not in', ['Administrator', 'Guest']],
+			'enabled': 1,
+			'user_type': ['!=', 'Website User']
+		},
+		order_by = 'name ASC')
+	all_users_list = list(map(lambda x: x['name'], all_users))
+
 	filters = [
 		['type', '!=', 'Review'],
+		['user', 'in', all_users_list]
 	]
 	if date_range:
 		date_range = frappe.parse_json(date_range)
@@ -28,15 +38,7 @@ def get_energy_point_leaderboard(date_range, company = None, field = None, limit
 		group_by = 'user',
 		order_by = 'value desc'
 	)
-	all_users = frappe.db.get_all('User',
-		filters = {
-			'name': ['not in', ['Administrator', 'Guest']],
-			'enabled': 1,
-			'user_type': ['!=', 'Website User']
-		},
-		order_by = 'name ASC')
 
-	all_users_list = list(map(lambda x: x['name'], all_users))
 	energy_point_users_list = list(map(lambda x: x['name'], energy_point_users))
 	for user in all_users_list:
 		if user not in energy_point_users_list:

frappe/desk/page/leaderboard/leaderboard.js

@@ -141,7 +141,7 @@ class Leaderboard {
 	}
 
 	create_date_range_field() {
-		let timespan_field = $(this.parent).find(`.frappe-control[data-original-title='Timespan']`);
+		let timespan_field = $(this.parent).find(`.frappe-control[data-original-title=${__('Timespan')}]`);
 		this.date_range_field = $(`<div class="from-date-field"></div>`).insertAfter(timespan_field).hide();
 
 		let date_field = frappe.ui.form.make_control({

frappe/desk/page/setup_wizard/setup_wizard.js

@@ -201,7 +201,7 @@ frappe.setup.SetupWizard = class SetupWizard extends frappe.ui.Slides {
 					this.abort_setup(r.message.fail);
 				}
 			},
-			error: this.abort_setup("Error in setup", true)
+			error: () => this.abort_setup("Error in setup")
 		});
 	}
 

frappe/desk/page/user_profile/user_profile_controller.js

@@ -151,32 +151,30 @@ class UserProfile {
 
 	// eslint-disable-next-line no-unused-vars
 	render_percentage_chart(field, title) {
-		// REDESIGN-TODO: chart seems to be broken. Enable this once fixed.
-		this.wrapper.find('.percentage-chart-container').hide();
-		// frappe.xcall('frappe.desk.page.user_profile.user_profile.get_energy_points_percentage_chart_data', {
-		// 	user: this.user_id,
-		// 	field: field
-		// }).then(chart => {
-		// 	if (chart.labels.length) {
-		// 		this.percentage_chart = new frappe.Chart('.performance-percentage-chart', {
-		// 			type: 'percentage',
-		// 			data: {
-		// 				labels: chart.labels,
-		// 				datasets: chart.datasets
-		// 			},
-		// 			truncateLegends: 1,
-		// 			barOptions: {
-		// 				height: 11,
-		// 				depth: 1
-		// 			},
-		// 			height: 200,
-		// 			maxSlices: 8,
-		// 			colors: ['purple', 'blue', 'cyan', 'teal', 'pink', 'red', 'orange', 'yellow'],
-		// 		});
-		// 	} else {
-		// 		this.wrapper.find('.percentage-chart-container').hide();
-		// 	}
-		// });
+		frappe.xcall('frappe.desk.page.user_profile.user_profile.get_energy_points_percentage_chart_data', {
+			user: this.user_id,
+			field: field
+		}).then(chart => {
+			if (chart.labels.length) {
+				this.percentage_chart = new frappe.Chart('.performance-percentage-chart', {
+					type: 'percentage',
+					data: {
+						labels: chart.labels,
+						datasets: chart.datasets
+					},
+					truncateLegends: 1,
+					barOptions: {
+						height: 11,
+						depth: 1
+					},
+					height: 200,
+					maxSlices: 8,
+					colors: ['purple', 'blue', 'cyan', 'teal', 'pink', 'red', 'orange', 'yellow'],
+				});
+			} else {
+				this.wrapper.find('.percentage-chart-container').hide();
+			}
+		});
 	}
 
 	create_line_chart_filters() {
@@ -452,4 +450,4 @@ class UserProfileTimeline extends BaseTimeline {
 }
 
 frappe.provide('frappe.ui');
-frappe.ui.UserProfile = UserProfile;
+frappe.ui.UserProfile = UserProfile;

frappe/desk/query_report.py

@@ -164,10 +164,14 @@ def get_script(report_name):
 	module = report.module or frappe.db.get_value(
 		"DocType", report.ref_doctype, "module"
 	)
-	module_path = get_module_path(module)
-	report_folder = os.path.join(module_path, "report", scrub(report.name))
-	script_path = os.path.join(report_folder, scrub(report.name) + ".js")
-	print_path = os.path.join(report_folder, scrub(report.name) + ".html")
+
+	is_custom_module = frappe.get_cached_value("Module Def", module, "custom")
+
+	# custom modules are virtual modules those exists in DB but not in disk.
+	module_path = '' if is_custom_module else get_module_path(module)
+	report_folder = module_path and os.path.join(module_path, "report", scrub(report.name))
+	script_path = report_folder and os.path.join(report_folder, scrub(report.name) + ".js")
+	print_path = report_folder and os.path.join(report_folder, scrub(report.name) + ".html")
 
 	script = None
 	if os.path.exists(script_path):

frappe/desk/search.py

@@ -80,13 +80,15 @@ def search_widget(doctype, txt, query=None, searchfield=None, start=0,
 			is_whitelisted(frappe.get_attr(query))
 			frappe.response["values"] = frappe.call(query, doctype, txt,
 				searchfield, start, page_length, filters, as_dict=as_dict)
-		except Exception as e:
+		except frappe.exceptions.PermissionError as e:
 			if frappe.local.conf.developer_mode:
 				raise e
 			else:
 				frappe.respond_as_web_page(title='Invalid Method', html='Method not found',
 				indicator_color='red', http_status_code=404)
 			return
+		except Exception as e:
+			raise e
 	elif not query and doctype in standard_queries:
 		# from standard queries
 		search_widget(doctype, txt, standard_queries[doctype][0],

frappe/desk/utils.py

@@ -7,12 +7,13 @@ def validate_route_conflict(doctype, name):
 	'''
 	Raises exception if name clashes with routes from other documents for /app routing
 	'''
-	if frappe.flags.ignore_route_conflict_validation:
-		return
 
 	all_names = []
 	for _doctype in ['Page', 'Workspace', 'DocType']:
-		all_names.extend([slug(d) for d in frappe.get_all(_doctype, pluck='name') if (doctype != _doctype and d != name)])
+		try:
+			all_names.extend([slug(d) for d in frappe.get_all(_doctype, pluck='name') if (doctype != _doctype and d != name)])
+		except frappe.db.TableMissingError:
+			pass
 
 	if slug(name) in all_names:
 		frappe.msgprint(frappe._('Name already taken, please set a new name'))

frappe/email/doctype/email_account/email_account.py

@@ -90,6 +90,29 @@ class EmailAccount(Document):
 			if self.append_to not in valid_doctypes:
 				frappe.throw(_("Append To can be one of {0}").format(comma_or(valid_doctypes)))
 
+	def before_save(self):
+		messages = []
+		as_list = 1
+		if not self.enable_incoming and self.default_incoming:
+			self.default_incoming = False
+			messages.append(_("{} has been disabled. It can only be enabled if {} is checked.")
+				.format(
+					frappe.bold(_('Default Incoming')),
+					frappe.bold(_('Enable Incoming'))
+				)
+			)
+		if not self.enable_outgoing and self.default_outgoing:
+			self.default_outgoing = False
+			messages.append(_("{} has been disabled. It can only be enabled if {} is checked.")
+				.format(
+						frappe.bold(_('Default Outgoing')),
+						frappe.bold(_('Enable Outgoing'))
+					)
+				)
+		if messages:
+			if len(messages) == 1: (as_list, messages) = (0, messages[0])
+			frappe.msgprint(messages, as_list= as_list, indicator='orange', title=_("Defaults Updated"))
+
 	def on_update(self):
 		"""Check there is only one default of each type."""
 		from frappe.core.doctype.user.user import setup_user_email_inbox

frappe/email/doctype/newsletter/newsletter.json

@@ -19,9 +19,12 @@
   "message",
   "message_md",
   "message_html",
+  "section_break_13",
   "send_unsubscribe_link",
   "send_attachments",
+  "column_break_9",
   "published",
+  "send_webview_link",
   "route",
   "test_the_newsletter",
   "test_email_id",
@@ -160,6 +163,21 @@
    "fieldtype": "Check",
    "label": "Schedule Sending",
    "read_only_depends_on": "eval: doc.email_sent"
+  },
+  {
+   "fieldname": "column_break_9",
+   "fieldtype": "Column Break"
+  },
+  {
+   "default": "0",
+   "depends_on": "published",
+   "fieldname": "send_webview_link",
+   "fieldtype": "Check",
+   "label": "Send Web View Link"
+  },
+  {
+   "fieldname": "section_break_13",
+   "fieldtype": "Section Break"
   }
  ],
  "has_web_view": 1,
@@ -169,7 +187,7 @@
  "is_published_field": "published",
  "links": [],
  "max_attachments": 3,
- "modified": "2020-08-24 19:59:37.262500",
+ "modified": "2021-02-22 14:33:56.095380",
  "modified_by": "Administrator",
  "module": "Email",
  "name": "Newsletter",

frappe/email/doctype/newsletter/newsletter.py

@@ -68,13 +68,17 @@ class Newsletter(WebsiteGenerator):
 				except IOError:
 					frappe.throw(_("Unable to find attachment {0}").format(file.name))
 
-		send(recipients=self.recipients, sender=sender,
-			subject=self.subject, message=self.get_message(),
+		args = {
+			"message": self.get_message(),
+			"name": self.name
+		}
+		frappe.sendmail(recipients=self.recipients, sender=sender,
+			subject=self.subject, message=self.get_message(), template="newsletter",
 			reference_doctype=self.doctype, reference_name=self.name,
 			add_unsubscribe_link=self.send_unsubscribe_link, attachments=attachments,
 			unsubscribe_method="/unsubscribe",
 			unsubscribe_params={"name": self.name},
-			send_priority=0, queue_separately=True)
+			send_priority=0, queue_separately=True, args=args)
 
 		if not frappe.flags.in_test:
 			frappe.db.auto_commit_on_many_writes = False

frappe/email/email_body.py

@@ -297,8 +297,9 @@ def inline_style_in_html(html):
 
 	for app in apps:
 		path = 'assets/{0}/css/email.css'.format(app)
-		if os.path.exists(os.path.abspath(path)):
-			css_files.append(path)
+		css_files.append(path)
+
+	css_files = [css_file for css_file in css_files if os.path.exists(os.path.abspath(css_file))]
 
 	p = Premailer(html=html, external_styles=css_files, strip_important=False)
 

frappe/email/test_email_body.py

@@ -17,7 +17,7 @@ class TestEmailBody(unittest.TestCase):
 <div>
 	<h3>Hey John Doe!</h3>
 	<p>This is embedded image you asked for</p>
-	<img embed="assets/frappe/images/favicon.png" />
+	<img embed="assets/frappe/images/frappe-favicon.svg" />
 </div>
 '''
 		email_text = '''
@@ -25,7 +25,7 @@ Hey John Doe!
 This is the text version of this email
 '''
 
-		img_path = os.path.abspath('assets/frappe/images/favicon.png')
+		img_path = os.path.abspath('assets/frappe/images/frappe-favicon.svg')
 		with open(img_path, 'rb') as f:
 			img_content = f.read()
 			img_base64 = base64.b64encode(img_content).decode()
@@ -77,12 +77,11 @@ This is the text version of this email
 
 	def test_image(self):
 		img_signature = '''
-Content-Type: image/png
+Content-Type: image/svg+xml
 MIME-Version: 1.0
 Content-Transfer-Encoding: base64
-Content-Disposition: inline; filename="favicon.png"
+Content-Disposition: inline; filename="frappe-favicon.svg"
 '''
-
 		self.assertTrue(img_signature in self.email_string)
 		self.assertTrue(self.img_base64 in self.email_string)
 
@@ -117,7 +116,7 @@ w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
 	def test_replace_filename_with_cid(self):
 		original_message = '''
 			<div>
-				<img embed="assets/frappe/images/favicon.png" alt="test" />
+				<img embed="assets/frappe/images/frappe-favicon.svg" alt="test" />
 				<img embed="notexists.jpg" />
 			</div>
 		'''

frappe/exceptions.py

@@ -33,6 +33,9 @@ class PermissionError(Exception):
 class DoesNotExistError(ValidationError):
 	http_status_code = 404
 
+class PageDoesNotExistError(ValidationError):
+	http_status_code = 404
+
 class NameError(Exception):
 	http_status_code = 409
 

frappe/geo/country_info.json

@@ -2729,11 +2729,11 @@
  },
  "Zimbabwe": {
   "code": "zw",
-  "currency": "ZWD",
-  "currency_fraction": "Thebe",
+  "currency": "ZWL",
+  "currency_fraction": "Cent",
   "currency_fraction_units": 100,
   "currency_name": "Zimbabwe Dollar",
-  "currency_symbol": "P",
+  "currency_symbol": "ZWL$",
   "number_format": "# ###.##",
   "timezones": [
    "Africa/Harare"

frappe/hooks.py

@@ -58,6 +58,11 @@ website_route_rules = [
 	{"from_route": "/kb/<category>", "to_route": "Help Article"},
 	{"from_route": "/newsletters", "to_route": "Newsletter"},
 	{"from_route": "/profile", "to_route": "me"},
+	{"from_route": "/app/<path:app_path>", "to_route": "app"},
+]
+
+website_redirects = [
+	{"source": r"/desk(.*)", "target": r"/app\1"},
 ]
 
 base_template = "templates/base.html"
@@ -202,8 +207,7 @@ scheduler_events = {
 		"frappe.deferred_insert.save_to_db",
 		"frappe.desk.form.document_follow.send_hourly_updates",
 		"frappe.integrations.doctype.google_calendar.google_calendar.sync",
-		"frappe.email.doctype.newsletter.newsletter.send_scheduled_email",
-		"frappe.utils.password.delete_password_reset_cache"
+		"frappe.email.doctype.newsletter.newsletter.send_scheduled_email"
 	],
 	"daily": [
 		"frappe.email.queue.set_expiry_for_email_queue",
@@ -286,61 +290,70 @@ before_migrate = ['frappe.patches.v11_0.sync_user_permission_doctype_before_migr
 after_migrate = ['frappe.website.doctype.website_theme.website_theme.after_migrate']
 
 otp_methods = ['OTP App','Email','SMS']
-user_privacy_documents = [
-	{
-		'doctype': 'File',
-		'match_field': 'attached_to_name',
-		'personal_fields': ['file_name', 'file_url'],
-		'applies_to_website_user': 1
-	},
-	{
-		'doctype': 'Email Group Member',
-		'match_field': 'email',
-	},
-	{
-		'doctype': 'Email Unsubscribe',
-		'match_field': 'email',
-	},
-	{
-		'doctype': 'Email Queue',
-		'match_field': 'sender',
-	},
-	{
-		'doctype': 'Email Queue Recipient',
-		'match_field': 'recipient',
-	},
-	{
-		'doctype': 'Contact',
-		'match_field': 'email_id',
-		'personal_fields': ['first_name', 'last_name', 'phone', 'mobile_no'],
-	},
-	{
-		'doctype': 'Contact Email',
-		'match_field': 'email_id',
-	},
-	{
-		'doctype': 'Address',
-		'match_field': 'email_id',
-		'personal_fields': ['address_title', 'address_line1', 'address_line2', 'city', 'county', 'state', 'pincode',
-			'phone', 'fax'],
-	},
-	{
-		'doctype': 'Communication',
-		'match_field': 'sender',
-		'personal_fields': ['sender_full_name', 'phone_no', 'content'],
-	},
-	{
-		'doctype': 'Communication',
-		'match_field': 'recipients',
-	},
-	{
-		'doctype': 'User',
-		'match_field': 'name',
-		'personal_fields': ['email', 'username', 'first_name', 'middle_name', 'last_name', 'full_name', 'birth_date',
-			'user_image', 'phone', 'mobile_no', 'location', 'banner_image', 'interest', 'bio', 'email_signature'],
-		'applies_to_website_user': 1
-	},
 
+user_data_fields = [
+	{"doctype": "Access Log", "strict": True},
+	{"doctype": "Activity Log", "strict": True},
+	{"doctype": "Comment", "strict": True},
+	{
+		"doctype": "Contact",
+		"filter_by": "email_id",
+		"redact_fields": ["first_name", "last_name", "phone", "mobile_no"],
+		"rename": True,
+	},
+	{"doctype": "Contact Email", "filter_by": "email_id"},
+	{
+		"doctype": "Address",
+		"filter_by": "email_id",
+		"redact_fields": [
+			"address_title",
+			"address_line1",
+			"address_line2",
+			"city",
+			"county",
+			"state",
+			"pincode",
+			"phone",
+			"fax",
+		],
+	},
+	{
+		"doctype": "Communication",
+		"filter_by": "sender",
+		"redact_fields": ["sender_full_name", "phone_no", "content"],
+	},
+	{"doctype": "Communication", "filter_by": "recipients"},
+	{"doctype": "Email Group Member", "filter_by": "email"},
+	{"doctype": "Email Unsubscribe", "filter_by": "email", "partial": True},
+	{"doctype": "Email Queue", "filter_by": "sender"},
+	{"doctype": "Email Queue Recipient", "filter_by": "recipient"},
+	{
+		"doctype": "File",
+		"filter_by": "attached_to_name",
+		"redact_fields": ["file_name", "file_url"],
+	},
+	{
+		"doctype": "User",
+		"filter_by": "name",
+		"redact_fields": [
+			"email",
+			"username",
+			"first_name",
+			"middle_name",
+			"last_name",
+			"full_name",
+			"birth_date",
+			"user_image",
+			"phone",
+			"mobile_no",
+			"location",
+			"banner_image",
+			"interest",
+			"bio",
+			"email_signature",
+		],
+	},
+	{"doctype": "Version", "strict": True},
 ]
 
 global_search_doctypes = {