Merge branch 'develop' into refactor-website

2021-04-19 10:35:11 +05:30 · 2021-04-19 10:35:11 +05:30 · 3209cd5c09
commit 3209cd5c09
parent a5f274b5a3 85af21cd03
313 changed files with 6687 additions and 3622 deletions
--- a/.flake8
+++ b/.flake8
@ -0,0 +1,32 @@
+[flake8]
+ignore =
+    E121,
+    E126,
+    E127,
+    E128,
+    E203,
+    E225,
+    E226,
+    E231,
+    E241,
+    E251,
+    E261,
+    E265,
+    E302,
+    E303,
+    E305,
+    E402,
+    E501,
+    E741,
+    W291,
+    W292,
+    W293,
+    W391,
+    W503,
+    W504,
+    F403,
+    B007,
+    B950,
+    W191,
+
+max-line-length = 200
--- a/.github/helper/consumer_db/mariadb.json
+++ b/.github/helper/consumer_db/mariadb.json
@ -1,5 +1,6 @@
 {
-    "db_host": "localhost",
+    "db_host": "127.0.0.1",
+    "db_port": 3306,
    "db_name": "test_frappe_consumer",
    "db_password": "test_frappe",
    "allow_tests": true,
--- a/.github/helper/consumer_db/postgres.json
+++ b/.github/helper/consumer_db/postgres.json
@ -1,5 +1,6 @@
 {
-    "db_host": "localhost",
+    "db_host": "127.0.0.1",
+    "db_port": 5432,
    "db_name": "test_frappe_consumer",
    "db_password": "test_frappe",
    "db_type": "postgres",
--- a/.github/helper/install.sh
+++ b/.github/helper/install.sh
@ -0,0 +1,61 @@
+#!/bin/bash
+
+set -e
+
+cd ~ || exit
+
+pip install frappe-bench
+
+bench init frappe-bench --skip-assets --python "$(which python)" --frappe-path "${GITHUB_WORKSPACE}"
+
+mkdir ~/frappe-bench/sites/test_site
+cp "${GITHUB_WORKSPACE}/.github/helper/consumer_db/$DB.json" ~/frappe-bench/sites/test_site/site_config.json
+
+if [ "$TYPE" == "server" ]; then
+      mkdir ~/frappe-bench/sites/test_site_producer;
+      cp "${GITHUB_WORKSPACE}/.github/helper/producer_db/$DB.json" ~/frappe-bench/sites/test_site_producer/site_config.json;
+fi
+
+if [ "$DB" == "mariadb" ];then
+      mysql --host 127.0.0.1 --port 3306 -u root -e "SET GLOBAL character_set_server = 'utf8mb4'";
+      mysql --host 127.0.0.1 --port 3306 -u root -e "SET GLOBAL collation_server = 'utf8mb4_unicode_ci'";
+
+      mysql --host 127.0.0.1 --port 3306 -u root -e "CREATE DATABASE test_frappe_consumer";
+      mysql --host 127.0.0.1 --port 3306 -u root -e "CREATE USER 'test_frappe_consumer'@'localhost' IDENTIFIED BY 'test_frappe_consumer'";
+      mysql --host 127.0.0.1 --port 3306 -u root -e "GRANT ALL PRIVILEGES ON \`test_frappe_consumer\`.* TO 'test_frappe_consumer'@'localhost'";
+
+      mysql --host 127.0.0.1 --port 3306 -u root -e "CREATE DATABASE test_frappe_producer";
+      mysql --host 127.0.0.1 --port 3306 -u root -e "CREATE USER 'test_frappe_producer'@'localhost' IDENTIFIED BY 'test_frappe_producer'";
+      mysql --host 127.0.0.1 --port 3306 -u root -e "GRANT ALL PRIVILEGES ON \`test_frappe_producer\`.* TO 'test_frappe_producer'@'localhost'";
+
+      mysql --host 127.0.0.1 --port 3306 -u root -e "UPDATE mysql.user SET Password=PASSWORD('travis') WHERE User='root'";
+      mysql --host 127.0.0.1 --port 3306 -u root -e "FLUSH PRIVILEGES";
+    fi
+
+if [ "$DB" == "postgres" ];then
+    echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE DATABASE test_frappe_consumer" -U postgres;
+    echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE USER test_frappe_consumer WITH PASSWORD 'test_frappe'" -U postgres;
+
+    echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE DATABASE test_frappe_producer" -U postgres;
+    echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE USER test_frappe_producer WITH PASSWORD 'test_frappe'" -U postgres;
+fi
+
+cd ./frappe-bench || exit
+
+sed -i 's/^watch:/# watch:/g' Procfile
+sed -i 's/^schedule:/# schedule:/g' Procfile
+
+if [ "$TYPE" == "server" ]; then sed -i 's/^socketio:/# socketio:/g' Procfile; fi
+if [ "$TYPE" == "server" ]; then sed -i 's/^redis_socketio:/# redis_socketio:/g' Procfile; fi
+
+if [ "$TYPE" == "ui" ]; then bench setup requirements --node; fi
+
+# install node-sass which is required for website theme test
+cd ./apps/frappe || exit
+yarn add node-sass@4.13.1
+cd ../..
+
+bench start &
+bench --site test_site reinstall --yes
+if [ "$TYPE" == "server" ]; then bench --site test_site_producer reinstall --yes; fi
+bench build --app frappe
--- a/.github/helper/install_dependencies.sh
+++ b/.github/helper/install_dependencies.sh
@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -e
+
+# python "${GITHUB_WORKSPACE}/.github/helper/roulette.py"
+# if [[ $? != 2 ]];then
+#   exit;
+# fi
+
+ # install wkhtmltopdf
+wget -O /tmp/wkhtmltox.tar.xz https://github.com/frappe/wkhtmltopdf/raw/master/wkhtmltox-0.12.3_linux-generic-amd64.tar.xz
+tar -xf /tmp/wkhtmltox.tar.xz -C /tmp
+sudo mv /tmp/wkhtmltox/bin/wkhtmltopdf /usr/local/bin/wkhtmltopdf
+sudo chmod o+x /usr/local/bin/wkhtmltopdf
+
+# install cups
+sudo apt-get install libcups2-dev
+
+# install redis
+sudo apt-get install redis-server
+
--- a/.github/helper/producer_db/mariadb.json
+++ b/.github/helper/producer_db/mariadb.json
@ -1,5 +1,6 @@
 {
-    "db_host": "localhost",
+    "db_host": "127.0.0.1",
+    "db_port": 3306,
    "db_name": "test_frappe_producer",
    "db_password": "test_frappe",
    "allow_tests": true,
--- a/.github/helper/producer_db/postgres.json
+++ b/.github/helper/producer_db/postgres.json
@ -1,5 +1,6 @@
 {
-    "db_host": "localhost",
+    "db_host": "127.0.0.1",
+    "db_port": 5432,
    "db_name": "test_frappe_producer",
    "db_password": "test_frappe",
    "db_type": "postgres",
--- a/.github/helper/roulette.py
+++ b/.github/helper/roulette.py
@ -24,10 +24,12 @@ def is_docs(file):

 if __name__ == "__main__":
    build_type = os.environ.get("TYPE")
-    commit_range = os.environ.get("TRAVIS_COMMIT_RANGE")
+    before = os.environ.get("BEFORE")
+    after = os.environ.get("AFTER")
+    commit_range = before + '...' + after
    print("Build Type: {}".format(build_type))
    print("Commit Range: {}".format(commit_range))
-    
+
    try:
        files_changed = get_output("git diff --name-only {}".format(commit_range), shell=False)
    except Exception:
--- a/.github/helper/semgrep_rules/README.md
+++ b/.github/helper/semgrep_rules/README.md
@ -0,0 +1,38 @@
+# Semgrep linting
+
+## What is semgrep?
+Semgrep or "semantic grep" is language agnostic static analysis tool. In simple terms semgrep is syntax-aware `grep`, so unlike regex it doesn't get confused by different ways of writing same thing or whitespaces or code split in multiple lines etc.
+
+Example:
+
+To check if a translate function is using f-string or not the regex would be `r"_\(\s*f[\"']"` while equivalent rule in semgrep would be `_(f"...")`. As semgrep knows grammer of language it takes care of unnecessary whitespace, type of quotation marks etc.
+
+You can read more such examples in `.github/helper/semgrep_rules` directory.
+
+# Why/when to use this?
+We want to maintain quality of contributions, at the same time remembering all the good practices can be pain to deal with while evaluating contributions. Using semgrep if you can translate "best practice" into a rule then it can automate the task for us.
+
+## Running locally
+
+Install semgrep using homebrew `brew install semgrep` or pip `pip install semgrep`.
+
+To run locally use following command:
+
+`semgrep --config=.github/helper/semgrep_rules [file/folder names]`
+
+## Testing
+semgrep allows testing the tests. Refer to this page: https://semgrep.dev/docs/writing-rules/testing-rules/
+
+When writing new rules you should write few positive and few negative cases as shown in the guide and current tests.
+
+To run current tests: `semgrep --test --test-ignore-todo .github/helper/semgrep_rules`
+
+
+## Reference
+
+If you are new to Semgrep read following pages to get started on writing/modifying rules:
+
+- https://semgrep.dev/docs/getting-started/
+- https://semgrep.dev/docs/writing-rules/rule-syntax
+- https://semgrep.dev/docs/writing-rules/pattern-examples/
+- https://semgrep.dev/docs/writing-rules/rule-ideas/#common-use-cases
--- a/.github/helper/semgrep_rules/security.py
+++ b/.github/helper/semgrep_rules/security.py
@ -0,0 +1,6 @@
+def function_name(input):
+	# ruleid: frappe-codeinjection-eval
+	eval(input)
+
+# ok: frappe-codeinjection-eval
+eval("1 + 1")
--- a/.github/helper/semgrep_rules/security.yml
+++ b/.github/helper/semgrep_rules/security.yml
@ -0,0 +1,14 @@
+rules:
+- id: frappe-codeinjection-eval
+  patterns:
+  - pattern-not: eval("...")
+  - pattern: eval(...)
+  message: |
+    Detected the use of eval(). eval() can be dangerous if used to evaluate
+    dynamic content. Avoid it or use safe_eval().
+  languages: [python]
+  severity: ERROR
+  paths:
+    exclude:
+      - frappe/__init__.py
+      - frappe/commands/utils.py
--- a/.github/helper/semgrep_rules/translate.js
+++ b/.github/helper/semgrep_rules/translate.js
@ -0,0 +1,37 @@
+// ruleid: frappe-translation-empty-string
+__("")
+// ruleid: frappe-translation-empty-string
+__('')
+
+// ok: frappe-translation-js-formatting
+__('Welcome {0}, get started with ERPNext in just a few clicks.', [full_name]);
+
+// ruleid: frappe-translation-js-formatting
+__(`Welcome ${full_name}, get started with ERPNext in just a few clicks.`);
+
+// ok: frappe-translation-js-formatting
+__('This is fine');
+
+
+// ok: frappe-translation-trailing-spaces
+__('This is fine');
+
+// ruleid: frappe-translation-trailing-spaces
+__(' this is not ok ');
+// ruleid: frappe-translation-trailing-spaces
+__('this is not ok ');
+// ruleid: frappe-translation-trailing-spaces
+__(' this is not ok');
+
+// ok: frappe-translation-js-splitting
+__('You have {0} subscribers in your mailing list.', [subscribers.length])
+
+// todoruleid: frappe-translation-js-splitting
+__('You have') + subscribers.length + __('subscribers in your mailing list.')
+
+// ruleid: frappe-translation-js-splitting
+__('You have' + 'subscribers in your mailing list.')
+
+// ruleid: frappe-translation-js-splitting
+__('You have {0} subscribers' +
+    'in your mailing list', [subscribers.length])
--- a/.github/helper/semgrep_rules/translate.py
+++ b/.github/helper/semgrep_rules/translate.py
@ -0,0 +1,53 @@
+# Examples taken from https://frappeframework.com/docs/user/en/translations
+# This file is used for testing the tests.
+
+from frappe import _
+
+full_name = "Jon Doe"
+# ok: frappe-translation-python-formatting
+_('Welcome {0}, get started with ERPNext in just a few clicks.').format(full_name)
+
+# ruleid: frappe-translation-python-formatting
+_('Welcome %s, get started with ERPNext in just a few clicks.' % full_name)
+# ruleid: frappe-translation-python-formatting
+_('Welcome %(name)s, get started with ERPNext in just a few clicks.' % {'name': full_name})
+
+# ruleid: frappe-translation-python-formatting
+_('Welcome {0}, get started with ERPNext in just a few clicks.'.format(full_name))
+
+
+subscribers = ["Jon", "Doe"]
+# ok: frappe-translation-python-formatting
+_('You have {0} subscribers in your mailing list.').format(len(subscribers))
+
+# ruleid: frappe-translation-python-splitting
+_('You have') + len(subscribers) + _('subscribers in your mailing list.')
+
+# ruleid: frappe-translation-python-splitting
+_('You have {0} subscribers \
+    in your mailing list').format(len(subscribers))
+
+# ok: frappe-translation-python-splitting
+_('You have {0} subscribers') \
+    + 'in your mailing list'
+
+# ruleid: frappe-translation-trailing-spaces
+msg = _(" You have {0} pending invoice ")
+# ruleid: frappe-translation-trailing-spaces
+msg = _("You have {0} pending invoice ")
+# ruleid: frappe-translation-trailing-spaces
+msg = _(" You have {0} pending invoice")
+
+# ok: frappe-translation-trailing-spaces
+msg = ' ' + _("You have {0} pending invoices") + ' '
+
+# ruleid: frappe-translation-python-formatting
+_(f"can not format like this - {subscribers}")
+# ruleid: frappe-translation-python-splitting
+_(f"what" + f"this is also not cool")
+
+
+# ruleid: frappe-translation-empty-string
+_("")
+# ruleid: frappe-translation-empty-string
+_('')
--- a/.github/helper/semgrep_rules/translate.yml
+++ b/.github/helper/semgrep_rules/translate.yml
@ -0,0 +1,63 @@
+rules:
+- id: frappe-translation-empty-string
+  pattern-either:
+  - pattern: _("")
+  - pattern: __("")
+  message: |
+    Empty string is useless for translation.
+    Please refer: https://frappeframework.com/docs/user/en/translations
+  languages: [python, javascript, json]
+  severity: ERROR
+
+- id: frappe-translation-trailing-spaces
+  pattern-either:
+    - pattern: _("=~/(^[ \t]+|[ \t]+$)/")
+    - pattern: __("=~/(^[ \t]+|[ \t]+$)/")
+  message: |
+    Trailing or leading whitespace not allowed in translate strings.
+    Please refer: https://frappeframework.com/docs/user/en/translations
+  languages: [python, javascript, json]
+  severity: ERROR
+
+- id: frappe-translation-python-formatting
+  pattern-either:
+  - pattern: _("..." % ...)
+  - pattern: _("...".format(...))
+  - pattern: _(f"...")
+  message: |
+    Only positional formatters are allowed and formatting should not be done before translating.
+    Please refer: https://frappeframework.com/docs/user/en/translations
+  languages: [python]
+  severity: ERROR
+
+- id: frappe-translation-js-formatting
+  patterns:
+  - pattern: __(`...`)
+  - pattern-not: __("...")
+  message: |
+    Template strings are not allowed for text formatting.
+    Please refer: https://frappeframework.com/docs/user/en/translations
+  languages: [javascript, json]
+  severity: ERROR
+
+- id: frappe-translation-python-splitting
+  pattern-either:
+  - pattern: _(...) + ... + _(...)
+  - pattern: _("..." + "...")
+  - pattern-regex: '_\([^\)]*\\\s*'
+  message: |
+    Do not split strings inside translate function. Do not concatenate using translate functions.
+    Please refer: https://frappeframework.com/docs/user/en/translations
+  languages: [python]
+  severity: ERROR
+
+- id: frappe-translation-js-splitting
+  pattern-either:
+  - pattern-regex: '__\([^\)]*[\+\\]\s*'
+  - pattern: __('...' + '...')
+  - pattern: __('...') + __('...')
+  message: |
+    Do not split strings inside translate function. Do not concatenate using translate functions.
+    Please refer: https://frappeframework.com/docs/user/en/translations
+  languages: [javascript, json]
+  severity: ERROR
--- a/.github/stale.yml
+++ b/.github/stale.yml
@ -1,7 +1,7 @@
 # Configuration for probot-stale - https://github.com/probot/stale

 # Number of days of inactivity before an Issue or Pull Request becomes stale
-daysUntilStale: 10
+daysUntilStale: 7

 # Number of days of inactivity before a stale Issue or Pull Request is closed.
 # Set to false to disable. If disabled, issues still need to be closed manually, but will remain marked as stale.
@ -28,7 +28,7 @@ markComment: >
  you can always reopen the PR when you're ready. Thank you for contributing.

 # Limit the number of actions per hour, from 1-30. Default is 30
-limitPerRun: 30
+limitPerRun: 10

 # Limit to only `issues` or `pulls`
 only: pulls
--- a/.github/workflows/ci-tests.yml
+++ b/.github/workflows/ci-tests.yml
@ -0,0 +1,157 @@
+name: CI
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, labeled, unlabeled]
+  workflow_dispatch:
+  push:
+
+jobs:
+  test:
+    runs-on: ubuntu-18.04
+
+    strategy:
+      fail-fast: false
+      matrix:
+       include:
+        - DB: "mariadb"
+          TYPE: "server"
+          JOB_NAME: "Python MariaDB"
+          RUN_COMMAND: bench --site test_site run-tests --coverage
+
+        - DB: "postgres"
+          TYPE: "server"
+          JOB_NAME: "Python PostgreSQL"
+          RUN_COMMAND: bench --site test_site run-tests --coverage
+
+        - DB: "mariadb"
+          TYPE: "ui"
+          JOB_NAME: "UI MariaDB"
+          RUN_COMMAND:  bench --site test_site run-ui-tests frappe --headless
+
+    name: ${{ matrix.JOB_NAME }}
+
+    services:
+      mysql:
+        image: mariadb:10.3
+        env:
+          MYSQL_ALLOW_EMPTY_PASSWORD: YES
+        ports:
+          - 3306:3306
+        options: --health-cmd="mysqladmin ping" --health-interval=5s --health-timeout=2s --health-retries=3
+
+      postgres:
+        image: postgres:12.4
+        env:
+          POSTGRES_PASSWORD: travis
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
+    steps:
+      - name: Clone
+        uses: actions/checkout@v2
+
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+
+      - uses: actions/setup-node@v2
+        with:
+          node-version: '12'
+          check-latest: true
+
+      - name: Add to Hosts
+        run: |
+          echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts
+          echo "127.0.0.1 test_site_producer" | sudo tee -a /etc/hosts
+
+      - name: Cache pip
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+            ${{ runner.os }}-
+
+      - name: Cache node modules
+        uses: actions/cache@v2
+        env:
+          cache-name: cache-node-modules
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-build-${{ env.cache-name }}-
+            ${{ runner.os }}-build-
+            ${{ runner.os }}-
+
+      - name: Get yarn cache directory path
+        id: yarn-cache-dir-path
+        run: echo "::set-output name=dir::$(yarn cache dir)"
+
+      - uses: actions/cache@v2
+        id: yarn-cache
+        with:
+          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
+          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-yarn-
+
+      - name: Cache cypress binary
+        if: matrix.TYPE == 'ui'
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache
+          key: ${{ runner.os }}-cypress-
+          restore-keys: |
+            ${{ runner.os }}-cypress-
+            ${{ runner.os }}-
+
+      - name: Install Dependencies
+        run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh
+        env:
+          BEFORE: ${{ env.GITHUB_EVENT_PATH.before }}
+          AFTER: ${{ env.GITHUB_EVENT_PATH.after }}
+          TYPE: ${{ matrix.TYPE }}
+
+      - name: Install
+        run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh
+        env:
+          DB: ${{ matrix.DB }}
+          TYPE: ${{ matrix.TYPE }}
+
+      - name: Run Set-Up
+        if: matrix.TYPE == 'ui'
+        run: cd ~/frappe-bench/ && bench --site test_site execute frappe.utils.install.complete_setup_wizard
+        env:
+          DB: ${{ matrix.DB }}
+          TYPE: ${{ matrix.TYPE }}
+
+      - name: Setup tmate session
+        if: contains(github.event.pull_request.labels.*.name, 'debug-gha')
+        uses: mxschmitt/action-tmate@v3
+
+      - name: Run Tests
+        run: cd ~/frappe-bench/ && ${{ matrix.RUN_COMMAND }}
+        env:
+          DB: ${{ matrix.DB }}
+          TYPE: ${{ matrix.TYPE }}
+
+      - name: Coverage
+        if: matrix.TYPE == 'server'
+        run: |
+          cp ~/frappe-bench/sites/.coverage ${GITHUB_WORKSPACE}
+          cd ${GITHUB_WORKSPACE}
+          pip install coveralls==2.2.0
+          pip install coverage==4.5.4
+          coveralls
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_TOKEN }}
--- a/.github/workflows/semgrep.yml
+++ b/.github/workflows/semgrep.yml
@ -1,13 +1,22 @@
 name: Semgrep

 on:
-  pull_request: {}
-
+  pull_request:
+    branches:
+      - develop
 jobs:
  semgrep:
    name: Frappe Linter
    runs-on: ubuntu-latest
    steps:
-      - uses: actions/checkout@v2
-      - uses: returntocorp/semgrep-action@v1
-
+    - uses: actions/checkout@v2
+    - name: Setup python3
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+    - name: Run semgrep
+      run: |
+        python -m pip install -q semgrep
+        git fetch origin $GITHUB_BASE_REF:$GITHUB_BASE_REF -q
+        files=$(git diff --name-only --diff-filter=d $GITHUB_BASE_REF)
+        [[ -d .github/helper/semgrep_rules ]] && semgrep --config=.github/helper/semgrep_rules --quiet --error $files
--- a/.mergify.yml
+++ b/.mergify.yml
@ -3,7 +3,9 @@ pull_request_rules:
    conditions:
      - status-success=Sider
      - status-success=Semantic Pull Request
-      - status-success=Travis CI - Pull Request
+      - status-success=Python MariaDB
+      - status-success=Python PostgreSQL
+      - status-success=UI MariaDB
      - status-success=security/snyk (frappe)
      - label!=dont-merge
      - label!=squash
@ -14,7 +16,9 @@ pull_request_rules:
  - name: Automatic squash on CI success and review
    conditions:
      - status-success=Sider
-      - status-success=Travis CI - Pull Request
+      - status-success=Python MariaDB
+      - status-success=Python PostgreSQL
+      - status-success=UI MariaDB
      - status-success=security/snyk (frappe)
      - label!=dont-merge
      - label=squash
--- a/.semgrep.yml
+++ b/.semgrep.yml
@ -1,29 +0,0 @@
-#Reference: https://semgrep.dev/docs/writing-rules/rule-syntax/
-
-rules:
- id: eval
-  patterns:
-  - pattern-not: eval("...")
-  - pattern: eval(...)
-  message: |
-    Detected the use of eval(). eval() can be dangerous if used to evaluate
-    dynamic content. Avoid it or use safe_eval().
-  languages:
-  - python
-  severity: ERROR
-
-# translations
- id: frappe-translation-syntax-python
-  pattern-either:
-  - pattern: _(f"...")  # f-strings not allowed
-  - pattern: _("..." + "...")  # concatenation not allowed
-  - pattern: _("")  # empty string is meaningless
-  - pattern: _("..." % ...)  # Only positional formatters are allowed.
-  - pattern: _("...".format(...))  # format should not be used before translating
-  - pattern: _("...") + ... + _("...")  # don't split strings
-  message: |
-    Incorrect use of translation function detected.
-    Please refer: https://frappeframework.com/docs/user/en/translations
-  languages:
-  - python
-  severity: ERROR
--- a/.travis.yml
+++ b/.travis.yml
@ -1,129 +0,0 @@
-language: python
-dist: bionic
-
-addons:
-  hosts:
-    - test_site
-    - test_site_producer
-  mariadb: 10.3
-  postgresql: 9.5
-  firefox: latest
-
-services:
-  - xvfb
-  - mysql
-
-git:
-  depth: 1
-
-cache:
-  pip: true
-  npm: true
-  yarn: true
-  directories:
-    # we also need to cache folder with Cypress binary
-    # https://docs.cypress.io/guides/guides/continuous-integration.html#Caching
-    - ~/.cache
-
-
-matrix:
-  include:
-  - name: "Python 3.7 MariaDB"
-    python: 3.7
-    env: DB=mariadb TYPE=server
-    script: bench --verbose --site test_site run-tests --coverage
-
-  - name: "Python 3.7 PostgreSQL"
-    python: 3.7
-    env: DB=postgres TYPE=server
-    script: bench --verbose --site test_site run-tests --coverage
-
-  - name: "Cypress"
-    python: 3.7
-    env: DB=mariadb TYPE=ui
-    before_script:
-    - bench --site test_site execute frappe.utils.install.complete_setup_wizard
-    script: bench --site test_site run-ui-tests frappe --headless
-
-before_install:
- # do we really want to run travis?
-  - |
-    python ./.travis/roulette.py
-    if [[ $? != 2 ]];then
-      exit;
-    fi
-
- # install wkhtmltopdf
-  - wget -O /tmp/wkhtmltox.tar.xz https://github.com/frappe/wkhtmltopdf/raw/master/wkhtmltox-0.12.3_linux-generic-amd64.tar.xz
-  - tar -xf /tmp/wkhtmltox.tar.xz -C /tmp
-  - sudo mv /tmp/wkhtmltox/bin/wkhtmltopdf /usr/local/bin/wkhtmltopdf
-  - sudo chmod o+x /usr/local/bin/wkhtmltopdf
-
-  # install cups
-  - sudo apt-get install libcups2-dev
-
-install:
-  - cd ~
-  - source ./.nvm/nvm.sh
-  - nvm install 12
-
-  - pip install frappe-bench
-
-  - bench init frappe-bench --skip-assets --python $(which python) --frappe-path $TRAVIS_BUILD_DIR
-
-  - mkdir ~/frappe-bench/sites/test_site
-  - cp $TRAVIS_BUILD_DIR/.travis/consumer_db/$DB.json ~/frappe-bench/sites/test_site/site_config.json
-
-  - if [ $TYPE == "server" ]; then
-      mkdir ~/frappe-bench/sites/test_site_producer;
-      cp $TRAVIS_BUILD_DIR/.travis/producer_db/$DB.json ~/frappe-bench/sites/test_site_producer/site_config.json;
-    fi
-
-  - if [ $DB == "mariadb" ];then
-      mysql -u root -e "SET GLOBAL character_set_server = 'utf8mb4'";
-      mysql -u root -e "SET GLOBAL collation_server = 'utf8mb4_unicode_ci'";
-
-      mysql -u root -e "CREATE DATABASE test_frappe_consumer";
-      mysql -u root -e "CREATE USER 'test_frappe_consumer'@'localhost' IDENTIFIED BY 'test_frappe_consumer'";
-      mysql -u root -e "GRANT ALL PRIVILEGES ON \`test_frappe_consumer\`.* TO 'test_frappe_consumer'@'localhost'";
-
-      mysql -u root -e "CREATE DATABASE test_frappe_producer";
-      mysql -u root -e "CREATE USER 'test_frappe_producer'@'localhost' IDENTIFIED BY 'test_frappe_producer'";
-      mysql -u root -e "GRANT ALL PRIVILEGES ON \`test_frappe_producer\`.* TO 'test_frappe_producer'@'localhost'";
-
-      mysql -u root -e "UPDATE mysql.user SET Password=PASSWORD('travis') WHERE User='root'";
-      mysql -u root -e "FLUSH PRIVILEGES";
-    fi
-
-  - if [ $DB == "postgres" ];then
-      psql -c "CREATE DATABASE test_frappe_consumer" -U postgres;
-      psql -c "CREATE USER test_frappe_consumer WITH PASSWORD 'test_frappe'" -U postgres;
-
-      psql -c "CREATE DATABASE test_frappe_producer" -U postgres;
-      psql -c "CREATE USER test_frappe_producer WITH PASSWORD 'test_frappe'" -U postgres;
-    fi
-
-  - cd ./frappe-bench
-
-  - sed -i 's/^watch:/# watch:/g' Procfile
-  - sed -i 's/^schedule:/# schedule:/g' Procfile
-
-  - if [ $TYPE == "server" ]; then sed -i 's/^socketio:/# socketio:/g' Procfile; fi
-  - if [ $TYPE == "server" ]; then sed -i 's/^redis_socketio:/# redis_socketio:/g' Procfile; fi
-
-  - if [ $TYPE == "ui" ]; then bench setup requirements --node; fi
-
-  # install node-sass which is required for website theme test
-  - cd ./apps/frappe
-  - yarn add node-sass@4.13.1
-  - cd ../..
-
-  - bench start &
-  - bench --site test_site reinstall --yes
-  - if [ $TYPE == "server" ]; then bench --site test_site_producer reinstall --yes; fi
-  - bench build --app frappe
-
-after_script:
-  - pip install coverage==4.5.4
-  - pip install python-coveralls
-  - coveralls -b apps/frappe -d ../../sites/.coverage
--- a/README.md
+++ b/README.md
@ -14,8 +14,8 @@
 </div>

 <div align="center">
-    <a href="https://travis-ci.com/frappe/frappe">
-        <img src="https://travis-ci.com/frappe/frappe.svg?branch=develop">
+    <a href="https://github.com/frappe/frappe/actions/workflows/ci-tests.yml">
+        <img src="https://github.com/frappe/frappe/actions/workflows/ci-tests.yml/badge.svg?branch=develop">
    </a>
    <a href='https://frappeframework.com/docs'>
        <img src='https://img.shields.io/badge/docs-📖-7575FF.svg?style=flat-square'/>
--- a/cypress/integration/form.js
+++ b/cypress/integration/form.js
@ -8,7 +8,7 @@ context('Form', () => {
 	});
 	it('create a new form', () => {
 		cy.visit('/app/todo/new');
-		cy.fill_field('description', 'this is a test todo', 'Text Editor').blur();
+		cy.fill_field('description', 'this is a test todo', 'Text Editor');
 		cy.wait(300);
 		cy.get('.page-title').should('contain', 'Not Saved');
 		cy.intercept({
@ -42,18 +42,33 @@ context('Form', () => {
 	it('validates behaviour of Data options validations in child table', () => {
 		// test email validations for set_invalid controller
 		let website_input = 'website.in';
+		let valid_email = 'user@email.com';
 		let expectBackgroundColor = 'rgb(255, 245, 245)';

 		cy.visit('/app/contact/new');
 		cy.get('.frappe-control[data-fieldname="email_ids"]').as('table');
 		cy.get('@table').find('button.grid-add-row').click();
-		cy.get('.grid-body .rows [data-fieldname="email_id"]').click();
-		cy.get('@table').find('input.input-with-feedback.form-control').as('email_input');
-		cy.get('@email_input').type(website_input, { waitForAnimations: false });
+		cy.get('@table').find('button.grid-add-row').click();
+		cy.get('@table').find('[data-idx="1"]').as('row1');
+		cy.get('@table').find('[data-idx="2"]').as('row2');
+		cy.get('@row1').click();
+		cy.get('@row1').find('input.input-with-feedback.form-control').as('email_input1');
+
+		cy.get('@email_input1').type(website_input, { waitForAnimations: false });
 		cy.fill_field('company_name', 'Test Company');
-		cy.get('@email_input').should($div => {
+
+		cy.get('@row2').click();
+		cy.get('@row2').find('input.input-with-feedback.form-control').as('email_input2');
+		cy.get('@email_input2').type(valid_email, { waitForAnimations: false });
+
+		cy.get('@row1').click();
+		cy.get('@email_input1').should($div => {
 			const style = window.getComputedStyle($div[0]);
 			expect(style.backgroundColor).to.equal(expectBackgroundColor);
 		});
+		cy.get('@email_input1').should('have.class', 'invalid');
+
+		cy.get('@row2').click();
+		cy.get('@email_input2').should('not.have.class', 'invalid');
 	});
 });
--- a/frappe/init.py
+++ b/frappe/init.py
@ -15,6 +15,7 @@ from __future__ import unicode_literals, print_function
 from six import iteritems, binary_type, text_type, string_types, PY2
 from werkzeug.local import Local, release_local
 import os, sys, importlib, inspect, json
+import typing
 from past.builtins import cmp
 import click

@ -34,6 +35,7 @@ if PY2:
 	sys.setdefaultencoding("utf-8")

 __version__ = '13.0.0-dev'
+
 __title__ = "Frappe Framework"

 local = Local()
@ -133,6 +135,14 @@ message_log = local("message_log")

 lang = local("lang")

+# This if block is never executed when running the code. It is only used for
+# telling static code analyzer where to find dynamically defined attributes.
+if typing.TYPE_CHECKING:
+	from frappe.database.mariadb.database import MariaDBDatabase
+	from frappe.database.postgres.database import PostgresDatabase
+	db: typing.Union[MariaDBDatabase, PostgresDatabase]
+# end: static analysis hack
+
 def init(site, sites_path=None, new_site=False):
 	"""Initialize frappe for the current site. Reset thread locals `frappe.local`"""
 	if getattr(local, "initialised", None):
@ -555,8 +565,15 @@ def whitelist(allow_guest=False, xss_safe=False, methods=None):

 	def innerfn(fn):
 		global whitelisted, guest_methods, xss_safe_methods, allowed_http_methods_for_whitelisted_func
-		whitelisted.append(fn)

+		# get function from the unbound / bound method
+		# this is needed because functions can be compared, but not methods
+		method = None
+		if hasattr(fn, '__func__'):
+			method = fn
+			fn = method.__func__
+
+		whitelisted.append(fn)
 		allowed_http_methods_for_whitelisted_func[fn] = methods

 		if allow_guest:
@ -565,10 +582,24 @@ def whitelist(allow_guest=False, xss_safe=False, methods=None):
 			if xss_safe:
 				xss_safe_methods.append(fn)

-		return fn
+		return method or fn

 	return innerfn

+def is_whitelisted(method):
+	from frappe.utils import sanitize_html
+
+	is_guest = session['user'] == 'Guest'
+	if method not in whitelisted or is_guest and method not in guest_methods:
+		throw(_("Not permitted"), PermissionError)
+
+	if is_guest and method not in xss_safe_methods:
+		# strictly sanitize form_dict
+		# escapes html characters like <> except for predefined tags like a, b, ul etc.
+		for key, value in form_dict.items():
+			if isinstance(value, string_types):
+				form_dict[key] = sanitize_html(value)
+
 def read_only():
 	def innfn(fn):
 		def wrapper_fn(*args, **kwargs):
@ -854,8 +885,8 @@ def get_meta_module(doctype):
 	import frappe.modules
 	return frappe.modules.load_doctype_module(doctype)

-def delete_doc(doctype=None, name=None, force=0, ignore_doctypes=None,
-	for_reload=False, ignore_permissions=False, flags=None, ignore_on_trash=False, ignore_missing=True):
+def delete_doc(doctype=None, name=None, force=0, ignore_doctypes=None, for_reload=False,
+	ignore_permissions=False, flags=None, ignore_on_trash=False, ignore_missing=True, delete_permanently=False):
 	"""Delete a document. Calls `frappe.model.delete_doc.delete_doc`.

 	:param doctype: DocType of document to be delete.
@ -863,10 +894,11 @@ def delete_doc(doctype=None, name=None, force=0, ignore_doctypes=None,
 	:param force: Allow even if document is linked. Warning: This may lead to data integrity errors.
 	:param ignore_doctypes: Ignore if child table is one of these.
 	:param for_reload: Call `before_reload` trigger before deleting.
-	:param ignore_permissions: Ignore user permissions."""
+	:param ignore_permissions: Ignore user permissions.
+	:param delete_permanently: Do not create a Deleted Document for the document."""
 	import frappe.model.delete_doc
 	frappe.model.delete_doc.delete_doc(doctype, name, force, ignore_doctypes, for_reload,
-		ignore_permissions, flags, ignore_on_trash, ignore_missing)
+		ignore_permissions, flags, ignore_on_trash, ignore_missing, delete_permanently)

 def delete_doc_if_exists(doctype, name, force=0):
 	"""Delete document if exists."""
@ -1202,10 +1234,10 @@ def make_property_setter(args, ignore_validate=False, validate_fields_for_doctyp
 		ps.validate_fieldtype_change()
 		ps.insert()

-def import_doc(path, ignore_links=False, ignore_insert=False, insert=False):
+def import_doc(path):
 	"""Import a file using Data Import."""
 	from frappe.core.doctype.data_import.data_import import import_doc
-	import_doc(path, ignore_links=ignore_links, ignore_insert=ignore_insert, insert=insert)
+	import_doc(path)

 def copy_doc(doc, ignore_no_copy=True):
 	""" No_copy fields also get copied."""
@ -1377,7 +1409,7 @@ def get_list(doctype, *args, **kwargs):
 		frappe.get_list("ToDo", fields="*", filters = {"description": ("like", "test%")})
 	"""
 	import frappe.model.db_query
-	return frappe.model.db_query.DatabaseQuery(doctype).execute(None, *args, **kwargs)
+	return frappe.model.db_query.DatabaseQuery(doctype).execute(*args, **kwargs)

 def get_all(doctype, *args, **kwargs):
 	"""List database query via `frappe.model.db_query`. Will **not** check for permissions.
--- a/frappe/auth.py
+++ b/frappe/auth.py
@ -215,35 +215,25 @@ class LoginManager:
 		if not (user and pwd):
 			self.fail(_('Incomplete login details'), user=user)

-		# Ignore password check if tmp_id is set, 2FA takes care of authentication.
-		validate_password = not bool(frappe.form_dict.get('tmp_id'))
-		user = User.find_by_credentials(user, pwd, validate_password=validate_password)
+		user = User.find_by_credentials(user, pwd)

 		if not user:
 			self.fail('Invalid login credentials')

-		sys_settings = frappe.get_doc("System Settings")
-		track_login_attempts = (sys_settings.allow_consecutive_login_attempts >0)
-
-		tracker_kwargs = {}
-		if track_login_attempts:
-			tracker_kwargs['lock_interval'] = sys_settings.allow_login_after_fail
-			tracker_kwargs['max_consecutive_login_attempts'] = sys_settings.allow_consecutive_login_attempts
-
-		tracker = LoginAttemptTracker(user.name, **tracker_kwargs)
-
-		if track_login_attempts and not tracker.is_user_allowed():
-			frappe.throw(_("Your account has been locked and will resume after {0} seconds")
-				.format(sys_settings.allow_login_after_fail), frappe.SecurityException)
+		# Current login flow uses cached credentials for authentication while checking OTP.
+		# Incase of OTP check, tracker for auth needs to be disabled(If not, it can remove tracker history as it is going to succeed anyway)
+		# Tracker is activated for 2FA incase of OTP.
+		ignore_tracker = should_run_2fa(user.name) and ('otp' in frappe.form_dict)
+		tracker = None if ignore_tracker else get_login_attempt_tracker(user.name)

 		if not user.is_authenticated:
-			tracker.add_failure_attempt()
+			tracker and tracker.add_failure_attempt()
 			self.fail('Invalid login credentials', user=user.name)
 		elif not (user.name == 'Administrator' or user.enabled):
-			tracker.add_failure_attempt()
+			tracker and tracker.add_failure_attempt()
 			self.fail('User disabled or missing', user=user.name)
 		else:
-			tracker.add_success_attempt()
+			tracker and tracker.add_success_attempt()
 		self.user = user.name

 	def force_user_to_reset_password(self):
@ -406,6 +396,27 @@ def validate_ip_address(user):

 	frappe.throw(_("Access not allowed from this IP Address"), frappe.AuthenticationError)

+def get_login_attempt_tracker(user_name: str, raise_locked_exception: bool = True):
+	"""Get login attempt tracker instance.
+
+	:param user_name: Name of the loggedin user
+	:param raise_locked_exception: If set, raises an exception incase of user not allowed to login
+	"""
+	sys_settings = frappe.get_doc("System Settings")
+	track_login_attempts = (sys_settings.allow_consecutive_login_attempts >0)
+	tracker_kwargs = {}
+
+	if track_login_attempts:
+		tracker_kwargs['lock_interval'] = sys_settings.allow_login_after_fail
+		tracker_kwargs['max_consecutive_login_attempts'] = sys_settings.allow_consecutive_login_attempts
+
+	tracker = LoginAttemptTracker(user_name, **tracker_kwargs)
+
+	if raise_locked_exception and track_login_attempts and not tracker.is_user_allowed():
+		frappe.throw(_("Your account has been locked and will resume after {0} seconds")
+			.format(sys_settings.allow_login_after_fail), frappe.SecurityException)
+	return tracker
+

 class LoginAttemptTracker(object):
 	"""Track login attemts of a user.
--- a/frappe/automation/doctype/auto_repeat/auto_repeat.py
+++ b/frappe/automation/doctype/auto_repeat/auto_repeat.py
@ -118,6 +118,7 @@ class AutoRepeat(Document):
 	def is_completed(self):
 		return self.end_date and getdate(self.end_date) < getdate(today())

+	@frappe.whitelist()
 	def get_auto_repeat_schedule(self):
 		schedule_details = []
 		start_date = getdate(self.start_date)
@ -328,6 +329,7 @@ class AutoRepeat(Document):
 		make(doctype=new_doc.doctype, name=new_doc.name, recipients=recipients,
 			subject=subject, content=message, attachments=attachments, send_email=1)

+	@frappe.whitelist()
 	def fetch_linked_contacts(self):
 		if self.reference_doctype and self.reference_document:
 			res = get_contacts_linking_to(self.reference_doctype, self.reference_document, fields=['email_id'])
--- a/frappe/automation/doctype/auto_repeat/test_auto_repeat.py
+++ b/frappe/automation/doctype/auto_repeat/test_auto_repeat.py
@ -196,7 +196,7 @@ def make_auto_repeat(**args):
 	return doc


-def create_submittable_doctype(doctype):
+def create_submittable_doctype(doctype, submit_perms=1):
 	if frappe.db.exists('DocType', doctype):
 		return
 	else:
@ -217,9 +217,9 @@ def create_submittable_doctype(doctype):
 				'write': 1,
 				'create': 1,
 				'delete': 1,
-				'submit': 1,
-				'cancel': 1,
-				'amend': 1
+				'submit': submit_perms,
+				'cancel': submit_perms,
+				'amend': submit_perms
 			}]
 		}).insert()

--- a/frappe/boot.py
+++ b/frappe/boot.py
@ -42,6 +42,8 @@ def get_bootinfo():
 		bootinfo.user_info = get_user_info()
 		bootinfo.sid = frappe.session['sid']

+	bootinfo.user_groups = frappe.get_all('User Group', pluck="name")
+
 	bootinfo.modules = {}
 	bootinfo.module_list = []
 	load_desktop_data(bootinfo)
--- a/frappe/change_log/v13/v13_0_0.md
+++ b/frappe/change_log/v13/v13_0_0.md
@ -0,0 +1,54 @@
+# Version 13.0.0 Release Notes
+
+## Highlights
+
+- Re-branded UI 💎 ✨🎊 ([#12277](https://github.com/frappe/frappe/pull/12277))
+- New Page Builder in Web Page ([#10035](https://github.com/frappe/frappe/pull/10035))
+- Customizable desk ([#9617](https://github.com/frappe/frappe/pull/9617))
+- Custom Dashboard for DocTypes ([#9872](https://github.com/frappe/frappe/pull/9872))
+- Widgets to make dashboards ([#9693](https://github.com/frappe/frappe/pull/9693))
+- Events Streaming ([#8567](https://github.com/frappe/frappe/pull/8567))
+- Contextual translation and Translation Tool ([#9636](https://github.com/frappe/frappe/pull/9636))
+
+### Other Features & Enhancements
+
+- Added permission to grant only `Select` access ([#12063](https://github.com/frappe/frappe/pull/12063))
+- Add columns and filters for reports via configuration ([#11287](https://github.com/frappe/frappe/pull/11287))
+- Configurable Navbar logo and dropdowns ([#11213](https://github.com/frappe/frappe/pull/11213))
+- Rule based naming of documents ([#11439](https://github.com/frappe/frappe/pull/11439))
+- New routing style, not using hashes, also /desk -> /app ([#11917](https://github.com/frappe/frappe/pull/11917))
+- Web Page tracking ([#9959](https://github.com/frappe/frappe/pull/9959))
+- Introduced "Yesterday" and "Tomorrow" options for Timespan filter ([12179](https://github.com/frappe/frappe/pull/12179))
+- Child table pagination ([#8786](https://github.com/frappe/frappe/pull/8786))
+- Introduced Duration Control ([#10248](https://github.com/frappe/frappe/pull/10248))
+- Form Tour feature ([#10287](https://github.com/frappe/frappe/pull/10287))
+<details>
+<summary>More</summary>
+
+- Introduced Map View ([#11202](https://github.com/frappe/frappe/pull/11202))
+- Custom JS & CSS support in Web Form ([#9121](https://github.com/frappe/frappe/pull/9121)) ([#9610](https://github.com/frappe/frappe/pull/9610))
+- Ability to attach photo from webcam ([#12160](https://github.com/frappe/frappe/pull/12160))
+- Added a System Console to help in debugging ([#11306](https://github.com/frappe/frappe/pull/11306))
+- Introduced System Settings to automatically delete old Prepared Reports ([#9751](https://github.com/frappe/frappe/pull/9751))
+- "Mandatory Depends On" and "Read Only Depends On" option for document fields ([#8820](https://github.com/frappe/frappe/pull/8820))
+- Added 2FA for LDAP users ([#10001](https://github.com/frappe/frappe/pull/10001))
+- Introduced Help Article Feedback system ([#10260](https://github.com/frappe/frappe/pull/10260))
+- Introduced Razorpay client ([#11418](https://github.com/frappe/frappe/pull/11418))
+- Rate Limiting ([#10310](https://github.com/frappe/frappe/pull/10310))
+- Introduced Log Settings ([#11699](https://github.com/frappe/frappe/pull/11699))
+- Enhancements in notifications ([#11398](https://github.com/frappe/frappe/pull/11398)) ([#11409](https://github.com/frappe/frappe/pull/11409))
+- Added a field-level permission check for report data ([12163](https://github.com/frappe/frappe/pull/12163))
+- Ability to cancel all linked document with a single click ([#8905](https://github.com/frappe/frappe/pull/8905))
+- Made checkboxes navigable via tab key ([#11030](https://github.com/frappe/frappe/pull/11030))
+- Renamed "Custom Script" to "Client Script" ([#12324](https://github.com/frappe/frappe/pull/12324))
+
+</details>
+
+### Performance
+
+- Faster application load ([#12364](https://github.com/frappe/frappe/pull/12364)) ([#10229](https://github.com/frappe/frappe/pull/10229)) ([#10147](https://github.com/frappe/frappe/pull/10147)) ([#9930](https://github.com/frappe/frappe/pull/9930))
+- Theme files will now be compressed to make the website load faster ([#11048](https://github.com/frappe/frappe/pull/11048))
+- Confirmation emails will be sent instantly ([#10790](https://github.com/frappe/frappe/pull/10790))
+- Faster scheduled job processing ([#9928](https://github.com/frappe/frappe/pull/9928))
+- Faster data imports ([#12565](https://github.com/frappe/frappe/pull/12565))
+- Faster CLI commands ([#12447](https://github.com/frappe/frappe/pull/12447))
--- a/frappe/client.py
+++ b/frappe/client.py
@ -8,6 +8,8 @@ import frappe.model
 import frappe.utils
 import json, os
 from frappe.utils import get_safe_filters
+from frappe.desk.reportview import validate_args
+from frappe.model.db_query import check_parent_permission

 from six import iteritems, string_types, integer_types

@ -19,7 +21,7 @@ Requests via FrappeClient are also handled here.

@frappe.whitelist()
 def get_list(doctype, fields=None, filters=None, order_by=None,
-	limit_start=None, limit_page_length=20, parent=None):
+	limit_start=None, limit_page_length=20, parent=None, debug=False, as_dict=True):
 	'''Returns a list of records by filters, fields, ordering and limit

 	:param doctype: DocType of the data to be queried
@ -31,8 +33,19 @@ def get_list(doctype, fields=None, filters=None, order_by=None,
 	if frappe.is_table(doctype):
 		check_parent_permission(parent, doctype)

-	return frappe.get_list(doctype, fields=fields, filters=filters, order_by=order_by,
-		limit_start=limit_start, limit_page_length=limit_page_length, ignore_permissions=False)
+	args = frappe._dict(
+		doctype=doctype,
+		fields=fields,
+		filters=filters,
+		order_by=order_by,
+		limit_start=limit_start,
+		limit_page_length=limit_page_length,
+		debug=debug,
+		as_list=not as_dict
+	)
+
+	validate_args(args)
+	return frappe.get_list(**args)

@frappe.whitelist()
 def get_count(doctype, filters=None, debug=False, cache=False):
@ -91,14 +104,15 @@ def get_value(doctype, fieldname, filters=None, as_dict=True, debug=False, paren
 	if frappe.get_meta(doctype).issingle:
 		value = frappe.db.get_values_from_single(fields, filters, doctype, as_dict=as_dict, debug=debug)
 	else:
-		value = frappe.get_list(doctype, filters=filters, fields=fields, debug=debug, limit=1)
+		value = get_list(doctype, filters=filters, fields=fields, debug=debug, limit_page_length=1, parent=parent, as_dict=as_dict)

 	if as_dict:
-		value = value[0] if value else {}
-	else:
-		value = value[0].fieldname
+		return value[0] if value else {}

-	return value
+	if not value:
+		return
+
+	return value[0] if len(fields) > 1 else value[0][0]

@frappe.whitelist()
 def get_single_value(doctype, field):
@ -378,18 +392,6 @@ def attach_file(filename=None, filedata=None, doctype=None, docname=None, folder
 def get_hooks(hook, app_name=None):
 	return frappe.get_hooks(hook, app_name)

-def check_parent_permission(parent, child_doctype):
-	if parent:
-		# User may pass fake parent and get the information from the child table
-		if child_doctype and not frappe.db.exists('DocField',
-			{'parent': parent, 'options': child_doctype}):
-			raise frappe.PermissionError
-
-		if frappe.permissions.has_permission(parent):
-			return
-	# Either parent not passed or the user doesn't have permission on parent doctype of child table!
-	raise frappe.PermissionError
-
@frappe.whitelist()
 def is_document_amended(doctype, docname):
 	if frappe.permissions.has_permission(doctype):
@ -400,4 +402,4 @@ def is_document_amended(doctype, docname):
 		except frappe.db.InternalError:
 			pass

-	return False
+	return False
--- a/frappe/commands/utils.py
+++ b/frappe/commands/utils.py
@ -293,7 +293,7 @@ def import_doc(context, path, force=False):
 		try:
 			frappe.init(site=site)
 			frappe.connect()
-			import_doc(path, overwrite=context.force)
+			import_doc(path)
 		finally:
 			frappe.destroy()
 	if not context.sites:
@ -483,7 +483,6 @@ def console(context):
@click.option('--doctype', help="For DocType")
@click.option('--doctype-list-path', help="Path to .txt file for list of doctypes. Example erpnext/tests/server/agriculture.txt")
@click.option('--test', multiple=True, help="Specific test")
-@click.option('--driver', help="For Travis")
@click.option('--ui-tests', is_flag=True, default=False, help="Run UI Tests")
@click.option('--module', help="Run tests in a module")
@click.option('--profile', is_flag=True, default=False)
@ -493,9 +492,9 @@ def console(context):
@click.option('--junit-xml-output', help="Destination file path for junit xml report")
@click.option('--failfast', is_flag=True, default=False)
@pass_context
-def run_tests(context, app=None, module=None, doctype=None, test=(),
-	driver=None, profile=False, coverage=False, junit_xml_output=False, ui_tests = False,
-	doctype_list_path=None, skip_test_records=False, skip_before_tests=False, failfast=False):
+def run_tests(context, app=None, module=None, doctype=None, test=(), profile=False,
+		coverage=False, junit_xml_output=False, ui_tests = False, doctype_list_path=None,
+		skip_test_records=False, skip_before_tests=False, failfast=False):

 	"Run tests"
 	import frappe.test_runner
@ -535,8 +534,8 @@ def run_tests(context, app=None, module=None, doctype=None, test=(),
 		cov.start()

 	ret = frappe.test_runner.main(app, module, doctype, context.verbose, tests=tests,
-								force=context.force, profile=profile, junit_xml_output=junit_xml_output,
-								ui_tests=ui_tests, doctype_list_path=doctype_list_path, failfast=failfast)
+		force=context.force, profile=profile, junit_xml_output=junit_xml_output,
+		ui_tests=ui_tests, doctype_list_path=doctype_list_path, failfast=failfast)

 	if coverage:
 		cov.stop()
--- a/frappe/config/init.py
+++ b/frappe/config/init.py
@ -36,48 +36,10 @@ def get_modules_from_all_apps():
 	return modules_list

 def get_modules_from_app(app):
-	try:
-		modules = frappe.get_attr(app + '.config.desktop.get_data')() or {}
-	except ImportError:
-		return []
-
-	active_domains = frappe.get_active_domains()
-
-	if isinstance(modules, dict):
-		active_modules_list = []
-		for m, module in iteritems(modules):
-			module['module_name'] = m
-			module['app'] = app
-			active_modules_list.append(module)
-	else:
-		for m in modules:
-			if m.get("type") == "module" and "category" not in m:
-				m["category"] = "Modules"
-
-		# Only newly formatted modules that have a category to be shown on desk
-		modules = [m for m in modules if m.get("category")]
-		active_modules_list = []
-
-		for m in modules:
-			to_add = True
-			module_name = m.get("module_name")
-
-			# Check Domain
-			if is_domain(m) and module_name not in active_domains:
-				to_add = False
-
-			# Check if config
-			if is_module(m) and not config_exists(app, frappe.scrub(module_name)):
-				to_add = False
-
-			if "condition" in m and not m["condition"]:
-				to_add = False
-
-			if to_add:
-				m["app"] = app
-				active_modules_list.append(m)
-
-	return active_modules_list
+	return frappe.get_all('Module Def',
+		filters={'app_name': app},
+		fields=['module_name', 'app_name as app']
+	)

 def get_all_empty_tables_by_module():
 	empty_tables = set(r[0] for r in frappe.db.multisql({
--- a/frappe/core/doctype/comment/comment.py
+++ b/frappe/core/doctype/comment/comment.py
@ -159,7 +159,7 @@ def update_comments_in_parent(reference_doctype, reference_name, _comments):
 	"""Updates `_comments` property in parent Document with given dict.

 	:param _comments: Dict of comments."""
-	if not reference_doctype or not reference_name or frappe.db.get_value("DocType", reference_doctype, "issingle"):
+	if not reference_doctype or not reference_name or frappe.db.get_value("DocType", reference_doctype, "issingle") or frappe.db.get_value("DocType", reference_doctype, "is_virtual"):
 		return

 	try:
--- a/frappe/core/doctype/communication/communication.json
+++ b/frappe/core/doctype/communication/communication.json
@ -152,7 +152,7 @@
   "fieldname": "communication_type",
   "fieldtype": "Select",
   "label": "Communication Type",
-   "options": "Communication\nComment\nChat\nBot\nNotification\nFeedback",
+   "options": "Communication\nComment\nChat\nBot\nNotification\nFeedback\nAutomated Message",
   "read_only": 1,
   "reqd": 1
  },
@ -387,7 +387,7 @@
 "icon": "fa fa-comment",
 "idx": 1,
 "links": [],
- "modified": "2019-12-27 14:44:04.880373",
+ "modified": "2021-03-25 09:44:28.963538",
 "modified_by": "Administrator",
 "module": "Core",
 "name": "Communication",
@ -426,13 +426,13 @@
   "write": 1
  },
  {
-    "create": 1,
-    "delete": 1,
-    "email": 1,
-    "export":1,
-    "print":1,
-    "read": 1,
-    "role": "Inbox User"
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "export": 1,
+   "print": 1,
+   "read": 1,
+   "role": "Inbox User"
  },
  {
   "delete": 1,
@ -450,4 +450,4 @@
 "title_field": "subject",
 "track_changes": 1,
 "track_seen": 1
-}
+}
--- a/frappe/core/doctype/communication/communication_list.js
+++ b/frappe/core/doctype/communication/communication_list.js
@ -20,6 +20,6 @@ frappe.listview_settings['Communication'] = {
 	},

 	primary_action: function() {
-		new frappe.views.CommunicationComposer({ doc: {} });
+		new frappe.views.CommunicationComposer();
 	}
 };
--- a/frappe/core/doctype/communication/email.py
+++ b/frappe/core/doctype/communication/email.py
@ -8,8 +8,8 @@ import frappe
 import json
 from email.utils import formataddr
 from frappe.core.utils import get_parent_doc
-from frappe.utils import (get_url, get_formatted_email, cint,
-  validate_email_address, split_emails, parse_addr, get_datetime)
+from frappe.utils import (get_url, get_formatted_email, cint, list_to_str,
+	validate_email_address, split_emails, parse_addr, get_datetime)
 from frappe.email.email_body import get_message_id
 import frappe.email.smtp
 import time
@ -20,7 +20,8 @@ from frappe.utils.background_jobs import enqueue
 def make(doctype=None, name=None, content=None, subject=None, sent_or_received = "Sent",
 	sender=None, sender_full_name=None, recipients=None, communication_medium="Email", send_email=False,
 	print_html=None, print_format=None, attachments='[]', send_me_a_copy=False, cc=None, bcc=None,
-	flags=None, read_receipt=None, print_letterhead=True, email_template=None):
+	flags=None, read_receipt=None, print_letterhead=True, email_template=None, communication_type=None,
+	ignore_permissions=False):
 	"""Make a new communication.

 	:param doctype: Reference DocType.
@ -42,15 +43,17 @@ def make(doctype=None, name=None, content=None, subject=None, sent_or_received =
 	is_error_report = (doctype=="User" and name==frappe.session.user and subject=="Error Report")
 	send_me_a_copy = cint(send_me_a_copy)

-	if doctype and name and not is_error_report and not frappe.has_permission(doctype, "email", name) and not (flags or {}).get('ignore_doctype_permissions'):
-		raise frappe.PermissionError("You are not allowed to send emails related to: {doctype} {name}".format(
-			doctype=doctype, name=name))
+	if not ignore_permissions:
+		if doctype and name and not is_error_report and not frappe.has_permission(doctype, "email", name) and not (flags or {}).get('ignore_doctype_permissions'):
+			raise frappe.PermissionError("You are not allowed to send emails related to: {doctype} {name}".format(
+				doctype=doctype, name=name))

 	if not sender:
 		sender = get_formatted_email(frappe.session.user)

-	if isinstance(recipients, list):
-		recipients = ', '.join(recipients)
+	recipients = list_to_str(recipients) if isinstance(recipients, list) else recipients
+	cc = list_to_str(cc) if isinstance(cc, list) else cc
+	bcc = list_to_str(bcc) if isinstance(bcc, list) else bcc

 	comm = frappe.get_doc({
 		"doctype":"Communication",
@ -68,7 +71,8 @@ def make(doctype=None, name=None, content=None, subject=None, sent_or_received =
 		"email_template": email_template,
 		"message_id":get_message_id().strip(" <>"),
 		"read_receipt":read_receipt,
-		"has_attachment": 1 if attachments else 0
+		"has_attachment": 1 if attachments else 0,
+		"communication_type": communication_type
 	}).insert(ignore_permissions=True)

 	comm.save(ignore_permissions=True)
--- a/frappe/core/doctype/data_import/data_import.js
+++ b/frappe/core/doctype/data_import/data_import.js
@ -479,43 +479,4 @@ frappe.ui.form.on('Data Import', {
 			</table>
 		`);
 	},
-
-	show_missing_link_values(frm, missing_link_values) {
-		let can_be_created_automatically = missing_link_values.every(
-			d => d.has_one_mandatory_field
-		);
-
-		let html = missing_link_values
-			.map(d => {
-				let doctype = d.doctype;
-				let values = d.missing_values;
-				return `
-					<h5>${doctype}</h5>
-					<ul>${values.map(v => `<li>${v}</li>`).join('')}</ul>
-				`;
-			})
-			.join('');
-
-		if (can_be_created_automatically) {
-			// prettier-ignore
-			let message = __('There are some linked records which needs to be created before we can import your file. Do you want to create the following missing records automatically?');
-			frappe.confirm(message + html, () => {
-				frm
-					.call('create_missing_link_values', {
-						missing_link_values
-					})
-					.then(r => {
-						let records = r.message;
-						frappe.msgprint(
-							__('Created {0} records successfully.', [records.length])
-						);
-					});
-			});
-		} else {
-			frappe.msgprint(
-				// prettier-ignore
-				__('The following records needs to be created before we can import your file.') + html
-			);
-		}
-	}
 });
--- a/frappe/core/doctype/data_import/data_import.json
+++ b/frappe/core/doctype/data_import/data_import.json
@ -53,7 +53,8 @@
   "fieldname": "import_file",
   "fieldtype": "Attach",
   "in_list_view": 1,
-   "label": "Import File"
+   "label": "Import File",
+   "read_only_depends_on": "eval: ['Success', 'Partial Success'].includes(doc.status)"
  },
  {
   "fieldname": "import_preview",
@ -156,10 +157,11 @@
   "description": "Must be a publicly accessible Google Sheets URL",
   "fieldname": "google_sheets_url",
   "fieldtype": "Data",
-   "label": "Import from Google Sheets"
+   "label": "Import from Google Sheets",
+   "read_only_depends_on": "eval: ['Success', 'Partial Success'].includes(doc.status)"
  },
  {
-   "depends_on": "eval:doc.google_sheets_url && !doc.__unsaved",
+   "depends_on": "eval:doc.google_sheets_url && !doc.__unsaved &&  ['Success', 'Partial Success'].includes(doc.status)",
   "fieldname": "refresh_google_sheet",
   "fieldtype": "Button",
   "label": "Refresh Google Sheet"
@ -167,7 +169,7 @@
 ],
 "hide_toolbar": 1,
 "links": [],
- "modified": "2020-06-24 14:33:03.173876",
+ "modified": "2021-04-11 01:50:42.074623",
 "modified_by": "Administrator",
 "module": "Core",
 "name": "Data Import",
--- a/frappe/core/doctype/data_import/data_import.py
+++ b/frappe/core/doctype/data_import/data_import.py
@ -2,16 +2,16 @@
 # Copyright (c) 2019, Frappe Technologies and contributors
 # For license information, please see license.txt

-from __future__ import unicode_literals
 import os
-import frappe
-from frappe.model.document import Document

-from frappe.core.doctype.data_import.importer import Importer
+import frappe
+from frappe import _
 from frappe.core.doctype.data_import.exporter import Exporter
+from frappe.core.doctype.data_import.importer import Importer
+from frappe.model.document import Document
+from frappe.modules.import_file import import_file_by_path
 from frappe.utils.background_jobs import enqueue
 from frappe.utils.csvutils import validate_google_sheets_url
-from frappe import _


 class DataImport(Document):
@ -38,6 +38,7 @@ class DataImport(Document):
 			return
 		validate_google_sheets_url(self.google_sheets_url)

+	@frappe.whitelist()
 	def get_preview_from_template(self, import_file=None, google_sheets_url=None):
 		if import_file:
 			self.import_file = import_file
@ -173,15 +174,7 @@ def import_file(
 ##############


-def import_doc(
-	path,
-	overwrite=False,
-	ignore_links=False,
-	ignore_insert=False,
-	insert=False,
-	submit=False,
-	pre_process=None,
-):
+def import_doc(path, pre_process=None):
 	if os.path.isdir(path):
 		files = [os.path.join(path, f) for f in os.listdir(path)]
 	else:
@ -190,30 +183,21 @@ def import_doc(
 	for f in files:
 		if f.endswith(".json"):
 			frappe.flags.mute_emails = True
-			frappe.modules.import_file.import_file_by_path(
-				f, data_import=True, force=True, pre_process=pre_process, reset_permissions=True
+			import_file_by_path(
+				f,
+				data_import=True,
+				force=True,
+				pre_process=pre_process,
+				reset_permissions=True
 			)
 			frappe.flags.mute_emails = False
 			frappe.db.commit()
 		elif f.endswith(".csv"):
-			import_file_by_path(
-				f,
-				ignore_links=ignore_links,
-				overwrite=overwrite,
-				submit=submit,
-				pre_process=pre_process,
-			)
+			validate_csv_import_file(f)
 			frappe.db.commit()


-def import_file_by_path(
-	path,
-	ignore_links=False,
-	overwrite=False,
-	submit=False,
-	pre_process=None,
-	no_email=True,
-):
+def validate_csv_import_file(path):
 	if path.endswith(".csv"):
 		print()
 		print("This method is deprecated.")
--- a/frappe/core/doctype/data_import/exporter.py
+++ b/frappe/core/doctype/data_import/exporter.py
@ -2,13 +2,15 @@
 # Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and Contributors
 # MIT License. See license.txt

+import typing
+
 import frappe
 from frappe.model import (
 	display_fieldtypes,
 	no_value_fields,
 	table_fields as table_fieldtypes,
 )
-from frappe.utils import flt, format_duration
+from frappe.utils import flt, format_duration, groupby_metric
 from frappe.utils.csvutils import build_csv_response
 from frappe.utils.xlsxutils import build_xlsx_response

@ -116,7 +118,6 @@ class Exporter:

 	def get_data_to_export(self):
 		frappe.permissions.can_export(self.doctype, raise_exception=True)
-		data_to_export = []

 		table_fields = [f for f in self.exportable_fields if f != self.doctype]
 		data = self.get_data_as_docs()
@ -128,14 +129,13 @@ class Exporter:
 			if table_fields:
 				# add child table data
 				for f in table_fields:
-					for i, child_row in enumerate(doc[f]):
+					for i, child_row in enumerate(doc.get(f, [])):
 						table_df = self.meta.get_field(f)
 						child_doctype = table_df.options
 						rows = self.add_data_row(child_doctype, child_row.parentfield, child_row, rows, i)

-			data_to_export += rows
-
-		return data_to_export
+			for row in rows:
+				yield row

 	def add_data_row(self, doctype, parentfield, doc, rows, row_idx):
 		if len(rows) < row_idx + 1:
@ -204,17 +204,13 @@ class Exporter:
 			)
 			child_data[key] = data

-		return self.merge_data(parent_data, child_data)
-
-	def merge_data(self, parent_data, child_data):
+		# Group children data by parent name
+		grouped_children_data = self.group_children_data_by_parent(child_data)
 		for doc in parent_data:
-			for table_field, table_rows in child_data.items():
-				doc[table_field] = [row for row in table_rows if row.parent == doc.name]
-
-		return parent_data
+			related_children_docs = grouped_children_data.get(doc.name, {})
+			yield {**doc, **related_children_docs}

 	def add_header(self):
-
 		header = []
 		for df in self.fields:
 			is_parent = not df.is_child_table_field
@ -261,3 +257,6 @@ class Exporter:

 	def build_xlsx_response(self):
 		build_xlsx_response(self.get_csv_array_for_export(), self.doctype)
+
+	def group_children_data_by_parent(self, children_data: typing.Dict[str, list]):
+		return groupby_metric(children_data, key='parent')
--- a/frappe/core/doctype/data_import/importer.py
+++ b/frappe/core/doctype/data_import/importer.py
@ -449,8 +449,8 @@ class ImportFile:
 			data_without_first_row = data[1:]
 			for row in data_without_first_row:
 				row_values = row.get_values(parent_column_indexes)
-				# if the row is blank or same content as the previous parent row, it's a child row doc
-				if all([v in INVALID_VALUES for v in row_values]) or row_values == parent_row_values:
+				# if the row is blank, it's a child row doc
+				if all([v in INVALID_VALUES for v in row_values]):
 					rows.append(row)
 					continue
 				# if we encounter a row which has values in parent columns,
--- a/frappe/core/doctype/data_import_legacy/data_import_legacy.py
+++ b/frappe/core/doctype/data_import_legacy/data_import_legacy.py
@ -2,20 +2,22 @@
 # Copyright (c) 2017, Frappe Technologies and contributors
 # For license information, please see license.txt

-from __future__ import unicode_literals
-import frappe, os
-from frappe import _
+import os
+
+import frappe
 import frappe.modules.import_file
-from frappe.model.document import Document
-from frappe.utils.data import format_datetime
+from frappe import _
 from frappe.core.doctype.data_import_legacy.importer import upload
+from frappe.model.document import Document
+from frappe.modules.import_file import import_file_by_path as _import_file_by_path
 from frappe.utils.background_jobs import enqueue
+from frappe.utils.data import format_datetime


 class DataImportLegacy(Document):
 	def autoname(self):
 		if not self.name:
-			self.name = "Import on " +format_datetime(self.creation)
+			self.name = "Import on " + format_datetime(self.creation)

 	def validate(self):
 		if not self.import_file:
@ -33,6 +35,7 @@ class DataImportLegacy(Document):
 def get_importable_doctypes():
 	return frappe.cache().hget("can_import", frappe.session.user)

+
@frappe.whitelist()
 def import_data(data_import):
 	frappe.db.set_value("Data Import Legacy", data_import, "import_status", "In Progress", update_modified=False)
@ -57,7 +60,7 @@ def import_doc(path, overwrite=False, ignore_links=False, ignore_insert=False,
 	for f in files:
 		if f.endswith(".json"):
 			frappe.flags.mute_emails = True
-			frappe.modules.import_file.import_file_by_path(f, data_import=True, force=True, pre_process=pre_process, reset_permissions=True)
+			_import_file_by_path(f, data_import=True, force=True, pre_process=pre_process, reset_permissions=True)
 			frappe.flags.mute_emails = False
 			frappe.db.commit()
 		elif f.endswith(".csv"):
@ -69,7 +72,7 @@ def import_file_by_path(path, ignore_links=False, overwrite=False, submit=False,
 	from frappe.utils.csvutils import read_csv_content
 	print("Importing " + path)
 	with open(path, "r") as infile:
-		upload(rows = read_csv_content(infile.read()), ignore_links=ignore_links, no_email=no_email, overwrite=overwrite,
+		upload(rows=read_csv_content(infile.read()), ignore_links=ignore_links, no_email=no_email, overwrite=overwrite,
 			submit_after_import=submit, pre_process=pre_process)


--- a/frappe/core/doctype/docshare/docshare.json
+++ b/frappe/core/doctype/docshare/docshare.json
@ -1,293 +1,110 @@
 {
- "allow_copy": 0,
+ "actions": [],
 "allow_import": 1,
- "allow_rename": 0,
 "autoname": "hash",
- "beta": 0,
 "creation": "2015-02-04 04:33:36.330477",
- "custom": 0,
 "description": "Internal record of document shares",
- "docstatus": 0,
 "doctype": "DocType",
 "document_type": "System",
- "editable_grid": 0,
+ "engine": "InnoDB",
+ "field_order": [
+  "user",
+  "share_doctype",
+  "share_name",
+  "read",
+  "write",
+  "share",
+  "submit",
+  "everyone",
+  "notify_by_email"
+ ],
 "fields": [
  {
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
   "fieldname": "user",
   "fieldtype": "Link",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
   "in_list_view": 1,
-   "in_standard_filter": 0,
   "label": "User",
-   "length": 0,
-   "no_copy": 0,
   "options": "User",
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 1,
-   "set_only_once": 0,
-   "unique": 0
+   "search_index": 1
  },
  {
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
   "fieldname": "share_doctype",
   "fieldtype": "Link",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
   "in_list_view": 1,
-   "in_standard_filter": 0,
   "label": "Document Type",
-   "length": 0,
-   "no_copy": 0,
   "options": "DocType",
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
   "reqd": 1,
-   "search_index": 1,
-   "set_only_once": 0,
-   "unique": 0
+   "search_index": 1
  },
  {
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
   "fieldname": "share_name",
   "fieldtype": "Dynamic Link",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
   "in_list_view": 1,
-   "in_standard_filter": 0,
   "label": "Document Name",
-   "length": 0,
-   "no_copy": 0,
   "options": "share_doctype",
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
   "reqd": 1,
-   "search_index": 1,
-   "set_only_once": 0,
-   "unique": 0
+   "search_index": 1
  },
  {
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
   "default": "0",
   "fieldname": "read",
   "fieldtype": "Check",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
-   "label": "Read",
-   "length": 0,
-   "no_copy": 0,
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "unique": 0
+   "label": "Read"
  },
  {
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
   "default": "0",
   "fieldname": "write",
   "fieldtype": "Check",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
-   "label": "Write",
-   "length": 0,
-   "no_copy": 0,
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "unique": 0
+   "label": "Write"
  },
  {
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
   "default": "0",
   "fieldname": "share",
   "fieldtype": "Check",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
-   "label": "Share",
-   "length": 0,
-   "no_copy": 0,
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "unique": 0
+   "label": "Share"
  },
  {
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
+   "default": "0",
   "fieldname": "everyone",
   "fieldtype": "Check",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
-   "label": "Everyone",
-   "length": 0,
-   "no_copy": 0,
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "unique": 0
+   "label": "Everyone"
  },
  {
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
   "default": "1",
   "fieldname": "notify_by_email",
   "fieldtype": "Check",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
   "label": "Notify by email",
-   "length": 0,
-   "no_copy": 0,
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 1,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "unique": 0
+   "print_hide": 1
+  },
+  {
+   "default": "0",
+   "fieldname": "submit",
+   "fieldtype": "Check",
+   "label": "Submit"
  }
 ],
- "has_web_view": 0,
- "hide_heading": 0,
- "hide_toolbar": 0,
- "idx": 0,
- "image_view": 0,
 "in_create": 1,
- "is_submittable": 0,
- "issingle": 0,
- "istable": 0,
- "max_attachments": 0,
- "modified": "2017-09-15 15:58:34.126438",
+ "links": [],
+ "modified": "2021-04-04 11:38:50.813312",
 "modified_by": "Administrator",
 "module": "Core",
 "name": "DocShare",
- "name_case": "",
 "owner": "Administrator",
 "permissions": [
  {
-   "amend": 0,
-   "apply_user_permissions": 0,
-   "cancel": 0,
   "create": 1,
   "delete": 1,
-   "email": 0,
   "export": 1,
-   "if_owner": 0,
   "import": 1,
-   "permlevel": 0,
-   "print": 0,
   "read": 1,
   "report": 1,
   "role": "System Manager",
-   "set_user_permissions": 0,
   "share": 1,
-   "submit": 0,
   "write": 1
  }
 ],
- "quick_entry": 0,
 "read_only": 1,
- "read_only_onload": 0,
- "show_name_in_global_search": 0,
 "sort_field": "modified",
 "sort_order": "DESC",
- "track_changes": 1,
- "track_seen": 0
-}
+ "track_changes": 1
+}
--- a/frappe/core/doctype/docshare/docshare.py
+++ b/frappe/core/doctype/docshare/docshare.py
@ -5,7 +5,7 @@ from __future__ import unicode_literals
 import frappe
 from frappe.model.document import Document
 from frappe import _
-from frappe.utils import get_fullname
+from frappe.utils import get_fullname, cint

 exclude_from_linked_with = True

@ -15,12 +15,15 @@ class DocShare(Document):
 	def validate(self):
 		self.validate_user()
 		self.check_share_permission()
+		self.check_is_submittable()
 		self.cascade_permissions_downwards()
 		self.get_doc().run_method("validate_share", self)

 	def cascade_permissions_downwards(self):
-		if self.share or self.write:
+		if self.share or self.write or self.submit:
 			self.read = 1
+		if self.submit:
+			self.write = 1

 	def get_doc(self):
 		if not getattr(self, "_doc", None):
@ -39,6 +42,11 @@ class DocShare(Document):

 			frappe.throw(_('You need to have "Share" permission'), frappe.PermissionError)

+	def check_is_submittable(self):
+		if self.submit and not cint(frappe.db.get_value("DocType", self.share_doctype, "is_submittable")):
+			frappe.throw(_("Cannot share {0} with submit permission as the doctype {1} is not submittable").format(
+				frappe.bold(self.share_name), frappe.bold(self.share_doctype)))
+
 	def after_insert(self):
 		doc = self.get_doc()
 		owner = get_fullname(self.owner)
--- a/frappe/core/doctype/docshare/test_docshare.py
+++ b/frappe/core/doctype/docshare/test_docshare.py
@ -5,6 +5,7 @@ from __future__ import unicode_literals
 import frappe
 import frappe.share
 import unittest
+from frappe.automation.doctype.auto_repeat.test_auto_repeat import create_submittable_doctype

 class TestDocShare(unittest.TestCase):
 	def setUp(self):
@ -91,3 +92,24 @@ class TestDocShare(unittest.TestCase):
 		self.assertTrue(self.event.name not in frappe.share.get_shared("Event", self.user))
 		self.assertTrue(self.event.name not in frappe.share.get_shared("Event", "test1@example.com"))
 		self.assertTrue(self.event.name not in frappe.share.get_shared("Event", "Guest"))
+
+	def test_share_with_submit_perm(self):
+		doctype = "Test DocShare with Submit"
+		create_submittable_doctype(doctype, submit_perms=0)
+
+		submittable_doc = frappe.get_doc(dict(doctype=doctype, test="test docshare with submit")).insert()
+
+		frappe.set_user(self.user)
+		self.assertFalse(frappe.has_permission(doctype, "submit", user=self.user))
+
+		frappe.set_user("Administrator")
+		frappe.share.add(doctype, submittable_doc.name, self.user, submit=1)
+
+		frappe.set_user(self.user)
+		self.assertTrue(frappe.has_permission(doctype, "submit", doc=submittable_doc.name, user=self.user))
+
+		# test cascade
+		self.assertTrue(frappe.has_permission(doctype, "read", doc=submittable_doc.name, user=self.user))
+		self.assertTrue(frappe.has_permission(doctype, "write", doc=submittable_doc.name, user=self.user))
+
+		frappe.share.remove(doctype, submittable_doc.name, self.user)
--- a/frappe/core/doctype/doctype/boilerplate/controller._py
+++ b/frappe/core/doctype/doctype/boilerplate/controller._py
@ -7,4 +7,4 @@ from __future__ import unicode_literals
 {base_class_import}

 class {classname}({base_class}):
-	pass
+	{custom_controller}
--- a/frappe/core/doctype/doctype/doctype.js
+++ b/frappe/core/doctype/doctype/doctype.js
@ -13,11 +13,21 @@

 frappe.ui.form.on('DocType', {
 	refresh: function(frm) {
+		frm.set_query('role', 'permissions', function(doc) {
+			if (doc.custom && frappe.session.user != 'Administrator') {
+				return {
+					query: "frappe.core.doctype.role.role.role_query",
+					filters: [['Role', 'name', '!=', 'All']]
+				};
+			}
+		});
+
 		if(frappe.session.user !== "Administrator" || !frappe.boot.developer_mode) {
 			if(frm.is_new()) {
 				frm.set_value("custom", 1);
 			}
 			frm.toggle_enable("custom", 0);
+			frm.toggle_enable("is_virtual", 0);
 			frm.toggle_enable("beta", 0);
 		}

--- a/frappe/core/doctype/doctype/doctype.json
+++ b/frappe/core/doctype/doctype/doctype.json
@ -22,6 +22,7 @@
  "track_views",
  "custom",
  "beta",
+  "is_virtual",
  "fields_section_break",
  "fields",
  "sb1",
@ -55,6 +56,8 @@
  "show_preview_popup",
  "show_name_in_global_search",
  "email_settings_sb",
+  "default_email_template",
+  "column_break_51",
  "email_append_to",
  "sender_field",
  "subject_field",
@ -528,6 +531,22 @@
   "fieldname": "index_web_pages_for_search",
   "fieldtype": "Check",
   "label": "Index Web Pages for Search"
+  },
+  {
+   "default": "0",
+   "fieldname": "is_virtual",
+   "fieldtype": "Check",
+   "label": "Is Virtual"
+  },
+  {
+   "fieldname": "default_email_template",
+   "fieldtype": "Link",
+   "label": "Default Email Template",
+   "options": "Email Template"
+  },
+  {
+   "fieldname": "column_break_51",
+   "fieldtype": "Column Break"
  }
 ],
 "icon": "fa fa-bolt",
@ -609,7 +628,7 @@
   "link_fieldname": "reference_doctype"
  }
 ],
- "modified": "2021-02-04 15:10:09.227205",
+ "modified": "2021-04-16 12:26:41.031135",
 "modified_by": "Administrator",
 "module": "Core",
 "name": "DocType",
--- a/frappe/core/doctype/doctype/doctype.py
+++ b/frappe/core/doctype/doctype/doctype.py
@ -67,7 +67,7 @@ class DocType(Document):
 		self.scrub_field_names()
 		self.set_default_in_list_view()
 		self.set_default_translatable()
-		self.validate_series()
+		validate_series(self)
 		self.validate_document_type()
 		validate_fields(self)

@ -127,6 +127,10 @@ class DocType(Document):
 		if not frappe.conf.get("developer_mode") and not self.custom:
 			frappe.throw(_("Not in Developer Mode! Set in site_config.json or make 'Custom' DocType."), CannotCreateStandardDoctypeError)

+		if self.is_virtual and self.custom:
+			frappe.throw(_("Not allowed to create custom Virtual DocType."), CannotCreateStandardDoctypeError)
+
+
 		if frappe.conf.get('developer_mode'):
 			self.owner = 'Administrator'
 			self.modified_by = 'Administrator'
@ -238,44 +242,6 @@ class DocType(Document):
 			# unique is automatically an index
 			if d.unique: d.search_index = 0

-	def validate_series(self, autoname=None, name=None):
-		"""Validate if `autoname` property is correctly set."""
-		if not autoname: autoname = self.autoname
-		if not name: name = self.name
-
-		if not autoname and self.get("fields", {"fieldname":"naming_series"}):
-			self.autoname = "naming_series:"
-		elif self.autoname == "naming_series:" and not self.get("fields", {"fieldname":"naming_series"}):
-			frappe.throw(_("Invalid fieldname '{0}' in autoname").format(self.autoname))
-
-		# validate field name if autoname field:fieldname is used
-		# Create unique index on autoname field automatically.
-		if autoname and autoname.startswith('field:'):
-			field = autoname.split(":")[1]
-			if not field or field not in [ df.fieldname for df in self.fields ]:
-				frappe.throw(_("Invalid fieldname '{0}' in autoname").format(field))
-			else:
-				for df in self.fields:
-					if df.fieldname == field:
-						df.unique = 1
-						break
-
-		if autoname and (not autoname.startswith('field:')) \
-			and (not autoname.startswith('eval:')) \
-			and (not autoname.lower() in ('prompt', 'hash')) \
-			and (not autoname.startswith('naming_series:')) \
-			and (not autoname.startswith('format:')):
-
-			prefix = autoname.split('.')[0]
-			used_in = frappe.db.sql("""
-				SELECT `name`
-				FROM `tabDocType`
-				WHERE `autoname` LIKE CONCAT(%s, '.%%')
-				AND `name`!=%s
-			""", (prefix, name))
-			if used_in:
-				frappe.throw(_("Series {0} already used in {1}").format(prefix, used_in[0][0]))
-
 	def on_update(self):
 		"""Update database schema, make controller templates if `custom` is not set and clear cache."""
 		try:
@ -666,6 +632,46 @@ class DocType(Document):

 		validate_route_conflict(self.doctype, self.name)

+def validate_series(dt, autoname=None, name=None):
+	"""Validate if `autoname` property is correctly set."""
+	if not autoname:
+		autoname = dt.autoname
+	if not name:
+		name = dt.name
+
+	if not autoname and dt.get("fields", {"fieldname":"naming_series"}):
+		dt.autoname = "naming_series:"
+	elif dt.autoname == "naming_series:" and not dt.get("fields", {"fieldname":"naming_series"}):
+		frappe.throw(_("Invalid fieldname '{0}' in autoname").format(dt.autoname))
+
+	# validate field name if autoname field:fieldname is used
+	# Create unique index on autoname field automatically.
+	if autoname and autoname.startswith('field:'):
+		field = autoname.split(":")[1]
+		if not field or field not in [df.fieldname for df in dt.fields]:
+			frappe.throw(_("Invalid fieldname '{0}' in autoname").format(field))
+		else:
+			for df in dt.fields:
+				if df.fieldname == field:
+					df.unique = 1
+					break
+
+	if autoname and (not autoname.startswith('field:')) \
+		and (not autoname.startswith('eval:')) \
+		and (not autoname.lower() in ('prompt', 'hash')) \
+		and (not autoname.startswith('naming_series:')) \
+		and (not autoname.startswith('format:')):
+
+		prefix = autoname.split('.')[0]
+		used_in = frappe.db.sql("""
+			SELECT `name`
+			FROM `tabDocType`
+			WHERE `autoname` LIKE CONCAT(%s, '.%%')
+			AND `name`!=%s
+		""", (prefix, name))
+		if used_in:
+			frappe.throw(_("Series {0} already used in {1}").format(prefix, used_in[0][0]))
+
 def validate_links_table_fieldnames(meta):
 	"""Validate fieldnames in Links table"""
 	if frappe.flags.in_patch: return
@ -1110,6 +1116,21 @@ def validate_permissions(doctype, for_remove=False, alert=False):
 		if d.get("import") and not isimportable:
 			frappe.throw(_("{0}: Cannot set import as {1} is not importable").format(get_txt(d), doctype))

+	def validate_permission_for_all_role(d):
+		if frappe.session.user == 'Administrator':
+			return
+
+		if doctype.custom:
+			if d.role == 'All':
+				frappe.throw(_('Row # {0}: Non administrator user can not set the role {1} to the custom doctype')
+					.format(d.idx, frappe.bold(_('All'))), title=_('Permissions Error'))
+
+			roles = [row.name for row in frappe.get_all('Role', filters={'is_custom': 1})]
+
+			if d.role in roles:
+				frappe.throw(_('Row # {0}: Non administrator user can not set the role {1} to the custom doctype')
+					.format(d.idx, frappe.bold(_(d.role))), title=_('Permissions Error'))
+
 	for d in permissions:
 		if not d.permlevel:
 			d.permlevel=0
@ -1121,6 +1142,7 @@ def validate_permissions(doctype, for_remove=False, alert=False):
 			check_if_importable(d)
 		check_level_zero_is_set(d)
 		remove_rights_for_single(d)
+		validate_permission_for_all_role(d)

 def make_module_and_roles(doc, perm_fieldname="permissions"):
 	"""Make `Module Def` and `Role` records if already not made. Called while installing."""
--- a/frappe/core/doctype/doctype/test_doctype.py
+++ b/frappe/core/doctype/doctype/test_doctype.py
@ -480,8 +480,19 @@ class TestDocType(unittest.TestCase):
 			'link_doctype': "User",
 			'link_fieldname': "a_field_that_does_not_exists"
 		})
+
 		self.assertRaises(InvalidFieldNameError, validate_links_table_fieldnames, doc)

+	def test_create_virtual_doctype(self):
+		"""Test virtual DOcTYpe."""
+		virtual_doc = new_doctype('Test Virtual Doctype')
+		virtual_doc.is_virtual = 1
+		virtual_doc.insert()
+		virtual_doc.save()
+		doc = frappe.get_doc("DocType", "Test Virtual Doctype")
+
+		self.assertEqual(doc.is_virtual, 1)
+		self.assertFalse(frappe.db.table_exists('Test Virtual Doctype'))

 def new_doctype(name, unique=0, depends_on='', fields=None):
 	doc = frappe.get_doc({
--- a/frappe/core/doctype/document_naming_rule/document_naming_rule.js
+++ b/frappe/core/doctype/document_naming_rule/document_naming_rule.js
@ -15,8 +15,9 @@ frappe.ui.form.on('Document Naming Rule', {
 					}).map((d) => {
 						return {label: `${d.label} (${d.fieldname})`, value: d.fieldname};
 					});
-				frappe.meta.get_docfield('Document Naming Rule Condition', 'field', frm.doc.name).options = fieldnames;
-				frm.refresh_field('conditions');
+				frm.fields_dict.conditions.grid.update_docfield_property(
+					'field', 'options', fieldnames
+				);
 			});
 		}
 	}
--- a/frappe/core/doctype/file/file.py
+++ b/frappe/core/doctype/file/file.py
@ -75,7 +75,7 @@ class File(Document):
 			self.add_comment_in_reference_doc('Attachment',
 				_('Added {0}').format("<a href='{file_url}' target='_blank'>{file_name}</a>{icon}".format(**{
 					"icon": ' <i class="fa fa-lock text-warning"></i>' if self.is_private else "",
-					"file_url": quote(self.file_url) if self.file_url else self.file_name,
+					"file_url": quote(frappe.safe_encode(self.file_url)) if self.file_url else self.file_name,
 					"file_name": self.file_name or self.file_url
 				})))

@ -94,52 +94,89 @@ class File(Document):
 			self.set_file_name()
 			self.validate_duplicate_entry()
 			self.validate_attachment_limit()
+
 		self.validate_folder()

-		if not self.file_url and not self.flags.ignore_file_validate:
-			if not self.is_folder:
+		if self.is_folder:
+			self.file_url = ""
+		else:
+			self.validate_url()
+
+		self.file_size = frappe.form_dict.file_size or self.file_size
+
+	def validate_url(self):
+		if not self.file_url or self.file_url.startswith(("http://", "https://")):
+			if not self.flags.ignore_file_validate:
 				self.validate_file()
-			self.generate_content_hash()

-		if frappe.db.exists('File', {'name': self.name, 'is_folder': 0}):
-			old_file_url = self.file_url
-			if not self.is_folder and (self.is_private != self.db_get('is_private')):
-				private_files = frappe.get_site_path('private', 'files')
-				public_files = frappe.get_site_path('public', 'files')
+			return

-				file_name = self.file_url.split('/')[-1]
-				if not self.is_private:
-					shutil.move(os.path.join(private_files, file_name),
-						os.path.join(public_files, file_name))
+		# Probably an invalid web URL
+		if not self.file_url.startswith(("/files/", "/private/files/")):
+			frappe.throw(
+				_("URL must start with http:// or https://"),
+				title=_('Invalid URL')
+			)

-					self.file_url = "/files/{0}".format(file_name)
+		# Ensure correct formatting and type
+		self.file_url = unquote(self.file_url)
+		self.is_private = cint(self.is_private)

-				else:
-					shutil.move(os.path.join(public_files, file_name),
-						os.path.join(private_files, file_name))
+		self.handle_is_private_changed()

-					self.file_url = "/private/files/{0}".format(file_name)
+		base_path = os.path.realpath(get_files_path(is_private=self.is_private))
+		if not os.path.realpath(self.get_full_path()).startswith(base_path):
+			frappe.throw(
+				_("The File URL you've entered is incorrect"),
+				title=_('Invalid File URL')
+			)

-				update_existing_file_docs(self)
+	def handle_is_private_changed(self):
+		if not frappe.db.exists(
+			'File', {
+				'name': self.name,
+				'is_private': cint(not self.is_private)
+			}
+		):
+			return

-			# update documents image url with new file url
-			if self.attached_to_doctype and self.attached_to_name:
-				if not self.attached_to_field:
-					field_name = None
-					reference_dict = frappe.get_doc(self.attached_to_doctype, self.attached_to_name).as_dict()
-					for key, value in reference_dict.items():
-						if value == old_file_url:
-							field_name = key
-							break
-					self.attached_to_field = field_name
-				if self.attached_to_field:
-					frappe.db.set_value(self.attached_to_doctype, self.attached_to_name,
-						self.attached_to_field, self.file_url)
+		old_file_url = self.file_url

-		self.validate_url()
+		file_name = self.file_url.split('/')[-1]
+		private_file_path = frappe.get_site_path('private', 'files', file_name)
+		public_file_path = frappe.get_site_path('public', 'files', file_name)

-		if self.file_url and (self.is_private != self.file_url.startswith('/private')):
-			frappe.throw(_('Invalid file URL. Please contact System Administrator.'))
+		if self.is_private:
+			shutil.move(public_file_path, private_file_path)
+			url_starts_with = "/private/files/"
+		else:
+			shutil.move(private_file_path, public_file_path)
+			url_starts_with = "/files/"
+
+		self.file_url = "{0}{1}".format(url_starts_with, file_name)
+		update_existing_file_docs(self)
+
+		if (
+			not self.attached_to_doctype
+			or not self.attached_to_name
+			or not self.fetch_attached_to_field(old_file_url)
+		):
+			return
+
+		frappe.db.set_value(self.attached_to_doctype, self.attached_to_name,
+			self.attached_to_field, self.file_url)
+
+	def fetch_attached_to_field(self, old_file_url):
+		if self.attached_to_field:
+			return True
+
+		reference_dict = frappe.get_doc(
+			self.attached_to_doctype, self.attached_to_name).as_dict()
+
+		for key, value in reference_dict.items():
+			if value == old_file_url:
+				self.attached_to_field = key
+				return True

 	def validate_attachment_limit(self):
 		attachment_limit = 0
@ -335,8 +372,13 @@ class File(Document):

 	def get_content(self):
 		"""Returns [`file_name`, `content`] for given file name `fname`"""
+		if self.is_folder:
+			frappe.throw(_("Cannot get file contents of a Folder"))
+
 		if self.get('content'):
 			return self.content
+
+		self.validate_url()
 		file_path = self.get_full_path()

 		# read the file
@ -423,23 +465,6 @@ class File(Document):
 		else:
 			raise Exception

-
-	def validate_url(self, df=None):
-		if self.file_url:
-			if not self.file_url.startswith(("http://", "https://", "/files/", "/private/files/")):
-				frappe.throw(_("URL must start with 'http://' or 'https://'"))
-				return
-
-			if not self.file_url.startswith(("http://", "https://")):
-				# local file
-				root_files_path = get_files_path(is_private=self.is_private)
-				if not os.path.commonpath([root_files_path]) == os.path.commonpath([root_files_path, self.get_full_path()]):
-					# basically the file url is skewed to not point to /files/ or /private/files
-					frappe.throw(_("{0} is not a valid file url").format(self.file_url))
-			self.file_url = unquote(self.file_url)
-			self.file_size = frappe.form_dict.file_size or self.file_size
-
-
 	def get_uploaded_content(self):
 		# should not be unicode when reading a file, hence using frappe.form
 		if 'filedata' in frappe.form_dict:
@ -718,7 +743,7 @@ def delete_file(path):
 			os.remove(path)


-def remove_file(fid=None, attached_to_doctype=None, attached_to_name=None, from_delete=False):
+def remove_file(fid=None, attached_to_doctype=None, attached_to_name=None, from_delete=False, delete_permanently=False):
 	"""Remove file and File entry"""
 	file_name = None
 	if not (attached_to_doctype and attached_to_name):
@ -736,7 +761,7 @@ def remove_file(fid=None, attached_to_doctype=None, attached_to_name=None, from_
 		if not file_name:
 			file_name = frappe.db.get_value("File", fid, "file_name")
 		comment = doc.add_comment("Attachment Removed", _("Removed {0}").format(file_name))
-		frappe.delete_doc("File", fid, ignore_permissions=ignore_permissions)
+		frappe.delete_doc("File", fid, ignore_permissions=ignore_permissions, delete_permanently=delete_permanently)

 	return comment

@ -745,17 +770,18 @@ def get_max_file_size():
 	return cint(conf.get('max_file_size')) or 10485760


-def remove_all(dt, dn, from_delete=False):
+def remove_all(dt, dn, from_delete=False, delete_permanently=False):
 	"""remove all files in a transaction"""
 	try:
 		for fid in frappe.db.sql_list("""select name from `tabFile` where
 			attached_to_doctype=%s and attached_to_name=%s""", (dt, dn)):
 			if from_delete:
 				# If deleting a doc, directly delete files
-				frappe.delete_doc("File", fid, ignore_permissions=True)
+				frappe.delete_doc("File", fid, ignore_permissions=True, delete_permanently=delete_permanently)
 			else:
 				# Removes file and adds a comment in the document it is attached to
-				remove_file(fid=fid, attached_to_doctype=dt, attached_to_name=dn, from_delete=from_delete)
+				remove_file(fid=fid, attached_to_doctype=dt, attached_to_name=dn,
+					from_delete=from_delete, delete_permanently=delete_permanently)
 	except Exception as e:
 		if e.args[0]!=1054: raise # (temp till for patched)

@ -944,12 +970,22 @@ def get_files_in_folder(folder, start=0, page_length=20):
 	start = cint(start)
 	page_length = cint(page_length)

-	files = frappe.db.get_all('File',
+	attachment_folder = frappe.db.get_value('File',
+		'Home/Attachments',
+		['name', 'file_name', 'file_url', 'is_folder', 'modified'],
+		as_dict=1
+	)
+
+	files = frappe.db.get_list('File',
 		{ 'folder': folder },
 		['name', 'file_name', 'file_url', 'is_folder', 'modified'],
 		start=start,
 		page_length=page_length + 1
 	)
+
+	if folder == 'Home' and attachment_folder not in files:
+		files.insert(0, attachment_folder)
+
 	return {
 		'files': files[:page_length],
 		'has_more': len(files) > page_length
--- a/frappe/core/doctype/file/test_file.py
+++ b/frappe/core/doctype/file/test_file.py
@ -8,7 +8,7 @@ import frappe
 import os
 import unittest
 from frappe import _
-from frappe.core.doctype.file.file import move_file
+from frappe.core.doctype.file.file import move_file, get_files_in_folder
 from frappe.utils import get_files_path
 # test_records = frappe.get_test_records('File')

@ -192,13 +192,10 @@ class TestSameContent(unittest.TestCase):


 class TestFile(unittest.TestCase):
-
-
 	def setUp(self):
 		self.delete_test_data()
 		self.upload_file()

-
 	def tearDown(self):
 		try:
 			frappe.get_doc("File", {"file_name": "file_copy.txt"}).delete()
@ -352,6 +349,22 @@ class TestFile(unittest.TestCase):
 		self.assertEqual(file1.file_url, file2.file_url)
 		self.assertTrue(os.path.exists(file2.get_full_path()))

+	def test_parent_directory_validation_in_file_url(self):
+		file1 = frappe.get_doc({
+			"doctype": "File",
+			"file_name": 'parent_dir.txt',
+			"attached_to_doctype": "",
+			"attached_to_name": "",
+			"is_private": 1,
+			"content": test_content1}).insert()
+
+		file1.file_url = '/private/files/../test.txt'
+		self.assertRaises(frappe.exceptions.ValidationError, file1.save)
+
+		# No validation to see if file exists
+		file1.reload()
+		file1.file_url = '/private/files/parent_dir2.txt'
+		file1.save()

 class TestAttachment(unittest.TestCase):
 	test_doctype = 'Test For Attachment'
@ -399,3 +412,61 @@ class TestAttachment(unittest.TestCase):
 		})

 		self.assertTrue(exists)
+
+
+class TestAttachmentsAccess(unittest.TestCase):
+
+	def test_attachments_access(self):
+
+		frappe.set_user('test4@example.com')
+		self.attached_to_doctype, self.attached_to_docname = make_test_doc()
+
+		frappe.get_doc({
+			"doctype": "File",
+			"file_name": 'test_user.txt',
+			"attached_to_doctype": self.attached_to_doctype,
+			"attached_to_name": self.attached_to_docname,
+			"content": 'Testing User'
+		}).insert()
+
+		frappe.get_doc({
+			"doctype": "File",
+			"file_name": "test_user_home.txt",
+			"content": 'User Home',
+		}).insert()
+
+		frappe.set_user('test@example.com')
+
+		frappe.get_doc({
+			"doctype": "File",
+			"file_name": 'test_system_manager.txt',
+			"attached_to_doctype": self.attached_to_doctype,
+			"attached_to_name": self.attached_to_docname,
+			"content": 'Testing System Manager'
+		}).insert()
+
+		frappe.get_doc({
+			"doctype": "File",
+			"file_name": "test_sm_home.txt",
+			"content": 'System Manager Home',
+		}).insert()
+
+		system_manager_files = [file.file_name for file in get_files_in_folder('Home')['files']]
+		system_manager_attachments_files = [file.file_name for file in get_files_in_folder('Home/Attachments')['files']]
+
+		frappe.set_user('test4@example.com')
+		user_files = [file.file_name for file in get_files_in_folder('Home')['files']]
+		user_attachments_files = [file.file_name for file in get_files_in_folder('Home/Attachments')['files']]
+
+		self.assertIn('test_sm_home.txt', system_manager_files)
+		self.assertNotIn('test_sm_home.txt', user_files)
+		self.assertIn('test_user_home.txt', system_manager_files)
+		self.assertIn('test_user_home.txt', user_files)
+
+		self.assertIn('test_system_manager.txt', system_manager_attachments_files)
+		self.assertNotIn('test_system_manager.txt', user_attachments_files)
+		self.assertIn('test_user.txt', system_manager_attachments_files)
+		self.assertIn('test_user.txt', user_attachments_files)
+
+		frappe.set_user('Administrator')
+		frappe.db.rollback()
--- a/frappe/core/doctype/prepared_report/prepared_report.py
+++ b/frappe/core/doctype/prepared_report/prepared_report.py
@ -24,8 +24,6 @@ class PreparedReport(Document):
 	def enqueue_report(self):
 		enqueue(run_background, prepared_report=self.name, timeout=6000)

-	def on_trash(self):
-		remove_all("Prepared Report", self.name)


 def run_background(prepared_report):
@ -39,7 +37,10 @@ def run_background(prepared_report):
 			custom_report_doc = report
 			reference_report = custom_report_doc.reference_report
 			report = frappe.get_doc("Report", reference_report)
-			report.custom_columns = custom_report_doc.json
+			if custom_report_doc.json:
+				data = json.loads(custom_report_doc.json)
+				if data:
+					report.custom_columns = data["columns"]

 		result = generate_report_result(
 			report=report,
@ -100,7 +101,7 @@ def delete_expired_prepared_reports():
 def delete_prepared_reports(reports):
 	reports = frappe.parse_json(reports)
 	for report in reports:
-		frappe.delete_doc('Prepared Report', report['name'], ignore_permissions=True)
+		frappe.delete_doc('Prepared Report', report['name'], ignore_permissions=True, delete_permanently=True)

 def create_json_gz_file(data, dt, dn):
 	# Storing data in CSV file causes information loss
--- a/frappe/core/doctype/report/report.js
+++ b/frappe/core/doctype/report/report.js
@ -25,7 +25,7 @@ frappe.ui.form.on('Report', {
 			}
 		}, "fa fa-table");

-		if (doc.is_standard === "Yes") {
+		if (doc.is_standard === "Yes" && frm.perm[0].write) {
 			frm.add_custom_button(doc.disabled ? __("Enable Report") : __("Disable Report"), function() {
 				frm.call('toggle_disable', {
 					disable: doc.disabled ? 0 : 1
--- a/frappe/core/doctype/report/report.py
+++ b/frappe/core/doctype/report/report.py
@ -58,6 +58,7 @@ class Report(Document):
 	def get_columns(self):
 		return [d.as_dict(no_default_fields = True) for d in self.columns]

+	@frappe.whitelist()
 	def set_doctype_roles(self):
 		if not self.get('roles') and self.is_standard == 'No':
 			meta = frappe.get_meta(self.ref_doctype)
@ -304,8 +305,11 @@ class Report(Document):

 		return data

-	@Document.whitelist
+	@frappe.whitelist()
 	def toggle_disable(self, disable):
+		if not self.has_permission('write'):
+			frappe.throw(_("You are not allowed to edit the report."))
+
 		self.db_set("disabled", cint(disable))

@frappe.whitelist()
--- a/frappe/core/doctype/report/test_report.py
+++ b/frappe/core/doctype/report/test_report.py
@ -201,3 +201,27 @@ result = [

 		# check values
 		self.assertTrue('System User' in [d.get('type') for d in data[1]])
+
+	def test_toggle_disabled(self):
+		"""Make sure that authorization is respected.
+		"""
+		# Assuming that there will be reports in the system.
+		reports = frappe.get_all(doctype='Report', limit=1)
+		report_name = reports[0]['name']
+		doc = frappe.get_doc('Report', report_name)
+		status = doc.disabled
+
+		# User has write permission on reports and should pass through
+		frappe.set_user('test@example.com')
+		doc.toggle_disable(not status)
+		doc.reload()
+		self.assertNotEqual(status, doc.disabled)
+
+		# User has no write permission on reports, permission error is expected.
+		frappe.set_user('test1@example.com')
+		doc = frappe.get_doc('Report', report_name)
+		with self.assertRaises(frappe.exceptions.ValidationError):
+			doc.toggle_disable(1)
+
+		# Set user back to administrator
+		frappe.set_user('Administrator')
--- a/frappe/core/doctype/role/role.js
+++ b/frappe/core/doctype/role/role.js
@ -3,6 +3,8 @@

 frappe.ui.form.on('Role', {
 	refresh: function(frm) {
+		frm.set_df_property('is_custom', 'read_only', frappe.session.user !== 'Administrator');
+
 		frm.add_custom_button("Role Permissions Manager", function() {
 			frappe.route_options = {"role": frm.doc.name};
 			frappe.set_route("permission-manager");
--- a/frappe/core/doctype/role/role.json
+++ b/frappe/core/doctype/role/role.json
@ -25,7 +25,8 @@
  "form_settings_section",
  "form_sidebar",
  "timeline",
-  "dashboard"
+  "dashboard",
+  "is_custom"
 ],
 "fields": [
  {
@ -141,13 +142,20 @@
   "fieldname": "notifications",
   "fieldtype": "Check",
   "label": "Notifications"
+  },
+  {
+   "default": "0",
+   "fieldname": "is_custom",
+   "fieldtype": "Check",
+   "in_list_view": 1,
+   "label": "Is Custom"
  }
 ],
 "icon": "fa fa-bookmark",
 "idx": 1,
 "index_web_pages_for_search": 1,
 "links": [],
- "modified": "2020-12-03 14:08:38.181035",
+ "modified": "2021-01-27 10:35:37.638350",
 "modified_by": "Administrator",
 "module": "Core",
 "name": "Role",
--- a/frappe/core/doctype/role/role.py
+++ b/frappe/core/doctype/role/role.py
@ -33,7 +33,7 @@ class Role(Document):
 		# set if desk_access is not allowed, unset all desk properties
 		if self.name == 'Guest':
 			self.desk_access = 0
-			
+
 		if not self.desk_access:
 			for key in desk_properties:
 				self.set(key, 0)
@ -53,7 +53,6 @@ class Role(Document):
 				if user_type != user.user_type:
 					user.save()

-
 def get_info_based_on_role(role, field='email'):
 	''' Get information of all users that have been assigned this role '''
 	users = frappe.get_list("Has Role", filters={"role": role, "parenttype": "User"},
@ -73,3 +72,15 @@ def get_user_info(users, field='email'):
 def get_users(role):
 	return [d.parent for d in frappe.get_all("Has Role", filters={"role": role, "parenttype": "User"},
 		fields=["parent"])]
+
+
+# searches for active employees
+@frappe.whitelist()
+@frappe.validate_and_sanitize_search_inputs
+def role_query(doctype, txt, searchfield, start, page_len, filters):
+	report_filters = [['Role', 'name', 'like', '%{}%'.format(txt)], ['Role', 'is_custom', '=', 0]]
+	if filters and isinstance(filters, list):
+		report_filters.extend(filters)
+
+	return frappe.get_all('Role', limit_start=start, limit_page_length=page_len,
+		filters=report_filters, as_list=1)
--- a/frappe/core/doctype/role_permission_for_page_and_report/role_permission_for_page_and_report.py
+++ b/frappe/core/doctype/role_permission_for_page_and_report/role_permission_for_page_and_report.py
@ -8,6 +8,7 @@ from frappe.core.doctype.report.report import is_prepared_report_disabled
 from frappe.model.document import Document

 class RolePermissionforPageandReport(Document):
+	@frappe.whitelist()
 	def set_report_page_data(self):
 		self.set_custom_roles()
 		self.check_prepared_report_disabled()
@ -35,12 +36,14 @@ class RolePermissionforPageandReport(Document):
 		doc = frappe.get_doc(doctype, docname)
 		return doc.roles

+	@frappe.whitelist()
 	def reset_roles(self):
 		roles = self.get_standard_roles()
 		self.set('roles', roles)
 		self.update_custom_roles()
 		self.update_disable_prepared_report()

+	@frappe.whitelist()
 	def update_report_page_data(self):
 		self.update_custom_roles()
 		self.update_disable_prepared_report()
--- a/frappe/core/doctype/system_settings/system_settings.json
+++ b/frappe/core/doctype/system_settings/system_settings.json
@ -67,8 +67,7 @@
  "enable_prepared_report_auto_deletion",
  "prepared_report_expiry_period",
  "chat",
-  "enable_chat",
-  "use_socketio_to_upload_file"
+  "enable_chat"
 ],
 "fields": [
  {
@ -394,12 +393,6 @@
   "fieldtype": "Check",
   "label": "Enable Chat"
  },
-  {
-   "default": "1",
-   "fieldname": "use_socketio_to_upload_file",
-   "fieldtype": "Check",
-   "label": "Use socketio to upload file"
-  },
  {
   "fieldname": "column_break_21",
   "fieldtype": "Column Break"
@ -446,7 +439,7 @@
  {
   "default": "30",
   "depends_on": "enable_prepared_report_auto_deletion",
-   "description": "System will automatically delete Prepared Reports after these many days since creation",
+   "description": "System will auto-delete Prepared Reports permanently after these many days since creation",
   "fieldname": "prepared_report_expiry_period",
   "fieldtype": "Int",
   "label": "Prepared Report Expiry Period (Days)"
@ -465,16 +458,11 @@
  },
  {
   "default": "Frappe",
+   "description": "The application name will be used in the Login page.",
   "fieldname": "app_name",
   "fieldtype": "Data",
-   "label": "App Name"
-  },
-  {
-   "default": "3",
-   "description": "Hourly rate limit for generating password reset links",
-   "fieldname": "password_reset_limit",
-   "fieldtype": "Int",
-   "label": "Password Reset Link Generation Limit"
+   "hidden": 1,
+   "label": "Application Name"
  },
  {
   "default": "1",
@ -486,7 +474,7 @@
 "icon": "fa fa-cog",
 "issingle": 1,
 "links": [],
- "modified": "2020-12-30 18:52:22.161391",
+ "modified": "2021-03-30 11:47:47.330437",
 "modified_by": "Administrator",
 "module": "Core",
 "name": "System Settings",
--- a/frappe/core/doctype/test/init.py
+++ b/frappe/core/doctype/test/init.py
--- a/frappe/core/doctype/test/test.js
+++ b/frappe/core/doctype/test/test.js
@ -0,0 +1,8 @@
+// Copyright (c) 2021, Frappe Technologies and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on('test', {
+	// refresh: function(frm) {
+
+	// }
+});
--- a/frappe/core/doctype/test/test.json
+++ b/frappe/core/doctype/test/test.json
@ -0,0 +1,42 @@
+{
+ "actions": [],
+ "creation": "2021-03-31 10:06:57.919697",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "test"
+ ],
+ "fields": [
+  {
+   "fieldname": "test",
+   "fieldtype": "Data",
+   "label": "Test"
+  }
+ ],
+ "index_web_pages_for_search": 1,
+ "is_virtual": 1,
+ "links": [],
+ "modified": "2021-03-31 10:06:57.919697",
+ "modified_by": "Administrator",
+ "module": "Core",
+ "name": "test",
+ "owner": "Administrator",
+ "permissions": [
+  {
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "export": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "System Manager",
+   "share": 1,
+   "write": 1
+  }
+ ],
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "track_changes": 1
+}
--- a/frappe/core/doctype/test/test.py
+++ b/frappe/core/doctype/test/test.py
@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2021, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+# import frappe
+from frappe.model.document import Document
+import json
+
+
+class test(Document):
+
+	def db_insert(self):
+		d = self.get_valid_dict(convert_dates_to_str=True)
+		with open("data_file.json", "w+") as read_file:
+			json.dump(d, read_file)
+
+	def load_from_db(self):
+		with open("data_file.json", "r") as read_file:
+			d = json.load(read_file)
+			super(Document, self).__init__(d)
+
+	def db_update(self):
+		d = self.get_valid_dict(convert_dates_to_str=True)
+		with open("data_file.json", "w+") as read_file:
+			json.dump(d, read_file)
+
+	def get_list(self, args):
+		with open("data_file.json", "r") as read_file:
+			return [json.load(read_file)]
+
+	def get_value(self, fields, filters, **kwargs):
+		# return []
+		with open("data_file.json", "r") as read_file:
+			return [json.load(read_file)]
--- a/frappe/core/doctype/test/test_test.py
+++ b/frappe/core/doctype/test/test_test.py
@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2021, Frappe Technologies and Contributors
+# See license.txt
+from __future__ import unicode_literals
+
+# import frappe
+import unittest
+
+class Testtest(unittest.TestCase):
+	pass
--- a/frappe/core/doctype/user/test_records.json
+++ b/frappe/core/doctype/user/test_records.json
@ -38,6 +38,13 @@
  "new_password": "Eastern_43A1W",
  "enabled": 1
 },
+ {
+  "doctype": "User",
+  "email": "test4@example.com",
+  "first_name": "_Test4",
+  "new_password": "Eastern_43A1W",
+  "enabled": 1
+ },
 {
  "doctype": "User",
  "email": "testperm@example.com",
--- a/frappe/core/doctype/user/test_user.py
+++ b/frappe/core/doctype/user/test_user.py
@ -229,6 +229,28 @@ class TestUser(unittest.TestCase):
 		self.assertEqual(extract_mentions(comment)[0], "test_user@example.com")
 		self.assertEqual(extract_mentions(comment)[1], "test.again@example1.com")

+		doc = frappe.get_doc({
+			'doctype': 'User Group',
+			'name': 'Team',
+			'user_group_members': [{
+				'user': 'test@example.com'
+			}, {
+				'user': 'test1@example.com'
+			}]
+		})
+		doc.insert(ignore_if_duplicate=True)
+
+		comment = '''
+			<div>
+				Testing comment for
+				<span class="mention" data-id="Team" data-value="Team" data-is-group="true" data-denotation-char="@">
+					<span><span class="ql-mention-denotation-char">@</span>Team</span>
+				</span>
+				please check
+			</div>
+		'''
+		self.assertListEqual(extract_mentions(comment), ['test@example.com', 'test1@example.com'])
+
 	def test_rate_limiting_for_reset_password(self):
 		# Allow only one reset request for a day
 		frappe.db.set_value("System Settings", "System Settings", "password_reset_limit", 1)
@ -247,29 +269,31 @@ class TestUser(unittest.TestCase):
 		self.assertEqual(res1.status_code, 200)
 		self.assertEqual(res2.status_code, 417)

-	def test_user_rollback(self):
-		""" """
-		frappe.db.commit()
-		frappe.db.begin()
-		user_id = str(uuid.uuid4())
-		email = f'{user_id}@example.com'
-		try:
-			frappe.flags.in_import = True  # disable throttling
-			frappe.get_doc(dict(
-				doctype='User',
-				email=email,
-				first_name=user_id,
-			)).insert()
-		finally:
-			frappe.flags.in_import = False
+	# def test_user_rollback(self):
+	# 	"""
+	#	FIXME: This is failing with PR #12693 as Rollback can't happen if notifications sent on user creation.
+	#	Make sure that notifications disabled.
+	# 	"""
+	# 	frappe.db.commit()
+	# 	frappe.db.begin()
+	# 	user_id = str(uuid.uuid4())
+	# 	email = f'{user_id}@example.com'
+	# 	try:
+	# 		frappe.flags.in_import = True  # disable throttling
+	# 		frappe.get_doc(dict(
+	# 			doctype='User',
+	# 			email=email,
+	# 			first_name=user_id,
+	# 		)).insert()
+	# 	finally:
+	# 		frappe.flags.in_import = False

-		# Check user has been added
-		self.assertIsNotNone(frappe.db.get("User", {"email": email}))
-
-		# Check that rollback works
-		frappe.db.rollback()
-		self.assertIsNone(frappe.db.get("User", {"email": email}))
+	# 	# Check user has been added
+	# 	self.assertIsNotNone(frappe.db.get("User", {"email": email}))

+	# 	# Check that rollback works
+	# 	frappe.db.rollback()
+	# 	self.assertIsNone(frappe.db.get("User", {"email": email}))

 def delete_contact(user):
 	frappe.db.sql("DELETE FROM `tabContact` WHERE `email_id`= %s", user)
--- a/frappe/core/doctype/user/user.js
+++ b/frappe/core/doctype/user/user.js
@ -59,15 +59,18 @@ frappe.ui.form.on('User', {
 	onload: function(frm) {
 		frm.can_edit_roles = has_access_to_edit_user();

-		if (frm.can_edit_roles && !frm.is_new()) {
+		if (frm.can_edit_roles && !frm.is_new() && in_list(['System User', 'Website User'], frm.doc.user_type)) {
 			if (!frm.roles_editor) {
 				const role_area = $('<div class="role-editor">')
 					.appendTo(frm.fields_dict.roles_html.wrapper);
+
 				frm.roles_editor = new frappe.RoleEditor(role_area, frm, frm.doc.role_profile_name ? 1 : 0);

-				var module_area = $('<div>')
-					.appendTo(frm.fields_dict.modules_html.wrapper);
-				frm.module_editor = new frappe.ModuleEditor(frm, module_area);
+				if (frm.doc.user_type == 'System User') {
+					var module_area = $('<div>')
+						.appendTo(frm.fields_dict.modules_html.wrapper);
+					frm.module_editor = new frappe.ModuleEditor(frm, module_area);
+				}
 			} else {
 				frm.roles_editor.show();
 			}
@ -75,7 +78,8 @@ frappe.ui.form.on('User', {
 	},
 	refresh: function(frm) {
 		var doc = frm.doc;
-		if(!frm.is_new() && !frm.roles_editor && frm.can_edit_roles) {
+		if (in_list(['System User', 'Website User'], frm.doc.user_type)
+			&& !frm.is_new() && !frm.roles_editor && frm.can_edit_roles) {
 			frm.reload_doc();
 			return;
 		}
@ -250,15 +254,15 @@ frappe.ui.form.on('User', {
 			}
 		});
 	},
-	generate_keys: function(frm){
+	generate_keys: function(frm) {
 		frappe.call({
 			method: 'frappe.core.doctype.user.user.generate_keys',
 			args: {
 				user: frm.doc.name
 			},
-			callback: function(r){
-				if(r.message){
-					frappe.msgprint(__("Save API Secret: ") + r.message.api_secret);
+			callback: function(r) {
+				if (r.message) {
+					frappe.msgprint(__("Save API Secret: {0}", [r.message.api_secret]));
 				}
 			}
 		});
--- a/frappe/core/doctype/user/user.json
+++ b/frappe/core/doctype/user/user.json
@ -191,7 +191,7 @@
   "print_hide": 1
  },
  {
-   "depends_on": "enabled",
+   "depends_on": "eval:in_list(['System User', 'Website User'], doc.user_type) && doc.enabled == 1",
   "fieldname": "sb1",
   "fieldtype": "Section Break",
   "label": "Roles",
@ -391,6 +391,7 @@
  },
  {
   "collapsible": 1,
+   "depends_on": "eval:in_list(['System User'], doc.user_type)",
   "fieldname": "sb_allow_modules",
   "fieldtype": "Section Break",
   "label": "Allow Modules",
@ -453,18 +454,18 @@
   "label": "Simultaneous Sessions"
  },
  {
+   "bold": 1,
   "default": "System User",
   "description": "If the user has any role checked, then the user becomes a \"System User\". \"System User\" has access to the desktop",
   "fieldname": "user_type",
-   "fieldtype": "Select",
+   "fieldtype": "Link",
   "in_list_view": 1,
   "in_standard_filter": 1,
   "label": "User Type",
   "oldfieldname": "user_type",
   "oldfieldtype": "Select",
-   "options": "System User\nWebsite User",
-   "permlevel": 1,
-   "read_only": 1
+   "options": "User Type",
+   "permlevel": 1
  },
  {
   "description": "Allow user to login only after this hour (0-24)",
@ -669,7 +670,7 @@
  }
 ],
 "max_attachments": 5,
- "modified": "2021-02-01 16:11:06.037543",
+ "modified": "2021-02-02 16:11:06.037543",
 "modified_by": "Administrator",
 "module": "Core",
 "name": "User",
--- a/frappe/core/doctype/user/user.py
+++ b/frappe/core/doctype/user/user.py
@ -10,7 +10,8 @@ import frappe.share
 import frappe.defaults
 import frappe.permissions
 from frappe.model.document import Document
-from frappe.utils import cint, flt, has_gravatar, escape_html, format_datetime, now_datetime, get_formatted_email, today
+from frappe.utils import (cint, flt, has_gravatar, escape_html, format_datetime,
+	now_datetime, get_formatted_email, today)
 from frappe import throw, msgprint, _
 from frappe.utils.password import update_password as _update_password, check_password, get_password_reset_limit
 from frappe.desk.notifications import clear_notifications
@ -19,6 +20,7 @@ from frappe.utils.user import get_system_managers
 from frappe.website.utils import is_signup_enabled
 from frappe.rate_limiter import rate_limit
 from frappe.utils.background_jobs import enqueue
+from frappe.core.doctype.user_type.user_type import user_linked_with_permission_on_doctype


 STANDARD_USERS = ("Guest", "Administrator")
@ -186,11 +188,36 @@ class User(Document):
 			_update_password(user=self.name, pwd=new_password, logout_all_sessions=self.logout_all_sessions)

 	def set_system_user(self):
-		'''Set as System User if any of the given roles has desk_access'''
-		if self.has_desk_access() or self.name == 'Administrator':
-			self.user_type = 'System User'
+		'''For the standard users like admin and guest, the user type is fixed.'''
+		user_type_mapper = {
+			'Administrator': 'System User',
+			'Guest': 'Website User'
+		}
+
+		if self.user_type and not frappe.get_cached_value('User Type', self.user_type, 'is_standard'):
+			if user_type_mapper.get(self.name):
+				self.user_type = user_type_mapper.get(self.name)
+			else:
+				self.set_roles_and_modules_based_on_user_type()
 		else:
-			self.user_type = 'Website User'
+			'''Set as System User if any of the given roles has desk_access'''
+			self.user_type = 'System User' if self.has_desk_access() else 'Website User'
+
+	def set_roles_and_modules_based_on_user_type(self):
+		user_type_doc = frappe.get_cached_doc('User Type', self.user_type)
+		if user_type_doc.role:
+			self.roles = []
+
+			# Check whether User has linked with the 'Apply User Permission On' doctype or not
+			if user_linked_with_permission_on_doctype(user_type_doc, self.name):
+				self.append('roles', {
+					'role': user_type_doc.role
+				})
+
+				frappe.msgprint(_('Role has been set as per the user type {0}')
+					.format(self.user_type), alert=True)
+
+		user_type_doc.update_modules_in_user(self)

 	def has_desk_access(self):
 		'''Return true if any of the set roles has desk access'''
@ -534,24 +561,36 @@ class User(Document):
 	@classmethod
 	def find_by_credentials(cls, user_name: str, password: str, validate_password: bool = True):
 		"""Find the user by credentials.
-		"""
-		login_with_mobile = cint(frappe.db.get_value("System Settings", "System Settings", "allow_login_using_mobile_number"))
-		filter = {"mobile_no": user_name} if login_with_mobile else {"name": user_name}

-		user = frappe.db.get_value("User", filters=filter, fieldname=['name', 'enabled'], as_dict=True) or {}
-		if not user:
+		This is a login utility that needs to check login related system settings while finding the user.
+		1. Find user by email ID by default
+		2. If allow_login_using_mobile_number is set, you can use mobile number while finding the user.
+		3. If allow_login_using_user_name is set, you can use username while finding the user.
+		"""
+
+		login_with_mobile = cint(frappe.db.get_value("System Settings", "System Settings", "allow_login_using_mobile_number"))
+		login_with_username = cint(frappe.db.get_value("System Settings", "System Settings", "allow_login_using_user_name"))
+
+		or_filters = [{"name": user_name}]
+		if login_with_mobile:
+			or_filters.append({"mobile_no": user_name})
+		if login_with_username:
+			or_filters.append({"username": user_name})
+
+		users = frappe.db.get_all('User', fields=['name', 'enabled'], or_filters=or_filters, limit=1)
+		if not users:
 			return

+		user = users[0]
 		user['is_authenticated'] = True
 		if validate_password:
 			try:
-				check_password(user_name, password)
+				check_password(user['name'], password, delete_tracker_cache=False)
 			except frappe.AuthenticationError:
 				user['is_authenticated'] = False

 		return user

-
@frappe.whitelist()
 def get_timezones():
 	import pytz
@ -863,11 +902,13 @@ def reset_password(user):
@frappe.whitelist()
@frappe.validate_and_sanitize_search_inputs
 def user_query(doctype, txt, searchfield, start, page_len, filters):
-	from frappe.desk.reportview import get_match_cond
+	from frappe.desk.reportview import get_match_cond, get_filters_cond
+	conditions=[]

-	user_type_condition = "and user_type = 'System User'"
+	user_type_condition = "and user_type != 'Website User'"
 	if filters and filters.get('ignore_user_type'):
 		user_type_condition = ''
+		filters.pop('ignore_user_type')

 	txt = "%{}%".format(txt)
 	return frappe.db.sql("""SELECT `name`, CONCAT_WS(' ', first_name, middle_name, last_name)
@ -878,17 +919,22 @@ def user_query(doctype, txt, searchfield, start, page_len, filters):
 			AND `name` NOT IN ({standard_users})
 			AND ({key} LIKE %(txt)s
 				OR CONCAT_WS(' ', first_name, middle_name, last_name) LIKE %(txt)s)
-			{mcond}
+			{fcond} {mcond}
 		ORDER BY
 			CASE WHEN `name` LIKE %(txt)s THEN 0 ELSE 1 END,
 			CASE WHEN concat_ws(' ', first_name, middle_name, last_name) LIKE %(txt)s
 				THEN 0 ELSE 1 END,
 			NAME asc
-		LIMIT %(page_len)s OFFSET %(start)s""".format(
+		LIMIT %(page_len)s OFFSET %(start)s
+	""".format(
 			user_type_condition = user_type_condition,
 			standard_users=", ".join([frappe.db.escape(u) for u in STANDARD_USERS]),
-			key=searchfield, mcond=get_match_cond(doctype)),
-			dict(start=start, page_len=page_len, txt=txt))
+			key=searchfield,
+			fcond=get_filters_cond(doctype, filters, conditions),
+			mcond=get_match_cond(doctype)
+		),
+		dict(start=start, page_len=page_len, txt=txt)
+	)

 def get_total_users():
 	"""Returns total no. of system users"""
@ -972,8 +1018,16 @@ def extract_mentions(txt):
 	soup = BeautifulSoup(txt, 'html.parser')
 	emails = []
 	for mention in soup.find_all(class_='mention'):
+		if mention.get('data-is-group') == 'true':
+			try:
+				user_group = frappe.get_cached_doc('User Group', mention['data-id'])
+				emails += [d.user for d in user_group.user_group_members]
+			except frappe.DoesNotExistError:
+				pass
+			continue
 		email = mention['data-id']
 		emails.append(email)
+
 	return emails

 def handle_password_test_fail(result):
--- a/frappe/core/doctype/user_document_type/init.py
+++ b/frappe/core/doctype/user_document_type/init.py
--- a/frappe/core/doctype/user_document_type/user_document_type.json
+++ b/frappe/core/doctype/user_document_type/user_document_type.json
@ -0,0 +1,109 @@
+{
+ "actions": [],
+ "creation": "2021-01-13 01:51:40.158521",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "document_type",
+  "column_break_2",
+  "is_custom",
+  "permissions_section",
+  "read",
+  "write",
+  "create",
+  "column_break_8",
+  "submit",
+  "cancel",
+  "amend",
+  "delete"
+ ],
+ "fields": [
+  {
+   "fieldname": "document_type",
+   "fieldtype": "Link",
+   "in_list_view": 1,
+   "label": "Document Type",
+   "options": "DocType",
+   "reqd": 1
+  },
+  {
+   "fieldname": "permissions_section",
+   "fieldtype": "Section Break",
+   "label": "Role Permissions"
+  },
+  {
+   "default": "1",
+   "fieldname": "read",
+   "fieldtype": "Check",
+   "in_list_view": 1,
+   "label": "Read"
+  },
+  {
+   "default": "0",
+   "fieldname": "write",
+   "fieldtype": "Check",
+   "in_list_view": 1,
+   "label": "Write"
+  },
+  {
+   "default": "0",
+   "fieldname": "create",
+   "fieldtype": "Check",
+   "in_list_view": 1,
+   "label": "Create"
+  },
+  {
+   "fieldname": "column_break_2",
+   "fieldtype": "Column Break"
+  },
+  {
+   "default": "0",
+   "fetch_from": "document_type.custom",
+   "fieldname": "is_custom",
+   "fieldtype": "Check",
+   "label": "Is Custom",
+   "read_only": 1
+  },
+  {
+   "fieldname": "column_break_8",
+   "fieldtype": "Column Break"
+  },
+  {
+   "default": "0",
+   "fieldname": "submit",
+   "fieldtype": "Check",
+   "label": "Submit"
+  },
+  {
+   "default": "0",
+   "fieldname": "cancel",
+   "fieldtype": "Check",
+   "label": "Cancel"
+  },
+  {
+   "default": "0",
+   "fieldname": "amend",
+   "fieldtype": "Check",
+   "label": "Amend"
+  },
+  {
+   "default": "0",
+   "fieldname": "delete",
+   "fieldtype": "Check",
+   "label": "Delete"
+  }
+ ],
+ "index_web_pages_for_search": 1,
+ "istable": 1,
+ "links": [],
+ "modified": "2021-03-16 00:32:24.414313",
+ "modified_by": "Administrator",
+ "module": "Core",
+ "name": "User Document Type",
+ "owner": "Administrator",
+ "permissions": [],
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "track_changes": 1
+}
--- a/frappe/core/doctype/user_document_type/user_document_type.py
+++ b/frappe/core/doctype/user_document_type/user_document_type.py
@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2021, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+# import frappe
+from frappe.model.document import Document
+
+class UserDocumentType(Document):
+	pass
--- a/frappe/core/doctype/user_group/init.py
+++ b/frappe/core/doctype/user_group/init.py
--- a/frappe/core/doctype/user_group/test_user_group.py
+++ b/frappe/core/doctype/user_group/test_user_group.py
@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2021, Frappe Technologies and Contributors
+# See license.txt
+from __future__ import unicode_literals
+
+# import frappe
+import unittest
+
+class TestUserGroup(unittest.TestCase):
+	pass
--- a/frappe/core/doctype/user_group/user_group.js
+++ b/frappe/core/doctype/user_group/user_group.js
@ -0,0 +1,8 @@
+// Copyright (c) 2021, Frappe Technologies and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on('User Group', {
+	// refresh: function(frm) {
+
+	// }
+});
--- a/frappe/core/doctype/user_group/user_group.json
+++ b/frappe/core/doctype/user_group/user_group.json
@ -0,0 +1,48 @@
+{
+ "actions": [],
+ "autoname": "Prompt",
+ "creation": "2021-04-12 15:17:24.751710",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "user_group_members"
+ ],
+ "fields": [
+  {
+   "fieldname": "user_group_members",
+   "fieldtype": "Table MultiSelect",
+   "label": "User Group Members",
+   "options": "User Group Member",
+   "reqd": 1
+  }
+ ],
+ "index_web_pages_for_search": 1,
+ "links": [],
+ "modified": "2021-04-15 16:12:31.455401",
+ "modified_by": "Administrator",
+ "module": "Core",
+ "name": "User Group",
+ "owner": "Administrator",
+ "permissions": [
+  {
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "export": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "System Manager",
+   "share": 1,
+   "write": 1
+  },
+  {
+   "read": 1,
+   "role": "All"
+  }
+ ],
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "track_changes": 1
+}
--- a/frappe/core/doctype/user_group/user_group.py
+++ b/frappe/core/doctype/user_group/user_group.py
@ -0,0 +1,15 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2021, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+# import frappe
+from frappe.model.document import Document
+import frappe
+
+class UserGroup(Document):
+	def after_insert(self):
+		frappe.publish_realtime('user_group_added', self.name)
+
+	def on_trash(self):
+		frappe.publish_realtime('user_group_deleted', self.name)
--- a/frappe/core/doctype/user_group_member/init.py
+++ b/frappe/core/doctype/user_group_member/init.py
--- a/frappe/core/doctype/user_group_member/test_user_group_member.py
+++ b/frappe/core/doctype/user_group_member/test_user_group_member.py
@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2021, Frappe Technologies and Contributors
+# See license.txt
+from __future__ import unicode_literals
+
+# import frappe
+import unittest
+
+class TestUserGroupMember(unittest.TestCase):
+	pass
--- a/frappe/core/doctype/user_group_member/user_group_member.js
+++ b/frappe/core/doctype/user_group_member/user_group_member.js
@ -0,0 +1,8 @@
+// Copyright (c) 2021, Frappe Technologies and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on('User Group Member', {
+	// refresh: function(frm) {
+
+	// }
+});
--- a/frappe/core/doctype/user_group_member/user_group_member.json
+++ b/frappe/core/doctype/user_group_member/user_group_member.json
@ -0,0 +1,32 @@
+{
+ "actions": [],
+ "creation": "2021-04-12 15:16:29.279107",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "user"
+ ],
+ "fields": [
+  {
+   "fieldname": "user",
+   "fieldtype": "Link",
+   "in_list_view": 1,
+   "label": "User",
+   "options": "User",
+   "reqd": 1
+  }
+ ],
+ "index_web_pages_for_search": 1,
+ "istable": 1,
+ "links": [],
+ "modified": "2021-04-12 15:17:18.773046",
+ "modified_by": "Administrator",
+ "module": "Core",
+ "name": "User Group Member",
+ "owner": "Administrator",
+ "permissions": [],
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "track_changes": 1
+}
--- a/frappe/core/doctype/user_group_member/user_group_member.py
+++ b/frappe/core/doctype/user_group_member/user_group_member.py
@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2021, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+# import frappe
+from frappe.model.document import Document
+
+class UserGroupMember(Document):
+	pass
--- a/frappe/core/doctype/user_permission/test_user_permission.py
+++ b/frappe/core/doctype/user_permission/test_user_permission.py
@ -2,8 +2,9 @@
 # Copyright (c) 2017, Frappe Technologies and Contributors
 # See license.txt
 from __future__ import unicode_literals
-from frappe.core.doctype.user_permission.user_permission import add_user_permissions
+from frappe.core.doctype.user_permission.user_permission import add_user_permissions, remove_applicable
 from frappe.permissions import has_user_permission
+from frappe.core.doctype.doctype.test_doctype import new_doctype

 import frappe
 import unittest
@ -17,6 +18,8 @@ class TestUserPermission(unittest.TestCase):
 				'nested_doc_user@example.com')""")
 		frappe.delete_doc_if_exists("DocType", "Person")
 		frappe.db.sql_ddl("DROP TABLE IF EXISTS `tabPerson`")
+		frappe.delete_doc_if_exists("DocType", "Doc A")
+		frappe.db.sql_ddl("DROP TABLE IF EXISTS `tabDoc A`")

 	def test_default_user_permission_validation(self):
 		user = create_user('test_default_permission@example.com')
@ -153,16 +156,98 @@ class TestUserPermission(unittest.TestCase):
 		self.assertTrue(has_user_permission(frappe.get_doc("Person", parent_record.name), user.name))
 		self.assertFalse(has_user_permission(frappe.get_doc("Person", child_record.name), user.name))

-def create_user(email, role="System Manager"):
+	def test_user_perm_on_new_doc_with_field_default(self):
+		"""Test User Perm impact on frappe.new_doc. with *field* default value"""
+		frappe.set_user('Administrator')
+		user = create_user("new_doc_test@example.com", "Blogger")
+
+		# make a doctype "Doc A" with 'doctype' link field and default value ToDo
+		if not frappe.db.exists("DocType", "Doc A"):
+			doc = new_doctype("Doc A",
+				fields=[
+					{
+						"label": "DocType",
+						"fieldname": "doc",
+						"fieldtype": "Link",
+						"options": "DocType",
+						"default": "ToDo"
+					}
+				], unique=0)
+			doc.insert()
+
+		# make User Perm on DocType 'ToDo' in Assignment Rule (unrelated doctype)
+		add_user_permissions(get_params(user, "DocType", "ToDo", applicable=["Assignment Rule"]))
+		frappe.set_user("new_doc_test@example.com")
+
+		new_doc = frappe.new_doc("Doc A")
+
+		# User perm is created on ToDo but for doctype Assignment Rule only
+		# it should not have impact on Doc A
+		self.assertEquals(new_doc.doc, "ToDo")
+
+		frappe.set_user('Administrator')
+		remove_applicable(["Assignment Rule"], "new_doc_test@example.com", "DocType", "ToDo")
+
+	def test_user_perm_on_new_doc_with_user_default(self):
+		"""Test User Perm impact on frappe.new_doc. with *user* default value"""
+		from frappe.core.doctype.session_default_settings.session_default_settings import (clear_session_defaults,
+			set_session_default_values)
+
+		frappe.set_user('Administrator')
+		user = create_user("user_default_test@example.com", "Blogger")
+
+		# make a doctype "Doc A" with 'doctype' link field
+		if not frappe.db.exists("DocType", "Doc A"):
+			doc = new_doctype("Doc A",
+				fields=[
+					{
+						"label": "DocType",
+						"fieldname": "doc",
+						"fieldtype": "Link",
+						"options": "DocType",
+					}
+				], unique=0)
+			doc.insert()
+
+		# create a 'DocType' session default field
+		if not frappe.db.exists("Session Default", {"ref_doctype": "DocType"}):
+			settings = frappe.get_single('Session Default Settings')
+			settings.append("session_defaults", {
+				"ref_doctype": "DocType"
+			})
+			settings.save()
+
+		# make User Perm on DocType 'ToDo' in Assignment Rule (unrelated doctype)
+		add_user_permissions(get_params(user, "DocType", "ToDo", applicable=["Assignment Rule"]))
+
+		# User default Doctype value is ToDo via Session Defaults
+		frappe.set_user("user_default_test@example.com")
+		set_session_default_values({"doc": "ToDo"})
+
+		new_doc = frappe.new_doc("Doc A")
+
+		# User perm is created on ToDo but for doctype Assignment Rule only
+		# it should not have impact on Doc A
+		self.assertEquals(new_doc.doc, "ToDo")
+
+		frappe.set_user('Administrator')
+		clear_session_defaults()
+		remove_applicable(["Assignment Rule"], "user_default_test@example.com", "DocType", "ToDo")
+
+def create_user(email, *roles):
 	''' create user with role system manager '''
 	if frappe.db.exists('User', email):
 		return frappe.get_doc('User', email)
-	else:
-		user = frappe.new_doc('User')
-		user.email = email
-		user.first_name = email.split("@")[0]
-		user.add_roles(role)
-		return user
+
+	user = frappe.new_doc('User')
+	user.email = email
+	user.first_name = email.split("@")[0]
+
+	if not roles:
+		roles = ('System Manager',)
+
+	user.add_roles(*roles)
+	return user

 def get_params(user, doctype, docname, is_default=0, hide_descendants=0, applicable=None):
 	''' Return param to insert '''
--- a/frappe/core/doctype/user_select_document_type/init.py
+++ b/frappe/core/doctype/user_select_document_type/init.py
--- a/frappe/core/doctype/user_select_document_type/user_select_document_type.json
+++ b/frappe/core/doctype/user_select_document_type/user_select_document_type.json
@ -0,0 +1,33 @@
+{
+ "actions": [],
+ "creation": "2021-01-17 18:28:14.208576",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "document_type"
+ ],
+ "fields": [
+  {
+   "fieldname": "document_type",
+   "fieldtype": "Link",
+   "in_list_view": 1,
+   "label": "Document Type",
+   "options": "DocType",
+   "reqd": 1
+  }
+ ],
+ "index_web_pages_for_search": 1,
+ "istable": 1,
+ "links": [],
+ "modified": "2021-01-17 18:45:44.993190",
+ "modified_by": "Administrator",
+ "module": "Core",
+ "name": "User Select Document Type",
+ "owner": "Administrator",
+ "permissions": [],
+ "quick_entry": 1,
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "track_changes": 1
+}
--- a/frappe/core/doctype/user_select_document_type/user_select_document_type.py
+++ b/frappe/core/doctype/user_select_document_type/user_select_document_type.py
@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2021, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+# import frappe
+from frappe.model.document import Document
+
+class UserSelectDocumentType(Document):
+	pass
--- a/frappe/core/doctype/user_type/init.py
+++ b/frappe/core/doctype/user_type/init.py
--- a/frappe/core/doctype/user_type/test_user_type.py
+++ b/frappe/core/doctype/user_type/test_user_type.py
@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2021, Frappe Technologies and Contributors
+# See license.txt
+from __future__ import unicode_literals
+
+# import frappe
+import unittest
+
+class TestUserType(unittest.TestCase):
+	pass
--- a/frappe/core/doctype/user_type/user_type.js
+++ b/frappe/core/doctype/user_type/user_type.js
@ -0,0 +1,77 @@
+// Copyright (c) 2021, Frappe Technologies and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on('User Type', {
+	refresh: function(frm) {
+		frm.toggle_display('is_standard', frappe.boot.developer_mode);
+		frm.set_df_property('is_standard', 'read_only', !frappe.boot.developer_mode);
+
+		const fields = ['role', 'apply_user_permission_on', 'user_id_field',
+			'user_doctypes', 'user_type_modules'];
+
+		frm.toggle_display(fields, !frm.doc.is_standard);
+
+		frm.set_query('document_type', 'user_doctypes', function() {
+			return {
+				filters: {
+					istable: 0
+				}
+			};
+		});
+
+		frm.set_query('document_type', 'select_doctypes', function() {
+			return {
+				filters: {
+					istable: 0
+				}
+			};
+		});
+
+		frm.set_query('document_type', 'custom_select_doctypes', function() {
+			return {
+				filters: {
+					istable: 0
+				}
+			};
+		});
+
+		frm.set_query('role', function() {
+			return {
+				filters: {
+					is_custom: 1,
+					disabled: 0,
+					desk_access: 1
+				}
+			};
+		});
+
+		frm.set_query('apply_user_permission_on', function() {
+			return {
+				query: "frappe.core.doctype.user_type.user_type.get_user_linked_doctypes"
+			};
+		});
+	},
+
+	onload: function(frm) {
+		frm.trigger('get_user_id_fields');
+	},
+
+	apply_user_permission_on: function(frm) {
+		frm.set_value('user_id_field', '');
+		frm.trigger('get_user_id_fields');
+	},
+
+	get_user_id_fields: function(frm) {
+		if (frm.doc.apply_user_permission_on) {
+			frappe.call({
+				method: 'frappe.core.doctype.user_type.user_type.get_user_id',
+				args: {
+					parent: frm.doc.apply_user_permission_on
+				},
+				callback: function(r) {
+					set_field_options('user_id_field', [""].concat(r.message));
+				}
+			});
+		}
+	}
+});
--- a/frappe/core/doctype/user_type/user_type.json
+++ b/frappe/core/doctype/user_type/user_type.json
@ -0,0 +1,141 @@
+{
+ "actions": [],
+ "autoname": "Prompt",
+ "creation": "2021-01-13 01:48:02.378548",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "is_standard",
+  "section_break_2",
+  "role",
+  "column_break_4",
+  "apply_user_permission_on",
+  "user_id_field",
+  "section_break_6",
+  "user_doctypes",
+  "custom_select_doctypes",
+  "select_doctypes",
+  "allowed_modules_section",
+  "user_type_modules"
+ ],
+ "fields": [
+  {
+   "default": "0",
+   "fieldname": "is_standard",
+   "fieldtype": "Check",
+   "label": "Is Standard"
+  },
+  {
+   "depends_on": "eval: !doc.is_standard",
+   "fieldname": "section_break_2",
+   "fieldtype": "Section Break",
+   "label": "Document Types and Permissions"
+  },
+  {
+   "fieldname": "user_doctypes",
+   "fieldtype": "Table",
+   "label": "Document Types",
+   "mandatory_depends_on": "eval: !doc.is_standard",
+   "options": "User Document Type",
+   "read_only": 1
+  },
+  {
+   "fieldname": "role",
+   "fieldtype": "Link",
+   "in_list_view": 1,
+   "label": "Role",
+   "mandatory_depends_on": "eval: !doc.is_standard",
+   "options": "Role",
+   "read_only": 1
+  },
+  {
+   "fieldname": "select_doctypes",
+   "fieldtype": "Table",
+   "hidden": 1,
+   "label": "Document Types (Select Permissions Only)",
+   "options": "User Select Document Type",
+   "read_only": 1
+  },
+  {
+   "fieldname": "column_break_4",
+   "fieldtype": "Column Break"
+  },
+  {
+   "description": "Can only list down the document types which has been linked to the User document type.",
+   "fieldname": "apply_user_permission_on",
+   "fieldtype": "Link",
+   "label": "Apply User Permission On",
+   "mandatory_depends_on": "eval: !doc.is_standard",
+   "options": "DocType",
+   "read_only": 1
+  },
+  {
+   "depends_on": "eval: !doc.is_standard",
+   "fieldname": "section_break_6",
+   "fieldtype": "Section Break",
+   "hide_border": 1
+  },
+  {
+   "depends_on": "apply_user_permission_on",
+   "fieldname": "user_id_field",
+   "fieldtype": "Select",
+   "label": "User Id Field",
+   "mandatory_depends_on": "eval: !doc.is_standard",
+   "read_only": 1
+  },
+  {
+   "depends_on": "eval: !doc.is_standard",
+   "fieldname": "allowed_modules_section",
+   "fieldtype": "Section Break",
+   "label": "Allowed Modules"
+  },
+  {
+   "fieldname": "user_type_modules",
+   "fieldtype": "Table",
+   "no_copy": 1,
+   "options": "User Type Module",
+   "print_hide": 1,
+   "read_only": 1
+  },
+  {
+   "fieldname": "custom_select_doctypes",
+   "fieldtype": "Table",
+   "label": "Custom Document Types (Select Permission)",
+   "options": "User Select Document Type"
+  }
+ ],
+ "index_web_pages_for_search": 1,
+ "links": [],
+ "modified": "2021-03-12 16:25:18.639050",
+ "modified_by": "Administrator",
+ "module": "Core",
+ "name": "User Type",
+ "owner": "Administrator",
+ "permissions": [
+  {
+   "create": 1,
+   "email": 1,
+   "export": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "Administrator",
+   "share": 1,
+   "write": 1
+  },
+  {
+   "email": 1,
+   "export": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "System Manager",
+   "share": 1,
+   "write": 1
+  }
+ ],
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "track_changes": 1
+}
--- a/frappe/core/doctype/user_type/user_type.py
+++ b/frappe/core/doctype/user_type/user_type.py
@ -0,0 +1,270 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2021, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+import frappe
+from frappe import _
+from six import iteritems
+from frappe.utils import get_link_to_form
+from frappe.config import get_modules_from_app
+from frappe.permissions import add_permission, add_user_permission
+from frappe.model.document import Document
+
+class UserType(Document):
+	def validate(self):
+		self.set_modules()
+		self.add_select_perm_doctypes()
+
+	def on_update(self):
+		if self.is_standard:
+			return
+
+		self.validate_document_type_limit()
+		self.validate_role()
+		self.add_role_permissions_for_user_doctypes()
+		self.add_role_permissions_for_select_doctypes()
+		self.add_role_permissions_for_file()
+		self.update_users()
+		get_non_standard_user_type_details()
+		self.remove_permission_for_deleted_doctypes()
+
+	def on_trash(self):
+		if self.is_standard:
+			frappe.throw(_('Standard user type {0} can not be deleted.')
+				.format(frappe.bold(self.name)))
+
+	def set_modules(self):
+		if not self.user_doctypes:
+			return
+
+		modules = frappe.get_all('DocType', fields=['distinct module as module'],
+			filters={'name': ('in', [d.document_type for d in self.user_doctypes])})
+
+		self.set('user_type_modules', [])
+		for row in modules:
+			self.append('user_type_modules', {
+				'module': row.module
+			})
+
+	def validate_document_type_limit(self):
+		limit = frappe.conf.get('user_type_doctype_limit', {}).get(frappe.scrub(self.name))
+
+		if not limit and frappe.session.user != 'Administrator':
+			frappe.throw(_('User does not have permission to create the new {0}')
+				.format(frappe.bold(_('User Type'))), title=_('Permission Error'))
+
+		if not limit:
+			frappe.throw(_('The limit has not set for the user type {0} in the site config file.')
+				.format(frappe.bold(self.name)), title=_('Set Limit'))
+
+		if self.user_doctypes and len(self.user_doctypes) > limit:
+			frappe.throw(_('The total number of user document types limit has been crossed.'),
+				title=_('User Document Types Limit Exceeded'))
+
+		custom_doctypes = [row.document_type for row in self.user_doctypes if row.is_custom]
+		if custom_doctypes and len(custom_doctypes) > 3:
+			frappe.throw(_('You can only set the 3 custom doctypes in the Document Types table.'),
+				title=_('Custom Document Types Limit Exceeded'))
+
+	def validate_role(self):
+		if not self.role:
+			frappe.throw(_("The field {0} is mandatory")
+				.format(frappe.bold(_('Role'))))
+
+		if not frappe.db.get_value('Role', self.role, 'is_custom'):
+			frappe.throw(_("The role {0} should be a custom role.")
+				.format(frappe.bold(get_link_to_form('Role', self.role))))
+
+	def update_users(self):
+		for row in frappe.get_all('User', filters = {'user_type': self.name}):
+			user = frappe.get_cached_doc('User', row.name)
+			self.update_roles_in_user(user)
+			self.update_modules_in_user(user)
+			user.update_children()
+
+	def update_roles_in_user(self, user):
+		user.set('roles', [])
+		user.append('roles', {
+			'role': self.role
+		})
+
+	def update_modules_in_user(self, user):
+		block_modules = frappe.get_all('Module Def', fields = ['name as module'],
+			filters={'name': ['not in', [d.module for d in self.user_type_modules]]})
+
+		if block_modules:
+			user.set('block_modules', block_modules)
+
+	def add_role_permissions_for_user_doctypes(self):
+		perms = ['read', 'write', 'create', 'submit', 'cancel', 'amend', 'delete']
+		for row in self.user_doctypes:
+			docperm = add_role_permissions(row.document_type, self.role)
+
+			values = {perm:row.get(perm) or 0 for perm in perms}
+			for perm in ['print', 'email', 'share']:
+				values[perm] = 1
+
+			frappe.db.set_value('Custom DocPerm', docperm, values)
+
+	def add_select_perm_doctypes(self):
+		if frappe.flags.ignore_select_perm:
+			return
+
+		self.select_doctypes = []
+
+		select_doctypes = []
+		user_doctypes = tuple([row.document_type for row in self.user_doctypes])
+
+		for doctype in user_doctypes:
+			doc = frappe.get_meta(doctype)
+			self.prepare_select_perm_doctypes(doc, user_doctypes, select_doctypes)
+
+			for child_table in doc.get_table_fields():
+				child_doc = frappe.get_meta(child_table.options)
+				if not child_doc.istable:
+					self.prepare_select_perm_doctypes(child_doc, user_doctypes, select_doctypes)
+
+		if select_doctypes:
+			select_doctypes = set(select_doctypes)
+			for select_doctype in select_doctypes:
+				self.append('select_doctypes', {
+					'document_type': select_doctype
+				})
+
+	def prepare_select_perm_doctypes(self, doc, user_doctypes, select_doctypes):
+		for field in doc.get_link_fields():
+			if field.options not in user_doctypes:
+				select_doctypes.append(field.options)
+
+	def add_role_permissions_for_select_doctypes(self):
+		for doctype in ['select_doctypes', 'custom_select_doctypes']:
+			for row in self.get(doctype):
+				docperm = add_role_permissions(row.document_type, self.role)
+				frappe.db.set_value('Custom DocPerm', docperm,
+					{'select': 1, 'read': 0, 'create': 0, 'write': 0})
+
+	def add_role_permissions_for_file(self):
+		docperm = add_role_permissions('File', self.role)
+		frappe.db.set_value('Custom DocPerm', docperm,
+			{'read': 1, 'create': 1, 'write': 1})
+
+	def remove_permission_for_deleted_doctypes(self):
+		doctypes = [d.document_type for d in self.user_doctypes]
+
+		# Do not remove the doc permission for the file doctype
+		doctypes.append('File')
+
+		for doctype in ['select_doctypes', 'custom_select_doctypes']:
+			for dt in self.get(doctype):
+				doctypes.append(dt.document_type)
+
+		for perm in frappe.get_all('Custom DocPerm',
+			filters = {'role': self.role, 'parent': ['not in', doctypes]}):
+			frappe.delete_doc('Custom DocPerm', perm.name)
+
+def add_role_permissions(doctype, role):
+	name = frappe.get_value('Custom DocPerm', dict(parent=doctype,
+		role=role, permlevel=0))
+
+	if not name:
+		name = add_permission(doctype, role, 0)
+
+	return name
+
+def get_non_standard_user_type_details():
+	user_types = frappe.get_all('User Type',
+		fields=['apply_user_permission_on', 'name', 'user_id_field'],
+		filters={'is_standard': 0})
+
+	if user_types:
+		user_type_details = {d.name: [d.apply_user_permission_on, d.user_id_field] for d in user_types}
+
+		frappe.cache().set_value('non_standard_user_types', user_type_details)
+
+		return user_type_details
+
+@frappe.whitelist()
+@frappe.validate_and_sanitize_search_inputs
+def get_user_linked_doctypes(doctype, txt, searchfield, start, page_len, filters):
+	modules = [d.get('module_name') for d in get_modules_from_app('frappe')]
+
+	filters = [['DocField', 'options', '=', 'User'], ['DocType', 'is_submittable', '=', 0],
+		['DocType', 'issingle', '=', 0], ['DocType', 'module', 'not in', modules],
+		['DocType', 'read_only', '=', 0], ['DocType', 'name', 'like', '%{0}%'.format(txt)]]
+
+	doctypes = frappe.get_all('DocType', fields = ['`tabDocType`.`name`'], filters=filters,
+		order_by = '`tabDocType`.`idx` desc', limit_start=start, limit_page_length=page_len, as_list=1, debug=1)
+
+	custom_dt_filters = [['Custom Field', 'dt', 'like', '%{0}%'.format(txt)],
+		['Custom Field', 'options', '=', 'User'], ['Custom Field', 'fieldtype', '=', 'Link']]
+
+	custom_doctypes = frappe.get_all('Custom Field', fields = ['dt as name'],
+		filters= custom_dt_filters, as_list=1)
+
+	return doctypes + custom_doctypes
+
+@frappe.whitelist()
+def get_user_id(parent):
+	data = frappe.get_all('DocField', fields = ['label', 'fieldname as value'],
+		filters= {'options': 'User', 'fieldtype': 'Link', 'parent': parent}) or []
+
+	data.extend(frappe.get_all('Custom Field', fields = ['label', 'fieldname as value'],
+		filters= {'options': 'User', 'fieldtype': 'Link', 'dt': parent}))
+
+	return data
+
+def user_linked_with_permission_on_doctype(doc, user):
+	if not doc.apply_user_permission_on:
+		return True
+
+	if not doc.user_id_field:
+		frappe.throw(_('User Id Field is mandatory in the user type {0}')
+			.format(frappe.bold(doc.name)))
+
+	if frappe.db.get_value(doc.apply_user_permission_on,
+		{doc.user_id_field: user}, 'name'):
+		return True
+	else:
+		label = frappe.get_meta(doc.apply_user_permission_on).get_field(doc.user_id_field).label
+
+		frappe.msgprint(_("To set the role {0} in the user {1}, kindly set the {2} field as {3} in one of the {4} record.")
+			.format(frappe.bold(doc.role), frappe.bold(user), frappe.bold(label),
+				frappe.bold(user), frappe.bold(doc.apply_user_permission_on)))
+
+		return False
+
+def apply_permissions_for_non_standard_user_type(doc, method=None):
+	'''Create user permission for the non standard user type'''
+	if not frappe.db.table_exists('User Type'):
+		return
+
+	user_types = frappe.cache().get_value('non_standard_user_types')
+
+	if not user_types:
+		user_types = get_non_standard_user_type_details()
+
+	if not user_types:
+		return
+
+	for user_type, data in iteritems(user_types):
+		if (not doc.get(data[1]) or doc.doctype != data[0]):
+			continue
+
+		if frappe.get_cached_value('User', doc.get(data[1]), 'user_type') != user_type:
+			return
+
+		if (doc.get(data[1]) and (not doc._doc_before_save or doc.get(data[1]) != doc._doc_before_save.get(data[1])
+			or not frappe.db.get_value('User Permission',
+				{'user': doc.get(data[1]), 'allow': data[0], 'for_value': doc.name}, 'name'))):
+
+			perm_data = frappe.db.get_value('User Permission',
+				{'allow': doc.doctype, 'for_value': doc.name}, ['name', 'user'])
+
+			if not perm_data:
+				user_doc = frappe.get_cached_doc('User', doc.get(data[1]))
+				user_doc.set_roles_and_modules_based_on_user_type()
+				user_doc.update_children()
+				add_user_permission(doc.doctype, doc.name, doc.get(data[1]))
+			else:
+				frappe.db.set_value('User Permission', perm_data[0], 'user', doc.get(data[1]))
--- a/frappe/core/doctype/user_type/user_type_dashboard.py
+++ b/frappe/core/doctype/user_type/user_type_dashboard.py
@ -0,0 +1,13 @@
+from __future__ import unicode_literals
+from frappe import _
+
+def get_data():
+	return {
+		'fieldname': 'user_type',
+		'transactions': [
+			{
+				'label': _('Reference'),
+				'items': ['User']
+			}
+		]
+	}
--- a/frappe/core/doctype/user_type/user_type_list.js
+++ b/frappe/core/doctype/user_type/user_type_list.js
@ -0,0 +1,10 @@
+frappe.listview_settings['User Type'] = {
+	add_fields: ["is_standard"],
+	get_indicator: function (doc) {
+		if (doc.is_standard) {
+			return [__("Standard"), "green", "is_standard,=,1"];
+		} else {
+			return [__("Custom"), "blue", "is_standard,=,0"];
+		}
+	}
+};
--- a/frappe/core/doctype/user_type_module/init.py
+++ b/frappe/core/doctype/user_type_module/init.py
--- a/frappe/core/doctype/user_type_module/user_type_module.json
+++ b/frappe/core/doctype/user_type_module/user_type_module.json
@ -0,0 +1,33 @@
+{
+ "actions": [],
+ "creation": "2021-01-24 03:05:24.634719",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "module"
+ ],
+ "fields": [
+  {
+   "fieldname": "module",
+   "fieldtype": "Link",
+   "in_list_view": 1,
+   "label": "Module",
+   "options": "Module Def",
+   "reqd": 1
+  }
+ ],
+ "index_web_pages_for_search": 1,
+ "istable": 1,
+ "links": [],
+ "modified": "2021-01-24 03:07:43.602927",
+ "modified_by": "Administrator",
+ "module": "Core",
+ "name": "User Type Module",
+ "owner": "Administrator",
+ "permissions": [],
+ "quick_entry": 1,
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "track_changes": 1
+}
--- a/frappe/core/doctype/user_type_module/user_type_module.py
+++ b/frappe/core/doctype/user_type_module/user_type_module.py
@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2021, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+# import frappe
+from frappe.model.document import Document
+
+class UserTypeModule(Document):
+	pass
--- a/frappe/core/doctype/version/test_version.py
+++ b/frappe/core/doctype/version/test_version.py
@ -9,6 +9,7 @@ from frappe.core.doctype.version.version import get_diff

 class TestVersion(unittest.TestCase):
 	def test_get_diff(self):
+		frappe.set_user('Administrator')
 		test_records = make_test_objects('Event', reset = True)
 		old_doc = frappe.get_doc("Event", test_records[0])
 		new_doc = copy.deepcopy(old_doc)
--- a/frappe/core/page/dashboard_view/dashboard_view.js
+++ b/frappe/core/page/dashboard_view/dashboard_view.js
@ -36,17 +36,17 @@ class Dashboard {
 		} else {
 			// last opened
 			if (frappe.last_dashboard) {
-				frappe.set_route('dashboard-view', frappe.last_dashboard);
+				frappe.set_re_route('dashboard-view', frappe.last_dashboard);
 			} else {
 				// default dashboard
 				frappe.db.get_list('Dashboard', {filters: {is_default: 1}}).then(data => {
 					if (data && data.length) {
-						frappe.set_route('dashboard-view', data[0].name);
+						frappe.set_re_route('dashboard-view', data[0].name);
 					} else {
 						// no default, get the latest one
 						frappe.db.get_list('Dashboard', {limit: 1}).then(data => {
 							if (data && data.length) {
-								frappe.set_route('dashboard-view', data[0].name);
+								frappe.set_re_route('dashboard-view', data[0].name);
 							} else {
 								// create a new dashboard!
 								frappe.new_doc('Dashboard');
--- a/Show more
+++ b/Show more