diff --git a/.eslintrc b/.eslintrc index c55acc5bac..cc7f555669 100644 --- a/.eslintrc +++ b/.eslintrc @@ -80,6 +80,7 @@ "validate_email": true, "validate_name": true, "validate_phone": true, + "validate_url": true, "get_number_format": true, "format_number": true, "format_currency": true, @@ -148,6 +149,7 @@ "before": true, "beforeEach": true, "qz": true, - "localforage": true + "localforage": true, + "extend_cscript": true } } diff --git a/.flake8 b/.flake8 index 399b176e1d..56c9b9a369 100644 --- a/.flake8 +++ b/.flake8 @@ -29,4 +29,5 @@ ignore = B950, W191, -max-line-length = 200 \ No newline at end of file +max-line-length = 200 +exclude=.github/helper/semgrep_rules diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 0000000000..4faece896a --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,12 @@ +# Since version 2.23 (released in August 2019), git-blame has a feature +# to ignore or bypass certain commits. +# +# This file contains a list of commits that are not likely what you +# are looking for in a blame, such as mass reformatting or renaming. +# You can set this file as a default ignore file for blame by running +# the following command. +# +# $ git config blame.ignoreRevsFile .git-blame-ignore-revs + +# Replace use of Class.extend with native JS class +fe20515c23a3ac41f1092bf0eaf0a0a452ec2e85 diff --git a/.github/helper/roulette.py b/.github/helper/roulette.py index ba775d6794..ea4f07b9f7 100644 --- a/.github/helper/roulette.py +++ b/.github/helper/roulette.py @@ -18,7 +18,7 @@ def is_js(file): return file.endswith("js") def is_docs(file): - regex = re.compile('\.(md|png|jpg|jpeg)$|^.github|LICENSE') + regex = re.compile(r'\.(md|png|jpg|jpeg)$|^.github|LICENSE') return bool(regex.search(file)) diff --git a/.github/helper/semgrep_rules/frappe_correctness.py b/.github/helper/semgrep_rules/frappe_correctness.py index 37889fbbb1..745e6463b8 100644 --- a/.github/helper/semgrep_rules/frappe_correctness.py +++ b/.github/helper/semgrep_rules/frappe_correctness.py @@ -4,25 +4,61 @@ from frappe import _, flt from frappe.model.document import Document +# ruleid: frappe-modifying-but-not-comitting def on_submit(self): if self.value_of_goods == 0: frappe.throw(_('Value of goods cannot be 0')) - # ruleid: frappe-modifying-after-submit self.status = 'Submitted' -def on_submit(self): # noqa - if flt(self.per_billed) < 100: - self.update_billing_status() - else: - # todook: frappe-modifying-after-submit - self.status = "Completed" - self.db_set("status", "Completed") -class TestDoc(Document): - pass +# ok: frappe-modifying-but-not-comitting +def on_submit(self): + if self.value_of_goods == 0: + frappe.throw(_('Value of goods cannot be 0')) + self.status = 'Submitted' + self.db_set('status', 'Submitted') - def validate(self): - #ruleid: frappe-modifying-child-tables-while-iterating - for item in self.child_table: - if item.value < 0: - self.remove(item) +# ok: frappe-modifying-but-not-comitting +def on_submit(self): + if self.value_of_goods == 0: + frappe.throw(_('Value of goods cannot be 0')) + x = "y" + self.status = x + self.db_set('status', x) + + +# ok: frappe-modifying-but-not-comitting +def on_submit(self): + x = "y" + self.status = x + self.save() + +# ruleid: frappe-modifying-but-not-comitting-other-method +class DoctypeClass(Document): + def on_submit(self): + self.good_method() + self.tainted_method() + + def tainted_method(self): + self.status = "uptate" + + +# ok: frappe-modifying-but-not-comitting-other-method +class DoctypeClass(Document): + def on_submit(self): + self.good_method() + self.tainted_method() + + def tainted_method(self): + self.status = "update" + self.db_set("status", "update") + +# ok: frappe-modifying-but-not-comitting-other-method +class DoctypeClass(Document): + def on_submit(self): + self.good_method() + self.tainted_method() + self.save() + + def tainted_method(self): + self.status = "uptate" diff --git a/.github/helper/semgrep_rules/translate.js b/.github/helper/semgrep_rules/translate.js index 7b92fe2dff..9cdfb75d0b 100644 --- a/.github/helper/semgrep_rules/translate.js +++ b/.github/helper/semgrep_rules/translate.js @@ -35,3 +35,10 @@ __('You have' + 'subscribers in your mailing list.') // ruleid: frappe-translation-js-splitting __('You have {0} subscribers' + 'in your mailing list', [subscribers.length]) + +// ok: frappe-translation-js-splitting +__("Ctrl+Enter to add comment") + +// ruleid: frappe-translation-js-splitting +__('You have {0} subscribers \ + in your mailing list', [subscribers.length]) diff --git a/.github/helper/semgrep_rules/translate.py b/.github/helper/semgrep_rules/translate.py index bd6cd9126c..9de6aa94f0 100644 --- a/.github/helper/semgrep_rules/translate.py +++ b/.github/helper/semgrep_rules/translate.py @@ -51,3 +51,11 @@ _(f"what" + f"this is also not cool") _("") # ruleid: frappe-translation-empty-string _('') + + +class Test: + # ok: frappe-translation-python-splitting + def __init__( + args + ): + pass diff --git a/.github/helper/semgrep_rules/translate.yml b/.github/helper/semgrep_rules/translate.yml index df55089b9f..5f03fb9fd0 100644 --- a/.github/helper/semgrep_rules/translate.yml +++ b/.github/helper/semgrep_rules/translate.yml @@ -42,10 +42,10 @@ rules: - id: frappe-translation-python-splitting pattern-either: - - pattern: _(...) + ... + _(...) + - pattern: _(...) + _(...) - pattern: _("..." + "...") - - pattern-regex: '_\([^\)]*\\\s*' # lines broken by `\` - - pattern-regex: '_\(\s*\n' # line breaks allowed by python for using ( ) + - pattern-regex: '[\s\.]_\([^\)]*\\\s*' # lines broken by `\` + - pattern-regex: '[\s\.]_\(\s*\n' # line breaks allowed by python for using ( ) message: | Do not split strings inside translate function. Do not concatenate using translate functions. Please refer: https://frappeframework.com/docs/user/en/translations @@ -54,8 +54,8 @@ rules: - id: frappe-translation-js-splitting pattern-either: - - pattern-regex: '__\([^\)]*[\+\\]\s*' - - pattern: __('...' + '...') + - pattern-regex: '__\([^\)]*[\\]\s+' + - pattern: __('...' + '...', ...) - pattern: __('...') + __('...') message: | Do not split strings inside translate function. Do not concatenate using translate functions. diff --git a/.github/helper/semgrep_rules/ux.js b/.github/helper/semgrep_rules/ux.js new file mode 100644 index 0000000000..ae73f9cc60 --- /dev/null +++ b/.github/helper/semgrep_rules/ux.js @@ -0,0 +1,9 @@ + +// ok: frappe-missing-translate-function-js +frappe.msgprint('{{ _("Both login and password required") }}'); + +// ruleid: frappe-missing-translate-function-js +frappe.msgprint('What'); + +// ok: frappe-missing-translate-function-js +frappe.throw(' {{ _("Both login and password required") }}. '); diff --git a/.github/helper/semgrep_rules/ux.py b/.github/helper/semgrep_rules/ux.py index 4a74457435..a00d3cd8ae 100644 --- a/.github/helper/semgrep_rules/ux.py +++ b/.github/helper/semgrep_rules/ux.py @@ -2,30 +2,30 @@ import frappe from frappe import msgprint, throw, _ -# ruleid: frappe-missing-translate-function +# ruleid: frappe-missing-translate-function-python throw("Error Occured") -# ruleid: frappe-missing-translate-function +# ruleid: frappe-missing-translate-function-python frappe.throw("Error Occured") -# ruleid: frappe-missing-translate-function +# ruleid: frappe-missing-translate-function-python frappe.msgprint("Useful message") -# ruleid: frappe-missing-translate-function +# ruleid: frappe-missing-translate-function-python msgprint("Useful message") -# ok: frappe-missing-translate-function +# ok: frappe-missing-translate-function-python translatedmessage = _("Hello") -# ok: frappe-missing-translate-function +# ok: frappe-missing-translate-function-python throw(translatedmessage) -# ok: frappe-missing-translate-function +# ok: frappe-missing-translate-function-python msgprint(translatedmessage) -# ok: frappe-missing-translate-function +# ok: frappe-missing-translate-function-python msgprint(_("Helpful message")) -# ok: frappe-missing-translate-function +# ok: frappe-missing-translate-function-python frappe.throw(_("Error occured")) diff --git a/.github/helper/semgrep_rules/ux.yml b/.github/helper/semgrep_rules/ux.yml index ed06a6a80c..dd667f36c0 100644 --- a/.github/helper/semgrep_rules/ux.yml +++ b/.github/helper/semgrep_rules/ux.yml @@ -1,15 +1,30 @@ rules: -- id: frappe-missing-translate-function +- id: frappe-missing-translate-function-python pattern-either: - patterns: - pattern: frappe.msgprint("...", ...) - pattern-not: frappe.msgprint(_("..."), ...) - - pattern-not: frappe.msgprint(__("..."), ...) - patterns: - pattern: frappe.throw("...", ...) - pattern-not: frappe.throw(_("..."), ...) - - pattern-not: frappe.throw(__("..."), ...) message: | All user facing text must be wrapped in translate function. Please refer to translation documentation. https://frappeframework.com/docs/user/en/guides/basics/translations - languages: [python, javascript, json] + languages: [python] + severity: ERROR + +- id: frappe-missing-translate-function-js + pattern-either: + - patterns: + - pattern: frappe.msgprint("...", ...) + - pattern-not: frappe.msgprint(__("..."), ...) + # ignore microtemplating e.g. msgprint("{{ _("server side translation") }}") + - pattern-not: frappe.msgprint("=~/\{\{.*\_.*\}\}/i", ...) + - patterns: + - pattern: frappe.throw("...", ...) + - pattern-not: frappe.throw(__("..."), ...) + # ignore microtemplating + - pattern-not: frappe.throw("=~/\{\{.*\_.*\}\}/i", ...) + message: | + All user facing text must be wrapped in translate function. Please refer to translation documentation. https://frappeframework.com/docs/user/en/guides/basics/translations + languages: [javascript] severity: ERROR diff --git a/.github/workflows/patch-mariadb-tests.yml b/.github/workflows/patch-mariadb-tests.yml new file mode 100644 index 0000000000..e8627a01fb --- /dev/null +++ b/.github/workflows/patch-mariadb-tests.yml @@ -0,0 +1,83 @@ +name: Patch + +on: [pull_request, workflow_dispatch] + +jobs: + test: + runs-on: ubuntu-18.04 + + name: Patch Test + + services: + mysql: + image: mariadb:10.3 + env: + MYSQL_ALLOW_EMPTY_PASSWORD: YES + ports: + - 3306:3306 + options: --health-cmd="mysqladmin ping" --health-interval=5s --health-timeout=2s --health-retries=3 + + steps: + - name: Clone + uses: actions/checkout@v2 + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.7 + + - name: Add to Hosts + run: echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts + + - name: Cache pip + uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + ${{ runner.os }}- + + - name: Cache node modules + uses: actions/cache@v2 + env: + cache-name: cache-node-modules + with: + path: ~/.npm + key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-build-${{ env.cache-name }}- + ${{ runner.os }}-build- + ${{ runner.os }}- + + - name: Get yarn cache directory path + id: yarn-cache-dir-path + run: echo "::set-output name=dir::$(yarn cache dir)" + + - uses: actions/cache@v2 + id: yarn-cache + with: + path: ${{ steps.yarn-cache-dir-path.outputs.dir }} + key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} + restore-keys: | + ${{ runner.os }}-yarn- + + - name: Install Dependencies + run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh + env: + BEFORE: ${{ env.GITHUB_EVENT_PATH.before }} + AFTER: ${{ env.GITHUB_EVENT_PATH.after }} + TYPE: server + + - name: Install + run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh + env: + DB: mariadb + TYPE: server + + - name: Run Patch Tests + run: | + cd ~/frappe-bench/ + wget https://frappeframework.com/files/v10-frappe.sql.gz + bench --site test_site --force restore ~/frappe-bench/v10-frappe.sql.gz + bench --site test_site migrate diff --git a/.github/workflows/publish-assets-develop.yml b/.github/workflows/publish-assets-develop.yml index 2a934a6795..a23885b508 100644 --- a/.github/workflows/publish-assets-develop.yml +++ b/.github/workflows/publish-assets-develop.yml @@ -15,11 +15,11 @@ jobs: path: 'frappe' - uses: actions/setup-node@v1 with: - python-version: '12.x' + node-version: 14 - uses: actions/setup-python@v2 with: python-version: '3.6' - - name: Set up bench for current push + - name: Set up bench and build assets run: | npm install -g yarn pip3 install -U frappe-bench @@ -29,7 +29,7 @@ jobs: - name: Package assets run: | mkdir -p $GITHUB_WORKSPACE/build - tar -cvpzf $GITHUB_WORKSPACE/build/$GITHUB_SHA.tar.gz ./frappe-bench/sites/assets/js ./frappe-bench/sites/assets/css + tar -cvpzf $GITHUB_WORKSPACE/build/$GITHUB_SHA.tar.gz ./frappe-bench/sites/assets/frappe/dist - name: Publish assets to S3 uses: jakejarvis/s3-sync-action@master diff --git a/.github/workflows/publish-assets-releases.yml b/.github/workflows/publish-assets-releases.yml index e86f884f35..a697517c23 100644 --- a/.github/workflows/publish-assets-releases.yml +++ b/.github/workflows/publish-assets-releases.yml @@ -22,7 +22,7 @@ jobs: - uses: actions/setup-python@v2 with: python-version: '3.6' - - name: Set up bench for current push + - name: Set up bench and build assets run: | npm install -g yarn pip3 install -U frappe-bench @@ -32,7 +32,7 @@ jobs: - name: Package assets run: | mkdir -p $GITHUB_WORKSPACE/build - tar -cvpzf $GITHUB_WORKSPACE/build/assets.tar.gz ./frappe-bench/sites/assets/js ./frappe-bench/sites/assets/css + tar -cvpzf $GITHUB_WORKSPACE/build/assets.tar.gz ./frappe-bench/sites/assets/frappe/dist - name: Get release id: get_release diff --git a/.github/workflows/semgrep.yml b/.github/workflows/semgrep.yml index 5092bf4705..389524e968 100644 --- a/.github/workflows/semgrep.yml +++ b/.github/workflows/semgrep.yml @@ -4,6 +4,8 @@ on: pull_request: branches: - develop + - version-13-hotfix + - version-13-pre-release jobs: semgrep: name: Frappe Linter diff --git a/.github/workflows/server-mariadb-tests.yml b/.github/workflows/server-mariadb-tests.yml new file mode 100644 index 0000000000..2476102e3d --- /dev/null +++ b/.github/workflows/server-mariadb-tests.yml @@ -0,0 +1,130 @@ +name: Server + +on: + pull_request: + workflow_dispatch: + push: + branches: [ develop ] + +jobs: + test: + runs-on: ubuntu-18.04 + + strategy: + fail-fast: false + matrix: + container: [1, 2] + + name: Python Unit Tests (MariaDB) + + services: + mysql: + image: mariadb:10.3 + env: + MYSQL_ALLOW_EMPTY_PASSWORD: YES + ports: + - 3306:3306 + options: --health-cmd="mysqladmin ping" --health-interval=5s --health-timeout=2s --health-retries=3 + + steps: + - name: Clone + uses: actions/checkout@v2 + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.7 + + - uses: actions/setup-node@v2 + with: + node-version: 14 + check-latest: true + + - name: Add to Hosts + run: | + echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts + echo "127.0.0.1 test_site_producer" | sudo tee -a /etc/hosts + + - name: Cache pip + uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + ${{ runner.os }}- + + - name: Cache node modules + uses: actions/cache@v2 + env: + cache-name: cache-node-modules + with: + path: ~/.npm + key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-build-${{ env.cache-name }}- + ${{ runner.os }}-build- + ${{ runner.os }}- + + - name: Get yarn cache directory path + id: yarn-cache-dir-path + run: echo "::set-output name=dir::$(yarn cache dir)" + + - uses: actions/cache@v2 + id: yarn-cache + with: + path: ${{ steps.yarn-cache-dir-path.outputs.dir }} + key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} + restore-keys: | + ${{ runner.os }}-yarn- + + - name: Install Dependencies + run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh + env: + BEFORE: ${{ env.GITHUB_EVENT_PATH.before }} + AFTER: ${{ env.GITHUB_EVENT_PATH.after }} + TYPE: server + + - name: Install + run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh + env: + DB: mariadb + TYPE: server + + - name: Run Tests + run: cd ~/frappe-bench/ && bench --site test_site run-parallel-tests --use-orchestrator --with-coverage + env: + CI_BUILD_ID: ${{ github.run_id }} + ORCHESTRATOR_URL: http://test-orchestrator.frappe.io + + - name: Upload Coverage Data + run: | + cp ~/frappe-bench/sites/.coverage ${GITHUB_WORKSPACE} + cd ${GITHUB_WORKSPACE} + pip3 install coverage==5.5 + pip3 install coveralls==3.0.1 + coveralls + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_TOKEN }} + COVERALLS_FLAG_NAME: run-${{ matrix.container }} + COVERALLS_SERVICE_NAME: ${{ github.event_name == 'pull_request' && 'github' || 'github-actions' }} + COVERALLS_PARALLEL: true + + coveralls: + name: Coverage Wrap Up + needs: test + container: python:3-slim + runs-on: ubuntu-18.04 + steps: + - name: Clone + uses: actions/checkout@v2 + + - name: Coveralls Finished + run: | + cd ${GITHUB_WORKSPACE} + pip3 install coverage==5.5 + pip3 install coveralls==3.0.1 + coveralls --finish + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/server-postgres-tests.yml b/.github/workflows/server-postgres-tests.yml new file mode 100644 index 0000000000..4325eebaad --- /dev/null +++ b/.github/workflows/server-postgres-tests.yml @@ -0,0 +1,100 @@ +name: Server + +on: + pull_request: + workflow_dispatch: + +jobs: + test: + runs-on: ubuntu-18.04 + + strategy: + fail-fast: false + matrix: + container: [1, 2] + + name: Python Unit Tests (Postgres) + + services: + postgres: + image: postgres:12.4 + env: + POSTGRES_PASSWORD: travis + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + + steps: + - name: Clone + uses: actions/checkout@v2 + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.7 + + - uses: actions/setup-node@v2 + with: + node-version: '14' + check-latest: true + + - name: Add to Hosts + run: | + echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts + echo "127.0.0.1 test_site_producer" | sudo tee -a /etc/hosts + + - name: Cache pip + uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + ${{ runner.os }}- + + - name: Cache node modules + uses: actions/cache@v2 + env: + cache-name: cache-node-modules + with: + path: ~/.npm + key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-build-${{ env.cache-name }}- + ${{ runner.os }}-build- + ${{ runner.os }}- + + - name: Get yarn cache directory path + id: yarn-cache-dir-path + run: echo "::set-output name=dir::$(yarn cache dir)" + + - uses: actions/cache@v2 + id: yarn-cache + with: + path: ${{ steps.yarn-cache-dir-path.outputs.dir }} + key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} + restore-keys: | + ${{ runner.os }}-yarn- + + - name: Install Dependencies + run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh + env: + BEFORE: ${{ env.GITHUB_EVENT_PATH.before }} + AFTER: ${{ env.GITHUB_EVENT_PATH.after }} + TYPE: server + + - name: Install + run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh + env: + DB: postgres + TYPE: server + + - name: Run Tests + run: cd ~/frappe-bench/ && bench --site test_site run-parallel-tests --use-orchestrator + env: + CI_BUILD_ID: ${{ github.run_id }} + ORCHESTRATOR_URL: http://test-orchestrator.frappe.io diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ui-tests.yml similarity index 57% rename from .github/workflows/ci-tests.yml rename to .github/workflows/ui-tests.yml index e21a1f7ac6..f2f43f10f8 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ui-tests.yml @@ -1,10 +1,10 @@ -name: CI +name: UI on: pull_request: - types: [opened, synchronize, reopened, labeled, unlabeled] workflow_dispatch: push: + branches: [ develop ] jobs: test: @@ -13,23 +13,9 @@ jobs: strategy: fail-fast: false matrix: - include: - - DB: "mariadb" - TYPE: "server" - JOB_NAME: "Python MariaDB" - RUN_COMMAND: bench --site test_site run-tests --coverage + containers: [1, 2] - - DB: "postgres" - TYPE: "server" - JOB_NAME: "Python PostgreSQL" - RUN_COMMAND: bench --site test_site run-tests --coverage - - - DB: "mariadb" - TYPE: "ui" - JOB_NAME: "UI MariaDB" - RUN_COMMAND: bench --site test_site run-ui-tests frappe --headless - - name: ${{ matrix.JOB_NAME }} + name: UI Tests (Cypress) services: mysql: @@ -40,18 +26,6 @@ jobs: - 3306:3306 options: --health-cmd="mysqladmin ping" --health-interval=5s --health-timeout=2s --health-retries=3 - postgres: - image: postgres:12.4 - env: - POSTGRES_PASSWORD: travis - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 5432:5432 - steps: - name: Clone uses: actions/checkout@v2 @@ -63,7 +37,7 @@ jobs: - uses: actions/setup-node@v2 with: - node-version: '12' + node-version: 14 check-latest: true - name: Add to Hosts @@ -105,7 +79,6 @@ jobs: ${{ runner.os }}-yarn- - name: Cache cypress binary - if: matrix.TYPE == 'ui' uses: actions/cache@v2 with: path: ~/.cache @@ -119,40 +92,16 @@ jobs: env: BEFORE: ${{ env.GITHUB_EVENT_PATH.before }} AFTER: ${{ env.GITHUB_EVENT_PATH.after }} - TYPE: ${{ matrix.TYPE }} + TYPE: ui - name: Install run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh env: - DB: ${{ matrix.DB }} - TYPE: ${{ matrix.TYPE }} + DB: mariadb + TYPE: ui - - name: Run Set-Up - if: matrix.TYPE == 'ui' + - name: Site Setup run: cd ~/frappe-bench/ && bench --site test_site execute frappe.utils.install.complete_setup_wizard - env: - DB: ${{ matrix.DB }} - TYPE: ${{ matrix.TYPE }} - - name: Setup tmate session - if: contains(github.event.pull_request.labels.*.name, 'debug-gha') - uses: mxschmitt/action-tmate@v3 - - - name: Run Tests - run: cd ~/frappe-bench/ && ${{ matrix.RUN_COMMAND }} - env: - DB: ${{ matrix.DB }} - TYPE: ${{ matrix.TYPE }} - - - name: Coverage - if: matrix.TYPE == 'server' - run: | - cp ~/frappe-bench/sites/.coverage ${GITHUB_WORKSPACE} - cd ${GITHUB_WORKSPACE} - pip install coveralls==2.2.0 - pip install coverage==4.5.4 - coveralls --service=github - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_TOKEN }} - COVERALLS_SERVICE_NAME: github + - name: UI Tests + run: cd ~/frappe-bench/ && bench --site test_site run-ui-tests frappe --headless --parallel --ci-build-id $GITHUB_RUN_ID diff --git a/.gitignore b/.gitignore index 766288fe2e..1ff3122d70 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ locale dist/ # build/ frappe/docs/current +frappe/public/dist .vscode node_modules .kdev4/ diff --git a/.mergify.yml b/.mergify.yml index 82f710a5a8..c759c1e3ec 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -3,9 +3,12 @@ pull_request_rules: conditions: - status-success=Sider - status-success=Semantic Pull Request - - status-success=Python MariaDB - - status-success=Python PostgreSQL - - status-success=UI MariaDB + - status-success=Python Unit Tests (MariaDB) (1) + - status-success=Python Unit Tests (MariaDB) (2) + - status-success=Python Unit Tests (Postgres) (1) + - status-success=Python Unit Tests (Postgres) (2) + - status-success=UI Tests (Cypress) (1) + - status-success=UI Tests (Cypress) (2) - status-success=security/snyk (frappe) - label!=dont-merge - label!=squash @@ -16,9 +19,12 @@ pull_request_rules: - name: Automatic squash on CI success and review conditions: - status-success=Sider - - status-success=Python MariaDB - - status-success=Python PostgreSQL - - status-success=UI MariaDB + - status-success=Python Unit Tests (MariaDB) (1) + - status-success=Python Unit Tests (MariaDB) (2) + - status-success=Python Unit Tests (Postgres) (1) + - status-success=Python Unit Tests (Postgres) (2) + - status-success=UI Tests (Cypress) (1) + - status-success=UI Tests (Cypress) (2) - status-success=security/snyk (frappe) - label!=dont-merge - label=squash diff --git a/README.md b/README.md index e00bea7857..11343a632a 100644 --- a/README.md +++ b/README.md @@ -14,18 +14,21 @@
- - - - - - + + + + + + + + + - - - + + +
diff --git a/cypress/fixtures/data_field_validation_doctype.js b/cypress/fixtures/data_field_validation_doctype.js new file mode 100644 index 0000000000..da091af7e5 --- /dev/null +++ b/cypress/fixtures/data_field_validation_doctype.js @@ -0,0 +1,65 @@ +export default { + name: 'Validation Test', + custom: 1, + actions: [], + creation: '2019-03-15 06:29:07.215072', + doctype: 'DocType', + editable_grid: 1, + engine: 'InnoDB', + fields: [ + { + fieldname: 'email', + fieldtype: 'Data', + label: 'Email', + options: 'Email' + }, + { + fieldname: 'URL', + fieldtype: 'Data', + label: 'URL', + options: 'URL' + }, + { + fieldname: 'Phone', + fieldtype: 'Data', + label: 'Phone', + options: 'Phone' + }, + { + fieldname: 'person_name', + fieldtype: 'Data', + label: 'Person Name', + options: 'Name' + }, + { + fieldname: 'read_only_url', + fieldtype: 'Data', + label: 'Read Only URL', + options: 'URL', + read_only: '1', + default: 'https://frappe.io' + } + ], + issingle: 1, + links: [], + modified: '2021-04-19 14:40:53.127615', + modified_by: 'Administrator', + module: 'Custom', + owner: 'Administrator', + permissions: [ + { + create: 1, + delete: 1, + email: 1, + print: 1, + read: 1, + role: 'System Manager', + share: 1, + write: 1 + } + ], + quick_entry: 1, + sort_field: 'modified', + sort_order: 'ASC', + track_changes: 1 +}; diff --git a/cypress/integration/data_field_form_validation.js b/cypress/integration/data_field_form_validation.js new file mode 100644 index 0000000000..c6feea5550 --- /dev/null +++ b/cypress/integration/data_field_form_validation.js @@ -0,0 +1,43 @@ +import data_field_validation_doctype from '../fixtures/data_field_validation_doctype'; +const doctype_name = data_field_validation_doctype.name; + + +context('Data Field Input Validation in New Form', () => { + before(() => { + cy.login(); + cy.visit('/app/website'); + return cy.insert_doc('DocType', data_field_validation_doctype, true); + }); + + function validateField(fieldname, invalid_value, valid_value) { + // Invalid, should have has-error class + cy.get_field(fieldname).clear().type(invalid_value).blur(); + cy.get(`.frappe-control[data-fieldname="${fieldname}"]`).should('have.class', 'has-error'); + // Valid value, should not have has-error class + cy.get_field(fieldname).clear().type(valid_value); + cy.get(`.frappe-control[data-fieldname="${fieldname}"]`).should('not.have.class', 'has-error'); + } + + describe('Data Field Options', () => { + it('should validate email address', () => { + cy.new_form(doctype_name); + validateField('email', 'captian', 'hello@test.com'); + }); + + it('should validate URL', () => { + validateField('url', 'jkl', 'https://frappe.io'); + validateField('url', 'abcd.com', 'http://google.com/home'); + validateField('url', '&&http://google.uae', 'gopher://frappe.io'); + validateField('url', 'ftt2:://google.in?q=news', 'ftps2://frappe.io/__/#home'); + validateField('url', 'ftt2://', 'ntps://localhost'); // For intranet URLs + }); + + it('should validate phone number', () => { + validateField('phone', 'america', '89787878'); + }); + + it('should validate name', () => { + validateField('person_name', ' 777Hello', 'James Bond'); + }); + }); +}); \ No newline at end of file diff --git a/cypress/integration/recorder.js b/cypress/integration/recorder.js index d30cc3568c..5b7692d8ff 100644 --- a/cypress/integration/recorder.js +++ b/cypress/integration/recorder.js @@ -50,7 +50,7 @@ context('Recorder', () => { cy.get('.result-list').should('contain', '/api/method/frappe.desk.reportview.get'); }); - it.only('Recorder View Request', () => { + it('Recorder View Request', () => { cy.get('.primary-action').should('contain', 'Start').click(); cy.visit('/app/List/DocType/List'); diff --git a/cypress/integration/url_data_field.js b/cypress/integration/url_data_field.js new file mode 100644 index 0000000000..cf22c62363 --- /dev/null +++ b/cypress/integration/url_data_field.js @@ -0,0 +1,43 @@ +import data_field_validation_doctype from '../fixtures/data_field_validation_doctype'; + +const doctype_name = data_field_validation_doctype.name; + +context('URL Data Field Input', () => { + before(() => { + cy.login(); + cy.visit('/app/website'); + return cy.insert_doc('DocType', data_field_validation_doctype, true); + }); + + + describe('URL Data Field Input ', () => { + it('should not show URL link button without focus', () => { + cy.new_form(doctype_name); + cy.get_field('url').clear().type('https://frappe.io'); + cy.get_field('url').blur().wait(500); + cy.get('.link-btn').should('not.be.visible'); + }); + + it('should show URL link button on focus', () => { + cy.get_field('url').focus().wait(500); + cy.get('.link-btn').should('be.visible'); + }); + + it('should not show URL link button for invalid URL', () => { + cy.get_field('url').clear().type('fuzzbuzz'); + cy.get('.link-btn').should('not.be.visible'); + }); + + it('should have valid URL link with target _blank', () => { + cy.get_field('url').clear().type('https://frappe.io'); + cy.get('.link-btn .btn-open').should('have.attr', 'href', 'https://frappe.io'); + cy.get('.link-btn .btn-open').should('have.attr', 'target', '_blank'); + }); + + it('should inject anchor tag in read-only URL data field', () => { + cy.get('[data-fieldname="read_only_url"]') + .find('a') + .should('have.attr', 'target', '_blank'); + }); + }); +}); \ No newline at end of file diff --git a/esbuild/esbuild.js b/esbuild/esbuild.js new file mode 100644 index 0000000000..5154adb634 --- /dev/null +++ b/esbuild/esbuild.js @@ -0,0 +1,481 @@ +/* eslint-disable no-console */ +let path = require("path"); +let fs = require("fs"); +let glob = require("fast-glob"); +let esbuild = require("esbuild"); +let vue = require("esbuild-vue"); +let yargs = require("yargs"); +let cliui = require("cliui")(); +let chalk = require("chalk"); +let html_plugin = require("./frappe-html"); +let postCssPlugin = require("esbuild-plugin-postcss2").default; +let ignore_assets = require("./ignore-assets"); +let sass_options = require("./sass_options"); +let { + app_list, + assets_path, + apps_path, + sites_path, + get_app_path, + get_public_path, + log, + log_warn, + log_error, + bench_path, + get_redis_subscriber +} = require("./utils"); + +let argv = yargs + .usage("Usage: node esbuild [options]") + .option("apps", { + type: "string", + description: "Run build for specific apps" + }) + .option("skip_frappe", { + type: "boolean", + description: "Skip building frappe assets" + }) + .option("files", { + type: "string", + description: "Run build for specified bundles" + }) + .option("watch", { + type: "boolean", + description: "Run in watch mode and rebuild on file changes" + }) + .option("production", { + type: "boolean", + description: "Run build in production mode" + }) + .option("run-build-command", { + type: "boolean", + description: "Run build command for apps" + }) + .example( + "node esbuild --apps frappe,erpnext", + "Run build only for frappe and erpnext" + ) + .example( + "node esbuild --files frappe/website.bundle.js,frappe/desk.bundle.js", + "Run build only for specified bundles" + ) + .version(false).argv; + +const APPS = (!argv.apps ? app_list : argv.apps.split(",")).filter( + app => !(argv.skip_frappe && app == "frappe") +); +const FILES_TO_BUILD = argv.files ? argv.files.split(",") : []; +const WATCH_MODE = Boolean(argv.watch); +const PRODUCTION = Boolean(argv.production); +const RUN_BUILD_COMMAND = !WATCH_MODE && Boolean(argv["run-build-command"]); + +const TOTAL_BUILD_TIME = `${chalk.black.bgGreen(" DONE ")} Total Build Time`; +const NODE_PATHS = [].concat( + // node_modules of apps directly importable + app_list + .map(app => path.resolve(get_app_path(app), "../node_modules")) + .filter(fs.existsSync), + // import js file of any app if you provide the full path + app_list + .map(app => path.resolve(get_app_path(app), "..")) + .filter(fs.existsSync) +); + +execute() + .then(() => RUN_BUILD_COMMAND && run_build_command_for_apps(APPS)) + .catch(e => console.error(e)); + +if (WATCH_MODE) { + // listen for open files in editor event + open_in_editor(); +} + +async function execute() { + console.time(TOTAL_BUILD_TIME); + if (!FILES_TO_BUILD.length) { + await clean_dist_folders(APPS); + } + + let result; + try { + result = await build_assets_for_apps(APPS, FILES_TO_BUILD); + } catch (e) { + log_error("There were some problems during build"); + log(); + log(chalk.dim(e.stack)); + return; + } + + if (!WATCH_MODE) { + log_built_assets(result.metafile); + console.timeEnd(TOTAL_BUILD_TIME); + log(); + } else { + log("Watching for changes..."); + } + return await write_assets_json(result.metafile); +} + +function build_assets_for_apps(apps, files) { + let { include_patterns, ignore_patterns } = files.length + ? get_files_to_build(files) + : get_all_files_to_build(apps); + + return glob(include_patterns, { ignore: ignore_patterns }).then(files => { + let output_path = assets_path; + + let file_map = {}; + for (let file of files) { + let relative_app_path = path.relative(apps_path, file); + let app = relative_app_path.split(path.sep)[0]; + + let extension = path.extname(file); + let output_name = path.basename(file, extension); + if ( + [".css", ".scss", ".less", ".sass", ".styl"].includes(extension) + ) { + output_name = path.join("css", output_name); + } else if ([".js", ".ts"].includes(extension)) { + output_name = path.join("js", output_name); + } + output_name = path.join(app, "dist", output_name); + + if (Object.keys(file_map).includes(output_name)) { + log_warn( + `Duplicate output file ${output_name} generated from ${file}` + ); + } + + file_map[output_name] = file; + } + + return build_files({ + files: file_map, + outdir: output_path + }); + }); +} + +function get_all_files_to_build(apps) { + let include_patterns = []; + let ignore_patterns = []; + + for (let app of apps) { + let public_path = get_public_path(app); + include_patterns.push( + path.resolve( + public_path, + "**", + "*.bundle.{js,ts,css,sass,scss,less,styl}" + ) + ); + ignore_patterns.push( + path.resolve(public_path, "node_modules"), + path.resolve(public_path, "dist") + ); + } + + return { + include_patterns, + ignore_patterns + }; +} + +function get_files_to_build(files) { + // files: ['frappe/website.bundle.js', 'erpnext/main.bundle.js'] + let include_patterns = []; + let ignore_patterns = []; + + for (let file of files) { + let [app, bundle] = file.split("/"); + let public_path = get_public_path(app); + include_patterns.push(path.resolve(public_path, "**", bundle)); + ignore_patterns.push( + path.resolve(public_path, "node_modules"), + path.resolve(public_path, "dist") + ); + } + + return { + include_patterns, + ignore_patterns + }; +} + +function build_files({ files, outdir }) { + return esbuild.build({ + entryPoints: files, + entryNames: "[dir]/[name].[hash]", + outdir, + sourcemap: true, + bundle: true, + metafile: true, + minify: PRODUCTION, + nodePaths: NODE_PATHS, + define: { + "process.env.NODE_ENV": JSON.stringify( + PRODUCTION ? "production" : "development" + ) + }, + plugins: [ + html_plugin, + ignore_assets, + vue(), + postCssPlugin({ + plugins: [require("autoprefixer")], + sassOptions: sass_options + }) + ], + watch: get_watch_config() + }); +} + +function get_watch_config() { + if (WATCH_MODE) { + return { + async onRebuild(error, result) { + if (error) { + log_error("There was an error during rebuilding changes."); + log(); + log(chalk.dim(error.stack)); + notify_redis({ error }); + } else { + let { + assets_json, + prev_assets_json + } = await write_assets_json(result.metafile); + if (prev_assets_json) { + log_rebuilt_assets(prev_assets_json, assets_json); + } + notify_redis({ success: true }); + } + } + }; + } + return null; +} + +async function clean_dist_folders(apps) { + for (let app of apps) { + let public_path = get_public_path(app); + await fs.promises.rmdir(path.resolve(public_path, "dist", "js"), { + recursive: true + }); + await fs.promises.rmdir(path.resolve(public_path, "dist", "css"), { + recursive: true + }); + } +} + +function log_built_assets(metafile) { + let column_widths = [60, 20]; + cliui.div( + { + text: chalk.cyan.bold("File"), + width: column_widths[0] + }, + { + text: chalk.cyan.bold("Size"), + width: column_widths[1] + } + ); + cliui.div(""); + + let output_by_dist_path = {}; + for (let outfile in metafile.outputs) { + if (outfile.endsWith(".map")) continue; + let data = metafile.outputs[outfile]; + outfile = path.resolve(outfile); + outfile = path.relative(assets_path, outfile); + let filename = path.basename(outfile); + let dist_path = outfile.replace(filename, ""); + output_by_dist_path[dist_path] = output_by_dist_path[dist_path] || []; + output_by_dist_path[dist_path].push({ + name: filename, + size: (data.bytes / 1000).toFixed(2) + " Kb" + }); + } + + for (let dist_path in output_by_dist_path) { + let files = output_by_dist_path[dist_path]; + cliui.div({ + text: dist_path, + width: column_widths[0] + }); + + for (let i in files) { + let file = files[i]; + let branch = ""; + if (i < files.length - 1) { + branch = "├─ "; + } else { + branch = "└─ "; + } + let color = file.name.endsWith(".js") ? "green" : "blue"; + cliui.div( + { + text: branch + chalk[color]("" + file.name), + width: column_widths[0] + }, + { + text: file.size, + width: column_widths[1] + } + ); + } + cliui.div(""); + } + log(cliui.toString()); +} + +// to store previous build's assets.json for comparison +let prev_assets_json; +let curr_assets_json; + +async function write_assets_json(metafile) { + prev_assets_json = curr_assets_json; + let out = {}; + for (let output in metafile.outputs) { + let info = metafile.outputs[output]; + let asset_path = "/" + path.relative(sites_path, output); + if (info.entryPoint) { + out[path.basename(info.entryPoint)] = asset_path; + } + } + + let assets_json_path = path.resolve(assets_path, "assets.json"); + let assets_json; + try { + assets_json = await fs.promises.readFile(assets_json_path, "utf-8"); + } catch (error) { + assets_json = "{}"; + } + assets_json = JSON.parse(assets_json); + // update with new values + assets_json = Object.assign({}, assets_json, out); + curr_assets_json = assets_json; + + await fs.promises.writeFile( + assets_json_path, + JSON.stringify(assets_json, null, 4) + ); + await update_assets_json_in_cache(assets_json); + return { + assets_json, + prev_assets_json + }; +} + +function update_assets_json_in_cache(assets_json) { + // update assets_json cache in redis, so that it can be read directly by python + return new Promise(resolve => { + let client = get_redis_subscriber("redis_cache"); + // handle error event to avoid printing stack traces + client.on("error", _ => { + log_warn("Cannot connect to redis_cache to update assets_json"); + }); + client.set("assets_json", JSON.stringify(assets_json), err => { + client.unref(); + resolve(); + }); + }); +} + +function run_build_command_for_apps(apps) { + let cwd = process.cwd(); + let { execSync } = require("child_process"); + + for (let app of apps) { + if (app === "frappe") continue; + + let root_app_path = path.resolve(get_app_path(app), ".."); + let package_json = path.resolve(root_app_path, "package.json"); + if (fs.existsSync(package_json)) { + let { scripts } = require(package_json); + if (scripts && scripts.build) { + log("\nRunning build command for", chalk.bold(app)); + process.chdir(root_app_path); + execSync("yarn build", { encoding: "utf8", stdio: "inherit" }); + } + } + } + + process.chdir(cwd); +} + +async function notify_redis({ error, success }) { + // notify redis which in turns tells socketio to publish this to browser + let subscriber = get_redis_subscriber("redis_socketio"); + subscriber.on("error", _ => { + log_warn("Cannot connect to redis_socketio for browser events"); + }); + + let payload = null; + if (error) { + let formatted = await esbuild.formatMessages(error.errors, { + kind: "error", + terminalWidth: 100 + }); + let stack = error.stack.replace(new RegExp(bench_path, "g"), ""); + payload = { + error, + formatted, + stack + }; + } + if (success) { + payload = { + success: true + }; + } + + subscriber.publish( + "events", + JSON.stringify({ + event: "build_event", + message: payload + }) + ); +} + +function open_in_editor() { + let subscriber = get_redis_subscriber("redis_socketio"); + subscriber.on("error", _ => { + log_warn("Cannot connect to redis_socketio for open_in_editor events"); + }); + subscriber.on("message", (event, file) => { + if (event === "open_in_editor") { + file = JSON.parse(file); + let file_path = path.resolve(file.file); + log("Opening file in editor:", file_path); + let launch = require("launch-editor"); + launch(`${file_path}:${file.line}:${file.column}`); + } + }); + subscriber.subscribe("open_in_editor"); +} + +function log_rebuilt_assets(prev_assets, new_assets) { + let added_files = []; + let old_files = Object.values(prev_assets); + let new_files = Object.values(new_assets); + + for (let filepath of new_files) { + if (!old_files.includes(filepath)) { + added_files.push(filepath); + } + } + + log( + chalk.yellow( + `${new Date().toLocaleTimeString()}: Compiled ${ + added_files.length + } files...` + ) + ); + for (let filepath of added_files) { + let filename = path.basename(filepath); + log(" " + filename); + } + log(); +} diff --git a/esbuild/frappe-html.js b/esbuild/frappe-html.js new file mode 100644 index 0000000000..8c4b7ca3d7 --- /dev/null +++ b/esbuild/frappe-html.js @@ -0,0 +1,43 @@ +module.exports = { + name: "frappe-html", + setup(build) { + let path = require("path"); + let fs = require("fs/promises"); + + build.onResolve({ filter: /\.html$/ }, args => { + return { + path: path.join(args.resolveDir, args.path), + namespace: "frappe-html" + }; + }); + + build.onLoad({ filter: /.*/, namespace: "frappe-html" }, args => { + let filepath = args.path; + let filename = path.basename(filepath).split(".")[0]; + + return fs + .readFile(filepath, "utf-8") + .then(content => { + content = scrub_html_template(content); + return { + contents: `\n\tfrappe.templates['${filename}'] = \`${content}\`;\n` + }; + }) + .catch(() => { + return { + contents: "", + warnings: [ + { + text: `There was an error importing ${filepath}` + } + ] + }; + }); + }); + } +}; + +function scrub_html_template(content) { + content = content.replace(/`/g, "\\`"); + return content; +} diff --git a/esbuild/ignore-assets.js b/esbuild/ignore-assets.js new file mode 100644 index 0000000000..5edfef2110 --- /dev/null +++ b/esbuild/ignore-assets.js @@ -0,0 +1,11 @@ +module.exports = { + name: "frappe-ignore-asset", + setup(build) { + build.onResolve({ filter: /^\/assets\// }, args => { + return { + path: args.path, + external: true + }; + }); + } +}; diff --git a/esbuild/index.js b/esbuild/index.js new file mode 100644 index 0000000000..2721673702 --- /dev/null +++ b/esbuild/index.js @@ -0,0 +1 @@ +require("./esbuild"); diff --git a/esbuild/sass_options.js b/esbuild/sass_options.js new file mode 100644 index 0000000000..fcc7e04ccd --- /dev/null +++ b/esbuild/sass_options.js @@ -0,0 +1,29 @@ +let path = require("path"); +let { get_app_path, app_list } = require("./utils"); + +let node_modules_path = path.resolve( + get_app_path("frappe"), + "..", + "node_modules" +); +let app_paths = app_list + .map(get_app_path) + .map(app_path => path.resolve(app_path, "..")); + +module.exports = { + includePaths: [node_modules_path, ...app_paths], + importer: function(url) { + if (url.startsWith("~")) { + // strip ~ so that it can resolve from node_modules + url = url.slice(1); + } + if (url.endsWith(".css")) { + // strip .css from end of path + url = url.slice(0, -4); + } + // normal file, let it go + return { + file: url + }; + } +}; diff --git a/esbuild/utils.js b/esbuild/utils.js new file mode 100644 index 0000000000..82490adb36 --- /dev/null +++ b/esbuild/utils.js @@ -0,0 +1,145 @@ +const path = require("path"); +const fs = require("fs"); +const chalk = require("chalk"); + +const frappe_path = path.resolve(__dirname, ".."); +const bench_path = path.resolve(frappe_path, "..", ".."); +const sites_path = path.resolve(bench_path, "sites"); +const apps_path = path.resolve(bench_path, "apps"); +const assets_path = path.resolve(sites_path, "assets"); +const app_list = get_apps_list(); + +const app_paths = app_list.reduce((out, app) => { + out[app] = path.resolve(apps_path, app, app); + return out; +}, {}); +const public_paths = app_list.reduce((out, app) => { + out[app] = path.resolve(app_paths[app], "public"); + return out; +}, {}); +const public_js_paths = app_list.reduce((out, app) => { + out[app] = path.resolve(app_paths[app], "public/js"); + return out; +}, {}); + +const bundle_map = app_list.reduce((out, app) => { + const public_js_path = public_js_paths[app]; + if (fs.existsSync(public_js_path)) { + const all_files = fs.readdirSync(public_js_path); + const js_files = all_files.filter(file => file.endsWith(".js")); + + for (let js_file of js_files) { + const filename = path.basename(js_file).split(".")[0]; + out[path.join(app, "js", filename)] = path.resolve( + public_js_path, + js_file + ); + } + } + + return out; +}, {}); + +const get_public_path = app => public_paths[app]; + +const get_build_json_path = app => + path.resolve(get_public_path(app), "build.json"); + +function get_build_json(app) { + try { + return require(get_build_json_path(app)); + } catch (e) { + // build.json does not exist + return null; + } +} + +function delete_file(path) { + if (fs.existsSync(path)) { + fs.unlinkSync(path); + } +} + +function run_serially(tasks) { + let result = Promise.resolve(); + tasks.forEach(task => { + if (task) { + result = result.then ? result.then(task) : Promise.resolve(); + } + }); + return result; +} + +const get_app_path = app => app_paths[app]; + +function get_apps_list() { + return fs + .readFileSync(path.resolve(sites_path, "apps.txt"), { + encoding: "utf-8" + }) + .split("\n") + .filter(Boolean); +} + +function get_cli_arg(name) { + let args = process.argv.slice(2); + let arg = `--${name}`; + let index = args.indexOf(arg); + + let value = null; + if (index != -1) { + value = true; + } + if (value && args[index + 1]) { + value = args[index + 1]; + } + return value; +} + +function log_error(message, badge = "ERROR") { + badge = chalk.white.bgRed(` ${badge} `); + console.error(`${badge} ${message}`); // eslint-disable-line no-console +} + +function log_warn(message, badge = "WARN") { + badge = chalk.black.bgYellowBright(` ${badge} `); + console.warn(`${badge} ${message}`); // eslint-disable-line no-console +} + +function log(...args) { + console.log(...args); // eslint-disable-line no-console +} + +function get_redis_subscriber(kind) { + // get redis subscriber that aborts after 10 connection attempts + let { get_redis_subscriber: get_redis } = require("../node_utils"); + return get_redis(kind, { + retry_strategy: function(options) { + // abort after 10 connection attempts + if (options.attempt > 10) { + return undefined; + } + return Math.min(options.attempt * 100, 2000); + } + }); +} + +module.exports = { + app_list, + bench_path, + assets_path, + sites_path, + apps_path, + bundle_map, + get_public_path, + get_build_json_path, + get_build_json, + get_app_path, + delete_file, + run_serially, + get_cli_arg, + log, + log_warn, + log_error, + get_redis_subscriber +}; diff --git a/frappe/__init__.py b/frappe/__init__.py index 1a495d8ce1..9d8c5d3607 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -10,13 +10,17 @@ be used to build database driven apps. Read the documentation: https://frappeframework.com/docs """ -from __future__ import unicode_literals, print_function +import os, warnings + +_dev_server = os.environ.get('DEV_SERVER', False) + +if _dev_server: + warnings.simplefilter('always', DeprecationWarning) + warnings.simplefilter('always', PendingDeprecationWarning) -from six import iteritems, binary_type, text_type, string_types, PY2 from werkzeug.local import Local, release_local -import os, sys, importlib, inspect, json +import sys, importlib, inspect, json import typing -from past.builtins import cmp import click # Local application imports @@ -27,14 +31,7 @@ from .utils.lazy_loader import lazy_import # Lazy imports faker = lazy_import('faker') - -# Harmless for Python 3 -# For Python 2 set default encoding to utf-8 -if PY2: - reload(sys) - sys.setdefaultencoding("utf-8") - -__version__ = '13.2.2' +__version__ = '14.0.0-dev' __title__ = "Frappe Framework" @@ -97,14 +94,14 @@ def _(msg, lang=None, context=None): def as_unicode(text, encoding='utf-8'): '''Convert to unicode if required''' - if isinstance(text, text_type): + if isinstance(text, str): return text elif text==None: return '' - elif isinstance(text, binary_type): - return text_type(text, encoding) + elif isinstance(text, bytes): + return str(text, encoding) else: - return text_type(text) + return str(text) def get_lang_dict(fortype, name=None): """Returns the translated language dict for the given type and name. @@ -204,7 +201,7 @@ def init(site, sites_path=None, new_site=False): local.meta_cache = {} local.form_dict = _dict() local.session = _dict() - local.dev_server = os.environ.get('DEV_SERVER', False) + local.dev_server = _dev_server setup_module_map() @@ -530,16 +527,20 @@ def sendmail(recipients=[], sender="", subject="No Subject", message="No Message if not delayed: now = True - from frappe.email import queue - queue.send(recipients=recipients, sender=sender, + from frappe.email.doctype.email_queue.email_queue import QueueBuilder + builder = QueueBuilder(recipients=recipients, sender=sender, subject=subject, message=message, text_content=text_content, reference_doctype = doctype or reference_doctype, reference_name = name or reference_name, add_unsubscribe_link=add_unsubscribe_link, unsubscribe_method=unsubscribe_method, unsubscribe_params=unsubscribe_params, unsubscribe_message=unsubscribe_message, attachments=attachments, reply_to=reply_to, cc=cc, bcc=bcc, message_id=message_id, in_reply_to=in_reply_to, send_after=send_after, expose_recipients=expose_recipients, send_priority=send_priority, queue_separately=queue_separately, - communication=communication, now=now, read_receipt=read_receipt, is_notification=is_notification, + communication=communication, read_receipt=read_receipt, is_notification=is_notification, inline_images=inline_images, header=header, print_letterhead=print_letterhead, with_container=with_container) + # build email queue and send the email if send_now is True. + builder.process(send_now=now) + + whitelisted = [] guest_methods = [] xss_safe_methods = [] @@ -597,7 +598,7 @@ def is_whitelisted(method): # strictly sanitize form_dict # escapes html characters like <> except for predefined tags like a, b, ul etc. for key, value in form_dict.items(): - if isinstance(value, string_types): + if isinstance(value, str): form_dict[key] = sanitize_html(value) def read_only(): @@ -721,7 +722,7 @@ def has_website_permission(doc=None, ptype='read', user=None, verbose=False, doc user = session.user if doc: - if isinstance(doc, string_types): + if isinstance(doc, str): doc = get_doc(doctype, doc) doctype = doc.doctype @@ -790,7 +791,7 @@ def set_value(doctype, docname, fieldname, value=None): return frappe.client.set_value(doctype, docname, fieldname, value) def get_cached_doc(*args, **kwargs): - if args and len(args) > 1 and isinstance(args[1], text_type): + if args and len(args) > 1 and isinstance(args[1], str): key = get_document_cache_key(args[0], args[1]) # local cache doc = local.document_cache.get(key) @@ -821,7 +822,7 @@ def clear_document_cache(doctype, name): def get_cached_value(doctype, name, fieldname, as_dict=False): doc = get_cached_doc(doctype, name) - if isinstance(fieldname, string_types): + if isinstance(fieldname, str): if as_dict: throw('Cannot make dict for single fieldname') return doc.get(fieldname) @@ -1027,7 +1028,7 @@ def get_doc_hooks(): if not hasattr(local, 'doc_events_hooks'): hooks = get_hooks('doc_events', {}) out = {} - for key, value in iteritems(hooks): + for key, value in hooks.items(): if isinstance(key, tuple): for doctype in key: append_hook(out, doctype, value) @@ -1109,9 +1110,7 @@ def setup_module_map(): if not (local.app_modules and local.module_app): local.module_app, local.app_modules = {}, {} - for app in get_all_apps(True): - if app == "webnotes": - app = "frappe" + for app in get_all_apps(with_internal_apps=True): local.app_modules.setdefault(app, []) for module in get_module_list(app): module = scrub(module) @@ -1144,7 +1143,7 @@ def get_file_json(path): def read_file(path, raise_not_found=False): """Open a file and return its content as Unicode.""" - if isinstance(path, text_type): + if isinstance(path, str): path = path.encode("utf-8") if os.path.exists(path): @@ -1167,7 +1166,7 @@ def get_attr(method_string): def call(fn, *args, **kwargs): """Call a function and match arguments.""" - if isinstance(fn, string_types): + if isinstance(fn, str): fn = get_attr(fn) newargs = get_newargs(fn, kwargs) @@ -1178,13 +1177,9 @@ def get_newargs(fn, kwargs): if hasattr(fn, 'fnargs'): fnargs = fn.fnargs else: - try: - fnargs, varargs, varkw, defaults = inspect.getargspec(fn) - except ValueError: - fnargs = inspect.getfullargspec(fn).args - varargs = inspect.getfullargspec(fn).varargs - varkw = inspect.getfullargspec(fn).varkw - defaults = inspect.getfullargspec(fn).defaults + fnargs = inspect.getfullargspec(fn).args + fnargs.extend(inspect.getfullargspec(fn).kwonlyargs) + varkw = inspect.getfullargspec(fn).varkw newargs = {} for a in kwargs: @@ -1626,6 +1621,12 @@ def enqueue(*args, **kwargs): import frappe.utils.background_jobs return frappe.utils.background_jobs.enqueue(*args, **kwargs) +def task(**task_kwargs): + def decorator_task(f): + f.enqueue = lambda **fun_kwargs: enqueue(f, **task_kwargs, **fun_kwargs) + return f + return decorator_task + def enqueue_doc(*args, **kwargs): ''' Enqueue method to be executed using a background worker @@ -1693,6 +1694,23 @@ def safe_eval(code, eval_globals=None, eval_locals=None): "round": round } + UNSAFE_ATTRIBUTES = { + # Generator Attributes + "gi_frame", "gi_code", + # Coroutine Attributes + "cr_frame", "cr_code", "cr_origin", + # Async Generator Attributes + "ag_code", "ag_frame", + # Traceback Attributes + "tb_frame", "tb_next", + # Format Attributes + "format", "format_map", + } + + for attribute in UNSAFE_ATTRIBUTES: + if attribute in code: + throw('Illegal rule {0}. Cannot use "{1}"'.format(bold(code), attribute)) + if '__' in code: throw('Illegal rule {0}. Cannot use "__"'.format(bold(code))) diff --git a/frappe/api.py b/frappe/api.py index 9039ae0e5f..36d51e894c 100644 --- a/frappe/api.py +++ b/frappe/api.py @@ -1,6 +1,5 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt - import base64 import binascii import json @@ -11,6 +10,7 @@ import frappe.client import frappe.handler from frappe import _ from frappe.utils.response import build_response +from frappe.utils.data import sbool def handle(): @@ -108,25 +108,40 @@ def handle(): elif doctype: if frappe.local.request.method == "GET": - if frappe.local.form_dict.get('fields'): - frappe.local.form_dict['fields'] = json.loads(frappe.local.form_dict['fields']) - frappe.local.form_dict.setdefault('limit_page_length', 20) - frappe.local.response.update({ - "data": frappe.call( - frappe.client.get_list, - doctype, - **frappe.local.form_dict - ) - }) + # set fields for frappe.get_list + if frappe.local.form_dict.get("fields"): + frappe.local.form_dict["fields"] = json.loads(frappe.local.form_dict["fields"]) + + # set limit of records for frappe.get_list + frappe.local.form_dict.setdefault( + "limit_page_length", + frappe.local.form_dict.limit or frappe.local.form_dict.limit_page_length or 20, + ) + + # convert strings to native types - only as_dict and debug accept bool + for param in ["as_dict", "debug"]: + param_val = frappe.local.form_dict.get(param) + if param_val is not None: + frappe.local.form_dict[param] = sbool(param_val) + + # evaluate frappe.get_list + data = frappe.call(frappe.client.get_list, doctype, **frappe.local.form_dict) + + # set frappe.get_list result to response + frappe.local.response.update({"data": data}) if frappe.local.request.method == "POST": + # fetch data from from dict data = get_request_form_data() - data.update({ - "doctype": doctype - }) - frappe.local.response.update({ - "data": frappe.get_doc(data).insert().as_dict() - }) + data.update({"doctype": doctype}) + + # insert document from request data + doc = frappe.get_doc(data).insert() + + # set response data + frappe.local.response.update({"data": doc.as_dict()}) + + # commit for POST requests frappe.db.commit() else: raise frappe.DoesNotExistError diff --git a/frappe/app.py b/frappe/app.py index 794d0f18af..6f5023be93 100644 --- a/frappe/app.py +++ b/frappe/app.py @@ -1,10 +1,8 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import os -from six import iteritems import logging from werkzeug.local import LocalManager @@ -99,17 +97,7 @@ def application(request): frappe.monitor.stop(response) frappe.recorder.dump() - if hasattr(frappe.local, 'conf') and frappe.local.conf.enable_frappe_logger: - frappe.logger("frappe.web", allow_site=frappe.local.site).info({ - "site": get_site_name(request.host), - "remote_addr": getattr(request, "remote_addr", "NOTFOUND"), - "base_url": getattr(request, "base_url", "NOTFOUND"), - "full_path": getattr(request, "full_path", "NOTFOUND"), - "method": getattr(request, "method", "NOTFOUND"), - "scheme": getattr(request, "scheme", "NOTFOUND"), - "http_status_code": getattr(response, "status_code", "NOTFOUND") - }) - + log_request(request, response) process_response(response) frappe.destroy() @@ -137,6 +125,19 @@ def init_request(request): if request.method != "OPTIONS": frappe.local.http_request = frappe.auth.HTTPRequest() +def log_request(request, response): + if hasattr(frappe.local, 'conf') and frappe.local.conf.enable_frappe_logger: + frappe.logger("frappe.web", allow_site=frappe.local.site).info({ + "site": get_site_name(request.host), + "remote_addr": getattr(request, "remote_addr", "NOTFOUND"), + "base_url": getattr(request, "base_url", "NOTFOUND"), + "full_path": getattr(request, "full_path", "NOTFOUND"), + "method": getattr(request, "method", "NOTFOUND"), + "scheme": getattr(request, "scheme", "NOTFOUND"), + "http_status_code": getattr(response, "status_code", "NOTFOUND") + }) + + def process_response(response): if not response: return @@ -185,11 +186,12 @@ def make_form_dict(request): args = request.form or request.args if not isinstance(args, dict): - frappe.throw("Invalid request arguments") + frappe.throw(_("Invalid request arguments")) try: - frappe.local.form_dict = frappe._dict({ k:v[0] if isinstance(v, (list, tuple)) else v \ - for k, v in iteritems(args) }) + frappe.local.form_dict = frappe._dict({ + k: v[0] if isinstance(v, (list, tuple)) else v for k, v in args.items() + }) except IndexError: frappe.local.form_dict = frappe._dict(args) @@ -294,8 +296,9 @@ def serve(port=8000, profile=False, no_reload=False, no_threading=False, site=No _sites_path = sites_path from werkzeug.serving import run_simple + patch_werkzeug_reloader() - if profile: + if profile or os.environ.get('USE_PROFILER'): application = ProfilerMiddleware(application, sort_by=('cumtime', 'calls')) if not os.environ.get('NO_STATICS'): @@ -324,3 +327,23 @@ def serve(port=8000, profile=False, no_reload=False, no_threading=False, site=No use_debugger=not in_test_env, use_evalex=not in_test_env, threaded=not no_threading) + +def patch_werkzeug_reloader(): + """ + This function monkey patches Werkzeug reloader to ignore reloading files in + the __pycache__ directory. + + To be deprecated when upgrading to Werkzeug 2. + """ + + from werkzeug._reloader import WatchdogReloaderLoop + + trigger_reload = WatchdogReloaderLoop.trigger_reload + + def custom_trigger_reload(self, filename): + if os.path.basename(os.path.dirname(filename)) == "__pycache__": + return + + return trigger_reload(self, filename) + + WatchdogReloaderLoop.trigger_reload = custom_trigger_reload diff --git a/frappe/auth.py b/frappe/auth.py index 73cb8e8c15..ef79d96ddb 100644 --- a/frappe/auth.py +++ b/frappe/auth.py @@ -1,9 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt - -from __future__ import unicode_literals import datetime - from frappe import _ import frappe import frappe.database @@ -19,8 +16,7 @@ from frappe.core.doctype.activity_log.activity_log import add_authentication_log from frappe.twofactor import (should_run_2fa, authenticate_for_2factor, confirm_otp_token, get_cached_user_pass) from frappe.website.utils import get_home_page - -from six.moves.urllib.parse import quote +from urllib.parse import quote class HTTPRequest: diff --git a/frappe/automation/doctype/assignment_rule/assignment_rule.py b/frappe/automation/doctype/assignment_rule/assignment_rule.py index c673d5ceeb..ef579aca01 100644 --- a/frappe/automation/doctype/assignment_rule/assignment_rule.py +++ b/frappe/automation/doctype/assignment_rule/assignment_rule.py @@ -2,8 +2,6 @@ # Copyright (c) 2019, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals - import frappe from frappe.model.document import Document from frappe.desk.form import assign_to diff --git a/frappe/automation/doctype/assignment_rule/test_assignment_rule.py b/frappe/automation/doctype/assignment_rule/test_assignment_rule.py index cb1e0ff8f4..e287b83965 100644 --- a/frappe/automation/doctype/assignment_rule/test_assignment_rule.py +++ b/frappe/automation/doctype/assignment_rule/test_assignment_rule.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import frappe import unittest from frappe.utils import random_string diff --git a/frappe/automation/doctype/assignment_rule_day/assignment_rule_day.py b/frappe/automation/doctype/assignment_rule_day/assignment_rule_day.py index 27f9aa40e1..c734495c39 100644 --- a/frappe/automation/doctype/assignment_rule_day/assignment_rule_day.py +++ b/frappe/automation/doctype/assignment_rule_day/assignment_rule_day.py @@ -2,7 +2,6 @@ # Copyright (c) 2019, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals # import frappe from frappe.model.document import Document diff --git a/frappe/automation/doctype/assignment_rule_user/assignment_rule_user.py b/frappe/automation/doctype/assignment_rule_user/assignment_rule_user.py index ee8081c6d8..4d65efd5c1 100644 --- a/frappe/automation/doctype/assignment_rule_user/assignment_rule_user.py +++ b/frappe/automation/doctype/assignment_rule_user/assignment_rule_user.py @@ -2,7 +2,6 @@ # Copyright (c) 2019, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals # import frappe from frappe.model.document import Document diff --git a/frappe/automation/doctype/auto_repeat/auto_repeat.js b/frappe/automation/doctype/auto_repeat/auto_repeat.js index 7028ac486d..896a10dfe0 100644 --- a/frappe/automation/doctype/auto_repeat/auto_repeat.js +++ b/frappe/automation/doctype/auto_repeat/auto_repeat.js @@ -103,7 +103,7 @@ frappe.ui.form.on('Auto Repeat', { frappe.auto_repeat.render_schedule = function(frm) { if (!frm.is_dirty() && frm.doc.status !== 'Disabled') { frm.call("get_auto_repeat_schedule").then(r => { - frm.dashboard.wrapper.empty(); + frm.dashboard.reset(); frm.dashboard.add_section( frappe.render_template("auto_repeat_schedule", { schedule_details: r.message || [] diff --git a/frappe/automation/doctype/auto_repeat/auto_repeat.py b/frappe/automation/doctype/auto_repeat/auto_repeat.py index bf05baf5b6..998e73a42c 100644 --- a/frappe/automation/doctype/auto_repeat/auto_repeat.py +++ b/frappe/automation/doctype/auto_repeat/auto_repeat.py @@ -2,7 +2,6 @@ # Copyright (c) 2018, Frappe Technologies Pvt. Ltd. and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe import _ from datetime import timedelta diff --git a/frappe/automation/doctype/auto_repeat/test_auto_repeat.py b/frappe/automation/doctype/auto_repeat/test_auto_repeat.py index f41f31f3bb..567c1161af 100644 --- a/frappe/automation/doctype/auto_repeat/test_auto_repeat.py +++ b/frappe/automation/doctype/auto_repeat/test_auto_repeat.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2018, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import unittest import frappe @@ -173,7 +171,7 @@ class TestAutoRepeat(unittest.TestCase): fields=['docstatus'], limit=1 ) - self.assertEquals(docnames[0].docstatus, 1) + self.assertEqual(docnames[0].docstatus, 1) def make_auto_repeat(**args): diff --git a/frappe/automation/doctype/auto_repeat_day/auto_repeat_day.py b/frappe/automation/doctype/auto_repeat_day/auto_repeat_day.py index 3a7ced1370..8af3284cde 100644 --- a/frappe/automation/doctype/auto_repeat_day/auto_repeat_day.py +++ b/frappe/automation/doctype/auto_repeat_day/auto_repeat_day.py @@ -2,7 +2,6 @@ # Copyright (c) 2020, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals # import frappe from frappe.model.document import Document diff --git a/frappe/automation/doctype/milestone/milestone.py b/frappe/automation/doctype/milestone/milestone.py index 64c073a378..6ea6d7544a 100644 --- a/frappe/automation/doctype/milestone/milestone.py +++ b/frappe/automation/doctype/milestone/milestone.py @@ -2,8 +2,6 @@ # Copyright (c) 2019, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals - import frappe from frappe.model.document import Document diff --git a/frappe/automation/doctype/milestone/test_milestone.py b/frappe/automation/doctype/milestone/test_milestone.py index 75602d48db..175c56e552 100644 --- a/frappe/automation/doctype/milestone/test_milestone.py +++ b/frappe/automation/doctype/milestone/test_milestone.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - #import frappe import unittest diff --git a/frappe/automation/doctype/milestone_tracker/milestone_tracker.py b/frappe/automation/doctype/milestone_tracker/milestone_tracker.py index 388620bfb4..125cad7fa8 100644 --- a/frappe/automation/doctype/milestone_tracker/milestone_tracker.py +++ b/frappe/automation/doctype/milestone_tracker/milestone_tracker.py @@ -2,8 +2,6 @@ # Copyright (c) 2019, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals - import frappe from frappe.model.document import Document import frappe.cache_manager diff --git a/frappe/automation/doctype/milestone_tracker/test_milestone_tracker.py b/frappe/automation/doctype/milestone_tracker/test_milestone_tracker.py index 05db3b025e..21b2779018 100644 --- a/frappe/automation/doctype/milestone_tracker/test_milestone_tracker.py +++ b/frappe/automation/doctype/milestone_tracker/test_milestone_tracker.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import frappe import frappe.cache_manager import unittest diff --git a/frappe/boot.py b/frappe/boot.py index 65a07b15e5..0589e32ac8 100644 --- a/frappe/boot.py +++ b/frappe/boot.py @@ -1,10 +1,5 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt - -from __future__ import unicode_literals - -from six import iteritems, text_type - """ bootstrap client session """ @@ -42,8 +37,6 @@ def get_bootinfo(): bootinfo.user_info = get_user_info() bootinfo.sid = frappe.session['sid'] - bootinfo.user_groups = frappe.get_all('User Group', pluck="name") - bootinfo.modules = {} bootinfo.module_list = [] load_desktop_data(bootinfo) @@ -77,7 +70,7 @@ def get_bootinfo(): frappe.get_attr(method)(bootinfo) if bootinfo.lang: - bootinfo.lang = text_type(bootinfo.lang) + bootinfo.lang = str(bootinfo.lang) bootinfo.versions = {k: v['version'] for k, v in get_versions().items()} bootinfo.error_report_email = frappe.conf.error_report_email @@ -222,7 +215,7 @@ def load_translations(bootinfo): messages[name] = frappe._(name) # only untranslated - messages = {k:v for k, v in iteritems(messages) if k!=v} + messages = {k: v for k, v in messages.items() if k!=v} bootinfo["__messages"] = messages diff --git a/frappe/build.py b/frappe/build.py index baedb633b6..ed19574cfd 100644 --- a/frappe/build.py +++ b/frappe/build.py @@ -1,14 +1,12 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt - -from __future__ import print_function, unicode_literals - import os import re import json import shutil -import warnings -import tempfile +import subprocess +from io import StringIO +from tempfile import mkdtemp, mktemp from distutils.spawn import find_executable import frappe @@ -16,8 +14,9 @@ from frappe.utils.minify import JavascriptMinify import click import psutil -from six import iteritems, text_type -from six.moves.urllib.parse import urlparse +from urllib.parse import urlparse +from simple_chalk import green +from semantic_version import Version timestamps = {} @@ -39,35 +38,36 @@ def download_file(url, prefix): def build_missing_files(): - # check which files dont exist yet from the build.json and tell build.js to build only those! + '''Check which files dont exist yet from the assets.json and run build for those files''' + missing_assets = [] current_asset_files = [] - frappe_build = os.path.join("..", "apps", "frappe", "frappe", "public", "build.json") for type in ["css", "js"]: - current_asset_files.extend( - [ - "{0}/{1}".format(type, name) - for name in os.listdir(os.path.join(sites_path, "assets", type)) - ] - ) + folder = os.path.join(sites_path, "assets", "frappe", "dist", type) + current_asset_files.extend(os.listdir(folder)) - with open(frappe_build) as f: - all_asset_files = json.load(f).keys() + development = frappe.local.conf.developer_mode or frappe.local.dev_server + build_mode = "development" if development else "production" - for asset in all_asset_files: - if asset.replace("concat:", "") not in current_asset_files: - missing_assets.append(asset) + assets_json = frappe.read_file("assets/assets.json") + if assets_json: + assets_json = frappe.parse_json(assets_json) - if missing_assets: - from subprocess import check_call - from shlex import split + for bundle_file, output_file in assets_json.items(): + if not output_file.startswith('/assets/frappe'): + continue - click.secho("\nBuilding missing assets...\n", fg="yellow") - command = split( - "node rollup/build.js --files {0} --no-concat".format(",".join(missing_assets)) - ) - check_call(command, cwd=os.path.join("..", "apps", "frappe")) + if os.path.basename(output_file) not in current_asset_files: + missing_assets.append(bundle_file) + + if missing_assets: + click.secho("\nBuilding missing assets...\n", fg="yellow") + files_to_build = ["frappe/" + name for name in missing_assets] + bundle(build_mode, files=files_to_build) + else: + # no assets.json, run full build + bundle(build_mode, apps="frappe") def get_assets_link(frappe_head): @@ -75,8 +75,8 @@ def get_assets_link(frappe_head): from requests import head tag = getoutput( - "cd ../apps/frappe && git show-ref --tags -d | grep %s | sed -e 's,.*" - " refs/tags/,,' -e 's/\^{}//'" + r"cd ../apps/frappe && git show-ref --tags -d | grep %s | sed -e 's,.*" + r" refs/tags/,,' -e 's/\^{}//'" % frappe_head ) @@ -97,9 +97,7 @@ def download_frappe_assets(verbose=True): commit HEAD. Returns True if correctly setup else returns False. """ - from simple_chalk import green from subprocess import getoutput - from tempfile import mkdtemp assets_setup = False frappe_head = getoutput("cd ../apps/frappe && git rev-parse HEAD") @@ -166,7 +164,7 @@ def symlink(target, link_name, overwrite=False): # Create link to target with temporary filename while True: - temp_link_name = tempfile.mktemp(dir=link_dir) + temp_link_name = mktemp(dir=link_dir) # os.* functions mimic as closely as possible system functions # The POSIX symlink() returns EEXIST if link_name already exists @@ -193,7 +191,8 @@ def symlink(target, link_name, overwrite=False): def setup(): - global app_paths + global app_paths, assets_path + pymodules = [] for app in frappe.get_all_apps(True): try: @@ -201,51 +200,54 @@ def setup(): except ImportError: pass app_paths = [os.path.dirname(pymodule.__file__) for pymodule in pymodules] + assets_path = os.path.join(frappe.local.sites_path, "assets") -def get_node_pacman(): - exec_ = find_executable("yarn") - if exec_: - return exec_ - raise ValueError("Yarn not found") - - -def bundle(no_compress, app=None, make_copy=False, restore=False, verbose=False, skip_frappe=False): +def bundle(mode, apps=None, hard_link=False, make_copy=False, restore=False, verbose=False, skip_frappe=False, files=None): """concat / minify js files""" setup() - make_asset_dirs(make_copy=make_copy, restore=restore) + make_asset_dirs(hard_link=hard_link) - pacman = get_node_pacman() - mode = "build" if no_compress else "production" - command = "{pacman} run {mode}".format(pacman=pacman, mode=mode) + mode = "production" if mode == "production" else "build" + command = "yarn run {mode}".format(mode=mode) - if app: - command += " --app {app}".format(app=app) + if apps: + command += " --apps {apps}".format(apps=apps) if skip_frappe: command += " --skip_frappe" - frappe_app_path = os.path.abspath(os.path.join(app_paths[0], "..")) - check_yarn() + if files: + command += " --files {files}".format(files=','.join(files)) + + command += " --run-build-command" + + check_node_executable() + frappe_app_path = frappe.get_app_path("frappe", "..") frappe.commands.popen(command, cwd=frappe_app_path, env=get_node_env()) -def watch(no_compress): +def watch(apps=None): """watch and rebuild if necessary""" setup() - pacman = get_node_pacman() + command = "yarn run watch" + if apps: + command += " --apps {apps}".format(apps=apps) - frappe_app_path = os.path.abspath(os.path.join(app_paths[0], "..")) - check_yarn() + check_node_executable() frappe_app_path = frappe.get_app_path("frappe", "..") - frappe.commands.popen("{pacman} run watch".format(pacman=pacman), - cwd=frappe_app_path, env=get_node_env()) + frappe.commands.popen(command, cwd=frappe_app_path, env=get_node_env()) -def check_yarn(): +def check_node_executable(): + node_version = Version(subprocess.getoutput('node -v')[1:]) + warn = '⚠️ ' + if node_version.major < 14: + click.echo(f"{warn} Please update your node version to 14") if not find_executable("yarn"): - print("Please install yarn using below command and try again.\nnpm install -g yarn") + click.echo(f"{warn} Please install yarn using below command and try again.\nnpm install -g yarn") + click.echo() def get_node_env(): node_env = { @@ -266,75 +268,109 @@ def get_safe_max_old_space_size(): return safe_max_old_space_size -def make_asset_dirs(make_copy=False, restore=False): - # don't even think of making assets_path absolute - rm -rf ahead. - assets_path = os.path.join(frappe.local.sites_path, "assets") +def generate_assets_map(): + symlinks = {} - for dir_path in [os.path.join(assets_path, "js"), os.path.join(assets_path, "css")]: - if not os.path.exists(dir_path): - os.makedirs(dir_path) + for app_name in frappe.get_all_apps(): + app_doc_path = None - for app_name in frappe.get_all_apps(True): pymodule = frappe.get_module(app_name) app_base_path = os.path.abspath(os.path.dirname(pymodule.__file__)) - - symlinks = [] app_public_path = os.path.join(app_base_path, "public") - # app/public > assets/app - symlinks.append([app_public_path, os.path.join(assets_path, app_name)]) - # app/node_modules > assets/app/node_modules - if os.path.exists(os.path.abspath(app_public_path)): - symlinks.append( - [ - os.path.join(app_base_path, "..", "node_modules"), - os.path.join(assets_path, app_name, "node_modules"), - ] - ) + app_node_modules_path = os.path.join(app_base_path, "..", "node_modules") + app_docs_path = os.path.join(app_base_path, "docs") + app_www_docs_path = os.path.join(app_base_path, "www", "docs") - app_doc_path = None - if os.path.isdir(os.path.join(app_base_path, "docs")): + app_assets = os.path.abspath(app_public_path) + app_node_modules = os.path.abspath(app_node_modules_path) + + # {app}/public > assets/{app} + if os.path.isdir(app_assets): + symlinks[app_assets] = os.path.join(assets_path, app_name) + + # {app}/node_modules > assets/{app}/node_modules + if os.path.isdir(app_node_modules): + symlinks[app_node_modules] = os.path.join(assets_path, app_name, "node_modules") + + # {app}/docs > assets/{app}_docs + if os.path.isdir(app_docs_path): app_doc_path = os.path.join(app_base_path, "docs") - - elif os.path.isdir(os.path.join(app_base_path, "www", "docs")): + elif os.path.isdir(app_www_docs_path): app_doc_path = os.path.join(app_base_path, "www", "docs") - if app_doc_path: - symlinks.append([app_doc_path, os.path.join(assets_path, app_name + "_docs")]) + app_docs = os.path.abspath(app_doc_path) + symlinks[app_docs] = os.path.join(assets_path, app_name + "_docs") - for source, target in symlinks: - source = os.path.abspath(source) - if os.path.exists(source): - if restore: - if os.path.exists(target): - if os.path.islink(target): - os.unlink(target) - else: - shutil.rmtree(target) - shutil.copytree(source, target) - elif make_copy: - if os.path.exists(target): - warnings.warn("Target {target} already exists.".format(target=target)) - else: - shutil.copytree(source, target) - else: - if os.path.exists(target): - if os.path.islink(target): - os.unlink(target) - else: - shutil.rmtree(target) - try: - symlink(source, target, overwrite=True) - except OSError: - print("Cannot link {} to {}".format(source, target)) - else: - # warnings.warn('Source {source} does not exist.'.format(source = source)) - pass + return symlinks + + +def setup_assets_dirs(): + for dir_path in (os.path.join(assets_path, x) for x in ("js", "css")): + os.makedirs(dir_path, exist_ok=True) + + +def clear_broken_symlinks(): + for path in os.listdir(assets_path): + path = os.path.join(assets_path, path) + if os.path.islink(path) and not os.path.exists(path): + os.remove(path) + + + +def unstrip(message: str) -> str: + """Pads input string on the right side until the last available column in the terminal + """ + _len = len(message) + try: + max_str = os.get_terminal_size().columns + except Exception: + max_str = 80 + + if _len < max_str: + _rem = max_str - _len + else: + _rem = max_str % _len + + return f"{message}{' ' * _rem}" + + +def make_asset_dirs(hard_link=False): + setup_assets_dirs() + clear_broken_symlinks() + symlinks = generate_assets_map() + + for source, target in symlinks.items(): + start_message = unstrip(f"{'Copying assets from' if hard_link else 'Linking'} {source} to {target}") + fail_message = unstrip(f"Cannot {'copy' if hard_link else 'link'} {source} to {target}") + + # Used '\r' instead of '\x1b[1K\r' to print entire lines in smaller terminal sizes + try: + print(start_message, end="\r") + link_assets_dir(source, target, hard_link=hard_link) + except Exception: + print(fail_message, end="\r") + + print(unstrip(f"{green('✔')} Application Assets Linked") + "\n") + + +def link_assets_dir(source, target, hard_link=False): + if not os.path.exists(source): + return + + if os.path.exists(target): + if os.path.islink(target): + os.unlink(target) + else: + shutil.rmtree(target) + + if hard_link: + shutil.copytree(source, target, dirs_exist_ok=True) + else: + symlink(source, target, overwrite=True) def build(no_compress=False, verbose=False): - assets_path = os.path.join(frappe.local.sites_path, "assets") - - for target, sources in iteritems(get_build_maps()): + for target, sources in get_build_maps().items(): pack(os.path.join(assets_path, target), sources, no_compress, verbose) @@ -348,7 +384,7 @@ def get_build_maps(): if os.path.exists(path): with open(path) as f: try: - for target, sources in iteritems(json.loads(f.read())): + for target, sources in (json.loads(f.read() or "{}")).items(): # update app path source_paths = [] for source in sources: @@ -366,8 +402,6 @@ def get_build_maps(): def pack(target, sources, no_compress, verbose): - from six import StringIO - outtype, outtxt = target.split(".")[-1], "" jsm = JavascriptMinify() @@ -381,7 +415,7 @@ def pack(target, sources, no_compress, verbose): timestamps[f] = os.path.getmtime(f) try: with open(f, "r") as sourcefile: - data = text_type(sourcefile.read(), "utf-8", errors="ignore") + data = str(sourcefile.read(), "utf-8", errors="ignore") extn = f.rsplit(".", 1)[1] @@ -396,7 +430,7 @@ def pack(target, sources, no_compress, verbose): jsm.minify(tmpin, tmpout) minified = tmpout.getvalue() if minified: - outtxt += text_type(minified or "", "utf-8").strip("\n") + ";" + outtxt += str(minified or "", "utf-8").strip("\n") + ";" if verbose: print("{0}: {1}k".format(f, int(len(minified) / 1024))) @@ -426,16 +460,16 @@ def html_to_js_template(path, content): def scrub_html_template(content): """Returns HTML content with removed whitespace and comments""" # remove whitespace to a single space - content = re.sub("\s+", " ", content) + content = re.sub(r"\s+", " ", content) # strip comments - content = re.sub("()", "", content) + content = re.sub(r"()", "", content) return content.replace("'", "\'") def files_dirty(): - for target, sources in iteritems(get_build_maps()): + for target, sources in get_build_maps().items(): for f in sources: if ":" in f: f, suffix = f.split(":") diff --git a/frappe/cache_manager.py b/frappe/cache_manager.py index 4e0fe0cf44..52fba4568d 100644 --- a/frappe/cache_manager.py +++ b/frappe/cache_manager.py @@ -1,8 +1,6 @@ # Copyright (c) 2018, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals - import frappe, json from frappe.model.document import Document from frappe.desk.notifications import (delete_notification_count_for, @@ -13,6 +11,8 @@ common_default_keys = ["__default", "__global"] doctype_map_keys = ('energy_point_rule_map', 'assignment_rule_map', 'milestone_tracker_map', 'event_consumer_document_type_map') +bench_cache_keys = ('assets_json',) + global_cache_keys = ("app_hooks", "installed_apps", 'all_apps', "app_modules", "module_app", "system_settings", 'scheduler_events', 'time_zone', 'webhooks', 'active_domains', @@ -58,6 +58,7 @@ def clear_global_cache(): clear_doctype_cache() clear_website_cache() frappe.cache().delete_value(global_cache_keys) + frappe.cache().delete_value(bench_cache_keys) frappe.setup_module_map() def clear_defaults_cache(user=None): diff --git a/frappe/change_log/v13/v13_3_0.md b/frappe/change_log/v13/v13_3_0.md new file mode 100644 index 0000000000..6ab181ef09 --- /dev/null +++ b/frappe/change_log/v13/v13_3_0.md @@ -0,0 +1,49 @@ +# Version 13.3.0 Release Notes + +### Features & Enhancements + +- Deletion Steps in Data Deletion Tool ([#13124](https://github.com/frappe/frappe/pull/13124)) +- Format Option for list-apps in bench CLI ([#13125](https://github.com/frappe/frappe/pull/13125)) +- Add password fieldtype option for Web Form ([#13093](https://github.com/frappe/frappe/pull/13093)) +- Add simple __repr__ for DocTypes ([#13151](https://github.com/frappe/frappe/pull/13151)) +- Switch theme with left/right keys ([#13077](https://github.com/frappe/frappe/pull/13077)) +- sourceURL for injected javascript ([#13022](https://github.com/frappe/frappe/pull/13022)) + +### Fixes + +- Decode uri before importing file via weblink ([#13026](https://github.com/frappe/frappe/pull/13026)) +- Respond to /api requests as JSON by default ([#13053](https://github.com/frappe/frappe/pull/13053)) +- Disabled checkbox should be disabled ([#13021](https://github.com/frappe/frappe/pull/13021)) +- Moving Site folder across different FileSystems failed ([#13038](https://github.com/frappe/frappe/pull/13038)) +- Freeze screen till the background request is complete ([#13078](https://github.com/frappe/frappe/pull/13078)) +- Added conditional rendering for content field in split section w… ([#13075](https://github.com/frappe/frappe/pull/13075)) +- Show delete button on portal if user has permission to delete document ([#13149](https://github.com/frappe/frappe/pull/13149)) +- Dont disable dialog scroll on focusing a Link/Autocomplete field ([#13119](https://github.com/frappe/frappe/pull/13119)) +- Typo in RecorderDetail.vue ([#13086](https://github.com/frappe/frappe/pull/13086)) +- Error for bench drop-site. Added missing import. ([#13064](https://github.com/frappe/frappe/pull/13064)) +- Report column context ([#13090](https://github.com/frappe/frappe/pull/13090)) +- Different service name for push and pull request events ([#13094](https://github.com/frappe/frappe/pull/13094)) +- Moving Site folder across different FileSystems failed ([#13033](https://github.com/frappe/frappe/pull/13033)) +- Consistent checkboxes on all browsers ([#13042](https://github.com/frappe/frappe/pull/13042)) +- Changed shorcut widgets color picker to dropdown ([#13073](https://github.com/frappe/frappe/pull/13073)) +- Error while exporting reports with duration field ([#13118](https://github.com/frappe/frappe/pull/13118)) +- Add margin to download backup card ([#13079](https://github.com/frappe/frappe/pull/13079)) +- Move mention list generation logic to server-side ([#13074](https://github.com/frappe/frappe/pull/13074)) +- Make strings translatable ([#13046](https://github.com/frappe/frappe/pull/13046)) +- Don't evaluate dynamic properties to check if conflicts exist ([#13186](https://github.com/frappe/frappe/pull/13186)) +- Add __ function in vue global for translation in recorder ([#13089](https://github.com/frappe/frappe/pull/13089)) +- Make strings translatable ([#13076](https://github.com/frappe/frappe/pull/13076)) +- Show config in bench CLI ([#13128](https://github.com/frappe/frappe/pull/13128)) +- Add breadcrumbs for list view ([#13091](https://github.com/frappe/frappe/pull/13091)) +- Do not skip data in save while using shortcut ([#13182](https://github.com/frappe/frappe/pull/13182)) +- Use docfields from options if no docfields are returned from meta ([#13188](https://github.com/frappe/frappe/pull/13188)) +- Disable reloading files in `__pycache__` directory ([#13109](https://github.com/frappe/frappe/pull/13109)) +- RTL stylesheet route to load RTL style on demand. ([#13007](https://github.com/frappe/frappe/pull/13007)) +- Do not show messsage when exception is handled ([#13111](https://github.com/frappe/frappe/pull/13111)) +- Replace parseFloat by Number ([#13082](https://github.com/frappe/frappe/pull/13082)) +- Add margin to download backup card ([#13050](https://github.com/frappe/frappe/pull/13050)) +- Translate report column labels ([#13083](https://github.com/frappe/frappe/pull/13083)) +- Grid row color picker field not working ([#13040](https://github.com/frappe/frappe/pull/13040)) +- Improve oauthlib implementation ([#13045](https://github.com/frappe/frappe/pull/13045)) +- Replace filter_by like with full text filter ([#13126](https://github.com/frappe/frappe/pull/13126)) +- Focus jumps to first field ([#13067](https://github.com/frappe/frappe/pull/13067)) \ No newline at end of file diff --git a/frappe/chat/__init__.py b/frappe/chat/__init__.py index dea0030839..4c9b1c5db7 100644 --- a/frappe/chat/__init__.py +++ b/frappe/chat/__init__.py @@ -1,4 +1,4 @@ -from __future__ import unicode_literals + import frappe from frappe import _ diff --git a/frappe/chat/doctype/chat_message/chat_message.py b/frappe/chat/doctype/chat_message/chat_message.py index 5549aaa657..bc470a5e9c 100644 --- a/frappe/chat/doctype/chat_message/chat_message.py +++ b/frappe/chat/doctype/chat_message/chat_message.py @@ -1,5 +1,3 @@ -from __future__ import unicode_literals - # imports - standard imports import json diff --git a/frappe/chat/doctype/chat_profile/chat_profile.py b/frappe/chat/doctype/chat_profile/chat_profile.py index 698d992d35..da10a836c4 100644 --- a/frappe/chat/doctype/chat_profile/chat_profile.py +++ b/frappe/chat/doctype/chat_profile/chat_profile.py @@ -1,5 +1,3 @@ -from __future__ import unicode_literals - # imports - module imports from frappe.model.document import Document from frappe import _ diff --git a/frappe/chat/doctype/chat_room/chat_room.py b/frappe/chat/doctype/chat_room/chat_room.py index 609acaef7d..bdbee44d7a 100644 --- a/frappe/chat/doctype/chat_room/chat_room.py +++ b/frappe/chat/doctype/chat_room/chat_room.py @@ -1,5 +1,3 @@ -from __future__ import unicode_literals - # imports - module imports from frappe.model.document import Document from frappe import _ diff --git a/frappe/chat/doctype/chat_room_user/chat_room_user.py b/frappe/chat/doctype/chat_room_user/chat_room_user.py index f8e13add82..f6dbdc7659 100644 --- a/frappe/chat/doctype/chat_room_user/chat_room_user.py +++ b/frappe/chat/doctype/chat_room_user/chat_room_user.py @@ -1,5 +1,3 @@ -from __future__ import unicode_literals - # imports - module imports from frappe.model.document import Document import frappe diff --git a/frappe/chat/doctype/chat_token/chat_token.py b/frappe/chat/doctype/chat_token/chat_token.py index 30a76ef5bd..63d69a58be 100644 --- a/frappe/chat/doctype/chat_token/chat_token.py +++ b/frappe/chat/doctype/chat_token/chat_token.py @@ -2,7 +2,6 @@ # Copyright (c) 2018, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document diff --git a/frappe/chat/util/__init__.py b/frappe/chat/util/__init__.py index 15977af566..383df581cd 100644 --- a/frappe/chat/util/__init__.py +++ b/frappe/chat/util/__init__.py @@ -1,5 +1,3 @@ -from __future__ import unicode_literals - # imports - module imports from frappe.chat.util.util import ( get_user_doc, diff --git a/frappe/chat/util/test_util.py b/frappe/chat/util/test_util.py index 6d44a63d31..e2d05a4024 100644 --- a/frappe/chat/util/test_util.py +++ b/frappe/chat/util/test_util.py @@ -1,5 +1,3 @@ -from __future__ import unicode_literals - # imports - standard imports import unittest @@ -9,7 +7,6 @@ from frappe.chat.util import ( safe_json_loads ) import frappe -import six class TestChatUtil(unittest.TestCase): def test_safe_json_loads(self): @@ -20,7 +17,7 @@ class TestChatUtil(unittest.TestCase): self.assertEqual(type(number), float) string = safe_json_loads("foobar") - self.assertEqual(type(string), six.text_type) + self.assertEqual(type(string), str) array = safe_json_loads('[{ "foo": "bar" }]') self.assertEqual(type(array), list) diff --git a/frappe/chat/util/util.py b/frappe/chat/util/util.py index 82df6dd127..b7e7991c2b 100644 --- a/frappe/chat/util/util.py +++ b/frappe/chat/util/util.py @@ -1,5 +1,3 @@ -from __future__ import unicode_literals - # imports - standard imports import json from collections.abc import MutableMapping, MutableSequence, Sequence diff --git a/frappe/chat/website/__init__.py b/frappe/chat/website/__init__.py index f33f531cbf..12affd2782 100644 --- a/frappe/chat/website/__init__.py +++ b/frappe/chat/website/__init__.py @@ -1,4 +1,4 @@ -from __future__ import unicode_literals + import frappe from frappe.chat.util import filter_dict, safe_json_loads diff --git a/frappe/client.py b/frappe/client.py index a2e04452ff..66c457e893 100644 --- a/frappe/client.py +++ b/frappe/client.py @@ -1,7 +1,5 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt - -from __future__ import unicode_literals import frappe from frappe import _ import frappe.model @@ -11,7 +9,6 @@ from frappe.utils import get_safe_filters from frappe.desk.reportview import validate_args from frappe.model.db_query import check_parent_permission -from six import iteritems, string_types, integer_types ''' Handle RESTful requests that are mapped to the `/api/resource` route. @@ -86,7 +83,7 @@ def get_value(doctype, fieldname, filters=None, as_dict=True, debug=False, paren frappe.throw(_("No permission for {0}").format(doctype), frappe.PermissionError) filters = get_safe_filters(filters) - if isinstance(filters, string_types): + if isinstance(filters, str): filters = {"name": filters} try: @@ -135,7 +132,7 @@ def set_value(doctype, name, fieldname, value=None): if not value: values = fieldname - if isinstance(fieldname, string_types): + if isinstance(fieldname, str): try: values = json.loads(fieldname) except ValueError: @@ -161,7 +158,7 @@ def insert(doc=None): '''Insert a document :param doc: JSON or dict object to be inserted''' - if isinstance(doc, string_types): + if isinstance(doc, str): doc = json.loads(doc) if doc.get("parent") and doc.get("parenttype"): @@ -179,7 +176,7 @@ def insert_many(docs=None): '''Insert multiple documents :param docs: JSON or list of dict objects to be inserted in one request''' - if isinstance(docs, string_types): + if isinstance(docs, str): docs = json.loads(docs) out = [] @@ -205,7 +202,7 @@ def save(doc): '''Update (save) an existing document :param doc: JSON or dict object with the properties of the document to be updated''' - if isinstance(doc, string_types): + if isinstance(doc, str): doc = json.loads(doc) doc = frappe.get_doc(doc) @@ -228,7 +225,7 @@ def submit(doc): '''Submit a document :param doc: JSON or dict object to be submitted remotely''' - if isinstance(doc, string_types): + if isinstance(doc, str): doc = json.loads(doc) doc = frappe.get_doc(doc) @@ -266,7 +263,7 @@ def make_width_property_setter(doc): '''Set width Property Setter :param doc: Property Setter document with `width` property''' - if isinstance(doc, string_types): + if isinstance(doc, str): doc = json.loads(doc) if doc["doctype"]=="Property Setter" and doc["property"]=="width": frappe.get_doc(doc).insert(ignore_permissions = True) @@ -280,7 +277,7 @@ def bulk_update(docs): failed_docs = [] for doc in docs: try: - ddoc = {key: val for key, val in iteritems(doc) if key not in ['doctype', 'docname']} + ddoc = {key: val for key, val in doc.items() if key not in ['doctype', 'docname']} doctype = doc['doctype'] docname = doc['docname'] doc = frappe.get_doc(doctype, docname) diff --git a/frappe/commands/__init__.py b/frappe/commands/__init__.py index 61ee62d352..be9d107025 100644 --- a/frappe/commands/__init__.py +++ b/frappe/commands/__init__.py @@ -1,7 +1,6 @@ # Copyright (c) 2015, Web Notes Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals, absolute_import, print_function import sys import click import cProfile @@ -10,7 +9,7 @@ import frappe import frappe.utils import subprocess # nosec from functools import wraps -from six import StringIO +from io import StringIO from os import environ click.disable_unicode_literals_warning = True @@ -28,6 +27,10 @@ def pass_context(f): except frappe.exceptions.SiteNotSpecifiedError as e: click.secho(str(e), fg='yellow') sys.exit(1) + except frappe.exceptions.IncorrectSitePath: + site = ctx.obj.get("sites", "")[0] + click.secho(f'Site {site} does not exist!', fg='yellow') + sys.exit(1) if profile: pr.disable() diff --git a/frappe/commands/scheduler.py b/frappe/commands/scheduler.py index e9638800cd..d69ebb3024 100755 --- a/frappe/commands/scheduler.py +++ b/frappe/commands/scheduler.py @@ -1,4 +1,3 @@ -from __future__ import unicode_literals, absolute_import, print_function import click import sys import frappe diff --git a/frappe/commands/site.py b/frappe/commands/site.py index 022fe5f22d..22a063651c 100755 --- a/frappe/commands/site.py +++ b/frappe/commands/site.py @@ -203,10 +203,13 @@ def install_app(context, apps): @click.command("list-apps") +@click.option("--format", "-f", type=click.Choice(["text", "json"]), default="text") @pass_context -def list_apps(context): +def list_apps(context, format): "List apps in site" + summary_dict = {} + def fix_whitespaces(text): if site == context.sites[-1]: text = text.rstrip() @@ -235,18 +238,23 @@ def list_apps(context): ] applications_summary = "\n".join(installed_applications) summary = f"{site_title}\n{applications_summary}\n" + summary_dict[site] = [app.app_name for app in apps] else: - applications_summary = "\n".join(frappe.get_installed_apps()) + installed_applications = frappe.get_installed_apps() + applications_summary = "\n".join(installed_applications) summary = f"{site_title}\n{applications_summary}\n" + summary_dict[site] = installed_applications summary = fix_whitespaces(summary) - if applications_summary and summary: + if format == "text" and applications_summary and summary: print(summary) frappe.destroy() + if format == "json": + click.echo(frappe.as_json(summary_dict)) @click.command('add-system-manager') @click.argument('email') @@ -548,7 +556,7 @@ def move(dest_dir, site): site_dump_exists = os.path.exists(final_new_path) count = int(count or 0) + 1 - os.rename(old_path, final_new_path) + shutil.move(old_path, final_new_path) frappe.destroy() return final_new_path diff --git a/frappe/commands/translate.py b/frappe/commands/translate.py index 48a7fd1db7..68d210eaaa 100644 --- a/frappe/commands/translate.py +++ b/frappe/commands/translate.py @@ -1,4 +1,3 @@ -from __future__ import unicode_literals, absolute_import, print_function import click from frappe.commands import pass_context, get_site from frappe.exceptions import SiteNotSpecifiedError diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py index a203c8c6d9..8ef70d739c 100644 --- a/frappe/commands/utils.py +++ b/frappe/commands/utils.py @@ -16,33 +16,52 @@ from frappe.utils import get_bench_path, update_progress_bar, cint @click.command('build') @click.option('--app', help='Build assets for app') -@click.option('--make-copy', is_flag=True, default=False, help='Copy the files instead of symlinking') -@click.option('--restore', is_flag=True, default=False, help='Copy the files instead of symlinking with force') +@click.option('--apps', help='Build assets for specific apps') +@click.option('--hard-link', is_flag=True, default=False, help='Copy the files instead of symlinking') +@click.option('--make-copy', is_flag=True, default=False, help='[DEPRECATED] Copy the files instead of symlinking') +@click.option('--restore', is_flag=True, default=False, help='[DEPRECATED] Copy the files instead of symlinking with force') +@click.option('--production', is_flag=True, default=False, help='Build assets in production mode') @click.option('--verbose', is_flag=True, default=False, help='Verbose') @click.option('--force', is_flag=True, default=False, help='Force build assets instead of downloading available') -def build(app=None, make_copy=False, restore=False, verbose=False, force=False): - "Minify + concatenate JS and CSS files, build translations" - import frappe.build +def build(app=None, apps=None, hard_link=False, make_copy=False, restore=False, production=False, verbose=False, force=False): + "Compile JS and CSS source files" + from frappe.build import bundle, download_frappe_assets frappe.init('') - # don't minify in developer_mode for faster builds - no_compress = frappe.local.conf.developer_mode or False + + if not apps and app: + apps = app # dont try downloading assets if force used, app specified or running via CI - if not (force or app or os.environ.get('CI')): + if not (force or apps or os.environ.get('CI')): # skip building frappe if assets exist remotely - skip_frappe = frappe.build.download_frappe_assets(verbose=verbose) + skip_frappe = download_frappe_assets(verbose=verbose) else: skip_frappe = False - frappe.build.bundle(no_compress, app=app, make_copy=make_copy, restore=restore, verbose=verbose, skip_frappe=skip_frappe) + # don't minify in developer_mode for faster builds + development = frappe.local.conf.developer_mode or frappe.local.dev_server + mode = "development" if development else "production" + if production: + mode = "production" + + if make_copy or restore: + hard_link = make_copy or restore + click.secho( + "bench build: --make-copy and --restore options are deprecated in favour of --hard-link", + fg="yellow", + ) + + bundle(mode, apps=apps, hard_link=hard_link, verbose=verbose, skip_frappe=skip_frappe) + @click.command('watch') -def watch(): - "Watch and concatenate JS and CSS files as and when they change" - import frappe.build +@click.option('--apps', help='Watch assets for specific apps') +def watch(apps=None): + "Watch and compile JS and CSS files as and when they change" + from frappe.build import watch frappe.init('') - frappe.build.watch(True) + watch(apps) @click.command('clear-cache') @@ -96,22 +115,54 @@ def destroy_all_sessions(context, reason=None): raise SiteNotSpecifiedError @click.command('show-config') +@click.option("--format", "-f", type=click.Choice(["text", "json"]), default="text") @pass_context -def show_config(context): - "print configuration file" - print("\t\033[92m{:<50}\033[0m \033[92m{:<15}\033[0m".format('Config','Value')) - sites_path = os.path.join(frappe.utils.get_bench_path(), 'sites') - site_path = context.sites[0] - configuration = frappe.get_site_config(sites_path=sites_path, site_path=site_path) - print_config(configuration) +def show_config(context, format): + "Print configuration file to STDOUT in speified format" + if not context.sites: + raise SiteNotSpecifiedError -def print_config(config): - for conf, value in config.items(): - if isinstance(value, dict): - print_config(value) - else: - print("\t{:<50} {:<15}".format(conf, value)) + sites_config = {} + sites_path = os.getcwd() + + from frappe.utils.commands import render_table + + def transform_config(config, prefix=None): + prefix = f"{prefix}." if prefix else "" + site_config = [] + + for conf, value in config.items(): + if isinstance(value, dict): + site_config += transform_config(value, prefix=f"{prefix}{conf}") + else: + log_value = json.dumps(value) if isinstance(value, list) else value + site_config += [[f"{prefix}{conf}", log_value]] + + return site_config + + for site in context.sites: + frappe.init(site) + + if len(context.sites) != 1 and format == "text": + if context.sites.index(site) != 0: + click.echo() + click.secho(f"Site {site}", fg="yellow") + + configuration = frappe.get_site_config(sites_path=sites_path, site_path=site) + + if format == "text": + data = transform_config(configuration) + data.insert(0, ['Config','Value']) + render_table(data) + + if format == "json": + sites_config[site] = configuration + + frappe.destroy() + + if format == "json": + click.echo(frappe.as_json(sites_config)) @click.command('reset-perms') @@ -171,7 +222,7 @@ def execute(context, method, args=None, kwargs=None, profile=False): if profile: import pstats - from six import StringIO + from io import StringIO pr.disable() s = StringIO() @@ -470,6 +521,7 @@ def console(context): locals()[app] = __import__(app) except ModuleNotFoundError: failed_to_import.append(app) + all_apps.remove(app) print("Apps in this namespace:\n{}".format(", ".join(all_apps))) if failed_to_import: @@ -520,17 +572,29 @@ def run_tests(context, app=None, module=None, doctype=None, test=(), profile=Fal # Generate coverage report only for app that is being tested source_path = os.path.join(get_bench_path(), 'apps', app or 'frappe') - cov = Coverage(source=[source_path], omit=[ - '*.html', + incl = [ + '*.py', + ] + omit = [ '*.js', '*.xml', + '*.pyc', '*.css', '*.less', '*.scss', '*.vue', + '*.html', + '*/test_*', + '*/node_modules/*', '*/doctype/*/*_dashboard.py', - '*/patches/*' - ]) + '*/patches/*', + ] + + if not app or app == 'frappe': + omit.append('*/tests/*') + omit.append('*/commands/*') + + cov = Coverage(source=[source_path], omit=omit, include=incl) cov.start() ret = frappe.test_runner.main(app, module, doctype, context.verbose, tests=tests, @@ -547,12 +611,29 @@ def run_tests(context, app=None, module=None, doctype=None, test=(), profile=Fal if os.environ.get('CI'): sys.exit(ret) +@click.command('run-parallel-tests') +@click.option('--app', help="For App", default='frappe') +@click.option('--build-number', help="Build number", default=1) +@click.option('--total-builds', help="Total number of builds", default=1) +@click.option('--with-coverage', is_flag=True, help="Build coverage file") +@click.option('--use-orchestrator', is_flag=True, help="Use orchestrator to run parallel tests") +@pass_context +def run_parallel_tests(context, app, build_number, total_builds, with_coverage=False, use_orchestrator=False): + site = get_site(context) + if use_orchestrator: + from frappe.parallel_test_runner import ParallelTestWithOrchestrator + ParallelTestWithOrchestrator(app, site=site, with_coverage=with_coverage) + else: + from frappe.parallel_test_runner import ParallelTestRunner + ParallelTestRunner(app, site=site, build_number=build_number, total_builds=total_builds, with_coverage=with_coverage) @click.command('run-ui-tests') @click.argument('app') @click.option('--headless', is_flag=True, help="Run UI Test in headless mode") +@click.option('--parallel', is_flag=True, help="Run UI Test in parallel mode") +@click.option('--ci-build-id') @pass_context -def run_ui_tests(context, app, headless=False): +def run_ui_tests(context, app, headless=False, parallel=True, ci_build_id=None): "Run UI tests" site = get_site(context) app_base_path = os.path.abspath(os.path.join(frappe.get_app_path(app), '..')) @@ -584,6 +665,12 @@ def run_ui_tests(context, app, headless=False): command = '{site_env} {password_env} {cypress} {run_or_open}' formatted_command = command.format(site_env=site_env, password_env=password_env, cypress=cypress_path, run_or_open=run_or_open) + if parallel: + formatted_command += ' --parallel' + + if ci_build_id: + formatted_command += ' --ci-build-id {}'.format(ci_build_id) + click.secho("Running Cypress...", fg="yellow") frappe.commands.popen(formatted_command, cwd=app_base_path, raise_err=True) @@ -652,20 +739,27 @@ def make_app(destination, app_name): @click.command('set-config') @click.argument('key') @click.argument('value') -@click.option('-g', '--global', 'global_', is_flag = True, default = False, help = 'Set Global Site Config') -@click.option('--as-dict', is_flag=True, default=False) +@click.option('-g', '--global', 'global_', is_flag=True, default=False, help='Set value in bench config') +@click.option('-p', '--parse', is_flag=True, default=False, help='Evaluate as Python Object') +@click.option('--as-dict', is_flag=True, default=False, help='Legacy: Evaluate as Python Object') @pass_context -def set_config(context, key, value, global_ = False, as_dict=False): +def set_config(context, key, value, global_=False, parse=False, as_dict=False): "Insert/Update a value in site_config.json" from frappe.installer import update_site_config - import ast + if as_dict: + from frappe.utils.commands import warn + warn("--as-dict will be deprecated in v14. Use --parse instead", category=PendingDeprecationWarning) + parse = as_dict + + if parse: + import ast value = ast.literal_eval(value) if global_: - sites_path = os.getcwd() # big assumption. + sites_path = os.getcwd() common_site_config_path = os.path.join(sites_path, 'common_site_config.json') - update_site_config(key, value, validate = False, site_config_path = common_site_config_path) + update_site_config(key, value, validate=False, site_config_path=common_site_config_path) else: for site in context.sites: frappe.init(site=site) @@ -722,50 +816,6 @@ def rebuild_global_search(context, static_pages=False): if not context.sites: raise SiteNotSpecifiedError -@click.command('auto-deploy') -@click.argument('app') -@click.option('--migrate', is_flag=True, default=False, help='Migrate after pulling') -@click.option('--restart', is_flag=True, default=False, help='Restart after migration') -@click.option('--remote', default='upstream', help='Remote, default is "upstream"') -@pass_context -def auto_deploy(context, app, migrate=False, restart=False, remote='upstream'): - '''Pull and migrate sites that have new version''' - from frappe.utils.gitutils import get_app_branch - from frappe.utils import get_sites - - branch = get_app_branch(app) - app_path = frappe.get_app_path(app) - - # fetch - subprocess.check_output(['git', 'fetch', remote, branch], cwd = app_path) - - # get diff - if subprocess.check_output(['git', 'diff', '{0}..{1}/{0}'.format(branch, remote)], cwd = app_path): - print('Updates found for {0}'.format(app)) - if app=='frappe': - # run bench update - import shlex - subprocess.check_output(shlex.split('bench update --no-backup'), cwd = '..') - else: - updated = False - subprocess.check_output(['git', 'pull', '--rebase', remote, branch], - cwd = app_path) - # find all sites with that app - for site in get_sites(): - frappe.init(site) - if app in frappe.get_installed_apps(): - print('Updating {0}'.format(site)) - updated = True - subprocess.check_output(['bench', '--site', site, 'clear-cache'], cwd = '..') - if migrate: - subprocess.check_output(['bench', '--site', site, 'migrate'], cwd = '..') - frappe.destroy() - - if updated or restart: - subprocess.check_output(['bench', 'restart'], cwd = '..') - else: - print('No Updates') - commands = [ build, @@ -796,5 +846,6 @@ commands = [ watch, bulk_rename, add_to_email_queue, - rebuild_global_search + rebuild_global_search, + run_parallel_tests ] diff --git a/frappe/config/__init__.py b/frappe/config/__init__.py index 30be82d0df..62a877be24 100644 --- a/frappe/config/__init__.py +++ b/frappe/config/__init__.py @@ -1,6 +1,3 @@ -from __future__ import unicode_literals -import json -from six import iteritems import frappe from frappe import _ from frappe.desk.moduleview import (get_data, get_onboard_items, config_exists, get_module_link_items_from_list) diff --git a/frappe/contacts/address_and_contact.py b/frappe/contacts/address_and_contact.py index 3ca9547188..f21819ad98 100644 --- a/frappe/contacts/address_and_contact.py +++ b/frappe/contacts/address_and_contact.py @@ -1,7 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: GNU General Public License v3. See license.txt -from __future__ import unicode_literals import frappe from frappe import _ diff --git a/frappe/contacts/doctype/address/address.py b/frappe/contacts/doctype/address/address.py index 84b925d50e..bfcf91427d 100644 --- a/frappe/contacts/doctype/address/address.py +++ b/frappe/contacts/doctype/address/address.py @@ -2,7 +2,6 @@ # Copyright (c) 2015, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe import throw, _ @@ -10,15 +9,10 @@ from frappe.utils import cstr from frappe.model.document import Document from jinja2 import TemplateSyntaxError -from frappe.utils.user import is_website_user from frappe.model.naming import make_autoname from frappe.core.doctype.dynamic_link.dynamic_link import deduplicate_dynamic_links -from six import iteritems, string_types -from past.builtins import cmp from frappe.contacts.address_and_contact import set_link_title -import functools - class Address(Document): def __setup__(self): @@ -112,10 +106,13 @@ def get_default_address(doctype, name, sort_key='is_primary_address'): WHERE dl.parent = addr.name and dl.link_doctype = %s and dl.link_name = %s and ifnull(addr.disabled, 0) = 0 - """ %(sort_key, '%s', '%s'), (doctype, name)) + """ %(sort_key, '%s', '%s'), (doctype, name), as_dict=True) if out: - return sorted(out, key = functools.cmp_to_key(lambda x,y: cmp(y[1], x[1])))[0][0] + for contact in out: + if contact.get(sort_key): + return contact.name + return out[0].name else: return None @@ -141,7 +138,7 @@ def get_territory_from_address(address): if not address: return - if isinstance(address, string_types): + if isinstance(address, str): address = frappe.get_cached_doc("Address", address) territory = None @@ -174,14 +171,11 @@ def get_address_list(doctype, txt, filters, limit_start, limit_page_length = 20, def has_website_permission(doc, ptype, user, verbose=False): """Returns true if there is a related lead or contact related to this document""" contact_name = frappe.db.get_value("Contact", {"email_id": frappe.session.user}) + if contact_name: contact = frappe.get_doc('Contact', contact_name) return contact.has_common_link(doc) - lead_name = frappe.db.get_value("Lead", {"email_id": frappe.session.user}) - if lead_name: - return doc.has_link('Lead', lead_name) - return False def get_address_templates(address): @@ -214,7 +208,7 @@ def address_query(doctype, txt, searchfield, start, page_len, filters): condition = "" meta = frappe.get_meta("Address") - for fieldname, value in iteritems(filters): + for fieldname, value in filters.items(): if meta.get_field(fieldname) or fieldname in frappe.db.DEFAULT_COLUMNS: condition += " and {field}={value}".format( field=fieldname, diff --git a/frappe/contacts/doctype/address/test_address.py b/frappe/contacts/doctype/address/test_address.py index d6d4e50491..ed61b6f0ee 100644 --- a/frappe/contacts/doctype/address/test_address.py +++ b/frappe/contacts/doctype/address/test_address.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import frappe, unittest from frappe.contacts.doctype.address.address import get_address_display diff --git a/frappe/contacts/doctype/address_template/address_template.py b/frappe/contacts/doctype/address_template/address_template.py index 2ca9aebff5..2d69a792ab 100644 --- a/frappe/contacts/doctype/address_template/address_template.py +++ b/frappe/contacts/doctype/address_template/address_template.py @@ -2,7 +2,6 @@ # Copyright (c) 2015, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document from frappe.utils import cint diff --git a/frappe/contacts/doctype/address_template/test_address_template.py b/frappe/contacts/doctype/address_template/test_address_template.py index f40b56e7d9..6b519a3bb7 100644 --- a/frappe/contacts/doctype/address_template/test_address_template.py +++ b/frappe/contacts/doctype/address_template/test_address_template.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import frappe, unittest class TestAddressTemplate(unittest.TestCase): @@ -42,4 +40,4 @@ class TestAddressTemplate(unittest.TestCase): "doctype": "Address Template", "country": 'Brazil', "template": template - }).insert() \ No newline at end of file + }).insert() \ No newline at end of file diff --git a/frappe/contacts/doctype/contact/contact.py b/frappe/contacts/doctype/contact/contact.py index b3d4c6fc5c..d1dd1f1010 100644 --- a/frappe/contacts/doctype/contact/contact.py +++ b/frappe/contacts/doctype/contact/contact.py @@ -1,18 +1,13 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: GNU General Public License v3. See license.txt - -from __future__ import unicode_literals import frappe -from frappe.utils import cstr, has_gravatar, cint +from frappe.utils import cstr, has_gravatar from frappe import _ from frappe.model.document import Document from frappe.core.doctype.dynamic_link.dynamic_link import deduplicate_dynamic_links -from six import iteritems -from past.builtins import cmp from frappe.model.naming import append_number_if_name_exists from frappe.contacts.address_and_contact import set_link_title -import functools class Contact(Document): def autoname(self): @@ -120,7 +115,7 @@ class Contact(Document): if len(is_primary) > 1: frappe.throw(_("Only one {0} can be set as primary.").format(frappe.bold(frappe.unscrub(fieldname)))) - primary_number_exists = False + primary_number_exists = False for d in self.phone_nos: if d.get(field_name) == 1: primary_number_exists = True @@ -140,10 +135,13 @@ def get_default_contact(doctype, name): where dl.link_doctype=%s and dl.link_name=%s and - dl.parenttype = "Contact"''', (doctype, name)) + dl.parenttype = "Contact"''', (doctype, name), as_dict=True) if out: - return sorted(out, key = functools.cmp_to_key(lambda x,y: cmp(cint(y[1]), cint(x[1]))))[0][0] + for contact in out: + if contact.is_primary_contact: + return contact.parent + return out[0].parent else: return None diff --git a/frappe/contacts/doctype/contact/test_contact.py b/frappe/contacts/doctype/contact/test_contact.py index 4929873dc4..6c6089edeb 100644 --- a/frappe/contacts/doctype/contact/test_contact.py +++ b/frappe/contacts/doctype/contact/test_contact.py @@ -1,11 +1,10 @@ # -*- coding: utf-8 -*- # Copyright (c) 2017, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import frappe import unittest -from frappe.exceptions import ValidationError + +test_dependencies = ['Contact', 'Salutation'] class TestContact(unittest.TestCase): @@ -52,4 +51,4 @@ def create_contact(name, salutation, emails=None, phones=None, save=True): if save: doc.insert() - return doc \ No newline at end of file + return doc diff --git a/frappe/contacts/doctype/contact_email/contact_email.py b/frappe/contacts/doctype/contact_email/contact_email.py index 04e8b22989..5fc2fef316 100644 --- a/frappe/contacts/doctype/contact_email/contact_email.py +++ b/frappe/contacts/doctype/contact_email/contact_email.py @@ -2,7 +2,6 @@ # Copyright (c) 2019, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals # import frappe from frappe.model.document import Document diff --git a/frappe/contacts/doctype/contact_phone/contact_phone.py b/frappe/contacts/doctype/contact_phone/contact_phone.py index fe2f86a4bd..63f5f73cf1 100644 --- a/frappe/contacts/doctype/contact_phone/contact_phone.py +++ b/frappe/contacts/doctype/contact_phone/contact_phone.py @@ -2,7 +2,6 @@ # Copyright (c) 2019, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals # import frappe from frappe.model.document import Document diff --git a/frappe/contacts/doctype/gender/gender.py b/frappe/contacts/doctype/gender/gender.py index bfca5830c1..319800de7e 100644 --- a/frappe/contacts/doctype/gender/gender.py +++ b/frappe/contacts/doctype/gender/gender.py @@ -2,7 +2,6 @@ # Copyright (c) 2017, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals from frappe.model.document import Document class Gender(Document): diff --git a/frappe/contacts/doctype/gender/test_gender.py b/frappe/contacts/doctype/gender/test_gender.py index fbe3473bc3..071ed47df0 100644 --- a/frappe/contacts/doctype/gender/test_gender.py +++ b/frappe/contacts/doctype/gender/test_gender.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2017, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import unittest class TestGender(unittest.TestCase): diff --git a/frappe/contacts/doctype/salutation/salutation.py b/frappe/contacts/doctype/salutation/salutation.py index d9e4528c7d..d79ad66845 100644 --- a/frappe/contacts/doctype/salutation/salutation.py +++ b/frappe/contacts/doctype/salutation/salutation.py @@ -2,7 +2,6 @@ # Copyright (c) 2017, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals from frappe.model.document import Document class Salutation(Document): diff --git a/frappe/contacts/doctype/salutation/test_salutation.py b/frappe/contacts/doctype/salutation/test_salutation.py index 63d603e6a4..e2e9075855 100644 --- a/frappe/contacts/doctype/salutation/test_salutation.py +++ b/frappe/contacts/doctype/salutation/test_salutation.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2017, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import unittest class TestSalutation(unittest.TestCase): diff --git a/frappe/contacts/report/addresses_and_contacts/addresses_and_contacts.py b/frappe/contacts/report/addresses_and_contacts/addresses_and_contacts.py index 1b3982f251..bf48b6b185 100644 --- a/frappe/contacts/report/addresses_and_contacts/addresses_and_contacts.py +++ b/frappe/contacts/report/addresses_and_contacts/addresses_and_contacts.py @@ -1,8 +1,5 @@ # Copyright (c) 2013, Frappe Technologies Pvt. Ltd. and contributors # For license information, please see license.txt - -from __future__ import unicode_literals -from six import iteritems import frappe from frappe import _ @@ -58,7 +55,7 @@ def get_reference_addresses_and_contact(reference_doctype, reference_name): reference_details = get_reference_details(reference_doctype, "Address", reference_list, reference_details) reference_details = get_reference_details(reference_doctype, "Contact", reference_list, reference_details) - for reference_name, details in iteritems(reference_details): + for reference_name, details in reference_details.items(): addresses = details.get("address", []) contacts = details.get("contact", []) if not any([addresses, contacts]): diff --git a/frappe/contacts/report/addresses_and_contacts/test_addresses_and_contacts.py b/frappe/contacts/report/addresses_and_contacts/test_addresses_and_contacts.py index 9e98dcf6f6..f539722175 100644 --- a/frappe/contacts/report/addresses_and_contacts/test_addresses_and_contacts.py +++ b/frappe/contacts/report/addresses_and_contacts/test_addresses_and_contacts.py @@ -1,4 +1,4 @@ -from __future__ import unicode_literals + import frappe import frappe.defaults import unittest diff --git a/frappe/core/__init__.py b/frappe/core/__init__.py index 998a299158..f064a66c17 100644 --- a/frappe/core/__init__.py +++ b/frappe/core/__init__.py @@ -1,4 +1,2 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt - -from __future__ import unicode_literals \ No newline at end of file diff --git a/frappe/core/doctype/__init__.py b/frappe/core/doctype/__init__.py index 4dbcd0d163..0e57cb68c3 100644 --- a/frappe/core/doctype/__init__.py +++ b/frappe/core/doctype/__init__.py @@ -1,4 +1,3 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals diff --git a/frappe/core/doctype/access_log/access_log.py b/frappe/core/doctype/access_log/access_log.py index 43381e7f2e..d2fbee108b 100644 --- a/frappe/core/doctype/access_log/access_log.py +++ b/frappe/core/doctype/access_log/access_log.py @@ -3,8 +3,6 @@ # For license information, please see license.txt # imports - standard imports -from __future__ import unicode_literals - # imports - module imports import frappe from frappe.model.document import Document diff --git a/frappe/core/doctype/activity_log/activity_log.py b/frappe/core/doctype/activity_log/activity_log.py index 98dc91806d..efec0dc217 100644 --- a/frappe/core/doctype/activity_log/activity_log.py +++ b/frappe/core/doctype/activity_log/activity_log.py @@ -2,7 +2,6 @@ # Copyright (c) 2017, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals from frappe import _ from frappe.utils import get_fullname, now from frappe.model.document import Document diff --git a/frappe/core/doctype/activity_log/feed.py b/frappe/core/doctype/activity_log/feed.py index f51692fe9f..caa3cae613 100644 --- a/frappe/core/doctype/activity_log/feed.py +++ b/frappe/core/doctype/activity_log/feed.py @@ -1,13 +1,11 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: See license.txt -from __future__ import unicode_literals import frappe import frappe.permissions from frappe.utils import get_fullname from frappe import _ from frappe.core.doctype.activity_log.activity_log import add_authentication_log -from six import string_types def update_feed(doc, method=None): if frappe.flags.in_patch or frappe.flags.in_install or frappe.flags.in_import: @@ -23,7 +21,7 @@ def update_feed(doc, method=None): feed = doc.get_feed() if feed: - if isinstance(feed, string_types): + if isinstance(feed, str): feed = {"subject": feed} feed = frappe._dict(feed) diff --git a/frappe/core/doctype/activity_log/test_activity_log.py b/frappe/core/doctype/activity_log/test_activity_log.py index bd0ea08cc7..ed7b70cca1 100644 --- a/frappe/core/doctype/activity_log/test_activity_log.py +++ b/frappe/core/doctype/activity_log/test_activity_log.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import frappe import unittest import time @@ -65,12 +63,12 @@ class TestActivityLog(unittest.TestCase): frappe.local.login_manager = LoginManager() auth_log = self.get_auth_log() - self.assertEquals(auth_log.status, 'Success') + self.assertEqual(auth_log.status, 'Success') # test user logout log frappe.local.login_manager.logout() auth_log = self.get_auth_log(operation='Logout') - self.assertEquals(auth_log.status, 'Success') + self.assertEqual(auth_log.status, 'Success') # test invalid login frappe.form_dict.update({ 'pwd': 'password' }) @@ -90,4 +88,5 @@ class TestActivityLog(unittest.TestCase): def update_system_settings(args): doc = frappe.get_doc('System Settings') doc.update(args) + doc.flags.ignore_mandatory = 1 doc.save() diff --git a/frappe/core/doctype/block_module/block_module.py b/frappe/core/doctype/block_module/block_module.py index e7bb3cf045..d9723f9170 100644 --- a/frappe/core/doctype/block_module/block_module.py +++ b/frappe/core/doctype/block_module/block_module.py @@ -2,7 +2,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document diff --git a/frappe/core/doctype/comment/comment.py b/frappe/core/doctype/comment/comment.py index ad5d60500b..e29bae25a2 100644 --- a/frappe/core/doctype/comment/comment.py +++ b/frappe/core/doctype/comment/comment.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and contributors # For license information, please see license.txt - -from __future__ import unicode_literals, absolute_import import frappe from frappe import _ import json diff --git a/frappe/core/doctype/comment/test_comment.py b/frappe/core/doctype/comment/test_comment.py index 3cf8fbaa3f..13db92e7a8 100644 --- a/frappe/core/doctype/comment/test_comment.py +++ b/frappe/core/doctype/comment/test_comment.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import frappe, json import unittest diff --git a/frappe/core/doctype/communication/communication.py b/frappe/core/doctype/communication/communication.py index 5ebf714645..17b1290776 100644 --- a/frappe/core/doctype/communication/communication.py +++ b/frappe/core/doctype/communication/communication.py @@ -1,29 +1,31 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals, absolute_import from collections import Counter import frappe from frappe import _ from frappe.model.document import Document from frappe.utils import validate_email_address, strip_html, cstr, time_diff_in_seconds -from frappe.core.doctype.communication.email import validate_email, notify, _notify +from frappe.core.doctype.communication.email import validate_email +from frappe.core.doctype.communication.mixins import CommunicationEmailMixin from frappe.core.utils import get_parent_doc from frappe.utils.bot import BotReply -from frappe.utils import parse_addr +from frappe.utils import parse_addr, split_emails from frappe.core.doctype.comment.comment import update_comment_in_doc from email.utils import parseaddr -from six.moves.urllib.parse import unquote +from urllib.parse import unquote from frappe.utils.user import is_system_user from frappe.contacts.doctype.contact.contact import get_contact_name from frappe.automation.doctype.assignment_rule.assignment_rule import apply as apply_assignment_rule exclude_from_linked_with = True -class Communication(Document): +class Communication(Document, CommunicationEmailMixin): + """Communication represents an external communication like Email. + """ no_feed_on_delete = True + DOCTYPE = 'Communication' - """Communication represents an external communication like Email.""" def onload(self): """create email flag queue""" if self.communication_type == "Communication" and self.communication_medium == "Email" \ @@ -124,6 +126,45 @@ class Communication(Document): if self.communication_type == "Communication": self.notify_change('delete') + @property + def sender_mailid(self): + return parse_addr(self.sender)[1] if self.sender else "" + + @staticmethod + def _get_emails_list(emails=None, exclude_displayname = False): + """Returns list of emails from given email string. + + * Removes duplicate mailids + * Removes display name from email address if exclude_displayname is True + """ + emails = split_emails(emails) if isinstance(emails, str) else (emails or []) + if exclude_displayname: + return [email.lower() for email in set([parse_addr(email)[1] for email in emails]) if email] + return [email.lower() for email in set(emails) if email] + + def to_list(self, exclude_displayname = True): + """Returns to list. + """ + return self._get_emails_list(self.recipients, exclude_displayname=exclude_displayname) + + def cc_list(self, exclude_displayname = True): + """Returns cc list. + """ + return self._get_emails_list(self.cc, exclude_displayname=exclude_displayname) + + def bcc_list(self, exclude_displayname = True): + """Returns bcc list. + """ + return self._get_emails_list(self.bcc, exclude_displayname=exclude_displayname) + + def get_attachments(self): + attachments = frappe.get_all( + "File", + fields=["name", "file_name", "file_url", "is_private"], + filters = {"attached_to_name": self.name, "attached_to_doctype": self.DOCTYPE} + ) + return attachments + def notify_change(self, action): frappe.publish_realtime('update_docinfo_for_{}_{}'.format(self.reference_doctype, self.reference_name), { 'doc': self.as_dict(), @@ -149,6 +190,23 @@ class Communication(Document): self.email_status = "Spam" + @classmethod + def find(cls, name, ignore_error=False): + try: + return frappe.get_doc(cls.DOCTYPE, name) + except frappe.DoesNotExistError: + if ignore_error: + return + raise + + @classmethod + def find_one_by_filters(cls, *, order_by=None, **kwargs): + name = frappe.db.get_value(cls.DOCTYPE, kwargs, order_by=order_by) + return cls.find(name) if name else None + + def update_db(self, **kwargs): + frappe.db.set_value(self.DOCTYPE, self.name, kwargs) + def set_sender_full_name(self): if not self.sender_full_name and self.sender: if self.sender == "Administrator": @@ -180,36 +238,6 @@ class Communication(Document): if not self.sender_full_name: self.sender_full_name = sender_email - def send(self, print_html=None, print_format=None, attachments=None, - send_me_a_copy=False, recipients=None): - """Send communication via Email. - - :param print_html: Send given value as HTML attachment. - :param print_format: Attach print format of parent document.""" - - self.send_me_a_copy = send_me_a_copy - self.notify(print_html, print_format, attachments, recipients) - - def notify(self, print_html=None, print_format=None, attachments=None, - recipients=None, cc=None, bcc=None,fetched_from_email_account=False): - """Calls a delayed task 'sendmail' that enqueus email in Email Queue queue - - :param print_html: Send given value as HTML attachment - :param print_format: Attach print format of parent document - :param attachments: A list of filenames that should be attached when sending this email - :param recipients: Email recipients - :param cc: Send email as CC to - :param fetched_from_email_account: True when pulling email, the notification shouldn't go to the main recipient - - """ - notify(self, print_html, print_format, attachments, recipients, cc, bcc, - fetched_from_email_account) - - def _notify(self, print_html=None, print_format=None, attachments=None, - recipients=None, cc=None, bcc=None): - - _notify(self, print_html, print_format, attachments, recipients, cc, bcc) - def bot_reply(self): if self.comment_type == 'Bot' and self.communication_type == 'Chat': reply = BotReply().get_reply(self.content) @@ -485,4 +513,5 @@ def set_avg_response_time(parent, communication): response_times.append(response_time) if response_times: avg_response_time = sum(response_times) / len(response_times) - parent.db_set("avg_response_time", avg_response_time) \ No newline at end of file + parent.db_set("avg_response_time", avg_response_time) + diff --git a/frappe/core/doctype/communication/email.py b/frappe/core/doctype/communication/email.py index 731cb85d7c..d35c118550 100755 --- a/frappe/core/doctype/communication/email.py +++ b/frappe/core/doctype/communication/email.py @@ -1,9 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals, absolute_import -from six.moves import range -from six import string_types import frappe import json from email.utils import formataddr @@ -16,6 +13,11 @@ import time from frappe import _ from frappe.utils.background_jobs import enqueue +OUTGOING_EMAIL_ACCOUNT_MISSING = _(""" + Unable to send mail because of a missing email account. + Please setup default Email Account from Setup > Email > Email Account +""") + @frappe.whitelist() def make(doctype=None, name=None, content=None, subject=None, sent_or_received = "Sent", sender=None, sender_full_name=None, recipients=None, communication_medium="Email", send_email=False, @@ -39,7 +41,6 @@ def make(doctype=None, name=None, content=None, subject=None, sent_or_received = :param send_me_a_copy: Send a copy to the sender (default **False**). :param email_template: Template which is used to compose mail . """ - is_error_report = (doctype=="User" and name==frappe.session.user and subject=="Error Report") send_me_a_copy = cint(send_me_a_copy) @@ -77,7 +78,7 @@ def make(doctype=None, name=None, content=None, subject=None, sent_or_received = comm.save(ignore_permissions=True) - if isinstance(attachments, string_types): + if isinstance(attachments, str): attachments = json.loads(attachments) # if not committed, delayed task doesn't find the communication @@ -87,12 +88,16 @@ def make(doctype=None, name=None, content=None, subject=None, sent_or_received = frappe.db.commit() if cint(send_email): - frappe.flags.print_letterhead = cint(print_letterhead) - comm.send(print_html, print_format, attachments, send_me_a_copy=send_me_a_copy) + if not comm.get_outgoing_email_account(): + frappe.throw(msg=OUTGOING_EMAIL_ACCOUNT_MISSING, exc=frappe.OutgoingEmailError) + comm.send_email(print_html=print_html, print_format=print_format, + send_me_a_copy=send_me_a_copy, print_letterhead=print_letterhead) + + emails_not_sent_to = comm.exclude_emails_list(include_sender=send_me_a_copy) return { "name": comm.name, - "emails_not_sent_to": ", ".join(comm.emails_not_sent_to) if hasattr(comm, "emails_not_sent_to") else None + "emails_not_sent_to": ", ".join(emails_not_sent_to or []) } def validate_email(doc): @@ -113,261 +118,25 @@ def validate_email(doc): # validate sender -def notify(doc, print_html=None, print_format=None, attachments=None, - recipients=None, cc=None, bcc=None, fetched_from_email_account=False): - """Calls a delayed task 'sendmail' that enqueus email in Email Queue queue - - :param print_html: Send given value as HTML attachment - :param print_format: Attach print format of parent document - :param attachments: A list of filenames that should be attached when sending this email - :param recipients: Email recipients - :param cc: Send email as CC to - :param bcc: Send email as BCC to - :param fetched_from_email_account: True when pulling email, the notification shouldn't go to the main recipient - - """ - recipients, cc, bcc = get_recipients_cc_and_bcc(doc, recipients, cc, bcc, - fetched_from_email_account=fetched_from_email_account) - - if not recipients and not cc: - return - - doc.emails_not_sent_to = set(doc.all_email_addresses) - set(doc.sent_email_addresses) - - if frappe.flags.in_test: - # for test cases, run synchronously - doc._notify(print_html=print_html, print_format=print_format, attachments=attachments, - recipients=recipients, cc=cc, bcc=None) - else: - enqueue(sendmail, queue="default", timeout=300, event="sendmail", - communication_name=doc.name, - print_html=print_html, print_format=print_format, attachments=attachments, - recipients=recipients, cc=cc, bcc=bcc, lang=frappe.local.lang, - session=frappe.local.session, print_letterhead=frappe.flags.print_letterhead) - -def _notify(doc, print_html=None, print_format=None, attachments=None, - recipients=None, cc=None, bcc=None): - - prepare_to_notify(doc, print_html, print_format, attachments) - - if doc.outgoing_email_account.send_unsubscribe_message: - unsubscribe_message = _("Leave this conversation") - else: - unsubscribe_message = "" - - frappe.sendmail( - recipients=(recipients or []), - cc=(cc or []), - bcc=(bcc or []), - expose_recipients="header", - sender=doc.sender, - reply_to=doc.incoming_email_account, - subject=doc.subject, - content=doc.content, - reference_doctype=doc.reference_doctype, - reference_name=doc.reference_name, - attachments=doc.attachments, - message_id=doc.message_id, - unsubscribe_message=unsubscribe_message, - delayed=True, - communication=doc.name, - read_receipt=doc.read_receipt, - is_notification=True if doc.sent_or_received =="Received" else False, - print_letterhead=frappe.flags.print_letterhead - ) - -def get_recipients_cc_and_bcc(doc, recipients, cc, bcc, fetched_from_email_account=False): - doc.all_email_addresses = [] - doc.sent_email_addresses = [] - doc.previous_email_sender = None - - if not recipients: - recipients = get_recipients(doc, fetched_from_email_account=fetched_from_email_account) - - if not cc: - cc = get_cc(doc, recipients, fetched_from_email_account=fetched_from_email_account) - - if not bcc: - bcc = get_bcc(doc, recipients, fetched_from_email_account=fetched_from_email_account) - - if fetched_from_email_account: - # email was already sent to the original recipient by the sender's email service - original_recipients, recipients = recipients, [] - - # send email to the sender of the previous email in the thread which this email is a reply to - #provides erratic results and can send external - #if doc.previous_email_sender: - # recipients.append(doc.previous_email_sender) - - # cc that was received in the email - original_cc = split_emails(doc.cc) - - # don't cc to people who already received the mail from sender's email service - cc = list(set(cc) - set(original_cc) - set(original_recipients)) - remove_administrator_from_email_list(cc) - - original_bcc = split_emails(doc.bcc) - bcc = list(set(bcc) - set(original_bcc) - set(original_recipients)) - remove_administrator_from_email_list(bcc) - - remove_administrator_from_email_list(recipients) - - return recipients, cc, bcc - -def remove_administrator_from_email_list(email_list): - administrator_email = list(filter(lambda emails: "Administrator" in emails, email_list)) - if administrator_email: - email_list.remove(administrator_email[0]) - -def prepare_to_notify(doc, print_html=None, print_format=None, attachments=None): - """Prepare to make multipart MIME Email - - :param print_html: Send given value as HTML attachment. - :param print_format: Attach print format of parent document.""" - - view_link = frappe.utils.cint(frappe.db.get_value("System Settings", "System Settings", "attach_view_link")) - - if print_format and view_link: - doc.content += get_attach_link(doc, print_format) - - set_incoming_outgoing_accounts(doc) - - if not doc.sender: - doc.sender = doc.outgoing_email_account.email_id - - if not doc.sender_full_name: - doc.sender_full_name = doc.outgoing_email_account.name or _("Notification") - - if doc.sender: - # combine for sending to get the format 'Jane ' - doc.sender = get_formatted_email(doc.sender_full_name, mail=doc.sender) - - doc.attachments = [] - - if print_html or print_format: - doc.attachments.append({"print_format_attachment":1, "doctype":doc.reference_doctype, - "name":doc.reference_name, "print_format":print_format, "html":print_html}) - - if attachments: - if isinstance(attachments, string_types): - attachments = json.loads(attachments) - - for a in attachments: - if isinstance(a, string_types): - # is it a filename? - try: - # check for both filename and file id - file_id = frappe.db.get_list('File', or_filters={'file_name': a, 'name': a}, limit=1) - if not file_id: - frappe.throw(_("Unable to find attachment {0}").format(a)) - file_id = file_id[0]['name'] - _file = frappe.get_doc("File", file_id) - _file.get_content() - # these attachments will be attached on-demand - # and won't be stored in the message - doc.attachments.append({"fid": file_id}) - except IOError: - frappe.throw(_("Unable to find attachment {0}").format(a)) - else: - doc.attachments.append(a) - def set_incoming_outgoing_accounts(doc): - doc.incoming_email_account = doc.outgoing_email_account = None + from frappe.email.doctype.email_account.email_account import EmailAccount + incoming_email_account = EmailAccount.find_incoming( + match_by_email=doc.sender, match_by_doctype=doc.reference_doctype) + doc.incoming_email_account = incoming_email_account.email_id if incoming_email_account else None - if not doc.incoming_email_account and doc.sender: - doc.incoming_email_account = frappe.db.get_value("Email Account", - {"email_id": doc.sender, "enable_incoming": 1}, "email_id") - - if not doc.incoming_email_account and doc.reference_doctype: - doc.incoming_email_account = frappe.db.get_value("Email Account", - {"append_to": doc.reference_doctype, }, "email_id") - - if not doc.incoming_email_account: - doc.incoming_email_account = frappe.db.get_value("Email Account", - {"default_incoming": 1, "enable_incoming": 1}, "email_id") - - doc.outgoing_email_account = frappe.email.smtp.get_outgoing_email_account(raise_exception_not_set=False, - append_to=doc.doctype, sender=doc.sender) + doc.outgoing_email_account = EmailAccount.find_outgoing( + match_by_email=doc.sender, match_by_doctype=doc.reference_doctype) if doc.sent_or_received == "Sent": doc.db_set("email_account", doc.outgoing_email_account.name) -def get_recipients(doc, fetched_from_email_account=False): - """Build a list of email addresses for To""" - # [EDGE CASE] doc.recipients can be None when an email is sent as BCC - recipients = split_emails(doc.recipients) - - #if fetched_from_email_account and doc.in_reply_to: - # add sender of previous reply - #doc.previous_email_sender = frappe.db.get_value("Communication", doc.in_reply_to, "sender") - #recipients.append(doc.previous_email_sender) - - if recipients: - recipients = filter_email_list(doc, recipients, []) - - return recipients - -def get_cc(doc, recipients=None, fetched_from_email_account=False): - """Build a list of email addresses for CC""" - # get a copy of CC list - cc = split_emails(doc.cc) - - if doc.reference_doctype and doc.reference_name: - if fetched_from_email_account: - # if it is a fetched email, add follows to CC - cc.append(get_owner_email(doc)) - cc += get_assignees(doc) - - if getattr(doc, "send_me_a_copy", False) and doc.sender not in cc: - cc.append(doc.sender) - - if cc: - # exclude unfollows, recipients and unsubscribes - exclude = [] #added to remove account check - exclude += [d[0] for d in frappe.db.get_all("User", ["email"], {"thread_notify": 0}, as_list=True)] - exclude += [(parse_addr(email)[1] or "").lower() for email in recipients] - - if fetched_from_email_account: - # exclude sender when pulling email - exclude += [parse_addr(doc.sender)[1]] - - if doc.reference_doctype and doc.reference_name: - exclude += [d[0] for d in frappe.db.get_all("Email Unsubscribe", ["email"], - {"reference_doctype": doc.reference_doctype, "reference_name": doc.reference_name}, as_list=True)] - - cc = filter_email_list(doc, cc, exclude, is_cc=True) - - return cc - -def get_bcc(doc, recipients=None, fetched_from_email_account=False): - """Build a list of email addresses for BCC""" - bcc = split_emails(doc.bcc) - - if bcc: - exclude = [] - exclude += [d[0] for d in frappe.db.get_all("User", ["email"], {"thread_notify": 0}, as_list=True)] - exclude += [(parse_addr(email)[1] or "").lower() for email in recipients] - - if fetched_from_email_account: - # exclude sender when pulling email - exclude += [parse_addr(doc.sender)[1]] - - if doc.reference_doctype and doc.reference_name: - exclude += [d[0] for d in frappe.db.get_all("Email Unsubscribe", ["email"], - {"reference_doctype": doc.reference_doctype, "reference_name": doc.reference_name}, as_list=True)] - - bcc = filter_email_list(doc, bcc, exclude, is_bcc=True) - - return bcc - def add_attachments(name, attachments): '''Add attachments to the given Communication''' # loop through attachments for a in attachments: - if isinstance(a, string_types): + if isinstance(a, str): attach = frappe.db.get_value("File", {"name":a}, ["file_name", "file_url", "is_private"], as_dict=1) - # save attachments to new doc _file = frappe.get_doc({ "doctype": "File", @@ -379,103 +148,6 @@ def add_attachments(name, attachments): }) _file.save(ignore_permissions=True) -def filter_email_list(doc, email_list, exclude, is_cc=False, is_bcc=False): - # temp variables - filtered = [] - email_address_list = [] - - for email in list(set(email_list)): - email_address = (parse_addr(email)[1] or "").lower() - if not email_address: - continue - - # this will be used to eventually find email addresses that aren't sent to - doc.all_email_addresses.append(email_address) - - if (email in exclude) or (email_address in exclude): - continue - - if is_cc: - is_user_enabled = frappe.db.get_value("User", email_address, "enabled") - if is_user_enabled==0: - # don't send to disabled users - continue - - if is_bcc: - is_user_enabled = frappe.db.get_value("User", email_address, "enabled") - if is_user_enabled==0: - continue - - # make sure of case-insensitive uniqueness of email address - if email_address not in email_address_list: - # append the full email i.e. "Human " - filtered.append(email) - email_address_list.append(email_address) - - doc.sent_email_addresses.extend(email_address_list) - - return filtered - -def get_owner_email(doc): - owner = get_parent_doc(doc).owner - return get_formatted_email(owner) or owner - -def get_assignees(doc): - return [( get_formatted_email(d.owner) or d.owner ) for d in - frappe.db.get_all("ToDo", filters={ - "reference_type": doc.reference_doctype, - "reference_name": doc.reference_name, - "status": "Open" - }, fields=["owner"]) - ] - -def get_attach_link(doc, print_format): - """Returns public link for the attachment via `templates/emails/print_link.html`.""" - return frappe.get_template("templates/emails/print_link.html").render({ - "url": get_url(), - "doctype": doc.reference_doctype, - "name": doc.reference_name, - "print_format": print_format, - "key": get_parent_doc(doc).get_signature() - }) - -def sendmail(communication_name, print_html=None, print_format=None, attachments=None, - recipients=None, cc=None, bcc=None, lang=None, session=None, print_letterhead=None): - try: - - if lang: - frappe.local.lang = lang - - if session: - # hack to enable access to private files in PDF - session['data'] = frappe._dict(session['data']) - frappe.local.session.update(session) - - if print_letterhead: - frappe.flags.print_letterhead = print_letterhead - - # upto 3 retries - for i in range(3): - try: - communication = frappe.get_doc("Communication", communication_name) - communication._notify(print_html=print_html, print_format=print_format, attachments=attachments, - recipients=recipients, cc=cc, bcc=bcc) - - except frappe.db.InternalError as e: - # deadlock, try again - if frappe.db.is_deadlocked(e): - frappe.db.rollback() - time.sleep(1) - continue - else: - raise - else: - break - - except: - traceback = frappe.log_error("frappe.core.doctype.communication.email.sendmail") - raise - @frappe.whitelist(allow_guest=True) def mark_email_as_seen(name=None): try: diff --git a/frappe/core/doctype/communication/mixins.py b/frappe/core/doctype/communication/mixins.py new file mode 100644 index 0000000000..82a47d24d9 --- /dev/null +++ b/frappe/core/doctype/communication/mixins.py @@ -0,0 +1,297 @@ +import frappe +from frappe import _ +from frappe.core.utils import get_parent_doc +from frappe.utils import parse_addr, get_formatted_email, get_url +from frappe.email.doctype.email_account.email_account import EmailAccount + +class CommunicationEmailMixin: + """Mixin class to handle communication mails. + """ + def is_email_communication(self): + return self.communication_type=="Communication" and self.communication_medium == "Email" + + def get_owner(self): + """Get owner of the communication docs parent. + """ + parent_doc = get_parent_doc(self) + return parent_doc.owner if parent_doc else None + + def get_all_email_addresses(self, exclude_displayname=False): + """Get all Email addresses mentioned in the doc along with display name. + """ + return self.to_list(exclude_displayname=exclude_displayname) + \ + self.cc_list(exclude_displayname=exclude_displayname) + \ + self.bcc_list(exclude_displayname=exclude_displayname) + + def get_email_with_displayname(self, email_address): + """Returns email address after adding displayname. + """ + display_name, email = parse_addr(email_address) + if display_name and display_name != email: + return email_address + + # emailid to emailid with display name map. + email_map = {parse_addr(email)[1]: email for email in self.get_all_email_addresses()} + return email_map.get(email, email) + + def mail_recipients(self, is_inbound_mail_communcation=False): + """Build to(recipient) list to send an email. + """ + # Incase of inbound mail, recipients already received the mail, no need to send again. + if is_inbound_mail_communcation: + return [] + + if hasattr(self, '_final_recipients'): + return self._final_recipients + + to = self.to_list() + self._final_recipients = list(filter(lambda id: id != 'Administrator', to)) + return self._final_recipients + + def get_mail_recipients_with_displayname(self, is_inbound_mail_communcation=False): + """Build to(recipient) list to send an email including displayname in email. + """ + to_list = self.mail_recipients(is_inbound_mail_communcation=is_inbound_mail_communcation) + return [self.get_email_with_displayname(email) for email in to_list] + + def mail_cc(self, is_inbound_mail_communcation=False, include_sender = False): + """Build cc list to send an email. + + * if email copy is requested by sender, then add sender to CC. + * If this doc is created through inbound mail, then add doc owner to cc list + * remove all the thread_notify disabled users. + * Make sure that all users enabled in the system + * Remove admin from email list + + * FixMe: Removed adding TODO owners to cc list. Check if that is needed. + """ + if hasattr(self, '_final_cc'): + return self._final_cc + + cc = self.cc_list() + + # Need to inform parent document owner incase communication is created through inbound mail + if include_sender: + cc.append(self.sender_mailid) + if is_inbound_mail_communcation: + cc.append(self.get_owner()) + cc = set(cc) - {self.sender_mailid} + + cc = set(cc) - set(self.filter_thread_notification_disbled_users(cc)) + cc = cc - set(self.mail_recipients(is_inbound_mail_communcation=is_inbound_mail_communcation)) + cc = cc - set(self.filter_disabled_users(cc)) + + # # Incase of inbound mail, to and cc already received the mail, no need to send again. + if is_inbound_mail_communcation: + cc = cc - set(self.cc_list() + self.to_list()) + + self._final_cc = list(filter(lambda id: id != 'Administrator', cc)) + return self._final_cc + + def get_mail_cc_with_displayname(self, is_inbound_mail_communcation=False, include_sender = False): + cc_list = self.mail_cc(is_inbound_mail_communcation=False, include_sender = False) + return [self.get_email_with_displayname(email) for email in cc_list] + + def mail_bcc(self, is_inbound_mail_communcation=False): + """ + * Thread_notify check + * Email unsubscribe list + * User must be enabled in the system + * remove_administrator_from_email_list + """ + if hasattr(self, '_final_bcc'): + return self._final_bcc + + bcc = set(self.bcc_list()) + if is_inbound_mail_communcation: + bcc = bcc - {self.sender_mailid} + bcc = bcc - set(self.filter_thread_notification_disbled_users(bcc)) + bcc = bcc - set(self.mail_recipients(is_inbound_mail_communcation=is_inbound_mail_communcation)) + bcc = bcc - set(self.filter_disabled_users(bcc)) + + # Incase of inbound mail, to and cc & bcc already received the mail, no need to send again. + if is_inbound_mail_communcation: + bcc = bcc - set(self.bcc_list() + self.to_list()) + + self._final_bcc = list(filter(lambda id: id != 'Administrator', bcc)) + return self._final_bcc + + def get_mail_bcc_with_displayname(self, is_inbound_mail_communcation=False): + bcc_list = self.mail_bcc(is_inbound_mail_communcation=is_inbound_mail_communcation) + return [self.get_email_with_displayname(email) for email in bcc_list] + + def mail_sender(self): + email_account = self.get_outgoing_email_account() + if not self.sender_mailid and email_account: + return email_account.email_id + return self.sender_mailid + + def mail_sender_fullname(self): + email_account = self.get_outgoing_email_account() + if not self.sender_full_name: + return (email_account and email_account.name) or _("Notification") + return self.sender_full_name + + def get_mail_sender_with_displayname(self): + return get_formatted_email(self.mail_sender_fullname(), mail=self.mail_sender()) + + def get_content(self, print_format=None): + if print_format: + return self.content + self.get_attach_link(print_format) + return self.content + + def get_attach_link(self, print_format): + """Returns public link for the attachment via `templates/emails/print_link.html`.""" + return frappe.get_template("templates/emails/print_link.html").render({ + "url": get_url(), + "doctype": self.reference_doctype, + "name": self.reference_name, + "print_format": print_format, + "key": get_parent_doc(self).get_signature() + }) + + def get_outgoing_email_account(self): + if not hasattr(self, '_outgoing_email_account'): + if self.email_account: + self._outgoing_email_account = EmailAccount.find(self.email_account) + else: + self._outgoing_email_account = EmailAccount.find_outgoing( + match_by_email=self.sender_mailid, + match_by_doctype=self.reference_doctype + ) + + if self.sent_or_received == "Sent" and self._outgoing_email_account: + self.db_set("email_account", self._outgoing_email_account.name) + + return self._outgoing_email_account + + def get_incoming_email_account(self): + if not hasattr(self, '_incoming_email_account'): + self._incoming_email_account = EmailAccount.find_incoming( + match_by_email=self.sender_mailid, + match_by_doctype=self.reference_doctype + ) + return self._incoming_email_account + + def mail_attachments(self, print_format=None, print_html=None): + final_attachments = [] + + if print_format and print_html: + d = {'print_format': print_format, 'print_html': print_html, 'print_format_attachment': 1, + 'doctype': self.reference_doctype, 'name': self.reference_name} + final_attachments.append(d) + + for a in self.get_attachments() or []: + final_attachments.append({"fid": a['name']}) + + return final_attachments + + def get_unsubscribe_message(self): + email_account = self.get_outgoing_email_account() + if email_account and email_account.send_unsubscribe_message: + return _("Leave this conversation") + return '' + + def exclude_emails_list(self, is_inbound_mail_communcation=False, include_sender=False): + """List of mail id's excluded while sending mail. + """ + all_ids = self.get_all_email_addresses(exclude_displayname=True) + final_ids = self.mail_recipients(is_inbound_mail_communcation = is_inbound_mail_communcation) + \ + self.mail_bcc(is_inbound_mail_communcation = is_inbound_mail_communcation) + \ + self.mail_cc(is_inbound_mail_communcation = is_inbound_mail_communcation, include_sender=include_sender) + return set(all_ids) - set(final_ids) + + @staticmethod + def filter_thread_notification_disbled_users(emails): + """Filter users based on notifications for email threads setting is disabled. + """ + if not emails: + return [] + + disabled_users = frappe.db.sql_list(""" + SELECT + email + FROM + `tabUser` + where + email in %(emails)s + and + thread_notify=0 + """, {'emails': tuple(emails)}) + return disabled_users + + @staticmethod + def filter_disabled_users(emails): + """ + """ + if not emails: + return [] + + disabled_users = frappe.db.sql_list(""" + SELECT + email + FROM + `tabUser` + where + email in %(emails)s + and + enabled=0 + """, {'emails': tuple(emails)}) + return disabled_users + + def sendmail_input_dict(self, print_html=None, print_format=None, + send_me_a_copy=None, print_letterhead=None, is_inbound_mail_communcation=None): + + outgoing_email_account = self.get_outgoing_email_account() + if not outgoing_email_account: + return {} + + recipients = self.get_mail_recipients_with_displayname( + is_inbound_mail_communcation=is_inbound_mail_communcation + ) + cc = self.get_mail_cc_with_displayname( + is_inbound_mail_communcation=is_inbound_mail_communcation, + include_sender = send_me_a_copy + ) + bcc = self.get_mail_bcc_with_displayname( + is_inbound_mail_communcation=is_inbound_mail_communcation + ) + + if not (recipients or cc): + return {} + + final_attachments = self.mail_attachments(print_format=print_format, print_html=print_html) + incoming_email_account = self.get_incoming_email_account() + return { + "recipients": recipients, + "cc": cc, + "bcc": bcc, + "expose_recipients": "header", + "sender": self.get_mail_sender_with_displayname(), + "reply_to": incoming_email_account and incoming_email_account.email_id, + "subject": self.subject, + "content": self.get_content(print_format=print_format), + "reference_doctype": self.reference_doctype, + "reference_name": self.reference_name, + "attachments": final_attachments, + "message_id": self.message_id, + "unsubscribe_message": self.get_unsubscribe_message(), + "delayed": True, + "communication": self.name, + "read_receipt": self.read_receipt, + "is_notification": (self.sent_or_received =="Received" and True) or False, + "print_letterhead": print_letterhead + } + + def send_email(self, print_html=None, print_format=None, + send_me_a_copy=None, print_letterhead=None, is_inbound_mail_communcation=None): + input_dict = self.sendmail_input_dict( + print_html=print_html, + print_format=print_format, + send_me_a_copy=send_me_a_copy, + print_letterhead=print_letterhead, + is_inbound_mail_communcation=is_inbound_mail_communcation + ) + + if input_dict: + frappe.sendmail(**input_dict) diff --git a/frappe/core/doctype/communication/test_communication.py b/frappe/core/doctype/communication/test_communication.py index 6df90baaae..d50a4db88a 100644 --- a/frappe/core/doctype/communication/test_communication.py +++ b/frappe/core/doctype/communication/test_communication.py @@ -1,12 +1,12 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # See license.txt -from __future__ import unicode_literals +import unittest +from urllib.parse import quote import frappe -import unittest -from six.moves.urllib.parse import quote -test_records = frappe.get_test_records('Communication') +from frappe.email.doctype.email_queue.email_queue import EmailQueue +test_records = frappe.get_test_records('Communication') class TestCommunication(unittest.TestCase): @@ -201,6 +201,70 @@ class TestCommunication(unittest.TestCase): self.assertIn(("Note", note.name), doc_links) +class TestCommunicationEmailMixin(unittest.TestCase): + def new_communication(self, recipients=None, cc=None, bcc=None): + recipients = ', '.join(recipients or []) + cc = ', '.join(cc or []) + bcc = ', '.join(bcc or []) + + comm = frappe.get_doc({ + "doctype": "Communication", + "communication_type": "Communication", + "communication_medium": "Email", + "content": "Test content", + "recipients": recipients, + "cc": cc, + "bcc": bcc + }).insert(ignore_permissions=True) + return comm + + def new_user(self, email, **user_data): + user_data.setdefault('first_name', 'first_name') + user = frappe.new_doc('User') + user.email = email + user.update(user_data) + user.insert(ignore_permissions=True, ignore_if_duplicate=True) + return user + + def test_recipients(self): + to_list = ['to@test.com', 'receiver ', 'to@test.com'] + comm = self.new_communication(recipients = to_list) + res = comm.get_mail_recipients_with_displayname() + self.assertCountEqual(res, ['to@test.com', 'receiver ']) + comm.delete() + + def test_cc(self): + to_list = ['to@test.com'] + cc_list = ['cc+1@test.com', 'cc ', 'to@test.com'] + user = self.new_user(email='cc+1@test.com', thread_notify=0) + comm = self.new_communication(recipients=to_list, cc=cc_list) + res = comm.get_mail_cc_with_displayname() + self.assertCountEqual(res, ['cc ']) + user.delete() + comm.delete() + + def test_bcc(self): + bcc_list = ['bcc+1@test.com', 'cc ', ] + user = self.new_user(email='bcc+2@test.com', enabled=0) + comm = self.new_communication(bcc=bcc_list) + res = comm.get_mail_bcc_with_displayname() + self.assertCountEqual(res, ['bcc+1@test.com']) + user.delete() + comm.delete() + + def test_sendmail(self): + to_list = ['to '] + cc_list = ['cc ', 'cc '] + + comm = self.new_communication(recipients=to_list, cc=cc_list) + comm.send_email() + doc = EmailQueue.find_one_by_filters(communication=comm.name) + mail_receivers = [each.recipient for each in doc.recipients] + self.assertIsNotNone(doc) + self.assertCountEqual(to_list+cc_list, mail_receivers) + doc.delete() + comm.delete() + def create_email_account(): frappe.delete_doc_if_exists("Email Account", "_Test Comm Account 1") @@ -231,4 +295,4 @@ def create_email_account(): "enable_automatic_linking": 1 }).insert(ignore_permissions=True) - return email_account \ No newline at end of file + return email_account diff --git a/frappe/core/doctype/communication_link/communication_link.py b/frappe/core/doctype/communication_link/communication_link.py index d1612ef57e..d3307d1d32 100644 --- a/frappe/core/doctype/communication_link/communication_link.py +++ b/frappe/core/doctype/communication_link/communication_link.py @@ -2,7 +2,6 @@ # Copyright (c) 2019, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document diff --git a/frappe/core/doctype/custom_docperm/custom_docperm.py b/frappe/core/doctype/custom_docperm/custom_docperm.py index cce9788b73..225f5db79b 100644 --- a/frappe/core/doctype/custom_docperm/custom_docperm.py +++ b/frappe/core/doctype/custom_docperm/custom_docperm.py @@ -2,7 +2,6 @@ # Copyright (c) 2015, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document diff --git a/frappe/core/doctype/custom_docperm/test_custom_docperm.py b/frappe/core/doctype/custom_docperm/test_custom_docperm.py index bd6e17ccc9..6e0c82d1db 100644 --- a/frappe/core/doctype/custom_docperm/test_custom_docperm.py +++ b/frappe/core/doctype/custom_docperm/test_custom_docperm.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import frappe import unittest diff --git a/frappe/core/doctype/custom_role/custom_role.py b/frappe/core/doctype/custom_role/custom_role.py index 25257e1a23..89e478dd38 100644 --- a/frappe/core/doctype/custom_role/custom_role.py +++ b/frappe/core/doctype/custom_role/custom_role.py @@ -2,7 +2,6 @@ # Copyright (c) 2015, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document diff --git a/frappe/core/doctype/custom_role/test_custom_role.py b/frappe/core/doctype/custom_role/test_custom_role.py index 670b494b10..0ad77524fa 100644 --- a/frappe/core/doctype/custom_role/test_custom_role.py +++ b/frappe/core/doctype/custom_role/test_custom_role.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import frappe import unittest diff --git a/frappe/core/doctype/data_export/data_export.py b/frappe/core/doctype/data_export/data_export.py index fb4fae26d5..c376b25230 100644 --- a/frappe/core/doctype/data_export/data_export.py +++ b/frappe/core/doctype/data_export/data_export.py @@ -2,7 +2,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and contributors # For license information, please see license.txt -from __future__ import unicode_literals from frappe.model.document import Document class DataExport(Document): diff --git a/frappe/core/doctype/data_export/exporter.py b/frappe/core/doctype/data_export/exporter.py index bec8cde7ea..389948449e 100644 --- a/frappe/core/doctype/data_export/exporter.py +++ b/frappe/core/doctype/data_export/exporter.py @@ -1,8 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals - import frappe from frappe import _ import frappe.permissions @@ -10,7 +8,6 @@ import re, csv, os from frappe.utils.csvutils import UnicodeWriter from frappe.utils import cstr, formatdate, format_datetime, parse_json, cint, format_duration from frappe.core.doctype.data_import_legacy.importer import get_data_keys -from six import string_types from frappe.core.doctype.access_log.access_log import make_access_log reflags = { @@ -57,7 +54,7 @@ class DataExporter: self.docs_to_export = {} if self.doctype: - if isinstance(self.doctype, string_types): + if isinstance(self.doctype, str): self.doctype = [self.doctype] if len(self.doctype) > 1: @@ -282,7 +279,7 @@ class DataExporter: try: sflags = self.docs_to_export.get("flags", "I,U").upper() flags = 0 - for a in re.split('\W+',sflags): + for a in re.split(r'\W+', sflags): flags = flags | reflags.get(a,0) c = re.compile(names, flags) diff --git a/frappe/core/doctype/data_import/data_import.js b/frappe/core/doctype/data_import/data_import.js index e03c22a898..216db53c72 100644 --- a/frappe/core/doctype/data_import/data_import.js +++ b/frappe/core/doctype/data_import/data_import.js @@ -91,7 +91,7 @@ frappe.ui.form.on('Data Import', { if (frm.doc.status.includes('Success')) { frm.add_custom_button( - __('Go to {0} List', [frm.doc.reference_doctype]), + __('Go to {0} List', [__(frm.doc.reference_doctype)]), () => frappe.set_route('List', frm.doc.reference_doctype) ); } @@ -203,7 +203,7 @@ frappe.ui.form.on('Data Import', { }, download_template(frm) { - frappe.require('/assets/js/data_import_tools.min.js', () => { + frappe.require('data_import_tools.bundle.js', () => { frm.data_exporter = new frappe.data_import.DataExporter( frm.doc.reference_doctype, frm.doc.import_type @@ -287,7 +287,7 @@ frappe.ui.form.on('Data Import', { return; } - frappe.require('/assets/js/data_import_tools.min.js', () => { + frappe.require('data_import_tools.bundle.js', () => { frm.import_preview = new frappe.data_import.ImportPreview({ wrapper: frm.get_field('import_preview').$wrapper, doctype: frm.doc.reference_doctype, diff --git a/frappe/core/doctype/data_import/data_import.py b/frappe/core/doctype/data_import/data_import.py index 1c56f54303..7e8374a0a2 100644 --- a/frappe/core/doctype/data_import/data_import.py +++ b/frappe/core/doctype/data_import/data_import.py @@ -211,7 +211,12 @@ def export_json( doctype, path, filters=None, or_filters=None, name=None, order_by="creation asc" ): def post_process(out): - del_keys = ("modified_by", "creation", "owner", "idx") + # Note on Tree DocTypes: + # The tree structure is maintained in the database via the fields "lft" + # and "rgt". They are automatically set and kept up-to-date. Importing + # them would destroy any existing tree structure. For this reason they + # are not exported as well. + del_keys = ("modified_by", "creation", "owner", "idx", "lft", "rgt") for doc in out: for key in del_keys: if key in doc: diff --git a/frappe/core/doctype/data_import/importer.py b/frappe/core/doctype/data_import/importer.py index 388d9389f2..fed90b75ce 100644 --- a/frappe/core/doctype/data_import/importer.py +++ b/frappe/core/doctype/data_import/importer.py @@ -1,7 +1,6 @@ # Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import os import io import frappe @@ -233,7 +232,7 @@ class Importer: return updated_doc else: # throw if no changes - frappe.throw("No changes to update") + frappe.throw(_("No changes to update")) def get_eta(self, current, total, processing_time): self.last_eta = getattr(self, "last_eta", 0) @@ -319,7 +318,7 @@ class ImportFile: self.warnings = [] self.file_doc = self.file_path = self.google_sheets_url = None - if isinstance(file, frappe.string_types): + if isinstance(file, str): if frappe.db.exists("File", {"file_url": file}): self.file_doc = frappe.get_doc("File", {"file_url": file}) elif "docs.google.com/spreadsheets" in file: @@ -626,7 +625,7 @@ class Row: return elif df.fieldtype in ["Date", "Datetime"]: value = self.get_date(value, col) - if isinstance(value, frappe.string_types): + if isinstance(value, str): # value was not parsed as datetime object self.warnings.append( { @@ -641,7 +640,7 @@ class Row: return elif df.fieldtype == "Duration": import re - is_valid_duration = re.match("^(?:(\d+d)?((^|\s)\d+h)?((^|\s)\d+m)?((^|\s)\d+s)?)$", value) + is_valid_duration = re.match(r"^(?:(\d+d)?((^|\s)\d+h)?((^|\s)\d+m)?((^|\s)\d+s)?)$", value) if not is_valid_duration: self.warnings.append( { @@ -929,10 +928,7 @@ class Column: self.warnings.append( { "col": self.column_number, - "message": _( - "Date format could not be determined from the values in" - " this column. Defaulting to yyyy-mm-dd." - ), + "message": _("Date format could not be determined from the values in this column. Defaulting to yyyy-mm-dd."), "type": "info", } ) diff --git a/frappe/core/doctype/data_import/test_data_import.py b/frappe/core/doctype/data_import/test_data_import.py index 15fd57744a..c9366a97ba 100644 --- a/frappe/core/doctype/data_import/test_data_import.py +++ b/frappe/core/doctype/data_import/test_data_import.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - # import frappe import unittest diff --git a/frappe/core/doctype/data_import/test_exporter.py b/frappe/core/doctype/data_import/test_exporter.py index 8415af2e63..dfe9926906 100644 --- a/frappe/core/doctype/data_import/test_exporter.py +++ b/frappe/core/doctype/data_import/test_exporter.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import unittest import frappe from frappe.core.doctype.data_import.exporter import Exporter diff --git a/frappe/core/doctype/data_import/test_importer.py b/frappe/core/doctype/data_import/test_importer.py index f76d4504a4..54a7788a2d 100644 --- a/frappe/core/doctype/data_import/test_importer.py +++ b/frappe/core/doctype/data_import/test_importer.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import unittest import frappe from frappe.core.doctype.data_import.importer import Importer diff --git a/frappe/core/doctype/data_import_legacy/importer.py b/frappe/core/doctype/data_import_legacy/importer.py index 35569c7186..4080e70418 100644 --- a/frappe/core/doctype/data_import_legacy/importer.py +++ b/frappe/core/doctype/data_import_legacy/importer.py @@ -3,9 +3,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals, print_function - -from six.moves import range import requests import frappe, json import frappe.permissions @@ -16,7 +13,6 @@ from frappe.utils.csvutils import getlink from frappe.utils.dateutils import parse_date from frappe.utils import cint, cstr, flt, getdate, get_datetime, get_url, get_absolute_url, duration_to_seconds -from six import string_types @frappe.whitelist() @@ -42,7 +38,7 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False, frappe.cache().hdel("lang", user) frappe.set_user_lang(user) - if data_import_doc and isinstance(data_import_doc, string_types): + if data_import_doc and isinstance(data_import_doc, str): data_import_doc = frappe.get_doc("Data Import Legacy", data_import_doc) if data_import_doc and from_data_import == "Yes": no_email = data_import_doc.no_email @@ -152,7 +148,7 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False, elif fieldtype in ("Float", "Currency", "Percent"): d[fieldname] = flt(d[fieldname]) elif fieldtype == "Date": - if d[fieldname] and isinstance(d[fieldname], string_types): + if d[fieldname] and isinstance(d[fieldname], str): d[fieldname] = getdate(parse_date(d[fieldname])) elif fieldtype == "Datetime": if d[fieldname]: diff --git a/frappe/core/doctype/data_import_legacy/test_data_import_legacy.py b/frappe/core/doctype/data_import_legacy/test_data_import_legacy.py index e5b244e6a0..6f9964e8f5 100644 --- a/frappe/core/doctype/data_import_legacy/test_data_import_legacy.py +++ b/frappe/core/doctype/data_import_legacy/test_data_import_legacy.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - # import frappe import unittest diff --git a/frappe/core/doctype/defaultvalue/__init__.py b/frappe/core/doctype/defaultvalue/__init__.py index 4dbcd0d163..0e57cb68c3 100644 --- a/frappe/core/doctype/defaultvalue/__init__.py +++ b/frappe/core/doctype/defaultvalue/__init__.py @@ -1,4 +1,3 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals diff --git a/frappe/core/doctype/defaultvalue/defaultvalue.py b/frappe/core/doctype/defaultvalue/defaultvalue.py index d9cc145053..0ae088ee96 100644 --- a/frappe/core/doctype/defaultvalue/defaultvalue.py +++ b/frappe/core/doctype/defaultvalue/defaultvalue.py @@ -1,7 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document diff --git a/frappe/core/doctype/deleted_document/deleted_document.py b/frappe/core/doctype/deleted_document/deleted_document.py index 116fc5caf5..f4109c8197 100644 --- a/frappe/core/doctype/deleted_document/deleted_document.py +++ b/frappe/core/doctype/deleted_document/deleted_document.py @@ -2,7 +2,6 @@ # Copyright (c) 2015, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe import json from frappe.desk.doctype.bulk_update.bulk_update import show_progress diff --git a/frappe/core/doctype/deleted_document/test_deleted_document.py b/frappe/core/doctype/deleted_document/test_deleted_document.py index c45a2bd180..d9dc2bb2d1 100644 --- a/frappe/core/doctype/deleted_document/test_deleted_document.py +++ b/frappe/core/doctype/deleted_document/test_deleted_document.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import frappe import unittest diff --git a/frappe/core/doctype/docfield/__init__.py b/frappe/core/doctype/docfield/__init__.py index 4dbcd0d163..0e57cb68c3 100644 --- a/frappe/core/doctype/docfield/__init__.py +++ b/frappe/core/doctype/docfield/__init__.py @@ -1,4 +1,3 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals diff --git a/frappe/core/doctype/docfield/docfield.py b/frappe/core/doctype/docfield/docfield.py index b6e2d9b67d..175cba3c7c 100644 --- a/frappe/core/doctype/docfield/docfield.py +++ b/frappe/core/doctype/docfield/docfield.py @@ -1,8 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals - import frappe from frappe.model.document import Document diff --git a/frappe/core/doctype/docperm/__init__.py b/frappe/core/doctype/docperm/__init__.py index 4dbcd0d163..0e57cb68c3 100644 --- a/frappe/core/doctype/docperm/__init__.py +++ b/frappe/core/doctype/docperm/__init__.py @@ -1,4 +1,3 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals diff --git a/frappe/core/doctype/docperm/docperm.py b/frappe/core/doctype/docperm/docperm.py index 36ed9acbe6..9732cde920 100644 --- a/frappe/core/doctype/docperm/docperm.py +++ b/frappe/core/doctype/docperm/docperm.py @@ -1,7 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document diff --git a/frappe/core/doctype/docshare/docshare.py b/frappe/core/doctype/docshare/docshare.py index 26ed53a87d..2d7b6b9e48 100644 --- a/frappe/core/doctype/docshare/docshare.py +++ b/frappe/core/doctype/docshare/docshare.py @@ -1,7 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document from frappe import _ diff --git a/frappe/core/doctype/docshare/test_docshare.py b/frappe/core/doctype/docshare/test_docshare.py index d4ef1f92f8..6551dabbea 100644 --- a/frappe/core/doctype/docshare/test_docshare.py +++ b/frappe/core/doctype/docshare/test_docshare.py @@ -1,12 +1,13 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # See license.txt -from __future__ import unicode_literals import frappe import frappe.share import unittest from frappe.automation.doctype.auto_repeat.test_auto_repeat import create_submittable_doctype +test_dependencies = ['User'] + class TestDocShare(unittest.TestCase): def setUp(self): self.user = "test@example.com" @@ -112,4 +113,4 @@ class TestDocShare(unittest.TestCase): self.assertTrue(frappe.has_permission(doctype, "read", doc=submittable_doc.name, user=self.user)) self.assertTrue(frappe.has_permission(doctype, "write", doc=submittable_doc.name, user=self.user)) - frappe.share.remove(doctype, submittable_doc.name, self.user) \ No newline at end of file + frappe.share.remove(doctype, submittable_doc.name, self.user) diff --git a/frappe/core/doctype/doctype/__init__.py b/frappe/core/doctype/doctype/__init__.py index 4dbcd0d163..0e57cb68c3 100644 --- a/frappe/core/doctype/doctype/__init__.py +++ b/frappe/core/doctype/doctype/__init__.py @@ -1,4 +1,3 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals diff --git a/frappe/core/doctype/doctype/boilerplate/controller._py b/frappe/core/doctype/doctype/boilerplate/controller._py index 583bd30908..6db99def55 100644 --- a/frappe/core/doctype/doctype/boilerplate/controller._py +++ b/frappe/core/doctype/doctype/boilerplate/controller._py @@ -1,8 +1,6 @@ -# -*- coding: utf-8 -*- # Copyright (c) {year}, {app_publisher} and contributors # For license information, please see license.txt -from __future__ import unicode_literals # import frappe {base_class_import} diff --git a/frappe/core/doctype/doctype/boilerplate/test_controller._py b/frappe/core/doctype/doctype/boilerplate/test_controller._py index 8ed08ae15a..5f4150ce9b 100644 --- a/frappe/core/doctype/doctype/boilerplate/test_controller._py +++ b/frappe/core/doctype/doctype/boilerplate/test_controller._py @@ -1,7 +1,5 @@ -# -*- coding: utf-8 -*- # Copyright (c) {year}, {app_publisher} and Contributors # See license.txt -from __future__ import unicode_literals # import frappe import unittest diff --git a/frappe/core/doctype/doctype/doctype.js b/frappe/core/doctype/doctype/doctype.js index 1a173f7252..b4d3fb9a89 100644 --- a/frappe/core/doctype/doctype/doctype.js +++ b/frappe/core/doctype/doctype/doctype.js @@ -33,11 +33,11 @@ frappe.ui.form.on('DocType', { if (!frm.is_new() && !frm.doc.istable) { if (frm.doc.issingle) { - frm.add_custom_button(__('Go to {0}', [frm.doc.name]), () => { + frm.add_custom_button(__('Go to {0}', [__(frm.doc.name)]), () => { window.open(`/app/${frappe.router.slug(frm.doc.name)}`); }); } else { - frm.add_custom_button(__('Go to {0} List', [frm.doc.name]), () => { + frm.add_custom_button(__('Go to {0} List', [__(frm.doc.name)]), () => { window.open(`/app/${frappe.router.slug(frm.doc.name)}`); }); } diff --git a/frappe/core/doctype/doctype/doctype.json b/frappe/core/doctype/doctype/doctype.json index ceaa1240f2..7f93d3130a 100644 --- a/frappe/core/doctype/doctype/doctype.json +++ b/frappe/core/doctype/doctype/doctype.json @@ -628,7 +628,7 @@ "link_fieldname": "reference_doctype" } ], - "modified": "2021-03-22 12:26:41.031135", + "modified": "2021-04-16 12:26:41.031135", "modified_by": "Administrator", "module": "Core", "name": "DocType", diff --git a/frappe/core/doctype/doctype/doctype.py b/frappe/core/doctype/doctype/doctype.py index 3588cc553a..8a96fc89f6 100644 --- a/frappe/core/doctype/doctype/doctype.py +++ b/frappe/core/doctype/doctype/doctype.py @@ -2,15 +2,10 @@ # MIT License. See license.txt # imports - standard imports -from __future__ import unicode_literals import re, copy, os, shutil import json from frappe.cache_manager import clear_user_cache, clear_controller_cache -# imports - third party imports -import six -from six import iteritems - # imports - module imports import frappe import frappe.website.render @@ -18,6 +13,7 @@ from frappe import _ from frappe.utils import now, cint from frappe.model import no_value_fields, default_fields, data_fieldtypes, table_fields, data_field_options from frappe.model.document import Document +from frappe.model.base_document import get_controller from frappe.custom.doctype.property_setter.property_setter import make_property_setter from frappe.custom.doctype.custom_field.custom_field import create_custom_field from frappe.desk.notifications import delete_notification_count_for @@ -83,12 +79,62 @@ class DocType(Document): if not self.is_new(): self.before_update = frappe.get_doc('DocType', self.name) self.setup_fields_to_fetch() + self.validate_field_name_conflicts() check_email_append_to(self) if self.default_print_format and not self.custom: frappe.throw(_('Standard DocType cannot have default print format, use Customize Form')) + if frappe.conf.get('developer_mode'): + self.owner = 'Administrator' + self.modified_by = 'Administrator' + + def validate_field_name_conflicts(self): + """Check if field names dont conflict with controller properties and methods""" + core_doctypes = [ + "Custom DocPerm", + "DocPerm", + "Custom Field", + "Customize Form Field", + "DocField", + ] + + if self.name in core_doctypes: + return + + try: + controller = get_controller(self.name) + except ImportError: + controller = Document + + available_objects = {x for x in dir(controller) if isinstance(x, str)} + property_set = { + x for x in available_objects if isinstance(getattr(controller, x, None), property) + } + method_set = { + x for x in available_objects if x not in property_set and callable(getattr(controller, x, None)) + } + + for docfield in self.get("fields") or []: + if docfield.fieldtype in no_value_fields: + continue + + conflict_type = None + field = docfield.fieldname + field_label = docfield.label or docfield.fieldname + + if docfield.fieldname in method_set: + conflict_type = "controller method" + if docfield.fieldname in property_set: + conflict_type = "class property" + + if conflict_type: + frappe.throw( + _("Fieldname '{0}' conflicting with a {1} of the name {2} in {3}") + .format(field_label, conflict_type, field, self.name) + ) + def after_insert(self): # clear user cache so that on the next reload this doctype is included in boot clear_user_cache(frappe.session.user) @@ -435,7 +481,7 @@ class DocType(Document): # remove null and empty fields def remove_null_fields(o): to_remove = [] - for attr, value in iteritems(o): + for attr, value in o.items(): if isinstance(value, list): for v in value: remove_null_fields(v) @@ -619,15 +665,15 @@ class DocType(Document): if not name: name = self.name - flags = {"flags": re.ASCII} if six.PY3 else {} + flags = {"flags": re.ASCII} # a DocType name should not start or end with an empty space - if re.search("^[ \t\n\r]+|[ \t\n\r]+$", name, **flags): + if re.search(r"^[ \t\n\r]+|[ \t\n\r]+$", name, **flags): frappe.throw(_("DocType's name should not start or end with whitespace"), frappe.NameError) # a DocType's name should not start with a number or underscore # and should only contain letters, numbers and underscore - if not re.match("^(?![\W])[^\d_\s][\w ]+$", name, **flags): + if not re.match(r"^(?![\W])[^\d_\s][\w ]+$", name, **flags): frappe.throw(_("DocType's name should start with a letter and it can only consist of letters, numbers, spaces and underscores"), frappe.NameError) validate_route_conflict(self.doctype, self.name) @@ -915,7 +961,7 @@ def validate_fields(meta): for field in depends_on_fields: depends_on = docfield.get(field, None) if depends_on and ("=" in depends_on) and \ - re.match("""[\w\.:_]+\s*={1}\s*[\w\.@'"]+""", depends_on): + re.match(r'[\w\.:_]+\s*={1}\s*[\w\.@\'"]+', depends_on): frappe.throw(_("Invalid {0} condition").format(frappe.unscrub(field)), frappe.ValidationError) def check_table_multiselect_option(docfield): @@ -1174,11 +1220,19 @@ def make_module_and_roles(doc, perm_fieldname="permissions"): else: raise -def check_if_fieldname_conflicts_with_methods(doctype, fieldname): - doc = frappe.get_doc({"doctype": doctype}) - method_list = [method for method in dir(doc) if isinstance(method, str) and callable(getattr(doc, method))] +def check_fieldname_conflicts(doctype, fieldname): + """Checks if fieldname conflicts with methods or properties""" - if fieldname in method_list: + doc = frappe.get_doc({"doctype": doctype}) + available_objects = [x for x in dir(doc) if isinstance(x, str)] + property_list = [ + x for x in available_objects if isinstance(getattr(type(doc), x, None), property) + ] + method_list = [ + x for x in available_objects if x not in property_list and callable(getattr(doc, x)) + ] + + if fieldname in method_list + property_list: frappe.throw(_("Fieldname {0} conflicting with meta object").format(fieldname)) def clear_linked_doctype_cache(): diff --git a/frappe/core/doctype/doctype/test_doctype.py b/frappe/core/doctype/doctype/test_doctype.py index bfa9d0ec8a..1e1a01a685 100644 --- a/frappe/core/doctype/doctype/test_doctype.py +++ b/frappe/core/doctype/doctype/test_doctype.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # See license.txt -from __future__ import unicode_literals - import frappe import unittest from frappe.core.doctype.doctype.doctype import (UniqueFieldnameError, @@ -92,7 +90,7 @@ class TestDocType(unittest.TestCase): fields=["parent", "depends_on", "collapsible_depends_on", "mandatory_depends_on",\ "read_only_depends_on", "fieldname", "fieldtype"]) - pattern = """[\w\.:_]+\s*={1}\s*[\w\.@'"]+""" + pattern = r'[\w\.:_]+\s*={1}\s*[\w\.@\'"]+' for field in docfields: for depends_on in ["depends_on", "collapsible_depends_on", "mandatory_depends_on", "read_only_depends_on"]: condition = field.get(depends_on) @@ -517,4 +515,4 @@ def new_doctype(name, unique=0, depends_on='', fields=None): for f in fields: doc.append('fields', f) - return doc \ No newline at end of file + return doc diff --git a/frappe/core/doctype/doctype_action/doctype_action.py b/frappe/core/doctype/doctype_action/doctype_action.py index a745c7da40..203b06ec1b 100644 --- a/frappe/core/doctype/doctype_action/doctype_action.py +++ b/frappe/core/doctype/doctype_action/doctype_action.py @@ -2,7 +2,6 @@ # Copyright (c) 2019, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals # import frappe from frappe.model.document import Document diff --git a/frappe/core/doctype/doctype_link/doctype_link.py b/frappe/core/doctype/doctype_link/doctype_link.py index efe8b09809..07e0efdace 100644 --- a/frappe/core/doctype/doctype_link/doctype_link.py +++ b/frappe/core/doctype/doctype_link/doctype_link.py @@ -2,7 +2,6 @@ # Copyright (c) 2019, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals # import frappe from frappe.model.document import Document diff --git a/frappe/core/doctype/document_naming_rule/document_naming_rule.js b/frappe/core/doctype/document_naming_rule/document_naming_rule.js index 56b5c2fdf4..097a4e9a6e 100644 --- a/frappe/core/doctype/document_naming_rule/document_naming_rule.js +++ b/frappe/core/doctype/document_naming_rule/document_naming_rule.js @@ -4,6 +4,7 @@ frappe.ui.form.on('Document Naming Rule', { refresh: function(frm) { frm.trigger('document_type'); + if (!frm.doc.__islocal) frm.trigger("add_update_counter_button"); }, document_type: (frm) => { // update the select field options with fieldnames @@ -20,5 +21,44 @@ frappe.ui.form.on('Document Naming Rule', { ); }); } + }, + add_update_counter_button: (frm) => { + frm.add_custom_button(__('Update Counter'), function() { + + const fields = [{ + fieldtype: 'Data', + fieldname: 'new_counter', + label: __('New Counter'), + default: frm.doc.counter, + reqd: 1, + description: __('Warning: Updating counter may lead to document name conflicts if not done properly') + }]; + + let primary_action_label = __('Save'); + + let primary_action = (fields) => { + frappe.call({ + method: 'frappe.core.doctype.document_naming_rule.document_naming_rule.update_current', + args: { + name: frm.doc.name, + new_counter: fields.new_counter + }, + callback: function() { + frm.set_value("counter", fields.new_counter); + dialog.hide(); + } + }); + }; + + const dialog = new frappe.ui.Dialog({ + title: __('Update Counter Value for Prefix: {0}', [frm.doc.prefix]), + fields, + primary_action_label, + primary_action + }); + + dialog.show(); + + }); } }); diff --git a/frappe/core/doctype/document_naming_rule/document_naming_rule.py b/frappe/core/doctype/document_naming_rule/document_naming_rule.py index 4b34293af6..10099bd19a 100644 --- a/frappe/core/doctype/document_naming_rule/document_naming_rule.py +++ b/frappe/core/doctype/document_naming_rule/document_naming_rule.py @@ -2,7 +2,6 @@ # Copyright (c) 2020, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document from frappe.utils.data import evaluate_filters @@ -30,3 +29,8 @@ class DocumentNamingRule(Document): counter = frappe.db.get_value(self.doctype, self.name, 'counter', for_update=True) or 0 doc.name = self.prefix + ('%0'+str(self.prefix_digits)+'d') % (counter + 1) frappe.db.set_value(self.doctype, self.name, 'counter', counter + 1) + +@frappe.whitelist() +def update_current(name, new_counter): + frappe.only_for('System Manager') + frappe.db.set_value('Document Naming Rule', name, 'counter', new_counter) diff --git a/frappe/core/doctype/document_naming_rule/test_document_naming_rule.py b/frappe/core/doctype/document_naming_rule/test_document_naming_rule.py index 1b91f6a0cf..2206d173d7 100644 --- a/frappe/core/doctype/document_naming_rule/test_document_naming_rule.py +++ b/frappe/core/doctype/document_naming_rule/test_document_naming_rule.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import frappe import unittest diff --git a/frappe/core/doctype/document_naming_rule_condition/document_naming_rule_condition.py b/frappe/core/doctype/document_naming_rule_condition/document_naming_rule_condition.py index 0895c9f93f..dfca052d95 100644 --- a/frappe/core/doctype/document_naming_rule_condition/document_naming_rule_condition.py +++ b/frappe/core/doctype/document_naming_rule_condition/document_naming_rule_condition.py @@ -2,7 +2,6 @@ # Copyright (c) 2020, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals # import frappe from frappe.model.document import Document diff --git a/frappe/core/doctype/document_naming_rule_condition/test_document_naming_rule_condition.py b/frappe/core/doctype/document_naming_rule_condition/test_document_naming_rule_condition.py index 6f1376dc62..643e963bd7 100644 --- a/frappe/core/doctype/document_naming_rule_condition/test_document_naming_rule_condition.py +++ b/frappe/core/doctype/document_naming_rule_condition/test_document_naming_rule_condition.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - # import frappe import unittest diff --git a/frappe/core/doctype/domain/domain.py b/frappe/core/doctype/domain/domain.py index a4e9f503ab..681824bb02 100644 --- a/frappe/core/doctype/domain/domain.py +++ b/frappe/core/doctype/domain/domain.py @@ -2,7 +2,6 @@ # Copyright (c) 2017, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document diff --git a/frappe/core/doctype/domain/test_domain.py b/frappe/core/doctype/domain/test_domain.py index 8e0bc65c54..c2686a7566 100644 --- a/frappe/core/doctype/domain/test_domain.py +++ b/frappe/core/doctype/domain/test_domain.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2017, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import frappe import unittest diff --git a/frappe/core/doctype/domain_settings/domain_settings.py b/frappe/core/doctype/domain_settings/domain_settings.py index d4d394a5cb..7ad0aeff21 100644 --- a/frappe/core/doctype/domain_settings/domain_settings.py +++ b/frappe/core/doctype/domain_settings/domain_settings.py @@ -2,7 +2,6 @@ # Copyright (c) 2017, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document diff --git a/frappe/core/doctype/dynamic_link/dynamic_link.py b/frappe/core/doctype/dynamic_link/dynamic_link.py index 30e0ef1f1f..a7adb9ae72 100644 --- a/frappe/core/doctype/dynamic_link/dynamic_link.py +++ b/frappe/core/doctype/dynamic_link/dynamic_link.py @@ -2,7 +2,6 @@ # Copyright (c) 2015, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document diff --git a/frappe/core/doctype/error_log/error_log.py b/frappe/core/doctype/error_log/error_log.py index ec02aaf446..8223238c57 100644 --- a/frappe/core/doctype/error_log/error_log.py +++ b/frappe/core/doctype/error_log/error_log.py @@ -2,7 +2,6 @@ # Copyright (c) 2015, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document diff --git a/frappe/core/doctype/error_log/test_error_log.py b/frappe/core/doctype/error_log/test_error_log.py index d93fe07c61..d7444ab2a7 100644 --- a/frappe/core/doctype/error_log/test_error_log.py +++ b/frappe/core/doctype/error_log/test_error_log.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import frappe import unittest diff --git a/frappe/core/doctype/error_snapshot/error_snapshot.py b/frappe/core/doctype/error_snapshot/error_snapshot.py index 5badaad63f..247a796a6b 100644 --- a/frappe/core/doctype/error_snapshot/error_snapshot.py +++ b/frappe/core/doctype/error_snapshot/error_snapshot.py @@ -2,7 +2,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document diff --git a/frappe/core/doctype/error_snapshot/test_error_snapshot.py b/frappe/core/doctype/error_snapshot/test_error_snapshot.py index b6438eae1d..135136294a 100644 --- a/frappe/core/doctype/error_snapshot/test_error_snapshot.py +++ b/frappe/core/doctype/error_snapshot/test_error_snapshot.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # See license.txt -from __future__ import unicode_literals - import frappe import unittest diff --git a/frappe/core/doctype/file/file.py b/frappe/core/doctype/file/file.py index 017106e6f5..b4bfe1d21b 100755 --- a/frappe/core/doctype/file/file.py +++ b/frappe/core/doctype/file/file.py @@ -7,8 +7,6 @@ record of files naming for same name files: file.gif, file-1.gif, file-2.gif etc """ -from __future__ import unicode_literals - import base64 import hashlib import imghdr @@ -23,8 +21,8 @@ import zipfile import requests import requests.exceptions from PIL import Image, ImageFile, ImageOps -from six import PY2, StringIO, string_types, text_type -from six.moves.urllib.parse import quote, unquote +from io import StringIO +from urllib.parse import quote, unquote import frappe from frappe import _, conf @@ -382,18 +380,14 @@ class File(Document): file_path = self.get_full_path() # read the file - if PY2: - with open(encode(file_path)) as f: - content = f.read() - else: - with io.open(encode(file_path), mode='rb') as f: - content = f.read() - try: - # for plain text files - content = content.decode() - except UnicodeDecodeError: - # for .png, .jpg, etc - pass + with io.open(encode(file_path), mode='rb') as f: + content = f.read() + try: + # for plain text files + content = content.decode() + except UnicodeDecodeError: + # for .png, .jpg, etc + pass return content @@ -430,7 +424,7 @@ class File(Document): frappe.create_folder(file_path) # write the file self.content = self.get_content() - if isinstance(self.content, text_type): + if isinstance(self.content, str): self.content = self.content.encode() with open(os.path.join(file_path.encode('utf-8'), self.file_name.encode('utf-8')), 'wb+') as f: f.write(self.content) @@ -483,7 +477,7 @@ class File(Document): self.content = content if decode: - if isinstance(content, text_type): + if isinstance(content, str): self.content = content.encode("utf-8") if b"," in self.content: @@ -498,7 +492,7 @@ class File(Document): self.file_size = self.check_max_file_size() if ( - self.content_type and "image" in self.content_type + self.content_type and self.content_type == "image/jpeg" and frappe.get_system_settings("strip_exif_metadata_from_uploaded_images") ): self.content = strip_exif_data(self.content, self.content_type) @@ -632,7 +626,7 @@ def create_new_folder(file_name, folder): @frappe.whitelist() def move_file(file_list, new_parent, old_parent): - if isinstance(file_list, string_types): + if isinstance(file_list, str): file_list = json.loads(file_list) for file_obj in file_list: @@ -834,7 +828,7 @@ def remove_file_by_url(file_url, doctype=None, name=None): def get_content_hash(content): - if isinstance(content, text_type): + if isinstance(content, str): content = content.encode() return hashlib.md5(content).hexdigest() #nosec @@ -887,8 +881,8 @@ def extract_images_from_html(doc, content): filename = headers.split("filename=")[-1] # decode filename - if not isinstance(filename, text_type): - filename = text_type(filename, 'utf-8') + if not isinstance(filename, str): + filename = str(filename, 'utf-8') else: mtype = headers.split(";")[0] filename = get_random_filename(content_type=mtype) @@ -911,8 +905,8 @@ def extract_images_from_html(doc, content): return ']*src\s*=\s*["\'](?=data:)(.*?)["\']', _save_file, content) + if content and isinstance(content, str): + content = re.sub(r']*src\s*=\s*["\'](?=data:)(.*?)["\']', _save_file, content) return content @@ -941,7 +935,7 @@ def get_attached_images(doctype, names): '''get list of image urls attached in form returns {name: ['image.jpg', 'image.png']}''' - if isinstance(names, string_types): + if isinstance(names, str): names = json.loads(names) img_urls = frappe.db.get_list('File', filters={ diff --git a/frappe/core/doctype/file/test_file.py b/frappe/core/doctype/file/test_file.py index 2f8f437fc9..649010c468 100644 --- a/frappe/core/doctype/file/test_file.py +++ b/frappe/core/doctype/file/test_file.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # See license.txt -from __future__ import unicode_literals - import base64 import frappe import os @@ -193,6 +191,7 @@ class TestSameContent(unittest.TestCase): class TestFile(unittest.TestCase): def setUp(self): + frappe.set_user('Administrator') self.delete_test_data() self.upload_file() diff --git a/frappe/core/doctype/has_domain/has_domain.py b/frappe/core/doctype/has_domain/has_domain.py index 6381996035..2220656a2e 100644 --- a/frappe/core/doctype/has_domain/has_domain.py +++ b/frappe/core/doctype/has_domain/has_domain.py @@ -2,7 +2,6 @@ # Copyright (c) 2017, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document diff --git a/frappe/core/doctype/has_role/has_role.py b/frappe/core/doctype/has_role/has_role.py index 45e76c85a1..51d86c7b0a 100644 --- a/frappe/core/doctype/has_role/has_role.py +++ b/frappe/core/doctype/has_role/has_role.py @@ -2,7 +2,6 @@ # Copyright (c) 2015, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document diff --git a/frappe/core/doctype/installed_application/installed_application.py b/frappe/core/doctype/installed_application/installed_application.py index 6bb12afc49..f53a6424eb 100644 --- a/frappe/core/doctype/installed_application/installed_application.py +++ b/frappe/core/doctype/installed_application/installed_application.py @@ -2,7 +2,6 @@ # Copyright (c) 2020, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals # import frappe from frappe.model.document import Document diff --git a/frappe/core/doctype/installed_applications/installed_applications.py b/frappe/core/doctype/installed_applications/installed_applications.py index 4e6eadf07e..b61555f57e 100644 --- a/frappe/core/doctype/installed_applications/installed_applications.py +++ b/frappe/core/doctype/installed_applications/installed_applications.py @@ -2,7 +2,6 @@ # Copyright (c) 2020, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document diff --git a/frappe/core/doctype/installed_applications/test_installed_applications.py b/frappe/core/doctype/installed_applications/test_installed_applications.py index ab9b849fa1..1d57fd2cd8 100644 --- a/frappe/core/doctype/installed_applications/test_installed_applications.py +++ b/frappe/core/doctype/installed_applications/test_installed_applications.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - # import frappe import unittest diff --git a/frappe/core/doctype/language/language.py b/frappe/core/doctype/language/language.py index fb18abdf5e..01c8553e10 100644 --- a/frappe/core/doctype/language/language.py +++ b/frappe/core/doctype/language/language.py @@ -2,7 +2,6 @@ # Copyright (c) 2015, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe, json, re from frappe import _ from frappe.model.document import Document diff --git a/frappe/core/doctype/language/test_language.py b/frappe/core/doctype/language/test_language.py index a4f35dd77b..837594247f 100644 --- a/frappe/core/doctype/language/test_language.py +++ b/frappe/core/doctype/language/test_language.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import frappe import unittest diff --git a/frappe/core/doctype/log_setting_user/log_setting_user.py b/frappe/core/doctype/log_setting_user/log_setting_user.py index df6d55f0a9..64728b2c2b 100644 --- a/frappe/core/doctype/log_setting_user/log_setting_user.py +++ b/frappe/core/doctype/log_setting_user/log_setting_user.py @@ -2,7 +2,6 @@ # Copyright (c) 2020, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals # import frappe from frappe.model.document import Document diff --git a/frappe/core/doctype/log_setting_user/test_log_setting_user.py b/frappe/core/doctype/log_setting_user/test_log_setting_user.py index 507c02d87d..c58b8faa66 100644 --- a/frappe/core/doctype/log_setting_user/test_log_setting_user.py +++ b/frappe/core/doctype/log_setting_user/test_log_setting_user.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - # import frappe import unittest diff --git a/frappe/core/doctype/log_settings/log_settings.py b/frappe/core/doctype/log_settings/log_settings.py index 08e61d3289..e73aa8dac1 100644 --- a/frappe/core/doctype/log_settings/log_settings.py +++ b/frappe/core/doctype/log_settings/log_settings.py @@ -2,7 +2,6 @@ # Copyright (c) 2020, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe import _ from frappe.model.document import Document diff --git a/frappe/core/doctype/log_settings/test_log_settings.py b/frappe/core/doctype/log_settings/test_log_settings.py index 2824c71c88..8e0c9c3f23 100644 --- a/frappe/core/doctype/log_settings/test_log_settings.py +++ b/frappe/core/doctype/log_settings/test_log_settings.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - # import frappe import unittest diff --git a/frappe/core/doctype/module_def/__init__.py b/frappe/core/doctype/module_def/__init__.py index 4dbcd0d163..0e57cb68c3 100644 --- a/frappe/core/doctype/module_def/__init__.py +++ b/frappe/core/doctype/module_def/__init__.py @@ -1,4 +1,3 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals diff --git a/frappe/core/doctype/module_def/module_def.json b/frappe/core/doctype/module_def/module_def.json index 7a8bfd76a7..4de046bbb6 100644 --- a/frappe/core/doctype/module_def/module_def.json +++ b/frappe/core/doctype/module_def/module_def.json @@ -55,7 +55,7 @@ "link_fieldname": "module" } ], - "modified": "2020-08-06 12:39:30.740379", + "modified": "2021-06-02 13:04:53.118716", "modified_by": "Administrator", "module": "Core", "name": "Module Def", @@ -69,6 +69,7 @@ "read": 1, "report": 1, "role": "Administrator", + "select": 1, "share": 1, "write": 1 }, @@ -78,7 +79,14 @@ "read": 1, "report": 1, "role": "System Manager", + "select": 1, "write": 1 + }, + { + "read": 1, + "report": 1, + "role": "All", + "select": 1 } ], "show_name_in_global_search": 1, diff --git a/frappe/core/doctype/module_def/module_def.py b/frappe/core/doctype/module_def/module_def.py index 7e63572162..68025c83bb 100644 --- a/frappe/core/doctype/module_def/module_def.py +++ b/frappe/core/doctype/module_def/module_def.py @@ -1,7 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import frappe, os, json from frappe.model.document import Document diff --git a/frappe/core/doctype/module_def/test_module_def.py b/frappe/core/doctype/module_def/test_module_def.py index 1f9bea4768..3a3ceb4b57 100644 --- a/frappe/core/doctype/module_def/test_module_def.py +++ b/frappe/core/doctype/module_def/test_module_def.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # See license.txt -from __future__ import unicode_literals - import frappe import unittest diff --git a/frappe/core/doctype/module_profile/module_profile.py b/frappe/core/doctype/module_profile/module_profile.py index 4f392353ac..373e5078d0 100644 --- a/frappe/core/doctype/module_profile/module_profile.py +++ b/frappe/core/doctype/module_profile/module_profile.py @@ -2,7 +2,6 @@ # Copyright (c) 2020, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals from frappe.model.document import Document class ModuleProfile(Document): diff --git a/frappe/core/doctype/module_profile/test_module_profile.py b/frappe/core/doctype/module_profile/test_module_profile.py index 400053d22c..e0d9c13371 100644 --- a/frappe/core/doctype/module_profile/test_module_profile.py +++ b/frappe/core/doctype/module_profile/test_module_profile.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals import frappe import unittest diff --git a/frappe/core/doctype/navbar_item/navbar_item.py b/frappe/core/doctype/navbar_item/navbar_item.py index 614aee8eaf..a8fa611374 100644 --- a/frappe/core/doctype/navbar_item/navbar_item.py +++ b/frappe/core/doctype/navbar_item/navbar_item.py @@ -2,7 +2,6 @@ # Copyright (c) 2020, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals # import frappe from frappe.model.document import Document diff --git a/frappe/core/doctype/navbar_item/test_navbar_item.py b/frappe/core/doctype/navbar_item/test_navbar_item.py index 192e8fe42a..85852a45e8 100644 --- a/frappe/core/doctype/navbar_item/test_navbar_item.py +++ b/frappe/core/doctype/navbar_item/test_navbar_item.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - # import frappe import unittest diff --git a/frappe/core/doctype/navbar_settings/navbar_settings.py b/frappe/core/doctype/navbar_settings/navbar_settings.py index 2244bc9e4e..60aec67a00 100644 --- a/frappe/core/doctype/navbar_settings/navbar_settings.py +++ b/frappe/core/doctype/navbar_settings/navbar_settings.py @@ -2,7 +2,6 @@ # Copyright (c) 2020, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document from frappe import _ diff --git a/frappe/core/doctype/navbar_settings/test_navbar_settings.py b/frappe/core/doctype/navbar_settings/test_navbar_settings.py index ed423b0f27..4d1ee72815 100644 --- a/frappe/core/doctype/navbar_settings/test_navbar_settings.py +++ b/frappe/core/doctype/navbar_settings/test_navbar_settings.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - # import frappe import unittest diff --git a/frappe/core/doctype/page/__init__.py b/frappe/core/doctype/page/__init__.py index 4dbcd0d163..0e57cb68c3 100644 --- a/frappe/core/doctype/page/__init__.py +++ b/frappe/core/doctype/page/__init__.py @@ -1,4 +1,3 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals diff --git a/frappe/core/doctype/page/page.py b/frappe/core/doctype/page/page.py index bdec350efd..0ba0e309dd 100644 --- a/frappe/core/doctype/page/page.py +++ b/frappe/core/doctype/page/page.py @@ -1,7 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import frappe import os from frappe.model.document import Document @@ -11,7 +10,6 @@ from frappe import conf, _, safe_decode from frappe.desk.form.meta import get_code_files_via_hooks, get_js from frappe.desk.utils import validate_route_conflict from frappe.core.doctype.custom_role.custom_role import get_custom_allowed_roles -from six import text_type class Page(Document): def autoname(self): diff --git a/frappe/core/doctype/page/test_page.py b/frappe/core/doctype/page/test_page.py index f7b3952a5b..18b4aea2c8 100644 --- a/frappe/core/doctype/page/test_page.py +++ b/frappe/core/doctype/page/test_page.py @@ -1,7 +1,5 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # See license.txt -from __future__ import unicode_literals - import frappe import unittest diff --git a/frappe/core/doctype/patch_log/patch_log.py b/frappe/core/doctype/patch_log/patch_log.py index 3103d44af4..cc66955eb8 100644 --- a/frappe/core/doctype/patch_log/patch_log.py +++ b/frappe/core/doctype/patch_log/patch_log.py @@ -3,7 +3,6 @@ # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document diff --git a/frappe/core/doctype/patch_log/test_patch_log.py b/frappe/core/doctype/patch_log/test_patch_log.py index 0a7f22a78b..d0690ecee0 100644 --- a/frappe/core/doctype/patch_log/test_patch_log.py +++ b/frappe/core/doctype/patch_log/test_patch_log.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # See license.txt -from __future__ import unicode_literals - import frappe import unittest diff --git a/frappe/core/doctype/payment_gateway/payment_gateway.py b/frappe/core/doctype/payment_gateway/payment_gateway.py index 80799e311b..1459635b01 100644 --- a/frappe/core/doctype/payment_gateway/payment_gateway.py +++ b/frappe/core/doctype/payment_gateway/payment_gateway.py @@ -2,7 +2,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document diff --git a/frappe/core/doctype/payment_gateway/test_payment_gateway.py b/frappe/core/doctype/payment_gateway/test_payment_gateway.py index 2faf1a7fb4..66f899bd27 100644 --- a/frappe/core/doctype/payment_gateway/test_payment_gateway.py +++ b/frappe/core/doctype/payment_gateway/test_payment_gateway.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # See license.txt -from __future__ import unicode_literals - import frappe import unittest diff --git a/frappe/core/doctype/prepared_report/prepared_report.py b/frappe/core/doctype/prepared_report/prepared_report.py index c27853f460..c68bb6a4f1 100644 --- a/frappe/core/doctype/prepared_report/prepared_report.py +++ b/frappe/core/doctype/prepared_report/prepared_report.py @@ -3,8 +3,6 @@ # For license information, please see license.txt -from __future__ import unicode_literals - import json import frappe diff --git a/frappe/core/doctype/prepared_report/test_prepared_report.py b/frappe/core/doctype/prepared_report/test_prepared_report.py index 17845be521..ef324dd01a 100644 --- a/frappe/core/doctype/prepared_report/test_prepared_report.py +++ b/frappe/core/doctype/prepared_report/test_prepared_report.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2018, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import frappe import unittest import json diff --git a/frappe/core/doctype/report/__init__.py b/frappe/core/doctype/report/__init__.py index 4dbcd0d163..0e57cb68c3 100644 --- a/frappe/core/doctype/report/__init__.py +++ b/frappe/core/doctype/report/__init__.py @@ -1,4 +1,3 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals diff --git a/frappe/core/doctype/report/boilerplate/controller.py b/frappe/core/doctype/report/boilerplate/controller.py index 55c01e4f75..b8e9cb7467 100644 --- a/frappe/core/doctype/report/boilerplate/controller.py +++ b/frappe/core/doctype/report/boilerplate/controller.py @@ -1,7 +1,6 @@ # Copyright (c) 2013, {app_publisher} and contributors # For license information, please see license.txt -from __future__ import unicode_literals # import frappe def execute(filters=None): diff --git a/frappe/core/doctype/report/report.py b/frappe/core/doctype/report/report.py index 8a0f9a99f5..a5c61fa436 100644 --- a/frappe/core/doctype/report/report.py +++ b/frappe/core/doctype/report/report.py @@ -1,7 +1,5 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt - -from __future__ import unicode_literals import frappe import json, datetime from frappe import _, scrub @@ -13,7 +11,6 @@ from frappe.modules import make_boilerplate from frappe.core.doctype.page.page import delete_custom_role from frappe.core.doctype.custom_role.custom_role import get_custom_allowed_roles from frappe.desk.reportview import append_totals_row -from six import iteritems from frappe.utils.safe_exec import safe_exec @@ -238,7 +235,7 @@ class Report(Document): _filters = params.get('filters') or [] if filters: - for key, value in iteritems(filters): + for key, value in filters.items(): condition, _value = '=', value if isinstance(value, (list, tuple)): condition, _value = value diff --git a/frappe/core/doctype/report/test_report.py b/frappe/core/doctype/report/test_report.py index 9c76c839f3..9d0c0b9af0 100644 --- a/frappe/core/doctype/report/test_report.py +++ b/frappe/core/doctype/report/test_report.py @@ -1,7 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # See license.txt -from __future__ import unicode_literals import frappe, json, os import unittest from frappe.desk.query_report import run, save_report @@ -106,7 +105,7 @@ class TestReport(unittest.TestCase): else: report = frappe.get_doc('Report', 'Test Report') - self.assertNotEquals(report.is_permitted(), True) + self.assertNotEqual(report.is_permitted(), True) frappe.set_user('Administrator') # test for the `_format` method if report data doesn't have sort_by parameter diff --git a/frappe/core/doctype/report_column/report_column.py b/frappe/core/doctype/report_column/report_column.py index 69c88b7bda..f9078d820d 100644 --- a/frappe/core/doctype/report_column/report_column.py +++ b/frappe/core/doctype/report_column/report_column.py @@ -2,7 +2,6 @@ # Copyright (c) 2020, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals # import frappe from frappe.model.document import Document diff --git a/frappe/core/doctype/report_filter/report_filter.py b/frappe/core/doctype/report_filter/report_filter.py index d85a1a5a65..ccdcc0eb6f 100644 --- a/frappe/core/doctype/report_filter/report_filter.py +++ b/frappe/core/doctype/report_filter/report_filter.py @@ -2,7 +2,6 @@ # Copyright (c) 2020, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals # import frappe from frappe.model.document import Document diff --git a/frappe/core/doctype/role/__init__.py b/frappe/core/doctype/role/__init__.py index 4dbcd0d163..0e57cb68c3 100644 --- a/frappe/core/doctype/role/__init__.py +++ b/frappe/core/doctype/role/__init__.py @@ -1,4 +1,3 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals diff --git a/frappe/core/doctype/role/role.py b/frappe/core/doctype/role/role.py index a1523db0dd..02482c75ca 100644 --- a/frappe/core/doctype/role/role.py +++ b/frappe/core/doctype/role/role.py @@ -1,7 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document diff --git a/frappe/core/doctype/role/test_role.py b/frappe/core/doctype/role/test_role.py index 6459a72c98..471f6cac43 100644 --- a/frappe/core/doctype/role/test_role.py +++ b/frappe/core/doctype/role/test_role.py @@ -1,7 +1,5 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals - import frappe import unittest diff --git a/frappe/core/doctype/role_permission_for_page_and_report/role_permission_for_page_and_report.py b/frappe/core/doctype/role_permission_for_page_and_report/role_permission_for_page_and_report.py index 77b523987c..59f34a1483 100644 --- a/frappe/core/doctype/role_permission_for_page_and_report/role_permission_for_page_and_report.py +++ b/frappe/core/doctype/role_permission_for_page_and_report/role_permission_for_page_and_report.py @@ -2,7 +2,6 @@ # Copyright (c) 2015, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe.core.doctype.report.report import is_prepared_report_disabled from frappe.model.document import Document diff --git a/frappe/core/doctype/role_profile/role_profile.py b/frappe/core/doctype/role_profile/role_profile.py index 4def834adb..0f58da5b5e 100644 --- a/frappe/core/doctype/role_profile/role_profile.py +++ b/frappe/core/doctype/role_profile/role_profile.py @@ -2,7 +2,6 @@ # Copyright (c) 2017, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals from frappe.model.document import Document class RoleProfile(Document): diff --git a/frappe/core/doctype/role_profile/test_role_profile.py b/frappe/core/doctype/role_profile/test_role_profile.py index 624b85c315..53e0a1b043 100644 --- a/frappe/core/doctype/role_profile/test_role_profile.py +++ b/frappe/core/doctype/role_profile/test_role_profile.py @@ -1,10 +1,11 @@ # -*- coding: utf-8 -*- # Copyright (c) 2017, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals import frappe import unittest +test_dependencies = ['Role'] + class TestRoleProfile(unittest.TestCase): def test_make_new_role_profile(self): new_role_profile = frappe.get_doc(dict(doctype='Role Profile', role_profile='Test 1')).insert() @@ -21,4 +22,4 @@ class TestRoleProfile(unittest.TestCase): # clear roles new_role_profile.roles = [] new_role_profile.save() - self.assertEqual(new_role_profile.roles, []) \ No newline at end of file + self.assertEqual(new_role_profile.roles, []) diff --git a/frappe/core/doctype/scheduled_job_log/scheduled_job_log.py b/frappe/core/doctype/scheduled_job_log/scheduled_job_log.py index 26871c9adf..7f54a3b6ae 100644 --- a/frappe/core/doctype/scheduled_job_log/scheduled_job_log.py +++ b/frappe/core/doctype/scheduled_job_log/scheduled_job_log.py @@ -2,7 +2,6 @@ # Copyright (c) 2019, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals # import frappe from frappe.model.document import Document diff --git a/frappe/core/doctype/scheduled_job_log/test_scheduled_job_log.py b/frappe/core/doctype/scheduled_job_log/test_scheduled_job_log.py index 1e5290425b..85471d0d71 100644 --- a/frappe/core/doctype/scheduled_job_log/test_scheduled_job_log.py +++ b/frappe/core/doctype/scheduled_job_log/test_scheduled_job_log.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - # import frappe import unittest diff --git a/frappe/core/doctype/scheduled_job_type/test_scheduled_job_type.py b/frappe/core/doctype/scheduled_job_type/test_scheduled_job_type.py index d0a65defa4..a071cfe9a9 100644 --- a/frappe/core/doctype/scheduled_job_type/test_scheduled_job_type.py +++ b/frappe/core/doctype/scheduled_job_type/test_scheduled_job_type.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import frappe import unittest from frappe.utils import get_datetime diff --git a/frappe/core/doctype/server_script/server_script.py b/frappe/core/doctype/server_script/server_script.py index f80a067cf1..d26fe5a188 100644 --- a/frappe/core/doctype/server_script/server_script.py +++ b/frappe/core/doctype/server_script/server_script.py @@ -2,8 +2,6 @@ # Copyright (c) 2019, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals - import ast from types import FunctionType, MethodType, ModuleType from typing import Dict, List diff --git a/frappe/core/doctype/server_script/test_server_script.py b/frappe/core/doctype/server_script/test_server_script.py index aac8b3deed..c39fcfa0d0 100644 --- a/frappe/core/doctype/server_script/test_server_script.py +++ b/frappe/core/doctype/server_script/test_server_script.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import frappe import unittest import requests diff --git a/frappe/core/doctype/session_default/session_default.py b/frappe/core/doctype/session_default/session_default.py index 8a8db46ff1..70ff103111 100644 --- a/frappe/core/doctype/session_default/session_default.py +++ b/frappe/core/doctype/session_default/session_default.py @@ -2,7 +2,6 @@ # Copyright (c) 2019, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals # import frappe from frappe.model.document import Document diff --git a/frappe/core/doctype/session_default_settings/session_default_settings.py b/frappe/core/doctype/session_default_settings/session_default_settings.py index 7b4bd19e9a..25f7522c86 100644 --- a/frappe/core/doctype/session_default_settings/session_default_settings.py +++ b/frappe/core/doctype/session_default_settings/session_default_settings.py @@ -2,7 +2,6 @@ # Copyright (c) 2019, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe import _ import json diff --git a/frappe/core/doctype/session_default_settings/test_session_default_settings.py b/frappe/core/doctype/session_default_settings/test_session_default_settings.py index 12aa14d343..7d20015b66 100644 --- a/frappe/core/doctype/session_default_settings/test_session_default_settings.py +++ b/frappe/core/doctype/session_default_settings/test_session_default_settings.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import frappe import unittest from frappe.core.doctype.session_default_settings.session_default_settings import set_session_default_values, clear_session_defaults diff --git a/frappe/core/doctype/sms_parameter/__init__.py b/frappe/core/doctype/sms_parameter/__init__.py index baffc48825..8b13789179 100755 --- a/frappe/core/doctype/sms_parameter/__init__.py +++ b/frappe/core/doctype/sms_parameter/__init__.py @@ -1 +1 @@ -from __future__ import unicode_literals + diff --git a/frappe/core/doctype/sms_parameter/sms_parameter.py b/frappe/core/doctype/sms_parameter/sms_parameter.py index 08b220b61a..d1fb1c53db 100644 --- a/frappe/core/doctype/sms_parameter/sms_parameter.py +++ b/frappe/core/doctype/sms_parameter/sms_parameter.py @@ -1,7 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: GNU General Public License v3. See license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document diff --git a/frappe/core/doctype/sms_settings/__init__.py b/frappe/core/doctype/sms_settings/__init__.py index baffc48825..8b13789179 100755 --- a/frappe/core/doctype/sms_settings/__init__.py +++ b/frappe/core/doctype/sms_settings/__init__.py @@ -1 +1 @@ -from __future__ import unicode_literals + diff --git a/frappe/core/doctype/sms_settings/sms_settings.py b/frappe/core/doctype/sms_settings/sms_settings.py index ac835108c1..58a0ff08f6 100644 --- a/frappe/core/doctype/sms_settings/sms_settings.py +++ b/frappe/core/doctype/sms_settings/sms_settings.py @@ -2,15 +2,12 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: GNU General Public License v3. See license.txt -from __future__ import unicode_literals import frappe from frappe import _, throw, msgprint from frappe.utils import nowdate from frappe.model.document import Document -import six -from six import string_types class SMSSettings(Document): pass @@ -35,20 +32,20 @@ def validate_receiver_nos(receiver_list): @frappe.whitelist() def get_contact_number(contact_name, ref_doctype, ref_name): "returns mobile number of the contact" - number = frappe.db.sql("""select mobile_no, phone from tabContact - where name=%s + number = frappe.db.sql("""select mobile_no, phone from tabContact + where name=%s and exists( select name from `tabDynamic Link` where link_doctype=%s and link_name=%s ) """, (contact_name, ref_doctype, ref_name)) - + return number and (number[0][0] or number[0][1]) or '' @frappe.whitelist() def send_sms(receiver_list, msg, sender_name = '', success_msg = True): import json - if isinstance(receiver_list, string_types): + if isinstance(receiver_list, str): receiver_list = json.loads(receiver_list) if not isinstance(receiver_list, list): receiver_list = [receiver_list] diff --git a/frappe/core/doctype/sms_settings/test_sms_settings.py b/frappe/core/doctype/sms_settings/test_sms_settings.py index b14fd3e4a0..862f5e3965 100644 --- a/frappe/core/doctype/sms_settings/test_sms_settings.py +++ b/frappe/core/doctype/sms_settings/test_sms_settings.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2017, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import frappe import unittest diff --git a/frappe/core/doctype/success_action/success_action.py b/frappe/core/doctype/success_action/success_action.py index f8b99f1fea..4ebd3d250b 100644 --- a/frappe/core/doctype/success_action/success_action.py +++ b/frappe/core/doctype/success_action/success_action.py @@ -2,7 +2,6 @@ # Copyright (c) 2018, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals from frappe.model.document import Document class SuccessAction(Document): diff --git a/frappe/core/doctype/system_settings/system_settings.py b/frappe/core/doctype/system_settings/system_settings.py index d102526a9e..466914569f 100644 --- a/frappe/core/doctype/system_settings/system_settings.py +++ b/frappe/core/doctype/system_settings/system_settings.py @@ -1,7 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe import _ from frappe.model.document import Document @@ -42,7 +41,7 @@ class SystemSettings(Document): def on_update(self): for df in self.meta.get("fields"): - if df.fieldtype not in no_value_fields: + if df.fieldtype not in no_value_fields and self.has_value_changed(df.fieldname): frappe.db.set_default(df.fieldname, self.get(df.fieldname)) if self.language: diff --git a/frappe/core/doctype/system_settings/test_system_settings.py b/frappe/core/doctype/system_settings/test_system_settings.py index 82d0ddbd7c..a65c602abe 100644 --- a/frappe/core/doctype/system_settings/test_system_settings.py +++ b/frappe/core/doctype/system_settings/test_system_settings.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2017, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import frappe import unittest diff --git a/frappe/core/doctype/test/test.py b/frappe/core/doctype/test/test.py index 7e91b1cd4a..98e36e6a30 100644 --- a/frappe/core/doctype/test/test.py +++ b/frappe/core/doctype/test/test.py @@ -2,7 +2,6 @@ # Copyright (c) 2021, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals # import frappe from frappe.model.document import Document import json diff --git a/frappe/core/doctype/test/test_test.py b/frappe/core/doctype/test/test_test.py index 2a9b43bf95..d8ca975d63 100644 --- a/frappe/core/doctype/test/test_test.py +++ b/frappe/core/doctype/test/test_test.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2021, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - # import frappe import unittest diff --git a/frappe/core/doctype/transaction_log/test_transaction_log.py b/frappe/core/doctype/transaction_log/test_transaction_log.py index 164a683c38..0d9b9353d0 100644 --- a/frappe/core/doctype/transaction_log/test_transaction_log.py +++ b/frappe/core/doctype/transaction_log/test_transaction_log.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2018, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import frappe import unittest import hashlib @@ -35,7 +33,7 @@ class TestTransactionLog(unittest.TestCase): sha = hashlib.sha256() sha.update( - frappe.safe_encode(str(third_log.transaction_hash)) + + frappe.safe_encode(str(third_log.transaction_hash)) + frappe.safe_encode(str(second_log.chaining_hash)) ) diff --git a/frappe/core/doctype/transaction_log/transaction_log.py b/frappe/core/doctype/transaction_log/transaction_log.py index b7ea6cac60..58d0b3d176 100644 --- a/frappe/core/doctype/transaction_log/transaction_log.py +++ b/frappe/core/doctype/transaction_log/transaction_log.py @@ -2,7 +2,6 @@ # Copyright (c) 2018, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe import _ from frappe.model.document import Document diff --git a/frappe/core/doctype/translation/test_translation.py b/frappe/core/doctype/translation/test_translation.py index 12899dddf7..ae1293b38f 100644 --- a/frappe/core/doctype/translation/test_translation.py +++ b/frappe/core/doctype/translation/test_translation.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import frappe import unittest diff --git a/frappe/core/doctype/translation/translation.py b/frappe/core/doctype/translation/translation.py index 177dea401f..b1f4642791 100644 --- a/frappe/core/doctype/translation/translation.py +++ b/frappe/core/doctype/translation/translation.py @@ -2,7 +2,6 @@ # Copyright (c) 2015, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document from frappe.utils import strip_html_tags, is_html diff --git a/frappe/core/doctype/user/test_user.py b/frappe/core/doctype/user/test_user.py index 5bea767934..392128834d 100644 --- a/frappe/core/doctype/user/test_user.py +++ b/frappe/core/doctype/user/test_user.py @@ -1,7 +1,5 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals - import frappe, unittest, uuid from frappe.model.delete_doc import delete_doc diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py index 0462de8643..3fa31cbf80 100644 --- a/frappe/core/doctype/user/user.py +++ b/frappe/core/doctype/user/user.py @@ -1,10 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt - -from __future__ import unicode_literals, print_function - from bs4 import BeautifulSoup - import frappe import frappe.share import frappe.defaults @@ -56,6 +52,7 @@ class User(Document): def after_insert(self): create_notification_settings(self.name) + frappe.cache().delete_key('users_for_mentions') def validate(self): self.check_demo() @@ -129,6 +126,9 @@ class User(Document): if self.time_zone: frappe.defaults.set_default("time_zone", self.time_zone, self.name) + if self.has_value_changed('allow_in_mentions') or self.has_value_changed('user_type'): + frappe.cache().delete_key('users_for_mentions') + def has_website_permission(self, ptype, user, verbose=False): """Returns true if current user is the session user""" return self.name == frappe.session.user @@ -389,6 +389,9 @@ class User(Document): # delete notification settings frappe.delete_doc("Notification Settings", self.name, ignore_permissions=True) + if self.get('allow_in_mentions'): + frappe.cache().delete_key('users_for_mentions') + def before_rename(self, old_name, new_name, merge=False): self.check_demo() diff --git a/frappe/core/doctype/user_document_type/user_document_type.py b/frappe/core/doctype/user_document_type/user_document_type.py index 979bfcb250..48dbf87b3d 100644 --- a/frappe/core/doctype/user_document_type/user_document_type.py +++ b/frappe/core/doctype/user_document_type/user_document_type.py @@ -2,7 +2,6 @@ # Copyright (c) 2021, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals # import frappe from frappe.model.document import Document diff --git a/frappe/core/doctype/user_email/user_email.py b/frappe/core/doctype/user_email/user_email.py index a0ce2e169d..729aa03444 100644 --- a/frappe/core/doctype/user_email/user_email.py +++ b/frappe/core/doctype/user_email/user_email.py @@ -2,7 +2,6 @@ # Copyright (c) 2015, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document diff --git a/frappe/core/doctype/user_group/test_user_group.py b/frappe/core/doctype/user_group/test_user_group.py index c7e28f3d31..2f89d032e1 100644 --- a/frappe/core/doctype/user_group/test_user_group.py +++ b/frappe/core/doctype/user_group/test_user_group.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2021, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - # import frappe import unittest diff --git a/frappe/core/doctype/user_group/user_group.py b/frappe/core/doctype/user_group/user_group.py index 64bffa06d0..178775d407 100644 --- a/frappe/core/doctype/user_group/user_group.py +++ b/frappe/core/doctype/user_group/user_group.py @@ -2,14 +2,13 @@ # Copyright (c) 2021, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals # import frappe from frappe.model.document import Document import frappe class UserGroup(Document): def after_insert(self): - frappe.publish_realtime('user_group_added', self.name) + frappe.cache().delete_key('user_groups') def on_trash(self): - frappe.publish_realtime('user_group_deleted', self.name) + frappe.cache().delete_key('user_groups') diff --git a/frappe/core/doctype/user_group_member/test_user_group_member.py b/frappe/core/doctype/user_group_member/test_user_group_member.py index 38aade4608..8dbaed9e65 100644 --- a/frappe/core/doctype/user_group_member/test_user_group_member.py +++ b/frappe/core/doctype/user_group_member/test_user_group_member.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2021, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - # import frappe import unittest diff --git a/frappe/core/doctype/user_group_member/user_group_member.py b/frappe/core/doctype/user_group_member/user_group_member.py index 4d0656913d..f85ddc3209 100644 --- a/frappe/core/doctype/user_group_member/user_group_member.py +++ b/frappe/core/doctype/user_group_member/user_group_member.py @@ -2,7 +2,6 @@ # Copyright (c) 2021, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals # import frappe from frappe.model.document import Document diff --git a/frappe/core/doctype/user_permission/test_user_permission.py b/frappe/core/doctype/user_permission/test_user_permission.py index 2e9b832acc..1a442b53e7 100644 --- a/frappe/core/doctype/user_permission/test_user_permission.py +++ b/frappe/core/doctype/user_permission/test_user_permission.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2017, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals from frappe.core.doctype.user_permission.user_permission import add_user_permissions, remove_applicable from frappe.permissions import has_user_permission from frappe.core.doctype.doctype.test_doctype import new_doctype @@ -46,7 +45,7 @@ class TestUserPermission(unittest.TestCase): frappe.set_user('test_user_perm1@example.com') doc = frappe.new_doc("Blog Post") - self.assertEquals(doc.blog_category, 'general') + self.assertEqual(doc.blog_category, 'general') frappe.set_user('Administrator') def test_apply_to_all(self): @@ -54,7 +53,7 @@ class TestUserPermission(unittest.TestCase): user = create_user('test_bulk_creation_update@example.com') param = get_params(user, 'User', user.name) is_created = add_user_permissions(param) - self.assertEquals(is_created, 1) + self.assertEqual(is_created, 1) def test_for_apply_to_all_on_update_from_apply_all(self): user = create_user('test_bulk_creation_update@example.com') @@ -63,11 +62,11 @@ class TestUserPermission(unittest.TestCase): # Initially create User Permission document with apply_to_all checked is_created = add_user_permissions(param) - self.assertEquals(is_created, 1) + self.assertEqual(is_created, 1) is_created = add_user_permissions(param) # User Permission should not be changed - self.assertEquals(is_created, 0) + self.assertEqual(is_created, 0) def test_for_applicable_on_update_from_apply_to_all(self): ''' Update User Permission from all to some applicable Doctypes''' @@ -77,7 +76,7 @@ class TestUserPermission(unittest.TestCase): # Initially create User Permission document with apply_to_all checked is_created = add_user_permissions(get_params(user, 'User', user.name)) - self.assertEquals(is_created, 1) + self.assertEqual(is_created, 1) is_created = add_user_permissions(param) frappe.db.commit() @@ -92,7 +91,7 @@ class TestUserPermission(unittest.TestCase): # Check that User Permissions for applicable is created self.assertIsNotNone(is_created_applicable_first) self.assertIsNotNone(is_created_applicable_second) - self.assertEquals(is_created, 1) + self.assertEqual(is_created, 1) def test_for_apply_to_all_on_update_from_applicable(self): ''' Update User Permission from some to all applicable Doctypes''' @@ -102,7 +101,7 @@ class TestUserPermission(unittest.TestCase): # create User permissions that with applicable is_created = add_user_permissions(get_params(user, 'User', user.name, applicable = ["Chat Room", "Chat Message"])) - self.assertEquals(is_created, 1) + self.assertEqual(is_created, 1) is_created = add_user_permissions(param) is_created_apply_to_all = frappe.db.exists("User Permission", get_exists_param(user)) @@ -115,7 +114,7 @@ class TestUserPermission(unittest.TestCase): # Check that all User Permission with applicable is removed self.assertIsNone(removed_applicable_first) self.assertIsNone(removed_applicable_second) - self.assertEquals(is_created, 1) + self.assertEqual(is_created, 1) def test_user_perm_for_nested_doctype(self): """Test if descendants' visibility is controlled for a nested DocType.""" @@ -183,7 +182,7 @@ class TestUserPermission(unittest.TestCase): # User perm is created on ToDo but for doctype Assignment Rule only # it should not have impact on Doc A - self.assertEquals(new_doc.doc, "ToDo") + self.assertEqual(new_doc.doc, "ToDo") frappe.set_user('Administrator') remove_applicable(["Assignment Rule"], "new_doc_test@example.com", "DocType", "ToDo") @@ -228,7 +227,7 @@ class TestUserPermission(unittest.TestCase): # User perm is created on ToDo but for doctype Assignment Rule only # it should not have impact on Doc A - self.assertEquals(new_doc.doc, "ToDo") + self.assertEqual(new_doc.doc, "ToDo") frappe.set_user('Administrator') clear_session_defaults() diff --git a/frappe/core/doctype/user_permission/user_permission.py b/frappe/core/doctype/user_permission/user_permission.py index fbc788f6bf..42ca4d7a14 100644 --- a/frappe/core/doctype/user_permission/user_permission.py +++ b/frappe/core/doctype/user_permission/user_permission.py @@ -2,7 +2,6 @@ # Copyright (c) 2017, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe, json from frappe.model.document import Document from frappe.permissions import (get_valid_perms, update_permission_property) @@ -191,7 +190,7 @@ def clear_user_permissions(user, for_doctype): def add_user_permissions(data): ''' Add and update the user permissions ''' frappe.only_for('System Manager') - if isinstance(data, frappe.string_types): + if isinstance(data, str): data = json.loads(data) data = frappe._dict(data) diff --git a/frappe/core/doctype/user_select_document_type/user_select_document_type.py b/frappe/core/doctype/user_select_document_type/user_select_document_type.py index 373eaf7aa3..13e3f0d351 100644 --- a/frappe/core/doctype/user_select_document_type/user_select_document_type.py +++ b/frappe/core/doctype/user_select_document_type/user_select_document_type.py @@ -2,7 +2,6 @@ # Copyright (c) 2021, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals # import frappe from frappe.model.document import Document diff --git a/frappe/core/doctype/user_social_login/user_social_login.py b/frappe/core/doctype/user_social_login/user_social_login.py index cc6c3d0e05..4a34006d2b 100644 --- a/frappe/core/doctype/user_social_login/user_social_login.py +++ b/frappe/core/doctype/user_social_login/user_social_login.py @@ -2,7 +2,6 @@ # Copyright (c) 2017, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals from frappe.model.document import Document class UserSocialLogin(Document): diff --git a/frappe/core/doctype/user_type/test_user_type.py b/frappe/core/doctype/user_type/test_user_type.py index de61e0f476..1c47f02bbb 100644 --- a/frappe/core/doctype/user_type/test_user_type.py +++ b/frappe/core/doctype/user_type/test_user_type.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2021, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - # import frappe import unittest diff --git a/frappe/core/doctype/user_type/user_type.py b/frappe/core/doctype/user_type/user_type.py index 0e8b692416..e7d06c45f2 100644 --- a/frappe/core/doctype/user_type/user_type.py +++ b/frappe/core/doctype/user_type/user_type.py @@ -2,10 +2,8 @@ # Copyright (c) 2021, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe import _ -from six import iteritems from frappe.utils import get_link_to_form from frappe.config import get_modules_from_app from frappe.permissions import add_permission, add_user_permission @@ -247,7 +245,7 @@ def apply_permissions_for_non_standard_user_type(doc, method=None): if not user_types: return - for user_type, data in iteritems(user_types): + for user_type, data in user_types.items(): if (not doc.get(data[1]) or doc.doctype != data[0]): continue diff --git a/frappe/core/doctype/user_type/user_type_dashboard.py b/frappe/core/doctype/user_type/user_type_dashboard.py index 7e14198bca..6cdd2f82a5 100644 --- a/frappe/core/doctype/user_type/user_type_dashboard.py +++ b/frappe/core/doctype/user_type/user_type_dashboard.py @@ -1,4 +1,4 @@ -from __future__ import unicode_literals + from frappe import _ def get_data(): diff --git a/frappe/core/doctype/user_type_module/user_type_module.py b/frappe/core/doctype/user_type_module/user_type_module.py index 6cd2cbacdb..9afbcd294d 100644 --- a/frappe/core/doctype/user_type_module/user_type_module.py +++ b/frappe/core/doctype/user_type_module/user_type_module.py @@ -2,7 +2,6 @@ # Copyright (c) 2021, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals # import frappe from frappe.model.document import Document diff --git a/frappe/core/doctype/version/test_version.py b/frappe/core/doctype/version/test_version.py index 51b3c21f58..f6c099c4ea 100644 --- a/frappe/core/doctype/version/test_version.py +++ b/frappe/core/doctype/version/test_version.py @@ -1,7 +1,5 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # See license.txt -from __future__ import unicode_literals - import frappe import unittest, copy from frappe.test_runner import make_test_objects diff --git a/frappe/core/doctype/version/version.py b/frappe/core/doctype/version/version.py index 7654db4ae5..a1bd851346 100644 --- a/frappe/core/doctype/version/version.py +++ b/frappe/core/doctype/version/version.py @@ -3,7 +3,6 @@ # For license information, please see license.txt -from __future__ import unicode_literals import frappe, json from frappe.model.document import Document diff --git a/frappe/core/doctype/view_log/test_view_log.py b/frappe/core/doctype/view_log/test_view_log.py index 83967a39a4..025f3d8ad9 100644 --- a/frappe/core/doctype/view_log/test_view_log.py +++ b/frappe/core/doctype/view_log/test_view_log.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2018, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import frappe import unittest @@ -25,11 +23,11 @@ class TestViewLog(unittest.TestCase): # load the form getdoc('Event', ev.name) a = frappe.get_value( - doctype="View Log", + doctype="View Log", filters={ "reference_doctype": "Event", "reference_name": ev.name - }, + }, fieldname=['viewed_by'] ) diff --git a/frappe/core/doctype/view_log/view_log.py b/frappe/core/doctype/view_log/view_log.py index 45e98e37c7..242250be8b 100644 --- a/frappe/core/doctype/view_log/view_log.py +++ b/frappe/core/doctype/view_log/view_log.py @@ -2,7 +2,6 @@ # Copyright (c) 2018, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document diff --git a/frappe/core/notifications.py b/frappe/core/notifications.py index 771a15a2e7..707de43f28 100644 --- a/frappe/core/notifications.py +++ b/frappe/core/notifications.py @@ -1,7 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import frappe def get_notification_config(): diff --git a/frappe/core/page/__init__.py b/frappe/core/page/__init__.py index 4dbcd0d163..0e57cb68c3 100644 --- a/frappe/core/page/__init__.py +++ b/frappe/core/page/__init__.py @@ -1,4 +1,3 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals diff --git a/frappe/core/page/permission_manager/permission_manager.py b/frappe/core/page/permission_manager/permission_manager.py index 1c215eb6e1..15c7cb55ae 100644 --- a/frappe/core/page/permission_manager/permission_manager.py +++ b/frappe/core/page/permission_manager/permission_manager.py @@ -1,7 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import frappe from frappe import _ import frappe.defaults diff --git a/frappe/core/page/recorder/recorder.js b/frappe/core/page/recorder/recorder.js index b75ea6a41c..f1f74daf71 100644 --- a/frappe/core/page/recorder/recorder.js +++ b/frappe/core/page/recorder/recorder.js @@ -1,7 +1,7 @@ frappe.pages['recorder'].on_page_load = function(wrapper) { frappe.ui.make_app_page({ parent: wrapper, - title: 'Recorder', + title: __('Recorder'), single_column: true, card_layout: true }); @@ -11,7 +11,7 @@ frappe.pages['recorder'].on_page_load = function(wrapper) { frappe.recorder.show(); }); - frappe.require('/assets/js/frappe-recorder.min.js'); + frappe.require('recorder.bundle.js'); }; class Recorder { diff --git a/frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.py b/frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.py index c928939119..13602ca777 100644 --- a/frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.py +++ b/frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.py @@ -1,7 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import frappe from frappe import _, throw import frappe.utils.user diff --git a/frappe/core/report/transaction_log_report/transaction_log_report.py b/frappe/core/report/transaction_log_report/transaction_log_report.py index 9d84901f22..ff8d8345d6 100644 --- a/frappe/core/report/transaction_log_report/transaction_log_report.py +++ b/frappe/core/report/transaction_log_report/transaction_log_report.py @@ -1,7 +1,6 @@ # Copyright (c) 2019, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe import hashlib from frappe import _ diff --git a/frappe/core/utils.py b/frappe/core/utils.py index 55cfbc34d7..9b8ee3a326 100644 --- a/frappe/core/utils.py +++ b/frappe/core/utils.py @@ -1,7 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import frappe diff --git a/frappe/core/web_form/edit_profile/edit_profile.py b/frappe/core/web_form/edit_profile/edit_profile.py index 2334f8b26d..e1ada61927 100644 --- a/frappe/core/web_form/edit_profile/edit_profile.py +++ b/frappe/core/web_form/edit_profile/edit_profile.py @@ -1,5 +1,3 @@ -from __future__ import unicode_literals - import frappe def get_context(context): diff --git a/frappe/custom/doctype/client_script/__init__.py b/frappe/custom/doctype/client_script/__init__.py index 4dbcd0d163..0e57cb68c3 100644 --- a/frappe/custom/doctype/client_script/__init__.py +++ b/frappe/custom/doctype/client_script/__init__.py @@ -1,4 +1,3 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals diff --git a/frappe/custom/doctype/client_script/client_script.py b/frappe/custom/doctype/client_script/client_script.py index 049f979263..9c098fe8c9 100644 --- a/frappe/custom/doctype/client_script/client_script.py +++ b/frappe/custom/doctype/client_script/client_script.py @@ -1,6 +1,5 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import frappe from frappe import _ diff --git a/frappe/custom/doctype/client_script/test_client_script.py b/frappe/custom/doctype/client_script/test_client_script.py index de113c1ce7..b8358468b9 100644 --- a/frappe/custom/doctype/client_script/test_client_script.py +++ b/frappe/custom/doctype/client_script/test_client_script.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # See license.txt -from __future__ import unicode_literals - import frappe import unittest diff --git a/frappe/custom/doctype/custom_field/__init__.py b/frappe/custom/doctype/custom_field/__init__.py index 4dbcd0d163..0e57cb68c3 100644 --- a/frappe/custom/doctype/custom_field/__init__.py +++ b/frappe/custom/doctype/custom_field/__init__.py @@ -1,4 +1,3 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals diff --git a/frappe/custom/doctype/custom_field/custom_field.py b/frappe/custom/doctype/custom_field/custom_field.py index fb49aa5da0..7e6ea1875a 100644 --- a/frappe/custom/doctype/custom_field/custom_field.py +++ b/frappe/custom/doctype/custom_field/custom_field.py @@ -1,7 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import frappe import json from frappe.utils import cstr @@ -64,8 +63,8 @@ class CustomField(Document): self.translatable = 0 if not self.flags.ignore_validate: - from frappe.core.doctype.doctype.doctype import check_if_fieldname_conflicts_with_methods - check_if_fieldname_conflicts_with_methods(self.dt, self.fieldname) + from frappe.core.doctype.doctype.doctype import check_fieldname_conflicts + check_fieldname_conflicts(self.dt, self.fieldname) def on_update(self): if not frappe.flags.in_setup_wizard: diff --git a/frappe/custom/doctype/custom_field/test_custom_field.py b/frappe/custom/doctype/custom_field/test_custom_field.py index 819917050a..3196b66ee8 100644 --- a/frappe/custom/doctype/custom_field/test_custom_field.py +++ b/frappe/custom/doctype/custom_field/test_custom_field.py @@ -3,8 +3,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # See license.txt -from __future__ import unicode_literals - import frappe import unittest diff --git a/frappe/custom/doctype/customize_form/__init__.py b/frappe/custom/doctype/customize_form/__init__.py index 4dbcd0d163..0e57cb68c3 100644 --- a/frappe/custom/doctype/customize_form/__init__.py +++ b/frappe/custom/doctype/customize_form/__init__.py @@ -1,4 +1,3 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals diff --git a/frappe/custom/doctype/customize_form/customize_form.js b/frappe/custom/doctype/customize_form/customize_form.js index d9d8ae196e..4e00456f0d 100644 --- a/frappe/custom/doctype/customize_form/customize_form.js +++ b/frappe/custom/doctype/customize_form/customize_form.js @@ -117,7 +117,7 @@ frappe.ui.form.on("Customize Form", { frappe.customize_form.set_primary_action(frm); frm.add_custom_button( - __("Go to {0} List", [frm.doc.doc_type]), + __("Go to {0} List", [__(frm.doc.doc_type)]), function() { frappe.set_route("List", frm.doc.doc_type); }, diff --git a/frappe/custom/doctype/customize_form/customize_form.json b/frappe/custom/doctype/customize_form/customize_form.json index 8d6a6a8ca7..b9dde88126 100644 --- a/frappe/custom/doctype/customize_form/customize_form.json +++ b/frappe/custom/doctype/customize_form/customize_form.json @@ -278,6 +278,7 @@ }, { "collapsible": 1, + "depends_on": "doc_type", "fieldname": "naming_section", "fieldtype": "Section Break", "label": "Naming" @@ -287,16 +288,6 @@ "fieldname": "autoname", "fieldtype": "Data", "label": "Auto Name" - }, - { - "fieldname": "default_email_template", - "fieldtype": "Link", - "label": "Default Email Template", - "options": "Email Template" - }, - { - "fieldname": "column_break_26", - "fieldtype": "Column Break" } ], "hide_toolbar": 1, @@ -305,7 +296,7 @@ "index_web_pages_for_search": 1, "issingle": 1, "links": [], - "modified": "2021-03-22 12:27:15.462727", + "modified": "2021-06-02 06:49:16.782806", "modified_by": "Administrator", "module": "Custom", "name": "Customize Form", @@ -326,4 +317,4 @@ "sort_field": "modified", "sort_order": "DESC", "track_changes": 1 -} +} \ No newline at end of file diff --git a/frappe/custom/doctype/customize_form/customize_form.py b/frappe/custom/doctype/customize_form/customize_form.py index be0dded99c..8bcc6cf059 100644 --- a/frappe/custom/doctype/customize_form/customize_form.py +++ b/frappe/custom/doctype/customize_form/customize_form.py @@ -1,7 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals """ Customize Form is a Single DocType used to mask the Property Setter Thus providing a better UI from user perspective diff --git a/frappe/custom/doctype/customize_form/test_customize_form.py b/frappe/custom/doctype/customize_form/test_customize_form.py index f5e0371c1f..58bdcf9a18 100644 --- a/frappe/custom/doctype/customize_form/test_customize_form.py +++ b/frappe/custom/doctype/customize_form/test_customize_form.py @@ -1,7 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import frappe, unittest, json from frappe.test_runner import make_test_records_for_doctype from frappe.core.doctype.doctype.doctype import InvalidFieldNameError @@ -47,64 +46,64 @@ class TestCustomizeForm(unittest.TestCase): self.assertEqual(len(d.get("fields")), 0) d = self.get_customize_form("Event") - self.assertEquals(d.doc_type, "Event") - self.assertEquals(len(d.get("fields")), 36) + self.assertEqual(d.doc_type, "Event") + self.assertEqual(len(d.get("fields")), 36) d = self.get_customize_form("Event") - self.assertEquals(d.doc_type, "Event") + self.assertEqual(d.doc_type, "Event") self.assertEqual(len(d.get("fields")), len(frappe.get_doc("DocType", d.doc_type).fields) + 1) - self.assertEquals(d.get("fields")[-1].fieldname, "test_custom_field") - self.assertEquals(d.get("fields", {"fieldname": "event_type"})[0].in_list_view, 1) + self.assertEqual(d.get("fields")[-1].fieldname, "test_custom_field") + self.assertEqual(d.get("fields", {"fieldname": "event_type"})[0].in_list_view, 1) return d def test_save_customization_property(self): d = self.get_customize_form("Event") - self.assertEquals(frappe.db.get_value("Property Setter", + self.assertEqual(frappe.db.get_value("Property Setter", {"doc_type": "Event", "property": "allow_copy"}, "value"), None) d.allow_copy = 1 d.run_method("save_customization") - self.assertEquals(frappe.db.get_value("Property Setter", + self.assertEqual(frappe.db.get_value("Property Setter", {"doc_type": "Event", "property": "allow_copy"}, "value"), '1') d.allow_copy = 0 d.run_method("save_customization") - self.assertEquals(frappe.db.get_value("Property Setter", + self.assertEqual(frappe.db.get_value("Property Setter", {"doc_type": "Event", "property": "allow_copy"}, "value"), None) def test_save_customization_field_property(self): d = self.get_customize_form("Event") - self.assertEquals(frappe.db.get_value("Property Setter", + self.assertEqual(frappe.db.get_value("Property Setter", {"doc_type": "Event", "property": "reqd", "field_name": "repeat_this_event"}, "value"), None) repeat_this_event_field = d.get("fields", {"fieldname": "repeat_this_event"})[0] repeat_this_event_field.reqd = 1 d.run_method("save_customization") - self.assertEquals(frappe.db.get_value("Property Setter", + self.assertEqual(frappe.db.get_value("Property Setter", {"doc_type": "Event", "property": "reqd", "field_name": "repeat_this_event"}, "value"), '1') repeat_this_event_field = d.get("fields", {"fieldname": "repeat_this_event"})[0] repeat_this_event_field.reqd = 0 d.run_method("save_customization") - self.assertEquals(frappe.db.get_value("Property Setter", + self.assertEqual(frappe.db.get_value("Property Setter", {"doc_type": "Event", "property": "reqd", "field_name": "repeat_this_event"}, "value"), None) def test_save_customization_custom_field_property(self): d = self.get_customize_form("Event") - self.assertEquals(frappe.db.get_value("Custom Field", "Event-test_custom_field", "reqd"), 0) + self.assertEqual(frappe.db.get_value("Custom Field", "Event-test_custom_field", "reqd"), 0) custom_field = d.get("fields", {"fieldname": "test_custom_field"})[0] custom_field.reqd = 1 d.run_method("save_customization") - self.assertEquals(frappe.db.get_value("Custom Field", "Event-test_custom_field", "reqd"), 1) + self.assertEqual(frappe.db.get_value("Custom Field", "Event-test_custom_field", "reqd"), 1) custom_field = d.get("fields", {"is_custom_field": True})[0] custom_field.reqd = 0 d.run_method("save_customization") - self.assertEquals(frappe.db.get_value("Custom Field", "Event-test_custom_field", "reqd"), 0) + self.assertEqual(frappe.db.get_value("Custom Field", "Event-test_custom_field", "reqd"), 0) def test_save_customization_new_field(self): d = self.get_customize_form("Event") @@ -115,14 +114,14 @@ class TestCustomizeForm(unittest.TestCase): "is_custom_field": 1 }) d.run_method("save_customization") - self.assertEquals(frappe.db.get_value("Custom Field", + self.assertEqual(frappe.db.get_value("Custom Field", "Event-test_add_custom_field_via_customize_form", "fieldtype"), "Data") - self.assertEquals(frappe.db.get_value("Custom Field", + self.assertEqual(frappe.db.get_value("Custom Field", "Event-test_add_custom_field_via_customize_form", 'insert_after'), last_fieldname) frappe.delete_doc("Custom Field", "Event-test_add_custom_field_via_customize_form") - self.assertEquals(frappe.db.get_value("Custom Field", + self.assertEqual(frappe.db.get_value("Custom Field", "Event-test_add_custom_field_via_customize_form"), None) @@ -142,7 +141,7 @@ class TestCustomizeForm(unittest.TestCase): d.doc_type = "Event" d.run_method('reset_to_defaults') - self.assertEquals(d.get("fields", {"fieldname": "repeat_this_event"})[0].in_list_view, 0) + self.assertEqual(d.get("fields", {"fieldname": "repeat_this_event"})[0].in_list_view, 0) frappe.local.test_objects["Property Setter"] = [] make_test_records_for_doctype("Property Setter") @@ -156,7 +155,7 @@ class TestCustomizeForm(unittest.TestCase): d = self.get_customize_form("Event") # don't allow for standard fields - self.assertEquals(d.get("fields", {"fieldname": "subject"})[0].allow_on_submit or 0, 0) + self.assertEqual(d.get("fields", {"fieldname": "subject"})[0].allow_on_submit or 0, 0) # allow for custom field self.assertEqual(d.get("fields", {"fieldname": "test_custom_field"})[0].allow_on_submit, 1) diff --git a/frappe/custom/doctype/customize_form_field/__init__.py b/frappe/custom/doctype/customize_form_field/__init__.py index 4dbcd0d163..0e57cb68c3 100644 --- a/frappe/custom/doctype/customize_form_field/__init__.py +++ b/frappe/custom/doctype/customize_form_field/__init__.py @@ -1,4 +1,3 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals diff --git a/frappe/custom/doctype/customize_form_field/customize_form_field.py b/frappe/custom/doctype/customize_form_field/customize_form_field.py index 20c206328c..f288e70754 100644 --- a/frappe/custom/doctype/customize_form_field/customize_form_field.py +++ b/frappe/custom/doctype/customize_form_field/customize_form_field.py @@ -1,7 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document diff --git a/frappe/custom/doctype/doctype_layout/doctype_layout.py b/frappe/custom/doctype/doctype_layout/doctype_layout.py index a4fe9a9bce..0dc320353d 100644 --- a/frappe/custom/doctype/doctype_layout/doctype_layout.py +++ b/frappe/custom/doctype/doctype_layout/doctype_layout.py @@ -2,8 +2,6 @@ # Copyright (c) 2020, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals - from frappe.model.document import Document from frappe.desk.utils import slug diff --git a/frappe/custom/doctype/doctype_layout/test_doctype_layout.py b/frappe/custom/doctype/doctype_layout/test_doctype_layout.py index 5765c86262..dcde3c00a4 100644 --- a/frappe/custom/doctype/doctype_layout/test_doctype_layout.py +++ b/frappe/custom/doctype/doctype_layout/test_doctype_layout.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - # import frappe import unittest diff --git a/frappe/custom/doctype/doctype_layout_field/doctype_layout_field.py b/frappe/custom/doctype/doctype_layout_field/doctype_layout_field.py index 7f8c8edfce..c1e963602f 100644 --- a/frappe/custom/doctype/doctype_layout_field/doctype_layout_field.py +++ b/frappe/custom/doctype/doctype_layout_field/doctype_layout_field.py @@ -2,7 +2,6 @@ # Copyright (c) 2020, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals # import frappe from frappe.model.document import Document diff --git a/frappe/custom/doctype/property_setter/__init__.py b/frappe/custom/doctype/property_setter/__init__.py index 4dbcd0d163..0e57cb68c3 100644 --- a/frappe/custom/doctype/property_setter/__init__.py +++ b/frappe/custom/doctype/property_setter/__init__.py @@ -1,4 +1,3 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals diff --git a/frappe/custom/doctype/property_setter/property_setter.py b/frappe/custom/doctype/property_setter/property_setter.py index 56e5829271..2a6c06b70a 100644 --- a/frappe/custom/doctype/property_setter/property_setter.py +++ b/frappe/custom/doctype/property_setter/property_setter.py @@ -1,7 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import frappe from frappe import _ diff --git a/frappe/custom/doctype/property_setter/test_property_setter.py b/frappe/custom/doctype/property_setter/test_property_setter.py index 33e7d288a4..4d4de66d51 100644 --- a/frappe/custom/doctype/property_setter/test_property_setter.py +++ b/frappe/custom/doctype/property_setter/test_property_setter.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # See license.txt -from __future__ import unicode_literals - import frappe import unittest diff --git a/frappe/custom/doctype/test_rename_new/test_rename_new.py b/frappe/custom/doctype/test_rename_new/test_rename_new.py index aa5984e466..32d2396b2b 100644 --- a/frappe/custom/doctype/test_rename_new/test_rename_new.py +++ b/frappe/custom/doctype/test_rename_new/test_rename_new.py @@ -2,7 +2,6 @@ # Copyright (c) 2021, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals # import frappe from frappe.model.document import Document diff --git a/frappe/custom/doctype/test_rename_new/test_test_rename_new.py b/frappe/custom/doctype/test_rename_new/test_test_rename_new.py index 554efbae45..b3ea4818de 100644 --- a/frappe/custom/doctype/test_rename_new/test_test_rename_new.py +++ b/frappe/custom/doctype/test_rename_new/test_test_rename_new.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2021, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - # import frappe import unittest diff --git a/frappe/data_migration/doctype/data_migration_connector/connectors/base.py b/frappe/data_migration/doctype/data_migration_connector/connectors/base.py index 97f9f5f4a3..5eca7cfac5 100644 --- a/frappe/data_migration/doctype/data_migration_connector/connectors/base.py +++ b/frappe/data_migration/doctype/data_migration_connector/connectors/base.py @@ -1,10 +1,7 @@ -from __future__ import unicode_literals -from six import with_metaclass from abc import ABCMeta, abstractmethod from frappe.utils.password import get_decrypted_password -class BaseConnection(with_metaclass(ABCMeta)): - +class BaseConnection(metaclass=ABCMeta): @abstractmethod def get(self, remote_objectname, fields=None, filters=None, start=0, page_length=10): pass diff --git a/frappe/data_migration/doctype/data_migration_connector/connectors/frappe_connection.py b/frappe/data_migration/doctype/data_migration_connector/connectors/frappe_connection.py index 6ee41afdf2..473a15c2dc 100644 --- a/frappe/data_migration/doctype/data_migration_connector/connectors/frappe_connection.py +++ b/frappe/data_migration/doctype/data_migration_connector/connectors/frappe_connection.py @@ -1,4 +1,4 @@ -from __future__ import unicode_literals + import frappe from frappe.frappeclient import FrappeClient from .base import BaseConnection diff --git a/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.py b/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.py index 793dfe6694..d1137f2e67 100644 --- a/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.py +++ b/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.py @@ -2,7 +2,6 @@ # Copyright (c) 2017, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe, os from frappe.model.document import Document from frappe import _ @@ -76,8 +75,7 @@ def get_connection_class(python_module): return _class -connection_boilerplate = """from __future__ import unicode_literals -from frappe.data_migration.doctype.data_migration_connector.connectors.base import BaseConnection +connection_boilerplate = """from frappe.data_migration.doctype.data_migration_connector.connectors.base import BaseConnection class {connection_class}(BaseConnection): def __init__(self, connector): diff --git a/frappe/data_migration/doctype/data_migration_connector/test_data_migration_connector.py b/frappe/data_migration/doctype/data_migration_connector/test_data_migration_connector.py index a6e30fbe44..fd45f86ec1 100644 --- a/frappe/data_migration/doctype/data_migration_connector/test_data_migration_connector.py +++ b/frappe/data_migration/doctype/data_migration_connector/test_data_migration_connector.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2017, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals import unittest class TestDataMigrationConnector(unittest.TestCase): diff --git a/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.py b/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.py index 1cc54a0d1a..5cb20ba56c 100644 --- a/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.py +++ b/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.py @@ -2,7 +2,6 @@ # Copyright (c) 2017, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document from frappe.utils.safe_exec import get_safe_globals diff --git a/frappe/data_migration/doctype/data_migration_mapping/test_data_migration_mapping.py b/frappe/data_migration/doctype/data_migration_mapping/test_data_migration_mapping.py index e6f0ce2796..df11fc0522 100644 --- a/frappe/data_migration/doctype/data_migration_mapping/test_data_migration_mapping.py +++ b/frappe/data_migration/doctype/data_migration_mapping/test_data_migration_mapping.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2017, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals import unittest class TestDataMigrationMapping(unittest.TestCase): diff --git a/frappe/data_migration/doctype/data_migration_mapping_detail/data_migration_mapping_detail.py b/frappe/data_migration/doctype/data_migration_mapping_detail/data_migration_mapping_detail.py index 1ccdf76eed..6d3ef50937 100644 --- a/frappe/data_migration/doctype/data_migration_mapping_detail/data_migration_mapping_detail.py +++ b/frappe/data_migration/doctype/data_migration_mapping_detail/data_migration_mapping_detail.py @@ -2,7 +2,6 @@ # Copyright (c) 2017, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals from frappe.model.document import Document class DataMigrationMappingDetail(Document): diff --git a/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.py b/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.py index 5cd195f4fe..a8d0e40a4c 100644 --- a/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.py +++ b/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.py @@ -2,7 +2,6 @@ # Copyright (c) 2017, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe.modules import get_module_path, scrub_dt_dn from frappe.modules.export_file import export_to_files, create_init_py diff --git a/frappe/data_migration/doctype/data_migration_plan/test_data_migration_plan.py b/frappe/data_migration/doctype/data_migration_plan/test_data_migration_plan.py index 3a33039c3d..14c585a82d 100644 --- a/frappe/data_migration/doctype/data_migration_plan/test_data_migration_plan.py +++ b/frappe/data_migration/doctype/data_migration_plan/test_data_migration_plan.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2017, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals import unittest class TestDataMigrationPlan(unittest.TestCase): diff --git a/frappe/data_migration/doctype/data_migration_plan_mapping/data_migration_plan_mapping.py b/frappe/data_migration/doctype/data_migration_plan_mapping/data_migration_plan_mapping.py index 85f879069c..ba4cf28eb8 100644 --- a/frappe/data_migration/doctype/data_migration_plan_mapping/data_migration_plan_mapping.py +++ b/frappe/data_migration/doctype/data_migration_plan_mapping/data_migration_plan_mapping.py @@ -2,7 +2,6 @@ # Copyright (c) 2017, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals from frappe.model.document import Document class DataMigrationPlanMapping(Document): diff --git a/frappe/data_migration/doctype/data_migration_run/data_migration_run.py b/frappe/data_migration/doctype/data_migration_run/data_migration_run.py index aed9c6cb1d..c35af5827b 100644 --- a/frappe/data_migration/doctype/data_migration_run/data_migration_run.py +++ b/frappe/data_migration/doctype/data_migration_run/data_migration_run.py @@ -2,7 +2,6 @@ # Copyright (c) 2017, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe, json, math from frappe.model.document import Document from frappe import _ diff --git a/frappe/data_migration/doctype/data_migration_run/test_data_migration_run.py b/frappe/data_migration/doctype/data_migration_run/test_data_migration_run.py index c6c3ea138c..ef7b70dca2 100644 --- a/frappe/data_migration/doctype/data_migration_run/test_data_migration_run.py +++ b/frappe/data_migration/doctype/data_migration_run/test_data_migration_run.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2017, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals import frappe, unittest class TestDataMigrationRun(unittest.TestCase): diff --git a/frappe/database/__init__.py b/frappe/database/__init__.py index 1f0d3f9bf5..a899bec3d1 100644 --- a/frappe/database/__init__.py +++ b/frappe/database/__init__.py @@ -4,8 +4,6 @@ # Database Module # -------------------- -from __future__ import unicode_literals - def setup_database(force, source_sql=None, verbose=None, no_mariadb_socket=False): import frappe if frappe.conf.db_type == 'postgres': diff --git a/frappe/database/database.py b/frappe/database/database.py index 58e5c8a46e..7e8d2da43b 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -4,8 +4,6 @@ # Database Module # -------------------- -from __future__ import unicode_literals - import re import time import frappe @@ -19,13 +17,6 @@ from frappe.utils import now, getdate, cast_fieldtype, get_datetime from frappe.model.utils.link_count import flush_local_link_count from frappe.utils import cint -# imports - compatibility imports -from six import ( - integer_types, - string_types, - text_type, - iteritems -) class Database(object): """ @@ -277,7 +268,7 @@ class Database(object): for r in result: values = [] for value in r: - if as_utf8 and isinstance(value, text_type): + if as_utf8 and isinstance(value, str): value = value.encode('utf-8') values.append(value) @@ -294,7 +285,7 @@ class Database(object): """Returns true if the first row in the result has a Date, Datetime, Long Int.""" if result and result[0]: for v in result[0]: - if isinstance(v, (datetime.date, datetime.timedelta, datetime.datetime, integer_types)): + if isinstance(v, (datetime.date, datetime.timedelta, datetime.datetime, int)): return True if formatted and isinstance(v, (int, float)): return True @@ -312,7 +303,7 @@ class Database(object): for r in res: nr = [] for val in r: - if as_utf8 and isinstance(val, text_type): + if as_utf8 and isinstance(val, str): val = val.encode('utf-8') nr.append(val) nres.append(nr) @@ -363,7 +354,7 @@ class Database(object): # docname is a number, convert to string filters = str(filters) - if isinstance(filters, string_types): + if isinstance(filters, str): filters = { "name": filters } for f in filters: @@ -428,7 +419,7 @@ class Database(object): user = frappe.db.get_values("User", "test@example.com", "*")[0] """ out = None - if cache and isinstance(filters, string_types) and \ + if cache and isinstance(filters, str) and \ (doctype, filters, fieldname) in self.value_cache: return self.value_cache[(doctype, filters, fieldname)] @@ -440,7 +431,7 @@ class Database(object): else: fields = fieldname if fieldname!="*": - if isinstance(fieldname, string_types): + if isinstance(fieldname, str): fields = [fieldname] else: fields = fieldname @@ -461,7 +452,7 @@ class Database(object): else: out = self.get_values_from_single(fields, filters, doctype, as_dict, debug, update) - if cache and isinstance(filters, string_types): + if cache and isinstance(filters, str): self.value_cache[(doctype, filters, fieldname)] = out return out @@ -673,7 +664,7 @@ class Database(object): where field in ({0}) and doctype=%s'''.format(', '.join(['%s']*len(keys))), list(keys) + [dt], debug=debug) - for key, value in iteritems(to_update): + for key, value in to_update.items(): self.sql('''insert into `tabSingles` (doctype, field, value) values (%s, %s, %s)''', (dt, key, value), debug=debug) @@ -811,7 +802,7 @@ class Database(object): :param dt: DocType name. :param dn: Document name or filter dict.""" - if isinstance(dt, string_types): + if isinstance(dt, str): if dt!="DocType" and dt==dn: return True # single always exists (!) try: @@ -858,7 +849,7 @@ class Database(object): if not datetime: return '0001-01-01 00:00:00.000000' - if isinstance(datetime, frappe.string_types): + if isinstance(datetime, str): if ':' not in datetime: datetime = datetime + ' 00:00:00.000000' else: diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index 7d1d92408c..879c8394d7 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -1,5 +1,3 @@ -import warnings - import pymysql from pymysql.constants import ER, FIELD_TYPE from pymysql.converters import conversions, escape_string @@ -55,7 +53,6 @@ class MariaDBDatabase(Database): } def get_connection(self): - warnings.filterwarnings('ignore', category=pymysql.Warning) usessl = 0 if frappe.conf.db_ssl_ca and frappe.conf.db_ssl_cert and frappe.conf.db_ssl_key: usessl = 1 diff --git a/frappe/database/mariadb/schema.py b/frappe/database/mariadb/schema.py index 4bbecd2a2e..b40af59286 100644 --- a/frappe/database/mariadb/schema.py +++ b/frappe/database/mariadb/schema.py @@ -1,5 +1,3 @@ -from __future__ import unicode_literals - import frappe from frappe import _ from frappe.database.schema import DBTable diff --git a/frappe/database/mariadb/setup_db.py b/frappe/database/mariadb/setup_db.py index 9b73d77171..6be08c66bb 100644 --- a/frappe/database/mariadb/setup_db.py +++ b/frappe/database/mariadb/setup_db.py @@ -1,5 +1,3 @@ -from __future__ import unicode_literals - import frappe import os from frappe.database.db_manager import DbManager diff --git a/frappe/database/postgres/database.py b/frappe/database/postgres/database.py index 4faea78551..8235277e30 100644 --- a/frappe/database/postgres/database.py +++ b/frappe/database/postgres/database.py @@ -1,10 +1,7 @@ -from __future__ import unicode_literals - import re import frappe import psycopg2 import psycopg2.extensions -from six import string_types from frappe.utils import cstr from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT @@ -13,9 +10,9 @@ from frappe.database.postgres.schema import PostgresTable # cast decimals as floats DEC2FLOAT = psycopg2.extensions.new_type( - psycopg2.extensions.DECIMAL.values, - 'DEC2FLOAT', - lambda value, curs: float(value) if value is not None else None) + psycopg2.extensions.DECIMAL.values, + 'DEC2FLOAT', + lambda value, curs: float(value) if value is not None else None) psycopg2.extensions.register_type(DEC2FLOAT) @@ -65,7 +62,6 @@ class PostgresDatabase(Database): } def get_connection(self): - # warnings.filterwarnings('ignore', category=psycopg2.Warning) conn = psycopg2.connect("host='{}' dbname='{}' user='{}' password='{}' port={}".format( self.host, self.user, self.user, self.password, self.port )) @@ -114,7 +110,7 @@ class PostgresDatabase(Database): if not date: return '0001-01-01' - if not isinstance(date, frappe.string_types): + if not isinstance(date, str): date = date.strftime('%Y-%m-%d') return date @@ -256,7 +252,7 @@ class PostgresDatabase(Database): self.sql("""CREATE INDEX IF NOT EXISTS "{}" ON `{}`("{}")""".format(index_name, table_name, '", "'.join(fields))) def add_unique(self, doctype, fields, constraint_name=None): - if isinstance(fields, string_types): + if isinstance(fields, str): fields = [fields] if not constraint_name: constraint_name = "unique_" + "_".join(fields) diff --git a/frappe/database/postgres/setup_db.py b/frappe/database/postgres/setup_db.py index 3ee6b6a286..19ba681237 100644 --- a/frappe/database/postgres/setup_db.py +++ b/frappe/database/postgres/setup_db.py @@ -83,7 +83,6 @@ def get_root_connection(root_login=None, root_password=None): root_login = frappe.conf.get("root_login") or None if not root_login: - from six.moves import input root_login = input("Enter postgres super user: ") if not root_password: diff --git a/frappe/database/schema.py b/frappe/database/schema.py index 5f5ba06d8b..31f11dbd5e 100644 --- a/frappe/database/schema.py +++ b/frappe/database/schema.py @@ -1,5 +1,3 @@ -from __future__ import unicode_literals - import re import frappe diff --git a/frappe/defaults.py b/frappe/defaults.py index 4bec6677c7..fde48d71ff 100644 --- a/frappe/defaults.py +++ b/frappe/defaults.py @@ -1,7 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import frappe from frappe.desk.notifications import clear_notifications from frappe.cache_manager import clear_defaults_cache, common_default_keys diff --git a/frappe/desk/__init__.py b/frappe/desk/__init__.py index 4dbcd0d163..0e57cb68c3 100644 --- a/frappe/desk/__init__.py +++ b/frappe/desk/__init__.py @@ -1,4 +1,3 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals diff --git a/frappe/desk/calendar.py b/frappe/desk/calendar.py index 064d870092..f00f729415 100644 --- a/frappe/desk/calendar.py +++ b/frappe/desk/calendar.py @@ -1,8 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals - import frappe from frappe import _ import json diff --git a/frappe/desk/desk_page.py b/frappe/desk/desk_page.py index 6c5fdc6821..d373dbda0e 100644 --- a/frappe/desk/desk_page.py +++ b/frappe/desk/desk_page.py @@ -1,7 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import frappe from frappe.translate import send_translations diff --git a/frappe/desk/desktop.py b/frappe/desk/desktop.py index d1b5e27a2f..0a7d436169 100644 --- a/frappe/desk/desktop.py +++ b/frappe/desk/desktop.py @@ -2,12 +2,10 @@ # MIT License. See license.txt # Author - Shivam Mishra -from __future__ import unicode_literals import frappe from json import loads, dumps from frappe import _, DoesNotExistError, ValidationError, _dict from frappe.boot import get_allowed_pages, get_allowed_reports -from six import string_types from functools import wraps from frappe.cache_manager import ( build_domain_restriced_doctype_cache, @@ -61,7 +59,7 @@ class Workspace: shortcuts = self.doc.shortcuts + self.extended_shortcuts for section in cards: - links = loads(section.get('links')) if isinstance(section.get('links'), string_types) else section.get('links') + links = loads(section.get('links')) if isinstance(section.get('links'), str) else section.get('links') for item in links: if self.is_item_allowed(item.get('link_to'), item.get('link_type')): return True @@ -359,15 +357,18 @@ def get_desktop_page(page): Returns: dict: dictionary of cards, charts and shortcuts to be displayed on website """ - wspace = Workspace(page) - wspace.build_workspace() - return { - 'charts': wspace.charts, - 'shortcuts': wspace.shortcuts, - 'cards': wspace.cards, - 'onboarding': wspace.onboarding, - 'allow_customization': not wspace.doc.disable_user_customization - } + try: + wspace = Workspace(page) + wspace.build_workspace() + return { + 'charts': wspace.charts, + 'shortcuts': wspace.shortcuts, + 'cards': wspace.cards, + 'onboarding': wspace.onboarding, + 'allow_customization': not wspace.doc.disable_user_customization + } + except DoesNotExistError: + return {} @frappe.whitelist() def get_desk_sidebar_items(): @@ -608,3 +609,4 @@ def merge_cards_based_on_label(cards): cards_dict[label] = card return list(cards_dict.values()) + diff --git a/frappe/desk/doctype/bulk_update/bulk_update.py b/frappe/desk/doctype/bulk_update/bulk_update.py index 9b9f7d7a73..469ee839f1 100644 --- a/frappe/desk/doctype/bulk_update/bulk_update.py +++ b/frappe/desk/doctype/bulk_update/bulk_update.py @@ -2,7 +2,6 @@ # Copyright (c) 2015, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document from frappe import _ diff --git a/frappe/desk/doctype/calendar_view/calendar_view.py b/frappe/desk/doctype/calendar_view/calendar_view.py index ae8ab1eb46..3a986f3273 100644 --- a/frappe/desk/doctype/calendar_view/calendar_view.py +++ b/frappe/desk/doctype/calendar_view/calendar_view.py @@ -2,7 +2,6 @@ # Copyright (c) 2017, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals from frappe.model.document import Document class CalendarView(Document): diff --git a/frappe/desk/doctype/console_log/console_log.py b/frappe/desk/doctype/console_log/console_log.py index 635c4c1ba7..5d0f1cfa93 100644 --- a/frappe/desk/doctype/console_log/console_log.py +++ b/frappe/desk/doctype/console_log/console_log.py @@ -2,7 +2,6 @@ # Copyright (c) 2020, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals # import frappe from frappe.model.document import Document diff --git a/frappe/desk/doctype/console_log/test_console_log.py b/frappe/desk/doctype/console_log/test_console_log.py index 04dc4f241f..3bb1605204 100644 --- a/frappe/desk/doctype/console_log/test_console_log.py +++ b/frappe/desk/doctype/console_log/test_console_log.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - # import frappe import unittest diff --git a/frappe/desk/doctype/dashboard/dashboard.py b/frappe/desk/doctype/dashboard/dashboard.py index 4e66318769..1d333609db 100644 --- a/frappe/desk/doctype/dashboard/dashboard.py +++ b/frappe/desk/doctype/dashboard/dashboard.py @@ -2,7 +2,6 @@ # Copyright (c) 2019, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals from frappe.model.document import Document from frappe.modules.export_file import export_to_files from frappe.config import get_modules_from_all_apps_for_user @@ -22,7 +21,7 @@ class Dashboard(Document): def validate(self): if not frappe.conf.developer_mode and self.is_standard: - frappe.throw('Cannot edit Standard Dashboards') + frappe.throw(_("Cannot edit Standard Dashboards")) if self.is_standard: non_standard_docs_map = { diff --git a/frappe/desk/doctype/dashboard/test_dashboard.py b/frappe/desk/doctype/dashboard/test_dashboard.py index d5485d8f70..dd1bc31d86 100644 --- a/frappe/desk/doctype/dashboard/test_dashboard.py +++ b/frappe/desk/doctype/dashboard/test_dashboard.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import unittest class TestDashboard(unittest.TestCase): diff --git a/frappe/desk/doctype/dashboard_chart/dashboard_chart.py b/frappe/desk/doctype/dashboard_chart/dashboard_chart.py index 48b34e6cd9..db5964e7b2 100644 --- a/frappe/desk/doctype/dashboard_chart/dashboard_chart.py +++ b/frappe/desk/doctype/dashboard_chart/dashboard_chart.py @@ -2,15 +2,13 @@ # Copyright (c) 2019, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe import _ import datetime import json from frappe.utils.dashboard import cache_source from frappe.utils import nowdate, getdate, get_datetime, cint, now_datetime -from frappe.utils.dateutils import\ - get_period, get_period_beginning, get_from_date_from_timespan, get_dates_from_timegrain +from frappe.utils.dateutils import get_period, get_period_beginning, get_from_date_from_timespan, get_dates_from_timegrain from frappe.model.naming import append_number_if_name_exists from frappe.boot import get_allowed_reports from frappe.config import get_modules_from_all_apps_for_user @@ -326,7 +324,7 @@ class DashboardChart(Document): def validate(self): if not frappe.conf.developer_mode and self.is_standard: - frappe.throw('Cannot edit Standard charts') + frappe.throw(_("Cannot edit Standard charts")) if self.chart_type != 'Custom' and self.chart_type != 'Report': self.check_required_field() self.check_document_type() diff --git a/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py b/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py index 72ab18385d..78d133b2d5 100644 --- a/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py +++ b/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import unittest, frappe from frappe.utils import getdate, formatdate, get_last_day from frappe.utils.dateutils import get_period_ending, get_period diff --git a/frappe/desk/doctype/dashboard_chart_field/dashboard_chart_field.py b/frappe/desk/doctype/dashboard_chart_field/dashboard_chart_field.py index 734f27cc28..7d6f66daa2 100644 --- a/frappe/desk/doctype/dashboard_chart_field/dashboard_chart_field.py +++ b/frappe/desk/doctype/dashboard_chart_field/dashboard_chart_field.py @@ -2,7 +2,6 @@ # Copyright (c) 2020, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals # import frappe from frappe.model.document import Document diff --git a/frappe/desk/doctype/dashboard_chart_link/dashboard_chart_link.py b/frappe/desk/doctype/dashboard_chart_link/dashboard_chart_link.py index 7cd4f9daa3..359801a303 100644 --- a/frappe/desk/doctype/dashboard_chart_link/dashboard_chart_link.py +++ b/frappe/desk/doctype/dashboard_chart_link/dashboard_chart_link.py @@ -2,7 +2,6 @@ # Copyright (c) 2019, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals # import frappe from frappe.model.document import Document diff --git a/frappe/desk/doctype/dashboard_chart_source/dashboard_chart_source.py b/frappe/desk/doctype/dashboard_chart_source/dashboard_chart_source.py index 6685009078..791dbc563b 100644 --- a/frappe/desk/doctype/dashboard_chart_source/dashboard_chart_source.py +++ b/frappe/desk/doctype/dashboard_chart_source/dashboard_chart_source.py @@ -2,7 +2,6 @@ # Copyright (c) 2019, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe, os from frappe import _ from frappe.model.document import Document diff --git a/frappe/desk/doctype/dashboard_chart_source/test_dashboard_chart_source.py b/frappe/desk/doctype/dashboard_chart_source/test_dashboard_chart_source.py index 822526b591..53fe127dfb 100644 --- a/frappe/desk/doctype/dashboard_chart_source/test_dashboard_chart_source.py +++ b/frappe/desk/doctype/dashboard_chart_source/test_dashboard_chart_source.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import unittest class TestDashboardChartSource(unittest.TestCase): diff --git a/frappe/desk/doctype/dashboard_settings/dashboard_settings.py b/frappe/desk/doctype/dashboard_settings/dashboard_settings.py index 4697d897fc..df61c52114 100644 --- a/frappe/desk/doctype/dashboard_settings/dashboard_settings.py +++ b/frappe/desk/doctype/dashboard_settings/dashboard_settings.py @@ -2,7 +2,6 @@ # Copyright (c) 2020, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals # import frappe from frappe.model.document import Document import frappe diff --git a/frappe/desk/doctype/desktop_icon/desktop_icon.py b/frappe/desk/doctype/desktop_icon/desktop_icon.py index fcf10ef61d..81a79cdb09 100644 --- a/frappe/desk/doctype/desktop_icon/desktop_icon.py +++ b/frappe/desk/doctype/desktop_icon/desktop_icon.py @@ -2,14 +2,11 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and contributors # For license information, please see license.txt -from __future__ import unicode_literals - import frappe from frappe import _ import json import random from frappe.model.document import Document -from six import iteritems, string_types from frappe.utils.user import UserPermissions class DesktopIcon(Document): @@ -173,7 +170,7 @@ def add_user_icon(_doctype, _report=None, label=None, link=None, type='link', st @frappe.whitelist() def set_order(new_order, user=None): '''set new order by duplicating user icons (if user is set) or set global order''' - if isinstance(new_order, string_types): + if isinstance(new_order, str): new_order = json.loads(new_order) for i, module_name in enumerate(new_order): if module_name not in ('Explore',): @@ -232,7 +229,7 @@ def set_hidden_list(hidden_list, user=None): '''Sets property `hidden`=1 in **Desktop Icon** for given user. If user is None then it will set global values. It will also set the rest of the icons as shown (`hidden` = 0)''' - if isinstance(hidden_list, string_types): + if isinstance(hidden_list, str): hidden_list = json.loads(hidden_list) # set as hidden @@ -329,7 +326,7 @@ def sync_from_app(app): if isinstance(modules, dict): modules_list = [] - for m, desktop_icon in iteritems(modules): + for m, desktop_icon in modules.items(): desktop_icon['module_name'] = m modules_list.append(desktop_icon) else: diff --git a/frappe/desk/doctype/event/__init__.py b/frappe/desk/doctype/event/__init__.py index 4dbcd0d163..0e57cb68c3 100644 --- a/frappe/desk/doctype/event/__init__.py +++ b/frappe/desk/doctype/event/__init__.py @@ -1,4 +1,3 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals diff --git a/frappe/desk/doctype/event/event.py b/frappe/desk/doctype/event/event.py index 54905bed6a..57c89eaf2e 100644 --- a/frappe/desk/doctype/event/event.py +++ b/frappe/desk/doctype/event/event.py @@ -1,9 +1,7 @@ # Copyright (c) 2018, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals -from six.moves import range -from six import string_types + import frappe import json @@ -106,7 +104,7 @@ class Event(Document): @frappe.whitelist() def delete_communication(event, reference_doctype, reference_docname): deleted_participant = frappe.get_doc(reference_doctype, reference_docname) - if isinstance(event, string_types): + if isinstance(event, str): event = json.loads(event) filters = [ @@ -168,7 +166,7 @@ def get_events(start, end, user=None, for_reminder=False, filters=None): if not user: user = frappe.session.user - if isinstance(filters, string_types): + if isinstance(filters, str): filters = json.loads(filters) filter_condition = get_filters_cond('Event', filters, []) diff --git a/frappe/desk/doctype/event/test_event.py b/frappe/desk/doctype/event/test_event.py index 2926a74a55..77211946a9 100644 --- a/frappe/desk/doctype/event/test_event.py +++ b/frappe/desk/doctype/event/test_event.py @@ -1,7 +1,5 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals - """Use blog post test to test user permissions logic""" import frappe diff --git a/frappe/desk/doctype/event_participants/event_participants.py b/frappe/desk/doctype/event_participants/event_participants.py index 18e4672140..ca4fae9930 100644 --- a/frappe/desk/doctype/event_participants/event_participants.py +++ b/frappe/desk/doctype/event_participants/event_participants.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2018, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals from frappe.model.document import Document class EventParticipants(Document): diff --git a/frappe/desk/doctype/global_search_doctype/global_search_doctype.py b/frappe/desk/doctype/global_search_doctype/global_search_doctype.py index 4c9a948278..de8a48af01 100644 --- a/frappe/desk/doctype/global_search_doctype/global_search_doctype.py +++ b/frappe/desk/doctype/global_search_doctype/global_search_doctype.py @@ -2,7 +2,6 @@ # Copyright (c) 2019, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals # import frappe from frappe.model.document import Document diff --git a/frappe/desk/doctype/global_search_settings/global_search_settings.py b/frappe/desk/doctype/global_search_settings/global_search_settings.py index 85c9687ab3..28a1ed8239 100644 --- a/frappe/desk/doctype/global_search_settings/global_search_settings.py +++ b/frappe/desk/doctype/global_search_settings/global_search_settings.py @@ -2,7 +2,6 @@ # Copyright (c) 2019, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document from frappe import _ diff --git a/frappe/desk/doctype/kanban_board/kanban_board.py b/frappe/desk/doctype/kanban_board/kanban_board.py index a655e9e1da..5100727f43 100644 --- a/frappe/desk/doctype/kanban_board/kanban_board.py +++ b/frappe/desk/doctype/kanban_board/kanban_board.py @@ -2,12 +2,10 @@ # Copyright (c) 2015, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe import json from frappe import _ from frappe.model.document import Document -from six import iteritems class KanbanBoard(Document): @@ -107,7 +105,7 @@ def update_order(board_name, order): order_dict = json.loads(order) updated_cards = [] - for col_name, cards in iteritems(order_dict): + for col_name, cards in order_dict.items(): order_list = [] for card in cards: column = frappe.get_value( diff --git a/frappe/desk/doctype/kanban_board/test_kanban_board.py b/frappe/desk/doctype/kanban_board/test_kanban_board.py index 33947f4a54..f9503d736a 100644 --- a/frappe/desk/doctype/kanban_board/test_kanban_board.py +++ b/frappe/desk/doctype/kanban_board/test_kanban_board.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import frappe import unittest diff --git a/frappe/desk/doctype/kanban_board_column/kanban_board_column.py b/frappe/desk/doctype/kanban_board_column/kanban_board_column.py index 4ea30d21b2..aebba3351c 100644 --- a/frappe/desk/doctype/kanban_board_column/kanban_board_column.py +++ b/frappe/desk/doctype/kanban_board_column/kanban_board_column.py @@ -2,7 +2,6 @@ # Copyright (c) 2015, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document diff --git a/frappe/desk/doctype/list_filter/list_filter.py b/frappe/desk/doctype/list_filter/list_filter.py index 035f7e90b9..2467ae40a4 100644 --- a/frappe/desk/doctype/list_filter/list_filter.py +++ b/frappe/desk/doctype/list_filter/list_filter.py @@ -2,7 +2,6 @@ # Copyright (c) 2018, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe, json from frappe.model.document import Document diff --git a/frappe/desk/doctype/list_view_settings/list_view_settings.py b/frappe/desk/doctype/list_view_settings/list_view_settings.py index 74e029f499..f4a288b7ba 100644 --- a/frappe/desk/doctype/list_view_settings/list_view_settings.py +++ b/frappe/desk/doctype/list_view_settings/list_view_settings.py @@ -2,7 +2,6 @@ # Copyright (c) 2020, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document diff --git a/frappe/desk/doctype/list_view_settings/test_list_view_settings.py b/frappe/desk/doctype/list_view_settings/test_list_view_settings.py index c1b2f4a0da..00010d7604 100644 --- a/frappe/desk/doctype/list_view_settings/test_list_view_settings.py +++ b/frappe/desk/doctype/list_view_settings/test_list_view_settings.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - # import frappe import unittest diff --git a/frappe/desk/doctype/module_onboarding/module_onboarding.py b/frappe/desk/doctype/module_onboarding/module_onboarding.py index 8315c0b304..6f01e0fd8d 100644 --- a/frappe/desk/doctype/module_onboarding/module_onboarding.py +++ b/frappe/desk/doctype/module_onboarding/module_onboarding.py @@ -2,7 +2,6 @@ # Copyright (c) 2020, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document from frappe.modules.export_file import export_to_files diff --git a/frappe/desk/doctype/module_onboarding/test_module_onboarding.py b/frappe/desk/doctype/module_onboarding/test_module_onboarding.py index ef305667b1..39184401a1 100644 --- a/frappe/desk/doctype/module_onboarding/test_module_onboarding.py +++ b/frappe/desk/doctype/module_onboarding/test_module_onboarding.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - # import frappe import unittest diff --git a/frappe/desk/doctype/note/note.py b/frappe/desk/doctype/note/note.py index c54689418e..790f9a514c 100644 --- a/frappe/desk/doctype/note/note.py +++ b/frappe/desk/doctype/note/note.py @@ -1,7 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: See license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document diff --git a/frappe/desk/doctype/note/test_note.py b/frappe/desk/doctype/note/test_note.py index 38894a9c3d..1bb1730357 100644 --- a/frappe/desk/doctype/note/test_note.py +++ b/frappe/desk/doctype/note/test_note.py @@ -1,7 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors and Contributors # See license.txt -from __future__ import unicode_literals import frappe import unittest diff --git a/frappe/desk/doctype/note_seen_by/note_seen_by.py b/frappe/desk/doctype/note_seen_by/note_seen_by.py index 6123f20929..cec4628b20 100644 --- a/frappe/desk/doctype/note_seen_by/note_seen_by.py +++ b/frappe/desk/doctype/note_seen_by/note_seen_by.py @@ -2,7 +2,6 @@ # Copyright (c) 2015, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document diff --git a/frappe/desk/doctype/notification_log/notification_log.py b/frappe/desk/doctype/notification_log/notification_log.py index 20551559fd..414f272f59 100644 --- a/frappe/desk/doctype/notification_log/notification_log.py +++ b/frappe/desk/doctype/notification_log/notification_log.py @@ -2,7 +2,6 @@ # Copyright (c) 2019, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe import _ from frappe.model.document import Document @@ -46,7 +45,7 @@ def enqueue_create_notification(users, doc): doc = frappe._dict(doc) - if isinstance(users, frappe.string_types): + if isinstance(users, str): users = [user.strip() for user in users.split(',') if user.strip()] users = list(set(users)) diff --git a/frappe/desk/doctype/notification_log/test_notification_log.py b/frappe/desk/doctype/notification_log/test_notification_log.py index e59aee30c9..af4dee8df3 100644 --- a/frappe/desk/doctype/notification_log/test_notification_log.py +++ b/frappe/desk/doctype/notification_log/test_notification_log.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import frappe from frappe.desk.form.assign_to import add as assign_task import unittest diff --git a/frappe/desk/doctype/notification_settings/notification_settings.py b/frappe/desk/doctype/notification_settings/notification_settings.py index 4ab40bffe9..eb3a16435f 100644 --- a/frappe/desk/doctype/notification_settings/notification_settings.py +++ b/frappe/desk/doctype/notification_settings/notification_settings.py @@ -2,7 +2,6 @@ # Copyright (c) 2019, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document diff --git a/frappe/desk/doctype/notification_subscribed_document/notification_subscribed_document.py b/frappe/desk/doctype/notification_subscribed_document/notification_subscribed_document.py index f005efae76..6931e77754 100644 --- a/frappe/desk/doctype/notification_subscribed_document/notification_subscribed_document.py +++ b/frappe/desk/doctype/notification_subscribed_document/notification_subscribed_document.py @@ -2,7 +2,6 @@ # Copyright (c) 2019, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals # import frappe from frappe.model.document import Document diff --git a/frappe/desk/doctype/number_card/number_card.py b/frappe/desk/doctype/number_card/number_card.py index 7d1a697f6b..d8d5fe0953 100644 --- a/frappe/desk/doctype/number_card/number_card.py +++ b/frappe/desk/doctype/number_card/number_card.py @@ -2,7 +2,6 @@ # Copyright (c) 2020, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document from frappe.utils import cint diff --git a/frappe/desk/doctype/number_card/test_number_card.py b/frappe/desk/doctype/number_card/test_number_card.py index 4aa1ecf282..c395f5f915 100644 --- a/frappe/desk/doctype/number_card/test_number_card.py +++ b/frappe/desk/doctype/number_card/test_number_card.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - # import frappe import unittest diff --git a/frappe/desk/doctype/number_card_link/number_card_link.py b/frappe/desk/doctype/number_card_link/number_card_link.py index 67ad7e70cd..6c16f45f4b 100644 --- a/frappe/desk/doctype/number_card_link/number_card_link.py +++ b/frappe/desk/doctype/number_card_link/number_card_link.py @@ -2,7 +2,6 @@ # Copyright (c) 2020, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals # import frappe from frappe.model.document import Document diff --git a/frappe/desk/doctype/onboarding_permission/onboarding_permission.py b/frappe/desk/doctype/onboarding_permission/onboarding_permission.py index f8772480df..40d3dc33b1 100644 --- a/frappe/desk/doctype/onboarding_permission/onboarding_permission.py +++ b/frappe/desk/doctype/onboarding_permission/onboarding_permission.py @@ -2,7 +2,6 @@ # Copyright (c) 2020, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals # import frappe from frappe.model.document import Document diff --git a/frappe/desk/doctype/onboarding_permission/test_onboarding_permission.py b/frappe/desk/doctype/onboarding_permission/test_onboarding_permission.py index 9a7e8ae6fd..80b166de0a 100644 --- a/frappe/desk/doctype/onboarding_permission/test_onboarding_permission.py +++ b/frappe/desk/doctype/onboarding_permission/test_onboarding_permission.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - # import frappe import unittest diff --git a/frappe/desk/doctype/onboarding_step/onboarding_step.py b/frappe/desk/doctype/onboarding_step/onboarding_step.py index e1cc5dfba4..10bd8926ce 100644 --- a/frappe/desk/doctype/onboarding_step/onboarding_step.py +++ b/frappe/desk/doctype/onboarding_step/onboarding_step.py @@ -2,7 +2,6 @@ # Copyright (c) 2020, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals # import frappe from frappe.model.document import Document diff --git a/frappe/desk/doctype/onboarding_step/test_onboarding_step.py b/frappe/desk/doctype/onboarding_step/test_onboarding_step.py index 66bd0c6660..2425577478 100644 --- a/frappe/desk/doctype/onboarding_step/test_onboarding_step.py +++ b/frappe/desk/doctype/onboarding_step/test_onboarding_step.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - # import frappe import unittest diff --git a/frappe/desk/doctype/onboarding_step_map/onboarding_step_map.py b/frappe/desk/doctype/onboarding_step_map/onboarding_step_map.py index ea34de6088..c79244c4ad 100644 --- a/frappe/desk/doctype/onboarding_step_map/onboarding_step_map.py +++ b/frappe/desk/doctype/onboarding_step_map/onboarding_step_map.py @@ -2,7 +2,6 @@ # Copyright (c) 2020, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals # import frappe from frappe.model.document import Document diff --git a/frappe/desk/doctype/route_history/route_history.py b/frappe/desk/doctype/route_history/route_history.py index 12d898afa5..b82077f485 100644 --- a/frappe/desk/doctype/route_history/route_history.py +++ b/frappe/desk/doctype/route_history/route_history.py @@ -2,7 +2,6 @@ # Copyright (c) 2018, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document diff --git a/frappe/desk/doctype/system_console/system_console.py b/frappe/desk/doctype/system_console/system_console.py index 6c87ca8c36..e2b5656bc0 100644 --- a/frappe/desk/doctype/system_console/system_console.py +++ b/frappe/desk/doctype/system_console/system_console.py @@ -2,8 +2,6 @@ # Copyright (c) 2020, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals - import json import frappe diff --git a/frappe/desk/doctype/system_console/test_system_console.py b/frappe/desk/doctype/system_console/test_system_console.py index 55ef199122..743c2d6dde 100644 --- a/frappe/desk/doctype/system_console/test_system_console.py +++ b/frappe/desk/doctype/system_console/test_system_console.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import frappe import unittest diff --git a/frappe/desk/doctype/tag/tag.py b/frappe/desk/doctype/tag/tag.py index 7e016ee91b..3c67bb4668 100644 --- a/frappe/desk/doctype/tag/tag.py +++ b/frappe/desk/doctype/tag/tag.py @@ -2,7 +2,6 @@ # Copyright (c) 2019, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document from frappe.utils import unique diff --git a/frappe/desk/doctype/tag/test_tag.py b/frappe/desk/doctype/tag/test_tag.py index 8efd692f43..442a891fd8 100644 --- a/frappe/desk/doctype/tag/test_tag.py +++ b/frappe/desk/doctype/tag/test_tag.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - # import frappe import unittest diff --git a/frappe/desk/doctype/tag_link/tag_link.py b/frappe/desk/doctype/tag_link/tag_link.py index 87c8af7212..4c5149f42c 100644 --- a/frappe/desk/doctype/tag_link/tag_link.py +++ b/frappe/desk/doctype/tag_link/tag_link.py @@ -2,7 +2,6 @@ # Copyright (c) 2019, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals # import frappe from frappe.model.document import Document diff --git a/frappe/desk/doctype/tag_link/test_tag_link.py b/frappe/desk/doctype/tag_link/test_tag_link.py index 1c22ac18bc..297ee3cc96 100644 --- a/frappe/desk/doctype/tag_link/test_tag_link.py +++ b/frappe/desk/doctype/tag_link/test_tag_link.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - # import frappe import unittest diff --git a/frappe/desk/doctype/todo/__init__.py b/frappe/desk/doctype/todo/__init__.py index 4dbcd0d163..0e57cb68c3 100644 --- a/frappe/desk/doctype/todo/__init__.py +++ b/frappe/desk/doctype/todo/__init__.py @@ -1,4 +1,3 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals diff --git a/frappe/desk/doctype/todo/test_todo.py b/frappe/desk/doctype/todo/test_todo.py index b767fd4aef..b38e4a059a 100644 --- a/frappe/desk/doctype/todo/test_todo.py +++ b/frappe/desk/doctype/todo/test_todo.py @@ -1,16 +1,13 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # See license.txt -from __future__ import unicode_literals - import frappe import unittest from frappe.model.db_query import DatabaseQuery from frappe.permissions import add_permission, reset_perms from frappe.core.doctype.doctype.doctype import clear_permissions_cache -# test_records = frappe.get_test_records('ToDo') -test_user_records = frappe.get_test_records('User') +test_dependencies = ['User'] class TestToDo(unittest.TestCase): def test_delete(self): @@ -77,7 +74,7 @@ class TestToDo(unittest.TestCase): frappe.set_user('test4@example.com') #owner and assigned_by is test4 todo3 = create_new_todo('Test3', 'test4@example.com') - + # user without any role to read or write todo document self.assertFalse(todo1.has_permission("read")) self.assertFalse(todo1.has_permission("write")) diff --git a/frappe/desk/doctype/todo/todo.py b/frappe/desk/doctype/todo/todo.py index a766375fde..4696563445 100644 --- a/frappe/desk/doctype/todo/todo.py +++ b/frappe/desk/doctype/todo/todo.py @@ -1,7 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import frappe import json @@ -93,7 +92,7 @@ def get_permission_query_conditions(user): if not user: user = frappe.session.user todo_roles = frappe.permissions.get_doctype_roles('ToDo') - if 'All' in todo_roles: + if 'All' in todo_roles: todo_roles.remove('All') if any(check in todo_roles for check in frappe.get_roles(user)): @@ -105,7 +104,7 @@ def get_permission_query_conditions(user): def has_permission(doc, ptype="read", user=None): user = user or frappe.session.user todo_roles = frappe.permissions.get_doctype_roles('ToDo', ptype) - if 'All' in todo_roles: + if 'All' in todo_roles: todo_roles.remove('All') if any(check in todo_roles for check in frappe.get_roles(user)): diff --git a/frappe/desk/doctype/workspace/test_workspace.py b/frappe/desk/doctype/workspace/test_workspace.py index 7a3f122ee2..619b3608eb 100644 --- a/frappe/desk/doctype/workspace/test_workspace.py +++ b/frappe/desk/doctype/workspace/test_workspace.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - # import frappe import unittest diff --git a/frappe/desk/doctype/workspace/workspace.py b/frappe/desk/doctype/workspace/workspace.py index 0934138821..0329e0f7d2 100644 --- a/frappe/desk/doctype/workspace/workspace.py +++ b/frappe/desk/doctype/workspace/workspace.py @@ -2,7 +2,6 @@ # Copyright (c) 2020, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe import _ from frappe.modules.export_file import export_to_files diff --git a/frappe/desk/doctype/workspace_chart/workspace_chart.py b/frappe/desk/doctype/workspace_chart/workspace_chart.py index 0bb6194d2e..6ec7abfd3c 100644 --- a/frappe/desk/doctype/workspace_chart/workspace_chart.py +++ b/frappe/desk/doctype/workspace_chart/workspace_chart.py @@ -2,7 +2,6 @@ # Copyright (c) 2021, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals # import frappe from frappe.model.document import Document diff --git a/frappe/desk/doctype/workspace_link/workspace_link.json b/frappe/desk/doctype/workspace_link/workspace_link.json index 010fb3f316..53dadad83d 100644 --- a/frappe/desk/doctype/workspace_link/workspace_link.json +++ b/frappe/desk/doctype/workspace_link/workspace_link.json @@ -8,13 +8,13 @@ "type", "label", "icon", + "only_for", "hidden", "link_details_section", "link_type", "link_to", "column_break_7", "dependencies", - "only_for", "onboard", "is_query_report" ], @@ -84,7 +84,7 @@ { "fieldname": "only_for", "fieldtype": "Link", - "label": "Only for ", + "label": "Only for", "options": "Country" }, { @@ -104,7 +104,7 @@ "index_web_pages_for_search": 1, "istable": 1, "links": [], - "modified": "2021-01-12 13:13:12.379443", + "modified": "2021-05-13 13:10:18.128512", "modified_by": "Administrator", "module": "Desk", "name": "Workspace Link", diff --git a/frappe/desk/doctype/workspace_link/workspace_link.py b/frappe/desk/doctype/workspace_link/workspace_link.py index 8a139077a6..d6ccc5306a 100644 --- a/frappe/desk/doctype/workspace_link/workspace_link.py +++ b/frappe/desk/doctype/workspace_link/workspace_link.py @@ -2,7 +2,6 @@ # Copyright (c) 2021, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals # import frappe from frappe.model.document import Document diff --git a/frappe/desk/doctype/workspace_shortcut/workspace_shortcut.py b/frappe/desk/doctype/workspace_shortcut/workspace_shortcut.py index d676f08b73..83b446e454 100644 --- a/frappe/desk/doctype/workspace_shortcut/workspace_shortcut.py +++ b/frappe/desk/doctype/workspace_shortcut/workspace_shortcut.py @@ -2,7 +2,6 @@ # Copyright (c) 2021, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals # import frappe from frappe.model.document import Document diff --git a/frappe/desk/form/__init__.py b/frappe/desk/form/__init__.py index 4dbcd0d163..0e57cb68c3 100644 --- a/frappe/desk/form/__init__.py +++ b/frappe/desk/form/__init__.py @@ -1,4 +1,3 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals diff --git a/frappe/desk/form/assign_to.py b/frappe/desk/form/assign_to.py index aee7a8e52a..3eda291d1e 100644 --- a/frappe/desk/form/assign_to.py +++ b/frappe/desk/form/assign_to.py @@ -1,7 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals """assign/unassign to ToDo""" import frappe diff --git a/frappe/desk/form/document_follow.py b/frappe/desk/form/document_follow.py index f5e5c0ca9b..7f65f76a58 100644 --- a/frappe/desk/form/document_follow.py +++ b/frappe/desk/form/document_follow.py @@ -1,7 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import frappe import frappe.utils from frappe.utils import get_url_to_form diff --git a/frappe/desk/form/linked_with.py b/frappe/desk/form/linked_with.py index a62e2837d5..ae48b7fc6b 100644 --- a/frappe/desk/form/linked_with.py +++ b/frappe/desk/form/linked_with.py @@ -1,9 +1,8 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import json from collections import defaultdict -from six import string_types + import frappe import frappe.desk.form.load import frappe.desk.form.meta @@ -11,6 +10,7 @@ from frappe import _ from frappe.model.meta import is_single from frappe.modules import load_doctype_module + @frappe.whitelist() def get_submitted_linked_docs(doctype, name, docs=None, visited=None): """ @@ -87,7 +87,7 @@ def cancel_all_linked_docs(docs, ignore_doctypes_on_cancel_all=[]): """ docs = json.loads(docs) - if isinstance(ignore_doctypes_on_cancel_all, string_types): + if isinstance(ignore_doctypes_on_cancel_all, str): ignore_doctypes_on_cancel_all = json.loads(ignore_doctypes_on_cancel_all) for i, doc in enumerate(docs, 1): if validate_linked_doc(doc, ignore_doctypes_on_cancel_all): @@ -139,7 +139,7 @@ def get_exempted_doctypes(): @frappe.whitelist() def get_linked_docs(doctype, name, linkinfo=None, for_doctype=None): - if isinstance(linkinfo, string_types): + if isinstance(linkinfo, str): # additional fields are added in linkinfo linkinfo = json.loads(linkinfo) @@ -202,7 +202,8 @@ def get_linked_docs(doctype, name, linkinfo=None, for_doctype=None): else: link_fieldnames = link.get("fieldname") if link_fieldnames: - if isinstance(link_fieldnames, string_types): link_fieldnames = [link_fieldnames] + if isinstance(link_fieldnames, str): + link_fieldnames = [link_fieldnames] or_filters = [[dt, fieldname, '=', name] for fieldname in link_fieldnames] # dynamic link if link.get("doctype_fieldname"): diff --git a/frappe/desk/form/load.py b/frappe/desk/form/load.py index d81bb8c26c..a62bfd01d0 100644 --- a/frappe/desk/form/load.py +++ b/frappe/desk/form/load.py @@ -1,7 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import frappe, json import frappe.utils import frappe.share @@ -11,7 +10,7 @@ from frappe.model.utils.user_settings import get_user_settings from frappe.permissions import get_doc_permissions from frappe.desk.form.document_follow import is_document_followed from frappe import _ -from six.moves.urllib.parse import quote +from urllib.parse import quote @frappe.whitelist(allow_guest=True) def getdoc(doctype, name, user=None): diff --git a/frappe/desk/form/meta.py b/frappe/desk/form/meta.py index 087cc54d9d..cf3606e785 100644 --- a/frappe/desk/form/meta.py +++ b/frappe/desk/form/meta.py @@ -1,20 +1,16 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt - -# metadata - -from __future__ import unicode_literals -import frappe, os -from frappe.model.meta import Meta -from frappe.modules import scrub, get_module_path, load_doctype_module -from frappe.utils import get_html_format -from frappe.translate import make_dict_from_messages, extract_messages_from_code -from frappe.model.utils import render_include -from frappe.build import scrub_html_template - import io +import os + +import frappe +from frappe.build import scrub_html_template +from frappe.model.meta import Meta +from frappe.model.utils import render_include +from frappe.modules import get_module_path, load_doctype_module, scrub +from frappe.translate import extract_messages_from_code, make_dict_from_messages +from frappe.utils import get_html_format -from six import iteritems def get_meta(doctype, cached=True): # don't cache for developer mode as js files, templates may be edited @@ -199,7 +195,7 @@ class FormMeta(Meta): app = module.__name__.split(".")[0] templates = {} if hasattr(module, "form_grid_templates"): - for key, path in iteritems(module.form_grid_templates): + for key, path in module.form_grid_templates.items(): templates[key] = get_html_format(frappe.get_app_path(app, path)) self.set("__form_grid_templates", templates) diff --git a/frappe/desk/form/save.py b/frappe/desk/form/save.py index da43b14fce..a7a4b829d8 100644 --- a/frappe/desk/form/save.py +++ b/frappe/desk/form/save.py @@ -1,7 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import frappe, json from frappe.desk.form.load import run_onload diff --git a/frappe/desk/form/test_form.py b/frappe/desk/form/test_form.py index ff0343b6e0..f3c4132777 100644 --- a/frappe/desk/form/test_form.py +++ b/frappe/desk/form/test_form.py @@ -1,7 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import frappe, unittest from frappe.desk.form.linked_with import get_linked_docs, get_linked_doctypes diff --git a/frappe/desk/form/utils.py b/frappe/desk/form/utils.py index 395d2b9571..bfceee6ea2 100644 --- a/frappe/desk/form/utils.py +++ b/frappe/desk/form/utils.py @@ -1,7 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import frappe, json import frappe.desk.form.meta import frappe.desk.form.load @@ -9,7 +8,6 @@ from frappe.desk.form.document_follow import follow_document from frappe.utils.file_manager import extract_images_from_html from frappe import _ -from six import string_types @frappe.whitelist() def remove_attach(): @@ -90,7 +88,7 @@ def get_next(doctype, value, prev, filters=None, sort_order='desc', sort_field=' prev = int(prev) if not filters: filters = [] - if isinstance(filters, string_types): + if isinstance(filters, str): filters = json.loads(filters) # # condition based on sort order diff --git a/frappe/desk/gantt.py b/frappe/desk/gantt.py index 521884beaa..7f0889c751 100644 --- a/frappe/desk/gantt.py +++ b/frappe/desk/gantt.py @@ -1,8 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals - import frappe, json @frappe.whitelist() diff --git a/frappe/desk/leaderboard.py b/frappe/desk/leaderboard.py index d651687256..a98ae1a1c6 100644 --- a/frappe/desk/leaderboard.py +++ b/frappe/desk/leaderboard.py @@ -1,5 +1,3 @@ - -from __future__ import unicode_literals, print_function import frappe from frappe.utils import get_fullname diff --git a/frappe/desk/like.py b/frappe/desk/like.py index 6d2e9704af..d44d58a761 100644 --- a/frappe/desk/like.py +++ b/frappe/desk/like.py @@ -1,8 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals - """Allow adding of likes to documents""" import frappe, json diff --git a/frappe/desk/listview.py b/frappe/desk/listview.py index 91dc0f3ba9..d2c84d36bf 100644 --- a/frappe/desk/listview.py +++ b/frappe/desk/listview.py @@ -1,7 +1,5 @@ # Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals - import frappe @frappe.whitelist(allow_guest=True) diff --git a/frappe/desk/moduleview.py b/frappe/desk/moduleview.py index df25b77e2d..021698ac92 100644 --- a/frappe/desk/moduleview.py +++ b/frappe/desk/moduleview.py @@ -1,7 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import frappe import json from frappe import _ diff --git a/frappe/desk/notifications.py b/frappe/desk/notifications.py index 4b584a2429..c84027928e 100644 --- a/frappe/desk/notifications.py +++ b/frappe/desk/notifications.py @@ -1,11 +1,8 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals - import frappe from frappe.desk.doctype.notification_settings.notification_settings import get_subscribed_documents -from six import string_types import json @frappe.whitelist() @@ -149,7 +146,7 @@ def clear_doctype_notifications(doc, method=None, *args, **kwargs): config = get_notification_config() if not config: return - if isinstance(doc, string_types): + if isinstance(doc, str): doctype = doc # assuming doctype name was passed directly else: doctype = doc.doctype @@ -213,7 +210,7 @@ def get_filters_for(doctype): '''get open filters for doctype''' config = get_notification_config() doctype_config = config.get("for_doctype").get(doctype, {}) - filters = doctype_config if not isinstance(doctype_config, string_types) else None + filters = doctype_config if not isinstance(doctype_config, str) else None return filters diff --git a/frappe/desk/page/activity/__init__.py b/frappe/desk/page/activity/__init__.py index baffc48825..8b13789179 100644 --- a/frappe/desk/page/activity/__init__.py +++ b/frappe/desk/page/activity/__init__.py @@ -1 +1 @@ -from __future__ import unicode_literals + diff --git a/frappe/desk/page/activity/activity.js b/frappe/desk/page/activity/activity.js index 39de414122..7b4e8ddc1a 100644 --- a/frappe/desk/page/activity/activity.js +++ b/frappe/desk/page/activity/activity.js @@ -67,8 +67,8 @@ frappe.pages['activity'].on_page_show = function () { } frappe.activity.last_feed_date = false; -frappe.activity.Feed = Class.extend({ - init: function (row, data) { +frappe.activity.Feed = class Feed { + constructor(row, data) { this.scrub_data(data); this.add_date_separator(row, data); if (!data.add_class) @@ -97,8 +97,9 @@ frappe.activity.Feed = Class.extend({ $(row) .append(frappe.render_template("activity_row", data)) .find("a").addClass("grey"); - }, - scrub_data: function (data) { + } + + scrub_data(data) { data.by = frappe.user.full_name(data.owner); data.avatar = frappe.avatar(data.owner); @@ -113,9 +114,9 @@ frappe.activity.Feed = Class.extend({ data.when = comment_when(data.creation); data.feed_type = data.comment_type || data.communication_medium; - }, + } - add_date_separator: function (row, data) { + add_date_separator(row, data) { var date = frappe.datetime.str_to_obj(data.creation); var last = frappe.activity.last_feed_date; @@ -137,7 +138,7 @@ frappe.activity.Feed = Class.extend({ } frappe.activity.last_feed_date = date; } -}); +}; frappe.activity.render_heatmap = function (page) { $('
\ diff --git a/frappe/desk/page/activity/activity.py b/frappe/desk/page/activity/activity.py index 7de294d2f0..3abc8e0ea5 100644 --- a/frappe/desk/page/activity/activity.py +++ b/frappe/desk/page/activity/activity.py @@ -1,7 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: See license.txt -from __future__ import unicode_literals import frappe from frappe.utils import cint from frappe.core.doctype.activity_log.feed import get_feed_match_conditions diff --git a/frappe/desk/page/backups/backups.css b/frappe/desk/page/backups/backups.css index 13f093e0b1..32ccb88c37 100644 --- a/frappe/desk/page/backups/backups.css +++ b/frappe/desk/page/backups/backups.css @@ -5,6 +5,7 @@ .download-backup-card { display: block; text-decoration: none; + margin-bottom: var(--margin-lg); } .download-backup-card:hover { diff --git a/frappe/desk/page/backups/backups.js b/frappe/desk/page/backups/backups.js index c82407c6bd..337ad33f43 100644 --- a/frappe/desk/page/backups/backups.js +++ b/frappe/desk/page/backups/backups.js @@ -1,7 +1,7 @@ frappe.pages['backups'].on_page_load = function(wrapper) { var page = frappe.ui.make_app_page({ parent: wrapper, - title: 'Download Backups', + title: __('Download Backups'), single_column: true }); diff --git a/frappe/desk/page/backups/backups.py b/frappe/desk/page/backups/backups.py index eaa0c65143..2229a6d89e 100644 --- a/frappe/desk/page/backups/backups.py +++ b/frappe/desk/page/backups/backups.py @@ -1,4 +1,4 @@ -from __future__ import unicode_literals + import os import frappe from frappe import _ diff --git a/frappe/desk/page/leaderboard/leaderboard.py b/frappe/desk/page/leaderboard/leaderboard.py index 819e7fe9d1..9469096f50 100644 --- a/frappe/desk/page/leaderboard/leaderboard.py +++ b/frappe/desk/page/leaderboard/leaderboard.py @@ -1,7 +1,5 @@ # Copyright (c) 2017, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt - -from __future__ import unicode_literals, print_function import frappe @frappe.whitelist() diff --git a/frappe/desk/page/setup_wizard/install_fixtures.py b/frappe/desk/page/setup_wizard/install_fixtures.py index 6d3aaee22b..06301cdeaf 100644 --- a/frappe/desk/page/setup_wizard/install_fixtures.py +++ b/frappe/desk/page/setup_wizard/install_fixtures.py @@ -1,8 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: GNU General Public License v3. See license.txt -from __future__ import unicode_literals - import frappe from frappe import _ from frappe.desk.doctype.global_search_settings.global_search_settings import update_global_search_doctypes diff --git a/frappe/desk/page/setup_wizard/setup_wizard.py b/frappe/desk/page/setup_wizard/setup_wizard.py index 1ac5279508..5edb44e182 100755 --- a/frappe/desk/page/setup_wizard/setup_wizard.py +++ b/frappe/desk/page/setup_wizard/setup_wizard.py @@ -1,8 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: See license.txt -from __future__ import unicode_literals - import frappe, json, os from frappe.utils import strip, cint from frappe.translate import (set_default_language, get_dict, send_translations) @@ -10,7 +8,6 @@ from frappe.geo.country_info import get_country_info from frappe.utils.password import update_password from werkzeug.useragents import UserAgent from . import install_fixtures -from six import string_types def get_setup_stages(args): @@ -208,14 +205,14 @@ def update_user_name(args): def parse_args(args): if not args: args = frappe.local.form_dict - if isinstance(args, string_types): + if isinstance(args, str): args = json.loads(args) args = frappe._dict(args) # strip the whitespace for key, value in args.items(): - if isinstance(value, string_types): + if isinstance(value, str): args[key] = strip(value) return args @@ -294,7 +291,7 @@ def reset_is_first_startup(): def prettify_args(args): # remove attachments for key, val in args.items(): - if isinstance(val, string_types) and "data:image" in val: + if isinstance(val, str) and "data:image" in val: filename = val.split("data:image", 1)[0].strip(", ") size = round((len(val) * 3 / 4) / 1048576.0, 2) args[key] = "Image Attached: '{0}' of size {1} MB".format(filename, size) diff --git a/frappe/desk/page/translation_tool/translation_tool.js b/frappe/desk/page/translation_tool/translation_tool.js index b3f0c032e3..13f68e647a 100644 --- a/frappe/desk/page/translation_tool/translation_tool.js +++ b/frappe/desk/page/translation_tool/translation_tool.js @@ -1,7 +1,7 @@ frappe.pages['translation-tool'].on_page_load = function(wrapper) { var page = frappe.ui.make_app_page({ parent: wrapper, - title: 'Translation Tool', + title: __('Translation Tool'), single_column: true, card_layout: true, }); diff --git a/frappe/desk/page/user_profile/user_profile.html b/frappe/desk/page/user_profile/user_profile.html index 911ccc702d..f134441b74 100644 --- a/frappe/desk/page/user_profile/user_profile.html +++ b/frappe/desk/page/user_profile/user_profile.html @@ -8,7 +8,7 @@
- No Data to Show + {%=__("No Data to Show") %}
@@ -19,7 +19,7 @@
- No Data to Show + {%=__("No Data to Show") %}
@@ -30,7 +30,7 @@
- No Data to Show + {%=__("No Data to Show") %}
@@ -41,4 +41,4 @@ - \ No newline at end of file + diff --git a/frappe/desk/page/user_profile/user_profile.js b/frappe/desk/page/user_profile/user_profile.js index 3443a33942..5890975e69 100644 --- a/frappe/desk/page/user_profile/user_profile.js +++ b/frappe/desk/page/user_profile/user_profile.js @@ -1,6 +1,6 @@ frappe.pages['user-profile'].on_page_load = function (wrapper) { - frappe.require('assets/js/user_profile_controller.min.js', () => { + frappe.require('user_profile_controller.bundle.js', () => { let user_profile = new frappe.ui.UserProfile(wrapper); user_profile.show(); }); -}; \ No newline at end of file +}; diff --git a/frappe/desk/query_report.py b/frappe/desk/query_report.py index 9589507ca6..3c0ebf11c1 100644 --- a/frappe/desk/query_report.py +++ b/frappe/desk/query_report.py @@ -1,8 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals - import frappe import os import json @@ -22,7 +20,6 @@ from frappe.model.utils import render_include from frappe.translate import send_translations import frappe.desk.reportview from frappe.permissions import get_role_permissions -from six import string_types, iteritems from datetime import timedelta from frappe.core.utils import ljust_list @@ -66,7 +63,7 @@ def generate_report_result(report, filters=None, user=None, custom_columns=None) user = user or frappe.session.user filters = filters or [] - if filters and isinstance(filters, string_types): + if filters and isinstance(filters, str): filters = json.loads(filters) res = [] @@ -222,7 +219,7 @@ def run(report_name, filters=None, user=None, ignore_prepared_report=False, cust and not custom_columns ): if filters: - if isinstance(filters, string_types): + if isinstance(filters, str): filters = json.loads(filters) dn = filters.get("prepared_report_name") @@ -317,7 +314,7 @@ def export_query(): data.pop("cmd", None) data.pop("csrf_token", None) - if isinstance(data.get("filters"), string_types): + if isinstance(data.get("filters"), str): filters = json.loads(data["filters"]) if data.get("report_name"): @@ -332,7 +329,7 @@ def export_query(): include_indentation = data.get("include_indentation") visible_idx = data.get("visible_idx") - if isinstance(visible_idx, string_types): + if isinstance(visible_idx, str): visible_idx = json.loads(visible_idx) if file_format_type == "Excel": @@ -363,7 +360,7 @@ def export_query(): def handle_duration_fieldtype_values(result, columns): for i, col in enumerate(columns): fieldtype = None - if isinstance(col, string_types): + if isinstance(col, str): col = col.split(":") if len(col) > 1: if col[1]: @@ -377,10 +374,17 @@ def handle_duration_fieldtype_values(result, columns): if fieldtype == "Duration": for entry in range(0, len(result)): - val_in_seconds = result[entry][i] - if val_in_seconds: - duration_val = format_duration(val_in_seconds) - result[entry][i] = duration_val + row = result[entry] + if isinstance(row, dict): + val_in_seconds = row[col.fieldname] + if val_in_seconds: + duration_val = format_duration(val_in_seconds) + row[col.fieldname] = duration_val + else: + val_in_seconds = row[i] + if val_in_seconds: + duration_val = format_duration(val_in_seconds) + row[i] = duration_val return result @@ -426,7 +430,7 @@ def add_total_row(result, columns, meta=None): has_percent = [] for i, col in enumerate(columns): fieldtype, options, fieldname = None, None, None - if isinstance(col, string_types): + if isinstance(col, str): if meta: # get fieldtype from the meta field = meta.get_field(col) @@ -476,7 +480,7 @@ def add_total_row(result, columns, meta=None): total_row[i] = flt(total_row[i]) / len(result) first_col_fieldtype = None - if isinstance(columns[0], string_types): + if isinstance(columns[0], str): first_col = columns[0].split(":") if len(first_col) > 1: first_col_fieldtype = first_col[1].split("/")[0] @@ -694,7 +698,7 @@ def get_linked_doctypes(columns, data): if val and col not in columns_with_value: columns_with_value.append(col) - items = list(iteritems(linked_doctypes)) + items = list(linked_doctypes.items()) for doctype, key in items: if key not in columns_with_value: @@ -721,7 +725,7 @@ def get_column_as_dict(col): col_dict = frappe._dict() # string - if isinstance(col, string_types): + if isinstance(col, str): col = col.split(":") if len(col) > 1: if "/" in col[1]: diff --git a/frappe/desk/report/todo/todo.py b/frappe/desk/report/todo/todo.py index f4fe2dc805..6bd22b843e 100644 --- a/frappe/desk/report/todo/todo.py +++ b/frappe/desk/report/todo/todo.py @@ -1,7 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import frappe from frappe import _ from frappe.utils import getdate diff --git a/frappe/desk/report_dump.py b/frappe/desk/report_dump.py index 86b1765814..b2d3ca3443 100644 --- a/frappe/desk/report_dump.py +++ b/frappe/desk/report_dump.py @@ -1,8 +1,7 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals -from six.moves import range + import frappe import json import copy diff --git a/frappe/desk/reportview.py b/frappe/desk/reportview.py index 86f8ec0aa7..55515856f1 100644 --- a/frappe/desk/reportview.py +++ b/frappe/desk/reportview.py @@ -1,16 +1,14 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals """build query for doclistview and return results""" import frappe, json -from six.moves import range import frappe.permissions from frappe.model.db_query import DatabaseQuery from frappe.model import default_fields, optional_fields from frappe import _ -from six import string_types, StringIO +from io import StringIO from frappe.core.doctype.access_log.access_log import make_access_log from frappe.utils import cstr, format_duration from frappe.model.base_document import get_controller @@ -171,7 +169,7 @@ def get_meta_and_docfield(fieldname, data): return meta, df def update_wildcard_field_param(data): - if ((isinstance(data.fields, string_types) and data.fields == "*") + if ((isinstance(data.fields, str) and data.fields == "*") or (isinstance(data.fields, (list, tuple)) and len(data.fields) == 1 and data.fields[0] == "*")): data.fields = frappe.db.get_table_columns(data.doctype) return True @@ -191,15 +189,15 @@ def clean_params(data): def parse_json(data): - if isinstance(data.get("filters"), string_types): + if isinstance(data.get("filters"), str): data["filters"] = json.loads(data["filters"]) - if isinstance(data.get("or_filters"), string_types): + if isinstance(data.get("or_filters"), str): data["or_filters"] = json.loads(data["or_filters"]) - if isinstance(data.get("fields"), string_types): + if isinstance(data.get("fields"), str): data["fields"] = json.loads(data["fields"]) - if isinstance(data.get("docstatus"), string_types): + if isinstance(data.get("docstatus"), str): data["docstatus"] = json.loads(data["docstatus"]) - if isinstance(data.get("save_user_settings"), string_types): + if isinstance(data.get("save_user_settings"), str): data["save_user_settings"] = json.loads(data["save_user_settings"]) else: data["save_user_settings"] = True @@ -311,7 +309,7 @@ def export_query(): for r in data: # encode only unicode type strings and not int, floats etc. writer.writerow([handle_html(frappe.as_unicode(v)) \ - if isinstance(v, string_types) else v for v in r]) + if isinstance(v, str) else v for v in r]) f.seek(0) frappe.response['result'] = cstr(f.read()) @@ -540,7 +538,7 @@ def build_match_conditions(doctype, user=None, as_condition=True): return match_conditions def get_filters_cond(doctype, filters, conditions, ignore_permissions=None, with_match_conditions=False): - if isinstance(filters, string_types): + if isinstance(filters, str): filters = json.loads(filters) if filters: @@ -549,7 +547,7 @@ def get_filters_cond(doctype, filters, conditions, ignore_permissions=None, with filters = filters.items() flt = [] for f in filters: - if isinstance(f[1], string_types) and f[1][0] == '!': + if isinstance(f[1], str) and f[1][0] == '!': flt.append([doctype, f[0], '!=', f[1][1:]]) elif isinstance(f[1], (list, tuple)) and \ f[1][0] in (">", "<", ">=", "<=", "!=", "like", "not like", "in", "not in", "between"): diff --git a/frappe/desk/search.py b/frappe/desk/search.py index 6181261fc2..040a8c2118 100644 --- a/frappe/desk/search.py +++ b/frappe/desk/search.py @@ -2,12 +2,10 @@ # MIT License. See license.txt # Search -from __future__ import unicode_literals import frappe, json from frappe.utils import cstr, unique, cint from frappe.permissions import has_permission from frappe import _, is_whitelisted -from six import string_types import re import wrapt @@ -62,7 +60,7 @@ def search_widget(doctype, txt, query=None, searchfield=None, start=0, start = cint(start) - if isinstance(filters, string_types): + if isinstance(filters, str): filters = json.loads(filters) if searchfield: @@ -221,3 +219,37 @@ def validate_and_sanitize_search_inputs(fn, instance, args, kwargs): return [] return fn(**kwargs) + + +@frappe.whitelist() +def get_names_for_mentions(search_term): + users_for_mentions = frappe.cache().get_value('users_for_mentions', get_users_for_mentions) + user_groups = frappe.cache().get_value('user_groups', get_user_groups) + + filtered_mentions = [] + for mention_data in users_for_mentions + user_groups: + if search_term.lower() not in mention_data.value.lower(): + continue + + mention_data['link'] = frappe.utils.get_url_to_form( + 'User Group' if mention_data.get('is_group') else 'User Profile', + mention_data['id'] + ) + + filtered_mentions.append(mention_data) + + return sorted(filtered_mentions, key=lambda d: d['value']) + +def get_users_for_mentions(): + return frappe.get_all('User', + fields=['name as id', 'full_name as value'], + filters={ + 'name': ['not in', ('Administrator', 'Guest')], + 'allowed_in_mentions': True, + 'user_type': 'System User', + }) + +def get_user_groups(): + return frappe.get_all('User Group', fields=['name as id', 'name as value'], update={ + 'is_group': True + }) diff --git a/frappe/desk/treeview.py b/frappe/desk/treeview.py index 6f0d7d3d5f..66acde4cb2 100644 --- a/frappe/desk/treeview.py +++ b/frappe/desk/treeview.py @@ -1,7 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import frappe from frappe import _ diff --git a/frappe/email/__init__.py b/frappe/email/__init__.py index b05aef7639..3fb539398a 100644 --- a/frappe/email/__init__.py +++ b/frappe/email/__init__.py @@ -1,7 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import frappe from frappe.desk.reportview import build_match_conditions diff --git a/frappe/email/doctype/auto_email_report/auto_email_report.py b/frappe/email/doctype/auto_email_report/auto_email_report.py index 6f1cd8eebd..f30279e308 100644 --- a/frappe/email/doctype/auto_email_report/auto_email_report.py +++ b/frappe/email/doctype/auto_email_report/auto_email_report.py @@ -2,8 +2,6 @@ # Copyright (c) 2015, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals - import calendar from datetime import timedelta @@ -245,6 +243,7 @@ def send_monthly(): def make_links(columns, data): for row in data: + doc_name = row.get('name') for col in columns: if col.fieldtype == "Link" and col.options != "Currency": if col.options and row.get(col.fieldname): @@ -253,8 +252,9 @@ def make_links(columns, data): if col.options and row.get(col.fieldname) and row.get(col.options): row[col.fieldname] = get_link_to_form(row[col.options], row[col.fieldname]) elif col.fieldtype == "Currency" and row.get(col.fieldname): - row[col.fieldname] = frappe.format_value(row[col.fieldname], col) - + doc = frappe.get_doc(col.parent, doc_name) if doc_name else None + # Pass the Document to get the currency based on docfield option + row[col.fieldname] = frappe.format_value(row[col.fieldname], col, doc=doc) return columns, data def update_field_types(columns): @@ -262,4 +262,4 @@ def update_field_types(columns): if col.fieldtype in ("Link", "Dynamic Link", "Currency") and col.options != "Currency": col.fieldtype = "Data" col.options = "" - return columns \ No newline at end of file + return columns diff --git a/frappe/email/doctype/auto_email_report/test_auto_email_report.py b/frappe/email/doctype/auto_email_report/test_auto_email_report.py index e656ff18f7..211a141ec0 100644 --- a/frappe/email/doctype/auto_email_report/test_auto_email_report.py +++ b/frappe/email/doctype/auto_email_report/test_auto_email_report.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import json import unittest diff --git a/frappe/email/doctype/document_follow/document_follow.py b/frappe/email/doctype/document_follow/document_follow.py index aaabffab6b..a04f8ef4c2 100644 --- a/frappe/email/doctype/document_follow/document_follow.py +++ b/frappe/email/doctype/document_follow/document_follow.py @@ -2,7 +2,6 @@ # Copyright (c) 2019, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals from frappe.model.document import Document class DocumentFollow(Document): diff --git a/frappe/email/doctype/document_follow/test_document_follow.py b/frappe/email/doctype/document_follow/test_document_follow.py index 1ac2d19305..456c0931f8 100644 --- a/frappe/email/doctype/document_follow/test_document_follow.py +++ b/frappe/email/doctype/document_follow/test_document_follow.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import frappe import unittest import frappe.desk.form.document_follow as document_follow @@ -17,14 +15,14 @@ class TestDocumentFollow(unittest.TestCase): document_follow.unfollow_document("Event", event_doc.name, user.name) doc = document_follow.follow_document("Event", event_doc.name, user.name) - self.assertEquals(doc.user, user.name) + self.assertEqual(doc.user, user.name) document_follow.send_hourly_updates() email_queue_entry_name = frappe.get_all("Email Queue", limit=1)[0].name email_queue_entry_doc = frappe.get_doc("Email Queue", email_queue_entry_name) - self.assertEquals((email_queue_entry_doc.recipients[0].recipient), user.name) + self.assertEqual((email_queue_entry_doc.recipients[0].recipient), user.name) self.assertIn(event_doc.doctype, email_queue_entry_doc.message) self.assertIn(event_doc.name, email_queue_entry_doc.message) diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index 4869c5a9bf..7ca4e6f99c 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -1,37 +1,60 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and contributors # For license information, please see license.txt - -from __future__ import unicode_literals, print_function import frappe import imaplib import re import json import socket import time -from frappe import _ +import functools + +import email.utils + +from frappe import _, are_emails_muted from frappe.model.document import Document -from frappe.utils import validate_email_address, cint, cstr, get_datetime, DATE_FORMAT, strip, comma_or, sanitize_html, add_days +from frappe.utils import (validate_email_address, cint, cstr, get_datetime, + DATE_FORMAT, strip, comma_or, sanitize_html, add_days, parse_addr) from frappe.utils.user import is_system_user from frappe.utils.jinja import render_template from frappe.email.smtp import SMTPServer -from frappe.email.receive import EmailServer, Email +from frappe.email.receive import EmailServer, InboundMail, SentEmailInInboxError from poplib import error_proto from dateutil.relativedelta import relativedelta from datetime import datetime, timedelta from frappe.desk.form import assign_to from frappe.utils.user import get_system_managers from frappe.utils.background_jobs import enqueue, get_jobs -from frappe.core.doctype.communication.email import set_incoming_outgoing_accounts from frappe.utils.html_utils import clean_email_html +from frappe.utils.error import raise_error_on_no_output from frappe.email.utils import get_port +OUTGOING_EMAIL_ACCOUNT_MISSING = _("Please setup default Email Account from Setup > Email > Email Account") + class SentEmailInInbox(Exception): pass -class InvalidEmailCredentials(frappe.ValidationError): - pass +def cache_email_account(cache_name): + def decorator_cache_email_account(func): + @functools.wraps(func) + def wrapper_cache_email_account(*args, **kwargs): + if not hasattr(frappe.local, cache_name): + setattr(frappe.local, cache_name, {}) + + cached_accounts = getattr(frappe.local, cache_name) + match_by = list(kwargs.values()) + ['default'] + matched_accounts = list(filter(None, [cached_accounts.get(key) for key in match_by])) + if matched_accounts: + return matched_accounts[0] + + matched_accounts = func(*args, **kwargs) + cached_accounts.update(matched_accounts or {}) + return matched_accounts and list(matched_accounts.values())[0] + return wrapper_cache_email_account + return decorator_cache_email_account class EmailAccount(Document): + DOCTYPE = 'Email Account' + def autoname(self): """Set name as `email_account_name` or make title from Email Address.""" if not self.email_account_name: @@ -72,9 +95,8 @@ class EmailAccount(Document): self.get_incoming_server() self.no_failed = 0 - if self.enable_outgoing: - self.check_smtp() + self.validate_smtp_conn() else: if self.enable_incoming or (self.enable_outgoing and not self.no_smtp_authentication): frappe.throw(_("Password is required or select Awaiting Password")) @@ -90,6 +112,13 @@ class EmailAccount(Document): if self.append_to not in valid_doctypes: frappe.throw(_("Append To can be one of {0}").format(comma_or(valid_doctypes))) + def validate_smtp_conn(self): + if not self.smtp_server: + frappe.throw(_("SMTP Server is required")) + + server = self.get_smtp_server() + return server.session + def before_save(self): messages = [] as_list = 1 @@ -151,24 +180,6 @@ class EmailAccount(Document): except Exception: pass - def check_smtp(self): - """Checks SMTP settings.""" - if self.enable_outgoing: - if not self.smtp_server: - frappe.throw(_("{0} is required").format("SMTP Server")) - - server = SMTPServer( - login = getattr(self, "login_id", None) or self.email_id, - server=self.smtp_server, - port=cint(self.smtp_port), - use_tls=cint(self.use_tls), - use_ssl=cint(self.use_ssl_for_outgoing) - ) - if self.password and not self.no_smtp_authentication: - server.password = self.get_password() - - server.sess - def get_incoming_server(self, in_receive=False, email_sync_rule="UNSEEN"): """Returns logged in POP3/IMAP connection object.""" if frappe.cache().get_value("workers:no-internet") == True: @@ -231,7 +242,7 @@ class EmailAccount(Document): return None elif not in_receive and any(map(lambda t: t in message, auth_error_codes)): - self.throw_invalid_credentials_exception() + SMTPServer.throw_invalid_credentials_exception() else: frappe.throw(cstr(e)) @@ -249,13 +260,142 @@ class EmailAccount(Document): else: raise + @property + def _password(self): + raise_exception = not (self.no_smtp_authentication or frappe.flags.in_test) + return self.get_password(raise_exception=raise_exception) + + @property + def default_sender(self): + return email.utils.formataddr((self.name, self.get("email_id"))) + + def is_exists_in_db(self): + """Some of the Email Accounts we create from configs and those doesn't exists in DB. + This is is to check the specific email account exists in DB or not. + """ + return self.find_one_by_filters(name=self.name) + @classmethod - def throw_invalid_credentials_exception(cls): - frappe.throw( - _("Incorrect email or password. Please check your login credentials."), - exc=InvalidEmailCredentials, - title=_("Invalid Credentials") - ) + def from_record(cls, record): + email_account = frappe.new_doc(cls.DOCTYPE) + email_account.update(record) + return email_account + + @classmethod + def find(cls, name): + return frappe.get_doc(cls.DOCTYPE, name) + + @classmethod + def find_one_by_filters(cls, **kwargs): + name = frappe.db.get_value(cls.DOCTYPE, kwargs) + return cls.find(name) if name else None + + @classmethod + def find_from_config(cls): + config = cls.get_account_details_from_site_config() + return cls.from_record(config) if config else None + + @classmethod + def create_dummy(cls): + return cls.from_record({"sender": "notifications@example.com"}) + + @classmethod + @raise_error_on_no_output( + keep_quiet = lambda: not cint(frappe.get_system_settings('setup_complete')), + error_message = OUTGOING_EMAIL_ACCOUNT_MISSING, error_type = frappe.OutgoingEmailError) # noqa + @cache_email_account('outgoing_email_account') + def find_outgoing(cls, match_by_email=None, match_by_doctype=None, _raise_error=False): + """Find the outgoing Email account to use. + + :param match_by_email: Find account using emailID + :param match_by_doctype: Find account by matching `Append To` doctype + :param _raise_error: This is used by raise_error_on_no_output decorator to raise error. + """ + if match_by_email: + match_by_email = parse_addr(match_by_email)[1] + doc = cls.find_one_by_filters(enable_outgoing=1, email_id=match_by_email) + if doc: + return {match_by_email: doc} + + if match_by_doctype: + doc = cls.find_one_by_filters(enable_outgoing=1, enable_incoming=1, append_to=match_by_doctype) + if doc: + return {match_by_doctype: doc} + + doc = cls.find_default_outgoing() + if doc: + return {'default': doc} + + @classmethod + def find_default_outgoing(cls): + """ Find default outgoing account. + """ + doc = cls.find_one_by_filters(enable_outgoing=1, default_outgoing=1) + doc = doc or cls.find_from_config() + return doc or (are_emails_muted() and cls.create_dummy()) + + @classmethod + def find_incoming(cls, match_by_email=None, match_by_doctype=None): + """Find the incoming Email account to use. + :param match_by_email: Find account using emailID + :param match_by_doctype: Find account by matching `Append To` doctype + """ + doc = cls.find_one_by_filters(enable_incoming=1, email_id=match_by_email) + if doc: + return doc + + doc = cls.find_one_by_filters(enable_incoming=1, append_to=match_by_doctype) + if doc: + return doc + + doc = cls.find_default_incoming() + return doc + + @classmethod + def find_default_incoming(cls): + doc = cls.find_one_by_filters(enable_incoming=1, default_incoming=1) + return doc + + @classmethod + def get_account_details_from_site_config(cls): + if not frappe.conf.get("mail_server"): + return {} + + field_to_conf_name_map = { + 'smtp_server': {'conf_names': ('mail_server',)}, + 'smtp_port': {'conf_names': ('mail_port',)}, + 'use_tls': {'conf_names': ('use_tls', 'mail_login')}, + 'login_id': {'conf_names': ('mail_login',)}, + 'email_id': {'conf_names': ('auto_email_id', 'mail_login'), 'default': 'notifications@example.com'}, + 'password': {'conf_names': ('mail_password',)}, + 'always_use_account_email_id_as_sender': + {'conf_names': ('always_use_account_email_id_as_sender',), 'default': 0}, + 'always_use_account_name_as_sender_name': + {'conf_names': ('always_use_account_name_as_sender_name',), 'default': 0}, + 'name': {'conf_names': ('email_sender_name',), 'default': 'Frappe'}, + 'from_site_config': {'default': True} + } + + account_details = {} + for doc_field_name, d in field_to_conf_name_map.items(): + conf_names, default = d.get('conf_names') or [], d.get('default') + value = [frappe.conf.get(k) for k in conf_names if frappe.conf.get(k)] + account_details[doc_field_name] = (value and value[0]) or default + return account_details + + def sendmail_config(self): + return { + 'server': self.smtp_server, + 'port': cint(self.smtp_port), + 'login': getattr(self, "login_id", None) or self.email_id, + 'password': self._password, + 'use_ssl': cint(self.use_ssl_for_outgoing), + 'use_tls': cint(self.use_tls) + } + + def get_smtp_server(self): + config = self.sendmail_config() + return SMTPServer(**config) def handle_incoming_connect_error(self, description): if test_internet(): @@ -288,89 +428,74 @@ class EmailAccount(Document): def receive(self, test_mails=None): """Called by scheduler to receive emails from this EMail account using POP3/IMAP.""" - def get_seen(status): - if not status: - return None - seen = 1 if status == "SEEN" else 0 - return seen + exceptions = [] + inbound_mails = self.get_inbound_mails(test_mails=test_mails) + for mail in inbound_mails: + try: + communication = mail.process() + frappe.db.commit() + # If email already exists in the system + # then do not send notifications for the same email. + if communication and mail.flags.is_new_communication: + # notify all participants of this thread + if self.enable_auto_reply: + self.send_auto_reply(communication, mail) - if self.enable_incoming: - uid_list = [] - exceptions = [] - seen_status = [] - uid_reindexed = False - email_server = None - - if frappe.local.flags.in_test: - incoming_mails = test_mails or [] + communication.send_email(is_inbound_mail_communcation=True) + except SentEmailInInboxError: + frappe.db.rollback() + except Exception: + frappe.db.rollback() + frappe.log_error('email_account.receive') + if self.use_imap: + self.handle_bad_emails(mail.uid, mail.raw_message, frappe.get_traceback()) + exceptions.append(frappe.get_traceback()) else: - email_sync_rule = self.build_email_sync_rule() + frappe.db.commit() - try: - email_server = self.get_incoming_server(in_receive=True, email_sync_rule=email_sync_rule) - except Exception: - frappe.log_error(title=_("Error while connecting to email account {0}").format(self.name)) + #notify if user is linked to account + if len(inbound_mails)>0 and not frappe.local.flags.in_test: + frappe.publish_realtime('new_email', + {"account":self.email_account_name, "number":len(inbound_mails)} + ) - if not email_server: - return + if exceptions: + raise Exception(frappe.as_json(exceptions)) - emails = email_server.get_messages() - if not emails: - return + def get_inbound_mails(self, test_mails=None): + """retrive and return inbound mails. - incoming_mails = emails.get("latest_messages", []) - uid_list = emails.get("uid_list", []) - seen_status = emails.get("seen_status", []) - uid_reindexed = emails.get("uid_reindexed", False) + """ + if frappe.local.flags.in_test: + return [InboundMail(msg, self) for msg in test_mails or []] - for idx, msg in enumerate(incoming_mails): - uid = None if not uid_list else uid_list[idx] - self.flags.notify = True + if not self.enable_incoming: + return [] - try: - args = { - "uid": uid, - "seen": None if not seen_status else get_seen(seen_status.get(uid, None)), - "uid_reindexed": uid_reindexed - } - communication = self.insert_communication(msg, args=args) + email_sync_rule = self.build_email_sync_rule() + try: + email_server = self.get_incoming_server(in_receive=True, email_sync_rule=email_sync_rule) + messages = email_server.get_messages() or {} + except Exception: + frappe.log_error(title=_("Error while connecting to email account {0}").format(self.name)) + return [] - except SentEmailInInbox: - frappe.db.rollback() + mails = [] + for index, message in enumerate(messages.get("latest_messages", [])): + uid = messages['uid_list'][index] if messages.get('uid_list') else None + seen_status = 1 if messages.get('seen_status', {}).get(uid)=='SEEN' else 0 + mails.append(InboundMail(message, self, uid, seen_status)) - except Exception: - frappe.db.rollback() - frappe.log_error('email_account.receive') - if self.use_imap: - self.handle_bad_emails(email_server, uid, msg, frappe.get_traceback()) - exceptions.append(frappe.get_traceback()) + return mails - else: - frappe.db.commit() - if communication and self.flags.notify: - - # If email already exists in the system - # then do not send notifications for the same email. - - attachments = [] - - if hasattr(communication, '_attachments'): - attachments = [d.file_name for d in communication._attachments] - - communication.notify(attachments=attachments, fetched_from_email_account=True) - - #notify if user is linked to account - if len(incoming_mails)>0 and not frappe.local.flags.in_test: - frappe.publish_realtime('new_email', {"account":self.email_account_name, "number":len(incoming_mails)}) - - if exceptions: - raise Exception(frappe.as_json(exceptions)) - - def handle_bad_emails(self, email_server, uid, raw, reason): - if email_server and cint(email_server.settings.use_imap): + def handle_bad_emails(self, uid, raw, reason): + if cint(self.use_imap): import email try: - mail = email.message_from_string(raw) + if isinstance(raw, bytes): + mail = email.message_from_bytes(raw) + else: + mail = email.message_from_string(raw) message_id = mail.get('Message-ID') except Exception: @@ -382,278 +507,23 @@ class EmailAccount(Document): "reason":reason, "message_id": message_id, "doctype": "Unhandled Email", - "email_account": email_server.settings.email_account + "email_account": self.name }) unhandled_email.insert(ignore_permissions=True) frappe.db.commit() - def insert_communication(self, msg, args=None): - if isinstance(msg, list): - raw, uid, seen = msg - else: - raw = msg - uid = -1 - seen = 0 - if isinstance(args, dict): - if args.get("uid", -1): uid = args.get("uid", -1) - if args.get("seen", 0): seen = args.get("seen", 0) - - email = Email(raw) - - if email.from_email == self.email_id and not email.mail.get("Reply-To"): - # gmail shows sent emails in inbox - # and we don't want emails sent by us to be pulled back into the system again - # dont count emails sent by the system get those - if frappe.flags.in_test: - print('WARN: Cannot pull email. Sender sames as recipient inbox') - raise SentEmailInInbox - - if email.message_id: - # https://stackoverflow.com/a/18367248 - names = frappe.db.sql("""SELECT DISTINCT `name`, `creation` FROM `tabCommunication` - WHERE `message_id`='{message_id}' - ORDER BY `creation` DESC LIMIT 1""".format( - message_id=email.message_id - ), as_dict=True) - - if names: - name = names[0].get("name") - # email is already available update communication uid instead - frappe.db.set_value("Communication", name, "uid", uid, update_modified=False) - - self.flags.notify = False - - return frappe.get_doc("Communication", name) - - if email.content_type == 'text/html': - email.content = clean_email_html(email.content) - - communication = frappe.get_doc({ - "doctype": "Communication", - "subject": email.subject, - "content": email.content, - 'text_content': email.text_content, - "sent_or_received": "Received", - "sender_full_name": email.from_real_name, - "sender": email.from_email, - "recipients": email.mail.get("To"), - "cc": email.mail.get("CC"), - "email_account": self.name, - "communication_medium": "Email", - "uid": int(uid or -1), - "message_id": email.message_id, - "communication_date": email.date, - "has_attachment": 1 if email.attachments else 0, - "seen": seen or 0 - }) - - self.set_thread(communication, email) - if communication.seen: - # get email account user and set communication as seen - users = frappe.get_all("User Email", filters={ "email_account": self.name }, - fields=["parent"]) - users = list(set([ user.get("parent") for user in users ])) - communication._seen = json.dumps(users) - - communication.flags.in_receive = True - communication.insert(ignore_permissions=True) - - # save attachments - communication._attachments = email.save_attachments_in_doc(communication) - - # replace inline images - dirty = False - for file in communication._attachments: - if file.name in email.cid_map and email.cid_map[file.name]: - dirty = True - - email.content = email.content.replace("cid:{0}".format(email.cid_map[file.name]), - file.file_url) - - if dirty: - # not sure if using save() will trigger anything - communication.db_set("content", sanitize_html(email.content)) - - # notify all participants of this thread - if self.enable_auto_reply and getattr(communication, "is_first", False): - self.send_auto_reply(communication, email) - - return communication - - def set_thread(self, communication, email): - """Appends communication to parent based on thread ID. Will extract - parent communication and will link the communication to the reference of that - communication. Also set the status of parent transaction to Open or Replied. - - If no thread id is found and `append_to` is set for the email account, - it will create a new parent transaction (e.g. Issue)""" - parent = None - - parent = self.find_parent_from_in_reply_to(communication, email) - - if not parent and self.append_to: - self.set_sender_field_and_subject_field() - - if not parent and self.append_to: - parent = self.find_parent_based_on_subject_and_sender(communication, email) - - if not parent and self.append_to and self.append_to!="Communication": - parent = self.create_new_parent(communication, email) - - if parent: - communication.reference_doctype = parent.doctype - communication.reference_name = parent.name - - # check if message is notification and disable notifications for this message - isnotification = email.mail.get("isnotification") - if isnotification: - if "notification" in isnotification: - communication.unread_notification_sent = 1 - - def set_sender_field_and_subject_field(self): - '''Identify the sender and subject fields from the `append_to` DocType''' - # set subject_field and sender_field - meta = frappe.get_meta(self.append_to) - self.subject_field = None - self.sender_field = None - - if hasattr(meta, "subject_field"): - self.subject_field = meta.subject_field - - if hasattr(meta, "sender_field"): - self.sender_field = meta.sender_field - - def find_parent_based_on_subject_and_sender(self, communication, email): - '''Find parent document based on subject and sender match''' - parent = None - - if self.append_to and self.sender_field: - if self.subject_field: - if '#' in email.subject: - # try and match if ID is found - # document ID is appended to subject - # example "Re: Your email (#OPP-2020-2334343)" - parent_id = email.subject.rsplit('#', 1)[-1].strip(' ()') - if parent_id: - parent = frappe.db.get_all(self.append_to, filters = dict(name = parent_id), - fields = 'name') - - if not parent: - # try and match by subject and sender - # if sent by same sender with same subject, - # append it to old coversation - subject = frappe.as_unicode(strip(re.sub(r"(^\s*(fw|fwd|wg)[^:]*:|\s*(re|aw)[^:]*:\s*)*", - "", email.subject, 0, flags=re.IGNORECASE))) - - parent = frappe.db.get_all(self.append_to, filters={ - self.sender_field: email.from_email, - self.subject_field: ("like", "%{0}%".format(subject)), - "creation": (">", (get_datetime() - relativedelta(days=60)).strftime(DATE_FORMAT)) - }, fields = "name", limit = 1) - - if not parent and len(subject) > 10 and is_system_user(email.from_email): - # match only subject field - # when the from_email is of a user in the system - # and subject is atleast 10 chars long - parent = frappe.db.get_all(self.append_to, filters={ - self.subject_field: ("like", "%{0}%".format(subject)), - "creation": (">", (get_datetime() - relativedelta(days=60)).strftime(DATE_FORMAT)) - }, fields = "name", limit = 1) - - - - if parent: - parent = frappe._dict(doctype=self.append_to, name=parent[0].name) - return parent - - def create_new_parent(self, communication, email): - '''If no parent found, create a new reference document''' - - # no parent found, but must be tagged - # insert parent type doc - parent = frappe.new_doc(self.append_to) - - if self.subject_field: - parent.set(self.subject_field, frappe.as_unicode(email.subject)[:140]) - - if self.sender_field: - parent.set(self.sender_field, frappe.as_unicode(email.from_email)) - - if parent.meta.has_field("email_account"): - parent.email_account = self.name - - parent.flags.ignore_mandatory = True - - try: - parent.insert(ignore_permissions=True) - except frappe.DuplicateEntryError: - # try and find matching parent - parent_name = frappe.db.get_value(self.append_to, {self.sender_field: email.from_email}) - if parent_name: - parent.name = parent_name - else: - parent = None - - # NOTE if parent isn't found and there's no subject match, it is likely that it is a new conversation thread and hence is_first = True - communication.is_first = True - - return parent - - def find_parent_from_in_reply_to(self, communication, email): - '''Returns parent reference if embedded in In-Reply-To header - - Message-ID is formatted as `{message_id}@{site}`''' - parent = None - in_reply_to = (email.mail.get("In-Reply-To") or "").strip(" <>") - - if in_reply_to: - if "@{0}".format(frappe.local.site) in in_reply_to: - # reply to a communication sent from the system - email_queue = frappe.db.get_value('Email Queue', dict(message_id=in_reply_to), ['communication','reference_doctype', 'reference_name']) - if email_queue: - parent_communication, parent_doctype, parent_name = email_queue - if parent_communication: - communication.in_reply_to = parent_communication - else: - reference, domain = in_reply_to.split("@", 1) - parent_doctype, parent_name = 'Communication', reference - - if frappe.db.exists(parent_doctype, parent_name): - parent = frappe._dict(doctype=parent_doctype, name=parent_name) - - # set in_reply_to of current communication - if parent_doctype=='Communication': - # communication.in_reply_to = email_queue.communication - - if parent.reference_name: - # the true parent is the communication parent - parent = frappe.get_doc(parent.reference_doctype, - parent.reference_name) - else: - comm = frappe.db.get_value('Communication', - dict( - message_id=in_reply_to, - creation=['>=', add_days(get_datetime(), -30)]), - ['reference_doctype', 'reference_name'], as_dict=1) - if comm: - parent = frappe._dict(doctype=comm.reference_doctype, name=comm.reference_name) - - return parent - def send_auto_reply(self, communication, email): """Send auto reply if set.""" + from frappe.core.doctype.communication.email import set_incoming_outgoing_accounts if self.enable_auto_reply: set_incoming_outgoing_accounts(communication) - if self.send_unsubscribe_message: - unsubscribe_message = _("Leave this conversation") - else: - unsubscribe_message = "" + unsubscribe_message = (self.send_unsubscribe_message and _("Leave this conversation")) or "" frappe.sendmail(recipients = [email.from_email], sender = self.email_id, reply_to = communication.incoming_email_account, - subject = _("Re: ") + communication.subject, + subject = " ".join([_("Re:"), communication.subject]), content = render_template(self.auto_reply_message or "", communication.as_dict()) or \ frappe.get_template("templates/emails/auto_reply.html").render(communication.as_dict()), reference_doctype = communication.reference_doctype, diff --git a/frappe/email/doctype/email_account/test_email_account.py b/frappe/email/doctype/email_account/test_email_account.py index f87ee32bb1..35cacac45a 100644 --- a/frappe/email/doctype/email_account/test_email_account.py +++ b/frappe/email/doctype/email_account/test_email_account.py @@ -1,45 +1,56 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # See license.txt -from __future__ import unicode_literals -import frappe, os -import unittest, email +import os +import email +import unittest +from datetime import datetime, timedelta +from frappe.email.receive import InboundMail, SentEmailInInboxError, Email +from frappe.email.email_body import get_message_id +import frappe from frappe.test_runner import make_test_records +from frappe.core.doctype.communication.email import make +from frappe.desk.form.load import get_attachments +from frappe.email.doctype.email_account.email_account import notify_unreplied make_test_records("User") make_test_records("Email Account") -from frappe.core.doctype.communication.email import make -from frappe.desk.form.load import get_attachments -from frappe.email.doctype.email_account.email_account import notify_unreplied -from datetime import datetime, timedelta + class TestEmailAccount(unittest.TestCase): + @classmethod + def setUpClass(cls): + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + email_account.db_set("enable_incoming", 1) + email_account.db_set("enable_auto_reply", 1) + + @classmethod + def tearDownClass(cls): + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + email_account.db_set("enable_incoming", 0) + def setUp(self): frappe.flags.mute_emails = False frappe.flags.sent_mail = None - - email_account = frappe.get_doc("Email Account", "_Test Email Account 1") - email_account.db_set("enable_incoming", 1) frappe.db.sql('delete from `tabEmail Queue`') + frappe.db.sql('delete from `tabUnhandled Email`') - def tearDown(self): - email_account = frappe.get_doc("Email Account", "_Test Email Account 1") - email_account.db_set("enable_incoming", 0) + def get_test_mail(self, fname): + with open(os.path.join(os.path.dirname(__file__), "test_mails", fname), "r") as f: + return f.read() def test_incoming(self): cleanup("test_sender@example.com") - with open(os.path.join(os.path.dirname(__file__), "test_mails", "incoming-1.raw"), "r") as f: - test_mails = [f.read()] + test_mails = [self.get_test_mail('incoming-1.raw')] email_account = frappe.get_doc("Email Account", "_Test Email Account 1") email_account.receive(test_mails=test_mails) comm = frappe.get_doc("Communication", {"sender": "test_sender@example.com"}) self.assertTrue("test_receiver@example.com" in comm.recipients) - # check if todo is created self.assertTrue(frappe.db.get_value(comm.reference_doctype, comm.reference_name, "name")) @@ -88,7 +99,7 @@ class TestEmailAccount(unittest.TestCase): email_account.receive(test_mails=test_mails) comm = frappe.get_doc("Communication", {"sender": "test_sender@example.com"}) - self.assertTrue("From: \"Microsoft Outlook\" <test_sender@example.com>" in comm.content) + self.assertTrue("From: "Microsoft Outlook" <test_sender@example.com>" in comm.content) self.assertTrue("This is an e-mail message sent automatically by Microsoft Outlook while" in comm.content) def test_incoming_attached_email_from_outlook_layers(self): @@ -101,7 +112,7 @@ class TestEmailAccount(unittest.TestCase): email_account.receive(test_mails=test_mails) comm = frappe.get_doc("Communication", {"sender": "test_sender@example.com"}) - self.assertTrue("From: \"Microsoft Outlook\" <test_sender@example.com>" in comm.content) + self.assertTrue("From: "Microsoft Outlook" <test_sender@example.com>" in comm.content) self.assertTrue("This is an e-mail message sent automatically by Microsoft Outlook while" in comm.content) def test_outgoing(self): @@ -166,7 +177,6 @@ class TestEmailAccount(unittest.TestCase): comm_list = frappe.get_all("Communication", filters={"sender":"test_sender@example.com"}, fields=["name", "reference_doctype", "reference_name"]) - # both communications attached to the same reference self.assertEqual(comm_list[0].reference_doctype, comm_list[1].reference_doctype) self.assertEqual(comm_list[0].reference_name, comm_list[1].reference_name) @@ -199,6 +209,215 @@ class TestEmailAccount(unittest.TestCase): self.assertEqual(comm_list[0].reference_doctype, event.doctype) self.assertEqual(comm_list[0].reference_name, event.name) + def test_auto_reply(self): + cleanup("test_sender@example.com") + + test_mails = [self.get_test_mail('incoming-1.raw')] + + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + email_account.receive(test_mails=test_mails) + + comm = frappe.get_doc("Communication", {"sender": "test_sender@example.com"}) + self.assertTrue(frappe.db.get_value("Email Queue", {"reference_doctype": comm.reference_doctype, + "reference_name": comm.reference_name})) + + def test_handle_bad_emails(self): + mail_content = self.get_test_mail(fname="incoming-1.raw") + message_id = Email(mail_content).mail.get('Message-ID') + + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + email_account.handle_bad_emails(uid=-1, raw=mail_content, reason="Testing") + self.assertTrue(frappe.db.get_value("Unhandled Email", {'message_id': message_id})) + +class TestInboundMail(unittest.TestCase): + @classmethod + def setUpClass(cls): + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + email_account.db_set("enable_incoming", 1) + + @classmethod + def tearDownClass(cls): + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + email_account.db_set("enable_incoming", 0) + + def setUp(self): + cleanup() + frappe.db.sql('delete from `tabEmail Queue`') + frappe.db.sql('delete from `tabToDo`') + + def get_test_mail(self, fname): + with open(os.path.join(os.path.dirname(__file__), "test_mails", fname), "r") as f: + return f.read() + + def new_doc(self, doctype, **data): + doc = frappe.new_doc(doctype) + for field, value in data.items(): + setattr(doc, field, value) + doc.insert() + return doc + + def new_communication(self, **kwargs): + defaults = { + 'subject': "Test Subject" + } + d = {**defaults, **kwargs} + return self.new_doc('Communication', **d) + + def new_email_queue(self, **kwargs): + defaults = { + 'message_id': get_message_id().strip(" <>") + } + d = {**defaults, **kwargs} + return self.new_doc('Email Queue', **d) + + def new_todo(self, **kwargs): + defaults = { + 'description': "Description" + } + d = {**defaults, **kwargs} + return self.new_doc('ToDo', **d) + + def test_self_sent_mail(self): + """Check that we raise SentEmailInInboxError if the inbound mail is self sent mail. + """ + mail_content = self.get_test_mail(fname="incoming-self-sent.raw") + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + inbound_mail = InboundMail(mail_content, email_account, 1, 1) + with self.assertRaises(SentEmailInInboxError): + inbound_mail.process() + + def test_mail_exist_validation(self): + """Do not create communication record if the mail is already downloaded into the system. + """ + mail_content = self.get_test_mail(fname="incoming-1.raw") + message_id = Email(mail_content).message_id + # Create new communication record in DB + communication = self.new_communication(message_id=message_id) + + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + inbound_mail = InboundMail(mail_content, email_account, 12345, 1) + new_communiction = inbound_mail.process() + + # Make sure that uid is changed to new uid + self.assertEqual(new_communiction.uid, 12345) + self.assertEqual(communication.name, new_communiction.name) + + def test_find_parent_email_queue(self): + """If the mail is reply to the already sent mail, there will be a email queue record. + """ + # Create email queue record + queue_record = self.new_email_queue() + + mail_content = self.get_test_mail(fname="reply-4.raw").replace( + "{{ message_id }}", queue_record.message_id + ) + + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + inbound_mail = InboundMail(mail_content, email_account, 12345, 1) + parent_queue = inbound_mail.parent_email_queue() + self.assertEqual(queue_record.name, parent_queue.name) + + def test_find_parent_communication_through_queue(self): + """Find parent communication of an inbound mail. + Cases where parent communication does exist: + 1. No parent communication is the mail is not a reply. + + Cases where parent communication does not exist: + 2. If mail is not a reply to system sent mail, then there can exist co + """ + # Create email queue record + communication = self.new_communication() + queue_record = self.new_email_queue(communication=communication.name) + mail_content = self.get_test_mail(fname="reply-4.raw").replace( + "{{ message_id }}", queue_record.message_id + ) + + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + inbound_mail = InboundMail(mail_content, email_account, 12345, 1) + parent_communication = inbound_mail.parent_communication() + self.assertEqual(parent_communication.name, communication.name) + + def test_find_parent_communication_for_self_reply(self): + """If the inbound email is a reply but not reply to system sent mail. + + Ex: User replied to his/her mail. + """ + message_id = "new-message-id" + mail_content = self.get_test_mail(fname="reply-4.raw").replace( + "{{ message_id }}", message_id + ) + + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + inbound_mail = InboundMail(mail_content, email_account, 12345, 1) + parent_communication = inbound_mail.parent_communication() + self.assertFalse(parent_communication) + + communication = self.new_communication(message_id=message_id) + inbound_mail = InboundMail(mail_content, email_account, 12345, 1) + parent_communication = inbound_mail.parent_communication() + self.assertEqual(parent_communication.name, communication.name) + + def test_find_parent_communication_from_header(self): + """Incase of header contains parent communication name + """ + communication = self.new_communication() + mail_content = self.get_test_mail(fname="reply-4.raw").replace( + "{{ message_id }}", f"<{communication.name}@{frappe.local.site}>" + ) + + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + inbound_mail = InboundMail(mail_content, email_account, 12345, 1) + parent_communication = inbound_mail.parent_communication() + self.assertEqual(parent_communication.name, communication.name) + + def test_reference_document(self): + # Create email queue record + todo = self.new_todo() + # communication = self.new_communication(reference_doctype='ToDo', reference_name=todo.name) + queue_record = self.new_email_queue(reference_doctype='ToDo', reference_name=todo.name) + mail_content = self.get_test_mail(fname="reply-4.raw").replace( + "{{ message_id }}", queue_record.message_id + ) + + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + inbound_mail = InboundMail(mail_content, email_account, 12345, 1) + reference_doc = inbound_mail.reference_document() + self.assertEqual(todo.name, reference_doc.name) + + def test_reference_document_by_record_name_in_subject(self): + # Create email queue record + todo = self.new_todo() + + mail_content = self.get_test_mail(fname="incoming-subject-placeholder.raw").replace( + "{{ subject }}", f"RE: (#{todo.name})" + ) + + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + inbound_mail = InboundMail(mail_content, email_account, 12345, 1) + reference_doc = inbound_mail.reference_document() + self.assertEqual(todo.name, reference_doc.name) + + def test_reference_document_by_subject_match(self): + subject = "New todo" + todo = self.new_todo(sender='test_sender@example.com', description=subject) + + mail_content = self.get_test_mail(fname="incoming-subject-placeholder.raw").replace( + "{{ subject }}", f"RE: {subject}" + ) + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + inbound_mail = InboundMail(mail_content, email_account, 12345, 1) + reference_doc = inbound_mail.reference_document() + self.assertEqual(todo.name, reference_doc.name) + + def test_create_communication_from_mail(self): + # Create email queue record + mail_content = self.get_test_mail(fname="incoming-2.raw") + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + inbound_mail = InboundMail(mail_content, email_account, 12345, 1) + communication = inbound_mail.process() + self.assertTrue(communication.is_first) + self.assertTrue(communication._attachments) + def cleanup(sender=None): filters = {} if sender: @@ -207,4 +426,4 @@ def cleanup(sender=None): names = frappe.get_list("Communication", filters=filters, fields=["name"]) for name in names: frappe.delete_doc_if_exists("Communication", name.name) - frappe.delete_doc_if_exists("Communication Link", {"parent": name.name}) \ No newline at end of file + frappe.delete_doc_if_exists("Communication Link", {"parent": name.name}) diff --git a/frappe/email/doctype/email_account/test_mails/incoming-self-sent.raw b/frappe/email/doctype/email_account/test_mails/incoming-self-sent.raw new file mode 100644 index 0000000000..a16eecccd5 --- /dev/null +++ b/frappe/email/doctype/email_account/test_mails/incoming-self-sent.raw @@ -0,0 +1,91 @@ +Delivered-To: test_receiver@example.com +Received: by 10.96.153.227 with SMTP id vj3csp416144qdb; + Mon, 15 Sep 2014 03:35:07 -0700 (PDT) +X-Received: by 10.66.119.103 with SMTP id kt7mr36981968pab.95.1410777306321; + Mon, 15 Sep 2014 03:35:06 -0700 (PDT) +Return-Path: +Received: from mail-pa0-x230.google.com (mail-pa0-x230.google.com [2607:f8b0:400e:c03::230]) + by mx.google.com with ESMTPS id dg10si22178346pdb.115.2014.09.15.03.35.06 + for + (version=TLSv1 cipher=ECDHE-RSA-RC4-SHA bits=128/128); + Mon, 15 Sep 2014 03:35:06 -0700 (PDT) +Received-SPF: pass (google.com: domain of test@example.com designates 2607:f8b0:400e:c03::230 as permitted sender) client-ip=2607:f8b0:400e:c03::230; +Authentication-Results: mx.google.com; + spf=pass (google.com: domain of test@example.com designates 2607:f8b0:400e:c03::230 as permitted sender) smtp.mail=test@example.com; + dkim=pass header.i=@gmail.com; + dmarc=pass (p=NONE dis=NONE) header.from=gmail.com +Received: by mail-pa0-f48.google.com with SMTP id hz1so6118714pad.21 + for ; Mon, 15 Sep 2014 03:35:06 -0700 (PDT) +DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; + d=gmail.com; s=20120113; + h=from:content-type:subject:message-id:date:to:mime-version; + bh=rwiLijtF3lfy9M6cP/7dv2Hm7NJuBwFZn1OFsN8Tlvs=; + b=x7U4Ny3Kz2ULRJ7a04NDBrBTVhP2ImIB9n3LVNGQDnDonPUM5Ro/wZcxPTVnBWZ2L1 + o1bGfP+lhBrvYUlHsd5r4FYC0Uvpad6hbzLr0DGUQgPTxW4cGKbtDEAq+BR2JWd9f803 + vdjSWdGk8w2dt2qbngTqIZkm5U2XWjICDOAYuPIseLUgCFwi9lLyOSARFB7mjAa2YL7Q + Nswk7mbWU1hbnHP6jaBb0m8QanTc7Up944HpNDRxIrB1ZHgKzYhXtx8nhnOx588ZGIAe + E6tyG8IwogR11vLkkrBhtMaOme9PohYx4F1CSTiwspmDCadEzJFGRe//lEXKmZHAYH6g + 90Zg== +X-Received: by 10.70.38.135 with SMTP id g7mr22078275pdk.100.1410777305744; + Mon, 15 Sep 2014 03:35:05 -0700 (PDT) +Return-Path: +Received: from [192.168.0.100] ([27.106.4.70]) + by mx.google.com with ESMTPSA id zr6sm11025126pbc.50.2014.09.15.03.35.02 + for + (version=TLSv1 cipher=ECDHE-RSA-RC4-SHA bits=128/128); + Mon, 15 Sep 2014 03:35:04 -0700 (PDT) +From: Rushabh Mehta +Content-Type: multipart/alternative; boundary="Apple-Mail=_57F71261-5C3A-43F6-918B-4438B96F61AA" +Subject: test mail 🦄🌈😎 +Message-Id: <9143999C-8456-4399-9CF1-4A2DA9DD7711@gmail.com> +Date: Mon, 15 Sep 2014 16:04:57 +0530 +To: Rushabh Mehta +Mime-Version: 1.0 (Mac OS X Mail 7.3 \(1878.6\)) +X-Mailer: Apple Mail (2.1878.6) + + +--Apple-Mail=_57F71261-5C3A-43F6-918B-4438B96F61AA +Content-Transfer-Encoding: 7bit +Content-Type: text/plain; + charset=us-ascii + +test mail + + + +@rushabh_mehta +https://erpnext.org + + +--Apple-Mail=_57F71261-5C3A-43F6-918B-4438B96F61AA +Content-Transfer-Encoding: quoted-printable +Content-Type: text/html; + charset=us-ascii + +test = +mail
+



@rushabh_mehta
+
+
= + +--Apple-Mail=_57F71261-5C3A-43F6-918B-4438B96F61AA-- diff --git a/frappe/email/doctype/email_account/test_mails/incoming-subject-placeholder.raw b/frappe/email/doctype/email_account/test_mails/incoming-subject-placeholder.raw new file mode 100644 index 0000000000..35ddf06b01 --- /dev/null +++ b/frappe/email/doctype/email_account/test_mails/incoming-subject-placeholder.raw @@ -0,0 +1,183 @@ +Return-path: +Envelope-to: test_receiver@example.com +Delivery-date: Wed, 27 Jan 2016 16:24:20 +0800 +Received: from 23-59-23-10.perm.iinet.net.au ([23.59.23.10]:62191 helo=DESKTOP7C66I2M) + by webcloud85.au.syrahost.com with esmtp (Exim 4.86) + (envelope-from ) + id 1aOLOj-002xFL-CP + for test_receiver@example.com; Wed, 27 Jan 2016 16:24:20 +0800 +From: +To: +References: +In-Reply-To: +Subject: RE: {{ subject }} +Date: Wed, 27 Jan 2016 16:24:09 +0800 +Message-ID: <000001d158dc$1b8363a0$528a2ae0$@example.com> +MIME-Version: 1.0 +Content-Type: multipart/mixed; + boundary="----=_NextPart_000_0001_01D1591F.29A7DC20" +X-Mailer: Microsoft Outlook 14.0 +Thread-Index: AQJZfZxrgcB9KnMqoZ+S4Qq9hcoSeZ3+vGiQ +Content-Language: en-au + +This is a multipart message in MIME format. + +------=_NextPart_000_0001_01D1591F.29A7DC20 +Content-Type: multipart/alternative; + boundary="----=_NextPart_001_0002_01D1591F.29A7DC20" + + +------=_NextPart_001_0002_01D1591F.29A7DC20 +Content-Type: text/plain; + charset="utf-8" +Content-Transfer-Encoding: quoted-printable + +Test purely for testing with the debugger has email attached + +=20 + +From: Notification [mailto:test_receiver@example.com]=20 +Sent: Wednesday, 27 January 2016 9:30 AM +To: test_receiver@example.com +Subject: Sales Invoice: SINV-12276 + +=20 + +test no 6 sent from bench to outlook to be replied to with messaging + + + + +------=_NextPart_001_0002_01D1591F.29A7DC20 +Content-Type: text/html; + charset="utf-8" +Content-Transfer-Encoding: quoted-printable + +hi there

Test purely for testing with the debugger has email = +attached

 

From:= + = +Notification [mailto:test_receiver@example.com]
Sent: Wednesday, 27 = +January 2016 9:30 AM
To: = +test_receiver@example.com
Subject: Sales Invoice: = +SINV-12276

 

test no 3 sent from bench to outlook to be replied to with = +messaging

fizz buzz

This email was sent to test_receiver@example.= +com and copied to SuperUser

Leave this conversation = +

hi

+------=_NextPart_001_0002_01D1591F.29A7DC20-- + +------=_NextPart_000_0001_01D1591F.29A7DC20 +Content-Type: message/rfc822 +Content-Transfer-Encoding: 7bit +Content-Disposition: attachment + +Received: from 203-59-223-10.perm.iinet.net.au ([23.59.23.10]:49772 helo=DESKTOP7C66I2M) + by webcloud85.au.syrahost.com with esmtpsa (TLSv1.2:DHE-RSA-AES256-GCM-SHA384:256) + (Exim 4.86) + (envelope-from ) + id 1aOEtO-003tI4-Kv + for test_receiver@example.com; Wed, 27 Jan 2016 09:27:30 +0800 +Return-Path: +From: "Microsoft Outlook" +To: +Subject: Microsoft Outlook Test Message +MIME-Version: 1.0 +Content-Type: text/plain; + charset="utf-8" +Content-Transfer-Encoding: quoted-printable +X-Mailer: Microsoft Outlook 14.0 +Thread-Index: AdFYoeN8x8wUI/+QSoCJkp33NKPVmw== + +This is an e-mail message sent automatically by Microsoft Outlook while = +testing the settings for your account. diff --git a/frappe/email/doctype/email_account/test_records.json b/frappe/email/doctype/email_account/test_records.json index fbe7d9c281..15ca2a886e 100644 --- a/frappe/email/doctype/email_account/test_records.json +++ b/frappe/email/doctype/email_account/test_records.json @@ -19,7 +19,8 @@ "unreplied_for_mins": 20, "send_notification_to": "test_unreplied@example.com", "pop3_server": "pop.test.example.com", - "no_remaining":"0" + "no_remaining":"0", + "track_email_status": 1 }, { "doctype": "ToDo", diff --git a/frappe/email/doctype/email_domain/email_domain.py b/frappe/email/doctype/email_domain/email_domain.py index ce39523564..0856549eb7 100644 --- a/frappe/email/doctype/email_domain/email_domain.py +++ b/frappe/email/doctype/email_domain/email_domain.py @@ -2,7 +2,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe import _ from frappe.model.document import Document diff --git a/frappe/email/doctype/email_domain/test_email_domain.py b/frappe/email/doctype/email_domain/test_email_domain.py index 1c5306e9c2..8607151ca8 100644 --- a/frappe/email/doctype/email_domain/test_email_domain.py +++ b/frappe/email/doctype/email_domain/test_email_domain.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # See license.txt -from __future__ import unicode_literals - import frappe import unittest from frappe.test_runner import make_test_objects diff --git a/frappe/email/doctype/email_domain/test_records.json b/frappe/email/doctype/email_domain/test_records.json index 32bc66e150..a6ccc99f06 100644 --- a/frappe/email/doctype/email_domain/test_records.json +++ b/frappe/email/doctype/email_domain/test_records.json @@ -10,7 +10,8 @@ "incoming_port": "993", "attachment_limit": "1", "smtp_server": "smtp.test.com", - "smtp_port": "587" + "smtp_port": "587", + "password": "password" }, { "doctype": "Email Account", @@ -25,6 +26,7 @@ "incoming_port": "143", "attachment_limit": "1", "smtp_server": "smtp.test.com", - "smtp_port": "587" + "smtp_port": "587", + "password": "password" } ] diff --git a/frappe/email/doctype/email_flag_queue/email_flag_queue.py b/frappe/email/doctype/email_flag_queue/email_flag_queue.py index 487ef7db50..9bb30f08b2 100644 --- a/frappe/email/doctype/email_flag_queue/email_flag_queue.py +++ b/frappe/email/doctype/email_flag_queue/email_flag_queue.py @@ -2,7 +2,6 @@ # Copyright (c) 2015, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document diff --git a/frappe/email/doctype/email_flag_queue/test_email_flag_queue.py b/frappe/email/doctype/email_flag_queue/test_email_flag_queue.py index 644a2a8ff7..d09b823ce6 100644 --- a/frappe/email/doctype/email_flag_queue/test_email_flag_queue.py +++ b/frappe/email/doctype/email_flag_queue/test_email_flag_queue.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import frappe import unittest diff --git a/frappe/email/doctype/email_group/email_group.py b/frappe/email/doctype/email_group/email_group.py index b19a134713..2679353edf 100755 --- a/frappe/email/doctype/email_group/email_group.py +++ b/frappe/email/doctype/email_group/email_group.py @@ -2,7 +2,6 @@ # Copyright (c) 2015, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe import _ from frappe.utils import validate_email_address @@ -105,6 +104,6 @@ def send_welcome_email(welcome_email, email, email_group): email=email, email_group=email_group ) - - message = frappe.render_template(welcome_email.response, args) + email_message = welcome_email.response or welcome_email.response_html + message = frappe.render_template(email_message, args) frappe.sendmail(email, subject=welcome_email.subject, message=message) diff --git a/frappe/email/doctype/email_group/test_email_group.py b/frappe/email/doctype/email_group/test_email_group.py index 09f4f4c32c..3e894118df 100644 --- a/frappe/email/doctype/email_group/test_email_group.py +++ b/frappe/email/doctype/email_group/test_email_group.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import frappe import unittest diff --git a/frappe/email/doctype/email_group_member/email_group_member.py b/frappe/email/doctype/email_group_member/email_group_member.py index 23b279e755..1f9303b83e 100644 --- a/frappe/email/doctype/email_group_member/email_group_member.py +++ b/frappe/email/doctype/email_group_member/email_group_member.py @@ -2,7 +2,6 @@ # Copyright (c) 2015, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document diff --git a/frappe/email/doctype/email_group_member/test_email_group_member.py b/frappe/email/doctype/email_group_member/test_email_group_member.py index 35259617c1..829d686400 100644 --- a/frappe/email/doctype/email_group_member/test_email_group_member.py +++ b/frappe/email/doctype/email_group_member/test_email_group_member.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import frappe import unittest diff --git a/frappe/email/doctype/email_queue/email_queue.json b/frappe/email/doctype/email_queue/email_queue.json index 4529ea8211..f251786c90 100644 --- a/frappe/email/doctype/email_queue/email_queue.json +++ b/frappe/email/doctype/email_queue/email_queue.json @@ -24,7 +24,8 @@ "unsubscribe_method", "expose_recipients", "attachments", - "retry" + "retry", + "email_account" ], "fields": [ { @@ -139,13 +140,19 @@ "fieldtype": "Int", "label": "Retry", "read_only": 1 + }, + { + "fieldname": "email_account", + "fieldtype": "Link", + "label": "Email Account", + "options": "Email Account" } ], "icon": "fa fa-envelope", "idx": 1, "in_create": 1, "links": [], - "modified": "2020-07-17 15:58:15.369419", + "modified": "2021-04-29 06:33:25.191729", "modified_by": "Administrator", "module": "Email", "name": "Email Queue", diff --git a/frappe/email/doctype/email_queue/email_queue.py b/frappe/email/doctype/email_queue/email_queue.py index 267fbdfe9c..dad473b8aa 100644 --- a/frappe/email/doctype/email_queue/email_queue.py +++ b/frappe/email/doctype/email_queue/email_queue.py @@ -2,15 +2,30 @@ # Copyright (c) 2015, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals +import traceback +import json + +from rq.timeouts import JobTimeoutException +import smtplib +import quopri +from email.parser import Parser +from email.policy import SMTPUTF8 +from html2text import html2text +from six.moves import html_parser as HTMLParser + import frappe -from frappe import _ +from frappe import _, safe_encode, task from frappe.model.document import Document -from frappe.email.queue import send_one -from frappe.utils import now_datetime +from frappe.email.queue import get_unsubcribed_url, get_unsubscribe_message +from frappe.email.email_body import add_attachment, get_formatted_html, get_email +from frappe.utils import cint, split_emails, add_days, nowdate, cstr +from frappe.email.doctype.email_account.email_account import EmailAccount +MAX_RETRY_COUNT = 3 class EmailQueue(Document): + DOCTYPE = 'Email Queue' + def set_recipients(self, recipients): self.set("recipients", []) for r in recipients: @@ -30,6 +45,257 @@ class EmailQueue(Document): duplicate.set_recipients(recipients) return duplicate + @classmethod + def new(cls, doc_data, ignore_permissions=False): + data = doc_data.copy() + if not data.get('recipients'): + return + + recipients = data.pop('recipients') + doc = frappe.new_doc(cls.DOCTYPE) + doc.update(data) + doc.set_recipients(recipients) + doc.insert(ignore_permissions=ignore_permissions) + return doc + + @classmethod + def find(cls, name): + return frappe.get_doc(cls.DOCTYPE, name) + + @classmethod + def find_one_by_filters(cls, **kwargs): + name = frappe.db.get_value(cls.DOCTYPE, kwargs) + return cls.find(name) if name else None + + def update_db(self, commit=False, **kwargs): + frappe.db.set_value(self.DOCTYPE, self.name, kwargs) + if commit: + frappe.db.commit() + + def update_status(self, status, commit=False, **kwargs): + self.update_db(status = status, commit = commit, **kwargs) + if self.communication: + communication_doc = frappe.get_doc('Communication', self.communication) + communication_doc.set_delivery_status(commit=commit) + + @property + def cc(self): + return (self.show_as_cc and self.show_as_cc.split(",")) or [] + + @property + def to(self): + return [r.recipient for r in self.recipients if r.recipient not in self.cc] + + @property + def attachments_list(self): + return json.loads(self.attachments) if self.attachments else [] + + def get_email_account(self): + if self.email_account: + return frappe.get_doc('Email Account', self.email_account) + + return EmailAccount.find_outgoing( + match_by_email = self.sender, match_by_doctype = self.reference_doctype) + + def is_to_be_sent(self): + return self.status in ['Not Sent','Partially Sent'] + + def can_send_now(self): + hold_queue = (cint(frappe.defaults.get_defaults().get("hold_queue"))==1) + if frappe.are_emails_muted() or not self.is_to_be_sent() or hold_queue: + return False + + return True + + def send(self, is_background_task=False): + """ Send emails to recipients. + """ + if not self.can_send_now(): + frappe.db.rollback() + return + + with SendMailContext(self, is_background_task) as ctx: + message = None + for recipient in self.recipients: + if not recipient.is_mail_to_be_sent(): + continue + + message = ctx.build_message(recipient.recipient) + if not frappe.flags.in_test: + ctx.smtp_session.sendmail(from_addr=self.sender, to_addrs=recipient.recipient, msg=message) + ctx.add_to_sent_list(recipient) + + if frappe.flags.in_test: + frappe.flags.sent_mail = message + return + + if ctx.email_account_doc.append_emails_to_sent_folder and ctx.sent_to: + ctx.email_account_doc.append_email_to_sent_folder(message) + + +@task(queue = 'short') +def send_mail(email_queue_name, is_background_task=False): + """This is equalent to EmqilQueue.send. + + This provides a way to make sending mail as a background job. + """ + record = EmailQueue.find(email_queue_name) + record.send(is_background_task=is_background_task) + +class SendMailContext: + def __init__(self, queue_doc: Document, is_background_task: bool = False): + self.queue_doc = queue_doc + self.is_background_task = is_background_task + self.email_account_doc = queue_doc.get_email_account() + self.smtp_server = self.email_account_doc.get_smtp_server() + self.sent_to = [rec.recipient for rec in self.queue_doc.recipients if rec.is_main_sent()] + + def __enter__(self): + self.queue_doc.update_status(status='Sending', commit=True) + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + exceptions = [ + smtplib.SMTPServerDisconnected, + smtplib.SMTPAuthenticationError, + smtplib.SMTPRecipientsRefused, + smtplib.SMTPConnectError, + smtplib.SMTPHeloError, + JobTimeoutException + ] + + self.smtp_server.quit() + self.log_exception(exc_type, exc_val, exc_tb) + + if exc_type in exceptions: + email_status = (self.sent_to and 'Partially Sent') or 'Not Sent' + self.queue_doc.update_status(status = email_status, commit = True) + elif exc_type: + if self.queue_doc.retry < MAX_RETRY_COUNT: + update_fields = {'status': 'Not Sent', 'retry': self.queue_doc.retry + 1} + else: + update_fields = {'status': (self.sent_to and 'Partially Errored') or 'Error'} + self.queue_doc.update_status(**update_fields, commit = True) + else: + email_status = self.is_mail_sent_to_all() and 'Sent' + email_status = email_status or (self.sent_to and 'Partially Sent') or 'Not Sent' + self.queue_doc.update_status(status = email_status, commit = True) + + def log_exception(self, exc_type, exc_val, exc_tb): + if exc_type: + traceback_string = "".join(traceback.format_tb(exc_tb)) + traceback_string += f"\n Queue Name: {self.queue_doc.name}" + + if self.is_background_task: + frappe.log_error(title = 'frappe.email.queue.flush', message = traceback_string) + else: + frappe.log_error(message = traceback_string) + + @property + def smtp_session(self): + if frappe.flags.in_test: + return + return self.smtp_server.session + + def add_to_sent_list(self, recipient): + # Update recipient status + recipient.update_db(status='Sent', commit=True) + self.sent_to.append(recipient.recipient) + + def is_mail_sent_to_all(self): + return sorted(self.sent_to) == sorted([rec.recipient for rec in self.queue_doc.recipients]) + + def get_message_object(self, message): + return Parser(policy=SMTPUTF8).parsestr(message) + + def message_placeholder(self, placeholder_key): + map = { + 'tracker': '', + 'unsubscribe_url': '', + 'cc': '', + 'recipient': '', + } + return map.get(placeholder_key) + + def build_message(self, recipient_email): + """Build message specific to the recipient. + """ + message = self.queue_doc.message + if not message: + return "" + + message = message.replace(self.message_placeholder('tracker'), self.get_tracker_str()) + message = message.replace(self.message_placeholder('unsubscribe_url'), + self.get_unsubscribe_str(recipient_email)) + message = message.replace(self.message_placeholder('cc'), self.get_receivers_str()) + message = message.replace(self.message_placeholder('recipient'), + self.get_receipient_str(recipient_email)) + message = self.include_attachments(message) + return message + + def get_tracker_str(self): + tracker_url_html = \ + '' + + message = '' + if frappe.conf.use_ssl and self.email_account_doc.track_email_status: + message = quopri.encodestring( + tracker_url_html.format(frappe.local.site, self.queue_doc.communication).encode() + ).decode() + return message + + def get_unsubscribe_str(self, recipient_email): + unsubscribe_url = '' + if self.queue_doc.add_unsubscribe_link and self.queue_doc.reference_doctype: + doctype, doc_name = self.queue_doc.reference_doctype, self.queue_doc.reference_name + unsubscribe_url = get_unsubcribed_url(doctype, doc_name, recipient_email, + self.queue_doc.unsubscribe_method, self.queue_doc.unsubscribe_param) + + return quopri.encodestring(unsubscribe_url.encode()).decode() + + def get_receivers_str(self): + message = '' + if self.queue_doc.expose_recipients == "footer": + to_str = ', '.join(self.queue_doc.to) + cc_str = ', '.join(self.queue_doc.cc) + message = f"This email was sent to {to_str}" + message = message + f" and copied to {cc_str}" if cc_str else message + return message + + def get_receipient_str(self, recipient_email): + message = '' + if self.queue_doc.expose_recipients != "header": + message = recipient_email + return message + + def include_attachments(self, message): + message_obj = self.get_message_object(message) + attachments = self.queue_doc.attachments_list + + for attachment in attachments: + if attachment.get('fcontent'): + continue + + fid = attachment.get("fid") + if fid: + _file = frappe.get_doc("File", fid) + fcontent = _file.get_content() + attachment.update({ + 'fname': _file.file_name, + 'fcontent': fcontent, + 'parent': message_obj + }) + attachment.pop("fid", None) + add_attachment(**attachment) + + elif attachment.get("print_format_attachment") == 1: + attachment.pop("print_format_attachment", None) + print_format_file = frappe.attach_print(**attachment) + print_format_file.update({"parent": message_obj}) + add_attachment(**print_format_file) + + return safe_encode(message_obj.as_string()) + @frappe.whitelist() def retry_sending(name): doc = frappe.get_doc("Email Queue", name) @@ -42,8 +308,290 @@ def retry_sending(name): @frappe.whitelist() def send_now(name): - send_one(name, now=True) + record = EmailQueue.find(name) + if record: + record.send() def on_doctype_update(): """Add index in `tabCommunication` for `(reference_doctype, reference_name)`""" frappe.db.add_index('Email Queue', ('status', 'send_after', 'priority', 'creation'), 'index_bulk_flush') + +class QueueBuilder: + """Builds Email Queue from the given data + """ + def __init__(self, recipients=None, sender=None, subject=None, message=None, + text_content=None, reference_doctype=None, reference_name=None, + unsubscribe_method=None, unsubscribe_params=None, unsubscribe_message=None, + attachments=None, reply_to=None, cc=None, bcc=None, message_id=None, in_reply_to=None, + send_after=None, expose_recipients=None, send_priority=1, communication=None, + read_receipt=None, queue_separately=False, is_notification=False, + add_unsubscribe_link=1, inline_images=None, header=None, + print_letterhead=False, with_container=False): + """Add email to sending queue (Email Queue) + + :param recipients: List of recipients. + :param sender: Email sender. + :param subject: Email subject. + :param message: Email message. + :param text_content: Text version of email message. + :param reference_doctype: Reference DocType of caller document. + :param reference_name: Reference name of caller document. + :param send_priority: Priority for Email Queue, default 1. + :param unsubscribe_method: URL method for unsubscribe. Default is `/api/method/frappe.email.queue.unsubscribe`. + :param unsubscribe_params: additional params for unsubscribed links. default are name, doctype, email + :param attachments: Attachments to be sent. + :param reply_to: Reply to be captured here (default inbox) + :param in_reply_to: Used to send the Message-Id of a received email back as In-Reply-To. + :param send_after: Send this email after the given datetime. If value is in integer, then `send_after` will be the automatically set to no of days from current date. + :param communication: Communication link to be set in Email Queue record + :param queue_separately: Queue each email separately + :param is_notification: Marks email as notification so will not trigger notifications from system + :param add_unsubscribe_link: Send unsubscribe link in the footer of the Email, default 1. + :param inline_images: List of inline images as {"filename", "filecontent"}. All src properties will be replaced with random Content-Id + :param header: Append header in email (boolean) + :param with_container: Wraps email inside styled container + """ + + self._unsubscribe_method = unsubscribe_method + self._recipients = recipients + self._cc = cc + self._bcc = bcc + self._send_after = send_after + self._sender = sender + self._text_content = text_content + self._message = message + self._add_unsubscribe_link = add_unsubscribe_link + self._unsubscribe_message = unsubscribe_message + self._attachments = attachments + + self._unsubscribed_user_emails = None + self._email_account = None + + self.unsubscribe_params = unsubscribe_params + self.subject = subject + self.reference_doctype = reference_doctype + self.reference_name = reference_name + self.expose_recipients = expose_recipients + self.with_container = with_container + self.header = header + self.reply_to = reply_to + self.message_id = message_id + self.in_reply_to = in_reply_to + self.send_priority = send_priority + self.communication = communication + self.read_receipt = read_receipt + self.queue_separately = queue_separately + self.is_notification = is_notification + self.inline_images = inline_images + self.print_letterhead = print_letterhead + + @property + def unsubscribe_method(self): + return self._unsubscribe_method or '/api/method/frappe.email.queue.unsubscribe' + + def _get_emails_list(self, emails=None): + emails = split_emails(emails) if isinstance(emails, str) else (emails or []) + return [each for each in set(emails) if each] + + @property + def recipients(self): + return self._get_emails_list(self._recipients) + + @property + def cc(self): + return self._get_emails_list(self._cc) + + @property + def bcc(self): + return self._get_emails_list(self._bcc) + + @property + def send_after(self): + if isinstance(self._send_after, int): + return add_days(nowdate(), self._send_after) + return self._send_after + + @property + def sender(self): + if not self._sender or self._sender == "Administrator": + email_account = self.get_outgoing_email_account() + return email_account.default_sender + return self._sender + + def email_text_content(self): + unsubscribe_msg = self.unsubscribe_message() + unsubscribe_text_message = (unsubscribe_msg and unsubscribe_msg.text) or '' + + if self._text_content: + return self._text_content + unsubscribe_text_message + + try: + text_content = html2text(self._message) + except HTMLParser.HTMLParseError: + text_content = "See html attachment" + return text_content + unsubscribe_text_message + + def email_html_content(self): + email_account = self.get_outgoing_email_account() + return get_formatted_html(self.subject, self._message, header=self.header, + email_account=email_account, unsubscribe_link=self.unsubscribe_message(), + with_container=self.with_container) + + def should_include_unsubscribe_link(self): + return (self._add_unsubscribe_link == 1 + and self.reference_doctype + and (self._unsubscribe_message or self.reference_doctype=="Newsletter")) + + def unsubscribe_message(self): + if self.should_include_unsubscribe_link(): + return get_unsubscribe_message(self._unsubscribe_message, self.expose_recipients) + + def get_outgoing_email_account(self): + if self._email_account: + return self._email_account + + self._email_account = EmailAccount.find_outgoing( + match_by_doctype=self.reference_doctype, match_by_email=self._sender, _raise_error=True) + return self._email_account + + def get_unsubscribed_user_emails(self): + if self._unsubscribed_user_emails is not None: + return self._unsubscribed_user_emails + + all_ids = tuple(set(self.recipients + self.cc)) + + unsubscribed = frappe.db.sql_list(''' + SELECT + distinct email + from + `tabEmail Unsubscribe` + where + email in %(all_ids)s + and ( + ( + reference_doctype = %(reference_doctype)s + and reference_name = %(reference_name)s + ) + or global_unsubscribe = 1 + ) + ''', { + 'all_ids': all_ids, + 'reference_doctype': self.reference_doctype, + 'reference_name': self.reference_name, + }) + + self._unsubscribed_user_emails = unsubscribed or [] + return self._unsubscribed_user_emails + + def final_recipients(self): + unsubscribed_emails = self.get_unsubscribed_user_emails() + return [mail_id for mail_id in self.recipients if mail_id not in unsubscribed_emails] + + def final_cc(self): + unsubscribed_emails = self.get_unsubscribed_user_emails() + return [mail_id for mail_id in self.cc if mail_id not in unsubscribed_emails] + + def get_attachments(self): + attachments = [] + if self._attachments: + # store attachments with fid or print format details, to be attached on-demand later + for att in self._attachments: + if att.get('fid'): + attachments.append(att) + elif att.get("print_format_attachment") == 1: + if not att.get('lang', None): + att['lang'] = frappe.local.lang + att['print_letterhead'] = self.print_letterhead + attachments.append(att) + return attachments + + def prepare_email_content(self): + mail = get_email(recipients=self.final_recipients(), + sender=self.sender, + subject=self.subject, + formatted=self.email_html_content(), + text_content=self.email_text_content(), + attachments=self._attachments, + reply_to=self.reply_to, + cc=self.final_cc(), + bcc=self.bcc, + email_account=self.get_outgoing_email_account(), + expose_recipients=self.expose_recipients, + inline_images=self.inline_images, + header=self.header) + + mail.set_message_id(self.message_id, self.is_notification) + if self.read_receipt: + mail.msg_root["Disposition-Notification-To"] = self.sender + if self.in_reply_to: + mail.set_in_reply_to(self.in_reply_to) + return mail + + def process(self, send_now=False): + """Build and return the email queues those are created. + + Sends email incase if it is requested to send now. + """ + final_recipients = self.final_recipients() + queue_separately = (final_recipients and self.queue_separately) or len(final_recipients) > 20 + if not (final_recipients + self.final_cc()): + return [] + + email_queues = [] + queue_data = self.as_dict(include_recipients=False) + if not queue_data: + return [] + + if not queue_separately: + recipients = list(set(final_recipients + self.final_cc() + self.bcc)) + q = EmailQueue.new({**queue_data, **{'recipients': recipients}}, ignore_permissions=True) + email_queues.append(q) + else: + for r in final_recipients: + recipients = [r] if email_queues else list(set([r] + self.final_cc() + self.bcc)) + q = EmailQueue.new({**queue_data, **{'recipients': recipients}}, ignore_permissions=True) + email_queues.append(q) + + if send_now: + for doc in email_queues: + doc.send() + return email_queues + + def as_dict(self, include_recipients=True): + email_account = self.get_outgoing_email_account() + email_account_name = email_account and email_account.is_exists_in_db() and email_account.name + + mail = self.prepare_email_content() + try: + mail_to_string = cstr(mail.as_string()) + except frappe.InvalidEmailAddressError: + # bad Email Address - don't add to queue + frappe.log_error('Invalid Email ID Sender: {0}, Recipients: {1}, \nTraceback: {2} ' + .format(self.sender, ', '.join(self.final_recipients()), traceback.format_exc()), + 'Email Not Sent' + ) + return + + d = { + 'priority': self.send_priority, + 'attachments': json.dumps(self.get_attachments()), + 'message_id': mail.msg_root["Message-Id"].strip(" <>"), + 'message': mail_to_string, + 'sender': self.sender, + 'reference_doctype': self.reference_doctype, + 'reference_name': self.reference_name, + 'add_unsubscribe_link': self._add_unsubscribe_link, + 'unsubscribe_method': self.unsubscribe_method, + 'unsubscribe_params': self.unsubscribe_params, + 'expose_recipients': self.expose_recipients, + 'communication': self.communication, + 'send_after': self.send_after, + 'show_as_cc': ",".join(self.final_cc()), + 'show_as_bcc': ','.join(self.bcc), + 'email_account': email_account_name or None + } + + if include_recipients: + d['recipients'] = self.final_recipients() + + return d diff --git a/frappe/email/doctype/email_queue/test_email_queue.py b/frappe/email/doctype/email_queue/test_email_queue.py index 7cd79f9259..b76d6347b9 100644 --- a/frappe/email/doctype/email_queue/test_email_queue.py +++ b/frappe/email/doctype/email_queue/test_email_queue.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import frappe import unittest diff --git a/frappe/email/doctype/email_queue_recipient/email_queue_recipient.py b/frappe/email/doctype/email_queue_recipient/email_queue_recipient.py index 42956a1180..055bdb3fc1 100644 --- a/frappe/email/doctype/email_queue_recipient/email_queue_recipient.py +++ b/frappe/email/doctype/email_queue_recipient/email_queue_recipient.py @@ -2,9 +2,20 @@ # Copyright (c) 2015, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document class EmailQueueRecipient(Document): - pass + DOCTYPE = 'Email Queue Recipient' + + def is_mail_to_be_sent(self): + return self.status == 'Not Sent' + + def is_main_sent(self): + return self.status == 'Sent' + + def update_db(self, commit=False, **kwargs): + frappe.db.set_value(self.DOCTYPE, self.name, kwargs) + if commit: + frappe.db.commit() + diff --git a/frappe/email/doctype/email_rule/email_rule.py b/frappe/email/doctype/email_rule/email_rule.py index 220798bbdc..9807724ef1 100644 --- a/frappe/email/doctype/email_rule/email_rule.py +++ b/frappe/email/doctype/email_rule/email_rule.py @@ -2,7 +2,6 @@ # Copyright (c) 2017, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document diff --git a/frappe/email/doctype/email_rule/test_email_rule.py b/frappe/email/doctype/email_rule/test_email_rule.py index 3c7f9c83e6..b2213f7405 100644 --- a/frappe/email/doctype/email_rule/test_email_rule.py +++ b/frappe/email/doctype/email_rule/test_email_rule.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2017, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import frappe import unittest diff --git a/frappe/email/doctype/email_template/email_template.py b/frappe/email/doctype/email_template/email_template.py index 6708e9dd3f..4711451fd2 100644 --- a/frappe/email/doctype/email_template/email_template.py +++ b/frappe/email/doctype/email_template/email_template.py @@ -1,11 +1,9 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe, json from frappe.model.document import Document from frappe.utils.jinja import validate_template -from six import string_types class EmailTemplate(Document): def validate(self): @@ -24,7 +22,7 @@ class EmailTemplate(Document): return frappe.render_template(self.response, doc) def get_formatted_email(self, doc): - if isinstance(doc, string_types): + if isinstance(doc, str): doc = json.loads(doc) return { @@ -36,7 +34,7 @@ class EmailTemplate(Document): @frappe.whitelist() def get_email_template(template_name, doc): '''Returns the processed HTML of a email template with the given doc''' - if isinstance(doc, string_types): + if isinstance(doc, str): doc = json.loads(doc) email_template = frappe.get_doc("Email Template", template_name) diff --git a/frappe/email/doctype/email_template/test_email_template.py b/frappe/email/doctype/email_template/test_email_template.py index a48ce94ac5..5a9ee969c6 100644 --- a/frappe/email/doctype/email_template/test_email_template.py +++ b/frappe/email/doctype/email_template/test_email_template.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2018, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import unittest class TestEmailTemplate(unittest.TestCase): diff --git a/frappe/email/doctype/email_unsubscribe/email_unsubscribe.py b/frappe/email/doctype/email_unsubscribe/email_unsubscribe.py index e532e2b7eb..6c47d8c538 100644 --- a/frappe/email/doctype/email_unsubscribe/email_unsubscribe.py +++ b/frappe/email/doctype/email_unsubscribe/email_unsubscribe.py @@ -2,7 +2,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document from frappe import _ diff --git a/frappe/email/doctype/email_unsubscribe/test_email_unsubscribe.py b/frappe/email/doctype/email_unsubscribe/test_email_unsubscribe.py index ea84253ab6..602840fe3b 100644 --- a/frappe/email/doctype/email_unsubscribe/test_email_unsubscribe.py +++ b/frappe/email/doctype/email_unsubscribe/test_email_unsubscribe.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # See license.txt -from __future__ import unicode_literals - import frappe import unittest diff --git a/frappe/email/doctype/newsletter/newsletter..json b/frappe/email/doctype/newsletter/newsletter..json deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/email/doctype/newsletter/newsletter.py b/frappe/email/doctype/newsletter/newsletter.py index 6412338e96..97d77549b7 100755 --- a/frappe/email/doctype/newsletter/newsletter.py +++ b/frappe/email/doctype/newsletter/newsletter.py @@ -1,14 +1,11 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: GNU General Public License v3. See license.txt -from __future__ import unicode_literals - import frappe import frappe.utils from frappe import throw, _ from frappe.website.website_generator import WebsiteGenerator from frappe.utils.verified_command import get_signed_params, verify_request -from frappe.email.queue import send from frappe.email.doctype.email_group.email_group import add_subscribers from frappe.utils import parse_addr, now_datetime, markdown, validate_email_address diff --git a/frappe/email/doctype/newsletter/test_newsletter.py b/frappe/email/doctype/newsletter/test_newsletter.py index bd8fadc29c..cfd0df53a9 100644 --- a/frappe/email/doctype/newsletter/test_newsletter.py +++ b/frappe/email/doctype/newsletter/test_newsletter.py @@ -1,7 +1,5 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: GNU General Public License v3. See license.txt -from __future__ import unicode_literals - import unittest from random import choice diff --git a/frappe/email/doctype/newsletter_email_group/newsletter_email_group.py b/frappe/email/doctype/newsletter_email_group/newsletter_email_group.py index a59ac372fd..a453dda9e4 100644 --- a/frappe/email/doctype/newsletter_email_group/newsletter_email_group.py +++ b/frappe/email/doctype/newsletter_email_group/newsletter_email_group.py @@ -2,7 +2,6 @@ # Copyright (c) 2015, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document diff --git a/frappe/email/doctype/notification/notification.json b/frappe/email/doctype/notification/notification.json index c1c877efd4..8b6900a3c9 100644 --- a/frappe/email/doctype/notification/notification.json +++ b/frappe/email/doctype/notification/notification.json @@ -102,7 +102,8 @@ "default": "0", "fieldname": "is_standard", "fieldtype": "Check", - "label": "Is Standard" + "label": "Is Standard", + "no_copy": 1 }, { "depends_on": "is_standard", @@ -281,7 +282,7 @@ "icon": "fa fa-envelope", "index_web_pages_for_search": 1, "links": [], - "modified": "2020-11-24 14:25:43.245677", + "modified": "2021-05-04 11:17:11.882314", "modified_by": "Administrator", "module": "Email", "name": "Notification", diff --git a/frappe/email/doctype/notification/notification.py b/frappe/email/doctype/notification/notification.py index 2940a34f63..57418515f5 100644 --- a/frappe/email/doctype/notification/notification.py +++ b/frappe/email/doctype/notification/notification.py @@ -2,7 +2,6 @@ # Copyright (c) 2018, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe import json, os from frappe import _ @@ -12,7 +11,6 @@ from frappe.utils import validate_email_address, nowdate, parse_val, is_html, ad from frappe.utils.jinja import validate_template from frappe.utils.safe_exec import get_safe_globals from frappe.modules.utils import export_module_json, get_doc_module -from six import string_types from frappe.integrations.doctype.slack_webhook_url.slack_webhook_url import send_slack_message from frappe.core.doctype.sms_settings.sms_settings import send_sms from frappe.desk.doctype.notification_log.notification_log import enqueue_create_notification @@ -55,9 +53,7 @@ class Notification(Document): # py if not os.path.exists(path + '.py'): with open(path + '.py', 'w') as f: - f.write("""from __future__ import unicode_literals - -import frappe + f.write("""import frappe def get_context(context): # do your magic here @@ -397,7 +393,7 @@ def trigger_notifications(doc, method=None): def evaluate_alert(doc, alert, event): from jinja2 import TemplateError try: - if isinstance(alert, string_types): + if isinstance(alert, str): alert = frappe.get_doc("Notification", alert) context = get_context(doc) diff --git a/frappe/email/doctype/notification/test_notification.py b/frappe/email/doctype/notification/test_notification.py index 87c4b2527a..d6358ccbbe 100644 --- a/frappe/email/doctype/notification/test_notification.py +++ b/frappe/email/doctype/notification/test_notification.py @@ -1,15 +1,11 @@ # -*- coding: utf-8 -*- # Copyright (c) 2018, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import frappe, frappe.utils, frappe.utils.scheduler from frappe.desk.form import assign_to import unittest -test_records = frappe.get_test_records('Notification') - -test_dependencies = ["User"] +test_dependencies = ["User", "Notification"] class TestNotification(unittest.TestCase): def setUp(self): diff --git a/frappe/email/doctype/notification_recipient/notification_recipient.py b/frappe/email/doctype/notification_recipient/notification_recipient.py index a85ed62c04..d8480c5455 100644 --- a/frappe/email/doctype/notification_recipient/notification_recipient.py +++ b/frappe/email/doctype/notification_recipient/notification_recipient.py @@ -2,7 +2,6 @@ # Copyright (c) 2018, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document diff --git a/frappe/email/doctype/unhandled_email/test_unhandled_email.py b/frappe/email/doctype/unhandled_email/test_unhandled_email.py index 6cabcf6ec2..5606b8ff30 100644 --- a/frappe/email/doctype/unhandled_email/test_unhandled_email.py +++ b/frappe/email/doctype/unhandled_email/test_unhandled_email.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # See license.txt -from __future__ import unicode_literals - import frappe import unittest diff --git a/frappe/email/doctype/unhandled_email/unhandled_email.py b/frappe/email/doctype/unhandled_email/unhandled_email.py index 1276da71a1..6414dbece3 100644 --- a/frappe/email/doctype/unhandled_email/unhandled_email.py +++ b/frappe/email/doctype/unhandled_email/unhandled_email.py @@ -2,7 +2,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document diff --git a/frappe/email/email_body.py b/frappe/email/email_body.py index 3dcdf00a8e..ffb44d3412 100755 --- a/frappe/email/email_body.py +++ b/frappe/email/email_body.py @@ -1,14 +1,12 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import frappe, re, os from frappe.utils.pdf import get_pdf -from frappe.email.smtp import get_outgoing_email_account +from frappe.email.doctype.email_account.email_account import EmailAccount from frappe.utils import (get_url, scrub_urls, strip, expand_relative_urls, cint, split_emails, to_markdown, markdown, random_string, parse_addr) import email.utils -from six import iteritems, text_type, string_types from email.mime.multipart import MIMEMultipart from email.header import Header from email import policy @@ -55,7 +53,7 @@ class EMail: from email import charset as Charset Charset.add_charset('utf-8', Charset.QP, Charset.QP, 'utf-8') - if isinstance(recipients, string_types): + if isinstance(recipients, str): recipients = recipients.replace(';', ',').replace('\n', '') recipients = split_emails(recipients) @@ -75,7 +73,8 @@ class EMail: self.bcc = bcc or [] self.html_set = False - self.email_account = email_account or get_outgoing_email_account(sender=sender) + self.email_account = email_account or \ + EmailAccount.find_outgoing(match_by_email=sender, _raise_error=True) def set_html(self, message, text_content = None, footer=None, print_html=None, formatted=None, inline_images=None, header=None): @@ -224,7 +223,7 @@ class EMail: } # reset headers as values may be changed. - for key, val in iteritems(headers): + for key, val in headers.items(): if val: self.set_header(key, val) @@ -249,8 +248,8 @@ class EMail: def get_formatted_html(subject, message, footer=None, print_html=None, email_account=None, header=None, unsubscribe_link=None, sender=None, with_container=False): - if not email_account: - email_account = get_outgoing_email_account(False, sender=sender) + + email_account = email_account or EmailAccount.find_outgoing(match_by_email=sender) signature = None if "" not in message: @@ -291,18 +290,12 @@ def inline_style_in_html(html): ''' Convert email.css and html to inline-styled html ''' from premailer import Premailer + from frappe.utils.jinja_globals import bundled_asset - apps = frappe.get_installed_apps() - - # add frappe email css file - css_files = ['assets/css/email.css'] - if 'frappe' in apps: - apps.remove('frappe') - - for app in apps: - path = 'assets/{0}/css/email.css'.format(app) - css_files.append(path) - + # get email css files from hooks + css_files = frappe.get_hooks('email_css') + css_files = [bundled_asset(path) for path in css_files] + css_files = [path.lstrip('/') for path in css_files] css_files = [css_file for css_file in css_files if os.path.exists(os.path.abspath(css_file))] p = Premailer(html=html, external_styles=css_files, strip_important=False) @@ -333,7 +326,7 @@ def add_attachment(fname, fcontent, content_type=None, maintype, subtype = content_type.split('/', 1) if maintype == 'text': # Note: we should handle calculating the charset - if isinstance(fcontent, text_type): + if isinstance(fcontent, str): fcontent = fcontent.encode("utf-8") part = MIMEText(fcontent, _subtype=subtype, _charset="utf-8") elif maintype == 'image': @@ -350,7 +343,7 @@ def add_attachment(fname, fcontent, content_type=None, # Set the filename parameter if fname: attachment_type = 'inline' if inline else 'attachment' - part.add_header('Content-Disposition', attachment_type, filename=text_type(fname)) + part.add_header('Content-Disposition', attachment_type, filename=str(fname)) if content_id: part.add_header('Content-ID', '<{0}>'.format(content_id)) @@ -358,9 +351,7 @@ def add_attachment(fname, fcontent, content_type=None, def get_message_id(): '''Returns Message ID created from doctype and name''' - return "<{unique}@{site}>".format( - site=frappe.local.site, - unique=email.utils.make_msgid(random_string(10)).split('@')[0].split('<')[1]) + return email.utils.make_msgid(domain=frappe.local.site) def get_signature(email_account): if email_account and email_account.add_signature and email_account.signature: @@ -457,7 +448,7 @@ def get_header(header=None): if not header: return None - if isinstance(header, string_types): + if isinstance(header, str): # header = 'My Title' header = [header, None] if len(header) == 1: @@ -480,4 +471,4 @@ def sanitize_email_header(str): return str.replace('\r', '').replace('\n', '') def get_brand_logo(email_account): - return email_account.get('brand_logo') \ No newline at end of file + return email_account.get('brand_logo') diff --git a/frappe/email/inbox.py b/frappe/email/inbox.py index 395a2d3e2d..5f8f516772 100644 --- a/frappe/email/inbox.py +++ b/frappe/email/inbox.py @@ -1,4 +1,4 @@ -from __future__ import unicode_literals + import frappe import json diff --git a/frappe/email/queue.py b/frappe/email/queue.py index 2aff04edc9..ca96981aa8 100755 --- a/frappe/email/queue.py +++ b/frappe/email/queue.py @@ -1,253 +1,10 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import frappe -import sys -from six.moves import html_parser as HTMLParser -import smtplib, quopri, json -from frappe import msgprint, _, safe_decode, safe_encode, enqueue -from frappe.email.smtp import SMTPServer, get_outgoing_email_account -from frappe.email.email_body import get_email, get_formatted_html, add_attachment +from frappe import msgprint, _ from frappe.utils.verified_command import get_signed_params, verify_request -from html2text import html2text -from frappe.utils import get_url, nowdate, now_datetime, add_days, split_emails, cstr, cint -from rq.timeouts import JobTimeoutException -from six import text_type, string_types, PY3 -from email.parser import Parser - - -class EmailLimitCrossedError(frappe.ValidationError): pass - -def send(recipients=None, sender=None, subject=None, message=None, text_content=None, reference_doctype=None, - reference_name=None, unsubscribe_method=None, unsubscribe_params=None, unsubscribe_message=None, - attachments=None, reply_to=None, cc=None, bcc=None, message_id=None, in_reply_to=None, send_after=None, - expose_recipients=None, send_priority=1, communication=None, now=False, read_receipt=None, - queue_separately=False, is_notification=False, add_unsubscribe_link=1, inline_images=None, - header=None, print_letterhead=False, with_container=False): - """Add email to sending queue (Email Queue) - - :param recipients: List of recipients. - :param sender: Email sender. - :param subject: Email subject. - :param message: Email message. - :param text_content: Text version of email message. - :param reference_doctype: Reference DocType of caller document. - :param reference_name: Reference name of caller document. - :param send_priority: Priority for Email Queue, default 1. - :param unsubscribe_method: URL method for unsubscribe. Default is `/api/method/frappe.email.queue.unsubscribe`. - :param unsubscribe_params: additional params for unsubscribed links. default are name, doctype, email - :param attachments: Attachments to be sent. - :param reply_to: Reply to be captured here (default inbox) - :param in_reply_to: Used to send the Message-Id of a received email back as In-Reply-To. - :param send_after: Send this email after the given datetime. If value is in integer, then `send_after` will be the automatically set to no of days from current date. - :param communication: Communication link to be set in Email Queue record - :param now: Send immediately (don't send in the background) - :param queue_separately: Queue each email separately - :param is_notification: Marks email as notification so will not trigger notifications from system - :param add_unsubscribe_link: Send unsubscribe link in the footer of the Email, default 1. - :param inline_images: List of inline images as {"filename", "filecontent"}. All src properties will be replaced with random Content-Id - :param header: Append header in email (boolean) - :param with_container: Wraps email inside styled container - """ - if not unsubscribe_method: - unsubscribe_method = "/api/method/frappe.email.queue.unsubscribe" - - if not recipients and not cc: - return - - if not cc: - cc = [] - if not bcc: - bcc = [] - - if isinstance(recipients, string_types): - recipients = split_emails(recipients) - - if isinstance(cc, string_types): - cc = split_emails(cc) - - if isinstance(bcc, string_types): - bcc = split_emails(bcc) - - if isinstance(send_after, int): - send_after = add_days(nowdate(), send_after) - - email_account = get_outgoing_email_account(True, append_to=reference_doctype, sender=sender) - if not sender or sender == "Administrator": - sender = email_account.default_sender - - if not text_content: - try: - text_content = html2text(message) - except HTMLParser.HTMLParseError: - text_content = "See html attachment" - - recipients = list(set(recipients)) - cc = list(set(cc)) - - all_ids = tuple(recipients + cc) - - unsubscribed = frappe.db.sql_list(''' - SELECT - distinct email - from - `tabEmail Unsubscribe` - where - email in %(all_ids)s - and ( - ( - reference_doctype = %(reference_doctype)s - and reference_name = %(reference_name)s - ) - or global_unsubscribe = 1 - ) - ''', { - 'all_ids': all_ids, - 'reference_doctype': reference_doctype, - 'reference_name': reference_name, - }) - - recipients = [r for r in recipients if r and r not in unsubscribed] - - if cc: - cc = [r for r in cc if r and r not in unsubscribed] - - if not recipients and not cc: - # Recipients may have been unsubscribed, exit quietly - return - - email_text_context = text_content - - should_append_unsubscribe = (add_unsubscribe_link - and reference_doctype - and (unsubscribe_message or reference_doctype=="Newsletter") - and add_unsubscribe_link==1) - - unsubscribe_link = None - if should_append_unsubscribe: - unsubscribe_link = get_unsubscribe_message(unsubscribe_message, expose_recipients) - email_text_context += unsubscribe_link.text - - email_content = get_formatted_html(subject, message, - email_account=email_account, header=header, - unsubscribe_link=unsubscribe_link, with_container=with_container) - - # add to queue - add(recipients, sender, subject, - formatted=email_content, - text_content=email_text_context, - reference_doctype=reference_doctype, - reference_name=reference_name, - attachments=attachments, - reply_to=reply_to, - cc=cc, - bcc=bcc, - message_id=message_id, - in_reply_to=in_reply_to, - send_after=send_after, - send_priority=send_priority, - email_account=email_account, - communication=communication, - add_unsubscribe_link=add_unsubscribe_link, - unsubscribe_method=unsubscribe_method, - unsubscribe_params=unsubscribe_params, - expose_recipients=expose_recipients, - read_receipt=read_receipt, - queue_separately=queue_separately, - is_notification = is_notification, - inline_images = inline_images, - header=header, - now=now, - print_letterhead=print_letterhead) - - -def add(recipients, sender, subject, **kwargs): - """Add to Email Queue""" - if kwargs.get('queue_separately') or len(recipients) > 20: - email_queue = None - for r in recipients: - if not email_queue: - email_queue = get_email_queue([r], sender, subject, **kwargs) - if kwargs.get('now'): - send_one(email_queue.name, now=True) - else: - duplicate = email_queue.get_duplicate([r]) - duplicate.insert(ignore_permissions=True) - - if kwargs.get('now'): - send_one(duplicate.name, now=True) - - frappe.db.commit() - else: - email_queue = get_email_queue(recipients, sender, subject, **kwargs) - if kwargs.get('now'): - send_one(email_queue.name, now=True) - -def get_email_queue(recipients, sender, subject, **kwargs): - '''Make Email Queue object''' - e = frappe.new_doc('Email Queue') - e.priority = kwargs.get('send_priority') - attachments = kwargs.get('attachments') - if attachments: - # store attachments with fid or print format details, to be attached on-demand later - _attachments = [] - for att in attachments: - if att.get('fid'): - _attachments.append(att) - elif att.get("print_format_attachment") == 1: - if not att.get('lang', None): - att['lang'] = frappe.local.lang - att['print_letterhead'] = kwargs.get('print_letterhead') - _attachments.append(att) - e.attachments = json.dumps(_attachments) - - try: - mail = get_email(recipients, - sender=sender, - subject=subject, - formatted=kwargs.get('formatted'), - text_content=kwargs.get('text_content'), - attachments=kwargs.get('attachments'), - reply_to=kwargs.get('reply_to'), - cc=kwargs.get('cc'), - bcc=kwargs.get('bcc'), - email_account=kwargs.get('email_account'), - expose_recipients=kwargs.get('expose_recipients'), - inline_images=kwargs.get('inline_images'), - header=kwargs.get('header')) - - mail.set_message_id(kwargs.get('message_id'),kwargs.get('is_notification')) - if kwargs.get('read_receipt'): - mail.msg_root["Disposition-Notification-To"] = sender - if kwargs.get('in_reply_to'): - mail.set_in_reply_to(kwargs.get('in_reply_to')) - - e.message_id = mail.msg_root["Message-Id"].strip(" <>") - e.message = cstr(mail.as_string()) - e.sender = mail.sender - - except frappe.InvalidEmailAddressError: - # bad Email Address - don't add to queue - import traceback - frappe.log_error('Invalid Email ID Sender: {0}, Recipients: {1}, \nTraceback: {2} '.format(mail.sender, - ', '.join(mail.recipients), traceback.format_exc()), 'Email Not Sent') - - recipients = list(set(recipients + kwargs.get('cc', []) + kwargs.get('bcc', []))) - e.set_recipients(recipients) - e.reference_doctype = kwargs.get('reference_doctype') - e.reference_name = kwargs.get('reference_name') - e.add_unsubscribe_link = kwargs.get("add_unsubscribe_link") - e.unsubscribe_method = kwargs.get('unsubscribe_method') - e.unsubscribe_params = kwargs.get('unsubscribe_params') - e.expose_recipients = kwargs.get('expose_recipients') - e.communication = kwargs.get('communication') - e.send_after = kwargs.get('send_after') - e.show_as_cc = ",".join(kwargs.get('cc', [])) - e.show_as_bcc = ",".join(kwargs.get('bcc', [])) - e.insert(ignore_permissions=True) - - return e +from frappe.utils import get_url, now_datetime, cint def get_emails_sent_this_month(): return frappe.db.sql(""" @@ -328,44 +85,25 @@ def return_unsubscribed_page(email, doctype, name): indicator_color='green') def flush(from_test=False): - """flush email queue, every time: called from scheduler""" - # additional check - - auto_commit = not from_test + """flush email queue, every time: called from scheduler + """ + from frappe.email.doctype.email_queue.email_queue import send_mail + # To avoid running jobs inside unit tests if frappe.are_emails_muted(): msgprint(_("Emails are muted")) from_test = True - smtpserver_dict = frappe._dict() + if cint(frappe.defaults.get_defaults().get("hold_queue"))==1: + return - for email in get_queue(): + for row in get_queue(): + try: + func = send_mail if from_test else send_mail.enqueue + is_background_task = not from_test + func(email_queue_name = row.name, is_background_task = is_background_task) + except Exception: + frappe.log_error() - if cint(frappe.defaults.get_defaults().get("hold_queue"))==1: - break - - if email.name: - smtpserver = smtpserver_dict.get(email.sender) - if not smtpserver: - smtpserver = SMTPServer() - smtpserver_dict[email.sender] = smtpserver - - if from_test: - send_one(email.name, smtpserver, auto_commit) - else: - send_one_args = { - 'email': email.name, - 'smtpserver': smtpserver, - 'auto_commit': auto_commit, - } - enqueue( - method = 'frappe.email.queue.send_one', - queue = 'short', - **send_one_args - ) - - # NOTE: removing commit here because we pass auto_commit - # finally: - # frappe.db.commit() def get_queue(): return frappe.db.sql('''select name, sender @@ -378,213 +116,6 @@ def get_queue(): by priority desc, creation asc limit 500''', { 'now': now_datetime() }, as_dict=True) - -def send_one(email, smtpserver=None, auto_commit=True, now=False): - '''Send Email Queue with given smtpserver''' - - email = frappe.db.sql('''select - name, status, communication, message, sender, reference_doctype, - reference_name, unsubscribe_param, unsubscribe_method, expose_recipients, - show_as_cc, add_unsubscribe_link, attachments, retry - from - `tabEmail Queue` - where - name=%s - for update''', email, as_dict=True) - - if len(email): - email = email[0] - else: - return - - recipients_list = frappe.db.sql('''select name, recipient, status from - `tabEmail Queue Recipient` where parent=%s''', email.name, as_dict=1) - - if frappe.are_emails_muted(): - frappe.msgprint(_("Emails are muted")) - return - - if cint(frappe.defaults.get_defaults().get("hold_queue"))==1 : - return - - if email.status not in ('Not Sent','Partially Sent') : - # rollback to release lock and return - frappe.db.rollback() - return - - frappe.db.sql("""update `tabEmail Queue` set status='Sending', modified=%s where name=%s""", - (now_datetime(), email.name), auto_commit=auto_commit) - - if email.communication: - frappe.get_doc('Communication', email.communication).set_delivery_status(commit=auto_commit) - - email_sent_to_any_recipient = None - - try: - message = None - - if not frappe.flags.in_test: - if not smtpserver: - smtpserver = SMTPServer() - - # to avoid always using default email account for outgoing - if getattr(frappe.local, "outgoing_email_account", None): - frappe.local.outgoing_email_account = {} - - smtpserver.setup_email_account(email.reference_doctype, sender=email.sender) - - for recipient in recipients_list: - if recipient.status != "Not Sent": - continue - - message = prepare_message(email, recipient.recipient, recipients_list) - if not frappe.flags.in_test: - smtpserver.sess.sendmail(email.sender, recipient.recipient, message) - - recipient.status = "Sent" - frappe.db.sql("""update `tabEmail Queue Recipient` set status='Sent', modified=%s where name=%s""", - (now_datetime(), recipient.name), auto_commit=auto_commit) - - email_sent_to_any_recipient = any("Sent" == s.status for s in recipients_list) - - #if all are sent set status - if email_sent_to_any_recipient: - frappe.db.sql("""update `tabEmail Queue` set status='Sent', modified=%s where name=%s""", - (now_datetime(), email.name), auto_commit=auto_commit) - else: - frappe.db.sql("""update `tabEmail Queue` set status='Error', error=%s - where name=%s""", ("No recipients to send to", email.name), auto_commit=auto_commit) - if frappe.flags.in_test: - frappe.flags.sent_mail = message - return - if email.communication: - frappe.get_doc('Communication', email.communication).set_delivery_status(commit=auto_commit) - - if smtpserver.append_emails_to_sent_folder and email_sent_to_any_recipient: - smtpserver.email_account.append_email_to_sent_folder(message) - - except (smtplib.SMTPServerDisconnected, - smtplib.SMTPConnectError, - smtplib.SMTPHeloError, - smtplib.SMTPAuthenticationError, - smtplib.SMTPRecipientsRefused, - JobTimeoutException): - - # bad connection/timeout, retry later - - if email_sent_to_any_recipient: - frappe.db.sql("""update `tabEmail Queue` set status='Partially Sent', modified=%s where name=%s""", - (now_datetime(), email.name), auto_commit=auto_commit) - else: - frappe.db.sql("""update `tabEmail Queue` set status='Not Sent', modified=%s where name=%s""", - (now_datetime(), email.name), auto_commit=auto_commit) - - if email.communication: - frappe.get_doc('Communication', email.communication).set_delivery_status(commit=auto_commit) - - # no need to attempt further - return - - except Exception as e: - frappe.db.rollback() - - if email.retry < 3: - frappe.db.sql("""update `tabEmail Queue` set status='Not Sent', modified=%s, retry=retry+1 where name=%s""", - (now_datetime(), email.name), auto_commit=auto_commit) - else: - if email_sent_to_any_recipient: - frappe.db.sql("""update `tabEmail Queue` set status='Partially Errored', error=%s where name=%s""", - (text_type(e), email.name), auto_commit=auto_commit) - else: - frappe.db.sql("""update `tabEmail Queue` set status='Error', error=%s - where name=%s""", (text_type(e), email.name), auto_commit=auto_commit) - - if email.communication: - frappe.get_doc('Communication', email.communication).set_delivery_status(commit=auto_commit) - - if now: - print(frappe.get_traceback()) - raise e - - else: - # log to Error Log - frappe.log_error('frappe.email.queue.flush') - -def prepare_message(email, recipient, recipients_list): - message = email.message - if not message: - return "" - - # Parse "Email Account" from "Email Sender" - email_account = get_outgoing_email_account(raise_exception_not_set=False, sender=email.sender) - if frappe.conf.use_ssl and email_account.track_email_status: - # Using SSL => Publically available domain => Email Read Reciept Possible - message = message.replace("", quopri.encodestring(''.format(frappe.local.site, email.communication).encode()).decode()) - else: - # No SSL => No Email Read Reciept - message = message.replace("", quopri.encodestring("".encode()).decode()) - - if email.add_unsubscribe_link and email.reference_doctype: # is missing the check for unsubscribe message but will not add as there will be no unsubscribe url - unsubscribe_url = get_unsubcribed_url(email.reference_doctype, email.reference_name, recipient, - email.unsubscribe_method, email.unsubscribe_params) - message = message.replace("", quopri.encodestring(unsubscribe_url.encode()).decode()) - - if email.expose_recipients == "header": - pass - else: - if email.expose_recipients == "footer": - if isinstance(email.show_as_cc, string_types): - email.show_as_cc = email.show_as_cc.split(",") - email_sent_to = [r.recipient for r in recipients_list] - email_sent_cc = ", ".join([e for e in email_sent_to if e in email.show_as_cc]) - email_sent_to = ", ".join([e for e in email_sent_to if e not in email.show_as_cc]) - - if email_sent_cc: - email_sent_message = _("This email was sent to {0} and copied to {1}").format(email_sent_to,email_sent_cc) - else: - email_sent_message = _("This email was sent to {0}").format(email_sent_to) - message = message.replace("", quopri.encodestring(email_sent_message.encode()).decode()) - - message = message.replace("", recipient) - - message = (message and message.encode('utf8')) or '' - message = safe_decode(message) - - if PY3: - from email.policy import SMTPUTF8 - message = Parser(policy=SMTPUTF8).parsestr(message) - else: - message = Parser().parsestr(message) - - if email.attachments: - # On-demand attachments - - attachments = json.loads(email.attachments) - - for attachment in attachments: - if attachment.get('fcontent'): - continue - - fid = attachment.get("fid") - if fid: - _file = frappe.get_doc("File", fid) - fcontent = _file.get_content() - attachment.update({ - 'fname': _file.file_name, - 'fcontent': fcontent, - 'parent': message - }) - attachment.pop("fid", None) - add_attachment(**attachment) - - elif attachment.get("print_format_attachment") == 1: - attachment.pop("print_format_attachment", None) - print_format_file = frappe.attach_print(**attachment) - print_format_file.update({"parent": message}) - add_attachment(**print_format_file) - - return safe_encode(message.as_string()) - def clear_outbox(days=None): """Remove low priority older than 31 days in Outbox or configured in Log Settings. Note: Used separate query to avoid deadlock diff --git a/frappe/email/receive.py b/frappe/email/receive.py index 949da4a343..9ad560aa4a 100644 --- a/frappe/email/receive.py +++ b/frappe/email/receive.py @@ -8,11 +8,11 @@ import imaplib import poplib import re import time +import json from email.header import decode_header import _socket import chardet -import six from email_reply_parser import EmailReplyParser import frappe @@ -20,13 +20,26 @@ from frappe import _, safe_decode, safe_encode from frappe.core.doctype.file.file import (MaxFileSizeReachedError, get_random_filename) from frappe.utils import (cint, convert_utc_to_user_timezone, cstr, - extract_email_id, markdown, now, parse_addr, strip) + extract_email_id, markdown, now, parse_addr, strip, get_datetime, + add_days, sanitize_html) +from frappe.utils.user import is_system_user +from frappe.utils.html_utils import clean_email_html + +# fix due to a python bug in poplib that limits it to 2048 +poplib._MAXLINE = 20480 +imaplib._MAXLINE = 20480 + +# fix due to a python bug in poplib that limits it to 2048 +poplib._MAXLINE = 20480 +imaplib._MAXLINE = 20480 class EmailSizeExceededError(frappe.ValidationError): pass class EmailTimeoutError(frappe.ValidationError): pass class TotalSizeExceededError(frappe.ValidationError): pass class LoginLimitExceeded(frappe.ValidationError): pass +class SentEmailInInboxError(Exception): + pass class EmailServer: """Wrapper for POP server to pull emails.""" @@ -100,14 +113,11 @@ class EmailServer: def get_messages(self): """Returns new email messages in a list.""" - if not self.check_mails(): - return # nothing to do + if not (self.check_mails() or self.connect()): + return [] frappe.db.commit() - if not self.connect(): - return - uid_list = [] try: @@ -116,7 +126,6 @@ class EmailServer: self.latest_messages = [] self.seen_status = {} self.uid_reindexed = False - uid_list = email_list = self.get_new_mails() if not email_list: @@ -132,11 +141,7 @@ class EmailServer: self.max_email_size = cint(frappe.local.conf.get("max_email_size")) self.max_total_size = 5 * self.max_email_size - for i, message_meta in enumerate(email_list): - # do not pull more than NUM emails - if (i+1) > num: - break - + for i, message_meta in enumerate(email_list[:num]): try: self.retrieve_message(message_meta, i+1) except (TotalSizeExceededError, EmailTimeoutError, LoginLimitExceeded): @@ -152,7 +157,6 @@ class EmailServer: except Exception as e: if self.has_login_limit_exceeded(e): pass - else: raise @@ -284,7 +288,7 @@ class EmailServer: flags = [] for flag in imaplib.ParseFlags(flag_string) or []: - pattern = re.compile("\w+") + pattern = re.compile(r"\w+") match = re.search(pattern, frappe.as_unicode(flag)) flags.append(match.group(0)) @@ -361,14 +365,12 @@ class Email: """Parses headers, content, attachments from given raw message. :param content: Raw message.""" - if six.PY2: - self.mail = email.message_from_string(safe_encode(content)) + if isinstance(content, bytes): + self.mail = email.message_from_bytes(content) else: - if isinstance(content, bytes): - self.mail = email.message_from_bytes(content) - else: - self.mail = email.message_from_string(content) + self.mail = email.message_from_string(content) + self.raw_message = content self.text_content = '' self.html_content = '' self.attachments = [] @@ -391,6 +393,10 @@ class Email: if self.date > now(): self.date = now() + @property + def in_reply_to(self): + return (self.mail.get("In-Reply-To") or "").strip(" <>") + def parse(self): """Walk and process multi-part email.""" for part in self.mail.walk(): @@ -555,13 +561,330 @@ class Email: def get_thread_id(self): """Extract thread ID from `[]`""" - l = re.findall('(?<=\[)[\w/-]+', self.subject) + l = re.findall(r'(?<=\[)[\w/-]+', self.subject) return l and l[0] or None + def is_reply(self): + return bool(self.in_reply_to) -# fix due to a python bug in poplib that limits it to 2048 -poplib._MAXLINE = 20480 -imaplib._MAXLINE = 20480 +class InboundMail(Email): + """Class representation of incoming mail along with mail handlers. + """ + def __init__(self, content, email_account, uid=None, seen_status=None): + super().__init__(content) + self.email_account = email_account + self.uid = uid or -1 + self.seen_status = seen_status or 0 + + # System documents related to this mail + self._parent_email_queue = None + self._parent_communication = None + self._reference_document = None + + self.flags = frappe._dict() + + def get_content(self): + if self.content_type == 'text/html': + return clean_email_html(self.content) + + def process(self): + """Create communication record from email. + """ + if self.is_sender_same_as_receiver() and not self.is_reply(): + if frappe.flags.in_test: + print('WARN: Cannot pull email. Sender same as recipient inbox') + raise SentEmailInInboxError + + communication = self.is_exist_in_system() + if communication: + communication.update_db(uid=self.uid) + communication.reload() + return communication + + self.flags.is_new_communication = True + return self._build_communication_doc() + + def _build_communication_doc(self): + data = self.as_dict() + data['doctype'] = "Communication" + + if self.parent_communication(): + data['in_reply_to'] = self.parent_communication().name + + if self.reference_document(): + data['reference_doctype'] = self.reference_document().doctype + data['reference_name'] = self.reference_document().name + elif self.email_account.append_to and self.email_account.append_to != 'Communication': + reference_doc = self._create_reference_document(self.email_account.append_to) + if reference_doc: + data['reference_doctype'] = reference_doc.doctype + data['reference_name'] = reference_doc.name + data['is_first'] = True + + if self.is_notification(): + # Disable notifications for notification. + data['unread_notification_sent'] = 1 + + if self.seen_status: + data['_seen'] = json.dumps(self.get_users_linked_to_account(self.email_account)) + + communication = frappe.get_doc(data) + communication.flags.in_receive = True + communication.insert(ignore_permissions=True) + + # save attachments + communication._attachments = self.save_attachments_in_doc(communication) + communication.content = sanitize_html(self.replace_inline_images(communication._attachments)) + communication.save() + return communication + + def replace_inline_images(self, attachments): + # replace inline images + content = self.content + for file in attachments: + if file.name in self.cid_map and self.cid_map[file.name]: + content = content.replace("cid:{0}".format(self.cid_map[file.name]), + file.file_url) + return content + + def is_notification(self): + isnotification = self.mail.get("isnotification") + return isnotification and ("notification" in isnotification) + + def is_exist_in_system(self): + """Check if this email already exists in the system(as communication document). + """ + from frappe.core.doctype.communication.communication import Communication + if not self.message_id: + return + + return Communication.find_one_by_filters(message_id = self.message_id, + order_by = 'creation DESC') + + def is_sender_same_as_receiver(self): + return self.from_email == self.email_account.email_id + + def is_reply_to_system_sent_mail(self): + """Is it a reply to already sent mail. + """ + return self.is_reply() and frappe.local.site in self.in_reply_to + + def parent_email_queue(self): + """Get parent record from `Email Queue`. + + If it is a reply to already sent mail, then there will be a parent record in EMail Queue. + """ + from frappe.email.doctype.email_queue.email_queue import EmailQueue + + if self._parent_email_queue is not None: + return self._parent_email_queue + + parent_email_queue = '' + if self.is_reply_to_system_sent_mail(): + parent_email_queue = EmailQueue.find_one_by_filters(message_id=self.in_reply_to) + + self._parent_email_queue = parent_email_queue or '' + return self._parent_email_queue + + def parent_communication(self): + """Find a related communication so that we can prepare a mail thread. + + The way it happens is by using in-reply-to header, and we can't make thread if it does not exist. + + Here are the cases to handle: + 1. If mail is a reply to already sent mail, then we can get parent communicaion from + Email Queue record. + 2. Sometimes we send communication name in message-ID directly, use that to get parent communication. + 3. Sender sent a reply but reply is on top of what (s)he sent before, + then parent record exists directly in communication. + """ + from frappe.core.doctype.communication.communication import Communication + if self._parent_communication is not None: + return self._parent_communication + + if not self.is_reply(): + return '' + + if not self.is_reply_to_system_sent_mail(): + communication = Communication.find_one_by_filters(message_id=self.in_reply_to, + creation = ['>=', self.get_relative_dt(-30)]) + elif self.parent_email_queue() and self.parent_email_queue().communication: + communication = Communication.find(self.parent_email_queue().communication, ignore_error=True) + else: + reference = self.in_reply_to + if '@' in self.in_reply_to: + reference, _ = self.in_reply_to.split("@", 1) + communication = Communication.find(reference, ignore_error=True) + + self._parent_communication = communication or '' + return self._parent_communication + + def reference_document(self): + """Reference document is a document to which mail relate to. + + We can get reference document from Parent record(EmailQueue | Communication) if exists. + Otherwise we do subject match to find reference document if we know the reference(append_to) doctype. + """ + if self._reference_document is not None: + return self._reference_document + + reference_document = "" + parent = self.parent_email_queue() or self.parent_communication() + + if parent and parent.reference_doctype: + reference_doctype, reference_name = parent.reference_doctype, parent.reference_name + reference_document = self.get_doc(reference_doctype, reference_name, ignore_error=True) + + if not reference_document and self.email_account.append_to: + reference_document = self.match_record_by_subject_and_sender(self.email_account.append_to) + + self._reference_document = reference_document or '' + return self._reference_document + + def get_reference_name_from_subject(self): + """ + Ex: "Re: Your email (#OPP-2020-2334343)" + """ + return self.subject.rsplit('#', 1)[-1].strip(' ()') + + def match_record_by_subject_and_sender(self, doctype): + """Find a record in the given doctype that matches with email subject and sender. + + Cases: + 1. Sometimes record name is part of subject. We can get document by parsing name from subject + 2. Find by matching sender and subject + 3. Find by matching subject alone (Special case) + Ex: when a System User is using Outlook and replies to an email from their own client, + it reaches the Email Account with the threading info lost and the (sender + subject match) + doesn't work because the sender in the first communication was someone different to whom + the system user is replying to via the common email account in Frappe. This fix bypasses + the sender match when the sender is a system user and subject is atleast 10 chars long + (for additional safety) + + NOTE: We consider not to match by subject if match record is very old. + """ + name = self.get_reference_name_from_subject() + email_fields = self.get_email_fields(doctype) + + record = self.get_doc(doctype, name, ignore_error=True) if name else None + + if not record: + subject = self.clean_subject(self.subject) + filters = { + email_fields.subject_field: ("like", f"%{subject}%"), + "creation": (">", self.get_relative_dt(days=-60)) + } + + # Sender check is not needed incase mail is from system user. + if not (len(subject) > 10 and is_system_user(self.from_email)): + filters[email_fields.sender_field] = self.from_email + + name = frappe.db.get_value(self.email_account.append_to, filters = filters) + record = self.get_doc(doctype, name, ignore_error=True) if name else None + return record + + def _create_reference_document(self, doctype): + """ Create reference document if it does not exist in the system. + """ + parent = frappe.new_doc(doctype) + email_fileds = self.get_email_fields(doctype) + + if email_fileds.subject_field: + parent.set(email_fileds.subject_field, frappe.as_unicode(self.subject)[:140]) + + if email_fileds.sender_field: + parent.set(email_fileds.sender_field, frappe.as_unicode(self.from_email)) + + parent.flags.ignore_mandatory = True + + try: + parent.insert(ignore_permissions=True) + except frappe.DuplicateEntryError: + # try and find matching parent + parent_name = frappe.db.get_value(self.email_account.append_to, + {email_fileds.sender_field: email.from_email} + ) + if parent_name: + parent.name = parent_name + else: + parent = None + return parent + + + @staticmethod + def get_doc(doctype, docname, ignore_error=False): + try: + return frappe.get_doc(doctype, docname) + except frappe.DoesNotExistError: + if ignore_error: + return + raise + + @staticmethod + def get_relative_dt(days): + """Get relative to current datetime. Only relative days are supported. + """ + return add_days(get_datetime(), days) + + @staticmethod + def get_users_linked_to_account(email_account): + """Get list of users who linked to Email account. + """ + users = frappe.get_all("User Email", filters={"email_account": email_account.name}, + fields=["parent"]) + return list(set([user.get("parent") for user in users])) + + @staticmethod + def clean_subject(subject): + """Remove Prefixes like 'fw', FWD', 're' etc from subject. + """ + # Match strings like "fw:", "re :" etc. + regex = r"(^\s*(fw|fwd|wg)[^:]*:|\s*(re|aw)[^:]*:\s*)*" + return frappe.as_unicode(strip(re.sub(regex, "", subject, 0, flags=re.IGNORECASE))) + + @staticmethod + def get_email_fields(doctype): + """Returns Email related fields of a doctype. + """ + fields = frappe._dict() + + email_fields = ['subject_field', 'sender_field'] + meta = frappe.get_meta(doctype) + + for field in email_fields: + if hasattr(meta, field): + fields[field] = getattr(meta, field) + return fields + + @staticmethod + def get_document(self, doctype, name): + """Is same as frappe.get_doc but suppresses the DoesNotExist error. + """ + try: + return frappe.get_doc(doctype, name) + except frappe.DoesNotExistError: + return None + + def as_dict(self): + """ + """ + return { + "subject": self.subject, + "content": self.get_content(), + 'text_content': self.text_content, + "sent_or_received": "Received", + "sender_full_name": self.from_real_name, + "sender": self.from_email, + "recipients": self.mail.get("To"), + "cc": self.mail.get("CC"), + "email_account": self.email_account.name, + "communication_medium": "Email", + "uid": self.uid, + "message_id": self.message_id, + "communication_date": self.date, + "has_attachment": 1 if self.attachments else 0, + "seen": self.seen_status or 0 + } class TimerMixin(object): def __init__(self, *args, **kwargs): diff --git a/frappe/email/smtp.py b/frappe/email/smtp.py index 9ba81fa146..74492c09c3 100644 --- a/frappe/email/smtp.py +++ b/frappe/email/smtp.py @@ -1,7 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import frappe import smtplib import email.utils @@ -9,11 +8,24 @@ import _socket, sys from frappe import _ from frappe.utils import cint, cstr, parse_addr +CONNECTION_FAILED = _('Could not connect to outgoing email server') +AUTH_ERROR_TITLE = _("Invalid Credentials") +AUTH_ERROR = _("Incorrect email or password. Please check your login credentials.") +SOCKET_ERROR_TITLE = _("Incorrect Configuration") +SOCKET_ERROR = _("Invalid Outgoing Mail Server or Port") +SEND_MAIL_FAILED = _("Unable to send emails at this time") +EMAIL_ACCOUNT_MISSING = _('Email Account not setup. Please create a new Email Account from Setup > Email > Email Account') + +class InvalidEmailCredentials(frappe.ValidationError): + pass + def send(email, append_to=None, retry=1): """Deprecated: Send the message or add it to Outbox Email""" def _send(retry): + from frappe.email.doctype.email_account.email_account import EmailAccount try: - smtpserver = SMTPServer(append_to=append_to) + email_account = EmailAccount.find_outgoing(match_by_doctype=append_to) + smtpserver = email_account.get_smtp_server() # validate is called in as_string email_body = email.as_string() @@ -34,224 +46,81 @@ def send(email, append_to=None, retry=1): _send(retry) -def get_outgoing_email_account(raise_exception_not_set=True, append_to=None, sender=None): - """Returns outgoing email account based on `append_to` or the default - outgoing account. If default outgoing account is not found, it will - try getting settings from `site_config.json`.""" - - sender_email_id = None - _email_account = None - - if sender: - sender_email_id = parse_addr(sender)[1] - - if not getattr(frappe.local, "outgoing_email_account", None): - frappe.local.outgoing_email_account = {} - - if not (frappe.local.outgoing_email_account.get(append_to) - or frappe.local.outgoing_email_account.get(sender_email_id) - or frappe.local.outgoing_email_account.get("default")): - email_account = None - - if sender_email_id: - # check if the sender has an email account with enable_outgoing - email_account = _get_email_account({"enable_outgoing": 1, - "email_id": sender_email_id}) - - if not email_account and append_to: - # append_to is only valid when enable_incoming is checked - email_accounts = frappe.db.get_values("Email Account", { - "enable_outgoing": 1, - "enable_incoming": 1, - "append_to": append_to, - }, cache=True) - - if email_accounts: - _email_account = email_accounts[0] - - else: - email_account = _get_email_account({ - "enable_outgoing": 1, - "enable_incoming": 1, - "append_to": append_to - }) - - if not email_account: - # sender don't have the outging email account - sender_email_id = None - email_account = get_default_outgoing_email_account(raise_exception_not_set=raise_exception_not_set) - - if not email_account and _email_account: - # if default email account is not configured then setup first email account based on append to - email_account = _email_account - - if not email_account and raise_exception_not_set and cint(frappe.db.get_single_value('System Settings', 'setup_complete')): - frappe.throw(_("Please setup default Email Account from Setup > Email > Email Account"), - frappe.OutgoingEmailError) - - if email_account: - if email_account.enable_outgoing and not getattr(email_account, 'from_site_config', False): - raise_exception = True - if email_account.smtp_server in ['localhost','127.0.0.1'] or email_account.no_smtp_authentication: - raise_exception = False - email_account.password = email_account.get_password(raise_exception=raise_exception) - email_account.default_sender = email.utils.formataddr((email_account.name, email_account.get("email_id"))) - - frappe.local.outgoing_email_account[append_to or sender_email_id or "default"] = email_account - - return frappe.local.outgoing_email_account.get(append_to) \ - or frappe.local.outgoing_email_account.get(sender_email_id) \ - or frappe.local.outgoing_email_account.get("default") - -def get_default_outgoing_email_account(raise_exception_not_set=True): - '''conf should be like: - { - "mail_server": "smtp.example.com", - "mail_port": 587, - "use_tls": 1, - "mail_login": "emails@example.com", - "mail_password": "Super.Secret.Password", - "auto_email_id": "emails@example.com", - "email_sender_name": "Example Notifications", - "always_use_account_email_id_as_sender": 0, - "always_use_account_name_as_sender_name": 0 - } - ''' - email_account = _get_email_account({"enable_outgoing": 1, "default_outgoing": 1}) - if email_account: - email_account.password = email_account.get_password(raise_exception=False) - - if not email_account and frappe.conf.get("mail_server"): - # from site_config.json - email_account = frappe.new_doc("Email Account") - email_account.update({ - "smtp_server": frappe.conf.get("mail_server"), - "smtp_port": frappe.conf.get("mail_port"), - - # legacy: use_ssl was used in site_config instead of use_tls, but meant the same thing - "use_tls": cint(frappe.conf.get("use_tls") or 0) or cint(frappe.conf.get("use_ssl") or 0), - "login_id": frappe.conf.get("mail_login"), - "email_id": frappe.conf.get("auto_email_id") or frappe.conf.get("mail_login") or 'notifications@example.com', - "password": frappe.conf.get("mail_password"), - "always_use_account_email_id_as_sender": frappe.conf.get("always_use_account_email_id_as_sender", 0), - "always_use_account_name_as_sender_name": frappe.conf.get("always_use_account_name_as_sender_name", 0) - }) - email_account.from_site_config = True - email_account.name = frappe.conf.get("email_sender_name") or "Frappe" - - if not email_account and not raise_exception_not_set: - return None - - if frappe.are_emails_muted(): - # create a stub - email_account = frappe.new_doc("Email Account") - email_account.update({ - "email_id": "notifications@example.com" - }) - - return email_account - -def _get_email_account(filters): - name = frappe.db.get_value("Email Account", filters) - return frappe.get_doc("Email Account", name) if name else None - class SMTPServer: - def __init__(self, login=None, password=None, server=None, port=None, use_tls=None, use_ssl=None, append_to=None): - # get defaults from mail settings + def __init__(self, server, login=None, password=None, port=None, use_tls=None, use_ssl=None): + self.login = login + self.password = password + self._server = server + self._port = port + self.use_tls = use_tls + self.use_ssl = use_ssl + self._session = None - self._sess = None - self.email_account = None - self.server = None - self.append_emails_to_sent_folder = None - - if server: - self.server = server - self.port = port - self.use_tls = cint(use_tls) - self.use_ssl = cint(use_ssl) - self.login = login - self.password = password - - else: - self.setup_email_account(append_to) - - def setup_email_account(self, append_to=None, sender=None): - self.email_account = get_outgoing_email_account(raise_exception_not_set=False, append_to=append_to, sender=sender) - if self.email_account: - self.server = self.email_account.smtp_server - self.login = (getattr(self.email_account, "login_id", None) or self.email_account.email_id) - if not self.email_account.no_smtp_authentication: - if self.email_account.ascii_encode_password: - self.password = frappe.safe_encode(self.email_account.password, 'ascii') - else: - self.password = self.email_account.password - else: - self.password = None - self.port = self.email_account.smtp_port - self.use_tls = self.email_account.use_tls - self.sender = self.email_account.email_id - self.use_ssl = self.email_account.use_ssl_for_outgoing - self.append_emails_to_sent_folder = self.email_account.append_emails_to_sent_folder - self.always_use_account_email_id_as_sender = cint(self.email_account.get("always_use_account_email_id_as_sender")) - self.always_use_account_name_as_sender_name = cint(self.email_account.get("always_use_account_name_as_sender_name")) + if not self.server: + frappe.msgprint(EMAIL_ACCOUNT_MISSING, raise_exception=frappe.OutgoingEmailError) @property - def sess(self): - """get session""" - if self._sess: - return self._sess + def port(self): + port = self._port or (self.use_ssl and 465) or (self.use_tls and 587) + return cint(port) - # check if email server specified - if not getattr(self, 'server'): - err_msg = _('Email Account not setup. Please create a new Email Account from Setup > Email > Email Account') - frappe.msgprint(err_msg) - raise frappe.OutgoingEmailError(err_msg) + @property + def server(self): + return cstr(self._server or "") + + def secure_session(self, conn): + """Secure the connection incase of TLS. + """ + if self.use_tls: + conn.ehlo() + conn.starttls() + conn.ehlo() + + @property + def session(self): + if self.is_session_active(): + return self._session + + SMTP = smtplib.SMTP_SSL if self.use_ssl else smtplib.SMTP try: - if self.use_ssl: - if not self.port: - self.port = 465 - - self._sess = smtplib.SMTP_SSL((self.server or ""), cint(self.port)) - else: - if self.use_tls and not self.port: - self.port = 587 - - self._sess = smtplib.SMTP(cstr(self.server or ""), - cint(self.port) or None) - - if not self._sess: - err_msg = _('Could not connect to outgoing email server') - frappe.msgprint(err_msg) - raise frappe.OutgoingEmailError(err_msg) - - if self.use_tls: - self._sess.ehlo() - self._sess.starttls() - self._sess.ehlo() + _session = SMTP(self.server, self.port) + if not _session: + frappe.msgprint(CONNECTION_FAILED, raise_exception=frappe.OutgoingEmailError) + self.secure_session(_session) if self.login and self.password: - ret = self._sess.login(str(self.login or ""), str(self.password or "")) + res = _session.login(str(self.login or ""), str(self.password or "")) # check if logged correctly - if ret[0]!=235: - frappe.msgprint(ret[1]) - raise frappe.OutgoingEmailError(ret[1]) + if res[0]!=235: + frappe.msgprint(res[1], raise_exception=frappe.OutgoingEmailError) - return self._sess + self._session = _session + return self._session except smtplib.SMTPAuthenticationError as e: - from frappe.email.doctype.email_account.email_account import EmailAccount - EmailAccount.throw_invalid_credentials_exception() + self.throw_invalid_credentials_exception() except _socket.error as e: # Invalid mail server -- due to refusing connection - frappe.throw( - _("Invalid Outgoing Mail Server or Port"), - exc=frappe.ValidationError, - title=_("Incorrect Configuration") - ) + frappe.throw(SOCKET_ERROR, title=SOCKET_ERROR_TITLE) except smtplib.SMTPException: - frappe.msgprint(_('Unable to send emails at this time')) + frappe.msgprint(SEND_MAIL_FAILED) raise + + def is_session_active(self): + if self._session: + try: + return self._session.noop()[0] == 250 + except Exception: + return False + + def quit(self): + if self.is_session_active(): + self._session.quit() + + @classmethod + def throw_invalid_credentials_exception(cls): + frappe.throw(AUTH_ERROR, title=AUTH_ERROR_TITLE, exc=InvalidEmailCredentials) diff --git a/frappe/email/test_email_body.py b/frappe/email/test_email_body.py index 3fcabb9495..8e637273ed 100644 --- a/frappe/email/test_email_body.py +++ b/frappe/email/test_email_body.py @@ -1,14 +1,11 @@ # Copyright (c) 2017, Frappe Technologies Pvt. Ltd. and Contributors # License: GNU General Public License v3. See license.txt -from __future__ import unicode_literals - import unittest, os, base64 from frappe import safe_decode from frappe.email.receive import Email from frappe.email.email_body import (replace_filename_with_cid, get_email, inline_style_in_html, get_header) -from frappe.email.queue import prepare_message, get_email_queue -from six import PY3 +from frappe.email.doctype.email_queue.email_queue import SendMailContext, QueueBuilder class TestEmailBody(unittest.TestCase): @@ -42,38 +39,31 @@ This is the text version of this email ).as_string().replace("\r\n", "\n") def test_prepare_message_returns_already_encoded_string(self): + uni_chr1 = chr(40960) + uni_chr2 = chr(1972) - if PY3: - uni_chr1 = chr(40960) - uni_chr2 = chr(1972) - else: - uni_chr1 = unichr(40960) - uni_chr2 = unichr(1972) - - email = get_email_queue( + queue_doc = QueueBuilder( recipients=['test@example.com'], sender='me@example.com', subject='Test Subject', - content='

' + uni_chr1 + 'abcd' + uni_chr2 + '

', - formatted='

' + uni_chr1 + 'abcd' + uni_chr2 + '

', - text_content='whatever') - result = prepare_message(email=email, recipient='test@test.com', recipients_list=[]) + message='

' + uni_chr1 + 'abcd' + uni_chr2 + '

', + text_content='whatever').process()[0] + mail_ctx = SendMailContext(queue_doc = queue_doc) + result = mail_ctx.build_message(recipient_email = 'test@test.com') self.assertTrue(b"

=EA=80=80abcd=DE=B4

" in result) def test_prepare_message_returns_cr_lf(self): - email = get_email_queue( + queue_doc = QueueBuilder( recipients=['test@example.com'], sender='me@example.com', subject='Test Subject', - content='

\n this is a test of newlines\n' + '

', - formatted='

\n this is a test of newlines\n' + '

', - text_content='whatever') - result = safe_decode(prepare_message(email=email, - recipient='test@test.com', recipients_list=[])) - if PY3: - self.assertTrue(result.count('\n') == result.count("\r")) - else: - self.assertTrue(True) + message='

\n this is a test of newlines\n' + '

', + text_content='whatever').process()[0] + + mail_ctx = SendMailContext(queue_doc = queue_doc) + result = safe_decode(mail_ctx.build_message(recipient_email='test@test.com')) + + self.assertTrue(result.count('\n') == result.count("\r")) def test_image(self): img_signature = ''' diff --git a/frappe/email/test_smtp.py b/frappe/email/test_smtp.py index 0b11c559a2..58e4fdd8a6 100644 --- a/frappe/email/test_smtp.py +++ b/frappe/email/test_smtp.py @@ -4,7 +4,7 @@ import unittest import frappe from frappe.email.smtp import SMTPServer -from frappe.email.smtp import get_outgoing_email_account +from frappe.email.doctype.email_account.email_account import EmailAccount class TestSMTP(unittest.TestCase): def test_smtp_ssl_session(self): @@ -33,13 +33,13 @@ class TestSMTP(unittest.TestCase): frappe.local.outgoing_email_account = {} # lowest preference given to email account with default incoming enabled - create_email_account(email_id="default_outgoing_enabled@gmail.com", password="***", enable_outgoing = 1, default_outgoing=1) - self.assertEqual(get_outgoing_email_account().email_id, "default_outgoing_enabled@gmail.com") + create_email_account(email_id="default_outgoing_enabled@gmail.com", password="password", enable_outgoing = 1, default_outgoing=1) + self.assertEqual(EmailAccount.find_outgoing().email_id, "default_outgoing_enabled@gmail.com") frappe.local.outgoing_email_account = {} # highest preference given to email account with append_to matching - create_email_account(email_id="append_to@gmail.com", password="***", enable_outgoing = 1, default_outgoing=1, append_to="Blog Post") - self.assertEqual(get_outgoing_email_account(append_to="Blog Post").email_id, "append_to@gmail.com") + create_email_account(email_id="append_to@gmail.com", password="password", enable_outgoing = 1, default_outgoing=1, append_to="Blog Post") + self.assertEqual(EmailAccount.find_outgoing(match_by_doctype="Blog Post").email_id, "append_to@gmail.com") # add back the mail_server frappe.conf['mail_server'] = mail_server @@ -75,4 +75,4 @@ def make_server(port, ssl, tls): use_tls = tls ) - server.sess + server.session diff --git a/frappe/email/utils.py b/frappe/email/utils.py index 8b4bd95ba0..24ce77b922 100644 --- a/frappe/email/utils.py +++ b/frappe/email/utils.py @@ -1,7 +1,5 @@ # Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and contributors # For license information, please see license.txt - -from __future__ import unicode_literals, print_function import imaplib, poplib from frappe.utils import cint diff --git a/frappe/event_streaming/doctype/document_type_field_mapping/document_type_field_mapping.py b/frappe/event_streaming/doctype/document_type_field_mapping/document_type_field_mapping.py index 1ab9534bdc..fc8164d8a4 100644 --- a/frappe/event_streaming/doctype/document_type_field_mapping/document_type_field_mapping.py +++ b/frappe/event_streaming/doctype/document_type_field_mapping/document_type_field_mapping.py @@ -2,7 +2,6 @@ # Copyright (c) 2019, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals # import frappe from frappe.model.document import Document diff --git a/frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.py b/frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.py index bf96e4e27b..2cf7282a5a 100644 --- a/frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.py +++ b/frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.py @@ -1,12 +1,9 @@ # -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and contributors # For license information, please see license.txt - -from __future__ import unicode_literals import frappe import json from frappe import _ -from six import iteritems from frappe.model.document import Document from frappe.model import default_fields @@ -100,7 +97,7 @@ class DocumentTypeMapping(Document): def get_mapped_dependency(self, mapping, producer_site, doc): inner_mapping = frappe.get_doc('Document Type Mapping', mapping.mapping) filters = json.loads(mapping.remote_value_filters) - for key, value in iteritems(filters): + for key, value in filters.items(): if value.startswith('eval:'): val = frappe.safe_eval(value[5:], None, dict(doc=doc)) filters[key] = val @@ -117,7 +114,7 @@ class DocumentTypeMapping(Document): def map_rows_removed(self, update_diff, mapping): removed = [] mapping['removed'] = update_diff.removed - for key, value in iteritems(update_diff.removed.copy()): + for key, value in update_diff.removed.copy().items(): local_table_name = frappe.db.get_value('Document Type Field Mapping', { 'remote_fieldname': key, 'parent': self.name @@ -133,7 +130,7 @@ class DocumentTypeMapping(Document): def map_rows(self, update_diff, mapping, producer_site, operation): remote_fields = [] - for tablename, entries in iteritems(update_diff.get(operation).copy()): + for tablename, entries in update_diff.get(operation).copy().items(): local_table_name = frappe.db.get_value('Document Type Field Mapping', {'remote_fieldname': tablename}, 'local_fieldname') table_map = frappe.db.get_value('Document Type Field Mapping', {'local_fieldname': local_table_name, 'parent': self.name}, 'mapping') table_map = frappe.get_doc('Document Type Mapping', table_map) diff --git a/frappe/event_streaming/doctype/document_type_mapping/test_document_type_mapping.py b/frappe/event_streaming/doctype/document_type_mapping/test_document_type_mapping.py index 178d7b6b6a..b1bb322855 100644 --- a/frappe/event_streaming/doctype/document_type_mapping/test_document_type_mapping.py +++ b/frappe/event_streaming/doctype/document_type_mapping/test_document_type_mapping.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - # import frappe import unittest diff --git a/frappe/event_streaming/doctype/event_consumer/event_consumer.py b/frappe/event_streaming/doctype/event_consumer/event_consumer.py index 5789e09e74..00d304f7f4 100644 --- a/frappe/event_streaming/doctype/event_consumer/event_consumer.py +++ b/frappe/event_streaming/doctype/event_consumer/event_consumer.py @@ -2,7 +2,6 @@ # Copyright (c) 2019, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe import json import requests @@ -31,7 +30,7 @@ class EventConsumer(Document): self.update_consumer_status() else: frappe.db.set_value(self.doctype, self.name, 'incoming_change', 0) - + frappe.cache().delete_value('event_consumer_document_type_map') def on_trash(self): diff --git a/frappe/event_streaming/doctype/event_consumer/test_event_consumer.py b/frappe/event_streaming/doctype/event_consumer/test_event_consumer.py index 9e344842bd..b8072ecabd 100644 --- a/frappe/event_streaming/doctype/event_consumer/test_event_consumer.py +++ b/frappe/event_streaming/doctype/event_consumer/test_event_consumer.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - # import frappe import unittest diff --git a/frappe/event_streaming/doctype/event_consumer_document_type/event_consumer_document_type.py b/frappe/event_streaming/doctype/event_consumer_document_type/event_consumer_document_type.py index 197338027f..cf5d18edfd 100644 --- a/frappe/event_streaming/doctype/event_consumer_document_type/event_consumer_document_type.py +++ b/frappe/event_streaming/doctype/event_consumer_document_type/event_consumer_document_type.py @@ -2,7 +2,6 @@ # Copyright (c) 2019, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals # import frappe from frappe.model.document import Document diff --git a/frappe/event_streaming/doctype/event_producer/event_producer.py b/frappe/event_streaming/doctype/event_producer/event_producer.py index e43b4d131c..4836276734 100644 --- a/frappe/event_streaming/doctype/event_producer/event_producer.py +++ b/frappe/event_streaming/doctype/event_producer/event_producer.py @@ -2,20 +2,19 @@ # Copyright (c) 2019, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals -import frappe import json import time + import requests -from six import iteritems + +import frappe from frappe import _ -from frappe.model.document import Document -from frappe.frappeclient import FrappeClient -from frappe.utils.background_jobs import get_jobs -from frappe.utils.data import get_url, get_link_to_form -from frappe.utils.password import get_decrypted_password from frappe.custom.doctype.custom_field.custom_field import create_custom_field -from frappe.integrations.oauth2 import validate_url +from frappe.frappeclient import FrappeClient +from frappe.model.document import Document +from frappe.utils.background_jobs import get_jobs +from frappe.utils.data import get_link_to_form, get_url +from frappe.utils.password import get_decrypted_password class EventProducer(Document): @@ -56,8 +55,8 @@ class EventProducer(Document): self.reload() def check_url(self): - if not validate_url(self.producer_url): - frappe.throw(_('Invalid URL')) + valid_url_schemes = ("http", "https") + frappe.utils.validate_url(self.producer_url, throw=True, valid_schemes=valid_url_schemes) # remove '/' from the end of the url like http://test_site.com/ # to prevent mismatch in get_url() results @@ -272,8 +271,8 @@ def set_insert(update, producer_site, event_producer): if update.mapping: if update.get('dependencies'): dependencies_created = sync_mapped_dependencies(update.dependencies, producer_site) - for fieldname, value in iteritems(dependencies_created): - doc.update({ fieldname : value }) + for fieldname, value in dependencies_created.items(): + doc.update({fieldname: value}) else: sync_dependencies(doc, producer_site) @@ -304,8 +303,8 @@ def set_update(update, producer_site): if update.mapping: if update.get('dependencies'): dependencies_created = sync_mapped_dependencies(update.dependencies, producer_site) - for fieldname, value in iteritems(dependencies_created): - local_doc.update({ fieldname : value }) + for fieldname, value in dependencies_created.items(): + local_doc.update({fieldname: value}) else: sync_dependencies(local_doc, producer_site) @@ -315,7 +314,7 @@ def set_update(update, producer_site): def update_row_removed(local_doc, removed): """Sync child table row deletion type update""" - for tablename, rownames in iteritems(removed): + for tablename, rownames in removed.items(): table = local_doc.get_table_field_doctype(tablename) for row in rownames: table_rows = local_doc.get(tablename) @@ -333,7 +332,7 @@ def get_child_table_row(table_rows, row): def update_row_changed(local_doc, changed): """Sync child table row updation type update""" - for tablename, rows in iteritems(changed): + for tablename, rows in changed.items(): old = local_doc.get(tablename) for doc in old: for row in rows: @@ -343,7 +342,7 @@ def update_row_changed(local_doc, changed): def update_row_added(local_doc, added): """Sync child table row addition type update""" - for tablename, rows in iteritems(added): + for tablename, rows in added.items(): local_doc.extend(tablename, rows) for child in rows: child_doc = frappe.get_doc(child) diff --git a/frappe/event_streaming/doctype/event_producer/test_event_producer.py b/frappe/event_streaming/doctype/event_producer/test_event_producer.py index 4c259c3729..883f4f2df2 100644 --- a/frappe/event_streaming/doctype/event_producer/test_event_producer.py +++ b/frappe/event_streaming/doctype/event_producer/test_event_producer.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import frappe import unittest import json @@ -154,7 +152,7 @@ class TestEventProducer(unittest.TestCase): def test_conditional_events(self): producer = get_remote_site() - + # Add Condition event_producer = frappe.get_doc('Event Producer', producer_url) note_producer_entry = [ @@ -192,7 +190,7 @@ class TestEventProducer(unittest.TestCase): def test_conditional_events_with_cmd(self): producer = get_remote_site() - + # Add Condition event_producer = frappe.get_doc('Event Producer', producer_url) note_producer_entry = [ diff --git a/frappe/event_streaming/doctype/event_producer_document_type/event_producer_document_type.py b/frappe/event_streaming/doctype/event_producer_document_type/event_producer_document_type.py index 2870d5330f..9ae70e0f97 100644 --- a/frappe/event_streaming/doctype/event_producer_document_type/event_producer_document_type.py +++ b/frappe/event_streaming/doctype/event_producer_document_type/event_producer_document_type.py @@ -2,7 +2,6 @@ # Copyright (c) 2019, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals # import frappe from frappe.model.document import Document diff --git a/frappe/event_streaming/doctype/event_producer_last_update/event_producer_last_update.py b/frappe/event_streaming/doctype/event_producer_last_update/event_producer_last_update.py index 02e297bdd5..391cf79c27 100644 --- a/frappe/event_streaming/doctype/event_producer_last_update/event_producer_last_update.py +++ b/frappe/event_streaming/doctype/event_producer_last_update/event_producer_last_update.py @@ -2,7 +2,6 @@ # Copyright (c) 2020, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals # import frappe from frappe.model.document import Document diff --git a/frappe/event_streaming/doctype/event_producer_last_update/test_event_producer_last_update.py b/frappe/event_streaming/doctype/event_producer_last_update/test_event_producer_last_update.py index 0311cb2df9..62ea71edab 100644 --- a/frappe/event_streaming/doctype/event_producer_last_update/test_event_producer_last_update.py +++ b/frappe/event_streaming/doctype/event_producer_last_update/test_event_producer_last_update.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - # import frappe import unittest diff --git a/frappe/event_streaming/doctype/event_sync_log/event_sync_log.py b/frappe/event_streaming/doctype/event_sync_log/event_sync_log.py index 31b1f863aa..1d255a5c30 100644 --- a/frappe/event_streaming/doctype/event_sync_log/event_sync_log.py +++ b/frappe/event_streaming/doctype/event_sync_log/event_sync_log.py @@ -2,7 +2,6 @@ # Copyright (c) 2019, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals # import frappe from frappe.model.document import Document diff --git a/frappe/event_streaming/doctype/event_sync_log/test_event_sync_log.py b/frappe/event_streaming/doctype/event_sync_log/test_event_sync_log.py index 6c621b8b0e..ef55dc0f16 100644 --- a/frappe/event_streaming/doctype/event_sync_log/test_event_sync_log.py +++ b/frappe/event_streaming/doctype/event_sync_log/test_event_sync_log.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - # import frappe import unittest diff --git a/frappe/event_streaming/doctype/event_update_log/event_update_log.py b/frappe/event_streaming/doctype/event_update_log/event_update_log.py index 1c31718c2b..ae851c70d1 100644 --- a/frappe/event_streaming/doctype/event_update_log/event_update_log.py +++ b/frappe/event_streaming/doctype/event_update_log/event_update_log.py @@ -2,7 +2,6 @@ # Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document from frappe.utils.background_jobs import get_jobs @@ -235,7 +234,7 @@ def get_update_logs_for_consumer(event_consumer, doctypes, last_update): if isinstance(doctypes, str): doctypes = frappe.parse_json(doctypes) - + from frappe.event_streaming.doctype.event_consumer.event_consumer import has_consumer_access consumer = frappe.get_doc('Event Consumer', event_consumer) diff --git a/frappe/event_streaming/doctype/event_update_log/test_event_update_log.py b/frappe/event_streaming/doctype/event_update_log/test_event_update_log.py index e00fc767d9..99ced3c209 100644 --- a/frappe/event_streaming/doctype/event_update_log/test_event_update_log.py +++ b/frappe/event_streaming/doctype/event_update_log/test_event_update_log.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and Contributors # See license.txt -from __future__ import unicode_literals - # import frappe import unittest diff --git a/frappe/event_streaming/doctype/event_update_log_consumer/event_update_log_consumer.py b/frappe/event_streaming/doctype/event_update_log_consumer/event_update_log_consumer.py index ee6d5d8ca9..80a59e4c31 100644 --- a/frappe/event_streaming/doctype/event_update_log_consumer/event_update_log_consumer.py +++ b/frappe/event_streaming/doctype/event_update_log_consumer/event_update_log_consumer.py @@ -2,7 +2,6 @@ # Copyright (c) 2020, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals # import frappe from frappe.model.document import Document diff --git a/frappe/exceptions.py b/frappe/exceptions.py index ab65e6e006..13abd8f4f8 100644 --- a/frappe/exceptions.py +++ b/frappe/exceptions.py @@ -1,18 +1,9 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals -import sys - # BEWARE don't put anything in this file except exceptions from werkzeug.exceptions import NotFound - -if sys.version_info.major == 2: - class FileNotFoundError(Exception): pass -else: - from builtins import FileNotFoundError - class SiteNotSpecifiedError(Exception): def __init__(self, *args, **kwargs): self.message = "Please specify --site sitename" diff --git a/frappe/frappeclient.py b/frappe/frappeclient.py index 054a8c9369..e57f82b60a 100644 --- a/frappe/frappeclient.py +++ b/frappe/frappeclient.py @@ -1,8 +1,6 @@ -from __future__ import print_function, unicode_literals import requests import json import frappe -from six import iteritems, string_types import base64 ''' @@ -88,7 +86,7 @@ class FrappeClient(object): def get_list(self, doctype, fields='["name"]', filters=None, limit_start=0, limit_page_length=0): """Returns list of records of a particular type""" - if not isinstance(fields, string_types): + if not isinstance(fields, str): fields = json.dumps(fields) params = { "fields": fields, @@ -310,7 +308,7 @@ class FrappeClient(object): def preprocess(self, params): """convert dicts, lists to json""" - for key, value in iteritems(params): + for key, value in params.items(): if isinstance(value, (dict, list)): params[key] = json.dumps(value) diff --git a/frappe/geo/country_info.json b/frappe/geo/country_info.json index 1e0ae161bc..7ffdf0a8bf 100644 --- a/frappe/geo/country_info.json +++ b/frappe/geo/country_info.json @@ -953,7 +953,7 @@ "currency_fraction_units": 100, "smallest_currency_fraction_value": 0.01, "currency_symbol": "\u20ac", - "number_format": "#,###.##", + "number_format": "#.###,##", "timezones": [ "Europe/Berlin" ] diff --git a/frappe/geo/country_info.py b/frappe/geo/country_info.py index 4f878325ad..ddebd1fb0e 100644 --- a/frappe/geo/country_info.py +++ b/frappe/geo/country_info.py @@ -2,8 +2,6 @@ # MIT License. See license.txt # all country info -from __future__ import unicode_literals - import os, json, frappe from frappe.utils.momentjs import get_all_timezones diff --git a/frappe/geo/doctype/country/__init__.py b/frappe/geo/doctype/country/__init__.py index baffc48825..8b13789179 100644 --- a/frappe/geo/doctype/country/__init__.py +++ b/frappe/geo/doctype/country/__init__.py @@ -1 +1 @@ -from __future__ import unicode_literals + diff --git a/frappe/geo/doctype/country/country.py b/frappe/geo/doctype/country/country.py index 5f8b6f7bd5..54935e6eaf 100644 --- a/frappe/geo/doctype/country/country.py +++ b/frappe/geo/doctype/country/country.py @@ -1,7 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: See license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document diff --git a/frappe/geo/doctype/country/test_country.py b/frappe/geo/doctype/country/test_country.py index 81849d6886..e00d6ecf37 100644 --- a/frappe/geo/doctype/country/test_country.py +++ b/frappe/geo/doctype/country/test_country.py @@ -1,6 +1,5 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: See license.txt -from __future__ import unicode_literals import frappe test_records = frappe.get_test_records('Country') \ No newline at end of file diff --git a/frappe/geo/doctype/currency/__init__.py b/frappe/geo/doctype/currency/__init__.py index baffc48825..8b13789179 100644 --- a/frappe/geo/doctype/currency/__init__.py +++ b/frappe/geo/doctype/currency/__init__.py @@ -1 +1 @@ -from __future__ import unicode_literals + diff --git a/frappe/geo/doctype/currency/currency.py b/frappe/geo/doctype/currency/currency.py index 688303fd50..b3ce67cc67 100644 --- a/frappe/geo/doctype/currency/currency.py +++ b/frappe/geo/doctype/currency/currency.py @@ -1,7 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: See license.txt -from __future__ import unicode_literals import frappe from frappe import throw, _ diff --git a/frappe/geo/doctype/currency/test_currency.py b/frappe/geo/doctype/currency/test_currency.py index 7945e193da..5552e675ec 100644 --- a/frappe/geo/doctype/currency/test_currency.py +++ b/frappe/geo/doctype/currency/test_currency.py @@ -3,6 +3,5 @@ # pre loaded -from __future__ import unicode_literals import frappe test_records = frappe.get_test_records('Currency') \ No newline at end of file diff --git a/frappe/geo/utils.py b/frappe/geo/utils.py index d94a13ea41..89de176f0b 100644 --- a/frappe/geo/utils.py +++ b/frappe/geo/utils.py @@ -2,8 +2,6 @@ # Copyright (c) 2020, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals - import frappe from pymysql import InternalError diff --git a/frappe/handler.py b/frappe/handler.py index a38feb90fa..de86c15c8f 100755 --- a/frappe/handler.py +++ b/frappe/handler.py @@ -1,8 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals - from werkzeug.wrappers import Response import frappe @@ -228,10 +226,7 @@ def run_doc_method(method, docs=None, dt=None, dn=None, arg=None, args=None): is_whitelisted(fn) is_valid_http_method(fn) - try: - fnargs = inspect.getargspec(method_obj)[0] - except ValueError: - fnargs = inspect.getfullargspec(method_obj).args + fnargs = inspect.getfullargspec(method_obj).args if not fnargs or (len(fnargs)==1 and fnargs[0]=="self"): response = doc.run_method(method) diff --git a/frappe/hooks.py b/frappe/hooks.py index 74c538c5df..ac42a03461 100644 --- a/frappe/hooks.py +++ b/frappe/hooks.py @@ -1,4 +1,4 @@ -from __future__ import unicode_literals + from . import __version__ as app_version @@ -29,16 +29,16 @@ page_js = { # website app_include_js = [ - "/assets/js/libs.min.js", - "/assets/js/desk.min.js", - "/assets/js/list.min.js", - "/assets/js/form.min.js", - "/assets/js/control.min.js", - "/assets/js/report.min.js", + "libs.bundle.js", + "desk.bundle.js", + "list.bundle.js", + "form.bundle.js", + "controls.bundle.js", + "report.bundle.js", ] app_include_css = [ - "/assets/css/desk.min.css", - "/assets/css/report.min.css", + "desk.bundle.css", + "report.bundle.css", ] doctype_js = { @@ -52,6 +52,8 @@ web_include_js = [ web_include_css = [] +email_css = ['email.bundle.css'] + website_route_rules = [ {"from_route": "/blog/", "to_route": "Blog Post"}, {"from_route": "/kb/", "to_route": "Help Article"}, @@ -130,6 +132,16 @@ has_website_permission = { "Address": "frappe.contacts.doctype.address.address.has_website_permission" } +jinja = { + "methods": "frappe.utils.jinja_globals", + "filters": [ + "frappe.utils.data.global_date_format", + "frappe.utils.markdown", + "frappe.website.utils.get_shade", + "frappe.website.utils.abs_url", + ] +} + standard_queries = { "User": "frappe.core.doctype.user.user.user_query" } @@ -216,7 +228,6 @@ scheduler_events = { "frappe.desk.doctype.event.event.send_event_digest", "frappe.sessions.clear_expired_sessions", "frappe.email.doctype.notification.notification.trigger_daily_alerts", - "frappe.realtime.remove_old_task_logs", "frappe.utils.scheduler.restrict_scheduler_events_if_dormant", "frappe.email.doctype.auto_email_report.auto_email_report.send_daily", "frappe.website.doctype.personal_data_deletion_request.personal_data_deletion_request.remove_unverified_record", diff --git a/frappe/installer.py b/frappe/installer.py index 0cd5b136ae..d7d885d60e 100755 --- a/frappe/installer.py +++ b/frappe/installer.py @@ -390,19 +390,16 @@ def get_conf_params(db_name=None, db_password=None): def make_site_dirs(): - site_public_path = os.path.join(frappe.local.site_path, 'public') - site_private_path = os.path.join(frappe.local.site_path, 'private') - for dir_path in ( - os.path.join(site_private_path, 'backups'), - os.path.join(site_public_path, 'files'), - os.path.join(site_private_path, 'files'), - os.path.join(frappe.local.site_path, 'logs'), - os.path.join(frappe.local.site_path, 'task-logs')): - if not os.path.exists(dir_path): - os.makedirs(dir_path) - locks_dir = frappe.get_site_path('locks') - if not os.path.exists(locks_dir): - os.makedirs(locks_dir) + for dir_path in [ + os.path.join("public", "files"), + os.path.join("private", "backups"), + os.path.join("private", "files"), + "error-snapshots", + "locks", + "logs", + ]: + path = frappe.get_site_path(dir_path) + os.makedirs(path, exist_ok=True) def add_module_defs(app): diff --git a/frappe/integrations/doctype/braintree_settings/braintree_settings.py b/frappe/integrations/doctype/braintree_settings/braintree_settings.py index 768f58c0a0..9dc9778bee 100644 --- a/frappe/integrations/doctype/braintree_settings/braintree_settings.py +++ b/frappe/integrations/doctype/braintree_settings/braintree_settings.py @@ -2,12 +2,11 @@ # Copyright (c) 2018, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document import braintree from frappe import _ -from six.moves.urllib.parse import urlencode +from urllib.parse import urlencode from frappe.utils import get_url, call_hook_method from frappe.integrations.utils import create_request_log, create_payment_gateway diff --git a/frappe/integrations/doctype/braintree_settings/test_braintree_settings.py b/frappe/integrations/doctype/braintree_settings/test_braintree_settings.py index 80fa3c54b8..72a678a92c 100644 --- a/frappe/integrations/doctype/braintree_settings/test_braintree_settings.py +++ b/frappe/integrations/doctype/braintree_settings/test_braintree_settings.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2018, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import unittest class TestBraintreeSettings(unittest.TestCase): diff --git a/frappe/integrations/doctype/connected_app/connected_app.json b/frappe/integrations/doctype/connected_app/connected_app.json index e5dbb0472a..b5330f4d4f 100644 --- a/frappe/integrations/doctype/connected_app/connected_app.json +++ b/frappe/integrations/doctype/connected_app/connected_app.json @@ -54,7 +54,8 @@ "fieldname": "client_id", "fieldtype": "Data", "in_list_view": 1, - "label": "Client Id" + "label": "Client Id", + "mandatory_depends_on": "eval:doc.redirect_uri" }, { "fieldname": "redirect_uri", @@ -96,12 +97,14 @@ { "fieldname": "authorization_uri", "fieldtype": "Data", - "label": "Authorization URI" + "label": "Authorization URI", + "mandatory_depends_on": "eval:doc.redirect_uri" }, { "fieldname": "token_uri", "fieldtype": "Data", - "label": "Token URI" + "label": "Token URI", + "mandatory_depends_on": "eval:doc.redirect_uri" }, { "fieldname": "revocation_uri", @@ -136,7 +139,7 @@ "link_fieldname": "connected_app" } ], - "modified": "2020-11-16 16:29:50.277405", + "modified": "2021-05-10 05:03:06.296863", "modified_by": "Administrator", "module": "Integrations", "name": "Connected App", diff --git a/frappe/integrations/doctype/connected_app/connected_app.py b/frappe/integrations/doctype/connected_app/connected_app.py index 95077ece77..449e30f6d0 100644 --- a/frappe/integrations/doctype/connected_app/connected_app.py +++ b/frappe/integrations/doctype/connected_app/connected_app.py @@ -26,20 +26,27 @@ class ConnectedApp(Document): self.redirect_uri = urljoin(base_url, callback_path) def get_oauth2_session(self, user=None, init=False): + """Return an auto-refreshing OAuth2 session which is an extension of a requests.Session()""" token = None token_updater = None + auto_refresh_kwargs = None if not init: user = user or frappe.session.user token_cache = self.get_user_token(user) token = token_cache.get_json() token_updater = token_cache.update_data + auto_refresh_kwargs = {'client_id': self.client_id} + client_secret = self.get_password('client_secret') + if client_secret: + auto_refresh_kwargs['client_secret'] = client_secret return OAuth2Session( client_id=self.client_id, token=token, token_updater=token_updater, auto_refresh_url=self.token_uri, + auto_refresh_kwargs=auto_refresh_kwargs, redirect_uri=self.redirect_uri, scope=self.get_scopes() ) diff --git a/frappe/integrations/doctype/connected_app/test_connected_app.py b/frappe/integrations/doctype/connected_app/test_connected_app.py index b4304f6ee8..d1ff19ecb2 100644 --- a/frappe/integrations/doctype/connected_app/test_connected_app.py +++ b/frappe/integrations/doctype/connected_app/test_connected_app.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and contributors # See license.txt -from __future__ import unicode_literals - import unittest import requests from urllib.parse import urljoin diff --git a/frappe/integrations/doctype/dropbox_settings/test_dropbox_settings.py b/frappe/integrations/doctype/dropbox_settings/test_dropbox_settings.py index 539fc417f2..d34e65de50 100644 --- a/frappe/integrations/doctype/dropbox_settings/test_dropbox_settings.py +++ b/frappe/integrations/doctype/dropbox_settings/test_dropbox_settings.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - # import frappe import unittest diff --git a/frappe/integrations/doctype/google_drive/test_google_drive.py b/frappe/integrations/doctype/google_drive/test_google_drive.py index f06e13572c..96e8577c7c 100644 --- a/frappe/integrations/doctype/google_drive/test_google_drive.py +++ b/frappe/integrations/doctype/google_drive/test_google_drive.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - # import frappe import unittest diff --git a/frappe/integrations/doctype/google_settings/google_settings.py b/frappe/integrations/doctype/google_settings/google_settings.py index ecc975235a..9a3f3c8ae2 100644 --- a/frappe/integrations/doctype/google_settings/google_settings.py +++ b/frappe/integrations/doctype/google_settings/google_settings.py @@ -2,7 +2,6 @@ # Copyright (c) 2019, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals # import frappe from frappe.model.document import Document diff --git a/frappe/integrations/doctype/integration_request/integration_request.py b/frappe/integrations/doctype/integration_request/integration_request.py index f1d59beb5a..4c4961d96d 100644 --- a/frappe/integrations/doctype/integration_request/integration_request.py +++ b/frappe/integrations/doctype/integration_request/integration_request.py @@ -2,11 +2,9 @@ # Copyright (c) 2015, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document import json -from six import string_types from frappe.integrations.utils import json_handler class IntegrationRequest(Document): @@ -25,14 +23,14 @@ class IntegrationRequest(Document): def handle_success(self, response): """update the output field with the response along with the relevant status""" - if isinstance(response, string_types): + if isinstance(response, str): response = json.loads(response) self.db_set("status", "Completed") self.db_set("output", json.dumps(response, default=json_handler)) def handle_failure(self, response): """update the error field with the response along with the relevant status""" - if isinstance(response, string_types): + if isinstance(response, str): response = json.loads(response) self.db_set("status", "Failed") self.db_set("error", json.dumps(response, default=json_handler)) \ No newline at end of file diff --git a/frappe/integrations/doctype/integration_request/test_integration_request.py b/frappe/integrations/doctype/integration_request/test_integration_request.py index 6b77b57de4..a26eb4ba93 100644 --- a/frappe/integrations/doctype/integration_request/test_integration_request.py +++ b/frappe/integrations/doctype/integration_request/test_integration_request.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import frappe import unittest diff --git a/frappe/integrations/doctype/ldap_group_mapping/ldap_group_mapping.py b/frappe/integrations/doctype/ldap_group_mapping/ldap_group_mapping.py index f9f2adeed0..b6bb77d964 100644 --- a/frappe/integrations/doctype/ldap_group_mapping/ldap_group_mapping.py +++ b/frappe/integrations/doctype/ldap_group_mapping/ldap_group_mapping.py @@ -2,7 +2,6 @@ # Copyright (c) 2019, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals # import frappe from frappe.model.document import Document diff --git a/frappe/integrations/doctype/ldap_settings/ldap_settings.py b/frappe/integrations/doctype/ldap_settings/ldap_settings.py index 80dfef2693..122096cf6f 100644 --- a/frappe/integrations/doctype/ldap_settings/ldap_settings.py +++ b/frappe/integrations/doctype/ldap_settings/ldap_settings.py @@ -2,7 +2,6 @@ # Copyright (c) 2015, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe import _, safe_encode from frappe.model.document import Document diff --git a/frappe/integrations/doctype/ldap_settings/test_ldap_settings.py b/frappe/integrations/doctype/ldap_settings/test_ldap_settings.py index e6cf4eef3a..113692b6c4 100644 --- a/frappe/integrations/doctype/ldap_settings/test_ldap_settings.py +++ b/frappe/integrations/doctype/ldap_settings/test_ldap_settings.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - # import frappe import unittest diff --git a/frappe/integrations/doctype/oauth_authorization_code/oauth_authorization_code.json b/frappe/integrations/doctype/oauth_authorization_code/oauth_authorization_code.json index 13150f6cb3..2cd21bcaf4 100644 --- a/frappe/integrations/doctype/oauth_authorization_code/oauth_authorization_code.json +++ b/frappe/integrations/doctype/oauth_authorization_code/oauth_authorization_code.json @@ -1,256 +1,112 @@ { - "allow_copy": 0, - "allow_import": 0, - "allow_rename": 0, - "autoname": "field:authorization_code", - "beta": 0, - "creation": "2016-08-24 14:12:13.647159", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "Document", - "editable_grid": 1, - "engine": "InnoDB", + "actions": [], + "autoname": "field:authorization_code", + "creation": "2016-08-24 14:12:13.647159", + "doctype": "DocType", + "document_type": "Document", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "client", + "user", + "scopes", + "authorization_code", + "expiration_time", + "redirect_uri_bound_to_authorization_code", + "validity", + "nonce", + "code_challenge", + "code_challenge_method" + ], "fields": [ { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "client", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 1, - "in_standard_filter": 1, - "label": "Client", - "length": 0, - "no_copy": 0, - "options": "OAuth Client", - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "client", + "fieldtype": "Link", + "in_list_view": 1, + "in_standard_filter": 1, + "label": "Client", + "options": "OAuth Client", + "read_only": 1 + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "user", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "User", - "length": 0, - "no_copy": 0, - "options": "User", - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "user", + "fieldtype": "Link", + "label": "User", + "options": "User", + "read_only": 1 + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "scopes", - "fieldtype": "Text", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Scopes", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "scopes", + "fieldtype": "Text", + "label": "Scopes", + "read_only": 1 + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "authorization_code", - "fieldtype": "Text", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Authorization Code", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "authorization_code", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Authorization Code", + "read_only": 1, + "unique": 1 + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "expiration_time", - "fieldtype": "Datetime", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Expiration time", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "expiration_time", + "fieldtype": "Datetime", + "label": "Expiration time", + "read_only": 1 + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "redirect_uri_bound_to_authorization_code", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Redirect URI Bound To Auth Code", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "redirect_uri_bound_to_authorization_code", + "fieldtype": "Data", + "label": "Redirect URI Bound To Auth Code", + "read_only": 1 + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "validity", - "fieldtype": "Select", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Validity", - "length": 0, - "no_copy": 0, - "options": "Valid\nInvalid", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "fieldname": "validity", + "fieldtype": "Select", + "in_list_view": 1, + "label": "Validity", + "options": "Valid\nInvalid", + "read_only": 1 + }, + { + "fieldname": "nonce", + "fieldtype": "Data", + "label": "nonce", + "read_only": 1 + }, + { + "fieldname": "code_challenge", + "fieldtype": "Data", + "label": "Code Challenge", + "read_only": 1 + }, + { + "fieldname": "code_challenge_method", + "fieldtype": "Select", + "label": "Code challenge method", + "options": "\ns256\nplain", + "read_only": 1 } - ], - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2017-03-08 14:40:04.113884", - "modified_by": "Administrator", - "module": "Integrations", - "name": "OAuth Authorization Code", - "name_case": "", - "owner": "Administrator", + ], + "links": [], + "modified": "2021-04-26 07:23:02.980612", + "modified_by": "Administrator", + "module": "Integrations", + "name": "OAuth Authorization Code", + "owner": "Administrator", "permissions": [ { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 0, - "delete": 1, - "email": 1, - "export": 1, - "if_owner": 0, - "import": 0, - "is_custom": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "set_user_permissions": 0, - "share": 1, - "submit": 0, - "write": 0 + "delete": 1, + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1 } - ], - "quick_entry": 0, - "read_only": 0, - "read_only_onload": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1, - "track_seen": 0 + ], + "sort_field": "modified", + "sort_order": "DESC" } \ No newline at end of file diff --git a/frappe/integrations/doctype/oauth_authorization_code/oauth_authorization_code.py b/frappe/integrations/doctype/oauth_authorization_code/oauth_authorization_code.py index f08e7eb5bb..0c7f02844c 100644 --- a/frappe/integrations/doctype/oauth_authorization_code/oauth_authorization_code.py +++ b/frappe/integrations/doctype/oauth_authorization_code/oauth_authorization_code.py @@ -2,7 +2,6 @@ # Copyright (c) 2015, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document diff --git a/frappe/integrations/doctype/oauth_authorization_code/test_oauth_authorization_code.py b/frappe/integrations/doctype/oauth_authorization_code/test_oauth_authorization_code.py index cecf187e61..6084dd64b4 100644 --- a/frappe/integrations/doctype/oauth_authorization_code/test_oauth_authorization_code.py +++ b/frappe/integrations/doctype/oauth_authorization_code/test_oauth_authorization_code.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import frappe import unittest diff --git a/frappe/integrations/doctype/oauth_bearer_token/oauth_bearer_token.json b/frappe/integrations/doctype/oauth_bearer_token/oauth_bearer_token.json index aec5320ccc..083f1c9c54 100644 --- a/frappe/integrations/doctype/oauth_bearer_token/oauth_bearer_token.json +++ b/frappe/integrations/doctype/oauth_bearer_token/oauth_bearer_token.json @@ -1,283 +1,96 @@ { - "allow_copy": 0, - "allow_import": 0, - "allow_rename": 0, - "autoname": "field:access_token", - "beta": 0, - "creation": "2016-08-24 14:10:17.471264", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "Document", - "editable_grid": 1, - "engine": "InnoDB", + "actions": [], + "autoname": "field:access_token", + "creation": "2016-08-24 14:10:17.471264", + "doctype": "DocType", + "document_type": "Document", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "client", + "user", + "scopes", + "access_token", + "refresh_token", + "expiration_time", + "expires_in", + "status" + ], "fields": [ { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "client", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 1, - "in_standard_filter": 1, - "label": "Client", - "length": 0, - "no_copy": 0, - "options": "OAuth Client", - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "client", + "fieldtype": "Link", + "in_list_view": 1, + "in_standard_filter": 1, + "label": "Client", + "options": "OAuth Client", + "read_only": 1 + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "user", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "User", - "length": 0, - "no_copy": 0, - "options": "User", - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "user", + "fieldtype": "Link", + "label": "User", + "options": "User", + "read_only": 1 + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "scopes", - "fieldtype": "Text", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Scopes", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "scopes", + "fieldtype": "Text", + "label": "Scopes", + "read_only": 1 + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "access_token", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Access Token", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "access_token", + "fieldtype": "Data", + "label": "Access Token", + "read_only": 1, + "unique": 1 + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "refresh_token", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Refresh Token", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "refresh_token", + "fieldtype": "Data", + "label": "Refresh Token", + "read_only": 1 + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "expiration_time", - "fieldtype": "Datetime", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Expiration time", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "expiration_time", + "fieldtype": "Datetime", + "label": "Expiration time", + "read_only": 1 + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "expires_in", - "fieldtype": "Int", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Expires In", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "expires_in", + "fieldtype": "Int", + "label": "Expires In", + "read_only": 1 + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "status", - "fieldtype": "Select", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 1, - "label": "Status", - "length": 0, - "no_copy": 0, - "options": "Active\nRevoked", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "fieldname": "status", + "fieldtype": "Select", + "in_standard_filter": 1, + "label": "Status", + "options": "Active\nRevoked", + "read_only": 1 } - ], - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2017-03-08 14:40:04.209039", - "modified_by": "Administrator", - "module": "Integrations", - "name": "OAuth Bearer Token", - "name_case": "", - "owner": "Administrator", + ], + "links": [], + "modified": "2021-04-26 06:40:34.922441", + "modified_by": "Administrator", + "module": "Integrations", + "name": "OAuth Bearer Token", + "owner": "Administrator", "permissions": [ { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 0, - "delete": 1, - "email": 1, - "export": 1, - "if_owner": 0, - "import": 0, - "is_custom": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "set_user_permissions": 0, - "share": 1, - "submit": 0, - "write": 0 + "delete": 1, + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1 } - ], - "quick_entry": 0, - "read_only": 0, - "read_only_onload": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1, - "track_seen": 0 + ], + "sort_field": "modified", + "sort_order": "DESC" } \ No newline at end of file diff --git a/frappe/integrations/doctype/oauth_bearer_token/oauth_bearer_token.py b/frappe/integrations/doctype/oauth_bearer_token/oauth_bearer_token.py index 09fd29075b..916d0205d2 100644 --- a/frappe/integrations/doctype/oauth_bearer_token/oauth_bearer_token.py +++ b/frappe/integrations/doctype/oauth_bearer_token/oauth_bearer_token.py @@ -2,7 +2,6 @@ # Copyright (c) 2015, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document diff --git a/frappe/integrations/doctype/oauth_bearer_token/test_oauth_bearer_token.py b/frappe/integrations/doctype/oauth_bearer_token/test_oauth_bearer_token.py index af7de360ab..6028cebcf9 100644 --- a/frappe/integrations/doctype/oauth_bearer_token/test_oauth_bearer_token.py +++ b/frappe/integrations/doctype/oauth_bearer_token/test_oauth_bearer_token.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import frappe import unittest diff --git a/frappe/integrations/doctype/oauth_client/oauth_client.py b/frappe/integrations/doctype/oauth_client/oauth_client.py index 02f5041dfb..0b449ff968 100644 --- a/frappe/integrations/doctype/oauth_client/oauth_client.py +++ b/frappe/integrations/doctype/oauth_client/oauth_client.py @@ -2,7 +2,6 @@ # Copyright (c) 2015, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe import _ from frappe.model.document import Document diff --git a/frappe/integrations/doctype/oauth_client/test_oauth_client.py b/frappe/integrations/doctype/oauth_client/test_oauth_client.py index ee119455e5..a4e50e15d8 100644 --- a/frappe/integrations/doctype/oauth_client/test_oauth_client.py +++ b/frappe/integrations/doctype/oauth_client/test_oauth_client.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import frappe import unittest diff --git a/frappe/integrations/doctype/oauth_provider_settings/oauth_provider_settings.py b/frappe/integrations/doctype/oauth_provider_settings/oauth_provider_settings.py index 2bf086e0fe..3ab5df92ac 100644 --- a/frappe/integrations/doctype/oauth_provider_settings/oauth_provider_settings.py +++ b/frappe/integrations/doctype/oauth_provider_settings/oauth_provider_settings.py @@ -2,7 +2,6 @@ # Copyright (c) 2015, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document from frappe import _ diff --git a/frappe/integrations/doctype/oauth_scope/oauth_scope.py b/frappe/integrations/doctype/oauth_scope/oauth_scope.py index a5dfe7e1ce..ae579e6b51 100644 --- a/frappe/integrations/doctype/oauth_scope/oauth_scope.py +++ b/frappe/integrations/doctype/oauth_scope/oauth_scope.py @@ -2,7 +2,6 @@ # Copyright (c) 2020, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals # import frappe from frappe.model.document import Document diff --git a/frappe/integrations/doctype/paypal_settings/paypal_settings.py b/frappe/integrations/doctype/paypal_settings/paypal_settings.py index efd1b03355..da045d2c6a 100644 --- a/frappe/integrations/doctype/paypal_settings/paypal_settings.py +++ b/frappe/integrations/doctype/paypal_settings/paypal_settings.py @@ -63,12 +63,11 @@ More Details: """ -from __future__ import unicode_literals import frappe import json import pytz from frappe import _ -from six.moves.urllib.parse import urlencode +from urllib.parse import urlencode from frappe.model.document import Document from frappe.integrations.utils import create_request_log, make_post_request, create_payment_gateway from frappe.utils import get_url, call_hook_method, cint, get_datetime diff --git a/frappe/integrations/doctype/paytm_settings/paytm_settings.py b/frappe/integrations/doctype/paytm_settings/paytm_settings.py index 616c3837d4..9f15d73f09 100644 --- a/frappe/integrations/doctype/paytm_settings/paytm_settings.py +++ b/frappe/integrations/doctype/paytm_settings/paytm_settings.py @@ -2,10 +2,9 @@ # Copyright (c) 2020, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import json import requests -from six.moves.urllib.parse import urlencode +from urllib.parse import urlencode import frappe from frappe.model.document import Document @@ -59,7 +58,7 @@ def get_paytm_params(payment_details, order_id, paytm_config): # initialize a dictionary paytm_params = dict() - + redirect_uri = get_request_site_address(True) + "/api/method/frappe.integrations.doctype.paytm_settings.paytm_settings.verify_transaction" diff --git a/frappe/integrations/doctype/paytm_settings/test_paytm_settings.py b/frappe/integrations/doctype/paytm_settings/test_paytm_settings.py index 77a16c82ae..a00ce86327 100644 --- a/frappe/integrations/doctype/paytm_settings/test_paytm_settings.py +++ b/frappe/integrations/doctype/paytm_settings/test_paytm_settings.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - # import frappe import unittest diff --git a/frappe/integrations/doctype/query_parameters/query_parameters.py b/frappe/integrations/doctype/query_parameters/query_parameters.py index bfb8eae0b6..13fb94dbe3 100644 --- a/frappe/integrations/doctype/query_parameters/query_parameters.py +++ b/frappe/integrations/doctype/query_parameters/query_parameters.py @@ -2,7 +2,6 @@ # Copyright (c) 2020, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals # import frappe from frappe.model.document import Document diff --git a/frappe/integrations/doctype/razorpay_settings/razorpay_settings.py b/frappe/integrations/doctype/razorpay_settings/razorpay_settings.py index af7686c9b0..d24e15f480 100644 --- a/frappe/integrations/doctype/razorpay_settings/razorpay_settings.py +++ b/frappe/integrations/doctype/razorpay_settings/razorpay_settings.py @@ -60,14 +60,13 @@ For razorpay payment status is Authorized """ -from __future__ import unicode_literals import frappe from frappe import _ import json import hmac import razorpay import hashlib -from six.moves.urllib.parse import urlencode +from urllib.parse import urlencode from frappe.model.document import Document from frappe.utils import get_url, call_hook_method, cint, get_timestamp from frappe.integrations.utils import (make_get_request, make_post_request, create_request_log, diff --git a/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.py b/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.py index 308d34c5c2..1346811652 100755 --- a/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.py +++ b/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2017, Frappe Technologies and contributors # For license information, please see license.txt - -from __future__ import print_function, unicode_literals import os import os.path import frappe diff --git a/frappe/integrations/doctype/s3_backup_settings/test_s3_backup_settings.py b/frappe/integrations/doctype/s3_backup_settings/test_s3_backup_settings.py index 04d90f9b44..3aecdf3489 100755 --- a/frappe/integrations/doctype/s3_backup_settings/test_s3_backup_settings.py +++ b/frappe/integrations/doctype/s3_backup_settings/test_s3_backup_settings.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2017, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import unittest class TestS3BackupSettings(unittest.TestCase): diff --git a/frappe/integrations/doctype/slack_webhook_url/slack_webhook_url.json b/frappe/integrations/doctype/slack_webhook_url/slack_webhook_url.json index c610d6286b..56a76b989b 100644 --- a/frappe/integrations/doctype/slack_webhook_url/slack_webhook_url.json +++ b/frappe/integrations/doctype/slack_webhook_url/slack_webhook_url.json @@ -1,126 +1,61 @@ { - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "autoname": "field:webhook_name", - "beta": 0, - "creation": "2018-05-22 13:20:51.450815", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", + "actions": [], + "autoname": "field:webhook_name", + "creation": "2018-05-22 13:20:51.450815", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "webhook_name", + "webhook_url", + "show_document_link" + ], "fields": [ { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "webhook_name", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Name", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "webhook_name", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Name", + "reqd": 1, + "unique": 1 + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "webhook_url", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Webhook URL", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "fieldname": "webhook_url", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Webhook URL", + "reqd": 1 + }, + { + "allow_in_quick_entry": 1, + "default": "1", + "fieldname": "show_document_link", + "fieldtype": "Check", + "label": "Show link to document" } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2018-05-22 13:25:24.621129", - "modified_by": "Administrator", - "module": "Integrations", - "name": "Slack Webhook URL", - "name_case": "", - "owner": "Administrator", + ], + "links": [], + "modified": "2021-05-12 18:24:37.810235", + "modified_by": "Administrator", + "module": "Integrations", + "name": "Slack Webhook URL", + "owner": "Administrator", "permissions": [ { - "amend": 0, - "cancel": 0, - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "set_user_permissions": 0, - "share": 1, - "submit": 0, + "create": 1, + "delete": 1, + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1, "write": 1 } - ], - "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1, - "track_seen": 0 + ], + "quick_entry": 1, + "sort_field": "modified", + "sort_order": "DESC", + "track_changes": 1 } \ No newline at end of file diff --git a/frappe/integrations/doctype/slack_webhook_url/slack_webhook_url.py b/frappe/integrations/doctype/slack_webhook_url/slack_webhook_url.py index b28a6c50e6..a970fc1f11 100644 --- a/frappe/integrations/doctype/slack_webhook_url/slack_webhook_url.py +++ b/frappe/integrations/doctype/slack_webhook_url/slack_webhook_url.py @@ -2,7 +2,6 @@ # Copyright (c) 2018, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document from frappe.utils import get_url_to_form @@ -25,22 +24,27 @@ class SlackWebhookURL(Document): def send_slack_message(webhook_url, message, reference_doctype, reference_name): - slack_url = frappe.db.get_value("Slack Webhook URL", webhook_url, "webhook_url") - doc_url = get_url_to_form(reference_doctype, reference_name) - attachments = [ - { + data = {"text": message, "attachments": []} + + slack_url, show_link = frappe.db.get_value( + "Slack Webhook URL", webhook_url, ["webhook_url", "show_document_link"] + ) + + if show_link: + doc_url = get_url_to_form(reference_doctype, reference_name) + link_to_doc = { "fallback": _("See the document at {0}").format(doc_url), "actions": [ { "type": "button", "text": _("Go to the document"), "url": doc_url, - "style": "primary" + "style": "primary", } - ] + ], } - ] - data = {"text": message, "attachments": attachments} + data["attachments"].append(link_to_doc) + r = requests.post(slack_url, data=json.dumps(data)) if not r.ok: diff --git a/frappe/integrations/doctype/slack_webhook_url/test_slack_webhook_url.py b/frappe/integrations/doctype/slack_webhook_url/test_slack_webhook_url.py index a7f9316ddd..4285c2c4bc 100644 --- a/frappe/integrations/doctype/slack_webhook_url/test_slack_webhook_url.py +++ b/frappe/integrations/doctype/slack_webhook_url/test_slack_webhook_url.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2018, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import unittest class TestSlackWebhookURL(unittest.TestCase): diff --git a/frappe/integrations/doctype/social_login_key/social_login_key.py b/frappe/integrations/doctype/social_login_key/social_login_key.py index dffb730513..4a4fcd44f4 100644 --- a/frappe/integrations/doctype/social_login_key/social_login_key.py +++ b/frappe/integrations/doctype/social_login_key/social_login_key.py @@ -2,7 +2,6 @@ # Copyright (c) 2017, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe, json from frappe import _ from frappe.model.document import Document diff --git a/frappe/integrations/doctype/social_login_key/test_social_login_key.py b/frappe/integrations/doctype/social_login_key/test_social_login_key.py index e0b99ad391..23effd6a44 100644 --- a/frappe/integrations/doctype/social_login_key/test_social_login_key.py +++ b/frappe/integrations/doctype/social_login_key/test_social_login_key.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2017, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import frappe from frappe.integrations.doctype.social_login_key.social_login_key import BaseUrlNotSetError import unittest diff --git a/frappe/integrations/doctype/social_login_keys/social_login_keys.py b/frappe/integrations/doctype/social_login_keys/social_login_keys.py index bd4cea01af..da9e21cd8e 100644 --- a/frappe/integrations/doctype/social_login_keys/social_login_keys.py +++ b/frappe/integrations/doctype/social_login_keys/social_login_keys.py @@ -1,5 +1,4 @@ # see license -from __future__ import unicode_literals from frappe.model.document import Document class SocialLoginKeys(Document): diff --git a/frappe/integrations/doctype/stripe_settings/stripe_settings.py b/frappe/integrations/doctype/stripe_settings/stripe_settings.py index 70ca6002e4..9bb9c60775 100644 --- a/frappe/integrations/doctype/stripe_settings/stripe_settings.py +++ b/frappe/integrations/doctype/stripe_settings/stripe_settings.py @@ -2,11 +2,10 @@ # Copyright (c) 2017, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document from frappe import _ -from six.moves.urllib.parse import urlencode +from urllib.parse import urlencode from frappe.utils import get_url, call_hook_method, cint, flt from frappe.integrations.utils import make_get_request, make_post_request, create_request_log, create_payment_gateway diff --git a/frappe/integrations/doctype/stripe_settings/test_stripe_settings.py b/frappe/integrations/doctype/stripe_settings/test_stripe_settings.py index 39e128192f..ba11c3c38b 100644 --- a/frappe/integrations/doctype/stripe_settings/test_stripe_settings.py +++ b/frappe/integrations/doctype/stripe_settings/test_stripe_settings.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2018, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import unittest class TestStripeSettings(unittest.TestCase): diff --git a/frappe/integrations/doctype/token_cache/test_token_cache.py b/frappe/integrations/doctype/token_cache/test_token_cache.py index 7aa069647d..2ffd57403b 100644 --- a/frappe/integrations/doctype/token_cache/test_token_cache.py +++ b/frappe/integrations/doctype/token_cache/test_token_cache.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and contributors # See license.txt -from __future__ import unicode_literals - import unittest import frappe diff --git a/frappe/integrations/doctype/token_cache/token_cache.py b/frappe/integrations/doctype/token_cache/token_cache.py index 7cac58fae0..3001d12b2b 100644 --- a/frappe/integrations/doctype/token_cache/token_cache.py +++ b/frappe/integrations/doctype/token_cache/token_cache.py @@ -2,7 +2,6 @@ # Copyright (c) 2019, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals from datetime import datetime, timedelta import frappe diff --git a/frappe/integrations/doctype/webhook/__init__.py b/frappe/integrations/doctype/webhook/__init__.py index 19233bd175..b92497f16c 100644 --- a/frappe/integrations/doctype/webhook/__init__.py +++ b/frappe/integrations/doctype/webhook/__init__.py @@ -2,8 +2,6 @@ # Copyright (c) 2017, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals - import frappe @@ -21,7 +19,7 @@ def run_webhooks(doc, method): if webhooks is None: # query webhooks webhooks_list = frappe.get_all('Webhook', - fields=["name", "`condition`", "webhook_docevent", "webhook_doctype"], + fields=["name", "`condition`", "webhook_docevent", "webhook_doctype"], filters={"enabled": True} ) diff --git a/frappe/integrations/doctype/webhook/test_webhook.py b/frappe/integrations/doctype/webhook/test_webhook.py index acf2f609e7..09ad56a190 100644 --- a/frappe/integrations/doctype/webhook/test_webhook.py +++ b/frappe/integrations/doctype/webhook/test_webhook.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2017, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import unittest import frappe @@ -86,7 +84,7 @@ class TestWebhook(unittest.TestCase): # Insert the user to db self.test_user.insert() - + self.assertTrue("User" in frappe.flags.webhooks) # only 1 hook (enabled) must be queued self.assertEqual( @@ -95,7 +93,7 @@ class TestWebhook(unittest.TestCase): ) self.assertTrue(self.test_user.email in frappe.flags.webhooks_executed) self.assertEqual( - frappe.flags.webhooks_executed.get(self.test_user.email)[0], + frappe.flags.webhooks_executed.get(self.test_user.email)[0], self.sample_webhooks[0].name ) diff --git a/frappe/integrations/doctype/webhook/webhook.py b/frappe/integrations/doctype/webhook/webhook.py index ad64d9f714..1fb2bc6743 100644 --- a/frappe/integrations/doctype/webhook/webhook.py +++ b/frappe/integrations/doctype/webhook/webhook.py @@ -2,8 +2,6 @@ # Copyright (c) 2017, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals - import base64 import datetime import hashlib @@ -12,7 +10,7 @@ import json from time import sleep import requests -from six.moves.urllib.parse import urlparse +from urllib.parse import urlparse import frappe from frappe import _ diff --git a/frappe/integrations/doctype/webhook_data/webhook_data.py b/frappe/integrations/doctype/webhook_data/webhook_data.py index b7d989410f..dbd9328482 100644 --- a/frappe/integrations/doctype/webhook_data/webhook_data.py +++ b/frappe/integrations/doctype/webhook_data/webhook_data.py @@ -2,7 +2,6 @@ # Copyright (c) 2017, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals # import frappe from frappe.model.document import Document diff --git a/frappe/integrations/doctype/webhook_header/webhook_header.py b/frappe/integrations/doctype/webhook_header/webhook_header.py index 11d3ee4085..428b287db2 100644 --- a/frappe/integrations/doctype/webhook_header/webhook_header.py +++ b/frappe/integrations/doctype/webhook_header/webhook_header.py @@ -2,7 +2,6 @@ # Copyright (c) 2017, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals # import frappe from frappe.model.document import Document diff --git a/frappe/integrations/oauth2.py b/frappe/integrations/oauth2.py index 3ebaaffcff..2b227f503d 100644 --- a/frappe/integrations/oauth2.py +++ b/frappe/integrations/oauth2.py @@ -1,195 +1,38 @@ -from __future__ import unicode_literals - -import hashlib import json -from urllib.parse import quote, urlencode, urlparse - -import jwt +from urllib.parse import quote, urlencode from oauthlib.oauth2 import FatalClientError, OAuth2Error +from oauthlib.openid.connect.core.endpoints.pre_configured import ( + Server as WebApplicationServer, +) import frappe -from frappe import _ -from frappe.oauth import OAuthWebRequestValidator, WebApplicationServer -from frappe.integrations.doctype.oauth_provider_settings.oauth_provider_settings import get_oauth_settings +from frappe.integrations.doctype.oauth_provider_settings.oauth_provider_settings import ( + get_oauth_settings, +) +from frappe.oauth import ( + OAuthWebRequestValidator, + generate_json_error_response, + get_server_url, + get_userinfo, +) + def get_oauth_server(): - if not getattr(frappe.local, 'oauth_server', None): + if not getattr(frappe.local, "oauth_server", None): oauth_validator = OAuthWebRequestValidator() frappe.local.oauth_server = WebApplicationServer(oauth_validator) return frappe.local.oauth_server + def sanitize_kwargs(param_kwargs): """Remove 'data' and 'cmd' keys, if present.""" arguments = param_kwargs - arguments.pop('data', None) - arguments.pop('cmd', None) + arguments.pop("data", None) + arguments.pop("cmd", None) return arguments -@frappe.whitelist() -def approve(*args, **kwargs): - r = frappe.request - - try: - scopes, frappe.flags.oauth_credentials = get_oauth_server().validate_authorization_request( - r.url, - r.method, - r.get_data(), - r.headers - ) - - headers, body, status = get_oauth_server().create_authorization_response( - uri=frappe.flags.oauth_credentials['redirect_uri'], - body=r.get_data(), - headers=r.headers, - scopes=scopes, - credentials=frappe.flags.oauth_credentials - ) - uri = headers.get('Location', None) - - frappe.local.response["type"] = "redirect" - frappe.local.response["location"] = uri - - except FatalClientError as e: - return e - except OAuth2Error as e: - return e - -@frappe.whitelist(allow_guest=True) -def authorize(**kwargs): - success_url = "/api/method/frappe.integrations.oauth2.approve?" + encode_params(sanitize_kwargs(kwargs)) - failure_url = frappe.form_dict["redirect_uri"] + "?error=access_denied" - - if frappe.session.user == 'Guest': - #Force login, redirect to preauth again. - frappe.local.response["type"] = "redirect" - frappe.local.response["location"] = "/login?" + encode_params({'redirect-to': frappe.request.url}) - else: - try: - r = frappe.request - scopes, frappe.flags.oauth_credentials = get_oauth_server().validate_authorization_request( - r.url, - r.method, - r.get_data(), - r.headers - ) - - skip_auth = frappe.db.get_value("OAuth Client", frappe.flags.oauth_credentials['client_id'], "skip_authorization") - unrevoked_tokens = frappe.get_all("OAuth Bearer Token", filters={"status":"Active"}) - - if skip_auth or (get_oauth_settings().skip_authorization == "Auto" and unrevoked_tokens): - frappe.local.response["type"] = "redirect" - frappe.local.response["location"] = success_url - else: - #Show Allow/Deny screen. - response_html_params = frappe._dict({ - "client_id": frappe.db.get_value("OAuth Client", kwargs['client_id'], "app_name"), - "success_url": success_url, - "failure_url": failure_url, - "details": scopes - }) - resp_html = frappe.render_template("templates/includes/oauth_confirmation.html", response_html_params) - frappe.respond_as_web_page("Confirm Access", resp_html) - except FatalClientError as e: - return e - except OAuth2Error as e: - return e - -@frappe.whitelist(allow_guest=True) -def get_token(*args, **kwargs): - #Check whether frappe server URL is set - frappe_server_url = frappe.db.get_value("Social Login Key", "frappe", "base_url") or None - if not frappe_server_url: - frappe.throw(_("Please set Base URL in Social Login Key for Frappe")) - - try: - r = frappe.request - headers, body, status = get_oauth_server().create_token_response( - r.url, - r.method, - r.form, - r.headers, - frappe.flags.oauth_credentials - ) - out = frappe._dict(json.loads(body)) - if not out.error and "openid" in out.scope: - token_user = frappe.db.get_value("OAuth Bearer Token", out.access_token, "user") - token_client = frappe.db.get_value("OAuth Bearer Token", out.access_token, "client") - client_secret = frappe.db.get_value("OAuth Client", token_client, "client_secret") - if token_user in ["Guest", "Administrator"]: - frappe.throw(_("Logged in as Guest or Administrator")) - - id_token_header = { - "typ":"jwt", - "alg":"HS256" - } - id_token = { - "aud": token_client, - "exp": int((frappe.db.get_value("OAuth Bearer Token", out.access_token, "expiration_time") - frappe.utils.datetime.datetime(1970, 1, 1)).total_seconds()), - "sub": frappe.db.get_value("User Social Login", {"parent":token_user, "provider": "frappe"}, "userid"), - "iss": frappe_server_url, - "at_hash": frappe.oauth.calculate_at_hash(out.access_token, hashlib.sha256) - } - - id_token_encoded = jwt.encode(id_token, client_secret, algorithm='HS256', headers=id_token_header) - out.update({"id_token": frappe.safe_decode(id_token_encoded)}) - - frappe.local.response = out - - except FatalClientError as e: - return e - - -@frappe.whitelist(allow_guest=True) -def revoke_token(*args, **kwargs): - r = frappe.request - headers, body, status = get_oauth_server().create_revocation_response( - r.url, - headers=r.headers, - body=r.form, - http_method=r.method - ) - - frappe.local.response['http_status_code'] = status - if status == 200: - return "success" - else: - return "bad request" - -@frappe.whitelist() -def openid_profile(*args, **kwargs): - picture = None - first_name, last_name, avatar, name = frappe.db.get_value("User", frappe.session.user, ["first_name", "last_name", "user_image", "name"]) - frappe_userid = frappe.db.get_value("User Social Login", {"parent":frappe.session.user, "provider": "frappe"}, "userid") - request_url = urlparse(frappe.request.url) - base_url = frappe.db.get_value("Social Login Key", "frappe", "base_url") or None - - if avatar: - if validate_url(avatar): - picture = avatar - elif base_url: - picture = base_url + '/' + avatar - else: - picture = request_url.scheme + "://" + request_url.netloc + avatar - - user_profile = frappe._dict({ - "sub": frappe_userid, - "name": " ".join(filter(None, [first_name, last_name])), - "given_name": first_name, - "family_name": last_name, - "email": name, - "picture": picture - }) - - frappe.local.response = user_profile - -def validate_url(url_string): - try: - result = urlparse(url_string) - return result.scheme and result.scheme in ["http", "https", "ftp", "ftps"] - except: - return False def encode_params(params): """ @@ -200,3 +43,215 @@ def encode_params(params): as a whitespace. """ return urlencode(params, quote_via=quote) + + +@frappe.whitelist() +def approve(*args, **kwargs): + r = frappe.request + + try: + ( + scopes, + frappe.flags.oauth_credentials, + ) = get_oauth_server().validate_authorization_request( + r.url, r.method, r.get_data(), r.headers + ) + + headers, body, status = get_oauth_server().create_authorization_response( + uri=frappe.flags.oauth_credentials["redirect_uri"], + body=r.get_data(), + headers=r.headers, + scopes=scopes, + credentials=frappe.flags.oauth_credentials, + ) + uri = headers.get("Location", None) + + frappe.local.response["type"] = "redirect" + frappe.local.response["location"] = uri + return + + except (FatalClientError, OAuth2Error) as e: + return generate_json_error_response(e) + + +@frappe.whitelist(allow_guest=True) +def authorize(**kwargs): + success_url = "/api/method/frappe.integrations.oauth2.approve?" + encode_params( + sanitize_kwargs(kwargs) + ) + failure_url = frappe.form_dict["redirect_uri"] + "?error=access_denied" + + if frappe.session.user == "Guest": + # Force login, redirect to preauth again. + frappe.local.response["type"] = "redirect" + frappe.local.response["location"] = "/login?" + encode_params( + {"redirect-to": frappe.request.url} + ) + else: + try: + r = frappe.request + ( + scopes, + frappe.flags.oauth_credentials, + ) = get_oauth_server().validate_authorization_request( + r.url, r.method, r.get_data(), r.headers + ) + + skip_auth = frappe.db.get_value( + "OAuth Client", + frappe.flags.oauth_credentials["client_id"], + "skip_authorization", + ) + unrevoked_tokens = frappe.get_all( + "OAuth Bearer Token", filters={"status": "Active"} + ) + + if skip_auth or ( + get_oauth_settings().skip_authorization == "Auto" and unrevoked_tokens + ): + frappe.local.response["type"] = "redirect" + frappe.local.response["location"] = success_url + else: + # Show Allow/Deny screen. + response_html_params = frappe._dict( + { + "client_id": frappe.db.get_value( + "OAuth Client", kwargs["client_id"], "app_name" + ), + "success_url": success_url, + "failure_url": failure_url, + "details": scopes, + } + ) + resp_html = frappe.render_template( + "templates/includes/oauth_confirmation.html", response_html_params + ) + frappe.respond_as_web_page("Confirm Access", resp_html) + except (FatalClientError, OAuth2Error) as e: + return generate_json_error_response(e) + + +@frappe.whitelist(allow_guest=True) +def get_token(*args, **kwargs): + try: + r = frappe.request + headers, body, status = get_oauth_server().create_token_response( + r.url, r.method, r.form, r.headers, frappe.flags.oauth_credentials + ) + body = frappe._dict(json.loads(body)) + + if body.error: + frappe.local.response = body + frappe.local.response["http_status_code"] = 400 + return + + frappe.local.response = body + return + + except (FatalClientError, OAuth2Error) as e: + return generate_json_error_response(e) + + +@frappe.whitelist(allow_guest=True) +def revoke_token(*args, **kwargs): + try: + r = frappe.request + headers, body, status = get_oauth_server().create_revocation_response( + r.url, + headers=r.headers, + body=r.form, + http_method=r.method, + ) + except (FatalClientError, OAuth2Error): + pass + + # status_code must be 200 + frappe.local.response = frappe._dict({}) + frappe.local.response["http_status_code"] = status or 200 + return + + +@frappe.whitelist() +def openid_profile(*args, **kwargs): + try: + r = frappe.request + headers, body, status = get_oauth_server().create_userinfo_response( + r.url, + headers=r.headers, + body=r.form, + ) + body = frappe._dict(json.loads(body)) + frappe.local.response = body + return + + except (FatalClientError, OAuth2Error) as e: + return generate_json_error_response(e) + + +@frappe.whitelist(allow_guest=True) +def openid_configuration(): + frappe_server_url = get_server_url() + frappe.local.response = frappe._dict( + { + "issuer": frappe_server_url, + "authorization_endpoint": f"{frappe_server_url}/api/method/frappe.integrations.oauth2.authorize", + "token_endpoint": f"{frappe_server_url}/api/method/frappe.integrations.oauth2.get_token", + "userinfo_endpoint": f"{frappe_server_url}/api/method/frappe.integrations.oauth2.openid_profile", + "revocation_endpoint": f"{frappe_server_url}/api/method/frappe.integrations.oauth2.revoke_token", + "introspection_endpoint": f"{frappe_server_url}/api/method/frappe.integrations.oauth2.introspect_token", + "response_types_supported": [ + "code", + "token", + "code id_token", + "code token id_token", + "id_token", + "id_token token", + ], + "subject_types_supported": ["public"], + "id_token_signing_alg_values_supported": ["HS256"], + } + ) + + +@frappe.whitelist(allow_guest=True) +def introspect_token(token=None, token_type_hint=None): + if token_type_hint not in ["access_token", "refresh_token"]: + token_type_hint = "access_token" + try: + bearer_token = None + if token_type_hint == "access_token": + bearer_token = frappe.get_doc("OAuth Bearer Token", {"access_token": token}) + elif token_type_hint == "refresh_token": + bearer_token = frappe.get_doc( + "OAuth Bearer Token", {"refresh_token": token} + ) + + client = frappe.get_doc("OAuth Client", bearer_token.client) + + token_response = frappe._dict( + { + "client_id": client.client_id, + "trusted_client": client.skip_authorization, + "active": bearer_token.status == "Active", + "exp": round(bearer_token.expiration_time.timestamp()), + "scope": bearer_token.scopes, + } + ) + + if "openid" in bearer_token.scopes: + sub = frappe.get_value( + "User Social Login", + {"provider": "frappe", "parent": bearer_token.user}, + "userid", + ) + + if sub: + token_response.update({"sub": sub}) + user = frappe.get_doc("User", bearer_token.user) + userinfo = get_userinfo(user) + token_response.update(userinfo) + + frappe.local.response = token_response + + except Exception: + frappe.local.response = frappe._dict({"active": False}) diff --git a/frappe/integrations/oauth2_logins.py b/frappe/integrations/oauth2_logins.py index 14a6bcc417..c38b43beb7 100644 --- a/frappe/integrations/oauth2_logins.py +++ b/frappe/integrations/oauth2_logins.py @@ -1,7 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import frappe import frappe.utils from frappe.utils.oauth import login_via_oauth2, login_via_oauth2_id_token @@ -33,7 +32,7 @@ def login_via_salesforce(code, state): @frappe.whitelist(allow_guest=True) def login_via_fairlogin(code, state): - login_via_oauth2("fairlogin", code, state, decoder=decoder_compat) + login_via_oauth2("fairlogin", code, state, decoder=decoder_compat) @frappe.whitelist(allow_guest=True) def custom(code, state): diff --git a/frappe/integrations/offsite_backup_utils.py b/frappe/integrations/offsite_backup_utils.py index 48a2c89107..7a263e9d04 100644 --- a/frappe/integrations/offsite_backup_utils.py +++ b/frappe/integrations/offsite_backup_utils.py @@ -2,7 +2,6 @@ # Copyright (c) 2019, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe import glob import os diff --git a/frappe/integrations/utils.py b/frappe/integrations/utils.py index 1af9682073..09c20568b5 100644 --- a/frappe/integrations/utils.py +++ b/frappe/integrations/utils.py @@ -2,11 +2,9 @@ # Copyright (c) 2019, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe import json,datetime -from six.moves.urllib.parse import parse_qs -from six import string_types, text_type +from urllib.parse import parse_qs from frappe.utils import get_request_session from frappe import _ @@ -50,10 +48,10 @@ def make_post_request(url, auth=None, headers=None, data=None): raise exc def create_request_log(data, integration_type, service_name, name=None, error=None): - if isinstance(data, string_types): + if isinstance(data, str): data = json.loads(data) - if isinstance(error, string_types): + if isinstance(error, str): error = json.loads(error) integration_request = frappe.get_doc({ @@ -116,4 +114,4 @@ def create_payment_gateway(gateway, settings=None, controller=None): def json_handler(obj): if isinstance(obj, (datetime.date, datetime.timedelta, datetime.datetime)): - return text_type(obj) + return str(obj) diff --git a/frappe/middlewares.py b/frappe/middlewares.py index 252be56c47..05944ec37a 100644 --- a/frappe/middlewares.py +++ b/frappe/middlewares.py @@ -1,8 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals - import frappe import os from werkzeug.exceptions import NotFound diff --git a/frappe/migrate.py b/frappe/migrate.py index 619510fe5e..d19e255639 100644 --- a/frappe/migrate.py +++ b/frappe/migrate.py @@ -1,8 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals - import json import os import sys diff --git a/frappe/model/__init__.py b/frappe/model/__init__.py index af06696621..dd93fbcc18 100644 --- a/frappe/model/__init__.py +++ b/frappe/model/__init__.py @@ -2,7 +2,6 @@ # MIT License. See license.txt # model __init__.py -from __future__ import unicode_literals import frappe data_fieldtypes = ( @@ -71,7 +70,8 @@ numeric_fieldtypes = ( data_field_options = ( 'Email', 'Name', - 'Phone' + 'Phone', + 'URL' ) default_fields = ( diff --git a/frappe/model/base_document.py b/frappe/model/base_document.py index 983511f7a4..2f5154cfd9 100644 --- a/frappe/model/base_document.py +++ b/frappe/model/base_document.py @@ -1,9 +1,5 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt - -from __future__ import unicode_literals -from six import iteritems, string_types - import frappe import datetime from frappe import _ @@ -34,8 +30,9 @@ def get_controller(doctype): from frappe.model.document import Document from frappe.utils.nestedset import NestedSet - module_name, custom = frappe.db.get_value("DocType", doctype, ("module", "custom"), cache=True) \ - or ["Core", False] + module_name, custom = frappe.db.get_value( + "DocType", doctype, ("module", "custom"), cache=True + ) or ["Core", False] if custom: if frappe.db.field_exists("DocType", "is_tree"): @@ -108,7 +105,7 @@ class BaseDocument(object): if key in d: self.set(key, d.get(key)) - for key, value in iteritems(d): + for key, value in d.items(): self.set(key, value) return self @@ -119,7 +116,7 @@ class BaseDocument(object): if "doctype" in d: self.set("doctype", d.get("doctype")) - for key, value in iteritems(d): + for key, value in d.items(): # dont_update_if_missing is a list of fieldnames, for which, you don't want to set default value if (self.get(key) is None) and (value is not None) and (key not in self.dont_update_if_missing): self.set(key, value) @@ -666,6 +663,12 @@ class BaseDocument(object): if data_field_options == "Phone": frappe.utils.validate_phone_number(data, throw=True) + if data_field_options == "URL": + if not data: + continue + + frappe.utils.validate_url(data, throw=True) + def _validate_constants(self): if frappe.flags.in_import or self.is_new() or self.flags.ignore_validate_constants: return @@ -698,7 +701,7 @@ class BaseDocument(object): type_map = frappe.db.type_map - for fieldname, value in iteritems(self.get_valid_dict()): + for fieldname, value in self.get_valid_dict().items(): df = self.meta.get_field(fieldname) if not df or df.fieldtype == 'Check': @@ -763,7 +766,7 @@ class BaseDocument(object): return for fieldname, value in self.get_valid_dict().items(): - if not value or not isinstance(value, string_types): + if not value or not isinstance(value, str): continue value = frappe.as_unicode(value) @@ -832,7 +835,7 @@ class BaseDocument(object): :param parentfield: If fieldname is in child table.""" from frappe.model.meta import get_field_precision - if parentfield and not isinstance(parentfield, string_types): + if parentfield and not isinstance(parentfield, str): parentfield = parentfield.parentfield cache_key = parentfield or "main" @@ -863,7 +866,7 @@ class BaseDocument(object): from frappe.model.meta import get_default_df df = get_default_df(fieldname) - if not currency: + if not currency and df: currency = self.get(df.get("options")) if not frappe.db.exists('Currency', currency, cache=True): currency = None @@ -979,7 +982,7 @@ def _filter(data, filters, limit=None): fval = ("not None", fval) elif fval is False: fval = ("None", fval) - elif isinstance(fval, string_types) and fval.startswith("^"): + elif isinstance(fval, str) and fval.startswith("^"): fval = ("^", fval[1:]) else: fval = ("=", fval) @@ -988,7 +991,7 @@ def _filter(data, filters, limit=None): for d in data: add = True - for f, fval in iteritems(_filters): + for f, fval in _filters.items(): if not frappe.compare(getattr(d, f, None), fval[0], fval[1]): add = False break diff --git a/frappe/model/create_new.py b/frappe/model/create_new.py index dc4fd97e4c..fba6765479 100644 --- a/frappe/model/create_new.py +++ b/frappe/model/create_new.py @@ -1,7 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals """ Create a new document with defaults set """ diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py index 1c863a1577..1acccdc142 100644 --- a/frappe/model/db_query.py +++ b/frappe/model/db_query.py @@ -1,10 +1,5 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt - -from __future__ import unicode_literals - -from six import iteritems, string_types - """build query for doclistview and return results""" import frappe.defaults @@ -54,7 +49,7 @@ class DatabaseQuery(object): filters, fields = fields, filters elif fields and isinstance(filters, list) \ - and len(filters) > 1 and isinstance(filters[0], string_types): + and len(filters) > 1 and isinstance(filters[0], str): # if `filters` is a list of strings, its probably fields filters, fields = fields, filters @@ -209,7 +204,7 @@ class DatabaseQuery(object): def parse_args(self): """Convert fields and filters from strings to list, dicts""" - if isinstance(self.fields, string_types): + if isinstance(self.fields, str): if self.fields == "*": self.fields = ["*"] else: @@ -223,13 +218,13 @@ class DatabaseQuery(object): for filter_name in ["filters", "or_filters"]: filters = getattr(self, filter_name) - if isinstance(filters, string_types): + if isinstance(filters, str): filters = json.loads(filters) if isinstance(filters, dict): fdict = filters filters = [] - for key, value in iteritems(fdict): + for key, value in fdict.items(): filters.append(make_filter_tuple(self.doctype, key, value)) setattr(self, filter_name, filters) @@ -357,7 +352,7 @@ class DatabaseQuery(object): # remove from filters to_remove = [] for each in self.filters: - if isinstance(each, string_types): + if isinstance(each, str): each = [each] for element in each: @@ -391,7 +386,7 @@ class DatabaseQuery(object): filters = [filters] for f in filters: - if isinstance(f, string_types): + if isinstance(f, str): conditions.append(f) else: conditions.append(self.prepare_filter_condition(f)) @@ -465,7 +460,7 @@ class DatabaseQuery(object): elif f.operator.lower() in ('in', 'not in'): values = f.value or '' - if isinstance(values, frappe.string_types): + if isinstance(values, str): values = values.split(",") fallback = "''" @@ -522,12 +517,12 @@ class DatabaseQuery(object): value = get_time(f.value).strftime("%H:%M:%S.%f") fallback = "'00:00:00'" - elif f.operator.lower() in ("like", "not like") or (isinstance(f.value, string_types) and + elif f.operator.lower() in ("like", "not like") or (isinstance(f.value, str) and (not df or df.fieldtype not in ["Float", "Int", "Currency", "Percent", "Check"])): value = "" if f.value==None else f.value fallback = "''" - if f.operator.lower() in ("like", "not like") and isinstance(value, string_types): + if f.operator.lower() in ("like", "not like") and isinstance(value, str): # because "like" uses backslash (\) for escaping value = value.replace("\\", "\\\\").replace("%", "%%") @@ -544,7 +539,7 @@ class DatabaseQuery(object): fallback = 0 # escape value - if isinstance(value, string_types) and not f.operator.lower() == 'between': + if isinstance(value, str) and not f.operator.lower() == 'between': value = "{0}".format(frappe.db.escape(value, percent=False)) if (self.ignore_ifnull @@ -735,7 +730,7 @@ class DatabaseQuery(object): args.order_by = "`tab{0}`.`{1}` {2}".format(self.doctype, sort_field or "modified", sort_order or "desc") # draft docs always on top - if meta.is_submittable: + if hasattr(meta, 'is_submittable') and meta.is_submittable: args.order_by = "`tab{0}`.docstatus asc, {1}".format(self.doctype, args.order_by) def validate_order_by_and_group_by(self, parameters): diff --git a/frappe/model/delete_doc.py b/frappe/model/delete_doc.py index 5fcc74a734..cc88cfa106 100644 --- a/frappe/model/delete_doc.py +++ b/frappe/model/delete_doc.py @@ -1,9 +1,7 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import os -from six import string_types, integer_types import shutil import frappe @@ -17,7 +15,7 @@ from frappe.utils.password import delete_all_passwords_for from frappe.model.naming import revert_series_if_last from frappe.utils.global_search import delete_for_document from frappe.desk.doctype.tag.tag import delete_tags_for_document -from frappe.exceptions import FileNotFoundError + doctypes_to_skip = ("Communication", "ToDo", "DocShare", "Email Unsubscribe", "Activity Log", "File", "Version", "Document Follow", "Comment" , "View Log", "Tag Link", "Notification Log", "Email Queue") @@ -35,7 +33,7 @@ def delete_doc(doctype=None, name=None, force=0, ignore_doctypes=None, for_reloa name = frappe.form_dict.get('dn') names = name - if isinstance(name, string_types) or isinstance(name, integer_types): + if isinstance(name, str) or isinstance(name, int): names = [name] for name in names or []: diff --git a/frappe/model/docfield.py b/frappe/model/docfield.py index 19b78e329d..6360c3866d 100644 --- a/frappe/model/docfield.py +++ b/frappe/model/docfield.py @@ -1,7 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals """docfield utililtes""" import frappe diff --git a/frappe/model/document.py b/frappe/model/document.py index 4169919091..61160e1f01 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -1,14 +1,11 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt - -from __future__ import unicode_literals, print_function import frappe import time from frappe import _, msgprint, is_whitelisted from frappe.utils import flt, cstr, now, get_datetime_str, file_lock, date_diff from frappe.model.base_document import BaseDocument, get_controller from frappe.model.naming import set_new_name -from six import iteritems, string_types from werkzeug.exceptions import NotFound, Forbidden import hashlib, json from frappe.model import optional_fields, table_fields @@ -18,6 +15,7 @@ from frappe.utils.global_search import update_global_search from frappe.integrations.doctype.webhook import run_webhooks from frappe.desk.form.document_follow import follow_document from frappe.core.doctype.server_script.server_script_utils import run_server_script_for_doc_event +from frappe.utils.data import get_absolute_url # once_only validation # methods @@ -53,7 +51,7 @@ def get_doc(*args, **kwargs): if isinstance(args[0], BaseDocument): # already a document return args[0] - elif isinstance(args[0], string_types): + elif isinstance(args[0], str): doctype = args[0] elif isinstance(args[0], dict): @@ -90,7 +88,7 @@ class Document(BaseDocument): self._default_new_docs = {} self.flags = frappe._dict() - if args and args[0] and isinstance(args[0], string_types): + if args and args[0] and isinstance(args[0], str): # first arugment is doctype if len(args)==1: # single @@ -437,7 +435,7 @@ class Document(BaseDocument): def get_values(): values = self.as_dict() # format values - for key, value in iteritems(values): + for key, value in values.items(): if value==None: values[key] = "" return values @@ -454,7 +452,7 @@ class Document(BaseDocument): def update_single(self, d): """Updates values for Single type Document in `tabSingles`.""" frappe.db.sql("""delete from `tabSingles` where doctype=%s""", self.doctype) - for field, value in iteritems(d): + for field, value in d.items(): if field != "doctype": frappe.db.sql("""insert into `tabSingles` (doctype, field, value) values (%s, %s, %s)""", (self.doctype, field, value)) @@ -1202,8 +1200,8 @@ class Document(BaseDocument): doc.set(fieldname, flt(doc.get(fieldname), self.precision(fieldname, doc.parentfield))) def get_url(self): - """Returns Desk URL for this document. `/app/Form/{doctype}/{name}`""" - return "/app/Form/{doctype}/{name}".format(doctype=self.doctype, name=self.name) + """Returns Desk URL for this document.""" + return get_absolute_url(self.doctype, self.name) def add_comment(self, comment_type='Comment', text=None, comment_email=None, link_doctype=None, link_name=None, comment_by=None): """Add a comment to this document. @@ -1347,6 +1345,22 @@ class Document(BaseDocument): from frappe.desk.doctype.tag.tag import DocTags return DocTags(self.doctype).get_tags(self.name).split(",")[1:] + def __repr__(self): + name = self.name or "unsaved" + doctype = self.__class__.__name__ + + docstatus = f" docstatus={self.docstatus}" if self.docstatus else "" + parent = f" parent={self.parent}" if self.parent else "" + + return f"<{doctype}: {name}{docstatus}{parent}>" + + def __str__(self): + name = self.name or "unsaved" + doctype = self.__class__.__name__ + + return f"{doctype}({name})" + + def execute_action(doctype, name, action, **kwargs): """Execute an action on a document (called by background worker)""" doc = frappe.get_doc(doctype, name) diff --git a/frappe/model/dynamic_links.py b/frappe/model/dynamic_links.py index 7404ba407e..676c86d7da 100644 --- a/frappe/model/dynamic_links.py +++ b/frappe/model/dynamic_links.py @@ -1,7 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import frappe # select doctypes that are accessed by the user (not read_only) first, so that the diff --git a/frappe/model/mapper.py b/frappe/model/mapper.py index d3014435e0..fa8858d950 100644 --- a/frappe/model/mapper.py +++ b/frappe/model/mapper.py @@ -1,12 +1,12 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt +import json -from __future__ import unicode_literals -import frappe, json +import frappe from frappe import _ -from frappe.utils import cstr from frappe.model import default_fields, table_fields -from six import string_types +from frappe.utils import cstr + @frappe.whitelist() def make_mapped_doc(method, source_name, selected_children=None, args=None): @@ -60,7 +60,7 @@ def get_mapped_doc(from_doctype, from_docname, table_maps, target_doc=None, # main if not target_doc: target_doc = frappe.new_doc(table_maps[from_doctype]["doctype"]) - elif isinstance(target_doc, string_types): + elif isinstance(target_doc, str): target_doc = frappe.get_doc(json.loads(target_doc)) if (not apply_strict_user_permissions @@ -137,10 +137,8 @@ def get_mapped_doc(from_doctype, from_docname, table_maps, target_doc=None, def map_doc(source_doc, target_doc, table_map, source_parent=None): if table_map.get("validation"): for key, condition in table_map["validation"].items(): - if condition[0]=="=": - if source_doc.get(key) != condition[1]: - frappe.throw(_("Cannot map because following condition fails: ") - + key + "=" + cstr(condition[1])) + if condition[0] == "=" and source_doc.get(key) != condition[1]: + frappe.throw(_("Cannot map because following condition fails:") + f" {key}={cstr(condition[1])}") map_fields(source_doc, target_doc, table_map, source_parent) diff --git a/frappe/model/meta.py b/frappe/model/meta.py index 7f58c28397..b67c41c990 100644 --- a/frappe/model/meta.py +++ b/frappe/model/meta.py @@ -14,10 +14,7 @@ Example: ''' - -from __future__ import unicode_literals, print_function from datetime import datetime -from six.moves import range import frappe, json, os from frappe.utils import cstr, cint, cast_fieldtype from frappe.model import default_fields, no_value_fields, optional_fields, data_fieldtypes, table_fields @@ -118,7 +115,7 @@ class Meta(Document): # non standard list object, skip continue - if (isinstance(value, (frappe.text_type, int, float, datetime, list, tuple)) + if (isinstance(value, (str, int, float, datetime, list, tuple)) or (not no_nulls and value is None)): out[key] = value diff --git a/frappe/model/naming.py b/frappe/model/naming.py index 1a3f90da37..fe136adce8 100644 --- a/frappe/model/naming.py +++ b/frappe/model/naming.py @@ -1,12 +1,10 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import frappe from frappe import _ from frappe.utils import now_datetime, cint, cstr import re -from six import string_types from frappe.model import log_types @@ -146,7 +144,7 @@ def make_autoname(key="", doctype="", doc=""): def parse_naming_series(parts, doctype='', doc=''): n = '' - if isinstance(parts, string_types): + if isinstance(parts, str): parts = parts.split('.') series_set = False today = now_datetime() @@ -177,7 +175,7 @@ def parse_naming_series(parts, doctype='', doc=''): else: part = e - if isinstance(part, string_types): + if isinstance(part, str): n += part return n @@ -199,10 +197,39 @@ def getseries(key, digits): def revert_series_if_last(key, name, doc=None): + """ + Reverts the series for particular naming series: + * key is naming series - SINV-.YYYY-.#### + * name is actual name - SINV-2021-0001 + + 1. This function split the key into two parts prefix (SINV-YYYY) & hashes (####). + 2. Use prefix to get the current index of that naming series from Series table + 3. Then revert the current index. + + *For custom naming series:* + 1. hash can exist anywhere, if it exist in hashes then it take normal flow. + 2. If hash doesn't exit in hashes, we get the hash from prefix, then update name and prefix accordingly. + + *Example:* + 1. key = SINV-.YYYY.- + * If key doesn't have hash it will add hash at the end + * prefix will be SINV-YYYY based on this will get current index from Series table. + 2. key = SINV-.####.-2021 + * now prefix = SINV-#### and hashes = 2021 (hash doesn't exist) + * will search hash in key then accordingly get prefix = SINV- + 3. key = ####.-2021 + * prefix = #### and hashes = 2021 (hash doesn't exist) + * will search hash in key then accordingly get prefix = "" + """ if ".#" in key: prefix, hashes = key.rsplit(".", 1) if "#" not in hashes: - return + # get the hash part from the key + hash = re.search("#+", key) + if not hash: + return + name = name.replace(hashes, "") + prefix = prefix.replace(hash.group(), "") else: prefix = key @@ -254,7 +281,7 @@ def append_number_if_name_exists(doctype, value, fieldname="name", separator="-" filters.update({fieldname: value}) exists = frappe.db.exists(doctype, filters) - regex = "^{value}{separator}\d+$".format(value=re.escape(value), separator=separator) + regex = "^{value}{separator}\\d+$".format(value=re.escape(value), separator=separator) if exists: last = frappe.db.sql("""SELECT `{fieldname}` FROM `tab{doctype}` diff --git a/frappe/model/rename_doc.py b/frappe/model/rename_doc.py index 2c9dc5d823..fc5b3ca9fe 100644 --- a/frappe/model/rename_doc.py +++ b/frappe/model/rename_doc.py @@ -1,8 +1,5 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt - -from __future__ import print_function, unicode_literals - import frappe from frappe import _, bold from frappe.model.dynamic_links import get_dynamic_link_map diff --git a/frappe/model/sync.py b/frappe/model/sync.py index 61983d322c..28f9deb25d 100644 --- a/frappe/model/sync.py +++ b/frappe/model/sync.py @@ -1,7 +1,5 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt - -from __future__ import unicode_literals, print_function """ Sync's doctype and docfields from txt files to database perms will get synced only if none exist diff --git a/frappe/model/utils/__init__.py b/frappe/model/utils/__init__.py index efbe46a4ab..47615182e4 100644 --- a/frappe/model/utils/__init__.py +++ b/frappe/model/utils/__init__.py @@ -1,15 +1,10 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt - -from __future__ import unicode_literals, print_function -from six.moves import range import frappe from frappe import _ from frappe.utils import cstr from frappe.build import html_to_js_template import re -from six import text_type - import io STANDARD_FIELD_CONVERSION_MAP = { diff --git a/frappe/model/utils/link_count.py b/frappe/model/utils/link_count.py index 5faa5ba44b..7562aaae45 100644 --- a/frappe/model/utils/link_count.py +++ b/frappe/model/utils/link_count.py @@ -1,10 +1,7 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals - import frappe -from six import iteritems ignore_doctypes = ("DocType", "Print Format", "Role", "Module Def", "Communication", "ToDo") @@ -39,7 +36,7 @@ def update_link_count(): link_count = frappe.cache().get_value('_link_count') if link_count: - for key, count in iteritems(link_count): + for key, count in link_count.items(): if key[0] not in ignore_doctypes: try: frappe.db.sql('update `tab{0}` set idx = idx + {1} where name=%s'.format(key[0], count), diff --git a/frappe/model/utils/rename_field.py b/frappe/model/utils/rename_field.py index 778f623092..9fe9d64041 100644 --- a/frappe/model/utils/rename_field.py +++ b/frappe/model/utils/rename_field.py @@ -1,8 +1,5 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt - -from __future__ import unicode_literals, print_function - import frappe import json from frappe.model import no_value_fields, table_fields diff --git a/frappe/model/utils/user_settings.py b/frappe/model/utils/user_settings.py index d59bda71a5..ad378ab93f 100644 --- a/frappe/model/utils/user_settings.py +++ b/frappe/model/utils/user_settings.py @@ -1,9 +1,8 @@ -from __future__ import unicode_literals + # Settings saved per user basis # such as page_limit, filters, last_view import frappe, json -from six import iteritems, string_types from frappe import safe_decode # dict for mapping the index and index type for the filters of different views @@ -36,7 +35,7 @@ def update_user_settings(doctype, user_settings, for_update=False): else: current = json.loads(get_user_settings(doctype, for_update = True)) - if isinstance(current, string_types): + if isinstance(current, str): # corrupt due to old code, remove this in a future release current = {} @@ -47,7 +46,7 @@ def update_user_settings(doctype, user_settings, for_update=False): def sync_user_settings(): '''Sync from cache to database (called asynchronously via the browser)''' - for key, data in iteritems(frappe.cache().hgetall('_user_settings')): + for key, data in frappe.cache().hgetall('_user_settings').items(): key = safe_decode(key) doctype, user = key.split('::') # WTF? frappe.db.multisql({ diff --git a/frappe/model/workflow.py b/frappe/model/workflow.py index 3e8125f9b1..fa2f557370 100644 --- a/frappe/model/workflow.py +++ b/frappe/model/workflow.py @@ -1,11 +1,9 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import frappe from frappe.utils import cint from frappe import _ -from six import string_types import json class WorkflowStateError(frappe.ValidationError): pass @@ -268,7 +266,7 @@ def print_workflow_log(messages, title, doctype, indicator): @frappe.whitelist() def get_common_transition_actions(docs, doctype): common_actions = [] - if isinstance(docs, string_types): + if isinstance(docs, str): docs = json.loads(docs) try: for (i, doc) in enumerate(docs, 1): diff --git a/frappe/modules/__init__.py b/frappe/modules/__init__.py index fef4829bb6..33411f8d74 100644 --- a/frappe/modules/__init__.py +++ b/frappe/modules/__init__.py @@ -1,2 +1,2 @@ -from __future__ import unicode_literals + from .utils import * \ No newline at end of file diff --git a/frappe/modules/export_file.py b/frappe/modules/export_file.py index 4b22c82105..ae9f11d53b 100644 --- a/frappe/modules/export_file.py +++ b/frappe/modules/export_file.py @@ -1,8 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals - import frappe, os import frappe.model from frappe.modules import scrub, get_module_path, scrub_dt_dn diff --git a/frappe/modules/import_file.py b/frappe/modules/import_file.py index 5970eae5ca..e743f0c3da 100644 --- a/frappe/modules/import_file.py +++ b/frappe/modules/import_file.py @@ -1,8 +1,5 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt - -from __future__ import unicode_literals, print_function - import frappe, os, json from frappe.modules import get_module_path, scrub_dt_dn from frappe.utils import get_datetime_str @@ -107,6 +104,15 @@ def import_doc(docdict, force=False, data_import=False, pre_process=None, doc = frappe.get_doc(docdict) + # Note on Tree DocTypes: + # The tree structure is maintained in the database via the fields "lft" and + # "rgt". They are automatically set and kept up-to-date. Importing them + # would destroy any existing tree structure. + if getattr(doc.meta, 'is_tree', None) and any([doc.lft, doc.rgt]): + print('Ignoring values of `lft` and `rgt` for {} "{}"'.format(doc.doctype, doc.name)) + doc.lft = None + doc.rgt = None + doc.run_method("before_import") doc.flags.ignore_version = ignore_version diff --git a/frappe/modules/patch_handler.py b/frappe/modules/patch_handler.py index 0ed10d1e0d..029234d5d9 100644 --- a/frappe/modules/patch_handler.py +++ b/frappe/modules/patch_handler.py @@ -1,7 +1,5 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt - -from __future__ import unicode_literals, print_function """ Execute Patch Files @@ -14,9 +12,6 @@ from __future__ import unicode_literals, print_function """ import frappe, frappe.permissions, time -# for patches -import os - class PatchError(Exception): pass def run_all(skip_failing=False): diff --git a/frappe/modules/utils.py b/frappe/modules/utils.py index 132aa1e2a5..0f3e57a5a0 100644 --- a/frappe/modules/utils.py +++ b/frappe/modules/utils.py @@ -1,7 +1,5 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt - -from __future__ import unicode_literals, print_function """ Utilities for using modules """ diff --git a/frappe/monitor.py b/frappe/monitor.py index 6802a59584..34ca7d67f7 100644 --- a/frappe/monitor.py +++ b/frappe/monitor.py @@ -2,8 +2,6 @@ # Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals - from datetime import datetime import json import traceback diff --git a/frappe/oauth.py b/frappe/oauth.py index 09af5ad809..67d346ad8a 100644 --- a/frappe/oauth.py +++ b/frappe/oauth.py @@ -1,65 +1,14 @@ -from __future__ import print_function, unicode_literals -import frappe -import pytz - -from frappe import _ -from frappe.auth import LoginManager +import base64 +import datetime +import hashlib +import re from http import cookies -from oauthlib.oauth2.rfc6749.tokens import BearerToken -from oauthlib.oauth2.rfc6749.grant_types import AuthorizationCodeGrant, ImplicitGrant, ResourceOwnerPasswordCredentialsGrant, ClientCredentialsGrant, RefreshTokenGrant -from oauthlib.oauth2 import RequestValidator -from oauthlib.oauth2.rfc6749.endpoints.authorization import AuthorizationEndpoint -from oauthlib.oauth2.rfc6749.endpoints.token import TokenEndpoint -from oauthlib.oauth2.rfc6749.endpoints.resource import ResourceEndpoint -from oauthlib.oauth2.rfc6749.endpoints.revocation import RevocationEndpoint -from oauthlib.common import Request -from six.moves.urllib.parse import unquote - -def get_url_delimiter(separator_character=" "): - return separator_character - -class WebApplicationServer(AuthorizationEndpoint, TokenEndpoint, ResourceEndpoint, - RevocationEndpoint): - - """An all-in-one endpoint featuring Authorization code grant and Bearer tokens.""" - - def __init__(self, request_validator, token_generator=None, - token_expires_in=None, refresh_token_generator=None, **kwargs): - """Construct a new web application server. - - :param request_validator: An implementation of - oauthlib.oauth2.RequestValidator. - :param token_expires_in: An int or a function to generate a token - expiration offset (in seconds) given a - oauthlib.common.Request object. - :param token_generator: A function to generate a token from a request. - :param refresh_token_generator: A function to generate a token from a - request for the refresh token. - :param kwargs: Extra parameters to pass to authorization-, - token-, resource-, and revocation-endpoint constructors. - """ - implicit_grant = ImplicitGrant(request_validator) - auth_grant = AuthorizationCodeGrant(request_validator) - refresh_grant = RefreshTokenGrant(request_validator) - resource_owner_password_credentials_grant = ResourceOwnerPasswordCredentialsGrant(request_validator) - bearer = BearerToken(request_validator, token_generator, - token_expires_in, refresh_token_generator) - AuthorizationEndpoint.__init__(self, default_response_type='code', - response_types={ - 'code': auth_grant, - 'token': implicit_grant - }, - default_token_type=bearer) - TokenEndpoint.__init__(self, default_grant_type='authorization_code', - grant_types={ - 'authorization_code': auth_grant, - 'refresh_token': refresh_grant, - 'password': resource_owner_password_credentials_grant - }, - default_token_type=bearer) - ResourceEndpoint.__init__(self, default_token='Bearer', - token_types={'Bearer': bearer}) - RevocationEndpoint.__init__(self, request_validator) +from urllib.parse import unquote, urlparse +import jwt +import pytz +from oauthlib.openid import RequestValidator +import frappe +from frappe.auth import LoginManager class OAuthWebRequestValidator(RequestValidator): @@ -67,7 +16,7 @@ class OAuthWebRequestValidator(RequestValidator): # Pre- and post-authorization. def validate_client_id(self, client_id, request, *args, **kwargs): # Simple validity check, does client exist? Not banned? - cli_id = frappe.db.get_value("OAuth Client",{ "name":client_id }) + cli_id = frappe.db.get_value("OAuth Client", {"name": client_id}) if cli_id: request.client = frappe.get_doc("OAuth Client", client_id).as_dict() return True @@ -78,7 +27,9 @@ class OAuthWebRequestValidator(RequestValidator): # Is the client allowed to use the supplied redirect_uri? i.e. has # the client previously registered this EXACT redirect uri. - redirect_uris = frappe.db.get_value("OAuth Client", client_id, 'redirect_uris').split(get_url_delimiter()) + redirect_uris = frappe.db.get_value( + "OAuth Client", client_id, "redirect_uris" + ).split(get_url_delimiter()) if redirect_uri in redirect_uris: return True @@ -89,7 +40,9 @@ class OAuthWebRequestValidator(RequestValidator): # The redirect used if none has been supplied. # Prefer your clients to pre register a redirect uri rather than # supplying one on each authorization request. - redirect_uri = frappe.db.get_value("OAuth Client", client_id, 'default_redirect_uri') + redirect_uri = frappe.db.get_value( + "OAuth Client", client_id, "default_redirect_uri" + ) return redirect_uri def validate_scopes(self, client_id, scopes, client, request, *args, **kwargs): @@ -101,19 +54,23 @@ class OAuthWebRequestValidator(RequestValidator): # Scopes a client will authorize for if none are supplied in the # authorization request. scopes = get_client_scopes(client_id) - request.scopes = scopes #Apparently this is possible. + request.scopes = scopes # Apparently this is possible. return scopes - def validate_response_type(self, client_id, response_type, client, request, *args, **kwargs): - # Clients should only be allowed to use one type of response type, the - # one associated with their one allowed grant type. - # In this case it must be "code". - allowed_response_types = [client.response_type.lower(), - "code token", "code id_token", "code token id_token", - "code+token", "code+id_token", "code+token id_token"] - - return (response_type in allowed_response_types) + def validate_response_type( + self, client_id, response_type, client, request, *args, **kwargs + ): + allowed_response_types = [ + # From OAuth Client response_type field + client.response_type.lower(), + # OIDC + "id_token", + "id_token token", + "code id_token", + "code token id_token", + ] + return response_type in allowed_response_types # Post-authorization @@ -121,38 +78,69 @@ class OAuthWebRequestValidator(RequestValidator): cookie_dict = get_cookie_dict_from_headers(request) - oac = frappe.new_doc('OAuth Authorization Code') + oac = frappe.new_doc("OAuth Authorization Code") oac.scopes = get_url_delimiter().join(request.scopes) oac.redirect_uri_bound_to_authorization_code = request.redirect_uri oac.client = client_id - oac.user = unquote(cookie_dict['user_id'].value) - oac.authorization_code = code['code'] + oac.user = unquote(cookie_dict["user_id"].value) + oac.authorization_code = code["code"] + + if request.nonce: + oac.nonce = request.nonce + + if request.code_challenge and request.code_challenge_method: + oac.code_challenge = request.code_challenge + oac.code_challenge_method = request.code_challenge_method.lower() + oac.save(ignore_permissions=True) frappe.db.commit() def authenticate_client(self, request, *args, **kwargs): - #Get ClientID in URL + # Get ClientID in URL if request.client_id: oc = frappe.get_doc("OAuth Client", request.client_id) else: - #Extract token, instantiate OAuth Bearer Token and use clientid from there. + # Extract token, instantiate OAuth Bearer Token and use clientid from there. if "refresh_token" in frappe.form_dict: - oc = frappe.get_doc("OAuth Client", frappe.db.get_value("OAuth Bearer Token", {"refresh_token": frappe.form_dict["refresh_token"]}, 'client')) + oc = frappe.get_doc( + "OAuth Client", + frappe.db.get_value( + "OAuth Bearer Token", + {"refresh_token": frappe.form_dict["refresh_token"]}, + "client", + ), + ) elif "token" in frappe.form_dict: - oc = frappe.get_doc("OAuth Client", frappe.db.get_value("OAuth Bearer Token", frappe.form_dict["token"], 'client')) + oc = frappe.get_doc( + "OAuth Client", + frappe.db.get_value( + "OAuth Bearer Token", frappe.form_dict["token"], "client" + ), + ) else: - oc = frappe.get_doc("OAuth Client", frappe.db.get_value("OAuth Bearer Token", frappe.get_request_header("Authorization").split(" ")[1], 'client')) + oc = frappe.get_doc( + "OAuth Client", + frappe.db.get_value( + "OAuth Bearer Token", + frappe.get_request_header("Authorization").split(" ")[1], + "client", + ), + ) try: request.client = request.client or oc.as_dict() except Exception as e: - print("Failed body authentication: Application %s does not exist".format(cid=request.client_id)) + return generate_json_error_response(e) cookie_dict = get_cookie_dict_from_headers(request) - user_id = unquote(cookie_dict.get('user_id').value) if 'user_id' in cookie_dict else "Guest" + user_id = ( + unquote(cookie_dict.get("user_id").value) + if "user_id" in cookie_dict + else "Guest" + ) return frappe.session.user == user_id def authenticate_client_id(self, client_id, request, *args, **kwargs): - cli_id = frappe.db.get_value('OAuth Client', client_id, 'name') + cli_id = frappe.db.get_value("OAuth Client", client_id, "name") if not cli_id: # Don't allow public (non-authenticated) clients return False @@ -164,28 +152,72 @@ class OAuthWebRequestValidator(RequestValidator): # Validate the code belongs to the client. Add associated scopes, # state and user to request.scopes and request.user. - validcodes = frappe.get_all("OAuth Authorization Code", filters={"client": client_id, "validity": "Valid"}) + validcodes = frappe.get_all( + "OAuth Authorization Code", + filters={"client": client_id, "validity": "Valid"}, + ) checkcodes = [] for vcode in validcodes: checkcodes.append(vcode["name"]) if code in checkcodes: - request.scopes = frappe.db.get_value("OAuth Authorization Code", code, 'scopes').split(get_url_delimiter()) - request.user = frappe.db.get_value("OAuth Authorization Code", code, 'user') - return True - else: - return False + request.scopes = frappe.db.get_value( + "OAuth Authorization Code", code, "scopes" + ).split(get_url_delimiter()) + request.user = frappe.db.get_value("OAuth Authorization Code", code, "user") + code_challenge_method = frappe.db.get_value( + "OAuth Authorization Code", code, "code_challenge_method" + ) + code_challenge = frappe.db.get_value( + "OAuth Authorization Code", code, "code_challenge" + ) - def confirm_redirect_uri(self, client_id, code, redirect_uri, client, *args, **kwargs): - saved_redirect_uri = frappe.db.get_value('OAuth Client', client_id, 'default_redirect_uri') + if code_challenge and not request.code_verifier: + if frappe.db.exists("OAuth Authorization Code", code): + frappe.delete_doc( + "OAuth Authorization Code", code, ignore_permissions=True + ) + frappe.db.commit() + return False + + if code_challenge_method == "s256": + m = hashlib.sha256() + m.update(bytes(request.code_verifier, "utf-8")) + code_verifier = base64.b64encode(m.digest()).decode("utf-8") + code_verifier = re.sub(r"\+", "-", code_verifier) + code_verifier = re.sub(r"\/", "_", code_verifier) + code_verifier = re.sub(r"=", "", code_verifier) + return code_challenge == code_verifier + + elif code_challenge_method == "plain": + return code_challenge == request.code_verifier + + return True + + return False + + def confirm_redirect_uri( + self, client_id, code, redirect_uri, client, *args, **kwargs + ): + saved_redirect_uri = frappe.db.get_value( + "OAuth Client", client_id, "default_redirect_uri" + ) + + redirect_uris = frappe.db.get_value("OAuth Client", client_id, "redirect_uris") + + if redirect_uris: + redirect_uris = redirect_uris.split(get_url_delimiter()) + return redirect_uri in redirect_uris return saved_redirect_uri == redirect_uri - def validate_grant_type(self, client_id, grant_type, client, request, *args, **kwargs): + def validate_grant_type( + self, client_id, grant_type, client, request, *args, **kwargs + ): # Clients should only be allowed to use one type of grant. # In this case, it must be "authorization_code" or "refresh_token" - return (grant_type in ["authorization_code", "refresh_token", "password"]) + return grant_type in ["authorization_code", "refresh_token", "password"] def save_bearer_token(self, token, request, *args, **kwargs): # Remember to associate it with request.scopes, request.user and @@ -195,19 +227,30 @@ class OAuthWebRequestValidator(RequestValidator): # access_token to now + expires_in seconds. otoken = frappe.new_doc("OAuth Bearer Token") - otoken.client = request.client['name'] + otoken.client = request.client["name"] try: - otoken.user = request.user if request.user else frappe.db.get_value("OAuth Bearer Token", {"refresh_token":request.body.get("refresh_token")}, "user") - except Exception as e: + otoken.user = ( + request.user + if request.user + else frappe.db.get_value( + "OAuth Bearer Token", + {"refresh_token": request.body.get("refresh_token")}, + "user", + ) + ) + except Exception: otoken.user = frappe.session.user + otoken.scopes = get_url_delimiter().join(request.scopes) - otoken.access_token = token['access_token'] - otoken.refresh_token = token.get('refresh_token') - otoken.expires_in = token['expires_in'] + otoken.access_token = token["access_token"] + otoken.refresh_token = token.get("refresh_token") + otoken.expires_in = token["expires_in"] otoken.save(ignore_permissions=True) frappe.db.commit() - default_redirect_uri = frappe.db.get_value("OAuth Client", request.client['name'], "default_redirect_uri") + default_redirect_uri = frappe.db.get_value( + "OAuth Client", request.client["name"], "default_redirect_uri" + ) return default_redirect_uri def invalidate_authorization_code(self, client_id, code, request, *args, **kwargs): @@ -222,24 +265,35 @@ class OAuthWebRequestValidator(RequestValidator): def validate_bearer_token(self, token, scopes, request): # Remember to check expiration and scope membership otoken = frappe.get_doc("OAuth Bearer Token", token) - token_expiration_local = otoken.expiration_time.replace(tzinfo=pytz.timezone(frappe.utils.get_time_zone())) + token_expiration_local = otoken.expiration_time.replace( + tzinfo=pytz.timezone(frappe.utils.get_time_zone()) + ) token_expiration_utc = token_expiration_local.astimezone(pytz.utc) - is_token_valid = (frappe.utils.datetime.datetime.utcnow().replace(tzinfo=pytz.utc) < token_expiration_utc) \ - and otoken.status != "Revoked" - client_scopes = frappe.db.get_value("OAuth Client", otoken.client, 'scopes').split(get_url_delimiter()) + is_token_valid = ( + frappe.utils.datetime.datetime.utcnow().replace(tzinfo=pytz.utc) + < token_expiration_utc + ) and otoken.status != "Revoked" + client_scopes = frappe.db.get_value( + "OAuth Client", otoken.client, "scopes" + ).split(get_url_delimiter()) are_scopes_valid = True for scp in scopes: - are_scopes_valid = are_scopes_valid and True if scp in client_scopes else False + are_scopes_valid = ( + are_scopes_valid and True if scp in client_scopes else False + ) return is_token_valid and are_scopes_valid # Token refresh request + def get_original_scopes(self, refresh_token, request, *args, **kwargs): # Obtain the token associated with the given refresh_token and # return its scopes, these will be passed on to the refreshed # access token if the client did not specify a scope during the # request. - obearer_token = frappe.get_doc("OAuth Bearer Token", {"refresh_token": refresh_token}) + obearer_token = frappe.get_doc( + "OAuth Bearer Token", {"refresh_token": refresh_token} + ) return obearer_token.scopes def revoke_token(self, token, token_type_hint, request, *args, **kwargs): @@ -250,36 +304,38 @@ class OAuthWebRequestValidator(RequestValidator): :param request: The HTTP Request (oauthlib.common.Request) Method is used by: - - Revocation Endpoint + - Revocation Endpoint """ - otoken = None - if token_type_hint == "access_token": - otoken = frappe.db.set_value("OAuth Bearer Token", token, 'status', 'Revoked') + frappe.db.set_value("OAuth Bearer Token", token, "status", "Revoked") elif token_type_hint == "refresh_token": - otoken = frappe.db.set_value("OAuth Bearer Token", {"refresh_token": token}, 'status', 'Revoked') + frappe.db.set_value( + "OAuth Bearer Token", {"refresh_token": token}, "status", "Revoked" + ) else: - otoken = frappe.db.set_value("OAuth Bearer Token", token, 'status', 'Revoked') + frappe.db.set_value("OAuth Bearer Token", token, "status", "Revoked") frappe.db.commit() def validate_refresh_token(self, refresh_token, client, request, *args, **kwargs): - # """Ensure the Bearer token is valid and authorized access to scopes. + """Ensure the Bearer token is valid and authorized access to scopes. - # OBS! The request.user attribute should be set to the resource owner - # associated with this refresh token. + OBS! The request.user attribute should be set to the resource owner + associated with this refresh token. - # :param refresh_token: Unicode refresh token - # :param client: Client object set by you, see authenticate_client. - # :param request: The HTTP Request (oauthlib.common.Request) - # :rtype: True or False + :param refresh_token: Unicode refresh token + :param client: Client object set by you, see authenticate_client. + :param request: The HTTP Request (oauthlib.common.Request) + :rtype: True or False - # Method is used by: - # - Authorization Code Grant (indirectly by issuing refresh tokens) - # - Resource Owner Password Credentials Grant (also indirectly) - # - Refresh Token Grant - # """ + Method is used by: + - Authorization Code Grant (indirectly by issuing refresh tokens) + - Resource Owner Password Credentials Grant (also indirectly) + - Refresh Token Grant + """ - otoken = frappe.get_doc("OAuth Bearer Token", {"refresh_token": refresh_token, "status": "Active"}) + otoken = frappe.get_doc( + "OAuth Bearer Token", {"refresh_token": refresh_token, "status": "Active"} + ) if not otoken: return False @@ -287,36 +343,84 @@ class OAuthWebRequestValidator(RequestValidator): return True # OpenID Connect - def get_id_token(self, token, token_handler, request): - """ - In the OpenID Connect workflows when an ID Token is requested this method is called. - Subclasses should implement the construction, signing and optional encryption of the - ID Token as described in the OpenID Connect spec. - In addition to the standard OAuth2 request properties, the request may also contain - these OIDC specific properties which are useful to this method: + def finalize_id_token(self, id_token, token, token_handler, request): + # Check whether frappe server URL is set + id_token_header = {"typ": "jwt", "alg": "HS256"} - - nonce, if workflow is implicit or hybrid and it was provided - - claims, if provided to the original Authorization Code request + user = frappe.get_doc( + "User", + frappe.session.user, + ) - The token parameter is a dict which may contain an ``access_token`` entry, in which - case the resulting ID Token *should* include a calculated ``at_hash`` claim. + if request.nonce: + id_token["nonce"] = request.nonce - Similarly, when the request parameter has a ``code`` property defined, the ID Token - *should* include a calculated ``c_hash`` claim. + userinfo = get_userinfo(user) - http://openid.net/specs/openid-connect-core-1_0.html (sections `3.1.3.6`_, `3.2.2.10`_, `3.3.2.11`_) + if userinfo.get("iss"): + id_token["iss"] = userinfo.get("iss") - .. _`3.1.3.6`: http://openid.net/specs/openid-connect-core-1_0.html#CodeIDToken - .. _`3.2.2.10`: http://openid.net/specs/openid-connect-core-1_0.html#ImplicitIDToken - .. _`3.3.2.11`: http://openid.net/specs/openid-connect-core-1_0.html#HybridIDToken + if "openid" in request.scopes: + id_token.update(userinfo) - :param token: A Bearer token dict - :param token_handler: the token handler (BearerToken class) - :param request: the HTTP Request (oauthlib.common.Request) - :return: The ID Token (a JWS signed JWT) - """ - # the request.scope should be used by the get_id_token() method to determine which claims to include in the resulting id_token + id_token_encoded = jwt.encode( + payload=id_token, + key=request.client.client_secret, + algorithm="HS256", + headers=id_token_header, + ) + + return frappe.safe_decode(id_token_encoded) + + def get_authorization_code_nonce(self, client_id, code, redirect_uri, request): + if frappe.get_value("OAuth Authorization Code", code, "validity") == "Valid": + return frappe.get_value("OAuth Authorization Code", code, "nonce") + + return None + + def get_authorization_code_scopes(self, client_id, code, redirect_uri, request): + scope = frappe.get_value("OAuth Client", client_id, "scopes") + if not scope: + scope = [] + else: + scope = scope.split(get_url_delimiter()) + + return scope + + def get_jwt_bearer_token(self, token, token_handler, request): + now = datetime.datetime.now() + id_token = dict( + aud=token.client_id, + iat=round(now.timestamp()), + at_hash=calculate_at_hash(token.access_token, hashlib.sha256), + ) + return self.finalize_id_token(id_token, token, token_handler, request) + + def get_userinfo_claims(self, request): + user = frappe.get_doc("User", frappe.session.user) + userinfo = get_userinfo(user) + return userinfo + + def validate_id_token(self, token, scopes, request): + try: + id_token = frappe.get_doc("OAuth Bearer Token", token) + if id_token.status == "Active": + return True + except Exception: + return False + + return False + + def validate_jwt_bearer_token(self, token, scopes, request): + try: + jwt = frappe.get_doc("OAuth Bearer Token", token) + if jwt.status == "Active": + return True + except Exception: + return False + + return False def validate_silent_authorization(self, request): """Ensure the logged in user has authorized silent OpenID authorization. @@ -328,9 +432,9 @@ class OAuthWebRequestValidator(RequestValidator): :rtype: True or False Method is used by: - - OpenIDConnectAuthCode - - OpenIDConnectImplicit - - OpenIDConnectHybrid + - OpenIDConnectAuthCode + - OpenIDConnectImplicit + - OpenIDConnectHybrid """ if request.prompt == "login": False @@ -351,9 +455,9 @@ class OAuthWebRequestValidator(RequestValidator): :rtype: True or False Method is used by: - - OpenIDConnectAuthCode - - OpenIDConnectImplicit - - OpenIDConnectHybrid + - OpenIDConnectAuthCode + - OpenIDConnectImplicit + - OpenIDConnectHybrid """ if frappe.session.user == "Guest" or request.prompt.lower() == "login": return False @@ -373,32 +477,78 @@ class OAuthWebRequestValidator(RequestValidator): :rtype: True or False Method is used by: - - OpenIDConnectAuthCode - - OpenIDConnectImplicit - - OpenIDConnectHybrid + - OpenIDConnectAuthCode + - OpenIDConnectImplicit + - OpenIDConnectHybrid """ - if id_token_hint and id_token_hint == frappe.db.get_value("User Social Login", {"parent":frappe.session.user, "provider": "frappe"}, "userid"): + if id_token_hint: + try: + user = None + payload = jwt.decode( + id_token_hint, + algorithms=["HS256"], + options={ + "verify_signature": False, + "verify_aud": False, + }, + ) + client_id, client_secret = frappe.get_value( + "OAuth Client", + payload.get("aud"), + ["client_id", "client_secret"], + ) + + if payload.get("sub") and client_id and client_secret: + user = frappe.db.get_value( + "User Social Login", + {"userid": payload.get("sub"), "provider": "frappe"}, + "parent", + ) + user = frappe.get_doc("User", user) + verified_payload = jwt.decode( + id_token_hint, + key=client_secret, + audience=client_id, + algorithms=["HS256"], + options={ + "verify_exp": False, + }, + ) + + if verified_payload: + return user.name == frappe.session.user + + except Exception: + return False + + elif frappe.session.user != "Guest": return True - else: - return False + + return False def validate_user(self, username, password, client, request, *args, **kwargs): """Ensure the username and password is valid. - Method is used by: - - Resource Owner Password Credentials Grant - """ + Method is used by: + - Resource Owner Password Credentials Grant + """ login_manager = LoginManager() login_manager.authenticate(username, password) + + if login_manager.user == "Guest": + return False + request.user = login_manager.user return True + def get_cookie_dict_from_headers(r): cookie = cookies.BaseCookie() - if r.headers.get('Cookie'): - cookie.load(r.headers.get('Cookie')) + if r.headers.get("Cookie"): + cookie.load(r.headers.get("Cookie")) return cookie + def calculate_at_hash(access_token, hash_alg): """Helper method for calculating an access token hash, as described in http://openid.net/specs/openid-connect-core-1_0.html#CodeIDToken @@ -409,21 +559,25 @@ def calculate_at_hash(access_token, hash_alg): then take the left-most 128 bits and base64url encode them. The at_hash value is a case sensitive string. Args: - access_token (str): An access token string. - hash_alg (callable): A callable returning a hash object, e.g. hashlib.sha256 + access_token (str): An access token string. + hash_alg (callable): A callable returning a hash object, e.g. hashlib.sha256 """ - hash_digest = hash_alg(access_token.encode('utf-8')).digest() + hash_digest = hash_alg(access_token.encode("utf-8")).digest() cut_at = int(len(hash_digest) / 2) truncated = hash_digest[:cut_at] from jwt.utils import base64url_encode + at_hash = base64url_encode(truncated) - return at_hash.decode('utf-8') + return at_hash.decode("utf-8") + def delete_oauth2_data(): # Delete Invalid Authorization Code and Revoked Token commit_code, commit_token = False, False - code_list = frappe.get_all("OAuth Authorization Code", filters={"validity":"Invalid"}) - token_list = frappe.get_all("OAuth Bearer Token", filters={"status":"Revoked"}) + code_list = frappe.get_all( + "OAuth Authorization Code", filters={"validity": "Invalid"} + ) + token_list = frappe.get_all("OAuth Bearer Token", filters={"status": "Revoked"}) if len(code_list) > 0: commit_code = True if len(token_list) > 0: @@ -439,3 +593,59 @@ def delete_oauth2_data(): def get_client_scopes(client_id): scopes_string = frappe.db.get_value("OAuth Client", client_id, "scopes") return scopes_string.split() + + +def get_userinfo(user): + picture = None + frappe_server_url = get_server_url() + valid_url_schemes = ("http", "https", "ftp", "ftps") + + if user.user_image: + if frappe.utils.validate_url(user.user_image, valid_schemes=valid_url_schemes): + picture = user.user_image + else: + picture = frappe_server_url + "/" + user.user_image + + userinfo = frappe._dict( + { + "sub": frappe.db.get_value( + "User Social Login", + {"parent": user.name, "provider": "frappe"}, + "userid", + ), + "name": " ".join(filter(None, [user.first_name, user.last_name])), + "given_name": user.first_name, + "family_name": user.last_name, + "email": user.email, + "picture": picture, + "roles": frappe.get_roles(user.name), + "iss": frappe_server_url, + } + ) + + return userinfo + + +def get_url_delimiter(separator_character=" "): + return separator_character + + +def generate_json_error_response(e): + if not e: + e = frappe._dict({}) + + frappe.local.response = frappe._dict( + { + "description": getattr(e, "description", "Internal Server Error"), + "status_code": getattr(e, "status_code", 500), + "error": getattr(e, "error", "internal_server_error"), + } + ) + frappe.local.response["http_status_code"] = getattr(e, "status_code", 500) + return + + +def get_server_url(): + request_url = urlparse(frappe.request.url) + request_url = f"{request_url.scheme}://{request_url.netloc}" + return frappe.get_value("Social Login Key", "frappe", "base_url") or request_url diff --git a/frappe/parallel_test_runner.py b/frappe/parallel_test_runner.py new file mode 100644 index 0000000000..2f83b88572 --- /dev/null +++ b/frappe/parallel_test_runner.py @@ -0,0 +1,299 @@ +import json +import os +import re +import sys +import time +import unittest +import click +import frappe +import requests + +from .test_runner import (SLOW_TEST_THRESHOLD, make_test_records, set_test_email_config) + +click_ctx = click.get_current_context(True) +if click_ctx: + click_ctx.color = True + +class ParallelTestRunner(): + def __init__(self, app, site, build_number=1, total_builds=1, with_coverage=False): + self.app = app + self.site = site + self.with_coverage = with_coverage + self.build_number = frappe.utils.cint(build_number) or 1 + self.total_builds = frappe.utils.cint(total_builds) + self.setup_test_site() + self.run_tests() + + def setup_test_site(self): + frappe.init(site=self.site) + if not frappe.db: + frappe.connect() + + frappe.flags.in_test = True + frappe.clear_cache() + frappe.utils.scheduler.disable_scheduler() + set_test_email_config() + self.before_test_setup() + + def before_test_setup(self): + start_time = time.time() + for fn in frappe.get_hooks("before_tests", app_name=self.app): + frappe.get_attr(fn)() + + test_module = frappe.get_module(f'{self.app}.tests') + + if hasattr(test_module, "global_test_dependencies"): + for doctype in test_module.global_test_dependencies: + make_test_records(doctype) + + elapsed = time.time() - start_time + elapsed = click.style(f' ({elapsed:.03}s)', fg='red') + click.echo(f'Before Test {elapsed}') + + def run_tests(self): + self.test_result = ParallelTestResult(stream=sys.stderr, descriptions=True, verbosity=2) + + self.start_coverage() + + for test_file_info in self.get_test_file_list(): + self.run_tests_for_file(test_file_info) + + self.save_coverage() + self.print_result() + + def run_tests_for_file(self, file_info): + if not file_info: return + + frappe.set_user('Administrator') + path, filename = file_info + module = self.get_module(path, filename) + self.create_test_dependency_records(module, path, filename) + test_suite = unittest.TestSuite() + module_test_cases = unittest.TestLoader().loadTestsFromModule(module) + test_suite.addTest(module_test_cases) + test_suite(self.test_result) + + def create_test_dependency_records(self, module, path, filename): + if hasattr(module, "test_dependencies"): + for doctype in module.test_dependencies: + make_test_records(doctype) + + if os.path.basename(os.path.dirname(path)) == "doctype": + # test_data_migration_connector.py > data_migration_connector.json + test_record_filename = re.sub('^test_', '', filename).replace(".py", ".json") + test_record_file_path = os.path.join(path, test_record_filename) + if os.path.exists(test_record_file_path): + with open(test_record_file_path, 'r') as f: + doc = json.loads(f.read()) + doctype = doc["name"] + make_test_records(doctype) + + def get_module(self, path, filename): + app_path = frappe.get_pymodule_path(self.app) + relative_path = os.path.relpath(path, app_path) + if relative_path == '.': + module_name = self.app + else: + relative_path = relative_path.replace('/', '.') + module_name = os.path.splitext(filename)[0] + module_name = f'{self.app}.{relative_path}.{module_name}' + + return frappe.get_module(module_name) + + def print_result(self): + self.test_result.printErrors() + click.echo(self.test_result) + if self.test_result.failures or self.test_result.errors: + if os.environ.get('CI'): + sys.exit(1) + + def start_coverage(self): + if self.with_coverage: + from coverage import Coverage + from frappe.utils import get_bench_path + + # Generate coverage report only for app that is being tested + source_path = os.path.join(get_bench_path(), 'apps', self.app) + incl = [ + '*.py', + ] + omit = [ + '*.js', + '*.xml', + '*.pyc', + '*.css', + '*.less', + '*.scss', + '*.vue', + '*.pyc', + '*.html', + '*/test_*', + '*/node_modules/*', + '*/doctype/*/*_dashboard.py', + '*/patches/*', + ] + + if self.app == 'frappe': + omit.append('*/tests/*') + omit.append('*/commands/*') + + self.coverage = Coverage(source=[source_path], omit=omit, include=incl) + self.coverage.start() + + def save_coverage(self): + if not self.with_coverage: + return + self.coverage.stop() + self.coverage.save() + + def get_test_file_list(self): + test_list = get_all_tests(self.app) + split_size = frappe.utils.ceil(len(test_list) / self.total_builds) + # [1,2,3,4,5,6] to [[1,2], [3,4], [4,6]] if split_size is 2 + test_chunks = [test_list[x:x+split_size] for x in range(0, len(test_list), split_size)] + return test_chunks[self.build_number - 1] + + +class ParallelTestResult(unittest.TextTestResult): + def startTest(self, test): + self._started_at = time.time() + super(unittest.TextTestResult, self).startTest(test) + test_class = unittest.util.strclass(test.__class__) + if not hasattr(self, 'current_test_class') or self.current_test_class != test_class: + click.echo(f"\n{unittest.util.strclass(test.__class__)}") + self.current_test_class = test_class + + def getTestMethodName(self, test): + return test._testMethodName if hasattr(test, '_testMethodName') else str(test) + + def addSuccess(self, test): + super(unittest.TextTestResult, self).addSuccess(test) + elapsed = time.time() - self._started_at + threshold_passed = elapsed >= SLOW_TEST_THRESHOLD + elapsed = click.style(f' ({elapsed:.03}s)', fg='red') if threshold_passed else '' + click.echo(f" {click.style(' ✔ ', fg='green')} {self.getTestMethodName(test)}{elapsed}") + + def addError(self, test, err): + super(unittest.TextTestResult, self).addError(test, err) + click.echo(f" {click.style(' ✖ ', fg='red')} {self.getTestMethodName(test)}") + + def addFailure(self, test, err): + super(unittest.TextTestResult, self).addFailure(test, err) + click.echo(f" {click.style(' ✖ ', fg='red')} {self.getTestMethodName(test)}") + + def addSkip(self, test, reason): + super(unittest.TextTestResult, self).addSkip(test, reason) + click.echo(f" {click.style(' = ', fg='white')} {self.getTestMethodName(test)}") + + def addExpectedFailure(self, test, err): + super(unittest.TextTestResult, self).addExpectedFailure(test, err) + click.echo(f" {click.style(' ✖ ', fg='red')} {self.getTestMethodName(test)}") + + def addUnexpectedSuccess(self, test): + super(unittest.TextTestResult, self).addUnexpectedSuccess(test) + click.echo(f" {click.style(' ✔ ', fg='green')} {self.getTestMethodName(test)}") + + def printErrors(self): + click.echo('\n') + self.printErrorList(' ERROR ', self.errors, 'red') + self.printErrorList(' FAIL ', self.failures, 'red') + + def printErrorList(self, flavour, errors, color): + for test, err in errors: + click.echo(self.separator1) + click.echo(f"{click.style(flavour, bg=color)} {self.getDescription(test)}") + click.echo(self.separator2) + click.echo(err) + + def __str__(self): + return f"Tests: {self.testsRun}, Failing: {len(self.failures)}, Errors: {len(self.errors)}" + +def get_all_tests(app): + test_file_list = [] + for path, folders, files in os.walk(frappe.get_pymodule_path(app)): + for dontwalk in ('locals', '.git', 'public', '__pycache__'): + if dontwalk in folders: + folders.remove(dontwalk) + + # for predictability + folders.sort() + files.sort() + + if os.path.sep.join(["doctype", "doctype", "boilerplate"]) in path: + # in /doctype/doctype/boilerplate/ + continue + + for filename in files: + if filename.startswith("test_") and filename.endswith(".py") \ + and filename != 'test_runner.py': + test_file_list.append([path, filename]) + + return test_file_list + + +class ParallelTestWithOrchestrator(ParallelTestRunner): + ''' + This can be used to balance-out test time across multiple instances + This is dependent on external orchestrator which returns next test to run + + orchestrator endpoints + - register-instance (, , test_spec_list) + - get-next-test-spec (, ) + - test-completed (, ) + ''' + def __init__(self, app, site, with_coverage=False): + self.orchestrator_url = os.environ.get('ORCHESTRATOR_URL') + if not self.orchestrator_url: + click.echo('ORCHESTRATOR_URL environment variable not found!') + click.echo('Pass public URL after hosting https://github.com/frappe/test-orchestrator') + sys.exit(1) + + self.ci_build_id = os.environ.get('CI_BUILD_ID') + self.ci_instance_id = os.environ.get('CI_INSTANCE_ID') or frappe.generate_hash(length=10) + if not self.ci_build_id: + click.echo('CI_BUILD_ID environment variable not found!') + sys.exit(1) + + ParallelTestRunner.__init__(self, app, site, with_coverage=with_coverage) + + def run_tests(self): + self.test_status = 'ongoing' + self.register_instance() + super().run_tests() + + def get_test_file_list(self): + while self.test_status == 'ongoing': + yield self.get_next_test() + + def register_instance(self): + test_spec_list = get_all_tests(self.app) + response_data = self.call_orchestrator('register-instance', data={ + 'test_spec_list': test_spec_list + }) + self.is_master = response_data.get('is_master') + + def get_next_test(self): + response_data = self.call_orchestrator('get-next-test-spec') + self.test_status = response_data.get('status') + return response_data.get('next_test') + + def print_result(self): + self.call_orchestrator('test-completed') + return super().print_result() + + def call_orchestrator(self, endpoint, data={}): + # add repo token header + # build id in header + headers = { + 'CI-BUILD-ID': self.ci_build_id, + 'CI-INSTANCE-ID': self.ci_instance_id, + 'REPO-TOKEN': '2948288382838DE' + } + url = f'{self.orchestrator_url}/{endpoint}' + res = requests.get(url, json=data, headers=headers) + res.raise_for_status() + response_data = {} + if 'application/json' in res.headers.get('content-type'): + response_data = res.json() + + return response_data diff --git a/frappe/patches.txt b/frappe/patches.txt index 516ddb6094..7605d8ea2b 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -1,11 +1,5 @@ frappe.patches.v12_0.remove_deprecated_fields_from_doctype #3 -execute:frappe.db.sql("""update `tabPatch Log` set patch=replace(patch, '.4_0.', '.v4_0.')""") #2014-05-12 -frappe.patches.v5_0.convert_to_barracuda_and_utf8mb4 execute:frappe.utils.global_search.setup_global_search_table() -frappe.patches.v8_0.update_global_search_table -frappe.patches.v7_0.update_auth -frappe.patches.v8_0.drop_in_dialog #2017-09-22 -frappe.patches.v7_2.remove_in_filter execute:frappe.reload_doc('core', 'doctype', 'doctype_action', force=True) #2019-09-23 execute:frappe.reload_doc('core', 'doctype', 'doctype_link', force=True) #2020-10-17 execute:frappe.reload_doc('core', 'doctype', 'doctype', force=True) #2017-09-22 @@ -14,7 +8,6 @@ frappe.patches.v11_0.drop_column_apply_user_permissions execute:frappe.reload_doc('core', 'doctype', 'custom_docperm') execute:frappe.reload_doc('core', 'doctype', 'docperm') #2018-05-29 execute:frappe.reload_doc('core', 'doctype', 'comment') -frappe.patches.v8_0.drop_is_custom_from_docperm execute:frappe.reload_doc('core', 'doctype', 'document_naming_rule', force=True) execute:frappe.reload_doc('core', 'doctype', 'module_def') #2020-08-28 execute:frappe.reload_doc('core', 'doctype', 'version') #2017-04-01 @@ -25,190 +18,40 @@ execute:frappe.reload_doc('core', 'doctype', 'communication') #2019-10-02 execute:frappe.reload_doc('core', 'doctype', 'server_script') frappe.patches.v11_0.replicate_old_user_permissions frappe.patches.v11_0.reload_and_rename_view_log #2019-01-03 -frappe.patches.v7_1.rename_scheduler_log_to_error_log -frappe.patches.v6_1.rename_file_data -frappe.patches.v7_0.re_route #2016-06-27 -frappe.patches.v8_0.update_records_in_global_search #11-05-2017 -frappe.patches.v8_0.update_published_in_global_search frappe.patches.v11_0.copy_fetch_data_from_options frappe.patches.v11_0.change_email_signature_fieldtype execute:frappe.reload_doc('core', 'doctype', 'activity_log') execute:frappe.reload_doc('core', 'doctype', 'deleted_document') execute:frappe.reload_doc('core', 'doctype', 'domain_settings') frappe.patches.v13_0.rename_custom_client_script -frappe.patches.v8_0.rename_page_role_to_has_role #2017-03-16 -frappe.patches.v7_2.setup_custom_perms #2017-01-19 -frappe.patches.v8_0.set_user_permission_for_page_and_report #2017-03-20 execute:frappe.reload_doc('core', 'doctype', 'role') #2017-05-23 execute:frappe.reload_doc('core', 'doctype', 'user') #2017-10-27 -execute:frappe.reload_doc('custom', 'doctype', 'custom_field') #2015-10-19 -execute:frappe.reload_doc('core', 'doctype', 'page') #2013-13-26 execute:frappe.reload_doc('core', 'doctype', 'report_column') execute:frappe.reload_doc('core', 'doctype', 'report_filter') execute:frappe.reload_doc('core', 'doctype', 'report') #2020-08-25 -execute:frappe.reload_doc('core', 'doctype', 'translation') #2016-03-03 -execute:frappe.reload_doc('email', 'doctype', 'email_alert') #2014-07-15 -execute:frappe.reload_doc('desk', 'doctype', 'todo') #2014-12-31-1 -execute:frappe.reload_doc('custom', 'doctype', 'property_setter') #2014-12-31-1 -execute:frappe.reload_doc('core', 'doctype', 'patch_log') #2016-10-31 -execute:frappe.reload_doctype("File") # 2015-10-19 execute:frappe.reload_doc('core', 'doctype', 'error_snapshot') -execute:frappe.clear_cache() -frappe.patches.v7_1.rename_scheduler_log_to_error_log -frappe.patches.v7_1.sync_language_doctype -frappe.patches.v7_0.rename_bulk_email_to_email_queue -frappe.patches.v7_1.rename_chinese_language_codes - -execute:frappe.db.sql("alter table `tabSessions` modify `user` varchar(255), engine=InnoDB") -execute:frappe.db.sql("delete from `tabDocField` where parent='0'") -frappe.patches.v4_0.change_varchar_length -frappe.patches.v6_4.reduce_varchar_length -frappe.patches.v5_2.change_checks_to_not_null -frappe.patches.v6_9.int_float_not_null #2015-11-25 -frappe.patches.v5_0.v4_to_v5 - -frappe.patches.v5_0.remove_shopping_cart_app -frappe.patches.v4_0.webnotes_to_frappe -execute:frappe.permissions.reset_perms("Module Def") -execute:import frappe.installer;frappe.installer.make_site_dirs() #2014-02-19 -frappe.patches.v4_0.rename_profile_to_user -frappe.patches.v4_0.deprecate_control_panel -frappe.patches.v4_0.remove_old_parent -frappe.patches.v4_0.rename_sitemap_to_route -frappe.patches.v4_0.website_sitemap_hierarchy -frappe.patches.v4_0.remove_index_sitemap -frappe.patches.v4_0.set_website_route_idx -frappe.patches.v4_0.add_delete_permission -frappe.patches.v4_0.set_todo_checked_as_closed -frappe.patches.v4_0.private_backups -frappe.patches.v4_0.set_module_in_report -frappe.patches.v4_0.update_datetime -frappe.patches.v4_0.file_manager_hooks execute:frappe.get_doc("User", "Guest").save() -frappe.patches.v4_0.update_custom_field_insert_after -frappe.patches.v4_0.deprecate_link_selects -frappe.patches.v4_0.set_user_gravatar -frappe.patches.v4_0.set_user_permissions -frappe.patches.v4_0.create_custom_field_for_owner_match -frappe.patches.v4_0.enable_scheduler_in_system_settings -execute:frappe.db.sql("update tabReport set apply_user_permissions=1") #2014-06-03 -frappe.patches.v4_0.replace_deprecated_timezones -execute:import frappe.website.render; frappe.website.render.clear_cache("login"); #2014-06-10 -frappe.patches.v4_0.fix_attach_field_file_url -execute:frappe.permissions.reset_perms("User") #2015-03-24 -execute:frappe.db.sql("""delete from `tabUserRole` where ifnull(parentfield, '')='' or ifnull(`role`, '')=''""") #2014-08-18 -frappe.patches.v4_0.remove_user_owner_custom_field -execute:frappe.delete_doc("DocType", "Website Template") -execute:frappe.db.sql("""update `tabProperty Setter` set property_type='Text' where property in ('options', 'default')""") #2014-06-20 -frappe.patches.v4_1.enable_outgoing_email_settings -execute:frappe.db.sql("""update `tabSingles` set `value`=`doctype` where `field`='name'""") #2014-07-04 -frappe.patches.v4_1.enable_print_as_pdf #2014-06-17 -execute:frappe.db.sql("""update `tabDocPerm` set email=1 where parent='User' and permlevel=0 and `role`='All' and `read`=1 and apply_user_permissions=1""") #2014-07-15 -execute:frappe.db.sql("""update `tabPrint Format` set print_format_type='Client' where ifnull(print_format_type, '')=''""") #2014-07-28 -frappe.patches.v4_1.file_manager_fix -frappe.patches.v4_2.print_with_letterhead execute:frappe.delete_doc("DocType", "Control Panel", force=1) -execute:frappe.reload_doc('website', 'doctype', 'web_form') #2014-09-04 -execute:frappe.reload_doc('website', 'doctype', 'web_form_field') #2014-09-04 -frappe.patches.v4_2.refactor_website_routing -frappe.patches.v4_2.set_assign_in_doc -frappe.patches.v4_3.remove_allow_on_submit_customization -frappe.patches.v5_0.rename_table_fieldnames -frappe.patches.v5_0.communication_parent -frappe.patches.v5_0.clear_website_group_and_notifications -frappe.patches.v5_0.update_shared -execute:frappe.reload_doc("core", "doctype", "docshare") #2015-07-21 -frappe.patches.v6_19.comment_feed_communication -frappe.patches.v6_16.star_to_like -frappe.patches.v5_0.bookmarks_to_stars -frappe.patches.v5_0.style_settings_to_website_theme -frappe.patches.v5_0.rename_ref_type_fieldnames -frappe.patches.v5_0.fix_email_alert -frappe.patches.v5_0.fix_null_date_datetime -frappe.patches.v5_0.force_sync_website execute:frappe.delete_doc("DocType", "Tag") execute:frappe.db.sql("delete from `tabProperty Setter` where `property` in ('idx', '_idx')") -frappe.patches.v5_0.move_scheduler_last_event_to_system_settings execute:frappe.db.sql("update tabUser set new_password='' where ifnull(new_password, '')!=''") -frappe.patches.v5_0.fix_text_editor_file_urls -frappe.patches.v5_0.modify_session -frappe.patches.v5_0.expire_old_scheduler_logs execute:frappe.permissions.reset_perms("DocType") execute:frappe.db.sql("delete from `tabProperty Setter` where `property` = 'idx'") -frappe.patches.v6_0.communication_status_and_permission -frappe.patches.v6_0.make_task_log_folder -frappe.patches.v6_0.document_type_rename -frappe.patches.v6_0.fix_ghana_currency -frappe.patches.v6_2.ignore_user_permissions_if_missing execute:frappe.db.sql("delete from tabSessions where user is null") -frappe.patches.v6_2.rename_backup_manager execute:frappe.delete_doc("DocType", "Backup Manager") -execute:frappe.db.sql("""update `tabCommunication` set parenttype=null, parent=null, parentfield=null""") #2015-10-22 execute:frappe.permissions.reset_perms("Web Page") -frappe.patches.v6_6.user_last_active -frappe.patches.v6_6.fix_file_url -frappe.patches.v6_11.rename_field_in_email_account -frappe.patches.v7_0.create_private_file_folder -frappe.patches.v6_15.remove_property_setter_for_previous_field #2015-12-29 -frappe.patches.v6_15.set_username execute:frappe.permissions.reset_perms("Error Snapshot") -frappe.patches.v6_16.feed_doc_owner -frappe.patches.v6_21.print_settings_repeat_header_footer -frappe.patches.v6_24.set_language_as_code -frappe.patches.v6_20x.update_insert_after -frappe.patches.v6_20x.set_allow_draft_for_print -frappe.patches.v6_20x.remove_roles_from_website_user -frappe.patches.v7_0.set_user_fullname -frappe.patches.v7_0.add_communication_in_doc -frappe.patches.v7_0.update_send_after_in_bulk_email -execute:frappe.db.sql('''delete from `tabSingles` where doctype="Email Settings"''') # 2016-06-13 execute:frappe.db.sql("delete from `tabWeb Page` where ifnull(template_path, '')!=''") -frappe.patches.v7_0.rename_newsletter_list_to_email_group -frappe.patches.v7_0.set_email_group -frappe.patches.v7_1.setup_integration_services #2016-10-27 -frappe.patches.v7_1.rename_chinese_language_codes execute:frappe.core.doctype.language.language.update_language_names() # 2017-04-12 execute:frappe.db.set_value("Print Settings", "Print Settings", "add_draft_heading", 1) -frappe.patches.v7_0.cleanup_list_settings execute:frappe.db.set_default('language', '') -frappe.patches.v7_1.refactor_integration_broker -frappe.patches.v7_1.set_backup_limit -frappe.patches.v7_2.set_doctype_engine -frappe.patches.v7_2.merge_knowledge_base -frappe.patches.v7_0.update_report_builder_json -frappe.patches.v7_2.set_in_standard_filter_property #1 -frappe.patches.v8_0.drop_unwanted_indexes execute:frappe.db.sql("update tabCommunication set communication_date = creation where time(communication_date) = 0") -frappe.patches.v7_2.fix_email_queue_recipient -frappe.patches.v7_2.update_feedback_request # 2017-02-27 execute:frappe.rename_doc('Country', 'Macedonia, Republic of', 'Macedonia', ignore_if_exists=True) execute:frappe.rename_doc('Country', 'Iran, Islamic Republic of', 'Iran', ignore_if_exists=True) execute:frappe.rename_doc('Country', 'Tanzania, United Republic of', 'Tanzania', ignore_if_exists=True) execute:frappe.rename_doc('Country', 'Syrian Arab Republic', 'Syria', ignore_if_exists=True) -frappe.patches.v8_0.rename_listsettings_to_usersettings -frappe.patches.v7_2.update_communications -frappe.patches.v8_0.deprecate_integration_broker -frappe.patches.v8_0.update_gender_and_salutation -frappe.patches.v8_0.setup_email_inbox #2017-03-29 -frappe.patches.v8_0.newsletter_childtable_migrate -frappe.patches.v8_0.set_doctype_values_in_custom_role -frappe.patches.v8_0.install_new_build_system_requirements -frappe.patches.v8_0.set_currency_field_precision # 2017-05-09 execute:frappe.reload_doc('desk', 'doctype', 'notification_log') -frappe.patches.v8_0.rename_print_to_printing -frappe.patches.v7_1.disabled_print_settings_for_custom_print_format execute:frappe.db.sql('update tabReport set module="Desk" where name="ToDo"') -frappe.patches.v8_1.enable_allow_error_traceback_in_system_settings -frappe.patches.v8_1.update_format_options_in_auto_email_report -frappe.patches.v8_1.delete_custom_docperm_if_doctype_not_exists -frappe.patches.v8_5.delete_email_group_member_with_invalid_emails -frappe.patches.v8_x.update_user_permission -frappe.patches.v8_5.patch_event_colors -frappe.patches.v8_10.delete_static_web_page_from_global_search -frappe.patches.v9_1.add_sms_sender_name_as_parameters -frappe.patches.v9_1.resave_domain_settings -frappe.patches.v9_1.revert_domain_settings -frappe.patches.v9_1.move_feed_to_activity_log execute:frappe.delete_doc('Page', 'data-import-tool', ignore_missing=True) frappe.patches.v10_0.reload_countries_and_currencies # 2021-02-03 frappe.patches.v10_0.refactor_social_login_keys @@ -335,3 +178,5 @@ frappe.patches.v13_0.rename_list_view_setting_to_list_view_settings frappe.patches.v13_0.remove_twilio_settings frappe.patches.v12_0.rename_uploaded_files_with_proper_name frappe.patches.v13_0.queryreport_columns +frappe.patches.v13_0.jinja_hook +frappe.patches.v13_0.update_notification_channel_if_empty diff --git a/frappe/patches/v10_0/enable_chat_by_default_within_system_settings.py b/frappe/patches/v10_0/enable_chat_by_default_within_system_settings.py index eddca78051..24f915c512 100644 --- a/frappe/patches/v10_0/enable_chat_by_default_within_system_settings.py +++ b/frappe/patches/v10_0/enable_chat_by_default_within_system_settings.py @@ -1,4 +1,4 @@ -from __future__ import unicode_literals + import frappe def execute(): diff --git a/frappe/patches/v10_0/enhance_security.py b/frappe/patches/v10_0/enhance_security.py index 865d18dcff..4f6ca4faa1 100644 --- a/frappe/patches/v10_0/enhance_security.py +++ b/frappe/patches/v10_0/enhance_security.py @@ -1,5 +1,3 @@ -from __future__ import unicode_literals - import frappe from frappe.utils import cint diff --git a/frappe/patches/v10_0/increase_single_table_column_length.py b/frappe/patches/v10_0/increase_single_table_column_length.py index 18de0cff9e..e578d192fc 100644 --- a/frappe/patches/v10_0/increase_single_table_column_length.py +++ b/frappe/patches/v10_0/increase_single_table_column_length.py @@ -1,4 +1,4 @@ -from __future__ import unicode_literals + """ Run this after updating country_info.json and or """ diff --git a/frappe/patches/v10_0/migrate_passwords_passlib.py b/frappe/patches/v10_0/migrate_passwords_passlib.py index 22b7a86f85..d0b36efbaa 100644 --- a/frappe/patches/v10_0/migrate_passwords_passlib.py +++ b/frappe/patches/v10_0/migrate_passwords_passlib.py @@ -1,4 +1,4 @@ -from __future__ import unicode_literals + import frappe from frappe.utils.password import LegacyPassword diff --git a/frappe/patches/v10_0/modify_naming_series_table.py b/frappe/patches/v10_0/modify_naming_series_table.py index 659e247a38..ca6114eb55 100644 --- a/frappe/patches/v10_0/modify_naming_series_table.py +++ b/frappe/patches/v10_0/modify_naming_series_table.py @@ -1,5 +1,3 @@ -from __future__ import unicode_literals - ''' Modify the Integer 10 Digits Value to BigInt 20 Digit value to generate long Naming Series diff --git a/frappe/patches/v10_0/modify_smallest_currency_fraction.py b/frappe/patches/v10_0/modify_smallest_currency_fraction.py index f875d6b87d..c9ae477359 100644 --- a/frappe/patches/v10_0/modify_smallest_currency_fraction.py +++ b/frappe/patches/v10_0/modify_smallest_currency_fraction.py @@ -1,7 +1,6 @@ # Copyright (c) 2018, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import frappe def execute(): diff --git a/frappe/patches/v10_0/refactor_social_login_keys.py b/frappe/patches/v10_0/refactor_social_login_keys.py index 07737912df..a3f08939ec 100644 --- a/frappe/patches/v10_0/refactor_social_login_keys.py +++ b/frappe/patches/v10_0/refactor_social_login_keys.py @@ -1,5 +1,3 @@ -from __future__ import unicode_literals - import frappe from frappe.utils import cstr diff --git a/frappe/patches/v10_0/reload_countries_and_currencies.py b/frappe/patches/v10_0/reload_countries_and_currencies.py index f83ed9c3aa..8d019a4855 100644 --- a/frappe/patches/v10_0/reload_countries_and_currencies.py +++ b/frappe/patches/v10_0/reload_countries_and_currencies.py @@ -1,4 +1,4 @@ -from __future__ import unicode_literals + """ Run this after updating country_info.json and or """ diff --git a/frappe/patches/v10_0/remove_custom_field_for_disabled_domain.py b/frappe/patches/v10_0/remove_custom_field_for_disabled_domain.py index f27639388e..54839cfe02 100644 --- a/frappe/patches/v10_0/remove_custom_field_for_disabled_domain.py +++ b/frappe/patches/v10_0/remove_custom_field_for_disabled_domain.py @@ -1,4 +1,4 @@ -from __future__ import unicode_literals + import frappe def execute(): diff --git a/frappe/patches/v10_0/set_default_locking_time.py b/frappe/patches/v10_0/set_default_locking_time.py index 1c9797a6cc..045fa0e3fa 100644 --- a/frappe/patches/v10_0/set_default_locking_time.py +++ b/frappe/patches/v10_0/set_default_locking_time.py @@ -1,7 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import frappe def execute(): diff --git a/frappe/patches/v10_0/set_no_copy_to_workflow_state.py b/frappe/patches/v10_0/set_no_copy_to_workflow_state.py index 800d4a4d1b..eb469b8452 100644 --- a/frappe/patches/v10_0/set_no_copy_to_workflow_state.py +++ b/frappe/patches/v10_0/set_no_copy_to_workflow_state.py @@ -1,4 +1,4 @@ -from __future__ import unicode_literals + import frappe def execute(): diff --git a/frappe/patches/v11_0/change_email_signature_fieldtype.py b/frappe/patches/v11_0/change_email_signature_fieldtype.py index f6d4bd5dcb..ccfa8541c3 100644 --- a/frappe/patches/v11_0/change_email_signature_fieldtype.py +++ b/frappe/patches/v11_0/change_email_signature_fieldtype.py @@ -1,7 +1,6 @@ # Copyright (c) 2018, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import frappe def execute(): diff --git a/frappe/patches/v11_0/copy_fetch_data_from_options.py b/frappe/patches/v11_0/copy_fetch_data_from_options.py index ae7788450a..e256c7085f 100644 --- a/frappe/patches/v11_0/copy_fetch_data_from_options.py +++ b/frappe/patches/v11_0/copy_fetch_data_from_options.py @@ -1,4 +1,4 @@ -from __future__ import unicode_literals + import frappe def execute(): diff --git a/frappe/patches/v11_0/create_contact_for_user.py b/frappe/patches/v11_0/create_contact_for_user.py index b4722ab3ae..5a483b630e 100644 --- a/frappe/patches/v11_0/create_contact_for_user.py +++ b/frappe/patches/v11_0/create_contact_for_user.py @@ -1,4 +1,4 @@ -from __future__ import unicode_literals + import frappe from frappe.core.doctype.user.user import create_contact import re @@ -8,7 +8,6 @@ def execute(): frappe.reload_doc('integrations', 'doctype', 'google_contacts') frappe.reload_doc('contacts', 'doctype', 'contact') frappe.reload_doc('core', 'doctype', 'dynamic_link') - frappe.reload_doc('communication', 'doctype', 'call_log') contact_meta = frappe.get_meta("Contact") if contact_meta.has_field("phone_nos") and contact_meta.has_field("email_ids"): diff --git a/frappe/patches/v11_0/delete_all_prepared_reports.py b/frappe/patches/v11_0/delete_all_prepared_reports.py index 1d722da7e6..77f041e3ee 100644 --- a/frappe/patches/v11_0/delete_all_prepared_reports.py +++ b/frappe/patches/v11_0/delete_all_prepared_reports.py @@ -1,4 +1,4 @@ -from __future__ import unicode_literals + import frappe def execute(): diff --git a/frappe/patches/v11_0/delete_duplicate_user_permissions.py b/frappe/patches/v11_0/delete_duplicate_user_permissions.py index 9d9d516ac5..518c1f7714 100644 --- a/frappe/patches/v11_0/delete_duplicate_user_permissions.py +++ b/frappe/patches/v11_0/delete_duplicate_user_permissions.py @@ -1,4 +1,4 @@ -from __future__ import unicode_literals + import frappe def execute(): diff --git a/frappe/patches/v11_0/drop_column_apply_user_permissions.py b/frappe/patches/v11_0/drop_column_apply_user_permissions.py index 4f46bc0907..629d5a5da4 100644 --- a/frappe/patches/v11_0/drop_column_apply_user_permissions.py +++ b/frappe/patches/v11_0/drop_column_apply_user_permissions.py @@ -1,4 +1,4 @@ -from __future__ import unicode_literals + import frappe def execute(): diff --git a/frappe/patches/v11_0/fix_order_by_in_reports_json.py b/frappe/patches/v11_0/fix_order_by_in_reports_json.py index 2cd82d442d..096e0e7654 100644 --- a/frappe/patches/v11_0/fix_order_by_in_reports_json.py +++ b/frappe/patches/v11_0/fix_order_by_in_reports_json.py @@ -1,4 +1,4 @@ -from __future__ import unicode_literals + import frappe, json def execute(): diff --git a/frappe/patches/v11_0/make_all_prepared_report_attachments_private.py b/frappe/patches/v11_0/make_all_prepared_report_attachments_private.py index f7b9e476a9..a099b89b40 100644 --- a/frappe/patches/v11_0/make_all_prepared_report_attachments_private.py +++ b/frappe/patches/v11_0/make_all_prepared_report_attachments_private.py @@ -1,4 +1,4 @@ -from __future__ import unicode_literals + import frappe diff --git a/frappe/patches/v11_0/migrate_report_settings_for_new_listview.py b/frappe/patches/v11_0/migrate_report_settings_for_new_listview.py index 5bef52c295..e5b18368db 100644 --- a/frappe/patches/v11_0/migrate_report_settings_for_new_listview.py +++ b/frappe/patches/v11_0/migrate_report_settings_for_new_listview.py @@ -1,4 +1,4 @@ -from __future__ import unicode_literals + import frappe, json def execute(): diff --git a/frappe/patches/v11_0/multiple_references_in_events.py b/frappe/patches/v11_0/multiple_references_in_events.py index 57d4787eca..9fa5968d8e 100644 --- a/frappe/patches/v11_0/multiple_references_in_events.py +++ b/frappe/patches/v11_0/multiple_references_in_events.py @@ -1,4 +1,4 @@ -from __future__ import unicode_literals + import frappe def execute(): diff --git a/frappe/patches/v11_0/reload_and_rename_view_log.py b/frappe/patches/v11_0/reload_and_rename_view_log.py index 12c71b746f..fa0432c4e2 100644 --- a/frappe/patches/v11_0/reload_and_rename_view_log.py +++ b/frappe/patches/v11_0/reload_and_rename_view_log.py @@ -1,4 +1,4 @@ -from __future__ import unicode_literals + import frappe def execute(): diff --git a/frappe/patches/v11_0/remove_doctype_user_permissions_for_page_and_report.py b/frappe/patches/v11_0/remove_doctype_user_permissions_for_page_and_report.py index e2c2ef5f0e..5c54b1e5c1 100644 --- a/frappe/patches/v11_0/remove_doctype_user_permissions_for_page_and_report.py +++ b/frappe/patches/v11_0/remove_doctype_user_permissions_for_page_and_report.py @@ -1,7 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import frappe def execute(): diff --git a/frappe/patches/v11_0/remove_skip_for_doctype.py b/frappe/patches/v11_0/remove_skip_for_doctype.py index edd385e317..638a5a0fd7 100644 --- a/frappe/patches/v11_0/remove_skip_for_doctype.py +++ b/frappe/patches/v11_0/remove_skip_for_doctype.py @@ -1,4 +1,4 @@ -from __future__ import unicode_literals + import frappe from frappe.desk.form.linked_with import get_linked_doctypes from frappe.patches.v11_0.replicate_old_user_permissions import get_doctypes_to_skip diff --git a/frappe/patches/v11_0/rename_email_alert_to_notification.py b/frappe/patches/v11_0/rename_email_alert_to_notification.py index 727055fcc4..365b76ea48 100644 --- a/frappe/patches/v11_0/rename_email_alert_to_notification.py +++ b/frappe/patches/v11_0/rename_email_alert_to_notification.py @@ -1,4 +1,4 @@ -from __future__ import unicode_literals + import frappe from frappe.model.rename_doc import rename_doc diff --git a/frappe/patches/v11_0/rename_google_maps_doctype.py b/frappe/patches/v11_0/rename_google_maps_doctype.py index 5420dcfc20..8091154b9c 100644 --- a/frappe/patches/v11_0/rename_google_maps_doctype.py +++ b/frappe/patches/v11_0/rename_google_maps_doctype.py @@ -1,8 +1,7 @@ -from __future__ import unicode_literals + import frappe from frappe.model.rename_doc import rename_doc def execute(): if frappe.db.exists("DocType","Google Maps") and not frappe.db.exists("DocType","Google Maps Settings"): rename_doc('DocType', 'Google Maps', 'Google Maps Settings') - frappe.reload_doc('integrations', 'doctype', 'google_maps_settings') \ No newline at end of file diff --git a/frappe/patches/v11_0/rename_standard_reply_to_email_template.py b/frappe/patches/v11_0/rename_standard_reply_to_email_template.py index 06869530e2..2906085738 100644 --- a/frappe/patches/v11_0/rename_standard_reply_to_email_template.py +++ b/frappe/patches/v11_0/rename_standard_reply_to_email_template.py @@ -1,4 +1,4 @@ -from __future__ import unicode_literals + import frappe from frappe.model.rename_doc import rename_doc diff --git a/frappe/patches/v11_0/rename_workflow_action_to_workflow_action_master.py b/frappe/patches/v11_0/rename_workflow_action_to_workflow_action_master.py index 32f17ac2d8..9a48104611 100644 --- a/frappe/patches/v11_0/rename_workflow_action_to_workflow_action_master.py +++ b/frappe/patches/v11_0/rename_workflow_action_to_workflow_action_master.py @@ -1,4 +1,4 @@ -from __future__ import unicode_literals + import frappe from frappe.model.rename_doc import rename_doc diff --git a/frappe/patches/v11_0/replicate_old_user_permissions.py b/frappe/patches/v11_0/replicate_old_user_permissions.py index d1ceae8a7f..50a81b5ce7 100644 --- a/frappe/patches/v11_0/replicate_old_user_permissions.py +++ b/frappe/patches/v11_0/replicate_old_user_permissions.py @@ -1,4 +1,4 @@ -from __future__ import unicode_literals + import frappe import json from frappe.utils import cint diff --git a/frappe/patches/v11_0/set_allow_self_approval_in_workflow.py b/frappe/patches/v11_0/set_allow_self_approval_in_workflow.py index 24c01e1a58..63ae5f949f 100644 --- a/frappe/patches/v11_0/set_allow_self_approval_in_workflow.py +++ b/frappe/patches/v11_0/set_allow_self_approval_in_workflow.py @@ -1,4 +1,4 @@ -from __future__ import unicode_literals + import frappe def execute(): diff --git a/frappe/patches/v11_0/set_default_letter_head_source.py b/frappe/patches/v11_0/set_default_letter_head_source.py index a43ea397e4..3639524e7d 100644 --- a/frappe/patches/v11_0/set_default_letter_head_source.py +++ b/frappe/patches/v11_0/set_default_letter_head_source.py @@ -1,5 +1,3 @@ -from __future__ import unicode_literals - import frappe def execute(): diff --git a/frappe/patches/v11_0/set_dropbox_file_backup.py b/frappe/patches/v11_0/set_dropbox_file_backup.py index 884fef320e..27492b3ab2 100644 --- a/frappe/patches/v11_0/set_dropbox_file_backup.py +++ b/frappe/patches/v11_0/set_dropbox_file_backup.py @@ -1,4 +1,4 @@ -from __future__ import unicode_literals + from frappe.utils import cint import frappe diff --git a/frappe/patches/v11_0/sync_stripe_settings_before_migrate.py b/frappe/patches/v11_0/sync_stripe_settings_before_migrate.py index 331b0eba32..a8e9bd4de1 100644 --- a/frappe/patches/v11_0/sync_stripe_settings_before_migrate.py +++ b/frappe/patches/v11_0/sync_stripe_settings_before_migrate.py @@ -1,4 +1,4 @@ -from __future__ import unicode_literals + import frappe from frappe.utils.password import get_decrypted_password diff --git a/frappe/patches/v11_0/sync_user_permission_doctype_before_migrate.py b/frappe/patches/v11_0/sync_user_permission_doctype_before_migrate.py index 738fea1a48..55a7b74f7e 100644 --- a/frappe/patches/v11_0/sync_user_permission_doctype_before_migrate.py +++ b/frappe/patches/v11_0/sync_user_permission_doctype_before_migrate.py @@ -1,4 +1,4 @@ -from __future__ import unicode_literals + import frappe def execute(): diff --git a/frappe/patches/v11_0/update_list_user_settings.py b/frappe/patches/v11_0/update_list_user_settings.py index d492ff1704..1b179d8cdf 100644 --- a/frappe/patches/v11_0/update_list_user_settings.py +++ b/frappe/patches/v11_0/update_list_user_settings.py @@ -1,4 +1,4 @@ -from __future__ import unicode_literals + import frappe, json from frappe.model.utils.user_settings import update_user_settings, sync_user_settings diff --git a/frappe/patches/v12_0/create_notification_settings_for_user.py b/frappe/patches/v12_0/create_notification_settings_for_user.py index 63eeccc07a..6edfd88872 100644 --- a/frappe/patches/v12_0/create_notification_settings_for_user.py +++ b/frappe/patches/v12_0/create_notification_settings_for_user.py @@ -1,4 +1,4 @@ -from __future__ import unicode_literals + import frappe from frappe.desk.doctype.notification_settings.notification_settings import create_notification_settings diff --git a/frappe/patches/v12_0/init_desk_settings.py b/frappe/patches/v12_0/init_desk_settings.py index ecd9c94d5b..fceb44b924 100644 --- a/frappe/patches/v12_0/init_desk_settings.py +++ b/frappe/patches/v12_0/init_desk_settings.py @@ -1,5 +1,3 @@ -from __future__ import unicode_literals - import json import frappe from frappe.config import get_modules_from_all_apps_for_user diff --git a/frappe/patches/v12_0/move_timeline_links_to_dynamic_links.py b/frappe/patches/v12_0/move_timeline_links_to_dynamic_links.py index 040fde1bee..85be3f7feb 100644 --- a/frappe/patches/v12_0/move_timeline_links_to_dynamic_links.py +++ b/frappe/patches/v12_0/move_timeline_links_to_dynamic_links.py @@ -1,5 +1,3 @@ -from __future__ import unicode_literals - import frappe def execute(): diff --git a/frappe/patches/v12_0/setup_comments_from_communications.py b/frappe/patches/v12_0/setup_comments_from_communications.py index 28c7aa93c0..039ceeff35 100644 --- a/frappe/patches/v12_0/setup_comments_from_communications.py +++ b/frappe/patches/v12_0/setup_comments_from_communications.py @@ -1,5 +1,3 @@ -from __future__ import unicode_literals - import frappe def execute(): diff --git a/frappe/patches/v12_0/setup_email_linking.py b/frappe/patches/v12_0/setup_email_linking.py index 08f57ca5e4..9e939e1245 100644 --- a/frappe/patches/v12_0/setup_email_linking.py +++ b/frappe/patches/v12_0/setup_email_linking.py @@ -1,5 +1,3 @@ -from __future__ import unicode_literals - from frappe.desk.page.setup_wizard.install_fixtures import setup_email_linking def execute(): diff --git a/frappe/patches/v12_0/update_auto_repeat_status_and_not_submittable.py b/frappe/patches/v12_0/update_auto_repeat_status_and_not_submittable.py index d696b6c53a..3a3dcec315 100644 --- a/frappe/patches/v12_0/update_auto_repeat_status_and_not_submittable.py +++ b/frappe/patches/v12_0/update_auto_repeat_status_and_not_submittable.py @@ -1,4 +1,4 @@ -from __future__ import unicode_literals + import frappe from frappe.custom.doctype.custom_field.custom_field import create_custom_field diff --git a/frappe/patches/v13_0/add_standard_navbar_items.py b/frappe/patches/v13_0/add_standard_navbar_items.py index 9982e6e3f5..4473cb8c07 100644 --- a/frappe/patches/v13_0/add_standard_navbar_items.py +++ b/frappe/patches/v13_0/add_standard_navbar_items.py @@ -1,4 +1,4 @@ -from __future__ import unicode_literals + import frappe from frappe.utils.install import add_standard_navbar_items diff --git a/frappe/patches/v13_0/add_switch_theme_to_navbar_settings.py b/frappe/patches/v13_0/add_switch_theme_to_navbar_settings.py index 29b99464b5..b5542c9c8a 100644 --- a/frappe/patches/v13_0/add_switch_theme_to_navbar_settings.py +++ b/frappe/patches/v13_0/add_switch_theme_to_navbar_settings.py @@ -1,4 +1,4 @@ -from __future__ import unicode_literals + import frappe def execute(): diff --git a/frappe/patches/v13_0/add_toggle_width_in_navbar_settings.py b/frappe/patches/v13_0/add_toggle_width_in_navbar_settings.py index 59acb77480..bd3367377c 100644 --- a/frappe/patches/v13_0/add_toggle_width_in_navbar_settings.py +++ b/frappe/patches/v13_0/add_toggle_width_in_navbar_settings.py @@ -1,4 +1,4 @@ -from __future__ import unicode_literals + import frappe def execute(): diff --git a/frappe/patches/v13_0/cleanup_desk_cards.py b/frappe/patches/v13_0/cleanup_desk_cards.py index 6ac8604041..b6fab66475 100644 --- a/frappe/patches/v13_0/cleanup_desk_cards.py +++ b/frappe/patches/v13_0/cleanup_desk_cards.py @@ -1,11 +1,10 @@ import frappe -from six import string_types from json import loads from frappe.desk.doctype.workspace.workspace import get_link_type, get_report_type def execute(): frappe.reload_doc('desk', 'doctype', 'workspace') - + pages = frappe.db.sql("Select `name` from `tabDesk Page`") # pages = frappe.get_all("Workspace", filters={"is_standard": 0}, pluck="name") @@ -21,14 +20,14 @@ def rebuild_links(page): doc = frappe.get_doc("Workspace", page) except frappe.DoesNotExistError: db_doc = get_doc_from_db(page) - + doc = frappe.get_doc(db_doc) doc.insert(ignore_permissions=True) - + doc.links = [] for card in get_all_cards(page): - if isinstance(card.links, string_types): + if isinstance(card.links, str): links = loads(card.links) else: links = card.links @@ -43,7 +42,7 @@ def rebuild_links(page): for link in links: if not frappe.db.exists(get_link_type(link.get('type')), link.get('name')): continue - + doc.append('links', { "label": link.get('label') or link.get('name'), "type": "Link", @@ -53,7 +52,7 @@ def rebuild_links(page): "dependencies": ', '.join(link.get('dependencies', [])), "is_query_report": get_report_type(link.get('name')) if link.get('type').lower() == "report" else 0 }) - + try: doc.save(ignore_permissions=True) except frappe.LinkValidationError: diff --git a/frappe/patches/v13_0/delete_event_producer_and_consumer_keys.py b/frappe/patches/v13_0/delete_event_producer_and_consumer_keys.py index 1eba5871c2..776e9c796e 100644 --- a/frappe/patches/v13_0/delete_event_producer_and_consumer_keys.py +++ b/frappe/patches/v13_0/delete_event_producer_and_consumer_keys.py @@ -1,7 +1,6 @@ # Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import frappe def execute(): diff --git a/frappe/patches/v13_0/delete_package_publish_tool.py b/frappe/patches/v13_0/delete_package_publish_tool.py index 25024f58dd..bf9aaf5a76 100644 --- a/frappe/patches/v13_0/delete_package_publish_tool.py +++ b/frappe/patches/v13_0/delete_package_publish_tool.py @@ -1,7 +1,6 @@ # Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import frappe diff --git a/frappe/patches/v13_0/enable_custom_script.py b/frappe/patches/v13_0/enable_custom_script.py index edc242e700..0684074fe7 100644 --- a/frappe/patches/v13_0/enable_custom_script.py +++ b/frappe/patches/v13_0/enable_custom_script.py @@ -1,7 +1,6 @@ # Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import frappe def execute(): diff --git a/frappe/patches/v13_0/generate_theme_files_in_public_folder.py b/frappe/patches/v13_0/generate_theme_files_in_public_folder.py index bcb47bec24..dd9fb1961a 100644 --- a/frappe/patches/v13_0/generate_theme_files_in_public_folder.py +++ b/frappe/patches/v13_0/generate_theme_files_in_public_folder.py @@ -1,7 +1,6 @@ # Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import frappe diff --git a/frappe/patches/v13_0/jinja_hook.py b/frappe/patches/v13_0/jinja_hook.py new file mode 100644 index 0000000000..990ae50f35 --- /dev/null +++ b/frappe/patches/v13_0/jinja_hook.py @@ -0,0 +1,12 @@ +# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors +# MIT License. See license.txt + +import frappe +from click import secho + +def execute(): + if frappe.get_hooks('jenv'): + print() + secho('WARNING: The hook "jenv" is deprecated. Follow the migration guide to use the new "jinja" hook.', fg='yellow') + secho('https://github.com/frappe/frappe/wiki/Migrating-to-Version-13', fg='yellow') + print() diff --git a/frappe/patches/v13_0/queryreport_columns.py b/frappe/patches/v13_0/queryreport_columns.py index 6c2a1b1219..5c381f4f3e 100644 --- a/frappe/patches/v13_0/queryreport_columns.py +++ b/frappe/patches/v13_0/queryreport_columns.py @@ -1,7 +1,6 @@ # Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import frappe import json diff --git a/frappe/patches/v13_0/remove_duplicate_navbar_items.py b/frappe/patches/v13_0/remove_duplicate_navbar_items.py index cb4de4ca07..b6c6033f64 100644 --- a/frappe/patches/v13_0/remove_duplicate_navbar_items.py +++ b/frappe/patches/v13_0/remove_duplicate_navbar_items.py @@ -1,4 +1,4 @@ -from __future__ import unicode_literals + import frappe def execute(): diff --git a/frappe/patches/v13_0/remove_tailwind_from_page_builder.py b/frappe/patches/v13_0/remove_tailwind_from_page_builder.py index 6e7bf67bac..2bf2c7bf87 100644 --- a/frappe/patches/v13_0/remove_tailwind_from_page_builder.py +++ b/frappe/patches/v13_0/remove_tailwind_from_page_builder.py @@ -1,7 +1,6 @@ # Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import frappe diff --git a/frappe/patches/v13_0/rename_list_view_setting_to_list_view_settings.py b/frappe/patches/v13_0/rename_list_view_setting_to_list_view_settings.py index 7c3aec9510..3122de8bea 100644 --- a/frappe/patches/v13_0/rename_list_view_setting_to_list_view_settings.py +++ b/frappe/patches/v13_0/rename_list_view_setting_to_list_view_settings.py @@ -1,7 +1,6 @@ # Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import frappe diff --git a/frappe/patches/v13_0/rename_notification_fields.py b/frappe/patches/v13_0/rename_notification_fields.py index 2984e6503c..1413d80358 100644 --- a/frappe/patches/v13_0/rename_notification_fields.py +++ b/frappe/patches/v13_0/rename_notification_fields.py @@ -1,7 +1,6 @@ # Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import frappe from frappe.model.utils.rename_field import rename_field diff --git a/frappe/patches/v13_0/rename_onboarding.py b/frappe/patches/v13_0/rename_onboarding.py index c506c6076e..852065dfd2 100644 --- a/frappe/patches/v13_0/rename_onboarding.py +++ b/frappe/patches/v13_0/rename_onboarding.py @@ -1,7 +1,6 @@ # Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import frappe def execute(): diff --git a/frappe/patches/v13_0/replace_old_data_import.py b/frappe/patches/v13_0/replace_old_data_import.py index 920ee7b553..838881b48e 100644 --- a/frappe/patches/v13_0/replace_old_data_import.py +++ b/frappe/patches/v13_0/replace_old_data_import.py @@ -1,7 +1,6 @@ # Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import frappe diff --git a/frappe/patches/v13_0/update_date_filters_in_user_settings.py b/frappe/patches/v13_0/update_date_filters_in_user_settings.py index d4c6aa1d03..3b1b07fe0a 100644 --- a/frappe/patches/v13_0/update_date_filters_in_user_settings.py +++ b/frappe/patches/v13_0/update_date_filters_in_user_settings.py @@ -1,4 +1,4 @@ -from __future__ import unicode_literals + import frappe, json from frappe.model.utils.user_settings import update_user_settings, sync_user_settings diff --git a/frappe/patches/v13_0/update_duration_options.py b/frappe/patches/v13_0/update_duration_options.py index 60eef8fc93..e0d8dea4ea 100644 --- a/frappe/patches/v13_0/update_duration_options.py +++ b/frappe/patches/v13_0/update_duration_options.py @@ -1,7 +1,6 @@ # Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import frappe def execute(): diff --git a/frappe/patches/v13_0/update_icons_in_customized_desk_pages.py b/frappe/patches/v13_0/update_icons_in_customized_desk_pages.py index 93bf5c766e..ff58f99c2f 100644 --- a/frappe/patches/v13_0/update_icons_in_customized_desk_pages.py +++ b/frappe/patches/v13_0/update_icons_in_customized_desk_pages.py @@ -1,4 +1,4 @@ -from __future__ import unicode_literals + import frappe def execute(): diff --git a/frappe/patches/v13_0/update_newsletter_content_type.py b/frappe/patches/v13_0/update_newsletter_content_type.py index 6f8dcc1935..5f047680ee 100644 --- a/frappe/patches/v13_0/update_newsletter_content_type.py +++ b/frappe/patches/v13_0/update_newsletter_content_type.py @@ -1,7 +1,6 @@ # Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import frappe def execute(): diff --git a/frappe/patches/v13_0/update_notification_channel_if_empty.py b/frappe/patches/v13_0/update_notification_channel_if_empty.py new file mode 100644 index 0000000000..bcf9a7b28c --- /dev/null +++ b/frappe/patches/v13_0/update_notification_channel_if_empty.py @@ -0,0 +1,14 @@ +# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors +# MIT License. See license.txt + +import frappe + +def execute(): + + frappe.reload_doc("Email", "doctype", "Notification") + + notifications = frappe.get_all('Notification', {'is_standard': 1}, {'name', 'channel'}) + for notification in notifications: + if not notification.channel: + frappe.db.set_value("Notification", notification.name, "channel", "Email", update_modified=False) + frappe.db.commit() diff --git a/frappe/patches/v13_0/web_template_set_module.py b/frappe/patches/v13_0/web_template_set_module.py index df008557d8..2ee9e3ba2d 100644 --- a/frappe/patches/v13_0/web_template_set_module.py +++ b/frappe/patches/v13_0/web_template_set_module.py @@ -1,7 +1,6 @@ # Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import frappe def execute(): diff --git a/frappe/patches/v4_0/add_delete_permission.py b/frappe/patches/v4_0/add_delete_permission.py deleted file mode 100644 index 091bdab3ff..0000000000 --- a/frappe/patches/v4_0/add_delete_permission.py +++ /dev/null @@ -1,13 +0,0 @@ -from __future__ import unicode_literals -import frappe - -def execute(): - frappe.reload_doc("core", "doctype", "docperm") - - # delete same as cancel (map old permissions) - frappe.db.sql("""update tabDocPerm set `delete`=ifnull(`cancel`,0)""") - - # can't cancel if can't submit - frappe.db.sql("""update tabDocPerm set `cancel`=0 where ifnull(`submit`,0)=0""") - - frappe.clear_cache() \ No newline at end of file diff --git a/frappe/patches/v4_0/change_varchar_length.py b/frappe/patches/v4_0/change_varchar_length.py deleted file mode 100644 index 29fe8f310d..0000000000 --- a/frappe/patches/v4_0/change_varchar_length.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors -# MIT License. See license.txt - -from __future__ import unicode_literals -import frappe - -def execute(): - frappe.db.sql('update tabDocField set search_index=0 where fieldtype="Small Text"') - frappe.db.sql('update tabDocField set in_list_view=0 where fieldtype="Image"') - - for dt in frappe.db.sql_list("""select name from `tabDocType` where issingle=0"""): - desc = dict((d["Field"], d) for d in frappe.db.sql("desc `tab{}`".format(dt), as_dict=True)) - alter_table = [] - - if desc["name"]["Type"] != "varchar(255)": - alter_table.append("change `name` `name` varchar(255) not null") - - for fieldname in ("modified_by", "owner", "parent", "parentfield", "parenttype"): - if desc[fieldname]["Type"] != "varchar(255)": - alter_table.append("change `{fieldname}` `{fieldname}` varchar(255)".format(fieldname=fieldname)) - - if alter_table: - alter_table_query = "alter table `tab{doctype}` {alter_table}".format(doctype=dt, alter_table=",\n".join(alter_table)) - # print alter_table_query - frappe.db.sql_ddl(alter_table_query) - diff --git a/frappe/patches/v4_0/create_custom_field_for_owner_match.py b/frappe/patches/v4_0/create_custom_field_for_owner_match.py deleted file mode 100644 index 60dafc27da..0000000000 --- a/frappe/patches/v4_0/create_custom_field_for_owner_match.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors -# MIT License. See license.txt - -from __future__ import unicode_literals, print_function -import frappe -from frappe.custom.doctype.custom_field.custom_field import create_custom_field - -def execute(): - if "match" in frappe.db.get_table_columns("DocPerm"): - create_custom_field_for_owner_match() - -def create_custom_field_for_owner_match(): - docperm_meta = frappe.get_meta('DocPerm') - if docperm_meta.get_field('apply_user_permissions'): - frappe.db.sql("""update `tabDocPerm` set apply_user_permissions=1 where `match`='owner'""") - - for dt in frappe.db.sql_list("""select distinct parent from `tabDocPerm` - where `match`='owner' and permlevel=0 and parent != 'User'"""): - - # a link field pointing to User already exists - if (frappe.db.get_value("DocField", {"parent": dt, "fieldtype": "Link", "options": "User", "default": "__user"}) - or frappe.db.get_value("Custom Field", {"dt": dt, "fieldtype": "Link", "options": "User", "default": "__user"})): - print("User link field already exists for", dt) - continue - - fieldname = "{}_owner".format(frappe.scrub(dt)) - - create_custom_field(dt, frappe._dict({ - "permlevel": 0, - "label": "{} Owner".format(dt), - "fieldname": fieldname, - "fieldtype": "Link", - "options": "User", - "default": "__user" - })) - - frappe.db.sql("""update `tab{doctype}` set `{fieldname}`=owner""".format(doctype=dt, - fieldname=fieldname)) - - # commit is required so that we don't lose these changes because of an error in next loop's ddl - frappe.db.commit() diff --git a/frappe/patches/v4_0/deprecate_control_panel.py b/frappe/patches/v4_0/deprecate_control_panel.py deleted file mode 100644 index 892d3043c4..0000000000 --- a/frappe/patches/v4_0/deprecate_control_panel.py +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors -# MIT License. See license.txt - -from __future__ import unicode_literals -import frappe - -def execute(): - frappe.db.sql("update `tabDefaultValue` set parenttype='__default' where parenttype='Control Panel'") - frappe.db.sql("update `tabDefaultValue` set parent='__default' where parent='Control Panel'") - frappe.clear_cache() diff --git a/frappe/patches/v4_0/deprecate_link_selects.py b/frappe/patches/v4_0/deprecate_link_selects.py deleted file mode 100644 index a3243cffb8..0000000000 --- a/frappe/patches/v4_0/deprecate_link_selects.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors -# MIT License. See license.txt - -from __future__ import unicode_literals -import frappe - -def execute(): - for name in frappe.db.sql_list("""select name from `tabCustom Field` - where fieldtype="Select" and options like "link:%" """): - custom_field = frappe.get_doc("Custom Field", name) - custom_field.fieldtype = "Link" - custom_field.options = custom_field.options[5:] - custom_field.save() diff --git a/frappe/patches/v4_0/enable_scheduler_in_system_settings.py b/frappe/patches/v4_0/enable_scheduler_in_system_settings.py deleted file mode 100644 index 5d1b836270..0000000000 --- a/frappe/patches/v4_0/enable_scheduler_in_system_settings.py +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors -# MIT License. See license.txt - -from __future__ import unicode_literals -import frappe -from frappe.utils.scheduler import disable_scheduler, enable_scheduler -from frappe.utils import cint - -def execute(): - frappe.reload_doc("core", "doctype", "system_settings") - if cint(frappe.db.get_global("disable_scheduler")): - disable_scheduler() - else: - enable_scheduler() diff --git a/frappe/patches/v4_0/file_manager_hooks.py b/frappe/patches/v4_0/file_manager_hooks.py deleted file mode 100644 index 6be3b25124..0000000000 --- a/frappe/patches/v4_0/file_manager_hooks.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors -# MIT License. See license.txt - -from __future__ import unicode_literals, print_function - -import frappe -import os -from frappe.utils import get_files_path -from frappe.core.doctype.file.file import get_content_hash - - -def execute(): - frappe.reload_doc('core', 'doctype', 'file_data') - for name, file_name, file_url in frappe.db.sql( - """select name, file_name, file_url from `tabFile` - where file_name is not null"""): - b = frappe.get_doc('File', name) - old_file_name = b.file_name - b.file_name = os.path.basename(old_file_name) - if old_file_name.startswith('files/') or old_file_name.startswith('/files/'): - b.file_url = os.path.normpath('/' + old_file_name) - else: - b.file_url = os.path.normpath('/files/' + old_file_name) - try: - _file = frappe.get_doc("File", {"file_name": name}) - content = _file.get_content() - b.content_hash = get_content_hash(content) - except IOError: - print('Warning: Error processing ', name) - _file_name = old_file_name - b.content_hash = None - - try: - b.save() - except frappe.DuplicateEntryError: - frappe.delete_doc(b.doctype, b.name) - diff --git a/frappe/patches/v4_0/fix_attach_field_file_url.py b/frappe/patches/v4_0/fix_attach_field_file_url.py deleted file mode 100644 index c29e5763f1..0000000000 --- a/frappe/patches/v4_0/fix_attach_field_file_url.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors -# MIT License. See license.txt - -from __future__ import unicode_literals -import frappe - -def execute(): - attach_fields = (frappe.db.sql("""select parent, fieldname from `tabDocField` where fieldtype in ('Attach', 'Attach Image')""") + - frappe.db.sql("""select dt, fieldname from `tabCustom Field` where fieldtype in ('Attach', 'Attach Image')""")) - - for doctype, fieldname in attach_fields: - frappe.db.sql("""update `tab{doctype}` set `{fieldname}`=concat("/", `{fieldname}`) - where `{fieldname}` like 'files/%'""".format(doctype=doctype, fieldname=fieldname)) diff --git a/frappe/patches/v4_0/private_backups.py b/frappe/patches/v4_0/private_backups.py deleted file mode 100644 index 016af0615d..0000000000 --- a/frappe/patches/v4_0/private_backups.py +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors -# MIT License. See license.txt - -from __future__ import unicode_literals -import frappe -from frappe.installer import make_site_dirs - -def execute(): - make_site_dirs() - if frappe.local.conf.backup_path and frappe.local.conf.backup_path.startswith("public"): - raise Exception("Backups path in conf set to public directory") diff --git a/frappe/patches/v4_0/remove_index_sitemap.py b/frappe/patches/v4_0/remove_index_sitemap.py deleted file mode 100644 index 5dcd0d79c7..0000000000 --- a/frappe/patches/v4_0/remove_index_sitemap.py +++ /dev/null @@ -1,5 +0,0 @@ -from __future__ import unicode_literals -import frappe - -def execute(): - pass diff --git a/frappe/patches/v4_0/remove_old_parent.py b/frappe/patches/v4_0/remove_old_parent.py deleted file mode 100644 index 7717f7b7e3..0000000000 --- a/frappe/patches/v4_0/remove_old_parent.py +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors -# MIT License. See license.txt - -from __future__ import unicode_literals -import frappe - -def execute(): - for doctype in frappe.db.sql_list("""select name from `tabDocType` where istable=1"""): - frappe.db.sql("""delete from `tab{0}` where parent like "old_par%:%" """.format(doctype)) - frappe.db.sql("""delete from `tabDocField` where parent="0" """) diff --git a/frappe/patches/v4_0/remove_user_owner_custom_field.py b/frappe/patches/v4_0/remove_user_owner_custom_field.py deleted file mode 100644 index be6a45e090..0000000000 --- a/frappe/patches/v4_0/remove_user_owner_custom_field.py +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors -# MIT License. See license.txt - -from __future__ import unicode_literals -import frappe - -def execute(): - user_owner = frappe.db.get_value("Custom Field", {"fieldname": "user_owner"}) - if user_owner: - frappe.delete_doc("Custom Field", user_owner) diff --git a/frappe/patches/v4_0/rename_profile_to_user.py b/frappe/patches/v4_0/rename_profile_to_user.py deleted file mode 100644 index 48555ead9e..0000000000 --- a/frappe/patches/v4_0/rename_profile_to_user.py +++ /dev/null @@ -1,15 +0,0 @@ -from __future__ import unicode_literals -import frappe - -from frappe.model.utils.rename_field import rename_field -from frappe.model.meta import get_table_columns - -def execute(): - tables = frappe.db.sql_list("show tables") - if "tabUser" not in tables: - frappe.rename_doc("DocType", "Profile", "User", force=True) - - frappe.reload_doc("website", "doctype", "blogger") - - if "profile" in get_table_columns("Blogger"): - rename_field("Blogger", "profile", "user") diff --git a/frappe/patches/v4_0/rename_sitemap_to_route.py b/frappe/patches/v4_0/rename_sitemap_to_route.py deleted file mode 100644 index 8ae5170b44..0000000000 --- a/frappe/patches/v4_0/rename_sitemap_to_route.py +++ /dev/null @@ -1,25 +0,0 @@ -from __future__ import unicode_literals -import frappe - -from frappe.model.utils.rename_field import rename_field - -def execute(): - tables = frappe.db.sql_list("show tables") - for doctype in ("Website Sitemap", "Website Sitemap Config"): - if "tab{}".format(doctype) in tables: - frappe.delete_doc("DocType", doctype, force=1) - frappe.db.sql("drop table `tab{}`".format(doctype)) - - for d in ("Blog Category", "Blog Post", "Web Page"): - frappe.reload_doc("website", "doctype", frappe.scrub(d)) - rename_field_if_exists(d, "parent_website_sitemap", "parent_website_route") - - for d in ("blog_category", "blog_post", "web_page", "post", "user_vote"): - frappe.reload_doc("website", "doctype", d) - -def rename_field_if_exists(doctype, old_fieldname, new_fieldname): - try: - rename_field(doctype, old_fieldname, new_fieldname) - except frappe.db.ProgrammingError as e: - if not frappe.db.is_column_missing(e): - raise diff --git a/frappe/patches/v4_0/replace_deprecated_timezones.py b/frappe/patches/v4_0/replace_deprecated_timezones.py deleted file mode 100644 index a491325ebc..0000000000 --- a/frappe/patches/v4_0/replace_deprecated_timezones.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors -# MIT License. See license.txt - -from __future__ import unicode_literals -import frappe -from frappe.utils.momentjs import data as momentjs_data - -def execute(): - frappe.reload_doc("core", "doctype", "user") - - ss = frappe.get_doc("System Settings", "System Settings") - if ss.time_zone in momentjs_data.get("links"): - ss.time_zone = momentjs_data["links"][ss.time_zone] - ss.flags.ignore_mandatory = True - ss.save() - - for user, time_zone in frappe.db.sql("select name, time_zone from `tabUser` where ifnull(time_zone, '')!=''"): - if time_zone in momentjs_data.get("links"): - user = frappe.get_doc("User", user) - user.time_zone = momentjs_data["links"][user.time_zone] - user.save() diff --git a/frappe/patches/v4_0/set_module_in_report.py b/frappe/patches/v4_0/set_module_in_report.py deleted file mode 100644 index 9760f7efb3..0000000000 --- a/frappe/patches/v4_0/set_module_in_report.py +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors -# MIT License. See license.txt - -from __future__ import unicode_literals -import frappe - -def execute(): - frappe.reload_doc("core", "doctype", "report") - frappe.db.sql("""update `tabReport` r set r.module=(select d.module from `tabDocType` d - where d.name=r.ref_doctype) where ifnull(r.module, '')=''""") \ No newline at end of file diff --git a/frappe/patches/v4_0/set_todo_checked_as_closed.py b/frappe/patches/v4_0/set_todo_checked_as_closed.py deleted file mode 100644 index 59e8df3793..0000000000 --- a/frappe/patches/v4_0/set_todo_checked_as_closed.py +++ /dev/null @@ -1,9 +0,0 @@ -from __future__ import unicode_literals -import frappe - -def execute(): - frappe.reload_doc("core", "doctype", "todo") - try: - frappe.db.sql("""update tabToDo set status = if(ifnull(checked,0)=0, 'Open', 'Closed')""") - except: - pass diff --git a/frappe/patches/v4_0/set_user_gravatar.py b/frappe/patches/v4_0/set_user_gravatar.py deleted file mode 100644 index 733b9bfe11..0000000000 --- a/frappe/patches/v4_0/set_user_gravatar.py +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors -# MIT License. See license.txt - -from __future__ import unicode_literals -import frappe - -def execute(): - for name in frappe.db.sql_list("select name from `tabUser` where ifnull(user_image, '')=''"): - user = frappe.get_doc("User", name) - user.update_gravatar() - user.db_set("user_image", user.user_image) diff --git a/frappe/patches/v4_0/set_user_permissions.py b/frappe/patches/v4_0/set_user_permissions.py deleted file mode 100644 index 726b9ee715..0000000000 --- a/frappe/patches/v4_0/set_user_permissions.py +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors -# MIT License. See license.txt - -from __future__ import unicode_literals -import frappe -import frappe.permissions - -def execute(): - frappe.reload_doc("core", "doctype", "docperm") - table_columns = frappe.db.get_table_columns("DocPerm") - - if "restricted" in table_columns: - frappe.db.sql("""update `tabDocPerm` set apply_user_permissions=1 where apply_user_permissions=0 - and restricted=1""") - - if "match" in table_columns: - frappe.db.sql("""update `tabDocPerm` set apply_user_permissions=1 - where apply_user_permissions=0 and ifnull(`match`, '')!=''""") - - # change Restriction to User Permission in tabDefaultValue - frappe.db.sql("""update `tabDefaultValue` set parenttype='User Permission' where parenttype='Restriction'""") - - frappe.clear_cache() - diff --git a/frappe/patches/v4_0/set_website_route_idx.py b/frappe/patches/v4_0/set_website_route_idx.py deleted file mode 100644 index 663a324008..0000000000 --- a/frappe/patches/v4_0/set_website_route_idx.py +++ /dev/null @@ -1,25 +0,0 @@ -from __future__ import unicode_literals -import frappe - -def execute(): - pass - # from frappe.website.doctype.website_template.website_template import \ - # get_pages_and_generators, get_template_controller - # - # frappe.reload_doc("website", "doctype", "website_template") - # frappe.reload_doc("website", "doctype", "website_route") - # - # for app in frappe.get_installed_apps(): - # pages, generators = get_pages_and_generators(app) - # for g in generators: - # doctype = frappe.get_attr(get_template_controller(app, g["path"], g["fname"]) + ".doctype") - # module = frappe.db.get_value("DocType", doctype, "module") - # frappe.reload_doc(frappe.scrub(module), "doctype", frappe.scrub(doctype)) - # - # frappe.db.sql("""update `tabBlog Category` set `title`=`name` where ifnull(`title`, '')=''""") - # frappe.db.sql("""update `tabWebsite Route` set idx=null""") - # for doctype in ["Blog Category", "Blog Post", "Web Page", "Website Group"]: - # frappe.db.sql("""update `tab{}` set idx=null""".format(doctype)) - # - # from frappe.website.doctype.website_template.website_template import rebuild_website_template - # rebuild_website_template() diff --git a/frappe/patches/v4_0/update_custom_field_insert_after.py b/frappe/patches/v4_0/update_custom_field_insert_after.py deleted file mode 100644 index ddb888c493..0000000000 --- a/frappe/patches/v4_0/update_custom_field_insert_after.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors -# MIT License. See license.txt - -from __future__ import unicode_literals -import frappe - -def execute(): - for d in frappe.db.sql("""select name, dt, insert_after from `tabCustom Field` - where docstatus < 2""", as_dict=1): - dt_meta = frappe.get_meta(d.dt) - if not dt_meta.get_field(d.insert_after): - cf = frappe.get_doc("Custom Field", d.name) - df = dt_meta.get("fields", {"label": d.insert_after}) - if df: - cf.insert_after = df[0].fieldname - else: - cf.insert_after = None - cf.save() diff --git a/frappe/patches/v4_0/update_datetime.py b/frappe/patches/v4_0/update_datetime.py deleted file mode 100644 index 0e91174780..0000000000 --- a/frappe/patches/v4_0/update_datetime.py +++ /dev/null @@ -1,12 +0,0 @@ -from __future__ import unicode_literals -import frappe - -def execute(): - for table in frappe.db.sql_list("show tables"): - for field in frappe.db.sql("desc `%s`" % table): - if field[1]=="datetime": - frappe.db.sql("alter table `%s` change `%s` `%s` datetime(6)" % \ - (table, field[0], field[0])) - elif field[1]=="time": - frappe.db.sql("alter table `%s` change `%s` `%s` time(6)" % \ - (table, field[0], field[0])) diff --git a/frappe/patches/v4_0/webnotes_to_frappe.py b/frappe/patches/v4_0/webnotes_to_frappe.py deleted file mode 100644 index 22b3848d5a..0000000000 --- a/frappe/patches/v4_0/webnotes_to_frappe.py +++ /dev/null @@ -1,12 +0,0 @@ -from __future__ import unicode_literals -import frappe, json - -def execute(): - frappe.clear_cache() - installed = frappe.get_installed_apps() - if "webnotes" in installed: - installed.remove("webnotes") - if "frappe" not in installed: - installed = ["frappe"] + installed - frappe.db.set_global("installed_apps", json.dumps(installed)) - frappe.clear_cache() diff --git a/frappe/patches/v4_0/website_sitemap_hierarchy.py b/frappe/patches/v4_0/website_sitemap_hierarchy.py deleted file mode 100644 index bb22144cd7..0000000000 --- a/frappe/patches/v4_0/website_sitemap_hierarchy.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors -# MIT License. See license.txt - -from __future__ import unicode_literals - -import frappe - -def execute(): - # frappe.db.sql("""update `tabWebsite Route` ws set ref_doctype=(select wsc.ref_doctype - # from `tabWebsite Template` wsc where wsc.name=ws.website_template) - # where ifnull(page_or_generator, '')!='Page'""") - - frappe.reload_doc("website", "doctype", "website_settings") - - # original_home_page = frappe.db.get_value("Website Settings", "Website Settings", "home_page") - # - # home_page = frappe.db.sql("""select name from `tabWebsite Route` - # where (name=%s or docname=%s) and name!='index'""", (original_home_page, original_home_page)) - # home_page = home_page[0][0] if home_page else original_home_page - # - # frappe.db.set_value("Website Settings", "Website Settings", "home_page", home_page) diff --git a/frappe/patches/v4_1/__init__.py b/frappe/patches/v4_1/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/patches/v4_1/enable_outgoing_email_settings.py b/frappe/patches/v4_1/enable_outgoing_email_settings.py deleted file mode 100644 index 7ffa84a278..0000000000 --- a/frappe/patches/v4_1/enable_outgoing_email_settings.py +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors -# MIT License. See license.txt - -from __future__ import unicode_literals -import frappe - -def execute(): - frappe.reload_doc("core", "doctype", "outgoing_email_settings") - if (frappe.db.get_value("Outgoing Email Settings", "Outgoing Email Settings", "mail_server") or "").strip(): - frappe.db.set_value("Outgoing Email Settings", "Outgoing Email Settings", "enabled", 1) diff --git a/frappe/patches/v4_1/enable_print_as_pdf.py b/frappe/patches/v4_1/enable_print_as_pdf.py deleted file mode 100644 index 74db9f72ca..0000000000 --- a/frappe/patches/v4_1/enable_print_as_pdf.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors -# MIT License. See license.txt - -from __future__ import unicode_literals -import frappe - -def execute(): - frappe.reload_doc("core", "doctype", "print_settings") - print_settings = frappe.get_doc("Print Settings") - print_settings.print_style = "Modern" - - try: - import pdfkit - except ImportError: - pass - else: - # if someone has already configured in Outgoing Email Settings - outgoing_email_settings = frappe.db.get_singles_dict("Outgoing Email Settings") - if "send_print_as_pdf" in outgoing_email_settings: - print_settings.send_print_as_pdf = outgoing_email_settings.send_print_as_pdf - print_settings.pdf_page_size = outgoing_email_settings.pdf_page_size - - else: - print_settings.send_print_as_pdf = 1 - - print_settings.save() diff --git a/frappe/patches/v4_1/file_manager_fix.py b/frappe/patches/v4_1/file_manager_fix.py deleted file mode 100644 index cd30c94177..0000000000 --- a/frappe/patches/v4_1/file_manager_fix.py +++ /dev/null @@ -1,101 +0,0 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors -# MIT License. See license.txt - -from __future__ import unicode_literals, print_function - -import frappe -import os -from frappe.core.doctype.file.file import get_content_hash, get_file_name -from frappe.utils import get_files_path, get_site_path - -# The files missed by the previous patch might have been replaced with new files -# with the same filename -# -# This patch does the following, -# * Detect which files were replaced and rename them with name{hash:5}.extn and -# update filedata record for the new file -# -# * make missing_files.txt in site dir with files that should be recovered from -# a backup from a time before version 3 migration -# -# * Patch remaining unpatched File records. -from six import iteritems - - -def execute(): - frappe.db.auto_commit_on_many_writes = True - rename_replacing_files() - for name, file_name, file_url in frappe.db.sql( - """select name, file_name, file_url from `tabFile` - where ifnull(file_name, '')!='' and ifnull(content_hash, '')=''"""): - b = frappe.get_doc('File', name) - old_file_name = b.file_name - b.file_name = os.path.basename(old_file_name) - if old_file_name.startswith('files/') or old_file_name.startswith('/files/'): - b.file_url = os.path.normpath('/' + old_file_name) - else: - b.file_url = os.path.normpath('/files/' + old_file_name) - try: - _file = frappe.get_doc("File", {"file_name": name}) - content = _file.get_content() - b.content_hash = get_content_hash(content) - except IOError: - print('Warning: Error processing ', name) - b.content_hash = None - b.flags.ignore_duplicate_entry_error = True - b.save() - frappe.db.auto_commit_on_many_writes = False - -def get_replaced_files(): - ret = [] - new_files = dict(frappe.db.sql("select name, file_name from `tabFile` where file_name not like 'files/%'")) - old_files = dict(frappe.db.sql("select name, file_name from `tabFile` where ifnull(content_hash, '')=''")) - invfiles = invert_dict(new_files) - - for nname, nfilename in iteritems(new_files): - if 'files/' + nfilename in old_files.values(): - ret.append((nfilename, invfiles[nfilename])) - return ret - -def rename_replacing_files(): - replaced_files = get_replaced_files() - if len(replaced_files): - missing_files = [v[0] for v in replaced_files] - with open(get_site_path('missing_files.txt'), 'w') as f: - f.write(('\n'.join(missing_files) + '\n').encode('utf-8')) - - for file_name, file_datas in replaced_files: - print ('processing ' + file_name) - content_hash = frappe.db.get_value('File', file_datas[0], 'content_hash') - if not content_hash: - continue - new_file_name = get_file_name(file_name, content_hash) - if os.path.exists(get_files_path(new_file_name)): - continue - print('skipping ' + file_name) - try: - os.rename(get_files_path(file_name), get_files_path(new_file_name)) - except OSError: - print('Error renaming ', file_name) - for name in file_datas: - f = frappe.get_doc('File', name) - f.file_name = new_file_name - f.file_url = '/files/' + new_file_name - f.save() - -def invert_dict(ddict): - ret = {} - for k,v in iteritems(ddict): - if not ret.get(v): - ret[v] = [k] - else: - ret[v].append(k) - return ret - -def get_file_name(fname, hash): - if '.' in fname: - partial, extn = fname.rsplit('.', 1) - else: - partial = fname - extn = '' - return '{partial}{suffix}.{extn}'.format(partial=partial, extn=extn, suffix=hash[:5]) diff --git a/frappe/patches/v4_2/__init__.py b/frappe/patches/v4_2/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/patches/v4_2/print_with_letterhead.py b/frappe/patches/v4_2/print_with_letterhead.py deleted file mode 100644 index 3e611ce073..0000000000 --- a/frappe/patches/v4_2/print_with_letterhead.py +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors -# MIT License. See license.txt - -from __future__ import unicode_literals -import frappe - -def execute(): - frappe.reload_doc("core", "doctype", "print_settings") - print_settings = frappe.get_doc("Print Settings") - print_settings.with_letterhead = 1 - print_settings.save() diff --git a/frappe/patches/v4_2/refactor_website_routing.py b/frappe/patches/v4_2/refactor_website_routing.py deleted file mode 100644 index a5856db1c9..0000000000 --- a/frappe/patches/v4_2/refactor_website_routing.py +++ /dev/null @@ -1,8 +0,0 @@ -from __future__ import unicode_literals -import frappe - -def execute(): - # clear all static web pages - frappe.delete_doc("DocType", "Website Route", force=1) - frappe.delete_doc("Page", "sitemap-browser", force=1) - frappe.db.sql("drop table if exists `tabWebsite Route`") diff --git a/frappe/patches/v4_2/set_assign_in_doc.py b/frappe/patches/v4_2/set_assign_in_doc.py deleted file mode 100644 index a6a06492a0..0000000000 --- a/frappe/patches/v4_2/set_assign_in_doc.py +++ /dev/null @@ -1,11 +0,0 @@ -from __future__ import unicode_literals -import frappe - -def execute(): - for name in frappe.db.sql_list("""select name from `tabToDo` - where ifnull(reference_type, '')!='' and ifnull(reference_name, '')!=''"""): - try: - frappe.get_doc("ToDo", name).on_update() - except Exception as e: - if not frappe.db.is_table_missing(e): - raise diff --git a/frappe/patches/v4_3/__init__.py b/frappe/patches/v4_3/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/patches/v4_3/remove_allow_on_submit_customization.py b/frappe/patches/v4_3/remove_allow_on_submit_customization.py deleted file mode 100644 index af6ade68e6..0000000000 --- a/frappe/patches/v4_3/remove_allow_on_submit_customization.py +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors -# MIT License. See license.txt - -from __future__ import unicode_literals -import frappe - -def execute(): - for d in frappe.get_all("Property Setter", fields=["name", "doc_type"], - filters={"doctype_or_field": "DocField", "property": "allow_on_submit", "value": "1"}): - frappe.delete_doc("Property Setter", d.name) - frappe.clear_cache(doctype=d.doc_type) diff --git a/frappe/patches/v5_0/__init__.py b/frappe/patches/v5_0/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/patches/v5_0/bookmarks_to_stars.py b/frappe/patches/v5_0/bookmarks_to_stars.py deleted file mode 100644 index 048d059701..0000000000 --- a/frappe/patches/v5_0/bookmarks_to_stars.py +++ /dev/null @@ -1,33 +0,0 @@ -from __future__ import unicode_literals -import json -import frappe -import frappe.defaults -from frappe.desk.like import _toggle_like -from six import string_types - -def execute(): - for user in frappe.get_all("User"): - username = user["name"] - bookmarks = frappe.db.get_default("_bookmarks", username) - - if not bookmarks: - continue - - if isinstance(bookmarks, string_types): - bookmarks = json.loads(bookmarks) - - for opts in bookmarks: - route = (opts.get("route") or "").strip("#/ ") - - if route and route.startswith("Form"): - try: - view, doctype, docname = opts["route"].split("/") - except ValueError: - continue - - if frappe.db.exists(doctype, docname): - if (doctype=="DocType" - or int(frappe.db.get_value("DocType", doctype, "issingle") or 0) - or not frappe.db.table_exists(doctype)): - continue - _toggle_like(doctype, docname, add="Yes", user=username) diff --git a/frappe/patches/v5_0/clear_website_group_and_notifications.py b/frappe/patches/v5_0/clear_website_group_and_notifications.py deleted file mode 100644 index bad50222a3..0000000000 --- a/frappe/patches/v5_0/clear_website_group_and_notifications.py +++ /dev/null @@ -1,9 +0,0 @@ -from __future__ import unicode_literals -import frappe - -def execute(): - frappe.delete_doc("DocType", "Post") - frappe.delete_doc("DocType", "Website Group") - frappe.delete_doc("DocType", "Website Route Permission") - frappe.delete_doc("DocType", "User Vote") - frappe.delete_doc("DocType", "Notification Count") diff --git a/frappe/patches/v5_0/communication_parent.py b/frappe/patches/v5_0/communication_parent.py deleted file mode 100644 index 2ea3b401c6..0000000000 --- a/frappe/patches/v5_0/communication_parent.py +++ /dev/null @@ -1,6 +0,0 @@ -from __future__ import unicode_literals -import frappe - -def execute(): - frappe.reload_doc("core", "doctype", "communication") - frappe.db.sql("""update tabCommunication set reference_doctype = parenttype, reference_name = parent""") diff --git a/frappe/patches/v5_0/convert_to_barracuda_and_utf8mb4.py b/frappe/patches/v5_0/convert_to_barracuda_and_utf8mb4.py deleted file mode 100644 index 0ea2ee2387..0000000000 --- a/frappe/patches/v5_0/convert_to_barracuda_and_utf8mb4.py +++ /dev/null @@ -1,16 +0,0 @@ -from __future__ import unicode_literals -import frappe -from frappe.database.mariadb.setup_db import check_database_settings -from frappe.model.meta import trim_tables - -def execute(): - check_database_settings() - - for table in frappe.db.get_tables(): - frappe.db.sql_ddl("""alter table `{0}` ENGINE=InnoDB ROW_FORMAT=COMPRESSED""".format(table)) - try: - frappe.db.sql_ddl("""alter table `{0}` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci""".format(table)) - except: - # if row size gets too large, let it be old charset! - pass - diff --git a/frappe/patches/v5_0/expire_old_scheduler_logs.py b/frappe/patches/v5_0/expire_old_scheduler_logs.py deleted file mode 100644 index 8b65ed5fb1..0000000000 --- a/frappe/patches/v5_0/expire_old_scheduler_logs.py +++ /dev/null @@ -1,8 +0,0 @@ -from __future__ import unicode_literals -import frappe - -def execute(): - frappe.reload_doctype("Error Log") - - from frappe.core.doctype.error_log.error_log import set_old_logs_as_seen - set_old_logs_as_seen() diff --git a/frappe/patches/v5_0/fix_email_alert.py b/frappe/patches/v5_0/fix_email_alert.py deleted file mode 100644 index 0676f50a9c..0000000000 --- a/frappe/patches/v5_0/fix_email_alert.py +++ /dev/null @@ -1,16 +0,0 @@ -from __future__ import unicode_literals - -import frappe - -def execute(): - frappe.reload_doctype("Notification") - for e in frappe.get_all("Notification"): - notification = frappe.get_doc("Notification", e.name) - if notification.event == "Date Change": - if notification.days_in_advance < 0: - notification.event = "Days After" - notification.days_in_advance = -email_alert.days_in_advance - else: - notification.event = "Days Before" - - notification.save() diff --git a/frappe/patches/v5_0/fix_null_date_datetime.py b/frappe/patches/v5_0/fix_null_date_datetime.py deleted file mode 100644 index e4f4e9e8b9..0000000000 --- a/frappe/patches/v5_0/fix_null_date_datetime.py +++ /dev/null @@ -1,20 +0,0 @@ -from __future__ import unicode_literals -import frappe - -def execute(): - for table in frappe.db.get_tables(): - changed = False - desc = frappe.db.sql("desc `{table}`".format(table=table), as_dict=True) - for field in desc: - if field["Type"] == "date": - frappe.db.sql("""update `{table}` set `{fieldname}`=null where `{fieldname}`='0000-00-00'""".format( - table=table, fieldname=field["Field"])) - changed = True - - elif field["Type"] == "datetime(6)": - frappe.db.sql("""update `{table}` set `{fieldname}`=null where `{fieldname}`='0000-00-00 00:00:00.000000'""".format( - table=table, fieldname=field["Field"])) - changed = True - - if changed: - frappe.db.commit() diff --git a/frappe/patches/v5_0/fix_text_editor_file_urls.py b/frappe/patches/v5_0/fix_text_editor_file_urls.py deleted file mode 100644 index d91aad0234..0000000000 --- a/frappe/patches/v5_0/fix_text_editor_file_urls.py +++ /dev/null @@ -1,45 +0,0 @@ -from __future__ import unicode_literals, print_function -import frappe -import re - -def execute(): - """Fix relative urls for image src="files/" to src="/files/" in DocTypes with text editor fields""" - doctypes_with_text_fields = frappe.get_all("DocField", fields=["parent", "fieldname"], - filters={"fieldtype": "Text Editor"}) - - done = [] - for opts in doctypes_with_text_fields: - if opts in done: - continue - - try: - result = frappe.get_all(opts.parent, fields=["name", opts.fieldname]) - except frappe.db.SQLError: - # bypass single tables - continue - - for data in result: - old_value = data[opts.fieldname] - if not old_value: - continue - - html = scrub_relative_urls(old_value) - if html != old_value: - # print_diff(html, old_value) - frappe.db.set_value(opts.parent, data.name, opts.fieldname, html, update_modified=False) - - done.append(opts) - -def scrub_relative_urls(html): - """prepend a slash before a relative url""" - try: - return re.sub("""src[\s]*=[\s]*['"]files/([^'"]*)['"]""", 'src="/files/\g<1>"', html) - # return re.sub("""(src|href)[^\w'"]*['"](?!http|ftp|mailto|/|#|%|{|cid:|\.com/www\.)([^'" >]+)['"]""", '\g<1>="/\g<2>"', html) - except: - print("Error", html) - raise - -def print_diff(html, old_value): - import difflib - diff = difflib.unified_diff(old_value.splitlines(1), html.splitlines(1), lineterm='') - print('\n'.join(list(diff))) diff --git a/frappe/patches/v5_0/force_sync_website.py b/frappe/patches/v5_0/force_sync_website.py deleted file mode 100644 index 5dcd0d79c7..0000000000 --- a/frappe/patches/v5_0/force_sync_website.py +++ /dev/null @@ -1,5 +0,0 @@ -from __future__ import unicode_literals -import frappe - -def execute(): - pass diff --git a/frappe/patches/v5_0/modify_session.py b/frappe/patches/v5_0/modify_session.py deleted file mode 100644 index f0e247a633..0000000000 --- a/frappe/patches/v5_0/modify_session.py +++ /dev/null @@ -1,6 +0,0 @@ -from __future__ import unicode_literals -import frappe - -def execute(): - if "device" not in frappe.db.get_table_columns("Sessions"): - frappe.db.sql("alter table tabSessions add column `device` varchar(255) default 'desktop'") diff --git a/frappe/patches/v5_0/move_scheduler_last_event_to_system_settings.py b/frappe/patches/v5_0/move_scheduler_last_event_to_system_settings.py deleted file mode 100644 index 0fa1dad1e5..0000000000 --- a/frappe/patches/v5_0/move_scheduler_last_event_to_system_settings.py +++ /dev/null @@ -1,8 +0,0 @@ -from __future__ import unicode_literals -import frappe - -def execute(): - frappe.reload_doctype('System Settings') - last = frappe.db.get_global('scheduler_last_event') - frappe.db.set_value('System Settings', 'System Settings', 'scheduler_last_event', last) - diff --git a/frappe/patches/v5_0/remove_shopping_cart_app.py b/frappe/patches/v5_0/remove_shopping_cart_app.py deleted file mode 100644 index babde585a1..0000000000 --- a/frappe/patches/v5_0/remove_shopping_cart_app.py +++ /dev/null @@ -1,7 +0,0 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors -# License: GNU General Public License v3. See license.txt -from __future__ import unicode_literals - -def execute(): - from frappe.installer import remove_from_installed_apps - remove_from_installed_apps("shopping_cart") diff --git a/frappe/patches/v5_0/rename_ref_type_fieldnames.py b/frappe/patches/v5_0/rename_ref_type_fieldnames.py deleted file mode 100644 index dd24f6e5b5..0000000000 --- a/frappe/patches/v5_0/rename_ref_type_fieldnames.py +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors -# See license.txt - -from __future__ import unicode_literals -import frappe - -def execute(): - try: - frappe.db.sql("alter table `tabEmail Queue` change `ref_docname` `reference_name` varchar(255)") - except Exception as e: - if not frappe.db.is_table_or_column_missing(e): - raise - - try: - frappe.db.sql("alter table `tabEmail Queue` change `ref_doctype` `reference_doctype` varchar(255)") - except Exception as e: - if not frappe.db.is_table_or_column_missing(e): - raise - frappe.reload_doctype("Email Queue") diff --git a/frappe/patches/v5_0/rename_table_fieldnames.py b/frappe/patches/v5_0/rename_table_fieldnames.py deleted file mode 100644 index b716599f28..0000000000 --- a/frappe/patches/v5_0/rename_table_fieldnames.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors -# License: GNU General Public License v3. See license.txt - -from __future__ import unicode_literals -import frappe -from frappe.model.utils.rename_field import rename_field -from frappe.modules import scrub, get_doctype_module - -rename_map = { - "Customize Form": [ - ["customize_form_fields", "fields"] - ], - "Email Alert": [ - ["email_alert_recipients", "recipients"] - ], - "Workflow": [ - ["workflow_document_states", "states"], - ["workflow_transitions", "transitions"] - ] -} - -def execute(): - frappe.reload_doc("custom", "doctype", "customize_form") - frappe.reload_doc("email", "doctype", "notification") - frappe.reload_doc("desk", "doctype", "event") - frappe.reload_doc("workflow", "doctype", "workflow") - - for dt, field_list in rename_map.items(): - for field in field_list: - rename_field(dt, field[0], field[1]) diff --git a/frappe/patches/v5_0/style_settings_to_website_theme.py b/frappe/patches/v5_0/style_settings_to_website_theme.py deleted file mode 100644 index 40414d4e20..0000000000 --- a/frappe/patches/v5_0/style_settings_to_website_theme.py +++ /dev/null @@ -1,59 +0,0 @@ -from __future__ import unicode_literals -import frappe -from frappe import _ -from frappe.utils import cint - -def execute(): - frappe.reload_doc("website", "doctype", "website_theme") - frappe.reload_doc("website", "website_theme", "standard") - frappe.reload_doctype("Website Settings") - migrate_style_settings() - frappe.delete_doc("website", "doctype", "style_settings") - -def migrate_style_settings(): - style_settings = frappe.db.get_singles_dict("Style Settings") - standard_website_theme = frappe.get_doc("Website Theme", "Standard") - - website_theme = frappe.copy_doc(standard_website_theme) - website_theme.custom = 1 - website_theme.theme = _("Custom") - - if style_settings: - map_color_fields(style_settings, website_theme) - map_other_fields(style_settings, website_theme) - - website_theme.no_sidebar = cint(frappe.db.get_single_value("Website Settings", "no_sidebar")) - - website_theme.save() - website_theme.set_as_default() - -def map_color_fields(style_settings, website_theme): - color_fields_map = { - "page_text": "text_color", - "page_links": "link_color", - "top_bar_background": "top_bar_color", - "top_bar_foreground": "top_bar_text_color", - "footer_background": "footer_color", - "footer_color": "footer_text_color", - } - - for from_fieldname, to_fieldname in color_fields_map.items(): - from_value = style_settings.get(from_fieldname) - - if from_value: - website_theme.set(to_fieldname, "#{0}".format(from_value)) - -def map_other_fields(style_settings, website_theme): - other_fields_map = { - "heading_text_as": "heading_style", - "google_web_font_for_heading": "heading_webfont", - "google_web_font_for_text": "text_webfont", - "add_css": "css" - } - - for from_fieldname, to_fieldname in other_fields_map.items(): - website_theme.set(to_fieldname, style_settings.get(from_fieldname)) - - for fieldname in ("apply_style", "background_image", "background_color", - "font_size"): - website_theme.set(fieldname, style_settings.get(fieldname)) diff --git a/frappe/patches/v5_0/update_shared.py b/frappe/patches/v5_0/update_shared.py deleted file mode 100644 index f2b77895d8..0000000000 --- a/frappe/patches/v5_0/update_shared.py +++ /dev/null @@ -1,37 +0,0 @@ -from __future__ import unicode_literals -import frappe -import frappe.share - -def execute(): - frappe.reload_doc("core", "doctype", "docperm") - frappe.reload_doc("core", "doctype", "docshare") - frappe.reload_doc('email', 'doctype', 'email_account') - - # default share to all writes - frappe.db.sql("""update tabDocPerm set `share`=1 where ifnull(`write`,0)=1 and ifnull(`permlevel`,0)=0""") - - # every user must have access to his / her own detail - users = frappe.get_all("User", filters={"user_type": "System User"}) - usernames = [user.name for user in users] - for user in usernames: - frappe.share.add("User", user, user, write=1, share=1) - - # move event user to shared - if frappe.db.exists("DocType", "Event User"): - for event in frappe.get_all("Event User", fields=["parent", "person"]): - if event.person in usernames: - if not frappe.db.exists("Event", event.parent): - frappe.db.sql("delete from `tabEvent User` where parent = %s",event.parent) - else: - frappe.share.add("Event", event.parent, event.person, write=1) - - frappe.delete_doc("DocType", "Event User") - - # move note user to shared - if frappe.db.exists("DocType", "Note User"): - for note in frappe.get_all("Note User", fields=["parent", "user", "permission"]): - perm = {"read": 1} if note.permission=="Read" else {"write": 1} - if note.user in usernames: - frappe.share.add("Note", note.parent, note.user, **perm) - - frappe.delete_doc("DocType", "Note User") diff --git a/frappe/patches/v5_0/v4_to_v5.py b/frappe/patches/v5_0/v4_to_v5.py deleted file mode 100644 index cd34f04c97..0000000000 --- a/frappe/patches/v5_0/v4_to_v5.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors -# MIT License. See license.txt - -from __future__ import unicode_literals -import frappe - -def execute(): - changed = ( - ("desk", ("feed", "event", "todo", "note")), - ("custom", ("custom_field", "custom_script", "customize_form", - "customize_form_field", "property_setter")), - ("email", ("email_queue", "notification", "notification_recipient", "standard_reply")), - ("geo", ("country", "currency")), - ("print", ("letter_head", "print_format", "print_settings")) - ) - for module in changed: - for doctype in module[1]: - frappe.reload_doc(module[0], "doctype", doctype) diff --git a/frappe/patches/v5_2/__init__.py b/frappe/patches/v5_2/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/patches/v5_2/change_checks_to_not_null.py b/frappe/patches/v5_2/change_checks_to_not_null.py deleted file mode 100644 index 23f5d659b5..0000000000 --- a/frappe/patches/v5_2/change_checks_to_not_null.py +++ /dev/null @@ -1,34 +0,0 @@ -from __future__ import unicode_literals -import frappe -from frappe.utils import cint -from frappe.model import default_fields - -def execute(): - for table in frappe.db.get_tables(): - doctype = table[3:] - if frappe.db.exists("DocType", doctype): - fieldnames = [df["fieldname"] for df in - frappe.get_all("DocField", fields=["fieldname"], filters={"parent": doctype})] - custom_fieldnames = [df["fieldname"] for df in - frappe.get_all("Custom Field", fields=["fieldname"], filters={"dt": doctype})] - - else: - fieldnames = custom_fieldnames = [] - - for column in frappe.db.sql("""desc `{0}`""".format(table), as_dict=True): - if column["Type"]=="int(1)": - fieldname = column["Field"] - - # only change for defined fields, ignore old fields that don't exist in meta - if not (fieldname in default_fields or fieldname in fieldnames or fieldname in custom_fieldnames): - continue - - # set 0 - frappe.db.sql("""update `{table}` set `{column}`=0 where `{column}` is null"""\ - .format(table=table, column=fieldname)) - frappe.db.commit() - - # change definition - frappe.db.sql_ddl("""alter table `{table}` - modify `{column}` int(1) not null default {default}"""\ - .format(table=table, column=fieldname, default=cint(column["Default"]))) diff --git a/frappe/patches/v5_3/__init__.py b/frappe/patches/v5_3/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/patches/v5_3/rename_chinese_languages.py b/frappe/patches/v5_3/rename_chinese_languages.py deleted file mode 100644 index 8bc954c04c..0000000000 --- a/frappe/patches/v5_3/rename_chinese_languages.py +++ /dev/null @@ -1,13 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals -import frappe -from frappe.translate import rename_language - -def execute(): - language_map = { - "中国(简体)": "簡體中文", - "中國(繁體)": "正體中文" - } - - for old_name, new_name in language_map.items(): - rename_language(old_name, new_name) diff --git a/frappe/patches/v6_0/__init__.py b/frappe/patches/v6_0/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/patches/v6_0/communication_status_and_permission.py b/frappe/patches/v6_0/communication_status_and_permission.py deleted file mode 100644 index c68ed9b4d6..0000000000 --- a/frappe/patches/v6_0/communication_status_and_permission.py +++ /dev/null @@ -1,19 +0,0 @@ -from __future__ import unicode_literals -import frappe -from frappe.permissions import reset_perms - -def execute(): - frappe.reload_doctype("Communication") - - # set status = "Linked" - frappe.db.sql("""update `tabCommunication` set status='Linked' - where ifnull(reference_doctype, '')!='' and ifnull(reference_name, '')!=''""") - - frappe.db.sql("""update `tabCommunication` set status='Closed' - where status='Archived'""") - - # reset permissions if owner of all DocPerms is Administrator - if not frappe.db.sql("""select name from `tabDocPerm` - where parent='Communication' and ifnull(owner, '')!='Administrator'"""): - - reset_perms("Communication") diff --git a/frappe/patches/v6_0/document_type_rename.py b/frappe/patches/v6_0/document_type_rename.py deleted file mode 100644 index 16c7d34286..0000000000 --- a/frappe/patches/v6_0/document_type_rename.py +++ /dev/null @@ -1,8 +0,0 @@ -from __future__ import unicode_literals -import frappe - -def execute(): - frappe.db.sql("""update tabDocType set document_type='Document' - where document_type='Transaction'""") - frappe.db.sql("""update tabDocType set document_type='Setup' - where document_type='Master'""") diff --git a/frappe/patches/v6_0/fix_ghana_currency.py b/frappe/patches/v6_0/fix_ghana_currency.py deleted file mode 100644 index 67f740d240..0000000000 --- a/frappe/patches/v6_0/fix_ghana_currency.py +++ /dev/null @@ -1,8 +0,0 @@ -from __future__ import unicode_literals - -def execute(): - from frappe.geo.country_info import get_all - import frappe.utils.install - - countries = get_all() - frappe.utils.install.add_country_and_currency("Ghana", frappe._dict(countries["Ghana"])) diff --git a/frappe/patches/v6_0/make_task_log_folder.py b/frappe/patches/v6_0/make_task_log_folder.py deleted file mode 100644 index 87d6e4126f..0000000000 --- a/frappe/patches/v6_0/make_task_log_folder.py +++ /dev/null @@ -1,7 +0,0 @@ -from __future__ import unicode_literals -import frappe.utils, os - -def execute(): - path = frappe.utils.get_site_path('task-logs') - if not os.path.exists(path): - os.makedirs(path) diff --git a/frappe/patches/v6_1/__init__.py b/frappe/patches/v6_1/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/patches/v6_1/rename_file_data.py b/frappe/patches/v6_1/rename_file_data.py deleted file mode 100644 index 83152271eb..0000000000 --- a/frappe/patches/v6_1/rename_file_data.py +++ /dev/null @@ -1,38 +0,0 @@ -from __future__ import print_function, unicode_literals -import frappe - -def execute(): - from frappe.core.doctype.file.file import make_home_folder - - if not frappe.db.exists("DocType", "File"): - frappe.rename_doc("DocType", "File Data", "File") - frappe.reload_doctype("File") - - if not frappe.db.exists("File", {"is_home_folder": 1}): - make_home_folder() - - # make missing folders and set parent folder - for file in frappe.get_all("File", filters={"is_folder": 0}): - file = frappe.get_doc("File", file.name) - file.flags.ignore_folder_validate = True - file.flags.ignore_file_validate = True - file.flags.ignore_duplicate_entry_error = True - file.flags.ignore_links = True - file.set_folder_name() - try: - file.save() - except: - print(frappe.get_traceback()) - raise - - from frappe.utils.nestedset import rebuild_tree - rebuild_tree("File", "folder") - - # reset file size - for folder in frappe.db.sql("""select name from tabFile f1 where is_folder = 1 and - (select count(*) from tabFile f2 where f2.folder = f1.name and f2.is_folder = 1) = 0"""): - folder = frappe.get_doc("File", folder[0]) - folder.save() - - - diff --git a/frappe/patches/v6_11/__init__.py b/frappe/patches/v6_11/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/patches/v6_11/rename_field_in_email_account.py b/frappe/patches/v6_11/rename_field_in_email_account.py deleted file mode 100644 index 319b569802..0000000000 --- a/frappe/patches/v6_11/rename_field_in_email_account.py +++ /dev/null @@ -1,7 +0,0 @@ -from __future__ import unicode_literals -import frappe - -def execute(): - frappe.reload_doc("email", "doctype", "email_account") - if frappe.db.has_column('Email Account', 'pop3_server'): - frappe.db.sql("update `tabEmail Account` set email_server = pop3_server") diff --git a/frappe/patches/v6_15/__init__.py b/frappe/patches/v6_15/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/patches/v6_15/remove_property_setter_for_previous_field.py b/frappe/patches/v6_15/remove_property_setter_for_previous_field.py deleted file mode 100644 index b24bf38442..0000000000 --- a/frappe/patches/v6_15/remove_property_setter_for_previous_field.py +++ /dev/null @@ -1,87 +0,0 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors -# MIT License. See license.txt - -from __future__ import unicode_literals -import frappe, json -from frappe.utils import cstr - -def execute(): - # deprecated on 2016-03-09 - # using insert_after instead - return - - frappe.db.sql("""delete from `tabProperty Setter` where property='previous_field'""") - - all_custom_fields = frappe._dict() - for d in frappe.db.sql("""select name, dt, fieldname, insert_after from `tabCustom Field` - where insert_after is not null and insert_after != ''""", as_dict=1): - all_custom_fields.setdefault(d.dt, frappe._dict()).setdefault(d.fieldname, d.insert_after) - - for dt, custom_fields in all_custom_fields.items(): - _idx = [] - existing_ps = frappe.db.get_value("Property Setter", - {"doc_type": dt, "property": "_idx"}, ["name", "value", "creation"], as_dict=1) - - # if no existsing property setter, build based on meta - if not existing_ps: - _idx = get_sorted_fields(dt, custom_fields) - else: - _idx = json.loads(existing_ps.value) - - idx_needs_to_be_fixed = False - for fieldname, insert_after in custom_fields.items(): - # Delete existing property setter if field is not there - if fieldname not in _idx: - idx_needs_to_be_fixed = True - break - else: - previous_field = _idx[_idx.index(fieldname) - 1] - - if previous_field != insert_after and cstr(existing_ps.creation) >= "2015-12-28": - idx_needs_to_be_fixed = True - break - - if idx_needs_to_be_fixed: - frappe.delete_doc("Property Setter", existing_ps.name) - _idx = get_sorted_fields(dt, custom_fields) - - if _idx: - frappe.make_property_setter({ - "doctype":dt, - "doctype_or_field": "DocType", - "property": "_idx", - "value": json.dumps(_idx), - "property_type": "Text" - }, validate_fields_for_doctype=False) - - -def get_sorted_fields(doctype, custom_fields): - """sort on basis of insert_after""" - fields_dict = frappe.get_meta(doctype).get("fields") - - standard_fields_count = frappe.db.sql("""select count(name) from `tabDocField` - where parent=%s""", doctype)[0][0] - - newlist = [] - pending = [d.fieldname for d in fields_dict] - - maxloops = len(custom_fields) + 20 - while (pending and maxloops>0): - maxloops -= 1 - for fieldname in pending[:]: - if fieldname in custom_fields and len(newlist) >= standard_fields_count: - # field already added - for n in newlist: - if n==custom_fields.get(fieldname): - newlist.insert(newlist.index(n)+1, fieldname) - pending.remove(fieldname) - break - else: - newlist.append(fieldname) - pending.remove(fieldname) - - # recurring at end - if pending: - newlist += pending - - return newlist diff --git a/frappe/patches/v6_15/set_username.py b/frappe/patches/v6_15/set_username.py deleted file mode 100644 index 513ff3301d..0000000000 --- a/frappe/patches/v6_15/set_username.py +++ /dev/null @@ -1,16 +0,0 @@ -from __future__ import unicode_literals -import frappe - -def execute(): - frappe.reload_doctype("User") - - # give preference to System Users - users = frappe.db.sql_list("""select name from `tabUser` order by if(user_type='System User', 0, 1)""") - for name in users: - user = frappe.get_doc("User", name) - if user.username or not user.first_name: - continue - - username = user.suggest_username() - if username: - user.db_set("username", username, update_modified=False) diff --git a/frappe/patches/v6_16/__init__.py b/frappe/patches/v6_16/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/patches/v6_16/feed_doc_owner.py b/frappe/patches/v6_16/feed_doc_owner.py deleted file mode 100644 index 2dac9a143d..0000000000 --- a/frappe/patches/v6_16/feed_doc_owner.py +++ /dev/null @@ -1,31 +0,0 @@ -from __future__ import unicode_literals -import frappe - -def execute(): - frappe.reload_doctype("Communication") - - for doctype, name in frappe.db.sql("""select distinct reference_doctype, reference_name - from `tabCommunication` - where - (reference_doctype is not null and reference_doctype != '') - and (reference_name is not null and reference_name != '') - and (reference_owner is null or reference_owner = '') - for update"""): - - owner = frappe.db.get_value(doctype, name, "owner") - - if not owner: - continue - - frappe.db.sql("""update `tabCommunication` - set reference_owner=%(owner)s - where - reference_doctype=%(doctype)s - and reference_name=%(name)s - and (reference_owner is null or reference_owner = '')""".format(doctype=doctype), { - "doctype": doctype, - "name": name, - "owner": owner - }) - - frappe.db.commit() diff --git a/frappe/patches/v6_16/star_to_like.py b/frappe/patches/v6_16/star_to_like.py deleted file mode 100644 index e859223d54..0000000000 --- a/frappe/patches/v6_16/star_to_like.py +++ /dev/null @@ -1,15 +0,0 @@ -from __future__ import unicode_literals -import frappe -from frappe.database.schema import add_column - -def execute(): - frappe.db.sql("""update `tabSingles` set field='_liked_by' where field='_starred_by'""") - frappe.db.commit() - - for table in frappe.db.get_tables(): - columns = [r[0] for r in frappe.db.sql("DESC `{0}`".format(table))] - if "_starred_by" in columns and '_liked_by' not in columns: - frappe.db.sql_ddl("""alter table `{0}` change `_starred_by` `_liked_by` Text """.format(table)) - - if not frappe.db.has_column("Communication", "_liked_by"): - add_column("Communication", "_liked_by", "Text") diff --git a/frappe/patches/v6_19/__init__.py b/frappe/patches/v6_19/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/patches/v6_19/comment_feed_communication.py b/frappe/patches/v6_19/comment_feed_communication.py deleted file mode 100644 index a7503c08ab..0000000000 --- a/frappe/patches/v6_19/comment_feed_communication.py +++ /dev/null @@ -1,307 +0,0 @@ -from __future__ import unicode_literals -import frappe -from frappe import _ -from frappe.model.rename_doc import get_link_fields -from frappe.model.dynamic_links import dynamic_link_queries -from frappe.permissions import reset_perms - -def execute(): - # comments stay comments in v12 - return - - frappe.reload_doctype("DocType") - frappe.reload_doctype("Communication") - reset_perms("Communication") - - migrate_comments() - frappe.delete_doc("DocType", "Comment") - # frappe.db.sql_ddl("drop table `tabComment`") - - migrate_feed() - frappe.delete_doc("DocType", "Feed") - # frappe.db.sql_ddl("drop table `tabFeed`") - - update_timeline_doc_for("Blogger") - -def migrate_comments(): - from_fields = "" - to_fields = "" - - if "reference_doctype" in frappe.db.get_table_columns("Comment"): - from_fields = "reference_doctype as link_doctype, reference_name as link_name," - to_fields = "link_doctype, link_name," - - # comments - frappe.db.sql("""insert ignore into `tabCommunication` ( - subject, - content, - sender, - sender_full_name, - comment_type, - communication_date, - reference_doctype, - reference_name, - {to_fields} - - name, - user, - owner, - creation, - modified_by, - modified, - status, - sent_or_received, - communication_type, - seen - ) - select - substring(comment, 1, 100) as subject, - comment as content, - comment_by as sender, - comment_by_fullname as sender_full_name, - comment_type, - ifnull(timestamp(comment_date, comment_time), creation) as communication_date, - comment_doctype as reference_doctype, - comment_docname as reference_name, - {from_fields} - - name, - owner as user, - owner, - creation, - modified_by, - modified, - 'Linked' as status, - 'Sent' as sent_or_received, - 'Comment' as communication_type, - 1 as seen - from `tabComment` where comment_doctype is not null and comment_doctype not in ('Message', 'My Company')""" - .format(to_fields=to_fields, from_fields=from_fields)) - - # chat and assignment notifications - frappe.db.sql("""insert ignore into `tabCommunication` ( - subject, - content, - sender, - sender_full_name, - comment_type, - communication_date, - reference_doctype, - reference_name, - {to_fields} - - name, - user, - owner, - creation, - modified_by, - modified, - status, - sent_or_received, - communication_type, - seen - ) - select - case - when parenttype='Assignment' then %(assignment)s - else substring(comment, 1, 100) - end - as subject, - comment as content, - comment_by as sender, - comment_by_fullname as sender_full_name, - comment_type, - ifnull(timestamp(comment_date, comment_time), creation) as communication_date, - 'User' as reference_doctype, - comment_docname as reference_name, - {from_fields} - - name, - owner as user, - owner, - creation, - modified_by, - modified, - 'Linked' as status, - 'Sent' as sent_or_received, - case - when parenttype='Assignment' then 'Notification' - else 'Chat' - end - as communication_type, - 1 as seen - from `tabComment` where comment_doctype in ('Message', 'My Company')""" - .format(to_fields=to_fields, from_fields=from_fields), {"assignment": _("Assignment")}) - -def migrate_feed(): - # migrate delete feed - for doctype in frappe.db.sql("""select distinct doc_type from `tabFeed` where subject=%(deleted)s""", {"deleted": _("Deleted")}): - frappe.db.sql("""insert ignore into `tabCommunication` ( - subject, - sender, - sender_full_name, - comment_type, - communication_date, - reference_doctype, - - name, - user, - owner, - creation, - modified_by, - modified, - status, - sent_or_received, - communication_type, - seen - ) - select - concat_ws(" ", %(_doctype)s, doc_name) as subject, - owner as sender, - full_name as sender_full_name, - 'Deleted' as comment_type, - creation as communication_date, - doc_type as reference_doctype, - - name, - owner as user, - owner, - creation, - modified_by, - modified, - 'Linked' as status, - 'Sent' as sent_or_received, - 'Comment' as communication_type, - 1 as seen - from `tabFeed` where subject=%(deleted)s and doc_type=%(doctype)s""", { - "deleted": _("Deleted"), - "doctype": doctype, - "_doctype": _(doctype) - }) - - # migrate feed type login or empty - frappe.db.sql("""insert ignore into `tabCommunication` ( - subject, - sender, - sender_full_name, - comment_type, - communication_date, - reference_doctype, - reference_name, - - name, - user, - owner, - creation, - modified_by, - modified, - status, - sent_or_received, - communication_type, - seen - ) - select - subject, - owner as sender, - full_name as sender_full_name, - case - when feed_type='Login' then 'Info' - else 'Updated' - end as comment_type, - creation as communication_date, - doc_type as reference_doctype, - doc_name as reference_name, - - name, - owner as user, - owner, - creation, - modified_by, - modified, - 'Linked' as status, - 'Sent' as sent_or_received, - 'Comment' as communication_type, - 1 as seen - from `tabFeed` where (feed_type in ('Login', '') or feed_type is null)""") - -def update_timeline_doc_for(timeline_doctype): - """NOTE: This method may be used by other apps for patching. It also has COMMIT after each update.""" - - # find linked doctypes - # link fields - update_for_linked_docs(timeline_doctype) - - # dynamic link fields - update_for_dynamically_linked_docs(timeline_doctype) - -def update_for_linked_docs(timeline_doctype): - for df in get_link_fields(timeline_doctype): - if df.issingle: - continue - - reference_doctype = df.parent - - if not is_valid_timeline_doctype(reference_doctype, timeline_doctype): - continue - - for doc in frappe.get_all(reference_doctype, fields=["name", df.fieldname]): - timeline_name = doc.get(df.fieldname) - update_communication(timeline_doctype, timeline_name, reference_doctype, doc.name) - -def update_for_dynamically_linked_docs(timeline_doctype): - dynamic_link_fields = [] - for query in dynamic_link_queries: - for df in frappe.db.sql(query, as_dict=True): - dynamic_link_fields.append(df) - - for df in dynamic_link_fields: - reference_doctype = df.parent - - if not is_valid_timeline_doctype(reference_doctype, timeline_doctype): - continue - - try: - docs = frappe.get_all(reference_doctype, fields=["name", df.fieldname], - filters={ df.options: timeline_doctype }) - except frappe.db.SQLError as e: - if frappe.db.is_table_missing(e): - # single - continue - else: - raise - - for doc in docs: - timeline_name = doc.get(df.fieldname) - update_communication(timeline_doctype, timeline_name, reference_doctype, doc.name) - -def update_communication(timeline_doctype, timeline_name, reference_doctype, reference_name): - if not timeline_name: - return - - frappe.db.sql("""update `tabCommunication` set timeline_doctype=%(timeline_doctype)s, timeline_name=%(timeline_name)s - where (reference_doctype=%(reference_doctype)s and reference_name=%(reference_name)s) - and (timeline_doctype is null or timeline_doctype='') - and (timeline_name is null or timeline_name='')""", { - "timeline_doctype": timeline_doctype, - "timeline_name": timeline_name, - "reference_doctype": reference_doctype, - "reference_name": reference_name - }) - - frappe.db.commit() - -def is_valid_timeline_doctype(reference_doctype, timeline_doctype): - # for reloading timeline_field - frappe.reload_doctype(reference_doctype) - - # make sure the timeline field's doctype is same as timeline doctype - meta = frappe.get_meta(reference_doctype) - if not meta.timeline_field: - return False - - doctype = meta.get_link_doctype(meta.timeline_field) - if doctype != timeline_doctype: - return False - - - return True diff --git a/frappe/patches/v6_2/__init__.py b/frappe/patches/v6_2/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/patches/v6_2/ignore_user_permissions_if_missing.py b/frappe/patches/v6_2/ignore_user_permissions_if_missing.py deleted file mode 100644 index 356d28989a..0000000000 --- a/frappe/patches/v6_2/ignore_user_permissions_if_missing.py +++ /dev/null @@ -1,8 +0,0 @@ -from __future__ import unicode_literals -import frappe - -def execute(): - frappe.reload_doctype("System Settings") - system_settings = frappe.get_doc("System Settings") - system_settings.flags.ignore_mandatory = 1 - system_settings.save() diff --git a/frappe/patches/v6_2/rename_backup_manager.py b/frappe/patches/v6_2/rename_backup_manager.py deleted file mode 100644 index af02e55878..0000000000 --- a/frappe/patches/v6_2/rename_backup_manager.py +++ /dev/null @@ -1,20 +0,0 @@ -from __future__ import unicode_literals -import frappe - -def execute(): - unset = False - frappe.reload_doc("integrations", "doctype", "dropbox_backup") - - dropbox_backup = frappe.get_doc("Dropbox Backup", "Dropbox Backup") - for df in dropbox_backup.meta.fields: - value = frappe.db.get_single_value("Backup Manager", df.fieldname) - if value: - if df.fieldname=="upload_backups_to_dropbox" and value=="Never": - value = "Daily" - unset = True - dropbox_backup.set(df.fieldname, value) - - if unset: - dropbox_backup.set("send_backups_to_dropbox", 0) - - dropbox_backup.save() diff --git a/frappe/patches/v6_20x/__init__.py b/frappe/patches/v6_20x/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/patches/v6_20x/remove_roles_from_website_user.py b/frappe/patches/v6_20x/remove_roles_from_website_user.py deleted file mode 100644 index a4d579a1f0..0000000000 --- a/frappe/patches/v6_20x/remove_roles_from_website_user.py +++ /dev/null @@ -1,11 +0,0 @@ -from __future__ import unicode_literals -import frappe - -def execute(): - frappe.reload_doc("core", "doctype", "user_email") - frappe.reload_doc("core", "doctype", "user") - for user_name in frappe.get_all('User', filters={'user_type': 'Website User'}): - user = frappe.get_doc('User', user_name) - if user.roles: - user.roles = [] - user.save() diff --git a/frappe/patches/v6_20x/set_allow_draft_for_print.py b/frappe/patches/v6_20x/set_allow_draft_for_print.py deleted file mode 100644 index 90c15e22b2..0000000000 --- a/frappe/patches/v6_20x/set_allow_draft_for_print.py +++ /dev/null @@ -1,5 +0,0 @@ -from __future__ import unicode_literals -import frappe - -def execute(): - frappe.db.set_value("Print Settings", "Print Settings", "allow_print_for_draft", 1) \ No newline at end of file diff --git a/frappe/patches/v6_20x/update_insert_after.py b/frappe/patches/v6_20x/update_insert_after.py deleted file mode 100644 index 5ebec52fc9..0000000000 --- a/frappe/patches/v6_20x/update_insert_after.py +++ /dev/null @@ -1,27 +0,0 @@ -from __future__ import unicode_literals -import frappe, json - -def execute(): - for ps in frappe.get_all('Property Setter', filters={'property': '_idx'}, - fields = ['doc_type', 'value']): - custom_fields = frappe.get_all('Custom Field', - filters = {'dt': ps.doc_type}, fields=['name', 'fieldname']) - - if custom_fields: - _idx = json.loads(ps.value) - - for custom_field in custom_fields: - if custom_field.fieldname in _idx: - custom_field_idx = _idx.index(custom_field.fieldname) - if custom_field_idx == 0: - prev_fieldname = "" - - else: - prev_fieldname = _idx[custom_field_idx - 1] - - else: - prev_fieldname = _idx[-1] - custom_field_idx = len(_idx) - - frappe.db.set_value('Custom Field', custom_field.name, 'insert_after', prev_fieldname) - frappe.db.set_value('Custom Field', custom_field.name, 'idx', custom_field_idx) diff --git a/frappe/patches/v6_21/__init__.py b/frappe/patches/v6_21/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/patches/v6_21/print_settings_repeat_header_footer.py b/frappe/patches/v6_21/print_settings_repeat_header_footer.py deleted file mode 100644 index 941a145a54..0000000000 --- a/frappe/patches/v6_21/print_settings_repeat_header_footer.py +++ /dev/null @@ -1,6 +0,0 @@ -from __future__ import unicode_literals -import frappe - -def execute(): - frappe.reload_doctype('Print Settings') - frappe.db.set_value('Print Settings', 'Print Settings', 'repeat_header_footer', 1) diff --git a/frappe/patches/v6_24/__init__.py b/frappe/patches/v6_24/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/patches/v6_24/set_language_as_code.py b/frappe/patches/v6_24/set_language_as_code.py deleted file mode 100644 index d685fd7d0e..0000000000 --- a/frappe/patches/v6_24/set_language_as_code.py +++ /dev/null @@ -1,8 +0,0 @@ -from __future__ import unicode_literals -import frappe - -from frappe.translate import get_lang_dict - -# migrate language from name to code -def execute(): - return diff --git a/frappe/patches/v6_4/__init__.py b/frappe/patches/v6_4/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/patches/v6_4/reduce_varchar_length.py b/frappe/patches/v6_4/reduce_varchar_length.py deleted file mode 100644 index 93a8be8c92..0000000000 --- a/frappe/patches/v6_4/reduce_varchar_length.py +++ /dev/null @@ -1,37 +0,0 @@ -from __future__ import unicode_literals, print_function -import frappe - -def execute(): - for doctype in frappe.get_all("DocType", filters={"issingle": 0}): - doctype = doctype.name - if not frappe.db.table_exists(doctype): - continue - - for column in frappe.db.sql("desc `tab{doctype}`".format(doctype=doctype), as_dict=True): - fieldname = column["Field"] - column_type = column["Type"] - - if not column_type.startswith("varchar"): - continue - - max_length = frappe.db.sql("""select max(char_length(`{fieldname}`)) from `tab{doctype}`"""\ - .format(fieldname=fieldname, doctype=doctype)) - - max_length = max_length[0][0] if max_length else None - - if max_length and 140 < max_length <= 255: - print( - "setting length of '{fieldname}' in '{doctype}' as {length}".format( - fieldname=fieldname, doctype=doctype, length=max_length) - ) - - # create property setter for length - frappe.make_property_setter({ - "doctype": doctype, - "fieldname": fieldname, - "property": "length", - "value": max_length, - "property_type": "Int" - }) - - frappe.clear_cache(doctype=doctype) diff --git a/frappe/patches/v6_4/rename_bengali_language.py b/frappe/patches/v6_4/rename_bengali_language.py deleted file mode 100644 index dbbcb62f8d..0000000000 --- a/frappe/patches/v6_4/rename_bengali_language.py +++ /dev/null @@ -1,7 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals -import frappe -from frappe.translate import rename_language - -def execute(): - rename_language("বাঙালি", "বাংলা") \ No newline at end of file diff --git a/frappe/patches/v6_6/__init__.py b/frappe/patches/v6_6/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/patches/v6_6/fix_file_url.py b/frappe/patches/v6_6/fix_file_url.py deleted file mode 100644 index 4f8956d343..0000000000 --- a/frappe/patches/v6_6/fix_file_url.py +++ /dev/null @@ -1,36 +0,0 @@ -from __future__ import unicode_literals -import frappe -from frappe.model.meta import is_single - -def execute(): - """Fix old style file urls that start with files/""" - fix_file_urls() - fix_attach_field_urls() - -def fix_file_urls(): - for file in frappe.db.sql_list("""select name from `tabFile` where file_url like 'files/%'"""): - file = frappe.get_doc("File", file) - file.db_set("file_url", "/" + file.file_url, update_modified=False) - try: - file.validate_file() - file.db_set("file_name", file.file_name, update_modified=False) - if not file.content_hash: - file.generate_content_hash() - file.db_set("content_hash", file.content_hash, update_modified=False) - - except IOError: - pass - -def fix_attach_field_urls(): - # taken from an old patch - attach_fields = (frappe.db.sql("""select parent, fieldname from `tabDocField` where fieldtype in ('Attach', 'Attach Image')""") + - frappe.db.sql("""select dt, fieldname from `tabCustom Field` where fieldtype in ('Attach', 'Attach Image')""")) - - for doctype, fieldname in attach_fields: - if is_single(doctype): - frappe.db.sql("""update `tabSingles` set value=concat("/", `value`) - where doctype=%(doctype)s and field=%(fieldname)s - and value like 'files/%%'""", {"doctype": doctype, "fieldname": fieldname}) - else: - frappe.db.sql("""update `tab{doctype}` set `{fieldname}`=concat("/", `{fieldname}`) - where `{fieldname}` like 'files/%'""".format(doctype=doctype, fieldname=fieldname)) diff --git a/frappe/patches/v6_6/rename_slovak_language.py b/frappe/patches/v6_6/rename_slovak_language.py deleted file mode 100644 index a942543372..0000000000 --- a/frappe/patches/v6_6/rename_slovak_language.py +++ /dev/null @@ -1,7 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals -import frappe -from frappe.translate import rename_language - -def execute(): - rename_language("slovenčina", "slovenčina (Slovak)") diff --git a/frappe/patches/v6_6/user_last_active.py b/frappe/patches/v6_6/user_last_active.py deleted file mode 100644 index fd55935174..0000000000 --- a/frappe/patches/v6_6/user_last_active.py +++ /dev/null @@ -1,6 +0,0 @@ -from __future__ import unicode_literals -import frappe - -def execute(): - frappe.reload_doctype("User") - frappe.db.sql("update `tabUser` set last_active=last_login") diff --git a/frappe/patches/v6_9/__init__.py b/frappe/patches/v6_9/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/patches/v6_9/int_float_not_null.py b/frappe/patches/v6_9/int_float_not_null.py deleted file mode 100644 index 97495f9077..0000000000 --- a/frappe/patches/v6_9/int_float_not_null.py +++ /dev/null @@ -1,30 +0,0 @@ -from __future__ import unicode_literals -import frappe -from frappe.utils import cint, flt - -def execute(): - for doctype in frappe.get_all("DocType", filters={"issingle": 0}): - doctype = doctype.name - meta = frappe.get_meta(doctype) - - for column in frappe.db.sql("desc `tab{doctype}`".format(doctype=doctype), as_dict=True): - fieldname = column["Field"] - column_type = column["Type"] - - if not (column_type.startswith("int") or column_type.startswith("decimal")): - continue - - frappe.db.sql("""update `tab{doctype}` set `{fieldname}`=0 where `{fieldname}` is null"""\ - .format(doctype=doctype, fieldname=fieldname)) - - # alter table - if column["Null"]=='YES': - if not meta.get_field(fieldname): - continue - - default = cint(column["Default"]) if column_type.startswith("int") else flt(column["Default"]) - frappe.db.sql_ddl("""alter table `tab{doctype}` - change `{fieldname}` `{fieldname}` {column_type} not null default '{default}'""".format( - doctype=doctype, fieldname=fieldname, column_type=column_type, default=default)) - - diff --git a/frappe/patches/v6_9/rename_burmese_language.py b/frappe/patches/v6_9/rename_burmese_language.py deleted file mode 100644 index 66477f7efe..0000000000 --- a/frappe/patches/v6_9/rename_burmese_language.py +++ /dev/null @@ -1,7 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals -import frappe -from frappe.translate import rename_language - -def execute(): - rename_language("Melayu", "မြန်မာ") diff --git a/frappe/patches/v7_0/__init__.py b/frappe/patches/v7_0/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/patches/v7_0/add_communication_in_doc.py b/frappe/patches/v7_0/add_communication_in_doc.py deleted file mode 100644 index 4db02c5bab..0000000000 --- a/frappe/patches/v7_0/add_communication_in_doc.py +++ /dev/null @@ -1,14 +0,0 @@ -from __future__ import unicode_literals -import frappe - -from frappe.core.doctype.comment.comment import update_comment_in_doc - -def execute(): - for d in frappe.db.get_all("Communication", - fields = ['name', 'reference_doctype', 'reference_name', 'SUBSTRING(content,1,102)', 'communication_type'], - filters = {"reference_name":None,"reference_doctype":None,'communication_type': 'Communication'}): - - try: - update_comment_in_doc(d) - except frappe.ImplicitCommitError: - pass diff --git a/frappe/patches/v7_0/cleanup_list_settings.py b/frappe/patches/v7_0/cleanup_list_settings.py deleted file mode 100644 index e03ff57406..0000000000 --- a/frappe/patches/v7_0/cleanup_list_settings.py +++ /dev/null @@ -1,20 +0,0 @@ -from __future__ import unicode_literals -import frappe, json - -def execute(): - if frappe.db.table_exists("__ListSettings"): - list_settings = frappe.db.sql("select user, doctype, data from __ListSettings", as_dict=1) - for ls in list_settings: - if ls and ls.data: - data = json.loads(ls.data) - if "fields" not in data: - continue - fields = data["fields"] - for field in fields: - if "name as" in field: - fields.remove(field) - data["fields"] = fields - - frappe.db.sql("update __ListSettings set data = %s where user=%s and doctype=%s", - (json.dumps(data), ls.user, ls.doctype)) - diff --git a/frappe/patches/v7_0/create_private_file_folder.py b/frappe/patches/v7_0/create_private_file_folder.py deleted file mode 100644 index bd26917a78..0000000000 --- a/frappe/patches/v7_0/create_private_file_folder.py +++ /dev/null @@ -1,6 +0,0 @@ -from __future__ import unicode_literals -import frappe, os - -def execute(): - if not os.path.exists(os.path.join(frappe.local.site_path, 'private', 'files')): - frappe.create_folder(os.path.join(frappe.local.site_path, 'private', 'files')) \ No newline at end of file diff --git a/frappe/patches/v7_0/re_route.py b/frappe/patches/v7_0/re_route.py deleted file mode 100644 index cc36594ae8..0000000000 --- a/frappe/patches/v7_0/re_route.py +++ /dev/null @@ -1,23 +0,0 @@ -from __future__ import unicode_literals -import frappe -from frappe.model.base_document import get_controller - -def execute(): - update_routes(['Blog Post', 'Blog Category', 'Web Page']) - -def update_routes(doctypes): - """Patch old routing system""" - for d in doctypes: - frappe.reload_doctype(d) - c = get_controller(d) - - condition = '' - if c.website.condition_field: - condition = 'where {0}=1'.format(c.website.condition_field) - - try: - frappe.db.sql("""update ignore `tab{0}` set route = concat(ifnull(parent_website_route, ""), - if(ifnull(parent_website_route, "")="", "", "/"), page_name) {1}""".format(d, condition)) - - except Exception as e: - if not frappe.db.is_missing_column(e): raise diff --git a/frappe/patches/v7_0/rename_bulk_email_to_email_queue.py b/frappe/patches/v7_0/rename_bulk_email_to_email_queue.py deleted file mode 100644 index 9a7a756144..0000000000 --- a/frappe/patches/v7_0/rename_bulk_email_to_email_queue.py +++ /dev/null @@ -1,5 +0,0 @@ -from __future__ import unicode_literals -import frappe - -def execute(): - frappe.rename_doc('DocType', 'Bulk Email', 'Email Queue') \ No newline at end of file diff --git a/frappe/patches/v7_0/rename_newsletter_list_to_email_group.py b/frappe/patches/v7_0/rename_newsletter_list_to_email_group.py deleted file mode 100644 index 79061d383c..0000000000 --- a/frappe/patches/v7_0/rename_newsletter_list_to_email_group.py +++ /dev/null @@ -1,6 +0,0 @@ -from __future__ import unicode_literals -import frappe - -def execute(): - frappe.rename_doc('DocType', 'Newsletter List', 'Email Group') - frappe.rename_doc('DocType', 'Newsletter List Subscriber', 'Email Group Member') \ No newline at end of file diff --git a/frappe/patches/v7_0/set_email_group.py b/frappe/patches/v7_0/set_email_group.py deleted file mode 100644 index e3dd66ebf3..0000000000 --- a/frappe/patches/v7_0/set_email_group.py +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors -# License: GNU General Public License v3. See license.txt - -from __future__ import unicode_literals -import frappe - -def execute(): - frappe.reload_doc("email", "doctype", "email_group_member") - if "newsletter_list" in frappe.db.get_table_columns("Email Group Member"): - frappe.db.sql("""update `tabEmail Group Member` set email_group = newsletter_list - where email_group is null or email_group = ''""") \ No newline at end of file diff --git a/frappe/patches/v7_0/set_user_fullname.py b/frappe/patches/v7_0/set_user_fullname.py deleted file mode 100644 index a7c6670f45..0000000000 --- a/frappe/patches/v7_0/set_user_fullname.py +++ /dev/null @@ -1,10 +0,0 @@ -from __future__ import unicode_literals -import frappe - -def execute(): - frappe.reload_doc("Core", "DocType", "User") - - for user in frappe.db.get_all('User'): - user = frappe.get_doc('User', user.name) - user.set_full_name() - user.db_set('full_name', user.full_name, update_modified = False) \ No newline at end of file diff --git a/frappe/patches/v7_0/update_auth.py b/frappe/patches/v7_0/update_auth.py deleted file mode 100644 index 3d47edf4b5..0000000000 --- a/frappe/patches/v7_0/update_auth.py +++ /dev/null @@ -1,42 +0,0 @@ -from __future__ import unicode_literals -import frappe -from frappe.utils.password import create_auth_table, set_encrypted_password - -def execute(): - if '__OldAuth' not in frappe.db.get_tables(): - frappe.db.sql_ddl('''alter table `__Auth` rename `__OldAuth`''') - - create_auth_table() - - # user passwords - frappe.db.sql('''insert ignore into `__Auth` (doctype, name, fieldname, `password`) - (select 'User', `name`, 'password', `password` from `__OldAuth`)''') - - frappe.db.commit() - - # other password fields - for doctype in frappe.db.sql_list('''select distinct parent from `tabDocField` - where fieldtype="Password" and parent != "User"'''): - - frappe.reload_doctype(doctype) - meta = frappe.get_meta(doctype) - - for df in meta.get('fields', {'fieldtype': 'Password'}): - if meta.issingle: - password = frappe.db.get_value(doctype, doctype, df.fieldname) - if password: - set_encrypted_password(doctype, doctype, password, fieldname=df.fieldname) - frappe.db.set_value(doctype, doctype, df.fieldname, '*'*len(password)) - - else: - for d in frappe.db.sql('''select name, `{fieldname}` from `tab{doctype}` - where `{fieldname}` is not null'''.format(fieldname=df.fieldname, doctype=doctype), as_dict=True): - - set_encrypted_password(doctype, d.name, d.get(df.fieldname), fieldname=df.fieldname) - - frappe.db.sql('''update `tab{doctype}` set `{fieldname}`=repeat("*", char_length(`{fieldname}`))''' - .format(doctype=doctype, fieldname=df.fieldname)) - - frappe.db.commit() - - frappe.db.sql_ddl('''drop table `__OldAuth`''') diff --git a/frappe/patches/v7_0/update_report_builder_json.py b/frappe/patches/v7_0/update_report_builder_json.py deleted file mode 100644 index a344ca5412..0000000000 --- a/frappe/patches/v7_0/update_report_builder_json.py +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright (c) 2013, Web Notes Technologies Pvt. Ltd. and Contributors -# License: GNU General Public License v3. See license.txt - -from __future__ import unicode_literals -import frappe - -def execute(): - for report in frappe.db.sql_list(""" select name from `tabReport` where report_type = 'Report Builder' - and is_standard = 'No' and `json` != '' and `json` is not null """): - doc = frappe.get_doc("Report", report) - doc.update_report_json() - doc.db_set("json", doc.json, update_modified=False) \ No newline at end of file diff --git a/frappe/patches/v7_0/update_send_after_in_bulk_email.py b/frappe/patches/v7_0/update_send_after_in_bulk_email.py deleted file mode 100644 index 1b08309b6a..0000000000 --- a/frappe/patches/v7_0/update_send_after_in_bulk_email.py +++ /dev/null @@ -1,6 +0,0 @@ -from __future__ import unicode_literals -import frappe -from frappe.utils import now_datetime - -def execute(): - frappe.db.sql('update `tabEmail Queue` set send_after=%s where send_after is null', now_datetime()) \ No newline at end of file diff --git a/frappe/patches/v7_1/__init__.py b/frappe/patches/v7_1/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/patches/v7_1/disabled_print_settings_for_custom_print_format.py b/frappe/patches/v7_1/disabled_print_settings_for_custom_print_format.py deleted file mode 100644 index c74d2d98f9..0000000000 --- a/frappe/patches/v7_1/disabled_print_settings_for_custom_print_format.py +++ /dev/null @@ -1,17 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors -# License: GNU General Public License v3. See license.txt - -from __future__ import unicode_literals -import frappe - -def execute(): - frappe.reload_doctype('Print Format') - frappe.db.sql(""" - update - `tabPrint Format` - set - align_labels_right = 0, line_breaks = 0, show_section_headings = 0 - where - custom_format = 1 - """) diff --git a/frappe/patches/v7_1/refactor_integration_broker.py b/frappe/patches/v7_1/refactor_integration_broker.py deleted file mode 100644 index 8c9aaa6795..0000000000 --- a/frappe/patches/v7_1/refactor_integration_broker.py +++ /dev/null @@ -1,52 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors -# License: GNU General Public License v3. See license.txt - -from __future__ import unicode_literals -import frappe -import json - -def execute(): - for doctype_name in ["Razorpay Log", "Razorpay Payment", "Razorpay Settings"]: - delete_doc("DocType", doctype_name) - - reload_doctypes() - setup_services() - -def delete_doc(doctype, doctype_name): - frappe.delete_doc(doctype, doctype_name) - -def reload_doctypes(): - for doctype in ("razorpay_settings", "paypal_settings", "dropbox_settings", "ldap_settings"): - frappe.reload_doc("integrations", "doctype", doctype) - -def setup_services(): - for service in [{"old_name": "Razorpay", "new_name": "Razorpay"}, - {"old_name": "PayPal", "new_name": "PayPal"}, - {"old_name": "Dropbox Integration", "new_name": "Dropbox"}, - {"old_name": "LDAP Auth", "new_name": "LDAP"}]: - - try: - service_doc = frappe.get_doc("Integration Service", service["old_name"]) - settings = json.loads(service_doc.custom_settings_json) - - service_settings = frappe.new_doc("{0} Settings".format(service["new_name"])) - service_settings.update(settings) - - service_settings.flags.ignore_mandatory = True - service_settings.save(ignore_permissions=True) - - if service["old_name"] in ["Dropbox Integration", "LDAP Auth"]: - delete_doc("Integration Service", service["old_name"]) - - new_service_doc = frappe.get_doc({ - "doctype": "Integration Service", - "service": service["new_name"], - "enabled": 1 - }) - - new_service_doc.flags.ignore_mandatory = True - new_service_doc.save(ignore_permissions=True) - - except Exception: - pass diff --git a/frappe/patches/v7_1/rename_chinese_language_codes.py b/frappe/patches/v7_1/rename_chinese_language_codes.py deleted file mode 100644 index 1ed25a4959..0000000000 --- a/frappe/patches/v7_1/rename_chinese_language_codes.py +++ /dev/null @@ -1,11 +0,0 @@ -from __future__ import unicode_literals -import frappe - -def execute(): - frappe.rename_doc('Language', 'zh-cn', 'zh', force=True, - merge=True if frappe.db.exists('Language', 'zh') else False) - if frappe.db.get_value('Language', 'zh-tw') == 'zh-tw': - frappe.rename_doc('Language', 'zh-tw', 'zh-TW', force=True) - - frappe.db.set_value('Language', 'zh', 'language_code', 'zh') - frappe.db.set_value('Language', 'zh-TW', 'language_code', 'zh-TW') \ No newline at end of file diff --git a/frappe/patches/v7_1/rename_scheduler_log_to_error_log.py b/frappe/patches/v7_1/rename_scheduler_log_to_error_log.py deleted file mode 100644 index 4d1a39538f..0000000000 --- a/frappe/patches/v7_1/rename_scheduler_log_to_error_log.py +++ /dev/null @@ -1,11 +0,0 @@ -from __future__ import unicode_literals -import frappe - -def execute(): - if not 'tabError Log' in frappe.db.get_tables(): - frappe.rename_doc('DocType', 'Scheduler Log', 'Error Log') - frappe.db.sql("""delete from `tabError Log` where datediff(curdate(), creation) > 30""") - frappe.db.commit() - frappe.db.sql('alter table `tabError Log` change column name name varchar(140)') - frappe.db.sql('alter table `tabError Log` change column parent parent varchar(140)') - frappe.db.sql('alter table `tabError Log` engine=MyISAM') diff --git a/frappe/patches/v7_1/set_backup_limit.py b/frappe/patches/v7_1/set_backup_limit.py deleted file mode 100644 index 7b0a344305..0000000000 --- a/frappe/patches/v7_1/set_backup_limit.py +++ /dev/null @@ -1,10 +0,0 @@ -from __future__ import unicode_literals -from frappe.utils import cint -import frappe - -def execute(): - frappe.reload_doctype('System Settings') - backup_limit = frappe.db.get_single_value('System Settings', 'backup_limit') - - if cint(backup_limit) == 0: - frappe.db.set_value('System Settings', 'System Settings', 'backup_limit', 3) diff --git a/frappe/patches/v7_1/setup_integration_services.py b/frappe/patches/v7_1/setup_integration_services.py deleted file mode 100644 index 1c70b8e835..0000000000 --- a/frappe/patches/v7_1/setup_integration_services.py +++ /dev/null @@ -1,118 +0,0 @@ -from __future__ import unicode_literals -import frappe -from frappe.exceptions import DataError -from frappe.utils.password import get_decrypted_password -from frappe.utils import cstr -import os - -app_list = [ - {"app_name": "razorpay_integration", "service_name": "Razorpay", "doctype": "Razorpay Settings", "remove": True}, - {"app_name": "paypal_integration", "service_name": "PayPal", "doctype": "PayPal Settings", "remove": True}, - {"app_name": "frappe", "service_name": "Dropbox", "doctype": "Dropbox Backup", "remove": False} -] - -def execute(): - installed_apps = frappe.get_installed_apps() - - for app_details in app_list: - if app_details["app_name"] in installed_apps: - settings = get_app_settings(app_details) - if app_details["remove"]: - uninstall_app(app_details["app_name"]) - - try: - setup_integration_service(app_details, settings) - except DataError: - pass - - frappe.delete_doc("DocType", "Dropbox Backup") - -def setup_integration_service(app_details, settings=None): - if not settings: - return - - setup_service_settings(app_details["service_name"], settings) - - doc_path = frappe.get_app_path("frappe", "integration_broker", "doctype", - "integration_service", "integration_service.json") - - if not os.path.exists(doc_path): - return - - frappe.reload_doc("integration_broker", "doctype", "integration_service") - - if frappe.db.exists("Integration Service", app_details["service_name"]): - integration_service = frappe.get_doc("Integration Service", app_details["service_name"]) - else: - integration_service = frappe.new_doc("Integration Service") - integration_service.service = app_details["service_name"] - - integration_service.enabled = 1 - integration_service.flags.ignore_mandatory = True - integration_service.save(ignore_permissions=True) - -def get_app_settings(app_details): - parameters = {} - doctype = docname = app_details["doctype"] - - app_settings = get_parameters(app_details) - if app_settings: - settings = app_settings["settings"] - frappe.reload_doc("integrations", "doctype", "{0}_settings".format(app_details["service_name"].lower())) - controller = frappe.get_meta("{0} Settings".format(app_details["service_name"])) - - for d in controller.fields: - if settings.get(d.fieldname): - if ''.join(set(cstr(settings.get(d.fieldname)))) == '*': - setattr(settings, d.fieldname, get_decrypted_password(doctype, docname, d.fieldname, raise_exception=True)) - - parameters.update({d.fieldname : settings.get(d.fieldname)}) - - return parameters - -def uninstall_app(app_name): - from frappe.installer import remove_from_installed_apps - remove_from_installed_apps(app_name) - -def get_parameters(app_details): - if app_details["service_name"] == "Razorpay": - return {"settings": frappe.get_doc(app_details["doctype"])} - - elif app_details["service_name"] == "PayPal": - if frappe.conf.paypal_username and frappe.conf.paypal_password and frappe.conf.paypal_signature: - return { - "settings": { - "api_username": frappe.conf.paypal_username, - "api_password": frappe.conf.paypal_password, - "signature": frappe.conf.paypal_signature - } - } - else: - return {"settings": frappe.get_doc(app_details["doctype"])} - - elif app_details["service_name"] == "Dropbox": - doc = frappe.db.get_value(app_details["doctype"], None, - ["dropbox_access_key", "dropbox_access_secret", "upload_backups_to_dropbox"], as_dict=1) - - if not doc: - return - - if not (frappe.conf.dropbox_access_key and frappe.conf.dropbox_secret_key): - return - - return { - "settings": { - "app_access_key": frappe.conf.dropbox_access_key, - "app_secret_key": frappe.conf.dropbox_secret_key, - "dropbox_access_key": doc.dropbox_access_key, - "dropbox_access_secret": doc.dropbox_access_secret, - "backup_frequency": doc.upload_backups_to_dropbox, - "enabled": doc.send_backups_to_dropbox - } - } - -def setup_service_settings(service_name, settings): - service_doc = frappe.get_doc("{0} Settings".format(service_name)) - service_doc.update(settings) - service_doc.flags.ignore_mandatory = True - service_doc.save(ignore_permissions=True) \ No newline at end of file diff --git a/frappe/patches/v7_1/sync_language_doctype.py b/frappe/patches/v7_1/sync_language_doctype.py deleted file mode 100644 index 83d1a4f5a6..0000000000 --- a/frappe/patches/v7_1/sync_language_doctype.py +++ /dev/null @@ -1,22 +0,0 @@ -from __future__ import unicode_literals -import frappe -from frappe.translate import get_lang_dict - -def execute(): - frappe.reload_doc('core', 'doctype', 'language') - - from frappe.core.doctype.language.language import sync_languages - sync_languages() - - # move language from old style to new style for old accounts - # i.e. from "english" to "en" - - lang_dict = get_lang_dict() - language = frappe.db.get_value('System Settings', None, 'language') - if language: - frappe.db.set_value('System Settings', None, 'language', lang_dict.get('language') or 'en') - - for user in frappe.get_all('User', fields=['name', 'language']): - if user.language: - frappe.db.set_value('User', user.name, 'language', - lang_dict.get('language') or 'en', update_modified=False) diff --git a/frappe/patches/v7_2/__init__.py b/frappe/patches/v7_2/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/patches/v7_2/fix_email_queue_recipient.py b/frappe/patches/v7_2/fix_email_queue_recipient.py deleted file mode 100644 index 645b17b5c9..0000000000 --- a/frappe/patches/v7_2/fix_email_queue_recipient.py +++ /dev/null @@ -1,6 +0,0 @@ -from __future__ import unicode_literals -import frappe - -def execute(): - frappe.reload_doc('email', 'doctype', 'email_queue_recipient') - frappe.db.sql('update `tabEmail Queue Recipient` set parenttype="recipients"') \ No newline at end of file diff --git a/frappe/patches/v7_2/merge_knowledge_base.py b/frappe/patches/v7_2/merge_knowledge_base.py deleted file mode 100644 index 301d15e1dd..0000000000 --- a/frappe/patches/v7_2/merge_knowledge_base.py +++ /dev/null @@ -1,24 +0,0 @@ -from __future__ import unicode_literals -import frappe - -from frappe.patches.v7_0.re_route import update_routes -from frappe.installer import remove_from_installed_apps - -def execute(): - if 'knowledge_base' in frappe.get_installed_apps(): - frappe.reload_doc('website', 'doctype', 'help_category') - frappe.reload_doc('website', 'doctype', 'help_article') - update_routes(['Help Category', 'Help Article']) - remove_from_installed_apps('knowledge_base') - - # remove module def - if frappe.db.exists('Module Def', 'Knowledge Base'): - frappe.delete_doc('Module Def', 'Knowledge Base') - - # set missing routes - for doctype in ('Help Category', 'Help Article'): - for d in frappe.get_all(doctype, fields=['name', 'route']): - if not d.route: - doc = frappe.get_doc(doctype, d.name) - doc.set_route() - doc.db_update() \ No newline at end of file diff --git a/frappe/patches/v7_2/remove_in_filter.py b/frappe/patches/v7_2/remove_in_filter.py deleted file mode 100644 index 36556d7c13..0000000000 --- a/frappe/patches/v7_2/remove_in_filter.py +++ /dev/null @@ -1,7 +0,0 @@ -from __future__ import unicode_literals -import frappe - -def execute(): - if frappe.db.has_column('DocField', 'in_filter'): - frappe.db.sql('alter table tabDocField drop column in_filter') - frappe.clear_cache(doctype="DocField") \ No newline at end of file diff --git a/frappe/patches/v7_2/set_doctype_engine.py b/frappe/patches/v7_2/set_doctype_engine.py deleted file mode 100644 index 3a5cc384a2..0000000000 --- a/frappe/patches/v7_2/set_doctype_engine.py +++ /dev/null @@ -1,7 +0,0 @@ -from __future__ import unicode_literals -import frappe - -def execute(): - for t in frappe.db.sql('show table status'): - if t[0].startswith('tab'): - frappe.db.sql('update tabDocType set engine=%s where name=%s', (t[1], t[0][3:])) \ No newline at end of file diff --git a/frappe/patches/v7_2/set_in_standard_filter_property.py b/frappe/patches/v7_2/set_in_standard_filter_property.py deleted file mode 100644 index 12f97f7f8e..0000000000 --- a/frappe/patches/v7_2/set_in_standard_filter_property.py +++ /dev/null @@ -1,20 +0,0 @@ -from __future__ import unicode_literals -import frappe - -def execute(): - frappe.reload_doc('custom', 'doctype', 'custom_field', force=True) - - try: - frappe.db.sql('update `tabCustom Field` set in_standard_filter = in_filter_dash') - except Exception as e: - if not frappe.db.is_missing_column(e): raise e - - for doctype in frappe.get_all("DocType", {"istable": 0, "issingle": 0, "custom": 0}): - try: - frappe.reload_doctype(doctype.name, force=True) - except KeyError: - pass - except frappe.db.DataError: - pass - except Exception: - pass diff --git a/frappe/patches/v7_2/setup_custom_perms.py b/frappe/patches/v7_2/setup_custom_perms.py deleted file mode 100644 index 1b3b86236c..0000000000 --- a/frappe/patches/v7_2/setup_custom_perms.py +++ /dev/null @@ -1,13 +0,0 @@ -from __future__ import unicode_literals -import frappe -from frappe.permissions import setup_custom_perms -from frappe.core.page.permission_manager.permission_manager import get_standard_permissions -from frappe.utils.reset_doc import setup_perms_for - -''' -Copy DocPerm to Custom DocPerm where permissions are set differently -''' - -def execute(): - for d in frappe.db.get_all('DocType', dict(istable=0, issingle=0, custom=0)): - setup_perms_for(d.name) diff --git a/frappe/patches/v7_2/setup_ldap_config.py b/frappe/patches/v7_2/setup_ldap_config.py deleted file mode 100644 index 31dd8ca6fe..0000000000 --- a/frappe/patches/v7_2/setup_ldap_config.py +++ /dev/null @@ -1,22 +0,0 @@ -from __future__ import unicode_literals -import frappe -from frappe.utils import cint - -def execute(): - frappe.reload_doc("integrations", "doctype", "ldap_settings") - - if not frappe.db.exists("DocType", "Integration Service"): - return - - if not frappe.db.exists("Integration Service", "LDAP"): - return - - if not cint(frappe.db.get_value("Integration Service", "LDAP", 'enabled')): - return - - import ldap - try: - ldap_settings = frappe.get_doc("LDAP Settings") - ldap_settings.save(ignore_permissions=True) - except ldap.LDAPError: - pass diff --git a/frappe/patches/v7_2/update_communications.py b/frappe/patches/v7_2/update_communications.py deleted file mode 100644 index f3d859b95a..0000000000 --- a/frappe/patches/v7_2/update_communications.py +++ /dev/null @@ -1,10 +0,0 @@ -from __future__ import unicode_literals -import frappe - -def execute(): - """ - in communication move feedback details to content - remove Guest None from sender full name - setup feedback request trigger's is_manual field - """ - return diff --git a/frappe/patches/v7_2/update_feedback_request.py b/frappe/patches/v7_2/update_feedback_request.py deleted file mode 100644 index 11e9eb8e92..0000000000 --- a/frappe/patches/v7_2/update_feedback_request.py +++ /dev/null @@ -1,10 +0,0 @@ -from __future__ import unicode_literals -import frappe - -def execute(): - """ - rename feedback request documents, - update the feedback request and save the rating and communication - reference in Feedback Request document - """ - return diff --git a/frappe/patches/v8_0/__init__.py b/frappe/patches/v8_0/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/patches/v8_0/deprecate_integration_broker.py b/frappe/patches/v8_0/deprecate_integration_broker.py deleted file mode 100644 index ad1a7d9571..0000000000 --- a/frappe/patches/v8_0/deprecate_integration_broker.py +++ /dev/null @@ -1,51 +0,0 @@ -from __future__ import unicode_literals -import frappe -from frappe.integrations.utils import create_payment_gateway - -def execute(): - setup_enabled_integrations() - - for doctype in ["integration_request", "oauth_authorization_code", "oauth_bearer_token", "oauth_client"]: - frappe.reload_doc('integrations', 'doctype', doctype) - - frappe.reload_doc("core", "doctype", "payment_gateway") - update_doctype_module() - create_payment_gateway_master_records() - - for doctype in ["Integration Service", "Integration Service Parameter"]: - frappe.delete_doc("DocType", doctype) - - if not frappe.db.get_value("DocType", {"module": "Integration Broker"}, "name"): - frappe.delete_doc("Module Def", "Integration Broker") - -def setup_enabled_integrations(): - if not frappe.db.exists("DocType", "Integration Service"): - return - - for service in frappe.get_all("Integration Service", - filters={"enabled": 1, "service": ('in', ("Dropbox", "LDAP"))}, fields=["name"]): - - doctype = "{0} Settings".format(service.name) - frappe.db.set_value(doctype, doctype, 'enabled', 1) - -def update_doctype_module(): - frappe.db.sql("""update tabDocType set module='Integrations' - where name in ('Integration Request', 'Oauth Authorization Code', - 'Oauth Bearer Token', 'Oauth Client') """) - - frappe.db.sql(""" update tabDocType set module='Core' where name = 'Payment Gateway'""") - -def create_payment_gateway_master_records(): - for payment_gateway in ["Razorpay", "PayPal"]: - doctype = "{0} Settings".format(payment_gateway) - doc = frappe.get_doc(doctype) - doc_meta = frappe.get_meta(doctype) - all_mandatory_fields_has_value = True - - for d in doc_meta.fields: - if d.reqd and not doc.get(d.fieldname): - all_mandatory_fields_has_value = False - break - - if all_mandatory_fields_has_value: - create_payment_gateway(payment_gateway) diff --git a/frappe/patches/v8_0/drop_in_dialog.py b/frappe/patches/v8_0/drop_in_dialog.py deleted file mode 100644 index 231d757f26..0000000000 --- a/frappe/patches/v8_0/drop_in_dialog.py +++ /dev/null @@ -1,7 +0,0 @@ -from __future__ import unicode_literals -import frappe - -def execute(): - if frappe.db.has_column('DocType', 'in_dialog'): - frappe.db.sql('alter table tabDocType drop column in_dialog') - frappe.clear_cache(doctype="DocType") \ No newline at end of file diff --git a/frappe/patches/v8_0/drop_is_custom_from_docperm.py b/frappe/patches/v8_0/drop_is_custom_from_docperm.py deleted file mode 100644 index 4530dcd2e0..0000000000 --- a/frappe/patches/v8_0/drop_is_custom_from_docperm.py +++ /dev/null @@ -1,8 +0,0 @@ -from __future__ import unicode_literals -import frappe - -def execute(): - frappe.reload_doctype('DocPerm') - if frappe.db.has_column('DocPerm', 'is_custom'): - frappe.db.commit() - frappe.db.sql('alter table `tabDocPerm` drop column is_custom') \ No newline at end of file diff --git a/frappe/patches/v8_0/drop_unwanted_indexes.py b/frappe/patches/v8_0/drop_unwanted_indexes.py deleted file mode 100644 index fc66ed43fd..0000000000 --- a/frappe/patches/v8_0/drop_unwanted_indexes.py +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright (c) 2017, Frappe and Contributors -# License: GNU General Public License v3. See license.txt -# -*- coding: utf-8 -*- - -from __future__ import unicode_literals -import frappe - -def execute(): - # communication - unwanted_indexes = ["communication_date_index", "message_id_index", "modified_index", - "creation_index", "reference_owner", "communication_date"] - - for k in unwanted_indexes: - try: - frappe.db.sql("drop index {0} on `tabCommunication`".format(k)) - except: - pass \ No newline at end of file diff --git a/frappe/patches/v8_0/install_new_build_system_requirements.py b/frappe/patches/v8_0/install_new_build_system_requirements.py deleted file mode 100644 index 536c2fcfb3..0000000000 --- a/frappe/patches/v8_0/install_new_build_system_requirements.py +++ /dev/null @@ -1,22 +0,0 @@ -from __future__ import print_function, unicode_literals -from subprocess import Popen, call, PIPE - -def execute(): - # update nodejs version if brew exists - p = Popen(['which', 'brew'], stdout=PIPE, stderr=PIPE) - output, err = p.communicate() - if output: - call(['brew', 'upgrade', 'node']) - else: - print('Please update your NodeJS version') - - call([ - 'npm', 'install', - 'babel-core', - 'less', - 'chokidar', - 'babel-preset-es2015', - 'babel-preset-es2016', - 'babel-preset-es2017', - 'babel-preset-babili' - ]) \ No newline at end of file diff --git a/frappe/patches/v8_0/newsletter_childtable_migrate.py b/frappe/patches/v8_0/newsletter_childtable_migrate.py deleted file mode 100644 index f652b37f56..0000000000 --- a/frappe/patches/v8_0/newsletter_childtable_migrate.py +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright (c) 2017, Frappe and Contributors -# License: GNU General Public License v3. See license.txt - -from __future__ import unicode_literals -import frappe - -def execute(): - frappe.reload_doc('email', 'doctype', 'newsletter_email_group') - frappe.reload_doctype('Newsletter') - - if "email_group" not in frappe.db.get_table_columns("Newsletter"): - return - - newsletters = frappe.get_all("Newsletter", fields=["name", "email_group"]) - for newsletter in newsletters: - if newsletter.email_group: - newsletter_doc = frappe.get_doc("Newsletter", newsletter.name) - if not newsletter_doc.get("email_group"): - newsletter_doc.append("email_group", { - "email_group": newsletter.email_group, - }) - newsletter_doc.flags.ignore_validate = True - newsletter_doc.flags.ignore_mandatory = True - newsletter_doc.save() diff --git a/frappe/patches/v8_0/rename_listsettings_to_usersettings.py b/frappe/patches/v8_0/rename_listsettings_to_usersettings.py deleted file mode 100644 index 584e4a1111..0000000000 --- a/frappe/patches/v8_0/rename_listsettings_to_usersettings.py +++ /dev/null @@ -1,46 +0,0 @@ -from __future__ import unicode_literals -from frappe.model.utils.user_settings import update_user_settings -import frappe, json -from six import iteritems - - -def execute(): - if frappe.db.table_exists("__ListSettings"): - for us in frappe.db.sql('''select user, doctype, data from __ListSettings''', as_dict=True): - try: - data = json.loads(us.data) - except: - continue - - if 'List' in data: - continue - - if 'limit' in data: - data['page_length'] = data['limit'] - del data['limit'] - - new_data = dict(List=data) - new_data = json.dumps(new_data) - - frappe.db.sql('''update __ListSettings - set data=%(new_data)s - where user=%(user)s - and doctype=%(doctype)s''', - {'new_data': new_data, 'user': us.user, 'doctype': us.doctype}) - - frappe.db.sql("RENAME TABLE __ListSettings to __UserSettings") - else: - if not frappe.db.table_exists("__UserSettings"): - frappe.db.create_user_settings_table() - - for user in frappe.db.get_all('User', {'user_type': 'System User'}): - defaults = frappe.defaults.get_defaults_for(user.name) - for key, value in iteritems(defaults): - if key.startswith('_list_settings:'): - doctype = key.replace('_list_settings:', '') - columns = ['`tab{1}`.`{0}`'.format(*c) for c in json.loads(value)] - for col in columns: - if "name as" in col: - columns.remove(col) - - update_user_settings(doctype, {'fields': columns}) \ No newline at end of file diff --git a/frappe/patches/v8_0/rename_page_role_to_has_role.py b/frappe/patches/v8_0/rename_page_role_to_has_role.py deleted file mode 100644 index 9c610d857d..0000000000 --- a/frappe/patches/v8_0/rename_page_role_to_has_role.py +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright (c) 2013, Web Notes Technologies Pvt. Ltd. and Contributors -# License: GNU General Public License v3. See license.txt - -from __future__ import unicode_literals -import frappe - -def execute(): - if not frappe.db.exists('DocType', 'Has Role'): - frappe.rename_doc('DocType', 'Page Role', 'Has Role') - reload_doc() - set_ref_doctype_roles_to_report() - copy_user_roles_to_has_roles() - remove_doctypes() - -def reload_doc(): - frappe.reload_doc("core", 'doctype', "page") - frappe.reload_doc("core", 'doctype', "report") - frappe.reload_doc("core", 'doctype', "user") - frappe.reload_doc("core", 'doctype', "has_role") - -def set_ref_doctype_roles_to_report(): - for data in frappe.get_all('Report', fields=["name"]): - doc = frappe.get_doc('Report', data.name) - if frappe.db.exists("DocType", doc.ref_doctype): - try: - doc.set_doctype_roles() - for row in doc.roles: - row.db_update() - except: - pass - -def copy_user_roles_to_has_roles(): - if frappe.db.exists('DocType', 'UserRole'): - for data in frappe.get_all('User', fields = ["name"]): - doc = frappe.get_doc('User', data.name) - doc.set('roles',[]) - for args in frappe.get_all('UserRole', fields = ["role"], - filters = {'parent': data.name, 'parenttype': 'User'}): - doc.append('roles', { - 'role': args.role - }) - for role in doc.roles: - role.db_update() - -def remove_doctypes(): - for doctype in ['UserRole', 'Event Role']: - if frappe.db.exists('DocType', doctype): - frappe.delete_doc('DocType', doctype) \ No newline at end of file diff --git a/frappe/patches/v8_0/rename_print_to_printing.py b/frappe/patches/v8_0/rename_print_to_printing.py deleted file mode 100644 index ecdbc3f7be..0000000000 --- a/frappe/patches/v8_0/rename_print_to_printing.py +++ /dev/null @@ -1,13 +0,0 @@ -from __future__ import unicode_literals -import frappe - -def execute(): - if frappe.db.exists('Module Def', 'Print'): - frappe.reload_doc('printing', 'doctype', 'print_format') - frappe.reload_doc('printing', 'doctype', 'print_settings') - frappe.reload_doc('printing', 'doctype', 'print_heading') - frappe.reload_doc('printing', 'doctype', 'letter_head') - frappe.reload_doc('printing', 'page', 'print_format_builder') - frappe.db.sql("""update `tabPrint Format` set module='Printing' where module='Print'""") - - frappe.delete_doc('Module Def', 'Print') \ No newline at end of file diff --git a/frappe/patches/v8_0/set_allow_traceback.py b/frappe/patches/v8_0/set_allow_traceback.py deleted file mode 100644 index 3eceb3e29c..0000000000 --- a/frappe/patches/v8_0/set_allow_traceback.py +++ /dev/null @@ -1,6 +0,0 @@ -from __future__ import unicode_literals -import frappe - -def execute(): - frappe.reload_doc('core', 'doctype', 'system_settings') - frappe.db.sql("update `tabSystem Settings` set allow_error_traceback=1") diff --git a/frappe/patches/v8_0/set_currency_field_precision.py b/frappe/patches/v8_0/set_currency_field_precision.py deleted file mode 100644 index 89835c8c1e..0000000000 --- a/frappe/patches/v8_0/set_currency_field_precision.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright (c) 2013, Web Notes Technologies Pvt. Ltd. and Contributors -# License: GNU General Public License v3. See license.txt - -from __future__ import unicode_literals -import frappe -from frappe.utils import get_number_format_info - -def execute(): - frappe.reload_doc('core', 'doctype', 'system_settings', force=True) - if not frappe.db.get_value("System Settings", None, "currency_precision"): - default_currency = frappe.db.get_default("currency") - number_format = frappe.db.get_value("Currency", default_currency, "number_format", cache=True) \ - or frappe.db.get_default("number_format") - if number_format: - precision = get_number_format_info(number_format)[2] - else: - precision = 2 - - ss = frappe.get_doc("System Settings") - ss.currency_precision = precision - ss.flags.ignore_mandatory = True - ss.save() diff --git a/frappe/patches/v8_0/set_doctype_values_in_custom_role.py b/frappe/patches/v8_0/set_doctype_values_in_custom_role.py deleted file mode 100644 index 58cdc4497d..0000000000 --- a/frappe/patches/v8_0/set_doctype_values_in_custom_role.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright (c) 2013, Web Notes Technologies Pvt. Ltd. and Contributors -# License: GNU General Public License v3. See license.txt - -from __future__ import unicode_literals -import frappe - -def execute(): - frappe.reload_doctype('Custom Role') - - # set ref doctype in custom role for reports - frappe.db.sql(""" update `tabCustom Role` set - `tabCustom Role`.ref_doctype = (select ref_doctype from `tabReport` where name = `tabCustom Role`.report) - where `tabCustom Role`.report is not null""") diff --git a/frappe/patches/v8_0/set_user_permission_for_page_and_report.py b/frappe/patches/v8_0/set_user_permission_for_page_and_report.py deleted file mode 100644 index 560ea46db2..0000000000 --- a/frappe/patches/v8_0/set_user_permission_for_page_and_report.py +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright (c) 2013, Web Notes Technologies Pvt. Ltd. and Contributors -# License: GNU General Public License v3. See license.txt - -from __future__ import unicode_literals -import frappe - -def execute(): - if not frappe.db.exists('DocType', 'Custom Role'): - frappe.reload_doc("core", 'doctype', "custom_role") - set_user_permission_for_page_and_report() - - update_ref_doctype_in_custom_role() - -def update_ref_doctype_in_custom_role(): - frappe.reload_doc("core", 'doctype', "custom_role") - frappe.db.sql("""update `tabCustom Role` - set - ref_doctype = (select ref_doctype from tabReport where name = `tabCustom Role`.report) - where report is not null""") - -def set_user_permission_for_page_and_report(): - make_custom_roles_for_page_and_report() - -def make_custom_roles_for_page_and_report(): - for doctype in ['Page', 'Report']: - for data in get_data(doctype): - doc = frappe.get_doc(doctype, data.name) - roles = get_roles(doctype, data, doc) - make_custom_roles(doctype, doc.name, roles) - -def get_data(doctype): - fields = ["name"] if doctype == 'Page' else ["name", "ref_doctype"] - return frappe.get_all(doctype, fields = fields) - -def get_roles(doctype, data, doc): - roles = [] - if doctype == 'Page': - for d in doc.roles: - if frappe.db.exists('Role', d.role): - roles.append({'role': d.role}) - else: - out = frappe.get_all('Custom DocPerm', fields='distinct role', filters=dict(parent = data.ref_doctype)) - for d in out: - roles.append({'role': d.role}) - return roles - -def make_custom_roles(doctype, name, roles): - field = doctype.lower() - - if roles: - custom_permission = frappe.get_doc({ - 'doctype': 'Custom Role', - field : name, - 'roles' : roles - }).insert() diff --git a/frappe/patches/v8_0/setup_email_inbox.py b/frappe/patches/v8_0/setup_email_inbox.py deleted file mode 100644 index 1bfe3b0b74..0000000000 --- a/frappe/patches/v8_0/setup_email_inbox.py +++ /dev/null @@ -1,26 +0,0 @@ -from __future__ import unicode_literals -import frappe, json -from frappe.core.doctype.user.user import ask_pass_update, setup_user_email_inbox - -def execute(): - """ - depricate email inbox page if exists - remove desktop icon for email inbox page if exists - patch to remove Custom DocPerm for communication - add user inbox child table entry for existing email account in not exists - """ - - if frappe.db.exists("Page", "email_inbox"): - frappe.delete_doc("Page", "email_inbox") - - frappe.db.sql("""update `tabCustom DocPerm` set `write`=0, email=1 where parent='Communication'""") - - frappe.reload_doc("core", "doctype", "user_email") - frappe.reload_doc("email", "doctype", "email_account") - - email_accounts = frappe.get_all("Email Account", filters={"enable_incoming": 1}, - fields=["name", "email_id", "awaiting_password", "enable_outgoing"]) - - for email_account in email_accounts: - setup_user_email_inbox(email_account.get("name"), email_account.get("awaiting_password"), - email_account.get("email_id"), email_account.get("enabled_outgoing")) diff --git a/frappe/patches/v8_0/update_gender_and_salutation.py b/frappe/patches/v8_0/update_gender_and_salutation.py deleted file mode 100644 index bcd9d4cbd7..0000000000 --- a/frappe/patches/v8_0/update_gender_and_salutation.py +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors - -from __future__ import unicode_literals -import frappe -from frappe.desk.page.setup_wizard.install_fixtures import update_genders, update_salutations - -def execute(): - frappe.db.set_value("DocType", "Contact", "module", "Contacts") - frappe.db.set_value("DocType", "Address", "module", "Contacts") - frappe.db.set_value("DocType", "Address Template", "module", "Contacts") - frappe.reload_doc('contacts', 'doctype', 'gender') - frappe.reload_doc('contacts', 'doctype', 'salutation') - - update_genders() - update_salutations() \ No newline at end of file diff --git a/frappe/patches/v8_0/update_global_search_table.py b/frappe/patches/v8_0/update_global_search_table.py deleted file mode 100644 index 3c0a70155b..0000000000 --- a/frappe/patches/v8_0/update_global_search_table.py +++ /dev/null @@ -1,13 +0,0 @@ -from __future__ import unicode_literals -import frappe - -def execute(): - if not 'published' in frappe.db.get_db_table_columns('__global_search'): - frappe.db.sql('''alter table __global_search - add column `title` varchar(140)''') - - frappe.db.sql('''alter table __global_search - add column `route` varchar(140)''') - - frappe.db.sql('''alter table __global_search - add column `published` int(1) not null default 0''') diff --git a/frappe/patches/v8_0/update_published_in_global_search.py b/frappe/patches/v8_0/update_published_in_global_search.py deleted file mode 100644 index a378f24732..0000000000 --- a/frappe/patches/v8_0/update_published_in_global_search.py +++ /dev/null @@ -1,12 +0,0 @@ -from __future__ import unicode_literals -import frappe - -def execute(): - from frappe.website.router import get_doctypes_with_web_view - from frappe.utils.global_search import rebuild_for_doctype - - for doctype in get_doctypes_with_web_view(): - try: - rebuild_for_doctype(doctype) - except frappe.DoesNotExistError: - pass diff --git a/frappe/patches/v8_0/update_records_in_global_search.py b/frappe/patches/v8_0/update_records_in_global_search.py deleted file mode 100644 index dafa1e76d3..0000000000 --- a/frappe/patches/v8_0/update_records_in_global_search.py +++ /dev/null @@ -1,12 +0,0 @@ -from __future__ import unicode_literals -import frappe -from frappe.utils.global_search import get_doctypes_with_global_search, rebuild_for_doctype -from frappe.utils import update_progress_bar - -def execute(): - frappe.cache().delete_value('doctypes_with_global_search') - doctypes_with_global_search = get_doctypes_with_global_search(with_child_tables=False) - - for i, doctype in enumerate(doctypes_with_global_search): - update_progress_bar("Updating Global Search", i, len(doctypes_with_global_search)) - rebuild_for_doctype(doctype) diff --git a/frappe/patches/v8_1/__init__.py b/frappe/patches/v8_1/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/patches/v8_1/delete_custom_docperm_if_doctype_not_exists.py b/frappe/patches/v8_1/delete_custom_docperm_if_doctype_not_exists.py deleted file mode 100644 index 92b54edfd4..0000000000 --- a/frappe/patches/v8_1/delete_custom_docperm_if_doctype_not_exists.py +++ /dev/null @@ -1,7 +0,0 @@ -from __future__ import unicode_literals -import frappe - -def execute(): - frappe.db.sql("""delete from `tabCustom DocPerm` - where parent not in ( select name from `tabDocType` ) - """) diff --git a/frappe/patches/v8_1/enable_allow_error_traceback_in_system_settings.py b/frappe/patches/v8_1/enable_allow_error_traceback_in_system_settings.py deleted file mode 100644 index 9bd9757a86..0000000000 --- a/frappe/patches/v8_1/enable_allow_error_traceback_in_system_settings.py +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright (c) 2017, Frappe and Contributors -# License: GNU General Public License v3. See license.txt - -from __future__ import unicode_literals -import frappe - -def execute(): - """ enable the allow_enable_traceback property in system settings """ - - frappe.reload_doc("core", "doctype", "system_settings") - doc = frappe.get_doc("System Settings", "System Settings") - doc.allow_error_traceback = 1 - doc.flags.ignore_permissions=True - doc.flags.ignore_mandatory=True - doc.save() \ No newline at end of file diff --git a/frappe/patches/v8_1/update_format_options_in_auto_email_report.py b/frappe/patches/v8_1/update_format_options_in_auto_email_report.py deleted file mode 100644 index 56609780cb..0000000000 --- a/frappe/patches/v8_1/update_format_options_in_auto_email_report.py +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright (c) 2017, Frappe and Contributors -# License: GNU General Public License v3. See license.txt - -from __future__ import unicode_literals -import frappe - -def execute(): - """ change the XLS option as XLSX in the auto email report """ - - frappe.reload_doc("email", "doctype", "auto_email_report") - - auto_email_list = frappe.get_all("Auto Email Report", filters={"format": "XLS"}) - for auto_email in auto_email_list: - frappe.db.set_value("Auto Email Report", auto_email.name, "format", "XLSX") diff --git a/frappe/patches/v8_10/__init__.py b/frappe/patches/v8_10/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/patches/v8_10/delete_static_web_page_from_global_search.py b/frappe/patches/v8_10/delete_static_web_page_from_global_search.py deleted file mode 100644 index 336562c157..0000000000 --- a/frappe/patches/v8_10/delete_static_web_page_from_global_search.py +++ /dev/null @@ -1,5 +0,0 @@ -from __future__ import unicode_literals -import frappe - -def execute(): - frappe.db.sql("""delete from `__global_search` where doctype='Static Web Page'"""); \ No newline at end of file diff --git a/frappe/patches/v8_5/__init__.py b/frappe/patches/v8_5/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/patches/v8_5/delete_email_group_member_with_invalid_emails.py b/frappe/patches/v8_5/delete_email_group_member_with_invalid_emails.py deleted file mode 100644 index 89a9a7a1b9..0000000000 --- a/frappe/patches/v8_5/delete_email_group_member_with_invalid_emails.py +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright (c) 2017, Frappe and Contributors -# License: GNU General Public License v3. See license.txt - -from __future__ import unicode_literals -import frappe -from frappe.utils import validate_email_address - -def execute(): - ''' update/delete the email group member with the wrong email ''' - - email_group_members = frappe.get_all("Email Group Member", fields=["name", "email"]) - for member in email_group_members: - validated_email = validate_email_address(member.email) - if (validated_email==member.email): - pass - else: - try: - frappe.db.set_value("Email Group Member", member.name, "email", validated_email) - except Exception: - frappe.delete_doc(doctype="Email Group Member", name=member.name, force=1, ignore_permissions=True) \ No newline at end of file diff --git a/frappe/patches/v8_5/patch_event_colors.py b/frappe/patches/v8_5/patch_event_colors.py deleted file mode 100644 index 8ac7aec238..0000000000 --- a/frappe/patches/v8_5/patch_event_colors.py +++ /dev/null @@ -1,25 +0,0 @@ -from __future__ import unicode_literals -import frappe - -def execute(): - - if not frappe.db.sql("SHOW COLUMNS FROM `tabEvent` LIKE 'color';"): - return - - colors = ['red', 'green', 'blue', 'yellow', 'skyblue', 'orange'] - hex_colors = ['#ffc4c4', '#cef6d1', '#d2d2ff', '#fffacd', '#d2f1ff', '#ffd2c2'] - - def get_hex_for_color(color): - index = colors.index(color) - return hex_colors[index] - - query = ''' - update tabEvent - set color='{hex}' - where color='{color}' - ''' - - for color in colors: - frappe.db.sql(query.format(color=color, hex=get_hex_for_color(color))) - - frappe.db.commit() diff --git a/frappe/patches/v8_7/__init__.py b/frappe/patches/v8_7/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/patches/v8_x/__init__.py b/frappe/patches/v8_x/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/patches/v8_x/update_user_permission.py b/frappe/patches/v8_x/update_user_permission.py deleted file mode 100644 index 693b87c974..0000000000 --- a/frappe/patches/v8_x/update_user_permission.py +++ /dev/null @@ -1,28 +0,0 @@ -from __future__ import unicode_literals -import frappe - -def execute(): - frappe.reload_doc('core', 'doctype', 'user_permission') - frappe.delete_doc('core', 'page', 'user-permissions') - for perm in frappe.db.sql(""" - select - name, parent, defkey, defvalue - from - tabDefaultValue - where - parent not in ('__default', '__global') - and - substr(defkey,1,1)!='_' - and - parenttype='User Permission' - """, as_dict=True): - if frappe.db.exists(perm.defkey, perm.defvalue) and frappe.db.exists('User', perm.parent): - frappe.get_doc(dict( - doctype='User Permission', - user=perm.parent, - allow=perm.defkey, - for_value=perm.defvalue, - apply_for_all_roles=0, - )).insert(ignore_permissions = True) - - frappe.db.sql('delete from tabDefaultValue where parenttype="User Permission"') diff --git a/frappe/patches/v9_1/__init__.py b/frappe/patches/v9_1/__init__.py deleted file mode 100644 index baffc48825..0000000000 --- a/frappe/patches/v9_1/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from __future__ import unicode_literals diff --git a/frappe/patches/v9_1/add_sms_sender_name_as_parameters.py b/frappe/patches/v9_1/add_sms_sender_name_as_parameters.py deleted file mode 100644 index 9d7c0f003f..0000000000 --- a/frappe/patches/v9_1/add_sms_sender_name_as_parameters.py +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright (c) 2017, Frappe and Contributors -# License: GNU General Public License v3. See license.txt - -from __future__ import unicode_literals -import frappe - -def execute(): - frappe.reload_doc("core", "doctype", "sms_parameter") - sms_sender_name = frappe.db.get_single_value("SMS Settings", "sms_sender_name") - if sms_sender_name: - frappe.reload_doc("core", "doctype", "sms_settings") - sms_settings = frappe.get_doc("SMS Settings") - sms_settings.append("parameters", { - "parameter": "sender_name", - "value": sms_sender_name - }) - sms_settings.flags.ignore_mandatory = True - sms_settings.flags.ignore_permissions = True - sms_settings.save() diff --git a/frappe/patches/v9_1/move_feed_to_activity_log.py b/frappe/patches/v9_1/move_feed_to_activity_log.py deleted file mode 100644 index db46b4e419..0000000000 --- a/frappe/patches/v9_1/move_feed_to_activity_log.py +++ /dev/null @@ -1,24 +0,0 @@ -from __future__ import unicode_literals -import frappe -from frappe.utils.background_jobs import enqueue - -def execute(): - comm_records_count = frappe.db.count("Communication", {"comment_type": "Updated"}) - if comm_records_count > 100000: - enqueue(method=move_data_from_communication_to_activity_log, queue='short', now=True) - else: - move_data_from_communication_to_activity_log() - -def move_data_from_communication_to_activity_log(): - frappe.reload_doc("core", "doctype", "communication") - frappe.reload_doc("core", "doctype", "activity_log") - - frappe.db.sql("""insert into `tabActivity Log` (name, owner, modified, creation, status, communication_date, - reference_doctype, reference_name, timeline_doctype, timeline_name, link_doctype, link_name, subject, content, user) - select name, owner, modified, creation, status, communication_date, - reference_doctype, reference_name, timeline_doctype, timeline_name, link_doctype, link_name, subject, content, user - from `tabCommunication` - where comment_type = 'Updated'""") - - frappe.db.sql("""delete from `tabCommunication` where comment_type = 'Updated'""") - frappe.delete_doc("DocType", "Authentication Log") \ No newline at end of file diff --git a/frappe/patches/v9_1/resave_domain_settings.py b/frappe/patches/v9_1/resave_domain_settings.py deleted file mode 100644 index 1e54ad3aa5..0000000000 --- a/frappe/patches/v9_1/resave_domain_settings.py +++ /dev/null @@ -1,13 +0,0 @@ -from __future__ import unicode_literals -import frappe - -def execute(): - domain_settings = frappe.get_doc('Domain Settings') - active_domains = [d.domain for d in domain_settings.active_domains] - try: - for d in ('Education', 'Healthcare', 'Hospitality'): - if d in active_domains and frappe.db.exists('Domain', d): - domain = frappe.get_doc('Domain', d) - domain.setup_domain() - except frappe.LinkValidationError: - pass diff --git a/frappe/patches/v9_1/revert_domain_settings.py b/frappe/patches/v9_1/revert_domain_settings.py deleted file mode 100644 index a14b48dae6..0000000000 --- a/frappe/patches/v9_1/revert_domain_settings.py +++ /dev/null @@ -1,11 +0,0 @@ -from __future__ import unicode_literals -import frappe - -def execute(): - domain_settings = frappe.get_doc('Domain Settings') - active_domains = [d.domain for d in domain_settings.active_domains] - - for domain_name in ('Education', 'Healthcare', 'Hospitality'): - if frappe.db.exists('Domain', domain_name) and domain_name not in active_domains: - domain = frappe.get_doc('Domain', domain_name) - domain.remove_domain() \ No newline at end of file diff --git a/frappe/permissions.py b/frappe/permissions.py index 19f101aab5..c25a7c3947 100644 --- a/frappe/permissions.py +++ b/frappe/permissions.py @@ -1,19 +1,15 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt +import copy -from __future__ import unicode_literals, print_function -from six import string_types -import frappe, copy, json +import frappe +import frappe.share from frappe import _, msgprint from frappe.utils import cint -import frappe.share + rights = ("select", "read", "write", "create", "delete", "submit", "cancel", "amend", "print", "email", "report", "import", "export", "set_user_permissions", "share") -# TODO: - -# optimize: meta.get_link_map (check if the doctype link exists for the given permission type) - def check_admin_or_system_manager(user=None): if not user: user = frappe.session.user @@ -58,7 +54,7 @@ def has_permission(doctype, ptype="read", doc=None, verbose=False, user=None, ra meta = frappe.get_meta(doctype) if doc: - if isinstance(doc, string_types): + if isinstance(doc, str): doc = frappe.get_doc(meta.name, doc) perm = get_doc_permissions(doc, user=user, ptype=ptype).get(ptype) if not perm: push_perm_check_log(_('User {0} does not have access to this document').format(frappe.bold(user))) @@ -159,7 +155,7 @@ def get_role_permissions(doctype_meta, user=None, is_owner=None): } } """ - if isinstance(doctype_meta, string_types): + if isinstance(doctype_meta, str): doctype_meta = frappe.get_meta(doctype_meta) # assuming doctype name was passed if not user: user = frappe.session.user @@ -534,7 +530,7 @@ def get_linked_doctypes(dt): def get_doc_name(doc): if not doc: return None - return doc if isinstance(doc, string_types) else doc.name + return doc if isinstance(doc, str) else doc.name def allow_everything(): ''' diff --git a/frappe/printing/doctype/letter_head/letter_head.py b/frappe/printing/doctype/letter_head/letter_head.py index 3a3b14faad..948be60b88 100644 --- a/frappe/printing/doctype/letter_head/letter_head.py +++ b/frappe/printing/doctype/letter_head/letter_head.py @@ -1,7 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import frappe from frappe.utils import is_image from frappe.model.document import Document @@ -19,7 +18,7 @@ class LetterHead(Document): def validate_disabled_and_default(self): if self.disabled and self.is_default: frappe.throw(_("Letter Head cannot be both disabled and default")) - + if not self.is_default and not self.disabled: if not frappe.db.exists('Letter Head', dict(is_default=1)): self.is_default = 1 diff --git a/frappe/printing/doctype/letter_head/test_letter_head.py b/frappe/printing/doctype/letter_head/test_letter_head.py index b69e9924ea..96dfc68705 100644 --- a/frappe/printing/doctype/letter_head/test_letter_head.py +++ b/frappe/printing/doctype/letter_head/test_letter_head.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2017, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import frappe import unittest diff --git a/frappe/printing/doctype/print_format/print_format.py b/frappe/printing/doctype/print_format/print_format.py index 1c11f2d519..5d4ff92fe2 100644 --- a/frappe/printing/doctype/print_format/print_format.py +++ b/frappe/printing/doctype/print_format/print_format.py @@ -2,7 +2,6 @@ # Copyright (c) 2017, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe import frappe.utils import json diff --git a/frappe/printing/doctype/print_format/test_print_format.py b/frappe/printing/doctype/print_format/test_print_format.py index 7e30bda23e..e65eb0183f 100644 --- a/frappe/printing/doctype/print_format/test_print_format.py +++ b/frappe/printing/doctype/print_format/test_print_format.py @@ -1,7 +1,5 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # See license.txt -from __future__ import unicode_literals, print_function - import frappe import unittest import re @@ -12,13 +10,13 @@ class TestPrintFormat(unittest.TestCase): def test_print_user(self, style=None): print_html = frappe.get_print("User", "Administrator", style=style) self.assertTrue("" in print_html) - self.assertTrue(re.findall('
[\s]*administrator[\s]*
', print_html)) + self.assertTrue(re.findall(r'
[\s]*administrator[\s]*
', print_html)) return print_html def test_print_user_standard(self): print_html = self.test_print_user("Standard") - self.assertTrue(re.findall('\.print-format {[\s]*font-size: 9pt;', print_html)) - self.assertFalse(re.findall('th {[\s]*background-color: #eee;[\s]*}', print_html)) + self.assertTrue(re.findall(r'\.print-format {[\s]*font-size: 9pt;', print_html)) + self.assertFalse(re.findall(r'th {[\s]*background-color: #eee;[\s]*}', print_html)) self.assertFalse("font-family: serif;" in print_html) def test_print_user_modern(self): diff --git a/frappe/printing/doctype/print_heading/print_heading.py b/frappe/printing/doctype/print_heading/print_heading.py index 1bb3e52dd5..f9955c019d 100644 --- a/frappe/printing/doctype/print_heading/print_heading.py +++ b/frappe/printing/doctype/print_heading/print_heading.py @@ -2,7 +2,6 @@ # Copyright (c) 2017, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document diff --git a/frappe/printing/doctype/print_heading/test_print_heading.py b/frappe/printing/doctype/print_heading/test_print_heading.py index 1a6435e783..ce99cde607 100644 --- a/frappe/printing/doctype/print_heading/test_print_heading.py +++ b/frappe/printing/doctype/print_heading/test_print_heading.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2017, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import frappe import unittest diff --git a/frappe/printing/doctype/print_settings/print_settings.py b/frappe/printing/doctype/print_settings/print_settings.py index cf6a71a8ac..610c083097 100644 --- a/frappe/printing/doctype/print_settings/print_settings.py +++ b/frappe/printing/doctype/print_settings/print_settings.py @@ -2,7 +2,6 @@ # Copyright (c) 2018, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe import _ from frappe.utils import cint diff --git a/frappe/printing/doctype/print_settings/test_print_settings.py b/frappe/printing/doctype/print_settings/test_print_settings.py index b8ad70a681..d1dec861b2 100644 --- a/frappe/printing/doctype/print_settings/test_print_settings.py +++ b/frappe/printing/doctype/print_settings/test_print_settings.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2018, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import unittest class TestPrintSettings(unittest.TestCase): diff --git a/frappe/printing/doctype/print_style/print_style.py b/frappe/printing/doctype/print_style/print_style.py index 310babd5df..a91786795c 100644 --- a/frappe/printing/doctype/print_style/print_style.py +++ b/frappe/printing/doctype/print_style/print_style.py @@ -2,7 +2,6 @@ # Copyright (c) 2017, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals import frappe from frappe.model.document import Document diff --git a/frappe/printing/doctype/print_style/test_print_style.py b/frappe/printing/doctype/print_style/test_print_style.py index cee57f8826..b717b23df8 100644 --- a/frappe/printing/doctype/print_style/test_print_style.py +++ b/frappe/printing/doctype/print_style/test_print_style.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2017, Frappe Technologies and Contributors # See license.txt -from __future__ import unicode_literals - import frappe import unittest diff --git a/frappe/printing/page/print/print.js b/frappe/printing/page/print/print.js index dfd93c4efa..233bbe0ce7 100644 --- a/frappe/printing/page/print/print.js +++ b/frappe/printing/page/print/print.js @@ -408,14 +408,17 @@ frappe.ui.form.PrintView = class { setup_print_format_dom(out, $print_format) { this.print_wrapper.find('.print-format-skeleton').remove(); + let base_url = frappe.urllib.get_base_url(); + let print_css = frappe.assets.bundled_asset('print.bundle.css'); this.$print_format_body.find('head').html( ` - ` + ` ); if (frappe.utils.is_rtl(this.lang_code)) { + let rtl_css = frappe.assets.bundled_asset('frappe-rtl.bundle.css'); this.$print_format_body.find('head').append( - `` + `` ); } diff --git a/frappe/printing/page/print_format_builder/print_format_builder.js b/frappe/printing/page/print_format_builder/print_format_builder.js index 7e58e295b5..ca2a8bc378 100644 --- a/frappe/printing/page/print_format_builder/print_format_builder.js +++ b/frappe/printing/page/print_format_builder/print_format_builder.js @@ -23,13 +23,13 @@ frappe.pages['print-format-builder'].on_page_show = function(wrapper) { } } -frappe.PrintFormatBuilder = Class.extend({ - init: function(parent) { +frappe.PrintFormatBuilder = class PrintFormatBuilder { + constructor(parent) { this.parent = parent; this.make(); this.refresh(); - }, - refresh: function() { + } + refresh() { this.custom_html_count = 0; if(!this.print_format) { this.show_start(); @@ -37,8 +37,8 @@ frappe.PrintFormatBuilder = Class.extend({ this.page.set_title(this.print_format.name); this.setup_print_format(); } - }, - make: function() { + } + make() { this.page = frappe.ui.make_app_page({ parent: this.parent, title: __("Print Format Builder"), @@ -56,15 +56,15 @@ frappe.PrintFormatBuilder = Class.extend({ this.setup_edit_custom_html(); // $(this.page.sidebar).css({"position": 'fixed'}); // $(this.page.main).parent().css({"margin-left": '16.67%'}); - }, - show_start: function() { + } + show_start() { this.page.main.html(frappe.render_template("print_format_builder_start", {})); this.page.clear_actions(); this.page.set_title(__("Print Format Builder")); this.start_edit_print_format(); this.start_new_print_format(); - }, - start_edit_print_format: function() { + } + start_edit_print_format() { // print format control var me = this; this.print_format_input = frappe.ui.form.make_control({ @@ -89,8 +89,8 @@ frappe.PrintFormatBuilder = Class.extend({ frappe.set_route('print-format-builder', name); }); }); - }, - start_new_print_format: function() { + } + start_new_print_format() { var me = this; this.doctype_input = frappe.ui.form.make_control({ parent: this.page.main.find(".doctype-selector"), @@ -125,8 +125,8 @@ frappe.PrintFormatBuilder = Class.extend({ me.setup_new_print_format(doctype, name); }); - }, - setup_new_print_format: function(doctype, name, based_on) { + } + setup_new_print_format(doctype, name, based_on) { frappe.call({ method: 'frappe.printing.page.print_format_builder.print_format_builder.create_custom_format', args: { @@ -143,8 +143,8 @@ frappe.PrintFormatBuilder = Class.extend({ } }, }); - }, - setup_print_format: function() { + } + setup_print_format() { var me = this; frappe.model.with_doctype(this.print_format.doc_type, function(doctype) { me.meta = frappe.get_meta(me.print_format.doc_type); @@ -163,23 +163,23 @@ frappe.PrintFormatBuilder = Class.extend({ frappe.set_route("Form", "Print Format", me.print_format.name); }); }); - }, - setup_sidebar: function() { + } + setup_sidebar() { // prepend custom HTML field var fields = [this.get_custom_html_field()].concat(this.meta.fields); this.page.sidebar.html( $(frappe.render_template("print_format_builder_sidebar", {fields: fields})) ); this.setup_field_filter(); - }, - get_custom_html_field: function() { + } + get_custom_html_field() { return { fieldtype: "Custom HTML", fieldname: "_custom_html", label: __("Custom HTML") - } - }, - render_layout: function() { + }; + } + render_layout() { this.page.main.empty(); this.prepare_data(); $(frappe.render_template("print_format_builder_layout", { @@ -190,8 +190,8 @@ frappe.PrintFormatBuilder = Class.extend({ this.setup_edit_heading(); this.setup_field_settings(); this.setup_html_data(); - }, - prepare_data: function() { + } + prepare_data() { this.print_heading_template = null; this.data = JSON.parse(this.print_format.format_data || "[]"); if(!this.data.length) { @@ -280,22 +280,22 @@ frappe.PrintFormatBuilder = Class.extend({ this.layout_data = $.map(this.layout_data, function(s) { return s.has_fields ? s : null }); - }, - get_new_section: function() { + } + get_new_section() { return {columns: [], no_of_columns: 0, label:''}; - }, - get_new_column: function() { + } + get_new_column() { return {fields: []} - }, - add_table_properties: function(f) { + } + add_table_properties(f) { // build table columns and widths in a dict // visible_columns var me = this; if(!f.visible_columns) { me.init_visible_columns(f); } - }, - init_visible_columns: function(f) { + } + init_visible_columns(f) { f.visible_columns = [] $.each(frappe.get_meta(f.options).fields, function(i, _f) { if(!in_list(["Section Break", "Column Break"], _f.fieldtype) && @@ -306,8 +306,8 @@ frappe.PrintFormatBuilder = Class.extend({ print_width: (_f.width || ""), print_hide:0}); } }); - }, - setup_sortable: function() { + } + setup_sortable() { var me = this; // drag from fields library @@ -332,8 +332,8 @@ frappe.PrintFormatBuilder = Class.extend({ Sortable.create(this.page.main.find(".print-format-builder-layout").get(0), { handle: ".print-format-builder-section-head" } ); - }, - setup_sortable_for_column: function(col) { + } + setup_sortable_for_column(col) { var me = this; Sortable.create(col, { group: { @@ -363,8 +363,8 @@ frappe.PrintFormatBuilder = Class.extend({ } }); - }, - setup_field_filter: function() { + } + setup_field_filter() { var me = this; this.page.sidebar.find(".filter-fields").on("keyup", function() { var text = $(this).val(); @@ -373,8 +373,8 @@ frappe.PrintFormatBuilder = Class.extend({ $(this).parent().toggle(show); }) }); - }, - setup_section_settings: function() { + } + setup_section_settings() { var me = this; this.page.main.on("click", ".section-settings", function() { var section = $(this).parent().parent(); @@ -431,8 +431,8 @@ frappe.PrintFormatBuilder = Class.extend({ return false; }); - }, - setup_field_settings: function() { + } + setup_field_settings() { this.page.main.find(".field-settings").on("click", e => { const field = $(e.currentTarget).parent(); // new dialog @@ -482,8 +482,8 @@ frappe.PrintFormatBuilder = Class.extend({ return false; }); - }, - setup_html_data: function() { + } + setup_html_data() { // set JQuery `data` for Custom HTML fields, since editing the HTML // directly causes problem becuase of HTML reformatting // @@ -496,8 +496,8 @@ frappe.PrintFormatBuilder = Class.extend({ var html = me.custom_html_dict[parseInt(content.attr('data-custom-html-id'))].options; content.data('content', html); }) - }, - update_columns_in_section: function(section, no_of_columns, new_no_of_columns) { + } + update_columns_in_section(section, no_of_columns, new_no_of_columns) { var col_size = 12 / new_no_of_columns, me = this, resize = function() { @@ -539,8 +539,8 @@ frappe.PrintFormatBuilder = Class.extend({ resize(); } - }, - setup_add_section: function() { + } + setup_add_section() { var me = this; this.page.main.find(".print-format-builder-add-section").on("click", function() { // boostrap new section info @@ -554,8 +554,8 @@ frappe.PrintFormatBuilder = Class.extend({ me.setup_sortable_for_column($section.find(".print-format-builder-column").get(0)); }); - }, - setup_edit_heading: function() { + } + setup_edit_heading() { var me = this; var $heading = this.page.main.find(".print-format-builder-print-heading"); @@ -565,8 +565,8 @@ frappe.PrintFormatBuilder = Class.extend({ this.page.main.find(".edit-heading").on("click", function() { var d = me.get_edit_html_dialog(__("Edit Heading"), __("Heading"), $heading); }) - }, - setup_column_selector: function() { + } + setup_column_selector() { var me = this; this.page.main.on("click", ".select-columns", function() { var parent = $(this).parents(".print-format-builder-field:first"), @@ -657,24 +657,24 @@ frappe.PrintFormatBuilder = Class.extend({ return false; }); - }, - get_visible_columns_string: function(f) { + } + get_visible_columns_string(f) { if(!f.visible_columns) { this.init_visible_columns(f); } return $.map(f.visible_columns, function(v) { return v.fieldname + "|" + (v.print_width || "") }).join(","); - }, - get_no_content: function() { + } + get_no_content() { return __("Edit to add content") - }, - setup_edit_custom_html: function() { + } + setup_edit_custom_html() { var me = this; this.page.main.on("click", ".edit-html", function() { me.get_edit_html_dialog(__("Edit Custom HTML"), __("Custom HTML"), $(this).parents(".print-format-builder-field:first").find(".html-content")); }); - }, - get_edit_html_dialog: function(title, label, $content) { + } + get_edit_html_dialog(title, label, $content) { var me = this; var d = new frappe.ui.Dialog({ title: title, @@ -710,8 +710,8 @@ frappe.PrintFormatBuilder = Class.extend({ d.show(); return d; - }, - save_print_format: function() { + } + save_print_format() { var data = [], me = this; @@ -789,4 +789,4 @@ frappe.PrintFormatBuilder = Class.extend({ } }); } -}); +}; diff --git a/frappe/public/html/print_template.html b/frappe/public/html/print_template.html index bdb09541c9..721bec7fa7 100644 --- a/frappe/public/html/print_template.html +++ b/frappe/public/html/print_template.html @@ -7,7 +7,7 @@ {{ title }} - + diff --git a/frappe/public/js/barcode_scanner.bundle.js b/frappe/public/js/barcode_scanner.bundle.js new file mode 100644 index 0000000000..294f20c08f --- /dev/null +++ b/frappe/public/js/barcode_scanner.bundle.js @@ -0,0 +1 @@ +import "./frappe/barcode_scanner/quagga"; diff --git a/frappe/public/js/bootstrap-4-web.bundle.js b/frappe/public/js/bootstrap-4-web.bundle.js new file mode 100644 index 0000000000..2e3c4d7145 --- /dev/null +++ b/frappe/public/js/bootstrap-4-web.bundle.js @@ -0,0 +1,64 @@ + +// multilevel dropdown +$('.dropdown-menu a.dropdown-toggle').on('click', function (e) { + e.preventDefault(); + e.stopImmediatePropagation(); + if (!$(this).next().hasClass('show')) { + $(this).parents('.dropdown-menu').first().find('.show').removeClass("show"); + } + var $subMenu = $(this).next(".dropdown-menu"); + $subMenu.toggleClass('show'); + + + $(this).parents('li.nav-item.dropdown.show').on('hidden.bs.dropdown', function () { + $('.dropdown-submenu .show').removeClass("show"); + }); + + return false; +}); + +frappe.get_modal = function (title, content) { + return $( + `` + ); +}; + +frappe.ui.Dialog = class Dialog extends frappe.ui.Dialog { + get_primary_btn() { + return this.$wrapper.find(".modal-footer .btn-primary"); + } + + set_primary_action(label, click) { + this.$wrapper.find('.modal-footer').removeClass('hidden'); + return super.set_primary_action(label, click) + .removeClass('hidden'); + } + + make() { + super.make(); + if (this.fields) { + this.$wrapper.find('.section-body').addClass('w-100'); + } + } +}; diff --git a/frappe/public/js/chat.bundle.js b/frappe/public/js/chat.bundle.js new file mode 100644 index 0000000000..5f9a91ebb7 --- /dev/null +++ b/frappe/public/js/chat.bundle.js @@ -0,0 +1 @@ +import "./frappe/chat"; diff --git a/frappe/public/js/checkout.bundle.js b/frappe/public/js/checkout.bundle.js new file mode 100644 index 0000000000..954e838fa8 --- /dev/null +++ b/frappe/public/js/checkout.bundle.js @@ -0,0 +1 @@ +import "./integrations/razorpay"; diff --git a/frappe/public/js/controls.bundle.js b/frappe/public/js/controls.bundle.js new file mode 100644 index 0000000000..30b5d43905 --- /dev/null +++ b/frappe/public/js/controls.bundle.js @@ -0,0 +1,18 @@ +import "air-datepicker/dist/js/datepicker.min.js"; +import "air-datepicker/dist/js/i18n/datepicker.cs.js"; +import "air-datepicker/dist/js/i18n/datepicker.da.js"; +import "air-datepicker/dist/js/i18n/datepicker.de.js"; +import "air-datepicker/dist/js/i18n/datepicker.en.js"; +import "air-datepicker/dist/js/i18n/datepicker.es.js"; +import "air-datepicker/dist/js/i18n/datepicker.fi.js"; +import "air-datepicker/dist/js/i18n/datepicker.fr.js"; +import "air-datepicker/dist/js/i18n/datepicker.hu.js"; +import "air-datepicker/dist/js/i18n/datepicker.nl.js"; +import "air-datepicker/dist/js/i18n/datepicker.pl.js"; +import "air-datepicker/dist/js/i18n/datepicker.pt-BR.js"; +import "air-datepicker/dist/js/i18n/datepicker.pt.js"; +import "air-datepicker/dist/js/i18n/datepicker.ro.js"; +import "air-datepicker/dist/js/i18n/datepicker.sk.js"; +import "air-datepicker/dist/js/i18n/datepicker.zh.js"; +import "./frappe/ui/capture.js"; +import "./frappe/form/controls/control.js"; diff --git a/frappe/public/js/data_import_tools.bundle.js b/frappe/public/js/data_import_tools.bundle.js new file mode 100644 index 0000000000..b6e4c11968 --- /dev/null +++ b/frappe/public/js/data_import_tools.bundle.js @@ -0,0 +1 @@ +import "./frappe/data_import"; diff --git a/frappe/public/js/desk.bundle.js b/frappe/public/js/desk.bundle.js new file mode 100644 index 0000000000..66eb72cda0 --- /dev/null +++ b/frappe/public/js/desk.bundle.js @@ -0,0 +1,105 @@ +import "./frappe/translate.js"; +import "./frappe/class.js"; +import "./frappe/polyfill.js"; +import "./frappe/provide.js"; +import "./frappe/assets.js"; +import "./frappe/format.js"; +import "./frappe/form/formatters.js"; +import "./frappe/dom.js"; +import "./frappe/ui/messages.js"; +import "./frappe/ui/keyboard.js"; +import "./frappe/ui/colors.js"; +import "./frappe/ui/sidebar.js"; +import "./frappe/ui/link_preview.js"; + +import "./frappe/request.js"; +import "./frappe/socketio_client.js"; +import "./frappe/utils/utils.js"; +import "./frappe/event_emitter.js"; +import "./frappe/router.js"; +import "./frappe/router_history.js"; +import "./frappe/defaults.js"; +import "./frappe/roles_editor.js"; +import "./frappe/module_editor.js"; +import "./frappe/microtemplate.js"; + +import "./frappe/ui/page.html"; +import "./frappe/ui/page.js"; +import "./frappe/ui/slides.js"; +// import "./frappe/ui/onboarding_dialog.js"; +import "./frappe/ui/find.js"; +import "./frappe/ui/iconbar.js"; +import "./frappe/form/layout.js"; +import "./frappe/ui/field_group.js"; +import "./frappe/form/link_selector.js"; +import "./frappe/form/multi_select_dialog.js"; +import "./frappe/ui/dialog.js"; +import "./frappe/ui/capture.js"; +import "./frappe/ui/app_icon.js"; +import "./frappe/ui/theme_switcher.js"; + +import "./frappe/model/model.js"; +import "./frappe/db.js"; +import "./frappe/model/meta.js"; +import "./frappe/model/sync.js"; +import "./frappe/model/create_new.js"; +import "./frappe/model/perm.js"; +import "./frappe/model/workflow.js"; +import "./frappe/model/user_settings.js"; + +import "./frappe/utils/user.js"; +import "./frappe/utils/common.js"; +import "./frappe/utils/urllib.js"; +import "./frappe/utils/pretty_date.js"; +import "./frappe/utils/tools.js"; +import "./frappe/utils/datetime.js"; +import "./frappe/utils/number_format.js"; +import "./frappe/utils/help.js"; +import "./frappe/utils/help_links.js"; +import "./frappe/utils/address_and_contact.js"; +import "./frappe/utils/preview_email.js"; +import "./frappe/utils/file_manager.js"; + +import "./frappe/upload.js"; +import "./frappe/ui/tree.js"; + +import "./frappe/views/container.js"; +import "./frappe/views/breadcrumbs.js"; +import "./frappe/views/factory.js"; +import "./frappe/views/pageview.js"; + +import "./frappe/ui/toolbar/awesome_bar.js"; +// import "./frappe/ui/toolbar/energy_points_notifications.js"; +import "./frappe/ui/notifications/notifications.js"; +import "./frappe/ui/toolbar/search.js"; +import "./frappe/ui/toolbar/tag_utils.js"; +import "./frappe/ui/toolbar/search.html"; +import "./frappe/ui/toolbar/search_utils.js"; +import "./frappe/ui/toolbar/about.js"; +import "./frappe/ui/toolbar/navbar.html"; +import "./frappe/ui/toolbar/toolbar.js"; +// import "./frappe/ui/toolbar/notifications.js"; +import "./frappe/views/communication.js"; +import "./frappe/views/translation_manager.js"; +import "./frappe/views/workspace/workspace.js"; + +import "./frappe/widgets/widget_group.js"; + +import "./frappe/ui/sort_selector.html"; +import "./frappe/ui/sort_selector.js"; + +import "./frappe/change_log.html"; +import "./frappe/ui/workspace_loading_skeleton.html"; +import "./frappe/desk.js"; +import "./frappe/query_string.js"; + +// import "./frappe/ui/comment.js"; + +import "./frappe/chat.js"; +import "./frappe/utils/energy_point_utils.js"; +import "./frappe/utils/dashboard_utils.js"; +import "./frappe/ui/chart.js"; +import "./frappe/ui/datatable.js"; +import "./frappe/ui/driver.js"; +import "./frappe/ui/plyr.js"; +import "./frappe/barcode_scanner/index.js"; diff --git a/frappe/public/js/dialog.bundle.js b/frappe/public/js/dialog.bundle.js new file mode 100644 index 0000000000..3100b42ca7 --- /dev/null +++ b/frappe/public/js/dialog.bundle.js @@ -0,0 +1,7 @@ +import "./frappe/dom.js"; +import "./frappe/form/formatters.js"; +import "./frappe/form/layout.js"; +import "./frappe/ui/field_group.js"; +import "./frappe/form/link_selector.js"; +import "./frappe/form/multi_select_dialog.js"; +import "./frappe/ui/dialog.js"; diff --git a/frappe/public/js/form.bundle.js b/frappe/public/js/form.bundle.js new file mode 100644 index 0000000000..5bed5c2cb8 --- /dev/null +++ b/frappe/public/js/form.bundle.js @@ -0,0 +1,17 @@ +import "./frappe/form/templates/address_list.html"; +import "./frappe/form/templates/contact_list.html"; +import "./frappe/form/templates/form_dashboard.html"; +import "./frappe/form/templates/form_footer.html"; +import "./frappe/form/templates/form_links.html"; +import "./frappe/form/templates/form_sidebar.html"; +import "./frappe/form/templates/print_layout.html"; +import "./frappe/form/templates/report_links.html"; +import "./frappe/form/templates/set_sharing.html"; +import "./frappe/form/templates/timeline_message_box.html"; +import "./frappe/form/templates/users_in_sidebar.html"; + +import "./frappe/form/controls/control.js"; +import "./frappe/views/formview.js"; +import "./frappe/form/form.js"; +import "./frappe/meta_tag.js"; + diff --git a/frappe/public/js/frappe-web.bundle.js b/frappe/public/js/frappe-web.bundle.js new file mode 100644 index 0000000000..9f7875f96b --- /dev/null +++ b/frappe/public/js/frappe-web.bundle.js @@ -0,0 +1,26 @@ +import "./jquery-bootstrap"; +import "./frappe/class.js"; +import "./frappe/polyfill.js"; +import "./lib/md5.min.js"; +import "./frappe/provide.js"; +import "./frappe/format.js"; +import "./frappe/utils/number_format.js"; +import "./frappe/utils/utils.js"; +import "./frappe/utils/common.js"; +import "./frappe/ui/messages.js"; +import "./frappe/translate.js"; +import "./frappe/utils/pretty_date.js"; +import "./frappe/microtemplate.js"; +import "./frappe/query_string.js"; + +import "./frappe/upload.js"; + +import "./frappe/model/meta.js"; +import "./frappe/model/model.js"; +import "./frappe/model/perm.js"; + +import "./bootstrap-4-web.bundle"; + + +import "../../website/js/website.js"; +import "./frappe/socketio_client.js"; diff --git a/frappe/public/js/frappe/assets.js b/frappe/public/js/frappe/assets.js index 76441af235..3fca8640f3 100644 --- a/frappe/public/js/frappe/assets.js +++ b/frappe/public/js/frappe/assets.js @@ -9,7 +9,14 @@ frappe.require = function(items, callback) { if(typeof items === "string") { items = [items]; } - frappe.assets.execute(items, callback); + items = items.map(item => frappe.assets.bundled_asset(item)); + + return new Promise(resolve => { + frappe.assets.execute(items, () => { + resolve(); + callback && callback(); + }); + }); }; frappe.assets = { @@ -160,4 +167,11 @@ frappe.assets = { frappe.dom.set_style(txt); } }, + + bundled_asset(path) { + if (!path.startsWith('/assets') && path.includes('.bundle.')) { + return frappe.boot.assets_json[path] || path; + } + return path; + } }; diff --git a/frappe/public/js/frappe/barcode_scanner/index.js b/frappe/public/js/frappe/barcode_scanner/index.js index c5e7a7600f..fa3975b578 100644 --- a/frappe/public/js/frappe/barcode_scanner/index.js +++ b/frappe/public/js/frappe/barcode_scanner/index.js @@ -13,7 +13,7 @@ frappe.barcode.scan_barcode = function() { } }, reject); } else { - frappe.require('/assets/js/barcode_scanner.min.js', () => { + frappe.require('barcode_scanner.bundle.js', () => { frappe.barcode.get_barcode().then(barcode => { resolve(barcode); }); diff --git a/frappe/public/js/frappe/build_events/BuildError.vue b/frappe/public/js/frappe/build_events/BuildError.vue new file mode 100644 index 0000000000..6e10852719 --- /dev/null +++ b/frappe/public/js/frappe/build_events/BuildError.vue @@ -0,0 +1,111 @@ + + + diff --git a/frappe/public/js/frappe/build_events/BuildSuccess.vue b/frappe/public/js/frappe/build_events/BuildSuccess.vue new file mode 100644 index 0000000000..75a365fdc2 --- /dev/null +++ b/frappe/public/js/frappe/build_events/BuildSuccess.vue @@ -0,0 +1,52 @@ + + + diff --git a/frappe/public/js/frappe/build_events/build_events.bundle.js b/frappe/public/js/frappe/build_events/build_events.bundle.js new file mode 100644 index 0000000000..6c8986af3f --- /dev/null +++ b/frappe/public/js/frappe/build_events/build_events.bundle.js @@ -0,0 +1,48 @@ +import BuildError from "./BuildError.vue"; +import BuildSuccess from "./BuildSuccess.vue"; + +let $container = $("#build-events-overlay"); +let success = null; +let error = null; + +frappe.realtime.on("build_event", data => { + if (data.success) { + show_build_success(data); + } else if (data.error) { + show_build_error(data); + } +}); + +function show_build_success() { + if (error) { + error.hide(); + } + if (!success) { + let target = $('
') + .appendTo($container) + .get(0); + let vm = new Vue({ + el: target, + render: h => h(BuildSuccess) + }); + success = vm.$children[0]; + } + success.show(); +} + +function show_build_error(data) { + if (success) { + success.hide(); + } + if (!error) { + let target = $('
') + .appendTo($container) + .get(0); + let vm = new Vue({ + el: target, + render: h => h(BuildError) + }); + error = vm.$children[0]; + } + error.show(data); +} diff --git a/frappe/public/js/frappe/class.js b/frappe/public/js/frappe/class.js index 4f6dd0dc97..79ef2792ae 100644 --- a/frappe/public/js/frappe/class.js +++ b/frappe/public/js/frappe/class.js @@ -80,4 +80,4 @@ To subclass, use: // export global.Class = Class; - })(this); \ No newline at end of file + })(window); diff --git a/frappe/public/js/frappe/data_import/data_exporter.js b/frappe/public/js/frappe/data_import/data_exporter.js index dee4839b34..03e6288856 100644 --- a/frappe/public/js/frappe/data_import/data_exporter.js +++ b/frappe/public/js/frappe/data_import/data_exporter.js @@ -72,8 +72,8 @@ frappe.data_import.DataExporter = class DataExporter { let child_fieldname = df.fieldname; let label = df.reqd ? // prettier-ignore - __('{0} ({1}) (1 row mandatory)', [df.label || df.fieldname, doctype]) - : __('{0} ({1})', [df.label || df.fieldname, doctype]); + __('{0} ({1}) (1 row mandatory)', [__(df.label || df.fieldname), __(doctype)]) + : __('{0} ({1})', [__(df.label || df.fieldname), __(doctype)]); return { label, fieldname: child_fieldname, diff --git a/frappe/public/js/frappe/desk.js b/frappe/public/js/frappe/desk.js index c093a73689..46812f5fb6 100644 --- a/frappe/public/js/frappe/desk.js +++ b/frappe/public/js/frappe/desk.js @@ -24,12 +24,12 @@ $(document).ready(function() { frappe.start_app(); }); -frappe.Application = Class.extend({ - init: function() { +frappe.Application = class Application { + constructor() { this.startup(); - }, + } - startup: function() { + startup() { frappe.socketio.init(); frappe.model.init(); @@ -114,10 +114,8 @@ frappe.Application = Class.extend({ dialog.get_close_btn().toggle(false); }); - this.setup_user_group_listeners(); - // listen to build errors - this.setup_build_error_listener(); + this.setup_build_events(); if (frappe.sys_defaults.email_user_password) { var email_list = frappe.sys_defaults.email_user_password.split(','); @@ -162,7 +160,7 @@ frappe.Application = Class.extend({ }, 600000); // check every 10 minutes } } - }, + } set_route() { frappe.flags.setting_original_route = true; @@ -177,14 +175,14 @@ frappe.Application = Class.extend({ frappe.router.on('change', () => { $(".tooltip").hide(); }); - }, + } setup_frappe_vue() { Vue.prototype.__ = window.__; Vue.prototype.frappe = window.frappe; - }, + } - set_password: function(user) { + set_password(user) { var me=this; frappe.call({ method: 'frappe.core.doctype.user.user.get_email_awaiting', @@ -201,9 +199,9 @@ frappe.Application = Class.extend({ } } }); - }, + } - email_password_prompt: function(email_account,user,i) { + email_password_prompt(email_account,user,i) { var me = this; let d = new frappe.ui.Dialog({ title: __('Password missing in Email Account'), @@ -257,8 +255,8 @@ frappe.Application = Class.extend({ }); }); d.show(); - }, - load_bootinfo: function() { + } + load_bootinfo() { if(frappe.boot) { this.setup_workspaces(); frappe.model.sync(frappe.boot.docs); @@ -280,7 +278,7 @@ frappe.Application = Class.extend({ } else { this.set_as_guest(); } - }, + } setup_workspaces() { frappe.modules = {}; @@ -291,26 +289,26 @@ frappe.Application = Class.extend({ } if (!frappe.workspaces['home']) { // default workspace is settings for Frappe - frappe.workspaces['home'] = frappe.workspaces['build']; + frappe.workspaces['home'] = frappe.workspaces[Object.keys(frappe.workspaces)[0]]; } - }, + } - load_user_permissions: function() { + load_user_permissions() { frappe.defaults.update_user_permissions(); frappe.realtime.on('update_user_permissions', frappe.utils.debounce(() => { frappe.defaults.update_user_permissions(); }, 500)); - }, + } - check_metadata_cache_status: function() { + check_metadata_cache_status() { if(frappe.boot.metadata_version != localStorage.metadata_version) { frappe.assets.clear_local_storage(); frappe.assets.init_local_storage(); } - }, + } - set_globals: function() { + set_globals() { frappe.session.user = frappe.boot.user.name; frappe.session.logged_in_user = frappe.boot.user.name; frappe.session.user_email = frappe.boot.user.email; @@ -362,8 +360,8 @@ frappe.Application = Class.extend({ } } }); - }, - sync_pages: function() { + } + sync_pages() { // clear cached pages if timestamp is not found if(localStorage["page_info"]) { frappe.boot.allowed_pages = []; @@ -378,8 +376,8 @@ frappe.Application = Class.extend({ frappe.boot.allowed_pages = Object.keys(frappe.boot.page_info); } localStorage["page_info"] = JSON.stringify(frappe.boot.page_info); - }, - set_as_guest: function() { + } + set_as_guest() { frappe.session.user = 'Guest'; frappe.session.user_email = ''; frappe.session.user_fullname = 'Guest'; @@ -387,23 +385,23 @@ frappe.Application = Class.extend({ frappe.user_defaults = {}; frappe.user_roles = ['Guest']; frappe.sys_defaults = {}; - }, - make_page_container: function() { + } + make_page_container() { if ($("#body").length) { $(".splash").remove(); frappe.temp_container = $("