Merge branch 'develop' into get-all-mod

.flake8

@@ -28,6 +28,10 @@ ignore =
     B007,
     B950,
     W191,
+    E124, # closing bracket, irritating while writing QB code
+    E131, # continuation line unaligned for hanging indent
+    E123, # closing bracket does not match indentation of opening bracket's line
+    E101, # ensured by use of black
 
 max-line-length = 200
 exclude=.github/helper/semgrep_rules

.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"

.github/helper/roulette.py

@@ -5,8 +5,10 @@ import shlex
 import subprocess
 import sys
 import urllib.request
+from functools import cache
 
 
+@cache
 def fetch_pr_data(pr_number, repo, endpoint):
 	api_url = f"https://api.github.com/repos/{repo}/pulls/{pr_number}"
 
@@ -26,7 +28,16 @@ def get_output(command, shell=True):
 	return subprocess.check_output(command, shell=shell, encoding="utf8").strip()
 
 def has_skip_ci_label(pr_number, repo="frappe/frappe"):
-	return any([label["name"] for label in fetch_pr_data(pr_number, repo, "")["labels"] if label["name"] == "Skip CI"])
+	return has_label(pr_number, "Skip CI", repo)
+
+def has_run_server_tests_label(pr_number, repo="frappe/frappe"):
+	return has_label(pr_number, "Run Server Tests", repo)
+
+def has_run_ui_tests_label(pr_number, repo="frappe/frappe"):
+	return has_label(pr_number, "Run UI Tests", repo)
+
+def has_label(pr_number, label, repo="frappe/frappe"):
+	return any([label["name"] for label in fetch_pr_data(pr_number, repo, "")["labels"] if label["name"] == label])
 
 def is_py(file):
 	return file.endswith("py")
@@ -66,22 +77,22 @@ if __name__ == "__main__":
 	updated_py_file_count = len(list(filter(is_py, files_list)))
 	only_py_changed = updated_py_file_count == len(files_list)
 
-	if ci_files_changed:
-		print("CI related files were updated, running all build processes.")
-
-	elif has_skip_ci_label(pr_number, repo):
+	if has_skip_ci_label(pr_number, repo):
 		print("Found `Skip CI` label on pr, stopping build process.")
 		sys.exit(0)
 
+	elif ci_files_changed:
+		print("CI related files were updated, running all build processes.")
+
 	elif only_docs_changed:
 		print("Only docs were updated, stopping build process.")
 		sys.exit(0)
 
-	elif only_frontend_code_changed and build_type == "server":
+	elif only_frontend_code_changed and build_type == "server" and not has_run_server_tests_label(pr_number, repo):
 		print("Only Frontend code was updated; Stopping Python build process.")
 		sys.exit(0)
 
-	elif build_type == "ui" and only_py_changed:
+	elif build_type == "ui" and only_py_changed and not has_run_ui_tests_label(pr_number, repo):
 		print("Only Python code was updated, stopping Cypress build process.")
 		sys.exit(0)
 

.github/workflows/deps-checker.yml

@@ -0,0 +1,22 @@
+name: 'Python Dependency Check'
+on:
+  pull_request:
+  workflow_dispatch:
+  push:
+    branches: [ develop ]
+
+permissions:
+  contents: read
+
+jobs:
+  deps-vulnerable-check:
+    name: 'Vulnerable Dependency'
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/setup-python@v4
+        with:
+          python-version: 3.8
+      - uses: actions/checkout@v3
+      - run: pip install pip-audit
+      - run: pip-audit ${GITHUB_WORKSPACE}

.github/workflows/docs-checker.yml

@@ -13,12 +13,12 @@ jobs:
 
     steps:
       - name: 'Setup Environment'
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: 3.8
 
       - name: 'Clone repo'
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Validate Docs
         env:

.github/workflows/linters.yml

@@ -9,23 +9,21 @@ jobs:
     name: Frappe Linter
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v2
+      - name: Set up Python
+        uses: actions/setup-python@v4
         with:
-          python-version: 3.8
+          python-version: '3.10'
 
       - name: Install and Run Pre-commit
-        uses: pre-commit/action@v2.0.3
+        uses: pre-commit/action@v3.0.0
 
       - name: Download Semgrep rules
         run: git clone --depth 1 https://github.com/frappe/semgrep-rules.git frappe-semgrep-rules
 
-      - uses: returntocorp/semgrep-action@v1
-        env:
-            SEMGREP_TIMEOUT: 120
-        with:
-            config: >-
-              r/python.lang.correctness
-              ./frappe-semgrep-rules/rules
+      - name: Download semgrep
+        run: pip install semgrep==0.97.0
+
+      - name: Run Semgrep rules
+        run: semgrep ci --config ./frappe-semgrep-rules/rules --config r/python.lang.correctness

.github/workflows/patch-mariadb-tests.yml

@@ -28,15 +28,15 @@ jobs:
 
     steps:
       - name: Clone
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Setup Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: '3.9'
 
       - name: Setup Node
-        uses: actions/setup-node@v2
+        uses: actions/setup-node@v3
         with:
           node-version: 14
           check-latest: true
@@ -56,17 +56,17 @@ jobs:
 
       - name: Cache pip
         if: ${{ steps.check-build.outputs.build == 'strawberry' }}
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/*requirements.txt', '**/pyproject.toml', '**/setup.py') }}
           restore-keys: |
             ${{ runner.os }}-pip-
             ${{ runner.os }}-
 
       - name: Cache node modules
         if: ${{ steps.check-build.outputs.build == 'strawberry' }}
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         env:
           cache-name: cache-node-modules
         with:
@@ -82,7 +82,7 @@ jobs:
         id: yarn-cache-dir-path
         run: echo "::set-output name=dir::$(yarn cache dir)"
 
-      - uses: actions/cache@v2
+      - uses: actions/cache@v3
         if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         id: yarn-cache
         with:
@@ -124,7 +124,7 @@ jobs:
               git fetch --depth 1 upstream $branch_name:$branch_name
 
               git checkout -q -f $branch_name
-              pip install -q -r requirements.txt
+              bench setup requirements --python
               bench --site test_site migrate
           done
 

.github/workflows/publish-assets-develop.yml

@@ -10,13 +10,13 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           path: 'frappe'
-      - uses: actions/setup-node@v1
+      - uses: actions/setup-node@v3
         with:
           node-version: 14
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v4
         with:
           python-version: '3.9'
       - name: Set up bench and build assets

.github/workflows/publish-assets-releases.yml

@@ -13,13 +13,13 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           path: 'frappe'
-      - uses: actions/setup-node@v1
+      - uses: actions/setup-node@v3
         with:
           python-version: '12.x'
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v4
         with:
           python-version: '3.9'
       - name: Set up bench and build assets
@@ -36,7 +36,7 @@ jobs:
 
       - name: Get release
         id: get_release
-        uses: bruceadams/get-release@v1.2.0
+        uses: bruceadams/get-release@v1.2.3
 
       - name: Upload built Assets to Release
         uses: actions/upload-release-asset@v1.0.2

.github/workflows/release.yml

@@ -12,12 +12,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Entire Repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 0
           persist-credentials: false
       - name: Setup Node.js v14
-        uses: actions/setup-node@v2
+        uses: actions/setup-node@v3
         with:
           node-version: 14
       - name: Setup dependencies

.github/workflows/server-mariadb-tests.yml

@@ -37,10 +37,10 @@ jobs:
 
     steps:
       - name: Clone
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Setup Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: '3.9'
 
@@ -53,7 +53,7 @@ jobs:
           PR_NUMBER: ${{ github.event.number }}
           REPO_NAME: ${{ github.repository }}
 
-      - uses: actions/setup-node@v2
+      - uses: actions/setup-node@v3
         if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         with:
           node-version: 14
@@ -67,17 +67,17 @@ jobs:
 
       - name: Cache pip
         if: ${{ steps.check-build.outputs.build == 'strawberry' }}
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/*requirements.txt', '**/pyproject.toml', '**/setup.py') }}
           restore-keys: |
             ${{ runner.os }}-pip-
             ${{ runner.os }}-
 
       - name: Cache node modules
         if: ${{ steps.check-build.outputs.build == 'strawberry' }}
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         env:
           cache-name: cache-node-modules
         with:
@@ -93,7 +93,7 @@ jobs:
         id: yarn-cache-dir-path
         run: echo "::set-output name=dir::$(yarn cache dir)"
 
-      - uses: actions/cache@v2
+      - uses: actions/cache@v3
         if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         id: yarn-cache
         with:
@@ -126,7 +126,7 @@ jobs:
 
       - name: Upload coverage data
         if: ${{ steps.check-build.outputs.build == 'strawberry' }}
-        uses: codecov/codecov-action@v2
+        uses: codecov/codecov-action@v3
         with:
           name: MariaDB
           fail_ci_if_error: true

.github/workflows/server-postgres-tests.yml

@@ -40,10 +40,10 @@ jobs:
 
     steps:
       - name: Clone
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Setup Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: '3.9'
 
@@ -56,7 +56,7 @@ jobs:
           PR_NUMBER: ${{ github.event.number }}
           REPO_NAME: ${{ github.repository }}
 
-      - uses: actions/setup-node@v2
+      - uses: actions/setup-node@v3
         if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         with:
           node-version: '14'
@@ -70,17 +70,17 @@ jobs:
 
       - name: Cache pip
         if: ${{ steps.check-build.outputs.build == 'strawberry' }}
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/*requirements.txt', '**/pyproject.toml', '**/setup.py') }}
           restore-keys: |
             ${{ runner.os }}-pip-
             ${{ runner.os }}-
 
       - name: Cache node modules
         if: ${{ steps.check-build.outputs.build == 'strawberry' }}
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         env:
           cache-name: cache-node-modules
         with:
@@ -96,7 +96,7 @@ jobs:
         id: yarn-cache-dir-path
         run: echo "::set-output name=dir::$(yarn cache dir)"
 
-      - uses: actions/cache@v2
+      - uses: actions/cache@v3
         if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         id: yarn-cache
         with:
@@ -129,7 +129,7 @@ jobs:
 
       - name: Upload coverage data
         if: ${{ steps.check-build.outputs.build == 'strawberry' }}
-        uses: codecov/codecov-action@v2
+        uses: codecov/codecov-action@v3
         with:
           name: Postgres
           fail_ci_if_error: true

.github/workflows/ui-tests.yml

@@ -21,7 +21,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-       containers: [1, 2]
+       containers: [1, 2, 3]
 
     name: UI Tests (Cypress)
 
@@ -36,10 +36,10 @@ jobs:
 
     steps:
       - name: Clone
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Setup Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: '3.9'
 
@@ -52,7 +52,7 @@ jobs:
           PR_NUMBER: ${{ github.event.number }}
           REPO_NAME: ${{ github.repository }}
 
-      - uses: actions/setup-node@v2
+      - uses: actions/setup-node@v3
         if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         with:
           node-version: 14
@@ -66,17 +66,17 @@ jobs:
 
       - name: Cache pip
         if: ${{ steps.check-build.outputs.build == 'strawberry' }}
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/*requirements.txt', '**/pyproject.toml', '**/setup.py') }}
           restore-keys: |
             ${{ runner.os }}-pip-
             ${{ runner.os }}-
 
       - name: Cache node modules
         if: ${{ steps.check-build.outputs.build == 'strawberry' }}
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         env:
           cache-name: cache-node-modules
         with:
@@ -92,7 +92,7 @@ jobs:
         id: yarn-cache-dir-path
         run: echo "::set-output name=dir::$(yarn cache dir)"
 
-      - uses: actions/cache@v2
+      - uses: actions/cache@v3
         if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         id: yarn-cache
         with:
@@ -103,7 +103,7 @@ jobs:
 
       - name: Cache cypress binary
         if: ${{ steps.check-build.outputs.build == 'strawberry' }}
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: ~/.cache
           key: ${{ runner.os }}-cypress-
@@ -158,7 +158,7 @@ jobs:
 
       - name: Upload Coverage Data
         if: ${{ steps.check-build.outputs.build == 'strawberry' && steps.check_coverage.outputs.files_exists == 'true' }}
-        uses: codecov/codecov-action@v2
+        uses: codecov/codecov-action@v3
         with:
           name: Cypress
           fail_ci_if_error: true
@@ -168,7 +168,7 @@ jobs:
 
       - name: Upload Server Coverage Data
         if: ${{ steps.check-build.outputs.build-server == 'strawberry' }}
-        uses: codecov/codecov-action@v2
+        uses: codecov/codecov-action@v3
         with:
           name: MariaDB
           fail_ci_if_error: true

.mergify.yml

@@ -7,6 +7,7 @@ pull_request_rules:
           - author!=gavindsouza
           - author!=deepeshgarg007
           - author!=ankush
+          - author!=mergify[bot]
         - or:
           - base=version-13
           - base=version-12
@@ -20,13 +21,13 @@ pull_request_rules:
   - name: Automatic merge on CI success and review
     conditions:
       - status-success=Sider
-      - status-success=Semantic Pull Request
       - status-success=Python Unit Tests (MariaDB) (1)
       - status-success=Python Unit Tests (MariaDB) (2)
       - status-success=Python Unit Tests (Postgres) (1)
       - status-success=Python Unit Tests (Postgres) (2)
       - status-success=UI Tests (Cypress) (1)
       - status-success=UI Tests (Cypress) (2)
+      - status-success=UI Tests (Cypress) (3)
       - status-success=security/snyk (frappe)
       - label!=dont-merge
       - label!=squash
@@ -43,6 +44,7 @@ pull_request_rules:
       - status-success=Python Unit Tests (Postgres) (2)
       - status-success=UI Tests (Cypress) (1)
       - status-success=UI Tests (Cypress) (2)
+      - status-success=UI Tests (Cypress) (3)
       - status-success=security/snyk (frappe)
       - label!=dont-merge
       - label=squash

CODEOWNERS

@@ -12,7 +12,7 @@ data_import*      @netchampfaris
 core/             @surajshetty3416
 database          @gavindsouza
 model             @gavindsouza
-requirements.txt  @gavindsouza
+pyproject.toml    @gavindsouza
 query_builder/    @gavindsouza
 commands/         @gavindsouza
 workspace         @shariquerik

cypress/integration/awesome_bar.js

@@ -28,6 +28,7 @@ context('Awesome Bar', () => {
 
 		cy.findByPlaceholderText('ID')
 			.should('have.value', '%test%');
+		cy.clear_filters();
 	});
 
 	it('navigates to new form', () => {

cypress/integration/control_date_range.js

@@ -0,0 +1,42 @@
+context('Date Range Control', () => {
+	before(() => {
+		cy.login();
+		cy.visit('/app');
+	});
+
+	function get_dialog() {
+		return cy.dialog({
+			title: 'Date Range',
+			fields: [{
+				"label": "Date Range",
+				"fieldname": "date_range",
+				"fieldtype": "Date Range",
+			}]
+		});
+	}
+
+	it('Selecting a date range from the datepicker', () => {
+		cy.clear_dialogs();
+		cy.clear_datepickers();
+
+		get_dialog().as('dialog');
+		cy.get_field('date_range', 'Date Range').click();
+		cy.get('.datepicker--nav-title').click();
+		cy.get('.datepicker--nav-title').click({force: true});
+
+		//Inputing date range values in the date range field
+		cy.get('.datepicker--years > .datepicker--cells > .datepicker--cell[data-year=2020]').click();
+		cy.get('.datepicker--months > .datepicker--cells > .datepicker--cell[data-month=0]').click();
+		cy.get('.datepicker--cell[data-date=1]:first').click({force: true});
+		cy.get('.datepicker--cell[data-date=15]:first').click({force: true});
+
+		// Verify if the selected date range values is set in the date range field
+		cy.window()
+			.its('cur_dialog')
+			.then(dialog => {
+				let date_range = dialog.get_value("date_range");
+				expect(date_range[0]).to.equal('2020-01-01');
+				expect(date_range[1]).to.equal('2020-01-15');
+			});
+	});
+});

cypress/integration/custom_buttons.js

@@ -4,6 +4,7 @@ const test_button_names = [
 	"Porcupine Tree (the GOAT)",
 	"AC / DC",
 	`Electronic Dance "music"`,
+	"l'imperatrice",
 ];
 
 const add_button = (label, group = "TestGroup") => {

cypress/integration/form.js

@@ -78,4 +78,20 @@ context('Form', () => {
 		cy.get('@row2').click();
 		cy.get('@email_input2').should('not.have.class', 'invalid');
 	});
+
+	it('Shows version conflict warning', { scrollBehavior: false }, () => {
+		cy.visit('/app/todo');
+
+		cy.insert_doc("ToDo", {"description": "old"}).then(doc => {
+			cy.visit(`/app/todo/${doc.name}`);
+			// make form dirty
+			cy.fill_field("status", "Cancelled", "Select");
+
+			// update doc using api - simulating parallel change by another user
+			cy.update_doc("ToDo", doc.name, {"status": "Closed"}).then(() => {
+				cy.findByRole("button", {name: "Refresh"}).click();
+				cy.get_field("status", "Select").should("have.value", "Closed");
+			})
+		})
+	});
 });

cypress/integration/routing.js

@@ -0,0 +1,40 @@
+const list_view = "/app/todo";
+
+// test round trip with filter types
+
+const test_queries = [
+	"?status=Open",
+	`?date=%5B"Between"%2C%5B"2022-06-01"%2C"2022-06-30"%5D%5D`,
+	`?date=%5B">"%2C"2022-06-01"%5D`,
+	`?name=%5B"like"%2C"%2542%25"%5D`,
+	`?status=%5B"not%20in"%2C%5B"Open"%2C"Closed"%5D%5D`,
+];
+
+describe("SPA Routing", { scrollBehavior: false }, () => {
+	before(() => {
+		cy.login();
+		cy.go_to_list("ToDo");
+	});
+
+	after(() => {
+		cy.clear_filters(); // avoid flake in future tests
+	});
+
+	it("should apply filter on list view from route", () => {
+		test_queries.forEach((query) => {
+			const full_url = `${list_view}${query}`;
+			cy.visit(full_url);
+			cy.findByTitle("To Do").should("exist");
+
+			const expected = new URLSearchParams(query);
+			cy.location().then((loc) => {
+				const actual = new URLSearchParams(loc.search);
+				// This might appear like a dumb test checking visited URL to itself
+				// but it's actually doing a round trip
+				// URL with params -> parsed filters -> new URL
+				// if it's same that means everything worked in between.
+				expect(actual.toString()).to.eq(expected.toString());
+			});
+		});
+	});
+});

cypress/integration/timeline_email.js

@@ -7,7 +7,7 @@ context('Timeline Email', () => {
 
 	it('Adding new ToDo', () => {
 		cy.click_listview_primary_button('Add ToDo');
-		cy.get('.custom-actions:visible > .btn').contains("Edit in full page").click({delay: 500});
+		cy.get('.custom-actions:visible > .btn').contains("Edit Full Form").click({delay: 500});
 		cy.fill_field("description", "Test ToDo",  "Text Editor");
 		cy.wait(500);
 		cy.get('.primary-action').contains('Save').click({force: true});

cypress/support/commands.js

@@ -271,10 +271,9 @@ Cypress.Commands.add('save', () => {
 	cy.get(`button[data-label="Save"]:visible`).click({scrollBehavior: false, force: true});
 	cy.wait('@api');
 });
-
 Cypress.Commands.add('hide_dialog', () => {
-	cy.wait(400);
-	cy.get('.btn-modal-close:visible').click({force: true});
+	cy.wait(300);
+	cy.get_open_dialog().focus().find('.btn-modal-close').click();
 	cy.get('.modal:visible').should('not.exist');
 });
 
@@ -292,7 +291,11 @@ Cypress.Commands.add('clear_datepickers', () => {
 	cy.get('.datepicker').should('not.exist');
 });
 
+
 Cypress.Commands.add('insert_doc', (doctype, args, ignore_duplicate) => {
+	if (!args.doctype) {
+		args.doctype = doctype;
+	}
 	return cy
 		.window()
 		.its('frappe.csrf_token')
@@ -314,12 +317,41 @@ Cypress.Commands.add('insert_doc', (doctype, args, ignore_duplicate) => {
 					if (ignore_duplicate) {
 						status_codes.push(409);
 					}
-					expect(res.status).to.be.oneOf(status_codes);
+
+					let message = null;
+					if (ignore_duplicate && !status_codes.includes(res.status)) {
+						message = `Document insert failed, response: ${JSON.stringify(res, null, '\t')}`;
+					}
+					expect(res.status).to.be.oneOf(status_codes, message);
 					return res.body.data;
 				});
 		});
 });
 
+Cypress.Commands.add('update_doc', (doctype, docname, args) => {
+	return cy
+		.window()
+		.its('frappe.csrf_token')
+		.then(csrf_token => {
+			return cy
+				.request({
+					method: 'PUT',
+					url: `/api/resource/${doctype}/${docname}`,
+					body: args,
+					headers: {
+						Accept: 'application/json',
+						'Content-Type': 'application/json',
+						'X-Frappe-CSRF-Token': csrf_token
+					},
+				})
+				.then(res => {
+					expect(res.status).to.eq(200);
+					return res.body.data;
+				});
+		});
+});
+
+
 Cypress.Commands.add('open_list_filter', () => {
 	cy.get('.filter-section .filter-button').click();
 	cy.wait(300);

dev-requirements.txt

@@ -1,4 +1,4 @@
 coverage==5.5
-Faker~=8.1.0
+Faker~=13.12.1
 pyngrok~=5.0.5
 unittest-xml-reporting~=3.0.4

frappe/__init__.py

@@ -15,8 +15,9 @@ import importlib
 import inspect
 import json
 import os
+import re
 import warnings
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union
 
 import click
 from werkzeug.local import Local, release_local
@@ -49,6 +50,11 @@ local = Local()
 STANDARD_USERS = ("Guest", "Administrator")
 
 _dev_server = int(sbool(os.environ.get("DEV_SERVER", False)))
+_qb_patched = {}
+re._MAXCACHE = (
+	50  # reduced from default 512 given we are already maintaining this on parent worker
+)
+
 
 if _dev_server:
 	warnings.simplefilter("always", DeprecationWarning)
@@ -77,7 +83,7 @@ class _dict(dict):
 		return _dict(self)
 
 
-def _(msg, lang=None, context=None):
+def _(msg, lang=None, context=None) -> str:
 	"""Returns translated string in current lang, if exists.
 	Usage:
 	        _('Change')
@@ -241,8 +247,10 @@ def init(site, sites_path=None, new_site=False):
 	local.qb = get_query_builder(local.conf.db_type or "mariadb")
 	local.qb.engine = get_qb_engine()
 	setup_module_map()
-	patch_query_execute()
-	patch_query_aggregation()
+
+	if not _qb_patched.get(local.conf.db_type):
+		patch_query_execute()
+		patch_query_aggregation()
 
 	local.initialised = True
 
@@ -429,9 +437,6 @@ def msgprint(
 
 	def _raise_exception():
 		if raise_exception:
-			if flags.rollback_on_exception:
-				db.rollback()
-
 			if inspect.isclass(raise_exception) and issubclass(raise_exception, Exception):
 				raise raise_exception(msg)
 			else:
@@ -873,6 +878,10 @@ def clear_cache(user=None, doctype=None):
 	local.role_permissions = {}
 	if hasattr(local, "request_cache"):
 		local.request_cache.clear()
+	if hasattr(local, "system_settings"):
+		del local.system_settings
+	if hasattr(local, "website_settings"):
+		del local.website_settings
 
 
 def only_has_select_perm(doctype, user=None, ignore_permissions=False):
@@ -919,7 +928,7 @@ def has_permission(
 
 	if throw and not out:
 		# mimics frappe.throw
-		document_label = f"{doc.doctype} {doc.name}" if doc else doctype
+		document_label = f"{_(doc.doctype)} {doc.name}" if doc else _(doctype)
 		msgprint(
 			_("No permission for {0}").format(document_label),
 			raise_exception=ValidationError,
@@ -1096,6 +1105,10 @@ def clear_document_cache(doctype, name):
 	if key in local.document_cache:
 		del local.document_cache[key]
 	cache().hdel("document_cache", key)
+	if doctype == "System Settings" and hasattr(local, "system_settings"):
+		delattr(local, "system_settings")
+	if doctype == "Website Settings" and hasattr(local, "website_settings"):
+		delattr(local, "website_settings")
 
 
 def get_cached_value(doctype, name, fieldname="name", as_dict=False):
@@ -1540,7 +1553,15 @@ def call(fn, *args, **kwargs):
 	return fn(*args, **newargs)
 
 
-def get_newargs(fn, kwargs):
+def get_newargs(fn: Callable, kwargs: Dict[str, Any]) -> Dict[str, Any]:
+	"""Remove any kwargs that are not supported by the function.
+
+	Example:
+	        >>> def fn(a=1, b=2): pass
+
+	        >>> get_newargs(fn, {"a": 2, "c": 1})
+	                {"a": 2}
+	"""
 
 	# if function has any **kwargs parameter that capture arbitrary keyword arguments
 	# Ref: https://docs.python.org/3/library/inspect.html#inspect.Parameter.kind
@@ -2208,8 +2229,18 @@ def safe_eval(code, eval_globals=None, eval_locals=None):
 	return eval(code, eval_globals, eval_locals)
 
 
+def get_website_settings(key):
+	if not hasattr(local, "website_settings"):
+		local.website_settings = db.get_singles_dict("Website Settings", cast=True)
+
+	return local.website_settings.get(key)
+
+
 def get_system_settings(key):
-	return db.get_single_value("System Settings", key, cache=True)
+	if not hasattr(local, "system_settings"):
+		local.system_settings = db.get_singles_dict("System Settings", cast=True)
+
+	return local.system_settings.get(key)
 
 
 def get_active_domains():

frappe/client.py

@@ -100,8 +100,8 @@ def get_value(doctype, fieldname, filters=None, as_dict=True, debug=False, paren
 	if frappe.is_table(doctype):
 		check_parent_permission(parent, doctype)
 
-	if not frappe.has_permission(doctype):
-		frappe.throw(_("No permission for {0}").format(doctype), frappe.PermissionError)
+	if not frappe.has_permission(doctype, parent_doctype=parent):
+		frappe.throw(_("No permission for {0}").format(_(doctype)), frappe.PermissionError)
 
 	filters = get_safe_filters(filters)
 	if isinstance(filters, str):
@@ -143,7 +143,7 @@ def get_value(doctype, fieldname, filters=None, as_dict=True, debug=False, paren
 @frappe.whitelist()
 def get_single_value(doctype, field):
 	if not frappe.has_permission(doctype):
-		frappe.throw(_("No permission for {0}").format(doctype), frappe.PermissionError)
+		frappe.throw(_("No permission for {0}").format(_(doctype)), frappe.PermissionError)
 	value = frappe.db.get_single_value(doctype, field)
 	return value
 
@@ -281,12 +281,6 @@ def set_default(key, value, parent=None):
 	frappe.clear_cache(user=frappe.session.user)
 
 
-@frappe.whitelist()
-def get_default(key, parent=None):
-	"""set a user default value"""
-	return frappe.db.get_default(key, parent)
-
-
 @frappe.whitelist(methods=["POST", "PUT"])
 def make_width_property_setter(doc):
 	"""Set width Property Setter

frappe/commands/site.py

@@ -9,6 +9,7 @@ import click
 # imports - module imports
 import frappe
 from frappe.commands import get_site, pass_context
+from frappe.core.doctype.log_settings.log_settings import LOG_DOCTYPES
 from frappe.exceptions import SiteNotSpecifiedError
 
 
@@ -1088,6 +1089,51 @@ def build_search_index(context):
 		frappe.destroy()
 
 
+@click.command("clear-log-table")
+@click.option("--doctype", default="text", type=click.Choice(LOG_DOCTYPES), help="Log DocType")
+@click.option("--days", type=int, help="Keep records for days")
+@click.option("--no-backup", is_flag=True, default=False, help="Do not backup the table")
+@pass_context
+def clear_log_table(context, doctype, days, no_backup):
+	"""If any logtype table grows too large then clearing it with DELETE query
+	is not feasible in reasonable time. This command copies recent data to new
+	table and replaces current table with new smaller table.
+
+
+	ref: https://mariadb.com/kb/en/big-deletes/#deleting-more-than-half-a-table
+	"""
+	from frappe.core.doctype.log_settings.log_settings import clear_log_table as clear_logs
+	from frappe.utils.backups import scheduled_backup
+
+	if not context.sites:
+		raise SiteNotSpecifiedError
+
+	if doctype not in LOG_DOCTYPES:
+		raise frappe.ValidationError(f"Unsupported logging DocType: {doctype}")
+
+	for site in context.sites:
+		frappe.init(site=site)
+		frappe.connect()
+
+		if not no_backup:
+			scheduled_backup(
+				ignore_conf=False,
+				include_doctypes=doctype,
+				ignore_files=True,
+				force=True,
+			)
+			click.echo(f"Backed up {doctype}")
+
+		try:
+			click.echo(f"Copying {doctype} records from last {days} days to temporary table.")
+			clear_logs(doctype, days=days)
+		except Exception as e:
+			click.echo(f"Log cleanup for {doctype} failed:\n{e}")
+			sys.exit(1)
+		else:
+			click.secho(f"Cleared {doctype} records older than {days} days", fg="green")
+
+
 @click.command("trim-database")
 @click.option("--dry-run", is_flag=True, default=False, help="Show what would be deleted")
 @click.option(
@@ -1260,4 +1306,5 @@ commands = [
 	partial_restore,
 	trim_tables,
 	trim_database,
+	clear_log_table,
 ]

frappe/commands/translate.py

@@ -48,11 +48,12 @@ def new_language(context, lang_code, app):
 
 
 @click.command("get-untranslated")
+@click.option("--app", default="_ALL_APPS")
 @click.argument("lang")
 @click.argument("untranslated_file")
 @click.option("--all", default=False, is_flag=True, help="Get all message strings")
 @pass_context
-def get_untranslated(context, lang, untranslated_file, all=None):
+def get_untranslated(context, lang, untranslated_file, app="_ALL_APPS", all=None):
 	"Get untranslated strings for language"
 	import frappe.translate
 
@@ -60,17 +61,18 @@ def get_untranslated(context, lang, untranslated_file, all=None):
 	try:
 		frappe.init(site=site)
 		frappe.connect()
-		frappe.translate.get_untranslated(lang, untranslated_file, get_all=all)
+		frappe.translate.get_untranslated(lang, untranslated_file, get_all=all, app=app)
 	finally:
 		frappe.destroy()
 
 
 @click.command("update-translations")
+@click.option("--app", default="_ALL_APPS")
 @click.argument("lang")
 @click.argument("untranslated_file")
 @click.argument("translated-file")
 @pass_context
-def update_translations(context, lang, untranslated_file, translated_file):
+def update_translations(context, lang, untranslated_file, translated_file, app="_ALL_APPS"):
 	"Update translated strings"
 	import frappe.translate
 
@@ -78,7 +80,7 @@ def update_translations(context, lang, untranslated_file, translated_file):
 	try:
 		frappe.init(site=site)
 		frappe.connect()
-		frappe.translate.update_translations(lang, untranslated_file, translated_file)
+		frappe.translate.update_translations(lang, untranslated_file, translated_file, app=app)
 	finally:
 		frappe.destroy()
 

frappe/contacts/address_and_contact.py

@@ -3,6 +3,7 @@
 
 import functools
 import re
+from typing import Dict, List
 
 import frappe
 from frappe import _
@@ -169,29 +170,35 @@ def delete_contact_and_address(doctype, docname):
 
 @frappe.whitelist()
 @frappe.validate_and_sanitize_search_inputs
-def filter_dynamic_link_doctypes(doctype, txt, searchfield, start, page_len, filters):
-	if not txt:
-		txt = ""
+def filter_dynamic_link_doctypes(
+	doctype, txt: str, searchfield, start, page_len, filters: Dict
+) -> List[List[str]]:
+	from frappe.permissions import get_doctypes_with_read
 
-	doctypes = frappe.db.get_all(
-		"DocField", filters=filters, fields=["parent"], distinct=True, as_list=True
+	txt = txt or ""
+	filters = filters or {}
+
+	_doctypes_from_df = frappe.get_all(
+		"DocField",
+		filters=filters,
+		pluck="parent",
+		distinct=True,
+		order_by=None,
 	)
+	doctypes_from_df = {d for d in _doctypes_from_df if txt.lower() in _(d).lower()}
 
-	doctypes = tuple(d for d in doctypes if re.search(txt + ".*", _(d[0]), re.IGNORECASE))
+	filters.update({"dt": ("not in", doctypes_from_df)})
+	_doctypes_from_cdf = frappe.get_all(
+		"Custom Field", filters=filters, pluck="dt", distinct=True, order_by=None
+	)
+	doctypes_from_cdf = {d for d in _doctypes_from_cdf if txt.lower() in _(d).lower()}
 
-	filters.update({"dt": ("not in", [d[0] for d in doctypes])})
+	all_doctypes = doctypes_from_df.union(doctypes_from_cdf)
+	allowed_doctypes = set(get_doctypes_with_read())
 
-	_doctypes = frappe.db.get_all("Custom Field", filters=filters, fields=["dt"], as_list=True)
+	valid_doctypes = sorted(all_doctypes.intersection(allowed_doctypes))
 
-	_doctypes = tuple([d for d in _doctypes if re.search(txt + ".*", _(d[0]), re.IGNORECASE)])
-
-	all_doctypes = [d[0] for d in doctypes + _doctypes]
-	allowed_doctypes = frappe.permissions.get_doctypes_with_read()
-
-	valid_doctypes = sorted(set(all_doctypes).intersection(set(allowed_doctypes)))
-	valid_doctypes = [[doctype] for doctype in valid_doctypes]
-
-	return valid_doctypes
+	return [[doctype] for doctype in valid_doctypes]
 
 
 def set_link_title(doc):

frappe/core/doctype/access_log/access_log.json

@@ -36,6 +36,7 @@
    "fieldname": "user",
    "fieldtype": "Link",
    "in_list_view": 1,
+   "in_standard_filter": 1,
    "label": "User ",
    "options": "User",
    "read_only": 1
@@ -51,6 +52,7 @@
    "fieldname": "reference_document",
    "fieldtype": "Data",
    "in_list_view": 1,
+   "in_standard_filter": 1,
    "label": "Reference Document",
    "read_only": 1
   },
@@ -129,7 +131,7 @@
   }
  ],
  "links": [],
- "modified": "2022-05-03 09:34:19.337551",
+ "modified": "2022-06-13 05:59:26.866004",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "Access Log",

frappe/core/doctype/activity_log/activity_log.py

@@ -25,6 +25,13 @@ class ActivityLog(Document):
 		if self.reference_doctype and self.reference_name:
 			self.status = "Linked"
 
+	@staticmethod
+	def clear_old_logs(days=None):
+		if not days:
+			days = 90
+		doctype = DocType("Activity Log")
+		frappe.db.delete(doctype, filters=(doctype.modified < (Now() - Interval(days=days))))
+
 
 def on_doctype_update():
 	"""Add indexes in `tabActivity Log`"""
@@ -43,12 +50,3 @@ def add_authentication_log(subject, user, operation="Login", status="Success"):
 			"operation": operation,
 		}
 	).insert(ignore_permissions=True, ignore_links=True)
-
-
-def clear_activity_logs(days=None):
-	"""clear 90 day old authentication logs or configured in log settings"""
-
-	if not days:
-		days = 90
-	doctype = DocType("Activity Log")
-	frappe.db.delete(doctype, filters=(doctype.creation < (Now() - Interval(days=days))))

frappe/core/doctype/activity_log/activity_log_list.js

@@ -4,5 +4,10 @@ frappe.listview_settings['Activity Log'] = {
 			return [__(doc.status), "green"];
 		else if(doc.operation == "Login" && doc.status == "Failed")
 			return [__(doc.status), "red"];
-	}
-};
+	},
+	onload: function(listview) {
+		frappe.require("logtypes.bundle.js", () => {
+			frappe.utils.logtypes.show_log_retention_message(cur_list.doctype);
+		})
+	},
+};

frappe/core/doctype/deleted_document/deleted_document.json

@@ -1,256 +1,81 @@
 {
- "allow_copy": 0, 
- "allow_import": 0, 
- "allow_rename": 0, 
- "beta": 0, 
- "creation": "2016-12-29 12:59:48.638970", 
- "custom": 0, 
- "docstatus": 0, 
- "doctype": "DocType", 
- "document_type": "", 
- "editable_grid": 1, 
- "engine": "InnoDB", 
+ "actions": [],
+ "creation": "2016-12-29 12:59:48.638970",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "deleted_name",
+  "deleted_doctype",
+  "column_break_3",
+  "restored",
+  "new_name",
+  "section_break_6",
+  "data"
+ ],
  "fields": [
   {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "deleted_name", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Deleted Name", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "fieldname": "deleted_name",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "in_standard_filter": 1,
+   "label": "Deleted Name",
+   "read_only": 1
+  },
   {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "deleted_doctype", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Deleted DocType", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "fieldname": "deleted_doctype",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "in_standard_filter": 1,
+   "label": "Deleted DocType",
+   "read_only": 1
+  },
   {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "column_break_3", 
-   "fieldtype": "Column Break", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "fieldname": "column_break_3",
+   "fieldtype": "Column Break"
+  },
   {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "restored", 
-   "fieldtype": "Check", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Restored", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 1, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "default": "0",
+   "fieldname": "restored",
+   "fieldtype": "Check",
+   "in_list_view": 1,
+   "label": "Restored",
+   "read_only": 1
+  },
   {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "new_name", 
-   "fieldtype": "Read Only", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "New Name", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "fieldname": "new_name",
+   "fieldtype": "Read Only",
+   "label": "New Name"
+  },
   {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "section_break_6", 
-   "fieldtype": "Section Break", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "fieldname": "section_break_6",
+   "fieldtype": "Section Break"
+  },
   {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "data", 
-   "fieldtype": "Code", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Data", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
+   "fieldname": "data",
+   "fieldtype": "Code",
+   "label": "Data",
+   "read_only": 1
   }
- ], 
- "hide_heading": 0, 
- "hide_toolbar": 0, 
- "idx": 0, 
- "image_view": 0, 
- "in_create": 1, 
-
- "is_submittable": 0, 
- "issingle": 0, 
- "istable": 0, 
- "max_attachments": 0, 
- "modified": "2016-12-29 14:39:45.724494", 
- "modified_by": "Administrator", 
- "module": "Core", 
- "name": "Deleted Document", 
- "name_case": "", 
- "owner": "Administrator", 
+ ],
+ "in_create": 1,
+ "links": [],
+ "modified": "2022-06-13 05:50:58.314908",
+ "modified_by": "Administrator",
+ "module": "Core",
+ "name": "Deleted Document",
+ "owner": "Administrator",
  "permissions": [
   {
-   "amend": 0, 
-   "apply_user_permissions": 0, 
-   "cancel": 0, 
-   "create": 0, 
-   "delete": 1, 
-   "email": 0, 
-   "export": 1, 
-   "if_owner": 0, 
-   "import": 0, 
-   "is_custom": 0, 
-   "permlevel": 0, 
-   "print": 0, 
-   "read": 1, 
-   "report": 0, 
-   "role": "System Manager", 
-   "set_user_permissions": 0, 
-   "share": 0, 
-   "submit": 0, 
-   "write": 0
+   "delete": 1,
+   "export": 1,
+   "read": 1,
+   "role": "System Manager"
   }
- ], 
- "quick_entry": 0, 
- "read_only": 0, 
- "read_only_onload": 0, 
- "sort_field": "modified", 
- "sort_order": "DESC", 
- "title_field": "deleted_name", 
- "track_changes": 1, 
- "track_seen": 0
+ ],
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "states": [],
+ "title_field": "deleted_name",
+ "track_changes": 1
 }

frappe/core/doctype/doctype/doctype.py

@@ -8,6 +8,7 @@ import os
 # imports - standard imports
 import re
 import shutil
+from typing import TYPE_CHECKING, Union
 
 # imports - module imports
 import frappe
@@ -35,6 +36,9 @@ from frappe.query_builder.functions import Concat
 from frappe.utils import cint
 from frappe.website.utils import clear_cache
 
+if TYPE_CHECKING:
+	from frappe.custom.doctype.customize_form.customize_form import CustomizeForm
+
 DEPENDS_ON_PATTERN = re.compile(r'[\w\.:_]+\s*={1}\s*[\w\.@\'"]+')
 ILLEGAL_FIELDNAME_PATTERN = re.compile("""['",./%@()<>{}]""")
 WHITESPACE_PADDING_PATTERN = re.compile(r"^[ \t\n\r]+|[ \t\n\r]+$", flags=re.ASCII)
@@ -167,7 +171,7 @@ class DocType(Document):
 
 			if docfield.fieldname in method_set:
 				conflict_type = "controller method"
-			if docfield.fieldname in property_set:
+			if docfield.fieldname in property_set and not docfield.is_virtual:
 				conflict_type = "class property"
 
 			if conflict_type:
@@ -814,7 +818,7 @@ class DocType(Document):
 		self.nsm_parent_field = parent_field_name
 
 	def validate_child_table(self):
-		if not self.get("istable") or self.is_new():
+		if not self.get("istable") or self.is_new() or self.get("is_virtual"):
 			# if the doctype is not a child table then return
 			# if the doctype is a new doctype and also a child table then
 			# don't move forward as it will be handled via schema
@@ -916,11 +920,11 @@ def validate_series(dt, autoname=None, name=None):
 			frappe.throw(_("Series {0} already used in {1}").format(prefix, used_in[0][0]))
 
 
-def validate_autoincrement_autoname(dt: DocType) -> bool:
+def validate_autoincrement_autoname(dt: Union[DocType, "CustomizeForm"]) -> bool:
 	"""Checks if can doctype can change to/from autoincrement autoname"""
 
-	def get_autoname_before_save(dt: DocType) -> str:
-		if dt.name == "Customize Form":
+	def get_autoname_before_save(dt: Union[DocType, "CustomizeForm"]) -> str:
+		if dt.doctype == "Customize Form":
 			property_value = frappe.db.get_value(
 				"Property Setter", {"doc_type": dt.doc_type, "property": "autoname"}, "value"
 			)
@@ -943,10 +947,10 @@ def validate_autoincrement_autoname(dt: DocType) -> bool:
 			or (not is_autoname_autoincrement and autoname_before_save == "autoincrement")
 		):
 
-			if frappe.get_meta(dt.name).issingle:
-				if dt.name == "Customize Form":
-					frappe.throw(_("Cannot change to/from autoincrement autoname in Customize Form"))
+			if dt.doctype == "Customize Form":
+				frappe.throw(_("Cannot change to/from autoincrement autoname in Customize Form"))
 
+			if frappe.get_meta(dt.name).issingle:
 				return False
 
 			if not frappe.get_all(dt.name, limit=1):

frappe/core/doctype/doctype/test_doctype.py

@@ -564,6 +564,46 @@ class TestDocType(unittest.TestCase):
 		self.assertEqual(doc.is_virtual, 1)
 		self.assertFalse(frappe.db.table_exists("Test Virtual Doctype"))
 
+	def test_create_virtual_doctype_as_child_table(self):
+		"""Test virtual DocType as Child Table below a normal DocType."""
+		frappe.delete_doc_if_exists("DocType", "Test Parent Virtual DocType", force=1)
+		frappe.delete_doc_if_exists("DocType", "Test Virtual DocType as Child Table", force=1)
+
+		virtual_doc = new_doctype("Test Virtual DocType as Child Table")
+		virtual_doc.is_virtual = 1
+		virtual_doc.istable = 1
+		virtual_doc.insert(ignore_permissions=True)
+
+		doc = frappe.get_doc("DocType", "Test Virtual DocType as Child Table")
+
+		self.assertEqual(doc.is_virtual, 1)
+		self.assertEqual(doc.istable, 1)
+		self.assertFalse(frappe.db.table_exists("Test Virtual DocType as Child Table"))
+
+		parent_doc = new_doctype("Test Parent Virtual DocType")
+		parent_doc.append(
+			"fields",
+			{
+				"fieldname": "virtual_child_table",
+				"fieldtype": "Table",
+				"options": "Test Virtual DocType as Child Table",
+			},
+		)
+		parent_doc.insert(ignore_permissions=True)
+
+		# create entry for parent doctype
+		parent_doc_entry = frappe.get_doc(
+			{"doctype": "Test Parent Virtual DocType", "some_fieldname": "Test"}
+		)
+		parent_doc_entry.insert(ignore_permissions=True)
+
+		# update the parent doc (should not abort because of any DB query to a virtual child table, as there is none)
+		parent_doc_entry.some_fieldname = "Test update"
+		parent_doc_entry.save(ignore_permissions=True)
+
+		# delete the parent doc (should not abort because of any DB query to a virtual child table, as there is none)
+		parent_doc_entry.delete()
+
 	def test_default_fieldname(self):
 		fields = [
 			{"label": "title", "fieldname": "title", "fieldtype": "Data", "default": "{some_fieldname}"}

frappe/core/doctype/error_log/error_log.js

@@ -1,8 +1,17 @@
-// Copyright (c) 2016, Frappe Technologies and contributors
+// Copyright (c) 2022, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Error Log', {
+frappe.ui.form.on("Error Log", {
 	refresh: function(frm) {
+		frm.disable_save();
 
-	}
+		if (frm.doc.reference_doctype && frm.doc.reference_name) {
+			frm.add_custom_button(__("Show Related Errors"), function() {
+				frappe.set_route("List", "Error Log", {
+					reference_doctype: frm.doc.reference_doctype,
+					reference_name: frm.doc.reference_name,
+				});
+			});
+		}
+	},
 });

frappe/core/doctype/error_log/error_log.json

@@ -6,10 +6,12 @@
  "engine": "MyISAM",
  "field_order": [
   "seen",
-  "method",
-  "error",
   "reference_doctype",
-  "reference_name"
+  "column_break_3",
+  "reference_name",
+  "section_break_5",
+  "method",
+  "error"
  ],
  "fields": [
   {
@@ -47,12 +49,21 @@
    "fieldtype": "Data",
    "label": "Reference Name",
    "read_only": 1
+  },
+  {
+   "fieldname": "column_break_3",
+   "fieldtype": "Column Break"
+  },
+  {
+   "fieldname": "section_break_5",
+   "fieldtype": "Section Break"
   }
  ],
  "icon": "fa fa-warning-sign",
  "idx": 1,
+ "in_create": 1,
  "links": [],
- "modified": "2022-05-19 05:32:16.026684",
+ "modified": "2022-06-13 06:34:05.158606",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "Error Log",
@@ -70,7 +81,6 @@
    "write": 1
   }
  ],
- "quick_entry": 1,
  "sort_field": "modified",
  "sort_order": "ASC",
  "states": [],

frappe/core/doctype/error_log/error_log.py

@@ -4,6 +4,8 @@
 
 import frappe
 from frappe.model.document import Document
+from frappe.query_builder import Interval
+from frappe.query_builder.functions import Now
 
 
 class ErrorLog(Document):
@@ -12,13 +14,10 @@ class ErrorLog(Document):
 			self.db_set("seen", 1, update_modified=0)
 			frappe.db.commit()
 
-
-def set_old_logs_as_seen():
-	# set logs as seen
-	frappe.db.sql(
-		"""UPDATE `tabError Log` SET `seen`=1
-		WHERE `seen`=0 AND `creation` < (NOW() - INTERVAL '7' DAY)"""
-	)
+	@staticmethod
+	def clear_old_logs(days=30):
+		table = frappe.qb.DocType("Error Log")
+		frappe.db.delete(table, filters=(table.modified < (Now() - Interval(days=days))))
 
 
 @frappe.whitelist()

frappe/core/doctype/error_log/error_log_list.js

@@ -1,7 +1,7 @@
-frappe.listview_settings['Error Log'] = {
+frappe.listview_settings["Error Log"] = {
 	add_fields: ["seen"],
 	get_indicator: function(doc) {
-		if(cint(doc.seen)) {
+		if (cint(doc.seen)) {
 			return [__("Seen"), "green", "seen,=,1"];
 		} else {
 			return [__("Not Seen"), "red", "seen,=,0"];
@@ -11,11 +11,15 @@ frappe.listview_settings['Error Log'] = {
 	onload: function(listview) {
 		listview.page.add_menu_item(__("Clear Error Logs"), function() {
 			frappe.call({
-				method:'frappe.core.doctype.error_log.error_log.clear_error_logs',
+				method: "frappe.core.doctype.error_log.error_log.clear_error_logs",
 				callback: function() {
 					listview.refresh();
-				}
+				},
 			});
 		});
-	}
+
+		frappe.require("logtypes.bundle.js", () => {
+			frappe.utils.logtypes.show_log_retention_message(cur_list.doctype);
+		})
+	},
 };

frappe/core/doctype/error_snapshot/error_snapshot.py

@@ -4,6 +4,8 @@
 
 import frappe
 from frappe.model.document import Document
+from frappe.query_builder import Interval
+from frappe.query_builder.functions import Now
 
 
 class ErrorSnapshot(Document):
@@ -32,3 +34,8 @@ class ErrorSnapshot(Document):
 			frappe.db.set_value("Error Snapshot", parent["name"], "relapses", parent["relapses"] + 1)
 			if parent["seen"]:
 				frappe.db.set_value("Error Snapshot", parent["name"], "seen", False)
+
+	@staticmethod
+	def clear_old_logs(days=30):
+		table = frappe.qb.DocType("Error Snapshot")
+		frappe.db.delete(table, filters=(table.modified < (Now() - Interval(days=days))))

frappe/core/doctype/error_snapshot/error_snapshot_list.js

@@ -10,5 +10,10 @@ frappe.listview_settings["Error Snapshot"] = {
 		} else {
 			return [__("First Level"), !doc.seen ? "red" : "green", "parent_error_snapshot,=,"];
 		}
-	}
+	},
+	onload: function(listview) {
+		frappe.require("logtypes.bundle.js", () => {
+			frappe.utils.logtypes.show_log_retention_message(cur_list.doctype);
+		})
+	},
 }

frappe/core/doctype/file/file.py

@@ -16,7 +16,7 @@ from requests.exceptions import HTTPError, SSLError
 import frappe
 from frappe import _
 from frappe.model.document import Document
-from frappe.utils import call_hook_method, cint, get_files_path, get_hook_method
+from frappe.utils import call_hook_method, cint, get_files_path, get_hook_method, get_url
 from frappe.utils.file_manager import is_safe_path
 from frappe.utils.image import optimize_image, strip_exif_data
 
@@ -61,7 +61,12 @@ class File(Document):
 		self.set_file_name()
 		self.validate_attachment_limit()
 
-		if not self.is_folder and not self.is_remote_file:
+		if self.is_folder:
+			return
+
+		if self.is_remote_file:
+			self.validate_remote_file()
+		else:
 			self.save_file(content=self.get_content())
 			self.flags.new_file = True
 			frappe.local.rollback_observers.append(self)
@@ -255,6 +260,12 @@ class File(Document):
 					title=_("Attachment Limit Reached"),
 				)
 
+	def validate_remote_file(self):
+		"""Validates if file uploaded using URL already exist"""
+		site_url = get_url()
+		if "/files/" in self.file_url and self.file_url.startswith(site_url):
+			self.file_url = self.file_url.split(site_url, 1)[1]
+
 	def set_folder_name(self):
 		"""Make parent folders if not exists based on reference doctype and name"""
 		if self.folder:
@@ -341,9 +352,9 @@ class File(Document):
 
 		size = width, height
 		if crop:
-			image = ImageOps.fit(image, size, Image.ANTIALIAS)
+			image = ImageOps.fit(image, size, Image.Resampling.LANCZOS)
 		else:
-			image.thumbnail(size, Image.ANTIALIAS)
+			image.thumbnail(size, Image.Resampling.LANCZOS)
 
 		thumbnail_url = f"{filename}_{suffix}.{extn}"
 		path = os.path.abspath(frappe.get_site_path("public", thumbnail_url.lstrip("/")))
@@ -445,6 +456,10 @@ class File(Document):
 
 		file_path = self.file_url or self.file_name
 
+		site_url = get_url()
+		if "/files/" in file_path and file_path.startswith(site_url):
+			file_path = file_path.split(site_url, 1)[1]
+
 		if "/" not in file_path:
 			if self.is_private:
 				file_path = f"/private/files/{file_path}"

frappe/core/doctype/log_settings/log_settings.js

@@ -1,8 +1,16 @@
 // Copyright (c) 2020, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Log Settings', {
-	// refresh: function(frm) {
-
-	// }
+frappe.ui.form.on("Log Settings", {
+	refresh: (frm) => {
+		frm.set_query("ref_doctype", "logs_to_clear", () => {
+			const added_doctypes = frm.doc.logs_to_clear.map((r) => r.ref_doctype);
+			return {
+				query: "frappe.core.doctype.log_settings.log_settings.get_log_doctypes",
+				filters: [
+					["name", "not in", added_doctypes],
+				],
+			};
+		});
+	},
 });

frappe/core/doctype/log_settings/log_settings.json

@@ -5,61 +5,20 @@
  "editable_grid": 1,
  "engine": "InnoDB",
  "field_order": [
-  "error_log_notification_section",
-  "users_to_notify",
-  "log_cleanup_section",
-  "clear_error_log_after",
-  "clear_activity_log_after",
-  "column_break_4",
-  "clear_email_queue_after"
+  "logs_to_clear"
  ],
  "fields": [
   {
-   "fieldname": "log_cleanup_section",
-   "fieldtype": "Section Break",
-   "label": "Log Cleanup"
-  },
-  {
-   "fieldname": "column_break_4",
-   "fieldtype": "Column Break"
-  },
-  {
-   "fieldname": "error_log_notification_section",
-   "fieldtype": "Section Break",
-   "label": "Error Log Notification"
-  },
-  {
-   "fieldname": "users_to_notify",
-   "fieldtype": "Table MultiSelect",
-   "label": "Users To Notify",
-   "options": "Log Setting User"
-  },
-  {
-   "default": "90",
-   "description": "In Days",
-   "fieldname": "clear_error_log_after",
-   "fieldtype": "Int",
-   "label": "Clear Error log After"
-  },
-  {
-   "default": "90",
-   "description": "In Days",
-   "fieldname": "clear_activity_log_after",
-   "fieldtype": "Int",
-   "label": "Clear Activity Log After"
-  },
-  {
-   "default": "30",
-   "description": "In Days",
-   "fieldname": "clear_email_queue_after",
-   "fieldtype": "Int",
-   "label": "Clear Email Queue After"
+   "fieldname": "logs_to_clear",
+   "fieldtype": "Table",
+   "label": "Logs to Clear",
+   "options": "Logs To Clear"
   }
  ],
  "index_web_pages_for_search": 1,
  "issingle": 1,
  "links": [],
- "modified": "2020-10-13 12:18:48.649038",
+ "modified": "2022-06-11 02:17:30.803721",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "Log Settings",
@@ -79,5 +38,6 @@
  "quick_entry": 1,
  "sort_field": "modified",
  "sort_order": "DESC",
+ "states": [],
  "track_changes": 1
-}
+}

frappe/core/doctype/log_settings/log_settings.py

@@ -2,49 +2,119 @@
 # Copyright (c) 2020, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
+from typing import Protocol, runtime_checkable
+
 import frappe
 from frappe import _
+from frappe.model.base_document import get_controller
 from frappe.model.document import Document
-from frappe.query_builder import DocType, Interval
-from frappe.query_builder.functions import Now
+from frappe.utils import cint
+from frappe.utils.caching import site_cache
+
+DEFAULT_LOGTYPES_RETENTION = {
+	"Error Log": 30,
+	"Activity Log": 90,
+	"Email Queue": 30,
+	"Error Snapshot": 30,
+	"Scheduled Job Log": 90,
+}
+
+
+@runtime_checkable
+class LogType(Protocol):
+	"""Interface requirement for doctypes that can be cleared using log settings."""
+
+	@staticmethod
+	def clear_old_logs(days: int) -> None:
+		...
+
+
+@site_cache
+def _supports_log_clearing(doctype: str) -> bool:
+	try:
+		controller = get_controller(doctype)
+		return issubclass(controller, LogType)
+	except Exception:
+		return False
 
 
 class LogSettings(Document):
-	def clear_logs(self, commit=False):
-		self.clear_email_queue()
-		if commit:
-			# Since since deleting many logs can take significant amount of time, commit is required to relase locks.
-			# Error log table doesn't require commit - myisam
-			# activity logs are deleted last so background job finishes and commits.
+	def validate(self):
+		self.validate_supported_doctypes()
+		self.validate_duplicates()
+		self.add_default_logtypes()
+
+	def validate_supported_doctypes(self):
+		for entry in self.logs_to_clear:
+			if _supports_log_clearing(entry.ref_doctype):
+				continue
+
+			msg = _("{} does not support automated log clearing.").format(frappe.bold(entry.ref_doctype))
+			if frappe.conf.developer_mode:
+				msg += "<br>" + _("Implement `clear_old_logs` method to enable auto error clearing.")
+			frappe.throw(msg, title=_("DocType not supported by Log Settings."))
+
+	def validate_duplicates(self):
+		seen = set()
+		for entry in self.logs_to_clear:
+			if entry.ref_doctype in seen:
+				frappe.throw(
+					_("{} appears more than once in configured log doctypes.").format(entry.ref_doctype)
+				)
+			seen.add(entry.ref_doctype)
+
+	def add_default_logtypes(self):
+		existing_logtypes = {d.ref_doctype for d in self.logs_to_clear}
+		added_logtypes = set()
+		for logtype, retention in DEFAULT_LOGTYPES_RETENTION.items():
+			if logtype not in existing_logtypes and _supports_log_clearing(logtype):
+				self.append("logs_to_clear", {"ref_doctype": logtype, "days": cint(retention)})
+				added_logtypes.add(logtype)
+
+		if added_logtypes:
+			frappe.msgprint(
+				_("Added default log doctypes: {}").format(",".join(added_logtypes)), alert=True
+			)
+
+	def clear_logs(self):
+		"""
+		Log settings can clear any log type that's registered to it and provides a method to delete old logs.
+
+		Check `LogDoctype` above for interface that doctypes need to implement.
+		"""
+
+		for entry in self.logs_to_clear:
+			controller: LogType = get_controller(entry.ref_doctype)
+			func = controller.clear_old_logs
+
+			# Only pass what the method can handle, this is considering any
+			# future addition that might happen to the required interface.
+			kwargs = frappe.get_newargs(func, {"days": entry.days})
+			func(**kwargs)
 			frappe.db.commit()
-		self.clear_error_logs()
-		self.clear_activity_logs()
 
-	def clear_error_logs(self):
-		table = DocType("Error Log")
-		frappe.db.delete(
-			table, filters=(table.creation < (Now() - Interval(days=self.clear_error_log_after)))
-		)
+	def register_doctype(self, doctype: str, days=30):
+		existing_logtypes = {d.ref_doctype for d in self.logs_to_clear}
 
-	def clear_activity_logs(self):
-		from frappe.core.doctype.activity_log.activity_log import clear_activity_logs
-
-		clear_activity_logs(days=self.clear_activity_log_after)
-
-	def clear_email_queue(self):
-		from frappe.email.queue import clear_outbox
-
-		clear_outbox(days=self.clear_email_queue_after)
+		if doctype not in existing_logtypes and _supports_log_clearing(doctype):
+			self.append("logs_to_clear", {"ref_doctype": doctype, "days": cint(days)})
+		else:
+			for entry in self.logs_to_clear:
+				if entry.ref_doctype == doctype:
+					entry.days = days
+					break
 
 
 def run_log_clean_up():
 	doc = frappe.get_doc("Log Settings")
-	doc.clear_logs(commit=True)
+	doc.add_default_logtypes()
+	doc.save()
+	doc.clear_logs()
 
 
 @frappe.whitelist()
-def has_unseen_error_log(user):
-	def _get_response(show_alert=True):
+def has_unseen_error_log():
+	if frappe.get_all("Error Log", filters={"seen": 0}, limit=1):
 		return {
 			"show_alert": True,
 			"message": _("You have unseen {0}").format(
@@ -52,13 +122,67 @@ def has_unseen_error_log(user):
 			),
 		}
 
-	if frappe.get_all("Error Log", filters={"seen": 0}, limit=1):
-		log_settings = frappe.get_cached_doc("Log Settings")
 
-		if log_settings.users_to_notify:
-			if user in [u.user for u in log_settings.users_to_notify]:
-				return _get_response()
-			else:
-				return _get_response(show_alert=False)
-		else:
-			return _get_response()
+@frappe.whitelist()
+@frappe.validate_and_sanitize_search_inputs
+def get_log_doctypes(doctype, txt, searchfield, start, page_len, filters):
+
+	filters = filters or {}
+
+	filters.extend(
+		[
+			["istable", "=", 0],
+			["issingle", "=", 0],
+			["name", "like", f"%%{txt}%%"],
+		]
+	)
+	doctypes = frappe.get_list("DocType", filters=filters, pluck="name")
+
+	supported_doctypes = [(d,) for d in doctypes if _supports_log_clearing(d)]
+
+	return supported_doctypes[start:page_len]
+
+
+LOG_DOCTYPES = [
+	"Scheduled Job Log",
+	"Activity Log",
+	"Route History",
+	"Email Queue",
+	"Email Queue Recipient",
+	"Error Snapshot",
+	"Error Log",
+]
+
+
+def clear_log_table(doctype, days=90):
+	"""If any logtype table grows too large then clearing it with DELETE query
+	is not feasible in reasonable time. This command copies recent data to new
+	table and replaces current table with new smaller table.
+
+	ref: https://mariadb.com/kb/en/big-deletes/#deleting-more-than-half-a-table
+	"""
+	from frappe.utils import get_table_name
+
+	if doctype not in LOG_DOCTYPES:
+		raise frappe.ValidationError(f"Unsupported logging DocType: {doctype}")
+
+	original = get_table_name(doctype)
+	temporary = f"{original} temp_table"
+	backup = f"{original} backup_table"
+
+	try:
+		frappe.db.sql_ddl(f"CREATE TABLE `{temporary}` LIKE `{original}`")
+
+		# Copy all recent data to new table
+		frappe.db.sql(
+			f"""INSERT INTO `{temporary}`
+				SELECT * FROM `{original}`
+				WHERE `{original}`.`modified` > NOW() - INTERVAL '{days}' DAY"""
+		)
+		frappe.db.sql_ddl(f"RENAME TABLE `{original}` TO `{backup}`, `{temporary}` TO `{original}`")
+	except Exception:
+		frappe.db.rollback()
+		frappe.db.sql_ddl(f"DROP TABLE IF EXISTS `{temporary}`")
+		raise
+	else:
+		frappe.db.sql_ddl(f"DROP TABLE `{backup}`")

frappe/core/doctype/log_settings/test_log_settings.py

@@ -4,7 +4,7 @@
 from datetime import datetime
 
 import frappe
-from frappe.core.doctype.log_settings.log_settings import run_log_clean_up
+from frappe.core.doctype.log_settings.log_settings import _supports_log_clearing, run_log_clean_up
 from frappe.tests.utils import FrappeTestCase
 from frappe.utils import add_to_date, now_datetime
 
@@ -56,6 +56,23 @@ class TestLogSettings(FrappeTestCase):
 		self.assertEqual(error_log_count, 0)
 		self.assertEqual(email_queue_count, 0)
 
+	def test_logtype_identification(self):
+		supported_types = [
+			"Error Log",
+			"Activity Log",
+			"Email Queue",
+			"Route History",
+			"Error Snapshot",
+			"Scheduled Job Log",
+		]
+
+		for lt in supported_types:
+			self.assertTrue(_supports_log_clearing(lt), f"{lt} should be recognized as log type")
+
+		unsupported_types = ["DocType", "User", "Non Existing dt"]
+		for dt in unsupported_types:
+			self.assertFalse(_supports_log_clearing(dt), f"{dt} shouldn't be recognized as log type")
+
 
 def setup_test_logs(past: datetime) -> None:
 	activity_log = frappe.get_doc(

frappe/core/doctype/logs_to_clear/logs_to_clear.json

@@ -0,0 +1,43 @@
+{
+ "actions": [],
+ "autoname": "hash",
+ "creation": "2022-06-11 02:02:39.472511",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "ref_doctype",
+  "days"
+ ],
+ "fields": [
+  {
+   "fieldname": "ref_doctype",
+   "fieldtype": "Link",
+   "in_list_view": 1,
+   "label": "Log DocType",
+   "options": "DocType",
+   "reqd": 1
+  },
+  {
+   "default": "30",
+   "fieldname": "days",
+   "fieldtype": "Int",
+   "in_list_view": 1,
+   "label": "Clear Logs After (days)",
+   "non_negative": 1,
+   "reqd": 1
+  }
+ ],
+ "index_web_pages_for_search": 1,
+ "istable": 1,
+ "links": [],
+ "modified": "2022-06-13 02:51:36.857786",
+ "modified_by": "Administrator",
+ "module": "Core",
+ "name": "Logs To Clear",
+ "owner": "Administrator",
+ "permissions": [],
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "states": []
+}

frappe/core/doctype/logs_to_clear/logs_to_clear.py

@@ -0,0 +1,9 @@
+# Copyright (c) 2022, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+# import frappe
+from frappe.model.document import Document
+
+
+class LogsToClear(Document):
+	pass

frappe/core/doctype/patch_log/patch_log.js

@@ -3,6 +3,6 @@
 
 frappe.ui.form.on('Patch Log', {
 	refresh: function(frm) {
-
+		frm.disable_save();
 	}
 });

frappe/core/doctype/patch_log/patch_log.json

@@ -1,87 +1,44 @@
 {
- "allow_copy": 0, 
- "allow_import": 0, 
- "allow_rename": 0, 
- "autoname": "PATCHLOG.#####", 
- "beta": 0, 
- "creation": "2013-01-17 11:36:45", 
- "custom": 0, 
- "description": "List of patches executed", 
- "docstatus": 0, 
- "doctype": "DocType", 
- "document_type": "System", 
- "editable_grid": 0, 
+ "actions": [],
+ "autoname": "PATCHLOG.#####",
+ "creation": "2013-01-17 11:36:45",
+ "description": "List of patches executed",
+ "doctype": "DocType",
+ "document_type": "System",
+ "engine": "InnoDB",
+ "field_order": [
+  "patch"
+ ],
  "fields": [
   {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "patch", 
-   "fieldtype": "Code", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Patch", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
+   "fieldname": "patch",
+   "fieldtype": "Code",
+   "label": "Patch",
+   "read_only": 1
   }
- ], 
- "hide_heading": 0, 
- "hide_toolbar": 0, 
- "icon": "fa fa-cog", 
- "idx": 1, 
- "image_view": 0, 
- "in_create": 0, 
-
- "is_submittable": 0, 
- "issingle": 0, 
- "istable": 0, 
- "max_attachments": 0, 
- "modified": "2016-12-29 14:40:35.048570", 
- "modified_by": "Administrator", 
- "module": "Core", 
- "name": "Patch Log", 
- "owner": "Administrator", 
+ ],
+ "icon": "fa fa-cog",
+ "idx": 1,
+ "links": [],
+ "modified": "2022-06-13 05:34:37.845368",
+ "modified_by": "Administrator",
+ "module": "Core",
+ "name": "Patch Log",
+ "naming_rule": "Expression (old style)",
+ "owner": "Administrator",
  "permissions": [
   {
-   "amend": 0, 
-   "apply_user_permissions": 0, 
-   "cancel": 0, 
-   "create": 0, 
-   "delete": 0, 
-   "email": 1, 
-   "export": 0, 
-   "if_owner": 0, 
-   "import": 0, 
-   "is_custom": 0, 
-   "permlevel": 0, 
-   "print": 1, 
-   "read": 1, 
-   "report": 1, 
-   "role": "Administrator", 
-   "set_user_permissions": 0, 
-   "share": 0, 
-   "submit": 0, 
-   "write": 0
+   "email": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "Administrator"
   }
- ], 
- "quick_entry": 1, 
- "read_only": 0, 
- "read_only_onload": 0, 
- "track_changes": 1, 
- "track_seen": 0
+ ],
+ "quick_entry": 1,
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "states": [],
+ "title_field": "patch",
+ "track_changes": 1
 }

frappe/core/doctype/prepared_report/prepared_report.json

@@ -23,15 +23,14 @@
   {
    "fieldname": "report_name",
    "fieldtype": "Data",
-   "hidden": 1,
    "label": "Report Name",
    "read_only": 1
   },
   {
    "fieldname": "ref_report_doctype",
    "fieldtype": "Link",
-   "hidden": 1,
-   "label": "Ref Report DocType",
+   "in_standard_filter": 1,
+   "label": "Report Type",
    "options": "Report",
    "read_only": 1
   },
@@ -41,6 +40,7 @@
    "fieldtype": "Select",
    "hidden": 1,
    "in_list_view": 1,
+   "in_standard_filter": 1,
    "label": "Status",
    "options": "Error\nQueued\nCompleted",
    "read_only": 1
@@ -103,10 +103,11 @@
  ],
  "in_create": 1,
  "links": [],
- "modified": "2020-03-05 10:52:56.598365",
+ "modified": "2022-06-13 06:20:34.496412",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "Prepared Report",
+ "naming_rule": "Expression (old style)",
  "owner": "Administrator",
  "permissions": [
   {
@@ -131,9 +132,9 @@
    "share": 1
   }
  ],
- "quick_entry": 1,
  "sort_field": "modified",
  "sort_order": "DESC",
- "title_field": "report_name",
+ "states": [],
+ "title_field": "ref_report_doctype",
  "track_changes": 1
 }

frappe/core/doctype/scheduled_job_log/scheduled_job_log.json

@@ -38,7 +38,7 @@
   }
  ],
  "links": [],
- "modified": "2021-10-25 00:00:00.000000",
+ "modified": "2022-06-13 05:41:21.090972",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "Scheduled Job Log",
@@ -59,5 +59,7 @@
  ],
  "quick_entry": 1,
  "sort_field": "modified",
- "sort_order": "DESC"
-}
+ "sort_order": "DESC",
+ "states": [],
+ "title_field": "scheduled_job_type"
+}

frappe/core/doctype/scheduled_job_log/scheduled_job_log.py

@@ -2,9 +2,14 @@
 # Copyright (c) 2019, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
-# import frappe
+import frappe
 from frappe.model.document import Document
+from frappe.query_builder import Interval
+from frappe.query_builder.functions import Now
 
 
 class ScheduledJobLog(Document):
-	pass
+	@staticmethod
+	def clear_old_logs(days=90):
+		table = frappe.qb.DocType("Scheduled Job Log")
+		frappe.db.delete(table, filters=(table.modified < (Now() - Interval(days=days))))

frappe/core/doctype/scheduled_job_log/scheduled_job_log_list.js

@@ -0,0 +1,7 @@
+frappe.listview_settings["Scheduled Job Log"] = {
+	onload: function(listview) {
+		frappe.require("logtypes.bundle.js", () => {
+			frappe.utils.logtypes.show_log_retention_message(cur_list.doctype);
+		})
+	},
+};

frappe/core/doctype/scheduled_job_type/scheduled_job_type.json

@@ -16,8 +16,11 @@
   "server_script",
   "frequency",
   "cron_format",
+  "create_log",
+  "status_section",
   "last_execution",
-  "create_log"
+  "column_break_9",
+  "next_execution"
  ],
  "fields": [
   {
@@ -72,6 +75,22 @@
    "options": "Server Script",
    "read_only": 1,
    "search_index": 1
+  },
+  {
+   "fieldname": "next_execution",
+   "fieldtype": "Datetime",
+   "is_virtual": 1,
+   "label": "Next Execution",
+   "read_only": 1
+  },
+  {
+   "fieldname": "status_section",
+   "fieldtype": "Section Break",
+   "label": "Status"
+  },
+  {
+   "fieldname": "column_break_9",
+   "fieldtype": "Column Break"
   }
  ],
  "in_create": 1,
@@ -81,7 +100,7 @@
    "link_fieldname": "scheduled_job_type"
   }
  ],
- "modified": "2020-10-07 10:39:24.519460",
+ "modified": "2022-06-28 02:55:12.470915",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "Scheduled Job Type",
@@ -103,5 +122,7 @@
  "quick_entry": 1,
  "sort_field": "modified",
  "sort_order": "DESC",
+ "states": [],
+ "title_field": "method",
  "track_changes": 1
 }

frappe/core/doctype/scheduled_job_type/scheduled_job_type.py

@@ -50,6 +50,10 @@ class ScheduledJobType(Document):
 		queued_jobs = get_jobs(site=frappe.local.site, key="job_type")[frappe.local.site]
 		return self.method in queued_jobs
 
+	@property
+	def next_execution(self):
+		return self.get_next_execution()
+
 	def get_next_execution(self):
 		CRON_MAP = {
 			"Yearly": "0 0 1 1 *",

frappe/core/doctype/server_script/server_script.json

@@ -25,6 +25,7 @@
    "fieldname": "script_type",
    "fieldtype": "Select",
    "in_list_view": 1,
+   "in_standard_filter": 1,
    "label": "Script Type",
    "options": "DocType Event\nScheduler Event\nPermission Query\nAPI",
    "reqd": 1
@@ -41,6 +42,7 @@
    "fieldname": "reference_doctype",
    "fieldtype": "Link",
    "in_list_view": 1,
+   "in_standard_filter": 1,
    "label": "Reference Document Type",
    "options": "DocType"
   },
@@ -109,7 +111,7 @@
    "link_fieldname": "server_script"
   }
  ],
- "modified": "2022-04-27 11:42:52.032963",
+ "modified": "2022-06-13 06:04:20.937969",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "Server Script",

frappe/core/doctype/system_settings/system_settings.json

@@ -63,6 +63,7 @@
   "otp_issuer_name",
   "email",
   "email_footer_address",
+  "email_retry_limit",
   "column_break_18",
   "disable_standard_email_footer",
   "hide_footer_in_auto_email_reports",
@@ -495,8 +496,8 @@
    "fieldname": "allow_older_web_view_links",
    "fieldtype": "Check",
    "label": "Allow Older Web View Links (Insecure)"
-   },
-   {
+  },
+  {
    "fieldname": "column_break_64",
    "fieldtype": "Column Break"
   },
@@ -518,12 +519,18 @@
    "fieldtype": "Duration",
    "label": "Reset Password Link Expiry Duration",
    "non_negative": 1
+  },
+  {
+   "default": "3",
+   "fieldname": "email_retry_limit",
+   "fieldtype": "Int",
+   "label": "Email Retry Limit"
   }
  ],
  "icon": "fa fa-cog",
  "issingle": 1,
  "links": [],
- "modified": "2022-05-19 00:00:18.095269",
+ "modified": "2022-06-21 13:55:04.796152",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "System Settings",

frappe/core/doctype/user/user.json

@@ -722,7 +722,7 @@
    "link_fieldname": "user"
   }
  ],
- "modified": "2022-03-09 01:47:56.745069",
+ "modified": "2022-05-25 01:00:51.345319",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "User",
@@ -747,6 +747,10 @@
    "read": 1,
    "role": "System Manager",
    "write": 1
+  },
+  {
+   "role": "All",
+   "select": 1
   }
  ],
  "quick_entry": 1,

frappe/core/doctype/user/user.py

@@ -163,6 +163,9 @@ class User(Document):
 		toggle_notifications(self.name, enable=cint(self.enabled))
 
 	def add_system_manager_role(self):
+		if self.is_system_manager_disabled():
+			return
+
 		# if adding system manager, do nothing
 		if not cint(self.enabled) or (
 			"System Manager" in [user_role.role for user_role in self.get("roles")]
@@ -189,6 +192,9 @@ class User(Document):
 				],
 			)
 
+	def is_system_manager_disabled(self):
+		return frappe.db.get_value("Role", {"name": "System Manager"}, ["disabled"])
+
 	def email_new_password(self, new_password=None):
 		if new_password and not self.flags.in_insert:
 			_update_password(user=self.name, pwd=new_password, logout_all_sessions=self.logout_all_sessions)
@@ -372,6 +378,9 @@ class User(Document):
 		)
 
 	def a_system_manager_should_exist(self):
+		if self.is_system_manager_disabled():
+			return
+
 		if not self.get_other_system_managers():
 			throw(_("There should remain at least one System Manager"))
 

frappe/core/doctype/user_type/user_type.js

@@ -3,13 +3,8 @@
 
 frappe.ui.form.on('User Type', {
 	refresh: function(frm) {
-		frm.toggle_display('is_standard', frappe.boot.developer_mode);
-		frm.set_df_property('is_standard', 'read_only', !frappe.boot.developer_mode);
-
-		const fields = ['role', 'apply_user_permission_on', 'user_id_field',
-			'user_doctypes', 'user_type_modules'];
-
-		frm.toggle_display(fields, !frm.doc.is_standard);
+		if (frm.is_new() && !frappe.boot.developer_mode)
+			frm.set_value('is_standard', 1);
 
 		frm.set_query('document_type', 'user_doctypes', function() {
 			return {

frappe/core/doctype/user_type/user_type.json

@@ -22,9 +22,11 @@
  "fields": [
   {
    "default": "0",
+   "depends_on": "eval: frappe.boot.developer_mode",
    "fieldname": "is_standard",
    "fieldtype": "Check",
-   "label": "Is Standard"
+   "label": "Is Standard",
+   "read_only_depends_on": "eval: !frappe.boot.developer_mode"
   },
   {
    "depends_on": "eval: !doc.is_standard",
@@ -33,21 +35,21 @@
    "label": "Document Types and Permissions"
   },
   {
+   "depends_on": "eval: !doc.is_standard",
    "fieldname": "user_doctypes",
    "fieldtype": "Table",
    "label": "Document Types",
    "mandatory_depends_on": "eval: !doc.is_standard",
-   "options": "User Document Type",
-   "read_only": 1
+   "options": "User Document Type"
   },
   {
+   "depends_on": "eval: !doc.is_standard",
    "fieldname": "role",
    "fieldtype": "Link",
    "in_list_view": 1,
    "label": "Role",
    "mandatory_depends_on": "eval: !doc.is_standard",
-   "options": "Role",
-   "read_only": 1
+   "options": "Role"
   },
   {
    "fieldname": "select_doctypes",
@@ -62,13 +64,13 @@
    "fieldtype": "Column Break"
   },
   {
+   "depends_on": "eval: !doc.is_standard",
    "description": "Can only list down the document types which has been linked to the User document type.",
    "fieldname": "apply_user_permission_on",
    "fieldtype": "Link",
    "label": "Apply User Permission On",
    "mandatory_depends_on": "eval: !doc.is_standard",
-   "options": "DocType",
-   "read_only": 1
+   "options": "DocType"
   },
   {
    "depends_on": "eval: !doc.is_standard",
@@ -81,8 +83,7 @@
    "fieldname": "user_id_field",
    "fieldtype": "Select",
    "label": "User Id Field",
-   "mandatory_depends_on": "eval: !doc.is_standard",
-   "read_only": 1
+   "mandatory_depends_on": "eval: !doc.is_standard"
   },
   {
    "depends_on": "eval: !doc.is_standard",
@@ -93,6 +94,7 @@
   {
    "fieldname": "user_type_modules",
    "fieldtype": "Table",
+   "label": "User Type Module",
    "no_copy": 1,
    "options": "User Type Module",
    "print_hide": 1,
@@ -107,10 +109,11 @@
  ],
  "index_web_pages_for_search": 1,
  "links": [],
- "modified": "2021-03-12 16:25:18.639050",
+ "modified": "2022-06-09 14:00:36.820306",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "User Type",
+ "naming_rule": "Set by user",
  "owner": "Administrator",
  "permissions": [
   {
@@ -137,5 +140,6 @@
  ],
  "sort_field": "modified",
  "sort_order": "DESC",
+ "states": [],
  "track_changes": 1
 }

frappe/core/doctype/version/test_version.py

@@ -32,6 +32,19 @@ class TestVersion(unittest.TestCase):
 		self.assertEqual(get_old_values(diff)[1], "01-01-2014 00:00:00")
 		self.assertEqual(get_new_values(diff)[1], "07-20-2017 00:00:00")
 
+	def test_no_version_on_new_doc(self):
+		from frappe.desk.form.load import get_versions
+
+		t = frappe.get_doc(doctype="ToDo", description="something")
+		t.save(ignore_version=False)
+
+		self.assertFalse(get_versions(t))
+
+		t = frappe.get_doc(t.doctype, t.name)
+		t.description = "changed"
+		t.save(ignore_version=False)
+		self.assertTrue(get_versions(t))
+
 
 def get_fieldnames(change_array):
 	return [d[0] for d in change_array]

frappe/core/doctype/version/version.py

@@ -2,6 +2,7 @@
 # License: MIT. See LICENSE
 
 import json
+from typing import Optional
 
 import frappe
 from frappe.model import no_value_fields, table_fields
@@ -9,7 +10,15 @@ from frappe.model.document import Document
 
 
 class Version(Document):
-	def set_diff(self, old, new):
+	def update_version_info(self, old: Optional[Document], new: Document) -> bool:
+		"""Update changed info and return true if change contains useful data."""
+		if not old:
+			# Check if doc has some information about creation source like data import
+			return self.for_insert(new)
+		else:
+			return self.set_diff(old, new)
+
+	def set_diff(self, old: Document, new: Document) -> bool:
 		"""Set the data property with the diff of the docs if present"""
 		diff = get_diff(old, new)
 		if diff:
@@ -20,8 +29,11 @@ class Version(Document):
 		else:
 			return False
 
-	def for_insert(self, doc):
+	def for_insert(self, doc: Document) -> bool:
 		updater_reference = doc.flags.updater_reference
+		if not updater_reference:
+			return False
+
 		data = {
 			"creation": doc.creation,
 			"updater_reference": updater_reference,
@@ -29,7 +41,8 @@ class Version(Document):
 		}
 		self.ref_doctype = doc.doctype
 		self.docname = doc.name
-		self.data = frappe.as_json(data)
+		self.data = frappe.as_json(data, indent=None, separators=(",", ":"))
+		return True
 
 	def get_data(self):
 		return json.loads(self.data)

frappe/core/page/permission_manager/permission_manager.js

@@ -284,7 +284,7 @@ frappe.PermissionEngine = class PermissionEngine {
 	}
 
 	setup_if_owner(d, role_cell) {
-		this.add_check(role_cell, d, "if_owner", "Only If Creator")
+		this.add_check(role_cell, d, "if_owner", "Only if Creator")
 			.removeClass("col-md-4")
 			.css({ "margin-top": "15px" });
 	}

frappe/core/page/permission_manager/permission_manager.py

@@ -1,6 +1,8 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
 
+from typing import Optional
+
 import frappe
 import frappe.defaults
 from frappe import _
@@ -8,6 +10,7 @@ from frappe.core.doctype.doctype.doctype import (
 	clear_permissions_cache,
 	validate_permissions_for_doctype,
 )
+from frappe.exceptions import DoesNotExistError
 from frappe.modules.import_file import get_file_path, read_doc_from_file
 from frappe.permissions import (
 	add_permission,
@@ -68,17 +71,19 @@ def get_roles_and_doctypes():
 
 
 @frappe.whitelist()
-def get_permissions(doctype=None, role=None):
+def get_permissions(doctype: Optional[str] = None, role: Optional[str] = None):
 	frappe.only_for("System Manager")
+
 	if role:
 		out = get_all_perms(role)
 		if doctype:
 			out = [p for p in out if p.parent == doctype]
+
 	else:
-		filters = dict(parent=doctype)
+		filters = {"parent": doctype}
 		if frappe.session.user != "Administrator":
-			custom_roles = frappe.get_all("Role", filters={"is_custom": 1})
-			filters["role"] = ["not in", [row.name for row in custom_roles]]
+			custom_roles = frappe.get_all("Role", filters={"is_custom": 1}, pluck="name")
+			filters["role"] = ["not in", custom_roles]
 
 		out = frappe.get_all("Custom DocPerm", fields="*", filters=filters, order_by="permlevel")
 		if not out:
@@ -86,11 +91,15 @@ def get_permissions(doctype=None, role=None):
 
 	linked_doctypes = {}
 	for d in out:
-		if not d.parent in linked_doctypes:
-			linked_doctypes[d.parent] = get_linked_doctypes(d.parent)
+		if d.parent not in linked_doctypes:
+			try:
+				linked_doctypes[d.parent] = get_linked_doctypes(d.parent)
+			except DoesNotExistError:
+				# exclude & continue if linked doctype is not found
+				frappe.clear_last_message()
+				continue
 		d.linked_doctypes = linked_doctypes[d.parent]
-		meta = frappe.get_meta(d.parent)
-		if meta:
+		if meta := frappe.get_meta(d.parent):
 			d.is_submittable = meta.is_submittable
 			d.in_create = meta.in_create
 

frappe/custom/doctype/custom_field/custom_field.json

@@ -67,7 +67,8 @@
    "fieldtype": "Link",
    "in_filter": 1,
    "in_list_view": 1,
-   "label": "Document",
+   "in_standard_filter": 1,
+   "label": "DocType",
    "oldfieldname": "dt",
    "oldfieldtype": "Link",
    "options": "DocType",
@@ -94,6 +95,7 @@
    "fieldname": "fieldname",
    "fieldtype": "Data",
    "in_list_view": 1,
+   "in_standard_filter": 1,
    "label": "Fieldname",
    "no_copy": 1,
    "oldfieldname": "fieldname",
@@ -439,7 +441,7 @@
  "idx": 1,
  "index_web_pages_for_search": 1,
  "links": [],
- "modified": "2022-04-14 09:46:58.849765",
+ "modified": "2022-06-13 06:39:03.319667",
  "modified_by": "Administrator",
  "module": "Custom",
  "name": "Custom Field",

frappe/data_migration/doctype/data_migration_connector/connectors/base.py

@@ -1,24 +0,0 @@
-from abc import ABCMeta, abstractmethod
-
-from frappe.utils.password import get_decrypted_password
-
-
-class BaseConnection(metaclass=ABCMeta):
-	@abstractmethod
-	def get(self, remote_objectname, fields=None, filters=None, start=0, page_length=10):
-		pass
-
-	@abstractmethod
-	def insert(self, doctype, doc):
-		pass
-
-	@abstractmethod
-	def update(self, doctype, doc, migration_id):
-		pass
-
-	@abstractmethod
-	def delete(self, doctype, migration_id):
-		pass
-
-	def get_password(self):
-		return get_decrypted_password("Data Migration Connector", self.connector.name)

frappe/data_migration/doctype/data_migration_connector/connectors/frappe_connection.py

@@ -1,32 +0,0 @@
-import frappe
-from frappe.frappeclient import FrappeClient
-
-from .base import BaseConnection
-
-
-class FrappeConnection(BaseConnection):
-	def __init__(self, connector):
-		self.connector = connector
-		self.connection = FrappeClient(
-			self.connector.hostname, self.connector.username, self.get_password()
-		)
-		self.name_field = "name"
-
-	def insert(self, doctype, doc):
-		doc = frappe._dict(doc)
-		doc.doctype = doctype
-		return self.connection.insert(doc)
-
-	def update(self, doctype, doc, migration_id):
-		doc = frappe._dict(doc)
-		doc.doctype = doctype
-		doc.name = migration_id
-		return self.connection.update(doc)
-
-	def delete(self, doctype, migration_id):
-		return self.connection.delete(doctype, migration_id)
-
-	def get(self, doctype, fields='"*"', filters=None, start=0, page_length=20):
-		return self.connection.get_list(
-			doctype, fields=fields, filters=filters, limit_start=start, limit_page_length=page_length
-		)

frappe/data_migration/doctype/data_migration_connector/data_migration_connector.js

@@ -1,47 +0,0 @@
-// Copyright (c) 2017, Frappe Technologies and contributors
-// For license information, please see license.txt
-
-frappe.ui.form.on('Data Migration Connector', {
-	onload(frm) {
-		if(frappe.boot.developer_mode) {
-			frm.add_custom_button(__('New Connection'), () => frm.events.new_connection(frm));
-		}
-	},
-	new_connection(frm) {
-		const d = new frappe.ui.Dialog({
-			title: __('New Connection'),
-			fields: [
-				{ label: __('Module'), fieldtype: 'Link', options: 'Module Def', reqd: 1 },
-				{ label: __('Connection Name'), fieldtype: 'Data', description: 'For e.g: Shopify Connection', reqd: 1 },
-			],
-			primary_action_label: __('Create'),
-			primary_action: (values) => {
-				let { module, connection_name } = values;
-
-				frm.events.create_new_connection(module, connection_name)
-					.then(r => {
-						if (r.message) {
-							const connector_name = connection_name
-								.replace('connection', 'Connector')
-								.replace('Connection', 'Connector')
-								.trim();
-
-							frm.set_value('connector_name', connector_name);
-							frm.set_value('connector_type', 'Custom');
-							frm.set_value('python_module', r.message);
-							frm.save();
-							frappe.show_alert(__("New module created {0}", [r.message]));
-							d.hide();
-						}
-					});
-			}
-		});
-
-		d.show();
-	},
-	create_new_connection(module, connection_name) {
-		return frappe.call('frappe.data_migration.doctype.data_migration_connector.data_migration_connector.create_new_connection', {
-			module, connection_name
-		});
-	}
-});

frappe/data_migration/doctype/data_migration_connector/data_migration_connector.json

@@ -1,307 +0,0 @@
-{
- "allow_copy": 0, 
- "allow_guest_to_view": 0, 
- "allow_import": 0, 
- "allow_rename": 1, 
- "autoname": "field:connector_name", 
- "beta": 1, 
- "creation": "2017-08-11 05:03:27.091416", 
- "custom": 0, 
- "docstatus": 0, 
- "doctype": "DocType", 
- "document_type": "", 
- "editable_grid": 1, 
- "engine": "InnoDB", 
- "fields": [
-  {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "connector_name", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Connector Name", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "depends_on": "eval:!doc.is_custom", 
-   "fieldname": "connector_type", 
-   "fieldtype": "Select", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Connector Type", 
-   "length": 0, 
-   "no_copy": 0, 
-   "options": "\nFrappe\nCustom", 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "depends_on": "eval:doc.connector_type == 'Custom'", 
-   "fieldname": "python_module", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Python Module", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "authentication_credentials", 
-   "fieldtype": "Section Break", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Authentication Credentials", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "default": "", 
-   "fieldname": "hostname", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Hostname", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "database_name", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Database Name", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "username", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Username", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "password", 
-   "fieldtype": "Password", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Password", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }
- ], 
- "has_web_view": 0, 
- "hide_heading": 0, 
- "hide_toolbar": 0, 
- "idx": 0, 
- "image_view": 0, 
- "in_create": 0, 
- "is_submittable": 0, 
- "issingle": 0, 
- "istable": 0, 
- "max_attachments": 0, 
- "modified": "2017-12-01 13:38:55.992499", 
- "modified_by": "Administrator", 
- "module": "Data Migration", 
- "name": "Data Migration Connector", 
- "name_case": "", 
- "owner": "Administrator", 
- "permissions": [
-  {
-   "amend": 0, 
-   "apply_user_permissions": 0, 
-   "cancel": 0, 
-   "create": 1, 
-   "delete": 1, 
-   "email": 1, 
-   "export": 1, 
-   "if_owner": 0, 
-   "import": 0, 
-   "permlevel": 0, 
-   "print": 1, 
-   "read": 1, 
-   "report": 1, 
-   "role": "System Manager", 
-   "set_user_permissions": 0, 
-   "share": 1, 
-   "submit": 0, 
-   "write": 1
-  }
- ], 
- "quick_entry": 0, 
- "read_only": 0, 
- "read_only_onload": 0, 
- "show_name_in_global_search": 0, 
- "sort_field": "modified", 
- "sort_order": "DESC", 
- "track_changes": 1, 
- "track_seen": 0
-}

frappe/data_migration/doctype/data_migration_connector/data_migration_connector.py

@@ -1,107 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright (c) 2017, Frappe Technologies and contributors
-# License: MIT. See LICENSE
-
-import os
-
-import frappe
-from frappe import _
-from frappe.model.document import Document
-from frappe.modules.export_file import create_init_py
-
-from .connectors.base import BaseConnection
-from .connectors.frappe_connection import FrappeConnection
-
-
-class DataMigrationConnector(Document):
-	def validate(self):
-		if not (self.python_module or self.connector_type):
-			frappe.throw(_("Enter python module or select connector type"))
-
-		if self.python_module:
-			try:
-				get_connection_class(self.python_module)
-			except:
-				frappe.throw(frappe._("Invalid module path"))
-
-	def get_connection(self):
-		if self.python_module:
-			_class = get_connection_class(self.python_module)
-			return _class(self)
-		else:
-			self.connection = FrappeConnection(self)
-
-		return self.connection
-
-
-@frappe.whitelist()
-def create_new_connection(module, connection_name):
-	if not frappe.conf.get("developer_mode"):
-		frappe.msgprint(_("Please enable developer mode to create new connection"))
-		return
-	# create folder
-	module_path = frappe.get_module_path(module)
-	connectors_folder = os.path.join(module_path, "connectors")
-	frappe.create_folder(connectors_folder)
-
-	# create init py
-	create_init_py(module_path, "connectors", "")
-
-	connection_class = connection_name.replace(" ", "")
-	file_name = frappe.scrub(connection_name) + ".py"
-	file_path = os.path.join(module_path, "connectors", file_name)
-
-	# create boilerplate file
-	with open(file_path, "w") as f:
-		f.write(connection_boilerplate.format(connection_class=connection_class))
-
-	# get python module string from file_path
-	app_name = frappe.db.get_value("Module Def", module, "app_name")
-	python_module = os.path.relpath(file_path, "../apps/{0}".format(app_name)).replace(
-		os.path.sep, "."
-	)[:-3]
-
-	return python_module
-
-
-def get_connection_class(python_module):
-	filename = python_module.rsplit(".", 1)[-1]
-	classname = frappe.unscrub(filename).replace(" ", "")
-	module = frappe.get_module(python_module)
-
-	raise_error = False
-	if hasattr(module, classname):
-		_class = getattr(module, classname)
-		if not issubclass(_class, BaseConnection):
-			raise_error = True
-	else:
-		raise_error = True
-
-	if raise_error:
-		raise ImportError(filename)
-
-	return _class
-
-
-connection_boilerplate = """from frappe.data_migration.doctype.data_migration_connector.connectors.base import BaseConnection
-
-class {connection_class}(BaseConnection):
-	def __init__(self, connector):
-		# self.connector = connector
-		# self.connection = YourModule(self.connector.username, self.get_password())
-		# self.name_field = 'id'
-		pass
-
-	def get(self, remote_objectname, fields=None, filters=None, start=0, page_length=10):
-		pass
-
-	def insert(self, doctype, doc):
-		pass
-
-	def update(self, doctype, doc, migration_id):
-		pass
-
-	def delete(self, doctype, migration_id):
-		pass
-
-"""

frappe/data_migration/doctype/data_migration_connector/test_data_migration_connector.py

@@ -1,8 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright (c) 2017, Frappe Technologies and Contributors
-# License: MIT. See LICENSE
-import unittest
-
-
-class TestDataMigrationConnector(unittest.TestCase):
-	pass

frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.js

@@ -1,8 +0,0 @@
-// Copyright (c) 2017, Frappe Technologies and contributors
-// For license information, please see license.txt
-
-frappe.ui.form.on('Data Migration Mapping', {
-	refresh: function() {
-
-	}
-});

frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.json

@@ -1,456 +0,0 @@
-{
- "allow_copy": 0, 
- "allow_guest_to_view": 0, 
- "allow_import": 0, 
- "allow_rename": 1, 
- "autoname": "field:mapping_name", 
- "beta": 1, 
- "creation": "2017-08-11 05:11:49.975801", 
- "custom": 0, 
- "docstatus": 0, 
- "doctype": "DocType", 
- "document_type": "", 
- "editable_grid": 1, 
- "engine": "InnoDB", 
- "fields": [
-  {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "mapping_name", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Mapping Name", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "remote_objectname", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Remote Objectname", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "remote_primary_key", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Remote Primary Key", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "local_doctype", 
-   "fieldtype": "Link", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Local DocType", 
-   "length": 0, 
-   "no_copy": 0, 
-   "options": "DocType", 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "local_primary_key", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Local Primary Key", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "column_break_5", 
-   "fieldtype": "Column Break", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "mapping_type", 
-   "fieldtype": "Select", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Mapping Type", 
-   "length": 0, 
-   "no_copy": 0, 
-   "options": "Push\nPull\nSync", 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "default": "10", 
-   "fieldname": "page_length", 
-   "fieldtype": "Int", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Page Length", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "migration_id_field", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Migration ID Field", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 1, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "mapping", 
-   "fieldtype": "Section Break", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Mapping", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "fields", 
-   "fieldtype": "Table", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Field Maps", 
-   "length": 0, 
-   "no_copy": 0, 
-   "options": "Data Migration Mapping Detail", 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 1, 
-   "columns": 0, 
-   "fieldname": "condition_detail", 
-   "fieldtype": "Section Break", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Condition Detail", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "condition", 
-   "fieldtype": "Code", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Condition", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }
- ], 
- "has_web_view": 0, 
- "hide_heading": 0, 
- "hide_toolbar": 0, 
- "idx": 0, 
- "image_view": 0, 
- "in_create": 0, 
- "is_submittable": 0, 
- "issingle": 0, 
- "istable": 0, 
- "max_attachments": 0, 
- "modified": "2017-09-27 18:06:43.275207", 
- "modified_by": "Administrator", 
- "module": "Data Migration", 
- "name": "Data Migration Mapping", 
- "name_case": "", 
- "owner": "Administrator", 
- "permissions": [
-  {
-   "amend": 0, 
-   "apply_user_permissions": 0, 
-   "cancel": 0, 
-   "create": 1, 
-   "delete": 1, 
-   "email": 1, 
-   "export": 1, 
-   "if_owner": 0, 
-   "import": 0, 
-   "permlevel": 0, 
-   "print": 1, 
-   "read": 1, 
-   "report": 1, 
-   "role": "System Manager", 
-   "set_user_permissions": 0, 
-   "share": 1, 
-   "submit": 0, 
-   "write": 1
-  }
- ], 
- "quick_entry": 1, 
- "read_only": 0, 
- "read_only_onload": 0, 
- "show_name_in_global_search": 0, 
- "sort_field": "modified", 
- "sort_order": "DESC", 
- "track_changes": 1, 
- "track_seen": 0
-}

frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.py

@@ -1,83 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright (c) 2017, Frappe Technologies and contributors
-# License: MIT. See LICENSE
-
-import frappe
-from frappe.model.document import Document
-from frappe.utils.safe_exec import get_safe_globals
-
-
-class DataMigrationMapping(Document):
-	def get_filters(self):
-		if self.condition:
-			return frappe.safe_eval(self.condition, get_safe_globals())
-
-	def get_fields(self):
-		fields = []
-		for f in self.fields:
-			if not (f.local_fieldname[0] in ('"', "'") or f.local_fieldname.startswith("eval:")):
-				fields.append(f.local_fieldname)
-
-		if frappe.db.has_column(self.local_doctype, self.migration_id_field):
-			fields.append(self.migration_id_field)
-
-		if "name" not in fields:
-			fields.append("name")
-
-		return fields
-
-	def get_mapped_record(self, doc):
-		"""Build a mapped record using information from the fields table"""
-		mapped = frappe._dict()
-
-		key_fieldname = "remote_fieldname"
-		value_fieldname = "local_fieldname"
-
-		if self.mapping_type == "Pull":
-			key_fieldname, value_fieldname = value_fieldname, key_fieldname
-
-		for field_map in self.fields:
-			key = get_source_value(field_map, key_fieldname)
-
-			if not field_map.is_child_table:
-				# field to field mapping
-				value = get_value_from_fieldname(field_map, value_fieldname, doc)
-			else:
-				# child table mapping
-				mapping_name = field_map.child_table_mapping
-				value = get_mapped_child_records(
-					mapping_name, doc.get(get_source_value(field_map, value_fieldname))
-				)
-
-			mapped[key] = value
-
-		return mapped
-
-
-def get_mapped_child_records(mapping_name, child_docs):
-	mapped_child_docs = []
-	mapping = frappe.get_doc("Data Migration Mapping", mapping_name)
-	for child_doc in child_docs:
-		mapped_child_docs.append(mapping.get_mapped_record(child_doc))
-
-	return mapped_child_docs
-
-
-def get_value_from_fieldname(field_map, fieldname_field, doc):
-	field_name = get_source_value(field_map, fieldname_field)
-
-	if field_name.startswith("eval:"):
-		value = frappe.safe_eval(field_name[5:], get_safe_globals())
-	elif field_name[0] in ('"', "'"):
-		value = field_name[1:-1]
-	else:
-		value = get_source_value(doc, field_name)
-	return value
-
-
-def get_source_value(source, key):
-	"""Get value from source (object or dict) based on key"""
-	if isinstance(source, dict):
-		return source.get(key)
-	else:
-		return getattr(source, key)

frappe/data_migration/doctype/data_migration_mapping/test_data_migration_mapping.py

@@ -1,8 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright (c) 2017, Frappe Technologies and Contributors
-# License: MIT. See LICENSE
-import unittest
-
-
-class TestDataMigrationMapping(unittest.TestCase):
-	pass

frappe/data_migration/doctype/data_migration_mapping_detail/data_migration_mapping_detail.json

@@ -1,163 +0,0 @@
-{
- "allow_copy": 0, 
- "allow_guest_to_view": 0, 
- "allow_import": 0, 
- "allow_rename": 0, 
- "beta": 0, 
- "creation": "2017-08-11 05:09:10.900237", 
- "custom": 0, 
- "docstatus": 0, 
- "doctype": "DocType", 
- "document_type": "", 
- "editable_grid": 1, 
- "engine": "InnoDB", 
- "fields": [
-  {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "remote_fieldname", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Remote Fieldname", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "local_fieldname", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Local Fieldname", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "is_child_table", 
-   "fieldtype": "Check", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Is Child Table", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "depends_on": "is_child_table", 
-   "fieldname": "child_table_mapping", 
-   "fieldtype": "Link", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Child Table Mapping", 
-   "length": 0, 
-   "no_copy": 0, 
-   "options": "Data Migration Mapping", 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }
- ], 
- "has_web_view": 0, 
- "hide_heading": 0, 
- "hide_toolbar": 0, 
- "idx": 0, 
- "image_view": 0, 
- "in_create": 0, 
- "is_submittable": 0, 
- "issingle": 0, 
- "istable": 1, 
- "max_attachments": 0, 
- "modified": "2017-09-28 17:13:31.337005", 
- "modified_by": "Administrator", 
- "module": "Data Migration", 
- "name": "Data Migration Mapping Detail", 
- "name_case": "", 
- "owner": "Administrator", 
- "permissions": [], 
- "quick_entry": 1, 
- "read_only": 0, 
- "read_only_onload": 0, 
- "show_name_in_global_search": 0, 
- "sort_field": "modified", 
- "sort_order": "DESC", 
- "track_changes": 1, 
- "track_seen": 0
-}

frappe/data_migration/doctype/data_migration_mapping_detail/data_migration_mapping_detail.py

@@ -1,9 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright (c) 2017, Frappe Technologies and contributors
-# License: MIT. See LICENSE
-
-from frappe.model.document import Document
-
-
-class DataMigrationMappingDetail(Document):
-	pass

frappe/data_migration/doctype/data_migration_plan/data_migration_plan.js

@@ -1,10 +0,0 @@
-// Copyright (c) 2017, Frappe Technologies and contributors
-// For license information, please see license.txt
-
-frappe.ui.form.on('Data Migration Plan', {
-	onload(frm) {
-		frm.add_custom_button(__('Run'), () => frappe.new_doc('Data Migration Run', {
-			data_migration_plan: frm.doc.name
-		}));
-	}
-});

frappe/data_migration/doctype/data_migration_plan/data_migration_plan.json

@@ -1,224 +0,0 @@
-{
- "allow_copy": 0, 
- "allow_guest_to_view": 0, 
- "allow_import": 0, 
- "allow_rename": 0, 
- "autoname": "field:plan_name", 
- "beta": 0, 
- "creation": "2017-08-11 05:15:51.482165", 
- "custom": 0, 
- "docstatus": 0, 
- "doctype": "DocType", 
- "document_type": "", 
- "editable_grid": 1, 
- "engine": "InnoDB", 
- "fields": [
-  {
-   "allow_bulk_edit": 0, 
-   "allow_in_quick_entry": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "plan_name", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Plan Name", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "translatable": 0, 
-   "unique": 1
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_in_quick_entry": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "module", 
-   "fieldtype": "Link", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Module", 
-   "length": 0, 
-   "no_copy": 0, 
-   "options": "Module Def", 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "translatable": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_in_quick_entry": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "mappings", 
-   "fieldtype": "Table", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Mappings", 
-   "length": 0, 
-   "no_copy": 0, 
-   "options": "Data Migration Plan Mapping", 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "translatable": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_in_quick_entry": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "preprocess_method", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Preprocess Method", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "translatable": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_in_quick_entry": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "postprocess_method", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Postprocess Method", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "translatable": 0, 
-   "unique": 0
-  }
- ], 
- "has_web_view": 0, 
- "hide_heading": 0, 
- "hide_toolbar": 0, 
- "idx": 0, 
- "image_view": 0, 
- "in_create": 0, 
- "is_submittable": 0, 
- "issingle": 0, 
- "istable": 0, 
- "max_attachments": 0, 
- "modified": "2020-09-18 17:26:09.703215", 
- "modified_by": "Administrator", 
- "module": "Data Migration", 
- "name": "Data Migration Plan", 
- "name_case": "", 
- "owner": "Administrator", 
- "permissions": [
-  {
-   "amend": 0, 
-   "cancel": 0, 
-   "create": 1, 
-   "delete": 1, 
-   "email": 1, 
-   "export": 1, 
-   "if_owner": 0, 
-   "import": 0, 
-   "permlevel": 0, 
-   "print": 1, 
-   "read": 1, 
-   "report": 1, 
-   "role": "System Manager", 
-   "set_user_permissions": 0, 
-   "share": 1, 
-   "submit": 0, 
-   "write": 1
-  }
- ], 
- "quick_entry": 1, 
- "read_only": 0, 
- "read_only_onload": 0, 
- "show_name_in_global_search": 0, 
- "sort_field": "modified", 
- "sort_order": "DESC", 
- "track_changes": 1, 
- "track_seen": 0
-}

frappe/data_migration/doctype/data_migration_plan/data_migration_plan.py

@@ -1,78 +0,0 @@
-# Copyright (c) 2021, Frappe Technologies and contributors
-# License: MIT. See LICENSE
-
-import frappe
-from frappe.custom.doctype.custom_field.custom_field import create_custom_field
-from frappe.model.document import Document
-from frappe.modules import get_module_path, scrub_dt_dn
-from frappe.modules.export_file import create_init_py, export_to_files
-
-
-def get_mapping_module(module, mapping_name):
-	app_name = frappe.db.get_value("Module Def", module, "app_name")
-	mapping_name = frappe.scrub(mapping_name)
-	module = frappe.scrub(module)
-
-	try:
-		return frappe.get_module(f"{app_name}.{module}.data_migration_mapping.{mapping_name}")
-	except ImportError:
-		return None
-
-
-class DataMigrationPlan(Document):
-	def on_update(self):
-		# update custom fields in mappings
-		self.make_custom_fields_for_mappings()
-
-		if frappe.flags.in_import or frappe.flags.in_test:
-			return
-
-		if frappe.local.conf.get("developer_mode"):
-			record_list = [["Data Migration Plan", self.name]]
-
-			for m in self.mappings:
-				record_list.append(["Data Migration Mapping", m.mapping])
-
-			export_to_files(record_list=record_list, record_module=self.module)
-
-			for m in self.mappings:
-				dt, dn = scrub_dt_dn("Data Migration Mapping", m.mapping)
-				create_init_py(get_module_path(self.module), dt, dn)
-
-	def make_custom_fields_for_mappings(self):
-		frappe.flags.ignore_in_install = True
-		label = self.name + " ID"
-		fieldname = frappe.scrub(label)
-
-		df = {
-			"label": label,
-			"fieldname": fieldname,
-			"fieldtype": "Data",
-			"hidden": 1,
-			"read_only": 1,
-			"unique": 1,
-			"no_copy": 1,
-		}
-
-		for m in self.mappings:
-			mapping = frappe.get_doc("Data Migration Mapping", m.mapping)
-			create_custom_field(mapping.local_doctype, df)
-			mapping.migration_id_field = fieldname
-			mapping.save()
-
-		# Create custom field in Deleted Document
-		create_custom_field("Deleted Document", df)
-		frappe.flags.ignore_in_install = False
-
-	def pre_process_doc(self, mapping_name, doc):
-		module = get_mapping_module(self.module, mapping_name)
-
-		if module and hasattr(module, "pre_process"):
-			return module.pre_process(doc)
-		return doc
-
-	def post_process_doc(self, mapping_name, local_doc=None, remote_doc=None):
-		module = get_mapping_module(self.module, mapping_name)
-
-		if module and hasattr(module, "post_process"):
-			return module.post_process(local_doc=local_doc, remote_doc=remote_doc)

frappe/data_migration/doctype/data_migration_plan/test_data_migration_plan.py

@@ -1,8 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright (c) 2017, Frappe Technologies and Contributors
-# License: MIT. See LICENSE
-import unittest
-
-
-class TestDataMigrationPlan(unittest.TestCase):
-	pass

frappe/data_migration/doctype/data_migration_plan_mapping/data_migration_plan_mapping.json

@@ -1,103 +0,0 @@
-{
- "allow_copy": 0, 
- "allow_guest_to_view": 0, 
- "allow_import": 0, 
- "allow_rename": 0, 
- "beta": 1, 
- "creation": "2017-08-11 05:15:38.390831", 
- "custom": 0, 
- "docstatus": 0, 
- "doctype": "DocType", 
- "document_type": "", 
- "editable_grid": 1, 
- "engine": "InnoDB", 
- "fields": [
-  {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "mapping", 
-   "fieldtype": "Link", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Mapping", 
-   "length": 0, 
-   "no_copy": 0, 
-   "options": "Data Migration Mapping", 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "default": "1", 
-   "fieldname": "enabled", 
-   "fieldtype": "Check", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Enabled", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }
- ], 
- "has_web_view": 0, 
- "hide_heading": 0, 
- "hide_toolbar": 0, 
- "idx": 0, 
- "image_view": 0, 
- "in_create": 0, 
- "is_submittable": 0, 
- "issingle": 0, 
- "istable": 1, 
- "max_attachments": 0, 
- "modified": "2017-09-20 21:43:04.908650", 
- "modified_by": "Administrator", 
- "module": "Data Migration", 
- "name": "Data Migration Plan Mapping", 
- "name_case": "", 
- "owner": "Administrator", 
- "permissions": [], 
- "quick_entry": 1, 
- "read_only": 0, 
- "read_only_onload": 0, 
- "show_name_in_global_search": 0, 
- "sort_field": "modified", 
- "sort_order": "DESC", 
- "track_changes": 1, 
- "track_seen": 0
-}

frappe/data_migration/doctype/data_migration_plan_mapping/data_migration_plan_mapping.py

@@ -1,9 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright (c) 2017, Frappe Technologies and contributors
-# License: MIT. See LICENSE
-
-from frappe.model.document import Document
-
-
-class DataMigrationPlanMapping(Document):
-	pass

frappe/data_migration/doctype/data_migration_run/data_migration_run.js

@@ -1,14 +0,0 @@
-// Copyright (c) 2017, Frappe Technologies and contributors
-// For license information, please see license.txt
-
-frappe.ui.form.on('Data Migration Run', {
-	refresh: function(frm) {
-		if (frm.doc.status !== 'Success') {
-			frm.add_custom_button(__('Run'), () => frm.call('run'));
-		}
-		if (frm.doc.status === 'Started') {
-			frm.dashboard.add_progress(__('Percent Complete'), frm.doc.percent_complete,
-				__('Currently updating {0}', [frm.doc.current_mapping]));
-		}
-	}
-});

frappe/data_migration/doctype/data_migration_run/data_migration_run.json

@@ -1,838 +0,0 @@
-{
- "allow_copy": 0, 
- "allow_guest_to_view": 0, 
- "allow_import": 0, 
- "allow_rename": 0, 
- "beta": 0, 
- "creation": "2017-09-11 12:55:27.597728", 
- "custom": 0, 
- "docstatus": 0, 
- "doctype": "DocType", 
- "document_type": "", 
- "editable_grid": 1, 
- "engine": "InnoDB", 
- "fields": [
-  {
-   "allow_bulk_edit": 0, 
-   "allow_in_quick_entry": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "data_migration_plan", 
-   "fieldtype": "Link", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Data Migration Plan", 
-   "length": 0, 
-   "no_copy": 0, 
-   "options": "Data Migration Plan", 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "translatable": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_in_quick_entry": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "data_migration_connector", 
-   "fieldtype": "Link", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Data Migration Connector", 
-   "length": 0, 
-   "no_copy": 0, 
-   "options": "Data Migration Connector", 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "translatable": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_in_quick_entry": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "default": "Pending", 
-   "fieldname": "status", 
-   "fieldtype": "Select", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Status", 
-   "length": 0, 
-   "no_copy": 1, 
-   "options": "Pending\nStarted\nPartial Success\nSuccess\nFail\nError", 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 1, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "translatable": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_in_quick_entry": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "start_time", 
-   "fieldtype": "Datetime", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Start Time", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "translatable": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_in_quick_entry": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "end_time", 
-   "fieldtype": "Datetime", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "End Time", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "translatable": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_in_quick_entry": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "remote_id", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Remote ID", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 1, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "translatable": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_in_quick_entry": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "current_mapping", 
-   "fieldtype": "Data", 
-   "hidden": 1, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Current Mapping", 
-   "length": 0, 
-   "no_copy": 1, 
-   "options": "", 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 1, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "translatable": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_in_quick_entry": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "current_mapping_start", 
-   "fieldtype": "Int", 
-   "hidden": 1, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Current Mapping Start", 
-   "length": 0, 
-   "no_copy": 1, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 1, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "translatable": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_in_quick_entry": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "current_mapping_delete_start", 
-   "fieldtype": "Int", 
-   "hidden": 1, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Current Mapping Delete Start", 
-   "length": 0, 
-   "no_copy": 1, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "translatable": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_in_quick_entry": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "current_mapping_type", 
-   "fieldtype": "Select", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Current Mapping Type", 
-   "length": 0, 
-   "no_copy": 0, 
-   "options": "Push\nPull", 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "translatable": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_in_quick_entry": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "depends_on": "eval:(doc.status !== 'Pending')", 
-   "fieldname": "current_mapping_action", 
-   "fieldtype": "Select", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Current Mapping Action", 
-   "length": 0, 
-   "no_copy": 1, 
-   "options": "Insert\nDelete", 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 1, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "translatable": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_in_quick_entry": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "total_pages", 
-   "fieldtype": "Int", 
-   "hidden": 1, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Total Pages", 
-   "length": 0, 
-   "no_copy": 1, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "translatable": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_in_quick_entry": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "percent_complete", 
-   "fieldtype": "Percent", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Percent Complete", 
-   "length": 0, 
-   "no_copy": 1, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 1, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "translatable": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_in_quick_entry": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "trigger_name", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Trigger Name", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "translatable": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_in_quick_entry": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "depends_on": "eval:(doc.status !== 'Pending')", 
-   "fieldname": "logs_sb", 
-   "fieldtype": "Section Break", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Logs", 
-   "length": 0, 
-   "no_copy": 1, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 1, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "translatable": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_in_quick_entry": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "push_insert", 
-   "fieldtype": "Int", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Push Insert", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "translatable": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_in_quick_entry": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "push_update", 
-   "fieldtype": "Int", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Push Update", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "translatable": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_in_quick_entry": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "push_delete", 
-   "fieldtype": "Int", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Push Delete", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "translatable": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_in_quick_entry": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "push_failed", 
-   "fieldtype": "Code", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Push Failed", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "translatable": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_in_quick_entry": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "column_break_16", 
-   "fieldtype": "Column Break", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "translatable": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_in_quick_entry": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "pull_insert", 
-   "fieldtype": "Int", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Pull Insert", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "translatable": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_in_quick_entry": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "pull_update", 
-   "fieldtype": "Int", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Pull Update", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "translatable": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_in_quick_entry": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "pull_failed", 
-   "fieldtype": "Code", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Pull Failed", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "translatable": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_in_quick_entry": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "depends_on": "eval:doc.failed_log !== '[]'", 
-   "fieldname": "log", 
-   "fieldtype": "Code", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Log", 
-   "length": 0, 
-   "no_copy": 1, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 1, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "translatable": 0, 
-   "unique": 0
-  }
- ], 
- "has_web_view": 0, 
- "hide_heading": 0, 
- "hide_toolbar": 0, 
- "idx": 0, 
- "image_view": 0, 
- "in_create": 0, 
- "is_submittable": 0, 
- "issingle": 0, 
- "istable": 0, 
- "max_attachments": 0, 
- "modified": "2020-09-18 17:26:09.703215", 
- "modified_by": "Administrator", 
- "module": "Data Migration", 
- "name": "Data Migration Run", 
- "name_case": "", 
- "owner": "Administrator", 
- "permissions": [
-  {
-   "amend": 0, 
-   "cancel": 0, 
-   "create": 1, 
-   "delete": 1, 
-   "email": 1, 
-   "export": 1, 
-   "if_owner": 0, 
-   "import": 0, 
-   "permlevel": 0, 
-   "print": 1, 
-   "read": 1, 
-   "report": 1, 
-   "role": "System Manager", 
-   "set_user_permissions": 0, 
-   "share": 1, 
-   "submit": 0, 
-   "write": 1
-  }
- ], 
- "quick_entry": 0, 
- "read_only": 0, 
- "read_only_onload": 0, 
- "show_name_in_global_search": 0, 
- "sort_field": "modified", 
- "sort_order": "DESC", 
- "track_changes": 1, 
- "track_seen": 0
-}

frappe/data_migration/doctype/data_migration_run/data_migration_run.py

@@ -1,514 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright (c) 2017, Frappe Technologies and contributors
-# License: MIT. See LICENSE
-
-import json
-import math
-
-import frappe
-from frappe import _
-from frappe.data_migration.doctype.data_migration_mapping.data_migration_mapping import (
-	get_source_value,
-)
-from frappe.model.document import Document
-from frappe.utils import cstr
-
-
-class DataMigrationRun(Document):
-	@frappe.whitelist()
-	def run(self):
-		self.begin()
-		if self.total_pages > 0:
-			self.enqueue_next_mapping()
-		else:
-			self.complete()
-
-	def enqueue_next_mapping(self):
-		next_mapping_name = self.get_next_mapping_name()
-		if next_mapping_name:
-			next_mapping = self.get_mapping(next_mapping_name)
-			self.db_set(
-				dict(
-					current_mapping=next_mapping.name,
-					current_mapping_start=0,
-					current_mapping_delete_start=0,
-					current_mapping_action="Insert",
-				),
-				notify=True,
-				commit=True,
-			)
-			frappe.enqueue_doc(self.doctype, self.name, "run_current_mapping", now=frappe.flags.in_test)
-		else:
-			self.complete()
-
-	def enqueue_next_page(self):
-		mapping = self.get_mapping(self.current_mapping)
-		percent_complete = self.percent_complete + (100.0 / self.total_pages)
-		fields = dict(percent_complete=percent_complete)
-		if self.current_mapping_action == "Insert":
-			start = self.current_mapping_start + mapping.page_length
-			fields["current_mapping_start"] = start
-		elif self.current_mapping_action == "Delete":
-			delete_start = self.current_mapping_delete_start + mapping.page_length
-			fields["current_mapping_delete_start"] = delete_start
-
-		self.db_set(fields, notify=True, commit=True)
-
-		if percent_complete < 100:
-			frappe.publish_realtime(
-				self.trigger_name, {"progress_percent": percent_complete}, user=frappe.session.user
-			)
-
-		frappe.enqueue_doc(self.doctype, self.name, "run_current_mapping", now=frappe.flags.in_test)
-
-	def run_current_mapping(self):
-		try:
-			mapping = self.get_mapping(self.current_mapping)
-
-			if mapping.mapping_type == "Push":
-				done = self.push()
-			elif mapping.mapping_type == "Pull":
-				done = self.pull()
-
-			if done:
-				self.enqueue_next_mapping()
-			else:
-				self.enqueue_next_page()
-
-		except Exception as e:
-			self.db_set("status", "Error", notify=True, commit=True)
-			print("Data Migration Run failed")
-			print(frappe.get_traceback())
-			self.execute_postprocess("Error")
-			raise e
-
-	def get_last_modified_condition(self):
-		last_run_timestamp = frappe.db.get_value(
-			"Data Migration Run",
-			dict(
-				data_migration_plan=self.data_migration_plan,
-				data_migration_connector=self.data_migration_connector,
-				name=("!=", self.name),
-			),
-			"modified",
-		)
-		if last_run_timestamp:
-			condition = dict(modified=(">", last_run_timestamp))
-		else:
-			condition = {}
-		return condition
-
-	def begin(self):
-		plan_active_mappings = [m for m in self.get_plan().mappings if m.enabled]
-		self.mappings = [
-			frappe.get_doc("Data Migration Mapping", m.mapping) for m in plan_active_mappings
-		]
-
-		total_pages = 0
-		for m in [mapping for mapping in self.mappings]:
-			if m.mapping_type == "Push":
-				count = float(self.get_count(m))
-				page_count = math.ceil(count / m.page_length)
-				total_pages += page_count
-			if m.mapping_type == "Pull":
-				total_pages += 10
-
-		self.db_set(
-			dict(
-				status="Started",
-				current_mapping=None,
-				current_mapping_start=0,
-				current_mapping_delete_start=0,
-				percent_complete=0,
-				current_mapping_action="Insert",
-				total_pages=total_pages,
-			),
-			notify=True,
-			commit=True,
-		)
-
-	def complete(self):
-		fields = dict()
-
-		push_failed = self.get_log("push_failed", [])
-		pull_failed = self.get_log("pull_failed", [])
-
-		status = "Partial Success"
-
-		if not push_failed and not pull_failed:
-			status = "Success"
-			fields["percent_complete"] = 100
-
-		fields["status"] = status
-
-		self.db_set(fields, notify=True, commit=True)
-
-		self.execute_postprocess(status)
-
-		frappe.publish_realtime(self.trigger_name, {"progress_percent": 100}, user=frappe.session.user)
-
-	def execute_postprocess(self, status):
-		# Execute post process
-		postprocess_method_path = self.get_plan().postprocess_method
-
-		if postprocess_method_path:
-			frappe.get_attr(postprocess_method_path)(
-				{
-					"status": status,
-					"stats": {
-						"push_insert": self.push_insert,
-						"push_update": self.push_update,
-						"push_delete": self.push_delete,
-						"pull_insert": self.pull_insert,
-						"pull_update": self.pull_update,
-					},
-				}
-			)
-
-	def get_plan(self):
-		if not hasattr(self, "plan"):
-			self.plan = frappe.get_doc("Data Migration Plan", self.data_migration_plan)
-		return self.plan
-
-	def get_mapping(self, mapping_name):
-		if hasattr(self, "mappings"):
-			for m in self.mappings:
-				if m.name == mapping_name:
-					return m
-		return frappe.get_doc("Data Migration Mapping", mapping_name)
-
-	def get_next_mapping_name(self):
-		mappings = [m for m in self.get_plan().mappings if m.enabled]
-		if not self.current_mapping:
-			# first
-			return mappings[0].mapping
-		for i, d in enumerate(mappings):
-			if i == len(mappings) - 1:
-				# last
-				return None
-			if d.mapping == self.current_mapping:
-				return mappings[i + 1].mapping
-
-		raise frappe.ValidationError("Mapping Broken")
-
-	def get_data(self, filters):
-		mapping = self.get_mapping(self.current_mapping)
-		or_filters = self.get_or_filters(mapping)
-		start = self.current_mapping_start
-
-		data = []
-		doclist = frappe.get_all(
-			mapping.local_doctype,
-			filters=filters,
-			or_filters=or_filters,
-			start=start,
-			page_length=mapping.page_length,
-		)
-
-		for d in doclist:
-			doc = frappe.get_doc(mapping.local_doctype, d["name"])
-			data.append(doc)
-		return data
-
-	def get_new_local_data(self):
-		"""Fetch newly inserted local data using `frappe.get_all`. Used during Push"""
-		mapping = self.get_mapping(self.current_mapping)
-		filters = mapping.get_filters() or {}
-
-		# new docs dont have migration field set
-		filters.update({mapping.migration_id_field: ""})
-
-		return self.get_data(filters)
-
-	def get_updated_local_data(self):
-		"""Fetch local updated data using `frappe.get_all`. Used during Push"""
-		mapping = self.get_mapping(self.current_mapping)
-		filters = mapping.get_filters() or {}
-
-		# existing docs must have migration field set
-		filters.update({mapping.migration_id_field: ("!=", "")})
-
-		return self.get_data(filters)
-
-	def get_deleted_local_data(self):
-		"""Fetch local deleted data using `frappe.get_all`. Used during Push"""
-		mapping = self.get_mapping(self.current_mapping)
-		filters = self.get_last_modified_condition()
-		filters.update({"deleted_doctype": mapping.local_doctype})
-
-		data = frappe.get_all("Deleted Document", fields=["name", "data"], filters=filters)
-
-		_data = []
-		for d in data:
-			doc = json.loads(d.data)
-			if doc.get(mapping.migration_id_field):
-				doc["_deleted_document_name"] = d["name"]
-				_data.append(doc)
-
-		return _data
-
-	def get_remote_data(self):
-		"""Fetch data from remote using `connection.get`. Used during Pull"""
-		mapping = self.get_mapping(self.current_mapping)
-		start = self.current_mapping_start
-		filters = mapping.get_filters() or {}
-		connection = self.get_connection()
-
-		return connection.get(
-			mapping.remote_objectname,
-			fields=["*"],
-			filters=filters,
-			start=start,
-			page_length=mapping.page_length,
-		)
-
-	def get_count(self, mapping):
-		filters = mapping.get_filters() or {}
-		or_filters = self.get_or_filters(mapping)
-
-		to_insert = frappe.get_all(
-			mapping.local_doctype, ["count(name) as total"], filters=filters, or_filters=or_filters
-		)[0].total
-
-		to_delete = frappe.get_all(
-			"Deleted Document",
-			["count(name) as total"],
-			filters={"deleted_doctype": mapping.local_doctype},
-			or_filters=or_filters,
-		)[0].total
-
-		return to_insert + to_delete
-
-	def get_or_filters(self, mapping):
-		or_filters = self.get_last_modified_condition()
-
-		# docs whose migration_id_field is not set
-		# failed in the previous run, include those too
-		or_filters.update({mapping.migration_id_field: ("=", "")})
-
-		return or_filters
-
-	def get_connection(self):
-		if not hasattr(self, "connection"):
-			self.connection = frappe.get_doc(
-				"Data Migration Connector", self.data_migration_connector
-			).get_connection()
-
-		return self.connection
-
-	def push(self):
-		self.db_set("current_mapping_type", "Push")
-		done = True
-
-		if self.current_mapping_action == "Insert":
-			done = self._push_insert()
-
-		elif self.current_mapping_action == "Update":
-			done = self._push_update()
-
-		elif self.current_mapping_action == "Delete":
-			done = self._push_delete()
-
-		return done
-
-	def _push_insert(self):
-		"""Inserts new local docs on remote"""
-		mapping = self.get_mapping(self.current_mapping)
-		connection = self.get_connection()
-		data = self.get_new_local_data()
-
-		for d in data:
-			# pre process before insert
-			doc = self.pre_process_doc(d)
-			doc = mapping.get_mapped_record(doc)
-
-			try:
-				response_doc = connection.insert(mapping.remote_objectname, doc)
-				frappe.db.set_value(
-					mapping.local_doctype,
-					d.name,
-					mapping.migration_id_field,
-					response_doc[connection.name_field],
-					update_modified=False,
-				)
-				frappe.db.commit()
-				self.update_log("push_insert", 1)
-				# post process after insert
-				self.post_process_doc(local_doc=d, remote_doc=response_doc)
-			except Exception as e:
-				self.update_log("push_failed", {d.name: cstr(e)})
-
-		# update page_start
-		self.db_set("current_mapping_start", self.current_mapping_start + mapping.page_length)
-
-		if len(data) < mapping.page_length:
-			# done, no more new data to insert
-			self.db_set({"current_mapping_action": "Update", "current_mapping_start": 0})
-			# not done with this mapping
-			return False
-
-	def _push_update(self):
-		"""Updates local modified docs on remote"""
-		mapping = self.get_mapping(self.current_mapping)
-		connection = self.get_connection()
-		data = self.get_updated_local_data()
-
-		for d in data:
-			migration_id_value = d.get(mapping.migration_id_field)
-			# pre process before update
-			doc = self.pre_process_doc(d)
-			doc = mapping.get_mapped_record(doc)
-			try:
-				response_doc = connection.update(mapping.remote_objectname, doc, migration_id_value)
-				self.update_log("push_update", 1)
-				# post process after update
-				self.post_process_doc(local_doc=d, remote_doc=response_doc)
-			except Exception as e:
-				self.update_log("push_failed", {d.name: cstr(e)})
-
-		# update page_start
-		self.db_set("current_mapping_start", self.current_mapping_start + mapping.page_length)
-
-		if len(data) < mapping.page_length:
-			# done, no more data to update
-			self.db_set({"current_mapping_action": "Delete", "current_mapping_start": 0})
-			# not done with this mapping
-			return False
-
-	def _push_delete(self):
-		"""Deletes docs deleted from local on remote"""
-		mapping = self.get_mapping(self.current_mapping)
-		connection = self.get_connection()
-		data = self.get_deleted_local_data()
-
-		for d in data:
-			# Deleted Document also has a custom field for migration_id
-			migration_id_value = d.get(mapping.migration_id_field)
-			# pre process before update
-			self.pre_process_doc(d)
-			try:
-				response_doc = connection.delete(mapping.remote_objectname, migration_id_value)
-				self.update_log("push_delete", 1)
-				# post process only when action is success
-				self.post_process_doc(local_doc=d, remote_doc=response_doc)
-			except Exception as e:
-				self.update_log("push_failed", {d.name: cstr(e)})
-
-		# update page_start
-		self.db_set("current_mapping_start", self.current_mapping_start + mapping.page_length)
-
-		if len(data) < mapping.page_length:
-			# done, no more new data to delete
-			# done with this mapping
-			return True
-
-	def pull(self):
-		self.db_set("current_mapping_type", "Pull")
-
-		connection = self.get_connection()
-		mapping = self.get_mapping(self.current_mapping)
-		data = self.get_remote_data()
-
-		for d in data:
-			migration_id_value = get_source_value(d, connection.name_field)
-			doc = self.pre_process_doc(d)
-			doc = mapping.get_mapped_record(doc)
-
-			if migration_id_value:
-				try:
-					if not local_doc_exists(mapping, migration_id_value):
-						# insert new local doc
-						local_doc = insert_local_doc(mapping, doc)
-
-						self.update_log("pull_insert", 1)
-						# set migration id
-						frappe.db.set_value(
-							mapping.local_doctype,
-							local_doc.name,
-							mapping.migration_id_field,
-							migration_id_value,
-							update_modified=False,
-						)
-						frappe.db.commit()
-					else:
-						# update doc
-						local_doc = update_local_doc(mapping, doc, migration_id_value)
-						self.update_log("pull_update", 1)
-					# post process doc after success
-					self.post_process_doc(remote_doc=d, local_doc=local_doc)
-				except Exception as e:
-					# failed, append to log
-					self.update_log("pull_failed", {migration_id_value: cstr(e)})
-
-		if len(data) < mapping.page_length:
-			# last page, done with pull
-			return True
-
-	def pre_process_doc(self, doc):
-		plan = self.get_plan()
-		doc = plan.pre_process_doc(self.current_mapping, doc)
-		return doc
-
-	def post_process_doc(self, local_doc=None, remote_doc=None):
-		plan = self.get_plan()
-		doc = plan.post_process_doc(self.current_mapping, local_doc=local_doc, remote_doc=remote_doc)
-		return doc
-
-	def set_log(self, key, value):
-		value = json.dumps(value) if "_failed" in key else value
-		self.db_set(key, value)
-
-	def update_log(self, key, value=None):
-		"""
-		Helper for updating logs,
-		push_failed and pull_failed are stored as json,
-		other keys are stored as int
-		"""
-		if "_failed" in key:
-			# json
-			self.set_log(key, self.get_log(key, []) + [value])
-		else:
-			# int
-			self.set_log(key, self.get_log(key, 0) + (value or 1))
-
-	def get_log(self, key, default=None):
-		value = self.db_get(key)
-		if "_failed" in key:
-			if not value:
-				value = json.dumps(default)
-			value = json.loads(value)
-		return value or default
-
-
-def insert_local_doc(mapping, doc):
-	try:
-		# insert new doc
-		if not doc.doctype:
-			doc.doctype = mapping.local_doctype
-		doc = frappe.get_doc(doc).insert()
-		return doc
-	except Exception:
-		print("Data Migration Run failed: Error in Pull insert")
-		print(frappe.get_traceback())
-		return None
-
-
-def update_local_doc(mapping, remote_doc, migration_id_value):
-	try:
-		# migration id value is set in migration_id_field in mapping.local_doctype
-		docname = frappe.db.get_value(
-			mapping.local_doctype, filters={mapping.migration_id_field: migration_id_value}
-		)
-
-		doc = frappe.get_doc(mapping.local_doctype, docname)
-		doc.update(remote_doc)
-		doc.save()
-		return doc
-	except Exception:
-		print("Data Migration Run failed: Error in Pull update")
-		print(frappe.get_traceback())
-		return None
-
-
-def local_doc_exists(mapping, migration_id_value):
-	return frappe.db.exists(mapping.local_doctype, {mapping.migration_id_field: migration_id_value})

frappe/data_migration/doctype/data_migration_run/test_data_migration_run.py

@@ -1,128 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright (c) 2017, Frappe Technologies and Contributors
-# License: MIT. See LICENSE
-import unittest
-
-import frappe
-
-
-class TestDataMigrationRun(unittest.TestCase):
-	def test_run(self):
-		create_plan()
-
-		description = "data migration todo"
-		new_todo = frappe.get_doc({"doctype": "ToDo", "description": description}).insert()
-
-		event_subject = "data migration event"
-		frappe.get_doc(
-			dict(
-				doctype="Event",
-				subject=event_subject,
-				repeat_on="Monthly",
-				starts_on=frappe.utils.now_datetime(),
-			)
-		).insert()
-
-		run = frappe.get_doc(
-			{
-				"doctype": "Data Migration Run",
-				"data_migration_plan": "ToDo Sync",
-				"data_migration_connector": "Local Connector",
-			}
-		).insert()
-
-		run.run()
-		self.assertEqual(run.db_get("status"), "Success")
-
-		self.assertEqual(run.db_get("push_insert"), 1)
-		self.assertEqual(run.db_get("pull_insert"), 1)
-
-		todo = frappe.get_doc("ToDo", new_todo.name)
-		self.assertTrue(todo.todo_sync_id)
-
-		# Pushed Event
-		event = frappe.get_doc("Event", todo.todo_sync_id)
-		self.assertEqual(event.subject, description)
-
-		# Pulled ToDo
-		created_todo = frappe.get_doc("ToDo", {"description": event_subject})
-		self.assertEqual(created_todo.description, event_subject)
-
-		todo_list = frappe.get_list(
-			"ToDo", filters={"description": "data migration todo"}, fields=["name"]
-		)
-		todo_name = todo_list[0].name
-
-		todo = frappe.get_doc("ToDo", todo_name)
-		todo.description = "data migration todo updated"
-		todo.save()
-
-		run = frappe.get_doc(
-			{
-				"doctype": "Data Migration Run",
-				"data_migration_plan": "ToDo Sync",
-				"data_migration_connector": "Local Connector",
-			}
-		).insert()
-
-		run.run()
-
-		# Update
-		self.assertEqual(run.db_get("status"), "Success")
-		self.assertEqual(run.db_get("pull_update"), 1)
-
-
-def create_plan():
-	frappe.get_doc(
-		{
-			"doctype": "Data Migration Mapping",
-			"mapping_name": "Todo to Event",
-			"remote_objectname": "Event",
-			"remote_primary_key": "name",
-			"mapping_type": "Push",
-			"local_doctype": "ToDo",
-			"fields": [
-				{"remote_fieldname": "subject", "local_fieldname": "description"},
-				{
-					"remote_fieldname": "starts_on",
-					"local_fieldname": "eval:frappe.utils.get_datetime_str(frappe.utils.get_datetime())",
-				},
-			],
-			"condition": '{"description": "data migration todo" }',
-		}
-	).insert(ignore_if_duplicate=True)
-
-	frappe.get_doc(
-		{
-			"doctype": "Data Migration Mapping",
-			"mapping_name": "Event to ToDo",
-			"remote_objectname": "Event",
-			"remote_primary_key": "name",
-			"local_doctype": "ToDo",
-			"local_primary_key": "name",
-			"mapping_type": "Pull",
-			"condition": '{"subject": "data migration event" }',
-			"fields": [{"remote_fieldname": "subject", "local_fieldname": "description"}],
-		}
-	).insert(ignore_if_duplicate=True)
-
-	frappe.get_doc(
-		{
-			"doctype": "Data Migration Plan",
-			"plan_name": "ToDo Sync",
-			"module": "Core",
-			"mappings": [{"mapping": "Todo to Event"}, {"mapping": "Event to ToDo"}],
-		}
-	).insert(ignore_if_duplicate=True)
-
-	frappe.get_doc(
-		{
-			"doctype": "Data Migration Connector",
-			"connector_name": "Local Connector",
-			"connector_type": "Frappe",
-			# connect to same host.
-			"hostname": frappe.conf.host_name or frappe.utils.get_site_url(frappe.local.site),
-			"username": "Administrator",
-			"password": frappe.conf.get("admin_password") or "admin",
-		}
-	).insert(ignore_if_duplicate=True)

frappe/database/database.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
+# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
 
 # Database Module
@@ -18,10 +18,12 @@ import frappe
 import frappe.defaults
 import frappe.model.meta
 from frappe import _
+from frappe.exceptions import DoesNotExistError
 from frappe.model.utils.link_count import flush_local_link_count
 from frappe.query_builder.functions import Count
 from frappe.query_builder.utils import DocType
-from frappe.utils import cast, get_datetime, get_table_name, getdate, now, sbool
+from frappe.utils import cast as cast_fieldtype
+from frappe.utils import get_datetime, get_table_name, getdate, now, sbool
 
 IFNULL_PATTERN = re.compile(r"ifnull\(", flags=re.IGNORECASE)
 INDEX_PATTERN = re.compile(r"\s*\([^)]+\)\s*")
@@ -29,6 +31,10 @@ SINGLE_WORD_PATTERN = re.compile(r'([`"]?)(tab([A-Z]\w+))\1')
 MULTI_WORD_PATTERN = re.compile(r'([`"])(tab([A-Z]\w+)( [A-Z]\w+)+)\1')
 
 
+def is_query_type(query: str, query_type: Union[str, Tuple[str]]) -> bool:
+	return query.lstrip().split(maxsplit=1)[0].lower().startswith(query_type)
+
+
 class Database(object):
 	"""
 	Open a database connection with the given parmeters, if use_default is True, use the
@@ -239,7 +245,7 @@ class Database(object):
 
 		# debug
 		if debug:
-			if explain and query.strip().lower().startswith("select"):
+			if explain and is_query_type(query, "select"):
 				self.explain_query(query, values)
 			frappe.errprint(self.mogrify(query, values))
 
@@ -296,7 +302,7 @@ class Database(object):
 		could cause the system to hang."""
 		self.check_implicit_commit(query)
 
-		if query and query.strip().lower() in ("commit", "rollback"):
+		if query and is_query_type(query, ("commit", "rollback")):
 			self.transaction_writes = 0
 
 		if query[:6].lower() in ("update", "insert", "delete"):
@@ -313,8 +319,7 @@ class Database(object):
 		if (
 			self.transaction_writes
 			and query
-			and query.strip().split()[0].lower()
-			in ["start", "alter", "drop", "create", "begin", "truncate"]
+			and is_query_type(query, ("start", "alter", "drop", "create", "begin", "truncate"))
 		):
 			raise Exception("This statement can cause implicit commit")
 
@@ -337,7 +342,7 @@ class Database(object):
 
 	@staticmethod
 	def clear_db_table_cache(query):
-		if query and query.strip().split()[0].lower() in {"drop", "create"}:
+		if query and is_query_type(query, ("drop", "create")):
 			frappe.cache().delete_key("db_tables")
 
 	@staticmethod
@@ -606,10 +611,13 @@ class Database(object):
 			else:
 				return r and [[i[1] for i in r]] or []
 
-	def get_singles_dict(self, doctype, debug=False, *, for_update=False):
+	def get_singles_dict(self, doctype, debug=False, *, for_update=False, cast=False):
 		"""Get Single DocType as dict.
 
 		:param doctype: DocType of the single object whose value is requested
+		:param debug: Execute query in debug mode - print to STDOUT
+		:param for_update: Take `FOR UPDATE` lock on the records
+		:param cast: Cast values to Python data types based on field type
 
 		Example:
 
@@ -621,9 +629,26 @@ class Database(object):
 			filters={"doctype": doctype},
 			fields=["field", "value"],
 			for_update=for_update,
-		).run()
+		).run(debug=debug)
 
-		return frappe._dict(queried_result)
+		if not cast:
+			return frappe._dict(queried_result)
+
+		try:
+			meta = frappe.get_meta(doctype)
+		except DoesNotExistError:
+			return frappe._dict(queried_result)
+
+		return_value = frappe._dict()
+
+		for fieldname, value in queried_result:
+			if df := meta.get_field(fieldname):
+				casted_value = cast_fieldtype(df.fieldtype, value)
+			else:
+				casted_value = value
+			return_value[fieldname] = casted_value
+
+		return return_value
 
 	@staticmethod
 	def get_all(*args, **kwargs):
@@ -686,7 +711,7 @@ class Database(object):
 				_("Invalid field name: {0}").format(frappe.bold(fieldname)), self.InvalidColumnName
 			)
 
-		val = cast(df.fieldtype, val)
+		val = cast_fieldtype(df.fieldtype, val)
 
 		self.value_cache[doctype][fieldname] = val
 
@@ -1191,7 +1216,7 @@ class Database(object):
 	def log_touched_tables(self, query, values=None):
 		if values:
 			query = frappe.safe_decode(self._cursor.mogrify(query, values))
-		if query.strip().lower().split()[0] in ("insert", "delete", "update", "alter", "drop", "rename"):
+		if is_query_type(query, ("insert", "delete", "update", "alter", "drop", "rename")):
 			# single_word_regex is designed to match following patterns
 			# `tabXxx`, tabXxx and "tabXxx"
 

frappe/database/postgres/database.py

@@ -403,13 +403,12 @@ def modify_query(query):
 
 
 def modify_values(values):
-	def stringify_value(value):
-		if isinstance(value, int):
+	def modify_value(value):
+		if isinstance(value, (list, tuple)):
+			value = tuple(modify_values(value))
+
+		elif isinstance(value, int):
 			value = str(value)
-		elif isinstance(value, float):
-			truncated_float = int(value)
-			if value == truncated_float:
-				value = str(truncated_float)
 
 		return value
 
@@ -418,14 +417,15 @@ def modify_values(values):
 
 	if isinstance(values, dict):
 		for k, v in values.items():
-			values[k] = stringify_value(v)
+			values[k] = modify_value(v)
 	elif isinstance(values, (tuple, list)):
 		new_values = []
 		for val in values:
-			new_values.append(stringify_value(val))
+			new_values.append(modify_value(val))
+
 		values = new_values
 	else:
-		values = stringify_value(values)
+		values = modify_value(values)
 
 	return values
 

frappe/desk/doctype/dashboard_chart/dashboard_chart.py

@@ -14,6 +14,7 @@ from frappe.model.naming import append_number_if_name_exists
 from frappe.modules.export_file import export_to_files
 from frappe.utils import cint, get_datetime, getdate, now_datetime, nowdate
 from frappe.utils.dashboard import cache_source
+from frappe.utils.data import format_date
 from frappe.utils.dateutils import (
 	get_dates_from_timegrain,
 	get_from_date_from_timespan,
@@ -221,13 +222,16 @@ def get_chart_config(chart, filters, timespan, timegrain, from_date, to_date):
 
 	result = get_result(data, timegrain, from_date, to_date, chart.chart_type)
 
-	chart_config = {
-		"labels": [get_period(r[0], timegrain) for r in result],
+	return {
+		"labels": [
+			format_date(get_period(r[0], timegrain))
+			if timegrain in ("Daily", "Weekly")
+			else get_period(r[0], timegrain)
+			for r in result
+		],
 		"datasets": [{"name": chart.name, "values": [r[1] for r in result]}],
 	}
 
-	return chart_config
-
 
 def get_heatmap_chart_config(chart, filters, heatmap_year):
 	aggregate_function = get_aggregate_function(chart.chart_type)