Merge branch 'develop' into get-distinct-records-from-backend

.github/workflows/docs-checker.yml

@@ -12,7 +12,7 @@ jobs:
       - name: 'Setup Environment'
         uses: actions/setup-python@v2
         with:
-          python-version: 3.6
+          python-version: 3.7
 
       - name: 'Clone repo'
         uses: actions/checkout@v2

.github/workflows/publish-assets-develop.yml

@@ -18,7 +18,7 @@ jobs:
           node-version: 14
       - uses: actions/setup-python@v2
         with:
-          python-version: '3.6'
+          python-version: '3.7'
       - name: Set up bench and build assets
         run: |
           npm install -g yarn

.github/workflows/publish-assets-releases.yml

@@ -21,7 +21,7 @@ jobs:
           python-version: '12.x'
       - uses: actions/setup-python@v2
         with:
-          python-version: '3.6'
+          python-version: '3.7'
       - name: Set up bench and build assets
         run: |
           npm install -g yarn

frappe/core/doctype/access_log/access_log.py

@@ -1,6 +1,7 @@
-# Copyright (c) 2019, Frappe Technologies and contributors
+# Copyright (c) 2021, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 import frappe
+from tenacity import retry, retry_if_exception_type, stop_after_attempt
 from frappe.model.document import Document
 
 
@@ -10,25 +11,40 @@ class AccessLog(Document):
 
 @frappe.whitelist()
 @frappe.write_only()
-def make_access_log(doctype=None, document=None, method=None, file_type=None,
-		report_name=None, filters=None, page=None, columns=None):
+@retry(
+	stop=stop_after_attempt(3), retry=retry_if_exception_type(frappe.DuplicateEntryError)
+)
+def make_access_log(
+	doctype=None,
+	document=None,
+	method=None,
+	file_type=None,
+	report_name=None,
+	filters=None,
+	page=None,
+	columns=None,
+):
 
 	user = frappe.session.user
+	in_request = frappe.request and frappe.request.method == "GET"
 
-	doc = frappe.get_doc({
-		'doctype': 'Access Log',
-		'user': user,
-		'export_from': doctype,
-		'reference_document': document,
-		'file_type': file_type,
-		'report_name': report_name,
-		'page': page,
-		'method': method,
-		'filters': frappe.utils.cstr(filters) if filters else None,
-		'columns': columns
-	})
+	doc = frappe.get_doc(
+		{
+			"doctype": "Access Log",
+			"user": user,
+			"export_from": doctype,
+			"reference_document": document,
+			"file_type": file_type,
+			"report_name": report_name,
+			"page": page,
+			"method": method,
+			"filters": frappe.utils.cstr(filters) if filters else None,
+			"columns": columns,
+		}
+	)
 	doc.insert(ignore_permissions=True)
 
 	# `frappe.db.commit` added because insert doesnt `commit` when called in GET requests like `printview`
-	if frappe.request and frappe.request.method == 'GET':
+	# dont commit in test mode
+	if not frappe.flags.in_test or in_request:
 		frappe.db.commit()

frappe/database/database.py

@@ -332,7 +332,7 @@ class Database(object):
 			values[key] = value
 			if isinstance(value, (list, tuple)):
 				# value is a tuple like ("!=", 0)
-				_operator = value[0]
+				_operator = value[0].lower()
 				values[key] = value[1]
 				if isinstance(value[1], (tuple, list)):
 					# value is a list in tuple ("in", ("A", "B"))

frappe/desk/doctype/workspace/workspace.json

@@ -165,8 +165,6 @@
    "default": "0",
    "fieldname": "is_standard",
    "fieldtype": "Check",
-   "in_list_view": 1,
-   "in_standard_filter": 1,
    "label": "Is Standard",
    "search_index": 1
   },
@@ -181,7 +179,6 @@
    "depends_on": "eval:doc.extends_another_page == 1 || doc.for_user",
    "fieldname": "extends",
    "fieldtype": "Link",
-   "in_standard_filter": 1,
    "label": "Extends",
    "options": "Workspace",
    "search_index": 1
@@ -228,6 +225,8 @@
    "default": "0",
    "fieldname": "public",
    "fieldtype": "Check",
+   "in_list_view": 1,
+   "in_standard_filter": 1,
    "label": "Public"
   },
   {
@@ -265,11 +264,13 @@
    "label": "Roles"
   }
  ],
+ "in_create": 1,
  "links": [],
- "modified": "2021-08-30 18:47:18.227154",
+ "modified": "2021-09-16 12:01:06.450621",
  "modified_by": "Administrator",
  "module": "Desk",
  "name": "Workspace",
+ "naming_rule": "By fieldname",
  "owner": "Administrator",
  "permissions": [
   {

frappe/desk/doctype/workspace/workspace.py

@@ -208,17 +208,17 @@ def save_page(title, icon, parent, public, sb_public_items, sb_private_items, de
 	if loads(deleted_pages):
 		return delete_pages(loads(deleted_pages))
 
-	return {"name": title, "public": public}
+	return {"name": title, "public": public, "label": doc.label}
 
 def delete_pages(deleted_pages):
 	for page in deleted_pages:
 		if page.get("public") and "Workspace Manager" not in frappe.get_roles():
-			return {"name": page.get("title"), "public": 1}
+			return {"name": page.get("title"), "public": 1, "label": page.get("label")}
 
 		if frappe.db.exists("Workspace", page.get("name")):
 			frappe.get_doc("Workspace", page.get("name")).delete(ignore_permissions=True)
 
-	return {"name": "Home", "public": 1}
+	return {"name": "Home", "public": 1, "label": "Home"}
 
 def sort_pages(sb_public_items, sb_private_items):
 	wspace_public_pages = get_page_list(['name', 'title'], {'public': 1})

frappe/model/db_query.py

@@ -4,6 +4,7 @@
 
 from typing import List
 import frappe.defaults
+from frappe.query_builder.utils import Column
 import frappe.share
 from frappe import _
 import frappe.permissions
@@ -491,7 +492,7 @@ class DatabaseQuery(object):
 				f.value = date_range
 				fallback = "'0001-01-01 00:00:00'"
 
-			if f.operator in ('>', '<') and (f.fieldname in ('creation', 'modified')):
+			if (f.fieldname in ('creation', 'modified')):
 				value = cstr(f.value)
 				fallback = "NULL"
 
@@ -547,8 +548,12 @@ class DatabaseQuery(object):
 				value = flt(f.value)
 				fallback = 0
 
+			if isinstance(f.value, Column):
+				quote = '"' if frappe.conf.db_type == 'postgres' else "`"
+				value = f"{tname}.{quote}{f.value.name}{quote}"
+
 			# escape value
-			if isinstance(value, str) and not f.operator.lower() == 'between':
+			elif isinstance(value, str) and not f.operator.lower() == 'between':
 				value = f"{frappe.db.escape(value, percent=False)}"
 
 		if (

frappe/public/js/frappe/form/grid.js

@@ -212,13 +212,12 @@ export default class Grid {
 
 	delete_all_rows() {
 		frappe.confirm(__("Are you sure you want to delete all rows?"), () => {
-			this.grid_rows.forEach(row => {
-				row.remove();
-			});
-			this.frm.script_manager.trigger(this.df.fieldname + "_delete", this.doctype);
-
-			this.wrapper.find('.grid-heading-row .grid-row-check:checked:first').prop('checked', 0);
+			this.frm.doc[this.df.fieldname] = [];
+			$(this.parent).find('.rows').empty();
+			this.grid_rows = [];
 			this.refresh();
+			this.frm && this.frm.script_manager.trigger(this.df.fieldname + "_delete", this.doctype);
+			this.frm && this.frm.dirty();
 			this.scroll_to_top();
 		});
 	}
@@ -244,8 +243,10 @@ export default class Grid {
 
 		this.remove_rows_button.toggleClass('hidden',
 			this.wrapper.find('.grid-body .grid-row-check:checked:first').length ? false : true);
-		this.remove_all_rows_button.toggleClass('hidden',
-			this.wrapper.find('.grid-heading-row .grid-row-check:checked:first').length ? false : true);
+
+		let select_all_checkbox_checked = this.wrapper.find('.grid-heading-row .grid-row-check:checked:first').length;
+		let show_delete_all_btn = select_all_checkbox_checked && this.data.length > this.get_selected_children().length;
+		this.remove_all_rows_button.toggleClass('hidden', !show_delete_all_btn);
 	}
 
 	get_selected() {
@@ -835,10 +836,11 @@ export default class Grid {
 									$.each(row, (ci, value) => {
 										var fieldname = fieldnames[ci];
 										var df = frappe.meta.get_docfield(me.df.options, fieldname);
-
-										d[fieldnames[ci]] = value_formatter_map[df.fieldtype]
-											? value_formatter_map[df.fieldtype](value)
-											: value;
+										if (df) {
+											d[fieldnames[ci]] = value_formatter_map[df.fieldtype]
+												? value_formatter_map[df.fieldtype](value)
+												: value;
+										}
 									});
 								}
 							}

frappe/public/js/frappe/form/grid_row_form.js

@@ -123,10 +123,12 @@ export default class GridRowForm {
 			.toggle(this.row.grid.is_editable());
 	}
 	refresh_field(fieldname) {
-		if(this.fields_dict[fieldname]) {
-			this.fields_dict[fieldname].refresh();
-			this.layout && this.layout.refresh_dependency();
-		}
+		const field = this.fields_dict[fieldname];
+		if (!field) return;
+
+		field.docname = this.row.doc.name;
+		field.refresh();
+		this.layout && this.layout.refresh_dependency();
 	}
 	set_focus() {
 		// wait for animation and then focus on the first row

frappe/public/js/frappe/list/list_view.js

@@ -607,9 +607,9 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList {
 
 		const subject_field = this.columns[0].df;
 		let subject_html = `
-			<input class="level-item list-check-all hidden-xs" type="checkbox"
+			<input class="level-item list-check-all" type="checkbox"
 				title="${__("Select All")}">
-			<span class="level-item list-liked-by-me">
+			<span class="level-item list-liked-by-me hidden-xs">
 				<span title="${__("Likes")}">${frappe.utils.icon('heart', 'sm', 'like-icon')}</span>
 			</span>
 			<span class="level-item">${__(subject_field.label)}</span>
@@ -646,7 +646,7 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList {
 				</div>
 				<div class="level-left checkbox-actions">
 					<div class="level list-subject">
-						<input class="level-item list-check-all hidden-xs" type="checkbox"
+						<input class="level-item list-check-all" type="checkbox"
 							title="${__("Select All")}">
 						<span class="level-item list-header-meta"></span>
 					</div>
@@ -954,9 +954,9 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList {
 
 		let subject_html = `
 			<span class="level-item select-like">
-				<input class="list-row-checkbox hidden-xs" type="checkbox"
+				<input class="list-row-checkbox" type="checkbox"
 					data-name="${escape(doc.name)}">
-				<span class="list-row-like style="margin-bottom: 1px;">
+				<span class="list-row-like hidden-xs style="margin-bottom: 1px;">
 					${this.get_like_html(doc)}
 				</span>
 			</span>
@@ -1163,6 +1163,7 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList {
 			if (
 				$target.hasClass("filterable") ||
 				$target.hasClass("select-like") ||
+				$target.hasClass("file-select") ||
 				$target.hasClass("list-row-like") ||
 				$target.is(":checkbox")
 			) {

frappe/public/js/frappe/utils/utils.js

@@ -927,7 +927,16 @@ Object.assign(frappe.utils, {
 		// decodes base64 to string
 		let parts = dataURI.split(',');
 		const encoded_data = parts[1];
-		return decodeURIComponent(escape(atob(encoded_data)));
+		let decoded = atob(encoded_data);
+		try {
+			const escaped = escape(decoded);
+			decoded = decodeURIComponent(escaped);
+
+		} catch (e) {
+			// pass decodeURIComponent failure
+			// just return atob response
+		}
+		return decoded;
 	},
 	copy_to_clipboard(string) {
 		let input = $("<input>");

frappe/public/js/frappe/views/file/file_view.js

@@ -380,8 +380,10 @@ frappe.views.FileView = class FileView extends frappe.views.ListView {
 
 		return `
 			<div class="list-row-col ellipsis list-subject level">
-				<input class="level-item list-row-checkbox hidden-xs"
-					type="checkbox" data-name="${file.name}">
+				<span class="level-item file-select">
+					<input class="list-row-checkbox hidden-xs"
+						type="checkbox" data-name="${file.name}">
+				</span>
 				<span class="level-item  ellipsis" title="${file.file_name}">
 					<a class="ellipsis" href="${route_url}" title="${file.file_name}">
 						${file.subject_html}

frappe/public/js/frappe/views/workspace/workspace.js

@@ -23,6 +23,7 @@ frappe.views.Workspace = class Workspace {
 		this.blocks = frappe.wspace_block.blocks;
 		this.is_read_only = true;
 		this.new_page = null;
+		this.pages = {};
 		this.sorted_public_items = [];
 		this.sorted_private_items = [];
 		this.deleted_sidebar_items = [];
@@ -35,42 +36,6 @@ frappe.views.Workspace = class Workspace {
 			'My Workspaces',
 			'Public'
 		];
-		this.tools = {
-			header: {
-				class: this.blocks['header'],
-				inlineToolbar: true
-			},
-			paragraph: {
-				class: this.blocks['paragraph'],
-				inlineToolbar: true
-			},
-			chart: {
-				class: this.blocks['chart'],
-				config: {
-					page_data: this.page_data || []
-				}
-			},
-			card: {
-				class: this.blocks['card'],
-				config: {
-					page_data: this.page_data || []
-				}
-			},
-			shortcut: {
-				class: this.blocks['shortcut'],
-				config: {
-					page_data: this.page_data || []
-				}
-			},
-			onboarding: {
-				class: this.blocks['onboarding'],
-				config: {
-					page_data: this.page_data || []
-				}
-			},
-			spacer: this.blocks['spacer'],
-			spacingTune: frappe.wspace_block.tunes['spacing_tune'],
-		};
 
 		this.prepare_container();
 		this.setup_pages();
@@ -86,7 +51,7 @@ frappe.views.Workspace = class Workspace {
 		this.body = this.wrapper.find(".layout-main-section");
 	}
 
-	setup_pages() {
+	setup_pages(reload) {
 		this.get_pages().then(pages => {
 			this.all_pages = pages.pages;
 			this.has_access = pages.has_access;
@@ -115,7 +80,7 @@ frappe.views.Workspace = class Workspace {
 					this.new_page = null;
 				}
 				this.make_sidebar();
-				frappe.router.route();
+				reload && this.show();
 			}
 		});
 	}
@@ -236,10 +201,7 @@ frappe.views.Workspace = class Workspace {
 			return;
 		}
 
-		let page = {
-			name: this.get_page_to_show().name,
-			public: this.get_page_to_show().public
-		};
+		let page = this.get_page_to_show();
 		this.page.set_title(`${__(page.name)}`);
 
 		this.show_page(page);
@@ -250,6 +212,11 @@ frappe.views.Workspace = class Workspace {
 			page: page
 		}).then(data => {
 			this.page_data = data;
+
+			// caching page data
+			this.pages[page.name] && delete this.pages[page.name];
+			this.pages[page.name] = data;
+
 			if (!this.page_data || Object.keys(this.page_data).length === 0) return;
 
 			return frappe.dashboard_utils.get_dashboard_settings().then(settings => {
@@ -260,6 +227,7 @@ frappe.views.Workspace = class Workspace {
 							chart.chart_settings = chart_config[chart.chart_name] || {};
 						});
 					}
+					this.pages[page.name] = this.page_data;
 				}
 			});
 		});
@@ -281,7 +249,7 @@ frappe.views.Workspace = class Workspace {
 		return { name: page, public: is_public };
 	}
 
-	show_page(page) {
+	async show_page(page) {
 		let section = this.current_page.public ? 'public' : 'private';
 		if (this.sidebar_items && this.sidebar_items[section] && this.sidebar_items[section][this.current_page.name]) {
 			this.sidebar_items[section][this.current_page.name][0].firstElementChild.classList.remove("selected");
@@ -316,12 +284,17 @@ frappe.views.Workspace = class Workspace {
 			this.add_custom_cards_in_content();
 
 			$('.item-anchor').addClass('disable-click');
-			this.get_data(this_page).then(() => {
-				this.prepare_editorjs();
-				$('.item-anchor').removeClass('disable-click');
-				this.$page.find('.codex-editor').removeClass('hidden');
-				this.$page.find('.workspace-skeleton').remove();
-			});
+
+			if (this.pages && this.pages[this_page.name]) {
+				this.page_data = this.pages[this_page.name];
+			} else {
+				await this.get_data(this_page);
+			}
+
+			this.prepare_editorjs();
+			$('.item-anchor').removeClass('disable-click');
+			this.$page.find('.codex-editor').removeClass('hidden');
+			this.$page.find('.workspace-skeleton').remove();
 		}
 	}
 
@@ -652,7 +625,7 @@ frappe.views.Workspace = class Workspace {
 		let $sidebar_section = is_public ? $sidebar[1] : $sidebar[0];
 
 		if (!parent) {
-			!is_public && $sidebar.last().removeClass('hidden');
+			!is_public && $sidebar.first().removeClass('hidden');
 			$sidebar_item.appendTo($sidebar_section);
 		} else {
 			let $item_container = $($sidebar_section).find(`[item-name="${parent}"]`);
@@ -670,6 +643,42 @@ frappe.views.Workspace = class Workspace {
 	}
 
 	initialize_editorjs(blocks) {
+		this.tools = {
+			header: {
+				class: this.blocks['header'],
+				inlineToolbar: true
+			},
+			paragraph: {
+				class: this.blocks['paragraph'],
+				inlineToolbar: true
+			},
+			chart: {
+				class: this.blocks['chart'],
+				config: {
+					page_data: this.page_data || []
+				}
+			},
+			card: {
+				class: this.blocks['card'],
+				config: {
+					page_data: this.page_data || []
+				}
+			},
+			shortcut: {
+				class: this.blocks['shortcut'],
+				config: {
+					page_data: this.page_data || []
+				}
+			},
+			onboarding: {
+				class: this.blocks['onboarding'],
+				config: {
+					page_data: this.page_data || []
+				}
+			},
+			spacer: this.blocks['spacer'],
+			spacingTune: frappe.wspace_block.tunes['spacing_tune'],
+		};
 		this.editor = new EditorJS({
 			data: {
 				blocks: blocks || []
@@ -730,6 +739,7 @@ frappe.views.Workspace = class Workspace {
 					frappe.dom.unfreeze();
 					if (res.message) {
 						me.new_page = res.message;
+						me.pages[res.message.label] && delete me.pages[res.message.label];
 						me.title = '';
 						me.icon = '';
 						me.parent = '';
@@ -751,7 +761,7 @@ frappe.views.Workspace = class Workspace {
 	reload() {
 		this.$page.prepend(frappe.render_template('workspace_loading_skeleton'));
 		this.$page.find('.codex-editor').addClass('hidden');
-		this.setup_pages();
+		this.setup_pages(true);
 		this.undo.readOnly = true;
 	}
 };

frappe/public/scss/desk/desktop.scss

@@ -886,6 +886,10 @@ body {
 		}
 	}
 
+	.codex-editor__loader {
+		display: none !important;
+	}
+
 	.codex-editor {
 		min-height: 630px;
 

frappe/public/scss/desk/list.scss

@@ -140,7 +140,7 @@
 		}
 	}
 
-	.select-like {
+	.select-like, .file-select {
 		padding: 15px 0px 15px 15px;
 	}
 }
@@ -178,7 +178,7 @@ $level-margin-right: 8px;
 		color: var(--text-color);
 	}
 
-	.level-item {
+	.level-item:not(.file-select) {
 		margin-right: $level-margin-right;
 	}
 

frappe/public/scss/desk/mobile.scss

@@ -202,8 +202,12 @@ body {
 	}
 
 	// listviews
-	.list-row {
-		padding: 13px 15px !important;
+	.select-like {
+		margin-right: unset !important;
+	}
+
+	.list-count {
+		display: contents;
 	}
 
 	.doclist-row {

frappe/public/scss/website/page_builder.scss

@@ -486,9 +486,12 @@
 }
 
 .collapsible-content {
+	color: $gray-700;
+}
+
+.collapsible-content p {
 	margin-top: 1rem;
 	margin-bottom: 0;
-	color: $gray-700;
 }
 
 .section-with-collapsible-content.align-center {

frappe/query_builder/__init__.py

@@ -1,2 +1,2 @@
 from pypika import *
-from frappe.query_builder.utils import get_query_builder, patch_query_execute
+from frappe.query_builder.utils import Column, get_query_builder, patch_query_execute

frappe/query_builder/utils.py

@@ -3,8 +3,10 @@ from typing import Any, Callable, Dict, get_type_hints
 from importlib import import_module
 
 from pypika import Query
+from pypika.queries import Column
 
 import frappe
+
 from .builder import MariaDB, Postgres
 
 

frappe/search/full_text_search.py

@@ -8,6 +8,8 @@ from whoosh.index import create_in, open_dir, EmptyIndexError
 from whoosh.fields import TEXT, ID, Schema
 from whoosh.qparser import MultifieldParser, FieldsPlugin, WildcardPlugin
 from whoosh.query import Prefix
+from whoosh.writing import AsyncWriter
+
 
 class FullTextSearch:
 	""" Frappe Wrapper for Whoosh """
@@ -75,7 +77,7 @@ class FullTextSearch:
 		ix = self.get_index()
 
 		with ix.searcher():
-			writer = ix.writer()
+			writer = AsyncWriter(ix)
 			writer.delete_by_term(self.id, document[self.id])
 			writer.add_document(**document)
 			writer.commit(optimize=True)
@@ -135,4 +137,4 @@ class FullTextSearch:
 		return out
 
 def get_index_path(index_name):
-	return frappe.get_site_path("indexes", index_name)
+	return frappe.get_site_path("indexes", index_name)

frappe/tests/test_db_query.py

@@ -4,6 +4,7 @@ import frappe, unittest
 
 from frappe.model.db_query import DatabaseQuery
 from frappe.desk.reportview import get_filters_cond
+from frappe.query_builder import Column
 
 from frappe.core.page.permission_manager.permission_manager import update, reset, add
 from frappe.permissions import add_user_permission, clear_user_permissions_for_doctype
@@ -373,6 +374,25 @@ class TestReportview(unittest.TestCase):
 		owners = DatabaseQuery("DocType").execute(filters={"name": "DocType"}, pluck="owner")
 		self.assertEqual(owners, ["Administrator"])
 
+	def test_column_comparison(self):
+		"""Test DatabaseQuery.execute to test column comparison
+		"""
+		users_unedited = frappe.get_all(
+			"User",
+			filters={"creation": Column("modified")},
+			fields=["name", "creation", "modified"],
+			limit=1,
+		)
+		users_edited = frappe.get_all(
+			"User",
+			filters={"creation": ("!=", Column("modified"))},
+			fields=["name", "creation", "modified"],
+			limit=1,
+		)
+
+		self.assertEqual(users_unedited[0].modified, users_unedited[0].creation)
+		self.assertNotEqual(users_edited[0].modified, users_edited[0].creation)
+
 	def test_reportview_get(self):
 		user = frappe.get_doc("User", "test@example.com")
 		add_child_table_to_blog_post()

frappe/website/web_template/section_with_collapsible_content/section_with_collapsible_content.html

@@ -21,7 +21,9 @@
 				</svg>
 			</a>
 			<div class="collapse collapsible-content from-markdown" id="{{ collapse_id }}">
-				{{ frappe.utils.md_to_html(item.content) }}
+				<div>
+					{{ frappe.utils.md_to_html(item.content) }}
+				</div>
 			</div>
 		</div>
 		{%- endfor -%}

requirements.txt

@@ -24,8 +24,7 @@ googlemaps~=4.4.5
 gunicorn~=20.1.0
 html2text==2020.1.16
 html5lib~=1.1
-ipython~=7.16.1
-jedi==0.17.2  # not directly required. Pinned to fix upstream IPython issue (https://github.com/ipython/ipython/issues/12740)
+ipython~=7.27.0
 Jinja2~=3.0.1
 ldap3~=2.9
 markdown2~=2.4.0

setup.py

@@ -57,5 +57,5 @@ setup(
 	{
 		'clean': CleanCommand
 	},
-	python_requires='>=3.6'
+	python_requires='>=3.7'
 )