From c61ff3bced267ccd37c9f0d4b16c1dd32cd822dd Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Fri, 30 Aug 2019 19:25:36 +0530 Subject: [PATCH 01/32] fix: Remove NestedSet from File Pros: - Multiple file uploads wont result in a deadlock Cons: - Folder size cannot be computed --- frappe/core/doctype/file/file.py | 37 ++----------------- .../public/js/frappe/views/file/file_view.js | 2 +- 2 files changed, 4 insertions(+), 35 deletions(-) diff --git a/frappe/core/doctype/file/file.py b/frappe/core/doctype/file/file.py index 847e0cd267..71c4fcd965 100755 --- a/frappe/core/doctype/file/file.py +++ b/frappe/core/doctype/file/file.py @@ -26,6 +26,7 @@ from frappe.utils import get_hook_method, get_files_path, random_string, encode, from frappe import _ from frappe import conf from frappe.utils.nestedset import NestedSet +from frappe.model.document import Document from frappe.utils import strip from PIL import Image, ImageOps from six import StringIO, string_types @@ -42,8 +43,7 @@ class FolderNotEmpty(frappe.ValidationError): pass exclude_from_linked_with = True -class File(NestedSet): - nsm_parent_field = 'folder' +class File(Document): no_feed_on_delete = True def before_insert(self): @@ -72,8 +72,6 @@ class File(NestedSet): self.name = frappe.generate_hash("", 10) def after_insert(self): - self.update_parent_folder_size() - if not self.is_folder: self.add_comment_in_reference_doc('Attachment', _('Added {0}').format("{file_name}{icon}".format(**{ @@ -100,7 +98,6 @@ class File(NestedSet): self.validate_file() self.generate_content_hash() - self.set_folder_size() self.validate_url() if frappe.db.exists('File', {'name': self.name, 'is_folder': 0}): @@ -136,31 +133,6 @@ class File(NestedSet): frappe.db.set_value(self.attached_to_doctype, self.attached_to_name, self.attached_to_field, self.file_url) - - def set_folder_size(self): - """Set folder size if folder""" - if self.is_folder and not self.is_new(): - self.file_size = cint(self.get_folder_size()) - self.db_set('file_size', self.file_size) - - for folder in self.get_ancestors(): - frappe.db.set_value("File", folder, "file_size", self.get_folder_size(folder)) - - def get_folder_size(self, folder=None): - """Returns folder size for current folder""" - if not folder: - folder = self.name - - file_size = frappe.db.sql("""select ifnull(sum(file_size), 0) - from tabFile where folder=%s """, (folder))[0][0] - - return file_size - - def update_parent_folder_size(self): - """Update size of parent folder""" - if self.folder and not self.is_folder: # it not home - frappe.get_doc("File", self.folder).set_folder_size() - def set_folder_name(self): """Make parent folders if not exists based on reference doctype and name""" if self.attached_to_doctype and not self.folder: @@ -225,7 +197,7 @@ class File(NestedSet): if self.is_home_folder or self.is_attachments_folder: frappe.throw(_("Cannot delete Home and Attachments folders")) self.check_folder_is_empty() - super(File, self).on_trash() + # super(File, self).on_trash() self.call_delete_file() if not self.is_folder: self.add_comment_in_reference_doc('Attachment Removed', _("Removed {0}").format(self.file_name)) @@ -267,9 +239,6 @@ class File(NestedSet): return thumbnail_url - def after_delete(self): - self.update_parent_folder_size() - def check_folder_is_empty(self): """Throw exception if folder is not empty""" files = frappe.get_all("File", filters={"folder": self.name}, fields=("name", "file_name")) diff --git a/frappe/public/js/frappe/views/file/file_view.js b/frappe/public/js/frappe/views/file/file_view.js index 6ee37aec3d..5f9ca642f4 100644 --- a/frappe/public/js/frappe/views/file/file_view.js +++ b/frappe/public/js/frappe/views/file/file_view.js @@ -259,7 +259,7 @@ frappe.views.FileView = class FileView extends frappe.views.ListView { get_left_html(file) { file = this.prepare_datum(file); - const file_size = frappe.form.formatters.FileSize(file.file_size); + const file_size = file.file_size ? frappe.form.formatters.FileSize(file.file_size) : ''; const route_url = this.get_route_url(file); let created_on; From 92f8b6ac1b1e94081b0ae889c1bf25e59912c696 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Sun, 1 Sep 2019 12:49:13 +0530 Subject: [PATCH 02/32] fix: Remove lft and rgt --- frappe/core/doctype/file/file.json | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/frappe/core/doctype/file/file.json b/frappe/core/doctype/file/file.json index 9bf7d03512..d9ab504db7 100644 --- a/frappe/core/doctype/file/file.json +++ b/frappe/core/doctype/file/file.json @@ -22,8 +22,6 @@ "column_break_10", "attached_to_name", "attached_to_field", - "lft", - "rgt", "old_parent", "content_hash", "uploaded_to_dropbox", @@ -145,18 +143,6 @@ "label": "Attached To Field", "read_only": 1 }, - { - "fieldname": "lft", - "fieldtype": "Int", - "hidden": 1, - "label": "lft" - }, - { - "fieldname": "rgt", - "fieldtype": "Int", - "hidden": 1, - "label": "rgt" - }, { "fieldname": "old_parent", "fieldtype": "Data", @@ -186,7 +172,7 @@ ], "icon": "fa fa-file", "idx": 1, - "modified": "2019-08-16 16:41:03.086023", + "modified": "2019-08-30 19:46:20.796453", "modified_by": "Administrator", "module": "Core", "name": "File", From db305d75687d0c6a2f270d67bcb4b8ac8940114a Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Sun, 1 Sep 2019 12:49:17 +0530 Subject: [PATCH 03/32] fix: File tests --- frappe/core/doctype/file/file.py | 15 +++++++++------ frappe/core/doctype/file/test_file.py | 13 +++---------- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/frappe/core/doctype/file/file.py b/frappe/core/doctype/file/file.py index 71c4fcd965..e324a0d879 100755 --- a/frappe/core/doctype/file/file.py +++ b/frappe/core/doctype/file/file.py @@ -555,7 +555,6 @@ class File(Document): def on_doctype_update(): frappe.db.add_index("File", ["attached_to_doctype", "attached_to_name"]) - frappe.db.add_index("File", ["lft", "rgt"]) def make_home_folder(): home = frappe.get_doc({ @@ -576,12 +575,16 @@ def make_home_folder(): @frappe.whitelist() def get_breadcrumbs(folder): """returns name, file_name of parent folder""" - values = frappe.db.get_value("File", folder, ["lft", "rgt"], as_dict=True) - if not values: - frappe.throw(_("Folder {0} does not exist").format(folder)) + path = folder.split('/') - return frappe.db.sql("""select name, file_name from tabFile - where lft < %s and rgt > %s order by lft asc""", (values.lft, values.rgt), as_dict=1) + folders = [] + for i, p in enumerate(path): + indexes = range(0, i) + folder = '/'.join([path[i] for i in indexes]) + if folder: + folders.append(folder) + + return [frappe._dict(file_name=f) for f in folders] @frappe.whitelist() def create_new_folder(file_name, folder): diff --git a/frappe/core/doctype/file/test_file.py b/frappe/core/doctype/file/test_file.py index e9bf999bec..de97fbc162 100644 --- a/frappe/core/doctype/file/test_file.py +++ b/frappe/core/doctype/file/test_file.py @@ -171,7 +171,7 @@ class TestFile(unittest.TestCase): def delete_test_data(self): for f in frappe.db.sql('''select name, file_name from tabFile where - is_home_folder = 0 and is_attachments_folder = 0 order by rgt-lft asc'''): + is_home_folder = 0 and is_attachments_folder = 0 order by creation desc'''): frappe.delete_doc("File", f[0]) @@ -200,11 +200,8 @@ class TestFile(unittest.TestCase): def tests_after_upload(self): self.assertEqual(self.saved_folder, _("Home/Test Folder 1")) - - folder_size = frappe.db.get_value("File", _("Home/Test Folder 1"), "file_size") - saved_file_size = frappe.db.get_value("File", self.saved_name, "file_size") - - self.assertEqual(folder_size, saved_file_size) + file_folder = frappe.db.get_value("File", self.saved_name, "folder") + self.assertEqual(file_folder, _("Home/Test Folder 1")) def test_file_copy(self): @@ -215,8 +212,6 @@ class TestFile(unittest.TestCase): file = frappe.get_doc("File", {"file_name": "file_copy.txt"}) self.assertEqual(_("Home/Test Folder 2"), file.folder) - self.assertEqual(frappe.db.get_value("File", _("Home/Test Folder 2"), "file_size"), file.file_size) - self.assertEqual(frappe.db.get_value("File", _("Home/Test Folder 1"), "file_size"), 0) def test_folder_copy(self): @@ -239,8 +234,6 @@ class TestFile(unittest.TestCase): frappe.get_doc("File", file_copy_txt).delete() self.assertEqual(_("Home/Test Folder 1/Test Folder 3"), file.folder) - self.assertEqual(frappe.db.get_value("File", _("Home/Test Folder 1"), "file_size"), file.file_size) - self.assertEqual(frappe.db.get_value("File", _("Home/Test Folder 2"), "file_size"), 0) def test_default_folder(self): From 8676de02a17533d13ab645fecf814055f8c58c51 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Sun, 1 Sep 2019 12:50:30 +0530 Subject: [PATCH 04/32] fix: Remove commented call --- frappe/core/doctype/file/file.py | 1 - 1 file changed, 1 deletion(-) diff --git a/frappe/core/doctype/file/file.py b/frappe/core/doctype/file/file.py index e324a0d879..b9973d848b 100755 --- a/frappe/core/doctype/file/file.py +++ b/frappe/core/doctype/file/file.py @@ -197,7 +197,6 @@ class File(Document): if self.is_home_folder or self.is_attachments_folder: frappe.throw(_("Cannot delete Home and Attachments folders")) self.check_folder_is_empty() - # super(File, self).on_trash() self.call_delete_file() if not self.is_folder: self.add_comment_in_reference_doc('Attachment Removed', _("Removed {0}").format(self.file_name)) From 4abb057a53ad260ca8130a5a26bf811f57da48e8 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Tue, 3 Sep 2019 13:43:29 +0530 Subject: [PATCH 05/32] fix: Nested Set test cases --- frappe/tests/test_db_query.py | 201 ++++++++++++++++++---------------- 1 file changed, 108 insertions(+), 93 deletions(-) diff --git a/frappe/tests/test_db_query.py b/frappe/tests/test_db_query.py index b1b12f642e..28849618ee 100644 --- a/frappe/tests/test_db_query.py +++ b/frappe/tests/test_db_query.py @@ -185,37 +185,29 @@ class TestReportview(unittest.TestCase): self.assertTrue('date_diff' in data[0]) def test_nested_permission(self): - clear_user_permissions_for_doctype("File") - delete_test_file_hierarchy() # delete already existing folders - from frappe.core.doctype.file.file import create_new_folder frappe.set_user('Administrator') - - create_new_folder('level1-A', 'Home') - create_new_folder('level2-A', 'Home/level1-A') - create_new_folder('level2-B', 'Home/level1-A') - create_new_folder('level3-A', 'Home/level1-A/level2-A') - - create_new_folder('level1-B', 'Home') - create_new_folder('level2-A', 'Home/level1-B') + create_nested_doctype() + create_nested_doctype_records() + clear_user_permissions_for_doctype('Nested DocType') # user permission for only one root folder - add_user_permission('File', 'Home/level1-A', 'test2@example.com') + add_user_permission('Nested DocType', 'Level 1 A', 'test2@example.com') from frappe.core.page.permission_manager.permission_manager import update - update('File', 'All', 0, 'if_owner', 0) # to avoid if_owner filter + # to avoid if_owner filter + update('Nested DocType', 'All', 0, 'if_owner', 0) frappe.set_user('test2@example.com') - data = DatabaseQuery("File").execute() + data = DatabaseQuery('Nested DocType').execute() # children of root folder (for which we added user permission) should be accessible - self.assertTrue({"name": "Home/level1-A/level2-A"} in data) - self.assertTrue({"name": "Home/level1-A/level2-B"} in data) - self.assertTrue({"name": "Home/level1-A/level2-A/level3-A"} in data) + self.assertTrue({'name': 'Level 2 A'} in data) + self.assertTrue({'name': 'Level 2 A'} in data) # other folders should not be accessible - self.assertFalse({"name": "Home/level1-B"} in data) - self.assertFalse({"name": "Home/level1-B/level2-B"} in data) - update('File', 'All', 0, 'if_owner', 1) + self.assertFalse({'name': 'Level 1 B'} in data) + self.assertFalse({'name': 'Level 2 B'} in data) + update('Nested DocType', 'All', 0, 'if_owner', 1) frappe.set_user('Administrator') def test_filter_sanitizer(self): @@ -259,83 +251,75 @@ class TestReportview(unittest.TestCase): self.assertTrue('DefaultValue' in [d['name'] for d in out]) def test_of_not_of_descendant_ancestors(self): - clear_user_permissions_for_doctype("File") - delete_test_file_hierarchy() # delete already existing folders - from frappe.core.doctype.file.file import create_new_folder - - create_new_folder('level1-A', 'Home') - create_new_folder('level2-A', 'Home/level1-A') - create_new_folder('level2-B', 'Home/level1-A') - create_new_folder('level3-A', 'Home/level1-A/level2-A') - - create_new_folder('level1-B', 'Home') - create_new_folder('level2-A', 'Home/level1-B') + create_nested_doctype() + create_nested_doctype_records() + clear_user_permissions_for_doctype('Nested DocType') # in descendants filter - data = frappe.get_all('File', {'name': ('descendants of', 'Home/level1-A/level2-A')}) - self.assertTrue({"name": "Home/level1-A/level2-A/level3-A"} in data) + data = frappe.get_all('Nested DocType', {'name': ('descendants of', 'Level 2 A')}) + self.assertTrue({"name": "Level 3 A"} in data) - data = frappe.get_all('File', {'name': ('descendants of', 'Home/level1-A')}) - self.assertTrue({"name": "Home/level1-A/level2-A/level3-A"} in data) - self.assertTrue({"name": "Home/level1-A/level2-A"} in data) - self.assertTrue({"name": "Home/level1-A/level2-B"} in data) - self.assertFalse({"name": "Home/level1-B"} in data) - self.assertFalse({"name": "Home/level1-A"} in data) - self.assertFalse({"name": "Home"} in data) + data = frappe.get_all('Nested DocType', {'name': ('descendants of', 'Level 1 A')}) + self.assertTrue({"name": "Level 3 A"} in data) + self.assertTrue({"name": "Level 2 A"} in data) + self.assertFalse({"name": "Level 2 B"} in data) + self.assertFalse({"name": "Level 1 B"} in data) + self.assertFalse({"name": "Level 1 A"} in data) + self.assertFalse({"name": "Root"} in data) # in ancestors of filter - data = frappe.get_all('File', {'name': ('ancestors of', 'Home/level1-A/level2-A')}) - self.assertFalse({"name": "Home/level1-A/level2-A/level3-A"} in data) - self.assertFalse({"name": "Home/level1-A/level2-A"} in data) - self.assertFalse({"name": "Home/level1-A/level2-B"} in data) - self.assertFalse({"name": "Home/level1-B"} in data) - self.assertTrue({"name": "Home/level1-A"} in data) - self.assertTrue({"name": "Home"} in data) + data = frappe.get_all('Nested DocType', {'name': ('ancestors of', 'Level 2 A')}) + self.assertFalse({"name": "Level 3 A"} in data) + self.assertFalse({"name": "Level 2 A"} in data) + self.assertFalse({"name": "Level 2 B"} in data) + self.assertFalse({"name": "Level 1 B"} in data) + self.assertTrue({"name": "Level 1 A"} in data) + self.assertTrue({"name": "Root"} in data) - data = frappe.get_all('File', {'name': ('ancestors of', 'Home/level1-A')}) - self.assertFalse({"name": "Home/level1-A/level2-A/level3-A"} in data) - self.assertFalse({"name": "Home/level1-A/level2-A"} in data) - self.assertFalse({"name": "Home/level1-A/level2-B"} in data) - self.assertFalse({"name": "Home/level1-B"} in data) - self.assertFalse({"name": "Home/level1-A"} in data) - self.assertTrue({"name": "Home"} in data) + data = frappe.get_all('Nested DocType', {'name': ('ancestors of', 'Level 1 A')}) + self.assertFalse({"name": "Level 3 A"} in data) + self.assertFalse({"name": "Level 2 A"} in data) + self.assertFalse({"name": "Level 2 B"} in data) + self.assertFalse({"name": "Level 1 B"} in data) + self.assertFalse({"name": "Level 1 A"} in data) + self.assertTrue({"name": "Root"} in data) # not descendants filter - data = frappe.get_all('File', {'name': ('not descendants of', 'Home/level1-A/level2-A')}) - self.assertFalse({"name": "Home/level1-A/level2-A/level3-A"} in data) - self.assertTrue({"name": "Home/level1-A/level2-A"} in data) - self.assertTrue({"name": "Home/level1-A/level2-B"} in data) - self.assertTrue({"name": "Home/level1-A"} in data) - self.assertTrue({"name": "Home"} in data) + data = frappe.get_all('Nested DocType', {'name': ('not descendants of', 'Level 2 A')}) + self.assertFalse({"name": "Level 3 A"} in data) + self.assertTrue({"name": "Level 2 A"} in data) + self.assertTrue({"name": "Level 2 B"} in data) + self.assertTrue({"name": "Level 1 A"} in data) + self.assertTrue({"name": "Root"} in data) - data = frappe.get_all('File', {'name': ('not descendants of', 'Home/level1-A')}) - self.assertFalse({"name": "Home/level1-A/level2-A/level3-A"} in data) - self.assertFalse({"name": "Home/level1-A/level2-A"} in data) - self.assertFalse({"name": "Home/level1-A/level2-B"} in data) - self.assertTrue({"name": "Home/level1-B"} in data) - self.assertTrue({"name": "Home/level1-A"} in data) - self.assertTrue({"name": "Home"} in data) + data = frappe.get_all('Nested DocType', {'name': ('not descendants of', 'Level 1 A')}) + self.assertFalse({"name": "Level 3 A"} in data) + self.assertFalse({"name": "Level 2 A"} in data) + self.assertTrue({"name": "Level 2 B"} in data) + self.assertTrue({"name": "Level 1 B"} in data) + self.assertTrue({"name": "Level 1 A"} in data) + self.assertTrue({"name": "Root"} in data) # not ancestors of filter - data = frappe.get_all('File', {'name': ('not ancestors of', 'Home/level1-A/level2-A')}) - self.assertTrue({"name": "Home/level1-A/level2-A/level3-A"} in data) - self.assertTrue({"name": "Home/level1-A/level2-A"} in data) - self.assertTrue({"name": "Home/level1-A/level2-B"} in data) - self.assertTrue({"name": "Home/level1-B"} in data) - self.assertTrue({"name": "Home/level1-A"} not in data) - self.assertTrue({"name": "Home"} not in data) + data = frappe.get_all('Nested DocType', {'name': ('not ancestors of', 'Level 2 A')}) + self.assertTrue({"name": "Level 3 A"} in data) + self.assertTrue({"name": "Level 2 A"} in data) + self.assertTrue({"name": "Level 2 B"} in data) + self.assertTrue({"name": "Level 1 B"} in data) + self.assertTrue({"name": "Level 1 A"} not in data) + self.assertTrue({"name": "Root"} not in data) - data = frappe.get_all('File', {'name': ('not ancestors of', 'Home/level1-A')}) - self.assertTrue({"name": "Home/level1-A/level2-A/level3-A"} in data) - self.assertTrue({"name": "Home/level1-A/level2-A"} in data) - self.assertTrue({"name": "Home/level1-A/level2-B"} in data) - self.assertTrue({"name": "Home/level1-B"} in data) - self.assertTrue({"name": "Home/level1-A"} in data) - self.assertFalse({"name": "Home"} in data) + data = frappe.get_all('Nested DocType', {'name': ('not ancestors of', 'Level 1 A')}) + self.assertTrue({"name": "Level 3 A"} in data) + self.assertTrue({"name": "Level 2 A"} in data) + self.assertTrue({"name": "Level 2 B"} in data) + self.assertTrue({"name": "Level 1 B"} in data) + self.assertTrue({"name": "Level 1 A"} in data) + self.assertFalse({"name": "Root"} in data) - data = frappe.get_all('File', {'name': ('ancestors of', 'Home')}) + data = frappe.get_all('Nested DocType', {'name': ('ancestors of', 'Root')}) self.assertTrue(len(data) == 0) - self.assertTrue(len(frappe.get_all('File', {'name': ('not ancestors of', 'Home')})) == len(frappe.get_all('File'))) + self.assertTrue(len(frappe.get_all('Nested DocType', {'name': ('not ancestors of', 'Root')})) == len(frappe.get_all('Nested DocType'))) def test_is_set_is_not_set(self): @@ -364,14 +348,45 @@ def create_event(subject="_Test Event", starts_on=None): return event -def delete_test_file_hierarchy(): - files_to_delete = [ - 'Home/level1-A/level2-A/level3-A', - 'Home/level1-A/level2-A', - 'Home/level1-A/level2-B', - 'Home/level1-A', - 'Home/level1-B/level2-A', - 'Home/level1-B' +def create_nested_doctype(): + if frappe.db.exists('DocType', 'Nested DocType'): + return + + frappe.get_doc({ + 'doctype': 'DocType', + 'name': 'Nested DocType', + 'module': 'Custom', + 'is_tree': 1, + 'custom': 1, + 'autoname': 'Prompt', + 'fields': [ + {'label': 'Description', 'fieldname': 'description'} + ], + 'permissions': [ + {'role': 'Blogger'} + ] + }).insert() + +def create_nested_doctype_records(): + ''' + Create a structure like: + - Root + - Level 1 A + - Level 2 A + - Level 3 A + - Level 1 B + - Level 2 B + ''' + records = [ + {'name': 'Root', 'is_group': 1}, + {'name': 'Level 1 A', 'parent_nested_doctype': 'Root', 'is_group': 1}, + {'name': 'Level 2 A', 'parent_nested_doctype': 'Level 1 A', 'is_group': 1}, + {'name': 'Level 3 A', 'parent_nested_doctype': 'Level 2 A'}, + {'name': 'Level 1 B', 'parent_nested_doctype': 'Root', 'is_group': 1}, + {'name': 'Level 2 B', 'parent_nested_doctype': 'Level 1 B'}, ] - for file_name in files_to_delete: - frappe.delete_doc('File', file_name) + + for r in records: + d = frappe.new_doc('Nested DocType') + d.update(r) + d.insert(ignore_permissions=True, ignore_if_duplicate=True) From ef665c4c87efbd0852c8d0af263538590aeaa362 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Tue, 10 Sep 2019 11:45:45 +0530 Subject: [PATCH 06/32] fix(pg test): get_list to get_all --- frappe/permissions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/permissions.py b/frappe/permissions.py index a27ba5ee75..a0d1677fac 100644 --- a/frappe/permissions.py +++ b/frappe/permissions.py @@ -423,7 +423,7 @@ def clear_user_permissions_for_doctype(doctype, user=None): filters = {'allow': doctype} if user: filters['user'] = user - user_permissions_for_doctype = frappe.db.get_list('User Permission', filters=filters) + user_permissions_for_doctype = frappe.db.get_all('User Permission', filters=filters) for d in user_permissions_for_doctype: frappe.delete_doc('User Permission', d.name) From 244bc16c27dd585dca9268257129612e3a34ae09 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Tue, 10 Sep 2019 13:32:06 +0530 Subject: [PATCH 07/32] fix: Fallback to localhost for db_host --- frappe/database/postgres/setup_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/database/postgres/setup_db.py b/frappe/database/postgres/setup_db.py index 79c7c3a304..01a97178f9 100644 --- a/frappe/database/postgres/setup_db.py +++ b/frappe/database/postgres/setup_db.py @@ -17,7 +17,7 @@ def setup_database(force, source_sql, verbose): subprocess_env['PGPASSWORD'] = str(frappe.conf.db_password) # bootstrap db subprocess.check_output([ - 'psql', frappe.conf.db_name, '-h', frappe.conf.db_host, '-U', + 'psql', frappe.conf.db_name, '-h', frappe.conf.db_host or 'localhost', '-U', frappe.conf.db_name, '-f', os.path.join(os.path.dirname(__file__), 'framework_postgres.sql') ], env=subprocess_env) From 4811b64b1fefc980ca45a643821a4be8e24954cf Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Tue, 10 Sep 2019 13:32:23 +0530 Subject: [PATCH 08/32] fix(test): db.commit after creating table --- frappe/tests/test_db_query.py | 1 + 1 file changed, 1 insertion(+) diff --git a/frappe/tests/test_db_query.py b/frappe/tests/test_db_query.py index 28849618ee..6ddd152920 100644 --- a/frappe/tests/test_db_query.py +++ b/frappe/tests/test_db_query.py @@ -366,6 +366,7 @@ def create_nested_doctype(): {'role': 'Blogger'} ] }).insert() + frappe.db.commit() def create_nested_doctype_records(): ''' From 1eecc3dadb8ee1b44d9dc006292ff71e3ea1f4da Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Tue, 10 Sep 2019 13:34:48 +0530 Subject: [PATCH 09/32] Revert "fix(pg test): get_list to get_all" This reverts commit ef665c4c87efbd0852c8d0af263538590aeaa362. --- frappe/permissions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/permissions.py b/frappe/permissions.py index a0d1677fac..a27ba5ee75 100644 --- a/frappe/permissions.py +++ b/frappe/permissions.py @@ -423,7 +423,7 @@ def clear_user_permissions_for_doctype(doctype, user=None): filters = {'allow': doctype} if user: filters['user'] = user - user_permissions_for_doctype = frappe.db.get_all('User Permission', filters=filters) + user_permissions_for_doctype = frappe.db.get_list('User Permission', filters=filters) for d in user_permissions_for_doctype: frappe.delete_doc('User Permission', d.name) From 30a527b91123cc57bf5d45c731cdb91f59c4dfbb Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Tue, 10 Sep 2019 17:14:27 +0530 Subject: [PATCH 10/32] fix: Remove db.commit --- frappe/tests/test_db_query.py | 1 - 1 file changed, 1 deletion(-) diff --git a/frappe/tests/test_db_query.py b/frappe/tests/test_db_query.py index 6ddd152920..28849618ee 100644 --- a/frappe/tests/test_db_query.py +++ b/frappe/tests/test_db_query.py @@ -366,7 +366,6 @@ def create_nested_doctype(): {'role': 'Blogger'} ] }).insert() - frappe.db.commit() def create_nested_doctype_records(): ''' From 5bf4e1616f9956986b3d59781f4a8a7e2fa7b26c Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Tue, 10 Sep 2019 18:30:57 +0530 Subject: [PATCH 11/32] fix: Remove duplicate calls --- frappe/tests/test_db_query.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/frappe/tests/test_db_query.py b/frappe/tests/test_db_query.py index 28849618ee..d933140cbe 100644 --- a/frappe/tests/test_db_query.py +++ b/frappe/tests/test_db_query.py @@ -251,8 +251,6 @@ class TestReportview(unittest.TestCase): self.assertTrue('DefaultValue' in [d['name'] for d in out]) def test_of_not_of_descendant_ancestors(self): - create_nested_doctype() - create_nested_doctype_records() clear_user_permissions_for_doctype('Nested DocType') # in descendants filter From e74eb649b7cc8a4e09a15905e6f51fbfec4e29bc Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Wed, 11 Sep 2019 13:16:46 +0530 Subject: [PATCH 12/32] fix(test): Delete blog after test --- frappe/core/doctype/comment/test_comment.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frappe/core/doctype/comment/test_comment.py b/frappe/core/doctype/comment/test_comment.py index 2adc5eb899..3cf8fbaa3f 100644 --- a/frappe/core/doctype/comment/test_comment.py +++ b/frappe/core/doctype/comment/test_comment.py @@ -53,5 +53,7 @@ class TestComment(unittest.TestCase): reference_name = test_blog.name ))), 0) + test_blog.delete() + From 01da83909daaba6d20f1d52e15bdc162ba912c05 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Wed, 11 Sep 2019 13:17:00 +0530 Subject: [PATCH 13/32] fix(test): Check for mandatory manually --- frappe/core/doctype/doctype/doctype.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/frappe/core/doctype/doctype/doctype.py b/frappe/core/doctype/doctype/doctype.py index e808cd3053..ef346d1992 100644 --- a/frappe/core/doctype/doctype/doctype.py +++ b/frappe/core/doctype/doctype/doctype.py @@ -586,9 +586,11 @@ class DocType(Document): if not self.is_tree: return self.add_nestedset_fields() - # set field as mandatory - field = self.meta.get_field('nsm_parent_field') - field.reqd = 1 + + if not self.nsm_parent_field: + field_label = frappe.bold(_("Parent Field (Tree)")) + frappe.throw(_("{0} is a mandatory field").format(field_label), frappe.MandatoryError) + # check if field is valid fieldnames = [df.fieldname for df in self.fields] if self.nsm_parent_field and self.nsm_parent_field not in fieldnames: From 7b9ee521eee29c9f36285e5df32bd8a70f91cdba Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Wed, 11 Sep 2019 15:58:36 +0530 Subject: [PATCH 14/32] fix: Use get_all instead of get_list --- frappe/permissions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/permissions.py b/frappe/permissions.py index a27ba5ee75..a0d1677fac 100644 --- a/frappe/permissions.py +++ b/frappe/permissions.py @@ -423,7 +423,7 @@ def clear_user_permissions_for_doctype(doctype, user=None): filters = {'allow': doctype} if user: filters['user'] = user - user_permissions_for_doctype = frappe.db.get_list('User Permission', filters=filters) + user_permissions_for_doctype = frappe.db.get_all('User Permission', filters=filters) for d in user_permissions_for_doctype: frappe.delete_doc('User Permission', d.name) From 46c4f7ba31c1a20c4cfc88c18c618f10443e4336 Mon Sep 17 00:00:00 2001 From: Himanshu Warekar Date: Thu, 12 Sep 2019 16:05:27 +0530 Subject: [PATCH 15/32] fix: dont return admin and guest placeholder email --- frappe/core/doctype/role/role.py | 2 +- frappe/email/doctype/notification/notification.py | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/frappe/core/doctype/role/role.py b/frappe/core/doctype/role/role.py index 1467eafc5d..7ce2537da3 100644 --- a/frappe/core/doctype/role/role.py +++ b/frappe/core/doctype/role/role.py @@ -31,7 +31,7 @@ def get_emails_from_role(role): for user in users: user_email, enabled = frappe.db.get_value("User", user.parent, ["email", "enabled"]) - if enabled: + if enabled and user_email not in ["admin@example.com", "guest@example.com"]: emails.append(user_email) return emails \ No newline at end of file diff --git a/frappe/email/doctype/notification/notification.py b/frappe/email/doctype/notification/notification.py index 8032e020d7..f8d3002a2c 100644 --- a/frappe/email/doctype/notification/notification.py +++ b/frappe/email/doctype/notification/notification.py @@ -309,9 +309,7 @@ def evaluate_alert(doc, alert, event): else: raise db_value = parse_val(db_value) - if (doc.get(alert.value_changed) == db_value) or \ - (not db_value and not doc.get(alert.value_changed)): - + if (doc.get(alert.value_changed) == db_value) or (not db_value and not doc.get(alert.value_changed)): return # value not changed if event != "Value Change" and not doc.is_new(): From d1c4e6e7dd4477942ba7fba038a91398f712194e Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Tue, 17 Sep 2019 11:51:00 +0530 Subject: [PATCH 16/32] fix: more button visibility in webforms --- frappe/public/js/frappe/web_form/web_form_list.js | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/frappe/public/js/frappe/web_form/web_form_list.js b/frappe/public/js/frappe/web_form/web_form_list.js index 7b14844c6f..db5f1da3a0 100644 --- a/frappe/public/js/frappe/web_form/web_form_list.js +++ b/frappe/public/js/frappe/web_form/web_form_list.js @@ -201,14 +201,13 @@ export default class WebFormList { () => (window.location.href = window.location.pathname + "?new=1") ); - if (!(this.rows.length < this.page_length)) { + if (this.rows.length <= this.page_length) { addButton(footer, "more", "secondary", false, "More", () => this.more()); } function addButton(wrapper, id, type, hidden, name, action) { const button = document.createElement("button"); - button.classList.add("btn", "btn-primary", "btn-sm", "ml-2"); - if (type == "secondary") + if (type == "secondary") { button.classList.add( "btn", "btn-secondary", @@ -216,7 +215,8 @@ export default class WebFormList { "ml-2", "text-white" ); - if (type == "danger") + } + else if (type == "danger") { button.classList.add( "btn", "btn-danger", @@ -224,6 +224,10 @@ export default class WebFormList { "btn-sm", "ml-2" ); + } + else { + button.classList.add("btn", "btn-primary", "btn-sm", "ml-2"); + } button.id = id; button.innerText = name; From 3de3ce02302c0b19220a14b7e3602cbaeca1e807 Mon Sep 17 00:00:00 2001 From: Rucha Mahabal Date: Tue, 17 Sep 2019 22:23:24 +0530 Subject: [PATCH 17/32] fix(report-view): show export all checkbox only if total row count exceeds page length --- .../js/frappe/views/reports/report_view.js | 32 +++++++++++-------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/frappe/public/js/frappe/views/reports/report_view.js b/frappe/public/js/frappe/views/reports/report_view.js index ad4380bb4a..aa0a17b7a2 100644 --- a/frappe/public/js/frappe/views/reports/report_view.js +++ b/frappe/public/js/frappe/views/reports/report_view.js @@ -1231,23 +1231,27 @@ frappe.views.ReportView = class ReportView extends frappe.views.ListView { action: () => { const args = this.get_args(); const selected_items = this.get_checked_items(true); + let fields = [ + { + fieldtype: 'Select', + label: __('Select File Type'), + fieldname:'file_format_type', + options: ['Excel', 'CSV'], + default: 'Excel' + } + ] + + if (this.total_count > args.page_length) { + fields.push({ + fieldtype: 'Check', + fieldname: 'export_all_rows', + label: __('Export All {0} rows?', [(this.total_count + "").bold()]) + }); + } const d = new frappe.ui.Dialog({ title: __("Export Report: {0}",[__(this.doctype)]), - fields: [ - { - fieldtype: 'Select', - label: __('Select File Type'), - fieldname:'file_format_type', - options: ['Excel', 'CSV'], - default: 'Excel' - }, - { - fieldtype: 'Check', - fieldname: 'export_all_rows', - label: __('Export All {0} rows?', [(this.total_count + "").bold()]) - } - ], + fields: fields, primary_action_label: __('Download'), primary_action: (data) => { args.cmd = 'frappe.desk.reportview.export_query'; From 2508ae13777434ef30f858b4451a1a4d0f91eec1 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Wed, 18 Sep 2019 11:04:34 +0530 Subject: [PATCH 18/32] test: Set user Administrator --- frappe/tests/test_db_query.py | 1 + 1 file changed, 1 insertion(+) diff --git a/frappe/tests/test_db_query.py b/frappe/tests/test_db_query.py index d933140cbe..18c19cbf3e 100644 --- a/frappe/tests/test_db_query.py +++ b/frappe/tests/test_db_query.py @@ -251,6 +251,7 @@ class TestReportview(unittest.TestCase): self.assertTrue('DefaultValue' in [d['name'] for d in out]) def test_of_not_of_descendant_ancestors(self): + frappe.set_user('Administrator') clear_user_permissions_for_doctype('Nested DocType') # in descendants filter From e6055d406e87c4c77e883ad730379c687528ed8b Mon Sep 17 00:00:00 2001 From: Rucha Mahabal Date: Wed, 18 Sep 2019 11:56:16 +0530 Subject: [PATCH 19/32] fix: codacy --- frappe/public/js/frappe/views/reports/report_view.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/views/reports/report_view.js b/frappe/public/js/frappe/views/reports/report_view.js index aa0a17b7a2..6ac26c00da 100644 --- a/frappe/public/js/frappe/views/reports/report_view.js +++ b/frappe/public/js/frappe/views/reports/report_view.js @@ -1239,7 +1239,7 @@ frappe.views.ReportView = class ReportView extends frappe.views.ListView { options: ['Excel', 'CSV'], default: 'Excel' } - ] + ]; if (this.total_count > args.page_length) { fields.push({ From 66bd2a9f5eb4db6aa4ebf33213d40dc1c247218b Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Wed, 18 Sep 2019 12:29:50 +0530 Subject: [PATCH 20/32] chore: Squash PRs when squash label is added (#8446) --- .mergify.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.mergify.yml b/.mergify.yml index b0e11e0e6d..ed29c6c917 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -11,3 +11,16 @@ pull_request_rules: actions: merge: method: merge + - name: Automatic squash on CI success and review + conditions: + - status-success=Codacy/PR Quality Review + - status-success=Semantic Pull Request + - status-success=continuous-integration/travis-ci/pr + - status-success=security/snyk - package.json (frappe) + - status-success=security/snyk - requirements.txt (frappe) + - label!=don't-merge + - label=squash + - "#approved-reviews-by>=1" + actions: + merge: + method: squash From 44947ee13a65870adae3acfb033dcd9a24826d25 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Wed, 18 Sep 2019 13:25:38 +0530 Subject: [PATCH 21/32] chore: Dont merge if squash label is added (#8447) --- .mergify.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.mergify.yml b/.mergify.yml index ed29c6c917..d8f1c09bee 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -7,6 +7,7 @@ pull_request_rules: - status-success=security/snyk - package.json (frappe) - status-success=security/snyk - requirements.txt (frappe) - label!=don't-merge + - label!=squash - "#approved-reviews-by>=1" actions: merge: From 7d2e664b74a4abe94c7d659a79af4356f46e240b Mon Sep 17 00:00:00 2001 From: gavin Date: Wed, 18 Sep 2019 15:57:04 +0530 Subject: [PATCH 22/32] fix: disables 'New' for Prepared Report (#8448) --- .../prepared_report/prepared_report.json | 551 ++++-------------- .../prepared_report/prepared_report.py | 45 +- 2 files changed, 132 insertions(+), 464 deletions(-) diff --git a/frappe/core/doctype/prepared_report/prepared_report.json b/frappe/core/doctype/prepared_report/prepared_report.json index ec89c6327a..ab6650d9e3 100644 --- a/frappe/core/doctype/prepared_report/prepared_report.json +++ b/frappe/core/doctype/prepared_report/prepared_report.json @@ -1,463 +1,128 @@ { - "allow_copy": 0, - "allow_events_in_timeline": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "autoname": "REP.#####", - "beta": 0, - "creation": "2018-06-25 18:39:11.152960", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", + "autoname": "REP.#####", + "creation": "2018-06-25 18:39:11.152960", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "report_name", + "ref_report_doctype", + "status", + "column_break_4", + "report_start_time", + "report_end_time", + "section_break_7", + "error_message", + "filters_sb", + "filters", + "filter_values", + "columns" + ], "fields": [ { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fetch_if_empty": 0, - "fieldname": "report_name", - "fieldtype": "Data", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Report Name", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "report_name", + "fieldtype": "Data", + "hidden": 1, + "label": "Report Name", + "read_only": 1 + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fetch_if_empty": 0, - "fieldname": "ref_report_doctype", - "fieldtype": "Link", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Ref Report DocType", - "length": 0, - "no_copy": 0, - "options": "Report", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "ref_report_doctype", + "fieldtype": "Link", + "hidden": 1, + "label": "Ref Report DocType", + "options": "Report", + "read_only": 1 + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "default": "Queued", - "fetch_if_empty": 0, - "fieldname": "status", - "fieldtype": "Select", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Status", - "length": 0, - "no_copy": 0, - "options": "Error\nQueued\nCompleted", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "default": "Queued", + "fieldname": "status", + "fieldtype": "Select", + "hidden": 1, + "in_list_view": 1, + "label": "Status", + "options": "Error\nQueued\nCompleted", + "read_only": 1 + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fetch_if_empty": 0, - "fieldname": "column_break_4", - "fieldtype": "Column Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "column_break_4", + "fieldtype": "Column Break" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fetch_if_empty": 0, - "fieldname": "report_start_time", - "fieldtype": "Datetime", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Report Start Time", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "report_start_time", + "fieldtype": "Datetime", + "label": "Report Start Time", + "read_only": 1 + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fetch_if_empty": 0, - "fieldname": "report_end_time", - "fieldtype": "Datetime", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Report End Time", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "report_end_time", + "fieldtype": "Datetime", + "label": "Report End Time", + "read_only": 1 + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "depends_on": "eval:doc.status == 'Error'", - "fetch_if_empty": 0, - "fieldname": "section_break_7", - "fieldtype": "Section Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "depends_on": "eval:doc.status == 'Error'", + "fieldname": "section_break_7", + "fieldtype": "Section Break" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fetch_if_empty": 0, - "fieldname": "error_message", - "fieldtype": "Text", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Error Message", - "length": 0, - "no_copy": 1, - "permlevel": 0, - "precision": "", - "print_hide": 1, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "error_message", + "fieldtype": "Text", + "label": "Error Message", + "no_copy": 1, + "print_hide": 1, + "read_only": 1 + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fetch_if_empty": 0, - "fieldname": "filters_sb", - "fieldtype": "Section Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Filters", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "filters_sb", + "fieldtype": "Section Break", + "label": "Filters" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fetch_if_empty": 0, - "fieldname": "filters", - "fieldtype": "Small Text", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Filters", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "filters", + "fieldtype": "Small Text", + "hidden": 1, + "label": "Filters", + "read_only": 1 + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fetch_if_empty": 0, - "fieldname": "filter_values", - "fieldtype": "HTML", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Filter Values", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "filter_values", + "fieldtype": "HTML", + "label": "Filter Values" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fetch_if_empty": 0, - "fieldname": "columns", - "fieldtype": "Code", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Columns", - "length": 0, - "no_copy": 1, - "permlevel": 0, - "precision": "", - "print_hide": 1, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "fieldname": "columns", + "fieldtype": "Code", + "hidden": 1, + "label": "Columns", + "no_copy": 1, + "print_hide": 1, + "read_only": 1 } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2019-04-19 12:39:47.211516", - "modified_by": "Administrator", - "module": "Core", - "name": "Prepared Report", - "name_case": "", - "owner": "Administrator", + ], + "in_create": 1, + "modified": "2019-09-18 04:00:55.644257", + "modified_by": "Administrator", + "module": "Core", + "name": "Prepared Report", + "owner": "Administrator", "permissions": [ { - "amend": 0, - "cancel": 0, - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "set_user_permissions": 0, - "share": 1, - "submit": 0, + "create": 1, + "delete": 1, + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1, "write": 1 } - ], - "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "title_field": "report_name", - "track_changes": 1, - "track_seen": 0, - "track_views": 0 + ], + "quick_entry": 1, + "sort_field": "modified", + "sort_order": "DESC", + "title_field": "report_name", + "track_changes": 1 } \ No newline at end of file diff --git a/frappe/core/doctype/prepared_report/prepared_report.py b/frappe/core/doctype/prepared_report/prepared_report.py index 29c069515f..989a99511a 100644 --- a/frappe/core/doctype/prepared_report/prepared_report.py +++ b/frappe/core/doctype/prepared_report/prepared_report.py @@ -4,32 +4,25 @@ from __future__ import unicode_literals -import base64 + import json -import io import frappe -from frappe.model.document import Document -from frappe.utils.background_jobs import enqueue -from frappe.desk.query_report import generate_report_result, get_columns_dict -from frappe.core.doctype.file.file import remove_all -from frappe.utils.csvutils import to_csv, read_csv_content_from_attached_file from frappe.desk.form.load import get_attachments +from frappe.desk.query_report import generate_report_result +from frappe.model.document import Document from frappe.utils import gzip_compress, gzip_decompress -from six import PY2 -from frappe.utils import encode +from frappe.utils.background_jobs import enqueue +from frappe.core.doctype.file.file import remove_all + class PreparedReport(Document): - def before_insert(self): self.status = "Queued" self.report_start_time = frappe.utils.now() def enqueue_report(self): - enqueue( - run_background, - prepared_report=self.name, timeout=6000 - ) + enqueue(run_background, prepared_report=self.name, timeout=6000) def on_trash(self): remove_all("PreparedReport", self.name, from_delete=True) @@ -42,14 +35,18 @@ def run_background(prepared_report): try: report.custom_columns = [] - if report.report_type == 'Custom Report': + if report.report_type == "Custom Report": custom_report_doc = report reference_report = custom_report_doc.reference_report report = frappe.get_doc("Report", reference_report) report.custom_columns = custom_report_doc.json - result = generate_report_result(report, filters=instance.filters, user=instance.owner) - create_json_gz_file(result['result'], 'Prepared Report', instance.name) + result = generate_report_result( + report=report, + filters=instance.filters, + user=instance.owner + ) + create_json_gz_file(result["result"], "Prepared Report", instance.name) instance.status = "Completed" instance.columns = json.dumps(result["columns"]) @@ -64,8 +61,11 @@ def run_background(prepared_report): instance.save(ignore_permissions=True) frappe.publish_realtime( - 'report_generated', - {"report_name": instance.report_name, "name": instance.name}, + "report_generated", + { + "report_name": instance.report_name, + "name": instance.name + }, user=frappe.session.user ) @@ -73,7 +73,9 @@ def run_background(prepared_report): def create_json_gz_file(data, dt, dn): # Storing data in CSV file causes information loss # Reports like P&L Statement were completely unsuable because of this - json_filename = '{0}.json.gz'.format(frappe.utils.data.format_datetime(frappe.utils.now(), "Y-m-d-H:M")) + json_filename = "{0}.json.gz".format( + frappe.utils.data.format_datetime(frappe.utils.now(), "Y-m-d-H:M") + ) encoded_content = frappe.safe_encode(frappe.as_json(data)) compressed_content = gzip_compress(encoded_content) @@ -87,10 +89,11 @@ def create_json_gz_file(data, dt, dn): }) _file.save() + @frappe.whitelist() def download_attachment(dn): attachment = get_attachments("Prepared Report", dn)[0] frappe.local.response.filename = attachment.file_name[:-2] - attached_file = frappe.get_doc('File', attachment.name) + attached_file = frappe.get_doc("File", attachment.name) frappe.local.response.filecontent = gzip_decompress(attached_file.get_content()) frappe.local.response.type = "binary" From 7ef5b20ca4df31c7e835616af8f5ae36a65ecfac Mon Sep 17 00:00:00 2001 From: gavin Date: Wed, 18 Sep 2019 22:55:35 +0530 Subject: [PATCH 23/32] fix: fixed email count for current month (#8452) --- frappe/email/queue.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/frappe/email/queue.py b/frappe/email/queue.py index bbd1948989..018c2dc275 100755 --- a/frappe/email/queue.py +++ b/frappe/email/queue.py @@ -243,8 +243,10 @@ def get_email_queue(recipients, sender, subject, **kwargs): return e def get_emails_sent_this_month(): - return frappe.db.sql("""SELECT COUNT(`name`) FROM `tabEmail Queue` WHERE - `status`='Sent' AND EXTRACT(MONTH FROM `creation`) = EXTRACT(MONTH FROM NOW())""")[0][0] + return frappe.db.sql(""" + SELECT COUNT(*) FROM `tabEmail Queue` + WHERE `status`='Sent' AND EXTRACT(YEAR_MONTH FROM `creation`) = EXTRACT(YEAR_MONTH FROM NOW()) + """)[0][0] def get_emails_sent_today(): return frappe.db.sql("""SELECT COUNT(`name`) FROM `tabEmail Queue` WHERE From 03587849b8d7bc03402882d5fbf1b0074daf9118 Mon Sep 17 00:00:00 2001 From: Rucha Mahabal Date: Thu, 19 Sep 2019 13:09:06 +0530 Subject: [PATCH 24/32] fix: export report not working for add group --- frappe/desk/reportview.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/frappe/desk/reportview.py b/frappe/desk/reportview.py index e3e408eeb1..a6ff8d7514 100644 --- a/frappe/desk/reportview.py +++ b/frappe/desk/reportview.py @@ -213,6 +213,10 @@ def get_labels(fields, doctype): for key in fields: key = key.split(" as ")[0] + if key.startswith('count('): continue + if key.startswith('sum('): continue + if key.startswith('avg('): continue + if "." in key: parenttype, fieldname = key.split(".")[0][4:-1], key.split(".")[1].strip("`") else: From d4ea2b4c1f66a2cd702347341f48b0c02c8225ae Mon Sep 17 00:00:00 2001 From: Rucha Mahabal Date: Thu, 19 Sep 2019 13:34:51 +0530 Subject: [PATCH 25/32] fix: merge key validations Co-Authored-By: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> --- frappe/desk/reportview.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/desk/reportview.py b/frappe/desk/reportview.py index a6ff8d7514..2b22845de9 100644 --- a/frappe/desk/reportview.py +++ b/frappe/desk/reportview.py @@ -213,7 +213,7 @@ def get_labels(fields, doctype): for key in fields: key = key.split(" as ")[0] - if key.startswith('count('): continue + if key.startswith(('count(', 'sum(', 'avg(')): continue if key.startswith('sum('): continue if key.startswith('avg('): continue From c120bd34661a7b36d5a1b9e4c622a2162d58601f Mon Sep 17 00:00:00 2001 From: Rucha Mahabal Date: Thu, 19 Sep 2019 13:39:56 +0530 Subject: [PATCH 26/32] fix: merge key validations --- frappe/desk/reportview.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/frappe/desk/reportview.py b/frappe/desk/reportview.py index 2b22845de9..dd984625fd 100644 --- a/frappe/desk/reportview.py +++ b/frappe/desk/reportview.py @@ -214,8 +214,6 @@ def get_labels(fields, doctype): key = key.split(" as ")[0] if key.startswith(('count(', 'sum(', 'avg(')): continue - if key.startswith('sum('): continue - if key.startswith('avg('): continue if "." in key: parenttype, fieldname = key.split(".")[0][4:-1], key.split(".")[1].strip("`") From c8a80a2e4d64e14d028db7f19a8b4a028c7b2042 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Thu, 19 Sep 2019 15:41:17 +0530 Subject: [PATCH 27/32] fix: frappe.db.field_exists - field_exists didn't work correctly - The call to field_exists itself was wrong --- frappe/database/database.py | 5 ++++- frappe/model/base_document.py | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index 6702ce2a5e..412051c76f 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -758,7 +758,10 @@ class Database(object): def field_exists(self, dt, fn): """Return true of field exists.""" - return self.sql("select name from tabDocField where fieldname=%s and parent=%s", (dt, fn)) + return self.exists('DocField', { + 'fieldname': fn, + 'parent': dt + }) def table_exists(self, doctype): """Returns True if table for given doctype exists.""" diff --git a/frappe/model/base_document.py b/frappe/model/base_document.py index 7480970711..6eb2efce6b 100644 --- a/frappe/model/base_document.py +++ b/frappe/model/base_document.py @@ -38,8 +38,8 @@ def get_controller(doctype): or ["Core", False] if custom: - if frappe.db.field_exists(doctype, "is_tree"): - is_tree = frappe.db.get_value("DocType", doctype, ("is_tree"), cache=True) + if frappe.db.field_exists("DocType", "is_tree"): + is_tree = frappe.db.get_value("DocType", doctype, "is_tree", cache=True) else: is_tree = False _class = NestedSet if is_tree else Document From ef30490865bfb50313fcff8a1634d42dd3850292 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Thu, 19 Sep 2019 16:14:19 +0530 Subject: [PATCH 28/32] style: Unused variable --- frappe/core/doctype/file/file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/core/doctype/file/file.py b/frappe/core/doctype/file/file.py index abb69c83ea..fa80d0ed6e 100755 --- a/frappe/core/doctype/file/file.py +++ b/frappe/core/doctype/file/file.py @@ -589,7 +589,7 @@ def get_breadcrumbs(folder): path = folder.split('/') folders = [] - for i, p in enumerate(path): + for i, _ in enumerate(path): indexes = range(0, i) folder = '/'.join([path[i] for i in indexes]) if folder: From 3e390470aee3e2a89dc7da10ce515edc12c60ed3 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Thu, 19 Sep 2019 17:10:04 +0530 Subject: [PATCH 29/32] fix: retain doc.name --- frappe/public/js/frappe/web_form/web_form.js | 1 - 1 file changed, 1 deletion(-) diff --git a/frappe/public/js/frappe/web_form/web_form.js b/frappe/public/js/frappe/web_form/web_form.js index 216bf2d831..ca44dd65d0 100644 --- a/frappe/public/js/frappe/web_form/web_form.js +++ b/frappe/public/js/frappe/web_form/web_form.js @@ -107,7 +107,6 @@ export default class WebForm extends frappe.ui.FieldGroup { let for_payment = Boolean(this.accept_payment && !this.doc.paid); this.doc.doctype = this.doc_type; - this.doc.name = this.doc_name; this.doc.web_form_name = this.name; // Save From 79be132e3dca1a5435a2be3f35ad176ced1e30fb Mon Sep 17 00:00:00 2001 From: Rohit Waghchaure Date: Fri, 20 Sep 2019 12:22:54 +0530 Subject: [PATCH 30/32] feat: added flag to skip redirect on error for quick entry --- frappe/public/js/frappe/form/quick_entry.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/form/quick_entry.js b/frappe/public/js/frappe/form/quick_entry.js index 38250f2ad8..e680c8ada1 100644 --- a/frappe/public/js/frappe/form/quick_entry.js +++ b/frappe/public/js/frappe/form/quick_entry.js @@ -186,7 +186,9 @@ frappe.ui.form.QuickEntryForm = Class.extend({ } }, error: function() { - me.open_doc(); + if (!me.skip_redirect_on_error) { + me.open_doc(); + } }, always: function() { me.dialog.working = false; From d67b9956762b48e14e25540a9b8d3471cd494bce Mon Sep 17 00:00:00 2001 From: Aditya Hase Date: Fri, 20 Sep 2019 14:40:12 +0530 Subject: [PATCH 31/32] fix(security): Use frappe.render_template instead of Template.render --- frappe/tests/test_twofactor.py | 5 ++--- frappe/twofactor.py | 15 ++++----------- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/frappe/tests/test_twofactor.py b/frappe/tests/test_twofactor.py index 4064853dac..586748fcd3 100644 --- a/frappe/tests/test_twofactor.py +++ b/frappe/tests/test_twofactor.py @@ -7,8 +7,7 @@ from frappe.auth import HTTPRequest from frappe.utils import cint from frappe.tests import set_request from frappe.twofactor import (should_run_2fa, authenticate_for_2factor, get_cached_user_pass, - two_factor_is_enabled_for_, confirm_otp_token, get_otpsecret_for_, get_verification_obj, - render_string_template) + two_factor_is_enabled_for_, confirm_otp_token, get_otpsecret_for_, get_verification_obj) import time @@ -123,7 +122,7 @@ class TestTwoFactor(unittest.TestCase): '''String template renders as expected with variables.''' args = {'issuer_name':'Frappe Technologies'} _str = 'Verification Code from {{issuer_name}}' - _str = render_string_template(_str,args) + _str = frappe.render_template(_str,args) self.assertEqual(_str,'Verification Code from Frappe Technologies') def test_bypass_restict_ip(self): diff --git a/frappe/twofactor.py b/frappe/twofactor.py index d8f68fb050..a539532f25 100644 --- a/frappe/twofactor.py +++ b/frappe/twofactor.py @@ -7,7 +7,6 @@ import frappe from frappe import _ import pyotp, os from frappe.utils.background_jobs import enqueue -from jinja2 import Template from pyqrcode import create as qrcreate from six import BytesIO from base64 import b64encode, b32encode @@ -223,33 +222,27 @@ def process_2fa_for_email(user, token, otp_secret, otp_issuer, method='Email'): def get_email_subject_for_2fa(kwargs_dict): '''Get email subject for 2fa.''' subject_template = _('Login Verification Code from {}').format(frappe.db.get_value('System Settings', 'System Settings', 'otp_issuer_name')) - subject = render_string_template(subject_template, kwargs_dict) + subject = frappe.render_template(subject_template, kwargs_dict) return subject def get_email_body_for_2fa(kwargs_dict): '''Get email body for 2fa.''' body_template = 'Enter this code to complete your login:

{{otp}}' - body = render_string_template(body_template, kwargs_dict) + body = frappe.render_template(body_template, kwargs_dict) return body def get_email_subject_for_qr_code(kwargs_dict): '''Get QRCode email subject.''' subject_template = _('One Time Password (OTP) Registration Code from {}').format(frappe.db.get_value('System Settings', 'System Settings', 'otp_issuer_name')) - subject = render_string_template(subject_template, kwargs_dict) + subject = frappe.render_template(subject_template, kwargs_dict) return subject def get_email_body_for_qr_code(kwargs_dict): '''Get QRCode email body.''' body_template = 'Please click on the following link and follow the instructions on the page.

{{qrcode_link}}' - body = render_string_template(body_template, kwargs_dict) + body = frappe.render_template(body_template, kwargs_dict) return body -def render_string_template(_str, kwargs_dict): - '''Render string with jinja.''' - s = Template(_str) - s = s.render(**kwargs_dict) - return s - def get_link_for_qrcode(user, totp_uri): '''Get link to temporary page showing QRCode.''' key = frappe.generate_hash(length=20) From bcba2e3ee1ca97099d970829b0b0825dc0fee591 Mon Sep 17 00:00:00 2001 From: Sahil Khan Date: Fri, 20 Sep 2019 15:34:17 +0550 Subject: [PATCH 32/32] bumped to version 12.0.14 --- frappe/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/__init__.py b/frappe/__init__.py index 78e0cdb355..8ea5148873 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -23,7 +23,7 @@ if sys.version[0] == '2': reload(sys) sys.setdefaultencoding("utf-8") -__version__ = '12.0.13' +__version__ = '12.0.14' __title__ = "Frappe Framework" local = Local()