From c08c5ab6d58dbee3453080ae37524a3139299e84 Mon Sep 17 00:00:00 2001 From: prssanna Date: Mon, 23 Dec 2019 18:34:01 +0530 Subject: [PATCH 1/3] fix: only scroll to top when going to a new page to add a row --- frappe/public/js/frappe/form/grid_pagination.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/form/grid_pagination.js b/frappe/public/js/frappe/form/grid_pagination.js index 190a58c5fe..9727485f63 100644 --- a/frappe/public/js/frappe/form/grid_pagination.js +++ b/frappe/public/js/frappe/form/grid_pagination.js @@ -117,10 +117,10 @@ export default class GridPagination { let page_length = this.page_length; if (this.grid.data.length == page_length*total_pages) { this.go_to_page(total_pages + 1); + frappe.utils.scroll_to(this.wrapper); } else { this.go_to_page(total_pages); } - frappe.utils.scroll_to(this.wrapper); } get_result_length() { From 8c614a15750aea4c9e4bad8c34ce4c3e376be5aa Mon Sep 17 00:00:00 2001 From: prssanna Date: Mon, 23 Dec 2019 18:41:29 +0530 Subject: [PATCH 2/3] fix: don't create filter tag if no filter is applied --- frappe/public/js/frappe/ui/filters/filter.js | 1 + 1 file changed, 1 insertion(+) diff --git a/frappe/public/js/frappe/ui/filters/filter.js b/frappe/public/js/frappe/ui/filters/filter.js index 4737f18bc0..5cdc3424f8 100644 --- a/frappe/public/js/frappe/ui/filters/filter.js +++ b/frappe/public/js/frappe/ui/filters/filter.js @@ -287,6 +287,7 @@ frappe.ui.Filter = class { } make_tag() { + if (!this.field) return; this.$filter_tag = this.get_filter_tag_element() .insertAfter(this.parent.find(".active-tag-filters .clear-filters")); this.set_filter_button_text(); From c9c4cbd6d114bcaa848a3a851072ee405f089b24 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Tue, 24 Dec 2019 13:12:57 +0530 Subject: [PATCH 3/3] perf: Speed improvements for moduleview get (#9079) * feat: api to setup boot cache on login * feat (perf): use cache data instead of query * feat: added doc hooks for rebuilding cache * feat: improved can import query * fix: ignore cache rebuild for import or install * fix: fix postgres syntax error Co-authored-by: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> --- frappe/auth.py | 6 ++++++ frappe/cache_manager.py | 43 +++++++++++++++++++++++++++++++++++++++ frappe/desk/moduleview.py | 26 ++++++++++++++--------- frappe/hooks.py | 7 +++++++ frappe/utils/user.py | 9 ++++---- 5 files changed, 77 insertions(+), 14 deletions(-) diff --git a/frappe/auth.py b/frappe/auth.py index 468f45c53e..dba8b05a62 100644 --- a/frappe/auth.py +++ b/frappe/auth.py @@ -142,6 +142,7 @@ class LoginManager: self.validate_hour() self.get_user_info() self.make_session() + self.setup_boot_cache() self.set_user_info() def get_user_info(self, resume=False): @@ -150,6 +151,11 @@ class LoginManager: self.user_type = self.info.user_type + def setup_boot_cache(self): + frappe.cache_manager.build_table_count_cache() + frappe.cache_manager.build_domain_restriced_doctype_cache() + frappe.cache_manager.build_domain_restriced_page_cache() + def set_user_info(self, resume=False): # set sid again frappe.local.cookie_manager.init_cookies() diff --git a/frappe/cache_manager.py b/frappe/cache_manager.py index 6e17327616..1e785e12f1 100644 --- a/frappe/cache_manager.py +++ b/frappe/cache_manager.py @@ -115,3 +115,46 @@ def get_doctype_map(doctype, name, filters, order_by=None): def clear_doctype_map(doctype, name): cache_key = frappe.scrub(doctype) + '_map' frappe.cache().hdel(cache_key, name) + +def build_table_count_cache(): + if frappe.flags.in_patch or frappe.flags.in_install or frappe.flags.in_import: + return + _cache = frappe.cache() + data = frappe.db.multisql({ + "mariadb": """ + SELECT table_name AS name, + table_rows AS count + FROM information_schema.tables""", + "postgres": """ + SELECT "relname" AS name, + "n_tup_ins" AS count + FROM "pg_stat_all_tables" + """ + }, as_dict=1) + + counts = {d.get('name').lstrip('tab'): d.get('count', None) for d in data} + _cache.set_value("information_schema:counts", counts) + + return counts + +def build_domain_restriced_doctype_cache(): + if frappe.flags.in_patch or frappe.flags.in_install or frappe.flags.in_import: + return + _cache = frappe.cache() + active_domains = frappe.get_active_domains() + doctypes = frappe.get_all("DocType", filters={'restrict_to_domain': ('IN', active_domains)}) + doctypes = [doc.name for doc in doctypes] + _cache.set_value("domain_restricted_doctypes", doctypes) + + return doctypes + +def build_domain_restriced_page_cache(): + if frappe.flags.in_patch or frappe.flags.in_install or frappe.flags.in_import: + return + _cache = frappe.cache() + active_domains = frappe.get_active_domains() + pages = frappe.get_all("Page", filters={'restrict_to_domain': ('IN', active_domains)}) + pages = [page.name for page in pages] + _cache.set_value("domain_restricted_pages", pages) + + return pages diff --git a/frappe/desk/moduleview.py b/frappe/desk/moduleview.py index 0bad171b04..604d47de9f 100644 --- a/frappe/desk/moduleview.py +++ b/frappe/desk/moduleview.py @@ -7,6 +7,7 @@ import json from frappe import _ from frappe.boot import get_allowed_pages, get_allowed_reports from frappe.desk.doctype.desktop_icon.desktop_icon import set_hidden, clear_desktop_icons_cache +from frappe.cache_manager import build_domain_restriced_doctype_cache, build_domain_restriced_page_cache, build_table_count_cache @frappe.whitelist() def get(module): @@ -25,6 +26,13 @@ def hide_module(module): set_hidden(module, frappe.session.user, 1) clear_desktop_icons_cache() +def get_table_with_counts(): + counts = frappe.cache().get_value("information_schema:counts") + if counts: + return counts + else: + return build_table_count_cache() + def get_data(module, build=True): """Get module data for the module view `desk/#Module/[name]`""" doctype_info = get_doctype_info(module) @@ -44,10 +52,10 @@ def get_data(module, build=True): # set_last_modified(data) if build: - exists_cache = {} + exists_cache = get_table_with_counts() def doctype_contains_a_record(name): exists = exists_cache.get(name) - if not exists: + if not type(exists) == int: if not frappe.db.get_value('DocType', name, 'issingle'): exists = frappe.db.count(name) else: @@ -96,19 +104,17 @@ def build_config_from_file(module): def filter_by_restrict_to_domain(data): """ filter Pages and DocType depending on the Active Module(s) """ - mapper = { - "page": "Page", - "doctype": "DocType" - } - active_domains = frappe.get_active_domains() + doctypes = frappe.cache().get_value("domain_restricted_doctypes") or build_domain_restriced_doctype_cache() + pages = frappe.cache().get_value("domain_restricted_pages") or build_domain_restriced_page_cache() for d in data: _items = [] for item in d.get("items", []): - doctype = mapper.get(item.get("type")) - doctype_domain = frappe.db.get_value(doctype, item.get("name"), "restrict_to_domain") or '' - if not doctype_domain or (doctype_domain in active_domains): + item_type = item.get("type") + item_name = item.get("name") + + if (item_name in pages) or (item_name in doctypes) or item_type == 'report': _items.append(item) d.update({ "items": _items }) diff --git a/frappe/hooks.py b/frappe/hooks.py index 5406d0fc57..4922911e82 100644 --- a/frappe/hooks.py +++ b/frappe/hooks.py @@ -139,6 +139,7 @@ doc_events = { "on_change": [ "frappe.social.doctype.energy_point_rule.energy_point_rule.process_energy_points" ], + "aftet_insert": "frappe.cache_manager.build_table_count_cache", }, "Event": { "after_insert": "frappe.integrations.doctype.google_calendar.google_calendar.insert_event_in_google_calendar", @@ -148,6 +149,12 @@ doc_events = { "Contact": { "after_insert": "frappe.integrations.doctype.google_contacts.google_contacts.insert_contacts_to_google_contacts", "on_update": "frappe.integrations.doctype.google_contacts.google_contacts.update_contacts_to_google_contacts", + }, + "DocType": { + "after_save": "frappe.cache_manager.build_domain_restriced_doctype_cache", + }, + "Page": { + "after_save": "frappe.cache_manager.build_domain_restriced_page_cache", } } diff --git a/frappe/utils/user.py b/frappe/utils/user.py index fa114d0e3c..64c129b63b 100755 --- a/frappe/utils/user.py +++ b/frappe/utils/user.py @@ -157,10 +157,11 @@ class UserPermissions: self.can_read.remove(dt) if "System Manager" in self.get_roles(): - docs = [x["name"] for x in frappe.get_all("DocType", "name")] - for docname in docs: - if frappe.get_meta(docname, cached=True).allow_import == 1: - self.can_import.append(docname) + docs = frappe.get_all("DocType", {'allow_import': 1}) + self.can_import += [doc.name for doc in docs] + + customizations = frappe.get_all("Property Setter", fields=['doc_type'], filters={'property': 'allow_import', 'value': 1}) + self.can_import += [custom.doc_type for custom in customizations] frappe.cache().hset("can_import", frappe.session.user, self.can_import)