From 3964db5d95db405f91ebbf827172dea8eac74ba5 Mon Sep 17 00:00:00 2001 From: Rushabh Mehta Date: Mon, 3 Mar 2014 17:53:02 +0530 Subject: [PATCH] refactored reportview.py and added frappe/model/db_query.py --- frappe/__init__.py | 6 +- frappe/model/db_query.py | 287 +++++++++++++++++ frappe/tests/test_db_query.py | 31 ++ .../doctype/blog_post/test_blog_post.py | 3 +- frappe/widgets/reportview.py | 296 +----------------- 5 files changed, 340 insertions(+), 283 deletions(-) create mode 100644 frappe/model/db_query.py create mode 100644 frappe/tests/test_db_query.py diff --git a/frappe/__init__.py b/frappe/__init__.py index feaf501d06..2ff45384be 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -560,10 +560,12 @@ def build_match_conditions(doctype, fields=None, as_condition=True): def get_list(doctype, filters=None, fields=None, docstatus=None, group_by=None, order_by=None, limit_start=0, limit_page_length=None, as_list=False, debug=False): - import frappe.widgets.reportview - return frappe.widgets.reportview.execute(doctype, filters=filters, fields=fields, docstatus=docstatus, + import frappe.model.db_query + return frappe.model.db_query.DatabaseQuery(doctype).execute(filters=filters, fields=fields, docstatus=docstatus, group_by=group_by, order_by=order_by, limit_start=limit_start, limit_page_length=limit_page_length, as_list=as_list, debug=debug) + +run_query = get_list def get_jenv(): if not local.jenv: diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py new file mode 100644 index 0000000000..a7c7e1ad6d --- /dev/null +++ b/frappe/model/db_query.py @@ -0,0 +1,287 @@ +# Copyright (c) 2013, Web Notes Technologies Pvt. Ltd. and Contributors +# MIT License. See license.txt + +from __future__ import unicode_literals +"""build query for doclistview and return results""" + +import frappe, json +import frappe.defaults +import frappe.permissions +import frappe.model.doctype +from frappe.utils import cstr, flt + +class DatabaseQuery(object): + def __init__(self, doctype): + self.doctype = doctype + + def execute(self, query=None, filters=None, fields=None, docstatus=None, + group_by=None, order_by=None, limit_start=0, limit_page_length=20, + as_list=False, with_childnames=False, debug=False): + self.fields = fields or ["name"] + self.filters = filters or [] + self.docstatus = docstatus or [] + self.group_by = group_by + self.order_by = order_by + self.limit_start = limit_start + self.limit_page_length = limit_page_length + self.with_childnames = with_childnames + self.debug = debug + self.as_list = as_list + self.tables = [] + self.meta = [] + + if query: + return self.run_custom_query(query) + else: + return self.build_and_run() + + def build_and_run(self): + args = self.prepare_args() + args.limit = self.add_limit() + + query = """select %(fields)s from %(tables)s where %(conditions)s + %(group_by)s order by %(order_by)s %(limit)s""" % args + + return frappe.db.sql(query, as_dict=not self.as_list, debug=self.debug) + + def prepare_args(self): + self.parse_args() + self.extract_tables() + self.load_metadata() + self.remove_user_tags() + self.build_conditions() + + args = frappe._dict() + + if self.with_childnames: + for t in self.tables: + if t != "`tab" + doctype + "`": + fields.append(t + ".name as '%s:name'" % t[4:-1]) + + # query dict + args.tables = ', '.join(self.tables) + args.conditions = ' and '.join(self.conditions) + args.fields = ', '.join(self.fields) + + args.order_by = self.order_by or self.tables[0] + '.modified desc' + args.group_by = self.group_by and (" group by " + group_by) or "" + + self.check_sort_by_table(args.order_by) + + return args + + + def parse_args(self): + if isinstance(self.filters, basestring): + self.filters = json.loads(self.filters) + if isinstance(self.fields, basestring): + self.filters = json.loads(self.fields) + if isinstance(self.filters, dict): + fdict = self.filters + self.filters = [] + for key, value in fdict.iteritems(): + self.filters.append(self.make_filter_tuple(key, value)) + + def make_filter_tuple(self, key, value): + if isinstance(value, (list, tuple)): + return (self.doctype, key, value[0], value[1]) + else: + return (self.doctype, key, "=", value) + + def extract_tables(self): + """extract tables from fields""" + self.tables = ['`tab' + self.doctype + '`'] + + # add tables from fields + if self.fields: + for f in self.fields: + if "." not in f: continue + + table_name = f.split('.')[0] + if table_name.lower().startswith('group_concat('): + table_name = table_name[13:] + if table_name.lower().startswith('ifnull('): + table_name = table_name[7:] + if not table_name[0]=='`': + table_name = '`' + table_name + '`' + if not table_name in self.tables: + self.tables.append(table_name) + + def load_metadata(self): + """load all doctypes and roles""" + self.meta = {} + + for t in self.tables: + if t.startswith('`'): + doctype = t[4:-1] + if self.meta.get(doctype): + continue + if not frappe.has_permission(doctype): + raise frappe.PermissionError, doctype + self.meta[doctype] = frappe.model.doctype.get(doctype) + + def remove_user_tags(self): + """remove column _user_tags if not in table""" + columns = frappe.db.get_table_columns(self.doctype) + to_remove = [] + for fld in self.fields: + for f in ("_user_tags", "_comments"): + if f in fld and not f in columns: + to_remove.append(fld) + + for fld in to_remove: + del self.fields[self.fields.index(fld)] + + def build_conditions(self): + self.conditions = [] + self.add_docstatus_conditions() + self.build_filter_conditions() + + # join parent, child tables + for tname in self.tables[1:]: + self.conditions.append(tname + '.parent = ' + self.tables[0] + '.name') + + # match conditions + match_conditions = self.build_match_conditions() + if match_conditions: + self.conditions.append(match_conditions) + + def add_docstatus_conditions(self): + if self.docstatus: + self.conditions.append(self.tables[0] + '.docstatus in (' + ','.join(docstatus) + ')') + else: + self.conditions.append(self.tables[0] + '.docstatus < 2') + + def build_filter_conditions(self): + """build conditions from user filters""" + doclist = {} + for f in self.filters: + if isinstance(f, basestring): + self.conditions.append(f) + else: + f = self.get_filter_tuple(f) + + tname = ('`tab' + f[0] + '`') + if not tname in self.tables: + self.tables.append(tname) + + if not tname in self.meta: + self.load_metadata() + + # prepare in condition + if f[2] in ['in', 'not in']: + opts = ["'" + t.strip().replace("'", "\\'") + "'" for t in f[3].split(',')] + f[3] = "(" + ', '.join(opts) + ")" + self.conditions.append('ifnull(' + tname + '.' + f[1] + ", '') " + f[2] + " " + f[3]) + else: + df = self.meta[f[0]].get({"doctype": "DocField", "fieldname": f[1]}) + + if f[2] == "like" or (isinstance(f[3], basestring) and + (not df or df[0].fieldtype not in ["Float", "Int", "Currency", "Percent"])): + value, default_val = ("'" + f[3].replace("'", "\\'") + "'"), '""' + else: + value, default_val = flt(f[3]), 0 + + self.conditions.append('ifnull({tname}.{fname}, {default_val}) {operator} {value}'.format( + tname=tname, fname=f[1], default_val=default_val, operator=f[2], + value=value)) + + def get_filter_tuple(self, f): + if isinstance(f, dict): + key, value = f.items()[0] + f = self.make_filter_tuple(key, value) + + if not isinstance(f, (list, tuple)): + frappe.throw("Filter must be a tuple or list (in a list)") + + if len(f) != 4: + frappe.throw("Filter must have 4 values (doctype, fieldname, condition, value): " + str(f)) + + return f + + def build_match_conditions(self, as_condition=True): + """add match conditions if applicable""" + self.match_filters = {} + self.match_conditions = [] + self.or_conditions = [] + + if not self.tables: self.extract_tables() + if not self.meta: self.load_metadata() + + # explict permissions + restricted_by_user = frappe.permissions.get_user_perms(self.meta[self.doctype]).restricted + + # get restrictions + restrictions = frappe.defaults.get_restrictions() + + if restricted_by_user: + self.or_conditions.append('`tab{doctype}`.`owner`="{user}"'.format(doctype=self.doctype, + user=frappe.local.session.user)) + self.match_filters["owner"] = frappe.session.user + + if restrictions: + self.add_restrictions(restrictions) + + if as_condition: + return self.build_match_condition_string() + else: + return self.match_filters + + def add_restrictions(self, restrictions): + fields_to_check = self.meta[self.doctype].get_restricted_fields(restrictions.keys()) + if self.doctype in restrictions: + fields_to_check.append(frappe._dict({"fieldname":"name", "options":self.doctype})) + + # check in links + for df in fields_to_check: + self.match_conditions.append('`tab{doctype}`.{fieldname} in ({values})'.format(doctype=self.doctype, + fieldname=df.fieldname, + values=", ".join([('"'+v.replace('"', '\"')+'"') \ + for v in restrictions[df.options]]))) + self.match_filters.setdefault(df.fieldname, []) + self.match_filters[df.fieldname]= restrictions[df.options] + + def build_match_condition_string(self): + conditions = " and ".join(self.match_conditions) + doctype_conditions = self.get_permission_query_conditions() + if doctype_conditions: + conditions += ' and ' + doctype_conditions if conditions else doctype_conditions + + if self.or_conditions: + if conditions: + conditions = '({conditions}) or {or_conditions}'.format(conditions=conditions, + or_conditions = ' or '.join(self.or_conditions)) + else: + conditions = " or ".join(self.or_conditions) + + return conditions + + def get_permission_query_conditions(self): + condition_methods = frappe.get_hooks("permission_query_conditions:" + self.doctype) + if condition_methods: + conditions = [] + for method in condition_methods: + c = frappe.get_attr(method)() + if c: + conditions.append(c) + + return " and ".join(conditions) if conditions else None + + def run_custom_query(self, query): + if '%(key)s' in query: + query = query.replace('%(key)s', 'name') + return frappe.db.sql(query, as_dict = (not self.as_list)) + + def check_sort_by_table(self, order_by): + if "." in order_by: + tbl = order_by.split('.')[0] + if tbl not in self.tables: + if tbl.startswith('`'): + tbl = tbl[4:-1] + frappe.throw("Please select atleast 1 column from '%s' to sort" % tbl) + + def add_limit(self): + if self.limit_page_length: + return 'limit %s, %s' % (self.limit_start, self.limit_page_length) + else: + return '' diff --git a/frappe/tests/test_db_query.py b/frappe/tests/test_db_query.py new file mode 100644 index 0000000000..f211d547a1 --- /dev/null +++ b/frappe/tests/test_db_query.py @@ -0,0 +1,31 @@ +# Copyright (c) 2013, Web Notes Technologies Pvt. Ltd. and Contributors +# MIT License. See license.txt + +import frappe, unittest + +from frappe.model.db_query import DatabaseQuery + +class TestReportview(unittest.TestCase): + def test_basic(self): + self.assertTrue({"name":"DocType"} in DatabaseQuery("DocType").execute()) + + def test_fields(self): + self.assertTrue({"name":"DocType", "issingle":0} \ + in DatabaseQuery("DocType").execute(fields=["name", "issingle"])) + + def test_filters_1(self): + self.assertFalse({"name":"DocType"} \ + in DatabaseQuery("DocType").execute(filters=[["DocType", "name", "like", "J%"]])) + + def test_filters_2(self): + self.assertFalse({"name":"DocType"} \ + in DatabaseQuery("DocType").execute(filters=[{"name": ["like", "J%"]}])) + + def test_filters_3(self): + self.assertFalse({"name":"DocType"} \ + in DatabaseQuery("DocType").execute(filters={"name": ["like", "J%"]})) + + def test_filters_4(self): + self.assertTrue({"name":"DocField"} \ + in DatabaseQuery("DocType").execute(filters={"name": "DocField"})) + \ No newline at end of file diff --git a/frappe/website/doctype/blog_post/test_blog_post.py b/frappe/website/doctype/blog_post/test_blog_post.py index bb81b59320..167345dfc4 100644 --- a/frappe/website/doctype/blog_post/test_blog_post.py +++ b/frappe/website/doctype/blog_post/test_blog_post.py @@ -69,8 +69,7 @@ class TestBlogPost(unittest.TestCase): def test_restriction_in_report(self): frappe.defaults.add_default("Blog Category", "_Test Blog Category 1", "test1@example.com", "Restriction") - frappe.local.reportview_doctypes = {} - + names = [d.name for d in frappe.get_list("Blog Post", fields=["name", "blog_category"])] self.assertTrue("_test-blog-post-1" in names) diff --git a/frappe/widgets/reportview.py b/frappe/widgets/reportview.py index be62217a47..a2a193cb20 100644 --- a/frappe/widgets/reportview.py +++ b/frappe/widgets/reportview.py @@ -5,13 +5,19 @@ from __future__ import unicode_literals """build query for doclistview and return results""" import frappe, json -import frappe.defaults import frappe.permissions +from frappe.model.db_query import DatabaseQuery @frappe.whitelist() def get(): return compress(execute(**get_form_params())) +def execute(doctype, query=None, filters=None, fields=None, docstatus=None, + group_by=None, order_by=None, limit_start=0, limit_page_length=20, + as_list=False, with_childnames=False, debug=False): + return DatabaseQuery(doctype).execute(query, filters, fields, docstatus, group_by, + order_by, limit_start, limit_page_length, as_list, with_childnames, debug) + def get_form_params(): data = frappe._dict(frappe.local.form_dict) @@ -25,63 +31,7 @@ def get_form_params(): data["docstatus"] = json.loads(data["docstatus"]) return data - -def execute(doctype, query=None, filters=None, fields=None, docstatus=None, - group_by=None, order_by=None, limit_start=0, limit_page_length=20, - as_list=False, with_childnames=False, debug=False): - """ - fields as list ["name", "owner"] or ["tabTask.name", "tabTask.owner"] - filters as list of list [["Task", "name", "=", "TASK00001"]] - """ - - if query: - return run_custom_query(query) - - if not filters: - filters = [] - if isinstance(filters, basestring): - filters = json.loads(filters) - if not docstatus: - docstatus = [] - if not fields: - fields = ["name"] - if isinstance(fields, basestring): - filters = json.loads(fields) - - args = prepare_args(doctype, filters, fields, docstatus, group_by, order_by, with_childnames) - args.limit = add_limit(limit_start, limit_page_length) - - query = """select %(fields)s from %(tables)s where %(conditions)s - %(group_by)s order by %(order_by)s %(limit)s""" % args - - return frappe.db.sql(query, as_dict=not as_list, debug=debug) - -def prepare_args(doctype, filters, fields, docstatus, group_by, order_by, with_childnames): - frappe.local.reportview_tables = get_tables(doctype, fields) - load_doctypes() - remove_user_tags(doctype, fields) - conditions = build_conditions(doctype, fields, filters, docstatus) - - args = frappe._dict() - - if with_childnames: - for t in frappe.local.reportview_tables: - if t != "`tab" + doctype + "`": - fields.append(t + ".name as '%s:name'" % t[4:-1]) - - # query dict - args.tables = ', '.join(frappe.local.reportview_tables) - args.conditions = ' and '.join(conditions) - args.fields = ', '.join(fields) - - args.order_by = order_by or frappe.local.reportview_tables[0] + '.modified desc' - args.group_by = group_by and (" group by " + group_by) or "" - - check_sort_by_table(args.order_by) - - return args - def compress(data): """separate keys and values""" if not data: return data @@ -97,214 +47,8 @@ def compress(data): "keys": keys, "values": values } - -def check_sort_by_table(sort_by): - """check atleast 1 column selected from the sort by table """ - if "." in sort_by: - tbl = sort_by.split('.')[0] - if tbl not in frappe.local.reportview_tables: - if tbl.startswith('`'): - tbl = tbl[4:-1] - frappe.msgprint("Please select atleast 1 column from '%s' to sort"\ - % tbl, raise_exception=1) - -def run_custom_query(query): - """run custom query""" - if '%(key)s' in query: - query = query.replace('%(key)s', 'name') - return frappe.db.sql(query, as_dict=1) - -def load_doctypes(): - """load all doctypes and roles""" - import frappe.model.doctype - - if not getattr(frappe.local, "reportview_doctypes", None): - frappe.local.reportview_doctypes = {} - - for t in frappe.local.reportview_tables: - if t.startswith('`'): - doctype = t[4:-1] - if frappe.local.reportview_doctypes.get(doctype): - continue - - if not frappe.has_permission(doctype): - raise frappe.PermissionError, doctype - frappe.local.reportview_doctypes[doctype] = frappe.model.doctype.get(doctype) - -def remove_user_tags(doctype, fields): - """remove column _user_tags if not in table""" - columns = get_table_columns(doctype) - del_user_tags = False - del_comments = False - for fld in fields: - if '_user_tags' in fld and not "_user_tags" in columns: - del_user_tags = fld - if '_comments' in fld and not "_comments" in columns: - del_comments = fld - - if del_user_tags: del fields[fields.index(del_user_tags)] - if del_comments: del fields[fields.index(del_comments)] - -def add_limit(limit_start, limit_page_length): - if limit_page_length: - return 'limit %s, %s' % (limit_start, limit_page_length) - else: - return '' -def build_conditions(doctype, fields, filters, docstatus): - """build conditions""" - if docstatus: - conditions = [frappe.local.reportview_tables[0] + '.docstatus in (' + ','.join(docstatus) + ')'] - else: - # default condition - conditions = [frappe.local.reportview_tables[0] + '.docstatus < 2'] - - # make conditions from filters - build_filter_conditions(filters, conditions) - - # join parent, child tables - for tname in frappe.local.reportview_tables[1:]: - conditions.append(tname + '.parent = ' + frappe.local.reportview_tables[0] + '.name') - - # match conditions - match_conditions = build_match_conditions(doctype, fields) - if match_conditions: - conditions.append(match_conditions) - - return conditions -def build_filter_conditions(filters, conditions): - """build conditions from user filters""" - from frappe.utils import cstr, flt - if not getattr(frappe.local, "reportview_tables", None): - frappe.local.reportview_tables = [] - - doclist = {} - for f in filters: - if isinstance(f, basestring): - conditions.append(f) - else: - if not isinstance(f, (list, tuple)): - frappe.throw("Filter must be a tuple or list (in a list)") - - if len(f) != 4: - frappe.throw("Filter must have 4 values (doctype, fieldname, condition, value): " + str(f)) - - tname = ('`tab' + f[0] + '`') - if not tname in frappe.local.reportview_tables: - frappe.local.reportview_tables.append(tname) - - if not hasattr(frappe.local, "reportview_doctypes") \ - or not frappe.local.reportview_doctypes.has_key(tname): - load_doctypes() - - # prepare in condition - if f[2] in ['in', 'not in']: - opts = ["'" + t.strip().replace("'", "\\'") + "'" for t in f[3].split(',')] - f[3] = "(" + ', '.join(opts) + ")" - conditions.append('ifnull(' + tname + '.' + f[1] + ", '') " + f[2] + " " + f[3]) - else: - df = frappe.local.reportview_doctypes[f[0]].get({"doctype": "DocField", - "fieldname": f[1]}) - - if f[2] == "like" or (isinstance(f[3], basestring) and - (not df or df[0].fieldtype not in ["Float", "Int", "Currency", "Percent"])): - value, default_val = ("'" + f[3].replace("'", "\\'") + "'"), '""' - else: - value, default_val = flt(f[3]), 0 - - conditions.append('ifnull({tname}.{fname}, {default_val}) {operator} {value}'.format( - tname=tname, fname=f[1], default_val=default_val, operator=f[2], - value=value)) - -def build_match_conditions(doctype, fields=None, as_condition=True): - """add match conditions if applicable""" - import frappe.permissions - match_filters = {} - match_conditions = [] - or_conditions = [] - - if not getattr(frappe.local, "reportview_tables", None): - frappe.local.reportview_tables = get_tables(doctype, fields) - - load_doctypes() - - # is restricted - restricted = frappe.permissions.get_user_perms(frappe.local.reportview_doctypes[doctype]).restricted - - # get restrictions - restrictions = frappe.defaults.get_restrictions() - - if restricted: - or_conditions.append('`tab{doctype}`.`owner`="{user}"'.format(doctype=doctype, - user=frappe.local.session.user)) - match_filters["owner"] = frappe.session.user - - if restrictions: - fields_to_check = frappe.local.reportview_doctypes[doctype].get_restricted_fields(restrictions.keys()) - if doctype in restrictions: - fields_to_check.append(frappe._dict({"fieldname":"name", "options":doctype})) - - # check in links - for df in fields_to_check: - if as_condition: - match_conditions.append('`tab{doctype}`.{fieldname} in ({values})'.format(doctype=doctype, - fieldname=df.fieldname, - values=", ".join([('"'+v.replace('"', '\"')+'"') \ - for v in restrictions[df.options]]))) - else: - match_filters.setdefault(df.fieldname, []) - match_filters[df.fieldname]= restrictions[df.options] - - if as_condition: - conditions = " and ".join(match_conditions) - doctype_conditions = get_permission_query_conditions(doctype) - if doctype_conditions: - conditions += ' and ' + doctype_conditions if conditions else doctype_conditions - - if or_conditions: - if conditions: - conditions = '({conditions}) or {or_conditions}'.format(conditions=conditions, - or_conditions = ' or '.join(or_conditions)) - else: - conditions = " or ".join(or_conditions) - - return conditions - else: - return match_filters - -def get_permission_query_conditions(doctype): - condition_methods = frappe.get_hooks("permission_query_conditions:" + doctype) - if condition_methods: - conditions = [] - for method in condition_methods: - c = frappe.get_attr(method)() - if c: - conditions.append(c) - - return " and ".join(conditions) if conditions else None - -def get_tables(doctype, fields): - """extract tables from fields""" - tables = ['`tab' + doctype + '`'] - - # add tables from fields - if fields: - for f in fields: - if "." not in f: continue - - table_name = f.split('.')[0] - if table_name.lower().startswith('group_concat('): - table_name = table_name[13:] - if table_name.lower().startswith('ifnull('): - table_name = table_name[7:] - if not table_name[0]=='`': - table_name = '`' + table_name + '`' - if not table_name in tables: - tables.append(table_name) - - return tables - @frappe.whitelist() def save_report(): """save report""" @@ -359,13 +103,13 @@ def export_query(): f.seek(0) frappe.response['result'] = unicode(f.read(), 'utf-8') frappe.response['type'] = 'csv' - frappe.response['doctype'] = [t[4:-1] for t in frappe.local.reportview_tables][0] + frappe.response['doctype'] = [t[4:-1] for t in self.tables][0] def get_labels(columns): """get column labels based on column names""" label_dict = {} - for doctype in frappe.local.reportview_doctypes: - for d in frappe.local.reportview_doctypes[doctype]: + for doctype in self.meta: + for d in self.meta[doctype]: if d.doctype=='DocField' and d.fieldname: label_dict[d.fieldname] = d.label @@ -397,7 +141,7 @@ def get_stats(stats, doctype): tags = json.loads(stats) stats = {} - columns = get_table_columns(doctype) + columns = frappe.db.get_table_columns(doctype) for tag in tags: if not tag in columns: continue tagcount = execute(doctype, fields=[tag, "count(*)"], @@ -429,16 +173,10 @@ def scrub_user_tags(tagcount): return rlist -def get_table_columns(table): - res = frappe.db.sql("DESC `tab%s`" % table, as_dict=1) - if res: return [r['Field'] for r in res] - # used in building query in queries.py -def get_match_cond(doctype, searchfield = 'name'): - cond = build_match_conditions(doctype) - - if cond: - cond = ' and ' + cond - else: - cond = '' - return cond +def get_match_cond(doctype): + cond = DatabaseQuery(doctype).build_match_conditions() + return (' and ' + cond) if cond else "" + +def build_match_conditions(doctype, as_condition=True): + return DatabaseQuery(doctype).build_match_conditions(as_condition=as_condition)