This is a test email
+This is a test email
''' self.assertTrue(transformed_html in inline_style_in_html(html)) diff --git a/frappe/hooks.py b/frappe/hooks.py index ac42a03461..f3d25d6bf4 100644 --- a/frappe/hooks.py +++ b/frappe/hooks.py @@ -171,6 +171,9 @@ doc_events = { "frappe.workflow.doctype.workflow_action.workflow_action.process_workflow_actions", "frappe.event_streaming.doctype.event_update_log.event_update_log.notify_consumers" ], + "on_update_after_submit": [ + "frappe.workflow.doctype.workflow_action.workflow_action.process_workflow_actions" + ], "on_change": [ "frappe.social.doctype.energy_point_rule.energy_point_rule.process_energy_points", "frappe.automation.doctype.milestone_tracker.milestone_tracker.evaluate_milestone" diff --git a/frappe/model/__init__.py b/frappe/model/__init__.py index f63c76d2da..362f4c79b3 100644 --- a/frappe/model/__init__.py +++ b/frappe/model/__init__.py @@ -153,32 +153,22 @@ def delete_fields(args_dict, delete=0): if not fields: continue - frappe.db.sql(""" - DELETE FROM `tabDocField` - WHERE parent='%s' AND fieldname IN (%s) - """ % (dt, ", ".join(["'{}'".format(f) for f in fields]))) + frappe.db.delete("DocField", { + "parent": dt, + "fieldname": ("in", fields), + }) # Delete the data/column only if delete is specified if not delete: continue if frappe.db.get_value("DocType", dt, "issingle"): - frappe.db.sql(""" - DELETE FROM `tabSingles` - WHERE doctype='%s' AND field IN (%s) - """ % (dt, ", ".join("'{}'".format(f) for f in fields))) + frappe.db.delete("Singles", { + "doctype": dt, + "field": ("in", fields), + }) else: - existing_fields = frappe.db.multisql({ - "mariadb": "DESC `tab%s`" % dt, - "postgres": """ - SELECT - COLUMN_NAME - FROM - information_schema.COLUMNS - WHERE - TABLE_NAME = 'tab%s'; - """ % dt, - }) + existing_fields = frappe.db.describe(dt) existing_fields = existing_fields and [e[0] for e in existing_fields] or [] fields_need_to_delete = set(fields) & set(existing_fields) if not fields_need_to_delete: diff --git a/frappe/model/delete_doc.py b/frappe/model/delete_doc.py index cc88cfa106..fbbf1a4852 100644 --- a/frappe/model/delete_doc.py +++ b/frappe/model/delete_doc.py @@ -65,12 +65,12 @@ def delete_doc(doctype=None, name=None, force=0, ignore_doctypes=None, for_reloa update_flags(doc, flags, ignore_permissions) check_permission_and_not_submitted(doc) - frappe.db.sql("delete from `tabCustom Field` where dt = %s", name) - frappe.db.sql("delete from `tabClient Script` where dt = %s", name) - frappe.db.sql("delete from `tabProperty Setter` where doc_type = %s", name) - frappe.db.sql("delete from `tabReport` where ref_doctype=%s", name) - frappe.db.sql("delete from `tabCustom DocPerm` where parent=%s", name) - frappe.db.sql("delete from `__global_search` where doctype=%s", name) + frappe.db.delete("Custom Field", {"dt": name}) + frappe.db.delete("Client Script", {"dt": name}) + frappe.db.delete("Property Setter", {"doc_type": name}) + frappe.db.delete("Report", {"ref_doctype": name}) + frappe.db.delete("Custom DocPerm", {"parent": name}) + frappe.db.delete("__global_search", {"doctype": name}) delete_from_table(doctype, name, ignore_doctypes, None) @@ -162,10 +162,9 @@ def update_naming_series(doc): def delete_from_table(doctype, name, ignore_doctypes, doc): if doctype!="DocType" and doctype==name: - frappe.db.sql("delete from `tabSingles` where `doctype`=%s", name) + frappe.db.delete("Singles", {"doctype": name}) else: - frappe.db.sql("delete from `tab{0}` where `name`=%s".format(doctype), name) - + frappe.db.delete(doctype, {"name": name}) # get child tables if doc: tables = [d.options for d in doc.meta.get_table_fields()] @@ -339,8 +338,10 @@ def clear_references(doctype, reference_doctype, reference_name, (reference_doctype, reference_name)) def clear_timeline_references(link_doctype, link_name): - frappe.db.sql("""DELETE FROM `tabCommunication Link` - WHERE `tabCommunication Link`.link_doctype=%s AND `tabCommunication Link`.link_name=%s""", (link_doctype, link_name)) + frappe.db.delete("Communication Link", { + "link_doctype": link_doctype, + "link_name": link_name + }) def insert_feed(doc): if ( diff --git a/frappe/model/document.py b/frappe/model/document.py index 61160e1f01..b44d95716e 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -390,10 +390,11 @@ class Document(BaseDocument): else: # no rows found, delete all rows - frappe.db.sql("""delete from `tab{0}` where parent=%s - and parenttype=%s and parentfield=%s""".format(df.options), - (self.name, self.doctype, fieldname)) - + frappe.db.delete(df.options, { + "parent": self.name, + "parenttype": self.doctype, + "parentfield": fieldname + }) def get_doc_before_save(self): return getattr(self, '_doc_before_save', None) @@ -451,7 +452,9 @@ class Document(BaseDocument): def update_single(self, d): """Updates values for Single type Document in `tabSingles`.""" - frappe.db.sql("""delete from `tabSingles` where doctype=%s""", self.doctype) + frappe.db.delete("Singles", { + "doctype": self.doctype + }) for field, value in d.items(): if field != "doctype": frappe.db.sql("""insert into `tabSingles` (doctype, field, value) diff --git a/frappe/parallel_test_runner.py b/frappe/parallel_test_runner.py index 2f83b88572..6265498c96 100644 --- a/frappe/parallel_test_runner.py +++ b/frappe/parallel_test_runner.py @@ -111,33 +111,16 @@ class ParallelTestRunner(): if self.with_coverage: from coverage import Coverage from frappe.utils import get_bench_path + from frappe.coverage import STANDARD_INCLUSIONS, STANDARD_EXCLUSIONS, FRAPPE_EXCLUSIONS # Generate coverage report only for app that is being tested source_path = os.path.join(get_bench_path(), 'apps', self.app) - incl = [ - '*.py', - ] - omit = [ - '*.js', - '*.xml', - '*.pyc', - '*.css', - '*.less', - '*.scss', - '*.vue', - '*.pyc', - '*.html', - '*/test_*', - '*/node_modules/*', - '*/doctype/*/*_dashboard.py', - '*/patches/*', - ] + omit = STANDARD_EXCLUSIONS[:] if self.app == 'frappe': - omit.append('*/tests/*') - omit.append('*/commands/*') + omit.extend(FRAPPE_EXCLUSIONS) - self.coverage = Coverage(source=[source_path], omit=omit, include=incl) + self.coverage = Coverage(source=[source_path], omit=omit, include=STANDARD_INCLUSIONS) self.coverage.start() def save_coverage(self): diff --git a/frappe/patches.txt b/frappe/patches.txt index 7605d8ea2b..493c4dc9f6 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -180,3 +180,4 @@ frappe.patches.v12_0.rename_uploaded_files_with_proper_name frappe.patches.v13_0.queryreport_columns frappe.patches.v13_0.jinja_hook frappe.patches.v13_0.update_notification_channel_if_empty +frappe.patches.v14_0.drop_data_import_legacy diff --git a/frappe/patches/v11_0/apply_customization_to_custom_doctype.py b/frappe/patches/v11_0/apply_customization_to_custom_doctype.py index 49b68ed240..7e84c5ae24 100644 --- a/frappe/patches/v11_0/apply_customization_to_custom_doctype.py +++ b/frappe/patches/v11_0/apply_customization_to_custom_doctype.py @@ -28,7 +28,7 @@ def execute(): for prop in property_setters: property_setter_map[prop.field_name] = prop - frappe.db.sql('DELETE FROM `tabProperty Setter` WHERE `name`=%s', prop.name) + frappe.db.delete("Property Setter", {"name": prop.name}) meta = frappe.get_meta(doctype.name) @@ -50,6 +50,6 @@ def execute(): df = frappe.new_doc('DocField', meta, 'fields') df.update(cf) meta.fields.append(df) - frappe.db.sql('DELETE FROM `tabCustom Field` WHERE name=%s', cf.name) + frappe.db.delete("Custom Field", {"name": cf.name}) meta.save() diff --git a/frappe/patches/v11_0/sync_stripe_settings_before_migrate.py b/frappe/patches/v11_0/sync_stripe_settings_before_migrate.py index a8e9bd4de1..901ab66bfd 100644 --- a/frappe/patches/v11_0/sync_stripe_settings_before_migrate.py +++ b/frappe/patches/v11_0/sync_stripe_settings_before_migrate.py @@ -17,4 +17,4 @@ def execute(): settings.secret_key = secret_key settings.save(ignore_permissions=True) - frappe.db.sql("""DELETE FROM tabSingles WHERE doctype='Stripe Settings'""") \ No newline at end of file + frappe.db.delete("Singles", {"doctype": "Stripe Settings"}) diff --git a/frappe/patches/v12_0/delete_feedback_request_if_exists.py b/frappe/patches/v12_0/delete_feedback_request_if_exists.py index fdbcecfc5a..c1bf46b14a 100644 --- a/frappe/patches/v12_0/delete_feedback_request_if_exists.py +++ b/frappe/patches/v12_0/delete_feedback_request_if_exists.py @@ -2,7 +2,4 @@ import frappe def execute(): - frappe.db.sql(''' - DELETE from `tabDocType` - WHERE name = 'Feedback Request' - ''') \ No newline at end of file + frappe.db.delete("DocType", {"name": "Feedback Request"}) diff --git a/frappe/patches/v12_0/remove_deprecated_fields_from_doctype.py b/frappe/patches/v12_0/remove_deprecated_fields_from_doctype.py index 60599066e6..9c9a79ccbf 100644 --- a/frappe/patches/v12_0/remove_deprecated_fields_from_doctype.py +++ b/frappe/patches/v12_0/remove_deprecated_fields_from_doctype.py @@ -8,7 +8,6 @@ def execute(): 'DocType': ['hide_heading', 'image_view', 'read_only_onload'] }, delete=1) - frappe.db.sql(''' - DELETE from `tabProperty Setter` - WHERE property = 'read_only_onload' - ''') + frappe.db.delete("Property Setter", { + "property": "read_only_onload" + }) \ No newline at end of file diff --git a/frappe/patches/v12_0/set_correct_assign_value_in_docs.py b/frappe/patches/v12_0/set_correct_assign_value_in_docs.py index 65a635c170..90766b5f64 100644 --- a/frappe/patches/v12_0/set_correct_assign_value_in_docs.py +++ b/frappe/patches/v12_0/set_correct_assign_value_in_docs.py @@ -1,32 +1,29 @@ import frappe +from frappe.query_builder.functions import GroupConcat, Coalesce def execute(): - frappe.reload_doc('desk', 'doctype', 'todo') + frappe.reload_doc("desk", "doctype", "todo") - query = ''' - SELECT - name, reference_type, reference_name, {} as assignees - FROM - `tabToDo` - WHERE - COALESCE(reference_type, '') != '' AND - COALESCE(reference_name, '') != '' AND - status != 'Cancelled' - GROUP BY - reference_type, reference_name - ''' + ToDo = frappe.qb.Table("ToDo") + assignees = GroupConcat("owner").distinct().as_("assignees") - assignments = frappe.db.multisql({ - 'mariadb': query.format('GROUP_CONCAT(DISTINCT `owner`)'), - 'postgres': query.format('STRING_AGG(DISTINCT "owner", ",")') - }, as_dict=True) + query = ( + frappe.qb.from_(ToDo) + .select(ToDo.name, ToDo.reference_type, assignees) + .where(Coalesce(ToDo.reference_type, "") != "") + .where(Coalesce(ToDo.reference_name, "") != "") + .where(ToDo.status != "Cancelled") + .groupby(ToDo.reference_type, ToDo.reference_name) + ) + + assignments = frappe.db.sql(query, as_dict=True) for doc in assignments: - assignments = doc.assignees.split(',') + assignments = doc.assignees.split(",") frappe.db.set_value( doc.reference_type, doc.reference_name, - '_assign', + "_assign", frappe.as_json(assignments), update_modified=False - ) + ) \ No newline at end of file diff --git a/frappe/patches/v12_0/set_primary_key_in_series.py b/frappe/patches/v12_0/set_primary_key_in_series.py index e5ed2204ba..83a903fc2d 100644 --- a/frappe/patches/v12_0/set_primary_key_in_series.py +++ b/frappe/patches/v12_0/set_primary_key_in_series.py @@ -1,21 +1,24 @@ import frappe def execute(): - #if current = 0, simply delete the key as it'll be recreated on first entry - frappe.db.sql('delete from `tabSeries` where current = 0') - duplicate_keys = frappe.db.sql(''' - SELECT name, max(current) as current - from - `tabSeries` - group by - name - having count(name) > 1 - ''', as_dict=True) - for row in duplicate_keys: - frappe.db.sql('delete from `tabSeries` where name = %(key)s', { - 'key': row.name - }) - if row.current: - frappe.db.sql('insert into `tabSeries`(`name`, `current`) values (%(name)s, %(current)s)', row) - frappe.db.commit() - frappe.db.sql('ALTER table `tabSeries` ADD PRIMARY KEY IF NOT EXISTS (name)') + #if current = 0, simply delete the key as it'll be recreated on first entry + frappe.db.delete("Series", {"current": 0}) + + duplicate_keys = frappe.db.sql(''' + SELECT name, max(current) as current + from + `tabSeries` + group by + name + having count(name) > 1 + ''', as_dict=True) + + for row in duplicate_keys: + frappe.db.delete("Series", { + "name": row.name + }) + if row.current: + frappe.db.sql('insert into `tabSeries`(`name`, `current`) values (%(name)s, %(current)s)', row) + frappe.db.commit() + + frappe.db.sql('ALTER table `tabSeries` ADD PRIMARY KEY IF NOT EXISTS (name)') diff --git a/frappe/patches/v12_0/setup_comments_from_communications.py b/frappe/patches/v12_0/setup_comments_from_communications.py index 039ceeff35..11e02965f1 100644 --- a/frappe/patches/v12_0/setup_comments_from_communications.py +++ b/frappe/patches/v12_0/setup_comments_from_communications.py @@ -29,4 +29,6 @@ def execute(): frappe.db.auto_commit_on_many_writes = False # clean up - frappe.db.sql("delete from `tabCommunication` where communication_type = 'Comment'") + frappe.db.delete("Communication", { + "communication_type": "Comment" + }) diff --git a/frappe/patches/v13_0/increase_password_length.py b/frappe/patches/v13_0/increase_password_length.py index 1bb1979051..62ca2ed779 100644 --- a/frappe/patches/v13_0/increase_password_length.py +++ b/frappe/patches/v13_0/increase_password_length.py @@ -1,7 +1,4 @@ import frappe def execute(): - frappe.db.multisql({ - "mariadb": "ALTER TABLE `__Auth` MODIFY `password` TEXT NOT NULL", - "postgres": 'ALTER TABLE "__Auth" ALTER COLUMN "password" TYPE TEXT' - }) + frappe.db.change_column_type(table="__Auth", column="password", type="TEXT") diff --git a/frappe/patches/v13_0/remove_twilio_settings.py b/frappe/patches/v13_0/remove_twilio_settings.py index 363cbdd4b6..7efaf876e2 100644 --- a/frappe/patches/v13_0/remove_twilio_settings.py +++ b/frappe/patches/v13_0/remove_twilio_settings.py @@ -12,7 +12,9 @@ def execute(): frappe.delete_doc_if_exists('DocType', 'Twilio Number Group') if twilio_settings_doctype_in_integrations(): frappe.delete_doc_if_exists('DocType', 'Twilio Settings') - frappe.db.sql("delete from `tabSingles` where `doctype`=%s", 'Twilio Settings') + frappe.db.delete("Singles", { + "doctype": "Twilio Settings" + }) def twilio_settings_doctype_in_integrations() -> bool: """Check Twilio Settings doctype exists in integrations module or not. diff --git a/frappe/patches/v14_0/__init__.py b/frappe/patches/v14_0/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/frappe/patches/v14_0/drop_data_import_legacy.py b/frappe/patches/v14_0/drop_data_import_legacy.py new file mode 100644 index 0000000000..2037930c9f --- /dev/null +++ b/frappe/patches/v14_0/drop_data_import_legacy.py @@ -0,0 +1,22 @@ +import frappe +import click + + +def execute(): + doctype = "Data Import Legacy" + table = frappe.utils.get_table_name(doctype) + + # delete the doctype record to avoid broken links + frappe.db.delete("DocType", {"name": doctype}) + + # leaving table in database for manual cleanup + click.secho( + f"`{doctype}` has been deprecated. The DocType is deleted, but the data still" + " exists on the database. If this data is worth recovering, you may export it" + f" using\n\n\tbench --site {frappe.local.site} backup -i '{doctype}'\n\nAfter" + " this, the table will continue to persist in the database, until you choose" + " to remove it yourself. If you want to drop the table, you may run\n\n\tbench" + f" --site {frappe.local.site} execute frappe.db.sql --args \"('DROP TABLE IF" + f" EXISTS `{table}`', )\"\n", + fg="yellow", + ) diff --git a/frappe/permissions.py b/frappe/permissions.py index 07b4a2e68f..33aef4ab41 100644 --- a/frappe/permissions.py +++ b/frappe/permissions.py @@ -7,9 +7,11 @@ import frappe.share from frappe import _, msgprint from frappe.utils import cint + rights = ("select", "read", "write", "create", "delete", "submit", "cancel", "amend", "print", "email", "report", "import", "export", "set_user_permissions", "share") + def check_admin_or_system_manager(user=None): if not user: user = frappe.session.user @@ -516,8 +518,7 @@ def reset_perms(doctype): """Reset permissions for given doctype.""" from frappe.desk.notifications import delete_notification_count_for delete_notification_count_for(doctype) - - frappe.db.sql("""delete from `tabCustom DocPerm` where parent=%s""", doctype) + frappe.db.delete("Custom DocPerm", {"parent": doctype}) def get_linked_doctypes(dt): return list(set([dt] + [d.options for d in diff --git a/frappe/printing/page/print/print.js b/frappe/printing/page/print/print.js index 908479fd02..ca2a340661 100644 --- a/frappe/printing/page/print/print.js +++ b/frappe/printing/page/print/print.js @@ -403,19 +403,14 @@ frappe.ui.form.PrintView = class { setup_print_format_dom(out, $print_format) { this.print_wrapper.find('.print-format-skeleton').remove(); let base_url = frappe.urllib.get_base_url(); - let print_css = frappe.assets.bundled_asset('print.bundle.css'); + let print_css = frappe.assets.bundled_asset('print.bundle.css', frappe.utils.is_rtl(this.lang_code)); + this.$print_format_body.find('html').attr('dir', frappe.utils.is_rtl(this.lang_code) ? 'rtl': 'ltr'); + this.$print_format_body.find('html').attr('lang', this.lang_code); this.$print_format_body.find('head').html( ` ` ); - if (frappe.utils.is_rtl(this.lang_code)) { - let rtl_css = frappe.assets.bundled_asset('frappe-rtl.bundle.css'); - this.$print_format_body.find('head').append( - `` - ); - } - this.$print_format_body.find('body').html( `| # | {% for col in columns %} {% if col.name && col._id !== "_check" %}{% for row in data %} |
|---|---|
| + {{ row._index + 1 }} + | {% for col in columns %} {% if col.name && col._id !== "_check" %} diff --git a/frappe/public/js/frappe/views/reports/query_report.js b/frappe/public/js/frappe/views/reports/query_report.js index 208d5b2f67..a4b3564e37 100644 --- a/frappe/public/js/frappe/views/reports/query_report.js +++ b/frappe/public/js/frappe/views/reports/query_report.js @@ -1264,7 +1264,9 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList { print_css: print_css, print_settings: print_settings, landscape: landscape, - columns: columns + columns: columns, + lang: frappe.boot.lang, + layout_direction: frappe.utils.is_rtl() ? "rtl" : "ltr" }); frappe.render_pdf(html, print_settings); diff --git a/frappe/public/scss/common/controls.scss b/frappe/public/scss/common/controls.scss index 4d66797bba..a10cd454a6 100644 --- a/frappe/public/scss/common/controls.scss +++ b/frappe/public/scss/common/controls.scss @@ -167,6 +167,9 @@ select.form-control { .ace_print-margin { background-color: var(--dark-border-color); } + .ace_scrollbar { + z-index: 3; + } } .frappe-control[data-fieldtype="Attach"], diff --git a/frappe/public/scss/common/css_variables.scss b/frappe/public/scss/common/css_variables.scss index 48a8a48f5f..112238bfe5 100644 --- a/frappe/public/scss/common/css_variables.scss +++ b/frappe/public/scss/common/css_variables.scss @@ -179,9 +179,20 @@ --text-on-pink: var(--pink-500); --text-on-cyan: var(--cyan-600); + // Layout Colors + --bg-color: var(--gray-50); + --fg-color: white; + --navbar-bg: white; + --fg-hover-color: var(--gray-100); + --card-bg: var(--fg-color); + --disabled-text-color: var(--gray-700); --disabled-control-bg: var(--gray-50); --control-bg: var(--gray-100); --control-bg-on-gray: var(--gray-200); + --awesomebar-focus-bg: var(--fg-color); + --modal-bg: white; + --toast-bg: var(--modal-bg); + --popover-bg: white; --awesomplete-hover-bg: var(--control-bg); diff --git a/frappe/public/scss/desk/breadcrumb.scss b/frappe/public/scss/desk/breadcrumb.scss index 6324e6f012..b466bab7ae 100644 --- a/frappe/public/scss/desk/breadcrumb.scss +++ b/frappe/public/scss/desk/breadcrumb.scss @@ -4,7 +4,7 @@ background-color: white; font-size: var(--text-sm); } - +/*! This comment will be included even in compressed mode. */ #navbar-breadcrumbs { margin-left: var(--margin-md); font-size: var(--text-sm); @@ -12,7 +12,7 @@ font-size: var(--text-md); margin-right: 10px; &:before { - content: var(--right-arrow-svg); + content: #{"/*!rtl:var(--left-arrow-svg);*/"}var(--right-arrow-svg); display: inline-block; margin-right: 10px; } diff --git a/frappe/public/scss/desk/css_variables.scss b/frappe/public/scss/desk/css_variables.scss index 5bb2614dcc..255d0a91e4 100644 --- a/frappe/public/scss/desk/css_variables.scss +++ b/frappe/public/scss/desk/css_variables.scss @@ -25,20 +25,7 @@ $input-height: 28px !default; --navbar-height: 60px; - // Layout Colors - --bg-color: var(--gray-50); - --fg-color: white; - --navbar-bg: white; - --fg-hover-color: var(--gray-100); - --card-bg: var(--fg-color); - --disabled-text-color: var(--gray-700); - --disabled-control-bg: var(--gray-50); - --control-bg: var(--gray-100); - --control-bg-on-gray: var(--gray-200); - --awesomebar-focus-bg: var(--fg-color); - --modal-bg: white; - --toast-bg: var(--modal-bg); - --popover-bg: white; + --appreciation-color: var(--dark-green-600); --appreciation-bg: var(--dark-green-100); @@ -77,4 +64,6 @@ $input-height: 28px !default; --skeleton-bg: var(--gray-100); --right-arrow-svg: url("data: image/svg+xml;utf8, "); + + --left-arrow-svg: url("data: image/svg+xml;utf8, "); } diff --git a/frappe/public/scss/desk/dark.scss b/frappe/public/scss/desk/dark.scss index 76dcf90bc3..7f0dfe73b8 100644 --- a/frappe/public/scss/desk/dark.scss +++ b/frappe/public/scss/desk/dark.scss @@ -157,4 +157,6 @@ --skeleton-bg: var(--gray-800); --right-arrow-svg: url("data: image/svg+xml;utf8, "); + + --left-arrow-svg: url("data: image/svg+xml;utf8, "); } diff --git a/frappe/public/scss/desk/global.scss b/frappe/public/scss/desk/global.scss index afcf2957cc..d1205e0e38 100644 --- a/frappe/public/scss/desk/global.scss +++ b/frappe/public/scss/desk/global.scss @@ -574,3 +574,15 @@ details > summary:focus { // font-family: 'Octicons'; // content: "\f00b"; // } + +/*rtl:raw: +.dropdown-menu { + right: auto; +} +.popover { + right: auto; +} +.chart-container { + direction: ltr; +} +*/ \ No newline at end of file diff --git a/frappe/public/scss/email.bundle.scss b/frappe/public/scss/email.bundle.scss index b1a8d6c485..9b58c130a2 100644 --- a/frappe/public/scss/email.bundle.scss +++ b/frappe/public/scss/email.bundle.scss @@ -36,7 +36,13 @@ a { } p { - margin: 5px 0 !important; + margin: 1em 0 !important; +} + +.with-container { + p { + margin: 5px 0 !important; + } } .ql-editor { diff --git a/frappe/public/scss/frappe-rtl.bundle.scss b/frappe/public/scss/frappe-rtl.bundle.scss deleted file mode 100644 index bc618270a8..0000000000 --- a/frappe/public/scss/frappe-rtl.bundle.scss +++ /dev/null @@ -1,3 +0,0 @@ -@import "frappe/public/css/bootstrap-rtl.css"; -@import "frappe/public/css/desk-rtl.css"; -@import "frappe/public/css/report-rtl.css"; diff --git a/frappe/pythonrc.py b/frappe/pythonrc.py deleted file mode 100755 index 6761ead05b..0000000000 --- a/frappe/pythonrc.py +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env python2.7 - -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors -# MIT License. See license.txt - -import os -import frappe -frappe.connect(site=os.environ.get("site")) \ No newline at end of file diff --git a/frappe/query_builder/__init__.py b/frappe/query_builder/__init__.py new file mode 100644 index 0000000000..798c34b6cc --- /dev/null +++ b/frappe/query_builder/__init__.py @@ -0,0 +1 @@ +from frappe.query_builder.utils import get_query_builder diff --git a/frappe/query_builder/builder.py b/frappe/query_builder/builder.py new file mode 100644 index 0000000000..da1533fb1a --- /dev/null +++ b/frappe/query_builder/builder.py @@ -0,0 +1,55 @@ +from pypika import MySQLQuery, Order, PostgreSQLQuery, terms +from pypika.queries import Schema, Table +from frappe.utils import get_table_name + + +class Base: + terms = terms + desc = Order.desc + Schema = Schema + + @staticmethod + def Table(table_name: str, *args, **kwargs) -> Table: + table_name = get_table_name(table_name) + return Table(table_name, *args, **kwargs) + + +class MariaDB(Base, MySQLQuery): + Field = terms.Field + + @classmethod + def from_(cls, table, *args, **kwargs): + if isinstance(table, str): + table = cls.Table(table) + return super().from_(table, *args, **kwargs) + + +class Postgres(Base, PostgreSQLQuery): + field_translation = {"table_name": "relname", "table_rows": "n_tup_ins"} + schema_translation = {"tables": "pg_stat_all_tables"} + # TODO: Find a better way to do this + # These are interdependent query changes that need fixing. These + # translations happen in the same query. But there is no check to see if + # the Fields are changed only when a particular `information_schema` schema + # is used. Replacing them is not straightforward because the "from_" + # function can not see the arguments passed to the "select" function as + # they are two different objects. The quick fix used here is to replace the + # Field names in the "Field" function. + + @classmethod + def Field(cls, field_name, *args, **kwargs): + if field_name in cls.field_translation: + field_name = cls.field_translation[field_name] + return terms.Field(field_name, *args, **kwargs) + + @classmethod + def from_(cls, table, *args, **kwargs): + if isinstance(table, Table): + if table._schema: + if table._schema._name == "information_schema": + table = cls.schema_translation[table._table_name] + + elif isinstance(table, str): + table = cls.Table(table) + + return super().from_(table, *args, **kwargs) diff --git a/frappe/query_builder/custom.py b/frappe/query_builder/custom.py new file mode 100644 index 0000000000..5aaed463d9 --- /dev/null +++ b/frappe/query_builder/custom.py @@ -0,0 +1,83 @@ +from typing import Optional + +from pypika.functions import DistinctOptionFunction +from pypika.utils import builder + +import frappe + + +class GROUP_CONCAT(DistinctOptionFunction): + def __init__(self, column: str, alias: Optional[str] = None): + """[ Implements the group concat function read more about it at https://www.geeksforgeeks.org/mysql-group_concat-function ] + Args: + column (str): [ name of the column you want to concat] + alias (Optional[str], optional): [ is this an alias? ]. Defaults to None. + """ + super(GROUP_CONCAT, self).__init__("GROUP_CONCAT", column, alias=alias) + + +class STRING_AGG(DistinctOptionFunction): + def __init__(self, column: str, separator: str = ",", alias: Optional[str] = None): + """[ Implements the group concat function read more about it at https://docs.microsoft.com/en-us/sql/t-sql/functions/string-agg-transact-sql?view=sql-server-ver15 ] + + Args: + column (str): [ name of the column you want to concat ] + separator (str, optional): [separator to be used]. Defaults to ",". + alias (Optional[str], optional): [description]. Defaults to None. + """ + super(STRING_AGG, self).__init__("STRING_AGG", column, separator, alias=alias) + + +class MATCH(DistinctOptionFunction): + def __init__(self, column: str, *args, **kwargs): + """[ Implementation of Match Against read more about it https://dev.mysql.com/doc/refman/8.0/en/fulltext-search.html#function_match ] + + Args: + column (str):[ column to search in ] + """ + alias = kwargs.get("alias") + super(MATCH, self).__init__(" MATCH", column, *args, alias=alias) + self._Against = False + + def get_function_sql(self, **kwargs): + s = super(DistinctOptionFunction, self).get_function_sql(**kwargs) + + if self._Against: + return f"{s} AGAINST ({frappe.db.escape(f'+{self._Against}*')} IN BOOLEAN MODE)" + return s + + @builder + def Against(self, text: str): + """[ Text that has to be searched against ] + + Args: + text (str): [ the text string that we match it against ] + """ + self._Against = text + + +class TO_TSVECTOR(DistinctOptionFunction): + def __init__(self, column: str, *args, **kwargs): + """[ Implementation of TO_TSVECTOR read more about it https://www.postgresql.org/docs/9.1/textsearch-controls.html] + + Args: + column (str): [ column to search in ] + """ + alias = kwargs.get("alias") + super(TO_TSVECTOR, self).__init__("TO_TSVECTOR", column, *args, alias=alias) + self._PLAINTO_TSQUERY = False + + def get_function_sql(self, **kwargs): + s = super(DistinctOptionFunction, self).get_function_sql(**kwargs) + if self._PLAINTO_TSQUERY: + return f"{s} @@ PLAINTO_TSQUERY({frappe.db.escape(self._PLAINTO_TSQUERY)})" + return s + + @builder + def Against(self, text: str): + """[ Text that has to be searched against ] + + Args: + text (str): [ the text string that we match it against ] + """ + self._PLAINTO_TSQUERY = text diff --git a/frappe/query_builder/functions.py b/frappe/query_builder/functions.py new file mode 100644 index 0000000000..5ccb266945 --- /dev/null +++ b/frappe/query_builder/functions.py @@ -0,0 +1,17 @@ +from pypika.functions import * +from frappe.query_builder.utils import ImportMapper, db_type_is +from frappe.query_builder.custom import GROUP_CONCAT, STRING_AGG, MATCH, TO_TSVECTOR + +GroupConcat = ImportMapper( + { + db_type_is.MARIADB: GROUP_CONCAT, + db_type_is.POSTGRES: STRING_AGG + } +) + +Match = ImportMapper( + { + db_type_is.MARIADB: MATCH, + db_type_is.POSTGRES: TO_TSVECTOR + } +) diff --git a/frappe/query_builder/utils.py b/frappe/query_builder/utils.py new file mode 100644 index 0000000000..b52a3606e8 --- /dev/null +++ b/frappe/query_builder/utils.py @@ -0,0 +1,34 @@ +from enum import Enum +from typing import Any, Callable, Dict + +from pypika import Query + +import frappe +from .builder import MariaDB, Postgres + + +class db_type_is(Enum): + MARIADB = "mariadb" + POSTGRES = "postgres" + +class ImportMapper: + def __init__(self, func_map: Dict[db_type_is, Callable]) -> None: + self.func_map = func_map + + def __call__(self, *args: Any, **kwds: Any) -> Callable: + db = db_type_is(frappe.conf.db_type or "mariadb") + return self.func_map[db](*args, **kwds) + + +def get_query_builder(type_of_db: str) -> Query: + """[return the query builder object] + + Args: + type_of_db (str): [string value of the db used] + + Returns: + Query: [Query object] + """ + db = db_type_is(type_of_db) + picks = {db_type_is.MARIADB: MariaDB, db_type_is.POSTGRES: Postgres} + return picks[db] diff --git a/frappe/search/website_search.py b/frappe/search/website_search.py index 49bdade936..70962e8172 100644 --- a/frappe/search/website_search.py +++ b/frappe/search/website_search.py @@ -35,10 +35,12 @@ class WebsiteSearch(FullTextSearch): if getattr(self, "_items_to_index", False): return self._items_to_index - routes = get_static_pages_from_all_apps() + slugs_with_web_view() - self._items_to_index = [] + + routes = get_static_pages_from_all_apps() + slugs_with_web_view(self._items_to_index) + + for i, route in enumerate(routes): update_progress_bar("Retrieving Routes", i, len(routes)) self._items_to_index += [self.get_document_to_index(route)] @@ -85,16 +87,23 @@ class WebsiteSearch(FullTextSearch): ) -def slugs_with_web_view(): +def slugs_with_web_view(_items_to_index): all_routes = [] filters = { "has_web_view": 1, "allow_guest_to_view": 1, "index_web_pages_for_search": 1} - fields = ["name", "is_published_field"] + fields = ["name", "is_published_field", 'website_search_field'] doctype_with_web_views = frappe.get_all("DocType", filters=filters, fields=fields) for doctype in doctype_with_web_views: if doctype.is_published_field: - routes = frappe.get_all(doctype.name, filters={doctype.is_published_field: 1}, fields="route") - all_routes += [route.route for route in routes] + docs = frappe.get_all(doctype.name, filters={doctype.is_published_field: 1}, fields=["route", doctype.website_search_field, 'title']) + if doctype.website_search_field: + for doc in docs: + content = frappe.utils.md_to_html(getattr(doc, doctype.website_search_field)) + soup = BeautifulSoup(content, "html.parser") + text_content = soup.text if soup else "" + _items_to_index += [frappe._dict(title=doc.title, content=text_content, path=doc.route)] + else: + all_routes += [route.route for route in docs] return all_routes diff --git a/frappe/sessions.py b/frappe/sessions.py index 4d922d6769..4f769ea88f 100644 --- a/frappe/sessions.py +++ b/frappe/sessions.py @@ -1,4 +1,4 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors +# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt """ Boot session from cache or build @@ -84,7 +84,7 @@ def delete_session(sid=None, user=None, reason="Session Expired"): if user_details: user = user_details[0].get("user") logout_feed(user, reason) - frappe.db.sql("""delete from tabSessions where sid=%s""", sid) + frappe.db.delete("Sessions", {"sid": sid}) frappe.db.commit() def clear_all_sessions(reason=None): @@ -249,7 +249,6 @@ class Session: data = self.get_session_record() if data: - # set language self.data.update({'data': data, 'user':data.user, 'sid': self.sid}) self.user = data.user validate_ip_address(self.user) diff --git a/frappe/social/doctype/energy_point_log/test_energy_point_log.py b/frappe/social/doctype/energy_point_log/test_energy_point_log.py index 8c4dba5d6b..4a6e86463e 100644 --- a/frappe/social/doctype/energy_point_log/test_energy_point_log.py +++ b/frappe/social/doctype/energy_point_log/test_energy_point_log.py @@ -273,6 +273,31 @@ class TestEnergyPointLog(unittest.TestCase): self.assertEqual(points_after_reverting_todo, points_after_closing_todo - rule_points) self.assertEqual(points_after_saving_todo_again, points_after_reverting_todo + rule_points) + def test_energy_points_disabled_user(self): + frappe.set_user('test@example.com') + user = frappe.get_doc('User', 'test@example.com') + user.enabled = 0 + user.save() + todo_point_rule = create_energy_point_rule_for_todo() + energy_point_of_user = get_points('test@example.com') + + created_todo = create_a_todo() + + created_todo.status = 'Closed' + created_todo.save() + points_after_closing_todo = get_points('test@example.com') + + # do not update energy points for disabled user + self.assertEqual(points_after_closing_todo, energy_point_of_user) + + user.enabled = 1 + user.save() + + created_todo.save() + points_after_re_saving_todo = get_points('test@example.com') + self.assertEqual(points_after_re_saving_todo, energy_point_of_user + todo_point_rule.points) + + def create_energy_point_rule_for_todo(multiplier_field=None, for_doc_event='Custom', max_points=None, for_assigned_users=0, field_to_check=None, apply_once=False, user_field='owner'): name = 'ToDo Closed' diff --git a/frappe/social/doctype/energy_point_rule/energy_point_rule.py b/frappe/social/doctype/energy_point_rule/energy_point_rule.py index 1c736528de..b673e8e199 100644 --- a/frappe/social/doctype/energy_point_rule/energy_point_rule.py +++ b/frappe/social/doctype/energy_point_rule/energy_point_rule.py @@ -5,6 +5,7 @@ import frappe from frappe import _ import frappe.cache_manager +from frappe.core.doctype.user.user import get_enabled_users from frappe.model import log_types from frappe.model.document import Document from frappe.social.doctype.energy_point_settings.energy_point_settings import is_energy_point_enabled @@ -44,7 +45,7 @@ class EnergyPointRule(Document): try: for user in users: - if not user or user == 'Administrator': continue + if not is_eligible_user(user): continue create_energy_points_log(reference_doctype, reference_name, { 'points': points, 'user': user, @@ -119,3 +120,8 @@ def get_energy_point_doctypes(): d.reference_doctype for d in frappe.get_all('Energy Point Rule', ['reference_doctype'], {'enabled': 1}) ] + +def is_eligible_user(user): + '''Checks if user is eligible to get energy points''' + enabled_users = get_enabled_users() + return user and user in enabled_users and user != 'Administrator' diff --git a/frappe/templates/print_formats/pdf_header_footer.html b/frappe/templates/print_formats/pdf_header_footer.html index a9c621f505..189cf43cd9 100644 --- a/frappe/templates/print_formats/pdf_header_footer.html +++ b/frappe/templates/print_formats/pdf_header_footer.html @@ -1,5 +1,5 @@ - + diff --git a/frappe/test_runner.py b/frappe/test_runner.py index 0c30fbbd00..8112362f34 100644 --- a/frappe/test_runner.py +++ b/frappe/test_runner.py @@ -56,6 +56,7 @@ def main(app=None, module=None, doctype=None, verbose=False, tests=(), frappe.clear_cache() frappe.utils.scheduler.disable_scheduler() set_test_email_config() + frappe.conf.update({'bench_id': 'test_bench', 'use_rq_auth': False}) if not frappe.flags.skip_before_tests: if verbose: diff --git a/frappe/tests/test_auth.py b/frappe/tests/test_auth.py index bc23cb591c..b02b53338d 100644 --- a/frappe/tests/test_auth.py +++ b/frappe/tests/test_auth.py @@ -4,8 +4,9 @@ import time import unittest import frappe -from frappe.auth import LoginAttemptTracker +from frappe.auth import HTTPRequest, LoginAttemptTracker from frappe.frappeclient import FrappeClient, AuthError +from frappe.utils import set_request class TestAuth(unittest.TestCase): def __init__(self, *args, **kwargs): diff --git a/frappe/tests/test_background_jobs.py b/frappe/tests/test_background_jobs.py index 88783f14f1..188f3e166f 100644 --- a/frappe/tests/test_background_jobs.py +++ b/frappe/tests/test_background_jobs.py @@ -4,7 +4,7 @@ from rq import Queue import frappe from frappe.core.page.background_jobs.background_jobs import remove_failed_jobs -from frappe.utils.background_jobs import get_redis_conn +from frappe.utils.background_jobs import get_redis_conn, generate_qname import time @@ -17,14 +17,14 @@ class TestBackgroundJobs(unittest.TestCase): queues = Queue.all(conn) for queue in queues: - if queue.name == "short": + if queue.name == generate_qname("short"): fail_registry = queue.failed_job_registry self.assertGreater(fail_registry.count, 0) remove_failed_jobs() for queue in queues: - if queue.name == "short": + if queue.name == generate_qname("short"): fail_registry = queue.failed_job_registry self.assertEqual(fail_registry.count, 0) diff --git a/frappe/tests/test_commands.py b/frappe/tests/test_commands.py index 54103f0151..f687f70228 100644 --- a/frappe/tests/test_commands.py +++ b/frappe/tests/test_commands.py @@ -433,6 +433,6 @@ class TestCommands(BaseTestCommands): for output in ["legacy", "plain", "table", "json"]: self.execute(f"bench version -f {output}") self.assertEqual(self.returncode, 0) - + self.execute("bench version -f invalid") self.assertEqual(self.returncode, 2) diff --git a/frappe/tests/test_db.py b/frappe/tests/test_db.py index 04c9a525b1..044ce455d9 100644 --- a/frappe/tests/test_db.py +++ b/frappe/tests/test_db.py @@ -12,6 +12,8 @@ from frappe.custom.doctype.custom_field.custom_field import create_custom_field from frappe.utils import random_string from frappe.utils.testutils import clear_custom_fields +from .test_query_builder import run_only_if, db_type_is + class TestDB(unittest.TestCase): def test_get_value(self): @@ -146,7 +148,7 @@ class TestDB(unittest.TestCase): # Create documents under that doctype and query them via ORM for _ in range(10): - docfields = { key.lower(): random_string(10) for key in fields } + docfields = {key.lower(): random_string(10) for key in fields} doc = frappe.get_doc({"doctype": test_doctype, "description": random_string(20), **docfields}) doc.insert() created_docs.append(doc.name) @@ -189,3 +191,98 @@ class TestDB(unittest.TestCase): for doc in created_docs: frappe.delete_doc(test_doctype, doc) clear_custom_fields(test_doctype) + +@run_only_if(db_type_is.MARIADB) +class TestDDLCommandsMaria(unittest.TestCase): + test_table_name = "TestNotes" + + def setUp(self) -> None: + frappe.db.commit() + frappe.db.sql( + f""" + CREATE TABLE `tab{self.test_table_name}` (`id` INT NULL,PRIMARY KEY (`id`)); + """ + ) + + def tearDown(self) -> None: + frappe.db.sql(f"DROP TABLE tab{self.test_table_name};") + self.test_table_name = "TestNotes" + + def test_rename(self) -> None: + new_table_name = f"{self.test_table_name}_new" + frappe.db.rename_table(self.test_table_name, new_table_name) + check_exists = frappe.db.sql( + f""" + SELECT * FROM INFORMATION_SCHEMA.TABLES + WHERE TABLE_NAME = N'tab{new_table_name}'; + """ + ) + self.assertGreater(len(check_exists), 0) + self.assertIn(f"tab{new_table_name}", check_exists[0]) + + # * so this table is deleted after the rename + self.test_table_name = new_table_name + + def test_describe(self) -> None: + self.assertEqual( + (("id", "int(11)", "NO", "PRI", None, ""),), + frappe.db.describe(self.test_table_name), + ) + + def test_change_type(self) -> None: + frappe.db.change_column_type("TestNotes", "id", "varchar(255)") + test_table_description = frappe.db.sql(f"DESC tab{self.test_table_name};") + self.assertGreater(len(test_table_description), 0) + self.assertIn("varchar(255)", test_table_description[0]) + + +@run_only_if(db_type_is.POSTGRES) +class TestDDLCommandsPost(unittest.TestCase): + test_table_name = "TestNotes" + + def setUp(self) -> None: + frappe.db.sql( + f""" + CREATE TABLE "tab{self.test_table_name}" ("id" INT NULL,PRIMARY KEY ("id")) + """ + ) + + def tearDown(self) -> None: + frappe.db.sql(f'DROP TABLE "tab{self.test_table_name}"') + self.test_table_name = "TestNotes" + + def test_rename(self) -> None: + new_table_name = f"{self.test_table_name}_new" + frappe.db.rename_table(self.test_table_name, new_table_name) + check_exists = frappe.db.sql( + f""" + SELECT EXISTS ( + SELECT FROM information_schema.tables + WHERE table_name = 'tab{new_table_name}' + ); + """ + ) + self.assertTrue(check_exists[0][0]) + + # * so this table is deleted after the rename + self.test_table_name = new_table_name + + def test_describe(self) -> None: + self.assertEqual([("id",)], frappe.db.describe(self.test_table_name)) + + def test_change_type(self) -> None: + frappe.db.change_column_type(self.test_table_name, "id", "varchar(255)") + check_change = frappe.db.sql( + f""" + SELECT + table_name, + column_name, + data_type + FROM + information_schema.columns + WHERE + table_name = 'tab{self.test_table_name}' + """ + ) + self.assertGreater(len(check_change), 0) + self.assertIn("character varying", check_change[0]) diff --git a/frappe/tests/test_query_builder.py b/frappe/tests/test_query_builder.py new file mode 100644 index 0000000000..d155dd95db --- /dev/null +++ b/frappe/tests/test_query_builder.py @@ -0,0 +1,74 @@ +import unittest +from typing import Callable + +import frappe +from frappe.query_builder.functions import GroupConcat, Match +from frappe.query_builder.utils import db_type_is + + +def run_only_if(dbtype: db_type_is) -> Callable: + return unittest.skipIf( + db_type_is(frappe.conf.db_type) != dbtype, f"Only runs for {dbtype.value}" + ) + + +@run_only_if(db_type_is.MARIADB) +class TestCustomFunctionsMariaDB(unittest.TestCase): + def test_concat(self): + self.assertEqual("GROUP_CONCAT('Notes')", GroupConcat("Notes").get_sql()) + + def test_match(self): + query = Match("Notes").Against("text") + self.assertEqual( + " MATCH('Notes') AGAINST ('+text*' IN BOOLEAN MODE)", query.get_sql() + ) + + +@run_only_if(db_type_is.POSTGRES) +class TestCustomFunctionsPostgres(unittest.TestCase): + def test_concat(self): + self.assertEqual("STRING_AGG('Notes',',')", GroupConcat("Notes").get_sql()) + + def test_match(self): + query = Match("Notes").Against("text") + self.assertEqual( + "TO_TSVECTOR('Notes') @@ PLAINTO_TSQUERY('text')", query.get_sql() + ) + + +class TestBuilderBase(object): + def test_adding_tabs(self): + self.assertEqual("tabNotes", frappe.qb.Table("Notes").get_sql()) + self.assertEqual("__Auth", frappe.qb.Table("__Auth").get_sql()) + + +@run_only_if(db_type_is.MARIADB) +class TestBuilderMaria(unittest.TestCase, TestBuilderBase): + def test_adding_tabs_in_from(self): + self.assertEqual( + "SELECT * FROM `tabNotes`", frappe.qb.from_("Notes").select("*").get_sql() + ) + self.assertEqual( + "SELECT * FROM `__Auth`", frappe.qb.from_("__Auth").select("*").get_sql() + ) + + +@run_only_if(db_type_is.POSTGRES) +class TestBuilderPostgres(unittest.TestCase, TestBuilderBase): + def test_adding_tabs_in_from(self): + self.assertEqual( + 'SELECT * FROM "tabNotes"', frappe.qb.from_("Notes").select("*").get_sql() + ) + self.assertEqual( + 'SELECT * FROM "__Auth"', frappe.qb.from_("__Auth").select("*").get_sql() + ) + + def test_replace_tables(self): + info_schema = frappe.qb.Schema("information_schema") + self.assertEqual( + 'SELECT * FROM "pg_stat_all_tables"', + frappe.qb.from_(info_schema.tables).select("*").get_sql(), + ) + + def test_replace_fields_post(self): + self.assertEqual("relname", frappe.qb.Field("table_name").get_sql()) diff --git a/frappe/tests/test_redis.py b/frappe/tests/test_redis.py new file mode 100644 index 0000000000..72af1ac699 --- /dev/null +++ b/frappe/tests/test_redis.py @@ -0,0 +1,70 @@ +import unittest +import functools + +import redis + +import frappe +from frappe.utils import get_bench_id +from frappe.utils.rq import RedisQueue +from frappe.utils.background_jobs import get_redis_conn + +def version_tuple(version): + return tuple(map(int, (version.split(".")))) + +def skip_if_redis_version_lt(version): + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + conn = get_redis_conn() + redis_version = conn.execute_command('info')['redis_version'] + if version_tuple(redis_version) < version_tuple(version): + return + return func(*args, **kwargs) + return wrapper + return decorator + +class TestRedisAuth(unittest.TestCase): + @skip_if_redis_version_lt('6.0') + def test_rq_gen_acllist(self): + """Make sure that ACL list is genrated + """ + acl_list = RedisQueue.gen_acl_list() + self.assertEqual(acl_list[1]['bench'][0], get_bench_id()) + + @skip_if_redis_version_lt('6.0') + def test_adding_redis_user(self): + acl_list = RedisQueue.gen_acl_list() + username, password = acl_list[1]['bench'] + conn = get_redis_conn() + + conn.acl_deluser(username) + _ = RedisQueue(conn).add_user(username, password) + self.assertTrue(conn.acl_getuser(username)) + conn.acl_deluser(username) + + @skip_if_redis_version_lt('6.0') + def test_rq_namespace(self): + """Make sure that user can access only their respective namespace. + """ + # Current bench ID + bench_id = frappe.conf.get('bench_id') + conn = get_redis_conn() + conn.set('rq:queue:test_bench1:abc', 'value') + conn.set(f'rq:queue:{bench_id}:abc', 'value') + + # Create new Redis Queue user + tmp_bench_id = 'test_bench1' + username, password = tmp_bench_id, 'password1' + conn.acl_deluser(username) + frappe.conf.update({'bench_id': tmp_bench_id}) + _ = RedisQueue(conn).add_user(username, password) + test_bench1_conn = RedisQueue.get_connection(username, password) + + self.assertEqual(test_bench1_conn.get('rq:queue:test_bench1:abc'), b'value') + + # User should not be able to access queues apart from their bench queues + with self.assertRaises(redis.exceptions.NoPermissionError): + test_bench1_conn.get(f'rq:queue:{bench_id}:abc') + + frappe.conf.update({'bench_id': bench_id}) + conn.acl_deluser(username) diff --git a/frappe/tests/test_translate.py b/frappe/tests/test_translate.py index f51f31d509..edab3a82c3 100644 --- a/frappe/tests/test_translate.py +++ b/frappe/tests/test_translate.py @@ -1,13 +1,26 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors +# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -import frappe, unittest, os +import os +import unittest +from random import choices +from unittest.mock import patch + +import frappe import frappe.translate from frappe import _ +from frappe.translate import get_language, get_parent_language +from frappe.utils import set_request dirname = os.path.dirname(__file__) translation_string_file = os.path.join(dirname, 'translation_test_file.txt') +first_lang, second_lang, third_lang, fourth_lang, fifth_lang = choices( + frappe.get_all("Language", pluck="name"), k=5 +) class TestTranslate(unittest.TestCase): + def tearDown(self): + frappe.form_dict.pop("_lang", None) + def test_extract_message_from_file(self): data = frappe.translate.get_messages_from_file(translation_string_file) self.assertListEqual(data, expected_output) @@ -20,6 +33,41 @@ class TestTranslate(unittest.TestCase): finally: frappe.local.lang = 'en' + def test_request_language_resolution_with_form_dict(self): + """Test for frappe.translate.get_language + + Case 1: frappe.form_dict._lang is set + """ + + frappe.form_dict._lang = first_lang + + with patch.object(frappe.translate, "get_preferred_language_cookie", return_value=second_lang): + return_val = get_language() + + self.assertIn(return_val, [first_lang, get_parent_language(first_lang)]) + + def test_request_language_resolution_with_cookie(self): + """Test for frappe.translate.get_language + + Case 2: frappe.form_dict._lang is not set, but preferred_language cookie is + """ + + with patch.object(frappe.translate, "get_preferred_language_cookie", return_value=second_lang): + set_request(method="POST", path="/", headers=[("Accept-Language", third_lang)]) + return_val = get_language() + + self.assertIn(return_val, [second_lang, get_parent_language(second_lang)]) + + def test_request_language_resolution_with_request_header(self): + """Test for frappe.translate.get_language + + Case 3: frappe.form_dict._lang & preferred_language cookie is not set, but Accept-Language header is + """ + set_request(method="POST", path="/", headers=[("Accept-Language", third_lang)]) + return_val = get_language() + self.assertIn(return_val, [third_lang, get_parent_language(third_lang)]) + + expected_output = [ ('apps/frappe/frappe/tests/translation_test_file.txt', 'Warning: Unable to find {0} in any table related to {1}', 'This is some context', 2), ('apps/frappe/frappe/tests/translation_test_file.txt', 'Warning: Unable to find {0} in any table related to {1}', None, 4), diff --git a/frappe/translate.py b/frappe/translate.py index 4ff50d3fd0..d5916f1761 100644 --- a/frappe/translate.py +++ b/frappe/translate.py @@ -1,4 +1,4 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors +# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt """ frappe.translate @@ -11,71 +11,100 @@ import io import itertools import json import operator +import functools import os import re from csv import reader +from typing import List, Union, Tuple import frappe from frappe.model.utils import InvalidIncludePath, render_include -from frappe.utils import is_html, strip, strip_html_tags +from frappe.utils import get_bench_path, is_html, strip, strip_html_tags -def guess_language(lang_list=None): - """Set `frappe.local.lang` from HTTP headers at beginning of request""" - user_preferred_language = frappe.request.cookies.get('preferred_language') - is_guest_user = not frappe.session.user or frappe.session.user == 'Guest' - if is_guest_user and user_preferred_language: - return user_preferred_language +def get_language(lang_list: List = None) -> str: + """Set `frappe.local.lang` from HTTP headers at beginning of request - lang_codes = frappe.request.accept_languages.values() - if not lang_codes: - return frappe.local.lang + Order of priority for setting language: + 1. Form Dict => _lang + 2. Cookie => preferred_language + 3. Request Header => Accept-Language + 4. User document => language + 5. System Settings => language + """ - guess = None - if not lang_list: - lang_list = get_all_languages() or [] + # fetch language from form_dict + if frappe.form_dict._lang: + language = get_lang_code( + frappe.form_dict._lang or get_parent_language(frappe.form_dict._lang) + ) + if language: + return language - for l in lang_codes: - code = l.strip() - if not isinstance(code, str): - code = str(code, 'utf-8') - if code in lang_list or code == "en": - guess = code - break + lang_set = set(lang_list or get_all_languages() or []) - # check if parent language (pt) is setup, if variant (pt-BR) - if "-" in code: - code = code.split("-")[0] - if code in lang_list: - guess = code - break + # fetch language from cookie + preferred_language_cookie = get_preferred_language_cookie() - return guess or frappe.local.lang + if preferred_language_cookie: + if preferred_language_cookie in lang_set: + return preferred_language_cookie -def get_user_lang(user=None): + parent_language = get_parent_language(language) + if parent_language in lang_set: + return parent_language + + # fetch language from request headers + accept_language = list(frappe.request.accept_languages.values()) + + for language in accept_language: + if language in lang_set: + return language + + parent_language = get_parent_language(language) + if parent_language in lang_set: + return parent_language + + # fallback to language set in User or System Settings + return frappe.local.lang + + +@functools.lru_cache() +def get_parent_language(lang: str) -> str: + """If the passed language is a variant, return its parent + + Eg: + 1. zh-TW -> zh + 2. sr-BA -> sr + """ + is_language_variant = "-" in lang + if is_language_variant: + return lang[:lang.index("-")] + + +def get_user_lang(user: str = None) -> str: """Set frappe.local.lang from user preferences on session beginning or resumption""" - if not user: - user = frappe.session.user - - # via cache + user = user or frappe.session.user lang = frappe.cache().hget("lang", user) if not lang: - - # if defined in user profile - lang = frappe.db.get_value("User", user, "language") - if not lang: - lang = frappe.db.get_default("lang") - - if not lang: - lang = frappe.local.lang or 'en' + # User.language => Session Defaults => frappe.local.lang => 'en' + lang = ( + frappe.db.get_value("User", user, "language") + or frappe.db.get_default("lang") + or frappe.local.lang + or "en" + ) frappe.cache().hset("lang", user, lang) return lang -def get_lang_code(lang): - return frappe.db.get_value('Language', {'language_name': lang}) or lang +def get_lang_code(lang: str) -> Union[str, None]: + return ( + frappe.db.get_value("Language", {"name": lang}) + or frappe.db.get_value("Language", {"language_name": lang}) + ) def set_default_language(lang): """Set Global default language""" @@ -529,7 +558,7 @@ def get_all_messages_from_js_files(app_name=None): return messages -def get_messages_from_file(path): +def get_messages_from_file(path: str) -> List[Tuple[str, str, str, str]]: """Returns a list of transatable strings from a code file :param path: path of the code file @@ -542,7 +571,7 @@ def get_messages_from_file(path): frappe.flags.scanned_files.append(path) - apps_path = get_bench_dir() + bench_path = get_bench_path() if os.path.exists(path): with open(path, 'r') as sourcefile: try: @@ -550,11 +579,12 @@ def get_messages_from_file(path): except Exception: print("Could not scan file for translation: {0}".format(path)) return [] - data = [(os.path.relpath(path, apps_path), message, context, line) \ - for line, message, context in extract_messages_from_code(file_contents)] - return data + + return [ + (os.path.relpath(path, bench_path), message, context, line) + for (line, message, context) in extract_messages_from_code(file_contents) + ] else: - # print "Translate: {0} missing".format(os.path.abspath(path)) return [] def extract_messages_from_code(code): @@ -768,9 +798,6 @@ def deduplicate_messages(messages): ret.append(next(g)) return ret -def get_bench_dir(): - return os.path.join(frappe.__file__, '..', '..', '..', '..') - def rename_language(old_name, new_name): if not frappe.db.exists('Language', new_name): return @@ -879,3 +906,6 @@ def get_all_languages(with_language_name=False): @frappe.whitelist(allow_guest=True) def set_preferred_language_cookie(preferred_language): frappe.local.cookie_manager.set_cookie("preferred_language", preferred_language) + +def get_preferred_language_cookie(): + return frappe.request.cookies.get("preferred_language") diff --git a/frappe/utils/__init__.py b/frappe/utils/__init__.py index af9d5de1ee..b97585aa04 100644 --- a/frappe/utils/__init__.py +++ b/frappe/utils/__init__.py @@ -383,6 +383,12 @@ def get_files_path(*path, **kwargs): def get_bench_path(): return os.path.realpath(os.path.join(os.path.dirname(frappe.__file__), '..', '..', '..')) +def get_bench_id(): + return frappe.get_conf().get('bench_id', get_bench_path().strip('/').replace('/', '-')) + +def get_site_id(site=None): + return f"{site or frappe.local.site}@{get_bench_id()}" + def get_backups_path(): return get_site_path("private", "backups") @@ -843,3 +849,6 @@ def groupby_metric(iterable: typing.Dict[str, list], key: str): for item in items: records.setdefault(item[key], {}).setdefault(category, []).append(item) return records + +def get_table_name(table_name: str) -> str: + return f"tab{table_name}" if not table_name.startswith("__") else table_name diff --git a/frappe/utils/background_jobs.py b/frappe/utils/background_jobs.py index 8456835ca7..f0bd06aff4 100755 --- a/frappe/utils/background_jobs.py +++ b/frappe/utils/background_jobs.py @@ -1,13 +1,21 @@ +import os +import socket +import time +from uuid import uuid4 +from collections import defaultdict + + import redis +from typing import List from rq import Connection, Queue, Worker from rq.logutils import setup_loghandlers -from frappe.utils import cstr -from collections import defaultdict + import frappe -import os, socket, time from frappe import _ -from uuid import uuid4 import frappe.monitor +from frappe.utils import cstr, get_bench_id +from frappe.utils.rq import RedisQueue +from frappe.utils.commands import log default_timeout = 300 @@ -131,21 +139,22 @@ def execute_job(site, method, event, job_name, kwargs, user=None, is_async=True, if is_async: frappe.destroy() -def start_worker(queue=None, quiet = False): +def start_worker(queue=None, quiet = False, rq_username=None, rq_password=None): '''Wrapper to start rq worker. Connects to redis and monitors these queues.''' with frappe.init_site(): # empty init is required to get redis_queue from common_site_config.json - redis_connection = get_redis_conn() + redis_connection = get_redis_conn(username=rq_username, password=rq_password) + queues = get_queue_list(queue, build_queue_name=True) + queue_name = queue and generate_qname(queue) if os.environ.get('CI'): setup_loghandlers('ERROR') with Connection(redis_connection): - queues = get_queue_list(queue) logging_level = "INFO" if quiet: logging_level = "WARNING" - Worker(queues, name=get_worker_name(queue)).work(logging_level = logging_level) + Worker(queues, name=get_worker_name(queue_name)).work(logging_level = logging_level) def get_worker_name(queue): '''When limiting worker to a specific queue, also append queue name to default worker name''' @@ -186,7 +195,7 @@ def get_jobs(site=None, queue=None, key='method'): return jobs_per_site -def get_queue_list(queue_list=None): +def get_queue_list(queue_list=None, build_queue_name=False): '''Defines possible queues. Also wraps a given queue in a list after validating.''' default_queue_list = list(queue_timeout) if queue_list: @@ -195,11 +204,9 @@ def get_queue_list(queue_list=None): for queue in queue_list: validate_queue(queue, default_queue_list) - - return queue_list - else: - return default_queue_list + queue_list = default_queue_list + return [generate_qname(qtype) for qtype in queue_list] if build_queue_name else queue_list def get_workers(queue): '''Returns a list of Worker objects tied to a queue object''' @@ -215,10 +222,10 @@ def get_running_jobs_in_queue(queue): jobs.append(current_job) return jobs -def get_queue(queue, is_async=True): +def get_queue(qtype, is_async=True): '''Returns a Queue object tied to a redis connection''' - validate_queue(queue) - return Queue(queue, connection=get_redis_conn(), is_async=is_async) + validate_queue(qtype) + return Queue(generate_qname(qtype), connection=get_redis_conn(), is_async=is_async) def validate_queue(queue, default_queue_list=None): if not default_queue_list: @@ -227,7 +234,7 @@ def validate_queue(queue, default_queue_list=None): if queue not in default_queue_list: frappe.throw(_("Queue should be one of {0}").format(', '.join(default_queue_list))) -def get_redis_conn(): +def get_redis_conn(username=None, password=None): if not hasattr(frappe.local, 'conf'): raise Exception('You need to call frappe.init') @@ -236,11 +243,50 @@ def get_redis_conn(): global redis_connection - if not redis_connection: - redis_connection = redis.from_url(frappe.local.conf.redis_queue) + cred = frappe._dict() + if frappe.conf.get('use_rq_auth'): + if username: + cred['username'] = username + cred['password'] = password + else: + cred['username'] = frappe.get_site_config().rq_username or get_bench_id() + cred['password'] = frappe.get_site_config().rq_password + + elif os.environ.get('RQ_ADMIN_PASWORD'): + cred['username'] = 'default' + cred['password'] = os.environ.get('RQ_ADMIN_PASWORD') + try: + redis_connection = RedisQueue.get_connection(**cred) + except (redis.exceptions.AuthenticationError, redis.exceptions.ResponseError): + log(f'Wrong credentials used for {cred.username or "default user"}. ' + 'You can reset credentials using `bench create-rq-users` CLI and restart the server', + colour='red') + raise + except Exception: + log(f'Please make sure that Redis Queue runs @ {frappe.get_conf().redis_queue}', colour='red') + raise return redis_connection +def get_queues() -> List[Queue]: + """Get all the queues linked to the current bench. + """ + queues = Queue.all(connection=get_redis_conn()) + return [q for q in queues if is_queue_accessible(q)] + +def generate_qname(qtype: str) -> str: + """Generate qname by combining bench ID and queue type. + + qnames are useful to define namespaces of customers. + """ + return f"{get_bench_id()}:{qtype}" + +def is_queue_accessible(qobj: Queue) -> bool: + """Checks whether queue is relate to current bench or not. + """ + accessible_queues = [generate_qname(q) for q in list(queue_timeout)] + return qobj.name in accessible_queues + def enqueue_test_job(): enqueue('frappe.utils.background_jobs.test_job', s=100) diff --git a/frappe/utils/backups.py b/frappe/utils/backups.py index 908be52452..f13710dcfe 100644 --- a/frappe/utils/backups.py +++ b/frappe/utils/backups.py @@ -116,16 +116,16 @@ class BackupGenerator: def setup_backup_tables(self): """Sets self.backup_includes, self.backup_excludes based on passed args""" - existing_doctypes = set([x.name for x in frappe.get_all("DocType")]) + existing_tables = frappe.db.get_tables() def get_tables(doctypes): tables = [] for doctype in doctypes: - if doctype and doctype in existing_doctypes: - if doctype.startswith("tab"): - tables.append(doctype) - else: - tables.append("tab" + doctype) + if not doctype: + continue + table = frappe.utils.get_table_name(doctype) + if table in existing_tables: + tables.append(table) return tables passed_tables = { diff --git a/frappe/utils/error.py b/frappe/utils/error.py index 07e34674fe..05b578d7e8 100644 --- a/frappe/utils/error.py +++ b/frappe/utils/error.py @@ -176,6 +176,7 @@ def collect_error_snapshots(): def clear_old_snapshots(): """Clear snapshots that are older than a month""" + frappe.db.sql("""delete from `tabError Snapshot` where creation < (NOW() - INTERVAL '1' MONTH)""") diff --git a/frappe/utils/global_search.py b/frappe/utils/global_search.py index 8fa2ea474f..072e3a7c62 100644 --- a/frappe/utils/global_search.py +++ b/frappe/utils/global_search.py @@ -23,8 +23,7 @@ def reset(): Deletes all data in __global_search :return: """ - frappe.db.sql('DELETE FROM `__global_search`') - + frappe.db.delete("__global_search") def get_doctypes_with_global_search(with_child_tables=True): """ @@ -146,10 +145,9 @@ def rebuild_for_doctype(doctype): def delete_global_search_records_for_doctype(doctype): - frappe.db.sql('''DELETE - FROM `__global_search` - WHERE doctype = %s''', doctype, as_dict=True) - + frappe.db.delete("__global_search", { + "doctype": doctype + }) def get_selected_fields(meta, global_search_fields): fieldnames = [df.fieldname for df in global_search_fields] @@ -399,12 +397,10 @@ def delete_for_document(doc): been deleted :param doc: Deleted document """ - - frappe.db.sql('''DELETE - FROM `__global_search` - WHERE doctype = %s - AND name = %s''', (doc.doctype, doc.name), as_dict=True) - + frappe.db.delete("__global_search", { + "doctype": doc.doctype, + "name": doc.name + }) @frappe.whitelist() def search(text, start=0, limit=20, doctype=""): @@ -415,51 +411,41 @@ def search(text, start=0, limit=20, doctype=""): :param limit: number of results to return, default 20 :return: Array of result objects """ - from frappe.desk.doctype.global_search_settings.global_search_settings import get_doctypes_for_global_search + from frappe.desk.doctype.global_search_settings.global_search_settings import ( + get_doctypes_for_global_search, + ) + from frappe.query_builder.functions import Match results = [] sorted_results = [] allowed_doctypes = get_doctypes_for_global_search() - for text in set(text.split('&')): + for text in set(text.split("&")): text = text.strip() if not text: continue - conditions = '1=1' - offset = '' - - mariadb_text = frappe.db.escape('+' + text + '*') - - mariadb_fields = '`doctype`, `name`, `content`, MATCH (`content`) AGAINST ({} IN BOOLEAN MODE) AS rank'.format(mariadb_text) - postgres_fields = '`doctype`, `name`, `content`, TO_TSVECTOR("content") @@ PLAINTO_TSQUERY({}) AS rank'.format(frappe.db.escape(text)) - - values = {} + global_search = frappe.qb.Table("__global_search") + rank = Match(global_search.content).Against(text).as_("rank") + query = ( + frappe.qb.from_(global_search) + .select( + global_search.doctype, global_search.name, global_search.content, rank + ) + .orderby("rank", order=frappe.qb.desc) + .limit(limit) + ) if doctype: - conditions = '`doctype` = %(doctype)s' - values['doctype'] = doctype + query = query.where(global_search.doctype == doctype) elif allowed_doctypes: - conditions = '`doctype` IN %(allowed_doctypes)s' - values['allowed_doctypes'] = tuple(allowed_doctypes) + query = query.where(global_search.doctype.isin(allowed_doctypes)) - if int(start) > 0: - offset = 'OFFSET {}'.format(start) + if start > 0: + query = query.offset(start) - common_query = """ - SELECT {fields} - FROM `__global_search` - WHERE {conditions} - ORDER BY rank DESC - LIMIT {limit} - {offset} - """ - - result = frappe.db.multisql({ - 'mariadb': common_query.format(fields=mariadb_fields, conditions=conditions, limit=limit, offset=offset), - 'postgres': common_query.format(fields=postgres_fields, conditions=conditions, limit=limit, offset=offset) - }, values=values, as_dict=True) + result = frappe.db.sql(query, as_dict=True) results.extend(result) @@ -470,7 +456,9 @@ def search(text, start=0, limit=20, doctype=""): try: meta = frappe.get_meta(r.doctype) if meta.image_field: - r.image = frappe.db.get_value(r.doctype, r.name, meta.image_field) + r.image = frappe.db.get_value( + r.doctype, r.name, meta.image_field + ) except Exception: frappe.clear_messages() diff --git a/frappe/utils/install.py b/frappe/utils/install.py index 91d8f04eb4..3d6a2fed97 100644 --- a/frappe/utils/install.py +++ b/frappe/utils/install.py @@ -111,9 +111,9 @@ def before_tests(): # don't run before tests if any other app is installed return - frappe.db.sql("delete from `tabCustom Field`") - frappe.db.sql("delete from `tabEvent`") - frappe.db.commit() + frappe.db.truncate("Custom Field") + frappe.db.truncate("Event") + frappe.clear_cache() # complete setup if missing diff --git a/frappe/utils/jinja_globals.py b/frappe/utils/jinja_globals.py index 2c14249672..67ca9d108a 100644 --- a/frappe/utils/jinja_globals.py +++ b/frappe/utils/jinja_globals.py @@ -73,17 +73,25 @@ def include_script(path): return f'' -def include_style(path): +def include_style(path, rtl=None): path = bundled_asset(path) return f'' -def bundled_asset(path): +def bundled_asset(path, rtl=None): from frappe.utils import get_assets_json from frappe.website.utils import abs_url if ".bundle." in path and not path.startswith("/assets"): bundled_assets = get_assets_json() + if path.endswith('.css') and is_rtl(rtl): + path = f"rtl_{path}" path = bundled_assets.get(path) or path return abs_url(path) + +def is_rtl(rtl=None): + from frappe import local + if rtl is None: + return local.lang in ["ar", "he", "fa", "ps"] + return rtl \ No newline at end of file diff --git a/frappe/utils/nestedset.py b/frappe/utils/nestedset.py index 3c024c40e4..4a65140449 100644 --- a/frappe/utils/nestedset.py +++ b/frappe/utils/nestedset.py @@ -57,7 +57,7 @@ def update_add_node(doc, parent, parent_field): # get the last sibling of the parent if parent: - left, right = frappe.db.sql("select lft, rgt from `tab{0}` where name=%s" + left, right = frappe.db.sql("select lft, rgt from `tab{0}` where name=%s for update" .format(doctype), parent)[0] validate_loop(doc.doctype, doc.name, left, right) else: # root @@ -89,7 +89,7 @@ def update_move_node(doc, parent_field): if parent: new_parent = frappe.db.sql("""select lft, rgt from `tab{0}` - where name = %s""".format(doc.doctype), parent, as_dict=1)[0] + where name = %s for update""".format(doc.doctype), parent, as_dict=1)[0] validate_loop(doc.doctype, doc.name, new_parent.lft, new_parent.rgt) @@ -108,7 +108,7 @@ def update_move_node(doc, parent_field): if parent: new_parent = frappe.db.sql("""select lft, rgt from `tab%s` - where name = %s""" % (doc.doctype, '%s'), parent, as_dict=1)[0] + where name = %s for update""" % (doc.doctype, '%s'), parent, as_dict=1)[0] # set parent lft, rgt diff --git a/frappe/utils/password.py b/frappe/utils/password.py index 428f2e9577..a097c58b31 100644 --- a/frappe/utils/password.py +++ b/frappe/utils/password.py @@ -65,11 +65,11 @@ def set_encrypted_password(doctype, name, pwd, fieldname='password'): def remove_encrypted_password(doctype, name, fieldname='password'): - frappe.db.sql( - 'DELETE FROM `__Auth` WHERE doctype = %s and name = %s and fieldname = %s', - values=[doctype, name, fieldname] - ) - + frappe.db.delete("__Auth", { + "doctype": doctype, + "name": name, + "fieldname": fieldname + }) def check_password(user, pwd, doctype='User', fieldname='password', delete_tracker_cache=True): '''Checks if user and password are correct, else raises frappe.AuthenticationError''' @@ -131,8 +131,10 @@ def update_password(user, pwd, doctype='User', fieldname='password', logout_all_ def delete_all_passwords_for(doctype, name): try: - frappe.db.sql("""delete from `__Auth` where `doctype`=%(doctype)s and `name`=%(name)s""", - { 'doctype': doctype, 'name': name }) + frappe.db.delete("__Auth", { + "doctype": doctype, + "name": name + }) except Exception as e: if not frappe.db.is_missing_column(e): raise diff --git a/frappe/utils/pdf.py b/frappe/utils/pdf.py index fcf483bea6..9cbece5dbe 100644 --- a/frappe/utils/pdf.py +++ b/frappe/utils/pdf.py @@ -13,7 +13,7 @@ from PyPDF2 import PdfFileReader, PdfFileWriter import frappe from frappe import _ from frappe.utils import scrub_urls -from frappe.utils.jinja_globals import bundled_asset +from frappe.utils.jinja_globals import bundled_asset, is_rtl PDF_CONTENT_ERRORS = ["ContentNotFoundError", "ContentOperationNotPermittedError", "UnknownContentError", "RemoteHostClosedError"] @@ -177,7 +177,9 @@ def prepare_header_footer(soup): "content": content, "styles": styles, "html_id": html_id, - "css": css + "css": css, + "lang": frappe.local.lang, + "layout_direction": "rtl" if is_rtl else "ltr" }) # create temp file diff --git a/frappe/utils/rq.py b/frappe/utils/rq.py new file mode 100644 index 0000000000..b344b0caa5 --- /dev/null +++ b/frappe/utils/rq.py @@ -0,0 +1,83 @@ +import redis + +import frappe +from frappe.utils import get_bench_id, random_string + +class RedisQueue: + def __init__(self, conn): + self.conn = conn + + def add_user(self, username, password=None): + """Create or update the user. + """ + password = password or self.conn.acl_genpass() + user_settings = self.get_new_user_settings(username, password) + is_created = self.conn.acl_setuser(**user_settings) + return frappe._dict(user_settings) if is_created else {} + + @classmethod + def get_connection(cls, username=None, password=None): + rq_url = frappe.local.conf.redis_queue + domain = rq_url.split("redis://", 1)[-1] + url = (username and f"redis://{username}:{password or ''}@{domain}") or rq_url + conn = redis.from_url(url) + conn.ping() + return conn + + @classmethod + def new(cls, username='default', password=None): + return cls(cls.get_connection(username, password)) + + @classmethod + def set_admin_password(cls, cur_password=None, new_password=None, reset_passwords=False): + username = 'default' + conn = cls.get_connection(username, cur_password) + password = '+'+(new_password or conn.acl_genpass()) + conn.acl_setuser( + username=username, enabled=True, reset_passwords=reset_passwords, passwords=password + ) + return password[1:] + + @classmethod + def get_new_user_settings(cls, username, password): + d = {} + d['username'] = username + d['passwords'] = '+'+password + d['reset_keys'] = True + d['enabled'] = True + d['keys'] = cls.get_acl_key_rules() + d['commands'] = cls.get_acl_command_rules() + return d + + @classmethod + def get_acl_key_rules(cls, include_key_prefix=False): + """FIXME: Find better way + """ + rules = ['rq:[^q]*', 'rq:queues', f'rq:queue:{get_bench_id()}:*'] + if include_key_prefix: + return ['~'+pattern for pattern in rules] + return rules + + @classmethod + def get_acl_command_rules(cls): + return ['+@all', '-@admin'] + + @classmethod + def gen_acl_list(cls, set_admin_password=False): + """Generate list of ACL users needed for this branch. + + This list contains default ACL user and the bench ACL user(used by all sites incase of ACL is enabled). + """ + bench_username = get_bench_id() + bench_user_rules = cls.get_acl_key_rules(include_key_prefix=True) + cls.get_acl_command_rules() + bench_user_rule_str = ' '.join(bench_user_rules).strip() + bench_user_password = random_string(20) + + default_username = 'default' + _default_user_password = random_string(20) if set_admin_password else '' + default_user_password = '>'+_default_user_password if _default_user_password else 'nopass' + + return [ + f'user {default_username} on {default_user_password} ~* &* +@all', + f'user {bench_username} on >{bench_user_password} {bench_user_rule_str}' + ], {'bench': (bench_username, bench_user_password), 'default': (default_username, _default_user_password)} diff --git a/frappe/utils/testutils.py b/frappe/utils/testutils.py index c451d090f1..9a2b2da791 100644 --- a/frappe/utils/testutils.py +++ b/frappe/utils/testutils.py @@ -12,5 +12,5 @@ def add_custom_field(doctype, fieldname, fieldtype='Data', options=None): }).insert() def clear_custom_fields(doctype): - frappe.db.sql('delete from `tabCustom Field` where dt=%s', doctype) + frappe.db.delete("Custom Field", {"dt": doctype}) frappe.clear_cache(doctype=doctype) diff --git a/frappe/website/doctype/web_form/web_form.py b/frappe/website/doctype/web_form/web_form.py index 6e4200b54b..32f7e030a6 100644 --- a/frappe/website/doctype/web_form/web_form.py +++ b/frappe/website/doctype/web_form/web_form.py @@ -542,7 +542,7 @@ def get_form_data(doctype, docname=None, web_form_name=None): # For Table fields, server-side processing for meta for field in out.web_form.web_form_fields: if field.fieldtype == "Table": - field.fields = get_in_list_view_fields(field.options) + field.fields = frappe.get_meta(field.options).fields out.update({field.fieldname: field.fields}) if field.fieldtype == "Link": diff --git a/frappe/website/page_renderers/not_found_page.py b/frappe/website/page_renderers/not_found_page.py index af510fecfc..de631f5dfe 100644 --- a/frappe/website/page_renderers/not_found_page.py +++ b/frappe/website/page_renderers/not_found_page.py @@ -8,11 +8,11 @@ from frappe.website.utils import can_cache HOMEPAGE_PATHS = ('/', '/index', 'index') class NotFoundPage(TemplatePage): - def __init__(self, path, http_status_code): + def __init__(self, path, http_status_code=None): self.request_path = path self.request_url = frappe.local.request.url if hasattr(frappe.local, 'request') else '' path = '404' - http_status_code = 404 + http_status_code = http_status_code or 404 super().__init__(path=path, http_status_code=http_status_code) def can_render(self): diff --git a/frappe/website/report/website_analytics/website_analytics.py b/frappe/website/report/website_analytics/website_analytics.py index d141972679..6cc0fb7f97 100644 --- a/frappe/website/report/website_analytics/website_analytics.py +++ b/frappe/website/report/website_analytics/website_analytics.py @@ -1,11 +1,14 @@ # Copyright (c) 2013, Frappe Technologies and contributors # For license information, please see license.txt -import frappe from datetime import datetime + +import frappe +from frappe.query_builder.functions import Coalesce, Count from frappe.utils import getdate from frappe.utils.dateutils import get_dates_from_timegrain + def execute(filters=None): return WebsiteAnalytics(filters).run() @@ -56,33 +59,21 @@ class WebsiteAnalytics(object): ] def get_data(self): - pg_query = """ - SELECT - path, - COUNT(*) as count, - COUNT(CASE WHEN CAST(is_unique as Integer) = 1 THEN 1 END) as unique_count - FROM `tabWeb Page View` - WHERE coalesce("tabWeb Page View".creation, '0001-01-01') BETWEEN %s AND %s - GROUP BY path - ORDER BY count desc - """ + WebPageView = frappe.qb.Table("Web Page View") + count_all = Count("*").as_("count") + case = frappe.qb.terms.Case().when(WebPageView.is_unique == "1", "1") + count_is_unique = Count(case).as_("unique_count") - mariadb_query = """ - SELECT - path, - COUNT(*) as count, - COUNT(CASE WHEN is_unique = 1 THEN 1 END) as unique_count - FROM `tabWeb Page View` - WHERE creation BETWEEN %s AND %s - GROUP BY path - ORDER BY count desc - """ - - data = frappe.db.multisql({ - "mariadb": mariadb_query, - "postgres": pg_query - }, (self.filters.from_date, self.filters.to_date)) - return data + query = ( + frappe.qb.from_(WebPageView) + .select("path", count_all, count_is_unique) + .where( + Coalesce(WebPageView.creation, "0001-01-01")[self.filters.from_date:self.filters.to_date] + ) + .groupby(WebPageView.path) + .orderby("count", Order=frappe.qb.desc) + ) + return frappe.db.sql(query) def _get_query_for_mariadb(self): filters_range = self.filters.range diff --git a/frappe/website/utils.py b/frappe/website/utils.py index e218afe8c6..472e86d8f5 100644 --- a/frappe/website/utils.py +++ b/frappe/website/utils.py @@ -488,11 +488,12 @@ def set_content_type(response, data, path): return data def add_preload_headers(response): - from bs4 import BeautifulSoup + from bs4 import BeautifulSoup, SoupStrainer try: preload = [] - soup = BeautifulSoup(response.data, "lxml") + strainer = SoupStrainer(re.compile("script|link")) + soup = BeautifulSoup(response.data, "lxml", parse_only=strainer) for elem in soup.find_all('script', src=re.compile(".*")): preload.append(("script", elem.get("src"))) diff --git a/frappe/workflow/doctype/workflow_action/workflow_action.py b/frappe/workflow/doctype/workflow_action/workflow_action.py index b70ffb2406..5eedc27d9c 100644 --- a/frappe/workflow/doctype/workflow_action/workflow_action.py +++ b/frappe/workflow/doctype/workflow_action/workflow_action.py @@ -133,9 +133,12 @@ def return_link_expired_page(doc, doc_workflow_state): def clear_old_workflow_actions(doc, user=None): user = user if user else frappe.session.user - frappe.db.sql("""DELETE FROM `tabWorkflow Action` - WHERE `reference_doctype`=%s AND `reference_name`=%s AND `user`!=%s AND `status`='Open'""", - (doc.get('doctype'), doc.get('name'), user)) + frappe.db.delete("Workflow Action", { + "reference_doctype": doc.get("doctype"), + "reference_name": doc.get("name"), + "user": ("!=", user), + "status": "Open" + }) def update_completed_workflow_actions(doc, user=None): user = user if user else frappe.session.user @@ -253,11 +256,10 @@ def is_workflow_action_already_created(doc): def clear_workflow_actions(doctype, name): if not (doctype and name): return - - frappe.db.sql('''delete from `tabWorkflow Action` - where reference_doctype=%s and reference_name=%s''', - (doctype, name)) - + frappe.db.delete("Workflow Action", { + "reference_doctype": doctype, + "reference_name": name + }) def get_doc_workflow_state(doc): workflow_name = get_workflow_name(doc.get('doctype')) workflow_state_field = get_workflow_state_field(workflow_name) diff --git a/frappe/www/app.html b/frappe/www/app.html index c8172693b9..68a6dc8e86 100644 --- a/frappe/www/app.html +++ b/frappe/www/app.html @@ -1,5 +1,5 @@ - + diff --git a/frappe/www/app.py b/frappe/www/app.py index 27505c8131..acf6dde000 100644 --- a/frappe/www/app.py +++ b/frappe/www/app.py @@ -6,6 +6,7 @@ import os, re import frappe from frappe import _ import frappe.sessions +from frappe.utils.jinja_globals import is_rtl def get_context(context): if frappe.session.user == "Guest": @@ -40,6 +41,8 @@ def get_context(context): "build_version": frappe.utils.get_build_version(), "include_js": hooks["app_include_js"], "include_css": hooks["app_include_css"], + "layout_direction": "rtl" if is_rtl() else "ltr", + "lang": frappe.local.lang, "sounds": hooks["sounds"], "boot": boot if context.get("for_mobile") else boot_json, "desk_theme": desk_theme or "Light", diff --git a/frappe/www/printview.html b/frappe/www/printview.html index 8bc6e0cb80..3f8d4201fb 100644 --- a/frappe/www/printview.html +++ b/frappe/www/printview.html @@ -1,14 +1,11 @@ - +