diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 5a96c3fea8..03efd1d30d 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -34,3 +34,6 @@ c0c5b2ebdddbe8898ce2d5e5365f4931ff73b6bf # db.get_all -> get_all 2eec621e95564c359ad22da79501a855c1f32b03 + +# minor formatting fix in `user.py` +f223bc02490902dfcc32892058f13f343d51fbaf diff --git a/frappe/__init__.py b/frappe/__init__.py index 5efdfd8ce9..e5a0b9c4aa 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -1049,18 +1049,26 @@ def reset_metadata_version(): def new_doc( doctype: str, + *, parent_doc: Optional["Document"] = None, parentfield: str | None = None, as_dict: bool = False, + **kwargs, ) -> "Document": """Returns a new document of the given DocType with defaults set. :param doctype: DocType of the new document. :param parent_doc: [optional] add to parent document. - :param parentfield: [optional] add against this `parentfield`.""" + :param parentfield: [optional] add against this `parentfield`. + :param as_dict: [optional] return as dictionary instead of Document. + :param kwargs: [optional] You can specify fields as field=value pairs in function call. + """ + from frappe.model.create_new import get_new_doc - return get_new_doc(doctype, parent_doc, parentfield, as_dict=as_dict) + new_doc = get_new_doc(doctype, parent_doc, parentfield, as_dict=as_dict) + + return new_doc.update(kwargs) def set_value(doctype, docname, fieldname, value=None): diff --git a/frappe/app.py b/frappe/app.py index fab8facd3f..55855efaf9 100644 --- a/frappe/app.py +++ b/frappe/app.py @@ -19,7 +19,6 @@ import frappe.recorder import frappe.utils.response from frappe import _ from frappe.auth import SAFE_HTTP_METHODS, UNSAFE_HTTP_METHODS, HTTPRequest -from frappe.core.doctype.comment.comment import update_comments_in_parent_after_request from frappe.middlewares import StaticDataMiddleware from frappe.utils import cint, get_site_name, sanitize_html from frappe.utils.error import make_error_snapshot @@ -351,8 +350,6 @@ def sync_database(rollback: bool) -> bool: frappe.db.commit() rollback = False - update_comments_in_parent_after_request() - return rollback diff --git a/frappe/core/doctype/comment/comment.py b/frappe/core/doctype/comment/comment.py index dff13e1170..c86c7811ad 100644 --- a/frappe/core/doctype/comment/comment.py +++ b/frappe/core/doctype/comment/comment.py @@ -152,14 +152,9 @@ def update_comments_in_parent(reference_doctype, reference_name, _comments): except Exception as e: if frappe.db.is_column_missing(e) and getattr(frappe.local, "request", None): - # missing column and in request, add column and update after commit - frappe.local._comments = getattr(frappe.local, "_comments", []) + [ - (reference_doctype, reference_name, _comments) - ] - + pass elif frappe.db.is_data_too_long(e): raise frappe.DataTooLongException - else: raise else: @@ -169,13 +164,3 @@ def update_comments_in_parent(reference_doctype, reference_name, _comments): # Clear route cache if route := frappe.get_cached_value(reference_doctype, reference_name, "route"): clear_cache(route) - - -def update_comments_in_parent_after_request(): - """update _comments in parent if _comments column is missing""" - if hasattr(frappe.local, "_comments"): - for (reference_doctype, reference_name, _comments) in frappe.local._comments: - add_column(reference_doctype, "_comments", "Text") - update_comments_in_parent(reference_doctype, reference_name, _comments) - - frappe.db.commit() diff --git a/frappe/core/doctype/doctype/doctype.py b/frappe/core/doctype/doctype/doctype.py index 91a317dbff..12545adb4e 100644 --- a/frappe/core/doctype/doctype/doctype.py +++ b/frappe/core/doctype/doctype/doctype.py @@ -33,7 +33,7 @@ from frappe.model.meta import Meta from frappe.modules import get_doc_path, make_boilerplate from frappe.modules.import_file import get_file_path from frappe.query_builder.functions import Concat -from frappe.utils import cint, random_string +from frappe.utils import cint, flt, random_string from frappe.website.utils import clear_cache if TYPE_CHECKING: @@ -1751,3 +1751,14 @@ def get_field(doc, fieldname): for field in doc.fields: if field.fieldname == fieldname: return field + + +@frappe.whitelist() +def get_row_size_utilization(doctype: str) -> float: + """Get row size utilization in percentage""" + + frappe.has_permission("DocType", throw=True) + try: + return flt(frappe.db.get_row_size(doctype) / frappe.db.MAX_ROW_SIZE_LIMIT * 100, 2) + except Exception: + return 0.0 diff --git a/frappe/core/doctype/patch_log/patch_log.js b/frappe/core/doctype/patch_log/patch_log.js index 171a1d3a0f..78580a0cb0 100644 --- a/frappe/core/doctype/patch_log/patch_log.js +++ b/frappe/core/doctype/patch_log/patch_log.js @@ -4,5 +4,9 @@ frappe.ui.form.on("Patch Log", { refresh: function (frm) { frm.disable_save(); + + frm.add_custom_button(__("Re-Run Patch"), () => { + frm.call("rerun_patch"); + }); }, }); diff --git a/frappe/core/doctype/patch_log/patch_log.py b/frappe/core/doctype/patch_log/patch_log.py index c7d619017e..284a80df35 100644 --- a/frappe/core/doctype/patch_log/patch_log.py +++ b/frappe/core/doctype/patch_log/patch_log.py @@ -4,11 +4,20 @@ # License: MIT. See LICENSE import frappe +from frappe import _ from frappe.model.document import Document class PatchLog(Document): - pass + @frappe.whitelist() + def rerun_patch(self): + from frappe.modules.patch_handler import run_single + + if not frappe.conf.developer_mode: + frappe.throw(_("Re-running patch is only allowed in developer mode.")) + + run_single(self.patch, force=True) + frappe.msgprint(_("Successfully re-ran patch: {0}").format(self.patch), alert=True) def before_migrate(): diff --git a/frappe/core/doctype/system_settings/system_settings.json b/frappe/core/doctype/system_settings/system_settings.json index 091dc1df1e..5efe87da25 100644 --- a/frappe/core/doctype/system_settings/system_settings.json +++ b/frappe/core/doctype/system_settings/system_settings.json @@ -72,6 +72,8 @@ "disable_standard_email_footer", "hide_footer_in_auto_email_reports", "attach_view_link", + "welcome_email_template", + "reset_password_template", "prepared_report_section", "max_auto_email_report_per_user", "system_updates_section", @@ -549,12 +551,24 @@ "fieldname": "enable_telemetry", "fieldtype": "Check", "label": "Allow Sending Usage Data for Improving Applications" + }, + { + "fieldname": "welcome_email_template", + "fieldtype": "Link", + "label": "Welcome Email Template", + "options": "Email Template" + }, + { + "fieldname": "reset_password_template", + "fieldtype": "Link", + "label": "Reset Password Template", + "options": "Email Template" } ], "icon": "fa fa-cog", "issingle": 1, "links": [], - "modified": "2023-04-23 11:14:59.302851", + "modified": "2023-05-25 13:02:54.808773", "modified_by": "Administrator", "module": "Core", "name": "System Settings", diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py index 8e00aa7f0f..94ea8b16a0 100644 --- a/frappe/core/doctype/user/user.py +++ b/frappe/core/doctype/user/user.py @@ -329,7 +329,16 @@ class User(Document): return (self.first_name or "") + (self.first_name and " " or "") + (self.last_name or "") def password_reset_mail(self, link): - self.send_login_mail(_("Password Reset"), "password_reset", {"link": link}, now=True) + + reset_password_template = frappe.db.get_system_setting("reset_password_template") + + self.send_login_mail( + _("Password Reset"), + "password_reset", + {"link": link}, + now=True, + custom_template=reset_password_template, + ) def send_welcome_mail_to_user(self): from frappe.utils import get_url @@ -346,6 +355,8 @@ class User(Document): else: subject = _("Complete Registration") + welcome_email_template = frappe.db.get_system_setting("welcome_email_template") + self.send_login_mail( subject, "new_user", @@ -353,9 +364,10 @@ class User(Document): link=link, site_url=get_url(), ), + custom_template=welcome_email_template, ) - def send_login_mail(self, subject, template, add_args, now=None): + def send_login_mail(self, subject, template, add_args, now=None, custom_template=None): """send mail with login details""" from frappe.utils import get_url from frappe.utils.user import get_user_fullname @@ -378,11 +390,19 @@ class User(Document): frappe.session.user not in STANDARD_USERS and get_formatted_email(frappe.session.user) or None ) + if custom_template: + from frappe.email.doctype.email_template.email_template import get_email_template + + email_template = get_email_template(custom_template, args) + subject = email_template.get("subject") + content = email_template.get("message") + frappe.sendmail( recipients=self.email, sender=sender, subject=subject, - template=template, + template=template if not custom_template else None, + content=content if custom_template else None, args=args, header=[subject, "green"], delayed=(not now) if now is not None else self.flags.delay_emails, diff --git a/frappe/core/doctype/view_log/view_log.py b/frappe/core/doctype/view_log/view_log.py index 8383af818e..5dde78d007 100644 --- a/frappe/core/doctype/view_log/view_log.py +++ b/frappe/core/doctype/view_log/view_log.py @@ -1,8 +1,15 @@ # Copyright (c) 2018, Frappe Technologies and contributors # License: MIT. See LICENSE +import frappe from frappe.model.document import Document class ViewLog(Document): - pass + @staticmethod + def clear_old_logs(days=180): + from frappe.query_builder import Interval + from frappe.query_builder.functions import Now + + table = frappe.qb.DocType("View Log") + frappe.db.delete(table, filters=(table.modified < (Now() - Interval(days=days)))) diff --git a/frappe/custom/doctype/customize_form/customize_form.py b/frappe/custom/doctype/customize_form/customize_form.py index 42cbf33f4f..9e6b8990d5 100644 --- a/frappe/custom/doctype/customize_form/customize_form.py +++ b/frappe/custom/doctype/customize_form/customize_form.py @@ -172,7 +172,18 @@ class CustomizeForm(Document): check_email_append_to(self) if self.flags.update_db: - frappe.db.updatedb(self.doc_type) + try: + frappe.db.updatedb(self.doc_type) + except Exception as e: + if frappe.db.is_db_table_size_limit(e): + frappe.throw( + _("You have hit the row size limit on database table: {0}").format( + "" + "Maximum Number of Fields in a Form" + ), + title=_("Database Table Row Size Limit"), + ) + raise if not hasattr(self, "hide_success") or not self.hide_success: frappe.msgprint(_("{0} updated").format(_(self.doc_type)), alert=True) diff --git a/frappe/database/database.py b/frappe/database/database.py index 8b077ce4f7..728d1e9584 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -105,6 +105,8 @@ class Database: self.password = password or frappe.conf.db_password self.value_cache = {} + self.logger = frappe.logger("database") + self.logger.setLevel("WARNING") # self.db_type: str # self.last_query (lazy) attribute of last sql query executed @@ -122,7 +124,7 @@ class Database: if execution_timeout := get_query_execution_timeout(): self.set_execution_timeout(execution_timeout) except Exception as e: - frappe.logger("database").warning(f"Couldn't set execution timeout {e}") + self.logger.warning(f"Couldn't set execution timeout {e}") def set_execution_timeout(self, seconds: int): """Set session speicifc timeout on exeuction of statements. @@ -285,7 +287,13 @@ class Database: return self.convert_to_lists(self.last_result) return self.last_result - def _log_query(self, mogrified_query: str, debug: bool = False, explain: bool = False) -> None: + def _log_query( + self, + mogrified_query: str, + debug: bool = False, + explain: bool = False, + unmogrified_query: str = "", + ) -> None: """Takes the query and logs it to various interfaces according to the settings.""" _query = None @@ -303,6 +311,12 @@ class Database: _query = _query or str(mogrified_query) frappe.log(f"<<<< query\n{_query}\n>>>>") + if unmogrified_query and is_query_type( + unmogrified_query, ("alter", "drop", "create", "truncate", "rename") + ): + _query = _query or str(mogrified_query) + self.logger.warning("DDL Query made to DB:\n" + _query) + if frappe.flags.in_migrate: _query = _query or str(mogrified_query) self.log_touched_tables(_query) @@ -314,7 +328,7 @@ class Database: # like cursor._transformed_statement from the cursor object. We can also avoid setting # mogrified_query if we don't need to log it. mogrified_query = self.lazy_mogrify(query, values) - self._log_query(mogrified_query, debug, explain) + self._log_query(mogrified_query, debug, explain, unmogrified_query=query) return mogrified_query def mogrify(self, query: Query, values: QueryValues): @@ -812,6 +826,7 @@ class Database: fields=fields, distinct=distinct, limit=limit, + validate_filters=True, ) if isinstance(fields, str) and fields == "*": as_dict = True @@ -840,6 +855,7 @@ class Database: order_by=order_by, distinct=distinct, limit=limit, + validate_filters=True, ).run(debug=debug, run=run, as_dict=as_dict, pluck=pluck) return {} @@ -889,7 +905,12 @@ class Database: field, val, modified=modified, modified_by=modified_by, update_modified=update_modified ) - query = frappe.qb.get_query(table=dt, filters=dn, update=True) + query = frappe.qb.get_query( + table=dt, + filters=dn, + update=True, + validate_filters=True, + ) if isinstance(dn, str): frappe.clear_document_cache(dt, dn) @@ -1057,9 +1078,13 @@ class Database: cache_count = frappe.cache().get_value(f"doctype:count:{dt}") if cache_count is not None: return cache_count - count = frappe.qb.get_query(table=dt, filters=filters, fields=Count("*"), distinct=distinct).run( - debug=debug - )[0][0] + count = frappe.qb.get_query( + table=dt, + filters=filters, + fields=Count("*"), + distinct=distinct, + validate_filters=True, + ).run(debug=debug)[0][0] if not filters and cache: frappe.cache().set_value(f"doctype:count:{dt}", count, expires_in_sec=86400) return count @@ -1179,7 +1204,12 @@ class Database: Doctype name can be passed directly, it will be pre-pended with `tab`. """ filters = filters or kwargs.get("conditions") - query = frappe.qb.get_query(table=doctype, filters=filters, delete=True) + query = frappe.qb.get_query( + table=doctype, + filters=filters, + delete=True, + validate_filters=True, + ) if "debug" not in kwargs: kwargs["debug"] = debug return query.run(**kwargs) @@ -1269,6 +1299,10 @@ class Database: return get_next_val(*args, **kwargs) + def get_row_size(self, doctype: str) -> int: + """Get estimated max row size of any table in bytes.""" + raise NotImplementedError + def enqueue_jobs_after_commit(): from frappe.utils.background_jobs import ( diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index 8e52cc7ffd..f14fce2710 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -76,6 +76,10 @@ class MariaDBExceptionUtil: def is_data_too_long(e: pymysql.Error) -> bool: return e.args[0] == ER.DATA_TOO_LONG + @staticmethod + def is_db_table_size_limit(e: pymysql.Error) -> bool: + return e.args[0] == ER.TOO_BIG_ROWSIZE + @staticmethod def is_primary_key_violation(e: pymysql.Error) -> bool: return ( @@ -145,6 +149,7 @@ class MariaDBDatabase(MariaDBConnectionUtil, MariaDBExceptionUtil, Database): UnicodeWithAttrs: escape_string, } default_port = "3306" + MAX_ROW_SIZE_LIMIT = 65_535 # bytes def setup_type_map(self): self.db_type = "mariadb" @@ -200,8 +205,8 @@ class MariaDBDatabase(MariaDBConnectionUtil, MariaDBExceptionUtil, Database): return db_size[0].get("database_size") def log_query(self, query, values, debug, explain): - self.last_query = query = self._cursor._executed - self._log_query(query, debug, explain) + self.last_query = self._cursor._executed + self._log_query(self.last_query, debug, explain, query) return self.last_query @staticmethod @@ -445,3 +450,56 @@ class MariaDBDatabase(MariaDBConnectionUtil, MariaDBExceptionUtil, Database): frappe.cache().set_value("db_tables", tables) return tables + + def get_row_size(self, doctype: str) -> int: + """Get estimated max row size of any table in bytes.""" + + # Query reused from this answer: https://dba.stackexchange.com/a/313889/274503 + # Modification: get values for particular table instead of full summary. + # Reference: https://mariadb.com/kb/en/data-type-storage-requirements/ + + est_row_size = frappe.db.sql( + """ + SELECT SUM(col_sizes.col_size) AS EST_MAX_ROW_SIZE + FROM ( + SELECT + cols.COLUMN_NAME, + CASE cols.DATA_TYPE + WHEN 'tinyint' THEN 1 + WHEN 'smallint' THEN 2 + WHEN 'mediumint' THEN 3 + WHEN 'int' THEN 4 + WHEN 'bigint' THEN 8 + WHEN 'float' THEN IF(cols.NUMERIC_PRECISION > 24, 8, 4) + WHEN 'double' THEN 8 + WHEN 'decimal' THEN ((cols.NUMERIC_PRECISION - cols.NUMERIC_SCALE) DIV 9)*4 + (cols.NUMERIC_SCALE DIV 9)*4 + CEIL(MOD(cols.NUMERIC_PRECISION - cols.NUMERIC_SCALE,9)/2) + CEIL(MOD(cols.NUMERIC_SCALE,9)/2) + WHEN 'bit' THEN (cols.NUMERIC_PRECISION + 7) DIV 8 + WHEN 'year' THEN 1 + WHEN 'date' THEN 3 + WHEN 'time' THEN 3 + CEIL(cols.DATETIME_PRECISION /2) + WHEN 'datetime' THEN 5 + CEIL(cols.DATETIME_PRECISION /2) + WHEN 'timestamp' THEN 4 + CEIL(cols.DATETIME_PRECISION /2) + WHEN 'char' THEN cols.CHARACTER_OCTET_LENGTH + WHEN 'binary' THEN cols.CHARACTER_OCTET_LENGTH + WHEN 'varchar' THEN IF(cols.CHARACTER_OCTET_LENGTH > 255, 2, 1) + cols.CHARACTER_OCTET_LENGTH + WHEN 'varbinary' THEN IF(cols.CHARACTER_OCTET_LENGTH > 255, 2, 1) + cols.CHARACTER_OCTET_LENGTH + WHEN 'tinyblob' THEN 9 + WHEN 'tinytext' THEN 9 + WHEN 'blob' THEN 10 + WHEN 'text' THEN 10 + WHEN 'mediumblob' THEN 11 + WHEN 'mediumtext' THEN 11 + WHEN 'longblob' THEN 12 + WHEN 'longtext' THEN 12 + WHEN 'enum' THEN 2 + WHEN 'set' THEN 8 + ELSE 0 + END AS col_size + FROM INFORMATION_SCHEMA.COLUMNS cols + WHERE cols.TABLE_NAME = %s + ) AS col_sizes;""", + (get_table_name(doctype),), + ) + + if est_row_size: + return int(est_row_size[0][0]) diff --git a/frappe/database/postgres/database.py b/frappe/database/postgres/database.py index 836a689251..2d5b3a893f 100644 --- a/frappe/database/postgres/database.py +++ b/frappe/database/postgres/database.py @@ -107,6 +107,10 @@ class PostgresExceptionUtil: def is_data_too_long(e): return getattr(e, "pgcode", None) == STRING_DATA_RIGHT_TRUNCATION + @staticmethod + def is_db_table_size_limit(e) -> bool: + return False + class PostgresDatabase(PostgresExceptionUtil, Database): REGEX_CHARACTER = "~" diff --git a/frappe/database/query.py b/frappe/database/query.py index 02beff9afc..06295d33a6 100644 --- a/frappe/database/query.py +++ b/frappe/database/query.py @@ -9,6 +9,7 @@ from pypika.queries import QueryBuilder, Table import frappe from frappe import _ from frappe.database.operator_map import OPERATOR_MAP +from frappe.database.schema import SPECIAL_CHAR_PATTERN from frappe.database.utils import DefaultOrderBy, get_doctype_name from frappe.query_builder import Criterion, Field, Order, functions from frappe.query_builder.functions import Function, SqlFunctions @@ -44,9 +45,12 @@ class Engine: update: bool = False, into: bool = False, delete: bool = False, + *, + validate_filters: bool = False, ) -> QueryBuilder: self.is_mariadb = frappe.db.db_type == "mariadb" self.is_postgres = frappe.db.db_type == "postgres" + self.validate_filters = validate_filters if isinstance(table, Table): self.table = table @@ -157,14 +161,16 @@ class Engine: _value = value _operator = operator - if isinstance(_field, Field): + if not isinstance(_field, str): pass - elif dynamic_field := DynamicTableField.parse(field, self.doctype): + elif not self.validate_filters and ( + dynamic_field := DynamicTableField.parse(field, self.doctype) + ): # apply implicit join if link field's field is referenced self.query = dynamic_field.apply_join(self.query) _field = dynamic_field.field - elif has_function(field): - _field = self.get_function_object(field) + elif self.validate_filters and SPECIAL_CHAR_PATTERN.search(_field): + frappe.throw(_("Invalid filter: {0}").format(_field)) elif not doctype or doctype == self.doctype: _field = self.table[field] elif doctype: diff --git a/frappe/desk/doctype/number_card/number_card.py b/frappe/desk/doctype/number_card/number_card.py index 451dc699fe..a8e4841953 100644 --- a/frappe/desk/doctype/number_card/number_card.py +++ b/frappe/desk/doctype/number_card/number_card.py @@ -202,7 +202,11 @@ def get_cards_for_user(doctype, txt, searchfield, start, page_len, filters): if txt: search_conditions = [numberCard[field].like(f"%{txt}%") for field in searchfields] - condition_query = frappe.qb.get_query(doctype, filters=filters) + condition_query = frappe.qb.get_query( + doctype, + filters=filters, + validate_filters=True, + ) return ( condition_query.select(numberCard.name, numberCard.label, numberCard.document_type) diff --git a/frappe/desk/listview.py b/frappe/desk/listview.py index 05d45ad9ac..a1db82810e 100644 --- a/frappe/desk/listview.py +++ b/frappe/desk/listview.py @@ -36,7 +36,12 @@ def get_group_by_count(doctype: str, current_filters: str, field: str) -> list[d ToDo = DocType("ToDo") User = DocType("User") count = Count("*").as_("count") - filtered_records = frappe.qb.get_query(doctype, filters=current_filters, fields=["name"]) + filtered_records = frappe.qb.get_query( + doctype, + filters=current_filters, + fields=["name"], + validate_filters=True, + ) return ( frappe.qb.from_(ToDo) diff --git a/frappe/integrations/doctype/webhook/test_webhook.py b/frappe/integrations/doctype/webhook/test_webhook.py index 8284db7fd3..7235447662 100644 --- a/frappe/integrations/doctype/webhook/test_webhook.py +++ b/frappe/integrations/doctype/webhook/test_webhook.py @@ -206,3 +206,59 @@ class TestWebhook(FrappeTestCase): enqueue_webhook(doc, wh) log = frappe.get_last_doc("Webhook Request Log") self.assertEqual(len(json.loads(log.response)["json"]), 3) + + def test_webhook_with_dynamic_url_enabled(self): + wh_config = { + "doctype": "Webhook", + "webhook_doctype": "Note", + "webhook_docevent": "after_insert", + "enabled": 1, + "request_url": "https://httpbin.org/anything/{{ doc.doctype }}", + "is_dynamic_url": 1, + "request_method": "POST", + "request_structure": "JSON", + "webhook_json": "{}", + "meets_condition": "Yes", + "webhook_headers": [ + { + "key": "Content-Type", + "value": "application/json", + } + ], + } + + with get_test_webhook(wh_config) as wh: + doc = frappe.new_doc("Note") + doc.title = "Test Webhook Note" + enqueue_webhook(doc, wh) + log = frappe.get_last_doc("Webhook Request Log") + self.assertEqual(json.loads(log.response)["url"], "https://httpbin.org/anything/Note") + + def test_webhook_with_dynamic_url_disabled(self): + wh_config = { + "doctype": "Webhook", + "webhook_doctype": "Note", + "webhook_docevent": "after_insert", + "enabled": 1, + "request_url": "https://httpbin.org/anything/{{doc.doctype}}", + "is_dynamic_url": 0, + "request_method": "POST", + "request_structure": "JSON", + "webhook_json": "{}", + "meets_condition": "Yes", + "webhook_headers": [ + { + "key": "Content-Type", + "value": "application/json", + } + ], + } + + with get_test_webhook(wh_config) as wh: + doc = frappe.new_doc("Note") + doc.title = "Test Webhook Note" + enqueue_webhook(doc, wh) + log = frappe.get_last_doc("Webhook Request Log") + self.assertEqual( + json.loads(log.response)["url"], "https://httpbin.org/anything/{{doc.doctype}}" + ) diff --git a/frappe/integrations/doctype/webhook/webhook.json b/frappe/integrations/doctype/webhook/webhook.json index c4fc4f675d..cfb2a2e01c 100644 --- a/frappe/integrations/doctype/webhook/webhook.json +++ b/frappe/integrations/doctype/webhook/webhook.json @@ -18,8 +18,9 @@ "html_condition", "sb_webhook", "request_url", - "request_method", + "is_dynamic_url", "cb_webhook", + "request_method", "request_structure", "sb_security", "enable_security", @@ -202,6 +203,13 @@ { "fieldname": "section_break_28", "fieldtype": "Section Break" + }, + { + "default": "0", + "description": "On checking this option, URL will be treated like a jinja template string", + "fieldname": "is_dynamic_url", + "fieldtype": "Check", + "label": "Is Dynamic URL?" } ], "links": [ @@ -210,7 +218,7 @@ "link_fieldname": "webhook" } ], - "modified": "2023-05-21 15:42:58.844826", + "modified": "2023-05-22 16:30:10.740512", "modified_by": "Administrator", "module": "Integrations", "name": "Webhook", diff --git a/frappe/integrations/doctype/webhook/webhook.py b/frappe/integrations/doctype/webhook/webhook.py index 7d168c659f..1b56a1b129 100644 --- a/frappe/integrations/doctype/webhook/webhook.py +++ b/frappe/integrations/doctype/webhook/webhook.py @@ -115,29 +115,34 @@ def enqueue_webhook(doc, webhook) -> None: webhook: Webhook = frappe.get_doc("Webhook", webhook.get("name")) headers = get_webhook_headers(doc, webhook) data = get_webhook_data(doc, webhook) - r = None + if webhook.is_dynamic_url: + request_url = frappe.render_template(webhook.request_url, get_context(doc)) + else: + request_url = webhook.request_url + + r = None for i in range(3): try: r = requests.request( method=webhook.request_method, - url=webhook.request_url, + url=request_url, data=json.dumps(data, default=str), headers=headers, timeout=5, ) r.raise_for_status() frappe.logger().debug({"webhook_success": r.text}) - log_request(webhook.name, doc.name, webhook.request_url, headers, data, r) + log_request(webhook.name, doc.name, request_url, headers, data, r) break except requests.exceptions.ReadTimeout as e: frappe.logger().debug({"webhook_error": e, "try": i + 1}) - log_request(webhook.name, doc.name, webhook.request_url, headers, data) + log_request(webhook.name, doc.name, request_url, headers, data) except Exception as e: frappe.logger().debug({"webhook_error": e, "try": i + 1}) - log_request(webhook.name, doc.name, webhook.request_url, headers, data, r) + log_request(webhook.name, doc.name, request_url, headers, data, r) sleep(3 * i + 1) if i != 2: continue diff --git a/frappe/model/base_document.py b/frappe/model/base_document.py index 811ba5894c..63188e749d 100644 --- a/frappe/model/base_document.py +++ b/frappe/model/base_document.py @@ -530,7 +530,7 @@ class BaseDocument: if not ignore_if_duplicate: frappe.msgprint( - _("{0} {1} already exists").format(self.doctype, frappe.bold(self.name)), + _("{0} {1} already exists").format(_(self.doctype), frappe.bold(self.name)), title=_("Duplicate Name"), indicator="red", ) diff --git a/frappe/model/mapper.py b/frappe/model/mapper.py index 9df79ef276..4b9051f59c 100644 --- a/frappe/model/mapper.py +++ b/frappe/model/mapper.py @@ -62,6 +62,7 @@ def get_mapped_doc( postprocess=None, ignore_permissions=False, ignore_child_tables=False, + cached=False, ): apply_strict_user_permissions = frappe.get_system_settings("apply_strict_user_permissions") @@ -79,7 +80,10 @@ def get_mapped_doc( ): target_doc.raise_no_permission_to("create") - source_doc = frappe.get_doc(from_doctype, from_docname) + if cached: + source_doc = frappe.get_cached_doc(from_doctype, from_docname) + else: + source_doc = frappe.get_doc(from_doctype, from_docname) if not ignore_permissions: if not source_doc.has_permission("read"): @@ -255,7 +259,9 @@ def map_fetch_fields(target_doc, df, no_copy_fields): def map_child_doc(source_d, target_parent, table_map, source_parent=None): target_child_doctype = table_map["doctype"] target_parentfield = target_parent.get_parentfield_of_doctype(target_child_doctype) - target_d = frappe.new_doc(target_child_doctype, target_parent, target_parentfield) + target_d = frappe.new_doc( + target_child_doctype, parent_doc=target_parent, parentfield=target_parentfield + ) map_doc(source_d, target_d, table_map, source_parent) diff --git a/frappe/patches/v11_0/apply_customization_to_custom_doctype.py b/frappe/patches/v11_0/apply_customization_to_custom_doctype.py index d652efcef7..90986e065a 100644 --- a/frappe/patches/v11_0/apply_customization_to_custom_doctype.py +++ b/frappe/patches/v11_0/apply_customization_to_custom_doctype.py @@ -44,7 +44,7 @@ def execute(): if field: field.update(cf) else: - df = frappe.new_doc("DocField", meta, "fields") + df = frappe.new_doc("DocField", parent_doc=meta, parentfield="fields") df.update(cf) meta.fields.append(df) frappe.db.delete("Custom Field", {"name": cf.name}) diff --git a/frappe/printing/doctype/print_format/print_format.json b/frappe/printing/doctype/print_format/print_format.json index 7f4408c950..664692ec45 100644 --- a/frappe/printing/doctype/print_format/print_format.json +++ b/frappe/printing/doctype/print_format/print_format.json @@ -245,6 +245,7 @@ "default": "14", "fieldname": "font_size", "fieldtype": "Int", + "hidden": 1, "label": "Font Size" }, { @@ -258,7 +259,7 @@ "icon": "fa fa-print", "idx": 1, "links": [], - "modified": "2022-11-09 15:29:46.709305", + "modified": "2023-05-31 15:40:52.919029", "modified_by": "Administrator", "module": "Printing", "name": "Print Format", diff --git a/frappe/printing/doctype/print_settings/print_settings.json b/frappe/printing/doctype/print_settings/print_settings.json index f45de7637d..a67440b54e 100644 --- a/frappe/printing/doctype/print_settings/print_settings.json +++ b/frappe/printing/doctype/print_settings/print_settings.json @@ -47,7 +47,7 @@ "default": "1", "fieldname": "repeat_header_footer", "fieldtype": "Check", - "label": "Repeat Header and Footer in PDF" + "label": "Repeat Header and Footer" }, { "fieldname": "column_break_4", @@ -176,7 +176,7 @@ "index_web_pages_for_search": 1, "issingle": 1, "links": [], - "modified": "2021-09-17 12:59:14.783694", + "modified": "2023-05-30 14:55:25.740691", "modified_by": "Administrator", "module": "Printing", "name": "Print Settings", @@ -193,5 +193,6 @@ "quick_entry": 1, "sort_field": "modified", "sort_order": "DESC", + "states": [], "track_changes": 1 } \ No newline at end of file diff --git a/frappe/printing/page/print/print.js b/frappe/printing/page/print/print.js index 8e5e165c78..f930359b58 100644 --- a/frappe/printing/page/print/print.js +++ b/frappe/printing/page/print/print.js @@ -91,7 +91,7 @@ frappe.ui.form.PrintView = class { fieldtype: "Link", fieldname: "print_format", options: "Print Format", - placeholder: __("Print Format"), + label: __("Print Format"), get_query: () => { return { filters: { doc_type: this.frm.doctype } }; }, @@ -101,7 +101,7 @@ frappe.ui.form.PrintView = class { this.language_selector = this.add_sidebar_item({ fieldtype: "Link", fieldname: "language", - placeholder: __("Language"), + label: __("Language"), options: "Language", change: () => { this.set_user_lang(); @@ -109,12 +109,27 @@ frappe.ui.form.PrintView = class { }, }).$input; + let description = ""; + if (!cint(this.print_settings.repeat_header_footer)) { + description = + "
" + + __("Footer might not be visible as {0} option is disabled
", [ + `${__( + "Repeat Header and Footer" + )}`, + ]); + } + const print_view = this; this.letterhead_selector = this.add_sidebar_item({ fieldtype: "Link", fieldname: "letterhead", options: "Letter Head", - placeholder: __("Letter Head"), - change: () => this.preview(), + label: __("Letter Head"), + description: description, + change: function () { + this.set_description(this.get_value() ? description : ""); + print_view.preview(); + }, }).$input; this.sidebar_dynamic_section = $(`
`).appendTo( this.sidebar diff --git a/frappe/public/js/frappe/doctype/index.js b/frappe/public/js/frappe/doctype/index.js index 0dc5fd0a34..a0023164d7 100644 --- a/frappe/public/js/frappe/doctype/index.js +++ b/frappe/public/js/frappe/doctype/index.js @@ -22,6 +22,29 @@ frappe.model.DocTypeController = class DocTypeController extends frappe.ui.form. }; } + refresh() { + this.show_db_utilization(); + } + + show_db_utilization() { + const doctype = this.frm.doc.doc_type || this.frm.doc.name; + frappe + .xcall("frappe.core.doctype.doctype.doctype.get_row_size_utilization", { + doctype, + }) + .then((r) => { + if (r < 50.0) return; + this.frm.dashboard.show_progress( + __("Database Row Size Utilization"), + r, + __( + "Database Table Row Size Utilization: {0}%, this limits number of fields you can add.", + [r] + ) + ); + }); + } + max_attachments() { if (!this.frm.doc.max_attachments) { return; diff --git a/frappe/public/js/frappe/form/controls/multicheck.js b/frappe/public/js/frappe/form/controls/multicheck.js index de4c330ff7..7b980299aa 100644 --- a/frappe/public/js/frappe/form/controls/multicheck.js +++ b/frappe/public/js/frappe/form/controls/multicheck.js @@ -18,13 +18,11 @@ frappe.ui.form.ControlMultiCheck = class ControlMultiCheck extends frappe.ui.for this.$checkbox_area = $(`
`).appendTo( this.wrapper ); - this.refresh(); } refresh() { this.set_options(); this.bind_checkboxes(); - this.refresh_input(); super.refresh(); } diff --git a/frappe/public/js/frappe/form/formatters.js b/frappe/public/js/frappe/form/formatters.js index f4371f901b..9739eed8bb 100644 --- a/frappe/public/js/frappe/form/formatters.js +++ b/frappe/public/js/frappe/form/formatters.js @@ -365,8 +365,14 @@ frappe.form.formatters = { ` : ""; }, + Attach: format_attachment_url, + AttachImage: format_attachment_url, }; +function format_attachment_url(url) { + return url ? `${url}` : ""; +} + frappe.form.get_formatter = function (fieldtype) { if (!fieldtype) fieldtype = "Data"; return frappe.form.formatters[fieldtype.replace(/ /g, "")] || frappe.form.formatters.Data; diff --git a/frappe/public/js/frappe/form/templates/form_links.html b/frappe/public/js/frappe/form/templates/form_links.html index 57edb69a15..cd423bb238 100644 --- a/frappe/public/js/frappe/form/templates/form_links.html +++ b/frappe/public/js/frappe/form/templates/form_links.html @@ -5,7 +5,7 @@ {% } %}
- {{ __(transactions[i].label) }} + {{ __(transactions[i].label) }}
{% for (let j=0; j < transactions[i].items.length; j++) { %} {% let doctype = transactions[i].items[j]; %} diff --git a/frappe/public/js/frappe/list/list_view.js b/frappe/public/js/frappe/list/list_view.js index 7fd8d2c55c..f009593f6f 100644 --- a/frappe/public/js/frappe/list/list_view.js +++ b/frappe/public/js/frappe/list/list_view.js @@ -1029,7 +1029,7 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList { return ` ${__(indicator[0])} - `; + `; } return ""; } diff --git a/frappe/public/js/frappe/views/communication.js b/frappe/public/js/frappe/views/communication.js index c3e998788d..6baf4893e9 100755 --- a/frappe/public/js/frappe/views/communication.js +++ b/frappe/public/js/frappe/views/communication.js @@ -747,7 +747,10 @@ frappe.views.CommunicationComposer = class { this.content_set = true; } - message += await this.get_signature(sender_email || null); + const signature = await this.get_signature(sender_email || ""); + if (!this.content_set || !strip_html(message).includes(strip_html(signature))) { + message += signature; + } if (this.is_a_reply && !this.reply_set) { message += this.get_earlier_reply(); diff --git a/frappe/public/js/frappe/views/workspace/workspace.js b/frappe/public/js/frappe/views/workspace/workspace.js index 7bb53c65cd..b5fb0e2e54 100644 --- a/frappe/public/js/frappe/views/workspace/workspace.js +++ b/frappe/public/js/frappe/views/workspace/workspace.js @@ -353,7 +353,7 @@ frappe.views.Workspace = class Workspace { let current_page = pages.filter((p) => p.title == page.name)[0]; this.content = current_page && JSON.parse(current_page.content); - this.add_custom_cards_in_content(); + this.content && this.add_custom_cards_in_content(); $(".item-anchor").addClass("disable-click"); diff --git a/frappe/public/js/onboarding_tours/onboarding_tours.js b/frappe/public/js/onboarding_tours/onboarding_tours.js index df3e3c3894..29790ce5de 100644 --- a/frappe/public/js/onboarding_tours/onboarding_tours.js +++ b/frappe/public/js/onboarding_tours/onboarding_tours.js @@ -29,6 +29,13 @@ frappe.ui.OnboardingTour = class OnboardingTour { step.popover.node.offsetTop + step.options.step_info.offset_y }px`; } + if (step.popover.node.offsetLeft < 0) { + step.popover.node.style.minWidth = "200px"; + step.popover.node.style.maxWidth = `${ + 350 + step.popover.node.offsetLeft + }px`; + step.popover.node.style.left = "0px"; + } if (step.popover.closeBtnNode) { step.popover.closeBtnNode.onclick = () => { this.on_finish && this.on_finish(); diff --git a/frappe/public/js/telemetry/index.js b/frappe/public/js/telemetry/index.js index 48afaa5258..b9dee3be1c 100644 --- a/frappe/public/js/telemetry/index.js +++ b/frappe/public/js/telemetry/index.js @@ -32,9 +32,9 @@ class TelemetryManager { } } - capture(event, app) { + capture(event, app, props) { if (!this.enabled) return; - posthog.capture(`${app}_${event}`); + posthog.capture(`${app}_${event}`, props); } disable() { @@ -49,7 +49,7 @@ class TelemetryManager { if (!last || moment(now).diff(moment(last), "hours") > 12) { localStorage.setItem(KEY, now.toISOString()); - this.capture("heartbeat", "frappe"); + this.capture("heartbeat", "frappe", { frappe_version: frappe.boot?.versions?.frappe }); } } diff --git a/frappe/public/scss/desk/print_preview.scss b/frappe/public/scss/desk/print_preview.scss index 468b37fe5a..ed85f8b933 100644 --- a/frappe/public/scss/desk/print_preview.scss +++ b/frappe/public/scss/desk/print_preview.scss @@ -45,9 +45,6 @@ .layout-side-section.print-preview-sidebar { padding-right: var(--padding-md); - .clearfix { - display: none; - } .label-area { white-space: nowrap; diff --git a/frappe/tests/test_document.py b/frappe/tests/test_document.py index 474971c935..4e575528ab 100644 --- a/frappe/tests/test_document.py +++ b/frappe/tests/test_document.py @@ -169,6 +169,10 @@ class TestDocument(FrappeTestCase): with self.assertQueryCount(0): user.db_set("user_type", "Magical Wizard") + def test_new_doc_with_fields(self): + user = frappe.new_doc("User", first_name="wizard") + self.assertEqual(user.first_name, "wizard") + def test_update_after_submit(self): d = self.test_insert() d.starts_on = "2014-09-09" diff --git a/frappe/tests/test_query.py b/frappe/tests/test_query.py index dfebf5e890..9242630104 100644 --- a/frappe/tests/test_query.py +++ b/frappe/tests/test_query.py @@ -218,13 +218,6 @@ class TestQuery(FrappeTestCase): @run_only_if(db_type_is.MARIADB) def test_filters(self): - self.assertEqual( - frappe.qb.get_query( - "User", filters={"IfNull(name, " ")": ("<", Now())}, fields=["Max(name)"] - ).run(), - frappe.qb.from_("User").select(Max(Field("name"))).where(Ifnull("name", "") < Now()).run(), - ) - self.assertEqual( frappe.qb.get_query( "DocType", @@ -258,6 +251,17 @@ class TestQuery(FrappeTestCase): ), ) + self.assertRaisesRegex( + frappe.ValidationError, + "Invalid filter", + lambda: frappe.qb.get_query( + "DocType", + fields=["name"], + filters={"permissions.role": "System Manager"}, + validate_filters=True, + ), + ) + self.assertEqual( frappe.qb.get_query( "DocType", diff --git a/frappe/utils/goal.py b/frappe/utils/goal.py index 709fdc1644..01cd9d835e 100644 --- a/frappe/utils/goal.py +++ b/frappe/utils/goal.py @@ -31,6 +31,7 @@ def get_monthly_results( Function(aggregation, goal_field), ], filters=filters, + validate_filters=True, ) .groupby("month_year") .run() diff --git a/frappe/website/doctype/web_form/web_form.json b/frappe/website/doctype/web_form/web_form.json index 96749e460d..e0883ba439 100644 --- a/frappe/website/doctype/web_form/web_form.json +++ b/frappe/website/doctype/web_form/web_form.json @@ -31,6 +31,10 @@ "allow_incomplete", "section_break_2", "max_attachment_size", + "section_break_xzqr", + "condition", + "column_break_tjgl", + "condition_description", "section_break_3", "list_setting_message", "show_list", @@ -279,10 +283,6 @@ "fieldtype": "Tab Break", "label": "Form" }, - { - "fieldname": "column_break_1", - "fieldtype": "Column Break" - }, { "fieldname": "section_break_1", "fieldtype": "Section Break" @@ -297,7 +297,6 @@ "fieldtype": "Column Break" }, { - "collapsible": 1, "fieldname": "section_break_2", "fieldtype": "Section Break" }, @@ -374,13 +373,33 @@ "fieldname": "anonymous", "fieldtype": "Check", "label": "Anonymous" + }, + { + "fieldname": "condition", + "fieldtype": "Code", + "label": "Condition", + "max_height": "150px" + }, + { + "fieldname": "section_break_xzqr", + "fieldtype": "Section Break" + }, + { + "fieldname": "column_break_tjgl", + "fieldtype": "Column Break" + }, + { + "fieldname": "condition_description", + "fieldtype": "HTML", + "label": "Condition Description", + "options": "

Multiple webforms can be created for a single doctype. Write a condition specific to this webform to display correct record after submission.

For Example:

\n

If you create a separate webform every year to capture feedback from employees add a \n field named year in doctype and add a condition doc.year==\"2023\"

\n" } ], "has_web_view": 1, "icon": "icon-edit", "is_published_field": "published", "links": [], - "modified": "2023-04-20 17:24:42.657731", + "modified": "2023-06-03 19:18:56.760479", "modified_by": "Administrator", "module": "Website", "name": "Web Form", diff --git a/frappe/website/doctype/web_form/web_form.py b/frappe/website/doctype/web_form/web_form.py index 3e2705bdbe..81c6001558 100644 --- a/frappe/website/doctype/web_form/web_form.py +++ b/frappe/website/doctype/web_form/web_form.py @@ -153,10 +153,16 @@ def get_context(context): and not frappe.form_dict.name and not frappe.form_dict.is_list ): - name = frappe.db.get_value(self.doc_type, {"owner": frappe.session.user}, "name") - if name: - context.in_view_mode = True - frappe.redirect(f"/{self.route}/{name}") + names = frappe.db.get_values(self.doc_type, {"owner": frappe.session.user}, pluck="name") + for name in names: + if self.condition: + doc = frappe.get_doc(self.doc_type, name) + if frappe.safe_eval(self.condition, None, {"doc": doc.as_dict()}): + context.in_view_mode = True + frappe.redirect(f"/{self.route}/{name}") + else: + context.in_view_mode = True + frappe.redirect(f"/{self.route}/{name}") # Show new form when # - User is Guest diff --git a/frappe/website/doctype/web_page_view/web_page_view.py b/frappe/website/doctype/web_page_view/web_page_view.py index bbf2a394a6..b284dc095c 100644 --- a/frappe/website/doctype/web_page_view/web_page_view.py +++ b/frappe/website/doctype/web_page_view/web_page_view.py @@ -9,7 +9,13 @@ from frappe.model.document import Document class WebPageView(Document): - pass + @staticmethod + def clear_old_logs(days=180): + from frappe.query_builder import Interval + from frappe.query_builder.functions import Now + + table = frappe.qb.DocType("Web Page View") + frappe.db.delete(table, filters=(table.modified < (Now() - Interval(days=days)))) @frappe.whitelist(allow_guest=True)