diff --git a/.github/workflows/lock.yml b/.github/workflows/lock.yml index 72712c3d5f..e990228185 100644 --- a/.github/workflows/lock.yml +++ b/.github/workflows/lock.yml @@ -14,7 +14,7 @@ jobs: lock: runs-on: ubuntu-latest steps: - - uses: dessant/lock-threads@v4 + - uses: dessant/lock-threads@v5 with: github-token: ${{ github.token }} issue-inactive-days: 14 diff --git a/frappe/__init__.py b/frappe/__init__.py index eb9c502905..90f142c14c 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -429,7 +429,7 @@ def print_sql(enable: bool = True) -> None: def log(msg: str) -> None: - """Add to `debug_log`. + """Add to `debug_log` :param msg: Message.""" if not request: diff --git a/frappe/auth.py b/frappe/auth.py index f3fbe272c5..941edb9277 100644 --- a/frappe/auth.py +++ b/frappe/auth.py @@ -647,7 +647,7 @@ def validate_auth_via_api_keys(authorization_header): frappe.InvalidAuthorizationToken, ) except (AttributeError, TypeError, ValueError): - raise frappe.AuthenticationError + pass def validate_api_key_secret(api_key, api_secret, frappe_authorization_source=None): diff --git a/frappe/contacts/doctype/address/address.json b/frappe/contacts/doctype/address/address.json index 0ec67103a7..54d6b03739 100644 --- a/frappe/contacts/doctype/address/address.json +++ b/frappe/contacts/doctype/address/address.json @@ -51,12 +51,14 @@ "fieldname": "address_line1", "fieldtype": "Data", "label": "Address Line 1", + "length": 240, "reqd": 1 }, { "fieldname": "address_line2", "fieldtype": "Data", - "label": "Address Line 2" + "label": "Address Line 2", + "length": 240 }, { "fieldname": "city", @@ -148,7 +150,7 @@ "icon": "fa fa-map-marker", "idx": 5, "links": [], - "modified": "2023-10-30 05:50:23.912366", + "modified": "2023-11-20 17:28:41.698356", "modified_by": "Administrator", "module": "Contacts", "name": "Address", diff --git a/frappe/core/doctype/communication/communication.py b/frappe/core/doctype/communication/communication.py index 9000901be2..adf145a6c7 100644 --- a/frappe/core/doctype/communication/communication.py +++ b/frappe/core/doctype/communication/communication.py @@ -478,7 +478,7 @@ class Communication(Document, CommunicationEmailMixin): return self.timeline_links def remove_link(self, link_doctype, link_name, autosave=False, ignore_permissions=True): - for l in self.timeline_links: + for l in list(self.timeline_links): if l.link_doctype == link_doctype and l.link_name == link_name: self.timeline_links.remove(l) diff --git a/frappe/core/doctype/docfield/docfield.json b/frappe/core/doctype/docfield/docfield.json index f67969f43e..22b84d1cbc 100644 --- a/frappe/core/doctype/docfield/docfield.json +++ b/frappe/core/doctype/docfield/docfield.json @@ -19,6 +19,7 @@ "reqd", "is_virtual", "search_index", + "not_nullable", "column_break_18", "options", "sort_options", @@ -566,13 +567,20 @@ "fieldname": "link_filters", "fieldtype": "JSON", "label": "Link Filters" + }, + { + "default": "0", + "depends_on": "eval:!in_list([\"Check\", \"Currency\", \"Float\", \"Int\", \"Percent\", \"Rating\", \"Select\", \"Table\", \"Table MultiSelect\"], doc.fieldtype)", + "fieldname": "not_nullable", + "fieldtype": "Check", + "label": "Not Nullable" } ], "idx": 1, "index_web_pages_for_search": 1, "istable": 1, "links": [], - "modified": "2023-11-13 11:48:51.502812", + "modified": "2023-11-16 11:26:56.364594", "modified_by": "Administrator", "module": "Core", "name": "DocField", diff --git a/frappe/core/doctype/docfield/docfield.py b/frappe/core/doctype/docfield/docfield.py index 351dd0caae..dc26c1f96f 100644 --- a/frappe/core/doctype/docfield/docfield.py +++ b/frappe/core/doctype/docfield/docfield.py @@ -92,6 +92,7 @@ class DocField(Document): max_height: DF.Data | None no_copy: DF.Check non_negative: DF.Check + not_nullable: DF.Check oldfieldname: DF.Data | None oldfieldtype: DF.Data | None options: DF.SmallText | None diff --git a/frappe/core/doctype/doctype/doctype.py b/frappe/core/doctype/doctype/doctype.py index c3a1aa0e41..bbdcfd1817 100644 --- a/frappe/core/doctype/doctype/doctype.py +++ b/frappe/core/doctype/doctype/doctype.py @@ -365,16 +365,23 @@ class DocType(Document): SET `{fieldname}` = source.`{source_fieldname}` FROM `tab{link_doctype}` as source WHERE `{link_fieldname}` = source.name - AND ifnull(`{fieldname}`, '')='' """ + if df.not_nullable: + update_query += "AND `{fieldname}`=''" + else: + update_query += "AND ifnull(`{fieldname}`, '')=''" + else: update_query = """ UPDATE `tab{doctype}` as target INNER JOIN `tab{link_doctype}` as source ON `target`.`{link_fieldname}` = `source`.`name` SET `target`.`{fieldname}` = `source`.`{source_fieldname}` - WHERE ifnull(`target`.`{fieldname}`, '')="" """ + if df.not_nullable: + update_query += "WHERE `target`.`{fieldname}`=''" + else: + update_query += "WHERE ifnull(`target`.`{fieldname}`, '')=''" self.flags.update_fields_to_fetch_queries.append( update_query.format( diff --git a/frappe/core/doctype/scheduled_job_log/scheduled_job_log.json b/frappe/core/doctype/scheduled_job_log/scheduled_job_log.json index 451c4108a0..782c0749f8 100644 --- a/frappe/core/doctype/scheduled_job_log/scheduled_job_log.json +++ b/frappe/core/doctype/scheduled_job_log/scheduled_job_log.json @@ -7,7 +7,8 @@ "field_order": [ "status", "scheduled_job_type", - "details" + "details", + "debug_log" ], "fields": [ { @@ -35,10 +36,16 @@ "options": "Scheduled Job Type", "read_only": 1, "reqd": 1 + }, + { + "fieldname": "debug_log", + "fieldtype": "Code", + "label": "Debug Log", + "read_only": 1 } ], "links": [], - "modified": "2022-06-13 05:41:21.090972", + "modified": "2023-11-09 12:06:41.781270", "modified_by": "Administrator", "module": "Core", "name": "Scheduled Job Log", diff --git a/frappe/core/doctype/scheduled_job_log/scheduled_job_log.py b/frappe/core/doctype/scheduled_job_log/scheduled_job_log.py index 40f6823057..e4bfe21e2d 100644 --- a/frappe/core/doctype/scheduled_job_log/scheduled_job_log.py +++ b/frappe/core/doctype/scheduled_job_log/scheduled_job_log.py @@ -16,6 +16,7 @@ class ScheduledJobLog(Document): if TYPE_CHECKING: from frappe.types import DF + debug_log: DF.Code | None details: DF.Code | None scheduled_job_type: DF.Link status: DF.Literal["Scheduled", "Complete", "Failed"] diff --git a/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py b/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py index 6f56180c89..d6ec8bfef6 100644 --- a/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py +++ b/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py @@ -145,6 +145,8 @@ class ScheduledJobType(Document): dict(doctype="Scheduled Job Log", scheduled_job_type=self.name) ).insert(ignore_permissions=True) self.scheduler_log.db_set("status", status) + if frappe.debug_log: + self.scheduler_log.db_set("debug_log", "\n".join(frappe.debug_log)) if status == "Failed": self.scheduler_log.db_set("details", frappe.get_traceback()) if status == "Start": diff --git a/frappe/database/database.py b/frappe/database/database.py index 9a6cb7772a..d04135e827 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -1,7 +1,6 @@ # Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import datetime import itertools import json import random @@ -14,7 +13,6 @@ from time import time from typing import Any from pypika.dialects import MySQLQueryBuilder, PostgreSQLQueryBuilder -from pypika.terms import Criterion, NullValue import frappe import frappe.defaults @@ -163,6 +161,8 @@ class Database: :param auto_commit: Commit after executing the query. :param update: Update this dict to all rows (if returned `as_dict`). :param run: Returns query without executing it if False. + :param pluck: Get the plucked field only. + :param explain: Print `EXPLAIN` in error log. Examples: # return customer names as dicts @@ -369,7 +369,7 @@ class Database: self.commit() self.sql(query, debug=debug) - def check_transaction_status(self, query): + def check_transaction_status(self, query: str): """Raises exception if more than 200,000 `INSERT`, `UPDATE` queries are executed in one transaction. This is to ensure that writes are always flushed otherwise this could cause the system to hang.""" @@ -388,13 +388,13 @@ class Database: msg += _("The changes have been reverted.") + "
" raise frappe.TooManyWritesError(msg) - def check_implicit_commit(self, query): + def check_implicit_commit(self, query: str): if ( self.transaction_writes and query and is_query_type(query, ("start", "alter", "drop", "create", "begin", "truncate")) ): - raise ImplicitCommitError("This statement can cause implicit commit") + raise ImplicitCommitError("This statement can cause implicit commit", query) def fetch_as_dict(self) -> list[frappe._dict]: """Internal. Converts results to dict.""" diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index df64bdc86a..1f087a243a 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -310,7 +310,7 @@ class MariaDBDatabase(MariaDBConnectionUtil, MariaDBExceptionUtil, Database): ) @staticmethod - def get_on_duplicate_update(key=None): + def get_on_duplicate_update(): return "ON DUPLICATE key UPDATE " def get_table_columns_description(self, table_name): @@ -329,7 +329,8 @@ class MariaDBDatabase(MariaDBConnectionUtil, MariaDBExceptionUtil, Database): and Seq_in_index = 1 limit 1 ), 0) as 'index', - column_key = 'UNI' as 'unique' + column_key = 'UNI' as 'unique', + (is_nullable = 'NO') AS 'not_nullable' from information_schema.columns as columns where table_name = '{table_name}' """.format( table_name=table_name diff --git a/frappe/database/mariadb/schema.py b/frappe/database/mariadb/schema.py index 209535b9fd..d57335f589 100644 --- a/frappe/database/mariadb/schema.py +++ b/frappe/database/mariadb/schema.py @@ -3,6 +3,7 @@ from pymysql.constants.ER import DUP_ENTRY import frappe from frappe import _ from frappe.database.schema import DBTable +from frappe.utils.defaults import get_not_null_defaults class MariaDBTable(DBTable): @@ -69,7 +70,7 @@ class MariaDBTable(DBTable): add_column_query = [ f"ADD COLUMN `{col.fieldname}` {col.get_definition()}" for col in self.add_column ] - columns_to_modify = set(self.change_type + self.set_default) + columns_to_modify = set(self.change_type + self.set_default + self.change_nullability) modify_column_query = [ f"MODIFY `{col.fieldname}` {col.get_definition(for_modification=True)}" for col in columns_to_modify @@ -102,12 +103,23 @@ class MariaDBTable(DBTable): if index_record := frappe.db.get_column_index(self.table_name, col.fieldname, unique=False): drop_index_query.append(f"DROP INDEX `{index_record.Key_name}`") + for col in self.change_nullability: + if col.not_nullable: + try: + table = frappe.qb.DocType(self.doctype) + frappe.qb.update(table).set( + col.fieldname, col.default or get_not_null_defaults(col.fieldtype) + ).where(table[col.fieldname].isnull()).run() + except Exception: + print(f"Failed to update data in {self.table_name} for {col.fieldname}") + raise try: for query_parts in [add_column_query, modify_column_query, add_index_query, drop_index_query]: if query_parts: query_body = ", ".join(query_parts) query = f"ALTER TABLE `{self.table_name}` {query_body}" - frappe.db.sql(query) + # nosemgrep + frappe.db.sql_ddl(query) except Exception as e: if query := locals().get("query"): # this weirdness is to avoid potentially unbounded vars diff --git a/frappe/database/postgres/database.py b/frappe/database/postgres/database.py index f0a75e2e68..37fc9601f2 100644 --- a/frappe/database/postgres/database.py +++ b/frappe/database/postgres/database.py @@ -392,7 +392,8 @@ class PostgresDatabase(PostgresExceptionUtil, Database): END AS type, BOOL_OR(b.index) AS index, SPLIT_PART(COALESCE(a.column_default, NULL), '::', 1) AS default, - BOOL_OR(b.unique) AS unique + BOOL_OR(b.unique) AS unique, + COALESCE(a.is_nullable = 'NO', false) AS not_nullable FROM information_schema.columns a LEFT JOIN (SELECT indexdef, tablename, @@ -402,7 +403,7 @@ class PostgresDatabase(PostgresExceptionUtil, Database): WHERE tablename='{table_name}') b ON SUBSTRING(b.indexdef, '(.*)') LIKE CONCAT('%', a.column_name, '%') WHERE a.table_name = '{table_name}' - GROUP BY a.column_name, a.data_type, a.column_default, a.character_maximum_length; + GROUP BY a.column_name, a.data_type, a.column_default, a.character_maximum_length, a.is_nullable; """.format( table_name=table_name ), diff --git a/frappe/database/postgres/schema.py b/frappe/database/postgres/schema.py index 073fafd8c7..946747aa47 100644 --- a/frappe/database/postgres/schema.py +++ b/frappe/database/postgres/schema.py @@ -2,6 +2,7 @@ import frappe from frappe import _ from frappe.database.schema import DBTable, get_definition from frappe.utils import cint, flt +from frappe.utils.defaults import get_not_null_defaults class PostgresTable(DBTable): @@ -45,7 +46,7 @@ class PostgresTable(DBTable): docstatus smallint not null default '0', idx bigint not null default '0', {additional_definitions} - )""" + )""", ) self.create_indexes() @@ -139,11 +140,34 @@ class PostgresTable(DBTable): if col.fieldname != "name": # if index key exists drop_contraint_query += f'DROP INDEX IF EXISTS "unique_{col.fieldname}" ;' + + change_nullability = [] + for col in self.change_nullability: + default = col.default or get_not_null_defaults(col.fieldtype) + if isinstance(default, str): + default = frappe.db.escape(default) + change_nullability.append( + f"ALTER COLUMN \"{col.fieldname}\" {'SET' if col.not_nullable else 'DROP'} NOT NULL" + ) + change_nullability.append(f'ALTER COLUMN "{col.fieldname}" SET DEFAULT {default}') + + if col.not_nullable: + try: + table = frappe.qb.DocType(self.doctype) + frappe.qb.update(table).set( + col.fieldname, col.default or get_not_null_defaults(col.fieldtype) + ).where(table[col.fieldname].isnull()).run() + except Exception: + print(f"Failed to update data in {self.table_name} for {col.fieldname}") + raise try: if query: final_alter_query = "ALTER TABLE `{}` {}".format(self.table_name, ", ".join(query)) # nosemgrep frappe.db.sql(final_alter_query) + if change_nullability: + # nosemgrep + frappe.db.sql(f"ALTER TABLE `{self.table_name}` {','.join(change_nullability)}") if create_contraint_query: # nosemgrep frappe.db.sql(create_contraint_query) diff --git a/frappe/database/schema.py b/frappe/database/schema.py index 24eea24fa9..90c3055452 100644 --- a/frappe/database/schema.py +++ b/frappe/database/schema.py @@ -3,6 +3,7 @@ import re import frappe from frappe import _ from frappe.utils import cint, cstr, flt +from frappe.utils.defaults import get_not_null_defaults SPECIAL_CHAR_PATTERN = re.compile(r"[\W]", flags=re.UNICODE) VARCHAR_CAST_PATTERN = re.compile(r"varchar\(([\d]+)\)") @@ -24,6 +25,7 @@ class DBTable: self.add_column: list[DbColumn] = [] self.change_type: list[DbColumn] = [] self.change_name: list[DbColumn] = [] + self.change_nullability: list[DbColumn] = [] self.add_unique: list[DbColumn] = [] self.add_index: list[DbColumn] = [] self.drop_unique: list[DbColumn] = [] @@ -89,15 +91,16 @@ class DBTable: continue self.columns[field.get("fieldname")] = DbColumn( - self, - field.get("fieldname"), - field.get("fieldtype"), - field.get("length"), - field.get("default"), - field.get("search_index"), - field.get("options"), - field.get("unique"), - field.get("precision"), + table=self, + fieldname=field.get("fieldname"), + fieldtype=field.get("fieldtype"), + length=field.get("length"), + default=field.get("default"), + set_index=field.get("search_index"), + options=field.get("options"), + unique=field.get("unique"), + precision=field.get("precision"), + not_nullable=field.get("not_nullable"), ) def validate(self): @@ -175,7 +178,18 @@ class DBTable: class DbColumn: def __init__( - self, table, fieldname, fieldtype, length, default, set_index, options, unique, precision + self, + *, + table, + fieldname, + fieldtype, + length, + default, + set_index, + options, + unique, + precision, + not_nullable, ): self.table = table self.fieldname = fieldname @@ -186,6 +200,7 @@ class DbColumn: self.options = options self.unique = unique self.precision = precision + self.not_nullable = not_nullable def get_definition(self, for_modification=False): column_def = get_definition(self.fieldtype, precision=self.precision, length=self.length) @@ -193,24 +208,43 @@ class DbColumn: if not column_def: return column_def + null = True + default = None + unique = False + if self.fieldtype in ("Check", "Int"): - default_value = cint(self.default) or 0 - column_def += f" not null default {default_value}" + default = cint(self.default) or 0 + null = False elif self.fieldtype in ("Currency", "Float", "Percent"): - default_value = flt(self.default) or 0 - column_def += f" not null default {default_value}" + default = flt(self.default) or 0 + null = False elif ( self.default and (self.default not in frappe.db.DEFAULT_SHORTCUTS) and not cstr(self.default).startswith(":") ): - column_def += f" default {frappe.db.escape(self.default)}" + default = frappe.db.escape(self.default) + + if self.not_nullable and null: + if default is None: + default = get_not_null_defaults(self.fieldtype) + if isinstance(default, str): + default = frappe.db.escape(default) + null = False if self.unique and not for_modification and (column_def not in ("text", "longtext")): - column_def += " unique" + unique = True + if not null: + column_def += " NOT NULL" + + if default is not None: + column_def += f" DEFAULT {default}" + + if unique: + column_def += " UNIQUE" return column_def def build_for_alter_table(self, current_def): @@ -250,6 +284,10 @@ class DbColumn: ): self.table.set_default.append(self) + # nullability + if self.not_nullable is not None and (self.not_nullable != current_def["not_nullable"]): + self.table.change_nullability.append(self) + # index should be applied or dropped irrespective of type change if (current_def["index"] and not self.set_index) and column_type not in ("text", "longtext"): self.table.drop_index.append(self) diff --git a/frappe/database/utils.py b/frappe/database/utils.py index 7cdab76dda..5d1de5792f 100644 --- a/frappe/database/utils.py +++ b/frappe/database/utils.py @@ -1,7 +1,6 @@ # Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import typing from functools import cached_property from types import NoneType @@ -9,9 +8,6 @@ import frappe from frappe.query_builder.builder import MariaDB, Postgres from frappe.query_builder.functions import Function -if typing.TYPE_CHECKING: - from frappe.query_builder import DocType - Query = str | MariaDB | Postgres QueryValues = tuple | list | dict | NoneType @@ -27,7 +23,7 @@ NestedSetHierarchy = ( ) -def is_query_type(query: str, query_type: str | tuple[str]) -> bool: +def is_query_type(query: str, query_type: str | tuple[str, ...]) -> bool: return query.lstrip().split(maxsplit=1)[0].lower().startswith(query_type) diff --git a/frappe/desk/doctype/event/event.py b/frappe/desk/doctype/event/event.py index 5463df6413..bf56498780 100644 --- a/frappe/desk/doctype/event/event.py +++ b/frappe/desk/doctype/event/event.py @@ -121,9 +121,7 @@ class Event(Document): ["Communication Link", "link_doctype", "=", participant.reference_doctype], ["Communication Link", "link_name", "=", participant.reference_docname], ] - comms = frappe.get_all("Communication", filters=filters, fields=["name"]) - - if comms: + if comms := frappe.get_all("Communication", filters=filters, fields=["name"], distinct=True): for comm in comms: communication = frappe.get_doc("Communication", comm.name) self.update_communication(participant, communication) diff --git a/frappe/desk/doctype/notification_log/notification_log.js b/frappe/desk/doctype/notification_log/notification_log.js index ea5fdc6400..63776fbf95 100644 --- a/frappe/desk/doctype/notification_log/notification_log.js +++ b/frappe/desk/doctype/notification_log/notification_log.js @@ -11,6 +11,10 @@ frappe.ui.form.on("Notification Log", { }, open_reference_document: function (frm) { + if (frm.doc?.link) { + frappe.set_route(frm.doc.link); + return; + } const dt = frm.doc.document_type; const dn = frm.doc.document_name; frappe.set_route("Form", dt, dn); diff --git a/frappe/desk/doctype/notification_log/notification_log.json b/frappe/desk/doctype/notification_log/notification_log.json index bafe28faf8..9fbe7324d3 100644 --- a/frappe/desk/doctype/notification_log/notification_log.json +++ b/frappe/desk/doctype/notification_log/notification_log.json @@ -15,7 +15,8 @@ "attached_file", "attachment_link", "open_reference_document", - "from_user" + "from_user", + "link" ], "fields": [ { @@ -91,12 +92,18 @@ "fieldname": "attachment_link", "fieldtype": "HTML", "label": "Attachment Link" + }, + { + "fieldname": "link", + "fieldtype": "Data", + "hidden": 1, + "label": "Link" } ], "hide_toolbar": 1, "in_create": 1, "links": [], - "modified": "2023-06-14 21:20:51.197943", + "modified": "2023-11-18 22:40:12.145940", "modified_by": "Administrator", "module": "Desk", "name": "Notification Log", diff --git a/frappe/desk/doctype/notification_log/notification_log.py b/frappe/desk/doctype/notification_log/notification_log.py index e459a63ef8..e6e13dafe7 100644 --- a/frappe/desk/doctype/notification_log/notification_log.py +++ b/frappe/desk/doctype/notification_log/notification_log.py @@ -25,6 +25,7 @@ class NotificationLog(Document): email_content: DF.TextEditor | None for_user: DF.Link | None from_user: DF.Link | None + link: DF.Data | None read: DF.Check subject: DF.Text | None type: DF.Literal["Mention", "Energy Point", "Assignment", "Share", "Alert"] @@ -125,21 +126,24 @@ def send_notification_email(doc): if not email: return - doc_link = get_url_to_form(doc.document_type, doc.document_name) header = get_email_header(doc) email_subject = strip_html(doc.subject) + args = { + "body_content": doc.subject, + "description": doc.email_content, + } + if doc.link: + args["doc_link"] = doc.link + else: + args["document_type"] = doc.document_type + args["document_name"] = doc.document_name + args["doc_link"] = get_url_to_form(doc.document_type, doc.document_name) frappe.sendmail( recipients=email, subject=email_subject, template="new_notification", - args={ - "body_content": doc.subject, - "description": doc.email_content, - "document_type": doc.document_type, - "document_name": doc.document_name, - "doc_link": doc_link, - }, + args=args, header=[header, "orange"], now=frappe.flags.in_test, ) diff --git a/frappe/desk/doctype/system_console/system_console.json b/frappe/desk/doctype/system_console/system_console.json index a851831909..9573e23b52 100644 --- a/frappe/desk/doctype/system_console/system_console.json +++ b/frappe/desk/doctype/system_console/system_console.json @@ -29,7 +29,7 @@ ], "fields": [ { - "description": "To print output use log(text)", + "description": "To print output use print(text)", "fieldname": "console", "fieldtype": "Code", "label": "Console", @@ -86,7 +86,7 @@ "index_web_pages_for_search": 1, "issingle": 1, "links": [], - "modified": "2022-04-15 14:15:58.398590", + "modified": "2023-11-03 13:02:00.706806", "modified_by": "Administrator", "module": "Desk", "name": "System Console", diff --git a/frappe/desk/doctype/workspace_shortcut/workspace_shortcut.json b/frappe/desk/doctype/workspace_shortcut/workspace_shortcut.json index e494aad152..854305ad80 100644 --- a/frappe/desk/doctype/workspace_shortcut/workspace_shortcut.json +++ b/frappe/desk/doctype/workspace_shortcut/workspace_shortcut.json @@ -44,7 +44,7 @@ "fieldtype": "Select", "in_list_view": 1, "label": "DocType View", - "options": "\nList\nReport Builder\nDashboard\nTree\nNew\nCalendar\nKanban" + "options": "\nList\nReport Builder\nDashboard\nTree\nNew\nCalendar\nKanban\nImage" }, { "fieldname": "column_break_4", diff --git a/frappe/handler.py b/frappe/handler.py index 6db6a7600f..d889c67b23 100644 --- a/frappe/handler.py +++ b/frappe/handler.py @@ -287,7 +287,6 @@ def get_attr(cmd): f"Calling shorthand for {cmd} is deprecated, please specify full path in RPC call." ) method = globals()[cmd] - frappe.log("method:" + cmd) return method diff --git a/frappe/model/base_document.py b/frappe/model/base_document.py index 78c20a2eae..bdf354ce99 100644 --- a/frappe/model/base_document.py +++ b/frappe/model/base_document.py @@ -31,6 +31,7 @@ from frappe.utils import ( sanitize_html, strip_html, ) +from frappe.utils.defaults import get_not_null_defaults from frappe.utils.html_utils import unescape_html if TYPE_CHECKING: @@ -384,6 +385,13 @@ class BaseDocument: if ignore_nulls and not is_virtual_field and value is None: continue + # If the docfield is not nullable - set a default non-null value + if value is None and getattr(df, "not_nullable", False): + if df.default: + value = df.default + else: + value = get_not_null_defaults(df.fieldtype) + d[fieldname] = value return d diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py index 62c0538298..76a03f5a76 100644 --- a/frappe/model/db_query.py +++ b/frappe/model/db_query.py @@ -109,7 +109,6 @@ class DatabaseQuery: save_user_settings=False, save_user_settings_fields=False, update=None, - add_total_row=None, user_settings=None, reference_doctype=None, run=True, @@ -734,12 +733,15 @@ class DatabaseQuery: f.update(get_additional_filter_field(additional_filters_config, f, f.value)) meta = frappe.get_meta(f.doctype) + df = meta.get("fields", {"fieldname": f.fieldname}) + df = df[0] if df else None + can_be_null = True + value = None + # prepare in condition if f.operator.lower() in NestedSetHierarchy: - values = f.value or "" - # TODO: handle list and tuple # if not isinstance(values, (list, tuple)): # values = values.split(",") @@ -785,30 +787,33 @@ class DatabaseQuery: "not in" if f.operator.lower() in ("not ancestors of", "not descendants of") else "in" ) - elif f.operator.lower() in ("in", "not in"): + if f.operator.lower() in ("in", "not in"): # if values contain '' or falsy values then only coalesce column # for `in` query this is only required if values contain '' or values are empty. # for `not in` queries we can't be sure as column values might contain null. + can_be_null = not getattr(df, "not_nullable", False) if f.operator.lower() == "in": - can_be_null = not f.value or any(v is None or v == "" for v in f.value) + can_be_null &= not f.value or any(v is None or v == "" for v in f.value) - values = f.value or "" - if isinstance(values, str): - values = values.split(",") + if value is None: + values = f.value or "" + if isinstance(values, str): + values = values.split(",") - fallback = "''" - value = [frappe.db.escape((cstr(v) or "").strip(), percent=False) for v in values] - if len(value): - value = f"({', '.join(value)})" - else: - value = "('')" + fallback = "''" + value = [frappe.db.escape((cstr(v) or "").strip(), percent=False) for v in values] + if len(value): + value = f"({', '.join(value)})" + else: + value = "('')" else: escape = True - df = meta.get("fields", {"fieldname": f.fieldname}) - df = df[0] if df else None - if df and df.fieldtype in ("Check", "Float", "Int", "Currency", "Percent"): + if df and ( + df.fieldtype in ("Check", "Float", "Int", "Currency", "Percent") + or getattr(df, "not_nullable", False) + ): can_be_null = False if f.operator.lower() in ("previous", "next", "timespan"): @@ -841,7 +846,7 @@ class DatabaseQuery: elif f.value == "not set": f.operator = "=" fallback = "''" - can_be_null = True + can_be_null = not getattr(df, "not_nullable", False) value = "" @@ -981,7 +986,6 @@ class DatabaseQuery: ) def add_user_permissions(self, user_permissions): - doctype_link_fields = [] doctype_link_fields = self.doctype_meta.get_link_fields() # append current doctype with fieldname as 'name' as first link field diff --git a/frappe/model/document.py b/frappe/model/document.py index 4d5eec64fc..9ed4e2a3f2 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -343,7 +343,7 @@ class Document(BaseDocument): :param ignore_permissions: Do not check permissions if True. :param ignore_version: Do not save version if True.""" if self.flags.in_print: - return + return self self.flags.notifications_executed = [] diff --git a/frappe/model/rename_doc.py b/frappe/model/rename_doc.py index e6cc83874d..9bad8f1028 100644 --- a/frappe/model/rename_doc.py +++ b/frappe/model/rename_doc.py @@ -464,11 +464,12 @@ def get_link_fields(doctype: str) -> list[dict]: cf = frappe.qb.DocType("Custom Field") ps = frappe.qb.DocType("Property Setter") - st_issingle = frappe.qb.from_(dt).select(dt.issingle).where(dt.name == df.parent).as_("issingle") standard_fields = ( frappe.qb.from_(df) - .select(df.parent, df.fieldname, st_issingle) - .where((df.options == doctype) & (df.fieldtype == "Link")) + .inner_join(dt) + .on(df.parent == dt.name) + .select(df.parent, df.fieldname, dt.issingle.as_("issingle")) + .where((df.options == doctype) & (df.fieldtype == "Link") & (dt.is_virtual == 0)) .run(as_dict=True) ) diff --git a/frappe/public/css/fonts/inter/inter.css b/frappe/public/css/fonts/inter/inter.css index 3be525aba0..3200c42576 100644 --- a/frappe/public/css/fonts/inter/inter.css +++ b/frappe/public/css/fonts/inter/inter.css @@ -8,6 +8,7 @@ src: url('/assets/frappe/css/fonts/inter/Inter.var.woff2?v=3.19') format('woff2-variations'), url('/assets/frappe/css/fonts/inter/Inter.var.woff2?v=3.19') format('woff2'); src: url('/assets/frappe/css/fonts/inter/Inter.var.woff2?v=3.19') format('woff2') tech('variations'); + unicode-range: U+0460-052F,U+1C80-1C88,U+20B4,U+2DE0-2DFF,U+A640-A69F,U+FE2E-FE2F; } @font-face { font-family: 'Inter V'; @@ -17,6 +18,7 @@ src: url('/assets/frappe/css/fonts/inter/Inter-Italic.var.woff2?v=3.19') format('woff2-variations'), url('/assets/frappe/css/fonts/inter/Inter-Italic.var.woff2?v=3.19') format('woff2'); src: url('/assets/frappe/css/fonts/inter/Inter-Italic.var.woff2?v=3.19') format('woff2') tech('variations'); + unicode-range: U+0460-052F,U+1C80-1C88,U+20B4,U+2DE0-2DFF,U+A640-A69F,U+FE2E-FE2F; } @font-face { font-family: 'Inter'; diff --git a/frappe/public/js/frappe/model/indicator.js b/frappe/public/js/frappe/model/indicator.js index d5c42c3799..6ae40c1f05 100644 --- a/frappe/public/js/frappe/model/indicator.js +++ b/frappe/public/js/frappe/model/indicator.js @@ -36,11 +36,18 @@ frappe.get_indicator = function (doc, doctype, show_workflow_state) { var settings = frappe.listview_settings[doctype] || {}; - var is_submittable = frappe.model.is_submittable(doctype), - workflow_fieldname = frappe.workflow.get_state_fieldname(doctype); + var is_submittable = frappe.model.is_submittable(doctype); + let workflow_fieldname = frappe.workflow.get_state_fieldname(doctype); + let avoid_status_override = (frappe.workflow.avoid_status_override[doctype] || []).includes( + doc[workflow_fieldname] + ); // workflow - if (workflow_fieldname && (!without_workflow || show_workflow_state)) { + if ( + workflow_fieldname && + (!without_workflow || show_workflow_state) && + !avoid_status_override + ) { var value = doc[workflow_fieldname]; if (value) { let colour = ""; diff --git a/frappe/public/js/frappe/model/workflow.js b/frappe/public/js/frappe/model/workflow.js index db4c2b25ae..359b9ba310 100644 --- a/frappe/public/js/frappe/model/workflow.js +++ b/frappe/public/js/frappe/model/workflow.js @@ -6,11 +6,15 @@ frappe.provide("frappe.workflow"); frappe.workflow = { state_fields: {}, workflows: {}, + avoid_status_override: {}, setup: function (doctype) { var wf = frappe.get_list("Workflow", { document_type: doctype }); if (wf.length) { frappe.workflow.workflows[doctype] = wf[0]; frappe.workflow.state_fields[doctype] = wf[0].workflow_state_field; + frappe.workflow.avoid_status_override[doctype] = wf[0].states + .filter((row) => row.avoid_status_override) + .map((d) => d.state); } else { frappe.workflow.state_fields[doctype] = null; } diff --git a/frappe/public/js/frappe/request.js b/frappe/public/js/frappe/request.js index 10a7158e4a..f0195089f8 100644 --- a/frappe/public/js/frappe/request.js +++ b/frappe/public/js/frappe/request.js @@ -483,8 +483,8 @@ frappe.request.cleanup = function (opts, r) { if (opts.args) { console.log("======== arguments ========"); console.log(opts.args); - console.log("========"); } + console.log("======== debug messages ========"); $.each(JSON.parse(r._debug_messages), function (i, v) { console.log(v); }); diff --git a/frappe/public/js/frappe/ui/notifications/notifications.js b/frappe/public/js/frappe/ui/notifications/notifications.js index 14ff7fcbfb..15f7de8d9f 100644 --- a/frappe/public/js/frappe/ui/notifications/notifications.js +++ b/frappe/public/js/frappe/ui/notifications/notifications.js @@ -325,6 +325,9 @@ class NotificationsView extends BaseNotificationsView { } get_item_link(notification_doc) { + if (notification_doc.link) { + return notification_doc.link; + } const link_doctype = notification_doc.document_type ? notification_doc.document_type : "Notification Log"; diff --git a/frappe/public/js/frappe/utils/utils.js b/frappe/public/js/frappe/utils/utils.js index 0a18ba1edc..b8458fb3c1 100644 --- a/frappe/public/js/frappe/utils/utils.js +++ b/frappe/public/js/frappe/utils/utils.js @@ -1300,6 +1300,9 @@ Object.assign(frappe.utils, { route += `/${item.kanban_board}`; } break; + case "Image": + route = `${doctype_slug}/view/image`; + break; default: route = doctype_slug; } diff --git a/frappe/public/js/frappe/web_form/web_form_list.js b/frappe/public/js/frappe/web_form/web_form_list.js index 67a29319ca..388e81677f 100644 --- a/frappe/public/js/frappe/web_form/web_form_list.js +++ b/frappe/public/js/frappe/web_form/web_form_list.js @@ -110,7 +110,7 @@ export default class WebFormList { fetch_data() { if (this.condition_json && JSON.parse(this.condition_json)) { - let filter = frappe.utils.get_filter_from_json(this.condition_json,this.doctype); + let filter = frappe.utils.get_filter_from_json(this.condition_json, this.doctype); filter = frappe.utils.get_filter_as_json(filter); this.filters = Object.assign(this.filters, JSON.parse(filter)); } diff --git a/frappe/public/js/frappe/widgets/widget_dialog.js b/frappe/public/js/frappe/widgets/widget_dialog.js index e22e56c47f..abe7352d9d 100644 --- a/frappe/public/js/frappe/widgets/widget_dialog.js +++ b/frappe/public/js/frappe/widgets/widget_dialog.js @@ -396,6 +396,7 @@ class ShortcutDialog extends WidgetDialog { const views = ["List", "Report Builder", "Dashboard", "New"]; if (meta.is_tree === 1) views.push("Tree"); + if (meta.image_field) views.push("Image"); if (frappe.boot.calendars.includes(doctype)) views.push("Calendar"); const response = await frappe.db.get_value( @@ -427,7 +428,7 @@ class ShortcutDialog extends WidgetDialog { fieldtype: "Select", fieldname: "doc_view", label: "DocType View", - options: "List\nReport Builder\nDashboard\nTree\nNew\nCalendar\nKanban", + options: "List\nReport Builder\nDashboard\nTree\nNew\nCalendar\nKanban\nImage", description: __( "Which view of the associated DocType should this shortcut take you to?" ), diff --git a/frappe/tests/test_api.py b/frappe/tests/test_api.py index b415a57f77..a5c76e2698 100644 --- a/frappe/tests/test_api.py +++ b/frappe/tests/test_api.py @@ -282,7 +282,7 @@ class TestMethodAPI(FrappeAPITestCase): response = self.get(self.method_path("frappe.auth.get_logged_user")) self.assertEqual(response.status_code, 401) - authorization_token = f"NonExistentKey:INCORRECT" + authorization_token = "NonExistentKey:INCORRECT" response = self.get(self.method_path("frappe.auth.get_logged_user")) self.assertEqual(response.status_code, 401) @@ -382,7 +382,7 @@ def after_request(*args, **kwargs): class TestResponse(FrappeAPITestCase): def test_generate_pdf(self): response = self.get( - f"/api/method/frappe.utils.print_format.download_pdf", + "/api/method/frappe.utils.print_format.download_pdf", {"sid": self.sid, "doctype": "User", "name": "Guest"}, ) self.assertEqual(response.status_code, 200) diff --git a/frappe/tests/test_non_nullable_docfield.py b/frappe/tests/test_non_nullable_docfield.py new file mode 100644 index 0000000000..d79e6cde64 --- /dev/null +++ b/frappe/tests/test_non_nullable_docfield.py @@ -0,0 +1,62 @@ +import frappe +from frappe.core.doctype.doctype.test_doctype import new_doctype +from frappe.database.schema import DBTable +from frappe.tests.utils import FrappeTestCase + + +class TestNonNullableDocfield(FrappeTestCase): + def setUp(self): + doc = new_doctype( + fields=[ + { + "fieldname": "test_field", + "fieldtype": "Data", + "label": "test_field", + "not_nullable": 1, + }, + ], + ) + doc.insert() + self.doctype_name = doc.name + + nullable_doc = new_doctype( + fields=[ + { + "fieldname": "test_field", + "fieldtype": "Data", + "label": "test_field", + } + ] + ) + nullable_doc.insert() + self.nullable_doctype_name = nullable_doc.name + + def test_non_nullable_field(self): + doc = frappe.new_doc(doctype=self.doctype_name) + doc.insert() + inserted_doc = frappe.db.get(self.doctype_name, {"name": doc.name}) + self.assertEqual(inserted_doc.test_field, "") + + def test_edit_field_nullable_status(self): + doc = frappe.new_doc(doctype=self.nullable_doctype_name) + doc.insert() + inserted_doc = frappe.db.get(self.nullable_doctype_name, {"name": doc.name}) + self.assertEqual(inserted_doc.test_field, None) + table = DBTable(self.nullable_doctype_name) + query = "SELECT column_name AS name, column_default is NULL AS default_null,is_nullable = 'NO' AS not_nullable FROM information_schema.columns WHERE table_name=%s" + for column in frappe.db.sql(query, table.table_name, as_dict=True): + if column.name == "test_field": + self.assertFalse(column.not_nullable) + + doctype_doc = frappe.get_doc("DocType", self.nullable_doctype_name) + for field in doctype_doc.fields: + if field.fieldname == "test_field": + field.not_nullable = 1 + break + doctype_doc.save() + for column in frappe.db.sql(query, table.table_name, as_dict=True): + if column.name == "test_field": + self.assertFalse(column.default_null) + self.assertTrue(column.not_nullable) + inserted_doc = frappe.db.get(self.nullable_doctype_name, {"name": doc.name}) + self.assertEqual(inserted_doc.test_field, "") diff --git a/frappe/tests/test_safe_exec.py b/frappe/tests/test_safe_exec.py index ba38cc93e9..3542fdfce1 100644 --- a/frappe/tests/test_safe_exec.py +++ b/frappe/tests/test_safe_exec.py @@ -121,6 +121,11 @@ class TestSafeExec(FrappeTestCase): # dont Allow modifying _dict class self.assertRaises(Exception, safe_exec, "_dict.x = 1") + def test_print(self): + test_str = frappe.generate_hash() + safe_exec(f"print('{test_str}')") + self.assertEqual(frappe.local.debug_log[-1], test_str) + class TestNoSafeExec(FrappeTestCase): def test_safe_exec_disabled_by_default(self): diff --git a/frappe/types/exporter.py b/frappe/types/exporter.py index 97551e0c01..cbdce7c6e0 100644 --- a/frappe/types/exporter.py +++ b/frappe/types/exporter.py @@ -162,6 +162,9 @@ class TypeExporter: if field.fieldtype in non_nullable_types: return False + if field.not_nullable: + return False + return not bool(field.reqd) def _generic_parameters(self, field) -> str | None: diff --git a/frappe/utils/defaults.py b/frappe/utils/defaults.py new file mode 100644 index 0000000000..152efef8e3 --- /dev/null +++ b/frappe/utils/defaults.py @@ -0,0 +1,50 @@ +from typing import Literal + + +def get_not_null_defaults(column_type: str) -> Literal["", 0] | None: + """ + Method to return a default value for a column type that is not NoneType + :param column_type: The type of column + :return: The value to be set + """ + column_type_map = { + "Data": str, + "Text": str, + "Autocomplete": str, + "Attach": str, + "AttachImage": str, + "Barcode": str, + "Check": int, + "Code": str, + "Color": str, + "Currency": float, + "Date": str, + "Datetime": str, + "Duration": int, + "DynamicLink": str, + "Float": float, + "HTMLEditor": str, + "Int": int, + "JSON": str, + "Link": str, + "LongText": str, + "MarkdownEditor": str, + "Password": str, + "Percent": float, + "Phone": str, + "ReadOnly": str, + "Rating": float, + "Select": str, + "SmallText": str, + "TextEditor": str, + "Time": str, + "Table": list, + "Table MultiSelect": list, + } + data_type = column_type_map.get(column_type.replace(" ", ""), str) + # data_type = eval(f"frappe.types.DF.{column_type.replace(' ', '')}") + if data_type == str: + return "" + if data_type in (int, float): + return 0 + return None diff --git a/frappe/utils/response.py b/frappe/utils/response.py index 3fcf1a365e..61372eee23 100644 --- a/frappe/utils/response.py +++ b/frappe/utils/response.py @@ -175,7 +175,7 @@ def _make_logs_v1(): if frappe.local.message_log: response["_server_messages"] = json.dumps([json.dumps(d) for d in frappe.local.message_log]) - if frappe.debug_log and frappe.conf.get("logging"): + if frappe.debug_log: response["_debug_messages"] = json.dumps(frappe.local.debug_log) if frappe.flags.error_message: @@ -188,7 +188,7 @@ def _make_logs_v2(): if frappe.local.message_log: response["messages"] = frappe.local.message_log - if frappe.debug_log and frappe.conf.get("logging"): + if frappe.debug_log: response["debug"] = [{"message": m} for m in frappe.local.debug_log] diff --git a/frappe/utils/safe_exec.py b/frappe/utils/safe_exec.py index 0b8435b100..f8ea926365 100644 --- a/frappe/utils/safe_exec.py +++ b/frappe/utils/safe_exec.py @@ -1,6 +1,7 @@ import ast import copy import inspect +import io import json import mimetypes import types @@ -8,7 +9,7 @@ from contextlib import contextmanager from functools import lru_cache import RestrictedPython.Guards -from RestrictedPython import compile_restricted, safe_globals +from RestrictedPython import PrintCollector, compile_restricted, safe_globals from RestrictedPython.transformer import RestrictingNodeTransformer import frappe @@ -60,6 +61,16 @@ class FrappeTransformer(RestrictingNodeTransformer): return super().check_name(node, name, *args, **kwargs) +class FrappePrintCollector(PrintCollector): + """Collect written text, and return it when called.""" + + def _call_print(self, *objects, **kwargs): + output = io.StringIO() + print(*objects, file=output, **kwargs) + frappe.log(output.getvalue().strip()) + output.close() + + def is_safe_exec_enabled() -> bool: # server scripts can only be enabled via common_site_config.json return bool(frappe.get_common_site_config().get(SAFE_EXEC_CONFIG_KEY)) @@ -261,6 +272,9 @@ def get_safe_globals(): out._getitem_ = _getitem out._getattr_ = _getattr_for_safe_exec + # Allow using `print()` calls with `safe_exec()` + out._print_ = FrappePrintCollector + # allow iterators and list comprehension out._getiter_ = iter out._iter_unpack_sequence_ = RestrictedPython.Guards.guarded_iter_unpack_sequence diff --git a/frappe/website/doctype/web_form/web_form.py b/frappe/website/doctype/web_form/web_form.py index 8e6156dfbb..95684dee99 100644 --- a/frappe/website/doctype/web_form/web_form.py +++ b/frappe/website/doctype/web_form/web_form.py @@ -684,7 +684,7 @@ def get_link_options(web_form_name, doctype, allow_read_on_all_link_options=Fals frappe.get_cached_value("DocType", doctype, "show_title_field_in_link") == 1 ) if not show_title_field_in_link: - value = frappe.get_cached_value( + value = frappe.db.get_value( "Property Setter", fieldname="value", filters={"property": "show_title_field_in_link", "doc_type": doctype}, diff --git a/frappe/workflow/doctype/workflow_document_state/workflow_document_state.json b/frappe/workflow/doctype/workflow_document_state/workflow_document_state.json index 9318cec308..4d19080497 100644 --- a/frappe/workflow/doctype/workflow_document_state/workflow_document_state.json +++ b/frappe/workflow/doctype/workflow_document_state/workflow_document_state.json @@ -13,6 +13,7 @@ "update_value", "column_break_4", "is_optional_state", + "avoid_status_override", "next_action_email_template", "allow_edit", "section_break_9", @@ -97,12 +98,19 @@ "fieldtype": "Data", "hidden": 1, "label": "Workflow Builder ID" + }, + { + "default": "0", + "description": "If Checked workflow status will not override status in list view", + "fieldname": "avoid_status_override", + "fieldtype": "Check", + "label": "Don't Override Status" } ], "idx": 1, "istable": 1, "links": [], - "modified": "2023-11-16 12:10:15.472947", + "modified": "2023-11-13 18:27:08.633239", "modified_by": "Administrator", "module": "Workflow", "name": "Workflow Document State", diff --git a/frappe/workflow/doctype/workflow_document_state/workflow_document_state.py b/frappe/workflow/doctype/workflow_document_state/workflow_document_state.py index ed65665c87..d1644578d8 100644 --- a/frappe/workflow/doctype/workflow_document_state/workflow_document_state.py +++ b/frappe/workflow/doctype/workflow_document_state/workflow_document_state.py @@ -15,6 +15,7 @@ class WorkflowDocumentState(Document): from frappe.types import DF allow_edit: DF.Link + avoid_status_override: DF.Check doc_status: DF.Literal["0", "1", "2"] is_optional_state: DF.Check message: DF.Text | None