Merge branch 'develop' into select-print-language

This commit is contained in:
Raffael Meyer 2022-11-13 23:28:53 +01:00 committed by GitHub
commit 1fa770ef6e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
57 changed files with 707 additions and 219 deletions

View file

@ -71,15 +71,6 @@ pull_request_rules:
assignees:
- "{{ author }}"
- name: backport to develop
conditions:
- label="backport develop"
actions:
backport:
branches:
- develop
assignees:
- "{{ author }}"
- name: backport to version-13-pre-release
conditions:

View file

@ -1018,19 +1018,15 @@ def get_precision(
return get_field_precision(get_meta(doctype).get_field(fieldname), doc, currency)
def generate_hash(txt: str | None = None, length: int | None = None) -> str:
"""Generates random hash for given text + current timestamp + random string."""
import hashlib
import time
def generate_hash(txt: str | None = None, length: int = 56) -> str:
"""Generate random hash using best available randomness source."""
import math
import secrets
from .utils import random_string
if not length:
length = 56
digest = hashlib.sha224(
((txt or "") + repr(time.time()) + repr(random_string(8))).encode()
).hexdigest()
if length:
digest = digest[:length]
return digest
return secrets.token_hex(math.ceil(length / 2))[:length]
def reset_metadata_version():

View file

@ -432,7 +432,7 @@ class DataExporter:
row[_column_start_end.start + i + 1] = value
def build_response_as_excel(self):
filename = frappe.generate_hash("", 10)
filename = frappe.generate_hash(length=10)
with open(filename, "wb") as f:
f.write(cstr(self.writer.getvalue()).encode("utf-8"))
f = open(filename)

View file

@ -97,7 +97,7 @@ class TestImporter(FrappeTestCase):
def test_data_import_update(self):
existing_doc = frappe.get_doc(
doctype=doctype_name,
title=frappe.generate_hash(doctype_name, 8),
title=frappe.generate_hash(length=8),
table_field_1=[{"child_title": "child title to update"}],
)
existing_doc.save()

View file

@ -1,8 +1,8 @@
// Copyright (c) {year}, {app_publisher} and contributors
// For license information, please see license.txt
frappe.ui.form.on('{doctype}', {{
// refresh: function(frm) {{
// frappe.ui.form.on("{doctype}", {{
// refresh(frm) {{
// }}
}});
// }},
// }});

View file

@ -1,5 +1,5 @@
/* eslint-disable */
frappe.listview_settings['{doctype}'] = {{
// add_fields: ["status"],
// filters:[["status","=", "Open"]]
}};
// frappe.listview_settings["{doctype}"] = {{
// add_fields: ["status"],
// filters: [["status","=", "Open"]],
// }};

View file

@ -24,6 +24,7 @@
"custom",
"beta",
"is_virtual",
"queue_in_background",
"fields_section_break",
"fields",
"sb1",
@ -600,6 +601,13 @@
"fieldtype": "Check",
"label": "Make Attachments Public by Default"
},
{
"default": "0",
"depends_on": "eval: doc.is_submittable",
"fieldname": "queue_in_background",
"fieldtype": "Check",
"label": "Queue in Background"
},
{
"fieldname": "default_view",
"fieldtype": "Select",

View file

@ -329,7 +329,7 @@ class DocType(Document):
"DocField", "parent", dict(fieldtype=["in", frappe.model.table_fields], options=self.name)
)
for p in parent_list:
frappe.db.update("DocType", p.parent, {}, for_update=False)
frappe.db.set_value("DocType", p.parent, {}, for_update=False)
def scrub_field_names(self):
"""Sluggify fieldnames if not set from Label."""

View file

@ -17,6 +17,7 @@ DEFAULT_LOGTYPES_RETENTION = {
"Error Snapshot": 30,
"Scheduled Job Log": 90,
"Route History": 90,
"Submission Queue": 30,
}
@ -151,6 +152,7 @@ LOG_DOCTYPES = [
"Email Queue Recipient",
"Error Snapshot",
"Error Log",
"Submission Queue",
]

View file

@ -211,3 +211,25 @@ frappe.qb.from_(todo).select(todo.name).where(todo.name == "{todo.name}").run()
"""
script.save()
script.execute_method()
def test_scripts_all_the_way_down(self):
# why not
script = frappe.get_doc(
doctype="Server Script",
name="test_nested_scripts_1",
script_type="API",
api_method="test_nested_scripts_1",
script=f"""log("nothing")""",
)
script.insert()
script.execute_method()
script = frappe.get_doc(
doctype="Server Script",
name="test_nested_scripts_2",
script_type="API",
api_method="test_nested_scripts_2",
script=f"""frappe.call("test_nested_scripts_1")""",
)
script.insert()
script.execute_method()

View file

@ -0,0 +1,14 @@
// Copyright (c) 2022, Frappe Technologies and contributors
// For license information, please see license.txt
frappe.ui.form.on("Submission Queue", {
refresh: function (frm) {
if (frm.doc.status === "Queued" && frm.doc.job_id) {
frm.add_custom_button(__("Unlock Reference Document"), () => {
frappe.confirm(__("Are you sure you want to go ahead with this action?"), () => {
frm.call("unlock_doc");
});
});
}
},
});

View file

@ -0,0 +1,123 @@
{
"actions": [],
"autoname": "hash",
"creation": "2022-10-04 00:41:00.028163",
"doctype": "DocType",
"editable_grid": 1,
"engine": "InnoDB",
"field_order": [
"status",
"created_at",
"enqueued_by",
"job_id",
"column_break_5",
"ended_at",
"ref_doctype",
"ref_docname",
"section_break_8",
"exception"
],
"fields": [
{
"fieldname": "job_id",
"fieldtype": "Data",
"label": "Job Id",
"read_only": 1
},
{
"fieldname": "ref_doctype",
"fieldtype": "Link",
"in_list_view": 1,
"label": "Reference DocType",
"options": "DocType",
"read_only": 1
},
{
"fieldname": "ref_docname",
"fieldtype": "Dynamic Link",
"in_list_view": 1,
"label": "Reference Docname",
"options": "ref_doctype",
"read_only": 1,
"search_index": 1
},
{
"fieldname": "status",
"fieldtype": "Select",
"hidden": 1,
"in_list_view": 1,
"label": "Status",
"options": "Queued\nFinished\nFailed",
"read_only": 1
},
{
"fieldname": "column_break_5",
"fieldtype": "Column Break"
},
{
"fieldname": "section_break_8",
"fieldtype": "Section Break"
},
{
"fieldname": "enqueued_by",
"fieldtype": "Data",
"is_virtual": 1,
"label": "Enqueued By",
"read_only": 1
},
{
"fieldname": "ended_at",
"fieldtype": "Datetime",
"label": "Ended At",
"read_only": 1
},
{
"fieldname": "created_at",
"fieldtype": "Datetime",
"is_virtual": 1,
"label": "Created At",
"read_only": 1
},
{
"fieldname": "exception",
"fieldtype": "Text",
"label": "Exception",
"read_only": 1
}
],
"index_web_pages_for_search": 1,
"links": [],
"modified": "2022-11-12 16:48:37.797232",
"modified_by": "Administrator",
"module": "Core",
"name": "Submission Queue",
"naming_rule": "Random",
"owner": "Administrator",
"permissions": [
{
"email": 1,
"export": 1,
"print": 1,
"read": 1,
"report": 1,
"role": "System Manager",
"share": 1
}
],
"sort_field": "modified",
"sort_order": "DESC",
"states": [
{
"color": "Blue",
"title": "Queued"
},
{
"color": "Red",
"title": "Failed"
},
{
"color": "Green",
"title": "Finished"
}
]
}

View file

@ -0,0 +1,193 @@
# Copyright (c) 2022, Frappe Technologies and contributors
# For license information, please see license.txt
from urllib.parse import quote
from rq import get_current_job
from rq.exceptions import NoSuchJobError
from rq.job import Job
import frappe
from frappe import _
from frappe.desk.doctype.notification_log.notification_log import enqueue_create_notification
from frappe.model.document import Document
from frappe.monitor import add_data_to_monitor
from frappe.utils import now, time_diff_in_seconds
from frappe.utils.background_jobs import get_redis_conn
from frappe.utils.data import cint
class SubmissionQueue(Document):
@property
def created_at(self):
return self.creation
@property
def enqueued_by(self):
return self.owner
@property
def queued_doc(self):
return getattr(self, "to_be_queued_doc", frappe.get_doc(self.ref_doctype, self.ref_docname))
@staticmethod
def clear_old_logs(days=30):
from frappe.query_builder import Interval
from frappe.query_builder.functions import Now
table = frappe.qb.DocType("Submission Queue")
frappe.db.delete(table, filters=(table.modified < (Now() - Interval(days=days))))
def insert(self, to_be_queued_doc: Document, action: str):
self.to_be_queued_doc = to_be_queued_doc
self.action_for_queuing = action
super().insert(ignore_permissions=True)
def lock(self):
self.queued_doc.lock()
def unlock(self):
self.queued_doc.unlock()
def update_job_id(self, job_id):
frappe.db.set_value(
self.doctype,
self.name,
{"job_id": job_id},
update_modified=False,
)
frappe.db.commit()
def after_insert(self):
self.queue_action(
"background_submission",
to_be_queued_doc=self.queued_doc,
action_for_queuing=self.action_for_queuing,
timeout=600,
enqueue_after_commit=True,
)
def background_submission(self, to_be_queued_doc: Document, action_for_queuing: str):
# Set the job id for that submission doctype
self.update_job_id(get_current_job().id)
_action = action_for_queuing.lower()
if _action == "update":
_action = "submit"
try:
getattr(to_be_queued_doc, _action)()
add_data_to_monitor(
doctype=to_be_queued_doc.doctype,
docname=to_be_queued_doc.name,
action=_action,
execution_time=time_diff_in_seconds(now(), self.created_at),
enqueued_by=self.enqueued_by,
)
values = {"status": "Finished"}
except Exception:
values = {"status": "Failed", "exception": frappe.get_traceback()}
frappe.db.rollback()
values["ended_at"] = now()
frappe.db.set_value(self.doctype, self.name, values, update_modified=False)
self.notify(values["status"], action_for_queuing)
def notify(self, submission_status: str, action: str):
if submission_status == "Failed":
doctype = self.doctype
docname = self.name
message = _("Submission of {0} {1} with action {2} failed")
else:
doctype = self.ref_doctype
docname = self.ref_docname
message = _("Submission of {0} {1} with action {2} completed successfully")
message = message.format(
frappe.bold(str(self.ref_doctype)), frappe.bold(self.ref_docname), frappe.bold(action)
)
time_diff = time_diff_in_seconds(now(), self.created_at)
if cint(time_diff) <= 60:
frappe.publish_realtime(
"msgprint",
{
"message": message
+ f". View it <a href='/app/{quote(doctype.lower().replace(' ', '-'))}/{quote(docname)}'><b>here</b></a>",
"alert": True,
"indicator": "red" if submission_status == "Failed" else "green",
},
user=self.enqueued_by,
)
else:
notification_doc = {
"type": "Alert",
"document_type": doctype,
"document_name": docname,
"subject": message,
}
notify_to = frappe.db.get_value("User", self.enqueued_by, fieldname="email")
enqueue_create_notification([notify_to], notification_doc)
def _unlock_reference_doc(self):
"""
Only execute if self.job_id is defined.
"""
try:
job = Job.fetch(self.job_id, connection=get_redis_conn())
status = job.get_status(refresh=True)
exc = job.exc_info
except NoSuchJobError:
exc = None
status = "failed"
if status in ("queued", "started"):
frappe.msgprint(_("Document in queue for execution!"))
return
self.queued_doc.unlock()
values = (
{"status": "Finished"} if status == "finished" else {"status": "Failed", "exception": exc}
)
frappe.db.set_value(self.doctype, self.name, values, update_modified=False)
frappe.msgprint(_("Document Unlocked"))
@frappe.whitelist()
def unlock_doc(self):
# NOTE: this can lead to some weird unlocking/locking behaviours.
# for example: hitting unlock on a submission could lead to unlocking of another submission
# of the same reference document.
if self.status != "Queued" and not self.job_id:
return
self._unlock_reference_doc()
def queue_submission(doc: Document, action: str, alert: bool = True):
queue = frappe.new_doc("Submission Queue")
queue.state = "Queued"
queue.ref_doctype = doc.doctype
queue.ref_docname = doc.name
queue.insert(doc, action)
if alert:
frappe.msgprint(
_("Queued for Submission. You can track the progress over {0}.").format(
f"<a href='/app/submission-queue/{queue.name}'><b>here</b></a>"
),
indicator="green",
alert=True,
)
@frappe.whitelist()
def get_latest_submissions(doctype, docname):
# NOTE: not used creation as orderby intentianlly as we have used update_modified=False everywhere
# hence assuming modified will be equal to creation for submission queue documents
dt = "Submission Queue"
filters = {"ref_doctype": doctype, "ref_docname": docname}
return {
"latest_submission": frappe.db.get_value(dt, filters),
"latest_failed_submission": frappe.db.get_value(dt, filters | {"status": "Failed"}),
}

View file

@ -0,0 +1,51 @@
# Copyright (c) 2022, Frappe Technologies and Contributors
# See license.txt
import time
import typing
import frappe
from frappe.tests.utils import FrappeTestCase, timeout
from frappe.utils.background_jobs import get_queue
if typing.TYPE_CHECKING:
from rq.job import Job
class TestSubmissionQueue(FrappeTestCase):
queue = get_queue(qtype="default")
@timeout(seconds=20)
def check_status(self, job: "Job", status, wait=True):
if wait:
while True:
if job.is_queued or job.is_started:
time.sleep(0.2)
else:
break
self.assertEqual(frappe.get_doc("RQ Job", job.id).status, status)
def test_queue_operation(self):
from frappe.core.doctype.doctype.test_doctype import new_doctype
from frappe.core.doctype.submission_queue.submission_queue import queue_submission
if not frappe.db.table_exists("Test Submission Queue", cached=False):
doc = new_doctype("Test Submission Queue", is_submittable=True, queue_in_background=True)
doc.insert()
d = frappe.new_doc("Test Submission Queue")
d.update({"some_fieldname": "Random"})
d.insert()
frappe.db.commit()
queue_submission(d, "submit")
frappe.db.commit()
# Waiting for execution
time.sleep(4)
submission_queue = frappe.get_last_doc("Submission Queue")
# Test queueing / starting
job = self.queue.fetch_job(submission_queue.job_id)
# Test completion
self.check_status(job, status="finished")

View file

@ -471,7 +471,7 @@ class User(Document):
frappe.rename_doc("Notification Settings", old_name, new_name, force=True, show_alert=False)
# set email
frappe.db.update("User", new_name, "email", new_name)
frappe.db.set_value("User", new_name, "email", new_name)
def append_roles(self, *roles):
"""Add roles to user"""

View file

@ -149,6 +149,10 @@ frappe.ui.form.on("Customize Form", {
const is_autoname_autoincrement = frm.doc.autoname === "autoincrement";
frm.set_df_property("naming_rule", "hidden", is_autoname_autoincrement);
frm.set_df_property("autoname", "read_only", is_autoname_autoincrement);
frm.toggle_display(
["queue_in_background"],
frappe.get_meta(frm.doc.doc_type).is_submittable || 0
);
}
frm.events.setup_export(frm);

View file

@ -20,6 +20,7 @@
"track_views",
"allow_auto_repeat",
"allow_import",
"queue_in_background",
"fields_section_break",
"fields",
"naming_section",
@ -341,6 +342,12 @@
"fieldtype": "Check",
"label": "Make Attachments Public by Default"
},
{
"default": "0",
"fieldname": "queue_in_background",
"fieldtype": "Check",
"label": "Queue in Background"
},
{
"fieldname": "default_view",
"fieldtype": "Select",
@ -367,7 +374,7 @@
"index_web_pages_for_search": 1,
"issingle": 1,
"links": [],
"modified": "2022-08-30 11:45:16.772277",
"modified": "2022-10-30 23:39:49.628093",
"modified_by": "Administrator",
"module": "Custom",
"name": "Customize Form",

View file

@ -571,6 +571,7 @@ doctype_properties = {
"allow_copy": "Check",
"istable": "Check",
"quick_entry": "Check",
"queue_in_background": "Check",
"editable_grid": "Check",
"max_attachments": "Int",
"make_attachments_public": "Check",

View file

@ -30,7 +30,7 @@ from frappe.model.utils.link_count import flush_local_link_count
from frappe.query_builder.functions import Count
from frappe.utils import cast as cast_fieldtype
from frappe.utils import cint, get_datetime, get_table_name, getdate, now, sbool
from frappe.utils.deprecations import deprecated, deprecation_warning
from frappe.utils.deprecations import deprecated
IFNULL_PATTERN = re.compile(r"ifnull\(", flags=re.IGNORECASE)
INDEX_PATTERN = re.compile(r"\s*\([^)]+\)\s*")
@ -147,12 +147,11 @@ class Database:
self,
query: Query,
values: QueryValues = EmptyQueryValues,
*,
as_dict=0,
as_list=0,
formatted=0,
debug=0,
ignore_ddl=0,
as_utf8=0,
auto_commit=0,
update=None,
explain=False,
@ -165,10 +164,8 @@ class Database:
:param values: Tuple / List / Dict of values to be escaped and substituted in the query.
:param as_dict: Return as a dictionary.
:param as_list: Always return as a list.
:param formatted: Format values like date etc.
:param debug: Print query and `EXPLAIN` in debug log.
:param ignore_ddl: Catch exception if table, column missing.
:param as_utf8: Encode values as UTF 8.
:param auto_commit: Commit after executing the query.
:param update: Update this dict to all rows (if returned `as_dict`).
:param run: Returns query without executing it if False.
@ -274,20 +271,15 @@ class Database:
if pluck:
return [r[0] for r in self.last_result]
if as_utf8:
deprecation_warning("as_utf8 parameter is deprecated and will be removed in version 15.")
if formatted:
deprecation_warning("formatted parameter is deprecated and will be removed in version 15.")
# scrub output if required
if as_dict:
ret = self.fetch_as_dict(formatted, as_utf8)
ret = self.fetch_as_dict()
if update:
for r in ret:
r.update(update)
return ret
elif as_list or as_utf8:
return self.convert_to_lists(self.last_result, formatted, as_utf8)
elif as_list:
return self.convert_to_lists(self.last_result)
return self.last_result
def _log_query(self, mogrified_query: str, debug: bool = False, explain: bool = False) -> None:
@ -394,62 +386,27 @@ class Database:
):
raise ImplicitCommitError("This statement can cause implicit commit")
def fetch_as_dict(self, formatted=0, as_utf8=0) -> list[frappe._dict]:
def fetch_as_dict(self) -> list[frappe._dict]:
"""Internal. Converts results to dict."""
result = self.last_result
if result:
keys = [column[0] for column in self._cursor.description]
if not as_utf8:
return [frappe._dict(zip(keys, row)) for row in result]
ret = []
for r in result:
values = []
for value in r:
if as_utf8 and isinstance(value, str):
value = value.encode("utf-8")
values.append(value)
ret.append(frappe._dict(zip(keys, values)))
return ret
return [frappe._dict(zip(keys, row)) for row in result]
@staticmethod
def clear_db_table_cache(query):
if query and is_query_type(query, ("drop", "create")):
frappe.cache().delete_key("db_tables")
@staticmethod
def needs_formatting(result, formatted):
"""Returns true if the first row in the result has a Date, Datetime, Long Int."""
if result and result[0]:
for v in result[0]:
if isinstance(v, (datetime.date, datetime.timedelta, datetime.datetime, int)):
return True
if formatted and isinstance(v, (int, float)):
return True
return False
def get_description(self):
"""Returns result metadata."""
return self._cursor.description
@staticmethod
def convert_to_lists(res, formatted=0, as_utf8=0):
def convert_to_lists(res):
"""Convert tuple output to lists (internal)."""
if not as_utf8:
return [[value for value in row] for row in res]
nres = []
for r in res:
nr = []
for val in r:
if as_utf8 and isinstance(val, str):
val = val.encode("utf-8")
nr.append(val)
nres.append(nr)
return nres
return [[value for value in row] for row in res]
def get(self, doctype, filters=None, as_dict=True, cache=False):
"""Returns `get_value` with fieldname='*'"""
@ -849,11 +806,6 @@ class Database:
).run(debug=debug, run=run, as_dict=as_dict)
return {}
@deprecated
def update(self, *args, **kwargs):
"""Update multiple values. Alias for `set_value`."""
return self.set_value(*args, **kwargs)
def set_value(
self,
dt,
@ -879,7 +831,6 @@ class Database:
:param modified_by: Set this user as `modified_by`.
:param update_modified: default True. Set as false, if you don't want to update the timestamp.
:param debug: Print the query in the developer / js console.
:param for_update: [DEPRECATED] This function now performs updates in single query, locking is not required.
"""
is_single_doctype = not (dn and dt != dn)
to_update = field if isinstance(field, dict) else {field: val}
@ -889,9 +840,6 @@ class Database:
modified_by = modified_by or frappe.session.user
to_update.update({"modified": modified, "modified_by": modified_by})
if for_update:
deprecation_warning("for_update parameter is deprecated and will be removed in v15.")
if is_single_doctype:
frappe.db.delete(
"Singles", filters={"field": ("in", tuple(to_update)), "doctype": dt}, debug=debug
@ -922,32 +870,6 @@ class Database:
if dt in self.value_cache:
del self.value_cache[dt]
@deprecated
@staticmethod
def set(doc, field, val):
"""Set value in document. **Avoid**"""
doc.db_set(field, val)
@deprecated
def touch(self, doctype, docname):
"""Update the modified timestamp of this document."""
modified = now()
DocType = frappe.qb.DocType(doctype)
frappe.qb.update(DocType).set(DocType.modified, modified).where(DocType.name == docname).run()
return modified
@staticmethod
def set_temp(value):
"""Set a temperory value and return a key."""
key = frappe.generate_hash()
frappe.cache().hset("temp", key, value)
return key
@staticmethod
def get_temp(key):
"""Return the temperory value and delete it."""
return frappe.cache().hget("temp", key)
def set_global(self, key, val, user="__global"):
"""Save a global key value. Global values will be automatically set if they match fieldname."""
self.set_default(key, val, user)
@ -1107,7 +1029,7 @@ class Database:
return getdate(date).strftime("%Y-%m-%d")
@staticmethod
def format_datetime(datetime):
def format_datetime(datetime): # noqa: F811
if not datetime:
return FallBackDateTimeStr
@ -1251,10 +1173,6 @@ class Database:
"""
return self.sql_ddl(f"truncate `{get_table_name(doctype)}`")
@deprecated
def clear_table(self, doctype):
return self.truncate(doctype)
def get_last_created(self, doctype):
last_record = self.get_all(doctype, ("creation"), limit=1, order_by="creation desc")
if last_record:

View file

@ -3,8 +3,10 @@
import frappe
from frappe import _
from frappe.core.doctype.submission_queue.submission_queue import queue_submission
from frappe.model.document import Document
from frappe.utils import cint
from frappe.utils.scheduler import is_scheduler_inactive
class BulkUpdate(Document):
@ -44,8 +46,12 @@ def submit_cancel_or_update_docs(doctype, docnames, action="submit", data=None):
try:
message = ""
if action == "submit" and doc.docstatus.is_draft():
doc.submit()
message = _("Submitting {0}").format(doctype)
if doc.meta.queue_in_background and not is_scheduler_inactive():
queue_submission(doc, action)
message = _("Queuing {0} for Submission").format(doctype)
else:
doc.submit()
message = _("Submitting {0}").format(doctype)
elif action == "cancel" and doc.docstatus.is_submitted():
doc.cancel()
message = _("Cancelling {0}").format(doctype)

View file

@ -4,8 +4,10 @@
import json
import frappe
from frappe.core.doctype.submission_queue.submission_queue import queue_submission
from frappe.desk.form.load import run_onload
from frappe.monitor import add_data_to_monitor
from frappe.utils.scheduler import is_scheduler_inactive
@frappe.whitelist()
@ -16,8 +18,10 @@ def savedocs(doc, action):
# action
doc.docstatus = {"Save": 0, "Submit": 1, "Update": 1, "Cancel": 2}[action]
if doc.docstatus == 1:
if action == "Submit" and doc.meta.queue_in_background and not is_scheduler_inactive():
queue_submission(doc, action)
return
doc.submit()
else:
doc.save()
@ -27,7 +31,6 @@ def savedocs(doc, action):
send_updated_docs(doc)
add_data_to_monitor(doctype=doc.doctype, action=action)
frappe.msgprint(frappe._("Saved"), indicator="green", alert=True)

View file

@ -91,7 +91,7 @@ def validate_args(data):
def validate_fields(data):
wildcard = update_wildcard_field_param(data)
for field in data.fields or []:
for field in list(data.fields or []):
fieldname = extract_fieldname(field)
if is_standard(fieldname):
continue

View file

@ -74,7 +74,7 @@ class TestNewsletterMixin:
).insert(ignore_if_duplicate=True)
except Exception:
frappe.db.rollback(save_point=savepoint)
frappe.db.update(doctype, email_filters, "unsubscribed", 0)
frappe.db.set_value(doctype, email_filters, "unsubscribed", 0)
frappe.db.release_savepoint(savepoint)

View file

@ -132,19 +132,18 @@ def oauth_access(email_account: str, service: str):
if not service:
frappe.throw(frappe._("No Service is selected. Please select one and try again!"))
doctype = "Email Account"
if service == "GMail":
return authorize_google_access(email_account, doctype)
return authorize_google_access(email_account)
raise NotImplementedError(f"Service {service} currently doesn't have oauth implementation.")
def authorize_google_access(email_account, doctype: str = "Email Account", code: str = None):
def authorize_google_access(email_account: str, code: str = None):
"""Facilitates google oauth for email.
This is invoked 2 times - first time when user clicks `Authorze API Access` for getting the authorization url
This is invoked 2 times - first time when user clicks `Authorize API Access` for getting the authorization url
and second time for setting the refresh and access token in db when google redirects back with oauth code."""
doctype = "Email Account"
oauth_obj = GoogleOAuth("mail")
if not code:

View file

@ -100,12 +100,7 @@ class BaseDocument:
if d.get("doctype"):
self.doctype = d["doctype"]
self._table_fieldnames = (
d["_table_fieldnames"] # from cache
if "_table_fieldnames" in d
else {df.fieldname for df in self._get_table_fields()}
)
self._table_fieldnames = {df.fieldname for df in self._get_table_fields()}
self.update(d)
self.dont_update_if_missing = []

View file

@ -120,6 +120,10 @@ class Document(BaseDocument):
# incorrect arguments. let's not proceed.
raise ValueError("Illegal arguments")
@property
def is_locked(self):
return file_lock.lock_exists(self.get_signature())
@staticmethod
def whitelist(fn):
"""Decorator: Whitelist method to be called remotely via REST API."""
@ -142,9 +146,14 @@ class Document(BaseDocument):
self._fix_numeric_types()
else:
get_value_kwargs = {"for_update": self.flags.for_update, "as_dict": True}
if not isinstance(self.name, (dict, list)):
get_value_kwargs["order_by"] = None
d = frappe.db.get_value(
self.doctype, self.name, "*", as_dict=1, for_update=self.flags.for_update
doctype=self.doctype, filters=self.name, fieldname="*", **get_value_kwargs
)
if not d:
frappe.throw(
_("{0} {1} not found").format(_(self.doctype), self.name), frappe.DoesNotExistError
@ -295,6 +304,10 @@ class Document(BaseDocument):
follow_document(self.doctype, self.name, frappe.session.user)
return self
def check_if_locked(self):
if self.creation and self.is_locked:
raise frappe.DocumentLockedError
def save(self, *args, **kwargs):
"""Wrapper for _save"""
return self._save(*args, **kwargs)
@ -321,6 +334,7 @@ class Document(BaseDocument):
if self.get("__islocal") or not self.get("name"):
return self.insert()
self.check_if_locked()
self.check_permission("write", "save")
self.set_user_and_timestamp()
@ -744,12 +758,13 @@ class Document(BaseDocument):
Will also validate document transitions (Save > Submit > Cancel) calling
`self.check_docstatus_transition`."""
self.load_doc_before_save()
self.load_doc_before_save(raise_exception=True)
self._action = "save"
previous = self.get_doc_before_save()
previous = self._doc_before_save
if not previous or self.meta.get("is_virtual"):
# previous is None for new document insert
if not previous:
self.check_docstatus_transition(0)
return
@ -1048,7 +1063,7 @@ class Document(BaseDocument):
self.set_title_field()
def load_doc_before_save(self):
def load_doc_before_save(self, *, raise_exception: bool = False):
"""load existing document from db before saving"""
self._doc_before_save = None
@ -1059,6 +1074,9 @@ class Document(BaseDocument):
try:
self._doc_before_save = frappe.get_doc(self.doctype, self.name, for_update=True)
except frappe.DoesNotExistError:
if raise_exception:
raise
frappe.clear_last_message()
def run_post_save_methods(self):

View file

@ -278,7 +278,7 @@ def make_autoname(key="", doctype="", doc=""):
DE/09/01/00001 where 09 is the year, 01 is the month and 00001 is the series
"""
if key == "hash":
return frappe.generate_hash(doctype, 10)
return frappe.generate_hash(length=10)
series = NamingSeries(key)
return series.generate_next_name(doc)

View file

@ -1,12 +1,17 @@
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# License: MIT. See LICENSE
import json
from typing import TYPE_CHECKING, Union
import frappe
from frappe import _
from frappe.model.docstatus import DocStatus
from frappe.utils import cint
if TYPE_CHECKING:
from frappe.model.document import Document
from frappe.workflow.doctype.workflow.workflow import Workflow
class WorkflowStateError(frappe.ValidationError):
pass
@ -32,20 +37,22 @@ def get_workflow_name(doctype):
@frappe.whitelist()
def get_transitions(doc, workflow=None, raise_exception=False):
def get_transitions(
doc: Union["Document", str, dict], workflow: "Workflow" = None, raise_exception: bool = False
) -> list[dict]:
"""Return list of possible transitions for the given doc"""
doc = frappe.get_doc(frappe.parse_json(doc))
from frappe.model.document import Document
if not isinstance(doc, Document):
doc = frappe.get_doc(frappe.parse_json(doc))
doc.load_from_db()
if doc.is_new():
return []
doc.load_from_db()
doc.check_permission("read")
frappe.has_permission(doc, "read", throw=True)
roles = frappe.get_roles()
if not workflow:
workflow = get_workflow(doc.doctype)
workflow = workflow or get_workflow(doc.doctype)
current_state = doc.get(workflow.workflow_state_field)
if not current_state:
@ -55,11 +62,14 @@ def get_transitions(doc, workflow=None, raise_exception=False):
frappe.throw(_("Workflow State not set"), WorkflowStateError)
transitions = []
roles = frappe.get_roles()
for transition in workflow.transitions:
if transition.state == current_state and transition.allowed in roles:
if not is_transition_condition_satisfied(transition, doc):
continue
transitions.append(transition.as_dict())
return transitions
@ -79,7 +89,7 @@ def get_workflow_safe_globals():
)
def is_transition_condition_satisfied(transition, doc):
def is_transition_condition_satisfied(transition, doc) -> bool:
if not transition.condition:
return True
else:
@ -198,7 +208,7 @@ def validate_workflow(doc):
)
def get_workflow(doctype):
def get_workflow(doctype) -> "Workflow":
return frappe.get_doc("Workflow", get_workflow_name(doctype))

View file

@ -60,7 +60,7 @@ def execute():
# Maintain sequence (name, user, allow, for_value, applicable_for, apply_to_all_doctypes, creation, modified)
new_user_permissions_list.append(
(
frappe.generate_hash("", 10),
frappe.generate_hash(length=10),
user_permission.user,
user_permission.allow,
user_permission.for_value,

View file

@ -27,7 +27,7 @@ def execute():
email_values.append(
(
1,
frappe.generate_hash(contact_detail.email_id, 10),
frappe.generate_hash(length=10),
contact_detail.email_id,
"email_ids",
"Contact",
@ -44,7 +44,7 @@ def execute():
phone_values.append(
(
phone_counter,
frappe.generate_hash(contact_detail.email_id, 10),
frappe.generate_hash(length=10),
contact_detail.phone,
"phone_nos",
"Contact",
@ -63,7 +63,7 @@ def execute():
phone_values.append(
(
phone_counter,
frappe.generate_hash(contact_detail.email_id, 10),
frappe.generate_hash(length=10),
contact_detail.mobile_no,
"phone_nos",
"Contact",

View file

@ -28,7 +28,7 @@ def execute():
tag_list.append((tag.strip(), time, time, "Administrator"))
tag_link_name = frappe.generate_hash(_user_tags.name + tag.strip() + doctype.name, 10)
tag_link_name = frappe.generate_hash(length=10)
tag_links.append(
(tag_link_name, doctype.name, _user_tags.name, tag.strip(), time, time, "Administrator")
)

View file

@ -15,4 +15,4 @@ def execute():
if isinstance(data, list):
# double escape braces
jstr = f'{{"columns":{jstr}}}'
frappe.db.update("Report", record["name"], "json", jstr)
frappe.db.set_value("Report", record["name"], "json", jstr)

View file

@ -32,4 +32,4 @@ def execute():
for agg in ["avg", "max", "min", "sum"]:
script = re.sub(f"frappe.db.{agg}\\(", f"frappe.qb.{agg}(", script)
frappe.db.update("Server Script", name, "script", script)
frappe.db.set_value("Server Script", name, "script", script)

View file

@ -450,6 +450,7 @@ frappe.ui.form.Form = class FrappeForm {
.toggleClass("cancelled-form", this.doc.docstatus === 2);
this.show_conflict_message();
this.show_submission_queue_banner();
if (frappe.boot.read_only) {
this.disable_form();
@ -2036,6 +2037,83 @@ frappe.ui.form.Form = class FrappeForm {
.filter((user) => !["Administrator", frappe.session.user].includes(user))
.filter(Boolean);
}
show_submission_queue_banner() {
let wrapper = this.layout.wrapper.find(".submission-queue-banner");
if (
!(
this.meta.is_submittable &&
this.meta.queue_in_background &&
!this.doc.__islocal &&
this.doc.docstatus === 0
)
) {
if (wrapper.length) {
wrapper.hide();
wrapper.html("");
}
return;
}
if (!wrapper.length) {
wrapper = $('<div class="submission-queue-banner form-message yellow">');
this.layout.wrapper.prepend(wrapper);
}
frappe
.call({
method: "frappe.core.doctype.submission_queue.submission_queue.get_latest_submissions",
args: { doctype: this.doctype, docname: this.docname },
})
.then((r) => {
if (r.message.latest_submission) {
// if we are here that means some submission(s) were queued and are in queued/failed state
let col_width = 4;
let failed_link = "";
let submission_label = __("Previous Submission");
if (r.message.latest_failed_submission) {
if (r.message.latest_failed_submission !== r.message.latest_submission) {
col_width = 3;
failed_link = `<div class="col-md-3">
<a href='/app/submission-queue/${r.message.latest_failed_submission}'>${__(
"Previous Falied Submission"
)}</a>
</div>`;
} else {
submission_label = __("Previous Falied Submission");
}
}
let html = `
<div class="row">
<div class="col-md-${col_width}">
<strong>${__("Submission Status:")}</strong>
</div>
<div class="col-md-${col_width}">
<a href='/app/submission-queue/${r.message.latest_submission}'>${submission_label}</a>
</div>
${failed_link}
<div class="col-md-${col_width}">
<a href='/app/submission-queue?ref_doctype=${encodeURIComponent(
this.doctype
)}&ref_docname=${encodeURIComponent(this.docname)}'>${__(
"All Submissions"
)}</a>
</div>
</div>
`;
wrapper.show();
wrapper.html(html);
} else {
wrapper.hide();
wrapper.html("");
}
});
}
};
frappe.validated = 0;

View file

@ -315,7 +315,7 @@ frappe.views.ListViewSelect = class ListViewSelect {
accounts.forEach((account) => {
let email_account =
account.email_id == "All Accounts" ? "All Accounts" : account.email_account;
let route = `/app/communication/inbox/${email_account}`;
let route = `/app/communication/view/inbox/${email_account}`;
let display_name = ["All Accounts", "Sent Mail", "Spam", "Trash"].includes(
account.email_id
)

View file

@ -327,13 +327,14 @@ frappe.ui.Page = class Page {
//--- Menu --//
add_menu_item(label, click, standard, shortcut) {
add_menu_item(label, click, standard, shortcut, show_parent) {
return this.add_dropdown_item({
label,
click,
standard,
parent: this.menu,
shortcut,
show_parent,
});
}
@ -424,7 +425,7 @@ frappe.ui.Page = class Page {
icon = null,
}) {
if (show_parent) {
parent.parent().removeClass("hide");
parent.parent().removeClass("hide hidden-xl");
}
let $link = this.is_in_group_button_dropdown(parent, "li > a.grey-link > span", label);
@ -602,8 +603,11 @@ frappe.ui.Page = class Page {
};
// Add actions as menu item in Mobile View
let menu_item_label = group ? `${group} > ${label}` : label;
let menu_item = this.add_menu_item(menu_item_label, _action, false);
let menu_item = this.add_menu_item(menu_item_label, _action, false, false, false);
menu_item.parent().addClass("hidden-xl");
if (this.menu_btn_group.hasClass("hide")) {
this.menu_btn_group.removeClass("hide").addClass("hidden-xl");
}
if (group) {
var $group = this.get_or_add_inner_group_button(group);

View file

@ -131,9 +131,8 @@ frappe.breadcrumbs = {
}
},
async set_list_breadcrumb(breadcrumbs) {
set_list_breadcrumb(breadcrumbs) {
const doctype = breadcrumbs.doctype;
await frappe.model.with_doctype(doctype);
const doctype_meta = frappe.get_doc("DocType", doctype);
if (
(doctype === "User" && !frappe.user.has_role("System Manager")) ||

View file

@ -3,7 +3,7 @@
{% if parent %}
{%- set dropdown_id = 'id-' + frappe.utils.generate_hash('Dropdown', 12) -%}
{%- set dropdown_id = 'id-' + frappe.utils.generate_hash(length=12) -%}
<li class="nav-item dropdown {% if submenu %} dropdown-submenu {% endif %}">
<a class="nav-link dropdown-toggle" href="#" id="{{ dropdown_id }}" role="button"
data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
@ -16,7 +16,7 @@
</ul>
</li>
{% else %}
{%- set dropdown_id = 'id-' + frappe.utils.generate_hash('Dropdown', 12) -%}
{%- set dropdown_id = 'id-' + frappe.utils.generate_hash(length=12) -%}
<li class="dropdown {% if submenu %} dropdown-submenu {% endif %}">
<a class="dropdown-item dropdown-toggle" href="#" id="{{ dropdown_id }}" role="button"
data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">

View file

@ -2,8 +2,8 @@
<li>
<form action='/search'>
<input name='q' class='form-control navbar-search' type='text'
value='{{ frappe.form_dict.q or ''}}'
value='{{ frappe.form_dict.q|e or ''}}'
{% if not frappe.form_dict.q%}placeholder="{{ _("Search...") }}"{% endif %}>
</form>
</li>
{% endif %}
{% endif %}

View file

@ -791,23 +791,6 @@ class TestDBSetValue(FrappeTestCase):
cached_doc = frappe.get_cached_doc(self.todo2.doctype, self.todo2.name)
self.assertEqual(cached_doc.description, description)
def test_update_alias(self):
args = (self.todo1.doctype, self.todo1.name, "description", "Updated by `test_update_alias`")
kwargs = {
"for_update": False,
"modified": None,
"modified_by": None,
"update_modified": True,
"debug": False,
}
self.assertTrue("return self.set_value(" in inspect.getsource(frappe.db.update))
with patch.object(Database, "set_value") as set_value:
frappe.db.update(*args, **kwargs)
set_value.assert_called_once()
set_value.assert_called_with(*args, **kwargs)
@classmethod
def tearDownClass(cls):
frappe.db.rollback()

View file

@ -411,6 +411,19 @@ class TestDocument(FrappeTestCase):
todo.save()
self.assertEqual(todo.notify_update.call_count, 1)
def test_error_on_saving_new_doc_with_name(self):
"""Trying to save a new doc with name should raise DoesNotExistError"""
doc = frappe.get_doc(
{
"doctype": "ToDo",
"description": "this should raise frappe.DoesNotExistError",
"name": "lets-trick-doc-save",
}
)
self.assertRaises(frappe.DoesNotExistError, doc.save)
class TestDocumentWebView(FrappeTestCase):
def get(self, path, user="Guest"):

View file

@ -16,3 +16,23 @@ class TestDocumentLocks(FrappeTestCase):
todo_1.lock()
self.assertRaises(frappe.DocumentLockedError, todo.lock)
todo_1.unlock()
def test_operations_on_locked_documents(self):
todo = frappe.get_doc(dict(doctype="ToDo", description="testing operations")).insert()
todo.lock()
with self.assertRaises(frappe.DocumentLockedError):
todo.description = "Random"
todo.save()
# Checking for persistant locks across all instances.
doc = frappe.get_doc("ToDo", todo.name)
self.assertEqual(doc.is_locked, True)
with self.assertRaises(frappe.DocumentLockedError):
doc.description = "Random"
doc.save()
doc.unlock()
self.assertEqual(doc.is_locked, False)
self.assertEqual(todo.is_locked, False)

View file

@ -310,6 +310,20 @@ class TestEmail(FrappeTestCase):
email_account.enable_incoming = False
class TestVerifiedRequests(FrappeTestCase):
def test_round_trip(self):
from frappe.utils import set_request
from frappe.utils.verified_command import get_signed_params, verify_request
test_cases = [{"xyz": "abc"}, {"email": "a@b.com", "user": "xyz"}]
for params in test_cases:
signed_url = get_signed_params(params)
set_request(method="GET", path="?" + signed_url)
self.assertTrue(verify_request())
frappe.local.request = None
if __name__ == "__main__":
import unittest

View file

@ -47,7 +47,7 @@ class TestUtils(FrappeTestCase):
if self._testMethodName == "test_export_doc":
self.note = frappe.new_doc("Note")
self.note.title = frappe.generate_hash("Note", length=10)
self.note.title = frappe.generate_hash(length=10)
self.note.save()
if self._testMethodName == "test_make_boilerplate":

View file

@ -493,6 +493,10 @@ class TestDateUtils(FrappeTestCase):
frappe.utils.get_last_day_of_week("2020-12-28"), frappe.utils.getdate("2021-01-02")
)
def test_is_last_day_of_the_month(self):
self.assertEqual(frappe.utils.is_last_day_of_the_month("2020-12-24"), False)
self.assertEqual(frappe.utils.is_last_day_of_the_month("2020-12-31"), True)
def test_get_time(self):
datetime_input = now_datetime()
timedelta_input = get_timedelta()

View file

@ -52,6 +52,8 @@ def enqueue(
method,
queue="default",
timeout=None,
on_success=None,
on_failure=None,
event=None,
is_async=True,
job_name=None,
@ -116,6 +118,8 @@ def enqueue(
return q.enqueue_call(
execute_job,
on_success=on_success,
on_failure=on_failure,
timeout=timeout,
kwargs=queue_args,
at_front=at_front,

View file

@ -471,6 +471,12 @@ def get_last_day(dt):
return get_first_day(dt, 0, 1) + datetime.timedelta(-1)
def is_last_day_of_the_month(dt):
last_day_of_the_month = get_last_day(dt)
return getdate(dt) == getdate(last_day_of_the_month)
def get_quarter_ending(date):
date = getdate(date)

View file

@ -92,9 +92,7 @@ def web_blocks(blocks):
def get_dom_id(seed=None):
from frappe import generate_hash
if not seed:
seed = "DOM"
return "id-" + generate_hash(seed, 12)
return "id-" + generate_hash(12)
def include_script(path, preload=True):

View file

@ -266,7 +266,7 @@ def update_oauth_user(user, data, provider):
"email": get_email(data),
"gender": gender,
"enabled": 1,
"new_password": frappe.generate_hash(get_email(data)),
"new_password": frappe.generate_hash(),
"location": data.get("location"),
"user_type": "Website User",
"user_image": data.get("picture") or data.get("avatar_url"),

View file

@ -269,12 +269,15 @@ def call_with_form_dict(function, kwargs):
@contextmanager
def patched_qb():
require_patching = isinstance(frappe.qb.terms, types.ModuleType)
try:
_terms = frappe.qb.terms
frappe.qb.terms = _flatten(frappe.qb.terms)
if require_patching:
_terms = frappe.qb.terms
frappe.qb.terms = _flatten(frappe.qb.terms)
yield
finally:
frappe.qb.terms = _terms
if require_patching:
frappe.qb.terms = _terms
@lru_cache

View file

@ -16,15 +16,14 @@ def get_signed_params(params):
if not isinstance(params, str):
params = urlencode(params)
signature = hmac.new(params.encode(), digestmod=hashlib.md5)
signature.update(get_secret().encode())
return params + "&_signature=" + signature.hexdigest()
signature = _sign_message(params)
return params + "&_signature=" + signature
def get_secret():
return frappe.local.conf.get("secret") or str(
frappe.db.get_value("User", "Administrator", "creation")
)
from frappe.utils.password import get_encryption_key
return frappe.local.conf.get("secret") or get_encryption_key()
def verify_request():
@ -35,12 +34,10 @@ def verify_request():
signature_string = "&_signature="
if signature_string in query_string:
params, signature = query_string.split(signature_string)
params, given_signature = query_string.split(signature_string)
given_signature = hmac.new(params.encode("utf-8"), digestmod=hashlib.md5)
given_signature.update(get_secret().encode())
valid_signature = signature == given_signature.hexdigest()
computed_signature = _sign_message(params)
valid_signature = hmac.compare_digest(given_signature, computed_signature)
valid_method = frappe.request.method == "GET"
valid_request_data = not (frappe.request.form or frappe.request.data)
@ -55,6 +52,10 @@ def verify_request():
return False
def _sign_message(message: str) -> str:
return hmac.new(get_secret().encode(), message.encode(), digestmod=hashlib.sha512).hexdigest()
def get_url(cmd, params, nonce=None, secret=None):
if not nonce:
nonce = params

View file

@ -75,7 +75,7 @@ class WebsiteTheme(Document):
self.delete_old_theme_files(folder_path)
# add a random suffix
suffix = frappe.generate_hash("Website Theme", 8) if self.custom else "style"
suffix = frappe.generate_hash(length=8) if self.custom else "style"
file_name = frappe.scrub(self.name) + "_" + suffix + ".css"
output_path = join_path(folder_path, file_name)

View file

@ -7,7 +7,7 @@
<div class="collapsible-items">
{%- for item in items -%}
<div class="collapsible-item">
{%- set collapse_id = 'id-' + frappe.utils.generate_hash('Collapse', 12) -%}
{%- set collapse_id = 'id-' + frappe.utils.generate_hash(length=12) -%}
<a class="collapsible-title" data-toggle="collapse" href="#{{ collapse_id }}" role="button"
aria-expanded="false" aria-controls="{{ collapse_id }}">
<div class="collapsible-item-title">{{ _(item.title) }}</div>

View file

@ -11,8 +11,8 @@
{%- for index in ['1', '2', '3', '4', '5', '6'] -%}
{%- set buttonid = 'id-' + frappe.utils.generate_hash('TabButton', 12) -%}
{%- set panelid = 'id-' + frappe.utils.generate_hash('TabPanel', 12) -%}
{%- set buttonid = 'id-' + frappe.utils.generate_hash(length=12) -%}
{%- set panelid = 'id-' + frappe.utils.generate_hash(length=12) -%}
{%- set tab = {
'title': values['tab_' + index + '_title'],

View file

@ -1,6 +1,6 @@
{%- set slideshow = frappe.get_doc('Website Slideshow', website_slideshow) -%}
{%- set slides = slideshow.slideshow_items -%}
{%- set slideshow_id = 'id-' + frappe.utils.generate_hash('Slideshow', 12) -%}
{%- set slideshow_id = 'id-' + frappe.utils.generate_hash(length=12) -%}
{{ slideshow.header or '' }}

View file

@ -3170,9 +3170,9 @@ socket.io-client@^4.5.1:
socket.io-parser "~4.2.0"
socket.io-parser@~4.0.4:
version "4.0.4"
resolved "https://registry.yarnpkg.com/socket.io-parser/-/socket.io-parser-4.0.4.tgz#9ea21b0d61508d18196ef04a2c6b9ab630f4c2b0"
integrity sha512-t+b0SS+IxG7Rxzda2EVvyBZbvFPBCjJoyHuE0P//7OAsN23GItzDRdWa6ALxZI/8R5ygK7jAR6t028/z+7295g==
version "4.0.5"
resolved "https://registry.yarnpkg.com/socket.io-parser/-/socket.io-parser-4.0.5.tgz#cb404382c32324cc962f27f3a44058cf6e0552df"
integrity sha512-sNjbT9dX63nqUFIOv95tTVm6elyIU4RvB1m8dOeZt+IgWwcWklFDOdmGcfo3zSiRsnR/3pJkjY5lfoGqEe4Eig==
dependencies:
"@types/component-emitter" "^1.2.10"
component-emitter "~1.3.0"