vassili/seitime-frappe
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Merge branch 'develop' into form-builder-vue3

Browse source
This commit is contained in:
Shariq Ansari 2022-11-16 18:00:19 +05:30 • committed by GitHub
commit 4f13ad24b1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
45 changed files with 682 additions and 234 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.github/workflows/ui-tests.yml vendored
View file

@ -121,7 +121,6 @@ jobs:
DB: mariadb
- name: Verify yarn.lock
if: ${{ steps.check-build.outputs.build == 'strawberry' }}
run: |
cd ~/frappe-bench/apps/frappe
yarn install --immutable --immutable-cache --check-cache

9
.mergify.yml
View file

@ -71,15 +71,6 @@ pull_request_rules:
assignees:
- "{{ author }}"
- name: backport to develop
conditions:
- label="backport develop"
actions:
backport:
branches:
- develop
assignees:
- "{{ author }}"
- name: backport to version-13-pre-release
conditions:

4
.pre-commit-config.yaml
View file

@ -54,8 +54,8 @@ repos:
hooks:
- id: isort
- repo: https://gitlab.com/pycqa/flake8
rev: 3.9.2
- repo: https://github.com/PyCQA/flake8
rev: 5.0.4
hooks:
- id: flake8
additional_dependencies: ['flake8-bugbear',]

3
frappe/build.py
View file

@ -4,7 +4,6 @@ import os
import re
import shutil
import subprocess
from distutils.spawn import find_executable
from subprocess import getoutput
from tempfile import mkdtemp, mktemp
from urllib.parse import urlparse
@ -280,7 +279,7 @@ def check_node_executable():
warn = "⚠️ "
if node_version.major < 14:
click.echo(f"{warn} Please update your node version to 14")
if not find_executable("yarn"):
if not shutil.which("yarn"):
click.echo(f"{warn} Please install yarn using below command and try again.\nnpm install -g yarn")
click.echo()

7
frappe/commands/utils.py
View file

@ -2,7 +2,7 @@ import json
import os
import subprocess
import sys
from distutils.spawn import find_executable
from shutil import which
import click
@ -12,6 +12,7 @@ from frappe.coverage import CodeCoverage
from frappe.exceptions import SiteNotSpecifiedError
from frappe.utils import cint, update_progress_bar
find_executable = which # backwards compatibility
DATA_IMPORT_DEPRECATION = (
"[DEPRECATED] The `import-csv` command used 'Data Import Legacy' which has been deprecated.\n"
"Use `data-import` command instead to import data via 'Data Import'."
@ -525,7 +526,7 @@ def postgres(context):
def _mariadb():
from frappe.database.mariadb.database import MariaDBDatabase
mysql = find_executable("mysql")
mysql = which("mysql")
command = [
mysql,
"--port",
@ -544,7 +545,7 @@ def _mariadb():
def _psql():
psql = find_executable("psql")
psql = which("psql")
subprocess.run([psql, "-d", frappe.conf.db_name])

8
frappe/core/doctype/doctype/boilerplate/controller.js
View file

@ -1,8 +1,8 @@
// Copyright (c) {year}, {app_publisher} and contributors
// For license information, please see license.txt
frappe.ui.form.on('{doctype}', {{
// refresh: function(frm) {{
// frappe.ui.form.on("{doctype}", {{
// refresh(frm) {{
// }}
}});
// }},
// }});

8
frappe/core/doctype/doctype/boilerplate/controller_list.js
View file

@ -1,5 +1,5 @@
/* eslint-disable */
frappe.listview_settings['{doctype}'] = {{
// add_fields: ["status"],
// filters:[["status","=", "Open"]]
}};
// frappe.listview_settings["{doctype}"] = {{
// add_fields: ["status"],
// filters: [["status","=", "Open"]],
// }};

8
frappe/core/doctype/doctype/doctype.json
View file

@ -24,6 +24,7 @@
"custom",
"beta",
"is_virtual",
"queue_in_background",
"fields_section_break",
"fields",
"sb1",
@ -600,6 +601,13 @@
"fieldtype": "Check",
"label": "Make Attachments Public by Default"
},
{
"default": "0",
"depends_on": "eval: doc.is_submittable",
"fieldname": "queue_in_background",
"fieldtype": "Check",
"label": "Queue in Background"
},
{
"fieldname": "default_view",
"fieldtype": "Select",

2
frappe/core/doctype/doctype/doctype.py
View file

@ -329,7 +329,7 @@ class DocType(Document):
"DocField", "parent", dict(fieldtype=["in", frappe.model.table_fields], options=self.name)
)
for p in parent_list:
frappe.db.update("DocType", p.parent, {}, for_update=False)
frappe.db.set_value("DocType", p.parent, {}, for_update=False)
def scrub_field_names(self):
"""Sluggify fieldnames if not set from Label."""

25
frappe/core/doctype/file/file.py
View file

@ -78,21 +78,38 @@ class File(Document):
self.validate_duplicate_entry()
def validate(self):
if self.is_folder:
return
# Ensure correct formatting and type
self.file_url = unquote(self.file_url) if self.file_url else ""
self.validate_attachment_references()
# when dict is passed to get_doc for creation of new_doc, is_new returns None
# this case is handled inside handle_is_private_changed
if not self.is_new() and self.has_value_changed("is_private"):
self.handle_is_private_changed()
if not self.is_folder:
self.validate_file_path()
self.validate_file_url()
self.validate_file_on_disk()
self.validate_file_path()
self.validate_file_url()
self.validate_file_on_disk()
self.file_size = frappe.form_dict.file_size or self.file_size
def validate_attachment_references(self):
if not self.attached_to_doctype:
return
if self.attached_to_name and not isinstance(self.attached_to_name, str):
frappe.throw(_("Attached To Name must be a string"), TypeError)
if not self.attached_to_field:
return
if not frappe.get_meta(self.attached_to_doctype).has_field(self.attached_to_field):
frappe.throw(_("The fieldname you've specified in Attached To Field is invalid"))
def after_rename(self, *args, **kwargs):
for successor in self.get_successors():
setup_folder_path(successor, self.name)

5
frappe/core/doctype/log_settings/log_settings.py
View file

@ -17,6 +17,7 @@ DEFAULT_LOGTYPES_RETENTION = {
"Error Snapshot": 30,
"Scheduled Job Log": 90,
"Route History": 90,
"Submission Queue": 30,
}
@ -68,6 +69,9 @@ class LogSettings(Document):
added_logtypes = set()
for logtype, retention in DEFAULT_LOGTYPES_RETENTION.items():
if logtype not in existing_logtypes and _supports_log_clearing(logtype):
if not frappe.db.exists("DocType", logtype):
continue
self.append("logs_to_clear", {"ref_doctype": logtype, "days": cint(retention)})
added_logtypes.add(logtype)
@ -151,6 +155,7 @@ LOG_DOCTYPES = [
"Email Queue Recipient",
"Error Snapshot",
"Error Log",
"Submission Queue",
]

12
frappe/core/doctype/rq_worker/rq_worker.json
View file

@ -20,7 +20,8 @@
"column_break_12",
"birth_date",
"last_heartbeat",
"total_working_time"
"total_working_time",
"utilization_percent"
],
"fields": [
{
@ -59,7 +60,6 @@
{
"fieldname": "successful_job_count",
"fieldtype": "Int",
"in_list_view": 1,
"label": "Successful Job Count"
},
{
@ -102,12 +102,18 @@
{
"fieldname": "column_break_12",
"fieldtype": "Column Break"
},
{
"fieldname": "utilization_percent",
"fieldtype": "Percent",
"in_list_view": 1,
"label": "Utilization %"
}
],
"in_create": 1,
"is_virtual": 1,
"links": [],
"modified": "2022-09-11 05:02:53.981705",
"modified": "2022-11-14 15:35:32.786012",
"modified_by": "Administrator",
"module": "Core",
"name": "RQ Worker",

10
frappe/core/doctype/rq_worker/rq_worker.py
View file

@ -1,6 +1,9 @@
# Copyright (c) 2022, Frappe Technologies and contributors
# For license information, please see license.txt
import datetime
from contextlib import suppress
from rq import Worker
import frappe
@ -66,4 +69,11 @@ def serialize_worker(worker: Worker) -> frappe._dict:
_comment_count=0,
modified=convert_utc_to_user_timezone(worker.last_heartbeat),
creation=convert_utc_to_user_timezone(worker.birth_date),
utilization_percent=compute_utilization(worker),
)
def compute_utilization(worker: Worker) -> float:
with suppress(Exception):
total_time = (datetime.datetime.utcnow() - worker.birth_date).total_seconds()
return worker.total_working_time / total_time * 100

22
frappe/core/doctype/server_script/test_server_script.py
View file

@ -211,3 +211,25 @@ frappe.qb.from_(todo).select(todo.name).where(todo.name == "{todo.name}").run()
"""
script.save()
script.execute_method()
def test_scripts_all_the_way_down(self):
# why not
script = frappe.get_doc(
doctype="Server Script",
name="test_nested_scripts_1",
script_type="API",
api_method="test_nested_scripts_1",
script=f"""log("nothing")""",
)
script.insert()
script.execute_method()
script = frappe.get_doc(
doctype="Server Script",
name="test_nested_scripts_2",
script_type="API",
api_method="test_nested_scripts_2",
script=f"""frappe.call("test_nested_scripts_1")""",
)
script.insert()
script.execute_method()

0
frappe/core/doctype/submission_queue/__init__.py Normal file
View file

14
frappe/core/doctype/submission_queue/submission_queue.js Normal file
View file

@ -0,0 +1,14 @@
// Copyright (c) 2022, Frappe Technologies and contributors
// For license information, please see license.txt
frappe.ui.form.on("Submission Queue", {
refresh: function (frm) {
if (frm.doc.status === "Queued" && frm.doc.job_id) {
frm.add_custom_button(__("Unlock Reference Document"), () => {
frappe.confirm(__("Are you sure you want to go ahead with this action?"), () => {
frm.call("unlock_doc");
});
});
}
},
});

123
frappe/core/doctype/submission_queue/submission_queue.json Normal file
View file

@ -0,0 +1,123 @@
{
"actions": [],
"autoname": "hash",
"creation": "2022-10-04 00:41:00.028163",
"doctype": "DocType",
"editable_grid": 1,
"engine": "InnoDB",
"field_order": [
"status",
"created_at",
"enqueued_by",
"job_id",
"column_break_5",
"ended_at",
"ref_doctype",
"ref_docname",
"section_break_8",
"exception"
],
"fields": [
{
"fieldname": "job_id",
"fieldtype": "Data",
"label": "Job Id",
"read_only": 1
},
{
"fieldname": "ref_doctype",
"fieldtype": "Link",
"in_list_view": 1,
"label": "Reference DocType",
"options": "DocType",
"read_only": 1
},
{
"fieldname": "ref_docname",
"fieldtype": "Dynamic Link",
"in_list_view": 1,
"label": "Reference Docname",
"options": "ref_doctype",
"read_only": 1,
"search_index": 1
},
{
"fieldname": "status",
"fieldtype": "Select",
"hidden": 1,
"in_list_view": 1,
"label": "Status",
"options": "Queued\nFinished\nFailed",
"read_only": 1
},
{
"fieldname": "column_break_5",
"fieldtype": "Column Break"
},
{
"fieldname": "section_break_8",
"fieldtype": "Section Break"
},
{
"fieldname": "enqueued_by",
"fieldtype": "Data",
"is_virtual": 1,
"label": "Enqueued By",
"read_only": 1
},
{
"fieldname": "ended_at",
"fieldtype": "Datetime",
"label": "Ended At",
"read_only": 1
},
{
"fieldname": "created_at",
"fieldtype": "Datetime",
"is_virtual": 1,
"label": "Created At",
"read_only": 1
},
{
"fieldname": "exception",
"fieldtype": "Text",
"label": "Exception",
"read_only": 1
}
],
"index_web_pages_for_search": 1,
"links": [],
"modified": "2022-11-12 16:48:37.797232",
"modified_by": "Administrator",
"module": "Core",
"name": "Submission Queue",
"naming_rule": "Random",
"owner": "Administrator",
"permissions": [
{
"email": 1,
"export": 1,
"print": 1,
"read": 1,
"report": 1,
"role": "System Manager",
"share": 1
}
],
"sort_field": "modified",
"sort_order": "DESC",
"states": [
{
"color": "Blue",
"title": "Queued"
},
{
"color": "Red",
"title": "Failed"
},
{
"color": "Green",
"title": "Finished"
}
]
}

193
frappe/core/doctype/submission_queue/submission_queue.py Normal file
View file

@ -0,0 +1,193 @@
# Copyright (c) 2022, Frappe Technologies and contributors
# For license information, please see license.txt
from urllib.parse import quote
from rq import get_current_job
from rq.exceptions import NoSuchJobError
from rq.job import Job
import frappe
from frappe import _
from frappe.desk.doctype.notification_log.notification_log import enqueue_create_notification
from frappe.model.document import Document
from frappe.monitor import add_data_to_monitor
from frappe.utils import now, time_diff_in_seconds
from frappe.utils.background_jobs import get_redis_conn
from frappe.utils.data import cint
class SubmissionQueue(Document):
@property
def created_at(self):
return self.creation
@property
def enqueued_by(self):
return self.owner
@property
def queued_doc(self):
return getattr(self, "to_be_queued_doc", frappe.get_doc(self.ref_doctype, self.ref_docname))
@staticmethod
def clear_old_logs(days=30):
from frappe.query_builder import Interval
from frappe.query_builder.functions import Now
table = frappe.qb.DocType("Submission Queue")
frappe.db.delete(table, filters=(table.modified < (Now() - Interval(days=days))))
def insert(self, to_be_queued_doc: Document, action: str):
self.to_be_queued_doc = to_be_queued_doc
self.action_for_queuing = action
super().insert(ignore_permissions=True)
def lock(self):
self.queued_doc.lock()
def unlock(self):
self.queued_doc.unlock()
def update_job_id(self, job_id):
frappe.db.set_value(
self.doctype,
self.name,
{"job_id": job_id},
update_modified=False,
)
frappe.db.commit()
def after_insert(self):
self.queue_action(
"background_submission",
to_be_queued_doc=self.queued_doc,
action_for_queuing=self.action_for_queuing,
timeout=600,
enqueue_after_commit=True,
)
def background_submission(self, to_be_queued_doc: Document, action_for_queuing: str):
# Set the job id for that submission doctype
self.update_job_id(get_current_job().id)
_action = action_for_queuing.lower()
if _action == "update":
_action = "submit"
try:
getattr(to_be_queued_doc, _action)()
add_data_to_monitor(
doctype=to_be_queued_doc.doctype,
docname=to_be_queued_doc.name,
action=_action,
execution_time=time_diff_in_seconds(now(), self.created_at),
enqueued_by=self.enqueued_by,
)
values = {"status": "Finished"}
except Exception:
values = {"status": "Failed", "exception": frappe.get_traceback()}
frappe.db.rollback()
values["ended_at"] = now()
frappe.db.set_value(self.doctype, self.name, values, update_modified=False)
self.notify(values["status"], action_for_queuing)
def notify(self, submission_status: str, action: str):
if submission_status == "Failed":
doctype = self.doctype
docname = self.name
message = _("Submission of {0} {1} with action {2} failed")
else:
doctype = self.ref_doctype
docname = self.ref_docname
message = _("Submission of {0} {1} with action {2} completed successfully")
message = message.format(
frappe.bold(str(self.ref_doctype)), frappe.bold(self.ref_docname), frappe.bold(action)
)
time_diff = time_diff_in_seconds(now(), self.created_at)
if cint(time_diff) <= 60:
frappe.publish_realtime(
"msgprint",
{
"message": message
+ f". View it <a href='/app/{quote(doctype.lower().replace(' ', '-'))}/{quote(docname)}'><b>here</b></a>",
"alert": True,
"indicator": "red" if submission_status == "Failed" else "green",
},
user=self.enqueued_by,
)
else:
notification_doc = {
"type": "Alert",
"document_type": doctype,
"document_name": docname,
"subject": message,
}
notify_to = frappe.db.get_value("User", self.enqueued_by, fieldname="email")
enqueue_create_notification([notify_to], notification_doc)
def _unlock_reference_doc(self):
"""
Only execute if self.job_id is defined.
"""
try:
job = Job.fetch(self.job_id, connection=get_redis_conn())
status = job.get_status(refresh=True)
exc = job.exc_info
except NoSuchJobError:
exc = None
status = "failed"
if status in ("queued", "started"):
frappe.msgprint(_("Document in queue for execution!"))
return
self.queued_doc.unlock()
values = (
{"status": "Finished"} if status == "finished" else {"status": "Failed", "exception": exc}
)
frappe.db.set_value(self.doctype, self.name, values, update_modified=False)
frappe.msgprint(_("Document Unlocked"))
@frappe.whitelist()
def unlock_doc(self):
# NOTE: this can lead to some weird unlocking/locking behaviours.
# for example: hitting unlock on a submission could lead to unlocking of another submission
# of the same reference document.
if self.status != "Queued" and not self.job_id:
return
self._unlock_reference_doc()
def queue_submission(doc: Document, action: str, alert: bool = True):
queue = frappe.new_doc("Submission Queue")
queue.state = "Queued"
queue.ref_doctype = doc.doctype
queue.ref_docname = doc.name
queue.insert(doc, action)
if alert:
frappe.msgprint(
_("Queued for Submission. You can track the progress over {0}.").format(
f"<a href='/app/submission-queue/{queue.name}'><b>here</b></a>"
),
indicator="green",
alert=True,
)
@frappe.whitelist()
def get_latest_submissions(doctype, docname):
# NOTE: not used creation as orderby intentianlly as we have used update_modified=False everywhere
# hence assuming modified will be equal to creation for submission queue documents
dt = "Submission Queue"
filters = {"ref_doctype": doctype, "ref_docname": docname}
return {
"latest_submission": frappe.db.get_value(dt, filters),
"latest_failed_submission": frappe.db.get_value(dt, filters | {"status": "Failed"}),
}

51
frappe/core/doctype/submission_queue/test_submission_queue.py Normal file
View file

@ -0,0 +1,51 @@
# Copyright (c) 2022, Frappe Technologies and Contributors
# See license.txt
import time
import typing
import frappe
from frappe.tests.utils import FrappeTestCase, timeout
from frappe.utils.background_jobs import get_queue
if typing.TYPE_CHECKING:
from rq.job import Job
class TestSubmissionQueue(FrappeTestCase):
queue = get_queue(qtype="default")
@timeout(seconds=20)
def check_status(self, job: "Job", status, wait=True):
if wait:
while True:
if job.is_queued or job.is_started:
time.sleep(0.2)
else:
break
self.assertEqual(frappe.get_doc("RQ Job", job.id).status, status)
def test_queue_operation(self):
from frappe.core.doctype.doctype.test_doctype import new_doctype
from frappe.core.doctype.submission_queue.submission_queue import queue_submission
if not frappe.db.table_exists("Test Submission Queue", cached=False):
doc = new_doctype("Test Submission Queue", is_submittable=True, queue_in_background=True)
doc.insert()
d = frappe.new_doc("Test Submission Queue")
d.update({"some_fieldname": "Random"})
d.insert()
frappe.db.commit()
queue_submission(d, "submit")
frappe.db.commit()
# Waiting for execution
time.sleep(4)
submission_queue = frappe.get_last_doc("Submission Queue")
# Test queueing / starting
job = self.queue.fetch_job(submission_queue.job_id)
# Test completion
self.check_status(job, status="finished")

2
frappe/core/doctype/user/user.py
View file

@ -471,7 +471,7 @@ class User(Document):
frappe.rename_doc("Notification Settings", old_name, new_name, force=True, show_alert=False)
# set email
frappe.db.update("User", new_name, "email", new_name)
frappe.db.set_value("User", new_name, "email", new_name)
def append_roles(self, *roles):
"""Add roles to user"""

4
frappe/custom/doctype/customize_form/customize_form.js
View file

@ -155,6 +155,10 @@ frappe.ui.form.on("Customize Form", {
const is_autoname_autoincrement = frm.doc.autoname === "autoincrement";
frm.set_df_property("naming_rule", "hidden", is_autoname_autoincrement);
frm.set_df_property("autoname", "read_only", is_autoname_autoincrement);
frm.toggle_display(
["queue_in_background"],
frappe.get_meta(frm.doc.doc_type).is_submittable || 0
);
}
frm.events.setup_export(frm);

9
frappe/custom/doctype/customize_form/customize_form.json
View file

@ -20,6 +20,7 @@
"track_views",
"allow_auto_repeat",
"allow_import",
"queue_in_background",
"fields_section_break",
"fields",
"naming_section",
@ -341,6 +342,12 @@
"fieldtype": "Check",
"label": "Make Attachments Public by Default"
},
{
"default": "0",
"fieldname": "queue_in_background",
"fieldtype": "Check",
"label": "Queue in Background"
},
{
"fieldname": "default_view",
"fieldtype": "Select",
@ -367,7 +374,7 @@
"index_web_pages_for_search": 1,
"issingle": 1,
"links": [],
"modified": "2022-08-30 11:45:16.772277",
"modified": "2022-10-30 23:39:49.628093",
"modified_by": "Administrator",
"module": "Custom",
"name": "Customize Form",

1
frappe/custom/doctype/customize_form/customize_form.py
View file

@ -571,6 +571,7 @@ doctype_properties = {
"allow_copy": "Check",
"istable": "Check",
"quick_entry": "Check",
"queue_in_background": "Check",
"editable_grid": "Check",
"max_attachments": "Int",
"make_attachments_public": "Check",

102
frappe/database/database.py
View file

@ -30,7 +30,7 @@ from frappe.model.utils.link_count import flush_local_link_count
from frappe.query_builder.functions import Count
from frappe.utils import cast as cast_fieldtype
from frappe.utils import cint, get_datetime, get_table_name, getdate, now, sbool
from frappe.utils.deprecations import deprecated, deprecation_warning
from frappe.utils.deprecations import deprecated
IFNULL_PATTERN = re.compile(r"ifnull\(", flags=re.IGNORECASE)
INDEX_PATTERN = re.compile(r"\s*\([^)]+\)\s*")
@ -147,12 +147,11 @@ class Database:
self,
query: Query,
values: QueryValues = EmptyQueryValues,
*,
as_dict=0,
as_list=0,
formatted=0,
debug=0,
ignore_ddl=0,
as_utf8=0,
auto_commit=0,
update=None,
explain=False,
@ -165,10 +164,8 @@ class Database:
:param values: Tuple / List / Dict of values to be escaped and substituted in the query.
:param as_dict: Return as a dictionary.
:param as_list: Always return as a list.
:param formatted: Format values like date etc.
:param debug: Print query and `EXPLAIN` in debug log.
:param ignore_ddl: Catch exception if table, column missing.
:param as_utf8: Encode values as UTF 8.
:param auto_commit: Commit after executing the query.
:param update: Update this dict to all rows (if returned `as_dict`).
:param run: Returns query without executing it if False.
@ -274,20 +271,15 @@ class Database:
if pluck:
return [r[0] for r in self.last_result]
if as_utf8:
deprecation_warning("as_utf8 parameter is deprecated and will be removed in version 15.")
if formatted:
deprecation_warning("formatted parameter is deprecated and will be removed in version 15.")
# scrub output if required
if as_dict:
ret = self.fetch_as_dict(formatted, as_utf8)
ret = self.fetch_as_dict()
if update:
for r in ret:
r.update(update)
return ret
elif as_list or as_utf8:
return self.convert_to_lists(self.last_result, formatted, as_utf8)
elif as_list:
return self.convert_to_lists(self.last_result)
return self.last_result
def _log_query(self, mogrified_query: str, debug: bool = False, explain: bool = False) -> None:
@ -394,62 +386,27 @@ class Database:
):
raise ImplicitCommitError("This statement can cause implicit commit")
def fetch_as_dict(self, formatted=0, as_utf8=0) -> list[frappe._dict]:
def fetch_as_dict(self) -> list[frappe._dict]:
"""Internal. Converts results to dict."""
result = self.last_result
if result:
keys = [column[0] for column in self._cursor.description]
if not as_utf8:
return [frappe._dict(zip(keys, row)) for row in result]
ret = []
for r in result:
values = []
for value in r:
if as_utf8 and isinstance(value, str):
value = value.encode("utf-8")
values.append(value)
ret.append(frappe._dict(zip(keys, values)))
return ret
return [frappe._dict(zip(keys, row)) for row in result]
@staticmethod
def clear_db_table_cache(query):
if query and is_query_type(query, ("drop", "create")):
frappe.cache().delete_key("db_tables")
@staticmethod
def needs_formatting(result, formatted):
"""Returns true if the first row in the result has a Date, Datetime, Long Int."""
if result and result[0]:
for v in result[0]:
if isinstance(v, (datetime.date, datetime.timedelta, datetime.datetime, int)):
return True
if formatted and isinstance(v, (int, float)):
return True
return False
def get_description(self):
"""Returns result metadata."""
return self._cursor.description
@staticmethod
def convert_to_lists(res, formatted=0, as_utf8=0):
def convert_to_lists(res):
"""Convert tuple output to lists (internal)."""
if not as_utf8:
return [[value for value in row] for row in res]
nres = []
for r in res:
nr = []
for val in r:
if as_utf8 and isinstance(val, str):
val = val.encode("utf-8")
nr.append(val)
nres.append(nr)
return nres
return [[value for value in row] for row in res]
def get(self, doctype, filters=None, as_dict=True, cache=False):
"""Returns `get_value` with fieldname='*'"""
@ -849,11 +806,6 @@ class Database:
).run(debug=debug, run=run, as_dict=as_dict)
return {}
@deprecated
def update(self, *args, **kwargs):
"""Update multiple values. Alias for `set_value`."""
return self.set_value(*args, **kwargs)
def set_value(
self,
dt,
@ -879,7 +831,6 @@ class Database:
:param modified_by: Set this user as `modified_by`.
:param update_modified: default True. Set as false, if you don't want to update the timestamp.
:param debug: Print the query in the developer / js console.
:param for_update: [DEPRECATED] This function now performs updates in single query, locking is not required.
"""
is_single_doctype = not (dn and dt != dn)
to_update = field if isinstance(field, dict) else {field: val}
@ -889,9 +840,6 @@ class Database:
modified_by = modified_by or frappe.session.user
to_update.update({"modified": modified, "modified_by": modified_by})
if for_update:
deprecation_warning("for_update parameter is deprecated and will be removed in v15.")
if is_single_doctype:
frappe.db.delete(
"Singles", filters={"field": ("in", tuple(to_update)), "doctype": dt}, debug=debug
@ -922,32 +870,6 @@ class Database:
if dt in self.value_cache:
del self.value_cache[dt]
@staticmethod
@deprecated
def set(doc, field, val):
"""Set value in document. **Avoid**"""
doc.db_set(field, val)
@deprecated
def touch(self, doctype, docname):
"""Update the modified timestamp of this document."""
modified = now()
DocType = frappe.qb.DocType(doctype)
frappe.qb.update(DocType).set(DocType.modified, modified).where(DocType.name == docname).run()
return modified
@staticmethod
def set_temp(value):
"""Set a temperory value and return a key."""
key = frappe.generate_hash()
frappe.cache().hset("temp", key, value)
return key
@staticmethod
def get_temp(key):
"""Return the temperory value and delete it."""
return frappe.cache().hget("temp", key)
def set_global(self, key, val, user="__global"):
"""Save a global key value. Global values will be automatically set if they match fieldname."""
self.set_default(key, val, user)
@ -1107,7 +1029,7 @@ class Database:
return getdate(date).strftime("%Y-%m-%d")
@staticmethod
def format_datetime(datetime):
def format_datetime(datetime): # noqa: F811
if not datetime:
return FallBackDateTimeStr
@ -1251,10 +1173,6 @@ class Database:
"""
return self.sql_ddl(f"truncate `{get_table_name(doctype)}`")
@deprecated
def clear_table(self, doctype):
return self.truncate(doctype)
def get_last_created(self, doctype):
last_record = self.get_all(doctype, ("creation"), limit=1, order_by="creation desc")
if last_record:

4
frappe/database/db_manager.py
View file

@ -51,12 +51,12 @@ class DbManager:
@staticmethod
def restore_database(target, source, user, password):
import os
from distutils.spawn import find_executable
from shutil import which
from frappe.utils import make_esc
esc = make_esc("$ ")
pv = find_executable("pv")
pv = which("pv")
if pv:
pipe = f"{pv} {source} |"

10
frappe/desk/doctype/bulk_update/bulk_update.py
View file

@ -3,8 +3,10 @@
import frappe
from frappe import _
from frappe.core.doctype.submission_queue.submission_queue import queue_submission
from frappe.model.document import Document
from frappe.utils import cint
from frappe.utils.scheduler import is_scheduler_inactive
class BulkUpdate(Document):
@ -44,8 +46,12 @@ def submit_cancel_or_update_docs(doctype, docnames, action="submit", data=None):
try:
message = ""
if action == "submit" and doc.docstatus.is_draft():
doc.submit()
message = _("Submitting {0}").format(doctype)
if doc.meta.queue_in_background and not is_scheduler_inactive():
queue_submission(doc, action)
message = _("Queuing {0} for Submission").format(doctype)
else:
doc.submit()
message = _("Submitting {0}").format(doctype)
elif action == "cancel" and doc.docstatus.is_submitted():
doc.cancel()
message = _("Cancelling {0}").format(doctype)

7
frappe/desk/form/save.py
View file

@ -4,8 +4,10 @@
import json
import frappe
from frappe.core.doctype.submission_queue.submission_queue import queue_submission
from frappe.desk.form.load import run_onload
from frappe.monitor import add_data_to_monitor
from frappe.utils.scheduler import is_scheduler_inactive
@frappe.whitelist()
@ -16,8 +18,10 @@ def savedocs(doc, action):
# action
doc.docstatus = {"Save": 0, "Submit": 1, "Update": 1, "Cancel": 2}[action]
if doc.docstatus == 1:
if action == "Submit" and doc.meta.queue_in_background and not is_scheduler_inactive():
queue_submission(doc, action)
return
doc.submit()
else:
doc.save()
@ -27,7 +31,6 @@ def savedocs(doc, action):
send_updated_docs(doc)
add_data_to_monitor(doctype=doc.doctype, action=action)
frappe.msgprint(frappe._("Saved"), indicator="green", alert=True)

2
frappe/email/doctype/newsletter/test_newsletter.py
View file

@ -74,7 +74,7 @@ class TestNewsletterMixin:
).insert(ignore_if_duplicate=True)
except Exception:
frappe.db.rollback(save_point=savepoint)
frappe.db.update(doctype, email_filters, "unsubscribed", 0)
frappe.db.set_value(doctype, email_filters, "unsubscribed", 0)
frappe.db.release_savepoint(savepoint)

9
frappe/email/oauth.py
View file

@ -132,19 +132,18 @@ def oauth_access(email_account: str, service: str):
if not service:
frappe.throw(frappe._("No Service is selected. Please select one and try again!"))
doctype = "Email Account"
if service == "GMail":
return authorize_google_access(email_account, doctype)
return authorize_google_access(email_account)
raise NotImplementedError(f"Service {service} currently doesn't have oauth implementation.")
def authorize_google_access(email_account, doctype: str = "Email Account", code: str = None):
def authorize_google_access(email_account: str, code: str = None):
"""Facilitates google oauth for email.
This is invoked 2 times - first time when user clicks `Authorze API Access` for getting the authorization url
This is invoked 2 times - first time when user clicks `Authorize API Access` for getting the authorization url
and second time for setting the refresh and access token in db when google redirects back with oauth code."""
doctype = "Email Account"
oauth_obj = GoogleOAuth("mail")
if not code:

7
frappe/model/base_document.py
View file

@ -100,12 +100,7 @@ class BaseDocument:
if d.get("doctype"):
self.doctype = d["doctype"]
self._table_fieldnames = (
d["_table_fieldnames"] # from cache
if "_table_fieldnames" in d
else {df.fieldname for df in self._get_table_fields()}
)
self._table_fieldnames = {df.fieldname for df in self._get_table_fields()}
self.update(d)
self.dont_update_if_missing = []

9
frappe/model/document.py
View file

@ -120,6 +120,10 @@ class Document(BaseDocument):
# incorrect arguments. let's not proceed.
raise ValueError("Illegal arguments")
@property
def is_locked(self):
return file_lock.lock_exists(self.get_signature())
@staticmethod
def whitelist(fn):
"""Decorator: Whitelist method to be called remotely via REST API."""
@ -300,6 +304,10 @@ class Document(BaseDocument):
follow_document(self.doctype, self.name, frappe.session.user)
return self
def check_if_locked(self):
if self.creation and self.is_locked:
raise frappe.DocumentLockedError
def save(self, *args, **kwargs):
"""Wrapper for _save"""
return self._save(*args, **kwargs)
@ -326,6 +334,7 @@ class Document(BaseDocument):
if self.get("__islocal") or not self.get("name"):
return self.insert()
self.check_if_locked()
self.check_permission("write", "save")
self.set_user_and_timestamp()

2
frappe/patches/v13_0/queryreport_columns.py
View file

@ -15,4 +15,4 @@ def execute():
if isinstance(data, list):
# double escape braces
jstr = f'{{"columns":{jstr}}}'
frappe.db.update("Report", record["name"], "json", jstr)
frappe.db.set_value("Report", record["name"], "json", jstr)

2
frappe/patches/v14_0/remove_db_aggregation.py
View file

@ -32,4 +32,4 @@ def execute():
for agg in ["avg", "max", "min", "sum"]:
script = re.sub(f"frappe.db.{agg}\\(", f"frappe.qb.{agg}(", script)
frappe.db.update("Server Script", name, "script", script)
frappe.db.set_value("Server Script", name, "script", script)

78
frappe/public/js/frappe/form/form.js
View file

@ -445,6 +445,7 @@ frappe.ui.form.Form = class FrappeForm {
.toggleClass("cancelled-form", this.doc.docstatus === 2);
this.show_conflict_message();
this.show_submission_queue_banner();
if (frappe.boot.read_only) {
this.disable_form();
@ -2031,6 +2032,83 @@ frappe.ui.form.Form = class FrappeForm {
.filter((user) => !["Administrator", frappe.session.user].includes(user))
.filter(Boolean);
}
show_submission_queue_banner() {
let wrapper = this.layout.wrapper.find(".submission-queue-banner");
if (
!(
this.meta.is_submittable &&
this.meta.queue_in_background &&
!this.doc.__islocal &&
this.doc.docstatus === 0
)
) {
if (wrapper.length) {
wrapper.hide();
wrapper.html("");
}
return;
}
if (!wrapper.length) {
wrapper = $('<div class="submission-queue-banner form-message yellow">');
this.layout.wrapper.prepend(wrapper);
}
frappe
.call({
method: "frappe.core.doctype.submission_queue.submission_queue.get_latest_submissions",
args: { doctype: this.doctype, docname: this.docname },
})
.then((r) => {
if (r.message.latest_submission) {
// if we are here that means some submission(s) were queued and are in queued/failed state
let col_width = 4;
let failed_link = "";
let submission_label = __("Previous Submission");
if (r.message.latest_failed_submission) {
if (r.message.latest_failed_submission !== r.message.latest_submission) {
col_width = 3;
failed_link = `<div class="col-md-3">
<a href='/app/submission-queue/${r.message.latest_failed_submission}'>${__(
"Previous Falied Submission"
)}</a>
</div>`;
} else {
submission_label = __("Previous Falied Submission");
}
}
let html = `
<div class="row">
<div class="col-md-${col_width}">
<strong>${__("Submission Status:")}</strong>
</div>
<div class="col-md-${col_width}">
<a href='/app/submission-queue/${r.message.latest_submission}'>${submission_label}</a>
</div>
${failed_link}
<div class="col-md-${col_width}">
<a href='/app/submission-queue?ref_doctype=${encodeURIComponent(
this.doctype
)}&ref_docname=${encodeURIComponent(this.docname)}'>${__(
"All Submissions"
)}</a>
</div>
</div>
`;
wrapper.show();
wrapper.html(html);
} else {
wrapper.hide();
wrapper.html("");
}
});
}
};
frappe.validated = 0;

4
frappe/templates/includes/navbar/navbar_search.html
View file

@ -2,8 +2,8 @@
<li>
<form action='/search'>
<input name='q' class='form-control navbar-search' type='text'
value='{{ frappe.form_dict.q or ''}}'
value='{{ frappe.form_dict.q|e or ''}}'
{% if not frappe.form_dict.q%}placeholder="{{ _("Search...") }}"{% endif %}>
</form>
</li>
{% endif %}
{% endif %}

17
frappe/tests/test_db.py
View file

@ -791,23 +791,6 @@ class TestDBSetValue(FrappeTestCase):
cached_doc = frappe.get_cached_doc(self.todo2.doctype, self.todo2.name)
self.assertEqual(cached_doc.description, description)
def test_update_alias(self):
args = (self.todo1.doctype, self.todo1.name, "description", "Updated by `test_update_alias`")
kwargs = {
"for_update": False,
"modified": None,
"modified_by": None,
"update_modified": True,
"debug": False,
}
self.assertTrue("return self.set_value(" in inspect.getsource(frappe.db.update))
with patch.object(Database, "set_value") as set_value:
frappe.db.update(*args, **kwargs)
set_value.assert_called_once()
set_value.assert_called_with(*args, **kwargs)
@classmethod
def tearDownClass(cls):
frappe.db.rollback()

20
frappe/tests/test_document_locks.py
View file

@ -16,3 +16,23 @@ class TestDocumentLocks(FrappeTestCase):
todo_1.lock()
self.assertRaises(frappe.DocumentLockedError, todo.lock)
todo_1.unlock()
def test_operations_on_locked_documents(self):
todo = frappe.get_doc(dict(doctype="ToDo", description="testing operations")).insert()
todo.lock()
with self.assertRaises(frappe.DocumentLockedError):
todo.description = "Random"
todo.save()
# Checking for persistant locks across all instances.
doc = frappe.get_doc("ToDo", todo.name)
self.assertEqual(doc.is_locked, True)
with self.assertRaises(frappe.DocumentLockedError):
doc.description = "Random"
doc.save()
doc.unlock()
self.assertEqual(doc.is_locked, False)
self.assertEqual(todo.is_locked, False)

14
frappe/tests/test_email.py
View file

@ -310,6 +310,20 @@ class TestEmail(FrappeTestCase):
email_account.enable_incoming = False
class TestVerifiedRequests(FrappeTestCase):
def test_round_trip(self):
from frappe.utils import set_request
from frappe.utils.verified_command import get_signed_params, verify_request
test_cases = [{"xyz": "abc"}, {"email": "a@b.com", "user": "xyz"}]
for params in test_cases:
signed_url = get_signed_params(params)
set_request(method="GET", path="?" + signed_url)
self.assertTrue(verify_request())
frappe.local.request = None
if __name__ == "__main__":
import unittest

27
frappe/tests/test_zbg_job_sanity_test.py
View file

@ -1,27 +0,0 @@
""" smoak tests to check that all registered background jobs execute without error.
Note: Filename is intentional to run this test roughly at end. Don't change."""
import time
import frappe
from frappe.core.doctype.rq_job.rq_job import RQJob, remove_failed_jobs
from frappe.tests.utils import FrappeTestCase, timeout
class TestScheduledJobSanity(FrappeTestCase):
def setUp(self):
remove_failed_jobs()
@timeout(90)
def test_bg_jobs_run(self):
"""Enqueue all scheduled jobs, wait for finish and verify that none failed."""
for scheduled_job_type in frappe.get_all("Scheduled Job Type", pluck="name"):
frappe.get_doc("Scheduled Job Type", scheduled_job_type).enqueue(force=True)
while RQJob.get_list({"filters": [["RQ Job", "status", "in", ("queued", "started")]]}):
time.sleep(0.5)
# Check no failed, if failed print full details
failed_jobs = RQJob.get_list({"filters": [["RQ Job", "status", "=", "failed"]]})
self.assertEqual(len(failed_jobs), 0, "Jobs failed: " + str(failed_jobs))

4
frappe/utils/background_jobs.py
View file

@ -52,6 +52,8 @@ def enqueue(
method,
queue="default",
timeout=None,
on_success=None,
on_failure=None,
event=None,
is_async=True,
job_name=None,
@ -116,6 +118,8 @@ def enqueue(
return q.enqueue_call(
execute_job,
on_success=on_success,
on_failure=on_failure,
timeout=timeout,
kwargs=queue_args,
at_front=at_front,

1
frappe/utils/html_utils.py
View file

@ -277,6 +277,7 @@ acceptable_elements = [
"li",
"m",
"map",
"mark",
"menu",
"meter",
"multicol",

9
frappe/utils/safe_exec.py
View file

@ -269,12 +269,15 @@ def call_with_form_dict(function, kwargs):
@contextmanager
def patched_qb():
require_patching = isinstance(frappe.qb.terms, types.ModuleType)
try:
_terms = frappe.qb.terms
frappe.qb.terms = _flatten(frappe.qb.terms)
if require_patching:
_terms = frappe.qb.terms
frappe.qb.terms = _flatten(frappe.qb.terms)
yield
finally:
frappe.qb.terms = _terms
if require_patching:
frappe.qb.terms = _terms
@lru_cache

23
frappe/utils/verified_command.py
View file

@ -16,15 +16,14 @@ def get_signed_params(params):
if not isinstance(params, str):
params = urlencode(params)
signature = hmac.new(params.encode(), digestmod=hashlib.md5)
signature.update(get_secret().encode())
return params + "&_signature=" + signature.hexdigest()
signature = _sign_message(params)
return params + "&_signature=" + signature
def get_secret():
return frappe.local.conf.get("secret") or str(
frappe.db.get_value("User", "Administrator", "creation")
)
from frappe.utils.password import get_encryption_key
return frappe.local.conf.get("secret") or get_encryption_key()
def verify_request():
@ -35,12 +34,10 @@ def verify_request():
signature_string = "&_signature="
if signature_string in query_string:
params, signature = query_string.split(signature_string)
params, given_signature = query_string.split(signature_string)
given_signature = hmac.new(params.encode("utf-8"), digestmod=hashlib.md5)
given_signature.update(get_secret().encode())
valid_signature = signature == given_signature.hexdigest()
computed_signature = _sign_message(params)
valid_signature = hmac.compare_digest(given_signature, computed_signature)
valid_method = frappe.request.method == "GET"
valid_request_data = not (frappe.request.form or frappe.request.data)
@ -55,6 +52,10 @@ def verify_request():
return False
def _sign_message(message: str) -> str:
return hmac.new(get_secret().encode(), message.encode(), digestmod=hashlib.sha512).hexdigest()
def get_url(cmd, params, nonce=None, secret=None):
if not nonce:
nonce = params

26
package.json
View file

@ -22,17 +22,25 @@
"homepage": "https://frappeframework.com",
"dependencies": {
"@editorjs/editorjs": "2.20.0",
"@frappe/esbuild-plugin-postcss2": "^0.1.3",
"@vueuse/core":"^9.5.0",
"@vue/component-compiler": "^4.2.4",
"ace-builds": "^1.4.8",
"air-datepicker": "github:frappe/air-datepicker",
"autoprefixer": "10",
"awesomplete": "^1.1.5",
"bootstrap": "4.5.0",
"chalk": "^2.3.2",
"cliui": "^7.0.4",
"cookie": "^0.4.0",
"cropperjs": "^1.5.12",
"cssnano": "^5.0.0",
"driver.js": "^0.9.8",
"editorjs-undo": "0.1.6",
"esbuild": "^0.14.29",
"esbuild-plugin-vue3": "^0.3.0",
"fast-deep-equal": "^2.0.1",
"fast-glob": "^3.2.5",
"frappe-charts": "2.0.0-rc22",
"frappe-datatable": "^1.16.4",
"frappe-gantt": "^0.6.0",
@ -41,16 +49,21 @@
"jquery": "3.6.0",
"js-sha256": "^0.9.0",
"jsbarcode": "^3.9.0",
"launch-editor": "^2.2.1",
"localforage": "^1.9.0",
"md5": "^2.3.0",
"moment": "^2.29.4",
"moment-timezone": "^0.5.35",
"pinia": "^2.0.23",
"plyr": "^3.7.2",
"popper.js": "^1.16.0",
"postcss": "8",
"quill": "2.0.0-dev.4",
"quill-image-resize": "^3.0.9",
"quill-magic-url": "^3.0.0",
"qz-tray": "^2.0.8",
"redis": "^3.1.1",
"rtlcss": "^3.2.1",
"sass": "^1.53.0",
"showdown": "^2.1.0",
"snyk": "^1.996.0",
@ -63,19 +76,6 @@
"vue-router": "^4.1.5",
"vuedraggable": "^4.1.0",
"vuex": "4.0.2",
"pinia": "^2.0.23",
"@frappe/esbuild-plugin-postcss2": "^0.1.3",
"@vue/component-compiler": "^4.2.4",
"autoprefixer": "10",
"chalk": "^2.3.2",
"cliui": "^7.0.4",
"esbuild": "^0.14.29",
"esbuild-plugin-vue3": "^0.3.0",
"fast-glob": "^3.2.5",
"launch-editor": "^2.2.1",
"md5": "^2.3.0",
"postcss": "8",
"rtlcss": "^3.2.1",
"yargs": "^17.5.1"
},
"snyk": true,

18
yarn.lock
View file

@ -559,20 +559,10 @@ caniuse-api@^3.0.0:
lodash.memoize "^4.1.2"
lodash.uniq "^4.5.0"
caniuse-lite@^1.0.0, caniuse-lite@^1.0.30001219, caniuse-lite@^1.0.30001358:
version "1.0.30001359"
resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001359.tgz#a1c1cbe1c2da9e689638813618b4219acbd4925e"
integrity sha512-Xln/BAsPzEuiVLgJ2/45IaqD9jShtk3Y33anKb4+yLwQzws3+v6odKfpgES/cDEaZMLzSChpIGdbOYtH9MyuHw==
caniuse-lite@^1.0.30001196:
version "1.0.30001296"
resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001296.tgz"
integrity sha512-WfrtPEoNSoeATDlf4y3QvkwiELl9GyPLISV5GejTbbQRtQx4LhsXmc9IQ6XCL2d7UxCyEzToEZNMeqR79OUw8Q==
caniuse-lite@^1.0.30001297, caniuse-lite@^1.0.30001313:
version "1.0.30001316"
resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001316.tgz#b44a1f419f82d2e119aa0bbdab5ec15471796358"
integrity sha512-JgUdNoZKxPZFzbzJwy4hDSyGuH/gXz2rN51QmoR8cBQsVo58llD3A0vlRKKRt8FGf5u69P9eQyIH8/z9vN/S0Q==
caniuse-lite@^1.0.0, caniuse-lite@^1.0.30001196, caniuse-lite@^1.0.30001219, caniuse-lite@^1.0.30001297, caniuse-lite@^1.0.30001313, caniuse-lite@^1.0.30001358:
version "1.0.30001431"
resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001431.tgz"
integrity sha512-zBUoFU0ZcxpvSt9IU66dXVT/3ctO1cy4y9cscs1szkPlcWb6pasYM144GqrUygUbT+k7cmUCW61cvskjcv0enQ==
chalk@^1.1.3:
version "1.1.3"