Merge branch 'develop' into pass-strength-progress-bar

This commit is contained in:
Shariq Ansari 2023-01-23 16:29:47 +05:30 committed by GitHub
commit bbcdd3102e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 306 additions and 230 deletions

View file

@ -1030,6 +1030,16 @@ def make_app(destination, app_name, no_git=False):
make_boilerplate(destination, app_name, no_git=no_git)
@click.command("create-patch")
def create_patch():
"Creates a new patch interactively"
from frappe.utils.boilerplate import PatchCreator
pc = PatchCreator()
pc.fetch_user_inputs()
pc.create_patch_file()
@click.command("set-config")
@click.argument("key")
@click.argument("value")
@ -1176,6 +1186,7 @@ commands = [
data_import,
import_doc,
make_app,
create_patch,
mariadb,
postgres,
request,

View file

@ -604,6 +604,7 @@
{
"default": "0",
"depends_on": "eval: doc.is_submittable",
"description": "Enabling this will submit documents in background",
"fieldname": "queue_in_background",
"fieldtype": "Check",
"label": "Queue in Background"
@ -707,7 +708,7 @@
"link_fieldname": "reference_doctype"
}
],
"modified": "2022-12-14 09:47:27.315351",
"modified": "2023-01-04 17:23:09.206018",
"modified_by": "Administrator",
"module": "Core",
"name": "DocType",
@ -744,4 +745,4 @@
"states": [],
"track_changes": 1,
"translated_doctype": 1
}
}

View file

@ -3,11 +3,17 @@
frappe.ui.form.on("Submission Queue", {
refresh: function (frm) {
if (frm.doc.status === "Queued" && frm.doc.job_id) {
if (frm.doc.status === "Queued" && frappe.boot.user.roles.includes("System Manager")) {
frm.add_custom_button(__("Unlock Reference Document"), () => {
frappe.confirm(__("Are you sure you want to go ahead with this action?"), () => {
frm.call("unlock_doc");
});
frappe.confirm(
`
Are you sure you want to go ahead with this action?
Doing this could unlock other submissions of this document which are in queue (if present)
and could lead to non-ideal conditions.`,
() => {
frm.call("unlock_doc");
}
);
});
}
},

View file

@ -20,8 +20,9 @@
"fields": [
{
"fieldname": "job_id",
"fieldtype": "Data",
"fieldtype": "Link",
"label": "Job Id",
"options": "RQ Job",
"read_only": 1
},
{
@ -80,14 +81,14 @@
},
{
"fieldname": "exception",
"fieldtype": "Text",
"fieldtype": "Long Text",
"label": "Exception",
"read_only": 1
}
],
"index_web_pages_for_search": 1,
"links": [],
"modified": "2022-11-12 16:48:37.797232",
"modified": "2023-01-23 12:45:53.997708",
"modified_by": "Administrator",
"module": "Core",
"name": "Submission Queue",
@ -102,6 +103,11 @@
"report": 1,
"role": "System Manager",
"share": 1
},
{
"if_owner": 1,
"read": 1,
"role": "All"
}
],
"sort_field": "modified",

View file

@ -4,8 +4,6 @@
from urllib.parse import quote
from rq import get_current_job
from rq.exceptions import NoSuchJobError
from rq.job import Job
import frappe
from frappe import _
@ -13,7 +11,6 @@ from frappe.desk.doctype.notification_log.notification_log import enqueue_create
from frappe.model.document import Document
from frappe.monitor import add_data_to_monitor
from frappe.utils import now, time_diff_in_seconds
from frappe.utils.background_jobs import get_redis_conn
from frappe.utils.data import cint
@ -39,6 +36,7 @@ class SubmissionQueue(Document):
frappe.db.delete(table, filters=(table.modified < (Now() - Interval(days=days))))
def insert(self, to_be_queued_doc: Document, action: str):
self.status = "Queued"
self.to_be_queued_doc = to_be_queued_doc
self.action_for_queuing = action
super().insert(ignore_permissions=True)
@ -70,6 +68,7 @@ class SubmissionQueue(Document):
def background_submission(self, to_be_queued_doc: Document, action_for_queuing: str):
# Set the job id for that submission doctype
self.update_job_id(get_current_job().id)
_action = action_for_queuing.lower()
if _action == "update":
_action = "submit"
@ -85,7 +84,7 @@ class SubmissionQueue(Document):
)
values = {"status": "Finished"}
except Exception:
values = {"status": "Failed", "exception": frappe.get_traceback()}
values = {"status": "Failed", "exception": frappe.get_traceback(with_context=True)}
frappe.db.rollback()
values["ended_at"] = now()
@ -96,22 +95,27 @@ class SubmissionQueue(Document):
if submission_status == "Failed":
doctype = self.doctype
docname = self.name
message = _("Submission of {0} {1} with action {2} failed")
message = _("Action {0} failed on {1} {2}. View it {3}")
else:
doctype = self.ref_doctype
docname = self.ref_docname
message = _("Submission of {0} {1} with action {2} completed successfully")
message = _("Action {0} completed successfully on {1} {2}. View it {3}")
message = message.format(
frappe.bold(str(self.ref_doctype)), frappe.bold(self.ref_docname), frappe.bold(action)
message_replacements = (
frappe.bold(action),
frappe.bold(str(self.ref_doctype)),
frappe.bold(str(self.ref_docname)),
)
time_diff = time_diff_in_seconds(now(), self.created_at)
if cint(time_diff) <= 60:
frappe.publish_realtime(
"msgprint",
{
"message": message
+ f". View it <a href='/app/{quote(doctype.lower().replace(' ', '-'))}/{quote(docname)}'><b>here</b></a>",
"message": message.format(
*message_replacements,
f"<a href='/app/{quote(doctype.lower().replace(' ', '-'))}/{quote(docname)}'><b>here</b></a>",
),
"alert": True,
"indicator": "red" if submission_status == "Failed" else "green",
},
@ -122,50 +126,27 @@ class SubmissionQueue(Document):
"type": "Alert",
"document_type": doctype,
"document_name": docname,
"subject": message,
"subject": message.format(*message_replacements, "here"),
}
notify_to = frappe.db.get_value("User", self.enqueued_by, fieldname="email")
enqueue_create_notification([notify_to], notification_doc)
def _unlock_reference_doc(self):
"""
Only execute if self.job_id is defined.
"""
try:
job = Job.fetch(self.job_id, connection=get_redis_conn())
status = job.get_status(refresh=True)
exc = job.exc_info
except NoSuchJobError:
exc = None
status = "failed"
if status in ("queued", "started"):
frappe.msgprint(_("Document in queue for execution!"))
return
self.queued_doc.unlock()
values = (
{"status": "Finished"} if status == "finished" else {"status": "Failed", "exception": exc}
)
frappe.db.set_value(self.doctype, self.name, values, update_modified=False)
frappe.msgprint(_("Document Unlocked"))
@frappe.whitelist()
def unlock_doc(self):
# NOTE: this can lead to some weird unlocking/locking behaviours.
# for example: hitting unlock on a submission could lead to unlocking of another submission
# of the same reference document.
if self.status != "Queued" and not self.job_id:
if self.status != "Queued":
return
self._unlock_reference_doc()
self.queued_doc.unlock()
frappe.msgprint(_("Document Unlocked"))
def queue_submission(doc: Document, action: str, alert: bool = True):
queue = frappe.new_doc("Submission Queue")
queue.state = "Queued"
queue.ref_doctype = doc.doctype
queue.ref_docname = doc.name
queue.insert(doc, action)
@ -185,9 +166,25 @@ def get_latest_submissions(doctype, docname):
# NOTE: not used creation as orderby intentianlly as we have used update_modified=False everywhere
# hence assuming modified will be equal to creation for submission queue documents
dt = "Submission Queue"
filters = {"ref_doctype": doctype, "ref_docname": docname}
return {
"latest_submission": frappe.db.get_value(dt, filters),
"latest_failed_submission": frappe.db.get_value(dt, filters | {"status": "Failed"}),
}
latest_submission = frappe.db.get_value(
"Submission Queue",
filters={"ref_doctype": doctype, "ref_docname": docname},
fieldname=["name", "exception", "status"],
)
out = None
if latest_submission:
out = {
"latest_submission": latest_submission[0],
"exc": format_tb(latest_submission[1]),
"status": latest_submission[2],
}
return out
def format_tb(traceback: str | None = None):
if not traceback:
return
return traceback.strip().split("\n")[-1]

View file

@ -59,8 +59,8 @@ class NamingSeries:
if not NAMING_SERIES_PATTERN.match(self.series):
frappe.throw(
_(
'Special Characters except "-", "#", ".", "/", "{" and "}" not allowed in naming series',
),
"Special Characters except '-', '#', '.', '/', '{{' and '}}' not allowed in naming series {0}"
).format(frappe.bold(self.series)),
exc=InvalidNamingSeriesError,
)

View file

@ -2047,16 +2047,12 @@ frappe.ui.form.Form = class FrappeForm {
this.doc.docstatus === 0
)
) {
if (wrapper.length) {
wrapper.hide();
wrapper.html("");
}
wrapper.length && wrapper.remove();
return;
}
if (!wrapper.length) {
wrapper = $('<div class="submission-queue-banner form-message yellow">');
wrapper = $('<div class="submission-queue-banner form-message">');
this.layout.wrapper.prepend(wrapper);
}
@ -2066,49 +2062,40 @@ frappe.ui.form.Form = class FrappeForm {
args: { doctype: this.doctype, docname: this.docname },
})
.then((r) => {
if (r.message.latest_submission) {
if (r.message?.latest_submission) {
// if we are here that means some submission(s) were queued and are in queued/failed state
let col_width = 4;
let failed_link = "";
let submission_label = __("Previous Submission");
let secondary = "";
let div_class = "col-md-12";
if (r.message.latest_failed_submission) {
if (r.message.latest_failed_submission !== r.message.latest_submission) {
col_width = 3;
failed_link = `<div class="col-md-3">
<a href='/app/submission-queue/${r.message.latest_failed_submission}'>${__(
"Previous Falied Submission"
)}</a>
</div>`;
} else {
submission_label = __("Previous Falied Submission");
}
if (r.message.exc) {
secondary = `: <span>${r.message.exc}</span>`;
} else {
div_class = "col-md-6";
secondary = `
</div>
<div class="col-md-6">
<a href='/app/submission-queue?ref_doctype=${encodeURIComponent(
this.doctype
)}&ref_docname=${encodeURIComponent(this.docname)}'>${__(
"All Submissions"
)}</a>
`;
}
let html = `
<div class="row">
<div class="col-md-${col_width}">
<strong>${__("Submission Status:")}</strong>
<div class="row">
<div class="${div_class}">
<a href='/app/submission-queue/${r.message.latest_submission}'>${submission_label} (${r.message.status})</a>${secondary}
</div>
</div>
<div class="col-md-${col_width}">
<a href='/app/submission-queue/${r.message.latest_submission}'>${submission_label}</a>
</div>
${failed_link}
<div class="col-md-${col_width}">
<a href='/app/submission-queue?ref_doctype=${encodeURIComponent(
this.doctype
)}&ref_docname=${encodeURIComponent(this.docname)}'>${__(
"All Submissions"
)}</a>
</div>
</div>
`;
`;
wrapper.show();
wrapper.removeClass("red").removeClass("yellow");
wrapper.addClass(r.message.status == "Failed" ? "red" : "yellow");
wrapper.html(html);
} else {
wrapper.hide();
wrapper.html("");
wrapper.remove();
}
});
}

View file

@ -163,10 +163,12 @@ $input-height: 28px !default;
--bg-green: var(--dark-green-50);
--bg-yellow: var(--yellow-50);
--bg-orange: var(--orange-50);
--bg-red: var(--red-50);
--bg-red: var(--red-100);
--bg-gray: var(--gray-200);
--bg-grey: var(--gray-200);
--bg-light-gray: var(--gray-100);
--bg-dark-gray: var(--gray-900);
--bg-dark-gray: var(--gray-400);
--bg-darkgrey: var(--gray-400);
--bg-purple: var(--purple-100);
--bg-pink: var(--pink-50);
--bg-cyan: var(--cyan-50);
@ -186,12 +188,15 @@ $input-height: 28px !default;
--text-on-dark-blue: var(--blue-800);
--text-on-green: var(--dark-green-700);
--text-on-yellow: var(--yellow-700);
--text-on-orange: var(--orange-600);
--text-on-orange: var(--orange-700);
--text-on-red: var(--red-600);
--text-on-gray: var(--gray-600);
--text-on-gray: var(--gray-700);
--text-on-grey: var(--gray-700);
--text-on-darkgrey: var(--gray-800);
--text-on-dark-gray: var(--gray-800);
--text-on-light-gray: var(--gray-800);
--text-on-purple: var(--purple-700);
--text-on-pink: var(--pink-600);
--text-on-pink: var(--pink-700);
--text-on-cyan: var(--cyan-800);
// alert colors

View file

@ -1,19 +1,3 @@
@mixin indicator-pill-color($color) {
background: var(--bg-#{$color});
color: var(--text-on-#{$color});
&::before,
&::after {
background: var(--text-on-#{$color});
}
}
@mixin indicator-color($color) {
&::before,
&::after {
background: var(--text-on-#{$color});
}
}
.indicator,
.indicator-pill,
.indicator-pill-right,
@ -67,111 +51,30 @@
margin: 0 0 0 4px;
}
.indicator.green {
@include indicator-color('green');
$indicator-colors: green, cyan, blue, orange, yellow, gray, grey, red, pink, darkgrey, purple, light-blue;
@each $color in $indicator-colors {
.indicator.#{"" + $color} {
&::before,
&::after {
background: var(--indicator-dot-#{$color});
}
}
.indicator-pill.#{"" + $color},
.indicator-pill-right.#{"" + $color},
.indicator-pill-round.#{"" + $color} {
background: var(--bg-#{$color});
color: var(--text-on-#{$color});
&::before,
&::after {
background: var(--text-on-#{$color});
}
}
.indicator {
--indicator-dot-#{"" + $color}: var(--text-on-#{$color});
}
}
.indicator-pill.green,
.indicator-pill-right.green,
.indicator-pill-round.green {
@include indicator-pill-color('green');
}
.indicator.cyan {
@include indicator-color('cyan');
}
.indicator-pill.cyan,
.indicator-pill-right.cyan,
.indicator-pill-round.cyan {
@include indicator-pill-color('cyan');
}
.indicator.blue {
@include indicator-color('blue');
}
.indicator-pill.blue,
.indicator-pill-right.blue,
.indicator-pill-round.blue {
@include indicator-pill-color('blue');
}
.indicator.orange {
@include indicator-color('orange');
}
.indicator-pill.orange,
.indicator-pill-right.orange
.indicator-pill-round.orange {
@include indicator-pill-color('orange');
}
.indicator.yellow {
@include indicator-color('yellow');
}
.indicator-pill.yellow,
.indicator-pill-right.yellow
.indicator-pill-round.yellow {
@include indicator-pill-color('yellow');
}
.indicator.gray,
.indicator.grey {
@include indicator-color('gray');
}
.indicator-pill.gray,
.indicator-pill-right.gray,
.indicator-pill-round.gray,
.indicator-pill.grey,
.indicator-pill-right.grey,
.indicator-pill-round.grey {
@include indicator-pill-color('light-gray');
}
.indicator.red {
@include indicator-color('red');
}
.indicator-pill.red,
.indicator-pill-right.red,
.indicator-pill-round.red {
@include indicator-pill-color('red');
}
.indicator.pink {
@include indicator-color('pink');
}
.indicator-pill.pink,
.indicator-pill-right.pink,
.indicator-pill-round.pink {
@include indicator-pill-color('pink');
}
.indicator-pill.darkgrey,
.indicator-pill-right.darkgrey,
.indicator-pill-round.darkgrey {
@include indicator-pill-color('gray');
}
.indicator-pill.purple,
.indicator-pill-right.purple,
.indicator-pill-round.purple {
@include indicator-pill-color('purple');
}
.indicator.light-blue {
@include indicator-color('light-blue');
}
.indicator-pill.light-blue,
.indicator-pill-right.light-blue,
.indicator-pill-round.light-blue {
@include indicator-pill-color('light-blue');
}
.indicator.blink {
animation: blink 1s linear infinite;

View file

@ -47,27 +47,36 @@
// Background Text Color Pairs
--bg-blue: var(--blue-600);
--bg-light-blue: var(--blue-400);
--bg-light-blue: var(--blue-600);
--bg-dark-blue: var(--blue-900);
--bg-green: var(--dark-green-500);
--bg-yellow: var(--yellow-500);
--bg-orange: var(--orange-500);
--bg-red: var(--red-500);
--bg-gray: var(--gray-600);
--bg-green: var(--green-800);
--bg-yellow: var(--yellow-700);
--bg-orange: var(--orange-700);
--bg-red: var(--red-600);
--bg-gray: var(--gray-400);
--bg-grey: var(--gray-400);
--bg-darkgrey: var(--gray-600);
--bg-dark-gray: var(--gray-600);
--bg-light-gray: var(--gray-700);
--bg-dark-gray: var(--gray-300);
--bg-purple: var(--purple-600);
--bg-purple: var(--purple-700);
--bg-pink: var(--pink-700);
--bg-cyan: var(--cyan-800);
--text-on-blue: var(--blue-50);
--text-on-light-blue: var(--blue-100);
--text-on-light-blue: var(--blue-50);
--text-on-dark-blue: var(--blue-300);
--text-on-green: var(--dark-green-50);
--text-on-yellow: var(--yellow-50);
--text-on-orange: var(--orange-100);
--text-on-red: var(--red-50);
--text-on-gray: var(--gray-300);
--text-on-gray: var(--gray-50);
--text-on-grey: var(--gray-50);
--text-on-darkgrey: var(--gray-200);
--text-on-dark-gray: var(--gray-200);
--text-on-light-gray: var(--gray-100);
--text-on-purple: var(--purple-100);
--text-on-pink: var(--pink-100);
--text-on-cyan: var(--cyan-100);
// alert colors
--alert-text-danger: var(--red-300);
@ -190,4 +199,11 @@
color: var(--text-color);
background: var(--gray-500);
}
$indicator-colors: green, cyan, blue, orange, yellow, gray, grey, red, pink, darkgrey, purple, light-blue;
@each $color in $indicator-colors {
.indicator {
--indicator-dot-#{"" + $color}: var(--bg-#{$color});
}
}
}

View file

@ -2,22 +2,25 @@ import ast
import copy
import glob
import os
import pathlib
import shutil
import unittest
from io import StringIO
from unittest.mock import patch
import yaml
import frappe
from frappe.tests.utils import FrappeTestCase
from frappe.modules.patch_handler import get_all_patches
from frappe.utils.boilerplate import (
PatchCreator,
_create_app_boilerplate,
_get_user_inputs,
github_workflow_template,
)
class TestBoilerPlate(FrappeTestCase):
class TestBoilerPlate(unittest.TestCase):
@classmethod
def setUpClass(cls):
super().setUpClass()
@ -180,3 +183,30 @@ class TestBoilerPlate(FrappeTestCase):
ast.parse(p.read())
except Exception as e:
self.fail(f"Can't parse python file in new app: {python_file}\n" + str(e))
def test_new_patch_util(self):
user_inputs = {
"app_name": "frappe",
"doctype": "User",
"docstring": "Delete all users",
"file_name": "", # Accept default
"patch_folder_confirmation": "Y",
}
patches_txt = pathlib.Path(pathlib.Path(frappe.get_app_path("frappe", "patches.txt")))
original_patches = patches_txt.read_text()
with patch("sys.stdin", self.get_user_input_stream(user_inputs)):
patch_creator = PatchCreator()
patch_creator.fetch_user_inputs()
patch_creator.create_patch_file()
patches = get_all_patches()
expected_patch = "frappe.core.doctype.user.patches.delete_all_users"
self.assertIn(expected_patch, patches)
self.assertTrue(patch_creator.patch_file.exists())
# Cleanup
shutil.rmtree(patch_creator.patch_file.parents[0])
patches_txt.write_text(original_patches)

View file

@ -1,9 +1,13 @@
# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors
# License: MIT. See LICENSE
import contextlib
import glob
import json
import os
import pathlib
import re
import textwrap
import click
import git
@ -162,6 +166,113 @@ def _create_github_workflow_files(dest, hooks):
f.write(github_workflow_template.format(**hooks))
PATCH_TEMPLATE = textwrap.dedent(
'''
import frappe
def execute():
"""{docstring}"""
# Write your patch here.
pass
'''
)
class PatchCreator:
def __init__(self):
self.all_apps = frappe.get_all_apps(sites_path=".", with_internal_apps=False)
self.app = None
self.app_dir = None
self.patch_dir = None
self.filename = None
self.docstring = None
self.patch_file = None
def fetch_user_inputs(self):
self._ask_app_name()
self._ask_doctype_name()
self._ask_patch_meta_info()
def _ask_app_name(self):
self.app = click.prompt("Select app for new patch", type=click.Choice(self.all_apps))
self.app_dir = pathlib.Path(frappe.get_app_path(self.app))
def _ask_doctype_name(self):
def _doctype_name(filename):
with contextlib.suppress(Exception):
with open(filename) as f:
return json.load(f).get("name")
doctype_files = list(glob.glob(f"{self.app_dir}/**/doctype/**/*.json"))
doctype_map = {_doctype_name(file): file for file in doctype_files}
doctype_map.pop(None, None)
doctype = click.prompt(
"Provide DocType name on which this patch will apply",
type=click.Choice(doctype_map.keys()),
show_choices=False,
)
self.patch_dir = pathlib.Path(doctype_map[doctype]).parents[0] / "patches"
def _ask_patch_meta_info(self):
self.docstring = click.prompt("Describe what this patch does", type=str)
default_filename = frappe.scrub(self.docstring) + ".py"
def _valid_filename(name):
if not name:
return
match name.partition("."):
case filename, ".", "py" if filename.isidentifier():
return True
case _:
click.echo(f"{name} is not a valid python file name")
while not _valid_filename(self.filename):
self.filename = click.prompt(
"Provide filename for this patch", type=str, default=default_filename
)
def create_patch_file(self):
self._create_parent_folder_if_not_exists()
self.patch_file = self.patch_dir / self.filename
if self.patch_file.exists():
raise Exception(f"Patch {self.patch_file} already exists")
*path, _filename = self.patch_file.relative_to(self.app_dir.parents[0]).parts
dotted_path = ".".join(path + [self.patch_file.stem])
patches_txt = self.app_dir / "patches.txt"
existing_patches = patches_txt.read_text()
if dotted_path in existing_patches:
raise Exception(f"Patch {dotted_path} is already present in patches.txt")
self.patch_file.write_text(PATCH_TEMPLATE.format(docstring=self.docstring))
with open(patches_txt, "a+") as f:
if not existing_patches.endswith("\n"):
f.write("\n") # ensure EOF
f.write(dotted_path + "\n")
click.echo(f"Created {self.patch_file} and updated patches.txt")
def _create_parent_folder_if_not_exists(self):
if not self.patch_dir.exists():
click.confirm(
f"Patch folder '{self.patch_dir}' doesn't exist, create it?",
abort=True,
default=True,
)
self.patch_dir.mkdir()
init_py = self.patch_dir / "__init__.py"
init_py.touch()
manifest_template = """include MANIFEST.in
include requirements.txt
include *.json

View file

@ -62,7 +62,10 @@ def get_decrypted_password(doctype, name, fieldname="password", raise_exception=
return decrypt(result[0][0])
elif raise_exception:
frappe.throw(_("Password not found"), frappe.AuthenticationError)
frappe.throw(
_("Password not found for {0} {1} {2}").format(doctype, name, fieldname),
frappe.AuthenticationError,
)
def set_encrypted_password(doctype, name, pwd, fieldname="password"):