vassili/seitime-frappe
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Merge branch 'develop' into qb-fixes

Browse source
This commit is contained in:
Aradhya Tripathi 2022-09-21 02:24:23 +05:30 • committed by GitHub
commit c15da987eb
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
190 changed files with 1952 additions and 4093 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.github/helper/consumer_db/mariadb.json → .github/helper/db/mariadb.json vendored
View file

@ -1,7 +1,7 @@
{
"db_host": "127.0.0.1",
"db_port": 3306,
"db_name": "test_frappe_consumer",
"db_name": "test_frappe",
"db_password": "test_frappe",
"allow_tests": true,
"db_type": "mariadb",

2
.github/helper/consumer_db/postgres.json → .github/helper/db/postgres.json vendored
View file

@ -1,7 +1,7 @@
{
"db_host": "127.0.0.1",
"db_port": 5432,
"db_name": "test_frappe_consumer",
"db_name": "test_frappe",
"db_password": "test_frappe",
"db_type": "postgres",
"allow_tests": true,

37
.github/helper/install.sh vendored
View file

@ -17,37 +17,23 @@ fi
echo "Setting Up Sites & Database..."
mkdir ~/frappe-bench/sites/test_site
cp "${GITHUB_WORKSPACE}/.github/helper/consumer_db/$DB.json" ~/frappe-bench/sites/test_site/site_config.json
if [ "$TYPE" == "server" ]
then
mkdir ~/frappe-bench/sites/test_site_producer
cp "${GITHUB_WORKSPACE}/.github/helper/producer_db/$DB.json" ~/frappe-bench/sites/test_site_producer/site_config.json
fi
cp "${GITHUB_WORKSPACE}/.github/helper/db/$DB.json" ~/frappe-bench/sites/test_site/site_config.json
if [ "$DB" == "mariadb" ]
then
mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "SET GLOBAL character_set_server = 'utf8mb4'"
mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "SET GLOBAL collation_server = 'utf8mb4_unicode_ci'"
mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "SET GLOBAL character_set_server = 'utf8mb4'";
mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "SET GLOBAL collation_server = 'utf8mb4_unicode_ci'";
mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "CREATE DATABASE test_frappe_consumer"
mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "CREATE USER 'test_frappe_consumer'@'localhost' IDENTIFIED BY 'test_frappe_consumer'"
mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "GRANT ALL PRIVILEGES ON \`test_frappe_consumer\`.* TO 'test_frappe_consumer'@'localhost'"
mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "CREATE DATABASE test_frappe";
mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "CREATE USER 'test_frappe'@'localhost' IDENTIFIED BY 'test_frappe'";
mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "GRANT ALL PRIVILEGES ON \`test_frappe\`.* TO 'test_frappe'@'localhost'";
mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "CREATE DATABASE test_frappe_producer"
mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "CREATE USER 'test_frappe_producer'@'localhost' IDENTIFIED BY 'test_frappe_producer'"
mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "GRANT ALL PRIVILEGES ON \`test_frappe_producer\`.* TO 'test_frappe_producer'@'localhost'"
mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "FLUSH PRIVILEGES"
mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "FLUSH PRIVILEGES";
fi
if [ "$DB" == "postgres" ]
then
echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE DATABASE test_frappe_consumer" -U postgres
echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE USER test_frappe_consumer WITH PASSWORD 'test_frappe'" -U postgres
echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE DATABASE test_frappe_producer" -U postgres
echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE USER test_frappe_producer WITH PASSWORD 'test_frappe'" -U postgres
echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE DATABASE test_frappe" -U postgres;
echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE USER test_frappe WITH PASSWORD 'test_frappe'" -U postgres;
fi
echo "Setting Up Procfile..."
@ -78,11 +64,6 @@ fi
bench --site test_site reinstall --yes
if [ "$TYPE" == "server" ]
then
bench --site test_site_producer reinstall --yes
fi
if [ "$TYPE" == "server" ]
then
# wait till assets are built succesfully

16
.github/helper/producer_db/mariadb.json vendored
View file

@ -1,16 +0,0 @@
{
"db_host": "127.0.0.1",
"db_port": 3306,
"db_name": "test_frappe_producer",
"db_password": "test_frappe",
"allow_tests": true,
"db_type": "mariadb",
"auto_email_id": "test@example.com",
"mail_server": "smtp.example.com",
"mail_login": "test@example.com",
"mail_password": "test",
"admin_password": "admin",
"root_login": "root",
"root_password": "travis",
"host_name": "http://test_site_producer:8000"
}

16
.github/helper/producer_db/postgres.json vendored
View file

@ -1,16 +0,0 @@
{
"db_host": "127.0.0.1",
"db_port": 5432,
"db_name": "test_frappe_producer",
"db_password": "test_frappe",
"db_type": "postgres",
"allow_tests": true,
"auto_email_id": "test@example.com",
"mail_server": "smtp.example.com",
"mail_login": "test@example.com",
"mail_password": "test",
"admin_password": "admin",
"root_login": "postgres",
"root_password": "travis",
"host_name": "http://test_site_producer:8000"
}

26
.github/workflows/backport.yml vendored Normal file
View file

@ -0,0 +1,26 @@
name: Backport
on:
pull_request_target:
types:
- closed
- labeled
jobs:
main:
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- name: Checkout Actions
uses: actions/checkout@v2
with:
repository: "frappe/backport"
path: ./actions
ref: develop
- name: Install Actions
run: npm install --production --prefix ./actions
- name: Run backport
uses: ./actions/backport
with:
token: ${{secrets.RELEASE_TOKEN}}
labelsToAdd: "backport"
title: "{{originalTitle}}"

2
.github/workflows/server-mariadb-tests.yml vendored
View file

@ -74,7 +74,6 @@ jobs:
fi
- uses: actions/setup-node@v3
if: ${{ steps.check-build.outputs.build == 'strawberry' }}
with:
node-version: 16
check-latest: true
@ -82,7 +81,6 @@ jobs:
- name: Add to Hosts
run: |
echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts
echo "127.0.0.1 test_site_producer" | sudo tee -a /etc/hosts
- name: Cache pip
uses: actions/cache@v3

1
.github/workflows/server-postgres-tests.yml vendored
View file

@ -84,7 +84,6 @@ jobs:
- name: Add to Hosts
run: |
echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts
echo "127.0.0.1 test_site_producer" | sudo tee -a /etc/hosts
- name: Cache pip
uses: actions/cache@v3

6
.github/workflows/ui-tests.yml vendored
View file

@ -82,7 +82,6 @@ jobs:
- name: Add to Hosts
run: |
echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts
echo "127.0.0.1 test_site_producer" | sudo tee -a /etc/hosts
- name: Cache pip
uses: actions/cache@v3
@ -128,7 +127,10 @@ jobs:
run: cd ~/frappe-bench/ && bench build --apps frappe
- name: Site Setup
run: cd ~/frappe-bench/ && bench --site test_site execute frappe.utils.install.complete_setup_wizard
run: |
cd ~/frappe-bench/
bench --site test_site execute frappe.utils.install.complete_setup_wizard
bench --site test_site execute frappe.tests.ui_test_helpers.create_test_user
- name: UI Tests
run: cd ~/frappe-bench/ && bench --site test_site run-ui-tests frappe --with-coverage --headless --parallel --ci-build-id $GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT

2
.mergify.yml
View file

@ -4,9 +4,9 @@ pull_request_rules:
- and:
- and:
- author!=surajshetty3416
- author!=gavindsouza
- author!=deepeshgarg007
- author!=ankush
- author!=frappe-pr-bot
- author!=mergify[bot]
- or:
- base=version-15

1
CODEOWNERS
View file

@ -7,7 +7,6 @@
templates/ @surajshetty3416
www/ @surajshetty3416
patches/ @surajshetty3416
event_streaming/ @ruchamahabal
data_import* @netchampfaris
core/ @surajshetty3416
workspace @shariquerik

2
README.md
View file

@ -75,3 +75,5 @@ Full-stack web application framework that uses Python and MariaDB on the server
## License
This repository has been released under the [MIT License](LICENSE).
By contributing to Frappe, you agree that your contributions will be licensed under its MIT License.

1
cypress.config.js
View file

@ -3,6 +3,7 @@ const { defineConfig } = require("cypress");
module.exports = defineConfig({
projectId: "92odwv",
adminPassword: "admin",
testUser: "frappe@example.com",
defaultCommandTimeout: 20000,
pageLoadTimeout: 15000,
video: true,

10
cypress/integration/control_link.js
View file

@ -177,14 +177,14 @@ context("Control Link", () => {
cy.intercept("POST", "/api/method/frappe.client.validate_link").as("validate_link");
cy.get(".frappe-control[data-fieldname=assigned_by] input").focus().as("input");
cy.get("@input").type("Administrator", { delay: 100 }).blur();
cy.get("@input").type(cy.config("testUser"), { delay: 100 }).blur();
cy.wait("@validate_link");
cy.get(".frappe-control[data-fieldname=assigned_by_full_name] .control-value").should(
"contain",
"Administrator"
"Frappe"
);
cy.window().its("cur_frm.doc.assigned_by").should("eq", "Administrator");
cy.window().its("cur_frm.doc.assigned_by").should("eq", cy.config("testUser"));
// invalid input
cy.get("@input").clear().type("invalid input", { delay: 100 }).blur();
@ -198,10 +198,10 @@ context("Control Link", () => {
// set valid value again
cy.get("@input").clear().focus();
cy.wait("@search_link");
cy.get("@input").type("Administrator", { delay: 100 }).blur();
cy.get("@input").type(cy.config("testUser"), { delay: 100 }).blur();
cy.wait("@validate_link");
cy.window().its("cur_frm.doc.assigned_by").should("eq", "Administrator");
cy.window().its("cur_frm.doc.assigned_by").should("eq", cy.config("testUser"));
// clear input
cy.get("@input").clear().blur();

9
cypress/integration/control_markdown_editor.js
View file

@ -7,9 +7,12 @@ context("Control Markdown Editor", () => {
it("should allow inserting images by drag and drop", () => {
cy.visit("/app/web-page/new");
cy.fill_field("content_type", "Markdown", "Select");
cy.get_field("main_section_md", "Markdown Editor").attachFile("sample_image.jpg", {
subjectType: "drag-n-drop",
});
cy.get_field("main_section_md", "Markdown Editor").selectFile(
"cypress/fixtures/sample_image.jpg",
{
action: "drag-drop",
}
);
cy.click_modal_primary_button("Upload");
cy.get_field("main_section_md", "Markdown Editor").should(
"contain",

15
cypress/integration/dashboard_links.js
View file

@ -8,7 +8,7 @@ const child_table_doctype_name = child_table_doctype.name;
context("Dashboard links", () => {
before(() => {
cy.visit("/login");
cy.login();
cy.login("Administrator");
cy.insert_doc("DocType", child_table_doctype, true);
cy.insert_doc("DocType", child_table_doctype_1, true);
cy.insert_doc("DocType", doctype_with_child_table, true);
@ -27,8 +27,7 @@ context("Dashboard links", () => {
cy.visit("/app/contact");
cy.clear_filters();
cy.visit("/app/user");
cy.get(".list-row-col > .level-item > .ellipsis").eq(0).click({ force: true });
cy.visit(`/app/user/${cy.config("testUser")}`);
//To check if initially the dashboard contains only the "Contact" link and there is no counter
cy.select_form_tab("Connections");
@ -41,12 +40,11 @@ context("Dashboard links", () => {
cy.findByRole("button", { name: "Add Contact" }).click();
cy.get('[data-doctype="Contact"][data-fieldname="first_name"]').type("Admin");
cy.findByRole("button", { name: "Save" }).click();
cy.visit("/app/user");
cy.get(".list-row-col > .level-item > .ellipsis").eq(0).click({ force: true });
cy.visit(`/app/user/${cy.config("testUser")}`);
//To check if the counter for contact doc is "1" after adding the contact
//To check if the counter for contact doc is "2" after adding additional contact
cy.select_form_tab("Connections");
cy.get('[data-doctype="Contact"] > .count').should("contain", "1");
cy.get('[data-doctype="Contact"] > .count').should("contain", "2");
cy.get('[data-doctype="Contact"]').contains("Contact").click();
//Deleting the newly created contact
@ -64,8 +62,7 @@ context("Dashboard links", () => {
});
it("Report link in dashboard", () => {
cy.visit("/app/user");
cy.visit("/app/user/Administrator");
cy.visit(`/app/user/${cy.config("testUser")}`);
cy.select_form_tab("Connections");
cy.get('.document-link[data-doctype="Contact"]').contains("Contact");
cy.window()

16
cypress/integration/file_uploader.js
View file

@ -21,9 +21,11 @@ context("FileUploader", () => {
it("should accept dropped files", () => {
open_upload_dialog();
cy.get_open_dialog().find(".file-upload-area").attachFile("example.json", {
subjectType: "drag-n-drop",
});
cy.get_open_dialog()
.find(".file-upload-area")
.selectFile("cypress/fixtures/example.json", {
action: "drag-drop",
});
cy.get_open_dialog().find(".file-name").should("contain", "example.json");
cy.intercept("POST", "/api/method/upload_file").as("upload_file");
@ -64,9 +66,11 @@ context("FileUploader", () => {
it("should allow cropping and optimization for valid images", () => {
open_upload_dialog();
cy.get_open_dialog().find(".file-upload-area").attachFile("sample_image.jpg", {
subjectType: "drag-n-drop",
});
cy.get_open_dialog()
.find(".file-upload-area")
.selectFile("cypress/fixtures/sample_image.jpg", {
action: "drag-drop",
});
cy.get_open_dialog().findAllByText("sample_image.jpg").should("exist");
cy.get_open_dialog().find(".btn-crop").first().click();

4
cypress/integration/login.js
View file

@ -32,7 +32,7 @@ context("Login", () => {
it("logs in using correct credentials", () => {
cy.get("#login_email").type("Administrator");
cy.get("#login_password").type(Cypress.config("adminPassword"));
cy.get("#login_password").type(Cypress.env("adminPassword"));
cy.findByRole("button", { name: "Login" }).click();
cy.location("pathname").should("eq", "/app");
@ -56,7 +56,7 @@ context("Login", () => {
);
cy.get("#login_email").type("Administrator");
cy.get("#login_password").type(Cypress.config("adminPassword"));
cy.get("#login_password").type(Cypress.env("adminPassword"));
cy.findByRole("button", { name: "Login" }).click();

15
cypress/integration/sidebar.js
View file

@ -4,9 +4,11 @@ const verify_attachment_visibility = (document, is_private) => {
const assertion = is_private ? "be.checked" : "not.be.checked";
cy.findByRole("button", { name: "Attach File" }).click();
cy.get_open_dialog().find(".file-upload-area").attachFile("sample_image.jpg", {
subjectType: "drag-n-drop",
});
cy.get_open_dialog()
.find(".file-upload-area")
.selectFile("cypress/fixtures/sample_image.jpg", {
action: "drag-drop",
});
cy.get_open_dialog().findByRole("checkbox", { name: "Private" }).should(assertion);
};
@ -36,11 +38,6 @@ context("Sidebar", () => {
//To check if no filter is available in "Assigned To" dropdown
cy.get(".empty-state").should("contain", "No filters found");
cy.click_sidebar_button("Created By");
//To check if "Created By" dropdown contains filter
cy.get(".group-by-item > .dropdown-item").should("contain", "Me");
//Assigning a doctype to a user
cy.visit("/app/doctype/ToDo");
cy.get(".form-assignments > .flex > .text-muted").click();
@ -70,7 +67,7 @@ context("Sidebar", () => {
cy.get(".condition").should("have.value", "like");
cy.get(".filter-field > .form-group > .input-with-feedback").should(
"have.value",
"%Administrator%"
`%${cy.config("testUser")}%`
);
cy.click_filter_button();

6
cypress/integration/table_multiselect.js
View file

@ -13,7 +13,9 @@ context("Table MultiSelect", () => {
cy.fill_field("assign_condition", 'status=="Open"', "Code");
cy.get('input[data-fieldname="users"]').focus().as("input");
cy.get('input[data-fieldname="users"] + ul').should("be.visible");
cy.get("@input").type("test{enter}", { delay: 100 });
cy.get("@input").type("test@erpnext", { delay: 100 });
cy.wait(500);
cy.get("@input").type("{enter}");
cy.get(
'.frappe-control[data-fieldname="users"] .form-control .tb-selected-value .btn-link-to-form'
).as("selected-value");
@ -52,6 +54,6 @@ context("Table MultiSelect", () => {
"existing_value"
);
cy.get("@existing_value").find(".btn-link-to-form").click();
cy.location("pathname").should("contain", "/user/test@erpnext.com");
cy.location("pathname").should("contain", "/user/test%40erpnext.com");
});
});

4
cypress/integration/timeline.js
View file

@ -72,14 +72,14 @@ context("Timeline", () => {
cy.click_listview_row_item(0);
//To check if the submission of the documemt is visible in the timeline content
cy.get(".timeline-content").should("contain", "Administrator submitted this document");
cy.get(".timeline-content").should("contain", "Frappe submitted this document");
cy.get('[id="page-Custom Submittable DocType"] .page-actions')
.findByRole("button", { name: "Cancel" })
.click();
cy.get_open_dialog().findByRole("button", { name: "Yes" }).click();
//To check if the cancellation of the documemt is visible in the timeline content
cy.get(".timeline-content").should("contain", "Administrator cancelled this document");
cy.get(".timeline-content").should("contain", "Frappe cancelled this document");
//Deleting the document
cy.visit("/app/custom-submittable-doctype");

16
cypress/integration/web_form.js
View file

@ -1,6 +1,13 @@
context("Web Form", () => {
before(() => {
cy.login();
cy.login("Administrator");
cy.visit("/app/");
return cy
.window()
.its("frappe")
.then((frappe) => {
return frappe.xcall("frappe.tests.ui_test_helpers.clear_notes");
});
});
it("Create Web Form", () => {
@ -42,7 +49,7 @@ context("Web Form", () => {
});
it("Login Required", () => {
cy.login();
cy.login("Administrator");
cy.visit("/app/web-form/note");
cy.findByRole("tab", { name: "Settings" }).click();
@ -51,7 +58,6 @@ context("Web Form", () => {
cy.save();
cy.visit("/note");
cy.url().should("include", "/note/Note%201");
cy.call("logout");
@ -62,7 +68,7 @@ context("Web Form", () => {
});
it("Show List", () => {
cy.login();
cy.login("Administrator");
cy.visit("/app/web-form/note");
cy.findByRole("tab", { name: "Settings" }).click();
@ -156,7 +162,7 @@ context("Web Form", () => {
});
it("Read Only", () => {
cy.login();
cy.login("Administrator");
cy.visit("/note");
cy.url().should("include", "/note/list");

3
cypress/support/commands.js
View file

@ -1,4 +1,3 @@
import "cypress-file-upload";
import "@testing-library/cypress/add-commands";
import "@4tw/cypress-drag-drop";
import "cypress-real-events/support";
@ -30,7 +29,7 @@ import "cypress-real-events/support";
Cypress.Commands.add("login", (email, password) => {
if (!email) {
email = "Administrator";
email = Cypress.config("testUser") || "Administrator";
}
if (!password) {
password = Cypress.env("adminPassword");

2
esbuild/esbuild.js
View file

@ -103,7 +103,7 @@ async function execute() {
log_error("There were some problems during build");
log();
log(chalk.dim(e.stack));
if (process.env.CI) {
if (process.env.CI || PRODUCTION) {
process.kill(process.pid);
}
return;

49
frappe/__init__.py
View file

@ -203,6 +203,7 @@ def init(site: str, sites_path: str = ".", new_site: bool = False) -> None:
"mute_emails": False,
"has_dataurl": False,
"new_site": new_site,
"read_only": False,
}
)
local.rollback_observers = []
@ -238,7 +239,6 @@ def init(site: str, sites_path: str = ".", new_site: bool = False) -> None:
local.jloader = None
local.cache = {}
local.document_cache = {}
local.meta_cache = {}
local.form_dict = _dict()
local.preload_assets = {"style": [], "script": []}
local.session = _dict()
@ -284,9 +284,7 @@ def connect_replica():
user = local.conf.replica_db_name
password = local.conf.replica_db_password
local.replica_db = get_db(
host=local.conf.replica_host, user=user, password=password, port=port, read_only=True
)
local.replica_db = get_db(host=local.conf.replica_host, user=user, password=password, port=port)
# swap db connections
local.primary_db = local.db
@ -1065,21 +1063,9 @@ def set_value(doctype, docname, fieldname, value=None):
return frappe.client.set_value(doctype, docname, fieldname, value)
@overload
def get_cached_doc(doctype, docname, _allow_dict=True) -> dict:
...
@overload
def get_cached_doc(*args, **kwargs) -> "Document":
...
def get_cached_doc(*args, **kwargs):
allow_dict = kwargs.pop("_allow_dict", False)
def _respond(doc, from_redis=False):
if not allow_dict and isinstance(doc, dict):
if isinstance(doc, dict):
local.document_cache[key] = doc = get_doc(doc)
elif from_redis:
@ -1103,6 +1089,12 @@ def get_cached_doc(*args, **kwargs):
if not key:
key = get_document_cache_key(doc.doctype, doc.name)
_set_document_in_cache(key, doc)
return doc
def _set_document_in_cache(key: str, doc: "Document") -> None:
local.document_cache[key] = doc
# Avoid setting in local.cache since we're already using local.document_cache above
@ -1112,8 +1104,6 @@ def get_cached_doc(*args, **kwargs):
except Exception:
cache().hset("document_cache", key, doc.as_dict(), cache_locally=False)
return doc
def can_cache_doc(args) -> str | None:
"""
@ -1152,7 +1142,7 @@ def get_cached_value(
doctype: str, name: str, fieldname: str = "name", as_dict: bool = False
) -> Any:
try:
doc = get_cached_doc(doctype, name, _allow_dict=True)
doc = get_cached_doc(doctype, name)
except DoesNotExistError:
clear_last_message()
return
@ -1188,13 +1178,9 @@ def get_doc(*args, **kwargs) -> "Document":
doc = frappe.model.document.get_doc(*args, **kwargs)
# Replace cache
if key := can_cache_doc(args):
if key in local.document_cache:
local.document_cache[key] = doc
if cache().hexists("document_cache", key):
cache().hset("document_cache", key, doc.as_dict())
# Replace cache if stale one exists
if (key := can_cache_doc(args)) and cache().hexists("document_cache", key):
_set_document_in_cache(key, doc)
return doc
@ -2224,13 +2210,18 @@ def log_error(title=None, message=None, reference_doctype=None, reference_name=N
title = title or "Error"
traceback = as_unicode(traceback or get_traceback(with_context=True))
return get_doc(
error_log = get_doc(
doctype="Error Log",
error=traceback,
method=title,
reference_doctype=reference_doctype,
reference_name=reference_name,
).insert(ignore_permissions=True)
)
if flags.read_only:
error_log.deferred_insert()
else:
return error_log.insert(ignore_permissions=True)
def get_desk_link(doctype, name):

48
frappe/app.py
View file

@ -33,17 +33,6 @@ SAFE_HTTP_METHODS = ("GET", "HEAD", "OPTIONS")
UNSAFE_HTTP_METHODS = ("POST", "PUT", "DELETE", "PATCH")
class RequestContext:
def __init__(self, environ):
self.request = Request(environ)
def __enter__(self):
init_request(self.request)
def __exit__(self, type, value, traceback):
frappe.destroy()
@local_manager.middleware
@Request.application
def application(request: Request):
@ -83,9 +72,6 @@ def application(request: Request):
except HTTPException as e:
return e
except frappe.SessionStopped as e:
response = frappe.utils.response.handle_session_stopped()
except Exception as e:
response = handle_exception(e)
@ -118,9 +104,12 @@ def init_request(request):
# site does not exist
raise NotFound
if frappe.local.conf.get("maintenance_mode"):
if frappe.local.conf.maintenance_mode:
frappe.connect()
raise frappe.SessionStopped("Session Stopped")
if frappe.local.conf.allow_reads_during_maintenance:
setup_read_only_mode()
else:
raise frappe.SessionStopped("Session Stopped")
else:
frappe.connect(set_admin_as_user=False)
@ -132,6 +121,24 @@ def init_request(request):
frappe.local.http_request = frappe.auth.HTTPRequest()
def setup_read_only_mode():
"""During maintenance_mode reads to DB can still be performed to reduce downtime. This
function sets up read only mode
- Setting global flag so other pages, desk and database can know that we are in read only mode.
- Setup read only database access either by:
- Connecting to read replica if one exists
- Or setting up read only SQL transactions.
"""
frappe.flags.read_only = True
# If replica is available then just connect replica, else setup read only transaction.
if frappe.conf.read_from_replica:
frappe.connect_replica()
else:
frappe.db.begin(read_only=True)
def log_request(request, response):
if hasattr(frappe.local, "conf") and frappe.local.conf.enable_frappe_logger:
frappe.logger("frappe.web", allow_site=frappe.local.site).info(
@ -233,11 +240,20 @@ def handle_exception(e):
or (frappe.local.request.path.startswith("/api/") and not accept_header.startswith("text"))
)
if not frappe.session.user:
# If session creation fails then user won't be unset. This causes a lot of code that
# assumes presence of this to fail. Session creation fails => guest or expired login
# usually.
frappe.session.user = "Guest"
if respond_as_json:
# handle ajax responses first
# if the request is ajax, send back the trace or error message
response = frappe.utils.response.report_error(http_status_code)
elif isinstance(e, frappe.SessionStopped):
response = frappe.utils.response.handle_session_stopped()
elif (
http_status_code == 500
and (frappe.db and isinstance(e, frappe.db.InternalError))

19
frappe/auth.py
View file

@ -6,9 +6,8 @@ import frappe
import frappe.database
import frappe.utils
import frappe.utils.user
from frappe import _, conf
from frappe import _
from frappe.core.doctype.activity_log.activity_log import add_authentication_log
from frappe.modules.patch_handler import check_session_stopped
from frappe.sessions import Session, clear_sessions, delete_session
from frappe.translate import get_language
from frappe.twofactor import (
@ -30,9 +29,6 @@ class HTTPRequest:
# load cookies
self.set_cookies()
# set frappe.local.db
self.connect()
# login and start/resume user session
self.set_session()
@ -45,9 +41,6 @@ class HTTPRequest:
# write out latest cookies
frappe.local.cookie_manager.init_cookies()
# check session status
check_session_stopped()
@property
def domain(self):
if not getattr(self, "_domain", None):
@ -97,16 +90,6 @@ class HTTPRequest:
def set_lang(self):
frappe.local.lang = get_language()
def get_db_name(self):
"""get database name from conf"""
return conf.db_name
def connect(self):
"""connect to db, from ac_name or db_name"""
frappe.local.db = frappe.database.get_db(
user=self.get_db_name(), password=getattr(conf, "db_password", "")
)
class LoginManager:

70
frappe/automation/workspace/tools/tools.json
View file

@ -1,6 +1,6 @@
{
"charts": [],
"content": "[{\"type\":\"header\",\"data\":{\"text\":\"<span class=\\\"h4\\\"><b>Your Shortcuts</b></span>\",\"col\":12}},{\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"ToDo\",\"col\":3}},{\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"Note\",\"col\":3}},{\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"File\",\"col\":3}},{\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"Assignment Rule\",\"col\":3}},{\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"Auto Repeat\",\"col\":3}},{\"type\":\"spacer\",\"data\":{\"col\":12}},{\"type\":\"header\",\"data\":{\"text\":\"<span class=\\\"h4\\\"><b>Reports & Masters</b></span>\",\"col\":12}},{\"type\":\"card\",\"data\":{\"card_name\":\"Tools\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Email\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Automation\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Event Streaming\",\"col\":4}}]",
"content": "[{\"type\":\"header\",\"data\":{\"text\":\"<span class=\\\"h4\\\"><b>Your Shortcuts</b></span>\",\"col\":12}},{\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"ToDo\",\"col\":3}},{\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"Note\",\"col\":3}},{\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"File\",\"col\":3}},{\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"Assignment Rule\",\"col\":3}},{\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"Auto Repeat\",\"col\":3}},{\"type\":\"spacer\",\"data\":{\"col\":12}},{\"type\":\"header\",\"data\":{\"text\":\"<span class=\\\"h4\\\"><b>Reports &amp; Masters</b></span>\",\"col\":12}},{\"type\":\"card\",\"data\":{\"card_name\":\"Tools\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Email\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Automation\",\"col\":4}}]",
"creation": "2020-03-02 14:53:24.980279",
"docstatus": 0,
"doctype": "Workspace",
@ -107,7 +107,7 @@
"hidden": 0,
"is_query_report": 0,
"label": "Automation",
"link_count": 0,
"link_count": 3,
"onboard": 0,
"type": "Card Break"
},
@ -143,78 +143,16 @@
"link_type": "DocType",
"onboard": 0,
"type": "Link"
},
{
"hidden": 0,
"is_query_report": 0,
"label": "Event Streaming",
"link_count": 0,
"onboard": 0,
"type": "Card Break"
},
{
"dependencies": "",
"hidden": 0,
"is_query_report": 0,
"label": "Event Producer",
"link_count": 0,
"link_to": "Event Producer",
"link_type": "DocType",
"onboard": 0,
"type": "Link"
},
{
"dependencies": "",
"hidden": 0,
"is_query_report": 0,
"label": "Event Consumer",
"link_count": 0,
"link_to": "Event Consumer",
"link_type": "DocType",
"onboard": 0,
"type": "Link"
},
{
"dependencies": "",
"hidden": 0,
"is_query_report": 0,
"label": "Event Update Log",
"link_count": 0,
"link_to": "Event Update Log",
"link_type": "DocType",
"onboard": 0,
"type": "Link"
},
{
"dependencies": "",
"hidden": 0,
"is_query_report": 0,
"label": "Event Sync Log",
"link_count": 0,
"link_to": "Event Sync Log",
"link_type": "DocType",
"onboard": 0,
"type": "Link"
},
{
"dependencies": "",
"hidden": 0,
"is_query_report": 0,
"label": "Document Type Mapping",
"link_count": 0,
"link_to": "Document Type Mapping",
"link_type": "DocType",
"onboard": 0,
"type": "Link"
}
],
"modified": "2022-01-13 17:48:48.456763",
"modified": "2022-08-23 14:42:58.364898",
"modified_by": "Administrator",
"module": "Automation",
"name": "Tools",
"owner": "Administrator",
"parent_page": "",
"public": 1,
"quick_lists": [],
"restrict_to_domain": "",
"roles": [],
"sequence_id": 26.0,

8
frappe/cache_manager.py
View file

@ -12,7 +12,6 @@ doctype_map_keys = (
"energy_point_rule_map",
"assignment_rule_map",
"milestone_tracker_map",
"event_consumer_document_type_map",
)
bench_cache_keys = ("assets_json",)
@ -59,8 +58,8 @@ user_cache_keys = (
)
doctype_cache_keys = (
"meta",
"form_meta",
"doctype_meta",
"doctype_form_meta",
"table_columns",
"last_modified",
"linked_doctypes",
@ -117,9 +116,6 @@ def clear_doctype_cache(doctype=None):
clear_controller_cache(doctype)
cache = frappe.cache()
if getattr(frappe.local, "meta_cache") and (doctype in frappe.local.meta_cache):
del frappe.local.meta_cache[doctype]
for key in ("is_table", "doctype_modules", "document_cache"):
cache.delete_value(key)

3
frappe/commands/site.py
View file

@ -420,6 +420,9 @@ def install_app(context, apps, force=False):
print(f"An error occurred while installing {app}{err_msg}")
exit_code = 1
if not exit_code:
frappe.db.commit()
frappe.destroy()
sys.exit(exit_code)

4
frappe/commands/utils.py
View file

@ -874,7 +874,6 @@ def run_ui_tests(
node_bin = subprocess.getoutput("npm bin")
cypress_path = f"{node_bin}/cypress"
plugin_path = f"{node_bin}/../cypress-file-upload"
drag_drop_plugin_path = f"{node_bin}/../@4tw/cypress-drag-drop"
real_events_plugin_path = f"{node_bin}/../cypress-real-events"
testing_library_path = f"{node_bin}/../@testing-library"
@ -883,7 +882,6 @@ def run_ui_tests(
# check if cypress in path...if not, install it.
if not (
os.path.exists(cypress_path)
and os.path.exists(plugin_path)
and os.path.exists(drag_drop_plugin_path)
and os.path.exists(real_events_plugin_path)
and os.path.exists(testing_library_path)
@ -894,10 +892,10 @@ def run_ui_tests(
packages = " ".join(
[
"cypress@^10",
"cypress-file-upload@^5",
"@4tw/cypress-drag-drop@^2",
"cypress-real-events",
"@testing-library/cypress@^8",
"@testing-library/dom@8.17.1",
"@cypress/code-coverage@^3",
]
)

8
frappe/core/doctype/activity_log/activity_log.json
View file

@ -102,8 +102,7 @@
"fetch_from": "reference_name.owner",
"fieldname": "reference_owner",
"fieldtype": "Read Only",
"label": "Reference Owner",
"search_index": 1
"label": "Reference Owner"
},
{
"fieldname": "column_break_14",
@ -154,7 +153,7 @@
"icon": "fa fa-comment",
"index_web_pages_for_search": 1,
"links": [],
"modified": "2021-10-25 11:43:57.504565",
"modified": "2022-09-13 15:19:42.474114",
"modified_by": "Administrator",
"module": "Core",
"name": "Activity Log",
@ -181,6 +180,7 @@
"search_fields": "subject",
"sort_field": "modified",
"sort_order": "DESC",
"states": [],
"title_field": "subject",
"track_seen": 1
}
}

1
frappe/core/doctype/activity_log/activity_log.py
View file

@ -37,7 +37,6 @@ def on_doctype_update():
"""Add indexes in `tabActivity Log`"""
frappe.db.add_index("Activity Log", ["reference_doctype", "reference_name"])
frappe.db.add_index("Activity Log", ["timeline_doctype", "timeline_name"])
frappe.db.add_index("Activity Log", ["link_doctype", "link_name"])
def add_authentication_log(subject, user, operation="Login", status="Success"):

1
frappe/core/doctype/comment/comment.py
View file

@ -60,7 +60,6 @@ class Comment(Document):
def on_doctype_update():
frappe.db.add_index("Comment", ["reference_doctype", "reference_name"])
frappe.db.add_index("Comment", ["link_doctype", "link_name"])
def update_comment_in_doc(doc):

3
frappe/core/doctype/custom_docperm/custom_docperm.json
View file

@ -212,7 +212,8 @@
"fieldname": "parent",
"fieldtype": "Data",
"label": "Reference Document Type",
"read_only": 1
"read_only": 1,
"search_index": 1
},
{
"default": "0",

7
frappe/core/doctype/data_import/data_import.py
View file

@ -9,7 +9,7 @@ from frappe.core.doctype.data_import.exporter import Exporter
from frappe.core.doctype.data_import.importer import Importer
from frappe.model.document import Document
from frappe.modules.import_file import import_file_by_path
from frappe.utils.background_jobs import enqueue
from frappe.utils.background_jobs import enqueue, is_job_queued
from frappe.utils.csvutils import validate_google_sheets_url
@ -59,15 +59,12 @@ class DataImport(Document):
return i.get_data_for_import_preview()
def start_import(self):
from frappe.core.page.background_jobs.background_jobs import get_info
from frappe.utils.scheduler import is_scheduler_inactive
if is_scheduler_inactive() and not frappe.flags.in_test:
frappe.throw(_("Scheduler is inactive. Cannot import data."), title=_("Scheduler Inactive"))
enqueued_jobs = [d.get("job_name") for d in get_info()]
if self.name not in enqueued_jobs:
if not is_job_queued(self.name):
enqueue(
start_import,
queue="default",

7
frappe/core/doctype/doctype/doctype.py
View file

@ -414,10 +414,6 @@ class DocType(Document):
if not frappe.flags.in_install and hasattr(self, "before_update"):
self.sync_global_search()
# clear from local cache
if self.name in frappe.local.meta_cache:
del frappe.local.meta_cache[self.name]
clear_linked_doctype_cache()
def setup_autoincrement_and_sequence(self):
@ -1198,6 +1194,9 @@ def validate_fields(meta):
frappe.throw(_("Precision should be between 1 and 6"))
def check_unique_and_text(docname, d):
if meta.is_virtual:
return
if meta.issingle:
d.unique = 0
d.search_index = 0

2
frappe/core/doctype/document_naming_settings/document_naming_settings.py
View file

@ -113,6 +113,8 @@ class DocumentNamingSettings(Document):
option_string = "\n".join(options)
# Erase default first, it might not be in new options.
self.update_naming_series_property_setter(doctype, "default", "")
self.update_naming_series_property_setter(doctype, "options", option_string)
self.update_naming_series_property_setter(doctype, "default", default)

2
frappe/core/doctype/error_log/error_log.py
View file

@ -9,7 +9,7 @@ from frappe.query_builder.functions import Now
class ErrorLog(Document):
def onload(self):
if not self.seen:
if not self.seen and not frappe.flags.read_only:
self.db_set("seen", 1, update_modified=0)
frappe.db.commit()

12
frappe/core/doctype/file/file.json
View file

@ -64,8 +64,7 @@
"fieldname": "is_home_folder",
"fieldtype": "Check",
"hidden": 1,
"label": "Is Home Folder",
"search_index": 1
"label": "Is Home Folder"
},
{
"default": "0",
@ -125,8 +124,7 @@
"in_standard_filter": 1,
"label": "Attached To DocType",
"options": "DocType",
"read_only": 1,
"search_index": 1
"read_only": 1
},
{
"fieldname": "column_break_10",
@ -136,8 +134,7 @@
"fieldname": "attached_to_name",
"fieldtype": "Data",
"label": "Attached To Name",
"read_only": 1,
"search_index": 1
"read_only": 1
},
{
"fieldname": "attached_to_field",
@ -175,7 +172,7 @@
"icon": "fa fa-file",
"idx": 1,
"links": [],
"modified": "2020-06-28 12:21:30.772386",
"modified": "2022-09-13 15:50:15.508250",
"modified_by": "Administrator",
"module": "Core",
"name": "File",
@ -210,6 +207,7 @@
],
"sort_field": "modified",
"sort_order": "ASC",
"states": [],
"title_field": "file_name",
"track_changes": 1
}

5
frappe/core/doctype/report/report.json
View file

@ -128,7 +128,6 @@
"fieldtype": "Section Break"
},
{
"depends_on": "eval:doc.is_standard == 'Yes'",
"fieldname": "roles",
"fieldtype": "Table",
"label": "Roles",
@ -192,10 +191,11 @@
"idx": 1,
"index_web_pages_for_search": 1,
"links": [],
"modified": "2020-08-17 16:49:28.474274",
"modified": "2022-09-15 13:37:24.531848",
"modified_by": "Administrator",
"module": "Core",
"name": "Report",
"naming_rule": "By fieldname",
"owner": "Administrator",
"permissions": [
{
@ -242,5 +242,6 @@
"show_name_in_global_search": 1,
"sort_field": "modified",
"sort_order": "DESC",
"states": [],
"track_changes": 1
}

0
frappe/core/page/background_jobs/__init__.py → frappe/core/doctype/rq_job/__init__.py
View file

30
frappe/core/doctype/rq_job/rq_job.js Normal file
View file

@ -0,0 +1,30 @@
// Copyright (c) 2022, Frappe Technologies and contributors
// For license information, please see license.txt
frappe.ui.form.on("RQ Job", {
refresh: function (frm) {
// Nothing in this form is supposed to be editable.
frm.disable_form();
frm.dashboard.set_headline_alert(
"This is a virtual doctype and data is cleared periodically."
);
if (["started", "queued"].includes(frm.doc.status)) {
frm.add_custom_button(__("Force Stop job"), () => {
frappe.confirm(
"This will terminate the job immediately and might be dangerous, are you sure? ",
() => {
frappe
.xcall("frappe.core.doctype.rq_job.rq_job.stop_job", {
job_id: frm.doc.name,
})
.then((r) => {
frappe.show_alert("Job Stopped Succefully");
frm.reload_doc();
});
}
);
});
}
},
});

162
frappe/core/doctype/rq_job/rq_job.json Normal file
View file

@ -0,0 +1,162 @@
{
"actions": [],
"allow_copy": 1,
"autoname": "field:job_id",
"creation": "2022-09-10 16:19:37.934903",
"doctype": "DocType",
"editable_grid": 1,
"engine": "InnoDB",
"field_order": [
"job_info_section",
"job_id",
"job_name",
"queue",
"timeout",
"column_break_5",
"arguments",
"job_status_section",
"status",
"time_taken",
"column_break_11",
"started_at",
"ended_at",
"exception_section",
"exc_info"
],
"fields": [
{
"fieldname": "queue",
"fieldtype": "Select",
"in_list_view": 1,
"in_standard_filter": 1,
"label": "Queue",
"options": "default\nshort\nlong"
},
{
"fieldname": "status",
"fieldtype": "Select",
"in_list_view": 1,
"in_standard_filter": 1,
"label": "Status",
"options": "queued\nstarted\nfinished\nfailed\ndeferred\nscheduled\ncanceled"
},
{
"fieldname": "job_id",
"fieldtype": "Data",
"label": "Job ID",
"unique": 1
},
{
"fieldname": "exc_info",
"fieldtype": "Code",
"label": "Exception"
},
{
"fieldname": "job_name",
"fieldtype": "Data",
"label": "Job Name"
},
{
"fieldname": "arguments",
"fieldtype": "Code",
"label": "Arguments"
},
{
"fieldname": "timeout",
"fieldtype": "Duration",
"label": "Timeout"
},
{
"fieldname": "time_taken",
"fieldtype": "Duration",
"label": "Time Taken"
},
{
"fieldname": "started_at",
"fieldtype": "Datetime",
"label": "Started At"
},
{
"fieldname": "ended_at",
"fieldtype": "Datetime",
"label": "Ended At"
},
{
"fieldname": "job_info_section",
"fieldtype": "Section Break",
"label": "Job Info"
},
{
"fieldname": "job_status_section",
"fieldtype": "Section Break",
"label": "Job Status"
},
{
"fieldname": "column_break_5",
"fieldtype": "Column Break"
},
{
"fieldname": "column_break_11",
"fieldtype": "Column Break"
},
{
"fieldname": "exception_section",
"fieldtype": "Section Break"
}
],
"in_create": 1,
"is_virtual": 1,
"links": [],
"modified": "2022-09-11 05:27:50.878534",
"modified_by": "Administrator",
"module": "Core",
"name": "RQ Job",
"naming_rule": "By fieldname",
"owner": "Administrator",
"permissions": [
{
"email": 1,
"export": 1,
"print": 1,
"read": 1,
"report": 1,
"role": "System Manager",
"share": 1
},
{
"delete": 1,
"email": 1,
"export": 1,
"print": 1,
"read": 1,
"report": 1,
"role": "Administrator",
"share": 1
}
],
"sort_field": "modified",
"sort_order": "DESC",
"states": [
{
"color": "Yellow",
"title": "queued"
},
{
"color": "Blue",
"title": "started"
},
{
"color": "Red",
"title": "failed"
},
{
"color": "Green",
"title": "finished"
},
{
"color": "Orange",
"title": "cancelled"
}
],
"title_field": "job_name"
}

193
frappe/core/doctype/rq_job/rq_job.py Normal file
View file

@ -0,0 +1,193 @@
# Copyright (c) 2022, Frappe Technologies and contributors
# For license information, please see license.txt
import functools
import re
from rq.command import send_stop_job_command
from rq.job import Job
from rq.queue import Queue
import frappe
from frappe.model.document import Document
from frappe.utils import (
cint,
compare,
convert_utc_to_user_timezone,
create_batch,
make_filter_dict,
)
from frappe.utils.background_jobs import get_queues, get_redis_conn
QUEUES = ["default", "long", "short"]
JOB_STATUSES = ["queued", "started", "failed", "finished", "deferred", "scheduled", "canceled"]
def check_permissions(method):
@functools.wraps(method)
def wrapper(*args, **kwargs):
frappe.only_for("System Manager")
job = args[0].job
if not for_current_site(job):
raise frappe.PermissionError
return method(*args, **kwargs)
return wrapper
class RQJob(Document):
def load_from_db(self):
job = Job.fetch(self.name, connection=get_redis_conn())
if not for_current_site(job):
raise frappe.PermissionError
super(Document, self).__init__(serialize_job(job))
self._job_obj = job
@property
def job(self):
return self._job_obj
@staticmethod
def get_list(args):
start = cint(args.get("start")) or 0
page_length = cint(args.get("page_length")) or 20
order_desc = "desc" in args.get("order_by", "")
matched_job_ids = RQJob.get_matching_job_ids(args)
jobs = []
for job_ids in create_batch(matched_job_ids, 100):
jobs.extend(
serialize_job(job)
for job in Job.fetch_many(job_ids=job_ids, connection=get_redis_conn())
if job and for_current_site(job)
)
if len(jobs) > start + page_length:
# we have fetched enough. This is inefficient but because of site filtering TINA
break
return sorted(jobs, key=lambda j: j.modified, reverse=order_desc)[start : start + page_length]
@staticmethod
def get_matching_job_ids(args):
filters = make_filter_dict(args.get("filters"))
queues = _eval_filters(filters.get("queue"), QUEUES)
statuses = _eval_filters(filters.get("status"), JOB_STATUSES)
matched_job_ids = []
for queue in get_queues():
if not queue.name.endswith(tuple(queues)):
continue
for status in statuses:
matched_job_ids.extend(fetch_job_ids(queue, status))
return matched_job_ids
@check_permissions
def delete(self):
self.job.delete()
@check_permissions
def stop_job(self):
send_stop_job_command(connection=get_redis_conn(), job_id=self.job_id)
@staticmethod
def get_count(args) -> int:
# Can not be implemented efficiently due to site filtering hence ignored.
return 0
# None of these methods apply to virtual job doctype, overriden for sanity.
@staticmethod
def get_stats(args):
return {}
def db_insert(self, *args, **kwargs):
pass
def db_update(self, *args, **kwargs):
pass
def serialize_job(job: Job) -> frappe._dict:
modified = job.last_heartbeat or job.ended_at or job.started_at or job.created_at
job_name = job.kwargs.get("kwargs", {}).get("job_type") or str(job.kwargs.get("job_name"))
# function objects have this repr: '<function functionname at 0xmemory_address >'
# This regex just removes unnecessary things around it.
if matches := re.match(r"<function (?P<func_name>.*) at 0x.*>", job_name):
job_name = matches.group("func_name")
return frappe._dict(
name=job.id,
job_id=job.id,
queue=job.origin.rsplit(":", 1)[1],
job_name=job_name,
status=job.get_status(),
started_at=convert_utc_to_user_timezone(job.started_at) if job.started_at else "",
ended_at=convert_utc_to_user_timezone(job.ended_at) if job.ended_at else "",
time_taken=(job.ended_at - job.started_at).total_seconds() if job.ended_at else "",
exc_info=job.exc_info,
arguments=frappe.as_json(job.kwargs),
timeout=job.timeout,
creation=convert_utc_to_user_timezone(job.created_at),
modified=convert_utc_to_user_timezone(modified),
_comment_count=0,
)
def for_current_site(job: Job) -> bool:
return job.kwargs.get("site") == frappe.local.site
def _eval_filters(filter, values: list[str]) -> list[str]:
if filter:
operator, operand = filter
return [val for val in values if compare(val, operator, operand)]
return values
def fetch_job_ids(queue: Queue, status: str) -> list[str]:
registry_map = {
"queued": queue, # self
"started": queue.started_job_registry,
"finished": queue.finished_job_registry,
"failed": queue.failed_job_registry,
"deferred": queue.deferred_job_registry,
"scheduled": queue.scheduled_job_registry,
"canceled": queue.canceled_job_registry,
}
registry = registry_map.get(status)
if registry is not None:
job_ids = registry.get_job_ids()
return [j for j in job_ids if j]
return []
@frappe.whitelist()
def remove_failed_jobs():
frappe.only_for("System Manager")
for queue in get_queues():
fail_registry = queue.failed_job_registry
for job_ids in create_batch(fail_registry.get_job_ids(), 100):
for job in Job.fetch_many(job_ids=job_ids, connection=get_redis_conn()):
if job and for_current_site(job):
fail_registry.remove(job, delete_job=True)
def get_all_queued_jobs():
jobs = []
for q in get_queues():
jobs.extend(q.get_jobs())
return [job for job in jobs if for_current_site(job)]
@frappe.whitelist()
def stop_job(job_id):
frappe.get_doc("RQ Job", job_id).stop_job()

32
frappe/core/doctype/rq_job/rq_job_list.js Normal file
View file

@ -0,0 +1,32 @@
frappe.listview_settings["RQ Job"] = {
hide_name_column: true,
onload(listview) {
if (!has_common(frappe.user_roles, ["Administrator", "System Manager"])) return;
listview.page.add_inner_button(__("Remove Failed Jobs"), () => {
frappe.confirm(__("Are you sure you want to remove all failed jobs?"), () => {
frappe.xcall("frappe.core.doctype.rq_job.rq_job.remove_failed_jobs");
});
});
if (listview.list_view_settings) {
listview.list_view_settings.disable_count = 1;
listview.list_view_settings.disable_sidebar_stats = 1;
}
frappe.xcall("frappe.utils.scheduler.get_scheduler_status").then(({ status }) => {
if (status === "active") {
listview.page.set_indicator(__("Scheduler: Active"), "green");
} else {
listview.page.set_indicator(__("Scheduler: Inactive"), "red");
}
});
setInterval(() => {
if (!listview.list_view_settings.disable_auto_refresh) {
listview.refresh();
}
}, 5000);
},
};

103
frappe/core/doctype/rq_job/test_rq_job.py Normal file
View file

@ -0,0 +1,103 @@
# Copyright (c) 2022, Frappe Technologies and Contributors
# See license.txt
import time
from rq import exceptions as rq_exc
from rq.job import Job
import frappe
from frappe.core.doctype.rq_job.rq_job import RQJob, remove_failed_jobs, stop_job
from frappe.tests.utils import FrappeTestCase, timeout
from frappe.utils.background_jobs import is_job_queued
class TestRQJob(FrappeTestCase):
BG_JOB = "frappe.core.doctype.rq_job.test_rq_job.test_func"
@timeout(seconds=20)
def check_status(self, job: Job, status, wait=True):
if wait:
while True:
if job.is_queued or job.is_started:
time.sleep(0.2)
else:
break
self.assertEqual(frappe.get_doc("RQ Job", job.id).status, status)
def test_serialization(self):
job = frappe.enqueue(method=self.BG_JOB, queue="short")
rq_job = frappe.get_doc("RQ Job", job.id)
self.assertEqual(job, rq_job.job)
self.assertDocumentEqual(
{
"name": job.id,
"queue": "short",
"job_name": self.BG_JOB,
"status": "queued",
"exc_info": None,
},
rq_job,
)
self.check_status(job, "finished")
def test_func_obj_serialization(self):
job = frappe.enqueue(method=test_func, queue="short")
rq_job = frappe.get_doc("RQ Job", job.id)
self.assertEqual(rq_job.job_name, "test_func")
def test_get_list_filtering(self):
# Check failed job clearning and filtering
remove_failed_jobs()
jobs = RQJob.get_list({"filters": [["RQ Job", "status", "=", "failed"]]})
self.assertEqual(jobs, [])
# Fail a job
job = frappe.enqueue(method=self.BG_JOB, queue="short", fail=True)
self.check_status(job, "failed")
jobs = RQJob.get_list({"filters": [["RQ Job", "status", "=", "failed"]]})
self.assertEqual(len(jobs), 1)
self.assertTrue(jobs[0].exc_info)
# Assert that non-failed job still exists
non_failed_jobs = RQJob.get_list({"filters": [["RQ Job", "status", "!=", "failed"]]})
self.assertGreaterEqual(len(non_failed_jobs), 1)
# Create a slow job and check if it's stuck in "Started"
job = frappe.enqueue(method=self.BG_JOB, queue="short", sleep=1000)
time.sleep(3)
self.check_status(job, "started", wait=False)
stop_job(job_id=job.id)
self.check_status(job, "stopped")
def test_delete_doc(self):
job = frappe.enqueue(method=self.BG_JOB, queue="short")
frappe.get_doc("RQ Job", job.id).delete()
with self.assertRaises(rq_exc.NoSuchJobError):
job.refresh()
def test_is_enqueued(self):
job_name = "uniq_test_job"
dummy_job = frappe.enqueue(self.BG_JOB, sleep=100, queue="short")
actual_job = frappe.enqueue(self.BG_JOB, job_name=job_name, queue="short")
self.assertTrue(is_job_queued(job_name))
stop_job(dummy_job.id)
self.check_status(actual_job, "finished")
self.assertFalse(is_job_queued(job_name))
def test_func(fail=False, sleep=0):
if fail:
42 / 0
if sleep:
time.sleep(sleep)
return True

0
frappe/event_streaming/__init__.py → frappe/core/doctype/rq_worker/__init__.py
View file

9
frappe/core/doctype/rq_worker/rq_worker.js Normal file
View file

@ -0,0 +1,9 @@
// Copyright (c) 2022, Frappe Technologies and contributors
// For license information, please see license.txt
frappe.ui.form.on("RQ Worker", {
refresh: function (frm) {
// Nothing in this form is supposed to be editable.
frm.disable_form();
},
});

138
frappe/core/doctype/rq_worker/rq_worker.json Normal file
View file

@ -0,0 +1,138 @@
{
"actions": [],
"allow_copy": 1,
"creation": "2022-09-10 14:54:57.342170",
"doctype": "DocType",
"editable_grid": 1,
"engine": "InnoDB",
"field_order": [
"worker_information_section",
"queue",
"queue_type",
"column_break_4",
"worker_name",
"statistics_section",
"status",
"pid",
"current_job_id",
"successful_job_count",
"failed_job_count",
"column_break_12",
"birth_date",
"last_heartbeat",
"total_working_time"
],
"fields": [
{
"fieldname": "worker_name",
"fieldtype": "Data",
"label": "Worker Name",
"unique": 1
},
{
"fieldname": "status",
"fieldtype": "Data",
"in_list_view": 1,
"label": "Status"
},
{
"fieldname": "current_job_id",
"fieldtype": "Link",
"label": "Current Job ID",
"options": "RQ Job"
},
{
"fieldname": "pid",
"fieldtype": "Data",
"label": "PID"
},
{
"fieldname": "last_heartbeat",
"fieldtype": "Datetime",
"label": "Last Heartbeat"
},
{
"fieldname": "birth_date",
"fieldtype": "Datetime",
"label": "Start Time"
},
{
"fieldname": "successful_job_count",
"fieldtype": "Int",
"in_list_view": 1,
"label": "Successful Job Count"
},
{
"fieldname": "failed_job_count",
"fieldtype": "Int",
"in_list_view": 1,
"label": "Failed Job Count"
},
{
"fieldname": "total_working_time",
"fieldtype": "Duration",
"label": "Total Working Time"
},
{
"fieldname": "queue",
"fieldtype": "Data",
"label": "Queue"
},
{
"fieldname": "queue_type",
"fieldtype": "Select",
"in_list_view": 1,
"label": "Queue Type",
"options": "default\nlong\nshort"
},
{
"fieldname": "worker_information_section",
"fieldtype": "Section Break",
"label": "Worker Information"
},
{
"fieldname": "statistics_section",
"fieldtype": "Section Break",
"label": "Statistics"
},
{
"fieldname": "column_break_4",
"fieldtype": "Column Break"
},
{
"fieldname": "column_break_12",
"fieldtype": "Column Break"
}
],
"in_create": 1,
"is_virtual": 1,
"links": [],
"modified": "2022-09-11 05:02:53.981705",
"modified_by": "Administrator",
"module": "Core",
"name": "RQ Worker",
"owner": "Administrator",
"permissions": [
{
"email": 1,
"export": 1,
"print": 1,
"read": 1,
"report": 1,
"role": "System Manager",
"share": 1
}
],
"sort_field": "modified",
"sort_order": "DESC",
"states": [
{
"color": "Blue",
"title": "idle"
},
{
"color": "Yellow",
"title": "busy"
}
]
}

67
frappe/core/doctype/rq_worker/rq_worker.py Normal file
View file

@ -0,0 +1,67 @@
# Copyright (c) 2022, Frappe Technologies and contributors
# For license information, please see license.txt
from rq import Worker
import frappe
from frappe.model.document import Document
from frappe.utils import cint, convert_utc_to_user_timezone
from frappe.utils.background_jobs import get_workers
class RQWorker(Document):
def load_from_db(self):
all_workers = get_workers()
worker = [w for w in all_workers if w.pid == cint(self.name)][0]
d = serialize_worker(worker)
super(Document, self).__init__(d)
@staticmethod
def get_list(args):
start = cint(args.get("start")) or 0
page_length = cint(args.get("page_length")) or 20
workers = get_workers()[start : start + page_length]
return [serialize_worker(worker) for worker in workers]
@staticmethod
def get_count(args) -> int:
return len(get_workers())
# None of these methods apply to virtual workers, overriden for sanity.
@staticmethod
def get_stats(args):
return {}
def db_insert(self, *args, **kwargs):
pass
def db_update(self, *args, **kwargs):
pass
def delete(self):
pass
def serialize_worker(worker: Worker) -> frappe._dict:
queue = ", ".join(worker.queue_names())
return frappe._dict(
name=worker.pid,
queue=queue,
queue_type=queue.rsplit(":", 1)[1],
worker_name=worker.name,
status=worker.get_state(),
pid=worker.pid,
current_job_id=worker.get_current_job_id(),
last_heartbeat=convert_utc_to_user_timezone(worker.last_heartbeat),
birth_date=convert_utc_to_user_timezone(worker.birth_date),
successful_job_count=worker.successful_job_count,
failed_job_count=worker.failed_job_count,
total_working_time=worker.total_working_time,
_comment_count=0,
modified=convert_utc_to_user_timezone(worker.last_heartbeat),
creation=convert_utc_to_user_timezone(worker.birth_date),
)

17
frappe/core/doctype/rq_worker/test_rq_worker.py Normal file
View file

@ -0,0 +1,17 @@
# Copyright (c) 2022, Frappe Technologies and Contributors
# See license.txt
import frappe
from frappe.core.doctype.rq_worker.rq_worker import RQWorker
from frappe.tests.utils import FrappeTestCase
class TestRQWorker(FrappeTestCase):
def test_get_worker_list(self):
workers = RQWorker.get_list({})
self.assertGreaterEqual(len(workers), 1)
self.assertTrue(any(w.queue_type == "short" for w in workers))
def test_worker_serialization(self):
workers = RQWorker.get_list({})
frappe.get_doc("RQ Worker", workers[0].pid)

6
frappe/core/doctype/user/user.json
View file

@ -131,7 +131,7 @@
{
"fieldname": "middle_name",
"fieldtype": "Data",
"label": "Middle Name (Optional)",
"label": "Middle Name",
"oldfieldname": "middle_name",
"oldfieldtype": "Data"
},
@ -496,7 +496,7 @@
{
"description": "Restrict user from this IP address only. Multiple IP addresses can be added by separating with commas. Also accepts partial IP addresses like (111.111.111)",
"fieldname": "restrict_ip",
"fieldtype": "Data",
"fieldtype": "Small Text",
"label": "Restrict IP",
"permlevel": 1
},
@ -753,7 +753,7 @@
"link_fieldname": "user"
}
],
"modified": "2022-08-11 14:47:04.100892",
"modified": "2022-09-19 16:05:46.485242",
"modified_by": "Administrator",
"module": "Core",
"name": "User",

4
frappe/core/doctype/view_log/view_log.json
View file

@ -14,7 +14,6 @@
"fieldname": "viewed_by",
"fieldtype": "Data",
"label": "Viewed By",
"search_index": 1,
"set_only_once": 1
},
{
@ -22,7 +21,6 @@
"fieldtype": "Link",
"label": "Reference Document Type",
"options": "DocType",
"search_index": 1,
"set_only_once": 1
},
{
@ -35,7 +33,7 @@
}
],
"links": [],
"modified": "2022-08-03 12:20:52.857103",
"modified": "2022-09-07 05:16:14.587628",
"modified_by": "Administrator",
"module": "Core",
"name": "View Log",

47
frappe/core/page/background_jobs/background_jobs.css
View file

@ -1,47 +0,0 @@
.table-background-jobs {
margin-bottom: 0px;
margin-top: 0px;
font-size: var(--text-md);
table-layout: fixed;
}
.table-background-jobs th {
font-weight: normal;
color: var(--text-muted);
}
.table-background-jobs td {
color: var(--text-light);
}
.table-background-jobs th, .table-background-jobs td {
padding: var(--padding-sm) var(--padding-md);
}
.table-background-jobs tbody tr:hover {
background-color: var(--highlight-color);
}
.job-name {
font-size: var(--text-md);
font-family: var(--font-family-monospace);
word-break: break-word;
}
.no-background-jobs {
min-height: 320px;
display: flex;
align-items: center;
justify-content: center;
flex-direction: column;
}
.no-background-jobs > img {
margin-bottom: var(--margin-md);
max-height: 100px;
}
.footer {
padding: var(--padding-md);
}

58
frappe/core/page/background_jobs/background_jobs.html
View file

@ -1,58 +0,0 @@
{% if jobs.length %}
<table class="table table-background-jobs">
<thead>
<tr>
<th style="width: 10%">{{ __("Queue") }}</th>
<th>{{ __("Job") }}</th>
<th style="width: 10%">{{ __("Status") }}</th>
<th style="width: 15%">{{ __("Created") }}</th>
</tr>
</thead>
<tbody>
{% for j in jobs %}
<tr>
<td class="worker-name">
{{ toTitle(j.queue.split(":").slice(-1)[0]) }}
</td>
<td>
<div>
<span class="job-name">
{{ frappe.utils.encode_tags(j.job_name) }}
</span>
</div>
{% if j.exc_info %}
<details>
<summary>{{ __("Exception") }}</summary>
<div class="exc_info">
<pre>{{ frappe.utils.encode_tags(j.exc_info) }}</pre>
</div>
</details>
{% endif %}
</td>
<td>
<span class="indicator-pill {{ j.color }}">
{{ toTitle(j.status) }}
</span>
</td>
<td class="creation text-muted">
{{ frappe.datetime.prettyDate(j.creation) }}
</td>
</tr>
{% endfor %}
</tbody>
</table>
{% else %}
<div class="no-background-jobs">
<img
src="/assets/frappe/images/ui-states/list-empty-state.svg"
alt="Empty State"
/>
<p class="text-muted">{{ __("No jobs found on this site") }}</p>
</div>
{% endif %}
<div class="footer">
<div class="text-muted">
{{ __("Last refreshed") }}
{{ frappe.datetime.now_datetime(true).toLocaleString() }}
</div>
</div>

136
frappe/core/page/background_jobs/background_jobs.js
View file

@ -1,136 +0,0 @@
frappe.pages["background_jobs"].on_page_load = (wrapper) => {
const background_job = new BackgroundJobs(wrapper);
$(wrapper).bind("show", () => {
background_job.show();
});
window.background_jobs = background_job;
};
class BackgroundJobs {
constructor(wrapper) {
this.page = frappe.ui.make_app_page({
parent: wrapper,
title: __("Background Jobs"),
single_column: true,
});
this.page.add_inner_button(__("Remove Failed Jobs"), () => {
frappe.confirm(__("Are you sure you want to remove all failed jobs?"), () => {
frappe
.call("frappe.core.page.background_jobs.background_jobs.remove_failed_jobs")
.then(() => this.refresh_jobs());
});
});
this.page.main.addClass("frappe-card");
this.page.body.append('<div class="table-area"></div>');
this.$content = $(this.page.body).find(".table-area");
this.make_filters();
this.refresh_jobs = frappe.utils.throttle(this.refresh_jobs.bind(this), 1000);
}
make_filters() {
this.view = this.page.add_field({
label: __("View"),
fieldname: "view",
fieldtype: "Select",
options: ["Jobs", "Workers"],
default: "Jobs",
change: () => {
this.queue_timeout.toggle(this.view.get_value() === "Jobs");
this.job_status.toggle(this.view.get_value() === "Jobs");
},
});
this.queue_timeout = this.page.add_field({
label: __("Queue"),
fieldname: "queue_timeout",
fieldtype: "Select",
options: [
{ label: "All Queues", value: "all" },
{ label: "Default", value: "default" },
{ label: "Short", value: "short" },
{ label: "Long", value: "long" },
],
default: "all",
});
this.job_status = this.page.add_field({
label: __("Job Status"),
fieldname: "job_status",
fieldtype: "Select",
options: [
{ label: "All Jobs", value: "all" },
{ label: "Queued", value: "queued" },
{ label: "Deferred", value: "deferred" },
{ label: "Started", value: "started" },
{ label: "Finished", value: "finished" },
{ label: "Failed", value: "failed" },
],
default: "all",
});
this.auto_refresh = this.page.add_field({
label: __("Auto Refresh"),
fieldname: "auto_refresh",
fieldtype: "Check",
default: 1,
change: () => {
if (this.auto_refresh.get_value()) {
this.refresh_jobs();
}
},
});
}
show() {
this.refresh_jobs();
this.update_scheduler_status();
}
update_scheduler_status() {
frappe.call({
method: "frappe.core.page.background_jobs.background_jobs.get_scheduler_status",
callback: (r) => {
let { status } = r.message;
if (status === "active") {
this.page.set_indicator(__("Scheduler: Active"), "green");
} else {
this.page.set_indicator(__("Scheduler: Inactive"), "red");
}
},
});
}
refresh_jobs() {
let view = this.view.get_value();
let args;
let { queue_timeout, job_status } = this.page.get_form_values();
if (view === "Jobs") {
args = { view, queue_timeout, job_status };
} else {
args = { view };
}
this.page.add_inner_message(__("Refreshing..."));
frappe.call({
method: "frappe.core.page.background_jobs.background_jobs.get_info",
args,
callback: (res) => {
this.page.add_inner_message("");
let template = view === "Jobs" ? "background_jobs" : "background_workers";
this.$content.html(
frappe.render_template(template, {
jobs: res.message || [],
})
);
let auto_refresh = this.auto_refresh.get_value();
if (frappe.get_route()[0] === "background_jobs" && auto_refresh) {
setTimeout(() => this.refresh_jobs(), 2000);
}
},
});
}
}

22
frappe/core/page/background_jobs/background_jobs.json
View file

@ -1,22 +0,0 @@
{
"content": null,
"creation": "2016-08-18 16:44:14.322642",
"docstatus": 0,
"doctype": "Page",
"idx": 0,
"modified": "2016-08-18 16:48:11.577611",
"modified_by": "Administrator",
"module": "Core",
"name": "background_jobs",
"owner": "Administrator",
"page_name": "background_jobs",
"roles": [
{
"role": "System Manager"
}
],
"script": null,
"standard": "Yes",
"style": null,
"title": "Background Jobs"
}

78
frappe/core/page/background_jobs/background_jobs.py
View file

@ -1,78 +0,0 @@
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# License: MIT. See LICENSE
from typing import TYPE_CHECKING
import frappe
from frappe.utils import convert_utc_to_user_timezone
from frappe.utils.background_jobs import get_queues, get_workers
from frappe.utils.scheduler import is_scheduler_inactive
if TYPE_CHECKING:
from rq.job import Job
JOB_COLORS = {"queued": "orange", "failed": "red", "started": "blue", "finished": "green"}
@frappe.whitelist()
def get_info(view=None, queue_timeout=None, job_status=None) -> list[dict]:
jobs = []
def add_job(job: "Job", queue: str) -> None:
if job.kwargs.get("site") == frappe.local.site:
job_info = {
"job_name": job.kwargs.get("kwargs", {}).get("playbook_method")
or job.kwargs.get("kwargs", {}).get("job_type")
or str(job.kwargs.get("job_name")),
"status": job.get_status(),
"queue": queue,
"creation": convert_utc_to_user_timezone(job.created_at),
"color": JOB_COLORS[job.get_status()],
}
if job.exc_info:
job_info["exc_info"] = job.exc_info
jobs.append(job_info)
if view == "Jobs":
queues = get_queues()
for queue in queues:
for job in queue.jobs:
if job_status != "all" and job.get_status() != job_status:
return
if queue_timeout != "all" and not queue.name.endswith(f":{queue_timeout}"):
return
add_job(job, queue.name)
elif view == "Workers":
workers = get_workers()
for worker in workers:
current_job = worker.get_current_job()
if current_job:
if hasattr(current_job, "kwargs") and current_job.kwargs.get("site") == frappe.local.site:
add_job(current_job, current_job.origin)
else:
jobs.append({"queue": worker.name, "job_name": "busy", "status": "", "creation": ""})
else:
jobs.append({"queue": worker.name, "job_name": "idle", "status": "", "creation": ""})
return jobs
@frappe.whitelist()
def remove_failed_jobs():
queues = get_queues()
for queue in queues:
fail_registry = queue.failed_job_registry
for job_id in fail_registry.get_job_ids():
job = queue.fetch_job(job_id)
fail_registry.remove(job, delete_job=True)
@frappe.whitelist()
def get_scheduler_status():
if is_scheduler_inactive():
return {"status": "inactive"}
return {"status": "active"}

51
frappe/core/page/background_jobs/background_workers.html
View file

@ -1,51 +0,0 @@
{% if jobs.length %}
<table class="table table-background-jobs">
<thead>
<tr>
<th style="width: 40%">{{ __("Worker") }}</th>
<th>{{ __("Current Job") }}</th>
<th style="width: 10%">{{ __("Status") }}</th>
<th style="width: 15%">{{ __("Created") }}</th>
</tr>
</thead>
<tbody>
{% for j in jobs %}
<tr>
<td class="worker-name">
{{ j.queue }}
</td>
<td>
<div>
<span class="job-name">
{{ frappe.utils.encode_tags(j.job_name) }}
</span>
</div>
{% if j.exc_info %}
<details>
<summary>{{ __("Exception") }}</summary>
<div class="exc_info">
<pre>{{ frappe.utils.encode_tags(j.exc_info) }}</pre>
</div>
</details>
{% endif %}
</td>
<td>
<span class="indicator-pill {{ j.color }}">{{ toTitle(j.status) }}</span>
</td>
<td class="creation text-muted">{{ frappe.datetime.prettyDate(j.creation) }}</td>
</tr>
{% endfor %}
</tbody>
</table>
{% else %}
<div class="no-background-jobs">
<img src="/assets/frappe/images/ui-states/list-empty-state.svg" alt="Empty State">
<p class="text-muted">{{ __("No workers online on this site") }}</p>
</div>
{% endif %}
<div class="footer">
<div class="text-muted">
{{ __("Last refreshed") }}
{{ frappe.datetime.now_datetime(true).toLocaleString() }}
</div>
</div>

6
frappe/core/workspace/build/build.json
View file

@ -217,8 +217,8 @@
"is_query_report": 0,
"label": "Background Jobs",
"link_count": 0,
"link_to": "background_jobs",
"link_type": "Page",
"link_to": "RQ Job",
"link_type": "DocType",
"onboard": 0,
"type": "Link"
},
@ -273,7 +273,7 @@
"type": "Link"
}
],
"modified": "2022-09-02 01:48:28.029135",
"modified": "2022-09-11 06:41:31.095300",
"modified_by": "Administrator",
"module": "Core",
"name": "Build",

10
frappe/database/__init__.py
View file

@ -39,18 +39,14 @@ def drop_user_and_database(db_name, root_login=None, root_password=None):
)
def get_db(host=None, user=None, password=None, port=None, read_only=False):
def get_db(host=None, user=None, password=None, port=None):
import frappe
if frappe.conf.db_type == "postgres":
import frappe.database.postgres.database
return frappe.database.postgres.database.PostgresDatabase(
host, user, password, port=port, read_only=read_only
)
return frappe.database.postgres.database.PostgresDatabase(host, user, password, port=port)
else:
import frappe.database.mariadb.database
return frappe.database.mariadb.database.MariaDBDatabase(
host, user, password, port=port, read_only=read_only
)
return frappe.database.mariadb.database.MariaDBDatabase(host, user, password, port=port)

36
frappe/database/database.py
View file

@ -83,14 +83,12 @@ class Database:
ac_name=None,
use_default=0,
port=None,
read_only=False,
):
self.setup_type_map()
self.host = host or frappe.conf.db_host or "127.0.0.1"
self.port = port or frappe.conf.db_port or ""
self.user = user or frappe.conf.db_name
self.db_name = frappe.conf.db_name
self.read_only = read_only # Uses READ ONLY connection if set
self._conn = None
if ac_name:
@ -221,6 +219,15 @@ class Database:
elif self.is_timedout(e):
raise frappe.QueryTimeoutError(e) from e
elif self.is_read_only_mode_error(e):
frappe.throw(
_(
"Site is running in read only mode, this action can not be performed right now. Please try again later."
),
title=_("In Read Only Mode"),
exc=frappe.InReadOnlyMode,
)
# TODO: added temporarily
elif self.db_type == "postgres":
traceback.print_stack()
@ -956,8 +963,10 @@ class Database:
return defaults.get(frappe.scrub(key))
def begin(self):
self.sql("START TRANSACTION")
def begin(self, *, read_only=False):
read_only = read_only or frappe.flags.read_only
mode = "READ ONLY" if read_only else ""
self.sql(f"START TRANSACTION {mode}")
def commit(self):
"""Commit current transaction. Calls SQL `COMMIT`."""
@ -965,9 +974,7 @@ class Database:
frappe.call(method[0], *(method[1] or []), **(method[2] or {}))
self.sql("commit")
if self.db_type == "postgres":
# Postgres requires explicitly starting new transaction
self.begin()
self.begin() # explicitly start a new transaction
frappe.local.rollback_observers = []
self.flush_realtime_log()
@ -1298,12 +1305,23 @@ class Database:
def enqueue_jobs_after_commit():
from frappe.utils.background_jobs import execute_job, get_queue
from frappe.utils.background_jobs import (
RQ_JOB_FAILURE_TTL,
RQ_RESULTS_TTL,
execute_job,
get_queue,
)
if frappe.flags.enqueue_after_commit and len(frappe.flags.enqueue_after_commit) > 0:
for job in frappe.flags.enqueue_after_commit:
q = get_queue(job.get("queue"), is_async=job.get("is_async"))
q.enqueue_call(execute_job, timeout=job.get("timeout"), kwargs=job.get("queue_args"))
q.enqueue_call(
execute_job,
timeout=job.get("timeout"),
kwargs=job.get("queue_args"),
failure_ttl=RQ_JOB_FAILURE_TTL,
result_ttl=RQ_RESULTS_TTL,
)
frappe.flags.enqueue_after_commit = []

4
frappe/database/mariadb/database.py
View file

@ -32,6 +32,10 @@ class MariaDBExceptionUtil:
def is_timedout(e: pymysql.Error) -> bool:
return e.args[0] == ER.LOCK_WAIT_TIMEOUT
@staticmethod
def is_read_only_mode_error(e: pymysql.Error) -> bool:
return e.args[0] == 1792
@staticmethod
def is_table_missing(e: pymysql.Error) -> bool:
return e.args[0] == ER.NO_SUCH_TABLE

31
frappe/database/mariadb/schema.py
View file

@ -6,7 +6,7 @@ from frappe.model import log_types
class MariaDBTable(DBTable):
def create(self):
additional_definitions = ""
additional_definitions = []
engine = self.meta.get("engine") or "InnoDB"
varchar_len = frappe.db.VARCHAR_LEN
name_column = f"name varchar({varchar_len}) primary key"
@ -14,26 +14,24 @@ class MariaDBTable(DBTable):
# columns
column_defs = self.get_column_definitions()
if column_defs:
additional_definitions += ",\n".join(column_defs) + ",\n"
additional_definitions += column_defs
# index
index_defs = self.get_index_definitions()
if index_defs:
additional_definitions += ",\n".join(index_defs) + ",\n"
additional_definitions += index_defs
# child table columns
if self.meta.get("istable") or 0:
additional_definitions += (
",\n".join(
(
f"parent varchar({varchar_len})",
f"parentfield varchar({varchar_len})",
f"parenttype varchar({varchar_len})",
"index parent(parent)",
)
)
+ ",\n"
)
additional_definitions += [
f"parent varchar({varchar_len})",
f"parentfield varchar({varchar_len})",
f"parenttype varchar({varchar_len})",
"index parent(parent)",
]
else:
# parent types
additional_definitions.append("index modified(modified)")
# creating sequence(s)
if (
@ -47,6 +45,8 @@ class MariaDBTable(DBTable):
# issue link: https://jira.mariadb.org/browse/MDEV-20070
name_column = "name bigint primary key"
additional_definitions = ",\n".join(additional_definitions)
# create table
query = f"""create table `{self.table_name}` (
{name_column},
@ -56,8 +56,7 @@ class MariaDBTable(DBTable):
owner varchar({varchar_len}),
docstatus int(1) not null default '0',
idx int(8) not null default '0',
{additional_definitions}
index modified(modified))
{additional_definitions})
ENGINE={engine}
ROW_FORMAT=DYNAMIC
CHARACTER SET=utf8mb4

6
frappe/database/postgres/database.py
View file

@ -12,7 +12,7 @@ from psycopg2.errorcodes import (
UNDEFINED_TABLE,
UNIQUE_VIOLATION,
)
from psycopg2.errors import SequenceGeneratorLimitExceeded, SyntaxError
from psycopg2.errors import ReadOnlySqlTransaction, SequenceGeneratorLimitExceeded, SyntaxError
from psycopg2.extensions import ISOLATION_LEVEL_REPEATABLE_READ
import frappe
@ -55,6 +55,10 @@ class PostgresExceptionUtil:
# http://initd.org/psycopg/docs/extensions.html?highlight=datatype#psycopg2.extensions.QueryCanceledError
return isinstance(e, psycopg2.extensions.QueryCanceledError)
@staticmethod
def is_read_only_mode_error(e) -> bool:
return isinstance(e, ReadOnlySqlTransaction)
@staticmethod
def is_syntax_error(e):
return isinstance(e, SyntaxError)

9
frappe/desk/desktop.py
View file

@ -291,12 +291,13 @@ class Workspace:
quick_lists = self.doc.quick_lists
for item in quick_lists:
new_item = item.as_dict().copy()
if self.is_item_allowed(item.document_type, "doctype"):
new_item = item.as_dict().copy()
# Translate label
new_item["label"] = _(item.label) if item.label else _(item.document_type)
# Translate label
new_item["label"] = _(item.label) if item.label else _(item.document_type)
items.append(new_item)
items.append(new_item)
return items

30
frappe/desk/doctype/dashboard_chart/dashboard_chart.js
View file

@ -532,25 +532,21 @@ frappe.ui.form.on("Dashboard Chart", {
frm.set_df_property("parent_document_type", "hidden", !doc_is_table);
if (document_type && doc_is_table) {
let parent = await frappe.db.get_list("DocField", {
filters: {
fieldtype: "Table",
options: document_type,
},
fields: ["parent"],
let parents = await frappe.xcall(
"frappe.desk.doctype.dashboard_chart.dashboard_chart.get_parent_doctypes",
{ child_type: document_type }
);
frm.set_query("parent_document_type", function () {
return {
filters: {
name: ["in", parents],
},
};
});
parent &&
frm.set_query("parent_document_type", function () {
return {
filters: {
name: ["in", parent.map(({ parent }) => parent)],
},
};
});
if (parent.length === 1) {
frm.set_value("parent_document_type", parent[0].parent);
if (parents.length === 1) {
frm.set_value("parent_document_type", parents[0]);
}
}
},

22
frappe/desk/doctype/dashboard_chart/dashboard_chart.py
View file

@ -392,3 +392,25 @@ class DashboardChart(Document):
json.loads(self.custom_options)
except ValueError as error:
frappe.throw(_("Invalid json added in the custom options: {0}").format(error))
@frappe.whitelist()
def get_parent_doctypes(child_type: str) -> list[str]:
"""Get all parent doctypes that have the child doctype."""
assert isinstance(child_type, str)
standard = frappe.get_all(
"DocField",
fields="parent",
filters={"fieldtype": "Table", "options": child_type},
pluck="parent",
)
custom = frappe.get_all(
"Custom Field",
fields="dt",
filters={"fieldtype": "Table", "options": child_type},
pluck="dt",
)
return standard + custom

4
frappe/desk/doctype/event/event.json
View file

@ -221,7 +221,7 @@
"in_list_view": 1,
"in_standard_filter": 1,
"label": "Status",
"options": "Open\nClosed"
"options": "Open\nCompleted\nClosed"
},
{
"collapsible": 1,
@ -318,4 +318,4 @@
"track_changes": 1,
"track_seen": 1,
"track_views": 1
}
}

51
frappe/desk/doctype/notification_log/notification_log.json
View file

@ -22,18 +22,14 @@
"fieldname": "subject",
"fieldtype": "Text",
"in_list_view": 1,
"label": "Subject",
"show_days": 1,
"show_seconds": 1
"label": "Subject"
},
{
"fieldname": "for_user",
"fieldtype": "Link",
"hidden": 1,
"label": "For User",
"options": "User",
"show_days": 1,
"show_seconds": 1
"options": "User"
},
{
"fieldname": "type",
@ -42,36 +38,26 @@
"in_list_view": 1,
"in_standard_filter": 1,
"label": "Type",
"options": "Mention\nEnergy Point\nAssignment\nShare\nAlert",
"search_index": 1,
"show_days": 1,
"show_seconds": 1
"options": "Mention\nEnergy Point\nAssignment\nShare\nAlert"
},
{
"fieldname": "email_content",
"fieldtype": "Text Editor",
"label": "Message",
"show_days": 1,
"show_seconds": 1
"label": "Message"
},
{
"fieldname": "document_type",
"fieldtype": "Link",
"hidden": 1,
"label": "Document Type",
"options": "DocType",
"search_index": 1,
"show_days": 1,
"show_seconds": 1
"options": "DocType"
},
{
"fieldname": "document_name",
"fieldtype": "Data",
"hidden": 1,
"label": "Document Link",
"search_index": 1,
"show_days": 1,
"show_seconds": 1
"search_index": 1
},
{
"fieldname": "from_user",
@ -79,9 +65,7 @@
"hidden": 1,
"label": "From User",
"options": "User",
"search_index": 1,
"show_days": 1,
"show_seconds": 1
"search_index": 1
},
{
"default": "0",
@ -89,38 +73,30 @@
"fieldtype": "Check",
"hidden": 1,
"ignore_user_permissions": 1,
"label": "Read",
"show_days": 1,
"show_seconds": 1
"label": "Read"
},
{
"fieldname": "open_reference_document",
"fieldtype": "Button",
"label": "Open Reference Document",
"show_days": 1,
"show_seconds": 1
"label": "Open Reference Document"
},
{
"fieldname": "attached_file",
"fieldtype": "Code",
"hidden": 1,
"label": "Attached File",
"options": "JSON",
"show_days": 1,
"show_seconds": 1
"options": "JSON"
},
{
"fieldname": "attachment_link",
"fieldtype": "HTML",
"label": "Attachment Link",
"show_days": 1,
"show_seconds": 1
"label": "Attachment Link"
}
],
"hide_toolbar": 1,
"in_create": 1,
"links": [],
"modified": "2021-10-25 17:26:09.703215",
"modified": "2022-09-13 16:08:48.153934",
"modified_by": "Administrator",
"module": "Desk",
"name": "Notification Log",
@ -138,6 +114,7 @@
],
"sort_field": "modified",
"sort_order": "DESC",
"states": [],
"title_field": "subject",
"track_seen": 1
}
}

2
frappe/desk/doctype/notification_log/notification_log.py
View file

@ -136,7 +136,7 @@ def get_email_header(doc):
@frappe.whitelist()
def get_notification_logs(limit=20):
notification_logs = frappe.db.get_list(
"Notification Log", fields=["*"], limit=limit, order_by="creation desc"
"Notification Log", fields=["*"], limit=limit, order_by="modified desc"
)
users = [log.from_user for log in notification_logs]

30
frappe/desk/doctype/number_card/number_card.js
View file

@ -472,25 +472,21 @@ frappe.ui.form.on("Number Card", {
frm.set_df_property("parent_document_type", "hidden", !doc_is_table);
if (document_type && doc_is_table) {
let parent = await frappe.db.get_list("DocField", {
filters: {
fieldtype: "Table",
options: document_type,
},
fields: ["parent"],
let parents = await frappe.xcall(
"frappe.desk.doctype.dashboard_chart.dashboard_chart.get_parent_doctypes",
{ child_type: document_type }
);
frm.set_query("parent_document_type", function () {
return {
filters: {
name: ["in", parents],
},
};
});
parent &&
frm.set_query("parent_document_type", function () {
return {
filters: {
name: ["in", parent.map(({ parent }) => parent)],
},
};
});
if (parent.length === 1) {
frm.set_value("parent_document_type", parent[0].parent);
if (parents.length === 1) {
frm.set_value("parent_document_type", parents[0]);
}
}
},

1
frappe/desk/doctype/workspace/workspace.py
View file

@ -266,6 +266,7 @@ def duplicate_page(page_name, new_page):
doc.public = new_page.get("is_public")
doc.for_user = ""
doc.label = doc.title
doc.module = ""
if not doc.public:
doc.for_user = doc.for_user or frappe.session.user
doc.label = f"{doc.title}-{doc.for_user}"

12
frappe/desk/form/load.py
View file

@ -64,11 +64,9 @@ def getdoctype(doctype, with_parent=False, cached_timestamp=None):
parent_dt = None
# with parent (called from report builder)
if with_parent:
parent_dt = frappe.model.meta.get_parent_dt(doctype)
if parent_dt:
docs = get_meta_bundle(parent_dt)
frappe.response["parent_dt"] = parent_dt
if with_parent and (parent_dt := frappe.model.meta.get_parent_dt(doctype)):
docs = get_meta_bundle(parent_dt)
frappe.response["parent_dt"] = parent_dt
if not docs:
docs = get_meta_bundle(doctype)
@ -110,6 +108,8 @@ def get_docinfo(doc=None, doctype=None, name=None):
docinfo.update(
{
"doctype": doc.doctype,
"name": doc.name,
"attachments": get_attachments(doc.doctype, doc.name),
"communications": communications_except_auto_messages,
"automated_messages": automated_messages,
@ -373,7 +373,7 @@ def run_onload(doc):
def get_view_logs(doctype, docname):
"""get and return the latest view logs if available"""
logs = []
if hasattr(frappe.get_meta(doctype), "track_views") and frappe.get_meta(doctype).track_views:
if getattr(frappe.get_meta(doctype), "track_views", None):
view_logs = frappe.get_all(
"View Log",
filters={

8
frappe/desk/form/meta.py
View file

@ -35,12 +35,10 @@ ASSET_KEYS = (
def get_meta(doctype, cached=True):
# don't cache for developer mode as js files, templates may be edited
if cached and not frappe.conf.developer_mode:
meta = frappe.cache().hget("form_meta", doctype)
if meta:
meta = FormMeta(meta)
else:
meta = frappe.cache().hget("doctype_form_meta", doctype)
if not meta:
meta = FormMeta(doctype)
frappe.cache().hset("form_meta", doctype, meta.as_dict())
frappe.cache().hset("doctype_form_meta", doctype, meta)
else:
meta = FormMeta(doctype)

9
frappe/email/doctype/auto_email_report/auto_email_report.js
View file

@ -35,6 +35,15 @@ frappe.ui.form.on("Auto Email Report", {
frm.set_value("email_to", frappe.session.user);
}
}
frm.set_query("sender", function () {
return {
filters: {
enable_outgoing: 1,
awaiting_password: 0,
},
};
});
},
report: function (frm) {
frm.set_value("filters", "");

484
frappe/email/doctype/auto_email_report/auto_email_report.json
View file

@ -1,238 +1,248 @@
{
"allow_rename": 1,
"creation": "2016-09-01 01:34:34.985457",
"doctype": "DocType",
"editable_grid": 1,
"engine": "InnoDB",
"field_order": [
"report",
"user",
"enabled",
"column_break_4",
"report_type",
"reference_report",
"filter_data",
"send_if_data",
"data_modified_till",
"no_of_rows",
"report_filters",
"filters_display",
"filters",
"filter_meta",
"dynamic_report_filters_section",
"from_date_field",
"to_date_field",
"column_break_17",
"dynamic_date_period",
"email_settings",
"email_to",
"day_of_week",
"column_break_13",
"frequency",
"format",
"section_break_15",
"description"
],
"fields": [
{
"fieldname": "report",
"fieldtype": "Link",
"label": "Report",
"options": "Report",
"reqd": 1
},
{
"default": "User",
"fieldname": "user",
"fieldtype": "Link",
"label": "Based on Permissions For User",
"options": "User",
"reqd": 1
},
{
"default": "1",
"fieldname": "enabled",
"fieldtype": "Check",
"label": "Enabled"
},
{
"fieldname": "column_break_4",
"fieldtype": "Column Break"
},
{
"fetch_from": "report.report_type",
"fieldname": "report_type",
"fieldtype": "Read Only",
"label": "Report Type"
},
{
"fieldname": "filter_data",
"fieldtype": "Section Break",
"label": "Filter Data"
},
{
"default": "1",
"fieldname": "send_if_data",
"fieldtype": "Check",
"label": "Send only if there is any data"
},
{
"depends_on": "eval:doc.report_type=='Report Builder'",
"description": "Zero means send records updated at anytime",
"fieldname": "data_modified_till",
"fieldtype": "Int",
"label": "Only Send Records Updated in Last X Hours"
},
{
"default": "100",
"fieldname": "no_of_rows",
"fieldtype": "Int",
"label": "No of Rows (Max 500)"
},
{
"collapsible": 1,
"depends_on": "eval:doc.report_type !== 'Report Builder'",
"fieldname": "report_filters",
"fieldtype": "Section Break",
"label": "Report Filters"
},
{
"fieldname": "filters_display",
"fieldtype": "HTML",
"label": "Filters Display"
},
{
"fieldname": "filters",
"fieldtype": "Text",
"hidden": 1,
"label": "Filters"
},
{
"fieldname": "filter_meta",
"fieldtype": "Text",
"hidden": 1,
"label": "Filter Meta",
"read_only": 1
},
{
"collapsible": 1,
"depends_on": "eval:doc.report_type !== 'Report Builder'",
"fieldname": "dynamic_report_filters_section",
"fieldtype": "Section Break",
"label": "Dynamic Report Filters"
},
{
"fieldname": "from_date_field",
"fieldtype": "Select",
"label": "From Date Field"
},
{
"fieldname": "to_date_field",
"fieldtype": "Select",
"label": "To Date Field"
},
{
"fieldname": "column_break_17",
"fieldtype": "Column Break"
},
{
"fieldname": "dynamic_date_period",
"fieldtype": "Select",
"label": "Period",
"options": "\nDaily\nWeekly\nMonthly\nQuarterly\nHalf Yearly\nYearly"
},
{
"fieldname": "email_settings",
"fieldtype": "Section Break",
"label": "Email Settings"
},
{
"description": "For multiple addresses, enter the address on different line. e.g. test@test.com \u23ce test1@test.com",
"fieldname": "email_to",
"fieldtype": "Small Text",
"label": "Email To",
"reqd": 1
},
{
"default": "Monday",
"depends_on": "eval:doc.frequency=='Weekly'",
"fieldname": "day_of_week",
"fieldtype": "Select",
"label": "Day of Week",
"options": "Monday\nTuesday\nWednesday\nThursday\nFriday\nSaturday\nSunday"
},
{
"fieldname": "column_break_13",
"fieldtype": "Column Break"
},
{
"fieldname": "frequency",
"fieldtype": "Select",
"in_list_view": 1,
"in_standard_filter": 1,
"label": "Frequency",
"options": "Daily\nWeekdays\nWeekly\nMonthly",
"reqd": 1
},
{
"fieldname": "format",
"fieldtype": "Select",
"label": "Format",
"options": "HTML\nXLSX\nCSV",
"reqd": 1
},
{
"collapsible": 1,
"fieldname": "section_break_15",
"fieldtype": "Section Break",
"label": "Message"
},
{
"fieldname": "description",
"fieldtype": "Text Editor",
"label": "Message"
},
{
"fetch_from": "report.reference_report",
"fieldname": "reference_report",
"fieldtype": "Data",
"hidden": 1,
"label": "Reference Report",
"read_only": 1
}
],
"modified": "2021-01-28 15:59:43.151995",
"modified_by": "Administrator",
"module": "Email",
"name": "Auto Email Report",
"owner": "Administrator",
"permissions": [
{
"create": 1,
"delete": 1,
"email": 1,
"export": 1,
"print": 1,
"read": 1,
"report": 1,
"role": "System Manager",
"share": 1,
"write": 1
},
{
"create": 1,
"delete": 1,
"email": 1,
"export": 1,
"print": 1,
"read": 1,
"report": 1,
"role": "Report Manager",
"share": 1,
"write": 1
}
],
"sort_field": "modified",
"sort_order": "DESC",
"track_changes": 1
}
"actions": [],
"allow_rename": 1,
"creation": "2016-09-01 01:34:34.985457",
"doctype": "DocType",
"editable_grid": 1,
"engine": "InnoDB",
"field_order": [
"report",
"user",
"enabled",
"column_break_4",
"report_type",
"reference_report",
"filter_data",
"send_if_data",
"data_modified_till",
"no_of_rows",
"report_filters",
"filters_display",
"filters",
"filter_meta",
"dynamic_report_filters_section",
"from_date_field",
"to_date_field",
"column_break_17",
"dynamic_date_period",
"email_settings",
"email_to",
"day_of_week",
"column_break_13",
"sender",
"frequency",
"format",
"section_break_15",
"description"
],
"fields": [
{
"fieldname": "report",
"fieldtype": "Link",
"label": "Report",
"options": "Report",
"reqd": 1
},
{
"default": "User",
"fieldname": "user",
"fieldtype": "Link",
"label": "Based on Permissions For User",
"options": "User",
"reqd": 1
},
{
"default": "1",
"fieldname": "enabled",
"fieldtype": "Check",
"label": "Enabled"
},
{
"fieldname": "column_break_4",
"fieldtype": "Column Break"
},
{
"fetch_from": "report.report_type",
"fieldname": "report_type",
"fieldtype": "Read Only",
"label": "Report Type"
},
{
"fieldname": "filter_data",
"fieldtype": "Section Break",
"label": "Filter Data"
},
{
"default": "1",
"fieldname": "send_if_data",
"fieldtype": "Check",
"label": "Send only if there is any data"
},
{
"depends_on": "eval:doc.report_type=='Report Builder'",
"description": "Zero means send records updated at anytime",
"fieldname": "data_modified_till",
"fieldtype": "Int",
"label": "Only Send Records Updated in Last X Hours"
},
{
"default": "100",
"fieldname": "no_of_rows",
"fieldtype": "Int",
"label": "No of Rows (Max 500)"
},
{
"collapsible": 1,
"depends_on": "eval:doc.report_type !== 'Report Builder'",
"fieldname": "report_filters",
"fieldtype": "Section Break",
"label": "Report Filters"
},
{
"fieldname": "filters_display",
"fieldtype": "HTML",
"label": "Filters Display"
},
{
"fieldname": "filters",
"fieldtype": "Text",
"hidden": 1,
"label": "Filters"
},
{
"fieldname": "filter_meta",
"fieldtype": "Text",
"hidden": 1,
"label": "Filter Meta",
"read_only": 1
},
{
"collapsible": 1,
"depends_on": "eval:doc.report_type !== 'Report Builder'",
"fieldname": "dynamic_report_filters_section",
"fieldtype": "Section Break",
"label": "Dynamic Report Filters"
},
{
"fieldname": "from_date_field",
"fieldtype": "Select",
"label": "From Date Field"
},
{
"fieldname": "to_date_field",
"fieldtype": "Select",
"label": "To Date Field"
},
{
"fieldname": "column_break_17",
"fieldtype": "Column Break"
},
{
"fieldname": "dynamic_date_period",
"fieldtype": "Select",
"label": "Period",
"options": "\nDaily\nWeekly\nMonthly\nQuarterly\nHalf Yearly\nYearly"
},
{
"fieldname": "email_settings",
"fieldtype": "Section Break",
"label": "Email Settings"
},
{
"description": "For multiple addresses, enter the address on different line. e.g. test@test.com \u23ce test1@test.com",
"fieldname": "email_to",
"fieldtype": "Small Text",
"label": "Email To",
"reqd": 1
},
{
"default": "Monday",
"depends_on": "eval:doc.frequency=='Weekly'",
"fieldname": "day_of_week",
"fieldtype": "Select",
"label": "Day of Week",
"options": "Monday\nTuesday\nWednesday\nThursday\nFriday\nSaturday\nSunday"
},
{
"fieldname": "column_break_13",
"fieldtype": "Column Break"
},
{
"fieldname": "frequency",
"fieldtype": "Select",
"in_list_view": 1,
"in_standard_filter": 1,
"label": "Frequency",
"options": "Daily\nWeekdays\nWeekly\nMonthly",
"reqd": 1
},
{
"fieldname": "format",
"fieldtype": "Select",
"label": "Format",
"options": "HTML\nXLSX\nCSV",
"reqd": 1
},
{
"collapsible": 1,
"fieldname": "section_break_15",
"fieldtype": "Section Break",
"label": "Message"
},
{
"fieldname": "description",
"fieldtype": "Text Editor",
"label": "Message"
},
{
"fetch_from": "report.reference_report",
"fieldname": "reference_report",
"fieldtype": "Data",
"hidden": 1,
"label": "Reference Report",
"read_only": 1
},
{
"fieldname": "sender",
"fieldtype": "Link",
"label": "Sender",
"options": "Email Account"
}
],
"links": [],
"modified": "2022-09-08 15:31:55.031023",
"modified_by": "Administrator",
"module": "Email",
"name": "Auto Email Report",
"owner": "Administrator",
"permissions": [
{
"create": 1,
"delete": 1,
"email": 1,
"export": 1,
"print": 1,
"read": 1,
"report": 1,
"role": "System Manager",
"share": 1,
"write": 1
},
{
"create": 1,
"delete": 1,
"email": 1,
"export": 1,
"print": 1,
"read": 1,
"report": 1,
"role": "Report Manager",
"share": 1,
"write": 1
}
],
"sort_field": "modified",
"sort_order": "DESC",
"states": [],
"track_changes": 1
}

7
frappe/email/doctype/auto_email_report/auto_email_report.py
View file

@ -3,6 +3,7 @@
import calendar
from datetime import timedelta
from email.utils import formataddr
import frappe
from frappe import _
@ -37,6 +38,11 @@ class AutoEmailReport(Document):
self.validate_report_format()
self.validate_mandatory_fields()
@property
def sender_email(self):
email_id, login_id = frappe.db.get_value("Email Account", self.sender, ["email_id", "login_id"])
return login_id if login_id else email_id
def validate_emails(self):
"""Cleanup list of emails"""
if "," in self.email_to:
@ -203,6 +209,7 @@ class AutoEmailReport(Document):
frappe.sendmail(
recipients=self.email_to.split(),
sender=formataddr((self.sender, self.sender_email)) if self.sender else "",
subject=self.name,
message=message,
attachments=attachments,

4
frappe/email/doctype/email_queue_recipient/email_queue_recipient.json
View file

@ -22,7 +22,7 @@
"fieldtype": "Select",
"in_list_view": 1,
"label": "Status",
"options": "\nNot Sent\nSending\nSent\nError\nExpired",
"options": "\nNot Sent\nSent",
"search_index": 1
},
{
@ -33,7 +33,7 @@
],
"istable": 1,
"links": [],
"modified": "2022-07-11 16:38:10.644417",
"modified": "2022-09-06 13:38:10.644417",
"modified_by": "Administrator",
"module": "Email",
"name": "Email Queue Recipient",

11
frappe/email/doctype/newsletter/newsletter.py
View file

@ -128,15 +128,11 @@ class Newsletter(WebsiteGenerator):
pluck="name",
)
def get_success_recipients(self) -> list[str]:
"""Recipients who have already received the newsletter.
Couldn't think of a better name ;)
"""
def get_queued_recipients(self) -> list[str]:
"""Recipients who have already been queued for receiving the newsletter."""
return frappe.get_all(
"Email Queue Recipient",
filters={
"status": ("in", ["Not Sent", "Sending", "Sent"]),
"parent": ("in", self.get_linked_email_queue()),
},
pluck="recipient",
@ -146,8 +142,7 @@ class Newsletter(WebsiteGenerator):
"""Get list of pending recipients of the newsletter. These
recipients may not have receive the newsletter in the previous iteration.
"""
success_recipients = set(self.get_success_recipients())
return [x for x in self.newsletter_recipients if x not in success_recipients]
return [x for x in self.newsletter_recipients if x not in self.get_queued_recipients()]
def queue_all(self):
"""Queue Newsletter to all the recipients generated from the `Email Group` table"""

14
frappe/email/doctype/newsletter/test_newsletter.py
View file

@ -236,13 +236,15 @@ class TestNewsletter(TestNewsletterMixin, FrappeTestCase):
email_queue_list = [frappe.get_doc("Email Queue", e.name) for e in frappe.get_all("Email Queue")]
self.assertEqual(len(email_queue_list), 4)
# emulate partial send
email_queue_list[0].status = "Error"
email_queue_list[0].recipients[0].status = "Error"
email_queue_list[0].save()
# delete a queue document to emulate partial send
queue_recipient_name = email_queue_list[0].recipients[0].recipient
email_queue_list[0].delete()
newsletter.email_sent = False
# make sure the pending recipient is only the one which has been deleted
self.assertEqual(newsletter.get_pending_recipients(), [queue_recipient_name])
# retry
newsletter.send_emails()
email_queue_list = [frappe.get_doc("Email Queue", e.name) for e in frappe.get_all("Email Queue")]
self.assertEqual(len(email_queue_list), 5)
self.assertEqual(frappe.db.count("Email Queue"), 4)
self.assertTrue(newsletter.email_sent)

2
frappe/email/doctype/notification/notification.js
View file

@ -175,7 +175,7 @@ frappe.ui.form.on("Notification", {
notification: frm.doc.name,
},
callback: function (r) {
if (r.message) {
if (r.message && r.message.length > 0) {
frappe.msgprint(r.message);
} else {
frappe.msgprint(__("No alerts for today"));

0
frappe/event_streaming/doctype/__init__.py
View file

0
frappe/event_streaming/doctype/document_type_field_mapping/__init__.py
View file

73
frappe/event_streaming/doctype/document_type_field_mapping/document_type_field_mapping.json
View file

@ -1,73 +0,0 @@
{
"actions": [],
"creation": "2019-09-27 12:46:50.165135",
"doctype": "DocType",
"editable_grid": 1,
"engine": "InnoDB",
"field_order": [
"local_fieldname",
"mapping_type",
"mapping",
"remote_value_filters",
"column_break_5",
"remote_fieldname",
"default_value"
],
"fields": [
{
"fieldname": "remote_fieldname",
"fieldtype": "Data",
"in_list_view": 1,
"label": "Remote Fieldname"
},
{
"fieldname": "local_fieldname",
"fieldtype": "Data",
"in_list_view": 1,
"label": "Local Fieldname",
"reqd": 1
},
{
"fieldname": "column_break_5",
"fieldtype": "Column Break"
},
{
"fieldname": "default_value",
"fieldtype": "Data",
"label": "Default Value"
},
{
"fieldname": "mapping_type",
"fieldtype": "Select",
"label": "Mapping Type",
"options": "\nChild Table\nDocument"
},
{
"depends_on": "eval:doc.mapping_type;",
"fieldname": "mapping",
"fieldtype": "Link",
"label": "Mapping",
"options": "Document Type Mapping"
},
{
"depends_on": "eval:doc.mapping_type==\"Document\";",
"fieldname": "remote_value_filters",
"fieldtype": "Code",
"label": "Remote Value Filters",
"mandatory_depends_on": "eval:doc.mapping_type===\"Document\";",
"options": "JSON"
}
],
"istable": 1,
"links": [],
"modified": "2020-03-19 13:56:36.223799",
"modified_by": "Administrator",
"module": "Event Streaming",
"name": "Document Type Field Mapping",
"owner": "Administrator",
"permissions": [],
"quick_entry": 1,
"sort_field": "modified",
"sort_order": "DESC",
"track_changes": 1
}

9
frappe/event_streaming/doctype/document_type_field_mapping/document_type_field_mapping.py
View file

@ -1,9 +0,0 @@
# Copyright (c) 2019, Frappe Technologies and contributors
# License: MIT. See LICENSE
# import frappe
from frappe.model.document import Document
class DocumentTypeFieldMapping(Document):
pass

0
frappe/event_streaming/doctype/document_type_mapping/__init__.py
View file

37
frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.js
View file

@ -1,37 +0,0 @@
// Copyright (c) 2019, Frappe Technologies and contributors
// For license information, please see license.txt
frappe.ui.form.on("Document Type Mapping", {
local_doctype: function (frm) {
if (frm.doc.local_doctype) {
frappe.model.clear_table(frm.doc, "field_mapping");
let fields = frm.events.get_fields(frm);
$.each(fields, function (i, data) {
let row = frappe.model.add_child(
frm.doc,
"Document Type Field Mapping",
"field_mapping"
);
row.local_fieldname = data;
});
refresh_field("field_mapping");
}
},
get_fields: function (frm) {
let filtered_fields = [];
frappe.model.with_doctype(frm.doc.local_doctype, () => {
frappe.get_meta(frm.doc.local_doctype).fields.map((field) => {
if (
field.fieldname !== "remote_docname" &&
field.fieldname !== "remote_site_name" &&
frappe.model.is_value_type(field) &&
!field.hidden
) {
filtered_fields.push(field.fieldname);
}
});
});
return filtered_fields;
},
});

71
frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.json
View file

@ -1,71 +0,0 @@
{
"autoname": "field:mapping_name",
"creation": "2019-09-27 12:45:56.529124",
"doctype": "DocType",
"editable_grid": 1,
"engine": "InnoDB",
"field_order": [
"mapping_name",
"local_doctype",
"remote_doctype",
"section_break_3",
"field_mapping"
],
"fields": [
{
"fieldname": "local_doctype",
"fieldtype": "Link",
"in_list_view": 1,
"label": "Local Document Type",
"options": "DocType",
"reqd": 1
},
{
"fieldname": "remote_doctype",
"fieldtype": "Data",
"in_list_view": 1,
"label": "Remote Document Type",
"reqd": 1
},
{
"fieldname": "section_break_3",
"fieldtype": "Section Break"
},
{
"fieldname": "field_mapping",
"fieldtype": "Table",
"label": "Field Mapping",
"options": "Document Type Field Mapping"
},
{
"fieldname": "mapping_name",
"fieldtype": "Data",
"label": "Mapping Name",
"reqd": 1,
"unique": 1
}
],
"modified": "2019-10-09 08:36:04.621397",
"modified_by": "Administrator",
"module": "Event Streaming",
"name": "Document Type Mapping",
"owner": "Administrator",
"permissions": [
{
"create": 1,
"delete": 1,
"email": 1,
"export": 1,
"print": 1,
"read": 1,
"report": 1,
"role": "System Manager",
"share": 1,
"write": 1
}
],
"quick_entry": 1,
"sort_field": "modified",
"sort_order": "DESC",
"track_changes": 1
}

181
frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.py
View file

@ -1,181 +0,0 @@
# Copyright (c) 2019, Frappe Technologies and contributors
# License: MIT. See LICENSE
import json
import frappe
from frappe import _
from frappe.model import child_table_fields, default_fields
from frappe.model.document import Document
class DocumentTypeMapping(Document):
def validate(self):
self.validate_inner_mapping()
def validate_inner_mapping(self):
meta = frappe.get_meta(self.local_doctype)
for field_map in self.field_mapping:
if field_map.local_fieldname not in (default_fields + child_table_fields):
field = meta.get_field(field_map.local_fieldname)
if not field:
frappe.throw(_("Row #{0}: Invalid Local Fieldname").format(field_map.idx))
fieldtype = field.get("fieldtype")
if fieldtype in ["Link", "Dynamic Link", "Table"]:
if not field_map.mapping and not field_map.default_value:
msg = _(
"Row #{0}: Please set Mapping or Default Value for the field {1} since its a dependency field"
).format(field_map.idx, frappe.bold(field_map.local_fieldname))
frappe.throw(msg, title="Inner Mapping Missing")
if field_map.mapping_type == "Document" and not field_map.remote_value_filters:
msg = _(
"Row #{0}: Please set remote value filters for the field {1} to fetch the unique remote dependency document"
).format(field_map.idx, frappe.bold(field_map.remote_fieldname))
frappe.throw(msg, title="Remote Value Filters Missing")
def get_mapping(self, doc, producer_site, update_type):
remote_fields = []
# list of tuples (local_fieldname, dependent_doc)
dependencies = []
for mapping in self.field_mapping:
if doc.get(mapping.remote_fieldname):
if mapping.mapping_type == "Document":
if not mapping.default_value:
dependency = self.get_mapped_dependency(mapping, producer_site, doc)
if dependency:
dependencies.append((mapping.local_fieldname, dependency))
else:
doc[mapping.local_fieldname] = mapping.default_value
if mapping.mapping_type == "Child Table" and update_type != "Update":
doc[mapping.local_fieldname] = get_mapped_child_table_docs(
mapping.mapping, doc[mapping.remote_fieldname], producer_site
)
else:
# copy value into local fieldname key and remove remote fieldname key
doc[mapping.local_fieldname] = doc[mapping.remote_fieldname]
if mapping.local_fieldname != mapping.remote_fieldname:
remote_fields.append(mapping.remote_fieldname)
if not doc.get(mapping.remote_fieldname) and mapping.default_value and update_type != "Update":
doc[mapping.local_fieldname] = mapping.default_value
# remove the remote fieldnames
for field in remote_fields:
doc.pop(field, None)
if update_type != "Update":
doc["doctype"] = self.local_doctype
mapping = {"doc": frappe.as_json(doc)}
if len(dependencies):
mapping["dependencies"] = dependencies
return mapping
def get_mapped_update(self, update, producer_site):
update_diff = frappe._dict(json.loads(update.data))
mapping = update_diff
dependencies = []
if update_diff.changed:
doc_map = self.get_mapping(update_diff.changed, producer_site, "Update")
mapped_doc = doc_map.get("doc")
mapping.changed = json.loads(mapped_doc)
if doc_map.get("dependencies"):
dependencies += doc_map.get("dependencies")
if update_diff.removed:
mapping = self.map_rows_removed(update_diff, mapping)
if update_diff.added:
mapping = self.map_rows(update_diff, mapping, producer_site, operation="added")
if update_diff.row_changed:
mapping = self.map_rows(update_diff, mapping, producer_site, operation="row_changed")
update = {"doc": frappe.as_json(mapping)}
if len(dependencies):
update["dependencies"] = dependencies
return update
def get_mapped_dependency(self, mapping, producer_site, doc):
inner_mapping = frappe.get_doc("Document Type Mapping", mapping.mapping)
filters = json.loads(mapping.remote_value_filters)
for key, value in filters.items():
if value.startswith("eval:"):
val = frappe.safe_eval(value[5:], None, dict(doc=doc))
filters[key] = val
if doc.get(value):
filters[key] = doc.get(value)
matching_docs = producer_site.get_doc(inner_mapping.remote_doctype, filters=filters)
if len(matching_docs):
remote_docname = matching_docs[0].get("name")
remote_doc = producer_site.get_doc(inner_mapping.remote_doctype, remote_docname)
doc = inner_mapping.get_mapping(remote_doc, producer_site, "Insert").get("doc")
return doc
return
def map_rows_removed(self, update_diff, mapping):
removed = []
mapping["removed"] = update_diff.removed
for key, value in update_diff.removed.copy().items():
local_table_name = frappe.db.get_value(
"Document Type Field Mapping",
{"remote_fieldname": key, "parent": self.name},
"local_fieldname",
)
mapping.removed[local_table_name] = value
if local_table_name != key:
removed.append(key)
# remove the remote fieldnames
for field in removed:
mapping.removed.pop(field, None)
return mapping
def map_rows(self, update_diff, mapping, producer_site, operation):
remote_fields = []
for tablename, entries in update_diff.get(operation).copy().items():
local_table_name = frappe.db.get_value(
"Document Type Field Mapping", {"remote_fieldname": tablename}, "local_fieldname"
)
table_map = frappe.db.get_value(
"Document Type Field Mapping",
{"local_fieldname": local_table_name, "parent": self.name},
"mapping",
)
table_map = frappe.get_doc("Document Type Mapping", table_map)
docs = []
for entry in entries:
mapped_doc = table_map.get_mapping(entry, producer_site, "Update").get("doc")
docs.append(json.loads(mapped_doc))
mapping.get(operation)[local_table_name] = docs
if local_table_name != tablename:
remote_fields.append(tablename)
# remove the remote fieldnames
for field in remote_fields:
mapping.get(operation).pop(field, None)
return mapping
def get_mapped_child_table_docs(child_map, table_entries, producer_site):
"""Get mapping for child doctypes"""
child_map = frappe.get_doc("Document Type Mapping", child_map)
mapped_entries = []
remote_fields = []
for child_doc in table_entries:
for mapping in child_map.field_mapping:
if child_doc.get(mapping.remote_fieldname):
child_doc[mapping.local_fieldname] = child_doc[mapping.remote_fieldname]
if mapping.local_fieldname != mapping.remote_fieldname:
child_doc.pop(mapping.remote_fieldname, None)
mapped_entries.append(child_doc)
# remove the remote fieldnames
for field in remote_fields:
child_doc.pop(field, None)
child_doc["doctype"] = child_map.local_doctype
return mapped_entries

8
frappe/event_streaming/doctype/document_type_mapping/test_document_type_mapping.py
View file

@ -1,8 +0,0 @@
# Copyright (c) 2019, Frappe Technologies and Contributors
# License: MIT. See LICENSE
# import frappe
from frappe.tests.utils import FrappeTestCase
class TestDocumentTypeMapping(FrappeTestCase):
pass

0
frappe/event_streaming/doctype/event_consumer/__init__.py
View file

17
frappe/event_streaming/doctype/event_consumer/event_consumer.js
View file

@ -1,17 +0,0 @@
// Copyright (c) 2019, Frappe Technologies and contributors
// For license information, please see license.txt
frappe.ui.form.on("Event Consumer", {
refresh: function (frm) {
// formatter for subscribed doctype approval status
frm.set_indicator_formatter("status", function (doc) {
let indicator = "orange";
if (doc.status == "Approved") {
indicator = "green";
} else if (doc.status == "Rejected") {
indicator = "red";
}
return indicator;
});
},
});

97
frappe/event_streaming/doctype/event_consumer/event_consumer.json
View file

@ -1,97 +0,0 @@
{
"actions": [],
"autoname": "field:callback_url",
"creation": "2019-08-26 17:45:15.479530",
"doctype": "DocType",
"editable_grid": 1,
"engine": "InnoDB",
"field_order": [
"consumer_doctypes",
"callback_url",
"section_break_3",
"api_key",
"api_secret",
"column_break_6",
"user",
"incoming_change"
],
"fields": [
{
"fieldname": "callback_url",
"fieldtype": "Data",
"in_list_view": 1,
"label": "Callback URL",
"read_only": 1,
"reqd": 1,
"unique": 1
},
{
"fieldname": "api_key",
"fieldtype": "Data",
"label": "API Key",
"reqd": 1
},
{
"fieldname": "api_secret",
"fieldtype": "Password",
"label": "API Secret",
"reqd": 1
},
{
"fieldname": "user",
"fieldtype": "Link",
"label": "Event Subscriber",
"options": "User",
"read_only": 1,
"reqd": 1
},
{
"fieldname": "section_break_3",
"fieldtype": "Section Break"
},
{
"fieldname": "column_break_6",
"fieldtype": "Column Break"
},
{
"default": "0",
"fieldname": "incoming_change",
"fieldtype": "Check",
"hidden": 1,
"label": "Incoming Change",
"read_only": 1
},
{
"fieldname": "consumer_doctypes",
"fieldtype": "Table",
"label": "Event Consumer Document Types",
"options": "Event Consumer Document Type",
"reqd": 1
}
],
"in_create": 1,
"links": [],
"modified": "2020-09-08 16:42:39.828085",
"modified_by": "Administrator",
"module": "Event Streaming",
"name": "Event Consumer",
"owner": "Administrator",
"permissions": [
{
"create": 1,
"delete": 1,
"email": 1,
"export": 1,
"print": 1,
"read": 1,
"report": 1,
"role": "System Manager",
"share": 1,
"write": 1
}
],
"quick_entry": 1,
"sort_field": "modified",
"sort_order": "DESC",
"track_changes": 1
}

216
frappe/event_streaming/doctype/event_consumer/event_consumer.py
View file

@ -1,216 +0,0 @@
# Copyright (c) 2019, Frappe Technologies and contributors
# License: MIT. See LICENSE
import json
import os
import requests
import frappe
from frappe import _
from frappe.frappeclient import FrappeClient
from frappe.model.document import Document
from frappe.utils.background_jobs import get_jobs
from frappe.utils.data import get_url
class EventConsumer(Document):
def validate(self):
# approve subscribed doctypes for tests
# frappe.flags.in_test won't work here as tests are running on the consumer site
if os.environ.get("CI"):
for entry in self.consumer_doctypes:
entry.status = "Approved"
def on_update(self):
if not self.incoming_change:
doc_before_save = self.get_doc_before_save()
if doc_before_save.api_key != self.api_key or doc_before_save.api_secret != self.api_secret:
return
self.update_consumer_status()
else:
frappe.db.set_value(self.doctype, self.name, "incoming_change", 0)
frappe.cache().delete_value("event_consumer_document_type_map")
def on_trash(self):
for i in frappe.get_all("Event Update Log Consumer", {"consumer": self.name}):
frappe.delete_doc("Event Update Log Consumer", i.name)
frappe.cache().delete_value("event_consumer_document_type_map")
def update_consumer_status(self):
consumer_site = get_consumer_site(self.callback_url)
event_producer = consumer_site.get_doc("Event Producer", get_url())
event_producer = frappe._dict(event_producer)
config = event_producer.producer_doctypes
event_producer.producer_doctypes = []
for entry in config:
if entry.get("has_mapping"):
ref_doctype = consumer_site.get_value(
"Document Type Mapping", "remote_doctype", entry.get("mapping")
).get("remote_doctype")
else:
ref_doctype = entry.get("ref_doctype")
entry["status"] = frappe.db.get_value(
"Event Consumer Document Type", {"parent": self.name, "ref_doctype": ref_doctype}, "status"
)
event_producer.producer_doctypes = config
# when producer doc is updated it updates the consumer doc
# set flag to avoid deadlock
event_producer.incoming_change = True
consumer_site.update(event_producer)
def get_consumer_status(self):
response = requests.get(self.callback_url)
if response.status_code != 200:
return "offline"
return "online"
@frappe.whitelist()
def register_consumer(data):
"""create an event consumer document for registering a consumer"""
data = json.loads(data)
# to ensure that consumer is created only once
if frappe.db.exists("Event Consumer", data["event_consumer"]):
return None
user = data["user"]
if not frappe.db.exists("User", user):
frappe.throw(_("User {0} not found on the producer site").format(user))
if "System Manager" not in frappe.get_roles(user):
frappe.throw(_("Event Subscriber has to be a System Manager."))
consumer = frappe.new_doc("Event Consumer")
consumer.callback_url = data["event_consumer"]
consumer.user = data["user"]
consumer.api_key = data["api_key"]
consumer.api_secret = data["api_secret"]
consumer.incoming_change = True
consumer_doctypes = json.loads(data["consumer_doctypes"])
for entry in consumer_doctypes:
consumer.append(
"consumer_doctypes",
{"ref_doctype": entry.get("doctype"), "status": "Pending", "condition": entry.get("condition")},
)
consumer.insert()
# consumer's 'last_update' field should point to the latest update
# in producer's update log when subscribing
# so that, updates after subscribing are consumed and not the old ones.
last_update = str(get_last_update())
return json.dumps({"last_update": last_update})
def get_consumer_site(consumer_url):
"""create a FrappeClient object for event consumer site"""
consumer_doc = frappe.get_doc("Event Consumer", consumer_url)
consumer_site = FrappeClient(
url=consumer_url,
api_key=consumer_doc.api_key,
api_secret=consumer_doc.get_password("api_secret"),
)
return consumer_site
def get_last_update():
"""get the creation timestamp of last update consumed"""
updates = frappe.get_list(
"Event Update Log", "creation", ignore_permissions=True, limit=1, order_by="creation desc"
)
if updates:
return updates[0].creation
return frappe.utils.now_datetime()
@frappe.whitelist()
def notify_event_consumers(doctype):
"""get all event consumers and set flag for notification status"""
event_consumers = frappe.get_all(
"Event Consumer Document Type", ["parent"], {"ref_doctype": doctype, "status": "Approved"}
)
for entry in event_consumers:
consumer = frappe.get_doc("Event Consumer", entry.parent)
consumer.flags.notified = False
notify(consumer)
@frappe.whitelist()
def notify(consumer):
"""notify individual event consumers about a new update"""
consumer_status = consumer.get_consumer_status()
if consumer_status == "online":
try:
client = get_consumer_site(consumer.callback_url)
client.post_request(
{
"cmd": "frappe.event_streaming.doctype.event_producer.event_producer.new_event_notification",
"producer_url": get_url(),
}
)
consumer.flags.notified = True
except Exception:
consumer.flags.notified = False
else:
consumer.flags.notified = False
# enqueue another job if the site was not notified
if not consumer.flags.notified:
enqueued_method = "frappe.event_streaming.doctype.event_consumer.event_consumer.notify"
jobs = get_jobs()
if not jobs or enqueued_method not in jobs[frappe.local.site] and not consumer.flags.notifed:
frappe.enqueue(
enqueued_method, queue="long", enqueue_after_commit=True, **{"consumer": consumer}
)
def has_consumer_access(consumer, update_log):
"""Checks if consumer has completely satisfied all the conditions on the doc"""
if isinstance(consumer, str):
consumer = frappe.get_doc("Event Consumer", consumer)
if not frappe.db.exists(update_log.ref_doctype, update_log.docname):
# Delete Log
# Check if the last Update Log of this document was read by this consumer
last_update_log = frappe.get_all(
"Event Update Log",
filters={
"ref_doctype": update_log.ref_doctype,
"docname": update_log.docname,
"creation": ["<", update_log.creation],
},
order_by="creation desc",
limit_page_length=1,
)
if not len(last_update_log):
return False
last_update_log = frappe.get_doc("Event Update Log", last_update_log[0].name)
return len([x for x in last_update_log.consumers if x.consumer == consumer.name])
doc = frappe.get_doc(update_log.ref_doctype, update_log.docname)
try:
for dt_entry in consumer.consumer_doctypes:
if dt_entry.ref_doctype != update_log.ref_doctype:
continue
if not dt_entry.condition:
return True
condition: str = dt_entry.condition
if condition.startswith("cmd:"):
cmd = condition.split("cmd:")[1].strip()
args = {"consumer": consumer, "doc": doc, "update_log": update_log}
return frappe.call(cmd, **args)
else:
return frappe.safe_eval(condition, frappe._dict(doc=doc))
except Exception as e:
consumer.log_error("has_consumer_access error")
return False

8
frappe/event_streaming/doctype/event_consumer/test_event_consumer.py
View file

@ -1,8 +0,0 @@
# Copyright (c) 2019, Frappe Technologies and Contributors
# License: MIT. See LICENSE
# import frappe
from frappe.tests.utils import FrappeTestCase
class TestEventConsumer(FrappeTestCase):
pass

0
frappe/event_streaming/doctype/event_consumer_document_type/__init__.py
View file

61
frappe/event_streaming/doctype/event_consumer_document_type/event_consumer_document_type.json
View file

@ -1,61 +0,0 @@
{
"actions": [],
"creation": "2019-10-03 21:10:54.754651",
"doctype": "DocType",
"editable_grid": 1,
"engine": "InnoDB",
"field_order": [
"ref_doctype",
"status",
"unsubscribed",
"condition"
],
"fields": [
{
"columns": 4,
"fieldname": "ref_doctype",
"fieldtype": "Link",
"in_list_view": 1,
"label": "Document Type",
"options": "DocType",
"read_only": 1,
"reqd": 1
},
{
"columns": 4,
"default": "Pending",
"fieldname": "status",
"fieldtype": "Select",
"in_list_view": 1,
"label": "Approval Status",
"options": "Pending\nApproved\nRejected"
},
{
"columns": 2,
"default": "0",
"fieldname": "unsubscribed",
"fieldtype": "Check",
"in_list_view": 1,
"label": "Unsubscribed",
"read_only": 1
},
{
"fieldname": "condition",
"fieldtype": "Code",
"label": "Condition",
"read_only": 1
}
],
"istable": 1,
"links": [],
"modified": "2020-11-07 09:26:49.894294",
"modified_by": "Administrator",
"module": "Event Streaming",
"name": "Event Consumer Document Type",
"owner": "Administrator",
"permissions": [],
"quick_entry": 1,
"sort_field": "modified",
"sort_order": "DESC",
"track_changes": 1
}

9
frappe/event_streaming/doctype/event_consumer_document_type/event_consumer_document_type.py
View file

@ -1,9 +0,0 @@
# Copyright (c) 2019, Frappe Technologies and contributors
# License: MIT. See LICENSE
# import frappe
from frappe.model.document import Document
class EventConsumerDocumentType(Document):
pass

Some files were not shown because too many files have changed in this diff Show more