${this.get_indicator_html(doc)}
diff --git a/frappe/public/js/frappe/utils/dashboard_utils.js b/frappe/public/js/frappe/utils/dashboard_utils.js
index 488aa20742..1058101fff 100644
--- a/frappe/public/js/frappe/utils/dashboard_utils.js
+++ b/frappe/public/js/frappe/utils/dashboard_utils.js
@@ -202,14 +202,16 @@ frappe.dashboard_utils = {
},
get_all_filters(doc) {
- let filters = JSON.parse(doc.filters_json || "null");
- let dynamic_filters = JSON.parse(doc.dynamic_filters_json || "null");
+ let filters = doc.filters_json ? JSON.parse(doc.filters_json) : null;
+ let dynamic_filters = doc.dynamic_filters_json
+ ? JSON.parse(doc.dynamic_filters_json)
+ : null;
- if (!dynamic_filters) {
+ if (!dynamic_filters || !Object.keys(dynamic_filters).length) {
return filters;
}
- if ($.isArray(dynamic_filters)) {
+ if (Array.isArray(dynamic_filters)) {
dynamic_filters.forEach((f) => {
try {
f[3] = eval(f[3]);
diff --git a/frappe/public/scss/common/mixins.scss b/frappe/public/scss/common/mixins.scss
index 6d71ea9d6f..55b54d9de3 100644
--- a/frappe/public/scss/common/mixins.scss
+++ b/frappe/public/scss/common/mixins.scss
@@ -45,42 +45,5 @@
$background-color: var(--bg-color),
$border-radius: var(--border-radius),
) {
-
- @if $content {
- img:after {
- content: url($content);
- }
- } @else {
- img:after {
- content: url("data:image/svg+xml;utf8,
");
- }
- }
-
- img[alt]:after {
- height: $height;
- top: $top;
- left: $left;
- background-color: $background-color;
- border-radius: $border-radius;
- width: 100%;
- position: absolute;
- @include flex();
- z-index: 1;
- }
-}
-
-// @mixin img-foreground() {
-// content: "\f1c5";
-// display: block;
-// font-style: normal;
-// font-family: FontAwesome;
-// font-size: 32px;
-// color: var(--text-muted);
-
-// position: absolute;
-// top: 50%;
-// transform: translateY(-50%);
-// left: 0;
-// width: 100%;
-// text-align: center;
-// }
\ No newline at end of file
+ // Deprecated: Does not work as expected anymore. Also, this never worked in Safari.
+}
\ No newline at end of file
diff --git a/frappe/public/scss/desk/file_view.scss b/frappe/public/scss/desk/file_view.scss
index d074a7efdd..29e49ab65e 100644
--- a/frappe/public/scss/desk/file_view.scss
+++ b/frappe/public/scss/desk/file_view.scss
@@ -97,11 +97,6 @@
color: transparent;
position: relative;
}
-
- @include broken-img(
- $height: 70px,
- $top: -15px,
- );
}
}
diff --git a/frappe/public/scss/desk/image_view.scss b/frappe/public/scss/desk/image_view.scss
index 3b9d033406..063af27923 100644
--- a/frappe/public/scss/desk/image_view.scss
+++ b/frappe/public/scss/desk/image_view.scss
@@ -153,11 +153,6 @@
position: relative;
width: 100%;
}
-
- @include broken-img(
- $height: 175px,
- $border-radius: 0
- );
}
.image-title {
diff --git a/frappe/public/scss/desk/kanban.scss b/frappe/public/scss/desk/kanban.scss
index 8f286b7b35..5a748c581c 100644
--- a/frappe/public/scss/desk/kanban.scss
+++ b/frappe/public/scss/desk/kanban.scss
@@ -181,11 +181,6 @@
color: transparent;
position: relative;
}
-
- @include broken-img(
- $height: 125px,
- $top: -4px,
- )
}
.kanban-card-body {
diff --git a/frappe/public/scss/desk/report.scss b/frappe/public/scss/desk/report.scss
index 03e2763cd7..1db38529ed 100644
--- a/frappe/public/scss/desk/report.scss
+++ b/frappe/public/scss/desk/report.scss
@@ -100,11 +100,11 @@
.group-by-button {
margin: 5px;
- max-width: 125px;
}
.group-by-button.btn-primary-light {
color: var(--text-on-blue);
+ outline: 1px solid var(--bg-dark-blue);
}
.group-by-icon.active {
diff --git a/frappe/realtime.py b/frappe/realtime.py
index 410112b164..47b3d8a95c 100644
--- a/frappe/realtime.py
+++ b/frappe/realtime.py
@@ -100,10 +100,10 @@ def emit_via_redis(event, message, room):
:param event: Event name, like `task_progress` etc.
:param message: JSON message object. For async must contain `task_id`
:param room: name of the room"""
- from frappe.utils.background_jobs import get_redis_conn
+ from frappe.utils.background_jobs import get_redis_connection_without_auth
with suppress(redis.exceptions.ConnectionError):
- r = get_redis_conn()
+ r = get_redis_connection_without_auth()
r.publish("events", frappe.as_json({"event": event, "message": message, "room": room}))
diff --git a/frappe/sessions.py b/frappe/sessions.py
index 64a1a6b663..2709de8fab 100644
--- a/frappe/sessions.py
+++ b/frappe/sessions.py
@@ -18,10 +18,9 @@ import frappe.translate
import frappe.utils
from frappe import _
from frappe.cache_manager import clear_user_cache
-from frappe.query_builder import DocType, Order
-from frappe.query_builder.functions import Now
-from frappe.query_builder.utils import PseudoColumn
+from frappe.query_builder import Order
from frappe.utils import cint, cstr, get_assets_json
+from frappe.utils.data import add_to_date
@frappe.whitelist()
@@ -62,7 +61,7 @@ def get_sessions_to_clear(user=None, keep_current=False):
simultaneous_sessions = frappe.db.get_value("User", user, "simultaneous_sessions") or 1
offset = simultaneous_sessions - 1
- session = DocType("Sessions")
+ session = frappe.qb.DocType("Sessions")
session_id = frappe.qb.from_(session).where(session.user == user)
if keep_current:
session_id = session_id.where(session.sid != frappe.session.sid)
@@ -88,7 +87,7 @@ def delete_session(sid=None, user=None, reason="Session Expired"):
frappe.cache.hdel("session", sid)
frappe.cache.hdel("last_db_session_update", sid)
if sid and not user:
- table = DocType("Sessions")
+ table = frappe.qb.DocType("Sessions")
user_details = (
frappe.qb.from_(table).where(table.sid == sid).select(table.user).run(as_dict=True)
)
@@ -112,17 +111,12 @@ def clear_all_sessions(reason=None):
def get_expired_sessions():
"""Returns list of expired sessions"""
- sessions = DocType("Sessions")
-
- return frappe.db.get_values(
- sessions,
- filters=(
- PseudoColumn(f"({Now()} - {sessions.lastupdate.get_sql()})") > get_expiry_period_for_query()
- ),
- fieldname="sid",
- order_by=None,
- pluck=True,
- )
+ sessions = frappe.qb.DocType("Sessions")
+ return (
+ frappe.qb.from_(sessions)
+ .select(sessions.sid)
+ .where(sessions.lastupdate < get_expired_threshold())
+ ).run(pluck=True)
def clear_expired_sessions():
@@ -232,7 +226,7 @@ class Session:
sid = frappe.generate_hash()
self.data.user = self.user
- self.data.sid = sid
+ self.sid = self.data.sid = sid
self.data.data.user = self.user
self.data.data.session_ip = frappe.local.request_ip
if self.user != "Guest":
@@ -268,14 +262,17 @@ class Session:
frappe.db.commit()
def insert_session_record(self):
- frappe.db.sql(
- """insert into `tabSessions`
- (`sessiondata`, `user`, `lastupdate`, `sid`, `status`)
- values (%s , %s, NOW(), %s, 'Active')""",
- (str(self.data["data"]), self.data["user"], self.data["sid"]),
- )
- # also add to memcache
+ Sessions = frappe.qb.DocType("Sessions")
+ now = frappe.utils.now()
+
+ (
+ frappe.qb.into(Sessions)
+ .columns(
+ Sessions.sessiondata, Sessions.user, Sessions.lastupdate, Sessions.sid, Sessions.status
+ )
+ .insert((str(self.data["data"]), self.data["user"], now, self.data["sid"], "Active"))
+ ).run()
frappe.cache.hset("session", self.data.sid, self.data)
def resume(self):
@@ -338,20 +335,18 @@ class Session:
return data and data.data
def get_session_data_from_db(self):
- sessions = DocType("Sessions")
- rec = frappe.db.get_values(
- sessions,
- filters=(sessions.sid == self.sid)
- & (
- PseudoColumn(f"({Now()} - {sessions.lastupdate.get_sql()})") < get_expiry_period_for_query()
- ),
- fieldname=["user", "sessiondata"],
- order_by=None,
- )
+ sessions = frappe.qb.DocType("Sessions")
- if rec:
- data = frappe._dict(frappe.safe_eval(rec and rec[0][1] or "{}"))
- data.user = rec[0][0]
+ record = (
+ frappe.qb.from_(sessions)
+ .select(sessions.user, sessions.sessiondata)
+ .where(sessions.sid == self.sid)
+ .where(sessions.lastupdate > get_expired_threshold())
+ ).run()
+
+ if record:
+ data = frappe._dict(frappe.safe_eval(record and record[0][1] or "{}"))
+ data.user = record[0][0]
else:
self._delete_session()
data = None
@@ -373,6 +368,8 @@ class Session:
now = frappe.utils.now()
+ Sessions = frappe.qb.DocType("Sessions")
+
self.data["data"]["last_updated"] = now
self.data["data"]["lang"] = str(frappe.lang)
@@ -384,17 +381,14 @@ class Session:
updated_in_db = False
if (force or (time_diff is None) or (time_diff > 600)) and not frappe.flags.read_only:
# update sessions table
- frappe.db.sql(
- """update `tabSessions` set sessiondata=%s,
- lastupdate=NOW() where sid=%s""",
- (str(self.data["data"]), self.data["sid"]),
- )
+ (
+ frappe.qb.update(Sessions)
+ .where(Sessions.sid == self.data["sid"])
+ .set(Sessions.sessiondata, str(self.data["data"]))
+ .set(Sessions.lastupdate, now)
+ ).run()
- # update last active in user table
- frappe.db.sql(
- """update `tabUser` set last_active=%(now)s where name=%(name)s""",
- {"now": now, "name": frappe.session.user},
- )
+ frappe.db.set_value("User", frappe.session.user, "last_active", now, update_modified=False)
frappe.db.commit()
frappe.cache.hset("last_db_session_update", self.sid, now)
@@ -421,6 +415,15 @@ def get_expiry_in_seconds(expiry=None):
return (cint(parts[0]) * 3600) + (cint(parts[1]) * 60) + cint(parts[2])
+def get_expired_threshold():
+ """Get cutoff time before which all sessions are considered expired."""
+
+ now = frappe.utils.now()
+ expiry_in_seconds = get_expiry_in_seconds()
+
+ return add_to_date(now, seconds=-expiry_in_seconds, as_string=True)
+
+
def get_expiry_period():
exp_sec = frappe.defaults.get_global_default("session_expiry") or "06:00:00"
diff --git a/frappe/share.py b/frappe/share.py
index 6c2fb356a6..c068e063b2 100644
--- a/frappe/share.py
+++ b/frappe/share.py
@@ -141,7 +141,7 @@ def get_users(doctype, name):
)
-def get_shared(doctype, user=None, rights=None):
+def get_shared(doctype, user=None, rights=None, *, filters=None, limit=None):
"""Get list of shared document names for given user and DocType.
:param doctype: DocType of which shared names are queried.
@@ -154,14 +154,22 @@ def get_shared(doctype, user=None, rights=None):
if not rights:
rights = ["read"]
- filters = [[right, "=", 1] for right in rights]
- filters += [["share_doctype", "=", doctype]]
+ share_filters = [[right, "=", 1] for right in rights]
+ share_filters += [["share_doctype", "=", doctype]]
+ if filters:
+ share_filters += filters
+
or_filters = [["user", "=", user]]
if user != "Guest":
or_filters += [["everyone", "=", 1]]
shared_docs = frappe.get_all(
- "DocShare", fields=["share_name"], filters=filters, or_filters=or_filters, order_by=None
+ "DocShare",
+ fields=["share_name"],
+ filters=share_filters,
+ or_filters=or_filters,
+ order_by=None,
+ limit_page_length=limit,
)
return [doc.share_name for doc in shared_docs]
diff --git a/frappe/tests/test_auth.py b/frappe/tests/test_auth.py
index d4cfe4451e..6aa8d2cc2f 100644
--- a/frappe/tests/test_auth.py
+++ b/frappe/tests/test_auth.py
@@ -1,14 +1,18 @@
# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors
# License: MIT. See LICENSE
import time
+from unittest.mock import patch
import requests
import frappe
-import frappe.utils
from frappe.auth import LoginAttemptTracker
from frappe.frappeclient import AuthError, FrappeClient
+from frappe.sessions import Session, get_expired_sessions, get_expiry_in_seconds
+from frappe.tests.test_api import FrappeAPITestCase
from frappe.tests.utils import FrappeTestCase
+from frappe.utils import get_site_url, now
+from frappe.utils.data import add_to_date
from frappe.www.login import _generate_temporary_login_link
@@ -26,9 +30,7 @@ class TestAuth(FrappeTestCase):
@classmethod
def setUpClass(cls):
super().setUpClass()
- cls.HOST_NAME = frappe.get_site_config().host_name or frappe.utils.get_site_url(
- frappe.local.site
- )
+ cls.HOST_NAME = frappe.get_site_config().host_name or get_site_url(frappe.local.site)
cls.test_user_email = "test_auth@test.com"
cls.test_user_name = "test_auth_user"
cls.test_user_mobile = "+911234567890"
@@ -197,3 +199,27 @@ class TestLoginAttemptTracker(FrappeTestCase):
tracker.add_failure_attempt()
self.assertTrue(tracker.is_user_allowed())
+
+
+class TestSessionExpirty(FrappeAPITestCase):
+ def test_session_expires(self):
+ sid = self.sid # triggers login for test case login
+ s: Session = frappe.local.session_obj
+
+ expiry_in = get_expiry_in_seconds()
+ session_created = now()
+
+ # Try with 1% increments of times, it should always work
+ for step in range(0, 100, 1):
+ seconds_elapsed = expiry_in * step / 100
+
+ time_now = add_to_date(session_created, seconds=seconds_elapsed, as_string=True)
+ with patch("frappe.utils.now", return_value=time_now):
+ data = s.get_session_data_from_db()
+ self.assertEqual(data.user, "Administrator")
+
+ # 1% higher should immediately expire
+ time_now = add_to_date(session_created, seconds=expiry_in * 1.01, as_string=True)
+ with patch("frappe.utils.now", return_value=time_now):
+ self.assertIn(sid, get_expired_sessions())
+ self.assertFalse(s.get_session_data_from_db())
diff --git a/frappe/tests/test_background_jobs.py b/frappe/tests/test_background_jobs.py
index b6c1a0d694..99373a84a6 100644
--- a/frappe/tests/test_background_jobs.py
+++ b/frappe/tests/test_background_jobs.py
@@ -10,6 +10,7 @@ from frappe.tests.utils import FrappeTestCase
from frappe.utils.background_jobs import (
RQ_JOB_FAILURE_TTL,
RQ_RESULTS_TTL,
+ create_job_id,
execute_job,
generate_qname,
get_redis_conn,
@@ -54,11 +55,12 @@ class TestBackgroundJobs(FrappeTestCase):
def test_enqueue_call(self):
with patch.object(Queue, "enqueue_call") as mock_enqueue_call:
- frappe.enqueue(
+ job = frappe.enqueue(
"frappe.handler.ping",
queue="short",
timeout=300,
kwargs={"site": frappe.local.site},
+ job_id="test",
)
mock_enqueue_call.assert_called_once_with(
@@ -78,7 +80,7 @@ class TestBackgroundJobs(FrappeTestCase):
at_front=False,
failure_ttl=RQ_JOB_FAILURE_TTL,
result_ttl=RQ_RESULTS_TTL,
- job_id=None,
+ job_id=create_job_id("test"),
)
def test_job_hooks(self):
diff --git a/frappe/tests/test_commands.py b/frappe/tests/test_commands.py
index 7bda577bb6..cdf93a1870 100644
--- a/frappe/tests/test_commands.py
+++ b/frappe/tests/test_commands.py
@@ -20,9 +20,11 @@ from unittest.mock import patch
import click
from click import Command
from click.testing import CliRunner, Result
+from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_fixed
# imports - module imports
import frappe
+import frappe.commands.scheduler
import frappe.commands.site
import frappe.commands.utils
import frappe.recorder
@@ -760,6 +762,19 @@ class TestBenchBuild(BaseTestCommands):
)
+class TestSchedulerUtils(BaseTestCommands):
+ # Retry just in case there are stuck queued jobs
+ @retry(
+ retry=retry_if_exception_type(AssertionError),
+ stop=stop_after_attempt(3),
+ wait=wait_fixed(3),
+ reraise=True,
+ )
+ def test_ready_for_migrate(self):
+ with cli(frappe.commands.scheduler.ready_for_migration) as result:
+ self.assertEqual(result.exit_code, 0)
+
+
class TestCommandUtils(FrappeTestCase):
def test_bench_helper(self):
from frappe.utils.bench_helper import get_app_groups
diff --git a/frappe/tests/test_db.py b/frappe/tests/test_db.py
index f026726cfa..e8afba5d96 100644
--- a/frappe/tests/test_db.py
+++ b/frappe/tests/test_db.py
@@ -140,27 +140,6 @@ class TestDB(FrappeTestCase):
frappe.db.get_value("DocType", "DocField", order_by="creation desc, modified asc, name", run=0),
)
- def test_get_value_limits(self):
- # check both dict and list style filters
- filters = [{"enabled": 1}, [["enabled", "=", 1]]]
- for filter in filters:
- self.assertEqual(1, len(frappe.db.get_values("User", filters=filter, limit=1)))
- # count of last touched rows as per DB-API 2.0 https://peps.python.org/pep-0249/#rowcount
- self.assertGreaterEqual(1, cint(frappe.db._cursor.rowcount))
- self.assertEqual(2, len(frappe.db.get_values("User", filters=filter, limit=2)))
- self.assertGreaterEqual(2, cint(frappe.db._cursor.rowcount))
-
- # without limits length == count
- self.assertEqual(
- len(frappe.db.get_values("User", filters=filter)), frappe.db.count("User", filter)
- )
-
- frappe.db.get_value("User", filters=filter)
- self.assertGreaterEqual(1, cint(frappe.db._cursor.rowcount))
-
- frappe.db.exists("User", filter)
- self.assertGreaterEqual(1, cint(frappe.db._cursor.rowcount))
-
def test_escape(self):
frappe.db.escape("香港濟生堂製藥有限公司 - IT".encode())
diff --git a/frappe/tests/test_perf.py b/frappe/tests/test_perf.py
index 65f03c2f13..cc7d0b031d 100644
--- a/frappe/tests/test_perf.py
+++ b/frappe/tests/test_perf.py
@@ -27,6 +27,7 @@ from frappe.model.base_document import get_controller
from frappe.query_builder.utils import db_type_is
from frappe.tests.test_query_builder import run_only_if
from frappe.tests.utils import FrappeTestCase
+from frappe.utils import cint
from frappe.website.path_resolver import PathResolver
@@ -70,6 +71,25 @@ class TestPerformance(FrappeTestCase):
with self.assertQueryCount(0):
get_controller("User")
+ def test_get_value_limits(self):
+ # check both dict and list style filters
+ filters = [{"enabled": 1}, [["enabled", "=", 1]]]
+ for filter in filters:
+ with self.assertRowsRead(1):
+ self.assertEqual(1, len(frappe.db.get_values("User", filters=filter, limit=1)))
+ with self.assertRowsRead(2):
+ self.assertEqual(2, len(frappe.db.get_values("User", filters=filter, limit=2)))
+
+ self.assertEqual(
+ len(frappe.db.get_values("User", filters=filter)), frappe.db.count("User", filter)
+ )
+
+ with self.assertRowsRead(1):
+ frappe.db.get_value("User", filters=filter)
+
+ with self.assertRowsRead(1):
+ frappe.db.exists("User", filter)
+
def test_db_value_cache(self):
"""Link validation if repeated should just use db.value_cache, hence no extra queries"""
doc = frappe.get_last_doc("User")
diff --git a/frappe/tests/test_website.py b/frappe/tests/test_website.py
index 6c319fff0a..16e82850de 100644
--- a/frappe/tests/test_website.py
+++ b/frappe/tests/test_website.py
@@ -11,10 +11,16 @@ from frappe.website.utils import build_response, clear_website_cache, get_home_p
class TestWebsite(FrappeTestCase):
def setUp(self):
frappe.set_user("Guest")
+ self._clearRequest()
def tearDown(self):
frappe.db.delete("Access Log")
frappe.set_user("Administrator")
+ self._clearRequest()
+
+ def _clearRequest(self):
+ if hasattr(frappe.local, "request"):
+ delattr(frappe.local, "request")
def test_home_page(self):
frappe.set_user("Administrator")
@@ -340,8 +346,9 @@ class TestWebsite(FrappeTestCase):
FILES_TO_SKIP = choices(list(WWW.glob("**/*.py*")), k=10)
for suffix in FILES_TO_SKIP:
- content = get_response_content(suffix.relative_to(WWW))
- self.assertIn("404", content)
+ path: str = suffix.relative_to(WWW).as_posix()
+ content = get_response_content(path)
+ self.assertIn("
Not Found", content)
def test_metatags(self):
content = get_response_content("/_test/_test_metatags")
diff --git a/frappe/tests/utils.py b/frappe/tests/utils.py
index 6e6c72052a..762cd885b0 100644
--- a/frappe/tests/utils.py
+++ b/frappe/tests/utils.py
@@ -92,6 +92,26 @@ class FrappeTestCase(unittest.TestCase):
finally:
frappe.db.sql = orig_sql
+ @contextmanager
+ def assertRowsRead(self, count):
+ rows_read = 0
+
+ def _sql_with_count(*args, **kwargs):
+ nonlocal rows_read
+
+ ret = orig_sql(*args, **kwargs)
+ # count of last touched rows as per DB-API 2.0 https://peps.python.org/pep-0249/#rowcount
+ rows_read += cint(frappe.db._cursor.rowcount)
+ return ret
+
+ try:
+ orig_sql = frappe.db.sql
+ frappe.db.sql = _sql_with_count
+ yield
+ self.assertLessEqual(rows_read, count, msg="Queries read more rows than expected")
+ finally:
+ frappe.db.sql = orig_sql
+
class MockedRequestTestCase(FrappeTestCase):
def setUp(self):
diff --git a/frappe/utils/background_jobs.py b/frappe/utils/background_jobs.py
index 6b0249e720..be5291b771 100755
--- a/frappe/utils/background_jobs.py
+++ b/frappe/utils/background_jobs.py
@@ -4,16 +4,17 @@ import socket
import time
from collections import defaultdict
from functools import lru_cache
-from typing import Any, Callable, Literal, NoReturn
+from typing import Any, Callable, NoReturn
from uuid import uuid4
import redis
from redis.exceptions import BusyLoadingError, ConnectionError
-from rq import Connection, Queue, Worker
+from rq import Queue, Worker
from rq.exceptions import NoSuchJobError
from rq.job import Job, JobStatus
from rq.logutils import setup_loghandlers
-from rq.worker import RandomWorker, RoundRobinWorker
+from rq.worker import DequeueStrategy
+from rq.worker_pool import WorkerPool
from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_fixed
import frappe
@@ -65,6 +66,7 @@ def enqueue(
on_failure: Callable = None,
at_front: bool = False,
job_id: str = None,
+ deduplicate=False,
**kwargs,
) -> Job | Any:
"""
@@ -78,14 +80,28 @@ def enqueue(
:param job_name: [DEPRECATED] can be used to name an enqueue call, which can be used to prevent duplicate calls
:param now: if now=True, the method is executed via frappe.call
:param kwargs: keyword arguments to be passed to the method
+ :param deduplicate: do not re-queue job if it's already queued, requires job_id.
:param job_id: Assigning unique job id, which can be checked using `is_job_enqueued`
"""
# To handle older implementations
is_async = kwargs.pop("async", is_async)
- if job_id:
- # namespace job ids to sites
- job_id = create_job_id(job_id)
+ if deduplicate:
+ if not job_id:
+ frappe.throw(_("`job_id` paramater is required for deduplication."))
+ job = get_job(job_id)
+ if job and job.get_status() in (JobStatus.QUEUED, JobStatus.STARTED):
+ frappe.logger().debug(f"Not queueing job {job.id} because it is in queue already")
+ return
+ elif job:
+ # delete job to avoid argument issues related to job args
+ # https://github.com/rq/rq/issues/793
+ job.delete()
+
+ # If job exists and is completed then delete it before re-queue
+
+ # namespace job ids to sites
+ job_id = create_job_id(job_id)
if job_name:
deprecation_warning("Using enqueue with `job_name` is deprecated, use `job_id` instead.")
@@ -230,14 +246,14 @@ def start_worker(
rq_username: str | None = None,
rq_password: str | None = None,
burst: bool = False,
- strategy: Literal["round_robin", "random"] | None = None,
+ strategy: DequeueStrategy | None = DequeueStrategy.DEFAULT,
) -> NoReturn | None: # pragma: no cover
"""Wrapper to start rq worker. Connects to redis and monitors these queues."""
- DEQUEUE_STRATEGIES = {"round_robin": RoundRobinWorker, "random": RandomWorker}
- if frappe._tune_gc:
- gc.collect()
- gc.freeze()
+ if not strategy:
+ strategy = DequeueStrategy.DEFAULT
+
+ _freeze_gc()
with frappe.init_site():
# empty init is required to get redis_queue from common_site_config.json
@@ -251,19 +267,59 @@ def start_worker(
if os.environ.get("CI"):
setup_loghandlers("ERROR")
- WorkerKlass = DEQUEUE_STRATEGIES.get(strategy, Worker)
+ logging_level = "INFO"
+ if quiet:
+ logging_level = "WARNING"
- with Connection(redis_connection):
- logging_level = "INFO"
- if quiet:
- logging_level = "WARNING"
- worker = WorkerKlass(queues, name=get_worker_name(queue_name))
- worker.work(
- logging_level=logging_level,
- burst=burst,
- date_format="%Y-%m-%d %H:%M:%S",
- log_format="%(asctime)s,%(msecs)03d %(message)s",
- )
+ worker = Worker(queues, name=get_worker_name(queue_name), connection=redis_connection)
+ worker.work(
+ logging_level=logging_level,
+ burst=burst,
+ date_format="%Y-%m-%d %H:%M:%S",
+ log_format="%(asctime)s,%(msecs)03d %(message)s",
+ dequeue_strategy=strategy,
+ )
+
+
+def start_worker_pool(
+ queue: str | None = None,
+ num_workers: int = 1,
+ quiet: bool = False,
+ burst: bool = False,
+) -> NoReturn:
+ """Start worker pool with specified number of workers.
+
+ WARNING: This feature is considered "EXPERIMENTAL".
+ """
+
+ _freeze_gc()
+
+ with frappe.init_site():
+ redis_connection = get_redis_conn()
+
+ if queue:
+ queue = [q.strip() for q in queue.split(",")]
+ queues = get_queue_list(queue, build_queue_name=True)
+
+ if os.environ.get("CI"):
+ setup_loghandlers("ERROR")
+
+ logging_level = "INFO"
+ if quiet:
+ logging_level = "WARNING"
+
+ pool = WorkerPool(
+ queues=queues,
+ connection=redis_connection,
+ num_workers=num_workers,
+ )
+ pool.start(logging_level=logging_level, burst=burst)
+
+
+def _freeze_gc():
+ if frappe._tune_gc:
+ gc.collect()
+ gc.freeze()
def get_worker_name(queue):
@@ -353,8 +409,8 @@ def validate_queue(queue, default_queue_list=None):
@retry(
- retry=retry_if_exception_type(BusyLoadingError) | retry_if_exception_type(ConnectionError),
- stop=stop_after_attempt(10),
+ retry=retry_if_exception_type((BusyLoadingError, ConnectionError)),
+ stop=stop_after_attempt(5),
wait=wait_fixed(1),
reraise=True,
)
@@ -382,9 +438,7 @@ def get_redis_conn(username=None, password=None):
try:
if not cred:
- if not _redis_queue_conn:
- _redis_queue_conn = RedisQueue.get_connection()
- return _redis_queue_conn
+ return get_redis_connection_without_auth()
else:
return RedisQueue.get_connection(**cred)
except (redis.exceptions.AuthenticationError, redis.exceptions.ResponseError):
@@ -399,6 +453,14 @@ def get_redis_conn(username=None, password=None):
raise
+def get_redis_connection_without_auth():
+ global _redis_queue_conn
+
+ if not _redis_queue_conn:
+ _redis_queue_conn = RedisQueue.get_connection()
+ return _redis_queue_conn
+
+
def get_queues() -> list[Queue]:
"""Get all the queues linked to the current bench."""
queues = Queue.all(connection=get_redis_conn())
@@ -434,6 +496,9 @@ def test_job(s):
def create_job_id(job_id: str) -> str:
"""Generate unique job id for deduplication"""
+
+ if not job_id:
+ job_id = str(uuid4())
return f"{frappe.local.site}::{job_id}"
@@ -443,9 +508,13 @@ def is_job_enqueued(job_id: str) -> bool:
def get_job_status(job_id: str) -> JobStatus | None:
"""Get RQ job status, returns None if job is not found."""
+ job = get_job(job_id)
+ if job:
+ return job.get_status()
+
+
+def get_job(job_id: str) -> Job:
try:
- job = Job.fetch(create_job_id(job_id), connection=get_redis_conn())
+ return Job.fetch(create_job_id(job_id), connection=get_redis_conn())
except NoSuchJobError:
return None
-
- return job.get_status()
diff --git a/frappe/utils/bench_helper.py b/frappe/utils/bench_helper.py
index 01e611f021..14db5c8449 100644
--- a/frappe/utils/bench_helper.py
+++ b/frappe/utils/bench_helper.py
@@ -4,6 +4,7 @@ import os
import traceback
import warnings
from pathlib import Path
+from textwrap import dedent
import click
@@ -52,10 +53,19 @@ def get_sites(site_arg: str) -> list[str]:
return [site_arg]
elif os.environ.get("FRAPPE_SITE"):
return [os.environ.get("FRAPPE_SITE")]
- elif os.path.exists("currentsite.txt"):
- with open("currentsite.txt") as f:
- if site := f.read().strip():
- return [site]
+ elif default_site := frappe.get_conf().default_site:
+ return [default_site]
+ # This is not supported, just added here for warning.
+ elif (site := frappe.read_file("currentsite.txt")) and site.strip():
+ click.secho(
+ dedent(
+ f"""
+ WARNING: currentsite.txt is not supported anymore for setting default site. Use following command to set it as default site.
+ $ bench use {site}"""
+ ),
+ fg="red",
+ )
+
return []
diff --git a/frappe/utils/boilerplate.py b/frappe/utils/boilerplate.py
index 0e646f9992..e2aaa26f22 100644
--- a/frappe/utils/boilerplate.py
+++ b/frappe/utils/boilerplate.py
@@ -593,7 +593,7 @@ jobs:
- name: Setup Node
uses: actions/setup-node@v3
with:
- node-version: 16
+ node-version: 18
check-latest: true
- name: Cache pip
diff --git a/frappe/utils/doctor.py b/frappe/utils/doctor.py
index 5b12a01990..002fd8b154 100644
--- a/frappe/utils/doctor.py
+++ b/frappe/utils/doctor.py
@@ -79,6 +79,15 @@ def get_pending_jobs(site=None):
return jobs_per_queue
+def any_job_pending(site: str) -> bool:
+ for queue in get_queue_list():
+ q = get_queue(queue)
+ for job_id in q.get_job_ids():
+ if job_id.startswith(site):
+ return True
+ return False
+
+
def check_number_of_workers():
return len(get_workers())
diff --git a/frappe/utils/error.py b/frappe/utils/error.py
index a9891cb532..d44bdef47f 100644
--- a/frappe/utils/error.py
+++ b/frappe/utils/error.py
@@ -1,17 +1,10 @@
# Copyright (c) 2015, Maxwell Morais and contributors
# License: MIT. See LICENSE
-import datetime
import functools
import inspect
-import json
-import linecache
-import os
-import sys
-import traceback
import frappe
-from frappe.utils import cstr, encode
EXCLUDE_EXCEPTIONS = (
frappe.AuthenticationError,
@@ -37,194 +30,57 @@ def _is_ldap_exception(e):
return False
-def make_error_snapshot(exception):
- if frappe.conf.disable_error_snapshot:
+def log_error(
+ title=None, message=None, reference_doctype=None, reference_name=None, *, defer_insert=False
+):
+ """Log error to Error Log"""
+ # Parameter ALERT:
+ # the title and message may be swapped
+ # the better API for this is log_error(title, message), and used in many cases this way
+ # this hack tries to be smart about whats a title (single line ;-)) and fixes it
+
+ traceback = None
+ if message:
+ if "\n" in title: # traceback sent as title
+ traceback, title = title, message
+ else:
+ traceback = message
+
+ title = title or "Error"
+ traceback = frappe.as_unicode(traceback or frappe.get_traceback(with_context=True))
+
+ if not frappe.db:
+ print(f"Failed to log error in db: {title}")
return
+ error_log = frappe.get_doc(
+ doctype="Error Log",
+ error=traceback,
+ method=title,
+ reference_doctype=reference_doctype,
+ reference_name=reference_name,
+ )
+
+ if frappe.flags.read_only or defer_insert:
+ error_log.deferred_insert()
+ else:
+ return error_log.insert(ignore_permissions=True)
+
+
+def log_error_snapshot(exception: Exception):
+
if isinstance(exception, EXCLUDE_EXCEPTIONS) or _is_ldap_exception(exception):
return
logger = frappe.logger(with_more_info=True)
try:
- error_id = "{timestamp:s}-{ip:s}-{hash:s}".format(
- timestamp=cstr(datetime.datetime.now()),
- ip=frappe.local.request_ip or "127.0.0.1",
- hash=frappe.generate_hash(length=3),
- )
- snapshot_folder = get_error_snapshot_path()
- frappe.create_folder(snapshot_folder)
-
- snapshot_file_path = os.path.join(snapshot_folder, f"{error_id}.json")
- snapshot = get_snapshot(exception)
-
- with open(encode(snapshot_file_path), "wb") as error_file:
- error_file.write(encode(frappe.as_json(snapshot)))
-
- logger.error(f"New Exception collected with id: {error_id}")
-
+ log_error(title=str(exception), defer_insert=True)
+ logger.error("New Exception collected in error log")
except Exception as e:
logger.error(f"Could not take error snapshot: {e}", exc_info=True)
-def get_snapshot(exception, context=10):
- import pydoc
-
- """
- Return a dict describing a given traceback (based on cgitb.text)
- """
-
- etype, evalue, etb = sys.exc_info()
- if isinstance(etype, type):
- etype = etype.__name__
-
- # creates a snapshot dict with some basic information
-
- s = {
- "pyver": "Python {version:s}: {executable:s} (prefix: {prefix:s})".format(
- version=sys.version.split(maxsplit=1)[0], executable=sys.executable, prefix=sys.prefix
- ),
- "timestamp": cstr(datetime.datetime.now()),
- "traceback": traceback.format_exc(),
- "frames": [],
- "etype": cstr(etype),
- "evalue": cstr(repr(evalue)),
- "exception": {},
- "locals": {},
- }
-
- # start to process frames
- records = inspect.getinnerframes(etb, 5)
-
- for frame, file, lnum, func, lines, index in records:
- file = file and os.path.abspath(file) or "?"
- args, varargs, varkw, locals = inspect.getargvalues(frame)
- call = ""
-
- if func != "?":
- call = inspect.formatargvalues(
- args, varargs, varkw, locals, formatvalue=lambda value: f"={pydoc.text.repr(value)}"
- )
-
- # basic frame information
- f = {"file": file, "func": func, "call": call, "lines": {}, "lnum": lnum}
-
- def reader(lnum=[lnum]): # noqa
- try:
- # B023: function is evaluated immediately, binding not necessary
- return linecache.getline(file, lnum[0]) # noqa: B023
- finally:
- lnum[0] += 1
-
- vars = _scanvars(reader, frame, locals)
-
- # if it is a view, replace with generated code
- # if file.endswith('html'):
- # lmin = lnum > context and (lnum - context) or 0
- # lmax = lnum + context
- # lines = code.split("\n")[lmin:lmax]
- # index = min(context, lnum) - 1
-
- if index is not None:
- i = lnum - index
- for line in lines:
- f["lines"][i] = line.rstrip()
- i += 1
-
- # dump local variable (referenced in current line only)
- f["dump"] = {}
- for name, where, value in vars:
- if name in f["dump"]:
- continue
- if value is not __UNDEF__:
- if where == "global":
- name = f"global {name:s}"
- elif where != "local":
- name = where + " " + name.split(".")[-1]
- f["dump"][name] = pydoc.text.repr(value)
- else:
- f["dump"][name] = "undefined"
-
- s["frames"].append(f)
-
- # add exception type, value and attributes
- if isinstance(evalue, BaseException):
- for name in dir(evalue):
- if name != "messages" and not name.startswith("__"):
- value = pydoc.text.repr(getattr(evalue, name))
- s["exception"][name] = encode(value)
-
- # add all local values (of last frame) to the snapshot
- for name, value in locals.items():
- s["locals"][name] = value if isinstance(value, str) else pydoc.text.repr(value)
-
- return s
-
-
-def collect_error_snapshots():
- """Scheduled task to collect error snapshots from files and push into Error Snapshot table"""
- if frappe.conf.disable_error_snapshot:
- return
-
- try:
- path = get_error_snapshot_path()
- if not os.path.exists(path):
- return
-
- for fname in os.listdir(path):
- fullpath = os.path.join(path, fname)
-
- try:
- with open(fullpath) as filedata:
- data = json.load(filedata)
-
- except ValueError:
- # empty file
- os.remove(fullpath)
- continue
-
- for field in ["locals", "exception", "frames"]:
- data[field] = frappe.as_json(data[field])
-
- doc = frappe.new_doc("Error Snapshot")
- doc.update(data)
- doc.save()
-
- frappe.db.commit()
-
- os.remove(fullpath)
-
- clear_old_snapshots()
-
- except Exception as e:
- make_error_snapshot(e)
-
- # prevent creation of unlimited error snapshots
- raise
-
-
-def clear_old_snapshots():
- """Clear snapshots that are older than a month"""
- from frappe.query_builder import DocType, Interval
- from frappe.query_builder.functions import Now
-
- ErrorSnapshot = DocType("Error Snapshot")
- frappe.db.delete(ErrorSnapshot, filters=(ErrorSnapshot.creation < (Now() - Interval(months=1))))
-
- path = get_error_snapshot_path()
- today = datetime.datetime.now()
-
- for file in os.listdir(path):
- p = os.path.join(path, file)
- ctime = datetime.datetime.fromtimestamp(os.path.getctime(p))
- if (today - ctime).days > 31:
- os.remove(os.path.join(path, p))
-
-
-def get_error_snapshot_path():
- return frappe.get_site_path("error-snapshots")
-
-
def get_default_args(func):
"""Get default arguments of a function from its signature."""
signature = inspect.signature(func)
@@ -270,56 +126,3 @@ def raise_error_on_no_output(error_message, error_type=None, keep_quiet=None):
return wrapper_raise_error_on_no_output
return decorator_raise_error_on_no_output
-
-
-# Vendored from cgitb standard library reused under PSF License:
-# https://github.com/python/cpython/blob/main/LICENSE
-
-
-import keyword
-import tokenize
-
-__UNDEF__ = [] # a special sentinel object
-
-
-def _scanvars(reader, frame, locals):
- """Scan one logical line of Python and look up values of variables used."""
- vars, lasttoken, parent, prefix, value = [], None, None, "", __UNDEF__
- for ttype, token, start, end, line in tokenize.generate_tokens(reader):
- if ttype == tokenize.NEWLINE:
- break
- if ttype == tokenize.NAME and token not in keyword.kwlist:
- if lasttoken == ".":
- if parent is not __UNDEF__:
- value = getattr(parent, token, __UNDEF__)
- vars.append((prefix + token, prefix, value))
- else:
- where, value = _lookup(token, frame, locals)
- vars.append((token, where, value))
- elif token == ".":
- prefix += lasttoken + "."
- parent = value
- else:
- parent, prefix = None, ""
- lasttoken = token
- return vars
-
-
-def _lookup(name, frame, locals):
- """Find the value for a given name in the given environment."""
- if name in locals:
- return "local", locals[name]
- if name in frame.f_globals:
- return "global", frame.f_globals[name]
- if "__builtins__" in frame.f_globals:
- builtins = frame.f_globals["__builtins__"]
- if type(builtins) is type({}): # noqa
- if name in builtins:
- return "builtin", builtins[name]
- else:
- if hasattr(builtins, name):
- return "builtin", getattr(builtins, name)
- return None, __UNDEF__
-
-
-# end: vendored code
diff --git a/frappe/website/doctype/web_form/web_form.py b/frappe/website/doctype/web_form/web_form.py
index fd9949c45f..75f8793b4a 100644
--- a/frappe/website/doctype/web_form/web_form.py
+++ b/frappe/website/doctype/web_form/web_form.py
@@ -631,7 +631,7 @@ def get_link_options(web_form_name, doctype, allow_read_on_all_link_options=Fals
if title_field and show_title_field_in_link:
return json.dumps(link_options, default=str)
else:
- return "\n".join([doc.value for doc in link_options])
+ return "\n".join([str(doc.value) for doc in link_options])
else:
raise frappe.PermissionError(
diff --git a/frappe/website/page_renderers/static_page.py b/frappe/website/page_renderers/static_page.py
index 04e58ff217..00162c0772 100644
--- a/frappe/website/page_renderers/static_page.py
+++ b/frappe/website/page_renderers/static_page.py
@@ -8,7 +8,18 @@ import frappe
from frappe.website.page_renderers.base_renderer import BaseRenderer
from frappe.website.utils import is_binary_file
-UNSUPPORTED_STATIC_PAGE_TYPES = ("html", "md", "js", "xml", "css", "txt", "py", "json")
+UNSUPPORTED_STATIC_PAGE_TYPES = (
+ "css",
+ "html",
+ "js",
+ "json",
+ "md",
+ "py",
+ "pyc",
+ "pyo",
+ "txt",
+ "xml",
+)
class StaticPage(BaseRenderer):
diff --git a/frappe/website/path_resolver.py b/frappe/website/path_resolver.py
index 37bfb3ee56..50ef2afcb4 100644
--- a/frappe/website/path_resolver.py
+++ b/frappe/website/path_resolver.py
@@ -51,7 +51,6 @@ class PathResolver:
TemplatePage,
ListPage,
PrintPage,
- NotFoundPage,
]
for renderer in renderers:
diff --git a/node_utils.js b/node_utils.js
index 0b8c455875..4835a84433 100644
--- a/node_utils.js
+++ b/node_utils.js
@@ -31,10 +31,6 @@ function get_conf() {
if (process.env.FRAPPE_SITE) {
conf.default_site = process.env.FRAPPE_SITE;
}
- if (fs.existsSync("sites/currentsite.txt")) {
- conf.default_site = fs.readFileSync("sites/currentsite.txt").toString().trim();
- }
-
return conf;
}