vassili/seitime-frappe
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

perf: use modified instead of creation in scheduler (#18234)

Browse source 
creation doesn't have index and does full table scan. On large site with
~3M activity logs it takes ~1.5 seconds just to begin scheduler.


[skip ci]
This commit is contained in:
Ankush Menat 2022-09-27 19:21:02 +05:30 • committed by GitHub
parent 07662ebaf3
commit 0642708f01
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 23 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
frappe/tests/test_scheduler.py
View file

@ -7,7 +7,12 @@ import frappe
from frappe.core.doctype.scheduled_job_type.scheduled_job_type import ScheduledJobType, sync_jobs
from frappe.utils import add_days, get_datetime
from frappe.utils.doctor import purge_pending_jobs
from frappe.utils.scheduler import enqueue_events, is_dormant, schedule_jobs_based_on_activity
from frappe.utils.scheduler import (
_get_last_modified_timestamp,
enqueue_events,
is_dormant,
schedule_jobs_based_on_activity,
)
def test_timeout_10():
@ -56,7 +61,7 @@ class TestScheduler(TestCase):
self.assertFalse(is_dormant(check_time=frappe.db.get_last_created("Activity Log")))
def test_once_a_day_for_dormant(self):
frappe.db.clear_table("Scheduled Job Log")
frappe.db.truncate("Scheduled Job Log")
self.assertTrue(schedule_jobs_based_on_activity(check_time=get_datetime("2100-01-01 00:00:00")))
self.assertTrue(
schedule_jobs_based_on_activity(
@ -68,19 +73,21 @@ class TestScheduler(TestCase):
job = get_test_job(method="frappe.tests.test_scheduler.test_method", frequency="Daily")
job.execute()
job_log = frappe.get_doc("Scheduled Job Log", dict(scheduled_job_type=job.name))
job_log.db_set("creation", add_days(frappe.db.get_last_created("Activity Log"), 5))
job_log.db_set(
"modified", add_days(_get_last_modified_timestamp("Activity Log"), 5), update_modified=False
)
# inactive site with recent job, don't run
self.assertFalse(
schedule_jobs_based_on_activity(
check_time=add_days(frappe.db.get_last_created("Activity Log"), 5)
check_time=add_days(_get_last_modified_timestamp("Activity Log"), 5)
)
)
# one more day has passed
self.assertTrue(
schedule_jobs_based_on_activity(
check_time=add_days(frappe.db.get_last_created("Activity Log"), 6)
check_time=add_days(_get_last_modified_timestamp("Activity Log"), 6)
)
)

14
frappe/utils/scheduler.py
View file

@ -18,7 +18,7 @@ import schedule
# imports - module imports
import frappe
from frappe.installer import update_site_config
from frappe.utils import get_sites, now_datetime
from frappe.utils import get_datetime, get_sites, now_datetime
from frappe.utils.background_jobs import get_jobs
DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S"
@ -146,7 +146,7 @@ def schedule_jobs_based_on_activity(check_time=None):
Returns True for inactive sites once in 24 hours"""
if is_dormant(check_time=check_time):
# ensure last job is one day old
last_job_timestamp = frappe.db.get_last_created("Scheduled Job Log")
last_job_timestamp = _get_last_modified_timestamp("Scheduled Job Log")
if not last_job_timestamp:
return True
else:
@ -162,7 +162,7 @@ def schedule_jobs_based_on_activity(check_time=None):
def is_dormant(check_time=None):
last_activity_log_timestamp = frappe.db.get_last_created("Activity Log")
last_activity_log_timestamp = _get_last_modified_timestamp("Activity Log")
since = (frappe.get_system_settings("dormant_days") or 4) * 86400
if not last_activity_log_timestamp:
return True
@ -171,6 +171,14 @@ def is_dormant(check_time=None):
return False
def _get_last_modified_timestamp(doctype):
timestamp = frappe.db.get_value(
doctype, filters={}, fieldname="modified", order_by="modified desc"
)
if timestamp:
return get_datetime(timestamp)
@frappe.whitelist()
def activate_scheduler():
if is_scheduler_disabled():