diff --git a/frappe/core/doctype/scheduled_job_log/scheduled_job_log.json b/frappe/core/doctype/scheduled_job_log/scheduled_job_log.json index cfa2f27d1a..9e7f72a722 100644 --- a/frappe/core/doctype/scheduled_job_log/scheduled_job_log.json +++ b/frappe/core/doctype/scheduled_job_log/scheduled_job_log.json @@ -1,11 +1,12 @@ { + "actions": [], "creation": "2019-09-23 14:36:36.935869", "doctype": "DocType", "editable_grid": 1, "engine": "InnoDB", "field_order": [ "status", - "scheduled_job", + "scheduled_job_type", "details" ], "fields": [ @@ -20,7 +21,13 @@ "reqd": 1 }, { - "fieldname": "scheduled_job", + "fieldname": "details", + "fieldtype": "Code", + "label": "Details", + "read_only": 1 + }, + { + "fieldname": "scheduled_job_type", "fieldtype": "Link", "in_list_view": 1, "in_standard_filter": 1, @@ -28,15 +35,10 @@ "options": "Scheduled Job Type", "read_only": 1, "reqd": 1 - }, - { - "fieldname": "details", - "fieldtype": "Code", - "label": "Details", - "read_only": 1 } ], - "modified": "2019-09-23 14:36:36.935869", + "links": [], + "modified": "2019-09-25 11:55:10.646458", "modified_by": "Administrator", "module": "Core", "name": "Scheduled Job Log", diff --git a/frappe/core/doctype/scheduled_job_type/scheduled_job_type.json b/frappe/core/doctype/scheduled_job_type/scheduled_job_type.json index ec68544ab5..d35b88e9de 100644 --- a/frappe/core/doctype/scheduled_job_type/scheduled_job_type.json +++ b/frappe/core/doctype/scheduled_job_type/scheduled_job_type.json @@ -69,10 +69,10 @@ "links": [ { "link_doctype": "Scheduled Job Log", - "link_fieldname": "scheduled_job" + "link_fieldname": "scheduled_job_type" } ], - "modified": "2019-09-24 11:45:34.748779", + "modified": "2019-09-25 11:54:58.013623", "modified_by": "Administrator", "module": "Core", "name": "Scheduled Job Type", diff --git a/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py b/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py index 7c6b2a0e0c..9325d62078 100644 --- a/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py +++ b/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py @@ -35,6 +35,10 @@ class ScheduledJobType(Document): queue = self.get_queue_name(), job_type=self.method) return True + else: + pass + #print('not yet due') + return False def is_event_due(self, current_time = None): @@ -90,7 +94,7 @@ class ScheduledJobType(Document): if not self.create_log: return if not self.scheduler_log: - self.scheduler_log = frappe.get_doc(dict(doctype = 'Scheduled Job Log', scheduled_job=self.name)).insert(ignore_permissions=True) + self.scheduler_log = frappe.get_doc(dict(doctype = 'Scheduled Job Log', scheduled_job_type=self.name)).insert(ignore_permissions=True) self.scheduler_log.db_set('status', status) if status == 'Failed': self.scheduler_log.db_set('details', frappe.get_traceback()) @@ -112,6 +116,7 @@ def execute_event(doc): def run_scheduled_job(job_type): '''This is a wrapper function that runs a hooks.scheduler_events method''' try: + print('executing job {}'.format(job_type)) frappe.get_doc('Scheduled Job Type', dict(method=job_type)).execute() except Exception: print(frappe.get_traceback()) diff --git a/frappe/database/database.py b/frappe/database/database.py index 1e6a85236e..5779c77e86 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -15,7 +15,7 @@ import frappe.model.meta from frappe import _ from time import time -from frappe.utils import now, getdate, cast_fieldtype +from frappe.utils import now, getdate, cast_fieldtype, get_datetime from frappe.utils.background_jobs import execute_job, get_queue from frappe.model.utils.link_count import flush_local_link_count from frappe.utils import cint @@ -940,6 +940,16 @@ class Database(object): else: frappe.throw(_('No conditions provided')) + def get_last_created(self, doctype): + last_record = self.get_all(doctype, ('creation'), limit=1, order_by='creation desc') + if last_record: + return get_datetime(last_record[0].creation) + else: + return None + + def clear_table(self, doctype): + self.sql('truncate `tab{}`'.format(doctype)) + def log_touched_tables(self, query, values=None): if values: query = frappe.safe_decode(self._cursor.mogrify(query, values)) diff --git a/frappe/tests/test_scheduler.py b/frappe/tests/test_scheduler.py index b8c46d376f..3617a3983b 100644 --- a/frappe/tests/test_scheduler.py +++ b/frappe/tests/test_scheduler.py @@ -4,15 +4,21 @@ from unittest import TestCase from dateutil.relativedelta import relativedelta from frappe.core.doctype.scheduled_job_type.scheduled_job_type import sync_jobs from frappe.utils.background_jobs import enqueue, get_jobs -from frappe.utils.scheduler import enqueue_events +from frappe.utils.scheduler import enqueue_events, is_dormant, schedule_jobs_based_on_activity +from frappe.utils import add_days, get_datetime import frappe import time def test_timeout(): - '''This function needs to be pickleable''' time.sleep(100) +def test_timeout_10(): + time.sleep(10) + +def test_method(): + pass + class TestScheduler(TestCase): def setUp(self): if not frappe.get_all('Scheduled Job Type', limit=1): @@ -30,24 +36,38 @@ class TestScheduler(TestCase): self.assertTrue('frappe.email.doctype.auto_email_report.auto_email_report.send_monthly', frappe.flags.enqueued_jobs) def test_queue_peeking(self): - if not frappe.db.exists('Scheduled Job Type', 'test_scheduler.test_timeout'): - job = frappe.get_doc(dict( - doctype = 'Scheduled Job Type', - method = 'frappe.tests.test_scheduler.test_timeout', - last_execution = '2010-01-01 00:00:00', - queue = 'All' - )).insert() - frappe.db.commit() - else: - job = frappe.get_doc('Scheduled Job Type', 'test_scheduler.test_timeout') - + job = get_test_job() self.assertTrue(job.enqueue()) job.db_set('last_execution', '2010-01-01 00:00:00') frappe.db.commit() - time.sleep(1) # wait if job is not yet queued + time.sleep(3) # wait if job is not yet queued self.assertFalse(job.enqueue()) job.delete() + def test_is_dormant(self): + self.assertTrue(is_dormant(check_time= get_datetime('2100-01-01 00:00:00'))) + self.assertTrue(is_dormant(check_time = add_days(frappe.db.get_last_created('Activity Log'), 5))) + self.assertFalse(is_dormant(check_time = frappe.db.get_last_created('Activity Log'))) + + def test_once_a_day_for_dormant(self): + frappe.db.clear_table('Scheduled Job Log') + self.assertTrue(schedule_jobs_based_on_activity(check_time= get_datetime('2100-01-01 00:00:00'))) + self.assertTrue(schedule_jobs_based_on_activity(check_time = add_days(frappe.db.get_last_created('Activity Log'), 5))) + + # create a fake job executed 5 days from now + job = get_test_job(method='frappe.tests.test_scheduler.test_method', queue='Daily') + job.execute() + job_log = frappe.get_doc('Scheduled Job Log', dict(scheduled_job_type=job.name)) + job_log.db_set('creation', add_days(frappe.db.get_last_created('Activity Log'), 5)) + + # inactive site with recent job, don't run + self.assertFalse(schedule_jobs_based_on_activity(check_time = add_days(frappe.db.get_last_created('Activity Log'), 5))) + + # one more day has passed + self.assertTrue(schedule_jobs_based_on_activity(check_time = add_days(frappe.db.get_last_created('Activity Log'), 6))) + + frappe.db.rollback() + def test_job_timeout(self): return job = enqueue(test_timeout, timeout=10) @@ -59,3 +79,20 @@ class TestScheduler(TestCase): break self.assertTrue(job.is_failed) + +def get_test_job(method='frappe.tests.test_scheduler.test_timeout_10', queue='All'): + if not frappe.db.exists('Scheduled Job Type', dict(method=method)): + job = frappe.get_doc(dict( + doctype = 'Scheduled Job Type', + method = method, + last_execution = '2010-01-01 00:00:00', + queue = queue + )).insert() + frappe.db.commit() + else: + job = frappe.get_doc('Scheduled Job Type', dict(method=method)) + job.db_set('last_execution', '2010-01-01 00:00:00') + job.db_set('queue', queue) + + return job + diff --git a/frappe/utils/scheduler.py b/frappe/utils/scheduler.py index 4d9f782542..fb606d6d3d 100755 --- a/frappe/utils/scheduler.py +++ b/frappe/utils/scheduler.py @@ -108,16 +108,16 @@ def enable_scheduler(): def disable_scheduler(): toggle_scheduler(False) -def schedule_jobs_based_on_activity(): +def schedule_jobs_based_on_activity(check_time=None): '''Returns True for active sites defined by Activity Log Returns True for inactive sites once in 24 hours''' - if is_dormant(): + if is_dormant(check_time=check_time): # ensure last job is one day old - last_job = frappe.db.get_all('Scheduled Job Log', ('creation'), limit=1, order_by='creation desc') - if not last_job: + last_job_timestamp = frappe.db.get_last_created('Scheduled Job Log') + if not last_job_timestamp: return True else: - if (now_datetime() - get_datetime(last_job[0].creation)).seconds > 86400: + if ((check_time or now_datetime()) - last_job_timestamp).total_seconds() >= 86400: # one day is passed since jobs are run, so lets do this return True else: @@ -127,12 +127,12 @@ def schedule_jobs_based_on_activity(): # site active, lets run the jobs return True -def is_dormant(): - last_activity_log = frappe.db.get_all('Activity Log', ('modified'), limit=1, order_by='modified desc') +def is_dormant(check_time=None): + last_activity_log_timestamp = frappe.db.get_last_created('Activity Log') since = (frappe.get_system_settings('dormant_days') or 4) * 86400 - if not last_activity_log: + if not last_activity_log_timestamp: return True - if (now_datetime() - get_datetime(last_activity_log[0].modified)).seconds > since: + if ((check_time or now_datetime()) - last_activity_log_timestamp).total_seconds() >= since: return True return False \ No newline at end of file