vassili/seitime-frappe
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

reactor(scheduler): created "Scheduler Job Type" and cleaned up scheduler

Browse source
This commit is contained in:
Rushabh Mehta 2019-09-24 00:16:44 +05:30
parent 7eb322f2be
commit 7cd329fac9
35 changed files with 561 additions and 333 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.pylintrc Normal file
View file

@ -0,0 +1,2 @@
disable=access-member-before-definition
disable=no-member

16
frappe/__init__.py
View file

@ -123,7 +123,6 @@ def init(site, sites_path=None, new_site=False):
local.debug_log = []
local.realtime_log = []
local.flags = _dict({
"ran_schedulers": [],
"currently_saving": [],
"redirect_location": "",
"in_install_db": False,
@ -1504,7 +1503,20 @@ def logger(module=None, with_more_info=True):
def log_error(message=None, title=None):
'''Log error to Error Log'''
return get_doc(dict(doctype='Error Log', error=as_unicode(message or get_traceback()),
# AI ALERT:
# the title and message may be swapped
# the better API for this is log_error(title, message), and used in many cases this way
# this hack tries to be smart about whats a title (single line ;-)) and fixes it
if message:
if '\n' not in message:
title = message
error = get_traceback()
else:
error = message
return get_doc(dict(doctype='Error Log', error=as_unicode(error),
method=title)).insert(ignore_permissions=True)
def get_desk_link(doctype, name):

13
frappe/core/doctype/communication/email.py
View file

@ -10,7 +10,6 @@ from email.utils import formataddr
from frappe.core.utils import get_parent_doc
from frappe.utils import (get_url, get_formatted_email, cint,
validate_email_address, split_emails, time_diff_in_seconds, parse_addr, get_datetime)
from frappe.utils.scheduler import log
from frappe.email.email_body import get_message_id
import frappe.email.smtp
import time
@ -509,17 +508,7 @@ def sendmail(communication_name, print_html=None, print_format=None, attachments
break
except:
traceback = log("frappe.core.doctype.communication.email.sendmail", frappe.as_json({
"communication_name": communication_name,
"print_html": print_html,
"print_format": print_format,
"attachments": attachments,
"recipients": recipients,
"cc": cc,
"bcc": bcc,
"lang": lang
}))
frappe.logger(__name__).error(traceback)
traceback = frappe.log_error("frappe.core.doctype.communication.email.sendmail")
raise
def update_mins_to_first_communication(parent, communication):

15
frappe/core/doctype/doctype/doctype.json
View file

@ -57,6 +57,8 @@
"restrict_to_domain",
"read_only",
"in_create",
"actions_section",
"actions",
"web_view",
"has_web_view",
"allow_guest_to_view",
@ -454,11 +456,22 @@
"fieldname": "nsm_parent_field",
"fieldtype": "Data",
"label": "Parent Field (Tree)"
},
{
"fieldname": "actions_section",
"fieldtype": "Section Break",
"label": "Actions"
},
{
"fieldname": "actions",
"fieldtype": "Table",
"label": "Actions",
"options": "DocType Action"
}
],
"icon": "fa fa-bolt",
"idx": 6,
"modified": "2019-09-07 14:28:05.392490",
"modified": "2019-09-23 16:29:21.209832",
"modified_by": "Administrator",
"module": "Core",
"name": "DocType",

0
frappe/core/doctype/doctype_action/__init__.py Normal file
View file

54
frappe/core/doctype/doctype_action/doctype_action.json Normal file
View file

@ -0,0 +1,54 @@
{
"actions": [],
"creation": "2019-09-23 16:28:13.953520",
"doctype": "DocType",
"editable_grid": 1,
"engine": "InnoDB",
"field_order": [
"label",
"action_type",
"method",
"group"
],
"fields": [
{
"columns": 2,
"fieldname": "label",
"fieldtype": "Data",
"in_list_view": 1,
"label": "Label",
"reqd": 1
},
{
"columns": 6,
"fieldname": "method",
"fieldtype": "Data",
"in_list_view": 1,
"label": "Method",
"reqd": 1
},
{
"fieldname": "group",
"fieldtype": "Data",
"in_list_view": 1,
"label": "Group"
},
{
"fieldname": "action_type",
"fieldtype": "Data",
"label": "Action Type",
"options": "Server Action"
}
],
"istable": 1,
"modified": "2019-09-23 21:34:39.971700",
"modified_by": "Administrator",
"module": "Core",
"name": "DocType Action",
"owner": "Administrator",
"permissions": [],
"quick_entry": 1,
"sort_field": "modified",
"sort_order": "DESC",
"track_changes": 1
}

10
frappe/core/doctype/doctype_action/doctype_action.py Normal file
View file

@ -0,0 +1,10 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2019, Frappe Technologies and contributors
# For license information, please see license.txt
from __future__ import unicode_literals
# import frappe
from frappe.model.document import Document
class DocTypeAction(Document):
pass

0
frappe/core/doctype/scheduled_job_log/__init__.py Normal file
View file

8
frappe/core/doctype/scheduled_job_log/scheduled_job_log.js Normal file
View file

@ -0,0 +1,8 @@
// Copyright (c) 2019, Frappe Technologies and contributors
// For license information, please see license.txt
frappe.ui.form.on('Scheduled Job Log', {
// refresh: function(frm) {
// }
});

62
frappe/core/doctype/scheduled_job_log/scheduled_job_log.json Normal file
View file

@ -0,0 +1,62 @@
{
"creation": "2019-09-23 14:36:36.935869",
"doctype": "DocType",
"editable_grid": 1,
"engine": "InnoDB",
"field_order": [
"status",
"scheduled_job",
"details"
],
"fields": [
{
"fieldname": "status",
"fieldtype": "Select",
"in_list_view": 1,
"in_standard_filter": 1,
"label": "Status",
"options": "Scheduled\nSuccess\nFailed",
"read_only": 1,
"reqd": 1
},
{
"fieldname": "scheduled_job",
"fieldtype": "Link",
"in_list_view": 1,
"in_standard_filter": 1,
"label": "Scheduled Job",
"options": "Scheduled Job Type",
"read_only": 1,
"reqd": 1
},
{
"fieldname": "details",
"fieldtype": "Code",
"label": "Details",
"read_only": 1
}
],
"modified": "2019-09-23 14:36:36.935869",
"modified_by": "Administrator",
"module": "Core",
"name": "Scheduled Job Log",
"owner": "Administrator",
"permissions": [
{
"create": 1,
"delete": 1,
"email": 1,
"export": 1,
"print": 1,
"read": 1,
"report": 1,
"role": "System Manager",
"share": 1,
"write": 1
}
],
"quick_entry": 1,
"sort_field": "modified",
"sort_order": "DESC",
"track_changes": 1
}

10
frappe/core/doctype/scheduled_job_log/scheduled_job_log.py Normal file
View file

@ -0,0 +1,10 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2019, Frappe Technologies and contributors
# For license information, please see license.txt
from __future__ import unicode_literals
# import frappe
from frappe.model.document import Document
class ScheduledJobLog(Document):
pass

10
frappe/core/doctype/scheduled_job_log/test_scheduled_job_log.py Normal file
View file

@ -0,0 +1,10 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2019, Frappe Technologies and Contributors
# See license.txt
from __future__ import unicode_literals
# import frappe
import unittest
class TestScheduledJobLog(unittest.TestCase):
pass

0
frappe/core/doctype/scheduled_job_type/__init__.py Normal file
View file

8
frappe/core/doctype/scheduled_job_type/scheduled_job_type.js Normal file
View file

@ -0,0 +1,8 @@
// Copyright (c) 2019, Frappe Technologies and contributors
// For license information, please see license.txt
frappe.ui.form.on('Scheduled Job Type', {
// refresh: function(frm) {
// }
});

92
frappe/core/doctype/scheduled_job_type/scheduled_job_type.json Normal file
View file

@ -0,0 +1,92 @@
{
"actions": [
{
"action_path": "frappe.core.doctype.scheduled_job_type.scheduled_job_type.execute_event",
"label": "Execute",
"method": "frappe.core.doctype.scheduled_job_type.scheduled_job_type.execute_event"
}
],
"creation": "2019-09-23 14:34:09.205368",
"doctype": "DocType",
"editable_grid": 1,
"engine": "InnoDB",
"field_order": [
"stopped",
"method",
"queue",
"cron_format",
"last_execution",
"create_log"
],
"fields": [
{
"fieldname": "method",
"fieldtype": "Data",
"in_list_view": 1,
"label": "Method",
"read_only": 1,
"reqd": 1
},
{
"fieldname": "queue",
"fieldtype": "Select",
"in_list_view": 1,
"in_standard_filter": 1,
"label": "Queue",
"options": "All\nHourly\nDaily\nDaily Long\nWeekly\nWeekly Long\nMonthly\nMonthly Long\nCron\nYearly\nAnnual",
"read_only": 1,
"reqd": 1
},
{
"default": "0",
"fieldname": "stopped",
"fieldtype": "Check",
"label": "Stopped"
},
{
"default": "0",
"depends_on": "eval:doc.queue==='All'",
"fieldname": "create_log",
"fieldtype": "Check",
"label": "Create Log"
},
{
"fieldname": "last_execution",
"fieldtype": "Datetime",
"label": "Last Execution",
"read_only": 1
},
{
"allow_in_quick_entry": 1,
"depends_on": "eval:doc.queue==='Cron'",
"fieldname": "cron_format",
"fieldtype": "Data",
"label": "Cron Format",
"read_only": 1
}
],
"in_create": 1,
"modified": "2019-09-23 22:19:22.594874",
"modified_by": "Administrator",
"module": "Core",
"name": "Scheduled Job Type",
"owner": "Administrator",
"permissions": [
{
"create": 1,
"delete": 1,
"email": 1,
"export": 1,
"print": 1,
"read": 1,
"report": 1,
"role": "System Manager",
"share": 1,
"write": 1
}
],
"quick_entry": 1,
"sort_field": "modified",
"sort_order": "DESC",
"track_changes": 1
}

129
frappe/core/doctype/scheduled_job_type/scheduled_job_type.py Normal file
View file

@ -0,0 +1,129 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2019, Frappe Technologies and contributors
# For license information, please see license.txt
from __future__ import unicode_literals
import frappe, json
from frappe.model.document import Document
from frappe.utils import now_datetime, get_datetime
from datetime import datetime
from croniter import croniter
from frappe.utils.background_jobs import enqueue
CRON_MAP = {
"Yearly": "0 0 1 1 *",
"Annual": "0 0 1 1 *",
"Monthly": "0 0 1 * *",
"Monthly Long": "0 0 1 * *",
"Weekly": "0 0 * * 0",
"Weekly Long": "0 0 * * 0",
"Daily": "0 0 * * *",
"Daily Long": "0 0 * * *",
"Hourly": "0 * * * *",
"Hourly Long": "0 * * * *",
"All": "0/" + str((frappe.get_conf().scheduler_interval or 240) // 60) + " * * * *",
}
class ScheduledJobType(Document):
def autoname(self):
self.name = '.'.join(self.method.split('.')[-2:])
def validate(self):
if self.queue != 'All':
# force logging for all events other than continuous ones (ALL)
self.create_log = 1
def enqueue(self):
# enqueue event if last execution is done
if self.is_event_due():
self.update_last_execution()
frappe.flags.enqueued_jobs.append(self.method)
enqueue('frappe.core.doctype.scheduled_job_type.scheduled_job_type.run_scheduled_job',
job_type=self.method)
def is_event_due(self, current_time = None):
'''Return true if event is due based on time lapsed since last execution'''
# save last execution in expected execution time as per cron
self.last_execution = self.get_next_execution()
# if the next scheduled event is before NOW, then its due!
return self.last_execution <= (current_time or now_datetime())
def get_next_execution(self):
if not self.cron_format:
self.cron_format = CRON_MAP[self.queue]
return croniter(self.cron_format,
get_datetime(self.last_execution)).get_next(datetime)
def execute(self):
try:
frappe.logger(__name__).info('Started Scheduled Job: {0} for {1}'.format(self.method, frappe.local.site))
frappe.get_attr(self.method)()
frappe.db.commit()
frappe.logger(__name__).info('Completed Scheduled Job: {0} for {1}'.format(self.method, frappe.local.site))
except Exception:
frappe.db.rollback()
frappe.log_error('{} failed'.format(self.method))
frappe.logger(__name__).info('Failed Scheduled Job: {0} for {1}'.format(self.method, frappe.local.site))
def update_last_execution(self):
self.db_set('last_execution', self.last_execution, update_modified=False)
frappe.db.commit()
def get_queue_name(self):
return self.queue.replace(' ', '_').lower()
@frappe.whitelist()
def execute_event(doc):
frappe.only_for('System Manager')
doc = json.loads(doc)
frappe.get_doc('Scheduled Job Type', doc.get('name')).execute()
def run_scheduled_job(job_type):
'''This is a wrapper function that runs a hooks.scheduler_events method'''
frappe.get_doc('Scheduled Job Type', dict(method=job_type)).execute()
def sync_jobs():
frappe.reload_doc('core', 'doctype', 'scheduled_job_type')
all_events = []
scheduler_events = frappe.get_hooks("scheduler_events")
insert_events(all_events, scheduler_events)
clear_events(all_events, scheduler_events)
def insert_events(all_events, scheduler_events):
for event_type in scheduler_events:
events = scheduler_events.get(event_type)
if isinstance(events, dict):
insert_cron_event(events, all_events)
else:
# hourly, daily etc
insert_event_list(events, event_type, all_events)
def insert_cron_event(events, all_events):
for cron_format in events:
for event in events.get(cron_format):
all_events.append(event)
insert_single_event('Cron', event, cron_format)
def insert_event_list(events, event_type, all_events):
for event in events:
all_events.append(event)
queue = event_type.replace('_', ' ').title()
insert_single_event(queue, event)
def insert_single_event(queue, event, cron_format = None):
if not frappe.db.exists('Scheduled Job Type', dict(method=event)):
frappe.get_doc(dict(
doctype = 'Scheduled Job Type',
method = event,
cron_format = cron_format,
queue = queue
)).insert()
def clear_events(all_events, scheduler_events):
for event in frappe.get_all('Scheduled Job Type', ('name', 'method')):
if event.method not in all_events:
frappe.db.delete_doc('Scheduled Job Type', event.name)

62
frappe/core/doctype/scheduled_job_type/test_scheduled_job_type.py Normal file
View file

@ -0,0 +1,62 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2019, Frappe Technologies and Contributors
# See license.txt
from __future__ import unicode_literals
import frappe
import unittest
from frappe.utils import get_datetime
from frappe.core.doctype.scheduled_job_type.scheduled_job_type import sync_jobs
class TestScheduledJobType(unittest.TestCase):
def setUp(self):
if not frappe.get_all('Scheduled Job Type', limit=1):
frappe.db.rollback()
frappe.db.sql('truncate `tabScheduled Job Type`')
sync_jobs()
frappe.db.commit()
def test_sync_jobs(self):
all_job = frappe.get_doc('Scheduled Job Type',
dict(method='frappe.email.queue.flush'))
self.assertEqual(all_job.queue, 'All')
daily_job = frappe.get_doc('Scheduled Job Type',
dict(method='frappe.email.queue.clear_outbox'))
self.assertEqual(daily_job.queue, 'Daily')
# check if cron jobs are synced
cron_job = frappe.get_doc('Scheduled Job Type',
dict(method='frappe.oauth.delete_oauth2_data'))
self.assertEqual(cron_job.queue, 'Cron')
self.assertEqual(cron_job.cron_format, '0/15 * * * *')
def test_daily_job(self):
job = frappe.get_doc('Scheduled Job Type', dict(method = 'frappe.email.queue.clear_outbox'))
job.db_set('last_execution', '2019-01-01 00:00:00')
self.assertTrue(job.is_event_due(get_datetime('2019-01-02 00:00:06')))
self.assertFalse(job.is_event_due(get_datetime('2019-01-01 00:00:06')))
self.assertFalse(job.is_event_due(get_datetime('2019-01-01 23:59:59')))
def test_weekly_job(self):
job = frappe.get_doc('Scheduled Job Type', dict(method = 'frappe.utils.change_log.check_for_update'))
job.db_set('last_execution', '2019-01-01 00:00:00')
self.assertTrue(job.is_event_due(get_datetime('2019-01-06 00:00:01')))
self.assertFalse(job.is_event_due(get_datetime('2019-01-02 00:00:06')))
self.assertFalse(job.is_event_due(get_datetime('2019-01-05 23:59:59')))
def test_monthly_job(self):
job = frappe.get_doc('Scheduled Job Type', dict(method = 'frappe.email.doctype.auto_email_report.auto_email_report.send_monthly'))
job.db_set('last_execution', '2019-01-01 00:00:00')
self.assertTrue(job.is_event_due(get_datetime('2019-02-01 00:00:01')))
self.assertFalse(job.is_event_due(get_datetime('2019-01-15 00:00:06')))
self.assertFalse(job.is_event_due(get_datetime('2019-01-31 23:59:59')))
def test_cron_job(self):
# runs every 15 mins
job = frappe.get_doc('Scheduled Job Type', dict(method = 'frappe.oauth.delete_oauth2_data'))
job.db_set('last_execution', '2019-01-01 00:00:00')
self.assertTrue(job.is_event_due(get_datetime('2019-01-01 00:15:01')))
self.assertFalse(job.is_event_due(get_datetime('2019-01-01 00:05:06')))
self.assertFalse(job.is_event_due(get_datetime('2019-01-01 00:14:59')))

2
frappe/core/doctype/version/version.py
View file

@ -50,7 +50,7 @@ def get_diff(old, new, for_child=False):
if df.fieldtype in no_value_fields and df.fieldtype not in table_fields:
continue
old_value, new_value = old.get(df.fieldname), new.get(df.fieldname)
old_value, new_value = old.get(df.fieldname) or [], new.get(df.fieldname) or []
if df.fieldtype in table_fields:
# make maps

3
frappe/email/doctype/email_account/email_account.py
View file

@ -21,7 +21,6 @@ from frappe.desk.form import assign_to
from frappe.utils.user import get_system_managers
from frappe.utils.background_jobs import enqueue, get_jobs
from frappe.core.doctype.communication.email import set_incoming_outgoing_accounts
from frappe.utils.scheduler import log
from frappe.utils.html_utils import clean_email_html
from frappe.email.utils import get_port
@ -284,7 +283,7 @@ class EmailAccount(Document):
except Exception:
frappe.db.rollback()
log('email_account.receive')
frappe.log_error('email_account.receive')
if self.use_imap:
self.handle_bad_emails(email_server, uid, msg, frappe.get_traceback())
exceptions.append(frappe.get_traceback())

3
frappe/email/doctype/newsletter/newsletter.py
View file

@ -9,7 +9,6 @@ from frappe import throw, _
from frappe.website.website_generator import WebsiteGenerator
from frappe.utils.verified_command import get_signed_params, verify_request
from frappe.utils.background_jobs import enqueue
from frappe.utils.scheduler import log
from frappe.email.queue import send
from frappe.email.doctype.email_group.email_group import add_subscribers
from frappe.utils import parse_addr
@ -213,7 +212,7 @@ def send_newsletter(newsletter):
doc.db_set("email_sent", 0)
frappe.db.commit()
log("send_newsletter")
frappe.log_error("send_newsletter")
raise

3
frappe/email/queue.py
View file

@ -12,7 +12,6 @@ from frappe.utils.verified_command import get_signed_params, verify_request
from html2text import html2text
from frappe.utils import get_url, nowdate, encode, now_datetime, add_days, split_emails, cstr, cint
from rq.timeouts import JobTimeoutException
from frappe.utils.scheduler import log
from six import text_type, string_types
class EmailLimitCrossedError(frappe.ValidationError): pass
@ -469,7 +468,7 @@ def send_one(email, smtpserver=None, auto_commit=True, now=False, from_test=Fals
else:
# log to Error Log
log('frappe.email.queue.flush', text_type(e))
frappe.log_error('frappe.email.queue.flush')
def prepare_message(email, recipient, recipients_list):
message = email.message

5
frappe/email/receive.py
View file

@ -12,7 +12,6 @@ import frappe
from frappe import _, safe_decode, safe_encode
from frappe.utils import (extract_email_id, convert_utc_to_user_timezone, now,
cint, cstr, strip, markdown, parse_addr)
from frappe.utils.scheduler import log
from frappe.core.doctype.file.file import get_random_filename, MaxFileSizeReachedError
class EmailSizeExceededError(frappe.ValidationError): pass
@ -80,7 +79,7 @@ class EmailServer:
except _socket.error:
# log performs rollback and logs error in Error Log
log("receive.connect_pop")
frappe.log_error("receive.connect_pop")
# Invalid mail server -- due to refusing connection
frappe.msgprint(_('Invalid Mail Server. Please rectify and try again.'))
@ -255,7 +254,7 @@ class EmailServer:
else:
# log performs rollback and logs error in Error Log
log("receive.get_messages", self.make_error_msg(msg_num, incoming_mail))
frappe.log_error("receive.get_messages", self.make_error_msg(msg_num, incoming_mail))
self.errors = True
frappe.db.rollback()

13
frappe/hooks.py
View file

@ -76,8 +76,7 @@ leaderboards = "frappe.desk.leaderboard.get_leaderboards"
on_session_creation = [
"frappe.core.doctype.activity_log.feed.login_feed",
"frappe.core.doctype.user.user.notify_admin_access_to_system_manager",
"frappe.utils.scheduler.reset_enabled_scheduler_events",
"frappe.core.doctype.user.user.notify_admin_access_to_system_manager"
]
on_logout = "frappe.core.doctype.session_default_settings.session_default_settings.clear_session_defaults"
@ -153,14 +152,18 @@ doc_events = {
}
scheduler_events = {
"cron": {
"0/15 * * * *": [
"frappe.oauth.delete_oauth2_data",
"frappe.website.doctype.web_page.web_page.check_publish_status",
"frappe.twofactor.delete_all_barcodes_for_users"
]
},
"all": [
"frappe.email.queue.flush",
"frappe.email.doctype.email_account.email_account.pull",
"frappe.email.doctype.email_account.email_account.notify_unreplied",
"frappe.oauth.delete_oauth2_data",
"frappe.integrations.doctype.razorpay_settings.razorpay_settings.capture_payment",
"frappe.twofactor.delete_all_barcodes_for_users",
"frappe.website.doctype.web_page.web_page.check_publish_status",
'frappe.utils.global_search.sync_global_search'
],
"hourly": [

3
frappe/migrate.py
View file

@ -15,6 +15,7 @@ from frappe.desk.notifications import clear_notifications
from frappe.website import render
from frappe.core.doctype.language.language import sync_languages
from frappe.modules.utils import sync_customizations
from frappe.core.doctype.scheduled_job_type.scheduled_job_type import sync_jobs
from frappe.utils import global_search
def migrate(verbose=True, rebuild_website=False, skip_failing=False):
@ -46,9 +47,11 @@ def migrate(verbose=True, rebuild_website=False, skip_failing=False):
# run patches
frappe.modules.patch_handler.run_all(skip_failing)
# sync
frappe.model.sync.sync_all(verbose=verbose)
frappe.translate.clear_cache()
sync_jobs()
sync_fixtures()
sync_customizations()
sync_languages()

2
frappe/model/__init__.py
View file

@ -45,7 +45,7 @@ default_fields = ('doctype','name','owner','creation','modified','modified_by',
'parent','parentfield','parenttype','idx','docstatus')
optional_fields = ("_user_tags", "_comments", "_assign", "_liked_by", "_seen")
table_fields = ('Table', 'Table MultiSelect')
core_doctypes_list = ('DocType', 'DocField', 'DocPerm', 'User', 'Role', 'Has Role',
core_doctypes_list = ('DocType', 'DocField', 'DocPerm', 'DocType Action','User', 'Role', 'Has Role',
'Page', 'Module Def', 'Print Format', 'Report', 'Customize Form',
'Customize Form Field', 'Property Setter', 'Custom Field', 'Custom Script')

8
frappe/model/base_document.py
View file

@ -22,6 +22,8 @@ max_positive_value = {
'bigint': 2 ** 63
}
DOCTYPES_FOR_DOCTYPE = ('DocType', 'DocField', 'DocPerm', 'DocType Action')
_classes = {}
def get_controller(doctype):
@ -255,7 +257,7 @@ class BaseDocument(object):
def get_valid_columns(self):
if self.doctype not in frappe.local.valid_columns:
if self.doctype in ("DocField", "DocPerm") and self.parent in ("DocType", "DocField", "DocPerm"):
if self.doctype in DOCTYPES_FOR_DOCTYPE:
from frappe.model.meta import get_table_columns
valid = get_table_columns(self.doctype)
else:
@ -312,7 +314,7 @@ class BaseDocument(object):
self.created_by = self.modified_by = frappe.session.user
# if doctype is "DocType", don't insert null values as we don't know who is valid yet
d = self.get_valid_dict(convert_dates_to_str=True, ignore_nulls = self.doctype in ('DocType', 'DocField', 'DocPerm'))
d = self.get_valid_dict(convert_dates_to_str=True, ignore_nulls = self.doctype in DOCTYPES_FOR_DOCTYPE)
columns = list(d)
try:
@ -347,7 +349,7 @@ class BaseDocument(object):
self.db_insert()
return
d = self.get_valid_dict(convert_dates_to_str=True, ignore_nulls = self.doctype in ('DocType', 'DocField', 'DocPerm'))
d = self.get_valid_dict(convert_dates_to_str=True, ignore_nulls = self.doctype in DOCTYPES_FOR_DOCTYPE)
# don't update name, as case might've been changed
name = d['name']

4
frappe/model/document.py
View file

@ -150,8 +150,8 @@ class Document(BaseDocument):
super(Document, self).__init__(d)
if self.name=="DocType" and self.doctype=="DocType":
from frappe.model.meta import doctype_table_fields
table_fields = doctype_table_fields
from frappe.model.meta import DOCTYPE_TABLE_FIELDS
table_fields = DOCTYPE_TABLE_FIELDS
else:
table_fields = self.meta.get_table_fields()

11
frappe/model/meta.py
View file

@ -151,7 +151,7 @@ class Meta(Document):
if self.name!="DocType":
self._table_fields = self.get('fields', {"fieldtype": ['in', table_fields]})
else:
self._table_fields = doctype_table_fields
self._table_fields = DOCTYPE_TABLE_FIELDS
return self._table_fields
@ -165,7 +165,7 @@ class Meta(Document):
def get_valid_columns(self):
if not hasattr(self, "_valid_columns"):
if self.name in ("DocType", "DocField", "DocPerm", "Property Setter"):
if self.name in ("DocType", "DocField", "DocPerm", 'DocType Action',"Property Setter"):
self._valid_columns = get_table_columns(self.name)
else:
self._valid_columns = self.default_fields + \
@ -174,7 +174,7 @@ class Meta(Document):
return self._valid_columns
def get_table_field_doctype(self, fieldname):
return { "fields": "DocField", "permissions": "DocPerm"}.get(fieldname)
return { "fields": "DocField", "permissions": "DocPerm", "actions": "DocType Action"}.get(fieldname)
def get_field(self, fieldname):
'''Return docfield from meta'''
@ -441,9 +441,10 @@ class Meta(Document):
def is_nested_set(self):
return self.has_field('lft') and self.has_field('rgt')
doctype_table_fields = [
DOCTYPE_TABLE_FIELDS = [
frappe._dict({"fieldname": "fields", "options": "DocField"}),
frappe._dict({"fieldname": "permissions", "options": "DocPerm"})
frappe._dict({"fieldname": "permissions", "options": "DocPerm"}),
frappe._dict({"fieldname": "actions", "options": "DocType Action"}),
]
#######

1
frappe/model/sync.py
View file

@ -29,6 +29,7 @@ def sync_for(app_name, force=0, sync_everything = False, verbose=False, reset_pe
# these need to go first at time of install
for d in (("core", "docfield"),
("core", "docperm"),
("core", "doctype_action"),
("core", "role"),
("core", "has_role"),
("core", "doctype"),

1
frappe/patches.txt
View file

@ -9,6 +9,7 @@ frappe.patches.v7_2.remove_in_filter
frappe.patches.v11_0.drop_column_apply_user_permissions
execute:frappe.reload_doc('core', 'doctype', 'doctype', force=True) #2017-09-22
execute:frappe.reload_doc('core', 'doctype', 'docfield', force=True) #2018-02-20
execute:frappe.reload_doc('core', 'doctype', 'doctype_action', force=True) #2019-09-23
execute:frappe.reload_doc('core', 'doctype', 'custom_docperm')
execute:frappe.reload_doc('core', 'doctype', 'docperm') #2018-05-29
execute:frappe.reload_doc('core', 'doctype', 'comment')

19
frappe/public/js/frappe/form/form.js
View file

@ -111,6 +111,7 @@ frappe.ui.form.Form = class FrappeForm {
$("body").attr("data-sidebar", 1);
}
this.setup_file_drop();
this.setup_doctype_actions();
this.setup_done = true;
}
@ -319,6 +320,24 @@ frappe.ui.form.Form = class FrappeForm {
}
}
// sets up the refresh event for custom buttons
// added via configuration
setup_doctype_actions() {
if (this.meta.actions) {
for (let action of this.meta.actions) {
frappe.ui.form.on(this.doctype, 'refresh', () => {
if (!this.is_new()) {
this.add_custom_button(action.label, () => {
frappe.xcall(action.method, {doc: this.doc}).then(() => {
frappe.msgprint({message:__('Event Executed'), alert:true});
});
}, action.group);
}
});
}
}
}
switch_doc(docname) {
// record switch
if(this.docname != docname && (!this.meta.in_dialog || this.in_form) && !this.meta.istable) {

2
frappe/test_runner.py
View file

@ -71,7 +71,7 @@ def main(app=None, module=None, doctype=None, verbose=False, tests=(),
else:
ret = run_all_tests(app, verbose, profile, ui_tests, failfast=failfast)
frappe.db.commit()
if frappe.db: frappe.db.commit()
# workaround! since there is no separate test db
frappe.clear_cache()

68
frappe/tests/test_scheduler.py
View file

@ -2,11 +2,9 @@ from __future__ import unicode_literals
from unittest import TestCase
from dateutil.relativedelta import relativedelta
from frappe.utils.scheduler import (enqueue_applicable_events, restrict_scheduler_events_if_dormant,
get_enabled_scheduler_events)
from frappe import _dict
from frappe.core.doctype.scheduled_job_type.scheduled_job_type import sync_jobs
from frappe.utils.background_jobs import enqueue
from frappe.utils import now_datetime, today, add_days, add_to_date
from frappe.utils.scheduler import enqueue_events
import frappe
import time
@ -17,60 +15,19 @@ def test_timeout():
class TestScheduler(TestCase):
def setUp(self):
frappe.db.set_global('enabled_scheduler_events', "")
frappe.flags.ran_schedulers = []
def test_all_events(self):
last = now_datetime() - relativedelta(hours=2)
enqueue_applicable_events(frappe.local.site, now_datetime(), last)
self.assertTrue("all" in frappe.flags.ran_schedulers)
def test_enabled_events(self):
frappe.flags.enabled_events = ["hourly", "hourly_long", "daily", "daily_long",
"weekly", "weekly_long", "monthly", "monthly_long"]
# maintain last_event and next_event on the same day
last_event = now_datetime().replace(hour=0, minute=0, second=0, microsecond=0)
next_event = last_event + relativedelta(minutes=30)
enqueue_applicable_events(frappe.local.site, next_event, last_event)
self.assertFalse("cron" in frappe.flags.ran_schedulers)
# maintain last_event and next_event on the same day
last_event = now_datetime().replace(hour=0, minute=0, second=0, microsecond=0)
next_event = last_event + relativedelta(hours=2)
frappe.flags.ran_schedulers = []
enqueue_applicable_events(frappe.local.site, next_event, last_event)
self.assertTrue("all" in frappe.flags.ran_schedulers)
self.assertTrue("hourly" in frappe.flags.ran_schedulers)
frappe.flags.enabled_events = None
def test_enabled_events_day_change(self):
# use flags instead of globals as this test fails intermittently
# the root cause has not been identified but the culprit seems cache
# since cache is mutable, it maybe be changed by a parallel process
frappe.flags.enabled_events = ["daily", "daily_long", "weekly", "weekly_long",
"monthly", "monthly_long"]
# maintain last_event and next_event on different days
next_event = now_datetime().replace(hour=0, minute=0, second=0, microsecond=0)
last_event = next_event - relativedelta(hours=2)
frappe.flags.ran_schedulers = []
enqueue_applicable_events(frappe.local.site, next_event, last_event)
self.assertTrue("all" in frappe.flags.ran_schedulers)
self.assertFalse("hourly" in frappe.flags.ran_schedulers)
frappe.flags.enabled_events = None
if not frappe.get_all('Scheduled Job Type', limit=1):
sync_jobs()
def test_enqueue_jobs(self):
frappe.db.sql('update `tabScheduled Job Type` set last_execution = "2010-01-01 00:00:00"')
enqueue_events(site = frappe.local.site)
self.assertTrue('frappe.email.queue.clear_outbox', frappe.flags.enqueued_jobs)
self.assertTrue('frappe.utils.change_log.check_for_update', frappe.flags.enqueued_jobs)
self.assertTrue('frappe.email.doctype.auto_email_report.auto_email_report.send_monthly', frappe.flags.enqueued_jobs)
def test_job_timeout(self):
return
job = enqueue(test_timeout, timeout=10)
count = 5
while count > 0:
@ -80,6 +37,3 @@ class TestScheduler(TestCase):
break
self.assertTrue(job.is_failed)
def tearDown(self):
frappe.flags.ran_schedulers = []

6
frappe/utils/background_jobs.py
View file

@ -79,8 +79,6 @@ def run_doc_method(doctype, name, doc_method, **kwargs):
def execute_job(site, method, event, job_name, kwargs, user=None, is_async=True, retry=0):
'''Executes job in a worker, performs commit/rollback and logs if there is any error'''
from frappe.utils.scheduler import log
if is_async:
frappe.connect(site)
if os.environ.get('CI'):
@ -115,12 +113,12 @@ def execute_job(site, method, event, job_name, kwargs, user=None, is_async=True,
is_async=is_async, retry=retry+1)
else:
log(method_name, message=repr(locals()))
frappe.log_error(method_name)
raise
except:
frappe.db.rollback()
log(method_name, message=repr(locals()))
frappe.log_error(method_name)
raise
else:

249
frappe/utils/scheduler.py
View file

@ -10,38 +10,15 @@ Events:
from __future__ import unicode_literals, print_function
import frappe
import json
import frappe, os, time
import schedule
import time
import frappe.utils
import os
from frappe.utils import now_datetime, get_datetime
from frappe.utils import get_sites
from datetime import datetime
from frappe.utils.background_jobs import enqueue, get_jobs, queue_timeout
from frappe.utils.data import get_datetime, now_datetime
from frappe.core.doctype.user.user import STANDARD_USERS
from frappe.installer import update_site_config
from six import string_types
from croniter import croniter
from frappe.utils.background_jobs import get_jobs
DATETIME_FORMAT = '%Y-%m-%d %H:%M:%S'
cron_map = {
"yearly": "0 0 1 1 *",
"annual": "0 0 1 1 *",
"monthly": "0 0 1 * *",
"monthly_long": "0 0 1 * *",
"weekly": "0 0 * * 0",
"weekly_long": "0 0 * * 0",
"daily": "0 0 * * *",
"daily_long": "0 0 * * *",
"midnight": "0 0 * * *",
"hourly": "0 * * * *",
"hourly_long": "0 * * * *",
"all": "0/" + str((frappe.get_conf().scheduler_interval or 240) // 60) + " * * * *",
}
def start_scheduler():
'''Run enqueue_events_for_all_sites every 2 minutes (default).
Specify scheduler_interval in seconds in common_site_config.json'''
@ -60,17 +37,16 @@ def enqueue_events_for_all_sites():
return
with frappe.init_site():
jobs_per_site = get_jobs()
sites = get_sites()
for site in sites:
try:
enqueue_events_for_site(site=site, queued_jobs=jobs_per_site[site])
enqueue_events_for_site(site=site)
except:
# it should try to enqueue other sites
print(frappe.get_traceback())
def enqueue_events_for_site(site, queued_jobs):
def enqueue_events_for_site(site):
def log_and_raise():
frappe.logger(__name__).error('Exception in Enqueue Events for Site {0}'.format(site) +
'\n' + frappe.get_traceback())
@ -82,7 +58,7 @@ def enqueue_events_for_site(site, queued_jobs):
if is_scheduler_inactive():
return
enqueue_events(site=site, queued_jobs=queued_jobs)
enqueue_events(site=site)
frappe.logger(__name__).debug('Queued events for site {0}'.format(site))
except frappe.db.OperationalError as e:
@ -96,128 +72,13 @@ def enqueue_events_for_site(site, queued_jobs):
finally:
frappe.destroy()
def enqueue_events(site, queued_jobs):
nowtime = frappe.utils.now_datetime()
last = frappe.db.get_value('System Settings', 'System Settings', 'scheduler_last_event')
# set scheduler last event
frappe.db.set_value('System Settings', 'System Settings',
'scheduler_last_event', nowtime.strftime(DATETIME_FORMAT),
update_modified=False)
frappe.db.commit()
out = []
if last:
last = datetime.strptime(last, DATETIME_FORMAT)
out = enqueue_applicable_events(site, nowtime, last, queued_jobs)
return '\n'.join(out)
def enqueue_applicable_events(site, nowtime, last, queued_jobs=()):
nowtime_str = nowtime.strftime(DATETIME_FORMAT)
out = []
enabled_events = get_enabled_scheduler_events()
def trigger_if_enabled(site, event, last, queued_jobs):
trigger(site, event, last, queued_jobs)
_log(event)
def _log(event):
out.append("{time} - {event} - queued".format(time=nowtime_str, event=event))
for event in enabled_events:
trigger_if_enabled(site, event, last, queued_jobs)
if "all" not in enabled_events:
trigger_if_enabled(site, "all", last, queued_jobs)
return out
def trigger(site, event, last=None, queued_jobs=(), now=False):
"""Trigger method in hooks.scheduler_events."""
queue = 'long' if event.endswith('_long') else 'short'
timeout = queue_timeout[queue]
if not queued_jobs and not now:
queued_jobs = get_jobs(site=site, queue=queue)
if frappe.flags.in_test:
frappe.flags.ran_schedulers.append(event)
events_from_hooks = get_scheduler_events(event)
if not events_from_hooks:
return
events = events_from_hooks
if not now:
events = []
if event == "cron":
for e in events_from_hooks:
e = cron_map.get(e, e)
if croniter.is_valid(e):
if croniter(e, last).get_next(datetime) <= frappe.utils.now_datetime():
events.extend(events_from_hooks[e])
else:
frappe.log_error("Cron string " + e + " is not valid", "Error triggering cron job")
frappe.logger(__name__).error('Exception in Trigger Events for Site {0}, Cron String {1}'.format(site, e))
else:
if croniter(cron_map[event], last).get_next(datetime) <= frappe.utils.now_datetime():
events.extend(events_from_hooks)
for handler in events:
if not now:
if handler not in queued_jobs:
enqueue(handler, queue, timeout, event)
else:
scheduler_task(site=site, event=event, handler=handler, now=True)
def get_scheduler_events(event):
'''Get scheduler events from hooks and integrations'''
scheduler_events = frappe.cache().get_value('scheduler_events')
if not scheduler_events:
scheduler_events = frappe.get_hooks("scheduler_events")
frappe.cache().set_value('scheduler_events', scheduler_events)
return scheduler_events.get(event) or []
def log(method, message=None):
"""log error in patch_log"""
message = frappe.utils.cstr(message) + "\n" if message else ""
message += frappe.get_traceback()
if not (frappe.db and frappe.db._conn):
frappe.connect()
frappe.db.rollback()
frappe.db.begin()
d = frappe.new_doc("Error Log")
d.method = method
d.error = message
d.insert(ignore_permissions=True)
frappe.db.commit()
return message
def get_enabled_scheduler_events():
if 'enabled_events' in frappe.flags and frappe.flags.enabled_events:
return frappe.flags.enabled_events
enabled_events = frappe.db.get_global("enabled_scheduler_events")
if frappe.flags.in_test:
# TEMP for debug: this test fails randomly
print('found enabled_scheduler_events {0}'.format(enabled_events))
if enabled_events:
if isinstance(enabled_events, string_types):
enabled_events = json.loads(enabled_events)
return enabled_events
return ["all", "hourly", "hourly_long", "daily", "daily_long",
"weekly", "weekly_long", "monthly", "monthly_long", "cron"]
def enqueue_events(site):
frappe.flags.enqueued_jobs = []
queued_jobs = get_jobs(key='job_type').get(site) or []
for job_type in frappe.get_all('Scheduled Job Type', dict(stopped=0)):
if not job_type.method in queued_jobs:
# don't add it to queue if still pending
frappe.get_doc('Scheduled Job Type', job_type.name).enqueue()
def is_scheduler_inactive():
if frappe.local.conf.maintenance_mode:
@ -229,6 +90,9 @@ def is_scheduler_inactive():
if is_scheduler_disabled():
return True
if is_dormant():
return True
return False
def is_scheduler_disabled():
@ -246,90 +110,15 @@ def enable_scheduler():
def disable_scheduler():
toggle_scheduler(False)
def get_errors(from_date, to_date, limit):
errors = frappe.db.sql("""select modified, method, error from `tabError Log`
where date(modified) between %s and %s
and error not like '%%[Errno 110] Connection timed out%%'
order by modified limit %s""", (from_date, to_date, limit), as_dict=True)
return ["""<p>Time: {modified}</p><pre><code>Method: {method}\n{error}</code></pre>""".format(**e)
for e in errors]
def get_error_report(from_date=None, to_date=None, limit=10):
from frappe.utils import get_url, now_datetime, add_days
if not from_date:
from_date = add_days(now_datetime().date(), -1)
if not to_date:
to_date = add_days(now_datetime().date(), -1)
errors = get_errors(from_date, to_date, limit)
if errors:
return 1, """<h4>Error Logs (max {limit}):</h4>
<p>URL: <a href="{url}" target="_blank">{url}</a></p><hr>{errors}""".format(
limit=limit, url=get_url(), errors="<hr>".join(errors))
else:
return 0, "<p>No error logs</p>"
def scheduler_task(site, event, handler, now=False):
'''This is a wrapper function that runs a hooks.scheduler_events method'''
frappe.logger(__name__).info('running {handler} for {site} for event: {event}'.format(handler=handler, site=site, event=event))
try:
if not now:
frappe.connect(site=site)
frappe.flags.in_scheduler = True
frappe.get_attr(handler)()
except Exception:
frappe.db.rollback()
traceback = log(handler, "Method: {event}, Handler: {handler}".format(event=event, handler=handler))
frappe.logger(__name__).error(traceback)
raise
else:
frappe.db.commit()
frappe.logger(__name__).info('ran {handler} for {site} for event: {event}'.format(handler=handler, site=site, event=event))
def reset_enabled_scheduler_events(login_manager):
if login_manager.info.user_type == "System User":
try:
if frappe.db.get_global('enabled_scheduler_events'):
# clear restricted events, someone logged in!
frappe.db.set_global('enabled_scheduler_events', None)
except frappe.db.InternalError as e:
if frappe.db.is_timedout(e):
frappe.log_error(frappe.get_traceback(), "Error in reset_enabled_scheduler_events")
else:
raise
else:
is_dormant = frappe.conf.get('dormant')
if is_dormant:
update_site_config('dormant', 'None')
def restrict_scheduler_events_if_dormant():
if is_dormant():
restrict_scheduler_events()
update_site_config('dormant', True)
def restrict_scheduler_events(*args, **kwargs):
val = json.dumps(["hourly", "hourly_long", "daily", "daily_long", "weekly", "weekly_long", "monthly", "monthly_long", "cron"])
frappe.db.set_global('enabled_scheduler_events', val)
def is_dormant(since = 345600):
last_user_activity = get_last_active()
if not last_user_activity:
# no user has ever logged in, so not yet used
return False
last_active = get_datetime(last_user_activity)
# Get now without tz info
now = now_datetime().replace(tzinfo=None)
time_since_last_active = now - last_active
if time_since_last_active.total_seconds() > since: # 4 days
if now_datetime() - get_datetime(last_user_activity) > since: # 4 days
return True
return False
def get_last_active():