[refactor] scheduler log is now error log and as MyISAM (commit not required) (#2152)

* [refactor] scheduler log is now error log

* [minor] new naming series

* [fix] engine

* [fix] saved by tests
This commit is contained in:
Rushabh Mehta 2016-10-06 17:17:55 +05:30 committed by GitHub
parent c3bb5930f7
commit 9d7ee83c5e
26 changed files with 149 additions and 78 deletions

View file

@ -1249,8 +1249,8 @@ def logger(module=None, with_more_info=True):
return get_logger(module or 'default', with_more_info=with_more_info)
def log_error(message, title=None):
'''Log error to Scheduler Log'''
get_doc(dict(doctype='Scheduler Log', error=str(message),
'''Log error to Error Log'''
get_doc(dict(doctype='Error Log', error=str(message),
method=title)).insert(ignore_permissions=True)
def get_desk_link(doctype, name):

View file

@ -43,7 +43,7 @@ def get_data():
"items": [
{
"type": "doctype",
"name": "Scheduler Log",
"name": "Error Log",
"description": _("Errors in Background Events"),
},
{

View file

@ -75,7 +75,7 @@ def get_data():
},
{
"type": "doctype",
"name": "Scheduler Log",
"name": "Error Log",
"description": _("Log of error on automated events (scheduler).")
},
{

View file

@ -11,11 +11,13 @@
"doctype": "DocType",
"document_type": "Document",
"editable_grid": 0,
"engine": "InnoDB",
"fields": [
{
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "sb0",
"fieldtype": "Section Break",
"hidden": 0,
@ -41,6 +43,7 @@
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "module",
"fieldtype": "Link",
"hidden": 0,
@ -68,6 +71,7 @@
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"description": "Child Tables are shown as a Grid in other DocTypes.",
"fieldname": "istable",
"fieldtype": "Check",
@ -95,6 +99,7 @@
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"default": "1",
"depends_on": "istable",
"fieldname": "editable_grid",
@ -122,6 +127,7 @@
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"description": "Single Types have only one record no tables associated. Values are stored in tabSingles",
"fieldname": "issingle",
"fieldtype": "Check",
@ -149,6 +155,7 @@
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "cb01",
"fieldtype": "Column Break",
"hidden": 0,
@ -172,6 +179,7 @@
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "document_type",
"fieldtype": "Select",
"hidden": 0,
@ -199,6 +207,35 @@
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"default": "InnoDB",
"fieldname": "engine",
"fieldtype": "Select",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_list_view": 0,
"label": "Database Engine",
"length": 0,
"no_copy": 0,
"options": "InnoDB\nMyISAM",
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"unique": 0
},
{
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "icon",
"fieldtype": "Data",
"hidden": 0,
@ -223,6 +260,7 @@
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "custom",
"fieldtype": "Check",
"hidden": 0,
@ -247,6 +285,7 @@
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "beta",
"fieldtype": "Check",
"hidden": 0,
@ -272,6 +311,7 @@
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"default": "0",
"depends_on": "eval: doc.image_field",
"fieldname": "image_view",
@ -299,6 +339,7 @@
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "app",
"fieldtype": "Data",
"hidden": 1,
@ -323,6 +364,7 @@
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "fields_section_break",
"fieldtype": "Section Break",
"hidden": 0,
@ -348,6 +390,7 @@
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "fields",
"fieldtype": "Table",
"hidden": 0,
@ -375,6 +418,7 @@
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "sb1",
"fieldtype": "Section Break",
"hidden": 0,
@ -399,6 +443,7 @@
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"description": "Naming Options:\n<ol><li><b>field:[fieldname]</b> - By Field</li><li><b>naming_series:</b> - By Naming Series (field called naming_series must be present</li><li><b>Prompt</b> - Prompt user for a name</li><li><b>[series]</b> - Series by prefix (separated by a dot); for example PRE.#####</li></ol>",
"fieldname": "autoname",
"fieldtype": "Data",
@ -426,6 +471,7 @@
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "name_case",
"fieldtype": "Select",
"hidden": 0,
@ -453,6 +499,7 @@
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "description",
"fieldtype": "Small Text",
"hidden": 0,
@ -479,6 +526,7 @@
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"depends_on": "",
"fieldname": "column_break_15",
"fieldtype": "Column Break",
@ -503,6 +551,7 @@
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"depends_on": "eval:!doc.istable",
"description": "Show this field as title",
"fieldname": "title_field",
@ -529,6 +578,7 @@
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"depends_on": "eval:!doc.istable",
"fieldname": "search_fields",
"fieldtype": "Data",
@ -556,6 +606,7 @@
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"description": "Must be of type \"Attach Image\"",
"fieldname": "image_field",
"fieldtype": "Data",
@ -582,6 +633,7 @@
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"default": "modified",
"depends_on": "eval:!doc.istable",
"description": "",
@ -609,6 +661,7 @@
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"default": "DESC",
"depends_on": "eval:!doc.istable",
"fieldname": "sort_order",
@ -636,6 +689,7 @@
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"depends_on": "eval:!doc.istable",
"description": "Comments and Communications will be associated with this linked document",
"fieldname": "timeline_field",
@ -663,6 +717,7 @@
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"depends_on": "eval:!doc.istable",
"fieldname": "sb2",
"fieldtype": "Section Break",
@ -688,6 +743,7 @@
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"depends_on": "",
"fieldname": "permissions",
"fieldtype": "Table",
@ -716,6 +772,7 @@
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"depends_on": "eval:!doc.istable",
"fieldname": "sb3",
"fieldtype": "Section Break",
@ -740,6 +797,7 @@
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "cb30",
"fieldtype": "Column Break",
"hidden": 0,
@ -764,6 +822,7 @@
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "in_create",
"fieldtype": "Check",
"hidden": 0,
@ -790,6 +849,7 @@
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "read_only",
"fieldtype": "Check",
"hidden": 0,
@ -816,6 +876,7 @@
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "is_submittable",
"fieldtype": "Check",
"hidden": 0,
@ -840,6 +901,7 @@
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"description": "Allow Import via Data Import Tool",
"fieldname": "allow_import",
"fieldtype": "Check",
@ -865,6 +927,7 @@
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "allow_rename",
"fieldtype": "Check",
"hidden": 0,
@ -891,6 +954,7 @@
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "in_dialog",
"fieldtype": "Check",
"hidden": 0,
@ -917,6 +981,7 @@
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "read_only_onload",
"fieldtype": "Check",
"hidden": 0,
@ -943,6 +1008,7 @@
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "max_attachments",
"fieldtype": "Int",
"hidden": 0,
@ -969,6 +1035,7 @@
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "cb31",
"fieldtype": "Column Break",
"hidden": 0,
@ -993,6 +1060,7 @@
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "hide_heading",
"fieldtype": "Check",
"hidden": 0,
@ -1019,6 +1087,7 @@
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "hide_toolbar",
"fieldtype": "Check",
"hidden": 0,
@ -1045,6 +1114,7 @@
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "allow_copy",
"fieldtype": "Check",
"hidden": 0,
@ -1071,6 +1141,7 @@
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "track_seen",
"fieldtype": "Check",
"hidden": 0,
@ -1096,6 +1167,7 @@
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"default": "1",
"fieldname": "quick_entry",
"fieldtype": "Check",
@ -1122,6 +1194,7 @@
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "default_print_format",
"fieldtype": "Data",
"hidden": 0,
@ -1154,7 +1227,7 @@
"issingle": 0,
"istable": 0,
"max_attachments": 0,
"modified": "2016-07-25 12:18:27.724194",
"modified": "2016-10-06 03:09:39.138857",
"modified_by": "Administrator",
"module": "Core",
"name": "DocType",
@ -1170,6 +1243,7 @@
"export": 0,
"if_owner": 0,
"import": 0,
"is_custom": 0,
"permlevel": 0,
"print": 1,
"read": 1,
@ -1190,6 +1264,7 @@
"export": 0,
"if_owner": 0,
"import": 0,
"is_custom": 0,
"permlevel": 0,
"print": 1,
"read": 1,

View file

@ -0,0 +1,8 @@
// Copyright (c) 2016, Frappe Technologies and contributors
// For license information, please see license.txt
frappe.ui.form.on('Error Log', {
refresh: function(frm) {
}
});

View file

@ -2,7 +2,7 @@
"allow_copy": 0,
"allow_import": 0,
"allow_rename": 0,
"autoname": "SCHLOG.#####",
"autoname": "Error-.#####",
"beta": 0,
"creation": "2013-01-16 13:09:40",
"custom": 0,
@ -10,11 +10,14 @@
"docstatus": 0,
"doctype": "DocType",
"document_type": "System",
"editable_grid": 0,
"engine": "MyISAM",
"fields": [
{
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"default": "0",
"fieldname": "seen",
"fieldtype": "Check",
@ -41,6 +44,7 @@
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "method",
"fieldtype": "Data",
"hidden": 0,
@ -48,7 +52,7 @@
"ignore_xss_filter": 0,
"in_filter": 0,
"in_list_view": 1,
"label": "Method",
"label": "Title",
"length": 0,
"no_copy": 0,
"permlevel": 0,
@ -65,6 +69,7 @@
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "error",
"fieldtype": "Code",
"hidden": 0,
@ -97,10 +102,10 @@
"issingle": 0,
"istable": 0,
"max_attachments": 0,
"modified": "2016-07-03 14:24:13.581374",
"modified": "2016-10-06 03:29:47.810715",
"modified_by": "Administrator",
"module": "Core",
"name": "Scheduler Log",
"name": "Error Log",
"owner": "Administrator",
"permissions": [
{
@ -113,6 +118,7 @@
"export": 0,
"if_owner": 0,
"import": 0,
"is_custom": 0,
"permlevel": 0,
"print": 1,
"read": 1,

View file

@ -1,19 +1,17 @@
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt
# -*- coding: utf-8 -*-
# Copyright (c) 2015, Frappe Technologies and contributors
# For license information, please see license.txt
from __future__ import unicode_literals
import frappe
from frappe.model.document import Document
class SchedulerLog(Document):
class ErrorLog(Document):
def onload(self):
if not self.seen:
self.seen = 1
self.save()
def set_old_logs_as_seen():
frappe.db.sql("""update `tabScheduler Log` set seen=1
frappe.db.sql("""update `tabError Log` set seen=1
where seen=0 and datediff(curdate(), creation) > 7""")

View file

@ -1,4 +1,4 @@
frappe.listview_settings['Scheduler Log'] = {
frappe.listview_settings['Error Log'] = {
add_fields: ["seen"],
get_indicator: function(doc) {
if(cint(doc.seen)) {

View file

@ -0,0 +1,12 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2015, Frappe Technologies and Contributors
# See license.txt
from __future__ import unicode_literals
import frappe
import unittest
# test_records = frappe.get_test_records('Error Log')
class TestErrorLog(unittest.TestCase):
pass

View file

@ -1 +0,0 @@
Log exceptions (errors) raised when executing the scheduler.

View file

@ -1,3 +0,0 @@
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt

View file

@ -1,8 +0,0 @@
// Copyright (c) 2016, Frappe Technologies Pvt. Ltd. and contributors
// For license information, please see license.txt
frappe.ui.form.on('Scheduler Log', {
refresh: function(frm) {
}
});

View file

@ -1,12 +0,0 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# See license.txt
from __future__ import unicode_literals
import frappe
import unittest
# test_records = frappe.get_test_records('Scheduler Log')
class TestSchedulerLog(unittest.TestCase):
pass

View file

@ -7,7 +7,7 @@ import frappe
def get_notification_config():
return {
"for_doctype": {
"Scheduler Log": {"seen": 0},
"Error Log": {"seen": 0},
"Communication": {"status": "Open", "communication_type": "Communication"},
"ToDo": "frappe.core.notifications.get_things_todo",
"Event": "frappe.core.notifications.get_todays_events",

View file

@ -146,6 +146,7 @@ CREATE TABLE `tabDocType` (
`tag_fields` varchar(255) DEFAULT NULL,
`subject` varchar(255) DEFAULT NULL,
`_last_update` varchar(32) DEFAULT NULL,
`engine` varchar(20) DEFAULT 'InnoDB',
`default_print_format` varchar(255) DEFAULT NULL,
`is_submittable` int(1) NOT NULL DEFAULT 0,
`_user_tags` varchar(255) DEFAULT NULL,

View file

@ -340,7 +340,7 @@ def send_one(email, smtpserver=None, auto_commit=True, now=False):
raise e
else:
# log to scheduler log
# log to Error Log
log('frappe.email.queue.flush', unicode(e))
def clear_outbox():

View file

@ -76,7 +76,7 @@ class EmailServer:
return True
except _socket.error:
# log performs rollback and logs error in scheduler log
# log performs rollback and logs error in Error Log
log("receive.connect_pop")
# Invalid mail server -- due to refusing connection
@ -185,7 +185,7 @@ class EmailServer:
raise LoginLimitExceeded, e
else:
# log performs rollback and logs error in scheduler log
# log performs rollback and logs error in Error Log
log("receive.get_messages", self.make_error_msg(msg_num, incoming_mail))
self.errors = True
frappe.db.rollback()

View file

@ -135,7 +135,7 @@ scheduler_events = {
"daily": [
"frappe.email.queue.clear_outbox",
"frappe.desk.notifications.clear_notifications",
"frappe.core.doctype.scheduler_log.scheduler_log.set_old_logs_as_seen",
"frappe.core.doctype.error_log.error_log.set_old_logs_as_seen",
"frappe.desk.doctype.event.event.send_event_digest",
"frappe.sessions.clear_expired_sessions",
"frappe.email.doctype.email_alert.email_alert.trigger_daily_alerts",

View file

@ -146,23 +146,16 @@ class Controller(IntegrationController):
redirect_message = data.get('notes', {}).get('redirect_message') or None
try:
for tries in xrange(3):
resp = self.get_request("https://api.razorpay.com/v1/payments/{0}"
.format(self.data.razorpay_payment_id), auth=(settings.api_key,
settings.api_secret))
resp = self.get_request("https://api.razorpay.com/v1/payments/{0}"
.format(self.data.razorpay_payment_id), auth=(settings.api_key,
settings.api_secret))
if resp.get('status') == 'created':
# just created wait for a few seconds and try again
import time
time.sleep(0.5)
if resp.get("status") == "authorized":
self.integration_request.db_set('status', 'Authorized', update_modified=False)
self.flags.status_changed_to = "Authorized"
elif resp.get("status") == "authorized":
self.integration_request.db_set('status', 'Authorized', update_modified=False)
self.flags.status_changed_to = "Authorized"
break
if self.flags.status_changed_to != 'Authorized':
frappe.log_error(str(resp), 'Razorpay Payment not authorized')
else:
frappe.log_error(str(resp), 'Razorpay Payment not authorized')
except:
frappe.log_error(frappe.get_traceback())

View file

@ -174,10 +174,11 @@ class DbTable:
parenttype varchar({varchar_len}),
idx int(8) not null default '0',
%sindex parent(parent))
ENGINE=InnoDB
ENGINE={engine}
ROW_FORMAT=COMPRESSED
CHARACTER SET=utf8mb4
COLLATE=utf8mb4_unicode_ci""".format(varchar_len=varchar_len) % (self.name, add_text))
COLLATE=utf8mb4_unicode_ci""".format(varchar_len=varchar_len,
engine=self.meta.engine or 'InnoDB') % (self.name, add_text))
def get_column_definitions(self):
column_list = [] + default_columns

View file

@ -1,4 +1,5 @@
execute:frappe.db.sql("""update `tabPatch Log` set patch=replace(patch, '.4_0.', '.v4_0.')""") #2014-05-12
frappe.patches.v7_1.rename_scheduler_log_to_error_log
frappe.patches.v5_0.convert_to_barracuda_and_utf8mb4
frappe.patches.v7_0.update_auth
frappe.patches.v6_1.rename_file_data
@ -140,4 +141,4 @@ frappe.patches.v7_1.rename_chinese_language_codes
execute:frappe.core.doctype.language.language.update_language_names()
execute:frappe.db.set_value("Print Settings", "Print Settings", "add_draft_heading", 1)
frappe.patches.v7_0.cleanup_list_settings
execute:frappe.db.set_default('language', '')
execute:frappe.db.set_default('language', '')

View file

@ -1,7 +1,7 @@
import frappe
def execute():
frappe.reload_doctype("Scheduler Log")
frappe.reload_doctype("Error Log")
from frappe.core.doctype.scheduler_log.scheduler_log import set_old_logs_as_seen
from frappe.core.doctype.error_log.error_log import set_old_logs_as_seen
set_old_logs_as_seen()

View file

@ -0,0 +1,7 @@
import frappe
def execute():
if not 'tabError Log' in frappe.db.get_tables():
frappe.rename_doc('DocType', 'Scheduler Log', 'Error Log')
frappe.db.commit()
frappe.db.sql('alter table `tabError Log` engine=MyISAM')

View file

@ -43,10 +43,4 @@ def make_payment(razorpay_payment_id, options, reference_doctype, reference_docn
"reference_doctype": reference_doctype
})
# let razorpay commit its payments!
import time
time.sleep(0.5)
data = Controller().create_request(data)
frappe.db.commit()
return data
return Controller().create_request(data)

View file

@ -14,10 +14,9 @@ import frappe
import json
import schedule
import time
import os
import MySQLdb
import frappe.utils
from frappe.utils import get_sites, get_site_path, touch_file
from frappe.utils import get_sites
from datetime import datetime
from background_jobs import enqueue, get_jobs, queue_timeout
from frappe.limits import has_expired
@ -165,7 +164,7 @@ def log(method, message=None):
frappe.db.rollback()
frappe.db.begin()
d = frappe.new_doc("Scheduler Log")
d = frappe.new_doc("Error Log")
d.method = method
d.error = message
d.insert(ignore_permissions=True)
@ -205,7 +204,7 @@ def disable_scheduler():
toggle_scheduler(False)
def get_errors(from_date, to_date, limit):
errors = frappe.db.sql("""select modified, method, error from `tabScheduler Log`
errors = frappe.db.sql("""select modified, method, error from `tabError Log`
where date(modified) between %s and %s
and error not like '%%[Errno 110] Connection timed out%%'
order by modified limit %s""", (from_date, to_date, limit), as_dict=True)
@ -223,11 +222,11 @@ def get_error_report(from_date=None, to_date=None, limit=10):
errors = get_errors(from_date, to_date, limit)
if errors:
return 1, """<h4>Scheduler Failed Events (max {limit}):</h4>
return 1, """<h4>Error Logs (max {limit}):</h4>
<p>URL: <a href="{url}" target="_blank">{url}</a></p><hr>{errors}""".format(
limit=limit, url=get_url(), errors="<hr>".join(errors))
else:
return 0, "<p>Scheduler didn't encounter any problems.</p>"
return 0, "<p>No error logs</p>"
def scheduler_task(site, event, handler, now=False):
'''This is a wrapper function that runs a hooks.scheduler_events method'''