vassili/seitime-frappe
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

feat: log settings with "interface"

Browse source 
We have hardcoded "Log settings" to only apply on 3 doctypes, there are
few more logging doctypes in core which are not cleared right now, on
top of that it's not easy for user to configure all logging behaviour
from one place.

This change adds a table on log settings where logging doctypes that
support the interface required by log settings can auto-register and
show up in settings.

Currently only supported configuration is "number of days" to keep.
This commit is contained in:
Ankush Menat 2022-06-11 12:26:09 +05:30
parent ef1200d94b
commit ea416f9d6b
15 changed files with 255 additions and 129 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
frappe/__init__.py
View file

@ -17,7 +17,7 @@ import json
import os
import re
import warnings
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union
import click
from werkzeug.local import Local, release_local
@ -1551,7 +1551,15 @@ def call(fn, *args, **kwargs):
return fn(*args, **newargs)
def get_newargs(fn, kwargs):
def get_newargs(fn: Callable, kwargs: Dict[str, Any]) -> Dict[str, Any]:
"""Remove any kwargs that are not supported by the function.
Example:
>>> def fn(a=1, b=2): pass
>>> get_newargs(fn, {"a": 2, "c": 1})
{"a": 2}
"""
# if function has any **kwargs parameter that capture arbitrary keyword arguments
# Ref: https://docs.python.org/3/library/inspect.html#inspect.Parameter.kind

16
frappe/core/doctype/activity_log/activity_log.py
View file

@ -25,6 +25,13 @@ class ActivityLog(Document):
if self.reference_doctype and self.reference_name:
self.status = "Linked"
@staticmethod
def clear_old_logs(days=None):
if not days:
days = 90
doctype = DocType("Activity Log")
frappe.db.delete(doctype, filters=(doctype.modified < (Now() - Interval(days=days))))
def on_doctype_update():
"""Add indexes in `tabActivity Log`"""
@ -43,12 +50,3 @@ def add_authentication_log(subject, user, operation="Login", status="Success"):
"operation": operation,
}
).insert(ignore_permissions=True, ignore_links=True)
def clear_activity_logs(days=None):
"""clear 90 day old authentication logs or configured in log settings"""
if not days:
days = 90
doctype = DocType("Activity Log")
frappe.db.delete(doctype, filters=(doctype.creation < (Now() - Interval(days=days))))

7
frappe/core/doctype/error_log/error_log.py
View file

@ -4,6 +4,8 @@
import frappe
from frappe.model.document import Document
from frappe.query_builder import Interval
from frappe.query_builder.functions import Now
class ErrorLog(Document):
@ -12,6 +14,11 @@ class ErrorLog(Document):
self.db_set("seen", 1, update_modified=0)
frappe.db.commit()
@staticmethod
def clear_old_logs(days=30):
table = frappe.qb.DocType("Error Log")
frappe.db.delete(table, filters=(table.creation < (Now() - Interval(days=days))))
@frappe.whitelist()
def clear_error_logs():

16
frappe/core/doctype/log_settings/log_settings.js
View file

@ -1,8 +1,16 @@
// Copyright (c) 2020, Frappe Technologies and contributors
// For license information, please see license.txt
frappe.ui.form.on('Log Settings', {
// refresh: function(frm) {
// }
frappe.ui.form.on("Log Settings", {
refresh: (frm) => {
frm.set_query("ref_doctype", "logs_to_clear", () => {
const added_doctypes = frm.doc.logs_to_clear.map((r) => r.ref_doctype);
return {
query: "frappe.core.doctype.log_settings.log_settings.get_log_doctypes",
filters: [
["name", "not in", added_doctypes],
],
};
});
},
});

56
frappe/core/doctype/log_settings/log_settings.json
View file

@ -5,61 +5,20 @@
"editable_grid": 1,
"engine": "InnoDB",
"field_order": [
"error_log_notification_section",
"users_to_notify",
"log_cleanup_section",
"clear_error_log_after",
"clear_activity_log_after",
"column_break_4",
"clear_email_queue_after"
"logs_to_clear"
],
"fields": [
{
"fieldname": "log_cleanup_section",
"fieldtype": "Section Break",
"label": "Log Cleanup"
},
{
"fieldname": "column_break_4",
"fieldtype": "Column Break"
},
{
"fieldname": "error_log_notification_section",
"fieldtype": "Section Break",
"label": "Error Log Notification"
},
{
"fieldname": "users_to_notify",
"fieldtype": "Table MultiSelect",
"label": "Users To Notify",
"options": "Log Setting User"
},
{
"default": "90",
"description": "In Days",
"fieldname": "clear_error_log_after",
"fieldtype": "Int",
"label": "Clear Error log After"
},
{
"default": "90",
"description": "In Days",
"fieldname": "clear_activity_log_after",
"fieldtype": "Int",
"label": "Clear Activity Log After"
},
{
"default": "30",
"description": "In Days",
"fieldname": "clear_email_queue_after",
"fieldtype": "Int",
"label": "Clear Email Queue After"
"fieldname": "logs_to_clear",
"fieldtype": "Table",
"label": "Logs to Clear",
"options": "Logs To Clear"
}
],
"index_web_pages_for_search": 1,
"issingle": 1,
"links": [],
"modified": "2020-10-13 12:18:48.649038",
"modified": "2022-06-11 02:17:30.803721",
"modified_by": "Administrator",
"module": "Core",
"name": "Log Settings",
@ -79,5 +38,6 @@
"quick_entry": 1,
"sort_field": "modified",
"sort_order": "DESC",
"states": [],
"track_changes": 1
}
}

122
frappe/core/doctype/log_settings/log_settings.py
View file

@ -2,49 +2,88 @@
# Copyright (c) 2020, Frappe Technologies and contributors
# License: MIT. See LICENSE
from typing import Protocol, runtime_checkable
import frappe
from frappe import _
from frappe.model.base_document import get_controller
from frappe.model.document import Document
from frappe.query_builder import DocType, Interval
from frappe.query_builder.functions import Now
from frappe.utils import cint
from frappe.utils.caching import site_cache
@runtime_checkable
class LogType(Protocol):
"""Interface requirement for doctypes that can be cleared using log settings."""
@staticmethod
def clear_old_logs(days: int) -> None:
...
@site_cache
def _supports_log_clearing(doctype: str) -> bool:
try:
controller = get_controller(doctype)
return issubclass(controller, LogType)
except Exception:
return False
class LogSettings(Document):
def clear_logs(self, commit=False):
self.clear_email_queue()
if commit:
# Since since deleting many logs can take significant amount of time, commit is required to relase locks.
# Error log table doesn't require commit - myisam
# activity logs are deleted last so background job finishes and commits.
def validate(self):
self.validate_supported_doctypes()
self.validate_duplicates()
def validate_supported_doctypes(self):
for entry in self.logs_to_clear:
if _supports_log_clearing(entry.ref_doctype):
continue
msg = _("{} does not support automated log clearing.").format(frappe.bold(entry.ref_doctype))
if frappe.conf.developer_mode:
msg += "<br>" + _("Implement `clear_old_logs` method to enable auto error clearing.")
frappe.throw(msg, title=_("DocType not supported by Log Settings."))
def validate_duplicates(self):
seen = set()
for entry in self.logs_to_clear:
if entry.ref_doctype in seen:
frappe.throw(
_("{} appears more than once in configured log doctypes.").format(entry.ref_doctype)
)
seen.add(entry.ref_doctype)
def clear_logs(self):
"""
Log settings can clear any log type that's registered to it and provides a method to delete old logs.
Check `LogDoctype` above for interface that doctypes need to implement.
"""
for entry in self.logs_to_clear:
controller: LogType = get_controller(entry.ref_doctype)
func = controller.clear_old_logs
# Only pass what the method can handle, this is considering any
# future addition that might happen to the required interface.
kwargs = frappe.get_newargs(func, {"days": entry.days})
func(**kwargs)
frappe.db.commit()
self.clear_error_logs()
self.clear_activity_logs()
def clear_error_logs(self):
table = DocType("Error Log")
frappe.db.delete(
table, filters=(table.creation < (Now() - Interval(days=self.clear_error_log_after)))
)
def clear_activity_logs(self):
from frappe.core.doctype.activity_log.activity_log import clear_activity_logs
clear_activity_logs(days=self.clear_activity_log_after)
def clear_email_queue(self):
from frappe.email.queue import clear_outbox
clear_outbox(days=self.clear_email_queue_after)
def register_doctype(self, doctype: str, days=30):
if doctype not in {d.ref_doctype for d in self.logs_to_clear}:
self.append("logs_to_clear", {"ref_doctype": doctype, "days": cint(days)})
def run_log_clean_up():
doc = frappe.get_doc("Log Settings")
doc.clear_logs(commit=True)
doc.clear_logs()
@frappe.whitelist()
def has_unseen_error_log(user):
def _get_response(show_alert=True):
def has_unseen_error_log():
if frappe.get_all("Error Log", filters={"seen": 0}, limit=1):
return {
"show_alert": True,
"message": _("You have unseen {0}").format(
@ -52,13 +91,22 @@ def has_unseen_error_log(user):
),
}
if frappe.get_all("Error Log", filters={"seen": 0}, limit=1):
log_settings = frappe.get_cached_doc("Log Settings")
if log_settings.users_to_notify:
if user in [u.user for u in log_settings.users_to_notify]:
return _get_response()
else:
return _get_response(show_alert=False)
else:
return _get_response()
@frappe.whitelist()
@frappe.validate_and_sanitize_search_inputs
def get_log_doctypes(doctype, txt, searchfield, start, page_len, filters):
filters = filters or {}
filters.extend(
[
["istable", "=", 0],
["issingle", "=", 0],
["name", "like", f"%%{txt}%%"],
]
)
doctypes = frappe.get_list("DocType", filters=filters, pluck="name")
supported_doctypes = [(d,) for d in doctypes if _supports_log_clearing(d)]
return supported_doctypes[start:page_len]

19
frappe/core/doctype/log_settings/test_log_settings.py
View file

@ -4,7 +4,7 @@
from datetime import datetime
import frappe
from frappe.core.doctype.log_settings.log_settings import run_log_clean_up
from frappe.core.doctype.log_settings.log_settings import _supports_log_clearing, run_log_clean_up
from frappe.tests.utils import FrappeTestCase
from frappe.utils import add_to_date, now_datetime
@ -56,6 +56,23 @@ class TestLogSettings(FrappeTestCase):
self.assertEqual(error_log_count, 0)
self.assertEqual(email_queue_count, 0)
def test_logtype_identification(self):
supported_types = [
"Error Log",
"Activity Log",
"Email Queue",
"Route History",
"Error Snapshot",
"Scheduled Job Log",
]
for lt in supported_types:
self.assertTrue(_supports_log_clearing(lt), f"{lt} should be recognized as log type")
unsupported_types = ["DocType", "User", "Non Existing dt"]
for dt in unsupported_types:
self.assertFalse(_supports_log_clearing(dt), f"{dt} shouldn't be recognized as log type")
def setup_test_logs(past: datetime) -> None:
activity_log = frappe.get_doc(

0
frappe/core/doctype/logs_to_clear/__init__.py Normal file
View file

42
frappe/core/doctype/logs_to_clear/logs_to_clear.json Normal file
View file

@ -0,0 +1,42 @@
{
"actions": [],
"autoname": "autoincrement",
"creation": "2022-06-11 02:02:39.472511",
"doctype": "DocType",
"engine": "InnoDB",
"field_order": [
"ref_doctype",
"days"
],
"fields": [
{
"fieldname": "ref_doctype",
"fieldtype": "Link",
"in_list_view": 1,
"label": "Log DocType",
"options": "DocType",
"reqd": 1
},
{
"fieldname": "days",
"fieldtype": "Int",
"in_list_view": 1,
"label": "Clear Logs After (days)",
"non_negative": 1,
"reqd": 1
}
],
"index_web_pages_for_search": 1,
"istable": 1,
"links": [],
"modified": "2022-06-11 03:20:47.721188",
"modified_by": "Administrator",
"module": "Core",
"name": "Logs To Clear",
"naming_rule": "Autoincrement",
"owner": "Administrator",
"permissions": [],
"sort_field": "modified",
"sort_order": "DESC",
"states": []
}

9
frappe/core/doctype/logs_to_clear/logs_to_clear.py Normal file
View file

@ -0,0 +1,9 @@
# Copyright (c) 2022, Frappe Technologies and contributors
# For license information, please see license.txt
# import frappe
from frappe.model.document import Document
class LogsToClear(Document):
pass

28
frappe/email/doctype/email_queue/email_queue.py
View file

@ -18,7 +18,8 @@ from frappe.email.doctype.email_account.email_account import EmailAccount
from frappe.email.email_body import add_attachment, get_email, get_formatted_html
from frappe.email.queue import get_unsubcribed_url, get_unsubscribe_message
from frappe.model.document import Document
from frappe.query_builder.utils import DocType
from frappe.query_builder import DocType, Interval
from frappe.query_builder.functions import Now
from frappe.utils import (
add_days,
cint,
@ -144,6 +145,31 @@ class EmailQueue(Document):
if ctx.email_account_doc.append_emails_to_sent_folder and ctx.sent_to:
ctx.email_account_doc.append_email_to_sent_folder(message)
@staticmethod
def clear_old_logs(days=30):
"""Remove low priority older than 31 days in Outbox or configured in Log Settings.
Note: Used separate query to avoid deadlock
"""
days = days or 31
email_queue = frappe.qb.DocType("Email Queue")
email_recipient = frappe.qb.DocType("Email Queue Recipient")
# Delete queue table
(
frappe.qb.from_(email_queue)
.delete()
.where((email_queue.modified < (Now() - Interval(days=days))))
).run()
# delete child tables, note that this has potential to leave some orphan
# child table behind if modified time was later than parent doc (rare).
# But it's safe since child table doesn't contain links.
(
frappe.qb.from_(email_recipient)
.delete()
.where((email_recipient.modified < (Now() - Interval(days=days))))
).run()
@task(queue="short")
def send_mail(email_queue_name, is_background_task=False):

5
frappe/email/doctype/email_queue/test_email_queue.py
View file

@ -3,12 +3,13 @@
# License: MIT. See LICENSE
import frappe
from frappe.email.queue import clear_outbox
from frappe.tests.utils import FrappeTestCase
class TestEmailQueue(FrappeTestCase):
def test_email_queue_deletion_based_on_modified_date(self):
from frappe.email.doctype.email_queue.email_queue import EmailQueue
old_record = frappe.get_doc(
{
"doctype": "Email Queue",
@ -32,7 +33,7 @@ class TestEmailQueue(FrappeTestCase):
new_record = frappe.copy_doc(old_record)
new_record.insert()
clear_outbox()
EmailQueue.clear_old_logs()
self.assertFalse(frappe.db.exists("Email Queue", old_record.name))
self.assertFalse(frappe.db.exists("Email Queue Recipient", {"parent": old_record.name}))

25
frappe/email/queue.py
View file

@ -190,31 +190,6 @@ def get_queue():
)
def clear_outbox(days: int = None) -> None:
"""Remove low priority older than 31 days in Outbox or configured in Log Settings.
Note: Used separate query to avoid deadlock
"""
days = days or 31
email_queue = frappe.qb.DocType("Email Queue")
email_recipient = frappe.qb.DocType("Email Queue Recipient")
# Delete queue table
(
frappe.qb.from_(email_queue)
.delete()
.where((email_queue.modified < (Now() - Interval(days=days))))
).run()
# delete child tables, note that this has potential to leave some orphan
# child table behind if modified time was later than parent doc (rare).
# But it's safe since child table doesn't contain links.
(
frappe.qb.from_(email_recipient)
.delete()
.where((email_recipient.modified < (Now() - Interval(days=days))))
).run()
def set_expiry_for_email_queue():
"""Mark emails as expire that has not sent for 7 days.
Called daily via scheduler.

1
frappe/patches.txt
View file

@ -191,6 +191,7 @@ frappe.patches.v14_0.remove_post_and_post_comment
frappe.patches.v14_0.reset_creation_datetime
frappe.patches.v14_0.remove_is_first_startup
frappe.patches.v14_0.reload_workspace_child_tables
frappe.patches.v14_0.log_settings_migration
[post_model_sync]
frappe.patches.v14_0.drop_data_import_legacy

26
frappe/patches/v14_0/log_settings_migration.py Normal file
View file

@ -0,0 +1,26 @@
import frappe
def execute():
logging_doctypes = {
"Error Log": get_current_setting("clear_error_log_after") or 30,
"Activity Log": get_current_setting("clear_activity_log_after") or 90,
"Email Queue": get_current_setting("clear_email_queue_after") or 30,
}
frappe.reload_doc("core", "doctype", "Logs To Clear")
frappe.reload_doc("core", "doctype", "Log Settings")
log_settings = frappe.get_doc("Log Settings")
for doctype, days in logging_doctypes.items():
log_settings.register_doctype(doctype, days)
log_settings.save()
def get_current_setting(fieldname):
try:
return frappe.db.get_single_value("Log Settings", fieldname)
except Exception:
pass