feat: clear-log-table to clear large log tables

frappe/commands/site.py

@@ -1088,6 +1088,81 @@ def build_search_index(context):
 		frappe.destroy()
 
 
+LOG_DOCTYPES = [
+	"Scheduled Job Log",
+	"Activity Log",
+	"Route History",
+	"Email Queue",
+	"Error Snapshot",
+	"Error Log",
+]
+
+
+@click.command("clear-log-table")
+@click.option("--doctype", default="text", type=click.Choice(LOG_DOCTYPES), help="Log DocType")
+@click.option("--days", type=int, help="Keep records for days")
+@click.option("--no-backup", is_flag=True, default=False, help="Do not backup the table")
+@pass_context
+def clear_log_table(context, doctype, days, no_backup):
+	"""If any logtype table grows too large then clearing it with DELETE query
+	is not feasible in reasonable time. This command copies recent data to new
+	table and replaces current table with new smaller table.
+
+
+	ref: https://mariadb.com/kb/en/big-deletes/#deleting-more-than-half-a-table
+	"""
+	from frappe.utils.backups import scheduled_backup
+
+	if not context.sites:
+		raise SiteNotSpecifiedError
+
+	if doctype not in LOG_DOCTYPES:
+		raise frappe.ValidationError(f"Unsupported logging DocType: {doctype}")
+
+	for site in context.sites:
+		frappe.init(site=site)
+		frappe.connect()
+
+		if frappe.db.db_type != "mariadb":
+			click.echo("Postgres database isn't supported by this command")
+			sys.exit(1)
+
+		if not no_backup:
+			scheduled_backup(
+				ignore_conf=False,
+				include_doctypes=doctype,
+				ignore_files=True,
+				force=True,
+			)
+			click.echo(f"Backed up {doctype}")
+
+		original = f"`tab{doctype}`"
+		temporary = f"`tab{doctype} temp_table`"
+		backup = f"`tab{doctype} backup_table`"
+
+		try:
+			frappe.db.sql(f"CREATE TABLE {temporary} LIKE {original}")
+
+			click.echo(f"Copying {doctype} records from last {days} days to temporary table.")
+			# Copy all recent data to new table
+			frappe.db.sql(
+				f"""INSERT INTO {temporary}
+					SELECT * FROM {original}
+					WHERE {original}.`modified` > NOW() - INTERVAL '{days}' DAY"""
+			)
+			frappe.db.sql(f"RENAME TABLE {original} TO {backup}, {temporary} TO {original}")
+		except Exception as e:
+			# Discard created tables
+			frappe.db.rollback()
+			frappe.db.sql(f"DROP TABLE IF EXISTS {temporary}")
+			click.echo(f"Log cleanup for {doctype} failed:\n{e}")
+			sys.exit(1)
+		else:
+			frappe.db.commit()
+			frappe.db.sql(f"DROP TABLE {backup}")
+			click.secho(f"Cleared {doctype} records older than {days} days", fg="green")
+
+
 @click.command("trim-database")
 @click.option("--dry-run", is_flag=True, default=False, help="Show what would be deleted")
 @click.option(
@@ -1260,4 +1335,5 @@ commands = [
 	partial_restore,
 	trim_tables,
 	trim_database,
+	clear_log_table,
 ]

frappe/tests/test_commands.py

@@ -27,6 +27,8 @@ import frappe.commands.site
 import frappe.commands.utils
 import frappe.recorder
 from frappe.installer import add_to_installed_apps, remove_app
+from frappe.query_builder.utils import db_type_is
+from frappe.tests.test_query_builder import run_only_if
 from frappe.utils import add_to_date, get_bench_path, get_bench_relative_path, now
 from frappe.utils.backups import fetch_latest_backups
 
@@ -518,6 +520,18 @@ class TestBackups(BaseTestCommands):
 		self.assertIsNotNone(after_backup["public"])
 		self.assertIsNotNone(after_backup["private"])
 
+	@run_only_if(db_type_is.MARIADB)
+	def test_clear_log_table(self):
+		d = frappe.get_doc(doctype="Error Log", title="Something").insert()
+		d.db_set("modified", "2010-01-01", update_modified=False)
+		frappe.db.commit()
+
+		self.execute("bench --site {site} clear-log-table --days=30 --doctype='Error Log'")
+		self.assertEqual(self.returncode, 0)
+		frappe.db.commit()
+
+		self.assertFalse(frappe.db.exists("Error Log", d.name))
+
 	def test_backup_with_custom_path(self):
 		"""Backup to a custom path (--backup-path)"""
 		backup_path = os.path.join(self.home, "backups")