fix: Expire stalled reports

Mark any report that take more than 60 minutes as failed

frappe/core/doctype/prepared_report/prepared_report.py

@@ -13,9 +13,13 @@ from frappe.desk.form.load import get_attachments
 from frappe.desk.query_report import generate_report_result
 from frappe.model.document import Document
 from frappe.monitor import add_data_to_monitor
-from frappe.utils import gzip_compress, gzip_decompress
+from frappe.utils import add_to_date, gzip_compress, gzip_decompress, now
 from frappe.utils.background_jobs import enqueue
 
+# If prepared report runs for longer than this time it's automatically considered as failed
+FAILURE_THRESHOLD = 60 * 60
+REPORT_TIMEOUT = 25 * 60
+
 
 class PreparedReport(Document):
 	@property
@@ -53,7 +57,7 @@ class PreparedReport(Document):
 			generate_report,
 			queue="long",
 			prepared_report=self.name,
-			timeout=1500,
+			timeout=REPORT_TIMEOUT,
 			enqueue_after_commit=True,
 		)
 
@@ -171,6 +175,21 @@ def get_completed_prepared_report(filters, user, report_name):
 	)
 
 
+def expire_stalled_report():
+	frappe.db.set_value(
+		"Prepared Report",
+		{
+			"status": "Started",
+			"modified": ("<", add_to_date(now(), seconds=-FAILURE_THRESHOLD, as_datetime=True)),
+		},
+		{
+			"status": "Failed",
+			"error_message": frappe._("Report timed out."),
+		},
+		update_modified=False,
+	)
+
+
 @frappe.whitelist()
 def delete_prepared_reports(reports):
 	reports = frappe.parse_json(reports)

frappe/hooks.py

@@ -196,9 +196,9 @@ scheduler_events = {
 			"frappe.email.doctype.email_account.email_account.pull",
 		],
 		# Hourly but offset by 30 minutes
-		# "30 * * * *": [
-		#
-		# ],
+		"30 * * * *": [
+			"frappe.core.doctype.prepared_report.prepared_report.expire_stalled_report",
+		],
 		# Daily but offset by 45 minutes
 		"45 0 * * *": [
 			"frappe.core.doctype.log_settings.log_settings.run_log_clean_up",