From bd2dc42b94d2fac738d92ca8708ade9032b539ef Mon Sep 17 00:00:00 2001
From: Ankush Menat <ankush@frappe.io>
Date: Mon, 8 Apr 2024 17:18:52 +0530
Subject: [PATCH] perf: speed up changelog fetching

- implicit limit
- bench level cache for each app
---
 .../changelog_feed/changelog_feed.json        |  5 ++-
 .../doctype/changelog_feed/changelog_feed.py  | 38 ++++++++++++-------
 frappe/hooks.py                               |  2 +-
 3 files changed, 30 insertions(+), 15 deletions(-)

diff --git a/frappe/desk/doctype/changelog_feed/changelog_feed.json b/frappe/desk/doctype/changelog_feed/changelog_feed.json
index 8463457a0a..ffdbddbda5 100644
--- a/frappe/desk/doctype/changelog_feed/changelog_feed.json
+++ b/frappe/desk/doctype/changelog_feed/changelog_feed.json
@@ -1,6 +1,7 @@
 {
  "actions": [],
  "allow_rename": 1,
+ "beta": 1,
  "creation": "2023-05-16 19:37:51.047664",
  "default_view": "List",
  "doctype": "DocType",
@@ -40,9 +41,10 @@
    "search_index": 1
   }
  ],
+ "in_create": 1,
  "index_web_pages_for_search": 1,
  "links": [],
- "modified": "2024-04-08 16:55:05.943403",
+ "modified": "2024-04-08 17:28:33.854732",
  "modified_by": "Administrator",
  "module": "Desk",
  "name": "Changelog Feed",
@@ -61,6 +63,7 @@
    "write": 1
   }
  ],
+ "read_only": 1,
  "sort_field": "modified",
  "sort_order": "DESC",
  "states": []
diff --git a/frappe/desk/doctype/changelog_feed/changelog_feed.py b/frappe/desk/doctype/changelog_feed/changelog_feed.py
index e4e430d7e4..296e9e8ee2 100644
--- a/frappe/desk/doctype/changelog_feed/changelog_feed.py
+++ b/frappe/desk/doctype/changelog_feed/changelog_feed.py
@@ -30,12 +30,13 @@ class ChangelogFeed(Document):
 
 
 def get_feed(since):
+	"""'What's New' feed implementation for Frappe"""
 	r = requests.get(f"https://frappe.io/api/method/fetch_changelog?since={since}").json()
 	changelog_posts = r["message"]
 	return changelog_posts
 
 
-def fetch_changelog_feed_items_from_source():
+def fetch_changelog_feed():
 	"""Fetches changelog feed items from source using `get_changelog_feed` hook and stores in the db"""
 	since = frappe.db.get_value(
 		"Changelog Feed",
@@ -45,19 +46,28 @@ def fetch_changelog_feed_items_from_source():
 	)
 
 	for fn in frappe.get_hooks("get_changelog_feed"):
-		with suppress(Exception):
-			for changelog_feed_item in frappe.call(fn, since=since):
-				change_log_feed_item_dict = {
-					"doctype": "Changelog Feed",
-					"title": changelog_feed_item["title"],
-					"app_name": changelog_feed_item["app_name"],
-					"link": changelog_feed_item["link"],
-					"posting_timestamp": changelog_feed_item["creation"],
+		try:
+			cache_key = f"changelog_feed::{fn}"
+			changelog_feed = frappe.cache.get_value(cache_key, shared=True)
+			if changelog_feed is None:
+				changelog_feed = frappe.call(fn, since=since)[:20] or []
+				frappe.cache.set_value(
+					cache_key, changelog_feed, expires_in_sec=7 * 24 * 60 * 60, shared=True
+				)
+
+			for feed_item in changelog_feed:
+				feed = {
+					"title": feed_item["title"],
+					"app_name": feed_item["app_name"],
+					"link": feed_item["link"],
+					"posting_timestamp": feed_item["creation"],
 				}
-				if not frappe.db.exists(change_log_feed_item_dict):
-					feed_doc = frappe.new_doc("Changelog Feed")
-					feed_doc.update(change_log_feed_item_dict)
-					feed_doc.insert()
+				if not frappe.db.exists("Changelog Feed", feed):
+					frappe.new_doc("Changelog Feed").update(feed).insert()
+		except Exception:
+			frappe.log_error(f"Failed to fetch changelog from {fn}")
+			# don't retry if it's broken for 1 week
+			frappe.cache.set_value(cache_key, [], expires_in_sec=7 * 24 * 60 * 60, shared=True)
 
 
 @frappe.whitelist()
@@ -67,6 +77,8 @@ def get_changelog_feed_items():
 	return frappe.get_all(
 		"Changelog Feed",
 		fields=["title", "app_name", "link", "posting_timestamp"],
+		# allow pubishing feed for many apps with single hook
+		filters={"app_name": ("in", frappe.get_installed_apps())},
 		order_by="posting_timestamp desc",
 		limit=10,
 	)
diff --git a/frappe/hooks.py b/frappe/hooks.py
index cdfdcf4434..0aa2f83307 100644
--- a/frappe/hooks.py
+++ b/frappe/hooks.py
@@ -259,7 +259,7 @@ scheduler_events = {
 		"frappe.desk.form.document_follow.send_weekly_updates",
 		"frappe.social.doctype.energy_point_log.energy_point_log.send_weekly_summary",
 		"frappe.integrations.doctype.google_drive.google_drive.weekly_backup",
-		"frappe.desk.doctype.changelog_feed.changelog_feed.fetch_changelog_feed_items_from_source",
+		"frappe.desk.doctype.changelog_feed.changelog_feed.fetch_changelog_feed",
 	],
 	"monthly": [
 		"frappe.email.doctype.auto_email_report.auto_email_report.send_monthly",