Merge pull request #37861 from ShrihariMahabal/get-docs

feat: get_docs to fetch instantiated document objects from db

frappe/__init__.py

@@ -1573,6 +1573,7 @@ from frappe.config import get_common_site_config, get_conf, get_site_config
 from frappe.core.doctype.system_settings.system_settings import get_system_settings
 from frappe.model.document import (
 	get_doc,
+	get_docs,
 	get_lazy_doc,
 	copy_doc,
 	new_doc,

frappe/core/doctype/user/user.py

@@ -645,18 +645,16 @@ class User(Document):
 		frappe.db.delete("List Filter", {"for_user": self.name})
 
 		# Remove user from Note's Seen By table
-		seen_notes = frappe.get_all("Note", filters=[["Note Seen By", "user", "=", self.name]], pluck="name")
-		for note_id in seen_notes:
-			note = frappe.get_doc("Note", note_id)
+		seen_notes = frappe.get_docs("Note", filters=[["Note Seen By", "user", "=", self.name]])
+		for note in seen_notes:
 			for row in note.seen_by:
 				if row.user == self.name:
 					note.remove(row)
 			note.save(ignore_permissions=True)
 
 		# Unlink user from all of its invitation docs
-		invites = frappe.db.get_all("User Invitation", filters={"email": self.name}, pluck="name")
-		for invite in invites:
-			invite_doc = frappe.get_doc("User Invitation", invite)
+		invites = frappe.get_docs("User Invitation", filters={"email": self.name})
+		for invite_doc in invites:
 			invite_doc.user = None
 			invite_doc.save(ignore_permissions=True)
 

frappe/core/doctype/user_invitation/user_invitation.py

@@ -206,12 +206,11 @@ class UserInvitation(Document):
 
 def mark_expired_invitations() -> None:
 	days = 3
-	invitations_to_expire = frappe.db.get_all(
+	invitations_to_expire = frappe.get_docs(
 		"User Invitation",
 		filters={"status": "Pending", "creation": ["<", frappe.utils.add_days(frappe.utils.now(), -days)]},
 	)
 	for invitation in invitations_to_expire:
-		invitation = frappe.get_doc("User Invitation", invitation.name)
 		invitation.expire()
 		# to avoid losing work in case the job times out without finishing
 		frappe.db.commit()  # nosemgrep

frappe/desk/doctype/event/event.py

@@ -137,7 +137,7 @@ class Event(Document):
 			return
 
 		for participant in self.event_participants:
-			if communications := frappe.get_all(
+			if communications := frappe.get_docs(
 				"Communication",
 				filters=[
 					["Communication", "reference_doctype", "=", self.doctype],
@@ -145,11 +145,9 @@ class Event(Document):
 					["Communication Link", "link_doctype", "=", participant.reference_doctype],
 					["Communication Link", "link_name", "=", participant.reference_docname],
 				],
-				pluck="name",
 				distinct=True,
 			):
-				for comm in communications:
-					communication = frappe.get_doc("Communication", comm)
+				for communication in communications:
 					self.update_communication(participant, communication)
 			else:
 				meta = frappe.get_meta(participant.reference_doctype)

frappe/email/doctype/auto_email_report/auto_email_report.py

@@ -359,8 +359,8 @@ def process_auto_email_report(report):
 
 def send_monthly():
 	"""Check reports to be sent monthly"""
-	for report in frappe.get_all("Auto Email Report", {"enabled": 1, "frequency": "Monthly"}):
-		frappe.get_doc("Auto Email Report", report.name).send()
+	for report in frappe.get_docs("Auto Email Report", filters={"enabled": 1, "frequency": "Monthly"}):
+		report.send()
 
 
 def make_links(columns, data):

frappe/model/document.py

@@ -156,6 +156,164 @@ def get_lazy_doc(
 	raise ImportError(doctype)
 
 
+def get_docs(
+	doctype: str,
+	filters: dict | None = None,
+	*,
+	chunk_size: int = 1000,
+	limit: int | None = None,
+	limit_start: int = 0,
+	order_by: str = "creation asc",
+	as_iterator: bool = False,
+	for_update: bool = False,
+	distinct: bool = False,
+) -> list["Document"] | Generator["Document"]:
+	"""Fetch fully instantiated Document objects from the database.
+
+	Returns a list of Documents by default. Pass `as_iterator=True` to get
+	a chunked generator that yields a list of Documents per chunk to reduce memory usage.
+
+	:param doctype: DocType of the records to fetch.
+	:param filters: Dict or list of filters to apply.
+	:param chunk_size: Number of records to fetch in each chunk if using `as_iterator`.
+	:param limit: Maximum total number of records to fetch.
+	:param limit_start: Start results at record #. Default 0.
+	:param order_by: Order By string, e.g. `creation desc`.
+	:param as_iterator: If True, returns a iterator yielding Documents.
+	:param for_update: If True, locks the fetched rows for update.
+	:param distinct: If True, return distinct rows.
+
+
+	Note: Chunk size controls memory usage vs # of queries tradeoff. Using chunk size larger than
+	10,000 is not advisable.
+	"""
+	if is_virtual_doctype(doctype):
+		frappe.throw(_("Virtual DocType {0} cannot be fetched in bulk.").format(doctype))
+
+	meta = frappe.get_meta(doctype)
+
+	if meta.issingle:
+		frappe.throw(_("Single DocType {0} cannot be fetched in bulk.").format(doctype))
+
+	if limit_start and limit is None:
+		frappe.throw(_("limit cannot be None when limit_start is used"))
+
+	if not order_by:
+		# Sort order is mandatory for iterator logic
+		order_by = "name asc"
+
+	child_tables = [
+		(df.fieldname, df.options) for df in meta.get_table_fields() if not is_virtual_doctype(df.options)
+	]
+	controller = get_controller(doctype)
+	for_update = for_update and frappe.db.db_type != "sqlite"
+
+	iterator = _get_docs_generator(
+		doctype,
+		controller,
+		child_tables,
+		filters=filters,
+		chunk_size=chunk_size,
+		limit_start=limit_start,
+		order_by=order_by,
+		for_update=for_update,
+		distinct=distinct,
+	)
+
+	iterator = itertools.islice(iterator, limit)
+
+	if as_iterator:
+		return iterator
+	return list(iterator)
+
+
+def _get_docs_generator(
+	doctype,
+	controller,
+	child_tables,
+	*,
+	filters,
+	chunk_size,
+	limit_start,
+	order_by,
+	for_update,
+	distinct,
+) -> Generator["Document"]:
+	offset = limit_start
+
+	while True:
+		chunk_data = _fetch_rows(
+			doctype,
+			filters=filters,
+			order_by=order_by,
+			limit=chunk_size,
+			offset=offset,
+			for_update=for_update,
+			child_tables=child_tables,
+			distinct=distinct,
+		)
+		if not chunk_data:
+			break
+		yield from _build_document_objects(controller, chunk_data, for_update)
+		offset += chunk_size
+
+
+def _fetch_rows(doctype, *, filters, order_by, limit, offset, for_update, child_tables, distinct=False):
+	kwargs = {}
+	if limit is not None:
+		kwargs["limit"] = limit
+	if offset:
+		kwargs["offset"] = offset
+
+	data = frappe.qb.get_query(
+		table=doctype,
+		filters=filters or {},
+		fields=["*"],
+		order_by=order_by,
+		for_update=for_update,
+		distinct=distinct,
+		**kwargs,
+	).run(as_dict=True)
+
+	if not data:
+		return []
+
+	for row in data:
+		row["doctype"] = doctype
+
+	fetched_docs_by_name = {row.name: row for row in data}
+	parent_names = list(fetched_docs_by_name.keys())
+
+	for fieldname, child_doctype in child_tables:
+		child_table_data = frappe.qb.get_query(
+			table=child_doctype,
+			filters={"parent": ("in", parent_names), "parenttype": doctype, "parentfield": fieldname},
+			fields=["*"],
+			order_by="idx asc",
+			for_update=for_update,
+		).run(as_dict=True)
+
+		for child in child_table_data:
+			child["doctype"] = child_doctype
+
+		for parent_doc in fetched_docs_by_name.values():
+			parent_doc[fieldname] = []
+
+		for child in child_table_data:
+			if child.parent in fetched_docs_by_name:
+				fetched_docs_by_name[child.parent][fieldname].append(child)
+
+	return list(fetched_docs_by_name.values())
+
+
+def _build_document_objects(controller, data: list, for_update: bool):
+	for row in data:
+		doc = controller(row)
+		if for_update:
+			doc.flags.for_update = True
+		yield doc
+
+
 def get_doc_permission_check(doc: "Document", check_permission: str | bool | None = None) -> "Document":
 	"""
 	Checks permissions for the given document, if specified.

frappe/tests/test_api.py

@@ -170,7 +170,7 @@ class TestResourceAPI(FrappeAPITestCase):
 
 	def test_unauthorized_call(self):
 		# test 1: fetch documents without auth
-		response = requests.get(self.resource(self.DOCTYPE))
+		response = requests.get(self.resource("User"))
 		self.assertEqual(response.status_code, 403)
 
 	def test_get_list(self):

frappe/tests/test_document.py

@@ -815,3 +815,99 @@ class TestLazyDocument(IntegrationTestCase):
 	def test_for_update(self):
 		guest = frappe.get_lazy_doc("User", "Guest", for_update=True)
 		self.assertTrue(guest.flags.for_update)
+
+
+class TestGetDocs(IntegrationTestCase):
+	@classmethod
+	def setUpClass(cls):
+		super().setUpClass()
+		cls.child_dt = "Test Get Docs Child"
+		cls.parent_dt = "Test Get Docs Parent"
+
+		cls.child_dt = new_doctype(istable=1).insert().name
+		cls.parent_dt = (
+			new_doctype(
+				fields=[
+					{"fieldtype": "Data", "fieldname": "title", "label": "Title"},
+					{
+						"fieldtype": "Table",
+						"fieldname": "child_table",
+						"options": cls.child_dt,
+						"label": "Child Table",
+					},
+				],
+			)
+			.insert()
+			.name
+		)
+		for i in range(5):
+			frappe.get_doc(
+				{
+					"doctype": cls.parent_dt,
+					"title": f"Record {i}",
+					"child_table": [
+						{"some_fieldname": f"child_{i}_0"},
+						{"some_fieldname": f"child_{i}_1"},
+					],
+				}
+			).insert()
+
+	@classmethod
+	def tearDownClass(cls):
+		frappe.db.delete(cls.child_dt)
+		frappe.db.delete(cls.parent_dt)
+		frappe.delete_doc("DocType", cls.parent_dt, force=True)
+		frappe.delete_doc("DocType", cls.child_dt, force=True)
+		super().tearDownClass()
+
+	def test_returns_document_instances(self):
+		docs = frappe.get_docs(self.parent_dt)
+		self.assertEqual(len(docs), 5)
+		self.assertIsInstance(docs[0], frappe.model.document.Document)
+		self.assertEqual(docs[0].doctype, self.parent_dt)
+
+	def test_child_tables_populated(self):
+		docs = frappe.get_docs(self.parent_dt)
+		for doc in docs:
+			self.assertEqual(len(doc.child_table), 2)
+			for child in doc.child_table:
+				self.assertIsInstance(child, frappe.model.document.Document)
+				self.assertEqual(child.doctype, self.child_dt)
+
+	def test_parity_with_get_doc(self):
+		docs = frappe.get_docs(self.parent_dt, limit=1)
+		doc_bulk = docs[0]
+		doc_single = frappe.get_doc(self.parent_dt, doc_bulk.name)
+
+		self.assertEqual(doc_bulk.as_dict(), doc_single.as_dict())
+
+	def test_filters(self):
+		docs = frappe.get_docs(self.parent_dt, filters={"title": "Record 0"})
+		self.assertEqual(len(docs), 1)
+		self.assertEqual(docs[0].title, "Record 0")
+
+	def test_limit(self):
+		docs = frappe.get_docs(self.parent_dt, limit=2)
+		self.assertEqual(len(docs), 2)
+
+	def test_limit_start(self):
+		all_docs = frappe.get_docs(self.parent_dt, order_by="creation asc")
+		offset_docs = frappe.get_docs(self.parent_dt, limit_start=2, limit=5, order_by="creation asc")
+		self.assertEqual(len(offset_docs), 3)
+		self.assertEqual(offset_docs[0].name, all_docs[2].name)
+
+	def test_order_by(self):
+		docs_asc = frappe.get_docs(self.parent_dt, order_by="creation asc")
+		docs_desc = frappe.get_docs(self.parent_dt, order_by="creation desc")
+		self.assertEqual(docs_asc[0].name, docs_desc[-1].name)
+
+	def test_generator_parity(self):
+		eager = frappe.get_docs(self.parent_dt, order_by="creation asc")
+		gen_docs = list(
+			frappe.get_docs(self.parent_dt, as_iterator=True, chunk_size=2, order_by="creation asc")
+		)
+		self.assertEqual([d.name for d in eager], [d.name for d in gen_docs])
+
+	def test_for_update_sets_flag(self):
+		docs = frappe.get_docs(self.parent_dt, limit=1, for_update=True)
+		self.assertTrue(docs[0].flags.for_update)

frappe/utils/scheduler.py

@@ -132,10 +132,9 @@ def enqueue_events_for_site(site: str) -> None:
 def enqueue_events() -> list[str] | None:
 	if schedule_jobs_based_on_activity():
 		enqueued_jobs = []
-		all_jobs = frappe.get_all("Scheduled Job Type", filters={"stopped": 0}, fields="*")
+		all_jobs = frappe.get_docs("Scheduled Job Type", filters={"stopped": 0})
 		random.shuffle(all_jobs)
 		for job_type in all_jobs:
-			job_type = frappe.get_doc(doctype="Scheduled Job Type", **job_type)
 			try:
 				if job_type.enqueue():
 					enqueued_jobs.append(job_type.method)