diff --git a/frappe/__init__.py b/frappe/__init__.py index 8996a7fbe3..2a9a508921 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -1573,6 +1573,7 @@ from frappe.config import get_common_site_config, get_conf, get_site_config from frappe.core.doctype.system_settings.system_settings import get_system_settings from frappe.model.document import ( get_doc, + get_docs, get_lazy_doc, copy_doc, new_doc, diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py index 08c1e7fa68..cfb86e4c8e 100644 --- a/frappe/core/doctype/user/user.py +++ b/frappe/core/doctype/user/user.py @@ -645,18 +645,16 @@ class User(Document): frappe.db.delete("List Filter", {"for_user": self.name}) # Remove user from Note's Seen By table - seen_notes = frappe.get_all("Note", filters=[["Note Seen By", "user", "=", self.name]], pluck="name") - for note_id in seen_notes: - note = frappe.get_doc("Note", note_id) + seen_notes = frappe.get_docs("Note", filters=[["Note Seen By", "user", "=", self.name]]) + for note in seen_notes: for row in note.seen_by: if row.user == self.name: note.remove(row) note.save(ignore_permissions=True) # Unlink user from all of its invitation docs - invites = frappe.db.get_all("User Invitation", filters={"email": self.name}, pluck="name") - for invite in invites: - invite_doc = frappe.get_doc("User Invitation", invite) + invites = frappe.get_docs("User Invitation", filters={"email": self.name}) + for invite_doc in invites: invite_doc.user = None invite_doc.save(ignore_permissions=True) diff --git a/frappe/core/doctype/user_invitation/user_invitation.py b/frappe/core/doctype/user_invitation/user_invitation.py index 4d098d5af7..582ee2dfbe 100644 --- a/frappe/core/doctype/user_invitation/user_invitation.py +++ b/frappe/core/doctype/user_invitation/user_invitation.py @@ -206,12 +206,11 @@ class UserInvitation(Document): def mark_expired_invitations() -> None: days = 3 - invitations_to_expire = frappe.db.get_all( + invitations_to_expire = frappe.get_docs( "User Invitation", filters={"status": "Pending", "creation": ["<", frappe.utils.add_days(frappe.utils.now(), -days)]}, ) for invitation in invitations_to_expire: - invitation = frappe.get_doc("User Invitation", invitation.name) invitation.expire() # to avoid losing work in case the job times out without finishing frappe.db.commit() # nosemgrep diff --git a/frappe/desk/doctype/event/event.py b/frappe/desk/doctype/event/event.py index 99af59a417..33e480bb8c 100644 --- a/frappe/desk/doctype/event/event.py +++ b/frappe/desk/doctype/event/event.py @@ -137,7 +137,7 @@ class Event(Document): return for participant in self.event_participants: - if communications := frappe.get_all( + if communications := frappe.get_docs( "Communication", filters=[ ["Communication", "reference_doctype", "=", self.doctype], @@ -145,11 +145,9 @@ class Event(Document): ["Communication Link", "link_doctype", "=", participant.reference_doctype], ["Communication Link", "link_name", "=", participant.reference_docname], ], - pluck="name", distinct=True, ): - for comm in communications: - communication = frappe.get_doc("Communication", comm) + for communication in communications: self.update_communication(participant, communication) else: meta = frappe.get_meta(participant.reference_doctype) diff --git a/frappe/email/doctype/auto_email_report/auto_email_report.py b/frappe/email/doctype/auto_email_report/auto_email_report.py index f3109511a2..96f3ae0c44 100644 --- a/frappe/email/doctype/auto_email_report/auto_email_report.py +++ b/frappe/email/doctype/auto_email_report/auto_email_report.py @@ -359,8 +359,8 @@ def process_auto_email_report(report): def send_monthly(): """Check reports to be sent monthly""" - for report in frappe.get_all("Auto Email Report", {"enabled": 1, "frequency": "Monthly"}): - frappe.get_doc("Auto Email Report", report.name).send() + for report in frappe.get_docs("Auto Email Report", filters={"enabled": 1, "frequency": "Monthly"}): + report.send() def make_links(columns, data): diff --git a/frappe/model/document.py b/frappe/model/document.py index f959331d81..50d51e0d83 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -156,6 +156,164 @@ def get_lazy_doc( raise ImportError(doctype) +def get_docs( + doctype: str, + filters: dict | None = None, + *, + chunk_size: int = 1000, + limit: int | None = None, + limit_start: int = 0, + order_by: str = "creation asc", + as_iterator: bool = False, + for_update: bool = False, + distinct: bool = False, +) -> list["Document"] | Generator["Document"]: + """Fetch fully instantiated Document objects from the database. + + Returns a list of Documents by default. Pass `as_iterator=True` to get + a chunked generator that yields a list of Documents per chunk to reduce memory usage. + + :param doctype: DocType of the records to fetch. + :param filters: Dict or list of filters to apply. + :param chunk_size: Number of records to fetch in each chunk if using `as_iterator`. + :param limit: Maximum total number of records to fetch. + :param limit_start: Start results at record #. Default 0. + :param order_by: Order By string, e.g. `creation desc`. + :param as_iterator: If True, returns a iterator yielding Documents. + :param for_update: If True, locks the fetched rows for update. + :param distinct: If True, return distinct rows. + + + Note: Chunk size controls memory usage vs # of queries tradeoff. Using chunk size larger than + 10,000 is not advisable. + """ + if is_virtual_doctype(doctype): + frappe.throw(_("Virtual DocType {0} cannot be fetched in bulk.").format(doctype)) + + meta = frappe.get_meta(doctype) + + if meta.issingle: + frappe.throw(_("Single DocType {0} cannot be fetched in bulk.").format(doctype)) + + if limit_start and limit is None: + frappe.throw(_("limit cannot be None when limit_start is used")) + + if not order_by: + # Sort order is mandatory for iterator logic + order_by = "name asc" + + child_tables = [ + (df.fieldname, df.options) for df in meta.get_table_fields() if not is_virtual_doctype(df.options) + ] + controller = get_controller(doctype) + for_update = for_update and frappe.db.db_type != "sqlite" + + iterator = _get_docs_generator( + doctype, + controller, + child_tables, + filters=filters, + chunk_size=chunk_size, + limit_start=limit_start, + order_by=order_by, + for_update=for_update, + distinct=distinct, + ) + + iterator = itertools.islice(iterator, limit) + + if as_iterator: + return iterator + return list(iterator) + + +def _get_docs_generator( + doctype, + controller, + child_tables, + *, + filters, + chunk_size, + limit_start, + order_by, + for_update, + distinct, +) -> Generator["Document"]: + offset = limit_start + + while True: + chunk_data = _fetch_rows( + doctype, + filters=filters, + order_by=order_by, + limit=chunk_size, + offset=offset, + for_update=for_update, + child_tables=child_tables, + distinct=distinct, + ) + if not chunk_data: + break + yield from _build_document_objects(controller, chunk_data, for_update) + offset += chunk_size + + +def _fetch_rows(doctype, *, filters, order_by, limit, offset, for_update, child_tables, distinct=False): + kwargs = {} + if limit is not None: + kwargs["limit"] = limit + if offset: + kwargs["offset"] = offset + + data = frappe.qb.get_query( + table=doctype, + filters=filters or {}, + fields=["*"], + order_by=order_by, + for_update=for_update, + distinct=distinct, + **kwargs, + ).run(as_dict=True) + + if not data: + return [] + + for row in data: + row["doctype"] = doctype + + fetched_docs_by_name = {row.name: row for row in data} + parent_names = list(fetched_docs_by_name.keys()) + + for fieldname, child_doctype in child_tables: + child_table_data = frappe.qb.get_query( + table=child_doctype, + filters={"parent": ("in", parent_names), "parenttype": doctype, "parentfield": fieldname}, + fields=["*"], + order_by="idx asc", + for_update=for_update, + ).run(as_dict=True) + + for child in child_table_data: + child["doctype"] = child_doctype + + for parent_doc in fetched_docs_by_name.values(): + parent_doc[fieldname] = [] + + for child in child_table_data: + if child.parent in fetched_docs_by_name: + fetched_docs_by_name[child.parent][fieldname].append(child) + + return list(fetched_docs_by_name.values()) + + +def _build_document_objects(controller, data: list, for_update: bool): + for row in data: + doc = controller(row) + if for_update: + doc.flags.for_update = True + yield doc + + def get_doc_permission_check(doc: "Document", check_permission: str | bool | None = None) -> "Document": """ Checks permissions for the given document, if specified. diff --git a/frappe/tests/test_api.py b/frappe/tests/test_api.py index 5132359b0c..080de33564 100644 --- a/frappe/tests/test_api.py +++ b/frappe/tests/test_api.py @@ -170,7 +170,7 @@ class TestResourceAPI(FrappeAPITestCase): def test_unauthorized_call(self): # test 1: fetch documents without auth - response = requests.get(self.resource(self.DOCTYPE)) + response = requests.get(self.resource("User")) self.assertEqual(response.status_code, 403) def test_get_list(self): diff --git a/frappe/tests/test_document.py b/frappe/tests/test_document.py index a438f10e58..e28ca060c0 100644 --- a/frappe/tests/test_document.py +++ b/frappe/tests/test_document.py @@ -815,3 +815,99 @@ class TestLazyDocument(IntegrationTestCase): def test_for_update(self): guest = frappe.get_lazy_doc("User", "Guest", for_update=True) self.assertTrue(guest.flags.for_update) + + +class TestGetDocs(IntegrationTestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.child_dt = "Test Get Docs Child" + cls.parent_dt = "Test Get Docs Parent" + + cls.child_dt = new_doctype(istable=1).insert().name + cls.parent_dt = ( + new_doctype( + fields=[ + {"fieldtype": "Data", "fieldname": "title", "label": "Title"}, + { + "fieldtype": "Table", + "fieldname": "child_table", + "options": cls.child_dt, + "label": "Child Table", + }, + ], + ) + .insert() + .name + ) + for i in range(5): + frappe.get_doc( + { + "doctype": cls.parent_dt, + "title": f"Record {i}", + "child_table": [ + {"some_fieldname": f"child_{i}_0"}, + {"some_fieldname": f"child_{i}_1"}, + ], + } + ).insert() + + @classmethod + def tearDownClass(cls): + frappe.db.delete(cls.child_dt) + frappe.db.delete(cls.parent_dt) + frappe.delete_doc("DocType", cls.parent_dt, force=True) + frappe.delete_doc("DocType", cls.child_dt, force=True) + super().tearDownClass() + + def test_returns_document_instances(self): + docs = frappe.get_docs(self.parent_dt) + self.assertEqual(len(docs), 5) + self.assertIsInstance(docs[0], frappe.model.document.Document) + self.assertEqual(docs[0].doctype, self.parent_dt) + + def test_child_tables_populated(self): + docs = frappe.get_docs(self.parent_dt) + for doc in docs: + self.assertEqual(len(doc.child_table), 2) + for child in doc.child_table: + self.assertIsInstance(child, frappe.model.document.Document) + self.assertEqual(child.doctype, self.child_dt) + + def test_parity_with_get_doc(self): + docs = frappe.get_docs(self.parent_dt, limit=1) + doc_bulk = docs[0] + doc_single = frappe.get_doc(self.parent_dt, doc_bulk.name) + + self.assertEqual(doc_bulk.as_dict(), doc_single.as_dict()) + + def test_filters(self): + docs = frappe.get_docs(self.parent_dt, filters={"title": "Record 0"}) + self.assertEqual(len(docs), 1) + self.assertEqual(docs[0].title, "Record 0") + + def test_limit(self): + docs = frappe.get_docs(self.parent_dt, limit=2) + self.assertEqual(len(docs), 2) + + def test_limit_start(self): + all_docs = frappe.get_docs(self.parent_dt, order_by="creation asc") + offset_docs = frappe.get_docs(self.parent_dt, limit_start=2, limit=5, order_by="creation asc") + self.assertEqual(len(offset_docs), 3) + self.assertEqual(offset_docs[0].name, all_docs[2].name) + + def test_order_by(self): + docs_asc = frappe.get_docs(self.parent_dt, order_by="creation asc") + docs_desc = frappe.get_docs(self.parent_dt, order_by="creation desc") + self.assertEqual(docs_asc[0].name, docs_desc[-1].name) + + def test_generator_parity(self): + eager = frappe.get_docs(self.parent_dt, order_by="creation asc") + gen_docs = list( + frappe.get_docs(self.parent_dt, as_iterator=True, chunk_size=2, order_by="creation asc") + ) + self.assertEqual([d.name for d in eager], [d.name for d in gen_docs]) + + def test_for_update_sets_flag(self): + docs = frappe.get_docs(self.parent_dt, limit=1, for_update=True) + self.assertTrue(docs[0].flags.for_update) diff --git a/frappe/utils/scheduler.py b/frappe/utils/scheduler.py index 2d8075f8e9..8555bdd639 100644 --- a/frappe/utils/scheduler.py +++ b/frappe/utils/scheduler.py @@ -132,10 +132,9 @@ def enqueue_events_for_site(site: str) -> None: def enqueue_events() -> list[str] | None: if schedule_jobs_based_on_activity(): enqueued_jobs = [] - all_jobs = frappe.get_all("Scheduled Job Type", filters={"stopped": 0}, fields="*") + all_jobs = frappe.get_docs("Scheduled Job Type", filters={"stopped": 0}) random.shuffle(all_jobs) for job_type in all_jobs: - job_type = frappe.get_doc(doctype="Scheduled Job Type", **job_type) try: if job_type.enqueue(): enqueued_jobs.append(job_type.method)