From a996ff7f5b9d8e5b8eaf915dcd7d896b6cca0397 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 18 Aug 2023 14:14:43 +0000 Subject: [PATCH 1/3] fix: Handle child record insertions via bulk_insert --- frappe/model/document.py | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/frappe/model/document.py b/frappe/model/document.py index 4f966c88b2..078a10be35 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -1643,12 +1643,31 @@ def bulk_insert( - Documents can be any iterable / generator containing Document objects """ - columns = frappe.get_meta(doctype).get_valid_columns() - values = _document_values_generator(documents, columns) + doctype_meta = frappe.get_meta(doctype) - frappe.db.bulk_insert( - doctype, columns, values, ignore_duplicates=ignore_duplicates, chunk_size=chunk_size - ) + valid_column_map = { + doctype: doctype_meta.get_valid_columns(), + } + values_map = { + doctype: _document_values_generator(documents, valid_column_map[doctype]), + } + + for child_table in doctype_meta.get_table_fields(): + valid_column_map[child_table.options] = frappe.get_meta(child_table.options).get_valid_columns() + values_map[child_table.options] = _document_values_generator( + ( + ch_doc + for ch_doc in ( + child_docs for doc in documents for child_docs in doc.get(child_table.fieldname) + ) + ), + valid_column_map[child_table.options], + ) + + for dt, docs in values_map.items(): + frappe.db.bulk_insert( + dt, valid_column_map[dt], docs, ignore_duplicates=ignore_duplicates, chunk_size=chunk_size + ) def _document_values_generator( From c89130caf07071b228025a88e0fbd27d8f7818c2 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 25 Aug 2023 11:56:52 +0530 Subject: [PATCH 2/3] test: Add checks for child records bulk inserts doc bulk_insert for "Role Profile" tables --- frappe/tests/test_document.py | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/frappe/tests/test_document.py b/frappe/tests/test_document.py index cdd1d08c62..aa4e788906 100644 --- a/frappe/tests/test_document.py +++ b/frappe/tests/test_document.py @@ -508,17 +508,34 @@ class TestDocumentWebView(FrappeTestCase): def test_bulk_inserts(self): from frappe.model.document import bulk_insert - doctype = "ToDo" - sent_todo = set() + doctype = "Role Profile" + child_field = "roles" + child_doctype = frappe.get_meta(doctype).get_field(child_field).options + + sent_docs = set() + sent_child_docs = set() def doc_generator(): - for i in range(690): + for _ in range(21): doc = frappe.new_doc(doctype) - doc.name = doc.description = frappe.generate_hash() - sent_todo.add(doc.name) + doc.role_profile = frappe.generate_hash() + doc.append("roles", {"role": "System Manager"}) + + doc.set_new_name() + doc.set_parent_in_children() + + sent_docs.add(doc.name) + sent_child_docs.add(doc.roles[0].name) + yield doc - bulk_insert(doctype, doc_generator(), chunk_size=100) + bulk_insert(doctype, doc_generator(), chunk_size=5) - all_todos = set(frappe.get_all("ToDo", pluck="name")) - self.assertEqual(sent_todo - all_todos, set(), "All docs should be inserted") + all_docs = set(frappe.get_all(doctype, pluck="name")) + all_child_docs = set( + frappe.get_all( + child_doctype, filters={"parenttype": doctype, "parentfield": child_field}, pluck="name" + ) + ) + self.assertEqual(sent_docs - all_docs, set(), "All docs should be inserted") + self.assertEqual(sent_child_docs - all_child_docs, set(), "All child docs should be inserted") From 6b1609d277c2211181316f8aee248859e9aa2b67 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 25 Aug 2023 16:12:48 +0530 Subject: [PATCH 3/3] fix: Cast documents iterable to list in db_insert --- frappe/model/document.py | 1 + 1 file changed, 1 insertion(+) diff --git a/frappe/model/document.py b/frappe/model/document.py index 078a10be35..8883a7e232 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -1644,6 +1644,7 @@ def bulk_insert( """ doctype_meta = frappe.get_meta(doctype) + documents = list(documents) valid_column_map = { doctype: doctype_meta.get_valid_columns(),