From db6fc6a5fb8ce88cc47b4aef4fd7c89d3e7fd0a6 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 3 Feb 2021 17:58:19 +0530 Subject: [PATCH 01/13] feat: Hash based comparison migration --- frappe/core/doctype/doctype/doctype.json | 1364 +++++++++++----------- frappe/migrate.py | 1 + frappe/modules/import_file.py | 33 +- 3 files changed, 717 insertions(+), 681 deletions(-) diff --git a/frappe/core/doctype/doctype/doctype.json b/frappe/core/doctype/doctype/doctype.json index 18435f8873..c85b4e8f67 100644 --- a/frappe/core/doctype/doctype/doctype.json +++ b/frappe/core/doctype/doctype/doctype.json @@ -1,680 +1,686 @@ { - "actions": [], - "allow_rename": 1, - "autoname": "Prompt", - "creation": "2013-02-18 13:36:19", - "description": "DocType is a Table / Form in the application.", - "doctype": "DocType", - "document_type": "Document", - "engine": "InnoDB", - "field_order": [ - "sb0", - "module", - "is_submittable", - "istable", - "issingle", - "is_tree", - "editable_grid", - "quick_entry", - "cb01", - "track_changes", - "track_seen", - "track_views", - "custom", - "beta", - "is_virtual", - "fields_section_break", - "fields", - "sb1", - "naming_rule", - "autoname", - "name_case", - "allow_rename", - "column_break_15", - "description", - "documentation", - "form_settings_section", - "image_field", - "timeline_field", - "nsm_parent_field", - "max_attachments", - "column_break_23", - "hide_toolbar", - "allow_copy", - "allow_import", - "allow_events_in_timeline", - "allow_auto_repeat", - "view_settings", - "title_field", - "search_fields", - "default_print_format", - "sort_field", - "sort_order", - "column_break_29", - "document_type", - "icon", - "color", - "show_preview_popup", - "show_name_in_global_search", - "email_settings_sb", - "default_email_template", - "column_break_51", - "email_append_to", - "sender_field", - "subject_field", - "sb2", - "permissions", - "restrict_to_domain", - "read_only", - "in_create", - "actions_section", - "actions", - "links_section", - "links", - "web_view", - "has_web_view", - "allow_guest_to_view", - "index_web_pages_for_search", - "route", - "is_published_field", - "website_search_field", - "advanced", - "engine" - ], - "fields": [ - { - "fieldname": "sb0", - "fieldtype": "Section Break", - "oldfieldtype": "Section Break" - }, - { - "fieldname": "module", - "fieldtype": "Link", - "in_list_view": 1, - "in_standard_filter": 1, - "label": "Module", - "oldfieldname": "module", - "oldfieldtype": "Link", - "options": "Module Def", - "reqd": 1, - "search_index": 1 - }, - { - "default": "0", - "depends_on": "eval:!doc.istable", - "description": "Once submitted, submittable documents cannot be changed. They can only be Cancelled and Amended.", - "fieldname": "is_submittable", - "fieldtype": "Check", - "label": "Is Submittable" - }, - { - "default": "0", - "description": "Child Tables are shown as a Grid in other DocTypes", - "fieldname": "istable", - "fieldtype": "Check", - "in_standard_filter": 1, - "label": "Is Child Table", - "oldfieldname": "istable", - "oldfieldtype": "Check" - }, - { - "default": "0", - "depends_on": "eval:!doc.istable", - "description": "Single Types have only one record no tables associated. Values are stored in tabSingles", - "fieldname": "issingle", - "fieldtype": "Check", - "in_standard_filter": 1, - "label": "Is Single", - "oldfieldname": "issingle", - "oldfieldtype": "Check", - "set_only_once": 1 - }, - { - "default": "1", - "depends_on": "istable", - "fieldname": "editable_grid", - "fieldtype": "Check", - "label": "Editable Grid" - }, - { - "default": "0", - "depends_on": "eval:!doc.istable && !doc.issingle", - "description": "Open a dialog with mandatory fields to create a new record quickly", - "fieldname": "quick_entry", - "fieldtype": "Check", - "label": "Quick Entry" - }, - { - "fieldname": "cb01", - "fieldtype": "Column Break" - }, - { - "default": "1", - "depends_on": "eval:!doc.istable", - "description": "If enabled, changes to the document are tracked and shown in timeline", - "fieldname": "track_changes", - "fieldtype": "Check", - "label": "Track Changes" - }, - { - "default": "0", - "depends_on": "eval:!doc.istable", - "description": "If enabled, the document is marked as seen, the first time a user opens it", - "fieldname": "track_seen", - "fieldtype": "Check", - "label": "Track Seen" - }, - { - "default": "0", - "depends_on": "eval:!doc.istable", - "description": "If enabled, document views are tracked, this can happen multiple times", - "fieldname": "track_views", - "fieldtype": "Check", - "label": "Track Views" - }, - { - "default": "0", - "fieldname": "custom", - "fieldtype": "Check", - "label": "Custom?" - }, - { - "default": "0", - "fieldname": "beta", - "fieldtype": "Check", - "label": "Beta" - }, - { - "fieldname": "fields_section_break", - "fieldtype": "Section Break", - "label": "Fields", - "oldfieldtype": "Section Break" - }, - { - "fieldname": "fields", - "fieldtype": "Table", - "label": "Fields", - "oldfieldname": "fields", - "oldfieldtype": "Table", - "options": "DocField" - }, - { - "fieldname": "sb1", - "fieldtype": "Section Break", - "label": "Naming" - }, - { - "description": "Naming Options:\n
  1. field:[fieldname] - By Field
  2. naming_series: - By Naming Series (field called naming_series must be present
  3. Prompt - Prompt user for a name
  4. [series] - Series by prefix (separated by a dot); for example PRE.#####
    5. \n
  5. format:EXAMPLE-{MM}morewords{fieldname1}-{fieldname2}-{#####} - Replace all braced words (fieldnames, date words (DD, MM, YY), series) with their value. Outside braces, any characters can be used.
", - "fieldname": "autoname", - "fieldtype": "Data", - "label": "Auto Name", - "oldfieldname": "autoname", - "oldfieldtype": "Data" - }, - { - "fieldname": "name_case", - "fieldtype": "Select", - "label": "Name Case", - "oldfieldname": "name_case", - "oldfieldtype": "Select", - "options": "\nTitle Case\nUPPER CASE" - }, - { - "fieldname": "column_break_15", - "fieldtype": "Column Break" - }, - { - "fieldname": "description", - "fieldtype": "Small Text", - "label": "Description", - "oldfieldname": "description", - "oldfieldtype": "Text" - }, - { - "collapsible": 1, - "fieldname": "form_settings_section", - "fieldtype": "Section Break", - "label": "Form Settings" - }, - { - "description": "Must be of type \"Attach Image\"", - "fieldname": "image_field", - "fieldtype": "Data", - "label": "Image Field" - }, - { - "depends_on": "eval:!doc.istable", - "description": "Comments and Communications will be associated with this linked document", - "fieldname": "timeline_field", - "fieldtype": "Data", - "label": "Timeline Field" - }, - { - "fieldname": "max_attachments", - "fieldtype": "Int", - "label": "Max Attachments", - "oldfieldname": "max_attachments", - "oldfieldtype": "Int" - }, - { - "fieldname": "column_break_23", - "fieldtype": "Column Break" - }, - { - "default": "0", - "fieldname": "hide_toolbar", - "fieldtype": "Check", - "label": "Hide Sidebar and Menu", - "oldfieldname": "hide_toolbar", - "oldfieldtype": "Check" - }, - { - "default": "0", - "fieldname": "allow_copy", - "fieldtype": "Check", - "label": "Hide Copy", - "oldfieldname": "allow_copy", - "oldfieldtype": "Check" - }, - { - "default": "1", - "fieldname": "allow_rename", - "fieldtype": "Check", - "label": "Allow Rename", - "oldfieldname": "allow_rename", - "oldfieldtype": "Check" - }, - { - "default": "0", - "fieldname": "allow_import", - "fieldtype": "Check", - "label": "Allow Import (via Data Import Tool)" - }, - { - "default": "0", - "fieldname": "allow_events_in_timeline", - "fieldtype": "Check", - "label": "Allow events in timeline" - }, - { - "default": "0", - "fieldname": "allow_auto_repeat", - "fieldtype": "Check", - "label": "Allow Auto Repeat" - }, - { - "collapsible": 1, - "fieldname": "view_settings", - "fieldtype": "Section Break", - "label": "View Settings" - }, - { - "depends_on": "eval:!doc.istable", - "fieldname": "title_field", - "fieldtype": "Data", - "label": "Title Field" - }, - { - "depends_on": "eval:!doc.istable", - "fieldname": "search_fields", - "fieldtype": "Data", - "label": "Search Fields", - "oldfieldname": "search_fields", - "oldfieldtype": "Data" - }, - { - "fieldname": "default_print_format", - "fieldtype": "Data", - "label": "Default Print Format" - }, - { - "default": "modified", - "depends_on": "eval:!doc.istable", - "fieldname": "sort_field", - "fieldtype": "Data", - "label": "Default Sort Field" - }, - { - "default": "DESC", - "depends_on": "eval:!doc.istable", - "fieldname": "sort_order", - "fieldtype": "Select", - "label": "Default Sort Order", - "options": "ASC\nDESC" - }, - { - "fieldname": "column_break_29", - "fieldtype": "Column Break" - }, - { - "fieldname": "document_type", - "fieldtype": "Select", - "label": "Show in Module Section", - "oldfieldname": "document_type", - "oldfieldtype": "Select", - "options": "\nDocument\nSetup\nSystem\nOther" - }, - { - "fieldname": "icon", - "fieldtype": "Data", - "label": "Icon" - }, - { - "fieldname": "color", - "fieldtype": "Data", - "label": "Color" - }, - { - "default": "0", - "fieldname": "show_preview_popup", - "fieldtype": "Check", - "label": "Show Preview Popup" - }, - { - "default": "0", - "fieldname": "show_name_in_global_search", - "fieldtype": "Check", - "label": "Make \"name\" searchable in Global Search" - }, - { - "depends_on": "eval:!doc.istable", - "fieldname": "sb2", - "fieldtype": "Section Break", - "label": "Permission Rules" - }, - { - "fieldname": "permissions", - "fieldtype": "Table", - "label": "Permissions", - "oldfieldname": "permissions", - "oldfieldtype": "Table", - "options": "DocPerm" - }, - { - "fieldname": "restrict_to_domain", - "fieldtype": "Link", - "label": "Restrict To Domain", - "options": "Domain" - }, - { - "default": "0", - "fieldname": "read_only", - "fieldtype": "Check", - "label": "User Cannot Search", - "oldfieldname": "read_only", - "oldfieldtype": "Check" - }, - { - "default": "0", - "fieldname": "in_create", - "fieldtype": "Check", - "label": "User Cannot Create", - "oldfieldname": "in_create", - "oldfieldtype": "Check" - }, - { - "depends_on": "eval:doc.custom===0", - "fieldname": "web_view", - "fieldtype": "Section Break", - "label": "Web View" - }, - { - "default": "0", - "fieldname": "has_web_view", - "fieldtype": "Check", - "label": "Has Web View" - }, - { - "default": "0", - "depends_on": "has_web_view", - "fieldname": "allow_guest_to_view", - "fieldtype": "Check", - "label": "Allow Guest to View" - }, - { - "depends_on": "eval:!doc.istable", - "fieldname": "route", - "fieldtype": "Data", - "label": "Route" - }, - { - "depends_on": "has_web_view", - "fieldname": "is_published_field", - "fieldtype": "Data", - "label": "Is Published Field" - }, - { - "collapsible": 1, - "fieldname": "advanced", - "fieldtype": "Section Break", - "hidden": 1, - "label": "Advanced" - }, - { - "default": "InnoDB", - "depends_on": "eval:!doc.issingle", - "fieldname": "engine", - "fieldtype": "Select", - "label": "Database Engine", - "options": "InnoDB\nMyISAM" - }, - { - "default": "0", - "description": "Tree structures are implemented using Nested Set", - "fieldname": "is_tree", - "fieldtype": "Check", - "label": "Is Tree" - }, - { - "depends_on": "is_tree", - "fieldname": "nsm_parent_field", - "fieldtype": "Data", - "label": "Parent Field (Tree)" - }, - { - "description": "URL for documentation or help", - "fieldname": "documentation", - "fieldtype": "Data", - "label": "Documentation Link" - }, - { - "collapsible": 1, - "collapsible_depends_on": "actions", - "fieldname": "actions_section", - "fieldtype": "Section Break", - "label": "Actions" - }, - { - "fieldname": "actions", - "fieldtype": "Table", - "label": "Actions", - "options": "DocType Action" - }, - { - "collapsible": 1, - "collapsible_depends_on": "links", - "fieldname": "links_section", - "fieldtype": "Section Break", - "label": "Linked Documents" - }, - { - "fieldname": "links", - "fieldtype": "Table", - "label": "Links", - "options": "DocType Link" - }, - { - "depends_on": "email_append_to", - "fieldname": "subject_field", - "fieldtype": "Data", - "label": "Subject Field" - }, - { - "depends_on": "email_append_to", - "fieldname": "sender_field", - "fieldtype": "Data", - "label": "Sender Field", - "mandatory_depends_on": "email_append_to" - }, - { - "default": "0", - "fieldname": "email_append_to", - "fieldtype": "Check", - "label": "Allow document creation via Email" - }, - { - "collapsible": 1, - "fieldname": "email_settings_sb", - "fieldtype": "Section Break", - "label": "Email Settings" - }, - { - "default": "1", - "fieldname": "index_web_pages_for_search", - "fieldtype": "Check", - "label": "Index Web Pages for Search" - }, - { - "default": "0", - "fieldname": "is_virtual", - "fieldtype": "Check", - "label": "Is Virtual" - }, - { - "fieldname": "default_email_template", - "fieldtype": "Link", - "label": "Default Email Template", - "options": "Email Template" - }, - { - "fieldname": "column_break_51", - "fieldtype": "Column Break" - }, - { - "depends_on": "has_web_view", - "fieldname": "website_search_field", - "fieldtype": "Data", - "label": "Website Search Field" - }, - { - "fieldname": "naming_rule", - "fieldtype": "Select", - "label": "Naming Rule", - "length": 40, - "options": "\nSet by user\nBy fieldname\nBy \"Naming Series\" field\nExpression\nExpression (old style)\nRandom\nBy script" - } - ], - "icon": "fa fa-bolt", - "idx": 6, - "links": [ - { - "group": "Views", - "link_doctype": "Report", - "link_fieldname": "ref_doctype" - }, - { - "group": "Workflow", - "link_doctype": "Workflow", - "link_fieldname": "document_type" - }, - { - "group": "Workflow", - "link_doctype": "Notification", - "link_fieldname": "document_type" - }, - { - "group": "Customization", - "link_doctype": "Custom Field", - "link_fieldname": "dt" - }, - { - "group": "Customization", - "link_doctype": "Client Script", - "link_fieldname": "dt" - }, - { - "group": "Customization", - "link_doctype": "Server Script", - "link_fieldname": "reference_doctype" - }, - { - "group": "Workflow", - "link_doctype": "Webhook", - "link_fieldname": "webhook_doctype" - }, - { - "group": "Views", - "link_doctype": "Print Format", - "link_fieldname": "doc_type" - }, - { - "group": "Views", - "link_doctype": "Web Form", - "link_fieldname": "doc_type" - }, - { - "group": "Views", - "link_doctype": "Calendar View", - "link_fieldname": "reference_doctype" - }, - { - "group": "Views", - "link_doctype": "Kanban Board", - "link_fieldname": "reference_doctype" - }, - { - "group": "Workflow", - "link_doctype": "Onboarding Step", - "link_fieldname": "reference_document" - }, - { - "group": "Rules", - "link_doctype": "Auto Repeat", - "link_fieldname": "reference_doctype" - }, - { - "group": "Rules", - "link_doctype": "Assignment Rule", - "link_fieldname": "document_type" - }, - { - "group": "Rules", - "link_doctype": "Energy Point Rule", - "link_fieldname": "reference_doctype" - } - ], - "modified": "2021-09-05 15:39:13.233403", - "modified_by": "Administrator", - "module": "Core", - "name": "DocType", - "owner": "Administrator", - "permissions": [ - { - "create": 1, - "delete": 1, - "email": 1, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "write": 1 - }, - { - "create": 1, - "delete": 1, - "email": 1, - "print": 1, - "read": 1, - "report": 1, - "role": "Administrator", - "share": 1, - "write": 1 - } - ], - "route": "doctype", - "search_fields": "module", - "show_name_in_global_search": 1, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1 -} \ No newline at end of file + "actions": [], + "allow_rename": 1, + "autoname": "Prompt", + "creation": "2013-02-18 13:36:19", + "description": "DocType is a Table / Form in the application.", + "doctype": "DocType", + "document_type": "Document", + "engine": "InnoDB", + "field_order": [ + "sb0", + "module", + "is_submittable", + "istable", + "issingle", + "is_tree", + "editable_grid", + "quick_entry", + "cb01", + "track_changes", + "track_seen", + "track_views", + "custom", + "beta", + "is_virtual", + "fields_section_break", + "fields", + "sb1", + "naming_rule", + "autoname", + "name_case", + "allow_rename", + "column_break_15", + "description", + "documentation", + "form_settings_section", + "image_field", + "timeline_field", + "nsm_parent_field", + "max_attachments", + "column_break_23", + "hide_toolbar", + "allow_copy", + "allow_import", + "allow_events_in_timeline", + "allow_auto_repeat", + "view_settings", + "title_field", + "search_fields", + "default_print_format", + "sort_field", + "sort_order", + "column_break_29", + "document_type", + "icon", + "color", + "show_preview_popup", + "show_name_in_global_search", + "email_settings_sb", + "default_email_template", + "column_break_51", + "email_append_to", + "sender_field", + "subject_field", + "sb2", + "permissions", + "restrict_to_domain", + "read_only", + "in_create", + "actions_section", + "actions", + "links_section", + "links", + "web_view", + "has_web_view", + "allow_guest_to_view", + "index_web_pages_for_search", + "route", + "is_published_field", + "website_search_field", + "advanced", + "engine", + "migration_hash" + ], + "fields": [ + { + "fieldname": "sb0", + "fieldtype": "Section Break", + "oldfieldtype": "Section Break" + }, + { + "fieldname": "module", + "fieldtype": "Link", + "in_list_view": 1, + "in_standard_filter": 1, + "label": "Module", + "oldfieldname": "module", + "oldfieldtype": "Link", + "options": "Module Def", + "reqd": 1, + "search_index": 1 + }, + { + "default": "0", + "depends_on": "eval:!doc.istable", + "description": "Once submitted, submittable documents cannot be changed. They can only be Cancelled and Amended.", + "fieldname": "is_submittable", + "fieldtype": "Check", + "label": "Is Submittable" + }, + { + "default": "0", + "description": "Child Tables are shown as a Grid in other DocTypes", + "fieldname": "istable", + "fieldtype": "Check", + "in_standard_filter": 1, + "label": "Is Child Table", + "oldfieldname": "istable", + "oldfieldtype": "Check" + }, + { + "default": "0", + "depends_on": "eval:!doc.istable", + "description": "Single Types have only one record no tables associated. Values are stored in tabSingles", + "fieldname": "issingle", + "fieldtype": "Check", + "in_standard_filter": 1, + "label": "Is Single", + "oldfieldname": "issingle", + "oldfieldtype": "Check", + "set_only_once": 1 + }, + { + "default": "1", + "depends_on": "istable", + "fieldname": "editable_grid", + "fieldtype": "Check", + "label": "Editable Grid" + }, + { + "default": "0", + "depends_on": "eval:!doc.istable && !doc.issingle", + "description": "Open a dialog with mandatory fields to create a new record quickly", + "fieldname": "quick_entry", + "fieldtype": "Check", + "label": "Quick Entry" + }, + { + "fieldname": "cb01", + "fieldtype": "Column Break" + }, + { + "default": "1", + "depends_on": "eval:!doc.istable", + "description": "If enabled, changes to the document are tracked and shown in timeline", + "fieldname": "track_changes", + "fieldtype": "Check", + "label": "Track Changes" + }, + { + "default": "0", + "depends_on": "eval:!doc.istable", + "description": "If enabled, the document is marked as seen, the first time a user opens it", + "fieldname": "track_seen", + "fieldtype": "Check", + "label": "Track Seen" + }, + { + "default": "0", + "depends_on": "eval:!doc.istable", + "description": "If enabled, document views are tracked, this can happen multiple times", + "fieldname": "track_views", + "fieldtype": "Check", + "label": "Track Views" + }, + { + "default": "0", + "fieldname": "custom", + "fieldtype": "Check", + "label": "Custom?" + }, + { + "default": "0", + "fieldname": "beta", + "fieldtype": "Check", + "label": "Beta" + }, + { + "fieldname": "fields_section_break", + "fieldtype": "Section Break", + "label": "Fields", + "oldfieldtype": "Section Break" + }, + { + "fieldname": "fields", + "fieldtype": "Table", + "label": "Fields", + "oldfieldname": "fields", + "oldfieldtype": "Table", + "options": "DocField" + }, + { + "fieldname": "sb1", + "fieldtype": "Section Break", + "label": "Naming" + }, + { + "description": "Naming Options:\n
  1. field:[fieldname] - By Field
  2. naming_series: - By Naming Series (field called naming_series must be present
  3. Prompt - Prompt user for a name
  4. [series] - Series by prefix (separated by a dot); for example PRE.#####
    5. \n
  5. format:EXAMPLE-{MM}morewords{fieldname1}-{fieldname2}-{#####} - Replace all braced words (fieldnames, date words (DD, MM, YY), series) with their value. Outside braces, any characters can be used.
", + "fieldname": "autoname", + "fieldtype": "Data", + "label": "Auto Name", + "oldfieldname": "autoname", + "oldfieldtype": "Data" + }, + { + "fieldname": "name_case", + "fieldtype": "Select", + "label": "Name Case", + "oldfieldname": "name_case", + "oldfieldtype": "Select", + "options": "\nTitle Case\nUPPER CASE" + }, + { + "fieldname": "column_break_15", + "fieldtype": "Column Break" + }, + { + "fieldname": "description", + "fieldtype": "Small Text", + "label": "Description", + "oldfieldname": "description", + "oldfieldtype": "Text" + }, + { + "collapsible": 1, + "fieldname": "form_settings_section", + "fieldtype": "Section Break", + "label": "Form Settings" + }, + { + "description": "Must be of type \"Attach Image\"", + "fieldname": "image_field", + "fieldtype": "Data", + "label": "Image Field" + }, + { + "depends_on": "eval:!doc.istable", + "description": "Comments and Communications will be associated with this linked document", + "fieldname": "timeline_field", + "fieldtype": "Data", + "label": "Timeline Field" + }, + { + "fieldname": "max_attachments", + "fieldtype": "Int", + "label": "Max Attachments", + "oldfieldname": "max_attachments", + "oldfieldtype": "Int" + }, + { + "fieldname": "column_break_23", + "fieldtype": "Column Break" + }, + { + "default": "0", + "fieldname": "hide_toolbar", + "fieldtype": "Check", + "label": "Hide Sidebar and Menu", + "oldfieldname": "hide_toolbar", + "oldfieldtype": "Check" + }, + { + "default": "0", + "fieldname": "allow_copy", + "fieldtype": "Check", + "label": "Hide Copy", + "oldfieldname": "allow_copy", + "oldfieldtype": "Check" + }, + { + "default": "1", + "fieldname": "allow_rename", + "fieldtype": "Check", + "label": "Allow Rename", + "oldfieldname": "allow_rename", + "oldfieldtype": "Check" + }, + { + "default": "0", + "fieldname": "allow_import", + "fieldtype": "Check", + "label": "Allow Import (via Data Import Tool)" + }, + { + "default": "0", + "fieldname": "allow_events_in_timeline", + "fieldtype": "Check", + "label": "Allow events in timeline" + }, + { + "default": "0", + "fieldname": "allow_auto_repeat", + "fieldtype": "Check", + "label": "Allow Auto Repeat" + }, + { + "collapsible": 1, + "fieldname": "view_settings", + "fieldtype": "Section Break", + "label": "View Settings" + }, + { + "depends_on": "eval:!doc.istable", + "fieldname": "title_field", + "fieldtype": "Data", + "label": "Title Field" + }, + { + "depends_on": "eval:!doc.istable", + "fieldname": "search_fields", + "fieldtype": "Data", + "label": "Search Fields", + "oldfieldname": "search_fields", + "oldfieldtype": "Data" + }, + { + "fieldname": "default_print_format", + "fieldtype": "Data", + "label": "Default Print Format" + }, + { + "default": "modified", + "depends_on": "eval:!doc.istable", + "fieldname": "sort_field", + "fieldtype": "Data", + "label": "Default Sort Field" + }, + { + "default": "DESC", + "depends_on": "eval:!doc.istable", + "fieldname": "sort_order", + "fieldtype": "Select", + "label": "Default Sort Order", + "options": "ASC\nDESC" + }, + { + "fieldname": "column_break_29", + "fieldtype": "Column Break" + }, + { + "fieldname": "document_type", + "fieldtype": "Select", + "label": "Show in Module Section", + "oldfieldname": "document_type", + "oldfieldtype": "Select", + "options": "\nDocument\nSetup\nSystem\nOther" + }, + { + "fieldname": "icon", + "fieldtype": "Data", + "label": "Icon" + }, + { + "fieldname": "color", + "fieldtype": "Data", + "label": "Color" + }, + { + "default": "0", + "fieldname": "show_preview_popup", + "fieldtype": "Check", + "label": "Show Preview Popup" + }, + { + "default": "0", + "fieldname": "show_name_in_global_search", + "fieldtype": "Check", + "label": "Make \"name\" searchable in Global Search" + }, + { + "depends_on": "eval:!doc.istable", + "fieldname": "sb2", + "fieldtype": "Section Break", + "label": "Permission Rules" + }, + { + "fieldname": "permissions", + "fieldtype": "Table", + "label": "Permissions", + "oldfieldname": "permissions", + "oldfieldtype": "Table", + "options": "DocPerm" + }, + { + "fieldname": "restrict_to_domain", + "fieldtype": "Link", + "label": "Restrict To Domain", + "options": "Domain" + }, + { + "default": "0", + "fieldname": "read_only", + "fieldtype": "Check", + "label": "User Cannot Search", + "oldfieldname": "read_only", + "oldfieldtype": "Check" + }, + { + "default": "0", + "fieldname": "in_create", + "fieldtype": "Check", + "label": "User Cannot Create", + "oldfieldname": "in_create", + "oldfieldtype": "Check" + }, + { + "depends_on": "eval:doc.custom===0", + "fieldname": "web_view", + "fieldtype": "Section Break", + "label": "Web View" + }, + { + "default": "0", + "fieldname": "has_web_view", + "fieldtype": "Check", + "label": "Has Web View" + }, + { + "default": "0", + "depends_on": "has_web_view", + "fieldname": "allow_guest_to_view", + "fieldtype": "Check", + "label": "Allow Guest to View" + }, + { + "depends_on": "eval:!doc.istable", + "fieldname": "route", + "fieldtype": "Data", + "label": "Route" + }, + { + "depends_on": "has_web_view", + "fieldname": "is_published_field", + "fieldtype": "Data", + "label": "Is Published Field" + }, + { + "collapsible": 1, + "fieldname": "advanced", + "fieldtype": "Section Break", + "hidden": 1, + "label": "Advanced" + }, + { + "default": "InnoDB", + "depends_on": "eval:!doc.issingle", + "fieldname": "engine", + "fieldtype": "Select", + "label": "Database Engine", + "options": "InnoDB\nMyISAM" + }, + { + "default": "0", + "description": "Tree structures are implemented using Nested Set", + "fieldname": "is_tree", + "fieldtype": "Check", + "label": "Is Tree" + }, + { + "depends_on": "is_tree", + "fieldname": "nsm_parent_field", + "fieldtype": "Data", + "label": "Parent Field (Tree)" + }, + { + "description": "URL for documentation or help", + "fieldname": "documentation", + "fieldtype": "Data", + "label": "Documentation Link" + }, + { + "collapsible": 1, + "collapsible_depends_on": "actions", + "fieldname": "actions_section", + "fieldtype": "Section Break", + "label": "Actions" + }, + { + "fieldname": "actions", + "fieldtype": "Table", + "label": "Actions", + "options": "DocType Action" + }, + { + "collapsible": 1, + "collapsible_depends_on": "links", + "fieldname": "links_section", + "fieldtype": "Section Break", + "label": "Linked Documents" + }, + { + "fieldname": "links", + "fieldtype": "Table", + "label": "Links", + "options": "DocType Link" + }, + { + "depends_on": "email_append_to", + "fieldname": "subject_field", + "fieldtype": "Data", + "label": "Subject Field" + }, + { + "depends_on": "email_append_to", + "fieldname": "sender_field", + "fieldtype": "Data", + "label": "Sender Field", + "mandatory_depends_on": "email_append_to" + }, + { + "default": "0", + "fieldname": "email_append_to", + "fieldtype": "Check", + "label": "Allow document creation via Email" + }, + { + "collapsible": 1, + "fieldname": "email_settings_sb", + "fieldtype": "Section Break", + "label": "Email Settings" + }, + { + "default": "1", + "fieldname": "index_web_pages_for_search", + "fieldtype": "Check", + "label": "Index Web Pages for Search" + }, + { + "default": "0", + "fieldname": "is_virtual", + "fieldtype": "Check", + "label": "Is Virtual" + }, + { + "fieldname": "default_email_template", + "fieldtype": "Link", + "label": "Default Email Template", + "options": "Email Template" + }, + { + "fieldname": "column_break_51", + "fieldtype": "Column Break" + }, + { + "depends_on": "has_web_view", + "fieldname": "website_search_field", + "fieldtype": "Data", + "label": "Website Search Field" + }, + { + "fieldname": "naming_rule", + "fieldtype": "Select", + "label": "Naming Rule", + "length": 40, + "options": "\nSet by user\nBy fieldname\nBy \"Naming Series\" field\nExpression\nExpression (old style)\nRandom\nBy script" + }, + { + "fieldname": "migration_hash", + "fieldtype": "Data", + "hidden": 1 + } + ], + "icon": "fa fa-bolt", + "idx": 6, + "links": [ + { + "group": "Views", + "link_doctype": "Report", + "link_fieldname": "ref_doctype" + }, + { + "group": "Workflow", + "link_doctype": "Workflow", + "link_fieldname": "document_type" + }, + { + "group": "Workflow", + "link_doctype": "Notification", + "link_fieldname": "document_type" + }, + { + "group": "Customization", + "link_doctype": "Custom Field", + "link_fieldname": "dt" + }, + { + "group": "Customization", + "link_doctype": "Client Script", + "link_fieldname": "dt" + }, + { + "group": "Customization", + "link_doctype": "Server Script", + "link_fieldname": "reference_doctype" + }, + { + "group": "Workflow", + "link_doctype": "Webhook", + "link_fieldname": "webhook_doctype" + }, + { + "group": "Views", + "link_doctype": "Print Format", + "link_fieldname": "doc_type" + }, + { + "group": "Views", + "link_doctype": "Web Form", + "link_fieldname": "doc_type" + }, + { + "group": "Views", + "link_doctype": "Calendar View", + "link_fieldname": "reference_doctype" + }, + { + "group": "Views", + "link_doctype": "Kanban Board", + "link_fieldname": "reference_doctype" + }, + { + "group": "Workflow", + "link_doctype": "Onboarding Step", + "link_fieldname": "reference_document" + }, + { + "group": "Rules", + "link_doctype": "Auto Repeat", + "link_fieldname": "reference_doctype" + }, + { + "group": "Rules", + "link_doctype": "Assignment Rule", + "link_fieldname": "document_type" + }, + { + "group": "Rules", + "link_doctype": "Energy Point Rule", + "link_fieldname": "reference_doctype" + } + ], + "modified": "2021-09-05 15:39:13.233403", + "modified_by": "Administrator", + "module": "Core", + "name": "DocType", + "owner": "Administrator", + "permissions": [ + { + "create": 1, + "delete": 1, + "email": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "write": 1 + }, + { + "create": 1, + "delete": 1, + "email": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "Administrator", + "share": 1, + "write": 1 + } + ], + "route": "doctype", + "search_fields": "module", + "show_name_in_global_search": 1, + "sort_field": "modified", + "sort_order": "DESC", + "track_changes": 1 +} diff --git a/frappe/migrate.py b/frappe/migrate.py index 92258502e4..a9c609e461 100644 --- a/frappe/migrate.py +++ b/frappe/migrate.py @@ -51,6 +51,7 @@ Otherwise, check the server logs and ensure that all the required services are r os.remove(touched_tables_file) try: + frappe.reload_doctype("DocType", force=True) frappe.flags.touched_tables = set() frappe.flags.in_migrate = True diff --git a/frappe/modules/import_file.py b/frappe/modules/import_file.py index e7a1f5f97c..83d789939d 100644 --- a/frappe/modules/import_file.py +++ b/frappe/modules/import_file.py @@ -5,6 +5,16 @@ from frappe.modules import get_module_path, scrub_dt_dn from frappe.utils import get_datetime_str from frappe.model.base_document import get_controller +import hashlib + +def md5(fname): + hash_md5 = hashlib.md5() + with open(fname, "rb") as f: + for chunk in iter(lambda: f.read(4096), b""): + hash_md5.update(chunk) + return hash_md5.hexdigest() + + ignore_values = { "Report": ["disabled", "prepared_report", "add_total_row"], "Print Format": ["disabled"], @@ -43,25 +53,44 @@ def get_file_path(module, dt, dn): def import_file_by_path(path, force=False, data_import=False, pre_process=None, ignore_version=None, reset_permissions=False, for_sync=False): + if not frappe.flags.dt: + frappe.flags.dt = [] try: docs = read_doc_from_file(path) except IOError: print (path + " missing") return + curr_hash = md5(path) + if docs: if not isinstance(docs, list): docs = [docs] for doc in docs: - if not force and not is_changed(doc): - return False + if not force: + try: + db_hash = frappe.db.get_value(doc['doctype'], doc['name'], 'migration_hash') + except: + frappe.flags.dt += [doc['doctype']] + db_hash = None + + if not db_hash: + db_modified = frappe.db.get_value(doc['doctype'], doc['name'], 'modified') + if db_modified and doc.get('modified') == get_datetime_str(db_modified): + return False + + if curr_hash == db_hash: + return False original_modified = doc.get("modified") import_doc(doc, force=force, data_import=data_import, pre_process=pre_process, ignore_version=ignore_version, reset_permissions=reset_permissions, path=path) + if doc['doctype'] == "DocType": + frappe.db.set_value(doc['doctype'], doc['name'], 'migration_hash', curr_hash) + if original_modified: update_modified(original_modified, doc) From c3be3d35e8a0b9a85fc4e6e77131de2f68c6dc07 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 3 Feb 2021 18:01:35 +0530 Subject: [PATCH 02/13] fix: "function"al programming++ --- frappe/model/sync.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/frappe/model/sync.py b/frappe/model/sync.py index 138f9eaad4..7ce9dba245 100644 --- a/frappe/model/sync.py +++ b/frappe/model/sync.py @@ -59,9 +59,10 @@ def sync_for(app_name, force=0, sync_everything = False, verbose=False, reset_pe for module_name in frappe.local.app_modules.get(app_name) or []: folder = os.path.dirname(frappe.get_module(app_name + "." + module_name).__file__) - get_doc_files(files, folder) + files = get_doc_files(files, folder, force, sync_everything, verbose=verbose) l = len(files) + if l: for i, doc_path in enumerate(files): import_file_by_path(doc_path, force=force, ignore_version=True, @@ -78,6 +79,9 @@ def sync_for(app_name, force=0, sync_everything = False, verbose=False, reset_pe def get_doc_files(files, start_path): """walk and sync all doctypes and pages""" + if not files: + files = [] + # load in sequence - warning for devs document_types = ['doctype', 'page', 'report', 'dashboard_chart_source', 'print_format', 'web_page', 'website_theme', 'web_form', 'web_template', @@ -95,3 +99,5 @@ def get_doc_files(files, start_path): if os.path.exists(doc_path): if not doc_path in files: files.append(doc_path) + + return files \ No newline at end of file From b2b391e90ae7fa82b4b0ff5a3eab8f2b9c6ba007 Mon Sep 17 00:00:00 2001 From: abhishek Date: Fri, 17 Sep 2021 14:49:51 +0530 Subject: [PATCH 03/13] style: misc --- frappe/model/sync.py | 3 +- frappe/modules/import_file.py | 82 +++++++++++-------- ..._user_permission_doctype_before_migrate.py | 4 +- 3 files changed, 51 insertions(+), 38 deletions(-) diff --git a/frappe/model/sync.py b/frappe/model/sync.py index 7ce9dba245..70afd79723 100644 --- a/frappe/model/sync.py +++ b/frappe/model/sync.py @@ -79,8 +79,7 @@ def sync_for(app_name, force=0, sync_everything = False, verbose=False, reset_pe def get_doc_files(files, start_path): """walk and sync all doctypes and pages""" - if not files: - files = [] + files = files or [] # load in sequence - warning for devs document_types = ['doctype', 'page', 'report', 'dashboard_chart_source', 'print_format', diff --git a/frappe/modules/import_file.py b/frappe/modules/import_file.py index 83d789939d..7c3ebfddb3 100644 --- a/frappe/modules/import_file.py +++ b/frappe/modules/import_file.py @@ -7,6 +7,7 @@ from frappe.model.base_document import get_controller import hashlib + def md5(fname): hash_md5 = hashlib.md5() with open(fname, "rb") as f: @@ -20,22 +21,24 @@ ignore_values = { "Print Format": ["disabled"], "Notification": ["enabled"], "Print Style": ["disabled"], - "Module Onboarding": ['is_complete'], - "Onboarding Step": ['is_complete', 'is_skipped'] + "Module Onboarding": ["is_complete"], + "Onboarding Step": ["is_complete", "is_skipped"], } ignore_doctypes = [""] + def import_files(module, dt=None, dn=None, force=False, pre_process=None, reset_permissions=False): if type(module) is list: out = [] for m in module: - out.append(import_file(m[0], m[1], m[2], force=force, pre_process=pre_process, - reset_permissions=reset_permissions)) + out.append( + import_file(m[0], m[1], m[2], force=force, pre_process=pre_process, reset_permissions=reset_permissions) + ) return out else: - return import_file(module, dt, dn, force=force, pre_process=pre_process, - reset_permissions=reset_permissions) + return import_file(module, dt, dn, force=force, pre_process=pre_process, reset_permissions=reset_permissions) + def import_file(module, dt, dn, force=False, pre_process=None, reset_permissions=False): """Sync a file from txt if modifed, return false if not updated""" @@ -43,22 +46,21 @@ def import_file(module, dt, dn, force=False, pre_process=None, reset_permissions ret = import_file_by_path(path, force, pre_process=pre_process, reset_permissions=reset_permissions) return ret + def get_file_path(module, dt, dn): dt, dn = scrub_dt_dn(dt, dn) - path = os.path.join(get_module_path(module), - os.path.join(dt, dn, dn + ".json")) + path = os.path.join(get_module_path(module), os.path.join(dt, dn, f"{dn}.json")) return path def import_file_by_path(path, force=False, data_import=False, pre_process=None, ignore_version=None, reset_permissions=False, for_sync=False): - if not frappe.flags.dt: - frappe.flags.dt = [] + frappe.flags.dt = frappe.flags.dt or [] try: docs = read_doc_from_file(path) except IOError: - print (path + " missing") + print(f"{path} missing") return curr_hash = md5(path) @@ -70,58 +72,68 @@ def import_file_by_path(path, force=False, data_import=False, pre_process=None, for doc in docs: if not force: try: - db_hash = frappe.db.get_value(doc['doctype'], doc['name'], 'migration_hash') + db_hash = frappe.db.get_value(doc["doctype"], doc["name"], "migration_hash") except: - frappe.flags.dt += [doc['doctype']] + frappe.flags.dt += [doc["doctype"]] db_hash = None if not db_hash: - db_modified = frappe.db.get_value(doc['doctype'], doc['name'], 'modified') - if db_modified and doc.get('modified') == get_datetime_str(db_modified): + db_modified = frappe.db.get_value(doc["doctype"], doc["name"], "modified") + if db_modified and doc.get("modified") == get_datetime_str(db_modified): return False - if curr_hash == db_hash: return False original_modified = doc.get("modified") - import_doc(doc, force=force, data_import=data_import, pre_process=pre_process, - ignore_version=ignore_version, reset_permissions=reset_permissions, path=path) + import_doc( + docdict=doc, + force=force, + data_import=data_import, + pre_process=pre_process, + ignore_version=ignore_version, + reset_permissions=reset_permissions, + path=path, + ) - if doc['doctype'] == "DocType": - frappe.db.set_value(doc['doctype'], doc['name'], 'migration_hash', curr_hash) + if doc["doctype"] == "DocType": + frappe.db.set_value(doc["doctype"], doc["name"], "migration_hash", curr_hash) if original_modified: update_modified(original_modified, doc) return True + def is_changed(doc): # check if timestamps match - db_modified = frappe.db.get_value(doc['doctype'], doc['name'], 'modified') - if db_modified and doc.get('modified')==get_datetime_str(db_modified): + db_modified = frappe.db.get_value(doc["doctype"], doc["name"], "modified") + if db_modified and doc.get("modified") == get_datetime_str(db_modified): return False return True + def read_doc_from_file(path): doc = None if os.path.exists(path): - with open(path, 'r') as f: + with open(path, "r") as f: try: doc = json.loads(f.read()) except ValueError: print("bad json: {0}".format(path)) raise else: - raise IOError('%s missing' % path) + raise IOError("%s missing" % path) return doc + def update_modified(original_modified, doc): # since there is a new timestamp on the file, update timestamp in - if doc["doctype"] == doc["name"] and doc["name"]!="DocType": - frappe.db.sql("""update tabSingles set value=%s where field="modified" and doctype=%s""", - (original_modified, doc["name"])) + if doc["doctype"] == doc["name"] and doc["name"] != "DocType": + frappe.db.sql( + """update tabSingles set value=%s where field="modified" and doctype=%s""", (original_modified, doc["name"]) + ) else: frappe.db.sql("update `tab%s` set modified=%s where name=%s" % (doc['doctype'], '%s', '%s'), (original_modified, doc['name'])) @@ -131,8 +143,8 @@ def import_doc(docdict, force=False, data_import=False, pre_process=None, frappe.flags.in_import = True docdict["__islocal"] = 1 - controller = get_controller(docdict['doctype']) - if controller and hasattr(controller, 'prepare_for_import') and callable(getattr(controller, 'prepare_for_import')): + controller = get_controller(docdict["doctype"]) + if controller and hasattr(controller, "prepare_for_import") and callable(getattr(controller, "prepare_for_import")): controller.prepare_for_import(docdict) doc = frappe.get_doc(docdict) @@ -161,15 +173,16 @@ def import_doc(docdict, force=False, data_import=False, pre_process=None, return doc + def load_code_properties(doc, path): - '''Load code files stored in separate files with extensions''' + """Load code files stored in separate files with extensions""" if path: - if hasattr(doc, 'get_code_fields'): + if hasattr(doc, "get_code_fields"): dirname, filename = os.path.split(path) for key, extn in doc.get_code_fields().items(): - codefile = os.path.join(dirname, filename.split('.')[0]+'.'+extn) + codefile = os.path.join(dirname, filename.split(".")[0] + "." + extn) if os.path.exists(codefile): - with open(codefile,'r') as txtfile: + with open(codefile, "r") as txtfile: doc.set(key, txtfile.read()) @@ -193,12 +206,13 @@ def delete_old_doc(doc, reset_permissions): doc.flags.ignore_children_type = ignore + def reset_tree_properties(doc): # Note on Tree DocTypes: # The tree structure is maintained in the database via the fields "lft" and # "rgt". They are automatically set and kept up-to-date. Importing them # would destroy any existing tree structure. - if getattr(doc.meta, 'is_tree', None) and any([doc.lft, doc.rgt]): + if getattr(doc.meta, "is_tree", None) and any([doc.lft, doc.rgt]): print('Ignoring values of `lft` and `rgt` for {} "{}"'.format(doc.doctype, doc.name)) doc.lft = None doc.rgt = None diff --git a/frappe/patches/v11_0/sync_user_permission_doctype_before_migrate.py b/frappe/patches/v11_0/sync_user_permission_doctype_before_migrate.py index 55a7b74f7e..6b7a7695f6 100644 --- a/frappe/patches/v11_0/sync_user_permission_doctype_before_migrate.py +++ b/frappe/patches/v11_0/sync_user_permission_doctype_before_migrate.py @@ -1,7 +1,7 @@ - import frappe + def execute(): frappe.flags.in_patch = True - frappe.reload_doc('core', 'doctype', 'user_permission') + frappe.reload_doc("core", "doctype", "user_permission") frappe.db.commit() From 278883edb0dd0a9cb5842de7480df207e00a2152 Mon Sep 17 00:00:00 2001 From: abhishek Date: Mon, 20 Sep 2021 14:52:42 +0530 Subject: [PATCH 04/13] fix: missing migration hash column --- frappe/database/mariadb/framework_mariadb.sql | 1 + frappe/database/postgres/framework_postgres.sql | 1 + frappe/migrate.py | 3 ++- frappe/model/sync.py | 11 ++++------- frappe/modules/import_file.py | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/frappe/database/mariadb/framework_mariadb.sql b/frappe/database/mariadb/framework_mariadb.sql index 670fb71aa2..73b98f0ff3 100644 --- a/frappe/database/mariadb/framework_mariadb.sql +++ b/frappe/database/mariadb/framework_mariadb.sql @@ -226,6 +226,7 @@ CREATE TABLE `tabDocType` ( `email_append_to` int(1) NOT NULL DEFAULT 0, `subject_field` varchar(255) DEFAULT NULL, `sender_field` varchar(255) DEFAULT NULL, + `migration_hash` varchar(255) DEFAULT NULL, PRIMARY KEY (`name`), KEY `parent` (`parent`) ) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; diff --git a/frappe/database/postgres/framework_postgres.sql b/frappe/database/postgres/framework_postgres.sql index 868f98fc98..e8e047f194 100644 --- a/frappe/database/postgres/framework_postgres.sql +++ b/frappe/database/postgres/framework_postgres.sql @@ -231,6 +231,7 @@ CREATE TABLE "tabDocType" ( "email_append_to" smallint NOT NULL DEFAULT 0, "subject_field" varchar(255) DEFAULT NULL, "sender_field" varchar(255) DEFAULT NULL, + "migration_hash" varchar(255) DEFAULT NULL, PRIMARY KEY ("name") ) ; diff --git a/frappe/migrate.py b/frappe/migrate.py index a9c609e461..ea60f3ab7f 100644 --- a/frappe/migrate.py +++ b/frappe/migrate.py @@ -18,6 +18,7 @@ from frappe.core.doctype.language.language import sync_languages from frappe.modules.utils import sync_customizations from frappe.core.doctype.scheduled_job_type.scheduled_job_type import sync_jobs from frappe.search.website_search import build_index_for_all_routes +from frappe.database.schema import add_column def migrate(verbose=True, skip_failing=False, skip_search_index=False): @@ -51,7 +52,7 @@ Otherwise, check the server logs and ensure that all the required services are r os.remove(touched_tables_file) try: - frappe.reload_doctype("DocType", force=True) + add_column(doctype="DocType", column_name="migration_hash", fieldtype="Data") frappe.flags.touched_tables = set() frappe.flags.in_migrate = True diff --git a/frappe/model/sync.py b/frappe/model/sync.py index 70afd79723..4850eefa33 100644 --- a/frappe/model/sync.py +++ b/frappe/model/sync.py @@ -76,19 +76,16 @@ def sync_for(app_name, force=0, sync_everything = False, verbose=False, reset_pe # print each progress bar on new line print() -def get_doc_files(files, start_path): +def get_doc_files(files, start_path, force=0, sync_everything = False, verbose=False): """walk and sync all doctypes and pages""" files = files or [] # load in sequence - warning for devs document_types = ['doctype', 'page', 'report', 'dashboard_chart_source', 'print_format', - 'web_page', 'website_theme', 'web_form', 'web_template', - 'notification', 'print_style', - 'data_migration_mapping', 'data_migration_plan', - 'workspace', 'onboarding_step', 'module_onboarding', 'form_tour', - 'client_script', 'server_script', 'custom_field', 'property_setter'] - + 'website_theme', 'web_form', 'web_template', 'notification', 'print_style', + 'data_migration_mapping', 'data_migration_plan', 'workspace', + 'onboarding_step', 'module_onboarding'] for doctype in document_types: doctype_path = os.path.join(start_path, doctype) if os.path.exists(doctype_path): diff --git a/frappe/modules/import_file.py b/frappe/modules/import_file.py index 7c3ebfddb3..097bb79cef 100644 --- a/frappe/modules/import_file.py +++ b/frappe/modules/import_file.py @@ -73,7 +73,7 @@ def import_file_by_path(path, force=False, data_import=False, pre_process=None, if not force: try: db_hash = frappe.db.get_value(doc["doctype"], doc["name"], "migration_hash") - except: + except Exception: frappe.flags.dt += [doc["doctype"]] db_hash = None From 822fdce4d6f32dfb39a394d820e3d52fe74d1bf9 Mon Sep 17 00:00:00 2001 From: abhishek Date: Wed, 29 Sep 2021 10:19:27 +0530 Subject: [PATCH 05/13] fix: doctype doc hash creation --- frappe/modules/import_file.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/frappe/modules/import_file.py b/frappe/modules/import_file.py index 097bb79cef..05c0cb82e9 100644 --- a/frappe/modules/import_file.py +++ b/frappe/modules/import_file.py @@ -97,7 +97,11 @@ def import_file_by_path(path, force=False, data_import=False, pre_process=None, ) if doc["doctype"] == "DocType": - frappe.db.set_value(doc["doctype"], doc["name"], "migration_hash", curr_hash) + if doc["name"] == "DocType": + Doctype_table=frappe.qb.DocType("DocType") + frappe.qb.update(Doctype_table).set(Doctype_table.migration_hash, curr_hash).where(Doctype_table.name == "DocType").run() + else: + frappe.db.set_value(doc["doctype"], doc["name"], "migration_hash", curr_hash) if original_modified: update_modified(original_modified, doc) From 69d8eddf5ae616d3c4a187d6acb2600419645745 Mon Sep 17 00:00:00 2001 From: abhishek Date: Mon, 4 Oct 2021 13:14:57 +0530 Subject: [PATCH 06/13] refactor: remove unused args refactor: remove unused args --- frappe/installer.py | 2 +- frappe/migrate.py | 7 ++++--- frappe/model/sync.py | 13 ++++++------- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/frappe/installer.py b/frappe/installer.py index e1b67ee10c..16198c8931 100755 --- a/frappe/installer.py +++ b/frappe/installer.py @@ -158,7 +158,7 @@ def install_app(name, verbose=False, set_as_patched=True): if name != "frappe": add_module_defs(name) - sync_for(name, force=True, sync_everything=True, verbose=verbose, reset_permissions=True) + sync_for(name, force=True, reset_permissions=True) add_to_installed_apps(name) diff --git a/frappe/migrate.py b/frappe/migrate.py index ea60f3ab7f..6abc38796f 100644 --- a/frappe/migrate.py +++ b/frappe/migrate.py @@ -27,9 +27,10 @@ def migrate(verbose=True, skip_failing=False, skip_search_index=False): - run patches - sync doctypes (schema) - sync dashboards + - sync jobs - sync fixtures - - sync desktop icons - - sync web pages (from /www) + - sync customizations + - sync languages - sync web pages (from /www) - run after migrate hooks ''' @@ -67,7 +68,7 @@ Otherwise, check the server logs and ensure that all the required services are r frappe.modules.patch_handler.run_all(skip_failing) # sync - frappe.model.sync.sync_all(verbose=verbose) + frappe.model.sync.sync_all() frappe.translate.clear_cache() sync_jobs() sync_fixtures() diff --git a/frappe/model/sync.py b/frappe/model/sync.py index 4850eefa33..e28c8ae4a7 100644 --- a/frappe/model/sync.py +++ b/frappe/model/sync.py @@ -10,17 +10,17 @@ from frappe.modules.import_file import import_file_by_path from frappe.modules.patch_handler import block_user from frappe.utils import update_progress_bar -def sync_all(force=0, verbose=False, reset_permissions=False): +def sync_all(force=0, reset_permissions=False): block_user(True) for app in frappe.get_installed_apps(): - sync_for(app, force, verbose=verbose, reset_permissions=reset_permissions) + sync_for(app, force, reset_permissions=reset_permissions) block_user(False) frappe.clear_cache() -def sync_for(app_name, force=0, sync_everything = False, verbose=False, reset_permissions=False): +def sync_for(app_name, force=0, reset_permissions=False): files = [] if app_name == "frappe": @@ -59,14 +59,13 @@ def sync_for(app_name, force=0, sync_everything = False, verbose=False, reset_pe for module_name in frappe.local.app_modules.get(app_name) or []: folder = os.path.dirname(frappe.get_module(app_name + "." + module_name).__file__) - files = get_doc_files(files, folder, force, sync_everything, verbose=verbose) + files = get_doc_files(files=files, start_path=folder) l = len(files) if l: for i, doc_path in enumerate(files): - import_file_by_path(doc_path, force=force, ignore_version=True, - reset_permissions=reset_permissions, for_sync=True) + import_file_by_path(doc_path, force=force, ignore_version=True, reset_permissions=reset_permissions, for_sync=True) frappe.db.commit() @@ -76,7 +75,7 @@ def sync_for(app_name, force=0, sync_everything = False, verbose=False, reset_pe # print each progress bar on new line print() -def get_doc_files(files, start_path, force=0, sync_everything = False, verbose=False): +def get_doc_files(files, start_path): """walk and sync all doctypes and pages""" files = files or [] From 3ac65adb94636b75ee1ece2a173c9879e4ac650d Mon Sep 17 00:00:00 2001 From: abhishek Date: Tue, 5 Oct 2021 20:24:04 +0530 Subject: [PATCH 07/13] style: misc changes --- frappe/modules/import_file.py | 59 ++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 28 deletions(-) diff --git a/frappe/modules/import_file.py b/frappe/modules/import_file.py index 05c0cb82e9..2c860055c9 100644 --- a/frappe/modules/import_file.py +++ b/frappe/modules/import_file.py @@ -1,16 +1,26 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe, os, json +import hashlib +import json +import os + +import frappe +from frappe.model.base_document import get_controller from frappe.modules import get_module_path, scrub_dt_dn from frappe.utils import get_datetime_str -from frappe.model.base_document import get_controller - -import hashlib -def md5(fname): +def caclulate_hash(path: str) -> str: + """Calculate md5 hash of the file in binary mode + + Args: + path (str): Path to the file to be hashed + + Returns: + str: The calculated hash + """ hash_md5 = hashlib.md5() - with open(fname, "rb") as f: + with open(path, "rb") as f: for chunk in iter(lambda: f.read(4096), b""): hash_md5.update(chunk) return hash_md5.hexdigest() @@ -32,9 +42,7 @@ def import_files(module, dt=None, dn=None, force=False, pre_process=None, reset_ if type(module) is list: out = [] for m in module: - out.append( - import_file(m[0], m[1], m[2], force=force, pre_process=pre_process, reset_permissions=reset_permissions) - ) + out.append(import_file(m[0], m[1], m[2], force=force, pre_process=pre_process, reset_permissions=reset_permissions)) return out else: return import_file(module, dt, dn, force=force, pre_process=pre_process, reset_permissions=reset_permissions) @@ -54,8 +62,7 @@ def get_file_path(module, dt, dn): return path -def import_file_by_path(path, force=False, data_import=False, pre_process=None, ignore_version=None, - reset_permissions=False, for_sync=False): +def import_file_by_path(path, force=False, data_import=False, pre_process=None, ignore_version=None, reset_permissions=False, for_sync=False): frappe.flags.dt = frappe.flags.dt or [] try: docs = read_doc_from_file(path) @@ -63,7 +70,7 @@ def import_file_by_path(path, force=False, data_import=False, pre_process=None, print(f"{path} missing") return - curr_hash = md5(path) + calculated_hash = caclulate_hash(path) if docs: if not isinstance(docs, list): @@ -72,16 +79,16 @@ def import_file_by_path(path, force=False, data_import=False, pre_process=None, for doc in docs: if not force: try: - db_hash = frappe.db.get_value(doc["doctype"], doc["name"], "migration_hash") + stored_hash = frappe.db.get_value(doc["doctype"], doc["name"], "migration_hash") except Exception: frappe.flags.dt += [doc["doctype"]] - db_hash = None + stored_hash = None - if not db_hash: - db_modified = frappe.db.get_value(doc["doctype"], doc["name"], "modified") - if db_modified and doc.get("modified") == get_datetime_str(db_modified): - return False - if curr_hash == db_hash: + # fallback if stored_hash doesn't exist + if not stored_hash and is_timestamp_changed(doc): + return False + + if calculated_hash == stored_hash: return False original_modified = doc.get("modified") @@ -96,12 +103,10 @@ def import_file_by_path(path, force=False, data_import=False, pre_process=None, path=path, ) + # not using db.set_value to avoid making changes in tabSingles if doc["doctype"] == "DocType": - if doc["name"] == "DocType": - Doctype_table=frappe.qb.DocType("DocType") - frappe.qb.update(Doctype_table).set(Doctype_table.migration_hash, curr_hash).where(Doctype_table.name == "DocType").run() - else: - frappe.db.set_value(doc["doctype"], doc["name"], "migration_hash", curr_hash) + doctype_table = frappe.qb.DocType("DocType") + frappe.qb.update(doctype_table).set(doctype_table.migration_hash, calculated_hash).where(doctype_table.name == doc["name"]).run() if original_modified: update_modified(original_modified, doc) @@ -109,12 +114,10 @@ def import_file_by_path(path, force=False, data_import=False, pre_process=None, return True -def is_changed(doc): +def is_timestamp_changed(doc): # check if timestamps match db_modified = frappe.db.get_value(doc["doctype"], doc["name"], "modified") - if db_modified and doc.get("modified") == get_datetime_str(db_modified): - return False - return True + return not (db_modified and doc.get("modified") == get_datetime_str(db_modified)) def read_doc_from_file(path): From 547111841ae62b8eca9821061aca7dacd97c5314 Mon Sep 17 00:00:00 2001 From: abhishek Date: Tue, 5 Oct 2021 20:25:19 +0530 Subject: [PATCH 08/13] refactor: sql in update_modified() to frappe.qb --- frappe/modules/import_file.py | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/frappe/modules/import_file.py b/frappe/modules/import_file.py index 2c860055c9..00f298655f 100644 --- a/frappe/modules/import_file.py +++ b/frappe/modules/import_file.py @@ -138,15 +138,28 @@ def read_doc_from_file(path): def update_modified(original_modified, doc): # since there is a new timestamp on the file, update timestamp in if doc["doctype"] == doc["name"] and doc["name"] != "DocType": - frappe.db.sql( - """update tabSingles set value=%s where field="modified" and doctype=%s""", (original_modified, doc["name"]) - ) - else: - frappe.db.sql("update `tab%s` set modified=%s where name=%s" % (doc['doctype'], - '%s', '%s'), (original_modified, doc['name'])) + singles_table = frappe.qb.DocType("Singles") -def import_doc(docdict, force=False, data_import=False, pre_process=None, - ignore_version=None, reset_permissions=False, path=None): + frappe.qb.update( + singles_table + ).set( + singles_table.value,original_modified + ).where( + singles_table.field == "modified" + ).where( + singles_table.doctype == doc["name"] + ).run() + else: + doctype_table = frappe.qb.DocType(doc['doctype']) + + frappe.qb.update(doctype_table + ).set( + doctype_table.modified, original_modified + ).where( + doctype_table.name == doc["name"] + ).run() + +def import_doc(docdict, force=False, data_import=False, pre_process=None, ignore_version=None, reset_permissions=False, path=None): frappe.flags.in_import = True docdict["__islocal"] = 1 From 0be59cee807ef6167883fe32680f2c177202f2d5 Mon Sep 17 00:00:00 2001 From: abhishek Date: Tue, 5 Oct 2021 20:26:18 +0530 Subject: [PATCH 09/13] chore: add missing document_types --- frappe/model/sync.py | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/frappe/model/sync.py b/frappe/model/sync.py index e28c8ae4a7..bd6dacf332 100644 --- a/frappe/model/sync.py +++ b/frappe/model/sync.py @@ -81,10 +81,29 @@ def get_doc_files(files, start_path): files = files or [] # load in sequence - warning for devs - document_types = ['doctype', 'page', 'report', 'dashboard_chart_source', 'print_format', - 'website_theme', 'web_form', 'web_template', 'notification', 'print_style', - 'data_migration_mapping', 'data_migration_plan', 'workspace', - 'onboarding_step', 'module_onboarding'] + document_types = [ + "doctype", + "page", + "report", + "dashboard_chart_source", + "print_format", + "web_page", + "website_theme", + "web_form", + "web_template", + "notification", + "print_style", + "data_migration_mapping", + "data_migration_plan", + "workspace", + "onboarding_step", + "module_onboarding", + "form_tour", + "client_script", + "server_script", + "custom_field", + "property_setter", + ] for doctype in document_types: doctype_path = os.path.join(start_path, doctype) if os.path.exists(doctype_path): @@ -95,4 +114,4 @@ def get_doc_files(files, start_path): if not doc_path in files: files.append(doc_path) - return files \ No newline at end of file + return files From f3d810414a3823de0c369b09aae756aea83c7457 Mon Sep 17 00:00:00 2001 From: abhishek Date: Wed, 6 Oct 2021 01:57:31 +0530 Subject: [PATCH 10/13] refactor: sync_for() - first install module --- frappe/model/sync.py | 68 ++++++++++++++++++++++++-------------------- 1 file changed, 37 insertions(+), 31 deletions(-) diff --git a/frappe/model/sync.py b/frappe/model/sync.py index bd6dacf332..6f829d6135 100644 --- a/frappe/model/sync.py +++ b/frappe/model/sync.py @@ -10,6 +10,7 @@ from frappe.modules.import_file import import_file_by_path from frappe.modules.patch_handler import block_user from frappe.utils import update_progress_bar + def sync_all(force=0, reset_permissions=False): block_user(True) @@ -20,42 +21,46 @@ def sync_all(force=0, reset_permissions=False): frappe.clear_cache() + def sync_for(app_name, force=0, reset_permissions=False): files = [] if app_name == "frappe": # these need to go first at time of install - for d in (("core", "docfield"), - ("core", "docperm"), - ("core", "doctype_action"), - ("core", "doctype_link"), - ("core", "role"), - ("core", "has_role"), - ("core", "doctype"), - ("core", "user"), - ("custom", "custom_field"), - ("custom", "property_setter"), - ("website", "web_form"), - ("website", "web_template"), - ("website", "web_form_field"), - ("website", "portal_menu_item"), - ("data_migration", "data_migration_mapping_detail"), - ("data_migration", "data_migration_mapping"), - ("data_migration", "data_migration_plan_mapping"), - ("data_migration", "data_migration_plan"), - ("desk", "number_card"), - ("desk", "dashboard_chart"), - ("desk", "dashboard"), - ("desk", "onboarding_permission"), - ("desk", "onboarding_step"), - ("desk", "onboarding_step_map"), - ("desk", "module_onboarding"), - ("desk", "workspace_link"), - ("desk", "workspace_chart"), - ("desk", "workspace_shortcut"), - ("desk", "workspace")): - files.append(os.path.join(frappe.get_app_path("frappe"), d[0], - "doctype", d[1], d[1] + ".json")) + + FRAPPE_PATH = frappe.get_app_path("frappe") + + for core_module in ["docfield", "docperm", "doctype_action", "doctype_link", "role", "has_role", "doctype"]: + files.append(os.path.join(FRAPPE_PATH, "core", "doctype", core_module, f"{core_module}.json")) + + for custom_module in ["custom_field", "property_setter"]: + files.append(os.path.join(FRAPPE_PATH, "custom", "doctype", custom_module, f"{custom_module}.json")) + + for website_module in ["web_form", "web_template", "web_form_field", "portal_menu_item"]: + files.append(os.path.join(FRAPPE_PATH, "website", "doctype", website_module, f"{website_module}.json")) + + for data_migration_module in [ + "data_migration_mapping_detail", + "data_migration_mapping", + "data_migration_plan_mapping", + "data_migration_plan", + ]: + files.append(os.path.join(FRAPPE_PATH, "data_migration", "doctype", data_migration_module, f"{data_migration_module}.json")) + + for desk_module in [ + "number_card", + "dashboard_chart", + "dashboard", + "onboarding_permission", + "onboarding_step", + "onboarding_step_map", + "module_onboarding", + "workspace_link", + "workspace_chart", + "workspace_shortcut", + "workspace", + ]: + files.append(os.path.join(FRAPPE_PATH, "desk", "doctype", desk_module, f"{desk_module}.json")) for module_name in frappe.local.app_modules.get(app_name) or []: folder = os.path.dirname(frappe.get_module(app_name + "." + module_name).__file__) @@ -75,6 +80,7 @@ def sync_for(app_name, force=0, reset_permissions=False): # print each progress bar on new line print() + def get_doc_files(files, start_path): """walk and sync all doctypes and pages""" From 2622f3398e61df302ab94a6ef839a1f55304e0d4 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 7 Oct 2021 17:21:13 +0530 Subject: [PATCH 11/13] style: Black-ish queries * feat(minor): Added DocType to frappe.query_builder namespace --- frappe/modules/import_file.py | 12 +++++++----- frappe/query_builder/__init__.py | 2 +- frappe/query_builder/utils.py | 3 +++ 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/frappe/modules/import_file.py b/frappe/modules/import_file.py index 00f298655f..230a59df6e 100644 --- a/frappe/modules/import_file.py +++ b/frappe/modules/import_file.py @@ -8,7 +8,7 @@ import frappe from frappe.model.base_document import get_controller from frappe.modules import get_module_path, scrub_dt_dn from frappe.utils import get_datetime_str - +from frappe.query_builder import DocType def caclulate_hash(path: str) -> str: """Calculate md5 hash of the file in binary mode @@ -106,7 +106,9 @@ def import_file_by_path(path, force=False, data_import=False, pre_process=None, # not using db.set_value to avoid making changes in tabSingles if doc["doctype"] == "DocType": doctype_table = frappe.qb.DocType("DocType") - frappe.qb.update(doctype_table).set(doctype_table.migration_hash, calculated_hash).where(doctype_table.name == doc["name"]).run() + frappe.qb.update(doctype_table).set( + doctype_table.migration_hash, calculated_hash + ).where(doctype_table.name == doc["name"]).run() if original_modified: update_modified(original_modified, doc) @@ -138,7 +140,7 @@ def read_doc_from_file(path): def update_modified(original_modified, doc): # since there is a new timestamp on the file, update timestamp in if doc["doctype"] == doc["name"] and doc["name"] != "DocType": - singles_table = frappe.qb.DocType("Singles") + singles_table = DocType("Singles") frappe.qb.update( singles_table @@ -150,8 +152,8 @@ def update_modified(original_modified, doc): singles_table.doctype == doc["name"] ).run() else: - doctype_table = frappe.qb.DocType(doc['doctype']) - + doctype_table = DocType(doc['doctype']) + frappe.qb.update(doctype_table ).set( doctype_table.modified, original_modified diff --git a/frappe/query_builder/__init__.py b/frappe/query_builder/__init__.py index 6987ba24ab..4a1fe8fb84 100644 --- a/frappe/query_builder/__init__.py +++ b/frappe/query_builder/__init__.py @@ -1,2 +1,2 @@ from pypika import * -from frappe.query_builder.utils import Column, get_query_builder, patch_query_execute +from frappe.query_builder.utils import Column, DocType, get_query_builder, patch_query_execute diff --git a/frappe/query_builder/utils.py b/frappe/query_builder/utils.py index abd700c396..386ddda751 100644 --- a/frappe/query_builder/utils.py +++ b/frappe/query_builder/utils.py @@ -44,6 +44,9 @@ def get_attr(method_string): methodname = method_string.split('.')[-1] return getattr(import_module(modulename), methodname) +def DocType(*args, **kwargs): + return frappe.qb.DocType(*args, **kwargs) + def patch_query_execute(): """Patch the Query Builder with helper execute method This excludes the use of `frappe.db.sql` method while From 86f29aeaa34db8456161e530da0e4ed7bf500911 Mon Sep 17 00:00:00 2001 From: abhishek Date: Tue, 12 Oct 2021 03:21:08 +0530 Subject: [PATCH 12/13] fix: missing logical cases - handle first time imports - update hash and timestamps --- frappe/modules/import_file.py | 40 +++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/frappe/modules/import_file.py b/frappe/modules/import_file.py index 230a59df6e..f4179fb811 100644 --- a/frappe/modules/import_file.py +++ b/frappe/modules/import_file.py @@ -7,8 +7,9 @@ import os import frappe from frappe.model.base_document import get_controller from frappe.modules import get_module_path, scrub_dt_dn -from frappe.utils import get_datetime_str from frappe.query_builder import DocType +from frappe.utils import get_datetime_str, now + def caclulate_hash(path: str) -> str: """Calculate md5 hash of the file in binary mode @@ -77,22 +78,26 @@ def import_file_by_path(path, force=False, data_import=False, pre_process=None, docs = [docs] for doc in docs: - if not force: + + # modified timestamp in db, none if doctype's first import + db_modified_timestamp = frappe.db.get_value(doc["doctype"], doc["name"], "modified") + is_db_timestamp_latest = db_modified_timestamp and doc.get("modified") <= get_datetime_str(db_modified_timestamp) + + if not force or db_modified_timestamp: try: stored_hash = frappe.db.get_value(doc["doctype"], doc["name"], "migration_hash") except Exception: frappe.flags.dt += [doc["doctype"]] stored_hash = None - # fallback if stored_hash doesn't exist - if not stored_hash and is_timestamp_changed(doc): + # if hash exists and is equal no need to update + if stored_hash and stored_hash == calculated_hash: return False - if calculated_hash == stored_hash: + # if hash doesn't exist, check if db timestamp is same as json timestamp, add hash if from doctype + if is_db_timestamp_latest and doc["doctype"] != "DocType": return False - original_modified = doc.get("modified") - import_doc( docdict=doc, force=force, @@ -103,15 +108,24 @@ def import_file_by_path(path, force=False, data_import=False, pre_process=None, path=path, ) - # not using db.set_value to avoid making changes in tabSingles if doc["doctype"] == "DocType": - doctype_table = frappe.qb.DocType("DocType") - frappe.qb.update(doctype_table).set( + doctype_table = DocType("DocType") + frappe.qb.update( + doctype_table + ).set( doctype_table.migration_hash, calculated_hash - ).where(doctype_table.name == doc["name"]).run() + ).where( + doctype_table.name == doc["name"] + ).run() - if original_modified: - update_modified(original_modified, doc) + new_modified_timestamp = doc.get("modified") + + # if db timestamp is newer, hash must have changed, must update db timestamp + if is_db_timestamp_latest and doc["doctype"] == "DocType": + new_modified_timestamp = now() + + if new_modified_timestamp: + update_modified(new_modified_timestamp, doc) return True From ba78852c73893fdf41c463539dbb19287f94cded Mon Sep 17 00:00:00 2001 From: abhishek Date: Tue, 12 Oct 2021 17:15:12 +0530 Subject: [PATCH 13/13] docs: add docstring for import_file_by_path --- frappe/model/sync.py | 2 +- frappe/modules/import_file.py | 30 +++++++++++++++++++++++++++++- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/frappe/model/sync.py b/frappe/model/sync.py index 6f829d6135..42bb16cbc2 100644 --- a/frappe/model/sync.py +++ b/frappe/model/sync.py @@ -70,7 +70,7 @@ def sync_for(app_name, force=0, reset_permissions=False): if l: for i, doc_path in enumerate(files): - import_file_by_path(doc_path, force=force, ignore_version=True, reset_permissions=reset_permissions, for_sync=True) + import_file_by_path(doc_path, force=force, ignore_version=True, reset_permissions=reset_permissions) frappe.db.commit() diff --git a/frappe/modules/import_file.py b/frappe/modules/import_file.py index f4179fb811..cf8ec46d76 100644 --- a/frappe/modules/import_file.py +++ b/frappe/modules/import_file.py @@ -63,7 +63,35 @@ def get_file_path(module, dt, dn): return path -def import_file_by_path(path, force=False, data_import=False, pre_process=None, ignore_version=None, reset_permissions=False, for_sync=False): + +def import_file_by_path(path: str,force: bool = False,data_import: bool = False,pre_process = None,ignore_version: bool = None,reset_permissions: bool = False): + """Import file from the given path + + Some conditions decide if a file should be imported or not. + Evaluation takes place in the order they are mentioned below. + + - Check if `force` is true. Import the file. If not, move ahead. + - Get `db_modified_timestamp`(value of the modified field in the database for the file). + If the return is `none,` this file doesn't exist in the DB, so Import the file. If not, move ahead. + - Check if there is a hash in DB for that file. If there is, Calculate the Hash of the file to import and compare it with the one in DB if they are not equal. + Import the file. If Hash doesn't exist, move ahead. + - Check if `db_modified_timestamp` is older than the timestamp in the file; if it is, we import the file. + + If timestamp comparison happens for doctypes, that means the Hash for it doesn't exist. + So, even if the timestamp is newer on DB (When comparing timestamps), we import the file and add the calculated Hash to the DB. + So in the subsequent imports, we can use hashes to compare. As a precautionary measure, the timestamp is updated to the current time as well. + + Args: + path (str): Path to the file. + force (bool, optional): Load the file without checking any conditions. Defaults to False. + data_import (bool, optional): [description]. Defaults to False. + pre_process ([type], optional): Any preprocesing that may need to take place on the doc. Defaults to None. + ignore_version (bool, optional): ignore current version. Defaults to None. + reset_permissions (bool, optional): reset permissions for the file. Defaults to False. + + Returns: + [bool]: True if import takes place. False if it wasn't imported. + """ frappe.flags.dt = frappe.flags.dt or [] try: docs = read_doc_from_file(path)