refactor: use a database file directly, instead of an SQL file

Signed-off-by: Akhil Narang <me@akhilnarang.dev>

frappe/database/db_manager.py

@@ -78,11 +78,12 @@ class DbManager:
 		else:
 			command.extend(["cat", source, "|"])
 
-		# Newer versions of MariaDB add in a line that'll break on older versions, so remove it
-		command.extend(["sed", r"'/\/\*M\{0,1\}!999999\\- enable the sandbox mode \*\//d'", "|"])
+		if frappe.conf.db_type == "mariadb":
+			# Newer versions of MariaDB add in a line that'll break on older versions, so remove it
+			command.extend(["sed", r"'/\/\*M\{0,1\}!999999\\- enable the sandbox mode \*\//d'", "|"])
 
-		# Remove view security definers
-		command.extend(["sed", r"'/\/\*![0-9]* DEFINER=[^ ]* SQL SECURITY DEFINER \*\//d'", "|"])
+			# Remove view security definers
+			command.extend(["sed", r"'/\/\*![0-9]* DEFINER=[^ ]* SQL SECURITY DEFINER \*\//d'", "|"])
 
 		# Generate the restore command
 		bin, args, bin_name = get_command(

frappe/database/sqlite/framework_sqlite.sql

@@ -1,320 +0,0 @@
--- Core Elements to install WNFramework
--- To be called from install.py
-
---
--- Table structure for table `tabDocField`
---
-
-DROP TABLE IF EXISTS `tabDocField`;
-CREATE TABLE `tabDocField` (
-  `name` TEXT NOT NULL,
-  `creation` TEXT DEFAULT NULL,
-  `modified` TEXT DEFAULT NULL,
-  `modified_by` TEXT DEFAULT NULL,
-  `owner` TEXT DEFAULT NULL,
-  `docstatus` INTEGER NOT NULL DEFAULT 0,
-  `parent` TEXT DEFAULT NULL,
-  `parentfield` TEXT DEFAULT NULL,
-  `parenttype` TEXT DEFAULT NULL,
-  `idx` INTEGER NOT NULL DEFAULT 0,
-  `fieldname` TEXT DEFAULT NULL,
-  `label` TEXT DEFAULT NULL,
-  `oldfieldname` TEXT DEFAULT NULL,
-  `fieldtype` TEXT DEFAULT NULL,
-  `oldfieldtype` TEXT DEFAULT NULL,
-  `options` TEXT,
-  `search_index` INTEGER NOT NULL DEFAULT 0,
-  `show_dashboard` INTEGER NOT NULL DEFAULT 0,
-  `hidden` INTEGER NOT NULL DEFAULT 0,
-  `set_only_once` INTEGER NOT NULL DEFAULT 0,
-  `allow_in_quick_entry` INTEGER NOT NULL DEFAULT 0,
-  `print_hide` INTEGER NOT NULL DEFAULT 0,
-  `report_hide` INTEGER NOT NULL DEFAULT 0,
-  `reqd` INTEGER NOT NULL DEFAULT 0,
-  `bold` INTEGER NOT NULL DEFAULT 0,
-  `in_global_search` INTEGER NOT NULL DEFAULT 0,
-  `collapsible` INTEGER NOT NULL DEFAULT 0,
-  `unique` INTEGER NOT NULL DEFAULT 0,
-  `no_copy` INTEGER NOT NULL DEFAULT 0,
-  `allow_on_submit` INTEGER NOT NULL DEFAULT 0,
-  `show_preview_popup` INTEGER NOT NULL DEFAULT 0,
-  `trigger` TEXT DEFAULT NULL,
-  `collapsible_depends_on` TEXT,
-  `mandatory_depends_on` TEXT,
-  `read_only_depends_on` TEXT,
-  `depends_on` TEXT,
-  `permlevel` INTEGER NOT NULL DEFAULT 0,
-  `ignore_user_permissions` INTEGER NOT NULL DEFAULT 0,
-  `width` TEXT DEFAULT NULL,
-  `print_width` TEXT DEFAULT NULL,
-  `columns` INTEGER NOT NULL DEFAULT 0,
-  `default` TEXT,
-  `description` TEXT,
-  `in_list_view` INTEGER NOT NULL DEFAULT 0,
-  `fetch_if_empty` INTEGER NOT NULL DEFAULT 0,
-  `in_filter` INTEGER NOT NULL DEFAULT 0,
-  `remember_last_selected_value` INTEGER NOT NULL DEFAULT 0,
-  `ignore_xss_filter` INTEGER NOT NULL DEFAULT 0,
-  `print_hide_if_no_value` INTEGER NOT NULL DEFAULT 0,
-  `allow_bulk_edit` INTEGER NOT NULL DEFAULT 0,
-  `in_standard_filter` INTEGER NOT NULL DEFAULT 0,
-  `in_preview` INTEGER NOT NULL DEFAULT 0,
-  `read_only` INTEGER NOT NULL DEFAULT 0,
-  `precision` TEXT DEFAULT NULL,
-  `max_height` TEXT DEFAULT NULL,
-  `length` INTEGER NOT NULL DEFAULT 0,
-  `translatable` INTEGER NOT NULL DEFAULT 0,
-  `hide_border` INTEGER NOT NULL DEFAULT 0,
-  `hide_days` INTEGER NOT NULL DEFAULT 0,
-  `hide_seconds` INTEGER NOT NULL DEFAULT 0,
-  PRIMARY KEY (`name`)
-);
-
---
--- Table structure for table `tabDocPerm`
---
-
-DROP TABLE IF EXISTS `tabDocPerm`;
-CREATE TABLE `tabDocPerm` (
-  `name` TEXT NOT NULL,
-  `creation` TEXT DEFAULT NULL,
-  `modified` TEXT DEFAULT NULL,
-  `modified_by` TEXT DEFAULT NULL,
-  `owner` TEXT DEFAULT NULL,
-  `docstatus` INTEGER NOT NULL DEFAULT 0,
-  `parent` TEXT DEFAULT NULL,
-  `parentfield` TEXT DEFAULT NULL,
-  `parenttype` TEXT DEFAULT NULL,
-  `idx` INTEGER NOT NULL DEFAULT 0,
-  `permlevel` INTEGER DEFAULT 0,
-  `role` TEXT DEFAULT NULL,
-  `match` TEXT DEFAULT NULL,
-  `read` INTEGER NOT NULL DEFAULT 1,
-  `write` INTEGER NOT NULL DEFAULT 1,
-  `create` INTEGER NOT NULL DEFAULT 1,
-  `submit` INTEGER NOT NULL DEFAULT 0,
-  `cancel` INTEGER NOT NULL DEFAULT 0,
-  `delete` INTEGER NOT NULL DEFAULT 1,
-  `amend` INTEGER NOT NULL DEFAULT 0,
-  `report` INTEGER NOT NULL DEFAULT 1,
-  `export` INTEGER NOT NULL DEFAULT 1,
-  `import` INTEGER NOT NULL DEFAULT 0,
-  `share` INTEGER NOT NULL DEFAULT 1,
-  `print` INTEGER NOT NULL DEFAULT 1,
-  `email` INTEGER NOT NULL DEFAULT 1,
-  PRIMARY KEY (`name`)
-);
-
---
--- Table structure for table `tabDocType Action`
---
-
-DROP TABLE IF EXISTS `tabDocType Action`;
-CREATE TABLE `tabDocType Action` (
-  `name` TEXT NOT NULL,
-  `creation` TEXT DEFAULT NULL,
-  `modified` TEXT DEFAULT NULL,
-  `modified_by` TEXT DEFAULT NULL,
-  `owner` TEXT DEFAULT NULL,
-  `docstatus` INTEGER NOT NULL DEFAULT 0,
-  `parent` TEXT DEFAULT NULL,
-  `parentfield` TEXT DEFAULT NULL,
-  `parenttype` TEXT DEFAULT NULL,
-  `idx` INTEGER NOT NULL DEFAULT 0,
-  `label` TEXT DEFAULT NULL,
-  `group` TEXT DEFAULT NULL,
-  `action_type` TEXT DEFAULT NULL,
-  `action` TEXT DEFAULT NULL,
-  PRIMARY KEY (`name`)
-);
-
---
--- Table structure for table `tabDocType Link`
---
-
-DROP TABLE IF EXISTS `tabDocType Link`;
-CREATE TABLE `tabDocType Link` (
-  `name` TEXT NOT NULL,
-  `creation` TEXT DEFAULT NULL,
-  `modified` TEXT DEFAULT NULL,
-  `modified_by` TEXT DEFAULT NULL,
-  `owner` TEXT DEFAULT NULL,
-  `docstatus` INTEGER NOT NULL DEFAULT 0,
-  `parent` TEXT DEFAULT NULL,
-  `parentfield` TEXT DEFAULT NULL,
-  `parenttype` TEXT DEFAULT NULL,
-  `idx` INTEGER NOT NULL DEFAULT 0,
-  `group` TEXT DEFAULT NULL,
-  `link_doctype` TEXT DEFAULT NULL,
-  `link_fieldname` TEXT DEFAULT NULL,
-  PRIMARY KEY (`name`)
-);
-
---
--- Table structure for table `tabDocType`
---
-
-DROP TABLE IF EXISTS `tabDocType`;
-CREATE TABLE `tabDocType` (
-  `name` TEXT NOT NULL,
-  `creation` TEXT DEFAULT NULL,
-  `modified` TEXT DEFAULT NULL,
-  `modified_by` TEXT DEFAULT NULL,
-  `owner` TEXT DEFAULT NULL,
-  `docstatus` INTEGER NOT NULL DEFAULT 0,
-  `idx` INTEGER NOT NULL DEFAULT 0,
-  `search_fields` TEXT DEFAULT NULL,
-  `issingle` INTEGER NOT NULL DEFAULT 0,
-  `is_virtual` INTEGER NOT NULL DEFAULT 0,
-  `is_tree` INTEGER NOT NULL DEFAULT 0,
-  `istable` INTEGER NOT NULL DEFAULT 0,
-  `editable_grid` INTEGER NOT NULL DEFAULT 1,
-  `track_changes` INTEGER NOT NULL DEFAULT 0,
-  `module` TEXT DEFAULT NULL,
-  `restrict_to_domain` TEXT DEFAULT NULL,
-  `app` TEXT DEFAULT NULL,
-  `autoname` TEXT DEFAULT NULL,
-  `naming_rule` TEXT DEFAULT NULL,
-  `title_field` TEXT DEFAULT NULL,
-  `image_field` TEXT DEFAULT NULL,
-  `timeline_field` TEXT DEFAULT NULL,
-  `sort_field` TEXT DEFAULT NULL,
-  `sort_order` TEXT DEFAULT NULL,
-  `description` TEXT,
-  `colour` TEXT DEFAULT NULL,
-  `read_only` INTEGER NOT NULL DEFAULT 0,
-  `in_create` INTEGER NOT NULL DEFAULT 0,
-  `menu_index` INTEGER DEFAULT NULL,
-  `parent_node` TEXT DEFAULT NULL,
-  `smallicon` TEXT DEFAULT NULL,
-  `allow_copy` INTEGER NOT NULL DEFAULT 0,
-  `allow_rename` INTEGER NOT NULL DEFAULT 0,
-  `allow_import` INTEGER NOT NULL DEFAULT 0,
-  `hide_toolbar` INTEGER NOT NULL DEFAULT 0,
-  `track_seen` INTEGER NOT NULL DEFAULT 0,
-  `max_attachments` INTEGER NOT NULL DEFAULT 0,
-  `print_outline` TEXT DEFAULT NULL,
-  `document_type` TEXT DEFAULT NULL,
-  `icon` TEXT DEFAULT NULL,
-  `color` TEXT DEFAULT NULL,
-  `tag_fields` TEXT DEFAULT NULL,
-  `subject` TEXT DEFAULT NULL,
-  `_last_update` TEXT DEFAULT NULL,
-  `engine` TEXT DEFAULT 'InnoDB',
-  `default_print_format` TEXT DEFAULT NULL,
-  `is_submittable` INTEGER NOT NULL DEFAULT 0,
-  `show_name_in_global_search` INTEGER NOT NULL DEFAULT 0,
-  `_user_tags` TEXT DEFAULT NULL,
-  `custom` INTEGER NOT NULL DEFAULT 0,
-  `beta` INTEGER NOT NULL DEFAULT 0,
-  `has_web_view` INTEGER NOT NULL DEFAULT 0,
-  `allow_guest_to_view` INTEGER NOT NULL DEFAULT 0,
-  `route` TEXT DEFAULT NULL,
-  `is_published_field` TEXT DEFAULT NULL,
-  `website_search_field` TEXT DEFAULT NULL,
-  `email_append_to` INTEGER NOT NULL DEFAULT 0,
-  `subject_field` TEXT DEFAULT NULL,
-  `sender_field` TEXT DEFAULT NULL,
-  `show_title_field_in_link` INTEGER NOT NULL DEFAULT 0,
-  `migration_hash` TEXT DEFAULT NULL,
-  `translated_doctype` INTEGER NOT NULL DEFAULT 0,
-  PRIMARY KEY (`name`)
-);
-
---
--- Table structure for table `tabSeries`
---
-
-DROP TABLE IF EXISTS `tabSeries`;
-CREATE TABLE `tabSeries` (
-  `name` TEXT NOT NULL,
-  `current` INTEGER NOT NULL DEFAULT 0,
-  PRIMARY KEY(`name`)
-);
-
---
--- Table structure for table `tabSessions`
---
-
-DROP TABLE IF EXISTS `tabSessions`;
-CREATE TABLE `tabSessions` (
-  `user` TEXT DEFAULT NULL,
-  `sid` TEXT DEFAULT NULL,
-  `sessiondata` TEXT,
-  `ipaddress` TEXT DEFAULT NULL,
-  `lastupdate` TEXT DEFAULT NULL,
-  `status` TEXT DEFAULT NULL,
-  PRIMARY KEY (`sid`)
-);
-
---
--- Table structure for table `tabSingles`
---
-
-DROP TABLE IF EXISTS `tabSingles`;
-CREATE TABLE `tabSingles` (
-  `doctype` TEXT DEFAULT NULL,
-  `field` TEXT DEFAULT NULL,
-  `value` TEXT,
-  PRIMARY KEY (`doctype`, `field`)
-);
-
---
--- Table structure for table `__Auth`
---
-
-DROP TABLE IF EXISTS `__Auth`;
-CREATE TABLE `__Auth` (
-  `doctype` TEXT NOT NULL,
-  `name` TEXT NOT NULL,
-  `fieldname` TEXT NOT NULL,
-  `password` TEXT NOT NULL,
-  `encrypted` INTEGER NOT NULL DEFAULT 0,
-  PRIMARY KEY (`doctype`, `name`, `fieldname`)
-);
-
---
--- Table structure for table `tabFile`
---
-
-DROP TABLE IF EXISTS `tabFile`;
-CREATE TABLE `tabFile` (
-  `name` TEXT NOT NULL,
-  `creation` TEXT DEFAULT NULL,
-  `modified` TEXT DEFAULT NULL,
-  `modified_by` TEXT DEFAULT NULL,
-  `owner` TEXT DEFAULT NULL,
-  `docstatus` INTEGER NOT NULL DEFAULT 0,
-  `parent` TEXT DEFAULT NULL,
-  `parentfield` TEXT DEFAULT NULL,
-  `parenttype` TEXT DEFAULT NULL,
-  `idx` INTEGER NOT NULL DEFAULT 0,
-  `file_name` TEXT DEFAULT NULL,
-  `file_url` TEXT DEFAULT NULL,
-  `module` TEXT DEFAULT NULL,
-  `attached_to_name` TEXT DEFAULT NULL,
-  `file_size` INTEGER NOT NULL DEFAULT 0,
-  `attached_to_doctype` TEXT DEFAULT NULL,
-  PRIMARY KEY (`name`)
-);
-
---
--- Table structure for table `tabDefaultValue`
---
-
-DROP TABLE IF EXISTS `tabDefaultValue`;
-CREATE TABLE `tabDefaultValue` (
-  `name` TEXT NOT NULL,
-  `creation` TEXT DEFAULT NULL,
-  `modified` TEXT DEFAULT NULL,
-  `modified_by` TEXT DEFAULT NULL,
-  `owner` TEXT DEFAULT NULL,
-  `docstatus` INTEGER NOT NULL DEFAULT 0,
-  `parent` TEXT DEFAULT NULL,
-  `parentfield` TEXT DEFAULT NULL,
-  `parenttype` TEXT DEFAULT NULL,
-  `idx` INTEGER NOT NULL DEFAULT 0,
-  `defvalue` TEXT,
-  `defkey` TEXT DEFAULT NULL,
-  PRIMARY KEY (`name`)
-);

frappe/database/sqlite/setup_db.py

@@ -1,4 +1,6 @@
 import os
+import shutil
+import sys
 from pathlib import Path
 
 import click
@@ -18,10 +20,10 @@ def setup_database(force, verbose):
 	root_conn.close()
 
 
-def bootstrap_database(verbose, source_sql=None):
+def bootstrap_database(verbose, source_db=None):
 	import sys
 
-	import_db_from_sql(source_sql, verbose)
+	copy_db(source_db, verbose)
 
 	frappe.connect()
 	if "tabDefaultValue" not in frappe.db.get_tables(cached=False):
@@ -36,15 +38,25 @@ def bootstrap_database(verbose, source_sql=None):
 		sys.exit(1)
 
 
-def import_db_from_sql(source_sql=None, verbose=False):
+def copy_db(db_file=None, verbose=False):
 	if verbose:
 		print("Starting database import...")
 	db_name = frappe.conf.db_name
-	if not source_sql:
-		source_sql = os.path.join(os.path.dirname(__file__), "framework_sqlite.sql")
-	DbManager().restore_database(verbose, db_name, source_sql, frappe.conf.db_user, frappe.conf.db_password)
+	if not db_file:
+		db_file = os.path.join(os.path.dirname(__file__), "framework_sqlite.db")
+	db_path = Path(db_file)
+	if not db_path.exists():
+		click.secho(f"Database {db_path.as_posix()} does not exist", fg="red")
+		sys.exit(1)
+
+	destination_db_folder = Path(frappe.get_site_path()) / "db"
+	if not destination_db_folder.exists():
+		destination_db_folder.mkdir()
+	destination_db_path = destination_db_folder / f"{db_name}.db"
+
+	shutil.copy(db_path, destination_db_path)
 	if verbose:
-		print("Imported from database {}".format(source_sql))
+		print("Imported from database {}".format(db_path))
 
 
 def drop_database(db_name: str):

frappe/utils/backups.py

@@ -376,6 +376,18 @@ class BackupGenerator:
 			n.write(c.read())
 
 	def take_dump(self):
+		if self.db_type == "sqlite":
+			from pathlib import Path
+
+			import frappe
+
+			db_path = Path(frappe.get_site_path()) / "db" / f"{self.db_name}.db"
+			command = f"gzip -k {db_path} -c > {self.backup_path_db}"
+
+			frappe.utils.execute_in_shell(command, low_priority=True, check_exit_code=True)
+
+			return
+
 		import shlex
 
 		import frappe.utils
@@ -431,12 +443,6 @@ class BackupGenerator:
 			elif self.backup_excludes:
 				extra.extend([f"--ignore-table={self.db_name}.{table}" for table in self.backup_excludes])
 
-		elif self.db_type == "sqlite":
-			if self.backup_includes:
-				extra.extend([f'"{table}"' for table in self.backup_includes])
-			elif self.backup_excludes:
-				click.secho("Excluding tables is not supported for SQLite", fg="yellow")
-
 		elif self.db_type == "postgres":
 			if self.backup_includes:
 				extra.extend([f'--table=public."{table}"' for table in self.backup_includes])