Merge pull request #22915 from akhilnarang/dont-extract-backup-unconditionally
perf: don't extract gzipped backups
This commit is contained in:
Ankush Menat
GitHub
commit
2dfef73ae8
6 changed files with 301 additions and 327 deletions
|
|
@ -72,13 +72,10 @@ def new_site(
|
|||
setup_db=True,
|
||||
):
|
||||
"Create a new site"
|
||||
from frappe.installer import _new_site, extract_sql_from_archive
|
||||
from frappe.installer import _new_site
|
||||
|
||||
frappe.init(site=site, new_site=True)
|
||||
|
||||
if source_sql:
|
||||
source_sql = extract_sql_from_archive(source_sql)
|
||||
|
||||
_new_site(
|
||||
db_name,
|
||||
site,
|
||||
|
|
@ -180,75 +177,113 @@ def _restore(
|
|||
with_public_files=None,
|
||||
with_private_files=None,
|
||||
):
|
||||
from frappe.installer import extract_files
|
||||
from frappe.utils.backups import decrypt_backup, get_or_generate_backup_encryption_key
|
||||
|
||||
from frappe.installer import (
|
||||
_new_site,
|
||||
extract_files,
|
||||
extract_sql_from_archive,
|
||||
is_downgrade,
|
||||
is_partial,
|
||||
validate_database_sql,
|
||||
)
|
||||
from frappe.utils.backups import Backup, get_or_generate_backup_encryption_key
|
||||
err, out = frappe.utils.execute_in_shell(f"file {sql_file_path}", check_exit_code=True)
|
||||
if err:
|
||||
click.secho("Failed to detect type of backup file", fg="red")
|
||||
sys.exit(1)
|
||||
|
||||
_backup = Backup(sql_file_path)
|
||||
|
||||
try:
|
||||
decompressed_file_name = extract_sql_from_archive(sql_file_path)
|
||||
if is_partial(decompressed_file_name):
|
||||
click.secho(
|
||||
"Partial Backup file detected. You cannot use a partial file to restore a Frappe site.",
|
||||
fg="red",
|
||||
)
|
||||
click.secho(
|
||||
"Use `bench partial-restore` to restore a partial backup to an existing site.",
|
||||
fg="yellow",
|
||||
)
|
||||
_backup.decryption_rollback()
|
||||
sys.exit(1)
|
||||
|
||||
except UnicodeDecodeError:
|
||||
_backup.decryption_rollback()
|
||||
if "cipher" in out.decode().split(":")[-1].strip():
|
||||
if encryption_key:
|
||||
click.secho("Encrypted backup file detected. Decrypting using provided key.", fg="yellow")
|
||||
_backup.backup_decryption(encryption_key)
|
||||
|
||||
else:
|
||||
click.secho("Encrypted backup file detected. Decrypting using site config.", fg="yellow")
|
||||
encryption_key = get_or_generate_backup_encryption_key()
|
||||
_backup.backup_decryption(encryption_key)
|
||||
|
||||
# Rollback on unsuccessful decryrption
|
||||
if not os.path.exists(sql_file_path):
|
||||
click.secho("Decryption failed. Please provide a valid key and try again.", fg="red")
|
||||
with decrypt_backup(sql_file_path, encryption_key):
|
||||
# Rollback on unsuccessful decryption
|
||||
if not os.path.exists(sql_file_path):
|
||||
click.secho("Decryption failed. Please provide a valid key and try again.", fg="red")
|
||||
sys.exit(1)
|
||||
|
||||
_backup.decryption_rollback()
|
||||
sys.exit(1)
|
||||
|
||||
decompressed_file_name = extract_sql_from_archive(sql_file_path)
|
||||
|
||||
if is_partial(decompressed_file_name):
|
||||
click.secho(
|
||||
"Partial Backup file detected. You cannot use a partial file to restore a Frappe site.",
|
||||
fg="red",
|
||||
restore_backup(
|
||||
sql_file_path,
|
||||
site,
|
||||
db_root_username,
|
||||
db_root_password,
|
||||
verbose,
|
||||
install_app,
|
||||
admin_password,
|
||||
force,
|
||||
)
|
||||
click.secho(
|
||||
"Use `bench partial-restore` to restore a partial backup to an existing site.",
|
||||
fg="yellow",
|
||||
)
|
||||
_backup.decryption_rollback()
|
||||
sys.exit(1)
|
||||
else:
|
||||
restore_backup(
|
||||
sql_file_path,
|
||||
site,
|
||||
db_root_username,
|
||||
db_root_password,
|
||||
verbose,
|
||||
install_app,
|
||||
admin_password,
|
||||
force,
|
||||
)
|
||||
|
||||
validate_database_sql(decompressed_file_name, _raise=not force)
|
||||
# Extract public and/or private files to the restored site, if user has given the path
|
||||
if with_public_files:
|
||||
# Decrypt data if there is a Key
|
||||
if encryption_key:
|
||||
with decrypt_backup(with_public_files, encryption_key):
|
||||
public = extract_files(site, with_public_files)
|
||||
else:
|
||||
public = extract_files(site, with_public_files)
|
||||
|
||||
# dont allow downgrading to older versions of frappe without force
|
||||
if not force and is_downgrade(decompressed_file_name, verbose=True):
|
||||
# Removing temporarily created file
|
||||
os.remove(public)
|
||||
|
||||
if with_private_files:
|
||||
# Decrypt data if there is a Key
|
||||
if encryption_key:
|
||||
with decrypt_backup(with_private_files, encryption_key):
|
||||
private = extract_files(site, with_private_files)
|
||||
else:
|
||||
private = extract_files(site, with_private_files)
|
||||
|
||||
# Removing temporarily created file
|
||||
os.remove(private)
|
||||
|
||||
success_message = "Site {} has been restored{}".format(
|
||||
site, " with files" if (with_public_files or with_private_files) else ""
|
||||
)
|
||||
click.secho(success_message, fg="green")
|
||||
|
||||
|
||||
def restore_backup(
|
||||
sql_file_path: str,
|
||||
site,
|
||||
db_root_username,
|
||||
db_root_password,
|
||||
verbose,
|
||||
install_app,
|
||||
admin_password,
|
||||
force,
|
||||
):
|
||||
from frappe.installer import _new_site, is_downgrade, is_partial, validate_database_sql
|
||||
|
||||
if is_partial(sql_file_path):
|
||||
click.secho(
|
||||
"Partial Backup file detected. You cannot use a partial file to restore a Frappe site.",
|
||||
fg="red",
|
||||
)
|
||||
click.secho(
|
||||
"Use `bench partial-restore` to restore a partial backup to an existing site.",
|
||||
fg="yellow",
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
# Check if the backup is of an older version of frappe and the user hasn't specified force
|
||||
if is_downgrade(sql_file_path, verbose=True) and not force:
|
||||
warn_message = (
|
||||
"This is not recommended and may lead to unexpected behaviour. "
|
||||
"Do you want to continue anyway?"
|
||||
)
|
||||
click.confirm(warn_message, abort=True)
|
||||
|
||||
# Validate the sql file
|
||||
validate_database_sql(sql_file_path, _raise=not force)
|
||||
|
||||
try:
|
||||
_new_site(
|
||||
frappe.conf.db_name,
|
||||
|
|
@ -258,53 +293,15 @@ def _restore(
|
|||
admin_password=admin_password,
|
||||
verbose=verbose,
|
||||
install_apps=install_app,
|
||||
source_sql=decompressed_file_name,
|
||||
source_sql=sql_file_path,
|
||||
force=True,
|
||||
db_type=frappe.conf.db_type,
|
||||
)
|
||||
|
||||
except Exception as err:
|
||||
print(err.args[1])
|
||||
_backup.decryption_rollback()
|
||||
sys.exit(1)
|
||||
|
||||
# Removing temporarily created file
|
||||
if decompressed_file_name != sql_file_path:
|
||||
os.remove(decompressed_file_name)
|
||||
_backup.decryption_rollback()
|
||||
|
||||
# Extract public and/or private files to the restored site, if user has given the path
|
||||
if with_public_files:
|
||||
# Decrypt data if there is a Key
|
||||
if encryption_key:
|
||||
_backup = Backup(with_public_files)
|
||||
_backup.backup_decryption(encryption_key)
|
||||
if not os.path.exists(with_public_files):
|
||||
_backup.decryption_rollback()
|
||||
public = extract_files(site, with_public_files)
|
||||
|
||||
# Removing temporarily created file
|
||||
os.remove(public)
|
||||
_backup.decryption_rollback()
|
||||
|
||||
if with_private_files:
|
||||
# Decrypt data if there is a Key
|
||||
if encryption_key:
|
||||
_backup = Backup(with_private_files)
|
||||
_backup.backup_decryption(encryption_key)
|
||||
if not os.path.exists(with_private_files):
|
||||
_backup.decryption_rollback()
|
||||
private = extract_files(site, with_private_files)
|
||||
|
||||
# Removing temporarily created file
|
||||
os.remove(private)
|
||||
_backup.decryption_rollback()
|
||||
|
||||
success_message = "Site {} has been restored{}".format(
|
||||
site, " with files" if (with_public_files or with_private_files) else ""
|
||||
)
|
||||
click.secho(success_message, fg="green")
|
||||
|
||||
|
||||
@click.command("partial-restore")
|
||||
@click.argument("sql-file-path")
|
||||
|
|
@ -312,38 +309,23 @@ def _restore(
|
|||
@click.option("--encryption-key", help="Backup encryption key")
|
||||
@pass_context
|
||||
def partial_restore(context, sql_file_path, verbose, encryption_key=None):
|
||||
from frappe.installer import extract_sql_from_archive, partial_restore
|
||||
from frappe.utils.backups import Backup, get_or_generate_backup_encryption_key
|
||||
from frappe.installer import is_partial, partial_restore
|
||||
from frappe.utils.backups import decrypt_backup, get_or_generate_backup_encryption_key
|
||||
|
||||
if not os.path.exists(sql_file_path):
|
||||
print("Invalid path", sql_file_path)
|
||||
sys.exit(1)
|
||||
|
||||
site = get_site(context)
|
||||
frappe.init(site=site)
|
||||
|
||||
_backup = Backup(sql_file_path)
|
||||
|
||||
verbose = context.verbose or verbose
|
||||
|
||||
frappe.init(site=site)
|
||||
frappe.connect(site=site)
|
||||
try:
|
||||
decompressed_file_name = extract_sql_from_archive(sql_file_path)
|
||||
err, out = frappe.utils.execute_in_shell(f"file {sql_file_path}", check_exit_code=True)
|
||||
if err:
|
||||
click.secho("Failed to detect type of backup file", fg="red")
|
||||
sys.exit(1)
|
||||
|
||||
with open(decompressed_file_name) as f:
|
||||
header = " ".join(f.readline() for _ in range(5))
|
||||
|
||||
# Check for full backup file
|
||||
if "Partial Backup" not in header:
|
||||
click.secho(
|
||||
"Full backup file detected.Use `bench restore` to restore a Frappe Site.",
|
||||
fg="red",
|
||||
)
|
||||
_backup.decryption_rollback()
|
||||
sys.exit(1)
|
||||
|
||||
except UnicodeDecodeError:
|
||||
_backup.decryption_rollback()
|
||||
if "cipher" in out.decode().split(":")[-1].strip():
|
||||
if encryption_key:
|
||||
click.secho("Encrypted backup file detected. Decrypting using provided key.", fg="yellow")
|
||||
key = encryption_key
|
||||
|
|
@ -352,35 +334,30 @@ def partial_restore(context, sql_file_path, verbose, encryption_key=None):
|
|||
click.secho("Encrypted backup file detected. Decrypting using site config.", fg="yellow")
|
||||
key = get_or_generate_backup_encryption_key()
|
||||
|
||||
_backup.backup_decryption(key)
|
||||
|
||||
# Rollback on unsuccessful decryrption
|
||||
if not os.path.exists(sql_file_path):
|
||||
click.secho("Decryption failed. Please provide a valid key and try again.", fg="red")
|
||||
_backup.decryption_rollback()
|
||||
sys.exit(1)
|
||||
|
||||
decompressed_file_name = extract_sql_from_archive(sql_file_path)
|
||||
|
||||
with open(decompressed_file_name) as f:
|
||||
header = " ".join(f.readline() for _ in range(5))
|
||||
|
||||
# Check for Full backup file.
|
||||
if "Partial Backup" not in header:
|
||||
with decrypt_backup(sql_file_path, key):
|
||||
if not is_partial(sql_file_path):
|
||||
click.secho(
|
||||
"Full Backup file detected.Use `bench restore` to restore a Frappe Site.",
|
||||
"Full backup file detected.Use `bench restore` to restore a Frappe Site.",
|
||||
fg="red",
|
||||
)
|
||||
_backup.decryption_rollback()
|
||||
sys.exit(1)
|
||||
|
||||
partial_restore(sql_file_path, verbose)
|
||||
partial_restore(sql_file_path, verbose)
|
||||
|
||||
# Removing temporarily created file
|
||||
_backup.decryption_rollback()
|
||||
if os.path.exists(sql_file_path.rstrip(".gz")):
|
||||
os.remove(sql_file_path.rstrip(".gz"))
|
||||
# Rollback on unsuccessful decryption
|
||||
if not os.path.exists(sql_file_path):
|
||||
click.secho("Decryption failed. Please provide a valid key and try again.", fg="red")
|
||||
sys.exit(1)
|
||||
|
||||
else:
|
||||
if not is_partial(sql_file_path):
|
||||
click.secho(
|
||||
"Full backup file detected.Use `bench restore` to restore a Frappe Site.",
|
||||
fg="red",
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
partial_restore(sql_file_path, verbose)
|
||||
frappe.destroy()
|
||||
|
||||
|
||||
|
|
@ -865,6 +842,9 @@ def use(site, sites_path="."):
|
|||
)
|
||||
@click.option("--verbose", default=False, is_flag=True, help="Add verbosity")
|
||||
@click.option("--compress", default=False, is_flag=True, help="Compress private and public files")
|
||||
@click.option(
|
||||
"--old-backup-metadata", default=False, is_flag=True, help="Use older backup metadata"
|
||||
)
|
||||
@pass_context
|
||||
def backup(
|
||||
context,
|
||||
|
|
@ -879,6 +859,7 @@ def backup(
|
|||
compress=False,
|
||||
include="",
|
||||
exclude="",
|
||||
old_backup_metadata=False,
|
||||
):
|
||||
"Backup"
|
||||
|
||||
|
|
@ -904,6 +885,7 @@ def backup(
|
|||
compress=compress,
|
||||
verbose=verbose,
|
||||
force=True,
|
||||
old_backup_metadata=old_backup_metadata,
|
||||
)
|
||||
except Exception:
|
||||
click.secho(
|
||||
|
|
|
|||
|
|
@ -57,14 +57,15 @@ class DbManager:
|
|||
from frappe.database import get_command
|
||||
from frappe.utils import execute_in_shell
|
||||
|
||||
pv = which("pv")
|
||||
|
||||
command = []
|
||||
|
||||
if pv:
|
||||
command.extend([pv, source, "|"])
|
||||
source = []
|
||||
print("Restoring Database file...")
|
||||
if source.endswith(".gz"):
|
||||
if gzip := which("gzip"):
|
||||
command.extend([gzip, "-cd", source, "|"])
|
||||
source = []
|
||||
else:
|
||||
raise Exception("`gzip` not installed")
|
||||
|
||||
else:
|
||||
source = ["<", source]
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import os
|
||||
|
||||
import frappe
|
||||
from frappe import _
|
||||
from frappe.database.db_manager import DbManager
|
||||
|
||||
|
||||
def setup_database():
|
||||
|
|
@ -36,45 +36,16 @@ def bootstrap_database(db_name, verbose, source_sql=None):
|
|||
|
||||
|
||||
def import_db_from_sql(source_sql=None, verbose=False):
|
||||
import shlex
|
||||
from shutil import which
|
||||
|
||||
from frappe.database import get_command
|
||||
from frappe.utils import execute_in_shell
|
||||
|
||||
# bootstrap db
|
||||
if verbose:
|
||||
print("Starting database import...")
|
||||
db_name = frappe.conf.db_name
|
||||
if not source_sql:
|
||||
source_sql = os.path.join(os.path.dirname(__file__), "framework_postgres.sql")
|
||||
|
||||
pv = which("pv")
|
||||
|
||||
command = []
|
||||
|
||||
if pv:
|
||||
command.extend([pv, source_sql, "|"])
|
||||
source = []
|
||||
print("Restoring Database file...")
|
||||
else:
|
||||
source = ["-f", source_sql]
|
||||
|
||||
bin, args, bin_name = get_command(
|
||||
host=frappe.conf.db_host,
|
||||
port=frappe.conf.db_port,
|
||||
user=frappe.conf.db_name,
|
||||
password=frappe.conf.db_password,
|
||||
db_name=frappe.conf.db_name,
|
||||
DbManager(frappe.local.db).restore_database(
|
||||
verbose, db_name, source_sql, db_name, frappe.conf.db_password
|
||||
)
|
||||
|
||||
if not bin:
|
||||
frappe.throw(
|
||||
_("{} not found in PATH! This is required to restore the database.").format(bin_name),
|
||||
exc=frappe.ExecutableNotFound,
|
||||
)
|
||||
command.append(bin)
|
||||
command.append(shlex.join(args))
|
||||
command.extend(source)
|
||||
execute_in_shell(" ".join(command), check_exit_code=True, verbose=verbose)
|
||||
frappe.cache.delete_keys("") # Delete all keys associated with this site.
|
||||
if verbose:
|
||||
print("Imported from database %s" % source_sql)
|
||||
|
||||
|
||||
def get_root_connection(root_login=None, root_password=None):
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors
|
||||
# License: MIT. See LICENSE
|
||||
|
||||
import configparser
|
||||
import gzip
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
|
|
@ -52,7 +53,7 @@ def _new_site(
|
|||
):
|
||||
"""Install a new Frappe site"""
|
||||
|
||||
from frappe.utils import get_site_path, scheduler, touch_file
|
||||
from frappe.utils import scheduler
|
||||
|
||||
if not force and os.path.exists(site):
|
||||
print(f"Site {site} already exists")
|
||||
|
|
@ -449,7 +450,6 @@ def _delete_modules(modules: list[str], dry_run: bool) -> list[str]:
|
|||
def _delete_linked_documents(
|
||||
module_name: str, doctype_linkfield_map: dict[str, str], dry_run: bool
|
||||
) -> None:
|
||||
|
||||
"""Deleted all records linked with module def"""
|
||||
for doctype, fieldname in doctype_linkfield_map.items():
|
||||
for record in frappe.get_all(doctype, filters={fieldname: module_name}, pluck="name"):
|
||||
|
|
@ -664,32 +664,6 @@ def remove_missing_apps():
|
|||
frappe.db.set_global("installed_apps", json.dumps(installed_apps))
|
||||
|
||||
|
||||
def extract_sql_from_archive(sql_file_path):
|
||||
"""Return the path of an SQL file if the passed argument is the path of a gzipped
|
||||
SQL file or an SQL file path. The path may be absolute or relative from the bench
|
||||
root directory or the sites sub-directory.
|
||||
|
||||
Args:
|
||||
sql_file_path (str): Path of the SQL file
|
||||
|
||||
Return:
|
||||
str: Path of the decompressed SQL file
|
||||
"""
|
||||
from frappe.utils import get_bench_relative_path
|
||||
|
||||
sql_file_path = get_bench_relative_path(sql_file_path)
|
||||
# Extract the gzip file if user has passed *.sql.gz file instead of *.sql file
|
||||
if sql_file_path.endswith("sql.gz"):
|
||||
decompressed_file_name = extract_sql_gzip(sql_file_path)
|
||||
else:
|
||||
decompressed_file_name = sql_file_path
|
||||
|
||||
# convert archive sql to latest compatible
|
||||
convert_archive_content(decompressed_file_name)
|
||||
|
||||
return decompressed_file_name
|
||||
|
||||
|
||||
def convert_archive_content(sql_file_path):
|
||||
if frappe.conf.db_type == "mariadb":
|
||||
# ever since mariaDB 10.6, row_format COMPRESSED has been deprecated and removed
|
||||
|
|
@ -723,20 +697,6 @@ def convert_archive_content(sql_file_path):
|
|||
old_sql_file_path.unlink()
|
||||
|
||||
|
||||
def extract_sql_gzip(sql_gz_path):
|
||||
import subprocess
|
||||
|
||||
try:
|
||||
original_file = sql_gz_path
|
||||
decompressed_file = original_file.rstrip(".gz")
|
||||
cmd = f"gzip --decompress --force < {original_file} > {decompressed_file}"
|
||||
subprocess.check_call(cmd, shell=True)
|
||||
except Exception:
|
||||
raise
|
||||
|
||||
return decompressed_file
|
||||
|
||||
|
||||
def _guess_mariadb_version() -> tuple[int] | None:
|
||||
# Using command-line because we *might* not have a connection yet and this command is required
|
||||
# in non-interactive mode.
|
||||
|
|
@ -793,53 +753,58 @@ def is_downgrade(sql_file_path, verbose=False):
|
|||
|
||||
from semantic_version import Version
|
||||
|
||||
head = "INSERT INTO `tabInstalled Application` VALUES"
|
||||
backup_version = extract_version_from_dump(sql_file_path)
|
||||
if backup_version is None:
|
||||
# This is likely an older backup, so try to extract another way
|
||||
header = get_db_dump_header(sql_file_path).split("\n")
|
||||
if "Version" in header[0]:
|
||||
backup_version = header[0].split(":")[-1].strip()
|
||||
|
||||
with open(sql_file_path) as f:
|
||||
for line in f:
|
||||
if head in line:
|
||||
# 'line' (str) format: ('2056588823','2020-05-11 18:21:31.488367','2020-06-12 11:49:31.079506','Administrator','Administrator',0,'Installed Applications','installed_applications','Installed Applications',1,'frappe','v10.1.71-74 (3c50d5e) (v10.x.x)','v10.x.x'),('855c640b8e','2020-05-11 18:21:31.488367','2020-06-12 11:49:31.079506','Administrator','Administrator',0,'Installed Applications','installed_applications','Installed Applications',2,'your_custom_app','0.0.1','master')
|
||||
line = line.strip().lstrip(head).rstrip(";").strip()
|
||||
app_rows = frappe.safe_eval(line)
|
||||
# check if iterable consists of tuples before trying to transform
|
||||
apps_list = (
|
||||
app_rows
|
||||
if all(isinstance(app_row, (tuple, list, set)) for app_row in app_rows)
|
||||
else (app_rows,)
|
||||
)
|
||||
# 'all_apps' (list) format: [('frappe', '12.x.x-develop ()', 'develop'), ('your_custom_app', '0.0.1', 'master')]
|
||||
all_apps = [x[-3:] for x in apps_list]
|
||||
# Assume it's not a downgrade if we can't determine backup version
|
||||
if backup_version is None:
|
||||
return False
|
||||
|
||||
for app in all_apps:
|
||||
app_name = app[0]
|
||||
app_version = app[1].split(" ", 1)[0]
|
||||
current_version = Version(frappe.__version__)
|
||||
downgrade = Version(backup_version) < current_version
|
||||
|
||||
if app_name == "frappe":
|
||||
try:
|
||||
current_version = Version(frappe.__version__)
|
||||
backup_version = Version(app_version[1:] if app_version[0] == "v" else app_version)
|
||||
except ValueError:
|
||||
return False
|
||||
if verbose and downgrade:
|
||||
print(f"Your site will be downgraded from Frappe {current_version} to {backup_version}")
|
||||
|
||||
downgrade = backup_version > current_version
|
||||
|
||||
if verbose and downgrade:
|
||||
print(f"Your site will be downgraded from Frappe {backup_version} to {current_version}")
|
||||
|
||||
return downgrade
|
||||
return downgrade
|
||||
|
||||
|
||||
def is_partial(sql_file_path):
|
||||
with open(sql_file_path) as f:
|
||||
header = " ".join(f.readline() for _ in range(5))
|
||||
if "Partial Backup" in header:
|
||||
return True
|
||||
return False
|
||||
def extract_version_from_dump(sql_file_path: str) -> str | None:
|
||||
"""
|
||||
Extract frappe version from DB dump
|
||||
|
||||
:param sql_file_path: The path to the dump file
|
||||
:return: The frappe version used to create the backup
|
||||
"""
|
||||
header = get_db_dump_header(sql_file_path).split("\n")
|
||||
metadata = ""
|
||||
if "begin frappe metadata" in header[0]:
|
||||
for line in header[1:]:
|
||||
if "end frappe metadata" in line:
|
||||
break
|
||||
metadata += line.replace("--", "").strip() + "\n"
|
||||
parser = configparser.ConfigParser()
|
||||
parser.read_string(metadata)
|
||||
return parser["frappe"]["version"]
|
||||
return None
|
||||
|
||||
|
||||
def is_partial(sql_file_path: str) -> bool:
|
||||
"""
|
||||
Function to return whether the database dump is a partial backup or not
|
||||
|
||||
:param sql_file_path: path to the database dump file
|
||||
:return: True if the database dump is a partial backup, False otherwise
|
||||
"""
|
||||
header = get_db_dump_header(sql_file_path)
|
||||
return "Partial Backup" in header
|
||||
|
||||
|
||||
def partial_restore(sql_file_path, verbose=False):
|
||||
sql_file = extract_sql_from_archive(sql_file_path)
|
||||
|
||||
if frappe.conf.db_type == "mariadb":
|
||||
from frappe.database.mariadb.setup_db import import_db_from_sql
|
||||
elif frappe.conf.db_type == "postgres":
|
||||
|
|
@ -853,43 +818,60 @@ def partial_restore(sql_file_path, verbose=False):
|
|||
fg="yellow",
|
||||
)
|
||||
warnings.warn(warn)
|
||||
else:
|
||||
click.secho("Unsupported database type", fg="red")
|
||||
return
|
||||
|
||||
import_db_from_sql(source_sql=sql_file, verbose=verbose)
|
||||
|
||||
# Removing temporarily created file
|
||||
if sql_file != sql_file_path:
|
||||
os.remove(sql_file)
|
||||
import_db_from_sql(source_sql=sql_file_path, verbose=verbose)
|
||||
|
||||
|
||||
def validate_database_sql(path, _raise=True):
|
||||
"""Check if file has contents and if DefaultValue table exists
|
||||
def validate_database_sql(path: str, _raise: bool = True) -> None:
|
||||
"""Check if file has contents and if `__Auth` table exists
|
||||
|
||||
Args:
|
||||
path (str): Path of the decompressed SQL file
|
||||
_raise (bool, optional): Raise exception if invalid file. Defaults to True.
|
||||
"""
|
||||
empty_file = False
|
||||
missing_table = True
|
||||
|
||||
error_message = ""
|
||||
if path.endswith(".gz"):
|
||||
executable_name = "zgrep"
|
||||
else:
|
||||
executable_name = "grep"
|
||||
|
||||
if not os.path.getsize(path):
|
||||
if os.path.getsize(path):
|
||||
if (executable := which(executable_name)) is None:
|
||||
frappe.throw(
|
||||
f"`{executable_name}` not found in PATH! This is required to take a backup.",
|
||||
exc=frappe.ExecutableNotFound,
|
||||
)
|
||||
try:
|
||||
frappe.utils.execute_in_shell(f"{executable} -m1 __Auth {path}", check_exit_code=True)
|
||||
return
|
||||
except Exception:
|
||||
error_message = "Table `__Auth` not found in file."
|
||||
else:
|
||||
error_message = f"{path} is an empty file!"
|
||||
empty_file = True
|
||||
|
||||
# dont bother checking if empty file
|
||||
if not empty_file:
|
||||
with open(path) as f:
|
||||
for line in f:
|
||||
if "tabDefaultValue" in line:
|
||||
missing_table = False
|
||||
break
|
||||
|
||||
if missing_table:
|
||||
error_message = "Table `tabDefaultValue` not found in file."
|
||||
|
||||
if error_message:
|
||||
click.secho(error_message, fg="red")
|
||||
|
||||
if _raise and (missing_table or empty_file):
|
||||
if _raise:
|
||||
raise frappe.InvalidDatabaseFile
|
||||
|
||||
|
||||
def get_db_dump_header(file_path: str, file_bytes: int = 256) -> str:
|
||||
"""
|
||||
Get the header of a database dump file
|
||||
|
||||
:param file_path: path to the database dump file
|
||||
:param file_bytes: number of bytes to read from the file
|
||||
:return: The first few bytes of the file as requested
|
||||
"""
|
||||
|
||||
# Use `gzip` to open the file if the extension is `.gz`
|
||||
if file_path.endswith(".gz"):
|
||||
with gzip.open(file_path, "rb") as f:
|
||||
return f.read(file_bytes).decode()
|
||||
|
||||
with open(file_path, "rb") as f:
|
||||
return f.read(file_bytes).decode()
|
||||
|
|
|
|||
|
|
@ -547,6 +547,42 @@ class TestBackups(BaseTestCommands):
|
|||
self.assertIn("successfully completed", self.stdout)
|
||||
self.assertNotEqual(before_backup["database"], after_backup["database"])
|
||||
|
||||
@skipIf(
|
||||
not (frappe.conf.db_type == "mariadb"),
|
||||
"Only for MariaDB",
|
||||
)
|
||||
def test_backup_extract_restore(self):
|
||||
"""Restore a backup after extracting"""
|
||||
self.execute("bench --site {site} backup")
|
||||
self.assertEqual(self.returncode, 0)
|
||||
backup = fetch_latest_backups()
|
||||
self.execute(f"gunzip {backup['database']}")
|
||||
self.assertEqual(self.returncode, 0)
|
||||
backup_sql = backup["database"].replace(".gz", "")
|
||||
assert os.path.isfile(backup_sql)
|
||||
self.execute(
|
||||
"bench --site {site} restore {backup_sql}",
|
||||
{
|
||||
"backup_sql": backup_sql,
|
||||
},
|
||||
)
|
||||
self.assertEqual(self.returncode, 0)
|
||||
|
||||
@skipIf(
|
||||
not (frappe.conf.db_type == "mariadb"),
|
||||
"Only for MariaDB",
|
||||
)
|
||||
def test_old_backup_restore(self):
|
||||
"""Restore a backup after extracting"""
|
||||
self.execute("bench --site {site} backup --old-backup-metadata")
|
||||
self.assertEqual(self.returncode, 0)
|
||||
backup = fetch_latest_backups()
|
||||
self.execute(
|
||||
"bench --site {site} restore {database}",
|
||||
backup,
|
||||
)
|
||||
self.assertEqual(self.returncode, 0)
|
||||
|
||||
def test_backup_fails_with_exit_code(self):
|
||||
"""Provide incorrect options to check if exit code is 1"""
|
||||
odb = BackupGenerator(
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
|
||||
# License: MIT. See LICENSE
|
||||
import contextlib
|
||||
|
||||
# imports - standard imports
|
||||
import gzip
|
||||
|
|
@ -54,6 +55,7 @@ class BackupGenerator:
|
|||
include_doctypes="",
|
||||
exclude_doctypes="",
|
||||
verbose=False,
|
||||
old_backup_metadata=False,
|
||||
):
|
||||
global _verbose
|
||||
self.compress_files = compress_files or compress
|
||||
|
|
@ -72,6 +74,7 @@ class BackupGenerator:
|
|||
self.include_doctypes = include_doctypes
|
||||
self.exclude_doctypes = exclude_doctypes
|
||||
self.partial = False
|
||||
self.old_backup_metadata = old_backup_metadata
|
||||
|
||||
site = frappe.local.site or frappe.generate_hash(length=8)
|
||||
self.site_slug = site.replace(".", "_")
|
||||
|
|
@ -372,10 +375,20 @@ class BackupGenerator:
|
|||
_("gzip not found in PATH! This is required to take a backup."), exc=frappe.ExecutableNotFound
|
||||
)
|
||||
|
||||
database_header_content = [
|
||||
f"Backup generated by Frappe {frappe.__version__} on branch {get_app_branch('frappe') or 'N/A'}",
|
||||
"",
|
||||
]
|
||||
if self.old_backup_metadata:
|
||||
database_header_content = [
|
||||
f"Backup generated by Frappe {frappe.__version__} on branch {get_app_branch('frappe') or 'N/A'}",
|
||||
"",
|
||||
]
|
||||
else:
|
||||
database_header_content = [
|
||||
"begin frappe metadata",
|
||||
"[frappe]",
|
||||
f"version = {frappe.__version__}",
|
||||
f"branch = {get_app_branch('frappe') or 'N/A'}",
|
||||
"end frappe metadata",
|
||||
"",
|
||||
]
|
||||
|
||||
if self.backup_includes:
|
||||
backup_info = ("Backing Up Tables: ", ", ".join(self.backup_includes))
|
||||
|
|
@ -511,6 +524,7 @@ def scheduled_backup(
|
|||
compress=False,
|
||||
force=False,
|
||||
verbose=False,
|
||||
old_backup_metadata=False,
|
||||
):
|
||||
"""this function is called from scheduler
|
||||
deletes backups older than 7 days
|
||||
|
|
@ -529,6 +543,7 @@ def scheduled_backup(
|
|||
compress=compress,
|
||||
force=force,
|
||||
verbose=verbose,
|
||||
old_backup_metadata=old_backup_metadata,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -546,6 +561,7 @@ def new_backup(
|
|||
compress=False,
|
||||
force=False,
|
||||
verbose=False,
|
||||
old_backup_metadata=False,
|
||||
):
|
||||
delete_temp_backups()
|
||||
odb = BackupGenerator(
|
||||
|
|
@ -565,6 +581,7 @@ def new_backup(
|
|||
exclude_doctypes=exclude_doctypes,
|
||||
verbose=verbose,
|
||||
compress_files=compress,
|
||||
old_backup_metadata=old_backup_metadata,
|
||||
)
|
||||
odb.get_backup(older_than, ignore_files, force=force)
|
||||
return odb
|
||||
|
|
@ -628,43 +645,29 @@ def get_or_generate_backup_encryption_key():
|
|||
return key
|
||||
|
||||
|
||||
class Backup:
|
||||
def __init__(self, file_path):
|
||||
self.file_path = file_path
|
||||
@contextlib.contextmanager
|
||||
def decrypt_backup(file_path: str, passphrase: str):
|
||||
if not os.path.exists(file_path):
|
||||
print("Invalid path: ", file_path)
|
||||
return
|
||||
else:
|
||||
file_path_with_ext = file_path + ".gpg"
|
||||
os.rename(file_path, file_path_with_ext)
|
||||
|
||||
def backup_decryption(self, passphrase):
|
||||
"""
|
||||
Decrypts backup at the given path using the passphrase.
|
||||
"""
|
||||
if not os.path.exists(self.file_path):
|
||||
print("Invalid path", self.file_path)
|
||||
return
|
||||
else:
|
||||
file_path_with_ext = self.file_path + ".gpg"
|
||||
os.rename(self.file_path, file_path_with_ext)
|
||||
|
||||
cmd_string = "gpg --yes --passphrase {passphrase} --pinentry-mode loopback -o {decrypted_file} -d {file_location}"
|
||||
command = cmd_string.format(
|
||||
passphrase=passphrase,
|
||||
file_location=file_path_with_ext,
|
||||
decrypted_file=self.file_path,
|
||||
)
|
||||
frappe.utils.execute_in_shell(command)
|
||||
|
||||
def decryption_rollback(self):
|
||||
"""
|
||||
Checks if the decrypted file exists at the given path.
|
||||
if exists
|
||||
Renames the orginal encrypted file.
|
||||
else
|
||||
Removes the decrypted file and rename the original file.
|
||||
"""
|
||||
if os.path.exists(self.file_path + ".gpg"):
|
||||
if os.path.exists(self.file_path):
|
||||
os.remove(self.file_path)
|
||||
if os.path.exists(self.file_path.rstrip(".gz")):
|
||||
os.remove(self.file_path.rstrip(".gz"))
|
||||
os.rename(self.file_path + ".gpg", self.file_path)
|
||||
cmd_string = "gpg --yes --passphrase {passphrase} --pinentry-mode loopback -o {decrypted_file} -d {file_location}"
|
||||
command = cmd_string.format(
|
||||
passphrase=passphrase,
|
||||
file_location=file_path_with_ext,
|
||||
decrypted_file=file_path,
|
||||
)
|
||||
frappe.utils.execute_in_shell(command)
|
||||
yield
|
||||
if os.path.exists(file_path + ".gpg"):
|
||||
if os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(file_path.rstrip(".gz")):
|
||||
os.remove(file_path.rstrip(".gz"))
|
||||
os.rename(file_path + ".gpg", file_path)
|
||||
|
||||
|
||||
def backup(
|
||||
|
|
@ -673,7 +676,6 @@ def backup(
|
|||
backup_path_files=None,
|
||||
backup_path_private_files=None,
|
||||
backup_path_conf=None,
|
||||
quiet=False,
|
||||
):
|
||||
"Backup"
|
||||
odb = scheduled_backup(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue