feat: Ability to exclude/include certain doctypes from conf or via CLI
This commit is contained in:
Gavin D'souza
parent
305af578cf
commit
6cb6a18b48
2 changed files with 90 additions and 20 deletions
|
|
@ -387,10 +387,13 @@ def use(site, sites_path='.'):
|
|||
|
||||
@click.command('backup')
|
||||
@click.option('--with-files', default=False, is_flag=True, help="Take backup with files")
|
||||
@click.option('--ignore-backup-conf', default=False, is_flag=True, help="Ignore excludes/includes set in config")
|
||||
@click.option('--include', default="", type=str, help="Specify the DocTypes to backup seperated by commas")
|
||||
@click.option('--exclude', default="", type=str, help="Specify the DocTypes to not backup seperated by commas")
|
||||
@click.option('--verbose', default=False, is_flag=True)
|
||||
@pass_context
|
||||
def backup(context, with_files=False, backup_path_db=None, backup_path_files=None,
|
||||
backup_path_private_files=None, quiet=False, verbose=False):
|
||||
backup_path_private_files=None, quiet=False, verbose=False, ignore_backup_conf=False, include="", exclude=""):
|
||||
"Backup"
|
||||
from frappe.utils.backups import scheduled_backup
|
||||
verbose = verbose or context.verbose
|
||||
|
|
@ -399,10 +402,21 @@ def backup(context, with_files=False, backup_path_db=None, backup_path_files=Non
|
|||
try:
|
||||
frappe.init(site=site)
|
||||
frappe.connect()
|
||||
odb = scheduled_backup(ignore_files=not with_files, backup_path_db=backup_path_db, backup_path_files=backup_path_files, backup_path_private_files=backup_path_private_files, force=True, verbose=verbose)
|
||||
odb = scheduled_backup(
|
||||
ignore_files=not with_files,
|
||||
backup_path_db=backup_path_db,
|
||||
backup_path_files=backup_path_files,
|
||||
backup_path_private_files=backup_path_private_files,
|
||||
ignore_conf=ignore_backup_conf,
|
||||
include_doctypes=include,
|
||||
exclude_doctypes=exclude,
|
||||
verbose=verbose,
|
||||
force=True,
|
||||
)
|
||||
except Exception as e:
|
||||
if verbose:
|
||||
print("Backup failed for {0}. Database or site_config.json may be corrupted".format(site))
|
||||
print(e)
|
||||
exit_code = 1
|
||||
continue
|
||||
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ from frappe.utils import cstr, get_url, now_datetime
|
|||
# backup variable for backwards compatibility
|
||||
verbose = False
|
||||
_verbose = verbose
|
||||
base_tables = ["__Auth", "__global_search", "__UserSettings"]
|
||||
|
||||
|
||||
class BackupGenerator:
|
||||
|
|
@ -29,7 +30,7 @@ class BackupGenerator:
|
|||
"""
|
||||
def __init__(self, db_name, user, password, backup_path_db=None, backup_path_files=None,
|
||||
backup_path_private_files=None, db_host="localhost", db_port=None, verbose=False,
|
||||
db_type='mariadb', backup_path_conf=None):
|
||||
db_type='mariadb', backup_path_conf=None, ignore_conf=False, include_doctypes="", exclude_doctypes=""):
|
||||
global _verbose
|
||||
self.db_host = db_host
|
||||
self.db_port = db_port
|
||||
|
|
@ -41,6 +42,9 @@ class BackupGenerator:
|
|||
self.backup_path_db = backup_path_db
|
||||
self.backup_path_files = backup_path_files
|
||||
self.backup_path_private_files = backup_path_private_files
|
||||
self.ignore_conf = ignore_conf
|
||||
self.include_doctypes = include_doctypes
|
||||
self.exclude_doctypes = exclude_doctypes
|
||||
|
||||
if not self.db_type:
|
||||
self.db_type = 'mariadb'
|
||||
|
|
@ -54,6 +58,7 @@ class BackupGenerator:
|
|||
self.site_slug = site.replace('.', '_')
|
||||
self.verbose = verbose
|
||||
self.setup_backup_directory()
|
||||
self.setup_backup_tables()
|
||||
_verbose = verbose
|
||||
|
||||
def setup_backup_directory(self):
|
||||
|
|
@ -68,6 +73,35 @@ class BackupGenerator:
|
|||
dir = os.path.dirname(file_path)
|
||||
os.makedirs(dir, exist_ok=True)
|
||||
|
||||
def setup_backup_tables(self):
|
||||
"""Sets self.backup_includes, self.backup_excludes based on passed args
|
||||
"""
|
||||
def get_tables(doctypes):
|
||||
tables = []
|
||||
for doctype in doctypes:
|
||||
if doctype:
|
||||
if doctype.startswith("tab"):
|
||||
tables.append(doctype)
|
||||
else:
|
||||
tables.append("tab" + doctype)
|
||||
return tables
|
||||
|
||||
passed_tables = {
|
||||
"include": get_tables(self.include_doctypes.strip().split(",")),
|
||||
"exclude": get_tables(self.exclude_doctypes.strip().split(","))
|
||||
}
|
||||
conf_tables = {
|
||||
"include": get_tables(frappe.conf.backup.get("includes", [])) + base_tables,
|
||||
"exclude": get_tables(frappe.conf.backup.get("excludes", []))
|
||||
}
|
||||
|
||||
self.backup_includes = passed_tables["include"]
|
||||
self.backup_excludes = passed_tables["exclude"]
|
||||
|
||||
if not self.ignore_conf:
|
||||
self.backup_includes = self.backup_includes or conf_tables["include"]
|
||||
self.backup_excludes = self.backup_excludes or conf_tables["exclude"]
|
||||
|
||||
@property
|
||||
def site_config_backup_path(self):
|
||||
# For backwards compatibility
|
||||
|
|
@ -189,23 +223,42 @@ class BackupGenerator:
|
|||
args = dict([item[0], frappe.utils.esc(str(item[1]), '$ ')]
|
||||
for item in self.__dict__.copy().items())
|
||||
|
||||
if self.verbose:
|
||||
print("Skipping Tables: {0}\n".format(", ".join(frappe.conf.ignore_tables_in_backup)))
|
||||
|
||||
args["skip_tables"] = " ".join(["--ignore-table={0}.{1}".format(frappe.conf.db_name, table) for table in frappe.conf.ignore_tables_in_backup or []])
|
||||
cmd_string = """mysqldump --single-transaction --quick --lock-tables=false -u %(user)s -p%(password)s %(db_name)s -h %(db_host)s -P %(db_port)s %(skip_tables)s | gzip > %(backup_path_db)s """ % args
|
||||
if self.backup_includes:
|
||||
print("Backing Up Tables: {0}\n".format(", ".join(self.backup_includes)))
|
||||
elif self.backup_excludes:
|
||||
print("Skipping Tables: {0}\n".format(", ".join(self.backup_excludes)))
|
||||
|
||||
if self.db_type == 'postgres':
|
||||
cmd_string = "pg_dump postgres://{user}:{password}@{db_host}:{db_port}/{db_name} | gzip > {backup_path_db}".format(
|
||||
user=args.get('user'),
|
||||
password=args.get('password'),
|
||||
db_host=args.get('db_host'),
|
||||
db_port=args.get('db_port'),
|
||||
db_name=args.get('db_name'),
|
||||
backup_path_db=args.get('backup_path_db')
|
||||
)
|
||||
if self.backup_includes:
|
||||
args["include"] = " ".join(["--table='{0}'".format(table) for table in self.backup_includes])
|
||||
elif self.backup_excludes:
|
||||
args["exclude"] = " ".join(["--exclude-table='{0}'".format(table) for table in self.backup_excludes])
|
||||
|
||||
err, out = frappe.utils.execute_in_shell(cmd_string)
|
||||
cmd_string = "pg_dump postgres://{user}:{password}@{db_host}:{db_port}/{db_name} {include} {exclude} | gzip > {backup_path_db}"
|
||||
|
||||
else:
|
||||
if self.backup_includes:
|
||||
args["include"] = " ".join(["'{0}'".format(x) for x in self.backup_includes])
|
||||
|
||||
elif self.backup_excludes:
|
||||
args["exclude"] = " ".join(["--ignore-table='{0}.{1}'".format(frappe.conf.db_name, table) for table in self.backup_excludes])
|
||||
|
||||
cmd_string = "mysqldump --single-transaction --quick --lock-tables=false -u {user} -p{password} {db_name} -h {db_host} -P {db_port} {include} {exclude} | gzip > {backup_path_db}"
|
||||
|
||||
command = cmd_string.format(
|
||||
user=args.get('user'),
|
||||
password=args.get('password'),
|
||||
db_host=args.get('db_host'),
|
||||
db_port=args.get('db_port'),
|
||||
db_name=args.get('db_name'),
|
||||
backup_path_db=args.get('backup_path_db'),
|
||||
exclude=args.get('exclude', ''),
|
||||
include=args.get('include', '')
|
||||
)
|
||||
|
||||
print(command)
|
||||
|
||||
err, out = frappe.utils.execute_in_shell(command)
|
||||
|
||||
def send_email(self):
|
||||
"""
|
||||
|
|
@ -279,14 +332,14 @@ def fetch_latest_backups():
|
|||
}
|
||||
|
||||
|
||||
def scheduled_backup(older_than=6, ignore_files=False, backup_path_db=None, backup_path_files=None, backup_path_private_files=None, force=False, verbose=False):
|
||||
def scheduled_backup(older_than=6, ignore_files=False, backup_path_db=None, backup_path_files=None, backup_path_private_files=None, force=False, verbose=False, ignore_conf=False, include_doctypes="", exclude_doctypes=""):
|
||||
"""this function is called from scheduler
|
||||
deletes backups older than 7 days
|
||||
takes backup"""
|
||||
odb = new_backup(older_than, ignore_files, backup_path_db=backup_path_db, backup_path_files=backup_path_files, force=force, verbose=verbose)
|
||||
odb = new_backup(older_than, ignore_files, backup_path_db=backup_path_db, backup_path_files=backup_path_files, force=force, verbose=verbose, ignore_conf=ignore_conf, include_doctypes=include_doctypes, exclude_doctypes=exclude_doctypes)
|
||||
return odb
|
||||
|
||||
def new_backup(older_than=6, ignore_files=False, backup_path_db=None, backup_path_files=None, backup_path_private_files=None, force=False, verbose=False):
|
||||
def new_backup(older_than=6, ignore_files=False, backup_path_db=None, backup_path_files=None, backup_path_private_files=None, force=False, verbose=False, ignore_conf=False, include_doctypes="", exclude_doctypes=""):
|
||||
delete_temp_backups(older_than = frappe.conf.keep_backups_for_hours or 24)
|
||||
odb = BackupGenerator(frappe.conf.db_name, frappe.conf.db_name,\
|
||||
frappe.conf.db_password,
|
||||
|
|
@ -295,6 +348,9 @@ def new_backup(older_than=6, ignore_files=False, backup_path_db=None, backup_pat
|
|||
db_host = frappe.db.host,
|
||||
db_port = frappe.db.port,
|
||||
db_type = frappe.conf.db_type,
|
||||
ignore_conf=ignore_conf,
|
||||
include_doctypes=include_doctypes,
|
||||
exclude_doctypes=exclude_doctypes,
|
||||
verbose=verbose)
|
||||
odb.get_backup(older_than, ignore_files, force=force)
|
||||
return odb
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue