diff --git a/frappe/build.py b/frappe/build.py index f14b250a92..f47a7cb32b 100644 --- a/frappe/build.py +++ b/frappe/build.py @@ -105,7 +105,7 @@ def download_frappe_assets(verbose=True): if frappe_head: try: url = get_assets_link(frappe_head) - click.secho("Retreiving assets...", fg="yellow") + click.secho("Retrieving assets...", fg="yellow") prefix = mkdtemp(prefix="frappe-assets-", suffix=frappe_head) assets_archive = download_file(url, prefix) print("\n{0} Downloaded Frappe assets from {1}".format(green('✔'), url)) diff --git a/frappe/commands/site.py b/frappe/commands/site.py index 51c352a931..bc65aa178c 100755 --- a/frappe/commands/site.py +++ b/frappe/commands/site.py @@ -9,7 +9,7 @@ import click import frappe from frappe.commands import get_site, pass_context from frappe.exceptions import SiteNotSpecifiedError -from frappe.utils import get_site_path, touch_file +from frappe.installer import _new_site @click.command('new-site') @@ -42,57 +42,6 @@ def new_site(site, mariadb_root_username=None, mariadb_root_password=None, admin if len(frappe.utils.get_sites()) == 1: use(site) -def _new_site(db_name, site, mariadb_root_username=None, mariadb_root_password=None, - admin_password=None, verbose=False, install_apps=None, source_sql=None, force=False, - no_mariadb_socket=False, reinstall=False, db_password=None, db_type=None, db_host=None, - db_port=None, new_site=False): - """Install a new Frappe site""" - - if not force and os.path.exists(site): - print('Site {0} already exists'.format(site)) - sys.exit(1) - - if no_mariadb_socket and not db_type == "mariadb": - print('--no-mariadb-socket requires db_type to be set to mariadb.') - sys.exit(1) - - if not db_name: - import hashlib - db_name = '_' + hashlib.sha1(site.encode()).hexdigest()[:16] - - from frappe.commands.scheduler import _is_scheduler_enabled - from frappe.installer import install_db, make_site_dirs - from frappe.installer import install_app as _install_app - import frappe.utils.scheduler - - frappe.init(site=site) - - try: - - # enable scheduler post install? - enable_scheduler = _is_scheduler_enabled() - except Exception: - enable_scheduler = False - - make_site_dirs() - - installing = touch_file(get_site_path('locks', 'installing.lock')) - - install_db(root_login=mariadb_root_username, root_password=mariadb_root_password, db_name=db_name, - admin_password=admin_password, verbose=verbose, source_sql=source_sql, force=force, reinstall=reinstall, - db_password=db_password, db_type=db_type, db_host=db_host, db_port=db_port, no_mariadb_socket=no_mariadb_socket) - apps_to_install = ['frappe'] + (frappe.conf.get("install_apps") or []) + (list(install_apps) or []) - for app in apps_to_install: - _install_app(app, verbose=verbose, set_as_patched=not source_sql) - - os.remove(installing) - - frappe.utils.scheduler.toggle_scheduler(enable_scheduler) - frappe.db.commit() - - scheduler_status = "disabled" if frappe.utils.scheduler.is_scheduler_disabled() else "enabled" - print("*** Scheduler is", scheduler_status, "***") - @click.command('restore') @click.argument('sql-file-path') @@ -107,33 +56,41 @@ def _new_site(db_name, site, mariadb_root_username=None, mariadb_root_password=N @pass_context def restore(context, sql_file_path, mariadb_root_username=None, mariadb_root_password=None, db_name=None, verbose=None, install_app=None, admin_password=None, force=None, with_public_files=None, with_private_files=None): "Restore site database from an sql file" - from frappe.installer import extract_sql_gzip, extract_files, is_downgrade, validate_database_sql + from frappe.installer import ( + extract_sql_from_archive, + extract_files, + is_downgrade, + is_partial, + validate_database_sql + ) + force = context.force or force + decompressed_file_name = extract_sql_from_archive(sql_file_path) - # Extract the gzip file if user has passed *.sql.gz file instead of *.sql file - if not os.path.exists(sql_file_path): - base_path = '..' - sql_file_path = os.path.join(base_path, sql_file_path) - if not os.path.exists(sql_file_path): - print('Invalid path {0}'.format(sql_file_path[3:])) - sys.exit(1) - elif sql_file_path.startswith(os.sep): - base_path = os.sep - else: - base_path = '.' - - if sql_file_path.endswith('sql.gz'): - decompressed_file_name = extract_sql_gzip(os.path.abspath(sql_file_path)) - else: - decompressed_file_name = sql_file_path + # check if partial backup + if is_partial(decompressed_file_name): + click.secho( + "Partial Backup file detected. You cannot use a partial file to restore a Frappe Site.", + fg="red" + ) + click.secho( + "Use `bench partial-restore` to restore a partial backup to an existing site.", + fg="yellow" + ) + sys.exit(1) + # check if valid SQL file validate_database_sql(decompressed_file_name, _raise=not force) + site = get_site(context) frappe.init(site=site) # dont allow downgrading to older versions of frappe without force if not force and is_downgrade(decompressed_file_name, verbose=True): - warn_message = "This is not recommended and may lead to unexpected behaviour. Do you want to continue anyway?" + warn_message = ( + "This is not recommended and may lead to unexpected behaviour. " + "Do you want to continue anyway?" + ) click.confirm(warn_message, abort=True) _new_site(frappe.conf.db_name, site, mariadb_root_username=mariadb_root_username, @@ -156,9 +113,28 @@ def restore(context, sql_file_path, mariadb_root_username=None, mariadb_root_pas if decompressed_file_name != sql_file_path: os.remove(decompressed_file_name) - success_message = "Site {0} has been restored{1}".format(site, " with files" if (with_public_files or with_private_files) else "") + success_message = "Site {0} has been restored{1}".format( + site, + " with files" if (with_public_files or with_private_files) else "" + ) click.secho(success_message, fg="green") + +@click.command('partial-restore') +@click.argument('sql-file-path') +@click.option("--verbose", "-v", is_flag=True) +@pass_context +def partial_restore(context, sql_file_path, verbose): + from frappe.installer import partial_restore + verbose = context.verbose or verbose + + site = get_site(context) + frappe.init(site=site) + frappe.connect(site=site) + partial_restore(sql_file_path, verbose) + frappe.destroy() + + @click.command('reinstall') @click.option('--admin-password', help='Administrator Password for reinstalled site') @click.option('--mariadb-root-username', help='Root username for MariaDB') @@ -416,16 +392,20 @@ def use(site, sites_path='.'): @click.command('backup') @click.option('--with-files', default=False, is_flag=True, help="Take backup with files") +@click.option('--include', '--only', '-i', default="", type=str, help="Specify the DocTypes to backup seperated by commas") +@click.option('--exclude', '-e', default="", type=str, help="Specify the DocTypes to not backup seperated by commas") @click.option('--backup-path', default=None, help="Set path for saving all the files in this operation") @click.option('--backup-path-db', default=None, help="Set path for saving database file") @click.option('--backup-path-files', default=None, help="Set path for saving public file") @click.option('--backup-path-private-files', default=None, help="Set path for saving private file") @click.option('--backup-path-conf', default=None, help="Set path for saving config file") +@click.option('--ignore-backup-conf', default=False, is_flag=True, help="Ignore excludes/includes set in config") @click.option('--verbose', default=False, is_flag=True, help="Add verbosity") @click.option('--compress', default=False, is_flag=True, help="Compress private and public files") @pass_context def backup(context, with_files=False, backup_path=None, backup_path_db=None, backup_path_files=None, - backup_path_private_files=None, backup_path_conf=None, verbose=False, compress=False): + backup_path_private_files=None, backup_path_conf=None, ignore_backup_conf=False, verbose=False, + compress=False, include="", exclude=""): "Backup" from frappe.utils.backups import scheduled_backup verbose = verbose or context.verbose @@ -435,11 +415,27 @@ def backup(context, with_files=False, backup_path=None, backup_path_db=None, bac try: frappe.init(site=site) frappe.connect() - odb = scheduled_backup(ignore_files=not with_files, backup_path=backup_path, backup_path_db=backup_path_db, backup_path_files=backup_path_files, backup_path_private_files=backup_path_private_files, backup_path_conf=backup_path_conf, force=True, verbose=verbose, compress=compress) + odb = scheduled_backup( + ignore_files=not with_files, + backup_path=backup_path, + backup_path_db=backup_path_db, + backup_path_files=backup_path_files, + backup_path_private_files=backup_path_private_files, + backup_path_conf=backup_path_conf, + ignore_conf=ignore_backup_conf, + include_doctypes=include, + exclude_doctypes=exclude, + compress=compress, + verbose=verbose, + force=True + ) except Exception: click.secho("Backup failed for Site {0}. Database or site_config.json may be corrupted".format(site), fg="red") + if verbose: + print(frappe.get_traceback()) exit_code = 1 continue + odb.print_summary() click.secho("Backup for Site {0} has been successfully completed{1}".format(site, " with files" if with_files else ""), fg="green") frappe.destroy() @@ -512,13 +508,14 @@ def _drop_site(site, root_login='root', root_password=None, archived_sites_path= if force: pass else: - click.echo("="*80) - click.echo("Error: The operation has stopped because backup of {s}'s database failed.".format(s=site)) - click.echo("Reason: {reason}{sep}".format(reason=str(err), sep="\n")) - click.echo("Fix the issue and try again.") - click.echo( - "Hint: Use 'bench drop-site {s} --force' to force the removal of {s}".format(sep="\n", tab="\t", s=site) - ) + messages = [ + "=" * 80, + "Error: The operation has stopped because backup of {0}'s database failed.".format(site), + "Reason: {0}\n".format(str(err)), + "Fix the issue and try again.", + "Hint: Use 'bench drop-site {0} --force' to force the removal of {0}".format(site) + ] + click.echo("\n".join(messages)) sys.exit(1) drop_user_and_database(frappe.conf.db_name, root_login, root_password) @@ -734,5 +731,6 @@ commands = [ stop_recording, add_to_hosts, start_ngrok, - build_search_index + build_search_index, + partial_restore ] diff --git a/frappe/core/doctype/doctype/doctype.py b/frappe/core/doctype/doctype/doctype.py index fd0cb1917d..cb3d06a29a 100644 --- a/frappe/core/doctype/doctype/doctype.py +++ b/frappe/core/doctype/doctype/doctype.py @@ -572,7 +572,8 @@ class DocType(Document): def make_repeatable(self): """If allow_auto_repeat is set, add auto_repeat custom field.""" if self.allow_auto_repeat: - if not frappe.db.exists('Custom Field', {'fieldname': 'auto_repeat', 'dt': self.name}): + if not frappe.db.exists('Custom Field', {'fieldname': 'auto_repeat', 'dt': self.name}) and \ + not frappe.db.exists('DocField', {'fieldname': 'auto_repeat', 'parent': self.name}): insert_after = self.fields[len(self.fields) - 1].fieldname df = dict(fieldname='auto_repeat', label='Auto Repeat', fieldtype='Link', options='Auto Repeat', insert_after=insert_after, read_only=1, no_copy=1, print_hide=1) create_custom_field(self.name, df) diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py index 2c5865fb69..7309528da6 100644 --- a/frappe/core/doctype/user/user.py +++ b/frappe/core/doctype/user/user.py @@ -13,7 +13,7 @@ from frappe.utils.user import get_system_managers from bs4 import BeautifulSoup import frappe.permissions import frappe.share - +import frappe.defaults from frappe.website.utils import is_signup_enabled from frappe.utils.background_jobs import enqueue @@ -107,6 +107,10 @@ class User(Document): ) if self.name not in ('Administrator', 'Guest') and not self.user_image: frappe.enqueue('frappe.core.doctype.user.user.update_gravatar', name=self.name) + + # Set user selected timezone + if self.time_zone: + frappe.defaults.set_default("time_zone", self.time_zone, self.name) def has_website_permission(self, ptype, user, verbose=False): """Returns true if current user is the session user""" @@ -1129,4 +1133,4 @@ def check_password_reset_limit(user, rate_limit): frappe.throw(_("You have reached the hourly limit for generating password reset links. Please try again later.")) def get_generated_link_count(user): - return cint(frappe.cache().hget("password_reset_link_count", user)) or 0 \ No newline at end of file + return cint(frappe.cache().hget("password_reset_link_count", user)) or 0 diff --git a/frappe/custom/doctype/customize_form/customize_form.py b/frappe/custom/doctype/customize_form/customize_form.py index 9ce602906c..cf674082ab 100644 --- a/frappe/custom/doctype/customize_form/customize_form.py +++ b/frappe/custom/doctype/customize_form/customize_form.py @@ -76,8 +76,8 @@ class CustomizeForm(Document): def create_auto_repeat_custom_field_if_requried(self, meta): if self.allow_auto_repeat: - if not frappe.db.exists('Custom Field', {'fieldname': 'auto_repeat', - 'dt': self.doc_type}): + if not frappe.db.exists('Custom Field', {'fieldname': 'auto_repeat', 'dt': self.doc_type}) and \ + not frappe.db.exists('DocField', {'fieldname': 'auto_repeat', 'parent': self.name}): insert_after = self.fields[len(self.fields) - 1].fieldname df = dict( fieldname='auto_repeat', diff --git a/frappe/database/db_manager.py b/frappe/database/db_manager.py index 3345fce735..b8ffae519b 100644 --- a/frappe/database/db_manager.py +++ b/frappe/database/db_manager.py @@ -3,7 +3,6 @@ import frappe class DbManager: - def __init__(self, db): """ Pass root_conn here for access to all databases. @@ -66,10 +65,10 @@ class DbManager: esc = make_esc('$ ') from distutils.spawn import find_executable - pipe = find_executable('pv') - if pipe: - pipe = '{pipe} {source} |'.format( - pipe=pipe, + pv = find_executable('pv') + if pv: + pipe = '{pv} {source} |'.format( + pv=pv, source=source ) source = '' @@ -78,7 +77,7 @@ class DbManager: source = '< {source}'.format(source=source) if pipe: - print('Creating Database...') + print('Restoring Database file...') command = '{pipe} mysql -u {user} -p{password} -h{host} ' + ('-P{port}' if frappe.db.port else '') + ' {target} {source}' command = command.format( diff --git a/frappe/database/mariadb/setup_db.py b/frappe/database/mariadb/setup_db.py index a4e4d624ae..9b73d77171 100644 --- a/frappe/database/mariadb/setup_db.py +++ b/frappe/database/mariadb/setup_db.py @@ -1,7 +1,7 @@ from __future__ import unicode_literals import frappe -import os, sys +import os from frappe.database.db_manager import DbManager expected_settings_10_2_earlier = { @@ -86,6 +86,8 @@ def drop_user_and_database(db_name, root_login, root_password): dbman.drop_database(db_name) def bootstrap_database(db_name, verbose, source_sql=None): + import sys + frappe.connect(db_name=db_name) if not check_database_settings(): print('Database settings do not match expected values; stopping database setup.') @@ -94,9 +96,17 @@ def bootstrap_database(db_name, verbose, source_sql=None): import_db_from_sql(source_sql, verbose) frappe.connect(db_name=db_name) - if not 'tabDefaultValue' in frappe.db.get_tables(): - print('''Database not installed, this can due to lack of permission, or that the database name exists. - Check your mysql root password, or use --force to reinstall''') + if 'tabDefaultValue' not in frappe.db.get_tables(): + from click import secho + + secho( + "Table 'tabDefaultValue' missing in the restored site. " + "Database not installed correctly, this can due to lack of " + "permission, or that the database name exists. Check your mysql" + " root password, validity of the backup file or use --force to" + " reinstall", + fg="red" + ) sys.exit(1) def import_db_from_sql(source_sql=None, verbose=False): diff --git a/frappe/database/postgres/setup_db.py b/frappe/database/postgres/setup_db.py index f53872db82..3ee6b6a286 100644 --- a/frappe/database/postgres/setup_db.py +++ b/frappe/database/postgres/setup_db.py @@ -1,5 +1,7 @@ -import frappe, subprocess, os -from six.moves import input +import os + +import frappe + def setup_database(force, source_sql=None, verbose=False): root_conn = get_root_connection() @@ -10,24 +12,62 @@ def setup_database(force, source_sql=None, verbose=False): root_conn.sql("CREATE user {0} password '{1}'".format(frappe.conf.db_name, frappe.conf.db_password)) root_conn.sql("GRANT ALL PRIVILEGES ON DATABASE `{0}` TO {0}".format(frappe.conf.db_name)) + root_conn.close() + + bootstrap_database(frappe.conf.db_name, verbose, source_sql=source_sql) + frappe.connect() + +def bootstrap_database(db_name, verbose, source_sql=None): + frappe.connect(db_name=db_name) + import_db_from_sql(source_sql, verbose) + frappe.connect(db_name=db_name) + + if 'tabDefaultValue' not in frappe.db.get_tables(): + import sys + from click import secho + + secho( + "Table 'tabDefaultValue' missing in the restored site. " + "This may be due to incorrect permissions or the result of a restore from a bad backup file. " + "Database not installed correctly.", + fg="red" + ) + sys.exit(1) + +def import_db_from_sql(source_sql=None, verbose=False): + from shutil import which + from subprocess import run, PIPE # we can't pass psql password in arguments in postgresql as mysql. So # set password connection parameter in environment variable subprocess_env = os.environ.copy() subprocess_env['PGPASSWORD'] = str(frappe.conf.db_password) + # bootstrap db if not source_sql: source_sql = os.path.join(os.path.dirname(__file__), 'framework_postgres.sql') - subprocess.check_output([ - 'psql', frappe.conf.db_name, - '-h', frappe.conf.db_host or 'localhost', - '-p', str(frappe.conf.db_port or '5432'), - '-U', frappe.conf.db_name, - '-f', source_sql - ], env=subprocess_env) + pv = which('pv') - frappe.connect() + _command = ( + f"psql {frappe.conf.db_name} " + f"-h {frappe.conf.db_host or 'localhost'} -p {str(frappe.conf.db_port or '5432')} " + f"-U {frappe.conf.db_name}" + ) + + if pv: + command = f"{pv} {source_sql} | " + _command + else: + command = _command + f" -f {source_sql}" + + print("Restoring Database file...") + if verbose: + print(command) + + restore_proc = run(command, env=subprocess_env, shell=True, stdout=PIPE) + + if verbose: + print(f"\nSTDOUT by psql:\n{restore_proc.stdout.decode()}\nImported from Database File: {source_sql}") def setup_help_database(help_db_name): root_conn = get_root_connection() @@ -38,19 +78,20 @@ def setup_help_database(help_db_name): root_conn.sql("GRANT ALL PRIVILEGES ON DATABASE `{0}` TO {0}".format(help_db_name)) def get_root_connection(root_login=None, root_password=None): - import getpass if not frappe.local.flags.root_connection: if not root_login: root_login = frappe.conf.get("root_login") or None if not root_login: + from six.moves import input root_login = input("Enter postgres super user: ") if not root_password: root_password = frappe.conf.get("root_password") or None if not root_password: - root_password = getpass.getpass("Postgres super user password: ") + from getpass import getpass + root_password = getpass("Postgres super user password: ") frappe.local.flags.root_connection = frappe.database.get_db(user=root_login, password=root_password) diff --git a/frappe/exceptions.py b/frappe/exceptions.py index 267f5410af..9b63b23787 100644 --- a/frappe/exceptions.py +++ b/frappe/exceptions.py @@ -110,4 +110,5 @@ class DocumentAlreadyRestored(Exception): pass class InvalidAuthorizationHeader(CSRFTokenError): pass class InvalidAuthorizationPrefix(CSRFTokenError): pass class InvalidAuthorizationToken(CSRFTokenError): pass -class InvalidDatabaseFile(ValidationError): pass \ No newline at end of file +class InvalidDatabaseFile(ValidationError): pass +class ExecutableNotFound(FileNotFoundError): pass diff --git a/frappe/installer.py b/frappe/installer.py index be9b04d453..6a77e5e713 100755 --- a/frappe/installer.py +++ b/frappe/installer.py @@ -3,8 +3,90 @@ import json import os -from frappe.defaults import _clear_cache +import sys + import frappe +from frappe.defaults import _clear_cache + + +def _new_site( + db_name, + site, + mariadb_root_username=None, + mariadb_root_password=None, + admin_password=None, + verbose=False, + install_apps=None, + source_sql=None, + force=False, + no_mariadb_socket=False, + reinstall=False, + db_password=None, + db_type=None, + db_host=None, + db_port=None, + new_site=False, +): + """Install a new Frappe site""" + + if not force and os.path.exists(site): + print("Site {0} already exists".format(site)) + sys.exit(1) + + if no_mariadb_socket and not db_type == "mariadb": + print("--no-mariadb-socket requires db_type to be set to mariadb.") + sys.exit(1) + + if not db_name: + import hashlib + db_name = "_" + hashlib.sha1(site.encode()).hexdigest()[:16] + + frappe.init(site=site) + + from frappe.commands.scheduler import _is_scheduler_enabled + from frappe.utils import get_site_path, scheduler, touch_file + + try: + # enable scheduler post install? + enable_scheduler = _is_scheduler_enabled() + except Exception: + enable_scheduler = False + + make_site_dirs() + + installing = touch_file(get_site_path("locks", "installing.lock")) + + install_db( + root_login=mariadb_root_username, + root_password=mariadb_root_password, + db_name=db_name, + admin_password=admin_password, + verbose=verbose, + source_sql=source_sql, + force=force, + reinstall=reinstall, + db_password=db_password, + db_type=db_type, + db_host=db_host, + db_port=db_port, + no_mariadb_socket=no_mariadb_socket, + ) + apps_to_install = ( + ["frappe"] + (frappe.conf.get("install_apps") or []) + (list(install_apps) or []) + ) + + for app in apps_to_install: + install_app(app, verbose=verbose, set_as_patched=not source_sql) + + os.remove(installing) + + scheduler.toggle_scheduler(enable_scheduler) + frappe.db.commit() + + scheduler_status = ( + "disabled" if frappe.utils.scheduler.is_scheduler_disabled() else "enabled" + ) + print("*** Scheduler is", scheduler_status, "***") def install_db(root_login="root", root_password=None, db_name=None, source_sql=None, @@ -36,9 +118,9 @@ def install_db(root_login="root", root_password=None, db_name=None, source_sql=N def install_app(name, verbose=False, set_as_patched=True): from frappe.core.doctype.scheduled_job_type.scheduled_job_type import sync_jobs - from frappe.utils.fixtures import sync_fixtures from frappe.model.sync import sync_for from frappe.modules.utils import sync_customizations + from frappe.utils.fixtures import sync_fixtures frappe.flags.in_install = name frappe.flags.ignore_in_install = False @@ -347,6 +429,37 @@ def remove_missing_apps(): frappe.db.set_global("installed_apps", json.dumps(installed_apps)) +def extract_sql_from_archive(sql_file_path): + """Return the path of an SQL file if the passed argument is the path of a gzipped + SQL file or an SQL file path. The path may be absolute or relative from the bench + root directory or the sites sub-directory. + + Args: + sql_file_path (str): Path of the SQL file + + Returns: + str: Path of the decompressed SQL file + """ + # Extract the gzip file if user has passed *.sql.gz file instead of *.sql file + if not os.path.exists(sql_file_path): + base_path = '..' + sql_file_path = os.path.join(base_path, sql_file_path) + if not os.path.exists(sql_file_path): + print('Invalid path {0}'.format(sql_file_path[3:])) + sys.exit(1) + elif sql_file_path.startswith(os.sep): + base_path = os.sep + else: + base_path = '.' + + if sql_file_path.endswith('sql.gz'): + decompressed_file_name = extract_sql_gzip(os.path.abspath(sql_file_path)) + else: + decompressed_file_name = sql_file_path + + return decompressed_file_name + + def extract_sql_gzip(sql_gz_path): import subprocess @@ -361,9 +474,10 @@ def extract_sql_gzip(sql_gz_path): return decompressed_file + def extract_files(site_name, file_path, folder_name): - import subprocess import shutil + import subprocess # Need to do frappe.init to maintain the site locals frappe.init(site=site_name) @@ -391,6 +505,12 @@ def extract_files(site_name, file_path, folder_name): def is_downgrade(sql_file_path, verbose=False): """checks if input db backup will get downgraded on current bench""" + + # This function is only tested with mariadb + # TODO: Add postgres support + if frappe.conf.db_type not in (None, "mariadb"): + return False + from semantic_version import Version head = "INSERT INTO `tabInstalled Application` VALUES" @@ -424,6 +544,37 @@ def is_downgrade(sql_file_path, verbose=False): return downgrade +def is_partial(sql_file_path): + with open(sql_file_path) as f: + header = " ".join([f.readline() for _ in range(5)]) + if "Partial Backup" in header: + return True + return False + + +def partial_restore(sql_file_path, verbose=False): + sql_file = extract_sql_from_archive(sql_file_path) + + if frappe.conf.db_type in (None, "mariadb"): + from frappe.database.mariadb.setup_db import import_db_from_sql + elif frappe.conf.db_type == "postgres": + from frappe.database.postgres.setup_db import import_db_from_sql + import warnings + from click import style + warn = style( + "Delete the tables you want to restore manually before attempting" + " partial restore operation for PostreSQL databases", + fg="yellow" + ) + warnings.warn(warn) + + import_db_from_sql(source_sql=sql_file, verbose=verbose) + + # Removing temporarily created file + if sql_file != sql_file_path: + os.remove(sql_file) + + def validate_database_sql(path, _raise=True): """Check if file has contents and if DefaultValue table exists @@ -431,23 +582,29 @@ def validate_database_sql(path, _raise=True): path (str): Path of the decompressed SQL file _raise (bool, optional): Raise exception if invalid file. Defaults to True. """ - to_raise = False + empty_file = False + missing_table = True + error_message = "" if not os.path.getsize(path): error_message = f"{path} is an empty file!" - to_raise = True + empty_file = True - if not _raise: + # dont bother checking if empty file + if not empty_file: with open(path, "r") as f: for line in f: if 'tabDefaultValue' in line: - error_message = "Table `tabDefaultValue` not found in file." - to_raise = True + missing_table = False + break + + if missing_table: + error_message = "Table `tabDefaultValue` not found in file." if error_message: import click click.secho(error_message, fg="red") - if _raise and to_raise: + if _raise and (missing_table or empty_file): raise frappe.InvalidDatabaseFile diff --git a/frappe/integrations/frappe_providers/frappecloud.py b/frappe/integrations/frappe_providers/frappecloud.py index e09f09a44b..f60344ee8f 100644 --- a/frappe/integrations/frappe_providers/frappecloud.py +++ b/frappe/integrations/frappe_providers/frappecloud.py @@ -6,7 +6,7 @@ import frappe def frappecloud_migrator(local_site): - print("Retreiving Site Migrator...") + print("Retrieving Site Migrator...") remote_site = frappe.conf.frappecloud_url or "frappecloud.com" request_url = "https://{}/api/method/press.api.script".format(remote_site) request = requests.get(request_url) diff --git a/frappe/tests/test_commands.py b/frappe/tests/test_commands.py index 9757a823a6..8c76ce2f48 100644 --- a/frappe/tests/test_commands.py +++ b/frappe/tests/test_commands.py @@ -1,24 +1,88 @@ # Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors # imports - standard imports +import gzip +import json import os import shlex import subprocess +import sys import unittest -from glob import glob +import glob # imports - module imports import frappe -from frappe.utils.backups import fetch_latest_backups import frappe.recorder +from frappe.installer import add_to_installed_apps +from frappe.utils import add_to_date, now +from frappe.utils.backups import fetch_latest_backups + + +# TODO: check frappe.cli.coloured_output to set coloured output! +def supports_color(): + """ + Returns True if the running system's terminal supports color, and False + otherwise. + """ + plat = sys.platform + supported_platform = plat != 'Pocket PC' and (plat != 'win32' or 'ANSICON' in os.environ) + # isatty is not always implemented, #6223. + is_a_tty = hasattr(sys.stdout, 'isatty') and sys.stdout.isatty() + return supported_platform and is_a_tty + + +class color(dict): + nc = "\033[0m" + blue = "\033[94m" + green = "\033[92m" + yellow = "\033[93m" + red = "\033[91m" + silver = "\033[90m" + + def __getattr__(self, key): + if supports_color(): + ret = self.get(key) + else: + ret = "" + return ret def clean(value): - if isinstance(value, (bytes, str)): - value = value.decode().strip() + """Strips and converts bytes to str + + Args: + value ([type]): [description] + + Returns: + [type]: [description] + """ + if isinstance(value, bytes): + value = value.decode() + if isinstance(value, str): + value = value.strip() return value +def exists_in_backup(doctypes, file): + """Checks if the list of doctypes exist in the database.sql.gz file supplied + + Args: + doctypes (list): List of DocTypes to be checked + file (str): Path of the database file + + Returns: + bool: True if all tables exist + """ + predicate = ( + 'COPY public."tab{}"' + if frappe.conf.db_type == "postgres" + else "CREATE TABLE `tab{}`" + ) + with gzip.open(file, "rb") as f: + content = f.read().decode("utf8") + return all([predicate.format(doctype).lower() in content.lower() for doctype in doctypes]) + + class BaseTestCommands(unittest.TestCase): def execute(self, command, kwargs=None): site = {"site": frappe.local.site} @@ -26,13 +90,26 @@ class BaseTestCommands(unittest.TestCase): kwargs.update(site) else: kwargs = site - command = command.replace("\n", " ").format(**kwargs) - command = shlex.split(command) + self.command = " ".join(command.split()).format(**kwargs) + print("{0}$ {1}{2}".format(color.silver, self.command, color.nc)) + command = shlex.split(self.command) self._proc = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) self.stdout = clean(self._proc.stdout) self.stderr = clean(self._proc.stderr) self.returncode = clean(self._proc.returncode) + def _formatMessage(self, msg, standardMsg): + output = super(BaseTestCommands, self)._formatMessage(msg, standardMsg) + cmd_execution_summary = "\n".join([ + "-" * 70, + "Last Command Execution Summary:", + "Command: {}".format(self.command) if self.command else "", + "Standard Output: {}".format(self.stdout) if self.stdout else "", + "Standard Error: {}".format(self.stderr) if self.stderr else "", + "Return Code: {}".format(self.returncode) if self.returncode else "", + ]).strip() + return "{}\n\n{}".format(output, cmd_execution_summary) + class TestCommands(BaseTestCommands): def test_execute(self): @@ -52,9 +129,24 @@ class TestCommands(BaseTestCommands): # The returned value has quotes which have been trimmed for the test self.execute("""bench --site {site} execute frappe.bold --kwargs '{{"text": "DocType"}}'""") self.assertEquals(self.returncode, 0) - self.assertEquals(self.stdout[1:-1], frappe.bold(text='DocType')) + self.assertEquals(self.stdout[1:-1], frappe.bold(text="DocType")) def test_backup(self): + backup = { + "includes": { + "includes": [ + "ToDo", + "Note", + ] + }, + "excludes": { + "excludes": [ + "Activity Log", + "Access Log", + "Error Log" + ] + } + } home = os.path.expanduser("~") site_backup_path = frappe.utils.get_site_path("private", "backups") @@ -94,16 +186,19 @@ class TestCommands(BaseTestCommands): "db_path": "database.sql.gz", "files_path": "public.tar", "private_path": "private.tar", - "conf_path": "config.json" + "conf_path": "config.json", }.items() } - self.execute("""bench + self.execute( + """bench --site {site} backup --with-files --backup-path-db {db_path} --backup-path-files {files_path} --backup-path-private-files {private_path} - --backup-path-conf {conf_path}""", kwargs) + --backup-path-conf {conf_path}""", + kwargs, + ) self.assertEquals(self.returncode, 0) for path in kwargs.values(): @@ -111,16 +206,122 @@ class TestCommands(BaseTestCommands): # test 5: take a backup with --compress self.execute("bench --site {site} backup --with-files --compress") - self.assertEquals(self.returncode, 0) - - compressed_files = glob(site_backup_path + "/*.tgz") + compressed_files = glob.glob(site_backup_path + "/*.tgz") self.assertGreater(len(compressed_files), 0) # test 6: take a backup with --verbose self.execute("bench --site {site} backup --verbose") self.assertEquals(self.returncode, 0) + # test 7: take a backup with frappe.conf.backup.includes + self.execute( + "bench --site {site} set-config backup '{includes}' --as-dict", + {"includes": json.dumps(backup["includes"])}, + ) + self.execute("bench --site {site} backup --verbose") + self.assertEquals(self.returncode, 0) + database = fetch_latest_backups(partial=True)["database"] + self.assertTrue(exists_in_backup(backup["includes"]["includes"], database)) + + # test 8: take a backup with frappe.conf.backup.excludes + self.execute( + "bench --site {site} set-config backup '{excludes}' --as-dict", + {"excludes": json.dumps(backup["excludes"])}, + ) + self.execute("bench --site {site} backup --verbose") + self.assertEquals(self.returncode, 0) + database = fetch_latest_backups(partial=True)["database"] + self.assertFalse(exists_in_backup(backup["excludes"]["excludes"], database)) + self.assertTrue(exists_in_backup(backup["includes"]["includes"], database)) + + # test 9: take a backup with --include (with frappe.conf.excludes still set) + self.execute( + "bench --site {site} backup --include '{include}'", + {"include": ",".join(backup["includes"]["includes"])}, + ) + self.assertEquals(self.returncode, 0) + database = fetch_latest_backups(partial=True)["database"] + self.assertTrue(exists_in_backup(backup["includes"]["includes"], database)) + + # test 10: take a backup with --exclude + self.execute( + "bench --site {site} backup --exclude '{exclude}'", + {"exclude": ",".join(backup["excludes"]["excludes"])}, + ) + self.assertEquals(self.returncode, 0) + database = fetch_latest_backups(partial=True)["database"] + self.assertFalse(exists_in_backup(backup["excludes"]["excludes"], database)) + + # test 11: take a backup with --ignore-backup-conf + self.execute("bench --site {site} backup --ignore-backup-conf") + self.assertEquals(self.returncode, 0) + database = fetch_latest_backups()["database"] + self.assertTrue(exists_in_backup(backup["excludes"]["excludes"], database)) + + def test_restore(self): + # step 0: create a site to run the test on + global_config = { + "admin_password": frappe.conf.admin_password, + "root_login": frappe.conf.root_login, + "root_password": frappe.conf.root_password, + "db_type": frappe.conf.db_type, + } + site_data = {"another_site": f"{frappe.local.site}-restore.test", **global_config} + for key, value in global_config.items(): + if value: + self.execute(f"bench set-config {key} {value} -g") + self.execute( + "bench new-site {another_site} --admin-password {admin_password} --db-type" + " {db_type}", + site_data, + ) + + # test 1: bench restore from full backup + self.execute("bench --site {another_site} backup --ignore-backup-conf", site_data) + self.execute( + "bench --site {another_site} execute frappe.utils.backups.fetch_latest_backups", + site_data, + ) + site_data.update({"database": json.loads(self.stdout)["database"]}) + self.execute("bench --site {another_site} restore {database}", site_data) + + # test 2: restore from partial backup + self.execute("bench --site {another_site} backup --exclude 'ToDo'", site_data) + site_data.update({"kw": "\"{'partial':True}\""}) + self.execute( + "bench --site {another_site} execute" + " frappe.utils.backups.fetch_latest_backups --kwargs {kw}", + site_data, + ) + site_data.update({"database": json.loads(self.stdout)["database"]}) + self.execute("bench --site {another_site} restore {database}", site_data) + self.assertEquals(self.returncode, 1) + + def test_partial_restore(self): + _now = now() + for num in range(10): + frappe.get_doc({ + "doctype": "ToDo", + "date": add_to_date(_now, days=num), + "description": frappe.mock("paragraph") + }).insert() + frappe.db.commit() + todo_count = frappe.db.count("ToDo") + + # check if todos exist, create a partial backup and see if the state is the same after restore + self.assertIsNot(todo_count, 0) + self.execute("bench --site {site} backup --only 'ToDo'") + db_path = fetch_latest_backups(partial=True)["database"] + self.assertTrue("partial" in db_path) + + frappe.db.sql_ddl("DROP TABLE IF EXISTS `tabToDo`") + frappe.db.commit() + + self.execute("bench --site {site} partial-restore {path}", {"path": db_path}) + self.assertEquals(self.returncode, 0) + self.assertEquals(frappe.db.count("ToDo"), todo_count) + def test_recorder(self): frappe.recorder.stop() @@ -133,7 +334,6 @@ class TestCommands(BaseTestCommands): self.assertEqual(frappe.recorder.status(), False) def test_remove_from_installed_apps(self): - from frappe.installer import add_to_installed_apps app = "test_remove_app" add_to_installed_apps(app) diff --git a/frappe/utils/backups.py b/frappe/utils/backups.py index bc13158954..3ae300d3c4 100644 --- a/frappe/utils/backups.py +++ b/frappe/utils/backups.py @@ -2,11 +2,12 @@ # MIT License. See license.txt # imports - standard imports -import json +import gzip import os from calendar import timegm from datetime import datetime from glob import glob +from shutil import which # imports - third party imports import click @@ -14,24 +15,42 @@ import click # imports - module imports import frappe from frappe import _, conf -from frappe.utils import get_url, now, now_datetime, get_file_size +from frappe.utils import get_file_size, get_url, now, now_datetime # backup variable for backwards compatibility verbose = False compress = False _verbose = verbose +base_tables = ["__Auth", "__global_search", "__UserSettings"] class BackupGenerator: """ - This class contains methods to perform On Demand Backup + This class contains methods to perform On Demand Backup - To initialize, specify (db_name, user, password, db_file_name=None, db_host="localhost") - If specifying db_file_name, also append ".sql.gz" + To initialize, specify (db_name, user, password, db_file_name=None, db_host="localhost") + If specifying db_file_name, also append ".sql.gz" """ - def __init__(self, db_name, user, password, backup_path=None, backup_path_db=None, - backup_path_files=None, backup_path_private_files=None, db_host="localhost", db_port=None, - verbose=False, db_type='mariadb', backup_path_conf=None, compress_files=False): + + def __init__( + self, + db_name, + user, + password, + backup_path=None, + backup_path_db=None, + backup_path_files=None, + backup_path_private_files=None, + db_host="localhost", + db_port=None, + db_type="mariadb", + backup_path_conf=None, + ignore_conf=False, + compress_files=False, + include_doctypes="", + exclude_doctypes="", + verbose=False, + ): global _verbose self.compress_files = compress_files or compress self.db_host = db_host @@ -45,23 +64,35 @@ class BackupGenerator: self.backup_path_db = backup_path_db self.backup_path_files = backup_path_files self.backup_path_private_files = backup_path_private_files + self.ignore_conf = ignore_conf + self.include_doctypes = include_doctypes + self.exclude_doctypes = exclude_doctypes + self.partial = False if not self.db_type: - self.db_type = 'mariadb' + self.db_type = "mariadb" - if not self.db_port and self.db_type == 'mariadb': - self.db_port = 3306 - elif not self.db_port and self.db_type == 'postgres': - self.db_port = 5432 + if not self.db_port: + if self.db_type == "mariadb": + self.db_port = 3306 + if self.db_type == "postgres": + self.db_port = 5432 site = frappe.local.site or frappe.generate_hash(length=8) - self.site_slug = site.replace('.', '_') + self.site_slug = site.replace(".", "_") self.verbose = verbose self.setup_backup_directory() + self.setup_backup_tables() _verbose = verbose def setup_backup_directory(self): - specified = self.backup_path or self.backup_path_db or self.backup_path_files or self.backup_path_private_files or self.backup_path_conf + specified = ( + self.backup_path + or self.backup_path_db + or self.backup_path_files + or self.backup_path_private_files + or self.backup_path_conf + ) if not specified: backups_folder = get_backup_path() @@ -71,32 +102,93 @@ class BackupGenerator: if self.backup_path: os.makedirs(self.backup_path, exist_ok=True) - for file_path in set([self.backup_path_files, self.backup_path_db, self.backup_path_private_files, self.backup_path_conf]): + for file_path in set( + [ + self.backup_path_files, + self.backup_path_db, + self.backup_path_private_files, + self.backup_path_conf, + ] + ): if file_path: dir = os.path.dirname(file_path) os.makedirs(dir, exist_ok=True) + def setup_backup_tables(self): + """Sets self.backup_includes, self.backup_excludes based on passed args""" + existing_doctypes = set([x.name for x in frappe.get_all("DocType")]) + + def get_tables(doctypes): + tables = [] + for doctype in doctypes: + if doctype and doctype in existing_doctypes: + if doctype.startswith("tab"): + tables.append(doctype) + else: + tables.append("tab" + doctype) + return tables + + passed_tables = { + "include": get_tables(self.include_doctypes.strip().split(",")), + "exclude": get_tables(self.exclude_doctypes.strip().split(",")), + } + specified_tables = get_tables(frappe.conf.get("backup", {}).get("includes", [])) + include_tables = (specified_tables + base_tables) if specified_tables else [] + + conf_tables = { + "include": include_tables, + "exclude": get_tables(frappe.conf.get("backup", {}).get("excludes", [])), + } + + self.backup_includes = passed_tables["include"] + self.backup_excludes = passed_tables["exclude"] + + if not (self.backup_includes or self.backup_excludes) and not self.ignore_conf: + self.backup_includes = self.backup_includes or conf_tables["include"] + self.backup_excludes = self.backup_excludes or conf_tables["exclude"] + + self.partial = (self.backup_includes or self.backup_excludes) and not self.ignore_conf + @property def site_config_backup_path(self): # For backwards compatibility - click.secho("BackupGenerator.site_config_backup_path has been deprecated in favour of BackupGenerator.backup_path_conf", fg="yellow") + click.secho( + "BackupGenerator.site_config_backup_path has been deprecated in favour of" + " BackupGenerator.backup_path_conf", + fg="yellow", + ) return getattr(self, "backup_path_conf", None) def get_backup(self, older_than=24, ignore_files=False, force=False): """ - Takes a new dump if existing file is old - and sends the link to the file as email + Takes a new dump if existing file is old + and sends the link to the file as email """ - #Check if file exists and is less than a day old - #If not Take Dump + # Check if file exists and is less than a day old + # If not Take Dump if not force: - last_db, last_file, last_private_file, site_config_backup_path = self.get_recent_backup(older_than) + ( + last_db, + last_file, + last_private_file, + site_config_backup_path, + ) = self.get_recent_backup(older_than) else: - last_db, last_file, last_private_file, site_config_backup_path = False, False, False, False + last_db, last_file, last_private_file, site_config_backup_path = ( + False, + False, + False, + False, + ) - self.todays_date = now_datetime().strftime('%Y%m%d_%H%M%S') + self.todays_date = now_datetime().strftime("%Y%m%d_%H%M%S") - if not (self.backup_path_conf and self.backup_path_db and self.backup_path_files and self.backup_path_private_files): + if not ( + self.backup_path_conf + and self.backup_path_db + and self.backup_path_files + and self.backup_path_private_files + ): self.set_backup_file_name() if not (last_db and last_file and last_private_file and site_config_backup_path): @@ -112,13 +204,13 @@ class BackupGenerator: self.backup_path_conf = site_config_backup_path def set_backup_file_name(self): - #Generate a random name using today's date and a 8 digit random number - for_conf = self.todays_date + "-" + self.site_slug + "-site_config_backup.json" - for_db = self.todays_date + "-" + self.site_slug + "-database.sql.gz" + partial = "-partial" if self.partial else "" ext = "tgz" if self.compress_files else "tar" - for_public_files = self.todays_date + "-" + self.site_slug + "-files." + ext - for_private_files = self.todays_date + "-" + self.site_slug + "-private-files." + ext + for_conf = f"{self.todays_date}-{self.site_slug}-site_config_backup.json" + for_db = f"{self.todays_date}-{self.site_slug}{partial}-database.sql.gz" + for_public_files = f"{self.todays_date}-{self.site_slug}-files.{ext}" + for_private_files = f"{self.todays_date}-{self.site_slug}-private-files.{ext}" backup_path = self.backup_path or get_backup_path() if not self.backup_path_conf: @@ -130,11 +222,11 @@ class BackupGenerator: if not self.backup_path_private_files: self.backup_path_private_files = os.path.join(backup_path, for_private_files) - def get_recent_backup(self, older_than): + def get_recent_backup(self, older_than, partial=False): backup_path = get_backup_path() file_type_slugs = { - "database": "*-{}-database.sql.gz", + "database": "*-{{}}-{}database.sql.gz".format('*' if partial else ''), "public": "*-{}-files.tar", "private": "*-{}-private-files.tar", "config": "*-{}-site_config_backup.json", @@ -158,8 +250,7 @@ class BackupGenerator: return file_path latest_backups = { - file_type: get_latest(pattern) - for file_type, pattern in file_type_slugs.items() + file_type: get_latest(pattern) for file_type, pattern in file_type_slugs.items() } recent_backups = { @@ -175,32 +266,40 @@ class BackupGenerator: def zip_files(self): # For backwards compatibility - pre v13 - click.secho("BackupGenerator.zip_files has been deprecated in favour of BackupGenerator.backup_files", fg="yellow") + click.secho( + "BackupGenerator.zip_files has been deprecated in favour of" + " BackupGenerator.backup_files", + fg="yellow", + ) return self.backup_files() def get_summary(self): summary = { "config": { "path": self.backup_path_conf, - "size": get_file_size(self.backup_path_conf, format=True) + "size": get_file_size(self.backup_path_conf, format=True), }, "database": { "path": self.backup_path_db, - "size": get_file_size(self.backup_path_db, format=True) - } + "size": get_file_size(self.backup_path_db, format=True), + }, } - if os.path.exists(self.backup_path_files) and os.path.exists(self.backup_path_private_files): - summary.update({ - "public": { - "path": self.backup_path_files, - "size": get_file_size(self.backup_path_files, format=True) - }, - "private": { - "path": self.backup_path_private_files, - "size": get_file_size(self.backup_path_private_files, format=True) + if os.path.exists(self.backup_path_files) and os.path.exists( + self.backup_path_private_files + ): + summary.update( + { + "public": { + "path": self.backup_path_files, + "size": get_file_size(self.backup_path_files, format=True), + }, + "private": { + "path": self.backup_path_private_files, + "size": get_file_size(self.backup_path_private_files, format=True), + }, } - }) + ) return summary @@ -208,21 +307,29 @@ class BackupGenerator: backup_summary = self.get_summary() print("Backup Summary for {0} at {1}".format(frappe.local.site, now())) + title = max([len(x) for x in backup_summary]) + path = max([len(x["path"]) for x in backup_summary.values()]) + for _type, info in backup_summary.items(): - print("{0:8}: {1:85} {2}".format(_type.title(), info["path"], info["size"])) + template = "{{0:{0}}}: {{1:{1}}} {{2}}".format(title, path) + print(template.format(_type.title(), info["path"], info["size"])) def backup_files(self): import subprocess for folder in ("public", "private"): files_path = frappe.get_site_path(folder, "files") - backup_path = self.backup_path_files if folder=="public" else self.backup_path_private_files + backup_path = ( + self.backup_path_files if folder == "public" else self.backup_path_private_files + ) if self.compress_files: cmd_string = "tar cf - {1} | gzip > {0}" else: cmd_string = "tar -cf {0} {1}" - output = subprocess.check_output(cmd_string.format(backup_path, files_path), shell=True) + output = subprocess.check_output( + cmd_string.format(backup_path, files_path), shell=True + ) if self.verbose and output: print(output.decode("utf8")) @@ -236,34 +343,114 @@ class BackupGenerator: def take_dump(self): import frappe.utils + from frappe.utils.change_log import get_app_branch + + db_exc = { + "mariadb": ("mysqldump", which("mysqldump")), + "postgres": ("pg_dump", which("pg_dump")), + }[self.db_type] + gzip_exc = which("gzip") + + if not (gzip_exc and db_exc[1]): + _exc = "gzip" if not gzip_exc else db_exc[0] + frappe.throw( + f"{_exc} not found in PATH! This is required to take a backup.", + exc=frappe.ExecutableNotFound + ) + db_exc = db_exc[0] + + database_header_content = [ + f"Backup generated by Frappe {frappe.__version__} on branch {get_app_branch('frappe') or 'N/A'}", + "", + ] # escape reserved characters - args = dict([item[0], frappe.utils.esc(str(item[1]), '$ ')] - for item in self.__dict__.copy().items()) + args = frappe._dict( + [item[0], frappe.utils.esc(str(item[1]), "$ ")] + for item in self.__dict__.copy().items() + ) - cmd_string = """mysqldump --single-transaction --quick --lock-tables=false -u %(user)s -p%(password)s %(db_name)s -h %(db_host)s -P %(db_port)s | gzip > %(backup_path_db)s """ % args + if self.backup_includes: + backup_info = ("Backing Up Tables: ", ", ".join(self.backup_includes)) + elif self.backup_excludes: + backup_info = ("Skipping Tables: ", ", ".join(self.backup_excludes)) - if self.db_type == 'postgres': - cmd_string = "pg_dump postgres://{user}:{password}@{db_host}:{db_port}/{db_name} | gzip > {backup_path_db}".format( - user=args.get('user'), - password=args.get('password'), - db_host=args.get('db_host'), - db_port=args.get('db_port'), - db_name=args.get('db_name'), - backup_path_db=args.get('backup_path_db') + if self.partial: + print(''.join(backup_info), "\n") + database_header_content.extend([ + f"Partial Backup of Frappe Site {frappe.local.site}", + ("Backup contains: " if self.backup_includes else "Backup excludes: ") + backup_info[1], + "", + ]) + + generated_header = "\n".join([f"-- {x}" for x in database_header_content]) + "\n" + + with gzip.open(args.backup_path_db, "wt") as f: + f.write(generated_header) + + if self.db_type == "postgres": + if self.backup_includes: + args["include"] = " ".join( + ["--table='public.\"{0}\"'".format(table) for table in self.backup_includes] + ) + elif self.backup_excludes: + args["exclude"] = " ".join( + ["--exclude-table-data='public.\"{0}\"'".format(table) for table in self.backup_excludes] + ) + + cmd_string = ( + "{db_exc} postgres://{user}:{password}@{db_host}:{db_port}/{db_name}" + " {include} {exclude} | {gzip} >> {backup_path_db}" ) - err, out = frappe.utils.execute_in_shell(cmd_string) + else: + if self.backup_includes: + args["include"] = " ".join(["'{0}'".format(x) for x in self.backup_includes]) + elif self.backup_excludes: + args["exclude"] = " ".join( + [ + "--ignore-table='{0}.{1}'".format(frappe.conf.db_name, table) + for table in self.backup_excludes + ] + ) + + cmd_string = ( + "{db_exc} --single-transaction --quick --lock-tables=false -u {user}" + " -p{password} {db_name} -h {db_host} -P {db_port} {include} {exclude}" + " | {gzip} >> {backup_path_db}" + ) + + command = cmd_string.format( + user=args.user, + password=args.password, + db_exc=db_exc, + db_host=args.db_host, + db_port=args.db_port, + db_name=args.db_name, + backup_path_db=args.backup_path_db, + exclude=args.get("exclude", ""), + include=args.get("include", ""), + gzip=gzip_exc, + ) + + if self.verbose: + print(command + "\n") + + err, out = frappe.utils.execute_in_shell(command) def send_email(self): """ - Sends the link to backup file located at erpnext/backups + Sends the link to backup file located at erpnext/backups """ from frappe.email import get_system_managers recipient_list = get_system_managers() - db_backup_url = get_url(os.path.join('backups', os.path.basename(self.backup_path_db))) - files_backup_url = get_url(os.path.join('backups', os.path.basename(self.backup_path_files))) + db_backup_url = get_url( + os.path.join("backups", os.path.basename(self.backup_path_db)) + ) + files_backup_url = get_url( + os.path.join("backups", os.path.basename(self.backup_path_files)) + ) msg = """Hello, @@ -275,11 +462,13 @@ Your backups are ready to be downloaded. This link will be valid for 24 hours. A new backup will be available for download only after 24 hours.""" % { "db_backup_url": db_backup_url, - "files_backup_url": files_backup_url + "files_backup_url": files_backup_url, } datetime_str = datetime.fromtimestamp(os.stat(self.backup_path_db).st_ctime) - subject = datetime_str.strftime("%d/%m/%Y %H:%M:%S") + """ - Backup ready to be downloaded""" + subject = ( + datetime_str.strftime("%d/%m/%Y %H:%M:%S") + """ - Backup ready to be downloaded""" + ) frappe.sendmail(recipients=recipient_list, msg=msg, subject=subject) return recipient_list @@ -288,20 +477,29 @@ download only after 24 hours.""" % { @frappe.whitelist() def get_backup(): """ - This function is executed when the user clicks on - Toos > Download Backup + This function is executed when the user clicks on + Toos > Download Backup """ delete_temp_backups() - odb = BackupGenerator(frappe.conf.db_name, frappe.conf.db_name,\ - frappe.conf.db_password, db_host = frappe.db.host,\ - db_type=frappe.conf.db_type, db_port=frappe.conf.db_port) + odb = BackupGenerator( + frappe.conf.db_name, + frappe.conf.db_name, + frappe.conf.db_password, + db_host=frappe.db.host, + db_type=frappe.conf.db_type, + db_port=frappe.conf.db_port, + ) odb.get_backup() recipient_list = odb.send_email() - frappe.msgprint(_("Download link for your backup will be emailed on the following email address: {0}").format(', '.join(recipient_list))) + frappe.msgprint( + _( + "Download link for your backup will be emailed on the following email address: {0}" + ).format(", ".join(recipient_list)) + ) @frappe.whitelist() -def fetch_latest_backups(): +def fetch_latest_backups(partial=False): """Fetches paths of the latest backup taken in the last 30 days Only for: System Managers @@ -317,43 +515,88 @@ def fetch_latest_backups(): db_type=frappe.conf.db_type, db_port=frappe.conf.db_port, ) - database, public, private, config = odb.get_recent_backup(older_than=24 * 30) + database, public, private, config = odb.get_recent_backup(older_than=24 * 30, partial=partial) - return { - "database": database, - "public": public, - "private": private, - "config": config - } + return {"database": database, "public": public, "private": private, "config": config} -def scheduled_backup(older_than=6, ignore_files=False, backup_path=None, backup_path_db=None, backup_path_files=None, backup_path_private_files=None, backup_path_conf=None, force=False, verbose=False, compress=False): +def scheduled_backup( + older_than=6, + ignore_files=False, + backup_path=None, + backup_path_db=None, + backup_path_files=None, + backup_path_private_files=None, + backup_path_conf=None, + ignore_conf=False, + include_doctypes="", + exclude_doctypes="", + compress=False, + force=False, + verbose=False, +): """this function is called from scheduler - deletes backups older than 7 days - takes backup""" - odb = new_backup(older_than, ignore_files, backup_path=backup_path, backup_path_db=backup_path_db, backup_path_files=backup_path_files, backup_path_private_files=backup_path_private_files, backup_path_conf=backup_path_conf, force=force, verbose=verbose, compress=compress) + deletes backups older than 7 days + takes backup""" + odb = new_backup( + older_than=older_than, + ignore_files=ignore_files, + backup_path=backup_path, + backup_path_db=backup_path_db, + backup_path_files=backup_path_files, + backup_path_private_files=backup_path_private_files, + backup_path_conf=backup_path_conf, + ignore_conf=ignore_conf, + include_doctypes=include_doctypes, + exclude_doctypes=exclude_doctypes, + compress=compress, + force=force, + verbose=verbose, + ) return odb -def new_backup(older_than=6, ignore_files=False, backup_path=None, backup_path_db=None, backup_path_files=None, backup_path_private_files=None, backup_path_conf=None, force=False, verbose=False, compress=False): - delete_temp_backups(older_than = frappe.conf.keep_backups_for_hours or 24) - odb = BackupGenerator(frappe.conf.db_name, frappe.conf.db_name,\ - frappe.conf.db_password, - backup_path=backup_path, - backup_path_db=backup_path_db, - backup_path_files=backup_path_files, - backup_path_private_files=backup_path_private_files, - backup_path_conf=backup_path_conf, - db_host = frappe.db.host, - db_port = frappe.db.port, - db_type = frappe.conf.db_type, - verbose=verbose, - compress_files=compress) + +def new_backup( + older_than=6, + ignore_files=False, + backup_path=None, + backup_path_db=None, + backup_path_files=None, + backup_path_private_files=None, + backup_path_conf=None, + ignore_conf=False, + include_doctypes="", + exclude_doctypes="", + compress=False, + force=False, + verbose=False, +): + delete_temp_backups(older_than=frappe.conf.keep_backups_for_hours or 24) + odb = BackupGenerator( + frappe.conf.db_name, + frappe.conf.db_name, + frappe.conf.db_password, + db_host=frappe.db.host, + db_port=frappe.db.port, + db_type=frappe.conf.db_type, + backup_path=backup_path, + backup_path_db=backup_path_db, + backup_path_files=backup_path_files, + backup_path_private_files=backup_path_private_files, + backup_path_conf=backup_path_conf, + ignore_conf=ignore_conf, + include_doctypes=include_doctypes, + exclude_doctypes=exclude_doctypes, + verbose=verbose, + compress_files=compress, + ) odb.get_backup(older_than, ignore_files, force=force) return odb + def delete_temp_backups(older_than=24): """ - Cleans up the backup_link_path directory by deleting files older than 24 hours + Cleans up the backup_link_path directory by deleting files older than 24 hours """ backup_path = get_backup_path() if os.path.exists(backup_path): @@ -363,54 +606,68 @@ def delete_temp_backups(older_than=24): if is_file_old(this_file_path, older_than): os.remove(this_file_path) -def is_file_old(db_file_name, older_than=24): - """ - Checks if file exists and is older than specified hours - Returns -> - True: file does not exist or file is old - False: file is new - """ - if os.path.isfile(db_file_name): - from datetime import timedelta - #Get timestamp of the file - file_datetime = datetime.fromtimestamp\ - (os.stat(db_file_name).st_ctime) - if datetime.today() - file_datetime >= timedelta(hours = older_than): - if _verbose: - print("File is old") - return True - else: - if _verbose: - print("File is recent") - return False + +def is_file_old(file_path, older_than=24): + """ + Checks if file exists and is older than specified hours + Returns -> + True: file does not exist or file is old + False: file is new + """ + if os.path.isfile(file_path): + from datetime import timedelta + + # Get timestamp of the file + file_datetime = datetime.fromtimestamp(os.stat(file_path).st_ctime) + if datetime.today() - file_datetime >= timedelta(hours=older_than): + if _verbose: + print(f"File {file_path} is older than {older_than} hours") + return True else: if _verbose: - print("File does not exist") - return True + print(f"File {file_path} is recent") + return False + else: + if _verbose: + print(f"File {file_path} does not exist") + return True + def get_backup_path(): backup_path = frappe.utils.get_site_path(conf.get("backup_path", "private/backups")) return backup_path -def backup(with_files=False, backup_path_db=None, backup_path_files=None, backup_path_private_files=None, backup_path_conf=None, quiet=False): + +def backup( + with_files=False, + backup_path_db=None, + backup_path_files=None, + backup_path_private_files=None, + backup_path_conf=None, + quiet=False, +): "Backup" - odb = scheduled_backup(ignore_files=not with_files, backup_path_db=backup_path_db, backup_path_files=backup_path_files, backup_path_private_files=backup_path_private_files, backup_path_conf=backup_path_conf, force=True) + odb = scheduled_backup( + ignore_files=not with_files, + backup_path_db=backup_path_db, + backup_path_files=backup_path_files, + backup_path_private_files=backup_path_private_files, + backup_path_conf=backup_path_conf, + force=True, + ) return { "backup_path_db": odb.backup_path_db, "backup_path_files": odb.backup_path_files, - "backup_path_private_files": odb.backup_path_private_files + "backup_path_private_files": odb.backup_path_private_files, } if __name__ == "__main__": - """ - is_file_old db_name user password db_host db_type db_port - get_backup db_name user password db_host db_type db_port - """ import sys + cmd = sys.argv[1] - db_type = 'mariadb' + db_type = "mariadb" try: db_type = sys.argv[6] except IndexError: @@ -423,19 +680,47 @@ if __name__ == "__main__": pass if cmd == "is_file_old": - odb = BackupGenerator(sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5] or "localhost", db_type=db_type, db_port=db_port) + odb = BackupGenerator( + sys.argv[2], + sys.argv[3], + sys.argv[4], + sys.argv[5] or "localhost", + db_type=db_type, + db_port=db_port, + ) is_file_old(odb.db_file_name) if cmd == "get_backup": - odb = BackupGenerator(sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5] or "localhost", db_type=db_type, db_port=db_port) + odb = BackupGenerator( + sys.argv[2], + sys.argv[3], + sys.argv[4], + sys.argv[5] or "localhost", + db_type=db_type, + db_port=db_port, + ) odb.get_backup() if cmd == "take_dump": - odb = BackupGenerator(sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5] or "localhost", db_type=db_type, db_port=db_port) + odb = BackupGenerator( + sys.argv[2], + sys.argv[3], + sys.argv[4], + sys.argv[5] or "localhost", + db_type=db_type, + db_port=db_port, + ) odb.take_dump() if cmd == "send_email": - odb = BackupGenerator(sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5] or "localhost", db_type=db_type, db_port=db_port) + odb = BackupGenerator( + sys.argv[2], + sys.argv[3], + sys.argv[4], + sys.argv[5] or "localhost", + db_type=db_type, + db_port=db_port, + ) odb.send_email("abc.sql.gz") if cmd == "delete_temp_backups":