687 lines
18 KiB
Python
687 lines
18 KiB
Python
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
|
|
# MIT License. See license.txt
|
|
|
|
# imports - standard imports
|
|
import os
|
|
from calendar import timegm
|
|
from datetime import datetime
|
|
from glob import glob
|
|
|
|
# imports - third party imports
|
|
import click
|
|
|
|
# imports - module imports
|
|
import frappe
|
|
from frappe import _, conf
|
|
from frappe.utils import get_url, now, now_datetime, get_file_size
|
|
|
|
# backup variable for backwards compatibility
|
|
verbose = False
|
|
compress = False
|
|
_verbose = verbose
|
|
base_tables = ["__Auth", "__global_search", "__UserSettings"]
|
|
|
|
|
|
class BackupGenerator:
|
|
"""
|
|
This class contains methods to perform On Demand Backup
|
|
|
|
To initialize, specify (db_name, user, password, db_file_name=None, db_host="localhost")
|
|
If specifying db_file_name, also append ".sql.gz"
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
db_name,
|
|
user,
|
|
password,
|
|
backup_path=None,
|
|
backup_path_db=None,
|
|
backup_path_files=None,
|
|
backup_path_private_files=None,
|
|
db_host="localhost",
|
|
db_port=None,
|
|
db_type="mariadb",
|
|
backup_path_conf=None,
|
|
ignore_conf=False,
|
|
compress_files=False,
|
|
include_doctypes="",
|
|
exclude_doctypes="",
|
|
verbose=False,
|
|
):
|
|
global _verbose
|
|
self.compress_files = compress_files or compress
|
|
self.db_host = db_host
|
|
self.db_port = db_port
|
|
self.db_name = db_name
|
|
self.db_type = db_type
|
|
self.user = user
|
|
self.password = password
|
|
self.backup_path = backup_path
|
|
self.backup_path_conf = backup_path_conf
|
|
self.backup_path_db = backup_path_db
|
|
self.backup_path_files = backup_path_files
|
|
self.backup_path_private_files = backup_path_private_files
|
|
self.ignore_conf = ignore_conf
|
|
self.include_doctypes = include_doctypes
|
|
self.exclude_doctypes = exclude_doctypes
|
|
|
|
if not self.db_type:
|
|
self.db_type = "mariadb"
|
|
|
|
if not self.db_port:
|
|
if self.db_type == "mariadb":
|
|
self.db_port = 3306
|
|
if self.db_type == "postgres":
|
|
self.db_port = 5432
|
|
|
|
site = frappe.local.site or frappe.generate_hash(length=8)
|
|
self.site_slug = site.replace(".", "_")
|
|
self.verbose = verbose
|
|
self.setup_backup_directory()
|
|
self.setup_backup_tables()
|
|
_verbose = verbose
|
|
|
|
def setup_backup_directory(self):
|
|
specified = (
|
|
self.backup_path
|
|
or self.backup_path_db
|
|
or self.backup_path_files
|
|
or self.backup_path_private_files
|
|
or self.backup_path_conf
|
|
)
|
|
|
|
if not specified:
|
|
backups_folder = get_backup_path()
|
|
if not os.path.exists(backups_folder):
|
|
os.makedirs(backups_folder, exist_ok=True)
|
|
else:
|
|
if self.backup_path:
|
|
os.makedirs(self.backup_path, exist_ok=True)
|
|
|
|
for file_path in set(
|
|
[
|
|
self.backup_path_files,
|
|
self.backup_path_db,
|
|
self.backup_path_private_files,
|
|
self.backup_path_conf,
|
|
]
|
|
):
|
|
if file_path:
|
|
dir = os.path.dirname(file_path)
|
|
os.makedirs(dir, exist_ok=True)
|
|
|
|
def setup_backup_tables(self):
|
|
"""Sets self.backup_includes, self.backup_excludes based on passed args"""
|
|
existing_doctypes = set([x.name for x in frappe.get_all("DocType")])
|
|
|
|
def get_tables(doctypes):
|
|
tables = []
|
|
for doctype in doctypes:
|
|
if doctype and doctype in existing_doctypes:
|
|
if doctype.startswith("tab"):
|
|
tables.append(doctype)
|
|
else:
|
|
tables.append("tab" + doctype)
|
|
return tables
|
|
|
|
passed_tables = {
|
|
"include": get_tables(self.include_doctypes.strip().split(",")),
|
|
"exclude": get_tables(self.exclude_doctypes.strip().split(",")),
|
|
}
|
|
specified_tables = get_tables(frappe.conf.get("backup", {}).get("includes", []))
|
|
include_tables = (specified_tables + base_tables) if specified_tables else []
|
|
|
|
conf_tables = {
|
|
"include": include_tables,
|
|
"exclude": get_tables(frappe.conf.get("backup", {}).get("excludes", [])),
|
|
}
|
|
|
|
self.backup_includes = passed_tables["include"]
|
|
self.backup_excludes = passed_tables["exclude"]
|
|
|
|
if not (self.backup_includes or self.backup_excludes) and not self.ignore_conf:
|
|
self.backup_includes = self.backup_includes or conf_tables["include"]
|
|
self.backup_excludes = self.backup_excludes or conf_tables["exclude"]
|
|
|
|
@property
|
|
def site_config_backup_path(self):
|
|
# For backwards compatibility
|
|
click.secho(
|
|
"BackupGenerator.site_config_backup_path has been deprecated in favour of"
|
|
" BackupGenerator.backup_path_conf",
|
|
fg="yellow",
|
|
)
|
|
return getattr(self, "backup_path_conf", None)
|
|
|
|
def get_backup(self, older_than=24, ignore_files=False, force=False):
|
|
"""
|
|
Takes a new dump if existing file is old
|
|
and sends the link to the file as email
|
|
"""
|
|
# Check if file exists and is less than a day old
|
|
# If not Take Dump
|
|
if not force:
|
|
(
|
|
last_db,
|
|
last_file,
|
|
last_private_file,
|
|
site_config_backup_path,
|
|
) = self.get_recent_backup(older_than)
|
|
else:
|
|
last_db, last_file, last_private_file, site_config_backup_path = (
|
|
False,
|
|
False,
|
|
False,
|
|
False,
|
|
)
|
|
|
|
self.todays_date = now_datetime().strftime("%Y%m%d_%H%M%S")
|
|
|
|
if not (
|
|
self.backup_path_conf
|
|
and self.backup_path_db
|
|
and self.backup_path_files
|
|
and self.backup_path_private_files
|
|
):
|
|
self.set_backup_file_name()
|
|
|
|
if not (last_db and last_file and last_private_file and site_config_backup_path):
|
|
self.take_dump()
|
|
self.copy_site_config()
|
|
if not ignore_files:
|
|
self.backup_files()
|
|
|
|
else:
|
|
self.backup_path_files = last_file
|
|
self.backup_path_db = last_db
|
|
self.backup_path_private_files = last_private_file
|
|
self.backup_path_conf = site_config_backup_path
|
|
|
|
def set_backup_file_name(self):
|
|
# Generate a random name using today's date and a 8 digit random number
|
|
for_conf = self.todays_date + "-" + self.site_slug + "-site_config_backup.json"
|
|
for_db = self.todays_date + "-" + self.site_slug + "-database.sql.gz"
|
|
ext = "tgz" if self.compress_files else "tar"
|
|
|
|
for_public_files = self.todays_date + "-" + self.site_slug + "-files." + ext
|
|
for_private_files = self.todays_date + "-" + self.site_slug + "-private-files." + ext
|
|
backup_path = self.backup_path or get_backup_path()
|
|
|
|
if not self.backup_path_conf:
|
|
self.backup_path_conf = os.path.join(backup_path, for_conf)
|
|
if not self.backup_path_db:
|
|
self.backup_path_db = os.path.join(backup_path, for_db)
|
|
if not self.backup_path_files:
|
|
self.backup_path_files = os.path.join(backup_path, for_public_files)
|
|
if not self.backup_path_private_files:
|
|
self.backup_path_private_files = os.path.join(backup_path, for_private_files)
|
|
|
|
def get_recent_backup(self, older_than):
|
|
backup_path = get_backup_path()
|
|
|
|
file_type_slugs = {
|
|
"database": "*-{}-database.sql.gz",
|
|
"public": "*-{}-files.tar",
|
|
"private": "*-{}-private-files.tar",
|
|
"config": "*-{}-site_config_backup.json",
|
|
}
|
|
|
|
def backup_time(file_path):
|
|
file_name = file_path.split(os.sep)[-1]
|
|
file_timestamp = file_name.split("-")[0]
|
|
return timegm(datetime.strptime(file_timestamp, "%Y%m%d_%H%M%S").utctimetuple())
|
|
|
|
def get_latest(file_pattern):
|
|
file_pattern = os.path.join(backup_path, file_pattern.format(self.site_slug))
|
|
file_list = glob(file_pattern)
|
|
if file_list:
|
|
return max(file_list, key=backup_time)
|
|
|
|
def old_enough(file_path):
|
|
if file_path:
|
|
if not os.path.isfile(file_path) or is_file_old(file_path, older_than):
|
|
return None
|
|
return file_path
|
|
|
|
latest_backups = {
|
|
file_type: get_latest(pattern) for file_type, pattern in file_type_slugs.items()
|
|
}
|
|
|
|
recent_backups = {
|
|
file_type: old_enough(file_name) for file_type, file_name in latest_backups.items()
|
|
}
|
|
|
|
return (
|
|
recent_backups.get("database"),
|
|
recent_backups.get("public"),
|
|
recent_backups.get("private"),
|
|
recent_backups.get("config"),
|
|
)
|
|
|
|
def zip_files(self):
|
|
# For backwards compatibility - pre v13
|
|
click.secho(
|
|
"BackupGenerator.zip_files has been deprecated in favour of"
|
|
" BackupGenerator.backup_files",
|
|
fg="yellow",
|
|
)
|
|
return self.backup_files()
|
|
|
|
def get_summary(self):
|
|
summary = {
|
|
"config": {
|
|
"path": self.backup_path_conf,
|
|
"size": get_file_size(self.backup_path_conf, format=True),
|
|
},
|
|
"database": {
|
|
"path": self.backup_path_db,
|
|
"size": get_file_size(self.backup_path_db, format=True),
|
|
},
|
|
}
|
|
|
|
if os.path.exists(self.backup_path_files) and os.path.exists(
|
|
self.backup_path_private_files
|
|
):
|
|
summary.update(
|
|
{
|
|
"public": {
|
|
"path": self.backup_path_files,
|
|
"size": get_file_size(self.backup_path_files, format=True),
|
|
},
|
|
"private": {
|
|
"path": self.backup_path_private_files,
|
|
"size": get_file_size(self.backup_path_private_files, format=True),
|
|
},
|
|
}
|
|
)
|
|
|
|
return summary
|
|
|
|
def print_summary(self):
|
|
backup_summary = self.get_summary()
|
|
print("Backup Summary for {0} at {1}".format(frappe.local.site, now()))
|
|
|
|
title = max([len(x) for x in backup_summary])
|
|
path = max([len(x["path"]) for x in backup_summary.values()])
|
|
|
|
for _type, info in backup_summary.items():
|
|
template = "{{0:{0}}}: {{1:{1}}} {{2}}".format(title, path)
|
|
print(template.format(_type.title(), info["path"], info["size"]))
|
|
|
|
def backup_files(self):
|
|
import subprocess
|
|
|
|
for folder in ("public", "private"):
|
|
files_path = frappe.get_site_path(folder, "files")
|
|
backup_path = (
|
|
self.backup_path_files if folder == "public" else self.backup_path_private_files
|
|
)
|
|
|
|
if self.compress_files:
|
|
cmd_string = "tar cf - {1} | gzip > {0}"
|
|
else:
|
|
cmd_string = "tar -cf {0} {1}"
|
|
output = subprocess.check_output(
|
|
cmd_string.format(backup_path, files_path), shell=True
|
|
)
|
|
|
|
if self.verbose and output:
|
|
print(output.decode("utf8"))
|
|
|
|
def copy_site_config(self):
|
|
site_config_backup_path = self.backup_path_conf
|
|
site_config_path = os.path.join(frappe.get_site_path(), "site_config.json")
|
|
|
|
with open(site_config_backup_path, "w") as n, open(site_config_path) as c:
|
|
n.write(c.read())
|
|
|
|
def take_dump(self):
|
|
import frappe.utils
|
|
|
|
# escape reserved characters
|
|
args = dict(
|
|
[item[0], frappe.utils.esc(str(item[1]), "$ ")]
|
|
for item in self.__dict__.copy().items()
|
|
)
|
|
|
|
if self.backup_includes:
|
|
print("Backing Up Tables: {0}\n".format(", ".join(self.backup_includes)))
|
|
elif self.backup_excludes:
|
|
print("Skipping Tables: {0}\n".format(", ".join(self.backup_excludes)))
|
|
|
|
if self.db_type == "postgres":
|
|
if self.backup_includes:
|
|
args["include"] = " ".join(
|
|
["--table='{0}'".format(table) for table in self.backup_includes]
|
|
)
|
|
elif self.backup_excludes:
|
|
args["exclude"] = " ".join(
|
|
["--exclude-table='{0}'".format(table) for table in self.backup_excludes]
|
|
)
|
|
|
|
cmd_string = (
|
|
"pg_dump postgres://{user}:{password}@{db_host}:{db_port}/{db_name}"
|
|
" {include} {exclude} | gzip > {backup_path_db}"
|
|
)
|
|
|
|
else:
|
|
if self.backup_includes:
|
|
args["include"] = " ".join(["'{0}'".format(x) for x in self.backup_includes])
|
|
elif self.backup_excludes:
|
|
args["exclude"] = " ".join(
|
|
[
|
|
"--ignore-table='{0}.{1}'".format(frappe.conf.db_name, table)
|
|
for table in self.backup_excludes
|
|
]
|
|
)
|
|
|
|
cmd_string = (
|
|
"mysqldump --single-transaction --quick --lock-tables=false -u {user}"
|
|
" -p{password} {db_name} -h {db_host} -P {db_port} {include} {exclude}"
|
|
" | gzip > {backup_path_db}"
|
|
)
|
|
|
|
command = cmd_string.format(
|
|
user=args.get("user"),
|
|
password=args.get("password"),
|
|
db_host=args.get("db_host"),
|
|
db_port=args.get("db_port"),
|
|
db_name=args.get("db_name"),
|
|
backup_path_db=args.get("backup_path_db"),
|
|
exclude=args.get("exclude", ""),
|
|
include=args.get("include", ""),
|
|
)
|
|
|
|
if self.verbose:
|
|
print(command + "\n")
|
|
|
|
err, out = frappe.utils.execute_in_shell(command)
|
|
|
|
def send_email(self):
|
|
"""
|
|
Sends the link to backup file located at erpnext/backups
|
|
"""
|
|
from frappe.email import get_system_managers
|
|
|
|
recipient_list = get_system_managers()
|
|
db_backup_url = get_url(
|
|
os.path.join("backups", os.path.basename(self.backup_path_db))
|
|
)
|
|
files_backup_url = get_url(
|
|
os.path.join("backups", os.path.basename(self.backup_path_files))
|
|
)
|
|
|
|
msg = """Hello,
|
|
|
|
Your backups are ready to be downloaded.
|
|
|
|
1. [Click here to download the database backup](%(db_backup_url)s)
|
|
2. [Click here to download the files backup](%(files_backup_url)s)
|
|
|
|
This link will be valid for 24 hours. A new backup will be available for
|
|
download only after 24 hours.""" % {
|
|
"db_backup_url": db_backup_url,
|
|
"files_backup_url": files_backup_url,
|
|
}
|
|
|
|
datetime_str = datetime.fromtimestamp(os.stat(self.backup_path_db).st_ctime)
|
|
subject = (
|
|
datetime_str.strftime("%d/%m/%Y %H:%M:%S") + """ - Backup ready to be downloaded"""
|
|
)
|
|
|
|
frappe.sendmail(recipients=recipient_list, msg=msg, subject=subject)
|
|
return recipient_list
|
|
|
|
|
|
@frappe.whitelist()
|
|
def get_backup():
|
|
"""
|
|
This function is executed when the user clicks on
|
|
Toos > Download Backup
|
|
"""
|
|
delete_temp_backups()
|
|
odb = BackupGenerator(
|
|
frappe.conf.db_name,
|
|
frappe.conf.db_name,
|
|
frappe.conf.db_password,
|
|
db_host=frappe.db.host,
|
|
db_type=frappe.conf.db_type,
|
|
db_port=frappe.conf.db_port,
|
|
)
|
|
odb.get_backup()
|
|
recipient_list = odb.send_email()
|
|
frappe.msgprint(
|
|
_(
|
|
"Download link for your backup will be emailed on the following email address: {0}"
|
|
).format(", ".join(recipient_list))
|
|
)
|
|
|
|
|
|
@frappe.whitelist()
|
|
def fetch_latest_backups():
|
|
"""Fetches paths of the latest backup taken in the last 30 days
|
|
Only for: System Managers
|
|
|
|
Returns:
|
|
dict: relative Backup Paths
|
|
"""
|
|
frappe.only_for("System Manager")
|
|
odb = BackupGenerator(
|
|
frappe.conf.db_name,
|
|
frappe.conf.db_name,
|
|
frappe.conf.db_password,
|
|
db_host=frappe.db.host,
|
|
db_type=frappe.conf.db_type,
|
|
db_port=frappe.conf.db_port,
|
|
)
|
|
database, public, private, config = odb.get_recent_backup(older_than=24 * 30)
|
|
|
|
return {"database": database, "public": public, "private": private, "config": config}
|
|
|
|
|
|
def scheduled_backup(
|
|
older_than=6,
|
|
ignore_files=False,
|
|
backup_path=None,
|
|
backup_path_db=None,
|
|
backup_path_files=None,
|
|
backup_path_private_files=None,
|
|
backup_path_conf=None,
|
|
ignore_conf=False,
|
|
include_doctypes="",
|
|
exclude_doctypes="",
|
|
compress=False,
|
|
force=False,
|
|
verbose=False,
|
|
):
|
|
"""this function is called from scheduler
|
|
deletes backups older than 7 days
|
|
takes backup"""
|
|
odb = new_backup(
|
|
older_than=older_than,
|
|
ignore_files=ignore_files,
|
|
backup_path=backup_path,
|
|
backup_path_db=backup_path_db,
|
|
backup_path_files=backup_path_files,
|
|
backup_path_private_files=backup_path_private_files,
|
|
backup_path_conf=backup_path_conf,
|
|
ignore_conf=ignore_conf,
|
|
include_doctypes=include_doctypes,
|
|
exclude_doctypes=exclude_doctypes,
|
|
compress=compress,
|
|
force=force,
|
|
verbose=verbose,
|
|
)
|
|
return odb
|
|
|
|
|
|
def new_backup(
|
|
older_than=6,
|
|
ignore_files=False,
|
|
backup_path=None,
|
|
backup_path_db=None,
|
|
backup_path_files=None,
|
|
backup_path_private_files=None,
|
|
backup_path_conf=None,
|
|
ignore_conf=False,
|
|
include_doctypes="",
|
|
exclude_doctypes="",
|
|
compress=False,
|
|
force=False,
|
|
verbose=False,
|
|
):
|
|
delete_temp_backups(older_than=frappe.conf.keep_backups_for_hours or 24)
|
|
odb = BackupGenerator(
|
|
frappe.conf.db_name,
|
|
frappe.conf.db_name,
|
|
frappe.conf.db_password,
|
|
db_host=frappe.db.host,
|
|
db_port=frappe.db.port,
|
|
db_type=frappe.conf.db_type,
|
|
backup_path=backup_path,
|
|
backup_path_db=backup_path_db,
|
|
backup_path_files=backup_path_files,
|
|
backup_path_private_files=backup_path_private_files,
|
|
backup_path_conf=backup_path_conf,
|
|
ignore_conf=ignore_conf,
|
|
include_doctypes=include_doctypes,
|
|
exclude_doctypes=exclude_doctypes,
|
|
verbose=verbose,
|
|
compress_files=compress,
|
|
)
|
|
odb.get_backup(older_than, ignore_files, force=force)
|
|
return odb
|
|
|
|
|
|
def delete_temp_backups(older_than=24):
|
|
"""
|
|
Cleans up the backup_link_path directory by deleting files older than 24 hours
|
|
"""
|
|
backup_path = get_backup_path()
|
|
if os.path.exists(backup_path):
|
|
file_list = os.listdir(get_backup_path())
|
|
for this_file in file_list:
|
|
this_file_path = os.path.join(get_backup_path(), this_file)
|
|
if is_file_old(this_file_path, older_than):
|
|
os.remove(this_file_path)
|
|
|
|
|
|
def is_file_old(db_file_name, older_than=24):
|
|
"""
|
|
Checks if file exists and is older than specified hours
|
|
Returns ->
|
|
True: file does not exist or file is old
|
|
False: file is new
|
|
"""
|
|
if os.path.isfile(db_file_name):
|
|
from datetime import timedelta
|
|
|
|
# Get timestamp of the file
|
|
file_datetime = datetime.fromtimestamp(os.stat(db_file_name).st_ctime)
|
|
if datetime.today() - file_datetime >= timedelta(hours=older_than):
|
|
if _verbose:
|
|
print("File is old")
|
|
return True
|
|
else:
|
|
if _verbose:
|
|
print("File is recent")
|
|
return False
|
|
else:
|
|
if _verbose:
|
|
print("File does not exist")
|
|
return True
|
|
|
|
|
|
def get_backup_path():
|
|
backup_path = frappe.utils.get_site_path(conf.get("backup_path", "private/backups"))
|
|
return backup_path
|
|
|
|
|
|
def backup(
|
|
with_files=False,
|
|
backup_path_db=None,
|
|
backup_path_files=None,
|
|
backup_path_private_files=None,
|
|
backup_path_conf=None,
|
|
quiet=False,
|
|
):
|
|
"Backup"
|
|
odb = scheduled_backup(
|
|
ignore_files=not with_files,
|
|
backup_path_db=backup_path_db,
|
|
backup_path_files=backup_path_files,
|
|
backup_path_private_files=backup_path_private_files,
|
|
backup_path_conf=backup_path_conf,
|
|
force=True,
|
|
)
|
|
return {
|
|
"backup_path_db": odb.backup_path_db,
|
|
"backup_path_files": odb.backup_path_files,
|
|
"backup_path_private_files": odb.backup_path_private_files,
|
|
}
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import sys
|
|
|
|
cmd = sys.argv[1]
|
|
|
|
db_type = "mariadb"
|
|
try:
|
|
db_type = sys.argv[6]
|
|
except IndexError:
|
|
pass
|
|
|
|
db_port = 3306
|
|
try:
|
|
db_port = int(sys.argv[7])
|
|
except IndexError:
|
|
pass
|
|
|
|
if cmd == "is_file_old":
|
|
odb = BackupGenerator(
|
|
sys.argv[2],
|
|
sys.argv[3],
|
|
sys.argv[4],
|
|
sys.argv[5] or "localhost",
|
|
db_type=db_type,
|
|
db_port=db_port,
|
|
)
|
|
is_file_old(odb.db_file_name)
|
|
|
|
if cmd == "get_backup":
|
|
odb = BackupGenerator(
|
|
sys.argv[2],
|
|
sys.argv[3],
|
|
sys.argv[4],
|
|
sys.argv[5] or "localhost",
|
|
db_type=db_type,
|
|
db_port=db_port,
|
|
)
|
|
odb.get_backup()
|
|
|
|
if cmd == "take_dump":
|
|
odb = BackupGenerator(
|
|
sys.argv[2],
|
|
sys.argv[3],
|
|
sys.argv[4],
|
|
sys.argv[5] or "localhost",
|
|
db_type=db_type,
|
|
db_port=db_port,
|
|
)
|
|
odb.take_dump()
|
|
|
|
if cmd == "send_email":
|
|
odb = BackupGenerator(
|
|
sys.argv[2],
|
|
sys.argv[3],
|
|
sys.argv[4],
|
|
sys.argv[5] or "localhost",
|
|
db_type=db_type,
|
|
db_port=db_port,
|
|
)
|
|
odb.send_email("abc.sql.gz")
|
|
|
|
if cmd == "delete_temp_backups":
|
|
delete_temp_backups()
|