seitime-frappe/frappe/utils/backups.py

# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt

"""This module handles the On Demand Backup utility"""

from __future__ import print_function, unicode_literals

import os
import json
from calendar import timegm
from datetime import datetime
from glob import glob

import frappe
from frappe import _, conf
from frappe.utils import cstr, get_url, now_datetime

# backup variable for backwards compatibility
verbose = False
compress = False
_verbose = verbose


class BackupGenerator:
	"""
		This class contains methods to perform On Demand Backup

		To initialize, specify (db_name, user, password, db_file_name=None, db_host="localhost")
		If specifying db_file_name, also append ".sql.gz"
	"""
	def __init__(self, db_name, user, password, backup_path_db=None, backup_path_files=None,
		backup_path_private_files=None, db_host="localhost", db_port=None, verbose=False,
		db_type='mariadb', backup_path_conf=None, compress_files=False):
		global _verbose
		self.compress_files = compress_files or compress
		self.db_host = db_host
		self.db_port = db_port
		self.db_name = db_name
		self.db_type = db_type
		self.user = user
		self.password = password
		self.backup_path_conf = backup_path_conf
		self.backup_path_db = backup_path_db
		self.backup_path_files = backup_path_files
		self.backup_path_private_files = backup_path_private_files

		if not self.db_type:
			self.db_type = 'mariadb'

		if not self.db_port and self.db_type == 'mariadb':
			self.db_port = 3306
		elif not self.db_port and self.db_type == 'postgres':
			self.db_port = 5432

		site = frappe.local.site or frappe.generate_hash(length=8)
		self.site_slug = site.replace('.', '_')
		self.verbose = verbose
		self.setup_backup_directory()
		_verbose = verbose

	def setup_backup_directory(self):
		specified = self.backup_path_db or self.backup_path_files or self.backup_path_private_files

		if not specified:
			backups_folder = get_backup_path()
			if not os.path.exists(backups_folder):
				os.makedirs(backups_folder)
		else:
			for file_path in [self.backup_path_files, self.backup_path_db, self.backup_path_private_files]:
				dir = os.path.dirname(file_path)
				os.makedirs(dir, exist_ok=True)


	def get_backup(self, older_than=24, ignore_files=False, force=False):
		"""
			Takes a new dump if existing file is old
			and sends the link to the file as email
		"""
		#Check if file exists and is less than a day old
		#If not Take Dump
		if not force:
			last_db, last_file, last_private_file, site_config_backup_path = self.get_recent_backup(older_than)
		else:
			last_db, last_file, last_private_file, site_config_backup_path = False, False, False, False

		self.todays_date = now_datetime().strftime('%Y%m%d_%H%M%S')

		if not (self.backup_path_files and self.backup_path_db and self.backup_path_private_files):
			self.set_backup_file_name()

		if not (last_db and last_file and last_private_file and site_config_backup_path):
			self.take_dump()
			self.copy_site_config()
			if not ignore_files:
				self.backup_files()

		else:
			self.backup_path_files = last_file
			self.backup_path_db = last_db
			self.backup_path_private_files = last_private_file
			self.site_config_backup_path = site_config_backup_path

	def set_backup_file_name(self):
		#Generate a random name using today's date and a 8 digit random number
		for_conf = self.todays_date + "-" + self.site_slug + "-site_config_backup.json"
		for_db = self.todays_date + "-" + self.site_slug + "-database.sql.gz"
		ext = "tgz" if self.compress_files else "tar"

		for_public_files = self.todays_date + "-" + self.site_slug + "-files." + ext
		for_private_files = self.todays_date + "-" + self.site_slug + "-private-files." + ext
		backup_path = get_backup_path()

		if not self.backup_path_conf:
			self.backup_path_conf = os.path.join(backup_path, for_conf)
		if not self.backup_path_db:
			self.backup_path_db = os.path.join(backup_path, for_db)
		if not self.backup_path_files:
			self.backup_path_files = os.path.join(backup_path, for_public_files)
		if not self.backup_path_private_files:
			self.backup_path_private_files = os.path.join(backup_path, for_private_files)

	def get_recent_backup(self, older_than):
		backup_path = get_backup_path()

		file_type_slugs = {
			"database": "*-{}-database.sql.gz",
			"public": "*-{}-files.tar",
			"private": "*-{}-private-files.tar",
			"config": "*-{}-site_config_backup.json",
		}

		def backup_time(file_path):
			file_name = file_path.split(os.sep)[-1]
			file_timestamp = file_name.split("-")[0]
			return timegm(datetime.strptime(file_timestamp, "%Y%m%d_%H%M%S").utctimetuple())

		def get_latest(file_pattern):
			file_pattern = os.path.join(backup_path, file_pattern.format(self.site_slug))
			file_list = glob(file_pattern)
			if file_list:
				return max(file_list, key=backup_time)

		def old_enough(file_path):
			if file_path:
				if not os.path.isfile(file_path) or is_file_old(file_path, older_than):
					return None
				return file_path

		latest_backups = {
			file_type: get_latest(pattern)
			for file_type, pattern in file_type_slugs.items()
		}

		recent_backups = {
			file_type: old_enough(file_name) for file_type, file_name in latest_backups.items()
		}

		return (
			recent_backups.get("database"),
			recent_backups.get("public"),
			recent_backups.get("private"),
			recent_backups.get("config"),
		)

	def zip_files(self):
		# For backwards compatibility - pre v13
		return self.backup_files()

	def backup_files(self):
		for folder in ("public", "private"):
			files_path = frappe.get_site_path(folder, "files")
			backup_path = self.backup_path_files if folder=="public" else self.backup_path_private_files

			if self.compress_files:
				cmd_string = "tar cf - {1} | gzip -v > {0}"
			else:
				cmd_string = "tar -cf {0} {1}"
			import subprocess
			output = subprocess.check_output(cmd_string.format(backup_path, files_path), shell=True)

			if self.verbose:
				print('{0}\nBacked up file: {1}'.format(output or "", os.path.abspath(backup_path)))

	def copy_site_config(self):
		site_config_backup_path = self.backup_path_conf
		site_config_path = os.path.join(frappe.get_site_path(), "site_config.json")

		with open(site_config_backup_path, "w") as n, open(site_config_path) as c:
			n.write(c.read())

	def take_dump(self):
		import frappe.utils

		# escape reserved characters
		args = dict([item[0], frappe.utils.esc(str(item[1]), '$ ')]
			for item in self.__dict__.copy().items())

		cmd_string = """mysqldump --single-transaction --quick --lock-tables=false -u %(user)s -p%(password)s %(db_name)s -h %(db_host)s -P %(db_port)s | gzip > %(backup_path_db)s """ % args

		if self.db_type == 'postgres':
			cmd_string = "pg_dump postgres://{user}:{password}@{db_host}:{db_port}/{db_name} | gzip > {backup_path_db}".format(
				user=args.get('user'),
				password=args.get('password'),
				db_host=args.get('db_host'),
				db_port=args.get('db_port'),
				db_name=args.get('db_name'),
				backup_path_db=args.get('backup_path_db')
			)

		err, out = frappe.utils.execute_in_shell(cmd_string)

	def send_email(self):
		"""
			Sends the link to backup file located at erpnext/backups
		"""
		from frappe.email import get_system_managers

		recipient_list = get_system_managers()
		db_backup_url = get_url(os.path.join('backups', os.path.basename(self.backup_path_db)))
		files_backup_url = get_url(os.path.join('backups', os.path.basename(self.backup_path_files)))

		msg = """Hello,

Your backups are ready to be downloaded.

1. [Click here to download the database backup](%(db_backup_url)s)
2. [Click here to download the files backup](%(files_backup_url)s)

This link will be valid for 24 hours. A new backup will be available for
download only after 24 hours.""" % {
			"db_backup_url": db_backup_url,
			"files_backup_url": files_backup_url
		}

		datetime_str = datetime.fromtimestamp(os.stat(self.backup_path_db).st_ctime)
		subject = datetime_str.strftime("%d/%m/%Y %H:%M:%S") + """ - Backup ready to be downloaded"""

		frappe.sendmail(recipients=recipient_list, msg=msg, subject=subject)
		return recipient_list


@frappe.whitelist()
def get_backup():
	"""
		This function is executed when the user clicks on
		Toos > Download Backup
	"""
	delete_temp_backups()
	odb = BackupGenerator(frappe.conf.db_name, frappe.conf.db_name,\
						  frappe.conf.db_password, db_host = frappe.db.host,\
							db_type=frappe.conf.db_type, db_port=frappe.conf.db_port)
	odb.get_backup()
	recipient_list = odb.send_email()
	frappe.msgprint(_("Download link for your backup will be emailed on the following email address: {0}").format(', '.join(recipient_list)))


@frappe.whitelist()
def fetch_latest_backups():
	"""Fetches paths of the latest backup taken in the last 30 days
	Only for: System Managers

	Returns:
		dict: relative Backup Paths
	"""
	frappe.only_for("System Manager")
	odb = BackupGenerator(
		frappe.conf.db_name,
		frappe.conf.db_name,
		frappe.conf.db_password,
		db_host=frappe.db.host,
		db_type=frappe.conf.db_type,
		db_port=frappe.conf.db_port,
	)
	database, public, private, config = odb.get_recent_backup(older_than=24 * 30)

	return {
		"database": database,
		"public": public,
		"private": private,
		"config": config
	}


def scheduled_backup(older_than=6, ignore_files=False, backup_path_db=None, backup_path_files=None, backup_path_private_files=None, backup_path_conf=None, force=False, verbose=False, compress=False):
	"""this function is called from scheduler
		deletes backups older than 7 days
		takes backup"""
	odb = new_backup(older_than, ignore_files, backup_path_db=backup_path_db, backup_path_files=backup_path_files, backup_path_private_files=backup_path_private_files, backup_path_conf=backup_path_conf, force=force, verbose=verbose, compress=compress)
	return odb

def new_backup(older_than=6, ignore_files=False, backup_path_db=None, backup_path_files=None, backup_path_private_files=None, backup_path_conf=None, force=False, verbose=False, compress=False):
	delete_temp_backups(older_than = frappe.conf.keep_backups_for_hours or 24)
	odb = BackupGenerator(frappe.conf.db_name, frappe.conf.db_name,\
						  frappe.conf.db_password,
						  backup_path_db=backup_path_db,
						  backup_path_files=backup_path_files,
						  backup_path_private_files=backup_path_private_files,
						  backup_path_conf=backup_path_conf,
						  db_host = frappe.db.host,
						  db_port = frappe.db.port,
						  db_type = frappe.conf.db_type,
						  verbose=verbose,
						  compress_files=compress)
	odb.get_backup(older_than, ignore_files, force=force)
	return odb

def delete_temp_backups(older_than=24):
	"""
		Cleans up the backup_link_path directory by deleting files older than 24 hours
	"""
	backup_path = get_backup_path()
	if os.path.exists(backup_path):
		file_list = os.listdir(get_backup_path())
		for this_file in file_list:
			this_file_path = os.path.join(get_backup_path(), this_file)
			if is_file_old(this_file_path, older_than):
				os.remove(this_file_path)

def is_file_old(db_file_name, older_than=24):
		"""
			Checks if file exists and is older than specified hours
			Returns ->
			True: file does not exist or file is old
			False: file is new
		"""
		if os.path.isfile(db_file_name):
			from datetime import timedelta
			#Get timestamp of the file
			file_datetime = datetime.fromtimestamp\
						(os.stat(db_file_name).st_ctime)
			if datetime.today() - file_datetime >= timedelta(hours = older_than):
				if _verbose:
					print("File is old")
				return True
			else:
				if _verbose:
					print("File is recent")
				return False
		else:
			if _verbose:
				print("File does not exist")
			return True

def get_backup_path():
	backup_path = frappe.utils.get_site_path(conf.get("backup_path", "private/backups"))
	return backup_path

def backup(with_files=False, backup_path_db=None, backup_path_files=None, backup_path_private_files=None, backup_path_conf=None, quiet=False):
	"Backup"
	odb = scheduled_backup(ignore_files=not with_files, backup_path_db=backup_path_db, backup_path_files=backup_path_files, backup_path_private_files=backup_path_private_files, backup_path_conf=backup_path_conf, force=True)
	return {
		"backup_path_db": odb.backup_path_db,
		"backup_path_files": odb.backup_path_files,
		"backup_path_private_files": odb.backup_path_private_files
	}


if __name__ == "__main__":
	"""
		is_file_old db_name user password db_host db_type db_port
		get_backup  db_name user password db_host db_type db_port
	"""
	import sys
	cmd = sys.argv[1]

	db_type = 'mariadb'
	try:
		db_type = sys.argv[6]
	except IndexError:
		pass

	db_port = 3306
	try:
		db_port = int(sys.argv[7])
	except IndexError:
		pass

	if cmd == "is_file_old":
		odb = BackupGenerator(sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5] or "localhost", db_type=db_type, db_port=db_port)
		is_file_old(odb.db_file_name)

	if cmd == "get_backup":
		odb = BackupGenerator(sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5] or "localhost", db_type=db_type, db_port=db_port)
		odb.get_backup()

	if cmd == "take_dump":
		odb = BackupGenerator(sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5] or "localhost", db_type=db_type, db_port=db_port)
		odb.take_dump()

	if cmd == "send_email":
		odb = BackupGenerator(sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5] or "localhost", db_type=db_type, db_port=db_port)
		odb.send_email("abc.sql.gz")

	if cmd == "delete_temp_backups":
		delete_temp_backups()