From 1bc97e8f8a7e625f5981b1b510488f59ef744cdb Mon Sep 17 00:00:00 2001 From: Saurabh Date: Tue, 7 Apr 2020 00:06:27 +0530 Subject: [PATCH] feat: Offsite backup (#8241) * feat: utility to get file size * feat: if backup file is greater than 1gb then consider latest backup site instead of taking new * fix: remove unwanted import * chore: fix condition inside offsite backup utils Signed-off-by: Chinmay D. Pai * chore: change email field to notify_email Signed-off-by: Chinmay D. Pai * chore: fix deepsource issues Signed-off-by: Chinmay D. Pai * feat: add offsite_backup support for google drive Signed-off-by: Chinmay D. Pai * chore: get recipients from within send_email pass email_field to send_email instead of calling two functions Signed-off-by: Chinmay D. Pai * chore: set flag within validate_file_size Signed-off-by: Chinmay D. Pai * feat: get latest file backup when specified Signed-off-by: Chinmay D. Pai * chore: fix deepsource issues Signed-off-by: Chinmay D. Pai * chore: fix incorrectly spelled dropbox settings Signed-off-by: Chinmay D. Pai * chore: implement file backup logic for aws s3 Signed-off-by: Chinmay D. Pai * chore: fix deepsource issues Signed-off-by: Chinmay D. Pai * chore: fix deepsource issues Signed-off-by: Chinmay D. Pai Co-authored-by: Chinmay D. Pai Co-authored-by: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> --- .../dropbox_settings/dropbox_settings.json | 576 ++++-------------- .../dropbox_settings/dropbox_settings.py | 51 +- .../dropbox_settings/test_dropbox_settings.py | 10 + .../doctype/google_drive/google_drive.py | 52 +- .../s3_backup_settings.json | 471 +++----------- .../s3_backup_settings/s3_backup_settings.py | 54 +- frappe/integrations/offsite_backup_utils.py | 83 +++ frappe/integrations/utils.py | 2 +- 8 files changed, 346 insertions(+), 953 deletions(-) create mode 100644 frappe/integrations/doctype/dropbox_settings/test_dropbox_settings.py create mode 100644 frappe/integrations/offsite_backup_utils.py diff --git a/frappe/integrations/doctype/dropbox_settings/dropbox_settings.json b/frappe/integrations/doctype/dropbox_settings/dropbox_settings.json index 33d34e0210..858469647a 100644 --- a/frappe/integrations/doctype/dropbox_settings/dropbox_settings.json +++ b/frappe/integrations/doctype/dropbox_settings/dropbox_settings.json @@ -1,487 +1,129 @@ { - "allow_copy": 0, - "allow_events_in_timeline": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 0, - "creation": "2016-09-21 10:12:57.399174", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "System", - "editable_grid": 1, + "creation": "2016-09-21 10:12:57.399174", + "doctype": "DocType", + "document_type": "System", + "editable_grid": 1, + "field_order": [ + "enabled", + "send_notifications_to", + "send_email_for_successful_backup", + "backup_frequency", + "limit_no_of_backups", + "no_of_backups", + "file_backup", + "app_access_key", + "app_secret_key", + "allow_dropbox_access", + "dropbox_access_key", + "dropbox_access_secret", + "dropbox_access_token" + ], "fields": [ { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "enabled", - "fieldtype": "Check", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Enabled", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "default": "0", + "fieldname": "enabled", + "fieldtype": "Check", + "label": "Enabled" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "send_notifications_to", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Send Notifications To", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "send_notifications_to", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Send Notifications To", + "reqd": 1 + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "default": "1", - "description": "Note: By default emails for failed backups are sent.", - "fieldname": "send_email_for_successful_backup", - "fieldtype": "Check", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Send Email for Successful Backup", - "length": 0, - "no_copy": 0, - "options": "", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "default": "1", + "description": "Note: By default emails for failed backups are sent.", + "fieldname": "send_email_for_successful_backup", + "fieldtype": "Check", + "label": "Send Email for Successful Backup" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "backup_frequency", - "fieldtype": "Select", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Backup Frequency", - "length": 0, - "no_copy": 0, - "options": "\nDaily\nWeekly", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "backup_frequency", + "fieldtype": "Select", + "in_list_view": 1, + "label": "Backup Frequency", + "options": "\nDaily\nWeekly", + "reqd": 1 + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "limit_no_of_backups", - "fieldtype": "Check", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Limit Number of DB Backups", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "default": "0", + "fieldname": "limit_no_of_backups", + "fieldtype": "Check", + "label": "Limit Number of DB Backups" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "default": "5", - "depends_on": "eval:doc.limit_no_of_backups", - "fieldname": "no_of_backups", - "fieldtype": "Int", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Number of DB Backups", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "default": "5", + "depends_on": "eval:doc.limit_no_of_backups", + "fieldname": "no_of_backups", + "fieldtype": "Int", + "label": "Number of DB Backups" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "default": "1", - "fieldname": "file_backup", - "fieldtype": "Check", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "File Backup", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "default": "1", + "fieldname": "file_backup", + "fieldtype": "Check", + "label": "File Backup" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "depends_on": "", - "fieldname": "app_access_key", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "App Access Key", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "app_access_key", + "fieldtype": "Data", + "label": "App Access Key" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "depends_on": "", - "fieldname": "app_secret_key", - "fieldtype": "Password", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "App Secret Key", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "app_secret_key", + "fieldtype": "Password", + "label": "App Secret Key" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "allow_dropbox_access", - "fieldtype": "Button", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Allow Dropbox Access", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "allow_dropbox_access", + "fieldtype": "Button", + "label": "Allow Dropbox Access" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "dropbox_access_key", - "fieldtype": "Password", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Dropbox Access Key", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "dropbox_access_key", + "fieldtype": "Password", + "hidden": 1, + "label": "Dropbox Access Key", + "read_only": 1 + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "dropbox_access_secret", - "fieldtype": "Password", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Dropbox Access Secret", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "dropbox_access_secret", + "fieldtype": "Password", + "hidden": 1, + "label": "Dropbox Access Secret", + "read_only": 1 + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "dropbox_access_token", - "fieldtype": "Password", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Dropbox Access Token", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "fieldname": "dropbox_access_token", + "fieldtype": "Password", + "hidden": 1, + "label": "Dropbox Access Token" } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 1, - "is_submittable": 0, - "issingle": 1, - "istable": 0, - "max_attachments": 0, - "modified": "2019-01-03 05:44:40.520943", - "modified_by": "Administrator", - "module": "Integrations", - "name": "Dropbox Settings", - "name_case": "", - "owner": "Administrator", + ], + "in_create": 1, + "issingle": 1, + "modified": "2019-08-22 16:26:44.468391", + "modified_by": "Administrator", + "module": "Integrations", + "name": "Dropbox Settings", + "owner": "Administrator", "permissions": [ { - "amend": 0, - "cancel": 0, - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 0, - "role": "System Manager", - "set_user_permissions": 0, - "share": 1, - "submit": 0, + "create": 1, + "delete": 1, + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "role": "System Manager", + "share": 1, "write": 1 } - ], - "quick_entry": 0, - "read_only": 1, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1, - "track_seen": 0, - "track_views": 0 -} + ], + "read_only": 1, + "sort_field": "modified", + "sort_order": "DESC", + "track_changes": 1 +} \ No newline at end of file diff --git a/frappe/integrations/doctype/dropbox_settings/dropbox_settings.py b/frappe/integrations/doctype/dropbox_settings/dropbox_settings.py index dd4768f8b3..2a036f4838 100644 --- a/frappe/integrations/doctype/dropbox_settings/dropbox_settings.py +++ b/frappe/integrations/doctype/dropbox_settings/dropbox_settings.py @@ -3,22 +3,25 @@ # For license information, please see license.txt from __future__ import unicode_literals +import dropbox +import json import frappe import os from frappe import _ from frappe.model.document import Document -import dropbox, json +from frappe.integrations.offsite_backup_utils import get_latest_backup_file, send_email, validate_file_size +from frappe.integrations.utils import make_post_request +from frappe.utils import (cint, get_request_site_address, + get_files_path, get_backups_path, get_url, encode) from frappe.utils.backups import new_backup from frappe.utils.background_jobs import enqueue from six.moves.urllib.parse import urlparse, parse_qs -from frappe.integrations.utils import make_post_request from rq.timeouts import JobTimeoutException -from frappe.utils import (cint, split_emails, get_request_site_address, - get_files_path, get_backups_path, get_url, encode) from six import text_type ignore_list = [".DS_Store"] + class DropboxSettings(Document): def onload(self): if not self.app_access_key and frappe.conf.dropbox_access_key: @@ -48,10 +51,12 @@ def take_backup_to_dropbox(retry_count=0, upload_db_backup=True): did_not_upload, error_log = [], [] try: if cint(frappe.db.get_value("Dropbox Settings", None, "enabled")): + validate_file_size() + did_not_upload, error_log = backup_to_dropbox(upload_db_backup) if did_not_upload: raise Exception - send_email(True, "Dropbox") + send_email(True, "Dropbox", "Dropbox Settings", "send_notifications_to") except JobTimeoutException: if retry_count < 2: args = { @@ -66,34 +71,8 @@ def take_backup_to_dropbox(retry_count=0, upload_db_backup=True): else: file_and_error = [" - ".join(f) for f in zip(did_not_upload, error_log)] error_message = ("\n".join(file_and_error) + "\n" + frappe.get_traceback()) - frappe.errprint(error_message) - send_email(False, "Dropbox", error_message) -def send_email(success, service_name, error_status=None): - if success: - if frappe.db.get_value("Dropbox Settings", None, "send_email_for_successful_backup") == '0': - return - - subject = "Backup Upload Successful" - message ="""

Backup Uploaded Successfully

Hi there, this is just to inform you - that your backup was successfully uploaded to your %s account. So relax!

- """ % service_name - - else: - subject = "[Warning] Backup Upload Failed" - message ="""

Backup Upload Failed

Oops, your automated backup to %s - failed.

-

Error message:
- 

%s
-

-

Please contact your system manager for more information.

- """ % (service_name, error_status) - - if not frappe.db: - frappe.connect() - - recipients = split_emails(frappe.db.get_value("Dropbox Settings", None, "send_notifications_to")) - frappe.sendmail(recipients=recipients, subject=subject, message=message) + send_email(False, "Dropbox", "Dropbox Settings", "send_notifications_to", error_message) def backup_to_dropbox(upload_db_backup=True): if not frappe.db: @@ -114,8 +93,12 @@ def backup_to_dropbox(upload_db_backup=True): dropbox_client = dropbox.Dropbox(dropbox_settings['access_token']) if upload_db_backup: - backup = new_backup(ignore_files=True) - filename = os.path.join(get_backups_path(), os.path.basename(backup.backup_path_db)) + if frappe.flags.create_new_backup: + backup = new_backup(ignore_files=True) + filename = os.path.join(get_backups_path(), os.path.basename(backup.backup_path_db)) + else: + filename = get_latest_backup_file() + upload_file_to_dropbox(filename, "/database", dropbox_client) # delete older databases diff --git a/frappe/integrations/doctype/dropbox_settings/test_dropbox_settings.py b/frappe/integrations/doctype/dropbox_settings/test_dropbox_settings.py new file mode 100644 index 0000000000..539fc417f2 --- /dev/null +++ b/frappe/integrations/doctype/dropbox_settings/test_dropbox_settings.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2019, Frappe Technologies and Contributors +# See license.txt +from __future__ import unicode_literals + +# import frappe +import unittest + +class TestDropboxSettings(unittest.TestCase): + pass diff --git a/frappe/integrations/doctype/google_drive/google_drive.py b/frappe/integrations/doctype/google_drive/google_drive.py index 8078c702c0..60ee173bbf 100644 --- a/frappe/integrations/doctype/google_drive/google_drive.py +++ b/frappe/integrations/doctype/google_drive/google_drive.py @@ -19,6 +19,7 @@ from apiclient.http import MediaFileUpload from frappe.utils import get_backups_path, get_bench_path from frappe.utils.backups import new_backup from frappe.integrations.doctype.google_settings.google_settings import get_auth_url +from frappe.integrations.offsite_backup_utils import get_latest_backup_file, send_email, validate_file_size SCOPES = "https://www.googleapis.com/auth/drive" @@ -183,13 +184,16 @@ def upload_system_backup_to_google_drive(): check_for_folder_in_google_drive() account.load_from_db() - progress(1, "Backing up Data.") - backup = new_backup() - - fileurl_backup = os.path.basename(backup.backup_path_db) - fileurl_public_files = os.path.basename(backup.backup_path_files) - fileurl_private_files = os.path.basename(backup.backup_path_private_files) + validate_file_size() + if frappe.flags.create_new_backup: + set_progress(1, "Backing up Data.") + backup = new_backup() + fileurl_backup = os.path.basename(backup.backup_path_db) + fileurl_public_files = os.path.basename(backup.backup_path_files) + fileurl_private_files = os.path.basename(backup.backup_path_private_files) + else: + fileurl_backup, fileurl_public_files, fileurl_private_files = get_latest_backup_file(with_files=True) for fileurl in [fileurl_backup, fileurl_public_files, fileurl_private_files]: file_metadata = { @@ -203,15 +207,14 @@ def upload_system_backup_to_google_drive(): frappe.throw(_("Google Drive - Could not locate locate - {0}").format(e)) try: - progress(2, "Uploading backup to Google Drive.") + set_progress(2, "Uploading backup to Google Drive.") google_drive.files().create(body=file_metadata, media_body=media, fields="id").execute() except HttpError as e: - send_email(success=False, error=e) - frappe.msgprint(_("Google Drive - Could not upload backup - Error {0}").format(e)) + send_email(False, "Google Drive", "Google Drive", "email", error_status=e) - progress(3, "Uploading successful.") + set_progress(3, "Uploading successful.") frappe.db.set_value("Google Drive", None, "last_backup_on", frappe.utils.now_datetime()) - send_email(success=True) + send_email(True, "Google Drive", "Google Drive", "email") return _("Google Drive Backup Successful.") def daily_backup(): @@ -226,30 +229,5 @@ def get_absolute_path(filename): file_path = os.path.join(get_backups_path()[2:], filename) return "{0}/sites/{1}".format(get_bench_path(), file_path) -def progress(progress, message): +def set_progress(progress, message): frappe.publish_realtime("upload_to_google_drive", dict(progress=progress, total=3, message=message), user=frappe.session.user) - -def send_email(success, error=None): - if success: - if not frappe.db.get_single_value("Google Drive", "send_email_for_successful_backup"): - return - - subject = "Backup Upload Successful" - message = """

Backup Uploaded Successfully

Hi there, this is just to inform you - that your backup was successfully uploaded to Google Drive.

- """ - else: - subject = "[Warning] Backup Upload Failed" - message = """

Backup Upload Failed

Oops, your automated backup to Google Drive - failed.

-

Error message:
- 

{0}
-

-

Please contact your system manager for more information.

- """.format(error) - - frappe.sendmail( - recipients=frappe.db.get_single_value("Google Drive", "email"), - subject=subject, - message=message - ) \ No newline at end of file diff --git a/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.json b/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.json index 93fff995f0..bbdbf74a67 100755 --- a/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.json +++ b/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.json @@ -1,397 +1,110 @@ { - "allow_copy": 0, - "allow_events_in_timeline": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 0, - "creation": "2017-09-04 20:57:20.129205", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", + "creation": "2017-09-04 20:57:20.129205", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "enabled", + "notify_email", + "send_email_for_successful_backup", + "frequency", + "access_key_id", + "secret_access_key", + "region", + "endpoint_url", + "bucket", + "backup_limit" + ], "fields": [ { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fetch_if_empty": 0, - "fieldname": "enabled", - "fieldtype": "Check", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Enable Automatic Backup", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "default": "0", + "fieldname": "enabled", + "fieldtype": "Check", + "label": "Enable Automatic Backup" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fetch_if_empty": 0, - "fieldname": "notify_email", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Send Notifications To", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "notify_email", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Send Notifications To", + "reqd": 1 + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "default": "1", - "description": "Note: By default emails for failed backups are sent.", - "fetch_if_empty": 0, - "fieldname": "send_email_for_successful_backup", - "fieldtype": "Check", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Send Email for Successful Backup", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "default": "1", + "description": "Note: By default emails for failed backups are sent.", + "fieldname": "send_email_for_successful_backup", + "fieldtype": "Check", + "label": "Send Email for Successful Backup" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fetch_if_empty": 0, - "fieldname": "frequency", - "fieldtype": "Select", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Backup Frequency", - "length": 0, - "no_copy": 0, - "options": "Daily\nWeekly\nMonthly\nNone", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "frequency", + "fieldtype": "Select", + "in_list_view": 1, + "label": "Backup Frequency", + "options": "Daily\nWeekly\nMonthly\nNone", + "reqd": 1 + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fetch_if_empty": 0, - "fieldname": "access_key_id", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Access Key ID", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "access_key_id", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Access Key ID", + "reqd": 1 + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fetch_if_empty": 0, - "fieldname": "secret_access_key", - "fieldtype": "Password", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Secret Access Key", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "secret_access_key", + "fieldtype": "Password", + "in_list_view": 1, + "label": "Secret Access Key", + "reqd": 1 + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "default": "us-east-1", - "description": "See https://docs.aws.amazon.com/de_de/general/latest/gr/rande.html#s3_region for details.", - "fetch_if_empty": 0, - "fieldname": "region", - "fieldtype": "Select", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Region", - "length": 0, - "no_copy": 0, - "options": "us-east-1\nus-east-2\nus-west-1\nus-west-2\nap-south-1\nap-southeast-1\nap-southeast-2\nap-northeast-1\nap-northeast-2\nap-northeast-3\nca-central-1\ncn-north-1\ncn-northwest-1\neu-central-1\neu-west-1\neu-west-2\neu-west-3\neu-north-1\nsa-east-1", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "default": "us-east-1", + "description": "See https://docs.aws.amazon.com/de_de/general/latest/gr/rande.html#s3_region for details.", + "fieldname": "region", + "fieldtype": "Select", + "label": "Region", + "options": "us-east-1\nus-east-2\nus-west-1\nus-west-2\nap-south-1\nap-southeast-1\nap-southeast-2\nap-northeast-1\nap-northeast-2\nap-northeast-3\nca-central-1\ncn-north-1\ncn-northwest-1\neu-central-1\neu-west-1\neu-west-2\neu-west-3\neu-north-1\nsa-east-1" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fetch_if_empty": 0, - "fieldname": "endpoint_url", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Endpoint URL", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "endpoint_url", + "fieldtype": "Data", + "label": "Endpoint URL" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fetch_if_empty": 0, - "fieldname": "bucket", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Bucket", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "bucket", + "fieldtype": "Data", + "label": "Bucket", + "reqd": 1 + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fetch_if_empty": 0, - "fieldname": "backup_limit", - "fieldtype": "Int", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Backup Limit", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "fieldname": "backup_limit", + "fieldtype": "Int", + "label": "Backup Limit", + "reqd": 1 } - ], - "has_web_view": 0, - "hide_toolbar": 1, - "idx": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 1, - "istable": 0, - "max_attachments": 0, - "menu_index": 0, - "modified": "2019-04-10 03:56:55.632017", - "modified_by": "Administrator", - "module": "Integrations", - "name": "S3 Backup Settings", - "name_case": "", - "owner": "Administrator", + ], + "hide_toolbar": 1, + "issingle": 1, + "modified": "2019-08-22 16:26:04.774571", + "modified_by": "Administrator", + "module": "Integrations", + "name": "S3 Backup Settings", + "owner": "Administrator", "permissions": [ { - "amend": 0, - "cancel": 0, - "create": 1, - "delete": 1, - "email": 1, - "export": 0, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 0, - "role": "System Manager", - "set_user_permissions": 0, - "share": 1, - "submit": 0, + "create": 1, + "delete": 1, + "email": 1, + "print": 1, + "read": 1, + "role": "System Manager", + "share": 1, "write": 1 } - ], - "quick_entry": 1, - "read_only": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1, - "track_seen": 0, - "track_views": 0 + ], + "quick_entry": 1, + "sort_field": "modified", + "sort_order": "DESC", + "track_changes": 1 } \ No newline at end of file diff --git a/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.py b/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.py index 55b9e63a4d..7e69da922c 100755 --- a/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.py +++ b/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.py @@ -8,12 +8,14 @@ import os.path import frappe import boto3 from frappe import _ +from frappe.integrations.offsite_backup_utils import get_latest_backup_file, send_email, validate_file_size from frappe.model.document import Document -from frappe.utils import cint, split_emails +from frappe.utils import cint from frappe.utils.background_jobs import enqueue from rq.timeouts import JobTimeoutException from botocore.exceptions import ClientError + class S3BackupSettings(Document): def validate(self): @@ -49,7 +51,7 @@ class S3BackupSettings(Document): @frappe.whitelist() def take_backup(): - "Enqueue longjob for taking backup to s3" + """Enqueue longjob for taking backup to s3""" enqueue("frappe.integrations.doctype.s3_backup_settings.s3_backup_settings.take_backups_s3", queue='long', timeout=1500) frappe.msgprint(_("Queued for backup. It may take a few minutes to an hour.")) @@ -65,22 +67,21 @@ def take_backups_weekly(): def take_backups_monthly(): take_backups_if("Monthly") - def take_backups_if(freq): if cint(frappe.db.get_value("S3 Backup Settings", None, "enabled")): if frappe.db.get_value("S3 Backup Settings", None, "frequency") == freq: take_backups_s3() - @frappe.whitelist() def take_backups_s3(retry_count=0): try: + validate_file_size() backup_to_s3() - send_email(True, "S3 Backup Settings") + send_email(True, "Amazon S3", "S3 Backup Settings", "notify_email") except JobTimeoutException: if retry_count < 2: args = { - "retry_count" :retry_count + 1 + "retry_count": retry_count + 1 } enqueue("frappe.integrations.doctype.s3_backup_settings.s3_backup_settings.take_backups_s3", queue='long', timeout=1500, **args) @@ -89,31 +90,10 @@ def take_backups_s3(retry_count=0): except Exception: notify() + def notify(): error_message = frappe.get_traceback() - frappe.errprint(error_message) - send_email(False, "S3 Backup Settings", error_message) - -def send_email(success, service_name, error_status=None): - if success: - if frappe.db.get_value("S3 Backup Settings", None, "send_email_for_successful_backup") == '0': - return - - subject = "Backup Upload Successful" - message = """

Backup Uploaded Successfully!

Hi there, this is just to inform you - that your backup was successfully uploaded to your Amazon S3 bucket. So relax!

""" - - else: - subject = "[Warning] Backup Upload Failed" - message = """

Backup Upload Failed!

Oops, your automated backup to Amazon S3 failed. -

Error message: %s

Please contact your system manager - for more information.

""" % error_status - - if not frappe.db: - frappe.connect() - - recipients = split_emails(frappe.db.get_value("S3 Backup Settings", None, "notify_email")) - frappe.sendmail(recipients=recipients, subject=subject, message=message) + send_email(False, 'Amazon S3', "S3 Backup Settings", "notify_email", error_message) def backup_to_s3(): @@ -130,11 +110,15 @@ def backup_to_s3(): endpoint_url=doc.endpoint_url or 'https://s3.amazonaws.com' ) - backup = new_backup(ignore_files=False, backup_path_db=None, + if frappe.flags.create_new_backup: + backup = new_backup(ignore_files=False, backup_path_db=None, backup_path_files=None, backup_path_private_files=None, force=True) - db_filename = os.path.join(get_backups_path(), os.path.basename(backup.backup_path_db)) - files_filename = os.path.join(get_backups_path(), os.path.basename(backup.backup_path_files)) - private_files = os.path.join(get_backups_path(), os.path.basename(backup.backup_path_private_files)) + db_filename = os.path.join(get_backups_path(), os.path.basename(backup.backup_path_db)) + files_filename = os.path.join(get_backups_path(), os.path.basename(backup.backup_path_files)) + private_files = os.path.join(get_backups_path(), os.path.basename(backup.backup_path_private_files)) + else: + db_filename, files_filename, private_files = get_latest_backup_file(with_files=True) + folder = os.path.basename(db_filename)[:15] + '/' # for adding datetime to folder name @@ -143,8 +127,8 @@ def backup_to_s3(): upload_file_to_s3(files_filename, folder, conn, bucket) delete_old_backups(doc.backup_limit, bucket) -def upload_file_to_s3(filename, folder, conn, bucket): +def upload_file_to_s3(filename, folder, conn, bucket): destpath = os.path.join(folder, os.path.basename(filename)) try: print("Uploading file:", filename) @@ -156,7 +140,7 @@ def upload_file_to_s3(filename, folder, conn, bucket): def delete_old_backups(limit, bucket): - all_backups = list() + all_backups = [] doc = frappe.get_single("S3 Backup Settings") backup_limit = int(limit) diff --git a/frappe/integrations/offsite_backup_utils.py b/frappe/integrations/offsite_backup_utils.py new file mode 100644 index 0000000000..c280a1d9dd --- /dev/null +++ b/frappe/integrations/offsite_backup_utils.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2019, Frappe Technologies and contributors +# For license information, please see license.txt + +from __future__ import unicode_literals +import frappe +import glob +import os +from frappe.utils import split_emails, get_backups_path + + +def send_email(success, service_name, doctype, email_field, error_status=None): + recipients = get_recipients(service_name, email_field) + if not recipients: + frappe.log_error("No Email Recipient found for {0}".format(service_name), + "{0}: Failed to send backup status email".format(service_name)) + return + + if success: + if not frappe.db.get_value(doctype, None, "send_email_for_successful_backup"): + return + + subject = "Backup Upload Successful" + message = """ +

Backup Uploaded Successfully!

+

Hi there, this is just to inform you that your backup was successfully uploaded to your {0} bucket. So relax!

""".format(service_name) + + else: + subject = "[Warning] Backup Upload Failed" + message = """ +

Backup Upload Failed!

+

Oops, your automated backup to {0} failed.

+

Error message: {1}

+

Please contact your system manager for more information.

""".format(service_name, error_status) + + frappe.sendmail(recipients=recipients, subject=subject, message=message) + + +def get_recipients(service_name, email_field): + if not frappe.db: + frappe.connect() + + return split_emails(frappe.db.get_value(service_name, None, email_field)) + + +def get_latest_backup_file(with_files=False): + + def get_latest(file_ext): + file_list = glob.glob(os.path.join(get_backups_path(), file_ext)) + return max(file_list, key=os.path.getctime) + + latest_file = get_latest('*.sql.gz') + + if with_files: + latest_public_file_bak = get_latest('*-files.tar') + latest_private_file_bak = get_latest('*-private-files.tar') + return latest_file, latest_public_file_bak, latest_private_file_bak + + return latest_file + + +def get_file_size(file_path, unit): + if not unit: + unit = 'MB' + + file_size = os.path.getsize(file_path) + + memory_size_unit_mapper = {'KB': 1, 'MB': 2, 'GB': 3, 'TB': 4} + i = 0 + while i < memory_size_unit_mapper[unit]: + file_size = file_size / 1000.0 + i += 1 + + return file_size + + +def validate_file_size(): + frappe.flags.create_new_backup = True + latest_file = get_latest_backup_file() + file_size = get_file_size(latest_file, unit='GB') + + if file_size > 1: + frappe.flags.create_new_backup = False diff --git a/frappe/integrations/utils.py b/frappe/integrations/utils.py index 811b007131..808affe47a 100644 --- a/frappe/integrations/utils.py +++ b/frappe/integrations/utils.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2015, Frappe Technologies and contributors +# Copyright (c) 2019, Frappe Technologies and contributors # For license information, please see license.txt from __future__ import unicode_literals