From 4e04d40d33a23a4626798a06bb27e15784c67451 Mon Sep 17 00:00:00 2001 From: Rushabh Mehta Date: Thu, 24 Sep 2015 18:15:58 +0530 Subject: [PATCH] [cleanup] added dropbox backup --- frappe/config/integrations.py | 4 +- .../doctype/communication/communication.json | 4 +- frappe/desk/page/backups/__init__.py | 0 frappe/desk/page/backups/backups.html | 32 +++ frappe/desk/page/backups/backups.js | 9 + frappe/desk/page/backups/backups.json | 21 ++ frappe/desk/page/backups/backups.py | 24 ++ frappe/hooks.py | 7 + .../doctype/dropbox_backup/__init__.py | 0 .../doctype/dropbox_backup/dropbox_backup.js | 37 +++ .../dropbox_backup/dropbox_backup.json | 210 ++++++++++++++++++ .../doctype/dropbox_backup/dropbox_backup.py | 207 +++++++++++++++++ frappe/patches.txt | 1 + frappe/patches/v6_2/rename_backup_manager.py | 10 + 14 files changed, 562 insertions(+), 4 deletions(-) create mode 100644 frappe/desk/page/backups/__init__.py create mode 100644 frappe/desk/page/backups/backups.html create mode 100644 frappe/desk/page/backups/backups.js create mode 100644 frappe/desk/page/backups/backups.json create mode 100644 frappe/desk/page/backups/backups.py create mode 100644 frappe/integrations/doctype/dropbox_backup/__init__.py create mode 100644 frappe/integrations/doctype/dropbox_backup/dropbox_backup.js create mode 100644 frappe/integrations/doctype/dropbox_backup/dropbox_backup.json create mode 100644 frappe/integrations/doctype/dropbox_backup/dropbox_backup.py create mode 100644 frappe/patches/v6_2/rename_backup_manager.py diff --git a/frappe/config/integrations.py b/frappe/config/integrations.py index 99ee288f5c..d4fc4d03ac 100644 --- a/frappe/config/integrations.py +++ b/frappe/config/integrations.py @@ -4,7 +4,7 @@ from frappe import _ def get_data(): return [ { - "label": _("Documents"), + "label": _("Setup"), "icon": "icon-star", "items": [ { @@ -14,7 +14,7 @@ def get_data(): }, { "type": "doctype", - "name": "Backup Manager", + "name": "Dropbox Backup", "description": _("Manage cloud backups on Dropbox"), "hide_count": True } diff --git a/frappe/core/doctype/communication/communication.json b/frappe/core/doctype/communication/communication.json index 16627d67fa..77ddb28d51 100644 --- a/frappe/core/doctype/communication/communication.json +++ b/frappe/core/doctype/communication/communication.json @@ -43,7 +43,7 @@ "hidden": 0, "ignore_user_permissions": 0, "in_filter": 0, - "in_list_view": 1, + "in_list_view": 0, "label": "Communication Medium", "no_copy": 0, "options": "\nChat\nPhone\nEmail\nSMS\nVisit\nOther", @@ -573,7 +573,7 @@ "is_submittable": 0, "issingle": 0, "istable": 0, - "modified": "2015-09-15 05:51:16.112080", + "modified": "2015-09-24 02:27:43.536919", "modified_by": "Administrator", "module": "Core", "name": "Communication", diff --git a/frappe/desk/page/backups/__init__.py b/frappe/desk/page/backups/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/frappe/desk/page/backups/backups.html b/frappe/desk/page/backups/backups.html new file mode 100644 index 0000000000..dca38315cc --- /dev/null +++ b/frappe/desk/page/backups/backups.html @@ -0,0 +1,32 @@ + + + + + + + + + + + + {% for f in files %} + + + + + + {% endfor %} + +
+ {{ _("Date") }} + + {{ _("File") }} + + {{ _("Size") }} +
+ {{ f[1] }} + + {{ f[0] }} + + {{ f[2] }} +
diff --git a/frappe/desk/page/backups/backups.js b/frappe/desk/page/backups/backups.js new file mode 100644 index 0000000000..01746f129c --- /dev/null +++ b/frappe/desk/page/backups/backups.js @@ -0,0 +1,9 @@ +frappe.pages['backups'].on_page_load = function(wrapper) { + var page = frappe.ui.make_app_page({ + parent: wrapper, + title: 'Download Backups', + single_column: true + }); + + $(frappe.render_template("backups")).appendTo(page.body.addClass("no-border")); +} diff --git a/frappe/desk/page/backups/backups.json b/frappe/desk/page/backups/backups.json new file mode 100644 index 0000000000..dd6e8d9f0a --- /dev/null +++ b/frappe/desk/page/backups/backups.json @@ -0,0 +1,21 @@ +{ + "content": null, + "creation": "2015-09-24 01:26:06.225378", + "docstatus": 0, + "doctype": "Page", + "modified": "2015-09-24 01:26:06.225378", + "modified_by": "Administrator", + "module": "Desk", + "name": "backups", + "owner": "Administrator", + "page_name": "backups", + "roles": [ + { + "role": "System Manager" + } + ], + "script": null, + "standard": "Yes", + "style": null, + "title": "Download Backups" +} \ No newline at end of file diff --git a/frappe/desk/page/backups/backups.py b/frappe/desk/page/backups/backups.py new file mode 100644 index 0000000000..9f798f6841 --- /dev/null +++ b/frappe/desk/page/backups/backups.py @@ -0,0 +1,24 @@ +import os +from frappe.utils import get_site_path +from frappe.utils.data import convert_utc_to_user_timezone +import datetime + +def get_context(context): + def get_time(path): + dt = os.path.getmtime(path) + return convert_utc_to_user_timezone(datetime.datetime.utcfromtimestamp(dt)).strftime('%Y-%m-%d %H:%M') + + def get_size(path): + size = os.path.getsize(path) + if size > 1048576: + return "{0:.1f}M".format(float(size) / 1048576) + else: + return "{0:.1f}K".format(float(size) / 1024) + + path = get_site_path('private', 'backups') + files = [x for x in os.listdir(path) if os.path.isfile(os.path.join(path, x))] + files = [('/backups/' + _file, + get_time(os.path.join(path, _file)), + get_size(os.path.join(path, _file))) for _file in files] + + return {"files": files} diff --git a/frappe/hooks.py b/frappe/hooks.py index a9017ef876..f296d4778d 100644 --- a/frappe/hooks.py +++ b/frappe/hooks.py @@ -148,7 +148,14 @@ scheduler_events = { "frappe.sessions.clear_expired_sessions", "frappe.email.doctype.email_alert.email_alert.trigger_daily_alerts", "frappe.async.remove_old_task_logs", + ], + "daily_long": [ + "frappe.integrations.doctype.dropbox_backup.dropbox_backup.take_backups_daily" + ], + "weekly_long": [ + "frappe.integrations.doctype.dropbox_backup.dropbox_backup.take_backups_weekly" ] + } default_background = "/assets/frappe/images/ui/into-the-dawn.jpg" diff --git a/frappe/integrations/doctype/dropbox_backup/__init__.py b/frappe/integrations/doctype/dropbox_backup/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/frappe/integrations/doctype/dropbox_backup/dropbox_backup.js b/frappe/integrations/doctype/dropbox_backup/dropbox_backup.js new file mode 100644 index 0000000000..99d39949e6 --- /dev/null +++ b/frappe/integrations/doctype/dropbox_backup/dropbox_backup.js @@ -0,0 +1,37 @@ +$.extend(cur_frm.cscript, { + onload_post_render: function() { + cur_frm.fields_dict.allow_dropbox_access.$input.addClass("btn-primary"); + }, + + refresh: function() { + cur_frm.disable_save(); + }, + + validate_send_notifications_to: function() { + if(!cur_frm.doc.send_notifications_to) { + msgprint(__("Please specify") + ": " + + __(frappe.meta.get_label(cur_frm.doctype, + "send_notifications_to"))); + return false; + } + + return true; + }, + + allow_dropbox_access: function() { + if(cur_frm.cscript.validate_send_notifications_to()) { + return frappe.call({ + method: "frappe.integrations.doctype.dropbox_backup.dropbox_backup.get_dropbox_authorize_url", + callback: function(r) { + if(!r.exc) { + cur_frm.set_value("dropbox_access_secret", r.message.secret); + cur_frm.set_value("dropbox_access_key", r.message.key); + cur_frm.save(null, function() { + window.open(r.message.url); + }); + } + } + }); + } + } +}); diff --git a/frappe/integrations/doctype/dropbox_backup/dropbox_backup.json b/frappe/integrations/doctype/dropbox_backup/dropbox_backup.json new file mode 100644 index 0000000000..938201cdf1 --- /dev/null +++ b/frappe/integrations/doctype/dropbox_backup/dropbox_backup.json @@ -0,0 +1,210 @@ +{ + "allow_copy": 0, + "allow_import": 0, + "allow_rename": 0, + "creation": "2015-09-24 01:16:21.711868", + "custom": 0, + "docstatus": 0, + "doctype": "DocType", + "document_type": "Setup", + "fields": [ + { + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "fieldname": "send_backups_to_dropbox", + "fieldtype": "Check", + "hidden": 0, + "ignore_user_permissions": 0, + "in_filter": 0, + "in_list_view": 0, + "label": "Send Backups to Dropbox", + "no_copy": 0, + "permlevel": 0, + "precision": "", + "print_hide": 0, + "read_only": 0, + "report_hide": 0, + "reqd": 0, + "search_index": 0, + "set_only_once": 0, + "unique": 0 + }, + { + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "default": "Daily", + "depends_on": "send_backups_to_dropbox", + "fieldname": "upload_backups_to_dropbox", + "fieldtype": "Select", + "hidden": 0, + "ignore_user_permissions": 0, + "in_filter": 0, + "in_list_view": 0, + "label": "Upload Frequency", + "no_copy": 0, + "options": "Daily\nWeekly", + "permlevel": 0, + "precision": "", + "print_hide": 0, + "read_only": 0, + "report_hide": 0, + "reqd": 0, + "search_index": 0, + "set_only_once": 0, + "unique": 0 + }, + { + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "depends_on": "send_backups_to_dropbox", + "fieldname": "send_notifications_to", + "fieldtype": "Data", + "hidden": 0, + "ignore_user_permissions": 0, + "in_filter": 0, + "in_list_view": 0, + "label": "Send Notifications To", + "no_copy": 0, + "permlevel": 0, + "precision": "", + "print_hide": 0, + "read_only": 0, + "report_hide": 0, + "reqd": 0, + "search_index": 0, + "set_only_once": 0, + "unique": 0 + }, + { + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "fieldname": "dropbox_access_key", + "fieldtype": "Data", + "hidden": 1, + "ignore_user_permissions": 0, + "in_filter": 0, + "in_list_view": 0, + "label": "Dropbox Access Key", + "no_copy": 0, + "permlevel": 0, + "precision": "", + "print_hide": 0, + "read_only": 0, + "report_hide": 0, + "reqd": 0, + "search_index": 0, + "set_only_once": 0, + "unique": 0 + }, + { + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "fieldname": "dropbox_access_secret", + "fieldtype": "Data", + "hidden": 1, + "ignore_user_permissions": 0, + "in_filter": 0, + "in_list_view": 0, + "label": "Dropbox Access Secret", + "no_copy": 0, + "permlevel": 0, + "precision": "", + "print_hide": 0, + "read_only": 0, + "report_hide": 0, + "reqd": 0, + "search_index": 0, + "set_only_once": 0, + "unique": 0 + }, + { + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "fieldname": "dropbox_access_allowed", + "fieldtype": "Data", + "hidden": 1, + "ignore_user_permissions": 0, + "in_filter": 0, + "in_list_view": 0, + "label": "Dropbox Access Allowed", + "no_copy": 0, + "permlevel": 0, + "precision": "", + "print_hide": 0, + "read_only": 0, + "report_hide": 0, + "reqd": 0, + "search_index": 0, + "set_only_once": 0, + "unique": 0 + }, + { + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "depends_on": "send_backups_to_dropbox", + "fieldname": "allow_dropbox_access", + "fieldtype": "Button", + "hidden": 0, + "ignore_user_permissions": 0, + "in_filter": 0, + "in_list_view": 0, + "label": "Allow Dropbox Access", + "no_copy": 0, + "permlevel": 0, + "precision": "", + "print_hide": 0, + "read_only": 0, + "report_hide": 0, + "reqd": 0, + "search_index": 0, + "set_only_once": 0, + "unique": 0 + } + ], + "hide_heading": 0, + "hide_toolbar": 0, + "in_create": 0, + "in_dialog": 0, + "is_submittable": 0, + "issingle": 1, + "istable": 0, + "modified": "2015-09-24 01:42:25.670481", + "modified_by": "Administrator", + "module": "Integrations", + "name": "Dropbox Backup", + "name_case": "", + "owner": "Administrator", + "permissions": [ + { + "amend": 0, + "apply_user_permissions": 0, + "cancel": 0, + "create": 1, + "delete": 1, + "email": 1, + "export": 0, + "if_owner": 0, + "import": 0, + "permlevel": 0, + "print": 1, + "read": 1, + "report": 0, + "role": "System Manager", + "set_user_permissions": 0, + "share": 1, + "submit": 0, + "write": 1 + } + ], + "read_only": 0, + "read_only_onload": 0, + "sort_field": "modified", + "sort_order": "DESC" +} \ No newline at end of file diff --git a/frappe/integrations/doctype/dropbox_backup/dropbox_backup.py b/frappe/integrations/doctype/dropbox_backup/dropbox_backup.py new file mode 100644 index 0000000000..46be927c9f --- /dev/null +++ b/frappe/integrations/doctype/dropbox_backup/dropbox_backup.py @@ -0,0 +1,207 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and contributors +# For license information, please see license.txt + +# SETUP: +# install pip install --upgrade dropbox +# +# Create new Dropbox App +# +# in conf.py, set oauth2 settings +# dropbox_access_key +# dropbox_access_secret + +from __future__ import unicode_literals +import frappe +from frappe.model.document import Document +from frappe.utils import cint, split_emails, get_request_site_address, cstr +import os +from frappe import _ + +ignore_list = [".DS_Store"] + +class DropboxBackup(Document): + pass + +def take_backups_daily(): + take_backups_if("Daily") + +def take_backups_weekly(): + take_backups_if("Weekly") + +def take_backups_if(freq): + if cint(frappe.db.get_value("Dropbox Backup", None, "send_backups_to_dropbox")): + if frappe.db.get_value("Dropbox Backup", None, "upload_backups_to_dropbox")==freq: + take_backups_dropbox() + +@frappe.whitelist() +def take_backups_dropbox(): + did_not_upload, error_log = [], [] + try: + from frappe.integrations.doctype.drobox_backup.dropbox_backup import backup_to_dropbox + did_not_upload, error_log = backup_to_dropbox() + if did_not_upload: raise Exception + + send_email(True, "Dropbox") + except Exception: + file_and_error = [" - ".join(f) for f in zip(did_not_upload, error_log)] + error_message = ("\n".join(file_and_error) + "\n" + frappe.get_traceback()) + frappe.errprint(error_message) + send_email(False, "Dropbox", error_message) + +def send_email(success, service_name, error_status=None): + if success: + subject = "Backup Upload Successful" + message ="""

Backup Uploaded Successfully

Hi there, this is just to inform you + that your backup was successfully uploaded to your %s account. So relax!

+ """ % service_name + + else: + subject = "[Warning] Backup Upload Failed" + message ="""

Backup Upload Failed

Oops, your automated backup to %s + failed.

+

Error message: %s

+

Please contact your system manager for more information.

+ """ % (service_name, error_status) + + if not frappe.db: + frappe.connect() + + recipients = split_emails(frappe.db.get_value("Dropbox Backup", None, "send_notifications_to")) + frappe.sendmail(recipients=recipients, subject=subject, message=message) + +@frappe.whitelist() +def get_dropbox_authorize_url(): + sess = get_dropbox_session() + request_token = sess.obtain_request_token() + return_address = get_request_site_address(True) \ + + "?cmd=frappe.integrations.doctype.drobox_backup.dropbox_backup.dropbox_callback" + + url = sess.build_authorize_url(request_token, return_address) + + return { + "url": url, + "key": request_token.key, + "secret": request_token.secret, + } + +@frappe.whitelist(allow_guest=True) +def dropbox_callback(oauth_token=None, not_approved=False): + from dropbox import client + if not not_approved: + if frappe.db.get_value("Dropbox Backup", None, "dropbox_access_key")==oauth_token: + allowed = 1 + message = "Dropbox access allowed." + + sess = get_dropbox_session() + sess.set_request_token(frappe.db.get_value("Dropbox Backup", None, "dropbox_access_key"), + frappe.db.get_value("Dropbox Backup", None, "dropbox_access_secret")) + access_token = sess.obtain_access_token() + frappe.db.set_value("Dropbox Backup", "Dropbox Backup", "dropbox_access_key", access_token.key) + frappe.db.set_value("Dropbox Backup", "Dropbox Backup", "dropbox_access_secret", access_token.secret) + frappe.db.set_value("Dropbox Backup", "Dropbox Backup", "dropbox_access_allowed", allowed) + frappe.db.set_value("Dropbox Backup", "Dropbox Backup", "send_backups_to_dropbox", 1) + dropbox_client = client.DropboxClient(sess) + try: + dropbox_client.file_create_folder("files") + except: + pass + + else: + allowed = 0 + message = "Illegal Access Token Please try again." + else: + allowed = 0 + message = "Dropbox Access not approved." + + frappe.local.message_title = "Dropbox Approval" + frappe.local.message = "

%s

Please close this window.

" % message + + if allowed: + frappe.local.message_success = True + + frappe.db.commit() + frappe.response['type'] = 'page' + frappe.response['page_name'] = 'message.html' + +def backup_to_dropbox(): + from dropbox import client, session + from frappe.utils.backups import new_backup + from frappe.utils import get_files_path, get_backups_path + if not frappe.db: + frappe.connect() + + sess = session.DropboxSession(frappe.conf.dropbox_access_key, frappe.conf.dropbox_secret_key, "app_folder") + + sess.set_token(frappe.db.get_value("Dropbox Backup", None, "dropbox_access_key"), + frappe.db.get_value("Dropbox Backup", None, "dropbox_access_secret")) + + dropbox_client = client.DropboxClient(sess) + + # upload database + backup = new_backup() + filename = os.path.join(get_backups_path(), os.path.basename(backup.backup_path_db)) + upload_file_to_dropbox(filename, "/database", dropbox_client) + + frappe.db.close() + response = dropbox_client.metadata("/files") + + # upload files to files folder + did_not_upload = [] + error_log = [] + path = get_files_path() + for filename in os.listdir(path): + filename = cstr(filename) + + if filename in ignore_list: + continue + + found = False + filepath = os.path.join(path, filename) + for file_metadata in response["contents"]: + if os.path.basename(filepath) == os.path.basename(file_metadata["path"]) and os.stat(filepath).st_size == int(file_metadata["bytes"]): + found = True + break + if not found: + try: + upload_file_to_dropbox(filepath, "/files", dropbox_client) + except Exception: + did_not_upload.append(filename) + error_log.append(frappe.get_traceback()) + + frappe.connect() + return did_not_upload, list(set(error_log)) + +def get_dropbox_session(): + try: + from dropbox import session + except: + frappe.msgprint(_("Please install dropbox python module"), raise_exception=1) + + if not (frappe.conf.dropbox_access_key or frappe.conf.dropbox_secret_key): + frappe.throw(_("Please set Dropbox access keys in your site config")) + + sess = session.DropboxSession(frappe.conf.dropbox_access_key, frappe.conf.dropbox_secret_key, "app_folder") + return sess + +def upload_file_to_dropbox(filename, folder, dropbox_client): + from dropbox import rest + size = os.stat(filename).st_size + + with open(filename, 'r') as f: + # if max packet size reached, use chunked uploader + max_packet_size = 4194304 + + if size > max_packet_size: + uploader = dropbox_client.get_chunked_uploader(f, size) + while uploader.offset < size: + try: + uploader.upload_chunked() + uploader.finish(folder + "/" + os.path.basename(filename), overwrite=True) + except rest.ErrorResponse: + pass + else: + dropbox_client.put_file(folder + "/" + os.path.basename(filename), f, overwrite=True) + +if __name__=="__main__": + backup_to_dropbox() diff --git a/frappe/patches.txt b/frappe/patches.txt index 5ff8529524..1746f616e2 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -92,3 +92,4 @@ frappe.patches.v6_0.document_type_rename frappe.patches.v6_0.fix_ghana_currency frappe.patches.v6_2.ignore_user_permissions_if_missing execute:frappe.db.sql("delete from tabSessions where user is null") +frappe.patches.v6_2.rename_backup_manager diff --git a/frappe/patches/v6_2/rename_backup_manager.py b/frappe/patches/v6_2/rename_backup_manager.py new file mode 100644 index 0000000000..98fc2fcc65 --- /dev/null +++ b/frappe/patches/v6_2/rename_backup_manager.py @@ -0,0 +1,10 @@ +import frappe + +def execute(): + dropbox_backup = frappe.get_doc("Dropbox Backup", "Dropbox Backup") + for df in dropbox_backup.meta.fields: + value = frappe.db.get_single_value("Backup Manager", df.fieldname) + if value: + dropbox_backup.set(df.fieldname, value) + + dropbox_backup.save()