[cleanup] added dropbox backup

This commit is contained in:
Rushabh Mehta 2015-09-24 18:15:58 +05:30 committed by Anand Doshi
parent 3a37c25144
commit 4e04d40d33
14 changed files with 562 additions and 4 deletions

View file

@ -4,7 +4,7 @@ from frappe import _
def get_data():
return [
{
"label": _("Documents"),
"label": _("Setup"),
"icon": "icon-star",
"items": [
{
@ -14,7 +14,7 @@ def get_data():
},
{
"type": "doctype",
"name": "Backup Manager",
"name": "Dropbox Backup",
"description": _("Manage cloud backups on Dropbox"),
"hide_count": True
}

View file

@ -43,7 +43,7 @@
"hidden": 0,
"ignore_user_permissions": 0,
"in_filter": 0,
"in_list_view": 1,
"in_list_view": 0,
"label": "Communication Medium",
"no_copy": 0,
"options": "\nChat\nPhone\nEmail\nSMS\nVisit\nOther",
@ -573,7 +573,7 @@
"is_submittable": 0,
"issingle": 0,
"istable": 0,
"modified": "2015-09-15 05:51:16.112080",
"modified": "2015-09-24 02:27:43.536919",
"modified_by": "Administrator",
"module": "Core",
"name": "Communication",

View file

View file

@ -0,0 +1,32 @@
<!-- jinja -->
<table class="table table-striped table-bordered" style="max-width: 600px;">
<thead>
<tr>
<th style="width: 30%;">
{{ _("Date") }}
</th>
<th style="width: 50%;">
{{ _("File") }}
</th>
<th>
{{ _("Size") }}
</th>
</tr>
</thead>
<tbody>
{% for f in files %}
<tr>
<td>
{{ f[1] }}
</td>
<td>
<a href="{{ f[0] }}" target="_blank">{{ f[0] }}</a>
</td>
<td>
{{ f[2] }}
</td>
</tr>
{% endfor %}
</tbody>
</table>

View file

@ -0,0 +1,9 @@
frappe.pages['backups'].on_page_load = function(wrapper) {
var page = frappe.ui.make_app_page({
parent: wrapper,
title: 'Download Backups',
single_column: true
});
$(frappe.render_template("backups")).appendTo(page.body.addClass("no-border"));
}

View file

@ -0,0 +1,21 @@
{
"content": null,
"creation": "2015-09-24 01:26:06.225378",
"docstatus": 0,
"doctype": "Page",
"modified": "2015-09-24 01:26:06.225378",
"modified_by": "Administrator",
"module": "Desk",
"name": "backups",
"owner": "Administrator",
"page_name": "backups",
"roles": [
{
"role": "System Manager"
}
],
"script": null,
"standard": "Yes",
"style": null,
"title": "Download Backups"
}

View file

@ -0,0 +1,24 @@
import os
from frappe.utils import get_site_path
from frappe.utils.data import convert_utc_to_user_timezone
import datetime
def get_context(context):
def get_time(path):
dt = os.path.getmtime(path)
return convert_utc_to_user_timezone(datetime.datetime.utcfromtimestamp(dt)).strftime('%Y-%m-%d %H:%M')
def get_size(path):
size = os.path.getsize(path)
if size > 1048576:
return "{0:.1f}M".format(float(size) / 1048576)
else:
return "{0:.1f}K".format(float(size) / 1024)
path = get_site_path('private', 'backups')
files = [x for x in os.listdir(path) if os.path.isfile(os.path.join(path, x))]
files = [('/backups/' + _file,
get_time(os.path.join(path, _file)),
get_size(os.path.join(path, _file))) for _file in files]
return {"files": files}

View file

@ -148,7 +148,14 @@ scheduler_events = {
"frappe.sessions.clear_expired_sessions",
"frappe.email.doctype.email_alert.email_alert.trigger_daily_alerts",
"frappe.async.remove_old_task_logs",
],
"daily_long": [
"frappe.integrations.doctype.dropbox_backup.dropbox_backup.take_backups_daily"
],
"weekly_long": [
"frappe.integrations.doctype.dropbox_backup.dropbox_backup.take_backups_weekly"
]
}
default_background = "/assets/frappe/images/ui/into-the-dawn.jpg"

View file

@ -0,0 +1,37 @@
$.extend(cur_frm.cscript, {
onload_post_render: function() {
cur_frm.fields_dict.allow_dropbox_access.$input.addClass("btn-primary");
},
refresh: function() {
cur_frm.disable_save();
},
validate_send_notifications_to: function() {
if(!cur_frm.doc.send_notifications_to) {
msgprint(__("Please specify") + ": " +
__(frappe.meta.get_label(cur_frm.doctype,
"send_notifications_to")));
return false;
}
return true;
},
allow_dropbox_access: function() {
if(cur_frm.cscript.validate_send_notifications_to()) {
return frappe.call({
method: "frappe.integrations.doctype.dropbox_backup.dropbox_backup.get_dropbox_authorize_url",
callback: function(r) {
if(!r.exc) {
cur_frm.set_value("dropbox_access_secret", r.message.secret);
cur_frm.set_value("dropbox_access_key", r.message.key);
cur_frm.save(null, function() {
window.open(r.message.url);
});
}
}
});
}
}
});

View file

@ -0,0 +1,210 @@
{
"allow_copy": 0,
"allow_import": 0,
"allow_rename": 0,
"creation": "2015-09-24 01:16:21.711868",
"custom": 0,
"docstatus": 0,
"doctype": "DocType",
"document_type": "Setup",
"fields": [
{
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"fieldname": "send_backups_to_dropbox",
"fieldtype": "Check",
"hidden": 0,
"ignore_user_permissions": 0,
"in_filter": 0,
"in_list_view": 0,
"label": "Send Backups to Dropbox",
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"read_only": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"unique": 0
},
{
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"default": "Daily",
"depends_on": "send_backups_to_dropbox",
"fieldname": "upload_backups_to_dropbox",
"fieldtype": "Select",
"hidden": 0,
"ignore_user_permissions": 0,
"in_filter": 0,
"in_list_view": 0,
"label": "Upload Frequency",
"no_copy": 0,
"options": "Daily\nWeekly",
"permlevel": 0,
"precision": "",
"print_hide": 0,
"read_only": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"unique": 0
},
{
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"depends_on": "send_backups_to_dropbox",
"fieldname": "send_notifications_to",
"fieldtype": "Data",
"hidden": 0,
"ignore_user_permissions": 0,
"in_filter": 0,
"in_list_view": 0,
"label": "Send Notifications To",
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"read_only": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"unique": 0
},
{
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"fieldname": "dropbox_access_key",
"fieldtype": "Data",
"hidden": 1,
"ignore_user_permissions": 0,
"in_filter": 0,
"in_list_view": 0,
"label": "Dropbox Access Key",
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"read_only": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"unique": 0
},
{
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"fieldname": "dropbox_access_secret",
"fieldtype": "Data",
"hidden": 1,
"ignore_user_permissions": 0,
"in_filter": 0,
"in_list_view": 0,
"label": "Dropbox Access Secret",
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"read_only": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"unique": 0
},
{
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"fieldname": "dropbox_access_allowed",
"fieldtype": "Data",
"hidden": 1,
"ignore_user_permissions": 0,
"in_filter": 0,
"in_list_view": 0,
"label": "Dropbox Access Allowed",
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"read_only": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"unique": 0
},
{
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"depends_on": "send_backups_to_dropbox",
"fieldname": "allow_dropbox_access",
"fieldtype": "Button",
"hidden": 0,
"ignore_user_permissions": 0,
"in_filter": 0,
"in_list_view": 0,
"label": "Allow Dropbox Access",
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"read_only": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"unique": 0
}
],
"hide_heading": 0,
"hide_toolbar": 0,
"in_create": 0,
"in_dialog": 0,
"is_submittable": 0,
"issingle": 1,
"istable": 0,
"modified": "2015-09-24 01:42:25.670481",
"modified_by": "Administrator",
"module": "Integrations",
"name": "Dropbox Backup",
"name_case": "",
"owner": "Administrator",
"permissions": [
{
"amend": 0,
"apply_user_permissions": 0,
"cancel": 0,
"create": 1,
"delete": 1,
"email": 1,
"export": 0,
"if_owner": 0,
"import": 0,
"permlevel": 0,
"print": 1,
"read": 1,
"report": 0,
"role": "System Manager",
"set_user_permissions": 0,
"share": 1,
"submit": 0,
"write": 1
}
],
"read_only": 0,
"read_only_onload": 0,
"sort_field": "modified",
"sort_order": "DESC"
}

View file

@ -0,0 +1,207 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and contributors
# For license information, please see license.txt
# SETUP:
# install pip install --upgrade dropbox
#
# Create new Dropbox App
#
# in conf.py, set oauth2 settings
# dropbox_access_key
# dropbox_access_secret
from __future__ import unicode_literals
import frappe
from frappe.model.document import Document
from frappe.utils import cint, split_emails, get_request_site_address, cstr
import os
from frappe import _
ignore_list = [".DS_Store"]
class DropboxBackup(Document):
pass
def take_backups_daily():
take_backups_if("Daily")
def take_backups_weekly():
take_backups_if("Weekly")
def take_backups_if(freq):
if cint(frappe.db.get_value("Dropbox Backup", None, "send_backups_to_dropbox")):
if frappe.db.get_value("Dropbox Backup", None, "upload_backups_to_dropbox")==freq:
take_backups_dropbox()
@frappe.whitelist()
def take_backups_dropbox():
did_not_upload, error_log = [], []
try:
from frappe.integrations.doctype.drobox_backup.dropbox_backup import backup_to_dropbox
did_not_upload, error_log = backup_to_dropbox()
if did_not_upload: raise Exception
send_email(True, "Dropbox")
except Exception:
file_and_error = [" - ".join(f) for f in zip(did_not_upload, error_log)]
error_message = ("\n".join(file_and_error) + "\n" + frappe.get_traceback())
frappe.errprint(error_message)
send_email(False, "Dropbox", error_message)
def send_email(success, service_name, error_status=None):
if success:
subject = "Backup Upload Successful"
message ="""<h3>Backup Uploaded Successfully</h3><p>Hi there, this is just to inform you
that your backup was successfully uploaded to your %s account. So relax!</p>
""" % service_name
else:
subject = "[Warning] Backup Upload Failed"
message ="""<h3>Backup Upload Failed</h3><p>Oops, your automated backup to %s
failed.</p>
<p>Error message: %s</p>
<p>Please contact your system manager for more information.</p>
""" % (service_name, error_status)
if not frappe.db:
frappe.connect()
recipients = split_emails(frappe.db.get_value("Dropbox Backup", None, "send_notifications_to"))
frappe.sendmail(recipients=recipients, subject=subject, message=message)
@frappe.whitelist()
def get_dropbox_authorize_url():
sess = get_dropbox_session()
request_token = sess.obtain_request_token()
return_address = get_request_site_address(True) \
+ "?cmd=frappe.integrations.doctype.drobox_backup.dropbox_backup.dropbox_callback"
url = sess.build_authorize_url(request_token, return_address)
return {
"url": url,
"key": request_token.key,
"secret": request_token.secret,
}
@frappe.whitelist(allow_guest=True)
def dropbox_callback(oauth_token=None, not_approved=False):
from dropbox import client
if not not_approved:
if frappe.db.get_value("Dropbox Backup", None, "dropbox_access_key")==oauth_token:
allowed = 1
message = "Dropbox access allowed."
sess = get_dropbox_session()
sess.set_request_token(frappe.db.get_value("Dropbox Backup", None, "dropbox_access_key"),
frappe.db.get_value("Dropbox Backup", None, "dropbox_access_secret"))
access_token = sess.obtain_access_token()
frappe.db.set_value("Dropbox Backup", "Dropbox Backup", "dropbox_access_key", access_token.key)
frappe.db.set_value("Dropbox Backup", "Dropbox Backup", "dropbox_access_secret", access_token.secret)
frappe.db.set_value("Dropbox Backup", "Dropbox Backup", "dropbox_access_allowed", allowed)
frappe.db.set_value("Dropbox Backup", "Dropbox Backup", "send_backups_to_dropbox", 1)
dropbox_client = client.DropboxClient(sess)
try:
dropbox_client.file_create_folder("files")
except:
pass
else:
allowed = 0
message = "Illegal Access Token Please try again."
else:
allowed = 0
message = "Dropbox Access not approved."
frappe.local.message_title = "Dropbox Approval"
frappe.local.message = "<h3>%s</h3><p>Please close this window.</p>" % message
if allowed:
frappe.local.message_success = True
frappe.db.commit()
frappe.response['type'] = 'page'
frappe.response['page_name'] = 'message.html'
def backup_to_dropbox():
from dropbox import client, session
from frappe.utils.backups import new_backup
from frappe.utils import get_files_path, get_backups_path
if not frappe.db:
frappe.connect()
sess = session.DropboxSession(frappe.conf.dropbox_access_key, frappe.conf.dropbox_secret_key, "app_folder")
sess.set_token(frappe.db.get_value("Dropbox Backup", None, "dropbox_access_key"),
frappe.db.get_value("Dropbox Backup", None, "dropbox_access_secret"))
dropbox_client = client.DropboxClient(sess)
# upload database
backup = new_backup()
filename = os.path.join(get_backups_path(), os.path.basename(backup.backup_path_db))
upload_file_to_dropbox(filename, "/database", dropbox_client)
frappe.db.close()
response = dropbox_client.metadata("/files")
# upload files to files folder
did_not_upload = []
error_log = []
path = get_files_path()
for filename in os.listdir(path):
filename = cstr(filename)
if filename in ignore_list:
continue
found = False
filepath = os.path.join(path, filename)
for file_metadata in response["contents"]:
if os.path.basename(filepath) == os.path.basename(file_metadata["path"]) and os.stat(filepath).st_size == int(file_metadata["bytes"]):
found = True
break
if not found:
try:
upload_file_to_dropbox(filepath, "/files", dropbox_client)
except Exception:
did_not_upload.append(filename)
error_log.append(frappe.get_traceback())
frappe.connect()
return did_not_upload, list(set(error_log))
def get_dropbox_session():
try:
from dropbox import session
except:
frappe.msgprint(_("Please install dropbox python module"), raise_exception=1)
if not (frappe.conf.dropbox_access_key or frappe.conf.dropbox_secret_key):
frappe.throw(_("Please set Dropbox access keys in your site config"))
sess = session.DropboxSession(frappe.conf.dropbox_access_key, frappe.conf.dropbox_secret_key, "app_folder")
return sess
def upload_file_to_dropbox(filename, folder, dropbox_client):
from dropbox import rest
size = os.stat(filename).st_size
with open(filename, 'r') as f:
# if max packet size reached, use chunked uploader
max_packet_size = 4194304
if size > max_packet_size:
uploader = dropbox_client.get_chunked_uploader(f, size)
while uploader.offset < size:
try:
uploader.upload_chunked()
uploader.finish(folder + "/" + os.path.basename(filename), overwrite=True)
except rest.ErrorResponse:
pass
else:
dropbox_client.put_file(folder + "/" + os.path.basename(filename), f, overwrite=True)
if __name__=="__main__":
backup_to_dropbox()

View file

@ -92,3 +92,4 @@ frappe.patches.v6_0.document_type_rename
frappe.patches.v6_0.fix_ghana_currency
frappe.patches.v6_2.ignore_user_permissions_if_missing
execute:frappe.db.sql("delete from tabSessions where user is null")
frappe.patches.v6_2.rename_backup_manager

View file

@ -0,0 +1,10 @@
import frappe
def execute():
dropbox_backup = frappe.get_doc("Dropbox Backup", "Dropbox Backup")
for df in dropbox_backup.meta.fields:
value = frappe.db.get_single_value("Backup Manager", df.fieldname)
if value:
dropbox_backup.set(df.fieldname, value)
dropbox_backup.save()