feat: Offsite backup (#8241)
* feat: utility to get file size * feat: if backup file is greater than 1gb then consider latest backup site instead of taking new * fix: remove unwanted import * chore: fix condition inside offsite backup utils Signed-off-by: Chinmay D. Pai <chinmaydpai@gmail.com> * chore: change email field to notify_email Signed-off-by: Chinmay D. Pai <chinmaydpai@gmail.com> * chore: fix deepsource issues Signed-off-by: Chinmay D. Pai <chinmaydpai@gmail.com> * feat: add offsite_backup support for google drive Signed-off-by: Chinmay D. Pai <chinmaydpai@gmail.com> * chore: get recipients from within send_email pass email_field to send_email instead of calling two functions Signed-off-by: Chinmay D. Pai <chinmaydpai@gmail.com> * chore: set flag within validate_file_size Signed-off-by: Chinmay D. Pai <chinmaydpai@gmail.com> * feat: get latest file backup when specified Signed-off-by: Chinmay D. Pai <chinmaydpai@gmail.com> * chore: fix deepsource issues Signed-off-by: Chinmay D. Pai <chinmaydpai@gmail.com> * chore: fix incorrectly spelled dropbox settings Signed-off-by: Chinmay D. Pai <chinmaydpai@gmail.com> * chore: implement file backup logic for aws s3 Signed-off-by: Chinmay D. Pai <chinmaydpai@gmail.com> * chore: fix deepsource issues Signed-off-by: Chinmay D. Pai <chinmaydpai@gmail.com> * chore: fix deepsource issues Signed-off-by: Chinmay D. Pai <chinmaydpai@gmail.com> Co-authored-by: Chinmay D. Pai <chinmaydpai@gmail.com> Co-authored-by: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com>
This commit is contained in:
Saurabh
GitHub
parent
446c0636ac
commit
1bc97e8f8a
8 changed files with 346 additions and 953 deletions
|
|
@ -1,487 +1,129 @@
|
|||
{
|
||||
"allow_copy": 0,
|
||||
"allow_events_in_timeline": 0,
|
||||
"allow_guest_to_view": 0,
|
||||
"allow_import": 0,
|
||||
"allow_rename": 0,
|
||||
"beta": 0,
|
||||
"creation": "2016-09-21 10:12:57.399174",
|
||||
"custom": 0,
|
||||
"docstatus": 0,
|
||||
"doctype": "DocType",
|
||||
"document_type": "System",
|
||||
"editable_grid": 1,
|
||||
"creation": "2016-09-21 10:12:57.399174",
|
||||
"doctype": "DocType",
|
||||
"document_type": "System",
|
||||
"editable_grid": 1,
|
||||
"field_order": [
|
||||
"enabled",
|
||||
"send_notifications_to",
|
||||
"send_email_for_successful_backup",
|
||||
"backup_frequency",
|
||||
"limit_no_of_backups",
|
||||
"no_of_backups",
|
||||
"file_backup",
|
||||
"app_access_key",
|
||||
"app_secret_key",
|
||||
"allow_dropbox_access",
|
||||
"dropbox_access_key",
|
||||
"dropbox_access_secret",
|
||||
"dropbox_access_token"
|
||||
],
|
||||
"fields": [
|
||||
{
|
||||
"allow_bulk_edit": 0,
|
||||
"allow_in_quick_entry": 0,
|
||||
"allow_on_submit": 0,
|
||||
"bold": 0,
|
||||
"collapsible": 0,
|
||||
"columns": 0,
|
||||
"fieldname": "enabled",
|
||||
"fieldtype": "Check",
|
||||
"hidden": 0,
|
||||
"ignore_user_permissions": 0,
|
||||
"ignore_xss_filter": 0,
|
||||
"in_filter": 0,
|
||||
"in_global_search": 0,
|
||||
"in_list_view": 0,
|
||||
"in_standard_filter": 0,
|
||||
"label": "Enabled",
|
||||
"length": 0,
|
||||
"no_copy": 0,
|
||||
"permlevel": 0,
|
||||
"precision": "",
|
||||
"print_hide": 0,
|
||||
"print_hide_if_no_value": 0,
|
||||
"read_only": 0,
|
||||
"remember_last_selected_value": 0,
|
||||
"report_hide": 0,
|
||||
"reqd": 0,
|
||||
"search_index": 0,
|
||||
"set_only_once": 0,
|
||||
"translatable": 0,
|
||||
"unique": 0
|
||||
},
|
||||
"default": "0",
|
||||
"fieldname": "enabled",
|
||||
"fieldtype": "Check",
|
||||
"label": "Enabled"
|
||||
},
|
||||
{
|
||||
"allow_bulk_edit": 0,
|
||||
"allow_in_quick_entry": 0,
|
||||
"allow_on_submit": 0,
|
||||
"bold": 0,
|
||||
"collapsible": 0,
|
||||
"columns": 0,
|
||||
"fieldname": "send_notifications_to",
|
||||
"fieldtype": "Data",
|
||||
"hidden": 0,
|
||||
"ignore_user_permissions": 0,
|
||||
"ignore_xss_filter": 0,
|
||||
"in_filter": 0,
|
||||
"in_global_search": 0,
|
||||
"in_list_view": 1,
|
||||
"in_standard_filter": 0,
|
||||
"label": "Send Notifications To",
|
||||
"length": 0,
|
||||
"no_copy": 0,
|
||||
"permlevel": 0,
|
||||
"precision": "",
|
||||
"print_hide": 0,
|
||||
"print_hide_if_no_value": 0,
|
||||
"read_only": 0,
|
||||
"remember_last_selected_value": 0,
|
||||
"report_hide": 0,
|
||||
"reqd": 1,
|
||||
"search_index": 0,
|
||||
"set_only_once": 0,
|
||||
"translatable": 0,
|
||||
"unique": 0
|
||||
},
|
||||
"fieldname": "send_notifications_to",
|
||||
"fieldtype": "Data",
|
||||
"in_list_view": 1,
|
||||
"label": "Send Notifications To",
|
||||
"reqd": 1
|
||||
},
|
||||
{
|
||||
"allow_bulk_edit": 0,
|
||||
"allow_in_quick_entry": 0,
|
||||
"allow_on_submit": 0,
|
||||
"bold": 0,
|
||||
"collapsible": 0,
|
||||
"columns": 0,
|
||||
"default": "1",
|
||||
"description": "Note: By default emails for failed backups are sent.",
|
||||
"fieldname": "send_email_for_successful_backup",
|
||||
"fieldtype": "Check",
|
||||
"hidden": 0,
|
||||
"ignore_user_permissions": 0,
|
||||
"ignore_xss_filter": 0,
|
||||
"in_filter": 0,
|
||||
"in_global_search": 0,
|
||||
"in_list_view": 0,
|
||||
"in_standard_filter": 0,
|
||||
"label": "Send Email for Successful Backup",
|
||||
"length": 0,
|
||||
"no_copy": 0,
|
||||
"options": "",
|
||||
"permlevel": 0,
|
||||
"precision": "",
|
||||
"print_hide": 0,
|
||||
"print_hide_if_no_value": 0,
|
||||
"read_only": 0,
|
||||
"remember_last_selected_value": 0,
|
||||
"report_hide": 0,
|
||||
"reqd": 0,
|
||||
"search_index": 0,
|
||||
"set_only_once": 0,
|
||||
"translatable": 0,
|
||||
"unique": 0
|
||||
},
|
||||
"default": "1",
|
||||
"description": "Note: By default emails for failed backups are sent.",
|
||||
"fieldname": "send_email_for_successful_backup",
|
||||
"fieldtype": "Check",
|
||||
"label": "Send Email for Successful Backup"
|
||||
},
|
||||
{
|
||||
"allow_bulk_edit": 0,
|
||||
"allow_in_quick_entry": 0,
|
||||
"allow_on_submit": 0,
|
||||
"bold": 0,
|
||||
"collapsible": 0,
|
||||
"columns": 0,
|
||||
"fieldname": "backup_frequency",
|
||||
"fieldtype": "Select",
|
||||
"hidden": 0,
|
||||
"ignore_user_permissions": 0,
|
||||
"ignore_xss_filter": 0,
|
||||
"in_filter": 0,
|
||||
"in_global_search": 0,
|
||||
"in_list_view": 1,
|
||||
"in_standard_filter": 0,
|
||||
"label": "Backup Frequency",
|
||||
"length": 0,
|
||||
"no_copy": 0,
|
||||
"options": "\nDaily\nWeekly",
|
||||
"permlevel": 0,
|
||||
"precision": "",
|
||||
"print_hide": 0,
|
||||
"print_hide_if_no_value": 0,
|
||||
"read_only": 0,
|
||||
"remember_last_selected_value": 0,
|
||||
"report_hide": 0,
|
||||
"reqd": 1,
|
||||
"search_index": 0,
|
||||
"set_only_once": 0,
|
||||
"translatable": 0,
|
||||
"unique": 0
|
||||
},
|
||||
"fieldname": "backup_frequency",
|
||||
"fieldtype": "Select",
|
||||
"in_list_view": 1,
|
||||
"label": "Backup Frequency",
|
||||
"options": "\nDaily\nWeekly",
|
||||
"reqd": 1
|
||||
},
|
||||
{
|
||||
"allow_bulk_edit": 0,
|
||||
"allow_in_quick_entry": 0,
|
||||
"allow_on_submit": 0,
|
||||
"bold": 0,
|
||||
"collapsible": 0,
|
||||
"columns": 0,
|
||||
"fieldname": "limit_no_of_backups",
|
||||
"fieldtype": "Check",
|
||||
"hidden": 0,
|
||||
"ignore_user_permissions": 0,
|
||||
"ignore_xss_filter": 0,
|
||||
"in_filter": 0,
|
||||
"in_global_search": 0,
|
||||
"in_list_view": 0,
|
||||
"in_standard_filter": 0,
|
||||
"label": "Limit Number of DB Backups",
|
||||
"length": 0,
|
||||
"no_copy": 0,
|
||||
"permlevel": 0,
|
||||
"precision": "",
|
||||
"print_hide": 0,
|
||||
"print_hide_if_no_value": 0,
|
||||
"read_only": 0,
|
||||
"remember_last_selected_value": 0,
|
||||
"report_hide": 0,
|
||||
"reqd": 0,
|
||||
"search_index": 0,
|
||||
"set_only_once": 0,
|
||||
"translatable": 0,
|
||||
"unique": 0
|
||||
},
|
||||
"default": "0",
|
||||
"fieldname": "limit_no_of_backups",
|
||||
"fieldtype": "Check",
|
||||
"label": "Limit Number of DB Backups"
|
||||
},
|
||||
{
|
||||
"allow_bulk_edit": 0,
|
||||
"allow_in_quick_entry": 0,
|
||||
"allow_on_submit": 0,
|
||||
"bold": 0,
|
||||
"collapsible": 0,
|
||||
"columns": 0,
|
||||
"default": "5",
|
||||
"depends_on": "eval:doc.limit_no_of_backups",
|
||||
"fieldname": "no_of_backups",
|
||||
"fieldtype": "Int",
|
||||
"hidden": 0,
|
||||
"ignore_user_permissions": 0,
|
||||
"ignore_xss_filter": 0,
|
||||
"in_filter": 0,
|
||||
"in_global_search": 0,
|
||||
"in_list_view": 0,
|
||||
"in_standard_filter": 0,
|
||||
"label": "Number of DB Backups",
|
||||
"length": 0,
|
||||
"no_copy": 0,
|
||||
"permlevel": 0,
|
||||
"precision": "",
|
||||
"print_hide": 0,
|
||||
"print_hide_if_no_value": 0,
|
||||
"read_only": 0,
|
||||
"remember_last_selected_value": 0,
|
||||
"report_hide": 0,
|
||||
"reqd": 0,
|
||||
"search_index": 0,
|
||||
"set_only_once": 0,
|
||||
"translatable": 0,
|
||||
"unique": 0
|
||||
},
|
||||
"default": "5",
|
||||
"depends_on": "eval:doc.limit_no_of_backups",
|
||||
"fieldname": "no_of_backups",
|
||||
"fieldtype": "Int",
|
||||
"label": "Number of DB Backups"
|
||||
},
|
||||
{
|
||||
"allow_bulk_edit": 0,
|
||||
"allow_in_quick_entry": 0,
|
||||
"allow_on_submit": 0,
|
||||
"bold": 0,
|
||||
"collapsible": 0,
|
||||
"columns": 0,
|
||||
"default": "1",
|
||||
"fieldname": "file_backup",
|
||||
"fieldtype": "Check",
|
||||
"hidden": 0,
|
||||
"ignore_user_permissions": 0,
|
||||
"ignore_xss_filter": 0,
|
||||
"in_filter": 0,
|
||||
"in_global_search": 0,
|
||||
"in_list_view": 0,
|
||||
"in_standard_filter": 0,
|
||||
"label": "File Backup",
|
||||
"length": 0,
|
||||
"no_copy": 0,
|
||||
"permlevel": 0,
|
||||
"precision": "",
|
||||
"print_hide": 0,
|
||||
"print_hide_if_no_value": 0,
|
||||
"read_only": 0,
|
||||
"remember_last_selected_value": 0,
|
||||
"report_hide": 0,
|
||||
"reqd": 0,
|
||||
"search_index": 0,
|
||||
"set_only_once": 0,
|
||||
"translatable": 0,
|
||||
"unique": 0
|
||||
},
|
||||
"default": "1",
|
||||
"fieldname": "file_backup",
|
||||
"fieldtype": "Check",
|
||||
"label": "File Backup"
|
||||
},
|
||||
{
|
||||
"allow_bulk_edit": 0,
|
||||
"allow_in_quick_entry": 0,
|
||||
"allow_on_submit": 0,
|
||||
"bold": 0,
|
||||
"collapsible": 0,
|
||||
"columns": 0,
|
||||
"depends_on": "",
|
||||
"fieldname": "app_access_key",
|
||||
"fieldtype": "Data",
|
||||
"hidden": 0,
|
||||
"ignore_user_permissions": 0,
|
||||
"ignore_xss_filter": 0,
|
||||
"in_filter": 0,
|
||||
"in_global_search": 0,
|
||||
"in_list_view": 0,
|
||||
"in_standard_filter": 0,
|
||||
"label": "App Access Key",
|
||||
"length": 0,
|
||||
"no_copy": 0,
|
||||
"permlevel": 0,
|
||||
"precision": "",
|
||||
"print_hide": 0,
|
||||
"print_hide_if_no_value": 0,
|
||||
"read_only": 0,
|
||||
"remember_last_selected_value": 0,
|
||||
"report_hide": 0,
|
||||
"reqd": 0,
|
||||
"search_index": 0,
|
||||
"set_only_once": 0,
|
||||
"translatable": 0,
|
||||
"unique": 0
|
||||
},
|
||||
"fieldname": "app_access_key",
|
||||
"fieldtype": "Data",
|
||||
"label": "App Access Key"
|
||||
},
|
||||
{
|
||||
"allow_bulk_edit": 0,
|
||||
"allow_in_quick_entry": 0,
|
||||
"allow_on_submit": 0,
|
||||
"bold": 0,
|
||||
"collapsible": 0,
|
||||
"columns": 0,
|
||||
"depends_on": "",
|
||||
"fieldname": "app_secret_key",
|
||||
"fieldtype": "Password",
|
||||
"hidden": 0,
|
||||
"ignore_user_permissions": 0,
|
||||
"ignore_xss_filter": 0,
|
||||
"in_filter": 0,
|
||||
"in_global_search": 0,
|
||||
"in_list_view": 0,
|
||||
"in_standard_filter": 0,
|
||||
"label": "App Secret Key",
|
||||
"length": 0,
|
||||
"no_copy": 0,
|
||||
"permlevel": 0,
|
||||
"precision": "",
|
||||
"print_hide": 0,
|
||||
"print_hide_if_no_value": 0,
|
||||
"read_only": 0,
|
||||
"remember_last_selected_value": 0,
|
||||
"report_hide": 0,
|
||||
"reqd": 0,
|
||||
"search_index": 0,
|
||||
"set_only_once": 0,
|
||||
"translatable": 0,
|
||||
"unique": 0
|
||||
},
|
||||
"fieldname": "app_secret_key",
|
||||
"fieldtype": "Password",
|
||||
"label": "App Secret Key"
|
||||
},
|
||||
{
|
||||
"allow_bulk_edit": 0,
|
||||
"allow_in_quick_entry": 0,
|
||||
"allow_on_submit": 0,
|
||||
"bold": 0,
|
||||
"collapsible": 0,
|
||||
"columns": 0,
|
||||
"fieldname": "allow_dropbox_access",
|
||||
"fieldtype": "Button",
|
||||
"hidden": 0,
|
||||
"ignore_user_permissions": 0,
|
||||
"ignore_xss_filter": 0,
|
||||
"in_filter": 0,
|
||||
"in_global_search": 0,
|
||||
"in_list_view": 0,
|
||||
"in_standard_filter": 0,
|
||||
"label": "Allow Dropbox Access",
|
||||
"length": 0,
|
||||
"no_copy": 0,
|
||||
"permlevel": 0,
|
||||
"precision": "",
|
||||
"print_hide": 0,
|
||||
"print_hide_if_no_value": 0,
|
||||
"read_only": 0,
|
||||
"remember_last_selected_value": 0,
|
||||
"report_hide": 0,
|
||||
"reqd": 0,
|
||||
"search_index": 0,
|
||||
"set_only_once": 0,
|
||||
"translatable": 0,
|
||||
"unique": 0
|
||||
},
|
||||
"fieldname": "allow_dropbox_access",
|
||||
"fieldtype": "Button",
|
||||
"label": "Allow Dropbox Access"
|
||||
},
|
||||
{
|
||||
"allow_bulk_edit": 0,
|
||||
"allow_in_quick_entry": 0,
|
||||
"allow_on_submit": 0,
|
||||
"bold": 0,
|
||||
"collapsible": 0,
|
||||
"columns": 0,
|
||||
"fieldname": "dropbox_access_key",
|
||||
"fieldtype": "Password",
|
||||
"hidden": 1,
|
||||
"ignore_user_permissions": 0,
|
||||
"ignore_xss_filter": 0,
|
||||
"in_filter": 0,
|
||||
"in_global_search": 0,
|
||||
"in_list_view": 0,
|
||||
"in_standard_filter": 0,
|
||||
"label": "Dropbox Access Key",
|
||||
"length": 0,
|
||||
"no_copy": 0,
|
||||
"permlevel": 0,
|
||||
"precision": "",
|
||||
"print_hide": 0,
|
||||
"print_hide_if_no_value": 0,
|
||||
"read_only": 1,
|
||||
"remember_last_selected_value": 0,
|
||||
"report_hide": 0,
|
||||
"reqd": 0,
|
||||
"search_index": 0,
|
||||
"set_only_once": 0,
|
||||
"translatable": 0,
|
||||
"unique": 0
|
||||
},
|
||||
"fieldname": "dropbox_access_key",
|
||||
"fieldtype": "Password",
|
||||
"hidden": 1,
|
||||
"label": "Dropbox Access Key",
|
||||
"read_only": 1
|
||||
},
|
||||
{
|
||||
"allow_bulk_edit": 0,
|
||||
"allow_in_quick_entry": 0,
|
||||
"allow_on_submit": 0,
|
||||
"bold": 0,
|
||||
"collapsible": 0,
|
||||
"columns": 0,
|
||||
"fieldname": "dropbox_access_secret",
|
||||
"fieldtype": "Password",
|
||||
"hidden": 1,
|
||||
"ignore_user_permissions": 0,
|
||||
"ignore_xss_filter": 0,
|
||||
"in_filter": 0,
|
||||
"in_global_search": 0,
|
||||
"in_list_view": 0,
|
||||
"in_standard_filter": 0,
|
||||
"label": "Dropbox Access Secret",
|
||||
"length": 0,
|
||||
"no_copy": 0,
|
||||
"permlevel": 0,
|
||||
"precision": "",
|
||||
"print_hide": 0,
|
||||
"print_hide_if_no_value": 0,
|
||||
"read_only": 1,
|
||||
"remember_last_selected_value": 0,
|
||||
"report_hide": 0,
|
||||
"reqd": 0,
|
||||
"search_index": 0,
|
||||
"set_only_once": 0,
|
||||
"translatable": 0,
|
||||
"unique": 0
|
||||
},
|
||||
"fieldname": "dropbox_access_secret",
|
||||
"fieldtype": "Password",
|
||||
"hidden": 1,
|
||||
"label": "Dropbox Access Secret",
|
||||
"read_only": 1
|
||||
},
|
||||
{
|
||||
"allow_bulk_edit": 0,
|
||||
"allow_in_quick_entry": 0,
|
||||
"allow_on_submit": 0,
|
||||
"bold": 0,
|
||||
"collapsible": 0,
|
||||
"columns": 0,
|
||||
"fieldname": "dropbox_access_token",
|
||||
"fieldtype": "Password",
|
||||
"hidden": 1,
|
||||
"ignore_user_permissions": 0,
|
||||
"ignore_xss_filter": 0,
|
||||
"in_filter": 0,
|
||||
"in_global_search": 0,
|
||||
"in_list_view": 0,
|
||||
"in_standard_filter": 0,
|
||||
"label": "Dropbox Access Token",
|
||||
"length": 0,
|
||||
"no_copy": 0,
|
||||
"permlevel": 0,
|
||||
"precision": "",
|
||||
"print_hide": 0,
|
||||
"print_hide_if_no_value": 0,
|
||||
"read_only": 0,
|
||||
"remember_last_selected_value": 0,
|
||||
"report_hide": 0,
|
||||
"reqd": 0,
|
||||
"search_index": 0,
|
||||
"set_only_once": 0,
|
||||
"translatable": 0,
|
||||
"unique": 0
|
||||
"fieldname": "dropbox_access_token",
|
||||
"fieldtype": "Password",
|
||||
"hidden": 1,
|
||||
"label": "Dropbox Access Token"
|
||||
}
|
||||
],
|
||||
"has_web_view": 0,
|
||||
"hide_heading": 0,
|
||||
"hide_toolbar": 0,
|
||||
"idx": 0,
|
||||
"image_view": 0,
|
||||
"in_create": 1,
|
||||
"is_submittable": 0,
|
||||
"issingle": 1,
|
||||
"istable": 0,
|
||||
"max_attachments": 0,
|
||||
"modified": "2019-01-03 05:44:40.520943",
|
||||
"modified_by": "Administrator",
|
||||
"module": "Integrations",
|
||||
"name": "Dropbox Settings",
|
||||
"name_case": "",
|
||||
"owner": "Administrator",
|
||||
],
|
||||
"in_create": 1,
|
||||
"issingle": 1,
|
||||
"modified": "2019-08-22 16:26:44.468391",
|
||||
"modified_by": "Administrator",
|
||||
"module": "Integrations",
|
||||
"name": "Dropbox Settings",
|
||||
"owner": "Administrator",
|
||||
"permissions": [
|
||||
{
|
||||
"amend": 0,
|
||||
"cancel": 0,
|
||||
"create": 1,
|
||||
"delete": 1,
|
||||
"email": 1,
|
||||
"export": 1,
|
||||
"if_owner": 0,
|
||||
"import": 0,
|
||||
"permlevel": 0,
|
||||
"print": 1,
|
||||
"read": 1,
|
||||
"report": 0,
|
||||
"role": "System Manager",
|
||||
"set_user_permissions": 0,
|
||||
"share": 1,
|
||||
"submit": 0,
|
||||
"create": 1,
|
||||
"delete": 1,
|
||||
"email": 1,
|
||||
"export": 1,
|
||||
"print": 1,
|
||||
"read": 1,
|
||||
"role": "System Manager",
|
||||
"share": 1,
|
||||
"write": 1
|
||||
}
|
||||
],
|
||||
"quick_entry": 0,
|
||||
"read_only": 1,
|
||||
"read_only_onload": 0,
|
||||
"show_name_in_global_search": 0,
|
||||
"sort_field": "modified",
|
||||
"sort_order": "DESC",
|
||||
"track_changes": 1,
|
||||
"track_seen": 0,
|
||||
"track_views": 0
|
||||
}
|
||||
],
|
||||
"read_only": 1,
|
||||
"sort_field": "modified",
|
||||
"sort_order": "DESC",
|
||||
"track_changes": 1
|
||||
}
|
||||
|
|
@ -3,22 +3,25 @@
|
|||
# For license information, please see license.txt
|
||||
|
||||
from __future__ import unicode_literals
|
||||
import dropbox
|
||||
import json
|
||||
import frappe
|
||||
import os
|
||||
from frappe import _
|
||||
from frappe.model.document import Document
|
||||
import dropbox, json
|
||||
from frappe.integrations.offsite_backup_utils import get_latest_backup_file, send_email, validate_file_size
|
||||
from frappe.integrations.utils import make_post_request
|
||||
from frappe.utils import (cint, get_request_site_address,
|
||||
get_files_path, get_backups_path, get_url, encode)
|
||||
from frappe.utils.backups import new_backup
|
||||
from frappe.utils.background_jobs import enqueue
|
||||
from six.moves.urllib.parse import urlparse, parse_qs
|
||||
from frappe.integrations.utils import make_post_request
|
||||
from rq.timeouts import JobTimeoutException
|
||||
from frappe.utils import (cint, split_emails, get_request_site_address,
|
||||
get_files_path, get_backups_path, get_url, encode)
|
||||
from six import text_type
|
||||
|
||||
ignore_list = [".DS_Store"]
|
||||
|
||||
|
||||
class DropboxSettings(Document):
|
||||
def onload(self):
|
||||
if not self.app_access_key and frappe.conf.dropbox_access_key:
|
||||
|
|
@ -48,10 +51,12 @@ def take_backup_to_dropbox(retry_count=0, upload_db_backup=True):
|
|||
did_not_upload, error_log = [], []
|
||||
try:
|
||||
if cint(frappe.db.get_value("Dropbox Settings", None, "enabled")):
|
||||
validate_file_size()
|
||||
|
||||
did_not_upload, error_log = backup_to_dropbox(upload_db_backup)
|
||||
if did_not_upload: raise Exception
|
||||
|
||||
send_email(True, "Dropbox")
|
||||
send_email(True, "Dropbox", "Dropbox Settings", "send_notifications_to")
|
||||
except JobTimeoutException:
|
||||
if retry_count < 2:
|
||||
args = {
|
||||
|
|
@ -66,34 +71,8 @@ def take_backup_to_dropbox(retry_count=0, upload_db_backup=True):
|
|||
else:
|
||||
file_and_error = [" - ".join(f) for f in zip(did_not_upload, error_log)]
|
||||
error_message = ("\n".join(file_and_error) + "\n" + frappe.get_traceback())
|
||||
frappe.errprint(error_message)
|
||||
send_email(False, "Dropbox", error_message)
|
||||
|
||||
def send_email(success, service_name, error_status=None):
|
||||
if success:
|
||||
if frappe.db.get_value("Dropbox Settings", None, "send_email_for_successful_backup") == '0':
|
||||
return
|
||||
|
||||
subject = "Backup Upload Successful"
|
||||
message ="""<h3>Backup Uploaded Successfully</h3><p>Hi there, this is just to inform you
|
||||
that your backup was successfully uploaded to your %s account. So relax!</p>
|
||||
""" % service_name
|
||||
|
||||
else:
|
||||
subject = "[Warning] Backup Upload Failed"
|
||||
message ="""<h3>Backup Upload Failed</h3><p>Oops, your automated backup to %s
|
||||
failed.</p>
|
||||
<p>Error message: <br>
|
||||
<pre><code>%s</code></pre>
|
||||
</p>
|
||||
<p>Please contact your system manager for more information.</p>
|
||||
""" % (service_name, error_status)
|
||||
|
||||
if not frappe.db:
|
||||
frappe.connect()
|
||||
|
||||
recipients = split_emails(frappe.db.get_value("Dropbox Settings", None, "send_notifications_to"))
|
||||
frappe.sendmail(recipients=recipients, subject=subject, message=message)
|
||||
send_email(False, "Dropbox", "Dropbox Settings", "send_notifications_to", error_message)
|
||||
|
||||
def backup_to_dropbox(upload_db_backup=True):
|
||||
if not frappe.db:
|
||||
|
|
@ -114,8 +93,12 @@ def backup_to_dropbox(upload_db_backup=True):
|
|||
dropbox_client = dropbox.Dropbox(dropbox_settings['access_token'])
|
||||
|
||||
if upload_db_backup:
|
||||
backup = new_backup(ignore_files=True)
|
||||
filename = os.path.join(get_backups_path(), os.path.basename(backup.backup_path_db))
|
||||
if frappe.flags.create_new_backup:
|
||||
backup = new_backup(ignore_files=True)
|
||||
filename = os.path.join(get_backups_path(), os.path.basename(backup.backup_path_db))
|
||||
else:
|
||||
filename = get_latest_backup_file()
|
||||
|
||||
upload_file_to_dropbox(filename, "/database", dropbox_client)
|
||||
|
||||
# delete older databases
|
||||
|
|
|
|||
|
|
@ -0,0 +1,10 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
# Copyright (c) 2019, Frappe Technologies and Contributors
|
||||
# See license.txt
|
||||
from __future__ import unicode_literals
|
||||
|
||||
# import frappe
|
||||
import unittest
|
||||
|
||||
class TestDropboxSettings(unittest.TestCase):
|
||||
pass
|
||||
|
|
@ -19,6 +19,7 @@ from apiclient.http import MediaFileUpload
|
|||
from frappe.utils import get_backups_path, get_bench_path
|
||||
from frappe.utils.backups import new_backup
|
||||
from frappe.integrations.doctype.google_settings.google_settings import get_auth_url
|
||||
from frappe.integrations.offsite_backup_utils import get_latest_backup_file, send_email, validate_file_size
|
||||
|
||||
SCOPES = "https://www.googleapis.com/auth/drive"
|
||||
|
||||
|
|
@ -183,13 +184,16 @@ def upload_system_backup_to_google_drive():
|
|||
check_for_folder_in_google_drive()
|
||||
account.load_from_db()
|
||||
|
||||
progress(1, "Backing up Data.")
|
||||
backup = new_backup()
|
||||
|
||||
fileurl_backup = os.path.basename(backup.backup_path_db)
|
||||
fileurl_public_files = os.path.basename(backup.backup_path_files)
|
||||
fileurl_private_files = os.path.basename(backup.backup_path_private_files)
|
||||
validate_file_size()
|
||||
|
||||
if frappe.flags.create_new_backup:
|
||||
set_progress(1, "Backing up Data.")
|
||||
backup = new_backup()
|
||||
fileurl_backup = os.path.basename(backup.backup_path_db)
|
||||
fileurl_public_files = os.path.basename(backup.backup_path_files)
|
||||
fileurl_private_files = os.path.basename(backup.backup_path_private_files)
|
||||
else:
|
||||
fileurl_backup, fileurl_public_files, fileurl_private_files = get_latest_backup_file(with_files=True)
|
||||
|
||||
for fileurl in [fileurl_backup, fileurl_public_files, fileurl_private_files]:
|
||||
file_metadata = {
|
||||
|
|
@ -203,15 +207,14 @@ def upload_system_backup_to_google_drive():
|
|||
frappe.throw(_("Google Drive - Could not locate locate - {0}").format(e))
|
||||
|
||||
try:
|
||||
progress(2, "Uploading backup to Google Drive.")
|
||||
set_progress(2, "Uploading backup to Google Drive.")
|
||||
google_drive.files().create(body=file_metadata, media_body=media, fields="id").execute()
|
||||
except HttpError as e:
|
||||
send_email(success=False, error=e)
|
||||
frappe.msgprint(_("Google Drive - Could not upload backup - Error {0}").format(e))
|
||||
send_email(False, "Google Drive", "Google Drive", "email", error_status=e)
|
||||
|
||||
progress(3, "Uploading successful.")
|
||||
set_progress(3, "Uploading successful.")
|
||||
frappe.db.set_value("Google Drive", None, "last_backup_on", frappe.utils.now_datetime())
|
||||
send_email(success=True)
|
||||
send_email(True, "Google Drive", "Google Drive", "email")
|
||||
return _("Google Drive Backup Successful.")
|
||||
|
||||
def daily_backup():
|
||||
|
|
@ -226,30 +229,5 @@ def get_absolute_path(filename):
|
|||
file_path = os.path.join(get_backups_path()[2:], filename)
|
||||
return "{0}/sites/{1}".format(get_bench_path(), file_path)
|
||||
|
||||
def progress(progress, message):
|
||||
def set_progress(progress, message):
|
||||
frappe.publish_realtime("upload_to_google_drive", dict(progress=progress, total=3, message=message), user=frappe.session.user)
|
||||
|
||||
def send_email(success, error=None):
|
||||
if success:
|
||||
if not frappe.db.get_single_value("Google Drive", "send_email_for_successful_backup"):
|
||||
return
|
||||
|
||||
subject = "Backup Upload Successful"
|
||||
message = """<h3>Backup Uploaded Successfully</h3><p>Hi there, this is just to inform you
|
||||
that your backup was successfully uploaded to Google Drive.</p>
|
||||
"""
|
||||
else:
|
||||
subject = "[Warning] Backup Upload Failed"
|
||||
message = """<h3>Backup Upload Failed</h3><p>Oops, your automated backup to Google Drive
|
||||
failed.</p>
|
||||
<p>Error message: <br>
|
||||
<pre><code>{0}</code></pre>
|
||||
</p>
|
||||
<p>Please contact your system manager for more information.</p>
|
||||
""".format(error)
|
||||
|
||||
frappe.sendmail(
|
||||
recipients=frappe.db.get_single_value("Google Drive", "email"),
|
||||
subject=subject,
|
||||
message=message
|
||||
)
|
||||
|
|
@ -1,397 +1,110 @@
|
|||
{
|
||||
"allow_copy": 0,
|
||||
"allow_events_in_timeline": 0,
|
||||
"allow_guest_to_view": 0,
|
||||
"allow_import": 0,
|
||||
"allow_rename": 0,
|
||||
"beta": 0,
|
||||
"creation": "2017-09-04 20:57:20.129205",
|
||||
"custom": 0,
|
||||
"docstatus": 0,
|
||||
"doctype": "DocType",
|
||||
"document_type": "",
|
||||
"editable_grid": 1,
|
||||
"engine": "InnoDB",
|
||||
"creation": "2017-09-04 20:57:20.129205",
|
||||
"doctype": "DocType",
|
||||
"editable_grid": 1,
|
||||
"engine": "InnoDB",
|
||||
"field_order": [
|
||||
"enabled",
|
||||
"notify_email",
|
||||
"send_email_for_successful_backup",
|
||||
"frequency",
|
||||
"access_key_id",
|
||||
"secret_access_key",
|
||||
"region",
|
||||
"endpoint_url",
|
||||
"bucket",
|
||||
"backup_limit"
|
||||
],
|
||||
"fields": [
|
||||
{
|
||||
"allow_bulk_edit": 0,
|
||||
"allow_in_quick_entry": 0,
|
||||
"allow_on_submit": 0,
|
||||
"bold": 0,
|
||||
"collapsible": 0,
|
||||
"columns": 0,
|
||||
"fetch_if_empty": 0,
|
||||
"fieldname": "enabled",
|
||||
"fieldtype": "Check",
|
||||
"hidden": 0,
|
||||
"ignore_user_permissions": 0,
|
||||
"ignore_xss_filter": 0,
|
||||
"in_filter": 0,
|
||||
"in_global_search": 0,
|
||||
"in_list_view": 0,
|
||||
"in_standard_filter": 0,
|
||||
"label": "Enable Automatic Backup",
|
||||
"length": 0,
|
||||
"no_copy": 0,
|
||||
"permlevel": 0,
|
||||
"precision": "",
|
||||
"print_hide": 0,
|
||||
"print_hide_if_no_value": 0,
|
||||
"read_only": 0,
|
||||
"remember_last_selected_value": 0,
|
||||
"report_hide": 0,
|
||||
"reqd": 0,
|
||||
"search_index": 0,
|
||||
"set_only_once": 0,
|
||||
"translatable": 0,
|
||||
"unique": 0
|
||||
},
|
||||
"default": "0",
|
||||
"fieldname": "enabled",
|
||||
"fieldtype": "Check",
|
||||
"label": "Enable Automatic Backup"
|
||||
},
|
||||
{
|
||||
"allow_bulk_edit": 0,
|
||||
"allow_in_quick_entry": 0,
|
||||
"allow_on_submit": 0,
|
||||
"bold": 0,
|
||||
"collapsible": 0,
|
||||
"columns": 0,
|
||||
"fetch_if_empty": 0,
|
||||
"fieldname": "notify_email",
|
||||
"fieldtype": "Data",
|
||||
"hidden": 0,
|
||||
"ignore_user_permissions": 0,
|
||||
"ignore_xss_filter": 0,
|
||||
"in_filter": 0,
|
||||
"in_global_search": 0,
|
||||
"in_list_view": 1,
|
||||
"in_standard_filter": 0,
|
||||
"label": "Send Notifications To",
|
||||
"length": 0,
|
||||
"no_copy": 0,
|
||||
"permlevel": 0,
|
||||
"precision": "",
|
||||
"print_hide": 0,
|
||||
"print_hide_if_no_value": 0,
|
||||
"read_only": 0,
|
||||
"remember_last_selected_value": 0,
|
||||
"report_hide": 0,
|
||||
"reqd": 1,
|
||||
"search_index": 0,
|
||||
"set_only_once": 0,
|
||||
"translatable": 0,
|
||||
"unique": 0
|
||||
},
|
||||
"fieldname": "notify_email",
|
||||
"fieldtype": "Data",
|
||||
"in_list_view": 1,
|
||||
"label": "Send Notifications To",
|
||||
"reqd": 1
|
||||
},
|
||||
{
|
||||
"allow_bulk_edit": 0,
|
||||
"allow_in_quick_entry": 0,
|
||||
"allow_on_submit": 0,
|
||||
"bold": 0,
|
||||
"collapsible": 0,
|
||||
"columns": 0,
|
||||
"default": "1",
|
||||
"description": "Note: By default emails for failed backups are sent.",
|
||||
"fetch_if_empty": 0,
|
||||
"fieldname": "send_email_for_successful_backup",
|
||||
"fieldtype": "Check",
|
||||
"hidden": 0,
|
||||
"ignore_user_permissions": 0,
|
||||
"ignore_xss_filter": 0,
|
||||
"in_filter": 0,
|
||||
"in_global_search": 0,
|
||||
"in_list_view": 0,
|
||||
"in_standard_filter": 0,
|
||||
"label": "Send Email for Successful Backup",
|
||||
"length": 0,
|
||||
"no_copy": 0,
|
||||
"permlevel": 0,
|
||||
"precision": "",
|
||||
"print_hide": 0,
|
||||
"print_hide_if_no_value": 0,
|
||||
"read_only": 0,
|
||||
"remember_last_selected_value": 0,
|
||||
"report_hide": 0,
|
||||
"reqd": 0,
|
||||
"search_index": 0,
|
||||
"set_only_once": 0,
|
||||
"translatable": 0,
|
||||
"unique": 0
|
||||
},
|
||||
"default": "1",
|
||||
"description": "Note: By default emails for failed backups are sent.",
|
||||
"fieldname": "send_email_for_successful_backup",
|
||||
"fieldtype": "Check",
|
||||
"label": "Send Email for Successful Backup"
|
||||
},
|
||||
{
|
||||
"allow_bulk_edit": 0,
|
||||
"allow_in_quick_entry": 0,
|
||||
"allow_on_submit": 0,
|
||||
"bold": 0,
|
||||
"collapsible": 0,
|
||||
"columns": 0,
|
||||
"fetch_if_empty": 0,
|
||||
"fieldname": "frequency",
|
||||
"fieldtype": "Select",
|
||||
"hidden": 0,
|
||||
"ignore_user_permissions": 0,
|
||||
"ignore_xss_filter": 0,
|
||||
"in_filter": 0,
|
||||
"in_global_search": 0,
|
||||
"in_list_view": 1,
|
||||
"in_standard_filter": 0,
|
||||
"label": "Backup Frequency",
|
||||
"length": 0,
|
||||
"no_copy": 0,
|
||||
"options": "Daily\nWeekly\nMonthly\nNone",
|
||||
"permlevel": 0,
|
||||
"precision": "",
|
||||
"print_hide": 0,
|
||||
"print_hide_if_no_value": 0,
|
||||
"read_only": 0,
|
||||
"remember_last_selected_value": 0,
|
||||
"report_hide": 0,
|
||||
"reqd": 1,
|
||||
"search_index": 0,
|
||||
"set_only_once": 0,
|
||||
"translatable": 0,
|
||||
"unique": 0
|
||||
},
|
||||
"fieldname": "frequency",
|
||||
"fieldtype": "Select",
|
||||
"in_list_view": 1,
|
||||
"label": "Backup Frequency",
|
||||
"options": "Daily\nWeekly\nMonthly\nNone",
|
||||
"reqd": 1
|
||||
},
|
||||
{
|
||||
"allow_bulk_edit": 0,
|
||||
"allow_in_quick_entry": 0,
|
||||
"allow_on_submit": 0,
|
||||
"bold": 0,
|
||||
"collapsible": 0,
|
||||
"columns": 0,
|
||||
"fetch_if_empty": 0,
|
||||
"fieldname": "access_key_id",
|
||||
"fieldtype": "Data",
|
||||
"hidden": 0,
|
||||
"ignore_user_permissions": 0,
|
||||
"ignore_xss_filter": 0,
|
||||
"in_filter": 0,
|
||||
"in_global_search": 0,
|
||||
"in_list_view": 1,
|
||||
"in_standard_filter": 0,
|
||||
"label": "Access Key ID",
|
||||
"length": 0,
|
||||
"no_copy": 0,
|
||||
"permlevel": 0,
|
||||
"precision": "",
|
||||
"print_hide": 0,
|
||||
"print_hide_if_no_value": 0,
|
||||
"read_only": 0,
|
||||
"remember_last_selected_value": 0,
|
||||
"report_hide": 0,
|
||||
"reqd": 1,
|
||||
"search_index": 0,
|
||||
"set_only_once": 0,
|
||||
"translatable": 0,
|
||||
"unique": 0
|
||||
},
|
||||
"fieldname": "access_key_id",
|
||||
"fieldtype": "Data",
|
||||
"in_list_view": 1,
|
||||
"label": "Access Key ID",
|
||||
"reqd": 1
|
||||
},
|
||||
{
|
||||
"allow_bulk_edit": 0,
|
||||
"allow_in_quick_entry": 0,
|
||||
"allow_on_submit": 0,
|
||||
"bold": 0,
|
||||
"collapsible": 0,
|
||||
"columns": 0,
|
||||
"fetch_if_empty": 0,
|
||||
"fieldname": "secret_access_key",
|
||||
"fieldtype": "Password",
|
||||
"hidden": 0,
|
||||
"ignore_user_permissions": 0,
|
||||
"ignore_xss_filter": 0,
|
||||
"in_filter": 0,
|
||||
"in_global_search": 0,
|
||||
"in_list_view": 1,
|
||||
"in_standard_filter": 0,
|
||||
"label": "Secret Access Key",
|
||||
"length": 0,
|
||||
"no_copy": 0,
|
||||
"permlevel": 0,
|
||||
"precision": "",
|
||||
"print_hide": 0,
|
||||
"print_hide_if_no_value": 0,
|
||||
"read_only": 0,
|
||||
"remember_last_selected_value": 0,
|
||||
"report_hide": 0,
|
||||
"reqd": 1,
|
||||
"search_index": 0,
|
||||
"set_only_once": 0,
|
||||
"translatable": 0,
|
||||
"unique": 0
|
||||
},
|
||||
"fieldname": "secret_access_key",
|
||||
"fieldtype": "Password",
|
||||
"in_list_view": 1,
|
||||
"label": "Secret Access Key",
|
||||
"reqd": 1
|
||||
},
|
||||
{
|
||||
"allow_bulk_edit": 0,
|
||||
"allow_in_quick_entry": 0,
|
||||
"allow_on_submit": 0,
|
||||
"bold": 0,
|
||||
"collapsible": 0,
|
||||
"columns": 0,
|
||||
"default": "us-east-1",
|
||||
"description": "See https://docs.aws.amazon.com/de_de/general/latest/gr/rande.html#s3_region for details.",
|
||||
"fetch_if_empty": 0,
|
||||
"fieldname": "region",
|
||||
"fieldtype": "Select",
|
||||
"hidden": 0,
|
||||
"ignore_user_permissions": 0,
|
||||
"ignore_xss_filter": 0,
|
||||
"in_filter": 0,
|
||||
"in_global_search": 0,
|
||||
"in_list_view": 0,
|
||||
"in_standard_filter": 0,
|
||||
"label": "Region",
|
||||
"length": 0,
|
||||
"no_copy": 0,
|
||||
"options": "us-east-1\nus-east-2\nus-west-1\nus-west-2\nap-south-1\nap-southeast-1\nap-southeast-2\nap-northeast-1\nap-northeast-2\nap-northeast-3\nca-central-1\ncn-north-1\ncn-northwest-1\neu-central-1\neu-west-1\neu-west-2\neu-west-3\neu-north-1\nsa-east-1",
|
||||
"permlevel": 0,
|
||||
"precision": "",
|
||||
"print_hide": 0,
|
||||
"print_hide_if_no_value": 0,
|
||||
"read_only": 0,
|
||||
"remember_last_selected_value": 0,
|
||||
"report_hide": 0,
|
||||
"reqd": 0,
|
||||
"search_index": 0,
|
||||
"set_only_once": 0,
|
||||
"translatable": 0,
|
||||
"unique": 0
|
||||
},
|
||||
"default": "us-east-1",
|
||||
"description": "See https://docs.aws.amazon.com/de_de/general/latest/gr/rande.html#s3_region for details.",
|
||||
"fieldname": "region",
|
||||
"fieldtype": "Select",
|
||||
"label": "Region",
|
||||
"options": "us-east-1\nus-east-2\nus-west-1\nus-west-2\nap-south-1\nap-southeast-1\nap-southeast-2\nap-northeast-1\nap-northeast-2\nap-northeast-3\nca-central-1\ncn-north-1\ncn-northwest-1\neu-central-1\neu-west-1\neu-west-2\neu-west-3\neu-north-1\nsa-east-1"
|
||||
},
|
||||
{
|
||||
"allow_bulk_edit": 0,
|
||||
"allow_in_quick_entry": 0,
|
||||
"allow_on_submit": 0,
|
||||
"bold": 0,
|
||||
"collapsible": 0,
|
||||
"columns": 0,
|
||||
"fetch_if_empty": 0,
|
||||
"fieldname": "endpoint_url",
|
||||
"fieldtype": "Data",
|
||||
"hidden": 0,
|
||||
"ignore_user_permissions": 0,
|
||||
"ignore_xss_filter": 0,
|
||||
"in_filter": 0,
|
||||
"in_global_search": 0,
|
||||
"in_list_view": 0,
|
||||
"in_standard_filter": 0,
|
||||
"label": "Endpoint URL",
|
||||
"length": 0,
|
||||
"no_copy": 0,
|
||||
"permlevel": 0,
|
||||
"precision": "",
|
||||
"print_hide": 0,
|
||||
"print_hide_if_no_value": 0,
|
||||
"read_only": 0,
|
||||
"remember_last_selected_value": 0,
|
||||
"report_hide": 0,
|
||||
"reqd": 0,
|
||||
"search_index": 0,
|
||||
"set_only_once": 0,
|
||||
"translatable": 0,
|
||||
"unique": 0
|
||||
},
|
||||
"fieldname": "endpoint_url",
|
||||
"fieldtype": "Data",
|
||||
"label": "Endpoint URL"
|
||||
},
|
||||
{
|
||||
"allow_bulk_edit": 0,
|
||||
"allow_in_quick_entry": 0,
|
||||
"allow_on_submit": 0,
|
||||
"bold": 0,
|
||||
"collapsible": 0,
|
||||
"columns": 0,
|
||||
"fetch_if_empty": 0,
|
||||
"fieldname": "bucket",
|
||||
"fieldtype": "Data",
|
||||
"hidden": 0,
|
||||
"ignore_user_permissions": 0,
|
||||
"ignore_xss_filter": 0,
|
||||
"in_filter": 0,
|
||||
"in_global_search": 0,
|
||||
"in_list_view": 0,
|
||||
"in_standard_filter": 0,
|
||||
"label": "Bucket",
|
||||
"length": 0,
|
||||
"no_copy": 0,
|
||||
"permlevel": 0,
|
||||
"precision": "",
|
||||
"print_hide": 0,
|
||||
"print_hide_if_no_value": 0,
|
||||
"read_only": 0,
|
||||
"remember_last_selected_value": 0,
|
||||
"report_hide": 0,
|
||||
"reqd": 1,
|
||||
"search_index": 0,
|
||||
"set_only_once": 0,
|
||||
"translatable": 0,
|
||||
"unique": 0
|
||||
},
|
||||
"fieldname": "bucket",
|
||||
"fieldtype": "Data",
|
||||
"label": "Bucket",
|
||||
"reqd": 1
|
||||
},
|
||||
{
|
||||
"allow_bulk_edit": 0,
|
||||
"allow_in_quick_entry": 0,
|
||||
"allow_on_submit": 0,
|
||||
"bold": 0,
|
||||
"collapsible": 0,
|
||||
"columns": 0,
|
||||
"fetch_if_empty": 0,
|
||||
"fieldname": "backup_limit",
|
||||
"fieldtype": "Int",
|
||||
"hidden": 0,
|
||||
"ignore_user_permissions": 0,
|
||||
"ignore_xss_filter": 0,
|
||||
"in_filter": 0,
|
||||
"in_global_search": 0,
|
||||
"in_list_view": 0,
|
||||
"in_standard_filter": 0,
|
||||
"label": "Backup Limit",
|
||||
"length": 0,
|
||||
"no_copy": 0,
|
||||
"permlevel": 0,
|
||||
"precision": "",
|
||||
"print_hide": 0,
|
||||
"print_hide_if_no_value": 0,
|
||||
"read_only": 0,
|
||||
"remember_last_selected_value": 0,
|
||||
"report_hide": 0,
|
||||
"reqd": 1,
|
||||
"search_index": 0,
|
||||
"set_only_once": 0,
|
||||
"translatable": 0,
|
||||
"unique": 0
|
||||
"fieldname": "backup_limit",
|
||||
"fieldtype": "Int",
|
||||
"label": "Backup Limit",
|
||||
"reqd": 1
|
||||
}
|
||||
],
|
||||
"has_web_view": 0,
|
||||
"hide_toolbar": 1,
|
||||
"idx": 0,
|
||||
"in_create": 0,
|
||||
"is_submittable": 0,
|
||||
"issingle": 1,
|
||||
"istable": 0,
|
||||
"max_attachments": 0,
|
||||
"menu_index": 0,
|
||||
"modified": "2019-04-10 03:56:55.632017",
|
||||
"modified_by": "Administrator",
|
||||
"module": "Integrations",
|
||||
"name": "S3 Backup Settings",
|
||||
"name_case": "",
|
||||
"owner": "Administrator",
|
||||
],
|
||||
"hide_toolbar": 1,
|
||||
"issingle": 1,
|
||||
"modified": "2019-08-22 16:26:04.774571",
|
||||
"modified_by": "Administrator",
|
||||
"module": "Integrations",
|
||||
"name": "S3 Backup Settings",
|
||||
"owner": "Administrator",
|
||||
"permissions": [
|
||||
{
|
||||
"amend": 0,
|
||||
"cancel": 0,
|
||||
"create": 1,
|
||||
"delete": 1,
|
||||
"email": 1,
|
||||
"export": 0,
|
||||
"if_owner": 0,
|
||||
"import": 0,
|
||||
"permlevel": 0,
|
||||
"print": 1,
|
||||
"read": 1,
|
||||
"report": 0,
|
||||
"role": "System Manager",
|
||||
"set_user_permissions": 0,
|
||||
"share": 1,
|
||||
"submit": 0,
|
||||
"create": 1,
|
||||
"delete": 1,
|
||||
"email": 1,
|
||||
"print": 1,
|
||||
"read": 1,
|
||||
"role": "System Manager",
|
||||
"share": 1,
|
||||
"write": 1
|
||||
}
|
||||
],
|
||||
"quick_entry": 1,
|
||||
"read_only": 0,
|
||||
"show_name_in_global_search": 0,
|
||||
"sort_field": "modified",
|
||||
"sort_order": "DESC",
|
||||
"track_changes": 1,
|
||||
"track_seen": 0,
|
||||
"track_views": 0
|
||||
],
|
||||
"quick_entry": 1,
|
||||
"sort_field": "modified",
|
||||
"sort_order": "DESC",
|
||||
"track_changes": 1
|
||||
}
|
||||
|
|
@ -8,12 +8,14 @@ import os.path
|
|||
import frappe
|
||||
import boto3
|
||||
from frappe import _
|
||||
from frappe.integrations.offsite_backup_utils import get_latest_backup_file, send_email, validate_file_size
|
||||
from frappe.model.document import Document
|
||||
from frappe.utils import cint, split_emails
|
||||
from frappe.utils import cint
|
||||
from frappe.utils.background_jobs import enqueue
|
||||
from rq.timeouts import JobTimeoutException
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
|
||||
class S3BackupSettings(Document):
|
||||
|
||||
def validate(self):
|
||||
|
|
@ -49,7 +51,7 @@ class S3BackupSettings(Document):
|
|||
|
||||
@frappe.whitelist()
|
||||
def take_backup():
|
||||
"Enqueue longjob for taking backup to s3"
|
||||
"""Enqueue longjob for taking backup to s3"""
|
||||
enqueue("frappe.integrations.doctype.s3_backup_settings.s3_backup_settings.take_backups_s3", queue='long', timeout=1500)
|
||||
frappe.msgprint(_("Queued for backup. It may take a few minutes to an hour."))
|
||||
|
||||
|
|
@ -65,22 +67,21 @@ def take_backups_weekly():
|
|||
def take_backups_monthly():
|
||||
take_backups_if("Monthly")
|
||||
|
||||
|
||||
def take_backups_if(freq):
|
||||
if cint(frappe.db.get_value("S3 Backup Settings", None, "enabled")):
|
||||
if frappe.db.get_value("S3 Backup Settings", None, "frequency") == freq:
|
||||
take_backups_s3()
|
||||
|
||||
|
||||
@frappe.whitelist()
|
||||
def take_backups_s3(retry_count=0):
|
||||
try:
|
||||
validate_file_size()
|
||||
backup_to_s3()
|
||||
send_email(True, "S3 Backup Settings")
|
||||
send_email(True, "Amazon S3", "S3 Backup Settings", "notify_email")
|
||||
except JobTimeoutException:
|
||||
if retry_count < 2:
|
||||
args = {
|
||||
"retry_count" :retry_count + 1
|
||||
"retry_count": retry_count + 1
|
||||
}
|
||||
enqueue("frappe.integrations.doctype.s3_backup_settings.s3_backup_settings.take_backups_s3",
|
||||
queue='long', timeout=1500, **args)
|
||||
|
|
@ -89,31 +90,10 @@ def take_backups_s3(retry_count=0):
|
|||
except Exception:
|
||||
notify()
|
||||
|
||||
|
||||
def notify():
|
||||
error_message = frappe.get_traceback()
|
||||
frappe.errprint(error_message)
|
||||
send_email(False, "S3 Backup Settings", error_message)
|
||||
|
||||
def send_email(success, service_name, error_status=None):
|
||||
if success:
|
||||
if frappe.db.get_value("S3 Backup Settings", None, "send_email_for_successful_backup") == '0':
|
||||
return
|
||||
|
||||
subject = "Backup Upload Successful"
|
||||
message = """<h3>Backup Uploaded Successfully! </h3><p>Hi there, this is just to inform you
|
||||
that your backup was successfully uploaded to your Amazon S3 bucket. So relax!</p> """
|
||||
|
||||
else:
|
||||
subject = "[Warning] Backup Upload Failed"
|
||||
message = """<h3>Backup Upload Failed! </h3><p>Oops, your automated backup to Amazon S3 failed.
|
||||
</p> <p>Error message: %s</p> <p>Please contact your system manager
|
||||
for more information.</p>""" % error_status
|
||||
|
||||
if not frappe.db:
|
||||
frappe.connect()
|
||||
|
||||
recipients = split_emails(frappe.db.get_value("S3 Backup Settings", None, "notify_email"))
|
||||
frappe.sendmail(recipients=recipients, subject=subject, message=message)
|
||||
send_email(False, 'Amazon S3', "S3 Backup Settings", "notify_email", error_message)
|
||||
|
||||
|
||||
def backup_to_s3():
|
||||
|
|
@ -130,11 +110,15 @@ def backup_to_s3():
|
|||
endpoint_url=doc.endpoint_url or 'https://s3.amazonaws.com'
|
||||
)
|
||||
|
||||
backup = new_backup(ignore_files=False, backup_path_db=None,
|
||||
if frappe.flags.create_new_backup:
|
||||
backup = new_backup(ignore_files=False, backup_path_db=None,
|
||||
backup_path_files=None, backup_path_private_files=None, force=True)
|
||||
db_filename = os.path.join(get_backups_path(), os.path.basename(backup.backup_path_db))
|
||||
files_filename = os.path.join(get_backups_path(), os.path.basename(backup.backup_path_files))
|
||||
private_files = os.path.join(get_backups_path(), os.path.basename(backup.backup_path_private_files))
|
||||
db_filename = os.path.join(get_backups_path(), os.path.basename(backup.backup_path_db))
|
||||
files_filename = os.path.join(get_backups_path(), os.path.basename(backup.backup_path_files))
|
||||
private_files = os.path.join(get_backups_path(), os.path.basename(backup.backup_path_private_files))
|
||||
else:
|
||||
db_filename, files_filename, private_files = get_latest_backup_file(with_files=True)
|
||||
|
||||
folder = os.path.basename(db_filename)[:15] + '/'
|
||||
# for adding datetime to folder name
|
||||
|
||||
|
|
@ -143,8 +127,8 @@ def backup_to_s3():
|
|||
upload_file_to_s3(files_filename, folder, conn, bucket)
|
||||
delete_old_backups(doc.backup_limit, bucket)
|
||||
|
||||
def upload_file_to_s3(filename, folder, conn, bucket):
|
||||
|
||||
def upload_file_to_s3(filename, folder, conn, bucket):
|
||||
destpath = os.path.join(folder, os.path.basename(filename))
|
||||
try:
|
||||
print("Uploading file:", filename)
|
||||
|
|
@ -156,7 +140,7 @@ def upload_file_to_s3(filename, folder, conn, bucket):
|
|||
|
||||
|
||||
def delete_old_backups(limit, bucket):
|
||||
all_backups = list()
|
||||
all_backups = []
|
||||
doc = frappe.get_single("S3 Backup Settings")
|
||||
backup_limit = int(limit)
|
||||
|
||||
|
|
|
|||
83
frappe/integrations/offsite_backup_utils.py
Normal file
83
frappe/integrations/offsite_backup_utils.py
Normal file
|
|
@ -0,0 +1,83 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
# Copyright (c) 2019, Frappe Technologies and contributors
|
||||
# For license information, please see license.txt
|
||||
|
||||
from __future__ import unicode_literals
|
||||
import frappe
|
||||
import glob
|
||||
import os
|
||||
from frappe.utils import split_emails, get_backups_path
|
||||
|
||||
|
||||
def send_email(success, service_name, doctype, email_field, error_status=None):
|
||||
recipients = get_recipients(service_name, email_field)
|
||||
if not recipients:
|
||||
frappe.log_error("No Email Recipient found for {0}".format(service_name),
|
||||
"{0}: Failed to send backup status email".format(service_name))
|
||||
return
|
||||
|
||||
if success:
|
||||
if not frappe.db.get_value(doctype, None, "send_email_for_successful_backup"):
|
||||
return
|
||||
|
||||
subject = "Backup Upload Successful"
|
||||
message = """
|
||||
<h3>Backup Uploaded Successfully!</h3>
|
||||
<p>Hi there, this is just to inform you that your backup was successfully uploaded to your {0} bucket. So relax!</p>""".format(service_name)
|
||||
|
||||
else:
|
||||
subject = "[Warning] Backup Upload Failed"
|
||||
message = """
|
||||
<h3>Backup Upload Failed!</h3>
|
||||
<p>Oops, your automated backup to {0} failed.</p>
|
||||
<p>Error message: {1}</p>
|
||||
<p>Please contact your system manager for more information.</p>""".format(service_name, error_status)
|
||||
|
||||
frappe.sendmail(recipients=recipients, subject=subject, message=message)
|
||||
|
||||
|
||||
def get_recipients(service_name, email_field):
|
||||
if not frappe.db:
|
||||
frappe.connect()
|
||||
|
||||
return split_emails(frappe.db.get_value(service_name, None, email_field))
|
||||
|
||||
|
||||
def get_latest_backup_file(with_files=False):
|
||||
|
||||
def get_latest(file_ext):
|
||||
file_list = glob.glob(os.path.join(get_backups_path(), file_ext))
|
||||
return max(file_list, key=os.path.getctime)
|
||||
|
||||
latest_file = get_latest('*.sql.gz')
|
||||
|
||||
if with_files:
|
||||
latest_public_file_bak = get_latest('*-files.tar')
|
||||
latest_private_file_bak = get_latest('*-private-files.tar')
|
||||
return latest_file, latest_public_file_bak, latest_private_file_bak
|
||||
|
||||
return latest_file
|
||||
|
||||
|
||||
def get_file_size(file_path, unit):
|
||||
if not unit:
|
||||
unit = 'MB'
|
||||
|
||||
file_size = os.path.getsize(file_path)
|
||||
|
||||
memory_size_unit_mapper = {'KB': 1, 'MB': 2, 'GB': 3, 'TB': 4}
|
||||
i = 0
|
||||
while i < memory_size_unit_mapper[unit]:
|
||||
file_size = file_size / 1000.0
|
||||
i += 1
|
||||
|
||||
return file_size
|
||||
|
||||
|
||||
def validate_file_size():
|
||||
frappe.flags.create_new_backup = True
|
||||
latest_file = get_latest_backup_file()
|
||||
file_size = get_file_size(latest_file, unit='GB')
|
||||
|
||||
if file_size > 1:
|
||||
frappe.flags.create_new_backup = False
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
# Copyright (c) 2015, Frappe Technologies and contributors
|
||||
# Copyright (c) 2019, Frappe Technologies and contributors
|
||||
# For license information, please see license.txt
|
||||
|
||||
from __future__ import unicode_literals
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue