From aabbfb8df823e59ff7ea565254c08b1526c1a499 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 28 May 2020 18:17:41 +0530 Subject: [PATCH 1/5] feat: switch teams if you are a part of multiple --- .../frappe_providers/frappecloud.py | 107 ++++++------------ frappe/utils/commands.py | 9 ++ 2 files changed, 45 insertions(+), 71 deletions(-) diff --git a/frappe/integrations/frappe_providers/frappecloud.py b/frappe/integrations/frappe_providers/frappecloud.py index 4f33c990f9..3503f9d684 100644 --- a/frappe/integrations/frappe_providers/frappecloud.py +++ b/frappe/integrations/frappe_providers/frappecloud.py @@ -13,7 +13,28 @@ import requests import frappe import frappe.utils.backups from frappe.utils import get_installed_apps_info -from frappe.utils.commands import render_table, add_line_after +from frappe.utils.commands import render_table, add_line_after, add_line_before + + +@add_line_before +def select_team(session): + # get team options + account_details_sc = session.post(account_details_url) + if account_details_sc.ok: + account_details = account_details_sc.json()["message"] + available_teams = account_details["teams"] + + # ask if they want to select, go ahead with if only one exists + if len(available_teams) == 1: + team = available_teams[0] + else: + render_teams_table(available_teams) + idx = click.prompt("Select Team", type=click.IntRange(1, len(available_teams))) - 1 + team = available_teams[idx] + + print("Team '{}' set for current session".format(team)) + + return team def get_new_site_options(): @@ -148,24 +169,6 @@ def filter_apps(app_groups): return selected_group["name"], filtered_apps -@add_line_after -def create_session(): - # take user input from STDIN - username = click.prompt("Username").strip() - password = getpass.unix_getpass() - - auth_credentials = {"usr": username, "pwd": password} - - session = requests.Session() - login_sc = session.post(login_url, auth_credentials) - - if login_sc.ok: - print("Authorization Successful! ✅") - session.headers.update({"X-Press-Team": username}) - return session - else: - print("Authorization Failed with Error Code {}".format(login_sc.status_code)) - @add_line_after def get_subdomain(domain): @@ -208,61 +211,23 @@ def upload_backup(local_site): return files_uploaded -def frappecloud_migrator(local_site, remote_site): - global login_url, upload_url, files_url, options_url, site_exists_url, session - - login_url = "https://{}/api/method/login".format(remote_site) - upload_url = "https://{}/api/method/press.api.site.new".format(remote_site) - files_url = "https://{}/api/method/upload_file".format(remote_site) - options_url = "https://{}/api/method/press.api.site.options_for_new".format(remote_site) - site_exists_url = "https://{}/api/method/press.api.site.exists".format(remote_site) - +@add_line_after +def create_session(): print("Frappe Cloud credentials @ {}".format(remote_site)) - # get credentials + auth user + start session - session = create_session() + # take user input from STDIN + username = click.prompt("Username").strip() + password = getpass.unix_getpass() - if session: - # connect to site db - frappe.init(site=local_site) - frappe.connect() + auth_credentials = {"usr": username, "pwd": password} - # get new site options - site_options = get_new_site_options() - - # set preferences from site options - subdomain = get_subdomain(site_options["domain"]) - plan = choose_plan(site_options["plans"]) - - app_groups = site_options["groups"] - selected_group, filtered_apps = filter_apps(app_groups) - files_uploaded = upload_backup(local_site) - - # push to frappe_cloud - payload = json.dumps({ - "site": { - "apps": filtered_apps, - "files": files_uploaded, - "group": selected_group, - "name": subdomain, - "plan": plan - } - }) - - session.headers.update({"Content-Type": "application/json; charset=utf-8"}) - site_creation_request = session.post(upload_url, payload) - frappe.destroy() - - if site_creation_request.ok: - site_url = site_creation_request.json()["message"] - print("Your site {} is being migrated ✨".format(local_site)) - print("View your site dashboard at {}/dashboard/#/sites/{}".format(remote_site, site_url)) - print("Your site URL: {}".format(site_url)) - else: - print("Request failed with error code {}".format(site_creation_request.status_code)) - reason = html2text(site_creation_request.text) - print(reason) - sys.exit(1) + session = requests.Session() + login_sc = session.post(login_url, auth_credentials) + if login_sc.ok: + print("Authorization Successful! ✅") + team = select_team(session) + session.headers.update({"X-Press-Team": team }) + return session else: - sys.exit(1) + handle_request_failure(message="Authorization Failed with Error Code {}".format(login_sc.status_code), traceback=False) diff --git a/frappe/utils/commands.py b/frappe/utils/commands.py index 99322b50ba..113014c135 100644 --- a/frappe/utils/commands.py +++ b/frappe/utils/commands.py @@ -27,6 +27,15 @@ def add_line_after(function): return empty_line +def add_line_before(function): + """Adds an extra line to STDOUT before the execution of a function this decorates""" + def empty_line(*args, **kwargs): + print() + result = function(*args, **kwargs) + return result + return empty_line + + def log(message, colour=''): """Coloured log outputs to STDOUT""" colours = { From b8138004d9fcc500f9f4590ce5b10f3e1062d3c3 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 28 May 2020 18:18:38 +0530 Subject: [PATCH 2/5] feat: allow restoring to existing FC sites --- .../frappe_providers/frappecloud.py | 225 +++++++++++++++--- 1 file changed, 197 insertions(+), 28 deletions(-) diff --git a/frappe/integrations/frappe_providers/frappecloud.py b/frappe/integrations/frappe_providers/frappecloud.py index 3503f9d684..39712ecc42 100644 --- a/frappe/integrations/frappe_providers/frappecloud.py +++ b/frappe/integrations/frappe_providers/frappecloud.py @@ -16,6 +16,107 @@ from frappe.utils import get_installed_apps_info from frappe.utils.commands import render_table, add_line_after, add_line_before +# TODO: check upgrade compatibility + + +def render_actions_table(): + actions_table = [["#", "Action"]] + actions = [] + + for n, action in enumerate(migrator_actions): + actions_table.append([n+1, action["title"]]) + actions.append(action["fn"]) + + render_table(actions_table) + return actions + + +def render_site_table(sites_info): + sites_table = [["#", "Site Name", "Status"]] + available_sites = [] + + for n, site_data in enumerate(sites_info): + name, status = site_data["name"], site_data["status"] + if status not in ("Inactive", "Suspended"): + sites_table.append([n + 1, name, status]) + available_sites.append(name) + + render_table(sites_table) + return available_sites + + +def render_teams_table(teams): + teams_table = [["#", "Team"]] + + for n, team in enumerate(teams): + teams_table.append([n+1, team]) + + render_table(teams_table) + + +def render_plan_table(plans_list): + plans_table = [["Plan", "CPU Time"]] + visible_headers = ["name", "cpu_time_per_day"] + + for plan in plans_list: + plan, cpu_time = [plan[header] for header in visible_headers] + plans_table.append([plan, "{} hour{}/day".format(cpu_time, "" if cpu_time < 2 else "s")]) + + render_table(plans_table) + + +def render_group_table(app_groups): + # title row + app_groups_table = [["#", "App Group", "Apps"]] + + # all rows + for idx, app_group in enumerate(app_groups): + apps_list = ", ".join(["{}:{}".format(app["scrubbed"], app["branch"]) for app in app_group["apps"]]) + row = [idx + 1, app_group["name"], apps_list] + app_groups_table.append(row) + + render_table(app_groups_table) + + +def handle_request_failure(request=None, message=None, traceback=True, exit_code=1): + message = message or "Request failed with error code {}".format(request.status_code) + response = html2text(request.text) if traceback else "" + + print("{0}{1}".format(message, "\n" + response)) + sys.exit(exit_code) + + +@add_line_after +def select_primary_action(): + actions = render_actions_table() + idx = click.prompt("What do you want to do?", type=click.IntRange(1, len(actions))) - 1 + + return actions[idx] + + +@add_line_after +def select_site(): + get_all_sites_request = session.post(all_site_url, headers={ + "accept": "application/json", + "accept-encoding": "gzip, deflate, br", + "content-type": "application/json; charset=utf-8" + }) + + if get_all_sites_request.ok: + all_sites = get_all_sites_request.json()["message"] + available_sites = render_site_table(all_sites) + + while True: + selected_site = click.prompt("Name of the site you want to restore to", type=str).strip() + if selected_site in available_sites: + return selected_site + else: + print("Site {} does not exist. Try again ❌".format(site)) + else: + print("Couldn't retrive sites list...Try again later") + sys.exit(1) + + @add_line_before def select_team(session): # get team options @@ -67,21 +168,6 @@ def is_subdomain_available(subdomain): return available -def render_plan_table(plans_list): - plans_table = [] - - # title row - visible_headers = ["name", "cpu_time_per_day"] - plans_table.append(["Plan", "CPU Time"]) - - # all rows - for plan in plans_list: - plan, cpu_time = [plan[header] for header in visible_headers] - plans_table.append([plan, "{} hour{}/day".format(cpu_time, "" if cpu_time < 2 else "s")]) - - render_table(plans_table) - - @add_line_after def choose_plan(plans_list): print("{} plans available".format(len(plans_list))) @@ -134,19 +220,6 @@ def check_app_compat(available_group): return is_compat, filtered_apps -def render_group_table(app_groups): - # title row - app_groups_table = [["#", "App Group", "Apps"]] - - # all rows - for idx, app_group in enumerate(app_groups): - apps_list = ", ".join(["{}:{}".format(app["scrubbed"], app["branch"]) for app in app_group["apps"]]) - row = [idx + 1, app_group["name"], apps_list] - app_groups_table.append(row) - - render_table(app_groups_table) - - @add_line_after def filter_apps(app_groups): render_group_table(app_groups) @@ -211,6 +284,68 @@ def upload_backup(local_site): return files_uploaded +def new_site(local_site): + # get new site options + site_options = get_new_site_options() + + # set preferences from site options + subdomain = get_subdomain(site_options["domain"]) + plan = choose_plan(site_options["plans"]) + + app_groups = site_options["groups"] + selected_group, filtered_apps = filter_apps(app_groups) + files_uploaded = upload_backup(local_site) + + # push to frappe_cloud + payload = json.dumps({ + "site": { + "apps": filtered_apps, + "files": files_uploaded, + "group": selected_group, + "name": subdomain, + "plan": plan + } + }) + + session.headers.update({"Content-Type": "application/json; charset=utf-8"}) + site_creation_request = session.post(upload_url, payload) + + if site_creation_request.ok: + site_url = site_creation_request.json()["message"] + print("Your site {} is being migrated ✨".format(local_site)) + print("View your site dashboard at {}/dashboard/#/sites/{}".format(remote_site, site_url)) + print("Your site URL: {}".format(site_url)) + else: + handle_request_failure(site_creation_request) + + +def restore_site(local_site): + # get list of existing sites they can restore + selected_site = select_site() + + # TODO: check if they can restore it + + click.confirm("This is an irreversible action. Are you sure you want to continue?", abort=True) + + # backup site + files_uploaded = upload_backup(local_site) + + # push to frappe_cloud + payload = json.dumps({ + "name": selected_site, + "files": files_uploaded + }) + headers = {"Content-Type": "application/json; charset=utf-8"} + site_restore_request = session.post(restore_site_url, payload, headers=headers) + + if site_restore_request.ok: + print("Your site {0} is being restored on {1} ✨".format(local_site, selected_site)) + print("View your site dashboard at {}/dashboard/#/sites/{}".format(remote_site, selected_site)) + print("Your site URL: {}".format(selected_site)) + else: + handle_request_failure(site_restore_request) + + @add_line_after def create_session(): print("Frappe Cloud credentials @ {}".format(remote_site)) @@ -231,3 +366,37 @@ def create_session(): return session else: handle_request_failure(message="Authorization Failed with Error Code {}".format(login_sc.status_code), traceback=False) + + +def frappecloud_migrator(local_site, frappecloud_site): + global login_url, upload_url, files_url, options_url, site_exists_url, restore_site_url, account_details_url, all_site_url + global session, migrator_actions, remote_site + + remote_site = frappecloud_site + + login_url = "https://{}/api/method/login".format(remote_site) + upload_url = "https://{}/api/method/press.api.site.new".format(remote_site) + files_url = "https://{}/api/method/upload_file".format(remote_site) + options_url = "https://{}/api/method/press.api.site.options_for_new".format(remote_site) + site_exists_url = "https://{}/api/method/press.api.site.exists".format(remote_site) + account_details_url = "https://{}/api/method/press.api.account.get".format(remote_site) + all_site_url = "https://{}/api/method/press.api.site.all".format(remote_site) + restore_site_url = "https://{}/api/method/press.api.site.restore".format(remote_site) + + migrator_actions = [ + { "title": "Create a new site", "fn": new_site }, + { "title": "Restore to an existing site", "fn": restore_site } + ] + + # get credentials + auth user + start session + session = create_session() + + # available actions defined in migrator_actions + primary_action = select_primary_action() + + frappe.init(site=local_site) + frappe.connect() + + primary_action(local_site) + + frappe.destroy() From a1ee529cb744617b24c49cf9aa544a6d88811f8a Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 28 May 2020 18:42:06 +0530 Subject: [PATCH 3/5] fix: show only Active and Broken sites in Sites List --- frappe/integrations/frappe_providers/frappecloud.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/integrations/frappe_providers/frappecloud.py b/frappe/integrations/frappe_providers/frappecloud.py index 39712ecc42..7013dbf1ba 100644 --- a/frappe/integrations/frappe_providers/frappecloud.py +++ b/frappe/integrations/frappe_providers/frappecloud.py @@ -37,7 +37,7 @@ def render_site_table(sites_info): for n, site_data in enumerate(sites_info): name, status = site_data["name"], site_data["status"] - if status not in ("Inactive", "Suspended"): + if status in ("Active", "Broken"): sites_table.append([n + 1, name, status]) available_sites.append(name) @@ -372,7 +372,7 @@ def frappecloud_migrator(local_site, frappecloud_site): global login_url, upload_url, files_url, options_url, site_exists_url, restore_site_url, account_details_url, all_site_url global session, migrator_actions, remote_site - remote_site = frappecloud_site + remote_site = "staging.frappe.cloud" #frappecloud_site login_url = "https://{}/api/method/login".format(remote_site) upload_url = "https://{}/api/method/press.api.site.new".format(remote_site) From a65be3e22a71fdd5d88a87655011df4cced89638 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 29 May 2020 11:07:14 +0530 Subject: [PATCH 4/5] fix: undefined variables --- frappe/integrations/frappe_providers/frappecloud.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/integrations/frappe_providers/frappecloud.py b/frappe/integrations/frappe_providers/frappecloud.py index 7013dbf1ba..d8741e0834 100644 --- a/frappe/integrations/frappe_providers/frappecloud.py +++ b/frappe/integrations/frappe_providers/frappecloud.py @@ -111,7 +111,7 @@ def select_site(): if selected_site in available_sites: return selected_site else: - print("Site {} does not exist. Try again ❌".format(site)) + print("Site {} does not exist. Try again ❌".format(selected_site)) else: print("Couldn't retrive sites list...Try again later") sys.exit(1) @@ -372,7 +372,7 @@ def frappecloud_migrator(local_site, frappecloud_site): global login_url, upload_url, files_url, options_url, site_exists_url, restore_site_url, account_details_url, all_site_url global session, migrator_actions, remote_site - remote_site = "staging.frappe.cloud" #frappecloud_site + remote_site = frappecloud_site login_url = "https://{}/api/method/login".format(remote_site) upload_url = "https://{}/api/method/press.api.site.new".format(remote_site) From 4f99b5879ec9e5aff46294b58474e76a54f65427 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 28 May 2020 20:28:55 +0530 Subject: [PATCH 5/5] fix: set frappecloud_url in conf to override specified url --- frappe/integrations/frappe_providers/__init__.py | 1 - frappe/integrations/frappe_providers/frappecloud.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/frappe/integrations/frappe_providers/__init__.py b/frappe/integrations/frappe_providers/__init__.py index 0b689478d2..887e191e16 100644 --- a/frappe/integrations/frappe_providers/__init__.py +++ b/frappe/integrations/frappe_providers/__init__.py @@ -7,7 +7,6 @@ from frappe.integrations.frappe_providers.frappecloud import frappecloud_migrato def migrate_to(local_site, frappe_provider): if frappe_provider in ("frappe.cloud", "frappecloud.com"): - frappe_provider = "frappecloud.com" return frappecloud_migrator(local_site, frappe_provider) else: print("{} is not supported yet".format(frappe_provider)) diff --git a/frappe/integrations/frappe_providers/frappecloud.py b/frappe/integrations/frappe_providers/frappecloud.py index d8741e0834..291b8af647 100644 --- a/frappe/integrations/frappe_providers/frappecloud.py +++ b/frappe/integrations/frappe_providers/frappecloud.py @@ -372,7 +372,7 @@ def frappecloud_migrator(local_site, frappecloud_site): global login_url, upload_url, files_url, options_url, site_exists_url, restore_site_url, account_details_url, all_site_url global session, migrator_actions, remote_site - remote_site = frappecloud_site + remote_site = frappe.conf.frappecloud_url or "frappecloud.com" login_url = "https://{}/api/method/login".format(remote_site) upload_url = "https://{}/api/method/press.api.site.new".format(remote_site)