From 1752e5b0e52f9cd1366306fe2cd1df7eda2753ab Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 7 May 2021 12:14:00 +0530 Subject: [PATCH] feat(cli): Show progress bar for website search index building --- frappe/search/full_text_search.py | 5 +++-- frappe/search/website_search.py | 35 ++++++++++++++++++++----------- frappe/utils/__init__.py | 21 +++++++++++++++---- frappe/website/context.py | 2 +- 4 files changed, 44 insertions(+), 19 deletions(-) diff --git a/frappe/search/full_text_search.py b/frappe/search/full_text_search.py index ecb018dbb4..9dd181323e 100644 --- a/frappe/search/full_text_search.py +++ b/frappe/search/full_text_search.py @@ -1,8 +1,8 @@ # Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import frappe +from frappe.utils import update_progress_bar from whoosh.index import create_in, open_dir, EmptyIndexError from whoosh.fields import TEXT, ID, Schema @@ -95,9 +95,10 @@ class FullTextSearch: ix = self.create_index() writer = ix.writer() - for document in self.documents: + for i, document in enumerate(self.documents): if document: writer.add_document(**document) + update_progress_bar("Building Index", i, len(self.documents)) writer.commit(optimize=True) diff --git a/frappe/search/website_search.py b/frappe/search/website_search.py index 87ef4b08ad..3b2c51ef41 100644 --- a/frappe/search/website_search.py +++ b/frappe/search/website_search.py @@ -1,15 +1,16 @@ # Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals -import frappe -from bs4 import BeautifulSoup -from whoosh.fields import TEXT, ID, Schema -from frappe.search.full_text_search import FullTextSearch -from frappe.website.render import render_page -from frappe.utils import set_request import os +from bs4 import BeautifulSoup +from whoosh.fields import ID, TEXT, Schema + +import frappe +from frappe.search.full_text_search import FullTextSearch +from frappe.utils import set_request, update_progress_bar +from frappe.website.render import render_page + INDEX_NAME = "web_routes" class WebsiteSearch(FullTextSearch): @@ -30,11 +31,21 @@ class WebsiteSearch(FullTextSearch): Returns: self (object): FullTextSearch Instance """ - routes = get_static_pages_from_all_apps() - routes += slugs_with_web_view() - documents = [self.get_document_to_index(route) for route in routes] - return documents + if getattr(self, "_items_to_index", False): + return self._items_to_index + + routes = get_static_pages_from_all_apps() + slugs_with_web_view() + + self._items_to_index = [] + + for i, route in enumerate(routes): + update_progress_bar(f"Retrieving Routes", i, len(routes)) + self._items_to_index += [self.get_document_to_index(route)] + + print() + + return self.get_items_to_index() def get_document_to_index(self, route): """Render a page and parse it using BeautifulSoup @@ -114,4 +125,4 @@ def remove_document_from_index(path): def build_index_for_all_routes(): ws = WebsiteSearch(INDEX_NAME) - return ws.build() \ No newline at end of file + return ws.build() diff --git a/frappe/utils/__init__.py b/frappe/utils/__init__.py index 1da4cd3bae..ccd5283b1a 100644 --- a/frappe/utils/__init__.py +++ b/frappe/utils/__init__.py @@ -225,14 +225,17 @@ def get_gravatar(email): return gravatar_url -def get_traceback(): +def get_traceback() -> str: """ Returns the traceback of the Exception """ exc_type, exc_value, exc_tb = sys.exc_info() + + if not any(exc_type, exc_value, exc_tb): + return "" + trace_list = traceback.format_exception(exc_type, exc_value, exc_tb) - body = "".join(cstr(t) for t in trace_list) - return body + return "".join(cstr(t) for t in trace_list) def log(event, details): frappe.logger().info(details) @@ -439,6 +442,16 @@ def call_hook_method(hook, *args, **kwargs): return out +def is_cli() -> bool: + """Returns True if current instance is being run via a terminal + """ + invoked_from_terminal = False + try: + invoked_from_terminal = bool(os.get_terminal_size()) + except Exception: + invoked_from_terminal = sys.stdin.isatty() + return invoked_from_terminal + def update_progress_bar(txt, i, l): if os.environ.get("CI"): if i == 0: @@ -448,7 +461,7 @@ def update_progress_bar(txt, i, l): sys.stdout.flush() return - if not getattr(frappe.local, 'request', None): + if not getattr(frappe.local, 'request', None) or is_cli(): lt = len(txt) try: col = 40 if os.get_terminal_size().columns > 80 else 20 diff --git a/frappe/website/context.py b/frappe/website/context.py index b39770dcd4..401912d407 100644 --- a/frappe/website/context.py +++ b/frappe/website/context.py @@ -61,7 +61,7 @@ def update_controller_context(context, controller): except (frappe.PermissionError, frappe.PageDoesNotExistError, frappe.Redirect): raise except: - if not frappe.flags.in_migrate: + if not any(frappe.flags.in_migrate, frappe.flags.in_website_search_build): frappe.errprint(frappe.utils.get_traceback()) if hasattr(module, "get_children"):