From 86897c1808063efccbc8929250c3d2ca203b9c9c Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Tue, 25 May 2021 10:53:35 +0530 Subject: [PATCH] refactor: Remove render.py and move all utility functions to utils.py - Replace or remove all render imports --- frappe/cache_manager.py | 2 +- frappe/commands/utils.py | 8 +- frappe/core/doctype/comment/comment.py | 2 +- frappe/core/doctype/doctype/doctype.py | 9 +-- frappe/installer.py | 4 +- frappe/migrate.py | 4 +- frappe/patches.txt | 2 +- .../templates/includes/comments/comments.py | 2 +- .../about_us_settings/about_us_settings.py | 6 +- .../doctype/blog_category/blog_category.py | 2 +- frappe/website/doctype/blog_post/blog_post.py | 2 +- .../doctype/blog_settings/blog_settings.py | 4 +- .../contact_us_settings.py | 2 +- .../doctype/help_article/help_article.py | 2 +- .../portal_settings/portal_settings.py | 2 +- .../doctype/web_template/web_template.py | 2 +- .../doctype/website_script/website_script.py | 2 +- .../website_settings/website_settings.py | 2 +- .../website_slideshow/website_slideshow.py | 2 +- .../website/page_renderers/document_page.py | 2 +- .../website/page_renderers/redirect_page.py | 2 +- .../website/page_renderers/template_page.py | 6 +- frappe/website/page_renderers/web_page.py | 4 - frappe/website/render.py | 76 ------------------- frappe/website/router.py | 49 +----------- frappe/website/utils.py | 60 +++++++++++++++ frappe/website/website_generator.py | 2 +- .../_test/_test_folder/new.csv/__init__.py | 0 28 files changed, 96 insertions(+), 166 deletions(-) delete mode 100644 frappe/website/render.py create mode 100644 frappe/www/_test/_test_folder/new.csv/__init__.py diff --git a/frappe/cache_manager.py b/frappe/cache_manager.py index 7330c83102..516f13de39 100644 --- a/frappe/cache_manager.py +++ b/frappe/cache_manager.py @@ -55,7 +55,7 @@ def clear_domain_cache(user=None): cache.delete_value(domain_cache_keys) def clear_global_cache(): - from frappe.website.render import clear_cache as clear_website_cache + from frappe.website.utils import clear_cache as clear_website_cache clear_doctype_cache() clear_website_cache() diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py index 4da0f6bb78..a6937a5dc4 100644 --- a/frappe/commands/utils.py +++ b/frappe/commands/utils.py @@ -69,14 +69,14 @@ def watch(apps=None): def clear_cache(context): "Clear cache, doctype cache and defaults" import frappe.sessions - import frappe.website.render + from frappe.website.utils import clear_cache as clear_website_cache from frappe.desk.notifications import clear_notifications for site in context.sites: try: frappe.connect(site) frappe.clear_cache() clear_notifications() - frappe.website.render.clear_cache() + clear_website_cache() finally: frappe.destroy() if not context.sites: @@ -86,12 +86,12 @@ def clear_cache(context): @pass_context def clear_website_cache(context): "Clear website cache" - import frappe.website.render + from frappe.website.utils import clear_cache as clear_website_cache for site in context.sites: try: frappe.init(site=site) frappe.connect() - frappe.website.render.clear_cache() + clear_website_cache() finally: frappe.destroy() if not context.sites: diff --git a/frappe/core/doctype/comment/comment.py b/frappe/core/doctype/comment/comment.py index ad5d60500b..6090947393 100644 --- a/frappe/core/doctype/comment/comment.py +++ b/frappe/core/doctype/comment/comment.py @@ -11,7 +11,7 @@ from frappe.core.doctype.user.user import extract_mentions from frappe.desk.doctype.notification_log.notification_log import enqueue_create_notification,\ get_title, get_title_html from frappe.utils import get_fullname -from frappe.website.render import clear_cache +from frappe.website.utils import clear_cache from frappe.database.schema import add_column from frappe.exceptions import ImplicitCommitError diff --git a/frappe/core/doctype/doctype/doctype.py b/frappe/core/doctype/doctype/doctype.py index 84673f990a..8186aac2b0 100644 --- a/frappe/core/doctype/doctype/doctype.py +++ b/frappe/core/doctype/doctype/doctype.py @@ -13,7 +13,6 @@ from six import iteritems # imports - module imports import frappe -import frappe.website.render from frappe import _ from frappe.utils import now, cint from frappe.model import no_value_fields, default_fields, data_fieldtypes, table_fields, data_field_options @@ -27,6 +26,7 @@ from frappe.model.docfield import supports_translation from frappe.modules.import_file import get_file_path from frappe.model.meta import Meta from frappe.desk.utils import validate_route_conflict +from frappe.website.utils import clear_cache class InvalidFieldNameError(frappe.ValidationError): pass class UniqueFieldnameError(frappe.ValidationError): pass @@ -251,7 +251,7 @@ class DocType(Document): frappe.throw(_('Field "route" is mandatory for Web Views'), title='Missing Field') # clear website cache - frappe.website.render.clear_cache() + clear_cache() def change_modified_of_parent(self): """Change the timestamp of parent DocType if the current one is a child to clear caches.""" @@ -553,11 +553,6 @@ class DocType(Document): from frappe.modules.export_file import export_to_files export_to_files(record_list=[['DocType', self.name]], create_init=True) - def import_doc(self): - """Import from standard folder `[module]/doctype/[name]/[name].json`.""" - from frappe.modules.import_module import import_from_files - import_from_files(record_list=[[self.module, 'doctype', self.name]]) - def make_controller_template(self): """Make boilerplate controller template.""" make_boilerplate("controller._py", self) diff --git a/frappe/installer.py b/frappe/installer.py index d7d885d60e..138d5abc1b 100755 --- a/frappe/installer.py +++ b/frappe/installer.py @@ -282,10 +282,10 @@ def remove_app(app_name, dry_run=False, yes=False, no_backup=False, force=False) def post_install(rebuild_website=False): - from frappe.website import render + from frappe.website.utils import clear_cache as clear_website_cache if rebuild_website: - render.clear_cache() + clear_website_cache() init_singles() frappe.db.commit() diff --git a/frappe/migrate.py b/frappe/migrate.py index 619510fe5e..0fe7c2e034 100644 --- a/frappe/migrate.py +++ b/frappe/migrate.py @@ -15,7 +15,7 @@ from frappe.utils.connections import check_connection from frappe.utils.dashboard import sync_dashboards from frappe.cache_manager import clear_global_cache from frappe.desk.notifications import clear_notifications -from frappe.website import render +from frappe.website.utils import clear_cache as clear_website_cache from frappe.core.doctype.language.language import sync_languages from frappe.modules.utils import sync_customizations from frappe.core.doctype.scheduled_job_type.scheduled_job_type import sync_jobs @@ -78,7 +78,7 @@ Otherwise, check the server logs and ensure that all the required services are r frappe.get_doc('Portal Settings', 'Portal Settings').sync_menu() # syncs statics - render.clear_cache() + clear_website_cache() # updating installed applications data frappe.get_single('Installed Applications').update_versions() diff --git a/frappe/patches.txt b/frappe/patches.txt index e70be0a37b..a2a3f73592 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -93,7 +93,7 @@ frappe.patches.v4_0.create_custom_field_for_owner_match frappe.patches.v4_0.enable_scheduler_in_system_settings execute:frappe.db.sql("update tabReport set apply_user_permissions=1") #2014-06-03 frappe.patches.v4_0.replace_deprecated_timezones -execute:import frappe.website.render; frappe.website.render.clear_cache("login"); #2014-06-10 +execute:import frappe.website.utils; frappe.website.utils.clear_cache("login"); #2014-06-10 frappe.patches.v4_0.fix_attach_field_file_url execute:frappe.permissions.reset_perms("User") #2015-03-24 execute:frappe.db.sql("""delete from `tabUserRole` where ifnull(parentfield, '')='' or ifnull(`role`, '')=''""") #2014-08-18 diff --git a/frappe/templates/includes/comments/comments.py b/frappe/templates/includes/comments/comments.py index d08eb12ba8..b1cd98f2b9 100644 --- a/frappe/templates/includes/comments/comments.py +++ b/frappe/templates/includes/comments/comments.py @@ -4,7 +4,7 @@ from __future__ import unicode_literals import frappe import re -from frappe.website.render import clear_cache +from frappe.website.utils import clear_cache from frappe.utils import add_to_date, now from frappe import _ diff --git a/frappe/website/doctype/about_us_settings/about_us_settings.py b/frappe/website/doctype/about_us_settings/about_us_settings.py index 5b93cdcede..1ef832ba99 100644 --- a/frappe/website/doctype/about_us_settings/about_us_settings.py +++ b/frappe/website/doctype/about_us_settings/about_us_settings.py @@ -9,11 +9,11 @@ import frappe from frappe.model.document import Document class AboutUsSettings(Document): - + def on_update(self): - from frappe.website.render import clear_cache + from frappe.website.utils import clear_cache clear_cache("about") - + def get_args(): obj = frappe.get_doc("About Us Settings") return { diff --git a/frappe/website/doctype/blog_category/blog_category.py b/frappe/website/doctype/blog_category/blog_category.py index 375ba5b6a3..b8252c993f 100644 --- a/frappe/website/doctype/blog_category/blog_category.py +++ b/frappe/website/doctype/blog_category/blog_category.py @@ -3,7 +3,7 @@ from __future__ import unicode_literals from frappe.website.website_generator import WebsiteGenerator -from frappe.website.render import clear_cache +from frappe.website.utils import clear_cache class BlogCategory(WebsiteGenerator): def autoname(self): diff --git a/frappe/website/doctype/blog_post/blog_post.py b/frappe/website/doctype/blog_post/blog_post.py index 5671540682..b8a443525f 100644 --- a/frappe/website/doctype/blog_post/blog_post.py +++ b/frappe/website/doctype/blog_post/blog_post.py @@ -6,7 +6,7 @@ from __future__ import unicode_literals import frappe from frappe import _ from frappe.website.website_generator import WebsiteGenerator -from frappe.website.render import clear_cache +from frappe.website.utils import clear_cache from frappe.utils import today, cint, global_date_format, get_fullname, strip_html_tags, markdown, sanitize_html from math import ceil from frappe.website.utils import (find_first_image, get_html_content_based_on_type, diff --git a/frappe/website/doctype/blog_settings/blog_settings.py b/frappe/website/doctype/blog_settings/blog_settings.py index 0ed98b9b87..2580806ef1 100644 --- a/frappe/website/doctype/blog_settings/blog_settings.py +++ b/frappe/website/doctype/blog_settings/blog_settings.py @@ -9,8 +9,8 @@ import frappe from frappe.model.document import Document class BlogSettings(Document): - + def on_update(self): - from frappe.website.render import clear_cache + from frappe.website.utils import clear_cache clear_cache("blog") clear_cache("writers") \ No newline at end of file diff --git a/frappe/website/doctype/contact_us_settings/contact_us_settings.py b/frappe/website/doctype/contact_us_settings/contact_us_settings.py index 24e9811a47..c237a13bda 100644 --- a/frappe/website/doctype/contact_us_settings/contact_us_settings.py +++ b/frappe/website/doctype/contact_us_settings/contact_us_settings.py @@ -11,5 +11,5 @@ from frappe.model.document import Document class ContactUsSettings(Document): def on_update(self): - from frappe.website.render import clear_cache + from frappe.website.utils import clear_cache clear_cache("contact") \ No newline at end of file diff --git a/frappe/website/doctype/help_article/help_article.py b/frappe/website/doctype/help_article/help_article.py index fa26cfef99..555d6c9268 100644 --- a/frappe/website/doctype/help_article/help_article.py +++ b/frappe/website/doctype/help_article/help_article.py @@ -93,7 +93,7 @@ def get_sidebar_items(): def clear_cache(): clear_website_cache() - from frappe.website.render import clear_cache + from frappe.website.utils import clear_cache clear_cache() def clear_website_cache(path=None): diff --git a/frappe/website/doctype/portal_settings/portal_settings.py b/frappe/website/doctype/portal_settings/portal_settings.py index 1bfbc70d60..84ee678c1e 100644 --- a/frappe/website/doctype/portal_settings/portal_settings.py +++ b/frappe/website/doctype/portal_settings/portal_settings.py @@ -45,7 +45,7 @@ class PortalSettings(Document): # clear web cache (for menus!) frappe.clear_cache(user='Guest') - from frappe.website.render import clear_cache + from frappe.website.utils import clear_cache clear_cache() # clears role based home pages diff --git a/frappe/website/doctype/web_template/web_template.py b/frappe/website/doctype/web_template/web_template.py index 2fd5bfa179..712cf3c1e4 100644 --- a/frappe/website/doctype/web_template/web_template.py +++ b/frappe/website/doctype/web_template/web_template.py @@ -9,7 +9,7 @@ from shutil import rmtree import frappe from frappe.model.document import Document -from frappe.website.render import clear_cache +from frappe.website.utils import clear_cache from frappe import _ from frappe.modules.export_file import ( write_document_file, diff --git a/frappe/website/doctype/website_script/website_script.py b/frappe/website/doctype/website_script/website_script.py index 5648c27fd6..d1f9f595ed 100644 --- a/frappe/website/doctype/website_script/website_script.py +++ b/frappe/website/doctype/website_script/website_script.py @@ -14,5 +14,5 @@ class WebsiteScript(Document): """clear cache""" frappe.clear_cache(user = 'Guest') - from frappe.website.render import clear_cache + from frappe.website.utils import clear_cache clear_cache() \ No newline at end of file diff --git a/frappe/website/doctype/website_settings/website_settings.py b/frappe/website/doctype/website_settings/website_settings.py index ae8e6bd7e1..e5dec24fbb 100644 --- a/frappe/website/doctype/website_settings/website_settings.py +++ b/frappe/website/doctype/website_settings/website_settings.py @@ -68,7 +68,7 @@ class WebsiteSettings(Document): # clear web cache (for menus!) frappe.clear_cache(user = 'Guest') - from frappe.website.render import clear_cache + from frappe.website.utils import clear_cache clear_cache() # clears role based home pages diff --git a/frappe/website/doctype/website_slideshow/website_slideshow.py b/frappe/website/doctype/website_slideshow/website_slideshow.py index 90f62d1bb1..5e6353c5cd 100644 --- a/frappe/website/doctype/website_slideshow/website_slideshow.py +++ b/frappe/website/doctype/website_slideshow/website_slideshow.py @@ -15,7 +15,7 @@ class WebsiteSlideshow(Document): def on_update(self): # a slide show can be in use and any change in it should get reflected - from frappe.website.render import clear_cache + from frappe.website.utils import clear_cache clear_cache() def validate_images(self): diff --git a/frappe/website/page_renderers/document_page.py b/frappe/website/page_renderers/document_page.py index 34dc42e203..2f272ffd15 100644 --- a/frappe/website/page_renderers/document_page.py +++ b/frappe/website/page_renderers/document_page.py @@ -1,7 +1,7 @@ import frappe from frappe.model.document import get_controller from frappe.website.page_renderers.base_template_page import BaseTemplatePage -from frappe.website.render import build_response +from frappe.website.utils import build_response from frappe.website.router import (get_doctypes_with_web_view, get_page_info_from_web_page_with_dynamic_routes) diff --git a/frappe/website/page_renderers/redirect_page.py b/frappe/website/page_renderers/redirect_page.py index 0bfa8897a3..2049c375e8 100644 --- a/frappe/website/page_renderers/redirect_page.py +++ b/frappe/website/page_renderers/redirect_page.py @@ -1,5 +1,5 @@ import frappe -from frappe.website.render import build_response +from frappe.website.utils import build_response class RedirectPage(object): def __init__(self, path, http_status_code=301): diff --git a/frappe/website/page_renderers/template_page.py b/frappe/website/page_renderers/template_page.py index 467d74b283..d8032c6b9c 100644 --- a/frappe/website/page_renderers/template_page.py +++ b/frappe/website/page_renderers/template_page.py @@ -4,11 +4,9 @@ import click import frappe from frappe.website.page_renderers.base_template_page import BaseTemplatePage -from frappe.website.utils import get_sidebar_items -from frappe.website.render import build_response from frappe.website.router import get_base_template -from frappe.website.utils import (extract_comment_tag, extract_title, - get_next_link, get_toc, get_frontmatter, cache_html) +from frappe.website.utils import (extract_comment_tag, extract_title, get_next_link, + get_toc, get_frontmatter, cache_html, get_sidebar_items, build_response) WEBPAGE_PY_MODULE_PROPERTIES = ("base_template_path", "template", "no_cache", "sitemap", "condition_field") diff --git a/frappe/website/page_renderers/web_page.py b/frappe/website/page_renderers/web_page.py index 0407afe532..5c3807c0ee 100644 --- a/frappe/website/page_renderers/web_page.py +++ b/frappe/website/page_renderers/web_page.py @@ -9,10 +9,6 @@ class WebPage(object): self.path = path.strip('/ ') self.basepath = '' - def get(self): - if self.validate(): - return self.render() - def can_render(self): pass diff --git a/frappe/website/render.py b/frappe/website/render.py deleted file mode 100644 index d5587ad98b..0000000000 --- a/frappe/website/render.py +++ /dev/null @@ -1,76 +0,0 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors -# MIT License. See license.txt - -import json -import mimetypes -import re - -from six import iteritems -from werkzeug.wrappers import Response - -import frappe -import frappe.sessions - - -def build_response(path, data, http_status_code, headers=None): - # build response - response = Response() - response.data = set_content_type(response, data, path) - response.status_code = http_status_code - response.headers["X-Page-Name"] = path.encode("ascii", errors="xmlcharrefreplace") - response.headers["X-From-Cache"] = frappe.local.response.from_cache or False - - add_preload_headers(response) - if headers: - for key, val in iteritems(headers): - response.headers[key] = val.encode("ascii", errors="xmlcharrefreplace") - - return response - -def set_content_type(response, data, path): - if isinstance(data, dict): - response.mimetype = 'application/json' - response.charset = 'utf-8' - data = json.dumps(data) - return data - - response.mimetype = 'text/html' - response.charset = 'utf-8' - - # ignore paths ending with .com to avoid unnecessary download - # https://bugs.python.org/issue22347 - if "." in path and not path.endswith('.com'): - content_type, encoding = mimetypes.guess_type(path) - if content_type: - response.mimetype = content_type - if encoding: - response.charset = encoding - - return data - -def add_preload_headers(response): - from bs4 import BeautifulSoup - - try: - preload = [] - soup = BeautifulSoup(response.data, "lxml") - for elem in soup.find_all('script', src=re.compile(".*")): - preload.append(("script", elem.get("src"))) - - for elem in soup.find_all('link', rel="stylesheet"): - preload.append(("style", elem.get("href"))) - - links = [] - for _type, link in preload: - links.append("<{}>; rel=preload; as={}".format(link, _type)) - - if links: - response.headers["Link"] = ",".join(links) - except Exception: - import traceback - traceback.print_exc() - -def clear_cache(path=None): - # TODO: Remove this - from frappe.website.utils import clear_cache - return clear_cache(path) diff --git a/frappe/website/router.py b/frappe/website/router.py index 9922647ebf..23fd5cb21f 100644 --- a/frappe/website/router.py +++ b/frappe/website/router.py @@ -6,7 +6,7 @@ import os import re import frappe -from frappe.website.utils import extract_comment_tag, extract_title +from frappe.website.utils import extract_title from werkzeug.routing import Map, Rule, NotFound def get_page_info_from_web_page_with_dynamic_routes(path): @@ -143,14 +143,12 @@ def get_page_info(path, app, start, basepath=None, app_path=None, fname=None): # get the source setup_source(page_info) - # extract properties from HTML comments - load_properties_from_source(page_info) + if not page_info.title: + page_info.title = extract_title(page_info.source, page_info.route) # extract properties from controller attributes load_properties_from_controller(page_info) - page_info.build_version = frappe.utils.get_build_version() - return page_info def get_frontmatter(string): @@ -252,47 +250,6 @@ def setup_index(page_info): if os.path.exists(index_txt_path): page_info.index = open(index_txt_path, 'r').read().splitlines() -def load_properties_from_source(page_info): - '''Load properties like no_cache, title from source html''' - - if not page_info.title: - page_info.title = extract_title(page_info.source, page_info.route) - - base_template = extract_comment_tag(page_info.source, 'base_template') - if base_template: - page_info.base_template = base_template - - if (page_info.base_template - and "{%- extends" not in page_info.source - and "{% extends" not in page_info.source - and "" not in page_info.source): - page_info.source = '''{{% extends "{0}" %}} - {{% block page_content %}}{1}{{% endblock %}}'''.format(page_info.base_template, page_info.source) - - if "" in page_info.source: - page_info.no_breadcrumbs = 1 - - if "" in page_info.source: - page_info.show_sidebar = 1 - - if "" in page_info.source: - page_info.add_breadcrumbs = 1 - - if "" in page_info.source: - page_info.no_header = 1 - - if "" in page_info.source: - page_info.add_next_prev_links = 1 - - if "" in page_info.source: - page_info.no_cache = 1 - - if "" in page_info.source: - page_info.sitemap = 0 - - if "" in page_info.source: - page_info.sitemap = 1 - def load_properties_from_controller(page_info): if not page_info.controller: return diff --git a/frappe/website/utils.py b/frappe/website/utils.py index 814bd0101e..f57cfa7ec5 100644 --- a/frappe/website/utils.py +++ b/frappe/website/utils.py @@ -2,6 +2,7 @@ # MIT License. See license.txt import functools import json +import mimetypes import os import re from functools import wraps @@ -9,6 +10,7 @@ from functools import wraps import yaml from past.builtins import cmp from six import iteritems +from werkzeug.wrappers import Response import frappe from frappe import _ @@ -446,3 +448,61 @@ def cache_html(func): return html return cache_html_decorator + +def build_response(path, data, http_status_code, headers=None): + # build response + response = Response() + response.data = set_content_type(response, data, path) + response.status_code = http_status_code + response.headers["X-Page-Name"] = path.encode("ascii", errors="xmlcharrefreplace") + response.headers["X-From-Cache"] = frappe.local.response.from_cache or False + + add_preload_headers(response) + if headers: + for key, val in iteritems(headers): + response.headers[key] = val.encode("ascii", errors="xmlcharrefreplace") + + return response + +def set_content_type(response, data, path): + if isinstance(data, dict): + response.mimetype = 'application/json' + response.charset = 'utf-8' + data = json.dumps(data) + return data + + response.mimetype = 'text/html' + response.charset = 'utf-8' + + # ignore paths ending with .com to avoid unnecessary download + # https://bugs.python.org/issue22347 + if "." in path and not path.endswith('.com'): + content_type, encoding = mimetypes.guess_type(path) + if content_type: + response.mimetype = content_type + if encoding: + response.charset = encoding + + return data + +def add_preload_headers(response): + from bs4 import BeautifulSoup + + try: + preload = [] + soup = BeautifulSoup(response.data, "lxml") + for elem in soup.find_all('script', src=re.compile(".*")): + preload.append(("script", elem.get("src"))) + + for elem in soup.find_all('link', rel="stylesheet"): + preload.append(("style", elem.get("href"))) + + links = [] + for _type, link in preload: + links.append("<{}>; rel=preload; as={}".format(link, _type)) + + if links: + response.headers["Link"] = ",".join(links) + except Exception: + import traceback + traceback.print_exc() diff --git a/frappe/website/website_generator.py b/frappe/website/website_generator.py index f2638fd86d..8b0a1bfd2d 100644 --- a/frappe/website/website_generator.py +++ b/frappe/website/website_generator.py @@ -5,7 +5,7 @@ from __future__ import unicode_literals import frappe from frappe.model.document import Document from frappe.website.utils import cleanup_page_name -from frappe.website.render import clear_cache +from frappe.website.utils import clear_cache from frappe.modules import get_module_name from frappe.search.website_search import update_index_for_path, remove_document_from_index diff --git a/frappe/www/_test/_test_folder/new.csv/__init__.py b/frappe/www/_test/_test_folder/new.csv/__init__.py new file mode 100644 index 0000000000..e69de29bb2