refactor: Remove render.py and move all utility functions to utils.py

- Replace or remove all render imports

frappe/cache_manager.py

@@ -55,7 +55,7 @@ def clear_domain_cache(user=None):
 	cache.delete_value(domain_cache_keys)
 
 def clear_global_cache():
-	from frappe.website.render import clear_cache as clear_website_cache
+	from frappe.website.utils import clear_cache as clear_website_cache
 
 	clear_doctype_cache()
 	clear_website_cache()

frappe/commands/utils.py

@@ -69,14 +69,14 @@ def watch(apps=None):
 def clear_cache(context):
 	"Clear cache, doctype cache and defaults"
 	import frappe.sessions
-	import frappe.website.render
+	from frappe.website.utils import clear_cache as clear_website_cache
 	from frappe.desk.notifications import clear_notifications
 	for site in context.sites:
 		try:
 			frappe.connect(site)
 			frappe.clear_cache()
 			clear_notifications()
-			frappe.website.render.clear_cache()
+			clear_website_cache()
 		finally:
 			frappe.destroy()
 	if not context.sites:
@@ -86,12 +86,12 @@ def clear_cache(context):
 @pass_context
 def clear_website_cache(context):
 	"Clear website cache"
-	import frappe.website.render
+	from frappe.website.utils import clear_cache as clear_website_cache
 	for site in context.sites:
 		try:
 			frappe.init(site=site)
 			frappe.connect()
-			frappe.website.render.clear_cache()
+			clear_website_cache()
 		finally:
 			frappe.destroy()
 	if not context.sites:

frappe/core/doctype/comment/comment.py

@@ -11,7 +11,7 @@ from frappe.core.doctype.user.user import extract_mentions
 from frappe.desk.doctype.notification_log.notification_log import enqueue_create_notification,\
 	get_title, get_title_html
 from frappe.utils import get_fullname
-from frappe.website.render import clear_cache
+from frappe.website.utils import clear_cache
 from frappe.database.schema import add_column
 from frappe.exceptions import ImplicitCommitError
 

frappe/core/doctype/doctype/doctype.py

@@ -13,7 +13,6 @@ from six import iteritems
 
 # imports - module imports
 import frappe
-import frappe.website.render
 from frappe import _
 from frappe.utils import now, cint
 from frappe.model import no_value_fields, default_fields, data_fieldtypes, table_fields, data_field_options
@@ -27,6 +26,7 @@ from frappe.model.docfield import supports_translation
 from frappe.modules.import_file import get_file_path
 from frappe.model.meta import Meta
 from frappe.desk.utils import validate_route_conflict
+from frappe.website.utils import clear_cache
 
 class InvalidFieldNameError(frappe.ValidationError): pass
 class UniqueFieldnameError(frappe.ValidationError): pass
@@ -251,7 +251,7 @@ class DocType(Document):
 				frappe.throw(_('Field "route" is mandatory for Web Views'), title='Missing Field')
 
 			# clear website cache
-			frappe.website.render.clear_cache()
+			clear_cache()
 
 	def change_modified_of_parent(self):
 		"""Change the timestamp of parent DocType if the current one is a child to clear caches."""
@@ -553,11 +553,6 @@ class DocType(Document):
 		from frappe.modules.export_file import export_to_files
 		export_to_files(record_list=[['DocType', self.name]], create_init=True)
 
-	def import_doc(self):
-		"""Import from standard folder `[module]/doctype/[name]/[name].json`."""
-		from frappe.modules.import_module import import_from_files
-		import_from_files(record_list=[[self.module, 'doctype', self.name]])
-
 	def make_controller_template(self):
 		"""Make boilerplate controller template."""
 		make_boilerplate("controller._py", self)

frappe/installer.py

@@ -282,10 +282,10 @@ def remove_app(app_name, dry_run=False, yes=False, no_backup=False, force=False)
 
 
 def post_install(rebuild_website=False):
-	from frappe.website import render
+	from frappe.website.utils import clear_cache as clear_website_cache
 
 	if rebuild_website:
-		render.clear_cache()
+		clear_website_cache()
 
 	init_singles()
 	frappe.db.commit()

frappe/migrate.py

@@ -15,7 +15,7 @@ from frappe.utils.connections import check_connection
 from frappe.utils.dashboard import sync_dashboards
 from frappe.cache_manager import clear_global_cache
 from frappe.desk.notifications import clear_notifications
-from frappe.website import render
+from frappe.website.utils import clear_cache as clear_website_cache
 from frappe.core.doctype.language.language import sync_languages
 from frappe.modules.utils import sync_customizations
 from frappe.core.doctype.scheduled_job_type.scheduled_job_type import sync_jobs
@@ -78,7 +78,7 @@ Otherwise, check the server logs and ensure that all the required services are r
 		frappe.get_doc('Portal Settings', 'Portal Settings').sync_menu()
 
 		# syncs statics
-		render.clear_cache()
+		clear_website_cache()
 
 		# updating installed applications data
 		frappe.get_single('Installed Applications').update_versions()

frappe/patches.txt

@@ -93,7 +93,7 @@ frappe.patches.v4_0.create_custom_field_for_owner_match
 frappe.patches.v4_0.enable_scheduler_in_system_settings
 execute:frappe.db.sql("update tabReport set apply_user_permissions=1") #2014-06-03
 frappe.patches.v4_0.replace_deprecated_timezones
-execute:import frappe.website.render; frappe.website.render.clear_cache("login"); #2014-06-10
+execute:import frappe.website.utils; frappe.website.utils.clear_cache("login"); #2014-06-10
 frappe.patches.v4_0.fix_attach_field_file_url
 execute:frappe.permissions.reset_perms("User") #2015-03-24
 execute:frappe.db.sql("""delete from `tabUserRole` where ifnull(parentfield, '')='' or ifnull(`role`, '')=''""") #2014-08-18

frappe/templates/includes/comments/comments.py

@@ -4,7 +4,7 @@ from __future__ import unicode_literals
 
 import frappe
 import re
-from frappe.website.render import clear_cache
+from frappe.website.utils import clear_cache
 from frappe.utils import add_to_date, now
 
 from frappe import _

frappe/website/doctype/about_us_settings/about_us_settings.py

@@ -9,11 +9,11 @@ import frappe
 from frappe.model.document import Document
 
 class AboutUsSettings(Document):
-			
+
 	def on_update(self):
-		from frappe.website.render import clear_cache
+		from frappe.website.utils import clear_cache
 		clear_cache("about")
-		
+
 def get_args():
 	obj = frappe.get_doc("About Us Settings")
 	return {

frappe/website/doctype/blog_category/blog_category.py

@@ -3,7 +3,7 @@
 
 from __future__ import unicode_literals
 from frappe.website.website_generator import WebsiteGenerator
-from frappe.website.render import clear_cache
+from frappe.website.utils import clear_cache
 
 class BlogCategory(WebsiteGenerator):
 	def autoname(self):

frappe/website/doctype/blog_post/blog_post.py

@@ -6,7 +6,7 @@ from __future__ import unicode_literals
 import frappe
 from frappe import _
 from frappe.website.website_generator import WebsiteGenerator
-from frappe.website.render import clear_cache
+from frappe.website.utils import clear_cache
 from frappe.utils import today, cint, global_date_format, get_fullname, strip_html_tags, markdown, sanitize_html
 from math import ceil
 from frappe.website.utils import (find_first_image, get_html_content_based_on_type,

frappe/website/doctype/blog_settings/blog_settings.py

@@ -9,8 +9,8 @@ import frappe
 from frappe.model.document import Document
 
 class BlogSettings(Document):
-		
+
 	def on_update(self):
-		from frappe.website.render import clear_cache
+		from frappe.website.utils import clear_cache
 		clear_cache("blog")
 		clear_cache("writers")

frappe/website/doctype/contact_us_settings/contact_us_settings.py

@@ -11,5 +11,5 @@ from frappe.model.document import Document
 class ContactUsSettings(Document):
 
 	def on_update(self):
-		from frappe.website.render import clear_cache
+		from frappe.website.utils import clear_cache
 		clear_cache("contact")

frappe/website/doctype/help_article/help_article.py

@@ -93,7 +93,7 @@ def get_sidebar_items():
 def clear_cache():
 	clear_website_cache()
 
-	from frappe.website.render import clear_cache
+	from frappe.website.utils import clear_cache
 	clear_cache()
 
 def clear_website_cache(path=None):

frappe/website/doctype/portal_settings/portal_settings.py

@@ -45,7 +45,7 @@ class PortalSettings(Document):
 		# clear web cache (for menus!)
 		frappe.clear_cache(user='Guest')
 
-		from frappe.website.render import clear_cache
+		from frappe.website.utils import clear_cache
 		clear_cache()
 
 		# clears role based home pages

frappe/website/doctype/web_template/web_template.py

@@ -9,7 +9,7 @@ from shutil import rmtree
 
 import frappe
 from frappe.model.document import Document
-from frappe.website.render import clear_cache
+from frappe.website.utils import clear_cache
 from frappe import _
 from frappe.modules.export_file import (
 	write_document_file,

frappe/website/doctype/website_script/website_script.py

@@ -14,5 +14,5 @@ class WebsiteScript(Document):
 		"""clear cache"""
 		frappe.clear_cache(user = 'Guest')
 
-		from frappe.website.render import clear_cache
+		from frappe.website.utils import clear_cache
 		clear_cache()

frappe/website/doctype/website_settings/website_settings.py

@@ -68,7 +68,7 @@ class WebsiteSettings(Document):
 		# clear web cache (for menus!)
 		frappe.clear_cache(user = 'Guest')
 
-		from frappe.website.render import clear_cache
+		from frappe.website.utils import clear_cache
 		clear_cache()
 
 		# clears role based home pages

frappe/website/doctype/website_slideshow/website_slideshow.py

@@ -15,7 +15,7 @@ class WebsiteSlideshow(Document):
 
 	def on_update(self):
 		# a slide show can be in use and any change in it should get reflected
-		from frappe.website.render import clear_cache
+		from frappe.website.utils import clear_cache
 		clear_cache()
 
 	def validate_images(self):

frappe/website/page_renderers/document_page.py

@@ -1,7 +1,7 @@
 import frappe
 from frappe.model.document import get_controller
 from frappe.website.page_renderers.base_template_page import BaseTemplatePage
-from frappe.website.render import build_response
+from frappe.website.utils import build_response
 from frappe.website.router import (get_doctypes_with_web_view,
 	get_page_info_from_web_page_with_dynamic_routes)
 

frappe/website/page_renderers/redirect_page.py

@@ -1,5 +1,5 @@
 import frappe
-from frappe.website.render import build_response
+from frappe.website.utils import build_response
 
 class RedirectPage(object):
 	def __init__(self, path, http_status_code=301):

frappe/website/page_renderers/template_page.py

@@ -4,11 +4,9 @@ import click
 
 import frappe
 from frappe.website.page_renderers.base_template_page import BaseTemplatePage
-from frappe.website.utils import get_sidebar_items
-from frappe.website.render import build_response
 from frappe.website.router import get_base_template
-from frappe.website.utils import (extract_comment_tag, extract_title,
-	get_next_link, get_toc, get_frontmatter, cache_html)
+from frappe.website.utils import (extract_comment_tag, extract_title, get_next_link,
+	get_toc, get_frontmatter, cache_html, get_sidebar_items, build_response)
 
 WEBPAGE_PY_MODULE_PROPERTIES = ("base_template_path", "template", "no_cache", "sitemap", "condition_field")
 

frappe/website/page_renderers/web_page.py

@@ -9,10 +9,6 @@ class WebPage(object):
 		self.path = path.strip('/ ')
 		self.basepath = ''
 
-	def get(self):
-		if self.validate():
-			return self.render()
-
 	def can_render(self):
 		pass
 

frappe/website/render.py

@@ -1,76 +0,0 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
-
-import json
-import mimetypes
-import re
-
-from six import iteritems
-from werkzeug.wrappers import Response
-
-import frappe
-import frappe.sessions
-
-
-def build_response(path, data, http_status_code, headers=None):
-	# build response
-	response = Response()
-	response.data = set_content_type(response, data, path)
-	response.status_code = http_status_code
-	response.headers["X-Page-Name"] = path.encode("ascii", errors="xmlcharrefreplace")
-	response.headers["X-From-Cache"] = frappe.local.response.from_cache or False
-
-	add_preload_headers(response)
-	if headers:
-		for key, val in iteritems(headers):
-			response.headers[key] = val.encode("ascii", errors="xmlcharrefreplace")
-
-	return response
-
-def set_content_type(response, data, path):
-	if isinstance(data, dict):
-		response.mimetype = 'application/json'
-		response.charset = 'utf-8'
-		data = json.dumps(data)
-		return data
-
-	response.mimetype = 'text/html'
-	response.charset = 'utf-8'
-
-	# ignore paths ending with .com to avoid unnecessary download
-	# https://bugs.python.org/issue22347
-	if "." in path and not path.endswith('.com'):
-		content_type, encoding = mimetypes.guess_type(path)
-		if content_type:
-			response.mimetype = content_type
-			if encoding:
-				response.charset = encoding
-
-	return data
-
-def add_preload_headers(response):
-	from bs4 import BeautifulSoup
-
-	try:
-		preload = []
-		soup = BeautifulSoup(response.data, "lxml")
-		for elem in soup.find_all('script', src=re.compile(".*")):
-			preload.append(("script", elem.get("src")))
-
-		for elem in soup.find_all('link', rel="stylesheet"):
-			preload.append(("style", elem.get("href")))
-
-		links = []
-		for _type, link in preload:
-			links.append("<{}>; rel=preload; as={}".format(link, _type))
-
-		if links:
-			response.headers["Link"] = ",".join(links)
-	except Exception:
-		import traceback
-		traceback.print_exc()
-
-def clear_cache(path=None):
-	# TODO: Remove this
-	from frappe.website.utils import clear_cache
-	return clear_cache(path)

frappe/website/router.py

@@ -6,7 +6,7 @@ import os
 import re
 
 import frappe
-from frappe.website.utils import extract_comment_tag, extract_title
+from frappe.website.utils import extract_title
 from werkzeug.routing import Map, Rule, NotFound
 
 def get_page_info_from_web_page_with_dynamic_routes(path):
@@ -143,14 +143,12 @@ def get_page_info(path, app, start, basepath=None, app_path=None, fname=None):
 	# get the source
 	setup_source(page_info)
 
-	# extract properties from HTML comments
-	load_properties_from_source(page_info)
+	if not page_info.title:
+		page_info.title = extract_title(page_info.source, page_info.route)
 
 	# extract properties from controller attributes
 	load_properties_from_controller(page_info)
 
-	page_info.build_version = frappe.utils.get_build_version()
-
 	return page_info
 
 def get_frontmatter(string):
@@ -252,47 +250,6 @@ def setup_index(page_info):
 		if os.path.exists(index_txt_path):
 			page_info.index = open(index_txt_path, 'r').read().splitlines()
 
-def load_properties_from_source(page_info):
-	'''Load properties like no_cache, title from source html'''
-
-	if not page_info.title:
-		page_info.title = extract_title(page_info.source, page_info.route)
-
-	base_template = extract_comment_tag(page_info.source, 'base_template')
-	if base_template:
-		page_info.base_template = base_template
-
-	if (page_info.base_template
-		and "{%- extends" not in page_info.source
-		and "{% extends" not in page_info.source
-		and "</body>" not in page_info.source):
-		page_info.source = '''{{% extends "{0}" %}}
-			{{% block page_content %}}{1}{{% endblock %}}'''.format(page_info.base_template, page_info.source)
-
-	if "<!-- no-breadcrumbs -->" in page_info.source:
-		page_info.no_breadcrumbs = 1
-
-	if "<!-- show-sidebar -->" in page_info.source:
-		page_info.show_sidebar = 1
-
-	if "<!-- add-breadcrumbs -->" in page_info.source:
-		page_info.add_breadcrumbs = 1
-
-	if "<!-- no-header -->" in page_info.source:
-		page_info.no_header = 1
-
-	if "<!-- add-next-prev-links -->" in page_info.source:
-		page_info.add_next_prev_links = 1
-
-	if "<!-- no-cache -->" in page_info.source:
-		page_info.no_cache = 1
-
-	if "<!-- no-sitemap -->" in page_info.source:
-		page_info.sitemap = 0
-
-	if "<!-- sitemap -->" in page_info.source:
-		page_info.sitemap = 1
-
 def load_properties_from_controller(page_info):
 	if not page_info.controller: return
 

frappe/website/utils.py

@@ -2,6 +2,7 @@
 # MIT License. See license.txt
 import functools
 import json
+import mimetypes
 import os
 import re
 from functools import wraps
@@ -9,6 +10,7 @@ from functools import wraps
 import yaml
 from past.builtins import cmp
 from six import iteritems
+from werkzeug.wrappers import Response
 
 import frappe
 from frappe import _
@@ -446,3 +448,61 @@ def cache_html(func):
 		return html
 
 	return cache_html_decorator
+
+def build_response(path, data, http_status_code, headers=None):
+	# build response
+	response = Response()
+	response.data = set_content_type(response, data, path)
+	response.status_code = http_status_code
+	response.headers["X-Page-Name"] = path.encode("ascii", errors="xmlcharrefreplace")
+	response.headers["X-From-Cache"] = frappe.local.response.from_cache or False
+
+	add_preload_headers(response)
+	if headers:
+		for key, val in iteritems(headers):
+			response.headers[key] = val.encode("ascii", errors="xmlcharrefreplace")
+
+	return response
+
+def set_content_type(response, data, path):
+	if isinstance(data, dict):
+		response.mimetype = 'application/json'
+		response.charset = 'utf-8'
+		data = json.dumps(data)
+		return data
+
+	response.mimetype = 'text/html'
+	response.charset = 'utf-8'
+
+	# ignore paths ending with .com to avoid unnecessary download
+	# https://bugs.python.org/issue22347
+	if "." in path and not path.endswith('.com'):
+		content_type, encoding = mimetypes.guess_type(path)
+		if content_type:
+			response.mimetype = content_type
+			if encoding:
+				response.charset = encoding
+
+	return data
+
+def add_preload_headers(response):
+	from bs4 import BeautifulSoup
+
+	try:
+		preload = []
+		soup = BeautifulSoup(response.data, "lxml")
+		for elem in soup.find_all('script', src=re.compile(".*")):
+			preload.append(("script", elem.get("src")))
+
+		for elem in soup.find_all('link', rel="stylesheet"):
+			preload.append(("style", elem.get("href")))
+
+		links = []
+		for _type, link in preload:
+			links.append("<{}>; rel=preload; as={}".format(link, _type))
+
+		if links:
+			response.headers["Link"] = ",".join(links)
+	except Exception:
+		import traceback
+		traceback.print_exc()

frappe/website/website_generator.py

@@ -5,7 +5,7 @@ from __future__ import unicode_literals
 import frappe
 from frappe.model.document import Document
 from frappe.website.utils import cleanup_page_name
-from frappe.website.render import clear_cache
+from frappe.website.utils import clear_cache
 from frappe.modules import get_module_name
 from frappe.search.website_search import update_index_for_path, remove_document_from_index