[redesign] website route is evaluated on request, no cold start

frappe/boot.py

@@ -162,4 +162,4 @@ def load_print(bootinfo, doclist):
 	load_print_css(bootinfo, print_settings)
 
 def load_print_css(bootinfo, print_settings):
-	bootinfo.print_css = frappe.get_attr("frappe.templates.pages.print.get_print_style")(print_settings.print_style or "Modern", for_legacy=True)
+	bootinfo.print_css = frappe.get_attr("frappe.www.print.get_print_style")(print_settings.print_style or "Modern", for_legacy=True)

frappe/utils/jinja.py

@@ -105,7 +105,7 @@ def get_allowed_functions_for_jenv():
 			"get_controller": get_controller
 		},
 		"get_visible_columns": \
-			frappe.get_attr("frappe.templates.pages.print.get_visible_columns"),
+			frappe.get_attr("frappe.www.print.get_visible_columns"),
 		"_": frappe._,
 		"get_shade": get_shade,
 		"scrub": scrub,

frappe/utils/oauth.py

@@ -19,7 +19,7 @@ def get_oauth2_providers():
 				"base_url": "https://www.googleapis.com",
 			},
 
-			"redirect_uri": "/api/method/frappe.templates.pages.login.login_via_google",
+			"redirect_uri": "/api/method/frappe.www.login.login_via_google",
 
 			"auth_url_data": {
 				"scope": "https://www.googleapis.com/auth/userinfo.profile https://www.googleapis.com/auth/userinfo.email",
@@ -38,7 +38,7 @@ def get_oauth2_providers():
 				"base_url": "https://api.github.com/"
 			},
 
-			"redirect_uri": "/api/method/frappe.templates.pages.login.login_via_github",
+			"redirect_uri": "/api/method/frappe.www.login.login_via_github",
 
 			# relative to base_url
 			"api_endpoint": "user"
@@ -52,7 +52,7 @@ def get_oauth2_providers():
 				"base_url": "https://graph.facebook.com"
 			},
 
-			"redirect_uri": "/api/method/frappe.templates.pages.login.login_via_facebook",
+			"redirect_uri": "/api/method/frappe.www.login.login_via_facebook",
 
 			"auth_url_data": {
 				"display": "page",

frappe/website/context.py

@@ -87,7 +87,7 @@ def build_context(context):
 
 def add_sidebar_data(context):
 	from frappe.utils.user import get_fullname_and_avatar
-	import frappe.templates.pages.list
+	import frappe.www.list
 
 	context.my_account_list = []
 	my_account_list = frappe.get_all('Portal Menu Item',
@@ -96,7 +96,7 @@ def add_sidebar_data(context):
 	for item in my_account_list:
 		if item.reference_doctype:
 			try:
-				item.count = len(frappe.templates.pages.list.get(item.reference_doctype).get('result'))
+				item.count = len(frappe.www.list.get(item.reference_doctype).get('result'))
 
 			except frappe.PermissionError:
 				pass

frappe/website/doctype/web_form/web_form.py

@@ -63,7 +63,7 @@ class WebForm(WebsiteGenerator):
 
 	def get_context(self, context):
 		context.show_sidebar=True
-		from frappe.templates.pages.list import get_context as get_list_context
+		from frappe.www.list import get_context as get_list_context
 
 		frappe.local.form_dict.is_web_form = 1
 		context.params = frappe.form_dict
@@ -267,7 +267,7 @@ def check_webform_perm(doctype, name):
 			return True
 
 def get_web_form_list(doctype, txt, filters, limit_start, limit_page_length=20):
-	from frappe.templates.pages.list import get_list
+	from frappe.www.list import get_list
 	if not filters:
 		filters = {}
 

frappe/website/render.py

@@ -20,40 +20,50 @@ class PageNotFoundError(Exception): pass
 def render(path, http_status_code=None):
 	"""render html page"""
 	path = resolve_path(path.strip("/ "))
+	data = None
 
-	try:
-		data = render_page_by_language(path)
-	except frappe.DoesNotExistError, e:
-		doctype, name = get_doctype_from_path(path)
-		if doctype and name:
-			path = "print"
-			frappe.local.form_dict.doctype = doctype
-			frappe.local.form_dict.name = name
-		elif doctype:
-			path = "list"
-			frappe.local.form_dict.doctype = doctype
-		else:
-			path = "404"
-			http_status_code = e.http_status_code
+	# if in list of already known 404s, send it
+	if can_cache() and frappe.cache().hget('website_404', path):
+		data = render_page('404')
+		http_status_code = 404
 
+	else:
 		try:
-			data = render_page(path)
+			data = render_page_by_language(path)
+		except frappe.DoesNotExistError, e:
+			doctype, name = get_doctype_from_path(path)
+			if doctype and name:
+				path = "print"
+				frappe.local.form_dict.doctype = doctype
+				frappe.local.form_dict.name = name
+			elif doctype:
+				path = "list"
+				frappe.local.form_dict.doctype = doctype
+			else:
+				# 404s are expensive, cache them!
+				frappe.cache().hset('website_404', path, True)
+				data = render_page('404')
+				http_status_code = 404
+
+			if not data:
+				try:
+					data = render_page(path)
+				except frappe.PermissionError, e:
+					data, http_status_code = render_403(e, path)
+
 		except frappe.PermissionError, e:
 			data, http_status_code = render_403(e, path)
 
-	except frappe.PermissionError, e:
-		data, http_status_code = render_403(e, path)
+		except frappe.Redirect, e:
+			return build_response(path, "", 301, {
+				"Location": frappe.flags.redirect_location,
+				"Cache-Control": "no-store, no-cache, must-revalidate"
+			})
 
-	except frappe.Redirect, e:
-		return build_response(path, "", 301, {
-			"Location": frappe.flags.redirect_location,
-			"Cache-Control": "no-store, no-cache, must-revalidate"
-		})
-
-	except Exception:
-		path = "error"
-		data = render_page(path)
-		http_status_code = 500
+		except Exception:
+			path = "error"
+			data = render_page(path)
+			http_status_code = 500
 
 	data = add_csrf_token(data)
 
@@ -199,7 +209,7 @@ def clear_cache(path=None):
 	if not path:
 		clear_sitemap()
 		frappe.clear_cache("Guest")
-		frappe.cache().delete_value("_website_pages")
+		frappe.cache().delete_value("website_404")
 		frappe.cache().delete_value("home_page")
 
 	for method in frappe.get_hooks("website_clear_cache"):

frappe/website/router.py

@@ -50,61 +50,79 @@ def resolve_route(path):
 		return get_page_context_from_template(path)
 
 def get_page_context_from_template(path):
-	found = filter(lambda p: p.page_name==path, get_pages())
-	return found[0] if found else None
+	'''Return page_info from path'''
+	for app in frappe.get_installed_apps():
+		app_path = frappe.get_app_path(app)
+
+		for start in ('www', 'templates/pages'):
+			search_path = os.path.join(app_path, start, path)
+			options = (search_path, search_path + '.html', search_path + '.md',
+				search_path + '/index.html', search_path + '/index.md')
+			for o in options:
+				if os.path.exists(o) and not os.path.isdir(o):
+					return get_page_info(o, app, app_path=app_path)
+
+	return None
 
 def get_page_context_from_doctype(path):
-	generator_routes = get_page_context_from_doctypes()
-	if path in generator_routes:
-		route = generator_routes[path]
-		return frappe.get_doc(route.get("doctype"), route.get("name")).get_route_context()
+	page_info = get_page_info_from_doctypes(path)
+	if page_info:
+		return frappe.get_doc(page_info.get("doctype"), page_info.get("name")).get_page_info()
 
 def clear_sitemap():
 	delete_page_cache("*")
 
-def get_page_context_from_doctypes():
+def get_all_page_context_from_doctypes():
+	'''Get all doctype generated routes (for sitemap.xml)'''
 	routes = frappe.cache().get_value("website_generator_routes")
 	if not routes:
-		routes = {}
-		for app in frappe.get_installed_apps():
-			for doctype in frappe.get_hooks("website_generators", app_name = app):
-				condition = ""
-				route_column_name = "page_name"
-				controller = get_controller(doctype)
-				meta = frappe.get_meta(doctype)
-
-				if meta.get_field("parent_website_route"):
-					route_column_name = """concat(ifnull(parent_website_route, ""),
-						if(ifnull(parent_website_route, "")="", "", "/"), page_name)"""
-
-				if controller.website.condition_field:
-					condition ="where {0}=1".format(controller.website.condition_field)
-
-				for r in frappe.db.sql("""select {0} as route, name, modified from `tab{1}`
-						{2}""".format(route_column_name, doctype, condition), as_dict=True):
-					routes[r.route] = {"doctype": doctype, "name": r.name, "modified": r.modified}
-
+		routes = get_page_info_from_doctypes()
 		frappe.cache().set_value("website_generator_routes", routes)
 
 	return routes
 
+def get_page_info_from_doctypes(path=None):
+	routes = {}
+	for app in frappe.get_installed_apps():
+		for doctype in frappe.get_hooks("website_generators", app_name = app):
+			condition = ""
+			values = []
+			route_column_name = "page_name"
+			controller = get_controller(doctype)
+			meta = frappe.get_meta(doctype)
+
+			if meta.get_field("parent_website_route"):
+				route_column_name = """concat(ifnull(parent_website_route, ""),
+					if(ifnull(parent_website_route, "")="", "", "/"), page_name)"""
+
+			if controller.website.condition_field:
+				condition ="where {0}=1".format(controller.website.condition_field)
+
+			if path:
+				condition += ' {0} {1}=%s limit 1'.format(('and' if 'where' in condition else 'where'),
+					route_column_name)
+				values.append(path)
+
+			for r in frappe.db.sql("""select {0} as route, name, modified from `tab{1}`
+					{2}""".format(route_column_name, doctype, condition), values=values, as_dict=True):
+				routes[r.route] = {"doctype": doctype, "name": r.name, "modified": r.modified}
+
+				# just want one path, return it!
+				if path:
+					return routes[r.route]
+
+	return routes
+
 def get_pages():
-	pages = frappe.cache().get_value("_website_pages") if can_cache() else []
+	'''Get all pages. Called for docs / sitemap'''
+	pages = []
+	for app in frappe.get_installed_apps():
+		app_path = frappe.get_app_path(app)
 
-	if not pages:
-		pages = []
-		for app in frappe.get_installed_apps():
-			app_path = frappe.get_app_path(app)
-
-			# old
-			path = os.path.join(app_path, "templates", "pages")
+		for start in ('templates/pages', 'www'):
+			path = os.path.join(app_path, start)
 			pages += get_pages_from_path(path, app, app_path)
 
-			# new
-			path = os.path.join(app_path, "www")
-			pages += get_pages_from_path(path, app, app_path)
-
-		frappe.cache().set_value("_website_pages", pages)
 	return pages
 
 def get_pages_from_path(path, app, app_path):
@@ -123,13 +141,22 @@ def get_pages_from_path(path, app, app_path):
 					continue
 
 				if extn in ("html", "xml", "js", "css", "md"):
-					pages.append(get_page_info(path, basepath, app, app_path, fname))
+					pages.append(get_page_info(path, app, basepath, app_path, fname))
 					# print frappe.as_json(pages[-1])
 
 	return pages
 
-def get_page_info(path, basepath, app, app_path, fname):
+def get_page_info(path, app, basepath=None, app_path=None, fname=None):
 	'''Load page info'''
+	if not fname:
+		fname = os.path.basename(path)
+
+	if not app_path:
+		app_path = frappe.get_app_path(app)
+
+	if not basepath:
+		basepath = os.path.dirname(path)
+
 	page_name, extn = fname.rsplit(".", 1)
 
 	# add website route
@@ -140,25 +167,27 @@ def get_page_info(path, basepath, app, app_path, fname):
 
 	page_info.template = os.path.relpath(os.path.join(basepath, fname), app_path)
 
-	if page_info.basename == 'index' and basepath != path:
-		page_info.basename = ''
+	if page_info.basename == 'index':
+		page_info.basename = os.path.dirname(path).strip('.').strip('/')
 
 	page_info.name = page_info.page_name = os.path.join(os.path.relpath(basepath, path),
-		page_info.basename).strip('/').strip('.').strip('/')
+		page_info.basename).strip('.').strip('/')
 
+	# controller
 	page_info.controller_path = os.path.join(basepath, page_name.replace("-", "_") + ".py")
 
+	if os.path.exists(page_info.controller_path):
+		controller = app + "." + os.path.relpath(page_info.controller_path,
+			app_path).replace(os.path.sep, ".")[:-3]
+		page_info.controller = controller
+
 	# get the source
 	page_info.source = get_source(page_info, basepath)
 
-	# extract properties from HTML comments
 	if page_info.only_content:
+		# extract properties from HTML comments
 		load_properties(page_info)
 
-	# controller
-	controller = app + "." + os.path.relpath(page_info.controller_path,
-		app_path).replace(os.path.sep, ".")[:-3]
-	page_info.controller = controller
 
 	return page_info
 

frappe/website/website_generator.py

@@ -117,7 +117,7 @@ class WebsiteGenerator(Document):
 						where parent_website_route like %s""".format(self.doctype),
 						(old_route, self.get_route(), now(), frappe.session.user, old_route + "%"))
 
-	def get_route_context(self):
+	def get_page_info(self):
 		route = frappe._dict()
 		route.update({
 			"doc": self,

frappe/www/me.py

@@ -5,7 +5,7 @@ from __future__ import unicode_literals
 import frappe
 from frappe import _
 from frappe.utils.user import get_fullname_and_avatar
-import frappe.templates.pages.list
+import frappe.www.list
 
 no_cache = 1
 no_sitemap = 1

frappe/www/message.html

@@ -1,6 +1,6 @@
 {% extends "templates/web.html" %}
 
-{% block title %}{{ title }}{% endblock %}
+{% block title %}{{ title or _("Message") }}{% endblock %}
 
 {% block header %}
 	{% if header is defined -%}
@@ -12,7 +12,7 @@
 
 <div class="message-content" style="min-height: 200px;">
 	<div>
-		{{ message }}
+		{{ message or _("No Message") }}
 	</div>
 </div>
 {% endblock %}

frappe/www/message.py

@@ -27,9 +27,4 @@ def get_context(context):
 			if message.get('http_status_code'):
 				frappe.local.response['http_status_code'] = message['http_status_code']
 
-		else:
-			message_context = {
-				'message': ''
-			}
-
 	return message_context

frappe/www/sitemap.py

@@ -6,7 +6,7 @@ from __future__ import unicode_literals
 import urllib
 import frappe
 from frappe.utils import get_request_site_address, get_datetime, nowdate
-from frappe.website.router import get_pages, get_page_context_from_doctypes
+from frappe.website.router import get_pages, get_all_page_context_from_doctypes
 
 no_cache = 1
 no_sitemap = 1
@@ -23,7 +23,7 @@ def get_context(context):
 				"lastmod": nowdate()
 			})
 
-	for route, data in get_page_context_from_doctypes().iteritems():
+	for route, data in get_all_page_context_from_doctypes().iteritems():
 		links.append({
 			"loc": urllib.basejoin(host, urllib.quote((route or "").encode("utf-8"))),
 			"lastmod": get_datetime(data.get("modified")).strftime("%Y-%m-%d")