[feature] robots.txt - uses Website Settings or frappe.local.conf.robots_txt (file path)

frappe/docs/user/en/guides/basics/site_config.md

@@ -38,3 +38,7 @@ Example:
 - `developer_mode`: If developer mode is set, DocType changes are automatically updated in files.
 - `disable_website_cache`: Don't cache website pages.
 - `logging`: writes logs if **1**, writes queries also if set to **2**.
+
+### Others
+
+- `robots_txt`: Path to robots.txt file to be rendered when going to frappe-site.com/robots.txt

frappe/docs/user/fr/guides/basics/site_config.md

@@ -38,3 +38,7 @@ Example:
 - `developer_mode`: If developer mode is set, DocType changes are automatically updated in files.
 - `disable_website_cache`: Don't cache website pages.
 - `logging`: writes logs if **1**, writes queries also if set to **2**.
+
+### Others
+
+- `robots_txt`: Path to robots.txt file to be rendered when going to frappe-site.com/robots.txt

frappe/website/doctype/website_settings/website_settings.json

@@ -713,7 +713,7 @@
    "ignore_xss_filter": 0, 
    "in_filter": 0, 
    "in_list_view": 0, 
-   "label": "Links in Head", 
+   "label": "HTML Header & Robots", 
    "length": 0, 
    "no_copy": 0, 
    "permlevel": 0, 
@@ -753,19 +753,45 @@
    "search_index": 0, 
    "set_only_once": 0, 
    "unique": 0
+  }, 
+  {
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "fieldname": "robots_txt", 
+   "fieldtype": "Code", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_list_view": 0, 
+   "label": "Robots.txt", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
   }
  ], 
  "hide_heading": 0, 
  "hide_toolbar": 0, 
  "icon": "icon-cog", 
  "idx": 1, 
+ "image_view": 0, 
  "in_create": 0, 
  "in_dialog": 0, 
  "is_submittable": 0, 
  "issingle": 1, 
  "istable": 0, 
  "max_attachments": 10, 
- "modified": "2016-05-20 14:12:40.887550", 
+ "modified": "2016-06-29 03:44:31.762764", 
  "modified_by": "Administrator", 
  "module": "Website", 
  "name": "Website Settings", 

frappe/www/desk.py

@@ -5,7 +5,7 @@ from __future__ import unicode_literals
 
 no_sitemap = 1
 no_cache = 1
-base_template_path = "templates/pages/desk.html"
+base_template_path = "templates/www/desk.html"
 
 import os, re
 import frappe

frappe/www/print.py

@@ -14,7 +14,7 @@ from frappe.utils.pdf import get_pdf
 no_cache = 1
 no_sitemap = 1
 
-base_template_path = "templates/pages/print.html"
+base_template_path = "templates/www/print.html"
 standard_format = "templates/print_formats/standard.html"
 
 def get_context(context):

frappe/www/robots.py

@@ -0,0 +1,11 @@
+import frappe
+
+no_sitemap = 1
+base_template_path = "templates/www/robots.txt"
+
+def get_context(context):
+	robots_txt = (frappe.db.get_single_value('Website Settings', 'robots_txt')
+					or (frappe.local.conf.robots_txt and frappe.read_file(frappe.local.conf.robots_txt))
+					or '')
+
+	return { 'robots_txt': robots_txt }

frappe/www/robots.txt

@@ -0,0 +1 @@
+{{ robots_txt }}

frappe/www/rss.py

@@ -7,7 +7,7 @@ import urllib
 from frappe.utils import escape_html, get_request_site_address, now, cstr
 
 no_cache = 1
-base_template_path = "templates/pages/rss.xml"
+base_template_path = "templates/www/rss.xml"
 
 def get_context(context):
 	"""generate rss feed"""

frappe/www/sitemap.py

@@ -10,7 +10,7 @@ from frappe.website.router import get_pages, get_all_page_context_from_doctypes
 
 no_cache = 1
 no_sitemap = 1
-base_template_path = "templates/pages/sitemap.xml"
+base_template_path = "templates/www/sitemap.xml"
 
 def get_context(context):
 	"""generate the sitemap XML"""

frappe/www/website_script.py

@@ -7,7 +7,7 @@ from frappe.utils import strip
 from frappe.website.doctype.website_theme.website_theme import get_active_theme
 
 no_sitemap = 1
-base_template_path = "templates/pages/website_script.js"
+base_template_path = "templates/www/website_script.js"
 
 def get_context(context):
 	context.javascript = frappe.db.get_single_value('Website Script', 'javascript')

frappe/www/website_theme.py

@@ -8,7 +8,7 @@ from frappe.website.utils import get_shade
 from frappe.website.doctype.website_theme.website_theme import get_active_theme
 
 no_sitemap = 1
-base_template_path = "templates/pages/website_theme.css"
+base_template_path = "templates/www/website_theme.css"
 
 default_properties = {
 	"background_color": "#ffffff",