From 91d0e47f1dacf544533dc840b848cb5a59646a6a Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Wed, 17 Jun 2020 20:03:38 +0530
Subject: [PATCH 01/56] feat: optimize while storing index

---
 frappe/modules/full_text_search.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/modules/full_text_search.py b/frappe/modules/full_text_search.py
index fce9983907..a9b93a1265 100644
--- a/frappe/modules/full_text_search.py
+++ b/frappe/modules/full_text_search.py
@@ -67,7 +67,7 @@ def build_index(index_name, documents):
 				title=document.title, path=document.path, content=document.content
 			)
 
-	writer.commit()
+	writer.commit(optimize=True)
 
 
 def search(index_name, text, scope=None, limit=20):

From aa0cf2bad4ba4a0ed749e0187bab7f39fcb5cd92 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Wed, 17 Jun 2020 20:05:08 +0530
Subject: [PATCH 02/56] feat: added re-index method

---
 frappe/modules/full_text_search.py | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/frappe/modules/full_text_search.py b/frappe/modules/full_text_search.py
index a9b93a1265..d06d925d08 100644
--- a/frappe/modules/full_text_search.py
+++ b/frappe/modules/full_text_search.py
@@ -101,6 +101,26 @@ def search(index_name, text, scope=None, limit=20):
 
 	return out
 
+def reindex_path(index_name, path):
+	document = get_document_to_index(path):
+	reindex(index_name, document)
+
+def reindex(index_name, document):
+	index_dir = get_index_path(index_name)
+	ix = open_dir(index_dir)
+
+	with ix.searcher() as searcher:
+		writer = ix.writer()
+		
+		# Remove the index of the particular file
+		writer.delete_by_term('path', document.path)
+
+		# that wasn't indexed before. So index it!
+		writer.add_document(
+			title=document.title, path=document.path, content=document.content
+		)
+
+		writer.commit(optimize=True)
 
 def get_index_path(index_name):
 	return frappe.get_site_path("indexes", index_name)

From d5d32770136fc71e6d0de8bbc3a291b825d916a0 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Wed, 17 Jun 2020 20:05:55 +0530
Subject: [PATCH 03/56] feat: move search portable

---
 frappe/templates/doc.html    | 66 +--------------------------
 frappe/website/js/website.js | 86 +++++++++++++++++++++++++++++++++++-
 2 files changed, 87 insertions(+), 65 deletions(-)

diff --git a/frappe/templates/doc.html b/frappe/templates/doc.html
index 3e1cc5509a..1c70aace98 100644
--- a/frappe/templates/doc.html
+++ b/frappe/templates/doc.html
@@ -22,7 +22,7 @@
 			</div>
 			<div class="col-12 col-lg-8">
 				<div class="doc-search-container">
-					<div class="doc-search">
+					<div class="doc-search" id="search-container">
 						<div class="dropdown">
 							<div class="search-icon">
 								<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24"
@@ -117,73 +117,11 @@ id="page-{{ name or route | e }}" data-path="{{ pathname | e }}"
 {%- block script -%}
 <script>
 	frappe.ready(() => {
-		setup_search();
+		frappe.setup_search('#search-container', '{{ docs_search_scope or "" }}');
 
 		$('.web-footer .container')
 			.removeClass('container')
 			.addClass('container-fluid doc-container');
 	});
-
-	function setup_search() {
-		let $dropdown = $('.doc-search .dropdown');
-		let $dropdown_menu = $('.doc-search .dropdown-menu');
-		let $input = $('.doc-search input');
-
-		$(document).on('keypress', e => {
-			if (e.key === '/') {
-				e.preventDefault();
-				$input.focus();
-			}
-		});
-
-		$input.on('input', frappe.utils.debounce(() => {
-			if (!$input.val()) {
-				clear_dropdown();
-				return;
-			}
-
-			frappe.call({
-				method: 'frappe.modules.full_text_search.web_search',
-				args: {
-					index_name: 'web_routes',
-					scope: '{{ docs_search_scope or "" }}' || null,
-					query: $input.val(),
-					limit: 5
-				}
-			}).then(r => {
-				let results = r.message || [];
-				let dropdown_html;
-				if (results.length == 0) {
-					dropdown_html = `<div class="dropdown-item">No results found</div>`;
-				} else {
-					dropdown_html = results.map(r => {
-						return `<a class="dropdown-item" href="/${r.path}">
-							<h6>${r.title_highlights || r.title}</h6>
-							<div style="white-space: normal;">${r.content_highlights}</div>
-						</a>`
-					}).join('')
-				}
-				$dropdown_menu.html(dropdown_html);
-				$dropdown_menu.addClass('show');
-			});
-		}, 500));
-
-		$input.on('focus', () => {
-			if (!$input.val()) {
-				clear_dropdown();
-			}
-		});
-
-		$input.on('blur', () => {
-			setTimeout(() => {
-				clear_dropdown();
-			}, 300);
-		});
-
-		function clear_dropdown() {
-			$dropdown_menu.html('');
-			$dropdown_menu.removeClass('show');
-		}
-	}
 </script>
 {%- endblock -%}
diff --git a/frappe/website/js/website.js b/frappe/website/js/website.js
index cd9609cfd3..b69077f482 100644
--- a/frappe/website/js/website.js
+++ b/frappe/website/js/website.js
@@ -380,9 +380,93 @@ $.extend(frappe, {
 	}
 });
 
+frappe.setup_search = function (target, search_scope) {
+	if (typeof target === "string") {
+		target = $(target);
+	}
+
+	let $search_input = $(`<div class="dropdown">
+			<div class="search-icon">
+				<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24"
+					fill="none"
+					stroke="currentColor" stroke-width="2" stroke-linecap="round"
+					stroke-linejoin="round"
+					class="feather feather-search">
+					<circle cx="11" cy="11" r="8"></circle>
+					<line x1="21" y1="21" x2="16.65" y2="16.65"></line>
+				</svg>
+			</div>
+			<input type="search" class="form-control" placeholder="Search the docs (Press ? to focus)" />
+			<div class="overflow-hidden shadow dropdown-menu w-100">
+			</div>
+		</div>`);
+
+	target.empty();
+	$search_input.appendTo(target);
+
+	let $dropdown = $search_input.find('.dropdown');
+	let $dropdown_menu = $search_input.find('.dropdown-menu');
+	let $input = $search_input.find('input');
+
+	$(document).on('keypress', e => {
+		if (e.key === '/') {
+			e.preventDefault();
+			$input.focus();
+		}
+	});
+
+	$input.on('input', frappe.utils.debounce(() => {
+		if (!$input.val()) {
+			clear_dropdown();
+			return;
+		}
+
+		frappe.call({
+			method: 'frappe.modules.full_text_search.web_search',
+			args: {
+				index_name: 'web_routes',
+				scope: search_scope || null,
+				query: $input.val(),
+				limit: 5
+			}
+		}).then(r => {
+			let results = r.message || [];
+			let dropdown_html;
+			if (results.length == 0) {
+				dropdown_html = `<div class="dropdown-item">No results found</div>`;
+			} else {
+				dropdown_html = results.map(r => {
+					return `<a class="dropdown-item" href="/${r.path}">
+						<h6>${r.title_highlights || r.title}</h6>
+						<div style="white-space: normal;">${r.content_highlights}</div>
+					</a>`
+				}).join('')
+			}
+			$dropdown_menu.html(dropdown_html);
+			$dropdown_menu.addClass('show');
+		});
+	}, 500));
+
+	$input.on('focus', () => {
+		if (!$input.val()) {
+			clear_dropdown();
+		}
+	});
+
+	$input.on('blur', () => {
+		setTimeout(() => {
+			clear_dropdown();
+		}, 300);
+	});
+
+	function clear_dropdown() {
+		$dropdown_menu.html('');
+		$dropdown_menu.removeClass('show');
+	}
+}
+
 
 // Utility functions
-
 window.valid_email = function(id) {
 	// eslint-disable-next-line
 	// copied regex from frappe/utils.js validate_type

From f33c05d128b518fbcffa84ab108ac7872a21ca81 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Wed, 17 Jun 2020 20:08:11 +0530
Subject: [PATCH 04/56] chore: better comments

---
 frappe/modules/full_text_search.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/frappe/modules/full_text_search.py b/frappe/modules/full_text_search.py
index d06d925d08..84e0d1cee5 100644
--- a/frappe/modules/full_text_search.py
+++ b/frappe/modules/full_text_search.py
@@ -106,16 +106,18 @@ def reindex_path(index_name, path):
 	reindex(index_name, document)
 
 def reindex(index_name, document):
+	# open index
 	index_dir = get_index_path(index_name)
 	ix = open_dir(index_dir)
 
+	# initiate search scope to find previous index
 	with ix.searcher() as searcher:
 		writer = ix.writer()
 		
 		# Remove the index of the particular file
 		writer.delete_by_term('path', document.path)
 
-		# that wasn't indexed before. So index it!
+		# re index it!
 		writer.add_document(
 			title=document.title, path=document.path, content=document.content
 		)

From ac887875935d0ffc9620fa799e385368bba4fcbd Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Wed, 17 Jun 2020 20:28:18 +0530
Subject: [PATCH 05/56] fix: syntax

---
 frappe/modules/full_text_search.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/modules/full_text_search.py b/frappe/modules/full_text_search.py
index 84e0d1cee5..5355546c36 100644
--- a/frappe/modules/full_text_search.py
+++ b/frappe/modules/full_text_search.py
@@ -102,7 +102,7 @@ def search(index_name, text, scope=None, limit=20):
 	return out
 
 def reindex_path(index_name, path):
-	document = get_document_to_index(path):
+	document = get_document_to_index(path)
 	reindex(index_name, document)
 
 def reindex(index_name, document):

From 32fbab7d183929efc6d4e711233da8779a098699 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Wed, 17 Jun 2020 20:46:54 +0530
Subject: [PATCH 06/56] feat: remove re_index by path

* get_document_to_index sets user to administrator, this is not desirable when running desk, keeping this API available is not wise
---
 frappe/modules/full_text_search.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/frappe/modules/full_text_search.py b/frappe/modules/full_text_search.py
index 5355546c36..7ccc68f877 100644
--- a/frappe/modules/full_text_search.py
+++ b/frappe/modules/full_text_search.py
@@ -100,11 +100,7 @@ def search(index_name, text, scope=None, limit=20):
 			)
 
 	return out
-
-def reindex_path(index_name, path):
-	document = get_document_to_index(path)
-	reindex(index_name, document)
-
+	
 def reindex(index_name, document):
 	# open index
 	index_dir = get_index_path(index_name)

From 9db450f815b18090ca9f04d01e68d541a7bd6f4e Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Wed, 17 Jun 2020 20:47:03 +0530
Subject: [PATCH 07/56] feat: add remove index option

---
 frappe/modules/full_text_search.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/frappe/modules/full_text_search.py b/frappe/modules/full_text_search.py
index 7ccc68f877..022200f3cc 100644
--- a/frappe/modules/full_text_search.py
+++ b/frappe/modules/full_text_search.py
@@ -120,5 +120,17 @@ def reindex(index_name, document):
 
 		writer.commit(optimize=True)
 
+def remove_from_index(index_name, path):
+	# open index
+	index_dir = get_index_path(index_name)
+	ix = open_dir(index_dir)
+
+	# initiate search scope to find previous index
+	with ix.searcher() as searcher:
+		writer = ix.writer()
+		
+		# Remove the index of the particular file
+		writer.delete_by_term('path', path)
+
 def get_index_path(index_name):
 	return frappe.get_site_path("indexes", index_name)

From 702a6f103abbc0fa4150787de645944c7b983fd1 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Wed, 17 Jun 2020 21:02:42 +0530
Subject: [PATCH 08/56] fix: commit after removing index

---
 frappe/modules/full_text_search.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/frappe/modules/full_text_search.py b/frappe/modules/full_text_search.py
index 022200f3cc..03fb8d8bd6 100644
--- a/frappe/modules/full_text_search.py
+++ b/frappe/modules/full_text_search.py
@@ -132,5 +132,7 @@ def remove_from_index(index_name, path):
 		# Remove the index of the particular file
 		writer.delete_by_term('path', path)
 
+		writer.commit(optimize=True)
+
 def get_index_path(index_name):
 	return frappe.get_site_path("indexes", index_name)

From f0547ce0d0fa51e37eb59eaff2e926583d72f83e Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Wed, 17 Jun 2020 23:10:31 +0530
Subject: [PATCH 09/56] feat: added index hooks to website generator

---
 frappe/modules/full_text_search.py  |  8 ++++++--
 frappe/website/website_generator.py | 16 +++++++++++++++-
 2 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/frappe/modules/full_text_search.py b/frappe/modules/full_text_search.py
index 03fb8d8bd6..e3519690c5 100644
--- a/frappe/modules/full_text_search.py
+++ b/frappe/modules/full_text_search.py
@@ -100,8 +100,12 @@ def search(index_name, text, scope=None, limit=20):
 			)
 
 	return out
+
+def update_index_for_path(index_name, path):
+	document = get_document_to_index(path)
+	update_index(index_name, document)
 	
-def reindex(index_name, document):
+def update_index(index_name, document):
 	# open index
 	index_dir = get_index_path(index_name)
 	ix = open_dir(index_dir)
@@ -120,7 +124,7 @@ def reindex(index_name, document):
 
 		writer.commit(optimize=True)
 
-def remove_from_index(index_name, path):
+def remove_document_from_index(index_name, path):
 	# open index
 	index_dir = get_index_path(index_name)
 	ix = open_dir(index_dir)
diff --git a/frappe/website/website_generator.py b/frappe/website/website_generator.py
index 1a32ef9ffa..e8a13e970c 100644
--- a/frappe/website/website_generator.py
+++ b/frappe/website/website_generator.py
@@ -7,6 +7,7 @@ from frappe.model.document import Document
 from frappe.website.utils import cleanup_page_name
 from frappe.website.render import clear_cache
 from frappe.modules import get_module_name
+from frappe.modules.full_text_search import update_index_for_path, remove_document_from_index
 
 class WebsiteGenerator(Document):
 	website = frappe._dict()
@@ -82,11 +83,18 @@ class WebsiteGenerator(Document):
 		pass
 
 	def on_update(self):
+		old_doc = self.get_doc_before_save()
+		if old_doc.route != self.route:
+			remove_document_from_index("web_routes", old_doc.route)
 		self.send_indexing_request()
 
+	def on_change(self):
+		self.update_website_search_index()
+
 	def on_trash(self):
 		self.clear_cache()
 		self.send_indexing_request('URL_DELETED')
+		remove_document_from_index("web_routes", self.route)
 
 	def is_website_published(self):
 		"""Return true if published in website"""
@@ -129,4 +137,10 @@ class WebsiteGenerator(Document):
 
 			url = frappe.utils.get_url(self.route)
 			frappe.enqueue('frappe.website.doctype.website_settings.google_indexing.publish_site', \
-				url=url, operation_type=operation_type)
\ No newline at end of file
+				url=url, operation_type=operation_type)
+
+	def update_website_search_index(self):
+		if not self.is_website_published():
+			remove_document_from_index("web_routes", self.route)
+		else:
+			frappe.enqueue(update_index_for_path, index_name="web_routes", path=self.route)
\ No newline at end of file

From 841914c110f324515c3be6536196c6b8cbc2280a Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Thu, 18 Jun 2020 13:10:55 +0530
Subject: [PATCH 10/56] feat: allow disabling indexing for certain routes

---
 frappe/website/doctype/web_form/web_form.py |  3 ++
 frappe/website/website_generator.py         | 32 ++++++++++++++++++---
 2 files changed, 31 insertions(+), 4 deletions(-)

diff --git a/frappe/website/doctype/web_form/web_form.py b/frappe/website/doctype/web_form/web_form.py
index a3867f1396..4ac91c852f 100644
--- a/frappe/website/doctype/web_form/web_form.py
+++ b/frappe/website/doctype/web_form/web_form.py
@@ -343,6 +343,9 @@ def get_context(context):
 			frappe.throw(_('Mandatory Information missing:') + '<br><br>'
 				+ '<br>'.join(['{0} ({1})'.format(d.label, d.fieldtype) for d in missing]))
 
+	def enable_website_search_indexing(self):
+		return False
+
 
 @frappe.whitelist(allow_guest=True)
 def accept(web_form, data, docname=None, for_payment=False):
diff --git a/frappe/website/website_generator.py b/frappe/website/website_generator.py
index e8a13e970c..075e5644ac 100644
--- a/frappe/website/website_generator.py
+++ b/frappe/website/website_generator.py
@@ -83,18 +83,20 @@ class WebsiteGenerator(Document):
 		pass
 
 	def on_update(self):
-		old_doc = self.get_doc_before_save()
-		if old_doc.route != self.route:
-			remove_document_from_index("web_routes", old_doc.route)
 		self.send_indexing_request()
+		self.remove_old_route_from_index()
 
 	def on_change(self):
+		# Update the index on change
+		# On change is triggered last in the event lifecycle
 		self.update_website_search_index()
 
 	def on_trash(self):
 		self.clear_cache()
 		self.send_indexing_request('URL_DELETED')
-		remove_document_from_index("web_routes", self.route)
+		# On deleting the doc, remove the page from the web_routes index
+		if self.enable_website_search_indexing():
+			remove_document_from_index("web_routes", self.route)
 
 	def is_website_published(self):
 		"""Return true if published in website"""
@@ -139,8 +141,30 @@ class WebsiteGenerator(Document):
 			frappe.enqueue('frappe.website.doctype.website_settings.google_indexing.publish_site', \
 				url=url, operation_type=operation_type)
 
+	# Override this method to disable indexing
+	def enable_website_search_indexing(self):
+		return True
+
+	def remove_old_route_from_index(self):
+		"""Remove page from the website index if the route has changed."""
+		if not self.enable_website_search_indexing():
+			return
+		old_doc = self.get_doc_before_save()
+		# Check if the route is changed
+		if old_doc.route != self.route:
+			# Remove the route from index if the route has changed
+			remove_document_from_index("web_routes", old_doc.route)
+
 	def update_website_search_index(self):
+		"""
+			Update the full test index executed on document change event.
+			- remove document from index if document is unpublished
+			- update index otherwise
+		"""
+		if not self.enable_website_search_indexing():
+			return
 		if not self.is_website_published():
+			# If the website is not published
 			remove_document_from_index("web_routes", self.route)
 		else:
 			frappe.enqueue(update_index_for_path, index_name="web_routes", path=self.route)
\ No newline at end of file

From f9b59a7b1029e2f756af08cc9c34e070ed7ffbca Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Thu, 18 Jun 2020 13:20:52 +0530
Subject: [PATCH 11/56] chore: add docstrings

---
 frappe/modules/full_text_search.py | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/frappe/modules/full_text_search.py b/frappe/modules/full_text_search.py
index e3519690c5..6f7be30999 100644
--- a/frappe/modules/full_text_search.py
+++ b/frappe/modules/full_text_search.py
@@ -102,10 +102,24 @@ def search(index_name, text, scope=None, limit=20):
 	return out
 
 def update_index_for_path(index_name, path):
+	"""Wraps `update_index` method, gets the document from path
+	and updates the index. This function changes the current user
+	and should only be run as administrator or in a background job.
+
+	Args:
+		index_name (str): name of the index
+		path (str): route of the page to be updated 
+	"""
 	document = get_document_to_index(path)
 	update_index(index_name, document)
 	
 def update_index(index_name, document):
+	"""Update search index for a document
+
+	Args:
+		index_name (str): Name of the index to be updated
+		document (dict): A dictionary with title, path and content
+	"""
 	# open index
 	index_dir = get_index_path(index_name)
 	ix = open_dir(index_dir)
@@ -125,6 +139,12 @@ def update_index(index_name, document):
 		writer.commit(optimize=True)
 
 def remove_document_from_index(index_name, path):
+	"""Remove document from search index
+
+	Args:
+		index_name (str): name of the index
+		path (str): route of the page to be removed 
+	"""
 	# open index
 	index_dir = get_index_path(index_name)
 	ix = open_dir(index_dir)

From d2d37a1e0493bf8c275f3ae3f4dfb5747a47eac1 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Thu, 18 Jun 2020 13:34:28 +0530
Subject: [PATCH 12/56] feat: move whoosh full_text_search build to migrate
 from hooks

---
 frappe/hooks.py   | 2 +-
 frappe/migrate.py | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/frappe/hooks.py b/frappe/hooks.py
index 1f209f00a2..0850cd6f91 100644
--- a/frappe/hooks.py
+++ b/frappe/hooks.py
@@ -273,7 +273,7 @@ setup_wizard_exception = [
 
 before_migrate = ['frappe.patches.v11_0.sync_user_permission_doctype_before_migrate.execute']
 after_migrate = [
-	'frappe.modules.full_text_search.build_index_for_all_routes'
+	'frappe.website.doctype.website_theme.website_theme.generate_theme_files_if_not_exist'
 ]
 
 otp_methods = ['OTP App','Email','SMS']
diff --git a/frappe/migrate.py b/frappe/migrate.py
index 9ec23d8ae7..88c7bfdbb2 100644
--- a/frappe/migrate.py
+++ b/frappe/migrate.py
@@ -19,6 +19,7 @@ from frappe.website import render
 from frappe.core.doctype.language.language import sync_languages
 from frappe.modules.utils import sync_customizations
 from frappe.core.doctype.scheduled_job_type.scheduled_job_type import sync_jobs
+from frappe.modules import full_text_search
 from frappe.utils import global_search
 
 
@@ -82,6 +83,7 @@ Otherwise, check the server logs and ensure that all the required services are r
 
 		# add static pages to global search
 		global_search.update_global_search_for_all_web_pages()
+		full_text_search.build_index_for_all_routes()
 
 		# updating installed applications data
 		frappe.get_single('Installed Applications').update_versions()

From 51b8e467add2b756f27013ce65e2de0dfc82ee63 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Thu, 18 Jun 2020 13:34:43 +0530
Subject: [PATCH 13/56] feat: add flag to disable search index rebuild on
 migrate

---
 frappe/commands/site.py | 10 ++++++++--
 frappe/migrate.py       |  7 ++++---
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/frappe/commands/site.py b/frappe/commands/site.py
index 26eb455338..d64a4609e3 100755
--- a/frappe/commands/site.py
+++ b/frappe/commands/site.py
@@ -274,8 +274,9 @@ def disable_user(context, email):
 @click.command('migrate')
 @click.option('--rebuild-website', help="Rebuild webpages after migration")
 @click.option('--skip-failing', is_flag=True, help="Skip patches that fail to run")
+@click.option('--skip-search-index', is_flag=True, help="Skip patches that fail to run")
 @pass_context
-def migrate(context, rebuild_website=False, skip_failing=False):
+def migrate(context, rebuild_website=False, skip_failing=False, skip_search_index=False):
 	"Run patches, sync schema and rebuild files/translations"
 	from frappe.migrate import migrate
 
@@ -284,7 +285,12 @@ def migrate(context, rebuild_website=False, skip_failing=False):
 		frappe.init(site=site)
 		frappe.connect()
 		try:
-			migrate(context.verbose, rebuild_website=rebuild_website, skip_failing=skip_failing)
+			migrate(
+				context.verbose,
+				rebuild_website=rebuild_website,
+				skip_failing=skip_failing,
+				skip_search_index=skip_search_index
+			)
 		finally:
 			frappe.destroy()
 	if not context.sites:
diff --git a/frappe/migrate.py b/frappe/migrate.py
index 88c7bfdbb2..544e0f6b6f 100644
--- a/frappe/migrate.py
+++ b/frappe/migrate.py
@@ -23,7 +23,7 @@ from frappe.modules import full_text_search
 from frappe.utils import global_search
 
 
-def migrate(verbose=True, rebuild_website=False, skip_failing=False):
+def migrate(verbose=True, rebuild_website=False, skip_failing=False, skip_search_index=False):
 	'''Migrate all apps to the latest version, will:
 	- run before migrate hooks
 	- run patches
@@ -82,8 +82,9 @@ Otherwise, check the server logs and ensure that all the required services are r
 		render.clear_cache()
 
 		# add static pages to global search
-		global_search.update_global_search_for_all_web_pages()
-		full_text_search.build_index_for_all_routes()
+		if not skip_search_index:
+			global_search.update_global_search_for_all_web_pages()
+			full_text_search.build_index_for_all_routes()
 
 		# updating installed applications data
 		frappe.get_single('Installed Applications').update_versions()

From 8bb473143b82ad925476627081e57653e685ca49 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Fri, 19 Jun 2020 14:20:03 +0530
Subject: [PATCH 14/56] refactor: FullTextSearch class

---
 frappe/modules/full_text_search.py | 303 ++++++++++++++++-------------
 1 file changed, 163 insertions(+), 140 deletions(-)

diff --git a/frappe/modules/full_text_search.py b/frappe/modules/full_text_search.py
index 6f7be30999..f0c83b7cb3 100644
--- a/frappe/modules/full_text_search.py
+++ b/frappe/modules/full_text_search.py
@@ -10,153 +10,176 @@ from whoosh.query import Prefix
 from bs4 import BeautifulSoup
 from frappe.website.render import render_page
 from frappe.utils import set_request, cint
-from frappe.utils.global_search import get_routes_to_index
+import os
+
+class FullTextSearch:
+	def __init__(self, index_name):
+		self.index_name = index_name
+		self.index_path = get_index_path(index_name)
+		self.schema = Schema(
+			title=TEXT(stored=True), path=ID(stored=True), content=TEXT(stored=True)
+		)
+
+	def get_routes_to_index(self):
+		routes = get_static_pages_from_all_apps()
+		return routes
+
+	def build(self):
+		print("Building search index for all web routes...")
+		routes = self.get_routes_to_index()
+		self.documents = [self.get_document_to_index(route) for route in routes]
+		self.build_index()
+
+	def get_document_to_index(self, route):
+		frappe.set_user("Guest")
+		frappe.local.no_cache = True
+
+		try:
+			set_request(method="GET", path=route)
+			content = render_page(route)
+			soup = BeautifulSoup(content, "html.parser")
+			page_content = soup.find(class_="page_content")
+			text_content = page_content.text if page_content else ""
+			title = soup.title.text.strip() if soup.title else route
+
+			frappe.set_user("Administrator")
+
+			return frappe._dict(title=title, content=text_content, path=route)
+		except (
+			frappe.PermissionError,
+			frappe.DoesNotExistError,
+			frappe.ValidationError,
+			Exception,
+		):
+			pass
+
+	def update_index_for_path(self, path):
+		"""Wraps `update_index` method, gets the document from path
+		and updates the index. This function changes the current user
+		and should only be run as administrator or in a background job.
+
+		Args:
+			self (object): FullTextSearch Instance
+			path (str): route of the page to be updated 
+		"""
+		document = self.get_document_to_index(path)
+		self.update_index(document)
+
+	def update_index(self, document):
+		"""Update search index for a document
+
+		Args:
+			self (object): FullTextSearch Instance
+			document (dict): A dictionary with title, path and content
+		"""
+		ix = open_dir(self.index_path)
+
+		with ix.searcher() as searcher:
+			writer = ix.writer()
+			writer.delete_by_term('path', document.path)
+			writer.add_document(
+				title=document.title, path=document.path, content=document.content
+			)
+			writer.commit(optimize=True)
+
+	def remove_document_from_index(self, path):
+		"""Remove document from search index
+
+		Args:
+			self (object): FullTextSearch Instance
+			path (str): route of the page to be removed 
+		"""
+		ix = open_dir(self.index_path)
+		with ix.searcher() as searcher:
+			writer = ix.writer()
+			writer.delete_by_term('path', path)
+			writer.commit(optimize=True)
+
+	
+	def build_index(self):
+		ix = create_in(self.index_path, self.schema)
+		writer = ix.writer()
+
+		for document in self.documents:
+			if document:
+				writer.add_document(
+					title=document.title, path=document.path, content=document.content
+				)
+
+		writer.commit(optimize=True)
+
+	def search(self, text, scope=None, limit=20):
+		ix = open_dir(self.index_path)
+
+		results = None
+		out = []
+		
+		with ix.searcher() as searcher:
+			parser = MultifieldParser(["title", "content"], ix.schema)
+			parser.remove_plugin_class(FieldsPlugin)
+			parser.remove_plugin_class(WildcardPlugin)
+			query = parser.parse(text)
+
+			filter_scoped = None
+			if scope:
+				filter_scoped = Prefix("path", scope)
+			results = searcher.search(query, limit=limit, filter=filter_scoped)
+
+			for r in results:
+				title_highlights = r.highlights("title")
+				content_highlights = r.highlights("content")
+				out.append(
+					frappe._dict(
+						title=r["title"],
+						path=r["path"],
+						title_highlights=title_highlights,
+						content_highlights=content_highlights,
+					)
+				)
+
+		return out
 
 
-def build_index_for_all_routes():
-	print("Building search index for all web routes...")
-	routes = get_routes_to_index()
-	documents = [get_document_to_index(route) for route in routes]
-	build_index("web_routes", documents)
+def get_doctype_routes_with_web_view():
+	doctype_with_web_views = frappe.get_all("DocType", { "has_web_view": 1 })
+	return []
 
+def get_static_pages_from_all_apps():
+	apps = frappe.get_installed_apps()
+
+	routes_to_index = []
+	for app in apps:
+		base = frappe.get_app_path(app, 'www')
+		path_to_index = frappe.get_app_path(app, 'www')
+
+		for dirpath, _, filenames in os.walk(path_to_index, topdown=True):
+			for f in filenames:
+				if f.endswith(('.md', '.html')):
+					filepath = os.path.join(dirpath, f)
+
+					route = os.path.relpath(filepath, base)
+					route = route.split('.')[0]
+
+					if route.endswith('index'):
+						route = route.rsplit('index', 1)[0]
+
+					routes_to_index.append(route)
+
+	return routes_to_index
 
 @frappe.whitelist(allow_guest=True)
 def web_search(index_name, query, scope=None, limit=20):
 	limit = cint(limit)
-	return search(index_name, query, scope, limit)
-
-
-def get_document_to_index(route):
-	frappe.set_user("Guest")
-	frappe.local.no_cache = True
-
-	try:
-		set_request(method="GET", path=route)
-		content = render_page(route)
-		soup = BeautifulSoup(content, "html.parser")
-		page_content = soup.find(class_="page_content")
-		text_content = page_content.text if page_content else ""
-		title = soup.title.text.strip() if soup.title else route
-
-		frappe.set_user("Administrator")
-
-		return frappe._dict(title=title, content=text_content, path=route)
-	except (
-		frappe.PermissionError,
-		frappe.DoesNotExistError,
-		frappe.ValidationError,
-		Exception,
-	):
-		pass
-
-
-def build_index(index_name, documents):
-	schema = Schema(
-		title=TEXT(stored=True), path=ID(stored=True), content=TEXT(stored=True)
-	)
-
-	index_dir = get_index_path(index_name)
-	frappe.create_folder(index_dir)
-
-	ix = create_in(index_dir, schema)
-	writer = ix.writer()
-
-	for document in documents:
-		if document:
-			writer.add_document(
-				title=document.title, path=document.path, content=document.content
-			)
-
-	writer.commit(optimize=True)
-
-
-def search(index_name, text, scope=None, limit=20):
-	index_dir = get_index_path(index_name)
-	ix = open_dir(index_dir)
-
-	results = None
-	out = []
-	with ix.searcher() as searcher:
-		parser = MultifieldParser(["title", "content"], ix.schema)
-		parser.remove_plugin_class(FieldsPlugin)
-		parser.remove_plugin_class(WildcardPlugin)
-		query = parser.parse(text)
-
-		filter_scoped = None
-		if scope:
-			filter_scoped = Prefix("path", scope)
-		results = searcher.search(query, limit=limit, filter=filter_scoped)
-
-		for r in results:
-			title_highlights = r.highlights("title")
-			content_highlights = r.highlights("content")
-			out.append(
-				frappe._dict(
-					title=r["title"],
-					path=r["path"],
-					title_highlights=title_highlights,
-					content_highlights=content_highlights,
-				)
-			)
-
-	return out
-
-def update_index_for_path(index_name, path):
-	"""Wraps `update_index` method, gets the document from path
-	and updates the index. This function changes the current user
-	and should only be run as administrator or in a background job.
-
-	Args:
-		index_name (str): name of the index
-		path (str): route of the page to be updated 
-	"""
-	document = get_document_to_index(path)
-	update_index(index_name, document)
-	
-def update_index(index_name, document):
-	"""Update search index for a document
-
-	Args:
-		index_name (str): Name of the index to be updated
-		document (dict): A dictionary with title, path and content
-	"""
-	# open index
-	index_dir = get_index_path(index_name)
-	ix = open_dir(index_dir)
-
-	# initiate search scope to find previous index
-	with ix.searcher() as searcher:
-		writer = ix.writer()
-		
-		# Remove the index of the particular file
-		writer.delete_by_term('path', document.path)
-
-		# re index it!
-		writer.add_document(
-			title=document.title, path=document.path, content=document.content
-		)
-
-		writer.commit(optimize=True)
-
-def remove_document_from_index(index_name, path):
-	"""Remove document from search index
-
-	Args:
-		index_name (str): name of the index
-		path (str): route of the page to be removed 
-	"""
-	# open index
-	index_dir = get_index_path(index_name)
-	ix = open_dir(index_dir)
-
-	# initiate search scope to find previous index
-	with ix.searcher() as searcher:
-		writer = ix.writer()
-		
-		# Remove the index of the particular file
-		writer.delete_by_term('path', path)
-
-		writer.commit(optimize=True)
+	fts = FullTextSearch(index_name)
+	return fts.search(query, scope, limit)
 
 def get_index_path(index_name):
 	return frappe.get_site_path("indexes", index_name)
+
+def update_index_for_path(index_name, path):
+	fts = FullTextSearch(index_name)
+	return fts.update_index_for_path(path)
+
+def remove_document_from_index(index_name, path):
+	fts = FullTextSearch(index_name)
+	return fts.remove_document_from_index(path)

From 59ce616153272fa641283f0991d434273be1e394 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Fri, 19 Jun 2020 14:29:34 +0530
Subject: [PATCH 15/56] chore: added docstrings

---
 frappe/modules/full_text_search.py | 31 +++++++++++++++++++++++++++++-
 1 file changed, 30 insertions(+), 1 deletion(-)

diff --git a/frappe/modules/full_text_search.py b/frappe/modules/full_text_search.py
index f0c83b7cb3..2c48df4d7b 100644
--- a/frappe/modules/full_text_search.py
+++ b/frappe/modules/full_text_search.py
@@ -13,6 +13,8 @@ from frappe.utils import set_request, cint
 import os
 
 class FullTextSearch:
+	""" Frappe Wrapper for Whoosh """
+
 	def __init__(self, index_name):
 		self.index_name = index_name
 		self.index_path = get_index_path(index_name)
@@ -21,16 +23,32 @@ class FullTextSearch:
 		)
 
 	def get_routes_to_index(self):
+		"""Get all routes to be indexed, this includes the static pages
+		in www/ and routes from published documents
+
+		Returns:
+			self (object): FullTextSearch Instance
+		"""
 		routes = get_static_pages_from_all_apps()
 		return routes
 
 	def build(self):
+		"""	Build search index for all web routes """
+
 		print("Building search index for all web routes...")
 		routes = self.get_routes_to_index()
 		self.documents = [self.get_document_to_index(route) for route in routes]
 		self.build_index()
 
 	def get_document_to_index(self, route):
+		"""Render a page and parse it using BeautifulSoup
+
+		Args:
+			path (str): route of the page to be parsed 
+
+		Returns:
+			document (_dict): A dictionary with title, path and content
+		"""
 		frappe.set_user("Guest")
 		frappe.local.no_cache = True
 
@@ -70,7 +88,7 @@ class FullTextSearch:
 
 		Args:
 			self (object): FullTextSearch Instance
-			document (dict): A dictionary with title, path and content
+			document (_dict): A dictionary with title, path and content
 		"""
 		ix = open_dir(self.index_path)
 
@@ -97,6 +115,7 @@ class FullTextSearch:
 
 	
 	def build_index(self):
+		"""Build index for all parsed documents"""
 		ix = create_in(self.index_path, self.schema)
 		writer = ix.writer()
 
@@ -109,6 +128,16 @@ class FullTextSearch:
 		writer.commit(optimize=True)
 
 	def search(self, text, scope=None, limit=20):
+		"""Search from the current index
+
+		Args:
+			text (str): String to search for
+			scope (str, optional): Scope to limit the search. Defaults to None.
+			limit (int, optional): Limit number of search results. Defaults to 20.
+
+		Returns:
+			[List(_dict)]: Search results
+		"""
 		ix = open_dir(self.index_path)
 
 		results = None

From eb9435c0124941d6e550424a9287f640a2c6722e Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Fri, 19 Jun 2020 14:55:33 +0530
Subject: [PATCH 16/56] refactor: rename enable_website_search_indexing to
 allow_website_search_indexing

---
 frappe/website/doctype/web_form/web_form.py | 2 +-
 frappe/website/website_generator.py         | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/frappe/website/doctype/web_form/web_form.py b/frappe/website/doctype/web_form/web_form.py
index 4ac91c852f..cbec84752e 100644
--- a/frappe/website/doctype/web_form/web_form.py
+++ b/frappe/website/doctype/web_form/web_form.py
@@ -343,7 +343,7 @@ def get_context(context):
 			frappe.throw(_('Mandatory Information missing:') + '<br><br>'
 				+ '<br>'.join(['{0} ({1})'.format(d.label, d.fieldtype) for d in missing]))
 
-	def enable_website_search_indexing(self):
+	def allow_website_search_indexing(self):
 		return False
 
 
diff --git a/frappe/website/website_generator.py b/frappe/website/website_generator.py
index 075e5644ac..12743af62e 100644
--- a/frappe/website/website_generator.py
+++ b/frappe/website/website_generator.py
@@ -95,7 +95,7 @@ class WebsiteGenerator(Document):
 		self.clear_cache()
 		self.send_indexing_request('URL_DELETED')
 		# On deleting the doc, remove the page from the web_routes index
-		if self.enable_website_search_indexing():
+		if self.allow_website_search_indexing():
 			remove_document_from_index("web_routes", self.route)
 
 	def is_website_published(self):
@@ -142,12 +142,12 @@ class WebsiteGenerator(Document):
 				url=url, operation_type=operation_type)
 
 	# Override this method to disable indexing
-	def enable_website_search_indexing(self):
+	def allow_website_search_indexing(self):
 		return True
 
 	def remove_old_route_from_index(self):
 		"""Remove page from the website index if the route has changed."""
-		if not self.enable_website_search_indexing():
+		if self.allow_website_search_indexing():
 			return
 		old_doc = self.get_doc_before_save()
 		# Check if the route is changed
@@ -161,7 +161,7 @@ class WebsiteGenerator(Document):
 			- remove document from index if document is unpublished
 			- update index otherwise
 		"""
-		if not self.enable_website_search_indexing():
+		if not self.allow_website_search_indexing():
 			return
 		if not self.is_website_published():
 			# If the website is not published

From 90456a979e45fde047bfd864f9647095ce85e399 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Fri, 19 Jun 2020 14:55:54 +0530
Subject: [PATCH 17/56] feat: allow indexing only if route is present

---
 frappe/website/website_generator.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/frappe/website/website_generator.py b/frappe/website/website_generator.py
index 12743af62e..f8289f53f8 100644
--- a/frappe/website/website_generator.py
+++ b/frappe/website/website_generator.py
@@ -143,7 +143,7 @@ class WebsiteGenerator(Document):
 
 	# Override this method to disable indexing
 	def allow_website_search_indexing(self):
-		return True
+		return bool(self.route)
 
 	def remove_old_route_from_index(self):
 		"""Remove page from the website index if the route has changed."""
@@ -151,7 +151,7 @@ class WebsiteGenerator(Document):
 			return
 		old_doc = self.get_doc_before_save()
 		# Check if the route is changed
-		if old_doc.route != self.route:
+		if old_doc and old_doc.route != self.route:
 			# Remove the route from index if the route has changed
 			remove_document_from_index("web_routes", old_doc.route)
 
@@ -163,8 +163,10 @@ class WebsiteGenerator(Document):
 		"""
 		if not self.allow_website_search_indexing():
 			return
-		if not self.is_website_published():
+		
+		if self.is_website_published():
+			frappe.enqueue(update_index_for_path, index_name="web_routes", path=self.route)
+		elif self.route:
 			# If the website is not published
 			remove_document_from_index("web_routes", self.route)
-		else:
-			frappe.enqueue(update_index_for_path, index_name="web_routes", path=self.route)
\ No newline at end of file
+			
\ No newline at end of file

From a60a6fc04a5e32a53a54b4e9f98051a72b42d7ee Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Fri, 19 Jun 2020 15:33:29 +0530
Subject: [PATCH 18/56] feat: function to get routes from published docitems

---
 frappe/modules/full_text_search.py | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/frappe/modules/full_text_search.py b/frappe/modules/full_text_search.py
index 2c48df4d7b..f626ea8490 100644
--- a/frappe/modules/full_text_search.py
+++ b/frappe/modules/full_text_search.py
@@ -30,6 +30,7 @@ class FullTextSearch:
 			self (object): FullTextSearch Instance
 		"""
 		routes = get_static_pages_from_all_apps()
+		routes += get_doctype_routes_with_web_view()
 		return routes
 
 	def build(self):
@@ -170,8 +171,17 @@ class FullTextSearch:
 
 
 def get_doctype_routes_with_web_view():
-	doctype_with_web_views = frappe.get_all("DocType", { "has_web_view": 1 })
-	return []
+	all_routes = []
+	filters = { "has_web_view": 1, "allow_guest_to_view": 1 }
+	fields = ["name", "is_published_field"]
+	doctype_with_web_views = frappe.get_all("DocType", filters=filters, fields=fields)
+
+	for doctype in doctype_with_web_views:
+		if doctype.is_published_field:
+			routes = frappe.get_all(doctype.name, filters={doctype.is_published_field: 1}, fields="route")
+			all_routes += [route.route for route in routes]
+
+	return all_routes
 
 def get_static_pages_from_all_apps():
 	apps = frappe.get_installed_apps()

From 19f42d187759d75f4d04d4766353ef93e8057ff2 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Mon, 22 Jun 2020 13:29:11 +0530
Subject: [PATCH 19/56] feat: added index all routes function

---
 frappe/migrate.py                  | 4 ++--
 frappe/modules/full_text_search.py | 4 ++++
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/frappe/migrate.py b/frappe/migrate.py
index 544e0f6b6f..40c9bd4068 100644
--- a/frappe/migrate.py
+++ b/frappe/migrate.py
@@ -83,8 +83,8 @@ Otherwise, check the server logs and ensure that all the required services are r
 
 		# add static pages to global search
 		if not skip_search_index:
-			global_search.update_global_search_for_all_web_pages()
-			full_text_search.build_index_for_all_routes()
+			# global_search.update_global_search_for_all_web_pages()
+			full_text_search.build_index_for_all_routes("web_routes")
 
 		# updating installed applications data
 		frappe.get_single('Installed Applications').update_versions()
diff --git a/frappe/modules/full_text_search.py b/frappe/modules/full_text_search.py
index f626ea8490..b1003ecc97 100644
--- a/frappe/modules/full_text_search.py
+++ b/frappe/modules/full_text_search.py
@@ -222,3 +222,7 @@ def update_index_for_path(index_name, path):
 def remove_document_from_index(index_name, path):
 	fts = FullTextSearch(index_name)
 	return fts.remove_document_from_index(path)
+
+def build_index_for_all_routes(index_name):
+	fts = FullTextSearch(index_name)
+	return fts.build()
\ No newline at end of file

From 16a2ef78ff3c3a5b33b7cd9e9ef99c5c7f19231a Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Mon, 22 Jun 2020 13:39:39 +0530
Subject: [PATCH 20/56] style: add missing semicolon

---
 frappe/website/js/website.js | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/frappe/website/js/website.js b/frappe/website/js/website.js
index b69077f482..1c739cc6ca 100644
--- a/frappe/website/js/website.js
+++ b/frappe/website/js/website.js
@@ -439,8 +439,8 @@ frappe.setup_search = function (target, search_scope) {
 					return `<a class="dropdown-item" href="/${r.path}">
 						<h6>${r.title_highlights || r.title}</h6>
 						<div style="white-space: normal;">${r.content_highlights}</div>
-					</a>`
-				}).join('')
+					</a>`;
+				}).join('');
 			}
 			$dropdown_menu.html(dropdown_html);
 			$dropdown_menu.addClass('show');
@@ -463,7 +463,7 @@ frappe.setup_search = function (target, search_scope) {
 		$dropdown_menu.html('');
 		$dropdown_menu.removeClass('show');
 	}
-}
+};
 
 
 // Utility functions

From edf13c92e10d9cba9784047899f8e7a765a7610a Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Mon, 22 Jun 2020 16:29:59 +0530
Subject: [PATCH 21/56] feat: separate search style rules from doc layout

---
 frappe/public/scss/doc.scss     | 35 --------------------------------
 frappe/public/scss/search.scss  | 36 +++++++++++++++++++++++++++++++++
 frappe/public/scss/website.scss |  1 +
 frappe/templates/doc.html       |  2 +-
 4 files changed, 38 insertions(+), 36 deletions(-)
 create mode 100644 frappe/public/scss/search.scss

diff --git a/frappe/public/scss/doc.scss b/frappe/public/scss/doc.scss
index 13a59ba45b..f258e2ee47 100644
--- a/frappe/public/scss/doc.scss
+++ b/frappe/public/scss/doc.scss
@@ -81,45 +81,10 @@ $navbar-height-lg: 4.5rem;
 }
 
 .doc-search {
-	position: relative;
-	width: 100%;
-
 	@include media-breakpoint-up(lg) {
 		padding-left: 4rem;
 		padding-right: 4rem;
 	}
-
-	.search-icon {
-		position: absolute;
-		left: 0;
-		top: 0;
-		width: 2.5rem;
-		height: 100%;
-		display: flex;
-		justify-content: center;
-		align-items: center;
-	}
-
-	svg {
-		color: $gray-600;
-	}
-
-	input {
-		padding-left: 2.5rem;
-	}
-
-	.dropdown-menu {
-		.dropdown-item {
-			padding: 1rem 0.75rem;
-		}
-
-		.match {
-			background-color: $primary-light;
-			color: $primary;
-			font-weight: 500;
-			padding: 0 0.125rem;
-		}
-	}
 }
 
 .doc-sidebar {
diff --git a/frappe/public/scss/search.scss b/frappe/public/scss/search.scss
new file mode 100644
index 0000000000..929b73032d
--- /dev/null
+++ b/frappe/public/scss/search.scss
@@ -0,0 +1,36 @@
+.website-search {
+	position: relative;
+	width: 100%;
+
+	.search-icon {
+		position: absolute;
+		left: 0;
+		top: 0;
+		width: 2.5rem;
+		height: 100%;
+		display: flex;
+		justify-content: center;
+		align-items: center;
+	}
+
+	svg {
+		color: $gray-600;
+	}
+
+	input {
+		padding-left: 2.5rem;
+	}
+
+	.dropdown-menu {
+		.dropdown-item {
+			padding: 1rem 0.75rem;
+		}
+
+		.match {
+			background-color: $primary-light;
+			color: $primary;
+			font-weight: 500;
+			padding: 0 0.125rem;
+		}
+	}
+}
\ No newline at end of file
diff --git a/frappe/public/scss/website.scss b/frappe/public/scss/website.scss
index 3c11d23252..e64c090ea8 100644
--- a/frappe/public/scss/website.scss
+++ b/frappe/public/scss/website.scss
@@ -10,6 +10,7 @@
 @import 'markdown';
 @import 'sidebar';
 @import 'portal';
+@import 'search';
 @import 'doc';
 
 .ql-editor.read-mode {
diff --git a/frappe/templates/doc.html b/frappe/templates/doc.html
index 1c70aace98..f45cf3d29e 100644
--- a/frappe/templates/doc.html
+++ b/frappe/templates/doc.html
@@ -22,7 +22,7 @@
 			</div>
 			<div class="col-12 col-lg-8">
 				<div class="doc-search-container">
-					<div class="doc-search" id="search-container">
+					<div class="website-search doc-search" id="search-container">
 						<div class="dropdown">
 							<div class="search-icon">
 								<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24"

From 5145c58b815dc048d15f15beaa98037adb991eb3 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Tue, 23 Jun 2020 13:09:02 +0530
Subject: [PATCH 22/56] feat: [breaking] remove global search from migrate

* Run search last as it changes the session user
---
 frappe/migrate.py | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/frappe/migrate.py b/frappe/migrate.py
index 40c9bd4068..26264bad6c 100644
--- a/frappe/migrate.py
+++ b/frappe/migrate.py
@@ -20,7 +20,6 @@ from frappe.core.doctype.language.language import sync_languages
 from frappe.modules.utils import sync_customizations
 from frappe.core.doctype.scheduled_job_type.scheduled_job_type import sync_jobs
 from frappe.modules import full_text_search
-from frappe.utils import global_search
 
 
 def migrate(verbose=True, rebuild_website=False, skip_failing=False, skip_search_index=False):
@@ -81,11 +80,6 @@ Otherwise, check the server logs and ensure that all the required services are r
 		# syncs statics
 		render.clear_cache()
 
-		# add static pages to global search
-		if not skip_search_index:
-			# global_search.update_global_search_for_all_web_pages()
-			full_text_search.build_index_for_all_routes("web_routes")
-
 		# updating installed applications data
 		frappe.get_single('Installed Applications').update_versions()
 
@@ -94,6 +88,11 @@ Otherwise, check the server logs and ensure that all the required services are r
 			for fn in frappe.get_hooks('after_migrate', app_name=app):
 				frappe.get_attr(fn)()
 
+		# add static pages to global search
+		if not skip_search_index:
+			# Run this last as it updates the current session
+			full_text_search.build_index_for_all_routes("web_routes")
+
 		frappe.db.commit()
 
 		clear_notifications()

From 602366e285eaac231d2836efa04a951c21df9c9f Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Tue, 21 Jul 2020 14:27:41 +0530
Subject: [PATCH 23/56] feat: separate FTS and website search inside
 frappe.search

---
 frappe/modules/full_text_search.py | 228 -----------------------------
 frappe/search/__init__.py          |  13 ++
 frappe/search/full_text_search.py  | 128 ++++++++++++++++
 frappe/search/website_search.py    | 123 ++++++++++++++++
 4 files changed, 264 insertions(+), 228 deletions(-)
 delete mode 100644 frappe/modules/full_text_search.py
 create mode 100644 frappe/search/__init__.py
 create mode 100644 frappe/search/full_text_search.py
 create mode 100644 frappe/search/website_search.py

diff --git a/frappe/modules/full_text_search.py b/frappe/modules/full_text_search.py
deleted file mode 100644
index b1003ecc97..0000000000
--- a/frappe/modules/full_text_search.py
+++ /dev/null
@@ -1,228 +0,0 @@
-# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
-
-from __future__ import unicode_literals
-import frappe
-from whoosh.index import create_in, open_dir
-from whoosh.fields import TEXT, ID, Schema
-from whoosh.qparser import MultifieldParser, FieldsPlugin, WildcardPlugin
-from whoosh.query import Prefix
-from bs4 import BeautifulSoup
-from frappe.website.render import render_page
-from frappe.utils import set_request, cint
-import os
-
-class FullTextSearch:
-	""" Frappe Wrapper for Whoosh """
-
-	def __init__(self, index_name):
-		self.index_name = index_name
-		self.index_path = get_index_path(index_name)
-		self.schema = Schema(
-			title=TEXT(stored=True), path=ID(stored=True), content=TEXT(stored=True)
-		)
-
-	def get_routes_to_index(self):
-		"""Get all routes to be indexed, this includes the static pages
-		in www/ and routes from published documents
-
-		Returns:
-			self (object): FullTextSearch Instance
-		"""
-		routes = get_static_pages_from_all_apps()
-		routes += get_doctype_routes_with_web_view()
-		return routes
-
-	def build(self):
-		"""	Build search index for all web routes """
-
-		print("Building search index for all web routes...")
-		routes = self.get_routes_to_index()
-		self.documents = [self.get_document_to_index(route) for route in routes]
-		self.build_index()
-
-	def get_document_to_index(self, route):
-		"""Render a page and parse it using BeautifulSoup
-
-		Args:
-			path (str): route of the page to be parsed 
-
-		Returns:
-			document (_dict): A dictionary with title, path and content
-		"""
-		frappe.set_user("Guest")
-		frappe.local.no_cache = True
-
-		try:
-			set_request(method="GET", path=route)
-			content = render_page(route)
-			soup = BeautifulSoup(content, "html.parser")
-			page_content = soup.find(class_="page_content")
-			text_content = page_content.text if page_content else ""
-			title = soup.title.text.strip() if soup.title else route
-
-			frappe.set_user("Administrator")
-
-			return frappe._dict(title=title, content=text_content, path=route)
-		except (
-			frappe.PermissionError,
-			frappe.DoesNotExistError,
-			frappe.ValidationError,
-			Exception,
-		):
-			pass
-
-	def update_index_for_path(self, path):
-		"""Wraps `update_index` method, gets the document from path
-		and updates the index. This function changes the current user
-		and should only be run as administrator or in a background job.
-
-		Args:
-			self (object): FullTextSearch Instance
-			path (str): route of the page to be updated 
-		"""
-		document = self.get_document_to_index(path)
-		self.update_index(document)
-
-	def update_index(self, document):
-		"""Update search index for a document
-
-		Args:
-			self (object): FullTextSearch Instance
-			document (_dict): A dictionary with title, path and content
-		"""
-		ix = open_dir(self.index_path)
-
-		with ix.searcher() as searcher:
-			writer = ix.writer()
-			writer.delete_by_term('path', document.path)
-			writer.add_document(
-				title=document.title, path=document.path, content=document.content
-			)
-			writer.commit(optimize=True)
-
-	def remove_document_from_index(self, path):
-		"""Remove document from search index
-
-		Args:
-			self (object): FullTextSearch Instance
-			path (str): route of the page to be removed 
-		"""
-		ix = open_dir(self.index_path)
-		with ix.searcher() as searcher:
-			writer = ix.writer()
-			writer.delete_by_term('path', path)
-			writer.commit(optimize=True)
-
-	
-	def build_index(self):
-		"""Build index for all parsed documents"""
-		ix = create_in(self.index_path, self.schema)
-		writer = ix.writer()
-
-		for document in self.documents:
-			if document:
-				writer.add_document(
-					title=document.title, path=document.path, content=document.content
-				)
-
-		writer.commit(optimize=True)
-
-	def search(self, text, scope=None, limit=20):
-		"""Search from the current index
-
-		Args:
-			text (str): String to search for
-			scope (str, optional): Scope to limit the search. Defaults to None.
-			limit (int, optional): Limit number of search results. Defaults to 20.
-
-		Returns:
-			[List(_dict)]: Search results
-		"""
-		ix = open_dir(self.index_path)
-
-		results = None
-		out = []
-		
-		with ix.searcher() as searcher:
-			parser = MultifieldParser(["title", "content"], ix.schema)
-			parser.remove_plugin_class(FieldsPlugin)
-			parser.remove_plugin_class(WildcardPlugin)
-			query = parser.parse(text)
-
-			filter_scoped = None
-			if scope:
-				filter_scoped = Prefix("path", scope)
-			results = searcher.search(query, limit=limit, filter=filter_scoped)
-
-			for r in results:
-				title_highlights = r.highlights("title")
-				content_highlights = r.highlights("content")
-				out.append(
-					frappe._dict(
-						title=r["title"],
-						path=r["path"],
-						title_highlights=title_highlights,
-						content_highlights=content_highlights,
-					)
-				)
-
-		return out
-
-
-def get_doctype_routes_with_web_view():
-	all_routes = []
-	filters = { "has_web_view": 1, "allow_guest_to_view": 1 }
-	fields = ["name", "is_published_field"]
-	doctype_with_web_views = frappe.get_all("DocType", filters=filters, fields=fields)
-
-	for doctype in doctype_with_web_views:
-		if doctype.is_published_field:
-			routes = frappe.get_all(doctype.name, filters={doctype.is_published_field: 1}, fields="route")
-			all_routes += [route.route for route in routes]
-
-	return all_routes
-
-def get_static_pages_from_all_apps():
-	apps = frappe.get_installed_apps()
-
-	routes_to_index = []
-	for app in apps:
-		base = frappe.get_app_path(app, 'www')
-		path_to_index = frappe.get_app_path(app, 'www')
-
-		for dirpath, _, filenames in os.walk(path_to_index, topdown=True):
-			for f in filenames:
-				if f.endswith(('.md', '.html')):
-					filepath = os.path.join(dirpath, f)
-
-					route = os.path.relpath(filepath, base)
-					route = route.split('.')[0]
-
-					if route.endswith('index'):
-						route = route.rsplit('index', 1)[0]
-
-					routes_to_index.append(route)
-
-	return routes_to_index
-
-@frappe.whitelist(allow_guest=True)
-def web_search(index_name, query, scope=None, limit=20):
-	limit = cint(limit)
-	fts = FullTextSearch(index_name)
-	return fts.search(query, scope, limit)
-
-def get_index_path(index_name):
-	return frappe.get_site_path("indexes", index_name)
-
-def update_index_for_path(index_name, path):
-	fts = FullTextSearch(index_name)
-	return fts.update_index_for_path(path)
-
-def remove_document_from_index(index_name, path):
-	fts = FullTextSearch(index_name)
-	return fts.remove_document_from_index(path)
-
-def build_index_for_all_routes(index_name):
-	fts = FullTextSearch(index_name)
-	return fts.build()
\ No newline at end of file
diff --git a/frappe/search/__init__.py b/frappe/search/__init__.py
new file mode 100644
index 0000000000..462418ca41
--- /dev/null
+++ b/frappe/search/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors
+# MIT License. See license.txt
+
+import frappe
+from frappe.utils import cint
+from frappe.search.website_search import WebsiteSearch
+from frappe.search.website_search import FullTextSearch
+
+@frappe.whitelist(allow_guest=True)
+def web_search(query, scope=None, limit=20):
+	limit = cint(limit)
+	ws = WebsiteSearch(index_name="web_routes")
+	return ws.search(query, scope, limit)
\ No newline at end of file
diff --git a/frappe/search/full_text_search.py b/frappe/search/full_text_search.py
new file mode 100644
index 0000000000..13fcf22e9b
--- /dev/null
+++ b/frappe/search/full_text_search.py
@@ -0,0 +1,128 @@
+# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors
+# MIT License. See license.txt
+
+from __future__ import unicode_literals
+import frappe
+
+from whoosh.index import create_in, open_dir
+from whoosh.fields import TEXT, ID, Schema
+from whoosh.qparser import MultifieldParser, FieldsPlugin, WildcardPlugin
+from whoosh.query import Prefix
+import os
+
+class FullTextSearch:
+	""" Frappe Wrapper for Whoosh """
+
+	def __init__(self, index_name):
+		self.index_name = index_name
+		self.index_path = get_index_path(index_name)
+		self.schema = self.get_schema()
+		self.id = self.get_id()
+
+	def get_schema(self):
+		return Schema(name=ID(stored=True), content=TEXT(stored=True))
+
+	def get_id(self):
+		return "name"
+
+	def get_items_to_index(self):
+		"""Get all documents to be indexed conforming to the schema"""
+		return []
+
+	def get_document_to_index(self):
+		return {}
+
+	def build(self):
+		"""	Build search index for all documents """
+
+		print("Building search index for all web routes...")
+		self.documents = self.get_items_to_index()
+		self.build_index()
+
+	def update_index_by_name(self, doc_name):
+		"""Wraps `update_index` method, gets the document from name
+		and updates the index. This function changes the current user
+		and should only be run as administrator or in a background job.
+
+		Args:
+			self (object): FullTextSearch Instance
+			doc_name (str): name of the document to be updated
+		"""
+		document = self.get_document_to_index(doc_name)
+		self.update_index(document)
+
+	def remove_document_from_index(self, doc_name):
+		"""Remove document from search index
+
+		Args:
+			self (object): FullTextSearch Instance
+			doc_name (str): name of the document to be removed
+		"""
+		ix = open_dir(self.index_path)
+		with ix.searcher() as searcher:
+			writer = ix.writer()
+			writer.delete_by_term(self.id, doc_name)
+			writer.commit(optimize=True)
+
+	def update_index(self, document):
+		"""Update search index for a document
+
+		Args:
+			self (object): FullTextSearch Instance
+			document (_dict): A dictionary with title, path and content
+		"""
+		ix = open_dir(self.index_path)
+
+		with ix.searcher() as searcher:
+			writer = ix.writer()
+			writer.delete_by_term(self.id, document[self.id])
+			writer.add_document(**document)
+			writer.commit(optimize=True)
+
+
+	def build_index(self):
+		"""Build index for all parsed documents"""
+		frappe.create_folder(self.index_path)
+		ix = create_in(self.index_path, self.schema)
+		writer = ix.writer()
+
+		for document in self.documents:
+			if document:
+				writer.add_document(**document)
+
+		writer.commit(optimize=True)
+
+	def search(self, text, scope=None, limit=20):
+		"""Search from the current index
+
+		Args:
+			text (str): String to search for
+			scope (str, optional): Scope to limit the search. Defaults to None.
+			limit (int, optional): Limit number of search results. Defaults to 20.
+
+		Returns:
+			[List(_dict)]: Search results
+		"""
+		ix = open_dir(self.index_path)
+
+		results = None
+		out = []
+
+		with ix.searcher() as searcher:
+			parser = MultifieldParser(["title", "content"], ix.schema)
+			parser.remove_plugin_class(FieldsPlugin)
+			parser.remove_plugin_class(WildcardPlugin)
+			query = parser.parse(text)
+
+			filter_scoped = None
+			if scope:
+				filter_scoped = Prefix(self.id, scope)
+			results = searcher.search(query, limit=limit, filter=filter_scoped)
+
+			for r in results:
+				out.append(self.parse_result(r))
+
+		return out
+
+def get_index_path(index_name):
+	return frappe.get_site_path("indexes", index_name)
\ No newline at end of file
diff --git a/frappe/search/website_search.py b/frappe/search/website_search.py
new file mode 100644
index 0000000000..bb8d0d8fa4
--- /dev/null
+++ b/frappe/search/website_search.py
@@ -0,0 +1,123 @@
+# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors
+# MIT License. See license.txt
+
+from __future__ import unicode_literals
+import frappe
+from bs4 import BeautifulSoup
+from whoosh.fields import TEXT, ID, Schema
+from frappe.search.full_text_search import FullTextSearch
+from frappe.website.render import render_page
+from frappe.utils import set_request
+import os
+
+INDEX_NAME = "web_routes"
+
+class WebsiteSearch(FullTextSearch):
+	""" Wrapper for WebsiteSearch """
+
+	def get_schema(self):
+		return Schema(
+			title=TEXT(stored=True), path=ID(stored=True), content=TEXT(stored=True)
+		)
+
+	def get_id(self):
+		return "path"
+
+	def get_items_to_index(self):
+		"""Get all routes to be indexed, this includes the static pages
+		in www/ and routes from published documents
+
+		Returns:
+			self (object): FullTextSearch Instance
+		"""
+		routes = get_static_pages_from_all_apps()
+		routes += get_doctype_routes_with_web_view()
+		print(routes)
+		documents = [self.get_document_to_index(route) for route in routes]
+		return documents
+
+	def get_document_to_index(self, route):
+		"""Render a page and parse it using BeautifulSoup
+
+		Args:
+			path (str): route of the page to be parsed
+
+		Returns:
+			document (_dict): A dictionary with title, path and content
+		"""
+		frappe.set_user("Guest")
+		frappe.local.no_cache = True
+
+		try:
+			set_request(method="GET", path=route)
+			content = render_page(route)
+			soup = BeautifulSoup(content, "html.parser")
+			page_content = soup.find(class_="page_content")
+			text_content = page_content.text if page_content else ""
+			title = soup.title.text.strip() if soup.title else route
+
+			frappe.set_user("Administrator")
+
+			return frappe._dict(title=title, content=text_content, path=route)
+		except Exception:
+			pass
+
+	def parse_result(self, result):
+		title_highlights = result.highlights("title")
+		content_highlights = result.highlights("content")
+
+		return frappe._dict(
+			title=result["title"],
+			path=result["path"],
+			title_highlights=title_highlights,
+			content_highlights=content_highlights,
+		)
+
+
+def get_doctype_routes_with_web_view():
+	all_routes = []
+	filters = { "has_web_view": 1, "allow_guest_to_view": 1 }
+	fields = ["name", "is_published_field"]
+	doctype_with_web_views = frappe.get_all("DocType", filters=filters, fields=fields)
+
+	for doctype in doctype_with_web_views:
+		if doctype.is_published_field:
+			routes = frappe.get_all(doctype.name, filters={doctype.is_published_field: 1}, fields="route")
+			all_routes += [route.route for route in routes]
+
+	return all_routes
+
+def get_static_pages_from_all_apps():
+	apps = frappe.get_installed_apps()
+
+	routes_to_index = []
+	for app in apps:
+		base = frappe.get_app_path(app, 'www')
+		path_to_index = frappe.get_app_path(app, 'www')
+
+		for dirpath, _, filenames in os.walk(path_to_index, topdown=True):
+			for f in filenames:
+				if f.endswith(('.md', '.html')):
+					filepath = os.path.join(dirpath, f)
+
+					route = os.path.relpath(filepath, base)
+					route = route.split('.')[0]
+
+					if route.endswith('index'):
+						route = route.rsplit('index', 1)[0]
+
+					routes_to_index.append(route)
+
+	return routes_to_index
+
+def update_index_for_path(INDEX_NAME, path):
+	ws = WebsiteSearch(INDEX_NAME)
+	return ws.update_index_by_name(path)
+
+def remove_document_from_index(INDEX_NAME, path):
+	ws = WebsiteSearch(INDEX_NAME)
+	return ws.remove_document_from_index(path)
+
+def build_index_for_all_routes():
+	ws = WebsiteSearch(INDEX_NAME)
+	return ws.build()
\ No newline at end of file

From b2100279d6adc3336532ab35d509f860f24d228e Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Tue, 21 Jul 2020 14:27:56 +0530
Subject: [PATCH 24/56] feat: update code for new search

---
 frappe/migrate.py                   | 4 ++--
 frappe/website/js/website.js        | 4 ++--
 frappe/website/website_generator.py | 5 ++---
 3 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/frappe/migrate.py b/frappe/migrate.py
index 26264bad6c..51840e2506 100644
--- a/frappe/migrate.py
+++ b/frappe/migrate.py
@@ -19,7 +19,7 @@ from frappe.website import render
 from frappe.core.doctype.language.language import sync_languages
 from frappe.modules.utils import sync_customizations
 from frappe.core.doctype.scheduled_job_type.scheduled_job_type import sync_jobs
-from frappe.modules import full_text_search
+from frappe.search.website_search import build_index_for_all_routes
 
 
 def migrate(verbose=True, rebuild_website=False, skip_failing=False, skip_search_index=False):
@@ -91,7 +91,7 @@ Otherwise, check the server logs and ensure that all the required services are r
 		# add static pages to global search
 		if not skip_search_index:
 			# Run this last as it updates the current session
-			full_text_search.build_index_for_all_routes("web_routes")
+			full_text_search.build_index_for_all_routes()
 
 		frappe.db.commit()
 
diff --git a/frappe/website/js/website.js b/frappe/website/js/website.js
index 1c739cc6ca..6c8b29059e 100644
--- a/frappe/website/js/website.js
+++ b/frappe/website/js/website.js
@@ -420,11 +420,11 @@ frappe.setup_search = function (target, search_scope) {
 			clear_dropdown();
 			return;
 		}
+		console.log(search_scope);
 
 		frappe.call({
-			method: 'frappe.modules.full_text_search.web_search',
+			method: 'frappe.search.web_search',
 			args: {
-				index_name: 'web_routes',
 				scope: search_scope || null,
 				query: $input.val(),
 				limit: 5
diff --git a/frappe/website/website_generator.py b/frappe/website/website_generator.py
index f8289f53f8..7b5d46d47d 100644
--- a/frappe/website/website_generator.py
+++ b/frappe/website/website_generator.py
@@ -7,7 +7,7 @@ from frappe.model.document import Document
 from frappe.website.utils import cleanup_page_name
 from frappe.website.render import clear_cache
 from frappe.modules import get_module_name
-from frappe.modules.full_text_search import update_index_for_path, remove_document_from_index
+from frappe.search.website_search import update_index_for_path, remove_document_from_index
 
 class WebsiteGenerator(Document):
 	website = frappe._dict()
@@ -163,10 +163,9 @@ class WebsiteGenerator(Document):
 		"""
 		if not self.allow_website_search_indexing():
 			return
-		
+
 		if self.is_website_published():
 			frappe.enqueue(update_index_for_path, index_name="web_routes", path=self.route)
 		elif self.route:
 			# If the website is not published
 			remove_document_from_index("web_routes", self.route)
-			
\ No newline at end of file

From 24a984a0eab8d6e17b7191c36fbce5a011b59f7d Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Tue, 21 Jul 2020 14:28:05 +0530
Subject: [PATCH 25/56] feat: add test for FTS

---
 frappe/search/test_full_text_search.py | 129 +++++++++++++++++++++++++
 1 file changed, 129 insertions(+)
 create mode 100644 frappe/search/test_full_text_search.py

diff --git a/frappe/search/test_full_text_search.py b/frappe/search/test_full_text_search.py
new file mode 100644
index 0000000000..bb0d2cef02
--- /dev/null
+++ b/frappe/search/test_full_text_search.py
@@ -0,0 +1,129 @@
+# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors
+# MIT License. See license.txt
+import frappe
+import unittest
+from frappe.search import FullTextSearch
+
+class TestFullTextSearch(unittest.TestCase):
+
+	def setUp(self):
+		index = get_index()
+		index.build()
+		self.index = index
+
+	def test_search_term(self):
+		# Search Wikipedia
+		res = self.index.search("multilingual online encyclopedia")
+		self.assertEqual(res[0], 'wikipedia')
+
+		res = self.index.search("Linux is a family")
+		self.assertEqual(res[0], 'linux')
+
+		res = self.index.search("Enterprise Resource Planning")
+		self.assertEqual(res[0], 'erpnext')
+
+	def test_search_limit(self):
+		res = self.index.search("CommonSearchTerm")
+		self.assertEqual(len(res), 5)
+
+		res = self.index.search("CommonSearchTerm", limit=3)
+		self.assertEqual(len(res), 3)
+
+		res = self.index.search("CommonSearchTerm", limit=20)
+		self.assertEqual(len(res), 5)
+
+	def test_search_scope(self):
+		# Search outside scope
+		res = self.index.search("multilingual online encyclopedia", scope=["os"])
+		self.assertEqual(len(res), 0)
+
+		# Search inside scope
+		res = self.index.search("CommonSearchTerm", scope=["os"])
+		self.assertEqual(len(res), 2)
+		self.assertTrue('os/linux' in res)
+		self.assertTrue('os/gnu' in res)
+
+	def test_remove_document_from_index(self):
+		self.remove_document_from_index("os/gnu")
+		res = self.index.search("GNU")
+		self.assertEqual(len(res), 0)
+
+	def test_update_index(self):
+		# Update existing index
+		self.update_index({
+			'name': "sw/erpnext",
+			'content': """AwesomeERPNext"""
+		})
+
+		res = self.index.search("CommonSearchTerm")
+		self.assertTrue('sw/erpnext' not in res)
+
+		res = self.index.search("AwesomeERPNext")
+		self.assertEqual(res[0], "sw/erpnext")
+
+		# Update new doc
+		self.update_index({
+			'name': "sw/frappebooks",
+			'content': """DesktopAccounting"""
+		})
+
+		res = self.index.search("DesktopAccounting")
+		self.assertEqual(res[0], "sw/frappebooks")
+
+
+
+class TestWrapper(FullTextSearch):
+	def get_items_to_index(self):
+		return get_documents()
+
+	def get_document_to_index(self, name):
+		documents = get_documents()
+		for doc in documents:
+			if doc["name"] == name:
+				return doc
+
+	def parse_result(self, result):
+		return result["name"]
+
+
+def get_index():
+	return TestWrapper("test_frappe_index")
+
+def get_documents():
+	docs = []
+	docs.append({
+		'name': "site/wikipedia",
+		'content': """Wikipedia is a multilingual online encyclopedia created and maintained
+			as an open collaboration project by a community of volunteer editors using a wiki-based editing system.
+			It is the largest and most popular general reference work on the World Wide Web. CommonSearchTerm"""
+	})
+
+	docs.append({
+		'name': "os/linux",
+		'content': """Linux is a family of open source Unix-like operating systems based on the
+			Linux kernel, an operating system kernel first released on September 17, 1991, by Linus Torvalds.
+			Linux is typically packaged in a Linux distribution. CommonSearchTerm"""
+	})
+
+	docs.append({
+		'name': "os/gnu",
+		'content': """GNU is an operating system and an extensive collection of computer software.
+			GNU is composed wholly of free software, most of which is licensed under the GNU Project's own
+			General Public License. GNU is a recursive acronym for "GNU's Not Unix! ",
+			chosen because GNU's design is Unix-like, but differs from Unix by being free software and containing no Unix code. CommonSearchTerm"""
+	})
+
+	docs.append({
+		'name': "sw/erpnext",
+		'content': """ERPNext is a free and open-source integrated Enterprise Resource Planning software developed by
+			Frappe Technologies Pvt. Ltd. and is built on MariaDB database system using a Python based server-side framework.
+			ERPNext is a generic ERP software used by manufacturers, distributors and services companies. CommonSearchTerm"""
+	})
+
+	docs.append({
+		'name': "sw/frappe",
+		'content': """Frappe Framework is a full-stack web framework, that includes everything you need to build and
+			deploy business applications with Rich Admin Interface. CommonSearchTerm"""
+	})
+
+	return docs
\ No newline at end of file

From 9b06881702aa0e21b0f5843680011c7be26064e4 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Tue, 21 Jul 2020 14:30:38 +0530
Subject: [PATCH 26/56] chore: remove log/print statements

---
 frappe/search/website_search.py | 2 +-
 frappe/website/js/website.js    | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/frappe/search/website_search.py b/frappe/search/website_search.py
index bb8d0d8fa4..3b9b5d266b 100644
--- a/frappe/search/website_search.py
+++ b/frappe/search/website_search.py
@@ -32,7 +32,7 @@ class WebsiteSearch(FullTextSearch):
 		"""
 		routes = get_static_pages_from_all_apps()
 		routes += get_doctype_routes_with_web_view()
-		print(routes)
+
 		documents = [self.get_document_to_index(route) for route in routes]
 		return documents
 
diff --git a/frappe/website/js/website.js b/frappe/website/js/website.js
index 6c8b29059e..aff752476c 100644
--- a/frappe/website/js/website.js
+++ b/frappe/website/js/website.js
@@ -420,7 +420,6 @@ frappe.setup_search = function (target, search_scope) {
 			clear_dropdown();
 			return;
 		}
-		console.log(search_scope);
 
 		frappe.call({
 			method: 'frappe.search.web_search',

From bc8e23ca151fa6a8b9cde62ae653a44fc106fc0c Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Tue, 21 Jul 2020 14:30:51 +0530
Subject: [PATCH 27/56] fix: update shortcut for search

---
 frappe/templates/doc.html    | 2 +-
 frappe/website/js/website.js | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/frappe/templates/doc.html b/frappe/templates/doc.html
index f45cf3d29e..3a566a1227 100644
--- a/frappe/templates/doc.html
+++ b/frappe/templates/doc.html
@@ -34,7 +34,7 @@
 									<line x1="21" y1="21" x2="16.65" y2="16.65"></line>
 								</svg>
 							</div>
-							<input type="search" class="form-control" placeholder="Search the docs (Press ? to focus)" />
+							<input type="search" class="form-control" placeholder="Search the docs (Press / to focus)" />
 							<div class="overflow-hidden shadow dropdown-menu w-100">
 							</div>
 						</div>
diff --git a/frappe/website/js/website.js b/frappe/website/js/website.js
index aff752476c..2fc052a73b 100644
--- a/frappe/website/js/website.js
+++ b/frappe/website/js/website.js
@@ -396,7 +396,7 @@ frappe.setup_search = function (target, search_scope) {
 					<line x1="21" y1="21" x2="16.65" y2="16.65"></line>
 				</svg>
 			</div>
-			<input type="search" class="form-control" placeholder="Search the docs (Press ? to focus)" />
+			<input type="search" class="form-control" placeholder="Search the docs (Press / to focus)" />
 			<div class="overflow-hidden shadow dropdown-menu w-100">
 			</div>
 		</div>`);

From 7b6463c40eed684a182a300e8ae77fe367c0eb51 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Tue, 21 Jul 2020 15:42:46 +0530
Subject: [PATCH 28/56] feat: add keyboard navigation to search dropdown

---
 frappe/public/scss/search.scss |  4 +++
 frappe/website/js/website.js   | 61 ++++++++++++++++++++++++++++++----
 2 files changed, 58 insertions(+), 7 deletions(-)

diff --git a/frappe/public/scss/search.scss b/frappe/public/scss/search.scss
index 929b73032d..aee2457f31 100644
--- a/frappe/public/scss/search.scss
+++ b/frappe/public/scss/search.scss
@@ -24,6 +24,10 @@
 	.dropdown-menu {
 		.dropdown-item {
 			padding: 1rem 0.75rem;
+
+			&:focus {
+				background-color: $gray-100;
+			}
 		}
 
 		.match {
diff --git a/frappe/website/js/website.js b/frappe/website/js/website.js
index 2fc052a73b..dfe397db4f 100644
--- a/frappe/website/js/website.js
+++ b/frappe/website/js/website.js
@@ -385,7 +385,7 @@ frappe.setup_search = function (target, search_scope) {
 		target = $(target);
 	}
 
-	let $search_input = $(`<div class="dropdown">
+	let $search_input = $(`<div class="dropdown" id="dropdownMenuSearch">
 			<div class="search-icon">
 				<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24"
 					fill="none"
@@ -397,16 +397,18 @@ frappe.setup_search = function (target, search_scope) {
 				</svg>
 			</div>
 			<input type="search" class="form-control" placeholder="Search the docs (Press / to focus)" />
-			<div class="overflow-hidden shadow dropdown-menu w-100">
+			<div class="overflow-hidden shadow dropdown-menu w-100" aria-labelledby="dropdownMenuSearch">
 			</div>
 		</div>`);
 
 	target.empty();
 	$search_input.appendTo(target);
 
-	let $dropdown = $search_input.find('.dropdown');
+	// let $dropdown = $search_input.find('.dropdown');
 	let $dropdown_menu = $search_input.find('.dropdown-menu');
 	let $input = $search_input.find('input');
+	let dropdownItems;
+	let offsetIndex = 0;
 
 	$(document).on('keypress', e => {
 		if (e.key === '/') {
@@ -443,25 +445,70 @@ frappe.setup_search = function (target, search_scope) {
 			}
 			$dropdown_menu.html(dropdown_html);
 			$dropdown_menu.addClass('show');
+			dropdownItems = $dropdown_menu.find(".dropdown-item");
 		});
 	}, 500));
 
 	$input.on('focus', () => {
 		if (!$input.val()) {
 			clear_dropdown();
+		} else {
+			$input.trigger('input');
 		}
 	});
 
-	$input.on('blur', () => {
-		setTimeout(() => {
-			clear_dropdown();
-		}, 300);
+	$input.keydown(function(e){
+		// up: 38, down: 40
+		if (e.which == 40) {
+			navigate(0);
+		}
 	});
 
+	$dropdown_menu.keydown(function(e){
+		// up: 38, down: 40
+		if (e.which == 38) {
+			navigate(-1);
+		} else if (e.which == 40) {
+			navigate(1);
+		} else if (e.which == 27) {
+			setTimeout(() => {
+				clear_dropdown();
+			}, 300);
+		}
+	});
+
+	// Clear dropdown when clicked
+	$(window).click(function() {
+		clear_dropdown();
+	});
+
+	$search_input.click(function(event){
+		event.stopPropagation();
+	});
+
+	// Navigate the list
+	var navigate = function(diff) {
+		offsetIndex += diff;
+
+		if (offsetIndex >= dropdownItems.length)
+			offsetIndex = 0;
+		if (offsetIndex < 0)
+			offsetIndex = dropdownItems.length - 1;
+		$input.off('blur');
+		dropdownItems.eq(offsetIndex).focus();
+	}
+
 	function clear_dropdown() {
+		offsetIndex = 0
 		$dropdown_menu.html('');
 		$dropdown_menu.removeClass('show');
+		dropdownItems = undefined;
 	}
+
+	// Remove focus state on hover
+	$dropdown_menu.mouseover(function(e){
+		dropdownItems.blur();
+	});
 };
 
 

From 05ac0591dfca5226bb68aa46017321b3176d6b23 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Tue, 21 Jul 2020 16:00:32 +0530
Subject: [PATCH 29/56] refactor: use glob to crawl static files

Co-authored-by: Chinmay D. Pai <chinmaydpai@gmail.com>
---
 frappe/search/website_search.py | 22 ++++++++--------------
 1 file changed, 8 insertions(+), 14 deletions(-)

diff --git a/frappe/search/website_search.py b/frappe/search/website_search.py
index 3b9b5d266b..8412afdf35 100644
--- a/frappe/search/website_search.py
+++ b/frappe/search/website_search.py
@@ -88,26 +88,20 @@ def get_doctype_routes_with_web_view():
 	return all_routes
 
 def get_static_pages_from_all_apps():
+	from glob import glob
 	apps = frappe.get_installed_apps()
 
 	routes_to_index = []
 	for app in apps:
-		base = frappe.get_app_path(app, 'www')
 		path_to_index = frappe.get_app_path(app, 'www')
 
-		for dirpath, _, filenames in os.walk(path_to_index, topdown=True):
-			for f in filenames:
-				if f.endswith(('.md', '.html')):
-					filepath = os.path.join(dirpath, f)
-
-					route = os.path.relpath(filepath, base)
-					route = route.split('.')[0]
-
-					if route.endswith('index'):
-						route = route.rsplit('index', 1)[0]
-
-					routes_to_index.append(route)
-
+		files_to_index = glob(path_to_index + '/**/*.html', recursive=True)
+		files_to_index.extend(glob(path_to_index + '/**/*.md', recursive=True))
+		for file in files_to_index:
+			route = os.path.relpath(file, path_to_index).split('.')[0]
+			if route.endswith('index'):
+				route = route.rsplit('index', 1)[0]
+			routes_to_index.append(route)
 	return routes_to_index
 
 def update_index_for_path(INDEX_NAME, path):

From 9e3d3de432987bcd766e61637823375d85425141 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Tue, 21 Jul 2020 16:17:40 +0530
Subject: [PATCH 30/56] feat: add site indexing command

---
 frappe/commands/site.py           | 19 ++++++++++++++++++-
 frappe/search/full_text_search.py |  2 --
 2 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/frappe/commands/site.py b/frappe/commands/site.py
index d64a4609e3..88e90af4dc 100755
--- a/frappe/commands/site.py
+++ b/frappe/commands/site.py
@@ -661,6 +661,22 @@ def start_ngrok(context):
 		frappe.destroy()
 		ngrok.kill()
 
+@click.command('build-search-index')
+@pass_context
+def build_search_index(context):
+	from frappe.search.website_search import build_index_for_all_routes
+	site = get_site(context)
+	if not site:
+		raise SiteNotSpecifiedError
+
+	print('Building search index for {}'.format(site))
+	frappe.init(site=site)
+	frappe.connect()
+	try:
+		build_index_for_all_routes()
+	finally:
+		frappe.destroy()
+
 commands = [
 	add_system_manager,
 	backup,
@@ -686,5 +702,6 @@ commands = [
 	start_recording,
 	stop_recording,
 	add_to_hosts,
-	start_ngrok
+	start_ngrok,
+	build_search_index
 ]
diff --git a/frappe/search/full_text_search.py b/frappe/search/full_text_search.py
index 13fcf22e9b..a8125ee59e 100644
--- a/frappe/search/full_text_search.py
+++ b/frappe/search/full_text_search.py
@@ -34,8 +34,6 @@ class FullTextSearch:
 
 	def build(self):
 		"""	Build search index for all documents """
-
-		print("Building search index for all web routes...")
 		self.documents = self.get_items_to_index()
 		self.build_index()
 

From 7521c3d14f42c4646b6211dd2fb19dd5448d6dcb Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Tue, 21 Jul 2020 16:25:59 +0530
Subject: [PATCH 31/56] feat: add allow web indexing check to doctype

---
 frappe/core/doctype/doctype/doctype.json | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/frappe/core/doctype/doctype/doctype.json b/frappe/core/doctype/doctype/doctype.json
index 379ea227cb..698289140e 100644
--- a/frappe/core/doctype/doctype/doctype.json
+++ b/frappe/core/doctype/doctype/doctype.json
@@ -70,6 +70,7 @@
   "web_view",
   "has_web_view",
   "allow_guest_to_view",
+  "index_web_pages_for_search",
   "route",
   "is_published_field",
   "advanced",
@@ -517,12 +518,18 @@
    "fieldname": "email_settings_sb",
    "fieldtype": "Section Break",
    "label": "Email Settings"
+  },
+  {
+   "default": "1",
+   "fieldname": "index_web_pages_for_search",
+   "fieldtype": "Check",
+   "label": "Index Web Pages for Search"
   }
  ],
  "icon": "fa fa-bolt",
  "idx": 6,
  "links": [],
- "modified": "2020-03-27 14:51:44.581128",
+ "modified": "2020-07-21 16:20:57.028802",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "DocType",

From 02ddaa5ee7c101fb2ca40d6c0ac5982bcc88a982 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Tue, 21 Jul 2020 16:26:26 +0530
Subject: [PATCH 32/56] feat: mark web search indexing check

---
 .../email/doctype/newsletter/newsletter.json  |   3 +-
 .../doctype/blog_category/blog_category.json  | 251 +++++-------------
 .../website/doctype/blog_post/blog_post.json  |   3 +-
 .../doctype/help_article/help_article.json    |  39 +--
 frappe/website/doctype/web_form/web_form.json |   4 +-
 frappe/website/doctype/web_page/web_page.json |   3 +-
 6 files changed, 82 insertions(+), 221 deletions(-)

diff --git a/frappe/email/doctype/newsletter/newsletter.json b/frappe/email/doctype/newsletter/newsletter.json
index 01f75be954..1ec64826da 100644
--- a/frappe/email/doctype/newsletter/newsletter.json
+++ b/frappe/email/doctype/newsletter/newsletter.json
@@ -132,10 +132,11 @@
  "has_web_view": 1,
  "icon": "fa fa-envelope",
  "idx": 1,
+ "index_web_pages_for_search": 1,
  "is_published_field": "published",
  "links": [],
  "max_attachments": 3,
- "modified": "2020-05-12 18:09:40.137138",
+ "modified": "2020-07-21 16:25:17.687476",
  "modified_by": "Administrator",
  "module": "Email",
  "name": "Newsletter",
diff --git a/frappe/website/doctype/blog_category/blog_category.json b/frappe/website/doctype/blog_category/blog_category.json
index 5921cef2d5..975ae73829 100644
--- a/frappe/website/doctype/blog_category/blog_category.json
+++ b/frappe/website/doctype/blog_category/blog_category.json
@@ -1,196 +1,83 @@
 {
- "allow_copy": 0, 
- "allow_guest_to_view": 1, 
- "allow_import": 1, 
- "allow_rename": 0, 
- "autoname": "field:category_name", 
- "beta": 0, 
- "creation": "2013-03-08 09:41:11", 
- "custom": 0, 
- "docstatus": 0, 
- "doctype": "DocType", 
- "document_type": "Setup", 
- "editable_grid": 0, 
+ "actions": [],
+ "allow_guest_to_view": 1,
+ "allow_import": 1,
+ "autoname": "field:category_name",
+ "creation": "2013-03-08 09:41:11",
+ "doctype": "DocType",
+ "document_type": "Setup",
+ "engine": "InnoDB",
+ "field_order": [
+  "category_name",
+  "title",
+  "published",
+  "route"
+ ],
  "fields": [
   {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "category_name", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Category Name", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "fieldname": "category_name",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Category Name",
+   "reqd": 1,
+   "unique": 1
+  },
   {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "title", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Title", 
-   "length": 0, 
-   "no_copy": 1, 
-   "permlevel": 0, 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "fieldname": "title",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Title",
+   "no_copy": 1,
+   "reqd": 1
+  },
   {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "published", 
-   "fieldtype": "Check", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Published", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "default": "0",
+   "fieldname": "published",
+   "fieldtype": "Check",
+   "in_list_view": 1,
+   "label": "Published"
+  },
   {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "depends_on": "published", 
-   "fieldname": "route", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Route", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
+   "depends_on": "published",
+   "fieldname": "route",
+   "fieldtype": "Data",
+   "label": "Route",
    "unique": 1
   }
- ], 
- "has_web_view": 1, 
- "hide_heading": 0, 
- "hide_toolbar": 0, 
- "icon": "fa fa-tag", 
- "idx": 1, 
- "image_view": 0, 
- "in_create": 0, 
- "is_published_field": "published", 
- "is_submittable": 0, 
- "issingle": 0, 
- "istable": 0, 
- "max_attachments": 0, 
- "modified": "2017-03-06 16:29:05.035486", 
- "modified_by": "Administrator", 
- "module": "Website", 
- "name": "Blog Category", 
- "owner": "Administrator", 
+ ],
+ "has_web_view": 1,
+ "icon": "fa fa-tag",
+ "idx": 1,
+ "index_web_pages_for_search": 1,
+ "is_published_field": "published",
+ "links": [],
+ "modified": "2020-07-21 16:25:18.759447",
+ "modified_by": "Administrator",
+ "module": "Website",
+ "name": "Blog Category",
+ "owner": "Administrator",
  "permissions": [
   {
-   "amend": 0, 
-   "apply_user_permissions": 0, 
-   "cancel": 0, 
-   "create": 1, 
-   "delete": 1, 
-   "email": 1, 
-   "export": 0, 
-   "if_owner": 0, 
-   "import": 0, 
-   "permlevel": 0, 
-   "print": 1, 
-   "read": 1, 
-   "report": 1, 
-   "role": "Website Manager", 
-   "set_user_permissions": 1, 
-   "share": 1, 
-   "submit": 0, 
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "Website Manager",
+   "set_user_permissions": 1,
+   "share": 1,
    "write": 1
-  }, 
+  },
   {
-   "amend": 0, 
-   "apply_user_permissions": 0, 
-   "cancel": 0, 
-   "create": 0, 
-   "delete": 0, 
-   "email": 1, 
-   "export": 0, 
-   "if_owner": 0, 
-   "import": 0, 
-   "permlevel": 0, 
-   "print": 1, 
-   "read": 1, 
-   "report": 0, 
-   "role": "Blogger", 
-   "set_user_permissions": 0, 
-   "share": 0, 
-   "submit": 0, 
-   "write": 0
+   "email": 1,
+   "print": 1,
+   "read": 1,
+   "role": "Blogger"
   }
- ], 
- "quick_entry": 1, 
- "read_only": 0, 
- "read_only_onload": 0, 
- "show_name_in_global_search": 0, 
- "sort_order": "DESC", 
- "track_changes": 1, 
- "track_seen": 0
+ ],
+ "quick_entry": 1,
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "track_changes": 1
 }
\ No newline at end of file
diff --git a/frappe/website/doctype/blog_post/blog_post.json b/frappe/website/doctype/blog_post/blog_post.json
index 8821be1555..c3220788b7 100644
--- a/frappe/website/doctype/blog_post/blog_post.json
+++ b/frappe/website/doctype/blog_post/blog_post.json
@@ -189,10 +189,11 @@
  "has_web_view": 1,
  "icon": "fa fa-quote-left",
  "idx": 1,
+ "index_web_pages_for_search": 1,
  "is_published_field": "published",
  "links": [],
  "max_attachments": 5,
- "modified": "2020-06-01 13:37:57.465434",
+ "modified": "2020-07-21 16:25:17.154911",
  "modified_by": "Administrator",
  "module": "Website",
  "name": "Blog Post",
diff --git a/frappe/website/doctype/help_article/help_article.json b/frappe/website/doctype/help_article/help_article.json
index ca659692c6..2c69de6ff2 100644
--- a/frappe/website/doctype/help_article/help_article.json
+++ b/frappe/website/doctype/help_article/help_article.json
@@ -4,6 +4,7 @@
  "allow_import": 1,
  "creation": "2014-10-30 14:25:53.780105",
  "doctype": "DocType",
+ "engine": "InnoDB",
  "field_order": [
   "title",
   "category",
@@ -15,11 +16,7 @@
   "content",
   "likes",
   "route",
-  "owner",
-  "feedback",
-  "helpful",
-  "cb_00",
-  "not_helpful"
+  "owner"
  ],
  "fields": [
   {
@@ -27,8 +24,7 @@
    "fieldtype": "Data",
    "in_global_search": 1,
    "label": "Title",
-   "reqd": 1,
-   "search_index": 1
+   "reqd": 1
   },
   {
    "fieldname": "category",
@@ -90,39 +86,14 @@
    "fieldtype": "Link",
    "label": "Owner",
    "options": "User"
-  },
-  {
-   "collapsible": 1,
-   "fieldname": "feedback",
-   "fieldtype": "Section Break",
-   "label": "Feedback"
-  },
-  {
-   "default": "0",
-   "fieldname": "helpful",
-   "fieldtype": "Int",
-   "in_list_view": 1,
-   "label": "Helpful",
-   "read_only": 1
-  },
-  {
-   "fieldname": "cb_00",
-   "fieldtype": "Column Break"
-  },
-  {
-   "default": "0",
-   "fieldname": "not_helpful",
-   "fieldtype": "Data",
-   "in_list_view": 1,
-   "label": "Not Helpful",
-   "read_only": 1
   }
  ],
  "has_web_view": 1,
  "icon": "icon-file-alt",
+ "index_web_pages_for_search": 1,
  "is_published_field": "published",
  "links": [],
- "modified": "2020-05-08 10:48:19.997789",
+ "modified": "2020-07-21 16:25:18.577325",
  "modified_by": "Administrator",
  "module": "Website",
  "name": "Help Article",
diff --git a/frappe/website/doctype/web_form/web_form.json b/frappe/website/doctype/web_form/web_form.json
index e6b7632e4d..f4de1e518f 100644
--- a/frappe/website/doctype/web_form/web_form.json
+++ b/frappe/website/doctype/web_form/web_form.json
@@ -352,8 +352,8 @@
  "icon": "icon-edit",
  "is_published_field": "published",
  "links": [],
- "modified": "2019-12-24 14:15:43.497431",
- "modified_by": "faris@erpnext.com",
+ "modified": "2020-07-21 16:25:37.028459",
+ "modified_by": "Administrator",
  "module": "Website",
  "name": "Web Form",
  "owner": "Administrator",
diff --git a/frappe/website/doctype/web_page/web_page.json b/frappe/website/doctype/web_page/web_page.json
index 4766c24045..8f03237363 100644
--- a/frappe/website/doctype/web_page/web_page.json
+++ b/frappe/website/doctype/web_page/web_page.json
@@ -288,10 +288,11 @@
  "has_web_view": 1,
  "icon": "fa fa-file-alt",
  "idx": 1,
+ "index_web_pages_for_search": 1,
  "is_published_field": "published",
  "links": [],
  "max_attachments": 20,
- "modified": "2020-04-25 20:40:39.253548",
+ "modified": "2020-07-21 16:25:17.899069",
  "modified_by": "Administrator",
  "module": "Website",
  "name": "Web Page",

From 59fe1873139ffae182eab797f95736e1849214c1 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Tue, 21 Jul 2020 16:28:43 +0530
Subject: [PATCH 33/56] feat: update search and website generator to check
 index_web_pages_for_search

---
 frappe/search/website_search.py     | 2 +-
 frappe/website/website_generator.py | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/frappe/search/website_search.py b/frappe/search/website_search.py
index 8412afdf35..f8dd2a20da 100644
--- a/frappe/search/website_search.py
+++ b/frappe/search/website_search.py
@@ -76,7 +76,7 @@ class WebsiteSearch(FullTextSearch):
 
 def get_doctype_routes_with_web_view():
 	all_routes = []
-	filters = { "has_web_view": 1, "allow_guest_to_view": 1 }
+	filters = { "has_web_view": 1, "allow_guest_to_view": 1, "index_web_pages_for_search": 1}
 	fields = ["name", "is_published_field"]
 	doctype_with_web_views = frappe.get_all("DocType", filters=filters, fields=fields)
 
diff --git a/frappe/website/website_generator.py b/frappe/website/website_generator.py
index 7b5d46d47d..94bc4b24be 100644
--- a/frappe/website/website_generator.py
+++ b/frappe/website/website_generator.py
@@ -141,9 +141,10 @@ class WebsiteGenerator(Document):
 			frappe.enqueue('frappe.website.doctype.website_settings.google_indexing.publish_site', \
 				url=url, operation_type=operation_type)
 
+	# Change the field value in doctype
 	# Override this method to disable indexing
 	def allow_website_search_indexing(self):
-		return bool(self.route)
+		return self.index_web_pages_for_search
 
 	def remove_old_route_from_index(self):
 		"""Remove page from the website index if the route has changed."""

From 8b7ae16cddb570193216fd2dffbc476724e332e6 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Tue, 21 Jul 2020 16:32:54 +0530
Subject: [PATCH 34/56] feat: make search indexing optional in migrate

---
 frappe/commands/site.py | 6 +++---
 frappe/migrate.py       | 6 +++---
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/frappe/commands/site.py b/frappe/commands/site.py
index 88e90af4dc..54773d6e50 100755
--- a/frappe/commands/site.py
+++ b/frappe/commands/site.py
@@ -274,9 +274,9 @@ def disable_user(context, email):
 @click.command('migrate')
 @click.option('--rebuild-website', help="Rebuild webpages after migration")
 @click.option('--skip-failing', is_flag=True, help="Skip patches that fail to run")
-@click.option('--skip-search-index', is_flag=True, help="Skip patches that fail to run")
+@click.option('--index-site', is_flag=True, help="Index public web pages")
 @pass_context
-def migrate(context, rebuild_website=False, skip_failing=False, skip_search_index=False):
+def migrate(context, rebuild_website=False, skip_failing=False, index_site=False):
 	"Run patches, sync schema and rebuild files/translations"
 	from frappe.migrate import migrate
 
@@ -289,7 +289,7 @@ def migrate(context, rebuild_website=False, skip_failing=False, skip_search_inde
 				context.verbose,
 				rebuild_website=rebuild_website,
 				skip_failing=skip_failing,
-				skip_search_index=skip_search_index
+				index_site=index_site
 			)
 		finally:
 			frappe.destroy()
diff --git a/frappe/migrate.py b/frappe/migrate.py
index 51840e2506..63a87a769d 100644
--- a/frappe/migrate.py
+++ b/frappe/migrate.py
@@ -22,7 +22,7 @@ from frappe.core.doctype.scheduled_job_type.scheduled_job_type import sync_jobs
 from frappe.search.website_search import build_index_for_all_routes
 
 
-def migrate(verbose=True, rebuild_website=False, skip_failing=False, skip_search_index=False):
+def migrate(verbose=True, rebuild_website=False, skip_failing=False, index_site=False):
 	'''Migrate all apps to the latest version, will:
 	- run before migrate hooks
 	- run patches
@@ -89,9 +89,9 @@ Otherwise, check the server logs and ensure that all the required services are r
 				frappe.get_attr(fn)()
 
 		# add static pages to global search
-		if not skip_search_index:
+		if index_site:
 			# Run this last as it updates the current session
-			full_text_search.build_index_for_all_routes()
+			build_index_for_all_routes()
 
 		frappe.db.commit()
 

From 20f06f0d24de9300d3cdacfb204d17b18a72adc6 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Fri, 24 Jul 2020 18:59:56 +0530
Subject: [PATCH 35/56] chore: revert "feat: make search indexing optional in
 migrate"

This reverts commit b5fe9d4f7be8dae3e3e946b4327bf44b9a35448c.
---
 frappe/commands/site.py | 6 +++---
 frappe/migrate.py       | 6 +++---
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/frappe/commands/site.py b/frappe/commands/site.py
index 54773d6e50..88e90af4dc 100755
--- a/frappe/commands/site.py
+++ b/frappe/commands/site.py
@@ -274,9 +274,9 @@ def disable_user(context, email):
 @click.command('migrate')
 @click.option('--rebuild-website', help="Rebuild webpages after migration")
 @click.option('--skip-failing', is_flag=True, help="Skip patches that fail to run")
-@click.option('--index-site', is_flag=True, help="Index public web pages")
+@click.option('--skip-search-index', is_flag=True, help="Skip patches that fail to run")
 @pass_context
-def migrate(context, rebuild_website=False, skip_failing=False, index_site=False):
+def migrate(context, rebuild_website=False, skip_failing=False, skip_search_index=False):
 	"Run patches, sync schema and rebuild files/translations"
 	from frappe.migrate import migrate
 
@@ -289,7 +289,7 @@ def migrate(context, rebuild_website=False, skip_failing=False, index_site=False
 				context.verbose,
 				rebuild_website=rebuild_website,
 				skip_failing=skip_failing,
-				index_site=index_site
+				skip_search_index=skip_search_index
 			)
 		finally:
 			frappe.destroy()
diff --git a/frappe/migrate.py b/frappe/migrate.py
index 63a87a769d..51840e2506 100644
--- a/frappe/migrate.py
+++ b/frappe/migrate.py
@@ -22,7 +22,7 @@ from frappe.core.doctype.scheduled_job_type.scheduled_job_type import sync_jobs
 from frappe.search.website_search import build_index_for_all_routes
 
 
-def migrate(verbose=True, rebuild_website=False, skip_failing=False, index_site=False):
+def migrate(verbose=True, rebuild_website=False, skip_failing=False, skip_search_index=False):
 	'''Migrate all apps to the latest version, will:
 	- run before migrate hooks
 	- run patches
@@ -89,9 +89,9 @@ Otherwise, check the server logs and ensure that all the required services are r
 				frappe.get_attr(fn)()
 
 		# add static pages to global search
-		if index_site:
+		if not skip_search_index:
 			# Run this last as it updates the current session
-			build_index_for_all_routes()
+			full_text_search.build_index_for_all_routes()
 
 		frappe.db.commit()
 

From 156ca5baf13097a1906638f7945a234022a56e63 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Fri, 24 Jul 2020 19:03:17 +0530
Subject: [PATCH 36/56] fix: update help text

---
 frappe/commands/site.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/commands/site.py b/frappe/commands/site.py
index 88e90af4dc..b72d98c433 100755
--- a/frappe/commands/site.py
+++ b/frappe/commands/site.py
@@ -274,7 +274,7 @@ def disable_user(context, email):
 @click.command('migrate')
 @click.option('--rebuild-website', help="Rebuild webpages after migration")
 @click.option('--skip-failing', is_flag=True, help="Skip patches that fail to run")
-@click.option('--skip-search-index', is_flag=True, help="Skip patches that fail to run")
+@click.option('--skip-search-index', is_flag=True, help="Skip search indexing for web documents")
 @pass_context
 def migrate(context, rebuild_website=False, skip_failing=False, skip_search_index=False):
 	"Run patches, sync schema and rebuild files/translations"

From 6f0c4a4bf18abec7f3d9469a7e85df257e3abbf4 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Mon, 27 Jul 2020 14:01:03 +0530
Subject: [PATCH 37/56] fix: import for build_index_for_all_routes

---
 frappe/migrate.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/frappe/migrate.py b/frappe/migrate.py
index 51840e2506..3af06e8c91 100644
--- a/frappe/migrate.py
+++ b/frappe/migrate.py
@@ -91,7 +91,8 @@ Otherwise, check the server logs and ensure that all the required services are r
 		# add static pages to global search
 		if not skip_search_index:
 			# Run this last as it updates the current session
-			full_text_search.build_index_for_all_routes()
+			print('Building search index for {}'.format(frappe.local.site))
+			build_index_for_all_routes()
 
 		frappe.db.commit()
 

From ef0182cf148170dd5d1dd71367a5637a33e57cba Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Mon, 27 Jul 2020 14:03:45 +0530
Subject: [PATCH 38/56] fix: remove generate theme fn from hooks

* ref - a6eae5665
---
 frappe/hooks.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/frappe/hooks.py b/frappe/hooks.py
index 0850cd6f91..894e72a121 100644
--- a/frappe/hooks.py
+++ b/frappe/hooks.py
@@ -272,9 +272,6 @@ setup_wizard_exception = [
 ]
 
 before_migrate = ['frappe.patches.v11_0.sync_user_permission_doctype_before_migrate.execute']
-after_migrate = [
-	'frappe.website.doctype.website_theme.website_theme.generate_theme_files_if_not_exist'
-]
 
 otp_methods = ['OTP App','Email','SMS']
 user_privacy_documents = [

From a18b81e39db12a3f251719d6c03f949dc3cfce0d Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Mon, 27 Jul 2020 14:07:05 +0530
Subject: [PATCH 39/56] chore: linting fixes

---
 frappe/search/__init__.py              |  2 +-
 frappe/search/full_text_search.py      |  5 ++---
 frappe/search/test_full_text_search.py |  3 +--
 frappe/website/js/website.js           | 12 ++++++------
 4 files changed, 10 insertions(+), 12 deletions(-)

diff --git a/frappe/search/__init__.py b/frappe/search/__init__.py
index 462418ca41..0436775417 100644
--- a/frappe/search/__init__.py
+++ b/frappe/search/__init__.py
@@ -4,7 +4,7 @@
 import frappe
 from frappe.utils import cint
 from frappe.search.website_search import WebsiteSearch
-from frappe.search.website_search import FullTextSearch
+from frappe.search.full_text_search import FullTextSearch
 
 @frappe.whitelist(allow_guest=True)
 def web_search(query, scope=None, limit=20):
diff --git a/frappe/search/full_text_search.py b/frappe/search/full_text_search.py
index a8125ee59e..e073a2cfc1 100644
--- a/frappe/search/full_text_search.py
+++ b/frappe/search/full_text_search.py
@@ -8,7 +8,6 @@ from whoosh.index import create_in, open_dir
 from whoosh.fields import TEXT, ID, Schema
 from whoosh.qparser import MultifieldParser, FieldsPlugin, WildcardPlugin
 from whoosh.query import Prefix
-import os
 
 class FullTextSearch:
 	""" Frappe Wrapper for Whoosh """
@@ -57,7 +56,7 @@ class FullTextSearch:
 			doc_name (str): name of the document to be removed
 		"""
 		ix = open_dir(self.index_path)
-		with ix.searcher() as searcher:
+		with ix.searcher():
 			writer = ix.writer()
 			writer.delete_by_term(self.id, doc_name)
 			writer.commit(optimize=True)
@@ -71,7 +70,7 @@ class FullTextSearch:
 		"""
 		ix = open_dir(self.index_path)
 
-		with ix.searcher() as searcher:
+		with ix.searcher():
 			writer = ix.writer()
 			writer.delete_by_term(self.id, document[self.id])
 			writer.add_document(**document)
diff --git a/frappe/search/test_full_text_search.py b/frappe/search/test_full_text_search.py
index bb0d2cef02..feaa8ca84a 100644
--- a/frappe/search/test_full_text_search.py
+++ b/frappe/search/test_full_text_search.py
@@ -1,8 +1,7 @@
 # Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors
 # MIT License. See license.txt
-import frappe
 import unittest
-from frappe.search import FullTextSearch
+from frappe.search.full_text_search import FullTextSearch
 
 class TestFullTextSearch(unittest.TestCase):
 
diff --git a/frappe/website/js/website.js b/frappe/website/js/website.js
index dfe397db4f..21e1bb93f4 100644
--- a/frappe/website/js/website.js
+++ b/frappe/website/js/website.js
@@ -457,14 +457,14 @@ frappe.setup_search = function (target, search_scope) {
 		}
 	});
 
-	$input.keydown(function(e){
+	$input.keydown(function(e) {
 		// up: 38, down: 40
 		if (e.which == 40) {
 			navigate(0);
 		}
 	});
 
-	$dropdown_menu.keydown(function(e){
+	$dropdown_menu.keydown(function(e) {
 		// up: 38, down: 40
 		if (e.which == 38) {
 			navigate(-1);
@@ -482,7 +482,7 @@ frappe.setup_search = function (target, search_scope) {
 		clear_dropdown();
 	});
 
-	$search_input.click(function(event){
+	$search_input.click(function(event) {
 		event.stopPropagation();
 	});
 
@@ -499,14 +499,14 @@ frappe.setup_search = function (target, search_scope) {
 	}
 
 	function clear_dropdown() {
-		offsetIndex = 0
+		offsetIndex = 0;
 		$dropdown_menu.html('');
 		$dropdown_menu.removeClass('show');
 		dropdownItems = undefined;
-	}
+	};
 
 	// Remove focus state on hover
-	$dropdown_menu.mouseover(function(e){
+	$dropdown_menu.mouseover(function() {
 		dropdownItems.blur();
 	});
 };

From 9c35b7fc884e2ee3246a38dbc027f17daf486417 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Mon, 27 Jul 2020 18:36:16 +0530
Subject: [PATCH 40/56] chore: remove unused variables

---
 frappe/search/website_search.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/frappe/search/website_search.py b/frappe/search/website_search.py
index f8dd2a20da..ca6f0fe56b 100644
--- a/frappe/search/website_search.py
+++ b/frappe/search/website_search.py
@@ -104,11 +104,11 @@ def get_static_pages_from_all_apps():
 			routes_to_index.append(route)
 	return routes_to_index
 
-def update_index_for_path(INDEX_NAME, path):
+def update_index_for_path(path):
 	ws = WebsiteSearch(INDEX_NAME)
 	return ws.update_index_by_name(path)
 
-def remove_document_from_index(INDEX_NAME, path):
+def remove_document_from_index(path):
 	ws = WebsiteSearch(INDEX_NAME)
 	return ws.remove_document_from_index(path)
 

From 68e4509792182f7c39de73d154ad0657384eda54 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Mon, 27 Jul 2020 18:36:27 +0530
Subject: [PATCH 41/56] chore: update comment

---
 frappe/migrate.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/migrate.py b/frappe/migrate.py
index 3af06e8c91..6d64799fdd 100644
--- a/frappe/migrate.py
+++ b/frappe/migrate.py
@@ -88,7 +88,7 @@ Otherwise, check the server logs and ensure that all the required services are r
 			for fn in frappe.get_hooks('after_migrate', app_name=app):
 				frappe.get_attr(fn)()
 
-		# add static pages to global search
+		# build web_routes index
 		if not skip_search_index:
 			# Run this last as it updates the current session
 			print('Building search index for {}'.format(frappe.local.site))

From 1b525f0f869428ae0ba528d0b07ca0bfdb9f6b94 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Mon, 27 Jul 2020 18:39:00 +0530
Subject: [PATCH 42/56] chore: linting fixes

---
 frappe/website/js/website.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/frappe/website/js/website.js b/frappe/website/js/website.js
index 21e1bb93f4..113e04c5b7 100644
--- a/frappe/website/js/website.js
+++ b/frappe/website/js/website.js
@@ -496,14 +496,14 @@ frappe.setup_search = function (target, search_scope) {
 			offsetIndex = dropdownItems.length - 1;
 		$input.off('blur');
 		dropdownItems.eq(offsetIndex).focus();
-	}
+	};
 
 	function clear_dropdown() {
 		offsetIndex = 0;
 		$dropdown_menu.html('');
 		$dropdown_menu.removeClass('show');
 		dropdownItems = undefined;
-	};
+	}
 
 	// Remove focus state on hover
 	$dropdown_menu.mouseover(function() {

From 6e8772b42dc0bd579ffdfe0fd828abe704adf788 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Wed, 29 Jul 2020 21:17:25 +0530
Subject: [PATCH 43/56] feat: don't index blog category

---
 frappe/website/doctype/blog_category/blog_category.json | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/frappe/website/doctype/blog_category/blog_category.json b/frappe/website/doctype/blog_category/blog_category.json
index 975ae73829..b2180047cd 100644
--- a/frappe/website/doctype/blog_category/blog_category.json
+++ b/frappe/website/doctype/blog_category/blog_category.json
@@ -48,10 +48,9 @@
  "has_web_view": 1,
  "icon": "fa fa-tag",
  "idx": 1,
- "index_web_pages_for_search": 1,
  "is_published_field": "published",
  "links": [],
- "modified": "2020-07-21 16:25:18.759447",
+ "modified": "2020-07-29 21:14:47.210446",
  "modified_by": "Administrator",
  "module": "Website",
  "name": "Blog Category",

From 560e9dc8dda1efec2fae5dbc577728d27bd9c746 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Wed, 29 Jul 2020 21:40:24 +0530
Subject: [PATCH 44/56] fix: fetch index_web_pages_for_search from meta

---
 frappe/website/website_generator.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/website/website_generator.py b/frappe/website/website_generator.py
index 94bc4b24be..52ae61cd75 100644
--- a/frappe/website/website_generator.py
+++ b/frappe/website/website_generator.py
@@ -144,7 +144,7 @@ class WebsiteGenerator(Document):
 	# Change the field value in doctype
 	# Override this method to disable indexing
 	def allow_website_search_indexing(self):
-		return self.index_web_pages_for_search
+		return self.meta.index_web_pages_for_search
 
 	def remove_old_route_from_index(self):
 		"""Remove page from the website index if the route has changed."""

From 6da04fcb97a6f6b45b23754f0e6327139583534e Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Thu, 30 Jul 2020 21:31:11 +0530
Subject: [PATCH 45/56] fix: arguments for update_index_for_path

---
 frappe/website/website_generator.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/website/website_generator.py b/frappe/website/website_generator.py
index 52ae61cd75..d02e88c5ba 100644
--- a/frappe/website/website_generator.py
+++ b/frappe/website/website_generator.py
@@ -166,7 +166,7 @@ class WebsiteGenerator(Document):
 			return
 
 		if self.is_website_published():
-			frappe.enqueue(update_index_for_path, index_name="web_routes", path=self.route)
+			frappe.enqueue(update_index_for_path, path=self.route)
 		elif self.route:
 			# If the website is not published
 			remove_document_from_index("web_routes", self.route)

From 788711bdad66d384a454ff8ee86aa68bab24b8bd Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Fri, 31 Jul 2020 13:35:36 +0530
Subject: [PATCH 46/56] fix: arguments for remove_document_from_index

---
 frappe/website/website_generator.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/website/website_generator.py b/frappe/website/website_generator.py
index d02e88c5ba..427a264725 100644
--- a/frappe/website/website_generator.py
+++ b/frappe/website/website_generator.py
@@ -96,7 +96,7 @@ class WebsiteGenerator(Document):
 		self.send_indexing_request('URL_DELETED')
 		# On deleting the doc, remove the page from the web_routes index
 		if self.allow_website_search_indexing():
-			remove_document_from_index("web_routes", self.route)
+			remove_document_from_index(self.route)
 
 	def is_website_published(self):
 		"""Return true if published in website"""

From eab9a27b61e565c3290880d3daf54ac7afb9e4f2 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Thu, 6 Aug 2020 19:14:44 +0530
Subject: [PATCH 47/56] feat: separate create_index function

---
 frappe/search/full_text_search.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/frappe/search/full_text_search.py b/frappe/search/full_text_search.py
index e073a2cfc1..9fb478140f 100644
--- a/frappe/search/full_text_search.py
+++ b/frappe/search/full_text_search.py
@@ -76,11 +76,13 @@ class FullTextSearch:
 			writer.add_document(**document)
 			writer.commit(optimize=True)
 
+	def create_index(self):
+		frappe.create_folder(self.index_path)
+		return create_in(self.index_path, self.schema)
 
 	def build_index(self):
 		"""Build index for all parsed documents"""
-		frappe.create_folder(self.index_path)
-		ix = create_in(self.index_path, self.schema)
+		ix = create_index()
 		writer = ix.writer()
 
 		for document in self.documents:

From 38909db9559d832f2c4391d19bf4ca7c865d2b76 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Thu, 6 Aug 2020 19:14:52 +0530
Subject: [PATCH 48/56] feat: create index if not exists

---
 frappe/search/full_text_search.py | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/frappe/search/full_text_search.py b/frappe/search/full_text_search.py
index 9fb478140f..4dc405a5d2 100644
--- a/frappe/search/full_text_search.py
+++ b/frappe/search/full_text_search.py
@@ -4,7 +4,7 @@
 from __future__ import unicode_literals
 import frappe
 
-from whoosh.index import create_in, open_dir
+from whoosh.index import create_in, open_dir, EmptyIndexError
 from whoosh.fields import TEXT, ID, Schema
 from whoosh.qparser import MultifieldParser, FieldsPlugin, WildcardPlugin
 from whoosh.query import Prefix
@@ -55,7 +55,7 @@ class FullTextSearch:
 			self (object): FullTextSearch Instance
 			doc_name (str): name of the document to be removed
 		"""
-		ix = open_dir(self.index_path)
+		ix = self.get_index()
 		with ix.searcher():
 			writer = ix.writer()
 			writer.delete_by_term(self.id, doc_name)
@@ -68,7 +68,7 @@ class FullTextSearch:
 			self (object): FullTextSearch Instance
 			document (_dict): A dictionary with title, path and content
 		"""
-		ix = open_dir(self.index_path)
+		ix = self.get_index()
 
 		with ix.searcher():
 			writer = ix.writer()
@@ -76,13 +76,19 @@ class FullTextSearch:
 			writer.add_document(**document)
 			writer.commit(optimize=True)
 
+	def get_index(self):
+		try:
+			return open_dir(self.index_path)
+		except EmptyIndexError:
+			return self.create_index()
+
 	def create_index(self):
 		frappe.create_folder(self.index_path)
 		return create_in(self.index_path, self.schema)
 
 	def build_index(self):
 		"""Build index for all parsed documents"""
-		ix = create_index()
+		ix = self.create_index()
 		writer = ix.writer()
 
 		for document in self.documents:
@@ -102,7 +108,7 @@ class FullTextSearch:
 		Returns:
 			[List(_dict)]: Search results
 		"""
-		ix = open_dir(self.index_path)
+		ix = self.get_index()
 
 		results = None
 		out = []

From b76a01f28d36d8d065f140c7e96234a26c1b84cd Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Mon, 10 Aug 2020 12:50:37 +0530
Subject: [PATCH 49/56] refactor: add set admin to finally

---
 frappe/search/website_search.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/frappe/search/website_search.py b/frappe/search/website_search.py
index ca6f0fe56b..de93fea3f5 100644
--- a/frappe/search/website_search.py
+++ b/frappe/search/website_search.py
@@ -56,11 +56,11 @@ class WebsiteSearch(FullTextSearch):
 			text_content = page_content.text if page_content else ""
 			title = soup.title.text.strip() if soup.title else route
 
-			frappe.set_user("Administrator")
-
 			return frappe._dict(title=title, content=text_content, path=route)
 		except Exception:
 			pass
+		finally:
+			frappe.set_user("Administrator")
 
 	def parse_result(self, result):
 		title_highlights = result.highlights("title")

From e1d9909b78da6dd2e15165fdf92b2f2f96b03280 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Mon, 10 Aug 2020 12:50:49 +0530
Subject: [PATCH 50/56] feat: do not update search index in tests

---
 frappe/website/website_generator.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/frappe/website/website_generator.py b/frappe/website/website_generator.py
index 427a264725..c406679e54 100644
--- a/frappe/website/website_generator.py
+++ b/frappe/website/website_generator.py
@@ -148,7 +148,7 @@ class WebsiteGenerator(Document):
 
 	def remove_old_route_from_index(self):
 		"""Remove page from the website index if the route has changed."""
-		if self.allow_website_search_indexing():
+		if self.allow_website_search_indexing() or frappe.flags.in_test:
 			return
 		old_doc = self.get_doc_before_save()
 		# Check if the route is changed
@@ -162,7 +162,7 @@ class WebsiteGenerator(Document):
 			- remove document from index if document is unpublished
 			- update index otherwise
 		"""
-		if not self.allow_website_search_indexing():
+		if not self.allow_website_search_indexing() or frappe.flags.in_test:
 			return
 
 		if self.is_website_published():

From c72a19f696218fd720083a7b13553453bc9242c4 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Mon, 10 Aug 2020 13:52:11 +0530
Subject: [PATCH 51/56] fix: reference errors in test

---
 frappe/search/test_full_text_search.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/frappe/search/test_full_text_search.py b/frappe/search/test_full_text_search.py
index feaa8ca84a..57e908da99 100644
--- a/frappe/search/test_full_text_search.py
+++ b/frappe/search/test_full_text_search.py
@@ -43,13 +43,13 @@ class TestFullTextSearch(unittest.TestCase):
 		self.assertTrue('os/gnu' in res)
 
 	def test_remove_document_from_index(self):
-		self.remove_document_from_index("os/gnu")
+		self.index.remove_document_from_index("os/gnu")
 		res = self.index.search("GNU")
 		self.assertEqual(len(res), 0)
 
 	def test_update_index(self):
 		# Update existing index
-		self.update_index({
+		self.index.update_index({
 			'name': "sw/erpnext",
 			'content': """AwesomeERPNext"""
 		})
@@ -61,7 +61,7 @@ class TestFullTextSearch(unittest.TestCase):
 		self.assertEqual(res[0], "sw/erpnext")
 
 		# Update new doc
-		self.update_index({
+		self.index.update_index({
 			'name': "sw/frappebooks",
 			'content': """DesktopAccounting"""
 		})

From 579f5dfc8bc5150812e244c777cf44f8e35078de Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Mon, 10 Aug 2020 14:23:44 +0530
Subject: [PATCH 52/56] fix: test conditions

---
 frappe/search/test_full_text_search.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/frappe/search/test_full_text_search.py b/frappe/search/test_full_text_search.py
index 57e908da99..daa58265bb 100644
--- a/frappe/search/test_full_text_search.py
+++ b/frappe/search/test_full_text_search.py
@@ -13,13 +13,13 @@ class TestFullTextSearch(unittest.TestCase):
 	def test_search_term(self):
 		# Search Wikipedia
 		res = self.index.search("multilingual online encyclopedia")
-		self.assertEqual(res[0], 'wikipedia')
+		self.assertEqual(res[0], 'site/wikipedia')
 
 		res = self.index.search("Linux is a family")
-		self.assertEqual(res[0], 'linux')
+		self.assertEqual(res[0], 'os/linux')
 
 		res = self.index.search("Enterprise Resource Planning")
-		self.assertEqual(res[0], 'erpnext')
+		self.assertEqual(res[0], 'sw/erpnext')
 
 	def test_search_limit(self):
 		res = self.index.search("CommonSearchTerm")

From 8d71dad60bb8f5eb519fda8fdbb1886582e62a82 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Mon, 10 Aug 2020 14:24:00 +0530
Subject: [PATCH 53/56] feat: delete doc only if valid name is provided

---
 frappe/search/full_text_search.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/frappe/search/full_text_search.py b/frappe/search/full_text_search.py
index 4dc405a5d2..dd6e69111d 100644
--- a/frappe/search/full_text_search.py
+++ b/frappe/search/full_text_search.py
@@ -55,6 +55,9 @@ class FullTextSearch:
 			self (object): FullTextSearch Instance
 			doc_name (str): name of the document to be removed
 		"""
+		if not doc_name:
+			return
+
 		ix = self.get_index()
 		with ix.searcher():
 			writer = ix.writer()

From 24c5fbf743f9adc14c79a661bdb8d19d90c6a8e4 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Mon, 10 Aug 2020 21:34:02 +0530
Subject: [PATCH 54/56] test: fix tests

---
 frappe/search/test_full_text_search.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/search/test_full_text_search.py b/frappe/search/test_full_text_search.py
index daa58265bb..be9669d3c5 100644
--- a/frappe/search/test_full_text_search.py
+++ b/frappe/search/test_full_text_search.py
@@ -15,7 +15,7 @@ class TestFullTextSearch(unittest.TestCase):
 		res = self.index.search("multilingual online encyclopedia")
 		self.assertEqual(res[0], 'site/wikipedia')
 
-		res = self.index.search("Linux is a family")
+		res = self.index.search("Linux kernel")
 		self.assertEqual(res[0], 'os/linux')
 
 		res = self.index.search("Enterprise Resource Planning")

From beaaf16758000bec232ec295ffa0b7548b2a66ca Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Fri, 14 Aug 2020 18:30:27 +0530
Subject: [PATCH 55/56] test: show traceback on smtp error

---
 frappe/email/smtp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/email/smtp.py b/frappe/email/smtp.py
index aa025465e5..2fb9534f61 100644
--- a/frappe/email/smtp.py
+++ b/frappe/email/smtp.py
@@ -244,7 +244,7 @@ class SMTPServer:
 
 		except _socket.error as e:
 			# Invalid mail server -- due to refusing connection
-			frappe.msgprint(_('Invalid Outgoing Mail Server or Port'))
+			frappe.msgprint(_('Invalid Outgoing Mail Server or Port <br> {}').format(frappe.get_traceback()))
 			traceback = sys.exc_info()[2]
 			raise_(frappe.ValidationError, e, traceback)
 

From 88d28d7757049ae8ba7b78734f4eb57e78f538d4 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Fri, 14 Aug 2020 18:47:10 +0530
Subject: [PATCH 56/56] chore(revert): show traceback on smtp error

This reverts commit beaaf16758000bec232ec295ffa0b7548b2a66ca.
---
 frappe/email/smtp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/email/smtp.py b/frappe/email/smtp.py
index 2fb9534f61..aa025465e5 100644
--- a/frappe/email/smtp.py
+++ b/frappe/email/smtp.py
@@ -244,7 +244,7 @@ class SMTPServer:
 
 		except _socket.error as e:
 			# Invalid mail server -- due to refusing connection
-			frappe.msgprint(_('Invalid Outgoing Mail Server or Port <br> {}').format(frappe.get_traceback()))
+			frappe.msgprint(_('Invalid Outgoing Mail Server or Port'))
 			traceback = sys.exc_info()[2]
 			raise_(frappe.ValidationError, e, traceback)