vassili/seitime-frappe
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Merge pull request #10723 from scmmishra/feat-search-api-changes

Browse source 
feat: Search API changes
This commit is contained in:
Shivam Mishra 2020-08-14 14:00:56 +00:00 • committed by GitHub
commit 3de0c18ab2
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
22 changed files with 735 additions and 441 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

29
frappe/commands/site.py
View file

@ -274,8 +274,9 @@ def disable_user(context, email):
@click.command('migrate')
@click.option('--rebuild-website', help="Rebuild webpages after migration")
@click.option('--skip-failing', is_flag=True, help="Skip patches that fail to run")
@click.option('--skip-search-index', is_flag=True, help="Skip search indexing for web documents")
@pass_context
def migrate(context, rebuild_website=False, skip_failing=False):
def migrate(context, rebuild_website=False, skip_failing=False, skip_search_index=False):
"Run patches, sync schema and rebuild files/translations"
from frappe.migrate import migrate
@ -284,7 +285,12 @@ def migrate(context, rebuild_website=False, skip_failing=False):
frappe.init(site=site)
frappe.connect()
try:
migrate(context.verbose, rebuild_website=rebuild_website, skip_failing=skip_failing)
migrate(
context.verbose,
rebuild_website=rebuild_website,
skip_failing=skip_failing,
skip_search_index=skip_search_index
)
finally:
frappe.destroy()
if not context.sites:
@ -655,6 +661,22 @@ def start_ngrok(context):
frappe.destroy()
ngrok.kill()
@click.command('build-search-index')
@pass_context
def build_search_index(context):
from frappe.search.website_search import build_index_for_all_routes
site = get_site(context)
if not site:
raise SiteNotSpecifiedError
print('Building search index for {}'.format(site))
frappe.init(site=site)
frappe.connect()
try:
build_index_for_all_routes()
finally:
frappe.destroy()
commands = [
add_system_manager,
backup,
@ -680,5 +702,6 @@ commands = [
start_recording,
stop_recording,
add_to_hosts,
start_ngrok
start_ngrok,
build_search_index
]

9
frappe/core/doctype/doctype/doctype.json
View file

@ -70,6 +70,7 @@
"web_view",
"has_web_view",
"allow_guest_to_view",
"index_web_pages_for_search",
"route",
"is_published_field",
"advanced",
@ -517,12 +518,18 @@
"fieldname": "email_settings_sb",
"fieldtype": "Section Break",
"label": "Email Settings"
},
{
"default": "1",
"fieldname": "index_web_pages_for_search",
"fieldtype": "Check",
"label": "Index Web Pages for Search"
}
],
"icon": "fa fa-bolt",
"idx": 6,
"links": [],
"modified": "2020-03-27 14:51:44.581128",
"modified": "2020-07-21 16:20:57.028802",
"modified_by": "Administrator",
"module": "Core",
"name": "DocType",

3
frappe/email/doctype/newsletter/newsletter.json
View file

@ -132,10 +132,11 @@
"has_web_view": 1,
"icon": "fa fa-envelope",
"idx": 1,
"index_web_pages_for_search": 1,
"is_published_field": "published",
"links": [],
"max_attachments": 3,
"modified": "2020-05-12 18:09:40.137138",
"modified": "2020-07-21 16:25:17.687476",
"modified_by": "Administrator",
"module": "Email",
"name": "Newsletter",

3
frappe/hooks.py
View file

@ -272,9 +272,6 @@ setup_wizard_exception = [
]
before_migrate = ['frappe.patches.v11_0.sync_user_permission_doctype_before_migrate.execute']
after_migrate = [
'frappe.modules.full_text_search.build_index_for_all_routes'
]
otp_methods = ['OTP App','Email','SMS']
user_privacy_documents = [

13
frappe/migrate.py
View file

@ -19,10 +19,10 @@ from frappe.website import render
from frappe.core.doctype.language.language import sync_languages
from frappe.modules.utils import sync_customizations
from frappe.core.doctype.scheduled_job_type.scheduled_job_type import sync_jobs
from frappe.utils import global_search
from frappe.search.website_search import build_index_for_all_routes
def migrate(verbose=True, rebuild_website=False, skip_failing=False):
def migrate(verbose=True, rebuild_website=False, skip_failing=False, skip_search_index=False):
'''Migrate all apps to the latest version, will:
- run before migrate hooks
- run patches
@ -80,9 +80,6 @@ Otherwise, check the server logs and ensure that all the required services are r
# syncs statics
render.clear_cache()
# add static pages to global search
global_search.update_global_search_for_all_web_pages()
# updating installed applications data
frappe.get_single('Installed Applications').update_versions()
@ -91,6 +88,12 @@ Otherwise, check the server logs and ensure that all the required services are r
for fn in frappe.get_hooks('after_migrate', app_name=app):
frappe.get_attr(fn)()
# build web_routes index
if not skip_search_index:
# Run this last as it updates the current session
print('Building search index for {}'.format(frappe.local.site))
build_index_for_all_routes()
frappe.db.commit()
clear_notifications()

106
frappe/modules/full_text_search.py
View file

@ -1,106 +0,0 @@
# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt
from __future__ import unicode_literals
import frappe
from whoosh.index import create_in, open_dir
from whoosh.fields import TEXT, ID, Schema
from whoosh.qparser import MultifieldParser, FieldsPlugin, WildcardPlugin
from whoosh.query import Prefix
from bs4 import BeautifulSoup
from frappe.website.render import render_page
from frappe.utils import set_request, cint
from frappe.utils.global_search import get_routes_to_index
def build_index_for_all_routes():
print("Building search index for all web routes...")
routes = get_routes_to_index()
documents = [get_document_to_index(route) for route in routes]
build_index("web_routes", documents)
@frappe.whitelist(allow_guest=True)
def web_search(index_name, query, scope=None, limit=20):
limit = cint(limit)
return search(index_name, query, scope, limit)
def get_document_to_index(route):
frappe.set_user("Guest")
frappe.local.no_cache = True
try:
set_request(method="GET", path=route)
content = render_page(route)
soup = BeautifulSoup(content, "html.parser")
page_content = soup.find(class_="page_content")
text_content = page_content.text if page_content else ""
title = soup.title.text.strip() if soup.title else route
frappe.set_user("Administrator")
return frappe._dict(title=title, content=text_content, path=route)
except (
frappe.PermissionError,
frappe.DoesNotExistError,
frappe.ValidationError,
Exception,
):
pass
def build_index(index_name, documents):
schema = Schema(
title=TEXT(stored=True), path=ID(stored=True), content=TEXT(stored=True)
)
index_dir = get_index_path(index_name)
frappe.create_folder(index_dir)
ix = create_in(index_dir, schema)
writer = ix.writer()
for document in documents:
if document:
writer.add_document(
title=document.title, path=document.path, content=document.content
)
writer.commit()
def search(index_name, text, scope=None, limit=20):
index_dir = get_index_path(index_name)
ix = open_dir(index_dir)
results = None
out = []
with ix.searcher() as searcher:
parser = MultifieldParser(["title", "content"], ix.schema)
parser.remove_plugin_class(FieldsPlugin)
parser.remove_plugin_class(WildcardPlugin)
query = parser.parse(text)
filter_scoped = None
if scope:
filter_scoped = Prefix("path", scope)
results = searcher.search(query, limit=limit, filter=filter_scoped)
for r in results:
title_highlights = r.highlights("title")
content_highlights = r.highlights("content")
out.append(
frappe._dict(
title=r["title"],
path=r["path"],
title_highlights=title_highlights,
content_highlights=content_highlights,
)
)
return out
def get_index_path(index_name):
return frappe.get_site_path("indexes", index_name)

35
frappe/public/scss/doc.scss
View file

@ -81,45 +81,10 @@ $navbar-height-lg: 4.5rem;
}
.doc-search {
position: relative;
width: 100%;
@include media-breakpoint-up(lg) {
padding-left: 4rem;
padding-right: 4rem;
}
.search-icon {
position: absolute;
left: 0;
top: 0;
width: 2.5rem;
height: 100%;
display: flex;
justify-content: center;
align-items: center;
}
svg {
color: $gray-600;
}
input {
padding-left: 2.5rem;
}
.dropdown-menu {
.dropdown-item {
padding: 1rem 0.75rem;
}
.match {
background-color: $primary-light;
color: $primary;
font-weight: 500;
padding: 0 0.125rem;
}
}
}
.doc-sidebar {

40
frappe/public/scss/search.scss Normal file
View file

@ -0,0 +1,40 @@
.website-search {
position: relative;
width: 100%;
.search-icon {
position: absolute;
left: 0;
top: 0;
width: 2.5rem;
height: 100%;
display: flex;
justify-content: center;
align-items: center;
}
svg {
color: $gray-600;
}
input {
padding-left: 2.5rem;
}
.dropdown-menu {
.dropdown-item {
padding: 1rem 0.75rem;
&:focus {
background-color: $gray-100;
}
}
.match {
background-color: $primary-light;
color: $primary;
font-weight: 500;
padding: 0 0.125rem;
}
}
}

1
frappe/public/scss/website.scss
View file

@ -10,6 +10,7 @@
@import 'markdown';
@import 'sidebar';
@import 'portal';
@import 'search';
@import 'doc';
.ql-editor.read-mode {

13
frappe/search/__init__.py Normal file
View file

@ -0,0 +1,13 @@
# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt
import frappe
from frappe.utils import cint
from frappe.search.website_search import WebsiteSearch
from frappe.search.full_text_search import FullTextSearch
@frappe.whitelist(allow_guest=True)
def web_search(query, scope=None, limit=20):
limit = cint(limit)
ws = WebsiteSearch(index_name="web_routes")
return ws.search(query, scope, limit)

136
frappe/search/full_text_search.py Normal file
View file

@ -0,0 +1,136 @@
# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt
from __future__ import unicode_literals
import frappe
from whoosh.index import create_in, open_dir, EmptyIndexError
from whoosh.fields import TEXT, ID, Schema
from whoosh.qparser import MultifieldParser, FieldsPlugin, WildcardPlugin
from whoosh.query import Prefix
class FullTextSearch:
""" Frappe Wrapper for Whoosh """
def __init__(self, index_name):
self.index_name = index_name
self.index_path = get_index_path(index_name)
self.schema = self.get_schema()
self.id = self.get_id()
def get_schema(self):
return Schema(name=ID(stored=True), content=TEXT(stored=True))
def get_id(self):
return "name"
def get_items_to_index(self):
"""Get all documents to be indexed conforming to the schema"""
return []
def get_document_to_index(self):
return {}
def build(self):
""" Build search index for all documents """
self.documents = self.get_items_to_index()
self.build_index()
def update_index_by_name(self, doc_name):
"""Wraps `update_index` method, gets the document from name
and updates the index. This function changes the current user
and should only be run as administrator or in a background job.
Args:
self (object): FullTextSearch Instance
doc_name (str): name of the document to be updated
"""
document = self.get_document_to_index(doc_name)
self.update_index(document)
def remove_document_from_index(self, doc_name):
"""Remove document from search index
Args:
self (object): FullTextSearch Instance
doc_name (str): name of the document to be removed
"""
if not doc_name:
return
ix = self.get_index()
with ix.searcher():
writer = ix.writer()
writer.delete_by_term(self.id, doc_name)
writer.commit(optimize=True)
def update_index(self, document):
"""Update search index for a document
Args:
self (object): FullTextSearch Instance
document (_dict): A dictionary with title, path and content
"""
ix = self.get_index()
with ix.searcher():
writer = ix.writer()
writer.delete_by_term(self.id, document[self.id])
writer.add_document(**document)
writer.commit(optimize=True)
def get_index(self):
try:
return open_dir(self.index_path)
except EmptyIndexError:
return self.create_index()
def create_index(self):
frappe.create_folder(self.index_path)
return create_in(self.index_path, self.schema)
def build_index(self):
"""Build index for all parsed documents"""
ix = self.create_index()
writer = ix.writer()
for document in self.documents:
if document:
writer.add_document(**document)
writer.commit(optimize=True)
def search(self, text, scope=None, limit=20):
"""Search from the current index
Args:
text (str): String to search for
scope (str, optional): Scope to limit the search. Defaults to None.
limit (int, optional): Limit number of search results. Defaults to 20.
Returns:
[List(_dict)]: Search results
"""
ix = self.get_index()
results = None
out = []
with ix.searcher() as searcher:
parser = MultifieldParser(["title", "content"], ix.schema)
parser.remove_plugin_class(FieldsPlugin)
parser.remove_plugin_class(WildcardPlugin)
query = parser.parse(text)
filter_scoped = None
if scope:
filter_scoped = Prefix(self.id, scope)
results = searcher.search(query, limit=limit, filter=filter_scoped)
for r in results:
out.append(self.parse_result(r))
return out
def get_index_path(index_name):
return frappe.get_site_path("indexes", index_name)

128
frappe/search/test_full_text_search.py Normal file
View file

@ -0,0 +1,128 @@
# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt
import unittest
from frappe.search.full_text_search import FullTextSearch
class TestFullTextSearch(unittest.TestCase):
def setUp(self):
index = get_index()
index.build()
self.index = index
def test_search_term(self):
# Search Wikipedia
res = self.index.search("multilingual online encyclopedia")
self.assertEqual(res[0], 'site/wikipedia')
res = self.index.search("Linux kernel")
self.assertEqual(res[0], 'os/linux')
res = self.index.search("Enterprise Resource Planning")
self.assertEqual(res[0], 'sw/erpnext')
def test_search_limit(self):
res = self.index.search("CommonSearchTerm")
self.assertEqual(len(res), 5)
res = self.index.search("CommonSearchTerm", limit=3)
self.assertEqual(len(res), 3)
res = self.index.search("CommonSearchTerm", limit=20)
self.assertEqual(len(res), 5)
def test_search_scope(self):
# Search outside scope
res = self.index.search("multilingual online encyclopedia", scope=["os"])
self.assertEqual(len(res), 0)
# Search inside scope
res = self.index.search("CommonSearchTerm", scope=["os"])
self.assertEqual(len(res), 2)
self.assertTrue('os/linux' in res)
self.assertTrue('os/gnu' in res)
def test_remove_document_from_index(self):
self.index.remove_document_from_index("os/gnu")
res = self.index.search("GNU")
self.assertEqual(len(res), 0)
def test_update_index(self):
# Update existing index
self.index.update_index({
'name': "sw/erpnext",
'content': """AwesomeERPNext"""
})
res = self.index.search("CommonSearchTerm")
self.assertTrue('sw/erpnext' not in res)
res = self.index.search("AwesomeERPNext")
self.assertEqual(res[0], "sw/erpnext")
# Update new doc
self.index.update_index({
'name': "sw/frappebooks",
'content': """DesktopAccounting"""
})
res = self.index.search("DesktopAccounting")
self.assertEqual(res[0], "sw/frappebooks")
class TestWrapper(FullTextSearch):
def get_items_to_index(self):
return get_documents()
def get_document_to_index(self, name):
documents = get_documents()
for doc in documents:
if doc["name"] == name:
return doc
def parse_result(self, result):
return result["name"]
def get_index():
return TestWrapper("test_frappe_index")
def get_documents():
docs = []
docs.append({
'name': "site/wikipedia",
'content': """Wikipedia is a multilingual online encyclopedia created and maintained
as an open collaboration project by a community of volunteer editors using a wiki-based editing system.
It is the largest and most popular general reference work on the World Wide Web. CommonSearchTerm"""
})
docs.append({
'name': "os/linux",
'content': """Linux is a family of open source Unix-like operating systems based on the
Linux kernel, an operating system kernel first released on September 17, 1991, by Linus Torvalds.
Linux is typically packaged in a Linux distribution. CommonSearchTerm"""
})
docs.append({
'name': "os/gnu",
'content': """GNU is an operating system and an extensive collection of computer software.
GNU is composed wholly of free software, most of which is licensed under the GNU Project's own
General Public License. GNU is a recursive acronym for "GNU's Not Unix! ",
chosen because GNU's design is Unix-like, but differs from Unix by being free software and containing no Unix code. CommonSearchTerm"""
})
docs.append({
'name': "sw/erpnext",
'content': """ERPNext is a free and open-source integrated Enterprise Resource Planning software developed by
Frappe Technologies Pvt. Ltd. and is built on MariaDB database system using a Python based server-side framework.
ERPNext is a generic ERP software used by manufacturers, distributors and services companies. CommonSearchTerm"""
})
docs.append({
'name': "sw/frappe",
'content': """Frappe Framework is a full-stack web framework, that includes everything you need to build and
deploy business applications with Rich Admin Interface. CommonSearchTerm"""
})
return docs

117
frappe/search/website_search.py Normal file
View file

@ -0,0 +1,117 @@
# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt
from __future__ import unicode_literals
import frappe
from bs4 import BeautifulSoup
from whoosh.fields import TEXT, ID, Schema
from frappe.search.full_text_search import FullTextSearch
from frappe.website.render import render_page
from frappe.utils import set_request
import os
INDEX_NAME = "web_routes"
class WebsiteSearch(FullTextSearch):
""" Wrapper for WebsiteSearch """
def get_schema(self):
return Schema(
title=TEXT(stored=True), path=ID(stored=True), content=TEXT(stored=True)
)
def get_id(self):
return "path"
def get_items_to_index(self):
"""Get all routes to be indexed, this includes the static pages
in www/ and routes from published documents
Returns:
self (object): FullTextSearch Instance
"""
routes = get_static_pages_from_all_apps()
routes += get_doctype_routes_with_web_view()
documents = [self.get_document_to_index(route) for route in routes]
return documents
def get_document_to_index(self, route):
"""Render a page and parse it using BeautifulSoup
Args:
path (str): route of the page to be parsed
Returns:
document (_dict): A dictionary with title, path and content
"""
frappe.set_user("Guest")
frappe.local.no_cache = True
try:
set_request(method="GET", path=route)
content = render_page(route)
soup = BeautifulSoup(content, "html.parser")
page_content = soup.find(class_="page_content")
text_content = page_content.text if page_content else ""
title = soup.title.text.strip() if soup.title else route
return frappe._dict(title=title, content=text_content, path=route)
except Exception:
pass
finally:
frappe.set_user("Administrator")
def parse_result(self, result):
title_highlights = result.highlights("title")
content_highlights = result.highlights("content")
return frappe._dict(
title=result["title"],
path=result["path"],
title_highlights=title_highlights,
content_highlights=content_highlights,
)
def get_doctype_routes_with_web_view():
all_routes = []
filters = { "has_web_view": 1, "allow_guest_to_view": 1, "index_web_pages_for_search": 1}
fields = ["name", "is_published_field"]
doctype_with_web_views = frappe.get_all("DocType", filters=filters, fields=fields)
for doctype in doctype_with_web_views:
if doctype.is_published_field:
routes = frappe.get_all(doctype.name, filters={doctype.is_published_field: 1}, fields="route")
all_routes += [route.route for route in routes]
return all_routes
def get_static_pages_from_all_apps():
from glob import glob
apps = frappe.get_installed_apps()
routes_to_index = []
for app in apps:
path_to_index = frappe.get_app_path(app, 'www')
files_to_index = glob(path_to_index + '/**/*.html', recursive=True)
files_to_index.extend(glob(path_to_index + '/**/*.md', recursive=True))
for file in files_to_index:
route = os.path.relpath(file, path_to_index).split('.')[0]
if route.endswith('index'):
route = route.rsplit('index', 1)[0]
routes_to_index.append(route)
return routes_to_index
def update_index_for_path(path):
ws = WebsiteSearch(INDEX_NAME)
return ws.update_index_by_name(path)
def remove_document_from_index(path):
ws = WebsiteSearch(INDEX_NAME)
return ws.remove_document_from_index(path)
def build_index_for_all_routes():
ws = WebsiteSearch(INDEX_NAME)
return ws.build()

68
frappe/templates/doc.html
View file

@ -22,7 +22,7 @@
</div>
<div class="col-12 col-lg-8">
<div class="doc-search-container">
<div class="doc-search">
<div class="website-search doc-search" id="search-container">
<div class="dropdown">
<div class="search-icon">
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24"
@ -34,7 +34,7 @@
<line x1="21" y1="21" x2="16.65" y2="16.65"></line>
</svg>
</div>
<input type="search" class="form-control" placeholder="Search the docs (Press ? to focus)" />
<input type="search" class="form-control" placeholder="Search the docs (Press / to focus)" />
<div class="overflow-hidden shadow dropdown-menu w-100">
</div>
</div>
@ -117,73 +117,11 @@ id="page-{{ name or route | e }}" data-path="{{ pathname | e }}"
{%- block script -%}
<script>
frappe.ready(() => {
setup_search();
frappe.setup_search('#search-container', '{{ docs_search_scope or "" }}');
$('.web-footer .container')
.removeClass('container')
.addClass('container-fluid doc-container');
});
function setup_search() {
let $dropdown = $('.doc-search .dropdown');
let $dropdown_menu = $('.doc-search .dropdown-menu');
let $input = $('.doc-search input');
$(document).on('keypress', e => {
if (e.key === '/') {
e.preventDefault();
$input.focus();
}
});
$input.on('input', frappe.utils.debounce(() => {
if (!$input.val()) {
clear_dropdown();
return;
}
frappe.call({
method: 'frappe.modules.full_text_search.web_search',
args: {
index_name: 'web_routes',
scope: '{{ docs_search_scope or "" }}' || null,
query: $input.val(),
limit: 5
}
}).then(r => {
let results = r.message || [];
let dropdown_html;
if (results.length == 0) {
dropdown_html = `<div class="dropdown-item">No results found</div>`;
} else {
dropdown_html = results.map(r => {
return `<a class="dropdown-item" href="/${r.path}">
<h6>${r.title_highlights || r.title}</h6>
<div style="white-space: normal;">${r.content_highlights}</div>
</a>`
}).join('')
}
$dropdown_menu.html(dropdown_html);
$dropdown_menu.addClass('show');
});
}, 500));
$input.on('focus', () => {
if (!$input.val()) {
clear_dropdown();
}
});
$input.on('blur', () => {
setTimeout(() => {
clear_dropdown();
}, 300);
});
function clear_dropdown() {
$dropdown_menu.html('');
$dropdown_menu.removeClass('show');
}
}
</script>
{%- endblock -%}

250
frappe/website/doctype/blog_category/blog_category.json
View file

@ -1,196 +1,82 @@
{
"allow_copy": 0,
"allow_guest_to_view": 1,
"allow_import": 1,
"allow_rename": 0,
"autoname": "field:category_name",
"beta": 0,
"creation": "2013-03-08 09:41:11",
"custom": 0,
"docstatus": 0,
"doctype": "DocType",
"document_type": "Setup",
"editable_grid": 0,
"actions": [],
"allow_guest_to_view": 1,
"allow_import": 1,
"autoname": "field:category_name",
"creation": "2013-03-08 09:41:11",
"doctype": "DocType",
"document_type": "Setup",
"engine": "InnoDB",
"field_order": [
"category_name",
"title",
"published",
"route"
],
"fields": [
{
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "category_name",
"fieldtype": "Data",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 1,
"in_standard_filter": 0,
"label": "Category Name",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 1,
"search_index": 0,
"set_only_once": 0,
"unique": 0
},
"fieldname": "category_name",
"fieldtype": "Data",
"in_list_view": 1,
"label": "Category Name",
"reqd": 1,
"unique": 1
},
{
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "title",
"fieldtype": "Data",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 1,
"in_standard_filter": 0,
"label": "Title",
"length": 0,
"no_copy": 1,
"permlevel": 0,
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 1,
"search_index": 0,
"set_only_once": 0,
"unique": 0
},
"fieldname": "title",
"fieldtype": "Data",
"in_list_view": 1,
"label": "Title",
"no_copy": 1,
"reqd": 1
},
{
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "published",
"fieldtype": "Check",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 1,
"in_standard_filter": 0,
"label": "Published",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"unique": 0
},
"default": "0",
"fieldname": "published",
"fieldtype": "Check",
"in_list_view": 1,
"label": "Published"
},
{
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"depends_on": "published",
"fieldname": "route",
"fieldtype": "Data",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Route",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"depends_on": "published",
"fieldname": "route",
"fieldtype": "Data",
"label": "Route",
"unique": 1
}
],
"has_web_view": 1,
"hide_heading": 0,
"hide_toolbar": 0,
"icon": "fa fa-tag",
"idx": 1,
"image_view": 0,
"in_create": 0,
"is_published_field": "published",
"is_submittable": 0,
"issingle": 0,
"istable": 0,
"max_attachments": 0,
"modified": "2017-03-06 16:29:05.035486",
"modified_by": "Administrator",
"module": "Website",
"name": "Blog Category",
"owner": "Administrator",
],
"has_web_view": 1,
"icon": "fa fa-tag",
"idx": 1,
"is_published_field": "published",
"links": [],
"modified": "2020-07-29 21:14:47.210446",
"modified_by": "Administrator",
"module": "Website",
"name": "Blog Category",
"owner": "Administrator",
"permissions": [
{
"amend": 0,
"apply_user_permissions": 0,
"cancel": 0,
"create": 1,
"delete": 1,
"email": 1,
"export": 0,
"if_owner": 0,
"import": 0,
"permlevel": 0,
"print": 1,
"read": 1,
"report": 1,
"role": "Website Manager",
"set_user_permissions": 1,
"share": 1,
"submit": 0,
"create": 1,
"delete": 1,
"email": 1,
"print": 1,
"read": 1,
"report": 1,
"role": "Website Manager",
"set_user_permissions": 1,
"share": 1,
"write": 1
},
},
{
"amend": 0,
"apply_user_permissions": 0,
"cancel": 0,
"create": 0,
"delete": 0,
"email": 1,
"export": 0,
"if_owner": 0,
"import": 0,
"permlevel": 0,
"print": 1,
"read": 1,
"report": 0,
"role": "Blogger",
"set_user_permissions": 0,
"share": 0,
"submit": 0,
"write": 0
"email": 1,
"print": 1,
"read": 1,
"role": "Blogger"
}
],
"quick_entry": 1,
"read_only": 0,
"read_only_onload": 0,
"show_name_in_global_search": 0,
"sort_order": "DESC",
"track_changes": 1,
"track_seen": 0
],
"quick_entry": 1,
"sort_field": "modified",
"sort_order": "DESC",
"track_changes": 1
}

3
frappe/website/doctype/blog_post/blog_post.json
View file

@ -189,10 +189,11 @@
"has_web_view": 1,
"icon": "fa fa-quote-left",
"idx": 1,
"index_web_pages_for_search": 1,
"is_published_field": "published",
"links": [],
"max_attachments": 5,
"modified": "2020-06-01 13:37:57.465434",
"modified": "2020-07-21 16:25:17.154911",
"modified_by": "Administrator",
"module": "Website",
"name": "Blog Post",

39
frappe/website/doctype/help_article/help_article.json
View file

@ -4,6 +4,7 @@
"allow_import": 1,
"creation": "2014-10-30 14:25:53.780105",
"doctype": "DocType",
"engine": "InnoDB",
"field_order": [
"title",
"category",
@ -15,11 +16,7 @@
"content",
"likes",
"route",
"owner",
"feedback",
"helpful",
"cb_00",
"not_helpful"
"owner"
],
"fields": [
{
@ -27,8 +24,7 @@
"fieldtype": "Data",
"in_global_search": 1,
"label": "Title",
"reqd": 1,
"search_index": 1
"reqd": 1
},
{
"fieldname": "category",
@ -90,39 +86,14 @@
"fieldtype": "Link",
"label": "Owner",
"options": "User"
},
{
"collapsible": 1,
"fieldname": "feedback",
"fieldtype": "Section Break",
"label": "Feedback"
},
{
"default": "0",
"fieldname": "helpful",
"fieldtype": "Int",
"in_list_view": 1,
"label": "Helpful",
"read_only": 1
},
{
"fieldname": "cb_00",
"fieldtype": "Column Break"
},
{
"default": "0",
"fieldname": "not_helpful",
"fieldtype": "Data",
"in_list_view": 1,
"label": "Not Helpful",
"read_only": 1
}
],
"has_web_view": 1,
"icon": "icon-file-alt",
"index_web_pages_for_search": 1,
"is_published_field": "published",
"links": [],
"modified": "2020-05-08 10:48:19.997789",
"modified": "2020-07-21 16:25:18.577325",
"modified_by": "Administrator",
"module": "Website",
"name": "Help Article",

2
frappe/website/doctype/web_form/web_form.json
View file

@ -359,7 +359,7 @@
"icon": "icon-edit",
"is_published_field": "published",
"links": [],
"modified": "2020-06-30 21:49:18.237443",
"modified": "2020-07-21 16:25:37.028459",
"modified_by": "Administrator",
"module": "Website",
"name": "Web Form",

4
frappe/website/doctype/web_form/web_form.py
View file

@ -343,6 +343,9 @@ def get_context(context):
frappe.throw(_('Mandatory Information missing:') + '<br><br>'
+ '<br>'.join(['{0} ({1})'.format(d.label, d.fieldtype) for d in missing]))
def allow_website_search_indexing(self):
return False
def has_web_form_permission(self, doctype, name, ptype='read'):
if frappe.session.user=="Guest":
return False
@ -364,7 +367,6 @@ def get_context(context):
return False
@frappe.whitelist(allow_guest=True)
def accept(web_form, data, docname=None, for_payment=False):
'''Save the web form'''

3
frappe/website/doctype/web_page/web_page.json
View file

@ -288,10 +288,11 @@
"has_web_view": 1,
"icon": "fa fa-file-alt",
"idx": 1,
"index_web_pages_for_search": 1,
"is_published_field": "published",
"links": [],
"max_attachments": 20,
"modified": "2020-04-25 20:40:39.253548",
"modified": "2020-07-21 16:25:17.899069",
"modified_by": "Administrator",
"module": "Website",
"name": "Web Page",

132
frappe/website/js/website.js
View file

@ -380,9 +380,139 @@ $.extend(frappe, {
}
});
frappe.setup_search = function (target, search_scope) {
if (typeof target === "string") {
target = $(target);
}
let $search_input = $(`<div class="dropdown" id="dropdownMenuSearch">
<div class="search-icon">
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24"
fill="none"
stroke="currentColor" stroke-width="2" stroke-linecap="round"
stroke-linejoin="round"
class="feather feather-search">
<circle cx="11" cy="11" r="8"></circle>
<line x1="21" y1="21" x2="16.65" y2="16.65"></line>
</svg>
</div>
<input type="search" class="form-control" placeholder="Search the docs (Press / to focus)" />
<div class="overflow-hidden shadow dropdown-menu w-100" aria-labelledby="dropdownMenuSearch">
</div>
</div>`);
target.empty();
$search_input.appendTo(target);
// let $dropdown = $search_input.find('.dropdown');
let $dropdown_menu = $search_input.find('.dropdown-menu');
let $input = $search_input.find('input');
let dropdownItems;
let offsetIndex = 0;
$(document).on('keypress', e => {
if (e.key === '/') {
e.preventDefault();
$input.focus();
}
});
$input.on('input', frappe.utils.debounce(() => {
if (!$input.val()) {
clear_dropdown();
return;
}
frappe.call({
method: 'frappe.search.web_search',
args: {
scope: search_scope || null,
query: $input.val(),
limit: 5
}
}).then(r => {
let results = r.message || [];
let dropdown_html;
if (results.length == 0) {
dropdown_html = `<div class="dropdown-item">No results found</div>`;
} else {
dropdown_html = results.map(r => {
return `<a class="dropdown-item" href="/${r.path}">
<h6>${r.title_highlights || r.title}</h6>
<div style="white-space: normal;">${r.content_highlights}</div>
</a>`;
}).join('');
}
$dropdown_menu.html(dropdown_html);
$dropdown_menu.addClass('show');
dropdownItems = $dropdown_menu.find(".dropdown-item");
});
}, 500));
$input.on('focus', () => {
if (!$input.val()) {
clear_dropdown();
} else {
$input.trigger('input');
}
});
$input.keydown(function(e) {
// up: 38, down: 40
if (e.which == 40) {
navigate(0);
}
});
$dropdown_menu.keydown(function(e) {
// up: 38, down: 40
if (e.which == 38) {
navigate(-1);
} else if (e.which == 40) {
navigate(1);
} else if (e.which == 27) {
setTimeout(() => {
clear_dropdown();
}, 300);
}
});
// Clear dropdown when clicked
$(window).click(function() {
clear_dropdown();
});
$search_input.click(function(event) {
event.stopPropagation();
});
// Navigate the list
var navigate = function(diff) {
offsetIndex += diff;
if (offsetIndex >= dropdownItems.length)
offsetIndex = 0;
if (offsetIndex < 0)
offsetIndex = dropdownItems.length - 1;
$input.off('blur');
dropdownItems.eq(offsetIndex).focus();
};
function clear_dropdown() {
offsetIndex = 0;
$dropdown_menu.html('');
$dropdown_menu.removeClass('show');
dropdownItems = undefined;
}
// Remove focus state on hover
$dropdown_menu.mouseover(function() {
dropdownItems.blur();
});
};
// Utility functions
window.valid_email = function(id) {
// eslint-disable-next-line
// copied regex from frappe/utils.js validate_type

42
frappe/website/website_generator.py
View file

@ -7,6 +7,7 @@ from frappe.model.document import Document
from frappe.website.utils import cleanup_page_name
from frappe.website.render import clear_cache
from frappe.modules import get_module_name
from frappe.search.website_search import update_index_for_path, remove_document_from_index
class WebsiteGenerator(Document):
website = frappe._dict()
@ -83,10 +84,19 @@ class WebsiteGenerator(Document):
def on_update(self):
self.send_indexing_request()
self.remove_old_route_from_index()
def on_change(self):
# Update the index on change
# On change is triggered last in the event lifecycle
self.update_website_search_index()
def on_trash(self):
self.clear_cache()
self.send_indexing_request('URL_DELETED')
# On deleting the doc, remove the page from the web_routes index
if self.allow_website_search_indexing():
remove_document_from_index(self.route)
def is_website_published(self):
"""Return true if published in website"""
@ -129,4 +139,34 @@ class WebsiteGenerator(Document):
url = frappe.utils.get_url(self.route)
frappe.enqueue('frappe.website.doctype.website_settings.google_indexing.publish_site', \
url=url, operation_type=operation_type)
url=url, operation_type=operation_type)
# Change the field value in doctype
# Override this method to disable indexing
def allow_website_search_indexing(self):
return self.meta.index_web_pages_for_search
def remove_old_route_from_index(self):
"""Remove page from the website index if the route has changed."""
if self.allow_website_search_indexing() or frappe.flags.in_test:
return
old_doc = self.get_doc_before_save()
# Check if the route is changed
if old_doc and old_doc.route != self.route:
# Remove the route from index if the route has changed
remove_document_from_index("web_routes", old_doc.route)
def update_website_search_index(self):
"""
Update the full test index executed on document change event.
- remove document from index if document is unpublished
- update index otherwise
"""
if not self.allow_website_search_indexing() or frappe.flags.in_test:
return
if self.is_website_published():
frappe.enqueue(update_index_for_path, path=self.route)
elif self.route:
# If the website is not published
remove_document_from_index("web_routes", self.route)