vassili/seitime-frappe
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 4

Merge branch 'develop' into add-total-row-if-filterd

Browse source
This commit is contained in:
Abdeali Chharchhoda 2026-02-08 12:47:42 +05:30
commit 07e894ac62
23 changed files with 37853 additions and 554 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
frappe/desk/desktop.py
View file

@ -467,6 +467,8 @@ def get_workspace_sidebar_items():
pages.append(page)
elif page.for_user == frappe.session.user:
private_pages.append(page)
elif not page.public and not page.for_user:
pages.append(page)
page["label"] = _(page.get("name"))
if not page["app"] and page["module"]:

20
frappe/desk/doctype/workspace/test_workspace.py
View file

@ -28,6 +28,26 @@ class TestWorkspace(IntegrationTestCase):
# else:
# self.assertEqual(len(cards), 1)
def test_role_restricted_non_public_workspace_visible_to_permitted_user(self):
"""Non-public workspace with roles should be visible to users with matching role."""
from frappe.desk.desktop import get_workspace_sidebar_items
workspace = frappe.new_doc("Workspace")
workspace.label = "Role Test Workspace"
workspace.title = "Role Test Workspace"
workspace.category = "Modules"
workspace.public = 0
workspace.module = "Desk"
workspace.append("roles", {"role": "System Manager"})
workspace.insert(ignore_if_duplicate=True)
try:
result = get_workspace_sidebar_items()
workspace_titles = [p.title for p in result["pages"]]
self.assertIn("Role Test Workspace", workspace_titles)
finally:
frappe.db.delete("Workspace", {"name": workspace.name})
def create_module(module_name):
module = frappe.get_doc({"doctype": "Module Def", "module_name": module_name, "app_name": "frappe"})

4
frappe/hooks.py
View file

@ -216,7 +216,6 @@ scheduler_events = {
"frappe.deferred_insert.save_to_db",
"frappe.automation.doctype.reminder.reminder.send_reminders",
"frappe.model.utils.link_count.update_link_count",
"frappe.search.sqlite_search.build_index_if_not_exists",
"frappe.utils.telemetry.pulse.client.send_queued_events",
],
# 10 minutes
@ -227,6 +226,9 @@ scheduler_events = {
"30 * * * *": [],
# Daily but offset by 45 minutes
"45 0 * * *": [],
"0 */3 * * *": [
"frappe.search.sqlite_search.build_index_if_not_exists",
],
},
"all": [
"frappe.email.queue.flush",

10
frappe/locale/it.po
View file

@ -3,7 +3,7 @@ msgstr ""
"Project-Id-Version: frappe\n"
"Report-Msgid-Bugs-To: developers@frappe.io\n"
"POT-Creation-Date: 2026-02-01 09:42+0000\n"
"PO-Revision-Date: 2026-02-02 16:52\n"
"PO-Revision-Date: 2026-02-06 17:55\n"
"Last-Translator: developers@frappe.io\n"
"Language-Team: Italian\n"
"MIME-Version: 1.0\n"
@ -22519,7 +22519,7 @@ msgstr "Ruoli"
#. Label of the roles_permissions_tab (Tab Break) field in DocType 'User'
#: frappe/core/doctype/user/user.json
msgid "Roles & Permissions"
msgstr "Ruoli e Permessi"
msgstr "Ruoli e permessi"
#. Label of the roles (Table) field in DocType 'Role Profile'
#. Label of the roles (Table) field in DocType 'User'
@ -28720,7 +28720,7 @@ msgstr ""
#: frappe/core/doctype/has_role/has_role.py:25
msgid "User '{0}' already has the role '{1}'"
msgstr ""
msgstr "L'utente '{0}' ha già il ruolo '{1}'"
#. Name of a DocType
#: frappe/core/doctype/report/user_activity_report.json
@ -28761,7 +28761,7 @@ msgstr ""
#. Label of the user_details_tab (Tab Break) field in DocType 'User'
#: frappe/core/doctype/user/user.json
msgid "User Details"
msgstr "Dettagli Utente"
msgstr "Dettagli utente"
#. Name of a report
#: frappe/core/report/user_doctype_permissions/user_doctype_permissions.json
@ -28990,7 +28990,7 @@ msgstr ""
#: frappe/permissions.py:171
msgid "User {0} does not have doctype access via role permission for document {1}"
msgstr ""
msgstr "L'utente {0} non ha accesso al doctype tramite autorizzazione di ruolo per il documento {1}"
#: frappe/desk/doctype/workspace/workspace.py:285
msgid "User {0} does not have the permission to create a Workspace."

36677
frappe/locale/mn.po Normal file
View file

File diff suppressed because it is too large Load diff

734
frappe/locale/pt_BR.po
View file

File diff suppressed because it is too large Load diff

3
frappe/model/meta.py
View file

@ -912,11 +912,10 @@ def get_field_precision(df, doc=None, currency=None):
def get_precision_from_currency_format(currency: str) -> int:
"""Get precision from currency format string if applicable."""
from frappe.locale import get_number_format
from frappe.utils.number_format import NumberFormat
use_format_from_currency = frappe.get_system_settings("use_number_format_from_currency")
number_format = get_number_format()
number_format = NumberFormat.from_string(frappe.db.get_default("number_format"))
if use_format_from_currency:
currency_format = frappe.db.get_value("Currency", currency, "number_format", cache=True)
number_format = NumberFormat.from_string(currency_format) if currency_format else number_format

29
frappe/model/naming.py
View file

@ -230,7 +230,22 @@ def set_name_from_naming_options(autoname, doc):
elif _autoname.startswith("format:"):
doc.name = _format_autoname(autoname, doc)
elif "#" in autoname:
doc.name = make_autoname(autoname, doc=doc)
# For Expression naming rule, first replace braced params, then normalize, then process series
# This handles patterns like {full_name}-{description}-.#####
def get_param_value_for_match(match):
param = match.group()
return parse_naming_series([param[1:-1]], doc=doc)
# Replace braced params first
name_with_params = BRACED_PARAMS_PATTERN.sub(get_param_value_for_match, autoname)
# Normalize pattern: convert '-.#####' to '.-.#####' to support both formats
# This handles cases like {fieldname}-.##### (without dot before dash)
# Pattern matches: dash followed by dot followed by one or more hashes, but only if not preceded by a dot
normalized_autoname = re.sub(r"(?<!\.)(-\.#+)", r".\1", name_with_params)
# Process the series
doc.name = make_autoname(normalized_autoname, doc=doc)
def set_naming_from_document_naming_rule(doc):
@ -584,11 +599,19 @@ def _format_autoname(autoname: str, doc):
Independent of remaining string or separators.
Example pattern: 'format:LOG-{MM}-{fieldname1}-{fieldname2}-{#####}'
Supports both patterns:
- {fieldname}.-.##### (with dot before dash)
- {fieldname}-.##### (without dot before dash)
"""
first_colon_index = autoname.find(":")
autoname_value = autoname[first_colon_index + 1 :]
# Normalize pattern: convert '-.#####' to '.-.#####' to support both formats
# This handles cases like {fieldname}-.##### (without dot before dash)
# Pattern matches: dash followed by dot followed by one or more hashes, but only if not preceded by a dot
autoname_value = re.sub(r"(?<!\.)(-\.#+)", r".\1", autoname_value)
def get_param_value_for_match(match):
param = match.group()
return parse_naming_series([param[1:-1]], doc=doc)
@ -596,4 +619,8 @@ def _format_autoname(autoname: str, doc):
# Replace braced params with their parsed value
name = BRACED_PARAMS_PATTERN.sub(get_param_value_for_match, autoname_value)
# If the result still contains unbraced hash patterns (like .#####), process them as naming series
if "#" in name and "{" not in name:
name = make_autoname(name, doc=doc)
return name

52
frappe/public/js/frappe/form/grid_row.js
View file

@ -1239,6 +1239,12 @@ export default class GridRow {
} else if (this.columns_list && this.columns_list.slice(0)[0] === column) {
field.$input.attr("data-first-input", 1);
}
if (df.fieldtype === "Currency") {
this.update_currency_symbol_in_grid_input(field, df);
field.$input.off("input.grid-currency").on("input.grid-currency", () => {
this.update_currency_symbol_in_grid_input(field, df);
});
}
}
this.set_arrow_keys(field);
@ -1565,6 +1571,9 @@ export default class GridRow {
// - after row removals via customize_form.js on links, actions and states child-tables
if (this.doc) field.docname = this.doc.name;
field.refresh();
if (df && df.fieldtype === "Currency") {
this.update_currency_symbol_in_grid_input(field, df);
}
}
// in form
@ -1572,6 +1581,49 @@ export default class GridRow {
this.grid_form.refresh_field(fieldname);
}
}
update_currency_symbol_in_grid_input(field, df) {
if (!field?.$input || !this.grid?.is_editable?.()) return;
const currency = frappe.meta.get_field_currency(df, this.doc);
const symbol = window.get_currency_symbol(currency);
const show_on_right =
cint(frappe.model.get_value(":Currency", currency, "symbol_on_right")) === 1;
let $wrapper = field.$input.parent();
if (!$wrapper.hasClass("grid-currency-input")) {
field.$input.wrap('<div class="grid-currency-input"></div>');
}
$wrapper.toggleClass("grid-currency-symbol-right", show_on_right);
let $prefix = $wrapper.find(".grid-currency-prefix");
let $suffix = $wrapper.find(".grid-currency-suffix");
if (!symbol) {
$prefix.remove();
$suffix.remove();
$wrapper.removeClass("grid-currency-has-value");
return;
}
if (show_on_right) {
if (!$suffix.length) {
$suffix = $('<span class="grid-currency-suffix"></span>').appendTo($wrapper);
}
$suffix.text(symbol);
$prefix.remove();
} else {
if (!$prefix.length) {
$prefix = $('<span class="grid-currency-prefix"></span>').prependTo($wrapper);
}
$prefix.text(symbol);
$suffix.remove();
}
const has_value = /\d/.test(field.$input.val() || "");
$wrapper.toggleClass("grid-currency-has-value", has_value);
}
get_field(fieldname) {
let field = this.on_grid_fields_dict[fieldname];
if (field) {

1
frappe/public/js/frappe/form/sidebar/assign_to.js
View file

@ -5,6 +5,7 @@ frappe.ui.form.AssignTo = class AssignTo {
constructor(opts) {
$.extend(this, opts);
this.btn = this.parent.find(".add-assignment-btn").on("click", () => this.add());
this.parent.find(".add-assignment-label").on("click", () => this.add());
this.btn_wrapper = this.btn.parent();
this.refresh();

13
frappe/public/js/frappe/form/sidebar/share.js
View file

@ -18,12 +18,17 @@ frappe.ui.form.Share = class Share {
this.parent.find(".share-doc-btn").hide();
}
this.parent
.find(".share-doc-btn")
.off("click")
.on("click", () => {
const bind_share_click = ($el) => {
$el.off("click").on("click", () => {
this.frm.share_doc();
});
};
const $share_btn = this.parent.find(".share-doc-btn");
const $share_label = this.parent.find(".share-label");
bind_share_click($share_btn);
bind_share_click($share_label);
this.shares.empty();

1
frappe/public/js/frappe/form/toolbar.js
View file

@ -670,6 +670,7 @@ frappe.ui.form.Toolbar = class Toolbar {
}
can_submit() {
return (
frappe.model.is_submittable(this.frm.doc.doctype) &&
this.get_docstatus() === 0 &&
!this.frm.doc.__islocal &&
!this.frm.doc.__unsaved &&

1
frappe/public/js/frappe/ui/sidebar/sidebar_header.js
View file

@ -176,6 +176,7 @@ frappe.ui.SidebarHeader = class SidebarHeader {
help_dropdown_items = custom_help_links.concat(help_dropdown_items);
navbar_settings.help_dropdown.forEach((element) => {
if (element.hidden) return;
let dropdown_children = {
name: element.name,
label: element.item_label,

11
frappe/public/js/frappe/ui/tags.js
View file

@ -37,6 +37,11 @@ frappe.ui.Tags = class {
me.$input.val("");
};
const activate_input = () => {
this.activate();
this.$input.focus();
};
this.$input.keypress((e) => {
if (e.which == 13 || e.keyCode == 13) {
// Triggers event when <enter> is pressed
@ -55,10 +60,8 @@ frappe.ui.Tags = class {
this.deactivate();
});
this.$placeholder.on("click", () => {
this.activate();
this.$input.focus(); // focus only when clicked
});
this.$placeholder.on("click", activate_input);
this.$ul.find(".tags-label").on("click", activate_input);
}
boot() {

3
frappe/public/js/frappe/utils/utils.js
View file

@ -1564,8 +1564,7 @@ Object.assign(frappe.utils, {
if (item.is_query_report) {
route = "query-report/" + item.name;
} else if (!item.is_query_report && item.report_ref_doctype) {
route =
frappe.router.slug(item.report_ref_doctype) + "/view/report/" + item.name;
route = frappe.router.slug(item.report_ref_doctype) + "/view/report/";
} else {
route = "report/" + item.name;
}

1
frappe/public/js/frappe/views/calendar/calendar.js
View file

@ -261,6 +261,7 @@ frappe.views.Calendar = class Calendar {
hour12: true,
},
firstDay: 1,
eventDisplay: "block",
headerToolbar: {
left: "prev,title,next",
center: "",

44
frappe/public/scss/common/grid.scss
View file

@ -449,6 +449,50 @@
z-index: 1;
}
}
.grid-currency-input {
position: relative;
display: flex;
align-items: center;
width: 100%;
}
.grid-currency-prefix,
.grid-currency-suffix {
position: absolute;
color: var(--text-muted);
pointer-events: none;
z-index: 2;
display: none;
}
.grid-currency-prefix {
left: 8px;
}
.grid-currency-suffix {
right: 8px;
}
// for left aligned currency symbol
.grid-currency-input.grid-currency-has-value:not(.grid-currency-symbol-right) .form-control {
padding-left: calc(4px + 1em);
position: relative;
z-index: 1;
text-align: left;
}
// for right aligned currency symbol
.grid-currency-input.grid-currency-has-value.grid-currency-symbol-right .form-control {
padding-right: calc(4px + 1em);
position: relative;
z-index: 1;
}
.grid-currency-input.grid-currency-has-value .grid-currency-prefix,
.grid-currency-input.grid-currency-has-value .grid-currency-suffix {
display: block;
}
}
@mixin base-grid() {

4
frappe/public/scss/desk/calendar.scss
View file

@ -65,8 +65,8 @@ th.fc-col-header-cell {
border: none !important;
}
.fc-event-main .fc-event-time {
display: none;
.fc-event-main-frame {
flex-direction: column;
}
.fc-time-grid-event {

8
frappe/public/scss/desk/form_sidebar.scss
View file

@ -407,6 +407,14 @@ body[data-route^="Form"] {
}
}
.add-assignment-label,
.tags-label,
.share-label {
&:hover {
cursor: pointer;
}
}
.liked-by-popover {
.popover-body {
min-height: 30px;

1
frappe/public/scss/desk/role_editor.scss
View file

@ -43,6 +43,7 @@ table.user-perm {
text-decoration: underline;
}
}
break-inside: avoid;
}
.module-block-list .checkbox {

684
frappe/search/sqlite_search.py
View file

@ -291,88 +291,294 @@ class SQLiteSearch(ABC):
},
}
def build_index(self):
"""Build the complete search index from scratch using atomic replacement."""
def build_index(self, batch_size=1000, is_continuation=False):
"""Build the search index incrementally with progress tracking.
The job runs until completion or until killed by the queue timeout.
Progress is tracked in the database so builds can resume from where they left off.
"""
if not self.is_search_enabled():
return
# Use temporary database path for atomic replacement
temp_db_path = self._get_db_path(is_temp=True)
# Use temporary database path for atomic replacement (only for new index builds)
temp_db_path = None
original_db_path = self.db_path
# Remove temp file if it exists
if os.path.exists(temp_db_path):
os.unlink(temp_db_path)
# Check if this is a completely fresh build or a continuation
if not is_continuation and not self.index_exists():
# Fresh build - use temporary database for atomic replacement
temp_db_path = self._get_db_path(is_temp=True)
# Temporarily switch to temp database for building
self.db_path = temp_db_path
# Remove temp file if it exists
if os.path.exists(temp_db_path):
os.unlink(temp_db_path)
# Switch to temp database for building
self.db_path = temp_db_path
elif is_continuation:
# Check if we're continuing a fresh build (temp db exists)
potential_temp_path = self._get_db_path(is_temp=True)
if os.path.exists(potential_temp_path):
# Continue with temporary database from fresh build
temp_db_path = potential_temp_path
self.db_path = temp_db_path
print(f"Continuation: Using temporary database {temp_db_path}")
else:
print(f"Continuation: Using regular database {self.db_path}")
# If no temp db exists, we're continuing with regular database (already set)
if temp_db_path:
print(f"Working with temporary database: {self.db_path}")
else:
print(f"Working with regular database: {self.db_path}")
try:
self._update_progress("Setting up search tables", 0, 100, absolute=True)
# Setup tables if needed (for fresh builds or when temp DB was just created)
if not is_continuation or (temp_db_path and not self._tables_exist()):
self._update_progress("Setting up search tables", 0, 100, absolute=True)
self._ensure_fts_table()
# Setup tables in temp database
self._ensure_fts_table()
# Clear existing index data for fresh build
if temp_db_path and not is_continuation:
self._with_connection(lambda cursor: cursor.execute("DELETE FROM search_fts"))
self._update_progress("Fetching records", 20, 100, absolute=True)
# Initialize progress tracking (only for completely fresh builds)
if not is_continuation:
self._initialize_index_progress()
records = self.get_documents()
documents = []
# Get current progress
progress = self._get_index_progress()
self._update_progress("Preparing documents", 30, 100, absolute=True)
# Check if indexing is already complete
if self._is_indexing_complete():
self._update_progress("Search index already complete", 100, 100, absolute=True)
return
total_records = len(records)
for i, doc in enumerate(records):
document = self.prepare_document(doc)
if document:
documents.append(document)
# Process each doctype incrementally
total_doctypes = len(self.doc_configs)
processed_doctypes = 0
# Update progress during document preparation
if i % 100 == 0:
progress = 30 + int((i / total_records) * 20) # 30-50% range
self._update_progress("Preparing documents", progress, 100, absolute=True)
for doctype in self.doc_configs.keys():
doctype_progress = progress.get(doctype, {})
self._update_progress("Indexing documents", 50, 100, absolute=True)
# Skip if doctype is already complete
if doctype_progress.get("is_complete"):
processed_doctypes += 1
continue
self._index_documents(documents)
self._update_progress(
f"Indexing {doctype}",
20 + (processed_doctypes * 60 // total_doctypes),
100,
absolute=True,
)
self._update_progress("Building spell correction vocabulary", 80, 100, absolute=True)
# Process this doctype in batches
last_indexed_modified = doctype_progress.get("last_indexed_modified")
batch_count = 0
# Build vocabulary for spelling correction
self._build_vocabulary(documents)
while True:
# Get batch of documents
docs = self.get_documents_paginated(
doctype, limit=batch_size, last_indexed_modified=last_indexed_modified
)
# Atomic replacement: move temp database to final location
if os.path.exists(original_db_path):
os.unlink(original_db_path)
os.rename(temp_db_path, original_db_path)
if not docs:
# No more documents for this doctype
self._mark_doctype_complete(doctype)
break
# Prepare and index documents
documents = []
for doc in docs:
document = self.prepare_document(doc)
if document:
documents.append(document)
if documents:
self._index_documents(documents)
# Update progress with last processed document's modification time
# Use hardcoded 'modified' field since it's reliable in all Frappe doctypes
last_doc_modified = docs[-1]["modified"]
last_doc_name = docs[-1]["name"]
self._update_index_progress(doctype, last_doc_name, last_doc_modified, len(documents))
last_indexed_modified = last_doc_modified
batch_count += 1
# Show progress based on total document counts across all doctypes
indexed_docs, total_docs = self._get_indexing_progress()
if total_docs > 0:
progress_percent = 20 + (indexed_docs * 60) // total_docs
self._update_progress(
f"Indexing {doctype} {indexed_docs}/{total_docs}",
progress_percent,
100,
absolute=True,
)
processed_doctypes += 1
# Check if all doctypes are indexed before building vocabulary
if not self._is_vocabulary_built_needed():
self._update_progress("All documents indexed, building vocabulary", 80, 100, absolute=True)
# Build vocabulary incrementally
self._build_vocabulary_incremental()
self._mark_vocabulary_built()
# Final atomic replacement if this was a fresh build
if temp_db_path and os.path.exists(temp_db_path):
if os.path.exists(original_db_path):
os.unlink(original_db_path)
os.rename(temp_db_path, original_db_path)
self._update_progress("Search index build complete", 100, 100, absolute=True)
# Print warning summary
self._print_warning_summary()
except Exception:
# Clean up temp file on error
if os.path.exists(temp_db_path):
os.unlink(temp_db_path)
except Exception as e:
# Log the error
frappe.log_error(
title="Search Index Build Error",
message=f"Error during search index build: {e}",
)
raise
finally:
# Restore original database path
self.db_path = original_db_path
if temp_db_path:
self.db_path = original_db_path
def _get_incomplete_count(self, where_clause):
"""Get count of incomplete records from search_index_progress table.
Args:
where_clause: SQL WHERE clause condition (without 'WHERE' keyword)
Returns:
int: Count of matching records, or -1 on error
"""
try:
result = self.sql(
f"""
SELECT COUNT(*) as incomplete_count
FROM search_index_progress
WHERE {where_clause}
""",
read_only=True,
)
return result[0]["incomplete_count"]
except sqlite3.Error:
return -1
def _is_vocabulary_built_needed(self):
"""Check if vocabulary still needs to be built."""
count = self._get_incomplete_count("is_complete = 0")
return count > 0 if count >= 0 else True
def _build_vocabulary_incremental(self):
"""Build vocabulary incrementally from indexed documents."""
import re
word_freq = defaultdict(int)
word_regex = re.compile(r"\w+")
# Get all indexed documents in batches to avoid memory issues
batch_size = 1000
offset = 0
while True:
try:
# Get batch of documents from FTS table
documents = self.sql(
f"""
SELECT title, content
FROM search_fts
LIMIT {batch_size} OFFSET {offset}
""",
read_only=True,
)
if not documents:
break
# Process this batch
for i, doc in enumerate(documents):
# Show progress for large document sets
if (offset + i) % 1000 == 0:
self._update_progress(
f"Processing vocabulary ({offset + i} docs)", 85, 100, absolute=True
)
# Process title and content together
combined_text = " ".join([(doc["title"] or "").lower(), (doc["content"] or "").lower()])
# Extract all words at once
words = word_regex.findall(combined_text)
for word in words:
if len(word) > MIN_WORD_LENGTH - 1 and word.isalpha():
word_freq[word] += 1
offset += batch_size
except sqlite3.Error:
break
# Build vocabulary tables as before
if word_freq:
# Clear existing data
def clear_vocabulary(cursor):
cursor.execute("DELETE FROM search_vocabulary")
cursor.execute("DELETE FROM search_trigrams")
self._with_connection(clear_vocabulary)
# Prepare batch data
vocab_data = []
trigram_data = []
trigram_set = set()
for word, freq in word_freq.items():
vocab_data.append((word, freq, len(word)))
trigrams = self._generate_trigrams(word)
for trigram in trigrams:
trigram_key = (trigram, word)
if trigram_key not in trigram_set:
trigram_set.add(trigram_key)
trigram_data.append(trigram_key)
# Batch insert
def insert_vocabulary(cursor):
cursor.executemany(
"INSERT INTO search_vocabulary (word, frequency, length) VALUES (?, ?, ?)", vocab_data
)
cursor.executemany("INSERT INTO search_trigrams (trigram, word) VALUES (?, ?)", trigram_data)
self._with_connection(insert_vocabulary)
# Status and Validation Methods
def _table_exists(self, table_name):
"""Check if a table exists in the database."""
try:
result = self.sql(
f"SELECT name FROM sqlite_master WHERE type='table' AND name='{table_name}'", read_only=True
)
return bool(result)
except sqlite3.Error:
return False
def index_exists(self):
"""Check if FTS index exists."""
if not os.path.exists(self.db_path):
return False
try:
result = self.sql(
"SELECT name FROM sqlite_master WHERE type='table' AND name='search_fts'", read_only=True
)
return bool(result)
except sqlite3.Error:
return False
return self._table_exists("search_fts")
def drop_index(self):
"""Drop the search index by removing the database file."""
@ -408,6 +614,161 @@ class SQLiteSearch(ABC):
return records
def get_documents_paginated(self, doctype, limit=1000, last_indexed_modified=None):
"""Get records for a specific doctype with pagination support."""
config = self.doc_configs.get(doctype)
if not config:
return []
filters = config.get("filters", {}).copy()
# Ensure 'modified' field is always included for progress tracking
fields = config["fields"].copy()
if "modified" not in fields:
fields.append("modified")
# Build query with proper ordering and pagination
# Order by modified field for reliable resume capability
query = frappe.qb.get_query(
doctype,
fields=fields,
filters=filters,
order_by="creation ASC, name ASC", # Secondary sort by name for consistency
limit=limit,
)
# If resuming from a specific timestamp, filter by modification time
# This is more reliable than name-based filtering for VARCHAR names
if last_indexed_modified:
Table = frappe.qb.DocType(doctype)
query = query.where(Table.modified > last_indexed_modified)
docs = query.run(as_dict=True)
for doc in docs:
doc.doctype = doctype
return docs
def _get_index_progress(self):
"""Get current indexing progress for all doctypes."""
try:
result = self.sql(
"""
SELECT doctype, last_indexed_name, last_indexed_modified,
total_docs, indexed_docs, batch_size, is_complete,
started_at, updated_at, vocabulary_built
FROM search_index_progress
""",
read_only=True,
)
progress = {}
for row in result:
progress[row["doctype"]] = dict(row)
return progress
except sqlite3.Error:
return {}
def _initialize_index_progress(self):
"""Initialize progress tracking for all doctypes."""
def init_progress(cursor):
# Clear existing progress
cursor.execute("DELETE FROM search_index_progress")
# Initialize progress for each doctype
for doctype in self.doc_configs.keys():
# Get total count for this doctype
config = self.doc_configs[doctype]
total_count = frappe.qb.get_query(
doctype, filters=config.get("filters", {}), fields=[{"COUNT": "name", "as": "count"}]
).run(as_dict=True)[0]["count"]
cursor.execute(
"""
INSERT INTO search_index_progress
(doctype, total_docs, indexed_docs, batch_size, is_complete, started_at, updated_at, vocabulary_built, last_indexed_modified)
VALUES (?, ?, 0, 1000, 0, datetime('now'), datetime('now'), 0, 0)
""",
(doctype, total_count),
)
self._with_connection(init_progress)
def _update_index_progress(self, doctype, last_indexed_name, last_indexed_modified, indexed_count):
"""Update progress for a specific doctype."""
def update_progress(cursor):
cursor.execute(
"""
UPDATE search_index_progress
SET last_indexed_name = ?,
last_indexed_modified = ?,
indexed_docs = indexed_docs + ?,
updated_at = datetime('now')
WHERE doctype = ?
""",
(last_indexed_name, last_indexed_modified, indexed_count, doctype),
)
self._with_connection(update_progress)
def _mark_doctype_complete(self, doctype):
"""Mark a doctype as completely indexed."""
def mark_complete(cursor):
cursor.execute(
"""
UPDATE search_index_progress
SET is_complete = 1, updated_at = datetime('now')
WHERE doctype = ?
""",
(doctype,),
)
self._with_connection(mark_complete)
def _mark_vocabulary_built(self):
"""Mark vocabulary as built."""
def mark_built(cursor):
cursor.execute("""
UPDATE search_index_progress
SET vocabulary_built = 1, updated_at = datetime('now')
""")
self._with_connection(mark_built)
def _is_indexing_complete(self):
"""Check if all doctypes are completely indexed and vocabulary is built."""
count = self._get_incomplete_count("is_complete = 0 OR vocabulary_built = 0")
return count == 0 if count >= 0 else False
def _get_indexing_progress(self):
"""Get overall indexing progress across all doctypes."""
try:
result = self.sql(
"""
SELECT SUM(total_docs) as total_docs, SUM(indexed_docs) as indexed_docs
FROM search_index_progress
""",
read_only=True,
)
if result and result[0]:
total_docs = result[0]["total_docs"] or 0
indexed_docs = result[0]["indexed_docs"] or 0
return indexed_docs, total_docs
return 0, 0
except sqlite3.Error:
return 0, 0
def _tables_exist(self):
"""Check if the required tables exist in the current database."""
return self._table_exists("search_fts")
# Private Implementation Methods
def _execute_search_query(self, fts_query, title_only, filters):
@ -522,6 +883,7 @@ class SQLiteSearch(ABC):
ORDER BY bm25_score
LIMIT ?
"""
print(sql)
return self.sql(sql, params, read_only=True)
def _process_search_results(self, raw_results, query):
@ -784,80 +1146,6 @@ class SQLiteSearch(ABC):
similarities.sort(key=lambda x: x[1], reverse=True)
return [word for word, score in similarities[:max_suggestions]]
def _build_vocabulary(self, documents):
"""Build vocabulary and trigram index from documents for spelling correction."""
import re
word_freq = defaultdict(int)
word_regex = re.compile(r"\w+") # Compile regex once for efficiency
# Extract words from all documents in batches
for i, doc in enumerate(documents):
# Show progress for large document sets
if i % 1000 == 0:
progress = 80 + int((i / len(documents)) * 15) # 80-95% range
self._update_progress(
f"Processing vocabulary ({i}/{len(documents)})", progress, 100, absolute=True
)
# Process title and content together for efficiency
combined_text = " ".join(
[(doc.get("title", "") or "").lower(), (doc.get("content", "") or "").lower()]
)
# Extract all words at once with compiled regex
words = word_regex.findall(combined_text)
for word in words:
if len(word) > MIN_WORD_LENGTH - 1 and word.isalpha(): # Filter out short words and non-alpha
word_freq[word] += 1
# Clear existing data in a single transaction
conn = self._get_connection()
try:
cursor = conn.cursor()
cursor.execute("DELETE FROM search_vocabulary")
cursor.execute("DELETE FROM search_trigrams")
conn.commit()
finally:
conn.close()
if not word_freq:
return
# Prepare batch data for vocabulary
vocab_data = []
trigram_data = []
trigram_set = set() # Use set to avoid duplicate trigrams
for word, freq in word_freq.items():
vocab_data.append((word, freq, len(word)))
# Generate trigrams for this word
trigrams = self._generate_trigrams(word)
for trigram in trigrams:
trigram_key = (trigram, word)
if trigram_key not in trigram_set:
trigram_set.add(trigram_key)
trigram_data.append(trigram_key)
# Use batch inserts with a single transaction
conn = self._get_connection()
try:
cursor = conn.cursor()
# Batch insert vocabulary
cursor.executemany(
"INSERT INTO search_vocabulary (word, frequency, length) VALUES (?, ?, ?)", vocab_data
)
# Batch insert trigrams (duplicates already removed)
cursor.executemany("INSERT INTO search_trigrams (trigram, word) VALUES (?, ?)", trigram_data)
conn.commit()
finally:
conn.close()
# Database and Infrastructure Methods
def _get_connection(self, read_only=False):
@ -886,6 +1174,26 @@ class SQLiteSearch(ABC):
if is_read:
cursor.execute("PRAGMA query_only = 1;") # Read-only optimization
def _with_connection(self, callback, read_only=False):
"""Execute a callback with a managed database connection.
Args:
callback: Function that takes (cursor) and performs database operations
read_only: Whether the connection is read-only
Returns:
The return value of the callback, if any
"""
conn = self._get_connection(read_only=read_only)
try:
cursor = conn.cursor()
result = callback(cursor)
if not read_only:
conn.commit()
return result
finally:
conn.close()
def _ensure_fts_table(self):
"""Create FTS table and related tables if they don't exist."""
# Get schema from subclass
@ -893,11 +1201,7 @@ class SQLiteSearch(ABC):
metadata_fields = self.schema["metadata_fields"]
tokenizer = self.schema["tokenizer"]
# Use a single transaction for all table creation operations
conn = self._get_connection()
try:
cursor = conn.cursor()
def create_tables(cursor):
# Create the FTS table with dynamic columns
cursor.execute(f"""
CREATE VIRTUAL TABLE IF NOT EXISTS search_fts USING fts5(
@ -925,14 +1229,34 @@ class SQLiteSearch(ABC):
)
""")
# Create the index progress tracking table
cursor.execute("""
CREATE TABLE IF NOT EXISTS search_index_progress (
id INTEGER PRIMARY KEY,
doctype TEXT,
last_indexed_name TEXT,
last_indexed_modified TEXT,
total_docs INTEGER DEFAULT 0,
indexed_docs INTEGER DEFAULT 0,
batch_size INTEGER DEFAULT 1000,
is_complete BOOLEAN DEFAULT 0,
started_at DATETIME,
updated_at DATETIME,
vocabulary_built BOOLEAN DEFAULT 0
)
""")
# Index for fast trigram lookups
cursor.execute("""
CREATE INDEX IF NOT EXISTS idx_trigram_lookup ON search_trigrams(trigram)
""")
conn.commit()
finally:
conn.close()
# Index for progress tracking lookups
cursor.execute("""
CREATE INDEX IF NOT EXISTS idx_progress_doctype ON search_index_progress(doctype)
""")
self._with_connection(create_tables)
def _index_documents(self, documents):
"""Bulk index documents into SQLite FTS."""
@ -955,10 +1279,8 @@ class SQLiteSearch(ABC):
# Process documents in chunks to prevent memory issues with large datasets
chunk_size = 1000
conn = self._get_connection()
try:
cursor = conn.cursor()
def index_chunks(cursor):
for i in range(0, len(documents), chunk_size):
chunk = documents[i : i + chunk_size]
doc_ids_to_delete = []
@ -994,6 +1316,7 @@ class SQLiteSearch(ABC):
values.append(doc.get(field, ""))
doc_ids_to_delete.append(doc_id)
values_to_insert.append(tuple(values))
# Delete existing rows for these doc_ids first using a single statement
@ -1006,9 +1329,7 @@ class SQLiteSearch(ABC):
if values_to_insert:
cursor.executemany(insert_sql, values_to_insert)
conn.commit()
finally:
conn.close()
self._with_connection(index_chunks)
def index_doc(self, doctype, docname):
"""Index a single document."""
@ -1369,15 +1690,34 @@ class SQLiteSearch(ABC):
def build_index_if_not_exists():
"""Build index if it doesn't exist."""
"""Build index if it doesn't exist or continue if temp DB exists.
Called by scheduler every 3 hours to continue incomplete builds.
"""
search_classes = get_search_classes()
for SearchClass in search_classes:
build_index(SearchClass, force=False)
search = SearchClass()
if not search.is_search_enabled():
continue
# Check if a temp DB exists (incomplete build from previous run)
temp_db_path = search._get_db_path(is_temp=True)
if os.path.exists(temp_db_path):
# Continue the incomplete build
print(f"{SearchClass.__name__}: Found temp DB, continuing build...")
build_index(SearchClass, force=True, is_continuation=True)
elif not search.index_exists():
# No index exists, start fresh build
print(f"{SearchClass.__name__}: No index exists, starting fresh build...")
build_index(SearchClass, force=False)
def build_index(
SearchClass: type[SQLiteSearch] | None = None, search_class_path: str | None = None, force: bool = False
SearchClass: type[SQLiteSearch] | None = None,
search_class_path: str | None = None,
force: bool = False,
is_continuation: bool = False,
):
"""Build search index for SearchClass"""
if not SearchClass and not search_class_path:
@ -1389,29 +1729,69 @@ def build_index(
search = SearchClass()
if not search.is_search_enabled():
return
if not search.index_exists() or force:
print(f"{SearchClass.__name__}: Index does not exist, building...")
search.build_index()
if search.index_exists() and not force:
return
# For continuation jobs, always proceed regardless of existing index
if is_continuation or force:
if is_continuation:
print(f"{SearchClass.__name__}: Continuing incremental index build...")
else:
print(f"{SearchClass.__name__}: Index does not exist or force=True, building...")
search.build_index(is_continuation=is_continuation)
def _enqueue_index_job(search_class_path: str, is_continuation: bool = False):
"""Enqueue a search index build job.
Args:
search_class_path: Full path to the search class (e.g., 'module.ClassName')
is_continuation: Whether this is a continuation of an incomplete build
"""
job_id = f"{search_class_path}_continuation" if is_continuation else search_class_path
job_type = "continuation" if is_continuation else "fresh build"
print(f"Enqueuing {job_type} for {search_class_path}.build_index")
# timeout for 2 hour 10 minutes to account for job queue delays
timeout = 2 * 60 * 60 + 10 * 60
enqueue_kwargs = {
"queue": "long",
"job_id": job_id,
"deduplicate": True,
"search_class_path": search_class_path,
"force": True,
"is_continuation": is_continuation,
"timeout": timeout,
}
frappe.enqueue("frappe.search.sqlite_search.build_index", **enqueue_kwargs)
def build_index_in_background():
"""Enqueue index building in background."""
"""Enqueue index building in background.
Called after migrate to start/continue index building.
"""
search_classes = get_search_classes()
for SearchClass in search_classes:
search = SearchClass()
if not search.is_search_enabled():
return
continue
search_class_path = f"{SearchClass.__module__}.{SearchClass.__name__}"
print(f"Enqueuing {search_class_path}.build_index")
frappe.enqueue(
"frappe.search.sqlite_search.build_index",
queue="long",
job_id=search_class_path,
deduplicate=True,
# build_index args
search_class_path=search_class_path,
force=True,
)
# Check if a temp DB exists (incomplete build from previous run)
temp_db_path = search._get_db_path(is_temp=True)
if os.path.exists(temp_db_path):
# Continue the incomplete build
_enqueue_index_job(search_class_path, is_continuation=True)
elif not search.index_exists():
# No index exists, start fresh build
_enqueue_index_job(search_class_path, is_continuation=False)
else:
print(f"Index for {search_class_path} already exists")
def update_doc_index(doc: Document, method=None):

58
frappe/tests/test_naming.py
View file

@ -148,6 +148,64 @@ class TestNaming(IntegrationTestCase):
self.assertEqual(todo.name, f"TODO-{week}-{series}")
def test_expression_autoname_multiple_fields_pattern_without_dot_before_dash(self):
"""
Test Expression naming rule: {field_1}-{field_2}-.#####
Should produce: field_1-field_2-00001
"""
doctype = new_doctype(
fields=[
{"fieldname": "test_field_a", "fieldtype": "Data", "label": "Test Field A"},
{"fieldname": "test_field_b", "fieldtype": "Data", "label": "Test Field B"},
],
autoname="{test_field_a}-{test_field_b}-.#####",
).insert()
test_field_a = "Sumit"
test_field_b = "Jain"
doc = frappe.new_doc(doctype.name)
doc.test_field_a = test_field_a
doc.test_field_b = test_field_b
doc.insert()
series = getseries(f"{test_field_a}-{test_field_b}-", 5)
series = int(series) - 1
self.assertEqual(doc.name, f"{test_field_a}-{test_field_b}-{series:05d}")
doc.delete()
doctype.delete()
def test_expression_autoname_multiple_fields_pattern_with_dots_between_fields(self):
"""
Test Expression naming rule: {field_1}.-.{field_2}.-.#####
Should produce: field_1-field_2-00001 (same as pattern without dots)
"""
doctype = new_doctype(
fields=[
{"fieldname": "test_field_x", "fieldtype": "Data", "label": "Test Field X"},
{"fieldname": "test_field_y", "fieldtype": "Data", "label": "Test Field Y"},
],
autoname="{test_field_x}.-.{test_field_y}.-.#####",
).insert()
test_field_x = "Sumit"
test_field_y = "Jain"
doc = frappe.new_doc(doctype.name)
doc.test_field_x = test_field_x
doc.test_field_y = test_field_y
doc.insert()
series = getseries(f"{test_field_x}-{test_field_y}-", 5)
series = int(series) - 1
self.assertEqual(doc.name, f"{test_field_x}-{test_field_y}-{series:05d}")
doc.delete()
doctype.delete()
def test_revert_series(self):
from datetime import datetime

46
frappe/tests/test_sqlite_search.py
View file

@ -483,6 +483,29 @@ class TestSQLiteSearchAPI(IntegrationTestCase):
disabled_search.build_index() # Should not raise error but do nothing
self.assertFalse(disabled_search.index_exists())
@patch("frappe.enqueue")
def test_background_operations(self, mock_enqueue):
"""Test background job integration and module-level functions."""
from frappe.search.sqlite_search import build_index_in_background, get_search_classes
# Test getting search classes
with patch("frappe.get_hooks") as mock_get_hooks:
mock_get_hooks.return_value = ["frappe.tests.test_sqlite_search.TestSQLiteSearch"]
classes = get_search_classes()
self.assertEqual(len(classes), 1)
self.assertEqual(classes[0], TestSQLiteSearch)
# Ensure index doesn't exist so build_index_in_background will enqueue a job
self.search.drop_index()
# Test background index building
with patch("frappe.get_hooks") as mock_get_hooks:
mock_get_hooks.return_value = ["frappe.tests.test_sqlite_search.TestSQLiteSearch"]
build_index_in_background()
# Should have enqueued a background job since index doesn't exist
self.assertTrue(mock_enqueue.called)
def test_deduplication_on_reindex(self):
"""Test that re-indexing the same document does not create duplicates."""
self.search.build_index()
@ -546,26 +569,3 @@ class TestSQLiteSearchAPI(IntegrationTestCase):
finally:
test_note.delete()
@patch("frappe.enqueue")
def test_background_operations(self, mock_enqueue):
"""Test background job integration and module-level functions."""
from frappe.search.sqlite_search import (
build_index_in_background,
get_search_classes,
)
# Test getting search classes
with patch("frappe.get_hooks") as mock_get_hooks:
mock_get_hooks.return_value = ["frappe.tests.test_sqlite_search.TestSQLiteSearch"]
classes = get_search_classes()
self.assertEqual(len(classes), 1)
self.assertEqual(classes[0], TestSQLiteSearch)
# Test background index building
with patch("frappe.get_hooks") as mock_get_hooks:
mock_get_hooks.return_value = ["frappe.tests.test_sqlite_search.TestSQLiteSearch"]
build_index_in_background()
# Should have enqueued a background job
self.assertTrue(mock_enqueue.called)