diff --git a/.github/workflows/docs-checker.yml b/.github/workflows/docs-checker.yml index 90453cd1b4..02a01bf4e4 100644 --- a/.github/workflows/docs-checker.yml +++ b/.github/workflows/docs-checker.yml @@ -12,7 +12,7 @@ jobs: - name: 'Setup Environment' uses: actions/setup-python@v2 with: - python-version: 3.6 + python-version: 3.7 - name: 'Clone repo' uses: actions/checkout@v2 diff --git a/.github/workflows/publish-assets-develop.yml b/.github/workflows/publish-assets-develop.yml index a23885b508..85f3f7c3b0 100644 --- a/.github/workflows/publish-assets-develop.yml +++ b/.github/workflows/publish-assets-develop.yml @@ -18,7 +18,7 @@ jobs: node-version: 14 - uses: actions/setup-python@v2 with: - python-version: '3.6' + python-version: '3.7' - name: Set up bench and build assets run: | npm install -g yarn diff --git a/.github/workflows/publish-assets-releases.yml b/.github/workflows/publish-assets-releases.yml index a697517c23..a5cc1f8872 100644 --- a/.github/workflows/publish-assets-releases.yml +++ b/.github/workflows/publish-assets-releases.yml @@ -21,7 +21,7 @@ jobs: python-version: '12.x' - uses: actions/setup-python@v2 with: - python-version: '3.6' + python-version: '3.7' - name: Set up bench and build assets run: | npm install -g yarn diff --git a/cypress/integration/multi_select_dialog.js b/cypress/integration/multi_select_dialog.js new file mode 100644 index 0000000000..a45fba8d32 --- /dev/null +++ b/cypress/integration/multi_select_dialog.js @@ -0,0 +1,58 @@ +context('MultiSelectDialog', () => { + before(() => { + cy.login(); + cy.visit('/app'); + }); + + function open_multi_select_dialog() { + cy.window().its('frappe').then(frappe => { + new frappe.ui.form.MultiSelectDialog({ + doctype: "Assignment Rule", + target: {}, + setters: { + document_type: null, + priority: null + }, + add_filters_group: 1, + allow_child_item_selection: 1, + child_fieldname: "assignment_days", + child_columns: ["day"] + }); + }); + } + + it('multi select dialog api works', () => { + open_multi_select_dialog(); + cy.get_open_dialog().should('contain', 'Select Assignment Rules'); + }); + + it('checks for filters', () => { + ['search_term', 'document_type', 'priority'].forEach(fieldname => { + cy.get_open_dialog().get(`.frappe-control[data-fieldname="${fieldname}"]`).should('exist'); + }); + + // add_filters_group: 1 should add a filter group + cy.get_open_dialog().get(`.frappe-control[data-fieldname="filter_area"]`).should('exist'); + + }); + + it('checks for child item selection', () => { + cy.get_open_dialog() + .get(`.dt-row-header`).should('not.exist'); + + cy.get_open_dialog() + .get(`.frappe-control[data-fieldname="allow_child_item_selection"]`) + .should('exist') + .click(); + + cy.get_open_dialog() + .get(`.frappe-control[data-fieldname="child_selection_area"]`) + .should('exist'); + + cy.get_open_dialog() + .get(`.dt-row-header`).should('contain', 'Assignment Rule'); + + cy.get_open_dialog() + .get(`.dt-row-header`).should('contain', 'Day'); + }); +}); \ No newline at end of file diff --git a/cypress/integration/navigation.js b/cypress/integration/navigation.js index 7e1426aa46..ba45137cbd 100644 --- a/cypress/integration/navigation.js +++ b/cypress/integration/navigation.js @@ -1,7 +1,6 @@ context('Navigation', () => { before(() => { cy.login(); - cy.visit('/app/website'); }); it('Navigate to route with hash in document name', () => { cy.insert_doc('ToDo', {'__newname': 'ABC#123', 'description': 'Test this', 'ignore_duplicate': true}); @@ -11,4 +10,15 @@ context('Navigation', () => { cy.go('back'); cy.title().should('eq', 'Website'); }); + + it.only('Navigate to previous page after login', () => { + cy.visit('/app/todo'); + cy.request('/api/method/logout'); + cy.reload(); + cy.get('.btn-primary').contains('Login').click(); + cy.location('pathname').should('eq', '/login'); + cy.login(); + cy.visit('/app'); + cy.location('pathname').should('eq', '/app/todo'); + }); }); diff --git a/frappe/commands/__init__.py b/frappe/commands/__init__.py index 6eccdac4fb..6fb33a51b7 100644 --- a/frappe/commands/__init__.py +++ b/frappe/commands/__init__.py @@ -104,7 +104,22 @@ def get_commands(): from .utils import commands as utils_commands from .redis import commands as redis_commands - all_commands = scheduler_commands + site_commands + translate_commands + utils_commands + redis_commands - return list(set(all_commands)) + clickable_link = ( + "\x1b]8;;https://frappeframework.com/docs\afrappeframework.com\x1b]8;;\a" + ) + all_commands = ( + scheduler_commands + + site_commands + + translate_commands + + utils_commands + + redis_commands + ) + + for command in all_commands: + if not command.help: + command.help = f"Refer to {clickable_link}" + + return all_commands + commands = get_commands() diff --git a/frappe/commands/site.py b/frappe/commands/site.py index 9098e31738..2bd3110481 100755 --- a/frappe/commands/site.py +++ b/frappe/commands/site.py @@ -67,6 +67,9 @@ def restore(context, sql_file_path, mariadb_root_username=None, mariadb_root_pas validate_database_sql ) + site = get_site(context) + frappe.init(site=site) + force = context.force or force decompressed_file_name = extract_sql_from_archive(sql_file_path) @@ -85,9 +88,6 @@ def restore(context, sql_file_path, mariadb_root_username=None, mariadb_root_pas # check if valid SQL file validate_database_sql(decompressed_file_name, _raise=not force) - site = get_site(context) - frappe.init(site=site) - # dont allow downgrading to older versions of frappe without force if not force and is_downgrade(decompressed_file_name, verbose=True): warn_message = ( @@ -474,7 +474,7 @@ def remove_from_installed_apps(context, app): @click.command('uninstall-app') @click.argument('app') -@click.option('--yes', '-y', help='To bypass confirmation prompt for uninstalling the app', is_flag=True, default=False, multiple=True) +@click.option('--yes', '-y', help='To bypass confirmation prompt for uninstalling the app', is_flag=True, default=False) @click.option('--dry-run', help='List all doctypes that will be deleted', is_flag=True, default=False) @click.option('--no-backup', help='Do not backup the site', is_flag=True, default=False) @click.option('--force', help='Force remove app from site', is_flag=True, default=False) @@ -738,6 +738,131 @@ def build_search_index(context): finally: frappe.destroy() +@click.command('trim-database') +@click.option('--dry-run', is_flag=True, default=False, help='Show what would be deleted') +@click.option('--format', '-f', default='text', type=click.Choice(['json', 'text']), help='Output format') +@click.option('--no-backup', is_flag=True, default=False, help='Do not backup the site') +@pass_context +def trim_database(context, dry_run, format, no_backup): + if not context.sites: + raise SiteNotSpecifiedError + + from frappe.utils.backups import scheduled_backup + + ALL_DATA = {} + + for site in context.sites: + frappe.init(site=site) + frappe.connect() + + TABLES_TO_DROP = [] + STANDARD_TABLES = get_standard_tables() + information_schema = frappe.qb.Schema("information_schema") + table_name = frappe.qb.Field("table_name").as_("name") + + queried_result = frappe.qb.from_( + information_schema.tables + ).select(table_name).where( + information_schema.tables.table_schema == frappe.conf.db_name + ).run() + + database_tables = [x[0] for x in queried_result] + doctype_tables = frappe.get_all("DocType", pluck="name") + + for x in database_tables: + doctype = x.lstrip("tab") + if not (doctype in doctype_tables or x.startswith("__") or x in STANDARD_TABLES): + TABLES_TO_DROP.append(x) + + if not TABLES_TO_DROP: + if format == "text": + click.secho(f"No ghost tables found in {frappe.local.site}...Great!", fg="green") + else: + if not (no_backup or dry_run): + if format == "text": + print(f"Backing Up Tables: {', '.join(TABLES_TO_DROP)}") + + odb = scheduled_backup( + ignore_conf=False, + include_doctypes=",".join(x.lstrip("tab") for x in TABLES_TO_DROP), + ignore_files=True, + force=True, + ) + if format == "text": + odb.print_summary() + print("\nTrimming Database") + + for table in TABLES_TO_DROP: + if format == "text": + print(f"* Dropping Table '{table}'...") + if not dry_run: + frappe.db.sql_ddl(f"drop table `{table}`") + + ALL_DATA[frappe.local.site] = TABLES_TO_DROP + frappe.destroy() + + if format == "json": + import json + print(json.dumps(ALL_DATA, indent=1)) + + +def get_standard_tables(): + import re + + tables = [] + sql_file = os.path.join( + "..", "apps", "frappe", "frappe", "database", frappe.conf.db_type, f'framework_{frappe.conf.db_type}.sql' + ) + content = open(sql_file).read().splitlines() + + for line in content: + table_found = re.search(r"""CREATE TABLE ("|`)(.*)?("|`) \(""", line) + if table_found: + tables.append(table_found.group(2)) + + return tables + +@click.command('trim-tables') +@click.option('--dry-run', is_flag=True, default=False, help='Show what would be deleted') +@click.option('--format', '-f', default='table', type=click.Choice(['json', 'table']), help='Output format') +@click.option('--no-backup', is_flag=True, default=False, help='Do not backup the site') +@pass_context +def trim_tables(context, dry_run, format, no_backup): + if not context.sites: + raise SiteNotSpecifiedError + + from frappe.model.meta import trim_tables + from frappe.utils.backups import scheduled_backup + + for site in context.sites: + frappe.init(site=site) + frappe.connect() + + if not (no_backup or dry_run): + click.secho(f"Taking backup for {frappe.local.site}", fg="green") + odb = scheduled_backup(ignore_files=False, force=True) + odb.print_summary() + + try: + trimmed_data = trim_tables(dry_run=dry_run, quiet=format == 'json') + + if format == 'table' and not dry_run: + click.secho(f"The following data have been removed from {frappe.local.site}", fg='green') + + handle_data(trimmed_data, format=format) + finally: + frappe.destroy() + +def handle_data(data: dict, format='json'): + if format == 'json': + import json + print(json.dumps({frappe.local.site: data}, indent=1, sort_keys=True)) + else: + from frappe.utils.commands import render_table + data = [["DocType", "Fields"]] + [[table, ", ".join(columns)] for table, columns in data.items()] + render_table(data) + + commands = [ add_system_manager, backup, @@ -766,5 +891,7 @@ commands = [ add_to_hosts, start_ngrok, build_search_index, - partial_restore + partial_restore, + trim_tables, + trim_database, ] diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py index b0151106db..90cd60c6ec 100644 --- a/frappe/commands/utils.py +++ b/frappe/commands/utils.py @@ -408,20 +408,47 @@ def bulk_rename(context, doctype, path): frappe.destroy() +@click.command('db-console') +@pass_context +def database(context): + """ + Enter into the Database console for given site. + """ + site = get_site(context) + if not site: + raise SiteNotSpecifiedError + frappe.init(site=site) + if not frappe.conf.db_type or frappe.conf.db_type == "mariadb": + _mariadb() + elif frappe.conf.db_type == "postgres": + _psql() + + @click.command('mariadb') @pass_context def mariadb(context): """ Enter into mariadb console for a given site. """ - import os - site = get_site(context) if not site: raise SiteNotSpecifiedError frappe.init(site=site) + _mariadb() - # This is assuming you're within the bench instance. + +@click.command('postgres') +@pass_context +def postgres(context): + """ + Enter into postgres console for a given site. + """ + site = get_site(context) + frappe.init(site=site) + _psql() + + +def _mariadb(): mysql = find_executable('mysql') os.execv(mysql, [ mysql, @@ -434,15 +461,7 @@ def mariadb(context): "-A"]) -@click.command('postgres') -@pass_context -def postgres(context): - """ - Enter into postgres console for a given site. - """ - site = get_site(context) - frappe.init(site=site) - # This is assuming you're within the bench instance. +def _psql(): psql = find_executable('psql') subprocess.run([ psql, '-d', frappe.conf.db_name]) @@ -525,6 +544,74 @@ def console(context, autoreload=False): terminal() +@click.command('transform-database', help="Change tables' internal settings changing engine and row formats") +@click.option('--table', required=True, help="Comma separated name of tables to convert. To convert all tables, pass 'all'") +@click.option('--engine', default=None, type=click.Choice(["InnoDB", "MyISAM"]), help="Choice of storage engine for said table(s)") +@click.option('--row_format', default=None, type=click.Choice(["DYNAMIC", "COMPACT", "REDUNDANT", "COMPRESSED"]), help="Set ROW_FORMAT parameter for said table(s)") +@click.option('--failfast', is_flag=True, default=False, help="Exit on first failure occurred") +@pass_context +def transform_database(context, table, engine, row_format, failfast): + "Transform site database through given parameters" + site = get_site(context) + check_table = [] + add_line = False + skipped = 0 + frappe.init(site=site) + + if frappe.conf.db_type and frappe.conf.db_type != "mariadb": + click.secho("This command only has support for MariaDB databases at this point", fg="yellow") + sys.exit(1) + + if not (engine or row_format): + click.secho("Values for `--engine` or `--row_format` must be set") + sys.exit(1) + + frappe.connect() + + if table == "all": + information_schema = frappe.qb.Schema("information_schema") + queried_tables = frappe.qb.from_( + information_schema.tables + ).select("table_name").where( + (information_schema.tables.row_format != row_format) + & (information_schema.tables.table_schema == frappe.conf.db_name) + ).run() + tables = [x[0] for x in queried_tables] + else: + tables = [x.strip() for x in table.split(",")] + + total = len(tables) + + for current, table in enumerate(tables): + values_to_set = "" + if engine: + values_to_set += f" ENGINE={engine}" + if row_format: + values_to_set += f" ROW_FORMAT={row_format}" + + try: + frappe.db.sql(f"ALTER TABLE `{table}`{values_to_set}") + update_progress_bar("Updating table schema", current - skipped, total) + add_line = True + + except Exception as e: + check_table.append([table, e.args]) + skipped += 1 + + if failfast: + break + + if add_line: + print() + + for errored_table in check_table: + table, err = errored_table + err_msg = f"{table}: ERROR {err[0]}: {err[1]}" + click.secho(err_msg, fg="yellow") + + frappe.destroy() + + @click.command('run-tests') @click.option('--app', help="For App") @click.option('--doctype', help="For DocType") @@ -811,6 +898,8 @@ commands = [ build, clear_cache, clear_website_cache, + database, + transform_database, jupyter, console, destroy_all_sessions, diff --git a/frappe/core/doctype/access_log/access_log.py b/frappe/core/doctype/access_log/access_log.py index d93da02d25..f631353d56 100644 --- a/frappe/core/doctype/access_log/access_log.py +++ b/frappe/core/doctype/access_log/access_log.py @@ -1,6 +1,7 @@ -# Copyright (c) 2019, Frappe Technologies and contributors +# Copyright (c) 2021, Frappe Technologies and contributors # License: MIT. See LICENSE import frappe +from tenacity import retry, retry_if_exception_type, stop_after_attempt from frappe.model.document import Document @@ -10,25 +11,40 @@ class AccessLog(Document): @frappe.whitelist() @frappe.write_only() -def make_access_log(doctype=None, document=None, method=None, file_type=None, - report_name=None, filters=None, page=None, columns=None): +@retry( + stop=stop_after_attempt(3), retry=retry_if_exception_type(frappe.DuplicateEntryError) +) +def make_access_log( + doctype=None, + document=None, + method=None, + file_type=None, + report_name=None, + filters=None, + page=None, + columns=None, +): user = frappe.session.user + in_request = frappe.request and frappe.request.method == "GET" - doc = frappe.get_doc({ - 'doctype': 'Access Log', - 'user': user, - 'export_from': doctype, - 'reference_document': document, - 'file_type': file_type, - 'report_name': report_name, - 'page': page, - 'method': method, - 'filters': frappe.utils.cstr(filters) if filters else None, - 'columns': columns - }) + doc = frappe.get_doc( + { + "doctype": "Access Log", + "user": user, + "export_from": doctype, + "reference_document": document, + "file_type": file_type, + "report_name": report_name, + "page": page, + "method": method, + "filters": frappe.utils.cstr(filters) if filters else None, + "columns": columns, + } + ) doc.insert(ignore_permissions=True) # `frappe.db.commit` added because insert doesnt `commit` when called in GET requests like `printview` - if frappe.request and frappe.request.method == 'GET': + # dont commit in test mode + if not frappe.flags.in_test or in_request: frappe.db.commit() diff --git a/frappe/core/doctype/document_naming_rule/document_naming_rule.json b/frappe/core/doctype/document_naming_rule/document_naming_rule.json index 4a88e3be6e..4e6f3f3fd1 100644 --- a/frappe/core/doctype/document_naming_rule/document_naming_rule.json +++ b/frappe/core/doctype/document_naming_rule/document_naming_rule.json @@ -41,6 +41,7 @@ "fieldname": "counter", "fieldtype": "Int", "label": "Counter", + "no_copy": 1, "read_only": 1 }, { @@ -79,7 +80,7 @@ ], "index_web_pages_for_search": 1, "links": [], - "modified": "2020-11-04 14:38:14.836056", + "modified": "2021-09-13 20:07:47.617615", "modified_by": "Administrator", "module": "Core", "name": "Document Naming Rule", diff --git a/frappe/core/doctype/version/version.py b/frappe/core/doctype/version/version.py index 08d0456dff..fcb558650a 100644 --- a/frappe/core/doctype/version/version.py +++ b/frappe/core/doctype/version/version.py @@ -1,8 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -# License: MIT. See LICENSE - import frappe, json from frappe.model.document import Document diff --git a/frappe/database/database.py b/frappe/database/database.py index 84bfa76cd7..0ee11ea075 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -332,7 +332,7 @@ class Database(object): values[key] = value if isinstance(value, (list, tuple)): # value is a tuple like ("!=", 0) - _operator = value[0] + _operator = value[0].lower() values[key] = value[1] if isinstance(value[1], (tuple, list)): # value is a list in tuple ("in", ("A", "B")) @@ -919,13 +919,13 @@ class Database(object): WHERE table_name = 'tab{0}' AND column_name = '{1}' '''.format(doctype, column))[0][0] def has_index(self, table_name, index_name): - pass + raise NotImplementedError def add_index(self, doctype, fields, index_name=None): - pass + raise NotImplementedError def add_unique(self, doctype, fields, constraint_name=None): - pass + raise NotImplementedError @staticmethod def get_index_name(fields): @@ -951,7 +951,7 @@ class Database(object): def escape(s, percent=True): """Excape quotes and percent in given string.""" # implemented in specific class - pass + raise NotImplementedError @staticmethod def is_column_missing(e): diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index 71acefe17c..5ed7991a82 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -135,8 +135,8 @@ class MariaDBDatabase(Database): table_name = get_table_name(doctype) return self.sql(f"DESC `{table_name}`") - def change_column_type(self, table: str, column: str, type: str) -> Union[List, Tuple]: - table_name = get_table_name(table) + def change_column_type(self, doctype: str, column: str, type: str) -> Union[List, Tuple]: + table_name = get_table_name(doctype) return self.sql(f"ALTER TABLE `{table_name}` MODIFY `{column}` {type} NOT NULL") # exception types @@ -195,7 +195,7 @@ class MariaDBDatabase(Database): `password` TEXT NOT NULL, `encrypted` INT(1) NOT NULL DEFAULT 0, PRIMARY KEY (`doctype`, `name`, `fieldname`) - ) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci""") + ) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci""") def create_global_search_table(self): if not '__global_search' in self.get_tables(): diff --git a/frappe/database/mariadb/framework_mariadb.sql b/frappe/database/mariadb/framework_mariadb.sql index 777e036049..670fb71aa2 100644 --- a/frappe/database/mariadb/framework_mariadb.sql +++ b/frappe/database/mariadb/framework_mariadb.sql @@ -72,7 +72,7 @@ CREATE TABLE `tabDocField` ( KEY `label` (`label`), KEY `fieldtype` (`fieldtype`), KEY `fieldname` (`fieldname`) -) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -- @@ -109,7 +109,7 @@ CREATE TABLE `tabDocPerm` ( `email` int(1) NOT NULL DEFAULT 1, PRIMARY KEY (`name`), KEY `parent` (`parent`) -) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -- -- Table structure for table `tabDocType Action` @@ -133,7 +133,7 @@ CREATE TABLE `tabDocType Action` ( PRIMARY KEY (`name`), KEY `parent` (`parent`), KEY `modified` (`modified`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci ROW_FORMAT=COMPRESSED; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci ROW_FORMAT=DYNAMIC; -- -- Table structure for table `tabDocType Action` @@ -156,7 +156,7 @@ CREATE TABLE `tabDocType Link` ( PRIMARY KEY (`name`), KEY `parent` (`parent`), KEY `modified` (`modified`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci ROW_FORMAT=COMPRESSED; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci ROW_FORMAT=DYNAMIC; -- -- Table structure for table `tabDocType` @@ -228,7 +228,7 @@ CREATE TABLE `tabDocType` ( `sender_field` varchar(255) DEFAULT NULL, PRIMARY KEY (`name`), KEY `parent` (`parent`) -) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -- -- Table structure for table `tabSeries` @@ -239,7 +239,7 @@ CREATE TABLE `tabSeries` ( `name` varchar(100), `current` int(10) NOT NULL DEFAULT 0, PRIMARY KEY(`name`) -) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -- @@ -256,7 +256,7 @@ CREATE TABLE `tabSessions` ( `device` varchar(255) DEFAULT 'desktop', `status` varchar(20) DEFAULT NULL, KEY `sid` (`sid`) -) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -- @@ -269,7 +269,7 @@ CREATE TABLE `tabSingles` ( `field` varchar(255) DEFAULT NULL, `value` text, KEY `singles_doctype_field_index` (`doctype`, `field`) -) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -- -- Table structure for table `__Auth` @@ -283,7 +283,7 @@ CREATE TABLE `__Auth` ( `password` TEXT NOT NULL, `encrypted` INT(1) NOT NULL DEFAULT 0, PRIMARY KEY (`doctype`, `name`, `fieldname`) -) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -- -- Table structure for table `tabFile` @@ -311,7 +311,7 @@ CREATE TABLE `tabFile` ( KEY `parent` (`parent`), KEY `attached_to_name` (`attached_to_name`), KEY `attached_to_doctype` (`attached_to_doctype`) -) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -- -- Table structure for table `tabDefaultValue` @@ -334,4 +334,4 @@ CREATE TABLE `tabDefaultValue` ( PRIMARY KEY (`name`), KEY `parent` (`parent`), KEY `defaultvalue_parent_defkey_index` (`parent`,`defkey`) -) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; diff --git a/frappe/database/mariadb/schema.py b/frappe/database/mariadb/schema.py index b40af59286..5768a2f23d 100644 --- a/frappe/database/mariadb/schema.py +++ b/frappe/database/mariadb/schema.py @@ -4,18 +4,22 @@ from frappe.database.schema import DBTable class MariaDBTable(DBTable): def create(self): - add_text = '' + additional_definitions = "" + engine = self.meta.get("engine") or "InnoDB" + varchar_len = frappe.db.VARCHAR_LEN # columns column_defs = self.get_column_definitions() - if column_defs: add_text += ',\n'.join(column_defs) + ',\n' + if column_defs: + additional_definitions += ',\n'.join(column_defs) + ',\n' # index index_defs = self.get_index_definitions() - if index_defs: add_text += ',\n'.join(index_defs) + ',\n' + if index_defs: + additional_definitions += ',\n'.join(index_defs) + ',\n' # create table - frappe.db.sql("""create table `%s` ( + query = f"""create table `{self.table_name}` ( name varchar({varchar_len}) not null primary key, creation datetime(6), modified datetime(6), @@ -26,13 +30,15 @@ class MariaDBTable(DBTable): parentfield varchar({varchar_len}), parenttype varchar({varchar_len}), idx int(8) not null default '0', - %sindex parent(parent), + {additional_definitions} + index parent(parent), index modified(modified)) ENGINE={engine} - ROW_FORMAT=COMPRESSED + ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 - COLLATE=utf8mb4_unicode_ci""".format(varchar_len=frappe.db.VARCHAR_LEN, - engine=self.meta.get("engine") or 'InnoDB') % (self.table_name, add_text)) + COLLATE=utf8mb4_unicode_ci""" + + frappe.db.sql(query) def alter(self): for col in self.columns.values(): diff --git a/frappe/database/postgres/database.py b/frappe/database/postgres/database.py index 264d3bbf14..a06abb1013 100644 --- a/frappe/database/postgres/database.py +++ b/frappe/database/postgres/database.py @@ -4,6 +4,7 @@ from typing import List, Tuple, Union import psycopg2 import psycopg2.extensions from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT +from psycopg2.errorcodes import STRING_DATA_RIGHT_TRUNCATION import frappe from frappe.database.database import Database @@ -171,7 +172,7 @@ class PostgresDatabase(Database): @staticmethod def is_data_too_long(e): - return e.pgcode == '22001' + return e.pgcode == STRING_DATA_RIGHT_TRUNCATION def rename_table(self, old_name: str, new_name: str) -> Union[List, Tuple]: old_name = get_table_name(old_name) @@ -182,8 +183,8 @@ class PostgresDatabase(Database): table_name = get_table_name(doctype) return self.sql(f"SELECT COLUMN_NAME FROM information_schema.COLUMNS WHERE TABLE_NAME = '{table_name}'") - def change_column_type(self, table: str, column: str, type: str) -> Union[List, Tuple]: - table_name = get_table_name(table) + def change_column_type(self, doctype: str, column: str, type: str) -> Union[List, Tuple]: + table_name = get_table_name(doctype) return self.sql(f'ALTER TABLE "{table_name}" ALTER COLUMN "{column}" TYPE {type}') def create_auth_table(self): diff --git a/frappe/desk/doctype/system_console/system_console.js b/frappe/desk/doctype/system_console/system_console.js index 2d097f01ad..0fe3932671 100644 --- a/frappe/desk/doctype/system_console/system_console.js +++ b/frappe/desk/doctype/system_console/system_console.js @@ -10,18 +10,56 @@ frappe.ui.form.on('System Console', { description: __('Execute Console script'), ignore_inputs: true, }); + frm.set_value("type", "Python"); }, refresh: function(frm) { frm.disable_save(); frm.page.set_primary_action(__("Execute"), $btn => { - $btn.text(__('Executing...')); - return frm.execute_action("Execute").then(() => { - $btn.text(__('Execute')); - }); + $btn.text(__("Executing...")); + return frm + .execute_action("Execute") + .then(() => frm.trigger("render_sql_output")) + .finally(() => $btn.text(__("Execute"))); }); }, + type: function(frm) { + if (frm.doc.type == "Python") { + frm.set_value("output", ""); + if (frm.sql_output) { + frm.sql_output.destroy(); + frm.get_field("sql_output").html(""); + } + } + }, + + render_sql_output: function(frm) { + if (frm.doc.type !== "SQL") return; + if (frm.sql_output) { + frm.sql_output.destroy(); + frm.get_field("sql_output").html(""); + } + + if (frm.doc.output.startsWith("Traceback")) { + return; + } + + let result = JSON.parse(frm.doc.output); + frm.set_value("output", `${result.length} ${result.length == 1 ? 'row' : 'rows'}`); + + if (result.length) { + let columns = Object.keys(result[0]); + frm.sql_output = new DataTable( + frm.get_field("sql_output").$wrapper.get(0), + { + columns, + data: result + } + ); + } + }, + show_processlist: function(frm) { if (frm.doc.show_processlist) { // keep refreshing every 5 seconds @@ -32,6 +70,7 @@ frappe.ui.form.on('System Console', { // end it clearInterval(frm.processlist_interval); + frm.get_field("processlist").html(''); } } }, diff --git a/frappe/desk/doctype/system_console/system_console.json b/frappe/desk/doctype/system_console/system_console.json index 753e672cdc..657e9df89d 100644 --- a/frappe/desk/doctype/system_console/system_console.json +++ b/frappe/desk/doctype/system_console/system_console.json @@ -18,9 +18,11 @@ "engine": "InnoDB", "field_order": [ "execute_section", + "type", "console", "commit", "output", + "sql_output", "database_processes_section", "show_processlist", "processlist" @@ -65,13 +67,26 @@ "fieldname": "processlist", "fieldtype": "HTML", "label": "processlist" + }, + { + "default": "Python", + "fieldname": "type", + "fieldtype": "Select", + "label": "Type", + "options": "Python\nSQL" + }, + { + "depends_on": "eval:doc.type == 'SQL'", + "fieldname": "sql_output", + "fieldtype": "HTML", + "label": "SQL Output" } ], "hide_toolbar": 1, "index_web_pages_for_search": 1, "issingle": 1, "links": [], - "modified": "2021-09-09 13:10:14.237113", + "modified": "2021-09-15 17:17:44.844767", "modified_by": "Administrator", "module": "Desk", "name": "System Console", diff --git a/frappe/desk/doctype/system_console/system_console.py b/frappe/desk/doctype/system_console/system_console.py index 8382dc8638..107ab2f932 100644 --- a/frappe/desk/doctype/system_console/system_console.py +++ b/frappe/desk/doctype/system_console/system_console.py @@ -5,7 +5,7 @@ import json import frappe -from frappe.utils.safe_exec import safe_exec +from frappe.utils.safe_exec import safe_exec, read_sql from frappe.model.document import Document class SystemConsole(Document): @@ -13,8 +13,11 @@ class SystemConsole(Document): frappe.only_for('System Manager') try: frappe.debug_log = [] - safe_exec(self.console) - self.output = '\n'.join(frappe.debug_log) + if self.type == 'Python': + safe_exec(self.console) + self.output = '\n'.join(frappe.debug_log) + elif self.type == 'SQL': + self.output = frappe.as_json(read_sql(self.console, as_dict=1)) except: # noqa: E722 self.output = frappe.get_traceback() diff --git a/frappe/desk/doctype/workspace/workspace.json b/frappe/desk/doctype/workspace/workspace.json index 1e111b8d12..756a40da4b 100644 --- a/frappe/desk/doctype/workspace/workspace.json +++ b/frappe/desk/doctype/workspace/workspace.json @@ -165,8 +165,6 @@ "default": "0", "fieldname": "is_standard", "fieldtype": "Check", - "in_list_view": 1, - "in_standard_filter": 1, "label": "Is Standard", "search_index": 1 }, @@ -181,7 +179,6 @@ "depends_on": "eval:doc.extends_another_page == 1 || doc.for_user", "fieldname": "extends", "fieldtype": "Link", - "in_standard_filter": 1, "label": "Extends", "options": "Workspace", "search_index": 1 @@ -228,6 +225,8 @@ "default": "0", "fieldname": "public", "fieldtype": "Check", + "in_list_view": 1, + "in_standard_filter": 1, "label": "Public" }, { @@ -265,11 +264,13 @@ "label": "Roles" } ], + "in_create": 1, "links": [], - "modified": "2021-08-30 18:47:18.227154", + "modified": "2021-09-16 12:01:06.450621", "modified_by": "Administrator", "module": "Desk", "name": "Workspace", + "naming_rule": "By fieldname", "owner": "Administrator", "permissions": [ { diff --git a/frappe/desk/doctype/workspace/workspace.py b/frappe/desk/doctype/workspace/workspace.py index 25dd9b26d2..a0a22a43fc 100644 --- a/frappe/desk/doctype/workspace/workspace.py +++ b/frappe/desk/doctype/workspace/workspace.py @@ -208,17 +208,17 @@ def save_page(title, icon, parent, public, sb_public_items, sb_private_items, de if loads(deleted_pages): return delete_pages(loads(deleted_pages)) - return {"name": title, "public": public} + return {"name": title, "public": public, "label": doc.label} def delete_pages(deleted_pages): for page in deleted_pages: if page.get("public") and "Workspace Manager" not in frappe.get_roles(): - return {"name": page.get("title"), "public": 1} + return {"name": page.get("title"), "public": 1, "label": page.get("label")} if frappe.db.exists("Workspace", page.get("name")): frappe.get_doc("Workspace", page.get("name")).delete(ignore_permissions=True) - return {"name": "Home", "public": 1} + return {"name": "Home", "public": 1, "label": "Home"} def sort_pages(sb_public_items, sb_private_items): wspace_public_pages = get_page_list(['name', 'title'], {'public': 1}) diff --git a/frappe/hooks.py b/frappe/hooks.py index 3cfdebc12e..2ae5a59066 100644 --- a/frappe/hooks.py +++ b/frappe/hooks.py @@ -12,11 +12,11 @@ source_link = "https://github.com/frappe/frappe" app_license = "MIT" app_logo_url = '/assets/frappe/images/frappe-framework-logo.svg' -develop_version = '13.x.x-develop' +develop_version = '14.x.x-develop' -app_email = "info@frappe.io" +app_email = "developers@frappe.io" -docs_app = "frappe_io" +docs_app = "frappe_docs" translator_url = "https://translate.erpnext.com" diff --git a/frappe/installer.py b/frappe/installer.py index 23247046f6..f0bf0cb51c 100755 --- a/frappe/installer.py +++ b/frappe/installer.py @@ -445,9 +445,21 @@ def extract_sql_from_archive(sql_file_path): else: decompressed_file_name = sql_file_path + # convert archive sql to latest compatible + convert_archive_content(decompressed_file_name) + return decompressed_file_name +def convert_archive_content(sql_file_path): + if frappe.conf.db_type == "mariadb": + # ever since mariaDB 10.6, row_format COMPRESSED has been deprecated and removed + # this step is added to ease restoring sites depending on older mariaDB servers + contents = open(sql_file_path).read() + with open(sql_file_path, "w") as f: + f.write(contents.replace("ROW_FORMAT=COMPRESSED", "ROW_FORMAT=DYNAMIC")) + + def extract_sql_gzip(sql_gz_path): import subprocess @@ -457,7 +469,7 @@ def extract_sql_gzip(sql_gz_path): decompressed_file = original_file.rstrip(".gz") cmd = 'gzip -dvf < {0} > {1}'.format(original_file, decompressed_file) subprocess.check_call(cmd, shell=True) - except: + except Exception: raise return decompressed_file diff --git a/frappe/model/base_document.py b/frappe/model/base_document.py index b9c8900839..5605ac61ed 100644 --- a/frappe/model/base_document.py +++ b/frappe/model/base_document.py @@ -307,7 +307,7 @@ class BaseDocument(object): doc["doctype"] = self.doctype for df in self.meta.get_table_fields(): children = self.get(df.fieldname) or [] - doc[df.fieldname] = [d.as_dict(convert_dates_to_str=convert_dates_to_str, no_nulls=no_nulls) for d in children] + doc[df.fieldname] = [d.as_dict(convert_dates_to_str=convert_dates_to_str, no_nulls=no_nulls, no_default_fields=no_default_fields) for d in children] if no_nulls: for k in list(doc): diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py index fd74a8cfe4..978f3062c5 100644 --- a/frappe/model/db_query.py +++ b/frappe/model/db_query.py @@ -4,6 +4,7 @@ from typing import List import frappe.defaults +from frappe.query_builder.utils import Column import frappe.share from frappe import _ import frappe.permissions @@ -491,7 +492,7 @@ class DatabaseQuery(object): f.value = date_range fallback = "'0001-01-01 00:00:00'" - if f.operator in ('>', '<') and (f.fieldname in ('creation', 'modified')): + if (f.fieldname in ('creation', 'modified')): value = cstr(f.value) fallback = "NULL" @@ -547,8 +548,12 @@ class DatabaseQuery(object): value = flt(f.value) fallback = 0 + if isinstance(f.value, Column): + quote = '"' if frappe.conf.db_type == 'postgres' else "`" + value = f"{tname}.{quote}{f.value.name}{quote}" + # escape value - if isinstance(value, str) and not f.operator.lower() == 'between': + elif isinstance(value, str) and not f.operator.lower() == 'between': value = f"{frappe.db.escape(value, percent=False)}" if ( diff --git a/frappe/model/meta.py b/frappe/model/meta.py index 207aca089b..cd0d8e0f3a 100644 --- a/frappe/model/meta.py +++ b/frappe/model/meta.py @@ -15,6 +15,7 @@ Example: ''' from datetime import datetime +import click import frappe, json, os from frappe.utils import cstr, cint, cast from frappe.model import default_fields, no_value_fields, optional_fields, data_fieldtypes, table_fields @@ -658,27 +659,48 @@ def get_default_df(fieldname): fieldtype = "Data" ) -def trim_tables(doctype=None): +def trim_tables(doctype=None, dry_run=False, quiet=False): """ Removes database fields that don't exist in the doctype (json or custom field). This may be needed as maintenance since removing a field in a DocType doesn't automatically delete the db field. """ - ignore_fields = default_fields + optional_fields - - filters={ "issingle": 0 } + UPDATED_TABLES = {} + filters = {"issingle": 0} if doctype: filters["name"] = doctype - for doctype in frappe.db.get_all("DocType", filters=filters): - doctype = doctype.name - columns = frappe.db.get_table_columns(doctype) - fields = frappe.get_meta(doctype).get_fieldnames_with_value() - columns_to_remove = [f for f in list(set(columns) - set(fields)) if f not in ignore_fields - and not f.startswith("_")] - if columns_to_remove: - print(doctype, "columns removed:", columns_to_remove) - columns_to_remove = ", ".join("drop `{0}`".format(c) for c in columns_to_remove) - query = """alter table `tab{doctype}` {columns}""".format( - doctype=doctype, columns=columns_to_remove) - frappe.db.sql_ddl(query) + for doctype in frappe.db.get_all("DocType", filters=filters, pluck="name"): + try: + dropped_columns = trim_table(doctype, dry_run=dry_run) + if dropped_columns: + UPDATED_TABLES[doctype] = dropped_columns + except frappe.db.TableMissingError: + if quiet: + continue + click.secho(f"Ignoring missing table for DocType: {doctype}", fg="yellow", err=True) + click.secho(f"Consider removing record in the DocType table for {doctype}", fg="yellow", err=True) + except Exception as e: + if quiet: + continue + click.echo(e, err=True) + + return UPDATED_TABLES + + +def trim_table(doctype, dry_run=True): + frappe.cache().hdel('table_columns', f"tab{doctype}") + ignore_fields = default_fields + optional_fields + columns = frappe.db.get_table_columns(doctype) + fields = frappe.get_meta(doctype, cached=False).get_fieldnames_with_value() + is_internal = lambda f: f not in ignore_fields and not f.startswith("_") + columns_to_remove = [ + f for f in list(set(columns) - set(fields)) if is_internal(f) + ] + DROPPED_COLUMNS = columns_to_remove[:] + + if columns_to_remove and not dry_run: + columns_to_remove = ", ".join(f"DROP `{c}`" for c in columns_to_remove) + frappe.db.sql_ddl(f"ALTER TABLE `tab{doctype}` {columns_to_remove}") + + return DROPPED_COLUMNS diff --git a/frappe/patches/v13_0/increase_password_length.py b/frappe/patches/v13_0/increase_password_length.py index 62ca2ed779..deb7d7e98a 100644 --- a/frappe/patches/v13_0/increase_password_length.py +++ b/frappe/patches/v13_0/increase_password_length.py @@ -1,4 +1,4 @@ import frappe def execute(): - frappe.db.change_column_type(table="__Auth", column="password", type="TEXT") + frappe.db.change_column_type("__Auth", column="password", type="TEXT") diff --git a/frappe/public/js/frappe/form/grid.js b/frappe/public/js/frappe/form/grid.js index dd1d622bab..e6629ba039 100644 --- a/frappe/public/js/frappe/form/grid.js +++ b/frappe/public/js/frappe/form/grid.js @@ -835,10 +835,11 @@ export default class Grid { $.each(row, (ci, value) => { var fieldname = fieldnames[ci]; var df = frappe.meta.get_docfield(me.df.options, fieldname); - - d[fieldnames[ci]] = value_formatter_map[df.fieldtype] - ? value_formatter_map[df.fieldtype](value) - : value; + if (df) { + d[fieldnames[ci]] = value_formatter_map[df.fieldtype] + ? value_formatter_map[df.fieldtype](value) + : value; + } }); } } diff --git a/frappe/public/js/frappe/form/grid_row_form.js b/frappe/public/js/frappe/form/grid_row_form.js index 73131a00ae..31295899b5 100644 --- a/frappe/public/js/frappe/form/grid_row_form.js +++ b/frappe/public/js/frappe/form/grid_row_form.js @@ -123,10 +123,12 @@ export default class GridRowForm { .toggle(this.row.grid.is_editable()); } refresh_field(fieldname) { - if(this.fields_dict[fieldname]) { - this.fields_dict[fieldname].refresh(); - this.layout && this.layout.refresh_dependency(); - } + const field = this.fields_dict[fieldname]; + if (!field) return; + + field.docname = this.row.doc.name; + field.refresh(); + this.layout && this.layout.refresh_dependency(); } set_focus() { // wait for animation and then focus on the first row diff --git a/frappe/public/js/frappe/form/multi_select_dialog.js b/frappe/public/js/frappe/form/multi_select_dialog.js index dd96b57fb5..ba522a4085 100644 --- a/frappe/public/js/frappe/form/multi_select_dialog.js +++ b/frappe/public/js/frappe/form/multi_select_dialog.js @@ -2,86 +2,191 @@ frappe.ui.form.MultiSelectDialog = class MultiSelectDialog { constructor(opts) { /* Options: doctype, target, setters, get_query, action, add_filters_group, data_fields, primary_action_label */ Object.assign(this, opts); - var me = this; - if (this.doctype != "[Select]") { - frappe.model.with_doctype(this.doctype, function () { - me.make(); - }); + this.for_select = this.doctype == "[Select]"; + if (!this.for_select) { + frappe.model.with_doctype(this.doctype, () => this.init()); } else { - this.make(); + this.init(); } } - make() { - let me = this; + init() { this.page_length = 20; this.start = 0; - let fields = this.get_primary_filters(); + this.fields = this.get_fields(); - // Make results area - fields = fields.concat([ - { fieldtype: "HTML", fieldname: "results_area" }, + this.make(); + } + + get_fields() { + const primary_fields = this.get_primary_filters(); + const result_fields = this.get_result_fields(); + const data_fields = this.get_data_fields(); + const child_selection_fields = this.get_child_selection_fields(); + + return [...primary_fields, ...result_fields, ...data_fields, ...child_selection_fields]; + } + + get_result_fields() { + const show_next_page = () => { + this.start += 20; + this.get_results(); + }; + return [ { - fieldtype: "Button", fieldname: "more_btn", label: __("More"), - click: () => { - this.start += 20; - this.get_results(); - } + fieldtype: "HTML", fieldname: "results_area" + }, + { + fieldtype: "Button", fieldname: "more_btn", + label: __("More"), click: show_next_page.bind(this) } - ]); + ]; + } - // Custom Data Fields - if (this.data_fields) { - fields.push({ fieldtype: "Section Break" }); - fields = fields.concat(this.data_fields); + get_data_fields() { + if (this.data_fields && this.data_fields.length) { + // Custom Data Fields + return [ + { fieldtype: "Section Break" }, + ...this.data_fields + ]; + } else { + return []; } + } + get_child_selection_fields() { + const fields = []; + if (this.allow_child_item_selection && this.child_fieldname) { + fields.push({ fieldtype: "HTML", fieldname: "child_selection_area" }); + } + return fields; + } + + make() { let doctype_plural = this.doctype.plural(); + let title = __("Select {0}", [this.for_select ? __("value") : __(doctype_plural)]); this.dialog = new frappe.ui.Dialog({ - title: __("Select {0}", [(this.doctype == '[Select]') ? __("value") : __(doctype_plural)]), - fields: fields, + title: title, + fields: this.fields, primary_action_label: this.primary_action_label || __("Get Items"), - secondary_action_label: __("Make {0}", [__(me.doctype)]), - primary_action: function () { - let filters_data = me.get_custom_filters(); - me.action(me.get_checked_values(), cur_dialog.get_values(), me.args, filters_data); + secondary_action_label: __("Make {0}", [__(this.doctype)]), + primary_action: () => { + let filters_data = this.get_custom_filters(); + const data_values = cur_dialog.get_values(); // to pass values of data fields + const filtered_children = this.get_selected_child_names(); + const selected_documents = [...this.get_checked_values(), ...this.get_parent_name_of_selected_children()]; + this.action(selected_documents, { + ...this.args, + ...data_values, + ...filters_data, + filtered_children + }); }, - secondary_action: function (e) { - // If user wants to close the modal - if (e) { - frappe.route_options = {}; - if (Array.isArray(me.setters)) { - for (let df of me.setters) { - frappe.route_options[df.fieldname] = me.dialog.fields_dict[df.fieldname].get_value() || undefined; - } - } else { - Object.keys(me.setters).forEach(function (setter) { - frappe.route_options[setter] = me.dialog.fields_dict[setter].get_value() || undefined; - }); - } - - frappe.new_doc(me.doctype, true); - } - } + secondary_action: this.make_new_document.bind(this) }); if (this.add_filters_group) { this.make_filter_area(); } + this.args = {}; + + this.setup_results(); + this.bind_events(); + this.get_results(); + this.dialog.show(); + } + + make_new_document(e) { + // If user wants to close the modal + if (e) { + this.set_route_options(); + frappe.new_doc(this.doctype, true); + } + } + + set_route_options() { + // set route options to get pre-filled form fields + frappe.route_options = {}; + if (Array.isArray(this.setters)) { + for (let df of this.setters) { + frappe.route_options[df.fieldname] = this.dialog.fields_dict[df.fieldname].get_value() || undefined; + } + } else { + Object.keys(this.setters).forEach(setter => { + frappe.route_options[setter] = this.dialog.fields_dict[setter].get_value() || undefined; + }); + } + } + + setup_results() { this.$parent = $(this.dialog.body); - this.$wrapper = this.dialog.fields_dict.results_area.$wrapper.append(`
`); this.$results = this.$wrapper.find('.results'); this.$results.append(this.make_list_row()); + } - this.args = {}; + toggle_child_selection() { + if (this.dialog.fields_dict['allow_child_item_selection'].get_value()) { + this.get_child_result().then(r => { + this.child_results = r.message || []; + this.render_child_datatable(); + + this.$wrapper.addClass('hidden'); + this.$child_wrapper.removeClass('hidden'); + this.dialog.fields_dict.more_btn.$wrapper.hide(); + }); + } else { + this.child_results = []; + this.get_results(); + this.$wrapper.removeClass('hidden'); + this.$child_wrapper.addClass('hidden'); + } + } - this.bind_events(); - this.get_results(); - this.dialog.show(); + render_child_datatable() { + if (!this.child_datatable) { + this.setup_child_datatable(); + } else { + setTimeout(() => { + this.child_datatable.rowmanager.checkMap = []; + this.child_datatable.refresh(this.get_child_datatable_rows()); + this.$child_wrapper.find('.dt-scrollable').css('height', '300px'); + }, 500); + } + } + + get_child_datatable_columns() { + const parent = this.doctype; + return [parent, ...this.child_columns].map(d => ({ name: frappe.unscrub(d), editable: false })); + } + + get_child_datatable_rows() { + return this.child_results.map(d => Object.values(d).slice(1)); // slice name field + } + + setup_child_datatable() { + const header_columns = this.get_child_datatable_columns(); + const rows = this.get_child_datatable_rows(); + this.$child_wrapper = this.dialog.fields_dict.child_selection_area.$wrapper; + this.$child_wrapper.addClass('mt-3'); + + this.child_datatable = new frappe.DataTable(this.$child_wrapper.get(0), { + columns: header_columns, + data: rows, + layout: 'fluid', + inlineFilters: true, + serialNoColumn: false, + checkboxColumn: true, + cellHeight: 35, + noDataMessage: __('No Data'), + disableReorderColumn: true + }); + this.$child_wrapper.find('.dt-scrollable').css('height', '300px'); } get_primary_filters() { @@ -94,7 +199,7 @@ frappe.ui.form.MultiSelectDialog = class MultiSelectDialog { columns[0] = [ { fieldtype: "Data", - label: __("Search"), + label: __("Name"), fieldname: "search_term" } ]; @@ -127,6 +232,16 @@ frappe.ui.form.MultiSelectDialog = class MultiSelectDialog { // now a is a fixed-size array with mutable entries } + if (this.allow_child_item_selection) { + this.child_doctype = frappe.meta.get_docfield(this.doctype, this.child_fieldname).options; + columns[0].push({ + fieldtype: "Check", + label: __("Select {0}", [this.child_doctype]), + fieldname: "allow_child_item_selection", + onchange: this.toggle_child_selection.bind(this) + }); + } + fields = [ ...columns[0], { fieldtype: "Column Break" }, @@ -156,6 +271,9 @@ frappe.ui.form.MultiSelectDialog = class MultiSelectDialog { this.get_results(); } }); + // 'Apply Filter' breaks since the filers are not in a popover + // Hence keeping it hidden + this.filter_group.wrapper.find('.apply-filters').hide(); } get_custom_filters() { @@ -166,7 +284,7 @@ frappe.ui.form.MultiSelectDialog = class MultiSelectDialog { }); }, {}); } else { - return []; + return {}; } } @@ -200,6 +318,34 @@ frappe.ui.form.MultiSelectDialog = class MultiSelectDialog { }); } + get_parent_name_of_selected_children() { + if (!this.child_datatable || !this.child_datatable.datamanager.rows.length) return []; + + let parent_names = this.child_datatable.rowmanager.checkMap.reduce((parent_names, checked, index) => { + if (checked == 1) { + const parent_name = this.child_results[index].parent; + parent_names.push(parent_name); + } + return parent_names; + }, []); + + return parent_names; + } + + get_selected_child_names() { + if (!this.child_datatable || !this.child_datatable.datamanager.rows.length) return []; + + let checked_names = this.child_datatable.rowmanager.checkMap.reduce((checked_names, checked, index) => { + if (checked == 1) { + const child_row_name = this.child_results[index].name; + checked_names.push(child_row_name); + } + return checked_names; + }, []); + + return checked_names; + } + get_checked_values() { // Return name of checked value. return this.$results.find('.list-item-container').map(function () { @@ -276,6 +422,8 @@ frappe.ui.form.MultiSelectDialog = class MultiSelectDialog { me.$results.append(me.make_list_row(result)); }); + this.$results.find(".list-item--head").css("z-index", 0); + if (frappe.flags.auto_scroll) { this.$results.animate({ scrollTop: me.$results.prop('scrollHeight') }, 500); } @@ -297,7 +445,7 @@ frappe.ui.form.MultiSelectDialog = class MultiSelectDialog { this.render_result_list(checked, 0, false); } - get_results() { + get_filters_from_setters() { let me = this; let filters = this.get_query ? this.get_query().filters : {} || {}; let filter_fields = []; @@ -321,12 +469,18 @@ frappe.ui.form.MultiSelectDialog = class MultiSelectDialog { }); } - let filter_group = this.get_custom_filters(); - Object.assign(filters, filter_group); + return [filters, filter_fields]; + } - let args = { - doctype: me.doctype, - txt: me.dialog.fields_dict["search_term"].get_value(), + get_args_for_search() { + let [filters, filter_fields] = this.get_filters_from_setters(); + + let custom_filters = this.get_custom_filters(); + Object.assign(filters, custom_filters); + + return { + doctype: this.doctype, + txt: this.dialog.fields_dict["search_term"].get_value(), filters: filters, filter_fields: filter_fields, start: this.start, @@ -334,25 +488,81 @@ frappe.ui.form.MultiSelectDialog = class MultiSelectDialog { query: this.get_query ? this.get_query().query : '', as_dict: 1 }; - frappe.call({ + } + + async perform_search(args) { + const res = await frappe.call({ type: "GET", method: 'frappe.desk.search.search_widget', no_spinner: true, args: args, - callback: function (r) { - let more = 0; - me.results = []; - if (r.values.length) { - if (r.values.length > me.page_length) { - r.values.pop(); - more = 1; - } - r.values.forEach(function (result) { - result.checked = 0; - me.results.push(result); - }); + }); + const more = res.values.length && res.values.length > this.page_length ? 1 : 0; + if (more) { + res.values.pop(); + } + + return [res, more]; + } + + async get_results() { + const args = this.get_args_for_search(); + const [res, more] = await this.perform_search(args); + + this.results = []; + if (res.values.length) { + res.values.forEach(result => { + result.checked = 0; + this.results.push(result); + }); + } + this.render_result_list(this.results, more); + } + + async get_filtered_parents_for_child_search() { + const parent_search_args = this.get_args_for_search(); + parent_search_args.filter_fields = ['name']; + // eslint-disable-next-line no-unused-vars + const [response, _] = await this.perform_search(parent_search_args); + + let parent_names = []; + if (response.values.length) { + parent_names = response.values.map(v => v.name); + } + return parent_names; + } + + async add_parent_filters(filters) { + const parent_names = await this.get_filtered_parents_for_child_search(); + if (parent_names.length) { + filters.push([ "parent", "in", parent_names ]); + } + } + + add_custom_child_filters(filters) { + if (this.add_filters_group && this.filter_group) { + this.filter_group.get_filters().forEach(filter => { + if (filter[0] == this.child_doctype) { + filters.push([filter[1], filter[2], filter[3]]); } - me.render_result_list(me.results, more); + }); + } + } + + async get_child_result() { + let filters = [["parentfield", "=", this.child_fieldname]]; + + await this.add_parent_filters(filters); + this.add_custom_child_filters(filters); + + return frappe.call({ + method: "frappe.client.get_list", + args: { + doctype: this.child_doctype, + filters: filters, + fields: ['name', 'parent', ...this.child_columns], + parent: this.doctype, + order_by: 'parent' } }); } diff --git a/frappe/public/js/frappe/list/base_list.js b/frappe/public/js/frappe/list/base_list.js index 3c7f8ac39a..7af2cb2007 100644 --- a/frappe/public/js/frappe/list/base_list.js +++ b/frappe/public/js/frappe/list/base_list.js @@ -3,16 +3,14 @@ frappe.provide("frappe.views"); frappe.views.BaseList = class BaseList { constructor(opts) { Object.assign(this, opts); - this.init_page() } show() { - this.meta = frappe.get_meta(this.doctype); - this.set_title(); - // in loading state? - if (!this.meta) return; - - frappe.run_serially([ + return frappe.run_serially([ + () => this.show_skeleton(), + () => this.fetch_meta(), + () => this.hide_skeleton(), + () => this.check_permissions(), () => this.init(), () => this.before_refresh(), () => this.refresh(), @@ -40,6 +38,8 @@ frappe.views.BaseList = class BaseList { setup_defaults() { this.page_name = frappe.get_route_str(); + this.page_title = this.page_title || frappe.router.doctype_layout || __(this.doctype); + this.meta = frappe.get_meta(this.doctype); this.settings = frappe.listview_settings[this.doctype] || {}; this.user_settings = frappe.get_user_settings(this.doctype); @@ -154,21 +154,29 @@ frappe.views.BaseList = class BaseList { } } - init_page() { + fetch_meta() { + return frappe.model.with_doctype(this.doctype); + } + + show_skeleton() { + + } + + hide_skeleton() { + + } + + check_permissions() { + return true; + } + + setup_page() { this.page = this.parent.page; - this.make_skeleton(); this.$page = $(this.parent); !this.hide_card_layout && this.page.main.addClass('frappe-card'); this.page.page_form.removeClass("row").addClass("flex"); this.hide_page_form && this.page.page_form.hide(); this.hide_sidebar && this.$page.addClass('no-list-sidebar'); - } - - make_skeleton() { - this.skeleton = $(`
`).prependTo(this.page.main.parent()); - } - - setup_page() { this.setup_page_head(); } @@ -179,7 +187,6 @@ frappe.views.BaseList = class BaseList { } set_title() { - this.page_title = this.page_title || frappe.router.doctype_layout || __(this.doctype); this.page.set_title(this.page_title); } @@ -293,7 +300,6 @@ frappe.views.BaseList = class BaseList { } setup_list_wrapper() { - this.skeleton.remove(); // clear skeleton this.$frappe_list = $('
').appendTo( this.page.main ); diff --git a/frappe/public/js/frappe/list/list_factory.js b/frappe/public/js/frappe/list/list_factory.js index d870fdb6fc..acad85fdcb 100644 --- a/frappe/public/js/frappe/list/list_factory.js +++ b/frappe/public/js/frappe/list/list_factory.js @@ -8,48 +8,32 @@ frappe.views.ListFactory = class ListFactory extends frappe.views.Factory { make (route) { var me = this; var doctype = route[1]; - const meta_loaded = frappe.get_meta(doctype) ? true : false; - const page_name = frappe.get_route_str(); - let view_class = this.get_view_class(route, doctype); - this.make_list_view_page(page_name, doctype, view_class); - if (view_class && view_class.load_last_view && view_class.load_last_view()) { - // view can have custom routing logic - return; - } - - frappe.model.with_doctype(doctype, function () { - if (!meta_loaded) { - frappe.views.list_view[page_name].show(); - } - if (locals['DocType'][doctype].issingle) { - frappe.set_re_route('Form', doctype); - } else { - frappe.container.change_to(page_name); - me.set_cur_list(); - } - }); - } - - get_view_class(route, doctype) { // List / Gantt / Kanban / etc // File is a special view const view_name = doctype !== 'File' ? frappe.utils.to_title_case(route[2] || 'List') : 'File'; let view_class = frappe.views[view_name + 'View']; if (!view_class) view_class = frappe.views.ListView; - return view_class; - } + if (view_class && view_class.load_last_view && view_class.load_last_view()) { + // view can have custom routing logic + return; + } - make_list_view_page(page_name, doctype, view_class) { frappe.provide('frappe.views.list_view.' + doctype); + const page_name = frappe.get_route_str(); if (!frappe.views.list_view[page_name]) { frappe.views.list_view[page_name] = new view_class({ doctype: doctype, - parent: this.make_page(true, page_name) + parent: me.make_page(true, page_name) }); + } else { + frappe.container.change_to(page_name); } + me.set_cur_list(); + + } show() { diff --git a/frappe/public/js/frappe/list/list_view.js b/frappe/public/js/frappe/list/list_view.js index c5db7df88c..bd2423384d 100644 --- a/frappe/public/js/frappe/list/list_view.js +++ b/frappe/public/js/frappe/list/list_view.js @@ -33,14 +33,38 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList { show() { this.parent.disable_scroll_to_top = true; + super.show(); + } + check_permissions() { if (!this.has_permissions()) { frappe.set_route(''); - frappe.msgprint(__("Not permitted to view {0}", [this.doctype])); - return; + frappe.throw(__("Not permitted to view {0}", [this.doctype])); } + } - super.show(); + show_skeleton() { + this.$list_skeleton = this.parent.page.container.find('.list-skeleton'); + if (!this.$list_skeleton.length) { + this.$list_skeleton = $(` +
+
+
+
+
+
+
+
+ `); + this.parent.page.container.find('.page-content').append(this.$list_skeleton); + } + this.parent.page.container.find('.layout-main').hide(); + this.$list_skeleton.show(); + } + + hide_skeleton() { + this.$list_skeleton && this.$list_skeleton.hide(); + this.parent.page.container.find('.layout-main').show(); } get view_name() { @@ -583,9 +607,9 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList { const subject_field = this.columns[0].df; let subject_html = ` - - + ${__(subject_field.label)} @@ -622,7 +646,7 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList {
-
@@ -930,9 +954,9 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList { let subject_html = ` - - + @@ -1139,6 +1163,7 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList { if ( $target.hasClass("filterable") || $target.hasClass("select-like") || + $target.hasClass("file-select") || $target.hasClass("list-row-like") || $target.is(":checkbox") ) { diff --git a/frappe/public/js/frappe/model/model.js b/frappe/public/js/frappe/model/model.js index 4bbd8ab391..50f957c714 100644 --- a/frappe/public/js/frappe/model/model.js +++ b/frappe/public/js/frappe/model/model.js @@ -131,6 +131,7 @@ $.extend(frappe.model, { with_doctype: function(doctype, callback, async) { if(locals.DocType[doctype]) { callback && callback(); + return Promise.resolve(); } else { let cached_timestamp = null; let cached_doc = null; diff --git a/frappe/public/js/frappe/ui/dialog.js b/frappe/public/js/frappe/ui/dialog.js index 00336a2137..58175381cf 100644 --- a/frappe/public/js/frappe/ui/dialog.js +++ b/frappe/public/js/frappe/ui/dialog.js @@ -153,7 +153,7 @@ frappe.ui.Dialog = class Dialog extends frappe.ui.FieldGroup { set_secondary_action(click) { this.footer.removeClass('hide'); - this.get_secondary_btn().removeClass('hide').on('click', click); + this.get_secondary_btn().removeClass('hide').off('click').on('click', click); } set_secondary_action_label(label) { diff --git a/frappe/public/js/frappe/utils/utils.js b/frappe/public/js/frappe/utils/utils.js index 21841296dc..f534dff1c6 100644 --- a/frappe/public/js/frappe/utils/utils.js +++ b/frappe/public/js/frappe/utils/utils.js @@ -927,7 +927,16 @@ Object.assign(frappe.utils, { // decodes base64 to string let parts = dataURI.split(','); const encoded_data = parts[1]; - return decodeURIComponent(escape(atob(encoded_data))); + let decoded = atob(encoded_data); + try { + const escaped = escape(decoded); + decoded = decodeURIComponent(escaped); + + } catch (e) { + // pass decodeURIComponent failure + // just return atob response + } + return decoded; }, copy_to_clipboard(string) { let input = $(""); diff --git a/frappe/public/js/frappe/views/file/file_view.js b/frappe/public/js/frappe/views/file/file_view.js index e020bff4dd..11204bb660 100644 --- a/frappe/public/js/frappe/views/file/file_view.js +++ b/frappe/public/js/frappe/views/file/file_view.js @@ -380,8 +380,10 @@ frappe.views.FileView = class FileView extends frappe.views.ListView { return `
- + + + ${file.subject_html} diff --git a/frappe/public/js/frappe/views/reports/query_report.js b/frappe/public/js/frappe/views/reports/query_report.js index 1053f9b7c5..7d68919821 100644 --- a/frappe/public/js/frappe/views/reports/query_report.js +++ b/frappe/public/js/frappe/views/reports/query_report.js @@ -832,6 +832,7 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList { if (this.raw_data.add_total_row) { data = data.slice(); data.splice(-1, 1); + this.$page.find('.layout-main-section')[0].style.setProperty('--report-total-height', '310px'); } this.$report.show(); @@ -854,10 +855,6 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList { } }; - if (this.raw_data.add_total_row) { - this.$page.find('.layout-main-section').css('--report-total-height', '310px'); - } - if (this.report_settings.get_datatable_options) { datatable_options = this.report_settings.get_datatable_options(datatable_options); } diff --git a/frappe/public/js/frappe/views/reports/report_view.js b/frappe/public/js/frappe/views/reports/report_view.js index b46e6fb374..2a92d93e30 100644 --- a/frappe/public/js/frappe/views/reports/report_view.js +++ b/frappe/public/js/frappe/views/reports/report_view.js @@ -3,6 +3,7 @@ */ import DataTable from 'frappe-datatable'; +window.DataTable = DataTable; frappe.provide('frappe.views'); frappe.views.ReportView = class ReportView extends frappe.views.ListView { diff --git a/frappe/public/js/frappe/views/workspace/workspace.js b/frappe/public/js/frappe/views/workspace/workspace.js index 1e0143c2a8..8989814349 100644 --- a/frappe/public/js/frappe/views/workspace/workspace.js +++ b/frappe/public/js/frappe/views/workspace/workspace.js @@ -23,6 +23,7 @@ frappe.views.Workspace = class Workspace { this.blocks = frappe.wspace_block.blocks; this.is_read_only = true; this.new_page = null; + this.pages = {}; this.sorted_public_items = []; this.sorted_private_items = []; this.deleted_sidebar_items = []; @@ -35,42 +36,6 @@ frappe.views.Workspace = class Workspace { 'My Workspaces', 'Public' ]; - this.tools = { - header: { - class: this.blocks['header'], - inlineToolbar: true - }, - paragraph: { - class: this.blocks['paragraph'], - inlineToolbar: true - }, - chart: { - class: this.blocks['chart'], - config: { - page_data: this.page_data || [] - } - }, - card: { - class: this.blocks['card'], - config: { - page_data: this.page_data || [] - } - }, - shortcut: { - class: this.blocks['shortcut'], - config: { - page_data: this.page_data || [] - } - }, - onboarding: { - class: this.blocks['onboarding'], - config: { - page_data: this.page_data || [] - } - }, - spacer: this.blocks['spacer'], - spacingTune: frappe.wspace_block.tunes['spacing_tune'], - }; this.prepare_container(); this.setup_pages(); @@ -86,7 +51,7 @@ frappe.views.Workspace = class Workspace { this.body = this.wrapper.find(".layout-main-section"); } - setup_pages() { + setup_pages(reload) { this.get_pages().then(pages => { this.all_pages = pages.pages; this.has_access = pages.has_access; @@ -115,7 +80,7 @@ frappe.views.Workspace = class Workspace { this.new_page = null; } this.make_sidebar(); - frappe.router.route(); + reload && this.show(); } }); } @@ -236,10 +201,7 @@ frappe.views.Workspace = class Workspace { return; } - let page = { - name: this.get_page_to_show().name, - public: this.get_page_to_show().public - }; + let page = this.get_page_to_show(); this.page.set_title(`${__(page.name)}`); this.show_page(page); @@ -250,6 +212,11 @@ frappe.views.Workspace = class Workspace { page: page }).then(data => { this.page_data = data; + + // caching page data + this.pages[page.name] && delete this.pages[page.name]; + this.pages[page.name] = data; + if (!this.page_data || Object.keys(this.page_data).length === 0) return; return frappe.dashboard_utils.get_dashboard_settings().then(settings => { @@ -260,6 +227,7 @@ frappe.views.Workspace = class Workspace { chart.chart_settings = chart_config[chart.chart_name] || {}; }); } + this.pages[page.name] = this.page_data; } }); }); @@ -281,7 +249,7 @@ frappe.views.Workspace = class Workspace { return { name: page, public: is_public }; } - show_page(page) { + async show_page(page) { let section = this.current_page.public ? 'public' : 'private'; if (this.sidebar_items && this.sidebar_items[section] && this.sidebar_items[section][this.current_page.name]) { this.sidebar_items[section][this.current_page.name][0].firstElementChild.classList.remove("selected"); @@ -316,12 +284,17 @@ frappe.views.Workspace = class Workspace { this.add_custom_cards_in_content(); $('.item-anchor').addClass('disable-click'); - this.get_data(this_page).then(() => { - this.prepare_editorjs(); - $('.item-anchor').removeClass('disable-click'); - this.$page.find('.codex-editor').removeClass('hidden'); - this.$page.find('.workspace-skeleton').remove(); - }); + + if (this.pages && this.pages[this_page.name]) { + this.page_data = this.pages[this_page.name]; + } else { + await this.get_data(this_page); + } + + this.prepare_editorjs(); + $('.item-anchor').removeClass('disable-click'); + this.$page.find('.codex-editor').removeClass('hidden'); + this.$page.find('.workspace-skeleton').remove(); } } @@ -652,7 +625,7 @@ frappe.views.Workspace = class Workspace { let $sidebar_section = is_public ? $sidebar[1] : $sidebar[0]; if (!parent) { - !is_public && $sidebar.last().removeClass('hidden'); + !is_public && $sidebar.first().removeClass('hidden'); $sidebar_item.appendTo($sidebar_section); } else { let $item_container = $($sidebar_section).find(`[item-name="${parent}"]`); @@ -670,6 +643,42 @@ frappe.views.Workspace = class Workspace { } initialize_editorjs(blocks) { + this.tools = { + header: { + class: this.blocks['header'], + inlineToolbar: true + }, + paragraph: { + class: this.blocks['paragraph'], + inlineToolbar: true + }, + chart: { + class: this.blocks['chart'], + config: { + page_data: this.page_data || [] + } + }, + card: { + class: this.blocks['card'], + config: { + page_data: this.page_data || [] + } + }, + shortcut: { + class: this.blocks['shortcut'], + config: { + page_data: this.page_data || [] + } + }, + onboarding: { + class: this.blocks['onboarding'], + config: { + page_data: this.page_data || [] + } + }, + spacer: this.blocks['spacer'], + spacingTune: frappe.wspace_block.tunes['spacing_tune'], + }; this.editor = new EditorJS({ data: { blocks: blocks || [] @@ -730,6 +739,7 @@ frappe.views.Workspace = class Workspace { frappe.dom.unfreeze(); if (res.message) { me.new_page = res.message; + me.pages[res.message.label] && delete me.pages[res.message.label]; me.title = ''; me.icon = ''; me.parent = ''; @@ -751,7 +761,7 @@ frappe.views.Workspace = class Workspace { reload() { this.$page.prepend(frappe.render_template('workspace_loading_skeleton')); this.$page.find('.codex-editor').addClass('hidden'); - this.setup_pages(); + this.setup_pages(true); this.undo.readOnly = true; } }; diff --git a/frappe/public/scss/common/global.scss b/frappe/public/scss/common/global.scss index 024e0cd2a4..44b6e9ce34 100644 --- a/frappe/public/scss/common/global.scss +++ b/frappe/public/scss/common/global.scss @@ -1,3 +1,7 @@ +html, body { + height: 100%; +} + /* checkbox */ .checkbox { label { diff --git a/frappe/public/scss/desk/desktop.scss b/frappe/public/scss/desk/desktop.scss index 0de7103cc3..2ab6d98e20 100644 --- a/frappe/public/scss/desk/desktop.scss +++ b/frappe/public/scss/desk/desktop.scss @@ -739,10 +739,6 @@ body { animation-duration: 400ms; } -.skeleton-bg { - background-color: var(--skeleton-bg); -} - .workspace-skeleton { transition: ease; .widget-group-title { @@ -890,6 +886,10 @@ body { } } + .codex-editor__loader { + display: none !important; + } + .codex-editor { min-height: 630px; diff --git a/frappe/public/scss/desk/global.scss b/frappe/public/scss/desk/global.scss index ec7fc35cfe..8c646395e9 100644 --- a/frappe/public/scss/desk/global.scss +++ b/frappe/public/scss/desk/global.scss @@ -1,5 +1,4 @@ html { - height: 100%; background-color: var(--bg-color); } @@ -327,6 +326,10 @@ select.input-xs { } } +// .frappe-card { +// @include card(); +// } + .head-title { font-size: var(--text-lg); font-weight: 700; @@ -587,4 +590,4 @@ details > summary:focus { .chart-container { direction: ltr; } -*/ +*/ \ No newline at end of file diff --git a/frappe/public/scss/desk/list.scss b/frappe/public/scss/desk/list.scss index 7fe04338ee..1818f6d8b3 100644 --- a/frappe/public/scss/desk/list.scss +++ b/frappe/public/scss/desk/list.scss @@ -30,6 +30,15 @@ } } +.list-skeleton { + min-height: calc(100vh - 200px); + + .list-skeleton-box { + background-color: var(--skeleton-bg); + height: 100%; + border-radius: var(--border-radius); + } +} .no-list-sidebar { &[data-page-route^="List/"], [data-page-route^="List/"]{ @@ -131,7 +140,7 @@ } } - .select-like { + .select-like, .file-select { padding: 15px 0px 15px 15px; } } @@ -169,7 +178,7 @@ $level-margin-right: 8px; color: var(--text-color); } - .level-item { + .level-item:not(.file-select) { margin-right: $level-margin-right; } diff --git a/frappe/public/scss/desk/mobile.scss b/frappe/public/scss/desk/mobile.scss index 839fca9bd2..14bee62c74 100644 --- a/frappe/public/scss/desk/mobile.scss +++ b/frappe/public/scss/desk/mobile.scss @@ -1,9 +1,4 @@ -html { - min-height: 100%; -} - body { - height: 100%; // The html and body elements cannot have any padding or margin. margin: 0px; padding: 0px !important; @@ -207,8 +202,12 @@ body { } // listviews - .list-row { - padding: 13px 15px !important; + .select-like { + margin-right: unset !important; + } + + .list-count { + display: contents; } .doclist-row { diff --git a/frappe/public/scss/login.bundle.scss b/frappe/public/scss/login.bundle.scss index 25fc6662e3..17f33b0a67 100644 --- a/frappe/public/scss/login.bundle.scss +++ b/frappe/public/scss/login.bundle.scss @@ -4,12 +4,17 @@ body { background-color: var(--bg-light-gray); } -.for-login, .for-forgot, .for-signup, .for-email-login { display: none; - margin: 70px 0; +} + +.for-login, +.for-forgot, +.for-signup, +.for-email-login { + padding: max(15vh, 70px) 0; @include media-breakpoint-up(sm) { .page-card { diff --git a/frappe/public/scss/website/footer.scss b/frappe/public/scss/website/footer.scss index 5208afaa11..dc73fd180e 100644 --- a/frappe/public/scss/website/footer.scss +++ b/frappe/public/scss/website/footer.scss @@ -85,4 +85,15 @@ .form-control { border: none; font-size: var(--text-md); +} + +.footer-logo-extension { + .input-group { + justify-content: flex-end; + #footer-subscribe-email, #footer-subscribe-button { + max-width: 300px; + border: 1px solid var(--dark-border-color); + box-shadow: none; + } + } } \ No newline at end of file diff --git a/frappe/public/scss/website/page_builder.scss b/frappe/public/scss/website/page_builder.scss index ff9f4ae1e6..252ad1bf9f 100644 --- a/frappe/public/scss/website/page_builder.scss +++ b/frappe/public/scss/website/page_builder.scss @@ -486,9 +486,12 @@ } .collapsible-content { + color: $gray-700; +} + +.collapsible-content p { margin-top: 1rem; margin-bottom: 0; - color: $gray-700; } .section-with-collapsible-content.align-center { diff --git a/frappe/public/scss/website/web_form.scss b/frappe/public/scss/website/web_form.scss index 32b1c46f84..6a6547d79e 100644 --- a/frappe/public/scss/website/web_form.scss +++ b/frappe/public/scss/website/web_form.scss @@ -3,6 +3,7 @@ .web-form-wrapper { .form-control { color: var(--text-color); + background-color: var(--control-bg); } .form-section { diff --git a/frappe/query_builder/__init__.py b/frappe/query_builder/__init__.py index f1503c88b8..6987ba24ab 100644 --- a/frappe/query_builder/__init__.py +++ b/frappe/query_builder/__init__.py @@ -1 +1,2 @@ -from frappe.query_builder.utils import get_query_builder, patch_query_execute +from pypika import * +from frappe.query_builder.utils import Column, get_query_builder, patch_query_execute diff --git a/frappe/query_builder/utils.py b/frappe/query_builder/utils.py index 67e2c392f3..c217d0975e 100644 --- a/frappe/query_builder/utils.py +++ b/frappe/query_builder/utils.py @@ -3,8 +3,10 @@ from typing import Any, Callable, Dict, get_type_hints from importlib import import_module from pypika import Query +from pypika.queries import Column import frappe + from .builder import MariaDB, Postgres diff --git a/frappe/search/full_text_search.py b/frappe/search/full_text_search.py index 104398b0ef..560ad55bf3 100644 --- a/frappe/search/full_text_search.py +++ b/frappe/search/full_text_search.py @@ -8,6 +8,8 @@ from whoosh.index import create_in, open_dir, EmptyIndexError from whoosh.fields import TEXT, ID, Schema from whoosh.qparser import MultifieldParser, FieldsPlugin, WildcardPlugin from whoosh.query import Prefix +from whoosh.writing import AsyncWriter + class FullTextSearch: """ Frappe Wrapper for Whoosh """ @@ -75,7 +77,7 @@ class FullTextSearch: ix = self.get_index() with ix.searcher(): - writer = ix.writer() + writer = AsyncWriter(ix) writer.delete_by_term(self.id, document[self.id]) writer.add_document(**document) writer.commit(optimize=True) @@ -135,4 +137,4 @@ class FullTextSearch: return out def get_index_path(index_name): - return frappe.get_site_path("indexes", index_name) \ No newline at end of file + return frappe.get_site_path("indexes", index_name) diff --git a/frappe/templates/includes/footer/footer_logo_extension.html b/frappe/templates/includes/footer/footer_logo_extension.html index 17f3218c45..87bb4d14af 100644 --- a/frappe/templates/includes/footer/footer_logo_extension.html +++ b/frappe/templates/includes/footer/footer_logo_extension.html @@ -1,13 +1,13 @@
-
+
{%- if footer_logo -%}
{%- endif -%}
-
+
{% block extension %} {% include "templates/includes/footer/footer_extension.html" %} {% endblock %} diff --git a/frappe/templates/includes/login/login.js b/frappe/templates/includes/login/login.js index 4400578862..6d5a2893ce 100644 --- a/frappe/templates/includes/login/login.js +++ b/frappe/templates/includes/login/login.js @@ -317,7 +317,7 @@ var continue_otp_app = function (setup, qrcode) { qrcode_div.append(direction); $('#otp_div').prepend(qrcode_div); } else { - direction = $('
').attr('id', 'qr_info').text('{{ _("OTP setup using OTP App was not completed. Please contact Administrator.") }}'); + direction = $('
').attr('id', 'qr_info').html('{{ _("OTP setup using OTP App was not completed. Please contact Administrator.") }}'); qrcode_div.append(direction); $('#otp_div').prepend(qrcode_div); } @@ -331,7 +331,7 @@ var continue_sms = function (setup, prompt) { sms_div.append(prompt) $('#otp_div').prepend(sms_div); } else { - direction = $('
').attr('id', 'qr_info').text(prompt || '{{ _("SMS was not sent. Please contact Administrator.") }}'); + direction = $('
').attr('id', 'qr_info').html(prompt || '{{ _("SMS was not sent. Please contact Administrator.") }}'); sms_div.append(direction); $('#otp_div').prepend(sms_div) } @@ -345,7 +345,7 @@ var continue_email = function (setup, prompt) { email_div.append(prompt) $('#otp_div').prepend(email_div); } else { - var direction = $('
').attr('id', 'qr_info').text(prompt || '{{ _("Verification code email not sent. Please contact Administrator.") }}'); + var direction = $('
').attr('id', 'qr_info').html(prompt || '{{ _("Verification code email not sent. Please contact Administrator.") }}'); email_div.append(direction); $('#otp_div').prepend(email_div); } diff --git a/frappe/templates/print_formats/standard_macros.html b/frappe/templates/print_formats/standard_macros.html index f8dc6c370c..55c76a00c2 100644 --- a/frappe/templates/print_formats/standard_macros.html +++ b/frappe/templates/print_formats/standard_macros.html @@ -135,11 +135,13 @@ data-fieldname="{{ df.fieldname }}" data-fieldtype="{{ df.fieldtype }}" {% elif df.fieldtype=="Check" and doc[df.fieldname] %} + {% elif df.fieldtype=="Check" and not doc[df.fieldname] %} + {% elif df.fieldtype in ("Image", "Attach Image") and frappe.utils.is_image(doc[doc.meta.get_field(df.fieldname).options]) %} = 3 %}{{ "" }} {%- elif df.align -%}{{ "text-" + df.align }} - {%- elif df.fieldtype in ("Int", "Float", "Currency", "Check", "Percent") -%}{{ "text-right" }} + {%- elif df.fieldtype in ("Int", "Float", "Currency", "Percent") -%}{{ "text-right" }} + {%- elif df.fieldtype in ("Check") -%}{{ "text-center" }} {%- else -%}{{ "" }} {%- endif -%} {% endmacro %} diff --git a/frappe/tests/test_db_query.py b/frappe/tests/test_db_query.py index c18db29259..84d0d31616 100644 --- a/frappe/tests/test_db_query.py +++ b/frappe/tests/test_db_query.py @@ -4,6 +4,7 @@ import frappe, unittest from frappe.model.db_query import DatabaseQuery from frappe.desk.reportview import get_filters_cond +from frappe.query_builder import Column from frappe.core.page.permission_manager.permission_manager import update, reset, add from frappe.permissions import add_user_permission, clear_user_permissions_for_doctype @@ -373,6 +374,25 @@ class TestReportview(unittest.TestCase): owners = DatabaseQuery("DocType").execute(filters={"name": "DocType"}, pluck="owner") self.assertEqual(owners, ["Administrator"]) + def test_column_comparison(self): + """Test DatabaseQuery.execute to test column comparison + """ + users_unedited = frappe.get_all( + "User", + filters={"creation": Column("modified")}, + fields=["name", "creation", "modified"], + limit=1, + ) + users_edited = frappe.get_all( + "User", + filters={"creation": ("!=", Column("modified"))}, + fields=["name", "creation", "modified"], + limit=1, + ) + + self.assertEqual(users_unedited[0].modified, users_unedited[0].creation) + self.assertNotEqual(users_edited[0].modified, users_edited[0].creation) + def test_reportview_get(self): user = frappe.get_doc("User", "test@example.com") add_child_table_to_blog_post() diff --git a/frappe/utils/backups.py b/frappe/utils/backups.py index b2b5ae8237..461ac7cb3d 100644 --- a/frappe/utils/backups.py +++ b/frappe/utils/backups.py @@ -374,7 +374,8 @@ class BackupGenerator: backup_info = ("Skipping Tables: ", ", ".join(self.backup_excludes)) if self.partial: - print(''.join(backup_info), "\n") + if self.verbose: + print(''.join(backup_info), "\n") database_header_content.extend([ f"Partial Backup of Frappe Site {frappe.local.site}", ("Backup contains: " if self.backup_includes else "Backup excludes: ") + backup_info[1], diff --git a/frappe/utils/password.py b/frappe/utils/password.py index 8f1670e9bd..b5218806c8 100644 --- a/frappe/utils/password.py +++ b/frappe/utils/password.py @@ -4,13 +4,16 @@ import string import frappe from frappe import _ +from frappe.query_builder import Table from frappe.utils import cstr, encode from cryptography.fernet import Fernet, InvalidToken from passlib.hash import pbkdf2_sha256, mysql41 from passlib.registry import register_crypt_handler from passlib.context import CryptContext -from pymysql.constants.ER import DATA_TOO_LONG -from psycopg2.errorcodes import STRING_DATA_RIGHT_TRUNCATION +from pypika.terms import Values + +Auth = Table("__Auth") + class LegacyPassword(pbkdf2_sha256): name = "frappe_legacy" @@ -38,28 +41,46 @@ passlibctx = CryptContext( ) -def get_decrypted_password(doctype, name, fieldname='password', raise_exception=True): - auth = frappe.db.sql('''select `password` from `__Auth` - where doctype=%(doctype)s and name=%(name)s and fieldname=%(fieldname)s and encrypted=1''', - { 'doctype': doctype, 'name': name, 'fieldname': fieldname }) +def get_decrypted_password(doctype, name, fieldname="password", raise_exception=True): + result = ( + frappe.qb.from_(Auth) + .select(Auth.password) + .where( + (Auth.doctype == doctype) + & (Auth.name == name) + & (Auth.fieldname == fieldname) + & (Auth.encrypted == 1) + ) + .limit(1) + ).run() - if auth and auth[0][0]: - return decrypt(auth[0][0]) + if result and result[0][0]: + return decrypt(result[0][0]) elif raise_exception: - frappe.throw(_('Password not found'), frappe.AuthenticationError) + frappe.throw(_("Password not found"), frappe.AuthenticationError) -def set_encrypted_password(doctype, name, pwd, fieldname='password'): +def set_encrypted_password(doctype, name, pwd, fieldname="password"): + query = ( + frappe.qb.into(Auth) + .columns(Auth.doctype, Auth.name, Auth.fieldname, Auth.password, Auth.encrypted) + .insert(doctype, name, fieldname, encrypt(pwd), 1) + ) + + # TODO: Simplify this via aliasing methods in `frappe.qb` + if frappe.db.db_type == "mariadb": + query = query.on_duplicate_key_update(Auth.password, Values(Auth.password)) + elif frappe.db.db_type == "postgres": + query = ( + query.on_conflict(Auth.doctype, Auth.name, Auth.fieldname).do_update(Auth.password) + ) + try: - frappe.db.sql("""insert into `__Auth` (doctype, name, fieldname, `password`, encrypted) - values (%(doctype)s, %(name)s, %(fieldname)s, %(pwd)s, 1) - {on_duplicate_update} `password`=%(pwd)s, encrypted=1""".format( - on_duplicate_update=frappe.db.get_on_duplicate_update(['doctype', 'name', 'fieldname']) - ), { 'doctype': doctype, 'name': name, 'fieldname': fieldname, 'pwd': encrypt(pwd) }) + query.run() + except frappe.db.DataError as e: - if ((frappe.db.db_type == 'mariadb' and e.args[0] == DATA_TOO_LONG) or - (frappe.db.db_type == 'postgres' and e.pgcode == STRING_DATA_RIGHT_TRUNCATION)): + if frappe.db.is_data_too_long(e): frappe.throw(_("Most probably your password is too long."), exc=e) raise e @@ -71,34 +92,44 @@ def remove_encrypted_password(doctype, name, fieldname='password'): "fieldname": fieldname }) -def check_password(user, pwd, doctype='User', fieldname='password', delete_tracker_cache=True): - '''Checks if user and password are correct, else raises frappe.AuthenticationError''' +def check_password(user, pwd, doctype="User", fieldname="password", delete_tracker_cache=True): + """Checks if user and password are correct, else raises frappe.AuthenticationError""" - auth = frappe.db.sql("""select `name`, `password` from `__Auth` - where `doctype`=%(doctype)s and `name`=%(name)s and `fieldname`=%(fieldname)s and `encrypted`=0""", - {'doctype': doctype, 'name': user, 'fieldname': fieldname}, as_dict=True) + result = ( + frappe.qb.from_(Auth) + .select(Auth.name, Auth.password) + .where( + (Auth.doctype == doctype) + & (Auth.name == user) + & (Auth.fieldname == fieldname) + & (Auth.encrypted == 0) + ) + .limit(1) + .run(as_dict=True) + ) - if not auth or not passlibctx.verify(pwd, auth[0].password): - raise frappe.AuthenticationError(_('Incorrect User or Password')) + if not result or not passlibctx.verify(pwd, result[0].password): + raise frappe.AuthenticationError(_("Incorrect User or Password")) # lettercase agnostic - user = auth[0].name + user = result[0].name # TODO: This need to be deleted after checking side effects of it. # We have a `LoginAttemptTracker` that can take care of tracking related cache. if delete_tracker_cache: delete_login_failed_cache(user) - if not passlibctx.needs_update(auth[0].password): + if not passlibctx.needs_update(result[0].password): update_password(user, pwd, doctype, fieldname) return user def delete_login_failed_cache(user): - frappe.cache().hdel('last_login_tried', user) - frappe.cache().hdel('login_failed_count', user) - frappe.cache().hdel('locked_account_time', user) + frappe.cache().hdel("last_login_tried", user) + frappe.cache().hdel("login_failed_count", user) + frappe.cache().hdel("locked_account_time", user) + def update_password(user, pwd, doctype='User', fieldname='password', logout_all_sessions=False): ''' @@ -111,17 +142,27 @@ def update_password(user, pwd, doctype='User', fieldname='password', logout_all_ :param logout_all_session: delete all other session ''' hashPwd = passlibctx.hash(pwd) - frappe.db.multisql({ - "mariadb": """INSERT INTO `__Auth` - (`doctype`, `name`, `fieldname`, `password`, `encrypted`) - VALUES (%(doctype)s, %(name)s, %(fieldname)s, %(pwd)s, 0) - ON DUPLICATE key UPDATE `password`=%(pwd)s, encrypted=0""", - "postgres": """INSERT INTO `__Auth` - (`doctype`, `name`, `fieldname`, `password`, `encrypted`) - VALUES (%(doctype)s, %(name)s, %(fieldname)s, %(pwd)s, 0) - ON CONFLICT("name", "doctype", "fieldname") DO UPDATE - SET `password`=%(pwd)s, encrypted=0""", - }, {'doctype': doctype, 'name': user, 'fieldname': fieldname, 'pwd': hashPwd}) + + query = ( + frappe.qb.into(Auth) + .columns(Auth.doctype, Auth.name, Auth.fieldname, Auth.password, Auth.encrypted) + .insert(doctype, user, fieldname, hashPwd, 0) + ) + + # TODO: Simplify this via aliasing methods in `frappe.qb` + if frappe.db.db_type == "mariadb": + query = ( + query.on_duplicate_key_update(Auth.password, hashPwd) + .on_duplicate_key_update(Auth.encrypted, 0) + ) + elif frappe.db.db_type == "postgres": + query = ( + query.on_conflict(Auth.doctype, Auth.name, Auth.fieldname) + .do_update(Auth.password, hashPwd) + .do_update(Auth.encrypted, 0) + ) + + query.run() # clear all the sessions except current if logout_all_sessions: @@ -142,15 +183,17 @@ def delete_all_passwords_for(doctype, name): def rename_password(doctype, old_name, new_name): # NOTE: fieldname is not considered, since the document is renamed - frappe.db.sql("""update `__Auth` set name=%(new_name)s - where doctype=%(doctype)s and name=%(old_name)s""", - { 'doctype': doctype, 'new_name': new_name, 'old_name': old_name }) + frappe.qb.update(Auth).set(Auth.name, new_name).where( + (Auth.doctype == doctype) + & (Auth.name == old_name) + ).run() def rename_password_field(doctype, old_fieldname, new_fieldname): - frappe.db.sql('''update `__Auth` set fieldname=%(new_fieldname)s - where doctype=%(doctype)s and fieldname=%(old_fieldname)s''', - { 'doctype': doctype, 'old_fieldname': old_fieldname, 'new_fieldname': new_fieldname }) + frappe.qb.update(Auth).set(Auth.fieldname, new_fieldname).where( + (Auth.doctype == doctype) + & (Auth.fieldname == old_fieldname) + ).run() def create_auth_table(): @@ -180,18 +223,23 @@ def decrypt(txt, encryption_key=None): return plain_text except InvalidToken: # encryption_key in site_config is changed and not valid - frappe.throw(_('Encryption key is invalid' + '!' if encryption_key else ', please check site_config.json.')) + frappe.throw( + _("Encryption key is invalid") + "!" + if encryption_key + else _(", please check site_config.json.") + ) def get_encryption_key(): from frappe.installer import update_site_config - if 'encryption_key' not in frappe.local.conf: + if "encryption_key" not in frappe.local.conf: encryption_key = Fernet.generate_key().decode() - update_site_config('encryption_key', encryption_key) + update_site_config("encryption_key", encryption_key) frappe.local.conf.encryption_key = encryption_key return frappe.local.conf.encryption_key + def get_password_reset_limit(): return frappe.db.get_single_value("System Settings", "password_reset_limit") or 0 diff --git a/frappe/website/web_template/section_with_collapsible_content/section_with_collapsible_content.html b/frappe/website/web_template/section_with_collapsible_content/section_with_collapsible_content.html index 355c25001c..84a009564c 100644 --- a/frappe/website/web_template/section_with_collapsible_content/section_with_collapsible_content.html +++ b/frappe/website/web_template/section_with_collapsible_content/section_with_collapsible_content.html @@ -21,7 +21,9 @@
- {{ frappe.utils.md_to_html(item.content) }} +
+ {{ frappe.utils.md_to_html(item.content) }} +
{%- endfor -%} diff --git a/frappe/www/login.html b/frappe/www/login.html index 90e235e679..4c0c54d76f 100644 --- a/frappe/www/login.html +++ b/frappe/www/login.html @@ -154,11 +154,11 @@
{%- else -%} -
+
{{_("Signup Disabled")}} + +
-

{{_("Signups have been disabled for this website.")}}

- {%- endif -%}
diff --git a/frappe/www/message.html b/frappe/www/message.html index 2919592901..db9deeba1f 100644 --- a/frappe/www/message.html +++ b/frappe/www/message.html @@ -39,8 +39,8 @@ html, body { {% endif %}