From 77895bbcfc3efa8614ba011732f27d5f69faa386 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 3 Aug 2021 12:43:01 +0530 Subject: [PATCH 01/82] feat: Support for Column comparison in DatabaseQuery --- frappe/model/db_query.py | 7 ++++++- frappe/query_builder/__init__.py | 2 +- frappe/query_builder/utils.py | 17 +++++++++++++++++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py index 7ed681644f..f007c4874b 100644 --- a/frappe/model/db_query.py +++ b/frappe/model/db_query.py @@ -3,6 +3,7 @@ """build query for doclistview and return results""" import frappe.defaults +from frappe.query_builder.utils import Column import frappe.share from frappe import _ import frappe.permissions @@ -546,8 +547,12 @@ class DatabaseQuery(object): value = flt(f.value) fallback = 0 + if isinstance(f.value, Column): + quote = '"' if frappe.conf.db_type == 'postgres' else "`" + value = f"{tname}.{quote}{f.value}{quote}" + # escape value - if isinstance(value, str) and not f.operator.lower() == 'between': + elif isinstance(value, str) and not f.operator.lower() == 'between': value = f"{frappe.db.escape(value, percent=False)}" if ( diff --git a/frappe/query_builder/__init__.py b/frappe/query_builder/__init__.py index 798c34b6cc..cf39550100 100644 --- a/frappe/query_builder/__init__.py +++ b/frappe/query_builder/__init__.py @@ -1 +1 @@ -from frappe.query_builder.utils import get_query_builder +from frappe.query_builder.utils import Column, Data, get_query_builder diff --git a/frappe/query_builder/utils.py b/frappe/query_builder/utils.py index b52a3606e8..30e48f7e60 100644 --- a/frappe/query_builder/utils.py +++ b/frappe/query_builder/utils.py @@ -4,9 +4,26 @@ from typing import Any, Callable, Dict from pypika import Query import frappe + from .builder import MariaDB, Postgres +class Column: + """Represents a Database Column""" + def __init__(self, name) -> None: + self.name = name + + def __str__(self) -> str: + return self.name + +class Data: + """Represents a Data value...Specifically non column types""" + def __init__(self, name) -> None: + self.name = name + + def __str__(self) -> str: + return self.name + class db_type_is(Enum): MARIADB = "mariadb" POSTGRES = "postgres" From f0cc541a716ca892f4772c022e8f4ac87fb38da0 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 17 Aug 2021 12:36:54 +0530 Subject: [PATCH 02/82] feat: Add util to convert compressed tables to DYNAMIC NOTE: This shouldn't be considered as a finished command. Only run this if you understand what you're doing --- frappe/commands/utils.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py index f2395ae490..54ee559cf5 100644 --- a/frappe/commands/utils.py +++ b/frappe/commands/utils.py @@ -512,6 +512,29 @@ def console(context): IPython.embed(display_banner="", header="", colors="neutral") +@click.command('convert-database') +@pass_context +def convert_database(context): + "convert row_formats to DNAMIC from older formats -- innodb mariadb v10.6.3" + site = get_site(context) + frappe.init(site=site) + frappe.connect() + + information_schema = frappe.qb.Schema("information_schema") + queried_tables = frappe.qb.from_( + information_schema.tables + ).select("table_name").where( + information_schema.tables.row_format=="Compressed" + ).run() + tables = [x[0] for x in queried_tables] + + for table in tables: + frappe.db.sql(f"ALTER TABLE `{table}` ROW_FORMAT=DYNAMIC") + + frappe.db.commit() + frappe.destroy() + + @click.command('run-tests') @click.option('--app', help="For App") @click.option('--doctype', help="For DocType") @@ -796,6 +819,7 @@ commands = [ build, clear_cache, clear_website_cache, + convert_database, jupyter, console, destroy_all_sessions, From f8826750d4016c606eebb55ea7bf5fbde5b734d5 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 17 Aug 2021 12:38:19 +0530 Subject: [PATCH 03/82] chore: Use DYNAMIC row_format instead of deprecated COMPRESSED refs: * MariaDB 10.6 gets rid of the COMPRESSED row_format * https://dev.mysql.com/worklog/task/?id=8307 * https://stackoverflow.com/questions/24321896/mysql-row-format-compressed-vs-dynamic --- frappe/database/mariadb/database.py | 2 +- frappe/database/mariadb/framework_mariadb.sql | 22 +++++++++---------- frappe/database/mariadb/schema.py | 2 +- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index d4a119804b..9fc485e133 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -195,7 +195,7 @@ class MariaDBDatabase(Database): `password` TEXT NOT NULL, `encrypted` INT(1) NOT NULL DEFAULT 0, PRIMARY KEY (`doctype`, `name`, `fieldname`) - ) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci""") + ) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci""") def create_global_search_table(self): if not '__global_search' in self.get_tables(): diff --git a/frappe/database/mariadb/framework_mariadb.sql b/frappe/database/mariadb/framework_mariadb.sql index f8841e9417..426fae6d3e 100644 --- a/frappe/database/mariadb/framework_mariadb.sql +++ b/frappe/database/mariadb/framework_mariadb.sql @@ -71,7 +71,7 @@ CREATE TABLE `tabDocField` ( KEY `label` (`label`), KEY `fieldtype` (`fieldtype`), KEY `fieldname` (`fieldname`) -) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -- @@ -108,7 +108,7 @@ CREATE TABLE `tabDocPerm` ( `email` int(1) NOT NULL DEFAULT 1, PRIMARY KEY (`name`), KEY `parent` (`parent`) -) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -- -- Table structure for table `tabDocType Action` @@ -132,7 +132,7 @@ CREATE TABLE `tabDocType Action` ( PRIMARY KEY (`name`), KEY `parent` (`parent`), KEY `modified` (`modified`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci ROW_FORMAT=COMPRESSED; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci ROW_FORMAT=DYNAMIC; -- -- Table structure for table `tabDocType Action` @@ -155,7 +155,7 @@ CREATE TABLE `tabDocType Link` ( PRIMARY KEY (`name`), KEY `parent` (`parent`), KEY `modified` (`modified`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci ROW_FORMAT=COMPRESSED; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci ROW_FORMAT=DYNAMIC; -- -- Table structure for table `tabDocType` @@ -226,7 +226,7 @@ CREATE TABLE `tabDocType` ( `sender_field` varchar(255) DEFAULT NULL, PRIMARY KEY (`name`), KEY `parent` (`parent`) -) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -- -- Table structure for table `tabSeries` @@ -237,7 +237,7 @@ CREATE TABLE `tabSeries` ( `name` varchar(100), `current` int(10) NOT NULL DEFAULT 0, PRIMARY KEY(`name`) -) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -- @@ -254,7 +254,7 @@ CREATE TABLE `tabSessions` ( `device` varchar(255) DEFAULT 'desktop', `status` varchar(20) DEFAULT NULL, KEY `sid` (`sid`) -) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -- @@ -267,7 +267,7 @@ CREATE TABLE `tabSingles` ( `field` varchar(255) DEFAULT NULL, `value` text, KEY `singles_doctype_field_index` (`doctype`, `field`) -) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -- -- Table structure for table `__Auth` @@ -281,7 +281,7 @@ CREATE TABLE `__Auth` ( `password` TEXT NOT NULL, `encrypted` INT(1) NOT NULL DEFAULT 0, PRIMARY KEY (`doctype`, `name`, `fieldname`) -) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -- -- Table structure for table `tabFile` @@ -309,7 +309,7 @@ CREATE TABLE `tabFile` ( KEY `parent` (`parent`), KEY `attached_to_name` (`attached_to_name`), KEY `attached_to_doctype` (`attached_to_doctype`) -) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -- -- Table structure for table `tabDefaultValue` @@ -332,4 +332,4 @@ CREATE TABLE `tabDefaultValue` ( PRIMARY KEY (`name`), KEY `parent` (`parent`), KEY `defaultvalue_parent_defkey_index` (`parent`,`defkey`) -) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; diff --git a/frappe/database/mariadb/schema.py b/frappe/database/mariadb/schema.py index b40af59286..c5091cfe8f 100644 --- a/frappe/database/mariadb/schema.py +++ b/frappe/database/mariadb/schema.py @@ -29,7 +29,7 @@ class MariaDBTable(DBTable): %sindex parent(parent), index modified(modified)) ENGINE={engine} - ROW_FORMAT=COMPRESSED + ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci""".format(varchar_len=frappe.db.VARCHAR_LEN, engine=self.meta.get("engine") or 'InnoDB') % (self.table_name, add_text)) From 7f338edca11fb367e6126e5c9f0e33112965c9f4 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 17 Aug 2021 12:50:11 +0530 Subject: [PATCH 04/82] feat: Command to trim tables Maintenance command to get rid of extra columns in your DocType tables. These columns are remnants of "deleted" fields through customizations or upgrades --- frappe/commands/site.py | 28 +++++++++++++++- frappe/database/mariadb/database.py | 4 +-- frappe/model/meta.py | 50 ++++++++++++++++++++--------- 3 files changed, 63 insertions(+), 19 deletions(-) diff --git a/frappe/commands/site.py b/frappe/commands/site.py index 9098e31738..27e8e737ea 100755 --- a/frappe/commands/site.py +++ b/frappe/commands/site.py @@ -738,6 +738,31 @@ def build_search_index(context): finally: frappe.destroy() +@click.command('trim-tables') +@click.option('--dry-run', is_flag=True, default=False, help='Show what would be deleted') +@click.option('--format', default='table', type=click.Choice(['json', 'table']), help='Output format') +@pass_context +def trim_tables(context, dry_run, format): + from frappe.model.meta import trim_tables + + for site in context.sites: + frappe.init(site=site) + frappe.connect() + try: + trimmed_data = trim_tables(dry_run=dry_run) + handle_data(trimmed_data, format=format) + finally: + frappe.destroy() + +def handle_data(data: dict, format='json'): + if format == 'json': + import json + print(json.dumps({frappe.local.site: data}, indent=1, sort_keys=True)) + else: + click.secho(f"Site {frappe.local.site}", fg='green') + for table, columns in data.items(): + print(f"{table}: {', '.join(columns)}") + commands = [ add_system_manager, backup, @@ -766,5 +791,6 @@ commands = [ add_to_hosts, start_ngrok, build_search_index, - partial_restore + partial_restore, + trim_tables, ] diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index d4a119804b..0bf6450cda 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -135,8 +135,8 @@ class MariaDBDatabase(Database): table_name = get_table_name(doctype) return self.sql(f"DESC `{table_name}`") - def change_column_type(self, table: str, column: str, type: str) -> Union[List, Tuple]: - table_name = get_table_name(table) + def change_column_type(self, doctype: str, column: str, type: str) -> Union[List, Tuple]: + table_name = get_table_name(doctype) return self.sql(f"ALTER TABLE `{table_name}` MODIFY `{column}` {type} NOT NULL") # exception types diff --git a/frappe/model/meta.py b/frappe/model/meta.py index de794ba77f..f42cac59c3 100644 --- a/frappe/model/meta.py +++ b/frappe/model/meta.py @@ -15,6 +15,7 @@ Example: ''' from datetime import datetime +import click import frappe, json, os from frappe.utils import cstr, cint, cast_fieldtype from frappe.model import default_fields, no_value_fields, optional_fields, data_fieldtypes, table_fields @@ -658,27 +659,44 @@ def get_default_df(fieldname): fieldtype = "Data" ) -def trim_tables(doctype=None): +def trim_tables(doctype=None, dry_run=False): """ Removes database fields that don't exist in the doctype (json or custom field). This may be needed as maintenance since removing a field in a DocType doesn't automatically delete the db field. """ - ignore_fields = default_fields + optional_fields - - filters={ "issingle": 0 } + UPDATED_TABLES = {} + filters = {"issingle": 0} if doctype: filters["name"] = doctype - for doctype in frappe.db.get_all("DocType", filters=filters): - doctype = doctype.name - columns = frappe.db.get_table_columns(doctype) - fields = frappe.get_meta(doctype).get_fieldnames_with_value() - columns_to_remove = [f for f in list(set(columns) - set(fields)) if f not in ignore_fields - and not f.startswith("_")] - if columns_to_remove: - print(doctype, "columns removed:", columns_to_remove) - columns_to_remove = ", ".join("drop `{0}`".format(c) for c in columns_to_remove) - query = """alter table `tab{doctype}` {columns}""".format( - doctype=doctype, columns=columns_to_remove) - frappe.db.sql_ddl(query) + for doctype in frappe.db.get_all("DocType", filters=filters, pluck="name"): + try: + dropped_columns = trim_table(doctype, dry_run=dry_run) + if dropped_columns: + UPDATED_TABLES[doctype] = dropped_columns + except frappe.db.TableMissingError: + click.secho(f"Ignoring missing table for DocType: {doctype}", fg="yellow", err=True) + click.secho(f"Consider removing record in the DocType table for {doctype}", fg="yellow", err=True) + except Exception as e: + click.echo(e, err=True) + + return UPDATED_TABLES + + +def trim_table(doctype, dry_run=True): + ignore_fields = default_fields + optional_fields + columns = frappe.db.get_table_columns(doctype) + fields = frappe.get_meta(doctype, cached=False).get_fieldnames_with_value() + is_internal = lambda f: f not in ignore_fields and not f.startswith("_") + columns_to_remove = [ + f for f in list(set(columns) - set(fields)) if is_internal(f) + ] + DROPPED_COLUMNS = columns_to_remove[:] + + if columns_to_remove and not dry_run: + columns_to_remove = ", ".join(f"DROP `{c}`" for c in columns_to_remove) + frappe.db.sql_ddl(f"ALTER TABLE `tab{doctype}` {columns_to_remove}") + # frappe.clear_cache(doctype=doctype) + + return DROPPED_COLUMNS From 41b30b7442015e76f3996b94fcc61669a9a319d0 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 30 Aug 2021 12:26:18 +0530 Subject: [PATCH 05/82] feat: Handle site restores to MariaDB 10.6 --- frappe/installer.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/frappe/installer.py b/frappe/installer.py index d4d8117fcb..41fd7b1cf0 100755 --- a/frappe/installer.py +++ b/frappe/installer.py @@ -445,9 +445,21 @@ def extract_sql_from_archive(sql_file_path): else: decompressed_file_name = sql_file_path + # convert archive sql to latest compatible + convert_archive_content(decompressed_file_name) + return decompressed_file_name +def convert_archive_content(sql_file_path): + if frappe.conf.db_type == "mariadb": + # ever since mariaDB 10.6, row_format COMPRESSED has been deprecated and removed + # this step is added to ease restoring sites depending on older mariaDB servers + contents = open(sql_file_path).read() + with open(sql_file_path, "w") as f: + f.write(contents.replace("ROW_FORMAT=COMPRESSED", "ROW_FORMAT=DYNAMIC")) + + def extract_sql_gzip(sql_gz_path): import subprocess @@ -457,7 +469,7 @@ def extract_sql_gzip(sql_gz_path): decompressed_file = original_file.rstrip(".gz") cmd = 'gzip -dvf < {0} > {1}'.format(original_file, decompressed_file) subprocess.check_call(cmd, shell=True) - except: + except Exception: raise return decompressed_file From 3f2d0ac0cfc4b7ded6fad83c82d35e009d6cce8b Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 1 Sep 2021 13:37:01 +0530 Subject: [PATCH 06/82] feat(cli): New command 'db-console' Don't remember what db console you have to open for what site! How to use: `bench --site framework.io db-console` --- frappe/commands/utils.py | 44 +++++++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 12 deletions(-) diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py index 54ee559cf5..490671b66a 100644 --- a/frappe/commands/utils.py +++ b/frappe/commands/utils.py @@ -408,20 +408,47 @@ def bulk_rename(context, doctype, path): frappe.destroy() +@click.command('db-console') +@pass_context +def database(context): + """ + Enter into the Database console for given site. + """ + site = get_site(context) + if not site: + raise SiteNotSpecifiedError + frappe.init(site=site) + if not frappe.conf.db_type or frappe.conf.db_type == "mariadb": + _mariadb() + elif frappe.conf.db_type == "postgres": + _psql() + + @click.command('mariadb') @pass_context def mariadb(context): """ Enter into mariadb console for a given site. """ - import os - site = get_site(context) if not site: raise SiteNotSpecifiedError frappe.init(site=site) + _mariadb() - # This is assuming you're within the bench instance. + +@click.command('postgres') +@pass_context +def postgres(context): + """ + Enter into postgres console for a given site. + """ + site = get_site(context) + frappe.init(site=site) + _psql() + + +def _mariadb(): mysql = find_executable('mysql') os.execv(mysql, [ mysql, @@ -434,15 +461,7 @@ def mariadb(context): "-A"]) -@click.command('postgres') -@pass_context -def postgres(context): - """ - Enter into postgres console for a given site. - """ - site = get_site(context) - frappe.init(site=site) - # This is assuming you're within the bench instance. +def _psql(): psql = find_executable('psql') subprocess.run([ psql, '-d', frappe.conf.db_name]) @@ -819,6 +838,7 @@ commands = [ build, clear_cache, clear_website_cache, + database, convert_database, jupyter, console, From c472fd359f0c5f402a284c7dcc4c299e8eb5b893 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 2 Sep 2021 12:53:07 +0530 Subject: [PATCH 07/82] feat: Transform tables for given site * Ability to update ROW_FORMAT for mentioned tables in --table option * Failfast to stop on first error occured * Show progressbar for overall conversion progress (remaining tales) --- frappe/commands/utils.py | 61 +++++++++++++++++++++++++++++++--------- 1 file changed, 47 insertions(+), 14 deletions(-) diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py index 490671b66a..5ca6bd6d3f 100644 --- a/frappe/commands/utils.py +++ b/frappe/commands/utils.py @@ -531,26 +531,59 @@ def console(context): IPython.embed(display_banner="", header="", colors="neutral") -@click.command('convert-database') +@click.command('transform-database') +@click.option('--table', default="all") +@click.option('--row_format', default="DYNAMIC", type=click.Choice(["DYNAMIC", "COMPACT", "REDUNDANT", "COMPRESSED"])) +@click.option('--failfast', is_flag=True, default=False) @pass_context -def convert_database(context): - "convert row_formats to DNAMIC from older formats -- innodb mariadb v10.6.3" +def transform_database(context, table, row_format, failfast): + "Transform site database through given parameters" site = get_site(context) + check_table = [] + add_line = False + skipped = 0 frappe.init(site=site) + + if frappe.conf.db_type and frappe.conf.db_type != "mariadb": + click.secho("This command only has support for MariaDB databases at this point", fg="yellow") + sys.exit(1) + frappe.connect() - information_schema = frappe.qb.Schema("information_schema") - queried_tables = frappe.qb.from_( - information_schema.tables - ).select("table_name").where( - information_schema.tables.row_format=="Compressed" - ).run() - tables = [x[0] for x in queried_tables] + if table == "all": + information_schema = frappe.qb.Schema("information_schema") + queried_tables = frappe.qb.from_( + information_schema.tables + ).select("table_name").where( + (information_schema.tables.row_format != row_format) + & (information_schema.tables.table_schema == frappe.conf.db_name) + ).run() + tables = [x[0] for x in queried_tables] + else: + tables = [x.strip() for x in table.split(",")] - for table in tables: - frappe.db.sql(f"ALTER TABLE `{table}` ROW_FORMAT=DYNAMIC") + total = len(tables) + + for current, table in enumerate(tables): + try: + frappe.db.sql(f"ALTER TABLE `{table}` ROW_FORMAT={row_format}") + update_progress_bar("Updating table schema", current - skipped, total) + add_line = True + except Exception as e: + check_table.append([table, e.args]) + skipped += 1 + + if failfast: + break + + if add_line: + print() + + for errored_table in check_table: + table, err = errored_table + err_msg = f"{table}: ERROR {err[0]}: {err[1]}" + click.secho(err_msg, fg="yellow") - frappe.db.commit() frappe.destroy() @@ -839,7 +872,7 @@ commands = [ clear_cache, clear_website_cache, database, - convert_database, + transform_database, jupyter, console, destroy_all_sessions, From 6ccf73026fc23ee874e1e3a9cc02436871179517 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 2 Sep 2021 15:33:27 +0530 Subject: [PATCH 08/82] feat(minor): transform-tables - Update table engine --- frappe/commands/utils.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py index 5ca6bd6d3f..19bc355034 100644 --- a/frappe/commands/utils.py +++ b/frappe/commands/utils.py @@ -532,11 +532,12 @@ def console(context): @click.command('transform-database') -@click.option('--table', default="all") -@click.option('--row_format', default="DYNAMIC", type=click.Choice(["DYNAMIC", "COMPACT", "REDUNDANT", "COMPRESSED"])) +@click.option('--table', required=True) +@click.option('--engine', default=None, type=click.Choice(["InnoDB", "MyISAM"])) +@click.option('--row_format', default=None, type=click.Choice(["DYNAMIC", "COMPACT", "REDUNDANT", "COMPRESSED"])) @click.option('--failfast', is_flag=True, default=False) @pass_context -def transform_database(context, table, row_format, failfast): +def transform_database(context, table, engine, row_format, failfast): "Transform site database through given parameters" site = get_site(context) check_table = [] @@ -548,6 +549,10 @@ def transform_database(context, table, row_format, failfast): click.secho("This command only has support for MariaDB databases at this point", fg="yellow") sys.exit(1) + if not (engine or row_format): + click.secho("Values for `--engine` or `--row_format` must be set") + sys.exit(1) + frappe.connect() if table == "all": @@ -566,9 +571,15 @@ def transform_database(context, table, row_format, failfast): for current, table in enumerate(tables): try: - frappe.db.sql(f"ALTER TABLE `{table}` ROW_FORMAT={row_format}") + if engine: + frappe.db.sql(f"ALTER TABLE `{table}` ENGINE={engine}") + + if row_format: + frappe.db.sql(f"ALTER TABLE `{table}` ROW_FORMAT={row_format}") + update_progress_bar("Updating table schema", current - skipped, total) add_line = True + except Exception as e: check_table.append([table, e.args]) skipped += 1 From 71dc7f474ff695839de253d41b95bef11fa98ad2 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 2 Sep 2021 15:51:49 +0530 Subject: [PATCH 09/82] perf: Single query to update a table --- frappe/commands/utils.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py index 2dba0dd678..bad26944b6 100644 --- a/frappe/commands/utils.py +++ b/frappe/commands/utils.py @@ -583,13 +583,14 @@ def transform_database(context, table, engine, row_format, failfast): total = len(tables) for current, table in enumerate(tables): + values_to_set = "" + if engine: + values_to_set += f" ENGINE={engine}" + if row_format: + values_to_set += f" ROW_FORMAT={row_format}" + try: - if engine: - frappe.db.sql(f"ALTER TABLE `{table}` ENGINE={engine}") - - if row_format: - frappe.db.sql(f"ALTER TABLE `{table}` ROW_FORMAT={row_format}") - + frappe.db.sql(f"ALTER TABLE `{table}`{values_to_set}") update_progress_bar("Updating table schema", current - skipped, total) add_line = True From a1c8a7b5f2866a76ba62351711fa402f55e03622 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 2 Sep 2021 16:00:07 +0530 Subject: [PATCH 10/82] fix(restore): Initialize site before decompressing file --- frappe/commands/site.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frappe/commands/site.py b/frappe/commands/site.py index 9098e31738..9b0ecee896 100755 --- a/frappe/commands/site.py +++ b/frappe/commands/site.py @@ -67,6 +67,9 @@ def restore(context, sql_file_path, mariadb_root_username=None, mariadb_root_pas validate_database_sql ) + site = get_site(context) + frappe.init(site=site) + force = context.force or force decompressed_file_name = extract_sql_from_archive(sql_file_path) @@ -85,9 +88,6 @@ def restore(context, sql_file_path, mariadb_root_username=None, mariadb_root_pas # check if valid SQL file validate_database_sql(decompressed_file_name, _raise=not force) - site = get_site(context) - frappe.init(site=site) - # dont allow downgrading to older versions of frappe without force if not force and is_downgrade(decompressed_file_name, verbose=True): warn_message = ( From 1c1320c6844f37dfb6f93f5c511231c899a93d75 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 3 Sep 2021 18:59:29 +0530 Subject: [PATCH 11/82] fix: backup before trimming a site database --- frappe/commands/site.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/frappe/commands/site.py b/frappe/commands/site.py index 27e8e737ea..c0b6ac2187 100755 --- a/frappe/commands/site.py +++ b/frappe/commands/site.py @@ -741,13 +741,19 @@ def build_search_index(context): @click.command('trim-tables') @click.option('--dry-run', is_flag=True, default=False, help='Show what would be deleted') @click.option('--format', default='table', type=click.Choice(['json', 'table']), help='Output format') +@click.option('--no-backup', is_flag=True, default=False, help='Do not backup the site') @pass_context -def trim_tables(context, dry_run, format): +def trim_tables(context, dry_run, format, no_backup): from frappe.model.meta import trim_tables + from frappe.utils.backups import scheduled_backup for site in context.sites: frappe.init(site=site) frappe.connect() + + if not (no_backup or dry_run): + scheduled_backup(ignore_files=False, force=True) + try: trimmed_data = trim_tables(dry_run=dry_run) handle_data(trimmed_data, format=format) From 4127f0f135edc3ab25791697da45e26abdc320e4 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 3 Sep 2021 18:59:56 +0530 Subject: [PATCH 12/82] fix: Clear table columns cache to fetch accurate results --- frappe/model/meta.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/model/meta.py b/frappe/model/meta.py index f42cac59c3..ab3dd9ab97 100644 --- a/frappe/model/meta.py +++ b/frappe/model/meta.py @@ -685,6 +685,7 @@ def trim_tables(doctype=None, dry_run=False): def trim_table(doctype, dry_run=True): + frappe.cache().hdel('table_columns', f"tab{doctype}") ignore_fields = default_fields + optional_fields columns = frappe.db.get_table_columns(doctype) fields = frappe.get_meta(doctype, cached=False).get_fieldnames_with_value() @@ -697,6 +698,5 @@ def trim_table(doctype, dry_run=True): if columns_to_remove and not dry_run: columns_to_remove = ", ".join(f"DROP `{c}`" for c in columns_to_remove) frappe.db.sql_ddl(f"ALTER TABLE `tab{doctype}` {columns_to_remove}") - # frappe.clear_cache(doctype=doctype) return DROPPED_COLUMNS From 28d1d58944ac249a2e8893908a9055ee9dde8fde Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 3 Sep 2021 20:02:44 +0530 Subject: [PATCH 13/82] feat: bench trim-database Get rid of ghost tables in the site's database --- frappe/commands/site.py | 54 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/frappe/commands/site.py b/frappe/commands/site.py index c0b6ac2187..6f66c3628b 100755 --- a/frappe/commands/site.py +++ b/frappe/commands/site.py @@ -738,6 +738,59 @@ def build_search_index(context): finally: frappe.destroy() +@click.command('trim-database') +@click.option('--dry-run', is_flag=True, default=False, help='Show what would be deleted') +@click.option('--no-backup', is_flag=True, default=False, help='Do not backup the site') +@pass_context +def trim_database(context, dry_run, no_backup): + from frappe.utils.backups import scheduled_backup + + for site in context.sites: + frappe.init(site=site) + frappe.connect() + + TABLES_TO_DROPPED = [] + STANDARD_TABLES = get_standard_tables() + information_schema = frappe.qb.Schema("information_schema") + queried_result = frappe.qb.from_( + information_schema.tables + ).select("table_name").where( + information_schema.tables.table_schema == frappe.conf.db_name + ).run() + database_tables = [x[0] for x in queried_result] + doctype_tables = frappe.get_all("DocType", pluck="name") + + for x in database_tables: + doctype = x.lstrip("tab") + if not (doctype in doctype_tables or x.startswith("__") or x in STANDARD_TABLES): + TABLES_TO_DROPPED.append(x) + + if not TABLES_TO_DROPPED: + click.secho(f"No ghost tables found in {frappe.local.site}...Great!", fg="green") + else: + if not (no_backup or dry_run): + click.secho(f"Taking backup for {frappe.local.site}", fg="green") + odb = scheduled_backup(ignore_files=False, force=True) + odb.print_summary() + + for table in TABLES_TO_DROPPED: + print(f"* dropping Table '{table}'...") + if not dry_run: + frappe.db.sql_ddl(f"drop table `{table}`") + + frappe.destroy() + +def get_standard_tables(): + import re + tables = [] + sql_file = os.path.join("..", "apps", "frappe", "frappe", "database", frappe.conf.db_type, f'framework_{frappe.conf.db_type}.sql') + content = open(sql_file).read().splitlines() + for line in content: + beep = re.search("""CREATE TABLE ("|`)(.*)?("|`) \(""", line) + if beep: + tables.append(beep.group(2)) + return tables + @click.command('trim-tables') @click.option('--dry-run', is_flag=True, default=False, help='Show what would be deleted') @click.option('--format', default='table', type=click.Choice(['json', 'table']), help='Output format') @@ -799,4 +852,5 @@ commands = [ build_search_index, partial_restore, trim_tables, + trim_database, ] From b833e40b804fd1d62b5721d31db329e2390fc0c0 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 3 Sep 2021 20:04:15 +0530 Subject: [PATCH 14/82] fix: Show backup status in trim-tables command --- frappe/commands/site.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frappe/commands/site.py b/frappe/commands/site.py index 6f66c3628b..0cd965557c 100755 --- a/frappe/commands/site.py +++ b/frappe/commands/site.py @@ -805,7 +805,9 @@ def trim_tables(context, dry_run, format, no_backup): frappe.connect() if not (no_backup or dry_run): - scheduled_backup(ignore_files=False, force=True) + click.secho(f"Taking backup for {frappe.local.site}", fg="green") + odb = scheduled_backup(ignore_files=False, force=True) + odb.print_summary() try: trimmed_data = trim_tables(dry_run=dry_run) From 62593c49fb40b71f9189eab457d8f6012b82d282 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 3 Sep 2021 20:14:40 +0530 Subject: [PATCH 15/82] refactor: Rename arg table to doctype This change was made to be "more accurate" about how the internal magic handled --- frappe/database/postgres/database.py | 4 ++-- frappe/patches/v13_0/increase_password_length.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/frappe/database/postgres/database.py b/frappe/database/postgres/database.py index 00e60fb8d2..b4dd7fa9e5 100644 --- a/frappe/database/postgres/database.py +++ b/frappe/database/postgres/database.py @@ -182,8 +182,8 @@ class PostgresDatabase(Database): table_name = get_table_name(doctype) return self.sql(f"SELECT COLUMN_NAME FROM information_schema.COLUMNS WHERE TABLE_NAME = '{table_name}'") - def change_column_type(self, table: str, column: str, type: str) -> Union[List, Tuple]: - table_name = get_table_name(table) + def change_column_type(self, doctype: str, column: str, type: str) -> Union[List, Tuple]: + table_name = get_table_name(doctype) return self.sql(f'ALTER TABLE "{table_name}" ALTER COLUMN "{column}" TYPE {type}') def create_auth_table(self): diff --git a/frappe/patches/v13_0/increase_password_length.py b/frappe/patches/v13_0/increase_password_length.py index 62ca2ed779..deb7d7e98a 100644 --- a/frappe/patches/v13_0/increase_password_length.py +++ b/frappe/patches/v13_0/increase_password_length.py @@ -1,4 +1,4 @@ import frappe def execute(): - frappe.db.change_column_type(table="__Auth", column="password", type="TEXT") + frappe.db.change_column_type("__Auth", column="password", type="TEXT") From 46c617afde8dd3792b78cb35ba17e061ad93daaf Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Mon, 6 Sep 2021 11:28:32 +0530 Subject: [PATCH 16/82] fix(UI): Checked icon not visible in PDF --- frappe/templates/print_formats/standard_macros.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/templates/print_formats/standard_macros.html b/frappe/templates/print_formats/standard_macros.html index f8dc6c370c..ec60af1ce0 100644 --- a/frappe/templates/print_formats/standard_macros.html +++ b/frappe/templates/print_formats/standard_macros.html @@ -135,7 +135,7 @@ data-fieldname="{{ df.fieldname }}" data-fieldtype="{{ df.fieldtype }}" {% elif df.fieldtype=="Check" and doc[df.fieldname] %} From c8d1c26ac159dcc83a39447ef6773f72934c6603 Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Tue, 7 Sep 2021 16:42:52 +0530 Subject: [PATCH 17/82] fix: Dashboard Setting already exist error --- .../js/frappe/views/workspace/workspace.js | 83 +++++++++---------- 1 file changed, 40 insertions(+), 43 deletions(-) diff --git a/frappe/public/js/frappe/views/workspace/workspace.js b/frappe/public/js/frappe/views/workspace/workspace.js index 1e0143c2a8..46aa0d389d 100644 --- a/frappe/public/js/frappe/views/workspace/workspace.js +++ b/frappe/public/js/frappe/views/workspace/workspace.js @@ -35,42 +35,6 @@ frappe.views.Workspace = class Workspace { 'My Workspaces', 'Public' ]; - this.tools = { - header: { - class: this.blocks['header'], - inlineToolbar: true - }, - paragraph: { - class: this.blocks['paragraph'], - inlineToolbar: true - }, - chart: { - class: this.blocks['chart'], - config: { - page_data: this.page_data || [] - } - }, - card: { - class: this.blocks['card'], - config: { - page_data: this.page_data || [] - } - }, - shortcut: { - class: this.blocks['shortcut'], - config: { - page_data: this.page_data || [] - } - }, - onboarding: { - class: this.blocks['onboarding'], - config: { - page_data: this.page_data || [] - } - }, - spacer: this.blocks['spacer'], - spacingTune: frappe.wspace_block.tunes['spacing_tune'], - }; this.prepare_container(); this.setup_pages(); @@ -86,7 +50,7 @@ frappe.views.Workspace = class Workspace { this.body = this.wrapper.find(".layout-main-section"); } - setup_pages() { + setup_pages(reload) { this.get_pages().then(pages => { this.all_pages = pages.pages; this.has_access = pages.has_access; @@ -115,7 +79,7 @@ frappe.views.Workspace = class Workspace { this.new_page = null; } this.make_sidebar(); - frappe.router.route(); + reload && this.show(); } }); } @@ -236,10 +200,7 @@ frappe.views.Workspace = class Workspace { return; } - let page = { - name: this.get_page_to_show().name, - public: this.get_page_to_show().public - }; + let page = this.get_page_to_show(); this.page.set_title(`${__(page.name)}`); this.show_page(page); @@ -670,6 +631,42 @@ frappe.views.Workspace = class Workspace { } initialize_editorjs(blocks) { + this.tools = { + header: { + class: this.blocks['header'], + inlineToolbar: true + }, + paragraph: { + class: this.blocks['paragraph'], + inlineToolbar: true + }, + chart: { + class: this.blocks['chart'], + config: { + page_data: this.page_data || [] + } + }, + card: { + class: this.blocks['card'], + config: { + page_data: this.page_data || [] + } + }, + shortcut: { + class: this.blocks['shortcut'], + config: { + page_data: this.page_data || [] + } + }, + onboarding: { + class: this.blocks['onboarding'], + config: { + page_data: this.page_data || [] + } + }, + spacer: this.blocks['spacer'], + spacingTune: frappe.wspace_block.tunes['spacing_tune'], + }; this.editor = new EditorJS({ data: { blocks: blocks || [] @@ -751,7 +748,7 @@ frappe.views.Workspace = class Workspace { reload() { this.$page.prepend(frappe.render_template('workspace_loading_skeleton')); this.$page.find('.codex-editor').addClass('hidden'); - this.setup_pages(); + this.setup_pages(true); this.undo.readOnly = true; } }; From a69020f2f3dd15d5dc10811567c10706556916aa Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Tue, 7 Sep 2021 19:01:41 +0530 Subject: [PATCH 18/82] fix: Page load optimization --- frappe/desk/doctype/workspace/workspace.py | 6 ++--- .../js/frappe/views/workspace/workspace.js | 27 ++++++++++++++----- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/frappe/desk/doctype/workspace/workspace.py b/frappe/desk/doctype/workspace/workspace.py index 25dd9b26d2..a0a22a43fc 100644 --- a/frappe/desk/doctype/workspace/workspace.py +++ b/frappe/desk/doctype/workspace/workspace.py @@ -208,17 +208,17 @@ def save_page(title, icon, parent, public, sb_public_items, sb_private_items, de if loads(deleted_pages): return delete_pages(loads(deleted_pages)) - return {"name": title, "public": public} + return {"name": title, "public": public, "label": doc.label} def delete_pages(deleted_pages): for page in deleted_pages: if page.get("public") and "Workspace Manager" not in frappe.get_roles(): - return {"name": page.get("title"), "public": 1} + return {"name": page.get("title"), "public": 1, "label": page.get("label")} if frappe.db.exists("Workspace", page.get("name")): frappe.get_doc("Workspace", page.get("name")).delete(ignore_permissions=True) - return {"name": "Home", "public": 1} + return {"name": "Home", "public": 1, "label": "Home"} def sort_pages(sb_public_items, sb_private_items): wspace_public_pages = get_page_list(['name', 'title'], {'public': 1}) diff --git a/frappe/public/js/frappe/views/workspace/workspace.js b/frappe/public/js/frappe/views/workspace/workspace.js index 46aa0d389d..36b2988935 100644 --- a/frappe/public/js/frappe/views/workspace/workspace.js +++ b/frappe/public/js/frappe/views/workspace/workspace.js @@ -23,6 +23,7 @@ frappe.views.Workspace = class Workspace { this.blocks = frappe.wspace_block.blocks; this.is_read_only = true; this.new_page = null; + this.pages = {}; this.sorted_public_items = []; this.sorted_private_items = []; this.deleted_sidebar_items = []; @@ -211,6 +212,11 @@ frappe.views.Workspace = class Workspace { page: page }).then(data => { this.page_data = data; + + // caching page data + this.pages[page.name] && delete this.pages[page.name]; + this.pages[page.name] = data; + if (!this.page_data || Object.keys(this.page_data).length === 0) return; return frappe.dashboard_utils.get_dashboard_settings().then(settings => { @@ -221,6 +227,7 @@ frappe.views.Workspace = class Workspace { chart.chart_settings = chart_config[chart.chart_name] || {}; }); } + this.pages[page.name] = this.page_data; } }); }); @@ -242,7 +249,7 @@ frappe.views.Workspace = class Workspace { return { name: page, public: is_public }; } - show_page(page) { + async show_page(page) { let section = this.current_page.public ? 'public' : 'private'; if (this.sidebar_items && this.sidebar_items[section] && this.sidebar_items[section][this.current_page.name]) { this.sidebar_items[section][this.current_page.name][0].firstElementChild.classList.remove("selected"); @@ -277,12 +284,17 @@ frappe.views.Workspace = class Workspace { this.add_custom_cards_in_content(); $('.item-anchor').addClass('disable-click'); - this.get_data(this_page).then(() => { - this.prepare_editorjs(); - $('.item-anchor').removeClass('disable-click'); - this.$page.find('.codex-editor').removeClass('hidden'); - this.$page.find('.workspace-skeleton').remove(); - }); + + if (this.pages && this.pages[this_page.name]) { + this.page_data = this.pages[this_page.name]; + } else { + await this.get_data(this_page); + } + + this.prepare_editorjs(); + $('.item-anchor').removeClass('disable-click'); + this.$page.find('.codex-editor').removeClass('hidden'); + this.$page.find('.workspace-skeleton').remove(); } } @@ -727,6 +739,7 @@ frappe.views.Workspace = class Workspace { frappe.dom.unfreeze(); if (res.message) { me.new_page = res.message; + me.pages[res.message.label] && delete me.pages[res.message.label]; me.title = ''; me.icon = ''; me.parent = ''; From fd18da985d0786fd65c8720e6ea12e45a8f31d7b Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Tue, 7 Sep 2021 20:22:45 +0530 Subject: [PATCH 19/82] fix: Add page to sidebar minor fix --- frappe/public/js/frappe/views/workspace/workspace.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/views/workspace/workspace.js b/frappe/public/js/frappe/views/workspace/workspace.js index 36b2988935..8989814349 100644 --- a/frappe/public/js/frappe/views/workspace/workspace.js +++ b/frappe/public/js/frappe/views/workspace/workspace.js @@ -625,7 +625,7 @@ frappe.views.Workspace = class Workspace { let $sidebar_section = is_public ? $sidebar[1] : $sidebar[0]; if (!parent) { - !is_public && $sidebar.last().removeClass('hidden'); + !is_public && $sidebar.first().removeClass('hidden'); $sidebar_item.appendTo($sidebar_section); } else { let $item_container = $($sidebar_section).find(`[item-name="${parent}"]`); From 646ceb4c9e022d6ee28bd19e1d43d96d3a88e947 Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Wed, 8 Sep 2021 12:31:05 +0530 Subject: [PATCH 20/82] fix: not checked cell should be empty and make check centered --- frappe/templates/print_formats/standard_macros.html | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/frappe/templates/print_formats/standard_macros.html b/frappe/templates/print_formats/standard_macros.html index ec60af1ce0..55c76a00c2 100644 --- a/frappe/templates/print_formats/standard_macros.html +++ b/frappe/templates/print_formats/standard_macros.html @@ -140,6 +140,8 @@ data-fieldname="{{ df.fieldname }}" data-fieldtype="{{ df.fieldtype }}" style="width: 12px; height: 12px; margin-top: 5px;"> + {% elif df.fieldtype=="Check" and not doc[df.fieldname] %} + {% elif df.fieldtype in ("Image", "Attach Image") and frappe.utils.is_image(doc[doc.meta.get_field(df.fieldname).options]) %} = 3 %}{{ "" }} {%- elif df.align -%}{{ "text-" + df.align }} - {%- elif df.fieldtype in ("Int", "Float", "Currency", "Check", "Percent") -%}{{ "text-right" }} + {%- elif df.fieldtype in ("Int", "Float", "Currency", "Percent") -%}{{ "text-right" }} + {%- elif df.fieldtype in ("Check") -%}{{ "text-center" }} {%- else -%}{{ "" }} {%- endif -%} {% endmacro %} From fd264159931aa5b6d8966bb93eb1940b20e09608 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Wed, 8 Sep 2021 16:57:10 +0530 Subject: [PATCH 21/82] fix: Show login page instantly --- frappe/public/scss/login.bundle.scss | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/frappe/public/scss/login.bundle.scss b/frappe/public/scss/login.bundle.scss index 25fc6662e3..17f33b0a67 100644 --- a/frappe/public/scss/login.bundle.scss +++ b/frappe/public/scss/login.bundle.scss @@ -4,12 +4,17 @@ body { background-color: var(--bg-light-gray); } -.for-login, .for-forgot, .for-signup, .for-email-login { display: none; - margin: 70px 0; +} + +.for-login, +.for-forgot, +.for-signup, +.for-email-login { + padding: max(15vh, 70px) 0; @include media-breakpoint-up(sm) { .page-card { From 7f7ab7647958dad62c2580b2712d3607fc72e292 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Wed, 8 Sep 2021 16:59:50 +0530 Subject: [PATCH 22/82] fix: Update login page & footer style --- frappe/public/scss/common/global.scss | 4 ++++ frappe/public/scss/desk/global.scss | 1 - frappe/public/scss/desk/mobile.scss | 5 ----- frappe/public/scss/website/footer.scss | 11 +++++++++++ frappe/public/scss/website/web_form.scss | 1 + .../includes/footer/footer_logo_extension.html | 4 ++-- frappe/www/login.html | 6 +++--- 7 files changed, 21 insertions(+), 11 deletions(-) diff --git a/frappe/public/scss/common/global.scss b/frappe/public/scss/common/global.scss index 024e0cd2a4..44b6e9ce34 100644 --- a/frappe/public/scss/common/global.scss +++ b/frappe/public/scss/common/global.scss @@ -1,3 +1,7 @@ +html, body { + height: 100%; +} + /* checkbox */ .checkbox { label { diff --git a/frappe/public/scss/desk/global.scss b/frappe/public/scss/desk/global.scss index 333ee30e4d..8c646395e9 100644 --- a/frappe/public/scss/desk/global.scss +++ b/frappe/public/scss/desk/global.scss @@ -1,5 +1,4 @@ html { - height: 100%; background-color: var(--bg-color); } diff --git a/frappe/public/scss/desk/mobile.scss b/frappe/public/scss/desk/mobile.scss index 839fca9bd2..14fa25e50f 100644 --- a/frappe/public/scss/desk/mobile.scss +++ b/frappe/public/scss/desk/mobile.scss @@ -1,9 +1,4 @@ -html { - min-height: 100%; -} - body { - height: 100%; // The html and body elements cannot have any padding or margin. margin: 0px; padding: 0px !important; diff --git a/frappe/public/scss/website/footer.scss b/frappe/public/scss/website/footer.scss index 5208afaa11..dc73fd180e 100644 --- a/frappe/public/scss/website/footer.scss +++ b/frappe/public/scss/website/footer.scss @@ -85,4 +85,15 @@ .form-control { border: none; font-size: var(--text-md); +} + +.footer-logo-extension { + .input-group { + justify-content: flex-end; + #footer-subscribe-email, #footer-subscribe-button { + max-width: 300px; + border: 1px solid var(--dark-border-color); + box-shadow: none; + } + } } \ No newline at end of file diff --git a/frappe/public/scss/website/web_form.scss b/frappe/public/scss/website/web_form.scss index 32b1c46f84..6a6547d79e 100644 --- a/frappe/public/scss/website/web_form.scss +++ b/frappe/public/scss/website/web_form.scss @@ -3,6 +3,7 @@ .web-form-wrapper { .form-control { color: var(--text-color); + background-color: var(--control-bg); } .form-section { diff --git a/frappe/templates/includes/footer/footer_logo_extension.html b/frappe/templates/includes/footer/footer_logo_extension.html index 17f3218c45..87bb4d14af 100644 --- a/frappe/templates/includes/footer/footer_logo_extension.html +++ b/frappe/templates/includes/footer/footer_logo_extension.html @@ -1,13 +1,13 @@
-
+
{%- if footer_logo -%}
{%- endif -%}
-
+
{% block extension %} {% include "templates/includes/footer/footer_extension.html" %} {% endblock %} diff --git a/frappe/www/login.html b/frappe/www/login.html index 90e235e679..4c0c54d76f 100644 --- a/frappe/www/login.html +++ b/frappe/www/login.html @@ -154,11 +154,11 @@
{%- else -%} -
+
{{_("Signup Disabled")}} + +
-

{{_("Signups have been disabled for this website.")}}

- {%- endif -%}
From 90bc9e23174b559fbfac0e3b708fee2e77cd6c53 Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Thu, 9 Sep 2021 13:20:12 +0530 Subject: [PATCH 23/82] fix(UI): Clicking area around file view row opens the file --- frappe/public/js/frappe/list/list_view.js | 1 + frappe/public/js/frappe/views/file/file_view.js | 6 ++++-- frappe/public/scss/desk/list.scss | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/frappe/public/js/frappe/list/list_view.js b/frappe/public/js/frappe/list/list_view.js index c5db7df88c..d7e808dd9f 100644 --- a/frappe/public/js/frappe/list/list_view.js +++ b/frappe/public/js/frappe/list/list_view.js @@ -1139,6 +1139,7 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList { if ( $target.hasClass("filterable") || $target.hasClass("select-like") || + $target.hasClass("file-select") || $target.hasClass("list-row-like") || $target.is(":checkbox") ) { diff --git a/frappe/public/js/frappe/views/file/file_view.js b/frappe/public/js/frappe/views/file/file_view.js index e020bff4dd..11204bb660 100644 --- a/frappe/public/js/frappe/views/file/file_view.js +++ b/frappe/public/js/frappe/views/file/file_view.js @@ -380,8 +380,10 @@ frappe.views.FileView = class FileView extends frappe.views.ListView { return `
- + + + ${file.subject_html} diff --git a/frappe/public/scss/desk/list.scss b/frappe/public/scss/desk/list.scss index 7fe04338ee..848e60b491 100644 --- a/frappe/public/scss/desk/list.scss +++ b/frappe/public/scss/desk/list.scss @@ -131,7 +131,7 @@ } } - .select-like { + .select-like, .file-select { padding: 15px 0px 15px 15px; } } From 1e2287855496576fb373632d6f1879e7c4239123 Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Thu, 9 Sep 2021 13:30:09 +0530 Subject: [PATCH 24/82] fix: remove unnecessary padding --- frappe/public/scss/desk/list.scss | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/scss/desk/list.scss b/frappe/public/scss/desk/list.scss index 848e60b491..bdbcacd977 100644 --- a/frappe/public/scss/desk/list.scss +++ b/frappe/public/scss/desk/list.scss @@ -169,7 +169,7 @@ $level-margin-right: 8px; color: var(--text-color); } - .level-item { + .level-item:not(.file-select) { margin-right: $level-margin-right; } From 4415351597daf6b6d4b0323ca6ece83c80e12855 Mon Sep 17 00:00:00 2001 From: Steffen Date: Thu, 9 Sep 2021 10:28:27 +0200 Subject: [PATCH 25/82] fix: hand no_default_fields parameter further down to recursive calls of as_dict --- frappe/model/base_document.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/model/base_document.py b/frappe/model/base_document.py index b9c8900839..5605ac61ed 100644 --- a/frappe/model/base_document.py +++ b/frappe/model/base_document.py @@ -307,7 +307,7 @@ class BaseDocument(object): doc["doctype"] = self.doctype for df in self.meta.get_table_fields(): children = self.get(df.fieldname) or [] - doc[df.fieldname] = [d.as_dict(convert_dates_to_str=convert_dates_to_str, no_nulls=no_nulls) for d in children] + doc[df.fieldname] = [d.as_dict(convert_dates_to_str=convert_dates_to_str, no_nulls=no_nulls, no_default_fields=no_default_fields) for d in children] if no_nulls: for k in list(doc): From 8b707278b9946e01ac824d655c312d35592ecb7f Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Thu, 9 Sep 2021 15:29:02 +0530 Subject: [PATCH 26/82] fix: Redirect after login --- frappe/www/message.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/www/message.html b/frappe/www/message.html index 2919592901..db9deeba1f 100644 --- a/frappe/www/message.html +++ b/frappe/www/message.html @@ -39,8 +39,8 @@ html, body { {% endif %}