vassili/seitime-frappe
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Merge pull request #13954 from gavindsouza/mariadb-10_6

Browse source 
feat: Support for MariaDB 10.6
This commit is contained in:
gavin 2021-09-15 11:58:03 +05:30 • committed by GitHub
commit a89e13a4ff
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 130 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
frappe/commands/site.py
View file

@ -67,6 +67,9 @@ def restore(context, sql_file_path, mariadb_root_username=None, mariadb_root_pas
validate_database_sql
)
site = get_site(context)
frappe.init(site=site)
force = context.force or force
decompressed_file_name = extract_sql_from_archive(sql_file_path)
@ -85,9 +88,6 @@ def restore(context, sql_file_path, mariadb_root_username=None, mariadb_root_pas
# check if valid SQL file
validate_database_sql(decompressed_file_name, _raise=not force)
site = get_site(context)
frappe.init(site=site)
# dont allow downgrading to older versions of frappe without force
if not force and is_downgrade(decompressed_file_name, verbose=True):
warn_message = (

113
frappe/commands/utils.py
View file

@ -408,20 +408,47 @@ def bulk_rename(context, doctype, path):
frappe.destroy()
@click.command('db-console')
@pass_context
def database(context):
"""
Enter into the Database console for given site.
"""
site = get_site(context)
if not site:
raise SiteNotSpecifiedError
frappe.init(site=site)
if not frappe.conf.db_type or frappe.conf.db_type == "mariadb":
_mariadb()
elif frappe.conf.db_type == "postgres":
_psql()
@click.command('mariadb')
@pass_context
def mariadb(context):
"""
Enter into mariadb console for a given site.
"""
import os
site = get_site(context)
if not site:
raise SiteNotSpecifiedError
frappe.init(site=site)
_mariadb()
# This is assuming you're within the bench instance.
@click.command('postgres')
@pass_context
def postgres(context):
"""
Enter into postgres console for a given site.
"""
site = get_site(context)
frappe.init(site=site)
_psql()
def _mariadb():
mysql = find_executable('mysql')
os.execv(mysql, [
mysql,
@ -434,15 +461,7 @@ def mariadb(context):
"-A"])
@click.command('postgres')
@pass_context
def postgres(context):
"""
Enter into postgres console for a given site.
"""
site = get_site(context)
frappe.init(site=site)
# This is assuming you're within the bench instance.
def _psql():
psql = find_executable('psql')
subprocess.run([ psql, '-d', frappe.conf.db_name])
@ -525,6 +544,74 @@ def console(context, autoreload=False):
terminal()
@click.command('transform-database', help="Change tables' internal settings changing engine and row formats")
@click.option('--table', required=True, help="Comma separated name of tables to convert. To convert all tables, pass 'all'")
@click.option('--engine', default=None, type=click.Choice(["InnoDB", "MyISAM"]), help="Choice of storage engine for said table(s)")
@click.option('--row_format', default=None, type=click.Choice(["DYNAMIC", "COMPACT", "REDUNDANT", "COMPRESSED"]), help="Set ROW_FORMAT parameter for said table(s)")
@click.option('--failfast', is_flag=True, default=False, help="Exit on first failure occurred")
@pass_context
def transform_database(context, table, engine, row_format, failfast):
"Transform site database through given parameters"
site = get_site(context)
check_table = []
add_line = False
skipped = 0
frappe.init(site=site)
if frappe.conf.db_type and frappe.conf.db_type != "mariadb":
click.secho("This command only has support for MariaDB databases at this point", fg="yellow")
sys.exit(1)
if not (engine or row_format):
click.secho("Values for `--engine` or `--row_format` must be set")
sys.exit(1)
frappe.connect()
if table == "all":
information_schema = frappe.qb.Schema("information_schema")
queried_tables = frappe.qb.from_(
information_schema.tables
).select("table_name").where(
(information_schema.tables.row_format != row_format)
& (information_schema.tables.table_schema == frappe.conf.db_name)
).run()
tables = [x[0] for x in queried_tables]
else:
tables = [x.strip() for x in table.split(",")]
total = len(tables)
for current, table in enumerate(tables):
values_to_set = ""
if engine:
values_to_set += f" ENGINE={engine}"
if row_format:
values_to_set += f" ROW_FORMAT={row_format}"
try:
frappe.db.sql(f"ALTER TABLE `{table}`{values_to_set}")
update_progress_bar("Updating table schema", current - skipped, total)
add_line = True
except Exception as e:
check_table.append([table, e.args])
skipped += 1
if failfast:
break
if add_line:
print()
for errored_table in check_table:
table, err = errored_table
err_msg = f"{table}: ERROR {err[0]}: {err[1]}"
click.secho(err_msg, fg="yellow")
frappe.destroy()
@click.command('run-tests')
@click.option('--app', help="For App")
@click.option('--doctype', help="For DocType")
@ -811,6 +898,8 @@ commands = [
build,
clear_cache,
clear_website_cache,
database,
transform_database,
jupyter,
console,
destroy_all_sessions,

2
frappe/database/mariadb/database.py
View file

@ -195,7 +195,7 @@ class MariaDBDatabase(Database):
`password` TEXT NOT NULL,
`encrypted` INT(1) NOT NULL DEFAULT 0,
PRIMARY KEY (`doctype`, `name`, `fieldname`)
) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci""")
) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci""")
def create_global_search_table(self):
if not '__global_search' in self.get_tables():

22
frappe/database/mariadb/framework_mariadb.sql
View file

@ -72,7 +72,7 @@ CREATE TABLE `tabDocField` (
KEY `label` (`label`),
KEY `fieldtype` (`fieldtype`),
KEY `fieldname` (`fieldname`)
) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
--
@ -109,7 +109,7 @@ CREATE TABLE `tabDocPerm` (
`email` int(1) NOT NULL DEFAULT 1,
PRIMARY KEY (`name`),
KEY `parent` (`parent`)
) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
--
-- Table structure for table `tabDocType Action`
@ -133,7 +133,7 @@ CREATE TABLE `tabDocType Action` (
PRIMARY KEY (`name`),
KEY `parent` (`parent`),
KEY `modified` (`modified`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci ROW_FORMAT=COMPRESSED;
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci ROW_FORMAT=DYNAMIC;
--
-- Table structure for table `tabDocType Action`
@ -156,7 +156,7 @@ CREATE TABLE `tabDocType Link` (
PRIMARY KEY (`name`),
KEY `parent` (`parent`),
KEY `modified` (`modified`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci ROW_FORMAT=COMPRESSED;
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci ROW_FORMAT=DYNAMIC;
--
-- Table structure for table `tabDocType`
@ -228,7 +228,7 @@ CREATE TABLE `tabDocType` (
`sender_field` varchar(255) DEFAULT NULL,
PRIMARY KEY (`name`),
KEY `parent` (`parent`)
) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
--
-- Table structure for table `tabSeries`
@ -239,7 +239,7 @@ CREATE TABLE `tabSeries` (
`name` varchar(100),
`current` int(10) NOT NULL DEFAULT 0,
PRIMARY KEY(`name`)
) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
--
@ -256,7 +256,7 @@ CREATE TABLE `tabSessions` (
`device` varchar(255) DEFAULT 'desktop',
`status` varchar(20) DEFAULT NULL,
KEY `sid` (`sid`)
) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
--
@ -269,7 +269,7 @@ CREATE TABLE `tabSingles` (
`field` varchar(255) DEFAULT NULL,
`value` text,
KEY `singles_doctype_field_index` (`doctype`, `field`)
) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
--
-- Table structure for table `__Auth`
@ -283,7 +283,7 @@ CREATE TABLE `__Auth` (
`password` TEXT NOT NULL,
`encrypted` INT(1) NOT NULL DEFAULT 0,
PRIMARY KEY (`doctype`, `name`, `fieldname`)
) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
--
-- Table structure for table `tabFile`
@ -311,7 +311,7 @@ CREATE TABLE `tabFile` (
KEY `parent` (`parent`),
KEY `attached_to_name` (`attached_to_name`),
KEY `attached_to_doctype` (`attached_to_doctype`)
) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
--
-- Table structure for table `tabDefaultValue`
@ -334,4 +334,4 @@ CREATE TABLE `tabDefaultValue` (
PRIMARY KEY (`name`),
KEY `parent` (`parent`),
KEY `defaultvalue_parent_defkey_index` (`parent`,`defkey`)
) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;

2
frappe/database/mariadb/schema.py
View file

@ -29,7 +29,7 @@ class MariaDBTable(DBTable):
%sindex parent(parent),
index modified(modified))
ENGINE={engine}
ROW_FORMAT=COMPRESSED
ROW_FORMAT=DYNAMIC
CHARACTER SET=utf8mb4
COLLATE=utf8mb4_unicode_ci""".format(varchar_len=frappe.db.VARCHAR_LEN,
engine=self.meta.get("engine") or 'InnoDB') % (self.table_name, add_text))

14
frappe/installer.py
View file

@ -445,9 +445,21 @@ def extract_sql_from_archive(sql_file_path):
else:
decompressed_file_name = sql_file_path
# convert archive sql to latest compatible
convert_archive_content(decompressed_file_name)
return decompressed_file_name
def convert_archive_content(sql_file_path):
if frappe.conf.db_type == "mariadb":
# ever since mariaDB 10.6, row_format COMPRESSED has been deprecated and removed
# this step is added to ease restoring sites depending on older mariaDB servers
contents = open(sql_file_path).read()
with open(sql_file_path, "w") as f:
f.write(contents.replace("ROW_FORMAT=COMPRESSED", "ROW_FORMAT=DYNAMIC"))
def extract_sql_gzip(sql_gz_path):
import subprocess
@ -457,7 +469,7 @@ def extract_sql_gzip(sql_gz_path):
decompressed_file = original_file.rstrip(".gz")
cmd = 'gzip -dvf < {0} > {1}'.format(original_file, decompressed_file)
subprocess.check_call(cmd, shell=True)
except:
except Exception:
raise
return decompressed_file