From 2e6a202652c331c25f68a392468a05587a37d98d Mon Sep 17 00:00:00 2001 From: Rushabh Mehta Date: Fri, 21 Sep 2018 10:20:48 +0530 Subject: [PATCH] Postgres support for Frappe (#5919) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [start] postgres * [wip] started refactoring db_schema * Add psycopg2 to requirements.txt * Add support for Postgres SQL - Separate frameworkSQL, database, schema, setup_db file for mariaDB and postgres - WIP * Remove quotes from sql to make it compatible with postgres as well * Moved some code from db_schema to database.py * Move code from db_schema to schema.py Add other required refactoring * Add schema chages * Remove redundant code in file * Add invalid column name exception class to exceptions.py * Add back tick in query wherever needed and replace ifnull with coalesce * Update get_column_description code in database.py file * Remove a print statement * Add keys to get on_duplicate query * Add bactick wherever necessary - Remove db_schema.py file * Remove DATE_SUB as it is incompatible with postgres - Fix prepare_filter_condition * Add backtick and quotes wherever necessary - Move get_database_size to frappe.db namespace - fix some left out bugs and errors * Add code to create key and unique index - added mysql and posgres in their respective database.py * Add more bacticks in queries and fix some errors - Pass keys to on_duplicate_update method - Replace MONTH with EXTRACT function - Remove DATEDIFF and CURDATE usage * Cast state value to int in toggle_two_factor_auth - since two_factor_auth has the datatype of Int * Refactor - Replace Timediff with normal arithmetic operator - Add MAX_COLUMN_LENGTH - Remove Redundant code - Add regexp character constant - Move create_help_table to database.py - Add get_full_text_search_condition method - Inherit MariaDBTable from DBTable * Replace Database instance with get_db method * Move db_manager to separate file * Refactor - Remove some unwanted code - Separate alter table code for postgres and mysql - Replace data_type with column_type in database.py * Make fulltext search changes in global_search.py * Add empty string check * Add root_password to site config * Create cli command for postgres console * Move setup of help database to setup_db.py * Add get_database_list method * Fix exception handling - Replace bad_field handler with missing_column handler * Fix tests and sql queries * Fix import error * Fix typo db -> database * Fix error with make_table in help.py * Try test for postgres * Remove pyhton 2.7 version to try postgres travis test * Add test fixes * Add db_type to the config of test_site_postgres * Enable query debug to check the reason for travis fail * Add backticks to check if the test passes * Update travis.yml - Add postgres addon * Try appending 'd_' to hash for db_name - since postgres does not support dbname starting with a number * Try adding db_type for global help to make travis work * Add print statements to debug travis failure * Enable transaction and remove debug flag * Fix help table creation query (postgres) * Fix import issue * Add some checks to prevent errors - Some doctypes used to get called even before they are created * Try fixes * Update travis config * Fix create index for help table * Remove unused code * Fix queries and update travis config * Fix ifnull replace logic (regex) * Add query fixes and code cleanup * Fix typo - get_column_description -> get_table_columns_description * Fix tests - Replace double quotes in query with single quote * Replace psycopg2 with psycopg2-binary to avoid warnings - http://initd.org/psycopg/docs/install.html#binary-install-from-pypi * Add multisql api * Add few multisql queries * Remove print statements * Remove get_fulltext_search_condition method and replace with multi query * Remove text slicing in create user * Set default for 'values' argument in multisql * Fix incorrect queries and remove few debug flags - Fix multisql bug * Force delete user to fix test - Fix Import error - Fix incorrect query * Fix query builder bug * Fix bad query * Fix query (minor) * Convert boolean text to int since is_private has datatype of int - Some query changes like removed double quotes and replace with interpolated string to pass multiple value pass in one of the query * Extend database class from an object to support python 2 * Fix query - Add quotes around value passed to the query for variable comparision * Try setting host_name for each test site - To avoid "RemoteDisconnected" error while testing data migration test - Update travis.yml to add hosts - Remove unwanted commit in setup_help_database * Set site hostname to data migration connector (in test file) - To connect the same site host * Fix duplicate entry issue - the problem is in naming series file. In previous commits I unknowingly changed a part of a series query due to which series were not getting reset * Replace few sql queries with orm methods * Fix codacy * Fix 'Doctype Sessions not found' issue * Fix bugs induced during codacy fixes * Fix Notification Test - Use ORM instead of raw sql * Set Date fallback value to 0001-01-01 - 0000-00-00 is invalid date in Postgres - 0001-01-01 works in both * Fix date filter method * Replace double quotes with single quote for literal value * Remove print statement * Replace double quotes with single * Fix tests - Replace few raw sql with ORM * Separate query for postgres - update_fields_to_fetch_query * Fix tests - replace locate with strpos for postgres * Fix tests - Skip test for datediff - convert bytes to str in escape method * Remove TestBot * Skip fieldname extraction * Replace docshare raw sql with ORM * Fix typo * Fix ancestor query test * Fix test data migration * Remove hardcoded hostname * Add default option and option list for db_type * Remove frappe.async module * Remove a debug flag from test * Fix codacy * fix import issue * Convert classmethod to static method * Convert few instance methods to static methods * Remove some unused imports * Fix codacy - Add exception type - Replace few instance methods with static methods - Remove unsued import * Fix codacy * Remove unused code * Remove some unused codes - Convert some instance methods to static function * Fix a issue with query modification * Fix add_index query * Fix query * Fix update_auth patch * Fix a issue with exception handling * Add try catch to a reload_doc * Add try-catch to file_manager_hook patch * import update_gravatar to set_user_gravatar patch * Undo all the wrong patch fixes * Fix db_setup code 😪 - previously it was not restoring db from source SQL which is why few old patched were breaking (because they were getting different schema structure) * Fix typo ! * Fix exception(is_missing_column) handling * Add deleted code - This code is only used in a erpnext patch. Can be moved to that patch file * Fix codacy * Replace a mariadb specific function in a query used in validate_series * Remove a debug flag * Revert changes (rename_parent_and_child) * Fix validate_one_root method * Fix date format issue * Fix codacy - Disable a pylint for variable argument warning - Convert an instance method to static method * Add bandit.yml The Codacy seems to use Bandit which generates warning for every subprocess import and its usage during pytest Since we have carefully used subprocess (avoided user input), warnings needs to be avoided. This can be removed if we have any alternative for subprocess usage. * Skip start_process_with_partial_path check * Fix typo * Add python 2.7 test * Move python versions in travis.yml * Add python versions to jobs * Overwrite python version inheritance for postgres in travis.yml * Add quotes around python version in .travis.yml * Add quotes around the name of the job * Try a travis fix * Try .travis.yml fix * Import missing subprocess * Refactor travis.yml * Refactor travis.yml - move install and tests commands to separate files - Use matrix to build combination of python version and db type * Make install.sh and run-tests.sh executable * Add sudo required to travis.yml to allow sudo cmmands in shell files * Load nvm * Remove verbose flag from scripts * Remove command-trace-print flag * Change to build dir in before script * Add absolute path for scripts * Fix tests * Fix typo * Fix codacy - fixes - "echo won't expand escape sequences." warning * Append (_) underscore instead of 'd' for db_name * Remove printf and use mysql execute flag --- .travis.yml | 52 +- .travis/install.sh | 22 + .travis/run-tests.sh | 22 + bandit.yml | 1 + frappe/__init__.py | 10 +- frappe/app.py | 10 +- frappe/auth.py | 2 +- frappe/boot.py | 22 +- frappe/cache_manager.py | 6 +- .../chat/doctype/chat_profile/chat_profile.py | 2 +- frappe/commands/site.py | 63 +- frappe/commands/utils.py | 26 +- frappe/contacts/doctype/address/address.py | 4 +- frappe/contacts/doctype/contact/contact.py | 4 +- .../core/doctype/activity_log/activity_log.py | 2 +- frappe/core/doctype/activity_log/feed.py | 4 +- .../doctype/activity_log/test_activity_log.py | 7 +- frappe/core/doctype/communication/comment.py | 8 +- frappe/core/doctype/communication/email.py | 8 +- frappe/core/doctype/data_export/exporter.py | 7 +- .../core/doctype/defaultvalue/defaultvalue.py | 17 +- frappe/core/doctype/doctype/doctype.py | 94 +-- frappe/core/doctype/error_log/error_log.py | 8 +- .../feedback_request/feedback_request.py | 2 +- frappe/core/doctype/file/file.py | 4 +- .../transaction_log/transaction_log.py | 16 +- frappe/core/doctype/user/test_user.py | 2 +- frappe/core/doctype/user/user.py | 72 +- .../user_permission/user_permission.py | 4 +- frappe/core/notifications.py | 10 +- .../permitted_documents_for_user.py | 4 +- .../doctype/custom_field/custom_field.py | 3 +- .../doctype/custom_field/test_custom_field.py | 2 - .../doctype/customize_form/customize_form.py | 12 +- .../property_setter/property_setter.py | 6 +- .../erpnext_civil_contracting.json | 14 - frappe/data/app_listing/erpnext_shopify.json | 14 - .../app_listing/mandrill_integration.json | 13 - .../app_listing/sendgrid_integration.json | 13 - frappe/data/sample_site_config.json | 42 -- .../test_data_migration_run.py | 17 +- frappe/database/__init__.py | 42 ++ frappe/{ => database}/database.py | 422 +++++------ frappe/database/db_manager.py | 88 +++ frappe/database/mariadb/__init__.py | 0 frappe/database/mariadb/database.py | 282 ++++++++ .../mariadb/framework_mariadb.sql} | 0 frappe/database/mariadb/schema.py | 86 +++ frappe/database/mariadb/setup_db.py | 148 ++++ frappe/database/postgres/__init__.py | 0 frappe/database/postgres/database.py | 309 +++++++++ .../database/postgres/framework_postgres.sql | 279 ++++++++ frappe/database/postgres/schema.py | 96 +++ frappe/database/postgres/setup_db.py | 41 ++ frappe/database/schema.py | 339 +++++++++ frappe/defaults.py | 2 +- frappe/desk/calendar.py | 2 +- .../doctype/auto_repeat/test_auto_repeat.py | 2 +- frappe/desk/doctype/event/event.py | 17 +- .../desk/doctype/kanban_board/kanban_board.py | 2 +- frappe/desk/doctype/todo/todo.py | 8 +- frappe/desk/form/assign_to.py | 19 +- frappe/desk/form/load.py | 30 +- frappe/desk/form/utils.py | 3 +- frappe/desk/like.py | 6 +- frappe/desk/moduleview.py | 2 +- frappe/desk/query_builder.py | 19 +- frappe/desk/reportview.py | 19 +- frappe/desk/search.py | 11 +- frappe/desk/tags.py | 12 +- frappe/email/__init__.py | 2 +- .../doctype/email_account/email_account.py | 11 +- .../email_account/test_email_account.py | 4 +- frappe/email/doctype/newsletter/newsletter.py | 25 +- .../doctype/notification/notification.py | 40 +- frappe/email/queue.py | 30 +- frappe/exceptions.py | 7 +- frappe/installer.py | 171 +---- frappe/limits.py | 12 +- frappe/model/__init__.py | 49 +- frappe/model/base_document.py | 75 +- frappe/model/create_new.py | 10 +- frappe/model/db_query.py | 110 +-- frappe/model/db_schema.py | 656 ------------------ frappe/model/delete_doc.py | 8 +- frappe/model/document.py | 5 +- frappe/model/dynamic_links.py | 14 +- frappe/model/meta.py | 15 +- frappe/model/naming.py | 22 +- frappe/model/rename_doc.py | 71 +- frappe/model/sync.py | 2 +- frappe/model/utils/link_count.py | 2 +- frappe/model/utils/user_settings.py | 15 +- frappe/modules/import_file.py | 1 + frappe/modules/utils.py | 3 +- .../v11_0/update_list_user_settings.py | 2 +- .../patches/v4_0/rename_sitemap_to_route.py | 4 +- frappe/patches/v4_2/set_assign_in_doc.py | 2 +- .../v5_0/convert_to_barracuda_and_utf8mb4.py | 2 +- .../patches/v5_0/fix_text_editor_file_urls.py | 2 +- .../v5_0/rename_ref_type_fieldnames.py | 4 +- frappe/patches/v6_16/star_to_like.py | 2 +- .../v6_19/comment_feed_communication.py | 4 +- frappe/patches/v7_0/re_route.py | 2 +- frappe/patches/v7_0/update_auth.py | 2 +- .../v7_2/set_in_standard_filter_property.py | 5 +- .../rename_listsettings_to_usersettings.py | 5 +- frappe/permissions.py | 6 +- .../doctype/print_format/test_records.json | 2 +- frappe/sessions.py | 45 +- frappe/share.py | 18 +- frappe/tests/test_api.py | 2 +- frappe/tests/test_bot.py | 26 +- frappe/tests/test_db.py | 8 +- frappe/tests/test_db_query.py | 14 +- frappe/tests/test_document.py | 5 +- frappe/tests/test_email.py | 4 +- frappe/tests/test_form_load.py | 11 +- frappe/tests/test_global_search.py | 8 +- frappe/tests/test_goal.py | 6 +- frappe/tests/test_password.py | 26 +- frappe/tests/test_permissions.py | 6 +- frappe/tests/test_scheduler.py | 2 +- frappe/tests/test_twofactor.py | 5 +- frappe/twofactor.py | 14 +- frappe/utils/background_jobs.py | 7 +- frappe/utils/change_log.py | 4 + frappe/utils/data.py | 8 +- frappe/utils/error.py | 2 +- frappe/utils/file_manager.py | 9 +- frappe/utils/global_search.py | 149 ++-- frappe/utils/goal.py | 29 +- frappe/utils/help.py | 50 +- frappe/utils/install.py | 2 +- frappe/utils/nestedset.py | 14 +- frappe/utils/password.py | 43 +- frappe/utils/scheduler.py | 102 ++- frappe/utils/user.py | 45 +- frappe/website/doctype/blog_post/blog_post.py | 12 +- .../doctype/website_theme/website_theme.py | 1 + frappe/website/router.py | 4 +- frappe/website/utils.py | 2 +- frappe/workflow/doctype/workflow/workflow.py | 15 +- .../workflow_action/workflow_action.py | 22 +- requirements.txt | 1 + test_sites/test_site/site_config.json | 3 +- .../test_site_postgres/site_config.json | 13 + 147 files changed, 2993 insertions(+), 2085 deletions(-) create mode 100755 .travis/install.sh create mode 100755 .travis/run-tests.sh create mode 100644 bandit.yml delete mode 100644 frappe/data/app_listing/erpnext_civil_contracting.json delete mode 100644 frappe/data/app_listing/erpnext_shopify.json delete mode 100644 frappe/data/app_listing/mandrill_integration.json delete mode 100644 frappe/data/app_listing/sendgrid_integration.json delete mode 100644 frappe/data/sample_site_config.json create mode 100644 frappe/database/__init__.py rename frappe/{ => database}/database.py (74%) create mode 100644 frappe/database/db_manager.py create mode 100644 frappe/database/mariadb/__init__.py create mode 100644 frappe/database/mariadb/database.py rename frappe/{data/Framework.sql => database/mariadb/framework_mariadb.sql} (100%) create mode 100644 frappe/database/mariadb/schema.py create mode 100644 frappe/database/mariadb/setup_db.py create mode 100644 frappe/database/postgres/__init__.py create mode 100644 frappe/database/postgres/database.py create mode 100644 frappe/database/postgres/framework_postgres.sql create mode 100644 frappe/database/postgres/schema.py create mode 100644 frappe/database/postgres/setup_db.py create mode 100644 frappe/database/schema.py delete mode 100644 frappe/model/db_schema.py create mode 100644 test_sites/test_site_postgres/site_config.json diff --git a/.travis.yml b/.travis.yml index 55481b7d4b..fb3af94182 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,49 +1,41 @@ language: python dist: trusty +sudo: required python: - - "2.7" - - "3.6" + - 2.7 + - 3.6 + +env: + - DB=mariadb + - DB=postgres services: - mysql +addons: + postgresql: "9.5" + hosts: + - test_site + - test_site_postgres + +matrix: + allow_failures: + - python: 2.7 + env: DB=postgres + fast_finish: true + install: - - sudo rm /etc/apt/sources.list.d/mongodb*.list - - sudo rm /etc/apt/sources.list.d/docker.list - - sudo apt-get install hhvm && rm -rf /home/travis/.kiex/ - - sudo apt-get purge -y mysql-common mysql-server mysql-client - - nvm install v8.10.0 - - - pip install python-coveralls - - - wget https://raw.githubusercontent.com/frappe/bench/master/playbooks/install.py - - - sudo python install.py --develop --user travis --without-bench-setup - - sudo pip install -e ~/bench - - - rm $TRAVIS_BUILD_DIR/.git/shallow - - cd ~/ && bench init frappe-bench --python $(which python) --frappe-path $TRAVIS_BUILD_DIR - - cp -r $TRAVIS_BUILD_DIR/test_sites/test_site ~/frappe-bench/sites/ + - $TRAVIS_BUILD_DIR/.travis/install.sh before_script: - - mysql -u root -ptravis -e 'create database test_frappe' - - echo "USE mysql;\nCREATE USER 'test_frappe'@'localhost' IDENTIFIED BY 'test_frappe';\nFLUSH PRIVILEGES;\n" | mysql -u root -ptravis - - echo "USE mysql;\nGRANT ALL PRIVILEGES ON \`test_frappe\`.* TO 'test_frappe'@'localhost';\n" | mysql -u root -ptravis - - - cd ~/frappe-bench - - bench use test_site - - bench reinstall --yes - - bench setup-help - - bench setup-global-help --mariadb_root_password travis - - bench scheduler disable - sed -i 's/9000/9001/g' sites/common_site_config.json - bench start & - sleep 10 script: - - bench run-tests --coverage + - $TRAVIS_BUILD_DIR/.travis/run-tests.sh after_script: - - coveralls -b apps/frappe -d ../../sites/.coverage + - coveralls -b apps/frappe -d ../../sites/.coverage \ No newline at end of file diff --git a/.travis/install.sh b/.travis/install.sh new file mode 100755 index 0000000000..9438167764 --- /dev/null +++ b/.travis/install.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +set -e + +sudo rm /etc/apt/sources.list.d/mongodb*.list +sudo rm /etc/apt/sources.list.d/docker.list +sudo apt-get install hhvm && rm -rf /home/travis/.kiex/ +sudo apt-get purge -y mysql-common mysql-server mysql-client +source ~/.nvm/nvm.sh +nvm install v8.10.0 + +pip install python-coveralls + +wget https://raw.githubusercontent.com/frappe/bench/master/playbooks/install.py + +sudo python install.py --develop --user travis --without-bench-setup +sudo pip install -e ~/bench + +rm $TRAVIS_BUILD_DIR/.git/shallow +cd ~/ && bench init frappe-bench --python $(which python) --frappe-path $TRAVIS_BUILD_DIR +cp -r $TRAVIS_BUILD_DIR/test_sites/test_site ~/frappe-bench/sites/ +cp -r $TRAVIS_BUILD_DIR/test_sites/test_site_postgres ~/frappe-bench/sites/ \ No newline at end of file diff --git a/.travis/run-tests.sh b/.travis/run-tests.sh new file mode 100755 index 0000000000..fe942098f4 --- /dev/null +++ b/.travis/run-tests.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +set -e + +if [[ $DB == 'mariadb' ]]; then + mysql -u root -ptravis -e 'create database test_frappe' + mysql -u root -ptravis -e "USE mysql; CREATE USER 'test_frappe'@'localhost' IDENTIFIED BY 'test_frappe'; FLUSH PRIVILEGES; " + mysql -u root -ptravis -e "USE mysql; GRANT ALL PRIVILEGES ON \`test_frappe\`.* TO 'test_frappe'@'localhost';" + bench --site test_site reinstall --yes + bench --site test_site setup-help + bench setup-global-help --root_password travis + bench --site test_site scheduler disable + bench --site test_site run-tests --coverage +elif [[ $DB == 'postgres' ]]; then + psql -c "CREATE DATABASE test_frappe;" -U postgres + psql -c "CREATE USER test_frappe WITH PASSWORD 'test_frappe';" -U postgres + bench --site test_site_postgres reinstall --yes + bench --site test_site_postgres setup-help + bench setup-global-help --db_type=postgres --root_password travis + bench --site test_site_postgres scheduler disable + bench --site test_site_postgres run-tests --coverage +fi diff --git a/bandit.yml b/bandit.yml new file mode 100644 index 0000000000..fce28629e8 --- /dev/null +++ b/bandit.yml @@ -0,0 +1 @@ +skips: ['B605', 'B404', 'B603', 'B607'] \ No newline at end of file diff --git a/frappe/__init__.py b/frappe/__init__.py index cf079ca32e..305e5b0e0c 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -168,17 +168,17 @@ def connect(site=None, db_name=None): :param site: If site is given, calls `frappe.init`. :param db_name: Optional. Will use from `site_config.json`.""" - from frappe.database import Database + from frappe.database import get_db if site: init(site) - local.db = Database(user=db_name or local.conf.db_name) + local.db = get_db(user=db_name or local.conf.db_name) set_user("Administrator") def connect_read_only(): - from frappe.database import Database + from frappe.database import get_db - local.read_only_db = Database(local.conf.slave_host, local.conf.slave_db_name, + local.read_only_db = get_db(local.conf.slave_host, local.conf.slave_db_name, local.conf.slave_db_password) # swap db connections @@ -259,7 +259,7 @@ def errprint(msg): :param msg: Message.""" msg = as_unicode(msg) if not request or (not "cmd" in local.form_dict) or conf.developer_mode: - print(msg.encode('utf-8')) + print(msg) error_log.append(msg) diff --git a/frappe/app.py b/frappe/app.py index d07c42dff2..b4c6c5b8aa 100644 --- a/frappe/app.py +++ b/frappe/app.py @@ -25,12 +25,6 @@ from frappe.utils.error import make_error_snapshot from frappe.core.doctype.communication.comment import update_comments_in_parent_after_request from frappe import _ -# imports - third-party imports -import pymysql -from pymysql.constants import ER - -# imports - module imports - local_manager = LocalManager([frappe.local]) _site = None @@ -148,8 +142,8 @@ def handle_exception(e): response = frappe.utils.response.report_error(http_status_code) elif (http_status_code==500 - and isinstance(e, pymysql.InternalError) - and e.args[0] in (ER.LOCK_WAIT_TIMEOUT, ER.LOCK_DEADLOCK)): + and (frappe.db and isinstance(e, frappe.db.InternalError)) + and (frappe.db and (frappe.db.is_deadlocked(e) or frappe.db.is_timedout(e)))): http_status_code = 508 elif http_status_code==401: diff --git a/frappe/auth.py b/frappe/auth.py index 1c1929b218..938710fe1f 100644 --- a/frappe/auth.py +++ b/frappe/auth.py @@ -91,7 +91,7 @@ class HTTPRequest: def connect(self, ac_name = None): """connect to db, from ac_name or db_name""" - frappe.local.db = frappe.database.Database(user = self.get_db_name(), \ + frappe.local.db = frappe.database.get_db(user = self.get_db_name(), \ password = getattr(conf, 'db_password', '')) class LoginManager: diff --git a/frappe/boot.py b/frappe/boot.py index 219a93ec35..075613d675 100644 --- a/frappe/boot.py +++ b/frappe/boot.py @@ -128,19 +128,19 @@ def get_user_pages_or_reports(parent): standard_roles = frappe.db.sql(""" select distinct - tab{parent}.name, - tab{parent}.modified, + `tab{parent}`.name as name, + `tab{parent}`.modified, {column} from `tabHas Role`, `tab{parent}` where `tabHas Role`.role in ({roles}) and `tabHas Role`.parent = `tab{parent}`.name - and tab{parent}.name not in ( + and `tab{parent}`.`name` not in ( select `tabCustom Role`.{field} from `tabCustom Role` where `tabCustom Role`.{field} is not null) {condition} """.format(parent=parent, column=column, roles = ', '.join(['%s']*len(roles)), - field=parent.lower(), condition="and tabReport.disabled=0" if parent == "Report" else ""), + field=parent.lower(), condition="and `tabReport`.disabled=0" if parent == "Report" else ""), roles, as_dict=True) for p in standard_roles: @@ -157,7 +157,7 @@ def get_user_pages_or_reports(parent): from `tab{parent}` where (select count(*) from `tabHas Role` - where `tabHas Role`.parent=tab{parent}.name) = 0 + where `tabHas Role`.parent=`tab{parent}`.`name`) = 0 """.format(parent=parent, column=column), as_dict=1) for p in pages_with_no_roles: @@ -173,7 +173,7 @@ def get_user_pages_or_reports(parent): def get_column(doctype): column = "`tabPage`.title as title" if doctype == "Report": - column = "`tabReport`.name as name, `tabReport`.name as title, `tabReport`.ref_doctype, `tabReport`.report_type" + column = "`tabReport`.`name` as title, `tabReport`.ref_doctype, `tabReport`.report_type" return column @@ -193,9 +193,9 @@ def load_translations(bootinfo): def get_fullnames(): """map of user fullnames""" - ret = frappe.db.sql("""select name, full_name as fullname, - user_image as image, gender, email, username - from tabUser where enabled=1 and user_type!="Website User" """, as_dict=1) + ret = frappe.db.sql("""select `name`, full_name as fullname, + user_image as image, gender, email, username + from tabUser where enabled=1 and user_type!='Website User'""", as_dict=1) d = {} for r in ret: @@ -245,10 +245,10 @@ def load_print_css(bootinfo, print_settings): bootinfo.print_css = frappe.www.printview.get_print_style(print_settings.print_style or "Modern", for_legacy=True) def get_unseen_notes(): - return frappe.db.sql('''select name, title, content, notify_on_every_login from tabNote where notify_on_login=1 + return frappe.db.sql('''select `name`, title, content, notify_on_every_login from `tabNote` where notify_on_login=1 and expire_notification_on > %s and %s not in (select user from `tabNote Seen By` nsb - where nsb.parent=tabNote.name)''', (frappe.utils.now(), frappe.session.user), as_dict=True) + where nsb.parent=`tabNote`.name)''', (frappe.utils.now(), frappe.session.user), as_dict=True) def get_gsuite_status(): return (frappe.get_value('Gsuite Settings', None, 'enable') == '1') diff --git a/frappe/cache_manager.py b/frappe/cache_manager.py index f2728122ef..f1b5d95056 100644 --- a/frappe/cache_manager.py +++ b/frappe/cache_manager.py @@ -67,9 +67,9 @@ def clear_doctype_cache(doctype=None): clear_single(doctype) # clear all parent doctypes - for dt in frappe.db.sql("""select parent from tabDocField - where fieldtype="Table" and options=%s""", (doctype,)): - clear_single(dt[0]) + + for dt in frappe.db.get_all('DocField', 'parent', dict(fieldtype='Table', options=doctype)): + clear_single(dt.parent) # clear all notifications delete_notification_count_for(doctype) diff --git a/frappe/chat/doctype/chat_profile/chat_profile.py b/frappe/chat/doctype/chat_profile/chat_profile.py index 61475f877e..a26c76984a 100644 --- a/frappe/chat/doctype/chat_profile/chat_profile.py +++ b/frappe/chat/doctype/chat_profile/chat_profile.py @@ -79,7 +79,7 @@ def create(user, exists_ok = False, fields = None): result = frappe.db.sql(""" SELECT * FROM `tabChat Profile` - WHERE user = "{user}" + WHERE `user` = '{user}' """.format(user = user)) if result: diff --git a/frappe/commands/site.py b/frappe/commands/site.py index 5ec3db3cb9..d3d2544b50 100755 --- a/frappe/commands/site.py +++ b/frappe/commands/site.py @@ -10,15 +10,10 @@ from frappe.installer import update_site_config from frappe.utils import touch_file, get_site_path from six import text_type -# imports - third-party imports -from pymysql.constants import ER - -# imports - module imports -from frappe.exceptions import SQLError - @click.command('new-site') @click.argument('site') @click.option('--db-name', help='Database name') +@click.option('--db-type', default='mariadb', type=click.Choice(['mariadb', 'postgres']), help='Optional "postgres" or "mariadb". Default is "mariadb"') @click.option('--mariadb-root-username', default='root', help='Root username for MariaDB') @click.option('--mariadb-root-password', help='Root password for MariaDB') @click.option('--admin-password', help='Administrator password for new site', default=None) @@ -26,22 +21,27 @@ from frappe.exceptions import SQLError @click.option('--force', help='Force restore if site/database already exists', is_flag=True, default=False) @click.option('--source_sql', help='Initiate database with a SQL file') @click.option('--install-app', multiple=True, help='Install app after installation') -def new_site(site, mariadb_root_username=None, mariadb_root_password=None, admin_password=None, verbose=False, install_apps=None, source_sql=None, force=None, install_app=None, db_name=None): +def new_site(site, mariadb_root_username=None, mariadb_root_password=None, admin_password=None, + verbose=False, install_apps=None, source_sql=None, force=None, install_app=None, + db_name=None, db_type=None): "Create a new site" frappe.init(site=site, new_site=True) - _new_site(db_name, site, mariadb_root_username=mariadb_root_username, mariadb_root_password=mariadb_root_password, admin_password=admin_password, - verbose=verbose, install_apps=install_app, source_sql=source_sql, force=force) + _new_site(db_name, site, mariadb_root_username=mariadb_root_username, + mariadb_root_password=mariadb_root_password, admin_password=admin_password, + verbose=verbose, install_apps=install_app, source_sql=source_sql, force=force, + db_type = db_type) if len(frappe.utils.get_sites()) == 1: use(site) -def _new_site(db_name, site, mariadb_root_username=None, mariadb_root_password=None, admin_password=None, - verbose=False, install_apps=None, source_sql=None,force=False, reinstall=False): +def _new_site(db_name, site, mariadb_root_username=None, mariadb_root_password=None, + admin_password=None, verbose=False, install_apps=None, source_sql=None,force=False, + reinstall=False, db_type=None): """Install a new Frappe site""" if not db_name: - db_name = hashlib.sha1(site.encode()).hexdigest()[:16] + db_name = '_' + hashlib.sha1(site.encode()).hexdigest()[:16] from frappe.installer import install_db, make_site_dirs from frappe.installer import install_app as _install_app @@ -61,8 +61,9 @@ def _new_site(db_name, site, mariadb_root_username=None, mariadb_root_password=N try: installing = touch_file(get_site_path('locks', 'installing.lock')) - install_db(root_login=mariadb_root_username, root_password=mariadb_root_password, db_name=db_name, - admin_password=admin_password, verbose=verbose, source_sql=source_sql,force=force, reinstall=reinstall) + install_db(root_login=mariadb_root_username, root_password=mariadb_root_password, + db_name=db_name, admin_password=admin_password, verbose=verbose, + source_sql=source_sql,force=force, reinstall=reinstall, db_type=db_type) apps_to_install = ['frappe'] + (frappe.conf.get("install_apps") or []) + (list(install_apps) or []) for app in apps_to_install: @@ -347,8 +348,7 @@ def drop_site(site, root_login='root', root_password=None, archived_sites_path=N def _drop_site(site, root_login='root', root_password=None, archived_sites_path=None, force=False): "Remove site from database and filesystem" - from frappe.installer import get_root_connection - from frappe.model.db_schema import DbManager + from frappe.database import drop_user_and_database from frappe.utils.backups import scheduled_backup frappe.init(site=site) @@ -356,25 +356,20 @@ def _drop_site(site, root_login='root', root_password=None, archived_sites_path= try: scheduled_backup(ignore_files=False, force=True) - except SQLError as err: - if err[0] == ER.NO_SUCH_TABLE: - if force: - pass - else: - click.echo("="*80) - click.echo("Error: The operation has stopped because backup of {s}'s database failed.".format(s=site)) - click.echo("Reason: {reason}{sep}".format(reason=err[1], sep="\n")) - click.echo("Fix the issue and try again.") - click.echo( - "Hint: Use 'bench drop-site {s} --force' to force the removal of {s}".format(sep="\n", tab="\t", s=site) - ) - sys.exit(1) + except Exception as err: + if force: + pass + else: + click.echo("="*80) + click.echo("Error: The operation has stopped because backup of {s}'s database failed.".format(s=site)) + click.echo("Reason: {reason}{sep}".format(reason=err[1], sep="\n")) + click.echo("Fix the issue and try again.") + click.echo( + "Hint: Use 'bench drop-site {s} --force' to force the removal of {s}".format(sep="\n", tab="\t", s=site) + ) + sys.exit(1) - db_name = frappe.local.conf.db_name - frappe.local.db = get_root_connection(root_login, root_password) - dbman = DbManager(frappe.local.db) - dbman.delete_user(db_name) - dbman.drop_database(db_name) + drop_user_and_database(frappe.conf.db_name, root_login, root_password) if not archived_sites_path: archived_sites_path = os.path.join(frappe.get_app_path('frappe'), '..', '..', '..', 'archived_sites') diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py index e76e68e64c..8c0b44d8d2 100644 --- a/frappe/commands/utils.py +++ b/frappe/commands/utils.py @@ -316,6 +316,18 @@ def mariadb(context): '--pager=less -SFX', "-A"]) +@click.command('postgres') +@pass_context +def postgres(context): + """ + Enter into postgres console for a given site. + """ + site = get_site(context) + frappe.init(site=site) + # This is assuming you're within the bench instance. + psql = find_executable('psql') + subprocess.run([ psql, '-d', frappe.conf.db_name]) + @click.command('jupyter') @pass_context def jupyter(context): @@ -554,8 +566,9 @@ def get_version(): @click.command('setup-global-help') -@click.option('--mariadb_root_password') -def setup_global_help(mariadb_root_password=None): +@click.option('--db_type') +@click.option('--root_password') +def setup_global_help(db_type=None, root_password=None): '''setup help table in a separate database that will be shared by the whole bench and set `global_help_setup` as 1 in common_site_config.json''' @@ -571,8 +584,12 @@ def setup_global_help(mariadb_root_password=None): update_site_config('global_help_setup', 1, site_config_path=os.path.join('.', 'common_site_config.json')) - if mariadb_root_password: - frappe.local.conf.root_password = mariadb_root_password + if root_password: + frappe.local.conf.root_password = root_password + + if not frappe.local.conf.db_type: + frappe.local.conf.db_type = db_type + from frappe.utils.help import sync sync() @@ -684,6 +701,7 @@ commands = [ make_app, mysql, mariadb, + postgres, request, reset_perms, run_tests, diff --git a/frappe/contacts/doctype/address/address.py b/frappe/contacts/doctype/address/address.py index 3477ef86ce..98cfb39305 100644 --- a/frappe/contacts/doctype/address/address.py +++ b/frappe/contacts/doctype/address/address.py @@ -250,10 +250,10 @@ def address_query(doctype, txt, searchfield, start, page_len, filters): `tabAddress`.idx desc, `tabAddress`.name limit %(start)s, %(page_len)s """.format( mcond=get_match_cond(doctype), - key=frappe.db.escape(searchfield), + key=searchfield, condition=condition or ""), { - 'txt': "%%%s%%" % frappe.db.escape(txt), + 'txt': frappe.db.escape('%' + txt + '%'), '_txt': txt.replace("%", ""), 'start': start, 'page_len': page_len, diff --git a/frappe/contacts/doctype/contact/contact.py b/frappe/contacts/doctype/contact/contact.py index bce248679c..805a7ee59d 100644 --- a/frappe/contacts/doctype/contact/contact.py +++ b/frappe/contacts/doctype/contact/contact.py @@ -151,9 +151,9 @@ def contact_query(doctype, txt, searchfield, start, page_len, filters): `tabContact`.idx desc, `tabContact`.name limit %(start)s, %(page_len)s """.format( mcond=get_match_cond(doctype), - key=frappe.db.escape(searchfield)), + key=searchfield), { - 'txt': "%%%s%%" % frappe.db.escape(txt), + 'txt': frappe.db.escape('%' + txt + '%'), '_txt': txt.replace("%", ""), 'start': start, 'page_len': page_len, diff --git a/frappe/core/doctype/activity_log/activity_log.py b/frappe/core/doctype/activity_log/activity_log.py index 94a4803279..7badf737e4 100644 --- a/frappe/core/doctype/activity_log/activity_log.py +++ b/frappe/core/doctype/activity_log/activity_log.py @@ -46,4 +46,4 @@ def add_authentication_log(subject, user, operation="Login", status="Success"): def clear_authentication_logs(): """clear 100 day old authentication logs""" frappe.db.sql("""delete from `tabActivity Log` where \ - creation 1 limit 1""".format( doctype=d.parent, fieldname=d.fieldname)) - except pymysql.InternalError as e: - if e.args and e.args[0] == ER.BAD_FIELD_ERROR: + except frappe.db.InternalError as e: + if frappe.db.is_missing_column(e): # ignore if missing column, else raise # this happens in case of Custom Field pass @@ -743,13 +756,15 @@ def validate_permissions_for_doctype(doctype, for_remove=False): def clear_permissions_cache(doctype): frappe.clear_cache(doctype=doctype) delete_notification_count_for(doctype) - for user in frappe.db.sql_list("""select - distinct `tabHas Role`.parent - from + for user in frappe.db.sql_list(""" + SELECT + DISTINCT `tabHas Role`.`parent` + FROM `tabHas Role`, - tabDocPerm - where tabDocPerm.parent = %s - and tabDocPerm.role = `tabHas Role`.role""", doctype): + `tabDocPerm` + WHERE `tabDocPerm`.`parent` = %s + AND `tabDocPerm`.`role` = `tabHas Role`.`role` + """, doctype): frappe.clear_cache(user=user) def validate_permissions(doctype, for_remove=False): @@ -852,7 +867,8 @@ def make_module_and_roles(doc, perm_fieldname="permissions"): not frappe.db.exists('Domain', doc.restrict_to_domain): frappe.get_doc(dict(doctype='Domain', domain=doc.restrict_to_domain)).insert() - if not frappe.db.exists("Module Def", doc.module): + if ("tabModule Def" in frappe.db.get_tables() + and not frappe.db.exists("Module Def", doc.module)): m = frappe.get_doc({"doctype": "Module Def", "module_name": doc.module}) m.app_name = frappe.local.module_app[frappe.scrub(doc.module)] m.flags.ignore_mandatory = m.flags.ignore_permissions = True @@ -868,8 +884,8 @@ def make_module_and_roles(doc, perm_fieldname="permissions"): r.insert() except frappe.DoesNotExistError as e: pass - except frappe.SQLError as e: - if e.args[0]==1146: + except frappe.db.ProgrammingError as e: + if frappe.db.is_table_missing(e): pass else: raise diff --git a/frappe/core/doctype/error_log/error_log.py b/frappe/core/doctype/error_log/error_log.py index c363dbad52..6cc8275404 100644 --- a/frappe/core/doctype/error_log/error_log.py +++ b/frappe/core/doctype/error_log/error_log.py @@ -14,14 +14,14 @@ class ErrorLog(Document): def set_old_logs_as_seen(): # set logs as seen - frappe.db.sql("""update `tabError Log` set seen=1 - where seen=0 and datediff(curdate(), creation) > 7""") + frappe.db.sql("""UPDATE `tabError Log` SET `seen`=1 + WHERE `seen`=0 AND `creation` < (NOW() - INTERVAL '7' DAY)""") # clear old logs - frappe.db.sql("""delete from `tabError Log` where datediff(curdate(), creation) > 30""") + frappe.db.sql("""DELETE FROM `tabError Log` WHERE `creation` < (NOW() - INTERVAL '30' DAY)""") @frappe.whitelist() def clear_error_logs(): '''Flush all Error Logs''' frappe.only_for('System Manager') - frappe.db.sql('''delete from `tabError Log`''') \ No newline at end of file + frappe.db.sql('''DELETE FROM `tabError Log`''') \ No newline at end of file diff --git a/frappe/core/doctype/feedback_request/feedback_request.py b/frappe/core/doctype/feedback_request/feedback_request.py index 6dcf84caae..8605d10f8a 100644 --- a/frappe/core/doctype/feedback_request/feedback_request.py +++ b/frappe/core/doctype/feedback_request/feedback_request.py @@ -35,4 +35,4 @@ def is_valid_feedback_request(key=None): def delete_feedback_request(): """ clear 100 days old feedback request """ - frappe.db.sql("""delete from `tabFeedback Request` where creation= DATE_SUB(NOW(),INTERVAL 1 YEAR) + and modified >= (NOW() - INTERVAL '1' YEAR) and comment_type='Like' and owner is not null and owner!=%(user)s and reference_owner=%(user)s @@ -60,12 +60,12 @@ def get_unread_emails(): SELECT count(*) FROM `tabCommunication` WHERE communication_type='Communication' - AND communication_medium="Email" - AND sent_or_received="Received" - AND email_status not in ("Spam", "Trash") + AND communication_medium='Email' + AND sent_or_received='Received' + AND email_status not in ('Spam', 'Trash') AND email_account in ( SELECT distinct email_account from `tabUser Email` WHERE parent=%(user)s ) - AND modified >= DATE_SUB(NOW(),INTERVAL 1 YEAR) + AND modified >= (NOW() - INTERVAL '1' YEAR) AND seen=0 """, {"user": frappe.session.user})[0][0] diff --git a/frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.py b/frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.py index fdbd49cc17..95a04360be 100644 --- a/frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.py +++ b/frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.py @@ -6,7 +6,7 @@ import frappe from frappe import _, throw import frappe.utils.user from frappe.permissions import check_admin_or_system_manager -from frappe.model.db_schema import type_map +from frappe.model import data_fieldtypes def execute(filters=None): user, doctype, show_permissions = filters.get("user"), filters.get("doctype"), filters.get("show_permissions") @@ -34,7 +34,7 @@ def get_columns_and_fields(doctype): columns = ["Name:Link/{}:200".format(doctype)] fields = ["`name`"] for df in frappe.get_meta(doctype).fields: - if df.in_list_view and df.fieldtype in type_map: + if df.in_list_view and df.fieldtype in data_fieldtypes: fields.append("`{0}`".format(df.fieldname)) fieldtype = "Link/{}".format(df.options) if df.fieldtype=="Link" else df.fieldtype columns.append("{label}:{fieldtype}:{width}".format(label=df.label, fieldtype=fieldtype, width=df.width or 100)) diff --git a/frappe/custom/doctype/custom_field/custom_field.py b/frappe/custom/doctype/custom_field/custom_field.py index 0f779cdbb0..e461ca5f46 100644 --- a/frappe/custom/doctype/custom_field/custom_field.py +++ b/frappe/custom/doctype/custom_field/custom_field.py @@ -63,8 +63,7 @@ class CustomField(Document): if not frappe.db.get_value('DocType', self.dt, 'issingle'): if (self.fieldname not in frappe.db.get_table_columns(self.dt) or getattr(self, "_old_fieldtype", None) != self.fieldtype): - from frappe.model.db_schema import updatedb - updatedb(self.dt) + frappe.db.updatedb(self.dt) def on_trash(self): # delete property setter entries diff --git a/frappe/custom/doctype/custom_field/test_custom_field.py b/frappe/custom/doctype/custom_field/test_custom_field.py index 1ee5da3642..819917050a 100644 --- a/frappe/custom/doctype/custom_field/test_custom_field.py +++ b/frappe/custom/doctype/custom_field/test_custom_field.py @@ -10,7 +10,5 @@ import unittest test_records = frappe.get_test_records('Custom Field') -from frappe.model.db_schema import InvalidColumnName - class TestCustomField(unittest.TestCase): pass diff --git a/frappe/custom/doctype/customize_form/customize_form.py b/frappe/custom/doctype/customize_form/customize_form.py index 319aa853a9..05e764c99c 100644 --- a/frappe/custom/doctype/customize_form/customize_form.py +++ b/frappe/custom/doctype/customize_form/customize_form.py @@ -149,8 +149,7 @@ class CustomizeForm(Document): validate_fields_for_doctype(self.doc_type) if self.flags.update_db: - from frappe.model.db_schema import updatedb - updatedb(self.doc_type) + frappe.db.updatedb(self.doc_type) if not hasattr(self, 'hide_success') or not self.hide_success: frappe.msgprint(_("{0} updated").format(_(self.doc_type))) @@ -183,7 +182,7 @@ class CustomizeForm(Document): continue elif property == "reqd" and \ - ((frappe.db.get_value("DocField", + ((frappe.db.get_value("DocField", {"parent":self.doc_type,"fieldname":df.fieldname}, "reqd") == 1) \ and (df.get(property) == 0)): frappe.msgprint(_("Row {0}: Not allowed to disable Mandatory for standard fields")\ @@ -322,7 +321,7 @@ class CustomizeForm(Document): try: property_value = frappe.db.get_value("DocType", self.doc_type, property_name) except Exception as e: - if e.args[0]==1054: + if frappe.db.is_column_missing(e): property_value = None else: raise @@ -342,7 +341,8 @@ class CustomizeForm(Document): if not self.doc_type: return - frappe.db.sql("""delete from `tabProperty Setter` where doc_type=%s - and !(`field_name`='naming_series' and `property`='options')""", self.doc_type) + frappe.db.sql("""DELETE FROM `tabProperty Setter` WHERE doc_type=%s + and `field_name`!='naming_series' + and `property`!='options'""", self.doc_type) frappe.clear_cache(doctype=self.doc_type) self.fetch_to_customize() diff --git a/frappe/custom/doctype/property_setter/property_setter.py b/frappe/custom/doctype/property_setter/property_setter.py index 070001fe02..2343e4bc44 100644 --- a/frappe/custom/doctype/property_setter/property_setter.py +++ b/frappe/custom/doctype/property_setter/property_setter.py @@ -33,7 +33,7 @@ class PropertySetter(Document): frappe.db.sql("""delete from `tabProperty Setter` where doctype_or_field = %(doctype_or_field)s and doc_type = %(doc_type)s - and ifnull(field_name,'') = ifnull(%(field_name)s, '') + and coalesce(field_name,'') = coalesce(%(field_name)s, '') and property = %(property)s""", self.get_valid_dict()) def get_property_list(self, dt): @@ -41,7 +41,7 @@ class PropertySetter(Document): from tabDocField where parent=%s and fieldtype not in ('Section Break', 'Column Break', 'HTML', 'Read Only', 'Table', 'Fold') - and ifnull(fieldname, '') != '' + and coalesce(fieldname, '') != '' order by label asc""", dt, as_dict=1) def get_setup_data(self): @@ -69,7 +69,7 @@ class PropertySetter(Document): from frappe.core.doctype.doctype.doctype import validate_fields_for_doctype validate_fields_for_doctype(self.doc_type) -def make_property_setter(doctype, fieldname, property, value, property_type, for_doctype = False, +def make_property_setter(doctype, fieldname, property, value, property_type, for_doctype = False, validate_fields_for_doctype=True): # WARNING: Ignores Permissions property_setter = frappe.get_doc({ diff --git a/frappe/data/app_listing/erpnext_civil_contracting.json b/frappe/data/app_listing/erpnext_civil_contracting.json deleted file mode 100644 index 2f00961002..0000000000 --- a/frappe/data/app_listing/erpnext_civil_contracting.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "app_url": "https://github.com/revant/civil_contracting.git", - "app_name": "civil_contracting", - "app_icon": "octicon octicon-file-directory", - "app_color": "grey", - "app_description": "Civil Contracting App to manage workers, wages and measurements", - "app_publisher": "Revant Nandgaonkar", - "app_email": "revant@mntechnique.com", - "repo_url": "https://github.com/revant/civil_contracting.git", - "app_title": "Civil Contracting", - "app_version": "1.5.0", - "app_category": "Integrations", - "featured": 1 -} diff --git a/frappe/data/app_listing/erpnext_shopify.json b/frappe/data/app_listing/erpnext_shopify.json deleted file mode 100644 index 88db8f6494..0000000000 --- a/frappe/data/app_listing/erpnext_shopify.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "app_url": "https://github.com/frappe/erpnext_shopify.git", - "app_name": "erpnext_shopify", - "app_icon": "octicon octicon-file-directory", - "app_color": "grey", - "app_description": "Shopify connector for ERPNext", - "app_publisher": "Frappe", - "app_email": "hello@frappe.io", - "repo_url": "https://github.com/frappe/erpnext_shopify.git", - "app_title": "ERPNext Shopify", - "app_version": "1.0.0", - "app_category": "Integrations", - "featured": 1 -} diff --git a/frappe/data/app_listing/mandrill_integration.json b/frappe/data/app_listing/mandrill_integration.json deleted file mode 100644 index addc403db0..0000000000 --- a/frappe/data/app_listing/mandrill_integration.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "app_url": "https://github.com/frappe/mandrill_integration", - "app_name": "mandrill_integration", - "app_icon": "octicon octicon-inbox", - "app_color": "#4CB6E6", - "app_description": "Set email communication status (Sent, Bounced etc) from Mandrill via webhooks.", - "app_publisher": "Frappe Technologies Pvt Ltd, Sponsored by Rohit Industries Group Pvt Ltd", - "app_email": "hello@frappe.io", - "repo_url": "https://github.com/frappe/mandrill_integration.git", - "app_title": "Mandrill Integration", - "app_version": "0.1.0", - "app_category": "Integrations" -} diff --git a/frappe/data/app_listing/sendgrid_integration.json b/frappe/data/app_listing/sendgrid_integration.json deleted file mode 100644 index 84987108c7..0000000000 --- a/frappe/data/app_listing/sendgrid_integration.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "app_url": "https://github.com/semilimes/sendgrid_integration", - "app_name": "sendgrid_integration", - "app_icon": "octicon octicon-inbox", - "app_color": "#4CB6E6", - "app_description": "Set email communication status from SendGrid via webhook.", - "app_publisher": "Semilimes", - "app_email": "all@semilimes.com", - "repo_url": "https://github.com/semilimes/sendgrid_integration.git", - "app_title": "SendGrid Integration", - "app_version": "0.0.1", - "app_category": "Integrations" -} diff --git a/frappe/data/sample_site_config.json b/frappe/data/sample_site_config.json deleted file mode 100644 index 1bf3914039..0000000000 --- a/frappe/data/sample_site_config.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "db_name": "testdb", - "db_password": "password", - "mute_emails": true, - - "limits": { - "emails": 1500, - "space": 0.157, - "expiry": "2016-07-25", - "users": 1 - } - - "developer_mode": 1, - "auto_cache_clear": true, - "disable_website_cache": true, - "max_file_size": 1000000, - - "mail_server": "localhost", - "mail_login": null, - "mail_password": null, - "mail_port": 25, - "use_ssl": 0, - "auto_email_id": "hello@example.com", - - "google_login": { - "client_id": "google_client_id", - "client_secret": "google_client_secret" - }, - "github_login": { - "client_id": "github_client_id", - "client_secret": "github_client_secret" - }, - "facebook_login": { - "client_id": "facebook_client_id", - "client_secret": "facebook_client_secret" - }, - - "celery_broker": "redis://localhost", - "celery_result_backend": null, - "scheduler_interval": 300, - "celery_queue_per_site": true -} diff --git a/frappe/data_migration/doctype/data_migration_run/test_data_migration_run.py b/frappe/data_migration/doctype/data_migration_run/test_data_migration_run.py index 6700790ea6..07cb8aa278 100644 --- a/frappe/data_migration/doctype/data_migration_run/test_data_migration_run.py +++ b/frappe/data_migration/doctype/data_migration_run/test_data_migration_run.py @@ -45,11 +45,11 @@ class TestDataMigrationRun(unittest.TestCase): created_todo = frappe.get_doc('ToDo', {'description': event_subject}) self.assertEqual(created_todo.description, event_subject) - todo_list = frappe.get_list('ToDo', filters={'description': 'Data migration todo'}, fields=['name']) + todo_list = frappe.get_list('ToDo', filters={'description': 'data migration todo'}, fields=['name']) todo_name = todo_list[0].name todo = frappe.get_doc('ToDo', todo_name) - todo.description = 'Data migration todo updated' + todo.description = 'data migration todo updated' todo.save() run = frappe.get_doc({ @@ -77,7 +77,7 @@ def create_plan(): { 'remote_fieldname': 'starts_on', 'local_fieldname': 'eval:frappe.utils.get_datetime_str(frappe.utils.get_datetime())' } ], 'condition': '{"description": "data migration todo" }' - }).insert() + }).insert(ignore_if_duplicate=True) frappe.get_doc({ 'doctype': 'Data Migration Mapping', @@ -91,23 +91,24 @@ def create_plan(): 'fields': [ { 'remote_fieldname': 'subject', 'local_fieldname': 'description' } ] - }).insert() + }).insert(ignore_if_duplicate=True) frappe.get_doc({ 'doctype': 'Data Migration Plan', - 'plan_name': 'ToDo sync', + 'plan_name': 'ToDo Sync', 'module': 'Core', 'mappings': [ { 'mapping': 'Todo to Event' }, { 'mapping': 'Event to ToDo' } ] - }).insert() + }).insert(ignore_if_duplicate=True) frappe.get_doc({ 'doctype': 'Data Migration Connector', 'connector_name': 'Local Connector', 'connector_type': 'Frappe', - 'hostname': 'http://localhost:8000', + # connect to same host. + 'hostname': frappe.conf.host_name, 'username': 'Administrator', 'password': 'admin' - }).insert() + }).insert(ignore_if_duplicate=True) diff --git a/frappe/database/__init__.py b/frappe/database/__init__.py new file mode 100644 index 0000000000..f00c6bf0b4 --- /dev/null +++ b/frappe/database/__init__.py @@ -0,0 +1,42 @@ +# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors +# MIT License. See license.txt + +# Database Module +# -------------------- + +from __future__ import unicode_literals + +def setup_database(force, source_sql=None, verbose=None): + import frappe + if frappe.conf.db_type == 'postgres': + import frappe.database.postgres.setup_db + return frappe.database.postgres.setup_db.setup_database(force, source_sql, verbose) + else: + import frappe.database.mariadb.setup_db + return frappe.database.mariadb.setup_db.setup_database(force, source_sql, verbose) + +def drop_user_and_database(db_name, root_login=None, root_password=None): + import frappe + if frappe.conf.db_type == 'postgres': + pass + else: + import frappe.database.mariadb.setup_db + return frappe.database.mariadb.setup_db.drop_user_and_database(db_name, root_login, root_password) + +def get_db(host=None, user=None, password=None): + import frappe + if frappe.conf.db_type == 'postgres': + import frappe.database.postgres.database + return frappe.database.postgres.database.PostgresDatabase(host, user, password) + else: + import frappe.database.mariadb.database + return frappe.database.mariadb.database.MariaDBDatabase(host, user, password) + +def setup_help_database(help_db_name): + import frappe + if frappe.conf.db_type == 'postgres': + import frappe.database.postgres.setup_db + return frappe.database.postgres.setup_db.setup_help_database(help_db_name) + else: + import frappe.database.mariadb.setup_db + return frappe.database.mariadb.setup_db.setup_help_database(help_db_name) \ No newline at end of file diff --git a/frappe/database.py b/frappe/database/database.py similarity index 74% rename from frappe/database.py rename to frappe/database/database.py index bbfe6ad40a..ef50ef8ad9 100644 --- a/frappe/database.py +++ b/frappe/database/database.py @@ -5,66 +5,55 @@ # -------------------- from __future__ import unicode_literals -import warnings -import datetime -import frappe -import frappe.defaults -from time import time + import re +import time +import frappe +import datetime +import frappe.defaults import frappe.model.meta -from frappe.utils import now, get_datetime, cstr, cast_fieldtype + from frappe import _ -from frappe.model.utils.link_count import flush_local_link_count -from frappe.model.utils import STANDARD_FIELD_CONVERSION_MAP +from time import time +from frappe.utils import now, getdate, cast_fieldtype from frappe.utils.background_jobs import execute_job, get_queue -from frappe import as_unicode -import six +from frappe.model.utils.link_count import flush_local_link_count # imports - compatibility imports from six import ( integer_types, string_types, - binary_type, text_type, iteritems ) -# imports - third-party imports -from markdown2 import UnicodeWithAttrs -from pymysql.times import TimeDelta -from pymysql.constants import ER, FIELD_TYPE -from pymysql.converters import conversions -import pymysql - -# Helpers -def _cast_result(doctype, result): - batch = [ ] - - try: - for field, value in result: - df = frappe.get_meta(doctype).get_field(field) - if df: - value = cast_fieldtype(df.fieldtype, value) - - batch.append(tuple([field, value])) - except frappe.exceptions.DoesNotExistError: - return result - - return tuple(batch) - -class Database: +class Database(object): """ Open a database connection with the given parmeters, if use_default is True, use the login details from `conf.py`. This is called by the request handler and is accessible using the `db` global variable. the `sql` method is also global to run queries """ - def __init__(self, host=None, user=None, password=None, ac_name=None, use_default = 0, local_infile = 0): + VARCHAR_LEN = 140 + MAX_COLUMN_LENGTH = 64 + + OPTIONAL_COLUMNS = ["_user_tags", "_comments", "_assign", "_liked_by"] + DEFAULT_SHORTCUTS = ['_Login', '__user', '_Full Name', 'Today', '__today', "now", "Now"] + STANDARD_VARCHAR_COLUMNS = ('name', 'owner', 'modified_by', 'parent', 'parentfield', 'parenttype') + DEFAULT_COLUMNS = ['name', 'creation', 'modified', 'modified_by', 'owner', 'docstatus', 'parent', + 'parentfield', 'parenttype', 'idx'] + + class InvalidColumnName(frappe.ValidationError): pass + + + def __init__(self, host=None, user=None, password=None, ac_name=None, use_default=0): + self.setup_type_map() self.host = host or frappe.conf.db_host or 'localhost' self.user = user or frappe.conf.db_name + self.db_name = frappe.conf.db_name self._conn = None if ac_name: - self.user = self.get_db_login(ac_name) or frappe.conf.db_name + self.user = ac_name or frappe.conf.db_name if use_default: self.user = frappe.conf.db_name @@ -75,63 +64,25 @@ class Database: self.password = password or frappe.conf.db_password self.value_cache = {} - # this param is to load CSV's with LOCAL keyword. - # it can be set in site_config as > bench set-config local_infile 1 - # once the local-infile is set on MySql Server, the client needs to connect with this option - # Connections without this option leads to: 'The used command is not allowed with this MariaDB version' error - self.local_infile = local_infile or frappe.conf.local_infile - - def get_db_login(self, ac_name): - return ac_name + def setup_type_map(self): + pass def connect(self): """Connects to a database as set in `site_config.json`.""" - warnings.filterwarnings('ignore', category=pymysql.Warning) - usessl = 0 - if frappe.conf.db_ssl_ca and frappe.conf.db_ssl_cert and frappe.conf.db_ssl_key: - usessl = 1 - self.ssl = { - 'ca':frappe.conf.db_ssl_ca, - 'cert':frappe.conf.db_ssl_cert, - 'key':frappe.conf.db_ssl_key - } - - conversions.update({ - FIELD_TYPE.NEWDECIMAL: float, - FIELD_TYPE.DATETIME: get_datetime, - UnicodeWithAttrs: conversions[text_type] - }) - - if six.PY2: - conversions.update({ - TimeDelta: conversions[binary_type] - }) - - if usessl: - self._conn = pymysql.connect(self.host, self.user or '', self.password or '', - charset='utf8mb4', use_unicode = True, ssl=self.ssl, conv = conversions, local_infile = self.local_infile) - else: - self._conn = pymysql.connect(self.host, self.user or '', self.password or '', - charset='utf8mb4', use_unicode = True, conv = conversions, local_infile = self.local_infile) - - # MYSQL_OPTION_MULTI_STATEMENTS_OFF = 1 - # # self._conn.set_server_option(MYSQL_OPTION_MULTI_STATEMENTS_OFF) - + self.cur_db_name = self.user + self._conn = self.get_connection() self._cursor = self._conn.cursor() - if self.user != 'root': - self.use(self.user) frappe.local.rollback_observers = [] def use(self, db_name): """`USE` db_name.""" self._conn.select_db(db_name) - self.cur_db_name = db_name - def validate_query(self, q): - """Throw exception for dangerous queries: `ALTER`, `DROP`, `TRUNCATE` if not `Administrator`.""" - cmd = q.strip().lower().split()[0] - if cmd in ['alter', 'drop', 'truncate'] and frappe.session.user != 'Administrator': - frappe.throw(_("Not permitted"), frappe.PermissionError) + def get_connection(self): + pass + + def get_database_size(self): + pass def sql(self, query, values=(), as_dict = 0, as_list = 0, formatted = 0, debug=0, ignore_ddl=0, as_utf8=0, auto_commit=0, update=None, explain=False): @@ -161,6 +112,10 @@ class Database: {"name": "a%", "owner":"test@example.com"}) """ + if re.search(r'ifnull\(', query, flags=re.IGNORECASE): + # replaces ifnull in query with coalesce + query = re.sub(r'ifnull\(', 'coalesce(', query, flags=re.IGNORECASE) + if not self._conn: self.connect() @@ -183,7 +138,7 @@ class Database: if not isinstance(values, (dict, tuple, list)): values = (values,) - if debug and query.lower().startswith('select'): + if debug: try: if explain: self.explain_query(query, values) @@ -214,23 +169,19 @@ class Database: frappe.errprint(("Execution time: {0} sec").format(round(time_end - time_start, 2))) except Exception as e: - if ignore_ddl and e.args[0] in (ER.BAD_FIELD_ERROR, ER.NO_SUCH_TABLE, - ER.CANT_DROP_FIELD_OR_KEY): - pass + if(frappe.conf.db_type == 'postgres'): + self.rollback() - # NOTE: causes deadlock - # elif e.args[0]==2006: - # # mysql has gone away - # self.connect() - # return self.sql(query=query, values=values, - # as_dict=as_dict, as_list=as_list, formatted=formatted, - # debug=debug, ignore_ddl=ignore_ddl, as_utf8=as_utf8, - # auto_commit=auto_commit, update=update) + if ignore_ddl and (self.is_missing_column(e) or self.is_missing_table(e) or self.cant_drop_field_or_key(e)): + pass else: raise if auto_commit: self.commit() + if not self._cursor.description: + return () + # scrub output if required if as_dict: ret = self.fetch_as_dict(formatted, as_utf8) @@ -256,7 +207,7 @@ class Database: import json frappe.errprint(json.dumps(self.fetch_as_dict(), indent=1)) frappe.errprint("--- query explain end ---") - except: + except Exception: frappe.errprint("error in query explain") def sql_list(self, query, values=(), debug=False): @@ -290,7 +241,7 @@ class Database: self.transaction_writes += 1 if self.transaction_writes > 200000: if self.auto_commit_on_many_writes: - frappe.db.commit() + self.commit() else: frappe.throw(_("Too many writes in one request. Please send smaller requests"), frappe.ValidationError) @@ -298,26 +249,21 @@ class Database: """Internal. Converts results to dict.""" result = self._cursor.fetchall() ret = [] - needs_formatting = self.needs_formatting(result, formatted) if result: keys = [column[0] for column in self._cursor.description] for r in result: values = [] - for i in range(len(r)): - if needs_formatting: - val = self.convert_to_simple_type(r[i], formatted) - else: - val = r[i] - - if as_utf8 and type(val) is text_type: - val = val.encode('utf-8') - values.append(val) + for value in r: + if as_utf8 and isinstance(value, text_type): + value = value.encode('utf-8') + values.append(value) ret.append(frappe._dict(zip(keys, values))) return ret - def needs_formatting(self, result, formatted): + @staticmethod + def needs_formatting(result, formatted): """Returns true if the first row in the result has a Date, Datetime, Long Int.""" if result and result[0]: for v in result[0]: @@ -332,65 +278,21 @@ class Database: """Returns result metadata.""" return self._cursor.description - def convert_to_simple_type(self, v, formatted=0): - """Format date, time, longint values.""" - return v - - from frappe.utils import formatdate, fmt_money - - if isinstance(v, (datetime.date, datetime.timedelta, datetime.datetime, integer_types)): - if isinstance(v, datetime.date): - v = text_type(v) - if formatted: - v = formatdate(v) - - # time - elif isinstance(v, (datetime.timedelta, datetime.datetime)): - v = text_type(v) - - # long - elif isinstance(v, integer_types): - v=int(v) - - # convert to strings... (if formatted) - if formatted: - if isinstance(v, float): - v=fmt_money(v) - elif isinstance(v, int): - v = text_type(v) - - return v - - def convert_to_lists(self, res, formatted=0, as_utf8=0): + @staticmethod + def convert_to_lists(res, formatted=0, as_utf8=0): """Convert tuple output to lists (internal).""" nres = [] - needs_formatting = self.needs_formatting(res, formatted) for r in res: nr = [] - for c in r: - if needs_formatting: - val = self.convert_to_simple_type(c, formatted) - else: - val = c - if as_utf8 and type(val) is text_type: + for val in r: + if as_utf8 and isinstance(val, text_type): val = val.encode('utf-8') nr.append(val) nres.append(nr) return nres - def convert_to_utf8(self, res, formatted=0): - """Encode result as UTF-8.""" - nres = [] - for r in res: - nr = [] - for c in r: - if type(c) is text_type: - c = c.encode('utf-8') - nr.append(self.convert_to_simple_type(c, formatted)) - nres.append(nr) - return nres - - def build_conditions(self, filters): + @staticmethod + def build_conditions(filters): """Convert filters sent as dict, lists to SQL conditions. filter's key is passed by map function, build conditions like: @@ -430,7 +332,7 @@ class Database: if "[" in key: split_key = key.split("[") - condition = "ifnull(`" + split_key[0] + "`, " + split_key[1][:-1] + ") " \ + condition = "coalesce(`" + split_key[0] + "`, " + split_key[1][:-1] + ") " \ + _operator + _rhs else: condition = "`" + key + "` " + _operator + _rhs @@ -527,10 +429,10 @@ class Database: try: out = self._get_values_from_table(fields, filters, doctype, as_dict, debug, order_by, update) except Exception as e: - if ignore and e.args[0] in (1146, 1054): + if ignore and (frappe.db.is_missing_column(e) or frappe.db.is_table_missing(e)): # table or column not found, return None out = None - elif (not ignore) and e.args[0]==1146: + elif (not ignore) and frappe.db.is_table_missing(e): # table not found, look in singles out = self.get_values_from_single(fields, filters, doctype, as_dict, debug, update) else: @@ -566,14 +468,13 @@ class Database: return values and [values] or [] if isinstance(fields, list): - return [map(lambda d: values.get(d), fields)] + return [map(values.get, fields)] else: r = self.sql("""select field, value - from tabSingles where field in (%s) and doctype=%s""" \ + from `tabSingles` where field in (%s) and doctype=%s""" % (', '.join(['%s'] * len(fields)), '%s'), tuple(fields) + (doctype,), as_dict=False, debug=debug) - # r = _cast_result(doctype, r) if as_dict: if r: @@ -607,10 +508,12 @@ class Database: return dict_ - def get_all(self, *args, **kwargs): + @staticmethod + def get_all(*args, **kwargs): return frappe.get_all(*args, **kwargs) - def get_list(self, *args, **kwargs): + @staticmethod + def get_list(*args, **kwargs): return frappe.get_list(*args, **kwargs) def get_single_value(self, doctype, fieldname, cache=False): @@ -631,8 +534,8 @@ class Database: if fieldname in self.value_cache[doctype]: return self.value_cache[doctype][fieldname] - val = self.sql("""select value from - tabSingles where doctype=%s and field=%s""", (doctype, fieldname)) + val = self.sql("""select `value` from + `tabSingles` where `doctype`=%s and `field`=%s""", (doctype, fieldname)) val = val[0][0] if val else None if val=="0" or val=="1": @@ -675,8 +578,10 @@ class Database: names = list(filter(None, names)) if names: - return dict(self.sql("select name, `%s` from `tab%s` where name in (%s)" \ - % (field, doctype, ", ".join(["%s"]*len(names))), names, debug=debug)) + return self.get_all(doctype, + fields=['name', field], + filters=[['name', 'in', names]], + debug=debug, as_list=1) else: return {} @@ -732,12 +637,12 @@ class Database: # for singles keys = list(to_update) self.sql(''' - delete from tabSingles + delete from `tabSingles` where field in ({0}) and doctype=%s'''.format(', '.join(['%s']*len(keys))), list(keys) + [dt], debug=debug) for key, value in iteritems(to_update): - self.sql('''insert into tabSingles(doctype, field, value) values (%s, %s, %s)''', + self.sql('''insert into `tabSingles` (doctype, field, value) values (%s, %s, %s)''', (dt, key, value), debug=debug) if dt in self.value_cache: @@ -745,25 +650,27 @@ class Database: frappe.clear_document_cache(dt, dn) - def set(self, doc, field, val): + @staticmethod + def set(doc, field, val): """Set value in document. **Avoid**""" doc.db_set(field, val) def touch(self, doctype, docname): """Update the modified timestamp of this document.""" - from frappe.utils import now modified = now() - frappe.db.sql("""update `tab{doctype}` set `modified`=%s + self.sql("""update `tab{doctype}` set `modified`=%s where name=%s""".format(doctype=doctype), (modified, docname)) return modified - def set_temp(self, value): + @staticmethod + def set_temp(value): """Set a temperory value and return a key.""" key = frappe.generate_hash() frappe.cache().hset("temp", key, value) return key - def get_temp(self, key): + @staticmethod + def get_temp(key): """Return the temperory value and delete it.""" return frappe.cache().hget("temp", key) @@ -775,20 +682,24 @@ class Database: """Returns a global key value.""" return self.get_default(key, user) - def set_default(self, key, val, parent="__default", parenttype=None): - """Sets a global / user default value.""" - frappe.defaults.set_default(key, val, parent, parenttype) - - def add_default(self, key, val, parent="__default", parenttype=None): - """Append a default value for a key, there can be multiple default values for a particular key.""" - frappe.defaults.add_default(key, val, parent, parenttype) - def get_default(self, key, parent="__default"): """Returns default value as a list if multiple or single""" d = self.get_defaults(key, parent) return isinstance(d, list) and d[0] or d - def get_defaults(self, key=None, parent="__default"): + @staticmethod + def set_default(key, val, parent="__default", parenttype=None): + """Sets a global / user default value.""" + frappe.defaults.set_default(key, val, parent, parenttype) + + @staticmethod + def add_default(key, val, parent="__default", parenttype=None): + """Append a default value for a key, there can be multiple default values for a particular key.""" + frappe.defaults.add_default(key, val, parent, parenttype) + + + @staticmethod + def get_defaults(key=None, parent="__default"): """Get all defaults""" if key: defaults = frappe.defaults.get_defaults(parent) @@ -800,7 +711,7 @@ class Database: return frappe.defaults.get_defaults(parent) def begin(self): - self.sql("start transaction") + self.sql("START TRANSACTION") def commit(self): """Commit current transaction. Calls SQL `COMMIT`.""" @@ -810,7 +721,8 @@ class Database: enqueue_jobs_after_commit() flush_local_link_count() - def flush_realtime_log(self): + @staticmethod + def flush_realtime_log(): for args in frappe.local.realtime_log: frappe.realtime.emit_via_redis(*args) @@ -834,7 +746,7 @@ class Database: return ("tab" + doctype) in self.get_tables() def get_tables(self): - return [d[0] for d in self.sql("show tables")] + return [d[0] for d in self.sql("select table_name from information_schema.tables where table_schema not in ('pg_catalog', 'information_schema')")] def a_row_exists(self, doctype): """Returns True if atleast one row exists.""" @@ -850,7 +762,7 @@ class Database: return True # single always exists (!) try: return self.get_value(dt, dn, "name", cache=cache) - except: + except Exception: return None elif isinstance(dt, dict) and dt.get('doctype'): @@ -858,10 +770,9 @@ class Database: conditions = [] for d in dt: if d == 'doctype': continue - conditions.append('`%s` = "%s"' % (d, cstr(dt[d]).replace('"', '\"'))) - return self.sql('select name from `tab%s` where %s' % \ - (dt['doctype'], " and ".join(conditions))) - except: + conditions.append([d, '=', dt[d]]) + return self.get_all(dt['doctype'], filters=conditions, as_list=1) + except Exception: return None def count(self, dt, filters=None, debug=False, cache=False): @@ -872,11 +783,11 @@ class Database: return cache_count if filters: conditions, filters = self.build_conditions(filters) - count = frappe.db.sql("""select count(*) + count = self.sql("""select count(*) from `tab%s` where %s""" % (dt, conditions), filters, debug=debug)[0][0] return count else: - count = frappe.db.sql("""select count(*) + count = self.sql("""select count(*) from `tab%s`""" % (dt,))[0][0] if cache: @@ -884,58 +795,69 @@ class Database: return count + @staticmethod + def format_date(date): + return getdate(date).strftime("%Y-%m-%d") + + @staticmethod + def format_datetime(datetime): + if not datetime: + return '0001-01-01 00:00:00.000000' + + if isinstance(datetime, frappe.string_types): + if ':' not in datetime: + datetime = datetime + ' 00:00:00.000000' + else: + datetime = datetime.strftime("%Y-%m-%d %H:%M:%S.%f") + + return datetime def get_creation_count(self, doctype, minutes): """Get count of records created in the last x minutes""" from frappe.utils import now_datetime from dateutil.relativedelta import relativedelta - return frappe.db.sql("""select count(name) from `tab{doctype}` + return self.sql("""select count(name) from `tab{doctype}` where creation >= %s""".format(doctype=doctype), now_datetime() - relativedelta(minutes=minutes))[0][0] def get_db_table_columns(self, table): """Returns list of column names from given table.""" - return [r[0] for r in self.sql("DESC `%s`" % table)] + return [r[0] for r in self.sql(''' + select column_name + from information_schema.columns + where table_name = %s ''', table)] def get_table_columns(self, doctype): """Returns list of column names from given doctype.""" - return self.get_db_table_columns('tab' + doctype) + columns = self.get_db_table_columns('tab' + doctype) + if not columns: + raise self.ProgrammingError + return columns def has_column(self, doctype, column): """Returns True if column exists in database.""" return column in self.get_table_columns(doctype) def get_column_type(self, doctype, column): - return frappe.db.sql('''SELECT column_type FROM INFORMATION_SCHEMA.COLUMNS - WHERE table_name = 'tab{0}' AND COLUMN_NAME = "{1}"'''.format(doctype, column))[0][0] + return self.sql('''SELECT column_type FROM INFORMATION_SCHEMA.COLUMNS + WHERE table_name = 'tab{0}' AND column_name = '{1}' '''.format(doctype, column))[0][0] + + def has_index(self, table_name, index_name): + pass def add_index(self, doctype, fields, index_name=None): - """Creates an index with given fields if not already created. - Index name will be `fieldname1_fieldname2_index`""" - if not index_name: - index_name = "_".join(fields) + "_index" - - # remove index length if present e.g. (10) from index name - index_name = re.sub(r"\s*\([^)]+\)\s*", r"", index_name) - - if not frappe.db.sql("""show index from `tab%s` where Key_name="%s" """ % (doctype, index_name)): - frappe.db.commit() - frappe.db.sql("""alter table `tab%s` - add index `%s`(%s)""" % (doctype, index_name, ", ".join(fields))) + pass def add_unique(self, doctype, fields, constraint_name=None): - if isinstance(fields, string_types): - fields = [fields] - if not constraint_name: - constraint_name = "unique_" + "_".join(fields) + pass - if not frappe.db.sql("""select CONSTRAINT_NAME from information_schema.TABLE_CONSTRAINTS - where table_name=%s and constraint_type='UNIQUE' and CONSTRAINT_NAME=%s""", - ('tab' + doctype, constraint_name)): - frappe.db.commit() - frappe.db.sql("""alter table `tab%s` - add unique `%s`(%s)""" % (doctype, constraint_name, ", ".join(fields))) + @staticmethod + def get_index_name(fields): + index_name = "_".join(fields) + "_index" + # remove index length if present e.g. (10) from index name + index_name = re.sub(r"\s*\([^)]+\)\s*", r"", index_name) + return index_name def get_system_setting(self, key): def _load_system_settings(): @@ -950,20 +872,11 @@ class Database: self._cursor = None self._conn = None - def escape(self, s, percent=True): + @staticmethod + def escape(s, percent=True): """Excape quotes and percent in given string.""" - # pymysql expects unicode argument to escape_string with Python 3 - s = as_unicode(pymysql.escape_string(as_unicode(s)), "utf-8").replace("`", "\\`") - - # NOTE separating % escape, because % escape should only be done when using LIKE operator - # or when you use python format string to generate query that already has a %s - # for example: sql("select name from `tabUser` where name=%s and {0}".format(conditions), something) - # defaulting it to True, as this is the most frequent use case - # ideally we shouldn't have to use ESCAPE and strive to pass values via the values argument of sql - if percent: - s = s.replace("%", "%%") - - return s + # implemented in specific class + pass def get_descendants(self, doctype, name): '''Return descendants of the current record''' @@ -971,6 +884,25 @@ class Database: return self.sql_list('''select name from `tab{doctype}` where lft > {lft} and rgt < {rgt}'''.format(doctype=doctype, lft=lft, rgt=rgt)) + def is_missing_table_or_column(self, e): + return self.is_missing_column(e) or self.is_missing_table(e) + + def multisql(self, sql_dict, values=(), **kwargs): + current_dialect = frappe.conf.db_type or 'mariadb' + query = sql_dict.get(current_dialect) + return self.sql(query, values, **kwargs) + + def delete(self, doctype, conditions): + if conditions: + conditions, values = self.build_conditions(conditions) + return self.sql("DELETE FROM `tab{doctype}` where {conditions}".format( + doctype=doctype, + conditions=conditions + ), values) + else: + frappe.throw('No conditions provided') + + def enqueue_jobs_after_commit(): if frappe.flags.enqueue_after_commit and len(frappe.flags.enqueue_after_commit) > 0: for job in frappe.flags.enqueue_after_commit: @@ -978,3 +910,19 @@ def enqueue_jobs_after_commit(): q.enqueue_call(execute_job, timeout=job.get("timeout"), kwargs=job.get("queue_args")) frappe.flags.enqueue_after_commit = [] + +# Helpers +def _cast_result(doctype, result): + batch = [ ] + + try: + for field, value in result: + df = frappe.get_meta(doctype).get_field(field) + if df: + value = cast_fieldtype(df.fieldtype, value) + + batch.append(tuple([field, value])) + except frappe.exceptions.DoesNotExistError: + return result + + return tuple(batch) diff --git a/frappe/database/db_manager.py b/frappe/database/db_manager.py new file mode 100644 index 0000000000..0954657b28 --- /dev/null +++ b/frappe/database/db_manager.py @@ -0,0 +1,88 @@ +import os +import frappe + + +class DbManager: + + def __init__(self, db): + """ + Pass root_conn here for access to all databases. + """ + if db: + self.db = db + + def get_current_host(self): + return self.db.sql("select user()")[0][0].split('@')[1] + + def create_user(self, user, password, host=None): + # Create user if it doesn't exist. + if not host: + host = self.get_current_host() + + if password: + self.db.sql("CREATE USER '%s'@'%s' IDENTIFIED BY '%s';" % (user, host, password)) + else: + self.db.sql("CREATE USER '%s'@'%s';" % (user, host)) + + def delete_user(self, target, host=None): + if not host: + host = self.get_current_host() + try: + self.db.sql("DROP USER '%s'@'%s';" % (target, host)) + except Exception as e: + if e.args[0] == 1396: + pass + else: + raise + + def create_database(self, target): + if target in self.get_database_list(): + self.drop_database(target) + + self.db.sql("CREATE DATABASE `%s` ;" % target) + + def drop_database(self, target): + self.db.sql("DROP DATABASE IF EXISTS `%s`;" % target) + + def grant_all_privileges(self, target, user, host=None): + if not host: + host = self.get_current_host() + + self.db.sql("GRANT ALL PRIVILEGES ON `%s`.* TO '%s'@'%s';" % (target, user, host)) + + def flush_privileges(self): + self.db.sql("FLUSH PRIVILEGES") + + def get_database_list(self): + """get list of databases""" + return [d[0] for d in self.db.sql("SHOW DATABASES")] + + @staticmethod + def restore_database(target, source, user, password): + from frappe.utils import make_esc + esc = make_esc('$ ') + + from distutils.spawn import find_executable + pipe = find_executable('pv') + if pipe: + pipe = '{pipe} {source} |'.format( + pipe=pipe, + source=source + ) + source = '' + else: + pipe = '' + source = '< {source}'.format(source=source) + + if pipe: + print('Creating Database...') + + command = '{pipe} mysql -u {user} -p{password} -h{host} {target} {source}'.format( + pipe=pipe, + user=esc(user), + password=esc(password), + host=esc(frappe.db.host), + target=esc(target), + source=source + ) + os.system(command) diff --git a/frappe/database/mariadb/__init__.py b/frappe/database/mariadb/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py new file mode 100644 index 0000000000..ef571866fa --- /dev/null +++ b/frappe/database/mariadb/database.py @@ -0,0 +1,282 @@ +from __future__ import unicode_literals + +import frappe +import warnings + +import pymysql +from pymysql.times import TimeDelta +from pymysql.constants import ER, FIELD_TYPE +from pymysql.converters import conversions + +from frappe.utils import get_datetime, cstr +from markdown2 import UnicodeWithAttrs +from frappe.database.database import Database +from six import PY2, binary_type, text_type, string_types +from frappe.database.mariadb.schema import MariaDBTable + + +class MariaDBDatabase(Database): + ProgrammingError = pymysql.err.ProgrammingError + OperationalError = pymysql.err.OperationalError + InternalError = pymysql.err.InternalError + SQLError = pymysql.err.ProgrammingError + DataError = pymysql.err.DataError + REGEX_CHARACTER = 'regexp' + + def setup_type_map(self): + self.type_map = { + 'Currency': ('decimal', '18,6'), + 'Int': ('int', '11'), + 'Long Int': ('bigint', '20'), # convert int to bigint if length is more than 11 + 'Float': ('decimal', '18,6'), + 'Percent': ('decimal', '18,6'), + 'Check': ('int', '1'), + 'Small Text': ('text', ''), + 'Long Text': ('longtext', ''), + 'Code': ('longtext', ''), + 'Text Editor': ('longtext', ''), + 'Date': ('date', ''), + 'Datetime': ('datetime', '6'), + 'Time': ('time', '6'), + 'Text': ('text', ''), + 'Data': ('varchar', self.VARCHAR_LEN), + 'Link': ('varchar', self.VARCHAR_LEN), + 'Dynamic Link': ('varchar', self.VARCHAR_LEN), + 'Password': ('varchar', self.VARCHAR_LEN), + 'Select': ('varchar', self.VARCHAR_LEN), + 'Read Only': ('varchar', self.VARCHAR_LEN), + 'Attach': ('text', ''), + 'Attach Image': ('text', ''), + 'Signature': ('longtext', ''), + 'Color': ('varchar', self.VARCHAR_LEN), + 'Barcode': ('longtext', ''), + 'Geolocation': ('longtext', '') + } + + def get_connection(self): + warnings.filterwarnings('ignore', category=pymysql.Warning) + usessl = 0 + if frappe.conf.db_ssl_ca and frappe.conf.db_ssl_cert and frappe.conf.db_ssl_key: + usessl = 1 + ssl_params = { + 'ca':frappe.conf.db_ssl_ca, + 'cert':frappe.conf.db_ssl_cert, + 'key':frappe.conf.db_ssl_key + } + + conversions.update({ + FIELD_TYPE.NEWDECIMAL: float, + FIELD_TYPE.DATETIME: get_datetime, + UnicodeWithAttrs: conversions[text_type] + }) + + if PY2: + conversions.update({ + TimeDelta: conversions[binary_type] + }) + + if usessl: + conn = pymysql.connect(self.host, self.user or '', self.password or '', + charset='utf8mb4', use_unicode = True, ssl=ssl_params, + conv = conversions, local_infile = frappe.conf.local_infile) + else: + conn = pymysql.connect(self.host, self.user or '', self.password or '', + charset='utf8mb4', use_unicode = True, conv = conversions, + local_infile = frappe.conf.local_infile) + + # MYSQL_OPTION_MULTI_STATEMENTS_OFF = 1 + # # self._conn.set_server_option(MYSQL_OPTION_MULTI_STATEMENTS_OFF) + + if self.user != 'root': + conn.select_db(self.user) + + return conn + + def get_database_size(self): + ''''Returns database size in MB''' + db_size = self.sql(''' + SELECT `table_schema` as `database_name`, + SUM(`data_length` + `index_length`) / 1024 / 1024 AS `database_size` + FROM information_schema.tables WHERE `table_schema` = %s GROUP BY `table_schema` + ''', self.db_name, as_dict=True) + + return db_size[0].get('database_size') + + @staticmethod + def escape(s, percent=True): + """Excape quotes and percent in given string.""" + # pymysql expects unicode argument to escape_string with Python 3 + s = frappe.as_unicode(pymysql.escape_string(frappe.as_unicode(s)), "utf-8").replace("`", "\\`") + + # NOTE separating % escape, because % escape should only be done when using LIKE operator + # or when you use python format string to generate query that already has a %s + # for example: sql("select name from `tabUser` where name=%s and {0}".format(conditions), something) + # defaulting it to True, as this is the most frequent use case + # ideally we shouldn't have to use ESCAPE and strive to pass values via the values argument of sql + if percent: + s = s.replace("%", "%%") + + return "'" + s + "'" + + # column type + @staticmethod + def is_type_number(code): + return code == pymysql.NUMBER + + @staticmethod + def is_type_datetime(code): + return code in (pymysql.DATE, pymysql.DATETIME) + + # exception types + @staticmethod + def is_deadlocked(e): + return e.args[0] == ER.LOCK_DEADLOCK + + @staticmethod + def is_timedout(e): + return e.args[0] == ER.LOCK_WAIT_TIMEOUT + + @staticmethod + def is_table_missing(e): + return e.args[0] == ER.NO_SUCH_TABLE + + @staticmethod + def is_missing_column(e): + return e.args[0] == ER.BAD_FIELD_ERROR + + @staticmethod + def is_duplicate_fieldname(e): + return e.args[0] == ER.DUP_FIELDNAME + + @staticmethod + def is_duplicate_entry(e): + return e.args[0] == ER.DUP_ENTRY + + @staticmethod + def is_access_denied( e): + return e.args[0] == ER.ACCESS_DENIED_ERROR + + @staticmethod + def cant_drop_field_or_key(e): + return e.args[0] == ER.CANT_DROP_FIELD_OR_KEY + + def is_primary_key_violation(self, e): + return self.is_duplicate_entry(e) and 'PRIMARY' in cstr(e.args[1]) + + def is_unique_key_violation(self, e): + return self.is_duplicate_entry(e) and 'Duplicate' in cstr(e.args[1]) + + + def create_auth_table(self): + self.sql_ddl("""create table if not exists `__Auth` ( + `doctype` VARCHAR(140) NOT NULL, + `name` VARCHAR(255) NOT NULL, + `fieldname` VARCHAR(140) NOT NULL, + `password` VARCHAR(255) NOT NULL, + `encrypted` INT(1) NOT NULL DEFAULT 0, + PRIMARY KEY (`doctype`, `name`, `fieldname`) + ) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci""") + + def create_global_search_table(self): + if not '__global_search' in self.get_tables(): + self.sql('''create table __global_search( + doctype varchar(100), + name varchar({0}), + title varchar({0}), + content text, + fulltext(content), + route varchar({0}), + published int(1) not null default 0, + unique `doctype_name` (doctype, name)) + COLLATE=utf8mb4_unicode_ci + ENGINE=MyISAM + CHARACTER SET=utf8mb4'''.format(self.VARCHAR_LEN)) + + def create_user_settings_table(self): + self.sql_ddl("""create table if not exists __UserSettings ( + `user` VARCHAR(180) NOT NULL, + `doctype` VARCHAR(180) NOT NULL, + `data` TEXT, + UNIQUE(user, doctype) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8""") + + def create_help_table(self): + self.sql('''create table help( + path varchar(255), + content text, + title text, + intro text, + full_path text, + fulltext(title), + fulltext(content), + index (path)) + COLLATE=utf8mb4_unicode_ci + ENGINE=MyISAM + CHARACTER SET=utf8mb4''') + + @staticmethod + def get_on_duplicate_update(key=None): + return 'ON DUPLICATE key UPDATE ' + + def get_table_columns_description(self, table_name): + """Returns list of column and its description""" + return self.sql('''select + column_name as 'name', + column_type as 'type', + column_default as 'default', + column_key = 'MUL' as 'index', + column_key = 'UNI' as 'unique' + from information_schema.columns + where table_name = '{table_name}' '''.format(table_name=table_name), as_dict=1) + + def has_index(self, table_name, index_name): + return self.sql("""SHOW INDEX FROM `{table_name}` + WHERE Key_name='{index_name}'""".format( + table_name=table_name, + index_name=index_name + )) + + def add_index(self, doctype, fields, index_name=None): + """Creates an index with given fields if not already created. + Index name will be `fieldname1_fieldname2_index`""" + index_name = index_name or self.get_index_name(fields) + table_name = 'tab' + doctype + if not self.has_index(table_name, index_name): + self.commit() + self.sql("""ALTER TABLE `%s` + ADD INDEX `%s`(%s)""" % (table_name, index_name, ", ".join(fields))) + + def add_unique(self, doctype, fields, constraint_name=None): + if isinstance(fields, string_types): + fields = [fields] + if not constraint_name: + constraint_name = "unique_" + "_".join(fields) + + if not self.sql("""select CONSTRAINT_NAME from information_schema.TABLE_CONSTRAINTS + where table_name=%s and constraint_type='UNIQUE' and CONSTRAINT_NAME=%s""", + ('tab' + doctype, constraint_name)): + self.commit() + self.sql("""alter table `tab%s` + add unique `%s`(%s)""" % (doctype, constraint_name, ", ".join(fields))) + + def updatedb(self, doctype, meta=None): + """ + Syncs a `DocType` to the table + * creates if required + * updates columns + * updates indices + """ + res = self.sql("select issingle from `tabDocType` where name=%s", (doctype,)) + if not res: + raise Exception('Wrong doctype {0} in updatedb'.format(doctype)) + + if not res[0][0]: + db_table = MariaDBTable(doctype, meta) + db_table.validate() + + self.commit() + db_table.sync() + self.begin() + + def get_database_list(self, target): + return [d[0] for d in self.sql("SHOW DATABASES;")] \ No newline at end of file diff --git a/frappe/data/Framework.sql b/frappe/database/mariadb/framework_mariadb.sql similarity index 100% rename from frappe/data/Framework.sql rename to frappe/database/mariadb/framework_mariadb.sql diff --git a/frappe/database/mariadb/schema.py b/frappe/database/mariadb/schema.py new file mode 100644 index 0000000000..7fd87ba4b0 --- /dev/null +++ b/frappe/database/mariadb/schema.py @@ -0,0 +1,86 @@ +from __future__ import unicode_literals + +import frappe +from frappe import _ +from frappe.database.schema import DBTable + +class MariaDBTable(DBTable): + def create(self): + add_text = '' + + # columns + column_defs = self.get_column_definitions() + if column_defs: add_text += ',\n'.join(column_defs) + ',\n' + + # index + index_defs = self.get_index_definitions() + if index_defs: add_text += ',\n'.join(index_defs) + ',\n' + + # create table + frappe.db.sql("""create table `%s` ( + name varchar({varchar_len}) not null primary key, + creation datetime(6), + modified datetime(6), + modified_by varchar({varchar_len}), + owner varchar({varchar_len}), + docstatus int(1) not null default '0', + parent varchar({varchar_len}), + parentfield varchar({varchar_len}), + parenttype varchar({varchar_len}), + idx int(8) not null default '0', + %sindex parent(parent), + index modified(modified)) + ENGINE={engine} + ROW_FORMAT=COMPRESSED + CHARACTER SET=utf8mb4 + COLLATE=utf8mb4_unicode_ci""".format(varchar_len=frappe.db.VARCHAR_LEN, + engine=self.meta.get("engine") or 'InnoDB') % (self.table_name, add_text)) + + def alter(self): + for col in self.columns.values(): + col.build_for_alter_table(self.current_columns.get(col.fieldname.lower())) + + add_column_query = [] + modify_column_query = [] + add_index_query = [] + drop_index_query = [] + + columns_to_modify = set(self.change_type + self.add_unique + self.set_default) + + for col in self.add_column: + add_column_query.append("ADD COLUMN `{}` {}".format(col.fieldname, col.get_definition())) + + for col in columns_to_modify: + modify_column_query.append("MODIFY `{}` {}".format(col.fieldname, col.get_definition())) + + for col in self.add_index: + # if index key not exists + if not frappe.db.sql("SHOW INDEX FROM `%s` WHERE key_name = %s" % + (self.table_name, '%s'), col.fieldname): + add_index_query.append("ADD INDEX `{}`(`{}`)".format(col.fieldname, col.fieldname)) + + for col in self.drop_index: + if col.fieldname != 'name': # primary key + # if index key exists + if frappe.db.sql("""SHOW INDEX FROM `{0}` + WHERE key_name=%s + AND Non_unique=%s""".format(self.table_name), (col.fieldname, col.unique)): + drop_index_query.append("drop index `{}`".format(col.fieldname)) + + try: + for query_parts in [add_column_query, modify_column_query, add_index_query, drop_index_query]: + if query_parts: + query_body = ", ".join(query_parts) + query = "ALTER TABLE `{}` {}".format(self.table_name, query_body) + frappe.db.sql(query) + + except Exception as e: + # sanitize + if e.args[0]==1060: + frappe.throw(str(e)) + elif e.args[0]==1062: + fieldname = str(e).split("'")[-2] + frappe.throw(_("{0} field cannot be set as unique in {1}, as there are non-unique existing values".format( + fieldname, self.table_name))) + else: + raise e diff --git a/frappe/database/mariadb/setup_db.py b/frappe/database/mariadb/setup_db.py new file mode 100644 index 0000000000..3cc5365d79 --- /dev/null +++ b/frappe/database/mariadb/setup_db.py @@ -0,0 +1,148 @@ +from __future__ import unicode_literals + +import frappe +import os, sys +from frappe.database.db_manager import DbManager + +def setup_database(force, source_sql, verbose): + frappe.local.session = frappe._dict({'user':'Administrator'}) + + db_name = frappe.local.conf.db_name + root_conn = get_root_connection(frappe.flags.root_login, frappe.flags.root_password) + dbman = DbManager(root_conn) + if force or (db_name not in dbman.get_database_list()): + dbman.delete_user(db_name) + dbman.drop_database(db_name) + else: + raise Exception("Database %s already exists" % (db_name,)) + + dbman.create_user(db_name, frappe.conf.db_password) + if verbose: print("Created user %s" % db_name) + + dbman.create_database(db_name) + if verbose: print("Created database %s" % db_name) + + dbman.grant_all_privileges(db_name, db_name) + dbman.flush_privileges() + if verbose: print("Granted privileges to user %s and database %s" % (db_name, db_name)) + + # close root connection + root_conn.close() + + bootstrap_database(db_name, verbose, source_sql) + +def setup_help_database(help_db_name): + dbman = DbManager(get_root_connection(frappe.flags.root_login, frappe.flags.root_password)) + dbman.drop_database(help_db_name) + + # make database + if not help_db_name in dbman.get_database_list(): + try: + dbman.create_user(help_db_name, help_db_name) + except Exception as e: + # user already exists + if e.args[0] != 1396: raise + dbman.create_database(help_db_name) + dbman.grant_all_privileges(help_db_name, help_db_name) + dbman.flush_privileges() + +def drop_user_and_database(db_name, root_login, root_password): + frappe.local.db = get_root_connection(root_login, root_password) + dbman = DbManager(frappe.local.db) + dbman.delete_user(db_name) + dbman.drop_database(db_name) + +def bootstrap_database(db_name, verbose, source_sql=None): + frappe.connect(db_name=db_name) + check_if_ready_for_barracuda() + import_db_from_sql(source_sql, verbose) + if not 'tabDefaultValue' in frappe.db.get_tables(): + print('''Database not installed, this can due to lack of permission, or that the database name exists. + Check your mysql root password, or use --force to reinstall''') + sys.exit(1) + +def import_db_from_sql(source_sql=None, verbose=False): + if verbose: print("Starting database import...") + db_name = frappe.conf.db_name + if not source_sql: + source_sql = os.path.join(os.path.dirname(__file__), 'framework_mariadb.sql') + DbManager(frappe.local.db).restore_database(db_name, source_sql, db_name, frappe.conf.db_password) + if verbose: print("Imported from database %s" % source_sql) + +def check_if_ready_for_barracuda(): + mariadb_variables = frappe._dict(frappe.db.sql("""show variables""")) + mariadb_minor_version = int(mariadb_variables.get('version').split('-')[0].split('.')[1]) + if mariadb_minor_version < 3: + check_database(mariadb_variables, { + "innodb_file_format": "Barracuda", + "innodb_file_per_table": "ON", + "innodb_large_prefix": "ON" + }) + check_database(mariadb_variables, { + "character_set_server": "utf8mb4", + "collation_server": "utf8mb4_unicode_ci" + }) + +def check_database(mariadb_variables, variables_dict): + mariadb_minor_version = int(mariadb_variables.get('version').split('-')[0].split('.')[1]) + for key, value in variables_dict.items(): + if mariadb_variables.get(key) != value: + site = frappe.local.site + msg = ("Creation of your site - {x} failed because MariaDB is not properly {sep}" + "configured to use the Barracuda storage engine. {sep}" + "Please add the settings below to MariaDB's my.cnf, restart MariaDB then {sep}" + "run `bench new-site {x}` again.{sep2}" + "").format(x=site, sep2="\n"*2, sep="\n") + + if mariadb_minor_version < 3: + print_db_config(msg, expected_config_for_barracuda_2) + else: + print_db_config(msg, expected_config_for_barracuda_3) + raise frappe.exceptions.ImproperDBConfigurationError( + reason="MariaDB default file format is not Barracuda" + ) + +def get_root_connection(root_login, root_password): + import getpass + if not frappe.local.flags.root_connection: + if not root_login: + root_login = 'root' + + if not root_password: + root_password = frappe.conf.get("root_password") or None + + if not root_password: + root_password = getpass.getpass("MySQL root password: ") + + frappe.local.flags.root_connection = frappe.database.get_db(user=root_login, password=root_password) + + return frappe.local.flags.root_connection + +def print_db_config(explanation, config_text): + print("="*80) + print(explanation) + print(config_text) + print("="*80) + +expected_config_for_barracuda_2 = """ +[mysqld] +innodb-file-format=barracuda +innodb-file-per-table=1 +innodb-large-prefix=1 +character-set-client-handshake = FALSE +character-set-server = utf8mb4 +collation-server = utf8mb4_unicode_ci + +[mysql] +default-character-set = utf8mb4 +""" + +expected_config_for_barracuda_3 = """ +[mysqld] +character-set-client-handshake = FALSE +character-set-server = utf8mb4 +collation-server = utf8mb4_unicode_ci + +[mysql] +default-character-set = utf8mb4 +""" diff --git a/frappe/database/postgres/__init__.py b/frappe/database/postgres/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/frappe/database/postgres/database.py b/frappe/database/postgres/database.py new file mode 100644 index 0000000000..2498cd19cf --- /dev/null +++ b/frappe/database/postgres/database.py @@ -0,0 +1,309 @@ +from __future__ import unicode_literals + +import re +import frappe +import psycopg2 +import psycopg2.extensions +from six import string_types +from frappe.utils import cstr +from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT + +from frappe.database.database import Database +from frappe.database.postgres.schema import PostgresTable + +# cast decimals as floats +DEC2FLOAT = psycopg2.extensions.new_type( + psycopg2.extensions.DECIMAL.values, + 'DEC2FLOAT', + lambda value, curs: float(value) if value is not None else None) + +psycopg2.extensions.register_type(DEC2FLOAT) + +class PostgresDatabase(Database): + ProgrammingError = psycopg2.ProgrammingError + OperationalError = psycopg2.OperationalError + InternalError = psycopg2.InternalError + SQLError = psycopg2.ProgrammingError + DataError = psycopg2.DataError + InterfaceError = psycopg2.InterfaceError + REGEX_CHARACTER = '~' + + def setup_type_map(self): + self.type_map = { + 'Currency': ('decimal', '18,6'), + 'Int': ('bigint', None), + 'Long Int': ('bigint', None), # convert int to bigint if length is more than 11 + 'Float': ('decimal', '18,6'), + 'Percent': ('decimal', '18,6'), + 'Check': ('smallint', None), + 'Small Text': ('text', ''), + 'Long Text': ('text', ''), + 'Code': ('text', ''), + 'Text Editor': ('text', ''), + 'Date': ('date', ''), + 'Datetime': ('timestamp', None), + 'Time': ('time', '6'), + 'Text': ('text', ''), + 'Data': ('varchar', self.VARCHAR_LEN), + 'Link': ('varchar', self.VARCHAR_LEN), + 'Dynamic Link': ('varchar', self.VARCHAR_LEN), + 'Password': ('varchar', self.VARCHAR_LEN), + 'Select': ('varchar', self.VARCHAR_LEN), + 'Read Only': ('varchar', self.VARCHAR_LEN), + 'Attach': ('text', ''), + 'Attach Image': ('text', ''), + 'Signature': ('text', ''), + 'Color': ('varchar', self.VARCHAR_LEN), + 'Barcode': ('text', ''), + 'Geolocation': ('text', '') + } + + def get_connection(self): + # warnings.filterwarnings('ignore', category=psycopg2.Warning) + conn = psycopg2.connect('host={} dbname={}'.format(self.host, self.user)) + conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) # TODO: Remove this + # conn = psycopg2.connect('host={} dbname={} user={} password={}'.format(self.host, + # self.user, self.user, self.password)) + + return conn + + def escape(self, s, percent=True): + """Excape quotes and percent in given string.""" + if isinstance(s, bytes): + s = s.decode('utf-8') + + if percent: + s = s.replace("%", "%%") + + s = s.encode('utf-8') + + return str(psycopg2.extensions.QuotedString(s)) + + def get_database_size(self): + ''''Returns database size in MB''' + db_size = self.sql("SELECT (pg_database_size(%s) / 1024 / 1024) as database_size", + self.db_name, as_dict=True) + return db_size[0].get('database_size') + + # pylint: disable=W0221 + def sql(self, *args, **kwargs): + if len(args): + # since tuple is immutable + args = list(args) + args[0] = modify_query(args[0]) + args = tuple(args) + elif kwargs.get('query'): + kwargs['query'] = modify_query(kwargs.get('query')) + + return super(PostgresDatabase, self).sql(*args, **kwargs) + + def get_tables(self): + return [d[0] for d in self.sql("""select table_name + from information_schema.tables + where table_catalog='{0}' + and table_type = 'BASE TABLE' + and table_schema='public'""".format(frappe.conf.db_name))] + + def format_date(self, date): + if not date: + return '0001-01-01::DATE' + + if isinstance(date, frappe.string_types): + if ':' not in date: + date = date + '::DATE' + else: + date = date.strftime('%Y-%m-%d') + '::DATE' + + return date + + # column type + @staticmethod + def is_type_number(code): + return code == psycopg2.NUMBER + + @staticmethod + def is_type_datetime(code): + return code == psycopg2.DATETIME + + # exception type + @staticmethod + def is_deadlocked(e): + return e.pgcode == '40P01' + + @staticmethod + def is_timedout(e): + # http://initd.org/psycopg/docs/extensions.html?highlight=datatype#psycopg2.extensions.QueryCanceledError + return isinstance(e, psycopg2.extensions.QueryCanceledError) + + @staticmethod + def is_table_missing(e): + return e.pgcode == '42P01' + + @staticmethod + def is_missing_column(e): + return e.pgcode == '42703' + + @staticmethod + def is_access_denied(e): + return e.pgcode == '42501' + + @staticmethod + def cant_drop_field_or_key(e): + return e.pgcode.startswith('23') + + @staticmethod + def is_duplicate_entry(e): + return e.pgcode == '23505' + + @staticmethod + def is_primary_key_violation(e): + return e.pgcode == '23505' and '_pkey' in cstr(e.args[0]) + + @staticmethod + def is_unique_key_violation(e): + return e.pgcode == '23505' and '_key' in cstr(e.args[0]) + + @staticmethod + def is_duplicate_fieldname(e): + return e.pgcode == '42701' + + def create_auth_table(self): + self.sql_ddl("""create table if not exists "__Auth" ( + "doctype" VARCHAR(140) NOT NULL, + "name" VARCHAR(255) NOT NULL, + "fieldname" VARCHAR(140) NOT NULL, + "password" VARCHAR(255) NOT NULL, + "encrypted" INT NOT NULL DEFAULT 0, + PRIMARY KEY ("doctype", "name", "fieldname") + )""") + + def create_global_search_table(self): + if not '__global_search' in self.get_tables(): + self.sql('''create table "__global_search"( + doctype varchar(100), + name varchar({0}), + title varchar({0}), + content text, + route varchar({0}), + published int not null default 0, + unique (doctype, name))'''.format(self.VARCHAR_LEN)) + + def create_user_settings_table(self): + self.sql_ddl("""create table if not exists "__UserSettings" ( + "user" VARCHAR(180) NOT NULL, + "doctype" VARCHAR(180) NOT NULL, + "data" TEXT, + UNIQUE ("user", "doctype") + )""") + + def create_help_table(self): + self.sql('''CREATE TABLE "help"( + "path" varchar(255), + "content" text, + "title" text, + "intro" text, + "full_path" text)''') + self.sql('''CREATE INDEX IF NOT EXISTS "help_index" ON "help" ("path")''') + + def updatedb(self, doctype, meta=None): + """ + Syncs a `DocType` to the table + * creates if required + * updates columns + * updates indices + """ + res = self.sql("select issingle from `tabDocType` where name='{}'".format(doctype)) + if not res: + raise Exception('Wrong doctype {0} in updatedb'.format(doctype)) + + if not res[0][0]: + db_table = PostgresTable(doctype, meta) + db_table.validate() + + self.commit() + db_table.sync() + self.begin() + + @staticmethod + def get_on_duplicate_update(key='name'): + if isinstance(key, list): + key = '", "'.join(key) + return 'ON CONFLICT ("{key}") DO UPDATE SET '.format( + key=key + ) + + def check_transaction_status(self, query): + pass + + def has_index(self, table_name, index_name): + return self.sql("""SELECT 1 FROM pg_indexes WHERE tablename='{table_name}' + and indexname='{index_name}' limit 1""".format(table_name=table_name, index_name=index_name)) + + def add_index(self, doctype, fields, index_name=None): + """Creates an index with given fields if not already created. + Index name will be `fieldname1_fieldname2_index`""" + index_name = index_name or self.get_index_name(fields) + table_name = 'tab' + doctype + + self.commit() + self.sql("""CREATE INDEX IF NOT EXISTS "{}" ON `{}`("{}")""".format(index_name, table_name, '", "'.join(fields))) + + def add_unique(self, doctype, fields, constraint_name=None): + if isinstance(fields, string_types): + fields = [fields] + if not constraint_name: + constraint_name = "unique_" + "_".join(fields) + + if not self.sql(""" + SELECT CONSTRAINT_NAME + FROM information_schema.TABLE_CONSTRAINTS + WHERE table_name=%s + AND constraint_type='UNIQUE' + AND CONSTRAINT_NAME=%s""", + ('tab' + doctype, constraint_name)): + self.commit() + self.sql("""ALTER TABLE `tab%s` + ADD CONSTRAINT %s UNIQUE (%s)""" % (doctype, constraint_name, ", ".join(fields))) + + def get_table_columns_description(self, table_name): + """Returns list of column and its description""" + # pylint: disable=W1401 + return self.sql(''' + SELECT a.column_name AS name, + CASE a.data_type + WHEN 'character varying' THEN CONCAT('varchar(', a.character_maximum_length ,')') + WHEN 'timestamp without TIME zone' THEN 'timestamp' + ELSE a.data_type + END AS type, + COUNT(b.indexdef) AS Index, + COALESCE(a.column_default, NULL) AS default, + BOOL_OR(b.unique) AS unique + FROM information_schema.columns a + LEFT JOIN + (SELECT indexdef, tablename, indexdef LIKE '%UNIQUE INDEX%' AS unique + FROM pg_indexes + WHERE tablename='{table_name}') b + ON SUBSTRING(b.indexdef, '\(.*\)') LIKE CONCAT('%', a.column_name, '%') + WHERE a.table_name = '{table_name}' + GROUP BY a.column_name, a.data_type, a.column_default, a.character_maximum_length;''' + .format(table_name=table_name), as_dict=1) + + def get_database_list(self, target): + return [d[0] for d in self.sql("SELECT datname FROM pg_database;")] + +def modify_query(query): + """"Modifies query according to the requirements of postgres""" + # replace ` with " for definitions + query = query.replace('`', '"') + query = replace_locate_with_strpos(query) + # select from requires "" + if re.search('from tab', query, flags=re.IGNORECASE): + query = re.sub('from tab([a-zA-Z]*)', r'from "tab\1"', query, flags=re.IGNORECASE) + + return query + +def replace_locate_with_strpos(query): + # strpos is the locate equivalent in postgres + if re.search(r'locate\(', query, flags=re.IGNORECASE): + query = re.sub(r'locate\(([^,]+),([^)]+)\)', r'strpos(\2, \1)', query, flags=re.IGNORECASE) + return query \ No newline at end of file diff --git a/frappe/database/postgres/framework_postgres.sql b/frappe/database/postgres/framework_postgres.sql new file mode 100644 index 0000000000..aac91fc03c --- /dev/null +++ b/frappe/database/postgres/framework_postgres.sql @@ -0,0 +1,279 @@ +-- Core Elements to install WNFramework +-- To be called from install.py + + +-- +-- Table structure for table "tabDocField" +-- + +DROP TABLE IF EXISTS "tabDocField"; +CREATE TABLE "tabDocField" ( + "name" varchar(255) NOT NULL, + "creation" timestamp(6) DEFAULT NULL, + "modified" timestamp(6) DEFAULT NULL, + "modified_by" varchar(255) DEFAULT NULL, + "owner" varchar(255) DEFAULT NULL, + "docstatus" smallint NOT NULL DEFAULT 0, + "parent" varchar(255) DEFAULT NULL, + "parentfield" varchar(255) DEFAULT NULL, + "parenttype" varchar(255) DEFAULT NULL, + "idx" bigint NOT NULL DEFAULT 0, + "fieldname" varchar(255) DEFAULT NULL, + "label" varchar(255) DEFAULT NULL, + "oldfieldname" varchar(255) DEFAULT NULL, + "fieldtype" varchar(255) DEFAULT NULL, + "oldfieldtype" varchar(255) DEFAULT NULL, + "options" text, + "search_index" smallint NOT NULL DEFAULT 0, + "hidden" smallint NOT NULL DEFAULT 0, + "set_only_once" smallint NOT NULL DEFAULT 0, + "allow_in_quick_entry" smallint NOT NULL DEFAULT 0, + "print_hide" smallint NOT NULL DEFAULT 0, + "report_hide" smallint NOT NULL DEFAULT 0, + "reqd" smallint NOT NULL DEFAULT 0, + "bold" smallint NOT NULL DEFAULT 0, + "in_global_search" smallint NOT NULL DEFAULT 0, + "collapsible" smallint NOT NULL DEFAULT 0, + "unique" smallint NOT NULL DEFAULT 0, + "no_copy" smallint NOT NULL DEFAULT 0, + "allow_on_submit" smallint NOT NULL DEFAULT 0, + "trigger" varchar(255) DEFAULT NULL, + "collapsible_depends_on" text, + "depends_on" text, + "permlevel" bigint NOT NULL DEFAULT 0, + "ignore_user_permissions" smallint NOT NULL DEFAULT 0, + "width" varchar(255) DEFAULT NULL, + "print_width" varchar(255) DEFAULT NULL, + "columns" bigint NOT NULL DEFAULT 0, + "default" text, + "description" text, + "in_list_view" smallint NOT NULL DEFAULT 0, + "in_standard_filter" smallint NOT NULL DEFAULT 0, + "read_only" smallint NOT NULL DEFAULT 0, + "precision" varchar(255) DEFAULT NULL, + "length" bigint NOT NULL DEFAULT 0, + "translatable" smallint NOT NULL DEFAULT 0, + PRIMARY KEY ("name") +) ; + +create index on "tabDocField" ("parent"); +create index on "tabDocField" ("label"); +create index on "tabDocField" ("fieldtype"); +create index on "tabDocField" ("fieldname"); + +-- +-- Table structure for table "tabDocPerm" +-- + +DROP TABLE IF EXISTS "tabDocPerm"; +CREATE TABLE "tabDocPerm" ( + "name" varchar(255) NOT NULL, + "creation" timestamp(6) DEFAULT NULL, + "modified" timestamp(6) DEFAULT NULL, + "modified_by" varchar(255) DEFAULT NULL, + "owner" varchar(255) DEFAULT NULL, + "docstatus" smallint NOT NULL DEFAULT 0, + "parent" varchar(255) DEFAULT NULL, + "parentfield" varchar(255) DEFAULT NULL, + "parenttype" varchar(255) DEFAULT NULL, + "idx" bigint NOT NULL DEFAULT 0, + "permlevel" bigint DEFAULT '0', + "role" varchar(255) DEFAULT NULL, + "match" varchar(255) DEFAULT NULL, + "read" smallint NOT NULL DEFAULT 1, + "write" smallint NOT NULL DEFAULT 1, + "create" smallint NOT NULL DEFAULT 1, + "submit" smallint NOT NULL DEFAULT 0, + "cancel" smallint NOT NULL DEFAULT 0, + "delete" smallint NOT NULL DEFAULT 1, + "amend" smallint NOT NULL DEFAULT 0, + "report" smallint NOT NULL DEFAULT 1, + "export" smallint NOT NULL DEFAULT 1, + "import" smallint NOT NULL DEFAULT 0, + "share" smallint NOT NULL DEFAULT 1, + "print" smallint NOT NULL DEFAULT 1, + "email" smallint NOT NULL DEFAULT 1, + PRIMARY KEY ("name") +) ; + +create index on "tabDocPerm" ("parent"); + +-- +-- Table structure for table "tabDocType" +-- + +DROP TABLE IF EXISTS "tabDocType"; +CREATE TABLE "tabDocType" ( + "name" varchar(255) NOT NULL DEFAULT '', + "creation" timestamp(6) DEFAULT NULL, + "modified" timestamp(6) DEFAULT NULL, + "modified_by" varchar(255) DEFAULT NULL, + "owner" varchar(255) DEFAULT NULL, + "docstatus" smallint NOT NULL DEFAULT 0, + "parent" varchar(255) DEFAULT NULL, + "parentfield" varchar(255) DEFAULT NULL, + "parenttype" varchar(255) DEFAULT NULL, + "idx" bigint NOT NULL DEFAULT 0, + "search_fields" varchar(255) DEFAULT NULL, + "issingle" smallint NOT NULL DEFAULT 0, + "istable" smallint NOT NULL DEFAULT 0, + "editable_grid" smallint NOT NULL DEFAULT 1, + "track_changes" smallint NOT NULL DEFAULT 0, + "module" varchar(255) DEFAULT NULL, + "restrict_to_domain" varchar(255) DEFAULT NULL, + "app" varchar(255) DEFAULT NULL, + "autoname" varchar(255) DEFAULT NULL, + "name_case" varchar(255) DEFAULT NULL, + "title_field" varchar(255) DEFAULT NULL, + "image_field" varchar(255) DEFAULT NULL, + "timeline_field" varchar(255) DEFAULT NULL, + "sort_field" varchar(255) DEFAULT NULL, + "sort_order" varchar(255) DEFAULT NULL, + "description" text, + "colour" varchar(255) DEFAULT NULL, + "read_only" smallint NOT NULL DEFAULT 0, + "in_create" smallint NOT NULL DEFAULT 0, + "menu_index" bigint DEFAULT NULL, + "parent_node" varchar(255) DEFAULT NULL, + "smallicon" varchar(255) DEFAULT NULL, + "allow_copy" smallint NOT NULL DEFAULT 0, + "allow_rename" smallint NOT NULL DEFAULT 0, + "allow_import" smallint NOT NULL DEFAULT 0, + "hide_toolbar" smallint NOT NULL DEFAULT 0, + "hide_heading" smallint NOT NULL DEFAULT 0, + "track_seen" smallint NOT NULL DEFAULT 0, + "max_attachments" bigint NOT NULL DEFAULT 0, + "print_outline" varchar(255) DEFAULT NULL, + "read_only_onload" smallint NOT NULL DEFAULT 0, + "document_type" varchar(255) DEFAULT NULL, + "icon" varchar(255) DEFAULT NULL, + "color" varchar(255) DEFAULT NULL, + "tag_fields" varchar(255) DEFAULT NULL, + "subject" varchar(255) DEFAULT NULL, + "_last_update" varchar(32) DEFAULT NULL, + "engine" varchar(20) DEFAULT 'InnoDB', + "default_print_format" varchar(255) DEFAULT NULL, + "is_submittable" smallint NOT NULL DEFAULT 0, + "show_name_in_global_search" smallint NOT NULL DEFAULT 0, + "_user_tags" varchar(255) DEFAULT NULL, + "custom" smallint NOT NULL DEFAULT 0, + "beta" smallint NOT NULL DEFAULT 0, + "image_view" smallint NOT NULL DEFAULT 0, + "has_web_view" smallint NOT NULL DEFAULT 0, + "allow_guest_to_view" smallint NOT NULL DEFAULT 0, + "route" varchar(255) DEFAULT NULL, + "is_published_field" varchar(255) DEFAULT NULL, + PRIMARY KEY ("name") +) ; + +-- +-- Table structure for table "tabSeries" +-- + +DROP TABLE IF EXISTS "tabSeries"; +CREATE TABLE "tabSeries" ( + "name" varchar(100) DEFAULT NULL, + "current" bigint NOT NULL DEFAULT 0 +) ; + +create index on "tabSeries" ("name"); + +-- +-- Table structure for table "tabSessions" +-- + +DROP TABLE IF EXISTS "tabSessions"; +CREATE TABLE "tabSessions" ( + "user" varchar(255) DEFAULT NULL, + "sid" varchar(255) DEFAULT NULL, + "sessiondata" text, + "ipaddress" varchar(16) DEFAULT NULL, + "lastupdate" timestamp(6) DEFAULT NULL, + "device" varchar(255) DEFAULT 'desktop', + "status" varchar(20) DEFAULT NULL +); + +create index on "tabSessions" ("sid"); + +-- +-- Table structure for table "tabSingles" +-- + +DROP TABLE IF EXISTS "tabSingles"; +CREATE TABLE "tabSingles" ( + "doctype" varchar(255) DEFAULT NULL, + "field" varchar(255) DEFAULT NULL, + "value" text +); + +create index on "tabSingles" ("doctype", "field"); + +-- +-- Table structure for table "__Auth" +-- + +DROP TABLE IF EXISTS "__Auth"; +CREATE TABLE "__Auth" ( + "doctype" VARCHAR(140) NOT NULL, + "name" VARCHAR(255) NOT NULL, + "fieldname" VARCHAR(140) NOT NULL, + "password" VARCHAR(255) NOT NULL, + "encrypted" int NOT NULL DEFAULT 0, + PRIMARY KEY ("doctype", "name", "fieldname") +); + +create index on "__Auth" ("doctype", "name", "fieldname"); + +-- +-- Table structure for table "tabFile" +-- + +DROP TABLE IF EXISTS "tabFile"; +CREATE TABLE "tabFile" ( + "name" varchar(255) NOT NULL, + "creation" timestamp(6) DEFAULT NULL, + "modified" timestamp(6) DEFAULT NULL, + "modified_by" varchar(255) DEFAULT NULL, + "owner" varchar(255) DEFAULT NULL, + "docstatus" smallint NOT NULL DEFAULT 0, + "parent" varchar(255) DEFAULT NULL, + "parentfield" varchar(255) DEFAULT NULL, + "parenttype" varchar(255) DEFAULT NULL, + "idx" bigint NOT NULL DEFAULT 0, + "file_name" varchar(255) DEFAULT NULL, + "file_url" varchar(255) DEFAULT NULL, + "module" varchar(255) DEFAULT NULL, + "attached_to_name" varchar(255) DEFAULT NULL, + "file_size" bigint NOT NULL DEFAULT 0, + "attached_to_doctype" varchar(255) DEFAULT NULL, + PRIMARY KEY ("name") +); + +create index on "tabFile" ("parent"); +create index on "tabFile" ("attached_to_name"); +create index on "tabFile" ("attached_to_doctype"); + +-- +-- Table structure for table "tabDefaultValue" +-- + +DROP TABLE IF EXISTS "tabDefaultValue"; +CREATE TABLE "tabDefaultValue" ( + "name" varchar(255) NOT NULL, + "creation" timestamp(6) DEFAULT NULL, + "modified" timestamp(6) DEFAULT NULL, + "modified_by" varchar(255) DEFAULT NULL, + "owner" varchar(255) DEFAULT NULL, + "docstatus" smallint NOT NULL DEFAULT 0, + "parent" varchar(255) DEFAULT NULL, + "parentfield" varchar(255) DEFAULT NULL, + "parenttype" varchar(255) DEFAULT NULL, + "idx" bigint NOT NULL DEFAULT 0, + "defvalue" text, + "defkey" varchar(255) DEFAULT NULL, + PRIMARY KEY ("name") +); + +create index on "tabDefaultValue" ("parent"); +create index on "tabDefaultValue" ("parent", "defkey"); + diff --git a/frappe/database/postgres/schema.py b/frappe/database/postgres/schema.py new file mode 100644 index 0000000000..05b6b19a9a --- /dev/null +++ b/frappe/database/postgres/schema.py @@ -0,0 +1,96 @@ +import frappe +from frappe import _ +from frappe.utils import cint, flt +from frappe.database.schema import DBTable, get_definition + +class PostgresTable(DBTable): + def create(self): + add_text = '' + + # columns + column_defs = self.get_column_definitions() + if column_defs: add_text += ',\n'.join(column_defs) + + # index + # index_defs = self.get_index_definitions() + # TODO: set docstatus length + # create table + frappe.db.sql("""create table `%s` ( + name varchar({varchar_len}) not null primary key, + creation timestamp(6), + modified timestamp(6), + modified_by varchar({varchar_len}), + owner varchar({varchar_len}), + docstatus smallint not null default '0', + parent varchar({varchar_len}), + parentfield varchar({varchar_len}), + parenttype varchar({varchar_len}), + idx bigint not null default '0', + %s)""".format(varchar_len=frappe.db.VARCHAR_LEN) % (self.table_name, add_text)) + + frappe.db.commit() + + def alter(self): + for col in self.columns.values(): + col.build_for_alter_table(self.current_columns.get(col.fieldname.lower())) + + query = [] + + for col in self.add_column: + query.append("ADD COLUMN `{}` {}".format(col.fieldname, col.get_definition())) + + for col in self.change_type: + query.append("ALTER COLUMN `{}` TYPE {}".format(col.fieldname, get_definition(col.fieldtype, precision=col.precision, length=col.length))) + + for col in self.set_default: + if col.fieldname=="name": + continue + + if col.fieldtype in ("Check", "Int"): + col_default = cint(col.default) + + elif col.fieldtype in ("Currency", "Float", "Percent"): + col_default = flt(col.default) + + elif not col.default: + col_default = "NULL" + + else: + col_default = "{}".format(frappe.db.escape(col.default)) + + query.append("ALTER COLUMN `{}` SET DEFAULT {}".format(col.fieldname, col_default)) + + create_index_query = "" + for col in self.add_index: + # if index key not exists + create_index_query += 'CREATE INDEX IF NOT EXISTS "{index_name}" ON `{table_name}`(`{field}`);'.format( + index_name=col.fieldname, + table_name=self.table_name, + field=col.fieldname) + + drop_index_query = "" + for col in self.drop_index: + # primary key + if col.fieldname != 'name': + # if index key exists + if not frappe.db.has_index(self.table_name, col.fieldname): + drop_index_query += 'DROP INDEX IF EXISTS "{}" ;'.format(col.fieldname) + + if query: + try: + final_alter_query = "ALTER TABLE `{}` {}".format(self.table_name, ", ".join(query)) + if final_alter_query: frappe.db.sql(final_alter_query) + if create_index_query: frappe.db.sql(create_index_query) + if drop_index_query: frappe.db.sql(drop_index_query) + except Exception as e: + # sanitize + if frappe.db.is_duplicate_fieldname(e): + frappe.throw(str(e)) + elif frappe.db.is_duplicate_entry(e): + fieldname = str(e).split("'")[-2] + frappe.throw(_("""{0} field cannot be set as unique in {1}, + as there are non-unique existing values""".format( + fieldname, self.table_name))) + raise e + else: + raise e \ No newline at end of file diff --git a/frappe/database/postgres/setup_db.py b/frappe/database/postgres/setup_db.py new file mode 100644 index 0000000000..0209d65839 --- /dev/null +++ b/frappe/database/postgres/setup_db.py @@ -0,0 +1,41 @@ +import frappe, subprocess, os + +def setup_database(force, source_sql, verbose): + root_conn = get_root_connection() + root_conn.commit() + root_conn.sql("DROP DATABASE IF EXISTS `{0}`".format(frappe.conf.db_name)) + root_conn.sql("DROP USER IF EXISTS {0}".format(frappe.conf.db_name)) + root_conn.sql("CREATE DATABASE `{0}`".format(frappe.conf.db_name)) + root_conn.sql("CREATE user {0} password '{1}'".format(frappe.conf.db_name, + frappe.conf.db_password)) + root_conn.sql("GRANT ALL PRIVILEGES ON DATABASE `{0}` TO {0}".format(frappe.conf.db_name)) + + # bootstrap db + subprocess.check_output(['psql', frappe.conf.db_name, '-qf', + os.path.join(os.path.dirname(__file__), 'framework_postgres.sql')]) + + frappe.connect() + +def setup_help_database(help_db_name): + root_conn = get_root_connection() + root_conn.sql("DROP DATABASE IF EXISTS `{0}`".format(help_db_name)) + root_conn.sql("DROP USER IF EXISTS {0}".format(help_db_name)) + root_conn.sql("CREATE DATABASE `{0}`".format(help_db_name)) + root_conn.sql("CREATE user {0} password '{1}'".format(help_db_name, help_db_name)) + root_conn.sql("GRANT ALL PRIVILEGES ON DATABASE `{0}` TO {0}".format(help_db_name)) + +def get_root_connection(root_login='postgres', root_password=None): + import getpass + if not frappe.local.flags.root_connection: + if not root_login: + root_login = 'root' + + if not root_password: + root_password = frappe.conf.get("root_password") or None + + if not root_password: + root_password = getpass.getpass("Postgres root password: ") + + frappe.local.flags.root_connection = frappe.database.get_db(user=root_login, password=root_password) + + return frappe.local.flags.root_connection diff --git a/frappe/database/schema.py b/frappe/database/schema.py new file mode 100644 index 0000000000..5fe4d756ec --- /dev/null +++ b/frappe/database/schema.py @@ -0,0 +1,339 @@ +from __future__ import unicode_literals + +import re +import frappe + +from frappe import _ +from frappe.utils import cstr, cint, flt + + +class InvalidColumnName(frappe.ValidationError): pass + +class DBTable: + def __init__(self, doctype, meta=None): + self.doctype = doctype + self.table_name = 'tab{}'.format(doctype) + self.meta = meta or frappe.get_meta(doctype) + self.columns = {} + self.current_columns = {} + + # lists for change + self.add_column = [] + self.change_type = [] + self.change_name = [] + self.add_unique = [] + self.add_index = [] + self.drop_index = [] + self.set_default = [] + + # load + self.get_columns_from_docfields() + + def sync(self): + if self.is_new(): + self.create() + else: + self.alter() + + def create(self): + pass + + def get_column_definitions(self): + column_list = [] + frappe.db.DEFAULT_COLUMNS + ret = [] + for k in list(self.columns): + if k not in column_list: + d = self.columns[k].get_definition() + if d: + ret.append('`'+ k + '` ' + d) + column_list.append(k) + return ret + + def get_index_definitions(self): + ret = [] + for key, col in self.columns.items(): + if (col.set_index + and not col.unique + and col.fieldtype in frappe.db.type_map + and frappe.db.type_map.get(col.fieldtype)[0] + not in ('text', 'longtext')): + ret.append('index `' + key + '`(`' + key + '`)') + return ret + + def get_columns_from_docfields(self): + """ + get columns from docfields and custom fields + """ + fl = frappe.db.sql("SELECT * FROM `tabDocField` WHERE parent = %s", self.doctype, as_dict = 1) + lengths = {} + precisions = {} + uniques = {} + + # optional fields like _comments + if not self.meta.istable: + for fieldname in frappe.db.OPTIONAL_COLUMNS: + fl.append({ + "fieldname": fieldname, + "fieldtype": "Text" + }) + + # add _seen column if track_seen + if getattr(self.meta, 'track_seen', False): + fl.append({ + 'fieldname': '_seen', + 'fieldtype': 'Text' + }) + + if (not frappe.flags.in_install_db + and (frappe.flags.in_install != "frappe" + or frappe.flags.ignore_in_install)): + custom_fl = frappe.db.sql(""" + SELECT * FROM `tabCustom Field` + WHERE dt = %s AND docstatus < 2 + """, (self.doctype,), as_dict=1) + if custom_fl: fl += custom_fl + + # apply length, precision and unique from property setters + for ps in frappe.get_all("Property Setter", + fields=["field_name", "property", "value"], + filters={ + "doc_type": self.doctype, + "doctype_or_field": "DocField", + "property": ["in", ["precision", "length", "unique"]] + }): + + if ps.property=="length": + lengths[ps.field_name] = cint(ps.value) + + elif ps.property=="precision": + precisions[ps.field_name] = cint(ps.value) + + elif ps.property=="unique": + uniques[ps.field_name] = cint(ps.value) + + for f in fl: + self.columns[f['fieldname']] = DbColumn(self, + f['fieldname'], + f['fieldtype'], + lengths.get(f["fieldname"]) or f.get('length'), + f.get('default'), + f.get('search_index'), + f.get('options'), + uniques.get(f["fieldname"], + f.get('unique')), + precisions.get(f['fieldname']) or f.get('precision')) + + def validate(self): + """Check if change in varchar length isn't truncating the columns""" + if self.is_new(): + return + + self.setup_table_columns() + + columns = [frappe._dict({"fieldname": f, "fieldtype": "Data"}) for f in + frappe.db.STANDARD_VARCHAR_COLUMNS] + columns += self.columns.values() + + for col in columns: + if len(col.fieldname) >= 64: + frappe.throw(_("Fieldname is limited to 64 characters ({0})") + .format(frappe.bold(col.fieldname))) + + if 'varchar' in frappe.db.type_map.get(col.fieldtype, ()): + + # validate length range + new_length = cint(col.length) or cint(frappe.db.VARCHAR_LEN) + if not (1 <= new_length <= 1000): + frappe.throw(_("Length of {0} should be between 1 and 1000").format(col.fieldname)) + + current_col = self.current_columns.get(col.fieldname, {}) + if not current_col: + continue + current_type = self.current_columns[col.fieldname]["type"] + current_length = re.findall(r'varchar\(([\d]+)\)', current_type) + if not current_length: + # case when the field is no longer a varchar + continue + current_length = current_length[0] + if cint(current_length) != cint(new_length): + try: + # check for truncation + max_length = frappe.db.sql("""SELECT MAX(CHAR_LENGTH(`{fieldname}`)) FROM `tab{doctype}`""" + .format(fieldname=col.fieldname, doctype=self.doctype)) + + except frappe.db.InternalError as e: + if frappe.db.is_missing_column(e): + # Unknown column 'column_name' in 'field list' + continue + else: + raise + + if max_length and max_length[0][0] and max_length[0][0] > new_length: + if col.fieldname in self.columns: + self.columns[col.fieldname].length = current_length + + frappe.msgprint(_("""Reverting length to {0} for '{1}' in '{2}'; + Setting the length as {3} will cause truncation of data.""") + .format(current_length, col.fieldname, self.doctype, new_length)) + + def is_new(self): + return self.table_name not in frappe.db.get_tables() + + def setup_table_columns(self): + # TODO: figure out a way to get key data + for c in frappe.db.get_table_columns_description(self.table_name): + self.current_columns[c.name.lower()] = c + + def alter(self): + pass + + +class DbColumn: + def __init__(self, table, fieldname, fieldtype, length, default, + set_index, options, unique, precision): + self.table = table + self.fieldname = fieldname + self.fieldtype = fieldtype + self.length = length + self.set_index = set_index + self.default = default + self.options = options + self.unique = unique + self.precision = precision + + def get_definition(self, with_default=1): + column_def = get_definition(self.fieldtype, precision=self.precision, length=self.length) + + if not column_def: + return column_def + + if self.fieldtype in ("Check", "Int"): + default_value = cint(self.default) or 0 + column_def += ' not null default {0}'.format(default_value) + + elif self.fieldtype in ("Currency", "Float", "Percent"): + default_value = flt(self.default) or 0 + column_def += ' not null default {0}'.format(default_value) + + elif self.default and (self.default not in frappe.db.DEFAULT_SHORTCUTS) \ + and not self.default.startswith(":") and column_def not in ('text', 'longtext'): + column_def += " default {}".format(frappe.db.escape(self.default)) + + if self.unique and (column_def not in ('text', 'longtext')): + column_def += ' unique' + + return column_def + + def build_for_alter_table(self, current_def): + column_type = get_definition(self.fieldtype, self.precision, self.length) + + # no columns + if not column_type: + return + + # to add? + if not current_def: + self.fieldname = validate_column_name(self.fieldname) + self.table.add_column.append(self) + return + + # type + if ((current_def['type']) != column_type): + self.table.change_type.append(self) + + # unique + if((self.unique and not current_def['unique']) and column_type not in ('text', 'longtext')): + self.table.add_unique.append(self) + + # default + if (self.default_changed(current_def) + and (self.default not in frappe.db.DEFAULT_SHORTCUTS) + and not cstr(self.default).startswith(":") + and not (column_type in ['text','longtext'])): + self.table.set_default.append(self) + + # index should be applied or dropped irrespective of type change + if ((current_def['index'] and not self.set_index and not self.unique) + or (current_def['unique'] and not self.unique)): + # to drop unique you have to drop index + self.table.drop_index.append(self) + + elif (not current_def['index'] and self.set_index) and not (column_type in ('text', 'longtext')): + self.table.add_index.append(self) + + def default_changed(self, current_def): + if "decimal" in current_def['type']: + return self.default_changed_for_decimal(current_def) + else: + return current_def['default'] != self.default + + def default_changed_for_decimal(self, current_def): + try: + if current_def['default'] in ("", None) and self.default in ("", None): + # both none, empty + return False + + elif current_def['default'] in ("", None): + try: + # check if new default value is valid + float(self.default) + return True + except ValueError: + return False + + elif self.default in ("", None): + # new default value is empty + return True + + else: + # NOTE float() raise ValueError when "" or None is passed + return float(current_def['default'])!=float(self.default) + except TypeError: + return True + +def validate_column_name(n): + special_characters = re.findall(r"[\W]", n, re.UNICODE) + if special_characters: + special_characters = ", ".join('"{0}"'.format(c) for c in special_characters) + frappe.throw(_("Fieldname {0} cannot have special characters like {1}").format( + frappe.bold(cstr(n)), special_characters), frappe.db.InvalidColumnName) + return n + +def validate_column_length(fieldname): + if len(fieldname) > frappe.db.MAX_COLUMN_LENGTH: + frappe.throw(_("Fieldname is limited to 64 characters ({0})").format(fieldname)) + +def get_definition(fieldtype, precision=None, length=None): + d = frappe.db.type_map.get(fieldtype) + + # convert int to long int if the length of the int is greater than 11 + if fieldtype == "Int" and length and length > 11: + d = frappe.db.type_map.get("Long Int") + + if not d: return + + coltype = d[0] + size = d[1] if d[1] else None + + if size: + if fieldtype in ["Float", "Currency", "Percent"] and cint(precision) > 6: + size = '21,9' + + if coltype == "varchar" and length: + size = length + + if size is not None: + coltype = "{coltype}({size})".format(coltype=coltype, size=size) + + return coltype + +def add_column(doctype, column_name, fieldtype, precision=None): + if column_name in frappe.db.get_table_columns(doctype): + # already exists + return + + frappe.db.commit() + frappe.db.sql("alter table `tab%s` add column %s %s" % (doctype, + column_name, get_definition(fieldtype, precision))) + + diff --git a/frappe/defaults.py b/frappe/defaults.py index 5d3aed28e0..3470a81c55 100644 --- a/frappe/defaults.py +++ b/frappe/defaults.py @@ -115,7 +115,7 @@ def set_default(key, value, parent, parenttype="__default"): select defkey from - tabDefaultValue + `tabDefaultValue` where defkey=%s and parent=%s for update''', (key, parent)): diff --git a/frappe/desk/calendar.py b/frappe/desk/calendar.py index d0b7fb8dd8..ce9fb7f177 100644 --- a/frappe/desk/calendar.py +++ b/frappe/desk/calendar.py @@ -46,7 +46,7 @@ def get_events(doctype, start, end, field_map, filters=None, fields=None): if field_map.color: fields.append(field_map.color) - start_date = "ifnull(%s, '0000-00-00 00:00:00')" % field_map.start + start_date = "ifnull(%s, '0001-01-01 00:00:00')" % field_map.start end_date = "ifnull(%s, '2199-12-31 00:00:00')" % field_map.end filters += [ diff --git a/frappe/desk/doctype/auto_repeat/test_auto_repeat.py b/frappe/desk/doctype/auto_repeat/test_auto_repeat.py index 4517a6fe52..fbf1c66936 100644 --- a/frappe/desk/doctype/auto_repeat/test_auto_repeat.py +++ b/frappe/desk/doctype/auto_repeat/test_auto_repeat.py @@ -20,7 +20,7 @@ def add_custom_fields(): class TestAutoRepeat(unittest.TestCase): def setUp(self): - if not frappe.db.sql('select name from `tabCustom Field` where name="auto_repeat"'): + if not frappe.db.sql("SELECT `name` FROM `tabCustom Field` WHERE `name`='auto_repeat'"): add_custom_fields() def test_daily_auto_repeat(self): diff --git a/frappe/desk/doctype/event/event.py b/frappe/desk/doctype/event/event.py index b99b0453f6..add3a647f2 100644 --- a/frappe/desk/doctype/event/event.py +++ b/frappe/desk/doctype/event/event.py @@ -32,9 +32,8 @@ class Event(Document): def get_permission_query_conditions(user): if not user: user = frappe.session.user - return """(tabEvent.event_type='Public' or tabEvent.owner='%(user)s')""" % { + return """(`tabEvent`.`event_type`='Public' or `tabEvent`.`owner`=%(user)s)""" % { "user": frappe.db.escape(user), - "roles": "', '".join([frappe.db.escape(r) for r in frappe.get_roles(user)]) } def has_permission(doc, user): @@ -72,28 +71,26 @@ def get_events(start, end, user=None, for_reminder=False, filters=None): user = frappe.session.user if isinstance(filters, string_types): filters = json.loads(filters) - roles = frappe.get_roles(user) - events = frappe.db.sql("""select name, subject, description, color, + events = frappe.db.sql("""select `name`, subject, description, color, starts_on, ends_on, owner, all_day, event_type, repeat_this_event, repeat_on,repeat_till, monday, tuesday, wednesday, thursday, friday, saturday, sunday - from tabEvent where (( + from `tabEvent` where (( (date(starts_on) between date(%(start)s) and date(%(end)s)) or (date(ends_on) between date(%(start)s) and date(%(end)s)) or (date(starts_on) <= date(%(start)s) and date(ends_on) >= date(%(end)s)) ) or ( date(starts_on) <= date(%(start)s) and repeat_this_event=1 and - ifnull(repeat_till, "3000-01-01") > date(%(start)s) + coalesce(repeat_till, '3000-01-01') > date(%(start)s) )) {reminder_condition} {filter_condition} and (event_type='Public' or owner=%(user)s or exists(select name from `tabDocShare` where - tabDocShare.share_doctype="Event" and `tabDocShare`.share_name=tabEvent.name - and tabDocShare.user=%(user)s)) + `tabDocShare`.share_doctype='Event' and `tabDocShare`.share_name=`tabEvent`.`name` + and `tabDocShare`.`user`=%(user)s)) order by starts_on""".format( filter_condition=get_filters_cond('Event', filters, []), - reminder_condition="and ifnull(send_reminder,0)=1" if for_reminder else "", - roles=", ".join('"{}"'.format(frappe.db.escape(r)) for r in roles) + reminder_condition="and coalesce(send_reminder, 0)=1" if for_reminder else "" ), { "start": start, "end": end, diff --git a/frappe/desk/doctype/kanban_board/kanban_board.py b/frappe/desk/doctype/kanban_board/kanban_board.py index 19f3daefe3..cc207c19e6 100644 --- a/frappe/desk/doctype/kanban_board/kanban_board.py +++ b/frappe/desk/doctype/kanban_board/kanban_board.py @@ -29,7 +29,7 @@ def get_permission_query_conditions(user): if user == "Administrator": return "" - return """(`tabKanban Board`.private=0 or `tabKanban Board`.owner="{user}")""".format(user=user) + return """(`tabKanban Board`.private=0 or `tabKanban Board`.owner='{user}')""".format(user=user) def has_permission(doc, ptype, user): if doc.private == 0 or user == "Administrator": diff --git a/frappe/desk/doctype/todo/todo.py b/frappe/desk/doctype/todo/todo.py index b9e0472f34..45c3874806 100644 --- a/frappe/desk/doctype/todo/todo.py +++ b/frappe/desk/doctype/todo/todo.py @@ -72,12 +72,12 @@ class ToDo(Document): "_assign", json.dumps(assignments), update_modified=False) except Exception as e: - if e.args[0] == 1146 and frappe.flags.in_install: + if frappe.db.is_table_missing(e) and frappe.flags.in_install: # no table return - elif e.args[0]==1054: - from frappe.model.db_schema import add_column + elif frappe.db.is_column_missing(e): + from frappe.database.schema import add_column add_column(self.reference_type, "_assign", "Text") self.update_in_reference() @@ -94,7 +94,7 @@ def get_permission_query_conditions(user): if "System Manager" in frappe.get_roles(user): return None else: - return """(tabToDo.owner = '{user}' or tabToDo.assigned_by = '{user}')"""\ + return """(tabToDo.owner = {user} or tabToDo.assigned_by = {user})"""\ .format(user=frappe.db.escape(user)) def has_permission(doc, user): diff --git a/frappe/desk/form/assign_to.py b/frappe/desk/form/assign_to.py index 15ad442f92..0e849c5ea1 100644 --- a/frappe/desk/form/assign_to.py +++ b/frappe/desk/form/assign_to.py @@ -18,9 +18,13 @@ def get(args=None): get_docinfo(frappe.get_doc(args.get("doctype"), args.get("name"))) - return frappe.db.sql("""select owner, description from `tabToDo` - where reference_type=%(doctype)s and reference_name=%(name)s and status="Open" - order by modified desc limit 5""", args, as_dict=True) + return frappe.db.sql("""SELECT `owner`, `description` + FROM `tabToDo` + WHERE reference_type=%(doctype)s + AND reference_name=%(name)s + AND status='Open' + ORDER BY modified DESC + LIMIT 5""", args, as_dict=True) @frappe.whitelist() def add(args=None): @@ -36,9 +40,12 @@ def add(args=None): if not args: args = frappe.local.form_dict - if frappe.db.sql("""select owner from `tabToDo` - where reference_type=%(doctype)s and reference_name=%(name)s and status="Open" - and owner=%(assign_to)s""", args): + if frappe.db.sql("""SELECT `owner` + FROM `tabToDo` + WHERE `reference_type`=%(doctype)s + AND `reference_name`=%(name)s + AND `status`='Open' + AND `owner`=%(assign_to)s""", args): frappe.throw(_("Already in user's To Do list"), DuplicateToDoError) else: diff --git a/frappe/desk/form/load.py b/frappe/desk/form/load.py index 64bcf4044e..2304eb9d31 100644 --- a/frappe/desk/form/load.py +++ b/frappe/desk/form/load.py @@ -138,24 +138,22 @@ def get_communication_data(doctype, name, start=0, limit=20, after=None, fields= group_by=None, as_dict=True): '''Returns list of communications for a given document''' if not fields: - fields = '''name, communication_type, - communication_medium, comment_type, communication_date, - content, sender, sender_full_name, creation, subject, delivery_status, _liked_by, - timeline_doctype, timeline_name, - reference_doctype, reference_name, - link_doctype, link_name, read_by_recipient, - rating, "Communication" as doctype''' + fields = '''`name`, `communication_type`,`communication_medium`, `comment_type`, + `communication_date`, `content`, `sender`, `sender_full_name`, + `creation`, `subject`, `delivery_status`, `_liked_by`, + `timeline_doctype`, `timeline_name`, `reference_doctype`, `reference_name`, + `link_doctype`, `link_name`, `read_by_recipient`, `rating` ''' - conditions = '''communication_type in ("Communication", "Comment", "Feedback") + conditions = '''communication_type in ('Communication', 'Comment', 'Feedback') and ( (reference_doctype=%(doctype)s and reference_name=%(name)s) or ( (timeline_doctype=%(doctype)s and timeline_name=%(name)s) and ( - communication_type="Communication" + communication_type='Communication' or ( - communication_type="Comment" - and comment_type in ("Created", "Updated", "Submitted", "Cancelled", "Deleted") + communication_type='Comment' + and comment_type in ('Created', 'Updated', 'Submitted', 'Cancelled', 'Deleted') ))) )''' @@ -165,12 +163,12 @@ def get_communication_data(doctype, name, start=0, limit=20, after=None, fields= conditions+= ' and creation > {0}'.format(after) if doctype=='User': - conditions+= ' and not (reference_doctype="User" and communication_type="Communication")' + conditions+= " and not (reference_doctype='User' and communication_type='Communication')" communications = frappe.db.sql("""select {fields} - from tabCommunication + from `tabCommunication` where {conditions} {group_by} - order by creation desc limit %(start)s, %(limit)s""".format( + order by creation desc LIMIT %(limit)s OFFSET %(start)s""".format( fields = fields, conditions=conditions, group_by=group_by or ""), { "doctype": doctype, "name": name, "start": frappe.utils.cint(start), "limit": limit }, as_dict=as_dict) @@ -178,8 +176,8 @@ def get_communication_data(doctype, name, start=0, limit=20, after=None, fields= return communications def get_assignments(dt, dn): - cl = frappe.db.sql("""select name, owner, description from `tabToDo` - where reference_type=%(doctype)s and reference_name=%(name)s and status="Open" + cl = frappe.db.sql("""select `name`, owner, description from `tabToDo` + where reference_type=%(doctype)s and reference_name=%(name)s and status='Open' order by modified desc limit 5""", { "doctype": dt, "name": dn diff --git a/frappe/desk/form/utils.py b/frappe/desk/form/utils.py index bad7657a8e..a787e597fa 100644 --- a/frappe/desk/form/utils.py +++ b/frappe/desk/form/utils.py @@ -30,8 +30,7 @@ def validate_link(): frappe.response['message'] = 'Ok' return - valid_value = frappe.db.sql("select name from `tab%s` where name=%s" % (frappe.db.escape(options), - '%s'), (value,)) + valid_value = frappe.db.get_all(options, filters=dict(name=value), as_list=1, limit=1) if valid_value: valid_value = valid_value[0][0] diff --git a/frappe/desk/like.py b/frappe/desk/like.py index 769fb63bcc..ec01eace83 100644 --- a/frappe/desk/like.py +++ b/frappe/desk/like.py @@ -6,7 +6,7 @@ from __future__ import unicode_literals """Allow adding of likes to documents""" import frappe, json -from frappe.model.db_schema import add_column +from frappe.database.schema import add_column from frappe import _ from frappe.utils import get_link_to_form @@ -54,8 +54,8 @@ def _toggle_like(doctype, name, add, user=None): frappe.db.set_value(doctype, name, "_liked_by", json.dumps(liked_by), update_modified=False) - except Exception as e: - if isinstance(e.args, (tuple, list)) and e.args and e.args[0]==1054: + except frappe.db.ProgrammingError as e: + if frappe.db.is_column_missing(e): add_column(doctype, "_liked_by", "Text") _toggle_like(doctype, name, add, user) else: diff --git a/frappe/desk/moduleview.py b/frappe/desk/moduleview.py index 0b4f1eb853..f5a1e47bf4 100644 --- a/frappe/desk/moduleview.py +++ b/frappe/desk/moduleview.py @@ -224,7 +224,7 @@ def get_last_modified(doctype): try: last_modified = frappe.get_all(doctype, fields=["max(modified)"], as_list=True, limit_page_length=1)[0][0] except Exception as e: - if e.args[0]==1146: + if frappe.db.is_table_missing(e): last_modified = None else: raise diff --git a/frappe/desk/query_builder.py b/frappe/desk/query_builder.py index 2acf0c4526..ef6240b8dd 100644 --- a/frappe/desk/query_builder.py +++ b/frappe/desk/query_builder.py @@ -1,5 +1,5 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors -# MIT License. See license.txt +# MIT License. See license.txt from __future__ import unicode_literals import frappe @@ -10,9 +10,6 @@ from frappe.utils import cint import frappe.defaults from six import text_type -# imports - third-party imports -import pymysql - def get_sql_tables(q): if q.find('WHERE') != -1: tl = q.split('FROM')[1].split('WHERE')[0].split(',') @@ -78,16 +75,16 @@ def add_match_conditions(q, tl): q = q[0] + condition_st + '(' + ' OR '.join(sl) + ') ' + condition_end + q[1] else: q = q + condition_st + '(' + ' OR '.join(sl) + ')' - + return q def guess_type(m): """ Returns fieldtype depending on the MySQLdb Description """ - if m in pymysql.NUMBER: + if frappe.db.is_type_number(m): return 'Currency' - elif m in pymysql.DATE: + elif m in frappe.is_type_datetime(m): return 'Date' else: return 'Data' @@ -97,7 +94,7 @@ def build_description_simple(): for m in frappe.db.get_description(): colnames.append(m[0]) - coltypes.append(guess_type[m[0]]) + coltypes.append(guess_type[m[1]]) coloptions.append('') colwidths.append('100') @@ -178,7 +175,7 @@ def runquery(q='', ret=0, from_export=0): meta = get_sql_meta(tl) q = add_match_conditions(q, tl) - + # replace special variables q = q.replace('__user', frappe.session.user) q = q.replace('__today', frappe.utils.nowdate()) @@ -264,9 +261,9 @@ def add_limit_to_query(query, args): if args.get('limit_page_length'): query += """ limit %(limit_start)s, %(limit_page_length)s""" - + import frappe.utils args['limit_start'] = frappe.utils.cint(args.get('limit_start')) args['limit_page_length'] = frappe.utils.cint(args.get('limit_page_length')) - + return query, args diff --git a/frappe/desk/reportview.py b/frappe/desk/reportview.py index de80a0ae8a..5d137eef56 100644 --- a/frappe/desk/reportview.py +++ b/frappe/desk/reportview.py @@ -11,9 +11,6 @@ from frappe.model.db_query import DatabaseQuery from frappe import _ from six import text_type, string_types, StringIO -# imports - third-party imports -import pymysql - @frappe.whitelist() @frappe.read_only() def get(): @@ -235,11 +232,11 @@ def delete_items(): @frappe.whitelist() @frappe.read_only() def get_sidebar_stats(stats, doctype, filters=[]): - cat_tags = frappe.db.sql("""select tag.parent as category, tag.tag_name as tag - from `tabTag Doc Category` as docCat - INNER JOIN tabTag as tag on tag.parent = docCat.parent - where docCat.tagdoc=%s - ORDER BY tag.parent asc,tag.idx""",doctype,as_dict=1) + cat_tags = frappe.db.sql("""select `tag`.parent as `category`, `tag`.tag_name as `tag` + from `tabTag Doc Category` as `docCat` + INNER JOIN `tabTag` as `tag` on `tag`.parent = `docCat`.parent + where `docCat`.tagdoc=%s + ORDER BY `tag`.parent asc, `tag`.idx""", doctype, as_dict=1) return {"defined_cat":cat_tags, "stats":get_stats(stats, doctype, filters)} @@ -254,7 +251,7 @@ def get_stats(stats, doctype, filters=[]): try: columns = frappe.db.get_table_columns(doctype) - except pymysql.InternalError: + except frappe.db.InternalError: # raised when _user_tags column is added on the fly columns = [] @@ -273,10 +270,10 @@ def get_stats(stats, doctype, filters=[]): else: stats[tag] = tagcount - except frappe.SQLError: + except frappe.db.SQLError: # does not work for child tables pass - except pymysql.InternalError: + except frappe.db.InternalError: # raised when _user_tags column is added on the fly pass return stats diff --git a/frappe/desk/search.py b/frappe/desk/search.py index 01a6192460..69af811e72 100644 --- a/frappe/desk/search.py +++ b/frappe/desk/search.py @@ -20,7 +20,7 @@ def sanitize_searchfield(searchfield): if len(searchfield) == 1: # do not allow special characters to pass as searchfields - regex = re.compile('^.*[=;*,\'"$\-+%#@()_].*') + regex = re.compile(r'^.*[=;*,\'"$\-+%#@()_].*') if regex.match(searchfield): _raise_exception(searchfield) @@ -43,7 +43,7 @@ def sanitize_searchfield(searchfield): _raise_exception(searchfield) else: - regex = re.compile('^.*[=;*,\'"$\-+%#@()].*') + regex = re.compile(r'^.*[=;*,\'"$\-+%#@()].*') if any(regex.match(f) for f in searchfield.split()): _raise_exception(searchfield) @@ -126,14 +126,15 @@ def search_widget(doctype, txt, query=None, searchfield=None, start=0, formatted_fields = ['`tab%s`.`%s`' % (meta.name, f.strip()) for f in fields] # find relevance as location of search term from the beginning of string `name`. used for sorting results. - formatted_fields.append("""locate("{_txt}", `tab{doctype}`.`name`) as `_relevance`""".format( - _txt=frappe.db.escape((txt or "").replace("%", "")), doctype=frappe.db.escape(doctype))) + formatted_fields.append("""locate({_txt}, `tab{doctype}`.`name`) as `_relevance`""".format( + _txt=frappe.db.escape((txt or "").replace("%", "")), doctype=doctype)) # In order_by, `idx` gets second priority, because it stores link count from frappe.model.db_query import get_order_by order_by_based_on_meta = get_order_by(doctype, meta) - order_by = "if(_relevance, _relevance, 99999), {0}, `tab{1}`.idx desc".format(order_by_based_on_meta, doctype) + # 2 is the index of _relevance column + order_by = "2 , {0}, `tab{1}`.idx desc".format(order_by_based_on_meta, doctype) ignore_permissions = True if doctype == "DocType" else (cint(ignore_user_permissions) and has_permission(doctype)) diff --git a/frappe/desk/tags.py b/frappe/desk/tags.py index 5d64e018cb..4d652cce93 100644 --- a/frappe/desk/tags.py +++ b/frappe/desk/tags.py @@ -33,7 +33,7 @@ def check_user_tags(dt): try: frappe.db.sql("select `_user_tags` from `tab%s` limit 1" % dt) except Exception as e: - if e.args[0] == 1054: + if frappe.db.is_column_missing(e): DocTags(dt).setup() @frappe.whitelist() @@ -62,11 +62,11 @@ def get_tags(doctype, txt, cat_tags): try: for _user_tags in frappe.db.sql_list("""select DISTINCT `_user_tags` from `tab{0}` - where _user_tags like '%{1}%' - limit 50""".format(frappe.db.escape(doctype), frappe.db.escape(txt))): + where _user_tags like '{1}' + limit 50""".format(doctype, frappe.db.escape('%' + txt + '%'))): tags.extend(_user_tags[1:].split(",")) except Exception as e: - if e.args[0]!=1054: raise + if not frappe.db.is_column_missing(e): raise return sorted(filter(lambda t: t and txt.lower() in t.lower(), list(set(tags)))) class DocTags: @@ -112,7 +112,7 @@ class DocTags: doc= frappe.get_doc(self.dt, dn) update_global_search(doc) except Exception as e: - if e.args[0]==1054: + if frappe.db.is_column_missing(e): if not tags: # no tags, nothing to do return @@ -123,5 +123,5 @@ class DocTags: def setup(self): """adds the _user_tags column if not exists""" - from frappe.model.db_schema import add_column + from frappe.database.schema import add_column add_column(self.dt, "_user_tags", "Data") diff --git a/frappe/email/__init__.py b/frappe/email/__init__.py index 53c6140d04..73622a4569 100644 --- a/frappe/email/__init__.py +++ b/frappe/email/__init__.py @@ -26,7 +26,7 @@ def get_contact_list(txt, page_length=20): where name like %(txt)s %(condition)s limit %(page_length)s - """, {'txt': "%%%s%%" % frappe.db.escape(txt), + """, {'txt': frappe.db.escape('%' + txt + '%'), 'condition': match_conditions, 'page_length': page_length}, as_dict=True) out = filter(None, out) diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index 591c755777..f61ee02806 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -342,9 +342,10 @@ class EmailAccount(Document): raise SentEmailInInbox if email.message_id: - names = frappe.db.sql("""select distinct name from tabCommunication - where message_id='{message_id}' - order by creation desc limit 1""".format( + # https://stackoverflow.com/a/18367248 + names = frappe.db.sql("""SELECT DISTINCT `name`, `creation` FROM `tabCommunication` + WHERE `message_id`='{message_id}' + ORDER BY `creation` DESC LIMIT 1""".format( message_id=email.message_id ), as_dict=True) @@ -462,7 +463,7 @@ class EmailAccount(Document): # try and match by subject and sender # if sent by same sender with same subject, # append it to old coversation - subject = frappe.as_unicode(strip(re.sub("(^\s*(fw|fwd|wg)[^:]*:|\s*(re|aw)[^:]*:\s*)*", + subject = frappe.as_unicode(strip(re.sub(r"(^\s*(fw|fwd|wg)[^:]*:|\s*(re|aw)[^:]*:\s*)*", "", email.subject, 0, flags=re.IGNORECASE))) parent = frappe.db.get_all(self.append_to, filters={ @@ -604,7 +605,7 @@ class EmailAccount(Document): return flags = frappe.db.sql("""select name, communication, uid, action from - `tabEmail Flag Queue` where is_completed=0 and email_account='{email_account}' + `tabEmail Flag Queue` where is_completed=0 and email_account={email_account} """.format(email_account=frappe.db.escape(self.name)), as_dict=True) uid_list = { flag.get("uid", None): flag.get("action", "Read") for flag in flags } diff --git a/frappe/email/doctype/email_account/test_email_account.py b/frappe/email/doctype/email_account/test_email_account.py index bae0229a8c..6fa7ff30ed 100644 --- a/frappe/email/doctype/email_account/test_email_account.py +++ b/frappe/email/doctype/email_account/test_email_account.py @@ -46,13 +46,13 @@ class TestEmailAccount(unittest.TestCase): comm = frappe.get_doc("Communication", {"sender": "test_sender@example.com"}) comm.db_set("creation", datetime.now() - timedelta(seconds = 30 * 60)) - frappe.db.sql("delete from `tabEmail Queue`") + frappe.db.sql("DELETE FROM `tabEmail Queue`") notify_unreplied() self.assertTrue(frappe.db.get_value("Email Queue", {"reference_doctype": comm.reference_doctype, "reference_name": comm.reference_name, "status":"Not Sent"})) def test_incoming_with_attach(self): - frappe.db.sql("delete from tabCommunication where sender='test_sender@example.com'") + frappe.db.sql("DELETE FROM `tabCommunication` WHERE sender='test_sender@example.com'") existing_file = frappe.get_doc({'doctype': 'File', 'file_name': 'erpnext-conf-14.png'}) frappe.delete_doc("File", existing_file.name) delete_file_from_filesystem(existing_file) diff --git a/frappe/email/doctype/newsletter/newsletter.py b/frappe/email/doctype/newsletter/newsletter.py index d69fae1f1d..9dd7555c45 100755 --- a/frappe/email/doctype/newsletter/newsletter.py +++ b/frappe/email/doctype/newsletter/newsletter.py @@ -252,12 +252,21 @@ def get_list_context(context=None): def get_newsletter_list(doctype, txt, filters, limit_start, limit_page_length=20, order_by="modified"): - email_group_list = frappe.db.sql('''select eg.name from `tabEmail Group` eg, `tabEmail Group Member` egm - where egm.unsubscribed=0 and eg.name=egm.email_group and egm.email = %s''', frappe.session.user) - if email_group_list: - return frappe.db.sql('''select n.name, n.subject, n.message, n.modified - from `tabNewsletter` n, `tabNewsletter Email Group` neg - where n.name = neg.parent and n.email_sent=1 and n.published=1 and neg.email_group in %s - order by n.modified desc limit {0}, {1} - '''.format(limit_start, limit_page_length), [email_group_list], as_dict=1) + email_group_list = frappe.db.sql('''SELECT eg.name + FROM `tabEmail Group` eg, `tabEmail Group Member` egm + WHERE egm.unsubscribed=0 + AND eg.name=egm.email_group + AND egm.email = %s''', frappe.session.user) + email_group_list = [d[0] for d in email_group_list] + + if email_group_list: + return frappe.db.sql('''SELECT n.name, n.subject, n.message, n.modified + FROM `tabNewsletter` n, `tabNewsletter Email Group` neg + WHERE n.name = neg.parent + AND n.email_sent=1 + AND n.published=1 + AND neg.email_group in ({0}) + ORDER BY n.modified DESC LIMIT {1} OFFSET {2} + '''.format(','.join(['%s'] * len(email_group_list)), + limit_page_length, limit_start), email_group_list, as_dict=1) diff --git a/frappe/email/doctype/notification/notification.py b/frappe/email/doctype/notification/notification.py index 60862b1ff1..46ff6f142f 100644 --- a/frappe/email/doctype/notification/notification.py +++ b/frappe/email/doctype/notification/notification.py @@ -8,16 +8,12 @@ import json, os from frappe import _ from frappe.model.document import Document from frappe.core.doctype.role.role import get_emails_from_role -from frappe.utils import validate_email_add, nowdate, parse_val, is_html +from frappe.utils import validate_email_add, nowdate, parse_val, is_html, add_to_date from frappe.utils.jinja import validate_template from frappe.modules.utils import export_module_json, get_doc_module from six import string_types from frappe.integrations.doctype.slack_webhook_url.slack_webhook_url import send_slack_message -# imports - third-party imports -import pymysql -from pymysql.constants import ER - class Notification(Document): def onload(self): '''load message''' @@ -91,11 +87,19 @@ def get_context(context): if self.event=="Days After": diff_days = -diff_days - for name in frappe.db.sql_list("""select name from `tab{0}` where - DATE(`{1}`) = ADDDATE(DATE(%s), INTERVAL %s DAY)""".format(self.document_type, - self.date_changed), (nowdate(), diff_days or 0)): + reference_date = add_to_date(nowdate(), days=diff_days) + reference_date_start = reference_date + ' 00:00:00.000000' + reference_date_end = reference_date + ' 23:59:59.000000' - doc = frappe.get_doc(self.document_type, name) + doc_list = frappe.get_all(self.document_type, + fields='name', + filters=[ + { self.date_changed: ('>=', reference_date_start) }, + { self.date_changed: ('<=', reference_date_end) } + ]) + + for d in doc_list: + doc = frappe.get_doc(self.document_type, d.name) if self.condition and not frappe.safe_eval(self.condition, None, get_context(doc)): continue @@ -246,9 +250,14 @@ def trigger_notifications(doc, method=None): return if method == "daily": - for alert in frappe.db.sql_list("""select name from `tabNotification` - where event in ('Days Before', 'Days After') and enabled=1"""): - alert = frappe.get_doc("Notification", alert) + doc_list = frappe.get_all('Notification', + filters={ + 'event': ('in', ('Days Before', 'Days After')), + 'enabled': 1 + }) + for d in doc_list: + alert = frappe.get_doc("Notification", d.name) + for doc in alert.get_documents_for_today(): evaluate_alert(doc, alert, alert.event) frappe.db.commit() @@ -268,12 +277,13 @@ def evaluate_alert(doc, alert, event): if event=="Value Change" and not doc.is_new(): try: db_value = frappe.db.get_value(doc.doctype, doc.name, alert.value_changed) - except pymysql.InternalError as e: - if e.args[0]== ER.BAD_FIELD_ERROR: + except Exception as e: + if frappe.db.is_missing_column(e): alert.db_set('enabled', 0) frappe.log_error('Notification {0} has been disabled due to missing field'.format(alert.name)) return - + else: + raise db_value = parse_val(db_value) if (doc.get(alert.value_changed) == db_value) or \ (not db_value and not doc.get(alert.value_changed)): diff --git a/frappe/email/queue.py b/frappe/email/queue.py index 528e60c6c8..d299658878 100755 --- a/frappe/email/queue.py +++ b/frappe/email/queue.py @@ -253,12 +253,12 @@ def check_email_limit(recipients): EmailLimitCrossedError) def get_emails_sent_this_month(): - return frappe.db.sql("""select count(name) from `tabEmail Queue` where - status='Sent' and MONTH(creation)=MONTH(CURDATE())""")[0][0] + return frappe.db.sql("""SELECT COUNT(`name`) FROM `tabEmail Queue` WHERE + `status`='Sent' AND EXTRACT(MONTH FROM `creation`) = EXTRACT(MONTH FROM NOW())""")[0][0] def get_emails_sent_today(): - return frappe.db.sql("""select count(name) from `tabEmail Queue` where - status='Sent' and creation>DATE_SUB(NOW(), INTERVAL 24 HOUR)""")[0][0] + return frappe.db.sql("""SELECT COUNT(`name`) FROM `tabEmail Queue` WHERE + `status`='Sent' AND `creation` > (NOW() - INTERVAL '24' HOUR)""")[0][0] def get_unsubscribe_message(unsubscribe_message, expose_recipients): if unsubscribe_message: @@ -554,17 +554,21 @@ def clear_outbox(): Note: Used separate query to avoid deadlock """ - email_queues = frappe.db.sql_list("""select name from `tabEmail Queue` - where priority=0 and datediff(now(), modified) > 31""") + email_queues = frappe.db.sql_list("""SELECT `name` FROM `tabEmail Queue` + WHERE `priority`=0 AND `modified` < (NOW() - INTERVAL '31' DAY)""") if email_queues: - frappe.db.sql("""delete from `tabEmail Queue` where name in (%s)""" - % ','.join(['%s']*len(email_queues)), tuple(email_queues)) + frappe.db.sql("""DELETE FROM `tabEmail Queue` WHERE `name` IN ({0})""".format( + ','.join(['%s']*len(email_queues) + )), tuple(email_queues)) - frappe.db.sql("""delete from `tabEmail Queue Recipient` where parent in (%s)""" - % ','.join(['%s']*len(email_queues)), tuple(email_queues)) + frappe.db.sql("""DELETE FROM `tabEmail Queue Recipient` WHERE `parent` IN ({0})""".format( + ','.join(['%s']*len(email_queues) + )), tuple(email_queues)) frappe.db.sql(""" - update `tabEmail Queue` - set status='Expired' - where datediff(curdate(), modified) > 7 and status='Not Sent' and (send_after is null or send_after < %(now)s)""", { 'now': now_datetime() }) + UPDATE `tabEmail Queue` + SET `status`='Expired' + WHERE `modified` < (NOW() - INTERVAL '7' DAY) + AND `status`='Not Sent' + AND (`send_after` IS NULL OR `send_after` < %(now)s)""", { 'now': now_datetime() }) diff --git a/frappe/exceptions.py b/frappe/exceptions.py index 3a8548a6e0..857030ff55 100644 --- a/frappe/exceptions.py +++ b/frappe/exceptions.py @@ -6,10 +6,6 @@ from __future__ import unicode_literals # BEWARE don't put anything in this file except exceptions from werkzeug.exceptions import NotFound -# imports - third-party imports -from pymysql import ProgrammingError as SQLError, Error -# from pymysql import OperationalError as DatabaseOperationalError - class ValidationError(Exception): http_status_code = 417 @@ -46,7 +42,7 @@ class Redirect(Exception): class CSRFTokenError(Exception): http_status_code = 400 -class ImproperDBConfigurationError(Error): +class ImproperDBConfigurationError(Exception): """ Used when frappe detects that database or tables are not properly configured @@ -84,3 +80,4 @@ class RetryBackgroundJobError(Exception): pass class DocumentLockedError(ValidationError): pass class CircularLinkingError(ValidationError): pass class SecurityException(Exception): pass +class InvalidColumnName(ValidationError): pass diff --git a/frappe/installer.py b/frappe/installer.py index 668f5a16ea..c4af23976f 100755 --- a/frappe/installer.py +++ b/frappe/installer.py @@ -8,104 +8,44 @@ from __future__ import unicode_literals, print_function from six.moves import input -import os, json, sys, subprocess, shutil +import os, json, subprocess, shutil import frappe import frappe.database -import getpass import importlib from frappe import _ -from frappe.model.db_schema import DbManager from frappe.model.sync import sync_for from frappe.utils.fixtures import sync_fixtures from frappe.website import render from frappe.desk.doctype.desktop_icon.desktop_icon import sync_from_app -from frappe.utils.password import create_auth_table -from frappe.utils.global_search import setup_global_search_table from frappe.modules.utils import sync_customizations +from frappe.database import setup_database def install_db(root_login="root", root_password=None, db_name=None, source_sql=None, - admin_password=None, verbose=True, force=0, site_config=None, reinstall=False): - make_conf(db_name, site_config=site_config) - frappe.flags.in_install_db = True - if reinstall: - frappe.connect(db_name=db_name) - dbman = DbManager(frappe.local.db) - dbman.create_database(db_name) + admin_password=None, verbose=True, force=0, site_config=None, reinstall=False, + db_type=None): + + if not db_type: + db_type = frappe.conf.db_type or 'mariadb' - else: - frappe.local.db = get_root_connection(root_login, root_password) - frappe.local.session = frappe._dict({'user':'Administrator'}) - create_database_and_user(force, verbose) + + make_conf(db_name, site_config=site_config, db_type=db_type) + frappe.flags.in_install_db = True + + frappe.flags.root_login = root_login + frappe.flags.root_password = root_password + setup_database(force, source_sql, verbose) frappe.conf.admin_password = frappe.conf.admin_password or admin_password - frappe.connect(db_name=db_name) - check_if_ready_for_barracuda() - import_db_from_sql(source_sql, verbose) - if not 'tabDefaultValue' in frappe.db.get_tables(): - print('''Database not installed, this can due to lack of permission, or that the database name exists. -Check your mysql root password, or use --force to reinstall''') - sys.exit(1) - remove_missing_apps() - create_auth_table() - setup_global_search_table() - create_user_settings_table() + frappe.db.create_auth_table() + frappe.db.create_global_search_table() + frappe.db.create_user_settings_table() frappe.flags.in_install_db = False -def create_database_and_user(force, verbose): - db_name = frappe.local.conf.db_name - dbman = DbManager(frappe.local.db) - if force or (db_name not in dbman.get_database_list()): - dbman.delete_user(db_name) - dbman.drop_database(db_name) - else: - raise Exception("Database %s already exists" % (db_name,)) - - dbman.create_user(db_name, frappe.conf.db_password) - if verbose: print("Created user %s" % db_name) - - dbman.create_database(db_name) - if verbose: print("Created database %s" % db_name) - - dbman.grant_all_privileges(db_name, db_name) - dbman.flush_privileges() - if verbose: print("Granted privileges to user %s and database %s" % (db_name, db_name)) - - # close root connection - frappe.db.close() - -def create_user_settings_table(): - frappe.db.sql_ddl("""create table if not exists __UserSettings ( - `user` VARCHAR(180) NOT NULL, - `doctype` VARCHAR(180) NOT NULL, - `data` TEXT, - UNIQUE(user, doctype) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8""") - -def import_db_from_sql(source_sql, verbose): - if verbose: print("Starting database import...") - db_name = frappe.conf.db_name - if not source_sql: - source_sql = os.path.join(os.path.dirname(frappe.__file__), 'data', 'Framework.sql') - DbManager(frappe.local.db).restore_database(db_name, source_sql, db_name, frappe.conf.db_password) - if verbose: print("Imported from database %s" % source_sql) - -def get_root_connection(root_login='root', root_password=None): - if not frappe.local.flags.root_connection: - if root_login: - if not root_password: - root_password = frappe.conf.get("root_password") or None - - if not root_password: - root_password = getpass.getpass("MySQL root password: ") - frappe.local.flags.root_connection = frappe.database.Database(user=root_login, password=root_password) - - return frappe.local.flags.root_connection - def install_app(name, verbose=False, set_as_patched=True): frappe.flags.in_install = name frappe.flags.ignore_in_install = False @@ -117,7 +57,7 @@ def install_app(name, verbose=False, set_as_patched=True): # install pre-requisites if app_hooks.required_apps: for app in app_hooks.required_apps: - install_app(app) + install_app(app, verbose=verbose) frappe.flags.in_install = name frappe.clear_cache() @@ -257,14 +197,14 @@ def init_singles(): doc.flags.ignore_validate=True doc.save() -def make_conf(db_name=None, db_password=None, site_config=None): +def make_conf(db_name=None, db_password=None, site_config=None, db_type=None): site = frappe.local.site - make_site_config(db_name, db_password, site_config) + make_site_config(db_name, db_password, site_config, db_type=db_type) sites_path = frappe.local.sites_path frappe.destroy() frappe.init(site, sites_path=sites_path) -def make_site_config(db_name=None, db_password=None, site_config=None): +def make_site_config(db_name=None, db_password=None, site_config=None, db_type=None): frappe.create_folder(os.path.join(frappe.local.site_path)) site_file = get_site_config_path() @@ -272,6 +212,9 @@ def make_site_config(db_name=None, db_password=None, site_config=None): if not (site_config and isinstance(site_config, dict)): site_config = get_conf_params(db_name, db_password) + if db_type: + site_config['db_type'] = db_type + with open(site_file, "w") as f: f.write(json.dumps(site_config, indent=1, sort_keys=True)) @@ -300,7 +243,7 @@ def update_site_config(key, value, validate=True, site_config_path=None): with open(site_config_path, "w") as f: f.write(json.dumps(site_config, indent=1, sort_keys=True)) - + if hasattr(frappe.local, "conf"): frappe.local.conf[key] = value @@ -353,47 +296,6 @@ def remove_missing_apps(): installed_apps.remove(app) frappe.db.set_global("installed_apps", json.dumps(installed_apps)) -def check_if_ready_for_barracuda(): - mariadb_variables = frappe._dict(frappe.db.sql("""show variables""")) - mariadb_minor_version = int(mariadb_variables.get('version').split('-')[0].split('.')[1]) - if mariadb_minor_version < 3: - check_database(mariadb_variables, { - "innodb_file_format": "Barracuda", - "innodb_file_per_table": "ON", - "innodb_large_prefix": "ON" - }) - check_database(mariadb_variables, { - "character_set_server": "utf8mb4", - "collation_server": "utf8mb4_unicode_ci" - }) - -def check_database(mariadb_variables, variables_dict): - mariadb_minor_version = int(mariadb_variables.get('version').split('-')[0].split('.')[1]) - for key, value in variables_dict.items(): - if mariadb_variables.get(key) != value: - site = frappe.local.site - msg = ("Creation of your site - {x} failed because MariaDB is not properly {sep}" - "configured to use the Barracuda storage engine. {sep}" - "Please add the settings below to MariaDB's my.cnf, restart MariaDB then {sep}" - "run `bench new-site {x}` again.{sep2}" - "").format(x=site, sep2="\n"*2, sep="\n") - - if mariadb_minor_version < 3: - print_db_config(msg, expected_config_for_barracuda_2) - else: - print_db_config(msg, expected_config_for_barracuda_3) - raise frappe.exceptions.ImproperDBConfigurationError( - reason="MariaDB default file format is not Barracuda" - ) - - -def print_db_config(explanation, config_text): - print("="*80) - print(explanation) - print(config_text) - print("="*80) - - def extract_sql_gzip(sql_gz_path): try: subprocess.check_call(['gzip', '-d', '-v', '-f', sql_gz_path]) @@ -421,25 +323,4 @@ def extract_tar_files(site_name, file_path, folder_name): finally: frappe.destroy() - return tar_path - -expected_config_for_barracuda_2 = """[mysqld] -innodb-file-format=barracuda -innodb-file-per-table=1 -innodb-large-prefix=1 -character-set-client-handshake = FALSE -character-set-server = utf8mb4 -collation-server = utf8mb4_unicode_ci - -[mysql] -default-character-set = utf8mb4 -""" - -expected_config_for_barracuda_3 = """[mysqld] -character-set-client-handshake = FALSE -character-set-server = utf8mb4 -collation-server = utf8mb4_unicode_ci - -[mysql] -default-character-set = utf8mb4 -""" + return tar_path \ No newline at end of file diff --git a/frappe/limits.py b/frappe/limits.py index 0330db2567..93e0e05ee0 100755 --- a/frappe/limits.py +++ b/frappe/limits.py @@ -207,7 +207,7 @@ def update_space_usage(): files_size += get_folder_size(frappe.get_site_path("private", "files")) backup_size = get_folder_size(frappe.get_site_path("private", "backups")) - database_size = get_database_size() + database_size = frappe.db.get_database_size() usage = { 'files_size': flt(files_size, 2), @@ -225,13 +225,3 @@ def get_folder_size(path): if os.path.exists(path): return flt(subprocess.check_output(['du', '-ms', path]).split()[0], 2) -def get_database_size(): - '''Returns approximate database size in MB''' - db_name = frappe.conf.db_name - - # This query will get the database size in MB - db_size = frappe.db.sql(''' - SELECT table_schema "database_name", sum( data_length + index_length ) / 1024 / 1024 "database_size" - FROM information_schema.TABLES WHERE table_schema = %s GROUP BY table_schema''', db_name, as_dict=True) - - return flt(db_size[0].get('database_size'), 2) diff --git a/frappe/model/__init__.py b/frappe/model/__init__.py index f4f8207ba0..b48b29702c 100644 --- a/frappe/model/__init__.py +++ b/frappe/model/__init__.py @@ -6,6 +6,34 @@ from __future__ import unicode_literals import frappe import json +data_fieldtypes = ( + 'Currency', + 'Int', + 'Long Int', + 'Float', + 'Percent', + 'Check', + 'Small Text', + 'Long Text', + 'Code', + 'Text Editor', + 'Date', + 'Datetime', + 'Time', + 'Text', + 'Data', + 'Link', + 'Dynamic Link', + 'Password', + 'Select', + 'Read Only', + 'Attach', + 'Attach Image', + 'Signature', + 'Color', + 'Barcode', + 'Geolocation' +) no_value_fields = ('Section Break', 'Column Break', 'HTML', 'Table', 'Button', 'Image', 'Fold', 'Heading') @@ -14,31 +42,12 @@ default_fields = ('doctype','name','owner','creation','modified','modified_by', 'parent','parentfield','parenttype','idx','docstatus') optional_fields = ("_user_tags", "_comments", "_assign", "_liked_by", "_seen") -def copytables(srctype, src, srcfield, tartype, tar, tarfield, srcfields, tarfields=[]): - if not tarfields: - tarfields = srcfields - l = [] - data = src.get(srcfield) - for d in data: - newrow = tar.append(tarfield) - newrow.idx = d.idx - - for i in range(len(srcfields)): - newrow.set(tarfields[i], d.get(srcfields[i])) - - l.append(newrow) - return l - -def db_exists(dt, dn): - return frappe.db.exists(dt, dn) - def delete_fields(args_dict, delete=0): """ Delete a field. * Deletes record from `tabDocField` * If not single doctype: Drops column from table * If single, deletes record from `tabSingles` - args_dict = { dt: [field names] } """ import frappe.utils @@ -65,4 +74,4 @@ def delete_fields(args_dict, delete=0): query = "ALTER TABLE `tab%s` " % dt + \ ", ".join(["DROP COLUMN `%s`" % f for f in fields if f in existing_fields]) frappe.db.commit() - frappe.db.sql(query) + frappe.db.sql(query) \ No newline at end of file diff --git a/frappe/model/base_document.py b/frappe/model/base_document.py index fe2165974c..aa6e309915 100644 --- a/frappe/model/base_document.py +++ b/frappe/model/base_document.py @@ -3,18 +3,18 @@ from __future__ import unicode_literals from six import iteritems, string_types + +import frappe import datetime -import frappe, sys from frappe import _ -from frappe.utils import (cint, flt, now, cstr, strip_html, getdate, get_datetime, to_timedelta, - sanitize_html, sanitize_email, cast_fieldtype) from frappe.model import default_fields from frappe.model.naming import set_new_name from frappe.model.utils.link_count import notify_link_count from frappe.modules import load_doctype_module -from frappe.model import display_fieldtypes -from frappe.model.db_schema import type_map, varchar_len +from frappe.model import display_fieldtypes, data_fieldtypes from frappe.utils.password import get_decrypted_password, set_encrypted_password +from frappe.utils import (cint, flt, now, cstr, strip_html, getdate, get_datetime, to_timedelta, + sanitize_html, sanitize_email, cast_fieldtype) _classes = {} @@ -298,37 +298,36 @@ class BaseDocument(object): if not self.creation: self.creation = self.modified = now() - self.created_by = self.modifield_by = frappe.session.user + self.created_by = self.modified_by = frappe.session.user d = self.get_valid_dict(convert_dates_to_str=True) columns = list(d) try: - frappe.db.sql("""insert into `tab{doctype}` - ({columns}) values ({values})""".format( + frappe.db.sql("""INSERT INTO `tab{doctype}` ({columns}) + VALUES ({values})""".format( doctype = self.doctype, columns = ", ".join(["`"+c+"`" for c in columns]), values = ", ".join(["%s"] * len(columns)) ), list(d.values())) except Exception as e: - if e.args[0]==1062: - if "PRIMARY" in cstr(e.args[1]): - if self.meta.autoname=="hash": - # hash collision? try again - self.name = None - self.db_insert() - return + if frappe.db.is_primary_key_violation(e): + if self.meta.autoname=="hash": + # hash collision? try again + self.name = None + self.db_insert() + return - frappe.msgprint(_("Duplicate name {0} {1}").format(self.doctype, self.name)) - raise frappe.DuplicateEntryError(self.doctype, self.name, e) + frappe.msgprint(_("Duplicate name {0} {1}").format(self.doctype, self.name)) + raise frappe.DuplicateEntryError(self.doctype, self.name, e) + + elif frappe.db.is_unique_key_violation(e): + # unique constraint + self.show_unique_validation_message(e) - elif "Duplicate" in cstr(e.args[1]): - # unique constraint - self.show_unique_validation_message(e) - else: - raise else: raise + self.set("__islocal", False) def db_update(self): @@ -345,31 +344,33 @@ class BaseDocument(object): columns = list(d) try: - frappe.db.sql("""update `tab{doctype}` - set {values} where name=%s""".format( + frappe.db.sql("""UPDATE `tab{doctype}` + SET {values} WHERE `name`=%s""".format( doctype = self.doctype, values = ", ".join(["`"+c+"`=%s" for c in columns]) ), list(d.values()) + [name]) except Exception as e: - if e.args[0]==1062 and "Duplicate" in cstr(e.args[1]): + if frappe.db.is_unique_key_violation(e): self.show_unique_validation_message(e) else: raise def show_unique_validation_message(self, e): - type, value, traceback = sys.exc_info() - fieldname, label = str(e).split("'")[-2], None + # TODO: Find a better way to extract fieldname + if frappe.conf.db_type != 'postgres': + fieldname = str(e).split("'")[-2] + label = None - # unique_first_fieldname_second_fieldname is the constraint name - # created using frappe.db.add_unique - if "unique_" in fieldname: - fieldname = fieldname.split("_", 1)[1] + # unique_first_fieldname_second_fieldname is the constraint name + # created using frappe.db.add_unique + if "unique_" in fieldname: + fieldname = fieldname.split("_", 1)[1] - df = self.meta.get_field(fieldname) - if df: - label = df.label + df = self.meta.get_field(fieldname) + if df: + label = df.label - frappe.msgprint(_("{0} must be unique".format(label or fieldname))) + frappe.msgprint(_("{0} must be unique".format(label or fieldname))) # this is used to preserve traceback raise frappe.UniqueValidationError(self.doctype, self.name, e) @@ -549,8 +550,8 @@ class BaseDocument(object): for fieldname, value in iteritems(self.get_valid_dict()): df = self.meta.get_field(fieldname) - if df and df.fieldtype in type_map and type_map[df.fieldtype][0]=="varchar": - max_length = cint(df.get("length")) or cint(varchar_len) + if df and df.fieldtype in data_fieldtypes and frappe.db.type_map[df.fieldtype][0]=="varchar": + max_length = cint(df.get("length")) or cint(frappe.db.VARCHAR_LEN) if len(cstr(value)) > max_length: if self.parentfield and self.idx: diff --git a/frappe/model/create_new.py b/frappe/model/create_new.py index 639ecafe03..a84503de6b 100644 --- a/frappe/model/create_new.py +++ b/frappe/model/create_new.py @@ -6,11 +6,11 @@ from __future__ import unicode_literals Create a new document with defaults set """ -import frappe -from frappe.utils import nowdate, nowtime, now_datetime -import frappe.defaults -from frappe.model.db_schema import type_map import copy +import frappe +import frappe.defaults +from frappe.model import data_fieldtypes +from frappe.utils import nowdate, nowtime, now_datetime from frappe.core.doctype.user_permission.user_permission import get_user_permissions def get_new_doc(doctype, parent_doc = None, parentfield = None, as_dict=False): @@ -52,7 +52,7 @@ def set_user_and_static_default_values(doc): defaults = frappe.defaults.get_defaults() for df in doc.meta.get("fields"): - if df.fieldtype in type_map: + if df.fieldtype in data_fieldtypes: user_default_value = get_user_default_value(df, defaults, user_permissions) if user_default_value is not None: doc.set(df.fieldname, user_default_value) diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py index 512a32f45c..ee3437f085 100644 --- a/frappe/model/db_query.py +++ b/frappe/model/db_query.py @@ -7,15 +7,15 @@ from six import iteritems, string_types """build query for doclistview and return results""" -import frappe, json, copy, re import frappe.defaults import frappe.share -import frappe.permissions -from frappe.utils import flt, cint, getdate, get_datetime, get_time, make_filter_tuple, get_filter, add_to_date from frappe import _ +import frappe.permissions +from datetime import datetime +import frappe, json, copy, re from frappe.model import optional_fields from frappe.model.utils.user_settings import get_user_settings, update_user_settings -from datetime import datetime +from frappe.utils import flt, cint, get_time, make_filter_tuple, get_filter, add_to_date, cstr class DatabaseQuery(object): def __init__(self, doctype, user=None): @@ -104,6 +104,7 @@ class DatabaseQuery(object): if self.distinct: args.fields = 'distinct ' + args.fields + args.order_by = '' # TODO: recheck for alternative query = """select %(fields)s from %(tables)s %(conditions)s %(group_by)s %(order_by)s %(limit)s""" % args @@ -210,10 +211,10 @@ class DatabaseQuery(object): if any("{0}(".format(keyword) in field.lower() for keyword in blacklisted_functions): _raise_exception() - if re.compile("[a-zA-Z]+\s*'").match(field): + if re.compile(r"[a-zA-Z]+\s*'").match(field): _raise_exception() - if re.compile('[a-zA-Z]+\s*,').match(field): + if re.compile(r'[a-zA-Z]+\s*,').match(field): _raise_exception() def extract_tables(self): @@ -223,7 +224,7 @@ class DatabaseQuery(object): # add tables from fields if self.fields: for f in self.fields: - if ( not ("tab" in f and "." in f) ) or ("locate(" in f) or ("count(" in f): + if ( not ("tab" in f and "." in f) ) or ("locate(" in f) or ("strpos(" in f) or ("count(" in f): continue table_name = f.split('.')[0] @@ -244,7 +245,7 @@ class DatabaseQuery(object): raise frappe.PermissionError(doctype) def set_field_tables(self): - '''If there are more than one table, the fieldname must not be ambigous. + '''If there are more than one table, the fieldname must not be ambiguous. If the fieldname is not explicitly mentioned, set the default table''' if len(self.tables) > 1: for i, f in enumerate(self.fields): @@ -345,16 +346,23 @@ class DatabaseQuery(object): # Get descendants elements of a DocType with a tree structure if f.operator.lower() in ('descendants of', 'not descendants of') : - result = frappe.db.sql_list("""select name from `tab{0}` - where lft>%s and rgt<%s order by lft asc""".format(ref_doctype), (lft, rgt)) + result = frappe.get_all(ref_doctype, filters={ + 'lft': ['>', lft], + 'rgt': ['<', rgt] + }, order_by='`lft` ASC') else : # Get ancestor elements of a DocType with a tree structure - result = frappe.db.sql_list("""select name from `tab{0}` - where lft<%s and rgt>%s order by lft desc""".format(ref_doctype), (lft, rgt)) + result = frappe.get_all(ref_doctype, filters={ + 'lft': ['<', lft], + 'rgt': ['>', rgt] + }, order_by='`lft` DESC') fallback = "''" - value = (frappe.db.escape((v or '').strip(), percent=False) for v in result) - value = '("{0}")'.format('", "'.join(value)) + value = [frappe.db.escape((v.name or '').strip(), percent=False) for v in result] + if len(value): + value = "({0})".format(", ".join(value)) + else: + value = "('')" # changing operator to IN as the above code fetches all the parent / child values and convert into tuple # which can be directly used with IN operator to query. f.operator = 'not in' if f.operator.lower() in ('not ancestors of', 'not descendants of') else 'in' @@ -366,8 +374,11 @@ class DatabaseQuery(object): values = values.split(",") fallback = "''" - value = (frappe.db.escape((v or '').strip(), percent=False) for v in values) - value = '("{0}")'.format('", "'.join(value)) + value = [frappe.db.escape((v or '').strip(), percent=False) for v in values] + if len(value): + value = "({0})".format(", ".join(value)) + else: + value = "('')" else: df = frappe.get_meta(f.doctype).get("fields", {"fieldname": f.fieldname}) df = df[0] if df else None @@ -375,19 +386,23 @@ class DatabaseQuery(object): if df and df.fieldtype in ("Check", "Float", "Int", "Currency", "Percent"): can_be_null = False - if f.operator.lower() == 'between' and \ + if f.operator in ('>', '<') and (f.fieldname in ('creation', 'modified')): + value = cstr(f.value) + fallback = "NULL" + + elif f.operator.lower() in ('between') and \ (f.fieldname in ('creation', 'modified') or (df and (df.fieldtype=="Date" or df.fieldtype=="Datetime"))): value = get_between_date_filter(f.value, df) - fallback = "'0000-00-00 00:00:00'" + fallback = "'0001-01-01 00:00:00'" elif df and df.fieldtype=="Date": - value = getdate(f.value).strftime("%Y-%m-%d") - fallback = "'0000-00-00'" + value = frappe.db.format_date(f.value) + fallback = "'0001-01-01'" elif (df and df.fieldtype=="Datetime") or isinstance(f.value, datetime): - value = get_datetime(f.value).strftime("%Y-%m-%d %H:%M:%S.%f") - fallback = "'0000-00-00 00:00:00'" + value = frappe.db.format_datetime(f.value) + fallback = "'0001-01-01 00:00:00'" elif df and df.fieldtype=="Time": value = get_time(f.value).strftime("%H:%M:%S.%f") @@ -396,19 +411,23 @@ class DatabaseQuery(object): elif f.operator.lower() in ("like", "not like") or (isinstance(f.value, string_types) and (not df or df.fieldtype not in ["Float", "Int", "Currency", "Percent", "Check"])): value = "" if f.value==None else f.value - fallback = '""' + fallback = "''" if f.operator.lower() in ("like", "not like") and isinstance(value, string_types): # because "like" uses backslash (\) for escaping value = value.replace("\\", "\\\\").replace("%", "%%") + elif f.operator == '=' and df and df.fieldtype in ['Link', 'Data']: # TODO: Refactor if possible + value = f.value or "''" + fallback = "''" + else: value = flt(f.value) fallback = 0 - # put it inside double quotes + # escape value if isinstance(value, string_types) and not f.operator.lower() == 'between': - value = '"{0}"'.format(frappe.db.escape(value, percent=False)) + value = "{0}".format(frappe.db.escape(value, percent=False)) if (self.ignore_ifnull or not can_be_null @@ -449,10 +468,12 @@ class DatabaseQuery(object): self.conditions.append(self.get_share_condition()) else: - if role_permissions.get("if_owner", {}).get("read"): #if has if_owner permission skip user perm check - self.match_conditions.append("`tab{0}`.owner = '{1}'".format(self.doctype, + #if has if_owner permission skip user perm check + if role_permissions.get("if_owner", {}).get("read"): + self.match_conditions.append("`tab{0}`.`owner` = {1}".format(self.doctype, frappe.db.escape(self.user, percent=False))) - elif role_permissions.get("read"): # add user permission only if role has read perm + # add user permission only if role has read perm + elif role_permissions.get("read"): # get user permissions user_permissions = frappe.permissions.get_user_permissions(self.user) self.add_user_permissions(user_permissions) @@ -478,7 +499,7 @@ class DatabaseQuery(object): return self.match_filters def get_share_condition(self): - return """`tab{0}`.name in ({1})""".format(self.doctype, ", ".join(["'%s'"] * len(self.shared))) % \ + return """`tab{0}`.name in ({1})""".format(self.doctype, ", ".join(["%s"] * len(self.shared))) % \ tuple([frappe.db.escape(s, percent=False) for s in self.shared]) def add_user_permissions(self, user_permissions): @@ -498,7 +519,7 @@ class DatabaseQuery(object): user_permission_values = user_permissions.get(df.get('options'), {}) if df.get('ignore_user_permissions'): continue - empty_value_condition = 'ifnull(`tab{doctype}`.`{fieldname}`, "")=""'.format( + empty_value_condition = "ifnull(`tab{doctype}`.`{fieldname}`, '')=''".format( doctype=self.doctype, fieldname=df.get('fieldname') ) @@ -511,7 +532,7 @@ class DatabaseQuery(object): condition += """`tab{doctype}`.`{fieldname}` in ({values})""".format( doctype=self.doctype, fieldname=df.get('fieldname'), - values=", ".join([('"'+frappe.db.escape(v, percent=False)+'"') + values=", ".join([(frappe.db.escape(v, percent=False)) for v in user_permission_values.get("docs")])) match_conditions.append("({condition})".format(condition=condition)) @@ -536,7 +557,7 @@ class DatabaseQuery(object): def run_custom_query(self, query): if '%(key)s' in query: - query = query.replace('%(key)s', 'name') + query = query.replace('%(key)s', '`name`') return frappe.db.sql(query, as_dict = (not self.as_list)) def set_order_by(self, args): @@ -594,7 +615,7 @@ class DatabaseQuery(object): def add_limit(self): if self.limit_page_length: - return 'limit %s, %s' % (self.limit_start, self.limit_page_length) + return 'limit %s offset %s' % (self.limit_page_length, self.limit_start) else: return '' @@ -672,22 +693,23 @@ def get_between_date_filter(value, df=None): return the formattted date as per the given example [u'2017-11-01', u'2017-11-03'] => '2017-11-01 00:00:00.000000' AND '2017-11-04 00:00:00.000000' ''' - from_date = None - to_date = None - date_format = "%Y-%m-%d %H:%M:%S.%f" - - if df: - date_format = "%Y-%m-%d %H:%M:%S.%f" if df.fieldtype == 'Datetime' else "%Y-%m-%d" + from_date = frappe.utils.nowdate() + to_date = frappe.utils.nowdate() if value and isinstance(value, (list, tuple)): if len(value) >= 1: from_date = value[0] if len(value) >= 2: to_date = value[1] if not df or (df and df.fieldtype == 'Datetime'): - to_date = add_to_date(to_date,days=1) + to_date = add_to_date(to_date, days=1) - data = "'%s' AND '%s'" % ( - get_datetime(from_date).strftime(date_format), - get_datetime(to_date).strftime(date_format)) + if df and df.fieldtype == 'Datetime': + data = "'%s' AND '%s'" % ( + frappe.db.format_datetime(from_date), + frappe.db.format_datetime(to_date)) + else: + data = "'%s' AND '%s'" % ( + frappe.db.format_date(from_date), + frappe.db.format_date(to_date)) - return data + return data \ No newline at end of file diff --git a/frappe/model/db_schema.py b/frappe/model/db_schema.py deleted file mode 100644 index c5c603836e..0000000000 --- a/frappe/model/db_schema.py +++ /dev/null @@ -1,656 +0,0 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors -# MIT License. See license.txt - -from __future__ import unicode_literals -""" -Syncs a database table to the `DocType` (metadata) - -.. note:: This module is only used internally - -""" -import re -import os -import frappe -from frappe import _ -from frappe.utils import cstr, cint, flt - -# imports - third-party imports -import pymysql -from pymysql.constants import ER - -class InvalidColumnName(frappe.ValidationError): pass - -varchar_len = '140' -standard_varchar_columns = ('name', 'owner', 'modified_by', 'parent', 'parentfield', 'parenttype') - -type_map = { - 'Currency': ('decimal', '18,6'), - 'Int': ('int', '11'), - 'Long Int': ('bigint', '20'), # convert int to bigint if length is more than 11 - 'Float': ('decimal', '18,6'), - 'Percent': ('decimal', '18,6'), - 'Check': ('int', '1'), - 'Small Text': ('text', ''), - 'Long Text': ('longtext', ''), - 'Code': ('longtext', ''), - 'Text Editor': ('longtext', ''), - 'Date': ('date', ''), - 'Datetime': ('datetime', '6'), - 'Time': ('time', '6'), - 'Text': ('text', ''), - 'Data': ('varchar', varchar_len), - 'Link': ('varchar', varchar_len), - 'Dynamic Link': ('varchar', varchar_len), - 'Password': ('varchar', varchar_len), - 'Select': ('varchar', varchar_len), - 'Read Only': ('varchar', varchar_len), - 'Attach': ('text', ''), - 'Attach Image': ('text', ''), - 'Signature': ('longtext', ''), - 'Color': ('varchar', varchar_len), - 'Barcode': ('longtext', ''), - 'Geolocation': ('longtext', '') -} - -default_columns = ['name', 'creation', 'modified', 'modified_by', 'owner', - 'docstatus', 'parent', 'parentfield', 'parenttype', 'idx'] -optional_columns = ["_user_tags", "_comments", "_assign", "_liked_by"] - -default_shortcuts = ['_Login', '__user', '_Full Name', 'Today', '__today', "now", "Now"] - -def updatedb(dt, meta=None): - """ - Syncs a `DocType` to the table - * creates if required - * updates columns - * updates indices - """ - res = frappe.db.sql("select issingle from tabDocType where name=%s", (dt,)) - if not res: - raise Exception('Wrong doctype "%s" in updatedb' % dt) - - if not res[0][0]: - tab = DbTable(dt, 'tab', meta) - tab.validate() - - frappe.db.commit() - tab.sync() - frappe.db.begin() - -class DbTable: - def __init__(self, doctype, prefix = 'tab', meta = None): - self.doctype = doctype - self.name = prefix + doctype - self.columns = {} - self.current_columns = {} - - self.meta = meta - if not self.meta: - self.meta = frappe.get_meta(self.doctype) - - # lists for change - self.add_column = [] - self.change_type = [] - - self.add_index = [] - self.drop_index = [] - self.set_default = [] - - # load - self.get_columns_from_docfields() - - def validate(self): - """Check if change in varchar length isn't truncating the columns""" - if self.is_new(): - return - - self.get_columns_from_db() - - columns = [frappe._dict({"fieldname": f, "fieldtype": "Data"}) for f in standard_varchar_columns] - columns += self.columns.values() - - for col in columns: - if len(col.fieldname) >= 64: - frappe.throw(_("Fieldname is limited to 64 characters ({0})") - .format(frappe.bold(col.fieldname))) - - if col.fieldtype in type_map and type_map[col.fieldtype][0]=="varchar": - - # validate length range - new_length = cint(col.length) or cint(varchar_len) - if not (1 <= new_length <= 1000): - frappe.throw(_("Length of {0} should be between 1 and 1000").format(col.fieldname)) - - current_col = self.current_columns.get(col.fieldname, {}) - if not current_col: - continue - current_type = self.current_columns[col.fieldname]["type"] - current_length = re.findall('varchar\(([\d]+)\)', current_type) - if not current_length: - # case when the field is no longer a varchar - continue - current_length = current_length[0] - if cint(current_length) != cint(new_length): - try: - # check for truncation - max_length = frappe.db.sql("""select max(char_length(`{fieldname}`)) from `tab{doctype}`"""\ - .format(fieldname=col.fieldname, doctype=self.doctype)) - - except pymysql.InternalError as e: - if e.args[0] == ER.BAD_FIELD_ERROR: - # Unknown column 'column_name' in 'field list' - continue - - else: - raise - - if max_length and max_length[0][0] and max_length[0][0] > new_length: - if col.fieldname in self.columns: - self.columns[col.fieldname].length = current_length - - frappe.msgprint(_("Reverting length to {0} for '{1}' in '{2}'; Setting the length as {3} will cause truncation of data.")\ - .format(current_length, col.fieldname, self.doctype, new_length)) - - - def sync(self): - if self.is_new(): - self.create() - else: - self.alter() - - def is_new(self): - return self.name not in DbManager(frappe.db).get_tables_list(frappe.db.cur_db_name) - - def create(self): - add_text = '' - - # columns - column_defs = self.get_column_definitions() - if column_defs: add_text += ',\n'.join(column_defs) + ',\n' - - # index - index_defs = self.get_index_definitions() - if index_defs: add_text += ',\n'.join(index_defs) + ',\n' - - # create table - frappe.db.sql("""create table `%s` ( - name varchar({varchar_len}) not null primary key, - creation datetime(6), - modified datetime(6), - modified_by varchar({varchar_len}), - owner varchar({varchar_len}), - docstatus int(1) not null default '0', - parent varchar({varchar_len}), - parentfield varchar({varchar_len}), - parenttype varchar({varchar_len}), - idx int(8) not null default '0', - %sindex parent(parent), - index modified(modified)) - ENGINE={engine} - ROW_FORMAT=COMPRESSED - CHARACTER SET=utf8mb4 - COLLATE=utf8mb4_unicode_ci""".format(varchar_len=varchar_len, - engine=self.meta.get("engine") or 'InnoDB') % (self.name, add_text)) - - def get_column_definitions(self): - column_list = [] + default_columns - ret = [] - for k in list(self.columns): - if k not in column_list: - d = self.columns[k].get_definition() - if d: - ret.append('`'+ k + '` ' + d) - column_list.append(k) - return ret - - def get_index_definitions(self): - ret = [] - for key, col in self.columns.items(): - if col.set_index and not col.unique and col.fieldtype in type_map and \ - type_map.get(col.fieldtype)[0] not in ('text', 'longtext'): - ret.append('index `' + key + '`(`' + key + '`)') - return ret - - def get_columns_from_docfields(self): - """ - get columns from docfields and custom fields - """ - fl = frappe.db.sql("SELECT * FROM tabDocField WHERE parent = %s", self.doctype, as_dict = 1) - lengths = {} - precisions = {} - uniques = {} - - # optional fields like _comments - if not self.meta.istable: - for fieldname in optional_columns: - fl.append({ - "fieldname": fieldname, - "fieldtype": "Text" - }) - - # add _seen column if track_seen - if getattr(self.meta, 'track_seen', False): - fl.append({ - 'fieldname': '_seen', - 'fieldtype': 'Text' - }) - - if not frappe.flags.in_install_db and (frappe.flags.in_install != "frappe" or frappe.flags.ignore_in_install): - custom_fl = frappe.db.sql("""\ - SELECT * FROM `tabCustom Field` - WHERE dt = %s AND docstatus < 2""", (self.doctype,), as_dict=1) - if custom_fl: fl += custom_fl - - # apply length, precision and unique from property setters - for ps in frappe.get_all("Property Setter", fields=["field_name", "property", "value"], - filters={ - "doc_type": self.doctype, - "doctype_or_field": "DocField", - "property": ["in", ["precision", "length", "unique"]] - }): - - if ps.property=="length": - lengths[ps.field_name] = cint(ps.value) - - elif ps.property=="precision": - precisions[ps.field_name] = cint(ps.value) - - elif ps.property=="unique": - uniques[ps.field_name] = cint(ps.value) - - for f in fl: - self.columns[f['fieldname']] = DbColumn(self, f['fieldname'], - f['fieldtype'], lengths.get(f["fieldname"]) or f.get('length'), f.get('default'), f.get('search_index'), - f.get('options'), uniques.get(f["fieldname"], f.get('unique')), precisions.get(f['fieldname']) or f.get('precision')) - - def get_columns_from_db(self): - self.show_columns = frappe.db.sql("desc `%s`" % self.name) - for c in self.show_columns: - self.current_columns[c[0].lower()] = {'name': c[0], - 'type':c[1], 'index':c[3]=="MUL", 'default':c[4], "unique":c[3]=="UNI"} - - # GET foreign keys - def get_foreign_keys(self): - fk_list = [] - txt = frappe.db.sql("show create table `%s`" % self.name)[0][1] - for line in txt.split('\n'): - if line.strip().startswith('CONSTRAINT') and line.find('FOREIGN')!=-1: - try: - fk_list.append((line.split('`')[3], line.split('`')[1])) - except IndexError: - pass - - return fk_list - - # Drop foreign keys - def drop_foreign_keys(self): - if not self.drop_foreign_key: - return - - fk_list = self.get_foreign_keys() - - # make dictionary of constraint names - fk_dict = {} - for f in fk_list: - fk_dict[f[0]] = f[1] - - # drop - for col in self.drop_foreign_key: - frappe.db.sql("set foreign_key_checks=0") - frappe.db.sql("alter table `%s` drop foreign key `%s`" % (self.name, fk_dict[col.fieldname])) - frappe.db.sql("set foreign_key_checks=1") - - def alter(self): - for col in self.columns.values(): - col.build_for_alter_table(self.current_columns.get(col.fieldname.lower(), None)) - - query = [] - - for col in self.add_column: - query.append("add column `{}` {}".format(col.fieldname, col.get_definition())) - - for col in self.change_type: - current_def = self.current_columns.get(col.fieldname.lower(), None) - query.append("change `{}` `{}` {}".format(current_def["name"], col.fieldname, col.get_definition())) - - for col in self.add_index: - # if index key not exists - if not frappe.db.sql("show index from `%s` where key_name = %s" % - (self.name, '%s'), col.fieldname): - query.append("add index `{}`(`{}`)".format(col.fieldname, col.fieldname)) - - for col in self.drop_index: - if col.fieldname != 'name': # primary key - # if index key exists - if frappe.db.sql("""show index from `{0}` - where key_name=%s - and Non_unique=%s""".format(self.name), (col.fieldname, col.unique)): - query.append("drop index `{}`".format(col.fieldname)) - - for col in self.set_default: - if col.fieldname=="name": - continue - - if col.fieldtype in ("Check", "Int"): - col_default = cint(col.default) - - elif col.fieldtype in ("Currency", "Float", "Percent"): - col_default = flt(col.default) - - elif not col.default: - col_default = "null" - - else: - col_default = '"{}"'.format(col.default.replace('"', '\\"')) - - query.append('alter column `{}` set default {}'.format(col.fieldname, col_default)) - - if query: - try: - frappe.db.sql("alter table `{}` {}".format(self.name, ", ".join(query))) - except Exception as e: - # sanitize - if e.args[0]==1060: - frappe.throw(str(e)) - elif e.args[0]==1062: - fieldname = str(e).split("'")[-2] - frappe.throw(_("{0} field cannot be set as unique in {1}, as there are non-unique existing values".format(fieldname, self.name))) - else: - raise e - -class DbColumn: - def __init__(self, table, fieldname, fieldtype, length, default, - set_index, options, unique, precision): - self.table = table - self.fieldname = fieldname - self.fieldtype = fieldtype - self.length = length - self.set_index = set_index - self.default = default - self.options = options - self.unique = unique - self.precision = precision - - def get_definition(self, with_default=1): - column_def = get_definition(self.fieldtype, precision=self.precision, length=self.length) - - if not column_def: - return column_def - - if self.fieldtype in ("Check", "Int"): - default_value = cint(self.default) or 0 - column_def += ' not null default {0}'.format(default_value) - - elif self.fieldtype in ("Currency", "Float", "Percent"): - default_value = flt(self.default) or 0 - column_def += ' not null default {0}'.format(default_value) - - elif self.default and (self.default not in default_shortcuts) \ - and not self.default.startswith(":") and column_def not in ('text', 'longtext'): - column_def += ' default "' + self.default.replace('"', '\"') + '"' - - if self.unique and (column_def not in ('text', 'longtext')): - column_def += ' unique' - - return column_def - - def build_for_alter_table(self, current_def): - column_def = get_definition(self.fieldtype, self.precision, self.length) - - # no columns - if not column_def: - return - - # to add? - if not current_def: - self.fieldname = validate_column_name(self.fieldname) - self.table.add_column.append(self) - return - - # type - if (current_def['type'] != column_def) or\ - self.fieldname != current_def['name'] or\ - ((self.unique and not current_def['unique']) and column_def not in ('text', 'longtext')): - self.table.change_type.append(self) - - else: - # default - if (self.default_changed(current_def) \ - and (self.default not in default_shortcuts) \ - and not cstr(self.default).startswith(":") \ - and not (column_def in ['text','longtext'])): - self.table.set_default.append(self) - - # index should be applied or dropped irrespective of type change - if ( (current_def['index'] and not self.set_index and not self.unique) - or (current_def['unique'] and not self.unique) ): - # to drop unique you have to drop index - self.table.drop_index.append(self) - - elif (not current_def['index'] and self.set_index) and not (column_def in ('text', 'longtext')): - self.table.add_index.append(self) - - def default_changed(self, current_def): - if "decimal" in current_def['type']: - return self.default_changed_for_decimal(current_def) - else: - return current_def['default'] != self.default - - def default_changed_for_decimal(self, current_def): - try: - if current_def['default'] in ("", None) and self.default in ("", None): - # both none, empty - return False - - elif current_def['default'] in ("", None): - try: - # check if new default value is valid - float(self.default) - return True - except ValueError: - return False - - elif self.default in ("", None): - # new default value is empty - return True - - else: - # NOTE float() raise ValueError when "" or None is passed - return float(current_def['default'])!=float(self.default) - except TypeError: - return True - -class DbManager: - """ - Basically, a wrapper for oft-used mysql commands. like show tables,databases, variables etc... - - #TODO: - 0. Simplify / create settings for the restore database source folder - 0a. Merge restore database and extract_sql(from frappe_server_tools). - 1. Setter and getter for different mysql variables. - 2. Setter and getter for mysql variables at global level?? - """ - def __init__(self,db): - """ - Pass root_conn here for access to all databases. - """ - if db: - self.db = db - - def get_current_host(self): - return self.db.sql("select user()")[0][0].split('@')[1] - - def get_variables(self,regex): - """ - Get variables that match the passed pattern regex - """ - return list(self.db.sql("SHOW VARIABLES LIKE '%s'"%regex)) - - def get_table_schema(self,table): - """ - Just returns the output of Desc tables. - """ - return list(self.db.sql("DESC `%s`"%table)) - - - def get_tables_list(self,target=None): - """get list of tables""" - if target: - self.db.use(target) - - return [t[0] for t in self.db.sql("SHOW TABLES")] - - def create_user(self, user, password, host=None): - #Create user if it doesn't exist. - if not host: - host = self.get_current_host() - - if password: - self.db.sql("CREATE USER '%s'@'%s' IDENTIFIED BY '%s';" % (user[:16], host, password)) - else: - self.db.sql("CREATE USER '%s'@'%s';" % (user[:16], host)) - - def delete_user(self, target, host=None): - if not host: - host = self.get_current_host() - try: - self.db.sql("DROP USER '%s'@'%s';" % (target, host)) - except Exception as e: - if e.args[0]==1396: - pass - else: - raise - - def create_database(self,target): - if target in self.get_database_list(): - self.drop_database(target) - - self.db.sql("CREATE DATABASE `%s` ;" % target) - - def drop_database(self,target): - self.db.sql("DROP DATABASE IF EXISTS `%s`;"%target) - - def grant_all_privileges(self, target, user, host=None): - if not host: - host = self.get_current_host() - - self.db.sql("GRANT ALL PRIVILEGES ON `%s`.* TO '%s'@'%s';" % (target, - user, host)) - - def grant_select_privilges(self, db, table, user, host=None): - if not host: - host = self.get_current_host() - - if table: - self.db.sql("GRANT SELECT ON %s.%s to '%s'@'%s';" % (db, table, user, host)) - else: - self.db.sql("GRANT SELECT ON %s.* to '%s'@'%s';" % (db, user, host)) - - def flush_privileges(self): - self.db.sql("FLUSH PRIVILEGES") - - def get_database_list(self): - """get list of databases""" - return [d[0] for d in self.db.sql("SHOW DATABASES")] - - def restore_database(self,target,source,user,password): - from frappe.utils import make_esc - esc = make_esc('$ ') - - from distutils.spawn import find_executable - pipe = find_executable('pv') - if pipe: - pipe = '{pipe} {source} |'.format( - pipe = pipe, - source = source - ) - source = '' - else: - pipe = '' - source = '< {source}'.format(source = source) - - if pipe: - print('Creating Database...') - - command = '{pipe} mysql -u {user} -p{password} -h{host} {target} {source}'.format( - pipe = pipe, - user = esc(user), - password = esc(password), - host = esc(frappe.db.host), - target = esc(target), - source = source - ) - os.system(command) - - def drop_table(self,table_name): - """drop table if exists""" - if not table_name in self.get_tables_list(): - return - - self.db.sql("DROP TABLE IF EXISTS %s "%(table_name)) - -def validate_column_name(n): - special_characters = re.findall("[\W]", n, re.UNICODE) - if special_characters: - special_characters = ", ".join('"{0}"'.format(c) for c in special_characters) - frappe.throw(_("Fieldname {0} cannot have special characters like {1}").format(frappe.bold(cstr(n)), special_characters), InvalidColumnName) - return n - -def validate_column_length(fieldname): - """ In MySQL maximum column length is 64 characters, - ref: https://dev.mysql.com/doc/refman/5.5/en/identifiers.html""" - - if len(fieldname) > 64: - frappe.throw(_("Fieldname is limited to 64 characters ({0})").format(fieldname)) - -def remove_all_foreign_keys(): - frappe.db.sql("set foreign_key_checks = 0") - frappe.db.commit() - for t in frappe.db.sql("select name from tabDocType where issingle=0"): - dbtab = DbTable(t[0]) - try: - fklist = dbtab.get_foreign_keys() - except Exception as e: - if e.args[0]==1146: - fklist = [] - else: - raise - - for f in fklist: - frappe.db.sql("alter table `tab%s` drop foreign key `%s`" % (t[0], f[1])) - -def get_definition(fieldtype, precision=None, length=None): - d = type_map.get(fieldtype) - - # convert int to long int if the length of the int is greater than 11 - if fieldtype == "Int" and length and length>11: - d = type_map.get("Long Int") - - if not d: - return - - coltype = d[0] - size = None - if d[1]: - size = d[1] - - if size: - if fieldtype in ["Float", "Currency", "Percent"] and cint(precision) > 6: - size = '21,9' - - if coltype == "varchar" and length: - size = length - - if size is not None: - coltype = "{coltype}({size})".format(coltype=coltype, size=size) - - return coltype - -def add_column(doctype, column_name, fieldtype, precision=None): - if column_name in frappe.db.get_table_columns(doctype): - # already exists - return - - frappe.db.commit() - frappe.db.sql("alter table `tab%s` add column %s %s" % (doctype, - column_name, get_definition(fieldtype, precision))) diff --git a/frappe/model/delete_doc.py b/frappe/model/delete_doc.py index 6348f66b55..34c5da61ab 100644 --- a/frappe/model/delete_doc.py +++ b/frappe/model/delete_doc.py @@ -131,9 +131,9 @@ def update_naming_series(doc): def delete_from_table(doctype, name, ignore_doctypes, doc): if doctype!="DocType" and doctype==name: - frappe.db.sql("delete from `tabSingles` where doctype=%s", name) + frappe.db.sql("delete from `tabSingles` where `doctype`=%s", name) else: - frappe.db.sql("delete from `tab{0}` where name=%s".format(doctype), name) + frappe.db.sql("delete from `tab{0}` where `name`=%s".format(doctype), name) # get child tables if doc: @@ -233,8 +233,8 @@ def check_if_doc_is_dynamically_linked(doc, method="Delete"): raise_link_exists_exception(doc, df.parent, df.parent) else: # dynamic link in table - df["table"] = ", parent, parenttype, idx" if meta.istable else "" - for refdoc in frappe.db.sql("""select name, docstatus{table} from `tab{parent}` where + df["table"] = ", `parent`, `parenttype`, `idx`" if meta.istable else "" + for refdoc in frappe.db.sql("""select `name`, `docstatus` {table} from `tab{parent}` where {options}=%s and {fieldname}=%s""".format(**df), (doc.doctype, doc.name), as_dict=True): if ((method=="Delete" and refdoc.docstatus < 2) or (method=="Cancel" and refdoc.docstatus==1)): diff --git a/frappe/model/document.py b/frappe/model/document.py index c8a0fd831f..9442e87ead 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -407,10 +407,10 @@ class Document(BaseDocument): def update_single(self, d): """Updates values for Single type Document in `tabSingles`.""" - frappe.db.sql("""delete from tabSingles where doctype=%s""", self.doctype) + frappe.db.sql("""delete from `tabSingles` where doctype=%s""", self.doctype) for field, value in iteritems(d): if field != "doctype": - frappe.db.sql("""insert into tabSingles(doctype, field, value) + frappe.db.sql("""insert into `tabSingles` (doctype, field, value) values (%s, %s, %s)""", (self.doctype, field, value)) if self.doctype in frappe.db.value_cache: @@ -468,6 +468,7 @@ class Document(BaseDocument): def validate_workflow(self): '''Validate if the workflow transition is valid''' + if frappe.flags.in_install == 'frappe': return if self.meta.get_workflow(): validate_workflow(self) diff --git a/frappe/model/dynamic_links.py b/frappe/model/dynamic_links.py index c34dd914ef..fcd0b9234b 100644 --- a/frappe/model/dynamic_links.py +++ b/frappe/model/dynamic_links.py @@ -3,17 +3,17 @@ import frappe -# select doctypes that are accessed by the user (not read_only) first, so that the -# the validation message shows the user-facing doctype first. +# select doctypes that are accessed by the user (not read_only) first, so that the +# the validation message shows the user-facing doctype first. # For example Journal Entry should be validated before GL Entry (which is an internal doctype) dynamic_link_queries = [ - """select tabDocField.parent, + """select `tabDocField`.parent, `tabDocType`.read_only, `tabDocType`.in_create, - tabDocField.fieldname, tabDocField.options - from tabDocField, `tabDocType` + `tabDocField`.fieldname, `tabDocField`.options + from `tabDocField`, `tabDocType` where `tabDocField`.fieldtype='Dynamic Link' and - `tabDocType`.name=`tabDocField`.parent + `tabDocType`.`name`=`tabDocField`.parent order by `tabDocType`.read_only, `tabDocType`.in_create""", """select `tabCustom Field`.dt as parent, @@ -21,7 +21,7 @@ dynamic_link_queries = [ `tabCustom Field`.fieldname, `tabCustom Field`.options from `tabCustom Field`, `tabDocType` where `tabCustom Field`.fieldtype='Dynamic Link' and - `tabDocType`.name=`tabCustom Field`.dt + `tabDocType`.`name`=`tabCustom Field`.dt order by `tabDocType`.read_only, `tabDocType`.in_create""", ] diff --git a/frappe/model/meta.py b/frappe/model/meta.py index 7060e05124..6232452475 100644 --- a/frappe/model/meta.py +++ b/frappe/model/meta.py @@ -20,10 +20,9 @@ from datetime import datetime from six.moves import range import frappe, json, os from frappe.utils import cstr, cint -from frappe.model import default_fields, no_value_fields, optional_fields +from frappe.model import default_fields, no_value_fields, optional_fields, data_fieldtypes from frappe.model.document import Document from frappe.model.base_document import BaseDocument -from frappe.model.db_schema import type_map from frappe.modules import load_doctype_module from frappe.model.workflow import get_workflow_name from frappe import _ @@ -171,7 +170,7 @@ class Meta(Document): self._valid_columns = get_table_columns(self.name) else: self._valid_columns = self.default_fields + \ - [df.fieldname for df in self.get("fields") if df.fieldtype in type_map] + [df.fieldname for df in self.get("fields") if df.fieldtype in data_fieldtypes] return self._valid_columns @@ -255,7 +254,7 @@ class Meta(Document): def get_list_fields(self): list_fields = ["name"] + [d.fieldname \ - for d in self.fields if (d.in_list_view and d.fieldtype in type_map)] + for d in self.fields if (d.in_list_view and d.fieldtype in data_fieldtypes)] if self.title_field and self.title_field not in list_fields: list_fields.append(self.title_field) return list_fields @@ -292,7 +291,7 @@ class Meta(Document): WHERE dt = %s AND docstatus < 2""", (self.name,), as_dict=1, update={"is_custom_field": 1})) except Exception as e: - if e.args[0]==1146: + if frappe.db.is_table_missing(e): return else: raise @@ -452,10 +451,8 @@ def is_single(doctype): raise Exception('Cannot determine whether %s is single' % doctype) def get_parent_dt(dt): - parent_dt = frappe.db.sql("""select parent from tabDocField - where fieldtype="Table" and options=%s and (parent not like "old_parent:%%") - limit 1""", dt) - return parent_dt and parent_dt[0][0] or '' + parent_dt = frappe.db.get_all('DocField', 'parent', dict(fieldtype='Table', options=dt), limit=1) + return parent_dt and parent_dt[0].parent or '' def set_fieldname(field_id, fieldname): frappe.db.set_value('DocField', field_id, 'fieldname', fieldname) diff --git a/frappe/model/naming.py b/frappe/model/naming.py index 1708e8f98a..5e1771c1df 100644 --- a/frappe/model/naming.py +++ b/frappe/model/naming.py @@ -154,15 +154,15 @@ def parse_naming_series(parts, doctype='', doc=''): def getseries(key, digits, doctype=''): # series created ? - current = frappe.db.sql("select `current` from `tabSeries` where name=%s for update", (key,)) + current = frappe.db.sql("SELECT `current` FROM `tabSeries` WHERE `name`=%s FOR UPDATE", (key,)) if current and current[0][0] is not None: current = current[0][0] # yes, update it - frappe.db.sql("update tabSeries set current = current+1 where name=%s", (key,)) + frappe.db.sql("UPDATE `tabSeries` SET `current` = `current` + 1 WHERE `name`=%s", (key,)) current = cint(current) + 1 else: # no, create it - frappe.db.sql("insert into tabSeries (name, current) values (%s, 1)", (key,)) + frappe.db.sql("INSERT INTO `tabSeries` (`name`, `current`) VALUES (%s, 1)", (key,)) current = 1 return ('%0'+str(digits)+'d') % current @@ -179,10 +179,10 @@ def revert_series_if_last(key, name): prefix = key count = cint(name.replace(prefix, "")) - current = frappe.db.sql("select `current` from `tabSeries` where name=%s for update", (prefix,)) + current = frappe.db.sql("SELECT `current` FROM `tabSeries` WHERE `name`=%s FOR UPDATE", (prefix,)) if current and current[0][0]==count: - frappe.db.sql("update tabSeries set current=current-1 where name=%s", prefix) + frappe.db.sql("UPDATE `tabSeries` SET `current` = `current` - 1 WHERE `name`=%s", prefix) def get_default_naming_series(doctype): @@ -226,10 +226,14 @@ def append_number_if_name_exists(doctype, value, fieldname='name', separator='-' regex = '^{value}{separator}\d+$'.format(value=re.escape(value), separator=separator) if exists: - last = frappe.db.sql("""select {fieldname} from `tab{doctype}` - where {fieldname} regexp %s - order by length({fieldname}) desc, - {fieldname} desc limit 1""".format(doctype=doctype, fieldname=fieldname), regex) + last = frappe.db.sql("""SELECT `{fieldname}` FROM `tab{doctype}` + WHERE `{fieldname}` {regex_character} %s + ORDER BY length({fieldname}) DESC, + `{fieldname}` DESC LIMIT 1""".format( + doctype=doctype, + fieldname=fieldname, + regex_character=frappe.db.REGEX_CHARACTER), + regex) if last: count = str(cint(last[0][0].rsplit(separator, 1)[1]) + 1) diff --git a/frappe/model/rename_doc.py b/frappe/model/rename_doc.py index 1a724eea9e..7f2361fc3a 100644 --- a/frappe/model/rename_doc.py +++ b/frappe/model/rename_doc.py @@ -70,8 +70,8 @@ def rename_doc(doctype, old, new, force=False, merge=False, ignore_permissions=F rename_password(doctype, old, new) # update user_permissions - frappe.db.sql("""update tabDefaultValue set defvalue=%s where parenttype='User Permission' - and defkey=%s and defvalue=%s""", (new, doctype, old)) + frappe.db.sql("""UPDATE `tabDefaultValue` SET `defvalue`=%s WHERE `parenttype`='User Permission' + AND `defkey`=%s AND `defvalue`=%s""", (new, doctype, old)) if merge: new_doc.add_comment('Edit', _("merged {0} into {1}").format(frappe.bold(old), frappe.bold(new))) @@ -98,9 +98,10 @@ def update_user_settings(old, new, link_fields): # find the user settings for the linked doctypes linked_doctypes = set([d.parent for d in link_fields if not d.issingle]) - user_settings_details = frappe.db.sql('''select user, doctype, data from `__UserSettings` where - data like "%%%s%%" and doctype in ({0})'''.format(", ".join(["%s"]*len(linked_doctypes))), - tuple([old] + list(linked_doctypes)), as_dict=1) + user_settings_details = frappe.db.sql('''SELECT `user`, `doctype`, `data` + FROM `__UserSettings` + WHERE `data` like %s + AND `doctype` IN ('{doctypes}')'''.format(doctypes="', '".join(linked_doctypes)), (old), as_dict=1) # create the dict using the doctype name as key and values as list of the user settings from collections import defaultdict @@ -123,18 +124,17 @@ def update_attachments(doctype, old, new): if old != "File Data" and doctype != "DocType": frappe.db.sql("""update `tabFile` set attached_to_name=%s where attached_to_name=%s and attached_to_doctype=%s""", (new, old, doctype)) - except Exception as e: - if e.args[0]!=1054: # in patch? + except frappe.db.ProgrammingError as e: + if not frappe.db.is_column_missing(e): raise def rename_versions(doctype, old, new): - frappe.db.sql("""update tabVersion set docname=%s where ref_doctype=%s and docname=%s""", + frappe.db.sql("""UPDATE `tabVersion` SET `docname`=%s WHERE `ref_doctype`=%s AND `docname`=%s""", (new, doctype, old)) def rename_parent_and_child(doctype, old, new, meta): # rename the doc - frappe.db.sql("update `tab%s` set name=%s where name=%s" % (frappe.db.escape(doctype), '%s', '%s'), - (new, old)) + frappe.db.sql("UPDATE `tab{0}` SET `name`={1} WHERE `name`={1}".format(doctype, '%s'), (new, old)) update_autoname_field(doctype, new, meta) update_child_docs(old, new, meta) @@ -143,12 +143,11 @@ def update_autoname_field(doctype, new, meta): if meta.get('autoname'): field = meta.get('autoname').split(':') if field and field[0] == "field": - frappe.db.sql("update `tab%s` set %s=%s where name=%s" % (frappe.db.escape(doctype), field[1], '%s', '%s'), - (new, new)) + frappe.db.sql("UPDATE `tab{0}` SET `{1}`={2} WHERE `name`={2}".format(doctype, field[1], '%s'), (new, new)) def validate_rename(doctype, new, meta, merge, force, ignore_permissions): # using for update so that it gets locked and someone else cannot edit it while this rename is going on! - exists = frappe.db.sql("select name from `tab{doctype}` where name=%s for update".format(doctype=frappe.db.escape(doctype)), new) + exists = frappe.db.sql("select name from `tab{doctype}` where name=%s for update".format(doctype=doctype), new) exists = exists[0][0] if exists else None if merge and not exists: @@ -190,7 +189,7 @@ def update_child_docs(old, new, meta): # update "parent" for df in meta.get_table_fields(): frappe.db.sql("update `tab%s` set parent=%s where parent=%s" \ - % (frappe.db.escape(df.options), '%s', '%s'), (new, old)) + % (df.options, '%s', '%s'), (new, old)) def update_link_field_values(link_fields, old, new, doctype): for field in link_fields: @@ -211,12 +210,11 @@ def update_link_field_values(link_fields, old, new, doctype): # because the table hasn't been renamed yet! parent = field['parent'] if field['parent']!=new else old - frappe.db.sql("""\ - update `tab%s` set `%s`=%s - where `%s`=%s""" \ - % (frappe.db.escape(parent), frappe.db.escape(field['fieldname']), '%s', - frappe.db.escape(field['fieldname']), '%s'), - (new, old)) + frappe.db.sql(""" + update `tab{table_name}` set `{fieldname}`=%s + where `{fieldname}`=%s""".format( + table_name=parent, + fieldname=field['fieldname']), (new, old)) # update cached link_fields as per new if doctype=='DocType' and field['parent'] == old: field['parent'] = new @@ -301,32 +299,30 @@ def get_select_fields(old, new): new line separated list """ # get link fields from tabDocField - select_fields = frappe.db.sql("""\ + select_fields = frappe.db.sql(""" select parent, fieldname, (select issingle from tabDocType dt where dt.name = df.parent) as issingle from tabDocField df where df.parent != %s and df.fieldtype = 'Select' and - df.options like "%%%%%s%%%%" """ \ - % ('%s', frappe.db.escape(old)), (new,), as_dict=1) + df.options like {0} """.format(frappe.db.escape('%' + old + '%')), (new,), as_dict=1) # get link fields from tabCustom Field - custom_select_fields = frappe.db.sql("""\ + custom_select_fields = frappe.db.sql(""" select dt as parent, fieldname, (select issingle from tabDocType dt where dt.name = df.dt) as issingle from `tabCustom Field` df where df.dt != %s and df.fieldtype = 'Select' and - df.options like "%%%%%s%%%%" """ \ - % ('%s', frappe.db.escape(old)), (new,), as_dict=1) + df.options like {0} """ .format(frappe.db.escape('%' + old + '%')), (new,), as_dict=1) # add custom link fields list to link fields list select_fields += custom_select_fields # remove fields whose options have been changed using property setter - property_setter_select_fields = frappe.db.sql("""\ + property_setter_select_fields = frappe.db.sql(""" select ps.doc_type as parent, ps.field_name as fieldname, (select issingle from tabDocType dt where dt.name = ps.doc_type) as issingle @@ -335,35 +331,34 @@ def get_select_fields(old, new): ps.doc_type != %s and ps.property_type='options' and ps.field_name is not null and - ps.value like "%%%%%s%%%%" """ \ - % ('%s', frappe.db.escape(old)), (new,), as_dict=1) + ps.value like {0} """.format(frappe.db.escape('%' + old + '%')), (new,), as_dict=1) select_fields += property_setter_select_fields return select_fields def update_select_field_values(old, new): - frappe.db.sql("""\ + frappe.db.sql(""" update `tabDocField` set options=replace(options, %s, %s) where parent != %s and fieldtype = 'Select' and - (options like "%%%%\\n%s%%%%" or options like "%%%%%s\\n%%%%")""" % \ - ('%s', '%s', '%s', frappe.db.escape(old), frappe.db.escape(old)), (old, new, new)) + (options like {0} or options like {1})""" + .format(frappe.db.escape('%' + '\n' + old + '%'), frappe.db.escape('%' + old + '\n' + '%')), (old, new, new)) - frappe.db.sql("""\ + frappe.db.sql(""" update `tabCustom Field` set options=replace(options, %s, %s) where dt != %s and fieldtype = 'Select' and - (options like "%%%%\\n%s%%%%" or options like "%%%%%s\\n%%%%")""" % \ - ('%s', '%s', '%s', frappe.db.escape(old), frappe.db.escape(old)), (old, new, new)) + (options like {0} or options like {1})""" + .format(frappe.db.escape('%' + '\n' + old + '%'), frappe.db.escape('%' + old + '\n' + '%')), (old, new, new)) - frappe.db.sql("""\ + frappe.db.sql(""" update `tabProperty Setter` set value=replace(value, %s, %s) where doc_type != %s and field_name is not null and property='options' and - (value like "%%%%\\n%s%%%%" or value like "%%%%%s\\n%%%%")""" % \ - ('%s', '%s', '%s', frappe.db.escape(old), frappe.db.escape(old)), (old, new, new)) + (value like {0} or value like {1})""" + .format(frappe.db.escape('%' + '\n' + old + '%'), frappe.db.escape('%' + old + '\n' + '%')), (old, new, new)) def update_parenttype_values(old, new): child_doctypes = frappe.db.sql("""\ diff --git a/frappe/model/sync.py b/frappe/model/sync.py index 3759fa0040..e59944bb1b 100644 --- a/frappe/model/sync.py +++ b/frappe/model/sync.py @@ -29,10 +29,10 @@ def sync_for(app_name, force=0, sync_everything = False, verbose=False, reset_pe # these need to go first at time of install for d in (("core", "docfield"), ("core", "docperm"), + ("core", "role"), ("core", "has_role"), ("core", "doctype"), ("core", "user"), - ("core", "role"), ("custom", "custom_field"), ("custom", "property_setter"), ("website", "web_form"), diff --git a/frappe/model/utils/link_count.py b/frappe/model/utils/link_count.py index a53ee84d59..5faa5ba44b 100644 --- a/frappe/model/utils/link_count.py +++ b/frappe/model/utils/link_count.py @@ -45,7 +45,7 @@ def update_link_count(): frappe.db.sql('update `tab{0}` set idx = idx + {1} where name=%s'.format(key[0], count), key[1], auto_commit=1) except Exception as e: - if e.args[0]!=1146: # table not found, single + if not frappe.db.is_table_missing(e): # table not found, single raise e # reset the count frappe.cache().delete_value('_link_count') diff --git a/frappe/model/utils/user_settings.py b/frappe/model/utils/user_settings.py index 798c78b919..2cbfa50026 100644 --- a/frappe/model/utils/user_settings.py +++ b/frappe/model/utils/user_settings.py @@ -18,8 +18,8 @@ def get_user_settings(doctype, for_update=False): '{0}::{1}'.format(doctype, frappe.session.user)) if user_settings is None: - user_settings = frappe.db.sql('''select data from __UserSettings - where user=%s and doctype=%s''', (frappe.session.user, doctype)) + user_settings = frappe.db.sql('''select data from `__UserSettings` + where `user`=%s and `doctype`=%s''', (frappe.session.user, doctype)) user_settings = user_settings and user_settings[0][0] or '{}' if not for_update: @@ -49,8 +49,15 @@ def sync_user_settings(): for key, data in iteritems(frappe.cache().hgetall('_user_settings')): key = safe_decode(key) doctype, user = key.split('::') # WTF? - frappe.db.sql('''insert into __UserSettings (user, doctype, data) values (%s, %s, %s) - on duplicate key update data=%s''', (user, doctype, data, data)) + frappe.db.multisql({ + 'mariadb': """INSERT INTO `__UserSettings`(`user`, `doctype`, `data`) + VALUES (%s, %s, %s) + ON DUPLICATE key UPDATE `data`=%s""", + 'postgres': """INSERT INTO `__UserSettings` (`user`, `doctype`, `data`) + VALUES (%s, %s, %s) + ON CONFLICT ("user", "doctype") DO UPDATE SET `data`=%s""", + }, (user, doctype, data, data), as_dict=1) + @frappe.whitelist() def save(doctype, user_settings): diff --git a/frappe/modules/import_file.py b/frappe/modules/import_file.py index eadfcb3dd5..a1fa77d669 100644 --- a/frappe/modules/import_file.py +++ b/frappe/modules/import_file.py @@ -106,6 +106,7 @@ def import_doc(docdict, force=False, data_import=False, pre_process=None, ignore = [] if frappe.db.exists(doc.doctype, doc.name): + # import pdb; pdb.set_trace() old_doc = frappe.get_doc(doc.doctype, doc.name) if doc.doctype in ignore_values: diff --git a/frappe/modules/utils.py b/frappe/modules/utils.py index 94d726c16b..0703a064d3 100644 --- a/frappe/modules/utils.py +++ b/frappe/modules/utils.py @@ -132,8 +132,7 @@ def sync_customizations_for_doctype(data, folder): validate_fields_for_doctype(doctype) if update_schema and not frappe.db.get_value('DocType', doctype, 'issingle'): - from frappe.model.db_schema import updatedb - updatedb(doctype) + frappe.db.updatedb(doctype) def scrub(txt): return frappe.scrub(txt) diff --git a/frappe/patches/v11_0/update_list_user_settings.py b/frappe/patches/v11_0/update_list_user_settings.py index b505f2b444..d492ff1704 100644 --- a/frappe/patches/v11_0/update_list_user_settings.py +++ b/frappe/patches/v11_0/update_list_user_settings.py @@ -10,7 +10,7 @@ def execute(): for user in users: # get user_settings for each user settings = frappe.db.sql("select * from `__UserSettings` \ - where user='{0}'".format(frappe.db.escape(user.user)), as_dict=True) + where user={0}".format(frappe.db.escape(user.user)), as_dict=True) # traverse through each doctype's settings for a user for d in settings: diff --git a/frappe/patches/v4_0/rename_sitemap_to_route.py b/frappe/patches/v4_0/rename_sitemap_to_route.py index 2c8ed3c2d1..8ae5170b44 100644 --- a/frappe/patches/v4_0/rename_sitemap_to_route.py +++ b/frappe/patches/v4_0/rename_sitemap_to_route.py @@ -20,6 +20,6 @@ def execute(): def rename_field_if_exists(doctype, old_fieldname, new_fieldname): try: rename_field(doctype, old_fieldname, new_fieldname) - except Exception as e: - if e.args[0] != 1054: + except frappe.db.ProgrammingError as e: + if not frappe.db.is_column_missing(e): raise diff --git a/frappe/patches/v4_2/set_assign_in_doc.py b/frappe/patches/v4_2/set_assign_in_doc.py index a7de3c6dec..a6a06492a0 100644 --- a/frappe/patches/v4_2/set_assign_in_doc.py +++ b/frappe/patches/v4_2/set_assign_in_doc.py @@ -7,5 +7,5 @@ def execute(): try: frappe.get_doc("ToDo", name).on_update() except Exception as e: - if e.args[0]!=1146: + if not frappe.db.is_table_missing(e): raise diff --git a/frappe/patches/v5_0/convert_to_barracuda_and_utf8mb4.py b/frappe/patches/v5_0/convert_to_barracuda_and_utf8mb4.py index 989e4828b8..0e4a334a76 100644 --- a/frappe/patches/v5_0/convert_to_barracuda_and_utf8mb4.py +++ b/frappe/patches/v5_0/convert_to_barracuda_and_utf8mb4.py @@ -1,6 +1,6 @@ from __future__ import unicode_literals import frappe -from frappe.installer import check_if_ready_for_barracuda +from frappe.database.mariadb.setup_db import check_if_ready_for_barracuda from frappe.model.meta import trim_tables def execute(): diff --git a/frappe/patches/v5_0/fix_text_editor_file_urls.py b/frappe/patches/v5_0/fix_text_editor_file_urls.py index a4b92803da..d91aad0234 100644 --- a/frappe/patches/v5_0/fix_text_editor_file_urls.py +++ b/frappe/patches/v5_0/fix_text_editor_file_urls.py @@ -14,7 +14,7 @@ def execute(): try: result = frappe.get_all(opts.parent, fields=["name", opts.fieldname]) - except frappe.SQLError as e: + except frappe.db.SQLError: # bypass single tables continue diff --git a/frappe/patches/v5_0/rename_ref_type_fieldnames.py b/frappe/patches/v5_0/rename_ref_type_fieldnames.py index 6276b9b395..dd24f6e5b5 100644 --- a/frappe/patches/v5_0/rename_ref_type_fieldnames.py +++ b/frappe/patches/v5_0/rename_ref_type_fieldnames.py @@ -8,12 +8,12 @@ def execute(): try: frappe.db.sql("alter table `tabEmail Queue` change `ref_docname` `reference_name` varchar(255)") except Exception as e: - if e.args[0] not in (1054, 1060): + if not frappe.db.is_table_or_column_missing(e): raise try: frappe.db.sql("alter table `tabEmail Queue` change `ref_doctype` `reference_doctype` varchar(255)") except Exception as e: - if e.args[0] not in (1054, 1060): + if not frappe.db.is_table_or_column_missing(e): raise frappe.reload_doctype("Email Queue") diff --git a/frappe/patches/v6_16/star_to_like.py b/frappe/patches/v6_16/star_to_like.py index ee25f92c7f..e859223d54 100644 --- a/frappe/patches/v6_16/star_to_like.py +++ b/frappe/patches/v6_16/star_to_like.py @@ -1,6 +1,6 @@ from __future__ import unicode_literals import frappe -from frappe.model.db_schema import add_column +from frappe.database.schema import add_column def execute(): frappe.db.sql("""update `tabSingles` set field='_liked_by' where field='_starred_by'""") diff --git a/frappe/patches/v6_19/comment_feed_communication.py b/frappe/patches/v6_19/comment_feed_communication.py index 7a44f2e2ae..6f91ba04f9 100644 --- a/frappe/patches/v6_19/comment_feed_communication.py +++ b/frappe/patches/v6_19/comment_feed_communication.py @@ -260,8 +260,8 @@ def update_for_dynamically_linked_docs(timeline_doctype): try: docs = frappe.get_all(reference_doctype, fields=["name", df.fieldname], filters={ df.options: timeline_doctype }) - except frappe.SQLError as e: - if e.args and e.args[0]==1146: + except frappe.db.SQLError as e: + if frappe.db.is_table_missing(e): # single continue else: diff --git a/frappe/patches/v7_0/re_route.py b/frappe/patches/v7_0/re_route.py index 909838a692..46e4771855 100644 --- a/frappe/patches/v7_0/re_route.py +++ b/frappe/patches/v7_0/re_route.py @@ -19,4 +19,4 @@ def update_routes(doctypes): if(ifnull(parent_website_route, "")="", "", "/"), page_name) {1}""".format(d, condition)) except Exception as e: - if e.args[0]!=1054: raise + if not frappe.db.is_missing_column(e): raise diff --git a/frappe/patches/v7_0/update_auth.py b/frappe/patches/v7_0/update_auth.py index d97babfd89..3d47edf4b5 100644 --- a/frappe/patches/v7_0/update_auth.py +++ b/frappe/patches/v7_0/update_auth.py @@ -10,7 +10,7 @@ def execute(): # user passwords frappe.db.sql('''insert ignore into `__Auth` (doctype, name, fieldname, `password`) - (select 'User', user, 'password', `password` from `__OldAuth`)''') + (select 'User', `name`, 'password', `password` from `__OldAuth`)''') frappe.db.commit() diff --git a/frappe/patches/v7_2/set_in_standard_filter_property.py b/frappe/patches/v7_2/set_in_standard_filter_property.py index b14e0408a8..9f0de7ebf2 100644 --- a/frappe/patches/v7_2/set_in_standard_filter_property.py +++ b/frappe/patches/v7_2/set_in_standard_filter_property.py @@ -1,5 +1,4 @@ import frappe -import pymysql def execute(): frappe.reload_doc('custom', 'doctype', 'custom_field', force=True) @@ -7,12 +6,12 @@ def execute(): try: frappe.db.sql('update `tabCustom Field` set in_standard_filter = in_filter_dash') except Exception as e: - if e.args[0]!=1054: raise e + if not frappe.db.is_missing_column(e): raise e for doctype in frappe.get_all("DocType", {"istable": 0, "issingle": 0, "custom": 0}): try: frappe.reload_doctype(doctype.name, force=True) except KeyError: pass - except pymysql.err.DataError: + except frappe.db.DataError: pass diff --git a/frappe/patches/v8_0/rename_listsettings_to_usersettings.py b/frappe/patches/v8_0/rename_listsettings_to_usersettings.py index 5f5614a602..f17e925cc3 100644 --- a/frappe/patches/v8_0/rename_listsettings_to_usersettings.py +++ b/frappe/patches/v8_0/rename_listsettings_to_usersettings.py @@ -1,4 +1,3 @@ -from frappe.installer import create_user_settings_table from frappe.model.utils.user_settings import update_user_settings import frappe, json from six import iteritems @@ -11,7 +10,7 @@ def execute(): data = json.loads(us.data) except: continue - + if 'List' in data: continue @@ -31,7 +30,7 @@ def execute(): frappe.db.sql("RENAME TABLE __ListSettings to __UserSettings") else: if not frappe.db.table_exists("__UserSettings"): - create_user_settings_table() + frappe.db.create_user_settings_table() for user in frappe.db.get_all('User', {'user_type': 'System User'}): defaults = frappe.defaults.get_defaults_for(user.name) diff --git a/frappe/permissions.py b/frappe/permissions.py index 526ed4ae03..bc09bc65c7 100644 --- a/frappe/permissions.py +++ b/frappe/permissions.py @@ -359,8 +359,10 @@ def remove_user_permission(doctype, name, user): frappe.delete_doc('User Permission', user_permission_name) def clear_user_permissions_for_doctype(doctype, user=None): - user_permissions_for_doctype = frappe.db.get_list('User Permission', - dict(user=user, allow=doctype)) + filters = {'allow': doctype} + if user: + filters['user'] = user + user_permissions_for_doctype = frappe.db.get_list('User Permission', filters=filters) for d in user_permissions_for_doctype: frappe.delete_doc('User Permission', d.name) diff --git a/frappe/printing/doctype/print_format/test_records.json b/frappe/printing/doctype/print_format/test_records.json index b0fac3c7ab..c77238326b 100644 --- a/frappe/printing/doctype/print_format/test_records.json +++ b/frappe/printing/doctype/print_format/test_records.json @@ -2,7 +2,7 @@ { "doctype": "Print Format", "name": "_Test Print Format 1", - "module": "core", + "module": "Core", "doc_type": "User", "html": "" } diff --git a/frappe/sessions.py b/frappe/sessions.py index d4555ddfc6..ff9385862e 100644 --- a/frappe/sessions.py +++ b/frappe/sessions.py @@ -57,18 +57,22 @@ def get_sessions_to_clear(user=None, keep_current=False, device=None): if not device: device = frappe.session.data.device or "desktop" - limit = 0 + offset = 0 if user == frappe.session.user: simultaneous_sessions = frappe.db.get_value('User', user, 'simultaneous_sessions') or 1 - limit = simultaneous_sessions - 1 + offset = simultaneous_sessions - 1 condition = '' if keep_current: - condition = ' and sid != "{0}"'.format(frappe.db.escape(frappe.session.sid)) + condition = ' AND sid != {0}'.format(frappe.db.escape(frappe.session.sid)) - return frappe.db.sql_list("""select sid from tabSessions - where user=%s and device=%s {condition} - order by lastupdate desc limit {limit}, 100""".format(condition=condition, limit=limit), + return frappe.db.sql_list(""" + SELECT `sid` FROM `tabSessions` + WHERE user=%s + AND device=%s + {condition} + ORDER BY `lastupdate` DESC + LIMIT 100 OFFSET {offset}""".format(condition=condition, offset=offset), (user, device)) def delete_session(sid=None, user=None, reason="Session Expired"): @@ -95,9 +99,10 @@ def get_expired_sessions(): '''Returns list of expired sessions''' expired = [] for device in ("desktop", "mobile"): - expired += frappe.db.sql_list("""select sid from tabSessions - where TIMEDIFF(NOW(), lastupdate) > TIME(%s) - and device = %s""", (get_expiry_period(device), device)) + expired += frappe.db.sql_list("""SELECT `sid` + FROM `tabSessions` + WHERE (NOW() - `lastupdate`) > %s + AND device = %s""", (get_expiry_period(device), device)) return expired @@ -224,8 +229,8 @@ class Session: self.insert_session_record() # update user - frappe.db.sql("""UPDATE tabUser SET last_login = %(now)s, last_ip = %(ip)s, last_active = %(now)s - where name=%(name)s""", { + frappe.db.sql("""UPDATE `tabUser` SET `last_login` = %(now)s, `last_ip` = %(ip)s, `last_active` = %(now)s + where `name`=%(name)s""", { "now": frappe.utils.now(), "ip": frappe.local.request_ip, "name": self.data['user'] @@ -234,8 +239,8 @@ class Session: frappe.db.commit() def insert_session_record(self): - frappe.db.sql("""insert into tabSessions - (sessiondata, user, lastupdate, sid, status, device) + frappe.db.sql("""insert into `tabSessions` + (`sessiondata`, `user`, `lastupdate`, `sid`, `status`, `device`) values (%s , %s, NOW(), %s, 'Active', %s)""", (str(self.data['data']), self.data['user'], self.data['sid'], self.device)) @@ -300,13 +305,15 @@ class Session: return data and data.data def get_session_data_from_db(self): - self.device = frappe.db.sql('select device from tabSessions where sid=%s', self.sid) + self.device = frappe.db.sql('SELECT `device` FROM `tabSessions` WHERE `sid`=%s', self.sid) self.device = self.device and self.device[0][0] or 'desktop' - rec = frappe.db.sql("""select user, sessiondata - from tabSessions where sid=%s and - TIMEDIFF(NOW(), lastupdate) < TIME(%s)""", (self.sid, - get_expiry_period(self.device))) + rec = frappe.db.sql(""" + SELECT `user`, `sessiondata` + FROM `tabSessions` WHERE `sid`=%s AND + (NOW() - lastupdate) < %s + """, (self.sid, get_expiry_period(self.device))) + if rec: data = frappe._dict(eval(rec and rec[0][1] or '{}')) data.user = rec[0][0] @@ -348,7 +355,7 @@ class Session: updated_in_db = False if force or (time_diff==None) or (time_diff > 600): # update sessions table - frappe.db.sql("""update tabSessions set sessiondata=%s, + frappe.db.sql("""update `tabSessions` set sessiondata=%s, lastupdate=NOW() where sid=%s""" , (str(self.data['data']), self.data['sid'])) diff --git a/frappe/share.py b/frappe/share.py index 574e00d2b4..7819b8e9b4 100644 --- a/frappe/share.py +++ b/frappe/share.py @@ -89,7 +89,7 @@ def get_users(doctype, name): return frappe.db.sql("""select `name`, `user`, `read`, `write`, `share`, `everyone` from - tabDocShare + `tabDocShare` where share_doctype=%s and share_name=%s""", (doctype, name), as_dict=True) @@ -107,12 +107,18 @@ def get_shared(doctype, user=None, rights=None): if not rights: rights = ["read"] - condition = " and ".join(["`{0}`=1".format(right) for right in rights]) + filters = [[right, '=', 1] for right in rights] + filters += [['share_doctype', '=', doctype]] + or_filters = [['user', '=', user]] + if user != 'Guest': + or_filters += [['everyone', '=', 1]] - return frappe.db.sql_list("""select share_name from tabDocShare - where (user=%s {everyone}) and share_doctype=%s and {condition}""".format( - condition=condition, everyone="or everyone=1" if user!="Guest" else ""), - (user, doctype)) + shared_docs = frappe.db.get_all('DocShare', + fields=['share_name'], + filters=filters, + or_filters=or_filters) + + return [doc.share_name for doc in shared_docs] def get_shared_doctypes(user=None): """Return list of doctypes in which documents are shared for the given user.""" diff --git a/frappe/tests/test_api.py b/frappe/tests/test_api.py index d916a3063c..f5db63a031 100644 --- a/frappe/tests/test_api.py +++ b/frappe/tests/test_api.py @@ -16,7 +16,7 @@ class TestAPI(unittest.TestCase): return from frappe.frappeclient import FrappeClient - frappe.db.sql('delete from `tabToDo` where description like "Test API%"') + frappe.db.sql("DELETE FROM `tabToDo` WHERE `description` LIKE 'Test API%'") frappe.db.commit() server = FrappeClient(get_url(), "Administrator", "admin", verify=False) diff --git a/frappe/tests/test_bot.py b/frappe/tests/test_bot.py index e606bcccb0..b098584a8f 100644 --- a/frappe/tests/test_bot.py +++ b/frappe/tests/test_bot.py @@ -6,29 +6,5 @@ from __future__ import unicode_literals import unittest -from frappe.utils.bot import BotReply - class TestBot(unittest.TestCase): - def test_hello(self): - reply = BotReply().get_reply('hello') - self.assertEqual(reply, 'Hello Administrator') - - def test_open_notifications(self): - reply = BotReply().get_reply('whatsup') - self.assertTrue('your attention' in reply) - - def test_find(self): - reply = BotReply().get_reply('find user in doctypes') - self.assertTrue('[User](#Form/DocType/User)' in reply) - - def test_not_found(self): - reply = BotReply().get_reply('find yoyo in doctypes') - self.assertTrue('Could not find' in reply) - - def test_list(self): - reply = BotReply().get_reply('list users') - self.assertTrue('(#Form/User/test@example.com)' in reply) - - def test_how_many(self): - reply = BotReply().get_reply('how many users') - self.assertTrue(int(reply) > 1) + pass diff --git a/frappe/tests/test_db.py b/frappe/tests/test_db.py index ea6c07ae76..7b5da292ab 100644 --- a/frappe/tests/test_db.py +++ b/frappe/tests/test_db.py @@ -15,15 +15,11 @@ class TestDB(unittest.TestCase): self.assertEqual(frappe.db.get_value("User", {"name": ["<", "B"]}), "Administrator") self.assertEqual(frappe.db.get_value("User", {"name": ["<=", "Administrator"]}), "Administrator") - self.assertEqual(frappe.db.sql("""select name from `tabUser` where name > "s" order by modified desc""")[0][0], + self.assertEqual(frappe.db.sql("""SELECT name FROM `tabUser` WHERE name > 's' ORDER BY MODIFIED DESC""")[0][0], frappe.db.get_value("User", {"name": [">", "s"]})) - self.assertEqual(frappe.db.sql("""select name from `tabUser` where name >= "t" order by modified desc""")[0][0], + self.assertEqual(frappe.db.sql("""SELECT name FROM `tabUser` WHERE name >= 't' ORDER BY MODIFIED DESC""")[0][0], frappe.db.get_value("User", {"name": [">=", "t"]})) def test_escape(self): frappe.db.escape("香港濟生堂製藥有限公司 - IT".encode("utf-8")) - - # def test_multiple_queries(self): - # # implicit commit - # self.assertRaises(frappe.SQLError, frappe.db.sql, """select name from `tabUser`; truncate `tabEmail Queue`""") diff --git a/frappe/tests/test_db_query.py b/frappe/tests/test_db_query.py index 478adc0bd1..038266bccb 100644 --- a/frappe/tests/test_db_query.py +++ b/frappe/tests/test_db_query.py @@ -8,6 +8,8 @@ from frappe.model.db_query import DatabaseQuery from frappe.desk.reportview import get_filters_cond from frappe.permissions import add_user_permission, clear_user_permissions_for_doctype +test_dependencies = ["User"] + class TestReportview(unittest.TestCase): def test_basic(self): self.assertTrue({"name":"DocType"} in DatabaseQuery("DocType").execute(limit_page_length=None)) @@ -124,9 +126,9 @@ class TestReportview(unittest.TestCase): self.assertRaises(frappe.DataError, DatabaseQuery("DocType").execute, fields=["name", "issingle,'"],limit_start=0, limit_page_length=1) - data = DatabaseQuery("DocType").execute(fields=["name", "issingle", "count(name)"], + data = DatabaseQuery("DocType").execute(fields=["count(`name`) as count"], limit_start=0, limit_page_length=1) - self.assertTrue('count(name)' in data[0]) + self.assertTrue('count' in data[0]) data = DatabaseQuery("DocType").execute(fields=["name", "issingle", "locate('', name) as _relevance"], limit_start=0, limit_page_length=1) @@ -136,9 +138,11 @@ class TestReportview(unittest.TestCase): limit_start=0, limit_page_length=1) self.assertTrue('creation' in data[0]) - data = DatabaseQuery("DocType").execute(fields=["name", "issingle", - "datediff(modified, creation) as date_diff"], limit_start=0, limit_page_length=1) - self.assertTrue('date_diff' in data[0]) + if frappe.conf.db_type != 'postgres': + # datediff function does not exist in postgres + data = DatabaseQuery("DocType").execute(fields=["name", "issingle", + "datediff(modified, creation) as date_diff"], limit_start=0, limit_page_length=1) + self.assertTrue('date_diff' in data[0]) def test_nested_permission(self): clear_user_permissions_for_doctype("File") diff --git a/frappe/tests/test_document.py b/frappe/tests/test_document.py index 015b0d261e..387ffaef82 100644 --- a/frappe/tests/test_document.py +++ b/frappe/tests/test_document.py @@ -62,7 +62,8 @@ class TestDocument(unittest.TestCase): self.assertEqual(frappe.db.get_value(d.doctype, d.name, "subject"), "subject changed") def test_mandatory(self): - frappe.delete_doc_if_exists("User", "test_mandatory@example.com") + # TODO: recheck if it is OK to force delete + frappe.delete_doc_if_exists("User", "test_mandatory@example.com", 1) d = frappe.get_doc({ "doctype": "User", @@ -102,7 +103,7 @@ class TestDocument(unittest.TestCase): frappe.set_user("Administrator") def test_link_validation(self): - frappe.delete_doc_if_exists("User", "test_link_validation@example.com") + frappe.delete_doc_if_exists("User", "test_link_validation@example.com", 1) d = frappe.get_doc({ "doctype": "User", diff --git a/frappe/tests/test_email.py b/frappe/tests/test_email.py index b2b17abb05..345cee8ff9 100644 --- a/frappe/tests/test_email.py +++ b/frappe/tests/test_email.py @@ -109,13 +109,13 @@ class TestEmail(unittest.TestCase): content = part.get_payload(decode=True) if content: - frappe.local.flags.signed_query_string = re.search('(?<=/api/method/frappe.email.queue.unsubscribe\?).*(?=\n)', content.decode()).group(0) + frappe.local.flags.signed_query_string = re.search(r'(?<=/api/method/frappe.email.queue.unsubscribe\?).*(?=\n)', content.decode()).group(0) self.assertTrue(verify_request()) break def test_expired(self): self.test_email_queue() - frappe.db.sql("update `tabEmail Queue` set modified=DATE_SUB(curdate(), interval 8 day)") + frappe.db.sql("UPDATE `tabEmail Queue` SET `modified`=(NOW() - INTERVAL '8' day)") from frappe.email.queue import clear_outbox clear_outbox() email_queue = frappe.db.sql("""select name from `tabEmail Queue` where status='Expired'""", as_dict=1) diff --git a/frappe/tests/test_form_load.py b/frappe/tests/test_form_load.py index d07e61c600..ab0ce48ce8 100644 --- a/frappe/tests/test_form_load.py +++ b/frappe/tests/test_form_load.py @@ -25,7 +25,10 @@ class TestFormLoad(unittest.TestCase): user.add_roles('Blogger') reset('Blog Post') - frappe.db.sql('update tabDocField set permlevel=1 where fieldname="published" and parent="Blog Post"') + frappe.db.set_value('DocField', { + 'fieldname': 'published', + 'parent': 'Blog Post' + }, 'permlevel', 1) update('Blog Post', 'Website Manager', 0, 'permlevel', 1) @@ -48,7 +51,11 @@ class TestFormLoad(unittest.TestCase): self.assertTrue(checked, True) - frappe.db.sql('update tabDocField set permlevel=0 where fieldname="published" and parent="Blog Post"') + frappe.db.set_value('DocField', { + 'fieldname': 'published', + 'parent': 'Blog Post' + }, 'permlevel', 0) + reset('Blog Post') frappe.clear_cache(doctype='Blog Post') diff --git a/frappe/tests/test_global_search.py b/frappe/tests/test_global_search.py index 4cae44060b..87201f7530 100644 --- a/frappe/tests/test_global_search.py +++ b/frappe/tests/test_global_search.py @@ -22,15 +22,15 @@ class TestGlobalSearch(unittest.TestCase): make_property_setter(doctype, "repeat_on", "in_global_search", 0, "Int") def tearDown(self): - frappe.db.sql('delete from `tabProperty Setter` where doc_type="Event"') + frappe.db.sql("DELETE FROM `tabProperty Setter` WHERE `doc_type`='Event'") frappe.clear_cache(doctype='Event') - frappe.db.sql('delete from `tabEvent`') - frappe.db.sql('delete from __global_search') + frappe.db.sql('DELETE FROM `tabEvent`') + frappe.db.sql('DELETE FROM `__global_search`') make_test_objects('Event') frappe.db.commit() def insert_test_events(self): - frappe.db.sql('delete from tabEvent') + frappe.db.sql('DELETE FROM `tabEvent`') phrases = ['"The Sixth Extinction II: Amor Fati" is the second episode of the seventh season of the American science fiction.', 'After Mulder awakens from his coma, he realizes his duty to prevent alien colonization. ', 'Carter explored themes of extraterrestrial involvement in ancient mass extinctions in this episode, the third in a trilogy.'] diff --git a/frappe/tests/test_goal.py b/frappe/tests/test_goal.py index d17dc909c7..b4c0736482 100644 --- a/frappe/tests/test_goal.py +++ b/frappe/tests/test_goal.py @@ -20,15 +20,15 @@ class TestGoal(unittest.TestCase): def test_get_monthly_results(self): '''Test monthly aggregation values of a field''' - result_dict = get_monthly_results('Event', 'subject', 'creation', 'event_type="Private"', 'count') + result_dict = get_monthly_results('Event', 'subject', 'creation', "event_type='Private'", 'count') from frappe.utils import today, formatdate - self.assertEqual(result_dict[formatdate(today(), "MM-yyyy")], 2) + self.assertEqual(result_dict.get(formatdate(today(), "MM-yyyy")), 2) def test_get_monthly_goal_graph_data(self): '''Test for accurate values in graph data (based on test_get_monthly_results)''' docname = frappe.get_list('Event', filters = {"subject": ["=", "_Test Event 1"]})[0]["name"] frappe.db.set_value('Event', docname, 'description', 1) data = get_monthly_goal_graph_data('Test', 'Event', docname, 'description', 'description', 'description', - 'Event', '', 'description', 'creation', 'starts_on = "2014-01-01"', 'count') + 'Event', '', 'description', 'creation', "starts_on = '2014-01-01'", 'count') self.assertEqual(float(data['data']['datasets'][0]['values'][-1]), 1) diff --git a/frappe/tests/test_password.py b/frappe/tests/test_password.py index 8b429e4270..157fcc7b28 100644 --- a/frappe/tests/test_password.py +++ b/frappe/tests/test_password.py @@ -17,10 +17,11 @@ class TestPassword(unittest.TestCase): doc.password = new_password doc.save() - self.assertEqual(doc.password, '*'*len(new_password)) + self.assertEqual(doc.password, '*' * len(new_password)) - auth_password = frappe.db.sql('''select `password` from `__Auth` - where doctype=%(doctype)s and name=%(name)s and fieldname="password"''', doc.as_dict())[0][0] + password_list = get_password_list(doc) + + auth_password = password_list[0].get('password', '') # encrypted self.assertTrue(auth_password != new_password) @@ -52,8 +53,7 @@ class TestPassword(unittest.TestCase): update_password(user, new_password) - auth = frappe.db.sql('''select `password` from `__Auth` - where doctype='User' and name=%s and fieldname="password"''', user, as_dict=True)[0] + auth = get_password_list(dict(doctype='User', name=user))[0] # is not plain text self.assertTrue(auth.password != new_password) @@ -83,17 +83,21 @@ class TestPassword(unittest.TestCase): new_doc = frappe.get_doc(doc.doctype, new_name) self.assertEqual(new_doc.get_password(), password) - self.assertTrue(not frappe.db.sql('''select `password` from `__Auth` - where doctype=%s and name=%s and fieldname="password"''', (doc.doctype, doc.name))) + self.assertTrue(not get_password_list(doc)) frappe.rename_doc(doc.doctype, new_name, old_name) - self.assertTrue(frappe.db.sql('''select `password` from `__Auth` - where doctype=%s and name=%s and fieldname="password"''', (doc.doctype, doc.name))) + self.assertTrue(get_password_list(doc)) def test_password_on_delete(self): doc = self.make_email_account() doc.delete() - self.assertTrue(not frappe.db.sql('''select `password` from `__Auth` - where doctype=%s and name=%s and fieldname="password"''', (doc.doctype, doc.name))) + self.assertTrue(not get_password_list(doc)) + +def get_password_list(doc): + return frappe.db.sql("""SELECT `password` + FROM `__Auth` + WHERE `doctype`=%s + AND `name`=%s + AND `fieldname`='password' LIMIT 1""", (doc.get('doctype'), doc.get('name')), as_dict=1) diff --git a/frappe/tests/test_permissions.py b/frappe/tests/test_permissions.py index eb442b60fa..a187475e6c 100644 --- a/frappe/tests/test_permissions.py +++ b/frappe/tests/test_permissions.py @@ -296,7 +296,7 @@ class TestPermissions(unittest.TestCase): doctype""" frappe.set_user('Administrator') - frappe.db.sql('delete from tabContact') + frappe.db.sql('DELETE FROM `tabContact`') reset('Salutation') reset('Contact') @@ -306,8 +306,8 @@ class TestPermissions(unittest.TestCase): add_user_permission("Salutation", "Mr", "test3@example.com") self.set_strict_user_permissions(0) - allowed_contact = frappe.get_doc('Contact', '_Test Contact for _Test Customer') - other_contact = frappe.get_doc('Contact', '_Test Contact for _Test Supplier') + allowed_contact = frappe.get_doc('Contact', '_Test Contact For _Test Customer') + other_contact = frappe.get_doc('Contact', '_Test Contact For _Test Supplier') frappe.set_user("test3@example.com") self.assertTrue(allowed_contact.has_permission('read')) diff --git a/frappe/tests/test_scheduler.py b/frappe/tests/test_scheduler.py index 6353f27c99..b4a1692498 100644 --- a/frappe/tests/test_scheduler.py +++ b/frappe/tests/test_scheduler.py @@ -61,7 +61,7 @@ class TestScheduler(TestCase): def test_restrict_scheduler_events(self): frappe.set_user("Administrator") dormant_date = add_days(today(), -5) - frappe.db.sql('update tabUser set last_active=%s', dormant_date) + frappe.db.sql('UPDATE `tabUser` SET `last_active`=%s', dormant_date) restrict_scheduler_events_if_dormant() frappe.local.conf = _dict(frappe.get_site_config()) diff --git a/frappe/tests/test_twofactor.py b/frappe/tests/test_twofactor.py index b5eccec81d..4a06916edb 100644 --- a/frappe/tests/test_twofactor.py +++ b/frappe/tests/test_twofactor.py @@ -137,6 +137,7 @@ class TestTwoFactor(unittest.TestCase): #1 user = frappe.get_doc('User', self.user) user.restrict_ip = "192.168.255.254" #Dummy IP + user.bypass_restrict_ip_check_if_2fa_enabled = 0 user.save() enable_2fa(bypass_restrict_ip_check=0) with self.assertRaises(frappe.AuthenticationError): @@ -188,8 +189,8 @@ def toggle_2fa_all_role(state=None): all_role = frappe.get_doc('Role','All') if state == None: state = False if all_role.two_factor_auth == True else False - if state not in [True,False]:return - all_role.two_factor_auth = state + if state not in [True, False]: return + all_role.two_factor_auth = cint(state) all_role.save(ignore_permissions=True) frappe.db.commit() diff --git a/frappe/twofactor.py b/frappe/twofactor.py index 35d7a28fcb..03d23744dd 100644 --- a/frappe/twofactor.py +++ b/frappe/twofactor.py @@ -11,7 +11,7 @@ from jinja2 import Template from pyqrcode import create as qrcreate from six import BytesIO from base64 import b64encode, b32encode -from frappe.utils import get_url, get_datetime, time_diff_in_seconds +from frappe.utils import get_url, get_datetime, time_diff_in_seconds, cint from six import iteritems, string_types class ExpiredLoginException(Exception): pass @@ -20,7 +20,7 @@ def toggle_two_factor_auth(state, roles=[]): '''Enable or disable 2FA in site_config and roles''' for role in roles: role = frappe.get_doc('Role', {'role_name': role}) - role.two_factor_auth = state + role.two_factor_auth = cint(state) role.save(ignore_permissions=True) def two_factor_is_enabled(user=None): @@ -92,10 +92,14 @@ def two_factor_is_enabled_for_(user): user = frappe.get_doc('User', user) roles = [frappe.db.escape(d.role) for d in user.roles or []] - roles.append('All') + roles.append("'All'") + + query = """SELECT `name` + FROM `tabRole` + WHERE `two_factor_auth`= 1 + AND `name` IN ({0}) + LIMIT 1""".format(", ".join(roles)) - query = """select name from `tabRole` where two_factor_auth=1 - and name in ({0}) limit 1""".format(', '.join('\"{}\"'.format(i) for i in roles)) if len(frappe.db.sql(query)) > 0: return True diff --git a/frappe/utils/background_jobs.py b/frappe/utils/background_jobs.py index daaed29cc5..ab6357cc45 100755 --- a/frappe/utils/background_jobs.py +++ b/frappe/utils/background_jobs.py @@ -10,8 +10,6 @@ from frappe import _ from six import string_types # imports - third-party imports -import pymysql -from pymysql.constants import ER default_timeout = 300 queue_timeout = { @@ -102,11 +100,12 @@ def execute_job(site, method, event, job_name, kwargs, user=None, is_async=True, try: method(**kwargs) - except (pymysql.InternalError, frappe.RetryBackgroundJobError) as e: + except (frappe.db.InternalError, frappe.RetryBackgroundJobError) as e: frappe.db.rollback() if (retry < 5 and - (isinstance(e, frappe.RetryBackgroundJobError) or e.args[0] in (ER.LOCK_DEADLOCK, ER.LOCK_WAIT_TIMEOUT))): + (isinstance(e, frappe.RetryBackgroundJobError) or + (frappe.db.is_deadlocked() or frappe.db.is_timedout()))): # retry the job if # 1213 = deadlock # 1205 = lock wait timeout diff --git a/frappe/utils/change_log.py b/frappe/utils/change_log.py index b2b91d88b5..4ac1427b4e 100644 --- a/frappe/utils/change_log.py +++ b/frappe/utils/change_log.py @@ -11,6 +11,7 @@ import subprocess # nosec from frappe.utils import cstr from frappe.utils.gitutils import get_app_last_commit_ref, get_app_branch from frappe import _ +import subprocess # nosec def get_change_log(user=None): if not user: user = frappe.session.user @@ -163,6 +164,9 @@ def check_release_on_github(app): # Passing this since some apps may not have git initializaed in them return None + if isinstance(remote_url, bytes): + remote_url = remote_url.decode() + if "github.com" not in remote_url: return None diff --git a/frappe/utils/data.py b/frappe/utils/data.py index 1ac19052d3..044be6b6ba 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -35,8 +35,8 @@ def getdate(string_date=None): elif isinstance(string_date, datetime.date): return string_date - # dateutil parser does not agree with dates like 0000-00-00 - if not string_date or string_date=="0000-00-00": + # dateutil parser does not agree with dates like 0001-01-01 + if not string_date or string_date=="0001-01-01": return None return parser.parse(string_date).date() @@ -53,8 +53,8 @@ def get_datetime(datetime_str=None): elif isinstance(datetime_str, datetime.date): return datetime.datetime.combine(datetime_str, datetime.time()) - # dateutil parser does not agree with dates like 0000-00-00 - if not datetime_str or (datetime_str or "").startswith("0000-00-00"): + # dateutil parser does not agree with dates like 0001-01-01 + if not datetime_str or (datetime_str or "").startswith("0001-01-01"): return None try: diff --git a/frappe/utils/error.py b/frappe/utils/error.py index 9269d6b3cb..7193e7eca0 100644 --- a/frappe/utils/error.py +++ b/frappe/utils/error.py @@ -187,7 +187,7 @@ def collect_error_snapshots(): def clear_old_snapshots(): """Clear snapshots that are older than a month""" frappe.db.sql("""delete from `tabError Snapshot` - where creation < date_sub(now(), interval 1 month)""") + where creation < (NOW() - INTERVAL '1' MONTH)""") path = get_error_snapshot_path() today = datetime.datetime.now() diff --git a/frappe/utils/file_manager.py b/frappe/utils/file_manager.py index 3b11a5e3f4..3151d00a43 100644 --- a/frappe/utils/file_manager.py +++ b/frappe/utils/file_manager.py @@ -151,7 +151,7 @@ def save_file(fname, content, dt, dn, folder=None, decode=False, is_private=0, d "folder": folder, "file_size": file_size, "content_hash": content_hash, - "is_private": is_private + "is_private": cint(is_private) }) f = frappe.get_doc(file_data) @@ -165,7 +165,10 @@ def save_file(fname, content, dt, dn, folder=None, decode=False, is_private=0, d def get_file_data_from_hash(content_hash, is_private=0): - for name in frappe.db.sql_list("select name from `tabFile` where content_hash=%s and is_private=%s", (content_hash, is_private)): + for name in frappe.db.sql_list("""SELECT `name` + FROM `tabFile` + WHERE `content_hash`=%s + AND `is_private`=%s""", (content_hash, cint(is_private))): b = frappe.get_doc('File', name) return {k: b.get(k) for k in frappe.get_hooks()['write_file_keys']} return False @@ -223,7 +226,7 @@ def remove_all(dt, dn, from_delete=False): attached_to_doctype=%s and attached_to_name=%s""", (dt, dn)): remove_file(fid, dt, dn, from_delete) except Exception as e: - if e.args[0]!=1054: raise # (temp till for patched) + if not frappe.db.is_missing_column(e): raise # (temp till for patched) def remove_file_by_url(file_url, doctype=None, name=None): diff --git a/frappe/utils/global_search.py b/frappe/utils/global_search.py index 1c078a1691..8cb519480f 100644 --- a/frappe/utils/global_search.py +++ b/frappe/utils/global_search.py @@ -8,27 +8,14 @@ import re import redis from frappe.utils import cint, strip_html_tags from frappe.model.base_document import get_controller -from frappe.model.db_schema import varchar_len from six import text_type def setup_global_search_table(): """ - Creates __global_seach table + Creates __global_search table :return: """ - if not '__global_search' in frappe.db.get_tables(): - frappe.db.sql('''create table __global_search( - doctype varchar(100), - name varchar({varchar_len}), - title varchar({varchar_len}), - content text, - fulltext(content), - route varchar({varchar_len}), - published int(1) not null default 0, - unique `doctype_name` (doctype, name)) - COLLATE=utf8mb4_unicode_ci - ENGINE=MyISAM - CHARACTER SET=utf8mb4'''.format(varchar_len=varchar_len)) + frappe.db.create_global_search_table() def reset(): @@ -36,7 +23,7 @@ def reset(): Deletes all data in __global_search :return: """ - frappe.db.sql('delete from __global_search') + frappe.db.sql('DELETE FROM `__global_search`') def get_doctypes_with_global_search(with_child_tables=True): @@ -142,19 +129,17 @@ def rebuild_for_doctype(doctype): "name": frappe.db.escape(doc.name), "content": frappe.db.escape(' ||| '.join(content or '')), "published": published, - "title": frappe.db.escape(title or '')[:int(varchar_len)], - "route": frappe.db.escape(route or '')[:int(varchar_len)] + "title": frappe.db.escape(title or '')[:int(frappe.db.VARCHAR_LEN)], + "route": frappe.db.escape(route or '')[:int(frappe.db.VARCHAR_LEN)] }) if all_contents: insert_values_for_multiple_docs(all_contents) def delete_global_search_records_for_doctype(doctype): - frappe.db.sql(''' - delete - from __global_search - where - doctype = %s''', doctype, as_dict=True) + frappe.db.sql('''DELETE + FROM `__global_search` + WHERE doctype = %s''', doctype, as_dict=True) def get_selected_fields(meta, global_search_fields): @@ -210,19 +195,22 @@ def get_children_data(doctype, meta): def insert_values_for_multiple_docs(all_contents): values = [] for content in all_contents: - values.append("( '{doctype}', '{name}', '{content}', '{published}', '{title}', '{route}')" + values.append("({doctype}, {name}, {content}, {published}, {title}, {route})" .format(**content)) batch_size = 50000 for i in range(0, len(values), batch_size): batch_values = values[i:i + batch_size] # ignoring duplicate keys for doctype_name - frappe.db.sql(''' - insert ignore into __global_search + frappe.db.multisql({ + 'mariadb': '''INSERT IGNORE INTO `__global_search` (doctype, name, content, published, title, route) - values - {0} - '''.format(", ".join(batch_values))) + VALUES {0} '''.format(", ".join(batch_values)), + 'postgres': '''INSERT INTO `__global_search` + (doctype, name, content, published, title, route) + VALUES {0} + ON CONFLICT("name", "doctype") DO NOTHING'''.format(", ".join(batch_values)) + }) def update_global_search(doc): @@ -261,16 +249,16 @@ def update_global_search(doc): if hasattr(doc, 'is_website_published') and doc.meta.allow_guest_to_view: published = 1 if doc.is_website_published() else 0 - title = (doc.get_title() or '')[:int(varchar_len)] + title = (doc.get_title() or '')[:int(frappe.db.VARCHAR_LEN)] route = doc.get('route') if doc else '' frappe.flags.update_global_search.append( dict( - doctype=doc.doctype, - name=doc.name, + doctype=doc.doctype, + name=doc.name, content=' ||| '.join(content or ''), - published=published, - title=title, + published=published, + title=title, route=route ) ) @@ -322,13 +310,16 @@ def sync_global_search(flags=None): # Can pass flags manually as frappe.flags.update_global_search isn't reliable at a later time, # when syncing is enqueued for value in flags: - frappe.db.sql(''' - insert into __global_search - (doctype, name, content, published, title, route) - values - (%(doctype)s, %(name)s, %(content)s, %(published)s, %(title)s, %(route)s) - on duplicate key update - content = %(content)s''', value) + frappe.db.multisql({ + 'mariadb': '''INSERT INTO `__global_search` + (`doctype`, `name`, `content`, `published`, `title`, `route`) + VALUES (%(doctype)s, %(name)s, %(content)s, %(published)s, %(title)s, %(route)s) + ON DUPLICATE key UPDATE `content`=%(content)s''', + 'postgres': '''INSERT INTO `__global_search` + (`doctype`, `name`, `content`, `published`, `title`, `route`) + VALUES (%(doctype)s, %(name)s, %(content)s, %(published)s, %(title)s, %(route)s) + ON CONFLICT("doctype", "name") DO UPDATE SET `content`=%(content)s''' + }, value) frappe.flags.update_global_search = [] @@ -340,12 +331,10 @@ def delete_for_document(doc): :param doc: Deleted document """ - frappe.db.sql(''' - delete - from __global_search - where - doctype = %s and - name = %s''', (doc.doctype, doc.name), as_dict=True) + frappe.db.sql('''DELETE + FROM `__global_search` + WHERE doctype = %s + AND name = %s''', (doc.doctype, doc.name), as_dict=True) @frappe.whitelist() @@ -360,31 +349,29 @@ def search(text, start=0, limit=20, doctype=""): results = [] texts = text.split('&') for text in texts: - text = "+" + text + "*" - if not doctype: - result = frappe.db.sql(''' - select - doctype, name, content - from - __global_search - where - match(content) against (%s IN BOOLEAN MODE) - limit {start}, {limit}'''.format(start=start, limit=limit), text+"*", as_dict=True) - else: - result = frappe.db.sql(''' - select - doctype, name, content - from - __global_search - where - doctype = %s AND - match(content) against (%s IN BOOLEAN MODE) - limit {start}, {limit}'''.format(start=start, limit=limit), (doctype, text), as_dict=True) + mariadb_conditions = '' + postgres_conditions = '' + if doctype: + mariadb_conditions = postgres_conditions = '`doctype` = {} AND '.format(doctype) + + mariadb_conditions += 'MATCH(`content`) AGAINST ({} IN BOOLEAN MODE)'.format(frappe.db.escape('+' + text + '*')) + postgres_conditions += 'TO_TSVECTOR("content") @@ PLAINTO_TSQUERY({})'.format(frappe.db.escape(text)) + + common_query = '''SELECT `doctype`, `name`, `content` + FROM `__global_search` + WHERE {conditions} + LIMIT {limit} OFFSET {start}''' + + result = frappe.db.multisql({ + 'mariadb': common_query.format(conditions=mariadb_conditions, limit=limit, start=start), + 'postgres': common_query.format(conditions=postgres_conditions, limit=limit, start=start) + }, as_dict=True) + tmp_result=[] for i in result: if i in results or not results: tmp_result.append(i) - results = tmp_result + results += tmp_result for r in results: try: @@ -409,22 +396,24 @@ def web_search(text, start=0, limit=20): results = [] texts = text.split('&') for text in texts: - text = "+" + text + "*" - result = frappe.db.sql(''' - select - doctype, name, content, title, route - from - __global_search - where - published = 1 and - match(content) against (%s IN BOOLEAN MODE) - limit {start}, {limit}'''.format(start=start, limit=limit), - text, as_dict=True) + common_query = ''' SELECT `doctype`, `name`, `content`, `title`, `route` + FROM `__global_search` + WHERE {conditions} + LIMIT {limit} OFFSET {start}''' + mariadb_conditions = postgres_conditions = "`published` = 1 AND " + + mariadb_conditions += 'MATCH(`content`) AGAINST ({} IN BOOLEAN MODE)'.format(frappe.db.escape('+' + text + '*')) + postgres_conditions += 'TO_TSVECTOR("content") @@ PLAINTO_TSQUERY({})'.format(frappe.db.escape(text)) + + result = frappe.db.multisql({ + 'mariadb': common_query.format(conditions=mariadb_conditions, limit=limit, start=start), + 'postgres': common_query.format(conditions=postgres_conditions, limit=limit, start=start) + }, as_dict=True) tmp_result=[] for i in result: if i in results or not results: tmp_result.append(i) - results = tmp_result + results += tmp_result return results diff --git a/frappe/utils/goal.py b/frappe/utils/goal.py index 23c7abcdc0..90940ba304 100644 --- a/frappe/utils/goal.py +++ b/frappe/utils/goal.py @@ -8,17 +8,24 @@ from six.moves import xrange def get_monthly_results(goal_doctype, goal_field, date_col, filter_str, aggregation = 'sum'): '''Get monthly aggregation values for given field of doctype''' + # TODO: move to ORM? + if(frappe.conf.db_type == 'postgres'): + month_year_format_query = '''to_char("{}", 'MM-YYYY')'''.format(date_col) + else: + month_year_format_query = 'date_format(`{}`, "%m-%Y")'.format(date_col) - where_clause = ('where ' + filter_str) if filter_str else '' - results = frappe.db.sql(''' - select - {0}({1}) as {1}, date_format({2}, '%m-%Y') as month_year - from - `{3}` - {4} - group by - month_year'''.format(aggregation, goal_field, date_col, "tab" + - goal_doctype, where_clause), as_dict=True) + conditions = ('where ' + filter_str) if filter_str else '' + results = frappe.db.sql('''SELECT {aggregation}(`{goal_field}`) AS {goal_field}, + {month_year_format_query} AS month_year + FROM `{table_name}` {conditions} + GROUP BY month_year''' + .format( + aggregation=aggregation, + goal_field=goal_field, + month_year_format_query=month_year_format_query, + table_name="tab" + goal_doctype, + conditions=conditions + ), as_dict=True) month_to_value_dict = {} for d in results: @@ -69,7 +76,7 @@ def get_monthly_goal_graph_data(title, doctype, docname, goal_value_field, goal_ month_to_value_dict = None if month_to_value_dict is None: - doc_filter = (goal_doctype_link + ' = "' + docname + '"') if doctype != goal_doctype else '' + doc_filter = (goal_doctype_link + " = '" + docname + "'") if doctype != goal_doctype else '' if filter_str: doc_filter += ' and ' + filter_str if doc_filter else filter_str month_to_value_dict = get_monthly_results(goal_doctype, goal_field, date_field, doc_filter, aggregation) diff --git a/frappe/utils/help.py b/frappe/utils/help.py index cd3972e483..bc1f3dac30 100644 --- a/frappe/utils/help.py +++ b/frappe/utils/help.py @@ -3,17 +3,13 @@ from __future__ import unicode_literals, print_function +import io import frappe import hashlib -from frappe.model.db_schema import DbManager -from frappe.installer import get_root_connection -from frappe.database import Database import os, subprocess -from bs4 import BeautifulSoup import jinja2.exceptions - -import io +from bs4 import BeautifulSoup def sync(): # make table @@ -59,47 +55,24 @@ class HelpDatabase(object): self.global_help_setup = frappe.conf.get('global_help_setup') if self.global_help_setup: bench_name = os.path.basename(os.path.abspath(frappe.get_app_path('frappe')).split('/apps/')[0]) - self.help_db_name = hashlib.sha224(bench_name.encode('utf-8')).hexdigest()[:15] + self.help_db_name = 'd' + hashlib.sha224(bench_name.encode('utf-8')).hexdigest()[:15] def make_database(self): '''make database for global help setup''' if not self.global_help_setup: return + frappe.database.setup_help_database(self.help_db_name) - dbman = DbManager(get_root_connection()) - dbman.drop_database(self.help_db_name) - - # make database - if not self.help_db_name in dbman.get_database_list(): - try: - dbman.create_user(self.help_db_name, self.help_db_name) - except Exception as e: - # user already exists - if e.args[0] != 1396: raise - dbman.create_database(self.help_db_name) - dbman.grant_all_privileges(self.help_db_name, self.help_db_name) - dbman.flush_privileges() def connect(self): if self.global_help_setup: - self.db = Database(user=self.help_db_name, password=self.help_db_name) + self.db = frappe.database.get_db(user=self.help_db_name, password=self.help_db_name) else: self.db = frappe.db def make_table(self): if not 'help' in self.db.get_tables(): - self.db.sql('''create table help( - path varchar(255), - content text, - title text, - intro text, - full_path text, - fulltext(title), - fulltext(content), - index (path)) - COLLATE=utf8mb4_unicode_ci - ENGINE=MyISAM - CHARACTER SET=utf8mb4''') + self.db.create_help_table() def search(self, words): self.connect() @@ -137,8 +110,11 @@ class HelpDatabase(object): def get_content(self, path): self.connect() - query = '''select title, content from help - where path like "{path}%" order by path desc limit 1''' + query = '''SELECT `title`, `content` + FROM `help` + WHERE `path` LIKE '{path}%' + ORDER BY `path` DESC + LIMIT 1''' result = None if not path.endswith('index'): @@ -185,8 +161,8 @@ class HelpDatabase(object): title = self.make_title(basepath, fname, content) intro = self.make_intro(content) content = self.make_content(content, fpath, relpath, app) - self.db.sql('''insert into help(path, content, title, intro, full_path) - values (%s, %s, %s, %s, %s)''', (relpath, content, title, intro, fpath)) + self.db.sql('''INSERT INTO `help`(`path`, `content`, `title`, `intro`, `full_path`) + VALUES (%s, %s, %s, %s, %s)''', (relpath, content, title, intro, fpath)) except jinja2.exceptions.TemplateSyntaxError: print("Invalid Jinja Template for {0}. Skipping".format(fpath)) diff --git a/frappe/utils/install.py b/frappe/utils/install.py index b84e292397..e1c5c12310 100644 --- a/frappe/utils/install.py +++ b/frappe/utils/install.py @@ -107,7 +107,7 @@ def before_tests(): from frappe.desk.page.setup_wizard.setup_wizard import setup_complete if not int(frappe.db.get_single_value('System Settings', 'setup_complete') or 0): setup_complete({ - "language" :"english", + "language" :"English", "email" :"test@erpnext.com", "full_name" :"Test User", "password" :"test", diff --git a/frappe/utils/nestedset.py b/frappe/utils/nestedset.py index bc982e8904..e8ec26c64c 100644 --- a/frappe/utils/nestedset.py +++ b/frappe/utils/nestedset.py @@ -63,8 +63,10 @@ def update_add_node(doc, parent, parent_field): .format(doctype), parent)[0] validate_loop(doc.doctype, doc.name, left, right) else: # root - right = frappe.db.sql("select ifnull(max(rgt),0)+1 from `tab%s` \ - where ifnull(`%s`,'') =''" % (doctype, parent_field))[0][0] + right = frappe.db.sql(""" + SELECT COALESCE(MAX(rgt), 0) + 1 FROM `tab{0}` + WHERE COALESCE(`{1}`, '') = '' + """.format(doctype, parent_field))[0][0] right = right or 1 # update all on the right @@ -235,10 +237,14 @@ class NestedSet(Document): def validate_one_root(self): if not self.get(self.nsm_parent_field): - if frappe.db.sql("""select count(*) from `tab%s` where - ifnull(%s, '')=''""" % (self.doctype, self.nsm_parent_field))[0][0] > 1: + if self.get_root_node_count() > 1: frappe.throw(_("""Multiple root nodes not allowed."""), NestedSetMultipleRootsError) + def get_root_node_count(self): + return frappe.db.count(self.doctype, { + self.nsm_parent_field: '' + }) + def validate_ledger(self, group_identifier="is_group"): if hasattr(self, group_identifier) and not bool(self.get(group_identifier)): if frappe.db.sql("""select name from `tab{0}` where {1}=%s and docstatus!=2""" diff --git a/frappe/utils/password.py b/frappe/utils/password.py index 911db4adcf..da5cdecc55 100644 --- a/frappe/utils/password.py +++ b/frappe/utils/password.py @@ -11,7 +11,6 @@ from passlib.hash import pbkdf2_sha256, mysql41 from passlib.registry import register_crypt_handler from passlib.context import CryptContext - class LegacyPassword(pbkdf2_sha256): name = "frappe_legacy" ident = "$frappel$" @@ -50,16 +49,17 @@ def get_decrypted_password(doctype, name, fieldname='password', raise_exception= frappe.throw(_('Password not found'), frappe.AuthenticationError) def set_encrypted_password(doctype, name, pwd, fieldname='password'): - frappe.db.sql("""insert into __Auth (doctype, name, fieldname, `password`, encrypted) + frappe.db.sql("""insert into `__Auth` (doctype, name, fieldname, `password`, encrypted) values (%(doctype)s, %(name)s, %(fieldname)s, %(pwd)s, 1) - on duplicate key update `password`=%(pwd)s, encrypted=1""", - { 'doctype': doctype, 'name': name, 'fieldname': fieldname, 'pwd': encrypt(pwd) }) + {on_duplicate_update} `password`=%(pwd)s, encrypted=1""".format( + on_duplicate_update=frappe.db.get_on_duplicate_update(['doctype', 'name', 'fieldname']) + ), { 'doctype': doctype, 'name': name, 'fieldname': fieldname, 'pwd': encrypt(pwd) }) def check_password(user, pwd, doctype='User', fieldname='password'): '''Checks if user and password are correct, else raises frappe.AuthenticationError''' - auth = frappe.db.sql("""select name, `password` from `__Auth` - where doctype=%(doctype)s and name=%(name)s and fieldname=%(fieldname)s and encrypted=0""", + auth = frappe.db.sql("""select `name`, `password` from `__Auth` + where `doctype`=%(doctype)s and `name`=%(name)s and `fieldname`=%(fieldname)s and `encrypted`=0""", {'doctype': doctype, 'name': user, 'fieldname': fieldname}, as_dict=True) if not auth or not passlibctx.verify(pwd, auth[0].password): @@ -90,11 +90,17 @@ def update_password(user, pwd, doctype='User', fieldname='password', logout_all_ :param logout_all_session: delete all other session ''' hashPwd = passlibctx.hash(pwd) - frappe.db.sql("""insert into __Auth (doctype, name, fieldname, `password`, encrypted) - values (%(doctype)s, %(name)s, %(fieldname)s, %(pwd)s, 0) - on duplicate key update - `password`=%(pwd)s, encrypted=0""", - {'doctype': doctype, 'name': user, 'fieldname': fieldname, 'pwd': hashPwd}) + frappe.db.multisql({ + "mariadb": """INSERT INTO `__Auth` + (`doctype`, `name`, `fieldname`, `password`, `encrypted`) + VALUES (%(doctype)s, %(name)s, %(fieldname)s, %(pwd)s, 0) + ON DUPLICATE key UPDATE `password`=%(pwd)s, encrypted=0""", + "postgres": """INSERT INTO `__Auth` + (`doctype`, `name`, `fieldname`, `password`, `encrypted`) + VALUES (%(doctype)s, %(name)s, %(fieldname)s, %(pwd)s, 0) + ON CONFLICT("name", "doctype", "fieldname") DO UPDATE + SET `password`=%(pwd)s, encrypted=0""", + }, {'doctype': doctype, 'name': user, 'fieldname': fieldname, 'pwd': hashPwd}) # clear all the sessions except current if logout_all_sessions: @@ -103,15 +109,15 @@ def update_password(user, pwd, doctype='User', fieldname='password', logout_all_ def delete_all_passwords_for(doctype, name): try: - frappe.db.sql("""delete from __Auth where doctype=%(doctype)s and name=%(name)s""", + frappe.db.sql("""delete from `__Auth` where `doctype`=%(doctype)s and `name`=%(name)s""", { 'doctype': doctype, 'name': name }) except Exception as e: - if e.args[0]!=1054: + if not frappe.db.is_missing_column(e): raise def rename_password(doctype, old_name, new_name): # NOTE: fieldname is not considered, since the document is renamed - frappe.db.sql("""update __Auth set name=%(new_name)s + frappe.db.sql("""update `__Auth` set name=%(new_name)s where doctype=%(doctype)s and name=%(old_name)s""", { 'doctype': doctype, 'new_name': new_name, 'old_name': old_name }) @@ -122,14 +128,7 @@ def rename_password_field(doctype, old_fieldname, new_fieldname): def create_auth_table(): # same as Framework.sql - frappe.db.sql_ddl("""create table if not exists __Auth ( - `doctype` VARCHAR(140) NOT NULL, - `name` VARCHAR(255) NOT NULL, - `fieldname` VARCHAR(140) NOT NULL, - `password` VARCHAR(255) NOT NULL, - `encrypted` INT(1) NOT NULL DEFAULT 0, - PRIMARY KEY (`doctype`, `name`, `fieldname`) - ) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci""") + frappe.db.create_auth_table() def encrypt(pwd): if len(pwd) > 100: diff --git a/frappe/utils/scheduler.py b/frappe/utils/scheduler.py index 8e7a834512..62de439b74 100755 --- a/frappe/utils/scheduler.py +++ b/frappe/utils/scheduler.py @@ -26,25 +26,21 @@ from frappe.installer import update_site_config from six import string_types from croniter import croniter -# imports - third-party libraries -import pymysql -from pymysql.constants import ER - DATETIME_FORMAT = '%Y-%m-%d %H:%M:%S' cron_map = { - "yearly": "0 0 1 1 *", - "annual": "0 0 1 1 *", - "monthly": "0 0 1 * *", - "monthly_long": "0 0 1 * *", - "weekly": "0 0 * * 0", - "weekly_long": "0 0 * * 0", - "daily": "0 0 * * *", - "daily_long": "0 0 * * *", - "midnight": "0 0 * * *", - "hourly": "0 * * * *", - "hourly_long": "0 * * * *", - "all": "0/" + str((frappe.get_conf().scheduler_interval or 240) // 60) + " * * * *", + "yearly": "0 0 1 1 *", + "annual": "0 0 1 1 *", + "monthly": "0 0 1 * *", + "monthly_long": "0 0 1 * *", + "weekly": "0 0 * * 0", + "weekly_long": "0 0 * * 0", + "daily": "0 0 * * *", + "daily_long": "0 0 * * *", + "midnight": "0 0 * * *", + "hourly": "0 * * * *", + "hourly_long": "0 * * * *", + "all": "0/" + str((frappe.get_conf().scheduler_interval or 240) // 60) + " * * * *", } def start_scheduler(): @@ -96,8 +92,8 @@ def enqueue_events_for_site(site, queued_jobs): enqueue_events(site=site, queued_jobs=queued_jobs) frappe.logger(__name__).debug('Queued events for site {0}'.format(site)) - except pymysql.OperationalError as e: - if e.args[0]==ER.ACCESS_DENIED_ERROR: + except frappe.db.OperationalError as e: + if frappe.db.is_access_denied(e): frappe.logger(__name__).debug('Access denied for site {0}'.format(site)) else: log_and_raise() @@ -146,43 +142,43 @@ def enqueue_applicable_events(site, nowtime, last, queued_jobs=()): return out def trigger(site, event, last=None, queued_jobs=(), now=False): - """Trigger method in hooks.scheduler_events.""" + """Trigger method in hooks.scheduler_events.""" - queue = 'long' if event.endswith('_long') else 'short' - timeout = queue_timeout[queue] - if not queued_jobs and not now: - queued_jobs = get_jobs(site=site, queue=queue) + queue = 'long' if event.endswith('_long') else 'short' + timeout = queue_timeout[queue] + if not queued_jobs and not now: + queued_jobs = get_jobs(site=site, queue=queue) - if frappe.flags.in_test: - frappe.flags.ran_schedulers.append(event) + if frappe.flags.in_test: + frappe.flags.ran_schedulers.append(event) - events_from_hooks = get_scheduler_events(event) - if not events_from_hooks: - return + events_from_hooks = get_scheduler_events(event) + if not events_from_hooks: + return - events = events_from_hooks - if not now: - events = [] - if event == "cron": - for e in events_from_hooks: - e = cron_map.get(e, e) - if croniter.is_valid(e): - if croniter(e, last).get_next(datetime) <= frappe.utils.now_datetime(): - events.extend(events_from_hooks[e]) - else: - frappe.log_error("Cron string " + e + " is not valid", "Error triggering cron job") - frappe.logger(__name__).error('Exception in Trigger Events for Site {0}, Cron String {1}'.format(site, e)) + events = events_from_hooks + if not now: + events = [] + if event == "cron": + for e in events_from_hooks: + e = cron_map.get(e, e) + if croniter.is_valid(e): + if croniter(e, last).get_next(datetime) <= frappe.utils.now_datetime(): + events.extend(events_from_hooks[e]) + else: + frappe.log_error("Cron string " + e + " is not valid", "Error triggering cron job") + frappe.logger(__name__).error('Exception in Trigger Events for Site {0}, Cron String {1}'.format(site, e)) - else: - if croniter(cron_map[event], last).get_next(datetime) <= frappe.utils.now_datetime(): - events.extend(events_from_hooks) + else: + if croniter(cron_map[event], last).get_next(datetime) <= frappe.utils.now_datetime(): + events.extend(events_from_hooks) - for handler in events: - if not now: - if handler not in queued_jobs: - enqueue(handler, queue, timeout, event) - else: - scheduler_task(site=site, event=event, handler=handler, now=True) + for handler in events: + if not now: + if handler not in queued_jobs: + enqueue(handler, queue, timeout, event) + else: + scheduler_task(site=site, event=event, handler=handler, now=True) def get_scheduler_events(event): '''Get scheduler events from hooks and integrations''' @@ -295,8 +291,8 @@ def reset_enabled_scheduler_events(login_manager): if frappe.db.get_global('enabled_scheduler_events'): # clear restricted events, someone logged in! frappe.db.set_global('enabled_scheduler_events', None) - except pymysql.InternalError as e: - if e.args[0]==ER.LOCK_WAIT_TIMEOUT: + except frappe.db.InternalError as e: + if frappe.db.is_timedout(e): frappe.log_error(frappe.get_traceback(), "Error in reset_enabled_scheduler_events") else: raise @@ -332,7 +328,7 @@ def is_dormant(since = 345600): return False def get_last_active(): - return frappe.db.sql("""select max(last_active) from `tabUser` - where user_type = 'System User' and name not in ({standard_users})"""\ + return frappe.db.sql("""SELECT MAX(`last_active`) FROM `tabUser` + WHERE `user_type` = 'System User' AND `name` NOT IN ({standard_users})""" .format(standard_users=", ".join(["%s"]*len(STANDARD_USERS))), STANDARD_USERS)[0][0] diff --git a/frappe/utils/user.py b/frappe/utils/user.py index 12c5cfc0c1..1d3f30c391 100755 --- a/frappe/utils/user.py +++ b/frappe/utils/user.py @@ -46,7 +46,7 @@ class UserPermissions: pass except Exception as e: # install boo-boo - if e.args[0] != 1146: raise + if not frappe.db.is_table_missing(e): raise return user @@ -238,14 +238,21 @@ def get_system_managers(only_name=False): """returns all system manager's user details""" import email.utils from frappe.core.doctype.user.user import STANDARD_USERS - system_managers = frappe.db.sql("""select distinct name, - concat_ws(" ", if(first_name="", null, first_name), if(last_name="", null, last_name)) - as fullname from tabUser p - where docstatus < 2 and enabled = 1 - and name not in ({}) - and exists (select * from `tabHas Role` ur - where ur.parent = p.name and ur.role="System Manager") - order by creation desc""".format(", ".join(["%s"]*len(STANDARD_USERS))), + system_managers = frappe.db.sql("""SELECT DISTINCT `name`, `creation`, + CONCAT_WS(' ', + CASE WHEN `first_name`= '' THEN NULL ELSE `first_name` END, + CASE WHEN `last_name`= '' THEN NULL ELSE `last_name` END + ) AS fullname + FROM `tabUser` AS p + WHERE `docstatus` < 2 + AND `enabled` = 1 + AND `name` NOT IN ({}) + AND exists + (SELECT * + FROM `tabHas Role` AS ur + WHERE ur.parent = p.name + AND ur.role='System Manager') + ORDER BY `creation` DESC""".format(", ".join(["%s"]*len(STANDARD_USERS))), STANDARD_USERS, as_dict=True) if only_name: @@ -272,13 +279,25 @@ def add_system_manager(email, first_name=None, last_name=None, send_welcome_emai user.insert() # add roles - roles = frappe.db.sql_list("""select name from `tabRole` - where name not in ("Administrator", "Guest", "All")""") + roles = frappe.get_all('Role', + fields=['name'], + filters={ + 'name': ['not in', ('Administrator', 'Guest', 'All')] + } + ) + roles = [role.name for role in roles] user.add_roles(*roles) def get_enabled_system_users(): - return frappe.db.sql("""select * from tabUser where - user_type='System User' and enabled=1 and name not in ('Administrator', 'Guest')""", as_dict=1) + # add more fields if required + return frappe.get_all('User', + fields=['email', 'language', 'name'], + filters={ + 'user_type': 'System User', + 'enabled': 1, + 'name': ['not in', ('Administrator', 'Guest')] + } + ) def is_website_user(): return frappe.db.get_value('User', frappe.session.user, 'user_type') == "Website User" diff --git a/frappe/website/doctype/blog_post/blog_post.py b/frappe/website/doctype/blog_post/blog_post.py index a20e9fa128..c2d1bf26d6 100644 --- a/frappe/website/doctype/blog_post/blog_post.py +++ b/frappe/website/doctype/blog_post/blog_post.py @@ -37,9 +37,9 @@ class BlogPost(WebsiteGenerator): self.published_on = today() # update posts - frappe.db.sql("""update tabBlogger set posts=(select count(*) from `tabBlog Post` - where ifnull(blogger,'')=tabBlogger.name) - where name=%s""", (self.blogger,)) + frappe.db.sql("""UPDATE `tabBlogger` SET `posts`=(SELECT COUNT(*) FROM `tabBlog Post` + WHERE IFNULL(`blogger`,'')=`tabBlogger`.`name`) + WHERE `name`=%s""", (self.blogger,)) def on_update(self): clear_cache("writers") @@ -140,12 +140,12 @@ def get_blog_list(doctype, txt=None, filters=None, limit_start=0, limit_page_len conditions = [] if filters: if filters.blogger: - conditions.append('t1.blogger="%s"' % frappe.db.escape(filters.blogger)) + conditions.append('t1.blogger=%s' % frappe.db.escape(filters.blogger)) if filters.blog_category: - conditions.append('t1.blog_category="%s"' % frappe.db.escape(filters.blog_category)) + conditions.append('t1.blog_category=%s' % frappe.db.escape(filters.blog_category)) if txt: - conditions.append('(t1.content like "%{0}%" or t1.title like "%{0}%")'.format(frappe.db.escape(txt))) + conditions.append('(t1.content like {0} or t1.title like {0}")'.format(frappe.db.escape('%' + txt + '%'))) if conditions: frappe.local.no_cache = 1 diff --git a/frappe/website/doctype/website_theme/website_theme.py b/frappe/website/doctype/website_theme/website_theme.py index 04ea545c2a..f9c9ba412b 100644 --- a/frappe/website/doctype/website_theme/website_theme.py +++ b/frappe/website/doctype/website_theme/website_theme.py @@ -47,6 +47,7 @@ class WebsiteTheme(Document): def clear_cache_if_current_theme(self): + if frappe.flags.in_install == 'frappe': return website_settings = frappe.get_doc("Website Settings", "Website Settings") if getattr(website_settings, "website_theme", None) == self.name: website_settings.clear_cache() diff --git a/frappe/website/router.py b/frappe/website/router.py index e9ab0a5269..91f3f7d5c2 100644 --- a/frappe/website/router.py +++ b/frappe/website/router.py @@ -115,7 +115,7 @@ def get_page_info_from_doctypes(path=None): if path: return routes[r.route] except Exception as e: - if e.args[0]!=1054: raise e + if not frappe.db.is_missing_column(e): raise e return routes @@ -321,7 +321,7 @@ def sync_global_search(): frappe.session.user = 'Guest' frappe.local.no_cache = True - frappe.db.sql('delete from __global_search where doctype="Static Web Page"') + frappe.db.sql("DELETE FROM `__global_search` WHERE `doctype`='Static Web Page'") for app in frappe.get_installed_apps(frappe_last=True): app_path = frappe.get_app_path(app) diff --git a/frappe/website/utils.py b/frappe/website/utils.py index d330e97488..38bacac335 100644 --- a/frappe/website/utils.py +++ b/frappe/website/utils.py @@ -41,7 +41,7 @@ def get_comment_list(doctype, name): and reference_doctype=%s and reference_name=%s and (comment_type is null or comment_type in ('Comment', 'Communication')) - and modified >= DATE_SUB(NOW(),INTERVAL 1 YEAR) + and modified >= (NOW() - INTERVAL '1' YEAR) order by creation""", (doctype, name), as_dict=1) or [] def get_home_page(): diff --git a/frappe/workflow/doctype/workflow/workflow.py b/frappe/workflow/doctype/workflow/workflow.py index 64b12dad31..6a10448479 100644 --- a/frappe/workflow/doctype/workflow/workflow.py +++ b/frappe/workflow/doctype/workflow/workflow.py @@ -45,9 +45,14 @@ class Workflow(Document): states = self.get("states") for d in states: if not d.doc_status in docstatus_map: - frappe.db.sql("""update `tab%s` set `%s` = %s where \ - ifnull(`%s`, '')='' and docstatus=%s""" % (self.document_type, self.workflow_state_field, - '%s', self.workflow_state_field, "%s"), (d.state, d.doc_status)) + frappe.db.sql(""" + UPDATE `tab{doctype}` + SET `{field}` = %s + WHERE ifnull(`{field}`, '') = '' + AND `docstatus` = %s + """.format(doctype=self.document_type, field=self.workflow_state_field), + (d.state, d.doc_status)) + docstatus_map[d.doc_status] = d.state def validate_docstatus(self): @@ -74,8 +79,8 @@ class Workflow(Document): def set_active(self): if int(self.is_active or 0): # clear all other - frappe.db.sql("""update tabWorkflow set is_active=0 - where document_type=%s""", + frappe.db.sql("""UPDATE `tabWorkflow` SET `is_active`=0 + WHERE `document_type`=%s""", self.document_type) @frappe.whitelist() diff --git a/frappe/workflow/doctype/workflow_action/workflow_action.py b/frappe/workflow/doctype/workflow_action/workflow_action.py index 20b2f0f84c..5eb7a84b0d 100644 --- a/frappe/workflow/doctype/workflow_action/workflow_action.py +++ b/frappe/workflow/doctype/workflow_action/workflow_action.py @@ -22,7 +22,7 @@ def get_permission_query_conditions(user): if user == "Administrator": return "" - return "(`tabWorkflow Action`.user='{user}')".format(user=user) + return "(`tabWorkflow Action`.`user`='{user}')".format(user=user) def has_permission(doc, user): @@ -128,14 +128,14 @@ def return_link_expired_page(doc, doc_workflow_state): def clear_old_workflow_actions(doc, user=None): user = user if user else frappe.session.user - frappe.db.sql('''delete from `tabWorkflow Action` - where reference_doctype=%s and reference_name=%s and user!=%s and status="Open"''', + frappe.db.sql("""DELETE FROM `tabWorkflow Action` + WHERE `reference_doctype`=%s AND `reference_name`=%s AND `user`!=%s AND `status`='Open'""", (doc.get('doctype'), doc.get('name'), user)) def update_completed_workflow_actions(doc, user=None): user = user if user else frappe.session.user - frappe.db.sql('''update `tabWorkflow Action` set status='Completed', completed_by=%s - where reference_doctype=%s and reference_name=%s and user=%s and status="Open"''', + frappe.db.sql("""UPDATE `tabWorkflow Action` SET `status`='Completed', `completed_by`=%s + WHERE `reference_doctype`=%s AND `reference_name`=%s AND `user`=%s AND `status`='Open'""", (user, doc.get('doctype'), doc.get('name'), user)) def get_next_possible_transitions(workflow_name, state): @@ -219,12 +219,12 @@ def get_confirm_workflow_action_url(doc, action, user): def get_users_with_role(role): - return [p[0] for p in frappe.db.sql("""select distinct tabUser.name - from `tabHas Role`, tabUser - where `tabHas Role`.role=%s - and tabUser.name != "Administrator" - and `tabHas Role`.parent = tabUser.name - and tabUser.enabled=1""", role)] + return [p[0] for p in frappe.db.sql("""SELECT DISTINCT `tabUser`.`name` + FROM `tabHas Role`, `tabUser` + WHERE `tabHas Role`.`role`=%s + AND `tabUser`.`name`!='Administrator' + AND `tabHas Role`.`parent`=`tabUser`.`name` + AND `tabUser`.`enabled`=1""", role)] def is_workflow_action_already_created(doc): return frappe.db.exists({ diff --git a/requirements.txt b/requirements.txt index bd676a94c4..ce2887ffac 100644 --- a/requirements.txt +++ b/requirements.txt @@ -54,4 +54,5 @@ google-auth-httplib2 google-auth-oauthlib faker stripe +psycopg2-binary coverage diff --git a/test_sites/test_site/site_config.json b/test_sites/test_site/site_config.json index 62bb059a65..40a2d62b6d 100644 --- a/test_sites/test_site/site_config.json +++ b/test_sites/test_site/site_config.json @@ -6,6 +6,7 @@ "mail_login": "test@example.com", "mail_password": "test", "admin_password": "admin", + "root_password": "travis", "run_selenium_tests": 1, - "host_name": "http://localhost:8000" + "host_name": "http://test_site:8000" } diff --git a/test_sites/test_site_postgres/site_config.json b/test_sites/test_site_postgres/site_config.json new file mode 100644 index 0000000000..809468ff40 --- /dev/null +++ b/test_sites/test_site_postgres/site_config.json @@ -0,0 +1,13 @@ +{ + "db_name": "test_frappe", + "db_password": "test_frappe", + "db_type": "postgres", + "auto_email_id": "test@example.com", + "mail_server": "smtp.example.com", + "mail_login": "test@example.com", + "mail_password": "test", + "admin_password": "admin", + "root_password": "travis", + "run_selenium_tests": 1, + "host_name": "http://test_site_postgres:8000" +}