diff --git a/frappe/__init__.py b/frappe/__init__.py index 9e744c2c07..a68c32fe03 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -153,6 +153,7 @@ def init(site, sites_path=None, new_site=False): local.site = site local.sites_path = sites_path local.site_path = os.path.join(sites_path, site) + local.all_apps = None local.request_ip = None local.response = _dict({"docs":[]}) @@ -231,8 +232,7 @@ def get_site_config(sites_path=None, site_path=None): if os.path.exists(site_config): config.update(get_file_json(site_config)) elif local.site and not local.flags.new_site: - print("Site {0} does not exist".format(local.site)) - sys.exit(1) + raise IncorrectSitePath("{0} does not exist".format(local.site)) return _dict(config) @@ -300,7 +300,7 @@ def log(msg): debug_log.append(as_unicode(msg)) -def msgprint(msg, title=None, raise_exception=0, as_table=False, indicator=None, alert=False, primary_action=None, is_minimizable=None): +def msgprint(msg, title=None, raise_exception=0, as_table=False, indicator=None, alert=False, primary_action=None, is_minimizable=None, wide=None): """Print a message to the user (via HTTP response). Messages are sent in the `__server_messages` property in the response JSON and shown in a pop-up / modal. @@ -310,6 +310,8 @@ def msgprint(msg, title=None, raise_exception=0, as_table=False, indicator=None, :param raise_exception: [optional] Raise given exception and show message. :param as_table: [optional] If `msg` is a list of lists, render as HTML table. :param primary_action: [optional] Bind a primary server/client side action. + :param is_minimizable: [optional] Allow users to minimize the modal + :param wide: [optional] Show wide modal """ from frappe.utils import encode @@ -367,6 +369,9 @@ def msgprint(msg, title=None, raise_exception=0, as_table=False, indicator=None, if primary_action: out.primary_action = primary_action + if wide: + out.wide = wide + message_log.append(json.dumps(out)) if raise_exception and hasattr(raise_exception, '__name__'): @@ -388,12 +393,12 @@ def clear_last_message(): if len(local.message_log) > 0: local.message_log = local.message_log[:-1] -def throw(msg, exc=ValidationError, title=None, is_minimizable=None): +def throw(msg, exc=ValidationError, title=None, is_minimizable=None, wide=None): """Throw execption and show message (`msgprint`). :param msg: Message. :param exc: Exception class. Default `frappe.ValidationError`""" - msgprint(msg, raise_exception=exc, title=title, indicator='red', is_minimizable=is_minimizable) + msgprint(msg, raise_exception=exc, title=title, indicator='red', is_minimizable=is_minimizable, wide=wide) def emit_js(js, user=False, **kwargs): if user == False: @@ -436,12 +441,8 @@ def get_roles(username=None): """Returns roles of current user.""" if not local.session: return ["Guest"] - - if username: - import frappe.permissions - return frappe.permissions.get_roles(username) - else: - return get_user().get_roles() + import frappe.permissions + return frappe.permissions.get_roles(username or local.session.user) def get_request_header(key, default=None): """Return HTTP request header. @@ -921,10 +922,13 @@ def get_installed_apps(sort=False, frappe_last=False): if not db: connect() + if not local.all_apps: + local.all_apps = get_all_apps(True) + installed = json.loads(db.get_global("installed_apps") or "[]") if sort: - installed = [app for app in get_all_apps(True) if app in installed] + installed = [app for app in local.all_apps if app in installed] if frappe_last: if 'frappe' in installed: diff --git a/frappe/app.py b/frappe/app.py index 725bec183a..39bff83122 100644 --- a/frappe/app.py +++ b/frappe/app.py @@ -99,15 +99,16 @@ def application(request): frappe.monitor.stop(response) frappe.recorder.dump() - frappe.logger("frappe.web", allow_site=frappe.local.site).info({ - "site": get_site_name(request.host), - "remote_addr": getattr(request, "remote_addr", "NOTFOUND"), - "base_url": getattr(request, "base_url", "NOTFOUND"), - "full_path": getattr(request, "full_path", "NOTFOUND"), - "method": getattr(request, "method", "NOTFOUND"), - "scheme": getattr(request, "scheme", "NOTFOUND"), - "http_status_code": getattr(response, "status_code", "NOTFOUND") - }) + if hasattr(frappe.local, 'conf') and frappe.local.conf.enable_frappe_logger: + frappe.logger("frappe.web", allow_site=frappe.local.site).info({ + "site": get_site_name(request.host), + "remote_addr": getattr(request, "remote_addr", "NOTFOUND"), + "base_url": getattr(request, "base_url", "NOTFOUND"), + "full_path": getattr(request, "full_path", "NOTFOUND"), + "method": getattr(request, "method", "NOTFOUND"), + "scheme": getattr(request, "scheme", "NOTFOUND"), + "http_status_code": getattr(response, "status_code", "NOTFOUND") + }) if response and hasattr(frappe.local, 'rate_limiter'): response.headers.extend(frappe.local.rate_limiter.headers()) diff --git a/frappe/auth.py b/frappe/auth.py index 64fea36748..998e97fe24 100644 --- a/frappe/auth.py +++ b/frappe/auth.py @@ -338,7 +338,7 @@ class CookieManager: self.set_cookie("country", frappe.session.session_country) def set_cookie(self, key, value, expires=None, secure=False, httponly=False, samesite="Lax"): - if not secure: + if not secure and hasattr(frappe.local, 'request'): secure = frappe.local.request.scheme == "https" self.cookies[key] = { "value": value, diff --git a/frappe/core/doctype/data_import/data_import.js b/frappe/core/doctype/data_import/data_import.js index 6a922618cb..0e827a42d8 100644 --- a/frappe/core/doctype/data_import/data_import.js +++ b/frappe/core/doctype/data_import/data_import.js @@ -317,6 +317,7 @@ frappe.ui.form.on('Data Import', { }, show_import_warnings(frm, preview_data) { + let columns = preview_data.columns; let warnings = JSON.parse(frm.doc.template_warnings || '[]'); warnings = warnings.concat(preview_data.warnings || []); @@ -367,11 +368,13 @@ frappe.ui.form.on('Data Import', { .map(warning => { let header = ''; if (warning.col) { - header = __('Column {0}', [warning.col]); + let column_number = `${__('Column {0}', [warning.col])}`; + let column_header = columns[warning.col].header_title; + header = `${column_number} (${column_header})`; } return `
-
${header}
+
${header}
${warning.message}
`; diff --git a/frappe/core/doctype/data_import/data_import_list.js b/frappe/core/doctype/data_import/data_import_list.js index 1dee4319f9..0eb05aa354 100644 --- a/frappe/core/doctype/data_import/data_import_list.js +++ b/frappe/core/doctype/data_import/data_import_list.js @@ -17,6 +17,7 @@ frappe.listview_settings['Data Import'] = { get_indicator: function(doc) { var colors = { 'Pending': 'orange', + 'Not Started': 'orange', 'Partial Success': 'orange', 'Success': 'green', 'In Progress': 'orange', @@ -26,6 +27,9 @@ frappe.listview_settings['Data Import'] = { if (imports_in_progress.includes(doc.name)) { status = 'In Progress'; } + if (status == 'Pending') { + status = 'Not Started'; + } return [__(status), colors[status], 'status,=,' + doc.status]; }, formatters: { diff --git a/frappe/core/doctype/data_import/importer.py b/frappe/core/doctype/data_import/importer.py index 14626eb5e3..485f7caf08 100644 --- a/frappe/core/doctype/data_import/importer.py +++ b/frappe/core/doctype/data_import/importer.py @@ -7,7 +7,7 @@ import io import frappe import timeit import json -from datetime import datetime +from datetime import datetime, date from frappe import _ from frappe.utils import cint, flt, update_progress_bar, cstr from frappe.utils.csvutils import read_csv_content, get_csv_content_from_google_sheets @@ -233,7 +233,7 @@ class Importer: return updated_doc else: # throw if no changes - frappe.throw('No changes to update') + frappe.throw("No changes to update") def get_eta(self, current, total, processing_time): self.last_eta = getattr(self, "last_eta", 0) @@ -322,7 +322,7 @@ class ImportFile: if isinstance(file, frappe.string_types): if frappe.db.exists("File", {"file_url": file}): self.file_doc = frappe.get_doc("File", {"file_url": file}) - elif 'docs.google.com/spreadsheets' in file: + elif "docs.google.com/spreadsheets" in file: self.google_sheets_url = file elif os.path.exists(file): self.file_path = file @@ -348,7 +348,7 @@ class ImportFile: elif self.google_sheets_url: content = get_csv_content_from_google_sheets(self.google_sheets_url) - extension = 'csv' + extension = "csv" if not content: frappe.throw(_("Invalid or corrupted content for import")) @@ -602,12 +602,20 @@ class Row: is_table = frappe.get_meta(doctype).istable is_update = self.import_type == UPDATE - if is_table and is_update and doc.get("name") in INVALID_VALUES: - # for table rows being inserted in update - # create a new doc with defaults set - new_doc = frappe.new_doc(doctype, as_dict=True) - new_doc.update(doc) - doc = new_doc + if is_table and is_update: + # check if the row already exists + # if yes, fetch the original doc so that it is not updated + # if no, create a new doc + id_field = get_id_field(doctype) + id_value = doc.get(id_field.fieldname) + if id_value and frappe.db.exists(doctype, id_value): + doc = frappe.get_doc(doctype, id_value) + else: + # for table rows being inserted in update + # create a new doc with defaults set + new_doc = frappe.new_doc(doctype, as_dict=True) + new_doc.update(doc) + doc = new_doc self.check_mandatory_fields(doctype, doc, table_df) return doc @@ -615,16 +623,12 @@ class Row: def validate_value(self, value, col): df = col.df if df.fieldtype == "Select": - select_options = [d for d in (df.options or '').split('\n') if d] + select_options = get_select_options(df) if select_options and value not in select_options: options_string = ", ".join([frappe.bold(d) for d in select_options]) msg = _("Value must be one of {0}").format(options_string) self.warnings.append( - { - "row": self.row_number, - "field": df_as_json(df), - "message": msg, - } + {"row": self.row_number, "field": df_as_json(df), "message": msg,} ) return @@ -635,11 +639,7 @@ class Row: frappe.bold(value), frappe.bold(df.options) ) self.warnings.append( - { - "row": self.row_number, - "field": df_as_json(df), - "message": msg, - } + {"row": self.row_number, "field": df_as_json(df), "message": msg,} ) return elif df.fieldtype in ["Date", "Datetime"]: @@ -668,7 +668,7 @@ class Row: def parse_value(self, value, col): df = col.df - if isinstance(value, datetime) and df.fieldtype in ["Date", "Datetime"]: + if isinstance(value, (datetime, date)) and df.fieldtype in ["Date", "Datetime"]: return value value = cstr(value) @@ -689,7 +689,7 @@ class Row: return value def get_date(self, value, column): - if isinstance(value, datetime): + if isinstance(value, (datetime, date)): return value date_format = column.date_format @@ -786,9 +786,7 @@ class Header(Row): for j, header in enumerate(row): column_values = [get_item_at_index(r, j) for r in raw_data] map_to_field = column_to_field_map.get(str(j)) - column = Column( - j, header, self.doctype, column_values, map_to_field, self.seen - ) + column = Column(j, header, self.doctype, column_values, map_to_field, self.seen) self.seen.append(header) self.columns.append(column) @@ -918,13 +916,20 @@ class Column: self.skip_import = skip_import def guess_date_format_for_column(self): - """ Guesses date format for a column by parsing all the values in the column, + """Guesses date format for a column by parsing all the values in the column, getting the date format and then returning the one which has the maximum frequency """ - date_formats = [ - frappe.utils.guess_date_format(d) for d in self.column_values if isinstance(d, str) - ] + def guess_date_format(d): + if isinstance(d, (datetime, date)): + if self.df.fieldtype == "Date": + return "%Y-%m-%d" + if self.df.fieldtype == "Datetime": + return "%Y-%m-%d %H:%M:%S" + if isinstance(d, str): + return frappe.utils.guess_date_format(d) + + date_formats = [guess_date_format(d) for d in self.column_values] date_formats = [d for d in date_formats if d] if not date_formats: return @@ -958,28 +963,58 @@ class Column: if self.skip_import: return - if self.df.fieldtype == 'Link': + if self.df.fieldtype == "Link": # find all values that dont exist values = list(set([cstr(v) for v in self.column_values[1:] if v])) - exists = [d.name for d in frappe.db.get_all(self.df.options, filters={'name': ('in', values)})] + exists = [ + d.name for d in frappe.db.get_all(self.df.options, filters={"name": ("in", values)}) + ] not_exists = list(set(values) - set(exists)) if not_exists: - missing_values = ', '.join(not_exists) - self.warnings.append({ - 'col': self.column_number, - 'message': "The following values do not exist for {}: {}".format(self.df.options, missing_values), - 'type': 'warning' - }) + missing_values = ", ".join(not_exists) + self.warnings.append( + { + "col": self.column_number, + "message": ( + "The following values do not exist for {}: {}".format( + self.df.options, missing_values + ) + ), + "type": "warning", + } + ) elif self.df.fieldtype in ("Date", "Time", "Datetime"): # guess date format self.date_format = self.guess_date_format_for_column() if not self.date_format: - self.date_format = '%Y-%m-%d' - self.warnings.append({ - 'col': self.column_number, - 'message': _("Date format could not determined from the values in this column. Defaulting to yyyy-mm-dd."), - 'type': 'info' - }) + self.date_format = "%Y-%m-%d" + self.warnings.append( + { + "col": self.column_number, + "message": _( + "Date format could not be determined from the values in" + " this column. Defaulting to yyyy-mm-dd." + ), + "type": "info", + } + ) + elif self.df.fieldtype == "Select": + options = get_select_options(self.df) + if options: + values = list(set([cstr(v) for v in self.column_values[1:] if v])) + invalid = list(set(values) - set(options)) + if invalid: + valid_values = ", ".join([frappe.bold(o) for o in options]) + invalid_values = ", ".join([frappe.bold(i) for i in invalid]) + self.warnings.append( + { + "col": self.column_number, + "message": ( + "The following values are invalid: {0}. Values must be" + " one of {1}".format(invalid_values, valid_values) + ), + } + ) def as_dict(self): d = frappe._dict() @@ -990,7 +1025,7 @@ class Column: d.map_to_field = self.map_to_field d.date_format = self.date_format d.df = self.df - if hasattr(self.df, 'is_child_table_field'): + if hasattr(self.df, "is_child_table_field"): d.is_child_table_field = self.df.is_child_table_field d.child_table_df = self.df.child_table_df d.skip_import = self.skip_import @@ -1070,7 +1105,7 @@ def build_fields_dict_for_column_matching(parent_doctype): # other fields fields = get_standard_fields(doctype) + frappe.get_meta(doctype).fields for df in fields: - label = (df.label or '').strip() + label = (df.label or "").strip() fieldtype = df.fieldtype or "Data" parent = df.parent or parent_doctype if fieldtype not in no_value_fields: @@ -1164,12 +1199,17 @@ def get_user_format(date_format): .replace("%d", "dd") ) + def df_as_json(df): return { - 'fieldname': df.fieldname, - 'fieldtype': df.fieldtype, - 'label': df.label, - 'options': df.options, - 'parent': df.parent, - 'default': df.default + "fieldname": df.fieldname, + "fieldtype": df.fieldtype, + "label": df.label, + "options": df.options, + "parent": df.parent, + "default": df.default, } + + +def get_select_options(df): + return [d for d in (df.options or "").split("\n") if d] diff --git a/frappe/data_migration/doctype/data_migration_run/test_data_migration_run.py b/frappe/data_migration/doctype/data_migration_run/test_data_migration_run.py index ba4a255b97..c6c3ea138c 100644 --- a/frappe/data_migration/doctype/data_migration_run/test_data_migration_run.py +++ b/frappe/data_migration/doctype/data_migration_run/test_data_migration_run.py @@ -108,7 +108,7 @@ def create_plan(): 'connector_name': 'Local Connector', 'connector_type': 'Frappe', # connect to same host. - 'hostname': frappe.conf.host_name, + 'hostname': frappe.conf.host_name or frappe.utils.get_site_url(frappe.local.site), 'username': 'Administrator', - 'password': 'admin' + 'password': frappe.conf.get("admin_password") or 'admin' }).insert(ignore_if_duplicate=True) diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index cf8c6e80c6..29cd890bf1 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -251,7 +251,7 @@ class EmailAccount(Document): email_server = None if frappe.local.flags.in_test: - incoming_mails = test_mails + incoming_mails = test_mails or [] else: email_sync_rule = self.build_email_sync_rule() diff --git a/frappe/geo/country_info.json b/frappe/geo/country_info.json index 1bf5ada467..6a56107333 100644 --- a/frappe/geo/country_info.json +++ b/frappe/geo/country_info.json @@ -620,7 +620,12 @@ }, "Congo, The Democratic Republic of the": { "code": "cd", - "number_format": "#,###.##" + "number_format": "#,###.##", + "currency": "CDF", + "currency_name": "Congolese franc", + "currency_symbol": "FC", + "currency_fraction": "Centime", + "currency_fraction_units": 100 }, "Cook Islands": { "code": "ck", diff --git a/frappe/public/js/frappe/data_import/data_exporter.js b/frappe/public/js/frappe/data_import/data_exporter.js index f6af338235..dee4839b34 100644 --- a/frappe/public/js/frappe/data_import/data_exporter.js +++ b/frappe/public/js/frappe/data_import/data_exporter.js @@ -274,23 +274,29 @@ frappe.data_import.DataExporter = class DataExporter { ? this.column_map[child_fieldname] : this.column_map[doctype]; - let is_field_mandatory = df => (df.fieldname === 'name' && !child_fieldname) - || (df.reqd && this.exporting_for == 'Insert New Records'); + let is_field_mandatory = df => { + if (df.reqd && this.exporting_for == 'Insert New Records') { + return true; + } + if (autoname_field && df.fieldname == autoname_field.fieldname) { + return true; + } + if (df.fieldname === 'name') { + return true; + } + return false; + }; return fields .filter(df => { - if (autoname_field && df.fieldname === autoname_field.fieldname) { + if (autoname_field && df.fieldname === 'name') { return false; } return true; }) .map(df => { - let label = __(df.label); - if (autoname_field && df.fieldname === 'name') { - label = label + ` (${__(autoname_field.label)})`; - } return { - label, + label: __(df.label), value: df.fieldname, danger: is_field_mandatory(df), checked: false, diff --git a/frappe/public/js/frappe/data_import/import_preview.js b/frappe/public/js/frappe/data_import/import_preview.js index 4edcb87aeb..6c17cb4351 100644 --- a/frappe/public/js/frappe/data_import/import_preview.js +++ b/frappe/public/js/frappe/data_import/import_preview.js @@ -98,6 +98,9 @@ frappe.data_import.ImportPreview = class ImportPreview { .replace('%y', 'yy') .replace('%m', 'mm') .replace('%d', 'dd') + .replace('%H', 'HH') + .replace('%M', 'mm') + .replace('%S', 'ss') : null; let column_title = ` @@ -354,4 +357,4 @@ function get_fields_as_options(doctype, column_map) { }); }) ); -} \ No newline at end of file +} diff --git a/frappe/tests/test_scheduler.py b/frappe/tests/test_scheduler.py index e554fd23be..30818e6f17 100644 --- a/frappe/tests/test_scheduler.py +++ b/frappe/tests/test_scheduler.py @@ -49,7 +49,7 @@ class TestScheduler(TestCase): # 2nd job not loaded self.assertFalse(job.enqueue()) - job.delete() + frappe.db.sql('DELETE FROM `tabScheduled Job Log` WHERE `scheduled_job_type`=%s', job.name) def test_is_dormant(self): self.assertTrue(is_dormant(check_time= get_datetime('2100-01-01 00:00:00'))) diff --git a/frappe/workflow/doctype/workflow/test_workflow.py b/frappe/workflow/doctype/workflow/test_workflow.py index 2719bc7cf0..84adc3a096 100644 --- a/frappe/workflow/doctype/workflow/test_workflow.py +++ b/frappe/workflow/doctype/workflow/test_workflow.py @@ -6,6 +6,9 @@ import frappe import unittest from frappe.utils import random_string from frappe.model.workflow import apply_workflow, WorkflowTransitionError, WorkflowPermissionError, get_common_transition_actions +from frappe.test_runner import make_test_records + +make_test_records("User") class TestWorkflow(unittest.TestCase): def setUp(self): @@ -78,7 +81,7 @@ class TestWorkflow(unittest.TestCase): frappe.set_user('test2@example.com') doc = self.test_default_condition() - workflow_actions = frappe.get_all('Workflow Action', fields=['status']) + workflow_actions = frappe.get_all('Workflow Action', fields=['status', 'reference_name']) self.assertEqual(len(workflow_actions), 1) # test if status of workflow actions are updated on approval @@ -102,6 +105,9 @@ class TestWorkflow(unittest.TestCase): todo.reload() self.assertEqual(todo.docstatus, 1) + self.workflow.states[1].doc_status = 0 + self.workflow.save() + def test_if_workflow_set_on_action(self): self.workflow.states[1].doc_status = 1 self.workflow.save() @@ -111,12 +117,17 @@ class TestWorkflow(unittest.TestCase): self.assertEqual(todo.docstatus, 1) self.assertEqual(todo.workflow_state, 'Approved') + self.workflow.states[1].doc_status = 0 + self.workflow.save() + def create_todo_workflow(): if frappe.db.exists('Workflow', 'Test ToDo'): return frappe.get_doc('Workflow', 'Test ToDo').save(ignore_permissions=True) else: frappe.get_doc(dict(doctype='Role', role_name='Test Approver')).insert(ignore_if_duplicate=True) + frappe.db.commit() + frappe.cache().hdel('roles', frappe.session.user) workflow = frappe.new_doc('Workflow') workflow.workflow_name = 'Test ToDo' workflow.document_type = 'ToDo'