vassili/seitime-frappe
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Merge branch 'develop' into workspace-ui-fixes

Browse source
This commit is contained in:
Shariq Ansari 2022-02-08 20:17:13 +05:30 • committed by GitHub
commit 0cb3e5691f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
117 changed files with 2566 additions and 1370 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

79
cypress/integration/control_link.js
View file

@ -58,6 +58,23 @@ context('Control Link', () => {
cy.get('.frappe-control[data-fieldname=link] input').should('have.value', '');
});
it("should be possible set empty value explicitly", () => {
get_dialog_with_link().as("dialog");
cy.intercept("POST", "/api/method/frappe.client.validate_link").as("validate_link");
cy.get(".frappe-control[data-fieldname=link] input")
.type(" ", { delay: 100 })
.blur();
cy.wait("@validate_link");
cy.get(".frappe-control[data-fieldname=link] input").should("have.value", "");
cy.window()
.its("cur_dialog")
.then((dialog) => {
expect(dialog.get_value("link")).to.equal('');
});
});
it('should route to form on arrow click', () => {
get_dialog_with_link().as('dialog');
@ -78,7 +95,7 @@ context('Control Link', () => {
});
});
it('should fetch valid value', () => {
it('should update dependant fields (via fetch_from)', () => {
cy.get('@todos').then(todos => {
cy.visit(`/app/todo/${todos[0]}`);
cy.intercept('POST', '/api/method/frappe.client.validate_link').as('validate_link');
@ -89,7 +106,67 @@ context('Control Link', () => {
cy.get('.frappe-control[data-fieldname=assigned_by_full_name] .control-value').should(
'contain', 'Administrator'
);
cy.window()
.its("cur_frm.doc.assigned_by")
.should("eq", "Administrator");
// invalid input
cy.get('@input').clear().type('invalid input', {delay: 100}).blur();
cy.get('.frappe-control[data-fieldname=assigned_by_full_name] .control-value').should(
'contain', ''
);
cy.window()
.its("cur_frm.doc.assigned_by")
.should("eq", null);
// set valid value again
cy.get('@input').clear().type('Administrator', {delay: 100}).blur();
cy.wait('@validate_link');
cy.window()
.its("cur_frm.doc.assigned_by")
.should("eq", "Administrator");
// clear input
cy.get('@input').clear().blur();
cy.get('.frappe-control[data-fieldname=assigned_by_full_name] .control-value').should(
'contain', ''
);
cy.window()
.its("cur_frm.doc.assigned_by")
.should("eq", "");
});
});
it("should set default values", () => {
cy.insert_doc("Property Setter", {
"doctype_or_field": "DocField",
"doc_type": "ToDo",
"field_name": "assigned_by",
"property": "default",
"property_type": "Text",
"value": "Administrator"
}, true);
cy.reload();
cy.new_form("ToDo");
cy.fill_field("description", "new", "Text Editor");
cy.intercept("POST", "/api/method/frappe.desk.form.save.savedocs").as("save_form");
cy.findByRole("button", {name: "Save"}).click();
cy.wait("@save_form");
cy.get(".frappe-control[data-fieldname=assigned_by_full_name] .control-value").should(
"contain", "Administrator"
);
// if user clears default value explicitly, system should not reset default again
cy.get_field("assigned_by").clear().blur();
cy.intercept("POST", "/api/method/frappe.desk.form.save.savedocs").as("save_form");
cy.findByRole("button", {name: "Save"}).click();
cy.wait("@save_form");
cy.get_field("assigned_by").should("have.value", "");
cy.get(".frappe-control[data-fieldname=assigned_by_full_name] .control-value").should(
"contain", ""
);
});
});

28
cypress/support/commands.js
View file

@ -110,34 +110,6 @@ Cypress.Commands.add('get_doc', (doctype, name) => {
});
});
Cypress.Commands.add('insert_doc', (doctype, args, ignore_duplicate) => {
return cy
.window()
.its('frappe.csrf_token')
.then(csrf_token => {
return cy
.request({
method: 'POST',
url: `/api/resource/${doctype}`,
body: args,
headers: {
Accept: 'application/json',
'Content-Type': 'application/json',
'X-Frappe-CSRF-Token': csrf_token
},
failOnStatusCode: !ignore_duplicate
})
.then(res => {
let status_codes = [200];
if (ignore_duplicate) {
status_codes.push(409);
}
expect(res.status).to.be.oneOf(status_codes);
return res.body;
});
});
});
Cypress.Commands.add('remove_doc', (doctype, name) => {
return cy
.window()

6
frappe/__init__.py
View file

@ -143,6 +143,8 @@ lang = local("lang")
# This if block is never executed when running the code. It is only used for
# telling static code analyzer where to find dynamically defined attributes.
if typing.TYPE_CHECKING:
from frappe.utils.redis_wrapper import RedisWrapper
from frappe.database.mariadb.database import MariaDBDatabase
from frappe.database.postgres.database import PostgresDatabase
from frappe.query_builder.builder import MariaDB, Postgres
@ -150,6 +152,7 @@ if typing.TYPE_CHECKING:
db: typing.Union[MariaDBDatabase, PostgresDatabase]
qb: typing.Union[MariaDB, Postgres]
# end: static analysis hack
def init(site, sites_path=None, new_site=False):
@ -311,9 +314,8 @@ def destroy():
release_local(local)
# memcache
redis_server = None
def cache():
def cache() -> "RedisWrapper":
"""Returns redis connection."""
global redis_server
if not redis_server:

3
frappe/api.py
View file

@ -94,7 +94,8 @@ def handle():
"data": doc.save().as_dict()
})
if doc.parenttype and doc.parent:
# check for child table doctype
if doc.get("parenttype"):
frappe.get_doc(doc.parenttype, doc.parent).save()
frappe.db.commit()

7
frappe/app.py
View file

@ -192,12 +192,7 @@ def make_form_dict(request):
if not isinstance(args, dict):
frappe.throw(_("Invalid request arguments"))
try:
frappe.local.form_dict = frappe._dict({
k: v[0] if isinstance(v, (list, tuple)) else v for k, v in args.items()
})
except IndexError:
frappe.local.form_dict = frappe._dict(args)
frappe.local.form_dict = frappe._dict(args)
if "_" in frappe.local.form_dict:
# _ is passed by $.ajax so that the request is not cached by the browser. So, remove _ from form_dict

26
frappe/client.py
View file

@ -99,7 +99,6 @@ def get_value(doctype, fieldname, filters=None, as_dict=True, debug=False, paren
if not filters:
filters = None
if frappe.get_meta(doctype).issingle:
value = frappe.db.get_values_from_single(fields, filters, doctype, as_dict=as_dict, debug=debug)
else:
@ -129,7 +128,7 @@ def set_value(doctype, name, fieldname, value=None):
:param fieldname: fieldname string or JSON / dict with key value pair
:param value: value if fieldname is JSON / dict'''
if fieldname!="idx" and fieldname in frappe.model.default_fields:
if fieldname in (frappe.model.default_fields + frappe.model.child_table_fields):
frappe.throw(_("Cannot edit standard fields"))
if not value:
@ -142,14 +141,15 @@ def set_value(doctype, name, fieldname, value=None):
else:
values = {fieldname: value}
doc = frappe.db.get_value(doctype, name, ["parenttype", "parent"], as_dict=True)
if doc and doc.parent and doc.parenttype:
# check for child table doctype
if not frappe.get_meta(doctype).istable:
doc = frappe.get_doc(doctype, name)
doc.update(values)
else:
doc = frappe.db.get_value(doctype, name, ["parenttype", "parent"], as_dict=True)
doc = frappe.get_doc(doc.parenttype, doc.parent)
child = doc.getone({"doctype": doctype, "name": name})
child.update(values)
else:
doc = frappe.get_doc(doctype, name)
doc.update(values)
doc.save()
@ -163,10 +163,10 @@ def insert(doc=None):
if isinstance(doc, str):
doc = json.loads(doc)
if doc.get("parent") and doc.get("parenttype"):
if doc.get("parenttype"):
# inserting a child record
parent = frappe.get_doc(doc.get("parenttype"), doc.get("parent"))
parent.append(doc.get("parentfield"), doc)
parent = frappe.get_doc(doc.parenttype, doc.parent)
parent.append(doc.parentfield, doc)
parent.save()
return parent.as_dict()
else:
@ -187,10 +187,10 @@ def insert_many(docs=None):
frappe.throw(_('Only 200 inserts allowed in one request'))
for doc in docs:
if doc.get("parent") and doc.get("parenttype"):
if doc.get("parenttype"):
# inserting a child record
parent = frappe.get_doc(doc.get("parenttype"), doc.get("parent"))
parent.append(doc.get("parentfield"), doc)
parent = frappe.get_doc(doc.parenttype, doc.parent)
parent.append(doc.parentfield, doc)
parent.save()
out.append(parent.name)
else:

5
frappe/commands/utils.py
View file

@ -623,6 +623,7 @@ def transform_database(context, table, engine, row_format, failfast):
@click.command('run-tests')
@click.option('--app', help="For App")
@click.option('--doctype', help="For DocType")
@click.option('--case', help="Select particular TestCase")
@click.option('--doctype-list-path', help="Path to .txt file for list of doctypes. Example erpnext/tests/server/agriculture.txt")
@click.option('--test', multiple=True, help="Specific test")
@click.option('--ui-tests', is_flag=True, default=False, help="Run UI Tests")
@ -636,7 +637,7 @@ def transform_database(context, table, engine, row_format, failfast):
@pass_context
def run_tests(context, app=None, module=None, doctype=None, test=(), profile=False,
coverage=False, junit_xml_output=False, ui_tests = False, doctype_list_path=None,
skip_test_records=False, skip_before_tests=False, failfast=False):
skip_test_records=False, skip_before_tests=False, failfast=False, case=None):
with CodeCoverage(coverage, app):
import frappe.test_runner
@ -658,7 +659,7 @@ def run_tests(context, app=None, module=None, doctype=None, test=(), profile=Fal
ret = frappe.test_runner.main(app, module, doctype, context.verbose, tests=tests,
force=context.force, profile=profile, junit_xml_output=junit_xml_output,
ui_tests=ui_tests, doctype_list_path=doctype_list_path, failfast=failfast)
ui_tests=ui_tests, doctype_list_path=doctype_list_path, failfast=failfast, case=case)
if len(ret.failures) == 0 and len(ret.errors) == 0:
ret = 0

291
frappe/core/doctype/data_import/data_import.js
View file

@ -44,6 +44,7 @@ frappe.ui.form.on('Data Import', {
}
frm.dashboard.show_progress(__('Import Progress'), percent, message);
frm.page.set_indicator(__('In Progress'), 'orange');
frm.trigger('update_primary_action');
// hide progress when complete
if (data.current === data.total) {
@ -80,7 +81,10 @@ frappe.ui.form.on('Data Import', {
frm.trigger('show_import_log');
frm.trigger('show_import_warnings');
frm.trigger('toggle_submit_after_import');
frm.trigger('show_import_status');
if (frm.doc.status != 'Pending')
frm.trigger('show_import_status');
frm.trigger('show_report_error_button');
if (frm.doc.status === 'Partial Success') {
@ -128,40 +132,49 @@ frappe.ui.form.on('Data Import', {
},
show_import_status(frm) {
let import_log = JSON.parse(frm.doc.import_log || '[]');
let successful_records = import_log.filter(log => log.success);
let failed_records = import_log.filter(log => !log.success);
if (successful_records.length === 0) return;
frappe.call({
'method': 'frappe.core.doctype.data_import.data_import.get_import_status',
'args': {
'data_import_name': frm.doc.name
},
'callback': function(r) {
let successful_records = cint(r.message.success);
let failed_records = cint(r.message.failed);
let total_records = cint(r.message.total_records);
let message;
if (failed_records.length === 0) {
let message_args = [successful_records.length];
if (frm.doc.import_type === 'Insert New Records') {
message =
successful_records.length > 1
? __('Successfully imported {0} records.', message_args)
: __('Successfully imported {0} record.', message_args);
} else {
message =
successful_records.length > 1
? __('Successfully updated {0} records.', message_args)
: __('Successfully updated {0} record.', message_args);
if (!total_records) return;
let message;
if (failed_records === 0) {
let message_args = [successful_records];
if (frm.doc.import_type === 'Insert New Records') {
message =
successful_records > 1
? __('Successfully imported {0} records.', message_args)
: __('Successfully imported {0} record.', message_args);
} else {
message =
successful_records > 1
? __('Successfully updated {0} records.', message_args)
: __('Successfully updated {0} record.', message_args);
}
} else {
let message_args = [successful_records, total_records];
if (frm.doc.import_type === 'Insert New Records') {
message =
successful_records > 1
? __('Successfully imported {0} records out of {1}. Click on Export Errored Rows, fix the errors and import again.', message_args)
: __('Successfully imported {0} record out of {1}. Click on Export Errored Rows, fix the errors and import again.', message_args);
} else {
message =
successful_records > 1
? __('Successfully updated {0} records out of {1}. Click on Export Errored Rows, fix the errors and import again.', message_args)
: __('Successfully updated {0} record out of {1}. Click on Export Errored Rows, fix the errors and import again.', message_args);
}
}
frm.dashboard.set_headline(message);
}
} else {
let message_args = [successful_records.length, import_log.length];
if (frm.doc.import_type === 'Insert New Records') {
message =
successful_records.length > 1
? __('Successfully imported {0} records out of {1}. Click on Export Errored Rows, fix the errors and import again.', message_args)
: __('Successfully imported {0} record out of {1}. Click on Export Errored Rows, fix the errors and import again.', message_args);
} else {
message =
successful_records.length > 1
? __('Successfully updated {0} records out of {1}. Click on Export Errored Rows, fix the errors and import again.', message_args)
: __('Successfully updated {0} record out of {1}. Click on Export Errored Rows, fix the errors and import again.', message_args);
}
}
frm.dashboard.set_headline(message);
});
},
show_report_error_button(frm) {
@ -275,7 +288,7 @@ frappe.ui.form.on('Data Import', {
},
show_import_preview(frm, preview_data) {
let import_log = JSON.parse(frm.doc.import_log || '[]');
let import_log = preview_data.import_log;
if (
frm.import_preview &&
@ -316,6 +329,15 @@ frappe.ui.form.on('Data Import', {
);
},
export_import_log(frm) {
open_url_post(
'/api/method/frappe.core.doctype.data_import.data_import.download_import_log',
{
data_import_name: frm.doc.name
}
);
},
show_import_warnings(frm, preview_data) {
let columns = preview_data.columns;
let warnings = JSON.parse(frm.doc.template_warnings || '[]');
@ -391,92 +413,131 @@ frappe.ui.form.on('Data Import', {
frm.trigger('show_import_log');
},
show_import_log(frm) {
let import_log = JSON.parse(frm.doc.import_log || '[]');
let logs = import_log;
frm.toggle_display('import_log', false);
frm.toggle_display('import_log_section', logs.length > 0);
render_import_log(frm) {
frappe.call({
'method': 'frappe.client.get_list',
'args': {
'doctype': 'Data Import Log',
'filters': {
'data_import': frm.doc.name
},
'fields': ['success', 'docname', 'messages', 'exception', 'row_indexes'],
'limit_page_length': 5000,
'order_by': 'log_index'
},
callback: function(r) {
let logs = r.message;
if (logs.length === 0) {
frm.get_field('import_log_preview').$wrapper.empty();
if (logs.length === 0) return;
frm.toggle_display('import_log_section', true);
let rows = logs
.map(log => {
let html = '';
if (log.success) {
if (frm.doc.import_type === 'Insert New Records') {
html = __('Successfully imported {0}', [
`<span class="underline">${frappe.utils.get_form_link(
frm.doc.reference_doctype,
log.docname,
true
)}<span>`
]);
} else {
html = __('Successfully updated {0}', [
`<span class="underline">${frappe.utils.get_form_link(
frm.doc.reference_doctype,
log.docname,
true
)}<span>`
]);
}
} else {
let messages = (JSON.parse(log.messages || '[]'))
.map(JSON.parse)
.map(m => {
let title = m.title ? `<strong>${m.title}</strong>` : '';
let message = m.message ? `<div>${m.message}</div>` : '';
return title + message;
})
.join('');
let id = frappe.dom.get_unique_id();
html = `${messages}
<button class="btn btn-default btn-xs" type="button" data-toggle="collapse" data-target="#${id}" aria-expanded="false" aria-controls="${id}" style="margin-top: 15px;">
${__('Show Traceback')}
</button>
<div class="collapse" id="${id}" style="margin-top: 15px;">
<div class="well">
<pre>${log.exception}</pre>
</div>
</div>`;
}
let indicator_color = log.success ? 'green' : 'red';
let title = log.success ? __('Success') : __('Failure');
if (frm.doc.show_failed_logs && log.success) {
return '';
}
return `<tr>
<td>${JSON.parse(log.row_indexes).join(', ')}</td>
<td>
<div class="indicator ${indicator_color}">${title}</div>
</td>
<td>
${html}
</td>
</tr>`;
})
.join('');
if (!rows && frm.doc.show_failed_logs) {
rows = `<tr><td class="text-center text-muted" colspan=3>
${__('No failed logs')}
</td></tr>`;
}
frm.get_field('import_log_preview').$wrapper.html(`
<table class="table table-bordered">
<tr class="text-muted">
<th width="10%">${__('Row Number')}</th>
<th width="10%">${__('Status')}</th>
<th width="80%">${__('Message')}</th>
</tr>
${rows}
</table>
`);
}
});
},
show_import_log(frm) {
frm.toggle_display('import_log_section', false);
if (frm.import_in_progress) {
return;
}
let rows = logs
.map(log => {
let html = '';
if (log.success) {
if (frm.doc.import_type === 'Insert New Records') {
html = __('Successfully imported {0}', [
`<span class="underline">${frappe.utils.get_form_link(
frm.doc.reference_doctype,
log.docname,
true
)}<span>`
]);
} else {
html = __('Successfully updated {0}', [
`<span class="underline">${frappe.utils.get_form_link(
frm.doc.reference_doctype,
log.docname,
true
)}<span>`
]);
}
frappe.call({
'method': 'frappe.client.get_count',
'args': {
'doctype': 'Data Import Log',
'filters': {
'data_import': frm.doc.name
}
},
'callback': function(r) {
let count = r.message;
if (count < 5000) {
frm.trigger('render_import_log');
} else {
let messages = log.messages
.map(JSON.parse)
.map(m => {
let title = m.title ? `<strong>${m.title}</strong>` : '';
let message = m.message ? `<div>${m.message}</div>` : '';
return title + message;
})
.join('');
let id = frappe.dom.get_unique_id();
html = `${messages}
<button class="btn btn-default btn-xs" type="button" data-toggle="collapse" data-target="#${id}" aria-expanded="false" aria-controls="${id}" style="margin-top: 15px;">
${__('Show Traceback')}
</button>
<div class="collapse" id="${id}" style="margin-top: 15px;">
<div class="well">
<pre>${log.exception}</pre>
</div>
</div>`;
frm.toggle_display('import_log_section', false);
frm.add_custom_button(__('Export Import Log'), () =>
frm.trigger('export_import_log')
);
}
let indicator_color = log.success ? 'green' : 'red';
let title = log.success ? __('Success') : __('Failure');
if (frm.doc.show_failed_logs && log.success) {
return '';
}
return `<tr>
<td>${log.row_indexes.join(', ')}</td>
<td>
<div class="indicator ${indicator_color}">${title}</div>
</td>
<td>
${html}
</td>
</tr>`;
})
.join('');
if (!rows && frm.doc.show_failed_logs) {
rows = `<tr><td class="text-center text-muted" colspan=3>
${__('No failed logs')}
</td></tr>`;
}
frm.get_field('import_log_preview').$wrapper.html(`
<table class="table table-bordered">
<tr class="text-muted">
<th width="10%">${__('Row Number')}</th>
<th width="10%">${__('Status')}</th>
<th width="80%">${__('Message')}</th>
</tr>
${rows}
</table>
`);
}
});
},
});

387
frappe/core/doctype/data_import/data_import.json
View file

@ -1,194 +1,197 @@
{
"actions": [],
"autoname": "format:{reference_doctype} Import on {creation}",
"beta": 1,
"creation": "2019-08-04 14:16:08.318714",
"doctype": "DocType",
"editable_grid": 1,
"engine": "InnoDB",
"field_order": [
"reference_doctype",
"import_type",
"download_template",
"import_file",
"html_5",
"google_sheets_url",
"refresh_google_sheet",
"column_break_5",
"status",
"submit_after_import",
"mute_emails",
"template_options",
"import_warnings_section",
"template_warnings",
"import_warnings",
"section_import_preview",
"import_preview",
"import_log_section",
"import_log",
"show_failed_logs",
"import_log_preview"
],
"fields": [
{
"fieldname": "reference_doctype",
"fieldtype": "Link",
"in_list_view": 1,
"label": "Document Type",
"options": "DocType",
"reqd": 1,
"set_only_once": 1
},
{
"fieldname": "import_type",
"fieldtype": "Select",
"in_list_view": 1,
"label": "Import Type",
"options": "\nInsert New Records\nUpdate Existing Records",
"reqd": 1,
"set_only_once": 1
},
{
"depends_on": "eval:!doc.__islocal",
"fieldname": "import_file",
"fieldtype": "Attach",
"in_list_view": 1,
"label": "Import File",
"read_only_depends_on": "eval: ['Success', 'Partial Success'].includes(doc.status)"
},
{
"fieldname": "import_preview",
"fieldtype": "HTML",
"label": "Import Preview"
},
{
"fieldname": "section_import_preview",
"fieldtype": "Section Break",
"label": "Preview"
},
{
"fieldname": "column_break_5",
"fieldtype": "Column Break"
},
{
"fieldname": "template_options",
"fieldtype": "Code",
"hidden": 1,
"label": "Template Options",
"options": "JSON",
"read_only": 1
},
{
"fieldname": "import_log",
"fieldtype": "Code",
"label": "Import Log",
"options": "JSON"
},
{
"fieldname": "import_log_section",
"fieldtype": "Section Break",
"label": "Import Log"
},
{
"fieldname": "import_log_preview",
"fieldtype": "HTML",
"label": "Import Log Preview"
},
{
"default": "Pending",
"fieldname": "status",
"fieldtype": "Select",
"hidden": 1,
"label": "Status",
"options": "Pending\nSuccess\nPartial Success\nError",
"read_only": 1
},
{
"fieldname": "template_warnings",
"fieldtype": "Code",
"hidden": 1,
"label": "Template Warnings",
"options": "JSON"
},
{
"default": "0",
"fieldname": "submit_after_import",
"fieldtype": "Check",
"label": "Submit After Import",
"set_only_once": 1
},
{
"fieldname": "import_warnings_section",
"fieldtype": "Section Break",
"label": "Import File Errors and Warnings"
},
{
"fieldname": "import_warnings",
"fieldtype": "HTML",
"label": "Import Warnings"
},
{
"depends_on": "eval:!doc.__islocal",
"fieldname": "download_template",
"fieldtype": "Button",
"label": "Download Template"
},
{
"default": "1",
"fieldname": "mute_emails",
"fieldtype": "Check",
"label": "Don't Send Emails",
"set_only_once": 1
},
{
"default": "0",
"fieldname": "show_failed_logs",
"fieldtype": "Check",
"label": "Show Failed Logs"
},
{
"depends_on": "eval:!doc.__islocal && !doc.import_file",
"fieldname": "html_5",
"fieldtype": "HTML",
"options": "<h5 class=\"text-muted uppercase\">Or</h5>"
},
{
"depends_on": "eval:!doc.__islocal && !doc.import_file\n",
"description": "Must be a publicly accessible Google Sheets URL",
"fieldname": "google_sheets_url",
"fieldtype": "Data",
"label": "Import from Google Sheets",
"read_only_depends_on": "eval: ['Success', 'Partial Success'].includes(doc.status)"
},
{
"depends_on": "eval:doc.google_sheets_url && !doc.__unsaved && ['Success', 'Partial Success'].includes(doc.status)",
"fieldname": "refresh_google_sheet",
"fieldtype": "Button",
"label": "Refresh Google Sheet"
}
],
"hide_toolbar": 1,
"links": [],
"modified": "2021-04-11 01:50:42.074623",
"modified_by": "Administrator",
"module": "Core",
"name": "Data Import",
"owner": "Administrator",
"permissions": [
{
"create": 1,
"delete": 1,
"email": 1,
"export": 1,
"print": 1,
"read": 1,
"report": 1,
"role": "System Manager",
"share": 1,
"write": 1
}
],
"sort_field": "modified",
"sort_order": "DESC",
"track_changes": 1
"actions": [],
"autoname": "format:{reference_doctype} Import on {creation}",
"beta": 1,
"creation": "2019-08-04 14:16:08.318714",
"doctype": "DocType",
"editable_grid": 1,
"engine": "InnoDB",
"field_order": [
"reference_doctype",
"import_type",
"download_template",
"import_file",
"payload_count",
"html_5",
"google_sheets_url",
"refresh_google_sheet",
"column_break_5",
"status",
"submit_after_import",
"mute_emails",
"template_options",
"import_warnings_section",
"template_warnings",
"import_warnings",
"section_import_preview",
"import_preview",
"import_log_section",
"show_failed_logs",
"import_log_preview"
],
"fields": [
{
"fieldname": "reference_doctype",
"fieldtype": "Link",
"in_list_view": 1,
"label": "Document Type",
"options": "DocType",
"reqd": 1,
"set_only_once": 1
},
{
"fieldname": "import_type",
"fieldtype": "Select",
"in_list_view": 1,
"label": "Import Type",
"options": "\nInsert New Records\nUpdate Existing Records",
"reqd": 1,
"set_only_once": 1
},
{
"depends_on": "eval:!doc.__islocal",
"fieldname": "import_file",
"fieldtype": "Attach",
"in_list_view": 1,
"label": "Import File",
"read_only_depends_on": "eval: ['Success', 'Partial Success'].includes(doc.status)"
},
{
"fieldname": "import_preview",
"fieldtype": "HTML",
"label": "Import Preview"
},
{
"fieldname": "section_import_preview",
"fieldtype": "Section Break",
"label": "Preview"
},
{
"fieldname": "column_break_5",
"fieldtype": "Column Break"
},
{
"fieldname": "template_options",
"fieldtype": "Code",
"hidden": 1,
"label": "Template Options",
"options": "JSON",
"read_only": 1
},
{
"fieldname": "import_log_section",
"fieldtype": "Section Break",
"label": "Import Log"
},
{
"fieldname": "import_log_preview",
"fieldtype": "HTML",
"label": "Import Log Preview"
},
{
"default": "Pending",
"fieldname": "status",
"fieldtype": "Select",
"hidden": 1,
"label": "Status",
"options": "Pending\nSuccess\nPartial Success\nError",
"read_only": 1
},
{
"fieldname": "template_warnings",
"fieldtype": "Code",
"hidden": 1,
"label": "Template Warnings",
"options": "JSON"
},
{
"default": "0",
"fieldname": "submit_after_import",
"fieldtype": "Check",
"label": "Submit After Import",
"set_only_once": 1
},
{
"fieldname": "import_warnings_section",
"fieldtype": "Section Break",
"label": "Import File Errors and Warnings"
},
{
"fieldname": "import_warnings",
"fieldtype": "HTML",
"label": "Import Warnings"
},
{
"depends_on": "eval:!doc.__islocal",
"fieldname": "download_template",
"fieldtype": "Button",
"label": "Download Template"
},
{
"default": "1",
"fieldname": "mute_emails",
"fieldtype": "Check",
"label": "Don't Send Emails",
"set_only_once": 1
},
{
"default": "0",
"fieldname": "show_failed_logs",
"fieldtype": "Check",
"label": "Show Failed Logs"
},
{
"depends_on": "eval:!doc.__islocal && !doc.import_file",
"fieldname": "html_5",
"fieldtype": "HTML",
"options": "<h5 class=\"text-muted uppercase\">Or</h5>"
},
{
"depends_on": "eval:!doc.__islocal && !doc.import_file\n",
"description": "Must be a publicly accessible Google Sheets URL",
"fieldname": "google_sheets_url",
"fieldtype": "Data",
"label": "Import from Google Sheets",
"read_only_depends_on": "eval: ['Success', 'Partial Success'].includes(doc.status)"
},
{
"depends_on": "eval:doc.google_sheets_url && !doc.__unsaved && ['Success', 'Partial Success'].includes(doc.status)",
"fieldname": "refresh_google_sheet",
"fieldtype": "Button",
"label": "Refresh Google Sheet"
},
{
"fieldname": "payload_count",
"fieldtype": "Int",
"hidden": 1,
"label": "Payload Count",
"read_only": 1
}
],
"hide_toolbar": 1,
"links": [],
"modified": "2022-02-01 20:08:37.624914",
"modified_by": "Administrator",
"module": "Core",
"name": "Data Import",
"naming_rule": "Expression",
"owner": "Administrator",
"permissions": [
{
"create": 1,
"delete": 1,
"email": 1,
"export": 1,
"print": 1,
"read": 1,
"report": 1,
"role": "System Manager",
"share": 1,
"write": 1
}
],
"sort_field": "modified",
"sort_order": "DESC",
"states": [],
"track_changes": 1
}

37
frappe/core/doctype/data_import/data_import.py
View file

@ -27,6 +27,7 @@ class DataImport(Document):
self.validate_import_file()
self.validate_google_sheets_url()
self.set_payload_count()
def validate_import_file(self):
if self.import_file:
@ -38,6 +39,12 @@ class DataImport(Document):
return
validate_google_sheets_url(self.google_sheets_url)
def set_payload_count(self):
if self.import_file:
i = self.get_importer()
payloads = i.import_file.get_payloads_for_import()
self.payload_count = len(payloads)
@frappe.whitelist()
def get_preview_from_template(self, import_file=None, google_sheets_url=None):
if import_file:
@ -67,7 +74,7 @@ class DataImport(Document):
enqueue(
start_import,
queue="default",
timeout=6000,
timeout=10000,
event="data_import",
job_name=self.name,
data_import=self.name,
@ -80,6 +87,9 @@ class DataImport(Document):
def export_errored_rows(self):
return self.get_importer().export_errored_rows()
def download_import_log(self):
return self.get_importer().export_import_log()
def get_importer(self):
return Importer(self.reference_doctype, data_import=self)
@ -90,7 +100,6 @@ def get_preview_from_template(data_import, import_file=None, google_sheets_url=N
import_file, google_sheets_url
)
@frappe.whitelist()
def form_start_import(data_import):
return frappe.get_doc("Data Import", data_import).start_import()
@ -145,6 +154,30 @@ def download_errored_template(data_import_name):
data_import = frappe.get_doc("Data Import", data_import_name)
data_import.export_errored_rows()
@frappe.whitelist()
def download_import_log(data_import_name):
data_import = frappe.get_doc("Data Import", data_import_name)
data_import.download_import_log()
@frappe.whitelist()
def get_import_status(data_import_name):
import_status = {}
logs = frappe.get_all('Data Import Log', fields=['count(*) as count', 'success'],
filters={'data_import': data_import_name},
group_by='success')
total_payload_count = frappe.db.get_value('Data Import', data_import_name, 'payload_count')
for log in logs:
if log.get('success'):
import_status['success'] = log.get('count')
else:
import_status['failed'] = log.get('count')
import_status['total_records'] = total_payload_count
return import_status
def import_file(
doctype, file_path, import_type, submit_after_import=False, console=False

2
frappe/core/doctype/data_import/data_import_list.js
View file

@ -24,12 +24,14 @@ frappe.listview_settings['Data Import'] = {
'Error': 'red'
};
let status = doc.status;
if (imports_in_progress.includes(doc.name)) {
status = 'In Progress';
}
if (status == 'Pending') {
status = 'Not Started';
}
return [__(status), colors[status], 'status,=,' + doc.status];
},
formatters: {

116
frappe/core/doctype/data_import/importer.py
View file

@ -47,7 +47,13 @@ class Importer:
)
def get_data_for_import_preview(self):
return self.import_file.get_data_for_import_preview()
out = self.import_file.get_data_for_import_preview()
out.import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes", "success"],
filters={"data_import": self.data_import.name},
order_by="log_index", limit=10)
return out
def before_import(self):
# set user lang for translations
@ -58,7 +64,6 @@ class Importer:
frappe.flags.in_import = True
frappe.flags.mute_emails = self.data_import.mute_emails
self.data_import.db_set("status", "Pending")
self.data_import.db_set("template_warnings", "")
def import_data(self):
@ -79,20 +84,25 @@ class Importer:
return
# setup import log
if self.data_import.import_log:
import_log = frappe.parse_json(self.data_import.import_log)
else:
import_log = []
import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes", "success", "log_index"],
filters={"data_import": self.data_import.name},
order_by="log_index") or []
# remove previous failures from import log
import_log = [log for log in import_log if log.get("success")]
log_index = 0
# Do not remove rows in case of retry after an error or pending data import
if self.data_import.status == "Partial Success" and len(import_log) >= self.data_import.payload_count:
# remove previous failures from import log only in case of retry after partial success
import_log = [log for log in import_log if log.get("success")]
# get successfully imported rows
imported_rows = []
for log in import_log:
log = frappe._dict(log)
if log.success:
imported_rows += log.row_indexes
if log.success or len(import_log) < self.data_import.payload_count:
imported_rows += json.loads(log.row_indexes)
log_index = log.log_index
# start import
total_payload_count = len(payloads)
@ -146,25 +156,41 @@ class Importer:
},
)
import_log.append(
frappe._dict(success=True, docname=doc.name, row_indexes=row_indexes)
)
create_import_log(self.data_import.name, log_index, {
'success': True,
'docname': doc.name,
'row_indexes': row_indexes
})
log_index += 1
if not self.data_import.status == "Partial Success":
self.data_import.db_set("status", "Partial Success")
# commit after every successful import
frappe.db.commit()
except Exception:
import_log.append(
frappe._dict(
success=False,
exception=frappe.get_traceback(),
messages=frappe.local.message_log,
row_indexes=row_indexes,
)
)
messages = frappe.local.message_log
frappe.clear_messages()
# rollback if exception
frappe.db.rollback()
create_import_log(self.data_import.name, log_index, {
'success': False,
'exception': frappe.get_traceback(),
'messages': messages,
'row_indexes': row_indexes
})
log_index += 1
# Logs are db inserted directly so will have to be fetched again
import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes", "success", "log_index"],
filters={"data_import": self.data_import.name},
order_by="log_index") or []
# set status
failures = [log for log in import_log if not log.get("success")]
if len(failures) == total_payload_count:
@ -178,7 +204,6 @@ class Importer:
self.print_import_log(import_log)
else:
self.data_import.db_set("status", status)
self.data_import.db_set("import_log", json.dumps(import_log))
self.after_import()
@ -248,11 +273,14 @@ class Importer:
if not self.data_import:
return
import_log = frappe.parse_json(self.data_import.import_log or "[]")
import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes", "success"],
filters={"data_import": self.data_import.name},
order_by="log_index") or []
failures = [log for log in import_log if not log.get("success")]
row_indexes = []
for f in failures:
row_indexes.extend(f.get("row_indexes", []))
row_indexes.extend(json.loads(f.get("row_indexes", [])))
# de duplicate
row_indexes = list(set(row_indexes))
@ -264,6 +292,30 @@ class Importer:
build_csv_response(rows, _(self.doctype))
def export_import_log(self):
from frappe.utils.csvutils import build_csv_response
if not self.data_import:
return
import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes", "success", "messages", "exception", "docname"],
filters={"data_import": self.data_import.name},
order_by="log_index")
header_row = ["Row Numbers", "Status", "Message", "Exception"]
rows = [header_row]
for log in import_log:
row_number = json.loads(log.get("row_indexes"))[0]
status = "Success" if log.get('success') else "Failure"
message = "Successfully Imported {0}".format(log.get('docname')) if log.get('success') else \
log.get("messages")
exception = frappe.utils.cstr(log.get("exception", ''))
rows += [[row_number, status, message, exception]]
build_csv_response(rows, self.doctype)
def print_import_log(self, import_log):
failed_records = [log for log in import_log if not log.success]
successful_records = [log for log in import_log if log.success]
@ -566,7 +618,7 @@ class Row:
)
# remove standard fields and __islocal
for key in frappe.model.default_fields + ("__islocal",):
for key in frappe.model.default_fields + frappe.model.child_table_fields + ("__islocal",):
doc.pop(key, None)
for col, value in zip(columns, values):
@ -1172,3 +1224,17 @@ def df_as_json(df):
def get_select_options(df):
return [d for d in (df.options or "").split("\n") if d]
def create_import_log(data_import, log_index, log_details):
frappe.get_doc({
'doctype': 'Data Import Log',
'log_index': log_index,
'success': log_details.get('success'),
'data_import': data_import,
'row_indexes': json.dumps(log_details.get('row_indexes')),
'docname': log_details.get('docname'),
'messages': json.dumps(log_details.get('messages', '[]')),
'exception': log_details.get('exception')
}).db_insert()

23
frappe/core/doctype/data_import/test_importer.py
View file

@ -4,6 +4,7 @@
import unittest
import frappe
from frappe.core.doctype.data_import.importer import Importer
from frappe.tests.test_query_builder import db_type_is, run_only_if
from frappe.utils import getdate, format_duration
doctype_name = 'DocType for Import'
@ -54,21 +55,27 @@ class TestImporter(unittest.TestCase):
self.assertEqual(len(preview.data), 4)
self.assertEqual(len(preview.columns), 16)
# ignored on postgres because myisam doesn't exist on pg
@run_only_if(db_type_is.MARIADB)
def test_data_import_without_mandatory_values(self):
import_file = get_import_file('sample_import_file_without_mandatory')
data_import = self.get_importer(doctype_name, import_file)
frappe.local.message_log = []
data_import.start_import()
data_import.reload()
import_log = frappe.parse_json(data_import.import_log)
self.assertEqual(import_log[0]['row_indexes'], [2,3])
expected_error = "Error: <strong>Child 1 of DocType for Import</strong> Row #1: Value missing for: Child Title"
self.assertEqual(frappe.parse_json(import_log[0]['messages'][0])['message'], expected_error)
expected_error = "Error: <strong>Child 1 of DocType for Import</strong> Row #2: Value missing for: Child Title"
self.assertEqual(frappe.parse_json(import_log[0]['messages'][1])['message'], expected_error)
self.assertEqual(import_log[1]['row_indexes'], [4])
self.assertEqual(frappe.parse_json(import_log[1]['messages'][0])['message'], "Title is required")
import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes", "success", "messages", "exception", "docname"],
filters={"data_import": data_import.name},
order_by="log_index")
self.assertEqual(frappe.parse_json(import_log[0]['row_indexes']), [2,3])
expected_error = "Error: <strong>Child 1 of DocType for Import</strong> Row #1: Value missing for: Child Title"
self.assertEqual(frappe.parse_json(frappe.parse_json(import_log[0]['messages'])[0])['message'], expected_error)
expected_error = "Error: <strong>Child 1 of DocType for Import</strong> Row #2: Value missing for: Child Title"
self.assertEqual(frappe.parse_json(frappe.parse_json(import_log[0]['messages'])[1])['message'], expected_error)
self.assertEqual(frappe.parse_json(import_log[1]['row_indexes']), [4])
self.assertEqual(frappe.parse_json(frappe.parse_json(import_log[1]['messages'])[0])['message'], "Title is required")
def test_data_import_update(self):
existing_doc = frappe.get_doc(

0
frappe/core/doctype/data_import_log/__init__.py Normal file
View file

8
frappe/core/doctype/data_import_log/data_import_log.js Normal file
View file

@ -0,0 +1,8 @@
// Copyright (c) 2021, Frappe Technologies and contributors
// For license information, please see license.txt
frappe.ui.form.on('Data Import Log', {
// refresh: function(frm) {
// }
});

84
frappe/core/doctype/data_import_log/data_import_log.json Normal file
View file

@ -0,0 +1,84 @@
{
"actions": [],
"creation": "2021-12-25 16:12:20.205889",
"doctype": "DocType",
"editable_grid": 1,
"engine": "MyISAM",
"field_order": [
"data_import",
"row_indexes",
"success",
"docname",
"messages",
"exception",
"log_index"
],
"fields": [
{
"fieldname": "data_import",
"fieldtype": "Link",
"in_list_view": 1,
"label": "Data Import",
"options": "Data Import"
},
{
"fieldname": "docname",
"fieldtype": "Data",
"label": "Reference Name"
},
{
"fieldname": "exception",
"fieldtype": "Text",
"label": "Exception"
},
{
"fieldname": "row_indexes",
"fieldtype": "Code",
"label": "Row Indexes",
"options": "JSON"
},
{
"default": "0",
"fieldname": "success",
"fieldtype": "Check",
"in_list_view": 1,
"label": "Success"
},
{
"fieldname": "log_index",
"fieldtype": "Int",
"in_list_view": 1,
"label": "Log Index"
},
{
"fieldname": "messages",
"fieldtype": "Code",
"label": "Messages",
"options": "JSON"
}
],
"in_create": 1,
"index_web_pages_for_search": 1,
"links": [],
"modified": "2021-12-29 11:19:19.646076",
"modified_by": "Administrator",
"module": "Core",
"name": "Data Import Log",
"owner": "Administrator",
"permissions": [
{
"create": 1,
"delete": 1,
"email": 1,
"export": 1,
"print": 1,
"read": 1,
"report": 1,
"role": "System Manager",
"share": 1,
"write": 1
}
],
"sort_field": "modified",
"sort_order": "DESC"
}

8
frappe/core/doctype/data_import_log/data_import_log.py Normal file
View file

@ -0,0 +1,8 @@
# Copyright (c) 2021, Frappe Technologies and contributors
# For license information, please see license.txt
# import frappe
from frappe.model.document import Document
class DataImportLog(Document):
pass

8
frappe/core/doctype/data_import_log/test_data_import_log.py Normal file
View file

@ -0,0 +1,8 @@
# Copyright (c) 2021, Frappe Technologies and Contributors
# See license.txt
# import frappe
import unittest
class TestDataImportLog(unittest.TestCase):
pass

30
frappe/core/doctype/doctype/doctype.py
View file

@ -10,7 +10,9 @@ from frappe.cache_manager import clear_user_cache, clear_controller_cache
import frappe
from frappe import _
from frappe.utils import now, cint
from frappe.model import no_value_fields, default_fields, data_fieldtypes, table_fields, data_field_options
from frappe.model import (
no_value_fields, default_fields, table_fields, data_field_options, child_table_fields
)
from frappe.model.document import Document
from frappe.model.base_document import get_controller
from frappe.custom.doctype.property_setter.property_setter import make_property_setter
@ -74,6 +76,7 @@ class DocType(Document):
self.make_amendable()
self.make_repeatable()
self.validate_nestedset()
self.validate_child_table()
self.validate_website()
self.ensure_minimum_max_attachment_limit()
validate_links_table_fieldnames(self)
@ -689,6 +692,22 @@ class DocType(Document):
})
self.nsm_parent_field = parent_field_name
def validate_child_table(self):
if not self.get("istable") or self.is_new():
# if the doctype is not a child table then return
# if the doctype is a new doctype and also a child table then
# don't move forward as it will be handled via schema
return
self.add_child_table_fields()
def add_child_table_fields(self):
from frappe.database.schema import add_column
add_column(self.name, "parent", "Data")
add_column(self.name, "parenttype", "Data")
add_column(self.name, "parentfield", "Data")
def get_max_idx(self):
"""Returns the highest `idx`"""
max_idx = frappe.db.sql("""select max(idx) from `tabDocField` where parent = %s""",
@ -699,6 +718,13 @@ class DocType(Document):
if not name:
name = self.name
# a Doctype name is the tablename created in database
# `tab<Doctype Name>` the length of tablename is limited to 64 characters
max_length = frappe.db.MAX_COLUMN_LENGTH - 3
if len(name) > max_length:
# length(tab + <Doctype Name>) should be equal to 64 characters hence doctype should be 61 characters
frappe.throw(_("Doctype name is limited to {0} characters ({1})").format(max_length, name), frappe.NameError)
flags = {"flags": re.ASCII}
# a DocType name should not start or end with an empty space
@ -1009,7 +1035,7 @@ def validate_fields(meta):
sort_fields = [d.split()[0] for d in meta.sort_field.split(',')]
for fieldname in sort_fields:
if not fieldname in fieldname_list + list(default_fields):
if fieldname not in (fieldname_list + list(default_fields) + list(child_table_fields)):
frappe.throw(_("Sort field {0} must be a valid fieldname").format(fieldname),
InvalidFieldNameError)

9
frappe/core/doctype/doctype/test_doctype.py
View file

@ -23,6 +23,7 @@ class TestDocType(unittest.TestCase):
self.assertRaises(frappe.NameError, new_doctype("_Some DocType").insert)
self.assertRaises(frappe.NameError, new_doctype("8Some DocType").insert)
self.assertRaises(frappe.NameError, new_doctype("Some (DocType)").insert)
self.assertRaises(frappe.NameError, new_doctype("Some Doctype with a name whose length is more than 61 characters").insert)
for name in ("Some DocType", "Some_DocType"):
if frappe.db.exists("DocType", name):
frappe.delete_doc("DocType", name)
@ -353,7 +354,6 @@ class TestDocType(unittest.TestCase):
dump_docs = json.dumps(docs.get('docs'))
cancel_all_linked_docs(dump_docs)
data_link_doc.cancel()
data_doc.name = '{}-CANC-0'.format(data_doc.name)
data_doc.load_from_db()
self.assertEqual(data_link_doc.docstatus, 2)
self.assertEqual(data_doc.docstatus, 2)
@ -377,7 +377,7 @@ class TestDocType(unittest.TestCase):
for data in link_doc.get('permissions'):
data.submit = 1
data.cancel = 1
link_doc.insert(ignore_if_duplicate=True)
link_doc.insert()
#create first parent doctype
test_doc_1 = new_doctype('Test Doctype 1')
@ -392,7 +392,7 @@ class TestDocType(unittest.TestCase):
for data in test_doc_1.get('permissions'):
data.submit = 1
data.cancel = 1
test_doc_1.insert(ignore_if_duplicate=True)
test_doc_1.insert()
#crete second parent doctype
doc = new_doctype('Test Doctype 2')
@ -407,7 +407,7 @@ class TestDocType(unittest.TestCase):
for data in link_doc.get('permissions'):
data.submit = 1
data.cancel = 1
doc.insert(ignore_if_duplicate=True)
doc.insert()
# create doctype data
data_link_doc_1 = frappe.new_doc('Test Linked Doctype 1')
@ -438,7 +438,6 @@ class TestDocType(unittest.TestCase):
# checking that doc for Test Doctype 2 is not canceled
self.assertRaises(frappe.LinkExistsError, data_link_doc_1.cancel)
data_doc_2.name = '{}-CANC-0'.format(data_doc_2.name)
data_doc.load_from_db()
data_doc_2.load_from_db()
self.assertEqual(data_link_doc_1.docstatus, 2)

39
frappe/core/doctype/file/file.py
View file

@ -1,4 +1,4 @@
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors
# License: MIT. See LICENSE
"""
@ -7,7 +7,6 @@ record of files
naming for same name files: file.gif, file-1.gif, file-2.gif etc
"""
import base64
import hashlib
import imghdr
import io
@ -17,9 +16,10 @@ import os
import re
import shutil
import zipfile
from typing import TYPE_CHECKING, Tuple
import requests
import requests.exceptions
from requests.exceptions import HTTPError, SSLError
from PIL import Image, ImageFile, ImageOps
from io import BytesIO
from urllib.parse import quote, unquote
@ -31,6 +31,11 @@ from frappe.utils import call_hook_method, cint, cstr, encode, get_files_path, g
from frappe.utils.image import strip_exif_data, optimize_image
from frappe.utils.file_manager import safe_b64decode
if TYPE_CHECKING:
from PIL.ImageFile import ImageFile
from requests.models import Response
class MaxFileSizeReachedError(frappe.ValidationError):
pass
@ -276,7 +281,7 @@ class File(Document):
image, filename, extn = get_local_image(self.file_url)
else:
image, filename, extn = get_web_image(self.file_url)
except (requests.exceptions.HTTPError, requests.exceptions.SSLError, IOError, TypeError):
except (HTTPError, SSLError, IOError, TypeError):
return
size = width, height
@ -648,9 +653,17 @@ def setup_folder_path(filename, new_parent):
from frappe.model.rename_doc import rename_doc
rename_doc("File", file.name, file.get_name_based_on_parent_folder(), ignore_permissions=True)
def get_extension(filename, extn, content):
def get_extension(filename, extn, content: bytes = None, response: "Response" = None) -> str:
mimetype = None
if response:
content_type = response.headers.get("Content-Type")
if content_type:
_extn = mimetypes.guess_extension(content_type)
if _extn:
return _extn[1:]
if extn:
# remove '?' char and parameters from extn if present
if '?' in extn:
@ -693,14 +706,14 @@ def get_local_image(file_url):
return image, filename, extn
def get_web_image(file_url):
def get_web_image(file_url: str) -> Tuple["ImageFile", str, str]:
# download
file_url = frappe.utils.get_url(file_url)
r = requests.get(file_url, stream=True)
try:
r.raise_for_status()
except requests.exceptions.HTTPError as e:
if "404" in e.args[0]:
except HTTPError:
if r.status_code == 404:
frappe.msgprint(_("File '{0}' not found").format(file_url))
else:
frappe.msgprint(_("Unable to read file format for {0}").format(file_url))
@ -719,7 +732,10 @@ def get_web_image(file_url):
filename = get_random_filename()
extn = None
extn = get_extension(filename, extn, r.content)
extn = get_extension(filename, extn, response=r)
if extn == "bin":
extn = get_extension(filename, extn, content=r.content) or "png"
filename = "/files/" + strip(unquote(filename))
return image, filename, extn
@ -862,8 +878,9 @@ def extract_images_from_html(doc, content, is_private=False):
else:
filename = get_random_filename(content_type=mtype)
doctype = doc.parenttype if doc.parent else doc.doctype
name = doc.parent or doc.name
# attaching a file to a child table doc, attaches it to the parent doc
doctype = doc.parenttype if doc.get("parent") else doc.doctype
name = doc.get("parent") or doc.name
_file = frappe.get_doc({
"doctype": "File",

41
frappe/core/doctype/file/test_file.py
View file

@ -1,15 +1,14 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors
# License: MIT. See LICENSE
import base64
import json
import frappe
import os
import unittest
from frappe import _
from frappe.core.doctype.file.file import get_attached_images, move_file, get_files_in_folder, unzip_file
from frappe.core.doctype.file.file import File, get_attached_images, move_file, get_files_in_folder, unzip_file
from frappe.utils import get_files_path
# test_records = frappe.get_test_records('File')
test_content1 = 'Hello'
test_content2 = 'Hello World'
@ -24,8 +23,6 @@ def make_test_doc():
class TestSimpleFile(unittest.TestCase):
def setUp(self):
self.attached_to_doctype, self.attached_to_docname = make_test_doc()
self.test_content = test_content1
@ -38,21 +35,13 @@ class TestSimpleFile(unittest.TestCase):
_file.save()
self.saved_file_url = _file.file_url
def test_save(self):
_file = frappe.get_doc("File", {"file_url": self.saved_file_url})
content = _file.get_content()
self.assertEqual(content, self.test_content)
def tearDown(self):
# File gets deleted on rollback, so blank
pass
class TestBase64File(unittest.TestCase):
def setUp(self):
self.attached_to_doctype, self.attached_to_docname = make_test_doc()
self.test_content = base64.b64encode(test_content1.encode('utf-8'))
@ -66,18 +55,12 @@ class TestBase64File(unittest.TestCase):
_file.save()
self.saved_file_url = _file.file_url
def test_saved_content(self):
_file = frappe.get_doc("File", {"file_url": self.saved_file_url})
content = _file.get_content()
self.assertEqual(content, test_content1)
def tearDown(self):
# File gets deleted on rollback, so blank
pass
class TestSameFileName(unittest.TestCase):
def test_saved_content(self):
self.attached_to_doctype, self.attached_to_docname = make_test_doc()
@ -130,8 +113,6 @@ class TestSameFileName(unittest.TestCase):
class TestSameContent(unittest.TestCase):
def setUp(self):
self.attached_to_doctype1, self.attached_to_docname1 = make_test_doc()
self.attached_to_doctype2, self.attached_to_docname2 = make_test_doc()
@ -186,10 +167,6 @@ class TestSameContent(unittest.TestCase):
limit_property.delete()
frappe.clear_cache(doctype='ToDo')
def tearDown(self):
# File gets deleted on rollback, so blank
pass
class TestFile(unittest.TestCase):
def setUp(self):
@ -398,7 +375,7 @@ class TestFile(unittest.TestCase):
def test_make_thumbnail(self):
# test web image
test_file = frappe.get_doc({
test_file: File = frappe.get_doc({
"doctype": "File",
"file_name": 'logo',
"file_url": frappe.utils.get_url('/_test/assets/image.jpg'),
@ -407,6 +384,16 @@ class TestFile(unittest.TestCase):
test_file.make_thumbnail()
self.assertEquals(test_file.thumbnail_url, '/files/image_small.jpg')
# test web image without extension
test_file = frappe.get_doc({
"doctype": "File",
"file_name": 'logo',
"file_url": frappe.utils.get_url('/_test/assets/image'),
}).insert(ignore_permissions=True)
test_file.make_thumbnail()
self.assertTrue(test_file.thumbnail_url.endswith("_small.jpeg"))
# test local image
test_file.db_set('thumbnail_url', None)
test_file.reload()

2
frappe/core/doctype/report/report.py
View file

@ -61,7 +61,7 @@ class Report(Document):
delete_permanently=True)
def get_columns(self):
return [d.as_dict(no_default_fields = True) for d in self.columns]
return [d.as_dict(no_default_fields=True, no_child_table_fields=True) for d in self.columns]
@frappe.whitelist()
def set_doctype_roles(self):

39
frappe/core/doctype/server_script/test_server_script.py
View file

@ -139,3 +139,42 @@ class TestServerScript(unittest.TestCase):
server_script.disabled = 1
server_script.save()
def test_restricted_qb(self):
todo = frappe.get_doc(doctype="ToDo", description="QbScriptTestNote")
todo.insert()
script = frappe.get_doc(
doctype='Server Script',
name='test_qb_restrictions',
script_type = 'API',
api_method = 'test_qb_restrictions',
allow_guest = 1,
# whitelisted update
script = f'''
frappe.db.set_value("ToDo", "{todo.name}", "description", "safe")
'''
)
script.insert()
script.execute_method()
todo.reload()
self.assertEqual(todo.description, "safe")
# unsafe update
script.script = f"""
todo = frappe.qb.DocType("ToDo")
frappe.qb.update(todo).set(todo.description, "unsafe").where(todo.name == "{todo.name}").run()
"""
script.save()
self.assertRaises(frappe.PermissionError, script.execute_method)
todo.reload()
self.assertEqual(todo.description, "safe")
# safe select
script.script = f"""
todo = frappe.qb.DocType("ToDo")
frappe.qb.from_(todo).select(todo.name).where(todo.name == "{todo.name}").run()
"""
script.save()
script.execute_method()

13
frappe/core/doctype/user_permission/test_user_permission.py
View file

@ -3,6 +3,7 @@
from frappe.core.doctype.user_permission.user_permission import add_user_permissions, remove_applicable
from frappe.permissions import has_user_permission
from frappe.core.doctype.doctype.test_doctype import new_doctype
from frappe.website.doctype.blog_post.test_blog_post import make_test_blog
import frappe
import unittest
@ -31,6 +32,18 @@ class TestUserPermission(unittest.TestCase):
param = get_params(user, 'User', perm_user.name, is_default=1)
self.assertRaises(frappe.ValidationError, add_user_permissions, param)
def test_default_user_permission_corectness(self):
user = create_user('test_default_corectness_permission_1@example.com')
param = get_params(user, 'User', user.name, is_default=1, hide_descendants= 1)
add_user_permissions(param)
#create a duplicate entry with default
perm_user = create_user('test_default_corectness2@example.com')
test_blog = make_test_blog()
param = get_params(perm_user, 'Blog Post', test_blog.name, is_default=1, hide_descendants= 1)
add_user_permissions(param)
frappe.db.delete('User Permission', filters={'for_value': test_blog.name})
frappe.delete_doc('Blog Post', test_blog.name)
def test_default_user_permission(self):
frappe.set_user('Administrator')
user = create_user('test_user_perm1@example.com', 'Website Manager')

1
frappe/core/doctype/user_permission/user_permission.py
View file

@ -48,7 +48,6 @@ class UserPermission(Document):
}, or_filters={
'applicable_for': cstr(self.applicable_for),
'apply_to_all_doctypes': 1,
'hide_descendants': cstr(self.hide_descendants)
}, limit=1)
if overlap_exists:
ref_link = frappe.get_desk_link(self.doctype, overlap_exists[0].name)

64
frappe/core/doctype/user_type/test_user_type.py
View file

@ -1,8 +1,68 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2021, Frappe Technologies and Contributors
# License: MIT. See LICENSE
# import frappe
import frappe
import unittest
from frappe.installer import update_site_config
class TestUserType(unittest.TestCase):
pass
def setUp(self):
create_role()
def test_add_select_perm_doctypes(self):
user_type = create_user_type('Test User Type')
# select perms added for all link fields
doc = frappe.get_meta('Contact')
link_fields = doc.get_link_fields()
select_doctypes = frappe.get_all('User Select Document Type', {'parent': user_type.name}, pluck='document_type')
for entry in link_fields:
self.assertTrue(entry.options in select_doctypes)
# select perms added for all child table link fields
link_fields = []
for child_table in doc.get_table_fields():
child_doc = frappe.get_meta(child_table.options)
link_fields.extend(child_doc.get_link_fields())
for entry in link_fields:
self.assertTrue(entry.options in select_doctypes)
def tearDown(self):
frappe.db.rollback()
def create_user_type(user_type):
if frappe.db.exists('User Type', user_type):
frappe.delete_doc('User Type', user_type)
user_type_limit = {frappe.scrub(user_type): 1}
update_site_config('user_type_doctype_limit', user_type_limit)
doc = frappe.get_doc({
'doctype': 'User Type',
'name': user_type,
'role': '_Test User Type',
'user_id_field': 'user',
'apply_user_permission_on': 'User'
})
doc.append('user_doctypes', {
'document_type': 'Contact',
'read': 1,
'write': 1
})
return doc.insert()
def create_role():
if not frappe.db.exists('Role', '_Test User Type'):
frappe.get_doc({
'doctype': 'Role',
'role_name': '_Test User Type',
'desk_access': 1,
'is_custom': 1
}).insert()

2
frappe/core/doctype/user_type/user_type.py
View file

@ -193,7 +193,7 @@ def get_user_linked_doctypes(doctype, txt, searchfield, start, page_len, filters
['DocType', 'read_only', '=', 0], ['DocType', 'name', 'like', '%{0}%'.format(txt)]]
doctypes = frappe.get_all('DocType', fields = ['`tabDocType`.`name`'], filters=filters,
order_by = '`tabDocType`.`idx` desc', limit_start=start, limit_page_length=page_len, as_list=1)
order_by='`tabDocType`.`idx` desc', limit_start=start, limit_page_length=page_len, as_list=1)
custom_dt_filters = [['Custom Field', 'dt', 'like', '%{0}%'.format(txt)],
['Custom Field', 'options', '=', 'User'], ['Custom Field', 'fieldtype', '=', 'Link']]

40
frappe/core/notifications.py
View file

@ -39,43 +39,3 @@ def get_todays_events(as_list=False):
today = nowdate()
events = get_events(today, today)
return events if as_list else len(events)
def get_unseen_likes():
"""Returns count of unseen likes"""
comment_doctype = DocType("Comment")
return frappe.db.count(comment_doctype,
filters=(
(comment_doctype.comment_type == "Like")
& (comment_doctype.modified >= Now() - Interval(years=1))
& (comment_doctype.owner.notnull())
& (comment_doctype.owner != frappe.session.user)
& (comment_doctype.reference_owner == frappe.session.user)
& (comment_doctype.seen == 0)
)
)
def get_unread_emails():
"returns count of unread emails for a user"
communication_doctype = DocType("Communication")
user_doctype = DocType("User")
distinct_email_accounts = (
frappe.qb.from_(user_doctype)
.select(user_doctype.email_account)
.where(user_doctype.parent == frappe.session.user)
.distinct()
)
return frappe.db.count(communication_doctype,
filters=(
(communication_doctype.communication_type == "Communication")
& (communication_doctype.communication_medium == "Email")
& (communication_doctype.sent_or_received == "Received")
& (communication_doctype.email_status.notin(["spam", "Trash"]))
& (communication_doctype.email_account.isin(distinct_email_accounts))
& (communication_doctype.modified >= Now() - Interval(years=1))
& (communication_doctype.seen == 0)
)
)

5
frappe/core/page/dashboard_view/dashboard_view.js
View file

@ -30,6 +30,7 @@ class Dashboard {
show() {
this.route = frappe.get_route();
this.set_breadcrumbs();
if (this.route.length > 1) {
// from route
this.show_dashboard(this.route.slice(-1)[0]);
@ -75,6 +76,10 @@ class Dashboard {
frappe.last_dashboard = current_dashboard_name;
}
set_breadcrumbs() {
frappe.breadcrumbs.add("Desk", "Dashboard");
}
refresh() {
frappe.run_serially([
() => this.render_cards(),

113
frappe/database/database.py
View file

@ -10,19 +10,20 @@ import re
import string
from contextlib import contextmanager
from time import time
from typing import Dict, List, Union, Tuple
from typing import Dict, List, Tuple, Union
from pypika.terms import Criterion, NullValue, PseudoColumn
import frappe
import frappe.defaults
import frappe.model.meta
from frappe import _
from frappe.utils import now, getdate, cast, get_datetime
from frappe.model.utils.link_count import flush_local_link_count
from frappe.query_builder.functions import Count
from frappe.query_builder.functions import Min, Max, Avg, Sum
from frappe.query_builder.utils import Column
from frappe.query_builder.utils import DocType
from frappe.utils import cast, get_datetime, getdate, now, sbool
from .query import Query
from pypika.terms import Criterion, PseudoColumn
class Database(object):
@ -36,9 +37,9 @@ class Database(object):
OPTIONAL_COLUMNS = ["_user_tags", "_comments", "_assign", "_liked_by"]
DEFAULT_SHORTCUTS = ['_Login', '__user', '_Full Name', 'Today', '__today', "now", "Now"]
STANDARD_VARCHAR_COLUMNS = ('name', 'owner', 'modified_by', 'parent', 'parentfield', 'parenttype')
DEFAULT_COLUMNS = ['name', 'creation', 'modified', 'modified_by', 'owner', 'docstatus', 'parent',
'parentfield', 'parenttype', 'idx']
STANDARD_VARCHAR_COLUMNS = ('name', 'owner', 'modified_by')
DEFAULT_COLUMNS = ['name', 'creation', 'modified', 'modified_by', 'owner', 'docstatus', 'idx']
CHILD_TABLE_COLUMNS = ('parent', 'parenttype', 'parentfield')
MAX_WRITES_PER_TRANSACTION = 200_000
class InvalidColumnName(frappe.ValidationError): pass
@ -278,7 +279,9 @@ class Database(object):
if self.auto_commit_on_many_writes:
self.commit()
else:
frappe.throw(_("Too many writes in one request. Please send smaller requests"), frappe.ValidationError)
msg = "<br><br>" + _("Too many changes to database in single action.") + "<br>"
msg += _("The changes have been reverted.") + "<br>"
raise frappe.TooManyWritesError(msg)
def check_implicit_commit(self, query):
if self.transaction_writes and \
@ -432,11 +435,9 @@ class Database(object):
else:
fields = fieldname
if fieldname!="*":
if fieldname != "*":
if isinstance(fieldname, str):
fields = [fieldname]
else:
fields = fieldname
if (filters is not None) and (filters!=doctype or doctype=="DocType"):
try:
@ -555,7 +556,21 @@ class Database(object):
def get_list(*args, **kwargs):
return frappe.get_list(*args, **kwargs)
def get_single_value(self, doctype, fieldname, cache=False):
def set_single_value(self, doctype, fieldname, value, *args, **kwargs):
"""Set field value of Single DocType.
:param doctype: DocType of the single object
:param fieldname: `fieldname` of the property
:param value: `value` of the property
Example:
# Update the `deny_multiple_sessions` field in System Settings DocType.
company = frappe.db.set_single_value("System Settings", "deny_multiple_sessions", True)
"""
return self.set_value(doctype, doctype, fieldname, value, *args, **kwargs)
def get_single_value(self, doctype, fieldname, cache=True):
"""Get property of Single DocType. Cache locally by default
:param doctype: DocType of the single object whose value is requested
@ -570,7 +585,7 @@ class Database(object):
if not doctype in self.value_cache:
self.value_cache[doctype] = {}
if fieldname in self.value_cache[doctype]:
if cache and fieldname in self.value_cache[doctype]:
return self.value_cache[doctype][fieldname]
val = self.query.get_sql(
@ -677,53 +692,55 @@ class Database(object):
:param debug: Print the query in the developer / js console.
:param for_update: Will add a row-level lock to the value that is being set so that it can be released on commit.
"""
if not modified:
modified = now()
if not modified_by:
modified_by = frappe.session.user
is_single_doctype = not (dn and dt != dn)
to_update = field if isinstance(field, dict) else {field: val}
to_update = {}
if update_modified:
to_update = {"modified": modified, "modified_by": modified_by}
modified = modified or now()
modified_by = modified_by or frappe.session.user
to_update.update({"modified": modified, "modified_by": modified_by})
if is_single_doctype:
frappe.db.delete(
"Singles",
filters={"field": ("in", tuple(to_update)), "doctype": dt}, debug=debug
)
singles_data = ((dt, key, sbool(value)) for key, value in to_update.items())
query = (
frappe.qb.into("Singles")
.columns("doctype", "field", "value")
.insert(*singles_data)
).run(debug=debug)
frappe.clear_document_cache(dt, dt)
if isinstance(field, dict):
to_update.update(field)
else:
to_update.update({field: val})
table = DocType(dt)
if dn and dt!=dn:
# with table
set_values = []
for key in to_update:
set_values.append('`{0}`=%({0})s'.format(key))
if for_update:
docnames = tuple(
self.get_values(dt, dn, "name", debug=debug, for_update=for_update, pluck=True)
) or (NullValue(),)
query = frappe.qb.update(table).where(table.name.isin(docnames))
for name in self.get_values(dt, dn, 'name', for_update=for_update, debug=debug):
values = dict(name=name[0])
values.update(to_update)
for docname in docnames:
frappe.clear_document_cache(dt, docname)
self.sql("""update `tab{0}`
set {1} where name=%(name)s""".format(dt, ', '.join(set_values)),
values, debug=debug)
else:
query = self.query.build_conditions(table=dt, filters=dn, update=True)
# TODO: Fix this; doesn't work rn - gavin@frappe.io
# frappe.cache().hdel_keys(dt, "document_cache")
# Workaround: clear all document caches
frappe.cache().delete_value('document_cache')
frappe.clear_document_cache(dt, values['name'])
else:
# for singles
keys = list(to_update)
self.sql('''
delete from `tabSingles`
where field in ({0}) and
doctype=%s'''.format(', '.join(['%s']*len(keys))),
list(keys) + [dt], debug=debug)
for key, value in to_update.items():
self.sql('''insert into `tabSingles` (doctype, field, value) values (%s, %s, %s)''',
(dt, key, value), debug=debug)
for column, value in to_update.items():
query = query.set(column, value)
frappe.clear_document_cache(dt, dn)
query.run(debug=debug)
if dt in self.value_cache:
del self.value_cache[dt]
@staticmethod
def set(doc, field, val):
"""Set value in document. **Avoid**"""

6
frappe/database/mariadb/framework_mariadb.sql
View file

@ -171,9 +171,6 @@ CREATE TABLE `tabDocType` (
`modified_by` varchar(255) DEFAULT NULL,
`owner` varchar(255) DEFAULT NULL,
`docstatus` int(1) NOT NULL DEFAULT 0,
`parent` varchar(255) DEFAULT NULL,
`parentfield` varchar(255) DEFAULT NULL,
`parenttype` varchar(255) DEFAULT NULL,
`idx` int(8) NOT NULL DEFAULT 0,
`search_fields` varchar(255) DEFAULT NULL,
`issingle` int(1) NOT NULL DEFAULT 0,
@ -228,8 +225,7 @@ CREATE TABLE `tabDocType` (
`subject_field` varchar(255) DEFAULT NULL,
`sender_field` varchar(255) DEFAULT NULL,
`migration_hash` varchar(255) DEFAULT NULL,
PRIMARY KEY (`name`),
KEY `parent` (`parent`)
PRIMARY KEY (`name`)
) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
--

15
frappe/database/mariadb/schema.py
View file

@ -18,6 +18,17 @@ class MariaDBTable(DBTable):
if index_defs:
additional_definitions += ',\n'.join(index_defs) + ',\n'
# child table columns
if self.meta.get("istable") or 0:
additional_definitions += ',\n'.join(
(
f"parent varchar({varchar_len})",
f"parentfield varchar({varchar_len})",
f"parenttype varchar({varchar_len})",
"index parent(parent)"
)
) + ',\n'
# create table
query = f"""create table `{self.table_name}` (
name varchar({varchar_len}) not null primary key,
@ -26,12 +37,8 @@ class MariaDBTable(DBTable):
modified_by varchar({varchar_len}),
owner varchar({varchar_len}),
docstatus int(1) not null default '0',
parent varchar({varchar_len}),
parentfield varchar({varchar_len}),
parenttype varchar({varchar_len}),
idx int(8) not null default '0',
{additional_definitions}
index parent(parent),
index modified(modified))
ENGINE={engine}
ROW_FORMAT=DYNAMIC

4
frappe/database/postgres/database.py
View file

@ -170,11 +170,11 @@ class PostgresDatabase(Database):
@staticmethod
def is_primary_key_violation(e):
return e.pgcode == '23505' and '_pkey' in cstr(e.args[0])
return getattr(e, "pgcode", None) == '23505' and '_pkey' in cstr(e.args[0])
@staticmethod
def is_unique_key_violation(e):
return e.pgcode == '23505' and '_key' in cstr(e.args[0])
return getattr(e, "pgcode", None) == '23505' and '_key' in cstr(e.args[0])
@staticmethod
def is_duplicate_fieldname(e):

3
frappe/database/postgres/framework_postgres.sql
View file

@ -176,9 +176,6 @@ CREATE TABLE "tabDocType" (
"modified_by" varchar(255) DEFAULT NULL,
"owner" varchar(255) DEFAULT NULL,
"docstatus" smallint NOT NULL DEFAULT 0,
"parent" varchar(255) DEFAULT NULL,
"parentfield" varchar(255) DEFAULT NULL,
"parenttype" varchar(255) DEFAULT NULL,
"idx" bigint NOT NULL DEFAULT 0,
"search_fields" varchar(255) DEFAULT NULL,
"issingle" smallint NOT NULL DEFAULT 0,

25
frappe/database/postgres/schema.py
View file

@ -5,26 +5,37 @@ from frappe.database.schema import DBTable, get_definition
class PostgresTable(DBTable):
def create(self):
add_text = ''
add_text = ""
# columns
column_defs = self.get_column_definitions()
if column_defs: add_text += ',\n'.join(column_defs)
if column_defs:
add_text += ",\n".join(column_defs)
# child table columns
if self.meta.get("istable") or 0:
if column_defs:
add_text += ",\n"
add_text += ",\n".join(
(
"parent varchar({varchar_len})",
"parentfield varchar({varchar_len})",
"parenttype varchar({varchar_len})"
)
)
# TODO: set docstatus length
# create table
frappe.db.sql("""create table `%s` (
frappe.db.sql(("""create table `%s` (
name varchar({varchar_len}) not null primary key,
creation timestamp(6),
modified timestamp(6),
modified_by varchar({varchar_len}),
owner varchar({varchar_len}),
docstatus smallint not null default '0',
parent varchar({varchar_len}),
parentfield varchar({varchar_len}),
parenttype varchar({varchar_len}),
idx bigint not null default '0',
%s)""".format(varchar_len=frappe.db.VARCHAR_LEN) % (self.table_name, add_text))
%s)""" % (self.table_name, add_text)).format(varchar_len=frappe.db.VARCHAR_LEN))
self.create_indexes()
frappe.db.commit()

47
frappe/database/schema.py
View file

@ -106,6 +106,9 @@ class DBTable:
columns = [frappe._dict({"fieldname": f, "fieldtype": "Data"}) for f in
frappe.db.STANDARD_VARCHAR_COLUMNS]
if self.meta.get("istable"):
columns += [frappe._dict({"fieldname": f, "fieldtype": "Data"}) for f in
frappe.db.CHILD_TABLE_COLUMNS]
columns += self.columns.values()
for col in columns:
@ -300,11 +303,12 @@ def validate_column_length(fieldname):
def get_definition(fieldtype, precision=None, length=None):
d = frappe.db.type_map.get(fieldtype)
# convert int to long int if the length of the int is greater than 11
if fieldtype == "Int" and length and length > 11:
d = frappe.db.type_map.get("Long Int")
if not d:
return
if not d: return
if fieldtype == "Int" and length and length > 11:
# convert int to long int if the length of the int is greater than 11
d = frappe.db.type_map.get("Long Int")
coltype = d[0]
size = d[1] if d[1] else None
@ -315,19 +319,44 @@ def get_definition(fieldtype, precision=None, length=None):
if fieldtype in ["Float", "Currency", "Percent"] and cint(precision) > 6:
size = '21,9'
if coltype == "varchar" and length:
size = length
if length:
if coltype == "varchar":
size = length
elif coltype == "int" and length < 11:
# allow setting custom length for int if length provided is less than 11
# NOTE: this will only be applicable for mariadb as frappe implements int
# in postgres as bigint (as seen in type_map)
size = length
if size is not None:
coltype = "{coltype}({size})".format(coltype=coltype, size=size)
return coltype
def add_column(doctype, column_name, fieldtype, precision=None):
def add_column(
doctype,
column_name,
fieldtype,
precision=None,
length=None,
default=None,
not_null=False
):
if column_name in frappe.db.get_table_columns(doctype):
# already exists
return
frappe.db.commit()
frappe.db.sql("alter table `tab%s` add column %s %s" % (doctype,
column_name, get_definition(fieldtype, precision)))
query = "alter table `tab%s` add column %s %s" % (
doctype,
column_name,
get_definition(fieldtype, precision, length)
)
if not_null:
query += " not null"
if default:
query += f" default '{default}'"
frappe.db.sql(query)

6
frappe/desk/doctype/bulk_update/bulk_update.py
View file

@ -42,13 +42,13 @@ def submit_cancel_or_update_docs(doctype, docnames, action='submit', data=None):
doc = frappe.get_doc(doctype, d)
try:
message = ''
if action == 'submit' and doc.docstatus==0:
if action == 'submit' and doc.docstatus.is_draft():
doc.submit()
message = _('Submiting {0}').format(doctype)
elif action == 'cancel' and doc.docstatus==1:
elif action == 'cancel' and doc.docstatus.is_submitted():
doc.cancel()
message = _('Cancelling {0}').format(doctype)
elif action == 'update' and doc.docstatus < 2:
elif action == 'update' and not doc.docstatus.is_cancelled():
doc.update(data)
doc.save()
message = _('Updating {0}').format(doctype)

2
frappe/desk/doctype/tag/tag.py
View file

@ -148,8 +148,6 @@ def update_tags(doc, tags):
"doctype": "Tag Link",
"document_type": doc.doctype,
"document_name": doc.name,
"parenttype": doc.doctype,
"parent": doc.name,
"title": doc.get_title() or '',
"tag": tag
}).insert(ignore_permissions=True)

48
frappe/desk/form/linked_with.py
View file

@ -389,8 +389,6 @@ def get_linked_docs(doctype, name, linkinfo=None, for_doctype=None):
else:
return results
me = frappe.db.get_value(doctype, name, ["parenttype", "parent"], as_dict=True)
for dt, link in linkinfo.items():
filters = []
link["doctype"] = dt
@ -413,11 +411,16 @@ def get_linked_docs(doctype, name, linkinfo=None, for_doctype=None):
ret = frappe.get_all(doctype=dt, fields=fields, filters=link.get("filters"))
elif link.get("get_parent"):
if me and me.parent and me.parenttype == dt:
ret = None
# check for child table
if not frappe.get_meta(doctype).istable:
continue
me = frappe.db.get_value(doctype, name, ["parenttype", "parent"], as_dict=True)
if me and me.parenttype == dt:
ret = frappe.get_all(doctype=dt, fields=fields,
filters=[[dt, "name", '=', me.parent]])
else:
ret = None
elif link.get("child_doctype"):
or_filters = [[link.get('child_doctype'), link_fieldnames, '=', name] for link_fieldnames in link.get("fieldname")]
@ -473,7 +476,7 @@ def _get_linked_doctypes(doctype, without_ignore_user_permissions_enabled=False)
ret.update(get_linked_fields(doctype, without_ignore_user_permissions_enabled))
ret.update(get_dynamic_linked_fields(doctype, without_ignore_user_permissions_enabled))
filters=[['fieldtype', 'in', frappe.model.table_fields], ['options', '=', doctype]]
filters = [['fieldtype', 'in', frappe.model.table_fields], ['options', '=', doctype]]
if without_ignore_user_permissions_enabled: filters.append(['ignore_user_permissions', '!=', 1])
# find links of parents
links = frappe.get_all("DocField", fields=["parent as dt"], filters=filters)
@ -498,12 +501,12 @@ def _get_linked_doctypes(doctype, without_ignore_user_permissions_enabled=False)
def get_linked_fields(doctype, without_ignore_user_permissions_enabled=False):
filters=[['fieldtype','=', 'Link'], ['options', '=', doctype]]
filters = [['fieldtype','=', 'Link'], ['options', '=', doctype]]
if without_ignore_user_permissions_enabled: filters.append(['ignore_user_permissions', '!=', 1])
# find links of parents
links = frappe.get_all("DocField", fields=["parent", "fieldname"], filters=filters, as_list=1)
links+= frappe.get_all("Custom Field", fields=["dt as parent", "fieldname"], filters=filters, as_list=1)
links += frappe.get_all("Custom Field", fields=["dt as parent", "fieldname"], filters=filters, as_list=1)
ret = {}
@ -529,34 +532,37 @@ def get_linked_fields(doctype, without_ignore_user_permissions_enabled=False):
def get_dynamic_linked_fields(doctype, without_ignore_user_permissions_enabled=False):
ret = {}
filters=[['fieldtype','=', 'Dynamic Link']]
filters = [['fieldtype','=', 'Dynamic Link']]
if without_ignore_user_permissions_enabled: filters.append(['ignore_user_permissions', '!=', 1])
# find dynamic links of parents
links = frappe.get_all("DocField", fields=["parent as doctype", "fieldname", "options as doctype_fieldname"], filters=filters)
links+= frappe.get_all("Custom Field", fields=["dt as doctype", "fieldname", "options as doctype_fieldname"], filters=filters)
links += frappe.get_all("Custom Field", fields=["dt as doctype", "fieldname", "options as doctype_fieldname"], filters=filters)
for df in links:
if is_single(df.doctype): continue
# optimized to get both link exists and parenttype
possible_link = frappe.get_all(df.doctype, filters={df.doctype_fieldname: doctype},
fields=['parenttype'], distinct=True)
is_child = frappe.get_meta(df.doctype).istable
possible_link = frappe.get_all(
df.doctype,
filters={df.doctype_fieldname: doctype},
fields=["parenttype"] if is_child else None,
distinct=True
)
if not possible_link: continue
for d in possible_link:
# is child
if d.parenttype:
if is_child:
for d in possible_link:
ret[d.parenttype] = {
"child_doctype": df.doctype,
"fieldname": [df.fieldname],
"doctype_fieldname": df.doctype_fieldname
}
else:
ret[df.doctype] = {
"fieldname": [df.fieldname],
"doctype_fieldname": df.doctype_fieldname
}
else:
ret[df.doctype] = {
"fieldname": [df.fieldname],
"doctype_fieldname": df.doctype_fieldname
}
return ret

5
frappe/desk/form/load.py
View file

@ -91,8 +91,8 @@ def get_docinfo(doc=None, doctype=None, name=None):
raise frappe.PermissionError
all_communications = _get_communications(doc.doctype, doc.name)
automated_messages = filter(lambda x: x['communication_type'] == 'Automated Message', all_communications)
communications_except_auto_messages = filter(lambda x: x['communication_type'] != 'Automated Message', all_communications)
automated_messages = [msg for msg in all_communications if msg['communication_type'] == 'Automated Message']
communications_except_auto_messages = [msg for msg in all_communications if msg['communication_type'] != 'Automated Message']
docinfo = frappe._dict(user_info = {})
@ -119,6 +119,7 @@ def get_docinfo(doc=None, doctype=None, name=None):
update_user_info(docinfo)
frappe.response["docinfo"] = docinfo
return docinfo
def add_comments(doc, docinfo):
# divide comments into separate lists

23
frappe/desk/reportview.py
View file

@ -6,7 +6,7 @@
import frappe, json
import frappe.permissions
from frappe.model.db_query import DatabaseQuery
from frappe.model import default_fields, optional_fields
from frappe.model import default_fields, optional_fields, child_table_fields
from frappe import _
from io import StringIO
from frappe.core.doctype.access_log.access_log import make_access_log
@ -156,7 +156,7 @@ def raise_invalid_field(fieldname):
def is_standard(fieldname):
if '.' in fieldname:
parenttype, fieldname = get_parenttype_and_fieldname(fieldname, None)
return fieldname in default_fields or fieldname in optional_fields
return fieldname in default_fields or fieldname in optional_fields or fieldname in child_table_fields
def extract_fieldname(field):
for text in (',', '/*', '#'):
@ -319,7 +319,7 @@ def export_query():
if add_totals_row:
ret = append_totals_row(ret)
data = [['Sr'] + get_labels(db_query.fields, doctype)]
data = [[_('Sr')] + get_labels(db_query.fields, doctype)]
for i, row in enumerate(ret):
data.append([i+1] + list(row))
@ -378,7 +378,8 @@ def get_labels(fields, doctype):
for key in fields:
key = key.split(" as ")[0]
if key.startswith(('count(', 'sum(', 'avg(')): continue
if key.startswith(('count(', 'sum(', 'avg(')):
continue
if "." in key:
parenttype, fieldname = key.split(".")[0][4:-1], key.split(".")[1].strip("`")
@ -386,10 +387,16 @@ def get_labels(fields, doctype):
parenttype = doctype
fieldname = fieldname.strip("`")
df = frappe.get_meta(parenttype).get_field(fieldname)
label = df.label if df else fieldname.title()
if label in labels:
label = doctype + ": " + label
if parenttype == doctype and fieldname == "name":
label = _("ID", context="Label of name column in report")
else:
df = frappe.get_meta(parenttype).get_field(fieldname)
label = _(df.label if df else fieldname.title())
if parenttype != doctype:
# If the column is from a child table, append the child doctype.
# For example, "Item Code (Sales Invoice Item)".
label += f" ({ _(parenttype) })"
labels.append(label)
return labels

2
frappe/email/doctype/auto_email_report/auto_email_report.py
View file

@ -252,7 +252,7 @@ def make_links(columns, data):
if col.options and row.get(col.options):
row[col.fieldname] = get_link_to_form(row[col.options], row[col.fieldname])
elif col.fieldtype == "Currency":
doc = frappe.get_doc(col.parent, doc_name) if doc_name and col.parent else None
doc = frappe.get_doc(col.parent, doc_name) if doc_name and col.get("parent") else None
# Pass the Document to get the currency based on docfield option
row[col.fieldname] = frappe.format_value(row[col.fieldname], col, doc=doc)
return columns, data

26
frappe/email/doctype/email_account/email_account.py
View file

@ -421,10 +421,10 @@ class EmailAccount(Document):
def get_failed_attempts_count(self):
return cint(frappe.cache().get('{0}:email-account-failed-attempts'.format(self.name)))
def receive(self, test_mails=None):
def receive(self):
"""Called by scheduler to receive emails from this EMail account using POP3/IMAP."""
exceptions = []
inbound_mails = self.get_inbound_mails(test_mails=test_mails)
inbound_mails = self.get_inbound_mails()
for mail in inbound_mails:
try:
communication = mail.process()
@ -457,20 +457,19 @@ class EmailAccount(Document):
if exceptions:
raise Exception(frappe.as_json(exceptions))
def get_inbound_mails(self, test_mails=None) -> List[InboundMail]:
def get_inbound_mails(self) -> List[InboundMail]:
"""retrive and return inbound mails.
"""
mails = []
def process_mail(messages):
def process_mail(messages, append_to=None):
for index, message in enumerate(messages.get("latest_messages", [])):
uid = messages['uid_list'][index] if messages.get('uid_list') else None
seen_status = 1 if messages.get('seen_status', {}).get(uid) == 'SEEN' else 0
mails.append(InboundMail(message, self, uid, seen_status))
if frappe.local.flags.in_test:
return [InboundMail(msg, self) for msg in test_mails or []]
seen_status = messages.get('seen_status', {}).get(uid)
if self.email_sync_option != 'UNSEEN' or seen_status != "SEEN":
# only append the emails with status != 'SEEN' if sync option is set to 'UNSEEN'
mails.append(InboundMail(message, self, uid, seen_status, append_to))
if not self.enable_incoming:
return []
@ -481,10 +480,10 @@ class EmailAccount(Document):
if self.use_imap:
# process all given imap folder
for folder in self.imap_folder:
email_server.select_imap_folder(folder.folder_name)
email_server.settings['uid_validity'] = folder.uidvalidity
messages = email_server.get_messages(folder=folder.folder_name) or {}
process_mail(messages)
if email_server.select_imap_folder(folder.folder_name):
email_server.settings['uid_validity'] = folder.uidvalidity
messages = email_server.get_messages(folder=f'"{folder.folder_name}"') or {}
process_mail(messages, folder.append_to)
else:
# process the pop3 account
messages = email_server.get_messages() or {}
@ -494,7 +493,6 @@ class EmailAccount(Document):
except Exception:
frappe.log_error(title=_("Error while connecting to email account {0}").format(self.name))
return []
return mails
def handle_bad_emails(self, uid, raw, reason):

218
frappe/email/doctype/email_account/test_email_account.py
View file

@ -14,11 +14,11 @@ from frappe.core.doctype.communication.email import make
from frappe.desk.form.load import get_attachments
from frappe.email.doctype.email_account.email_account import notify_unreplied
from unittest.mock import patch
make_test_records("User")
make_test_records("Email Account")
class TestEmailAccount(unittest.TestCase):
@classmethod
def setUpClass(cls):
@ -45,10 +45,21 @@ class TestEmailAccount(unittest.TestCase):
def test_incoming(self):
cleanup("test_sender@example.com")
test_mails = [self.get_test_mail('incoming-1.raw')]
messages = {
# append_to = ToDo
'"INBOX"': {
'latest_messages': [
self.get_test_mail('incoming-1.raw')
],
'seen_status': {
2: 'UNSEEN'
},
'uid_list': [2]
}
}
email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
email_account.receive(test_mails=test_mails)
TestEmailAccount.mocked_email_receive(email_account, messages)
comm = frappe.get_doc("Communication", {"sender": "test_sender@example.com"})
self.assertTrue("test_receiver@example.com" in comm.recipients)
@ -72,11 +83,21 @@ class TestEmailAccount(unittest.TestCase):
existing_file = frappe.get_doc({'doctype': 'File', 'file_name': 'erpnext-conf-14.png'})
frappe.delete_doc("File", existing_file.name)
with open(os.path.join(os.path.dirname(__file__), "test_mails", "incoming-2.raw"), "r") as testfile:
test_mails = [testfile.read()]
messages = {
# append_to = ToDo
'"INBOX"': {
'latest_messages': [
self.get_test_mail('incoming-2.raw')
],
'seen_status': {
2: 'UNSEEN'
},
'uid_list': [2]
}
}
email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
email_account.receive(test_mails=test_mails)
TestEmailAccount.mocked_email_receive(email_account, messages)
comm = frappe.get_doc("Communication", {"sender": "test_sender@example.com"})
self.assertTrue("test_receiver@example.com" in comm.recipients)
@ -93,11 +114,21 @@ class TestEmailAccount(unittest.TestCase):
def test_incoming_attached_email_from_outlook_plain_text_only(self):
cleanup("test_sender@example.com")
with open(os.path.join(os.path.dirname(__file__), "test_mails", "incoming-3.raw"), "r") as f:
test_mails = [f.read()]
messages = {
# append_to = ToDo
'"INBOX"': {
'latest_messages': [
self.get_test_mail('incoming-3.raw')
],
'seen_status': {
2: 'UNSEEN'
},
'uid_list': [2]
}
}
email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
email_account.receive(test_mails=test_mails)
TestEmailAccount.mocked_email_receive(email_account, messages)
comm = frappe.get_doc("Communication", {"sender": "test_sender@example.com"})
self.assertTrue("From: &quot;Microsoft Outlook&quot; &lt;test_sender@example.com&gt;" in comm.content)
@ -106,11 +137,21 @@ class TestEmailAccount(unittest.TestCase):
def test_incoming_attached_email_from_outlook_layers(self):
cleanup("test_sender@example.com")
with open(os.path.join(os.path.dirname(__file__), "test_mails", "incoming-4.raw"), "r") as f:
test_mails = [f.read()]
messages = {
# append_to = ToDo
'"INBOX"': {
'latest_messages': [
self.get_test_mail('incoming-4.raw')
],
'seen_status': {
2: 'UNSEEN'
},
'uid_list': [2]
}
}
email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
email_account.receive(test_mails=test_mails)
TestEmailAccount.mocked_email_receive(email_account, messages)
comm = frappe.get_doc("Communication", {"sender": "test_sender@example.com"})
self.assertTrue("From: &quot;Microsoft Outlook&quot; &lt;test_sender@example.com&gt;" in comm.content)
@ -151,11 +192,23 @@ class TestEmailAccount(unittest.TestCase):
with open(os.path.join(os.path.dirname(__file__), "test_mails", "reply-1.raw"), "r") as f:
raw = f.read()
raw = raw.replace("<-- in-reply-to -->", sent_mail.get("Message-Id"))
test_mails = [raw]
# parse reply
messages = {
# append_to = ToDo
'"INBOX"': {
'latest_messages': [
raw
],
'seen_status': {
2: 'UNSEEN'
},
'uid_list': [2]
}
}
email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
email_account.receive(test_mails=test_mails)
TestEmailAccount.mocked_email_receive(email_account, messages)
sent = frappe.get_doc("Communication", sent_name)
@ -173,8 +226,20 @@ class TestEmailAccount(unittest.TestCase):
test_mails.append(f.read())
# parse reply
messages = {
# append_to = ToDo
'"INBOX"': {
'latest_messages': test_mails,
'seen_status': {
2: 'UNSEEN',
3: 'UNSEEN'
},
'uid_list': [2, 3]
}
}
email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
email_account.receive(test_mails=test_mails)
TestEmailAccount.mocked_email_receive(email_account, messages)
comm_list = frappe.get_all("Communication", filters={"sender":"test_sender@example.com"},
fields=["name", "reference_doctype", "reference_name"])
@ -197,11 +262,22 @@ class TestEmailAccount(unittest.TestCase):
# get test mail with message-id as in-reply-to
with open(os.path.join(os.path.dirname(__file__), "test_mails", "reply-4.raw"), "r") as f:
test_mails = [f.read().replace('{{ message_id }}', last_mail.message_id)]
messages = {
# append_to = ToDo
'"INBOX"': {
'latest_messages': [
f.read().replace('{{ message_id }}', last_mail.message_id)
],
'seen_status': {
2: 'UNSEEN'
},
'uid_list': [2]
}
}
# pull the mail
email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
email_account.receive(test_mails=test_mails)
TestEmailAccount.mocked_email_receive(email_account, messages)
comm_list = frappe.get_all("Communication", filters={"sender":"test_sender@example.com"},
fields=["name", "reference_doctype", "reference_name"])
@ -213,10 +289,21 @@ class TestEmailAccount(unittest.TestCase):
def test_auto_reply(self):
cleanup("test_sender@example.com")
test_mails = [self.get_test_mail('incoming-1.raw')]
messages = {
# append_to = ToDo
'"INBOX"': {
'latest_messages': [
self.get_test_mail('incoming-1.raw')
],
'seen_status': {
2: 'UNSEEN'
},
'uid_list': [2]
}
}
email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
email_account.receive(test_mails=test_mails)
TestEmailAccount.mocked_email_receive(email_account, messages)
comm = frappe.get_doc("Communication", {"sender": "test_sender@example.com"})
self.assertTrue(frappe.db.get_value("Email Queue", {"reference_doctype": comm.reference_doctype,
@ -246,6 +333,91 @@ class TestEmailAccount(unittest.TestCase):
with self.assertRaises(Exception):
email_account.validate()
def test_append_to(self):
email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
mail_content = self.get_test_mail(fname="incoming-2.raw")
inbound_mail = InboundMail(mail_content, email_account, 12345, 1, 'ToDo')
communication = inbound_mail.process()
# the append_to for the email is set to ToDO in "_Test Email Account 1"
self.assertEqual(communication.reference_doctype, 'ToDo')
self.assertTrue(communication.reference_name)
self.assertTrue(frappe.db.exists(communication.reference_doctype, communication.reference_name))
def test_append_to_with_imap_folders(self):
mail_content_1 = self.get_test_mail(fname="incoming-1.raw")
mail_content_2 = self.get_test_mail(fname="incoming-2.raw")
mail_content_3 = self.get_test_mail(fname="incoming-3.raw")
messages = {
# append_to = ToDo
'"INBOX"': {
'latest_messages': [
mail_content_1,
mail_content_2
],
'seen_status': {
0: 'UNSEEN',
1: 'UNSEEN'
},
'uid_list': [0,1]
},
# append_to = Communication
'"Test Folder"': {
'latest_messages': [
mail_content_3
],
'seen_status': {
2: 'UNSEEN'
},
'uid_list': [2]
}
}
email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
mails = TestEmailAccount.mocked_get_inbound_mails(email_account, messages)
self.assertEqual(len(mails), 3)
inbox_mails = 0
test_folder_mails = 0
for mail in mails:
communication = mail.process()
if mail.append_to == 'ToDo':
inbox_mails += 1
self.assertEqual(communication.reference_doctype, 'ToDo')
self.assertTrue(communication.reference_name)
self.assertTrue(frappe.db.exists(communication.reference_doctype, communication.reference_name))
else:
test_folder_mails += 1
self.assertEqual(communication.reference_doctype, None)
self.assertEqual(inbox_mails, 2)
self.assertEqual(test_folder_mails, 1)
@patch("frappe.email.receive.EmailServer.select_imap_folder", return_value=True)
@patch("frappe.email.receive.EmailServer.logout", side_effect=lambda: None)
def mocked_get_inbound_mails(email_account, messages={}, mocked_logout=None, mocked_select_imap_folder=None):
from frappe.email.receive import EmailServer
def get_mocked_messages(**kwargs):
return messages.get(kwargs["folder"], {})
with patch.object(EmailServer, "get_messages", side_effect=get_mocked_messages):
mails = email_account.get_inbound_mails()
return mails
@patch("frappe.email.receive.EmailServer.select_imap_folder", return_value=True)
@patch("frappe.email.receive.EmailServer.logout", side_effect=lambda: None)
def mocked_email_receive(email_account, messages={}, mocked_logout=None, mocked_select_imap_folder=None):
def get_mocked_messages(**kwargs):
return messages.get(kwargs["folder"], {})
from frappe.email.receive import EmailServer
with patch.object(EmailServer, "get_messages", side_effect=get_mocked_messages):
email_account.receive()
class TestInboundMail(unittest.TestCase):
@classmethod
def setUpClass(cls):
@ -313,11 +485,11 @@ class TestInboundMail(unittest.TestCase):
email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
inbound_mail = InboundMail(mail_content, email_account, 12345, 1)
new_communiction = inbound_mail.process()
new_communication = inbound_mail.process()
# Make sure that uid is changed to new uid
self.assertEqual(new_communiction.uid, 12345)
self.assertEqual(communication.name, new_communiction.name)
self.assertEqual(new_communication.uid, 12345)
self.assertEqual(communication.name, new_communication.name)
def test_find_parent_email_queue(self):
"""If the mail is reply to the already sent mail, there will be a email queue record.

2
frappe/email/doctype/email_account/test_records.json
View file

@ -20,7 +20,7 @@
"pop3_server": "pop.test.example.com",
"no_remaining":"0",
"append_to": "ToDo",
"imap_folder": [{"folder_name": "INBOX", "append_to": "ToDo"}],
"imap_folder": [{"folder_name": "INBOX", "append_to": "ToDo"}, {"folder_name": "Test Folder", "append_to": "Communication"}],
"track_email_status": 1
},
{

31
frappe/email/doctype/email_queue/email_queue.py
View file

@ -479,21 +479,24 @@ class QueueBuilder:
EmailUnsubscribe = DocType("Email Unsubscribe")
unsubscribed = (
frappe.qb.from_(EmailUnsubscribe).select(
EmailUnsubscribe.email
).where(
EmailUnsubscribe.email.isin(all_ids)
& (
(
(EmailUnsubscribe.reference_doctype == self.reference_doctype)
& (EmailUnsubscribe.reference_name == self.reference_name)
) | (
EmailUnsubscribe.global_unsubscribe == 1
if len(all_ids) > 0:
unsubscribed = (
frappe.qb.from_(EmailUnsubscribe).select(
EmailUnsubscribe.email
).where(
EmailUnsubscribe.email.isin(all_ids)
& (
(
(EmailUnsubscribe.reference_doctype == self.reference_doctype)
& (EmailUnsubscribe.reference_name == self.reference_name)
) | (
EmailUnsubscribe.global_unsubscribe == 1
)
)
)
).distinct()
).run(pluck=True)
).distinct()
).run(pluck=True)
else:
unsubscribed = None
self._unsubscribed_user_emails = unsubscribed or []
return self._unsubscribed_user_emails

2
frappe/email/doctype/notification/notification.py
View file

@ -137,7 +137,7 @@ def get_context(context):
if self.set_property_after_alert:
allow_update = True
if doc.docstatus == 1 and not doc.meta.get_field(self.set_property_after_alert).allow_on_submit:
if doc.docstatus.is_submitted() and not doc.meta.get_field(self.set_property_after_alert).allow_on_submit:
allow_update = False
try:
if allow_update and not doc.flags.in_notification_update:

21
frappe/email/receive.py
View file

@ -108,7 +108,8 @@ class EmailServer:
raise
def select_imap_folder(self, folder):
self.imap.select(folder)
res = self.imap.select(f'"{folder}"')
return res[0] == 'OK' # The folder exsits TODO: handle other resoponses too
def logout(self):
if cint(self.settings.use_imap):
@ -582,10 +583,11 @@ class Email:
class InboundMail(Email):
"""Class representation of incoming mail along with mail handlers.
"""
def __init__(self, content, email_account, uid=None, seen_status=None):
def __init__(self, content, email_account, uid=None, seen_status=None, append_to=None):
super().__init__(content)
self.email_account = email_account
self.uid = uid or -1
self.append_to = append_to
self.seen_status = seen_status or 0
# System documents related to this mail
@ -623,15 +625,18 @@ class InboundMail(Email):
if self.parent_communication():
data['in_reply_to'] = self.parent_communication().name
append_to = self.append_to if self.email_account.use_imap else self.email_account.append_to
if self.reference_document():
data['reference_doctype'] = self.reference_document().doctype
data['reference_name'] = self.reference_document().name
elif self.email_account.append_to and self.email_account.append_to != 'Communication':
reference_doc = self._create_reference_document(self.email_account.append_to)
if reference_doc:
data['reference_doctype'] = reference_doc.doctype
data['reference_name'] = reference_doc.name
data['is_first'] = True
else:
if append_to and append_to != 'Communication':
reference_doc = self._create_reference_document(append_to)
if reference_doc:
data['reference_doctype'] = reference_doc.doctype
data['reference_name'] = reference_doc.name
data['is_first'] = True
if self.is_notification():
# Disable notifications for notification.

4
frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.py
View file

@ -5,7 +5,7 @@ import frappe
import json
from frappe import _
from frappe.model.document import Document
from frappe.model import default_fields
from frappe.model import default_fields, child_table_fields
class DocumentTypeMapping(Document):
def validate(self):
@ -14,7 +14,7 @@ class DocumentTypeMapping(Document):
def validate_inner_mapping(self):
meta = frappe.get_meta(self.local_doctype)
for field_map in self.field_mapping:
if field_map.local_fieldname not in default_fields:
if field_map.local_fieldname not in (default_fields + child_table_fields):
field = meta.get_field(field_map.local_fieldname)
if not field:
frappe.throw(_('Row #{0}: Invalid Local Fieldname').format(field_map.idx))

1
frappe/exceptions.py
View file

@ -103,6 +103,7 @@ class DocumentAlreadyRestored(ValidationError): pass
class AttachmentLimitReached(ValidationError): pass
class QueryTimeoutError(Exception): pass
class QueryDeadlockError(Exception): pass
class TooManyWritesError(Exception): pass
# OAuth exceptions
class InvalidAuthorizationHeader(CSRFTokenError): pass
class InvalidAuthorizationPrefix(CSRFTokenError): pass

9
frappe/model/__init__.py
View file

@ -90,11 +90,14 @@ default_fields = (
'creation',
'modified',
'modified_by',
'docstatus',
'idx'
)
child_table_fields = (
'parent',
'parentfield',
'parenttype',
'idx',
'docstatus'
'parenttype'
)
optional_fields = (

85
frappe/model/base_document.py
View file

@ -1,9 +1,10 @@
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# License: MIT. See LICENSE
import frappe
import datetime
from frappe import _
from frappe.model import default_fields, table_fields
from frappe.model import default_fields, table_fields, child_table_fields
from frappe.model.naming import set_new_name
from frappe.model.utils.link_count import notify_link_count
from frappe.modules import load_doctype_module
@ -11,6 +12,7 @@ from frappe.model import display_fieldtypes
from frappe.utils import (cint, flt, now, cstr, strip_html,
sanitize_html, sanitize_email, cast_fieldtype)
from frappe.utils.html_utils import unescape_html
from frappe.model.docstatus import DocStatus
max_positive_value = {
'smallint': 2 ** 15,
@ -20,6 +22,7 @@ max_positive_value = {
DOCTYPES_FOR_DOCTYPE = ('DocType', 'DocField', 'DocPerm', 'DocType Action', 'DocType Link')
def get_controller(doctype):
"""Returns the **class** object of the given DocType.
For `custom` type, returns `frappe.model.document.Document`.
@ -101,6 +104,10 @@ class BaseDocument(object):
"balance": 42000
})
"""
# QUESTION: why do we need the 1st for loop?
# we're essentially setting the values in d, in the 2nd for loop (?)
# first set default field values of base document
for key in default_fields:
if key in d:
@ -205,7 +212,10 @@ class BaseDocument(object):
raise ValueError
def remove(self, doc):
self.get(doc.parentfield).remove(doc)
# Usage: from the parent doc, pass the child table doc
# to remove that child doc from the child table, thus removing it from the parent doc
if doc.get("parentfield"):
self.get(doc.parentfield).remove(doc)
def _init_child(self, value, key):
if not self.doctype:
@ -224,7 +234,7 @@ class BaseDocument(object):
value.parentfield = key
if value.docstatus is None:
value.docstatus = 0
value.docstatus = DocStatus.draft()
if not getattr(value, "idx", None):
value.idx = len(self.get(key) or []) + 1
@ -282,8 +292,11 @@ class BaseDocument(object):
if key not in self.__dict__:
self.__dict__[key] = None
if key in ("idx", "docstatus") and self.__dict__[key] is None:
self.__dict__[key] = 0
if self.__dict__[key] is None:
if key == "docstatus":
self.docstatus = DocStatus.draft()
elif key == "idx":
self.__dict__[key] = 0
for key in self.get_valid_columns():
if key not in self.__dict__:
@ -304,12 +317,27 @@ class BaseDocument(object):
def is_new(self):
return self.get("__islocal")
def as_dict(self, no_nulls=False, no_default_fields=False, convert_dates_to_str=False):
@property
def docstatus(self):
return DocStatus(self.get("docstatus"))
@docstatus.setter
def docstatus(self, value):
self.__dict__["docstatus"] = DocStatus(cint(value))
def as_dict(self, no_nulls=False, no_default_fields=False, convert_dates_to_str=False, no_child_table_fields=False):
doc = self.get_valid_dict(convert_dates_to_str=convert_dates_to_str)
doc["doctype"] = self.doctype
for df in self.meta.get_table_fields():
children = self.get(df.fieldname) or []
doc[df.fieldname] = [d.as_dict(convert_dates_to_str=convert_dates_to_str, no_nulls=no_nulls, no_default_fields=no_default_fields) for d in children]
doc[df.fieldname] = [
d.as_dict(
convert_dates_to_str=convert_dates_to_str,
no_nulls=no_nulls,
no_default_fields=no_default_fields,
no_child_table_fields=no_child_table_fields
) for d in children
]
if no_nulls:
for k in list(doc):
@ -321,6 +349,11 @@ class BaseDocument(object):
if k in default_fields:
del doc[k]
if no_child_table_fields:
for k in list(doc):
if k in child_table_fields:
del doc[k]
for key in ("_user_tags", "__islocal", "__onload", "_liked_by", "__run_link_triggers", "__unsaved"):
if self.get(key):
doc[key] = self.get(key)
@ -492,7 +525,7 @@ class BaseDocument(object):
self.set(df.fieldname, flt(self.get(df.fieldname)))
if self.docstatus is not None:
self.docstatus = cint(self.docstatus)
self.docstatus = DocStatus(cint(self.docstatus))
def _get_missing_mandatory_fields(self):
"""Get mandatory fields that do not have any values"""
@ -500,12 +533,12 @@ class BaseDocument(object):
if df.fieldtype in table_fields:
return "{}: {}: {}".format(_("Error"), _("Data missing in table"), _(df.label))
elif self.parentfield:
# check if parentfield exists (only applicable for child table doctype)
elif self.get("parentfield"):
return "{}: {} {} #{}: {}: {}".format(_("Error"), frappe.bold(_(self.doctype)),
_("Row"), self.idx, _("Value missing for"), _(df.label))
else:
return _("Error: Value missing for {0}: {1}").format(_(df.parent), _(df.label))
return _("Error: Value missing for {0}: {1}").format(_(df.parent), _(df.label))
missing = []
@ -524,10 +557,11 @@ class BaseDocument(object):
def get_invalid_links(self, is_submittable=False):
"""Returns list of invalid links and also updates fetch values if not set"""
def get_msg(df, docname):
if self.parentfield:
# check if parentfield exists (only applicable for child table doctype)
if self.get("parentfield"):
return "{} #{}: {}: {}".format(_("Row"), self.idx, _(df.label), docname)
else:
return "{}: {}".format(_(df.label), docname)
return "{}: {}".format(_(df.label), docname)
invalid_links = []
cancelled_links = []
@ -581,7 +615,7 @@ class BaseDocument(object):
setattr(self, df.fieldname, values.name)
for _df in fields_to_fetch:
if self.is_new() or self.docstatus != 1 or _df.allow_on_submit:
if self.is_new() or not self.docstatus.is_submitted() or _df.allow_on_submit:
self.set_fetch_from_value(doctype, _df, values)
notify_link_count(doctype, docname)
@ -591,7 +625,7 @@ class BaseDocument(object):
elif (df.fieldname != "amended_from"
and (is_submittable or self.meta.is_submittable) and frappe.get_meta(doctype).is_submittable
and cint(frappe.db.get_value(doctype, docname, "docstatus"))==2):
and cint(frappe.db.get_value(doctype, docname, "docstatus")) == DocStatus.cancelled()):
cancelled_links.append((df.fieldname, docname, get_msg(df, docname)))
@ -601,11 +635,8 @@ class BaseDocument(object):
fetch_from_fieldname = df.fetch_from.split('.')[-1]
value = values[fetch_from_fieldname]
if df.fieldtype in ['Small Text', 'Text', 'Data']:
if fetch_from_fieldname in default_fields:
from frappe.model.meta import get_default_df
fetch_from_df = get_default_df(fetch_from_fieldname)
else:
fetch_from_df = frappe.get_meta(doctype).get_field(fetch_from_fieldname)
from frappe.model.meta import get_default_df
fetch_from_df = get_default_df(fetch_from_fieldname) or frappe.get_meta(doctype).get_field(fetch_from_fieldname)
if not fetch_from_df:
frappe.throw(
@ -740,9 +771,9 @@ class BaseDocument(object):
def throw_length_exceeded_error(self, df, max_length, value):
if self.parentfield and self.idx:
# check if parentfield exists (only applicable for child table doctype)
if self.get("parentfield"):
reference = _("{0}, Row {1}").format(_(self.doctype), self.idx)
else:
reference = "{0} {1}".format(_(self.doctype), self.name)
@ -805,8 +836,8 @@ class BaseDocument(object):
or df.get("fieldtype") in ("Attach", "Attach Image", "Barcode", "Code")
# cancelled and submit but not update after submit should be ignored
or self.docstatus==2
or (self.docstatus==1 and not df.get("allow_on_submit"))):
or self.docstatus.is_cancelled()
or (self.docstatus.is_submitted() and not df.get("allow_on_submit"))):
continue
else:
@ -853,7 +884,7 @@ class BaseDocument(object):
:param parentfield: If fieldname is in child table."""
from frappe.model.meta import get_field_precision
if parentfield and not isinstance(parentfield, str):
if parentfield and not isinstance(parentfield, str) and parentfield.get("parentfield"):
parentfield = parentfield.parentfield
cache_key = parentfield or "main"
@ -880,7 +911,7 @@ class BaseDocument(object):
from frappe.utils.formatters import format_value
df = self.meta.get_field(fieldname)
if not df and fieldname in default_fields:
if not df:
from frappe.model.meta import get_default_df
df = get_default_df(fieldname)

27
frappe/model/delete_doc.py
View file

@ -212,7 +212,7 @@ def check_permission_and_not_submitted(doc):
.format(doc.doctype, doc.name), raise_exception=frappe.PermissionError)
# check if submitted
if doc.docstatus == 1:
if doc.docstatus.is_submitted():
frappe.msgprint(_("{0} {1}: Submitted Record cannot be deleted. You must {2} Cancel {3} it first.").format(_(doc.doctype), doc.name, "<a href='https://docs.erpnext.com//docs/user/manual/en/setting-up/articles/delete-submitted-document' target='_blank'>", "</a>"),
raise_exception=True)
@ -222,32 +222,35 @@ def check_if_doc_is_linked(doc, method="Delete"):
"""
from frappe.model.rename_doc import get_link_fields
link_fields = get_link_fields(doc.doctype)
link_fields = [[lf['parent'], lf['fieldname'], lf['issingle']] for lf in link_fields]
ignore_linked_doctypes = doc.get('ignore_linked_doctypes') or []
for lf in link_fields:
link_dt, link_field, issingle = lf['parent'], lf['fieldname'], lf['issingle']
for link_dt, link_field, issingle in link_fields:
if not issingle:
for item in frappe.db.get_values(link_dt, {link_field:doc.name},
["name", "parent", "parenttype", "docstatus"], as_dict=True):
linked_doctype = item.parenttype if item.parent else link_dt
fields = ["name", "docstatus"]
if frappe.get_meta(link_dt).istable:
fields.extend(["parent", "parenttype"])
ignore_linked_doctypes = doc.get('ignore_linked_doctypes') or []
for item in frappe.db.get_values(link_dt, {link_field:doc.name}, fields , as_dict=True):
# available only in child table cases
item_parent = getattr(item, "parent", None)
linked_doctype = item.parenttype if item_parent else link_dt
if linked_doctype in doctypes_to_skip or (linked_doctype in ignore_linked_doctypes and method == 'Cancel'):
# don't check for communication and todo!
continue
if not item:
continue
elif method != "Delete" and (method != "Cancel" or item.docstatus != 1):
if method != "Delete" and (method != "Cancel" or item.docstatus != 1):
# don't raise exception if not
# linked to a non-cancelled doc when deleting or to a submitted doc when cancelling
continue
elif link_dt == doc.doctype and (item.parent or item.name) == doc.name:
elif link_dt == doc.doctype and (item_parent or item.name) == doc.name:
# don't raise exception if not
# linked to same item or doc having same name as the item
continue
else:
reference_docname = item.parent or item.name
reference_docname = item_parent or item.name
raise_link_exists_exception(doc, linked_doctype, reference_docname)
else:

25
frappe/model/docstatus.py Normal file
View file

@ -0,0 +1,25 @@
# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors
# License: MIT. See LICENSE
class DocStatus(int):
def is_draft(self):
return self == self.draft()
def is_submitted(self):
return self == self.submitted()
def is_cancelled(self):
return self == self.cancelled()
@classmethod
def draft(cls):
return cls(0)
@classmethod
def submitted(cls):
return cls(1)
@classmethod
def cancelled(cls):
return cls(2)

64
frappe/model/document.py
View file

@ -1,13 +1,16 @@
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# License: MIT. See LICENSE
import frappe
import hashlib
import json
import time
from werkzeug.exceptions import NotFound
import frappe
from frappe import _, msgprint, is_whitelisted
from frappe.utils import flt, cstr, now, get_datetime_str, file_lock, date_diff
from frappe.model.base_document import BaseDocument, get_controller
from frappe.model.naming import set_new_name, gen_new_name_for_cancelled_doc
from werkzeug.exceptions import NotFound, Forbidden
import hashlib, json
from frappe.model.naming import set_new_name
from frappe.model.docstatus import DocStatus
from frappe.model import optional_fields, table_fields
from frappe.model.workflow import validate_workflow
from frappe.model.workflow import set_workflow_state_on_action
@ -17,6 +20,7 @@ from frappe.desk.form.document_follow import follow_document
from frappe.core.doctype.server_script.server_script_utils import run_server_script_for_doc_event
from frappe.utils.data import get_absolute_url
# once_only validation
# methods
@ -307,9 +311,6 @@ class Document(BaseDocument):
self.check_permission("write", "save")
if self.docstatus == 2:
self._rename_doc_on_cancel()
self.set_user_and_timestamp()
self.set_docstatus()
self.check_if_latest()
@ -490,7 +491,7 @@ class Document(BaseDocument):
def set_docstatus(self):
if self.docstatus is None:
self.docstatus=0
self.docstatus = DocStatus.draft()
for d in self.get_all_children():
d.docstatus = self.docstatus
@ -526,7 +527,7 @@ class Document(BaseDocument):
def _validate_non_negative(self):
def get_msg(df):
if self.parentfield:
if self.get("parentfield"):
return "{} {} #{}: {} {}".format(frappe.bold(_(self.doctype)),
_("Row"), self.idx, _("Value cannot be negative for"), frappe.bold(_(df.label)))
else:
@ -720,6 +721,7 @@ class Document(BaseDocument):
else:
tmp = frappe.db.sql("""select modified, docstatus from `tab{0}`
where name = %s for update""".format(self.doctype), self.name, as_dict=True)
if not tmp:
frappe.throw(_("Record does not exist"))
else:
@ -740,7 +742,7 @@ class Document(BaseDocument):
else:
self.check_docstatus_transition(0)
def check_docstatus_transition(self, docstatus):
def check_docstatus_transition(self, to_docstatus):
"""Ensures valid `docstatus` transition.
Valid transitions are (number in brackets is `docstatus`):
@ -751,31 +753,32 @@ class Document(BaseDocument):
"""
if not self.docstatus:
self.docstatus = 0
if docstatus==0:
if self.docstatus==0:
self.docstatus = DocStatus.draft()
if to_docstatus == DocStatus.draft():
if self.docstatus.is_draft():
self._action = "save"
elif self.docstatus==1:
elif self.docstatus.is_submitted():
self._action = "submit"
self.check_permission("submit")
elif self.docstatus==2:
elif self.docstatus.is_cancelled():
raise frappe.DocstatusTransitionError(_("Cannot change docstatus from 0 (Draft) to 2 (Cancelled)"))
else:
raise frappe.ValidationError(_("Invalid docstatus"), self.docstatus)
elif docstatus==1:
if self.docstatus==1:
elif to_docstatus == DocStatus.submitted():
if self.docstatus.is_submitted():
self._action = "update_after_submit"
self.check_permission("submit")
elif self.docstatus==2:
elif self.docstatus.is_cancelled():
self._action = "cancel"
self.check_permission("cancel")
elif self.docstatus==0:
elif self.docstatus.is_draft():
raise frappe.DocstatusTransitionError(_("Cannot change docstatus from 1 (Submitted) to 0 (Draft)"))
else:
raise frappe.ValidationError(_("Invalid docstatus"), self.docstatus)
elif docstatus==2:
elif to_docstatus == DocStatus.cancelled():
raise frappe.ValidationError(_("Cannot edit cancelled document"))
def set_parent_in_children(self):
@ -929,14 +932,14 @@ class Document(BaseDocument):
@whitelist.__func__
def _submit(self):
"""Submit the document. Sets `docstatus` = 1, then saves."""
self.docstatus = 1
self.docstatus = DocStatus.submitted()
return self.save()
@whitelist.__func__
def _cancel(self):
"""Cancel the document. Sets `docstatus` = 2, then saves.
"""
self.docstatus = 2
self.docstatus = DocStatus.cancelled()
return self.save()
@whitelist.__func__
@ -954,7 +957,7 @@ class Document(BaseDocument):
frappe.delete_doc(self.doctype, self.name, ignore_permissions = ignore_permissions, flags=self.flags)
def run_before_save_methods(self):
"""Run standard methods before `INSERT` or `UPDATE`. Standard Methods are:
"""Run standard methods before `INSERT` or `UPDATE`. Standard Methods are:
- `validate`, `before_save` for **Save**.
- `validate`, `before_submit` for **Submit**.
@ -1199,7 +1202,7 @@ class Document(BaseDocument):
if not frappe.compare(val1, condition, val2):
label = doc.meta.get_label(fieldname)
condition_str = error_condition_map.get(condition, condition)
if doc.parentfield:
if doc.get("parentfield"):
msg = _("Incorrect value in row {0}: {1} must be {2} {3}").format(doc.idx, label, condition_str, val2)
else:
msg = _("Incorrect value: {0} must be {1} {2}").format(label, condition_str, val2)
@ -1223,7 +1226,7 @@ class Document(BaseDocument):
doc.meta.get("fields", {"fieldtype": ["in", ["Currency", "Float", "Percent"]]}))
for fieldname in fieldnames:
doc.set(fieldname, flt(doc.get(fieldname), self.precision(fieldname, doc.parentfield)))
doc.set(fieldname, flt(doc.get(fieldname), self.precision(fieldname, doc.get("parentfield"))))
def get_url(self):
"""Returns Desk URL for this document."""
@ -1371,19 +1374,16 @@ class Document(BaseDocument):
from frappe.desk.doctype.tag.tag import DocTags
return DocTags(self.doctype).get_tags(self.name).split(",")[1:]
def _rename_doc_on_cancel(self):
new_name = gen_new_name_for_cancelled_doc(self)
frappe.rename_doc(self.doctype, self.name, new_name, force=True, show_alert=False)
self.name = new_name
def __repr__(self):
name = self.name or "unsaved"
doctype = self.__class__.__name__
docstatus = f" docstatus={self.docstatus}" if self.docstatus else ""
parent = f" parent={self.parent}" if self.parent else ""
repr_str = f"<{doctype}: {name}{docstatus}"
return f"<{doctype}: {name}{docstatus}{parent}>"
if not hasattr(self, "parent"):
return repr_str + ">"
return f"{repr_str} parent={self.parent}>"
def __str__(self):
name = self.name or "unsaved"

3
frappe/model/mapper.py
View file

@ -4,7 +4,7 @@ import json
import frappe
from frappe import _
from frappe.model import default_fields, table_fields
from frappe.model import default_fields, table_fields, child_table_fields
from frappe.utils import cstr
@ -149,6 +149,7 @@ def map_fields(source_doc, target_doc, table_map, source_parent):
no_copy_fields = set([d.fieldname for d in source_doc.meta.get("fields") if (d.no_copy==1 or d.fieldtype in table_fields)]
+ [d.fieldname for d in target_doc.meta.get("fields") if (d.no_copy==1 or d.fieldtype in table_fields)]
+ list(default_fields)
+ list(child_table_fields)
+ list(table_map.get("field_no_map", [])))
for df in target_doc.meta.get("fields"):

28
frappe/model/meta.py
View file

@ -18,7 +18,7 @@ from datetime import datetime
import click
import frappe, json, os
from frappe.utils import cstr, cint, cast
from frappe.model import default_fields, no_value_fields, optional_fields, data_fieldtypes, table_fields
from frappe.model import default_fields, no_value_fields, optional_fields, data_fieldtypes, table_fields, child_table_fields
from frappe.model.document import Document
from frappe.model.base_document import BaseDocument
from frappe.modules import load_doctype_module
@ -191,6 +191,8 @@ class Meta(Document):
else:
self._valid_columns = self.default_fields + \
[df.fieldname for df in self.get("fields") if df.fieldtype in data_fieldtypes]
if self.istable:
self._valid_columns += list(child_table_fields)
return self._valid_columns
@ -520,7 +522,7 @@ class Meta(Document):
'''add `links` child table in standard link dashboard format'''
dashboard_links = []
if hasattr(self, 'links') and self.links:
if getattr(self, 'links', None):
dashboard_links.extend(self.links)
if not data.transactions:
@ -625,9 +627,9 @@ def get_field_currency(df, doc=None):
frappe.local.field_currency = frappe._dict()
if not (frappe.local.field_currency.get((doc.doctype, doc.name), {}).get(df.fieldname) or
(doc.parent and frappe.local.field_currency.get((doc.doctype, doc.parent), {}).get(df.fieldname))):
(doc.get("parent") and frappe.local.field_currency.get((doc.doctype, doc.parent), {}).get(df.fieldname))):
ref_docname = doc.parent or doc.name
ref_docname = doc.get("parent") or doc.name
if ":" in cstr(df.get("options")):
split_opts = df.get("options").split(":")
@ -635,7 +637,7 @@ def get_field_currency(df, doc=None):
currency = frappe.get_cached_value(split_opts[0], doc.get(split_opts[1]), split_opts[2])
else:
currency = doc.get(df.get("options"))
if doc.parent:
if doc.get("parenttype"):
if currency:
ref_docname = doc.name
else:
@ -648,7 +650,7 @@ def get_field_currency(df, doc=None):
.setdefault(df.fieldname, currency)
return frappe.local.field_currency.get((doc.doctype, doc.name), {}).get(df.fieldname) or \
(doc.parent and frappe.local.field_currency.get((doc.doctype, doc.parent), {}).get(df.fieldname))
(doc.get("parent") and frappe.local.field_currency.get((doc.doctype, doc.parent), {}).get(df.fieldname))
def get_field_precision(df, doc=None, currency=None):
"""get precision based on DocField options and fieldvalue in doc"""
@ -669,19 +671,25 @@ def get_field_precision(df, doc=None, currency=None):
def get_default_df(fieldname):
if fieldname in default_fields:
if fieldname in (default_fields + child_table_fields):
if fieldname in ("creation", "modified"):
return frappe._dict(
fieldname = fieldname,
fieldtype = "Datetime"
)
else:
elif fieldname in ("idx", "docstatus"):
return frappe._dict(
fieldname = fieldname,
fieldtype = "Data"
fieldtype = "Int"
)
return frappe._dict(
fieldname = fieldname,
fieldtype = "Data"
)
def trim_tables(doctype=None, dry_run=False, quiet=False):
"""
Removes database fields that don't exist in the doctype (json or custom field). This may be needed
@ -713,7 +721,7 @@ def trim_tables(doctype=None, dry_run=False, quiet=False):
def trim_table(doctype, dry_run=True):
frappe.cache().hdel('table_columns', f"tab{doctype}")
ignore_fields = default_fields + optional_fields
ignore_fields = default_fields + optional_fields + child_table_fields
columns = frappe.db.get_table_columns(doctype)
fields = frappe.get_meta(doctype, cached=False).get_fieldnames_with_value()
is_internal = lambda f: f not in ignore_fields and not f.startswith("_")

118
frappe/model/naming.py
View file

@ -1,14 +1,3 @@
"""utilities to generate a document name based on various rules defined.
NOTE:
Till version 13, whenever a submittable document is amended it's name is set to orig_name-X,
where X is a counter and it increments when amended again and so on.
From Version 14, The naming pattern is changed in a way that amended documents will
have the original name `orig_name` instead of `orig_name-X`. To make this happen
the cancelled document naming pattern is changed to 'orig_name-CANC-X'.
"""
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# License: MIT. See LICENSE
@ -40,7 +29,7 @@ def set_new_name(doc):
doc.name = None
if getattr(doc, "amended_from", None):
doc.name = _get_amended_name(doc)
_set_amended_name(doc)
return
elif getattr(doc.meta, "issingle", False):
@ -256,18 +245,6 @@ def revert_series_if_last(key, name, doc=None):
* prefix = #### and hashes = 2021 (hash doesn't exist)
* will search hash in key then accordingly get prefix = ""
"""
if hasattr(doc, 'amended_from'):
# Do not revert the series if the document is amended.
if doc.amended_from:
return
# Get document name by parsing incase of fist cancelled document
if doc.docstatus == 2 and not doc.amended_from:
if doc.name.endswith('-CANC'):
name, _ = NameParser.parse_docname(doc.name, sep='-CANC')
else:
name, _ = NameParser.parse_docname(doc.name, sep='-CANC-')
if ".#" in key:
prefix, hashes = key.rsplit(".", 1)
if "#" not in hashes:
@ -356,9 +333,16 @@ def append_number_if_name_exists(doctype, value, fieldname="name", separator="-"
return value
def _get_amended_name(doc):
name, _ = NameParser(doc).parse_amended_from()
return name
def _set_amended_name(doc):
am_id = 1
am_prefix = doc.amended_from
if frappe.db.get_value(doc.doctype, doc.amended_from, "amended_from"):
am_id = cint(doc.amended_from.split("-")[-1]) + 1
am_prefix = "-".join(doc.amended_from.split("-")[:-1]) # except the last hyphen
doc.name = am_prefix + "-" + str(am_id)
return doc.name
def _field_autoname(autoname, doc, skip_slicing=None):
"""
@ -399,83 +383,3 @@ def _format_autoname(autoname, doc):
name = re.sub(r"(\{[\w | #]+\})", get_param_value_for_match, autoname_value)
return name
class NameParser:
"""Parse document name and return parts of it.
NOTE: It handles cancellend and amended doc parsing for now. It can be expanded.
"""
def __init__(self, doc):
self.doc = doc
def parse_amended_from(self):
"""
Cancelled document naming will be in one of these formats
* original_name-X-CANC - This is introduced to migrate old style naming to new style
* original_name-CANC - This is introduced to migrate old style naming to new style
* original_name-CANC-X - This is the new style naming
New style naming: In new style naming amended documents will have original name. That says,
when a document gets cancelled we need rename the document by adding `-CANC-X` to the end
so that amended documents can use the original name.
Old style naming: cancelled documents stay with original name and when amended, amended one
gets a new name as `original_name-X`. To bring new style naming we had to change the existing
cancelled document names and that is done by adding `-CANC` to cancelled documents through patch.
"""
if not getattr(self.doc, 'amended_from', None):
return (None, None)
# Handle old style cancelled documents (original_name-X-CANC, original_name-CANC)
if self.doc.amended_from.endswith('-CANC'):
name, _ = self.parse_docname(self.doc.amended_from, '-CANC')
amended_from_doc = frappe.get_all(
self.doc.doctype,
filters = {'name': self.doc.amended_from},
fields = ['amended_from'],
limit=1)
# Handle format original_name-X-CANC.
if amended_from_doc and amended_from_doc[0].amended_from:
return self.parse_docname(name, '-')
return name, None
# Handle new style cancelled documents
return self.parse_docname(self.doc.amended_from, '-CANC-')
@classmethod
def parse_docname(cls, name, sep='-'):
split_list = name.rsplit(sep, 1)
if len(split_list) == 1:
return (name, None)
return (split_list[0], split_list[1])
def get_cancelled_doc_latest_counter(tname, docname):
"""Get the latest counter used for cancelled docs of given docname.
"""
name_prefix = f'{docname}-CANC-'
rows = frappe.db.sql("""
select
name
from `tab{tname}`
where
name like %(name_prefix)s and docstatus=2
""".format(tname=tname), {'name_prefix': name_prefix+'%'}, as_dict=1)
if not rows:
return -1
return max([int(row.name.replace(name_prefix, '') or -1) for row in rows])
def gen_new_name_for_cancelled_doc(doc):
"""Generate a new name for cancelled document.
"""
if getattr(doc, "amended_from", None):
name, _ = NameParser(doc).parse_amended_from()
else:
name = doc.name
counter = get_cancelled_doc_latest_counter(doc.doctype, name)
return f'{name}-CANC-{counter+1}'

17
frappe/model/workflow.py
View file

@ -1,10 +1,11 @@
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# License: MIT. See LICENSE
import json
import frappe
from frappe.utils import cint
from frappe import _
import json
from frappe.utils import cint
from frappe.model.docstatus import DocStatus
class WorkflowStateError(frappe.ValidationError): pass
class WorkflowTransitionError(frappe.ValidationError): pass
@ -102,13 +103,13 @@ def apply_workflow(doc, action):
doc.set(next_state.update_field, next_state.update_value)
new_docstatus = cint(next_state.doc_status)
if doc.docstatus == 0 and new_docstatus == 0:
if doc.docstatus.is_draft() and new_docstatus == DocStatus.draft():
doc.save()
elif doc.docstatus == 0 and new_docstatus == 1:
elif doc.docstatus.is_draft() and new_docstatus == DocStatus.submitted():
doc.submit()
elif doc.docstatus == 1 and new_docstatus == 1:
elif doc.docstatus.is_submitted() and new_docstatus == DocStatus.submitted():
doc.save()
elif doc.docstatus == 1 and new_docstatus == 2:
elif doc.docstatus.is_submitted() and new_docstatus == DocStatus.cancelled():
doc.cancel()
else:
frappe.throw(_('Illegal Document Status for {0}').format(next_state.state))
@ -212,10 +213,10 @@ def bulk_workflow_approval(docnames, doctype, action):
frappe.db.commit()
except Exception as e:
if not frappe.message_log:
# Exception is raised manually and not from msgprint or throw
# Exception is raised manually and not from msgprint or throw
message = "{0}".format(e.__class__.__name__)
if e.args:
message += " : {0}".format(e.args[0])
message += " : {0}".format(e.args[0])
message_dict = {"docname": docname, "message": message}
failed_transactions[docname].append(message_dict)

2
frappe/modules/export_file.py
View file

@ -47,7 +47,7 @@ def strip_default_fields(doc, doc_export):
for df in doc.meta.get_table_fields():
for d in doc_export.get(df.fieldname):
for fieldname in frappe.model.default_fields:
for fieldname in (frappe.model.default_fields + frappe.model.child_table_fields):
if fieldname in d:
del d[fieldname]

2
frappe/patches.txt
View file

@ -119,7 +119,6 @@ execute:frappe.delete_doc_if_exists('DocType', 'GSuite Settings')
execute:frappe.delete_doc_if_exists('DocType', 'GSuite Templates')
execute:frappe.delete_doc_if_exists('DocType', 'GCalendar Account')
execute:frappe.delete_doc_if_exists('DocType', 'GCalendar Settings')
frappe.patches.v12_0.remove_parent_and_parenttype_from_print_formats
frappe.patches.v12_0.remove_example_email_thread_notify
execute:from frappe.desk.page.setup_wizard.install_fixtures import update_genders;update_genders()
frappe.patches.v12_0.set_correct_url_in_files
@ -185,7 +184,6 @@ frappe.patches.v13_0.queryreport_columns
frappe.patches.v13_0.jinja_hook
frappe.patches.v13_0.update_notification_channel_if_empty
frappe.patches.v13_0.set_first_day_of_the_week
frappe.patches.v14_0.rename_cancelled_documents
frappe.patches.v14_0.update_workspace2 # 20.09.2021
frappe.patches.v14_0.save_ratings_in_fraction #23-12-2021
frappe.patches.v14_0.transform_todo_schema

14
frappe/patches/v12_0/remove_parent_and_parenttype_from_print_formats.py
View file

@ -1,14 +0,0 @@
import frappe
def execute():
frappe.db.sql("""
UPDATE
`tabPrint Format`
SET
`tabPrint Format`.`parent`='',
`tabPrint Format`.`parenttype`='',
`tabPrint Format`.parentfield=''
WHERE
`tabPrint Format`.parent != ''
OR `tabPrint Format`.parenttype != ''
""")

1
frappe/patches/v13_0/remove_chat.py
View file

@ -3,6 +3,7 @@ import click
def execute():
frappe.delete_doc_if_exists("DocType", "Chat Message")
frappe.delete_doc_if_exists("DocType", "Chat Message Attachment")
frappe.delete_doc_if_exists("DocType", "Chat Profile")
frappe.delete_doc_if_exists("DocType", "Chat Token")
frappe.delete_doc_if_exists("DocType", "Chat Room User")

213
frappe/patches/v14_0/rename_cancelled_documents.py
View file

@ -1,213 +0,0 @@
import functools
import traceback
import frappe
def execute():
"""Rename cancelled documents by adding a postfix.
"""
rename_cancelled_docs()
def get_submittable_doctypes():
"""Returns list of submittable doctypes in the system.
"""
return frappe.db.get_all('DocType', filters={'is_submittable': 1}, pluck='name')
def get_cancelled_doc_names(doctype):
"""Return names of cancelled document names those are in old format.
"""
docs = frappe.db.get_all(doctype, filters={'docstatus': 2}, pluck='name')
return [each for each in docs if not (each.endswith('-CANC') or ('-CANC-' in each))]
@functools.lru_cache()
def get_linked_doctypes():
"""Returns list of doctypes those are linked with given doctype using 'Link' fieldtype.
"""
filters=[['fieldtype','=', 'Link']]
links = frappe.get_all("DocField",
fields=["parent", "fieldname", "options as linked_to"],
filters=filters,
as_list=1)
links+= frappe.get_all("Custom Field",
fields=["dt as parent", "fieldname", "options as linked_to"],
filters=filters,
as_list=1)
links_by_doctype = {}
for doctype, fieldname, linked_to in links:
links_by_doctype.setdefault(linked_to, []).append((doctype, fieldname))
return links_by_doctype
@functools.lru_cache()
def get_single_doctypes():
return frappe.get_all("DocType", filters={'issingle': 1}, pluck='name')
@functools.lru_cache()
def get_dynamic_linked_doctypes():
filters=[['fieldtype','=', 'Dynamic Link']]
# find dynamic links of parents
links = frappe.get_all("DocField",
fields=["parent as doctype", "fieldname", "options as doctype_fieldname"],
filters=filters,
as_list=1)
links+= frappe.get_all("Custom Field",
fields=["dt as doctype", "fieldname", "options as doctype_fieldname"],
filters=filters,
as_list=1)
return links
@functools.lru_cache()
def get_child_tables():
"""
"""
filters =[['fieldtype', 'in', ('Table', 'Table MultiSelect')]]
links = frappe.get_all("DocField",
fields=["parent as doctype", "options as child_table"],
filters=filters,
as_list=1)
links+= frappe.get_all("Custom Field",
fields=["dt as doctype", "options as child_table"],
filters=filters,
as_list=1)
map = {}
for doctype, child_table in links:
map.setdefault(doctype, []).append(child_table)
return map
def update_cancelled_document_names(doctype, cancelled_doc_names):
return frappe.db.sql("""
update
`tab{doctype}`
set
name=CONCAT(name, '-CANC')
where
docstatus=2
and
name in %(cancelled_doc_names)s;
""".format(doctype=doctype), {'cancelled_doc_names': cancelled_doc_names})
def update_amended_field(doctype, cancelled_doc_names):
return frappe.db.sql("""
update
`tab{doctype}`
set
amended_from=CONCAT(amended_from, '-CANC')
where
amended_from in %(cancelled_doc_names)s;
""".format(doctype=doctype), {'cancelled_doc_names': cancelled_doc_names})
def update_attachments(doctype, cancelled_doc_names):
frappe.db.sql("""
update
`tabFile`
set
attached_to_name=CONCAT(attached_to_name, '-CANC')
where
attached_to_doctype=%(dt)s and attached_to_name in %(cancelled_doc_names)s
""", {'cancelled_doc_names': cancelled_doc_names, 'dt': doctype})
def update_versions(doctype, cancelled_doc_names):
frappe.db.sql("""
UPDATE
`tabVersion`
SET
docname=CONCAT(docname, '-CANC')
WHERE
ref_doctype=%(dt)s AND docname in %(cancelled_doc_names)s
""", {'cancelled_doc_names': cancelled_doc_names, 'dt': doctype})
def update_linked_doctypes(doctype, cancelled_doc_names):
single_doctypes = get_single_doctypes()
for linked_dt, field in get_linked_doctypes().get(doctype, []):
if linked_dt not in single_doctypes:
frappe.db.sql("""
update
`tab{linked_dt}`
set
`{column}`=CONCAT(`{column}`, '-CANC')
where
`{column}` in %(cancelled_doc_names)s;
""".format(linked_dt=linked_dt, column=field),
{'cancelled_doc_names': cancelled_doc_names})
else:
doc = frappe.get_single(linked_dt)
if getattr(doc, field) in cancelled_doc_names:
setattr(doc, field, getattr(doc, field)+'-CANC')
doc.flags.ignore_mandatory=True
doc.flags.ignore_validate=True
doc.save(ignore_permissions=True)
def update_dynamic_linked_doctypes(doctype, cancelled_doc_names):
single_doctypes = get_single_doctypes()
for linked_dt, fieldname, doctype_fieldname in get_dynamic_linked_doctypes():
if linked_dt not in single_doctypes:
frappe.db.sql("""
update
`tab{linked_dt}`
set
`{column}`=CONCAT(`{column}`, '-CANC')
where
`{column}` in %(cancelled_doc_names)s and {doctype_fieldname}=%(dt)s;
""".format(linked_dt=linked_dt, column=fieldname, doctype_fieldname=doctype_fieldname),
{'cancelled_doc_names': cancelled_doc_names, 'dt': doctype})
else:
doc = frappe.get_single(linked_dt)
if getattr(doc, doctype_fieldname) == doctype and getattr(doc, fieldname) in cancelled_doc_names:
setattr(doc, fieldname, getattr(doc, fieldname)+'-CANC')
doc.flags.ignore_mandatory=True
doc.flags.ignore_validate=True
doc.save(ignore_permissions=True)
def update_child_tables(doctype, cancelled_doc_names):
child_tables = get_child_tables().get(doctype, [])
single_doctypes = get_single_doctypes()
for table in child_tables:
if table not in single_doctypes:
frappe.db.sql("""
update
`tab{table}`
set
parent=CONCAT(parent, '-CANC')
where
parenttype=%(dt)s and parent in %(cancelled_doc_names)s;
""".format(table=table), {'cancelled_doc_names': cancelled_doc_names, 'dt': doctype})
else:
doc = frappe.get_single(table)
if getattr(doc, 'parenttype')==doctype and getattr(doc, 'parent') in cancelled_doc_names:
setattr(doc, 'parent', getattr(doc, 'parent')+'-CANC')
doc.flags.ignore_mandatory=True
doc.flags.ignore_validate=True
doc.save(ignore_permissions=True)
def rename_cancelled_docs():
submittable_doctypes = get_submittable_doctypes()
for dt in submittable_doctypes:
for retry in range(2):
try:
cancelled_doc_names = tuple(get_cancelled_doc_names(dt))
if not cancelled_doc_names:
break
update_cancelled_document_names(dt, cancelled_doc_names)
update_amended_field(dt, cancelled_doc_names)
update_child_tables(dt, cancelled_doc_names)
update_linked_doctypes(dt, cancelled_doc_names)
update_dynamic_linked_doctypes(dt, cancelled_doc_names)
update_attachments(dt, cancelled_doc_names)
update_versions(dt, cancelled_doc_names)
print(f"Renaming cancelled records of {dt} doctype")
frappe.db.commit()
break
except Exception:
if retry == 1:
print(f"Failed to rename the cancelled records of {dt} doctype, moving on!")
traceback.print_exc()
frappe.db.rollback()

1
frappe/public/js/frappe-web.bundle.js
View file

@ -2,6 +2,7 @@ import "./jquery-bootstrap";
import "./frappe/class.js";
import "./frappe/polyfill.js";
import "./lib/md5.min.js";
import "./lib/moment.js";
import "./frappe/provide.js";
import "./frappe/format.js";
import "./frappe/utils/number_format.js";

2
frappe/public/js/frappe/data_import/import_preview.js
View file

@ -331,7 +331,7 @@ frappe.data_import.ImportPreview = class ImportPreview {
is_row_imported(row) {
let serial_no = row[0].content;
return this.import_log.find(log => {
return log.success && log.row_indexes.includes(serial_no);
return log.success && JSON.parse(log.row_indexes || '[]').includes(serial_no);
});
}
};

9
frappe/public/js/frappe/file_uploader/FileUploader.vue
View file

@ -534,22 +534,21 @@ export default {
});
},
show_google_drive_picker() {
let dialog = cur_dialog;
dialog.hide();
this.close_dialog = true;
let google_drive = new GoogleDrivePicker({
pickerCallback: data => this.google_drive_callback(data, dialog),
pickerCallback: data => this.google_drive_callback(data),
...this.google_drive_settings
});
google_drive.loadPicker();
},
google_drive_callback(data, dialog) {
google_drive_callback(data) {
if (data.action == google.picker.Action.PICKED) {
this.upload_file({
file_url: data.docs[0].url,
file_name: data.docs[0].name
});
} else if (data.action == google.picker.Action.CANCEL) {
dialog.show();
cur_frm.attachments.new_attachment()
}
},
url_to_file(url, filename, mime_type) {

49
frappe/public/js/frappe/form/controls/link.js
View file

@ -374,10 +374,22 @@ frappe.ui.form.ControlLink = class ControlLink extends frappe.ui.form.ControlDat
}
set_custom_query(args) {
var set_nulls = function(obj) {
$.each(obj, function(key, value) {
if(value!==undefined) {
obj[key] = value;
const is_valid_value = (value, key) => {
if (value) return true;
// check if empty value is valid
if (this.frm) {
let field = frappe.meta.get_docfield(this.frm.doctype, key);
// empty value link fields is invalid
return !field || !["Link", "Dynamic Link"].includes(field.fieldtype);
} else {
return value !== undefined;
}
}
const set_nulls = (obj) => {
$.each(obj, (key, value) => {
if (!is_valid_value(value, key)) {
delete obj[key];
}
});
return obj;
@ -458,7 +470,6 @@ frappe.ui.form.ControlLink = class ControlLink extends frappe.ui.form.ControlDat
validate_link_and_fetch(df, options, docname, value) {
if (!options) return;
let field_value = "";
const fetch_map = this.fetch_map;
const columns_to_fetch = Object.values(fetch_map);
@ -467,16 +478,10 @@ frappe.ui.form.ControlLink = class ControlLink extends frappe.ui.form.ControlDat
return value;
}
return frappe.xcall("frappe.client.validate_link", {
doctype: options,
docname: value,
fields: columns_to_fetch,
}).then((response) => {
if (!docname || !columns_to_fetch.length) return response.name;
function update_dependant_fields(response) {
let field_value = "";
for (const [target_field, source_field] of Object.entries(fetch_map)) {
if (value) field_value = response[source_field];
frappe.model.set_value(
df.parent,
docname,
@ -485,9 +490,23 @@ frappe.ui.form.ControlLink = class ControlLink extends frappe.ui.form.ControlDat
df.fieldtype,
);
}
}
return response.name;
});
// to avoid unnecessary request
if (value) {
return frappe.xcall("frappe.client.validate_link", {
doctype: options,
docname: value,
fields: columns_to_fetch,
}).then((response) => {
if (!docname || !columns_to_fetch.length) return response.name;
update_dependant_fields(response);
return response.name;
});
} else {
update_dependant_fields({});
return value;
}
}
get fetch_map() {

24
frappe/public/js/frappe/form/form.js
View file

@ -860,36 +860,32 @@ frappe.ui.form.Form = class FrappeForm {
}
_cancel(btn, callback, on_error, skip_confirm) {
const me = this;
const cancel_doc = () => {
frappe.validated = true;
this.script_manager.trigger("before_cancel").then(() => {
me.script_manager.trigger("before_cancel").then(() => {
if (!frappe.validated) {
return this.handle_save_fail(btn, on_error);
return me.handle_save_fail(btn, on_error);
}
const original_name = this.docname;
const after_cancel = (r) => {
var after_cancel = function(r) {
if (r.exc) {
this.handle_save_fail(btn, on_error);
me.handle_save_fail(btn, on_error);
} else {
frappe.utils.play_sound("cancel");
me.refresh();
callback && callback();
this.script_manager.trigger("after_cancel");
frappe.run_serially([
() => this.rename_notify(this.doctype, original_name, r.docs[0].name),
() => frappe.router.clear_re_route(this.doctype, original_name),
() => this.refresh(),
]);
me.script_manager.trigger("after_cancel");
}
};
frappe.ui.form.save(this, "cancel", after_cancel, btn);
frappe.ui.form.save(me, "cancel", after_cancel, btn);
});
}
if (skip_confirm) {
cancel_doc();
} else {
frappe.confirm(__("Permanently Cancel {0}?", [this.docname]), cancel_doc, this.handle_save_fail(btn, on_error));
frappe.confirm(__("Permanently Cancel {0}?", [this.docname]), cancel_doc, me.handle_save_fail(btn, on_error));
}
};
@ -911,7 +907,7 @@ frappe.ui.form.Form = class FrappeForm {
'docname': this.doc.name
}).then(is_amended => {
if (is_amended) {
frappe.throw(__('This document is already amended, you cannot amend it again'));
frappe.throw(__('This document is already amended, you cannot ammend it again'));
}
this.validate_form_action("Amend");
var me = this;

26
frappe/public/js/frappe/form/multi_select_dialog.js
View file

@ -150,8 +150,12 @@ frappe.ui.form.MultiSelectDialog = class MultiSelectDialog {
});
}
is_child_selection_enabled() {
return this.dialog.fields_dict['allow_child_item_selection'].get_value();
}
toggle_child_selection() {
if (this.dialog.fields_dict['allow_child_item_selection'].get_value()) {
if (this.is_child_selection_enabled()) {
this.show_child_results();
} else {
this.child_results = [];
@ -289,7 +293,11 @@ frappe.ui.form.MultiSelectDialog = class MultiSelectDialog {
parent: this.dialog.get_field('filter_area').$wrapper,
doctype: this.doctype,
on_change: () => {
this.get_results();
if (this.is_child_selection_enabled()) {
this.show_child_results();
} else {
this.get_results();
}
}
});
// 'Apply Filter' breaks since the filers are not in a popover
@ -325,7 +333,11 @@ frappe.ui.form.MultiSelectDialog = class MultiSelectDialog {
this.$parent.find('.input-with-feedback').on('change', () => {
frappe.flags.auto_scroll = false;
this.get_results();
if (this.is_child_selection_enabled()) {
this.show_child_results();
} else {
this.get_results();
}
});
this.$parent.find('[data-fieldtype="Data"]').on('input', () => {
@ -333,8 +345,12 @@ frappe.ui.form.MultiSelectDialog = class MultiSelectDialog {
clearTimeout($this.data('timeout'));
$this.data('timeout', setTimeout(function () {
frappe.flags.auto_scroll = false;
me.empty_list();
me.get_results();
if (me.is_child_selection_enabled()) {
me.show_child_results();
} else {
me.empty_list();
me.get_results();
}
}, 300));
});
}

3
frappe/public/js/frappe/model/model.js
View file

@ -10,8 +10,7 @@ $.extend(frappe.model, {
layout_fields: ['Section Break', 'Column Break', 'Tab Break', 'Fold'],
std_fields_list: ['name', 'owner', 'creation', 'modified', 'modified_by',
'_user_tags', '_comments', '_assign', '_liked_by', 'docstatus',
'parent', 'parenttype', 'parentfield', 'idx'],
'_user_tags', '_comments', '_assign', '_liked_by', 'docstatus', 'idx'],
core_doctypes_list: ['DocType', 'DocField', 'DocPerm', 'User', 'Role', 'Has Role',
'Page', 'Module Def', 'Print Format', 'Report', 'Customize Form',

6
frappe/public/js/frappe/router.js
View file

@ -250,12 +250,6 @@ frappe.router = {
}
},
clear_re_route(doctype, docname) {
delete frappe.re_route[
`${encodeURIComponent(frappe.router.slug(doctype))}/${encodeURIComponent(docname)}`
];
},
set_title(sub_path) {
if (frappe.route_titles[sub_path]) {
frappe.utils.set_title(frappe.route_titles[sub_path]);

11
frappe/public/js/frappe/views/breadcrumbs.js
View file

@ -70,6 +70,9 @@ frappe.breadcrumbs = {
this.set_form_breadcrumb(breadcrumbs, view);
} else if (breadcrumbs.doctype && view === 'list') {
this.set_list_breadcrumb(breadcrumbs);
} else if (breadcrumbs.doctype && view == 'dashboard-view') {
this.set_list_breadcrumb(breadcrumbs);
this.set_dashboard_breadcrumb(breadcrumbs);
}
}
@ -164,6 +167,14 @@ frappe.breadcrumbs = {
},
set_dashboard_breadcrumb(breadcrumbs) {
const doctype = breadcrumbs.doctype;
const docname = frappe.get_route()[1];
let dashboard_route = `/app/${frappe.router.slug(doctype)}/${docname}`;
$(`<li><a href="${dashboard_route}">${__(docname)}</a></li>`)
.appendTo(this.$breadcrumbs);
},
setup_modules() {
if (!frappe.visible_modules) {
frappe.visible_modules = $.map(frappe.boot.allowed_workspaces, (m) => {

2
frappe/public/js/frappe/views/reports/report_view.js
View file

@ -866,7 +866,7 @@ frappe.views.ReportView = class ReportView extends frappe.views.ListView {
}
doctype_fields = [{
label: __('ID'),
label: __('ID', null, 'Label of name column in report'),
fieldname: 'name',
fieldtype: 'Data',
reqd: 1

29
frappe/public/js/integrations/google_drive_picker.js
View file

@ -44,9 +44,16 @@ export default class GoogleDrivePicker {
}
handleAuthResult(authResult) {
let error_map = {
"popup_closed_by_user": __("Google Authentication was closed abruptly by the user")
};
if (authResult && !authResult.error) {
frappe.boot.user.google_drive_token = authResult.access_token;
this.createPicker();
} else {
let error = error_map[authResult.error] || __("Google Authentication Error");
frappe.throw(error);
}
}
@ -58,20 +65,34 @@ export default class GoogleDrivePicker {
createPicker() {
// Create and render a Picker object for searching images.
if (this.pickerApiLoaded && frappe.boot.user.google_drive_token) {
var view = new google.picker.DocsView(google.picker.ViewId.DOCS)
this.view = new google.picker.DocsView(google.picker.ViewId.DOCS)
.setParent('root') // show the root folder by default
.setIncludeFolders(true); // also show folders, not just files
var picker = new google.picker.PickerBuilder()
this.picker = new google.picker.PickerBuilder()
.setAppId(this.appId)
.setDeveloperKey(this.developerKey)
.setOAuthToken(frappe.boot.user.google_drive_token)
.addView(view)
.addView(this.view)
.setLocale(frappe.boot.lang)
.setCallback(this.pickerCallback)
.build();
picker.setVisible(true);
this.picker.setVisible(true);
this.setupHide();
}
}
setupHide() {
let bg = $(".picker-dialog-bg");
for (let el of bg) {
el.onclick = () => {
this.picker.setVisible(false);
this.picker.Ob({
action: google.picker.Action.CANCEL
});
};
}
}
}

5
frappe/public/js/lib/moment.js Normal file
View file

@ -0,0 +1,5 @@
// This file is used to make sure that `moment` is bound to the window
// before the bundle finishes loading, due to imports (datetime.js) in the bundle
// that depend on `moment`.
import momentTimezone from "moment-timezone/builds/moment-timezone-with-data.js";
window.moment = momentTimezone;

5
frappe/public/js/libs.bundle.js
View file

@ -1,15 +1,12 @@
import "./jquery-bootstrap";
import Vue from "vue/dist/vue.esm.js";
import moment from "moment/min/moment-with-locales.js";
import momentTimezone from "moment-timezone/builds/moment-timezone-with-data.js";
import "./lib/moment";
import io from "socket.io-client/dist/socket.io.slim.js";
import Sortable from "./lib/Sortable.min.js";
// TODO: esbuild
// Don't think jquery.hotkeys is being used anywhere. Will remove this after being sure.
// import "./lib/jquery/jquery.hotkeys.js";
window.moment = moment;
window.moment = momentTimezone;
window.Vue = Vue;
window.Sortable = Sortable;
window.io = io;

4
frappe/public/js/print_format_builder/print_format_builder.bundle.js
View file

@ -21,7 +21,7 @@ class PrintFormatBuilder {
this.$component.toggle_preview();
}
);
this.page.add_button(__("Reset Changes"), () =>
let $reset_changes_btn = this.page.add_button(__("Reset Changes"), () =>
this.$component.$store.reset_changes()
);
this.page.add_menu_item(__("Edit Print Format"), () => {
@ -46,9 +46,11 @@ class PrintFormatBuilder {
if (value) {
this.page.set_indicator("Not Saved", "orange");
$toggle_preview_btn.hide();
$reset_changes_btn.show();
} else {
this.page.clear_indicator();
$toggle_preview_btn.show();
$reset_changes_btn.hide();
}
});
this.$component.$watch("show_preview", value => {

1
frappe/public/js/web_form.bundle.js
View file

@ -1,2 +1,3 @@
import "./lib/moment.js";
import "./frappe/utils/datetime.js";
import "./frappe/web_form/webform_script.js";

1
frappe/public/scss/common/css_variables.scss
View file

@ -249,6 +249,7 @@
--checkbox-right-margin: var(--margin-xs);
--checkbox-size: 14px;
--checkbox-focus-shadow: 0 0 0 2px var(--gray-300);
--checkbox-gradient: linear-gradient(180deg, #4AC3F8 -124.51%, var(--primary) 100%);
--right-arrow-svg: url("data: image/svg+xml;utf8, <svg width='6' height='8' viewBox='0 0 6 8' fill='none' xmlns='http://www.w3.org/2000/svg'><path d='M1.25 7.5L4.75 4L1.25 0.5' stroke='%231F272E' stroke-linecap='round' stroke-linejoin='round'/></svg>");
--left-arrow-svg: url("data: image/svg+xml;utf8, <svg width='6' height='8' viewBox='0 0 6 8' fill='none' xmlns='http://www.w3.org/2000/svg'><path d='M7.5 9.5L4 6l3.5-3.5' stroke='%231F272E' stroke-linecap='round' stroke-linejoin='round'></path></svg>");

6
frappe/public/scss/common/global.scss
View file

@ -54,7 +54,7 @@ input[type="radio"] {
}
&:checked::before {
background-color: var(--blue-500);
background-color: var(--primary);
border-radius: 16px;
box-shadow: inset 0 0 0 2px white;
}
@ -85,8 +85,8 @@ input[type="checkbox"] {
}
&:checked {
background-color: var(--blue-500);
background-image: $check-icon, linear-gradient(180deg, #4AC3F8 -124.51%, #2490EF 100%);
background-color: var(--primary);
background-image: $check-icon, var(--checkbox-gradient);
background-size: 57%, 100%;
box-shadow: none;
border: none;

2
frappe/public/scss/desk/css_variables.scss
View file

@ -54,4 +54,6 @@ $input-height: 28px !default;
// skeleton
--skeleton-bg: var(--gray-100);
// progress bar
--progress-bar-bg: var(--primary);
}

3
frappe/public/scss/desk/variables.scss
View file

@ -118,6 +118,9 @@ $custom-control-label-color: var(--text-color);
$custom-switch-indicator-size: 8px;
$custom-control-indicator-border-width: 2px;
// progress bar
$progress-bar-bg: var(--progress-bar-bg);
$navbar-nav-link-padding-x: 1rem !default;
$navbar-padding-y: 1rem !default;
$card-border-radius: 0.75rem !default;

21
frappe/query_builder/__init__.py
View file

@ -1,8 +1,21 @@
from frappe.query_builder.terms import ParameterizedValueWrapper, ParameterizedFunction
import pypika
import pypika.terms
from pypika import *
from pypika import Field
from pypika.utils import ignore_copy
from frappe.query_builder.terms import ParameterizedFunction, ParameterizedValueWrapper
from frappe.query_builder.utils import (
Column,
DocType,
get_query_builder,
patch_query_aggregation,
patch_query_execute,
)
pypika.terms.ValueWrapper = ParameterizedValueWrapper
pypika.terms.Function = ParameterizedFunction
from pypika import *
from frappe.query_builder.utils import Column, DocType, get_query_builder, patch_query_execute, patch_query_aggregation
# * Overrides the field() method and replaces it with the a `PseudoColumn` 'field' for consistency
pypika.queries.Selectable.__getattr__ = ignore_copy(lambda table, x: Field(x, table=table))
pypika.queries.Selectable.__getitem__ = ignore_copy(lambda table, x: Field(x, table=table))
pypika.queries.Selectable.field = pypika.terms.PseudoColumn("field")

20
frappe/query_builder/builder.py
View file

@ -1,8 +1,12 @@
from pypika import MySQLQuery, Order, PostgreSQLQuery, terms
from pypika.queries import Schema, Table
from frappe.utils import get_table_name
from pypika.dialects import MySQLQueryBuilder, PostgreSQLQueryBuilder
from pypika.queries import QueryBuilder, Schema, Table
from pypika.terms import Function
from frappe.query_builder.terms import ParameterizedValueWrapper
from frappe.utils import get_table_name
class Base:
terms = terms
desc = Order.desc
@ -19,13 +23,13 @@ class Base:
return Table(table_name, *args, **kwargs)
@classmethod
def into(cls, table, *args, **kwargs):
def into(cls, table, *args, **kwargs) -> QueryBuilder:
if isinstance(table, str):
table = cls.DocType(table)
return super().into(table, *args, **kwargs)
@classmethod
def update(cls, table, *args, **kwargs):
def update(cls, table, *args, **kwargs) -> QueryBuilder:
if isinstance(table, str):
table = cls.DocType(table)
return super().update(table, *args, **kwargs)
@ -34,6 +38,10 @@ class Base:
class MariaDB(Base, MySQLQuery):
Field = terms.Field
@classmethod
def _builder(cls, *args, **kwargs) -> "MySQLQueryBuilder":
return super()._builder(*args, wrapper_cls=ParameterizedValueWrapper, **kwargs)
@classmethod
def from_(cls, table, *args, **kwargs):
if isinstance(table, str):
@ -53,6 +61,10 @@ class Postgres(Base, PostgreSQLQuery):
# they are two different objects. The quick fix used here is to replace the
# Field names in the "Field" function.
@classmethod
def _builder(cls, *args, **kwargs) -> "PostgreSQLQueryBuilder":
return super()._builder(*args, wrapper_cls=ParameterizedValueWrapper, **kwargs)
@classmethod
def Field(cls, field_name, *args, **kwargs):
if field_name in cls.field_translation:

87
frappe/query_builder/terms.py
View file

@ -1,33 +1,77 @@
from datetime import timedelta
from typing import Any, Dict, Optional
from frappe.utils.data import format_timedelta
from pypika.terms import Function, ValueWrapper
from pypika.utils import format_alias_sql
class NamedParameterWrapper():
def __init__(self, parameters: Dict[str, Any]):
self.parameters = parameters
class NamedParameterWrapper:
"""Utility class to hold parameter values and keys"""
def update_parameters(self, param_key: Any, param_value: Any, **kwargs):
def __init__(self) -> None:
self.parameters = {}
def get_sql(self, param_value: Any, **kwargs) -> str:
"""returns SQL for a parameter, while adding the real value in a dict
Args:
param_value (Any): Value of the parameter
Returns:
str: parameter used in the SQL query
"""
param_key = f"%(param{len(self.parameters) + 1})s"
self.parameters[param_key[2:-2]] = param_value
return param_key
def get_sql(self, **kwargs):
return f'%(param{len(self.parameters) + 1})s'
def get_parameters(self) -> Dict[str, Any]:
"""get dict with parameters and values
Returns:
Dict[str, Any]: parameter dict
"""
return self.parameters
class ParameterizedValueWrapper(ValueWrapper):
def get_sql(self, quote_char: Optional[str] = None, secondary_quote_char: str = "'", param_wrapper= None, **kwargs: Any) -> str:
if param_wrapper is None:
sql = self.get_value_sql(quote_char=quote_char, secondary_quote_char=secondary_quote_char, **kwargs)
return format_alias_sql(sql, self.alias, quote_char=quote_char, **kwargs)
"""
Class to monkey patch ValueWrapper
Adds functionality to parameterize queries when a `param wrapper` is passed in get_sql()
"""
def get_sql(
self,
quote_char: Optional[str] = None,
secondary_quote_char: str = "'",
param_wrapper: Optional[NamedParameterWrapper] = None,
**kwargs: Any,
) -> str:
if param_wrapper and isinstance(self.value, str):
# add quotes if it's a string value
value_sql = self.get_value_sql(quote_char=quote_char, **kwargs)
sql = param_wrapper.get_sql(param_value=value_sql, **kwargs)
else:
value_sql = self.get_value_sql(quote_char=quote_char, **kwargs) if not isinstance(self.value,int) else self.value
param_sql = param_wrapper.get_sql(**kwargs)
param_wrapper.update_parameters(param_key=param_sql, param_value=value_sql, **kwargs)
return format_alias_sql(param_sql, self.alias, quote_char=quote_char, **kwargs)
# * BUG: pypika doesen't parse timedeltas
if isinstance(self.value, timedelta):
self.value = format_timedelta(self.value)
sql = self.get_value_sql(
quote_char=quote_char,
secondary_quote_char=secondary_quote_char,
param_wrapper=param_wrapper,
**kwargs,
)
return format_alias_sql(sql, self.alias, quote_char=quote_char, **kwargs)
class ParameterizedFunction(Function):
"""
Class to monkey patch pypika.terms.Functions
Only to pass `param_wrapper` in `get_function_sql`.
"""
def get_sql(self, **kwargs: Any) -> str:
with_alias = kwargs.pop("with_alias", False)
with_namespace = kwargs.pop("with_namespace", False)
@ -35,15 +79,24 @@ class ParameterizedFunction(Function):
dialect = kwargs.pop("dialect", None)
param_wrapper = kwargs.pop("param_wrapper", None)
function_sql = self.get_function_sql(with_namespace=with_namespace, quote_char=quote_char, param_wrapper=param_wrapper, dialect=dialect)
function_sql = self.get_function_sql(
with_namespace=with_namespace,
quote_char=quote_char,
param_wrapper=param_wrapper,
dialect=dialect,
)
if self.schema is not None:
function_sql = "{schema}.{function}".format(
schema=self.schema.get_sql(quote_char=quote_char, dialect=dialect, **kwargs),
schema=self.schema.get_sql(
quote_char=quote_char, dialect=dialect, **kwargs
),
function=function_sql,
)
if with_alias:
return format_alias_sql(function_sql, self.alias, quote_char=quote_char, **kwargs)
return format_alias_sql(
function_sql, self.alias, quote_char=quote_char, **kwargs
)
return function_sql

38
frappe/query_builder/utils.py
View file

@ -1,17 +1,17 @@
from enum import Enum
from typing import Any, Callable, Dict, Union, get_type_hints
from importlib import import_module
from typing import Any, Callable, Dict, Union, get_type_hints
from pypika import Query
from pypika.queries import Column
import frappe
from .builder import MariaDB, Postgres
from pypika.terms import PseudoColumn
import frappe
from frappe.query_builder.terms import NamedParameterWrapper
from .builder import MariaDB, Postgres
class db_type_is(Enum):
MARIADB = "mariadb"
POSTGRES = "postgres"
@ -59,11 +59,29 @@ def patch_query_execute():
return frappe.db.sql(query, params, *args, **kwargs) # nosemgrep
def prepare_query(query):
params = {}
query = query.get_sql(param_wrapper = NamedParameterWrapper(params))
import inspect
param_collector = NamedParameterWrapper()
query = query.get_sql(param_wrapper=param_collector)
if frappe.flags.in_safe_exec and not query.lower().strip().startswith("select"):
raise frappe.PermissionError('Only SELECT SQL allowed in scripting')
return query, params
callstack = inspect.stack()
if len(callstack) >= 3 and ".py" in callstack[2].filename:
# ignore any query builder methods called from python files
# assumption is that those functions are whitelisted already.
# since query objects are patched everywhere any query.run()
# will have callstack like this:
# frame0: this function prepare_query()
# frame1: execute_query()
# frame2: frame that called `query.run()`
#
# if frame2 is server script it wont have a filename and hence
# it shouldn't be allowed.
# ps. stack() returns `"<unknown>"` as filename.
pass
else:
raise frappe.PermissionError('Only SELECT SQL allowed in scripting')
return query, param_collector.get_parameters()
query_class = get_attr(str(frappe.qb).split("'")[1])
builder_class = get_type_hints(query_class._builder).get('return')
@ -78,7 +96,7 @@ def patch_query_execute():
def patch_query_aggregation():
"""Patch aggregation functions to frappe.qb
"""
from frappe.query_builder.functions import _max, _min, _avg, _sum
from frappe.query_builder.functions import _avg, _max, _min, _sum
frappe.qb.max = _max
frappe.qb.min = _min

4
frappe/search/full_text_search.py
View file

@ -66,7 +66,7 @@ class FullTextSearch:
ix = self.get_index()
with ix.searcher():
writer = ix.writer()
writer = AsyncWriter(ix)
writer.delete_by_term(self.id, doc_name)
writer.commit(optimize=True)
@ -98,7 +98,7 @@ class FullTextSearch:
def build_index(self):
"""Build index for all parsed documents"""
ix = self.create_index()
writer = ix.writer()
writer = AsyncWriter(ix)
for i, document in enumerate(self.documents):
if document:

6
frappe/social/doctype/energy_point_rule/energy_point_rule.py
View file

@ -59,9 +59,9 @@ class EnergyPointRule(Document):
# indicates that this was a new doc
return doc.get_doc_before_save() is None
if self.for_doc_event == 'Submit':
return doc.docstatus == 1
return doc.docstatus.is_submitted()
if self.for_doc_event == 'Cancel':
return doc.docstatus == 2
return doc.docstatus.is_cancelled()
if self.for_doc_event == 'Value Change':
field_to_check = self.field_to_check
if not field_to_check: return False
@ -96,7 +96,7 @@ def process_energy_points(doc, state):
old_doc = doc.get_doc_before_save()
# check if doc has been cancelled
if old_doc and old_doc.docstatus == 1 and doc.docstatus == 2:
if old_doc and old_doc.docstatus.is_submitted() and doc.docstatus.is_cancelled():
return revert_points_for_cancelled_doc(doc)
for d in frappe.cache_manager.get_doctype_map('Energy Point Rule', doc.doctype,

2
frappe/templates/base.html
View file

@ -105,8 +105,6 @@
// for backward compatibility of some libs
frappe.sys_defaults = frappe.boot.sysdefaults;
</script>
<script type="text/javascript" src="/assets/frappe/node_modules/moment/min/moment-with-locales.min.js"></script>
<script type="text/javascript" src="/assets/frappe/node_modules/moment-timezone/builds/moment-timezone-with-data.min.js"></script>
{{ include_script('frappe-web.bundle.js') }}
{% endblock %}

4
frappe/templates/print_format/print_format.css
View file

@ -57,6 +57,10 @@ body {
padding-left: {{ print_format.margin_left | int }}mm;
padding-bottom: {{ print_format.margin_bottom | int }}mm;
}
.child-table {
overflow-x: auto;
}
}
.section:not(:first-child) {

17
frappe/test_runner.py
View file

@ -30,7 +30,7 @@ def xmlrunner_wrapper(output):
def main(app=None, module=None, doctype=None, verbose=False, tests=(),
force=False, profile=False, junit_xml_output=None, ui_tests=False,
doctype_list_path=None, skip_test_records=False, failfast=False):
doctype_list_path=None, skip_test_records=False, failfast=False, case=None):
global unittest_runner
if doctype_list_path:
@ -76,7 +76,7 @@ def main(app=None, module=None, doctype=None, verbose=False, tests=(),
if doctype:
ret = run_tests_for_doctype(doctype, verbose, tests, force, profile, failfast=failfast, junit_xml_output=junit_xml_output)
elif module:
ret = run_tests_for_module(module, verbose, tests, profile, failfast=failfast, junit_xml_output=junit_xml_output)
ret = run_tests_for_module(module, verbose, tests, profile, failfast=failfast, junit_xml_output=junit_xml_output, case=case)
else:
ret = run_all_tests(app, verbose, profile, ui_tests, failfast=failfast, junit_xml_output=junit_xml_output)
@ -182,16 +182,16 @@ def run_tests_for_doctype(doctypes, verbose=False, tests=(), force=False, profil
return _run_unittest(modules, verbose=verbose, tests=tests, profile=profile, failfast=failfast, junit_xml_output=junit_xml_output)
def run_tests_for_module(module, verbose=False, tests=(), profile=False, failfast=False, junit_xml_output=False):
def run_tests_for_module(module, verbose=False, tests=(), profile=False, failfast=False, junit_xml_output=False, case=None):
module = importlib.import_module(module)
if hasattr(module, "test_dependencies"):
for doctype in module.test_dependencies:
make_test_records(doctype, verbose=verbose)
frappe.db.commit()
return _run_unittest(module, verbose=verbose, tests=tests, profile=profile, failfast=failfast, junit_xml_output=junit_xml_output)
return _run_unittest(module, verbose=verbose, tests=tests, profile=profile, failfast=failfast, junit_xml_output=junit_xml_output, case=case)
def _run_unittest(modules, verbose=False, tests=(), profile=False, failfast=False, junit_xml_output=False):
def _run_unittest(modules, verbose=False, tests=(), profile=False, failfast=False, junit_xml_output=False, case=None):
frappe.db.begin()
test_suite = unittest.TestSuite()
@ -200,7 +200,10 @@ def _run_unittest(modules, verbose=False, tests=(), profile=False, failfast=Fals
modules = [modules]
for module in modules:
module_test_cases = unittest.TestLoader().loadTestsFromModule(module)
if case:
module_test_cases = unittest.TestLoader().loadTestsFromTestCase(getattr(module, case))
else:
module_test_cases = unittest.TestLoader().loadTestsFromModule(module)
if tests:
for each in module_test_cases:
for test_case in each.__dict__["_tests"]:
@ -337,7 +340,7 @@ def make_test_records_for_doctype(doctype, verbose=0, force=False):
elif hasattr(test_module, "test_records"):
if doctype in frappe.local.test_objects:
frappe.local.test_objects[doctype] += make_test_objects(doctype, test_module.test_records, verbose, force)
else:
else:
frappe.local.test_objects[doctype] = make_test_objects(doctype, test_module.test_records, verbose, force)
else:

18
frappe/tests/test_base_document.py Normal file
View file

@ -0,0 +1,18 @@
import unittest
from frappe.model.base_document import BaseDocument
class TestBaseDocument(unittest.TestCase):
def test_docstatus(self):
doc = BaseDocument({"docstatus": 0})
self.assertTrue(doc.docstatus.is_draft())
self.assertEquals(doc.docstatus, 0)
doc.docstatus = 1
self.assertTrue(doc.docstatus.is_submitted())
self.assertEquals(doc.docstatus, 1)
doc.docstatus = 2
self.assertTrue(doc.docstatus.is_cancelled())
self.assertEquals(doc.docstatus, 2)

66
frappe/tests/test_child_table.py Normal file
View file

@ -0,0 +1,66 @@
import frappe
from frappe.model import child_table_fields
import unittest
from typing import Callable
class TestChildTable(unittest.TestCase):
def tearDown(self) -> None:
try:
frappe.delete_doc("DocType", self.doctype_name, force=1)
except Exception:
pass
def test_child_table_doctype_creation_and_transitioning(self) -> None:
'''
This method tests the creation of child table doctype
as well as it's transitioning from child table to normal and normal to child table doctype
'''
self.doctype_name = "Test Newy Child Table"
try:
doc = frappe.get_doc({
"doctype": "DocType",
"name": self.doctype_name,
"istable": 1,
"custom": 1,
"module": "Integrations",
"fields": [{
"label": "Some Field",
"fieldname": "some_fieldname",
"fieldtype": "Data",
"reqd": 1
}]
}).insert(ignore_permissions=True)
except Exception:
self.fail("Not able to create Child Table Doctype")
for column in child_table_fields:
self.assertTrue(frappe.db.has_column(self.doctype_name, column))
# check transitioning from child table to normal doctype
doc.istable = 0
try:
doc.save(ignore_permissions=True)
except Exception:
self.fail("Not able to transition from Child Table Doctype to Normal Doctype")
self.check_valid_columns(self.assertFalse)
# check transitioning from normal to child table doctype
doc.istable = 1
try:
doc.save(ignore_permissions=True)
except Exception:
self.fail("Not able to transition from Normal Doctype to Child Table Doctype")
self.check_valid_columns(self.assertTrue)
def check_valid_columns(self, assertion_method: Callable) -> None:
valid_columns = frappe.get_meta(self.doctype_name).get_valid_columns()
for column in child_table_fields:
assertion_method(column in valid_columns)

30
frappe/tests/test_client.py
View file

@ -101,3 +101,33 @@ class TestClient(unittest.TestCase):
execute_cmd,
frappe.local.form_dict.cmd
)
def test_array_values_in_request_args(self):
import requests
from frappe.auth import CookieManager, LoginManager
frappe.utils.set_request(path="/")
frappe.local.cookie_manager = CookieManager()
frappe.local.login_manager = LoginManager()
frappe.local.login_manager.login_as('Administrator')
params = {
'doctype': 'DocType',
'fields': ['name', 'modified'],
'sid': frappe.session.sid,
}
headers = {
'accept': 'application/json',
'content-type': 'application/json',
}
url = f'http://{frappe.local.site}:{frappe.conf.webserver_port}/api/method/frappe.client.get_list'
res = requests.post(
url,
json=params,
headers=headers
)
self.assertEqual(res.status_code, 200)
data = res.json()
first_item = data['message'][0]
self.assertTrue('name' in first_item)
self.assertTrue('modified' in first_item)
frappe.local.login_manager.logout()

182
frappe/tests/test_db.py
View file

@ -1,21 +1,21 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors
# License: MIT. See LICENSE
import datetime
import inspect
import unittest
from random import choice
import datetime
from unittest.mock import patch
import frappe
from frappe.custom.doctype.custom_field.custom_field import create_custom_field
from frappe.utils import random_string
from frappe.utils.testutils import clear_custom_fields
from frappe.query_builder import Field
from frappe.database import savepoint
from .test_query_builder import run_only_if, db_type_is
from frappe.database.database import Database
from frappe.query_builder import Field
from frappe.query_builder.functions import Concat_ws
from frappe.tests.test_query_builder import db_type_is, run_only_if
from frappe.utils import add_days, now, random_string
from frappe.utils.testutils import clear_custom_fields
class TestDB(unittest.TestCase):
@ -84,20 +84,6 @@ class TestDB(unittest.TestCase):
),
)
def test_set_value(self):
todo1 = frappe.get_doc(dict(doctype='ToDo', description = 'test_set_value 1')).insert()
todo2 = frappe.get_doc(dict(doctype='ToDo', description = 'test_set_value 2')).insert()
frappe.db.set_value('ToDo', todo1.name, 'description', 'test_set_value change 1')
self.assertEqual(frappe.db.get_value('ToDo', todo1.name, 'description'), 'test_set_value change 1')
# multiple set-value
frappe.db.set_value('ToDo', dict(description=('like', '%test_set_value%')),
'description', 'change 2')
self.assertEqual(frappe.db.get_value('ToDo', todo1.name, 'description'), 'change 2')
self.assertEqual(frappe.db.get_value('ToDo', todo2.name, 'description'), 'change 2')
def test_escape(self):
frappe.db.escape("香港濟生堂製藥有限公司 - IT".encode("utf-8"))
@ -246,7 +232,6 @@ class TestDB(unittest.TestCase):
frappe.delete_doc(test_doctype, doc)
clear_custom_fields(test_doctype)
def test_savepoints(self):
frappe.db.rollback()
save_point = "todonope"
@ -294,6 +279,18 @@ class TestDB(unittest.TestCase):
for d in created_docs:
self.assertTrue(frappe.db.exists("ToDo", d))
def test_transaction_writes_error(self):
from frappe.database.database import Database
frappe.db.rollback()
frappe.db.MAX_WRITES_PER_TRANSACTION = 1
note = frappe.get_last_doc("ToDo")
note.description = "changed"
with self.assertRaises(frappe.TooManyWritesError) as tmw:
note.save()
frappe.db.MAX_WRITES_PER_TRANSACTION = Database.MAX_WRITES_PER_TRANSACTION
@run_only_if(db_type_is.MARIADB)
class TestDDLCommandsMaria(unittest.TestCase):
@ -353,6 +350,143 @@ class TestDDLCommandsMaria(unittest.TestCase):
self.assertEquals(len(indexs_in_table), 2)
class TestDBSetValue(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.todo1 = frappe.get_doc(doctype="ToDo", description="test_set_value 1").insert()
cls.todo2 = frappe.get_doc(doctype="ToDo", description="test_set_value 2").insert()
def test_update_single_doctype_field(self):
value = frappe.db.get_single_value("System Settings", "deny_multiple_sessions")
changed_value = not value
frappe.db.set_value("System Settings", "System Settings", "deny_multiple_sessions", changed_value)
current_value = frappe.db.get_single_value("System Settings", "deny_multiple_sessions")
self.assertEqual(current_value, changed_value)
changed_value = not current_value
frappe.db.set_value("System Settings", None, "deny_multiple_sessions", changed_value)
current_value = frappe.db.get_single_value("System Settings", "deny_multiple_sessions")
self.assertEqual(current_value, changed_value)
changed_value = not current_value
frappe.db.set_single_value("System Settings", "deny_multiple_sessions", changed_value)
current_value = frappe.db.get_single_value("System Settings", "deny_multiple_sessions")
self.assertEqual(current_value, changed_value)
def test_update_single_row_single_column(self):
frappe.db.set_value("ToDo", self.todo1.name, "description", "test_set_value change 1")
updated_value = frappe.db.get_value("ToDo", self.todo1.name, "description")
self.assertEqual(updated_value, "test_set_value change 1")
def test_update_single_row_multiple_columns(self):
description, status = "Upated by test_update_single_row_multiple_columns", "Closed"
frappe.db.set_value("ToDo", self.todo1.name, {
"description": description,
"status": status,
}, update_modified=False)
updated_desciption, updated_status = frappe.db.get_value("ToDo",
filters={"name": self.todo1.name},
fieldname=["description", "status"]
)
self.assertEqual(description, updated_desciption)
self.assertEqual(status, updated_status)
def test_update_multiple_rows_single_column(self):
frappe.db.set_value("ToDo", {"description": ("like", "%test_set_value%")}, "description", "change 2")
self.assertEqual(frappe.db.get_value("ToDo", self.todo1.name, "description"), "change 2")
self.assertEqual(frappe.db.get_value("ToDo", self.todo2.name, "description"), "change 2")
def test_update_multiple_rows_multiple_columns(self):
todos_to_update = frappe.get_all("ToDo", filters={
"description": ("like", "%test_set_value%"),
"status": ("!=", "Closed")
}, pluck="name")
frappe.db.set_value("ToDo", {
"description": ("like", "%test_set_value%"),
"status": ("!=", "Closed")
}, {
"status": "Closed",
"priority": "High"
})
test_result = frappe.get_all("ToDo", filters={"name": ("in", todos_to_update)}, fields=["status", "priority"])
self.assertTrue(all(x for x in test_result if x["status"] == "Closed"))
self.assertTrue(all(x for x in test_result if x["priority"] == "High"))
def test_update_modified_options(self):
self.todo2.reload()
todo = self.todo2
updated_description = f"{todo.description} - by `test_update_modified_options`"
custom_modified = datetime.datetime.fromisoformat(add_days(now(), 10))
custom_modified_by = "user_that_doesnt_exist@example.com"
frappe.db.set_value("ToDo", todo.name, "description", updated_description, update_modified=False)
self.assertEqual(updated_description, frappe.db.get_value("ToDo", todo.name, "description"))
self.assertEqual(todo.modified, frappe.db.get_value("ToDo", todo.name, "modified"))
frappe.db.set_value("ToDo", todo.name, "description", "test_set_value change 1", modified=custom_modified, modified_by=custom_modified_by)
self.assertTupleEqual(
(custom_modified, custom_modified_by),
frappe.db.get_value("ToDo", todo.name, ["modified", "modified_by"])
)
def test_for_update(self):
self.todo1.reload()
with patch.object(Database, "sql") as sql_called:
frappe.db.set_value(
self.todo1.doctype,
self.todo1.name,
"description",
f"{self.todo1.description}-edit by `test_for_update`"
)
first_query = sql_called.call_args_list[0].args[0]
second_query = sql_called.call_args_list[1].args[0]
self.assertTrue(sql_called.call_count == 2)
self.assertTrue("FOR UPDATE" in first_query)
if frappe.conf.db_type == "postgres":
from frappe.database.postgres.database import modify_query
self.assertTrue(modify_query("UPDATE `tabToDo` SET") in second_query)
if frappe.conf.db_type == "mariadb":
self.assertTrue("UPDATE `tabToDo` SET" in second_query)
def test_cleared_cache(self):
self.todo2.reload()
with patch.object(frappe, "clear_document_cache") as clear_cache:
frappe.db.set_value(
self.todo2.doctype,
self.todo2.name,
"description",
f"{self.todo2.description}-edit by `test_cleared_cache`"
)
clear_cache.assert_called()
def test_update_alias(self):
args = (self.todo1.doctype, self.todo1.name, "description", "Updated by `test_update_alias`")
kwargs = {"for_update": False, "modified": None, "modified_by": None, "update_modified": True, "debug": False}
self.assertTrue("return self.set_value(" in inspect.getsource(frappe.db.update))
with patch.object(Database, "set_value") as set_value:
frappe.db.update(*args, **kwargs)
set_value.assert_called_once()
set_value.assert_called_with(*args, **kwargs)
@classmethod
def tearDownClass(cls):
frappe.db.rollback()
@run_only_if(db_type_is.POSTGRES)
class TestDDLCommandsPost(unittest.TestCase):
test_table_name = "TestNotes"

9
frappe/tests/test_db_update.py
View file

@ -103,10 +103,7 @@ def get_other_fields_meta(meta):
default_fields_map = {
'name': ('Data', 0),
'owner': ('Data', 0),
'parent': ('Data', 0),
'parentfield': ('Data', 0),
'modified_by': ('Data', 0),
'parenttype': ('Data', 0),
'creation': ('Datetime', 0),
'modified': ('Datetime', 0),
'idx': ('Int', 8),
@ -117,8 +114,12 @@ def get_other_fields_meta(meta):
if meta.track_seen:
optional_fields.append('_seen')
child_table_fields_map = {}
if meta.istable:
child_table_fields_map.update({field: ('Data', 0) for field in frappe.db.CHILD_TABLE_COLUMNS})
optional_fields_map = {field: ('Text', 0) for field in optional_fields}
fields = dict(default_fields_map, **optional_fields_map)
fields = dict(default_fields_map, **optional_fields_map, **child_table_fields_map)
field_map = [frappe._dict({'fieldname': field, 'fieldtype': _type, 'length': _length}) for field, (_type, _length) in fields.items()]
return field_map

26
frappe/tests/test_docstatus.py Normal file
View file

@ -0,0 +1,26 @@
import unittest
from frappe.model.docstatus import DocStatus
class TestDocStatus(unittest.TestCase):
def test_draft(self):
self.assertEqual(DocStatus(0), DocStatus.draft())
self.assertTrue(DocStatus.draft().is_draft())
self.assertFalse(DocStatus.draft().is_cancelled())
self.assertFalse(DocStatus.draft().is_submitted())
def test_submitted(self):
self.assertEqual(DocStatus(1), DocStatus.submitted())
self.assertFalse(DocStatus.submitted().is_draft())
self.assertTrue(DocStatus.submitted().is_submitted())
self.assertFalse(DocStatus.submitted().is_cancelled())
def test_cancelled(self):
self.assertEqual(DocStatus(2), DocStatus.cancelled())
self.assertFalse(DocStatus.cancelled().is_draft())
self.assertFalse(DocStatus.cancelled().is_submitted())
self.assertTrue(DocStatus.cancelled().is_cancelled())

27
frappe/tests/test_email.py
View file

@ -3,6 +3,8 @@
import unittest, frappe, re, email
from frappe.email.doctype.email_account.test_email_account import TestEmailAccount
test_dependencies = ['Email Account']
class TestEmail(unittest.TestCase):
@ -173,12 +175,35 @@ class TestEmail(unittest.TestCase):
frappe.db.delete("Communication", {"sender": "sukh@yyy.com"})
with open(frappe.get_app_path('frappe', 'tests', 'data', 'email_with_image.txt'), 'r') as raw:
mails = email_account.get_inbound_mails(test_mails=[raw.read()])
messages = {
# append_to = ToDo
'"INBOX"': {
'latest_messages': [
raw.read()
],
'seen_status': {
2: 'UNSEEN'
},
'uid_list': [2]
}
}
email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
changed_flag = False
if not email_account.enable_incoming:
email_account.enable_incoming = True
changed_flag = True
mails = TestEmailAccount.mocked_get_inbound_mails(email_account, messages)
# mails = email_account.get_inbound_mails(test_mails=[raw.read()])
communication = mails[0].process()
self.assertTrue(re.search('''<img[^>]*src=["']/private/files/rtco1.png[^>]*>''', communication.content))
self.assertTrue(re.search('''<img[^>]*src=["']/private/files/rtco2.png[^>]*>''', communication.content))
if changed_flag:
email_account.enable_incoming = False
if __name__ == '__main__':
frappe.connect()

Some files were not shown because too many files have changed in this diff Show more