diff --git a/.github/frappe_linter/translation.py b/.github/frappe_linter/translation.py deleted file mode 100644 index 5d33355a1b..0000000000 --- a/.github/frappe_linter/translation.py +++ /dev/null @@ -1,34 +0,0 @@ -import re -import sys - -errors_encounter = 0 -pattern = re.compile(r"_\(([\"']{,3})(?P((?!\1).)*)\1(\s*,\s*context\s*=\s*([\"'])(?P((?!\5).)*)\5)*(\s*,\s*(.)*?\s*(,\s*([\"'])(?P((?!\11).)*)\11)*)*\)") -start_pattern = re.compile(r"_{1,2}\([\"']{1,3}") - -# skip first argument -files = sys.argv[1:] -files_to_scan = [_file for _file in files if _file.endswith(('.py', '.js'))] - -for _file in files_to_scan: - with open(_file, 'r') as f: - print(f'Checking: {_file}') - file_lines = f.readlines() - for line_number, line in enumerate(file_lines, 1): - start_matches = start_pattern.search(line) - if start_matches: - match = pattern.search(line) - if not match and line.endswith(',\n'): - # concat remaining text to validate multiline pattern - line = "".join(file_lines[line_number - 1:]) - line = line[start_matches.start() + 1:] - match = pattern.match(line) - - if not match: - errors_encounter += 1 - print(f'\nTranslation syntax error at line number: {line_number + 1}\n{line.strip()[:100]}') - -if errors_encounter > 0: - print('\nYou can visit "https://frappeframework.com/docs/user/en/translations" to resolve this error.') - sys.exit(1) -else: - print('\nGood To Go!') diff --git a/.github/helper/documentation.py b/.github/helper/documentation.py new file mode 100644 index 0000000000..3fc14ba61b --- /dev/null +++ b/.github/helper/documentation.py @@ -0,0 +1,48 @@ +import sys +import requests +from urllib.parse import urlparse + + +docs_repos = [ + "frappe_docs", + "erpnext_documentation", + "erpnext_com", + "frappe_io", +] + + +def uri_validator(x): + result = urlparse(x) + return all([result.scheme, result.netloc, result.path]) + +def docs_link_exists(body): + for line in body.splitlines(): + for word in line.split(): + if word.startswith('http') and uri_validator(word): + parsed_url = urlparse(word) + if parsed_url.netloc == "github.com": + _, org, repo, _type, ref = parsed_url.path.split('/') + if org == "frappe" and repo in docs_repos: + return True + + +if __name__ == "__main__": + pr = sys.argv[1] + response = requests.get("https://api.github.com/repos/frappe/frappe/pulls/{}".format(pr)) + + if response.ok: + payload = response.json() + title = payload.get("title", "").lower() + head_sha = payload.get("head", {}).get("sha") + body = payload.get("body", "").lower() + + if title.startswith("feat") and head_sha and "no-docs" not in body: + if docs_link_exists(body): + print("Documentation Link Found. You're Awesome! 🎉") + + else: + print("Documentation Link Not Found! ⚠️") + sys.exit(1) + + else: + print("Skipping documentation checks... 🏃") diff --git a/.github/helper/translation.py b/.github/helper/translation.py new file mode 100644 index 0000000000..340f4f8772 --- /dev/null +++ b/.github/helper/translation.py @@ -0,0 +1,60 @@ +import re +import sys + +errors_encounter = 0 +pattern = re.compile(r"_\(([\"']{,3})(?P((?!\1).)*)\1(\s*,\s*context\s*=\s*([\"'])(?P((?!\5).)*)\5)*(\s*,\s*(.)*?\s*(,\s*([\"'])(?P((?!\11).)*)\11)*)*\)") +words_pattern = re.compile(r"_{1,2}\([\"'`]{1,3}.*?[a-zA-Z]") +start_pattern = re.compile(r"_{1,2}\([f\"'`]{1,3}") +f_string_pattern = re.compile(r"_\(f[\"']") +starts_with_f_pattern = re.compile(r"_\(f") + +# skip first argument +files = sys.argv[1:] +files_to_scan = [_file for _file in files if _file.endswith(('.py', '.js'))] + +for _file in files_to_scan: + with open(_file, 'r') as f: + print(f'Checking: {_file}') + file_lines = f.readlines() + for line_number, line in enumerate(file_lines, 1): + if 'frappe-lint: disable-translate' in line: + continue + + start_matches = start_pattern.search(line) + if start_matches: + starts_with_f = starts_with_f_pattern.search(line) + + if starts_with_f: + has_f_string = f_string_pattern.search(line) + if has_f_string: + errors_encounter += 1 + print(f'\nF-strings are not supported for translations at line number {line_number + 1}\n{line.strip()[:100]}') + continue + else: + continue + + match = pattern.search(line) + error_found = False + + if not match and line.endswith(',\n'): + # concat remaining text to validate multiline pattern + line = "".join(file_lines[line_number - 1:]) + line = line[start_matches.start() + 1:] + match = pattern.match(line) + + if not match: + error_found = True + print(f'\nTranslation syntax error at line number {line_number + 1}\n{line.strip()[:100]}') + + if not error_found and not words_pattern.search(line): + error_found = True + print(f'\nTranslation is useless because it has no words at line number {line_number + 1}\n{line.strip()[:100]}') + + if error_found: + errors_encounter += 1 + +if errors_encounter > 0: + print('\nVisit "https://frappeframework.com/docs/user/en/translations" to learn about valid translation strings.') + sys.exit(1) +else: + print('\nGood To Go!') diff --git a/.github/workflows/docs-checker.yml b/.github/workflows/docs-checker.yml new file mode 100644 index 0000000000..cdf676dd67 --- /dev/null +++ b/.github/workflows/docs-checker.yml @@ -0,0 +1,24 @@ +name: 'Documentation Required' +on: + pull_request: + types: [ opened, synchronize, reopened, edited ] + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: 'Setup Environment' + uses: actions/setup-python@v2 + with: + python-version: 3.6 + + - name: 'Clone repo' + uses: actions/checkout@v2 + + - name: Validate Docs + env: + PR_NUMBER: ${{ github.event.number }} + run: | + pip install requests --quiet + python $GITHUB_WORKSPACE/.github/helper/documentation.py $PR_NUMBER diff --git a/.github/workflows/translation_linter.yml b/.github/workflows/translation_linter.yml index bfa34f8147..4becaebd6b 100644 --- a/.github/workflows/translation_linter.yml +++ b/.github/workflows/translation_linter.yml @@ -19,4 +19,4 @@ jobs: run: | git fetch origin $GITHUB_BASE_REF:$GITHUB_BASE_REF -q files=$(git diff --name-only --diff-filter=d $GITHUB_BASE_REF) - python $GITHUB_WORKSPACE/.github/frappe_linter/translation.py $files \ No newline at end of file + python $GITHUB_WORKSPACE/.github/helper/translation.py $files diff --git a/README.md b/README.md index 57d4ca243d..f99988ae79 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ Full-stack web application framework that uses Python and MariaDB on the server ## Contributing -1. [Pull Request Requirements](https://github.com/frappe/erpnext/wiki/Pull-Request-Guidelines) +1. [Contribution Guidelines](https://github.com/frappe/erpnext/wiki/Contribution-Guidelines) 1. [Translations](https://translate.erpnext.com) ### Website diff --git a/cypress/integration/file_uploader.js b/cypress/integration/file_uploader.js index 5e9a264189..f9f44675db 100644 --- a/cypress/integration/file_uploader.js +++ b/cypress/integration/file_uploader.js @@ -20,10 +20,14 @@ context('FileUploader', () => { open_upload_dialog(); cy.fixture('example.json').then(fileContent => { - cy.get_open_dialog().find('.file-upload-area').upload( - { fileContent, fileName: 'example.json', mimeType: 'application/json' }, - { subjectType: 'drag-n-drop' }, - ); + cy.get_open_dialog().find('.file-upload-area').upload({ + fileContent, + fileName: 'example.json', + mimeType: 'application/json' + }, { + subjectType: 'drag-n-drop', + force: true + }); cy.get_open_dialog().find('.file-info').should('contain', 'example.json'); cy.server(); cy.route('POST', '/api/method/upload_file').as('upload_file'); diff --git a/frappe/automation/doctype/assignment_rule/assignment_rule.js b/frappe/automation/doctype/assignment_rule/assignment_rule.js index 3e86f6cefa..e8d17527bf 100644 --- a/frappe/automation/doctype/assignment_rule/assignment_rule.js +++ b/frappe/automation/doctype/assignment_rule/assignment_rule.js @@ -2,6 +2,9 @@ // For license information, please see license.txt frappe.ui.form.on('Assignment Rule', { + onload: (frm) => { + frm.trigger('set_due_date_field_options'); + }, refresh: function(frm) { // refresh description frm.events.rule(frm); @@ -12,5 +15,25 @@ frappe.ui.form.on('Assignment Rule', { } else { frm.get_field('rule').set_description(__('Assign to the one who has the least assignments')); } + }, + document_type: (frm) => { + frm.trigger('set_due_date_field_options'); + }, + set_due_date_field_options: (frm) => { + let doctype = frm.doc.document_type; + let datetime_fields = []; + if (doctype) { + frappe.model.with_doctype(doctype, () => { + frappe.get_meta(doctype).fields.map((df) => { + if (['Date', 'Datetime'].includes(df.fieldtype)) { + datetime_fields.push({ label: df.label, value: df.fieldname }); + } + }); + if (datetime_fields) { + frm.set_df_property('due_date_based_on', 'options', datetime_fields); + } + frm.set_df_property('due_date_based_on', 'hidden', !datetime_fields.length); + }); + } } }); diff --git a/frappe/automation/doctype/assignment_rule/assignment_rule.json b/frappe/automation/doctype/assignment_rule/assignment_rule.json index eb79b9e3a8..858ad8aac4 100644 --- a/frappe/automation/doctype/assignment_rule/assignment_rule.json +++ b/frappe/automation/doctype/assignment_rule/assignment_rule.json @@ -1,4 +1,5 @@ { + "actions": [], "allow_rename": 1, "autoname": "Prompt", "creation": "2019-02-28 17:12:18.815830", @@ -8,6 +9,7 @@ "engine": "InnoDB", "field_order": [ "document_type", + "due_date_based_on", "priority", "disabled", "column_break_4", @@ -129,9 +131,18 @@ "label": "Assignment Days", "options": "Assignment Rule Day", "reqd": 1 + }, + { + "depends_on": "document_type", + "fieldname": "due_date_based_on", + "fieldtype": "Select", + "label": "Due Date Based On", + "description": "Value from this field will be set as the due date in the ToDo" } ], - "modified": "2019-09-25 14:52:12.214514", + "index_web_pages_for_search": 1, + "links": [], + "modified": "2020-10-13 06:48:54.019735", "modified_by": "Administrator", "module": "Automation", "name": "Assignment Rule", diff --git a/frappe/automation/doctype/assignment_rule/assignment_rule.py b/frappe/automation/doctype/assignment_rule/assignment_rule.py index c4bd49b870..cd70799361 100644 --- a/frappe/automation/doctype/assignment_rule/assignment_rule.py +++ b/frappe/automation/doctype/assignment_rule/assignment_rule.py @@ -19,14 +19,14 @@ class AssignmentRule(Document): repeated_days = get_repeated(assignment_days) frappe.throw(_("Assignment Day {0} has been repeated.").format(frappe.bold(repeated_days))) - def on_update(self): # pylint: disable=no-self-use - frappe.cache_manager.clear_doctype_map('Assignment Rule', self.document_type) + def on_update(self): + clear_assignment_rule_cache(self) - def after_rename(self, old, new, merge): # pylint: disable=no-self-use - frappe.cache_manager.clear_doctype_map('Assignment Rule', self.document_type) + def after_rename(self, old, new, merge): + clear_assignment_rule_cache(self) - def on_trash(self): # pylint: disable=no-self-use - frappe.cache_manager.clear_doctype_map('Assignment Rule', self.document_type) + def on_trash(self): + clear_assignment_rule_cache(self) def apply_unassign(self, doc, assignments): if (self.unassign_condition and @@ -53,7 +53,8 @@ class AssignmentRule(Document): name = doc.get('name'), description = frappe.render_template(self.description, doc), assignment_rule = self.name, - notify = True + notify = True, + date = doc.get(self.due_date_based_on) if self.due_date_based_on else None )) # set for reference in round robin @@ -188,7 +189,7 @@ def apply(doc, method=None, doctype=None, name=None): # multiple auto assigns for d in assignment_rules: - assignment_rule_docs.append(frappe.get_doc('Assignment Rule', d.get('name'))) + assignment_rule_docs.append(frappe.get_cached_doc('Assignment Rule', d.get('name'))) if not assignment_rule_docs: return @@ -237,6 +238,40 @@ def apply(doc, method=None, doctype=None, name=None): break assignment_rule.close_assignments(doc) +def update_due_date(doc, state=None): + # called from hook + if (frappe.flags.in_patch + or frappe.flags.in_install + or frappe.flags.in_migrate + or frappe.flags.in_import + or frappe.flags.in_setup_wizard): + return + assignment_rules = frappe.cache_manager.get_doctype_map('Assignment Rule', 'due_date_rules_for_' + doc.doctype, dict( + document_type = doc.doctype, + disabled = 0, + due_date_based_on = ['is', 'set'] + )) + for rule in assignment_rules: + rule_doc = frappe.get_cached_doc('Assignment Rule', rule.get('name')) + due_date_field = rule_doc.due_date_based_on + if doc.meta.has_field(due_date_field) and \ + doc.has_value_changed(due_date_field) and rule.get('name'): + assignment_todos = frappe.get_all('ToDo', { + 'assignment_rule': rule.get('name'), + 'status': 'Open', + 'reference_type': doc.doctype, + 'reference_name': doc.name + }) + for todo in assignment_todos: + todo_doc = frappe.get_doc('ToDo', todo.name) + todo_doc.date = doc.get(due_date_field) + todo_doc.flags.updater_reference = { + 'doctype': 'Assignment Rule', + 'docname': rule.get('name'), + 'label': _('via Assignment Rule') + } + todo_doc.save(ignore_permissions=True) + def get_assignment_rules(): return [d.document_type for d in frappe.db.get_all('Assignment Rule', fields=['document_type'], filters=dict(disabled = 0))] @@ -250,3 +285,7 @@ def get_repeated(values): if value not in diff: diff.append(str(value)) return " ".join(diff) + +def clear_assignment_rule_cache(rule): + frappe.cache_manager.clear_doctype_map('Assignment Rule', rule.document_type) + frappe.cache_manager.clear_doctype_map('Assignment Rule', 'due_date_rules_for_' + rule.document_type) \ No newline at end of file diff --git a/frappe/automation/doctype/assignment_rule/test_assignment_rule.py b/frappe/automation/doctype/assignment_rule/test_assignment_rule.py index 7c68e63d95..dab842ad83 100644 --- a/frappe/automation/doctype/assignment_rule/test_assignment_rule.py +++ b/frappe/automation/doctype/assignment_rule/test_assignment_rule.py @@ -20,6 +20,7 @@ class TestAutoAssign(unittest.TestCase): dict(day = 'Friday'), dict(day = 'Saturday'), ] + self.days = days self.assignment_rule = get_assignment_rule([days, days]) clear_assignments() @@ -180,6 +181,55 @@ class TestAutoAssign(unittest.TestCase): status = 'Open' ), 'owner'), ['test3@example.com']) + def test_assignment_rule_condition(self): + frappe.db.sql("DELETE FROM `tabAssignment Rule`") + + # Add expiry_date custom field + from frappe.custom.doctype.custom_field.custom_field import create_custom_field + df = dict(fieldname='expiry_date', label='Expiry Date', fieldtype='Date') + create_custom_field('Note', df) + + assignment_rule = frappe.get_doc(dict( + name = 'Assignment with Due Date', + doctype = 'Assignment Rule', + document_type = 'Note', + assign_condition = 'public == 0', + due_date_based_on = 'expiry_date', + assignment_days = self.days, + users = [ + dict(user = 'test@example.com'), + ] + )).insert() + + expiry_date = frappe.utils.add_days(frappe.utils.nowdate(), 2) + note1 = make_note({'expiry_date': expiry_date}) + note2 = make_note({'expiry_date': expiry_date}) + + note1_todo = frappe.get_all('ToDo', filters=dict( + reference_type = 'Note', + reference_name = note1.name, + status = 'Open' + ))[0] + + note1_todo_doc = frappe.get_doc('ToDo', note1_todo.name) + self.assertEqual(frappe.utils.get_date_str(note1_todo_doc.date), expiry_date) + + # due date should be updated if the reference doc's date is updated. + note1.expiry_date = frappe.utils.add_days(expiry_date, 2) + note1.save() + note1_todo_doc.reload() + self.assertEqual(frappe.utils.get_date_str(note1_todo_doc.date), note1.expiry_date) + + # saving one note's expiry should not update other note todo's due date + note2_todo = frappe.get_all('ToDo', filters=dict( + reference_type = 'Note', + reference_name = note2.name, + status = 'Open' + ), fields=['name', 'date'])[0] + self.assertNotEqual(frappe.utils.get_date_str(note2_todo.date), note1.expiry_date) + self.assertEqual(frappe.utils.get_date_str(note2_todo.date), expiry_date) + assignment_rule.delete() + def clear_assignments(): frappe.db.sql("delete from tabToDo where reference_type = 'Note'") diff --git a/frappe/build.py b/frappe/build.py index 767217a9b9..f14b250a92 100644 --- a/frappe/build.py +++ b/frappe/build.py @@ -40,6 +40,7 @@ def build_missing_files(): # check which files dont exist yet from the build.json and tell build.js to build only those! missing_assets = [] current_asset_files = [] + frappe_build = os.path.join("..", "apps", "frappe", "frappe", "public", "build.json") for type in ["css", "js"]: current_asset_files.extend( @@ -49,7 +50,7 @@ def build_missing_files(): ] ) - with open(os.path.join(sites_path, "assets", "frappe", "build.json")) as f: + with open(frappe_build) as f: all_asset_files = json.load(f).keys() for asset in all_asset_files: @@ -111,13 +112,21 @@ def download_frappe_assets(verbose=True): if assets_archive: import tarfile + directories_created = set() click.secho("\nExtracting assets...\n", fg="yellow") with tarfile.open(assets_archive) as tar: for file in tar: if not file.isdir(): dest = "." + file.name.replace("./frappe-bench/sites", "") + asset_directory = os.path.dirname(dest) show = dest.replace("./assets/", "") + + if asset_directory not in directories_created: + if not os.path.exists(asset_directory): + os.makedirs(asset_directory, exist_ok=True) + directories_created.add(asset_directory) + tar.makefile(file, dest) print("{0} Restored {1}".format(green('✔'), show)) diff --git a/frappe/commands/site.py b/frappe/commands/site.py index fe8b238f32..1f4642658f 100755 --- a/frappe/commands/site.py +++ b/frappe/commands/site.py @@ -1,10 +1,5 @@ # imports - standard imports -import atexit -import compileall -import hashlib import os -import re -import shutil import sys # imports - third party imports @@ -13,9 +8,7 @@ import click # imports - module imports import frappe from frappe.commands import get_site, pass_context -from frappe.commands.scheduler import _is_scheduler_enabled from frappe.exceptions import SiteNotSpecifiedError -from frappe.installer import update_site_config from frappe.utils import get_site_path, touch_file @@ -64,8 +57,10 @@ def _new_site(db_name, site, mariadb_root_username=None, mariadb_root_password=N sys.exit(1) if not db_name: + import hashlib db_name = '_' + hashlib.sha1(site.encode()).hexdigest()[:16] + from frappe.commands.scheduler import _is_scheduler_enabled from frappe.installer import install_db, make_site_dirs from frappe.installer import install_app as _install_app import frappe.utils.scheduler @@ -73,6 +68,7 @@ def _new_site(db_name, site, mariadb_root_username=None, mariadb_root_password=N frappe.init(site=site) try: + # enable scheduler post install? enable_scheduler = _is_scheduler_enabled() except Exception: @@ -107,11 +103,11 @@ def _new_site(db_name, site, mariadb_root_username=None, mariadb_root_password=N @click.option('--install-app', multiple=True, help='Install app after installation') @click.option('--with-public-files', help='Restores the public files of the site, given path to its tar file') @click.option('--with-private-files', help='Restores the private files of the site, given path to its tar file') -@click.option('--force', is_flag=True, default=False, help='Use a bit of force to get the job done') +@click.option('--force', is_flag=True, default=False, help='Ignore the site downgrade warning, if applicable') @pass_context def restore(context, sql_file_path, mariadb_root_username=None, mariadb_root_password=None, db_name=None, verbose=None, install_app=None, admin_password=None, force=None, with_public_files=None, with_private_files=None): "Restore site database from an sql file" - from frappe.installer import extract_sql_gzip, extract_tar_files, is_downgrade + from frappe.installer import extract_sql_gzip, extract_files, is_downgrade force = context.force or force # Extract the gzip file if user has passed *.sql.gz file instead of *.sql file @@ -147,12 +143,12 @@ def restore(context, sql_file_path, mariadb_root_username=None, mariadb_root_pas # Extract public and/or private files to the restored site, if user has given the path if with_public_files: with_public_files = os.path.join(base_path, with_public_files) - public = extract_tar_files(site, with_public_files, 'public') + public = extract_files(site, with_public_files, 'public') os.remove(public) if with_private_files: with_private_files = os.path.join(base_path, with_private_files) - private = extract_tar_files(site, with_private_files, 'private') + private = extract_files(site, with_private_files, 'private') os.remove(private) # Removing temporarily created file @@ -276,6 +272,8 @@ def disable_user(context, email): @pass_context def migrate(context, skip_failing=False, skip_search_index=False): "Run patches, sync schema and rebuild files/translations" + import compileall + import re from frappe.migrate import migrate for site in context.sites: @@ -385,35 +383,34 @@ def use(site, sites_path='.'): @click.command('backup') @click.option('--with-files', default=False, is_flag=True, help="Take backup with files") -@click.option('--verbose', default=False, is_flag=True) +@click.option('--backup-path', default=None, help="Set path for saving all the files in this operation") +@click.option('--backup-path-db', default=None, help="Set path for saving database file") +@click.option('--backup-path-files', default=None, help="Set path for saving public file") +@click.option('--backup-path-private-files', default=None, help="Set path for saving private file") +@click.option('--backup-path-conf', default=None, help="Set path for saving config file") +@click.option('--verbose', default=False, is_flag=True, help="Add verbosity") +@click.option('--compress', default=False, is_flag=True, help="Compress private and public files") @pass_context -def backup(context, with_files=False, backup_path_db=None, backup_path_files=None, - backup_path_private_files=None, quiet=False, verbose=False): +def backup(context, with_files=False, backup_path=None, backup_path_db=None, backup_path_files=None, + backup_path_private_files=None, backup_path_conf=None, verbose=False, compress=False): "Backup" from frappe.utils.backups import scheduled_backup verbose = verbose or context.verbose exit_code = 0 + for site in context.sites: try: frappe.init(site=site) frappe.connect() - odb = scheduled_backup(ignore_files=not with_files, backup_path_db=backup_path_db, backup_path_files=backup_path_files, backup_path_private_files=backup_path_private_files, force=True, verbose=verbose) - except Exception as e: - if verbose: - print("Backup failed for {0}. Database or site_config.json may be corrupted".format(site)) + odb = scheduled_backup(ignore_files=not with_files, backup_path=backup_path, backup_path_db=backup_path_db, backup_path_files=backup_path_files, backup_path_private_files=backup_path_private_files, backup_path_conf=backup_path_conf, force=True, verbose=verbose, compress=compress) + except Exception: + click.secho("Backup failed for Site {0}. Database or site_config.json may be corrupted".format(site), fg="red") exit_code = 1 continue - - if verbose: - from frappe.utils import now - summary_title = "Backup Summary at {0}".format(now()) - print(summary_title + "\n" + "-" * len(summary_title)) - print("Database backup:", odb.backup_path_db) - if with_files: - print("Public files: ", odb.backup_path_files) - print("Private files: ", odb.backup_path_private_files) - + odb.print_summary() + click.secho("Backup for Site {0} has been successfully completed{1}".format(site, " with files" if with_files else ""), fg="green") frappe.destroy() + if not context.sites: raise SiteNotSpecifiedError diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py index 5a5986ff57..4a69486c1b 100644 --- a/frappe/commands/utils.py +++ b/frappe/commands/utils.py @@ -554,10 +554,24 @@ def run_ui_tests(context, app, headless=False): site_env = 'CYPRESS_baseUrl={}'.format(site_url) password_env = 'CYPRESS_adminPassword={}'.format(admin_password) if admin_password else '' + os.chdir(app_base_path) + + node_bin = subprocess.getoutput("npm bin") + cypress_path = "{0}/cypress".format(node_bin) + plugin_path = "{0}/cypress-file-upload".format(node_bin) + + # check if cypress in path...if not, install it. + if not (os.path.exists(cypress_path) or os.path.exists(plugin_path)): + # install cypress + click.secho("Installing Cypress...", fg="yellow") + frappe.commands.popen("yarn add cypress@3 cypress-file-upload@^3.1 --no-lockfile") + # run for headless mode run_or_open = 'run --browser chrome --record --key 4a48f41c-11b3-425b-aa88-c58048fa69eb' if headless else 'open' - command = '{site_env} {password_env} yarn run cypress {run_or_open}' - formatted_command = command.format(site_env=site_env, password_env=password_env, run_or_open=run_or_open) + command = '{site_env} {password_env} {cypress} {run_or_open}' + formatted_command = command.format(site_env=site_env, password_env=password_env, cypress=cypress_path, run_or_open=run_or_open) + + click.secho("Running Cypress...", fg="yellow") frappe.commands.popen(formatted_command, cwd=app_base_path, raise_err=True) diff --git a/frappe/config/settings.py b/frappe/config/settings.py index e43abd9fcb..0112c7ccff 100644 --- a/frappe/config/settings.py +++ b/frappe/config/settings.py @@ -23,6 +23,11 @@ def get_data(): "description": _("Company, Fiscal Year and Currency defaults"), "hide_count": True }, + { + "type": "doctype", + "name": "Log Settings", + "description": _("Log cleanup and notification configuration") + }, { "type": "doctype", "name": "Error Log", diff --git a/frappe/contacts/doctype/address/address.json b/frappe/contacts/doctype/address/address.json index 2f634074d2..ae7ab464b8 100644 --- a/frappe/contacts/doctype/address/address.json +++ b/frappe/contacts/doctype/address/address.json @@ -1,4 +1,5 @@ { + "actions": [], "allow_import": 1, "allow_rename": 1, "creation": "2013-01-10 16:34:32", @@ -24,7 +25,6 @@ "is_shipping_address", "disabled", "linked_with", - "is_your_company_address", "links" ], "fields": [ @@ -138,12 +138,6 @@ "label": "Reference", "options": "fa fa-pushpin" }, - { - "default": "0", - "fieldname": "is_your_company_address", - "fieldtype": "Check", - "label": "Is Your Company Address" - }, { "fieldname": "links", "fieldtype": "Table", @@ -153,7 +147,8 @@ ], "icon": "fa fa-map-marker", "idx": 5, - "modified": "2019-09-08 11:41:04.145589", + "links": [], + "modified": "2020-10-14 17:38:08.971776", "modified_by": "Administrator", "module": "Contacts", "name": "Address", diff --git a/frappe/contacts/doctype/address/address.py b/frappe/contacts/doctype/address/address.py index e82ab9b26e..84b925d50e 100644 --- a/frappe/contacts/doctype/address/address.py +++ b/frappe/contacts/doctype/address/address.py @@ -39,14 +39,13 @@ class Address(Document): def validate(self): self.link_address() - self.validate_reference() self.validate_preferred_address() set_link_title(self) deduplicate_dynamic_links(self) def link_address(self): """Link address based on owner""" - if not self.links and not self.is_your_company_address: + if not self.links: contact_name = frappe.db.get_value("Contact", {"email_id": self.owner}) if contact_name: contact = frappe.get_cached_doc('Contact', contact_name) @@ -56,12 +55,6 @@ class Address(Document): return False - def validate_reference(self): - if self.is_your_company_address: - if not [row for row in self.links if row.link_doctype == "Company"]: - frappe.throw(_("Address needs to be linked to a Company. Please add a row for Company in the Links table below."), - title =_("Company not Linked")) - def validate_preferred_address(self): preferred_fields = ['is_primary_address', 'is_shipping_address'] @@ -204,25 +197,6 @@ def get_address_templates(address): else: return result -@frappe.whitelist() -def get_shipping_address(company, address = None): - filters = [ - ["Dynamic Link", "link_doctype", "=", "Company"], - ["Dynamic Link", "link_name", "=", company], - ["Address", "is_your_company_address", "=", 1] - ] - fields = ["*"] - if address and frappe.db.get_value('Dynamic Link', - {'parent': address, 'link_name': company}): - filters.append(["Address", "name", "=", address]) - - address = frappe.get_all("Address", filters=filters, fields=fields) or {} - - if address: - address_as_dict = address[0] - name, address_template = get_address_templates(address_as_dict) - return address_as_dict.get("name"), frappe.render_template(address_template, address_as_dict) - def get_company_address(company): ret = frappe._dict() ret.company_address = get_default_address('Company', company) diff --git a/frappe/core/doctype/activity_log/activity_log.py b/frappe/core/doctype/activity_log/activity_log.py index 27a2892ca8..98dc91806d 100644 --- a/frappe/core/doctype/activity_log/activity_log.py +++ b/frappe/core/doctype/activity_log/activity_log.py @@ -40,7 +40,11 @@ def add_authentication_log(subject, user, operation="Login", status="Success"): "operation": operation, }).insert(ignore_permissions=True, ignore_links=True) -def clear_authentication_logs(): - """clear 100 day old authentication logs""" +def clear_activity_logs(days=None): + """clear 90 day old authentication logs or configured in log settings""" + + if not days: + days = 90 + frappe.db.sql("""delete from `tabActivity Log` where \ - creation< (NOW() - INTERVAL '100' DAY)""") \ No newline at end of file + creation< (NOW() - INTERVAL '{0}' DAY)""".format(days)) \ No newline at end of file diff --git a/frappe/core/doctype/communication/email.py b/frappe/core/doctype/communication/email.py index 4c531fbac6..c56a950fbd 100755 --- a/frappe/core/doctype/communication/email.py +++ b/frappe/core/doctype/communication/email.py @@ -55,7 +55,7 @@ def make(doctype=None, name=None, content=None, subject=None, sent_or_received = comm = frappe.get_doc({ "doctype":"Communication", "subject": subject, - "content": content, + "content": frappe.utils.sanitize_html(content), "sender": sender, "sender_full_name":sender_full_name, "recipients": recipients, diff --git a/frappe/core/doctype/error_log/error_log.py b/frappe/core/doctype/error_log/error_log.py index 6cc8275404..ec02aaf446 100644 --- a/frappe/core/doctype/error_log/error_log.py +++ b/frappe/core/doctype/error_log/error_log.py @@ -17,9 +17,6 @@ def set_old_logs_as_seen(): frappe.db.sql("""UPDATE `tabError Log` SET `seen`=1 WHERE `seen`=0 AND `creation` < (NOW() - INTERVAL '7' DAY)""") - # clear old logs - frappe.db.sql("""DELETE FROM `tabError Log` WHERE `creation` < (NOW() - INTERVAL '30' DAY)""") - @frappe.whitelist() def clear_error_logs(): '''Flush all Error Logs''' diff --git a/frappe/core/doctype/log_setting_user/__init__.py b/frappe/core/doctype/log_setting_user/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/frappe/core/doctype/log_setting_user/log_setting_user.js b/frappe/core/doctype/log_setting_user/log_setting_user.js new file mode 100644 index 0000000000..a1eb824e22 --- /dev/null +++ b/frappe/core/doctype/log_setting_user/log_setting_user.js @@ -0,0 +1,8 @@ +// Copyright (c) 2020, Frappe Technologies and contributors +// For license information, please see license.txt + +frappe.ui.form.on('Log Setting User', { + // refresh: function(frm) { + + // } +}); diff --git a/frappe/core/doctype/log_setting_user/log_setting_user.json b/frappe/core/doctype/log_setting_user/log_setting_user.json new file mode 100644 index 0000000000..7f4b0ef874 --- /dev/null +++ b/frappe/core/doctype/log_setting_user/log_setting_user.json @@ -0,0 +1,34 @@ +{ + "actions": [], + "autoname": "field:user", + "creation": "2020-10-08 13:09:36.034430", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "user" + ], + "fields": [ + { + "fieldname": "user", + "fieldtype": "Link", + "in_list_view": 1, + "label": "User", + "options": "User", + "reqd": 1, + "unique": 1 + } + ], + "index_web_pages_for_search": 1, + "istable": 1, + "links": [], + "modified": "2020-10-08 17:22:04.690348", + "modified_by": "Administrator", + "module": "Core", + "name": "Log Setting User", + "owner": "Administrator", + "permissions": [], + "sort_field": "modified", + "sort_order": "DESC", + "track_changes": 1 +} \ No newline at end of file diff --git a/frappe/core/doctype/log_setting_user/log_setting_user.py b/frappe/core/doctype/log_setting_user/log_setting_user.py new file mode 100644 index 0000000000..df6d55f0a9 --- /dev/null +++ b/frappe/core/doctype/log_setting_user/log_setting_user.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020, Frappe Technologies and contributors +# For license information, please see license.txt + +from __future__ import unicode_literals +# import frappe +from frappe.model.document import Document + +class LogSettingUser(Document): + pass diff --git a/frappe/core/doctype/log_setting_user/test_log_setting_user.py b/frappe/core/doctype/log_setting_user/test_log_setting_user.py new file mode 100644 index 0000000000..507c02d87d --- /dev/null +++ b/frappe/core/doctype/log_setting_user/test_log_setting_user.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020, Frappe Technologies and Contributors +# See license.txt +from __future__ import unicode_literals + +# import frappe +import unittest + +class TestLogSettingUser(unittest.TestCase): + pass diff --git a/frappe/core/doctype/log_settings/__init__.py b/frappe/core/doctype/log_settings/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/frappe/core/doctype/log_settings/log_settings.js b/frappe/core/doctype/log_settings/log_settings.js new file mode 100644 index 0000000000..09a2086a1d --- /dev/null +++ b/frappe/core/doctype/log_settings/log_settings.js @@ -0,0 +1,8 @@ +// Copyright (c) 2020, Frappe Technologies and contributors +// For license information, please see license.txt + +frappe.ui.form.on('Log Settings', { + // refresh: function(frm) { + + // } +}); diff --git a/frappe/core/doctype/log_settings/log_settings.json b/frappe/core/doctype/log_settings/log_settings.json new file mode 100644 index 0000000000..8a2596b35c --- /dev/null +++ b/frappe/core/doctype/log_settings/log_settings.json @@ -0,0 +1,83 @@ +{ + "actions": [], + "creation": "2020-10-08 12:12:21.694424", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "error_log_notification_section", + "users_to_notify", + "log_cleanup_section", + "clear_error_log_after", + "clear_activity_log_after", + "column_break_4", + "clear_email_queue_after" + ], + "fields": [ + { + "fieldname": "log_cleanup_section", + "fieldtype": "Section Break", + "label": "Log Cleanup" + }, + { + "fieldname": "column_break_4", + "fieldtype": "Column Break" + }, + { + "fieldname": "error_log_notification_section", + "fieldtype": "Section Break", + "label": "Error Log Notification" + }, + { + "fieldname": "users_to_notify", + "fieldtype": "Table MultiSelect", + "label": "Users To Notify", + "options": "Log Setting User" + }, + { + "default": "90", + "description": "In Days", + "fieldname": "clear_error_log_after", + "fieldtype": "Int", + "label": "Clear Error log After" + }, + { + "default": "90", + "description": "In Days", + "fieldname": "clear_activity_log_after", + "fieldtype": "Int", + "label": "Clear Activity Log After" + }, + { + "default": "90", + "description": "In Days", + "fieldname": "clear_email_queue_after", + "fieldtype": "Int", + "label": "Clear Email Queue After" + } + ], + "index_web_pages_for_search": 1, + "issingle": 1, + "links": [], + "modified": "2020-10-13 12:18:48.649038", + "modified_by": "Administrator", + "module": "Core", + "name": "Log Settings", + "owner": "Administrator", + "permissions": [ + { + "create": 1, + "delete": 1, + "email": 1, + "print": 1, + "read": 1, + "role": "System Manager", + "share": 1, + "write": 1 + } + ], + "quick_entry": 1, + "sort_field": "modified", + "sort_order": "DESC", + "track_changes": 1 +} \ No newline at end of file diff --git a/frappe/core/doctype/log_settings/log_settings.py b/frappe/core/doctype/log_settings/log_settings.py new file mode 100644 index 0000000000..6d59cdeb29 --- /dev/null +++ b/frappe/core/doctype/log_settings/log_settings.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020, Frappe Technologies and contributors +# For license information, please see license.txt + +from __future__ import unicode_literals +import frappe +from frappe import _ +from frappe.model.document import Document + +class LogSettings(Document): + def clear_logs(self): + self.clear_error_logs() + self.clear_activity_logs() + self.clear_email_queue() + + def clear_error_logs(self): + frappe.db.sql(""" DELETE FROM `tabError Log` + WHERE `creation` < (NOW() - INTERVAL '{0}' DAY) + """.format(self.clear_error_log_after)) + + def clear_activity_logs(self): + from frappe.core.doctype.activity_log.activity_log import clear_activity_logs + clear_activity_logs(days=self.clear_activity_log_after) + + def clear_email_queue(self): + from frappe.email.queue import clear_outbox + clear_outbox(days=self.clear_email_queue_after) + +def run_log_clean_up(): + doc = frappe.get_doc("Log Settings") + doc.clear_logs() + +@frappe.whitelist() +def has_unseen_error_log(user): + + def _get_response(show_alert=True): + return { + 'show_alert': True, + 'message': _("You have unseen {0}").format(' Error Logs ') + } + + if frappe.db.sql_list("select name from `tabError Log` where seen = 0 limit 1"): + log_settings = frappe.get_cached_doc('Log Settings') + + if log_settings.users_to_notify: + if user in [u.user for u in log_settings.users_to_notify]: + return _get_response() + else: + return _get_response(show_alert=False) + else: + return _get_response() \ No newline at end of file diff --git a/frappe/core/doctype/log_settings/test_log_settings.py b/frappe/core/doctype/log_settings/test_log_settings.py new file mode 100644 index 0000000000..2824c71c88 --- /dev/null +++ b/frappe/core/doctype/log_settings/test_log_settings.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020, Frappe Technologies and Contributors +# See license.txt +from __future__ import unicode_literals + +# import frappe +import unittest + +class TestLogSettings(unittest.TestCase): + pass diff --git a/frappe/core/doctype/report/report.py b/frappe/core/doctype/report/report.py index 8b7a03aa28..8634ad1084 100644 --- a/frappe/core/doctype/report/report.py +++ b/frappe/core/doctype/report/report.py @@ -49,6 +49,8 @@ class Report(Document): self.export_doc() def on_trash(self): + if self.is_standard == 'Yes' and not cint(getattr(frappe.local.conf, 'developer_mode',0)): + frappe.throw(_("You are not allowed to delete Standard Report")) delete_custom_role('report', self.name) def get_columns(self): diff --git a/frappe/core/doctype/scheduled_job_type/test_scheduled_job_type.py b/frappe/core/doctype/scheduled_job_type/test_scheduled_job_type.py index e7db6f9045..d0a65defa4 100644 --- a/frappe/core/doctype/scheduled_job_type/test_scheduled_job_type.py +++ b/frappe/core/doctype/scheduled_job_type/test_scheduled_job_type.py @@ -22,7 +22,7 @@ class TestScheduledJobType(unittest.TestCase): self.assertEqual(all_job.frequency, 'All') daily_job = frappe.get_doc('Scheduled Job Type', - dict(method='frappe.email.queue.clear_outbox')) + dict(method='frappe.email.queue.set_expiry_for_email_queue')) self.assertEqual(daily_job.frequency, 'Daily') # check if cron jobs are synced @@ -38,7 +38,7 @@ class TestScheduledJobType(unittest.TestCase): self.assertEqual(updated_scheduled_job.frequency, "Hourly") def test_daily_job(self): - job = frappe.get_doc('Scheduled Job Type', dict(method = 'frappe.email.queue.clear_outbox')) + job = frappe.get_doc('Scheduled Job Type', dict(method = 'frappe.email.queue.set_expiry_for_email_queue')) job.db_set('last_execution', '2019-01-01 00:00:00') self.assertTrue(job.is_event_due(get_datetime('2019-01-02 00:00:06'))) self.assertFalse(job.is_event_due(get_datetime('2019-01-01 00:00:06'))) diff --git a/frappe/custom/doctype/custom_field/custom_field.py b/frappe/custom/doctype/custom_field/custom_field.py index bc325b654e..ee6e3b9c61 100644 --- a/frappe/custom/doctype/custom_field/custom_field.py +++ b/frappe/custom/doctype/custom_field/custom_field.py @@ -32,6 +32,7 @@ class CustomField(Document): self.fieldname = self.fieldname.lower() def before_insert(self): + self.set_fieldname() meta = frappe.get_meta(self.dt, cached=False) fieldnames = [df.fieldname for df in meta.get("fields")] diff --git a/frappe/data/sample_site_config.json b/frappe/data/sample_site_config.json index 36818ef286..715cd7b9fa 100644 --- a/frappe/data/sample_site_config.json +++ b/frappe/data/sample_site_config.json @@ -22,6 +22,9 @@ "use_ssl": 0, "auto_email_id": "hello@example.com", + "google_analytics_id": "google_analytics_id", + "google_analytics_anonymize_ip": 1, + "google_login": { "client_id": "google_client_id", "client_secret": "google_client_secret" diff --git a/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.js b/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.js index 8d780969e1..0898fcf4e7 100644 --- a/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.js +++ b/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.js @@ -30,7 +30,7 @@ frappe.ui.form.on('Data Migration Connector', { frm.set_value('connector_type', 'Custom'); frm.set_value('python_module', r.message); frm.save(); - frappe.show_alert({message: __(`New module created ${r.message}`), indicator: 'success'}); + frappe.show_alert(__("New module created {0}", [r.message])); d.hide(); } }); diff --git a/frappe/database/postgres/setup_db.py b/frappe/database/postgres/setup_db.py index 1dc1ea4c97..f53872db82 100644 --- a/frappe/database/postgres/setup_db.py +++ b/frappe/database/postgres/setup_db.py @@ -20,8 +20,11 @@ def setup_database(force, source_sql=None, verbose=False): source_sql = os.path.join(os.path.dirname(__file__), 'framework_postgres.sql') subprocess.check_output([ - 'psql', frappe.conf.db_name, '-h', frappe.conf.db_host or 'localhost', '-U', - frappe.conf.db_name, '-f', source_sql + 'psql', frappe.conf.db_name, + '-h', frappe.conf.db_host or 'localhost', + '-p', str(frappe.conf.db_port or '5432'), + '-U', frappe.conf.db_name, + '-f', source_sql ], env=subprocess_env) frappe.connect() diff --git a/frappe/desk/doctype/dashboard_chart/dashboard_chart.js b/frappe/desk/doctype/dashboard_chart/dashboard_chart.js index 7f26bd9101..d763ce5009 100644 --- a/frappe/desk/doctype/dashboard_chart/dashboard_chart.js +++ b/frappe/desk/doctype/dashboard_chart/dashboard_chart.js @@ -169,7 +169,7 @@ frappe.ui.form.on('Dashboard Chart', { frm.field_options = frappe.report_utils.get_field_options_from_report(data.columns, data); frm.set_df_property('x_field', 'options', frm.field_options.non_numeric_fields); if (!frm.field_options.numeric_fields.length) { - frappe.msgprint(__(`Report has no numeric fields, please change the Report Name`)); + frappe.msgprint(__("Report has no numeric fields, please change the Report Name")); } else { let y_field_df = frappe.meta.get_docfield('Dashboard Chart Field', 'y_field', frm.doc.name); y_field_df.options = frm.field_options.numeric_fields; diff --git a/frappe/desk/doctype/module_onboarding/module_onboarding.py b/frappe/desk/doctype/module_onboarding/module_onboarding.py index 89160a60f0..8315c0b304 100644 --- a/frappe/desk/doctype/module_onboarding/module_onboarding.py +++ b/frappe/desk/doctype/module_onboarding/module_onboarding.py @@ -41,3 +41,15 @@ class ModuleOnboarding(Document): def before_export(self, doc): doc.is_complete = 0 + + def reset_onboarding(self): + frappe.only_for("Administrator") + + self.is_complete = 0 + steps = self.get_steps() + for step in steps: + step.is_complete = 0 + step.is_skipped = 0 + step.save() + + self.save() diff --git a/frappe/desk/doctype/number_card/number_card.js b/frappe/desk/doctype/number_card/number_card.js index 63b41b956e..6d1454a2cb 100644 --- a/frappe/desk/doctype/number_card/number_card.js +++ b/frappe/desk/doctype/number_card/number_card.js @@ -207,7 +207,7 @@ frappe.ui.form.on('Number Card', { frm.field_options = frappe.report_utils.get_field_options_from_report(data.columns, data); frm.set_df_property('report_field', 'options', frm.field_options.numeric_fields); if (!frm.field_options.numeric_fields.length) { - frappe.msgprint(__(`Report has no numeric fields, please change the Report Name`)); + frappe.msgprint(__("Report has no numeric fields, please change the Report Name")); } } else { frappe.msgprint(__('Report has no data, please modify the filters or change the Report Name')); diff --git a/frappe/desk/page/user_profile/user_profile.js b/frappe/desk/page/user_profile/user_profile.js index 0d0ea2d9f0..cbad658c3a 100644 --- a/frappe/desk/page/user_profile/user_profile.js +++ b/frappe/desk/page/user_profile/user_profile.js @@ -42,7 +42,7 @@ class UserProfile { } make_user_profile() { - frappe.set_route('user-profile', this.user_id); + frappe.set_route('user-profile', this.user_id, { redirect: true }); this.user = frappe.user_info(this.user_id); this.page.set_title(this.user.fullname); this.setup_user_search(); diff --git a/frappe/desk/query_report.py b/frappe/desk/query_report.py index eda61c6985..5a9aae8435 100644 --- a/frappe/desk/query_report.py +++ b/frappe/desk/query_report.py @@ -4,11 +4,19 @@ from __future__ import unicode_literals import frappe -import os, json +import os +import json from frappe import _ from frappe.modules import scrub, get_module_path -from frappe.utils import flt, cint, get_html_format, get_url_to_form, gzip_decompress, format_duration +from frappe.utils import ( + flt, + cint, + get_html_format, + get_url_to_form, + gzip_decompress, + format_duration, +) from frappe.model.utils import render_include from frappe.translate import send_translations import frappe.desk.reportview @@ -17,11 +25,12 @@ from six import string_types, iteritems from datetime import timedelta from frappe.core.utils import ljust_list + def get_report_doc(report_name): doc = frappe.get_doc("Report", report_name) doc.custom_columns = [] - if doc.report_type == 'Custom Report': + if doc.report_type == "Custom Report": custom_report_doc = doc reference_report = custom_report_doc.reference_report doc = frappe.get_doc("Report", reference_report) @@ -30,11 +39,18 @@ def get_report_doc(report_name): doc.is_custom_report = True if not doc.is_permitted(): - frappe.throw(_("You don't have access to Report: {0}").format(report_name), frappe.PermissionError) + frappe.throw( + _("You don't have access to Report: {0}").format(report_name), + frappe.PermissionError, + ) if not frappe.has_permission(doc.ref_doctype, "report"): - frappe.throw(_("You don't have permission to get a report on: {0}").format(doc.ref_doctype), - frappe.PermissionError) + frappe.throw( + _("You don't have permission to get a report on: {0}").format( + doc.ref_doctype + ), + frappe.PermissionError, + ) if doc.disabled: frappe.throw(_("Report {0} is disabled").format(report_name)) @@ -54,11 +70,10 @@ def generate_report_result(report, filters=None, user=None, custom_columns=None) if report.report_type == "Query Report": res = report.execute_query_report(filters) - elif report.report_type == 'Script Report': + elif report.report_type == "Script Report": res = report.execute_script_report(filters) - columns, result, message, chart, report_summary, skip_total_row = \ - ljust_list(res, 6) + columns, result, message, chart, report_summary, skip_total_row = ljust_list(res, 6) if report.custom_columns: # Original query columns, needed to reorder data as per custom columns @@ -74,7 +89,7 @@ def generate_report_result(report, filters=None, user=None, custom_columns=None) result = add_data_to_custom_columns(custom_columns, result) for custom_column in custom_columns: - columns.insert(custom_column['insert_after_index'] + 1, custom_column) + columns.insert(custom_column["insert_after_index"] + 1, custom_column) if result: result = get_filtered_data(report.ref_doctype, columns, result, user) @@ -90,17 +105,19 @@ def generate_report_result(report, filters=None, user=None, custom_columns=None) "report_summary": report_summary, "skip_total_row": skip_total_row or 0, "status": None, - "execution_time": frappe.cache().hget('report_execution_time', report.name) or 0 + "execution_time": frappe.cache().hget("report_execution_time", report.name) + or 0, } + @frappe.whitelist() def background_enqueue_run(report_name, filters=None, user=None): """run reports in background""" if not user: user = frappe.session.user report = get_report_doc(report_name) - track_instance = \ - frappe.get_doc({ + track_instance = frappe.get_doc( + { "doctype": "Prepared Report", "report_name": report_name, # This looks like an insanity but, without this it'd be very hard to find Prepared Reports matching given condition @@ -110,21 +127,24 @@ def background_enqueue_run(report_name, filters=None, user=None): "report_type": report.report_type, "query": report.query, "module": report.module, - }) + } + ) track_instance.insert(ignore_permissions=True) frappe.db.commit() track_instance.enqueue_report() return { "name": track_instance.name, - "redirect_url": get_url_to_form("Prepared Report", track_instance.name) + "redirect_url": get_url_to_form("Prepared Report", track_instance.name), } @frappe.whitelist() def get_script(report_name): report = get_report_doc(report_name) - module = report.module or frappe.db.get_value("DocType", report.ref_doctype, "module") + module = report.module or frappe.db.get_value( + "DocType", report.ref_doctype, "module" + ) module_path = get_module_path(module) report_folder = os.path.join(module_path, "report", scrub(report.name)) script_path = os.path.join(report_folder, scrub(report.name) + ".js") @@ -150,24 +170,38 @@ def get_script(report_name): return { "script": render_include(script), "html_format": html_format, - "execution_time": frappe.cache().hget('report_execution_time', report_name) or 0 + "execution_time": frappe.cache().hget("report_execution_time", report_name) + or 0, } @frappe.whitelist() @frappe.read_only() -def run(report_name, filters=None, user=None, ignore_prepared_report=False, custom_columns=None): +def run( + report_name, + filters=None, + user=None, + ignore_prepared_report=False, + custom_columns=None, +): report = get_report_doc(report_name) if not user: user = frappe.session.user if not frappe.has_permission(report.ref_doctype, "report"): - frappe.msgprint(_("Must have report permission to access this report."), - raise_exception=True) + frappe.msgprint( + _("Must have report permission to access this report."), + raise_exception=True, + ) result = None - if report.prepared_report and not report.disable_prepared_report and not ignore_prepared_report and not custom_columns: + if ( + report.prepared_report + and not report.disable_prepared_report + and not ignore_prepared_report + and not custom_columns + ): if filters: if isinstance(filters, string_types): filters = json.loads(filters) @@ -180,10 +214,13 @@ def run(report_name, filters=None, user=None, ignore_prepared_report=False, cust else: result = generate_report_result(report, filters, user, custom_columns) - result["add_total_row"] = report.add_total_row and not result.get('skip_total_row', False) + result["add_total_row"] = report.add_total_row and not result.get( + "skip_total_row", False + ) return result + def add_data_to_custom_columns(columns, result): custom_fields_data = get_data_for_custom_report(columns) @@ -195,25 +232,28 @@ def add_data_to_custom_columns(columns, result): if isinstance(row, list): for idx, column in enumerate(columns): - if column.get('link_field'): - row_obj[column['fieldname']] = None + if column.get("link_field"): + row_obj[column["fieldname"]] = None row.insert(idx, None) else: - row_obj[column['fieldname']] = row[idx] + row_obj[column["fieldname"]] = row[idx] data.append(row_obj) else: data.append(row) for row in data: for column in columns: - if column.get('link_field'): - fieldname = column['fieldname'] - key = (column['doctype'], fieldname) - link_field = column['link_field'] - row[fieldname] = custom_fields_data.get(key, {}).get(row.get(link_field)) + if column.get("link_field"): + fieldname = column["fieldname"] + key = (column["doctype"], fieldname) + link_field = column["link_field"] + row[fieldname] = custom_fields_data.get(key, {}).get( + row.get(link_field) + ) return data + def reorder_data_for_custom_columns(custom_columns, columns, result): if not result: return [] @@ -228,6 +268,7 @@ def reorder_data_for_custom_columns(custom_columns, columns, result): # columns do not need to be reordered if result is a list of dicts return result + def get_columns_from_list(columns, target_columns, result): reordered_result = [] @@ -244,6 +285,7 @@ def get_columns_from_list(columns, target_columns, result): return reordered_result + def get_prepared_report_result(report, filters, dn="", user=None): latest_report_data = {} doc = None @@ -252,14 +294,15 @@ def get_prepared_report_result(report, filters, dn="", user=None): doc = frappe.get_doc("Prepared Report", dn) else: # Only look for completed prepared reports with given filters. - doc_list = frappe.get_all("Prepared Report", + doc_list = frappe.get_all( + "Prepared Report", filters={ "status": "Completed", "filters": json.dumps(filters), "owner": user, - "report_name": report.get('custom_report') or report.get('report_name') + "report_name": report.get("custom_report") or report.get("report_name"), }, - order_by = 'creation desc' + order_by="creation desc", ) if doc_list: @@ -269,11 +312,15 @@ def get_prepared_report_result(report, filters, dn="", user=None): if doc: try: # Prepared Report data is stored in a GZip compressed JSON file - attached_file_name = frappe.db.get_value("File", {"attached_to_doctype": doc.doctype, "attached_to_name":doc.name}, "name") - attached_file = frappe.get_doc('File', attached_file_name) + attached_file_name = frappe.db.get_value( + "File", + {"attached_to_doctype": doc.doctype, "attached_to_name": doc.name}, + "name", + ) + attached_file = frappe.get_doc("File", attached_file_name) compressed_content = attached_file.get_content() uncompressed_content = gzip_decompress(compressed_content) - data = json.loads(uncompressed_content) + data = json.loads(uncompressed_content.decode("utf-8")) if data: columns = json.loads(doc.columns) if doc.columns else data[0] @@ -281,23 +328,18 @@ def get_prepared_report_result(report, filters, dn="", user=None): if isinstance(column, dict) and column.get("label"): column["label"] = _(column["label"]) - latest_report_data = { - "columns": columns, - "result": data - } + latest_report_data = {"columns": columns, "result": data} except Exception: frappe.log_error(frappe.get_traceback()) frappe.delete_doc("Prepared Report", doc.name) frappe.db.commit() doc = None - latest_report_data.update({ - "prepared_report": True, - "doc": doc - }) + latest_report_data.update({"prepared_report": True, "doc": doc}) return latest_report_data + @frappe.whitelist() def export_query(): """export from query reports""" @@ -313,8 +355,8 @@ def export_query(): if isinstance(data.get("report_name"), string_types): report_name = data["report_name"] frappe.permissions.can_export( - frappe.get_cached_value('Report', report_name, 'ref_doctype'), - raise_exception=True + frappe.get_cached_value("Report", report_name, "ref_doctype"), + raise_exception=True, ) if isinstance(data.get("file_format_type"), string_types): file_format_type = data["file_format_type"] @@ -331,20 +373,26 @@ def export_query(): data = run(report_name, filters, custom_columns=custom_columns) data = frappe._dict(data) if not data.columns: - frappe.respond_as_web_page(_("No data to export"), - _("You can try changing the filters of your report.")) + frappe.respond_as_web_page( + _("No data to export"), + _("You can try changing the filters of your report."), + ) return columns = get_columns_dict(data.columns) from frappe.utils.xlsxutils import make_xlsx - data['result'] = handle_duration_fieldtype_values(data.get('result'), data.get('columns')) + + data["result"] = handle_duration_fieldtype_values( + data.get("result"), data.get("columns") + ) xlsx_data = build_xlsx_data(columns, data, visible_idx, include_indentation) xlsx_file = make_xlsx(xlsx_data, "Query Report") - frappe.response['filename'] = report_name + '.xlsx' - frappe.response['filecontent'] = xlsx_file.getvalue() - frappe.response['type'] = 'binary' + frappe.response["filename"] = report_name + ".xlsx" + frappe.response["filecontent"] = xlsx_file.getvalue() + frappe.response["type"] = "binary" + def handle_duration_fieldtype_values(result, columns): for i, col in enumerate(columns): @@ -370,6 +418,7 @@ def handle_duration_fieldtype_values(result, columns): return result + def build_xlsx_data(columns, data, visible_idx, include_indentation): result = [[]] @@ -386,13 +435,13 @@ def build_xlsx_data(columns, data, visible_idx, include_indentation): if isinstance(row, dict) and row: for idx in range(len(data.columns)): - # check if column is not hidden + # check if column is not hidden if not columns[idx].get("hidden"): label = columns[idx]["label"] fieldname = columns[idx]["fieldname"] cell_value = row.get(fieldname, row.get(label, "")) - if cint(include_indentation) and 'indent' in row and idx == 0: - cell_value = (' ' * cint(row['indent'])) + cell_value + if cint(include_indentation) and "indent" in row and idx == 0: + cell_value = (" " * cint(row["indent"])) + cell_value row_data.append(cell_value) else: row_data = row @@ -401,8 +450,9 @@ def build_xlsx_data(columns, data, visible_idx, include_indentation): return result -def add_total_row(result, columns, meta = None): - total_row = [""]*len(columns) + +def add_total_row(result, columns, meta=None): + total_row = [""] * len(columns) has_percent = [] for i, col in enumerate(columns): fieldtype, options, fieldname = None, None, None @@ -428,10 +478,13 @@ def add_total_row(result, columns, meta = None): options = col.get("options") for row in result: - if i >= len(row): continue + if i >= len(row): + continue cell = row.get(fieldname) if isinstance(row, dict) else row[i] - if fieldtype in ["Currency", "Int", "Float", "Percent", "Duration"] and flt(cell): + if fieldtype in ["Currency", "Int", "Float", "Percent", "Duration"] and flt( + cell + ): total_row[i] = flt(total_row[i]) + flt(cell) if fieldtype == "Percent" and i not in has_percent: @@ -439,12 +492,15 @@ def add_total_row(result, columns, meta = None): if fieldtype == "Time" and cell: if not total_row[i]: - total_row[i]=timedelta(hours=0,minutes=0,seconds=0) - total_row[i] = total_row[i] + cell + total_row[i] = timedelta(hours=0, minutes=0, seconds=0) + total_row[i] = total_row[i] + cell - - if fieldtype=="Link" and options == "Currency": - total_row[i] = result[0].get(fieldname) if isinstance(result[0], dict) else result[0][i] + if fieldtype == "Link" and options == "Currency": + total_row[i] = ( + result[0].get(fieldname) + if isinstance(result[0], dict) + else result[0][i] + ) for i in has_percent: total_row[i] = flt(total_row[i]) / len(result) @@ -463,35 +519,44 @@ def add_total_row(result, columns, meta = None): result.append(total_row) return result + @frappe.whitelist() def get_data_for_custom_field(doctype, field): if not frappe.has_permission(doctype, "read"): frappe.throw(_("Not Permitted"), frappe.PermissionError) - value_map = frappe._dict(frappe.get_all(doctype, - fields=["name", field], - as_list=1)) + value_map = frappe._dict(frappe.get_all(doctype, fields=["name", field], as_list=1)) return value_map + def get_data_for_custom_report(columns): doc_field_value_map = {} for column in columns: - if column.get('link_field'): - fieldname = column.get('fieldname') - doctype = column.get('doctype') - doc_field_value_map[(doctype, fieldname)] = get_data_for_custom_field(doctype, fieldname) + if column.get("link_field"): + fieldname = column.get("fieldname") + doctype = column.get("doctype") + doc_field_value_map[(doctype, fieldname)] = get_data_for_custom_field( + doctype, fieldname + ) return doc_field_value_map + @frappe.whitelist() def save_report(reference_report, report_name, columns): report_doc = get_report_doc(reference_report) - docname = frappe.db.exists("Report", - {'report_name': report_name, 'is_standard': 'No', 'report_type': 'Custom Report'}) + docname = frappe.db.exists( + "Report", + { + "report_name": report_name, + "is_standard": "No", + "report_type": "Custom Report", + }, + ) if docname: report = frappe.get_doc("Report", docname) report.update({"json": columns}) @@ -500,15 +565,17 @@ def save_report(reference_report, report_name, columns): return docname else: - new_report = frappe.get_doc({ - 'doctype': 'Report', - 'report_name': report_name, - 'json': columns, - 'ref_doctype': report_doc.ref_doctype, - 'is_standard': 'No', - 'report_type': 'Custom Report', - 'reference_report': reference_report - }).insert(ignore_permissions = True) + new_report = frappe.get_doc( + { + "doctype": "Report", + "report_name": report_name, + "json": columns, + "ref_doctype": report_doc.ref_doctype, + "is_standard": "No", + "report_type": "Custom Report", + "reference_report": reference_report, + } + ).insert(ignore_permissions=True) frappe.msgprint(_("{0} saved successfully").format(new_report.name)) return new_report.name @@ -526,10 +593,22 @@ def get_filtered_data(ref_doctype, columns, data, user): if match_filters_per_doctype: for row in data: # Why linked_doctypes.get(ref_doctype)? because if column is empty, linked_doctypes[ref_doctype] is removed - if linked_doctypes.get(ref_doctype) and shared and row[linked_doctypes[ref_doctype]] in shared: + if ( + linked_doctypes.get(ref_doctype) + and shared + and row[linked_doctypes[ref_doctype]] in shared + ): result.append(row) - elif has_match(row, linked_doctypes, match_filters_per_doctype, ref_doctype, if_owner, columns_dict, user): + elif has_match( + row, + linked_doctypes, + match_filters_per_doctype, + ref_doctype, + if_owner, + columns_dict, + user, + ): result.append(row) else: result = list(data) @@ -537,17 +616,25 @@ def get_filtered_data(ref_doctype, columns, data, user): return result -def has_match(row, linked_doctypes, doctype_match_filters, ref_doctype, if_owner, columns_dict, user): +def has_match( + row, + linked_doctypes, + doctype_match_filters, + ref_doctype, + if_owner, + columns_dict, + user, +): """Returns True if after evaluating permissions for each linked doctype - - There is an owner match for the ref_doctype - - `and` There is a user permission match for all linked doctypes + - There is an owner match for the ref_doctype + - `and` There is a user permission match for all linked doctypes - Returns True if the row is empty + Returns True if the row is empty - Note: - Each doctype could have multiple conflicting user permission doctypes. - Hence even if one of the sets allows a match, it is true. - This behavior is equivalent to the trickling of user permissions of linked doctypes to the ref doctype. + Note: + Each doctype could have multiple conflicting user permission doctypes. + Hence even if one of the sets allows a match, it is true. + This behavior is equivalent to the trickling of user permissions of linked doctypes to the ref doctype. """ resultant_match = True @@ -558,20 +645,22 @@ def has_match(row, linked_doctypes, doctype_match_filters, ref_doctype, if_owner for doctype, filter_list in doctype_match_filters.items(): matched_for_doctype = False - if doctype==ref_doctype and if_owner: + if doctype == ref_doctype and if_owner: idx = linked_doctypes.get("User") - if (idx is not None - and row[idx]==user - and columns_dict[idx]==columns_dict.get("owner")): - # owner match is true - matched_for_doctype = True + if ( + idx is not None + and row[idx] == user + and columns_dict[idx] == columns_dict.get("owner") + ): + # owner match is true + matched_for_doctype = True if not matched_for_doctype: for match_filters in filter_list: match = True for dt, idx in linked_doctypes.items(): # case handled above - if dt=="User" and columns_dict[idx]==columns_dict.get("owner"): + if dt == "User" and columns_dict[idx] == columns_dict.get("owner"): continue cell_value = None @@ -580,7 +669,11 @@ def has_match(row, linked_doctypes, doctype_match_filters, ref_doctype, if_owner elif isinstance(row, (list, tuple)): cell_value = row[idx] - if dt in match_filters and cell_value not in match_filters.get(dt) and frappe.db.exists(dt, cell_value): + if ( + dt in match_filters + and cell_value not in match_filters.get(dt) + and frappe.db.exists(dt, cell_value) + ): match = False break @@ -599,6 +692,7 @@ def has_match(row, linked_doctypes, doctype_match_filters, ref_doctype, if_owner return resultant_match + def get_linked_doctypes(columns, data): linked_doctypes = {} @@ -606,7 +700,7 @@ def get_linked_doctypes(columns, data): for idx, col in enumerate(columns): df = columns_dict[idx] - if df.get("fieldtype")=="Link": + if df.get("fieldtype") == "Link": if data and isinstance(data[0], (list, tuple)): linked_doctypes[df["options"]] = idx else: @@ -635,10 +729,11 @@ def get_linked_doctypes(columns, data): return linked_doctypes + def get_columns_dict(columns): """Returns a dict with column docfield values as dict - The keys for the dict are both idx and fieldname, - so either index or fieldname can be used to search for a column's docfield properties + The keys for the dict are both idx and fieldname, + so either index or fieldname can be used to search for a column's docfield properties """ columns_dict = frappe._dict() for idx, col in enumerate(columns): @@ -648,6 +743,7 @@ def get_columns_dict(columns): return columns_dict + def get_column_as_dict(col): col_dict = frappe._dict() @@ -671,6 +767,7 @@ def get_column_as_dict(col): return col_dict + def get_user_match_filters(doctypes, user): match_filters = {} diff --git a/frappe/email/doctype/email_domain/email_domain.py b/frappe/email/doctype/email_domain/email_domain.py index 08583dc228..ce39523564 100644 --- a/frappe/email/doctype/email_domain/email_domain.py +++ b/frappe/email/doctype/email_domain/email_domain.py @@ -17,6 +17,8 @@ class EmailDomain(Document): def validate(self): """Validate email id and check POP3/IMAP and SMTP connections is enabled.""" + logger = frappe.logger() + if self.email_id: validate_email_address(self.email_id, True) @@ -26,19 +28,25 @@ class EmailDomain(Document): if not frappe.local.flags.in_install and not frappe.local.flags.in_patch: try: if self.use_imap: + logger.info('Checking incoming IMAP email server {host}:{port} ssl={ssl}...'.format( + host=self.email_server, port=get_port(self), ssl=self.use_ssl)) if self.use_ssl: test = imaplib.IMAP4_SSL(self.email_server, port=get_port(self)) else: test = imaplib.IMAP4(self.email_server, port=get_port(self)) else: + logger.info('Checking incoming POP3 email server {host}:{port} ssl={ssl}...'.format( + host=self.email_server, port=get_port(self), ssl=self.use_ssl)) if self.use_ssl: test = poplib.POP3_SSL(self.email_server, port=get_port(self)) else: test = poplib.POP3(self.email_server, port=get_port(self)) - except Exception: - frappe.throw(_("Incoming email account not correct")) + except Exception as e: + logger.warn('Incoming email account "{host}" not correct'.format(host=self.email_server), exc_info=e) + frappe.throw(title=_("Incoming email account not correct"), + msg='Error connecting IMAP/POP3 "{host}": {e}'.format(host=self.email_server, e=e)) finally: try: @@ -54,22 +62,28 @@ class EmailDomain(Document): if not self.get('smtp_port'): self.smtp_port = 465 + logger.info('Checking outgoing SMTPS email server {host}:{port}...'.format( + host=self.smtp_server, port=self.smtp_port)) sess = smtplib.SMTP_SSL((self.smtp_server or "").encode('utf-8'), cint(self.smtp_port) or None) else: if self.use_tls and not self.smtp_port: self.smtp_port = 587 + logger.info('Checking outgoing SMTP email server {host}:{port} STARTTLS={tls}...'.format( + host=self.smtp_server, port=self.get('smtp_port'), tls=self.use_tls)) sess = smtplib.SMTP(cstr(self.smtp_server or ""), cint(self.smtp_port) or None) sess.quit() - except Exception: - frappe.throw(_("Outgoing email account not correct")) + except Exception as e: + logger.warn('Outgoing email account "{host}" not correct'.format(host=self.smtp_server), exc_info=e) + frappe.throw(title=_("Outgoing email account not correct"), + msg='Error connecting SMTP "{host}": {e}'.format(host=self.smtp_server, e=e)) def on_update(self): """update all email accounts using this domain""" for email_account in frappe.get_all("Email Account", filters={"domain": self.name}): try: email_account = frappe.get_doc("Email Account", email_account.name) - for attr in ["email_server", "use_imap", "use_ssl", "use_tls", "attachment_limit", "smtp_server", "smtp_port", "use_ssl_for_outgoing", "append_emails_to_sent_folder"]: + for attr in ["email_server", "use_imap", "use_ssl", "use_tls", "attachment_limit", "smtp_server", "smtp_port", "use_ssl_for_outgoing", "append_emails_to_sent_folder", "incoming_port"]: email_account.set(attr, self.get(attr, default=0)) email_account.save() diff --git a/frappe/email/doctype/email_domain/test_email_domain.py b/frappe/email/doctype/email_domain/test_email_domain.py index 0050b3250a..1c5306e9c2 100644 --- a/frappe/email/doctype/email_domain/test_email_domain.py +++ b/frappe/email/doctype/email_domain/test_email_domain.py @@ -5,8 +5,35 @@ from __future__ import unicode_literals import frappe import unittest +from frappe.test_runner import make_test_objects -# test_records = frappe.get_test_records('Domain') +test_records = frappe.get_test_records('Email Domain') class TestDomain(unittest.TestCase): - pass + + def setUp(self): + make_test_objects('Email Domain', reset=True) + + def tearDown(self): + frappe.delete_doc("Email Account", "Test") + frappe.delete_doc("Email Domain", "test.com") + + def test_on_update(self): + mail_domain = frappe.get_doc("Email Domain", "test.com") + mail_account = frappe.get_doc("Email Account", "Test") + + # Initially, incoming_port is different in domain and account + self.assertNotEqual(mail_account.incoming_port, mail_domain.incoming_port) + # Trigger update of accounts using this domain + mail_domain.on_update() + mail_account = frappe.get_doc("Email Account", "Test") + # After update, incoming_port in account should match the domain + self.assertEqual(mail_account.incoming_port, mail_domain.incoming_port) + + # Also make sure that the other attributes match + self.assertEqual(mail_account.use_imap, mail_domain.use_imap) + self.assertEqual(mail_account.use_ssl, mail_domain.use_ssl) + self.assertEqual(mail_account.use_tls, mail_domain.use_tls) + self.assertEqual(mail_account.attachment_limit, mail_domain.attachment_limit) + self.assertEqual(mail_account.smtp_server, mail_domain.smtp_server) + self.assertEqual(mail_account.smtp_port, mail_domain.smtp_port) diff --git a/frappe/email/doctype/email_domain/test_records.json b/frappe/email/doctype/email_domain/test_records.json new file mode 100644 index 0000000000..32bc66e150 --- /dev/null +++ b/frappe/email/doctype/email_domain/test_records.json @@ -0,0 +1,30 @@ +[ + { + "doctype": "Email Domain", + "domain_name": "test.com", + "email_id": "_test@test.com", + "email_server": "imap.test.com", + "use_imap": "imap.test.com", + "use_ssl": 1, + "use_tls": 1, + "incoming_port": "993", + "attachment_limit": "1", + "smtp_server": "smtp.test.com", + "smtp_port": "587" + }, + { + "doctype": "Email Account", + "name": "_Test Email Account 1", + "enable_incoming": 1, + "email_id": "_test@test.com", + "domain": "test.com", + "email_server": "imap.test.com", + "use_imap": 1, + "use_ssl": 0, + "use_tls": 1, + "incoming_port": "143", + "attachment_limit": "1", + "smtp_server": "smtp.test.com", + "smtp_port": "587" + } +] diff --git a/frappe/email/doctype/notification/notification.py b/frappe/email/doctype/notification/notification.py index 9a40fb02b7..62be313b82 100644 --- a/frappe/email/doctype/notification/notification.py +++ b/frappe/email/doctype/notification/notification.py @@ -155,7 +155,12 @@ def get_context(context): allow_update = False try: if allow_update and not doc.flags.in_notification_update: - doc.set(self.set_property_after_alert, self.property_value) + fieldname = self.set_property_after_alert + value = self.property_value + if doc.meta.get_field(fieldname).fieldtype in frappe.model.numeric_fieldtypes: + value = frappe.utils.cint(value) + + doc.set(fieldname, value) doc.flags.updater_reference = { 'doctype': self.doctype, 'docname': self.name, @@ -177,7 +182,7 @@ def get_context(context): recipients, cc, bcc = self.get_list_of_recipients(doc, context) users = recipients + cc + bcc - + if not users: return diff --git a/frappe/email/queue.py b/frappe/email/queue.py index 25b6f93f83..f780aebdc1 100755 --- a/frappe/email/queue.py +++ b/frappe/email/queue.py @@ -584,14 +584,15 @@ def prepare_message(email, recipient, recipients_list): return safe_encode(message.as_string()) -def clear_outbox(): - """Remove low priority older than 31 days in Outbox and expire mails not sent for 7 days. - Called daily via scheduler. +def clear_outbox(days=None): + """Remove low priority older than 31 days in Outbox or configured in Log Settings. Note: Used separate query to avoid deadlock """ + if not days: + days=31 email_queues = frappe.db.sql_list("""SELECT `name` FROM `tabEmail Queue` - WHERE `priority`=0 AND `modified` < (NOW() - INTERVAL '31' DAY)""") + WHERE `priority`=0 AND `modified` < (NOW() - INTERVAL '{0}' DAY)""".format(days)) if email_queues: frappe.db.sql("""DELETE FROM `tabEmail Queue` WHERE `name` IN ({0})""".format( @@ -602,6 +603,11 @@ def clear_outbox(): ','.join(['%s']*len(email_queues) )), tuple(email_queues)) +def set_expiry_for_email_queue(): + ''' Mark emails as expire that has not sent for 7 days. + Called daily via scheduler. + ''' + frappe.db.sql(""" UPDATE `tabEmail Queue` SET `status`='Expired' diff --git a/frappe/geo/country_info.json b/frappe/geo/country_info.json index 6a56107333..e4c4e278b0 100644 --- a/frappe/geo/country_info.json +++ b/frappe/geo/country_info.json @@ -558,6 +558,8 @@ "code": "cn", "currency": "CNY", "currency_name": "Yuan Renminbi", + "currency_fraction_units": 100, + "smallest_currency_fraction_value": 0.01, "date_format": "yyyy-mm-dd", "number_format": "#,###.##", "timezones": [ @@ -1389,7 +1391,10 @@ "code": "la", "currency": "LAK", "currency_name": "Kip", - "number_format": "#,###.##" + "number_format": "#,###.##", + "timezones":[ + "Asia/Vientiane" + ] }, "Latvia": { "code": "lv", diff --git a/frappe/hooks.py b/frappe/hooks.py index dd267657fd..490c689090 100644 --- a/frappe/hooks.py +++ b/frappe/hooks.py @@ -141,6 +141,7 @@ doc_events = { "frappe.automation.doctype.milestone_tracker.milestone_tracker.evaluate_milestone", "frappe.core.doctype.file.file.attach_files_to_document", "frappe.event_streaming.doctype.event_update_log.event_update_log.notify_consumers", + "frappe.automation.doctype.assignment_rule.assignment_rule.update_due_date", ], "after_rename": "frappe.desk.notifications.clear_doctype_notifications", "on_cancel": [ @@ -203,7 +204,7 @@ scheduler_events = { "frappe.utils.password.delete_password_reset_cache" ], "daily": [ - "frappe.email.queue.clear_outbox", + "frappe.email.queue.set_expiry_for_email_queue", "frappe.desk.notifications.clear_notifications", "frappe.core.doctype.error_log.error_log.set_old_logs_as_seen", "frappe.desk.doctype.event.event.send_event_digest", @@ -212,7 +213,6 @@ scheduler_events = { "frappe.realtime.remove_old_task_logs", "frappe.utils.scheduler.restrict_scheduler_events_if_dormant", "frappe.email.doctype.auto_email_report.auto_email_report.send_daily", - "frappe.core.doctype.activity_log.activity_log.clear_authentication_logs", "frappe.website.doctype.personal_data_deletion_request.personal_data_deletion_request.remove_unverified_record", "frappe.desk.form.document_follow.send_daily_updates", "frappe.social.doctype.energy_point_settings.energy_point_settings.allocate_review_points", @@ -220,7 +220,8 @@ scheduler_events = { "frappe.automation.doctype.auto_repeat.auto_repeat.make_auto_repeat_entry", "frappe.automation.doctype.auto_repeat.auto_repeat.set_auto_repeat_as_completed", "frappe.email.doctype.unhandled_email.unhandled_email.remove_old_unhandled_emails", - "frappe.core.doctype.prepared_report.prepared_report.delete_expired_prepared_reports" + "frappe.core.doctype.prepared_report.prepared_report.delete_expired_prepared_reports", + "frappe.core.doctype.log_settings.log_settings.run_log_clean_up" ], "daily_long": [ "frappe.integrations.doctype.dropbox_settings.dropbox_settings.take_backups_daily", diff --git a/frappe/installer.py b/frappe/installer.py index 2a912695e5..c6549f16ee 100755 --- a/frappe/installer.py +++ b/frappe/installer.py @@ -345,8 +345,7 @@ def extract_sql_gzip(sql_gz_path): return decompressed_file - -def extract_tar_files(site_name, file_path, folder_name): +def extract_files(site_name, file_path, folder_name): import subprocess import shutil @@ -362,7 +361,10 @@ def extract_tar_files(site_name, file_path, folder_name): tar_path = os.path.join(abs_site_path, tar_name) try: - subprocess.check_output(['tar', 'xvf', tar_path, '--strip', '2'], cwd=abs_site_path) + if file_path.endswith(".tar"): + subprocess.check_output(['tar', 'xvf', tar_path, '--strip', '2'], cwd=abs_site_path) + elif file_path.endswith(".tgz"): + subprocess.check_output(['tar', 'zxvf', tar_path, '--strip', '2'], cwd=abs_site_path) except: raise finally: diff --git a/frappe/integrations/doctype/integration_request/integration_request.py b/frappe/integrations/doctype/integration_request/integration_request.py index 2cb656459b..f1d59beb5a 100644 --- a/frappe/integrations/doctype/integration_request/integration_request.py +++ b/frappe/integrations/doctype/integration_request/integration_request.py @@ -6,6 +6,8 @@ from __future__ import unicode_literals import frappe from frappe.model.document import Document import json +from six import string_types +from frappe.integrations.utils import json_handler class IntegrationRequest(Document): def autoname(self): @@ -20,3 +22,17 @@ class IntegrationRequest(Document): self.status = status self.save(ignore_permissions=True) frappe.db.commit() + + def handle_success(self, response): + """update the output field with the response along with the relevant status""" + if isinstance(response, string_types): + response = json.loads(response) + self.db_set("status", "Completed") + self.db_set("output", json.dumps(response, default=json_handler)) + + def handle_failure(self, response): + """update the error field with the response along with the relevant status""" + if isinstance(response, string_types): + response = json.loads(response) + self.db_set("status", "Failed") + self.db_set("error", json.dumps(response, default=json_handler)) \ No newline at end of file diff --git a/frappe/integrations/utils.py b/frappe/integrations/utils.py index 808affe47a..1af9682073 100644 --- a/frappe/integrations/utils.py +++ b/frappe/integrations/utils.py @@ -49,16 +49,20 @@ def make_post_request(url, auth=None, headers=None, data=None): frappe.log_error() raise exc -def create_request_log(data, integration_type, service_name, name=None): +def create_request_log(data, integration_type, service_name, name=None, error=None): if isinstance(data, string_types): data = json.loads(data) + if isinstance(error, string_types): + error = json.loads(error) + integration_request = frappe.get_doc({ "doctype": "Integration Request", "integration_type": integration_type, "integration_request_service": service_name, "reference_doctype": data.get("reference_doctype"), "reference_docname": data.get("reference_docname"), + "error": json.dumps(error, default=json_handler), "data": json.dumps(data, default=json_handler) }) diff --git a/frappe/model/base_document.py b/frappe/model/base_document.py index 5497090e72..0a219b4253 100644 --- a/frappe/model/base_document.py +++ b/frappe/model/base_document.py @@ -347,7 +347,7 @@ class BaseDocument(object): if self.meta.autoname=="hash": # hash collision? try again frappe.flags.retry_count = (frappe.flags.retry_count or 0) + 1 - if frappe.flags.retry_count > 5: + if frappe.flags.retry_count > 5 and not frappe.flags.in_test: raise self.name = None self.db_insert() diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py index fb8a027d20..596f69d2dd 100644 --- a/frappe/model/db_query.py +++ b/frappe/model/db_query.py @@ -58,7 +58,10 @@ class DatabaseQuery(object): if fields: self.fields = fields else: - self.fields = ["`tab{0}`.`name`".format(self.doctype)] + if pluck: + self.fields = ["`tab{0}`.`{1}`".format(self.doctype, pluck)] + else: + self.fields = ["`tab{0}`.`name`".format(self.doctype)] if start: limit_start = start if page_length: limit_page_length = page_length @@ -168,8 +171,16 @@ class DatabaseQuery(object): fields = [] + # Wrapping fields with grave quotes to allow support for sql keywords + # TODO: Add support for wrapping fields with sql functions and distinct keyword for field in self.fields: - if (field.strip().startswith(("`", "*")) or "(" in field): + stripped_field = field.strip().lower() + skip_wrapping = any([ + stripped_field.startswith(("`", "*", '"', "'")), + "(" in stripped_field, + "distinct" in stripped_field, + ]) + if skip_wrapping: fields.append(field) elif "as" in field.lower().split(" "): col, _, new = field.split() diff --git a/frappe/modules/utils.py b/frappe/modules/utils.py index bba2f62856..b3debfc43c 100644 --- a/frappe/modules/utils.py +++ b/frappe/modules/utils.py @@ -90,10 +90,11 @@ def sync_customizations(app=None): folder = frappe.get_app_path(app_name, module_name, 'custom') if os.path.exists(folder): for fname in os.listdir(folder): - with open(os.path.join(folder, fname), 'r') as f: - data = json.loads(f.read()) - if data.get('sync_on_migrate'): - sync_customizations_for_doctype(data, folder) + if fname.endswith('.json'): + with open(os.path.join(folder, fname), 'r') as f: + data = json.loads(f.read()) + if data.get('sync_on_migrate'): + sync_customizations_for_doctype(data, folder) def sync_customizations_for_doctype(data, folder): diff --git a/frappe/patches.txt b/frappe/patches.txt index f65a896d23..dd8654c81b 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -208,7 +208,7 @@ frappe.patches.v9_1.resave_domain_settings frappe.patches.v9_1.revert_domain_settings frappe.patches.v9_1.move_feed_to_activity_log execute:frappe.delete_doc('Page', 'data-import-tool', ignore_missing=True) -frappe.patches.v10_0.reload_countries_and_currencies # 20-10-2020 +frappe.patches.v10_0.reload_countries_and_currencies # 14-10-2020 frappe.patches.v10_0.refactor_social_login_keys frappe.patches.v10_0.enable_chat_by_default_within_system_settings frappe.patches.v10_0.remove_custom_field_for_disabled_domain @@ -314,4 +314,4 @@ frappe.patches.v13_0.enable_custom_script frappe.patches.v13_0.update_newsletter_content_type execute:frappe.db.set_value('Website Settings', 'Website Settings', {'navbar_template': 'Standard Navbar', 'footer_template': 'Standard Footer'}) frappe.patches.v13_0.delete_event_producer_and_consumer_keys -frappe.patches.v13_0.web_template_set_module +frappe.patches.v13_0.web_template_set_module #2020-10-05 diff --git a/frappe/patches/v13_0/generate_theme_files_in_public_folder.py b/frappe/patches/v13_0/generate_theme_files_in_public_folder.py index 4315f06ebe..bcb47bec24 100644 --- a/frappe/patches/v13_0/generate_theme_files_in_public_folder.py +++ b/frappe/patches/v13_0/generate_theme_files_in_public_folder.py @@ -6,6 +6,7 @@ import frappe def execute(): + frappe.reload_doc("website", "doctype", "website_theme_ignore_app") themes = frappe.db.get_all( "Website Theme", filters={"theme_url": ("not like", "/files/website_theme/%")} ) diff --git a/frappe/patches/v13_0/web_template_set_module.py b/frappe/patches/v13_0/web_template_set_module.py index b4ccb21ef2..df008557d8 100644 --- a/frappe/patches/v13_0/web_template_set_module.py +++ b/frappe/patches/v13_0/web_template_set_module.py @@ -6,7 +6,9 @@ import frappe def execute(): """Set default module for standard Web Template, if none.""" - frappe.reload_doc('website', 'doctype', 'Web Template') + frappe.reload_doc('website', 'doctype', 'Web Template Field') + frappe.reload_doc('website', 'doctype', 'web_template') + standard_templates = frappe.get_list('Web Template', {'standard': 1}) for template in standard_templates: doc = frappe.get_doc('Web Template', template.name) diff --git a/frappe/public/js/frappe/desk.js b/frappe/public/js/frappe/desk.js index 9658a9db08..2dd8be86e0 100644 --- a/frappe/public/js/frappe/desk.js +++ b/frappe/public/js/frappe/desk.js @@ -154,6 +154,26 @@ frappe.Application = Class.extend({ } }); }, 300000); // check every 5 minutes + + if(frappe.user.has_role("System Manager")){ + setInterval(function() { + frappe.call({ + method: 'frappe.core.doctype.log_settings.log_settings.has_unseen_error_log', + args: { + user: frappe.session.user + }, + callback: function(r) { + console.log(r); + if(r.message.show_alert){ + frappe.show_alert({ + indicator: 'red', + message: r.message.message + }); + } + } + }); + }, 600000); // check every 10 minutes + } } this.fetch_tags(); diff --git a/frappe/public/js/frappe/form/controls/dynamic_link.js b/frappe/public/js/frappe/form/controls/dynamic_link.js index 02e970091e..00bb02a5fc 100644 --- a/frappe/public/js/frappe/form/controls/dynamic_link.js +++ b/frappe/public/js/frappe/form/controls/dynamic_link.js @@ -1,14 +1,12 @@ frappe.ui.form.ControlDynamicLink = frappe.ui.form.ControlLink.extend({ get_options: function() { let options = ''; - if(this.df.get_options) { + if (this.df.get_options) { options = this.df.get_options(); - } - else if (this.docname==null && cur_dialog) { + } else if (this.docname==null && cur_dialog) { //for dialog box options = cur_dialog.get_value(this.df.options); - } - else if (!cur_frm) { + } else if (!cur_frm) { const selector = `input[data-fieldname="${this.df.options}"]`; let input = null; if (cur_list) { @@ -21,13 +19,12 @@ frappe.ui.form.ControlDynamicLink = frappe.ui.form.ControlLink.extend({ if (input) { options = input.val(); } - } - else { + } else { options = frappe.model.get_value(this.df.parent, this.docname, this.df.options); } if (frappe.model.is_single(options)) { - frappe.throw(__(`${options.bold()} is not a valid DocType for Dynamic Link`)); + frappe.throw(__("{0} is not a valid DocType for Dynamic Link", [options.bold()])); } return options; diff --git a/frappe/public/js/frappe/form/controls/int.js b/frappe/public/js/frappe/form/controls/int.js index 91dee45838..9b59444fd3 100644 --- a/frappe/public/js/frappe/form/controls/int.js +++ b/frappe/public/js/frappe/form/controls/int.js @@ -1,17 +1,17 @@ frappe.ui.form.ControlInt = frappe.ui.form.ControlData.extend({ - make: function() { + make: function () { this._super(); // $(this.label_area).addClass('pull-right'); // $(this.disp_area).addClass('text-right'); }, - make_input: function() { + make_input: function () { var me = this; this._super(); this.$input // .addClass("text-right") - .on("focus", function() { - setTimeout(function() { - if(!document.activeElement) return; + .on("focus", function () { + setTimeout(function () { + if (!document.activeElement) return; document.activeElement.value = me.validate(document.activeElement.value); document.activeElement.select(); @@ -19,7 +19,10 @@ frappe.ui.form.ControlInt = frappe.ui.form.ControlData.extend({ return false; }); }, - eval_expression: function(value) { + validate: function (value) { + return this.parse(value); + }, + eval_expression: function (value) { if (typeof value === 'string') { if (value.match(/^[0-9+\-/* ]+$/)) { // If it is a string containing operators @@ -33,7 +36,7 @@ frappe.ui.form.ControlInt = frappe.ui.form.ControlData.extend({ } return value; }, - parse: function(value) { + parse: function (value) { return cint(this.eval_expression(value), null); } }); diff --git a/frappe/public/js/frappe/form/form.js b/frappe/public/js/frappe/form/form.js index e34187f329..3c41b027a5 100644 --- a/frappe/public/js/frappe/form/form.js +++ b/frappe/public/js/frappe/form/form.js @@ -1522,11 +1522,12 @@ frappe.ui.form.Form = class FrappeForm { const escaped_name = encodeURIComponent(value); - return repl('%(label)s', { + return repl('%(label)s', { color: get_color(doc || {}), doctype: df.options, - name: escaped_name, - label: label + escaped_name: escaped_name, + label: label, + name: value }); } else { return ''; diff --git a/frappe/public/js/frappe/form/grid.js b/frappe/public/js/frappe/form/grid.js index 7697ca99f6..87c8106bd8 100644 --- a/frappe/public/js/frappe/form/grid.js +++ b/frappe/public/js/frappe/form/grid.js @@ -775,6 +775,13 @@ export default class Grid { // download this.setup_download(); + const value_formatter_map = { + "Date": val => val ? frappe.datetime.user_to_str(val) : val, + "Int": val => cint(val), + "Check": val => cint(val), + "Float": val => flt(val), + }; + // upload frappe.flags.no_socketio = true; $(this.wrapper).find(".grid-upload").removeClass('hidden').on("click", () => { @@ -802,16 +809,9 @@ export default class Grid { var fieldname = fieldnames[ci]; var df = frappe.meta.get_docfield(me.df.options, fieldname); - // convert date formatting - if (df.fieldtype === "Date" && value) { - value = frappe.datetime.user_to_str(value); - } - - if (df.fieldtype === "Int" || df.fieldtype === "Check") { - value = cint(value); - } - - d[fieldnames[ci]] = value; + d[fieldnames[ci]] = value_formatter_map[df.fieldtype] + ? value_formatter_map[df.fieldtype](value) + : value; }); } } diff --git a/frappe/public/js/frappe/form/sidebar/review.js b/frappe/public/js/frappe/form/sidebar/review.js index e4d6daa181..8e0cd64b98 100644 --- a/frappe/public/js/frappe/form/sidebar/review.js +++ b/frappe/public/js/frappe/form/sidebar/review.js @@ -4,7 +4,7 @@ frappe.ui.form.Review = class Review { - constructor({parent, frm}) { + constructor({ parent, frm }) { this.parent = parent; this.frm = frm; this.points = frappe.boot.points; @@ -49,7 +49,7 @@ frappe.ui.form.Review = class Review { const docinfo = this.frm.get_docinfo(); involved_users = involved_users.concat( - docinfo.communications.map(d => d.sender && d.delivery_status==='sent'), + docinfo.communications.map(d => d.sender && d.delivery_status === 'sent'), docinfo.comments.map(d => d.owner), docinfo.versions.map(d => d.owner), docinfo.assignments.map(d => d.owner) @@ -89,7 +89,7 @@ frappe.ui.form.Review = class Review { fieldtype: 'Int', label: __('Points'), reqd: 1, - description: __(`Currently you have ${this.points.review_points} review points`) + description: __("Currently you have {0} review points", [this.points.review_points]) }, { fieldtype: 'Small Text', fieldname: 'reason', @@ -132,7 +132,7 @@ frappe.ui.form.Review = class Review { this.reviews.empty(); review_logs.forEach(log => { let review_pill = $(` -
  • +
  • ${Math.abs(log.points)}
    @@ -166,7 +166,7 @@ frappe.ui.form.Review = class Review { delay: 500, placement: 'top', template: ` -