diff --git a/.gitignore b/.gitignore index 7df673c1f1..900ae1c7b7 100644 --- a/.gitignore +++ b/.gitignore @@ -188,4 +188,7 @@ typings/ # cypress cypress/screenshots -cypress/videos \ No newline at end of file +cypress/videos + +# JetBrains IDEs +.idea/ diff --git a/.travis.yml b/.travis.yml index 63895675ea..2331217363 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,12 +31,12 @@ matrix: - name: "Python 3.7 MariaDB" python: 3.7 env: DB=mariadb TYPE=server - script: bench --site test_site run-tests --coverage + script: bench --verbose --site test_site run-tests --coverage - name: "Python 3.7 PostgreSQL" python: 3.7 env: DB=postgres TYPE=server - script: bench --site test_site run-tests --coverage + script: bench --verbose --site test_site run-tests --coverage - name: "Cypress" python: 3.7 diff --git a/.travis/consumer_db/mariadb.json b/.travis/consumer_db/mariadb.json index e08ad70174..fb5b3bc976 100644 --- a/.travis/consumer_db/mariadb.json +++ b/.travis/consumer_db/mariadb.json @@ -2,6 +2,7 @@ "db_host": "localhost", "db_name": "test_frappe_consumer", "db_password": "test_frappe", + "allow_tests": true, "db_type": "mariadb", "auto_email_id": "test@example.com", "mail_server": "smtp.example.com", diff --git a/.travis/consumer_db/postgres.json b/.travis/consumer_db/postgres.json index 5923f602ba..fed9fdfde2 100644 --- a/.travis/consumer_db/postgres.json +++ b/.travis/consumer_db/postgres.json @@ -3,6 +3,7 @@ "db_name": "test_frappe_consumer", "db_password": "test_frappe", "db_type": "postgres", + "allow_tests": true, "auto_email_id": "test@example.com", "mail_server": "smtp.example.com", "mail_login": "test@example.com", diff --git a/.travis/producer_db/mariadb.json b/.travis/producer_db/mariadb.json index 8860241d80..988282a554 100644 --- a/.travis/producer_db/mariadb.json +++ b/.travis/producer_db/mariadb.json @@ -2,6 +2,7 @@ "db_host": "localhost", "db_name": "test_frappe_producer", "db_password": "test_frappe", + "allow_tests": true, "db_type": "mariadb", "auto_email_id": "test@example.com", "mail_server": "smtp.example.com", diff --git a/.travis/producer_db/postgres.json b/.travis/producer_db/postgres.json index 4d846eee98..6426e99058 100644 --- a/.travis/producer_db/postgres.json +++ b/.travis/producer_db/postgres.json @@ -3,6 +3,7 @@ "db_name": "test_frappe_producer", "db_password": "test_frappe", "db_type": "postgres", + "allow_tests": true, "auto_email_id": "test@example.com", "mail_server": "smtp.example.com", "mail_login": "test@example.com", diff --git a/frappe/__init__.py b/frappe/__init__.py index d644d2a473..36a8b48ecd 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -153,6 +153,7 @@ def init(site, sites_path=None, new_site=False): local.site = site local.sites_path = sites_path local.site_path = os.path.join(sites_path, site) + local.all_apps = None local.request_ip = None local.response = _dict({"docs":[]}) @@ -231,8 +232,7 @@ def get_site_config(sites_path=None, site_path=None): if os.path.exists(site_config): config.update(get_file_json(site_config)) elif local.site and not local.flags.new_site: - print("Site {0} does not exist".format(local.site)) - sys.exit(1) + raise IncorrectSitePath("{0} does not exist".format(local.site)) return _dict(config) @@ -267,7 +267,7 @@ def destroy(): # memcache redis_server = None def cache(): - """Returns memcache connection.""" + """Returns redis connection.""" global redis_server if not redis_server: from frappe.utils.redis_wrapper import RedisWrapper @@ -290,6 +290,9 @@ def errprint(msg): error_log.append({"exc": msg}) +def print_sql(enable=True): + return cache().set_value('flag_print_sql', enable) + def log(msg): """Add to `debug_log`. @@ -300,7 +303,7 @@ def log(msg): debug_log.append(as_unicode(msg)) -def msgprint(msg, title=None, raise_exception=0, as_table=False, indicator=None, alert=False, primary_action=None, is_minimizable=None): +def msgprint(msg, title=None, raise_exception=0, as_table=False, indicator=None, alert=False, primary_action=None, is_minimizable=None, wide=None): """Print a message to the user (via HTTP response). Messages are sent in the `__server_messages` property in the response JSON and shown in a pop-up / modal. @@ -310,6 +313,8 @@ def msgprint(msg, title=None, raise_exception=0, as_table=False, indicator=None, :param raise_exception: [optional] Raise given exception and show message. :param as_table: [optional] If `msg` is a list of lists, render as HTML table. :param primary_action: [optional] Bind a primary server/client side action. + :param is_minimizable: [optional] Allow users to minimize the modal + :param wide: [optional] Show wide modal """ from frappe.utils import encode @@ -367,6 +372,9 @@ def msgprint(msg, title=None, raise_exception=0, as_table=False, indicator=None, if primary_action: out.primary_action = primary_action + if wide: + out.wide = wide + message_log.append(json.dumps(out)) if raise_exception and hasattr(raise_exception, '__name__'): @@ -388,12 +396,12 @@ def clear_last_message(): if len(local.message_log) > 0: local.message_log = local.message_log[:-1] -def throw(msg, exc=ValidationError, title=None, is_minimizable=None): +def throw(msg, exc=ValidationError, title=None, is_minimizable=None, wide=None): """Throw execption and show message (`msgprint`). :param msg: Message. :param exc: Exception class. Default `frappe.ValidationError`""" - msgprint(msg, raise_exception=exc, title=title, indicator='red', is_minimizable=is_minimizable) + msgprint(msg, raise_exception=exc, title=title, indicator='red', is_minimizable=is_minimizable, wide=wide) def emit_js(js, user=False, **kwargs): if user == False: @@ -436,12 +444,8 @@ def get_roles(username=None): """Returns roles of current user.""" if not local.session: return ["Guest"] - - if username: - import frappe.permissions - return frappe.permissions.get_roles(username) - else: - return get_user().get_roles() + import frappe.permissions + return frappe.permissions.get_roles(username or local.session.user) def get_request_header(key, default=None): """Return HTTP request header. @@ -762,7 +766,7 @@ def get_doc(*args, **kwargs): # insert a new document todo = frappe.get_doc({"doctype":"ToDo", "description": "test"}) - tood.insert() + todo.insert() # open an existing document todo = frappe.get_doc("ToDo", "TD0001") @@ -921,10 +925,13 @@ def get_installed_apps(sort=False, frappe_last=False): if not db: connect() + if not local.all_apps: + local.all_apps = get_all_apps(True) + installed = json.loads(db.get_global("installed_apps") or "[]") if sort: - installed = [app for app in get_all_apps(True) if app in installed] + installed = [app for app in local.all_apps if app in installed] if frappe_last: if 'frappe' in installed: @@ -1559,10 +1566,10 @@ def get_doctype_app(doctype): loggers = {} log_level = None -def logger(module=None, with_more_info=False): +def logger(module=None, with_more_info=False, allow_site=True, filter=None, max_size=100_000, file_count=20): '''Returns a python logger that uses StreamHandler''' from frappe.utils.logger import get_logger - return get_logger(module=module, with_more_info=with_more_info) + return get_logger(module=module, with_more_info=with_more_info, allow_site=allow_site, filter=filter, max_size=max_size, file_count=file_count) def log_error(message=None, title=_("Error")): '''Log error to Error Log''' @@ -1707,3 +1714,7 @@ def mock(type, size=1, locale='en'): from frappe.chat.util import squashify return squashify(results) + +def validate_and_sanitize_search_inputs(fn): + from frappe.desk.search import validate_and_sanitize_search_inputs as func + return func(fn) diff --git a/frappe/app.py b/frappe/app.py index 57db867882..39bff83122 100644 --- a/frappe/app.py +++ b/frappe/app.py @@ -99,15 +99,16 @@ def application(request): frappe.monitor.stop(response) frappe.recorder.dump() - frappe.logger("frappe.web").info({ - "site": get_site_name(request.host), - "remote_addr": getattr(request, "remote_addr", "NOTFOUND"), - "base_url": getattr(request, "base_url", "NOTFOUND"), - "full_path": getattr(request, "full_path", "NOTFOUND"), - "method": getattr(request, "method", "NOTFOUND"), - "scheme": getattr(request, "scheme", "NOTFOUND"), - "http_status_code": getattr(response, "status_code", "NOTFOUND") - }) + if hasattr(frappe.local, 'conf') and frappe.local.conf.enable_frappe_logger: + frappe.logger("frappe.web", allow_site=frappe.local.site).info({ + "site": get_site_name(request.host), + "remote_addr": getattr(request, "remote_addr", "NOTFOUND"), + "base_url": getattr(request, "base_url", "NOTFOUND"), + "full_path": getattr(request, "full_path", "NOTFOUND"), + "method": getattr(request, "method", "NOTFOUND"), + "scheme": getattr(request, "scheme", "NOTFOUND"), + "http_status_code": getattr(response, "status_code", "NOTFOUND") + }) if response and hasattr(frappe.local, 'rate_limiter'): response.headers.extend(frappe.local.rate_limiter.headers()) @@ -256,9 +257,11 @@ def serve(port=8000, profile=False, no_reload=False, no_threading=False, site=No 'SERVER_NAME': 'localhost:8000' } + log = logging.getLogger('werkzeug') + log.propagate = False + in_test_env = os.environ.get('CI') if in_test_env: - log = logging.getLogger('werkzeug') log.setLevel(logging.ERROR) run_simple('0.0.0.0', int(port), application, diff --git a/frappe/auth.py b/frappe/auth.py index 64fea36748..6d51629c58 100644 --- a/frappe/auth.py +++ b/frappe/auth.py @@ -18,6 +18,7 @@ from frappe.utils.password import check_password, delete_login_failed_cache from frappe.core.doctype.activity_log.activity_log import add_authentication_log from frappe.twofactor import (should_run_2fa, authenticate_for_2factor, confirm_otp_token, get_cached_user_pass) +from frappe.website.utils import get_home_page from six.moves.urllib.parse import quote @@ -167,7 +168,7 @@ class LoginManager: frappe.local.cookie_manager.set_cookie("system_user", "no") if not resume: frappe.local.response["message"] = "No App" - frappe.local.response["home_page"] = get_website_user_home_page(self.user) + frappe.local.response["home_page"] = '/' + get_home_page() else: frappe.local.cookie_manager.set_cookie("system_user", "yes") if not resume: @@ -338,8 +339,13 @@ class CookieManager: self.set_cookie("country", frappe.session.session_country) def set_cookie(self, key, value, expires=None, secure=False, httponly=False, samesite="Lax"): - if not secure: + if not secure and hasattr(frappe.local, 'request'): secure = frappe.local.request.scheme == "https" + + # Cordova does not work with Lax + if frappe.local.session.data.device == "mobile": + samesite = None + self.cookies[key] = { "value": value, "expires": expires, @@ -377,16 +383,6 @@ def clear_cookies(): frappe.session.sid = "" frappe.local.cookie_manager.delete_cookie(["full_name", "user_id", "sid", "user_image", "system_user"]) -def get_website_user_home_page(user): - home_page_method = frappe.get_hooks('get_website_user_home_page') - if home_page_method: - home_page = frappe.get_attr(home_page_method[-1])(user) - return '/' + home_page.strip('/') - elif frappe.get_hooks('website_user_home_page'): - return '/' + frappe.get_hooks('website_user_home_page')[-1].strip('/') - else: - return '/me' - def get_last_tried_login_data(user, get_last_login=False): locked_account_time = frappe.cache().hget('locked_account_time', user) if get_last_login and locked_account_time: diff --git a/frappe/automation/desk_page/tools/tools.json b/frappe/automation/desk_page/tools/tools.json index 2164a4ce38..3fbaf62d02 100644 --- a/frappe/automation/desk_page/tools/tools.json +++ b/frappe/automation/desk_page/tools/tools.json @@ -3,7 +3,7 @@ { "hidden": 0, "label": "Tools", - "links": "[\n {\n \"description\": \"Documents assigned to you and by you.\",\n \"label\": \"To Do\",\n \"name\": \"ToDo\",\n \"onboard\": 1,\n \"type\": \"doctype\"\n },\n {\n \"description\": \"Event and other calendars.\",\n \"label\": \"Calendar\",\n \"link\": \"List/Event/Calendar\",\n \"name\": \"Event\",\n \"onboard\": 1,\n \"type\": \"doctype\"\n },\n {\n \"description\": \"Private and public Notes.\",\n \"label\": \"Note\",\n \"name\": \"Note\",\n \"onboard\": 1,\n \"type\": \"doctype\"\n },\n {\n \"label\": \"Files\",\n \"name\": \"File\",\n \"type\": \"doctype\"\n },\n {\n \"label\": \"Video\",\n \"name\": \"Video\",\n \"type\": \"doctype\"\n },\n {\n \"description\": \"Activity log of all users.\",\n \"label\": \"Activity\",\n \"name\": \"activity\",\n \"type\": \"page\"\n }\n]" + "links": "[\n {\n \"description\": \"Documents assigned to you and by you.\",\n \"label\": \"To Do\",\n \"name\": \"ToDo\",\n \"onboard\": 1,\n \"type\": \"doctype\"\n },\n {\n \"description\": \"Event and other calendars.\",\n \"label\": \"Calendar\",\n \"link\": \"List/Event/Calendar\",\n \"name\": \"Event\",\n \"onboard\": 1,\n \"type\": \"doctype\"\n },\n {\n \"description\": \"Private and public Notes.\",\n \"label\": \"Note\",\n \"name\": \"Note\",\n \"onboard\": 1,\n \"type\": \"doctype\"\n },\n {\n \"label\": \"Files\",\n \"name\": \"File\",\n \"type\": \"doctype\"\n },\n {\n \"description\": \"Activity log of all users.\",\n \"label\": \"Activity\",\n \"name\": \"activity\",\n \"type\": \"page\"\n }\n]" }, { "hidden": 0, @@ -29,10 +29,11 @@ "docstatus": 0, "doctype": "Desk Page", "extends_another_page": 0, + "hide_custom": 0, "idx": 0, "is_standard": 1, "label": "Tools", - "modified": "2020-04-20 18:21:14.152537", + "modified": "2020-07-21 19:32:18.480700", "modified_by": "Administrator", "module": "Automation", "name": "Tools", diff --git a/frappe/automation/doctype/assignment_rule/assignment_rule.py b/frappe/automation/doctype/assignment_rule/assignment_rule.py index 78f05e7fe9..c4bd49b870 100644 --- a/frappe/automation/doctype/assignment_rule/assignment_rule.py +++ b/frappe/automation/doctype/assignment_rule/assignment_rule.py @@ -9,6 +9,7 @@ from frappe.model.document import Document from frappe.desk.form import assign_to import frappe.cache_manager from frappe import _ +from frappe.model import log_types class AssignmentRule(Document): @@ -19,10 +20,13 @@ class AssignmentRule(Document): frappe.throw(_("Assignment Day {0} has been repeated.").format(frappe.bold(repeated_days))) def on_update(self): # pylint: disable=no-self-use - frappe.cache_manager.clear_doctype_map('Assignment Rule', self.name) + frappe.cache_manager.clear_doctype_map('Assignment Rule', self.document_type) def after_rename(self, old, new, merge): # pylint: disable=no-self-use - frappe.cache_manager.clear_doctype_map('Assignment Rule', self.name) + frappe.cache_manager.clear_doctype_map('Assignment Rule', self.document_type) + + def on_trash(self): # pylint: disable=no-self-use + frappe.cache_manager.clear_doctype_map('Assignment Rule', self.document_type) def apply_unassign(self, doc, assignments): if (self.unassign_condition and @@ -165,7 +169,13 @@ def reopen_closed_assignment(doc): return True def apply(doc, method=None, doctype=None, name=None): - if frappe.flags.in_patch or frappe.flags.in_install or frappe.flags.in_setup_wizard: + if not doctype: + doctype = doc.doctype + + if (frappe.flags.in_patch + or frappe.flags.in_install + or frappe.flags.in_setup_wizard + or doctype in log_types): return if not doc and doctype and name: diff --git a/frappe/automation/doctype/auto_repeat/auto_repeat.py b/frappe/automation/doctype/auto_repeat/auto_repeat.py index a946fcc81c..c09e347e71 100644 --- a/frappe/automation/doctype/auto_repeat/auto_repeat.py +++ b/frappe/automation/doctype/auto_repeat/auto_repeat.py @@ -374,6 +374,7 @@ def make_auto_repeat(doctype, docname, frequency = 'Daily', start_date = None, e # method for reference_doctype filter @frappe.whitelist() +@frappe.validate_and_sanitize_search_inputs def get_auto_repeat_doctypes(doctype, txt, searchfield, start, page_len, filters): res = frappe.db.get_all('Property Setter', { 'property': 'allow_auto_repeat', diff --git a/frappe/automation/doctype/milestone_tracker/milestone_tracker.py b/frappe/automation/doctype/milestone_tracker/milestone_tracker.py index 154cb599e1..388620bfb4 100644 --- a/frappe/automation/doctype/milestone_tracker/milestone_tracker.py +++ b/frappe/automation/doctype/milestone_tracker/milestone_tracker.py @@ -7,13 +7,14 @@ from __future__ import unicode_literals import frappe from frappe.model.document import Document import frappe.cache_manager +from frappe.model import log_types class MilestoneTracker(Document): def on_update(self): - frappe.cache_manager.clear_doctype_map('Milestone Tracker', self.name) + frappe.cache_manager.clear_doctype_map('Milestone Tracker', self.document_type) def on_trash(self): - frappe.cache_manager.clear_doctype_map('Milestone Tracker', self.name) + frappe.cache_manager.clear_doctype_map('Milestone Tracker', self.document_type) def apply(self, doc): before_save = doc.get_doc_before_save() @@ -32,8 +33,15 @@ class MilestoneTracker(Document): def evaluate_milestone(doc, event): if (frappe.flags.in_install or frappe.flags.in_migrate - or frappe.flags.in_setup_wizard): + or frappe.flags.in_setup_wizard + or doc.doctype in log_types): return - for d in frappe.cache_manager.get_doctype_map('Milestone Tracker', doc.doctype, - dict(document_type = doc.doctype, disabled=0)): - frappe.get_doc('Milestone Tracker', d.name).apply(doc) + + # track milestones related to this doctype + for d in get_milestone_trackers(doc.doctype): + frappe.get_doc('Milestone Tracker', d.get('name')).apply(doc) + +def get_milestone_trackers(doctype): + return frappe.cache_manager.get_doctype_map('Milestone Tracker', doctype, + dict(document_type = doctype, disabled=0)) + diff --git a/frappe/automation/doctype/milestone_tracker/test_milestone_tracker.py b/frappe/automation/doctype/milestone_tracker/test_milestone_tracker.py index c9bb6b7d5f..05db3b025e 100644 --- a/frappe/automation/doctype/milestone_tracker/test_milestone_tracker.py +++ b/frappe/automation/doctype/milestone_tracker/test_milestone_tracker.py @@ -4,12 +4,16 @@ from __future__ import unicode_literals import frappe +import frappe.cache_manager import unittest class TestMilestoneTracker(unittest.TestCase): def test_milestone(self): frappe.db.sql('delete from `tabMilestone Tracker`') - frappe.get_doc(dict( + + frappe.cache().delete_key('milestone_tracker_map') + + milestone_tracker = frappe.get_doc(dict( doctype = 'Milestone Tracker', document_type = 'ToDo', track_field = 'status' @@ -17,7 +21,8 @@ class TestMilestoneTracker(unittest.TestCase): todo = frappe.get_doc(dict( doctype = 'ToDo', - description = 'test milestone' + description = 'test milestone', + status = 'Open' )).insert() milestones = frappe.get_all('Milestone', @@ -40,3 +45,6 @@ class TestMilestoneTracker(unittest.TestCase): self.assertEqual(milestones[0].track_field, 'status') self.assertEqual(milestones[0].value, 'Closed') + # cleanup + frappe.db.sql('delete from tabMilestone') + milestone_tracker.delete() \ No newline at end of file diff --git a/frappe/boot.py b/frappe/boot.py index b552d7d703..5b1a1bf573 100644 --- a/frappe/boot.py +++ b/frappe/boot.py @@ -21,6 +21,7 @@ from frappe.website.doctype.web_page_view.web_page_view import is_tracking_enabl from frappe.social.doctype.energy_point_log.energy_point_log import get_energy_points from frappe.model.base_document import get_controller from frappe.social.doctype.post.post import frequently_visited_links +from frappe.core.doctype.navbar_settings.navbar_settings import get_navbar_settings def get_bootinfo(): """build and return boot info""" @@ -59,6 +60,7 @@ def get_bootinfo(): load_print(bootinfo, doclist) doclist.extend(get_meta_bundle("Page")) bootinfo.home_folder = frappe.db.get_value("File", {"is_home_folder": 1}) + bootinfo.navbar_settings = get_navbar_settings() # ipinfo if frappe.session.data.get('ipinfo'): diff --git a/frappe/cache_manager.py b/frappe/cache_manager.py index 97b6c235b5..dabc78a9f6 100644 --- a/frappe/cache_manager.py +++ b/frappe/cache_manager.py @@ -11,12 +11,15 @@ from frappe.desk.notifications import (delete_notification_count_for, common_default_keys = ["__default", "__global"] +doctype_map_keys = ('energy_point_rule_map', 'assignment_rule_map', + 'milestone_tracker_map', 'event_consumer_document_type_map') + global_cache_keys = ("app_hooks", "installed_apps", "app_modules", "module_app", "system_settings", 'scheduler_events', 'time_zone', 'webhooks', 'active_domains', 'active_modules', 'assignment_rule', 'server_script_map', 'wkhtmltopdf_version', 'domain_restricted_doctypes', 'domain_restricted_pages', 'information_schema:counts', - 'sitemap_routes', 'db_tables') + 'sitemap_routes', 'db_tables') + doctype_map_keys user_cache_keys = ("bootinfo", "user_recent", "roles", "user_doc", "lang", "defaults", "user_permissions", "home_page", "linked_with", @@ -24,8 +27,8 @@ user_cache_keys = ("bootinfo", "user_recent", "roles", "user_doc", "lang", "has_role:Page", "has_role:Report", "desk_sidebar_items") doctype_cache_keys = ("meta", "form_meta", "table_columns", "last_modified", - "linked_doctypes", 'notifications', 'workflow' ,'energy_point_rule_map', 'data_import_column_header_map') - + "linked_doctypes", 'notifications', 'workflow' , + 'data_import_column_header_map') + doctype_map_keys def clear_user_cache(user=None): cache = frappe.cache() @@ -102,19 +105,19 @@ def clear_doctype_cache(doctype=None): # Clear all document's cache. To clear documents of a specific DocType document_cache should be restructured clear_document_cache() -def get_doctype_map(doctype, name, filters, order_by=None): +def get_doctype_map(doctype, name, filters=None, order_by=None): cache = frappe.cache() cache_key = frappe.scrub(doctype) + '_map' doctype_map = cache.hget(cache_key, name) - if doctype_map: + if doctype_map is not None: # cached, return items = json.loads(doctype_map) else: # non cached, build cache try: items = frappe.get_all(doctype, filters=filters, order_by = order_by) - cache.hset(cache_key, doctype, json.dumps(items)) + cache.hset(cache_key, name, json.dumps(items)) except frappe.db.TableMissingError: # executed from inside patch, ignore items = [] @@ -122,8 +125,7 @@ def get_doctype_map(doctype, name, filters, order_by=None): return items def clear_doctype_map(doctype, name): - cache_key = frappe.scrub(doctype) + '_map' - frappe.cache().hdel(cache_key, name) + frappe.cache().hdel(frappe.scrub(doctype) + '_map', name) def build_table_count_cache(): if (frappe.flags.in_patch diff --git a/frappe/commands/site.py b/frappe/commands/site.py index 26eb455338..d343d10126 100755 --- a/frappe/commands/site.py +++ b/frappe/commands/site.py @@ -274,8 +274,9 @@ def disable_user(context, email): @click.command('migrate') @click.option('--rebuild-website', help="Rebuild webpages after migration") @click.option('--skip-failing', is_flag=True, help="Skip patches that fail to run") +@click.option('--skip-search-index', is_flag=True, help="Skip search indexing for web documents") @pass_context -def migrate(context, rebuild_website=False, skip_failing=False): +def migrate(context, rebuild_website=False, skip_failing=False, skip_search_index=False): "Run patches, sync schema and rebuild files/translations" from frappe.migrate import migrate @@ -284,13 +285,18 @@ def migrate(context, rebuild_website=False, skip_failing=False): frappe.init(site=site) frappe.connect() try: - migrate(context.verbose, rebuild_website=rebuild_website, skip_failing=skip_failing) + migrate( + context.verbose, + rebuild_website=rebuild_website, + skip_failing=skip_failing, + skip_search_index=skip_search_index + ) finally: frappe.destroy() if not context.sites: raise SiteNotSpecifiedError - print("Compiling Python Files...") + print("Compiling Python files...") compileall.compile_dir('../apps', quiet=1, rx=re.compile('.*node_modules.*')) @click.command('migrate-to') @@ -655,6 +661,22 @@ def start_ngrok(context): frappe.destroy() ngrok.kill() +@click.command('build-search-index') +@pass_context +def build_search_index(context): + from frappe.search.website_search import build_index_for_all_routes + site = get_site(context) + if not site: + raise SiteNotSpecifiedError + + print('Building search index for {}'.format(site)) + frappe.init(site=site) + frappe.connect() + try: + build_index_for_all_routes() + finally: + frappe.destroy() + commands = [ add_system_manager, backup, @@ -680,5 +702,6 @@ commands = [ start_recording, stop_recording, add_to_hosts, - start_ngrok + start_ngrok, + build_search_index ] diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py index 28b6344b8e..721376016c 100644 --- a/frappe/commands/utils.py +++ b/frappe/commands/utils.py @@ -528,7 +528,6 @@ def run_tests(context, app=None, module=None, doctype=None, test=(), @pass_context def run_ui_tests(context, app, headless=False): "Run UI tests" - site = get_site(context) app_base_path = os.path.abspath(os.path.join(frappe.get_app_path(app), '..')) site_url = frappe.utils.get_site_url(site) diff --git a/frappe/contacts/address_and_contact.py b/frappe/contacts/address_and_contact.py index 51f13fb1a1..3ca9547188 100644 --- a/frappe/contacts/address_and_contact.py +++ b/frappe/contacts/address_and_contact.py @@ -147,6 +147,7 @@ def delete_contact_and_address(doctype, docname): doc.delete() @frappe.whitelist() +@frappe.validate_and_sanitize_search_inputs def filter_dynamic_link_doctypes(doctype, txt, searchfield, start, page_len, filters): if not txt: txt = "" diff --git a/frappe/contacts/doctype/address/address.py b/frappe/contacts/doctype/address/address.py index 57dea8284c..e82ab9b26e 100644 --- a/frappe/contacts/doctype/address/address.py +++ b/frappe/contacts/doctype/address/address.py @@ -231,6 +231,7 @@ def get_company_address(company): return ret @frappe.whitelist() +@frappe.validate_and_sanitize_search_inputs def address_query(doctype, txt, searchfield, start, page_len, filters): from frappe.desk.reportview import get_match_cond diff --git a/frappe/contacts/doctype/contact/contact.py b/frappe/contacts/doctype/contact/contact.py index 8240940d2f..f82946dc5e 100644 --- a/frappe/contacts/doctype/contact/contact.py +++ b/frappe/contacts/doctype/contact/contact.py @@ -183,6 +183,7 @@ def update_contact(doc, method): contact.save(ignore_permissions=True) @frappe.whitelist() +@frappe.validate_and_sanitize_search_inputs def contact_query(doctype, txt, searchfield, start, page_len, filters): from frappe.desk.reportview import get_match_cond diff --git a/frappe/core/doctype/data_import/data_import.js b/frappe/core/doctype/data_import/data_import.js index 6a922618cb..0e827a42d8 100644 --- a/frappe/core/doctype/data_import/data_import.js +++ b/frappe/core/doctype/data_import/data_import.js @@ -317,6 +317,7 @@ frappe.ui.form.on('Data Import', { }, show_import_warnings(frm, preview_data) { + let columns = preview_data.columns; let warnings = JSON.parse(frm.doc.template_warnings || '[]'); warnings = warnings.concat(preview_data.warnings || []); @@ -367,11 +368,13 @@ frappe.ui.form.on('Data Import', { .map(warning => { let header = ''; if (warning.col) { - header = __('Column {0}', [warning.col]); + let column_number = `${__('Column {0}', [warning.col])}`; + let column_header = columns[warning.col].header_title; + header = `${column_number} (${column_header})`; } return `
-
${header}
+
${header}
${warning.message}
`; diff --git a/frappe/core/doctype/data_import/data_import_list.js b/frappe/core/doctype/data_import/data_import_list.js index 1dee4319f9..0eb05aa354 100644 --- a/frappe/core/doctype/data_import/data_import_list.js +++ b/frappe/core/doctype/data_import/data_import_list.js @@ -17,6 +17,7 @@ frappe.listview_settings['Data Import'] = { get_indicator: function(doc) { var colors = { 'Pending': 'orange', + 'Not Started': 'orange', 'Partial Success': 'orange', 'Success': 'green', 'In Progress': 'orange', @@ -26,6 +27,9 @@ frappe.listview_settings['Data Import'] = { if (imports_in_progress.includes(doc.name)) { status = 'In Progress'; } + if (status == 'Pending') { + status = 'Not Started'; + } return [__(status), colors[status], 'status,=,' + doc.status]; }, formatters: { diff --git a/frappe/core/doctype/data_import/importer.py b/frappe/core/doctype/data_import/importer.py index 910e42af1a..485f7caf08 100644 --- a/frappe/core/doctype/data_import/importer.py +++ b/frappe/core/doctype/data_import/importer.py @@ -7,7 +7,7 @@ import io import frappe import timeit import json -from datetime import datetime +from datetime import datetime, date from frappe import _ from frappe.utils import cint, flt, update_progress_bar, cstr from frappe.utils.csvutils import read_csv_content, get_csv_content_from_google_sheets @@ -233,7 +233,7 @@ class Importer: return updated_doc else: # throw if no changes - frappe.throw('No changes to update') + frappe.throw("No changes to update") def get_eta(self, current, total, processing_time): self.last_eta = getattr(self, "last_eta", 0) @@ -322,7 +322,7 @@ class ImportFile: if isinstance(file, frappe.string_types): if frappe.db.exists("File", {"file_url": file}): self.file_doc = frappe.get_doc("File", {"file_url": file}) - elif 'docs.google.com/spreadsheets' in file: + elif "docs.google.com/spreadsheets" in file: self.google_sheets_url = file elif os.path.exists(file): self.file_path = file @@ -348,7 +348,7 @@ class ImportFile: elif self.google_sheets_url: content = get_csv_content_from_google_sheets(self.google_sheets_url) - extension = 'csv' + extension = "csv" if not content: frappe.throw(_("Invalid or corrupted content for import")) @@ -602,12 +602,20 @@ class Row: is_table = frappe.get_meta(doctype).istable is_update = self.import_type == UPDATE - if is_table and is_update and doc.get("name") in INVALID_VALUES: - # for table rows being inserted in update - # create a new doc with defaults set - new_doc = frappe.new_doc(doctype, as_dict=True) - new_doc.update(doc) - doc = new_doc + if is_table and is_update: + # check if the row already exists + # if yes, fetch the original doc so that it is not updated + # if no, create a new doc + id_field = get_id_field(doctype) + id_value = doc.get(id_field.fieldname) + if id_value and frappe.db.exists(doctype, id_value): + doc = frappe.get_doc(doctype, id_value) + else: + # for table rows being inserted in update + # create a new doc with defaults set + new_doc = frappe.new_doc(doctype, as_dict=True) + new_doc.update(doc) + doc = new_doc self.check_mandatory_fields(doctype, doc, table_df) return doc @@ -615,16 +623,12 @@ class Row: def validate_value(self, value, col): df = col.df if df.fieldtype == "Select": - select_options = [d for d in (df.options or '').split('\n') if d] + select_options = get_select_options(df) if select_options and value not in select_options: options_string = ", ".join([frappe.bold(d) for d in select_options]) msg = _("Value must be one of {0}").format(options_string) self.warnings.append( - { - "row": self.row_number, - "field": df_as_json(df), - "message": msg, - } + {"row": self.row_number, "field": df_as_json(df), "message": msg,} ) return @@ -635,11 +639,7 @@ class Row: frappe.bold(value), frappe.bold(df.options) ) self.warnings.append( - { - "row": self.row_number, - "field": df_as_json(df), - "message": msg, - } + {"row": self.row_number, "field": df_as_json(df), "message": msg,} ) return elif df.fieldtype in ["Date", "Datetime"]: @@ -668,7 +668,7 @@ class Row: def parse_value(self, value, col): df = col.df - if isinstance(value, datetime) and df.fieldtype in ["Date", "Datetime"]: + if isinstance(value, (datetime, date)) and df.fieldtype in ["Date", "Datetime"]: return value value = cstr(value) @@ -689,7 +689,7 @@ class Row: return value def get_date(self, value, column): - if isinstance(value, datetime): + if isinstance(value, (datetime, date)): return value date_format = column.date_format @@ -786,9 +786,7 @@ class Header(Row): for j, header in enumerate(row): column_values = [get_item_at_index(r, j) for r in raw_data] map_to_field = column_to_field_map.get(str(j)) - column = Column( - j, header, self.doctype, column_values, map_to_field, self.seen - ) + column = Column(j, header, self.doctype, column_values, map_to_field, self.seen) self.seen.append(header) self.columns.append(column) @@ -918,13 +916,20 @@ class Column: self.skip_import = skip_import def guess_date_format_for_column(self): - """ Guesses date format for a column by parsing all the values in the column, + """Guesses date format for a column by parsing all the values in the column, getting the date format and then returning the one which has the maximum frequency """ - date_formats = [ - frappe.utils.guess_date_format(d) for d in self.column_values if isinstance(d, str) - ] + def guess_date_format(d): + if isinstance(d, (datetime, date)): + if self.df.fieldtype == "Date": + return "%Y-%m-%d" + if self.df.fieldtype == "Datetime": + return "%Y-%m-%d %H:%M:%S" + if isinstance(d, str): + return frappe.utils.guess_date_format(d) + + date_formats = [guess_date_format(d) for d in self.column_values] date_formats = [d for d in date_formats if d] if not date_formats: return @@ -955,28 +960,61 @@ class Column: if not self.df: return - if self.df.fieldtype == 'Link': + if self.skip_import: + return + + if self.df.fieldtype == "Link": # find all values that dont exist values = list(set([cstr(v) for v in self.column_values[1:] if v])) - exists = [d.name for d in frappe.db.get_all(self.df.options, filters={'name': ('in', values)})] + exists = [ + d.name for d in frappe.db.get_all(self.df.options, filters={"name": ("in", values)}) + ] not_exists = list(set(values) - set(exists)) if not_exists: - missing_values = ', '.join(not_exists) - self.warnings.append({ - 'col': self.column_number, - 'message': "The following values do not exist for {}: {}".format(self.df.options, missing_values), - 'type': 'warning' - }) + missing_values = ", ".join(not_exists) + self.warnings.append( + { + "col": self.column_number, + "message": ( + "The following values do not exist for {}: {}".format( + self.df.options, missing_values + ) + ), + "type": "warning", + } + ) elif self.df.fieldtype in ("Date", "Time", "Datetime"): # guess date format self.date_format = self.guess_date_format_for_column() if not self.date_format: - self.date_format = '%Y-%m-%d' - self.warnings.append({ - 'col': self.column_number, - 'message': _("Date format could not determined from the values in this column. Defaulting to yyyy-mm-dd."), - 'type': 'info' - }) + self.date_format = "%Y-%m-%d" + self.warnings.append( + { + "col": self.column_number, + "message": _( + "Date format could not be determined from the values in" + " this column. Defaulting to yyyy-mm-dd." + ), + "type": "info", + } + ) + elif self.df.fieldtype == "Select": + options = get_select_options(self.df) + if options: + values = list(set([cstr(v) for v in self.column_values[1:] if v])) + invalid = list(set(values) - set(options)) + if invalid: + valid_values = ", ".join([frappe.bold(o) for o in options]) + invalid_values = ", ".join([frappe.bold(i) for i in invalid]) + self.warnings.append( + { + "col": self.column_number, + "message": ( + "The following values are invalid: {0}. Values must be" + " one of {1}".format(invalid_values, valid_values) + ), + } + ) def as_dict(self): d = frappe._dict() @@ -987,7 +1025,7 @@ class Column: d.map_to_field = self.map_to_field d.date_format = self.date_format d.df = self.df - if hasattr(self.df, 'is_child_table_field'): + if hasattr(self.df, "is_child_table_field"): d.is_child_table_field = self.df.is_child_table_field d.child_table_df = self.df.child_table_df d.skip_import = self.skip_import @@ -1067,7 +1105,7 @@ def build_fields_dict_for_column_matching(parent_doctype): # other fields fields = get_standard_fields(doctype) + frappe.get_meta(doctype).fields for df in fields: - label = (df.label or '').strip() + label = (df.label or "").strip() fieldtype = df.fieldtype or "Data" parent = df.parent or parent_doctype if fieldtype not in no_value_fields: @@ -1161,12 +1199,17 @@ def get_user_format(date_format): .replace("%d", "dd") ) + def df_as_json(df): return { - 'fieldname': df.fieldname, - 'fieldtype': df.fieldtype, - 'label': df.label, - 'options': df.options, - 'parent': df.parent, - 'default': df.default + "fieldname": df.fieldname, + "fieldtype": df.fieldtype, + "label": df.label, + "options": df.options, + "parent": df.parent, + "default": df.default, } + + +def get_select_options(df): + return [d for d in (df.options or "").split("\n") if d] diff --git a/frappe/core/doctype/deleted_document/deleted_document.py b/frappe/core/doctype/deleted_document/deleted_document.py index bc2962ab3f..116fc5caf5 100644 --- a/frappe/core/doctype/deleted_document/deleted_document.py +++ b/frappe/core/doctype/deleted_document/deleted_document.py @@ -3,17 +3,26 @@ # For license information, please see license.txt from __future__ import unicode_literals -import frappe, json +import frappe +import json +from frappe.desk.doctype.bulk_update.bulk_update import show_progress from frappe.model.document import Document from frappe import _ + class DeletedDocument(Document): pass + @frappe.whitelist() -def restore(name): +def restore(name, alert=True): deleted = frappe.get_doc('Deleted Document', name) + + if deleted.restored: + frappe.throw(_("Document {0} Already Restored").format(name), exc=frappe.DocumentAlreadyRestored) + doc = frappe.get_doc(json.loads(deleted.data)) + try: doc.insert() except frappe.DocstatusTransitionError: @@ -27,4 +36,34 @@ def restore(name): deleted.restored = 1 deleted.db_update() - frappe.msgprint(_('Document Restored')) + if alert: + frappe.msgprint(_('Document Restored')) + + +@frappe.whitelist() +def bulk_restore(docnames): + docnames = frappe.parse_json(docnames) + message = _('Restoring Deleted Document') + restored, invalid, failed = [], [], [] + + for i, d in enumerate(docnames): + try: + show_progress(docnames, message, i + 1, d) + restore(d, alert=False) + frappe.db.commit() + restored.append(d) + + except frappe.DocumentAlreadyRestored: + frappe.message_log.pop() + invalid.append(d) + + except Exception: + frappe.message_log.pop() + failed.append(d) + frappe.db.rollback() + + return { + "restored": restored, + "invalid": invalid, + "failed": failed + } diff --git a/frappe/core/doctype/deleted_document/deleted_document_list.js b/frappe/core/doctype/deleted_document/deleted_document_list.js new file mode 100644 index 0000000000..f5e1147dfb --- /dev/null +++ b/frappe/core/doctype/deleted_document/deleted_document_list.js @@ -0,0 +1,40 @@ +frappe.listview_settings["Deleted Document"] = { + onload: function (doclist) { + const action = () => { + const selected_docs = doclist.get_checked_items(); + if (selected_docs.length > 0) { + let docnames = selected_docs.map(doc => doc.name); + frappe.call({ + method: "frappe.core.doctype.deleted_document.deleted_document.bulk_restore", + args: { docnames }, + callback: function (r) { + if (r.message) { + function body(docnames) { + const html = docnames.map(docname => { + return `
  • ${docname}
  • `; + }); + return "
    ": ""; + } + + const { restored, invalid, failed } = r.message; + const restored_summary = message(__("Documents restored successfully"), restored); + const invalid_summary = message(__("Documents that were already restored"), invalid); + const failed_summary = message(__("Documents that failed to restore"), failed); + const summary = restored_summary + invalid_summary + failed_summary; + + frappe.msgprint(summary, __("Document Restoration Summary"), true); + + if (restored.length > 0) { + doclist.refresh(); + } + } + }, + }); + } + }; + doclist.page.add_actions_menu_item(__("Restore"), action, false); + }, +}; diff --git a/frappe/core/doctype/doctype/doctype.json b/frappe/core/doctype/doctype/doctype.json index 379ea227cb..b890f743cc 100644 --- a/frappe/core/doctype/doctype/doctype.json +++ b/frappe/core/doctype/doctype/doctype.json @@ -70,6 +70,7 @@ "web_view", "has_web_view", "allow_guest_to_view", + "index_web_pages_for_search", "route", "is_published_field", "advanced", @@ -472,6 +473,8 @@ "label": "Documentation Link" }, { + "collapsible": 1, + "collapsible_depends_on": "actions", "fieldname": "actions_section", "fieldtype": "Section Break", "label": "Actions" @@ -483,6 +486,8 @@ "options": "DocType Action" }, { + "collapsible": 1, + "collapsible_depends_on": "links", "fieldname": "links_section", "fieldtype": "Section Break", "label": "Links Section" @@ -517,12 +522,94 @@ "fieldname": "email_settings_sb", "fieldtype": "Section Break", "label": "Email Settings" + }, + { + "default": "1", + "fieldname": "index_web_pages_for_search", + "fieldtype": "Check", + "label": "Index Web Pages for Search" } ], "icon": "fa fa-bolt", "idx": 6, - "links": [], - "modified": "2020-03-27 14:51:44.581128", + "links": [ + { + "group": "Views", + "link_doctype": "Report", + "link_fieldname": "ref_doctype" + }, + { + "group": "Workflow", + "link_doctype": "Workflow", + "link_fieldname": "document_type" + }, + { + "group": "Workflow", + "link_doctype": "Notification", + "link_fieldname": "document_type" + }, + { + "group": "Customization", + "link_doctype": "Custom Field", + "link_fieldname": "dt" + }, + { + "group": "Customization", + "link_doctype": "Custom Script", + "link_fieldname": "dt" + }, + { + "group": "Customization", + "link_doctype": "Server Script", + "link_fieldname": "reference_doctype" + }, + { + "group": "Workflow", + "link_doctype": "Webhook", + "link_fieldname": "webhook_doctype" + }, + { + "group": "Views", + "link_doctype": "Print Format", + "link_fieldname": "doc_type" + }, + { + "group": "Views", + "link_doctype": "Web Form", + "link_fieldname": "doc_type" + }, + { + "group": "Views", + "link_doctype": "Calendar View", + "link_fieldname": "reference_doctype" + }, + { + "group": "Views", + "link_doctype": "Kanban Board", + "link_fieldname": "reference_doctype" + }, + { + "group": "Workflow", + "link_doctype": "Onboarding Step", + "link_fieldname": "reference_document" + }, + { + "group": "Rules", + "link_doctype": "Auto Repeat", + "link_fieldname": "reference_doctype" + }, + { + "group": "Rules", + "link_doctype": "Assignment Rule", + "link_fieldname": "document_type" + }, + { + "group": "Rules", + "link_doctype": "Energy Point Rule", + "link_fieldname": "reference_doctype" + } + ], + "modified": "2020-08-06 12:59:32.369093", "modified_by": "Administrator", "module": "Core", "name": "DocType", diff --git a/frappe/core/doctype/doctype/test_doctype.py b/frappe/core/doctype/doctype/test_doctype.py index fe9f88b9b3..00e80ce4e7 100644 --- a/frappe/core/doctype/doctype/test_doctype.py +++ b/frappe/core/doctype/doctype/test_doctype.py @@ -376,3 +376,96 @@ class TestDocType(unittest.TestCase): link_doc.delete() doc.delete() frappe.db.commit() + + def test_ignore_cancelation_of_linked_doctype_during_cancell(self): + import json + from frappe.desk.form.linked_with import get_submitted_linked_docs, cancel_all_linked_docs + + #create linked doctype + link_doc = self.new_doctype('Test Linked Doctype 1') + link_doc.is_submittable = 1 + for data in link_doc.get('permissions'): + data.submit = 1 + data.cancel = 1 + link_doc.insert() + + #create first parent doctype + test_doc_1 = self.new_doctype('Test Doctype 1') + test_doc_1.is_submittable = 1 + + field_2 = test_doc_1.append('fields', {}) + field_2.label = 'Test Linked Doctype 1' + field_2.fieldname = 'test_linked_doctype_a' + field_2.fieldtype = 'Link' + field_2.options = 'Test Linked Doctype 1' + + for data in test_doc_1.get('permissions'): + data.submit = 1 + data.cancel = 1 + test_doc_1.insert() + + #crete second parent doctype + doc = self.new_doctype('Test Doctype 2') + doc.is_submittable = 1 + + field_2 = doc.append('fields', {}) + field_2.label = 'Test Linked Doctype 1' + field_2.fieldname = 'test_linked_doctype_a' + field_2.fieldtype = 'Link' + field_2.options = 'Test Linked Doctype 1' + + for data in link_doc.get('permissions'): + data.submit = 1 + data.cancel = 1 + doc.insert() + + # create doctype data + data_link_doc_1 = frappe.new_doc('Test Linked Doctype 1') + data_link_doc_1.some_fieldname = 'Data1' + data_link_doc_1.insert() + data_link_doc_1.save() + data_link_doc_1.submit() + + data_doc_2 = frappe.new_doc('Test Doctype 1') + data_doc_2.some_fieldname = 'Data1' + data_doc_2.test_linked_doctype_a = data_link_doc_1.name + data_doc_2.insert() + data_doc_2.save() + data_doc_2.submit() + + data_doc = frappe.new_doc('Test Doctype 2') + data_doc.some_fieldname = 'Data1' + data_doc.test_linked_doctype_a = data_link_doc_1.name + data_doc.insert() + data_doc.save() + data_doc.submit() + + docs = get_submitted_linked_docs(link_doc.name, data_link_doc_1.name) + dump_docs = json.dumps(docs.get('docs')) + + cancel_all_linked_docs(dump_docs, ignore_doctypes_on_cancel_all=["Test Doctype 2"]) + + # checking that doc for Test Doctype 2 is not canceled + self.assertRaises(frappe.LinkExistsError, data_link_doc_1.cancel) + + data_doc.load_from_db() + data_doc_2.load_from_db() + self.assertEqual(data_link_doc_1.docstatus, 2) + + #linked doc is canceled + self.assertEqual(data_doc_2.docstatus, 2) + + #ignored doctype 2 during cancel + self.assertEqual(data_doc.docstatus, 1) + + # delete doctype record + data_doc.cancel() + data_doc.delete() + data_doc_2.delete() + data_link_doc_1.delete() + + # delete doctype + link_doc.delete() + doc.delete() + test_doc_1.delete() + frappe.db.commit() diff --git a/frappe/core/doctype/module_def/module_def.js b/frappe/core/doctype/module_def/module_def.js index 5bb8db6d5c..c7a6cf85f9 100644 --- a/frappe/core/doctype/module_def/module_def.js +++ b/frappe/core/doctype/module_def/module_def.js @@ -3,6 +3,8 @@ frappe.ui.form.on('Module Def', { refresh: function(frm) { - + frappe.xcall('frappe.core.doctype.module_def.module_def.get_installed_apps').then(r => { + frm.set_df_property('app_name', 'options', JSON.parse(r)); + }); } }); diff --git a/frappe/core/doctype/module_def/module_def.json b/frappe/core/doctype/module_def/module_def.json index 4ff7a40877..b841eb1f0b 100644 --- a/frappe/core/doctype/module_def/module_def.json +++ b/frappe/core/doctype/module_def/module_def.json @@ -1,172 +1,88 @@ { - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 1, - "autoname": "field:module_name", - "beta": 0, - "creation": "2013-01-10 16:34:03", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "editable_grid": 0, - "engine": "InnoDB", + "actions": [], + "allow_rename": 1, + "autoname": "field:module_name", + "creation": "2013-01-10 16:34:03", + "doctype": "DocType", + "engine": "InnoDB", + "field_order": [ + "module_name", + "custom", + "app_name", + "restrict_to_domain" + ], "fields": [ { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "module_name", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Module Name", - "length": 0, - "no_copy": 0, - "oldfieldname": "module_name", - "oldfieldtype": "Data", - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "module_name", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Module Name", + "oldfieldname": "module_name", + "oldfieldtype": "Data", + "reqd": 1, + "unique": 1 + }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "app_name", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 1, - "label": "App Name", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "app_name", + "fieldtype": "Select", + "in_list_view": 1, + "in_standard_filter": 1, + "label": "App Name", + "reqd": 1 + }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "restrict_to_domain", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Restrict To Domain", - "length": 0, - "no_copy": 0, - "options": "Domain", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "fieldname": "restrict_to_domain", + "fieldtype": "Link", + "label": "Restrict To Domain", + "options": "Domain" + }, + { + "default": "0", + "fieldname": "custom", + "fieldtype": "Check", + "label": "Custom" } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "icon": "fa fa-sitemap", - "idx": 1, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2017-07-13 03:05:28.213656", - "modified_by": "Administrator", - "module": "Core", - "name": "Module Def", - "owner": "Administrator", + ], + "icon": "fa fa-sitemap", + "idx": 1, + "links": [ + { + "link_doctype": "DocType", + "link_fieldname": "module" + }, + { + "link_doctype": "Desk Page", + "link_fieldname": "module" + } + ], + "modified": "2020-08-06 12:39:30.740379", + "modified_by": "Administrator", + "module": "Core", + "name": "Module Def", + "owner": "Administrator", "permissions": [ { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 1, - "delete": 1, - "email": 1, - "export": 0, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 1, - "role": "Administrator", - "set_user_permissions": 0, - "share": 1, - "submit": 0, + "create": 1, + "delete": 1, + "email": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "Administrator", + "share": 1, "write": 1 - }, + }, { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 0, - "delete": 0, - "email": 0, - "export": 0, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 0, - "read": 1, - "report": 0, - "role": "System Manager", - "set_user_permissions": 0, - "share": 0, - "submit": 0, - "write": 0 + "create": 1, + "delete": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "write": 1 } - ], - "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 1, - "sort_order": "ASC", - "track_changes": 1, - "track_seen": 0 + ], + "show_name_in_global_search": 1, + "sort_field": "modified", + "sort_order": "ASC", + "track_changes": 1 } \ No newline at end of file diff --git a/frappe/core/doctype/module_def/module_def.py b/frappe/core/doctype/module_def/module_def.py index 755cb86dbe..930c46e60b 100644 --- a/frappe/core/doctype/module_def/module_def.py +++ b/frappe/core/doctype/module_def/module_def.py @@ -2,7 +2,7 @@ # MIT License. See license.txt from __future__ import unicode_literals -import frappe, os +import frappe, os, json from frappe.model.document import Document @@ -11,7 +11,7 @@ class ModuleDef(Document): """If in `developer_mode`, create folder for module and add in `modules.txt` of app if missing.""" frappe.clear_cache() - if frappe.conf.get("developer_mode"): + if not self.custom and frappe.conf.get("developer_mode"): self.create_modules_folder() self.add_to_modules_txt() @@ -43,7 +43,7 @@ class ModuleDef(Document): def on_trash(self): """Delete module name from modules.txt""" - if frappe.flags.in_uninstall: + if frappe.flags.in_uninstall or self.custom: return modules = None @@ -60,3 +60,7 @@ class ModuleDef(Document): frappe.clear_cache() frappe.setup_module_map() + +@frappe.whitelist() +def get_installed_apps(): + return json.dumps(frappe.get_installed_apps()) \ No newline at end of file diff --git a/frappe/core/doctype/video/__init__.py b/frappe/core/doctype/navbar_item/__init__.py similarity index 100% rename from frappe/core/doctype/video/__init__.py rename to frappe/core/doctype/navbar_item/__init__.py diff --git a/frappe/core/doctype/video/video.js b/frappe/core/doctype/navbar_item/navbar_item.js similarity index 81% rename from frappe/core/doctype/video/video.js rename to frappe/core/doctype/navbar_item/navbar_item.js index 36ea240a36..bd4274db49 100644 --- a/frappe/core/doctype/video/video.js +++ b/frappe/core/doctype/navbar_item/navbar_item.js @@ -1,7 +1,7 @@ // Copyright (c) 2020, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Video', { +frappe.ui.form.on('Navbar Item', { // refresh: function(frm) { // } diff --git a/frappe/core/doctype/navbar_item/navbar_item.json b/frappe/core/doctype/navbar_item/navbar_item.json new file mode 100644 index 0000000000..3bfea52558 --- /dev/null +++ b/frappe/core/doctype/navbar_item/navbar_item.json @@ -0,0 +1,87 @@ +{ + "actions": [], + "creation": "2020-08-01 23:38:41.783206", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "item_label", + "item_type", + "route", + "action", + "hidden", + "is_standard" + ], + "fields": [ + { + "columns": 2, + "fieldname": "item_label", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Item Label", + "mandatory_depends_on": "eval:doc.item_type == 'Route' || doc.item_type == 'Action'", + "show_days": 1, + "show_seconds": 1 + }, + { + "columns": 2, + "fieldname": "item_type", + "fieldtype": "Select", + "in_list_view": 1, + "label": "Item Type", + "options": "Route\nAction\nSeparator", + "show_days": 1, + "show_seconds": 1 + }, + { + "default": "0", + "fieldname": "hidden", + "fieldtype": "Check", + "in_list_view": 1, + "label": "Hidden", + "show_days": 1, + "show_seconds": 1 + }, + { + "default": "0", + "fieldname": "is_standard", + "fieldtype": "Check", + "label": "Is Standard", + "read_only": 1, + "show_days": 1, + "show_seconds": 1 + }, + { + "columns": 4, + "depends_on": "eval:doc.item_type == 'Route'", + "fieldname": "route", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Route", + "mandatory_depends_on": "eval:doc.item_type == 'Route'", + "show_days": 1, + "show_seconds": 1 + }, + { + "depends_on": "eval:doc.item_type == 'Action'", + "fieldname": "action", + "fieldtype": "Data", + "label": "Action", + "mandatory_depends_on": "eval:doc.item_type == 'Action'", + "show_days": 1, + "show_seconds": 1 + } + ], + "istable": 1, + "links": [], + "modified": "2020-08-06 16:32:49.597060", + "modified_by": "Administrator", + "module": "Core", + "name": "Navbar Item", + "owner": "Administrator", + "permissions": [], + "quick_entry": 1, + "sort_field": "modified", + "sort_order": "DESC", + "track_changes": 1 +} \ No newline at end of file diff --git a/frappe/core/doctype/video/video.py b/frappe/core/doctype/navbar_item/navbar_item.py similarity index 89% rename from frappe/core/doctype/video/video.py rename to frappe/core/doctype/navbar_item/navbar_item.py index fdbd3a1abe..614aee8eaf 100644 --- a/frappe/core/doctype/video/video.py +++ b/frappe/core/doctype/navbar_item/navbar_item.py @@ -6,5 +6,5 @@ from __future__ import unicode_literals # import frappe from frappe.model.document import Document -class Video(Document): +class NavbarItem(Document): pass diff --git a/frappe/core/doctype/video/test_video.py b/frappe/core/doctype/navbar_item/test_navbar_item.py similarity index 81% rename from frappe/core/doctype/video/test_video.py rename to frappe/core/doctype/navbar_item/test_navbar_item.py index 0bed1e98d6..192e8fe42a 100644 --- a/frappe/core/doctype/video/test_video.py +++ b/frappe/core/doctype/navbar_item/test_navbar_item.py @@ -6,5 +6,5 @@ from __future__ import unicode_literals # import frappe import unittest -class TestVideo(unittest.TestCase): +class TestNavbarItem(unittest.TestCase): pass diff --git a/frappe/core/doctype/navbar_settings/__init__.py b/frappe/core/doctype/navbar_settings/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/frappe/core/doctype/navbar_settings/navbar_settings.js b/frappe/core/doctype/navbar_settings/navbar_settings.js new file mode 100644 index 0000000000..e2c157fe6a --- /dev/null +++ b/frappe/core/doctype/navbar_settings/navbar_settings.js @@ -0,0 +1,8 @@ +// Copyright (c) 2020, Frappe Technologies and contributors +// For license information, please see license.txt + +frappe.ui.form.on('Navbar Settings', { + // refresh: function(frm) { + + // } +}); diff --git a/frappe/core/doctype/navbar_settings/navbar_settings.json b/frappe/core/doctype/navbar_settings/navbar_settings.json new file mode 100644 index 0000000000..8fc0c83c82 --- /dev/null +++ b/frappe/core/doctype/navbar_settings/navbar_settings.json @@ -0,0 +1,91 @@ +{ + "actions": [], + "creation": "2020-08-01 23:41:12.577160", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "logo_section", + "app_logo", + "column_break_3", + "logo_width", + "section_break_2", + "settings_dropdown", + "help_dropdown" + ], + "fields": [ + { + "fieldname": "app_logo", + "fieldtype": "Attach Image", + "label": "Application Logo", + "show_days": 1, + "show_seconds": 1 + }, + { + "fieldname": "settings_dropdown", + "fieldtype": "Table", + "label": "Settings Dropdown", + "options": "Navbar Item", + "show_days": 1, + "show_seconds": 1 + }, + { + "fieldname": "help_dropdown", + "fieldtype": "Table", + "label": "Help Dropdown", + "options": "Navbar Item", + "show_days": 1, + "show_seconds": 1 + }, + { + "fieldname": "section_break_2", + "fieldtype": "Section Break", + "label": "Dropdowns", + "show_days": 1, + "show_seconds": 1 + }, + { + "fieldname": "logo_section", + "fieldtype": "Section Break", + "label": "Application Logo", + "show_days": 1, + "show_seconds": 1 + }, + { + "fieldname": "column_break_3", + "fieldtype": "Column Break", + "show_days": 1, + "show_seconds": 1 + }, + { + "fieldname": "logo_width", + "fieldtype": "Int", + "label": "Logo Width", + "show_days": 1, + "show_seconds": 1 + } + ], + "issingle": 1, + "links": [], + "modified": "2020-08-06 18:11:29.955835", + "modified_by": "Administrator", + "module": "Core", + "name": "Navbar Settings", + "owner": "Administrator", + "permissions": [ + { + "create": 1, + "delete": 1, + "email": 1, + "print": 1, + "read": 1, + "role": "System Manager", + "share": 1, + "write": 1 + } + ], + "quick_entry": 1, + "sort_field": "modified", + "sort_order": "DESC", + "track_changes": 1 +} \ No newline at end of file diff --git a/frappe/core/doctype/navbar_settings/navbar_settings.py b/frappe/core/doctype/navbar_settings/navbar_settings.py new file mode 100644 index 0000000000..f7c437bf00 --- /dev/null +++ b/frappe/core/doctype/navbar_settings/navbar_settings.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020, Frappe Technologies and contributors +# For license information, please see license.txt + +from __future__ import unicode_literals +import frappe +from frappe.model.document import Document +from frappe import _ + +class NavbarSettings(Document): + def validate(self): + self.validate_standard_navbar_items() + + def validate_standard_navbar_items(self): + doc_before_save = self.get_doc_before_save() + + before_save_items = [item for item in \ + doc_before_save.help_dropdown + doc_before_save.settings_dropdown if item.is_standard] + + after_save_items = [item for item in \ + self.help_dropdown + self.settings_dropdown if item.is_standard] + + if not frappe.flags.in_patch and (len(before_save_items) > len(after_save_items)): + frappe.throw(_("Please hide the standard navbar items instead of deleting them")) + +@frappe.whitelist() +def get_app_logo(): + app_logo = frappe.db.get_single_value('Navbar Settings', 'app_logo') + if not app_logo: + app_logo = frappe.get_hooks('app_logo_url')[-1] + + return app_logo + +def get_navbar_settings(): + navbar_settings = frappe.get_single('Navbar Settings') + return navbar_settings + + + + diff --git a/frappe/core/doctype/navbar_settings/test_navbar_settings.py b/frappe/core/doctype/navbar_settings/test_navbar_settings.py new file mode 100644 index 0000000000..ed423b0f27 --- /dev/null +++ b/frappe/core/doctype/navbar_settings/test_navbar_settings.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020, Frappe Technologies and Contributors +# See license.txt +from __future__ import unicode_literals + +# import frappe +import unittest + +class TestNavbarSettings(unittest.TestCase): + pass diff --git a/frappe/core/doctype/report/report.js b/frappe/core/doctype/report/report.js index b4c30f5bbd..c410e9aa1a 100644 --- a/frappe/core/doctype/report/report.js +++ b/frappe/core/doctype/report/report.js @@ -1,6 +1,6 @@ frappe.ui.form.on('Report', { refresh: function(frm) { - if (frm.doc.is_standard && !frappe.boot.developer_mode) { + if (frm.doc.is_standard === "Yes" && !frappe.boot.developer_mode) { // make the document read-only frm.set_read_only(); } diff --git a/frappe/core/doctype/role/role.json b/frappe/core/doctype/role/role.json index 1eebb71a36..0ad15ba10b 100644 --- a/frappe/core/doctype/role/role.json +++ b/frappe/core/doctype/role/role.json @@ -1,216 +1,90 @@ { - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 1, - "autoname": "field:role_name", - "beta": 0, - "creation": "2013-01-08 15:50:01", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "Document", - "editable_grid": 0, + "actions": [], + "allow_rename": 1, + "autoname": "field:role_name", + "creation": "2013-01-08 15:50:01", + "doctype": "DocType", + "document_type": "Document", + "engine": "InnoDB", + "field_order": [ + "role_name", + "home_page", + "restrict_to_domain", + "column_break_4", + "disabled", + "desk_access", + "two_factor_auth" + ], "fields": [ { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "role_name", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Role Name", - "length": 0, - "no_copy": 0, - "oldfieldname": "role_name", - "oldfieldtype": "Data", - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "role_name", + "fieldtype": "Data", + "label": "Role Name", + "oldfieldname": "role_name", + "oldfieldtype": "Data", + "reqd": 1, + "unique": 1 + }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "description": "If disabled, this role will be removed from all users.", - "fieldname": "disabled", - "fieldtype": "Check", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Disabled", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "default": "0", + "description": "If disabled, this role will be removed from all users.", + "fieldname": "disabled", + "fieldtype": "Check", + "label": "Disabled" + }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "default": "1", - "fieldname": "desk_access", - "fieldtype": "Check", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Desk Access", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "default": "1", + "fieldname": "desk_access", + "fieldtype": "Check", + "in_list_view": 1, + "label": "Desk Access" + }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "default": "0", - "fieldname": "two_factor_auth", - "fieldtype": "Check", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Two Factor Authentication", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "default": "0", + "fieldname": "two_factor_auth", + "fieldtype": "Check", + "label": "Two Factor Authentication" + }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "restrict_to_domain", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Restrict To Domain", - "length": 0, - "no_copy": 0, - "options": "Domain", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "fieldname": "restrict_to_domain", + "fieldtype": "Link", + "label": "Restrict To Domain", + "options": "Domain" + }, + { + "description": "Route: Example \"/desk\"", + "fieldname": "home_page", + "fieldtype": "Data", + "label": "Home Page" + }, + { + "fieldname": "column_break_4", + "fieldtype": "Column Break" } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "icon": "fa fa-bookmark", - "idx": 1, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2017-07-06 12:42:57.097914", - "modified_by": "Administrator", - "module": "Core", - "name": "Role", - "owner": "Administrator", + ], + "icon": "fa fa-bookmark", + "idx": 1, + "links": [], + "modified": "2020-08-06 15:42:59.036960", + "modified_by": "Administrator", + "module": "Core", + "name": "Role", + "owner": "Administrator", "permissions": [ { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 1, - "delete": 1, - "email": 1, - "export": 0, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "set_user_permissions": 0, - "share": 1, - "submit": 0, + "create": 1, + "delete": 1, + "email": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1, "write": 1 } - ], - "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_order": "ASC", - "track_changes": 1, - "track_seen": 0 + ], + "quick_entry": 1, + "sort_field": "modified", + "sort_order": "ASC", + "track_changes": 1 } \ No newline at end of file diff --git a/frappe/core/doctype/role/role.py b/frappe/core/doctype/role/role.py index 657340ec24..e458b401e4 100644 --- a/frappe/core/doctype/role/role.py +++ b/frappe/core/doctype/role/role.py @@ -33,16 +33,22 @@ class Role(Document): if user_type != user.user_type: user.save() -# Get email addresses of all users that have been assigned this role -def get_emails_from_role(role): - emails = [] - for user in get_users(role): - user_email, enabled = frappe.db.get_value("User", user, ["email", "enabled"]) - if enabled and user_email not in ["admin@example.com", "guest@example.com"]: - emails.append(user_email) +def get_info_based_on_role(role, field='email'): + ''' Get information of all users that have been assigned this role ''' + users = frappe.get_list("Has Role", filters={"role": role, "parenttype": "User"}, + fields=["parent"]) - return emails + return get_user_info(users, field) + +def get_user_info(users, field='email'): + ''' Fetch details about users for the specified field ''' + info_list = [] + for user in users: + user_info, enabled = frappe.db.get_value("User", user.parent, [field, "enabled"]) + if enabled and user_info not in ["admin@example.com", "guest@example.com"]: + info_list.append(user_info) + return info_list def get_users(role): return [d.parent for d in frappe.get_all("Has Role", filters={"role": role, "parenttype": "User"}, diff --git a/frappe/core/doctype/server_script/server_script.js b/frappe/core/doctype/server_script/server_script.js index d7f4c3e536..78ef2d0509 100644 --- a/frappe/core/doctype/server_script/server_script.js +++ b/frappe/core/doctype/server_script/server_script.js @@ -2,8 +2,11 @@ // For license information, please see license.txt frappe.ui.form.on('Server Script', { + setup: function(frm) { + frm.trigger('setup_help'); + }, refresh: function(frm) { - if(frm.doc.script_type === 'Scheduler Event' && !frm.doc.disabled){ + if (frm.doc.script_type === 'Scheduler Event' && !frm.doc.disabled) { frm.add_custom_button('Schedule Script', function() { var d = new frappe.ui.Dialog({ title: "Schedule Script Execution", @@ -33,14 +36,50 @@ frappe.ui.form.on('Server Script', { } }, - schedule_script(frm, data){ + schedule_script(frm, data) { frm.call({ method: "frappe.core.doctype.server_script.server_script.setup_scheduler_events", args: { 'script_name': frm.doc.name, 'frequency': data.event_type } - }) + }); + }, + + setup_help(frm) { + frm.get_field('help_html').html(` +

    Examples

    +

    DocType Event

    +
    
    +# set property
    +if "test" in doc.description:
    +    doc.status = 'Closed'
    +
    +
    +# validate
    +if "validate" in doc.description:
    +    raise frappe.ValidationError
    +
    +# auto create another document
    +if doc.allocted_to:
    +    frappe.get_doc(dict(
    +        doctype = 'ToDo'
    +        owner = doc.allocated_to,
    +        description = doc.subject
    +    )).insert()
    +
    +
    + +

    API Call

    +
    
    +# respond to API
    +
    +if frappe.form_dict.message == "ping":
    +	frappe.response['message'] = "pong"
    +else:
    +	frappe.response['message'] = "ok"
    +
    +`); } }); diff --git a/frappe/core/doctype/server_script/server_script.json b/frappe/core/doctype/server_script/server_script.json index bef3dfc60c..3ed4076430 100644 --- a/frappe/core/doctype/server_script/server_script.json +++ b/frappe/core/doctype/server_script/server_script.json @@ -14,7 +14,9 @@ "api_method", "allow_guest", "section_break_8", - "script" + "script", + "help_section", + "help_html" ], "fields": [ { @@ -72,10 +74,19 @@ { "fieldname": "section_break_8", "fieldtype": "Section Break" + }, + { + "fieldname": "help_section", + "fieldtype": "Section Break", + "label": "Help" + }, + { + "fieldname": "help_html", + "fieldtype": "HTML" } ], "links": [], - "modified": "2020-04-06 11:24:38.161555", + "modified": "2020-08-07 13:13:02.483963", "modified_by": "Administrator", "module": "Core", "name": "Server Script", diff --git a/frappe/core/doctype/server_script/test_server_script.py b/frappe/core/doctype/server_script/test_server_script.py index 3e6b7a3a98..5c12858e8a 100644 --- a/frappe/core/doctype/server_script/test_server_script.py +++ b/frappe/core/doctype/server_script/test_server_script.py @@ -52,9 +52,10 @@ class TestServerScript(unittest.TestCase): frappe.db.commit() - # @classmethod - # def tearDownClass(cls): - # frappe.db.sql('truncate `tabServer Script`') + @classmethod + def tearDownClass(cls): + frappe.db.commit() + frappe.db.sql('truncate `tabServer Script`') def setUp(self): frappe.cache().delete_value('server_script_map') diff --git a/frappe/core/doctype/sms_settings/sms_settings.py b/frappe/core/doctype/sms_settings/sms_settings.py index f6134e045a..ac835108c1 100644 --- a/frappe/core/doctype/sms_settings/sms_settings.py +++ b/frappe/core/doctype/sms_settings/sms_settings.py @@ -18,6 +18,9 @@ class SMSSettings(Document): def validate_receiver_nos(receiver_list): validated_receiver_list = [] for d in receiver_list: + if not d: + break + # remove invalid character for x in [' ','-', '(', ')']: d = d.replace(x, '') diff --git a/frappe/core/doctype/user/user.json b/frappe/core/doctype/user/user.json index 7ed14e094c..9e6781ba64 100644 --- a/frappe/core/doctype/user/user.json +++ b/frappe/core/doctype/user/user.json @@ -588,9 +588,70 @@ "icon": "fa fa-user", "idx": 413, "image_field": "user_image", - "links": [], + "links": [ + { + "group": "Profile", + "link_doctype": "Contact", + "link_fieldname": "user" + }, + { + "group": "Profile", + "link_doctype": "Chat Profile", + "link_fieldname": "user" + }, + { + "group": "Profile", + "link_doctype": "Blogger", + "link_fieldname": "user" + }, + { + "group": "Logs", + "link_doctype": "Access Log", + "link_fieldname": "user" + }, + { + "group": "Logs", + "link_doctype": "Activity Log", + "link_fieldname": "user" + }, + { + "group": "Logs", + "link_doctype": "Energy Point Log", + "link_fieldname": "user" + }, + { + "group": "Logs", + "link_doctype": "Route History", + "link_fieldname": "user" + }, + { + "group": "Settings", + "link_doctype": "User Permission", + "link_fieldname": "user" + }, + { + "group": "Settings", + "link_doctype": "Assignment Rule", + "link_fieldname": "user" + }, + { + "group": "Settings", + "link_doctype": "Document Follow", + "link_fieldname": "user" + }, + { + "group": "Activity", + "link_doctype": "Communication", + "link_fieldname": "user" + }, + { + "group": "Activity", + "link_doctype": "ToDo", + "link_fieldname": "owner" + } + ], "max_attachments": 5, - "modified": "2020-04-08 12:27:36.716490", + "modified": "2020-08-06 19:48:49.677800", "modified_by": "Administrator", "module": "Core", "name": "User", @@ -606,13 +667,13 @@ "read": 1, "report": 1, "role": "System Manager", + "set_user_permissions": 1, "share": 1, "write": 1 }, { "permlevel": 1, "read": 1, - "report": 1, "role": "System Manager", "write": 1 } diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py index 64bff32189..ef2de679ae 100644 --- a/frappe/core/doctype/user/user.py +++ b/frappe/core/doctype/user/user.py @@ -812,6 +812,7 @@ def reset_password(user): return 'not found' @frappe.whitelist() +@frappe.validate_and_sanitize_search_inputs def user_query(doctype, txt, searchfield, start, page_len, filters): from frappe.desk.reportview import get_match_cond diff --git a/frappe/core/doctype/user_permission/user_permission.py b/frappe/core/doctype/user_permission/user_permission.py index 48d4fcb5d4..ba14583c2f 100644 --- a/frappe/core/doctype/user_permission/user_permission.py +++ b/frappe/core/doctype/user_permission/user_permission.py @@ -119,6 +119,8 @@ def user_permission_exists(user, allow, for_value, applicable_for=None): return has_same_user_permission +@frappe.whitelist() +@frappe.validate_and_sanitize_search_inputs def get_applicable_for_doctype_list(doctype, txt, searchfield, start, page_len, filters): linked_doctypes_map = get_linked_doctypes(doctype, True) diff --git a/frappe/core/doctype/video/video.json b/frappe/core/doctype/video/video.json deleted file mode 100644 index 26a407c05c..0000000000 --- a/frappe/core/doctype/video/video.json +++ /dev/null @@ -1,106 +0,0 @@ -{ - "actions": [], - "allow_import": 1, - "allow_rename": 1, - "autoname": "field:title", - "creation": "2018-10-17 05:47:13.087395", - "doctype": "DocType", - "editable_grid": 1, - "engine": "InnoDB", - "field_order": [ - "title", - "provider", - "url", - "column_break_4", - "publish_date", - "duration", - "section_break_7", - "description" - ], - "fields": [ - { - "fieldname": "title", - "fieldtype": "Data", - "in_list_view": 1, - "label": "Title", - "reqd": 1, - "unique": 1 - }, - { - "fieldname": "provider", - "fieldtype": "Select", - "in_list_view": 1, - "label": "Provider", - "options": "YouTube\nVimeo", - "reqd": 1 - }, - { - "fieldname": "url", - "fieldtype": "Data", - "in_list_view": 1, - "label": "URL", - "reqd": 1 - }, - { - "fieldname": "column_break_4", - "fieldtype": "Column Break" - }, - { - "fieldname": "publish_date", - "fieldtype": "Date", - "label": "Publish Date" - }, - { - "fieldname": "duration", - "fieldtype": "Data", - "label": "Duration" - }, - { - "fieldname": "section_break_7", - "fieldtype": "Section Break" - }, - { - "fieldname": "description", - "fieldtype": "Text Editor", - "in_list_view": 1, - "label": "Description", - "reqd": 1 - } - ], - "links": [], - "modified": "2020-04-22 12:09:49.057403", - "modified_by": "Administrator", - "module": "Core", - "name": "Video", - "owner": "Administrator", - "permissions": [ - { - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "if_owner": 1, - "print": 1, - "read": 1, - "report": 1, - "role": "All", - "share": 1, - "write": 1 - }, - { - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "share": 1, - "write": 1 - } - ], - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1 -} \ No newline at end of file diff --git a/frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.py b/frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.py index 8b2d1e01fa..97209cd8ea 100644 --- a/frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.py +++ b/frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.py @@ -41,6 +41,8 @@ def get_columns_and_fields(doctype): return columns, fields +@frappe.whitelist() +@frappe.validate_and_sanitize_search_inputs def query_doctypes(doctype, txt, searchfield, start, page_len, filters): user = filters.get("user") user_perms = frappe.utils.user.UserPermissions(user) diff --git a/frappe/data_migration/doctype/data_migration_run/test_data_migration_run.py b/frappe/data_migration/doctype/data_migration_run/test_data_migration_run.py index ba4a255b97..c6c3ea138c 100644 --- a/frappe/data_migration/doctype/data_migration_run/test_data_migration_run.py +++ b/frappe/data_migration/doctype/data_migration_run/test_data_migration_run.py @@ -108,7 +108,7 @@ def create_plan(): 'connector_name': 'Local Connector', 'connector_type': 'Frappe', # connect to same host. - 'hostname': frappe.conf.host_name, + 'hostname': frappe.conf.host_name or frappe.utils.get_site_url(frappe.local.site), 'username': 'Administrator', - 'password': 'admin' + 'password': frappe.conf.get("admin_password") or 'admin' }).insert(ignore_if_duplicate=True) diff --git a/frappe/database/database.py b/frappe/database/database.py index 101b97c915..2b8564481a 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -134,6 +134,8 @@ class Database(object): if debug: time_start = time() + self.log_query(query, values, debug, explain) + if values!=(): if isinstance(values, dict): values = dict(values) @@ -142,41 +144,18 @@ class Database(object): if not isinstance(values, (dict, tuple, list)): values = (values,) - if debug and query.strip().lower().startswith('select'): - try: - if explain: - self.explain_query(query, values) - frappe.errprint(query % values) - except TypeError: - frappe.errprint([query, values]) - if (frappe.conf.get("logging") or False)==2: - frappe.log("<<<< query") - frappe.log(query) - frappe.log("with values:") - frappe.log(values) - frappe.log(">>>>") self._cursor.execute(query, values) if frappe.flags.in_migrate: self.log_touched_tables(query, values) else: - if debug: - if explain: - self.explain_query(query) - frappe.errprint(query) - if (frappe.conf.get("logging") or False)==2: - frappe.log("<<<< query") - frappe.log(query) - frappe.log(">>>>") - self._cursor.execute(query) if frappe.flags.in_migrate: self.log_touched_tables(query) if debug: - frappe.errprint(self._cursor.mogrify(query, values)) time_end = time() frappe.errprint(("Execution time: {0} sec").format(round(time_end - time_start, 2))) @@ -213,6 +192,33 @@ class Database(object): else: return self._cursor.fetchall() + def log_query(self, query, values, debug, explain): + # for debugging in tests + if frappe.conf.get('allow_tests') and frappe.cache().get_value('flag_print_sql'): + print(self.mogrify(query, values)) + + # debug + if debug: + if explain and query.strip().lower().startswith('select'): + self.explain_query(query, values) + frappe.errprint(self.mogrify(query, values)) + + # info + if (frappe.conf.get("logging") or False)==2: + frappe.log("<<<< query") + frappe.log(self.mogrify(query, values)) + frappe.log(">>>>") + + def mogrify(self, query, values): + '''build the query string with values''' + if not values: + return query + else: + try: + return self._cursor.mogrify(query, values) + except: # noqa: E722 + return (query, values) + def explain_query(self, query, values=None): """Print `EXPLAIN` in error log.""" try: @@ -378,7 +384,7 @@ class Database(object): return self.get_value(doctype, filters, "*", as_dict=as_dict, cache=cache) def get_value(self, doctype, filters=None, fieldname="name", ignore=None, as_dict=False, - debug=False, order_by=None, cache=False): + debug=False, order_by=None, cache=False, for_update=False): """Returns a document property or list of properties. :param doctype: DocType name. @@ -405,12 +411,12 @@ class Database(object): """ ret = self.get_values(doctype, filters, fieldname, ignore, as_dict, debug, - order_by, cache=cache) + order_by, cache=cache, for_update=for_update) return ((len(ret[0]) > 1 or as_dict) and ret[0] or ret[0][0]) if ret else None def get_values(self, doctype, filters=None, fieldname="name", ignore=None, as_dict=False, - debug=False, order_by=None, update=None, cache=False): + debug=False, order_by=None, update=None, cache=False, for_update=False): """Returns multiple document properties. :param doctype: DocType name. @@ -449,7 +455,7 @@ class Database(object): if (filters is not None) and (filters!=doctype or doctype=="DocType"): try: - out = self._get_values_from_table(fields, filters, doctype, as_dict, debug, order_by, update) + out = self._get_values_from_table(fields, filters, doctype, as_dict, debug, order_by, update, for_update=for_update) except Exception as e: if ignore and (frappe.db.is_missing_column(e) or frappe.db.is_table_missing(e)): # table or column not found, return None @@ -576,7 +582,7 @@ class Database(object): """Alias for get_single_value""" return self.get_single_value(*args, **kwargs) - def _get_values_from_table(self, fields, filters, doctype, as_dict, debug, order_by=None, update=None): + def _get_values_from_table(self, fields, filters, doctype, as_dict, debug, order_by=None, update=None, for_update=False): fl = [] if isinstance(fields, (list, tuple)): for f in fields: @@ -594,9 +600,15 @@ class Database(object): order_by = ("order by " + order_by) if order_by else "" - r = self.sql("select {0} from `tab{1}` {2} {3} {4}" - .format(fl, doctype, "where" if conditions else "", conditions, order_by), values, - as_dict=as_dict, debug=debug, update=update) + r = self.sql("select {fields} from `tab{doctype}` {where} {conditions} {order_by} {for_update}" + .format( + for_update = 'for update' if for_update else '', + fields = fl, + doctype = doctype, + where = "where" if conditions else "", + conditions = conditions, + order_by = order_by), + values, as_dict=as_dict, debug=debug, update=update) return r @@ -616,7 +628,7 @@ class Database(object): return self.set_value(*args, **kwargs) def set_value(self, dt, dn, field, val=None, modified=None, modified_by=None, - update_modified=True, debug=False): + update_modified=True, debug=False, for_update=True): """Set a single value in the database, do not call the ORM triggers but update the modified timestamp (unless specified not to). @@ -630,6 +642,7 @@ class Database(object): :param modified_by: Set this user as `modified_by`. :param update_modified: default True. Set as false, if you don't want to update the timestamp. :param debug: Print the query in the developer / js console. + :param for_update: Will add a row-level lock to the value that is being set so that it can be released on commit. """ if not modified: modified = now() @@ -647,7 +660,9 @@ class Database(object): if dn and dt!=dn: # with table - conditions, values = self.build_conditions(dn) + values = dict( + name=self.get_value(dt, dn, 'name', for_update=for_update) + ) values.update(to_update) @@ -656,7 +671,7 @@ class Database(object): set_values.append('`{0}`=%({0})s'.format(key)) self.sql("""update `tab{0}` - set {1} where {2}""".format(dt, ', '.join(set_values), conditions), + set {1} where name=%(name)s""".format(dt, ', '.join(set_values)), values, debug=debug) else: diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index 4ec89c126d..3cbb2e4f0e 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -46,7 +46,7 @@ class MariaDBDatabase(Database): 'Data': ('varchar', self.VARCHAR_LEN), 'Link': ('varchar', self.VARCHAR_LEN), 'Dynamic Link': ('varchar', self.VARCHAR_LEN), - 'Password': ('varchar', self.VARCHAR_LEN), + 'Password': ('text', ''), 'Select': ('varchar', self.VARCHAR_LEN), 'Rating': ('int', '1'), 'Read Only': ('varchar', self.VARCHAR_LEN), @@ -186,7 +186,7 @@ class MariaDBDatabase(Database): `doctype` VARCHAR(140) NOT NULL, `name` VARCHAR(255) NOT NULL, `fieldname` VARCHAR(140) NOT NULL, - `password` VARCHAR(255) NOT NULL, + `password` TEXT NOT NULL, `encrypted` INT(1) NOT NULL DEFAULT 0, PRIMARY KEY (`doctype`, `name`, `fieldname`) ) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci""") diff --git a/frappe/database/mariadb/framework_mariadb.sql b/frappe/database/mariadb/framework_mariadb.sql index af537e0612..1e3749e030 100644 --- a/frappe/database/mariadb/framework_mariadb.sql +++ b/frappe/database/mariadb/framework_mariadb.sql @@ -277,7 +277,7 @@ CREATE TABLE `__Auth` ( `doctype` VARCHAR(140) NOT NULL, `name` VARCHAR(255) NOT NULL, `fieldname` VARCHAR(140) NOT NULL, - `password` VARCHAR(255) NOT NULL, + `password` TEXT NOT NULL, `encrypted` INT(1) NOT NULL DEFAULT 0, PRIMARY KEY (`doctype`, `name`, `fieldname`) ) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; diff --git a/frappe/database/postgres/database.py b/frappe/database/postgres/database.py index e348916705..3d997864e4 100644 --- a/frappe/database/postgres/database.py +++ b/frappe/database/postgres/database.py @@ -51,7 +51,7 @@ class PostgresDatabase(Database): 'Data': ('varchar', self.VARCHAR_LEN), 'Link': ('varchar', self.VARCHAR_LEN), 'Dynamic Link': ('varchar', self.VARCHAR_LEN), - 'Password': ('varchar', self.VARCHAR_LEN), + 'Password': ('text', ''), 'Select': ('varchar', self.VARCHAR_LEN), 'Rating': ('smallint', None), 'Read Only': ('varchar', self.VARCHAR_LEN), @@ -179,7 +179,7 @@ class PostgresDatabase(Database): "doctype" VARCHAR(140) NOT NULL, "name" VARCHAR(255) NOT NULL, "fieldname" VARCHAR(140) NOT NULL, - "password" VARCHAR(255) NOT NULL, + "password" TEXT NOT NULL, "encrypted" INT NOT NULL DEFAULT 0, PRIMARY KEY ("doctype", "name", "fieldname") )""") diff --git a/frappe/database/postgres/framework_postgres.sql b/frappe/database/postgres/framework_postgres.sql index 8f77ed6230..a946a7ee5c 100644 --- a/frappe/database/postgres/framework_postgres.sql +++ b/frappe/database/postgres/framework_postgres.sql @@ -281,7 +281,7 @@ CREATE TABLE "__Auth" ( "doctype" VARCHAR(140) NOT NULL, "name" VARCHAR(255) NOT NULL, "fieldname" VARCHAR(140) NOT NULL, - "password" VARCHAR(255) NOT NULL, + "password" TEXT NOT NULL, "encrypted" int NOT NULL DEFAULT 0, PRIMARY KEY ("doctype", "name", "fieldname") ); diff --git a/frappe/desk/desktop.py b/frappe/desk/desktop.py index 68b57a93d4..ae9d070976 100644 --- a/frappe/desk/desktop.py +++ b/frappe/desk/desktop.py @@ -221,6 +221,8 @@ class Workspace: incomplete_dependencies = [d for d in item.dependencies if not _doctype_contains_a_record(d)] if len(incomplete_dependencies): item.incomplete_dependencies = incomplete_dependencies + else: + item.incomplete_dependencies = "" if item.onboard: # Mark Spotlights for initial diff --git a/frappe/desk/doctype/dashboard_chart/dashboard_chart.js b/frappe/desk/doctype/dashboard_chart/dashboard_chart.js index 8d89cc2f31..7f26bd9101 100644 --- a/frappe/desk/doctype/dashboard_chart/dashboard_chart.js +++ b/frappe/desk/doctype/dashboard_chart/dashboard_chart.js @@ -23,7 +23,6 @@ frappe.ui.form.on('Dashboard Chart', { frm.chart_filters = null; if (!frappe.boot.developer_mode && frm.doc.is_standard) { - frm.set_df_property('chart_options_section', 'hidden', 1); frm.disable_form(); } @@ -57,11 +56,6 @@ frappe.ui.form.on('Dashboard Chart', { if (frm.doc.report_name) { frm.trigger('set_chart_report_filters'); } - - if (!frappe.boot.developer_mode) { - frm.set_df_property("custom_options", "hidden", 1); - } - }, is_standard: function(frm) { diff --git a/frappe/desk/doctype/dashboard_chart/dashboard_chart.py b/frappe/desk/doctype/dashboard_chart/dashboard_chart.py index a25e0cab9e..4ea61ec6a9 100644 --- a/frappe/desk/doctype/dashboard_chart/dashboard_chart.py +++ b/frappe/desk/doctype/dashboard_chart/dashboard_chart.py @@ -28,15 +28,28 @@ def get_permission_query_conditions(user): if "System Manager" in roles: return None - allowed_doctypes = ['"%s"' % doctype for doctype in frappe.permissions.get_doctypes_with_read()] - allowed_reports = ['"%s"' % key if type(key) == str else key.encode('UTF8') for key in get_allowed_reports()] + doctype_condition = False + report_condition = False + + allowed_doctypes = [frappe.db.escape(doctype) for doctype in frappe.permissions.get_doctypes_with_read()] + allowed_reports = [frappe.db.escape(key) if type(key) == str else key.encode('UTF8') for key in get_allowed_reports()] + + if allowed_doctypes: + doctype_condition = '`tabDashboard Chart`.`document_type` in ({allowed_doctypes})'.format( + allowed_doctypes=','.join(allowed_doctypes)) + if allowed_reports: + report_condition = '`tabDashboard Chart`.`report_name` in ({allowed_reports})'.format( + allowed_reports=','.join(allowed_reports)) return ''' - `tabDashboard Chart`.`document_type` in ({allowed_doctypes}) - or `tabDashboard Chart`.`report_name` in ({allowed_reports}) + (`tabDashboard Chart`.`chart_type` in ('Count', 'Sum', 'Average') + and {doctype_condition}) + or + (`tabDashboard Chart`.`chart_type` = 'Report' + and {report_condition}) '''.format( - allowed_doctypes=','.join(allowed_doctypes), - allowed_reports=','.join(allowed_reports) + doctype_condition=doctype_condition, + report_condition=report_condition ) @@ -130,7 +143,7 @@ def add_chart_to_dashboard(args): dashboard_link = frappe.new_doc('Dashboard Chart Link') dashboard_link.chart = args.chart_name or args.name - if args.set_standard: + if args.set_standard and dashboard.is_standard: chart = frappe.get_doc('Dashboard Chart', dashboard_link.chart) chart.is_standard = 1 chart.module = dashboard.module @@ -344,6 +357,8 @@ def get_year_ending(date): # last day of this month return add_to_date(date, days=-1) +@frappe.whitelist() +@frappe.validate_and_sanitize_search_inputs def get_charts_for_user(doctype, txt, searchfield, start, page_len, filters): or_filters = {'owner': frappe.session.user, 'is_public': 1} return frappe.db.get_list('Dashboard Chart', diff --git a/frappe/desk/doctype/desk_page/desk_page.py b/frappe/desk/doctype/desk_page/desk_page.py index f14535cb5f..cc2db53481 100644 --- a/frappe/desk/doctype/desk_page/desk_page.py +++ b/frappe/desk/doctype/desk_page/desk_page.py @@ -5,14 +5,23 @@ from __future__ import unicode_literals import frappe from frappe import _ +from frappe.utils.data import validate_json_string from frappe.modules.export_file import export_to_files from frappe.model.document import Document class DeskPage(Document): def validate(self): + self.validate_cards_json() if (self.is_standard and not frappe.conf.developer_mode and not disable_saving_as_standard()): frappe.throw(_("You need to be in developer mode to edit this document")) + def validate_cards_json(self): + for card in self.cards: + try: + validate_json_string(card.links) + except frappe.ValidationError: + frappe.throw(_("Invalid JSON in card links for {0}").format(frappe.bold(card.label))) + def on_update(self): if disable_saving_as_standard(): return diff --git a/frappe/desk/doctype/event/event.py b/frappe/desk/doctype/event/event.py index 45dc5d86c7..54905bed6a 100644 --- a/frappe/desk/doctype/event/event.py +++ b/frappe/desk/doctype/event/event.py @@ -82,6 +82,27 @@ class Event(Document): communication.add_link(participant.reference_doctype, participant.reference_docname) communication.save(ignore_permissions=True) + def add_participant(self, doctype, docname): + """Add a single participant to event participants + + Args: + doctype (string): Reference Doctype + docname (string): Reference Docname + """ + self.append("event_participants", { + "reference_doctype": doctype, + "reference_docname": docname, + }) + + def add_participants(self, participants): + """Add participant entry + + Args: + participants ([Array]): Array of a dict with doctype and docname + """ + for participant in participants: + self.add_participant(participant["doctype"], participant["docname"]) + @frappe.whitelist() def delete_communication(event, reference_doctype, reference_docname): deleted_participant = frappe.get_doc(reference_doctype, reference_docname) diff --git a/frappe/desk/doctype/number_card/number_card.py b/frappe/desk/doctype/number_card/number_card.py index 3eb08ec00a..d4a2b00c57 100644 --- a/frappe/desk/doctype/number_card/number_card.py +++ b/frappe/desk/doctype/number_card/number_card.py @@ -32,13 +32,17 @@ def get_permission_query_conditions(user=None): if "System Manager" in roles: return None - allowed_doctypes = ['"%s"' % doctype for doctype in frappe.permissions.get_doctypes_with_read()] + doctype_condition = False + + allowed_doctypes = [frappe.db.escape(doctype) for doctype in frappe.permissions.get_doctypes_with_read()] + + if allowed_doctypes: + doctype_condition = '`tabNumber Card`.`document_type` in ({allowed_doctypes})'.format( + allowed_doctypes=','.join(allowed_doctypes)) return ''' - `tabNumber Card`.`document_type` in ({allowed_doctypes}) - '''.format( - allowed_doctypes=','.join(allowed_doctypes) - ) + {doctype_condition} + '''.format(doctype_condition=doctype_condition) def has_permission(doc, ptype, user): roles = frappe.get_roles(user) @@ -124,11 +128,16 @@ def create_number_card(args): doc.insert(ignore_permissions=True) return doc +@frappe.whitelist() +@frappe.validate_and_sanitize_search_inputs def get_cards_for_user(doctype, txt, searchfield, start, page_len, filters): meta = frappe.get_meta(doctype) searchfields = meta.get_search_fields() search_conditions = [] + if not frappe.db.exists('DocType', doctype): + return + if txt: for field in searchfields: search_conditions.append('`tab{doctype}`.`{field}` like %(txt)s'.format(field=field, doctype=doctype, txt=txt)) @@ -172,7 +181,7 @@ def add_card_to_dashboard(args): dashboard_link = frappe.new_doc('Number Card Link') dashboard_link.card = args.name - if args.set_standard: + if args.set_standard and dashboard.is_standard: card = frappe.get_doc('Number Card', dashboard_link.card) card.is_standard = 1 card.module = dashboard.module diff --git a/frappe/desk/doctype/onboarding_step/onboarding_step.json b/frappe/desk/doctype/onboarding_step/onboarding_step.json index 365a1c7d21..79d659b1ed 100644 --- a/frappe/desk/doctype/onboarding_step/onboarding_step.json +++ b/frappe/desk/doctype/onboarding_step/onboarding_step.json @@ -184,7 +184,7 @@ } ], "links": [], - "modified": "2020-05-18 19:42:30.435604", + "modified": "2020-08-06 12:55:20.377679", "modified_by": "Administrator", "module": "Desk", "name": "Onboarding Step", diff --git a/frappe/desk/form/document_follow.py b/frappe/desk/form/document_follow.py index 333e9e1333..80f614b5b6 100644 --- a/frappe/desk/form/document_follow.py +++ b/frappe/desk/form/document_follow.py @@ -5,6 +5,7 @@ from __future__ import unicode_literals import frappe import frappe.utils from frappe.utils import get_url_to_form +from frappe.model import log_types from frappe import _ from itertools import groupby @@ -20,22 +21,26 @@ def follow_document(doctype, doc_name, user, force=False): avoided for some doctype follow only if track changes are set to 1 ''' - avoid_follow = ["Communication", "ToDo", "DocShare", "Email Unsubscribe", "Activity Log", - "File", "Version", "View Log", "Document Follow", "Comment"] + if (doctype in ("Communication", "ToDo", "Email Unsubscribe", "File", "Comment") + or doctype in log_types): + return - track_changes = frappe.get_meta(doctype).track_changes - exists = is_document_followed(doctype, doc_name, user) - if exists == 0: - user_can_follow = frappe.db.get_value("User", user, "document_follow_notify", ignore=True) - if user != "Administrator" and user_can_follow and track_changes and (doctype not in avoid_follow or force): - doc = frappe.new_doc("Document Follow") - doc.update({ - "ref_doctype": doctype, - "ref_docname": doc_name, - "user": user - }) - doc.save() - return doc + if ((not frappe.get_meta(doctype).track_changes) + or user == "Administrator"): + return + + if not frappe.db.get_value("User", user, "document_follow_notify", ignore=True, cache=True): + return + + if not is_document_followed(doctype, doc_name, user): + doc = frappe.new_doc("Document Follow") + doc.update({ + "ref_doctype": doctype, + "ref_docname": doc_name, + "user": user + }) + doc.save() + return doc @frappe.whitelist() def unfollow_document(doctype, doc_name, user): diff --git a/frappe/desk/form/linked_with.py b/frappe/desk/form/linked_with.py index 5bae49ea95..733ee1774c 100644 --- a/frappe/desk/form/linked_with.py +++ b/frappe/desk/form/linked_with.py @@ -11,7 +11,6 @@ from frappe import _ from frappe.model.meta import is_single from frappe.modules import load_doctype_module - @frappe.whitelist() def get_submitted_linked_docs(doctype, name, docs=None, visited=None): """ @@ -78,7 +77,7 @@ def get_submitted_linked_docs(doctype, name, docs=None, visited=None): @frappe.whitelist() -def cancel_all_linked_docs(docs): +def cancel_all_linked_docs(docs, ignore_doctypes_on_cancel_all=[]): """ Cancel all linked doctype @@ -87,14 +86,16 @@ def cancel_all_linked_docs(docs): """ docs = json.loads(docs) + if isinstance(ignore_doctypes_on_cancel_all, string_types): + ignore_doctypes_on_cancel_all = json.loads(ignore_doctypes_on_cancel_all) for i, doc in enumerate(docs, 1): - if validate_linked_doc(doc) is True: - frappe.publish_progress(percent=i * 100 / len(docs), title=_("Cancelling documents")) + if validate_linked_doc(doc, ignore_doctypes_on_cancel_all) is True: + frappe.publish_progress(percent=i * 100 / ((len(docs) - len(ignore_doctypes_on_cancel_all))), title=_("Cancelling documents")) linked_doc = frappe.get_doc(doc.get("doctype"), doc.get("name")) linked_doc.cancel() -def validate_linked_doc(docinfo): +def validate_linked_doc(docinfo, ignore_doctypes_on_cancel_all=[]): """ Validate a document to be submitted and non-exempted from auto-cancel. @@ -105,6 +106,10 @@ def validate_linked_doc(docinfo): bool: True if linked document passes all validations, else False """ + #ignore doctype to cancel + if docinfo.get("doctype") in ignore_doctypes_on_cancel_all: + return False + # skip non-submittable doctypes since they don't need to be cancelled if not frappe.get_meta(docinfo.get('doctype')).is_submittable: return False diff --git a/frappe/desk/query_report.py b/frappe/desk/query_report.py index 0edfd57d4f..d0a32ef076 100644 --- a/frappe/desk/query_report.py +++ b/frappe/desk/query_report.py @@ -67,8 +67,7 @@ def generate_report_result(report, filters=None, user=None, custom_columns=None) # Reordered columns columns = json.loads(report.custom_columns) - if report.report_type == 'Query Report': - result = reorder_data_for_custom_columns(columns, query_columns, result) + result = reorder_data_for_custom_columns(columns, query_columns, result, report.report_type) result = add_data_to_custom_columns(columns, result) @@ -216,15 +215,21 @@ def add_data_to_custom_columns(columns, result): return data -def reorder_data_for_custom_columns(custom_columns, columns, result): +def reorder_data_for_custom_columns(custom_columns, columns, result, report_type): + custom_column_labels = [col["label"] for col in custom_columns] + + if report_type == 'Query Report': + original_column_labels = [col.split(":")[0] for col in columns] + else: + original_column_labels = [col["label"] for col in columns] + reordered_result = [] - columns = [col.split(":")[0] for col in columns] for res in result: r = [] - for col in custom_columns: + for col_name in custom_column_labels: try: - idx = columns.index(col.get("label")) + idx = original_column_labels.index(col_name) r.append(res[idx]) except ValueError: pass diff --git a/frappe/desk/search.py b/frappe/desk/search.py index b4b54b4b6e..798e499bb9 100644 --- a/frappe/desk/search.py +++ b/frappe/desk/search.py @@ -10,6 +10,7 @@ from frappe.handler import is_whitelisted from frappe import _ from six import string_types import re +import wrapt UNTRANSLATED_DOCTYPES = ["DocType", "Role"] @@ -206,3 +207,15 @@ def scrub_custom_query(query, key, txt): if '%s' in query: query = query.replace('%s', ((txt or '') + '%')) return query + +@wrapt.decorator +def validate_and_sanitize_search_inputs(fn, instance, args, kwargs): + kwargs.update(dict(zip(fn.__code__.co_varnames, args))) + sanitize_searchfield(kwargs['searchfield']) + kwargs['start'] = cint(kwargs['start']) + kwargs['page_len'] = cint(kwargs['page_len']) + + if kwargs['doctype'] and not frappe.db.exists('DocType', kwargs['doctype']): + return [] + + return fn(**kwargs) \ No newline at end of file diff --git a/frappe/email/__init__.py b/frappe/email/__init__.py index d58b35040e..b05aef7639 100644 --- a/frappe/email/__init__.py +++ b/frappe/email/__init__.py @@ -57,6 +57,8 @@ def relink(name, reference_doctype=None, reference_name=None): communication_type = "Communication" and name = %s""", (reference_doctype, reference_name, name)) +@frappe.whitelist() +@frappe.validate_and_sanitize_search_inputs def get_communication_doctype(doctype, txt, searchfield, start, page_len, filters): user_perms = frappe.utils.user.UserPermissions(frappe.session.user) user_perms.build_permissions() diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index cf8c6e80c6..29cd890bf1 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -251,7 +251,7 @@ class EmailAccount(Document): email_server = None if frappe.local.flags.in_test: - incoming_mails = test_mails + incoming_mails = test_mails or [] else: email_sync_rule = self.build_email_sync_rule() diff --git a/frappe/email/doctype/email_account/test_email_account.py b/frappe/email/doctype/email_account/test_email_account.py index 29b54d7f8b..f87ee32bb1 100644 --- a/frappe/email/doctype/email_account/test_email_account.py +++ b/frappe/email/doctype/email_account/test_email_account.py @@ -5,7 +5,10 @@ from __future__ import unicode_literals import frappe, os import unittest, email -test_records = frappe.get_test_records('Email Account') +from frappe.test_runner import make_test_records + +make_test_records("User") +make_test_records("Email Account") from frappe.core.doctype.communication.email import make from frappe.desk.form.load import get_attachments diff --git a/frappe/email/doctype/newsletter/newsletter.json b/frappe/email/doctype/newsletter/newsletter.json index 01f75be954..1ec64826da 100644 --- a/frappe/email/doctype/newsletter/newsletter.json +++ b/frappe/email/doctype/newsletter/newsletter.json @@ -132,10 +132,11 @@ "has_web_view": 1, "icon": "fa fa-envelope", "idx": 1, + "index_web_pages_for_search": 1, "is_published_field": "published", "links": [], "max_attachments": 3, - "modified": "2020-05-12 18:09:40.137138", + "modified": "2020-07-21 16:25:17.687476", "modified_by": "Administrator", "module": "Email", "name": "Newsletter", diff --git a/frappe/email/doctype/notification/notification.js b/frappe/email/doctype/notification/notification.js index 02fc8512ca..454514f922 100644 --- a/frappe/email/doctype/notification/notification.js +++ b/frappe/email/doctype/notification/notification.js @@ -3,104 +3,175 @@ this.frm.add_fetch('sender', 'email_id', 'sender_email'); -this.frm.fields_dict.sender.get_query = function(){ +this.frm.fields_dict.sender.get_query = function() { return { filters: { - 'enable_outgoing': 1 + enable_outgoing: 1 } - } + }; }; frappe.notification = { setup_fieldname_select: function(frm) { // get the doctype to update fields - if(!frm.doc.document_type) { + if (!frm.doc.document_type) { return; } frappe.model.with_doctype(frm.doc.document_type, function() { let get_select_options = function(df) { - return {value: df.fieldname, label: df.fieldname + " (" + __(df.label) + ")"}; - } + return { + value: df.fieldname, + label: df.fieldname + ' (' + __(df.label) + ')' + }; + }; let get_date_change_options = function() { let date_options = $.map(fields, function(d) { - return (d.fieldtype=="Date" || d.fieldtype=="Datetime")? - get_select_options(d) : null; + return d.fieldtype == 'Date' || d.fieldtype == 'Datetime' + ? get_select_options(d) + : null; }); // append creation and modified date to Date Change field return date_options.concat([ - { value: "creation", label: `creation (${__('Created On')})` }, - { value: "modified", label: `modified (${__('Last Modified Date')})` } + { value: 'creation', label: `creation (${__('Created On')})` }, + { value: 'modified', label: `modified (${__('Last Modified Date')})` } ]); - } + }; - let fields = frappe.get_doc("DocType", frm.doc.document_type).fields; - let options = $.map(fields, - function(d) { return in_list(frappe.model.no_value_type, d.fieldtype) ? - null : get_select_options(d); }); + let fields = frappe.get_doc('DocType', frm.doc.document_type).fields; + let options = $.map(fields, function(d) { + return in_list(frappe.model.no_value_type, d.fieldtype) + ? null : get_select_options(d); + }); // set value changed options - frm.set_df_property("value_changed", "options", [""].concat(options)); - frm.set_df_property("set_property_after_alert", "options", [""].concat(options)); + frm.set_df_property('value_changed', 'options', [''].concat(options)); + frm.set_df_property( + 'set_property_after_alert', + 'options', + [''].concat(options) + ); // set date changed options - frm.set_df_property("date_changed", "options", get_date_change_options()); + frm.set_df_property('date_changed', 'options', get_date_change_options()); - let email_fields = $.map(fields, - function(d) { return (d.options == "Email" || - (d.options=='User' && d.fieldtype=='Link')) ? - get_select_options(d) : null; }); + let receiver_fields = []; + if (frm.doc.channel === 'Email') { + receiver_fields = $.map(fields, function(d) { + return d.options == 'Email' || + (d.options == 'User' && d.fieldtype == 'Link') + ? get_select_options(d) : null; + }); + } else if (in_list(['WhatsApp', 'SMS'], frm.doc.channel)) { + receiver_fields = $.map(fields, function(d) { + return d.options == 'Phone' ? get_select_options(d) : null; + }); + } // set email recipient options - frappe.meta.get_docfield("Notification Recipient", "email_by_document_field", + frappe.meta.get_docfield( + 'Notification Recipient', + 'receiver_by_document_field', // set first option as blank to allow notification not to be defaulted to the owner - frm.doc.name).options = [""].concat(["owner"].concat(email_fields)); + frm.doc.name + ).options = [''].concat(["owner"]).concat(receiver_fields); frm.fields_dict.recipients.grid.refresh(); }); - } -} + }, + setup_example_message: function(frm) { + let template = ''; + if (frm.doc.channel === 'WhatsApp') { + template = `
    Warning:
    Only Use Pre-Approved WhatsApp for Business Template +
    Message Example
    -frappe.ui.form.on("Notification", { +
    +Your {{ doc.name }} order of {{ doc.total }} has shipped and should be delivered on {{ doc.date }}. Details : {{doc.customer}}
    +
    `; + } else if (frm.doc.channel === 'Email') { + template = `
    Message Example
    + +
    <h3>Order Overdue</h3>
    +
    +<p>Transaction {{ doc.name }} has exceeded Due Date. Please take necessary action.</p>
    +
    +<!-- show last comment -->
    +{% if comments %}
    +Last comment: {{ comments[-1].comment }} by {{ comments[-1].by }}
    +{% endif %}
    +
    +<h4>Details</h4>
    +
    +<ul>
    +<li>Customer: {{ doc.customer }}
    +<li>Amount: {{ doc.grand_total }}
    +</ul>
    +
    + `; + } else { + template = `
    Message Example
    + +
    *Order Overdue*
    +
    +Transaction {{ doc.name }} has exceeded Due Date. Please take necessary action.
    +
    +
    +{% if comments %}
    +Last comment: {{ comments[-1].comment }} by {{ comments[-1].by }}
    +{% endif %}
    +
    +*Details*
    +
    +• Customer: {{ doc.customer }}
    +• Amount: {{ doc.grand_total }}
    +
    `; + } + frm.set_df_property('message_examples', 'options', template); + + } +}; + +frappe.ui.form.on('Notification', { onload: function(frm) { - frm.set_query("document_type", function() { + frm.set_query('document_type', function() { return { - "filters": { - "istable": 0 + filters: { + istable: 0 } - } + }; }); - frm.set_query("print_format", function() { + frm.set_query('print_format', function() { return { - "filters": { - "doc_type": frm.doc.document_type + filters: { + doc_type: frm.doc.document_type } - } + }; }); }, refresh: function(frm) { frappe.notification.setup_fieldname_select(frm); - frm.get_field("is_standard").toggle(frappe.boot.developer_mode); + frm.get_field('is_standard').toggle(frappe.boot.developer_mode); frm.trigger('event'); }, document_type: function(frm) { frappe.notification.setup_fieldname_select(frm); }, view_properties: function(frm) { - frappe.route_options = {doc_type:frm.doc.document_type}; - frappe.set_route("Form", "Customize Form"); + frappe.route_options = { doc_type: frm.doc.document_type }; + frappe.set_route('Form', 'Customize Form'); }, event: function(frm) { - if(in_list(['Days Before', 'Days After'], frm.doc.event)) { + if (in_list(['Days Before', 'Days After'], frm.doc.event)) { frm.add_custom_button(__('Get Alerts for Today'), function() { frappe.call({ - method: 'frappe.email.doctype.notification.notification.get_documents_for_today', + method: + 'frappe.email.doctype.notification.notification.get_documents_for_today', args: { notification: frm.doc.name }, callback: function(r) { - if(r.message) { + if (r.message) { frappe.msgprint(r.message); } else { frappe.msgprint(__('No alerts for today')); @@ -111,6 +182,14 @@ frappe.ui.form.on("Notification", { } }, channel: function(frm) { - frm.toggle_reqd("recipients", frm.doc.channel=="Email"); + frm.toggle_reqd('recipients', frm.doc.channel == 'Email'); + frappe.notification.setup_fieldname_select(frm); + frappe.notification.setup_example_message(frm); + if (frm.doc.channel === 'SMS' && frm.doc.__islocal) { + frm.set_df_property('channel', + 'description', `To use SMS Channel, initialize SMS Settings.`); + } else { + frm.set_df_property('channel', 'description', ` `); + } } }); diff --git a/frappe/email/doctype/notification/notification.json b/frappe/email/doctype/notification/notification.json index 932f0491a9..95f218ad73 100644 --- a/frappe/email/doctype/notification/notification.json +++ b/frappe/email/doctype/notification/notification.json @@ -10,6 +10,7 @@ "enabled", "column_break_2", "channel", + "twilio_number", "slack_webhook_url", "filters", "subject", @@ -37,7 +38,6 @@ "message_sb", "message", "message_examples", - "slack_message_examples", "view_properties", "column_break_25", "attach_print", @@ -60,11 +60,13 @@ "fieldname": "channel", "fieldtype": "Select", "label": "Channel", - "options": "Email\nSlack\nSystem Notification", - "reqd": 1 + "options": "Email\nSlack\nSystem Notification\nWhatsApp\nSMS", + "reqd": 1, + "set_only_once": 1 }, { "depends_on": "eval:doc.channel=='Slack'", + "description": "To use Slack Channel, add a Slack Webhook URL.", "fieldname": "slack_webhook_url", "fieldtype": "Link", "label": "Slack Channel", @@ -77,13 +79,14 @@ "label": "Filters" }, { + "depends_on": "eval: !in_list(['SMS', 'WhatsApp'], doc.channel)", "description": "To add dynamic subject, use jinja tags like\n\n
    {{ doc.name }} Delivered
    ", "fieldname": "subject", "fieldtype": "Data", "ignore_xss_filter": 1, "in_list_view": 1, "label": "Subject", - "reqd": 1 + "mandatory_depends_on": "eval:!in_list(['SMS', 'WhatsApp'], doc.channel)" }, { "fieldname": "document_type", @@ -153,6 +156,7 @@ "label": "Value Changed" }, { + "depends_on": "eval: doc.channel == 'Email'", "fieldname": "sender", "fieldtype": "Link", "label": "Sender", @@ -203,7 +207,7 @@ "label": "Value To Be Set" }, { - "depends_on": "eval:doc.channel!=='Slack'", + "depends_on": "eval:in_list(['Email', 'SMS', 'WhatsApp'], doc.channel)", "fieldname": "column_break_5", "fieldtype": "Section Break", "label": "Recipients" @@ -228,19 +232,11 @@ "label": "Message" }, { - "depends_on": "eval:doc.channel=='Email'", "fieldname": "message_examples", "fieldtype": "HTML", "label": "Message Examples", "options": "
    Message Example
    \n\n
    <h3>Order Overdue</h3>\n\n<p>Transaction {{ doc.name }} has exceeded Due Date. Please take necessary action.</p>\n\n<!-- show last comment -->\n{% if comments %}\nLast comment: {{ comments[-1].comment }} by {{ comments[-1].by }}\n{% endif %}\n\n<h4>Details</h4>\n\n<ul>\n<li>Customer: {{ doc.customer }}\n<li>Amount: {{ doc.grand_total }}\n</ul>\n
    " }, - { - "depends_on": "eval:doc.channel=='Slack'", - "fieldname": "slack_message_examples", - "fieldtype": "HTML", - "label": "Message Examples", - "options": "
    Message Example
    \n\n
    *Order Overdue*\n\nTransaction {{ doc.name }} has exceeded Due Date. Please take necessary action.\n\n\n{% if comments %}\nLast comment: {{ comments[-1].comment }} by {{ comments[-1].by }}\n{% endif %}\n\n*Details*\n\n\u2022 Customer: {{ doc.customer }}\n\u2022 Amount: {{ doc.grand_total }}\n
    " - }, { "fieldname": "view_properties", "fieldtype": "Button", @@ -266,6 +262,14 @@ "label": "Print Format", "options": "Print Format" }, + { + "depends_on": "eval: doc.channel==='WhatsApp'", + "description": "To use WhatsApp for Business, initialize Twilio Settings.", + "fieldname": "twilio_number", + "fieldtype": "Link", + "label": "Twilio Number", + "options": "Twilio Number Group" + }, { "default": "0", "depends_on": "eval: doc.channel !== 'System Notification'", @@ -277,7 +281,7 @@ ], "icon": "fa fa-envelope", "links": [], - "modified": "2020-06-23 14:01:25.462544", + "modified": "2020-08-11 19:24:35.479373", "modified_by": "Administrator", "module": "Email", "name": "Notification", diff --git a/frappe/email/doctype/notification/notification.py b/frappe/email/doctype/notification/notification.py index 81670756f6..2ec208c89d 100644 --- a/frappe/email/doctype/notification/notification.py +++ b/frappe/email/doctype/notification/notification.py @@ -7,12 +7,14 @@ import frappe import json, os from frappe import _ from frappe.model.document import Document -from frappe.core.doctype.role.role import get_emails_from_role +from frappe.core.doctype.role.role import get_info_based_on_role, get_user_info from frappe.utils import validate_email_address, nowdate, parse_val, is_html, add_to_date from frappe.utils.jinja import validate_template from frappe.modules.utils import export_module_json, get_doc_module from six import string_types from frappe.integrations.doctype.slack_webhook_url.slack_webhook_url import send_slack_message +from frappe.integrations.doctype.twilio_settings.twilio_settings import send_whatsapp_message +from frappe.core.doctype.sms_settings.sms_settings import send_sms from frappe.desk.doctype.notification_log.notification_log import enqueue_create_notification class Notification(Document): @@ -26,7 +28,9 @@ class Notification(Document): self.name = self.subject def validate(self): - validate_template(self.subject) + if self.channel not in ('WhatsApp', 'SMS'): + validate_template(self.subject) + validate_template(self.message) if self.event in ("Days Before", "Days After") and not self.date_changed: @@ -126,8 +130,15 @@ def get_context(context): if self.channel == 'Slack': self.send_a_slack_msg(doc, context) + if self.channel == 'WhatsApp': + self.send_whatsapp_msg(doc, context) + + if self.channel == 'SMS': + self.send_sms(doc, context) + if self.channel == 'System Notification' or self.send_system_notification: self.create_system_notification(doc, context) + except: frappe.log_error(title='Failed to send notification', message=frappe.get_traceback()) @@ -195,11 +206,24 @@ def get_context(context): and attachments[0].get('print_letterhead')) or False)) def send_a_slack_msg(self, doc, context): - send_slack_message( - webhook_url=self.slack_webhook_url, - message=frappe.render_template(self.message, context), - reference_doctype = doc.doctype, - reference_name = doc.name) + send_slack_message( + webhook_url=self.slack_webhook_url, + message=frappe.render_template(self.message, context), + reference_doctype=doc.doctype, + reference_name=doc.name) + + def send_whatsapp_msg(self, doc, context): + send_whatsapp_message( + sender=self.twilio_number, + receiver_list=self.get_receiver_list(doc, context), + message=frappe.render_template(self.message, context), + ) + + def send_sms(self, doc, context): + send_sms( + receiver_list=self.get_receiver_list(doc, context), + msg=frappe.render_template(self.message, context) + ) def get_list_of_recipients(self, doc, context): recipients = [] @@ -209,8 +233,8 @@ def get_context(context): if recipient.condition: if not frappe.safe_eval(recipient.condition, None, context): continue - if recipient.email_by_document_field: - email_ids_value = doc.get(recipient.email_by_document_field) + if recipient.receiver_by_document_field: + email_ids_value = doc.get(recipient.receiver_by_document_field) if validate_email_address(email_ids_value): email_ids = email_ids_value.replace(",", "\n") recipients = recipients + email_ids.split("\n") @@ -232,8 +256,8 @@ def get_context(context): bcc = bcc + recipient.bcc.split("\n") #For sending emails to specified role - if recipient.email_by_role: - emails = get_emails_from_role(recipient.email_by_role) + if recipient.receiver_by_role: + emails = get_info_based_on_role(recipient.receiver_by_role, 'email') for email in emails: recipients = recipients + email.split("\n") @@ -242,6 +266,27 @@ def get_context(context): return None, None, None return list(set(recipients)), list(set(cc)), list(set(bcc)) + def get_receiver_list(self, doc, context): + ''' return receiver list based on the doc field and role specified ''' + receiver_list = [] + for recipient in self.recipients: + if recipient.condition: + if not frappe.safe_eval(recipient.condition, None, context): + continue + + # For sending messages to the owner's mobile phone number + if recipient.receiver_by_document_field == 'owner': + receiver_list.append(get_user_info(doc.get('owner'), 'mobile_no')) + # For sending messages to the number specified in the receiver field + elif recipient.receiver_by_document_field: + receiver_list.append(doc.get(recipient.receiver_by_document_field)) + + #For sending messages to specified role + if recipient.receiver_by_role: + receiver_list += get_info_based_on_role(recipient.receiver_by_role, 'mobile_no') + + return receiver_list + def get_attachment(self, doc): """ check print settings are attach the pdf """ if not self.attach_print: diff --git a/frappe/email/doctype/notification/test_notification.py b/frappe/email/doctype/notification/test_notification.py index b9bbde172d..9bdf09375d 100644 --- a/frappe/email/doctype/notification/test_notification.py +++ b/frappe/email/doctype/notification/test_notification.py @@ -63,7 +63,7 @@ class TestNotification(unittest.TestCase): notification.message = "test" recipent = frappe.new_doc("Notification Recipient") - recipent.email_by_document_field = "owner" + recipent.receiver_by_document_field = "owner" notification.recipents = recipent notification.condition = "test" @@ -105,7 +105,7 @@ class TestNotification(unittest.TestCase): "value_changed": "description1", "message": "Description changed", "recipients": [ - { "email_by_document_field": "owner" } + { "receiver_by_document_field": "owner" } ] }).insert() frappe.db.commit() diff --git a/frappe/email/doctype/notification/test_records.json b/frappe/email/doctype/notification/test_records.json index 865b2ac021..665f800c0f 100644 --- a/frappe/email/doctype/notification/test_records.json +++ b/frappe/email/doctype/notification/test_records.json @@ -8,7 +8,7 @@ "message": "New comment {{ doc.content }} created", "condition": "doc.communication_type=='Comment'", "recipients": [ - { "email_by_document_field": "owner" } + { "receiver_by_document_field": "owner" } ] }, { @@ -20,7 +20,7 @@ "message": "New comment {{ doc.content }} saved", "condition": "doc.communication_type=='Comment'", "recipients": [ - { "email_by_document_field": "owner" } + { "receiver_by_document_field": "owner" } ], "set_property_after_alert": "subject", "property_value": "__testing__" @@ -34,7 +34,7 @@ "condition": "doc.event_type=='Public'", "message": "A new public event {{ doc.subject }} on {{ doc.starts_on }} is created", "recipients": [ - { "email_by_document_field": "owner" } + { "receiver_by_document_field": "owner" } ] }, { @@ -46,7 +46,7 @@ "value_changed": "description", "message": "Description changed", "recipients": [ - { "email_by_document_field": "owner" } + { "receiver_by_document_field": "owner" } ] }, { @@ -59,7 +59,7 @@ "days_in_advance": 2, "message": "Description changed", "recipients": [ - { "email_by_document_field": "owner" } + { "receiver_by_document_field": "owner" } ] }, { @@ -70,7 +70,7 @@ "attach_print": 0, "message": "New user {{ doc.name }} created", "recipients": [ - { "email_by_document_field": "owner", "cc": "{{ doc.email }}" } + { "receiver_by_document_field": "owner", "cc": "{{ doc.email }}" } ] } ] diff --git a/frappe/email/doctype/notification_recipient/notification_recipient.json b/frappe/email/doctype/notification_recipient/notification_recipient.json index ec35dccc63..201899cd57 100644 --- a/frappe/email/doctype/notification_recipient/notification_recipient.json +++ b/frappe/email/doctype/notification_recipient/notification_recipient.json @@ -1,204 +1,60 @@ { - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 0, - "creation": "2014-07-11 17:19:37.037109", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", + "actions": [], + "creation": "2014-07-11 17:19:37.037109", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "receiver_by_document_field", + "receiver_by_role", + "cc", + "bcc", + "condition" + ], "fields": [ { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "description": "", - "fieldname": "email_by_document_field", - "fieldtype": "Select", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Email By Document Field", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "depends_on": "eval:parent.channel=='Email'", + "description": "Optional: Always send to these ids. Each Email Address on a new row", + "fieldname": "cc", + "fieldtype": "Code", + "label": "CC" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "email_by_role", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Email By Role", - "length": 0, - "no_copy": 0, - "options": "Role", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "depends_on": "eval:parent.channel=='Email'", + "fieldname": "bcc", + "fieldtype": "Code", + "label": "BCC" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "description": "Optional: Always send to these ids. Each Email Address on a new row", - "fieldname": "cc", - "fieldtype": "Code", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "CC", - "length": 0, - "no_copy": 0, - "options": "", - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "description": "Expression, Optional", + "fieldname": "condition", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Condition" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "bcc", - "fieldtype": "Code", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "BCC", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "receiver_by_document_field", + "fieldtype": "Select", + "in_list_view": 1, + "label": "Receiver By Document Field" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "description": "Expression, Optional", - "fieldname": "condition", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Condition", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "fieldname": "receiver_by_role", + "fieldtype": "Link", + "in_list_view": 1, + "label": "Receiver By Role", + "options": "Role" } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 1, - "max_attachments": 0, - "modified": "2018-09-03 18:37:57.043251", - "modified_by": "Administrator", - "module": "Email", - "name": "Notification Recipient", - "name_case": "", - "owner": "Administrator", - "permissions": [], - "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 0, - "track_seen": 0, - "track_views": 0 + ], + "istable": 1, + "links": [], + "modified": "2020-02-21 11:18:40.125233", + "modified_by": "Administrator", + "module": "Email", + "name": "Notification Recipient", + "owner": "Administrator", + "permissions": [], + "quick_entry": 1, + "sort_field": "modified", + "sort_order": "DESC" } \ No newline at end of file diff --git a/frappe/event_streaming/doctype/event_consumer/event_consumer.py b/frappe/event_streaming/doctype/event_consumer/event_consumer.py index bd285e5401..a53d046be5 100644 --- a/frappe/event_streaming/doctype/event_consumer/event_consumer.py +++ b/frappe/event_streaming/doctype/event_consumer/event_consumer.py @@ -25,6 +25,11 @@ class EventConsumer(Document): else: frappe.db.set_value(self.doctype, self.name, 'incoming_change', 0) + frappe.cache().delete_value('event_consumer_document_type_map') + + def on_trash(self): + frappe.cache().delete_value('event_consumer_document_type_map') + def update_consumer_status(self): consumer_site = get_consumer_site(self.callback_url) event_producer = consumer_site.get_doc('Event Producer', get_url()) diff --git a/frappe/event_streaming/doctype/event_consumer_document_type/event_consumer_document_type.json b/frappe/event_streaming/doctype/event_consumer_document_type/event_consumer_document_type.json index 7463a518ba..71dcc63127 100644 --- a/frappe/event_streaming/doctype/event_consumer_document_type/event_consumer_document_type.json +++ b/frappe/event_streaming/doctype/event_consumer_document_type/event_consumer_document_type.json @@ -1,14 +1,17 @@ { + "actions": [], "creation": "2019-10-03 21:10:54.754651", "doctype": "DocType", "editable_grid": 1, "engine": "InnoDB", "field_order": [ "ref_doctype", - "status" + "status", + "unsubscribed" ], "fields": [ { + "columns": 4, "fieldname": "ref_doctype", "fieldtype": "Link", "in_list_view": 1, @@ -18,16 +21,27 @@ "reqd": 1 }, { + "columns": 4, "default": "Pending", "fieldname": "status", "fieldtype": "Select", "in_list_view": 1, "label": "Approval Status", "options": "Pending\nApproved\nRejected" + }, + { + "columns": 2, + "default": "0", + "fieldname": "unsubscribed", + "fieldtype": "Check", + "in_list_view": 1, + "label": "Unsubscribed", + "read_only": 1 } ], "istable": 1, - "modified": "2019-10-29 15:26:32.436528", + "links": [], + "modified": "2020-08-14 12:38:40.918620", "modified_by": "Administrator", "module": "Event Streaming", "name": "Event Consumer Document Type", diff --git a/frappe/event_streaming/doctype/event_producer/event_producer.py b/frappe/event_streaming/doctype/event_producer/event_producer.py index d4aa0914d8..73aea114ab 100644 --- a/frappe/event_streaming/doctype/event_producer/event_producer.py +++ b/frappe/event_streaming/doctype/event_producer/event_producer.py @@ -107,7 +107,8 @@ class EventProducer(Document): event_consumer.consumer_doctypes.append({ 'ref_doctype': ref_doctype, - 'status': get_approval_status(config, ref_doctype) + 'status': get_approval_status(config, ref_doctype), + 'unsubscribed': entry.unsubscribe }) if frappe.flags.in_test: event_consumer.in_test = True @@ -213,11 +214,12 @@ def sync(update, producer_site, event_producer, in_retry=False): except Exception: if in_retry: + if frappe.flags.in_test: + print(frappe.get_traceback()) return 'Failed' log_event_sync(update, event_producer.name, 'Failed', frappe.get_traceback()) - frappe.db.set_value('Event Producer', event_producer.name, 'last_update', update.creation) - event_producer.reload() + event_producer.db_set('last_update', update.creation) frappe.db.commit() diff --git a/frappe/event_streaming/doctype/event_producer/test_event_producer.py b/frappe/event_streaming/doctype/event_producer/test_event_producer.py index 0d414f5d63..4fea55eb39 100644 --- a/frappe/event_streaming/doctype/event_producer/test_event_producer.py +++ b/frappe/event_streaming/doctype/event_producer/test_event_producer.py @@ -12,9 +12,20 @@ from frappe.event_streaming.doctype.event_producer.event_producer import pull_fr producer_url = 'http://test_site_producer:8000' class TestEventProducer(unittest.TestCase): + # @classmethod + # def setUpClass(cls): + # frappe.print_sql(True) + + # @classmethod + # def tearDownClass(cls): + # frappe.print_sql(False) + def setUp(self): create_event_producer(producer_url) + def tearDown(self): + unsubscribe_doctypes(producer_url) + def test_insert(self): producer = get_remote_site() producer_doc = insert_into_producer(producer, 'test creation 1 sync') @@ -98,7 +109,7 @@ class TestEventProducer(unittest.TestCase): def test_dynamic_link_dependencies_synced(self): producer = get_remote_site() #unsubscribe for Note to check whether dependency is fulfilled - event_producer = frappe.get_doc('Event Producer', producer_url) + event_producer = frappe.get_doc('Event Producer', producer_url, for_update=True) event_producer.producer_doctypes = [] event_producer.append('producer_doctypes', { 'ref_doctype': 'ToDo', @@ -126,7 +137,7 @@ class TestEventProducer(unittest.TestCase): def test_naming_configuration(self): #test with use_same_name = 0 producer = get_remote_site() - event_producer = frappe.get_doc('Event Producer', producer_url) + event_producer = frappe.get_doc('Event Producer', producer_url, for_update=True) event_producer.producer_doctypes = [] event_producer.append('producer_doctypes', { 'ref_doctype': 'ToDo', @@ -167,7 +178,7 @@ class TestEventProducer(unittest.TestCase): def test_mapping(self): producer = get_remote_site() - event_producer = frappe.get_doc('Event Producer', producer_url) + event_producer = frappe.get_doc('Event Producer', producer_url, for_update=True) event_producer.producer_doctypes = [] mapping = [{ 'local_fieldname': 'description', @@ -205,36 +216,8 @@ class TestEventProducer(unittest.TestCase): def test_inner_mapping(self): producer = get_remote_site() - event_producer = frappe.get_doc('Event Producer', producer_url) - event_producer.producer_doctypes = [] - inner_mapping = [ - { - 'local_fieldname':'role_name', - 'remote_fieldname':'title' - } - ] - inner_map = get_mapping('Role to Note Dependency Creation', 'Role', 'Note', inner_mapping) - mapping = [ - { - 'local_fieldname':'description', - 'remote_fieldname':'content', - }, - { - 'local_fieldname': 'role', - 'remote_fieldname': 'title', - 'mapping_type': 'Document', - 'mapping': inner_map, - 'remote_value_filters': json.dumps({'title': 'title'}) - } - ] - event_producer.append('producer_doctypes', { - 'ref_doctype': 'ToDo', - 'use_same_name': 1, - 'has_mapping': 1, - 'mapping': get_mapping('ToDo to Note Mapping', 'ToDo', 'Note', mapping) - }) - event_producer.save() + setup_event_producer_for_inner_mapping() producer_note = frappe._dict(doctype='Note', title='Inner Mapping Tester', content='Test Inner Mapping') delete_on_remote_if_exists(producer, 'Note', {'title': producer_note.title}) producer_note = producer.insert(producer_note) @@ -248,6 +231,39 @@ class TestEventProducer(unittest.TestCase): reset_configuration(producer_url) +def setup_event_producer_for_inner_mapping(): + event_producer = frappe.get_doc('Event Producer', producer_url, for_update=True) + event_producer.producer_doctypes = [] + inner_mapping = [ + { + 'local_fieldname':'role_name', + 'remote_fieldname':'title' + } + ] + inner_map = get_mapping('Role to Note Dependency Creation', 'Role', 'Note', inner_mapping) + mapping = [ + { + 'local_fieldname':'description', + 'remote_fieldname':'content', + }, + { + 'local_fieldname': 'role', + 'remote_fieldname': 'title', + 'mapping_type': 'Document', + 'mapping': inner_map, + 'remote_value_filters': json.dumps({'title': 'title'}) + } + ] + event_producer.append('producer_doctypes', { + 'ref_doctype': 'ToDo', + 'use_same_name': 1, + 'has_mapping': 1, + 'mapping': get_mapping('ToDo to Note Mapping', 'ToDo', 'Note', mapping) + }) + event_producer.save() + return event_producer + + def insert_into_producer(producer, description): #create and insert todo on remote site todo = dict(doctype='ToDo', description=description, assigned_by='Administrator') @@ -276,7 +292,12 @@ def get_mapping(mapping_name, local, remote, field_map): def create_event_producer(producer_url): if frappe.db.exists('Event Producer', producer_url): + event_producer = frappe.get_doc('Event Producer', producer_url) + for entry in event_producer.producer_doctypes: + entry.unsubscribe = 0 + event_producer.save() return + event_producer = frappe.new_doc('Event Producer') event_producer.producer_doctypes = [] event_producer.producer_url = producer_url @@ -292,7 +313,7 @@ def create_event_producer(producer_url): event_producer.save() def reset_configuration(producer_url): - event_producer = frappe.get_doc('Event Producer', producer_url) + event_producer = frappe.get_doc('Event Producer', producer_url, for_update=True) event_producer.producer_doctypes = [] event_producer.producer_url = producer_url event_producer.append('producer_doctypes', { @@ -315,3 +336,9 @@ def get_remote_site(): frappe_authorization_source='Event Consumer' ) return producer_site + +def unsubscribe_doctypes(producer_url): + event_producer = frappe.get_doc('Event Producer', producer_url) + for entry in event_producer.producer_doctypes: + entry.unsubscribe = 1 + event_producer.save() \ No newline at end of file diff --git a/frappe/event_streaming/doctype/event_producer_document_type/event_producer_document_type.json b/frappe/event_streaming/doctype/event_producer_document_type/event_producer_document_type.json index f473879600..e5fe9497f8 100644 --- a/frappe/event_streaming/doctype/event_producer_document_type/event_producer_document_type.json +++ b/frappe/event_streaming/doctype/event_producer_document_type/event_producer_document_type.json @@ -8,11 +8,13 @@ "ref_doctype", "status", "use_same_name", + "unsubscribe", "has_mapping", "mapping" ], "fields": [ { + "columns": 3, "fieldname": "ref_doctype", "fieldtype": "Link", "in_list_view": 1, @@ -36,6 +38,7 @@ "options": "Document Type Mapping" }, { + "columns": 2, "default": "0", "description": "If this is checked the documents will have the same name as they have on the Event Producer's site", "fieldname": "use_same_name", @@ -44,6 +47,7 @@ "label": "Use Same Name" }, { + "columns": 3, "default": "Pending", "fieldname": "status", "fieldtype": "Select", @@ -51,11 +55,19 @@ "label": "Approval Status", "options": "Pending\nApproved\nRejected", "read_only": 1 + }, + { + "columns": 2, + "default": "0", + "fieldname": "unsubscribe", + "fieldtype": "Check", + "in_list_view": 1, + "label": "Unsubscribe" } ], "istable": 1, "links": [], - "modified": "2019-12-30 11:42:36.032351", + "modified": "2020-08-14 11:38:01.278996", "modified_by": "Administrator", "module": "Event Streaming", "name": "Event Producer Document Type", diff --git a/frappe/event_streaming/doctype/event_update_log/event_update_log.py b/frappe/event_streaming/doctype/event_update_log/event_update_log.py index 21e3b303e8..646331a02c 100644 --- a/frappe/event_streaming/doctype/event_update_log/event_update_log.py +++ b/frappe/event_streaming/doctype/event_update_log/event_update_log.py @@ -9,17 +9,40 @@ from frappe.utils.background_jobs import get_jobs from frappe.model import no_value_fields, table_fields class EventUpdateLog(Document): - pass + def after_insert(self): + """Send update notification updates to event consumers + whenever update log is generated""" + enqueued_method = 'frappe.event_streaming.doctype.event_consumer.event_consumer.notify_event_consumers' + jobs = get_jobs() + if not jobs or enqueued_method not in jobs[frappe.local.site]: + frappe.enqueue(enqueued_method, doctype=self.ref_doctype, queue='long', + enqueue_after_commit=True) +def notify_consumers(doc, event): + '''called via hooks''' + # make event update log for doctypes having event consumers + if frappe.flags.in_install or frappe.flags.in_migrate: + return -def notify_consumers(doc, _method=None): - """Send update notification updates to event consumers - whenever update log is generated""" - enqueued_method = 'frappe.event_streaming.doctype.event_consumer.event_consumer.notify_event_consumers' - jobs = get_jobs() - if not jobs or enqueued_method not in jobs[frappe.local.site]: - frappe.enqueue(enqueued_method, doctype=doc.ref_doctype, queue='long', enqueue_after_commit=True) + consumers = check_doctype_has_consumers(doc.doctype) + if consumers: + if event=='after_insert': + doc.flags.event_update_log = make_event_update_log(doc, update_type='Create') + elif event=='on_trash': + make_event_update_log(doc, update_type='Delete') + else: + # on_update + # called after saving + if not doc.flags.event_update_log: # if not already inserted + diff = get_update(doc.get_doc_before_save(), doc) + if diff: + doc.diff = diff + make_event_update_log(doc, update_type='Update') +def check_doctype_has_consumers(doctype): + """Check if doctype has event consumers for event streaming""" + return frappe.cache_manager.get_doctype_map('Event Consumer Document Type', doctype, + dict(ref_doctype=doctype, status='Approved', unsubscribed=0)) def get_update(old, new, for_child=False): """ @@ -60,6 +83,20 @@ def get_update(old, new, for_child=False): return out return None +def make_event_update_log(doc, update_type): + """Save update info for doctypes that have event consumers""" + if update_type != 'Delete': + # diff for update type, doc for create type + data = frappe.as_json(doc) if not doc.get('diff') else frappe.as_json(doc.diff) + else: + data = None + return frappe.get_doc({ + 'doctype': 'Event Update Log', + 'update_type': update_type, + 'ref_doctype': doc.doctype, + 'docname': doc.name, + 'data': data + }).insert(ignore_permissions=True) def make_maps(old_value, new_value): """make maps""" diff --git a/frappe/exceptions.py b/frappe/exceptions.py index 8ebda9c7b8..88428b875c 100644 --- a/frappe/exceptions.py +++ b/frappe/exceptions.py @@ -104,6 +104,7 @@ class IncompatibleApp(ValidationError): pass class InvalidDates(ValidationError): pass class DataTooLongException(ValidationError): pass class FileAlreadyAttachedException(Exception): pass +class DocumentAlreadyRestored(Exception): pass # OAuth exceptions class InvalidAuthorizationHeader(CSRFTokenError): pass class InvalidAuthorizationPrefix(CSRFTokenError): pass diff --git a/frappe/geo/country_info.json b/frappe/geo/country_info.json index 1bf5ada467..6a56107333 100644 --- a/frappe/geo/country_info.json +++ b/frappe/geo/country_info.json @@ -620,7 +620,12 @@ }, "Congo, The Democratic Republic of the": { "code": "cd", - "number_format": "#,###.##" + "number_format": "#,###.##", + "currency": "CDF", + "currency_name": "Congolese franc", + "currency_symbol": "FC", + "currency_fraction": "Centime", + "currency_fraction_units": 100 }, "Cook Islands": { "code": "ck", diff --git a/frappe/hooks.py b/frappe/hooks.py index 1f209f00a2..6277998b04 100644 --- a/frappe/hooks.py +++ b/frappe/hooks.py @@ -127,12 +127,16 @@ standard_queries = { doc_events = { "*": { + "after_insert": [ + "frappe.event_streaming.doctype.event_update_log.event_update_log.notify_consumers" + ], "on_update": [ "frappe.desk.notifications.clear_doctype_notifications", "frappe.core.doctype.activity_log.feed.update_feed", "frappe.workflow.doctype.workflow_action.workflow_action.process_workflow_actions", "frappe.automation.doctype.assignment_rule.assignment_rule.apply", - "frappe.automation.doctype.milestone_tracker.milestone_tracker.evaluate_milestone" + "frappe.automation.doctype.milestone_tracker.milestone_tracker.evaluate_milestone", + "frappe.event_streaming.doctype.event_update_log.event_update_log.notify_consumers" ], "after_rename": "frappe.desk.notifications.clear_doctype_notifications", "on_cancel": [ @@ -141,7 +145,8 @@ doc_events = { ], "on_trash": [ "frappe.desk.notifications.clear_doctype_notifications", - "frappe.workflow.doctype.workflow_action.workflow_action.process_workflow_actions" + "frappe.workflow.doctype.workflow_action.workflow_action.process_workflow_actions", + "frappe.event_streaming.doctype.event_update_log.event_update_log.notify_consumers" ], "on_change": [ "frappe.social.doctype.energy_point_rule.energy_point_rule.process_energy_points" @@ -163,9 +168,6 @@ doc_events = { "Page": { "after_insert": "frappe.cache_manager.build_domain_restriced_page_cache", "after_save": "frappe.cache_manager.build_domain_restriced_page_cache", - }, - "Event Update Log": { - "after_insert": "frappe.event_streaming.doctype.event_update_log.event_update_log.notify_consumers" } } @@ -272,9 +274,6 @@ setup_wizard_exception = [ ] before_migrate = ['frappe.patches.v11_0.sync_user_permission_doctype_before_migrate.execute'] -after_migrate = [ - 'frappe.modules.full_text_search.build_index_for_all_routes' -] otp_methods = ['OTP App','Email','SMS'] user_privacy_documents = [ diff --git a/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.json b/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.json index 830afbae53..123bb21e88 100755 --- a/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.json +++ b/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.json @@ -74,11 +74,11 @@ }, { "default": "us-east-1", - "description": "See https://docs.aws.amazon.com/de_de/general/latest/gr/rande.html#s3_region for details.", + "description": "See https://docs.aws.amazon.com/general/latest/gr/s3.html for details.", "fieldname": "region", "fieldtype": "Select", "label": "Region", - "options": "us-east-1\nus-east-2\nus-west-1\nus-west-2\nap-south-1\nap-southeast-1\nap-southeast-2\nap-northeast-1\nap-northeast-2\nap-northeast-3\nca-central-1\ncn-north-1\ncn-northwest-1\neu-central-1\neu-west-1\neu-west-2\neu-west-3\neu-north-1\nsa-east-1" + "options": "us-east-1\nus-east-2\nus-west-1\nus-west-2\naf-south-1\nap-east-1\nap-south-1\nap-southeast-1\nap-southeast-2\nap-northeast-1\nap-northeast-2\nap-northeast-3\nca-central-1\ncn-north-1\ncn-northwest-1\neu-central-1\neu-west-1\neu-west-2\neu-west-3\neu-south-1\neu-north-1\nme-south-1\nsa-east-1" }, { "fieldname": "endpoint_url", @@ -151,7 +151,7 @@ "hide_toolbar": 1, "issingle": 1, "links": [], - "modified": "2020-04-13 20:57:24.432183", + "modified": "2020-07-27 17:27:21.400000", "modified_by": "Administrator", "module": "Integrations", "name": "S3 Backup Settings", @@ -172,4 +172,4 @@ "sort_field": "modified", "sort_order": "DESC", "track_changes": 1 -} \ No newline at end of file +} diff --git a/frappe/integrations/doctype/twilio_number_group/__init__.py b/frappe/integrations/doctype/twilio_number_group/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/frappe/integrations/doctype/twilio_number_group/twilio_number_group.json b/frappe/integrations/doctype/twilio_number_group/twilio_number_group.json new file mode 100644 index 0000000000..1790581ca7 --- /dev/null +++ b/frappe/integrations/doctype/twilio_number_group/twilio_number_group.json @@ -0,0 +1,32 @@ +{ + "actions": [], + "autoname": "field:phone_number", + "creation": "2020-02-24 13:58:58.036914", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "phone_number" + ], + "fields": [ + { + "fieldname": "phone_number", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Phone Number", + "unique": 1 + } + ], + "istable": 1, + "links": [], + "modified": "2020-03-02 14:54:34.396254", + "modified_by": "Administrator", + "module": "Integrations", + "name": "Twilio Number Group", + "owner": "Administrator", + "permissions": [], + "quick_entry": 1, + "sort_field": "modified", + "sort_order": "DESC", + "track_changes": 1 +} \ No newline at end of file diff --git a/frappe/integrations/doctype/twilio_number_group/twilio_number_group.py b/frappe/integrations/doctype/twilio_number_group/twilio_number_group.py new file mode 100644 index 0000000000..04cb9ae146 --- /dev/null +++ b/frappe/integrations/doctype/twilio_number_group/twilio_number_group.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020, Frappe Technologies and contributors +# For license information, please see license.txt + +from __future__ import unicode_literals +# import frappe +from frappe.model.document import Document + +class TwilioNumberGroup(Document): + pass diff --git a/frappe/integrations/doctype/twilio_settings/__init__.py b/frappe/integrations/doctype/twilio_settings/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/frappe/integrations/doctype/twilio_settings/test_twilio_settings.py b/frappe/integrations/doctype/twilio_settings/test_twilio_settings.py new file mode 100644 index 0000000000..bcb1368d68 --- /dev/null +++ b/frappe/integrations/doctype/twilio_settings/test_twilio_settings.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020, Frappe Technologies and Contributors +# See license.txt +from __future__ import unicode_literals + +# import frappe +import unittest + +class TestTwilioSettings(unittest.TestCase): + pass diff --git a/frappe/integrations/doctype/twilio_settings/twilio_settings.js b/frappe/integrations/doctype/twilio_settings/twilio_settings.js new file mode 100644 index 0000000000..59ebcf2e7d --- /dev/null +++ b/frappe/integrations/doctype/twilio_settings/twilio_settings.js @@ -0,0 +1,8 @@ +// Copyright (c) 2020, Frappe Technologies and contributors +// For license information, please see license.txt + +frappe.ui.form.on('Twilio Settings', { + refresh: function(frm) { + frm.dashboard.set_headline(__("For more information, {0}.", [`${__('Click here')}`])); + } +}); diff --git a/frappe/integrations/doctype/twilio_settings/twilio_settings.json b/frappe/integrations/doctype/twilio_settings/twilio_settings.json new file mode 100644 index 0000000000..e54500fd5d --- /dev/null +++ b/frappe/integrations/doctype/twilio_settings/twilio_settings.json @@ -0,0 +1,57 @@ +{ + "actions": [], + "creation": "2020-01-28 15:21:44.457163", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "account_sid", + "auth_token", + "column_break_2", + "twilio_number" + ], + "fields": [ + { + "fieldname": "account_sid", + "fieldtype": "Data", + "label": "Account SID" + }, + { + "fieldname": "auth_token", + "fieldtype": "Password", + "label": "Auth Token" + }, + { + "fieldname": "column_break_2", + "fieldtype": "Column Break" + }, + { + "fieldname": "twilio_number", + "fieldtype": "Table", + "label": "Twilio Number", + "options": "Twilio Number Group" + } + ], + "issingle": 1, + "links": [], + "modified": "2020-08-11 15:28:57.860554", + "modified_by": "Administrator", + "module": "Integrations", + "name": "Twilio Settings", + "owner": "Administrator", + "permissions": [ + { + "create": 1, + "delete": 1, + "email": 1, + "print": 1, + "read": 1, + "role": "System Manager", + "share": 1, + "write": 1 + } + ], + "sort_field": "modified", + "sort_order": "DESC", + "track_changes": 1 +} \ No newline at end of file diff --git a/frappe/integrations/doctype/twilio_settings/twilio_settings.py b/frappe/integrations/doctype/twilio_settings/twilio_settings.py new file mode 100644 index 0000000000..ba0565b3af --- /dev/null +++ b/frappe/integrations/doctype/twilio_settings/twilio_settings.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020, Frappe Technologies and contributors +# For license information, please see license.txt + +from __future__ import unicode_literals +import frappe +from frappe.model.document import Document +from twilio.rest import Client +from frappe import _ +from frappe.utils.password import get_decrypted_password +from six import string_types + +class TwilioSettings(Document): + pass + +def send_whatsapp_message(sender, receiver_list, message): + import json + if isinstance(receiver_list, string_types): + receiver_list = json.loads(receiver_list) + if not isinstance(receiver_list, list): + receiver_list = [receiver_list] + + + twilio_settings = frappe.get_doc("Twilio Settings") + auth_token = get_decrypted_password("Twilio Settings", "Twilio Settings", 'auth_token') + client = Client(twilio_settings.account_sid, auth_token) + args = { + "from_": 'whatsapp:+{}'.format(sender), + "body": message + } + + failed_delivery = [] + + for rec in receiver_list: + args.update({"to": 'whatsapp:{}'.format(rec)}) + resp = _send_whatsapp(args, client) + if not resp or resp.error_message: + failed_delivery.append(rec) + + if failed_delivery: + frappe.log_error(_("The message wasn't correctly delivered to: {}".format(", ".join(failed_delivery))), _('Delivery Failed')) + + +def _send_whatsapp(message_dict, client): + response = frappe._dict() + try: + response = client.messages.create(**message_dict) + except Exception as e: + frappe.log_error(e, title = _('Twilio WhatsApp Message Error')) + + return response \ No newline at end of file diff --git a/frappe/integrations/offsite_backup_utils.py b/frappe/integrations/offsite_backup_utils.py index 9de176b2d0..a551d8edf1 100644 --- a/frappe/integrations/offsite_backup_utils.py +++ b/frappe/integrations/offsite_backup_utils.py @@ -12,8 +12,10 @@ from frappe.utils import split_emails, get_backups_path def send_email(success, service_name, doctype, email_field, error_status=None): recipients = get_recipients(doctype, email_field) if not recipients: - frappe.log_error("No Email Recipient found for {0}".format(service_name), - "{0}: Failed to send backup status email".format(service_name)) + frappe.log_error( + "No Email Recipient found for {0}".format(service_name), + "{0}: Failed to send backup status email".format(service_name), + ) return if success: @@ -23,7 +25,9 @@ def send_email(success, service_name, doctype, email_field, error_status=None): subject = "Backup Upload Successful" message = """

    Backup Uploaded Successfully!

    -

    Hi there, this is just to inform you that your backup was successfully uploaded to your {0} bucket. So relax!

    """.format(service_name) +

    Hi there, this is just to inform you that your backup was successfully uploaded to your {0} bucket. So relax!

    """.format( + service_name + ) else: subject = "[Warning] Backup Upload Failed" @@ -31,7 +35,9 @@ def send_email(success, service_name, doctype, email_field, error_status=None):

    Backup Upload Failed!

    Oops, your automated backup to {0} failed.

    Error message: {1}

    -

    Please contact your system manager for more information.

    """.format(service_name, error_status) +

    Please contact your system manager for more information.

    """.format( + service_name, error_status + ) frappe.sendmail(recipients=recipients, subject=subject, message=message) @@ -44,29 +50,31 @@ def get_recipients(doctype, email_field): def get_latest_backup_file(with_files=False): + from frappe.utils.backups import BackupGenerator - def get_latest(file_ext): - file_list = glob.glob(os.path.join(get_backups_path(), file_ext)) - return max(file_list, key=os.path.getctime) if file_list else None - - latest_file = get_latest('*.sql.gz') - latest_site_config = get_latest('*.json') + odb = BackupGenerator( + frappe.conf.db_name, + frappe.conf.db_name, + frappe.conf.db_password, + db_host=frappe.db.host, + db_type=frappe.conf.db_type, + db_port=frappe.conf.db_port, + ) + database, public, private, config = odb.get_recent_backup(older_than=24 * 30) if with_files: - latest_public_file_bak = get_latest('*-files.tar') - latest_private_file_bak = get_latest('*-private-files.tar') - return latest_file, latest_site_config, latest_public_file_bak, latest_private_file_bak + return database, config, public, private - return latest_file, latest_site_config + return database, config def get_file_size(file_path, unit): if not unit: - unit = 'MB' + unit = "MB" file_size = os.path.getsize(file_path) - memory_size_unit_mapper = {'KB': 1, 'MB': 2, 'GB': 3, 'TB': 4} + memory_size_unit_mapper = {"KB": 1, "MB": 2, "GB": 3, "TB": 4} i = 0 while i < memory_size_unit_mapper[unit]: file_size = file_size / 1000.0 @@ -78,7 +86,7 @@ def get_file_size(file_path, unit): def validate_file_size(): frappe.flags.create_new_backup = True latest_file, site_config = get_latest_backup_file() - file_size = get_file_size(latest_file, unit='GB') + file_size = get_file_size(latest_file, unit="GB") if file_size > 1: frappe.flags.create_new_backup = False diff --git a/frappe/migrate.py b/frappe/migrate.py index 9ec23d8ae7..6d64799fdd 100644 --- a/frappe/migrate.py +++ b/frappe/migrate.py @@ -19,10 +19,10 @@ from frappe.website import render from frappe.core.doctype.language.language import sync_languages from frappe.modules.utils import sync_customizations from frappe.core.doctype.scheduled_job_type.scheduled_job_type import sync_jobs -from frappe.utils import global_search +from frappe.search.website_search import build_index_for_all_routes -def migrate(verbose=True, rebuild_website=False, skip_failing=False): +def migrate(verbose=True, rebuild_website=False, skip_failing=False, skip_search_index=False): '''Migrate all apps to the latest version, will: - run before migrate hooks - run patches @@ -80,9 +80,6 @@ Otherwise, check the server logs and ensure that all the required services are r # syncs statics render.clear_cache() - # add static pages to global search - global_search.update_global_search_for_all_web_pages() - # updating installed applications data frappe.get_single('Installed Applications').update_versions() @@ -91,6 +88,12 @@ Otherwise, check the server logs and ensure that all the required services are r for fn in frappe.get_hooks('after_migrate', app_name=app): frappe.get_attr(fn)() + # build web_routes index + if not skip_search_index: + # Run this last as it updates the current session + print('Building search index for {}'.format(frappe.local.site)) + build_index_for_all_routes() + frappe.db.commit() clear_notifications() diff --git a/frappe/model/__init__.py b/frappe/model/__init__.py index 3c5d996439..e59d325c9a 100644 --- a/frappe/model/__init__.py +++ b/frappe/model/__init__.py @@ -38,36 +38,104 @@ data_fieldtypes = ( 'Duration' ) -no_value_fields = ('Section Break', 'Column Break', 'HTML', 'Table', 'Table MultiSelect', 'Button', 'Image', - 'Fold', 'Heading') -display_fieldtypes = ('Section Break', 'Column Break', 'HTML', 'Button', 'Image', 'Fold', 'Heading') -numeric_fieldtypes = ('Currency', 'Int', 'Long Int', 'Float', 'Percent', 'Check') -default_fields = ('doctype','name','owner','creation','modified','modified_by', - 'parent','parentfield','parenttype','idx','docstatus') -optional_fields = ("_user_tags", "_comments", "_assign", "_liked_by", "_seen") -table_fields = ('Table', 'Table MultiSelect') -core_doctypes_list = ('DocType', 'DocField', 'DocPerm', 'DocType Action', 'DocType Link', 'User', 'Role', 'Has Role', - 'Page', 'Module Def', 'Print Format', 'Report', 'Customize Form', - 'Customize Form Field', 'Property Setter', 'Custom Field', 'Custom Script') -data_field_options = ('Email', 'Name', 'Phone') +no_value_fields = ( + 'Section Break', + 'Column Break', + 'HTML', + 'Table', + 'Table MultiSelect', + 'Button', + 'Image', + 'Fold', + 'Heading' +) -def copytables(srctype, src, srcfield, tartype, tar, tarfield, srcfields, tarfields=[]): - if not tarfields: - tarfields = srcfields - l = [] - data = src.get(srcfield) - for d in data: - newrow = tar.append(tarfield) - newrow.idx = d.idx +display_fieldtypes = ( + 'Section Break', + 'Column Break', + 'HTML', + 'Button', + 'Image', + 'Fold', + 'Heading') - for i in range(len(srcfields)): - newrow.set(tarfields[i], d.get(srcfields[i])) +numeric_fieldtypes = ( + 'Currency', + 'Int', + 'Long Int', + 'Float', + 'Percent', + 'Check' +) - l.append(newrow) - return l +data_field_options = ( + 'Email', + 'Name', + 'Phone' +) -def db_exists(dt, dn): - return frappe.db.exists(dt, dn) +default_fields = ( + 'doctype', + 'name', + 'owner', + 'creation', + 'modified', + 'modified_by', + 'parent', + 'parentfield', + 'parenttype', + 'idx', + 'docstatus' +) + +optional_fields = ( + "_user_tags", + "_comments", + "_assign", + "_liked_by", + "_seen" +) + +table_fields = ( + 'Table', + 'Table MultiSelect' +) + +core_doctypes_list = ( + 'DocType', + 'DocField', + 'DocPerm', + 'DocType Action', + 'DocType Link', + 'User', + 'Role', + 'Has Role', + 'Page', + 'Module Def', + 'Print Format', + 'Report', + 'Customize Form', + 'Customize Form Field', + 'Property Setter', + 'Custom Field', + 'Custom Script' +) + +log_types = ( + 'Version', + 'Error Log', + 'Scheduled Job Log', + 'Event Sync Log', + 'Event Update Log', + 'Access Log', + 'View Log', + 'Activity Log', + 'Energy Point Log', + 'Notification Log', + 'Email Queue', + 'DocShare', + 'Document Follow' +) def delete_fields(args_dict, delete=0): """ diff --git a/frappe/model/delete_doc.py b/frappe/model/delete_doc.py index 98dbce1d8f..a38470e3f5 100644 --- a/frappe/model/delete_doc.py +++ b/frappe/model/delete_doc.py @@ -18,7 +18,6 @@ from frappe.model.naming import revert_series_if_last from frappe.utils.global_search import delete_for_document from frappe.desk.doctype.tag.tag import delete_tags_for_document from frappe.exceptions import FileNotFoundError -from frappe.model.document import make_event_update_log, check_doctype_has_consumers doctypes_to_skip = ("Communication", "ToDo", "DocShare", "Email Unsubscribe", "Activity Log", "File", "Version", "Document Follow", "Comment" , "View Log", "Tag Link", "Notification Log", "Email Queue") @@ -121,10 +120,6 @@ def delete_doc(doctype=None, name=None, force=0, ignore_doctypes=None, for_reloa # delete tag link entry delete_tags_for_document(doc) - # update log if doctype has event consumers - if not frappe.flags.in_install and not frappe.flags.in_migrate and check_doctype_has_consumers(doc.doctype): - make_event_update_log(doc, update_type='Delete') - if doc and not for_reload: add_to_deleted_document(doc) if not frappe.flags.in_patch: diff --git a/frappe/model/document.py b/frappe/model/document.py index 69a781d6d1..2b171547d1 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -28,6 +28,7 @@ def get_doc(*args, **kwargs): :param arg1: Document dict or DocType name. :param arg2: [optional] document name. + :param for_update: [optional] select document for update. There are multiple ways to call `get_doc` @@ -45,6 +46,9 @@ def get_doc(*args, **kwargs): # create new object with keyword arguments user = get_doc(doctype='User', email_id='test@example.com') + + # select a document for update + user = get_doc("User", "test@example.com", for_update=True) """ if args: if isinstance(args[0], BaseDocument): @@ -60,7 +64,7 @@ def get_doc(*args, **kwargs): else: raise ValueError('First non keyword argument must be a string or dict') - if kwargs: + if len(args) < 2 and kwargs: if 'doctype' in kwargs: doctype = kwargs['doctype'] else: @@ -103,6 +107,9 @@ class Document(BaseDocument): else: self.name = args[1] + if 'for_update' in kwargs: + self.flags.for_update = kwargs.get('for_update') + self.load_from_db() return @@ -144,7 +151,7 @@ class Document(BaseDocument): self._fix_numeric_types() else: - d = frappe.db.get_value(self.doctype, self.name, "*", as_dict=1) + d = frappe.db.get_value(self.doctype, self.name, "*", as_dict=1, for_update=self.flags.for_update) if not d: frappe.throw(_("{0} {1} not found").format(_(self.doctype), self.name), frappe.DoesNotExistError) @@ -233,9 +240,6 @@ class Document(BaseDocument): self.set_docstatus() self.flags.in_insert = False - # follow document on document creation - - # run validate, on update etc. # parent @@ -973,28 +977,13 @@ class Document(BaseDocument): update_global_search(self) - if getattr(self.meta, 'track_changes', False) and not self.flags.ignore_version \ - and not self.doctype == 'Version' and not frappe.flags.in_install: - self.save_version() + self.save_version() self.run_method('on_change') if (self.doctype, self.name) in frappe.flags.currently_saving: frappe.flags.currently_saving.remove((self.doctype, self.name)) - # make event update log for doctypes having event consumers - if not frappe.flags.in_install and not frappe.flags.in_migrate and check_doctype_has_consumers(self.doctype): - if self.flags.update_log_for_doc_creation: - make_event_update_log(self, update_type='Create') - self.flags.update_log_for_doc_creation = False - else: - from frappe.event_streaming.doctype.event_update_log.event_update_log import get_update - diff = get_update(doc_before_save, self) - if diff: - doc = self - doc.diff = diff - make_event_update_log(doc, update_type='Update') - self.latest = None def clear_cache(self): @@ -1071,7 +1060,14 @@ class Document(BaseDocument): def save_version(self): """Save version info""" - if not self._doc_before_save and frappe.flags.in_patch: return + + # don't track version under following conditions + if (not getattr(self.meta, 'track_changes', False) + or self.doctype == 'Version' + or self.flags.ignore_version + or frappe.flags.in_install + or (not self._doc_before_save and frappe.flags.in_patch)): + return version = frappe.new_doc('Version') if not self._doc_before_save: @@ -1080,6 +1076,7 @@ class Document(BaseDocument): elif version.set_diff(self._doc_before_save, self): version.insert(ignore_permissions=True) if not frappe.flags.in_migrate: + # follow since you made a change? follow_document(self.doctype, self.name, frappe.session.user) @staticmethod @@ -1307,6 +1304,16 @@ class Document(BaseDocument): users = set([assignment.owner for assignment in assignments]) return users + def add_tag(self, tag): + """Add a Tag to this document""" + from frappe.desk.doctype.tag.tag import DocTags + DocTags(self.doctype).add(self.name, tag) + + def get_tags(self): + """Return a list of Tags attached to this document""" + from frappe.desk.doctype.tag.tag import DocTags + return DocTags(self.doctype).get_tags(self.name).split(",")[1:] + def execute_action(doctype, name, action, **kwargs): """Execute an action on a document (called by background worker)""" doc = frappe.get_doc(doctype, name) @@ -1326,34 +1333,4 @@ def execute_action(doctype, name, action, **kwargs): doc.notify_update() -def make_event_update_log(doc, update_type): - """Save update info for doctypes that have event consumers""" - if update_type != 'Delete': - # diff for update type, doc for create type - data = frappe.as_json(doc) if not doc.get('diff') else frappe.as_json(doc.diff) - else: - data = None - log_doc = frappe.get_doc({ - 'doctype': 'Event Update Log', - 'update_type': update_type, - 'ref_doctype': doc.doctype, - 'docname': doc.name, - 'data': data - }) - log_doc.insert(ignore_permissions=True) - frappe.db.commit() - -def check_doctype_has_consumers(doctype): - """Check if doctype has event consumers for event streaming""" - if not frappe.db.exists('DocType', 'Event Consumer'): - return False - - event_consumers = frappe.get_all('Event Consumer Document Type', { - 'ref_doctype': doctype, - 'status': 'Approved' - }, limit=1) - - if len(event_consumers) and event_consumers[0]: - return True - return False diff --git a/frappe/model/workflow.py b/frappe/model/workflow.py index ea563dfc13..7239b202bd 100644 --- a/frappe/model/workflow.py +++ b/frappe/model/workflow.py @@ -272,9 +272,11 @@ def get_common_transition_actions(docs, doctype): doc['doctype'] = doctype actions = [t.get('action') for t in get_transitions(doc, raise_exception=True) \ if has_approval_access(frappe.session.user, doc, t)] - if not actions: return [] + if not actions: + return [] common_actions = actions if i == 1 else set(common_actions).intersection(actions) - if not common_actions: return [] + if not common_actions: + return [] except WorkflowStateError: pass @@ -307,4 +309,4 @@ def set_workflow_state_on_action(doc, workflow_name, action): for state in workflow.states: if state.doc_status == docstatus: doc.set(workflow_state_field, state.state) - return \ No newline at end of file + return diff --git a/frappe/modules/full_text_search.py b/frappe/modules/full_text_search.py deleted file mode 100644 index fce9983907..0000000000 --- a/frappe/modules/full_text_search.py +++ /dev/null @@ -1,106 +0,0 @@ -# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors -# MIT License. See license.txt - -from __future__ import unicode_literals -import frappe -from whoosh.index import create_in, open_dir -from whoosh.fields import TEXT, ID, Schema -from whoosh.qparser import MultifieldParser, FieldsPlugin, WildcardPlugin -from whoosh.query import Prefix -from bs4 import BeautifulSoup -from frappe.website.render import render_page -from frappe.utils import set_request, cint -from frappe.utils.global_search import get_routes_to_index - - -def build_index_for_all_routes(): - print("Building search index for all web routes...") - routes = get_routes_to_index() - documents = [get_document_to_index(route) for route in routes] - build_index("web_routes", documents) - - -@frappe.whitelist(allow_guest=True) -def web_search(index_name, query, scope=None, limit=20): - limit = cint(limit) - return search(index_name, query, scope, limit) - - -def get_document_to_index(route): - frappe.set_user("Guest") - frappe.local.no_cache = True - - try: - set_request(method="GET", path=route) - content = render_page(route) - soup = BeautifulSoup(content, "html.parser") - page_content = soup.find(class_="page_content") - text_content = page_content.text if page_content else "" - title = soup.title.text.strip() if soup.title else route - - frappe.set_user("Administrator") - - return frappe._dict(title=title, content=text_content, path=route) - except ( - frappe.PermissionError, - frappe.DoesNotExistError, - frappe.ValidationError, - Exception, - ): - pass - - -def build_index(index_name, documents): - schema = Schema( - title=TEXT(stored=True), path=ID(stored=True), content=TEXT(stored=True) - ) - - index_dir = get_index_path(index_name) - frappe.create_folder(index_dir) - - ix = create_in(index_dir, schema) - writer = ix.writer() - - for document in documents: - if document: - writer.add_document( - title=document.title, path=document.path, content=document.content - ) - - writer.commit() - - -def search(index_name, text, scope=None, limit=20): - index_dir = get_index_path(index_name) - ix = open_dir(index_dir) - - results = None - out = [] - with ix.searcher() as searcher: - parser = MultifieldParser(["title", "content"], ix.schema) - parser.remove_plugin_class(FieldsPlugin) - parser.remove_plugin_class(WildcardPlugin) - query = parser.parse(text) - - filter_scoped = None - if scope: - filter_scoped = Prefix("path", scope) - results = searcher.search(query, limit=limit, filter=filter_scoped) - - for r in results: - title_highlights = r.highlights("title") - content_highlights = r.highlights("content") - out.append( - frappe._dict( - title=r["title"], - path=r["path"], - title_highlights=title_highlights, - content_highlights=content_highlights, - ) - ) - - return out - - -def get_index_path(index_name): - return frappe.get_site_path("indexes", index_name) diff --git a/frappe/modules/import_file.py b/frappe/modules/import_file.py index 27649b8da9..5970eae5ca 100644 --- a/frappe/modules/import_file.py +++ b/frappe/modules/import_file.py @@ -9,7 +9,7 @@ from frappe.utils import get_datetime_str from frappe.model.base_document import get_controller ignore_values = { - "Report": ["disabled", "prepared_report"], + "Report": ["disabled", "prepared_report", "add_total_row"], "Print Format": ["disabled"], "Notification": ["enabled"], "Print Style": ["disabled"], diff --git a/frappe/oauth.py b/frappe/oauth.py index 4dc50366be..122c806072 100644 --- a/frappe/oauth.py +++ b/frappe/oauth.py @@ -4,6 +4,7 @@ import pytz from frappe import _ from frappe.auth import LoginManager +from http import cookies from oauthlib.oauth2.rfc6749.tokens import BearerToken from oauthlib.oauth2.rfc6749.grant_types import AuthorizationCodeGrant, ImplicitGrant, ResourceOwnerPasswordCredentialsGrant, ClientCredentialsGrant, RefreshTokenGrant from oauthlib.oauth2 import RequestValidator @@ -130,15 +131,12 @@ class OAuthWebRequestValidator(RequestValidator): oac.scopes = get_url_delimiter().join(request.scopes) oac.redirect_uri_bound_to_authorization_code = request.redirect_uri oac.client = client_id - oac.user = unquote(cookie_dict['user_id']) + oac.user = unquote(cookie_dict['user_id'].value) oac.authorization_code = code['code'] oac.save(ignore_permissions=True) frappe.db.commit() def authenticate_client(self, request, *args, **kwargs): - - cookie_dict = get_cookie_dict_from_headers(request) - #Get ClientID in URL if request.client_id: oc = frappe.get_doc("OAuth Client", request.client_id) @@ -155,7 +153,9 @@ class OAuthWebRequestValidator(RequestValidator): except Exception as e: print("Failed body authentication: Application %s does not exist".format(cid=request.client_id)) - return frappe.session.user == unquote(cookie_dict.get('user_id', "Guest")) + cookie_dict = get_cookie_dict_from_headers(request) + user_id = unquote(cookie_dict['user_id']) if 'user_id' in cookie_dict else "Guest" + return frappe.session.user == user_id def authenticate_client_id(self, client_id, request, *args, **kwargs): cli_id = frappe.db.get_value('OAuth Client', client_id, 'name') @@ -400,13 +400,10 @@ class OAuthWebRequestValidator(RequestValidator): return True def get_cookie_dict_from_headers(r): + cookie = cookies.BaseCookie() if r.headers.get('Cookie'): - cookie = r.headers.get('Cookie') - cookie = cookie.split("; ") - cookie_dict = {k:v for k,v in (x.split('=') for x in cookie)} - return cookie_dict - else: - return {} + cookie.load(r.headers.get('Cookie')) + return cookie def calculate_at_hash(access_token, hash_alg): """Helper method for calculating an access token diff --git a/frappe/patches.txt b/frappe/patches.txt index b207a325cb..8657be1fc5 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -296,3 +296,8 @@ frappe.patches.v13_0.update_duration_options frappe.patches.v13_0.replace_old_data_import # 2020-06-24 frappe.patches.v13_0.create_custom_dashboards_cards_and_charts frappe.patches.v13_0.rename_is_custom_field_in_dashboard_chart +frappe.patches.v13_0.add_standard_navbar_items +frappe.patches.v13_0.generate_theme_files_in_public_folder +frappe.patches.v13_0.increase_password_length +frappe.patches.v13_0.add_toggle_width_in_navbar_settings +frappe.patches.v13_0.rename_notification_fields diff --git a/frappe/patches/v13_0/add_standard_navbar_items.py b/frappe/patches/v13_0/add_standard_navbar_items.py new file mode 100644 index 0000000000..9982e6e3f5 --- /dev/null +++ b/frappe/patches/v13_0/add_standard_navbar_items.py @@ -0,0 +1,9 @@ +from __future__ import unicode_literals +import frappe +from frappe.utils.install import add_standard_navbar_items + +def execute(): + # Add standard navbar items for ERPNext in Navbar Settings + frappe.reload_doc('core', 'doctype', 'navbar_settings') + frappe.reload_doc('core', 'doctype', 'navbar_item') + add_standard_navbar_items() \ No newline at end of file diff --git a/frappe/patches/v13_0/add_toggle_width_in_navbar_settings.py b/frappe/patches/v13_0/add_toggle_width_in_navbar_settings.py new file mode 100644 index 0000000000..4d241cd8c7 --- /dev/null +++ b/frappe/patches/v13_0/add_toggle_width_in_navbar_settings.py @@ -0,0 +1,18 @@ +from __future__ import unicode_literals +import frappe + +def execute(): + navbar_settings = frappe.get_single("Navbar Settings") + + for navbar_item in navbar_settings.settings_dropdown[5:]: + navbar_item.idx = navbar_item.idx + 1 + + navbar_settings.append('settings_dropdown', { + 'item_label': 'Toggle Full Width', + 'item_type': 'Action', + 'action': 'frappe.ui.toolbar.toggle_full_width()', + 'is_standard': 1, + 'idx': 6 + }) + + navbar_settings.save() \ No newline at end of file diff --git a/frappe/patches/v13_0/generate_theme_files_in_public_folder.py b/frappe/patches/v13_0/generate_theme_files_in_public_folder.py new file mode 100644 index 0000000000..4315f06ebe --- /dev/null +++ b/frappe/patches/v13_0/generate_theme_files_in_public_folder.py @@ -0,0 +1,19 @@ +# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors +# MIT License. See license.txt + +from __future__ import unicode_literals +import frappe + + +def execute(): + themes = frappe.db.get_all( + "Website Theme", filters={"theme_url": ("not like", "/files/website_theme/%")} + ) + for theme in themes: + doc = frappe.get_doc("Website Theme", theme.name) + try: + doc.generate_bootstrap_theme() + doc.save() + except: # noqa: E722 + print('Ignoring....') + print(frappe.get_traceback()) diff --git a/frappe/patches/v13_0/increase_password_length.py b/frappe/patches/v13_0/increase_password_length.py new file mode 100644 index 0000000000..1bb1979051 --- /dev/null +++ b/frappe/patches/v13_0/increase_password_length.py @@ -0,0 +1,7 @@ +import frappe + +def execute(): + frappe.db.multisql({ + "mariadb": "ALTER TABLE `__Auth` MODIFY `password` TEXT NOT NULL", + "postgres": 'ALTER TABLE "__Auth" ALTER COLUMN "password" TYPE TEXT' + }) diff --git a/frappe/patches/v13_0/rename_notification_fields.py b/frappe/patches/v13_0/rename_notification_fields.py new file mode 100644 index 0000000000..2984e6503c --- /dev/null +++ b/frappe/patches/v13_0/rename_notification_fields.py @@ -0,0 +1,16 @@ +# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors +# MIT License. See license.txt + +from __future__ import unicode_literals +import frappe +from frappe.model.utils.rename_field import rename_field + +def execute(): + """ + Change notification recipient fields from email to receiver fields + """ + frappe.reload_doc("Email", "doctype", "Notification Recipient") + frappe.reload_doc("Email", "doctype", "Notification") + + rename_field("Notification Recipient", "email_by_document_field", "receiver_by_document_field") + rename_field("Notification Recipient", "email_by_role", "receiver_by_role") \ No newline at end of file diff --git a/frappe/printing/doctype/print_format/print_format.js b/frappe/printing/doctype/print_format/print_format.js index 252c706e51..e6599b2496 100644 --- a/frappe/printing/doctype/print_format/print_format.js +++ b/frappe/printing/doctype/print_format/print_format.js @@ -35,13 +35,20 @@ frappe.ui.form.on("Print Format", { else if (frm.doc.custom_format && !frm.doc.raw_printing) { frm.set_df_property("html", "reqd", 1); } - frm.add_custom_button(__("Make Default"), function () { - frappe.call({ - method: "frappe.printing.doctype.print_format.print_format.make_default", - args: { - name: frm.doc.name - } - }) + frappe.db.get_value('DocType', frm.doc.doc_type, 'default_print_format', (r) => { + if (r.default_print_format != frm.doc.name) { + frm.add_custom_button(__("Set as Default"), function () { + frappe.call({ + method: "frappe.printing.doctype.print_format.print_format.make_default", + args: { + name: frm.doc.name + }, + callback: function() { + frm.refresh(); + } + }); + }); + } }); } }, diff --git a/frappe/public/js/frappe/data_import/data_exporter.js b/frappe/public/js/frappe/data_import/data_exporter.js index f6af338235..dee4839b34 100644 --- a/frappe/public/js/frappe/data_import/data_exporter.js +++ b/frappe/public/js/frappe/data_import/data_exporter.js @@ -274,23 +274,29 @@ frappe.data_import.DataExporter = class DataExporter { ? this.column_map[child_fieldname] : this.column_map[doctype]; - let is_field_mandatory = df => (df.fieldname === 'name' && !child_fieldname) - || (df.reqd && this.exporting_for == 'Insert New Records'); + let is_field_mandatory = df => { + if (df.reqd && this.exporting_for == 'Insert New Records') { + return true; + } + if (autoname_field && df.fieldname == autoname_field.fieldname) { + return true; + } + if (df.fieldname === 'name') { + return true; + } + return false; + }; return fields .filter(df => { - if (autoname_field && df.fieldname === autoname_field.fieldname) { + if (autoname_field && df.fieldname === 'name') { return false; } return true; }) .map(df => { - let label = __(df.label); - if (autoname_field && df.fieldname === 'name') { - label = label + ` (${__(autoname_field.label)})`; - } return { - label, + label: __(df.label), value: df.fieldname, danger: is_field_mandatory(df), checked: false, diff --git a/frappe/public/js/frappe/data_import/import_preview.js b/frappe/public/js/frappe/data_import/import_preview.js index 4edcb87aeb..6c17cb4351 100644 --- a/frappe/public/js/frappe/data_import/import_preview.js +++ b/frappe/public/js/frappe/data_import/import_preview.js @@ -98,6 +98,9 @@ frappe.data_import.ImportPreview = class ImportPreview { .replace('%y', 'yy') .replace('%m', 'mm') .replace('%d', 'dd') + .replace('%H', 'HH') + .replace('%M', 'mm') + .replace('%S', 'ss') : null; let column_title = ` @@ -354,4 +357,4 @@ function get_fields_as_options(doctype, column_map) { }); }) ); -} \ No newline at end of file +} diff --git a/frappe/public/js/frappe/desk.js b/frappe/public/js/frappe/desk.js index 2e80dbfd85..1b064aa0f8 100644 --- a/frappe/public/js/frappe/desk.js +++ b/frappe/public/js/frappe/desk.js @@ -5,7 +5,7 @@ // __('Modules') __('Domains') __('Places') __('Administration') # for translation, don't remove frappe.start_app = function() { - if(!frappe.Application) + if (!frappe.Application) return; frappe.assets.check(); frappe.provide('frappe.app'); @@ -14,7 +14,7 @@ frappe.start_app = function() { }; $(document).ready(function() { - if(!frappe.utils.supportsES6) { + if (!frappe.utils.supportsES6) { frappe.msgprint({ indicator: 'red', title: __('Browser not supported'), @@ -427,9 +427,9 @@ frappe.Application = Class.extend({ }, set_app_logo_url: function() { - return frappe.call('frappe.client.get_hooks', { hook: 'app_logo_url' }) + return frappe.call('frappe.core.doctype.navbar_settings.navbar_settings.get_app_logo') .then(r => { - frappe.app.logo_url = (r.message || []).slice(-1)[0]; + frappe.app.logo_url = r.message; if (window.cordova) { let host = frappe.request.url; host = host.slice(0, host.length - 1); diff --git a/frappe/public/js/frappe/form/controls/code.js b/frappe/public/js/frappe/form/controls/code.js index 726a83db72..a547cfcf32 100644 --- a/frappe/public/js/frappe/form/controls/code.js +++ b/frappe/public/js/frappe/form/controls/code.js @@ -46,6 +46,8 @@ frappe.ui.form.ControlCode = frappe.ui.form.ControlText.extend({ const language_map = { 'Javascript': 'ace/mode/javascript', 'JS': 'ace/mode/javascript', + 'Python': 'ace/mode/python', + 'Py': 'ace/mode/python', 'HTML': 'ace/mode/html', 'CSS': 'ace/mode/css', 'Markdown': 'ace/mode/markdown', @@ -57,7 +59,7 @@ frappe.ui.form.ControlCode = frappe.ui.form.ControlText.extend({ const language = this.df.options; const valid_languages = Object.keys(language_map); - if (!valid_languages.includes(language)) { + if (language && !valid_languages.includes(language)) { // eslint-disable-next-line console.warn(`Invalid language option provided for field "${this.df.label}". Valid options are ${valid_languages.join(', ')}.`); } diff --git a/frappe/public/js/frappe/form/controls/multiselect_list.js b/frappe/public/js/frappe/form/controls/multiselect_list.js index cd86bdd767..2a7ee5cb10 100644 --- a/frappe/public/js/frappe/form/controls/multiselect_list.js +++ b/frappe/public/js/frappe/form/controls/multiselect_list.js @@ -3,7 +3,7 @@ frappe.ui.form.ControlMultiSelectList = frappe.ui.form.ControlData.extend({ let template = ` -