diff --git a/.github/workflows/semgrep.yml b/.github/workflows/semgrep.yml new file mode 100644 index 0000000000..26f1191a90 --- /dev/null +++ b/.github/workflows/semgrep.yml @@ -0,0 +1,13 @@ +name: Semgrep + +on: + pull_request: {} + +jobs: + semgrep: + name: Frappe Linter + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: returntocorp/semgrep-action@v1 + diff --git a/.semgrep.yml b/.semgrep.yml new file mode 100644 index 0000000000..99d237251e --- /dev/null +++ b/.semgrep.yml @@ -0,0 +1,29 @@ +#Reference: https://semgrep.dev/docs/writing-rules/rule-syntax/ + +rules: +- id: eval + patterns: + - pattern-not: eval("...") + - pattern: eval(...) + message: | + Detected the use of eval(). eval() can be dangerous if used to evaluate + dynamic content. Avoid it or use safe_eval(). + languages: + - python + severity: ERROR + +# translations +- id: frappe-translation-syntax-python + pattern-either: + - pattern: _(f"...") # f-strings not allowed + - pattern: _("..." + "...") # concatenation not allowed + - pattern: _("") # empty string is meaningless + - pattern: _("..." % ...) # Only positional formatters are allowed. + - pattern: _("...".format(...)) # format should not be used before translating + - pattern: _("...") + ... + _("...") # don't split strings + message: | + Incorrect use of translation function detected. + Please refer: https://frappeframework.com/docs/user/en/translations + languages: + - python + severity: ERROR diff --git a/.travis.yml b/.travis.yml index 53ad56a948..ffada0286f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ addons: - test_site_producer mariadb: 10.3 postgresql: 9.5 - chrome: stable + firefox: latest services: - xvfb diff --git a/CODEOWNERS b/CODEOWNERS index 1afa3f72e3..92723ab035 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -8,10 +8,10 @@ website/ @prssanna web_form/ @prssanna templates/ @surajshetty3416 www/ @surajshetty3416 -integrations/ @nextchamp-saqib +integrations/ @leela patches/ @surajshetty3416 dashboard/ @prssanna -email/ @saurabh6790 +email/ @leela event_streaming/ @ruchamahabal data_import* @netchampfaris core/ @surajshetty3416 diff --git a/cypress/integration/control_select.js b/cypress/integration/control_select.js new file mode 100644 index 0000000000..0bc719b4a7 --- /dev/null +++ b/cypress/integration/control_select.js @@ -0,0 +1,36 @@ +context('Control Select', () => { + before(() => { + cy.login(); + cy.visit('/app/website'); + }); + + function get_dialog_with_select() { + return cy.dialog({ + title: 'Select', + fields: [{ + 'fieldname': 'select_control', + 'fieldtype': 'Select', + 'placeholder': 'Select an Option', + 'options': ['', 'Option 1', 'Option 2', 'Option 2'], + }] + }); + } + + it('toggles placholder on clicking an option', () => { + get_dialog_with_select().as('dialog'); + + cy.get('.frappe-control[data-fieldname=select_control] .control-input').as('control'); + cy.get('.frappe-control[data-fieldname=select_control] .control-input select').as('select'); + cy.get('@control').get('.select-icon').should('exist'); + cy.get('@control').get('.placeholder').should('have.css', 'display', 'block'); + cy.get('@select').select('Option 1'); + cy.get('@control').get('.placeholder').should('have.css', 'display', 'none'); + cy.get('@select').invoke('val', ''); + cy.get('@control').get('.placeholder').should('have.css', 'display', 'block'); + + + cy.get('@dialog').then(dialog => { + dialog.hide(); + }); + }); +}); diff --git a/cypress/integration/table_multiselect.js b/cypress/integration/table_multiselect.js index 8b83a0d914..faa72d63a5 100644 --- a/cypress/integration/table_multiselect.js +++ b/cypress/integration/table_multiselect.js @@ -8,7 +8,7 @@ context('Table MultiSelect', () => { it('select value from multiselect dropdown', () => { cy.new_form('Assignment Rule'); cy.fill_field('__newname', name); - cy.fill_field('document_type', 'ToDo'); + cy.fill_field('document_type', 'Blog Post'); cy.fill_field('assign_condition', 'status=="Open"', 'Code'); cy.get('input[data-fieldname="users"]').focus().as('input'); cy.get('input[data-fieldname="users"] + ul').should('be.visible'); diff --git a/frappe/__init__.py b/frappe/__init__.py index bf5f117d81..871d1b9e92 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -17,11 +17,15 @@ from werkzeug.local import Local, release_local import os, sys, importlib, inspect, json from past.builtins import cmp import click -from faker import Faker -# public +# Local application imports from .exceptions import * from .utils.jinja import (get_jenv, get_template, render_template, get_email_from_template, get_jloader) +from .utils.lazy_loader import lazy_import + +# Lazy imports +faker = lazy_import('faker') + # Harmless for Python 3 # For Python 2 set default encoding to utf-8 @@ -196,17 +200,20 @@ def init(site, sites_path=None, new_site=False): local.initialised = True -def connect(site=None, db_name=None): +def connect(site=None, db_name=None, set_admin_as_user=True): """Connect to site database instance. :param site: If site is given, calls `frappe.init`. - :param db_name: Optional. Will use from `site_config.json`.""" + :param db_name: Optional. Will use from `site_config.json`. + :param set_admin_as_user: Set Administrator as current user. + """ from frappe.database import get_db if site: init(site) local.db = get_db(user=db_name or local.conf.db_name) - set_user("Administrator") + if set_admin_as_user: + set_user("Administrator") def connect_replica(): from frappe.database import get_db @@ -468,8 +475,8 @@ def get_request_header(key, default=None): def sendmail(recipients=[], sender="", subject="No Subject", message="No Message", as_markdown=False, delayed=True, reference_doctype=None, reference_name=None, - unsubscribe_method=None, unsubscribe_params=None, unsubscribe_message=None, - attachments=None, content=None, doctype=None, name=None, reply_to=None, + unsubscribe_method=None, unsubscribe_params=None, unsubscribe_message=None, add_unsubscribe_link=1, + attachments=None, content=None, doctype=None, name=None, reply_to=None, queue_separately=False, cc=[], bcc=[], message_id=None, in_reply_to=None, send_after=None, expose_recipients=None, send_priority=1, communication=None, retry=1, now=None, read_receipt=None, is_notification=False, inline_images=None, template=None, args=None, header=None, print_letterhead=False, with_container=False): @@ -516,10 +523,10 @@ def sendmail(recipients=[], sender="", subject="No Subject", message="No Message from frappe.email import queue queue.send(recipients=recipients, sender=sender, subject=subject, message=message, text_content=text_content, - reference_doctype = doctype or reference_doctype, reference_name = name or reference_name, + reference_doctype = doctype or reference_doctype, reference_name = name or reference_name, add_unsubscribe_link=add_unsubscribe_link, unsubscribe_method=unsubscribe_method, unsubscribe_params=unsubscribe_params, unsubscribe_message=unsubscribe_message, attachments=attachments, reply_to=reply_to, cc=cc, bcc=bcc, message_id=message_id, in_reply_to=in_reply_to, - send_after=send_after, expose_recipients=expose_recipients, send_priority=send_priority, + send_after=send_after, expose_recipients=expose_recipients, send_priority=send_priority, queue_separately=queue_separately, communication=communication, now=now, read_receipt=read_receipt, is_notification=is_notification, inline_images=inline_images, header=header, print_letterhead=print_letterhead, with_container=with_container) @@ -1748,12 +1755,12 @@ def parse_json(val): def mock(type, size=1, locale='en'): results = [] - faker = Faker(locale) - if not type in dir(faker): + fake = faker.Faker(locale) + if type not in dir(fake): raise ValueError('Not a valid mock type.') else: for i in range(size): - data = getattr(faker, type)() + data = getattr(fake, type)() results.append(data) from frappe.chat.util import squashify diff --git a/frappe/app.py b/frappe/app.py index 29ef69ef2d..607479ad52 100644 --- a/frappe/app.py +++ b/frappe/app.py @@ -128,6 +128,8 @@ def init_request(request): if frappe.local.conf.get('maintenance_mode'): frappe.connect() raise frappe.SessionStopped('Session Stopped') + else: + frappe.connect(set_admin_as_user=False) make_form_dict(request) @@ -152,10 +154,10 @@ def process_response(response): def set_cors_headers(response): origin = frappe.request.headers.get('Origin') - if not origin: + allow_cors = frappe.conf.allow_cors + if not (origin and allow_cors): return - allow_cors = frappe.conf.allow_cors if allow_cors != "*": if not isinstance(allow_cors, list): allow_cors = [allow_cors] diff --git a/frappe/auth.py b/frappe/auth.py index 2e0ec681d2..946a8c52d5 100644 --- a/frappe/auth.py +++ b/frappe/auth.py @@ -207,23 +207,44 @@ class LoginManager: if frappe.session.user != "Guest": clear_sessions(frappe.session.user, keep_current=True) - def authenticate(self, user=None, pwd=None): + def authenticate(self, user: str = None, pwd: str = None): + from frappe.core.doctype.user.user import User + if not (user and pwd): user, pwd = frappe.form_dict.get('usr'), frappe.form_dict.get('pwd') if not (user and pwd): self.fail(_('Incomplete login details'), user=user) - if cint(frappe.db.get_value("System Settings", "System Settings", "allow_login_using_mobile_number")): - user = frappe.db.get_value("User", filters={"mobile_no": user}, fieldname="name") or user + # Ignore password check if tmp_id is set, 2FA takes care of authentication. + validate_password = not bool(frappe.form_dict.get('tmp_id')) + user = User.find_by_credentials(user, pwd, validate_password=validate_password) - if cint(frappe.db.get_value("System Settings", "System Settings", "allow_login_using_user_name")): - user = frappe.db.get_value("User", filters={"username": user}, fieldname="name") or user + if not user: + self.fail('Invalid login credentials') - self.check_if_enabled(user) - if not frappe.form_dict.get('tmp_id'): - self.user = self.check_password(user, pwd) + sys_settings = frappe.get_doc("System Settings") + track_login_attempts = (sys_settings.allow_consecutive_login_attempts >0) + + tracker_kwargs = {} + if track_login_attempts: + tracker_kwargs['lock_interval'] = sys_settings.allow_login_after_fail + tracker_kwargs['max_consecutive_login_attempts'] = sys_settings.allow_consecutive_login_attempts + + tracker = LoginAttemptTracker(user.name, **tracker_kwargs) + + if track_login_attempts and not tracker.is_user_allowed(): + frappe.throw(_("Your account has been locked and will resume after {0} seconds") + .format(sys_settings.allow_login_after_fail), frappe.SecurityException) + + if not user.is_authenticated: + tracker.add_failure_attempt() + self.fail('Invalid login credentials', user=user.name) + elif not (user.name == 'Administrator' or user.enabled): + tracker.add_failure_attempt() + self.fail('User disabled or missing', user=user.name) else: - self.user = user + tracker.add_success_attempt() + self.user = user.name def force_user_to_reset_password(self): if not self.user: @@ -245,23 +266,12 @@ class LoginManager: if last_pwd_reset_days > reset_pwd_after_days: return True - def check_if_enabled(self, user): - """raise exception if user not enabled""" - doc = frappe.get_doc("System Settings") - if cint(doc.allow_consecutive_login_attempts) > 0: - check_consecutive_login_attempts(user, doc) - - if user=='Administrator': return - if not cint(frappe.db.get_value('User', user, 'enabled')): - self.fail('User disabled or missing', user=user) - def check_password(self, user, pwd): """check password""" try: # returns user in correct case return check_password(user, pwd) except frappe.AuthenticationError: - self.update_invalid_login(user) self.fail('Incorrect password', user=user) def fail(self, message, user=None): @@ -272,15 +282,6 @@ class LoginManager: frappe.db.commit() raise frappe.AuthenticationError - def update_invalid_login(self, user): - last_login_tried = get_last_tried_login_data(user) - - failed_count = 0 - if last_login_tried > get_datetime(): - failed_count = get_login_failed_count(user) - - frappe.cache().hset('login_failed_count', user, failed_count + 1) - def run_trigger(self, event='on_login'): for method in frappe.get_hooks().get(event, []): frappe.call(frappe.get_attr(method), login_manager=self) @@ -383,38 +384,6 @@ def clear_cookies(): frappe.session.sid = "" frappe.local.cookie_manager.delete_cookie(["full_name", "user_id", "sid", "user_image", "system_user"]) -def get_last_tried_login_data(user, get_last_login=False): - locked_account_time = frappe.cache().hget('locked_account_time', user) - if get_last_login and locked_account_time: - return locked_account_time - - last_login_tried = frappe.cache().hget('last_login_tried', user) - if not last_login_tried or last_login_tried < get_datetime(): - last_login_tried = get_datetime() + datetime.timedelta(seconds=60) - - frappe.cache().hset('last_login_tried', user, last_login_tried) - - return last_login_tried - -def get_login_failed_count(user): - return cint(frappe.cache().hget('login_failed_count', user)) or 0 - -def check_consecutive_login_attempts(user, doc): - login_failed_count = get_login_failed_count(user) - last_login_tried = (get_last_tried_login_data(user, True) - + datetime.timedelta(seconds=doc.allow_login_after_fail)) - - if login_failed_count >= cint(doc.allow_consecutive_login_attempts): - locked_account_time = frappe.cache().hget('locked_account_time', user) - if not locked_account_time: - frappe.cache().hset('locked_account_time', user, get_datetime()) - - if last_login_tried > get_datetime(): - frappe.throw(_("Your account has been locked and will resume after {0} seconds") - .format(doc.allow_login_after_fail), frappe.SecurityException) - else: - delete_login_failed_cache(user) - def validate_ip_address(user): """check if IP Address is valid""" user = frappe.get_cached_doc("User", user) if not frappe.flags.in_test else frappe.get_doc("User", user) @@ -436,3 +405,87 @@ def validate_ip_address(user): return frappe.throw(_("Access not allowed from this IP Address"), frappe.AuthenticationError) + + +class LoginAttemptTracker(object): + """Track login attemts of a user. + + Lock the account for s number of seconds if there have been n consecutive unsuccessful attempts to log in. + """ + def __init__(self, user_name: str, max_consecutive_login_attempts: int=3, lock_interval:int = 5*60): + """ Initialize the tracker. + + :param user_name: Name of the loggedin user + :param max_consecutive_login_attempts: Maximum allowed consecutive failed login attempts + :param lock_interval: Locking interval incase of maximum failed attempts + """ + self.user_name = user_name + self.lock_interval = datetime.timedelta(seconds=lock_interval) + self.max_failed_logins = max_consecutive_login_attempts + + @property + def login_failed_count(self): + return frappe.cache().hget('login_failed_count', self.user_name) + + @login_failed_count.setter + def login_failed_count(self, count): + frappe.cache().hset('login_failed_count', self.user_name, count) + + @login_failed_count.deleter + def login_failed_count(self): + frappe.cache().hdel('login_failed_count', self.user_name) + + @property + def login_failed_time(self): + """First failed login attempt time within lock interval. + + For every user we track only First failed login attempt time within lock interval of time. + """ + return frappe.cache().hget('login_failed_time', self.user_name) + + @login_failed_time.setter + def login_failed_time(self, timestamp): + frappe.cache().hset('login_failed_time', self.user_name, timestamp) + + @login_failed_time.deleter + def login_failed_time(self): + frappe.cache().hdel('login_failed_time', self.user_name) + + def add_failure_attempt(self): + """ Log user failure attempts into the system. + + Increase the failure count if new failure is with in current lock interval time period, if not reset the login failure count. + """ + login_failed_time = self.login_failed_time + login_failed_count = self.login_failed_count # Consecutive login failure count + current_time = get_datetime() + + if not (login_failed_time and login_failed_count): + login_failed_time, login_failed_count = current_time, 0 + + if login_failed_time + self.lock_interval > current_time: + login_failed_count += 1 + else: + login_failed_time, login_failed_count = current_time, 1 + + self.login_failed_time = login_failed_time + self.login_failed_count = login_failed_count + + def add_success_attempt(self): + """Reset login failures. + """ + del self.login_failed_count + del self.login_failed_time + + def is_user_allowed(self) -> bool: + """Is user allowed to login + + User is not allowed to login if login failures are greater than threshold within in lock interval from first login failure. + """ + login_failed_time = self.login_failed_time + login_failed_count = self.login_failed_count or 0 + current_time = get_datetime() + + if login_failed_time and login_failed_time + self.lock_interval > current_time and login_failed_count > self.max_failed_logins: + return False + return True diff --git a/frappe/automation/doctype/assignment_rule/assignment_rule.js b/frappe/automation/doctype/assignment_rule/assignment_rule.js index ee1a076465..97bed4f8f3 100644 --- a/frappe/automation/doctype/assignment_rule/assignment_rule.js +++ b/frappe/automation/doctype/assignment_rule/assignment_rule.js @@ -9,6 +9,16 @@ frappe.ui.form.on('Assignment Rule', { frm.events.rule(frm); }, + setup: function(frm) { + frm.set_query("document_type", () => { + return { + filters: { + name: ["!=", "ToDo"] + } + }; + }); + }, + document_type: function(frm) { frm.trigger('set_options'); }, diff --git a/frappe/automation/doctype/assignment_rule/assignment_rule.py b/frappe/automation/doctype/assignment_rule/assignment_rule.py index d20398d564..c673d5ceeb 100644 --- a/frappe/automation/doctype/assignment_rule/assignment_rule.py +++ b/frappe/automation/doctype/assignment_rule/assignment_rule.py @@ -18,6 +18,8 @@ class AssignmentRule(Document): if not len(set(assignment_days)) == len(assignment_days): repeated_days = get_repeated(assignment_days) frappe.throw(_("Assignment Day {0} has been repeated.").format(frappe.bold(repeated_days))) + if self.document_type == 'ToDo': + frappe.throw(_('Assignment Rule is not allowed on {0} document type').format(frappe.bold("ToDo"))) def on_update(self): clear_assignment_rule_cache(self) @@ -298,4 +300,4 @@ def get_repeated(values): def clear_assignment_rule_cache(rule): frappe.cache_manager.clear_doctype_map('Assignment Rule', rule.document_type) - frappe.cache_manager.clear_doctype_map('Assignment Rule', 'due_date_rules_for_' + rule.document_type) \ No newline at end of file + frappe.cache_manager.clear_doctype_map('Assignment Rule', 'due_date_rules_for_' + rule.document_type) diff --git a/frappe/build.py b/frappe/build.py index f47a7cb32b..baedb633b6 100644 --- a/frappe/build.py +++ b/frappe/build.py @@ -15,7 +15,7 @@ import frappe from frappe.utils.minify import JavascriptMinify import click -from requests import get +import psutil from six import iteritems, text_type from six.moves.urllib.parse import urlparse @@ -26,6 +26,8 @@ sites_path = os.path.abspath(os.getcwd()) def download_file(url, prefix): + from requests import get + filename = urlparse(url).path.split("/")[-1] local_filename = os.path.join(prefix, filename) with get(url, stream=True, allow_redirects=True) as r: @@ -225,7 +227,7 @@ def bundle(no_compress, app=None, make_copy=False, restore=False, verbose=False, frappe_app_path = os.path.abspath(os.path.join(app_paths[0], "..")) check_yarn() - frappe.commands.popen(command, cwd=frappe_app_path) + frappe.commands.popen(command, cwd=frappe_app_path, env=get_node_env()) def watch(no_compress): @@ -237,13 +239,32 @@ def watch(no_compress): frappe_app_path = os.path.abspath(os.path.join(app_paths[0], "..")) check_yarn() frappe_app_path = frappe.get_app_path("frappe", "..") - frappe.commands.popen("{pacman} run watch".format(pacman=pacman), cwd=frappe_app_path) + frappe.commands.popen("{pacman} run watch".format(pacman=pacman), + cwd=frappe_app_path, env=get_node_env()) def check_yarn(): if not find_executable("yarn"): print("Please install yarn using below command and try again.\nnpm install -g yarn") +def get_node_env(): + node_env = { + "NODE_OPTIONS": f"--max_old_space_size={get_safe_max_old_space_size()}" + } + return node_env + +def get_safe_max_old_space_size(): + safe_max_old_space_size = 0 + try: + total_memory = psutil.virtual_memory().total / (1024 * 1024) + # reference for the safe limit assumption + # https://nodejs.org/api/cli.html#cli_max_old_space_size_size_in_megabytes + # set minimum value 1GB + safe_max_old_space_size = max(1024, int(total_memory * 0.75)) + except Exception: + pass + + return safe_max_old_space_size def make_asset_dirs(make_copy=False, restore=False): # don't even think of making assets_path absolute - rm -rf ahead. diff --git a/frappe/commands/__init__.py b/frappe/commands/__init__.py index b7294fff77..b9ae02e112 100644 --- a/frappe/commands/__init__.py +++ b/frappe/commands/__init__.py @@ -11,6 +11,7 @@ import frappe.utils import subprocess # nosec from functools import wraps from six import StringIO +from os import environ click.disable_unicode_literals_warning = True @@ -53,16 +54,20 @@ def get_site(context, raise_err=True): return None def popen(command, *args, **kwargs): - output = kwargs.get('output', True) - cwd = kwargs.get('cwd') - shell = kwargs.get('shell', True) + output = kwargs.get('output', True) + cwd = kwargs.get('cwd') + shell = kwargs.get('shell', True) raise_err = kwargs.get('raise_err') + env = kwargs.get('env') + if env: + env = dict(environ, **env) proc = subprocess.Popen(command, - stdout = None if output else subprocess.PIPE, - stderr = None if output else subprocess.PIPE, - shell = shell, - cwd = cwd + stdout=None if output else subprocess.PIPE, + stderr=None if output else subprocess.PIPE, + shell=shell, + cwd=cwd, + env=env ) return_ = proc.wait() diff --git a/frappe/commands/site.py b/frappe/commands/site.py index 4a631be3ac..0fadf2a294 100755 --- a/frappe/commands/site.py +++ b/frappe/commands/site.py @@ -9,7 +9,6 @@ import click import frappe from frappe.commands import get_site, pass_context from frappe.exceptions import SiteNotSpecifiedError -from frappe.installer import _new_site @click.command('new-site') @@ -31,6 +30,8 @@ def new_site(site, mariadb_root_username=None, mariadb_root_password=None, admin verbose=False, install_apps=None, source_sql=None, force=None, no_mariadb_socket=False, install_app=None, db_name=None, db_password=None, db_type=None, db_host=None, db_port=None): "Create a new site" + from frappe.installer import _new_site + frappe.init(site=site, new_site=True) _new_site(db_name, site, mariadb_root_username=mariadb_root_username, @@ -57,6 +58,7 @@ def new_site(site, mariadb_root_username=None, mariadb_root_password=None, admin def restore(context, sql_file_path, mariadb_root_username=None, mariadb_root_password=None, db_name=None, verbose=None, install_app=None, admin_password=None, force=None, with_public_files=None, with_private_files=None): "Restore site database from an sql file" from frappe.installer import ( + _new_site, extract_sql_from_archive, extract_files, is_downgrade, @@ -145,6 +147,8 @@ def reinstall(context, admin_password=None, mariadb_root_username=None, mariadb_ _reinstall(site, admin_password, mariadb_root_username, mariadb_root_password, yes, verbose=context.verbose) def _reinstall(site, admin_password=None, mariadb_root_username=None, mariadb_root_password=None, yes=False, verbose=False): + from frappe.installer import _new_site + if not yes: click.confirm('This will wipe your database. Are you sure you want to reinstall?', abort=True) try: diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py index f3c25fb355..b3f85d13ad 100644 --- a/frappe/commands/utils.py +++ b/frappe/commands/utils.py @@ -578,7 +578,7 @@ def run_ui_tests(context, app, headless=False): frappe.commands.popen("yarn add cypress@^6 cypress-file-upload@^5 --no-lockfile") # run for headless mode - run_or_open = 'run --browser chrome --record --key 4a48f41c-11b3-425b-aa88-c58048fa69eb' if headless else 'open' + run_or_open = 'run --browser firefox --record --key 4a48f41c-11b3-425b-aa88-c58048fa69eb' if headless else 'open' command = '{site_env} {password_env} {cypress} {run_or_open}' formatted_command = command.format(site_env=site_env, password_env=password_env, cypress=cypress_path, run_or_open=run_or_open) diff --git a/frappe/core/doctype/activity_log/test_activity_log.py b/frappe/core/doctype/activity_log/test_activity_log.py index 4dbfd6700e..bd0ea08cc7 100644 --- a/frappe/core/doctype/activity_log/test_activity_log.py +++ b/frappe/core/doctype/activity_log/test_activity_log.py @@ -77,6 +77,10 @@ class TestActivityLog(unittest.TestCase): self.assertRaises(frappe.AuthenticationError, LoginManager) self.assertRaises(frappe.AuthenticationError, LoginManager) self.assertRaises(frappe.AuthenticationError, LoginManager) + + # REMOVE ME: current logic allows allow_consecutive_login_attempts+1 attempts + # before raising security exception, remove below line when that is fixed. + self.assertRaises(frappe.AuthenticationError, LoginManager) self.assertRaises(frappe.SecurityException, LoginManager) time.sleep(5) self.assertRaises(frappe.AuthenticationError, LoginManager) diff --git a/frappe/core/doctype/data_import/importer.py b/frappe/core/doctype/data_import/importer.py index dde3dfaee9..7caf69e668 100644 --- a/frappe/core/doctype/data_import/importer.py +++ b/frappe/core/doctype/data_import/importer.py @@ -449,8 +449,8 @@ class ImportFile: data_without_first_row = data[1:] for row in data_without_first_row: row_values = row.get_values(parent_column_indexes) - # if the row is blank, it's a child row doc - if all([v in INVALID_VALUES for v in row_values]): + # if the row is blank or same content as the previous parent row, it's a child row doc + if all([v in INVALID_VALUES for v in row_values]) or row_values == parent_row_values: rows.append(row) continue # if we encounter a row which has values in parent columns, @@ -472,32 +472,6 @@ class ImportFile: doc = parent_doc - if self.import_type == INSERT: - # check if there is atleast one row for mandatory table fields - meta = frappe.get_meta(self.doctype) - mandatory_table_fields = [ - df - for df in meta.fields - if df.fieldtype in table_fieldtypes - and df.reqd - and len(doc.get(df.fieldname, [])) == 0 - ] - if len(mandatory_table_fields) == 1: - self.warnings.append( - { - "row": first_row.row_number, - "message": _("There should be atleast one row for {0} table").format( - frappe.bold(mandatory_table_fields[0].label) - ), - } - ) - elif mandatory_table_fields: - fields_string = ", ".join([df.label for df in mandatory_table_fields]) - message = _("There should be atleast one row for the following tables: {0}").format( - fields_string - ) - self.warnings.append({"row": first_row.row_number, "message": message}) - return doc, rows, data[len(rows) :] def get_warnings(self): @@ -626,7 +600,6 @@ class Row: new_doc.update(doc) doc = new_doc - self.check_mandatory_fields(doctype, doc, table_df) return doc def validate_value(self, value, col): @@ -727,66 +700,6 @@ class Row: pass return value - def check_mandatory_fields(self, doctype, doc, table_df=None): - """If import type is Insert: - Check for mandatory fields (except table fields) in doc - if import type is Update: - Check for name field or autoname field in doc - """ - meta = frappe.get_meta(doctype) - if self.import_type == UPDATE: - if meta.istable: - # when updating records with table rows, - # there are two scenarios: - # 1. if row 'name' is provided in the template - # the table row will be updated - # 2. if row 'name' is not provided - # then a new row will be added - # so we dont need to check for mandatory - return - - # for update, only ID (name) field is mandatory - id_field = get_id_field(doctype) - if doc.get(id_field.fieldname) in INVALID_VALUES: - self.warnings.append( - { - "row": self.row_number, - "message": _("{0} is a mandatory field").format(id_field.label), - } - ) - return - - fields = [ - df - for df in meta.fields - if df.fieldtype not in table_fieldtypes - and df.reqd - and doc.get(df.fieldname) in INVALID_VALUES - ] - - if not fields: - return - - def get_field_label(df): - return "{0}{1}".format(df.label, " ({})".format(table_df.label) if table_df else "") - - if len(fields) == 1: - field_label = get_field_label(fields[0]) - self.warnings.append( - { - "row": self.row_number, - "message": _("{0} is a mandatory field").format(frappe.bold(field_label)), - } - ) - else: - fields_string = ", ".join([frappe.bold(get_field_label(df)) for df in fields]) - self.warnings.append( - { - "row": self.row_number, - "message": _("{0} are mandatory fields").format(fields_string), - } - ) - def get_values(self, indexes): return [self.data[i] for i in indexes] diff --git a/frappe/core/doctype/data_import/test_importer.py b/frappe/core/doctype/data_import/test_importer.py index b083b9eaaa..f76d4504a4 100644 --- a/frappe/core/doctype/data_import/test_importer.py +++ b/frappe/core/doctype/data_import/test_importer.py @@ -13,7 +13,7 @@ doctype_name = 'DocType for Import' class TestImporter(unittest.TestCase): @classmethod def setUpClass(cls): - create_doctype_if_not_exists(doctype_name) + create_doctype_if_not_exists(doctype_name,) def test_data_import_from_file(self): import_file = get_import_file('sample_import_file') @@ -59,18 +59,18 @@ class TestImporter(unittest.TestCase): def test_data_import_without_mandatory_values(self): import_file = get_import_file('sample_import_file_without_mandatory') data_import = self.get_importer(doctype_name, import_file) + frappe.local.message_log = [] data_import.start_import() data_import.reload() - warnings = frappe.parse_json(data_import.template_warnings) + import_log = frappe.parse_json(data_import.import_log) + self.assertEqual(import_log[0]['row_indexes'], [2,3]) + expected_error = "Error: Child 1 of DocType for Import Row #1: Value missing for: Child Title" + self.assertEqual(frappe.parse_json(import_log[0]['messages'][0])['message'], expected_error) + expected_error = "Error: Child 1 of DocType for Import Row #2: Value missing for: Child Title" + self.assertEqual(frappe.parse_json(import_log[0]['messages'][1])['message'], expected_error) - self.assertEqual(warnings[0]['row'], 2) - self.assertEqual(warnings[0]['message'], "Child Title (Table Field 1) is a mandatory field") - - self.assertEqual(warnings[1]['row'], 3) - self.assertEqual(warnings[1]['message'], "Child Title (Table Field 1 Again) is a mandatory field") - - self.assertEqual(warnings[2]['row'], 4) - self.assertEqual(warnings[2]['message'], "Title is a mandatory field") + self.assertEqual(import_log[1]['row_indexes'], [4]) + self.assertEqual(frappe.parse_json(import_log[1]['messages'][0])['message'], "Title is required") def test_data_import_update(self): existing_doc = frappe.get_doc( @@ -104,6 +104,8 @@ class TestImporter(unittest.TestCase): data_import.reference_doctype = doctype data_import.import_file = import_file.file_url data_import.insert() + # Commit so that the first import failure does not rollback the Data Import insert. + frappe.db.commit() return data_import diff --git a/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py b/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py index e02d9e5db0..92493a593a 100644 --- a/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py +++ b/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py @@ -117,6 +117,7 @@ def execute_event(doc: str): frappe.only_for("System Manager") doc = json.loads(doc) frappe.get_doc("Scheduled Job Type", doc.get("name")).enqueue(force=True) + return doc def run_scheduled_job(job_type: str): diff --git a/frappe/core/doctype/sms_settings/sms_settings.json b/frappe/core/doctype/sms_settings/sms_settings.json index 3bb89604af..073fb88bc7 100755 --- a/frappe/core/doctype/sms_settings/sms_settings.json +++ b/frappe/core/doctype/sms_settings/sms_settings.json @@ -202,7 +202,7 @@ "issingle": 1, "istable": 0, "max_attachments": 0, - "modified": "2017-11-01 12:57:20.943845", + "modified": "2021-03-02 18:06:00.868688", "modified_by": "Administrator", "module": "Core", "name": "SMS Settings", @@ -233,6 +233,6 @@ "read_only": 0, "read_only_onload": 0, "show_name_in_global_search": 0, - "track_changes": 0, + "track_changes": 1, "track_seen": 0 -} \ No newline at end of file +} diff --git a/frappe/core/doctype/user/test_user.py b/frappe/core/doctype/user/test_user.py index d16db5fecd..8a8071423e 100644 --- a/frappe/core/doctype/user/test_user.py +++ b/frappe/core/doctype/user/test_user.py @@ -11,6 +11,7 @@ from frappe.utils import get_url from frappe.core.doctype.user.user import get_total_users from frappe.core.doctype.user.user import MaxUsersReachedError, test_password_strength from frappe.core.doctype.user.user import extract_mentions +from frappe.frappeclient import FrappeClient test_records = frappe.get_test_records('User') @@ -229,16 +230,22 @@ class TestUser(unittest.TestCase): self.assertEqual(extract_mentions(comment)[1], "test.again@example1.com") def test_rate_limiting_for_reset_password(self): - from frappe.utils.password import delete_password_reset_cache - delete_password_reset_cache() - + # Allow only one reset request for a day frappe.db.set_value("System Settings", "System Settings", "password_reset_limit", 1) + frappe.db.commit() - user = frappe.get_doc("User", "testperm@example.com") - link = user.reset_password() - self.assertRegex(link, "\/update-password\?key=[A-Za-z0-9]*") + url = get_url() + data={'cmd': 'frappe.core.doctype.user.user.reset_password', 'user': 'test@test.com'} - self.assertRaises(frappe.ValidationError, user.reset_password, False) + # Clear rate limit tracker to start fresh + key = f"rl:{data['cmd']}:{data['user']}" + frappe.cache().delete(key) + + c = FrappeClient(url) + res1 = c.session.post(url, data=data, verify=c.verify, headers=c.headers) + res2 = c.session.post(url, data=data, verify=c.verify, headers=c.headers) + self.assertEqual(res1.status_code, 200) + self.assertEqual(res2.status_code, 417) def test_user_rollback(self): """ """ diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py index 142cc1ee26..c103ad7e4a 100644 --- a/frappe/core/doctype/user/user.py +++ b/frappe/core/doctype/user/user.py @@ -2,21 +2,25 @@ # MIT License. See license.txt from __future__ import unicode_literals, print_function + +from bs4 import BeautifulSoup + import frappe +import frappe.share +import frappe.defaults +import frappe.permissions from frappe.model.document import Document from frappe.utils import cint, flt, has_gravatar, escape_html, format_datetime, now_datetime, get_formatted_email, today from frappe import throw, msgprint, _ -from frappe.utils.password import update_password as _update_password +from frappe.utils.password import update_password as _update_password, check_password, get_password_reset_limit from frappe.desk.notifications import clear_notifications -from frappe.desk.doctype.notification_settings.notification_settings import create_notification_settings +from frappe.desk.doctype.notification_settings.notification_settings import create_notification_settings, toggle_notifications from frappe.utils.user import get_system_managers -from bs4 import BeautifulSoup -import frappe.permissions -import frappe.share -import frappe.defaults from frappe.website.utils import is_signup_enabled +from frappe.rate_limiter import rate_limit from frappe.utils.background_jobs import enqueue + STANDARD_USERS = ("Guest", "Administrator") @@ -146,6 +150,9 @@ class User(Document): if not cint(self.enabled) and getattr(frappe.local, "login_manager", None): frappe.local.login_manager.logout(user=self.name) + # toggle notifications based on the user's status + toggle_notifications(self.name, enable=cint(self.enabled)) + def add_system_manager_role(self): # if adding system manager, do nothing if not cint(self.enabled) or ("System Manager" in [user_role.role for user_role in @@ -238,11 +245,6 @@ class User(Document): def reset_password(self, send_email=False, password_expired=False): from frappe.utils import random_string, get_url - rate_limit = frappe.db.get_single_value("System Settings", "password_reset_limit") - - if rate_limit: - check_password_reset_limit(self.name, rate_limit) - key = random_string(32) self.db_set("reset_password_key", key) @@ -254,7 +256,6 @@ class User(Document): if send_email: self.password_reset_mail(link) - update_password_reset_limit(self.name) return link def get_other_system_managers(self): @@ -358,6 +359,9 @@ class User(Document): set `user`=null where `user`=%s""", (self.name)) + # delete notification settings + frappe.delete_doc("Notification Settings", self.name, ignore_permissions=True) + def before_rename(self, old_name, new_name, merge=False): self.check_demo() @@ -527,6 +531,27 @@ class User(Document): return [i.strip() for i in self.restrict_ip.split(",")] + @classmethod + def find_by_credentials(cls, user_name: str, password: str, validate_password: bool = True): + """Find the user by credentials. + """ + login_with_mobile = cint(frappe.db.get_value("System Settings", "System Settings", "allow_login_using_mobile_number")) + filter = {"mobile_no": user_name} if login_with_mobile else {"name": user_name} + + user = frappe.db.get_value("User", filters=filter, fieldname=['name', 'enabled'], as_dict=True) or {} + if not user: + return + + user['is_authenticated'] = True + if validate_password: + try: + check_password(user_name, password) + except frappe.AuthenticationError: + user['is_authenticated'] = False + + return user + + @frappe.whitelist() def get_timezones(): import pytz @@ -816,6 +841,7 @@ def sign_up(email, full_name, redirect_to): return 2, _("Please ask your administrator to verify your sign-up") @frappe.whitelist(allow_guest=True) +@rate_limit(key='user', limit=get_password_reset_limit, seconds = 24*60*60, methods=['POST']) def reset_password(user): if user=="Administrator": return 'not allowed' @@ -1147,16 +1173,3 @@ def generate_keys(user): def switch_theme(theme): if theme in ["Dark", "Light"]: frappe.db.set_value("User", frappe.session.user, "desk_theme", theme) - -def update_password_reset_limit(user): - generated_link_count = get_generated_link_count(user) - generated_link_count += 1 - frappe.cache().hset("password_reset_link_count", user, generated_link_count) - -def check_password_reset_limit(user, rate_limit): - generated_link_count = get_generated_link_count(user) - if generated_link_count >= rate_limit: - frappe.throw(_("You have reached the hourly limit for generating password reset links. Please try again later.")) - -def get_generated_link_count(user): - return cint(frappe.cache().hget("password_reset_link_count", user)) or 0 diff --git a/frappe/custom/doctype/customize_form/customize_form.js b/frappe/custom/doctype/customize_form/customize_form.js index 79978a49d7..d9d8ae196e 100644 --- a/frappe/custom/doctype/customize_form/customize_form.js +++ b/frappe/custom/doctype/customize_form/customize_form.js @@ -76,6 +76,7 @@ frappe.ui.form.on("Customize Form", { frm.trigger("setup_sortable"); } } + localStorage["customize_doctype"] = frm.doc.doc_type; } }); } else { diff --git a/frappe/database/database.py b/frappe/database/database.py index 179206a4af..4fcf10efda 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -16,7 +16,6 @@ import frappe.model.meta from frappe import _ from time import time from frappe.utils import now, getdate, cast_fieldtype, get_datetime -from frappe.utils.background_jobs import execute_job, get_queue from frappe.model.utils.link_count import flush_local_link_count from frappe.utils import cint @@ -1032,6 +1031,8 @@ class Database(object): insert_list = [] def enqueue_jobs_after_commit(): + from frappe.utils.background_jobs import execute_job, get_queue + if frappe.flags.enqueue_after_commit and len(frappe.flags.enqueue_after_commit) > 0: for job in frappe.flags.enqueue_after_commit: q = get_queue(job.get("queue"), is_async=job.get("is_async")) diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index 3cbb2e4f0e..f9997d1526 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -8,8 +8,7 @@ from pymysql.times import TimeDelta from pymysql.constants import ER, FIELD_TYPE from pymysql.converters import conversions -from frappe.utils import get_datetime, cstr -from markdown2 import UnicodeWithAttrs +from frappe.utils import get_datetime, cstr, UnicodeWithAttrs from frappe.database.database import Database from six import PY2, binary_type, text_type, string_types from frappe.database.mariadb.schema import MariaDBTable diff --git a/frappe/desk/desktop.py b/frappe/desk/desktop.py index 0ded8e0717..5b6e2fdd21 100644 --- a/frappe/desk/desktop.py +++ b/frappe/desk/desktop.py @@ -601,8 +601,8 @@ def merge_cards_based_on_label(cards): for card in cards: label = card.get('label') if label in cards_dict: - links = loads(cards_dict[label].links) + loads(card.links) - cards_dict[label].update(dict(links=dumps(links))) + links = cards_dict[label].links + card.links + cards_dict[label].update(dict(links=links)) cards_dict[label] = cards_dict.pop(label) else: cards_dict[label] = card diff --git a/frappe/desk/doctype/notification_settings/notification_settings.py b/frappe/desk/doctype/notification_settings/notification_settings.py index 34726bdf8a..4ab40bffe9 100644 --- a/frappe/desk/doctype/notification_settings/notification_settings.py +++ b/frappe/desk/doctype/notification_settings/notification_settings.py @@ -44,6 +44,11 @@ def create_notification_settings(user): _doc.insert(ignore_permissions=True) +def toggle_notifications(user, enable=False): + if frappe.db.exists("Notification Settings", user): + frappe.db.set_value("Notification Settings", user, 'enabled', enable) + + @frappe.whitelist() def get_subscribed_documents(): if not frappe.session.user: @@ -75,4 +80,4 @@ def get_permission_query_conditions(user): @frappe.whitelist() def set_seen_value(value, user): - frappe.db.set_value('Notification Settings', user, 'seen', value, update_modified=False) \ No newline at end of file + frappe.db.set_value('Notification Settings', user, 'seen', value, update_modified=False) diff --git a/frappe/desk/form/document_follow.py b/frappe/desk/form/document_follow.py index f5ace4d732..f5e5c0ca9b 100644 --- a/frappe/desk/form/document_follow.py +++ b/frappe/desk/form/document_follow.py @@ -147,6 +147,8 @@ def get_version(doctype, doc_name, frequency, user): return timeline def get_comments(doctype, doc_name, frequency, user): + from html2text import html2text + timeline = [] filters = get_filters("reference_name", doc_name, frequency, user) comments = frappe.get_all("Comment", @@ -166,7 +168,7 @@ def get_comments(doctype, doc_name, frequency, user): "time": comment.modified, "data": { "time": time, - "comment": frappe.utils.html2text(str(comment.content)), + "comment": html2text(str(comment.content)), "by": by }, "doctype": doctype, @@ -197,6 +199,8 @@ def get_follow_users(doctype, doc_name): ) def get_row_changed(row_changed, time, doctype, doc_name, v): + from html2text import html2text + items = [] for d in row_changed: d[2] = d[2] if d[2] else ' ' @@ -209,8 +213,8 @@ def get_row_changed(row_changed, time, doctype, doc_name, v): "table_field": d[0], "row": str(d[1]), "field": d[3][0][0], - "from": frappe.utils.html2text(str(d[3][0][1])), - "to": frappe.utils.html2text(str(d[3][0][2])) + "from": html2text(str(d[3][0][1])), + "to": html2text(str(d[3][0][2])) }, "doctype": doctype, "doc_name": doc_name, @@ -236,6 +240,8 @@ def get_added_row(added, time, doctype, doc_name, v): return items def get_field_changed(changed, time, doctype, doc_name, v): + from html2text import html2text + items = [] for d in changed: d[1] = d[1] if d[1] else ' ' @@ -246,8 +252,8 @@ def get_field_changed(changed, time, doctype, doc_name, v): "data": { "time": time, "field": d[0], - "from": frappe.utils.html2text(str(d[1])), - "to": frappe.utils.html2text(str(d[2])) + "from": html2text(str(d[1])), + "to": html2text(str(d[2])) }, "doctype": doctype, "doc_name": doc_name, diff --git a/frappe/desk/leaderboard.py b/frappe/desk/leaderboard.py index 8d00ea9bc2..2a981f061b 100644 --- a/frappe/desk/leaderboard.py +++ b/frappe/desk/leaderboard.py @@ -16,8 +16,18 @@ def get_leaderboards(): @frappe.whitelist() def get_energy_point_leaderboard(date_range, company = None, field = None, limit = None): + all_users = frappe.db.get_all('User', + filters = { + 'name': ['not in', ['Administrator', 'Guest']], + 'enabled': 1, + 'user_type': ['!=', 'Website User'] + }, + order_by = 'name ASC') + all_users_list = list(map(lambda x: x['name'], all_users)) + filters = [ ['type', '!=', 'Review'], + ['user', 'in', all_users_list] ] if date_range: date_range = frappe.parse_json(date_range) @@ -28,15 +38,7 @@ def get_energy_point_leaderboard(date_range, company = None, field = None, limit group_by = 'user', order_by = 'value desc' ) - all_users = frappe.db.get_all('User', - filters = { - 'name': ['not in', ['Administrator', 'Guest']], - 'enabled': 1, - 'user_type': ['!=', 'Website User'] - }, - order_by = 'name ASC') - all_users_list = list(map(lambda x: x['name'], all_users)) energy_point_users_list = list(map(lambda x: x['name'], energy_point_users)) for user in all_users_list: if user not in energy_point_users_list: diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index ca4dbb83e2..4869c5a9bf 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -90,6 +90,29 @@ class EmailAccount(Document): if self.append_to not in valid_doctypes: frappe.throw(_("Append To can be one of {0}").format(comma_or(valid_doctypes))) + def before_save(self): + messages = [] + as_list = 1 + if not self.enable_incoming and self.default_incoming: + self.default_incoming = False + messages.append(_("{} has been disabled. It can only be enabled if {} is checked.") + .format( + frappe.bold(_('Default Incoming')), + frappe.bold(_('Enable Incoming')) + ) + ) + if not self.enable_outgoing and self.default_outgoing: + self.default_outgoing = False + messages.append(_("{} has been disabled. It can only be enabled if {} is checked.") + .format( + frappe.bold(_('Default Outgoing')), + frappe.bold(_('Enable Outgoing')) + ) + ) + if messages: + if len(messages) == 1: (as_list, messages) = (0, messages[0]) + frappe.msgprint(messages, as_list= as_list, indicator='orange', title=_("Defaults Updated")) + def on_update(self): """Check there is only one default of each type.""" from frappe.core.doctype.user.user import setup_user_email_inbox diff --git a/frappe/email/doctype/newsletter/newsletter.json b/frappe/email/doctype/newsletter/newsletter.json index 1dd6115b43..dcd19ed33c 100644 --- a/frappe/email/doctype/newsletter/newsletter.json +++ b/frappe/email/doctype/newsletter/newsletter.json @@ -19,9 +19,12 @@ "message", "message_md", "message_html", + "section_break_13", "send_unsubscribe_link", "send_attachments", + "column_break_9", "published", + "send_webview_link", "route", "test_the_newsletter", "test_email_id", @@ -160,6 +163,21 @@ "fieldtype": "Check", "label": "Schedule Sending", "read_only_depends_on": "eval: doc.email_sent" + }, + { + "fieldname": "column_break_9", + "fieldtype": "Column Break" + }, + { + "default": "0", + "depends_on": "published", + "fieldname": "send_webview_link", + "fieldtype": "Check", + "label": "Send Web View Link" + }, + { + "fieldname": "section_break_13", + "fieldtype": "Section Break" } ], "has_web_view": 1, @@ -169,7 +187,7 @@ "is_published_field": "published", "links": [], "max_attachments": 3, - "modified": "2020-08-24 19:59:37.262500", + "modified": "2021-02-22 14:33:56.095380", "modified_by": "Administrator", "module": "Email", "name": "Newsletter", diff --git a/frappe/email/doctype/newsletter/newsletter.py b/frappe/email/doctype/newsletter/newsletter.py index 2791ebb75b..ad985ee20e 100755 --- a/frappe/email/doctype/newsletter/newsletter.py +++ b/frappe/email/doctype/newsletter/newsletter.py @@ -68,13 +68,17 @@ class Newsletter(WebsiteGenerator): except IOError: frappe.throw(_("Unable to find attachment {0}").format(file.name)) - send(recipients=self.recipients, sender=sender, - subject=self.subject, message=self.get_message(), + args = { + "message": self.get_message(), + "name": self.name + } + frappe.sendmail(recipients=self.recipients, sender=sender, + subject=self.subject, message=self.get_message(), template="newsletter", reference_doctype=self.doctype, reference_name=self.name, add_unsubscribe_link=self.send_unsubscribe_link, attachments=attachments, unsubscribe_method="/unsubscribe", unsubscribe_params={"name": self.name}, - send_priority=0, queue_separately=True) + send_priority=0, queue_separately=True, args=args) if not frappe.flags.in_test: frappe.db.auto_commit_on_many_writes = False diff --git a/frappe/exceptions.py b/frappe/exceptions.py index 82fbff7a90..ab65e6e006 100644 --- a/frappe/exceptions.py +++ b/frappe/exceptions.py @@ -33,6 +33,9 @@ class PermissionError(Exception): class DoesNotExistError(ValidationError): http_status_code = 404 +class PageDoesNotExistError(ValidationError): + http_status_code = 404 + class NameError(Exception): http_status_code = 409 diff --git a/frappe/hooks.py b/frappe/hooks.py index 3e206f0ad3..c9914237fe 100644 --- a/frappe/hooks.py +++ b/frappe/hooks.py @@ -207,8 +207,7 @@ scheduler_events = { "frappe.deferred_insert.save_to_db", "frappe.desk.form.document_follow.send_hourly_updates", "frappe.integrations.doctype.google_calendar.google_calendar.sync", - "frappe.email.doctype.newsletter.newsletter.send_scheduled_email", - "frappe.utils.password.delete_password_reset_cache" + "frappe.email.doctype.newsletter.newsletter.send_scheduled_email" ], "daily": [ "frappe.email.queue.set_expiry_for_email_queue", diff --git a/frappe/integrations/doctype/connected_app/test_connected_app.py b/frappe/integrations/doctype/connected_app/test_connected_app.py index 6faa542a60..b4304f6ee8 100644 --- a/frappe/integrations/doctype/connected_app/test_connected_app.py +++ b/frappe/integrations/doctype/connected_app/test_connected_app.py @@ -108,13 +108,8 @@ class TestConnectedApp(unittest.TestCase): session = requests.Session() - # first login of a new user on a new site fails with "401 UNAUTHORIZED" - # when anybody fixes that, the two lines below can be removed first_login = login() - self.assertEqual(first_login.status_code, 401) - - second_login = login() - self.assertEqual(second_login.status_code, 200) + self.assertEqual(first_login.status_code, 200) authorization_url = self.connected_app.initiate_web_application_flow(user=self.user_name) diff --git a/frappe/integrations/oauth2.py b/frappe/integrations/oauth2.py index 07db778a2d..c444964a16 100644 --- a/frappe/integrations/oauth2.py +++ b/frappe/integrations/oauth2.py @@ -163,10 +163,13 @@ def openid_profile(*args, **kwargs): first_name, last_name, avatar, name = frappe.db.get_value("User", frappe.session.user, ["first_name", "last_name", "user_image", "name"]) frappe_userid = frappe.db.get_value("User Social Login", {"parent":frappe.session.user, "provider": "frappe"}, "userid") request_url = urlparse(frappe.request.url) + base_url = frappe.db.get_value("Social Login Key", "frappe", "base_url") or None if avatar: if validate_url(avatar): picture = avatar + elif base_url: + picture = base_url + '/' + avatar else: picture = request_url.scheme + "://" + request_url.netloc + avatar diff --git a/frappe/model/base_document.py b/frappe/model/base_document.py index 295585665f..c1e5f01e03 100644 --- a/frappe/model/base_document.py +++ b/frappe/model/base_document.py @@ -12,11 +12,9 @@ from frappe.model.naming import set_new_name from frappe.model.utils.link_count import notify_link_count from frappe.modules import load_doctype_module from frappe.model import display_fieldtypes -from frappe.utils.password import get_decrypted_password, set_encrypted_password from frappe.utils import (cint, flt, now, cstr, strip_html, sanitize_html, sanitize_email, cast_fieldtype) from frappe.utils.html_utils import unescape_html -from bs4 import BeautifulSoup max_positive_value = { 'smallint': 2 ** 15, @@ -419,25 +417,60 @@ class BaseDocument(object): doc.db_update() def show_unique_validation_message(self, e): - # TODO: Find a better way to extract fieldname if frappe.db.db_type != 'postgres': fieldname = str(e).split("'")[-2] label = None - # unique_first_fieldname_second_fieldname is the constraint name - # created using frappe.db.add_unique - if "unique_" in fieldname: - fieldname = fieldname.split("_", 1)[1] + # MariaDB gives key_name in error. Extracting fieldname from key name + try: + fieldname = self.get_field_name_by_key_name(fieldname) + except IndexError: + pass - df = self.meta.get_field(fieldname) - if df: - label = df.label + label = self.get_label_from_fieldname(fieldname) frappe.msgprint(_("{0} must be unique").format(label or fieldname)) # this is used to preserve traceback raise frappe.UniqueValidationError(self.doctype, self.name, e) + def get_field_name_by_key_name(self, key_name): + """MariaDB stores a mapping between `key_name` and `column_name`. + This function returns the `column_name` associated with the `key_name` passed + + Args: + key_name (str): The name of the database index. + + Raises: + IndexError: If the key is not found in the table. + + Returns: + str: The column name associated with the key. + """ + return frappe.db.sql(f""" + SHOW + INDEX + FROM + `tab{self.doctype}` + WHERE + key_name=%s + AND + Non_unique=0 + """, key_name, as_dict=True)[0].get("Column_name") + + def get_label_from_fieldname(self, fieldname): + """Returns the associated label for fieldname + + Args: + fieldname (str): The fieldname in the DocType to use to pull the label. + + Returns: + str: The label associated with the fieldname, if found, otherwise `None`. + """ + df = self.meta.get_field(fieldname) + if df: + return df.label + def update_modified(self): """Update modified timestamp""" self.set("modified", now()) @@ -721,6 +754,8 @@ class BaseDocument(object): - Ignore if 'Ignore XSS Filter' is checked or fieldtype is 'Code' """ + from bs4 import BeautifulSoup + if frappe.flags.in_install: return @@ -757,6 +792,8 @@ class BaseDocument(object): def _save_passwords(self): """Save password field values in __Auth table""" + from frappe.utils.password import set_encrypted_password + if self.flags.ignore_save_passwords is True: return @@ -771,6 +808,8 @@ class BaseDocument(object): self.set(df.fieldname, '*'*len(new_password)) def get_password(self, fieldname='password', raise_exception=True): + from frappe.utils.password import get_decrypted_password + if self.get(fieldname) and not self.is_dummy_password(self.get(fieldname)): return self.get(fieldname) diff --git a/frappe/model/document.py b/frappe/model/document.py index 3ecc335cdd..d426dadd06 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -6,7 +6,6 @@ import frappe import time from frappe import _, msgprint from frappe.utils import flt, cstr, now, get_datetime_str, file_lock, date_diff -from frappe.utils.background_jobs import enqueue from frappe.model.base_document import BaseDocument, get_controller from frappe.model.naming import set_new_name from six import iteritems, string_types @@ -1269,6 +1268,8 @@ class Document(BaseDocument): # call _submit instead of submit, so you can override submit to call # run_delayed based on some action # See: Stock Reconciliation + from frappe.utils.background_jobs import enqueue + if hasattr(self, '_' + action): action = '_' + action diff --git a/frappe/patches.txt b/frappe/patches.txt index 5400c96354..a3f60ca210 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -35,6 +35,7 @@ frappe.patches.v11_0.change_email_signature_fieldtype execute:frappe.reload_doc('core', 'doctype', 'activity_log') execute:frappe.reload_doc('core', 'doctype', 'deleted_document') execute:frappe.reload_doc('core', 'doctype', 'domain_settings') +frappe.patches.v13_0.rename_custom_client_script frappe.patches.v8_0.rename_page_role_to_has_role #2017-03-16 frappe.patches.v7_2.setup_custom_perms #2017-01-19 frappe.patches.v8_0.set_user_permission_for_page_and_report #2017-03-20 @@ -330,4 +331,4 @@ execute:frappe.get_doc('Role', 'Guest').save() # remove desk access frappe.patches.v13_0.rename_desk_page_to_workspace # 02.02.2021 frappe.patches.v13_0.delete_package_publish_tool frappe.patches.v13_0.rename_list_view_setting_to_list_view_settings -frappe.patches.v13_0.rename_custom_client_script +frappe.patches.v13_0.remove_twilio_settings diff --git a/frappe/patches/v13_0/enable_custom_script.py b/frappe/patches/v13_0/enable_custom_script.py index 92284e6dcc..edc242e700 100644 --- a/frappe/patches/v13_0/enable_custom_script.py +++ b/frappe/patches/v13_0/enable_custom_script.py @@ -5,9 +5,8 @@ from __future__ import unicode_literals import frappe def execute(): - """Enable all the existing custom script""" - frappe.reload_doc("Custom", "doctype", "Custom Script") + """Enable all the existing Client script""" frappe.db.sql(""" - UPDATE `tabCustom Script` SET enabled=1 + UPDATE `tabClient Script` SET enabled=1 """) \ No newline at end of file diff --git a/frappe/patches/v13_0/remove_twilio_settings.py b/frappe/patches/v13_0/remove_twilio_settings.py new file mode 100644 index 0000000000..363cbdd4b6 --- /dev/null +++ b/frappe/patches/v13_0/remove_twilio_settings.py @@ -0,0 +1,20 @@ +# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors +# MIT License. See license.txt + +import frappe + + +def execute(): + """Add missing Twilio patch. + + While making Twilio as a standaone app, we missed to delete Twilio records from DB through migration. Adding the missing patch. + """ + frappe.delete_doc_if_exists('DocType', 'Twilio Number Group') + if twilio_settings_doctype_in_integrations(): + frappe.delete_doc_if_exists('DocType', 'Twilio Settings') + frappe.db.sql("delete from `tabSingles` where `doctype`=%s", 'Twilio Settings') + +def twilio_settings_doctype_in_integrations() -> bool: + """Check Twilio Settings doctype exists in integrations module or not. + """ + return frappe.db.exists("DocType", {'name': 'Twilio Settings', 'module': 'Integrations'}) diff --git a/frappe/printing/doctype/print_format/print_format.json b/frappe/printing/doctype/print_format/print_format.json index 92d4a67d14..4032cef209 100644 --- a/frappe/printing/doctype/print_format/print_format.json +++ b/frappe/printing/doctype/print_format/print_format.json @@ -171,7 +171,7 @@ "fieldname": "custom_html_help", "fieldtype": "HTML", "label": "Custom HTML Help", - "options": "

Custom CSS Help

\n\n

Notes:

\n\n
    \n
  1. All field groups (label + value) are set attributes data-fieldtype and data-fieldname
    2. \n
  2. All values are given class value
    3. \n
  3. All Section Breaks are given class section-break
    4. \n
  4. All Column Breaks are given class column-break
    5. \n
\n\n

Examples

\n\n

1. Left align integers

\n\n
[data-fieldtype=\"Int\"] .value { text-left: left; }
\n\n

1. Add border to sections except the last section

\n\n
.section-break { padding: 30px 0px; border-bottom: 1px solid #eee; }\n.section-break:last-child { padding-bottom: 0px; border-bottom: 0px;  }
\n" + "options": "

Custom CSS Help

\n\n

Notes:

\n\n
    \n
  1. All field groups (label + value) are set attributes data-fieldtype and data-fieldname
    2. \n
  2. All values are given class value
    3. \n
  3. All Section Breaks are given class section-break
    4. \n
  4. All Column Breaks are given class column-break
    5. \n
\n\n

Examples

\n\n

1. Left align integers

\n\n
[data-fieldtype=\"Int\"] .value { text-align: left; }
\n\n

1. Add border to sections except the last section

\n\n
.section-break { padding: 30px 0px; border-bottom: 1px solid #eee; }\n.section-break:last-child { padding-bottom: 0px; border-bottom: 0px;  }
\n" }, { "depends_on": "custom_format", @@ -211,7 +211,7 @@ "idx": 1, "index_web_pages_for_search": 1, "links": [], - "modified": "2020-12-14 11:38:49.132061", + "modified": "2021-03-01 15:25:46.578863", "modified_by": "Administrator", "module": "Printing", "name": "Print Format", diff --git a/frappe/printing/page/print/print.js b/frappe/printing/page/print/print.js index 0ae8786e95..dfd93c4efa 100644 --- a/frappe/printing/page/print/print.js +++ b/frappe/printing/page/print/print.js @@ -269,6 +269,7 @@ frappe.ui.form.PrintView = class { based_on: data.based_on, }; frappe.set_route('print-format-builder'); + this.print_sel.val(data.print_format_name); }, __('New Custom Print Format'), __('Start') @@ -641,10 +642,13 @@ frappe.ui.form.PrintView = class { refresh_print_options() { this.print_formats = frappe.meta.get_print_formats(this.frm.doctype); - return this.print_sel.empty().add_options([ + const print_format_select_val = this.print_sel.val(); + this.print_sel.empty().add_options([ this.get_default_option_for_select(__('Select Print Format')), ...this.print_formats ]); + return this.print_formats.includes(print_format_select_val) + && this.print_sel.val(print_format_select_val); } selected_format() { diff --git a/frappe/printing/page/print_format_builder/print_format_builder.js b/frappe/printing/page/print_format_builder/print_format_builder.js index eb87190ab5..7e58e295b5 100644 --- a/frappe/printing/page/print_format_builder/print_format_builder.js +++ b/frappe/printing/page/print_format_builder/print_format_builder.js @@ -784,6 +784,7 @@ frappe.PrintFormatBuilder = Class.extend({ btn: this.page.btn_primary, callback: function(r) { me.print_format = r.message; + locals['Print Format'][me.print_format.name] = r.message; frappe.show_alert({message: __("Saved"), indicator: 'green'}); } }); diff --git a/frappe/printing/page/print_format_builder/print_format_builder_sidebar.html b/frappe/printing/page/print_format_builder/print_format_builder_sidebar.html index 0cf8178f82..1ebb87ac31 100644 --- a/frappe/printing/page/print_format_builder/print_format_builder_sidebar.html +++ b/frappe/printing/page/print_format_builder/print_format_builder_sidebar.html @@ -13,7 +13,7 @@ - {%= __(f.label) %} + {%= __(f.label) || __(f.fieldname) %} {% } %} diff --git a/frappe/public/icons/timeless/icon-right-arrow.svg b/frappe/public/icons/timeless/icon-right-arrow.svg deleted file mode 100644 index 1e044d0e4d..0000000000 --- a/frappe/public/icons/timeless/icon-right-arrow.svg +++ /dev/null @@ -1,3 +0,0 @@ - - - \ No newline at end of file diff --git a/frappe/public/js/frappe/desk.js b/frappe/public/js/frappe/desk.js index 3912373bde..e033ae4c5b 100644 --- a/frappe/public/js/frappe/desk.js +++ b/frappe/public/js/frappe/desk.js @@ -128,7 +128,7 @@ frappe.Application = Class.extend({ } // REDESIGN-TODO: Fix preview popovers - //this.link_preview = new frappe.ui.LinkPreview(); + this.link_preview = new frappe.ui.LinkPreview(); if (!frappe.boot.developer_mode) { setInterval(function() { diff --git a/frappe/public/js/frappe/form/controls/attach.js b/frappe/public/js/frappe/form/controls/attach.js index fe662c1ada..604510bb52 100644 --- a/frappe/public/js/frappe/form/controls/attach.js +++ b/frappe/public/js/frappe/form/controls/attach.js @@ -1,6 +1,6 @@ frappe.ui.form.ControlAttach = frappe.ui.form.ControlData.extend({ make_input: function() { - var me = this; + let me = this; this.$input = $('