diff --git a/.github/helper/documentation.py b/.github/helper/documentation.py
index 3fc14ba61b..08d1d1aa9c 100644
--- a/.github/helper/documentation.py
+++ b/.github/helper/documentation.py
@@ -21,8 +21,8 @@ def docs_link_exists(body):
if word.startswith('http') and uri_validator(word):
parsed_url = urlparse(word)
if parsed_url.netloc == "github.com":
- _, org, repo, _type, ref = parsed_url.path.split('/')
- if org == "frappe" and repo in docs_repos:
+ parts = parsed_url.path.split('/')
+ if len(parts) == 5 and parts[1] == "frappe" and parts[2] in docs_repos:
return True
diff --git a/frappe/automation/doctype/assignment_rule/assignment_rule.js b/frappe/automation/doctype/assignment_rule/assignment_rule.js
index 774befc15e..ee1a076465 100644
--- a/frappe/automation/doctype/assignment_rule/assignment_rule.js
+++ b/frappe/automation/doctype/assignment_rule/assignment_rule.js
@@ -57,7 +57,8 @@ frappe.ui.form.on('Assignment Rule', {
frm.set_fields_as_options(
'field',
doctype,
- (df) => df.fieldtype == 'Link' && df.options == 'User',
+ (df) => ['Dynamic Link', 'Data'].includes(df.fieldtype)
+ || (df.fieldtype == 'Link' && df.options == 'User'),
[{ label: 'Owner', value: 'owner' }]
);
if (doctype) {
diff --git a/frappe/automation/doctype/assignment_rule/assignment_rule.py b/frappe/automation/doctype/assignment_rule/assignment_rule.py
index c85cb149ea..d20398d564 100644
--- a/frappe/automation/doctype/assignment_rule/assignment_rule.py
+++ b/frappe/automation/doctype/assignment_rule/assignment_rule.py
@@ -82,7 +82,7 @@ class AssignmentRule(Document):
elif self.rule == 'Load Balancing':
return self.get_user_load_balancing()
elif self.rule == 'Based on Field':
- return doc.get(self.field)
+ return self.get_user_based_on_field(doc)
def get_user_round_robin(self):
'''
@@ -119,6 +119,11 @@ class AssignmentRule(Document):
# pick the first user
return sorted_counts[0].get('user')
+ def get_user_based_on_field(self, doc):
+ val = doc.get(self.field)
+ if frappe.db.exists('User', val):
+ return val
+
def safe_eval(self, fieldname, doc):
try:
if self.get(fieldname):
diff --git a/frappe/build.py b/frappe/build.py
index f14b250a92..f47a7cb32b 100644
--- a/frappe/build.py
+++ b/frappe/build.py
@@ -105,7 +105,7 @@ def download_frappe_assets(verbose=True):
if frappe_head:
try:
url = get_assets_link(frappe_head)
- click.secho("Retreiving assets...", fg="yellow")
+ click.secho("Retrieving assets...", fg="yellow")
prefix = mkdtemp(prefix="frappe-assets-", suffix=frappe_head)
assets_archive = download_file(url, prefix)
print("\n{0} Downloaded Frappe assets from {1}".format(green('✔'), url))
diff --git a/frappe/commands/site.py b/frappe/commands/site.py
index 2be566f85f..bc65aa178c 100755
--- a/frappe/commands/site.py
+++ b/frappe/commands/site.py
@@ -9,7 +9,7 @@ import click
import frappe
from frappe.commands import get_site, pass_context
from frappe.exceptions import SiteNotSpecifiedError
-from frappe.utils import get_site_path, touch_file
+from frappe.installer import _new_site
@click.command('new-site')
@@ -42,57 +42,6 @@ def new_site(site, mariadb_root_username=None, mariadb_root_password=None, admin
if len(frappe.utils.get_sites()) == 1:
use(site)
-def _new_site(db_name, site, mariadb_root_username=None, mariadb_root_password=None,
- admin_password=None, verbose=False, install_apps=None, source_sql=None, force=False,
- no_mariadb_socket=False, reinstall=False, db_password=None, db_type=None, db_host=None,
- db_port=None, new_site=False):
- """Install a new Frappe site"""
-
- if not force and os.path.exists(site):
- print('Site {0} already exists'.format(site))
- sys.exit(1)
-
- if no_mariadb_socket and not db_type == "mariadb":
- print('--no-mariadb-socket requires db_type to be set to mariadb.')
- sys.exit(1)
-
- if not db_name:
- import hashlib
- db_name = '_' + hashlib.sha1(site.encode()).hexdigest()[:16]
-
- from frappe.commands.scheduler import _is_scheduler_enabled
- from frappe.installer import install_db, make_site_dirs
- from frappe.installer import install_app as _install_app
- import frappe.utils.scheduler
-
- frappe.init(site=site)
-
- try:
-
- # enable scheduler post install?
- enable_scheduler = _is_scheduler_enabled()
- except Exception:
- enable_scheduler = False
-
- make_site_dirs()
-
- installing = touch_file(get_site_path('locks', 'installing.lock'))
-
- install_db(root_login=mariadb_root_username, root_password=mariadb_root_password, db_name=db_name,
- admin_password=admin_password, verbose=verbose, source_sql=source_sql, force=force, reinstall=reinstall,
- db_password=db_password, db_type=db_type, db_host=db_host, db_port=db_port, no_mariadb_socket=no_mariadb_socket)
- apps_to_install = ['frappe'] + (frappe.conf.get("install_apps") or []) + (list(install_apps) or [])
- for app in apps_to_install:
- _install_app(app, verbose=verbose, set_as_patched=not source_sql)
-
- os.remove(installing)
-
- frappe.utils.scheduler.toggle_scheduler(enable_scheduler)
- frappe.db.commit()
-
- scheduler_status = "disabled" if frappe.utils.scheduler.is_scheduler_disabled() else "enabled"
- print("*** Scheduler is", scheduler_status, "***")
-
@click.command('restore')
@click.argument('sql-file-path')
@@ -107,33 +56,41 @@ def _new_site(db_name, site, mariadb_root_username=None, mariadb_root_password=N
@pass_context
def restore(context, sql_file_path, mariadb_root_username=None, mariadb_root_password=None, db_name=None, verbose=None, install_app=None, admin_password=None, force=None, with_public_files=None, with_private_files=None):
"Restore site database from an sql file"
- from frappe.installer import extract_sql_gzip, extract_files, is_downgrade, validate_database_sql
+ from frappe.installer import (
+ extract_sql_from_archive,
+ extract_files,
+ is_downgrade,
+ is_partial,
+ validate_database_sql
+ )
+
force = context.force or force
+ decompressed_file_name = extract_sql_from_archive(sql_file_path)
- # Extract the gzip file if user has passed *.sql.gz file instead of *.sql file
- if not os.path.exists(sql_file_path):
- base_path = '..'
- sql_file_path = os.path.join(base_path, sql_file_path)
- if not os.path.exists(sql_file_path):
- print('Invalid path {0}'.format(sql_file_path[3:]))
- sys.exit(1)
- elif sql_file_path.startswith(os.sep):
- base_path = os.sep
- else:
- base_path = '.'
+ # check if partial backup
+ if is_partial(decompressed_file_name):
+ click.secho(
+ "Partial Backup file detected. You cannot use a partial file to restore a Frappe Site.",
+ fg="red"
+ )
+ click.secho(
+ "Use `bench partial-restore` to restore a partial backup to an existing site.",
+ fg="yellow"
+ )
+ sys.exit(1)
- if sql_file_path.endswith('sql.gz'):
- decompressed_file_name = extract_sql_gzip(os.path.abspath(sql_file_path))
- else:
- decompressed_file_name = sql_file_path
+ # check if valid SQL file
+ validate_database_sql(decompressed_file_name, _raise=not force)
- validate_database_sql(decompressed_file_name, _raise=force)
site = get_site(context)
frappe.init(site=site)
# dont allow downgrading to older versions of frappe without force
if not force and is_downgrade(decompressed_file_name, verbose=True):
- warn_message = "This is not recommended and may lead to unexpected behaviour. Do you want to continue anyway?"
+ warn_message = (
+ "This is not recommended and may lead to unexpected behaviour. "
+ "Do you want to continue anyway?"
+ )
click.confirm(warn_message, abort=True)
_new_site(frappe.conf.db_name, site, mariadb_root_username=mariadb_root_username,
@@ -156,9 +113,28 @@ def restore(context, sql_file_path, mariadb_root_username=None, mariadb_root_pas
if decompressed_file_name != sql_file_path:
os.remove(decompressed_file_name)
- success_message = "Site {0} has been restored{1}".format(site, " with files" if (with_public_files or with_private_files) else "")
+ success_message = "Site {0} has been restored{1}".format(
+ site,
+ " with files" if (with_public_files or with_private_files) else ""
+ )
click.secho(success_message, fg="green")
+
+@click.command('partial-restore')
+@click.argument('sql-file-path')
+@click.option("--verbose", "-v", is_flag=True)
+@pass_context
+def partial_restore(context, sql_file_path, verbose):
+ from frappe.installer import partial_restore
+ verbose = context.verbose or verbose
+
+ site = get_site(context)
+ frappe.init(site=site)
+ frappe.connect(site=site)
+ partial_restore(sql_file_path, verbose)
+ frappe.destroy()
+
+
@click.command('reinstall')
@click.option('--admin-password', help='Administrator Password for reinstalled site')
@click.option('--mariadb-root-username', help='Root username for MariaDB')
@@ -416,16 +392,20 @@ def use(site, sites_path='.'):
@click.command('backup')
@click.option('--with-files', default=False, is_flag=True, help="Take backup with files")
+@click.option('--include', '--only', '-i', default="", type=str, help="Specify the DocTypes to backup seperated by commas")
+@click.option('--exclude', '-e', default="", type=str, help="Specify the DocTypes to not backup seperated by commas")
@click.option('--backup-path', default=None, help="Set path for saving all the files in this operation")
@click.option('--backup-path-db', default=None, help="Set path for saving database file")
@click.option('--backup-path-files', default=None, help="Set path for saving public file")
@click.option('--backup-path-private-files', default=None, help="Set path for saving private file")
@click.option('--backup-path-conf', default=None, help="Set path for saving config file")
+@click.option('--ignore-backup-conf', default=False, is_flag=True, help="Ignore excludes/includes set in config")
@click.option('--verbose', default=False, is_flag=True, help="Add verbosity")
@click.option('--compress', default=False, is_flag=True, help="Compress private and public files")
@pass_context
def backup(context, with_files=False, backup_path=None, backup_path_db=None, backup_path_files=None,
- backup_path_private_files=None, backup_path_conf=None, verbose=False, compress=False):
+ backup_path_private_files=None, backup_path_conf=None, ignore_backup_conf=False, verbose=False,
+ compress=False, include="", exclude=""):
"Backup"
from frappe.utils.backups import scheduled_backup
verbose = verbose or context.verbose
@@ -435,11 +415,27 @@ def backup(context, with_files=False, backup_path=None, backup_path_db=None, bac
try:
frappe.init(site=site)
frappe.connect()
- odb = scheduled_backup(ignore_files=not with_files, backup_path=backup_path, backup_path_db=backup_path_db, backup_path_files=backup_path_files, backup_path_private_files=backup_path_private_files, backup_path_conf=backup_path_conf, force=True, verbose=verbose, compress=compress)
+ odb = scheduled_backup(
+ ignore_files=not with_files,
+ backup_path=backup_path,
+ backup_path_db=backup_path_db,
+ backup_path_files=backup_path_files,
+ backup_path_private_files=backup_path_private_files,
+ backup_path_conf=backup_path_conf,
+ ignore_conf=ignore_backup_conf,
+ include_doctypes=include,
+ exclude_doctypes=exclude,
+ compress=compress,
+ verbose=verbose,
+ force=True
+ )
except Exception:
click.secho("Backup failed for Site {0}. Database or site_config.json may be corrupted".format(site), fg="red")
+ if verbose:
+ print(frappe.get_traceback())
exit_code = 1
continue
+
odb.print_summary()
click.secho("Backup for Site {0} has been successfully completed{1}".format(site, " with files" if with_files else ""), fg="green")
frappe.destroy()
@@ -512,13 +508,14 @@ def _drop_site(site, root_login='root', root_password=None, archived_sites_path=
if force:
pass
else:
- click.echo("="*80)
- click.echo("Error: The operation has stopped because backup of {s}'s database failed.".format(s=site))
- click.echo("Reason: {reason}{sep}".format(reason=str(err), sep="\n"))
- click.echo("Fix the issue and try again.")
- click.echo(
- "Hint: Use 'bench drop-site {s} --force' to force the removal of {s}".format(sep="\n", tab="\t", s=site)
- )
+ messages = [
+ "=" * 80,
+ "Error: The operation has stopped because backup of {0}'s database failed.".format(site),
+ "Reason: {0}\n".format(str(err)),
+ "Fix the issue and try again.",
+ "Hint: Use 'bench drop-site {0} --force' to force the removal of {0}".format(site)
+ ]
+ click.echo("\n".join(messages))
sys.exit(1)
drop_user_and_database(frappe.conf.db_name, root_login, root_password)
@@ -734,5 +731,6 @@ commands = [
stop_recording,
add_to_hosts,
start_ngrok,
- build_search_index
+ build_search_index,
+ partial_restore
]
diff --git a/frappe/core/doctype/doctype/doctype.py b/frappe/core/doctype/doctype/doctype.py
index fd0cb1917d..cb3d06a29a 100644
--- a/frappe/core/doctype/doctype/doctype.py
+++ b/frappe/core/doctype/doctype/doctype.py
@@ -572,7 +572,8 @@ class DocType(Document):
def make_repeatable(self):
"""If allow_auto_repeat is set, add auto_repeat custom field."""
if self.allow_auto_repeat:
- if not frappe.db.exists('Custom Field', {'fieldname': 'auto_repeat', 'dt': self.name}):
+ if not frappe.db.exists('Custom Field', {'fieldname': 'auto_repeat', 'dt': self.name}) and \
+ not frappe.db.exists('DocField', {'fieldname': 'auto_repeat', 'parent': self.name}):
insert_after = self.fields[len(self.fields) - 1].fieldname
df = dict(fieldname='auto_repeat', label='Auto Repeat', fieldtype='Link', options='Auto Repeat', insert_after=insert_after, read_only=1, no_copy=1, print_hide=1)
create_custom_field(self.name, df)
diff --git a/frappe/core/doctype/domain_settings/domain_settings.js b/frappe/core/doctype/domain_settings/domain_settings.js
index 1428727993..7178cb4cd6 100644
--- a/frappe/core/doctype/domain_settings/domain_settings.js
+++ b/frappe/core/doctype/domain_settings/domain_settings.js
@@ -18,6 +18,9 @@ frappe.ui.form.on('Domain Settings', {
checked: active_domains.includes(domain)
};
});
+ },
+ on_change: () => {
+ frm.dirty();
}
},
render_input: true
diff --git a/frappe/core/doctype/file/file.py b/frappe/core/doctype/file/file.py
index b8bed89a4d..473d810a9f 100755
--- a/frappe/core/doctype/file/file.py
+++ b/frappe/core/doctype/file/file.py
@@ -93,6 +93,7 @@ class File(Document):
self.set_is_private()
self.set_file_name()
self.validate_duplicate_entry()
+ self.validate_attachment_limit()
self.validate_folder()
if not self.file_url and not self.flags.ignore_file_validate:
@@ -140,6 +141,26 @@ class File(Document):
if self.file_url and (self.is_private != self.file_url.startswith('/private')):
frappe.throw(_('Invalid file URL. Please contact System Administrator.'))
+ def validate_attachment_limit(self):
+ attachment_limit = 0
+ if self.attached_to_doctype and self.attached_to_name:
+ attachment_limit = cint(frappe.get_meta(self.attached_to_doctype).max_attachments)
+
+ if attachment_limit:
+ current_attachment_count = len(frappe.get_all('File', filters={
+ 'attached_to_doctype': self.attached_to_doctype,
+ 'attached_to_name': self.attached_to_name,
+ }, limit=attachment_limit + 1))
+
+ if current_attachment_count >= attachment_limit:
+ frappe.throw(
+ _("Maximum Attachment Limit of {0} has been reached for {1} {2}.").format(
+ frappe.bold(attachment_limit), self.attached_to_doctype, self.attached_to_name
+ ),
+ exc=frappe.exceptions.AttachmentLimitReached,
+ title=_('Attachment Limit Reached')
+ )
+
def set_folder_name(self):
"""Make parent folders if not exists based on reference doctype and name"""
if self.attached_to_doctype and not self.folder:
@@ -612,7 +633,12 @@ def get_extension(filename, extn, content):
return extn
def get_local_image(file_url):
- file_path = frappe.get_site_path("public", file_url.lstrip("/"))
+ if file_url.startswith("/private"):
+ file_url_path = (file_url.lstrip("/"), )
+ else:
+ file_url_path = ("public", file_url.lstrip("/"))
+
+ file_path = frappe.get_site_path(*file_url_path)
try:
image = Image.open(file_path)
diff --git a/frappe/core/doctype/file/test_file.py b/frappe/core/doctype/file/test_file.py
index 85397ea1ee..e627558680 100644
--- a/frappe/core/doctype/file/test_file.py
+++ b/frappe/core/doctype/file/test_file.py
@@ -160,6 +160,31 @@ class TestSameContent(unittest.TestCase):
def test_saved_content(self):
self.assertFalse(os.path.exists(get_files_path(self.dup_filename)))
+ def test_attachment_limit(self):
+ doctype, docname = make_test_doc()
+ from frappe.custom.doctype.property_setter.property_setter import make_property_setter
+ limit_property = make_property_setter('ToDo', None, 'max_attachments', 1, 'int', for_doctype=True)
+ file1 = frappe.get_doc({
+ "doctype": "File",
+ "file_name": 'test-attachment',
+ "attached_to_doctype": doctype,
+ "attached_to_name": docname,
+ "content": 'test'
+ })
+
+ file1.insert()
+
+ file2 = frappe.get_doc({
+ "doctype": "File",
+ "file_name": 'test-attachment',
+ "attached_to_doctype": doctype,
+ "attached_to_name": docname,
+ "content": 'test2'
+ })
+
+ self.assertRaises(frappe.exceptions.AttachmentLimitReached, file2.insert)
+ limit_property.delete()
+ frappe.clear_cache(doctype='ToDo')
def tearDown(self):
# File gets deleted on rollback, so blank
diff --git a/frappe/core/doctype/prepared_report/prepared_report.py b/frappe/core/doctype/prepared_report/prepared_report.py
index 2c02d99dad..1d0d6ebb09 100644
--- a/frappe/core/doctype/prepared_report/prepared_report.py
+++ b/frappe/core/doctype/prepared_report/prepared_report.py
@@ -89,20 +89,18 @@ def delete_expired_prepared_reports():
'creation': ['<', frappe.utils.add_days(frappe.utils.now(), -expiry_period)]
})
- args = {
- 'reports': prepared_reports_to_delete,
- 'limit': 50
- }
-
- enqueue(method=delete_prepared_reports, job_name="delete_prepared_reports", **args)
+ batches = frappe.utils.create_batch(prepared_reports_to_delete, 100)
+ for batch in batches:
+ args = {
+ 'reports': batch,
+ }
+ enqueue(method=delete_prepared_reports, job_name="delete_prepared_reports", **args)
@frappe.whitelist()
-def delete_prepared_reports(reports, limit=None):
+def delete_prepared_reports(reports):
reports = frappe.parse_json(reports)
- for index, doc in enumerate(reports):
- if limit and index == limit:
- return
- frappe.delete_doc('Prepared Report', doc['name'], ignore_permissions=True)
+ for report in reports:
+ frappe.delete_doc('Prepared Report', report['name'], ignore_permissions=True)
def create_json_gz_file(data, dt, dn):
# Storing data in CSV file causes information loss
diff --git a/frappe/core/doctype/report/report.py b/frappe/core/doctype/report/report.py
index 9d30409a2a..01c32bcb57 100644
--- a/frappe/core/doctype/report/report.py
+++ b/frappe/core/doctype/report/report.py
@@ -61,8 +61,9 @@ class Report(Document):
def set_doctype_roles(self):
if not self.get('roles') and self.is_standard == 'No':
meta = frappe.get_meta(self.ref_doctype)
- roles = [{'role': d.role} for d in meta.permissions if d.permlevel==0]
- self.set('roles', roles)
+ if not meta.istable:
+ roles = [{'role': d.role} for d in meta.permissions if d.permlevel==0]
+ self.set('roles', roles)
def is_permitted(self):
"""Returns true if Has Role is not set or the user is allowed."""
diff --git a/frappe/core/doctype/role/role.py b/frappe/core/doctype/role/role.py
index e458b401e4..1920189f78 100644
--- a/frappe/core/doctype/role/role.py
+++ b/frappe/core/doctype/role/role.py
@@ -37,7 +37,7 @@ class Role(Document):
def get_info_based_on_role(role, field='email'):
''' Get information of all users that have been assigned this role '''
users = frappe.get_list("Has Role", filters={"role": role, "parenttype": "User"},
- fields=["parent"])
+ fields=["parent as user_name"])
return get_user_info(users, field)
@@ -45,7 +45,7 @@ def get_user_info(users, field='email'):
''' Fetch details about users for the specified field '''
info_list = []
for user in users:
- user_info, enabled = frappe.db.get_value("User", user.parent, [field, "enabled"])
+ user_info, enabled = frappe.db.get_value("User", user.get("user_name"), [field, "enabled"])
if enabled and user_info not in ["admin@example.com", "guest@example.com"]:
info_list.append(user_info)
return info_list
diff --git a/frappe/core/doctype/server_script/server_script.json b/frappe/core/doctype/server_script/server_script.json
index cc3995ad1d..420f96ec2f 100644
--- a/frappe/core/doctype/server_script/server_script.json
+++ b/frappe/core/doctype/server_script/server_script.json
@@ -31,6 +31,7 @@
"fieldname": "script",
"fieldtype": "Code",
"label": "Script",
+ "options": "Python",
"reqd": 1
},
{
@@ -87,7 +88,7 @@
],
"index_web_pages_for_search": 1,
"links": [],
- "modified": "2020-08-24 16:44:41.060350",
+ "modified": "2020-11-11 12:39:41.391052",
"modified_by": "Administrator",
"module": "Core",
"name": "Server Script",
diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py
index 2c5865fb69..7309528da6 100644
--- a/frappe/core/doctype/user/user.py
+++ b/frappe/core/doctype/user/user.py
@@ -13,7 +13,7 @@ from frappe.utils.user import get_system_managers
from bs4 import BeautifulSoup
import frappe.permissions
import frappe.share
-
+import frappe.defaults
from frappe.website.utils import is_signup_enabled
from frappe.utils.background_jobs import enqueue
@@ -107,6 +107,10 @@ class User(Document):
)
if self.name not in ('Administrator', 'Guest') and not self.user_image:
frappe.enqueue('frappe.core.doctype.user.user.update_gravatar', name=self.name)
+
+ # Set user selected timezone
+ if self.time_zone:
+ frappe.defaults.set_default("time_zone", self.time_zone, self.name)
def has_website_permission(self, ptype, user, verbose=False):
"""Returns true if current user is the session user"""
@@ -1129,4 +1133,4 @@ def check_password_reset_limit(user, rate_limit):
frappe.throw(_("You have reached the hourly limit for generating password reset links. Please try again later."))
def get_generated_link_count(user):
- return cint(frappe.cache().hget("password_reset_link_count", user)) or 0
\ No newline at end of file
+ return cint(frappe.cache().hget("password_reset_link_count", user)) or 0
diff --git a/frappe/custom/doctype/customize_form/customize_form.py b/frappe/custom/doctype/customize_form/customize_form.py
index 9ce602906c..82513783c7 100644
--- a/frappe/custom/doctype/customize_form/customize_form.py
+++ b/frappe/custom/doctype/customize_form/customize_form.py
@@ -39,7 +39,7 @@ class CustomizeForm(Document):
translation = self.get_name_translation()
self.label = translation.translated_text if translation else ''
- self.create_auto_repeat_custom_field_if_requried(meta)
+ self.create_auto_repeat_custom_field_if_required(meta)
# NOTE doc (self) is sent to clientside by run_method
@@ -74,19 +74,25 @@ class CustomizeForm(Document):
for d in meta.get(fieldname):
self.append(fieldname, d)
- def create_auto_repeat_custom_field_if_requried(self, meta):
+ def create_auto_repeat_custom_field_if_required(self, meta):
+ '''
+ Create auto repeat custom field if it's not already present
+ '''
if self.allow_auto_repeat:
- if not frappe.db.exists('Custom Field', {'fieldname': 'auto_repeat',
- 'dt': self.doc_type}):
- insert_after = self.fields[len(self.fields) - 1].fieldname
- df = dict(
- fieldname='auto_repeat',
- label='Auto Repeat',
- fieldtype='Link',
- options='Auto Repeat',
- insert_after=insert_after,
- read_only=1, no_copy=1, print_hide=1)
- create_custom_field(self.doc_type, df)
+ all_fields = [df.fieldname for df in meta.fields]
+
+ if "auto_repeat" in all_fields:
+ return
+
+ insert_after = self.fields[len(self.fields) - 1].fieldname
+ create_custom_field(self.doc_type, dict(
+ fieldname='auto_repeat',
+ label='Auto Repeat',
+ fieldtype='Link',
+ options='Auto Repeat',
+ insert_after=insert_after,
+ read_only=1, no_copy=1, print_hide=1
+ ))
def get_name_translation(self):
diff --git a/frappe/database/db_manager.py b/frappe/database/db_manager.py
index 3345fce735..b8ffae519b 100644
--- a/frappe/database/db_manager.py
+++ b/frappe/database/db_manager.py
@@ -3,7 +3,6 @@ import frappe
class DbManager:
-
def __init__(self, db):
"""
Pass root_conn here for access to all databases.
@@ -66,10 +65,10 @@ class DbManager:
esc = make_esc('$ ')
from distutils.spawn import find_executable
- pipe = find_executable('pv')
- if pipe:
- pipe = '{pipe} {source} |'.format(
- pipe=pipe,
+ pv = find_executable('pv')
+ if pv:
+ pipe = '{pv} {source} |'.format(
+ pv=pv,
source=source
)
source = ''
@@ -78,7 +77,7 @@ class DbManager:
source = '< {source}'.format(source=source)
if pipe:
- print('Creating Database...')
+ print('Restoring Database file...')
command = '{pipe} mysql -u {user} -p{password} -h{host} ' + ('-P{port}' if frappe.db.port else '') + ' {target} {source}'
command = command.format(
diff --git a/frappe/database/mariadb/setup_db.py b/frappe/database/mariadb/setup_db.py
index a4e4d624ae..9b73d77171 100644
--- a/frappe/database/mariadb/setup_db.py
+++ b/frappe/database/mariadb/setup_db.py
@@ -1,7 +1,7 @@
from __future__ import unicode_literals
import frappe
-import os, sys
+import os
from frappe.database.db_manager import DbManager
expected_settings_10_2_earlier = {
@@ -86,6 +86,8 @@ def drop_user_and_database(db_name, root_login, root_password):
dbman.drop_database(db_name)
def bootstrap_database(db_name, verbose, source_sql=None):
+ import sys
+
frappe.connect(db_name=db_name)
if not check_database_settings():
print('Database settings do not match expected values; stopping database setup.')
@@ -94,9 +96,17 @@ def bootstrap_database(db_name, verbose, source_sql=None):
import_db_from_sql(source_sql, verbose)
frappe.connect(db_name=db_name)
- if not 'tabDefaultValue' in frappe.db.get_tables():
- print('''Database not installed, this can due to lack of permission, or that the database name exists.
- Check your mysql root password, or use --force to reinstall''')
+ if 'tabDefaultValue' not in frappe.db.get_tables():
+ from click import secho
+
+ secho(
+ "Table 'tabDefaultValue' missing in the restored site. "
+ "Database not installed correctly, this can due to lack of "
+ "permission, or that the database name exists. Check your mysql"
+ " root password, validity of the backup file or use --force to"
+ " reinstall",
+ fg="red"
+ )
sys.exit(1)
def import_db_from_sql(source_sql=None, verbose=False):
diff --git a/frappe/database/postgres/setup_db.py b/frappe/database/postgres/setup_db.py
index f53872db82..3ee6b6a286 100644
--- a/frappe/database/postgres/setup_db.py
+++ b/frappe/database/postgres/setup_db.py
@@ -1,5 +1,7 @@
-import frappe, subprocess, os
-from six.moves import input
+import os
+
+import frappe
+
def setup_database(force, source_sql=None, verbose=False):
root_conn = get_root_connection()
@@ -10,24 +12,62 @@ def setup_database(force, source_sql=None, verbose=False):
root_conn.sql("CREATE user {0} password '{1}'".format(frappe.conf.db_name,
frappe.conf.db_password))
root_conn.sql("GRANT ALL PRIVILEGES ON DATABASE `{0}` TO {0}".format(frappe.conf.db_name))
+ root_conn.close()
+
+ bootstrap_database(frappe.conf.db_name, verbose, source_sql=source_sql)
+ frappe.connect()
+
+def bootstrap_database(db_name, verbose, source_sql=None):
+ frappe.connect(db_name=db_name)
+ import_db_from_sql(source_sql, verbose)
+ frappe.connect(db_name=db_name)
+
+ if 'tabDefaultValue' not in frappe.db.get_tables():
+ import sys
+ from click import secho
+
+ secho(
+ "Table 'tabDefaultValue' missing in the restored site. "
+ "This may be due to incorrect permissions or the result of a restore from a bad backup file. "
+ "Database not installed correctly.",
+ fg="red"
+ )
+ sys.exit(1)
+
+def import_db_from_sql(source_sql=None, verbose=False):
+ from shutil import which
+ from subprocess import run, PIPE
# we can't pass psql password in arguments in postgresql as mysql. So
# set password connection parameter in environment variable
subprocess_env = os.environ.copy()
subprocess_env['PGPASSWORD'] = str(frappe.conf.db_password)
+
# bootstrap db
if not source_sql:
source_sql = os.path.join(os.path.dirname(__file__), 'framework_postgres.sql')
- subprocess.check_output([
- 'psql', frappe.conf.db_name,
- '-h', frappe.conf.db_host or 'localhost',
- '-p', str(frappe.conf.db_port or '5432'),
- '-U', frappe.conf.db_name,
- '-f', source_sql
- ], env=subprocess_env)
+ pv = which('pv')
- frappe.connect()
+ _command = (
+ f"psql {frappe.conf.db_name} "
+ f"-h {frappe.conf.db_host or 'localhost'} -p {str(frappe.conf.db_port or '5432')} "
+ f"-U {frappe.conf.db_name}"
+ )
+
+ if pv:
+ command = f"{pv} {source_sql} | " + _command
+ else:
+ command = _command + f" -f {source_sql}"
+
+ print("Restoring Database file...")
+ if verbose:
+ print(command)
+
+ restore_proc = run(command, env=subprocess_env, shell=True, stdout=PIPE)
+
+ if verbose:
+ print(f"\nSTDOUT by psql:\n{restore_proc.stdout.decode()}\nImported from Database File: {source_sql}")
def setup_help_database(help_db_name):
root_conn = get_root_connection()
@@ -38,19 +78,20 @@ def setup_help_database(help_db_name):
root_conn.sql("GRANT ALL PRIVILEGES ON DATABASE `{0}` TO {0}".format(help_db_name))
def get_root_connection(root_login=None, root_password=None):
- import getpass
if not frappe.local.flags.root_connection:
if not root_login:
root_login = frappe.conf.get("root_login") or None
if not root_login:
+ from six.moves import input
root_login = input("Enter postgres super user: ")
if not root_password:
root_password = frappe.conf.get("root_password") or None
if not root_password:
- root_password = getpass.getpass("Postgres super user password: ")
+ from getpass import getpass
+ root_password = getpass("Postgres super user password: ")
frappe.local.flags.root_connection = frappe.database.get_db(user=root_login, password=root_password)
diff --git a/frappe/desk/doctype/dashboard_chart/dashboard_chart.py b/frappe/desk/doctype/dashboard_chart/dashboard_chart.py
index 7e2d952928..3f8d7c3c79 100644
--- a/frappe/desk/doctype/dashboard_chart/dashboard_chart.py
+++ b/frappe/desk/doctype/dashboard_chart/dashboard_chart.py
@@ -7,9 +7,10 @@ import frappe
from frappe import _
import datetime
import json
-from frappe.utils.dashboard import cache_source, get_from_date_from_timespan
-from frappe.utils import nowdate, add_to_date, getdate, get_last_day, formatdate,\
- get_datetime, cint, now_datetime
+from frappe.utils.dashboard import cache_source
+from frappe.utils import nowdate, getdate, get_datetime, cint, now_datetime
+from frappe.utils.dateutils import\
+ get_period, get_period_beginning, get_from_date_from_timespan, get_dates_from_timegrain
from frappe.model.naming import append_number_if_name_exists
from frappe.boot import get_allowed_reports
from frappe.model.document import Document
@@ -156,6 +157,7 @@ def add_chart_to_dashboard(args):
def get_chart_config(chart, filters, timespan, timegrain, from_date, to_date):
if not from_date:
from_date = get_from_date_from_timespan(to_date, timespan)
+ from_date = get_period_beginning(from_date, timegrain)
if not to_date:
to_date = now_datetime()
@@ -185,7 +187,7 @@ def get_chart_config(chart, filters, timespan, timegrain, from_date, to_date):
result = get_result(data, timegrain, from_date, to_date)
chart_config = {
- "labels": [formatdate(r[0].strftime('%Y-%m-%d')) for r in result],
+ "labels": [get_period(r[0], timegrain) for r in result],
"datasets": [{
"name": chart.name,
"values": [r[1] for r in result]
@@ -279,16 +281,8 @@ def get_aggregate_function(chart_type):
def get_result(data, timegrain, from_date, to_date):
- start_date = getdate(from_date)
- end_date = getdate(to_date)
-
- result = [[start_date, 0.0]]
-
- while start_date < end_date:
- next_date = get_next_expected_date(start_date, timegrain)
- result.append([next_date, 0.0])
- start_date = next_date
-
+ dates = get_dates_from_timegrain(from_date, to_date, timegrain)
+ result = [[date, 0] for date in dates]
data_index = 0
if data:
for i, d in enumerate(result):
@@ -298,65 +292,6 @@ def get_result(data, timegrain, from_date, to_date):
return result
-def get_next_expected_date(date, timegrain):
- next_date = None
- # given date is always assumed to be the period ending date
- next_date = get_period_ending(add_to_date(date, days=1), timegrain)
- return getdate(next_date)
-
-def get_period_ending(date, timegrain):
- date = getdate(date)
- if timegrain == 'Daily':
- pass
- elif timegrain == 'Weekly':
- date = get_week_ending(date)
- elif timegrain == 'Monthly':
- date = get_month_ending(date)
- elif timegrain == 'Quarterly':
- date = get_quarter_ending(date)
- elif timegrain == 'Yearly':
- date = get_year_ending(date)
-
- return getdate(date)
-
-def get_week_ending(date):
- # week starts on monday
- from datetime import timedelta
- start = date - timedelta(days = date.weekday())
- end = start + timedelta(days=6)
-
- return end
-
-def get_month_ending(date):
- month_of_the_year = int(date.strftime('%m'))
- # first day of next month (note month starts from 1)
-
- date = add_to_date('{}-01-01'.format(date.year), months = month_of_the_year)
- # last day of this month
- return add_to_date(date, days=-1)
-
-def get_quarter_ending(date):
- date = getdate(date)
-
- # find the earliest quarter ending date that is after
- # the given date
- for month in (3, 6, 9, 12):
- quarter_end_month = getdate('{}-{}-01'.format(date.year, month))
- quarter_end_date = getdate(get_last_day(quarter_end_month))
- if date <= quarter_end_date:
- date = quarter_end_date
- break
-
- return date
-
-def get_year_ending(date):
- ''' returns year ending of the given date '''
-
- # first day of next year (note year starts from 1)
- date = add_to_date('{}-01-01'.format(date.year), months = 12)
- # last day of this month
- return add_to_date(date, days=-1)
-
@frappe.whitelist()
@frappe.validate_and_sanitize_search_inputs
def get_charts_for_user(doctype, txt, searchfield, start, page_len, filters):
diff --git a/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py b/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py
index 5e39998e62..3c37ad4a09 100644
--- a/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py
+++ b/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py
@@ -5,8 +5,8 @@ from __future__ import unicode_literals
import unittest, frappe
from frappe.utils import getdate, formatdate, get_last_day
-from frappe.desk.doctype.dashboard_chart.dashboard_chart import (get,
- get_period_ending)
+from frappe.utils.dateutils import get_period_ending, get_period
+from frappe.desk.doctype.dashboard_chart.dashboard_chart import get
from datetime import datetime
from dateutil.relativedelta import relativedelta
@@ -53,15 +53,11 @@ class TestDashboardChart(unittest.TestCase):
cur_date = datetime.now() - relativedelta(years=1)
result = get(chart_name='Test Dashboard Chart', refresh=1)
- self.assertEqual(result.get('labels')[0], formatdate(cur_date.strftime('%Y-%m-%d')))
- if formatdate(cur_date.strftime('%Y-%m-%d')) == formatdate(get_last_day(cur_date).strftime('%Y-%m-%d')):
- cur_date += relativedelta(months=1)
-
- for idx in range(1, 13):
+ for idx in range(13):
month = get_last_day(cur_date)
month = formatdate(month.strftime('%Y-%m-%d'))
- self.assertEqual(result.get('labels')[idx], month)
+ self.assertEqual(result.get('labels')[idx], get_period(month))
cur_date += relativedelta(months=1)
frappe.db.rollback()
@@ -87,15 +83,11 @@ class TestDashboardChart(unittest.TestCase):
cur_date = datetime.now() - relativedelta(years=1)
result = get(chart_name ='Test Empty Dashboard Chart', refresh=1)
- self.assertEqual(result.get('labels')[0], formatdate(cur_date.strftime('%Y-%m-%d')))
- if formatdate(cur_date.strftime('%Y-%m-%d')) == formatdate(get_last_day(cur_date).strftime('%Y-%m-%d')):
- cur_date += relativedelta(months=1)
-
- for idx in range(1, 13):
+ for idx in range(13):
month = get_last_day(cur_date)
month = formatdate(month.strftime('%Y-%m-%d'))
- self.assertEqual(result.get('labels')[idx], month)
+ self.assertEqual(result.get('labels')[idx], get_period(month))
cur_date += relativedelta(months=1)
frappe.db.rollback()
@@ -124,15 +116,11 @@ class TestDashboardChart(unittest.TestCase):
cur_date = datetime.now() - relativedelta(years=1)
result = get(chart_name ='Test Empty Dashboard Chart 2', refresh = 1)
- self.assertEqual(result.get('labels')[0], formatdate(cur_date.strftime('%Y-%m-%d')))
- if formatdate(cur_date.strftime('%Y-%m-%d')) == formatdate(get_last_day(cur_date).strftime('%Y-%m-%d')):
- cur_date += relativedelta(months=1)
-
- for idx in range(1, 13):
+ for idx in range(13):
month = get_last_day(cur_date)
month = formatdate(month.strftime('%Y-%m-%d'))
- self.assertEqual(result.get('labels')[idx], month)
+ self.assertEqual(result.get('labels')[idx], get_period(month))
cur_date += relativedelta(months=1)
# only 1 data point with value
@@ -183,13 +171,12 @@ class TestDashboardChart(unittest.TestCase):
timeseries = 1
)).insert()
- result = get(chart_name ='Test Daily Dashboard Chart', refresh = 1)
+ result = get(chart_name = 'Test Daily Dashboard Chart', refresh = 1)
self.assertEqual(result.get('datasets')[0].get('values'), [200.0, 400.0, 300.0, 0.0, 100.0, 0.0])
self.assertEqual(
result.get('labels'),
- [formatdate('2019-01-06'), formatdate('2019-01-07'), formatdate('2019-01-08'),\
- formatdate('2019-01-09'), formatdate('2019-01-10'), formatdate('2019-01-11')]
+ ['06-01-19', '07-01-19', '08-01-19', '09-01-19', '10-01-19', '11-01-19']
)
frappe.db.rollback()
@@ -218,7 +205,10 @@ class TestDashboardChart(unittest.TestCase):
result = get(chart_name ='Test Weekly Dashboard Chart', refresh = 1)
self.assertEqual(result.get('datasets')[0].get('values'), [50.0, 300.0, 800.0, 0.0])
- self.assertEqual(result.get('labels'), [formatdate('2018-12-30'), formatdate('2019-01-06'), formatdate('2019-01-13'), formatdate('2019-01-20')])
+ self.assertEqual(
+ result.get('labels'),
+ ['30-12-18', '06-01-19', '13-01-19', '20-01-19']
+ )
frappe.db.rollback()
diff --git a/frappe/desk/form/document_follow.py b/frappe/desk/form/document_follow.py
index 3aa3a4fa88..66164948f2 100644
--- a/frappe/desk/form/document_follow.py
+++ b/frappe/desk/form/document_follow.py
@@ -21,7 +21,7 @@ def follow_document(doctype, doc_name, user, force=False):
avoided for some doctype
follow only if track changes are set to 1
'''
- if (doctype in ("Communication", "ToDo", "Email Unsubscribe", "File", "Comment")
+ if (doctype in ("Communication", "ToDo", "Email Unsubscribe", "File", "Comment", "Email Account", "Email Domain")
or doctype in log_types):
return
diff --git a/frappe/email/doctype/notification/notification.js b/frappe/email/doctype/notification/notification.js
index 2cc027acd6..27fcd0e453 100644
--- a/frappe/email/doctype/notification/notification.js
+++ b/frappe/email/doctype/notification/notification.js
@@ -97,14 +97,7 @@ frappe.notification = {
},
setup_example_message: function(frm) {
let template = '';
- if (frm.doc.channel === 'WhatsApp') {
- template = `Warning:
Only Use Pre-Approved WhatsApp for Business Template
-Message Example
-
-
-Your appointment is coming up on {{ doc.date }} at {{ doc.time }}
-`;
- } else if (frm.doc.channel === 'Email') {
+ if (frm.doc.channel === 'Email') {
template = `Message Example
<h3>Order Overdue</h3>
@@ -124,7 +117,7 @@ Last comment: {{ comments[-1].comment }} by {{ comments[-1].by }}
</ul>
`;
- } else {
+ } else if (in_list(['Slack', 'System Notification', 'SMS'], frm.doc.channel)) {
template = `Message Example
*Order Overdue*
@@ -142,7 +135,9 @@ Last comment: {{ comments[-1].comment }} by {{ comments[-1].by }}
• Amount: {{ doc.grand_total }}
`;
}
- frm.set_df_property('message_examples', 'options', template);
+ if (template) {
+ frm.set_df_property('message_examples', 'options', template);
+ }
}
};
diff --git a/frappe/email/doctype/notification/notification.json b/frappe/email/doctype/notification/notification.json
index 2a8ee1aeb1..73a84e1d3e 100644
--- a/frappe/email/doctype/notification/notification.json
+++ b/frappe/email/doctype/notification/notification.json
@@ -10,7 +10,6 @@
"enabled",
"column_break_2",
"channel",
- "twilio_number",
"slack_webhook_url",
"filters",
"subject",
@@ -61,7 +60,7 @@
"fieldname": "channel",
"fieldtype": "Select",
"label": "Channel",
- "options": "Email\nSlack\nSystem Notification\nWhatsApp\nSMS",
+ "options": "Email\nSlack\nSystem Notification\nSMS",
"reqd": 1,
"set_only_once": 1
},
@@ -80,14 +79,14 @@
"label": "Filters"
},
{
- "depends_on": "eval: !in_list(['SMS', 'WhatsApp'], doc.channel)",
+ "depends_on": "eval: in_list(['Email', 'Slack', 'System Notification'], doc.channel)",
"description": "To add dynamic subject, use jinja tags like\n\n",
"fieldname": "subject",
"fieldtype": "Data",
"ignore_xss_filter": 1,
"in_list_view": 1,
"label": "Subject",
- "mandatory_depends_on": "eval:!in_list(['SMS', 'WhatsApp'], doc.channel)"
+ "mandatory_depends_on": "eval: in_list(['Email', 'Slack', 'System Notification'], doc.channel)"
},
{
"fieldname": "document_type",
@@ -208,7 +207,7 @@
"label": "Value To Be Set"
},
{
- "depends_on": "eval:in_list(['Email', 'SMS', 'WhatsApp'], doc.channel)",
+ "depends_on": "eval:in_list(['Email', 'SMS'], doc.channel)",
"fieldname": "column_break_5",
"fieldtype": "Section Break",
"label": "Recipients"
@@ -263,15 +262,6 @@
"label": "Print Format",
"options": "Print Format"
},
- {
- "depends_on": "eval: doc.channel==='WhatsApp'",
- "description": "To use WhatsApp for Business, initialize Twilio Settings.",
- "fieldname": "twilio_number",
- "fieldtype": "Link",
- "label": "Twilio Number",
- "mandatory_depends_on": "eval: doc.channel==='WhatsApp'",
- "options": "Twilio Number Group"
- },
{
"default": "0",
"depends_on": "eval: doc.channel !== 'System Notification'",
@@ -291,7 +281,7 @@
"icon": "fa fa-envelope",
"index_web_pages_for_search": 1,
"links": [],
- "modified": "2020-09-03 10:33:23.084590",
+ "modified": "2020-10-28 11:04:54.955567",
"modified_by": "Administrator",
"module": "Email",
"name": "Notification",
diff --git a/frappe/email/doctype/notification/notification.py b/frappe/email/doctype/notification/notification.py
index 62be313b82..75281d427e 100644
--- a/frappe/email/doctype/notification/notification.py
+++ b/frappe/email/doctype/notification/notification.py
@@ -14,7 +14,6 @@ from frappe.utils.safe_exec import get_safe_globals
from frappe.modules.utils import export_module_json, get_doc_module
from six import string_types
from frappe.integrations.doctype.slack_webhook_url.slack_webhook_url import send_slack_message
-from frappe.integrations.doctype.twilio_settings.twilio_settings import send_whatsapp_message
from frappe.core.doctype.sms_settings.sms_settings import send_sms
from frappe.desk.doctype.notification_log.notification_log import enqueue_create_notification
@@ -29,7 +28,7 @@ class Notification(Document):
self.name = self.subject
def validate(self):
- if self.channel not in ('WhatsApp', 'SMS'):
+ if self.channel in ("Email", "Slack", "System Notification"):
validate_template(self.subject)
validate_template(self.message)
@@ -43,7 +42,6 @@ class Notification(Document):
self.validate_forbidden_types()
self.validate_condition()
self.validate_standard()
- self.validate_twilio_settings()
frappe.cache().hdel('notifications', self.document_type)
def on_update(self):
@@ -70,11 +68,6 @@ def get_context(context):
if self.is_standard and not frappe.conf.developer_mode:
frappe.throw(_('Cannot edit Standard Notification. To edit, please disable this and duplicate it'))
- def validate_twilio_settings(self):
- if self.enabled and self.channel == "WhatsApp" \
- and not frappe.db.get_single_value("Twilio Settings", "enabled"):
- frappe.throw(_("Please enable Twilio settings to send WhatsApp messages"))
-
def validate_condition(self):
temp_doc = frappe.new_doc(self.document_type)
if self.condition:
@@ -137,9 +130,6 @@ def get_context(context):
if self.channel == 'Slack':
self.send_a_slack_msg(doc, context)
- if self.channel == 'WhatsApp':
- self.send_whatsapp_msg(doc, context)
-
if self.channel == 'SMS':
self.send_sms(doc, context)
@@ -230,13 +220,6 @@ def get_context(context):
reference_doctype=doc.doctype,
reference_name=doc.name)
- def send_whatsapp_msg(self, doc, context):
- send_whatsapp_message(
- sender=self.twilio_number,
- receiver_list=self.get_receiver_list(doc, context),
- message=frappe.render_template(self.message, context),
- )
-
def send_sms(self, doc, context):
send_sms(
receiver_list=self.get_receiver_list(doc, context),
@@ -302,7 +285,7 @@ def get_context(context):
# For sending messages to the owner's mobile phone number
if recipient.receiver_by_document_field == 'owner':
- receiver_list.append(get_user_info(doc.get('owner'), 'mobile_no'))
+ receiver_list += get_user_info([dict(user_name=doc.get('owner'))], 'mobile_no')
# For sending messages to the number specified in the receiver field
elif recipient.receiver_by_document_field:
receiver_list.append(doc.get(recipient.receiver_by_document_field))
diff --git a/frappe/event_streaming/doctype/event_consumer/event_consumer.py b/frappe/event_streaming/doctype/event_consumer/event_consumer.py
index 1505c3a05d..5789e09e74 100644
--- a/frappe/event_streaming/doctype/event_consumer/event_consumer.py
+++ b/frappe/event_streaming/doctype/event_consumer/event_consumer.py
@@ -31,10 +31,12 @@ class EventConsumer(Document):
self.update_consumer_status()
else:
frappe.db.set_value(self.doctype, self.name, 'incoming_change', 0)
-
+
frappe.cache().delete_value('event_consumer_document_type_map')
def on_trash(self):
+ for i in frappe.get_all('Event Update Log Consumer', {'consumer': self.name}):
+ frappe.delete_doc('Event Update Log Consumer', i.name)
frappe.cache().delete_value('event_consumer_document_type_map')
def update_consumer_status(self):
@@ -88,8 +90,9 @@ def register_consumer(data):
for entry in consumer_doctypes:
consumer.append('consumer_doctypes', {
- 'ref_doctype': entry,
- 'status': 'Pending'
+ 'ref_doctype': entry.get('doctype'),
+ 'status': 'Pending',
+ 'condition': entry.get('condition')
})
consumer.insert()
@@ -153,3 +156,53 @@ def notify(consumer):
jobs = get_jobs()
if not jobs or enqueued_method not in jobs[frappe.local.site] and not consumer.flags.notifed:
frappe.enqueue(enqueued_method, queue='long', enqueue_after_commit=True, **{'consumer': consumer})
+
+
+def has_consumer_access(consumer, update_log):
+ """Checks if consumer has completely satisfied all the conditions on the doc"""
+
+ if isinstance(consumer, str):
+ consumer = frappe.get_doc('Event Consumer', consumer)
+
+ if not frappe.db.exists(update_log.ref_doctype, update_log.docname):
+ # Delete Log
+ # Check if the last Update Log of this document was read by this consumer
+ last_update_log = frappe.get_all(
+ 'Event Update Log',
+ filters={
+ 'ref_doctype': update_log.ref_doctype,
+ 'docname': update_log.docname,
+ 'creation': ['<', update_log.creation]
+ },
+ order_by='creation desc',
+ limit_page_length=1
+ )
+ if not len(last_update_log):
+ return False
+
+ last_update_log = frappe.get_doc('Event Update Log', last_update_log[0].name)
+ return len([x for x in last_update_log.consumers if x.consumer == consumer.name])
+
+ doc = frappe.get_doc(update_log.ref_doctype, update_log.docname)
+ try:
+ for dt_entry in consumer.consumer_doctypes:
+ if dt_entry.ref_doctype != update_log.ref_doctype:
+ continue
+
+ if not dt_entry.condition:
+ return True
+
+ condition: str = dt_entry.condition
+ if condition.startswith('cmd:'):
+ cmd = condition.split('cmd:')[1].strip()
+ args = {
+ 'consumer': consumer,
+ 'doc': doc,
+ 'update_log': update_log
+ }
+ return frappe.call(cmd, **args)
+ else:
+ return frappe.safe_eval(condition, frappe._dict(doc=doc))
+ except Exception as e:
+ frappe.log_error(title='has_consumer_access error', message=e)
+ return False
\ No newline at end of file
diff --git a/frappe/event_streaming/doctype/event_consumer_document_type/event_consumer_document_type.json b/frappe/event_streaming/doctype/event_consumer_document_type/event_consumer_document_type.json
index 71dcc63127..c243334a09 100644
--- a/frappe/event_streaming/doctype/event_consumer_document_type/event_consumer_document_type.json
+++ b/frappe/event_streaming/doctype/event_consumer_document_type/event_consumer_document_type.json
@@ -7,7 +7,8 @@
"field_order": [
"ref_doctype",
"status",
- "unsubscribed"
+ "unsubscribed",
+ "condition"
],
"fields": [
{
@@ -37,11 +38,17 @@
"in_list_view": 1,
"label": "Unsubscribed",
"read_only": 1
+ },
+ {
+ "fieldname": "condition",
+ "fieldtype": "Code",
+ "label": "Condition",
+ "read_only": 1
}
],
"istable": 1,
"links": [],
- "modified": "2020-08-14 12:38:40.918620",
+ "modified": "2020-11-07 09:26:49.894294",
"modified_by": "Administrator",
"module": "Event Streaming",
"name": "Event Consumer Document Type",
diff --git a/frappe/event_streaming/doctype/event_producer/event_producer.py b/frappe/event_streaming/doctype/event_producer/event_producer.py
index d458f3c24b..d8a6a55510 100644
--- a/frappe/event_streaming/doctype/event_producer/event_producer.py
+++ b/frappe/event_streaming/doctype/event_producer/event_producer.py
@@ -102,9 +102,13 @@ class EventProducer(Document):
for entry in self.producer_doctypes:
if entry.has_mapping:
# if mapping, subscribe to remote doctype on consumer's site
- consumer_doctypes.append(frappe.db.get_value('Document Type Mapping', entry.mapping, 'remote_doctype'))
+ dt = frappe.db.get_value('Document Type Mapping', entry.mapping, 'remote_doctype')
else:
- consumer_doctypes.append(entry.ref_doctype)
+ dt = entry.ref_doctype
+ consumer_doctypes.append({
+ "doctype": dt,
+ "condition": entry.condition
+ })
user_key = frappe.db.get_value('User', self.user, 'api_key')
user_secret = get_decrypted_password('User', self.user, 'api_secret')
@@ -145,7 +149,8 @@ class EventProducer(Document):
event_consumer.consumer_doctypes.append({
'ref_doctype': ref_doctype,
'status': get_approval_status(config, ref_doctype),
- 'unsubscribed': entry.unsubscribe
+ 'unsubscribed': entry.unsubscribe,
+ 'condition': entry.condition
})
event_consumer.user = self.user
event_consumer.incoming_change = True
@@ -347,13 +352,13 @@ def set_delete(update):
def get_updates(producer_site, last_update, doctypes):
"""Get all updates generated after the last update timestamp"""
- docs = producer_site.get_list(
- doctype='Event Update Log',
- filters={'ref_doctype': ('in', doctypes), 'creation': ('>', last_update)},
- fields=['update_type', 'ref_doctype', 'docname', 'data', 'name', 'creation']
- )
- docs.reverse()
- return [frappe._dict(d) for d in docs]
+ docs = producer_site.post_request({
+ 'cmd': 'frappe.event_streaming.doctype.event_update_log.event_update_log.get_update_logs_for_consumer',
+ 'event_consumer': get_url(),
+ 'doctypes': frappe.as_json(doctypes),
+ 'last_update': last_update
+ })
+ return [frappe._dict(d) for d in (docs or [])]
def get_local_doc(update):
diff --git a/frappe/event_streaming/doctype/event_producer/test_event_producer.py b/frappe/event_streaming/doctype/event_producer/test_event_producer.py
index fa2461a9d8..4c259c3729 100644
--- a/frappe/event_streaming/doctype/event_producer/test_event_producer.py
+++ b/frappe/event_streaming/doctype/event_producer/test_event_producer.py
@@ -152,6 +152,82 @@ class TestEventProducer(unittest.TestCase):
reset_configuration(producer_url)
+ def test_conditional_events(self):
+ producer = get_remote_site()
+
+ # Add Condition
+ event_producer = frappe.get_doc('Event Producer', producer_url)
+ note_producer_entry = [
+ x for x in event_producer.producer_doctypes if x.ref_doctype == 'Note'
+ ][0]
+ note_producer_entry.condition = 'doc.public == 1'
+ event_producer.save()
+
+ # Make test doc
+ producer_note1 = frappe._dict(doctype='Note', public=0, title='test conditional sync')
+ delete_on_remote_if_exists(producer, 'Note', {'title': producer_note1['title']})
+ producer_note1 = producer.insert(producer_note1)
+
+ # Make Update
+ producer_note1['content'] = 'Test Conditional Sync Content'
+ producer_note1 = producer.update(producer_note1)
+
+ self.pull_producer_data()
+
+ # Check if synced here
+ self.assertFalse(frappe.db.exists('Note', producer_note1.name))
+
+ # Lets satisfy the condition
+ producer_note1['public'] = 1
+ producer_note1 = producer.update(producer_note1)
+
+ self.pull_producer_data()
+
+ # it should sync now
+ self.assertTrue(frappe.db.exists('Note', producer_note1.name))
+ local_note = frappe.get_doc('Note', producer_note1.name)
+ self.assertEqual(local_note.content, producer_note1.content)
+
+ reset_configuration(producer_url)
+
+ def test_conditional_events_with_cmd(self):
+ producer = get_remote_site()
+
+ # Add Condition
+ event_producer = frappe.get_doc('Event Producer', producer_url)
+ note_producer_entry = [
+ x for x in event_producer.producer_doctypes if x.ref_doctype == 'Note'
+ ][0]
+ note_producer_entry.condition = 'cmd: frappe.event_streaming.doctype.event_producer.test_event_producer.can_sync_note'
+ event_producer.save()
+
+ # Make test doc
+ producer_note1 = frappe._dict(doctype='Note', public=0, title='test conditional sync cmd')
+ delete_on_remote_if_exists(producer, 'Note', {'title': producer_note1['title']})
+ producer_note1 = producer.insert(producer_note1)
+
+ # Make Update
+ producer_note1['content'] = 'Test Conditional Sync Content'
+ producer_note1 = producer.update(producer_note1)
+
+ self.pull_producer_data()
+
+ # Check if synced here
+ self.assertFalse(frappe.db.exists('Note', producer_note1.name))
+
+ # Lets satisfy the condition
+ producer_note1['public'] = 1
+ producer_note1 = producer.update(producer_note1)
+
+ self.pull_producer_data()
+
+ # it should sync now
+ self.assertTrue(frappe.db.exists('Note', producer_note1.name))
+ local_note = frappe.get_doc('Note', producer_note1.name)
+ self.assertEqual(local_note.content, producer_note1.content)
+
+ reset_configuration(producer_url)
+
def test_update_log(self):
producer = get_remote_site()
producer_doc = insert_into_producer(producer, 'test update log')
@@ -221,6 +297,8 @@ class TestEventProducer(unittest.TestCase):
reset_configuration(producer_url)
+def can_sync_note(consumer, doc, update_log):
+ return doc.public == 1
def setup_event_producer_for_inner_mapping():
event_producer = frappe.get_doc('Event Producer', producer_url, for_update=True)
@@ -322,6 +400,7 @@ def create_event_producer(producer_url):
def reset_configuration(producer_url):
event_producer = frappe.get_doc('Event Producer', producer_url, for_update=True)
event_producer.producer_doctypes = []
+ event_producer.conditions = []
event_producer.producer_url = producer_url
event_producer.append('producer_doctypes', {
'ref_doctype': 'ToDo',
diff --git a/frappe/event_streaming/doctype/event_producer_document_type/event_producer_document_type.json b/frappe/event_streaming/doctype/event_producer_document_type/event_producer_document_type.json
index e5fe9497f8..17fd51d12d 100644
--- a/frappe/event_streaming/doctype/event_producer_document_type/event_producer_document_type.json
+++ b/frappe/event_streaming/doctype/event_producer_document_type/event_producer_document_type.json
@@ -10,7 +10,8 @@
"use_same_name",
"unsubscribe",
"has_mapping",
- "mapping"
+ "mapping",
+ "condition"
],
"fields": [
{
@@ -63,11 +64,16 @@
"fieldtype": "Check",
"in_list_view": 1,
"label": "Unsubscribe"
+ },
+ {
+ "fieldname": "condition",
+ "fieldtype": "Code",
+ "label": "Condition"
}
],
"istable": 1,
"links": [],
- "modified": "2020-08-14 11:38:01.278996",
+ "modified": "2020-11-07 09:26:58.463868",
"modified_by": "Administrator",
"module": "Event Streaming",
"name": "Event Producer Document Type",
diff --git a/frappe/event_streaming/doctype/event_update_log/event_update_log.json b/frappe/event_streaming/doctype/event_update_log/event_update_log.json
index 452a656b8b..a42bc7ec87 100644
--- a/frappe/event_streaming/doctype/event_update_log/event_update_log.json
+++ b/frappe/event_streaming/doctype/event_update_log/event_update_log.json
@@ -1,4 +1,5 @@
{
+ "actions": [],
"creation": "2019-07-30 15:31:26.352527",
"doctype": "DocType",
"editable_grid": 1,
@@ -7,7 +8,8 @@
"update_type",
"ref_doctype",
"docname",
- "data"
+ "data",
+ "consumers"
],
"fields": [
{
@@ -31,7 +33,6 @@
"fieldtype": "Data",
"in_list_view": 1,
"label": "Document Name",
- "options": "ref_doctype",
"read_only": 1
},
{
@@ -39,10 +40,18 @@
"fieldtype": "Code",
"label": "Data",
"read_only": 1
+ },
+ {
+ "fieldname": "consumers",
+ "fieldtype": "Table MultiSelect",
+ "label": "Consumers",
+ "options": "Event Update Log Consumer",
+ "read_only": 1
}
],
"in_create": 1,
- "modified": "2019-09-24 23:16:07.207707",
+ "links": [],
+ "modified": "2020-09-04 07:31:52.599804",
"modified_by": "Administrator",
"module": "Event Streaming",
"name": "Event Update Log",
diff --git a/frappe/event_streaming/doctype/event_update_log/event_update_log.py b/frappe/event_streaming/doctype/event_update_log/event_update_log.py
index 646331a02c..1c31718c2b 100644
--- a/frappe/event_streaming/doctype/event_update_log/event_update_log.py
+++ b/frappe/event_streaming/doctype/event_update_log/event_update_log.py
@@ -140,3 +140,137 @@ def check_docstatus(out, old, new, for_child):
if not for_child and old.docstatus != new.docstatus:
out.changed['docstatus'] = new.docstatus
return out
+
+
+def is_consumer_uptodate(update_log, consumer):
+ """
+ Checks if Consumer has read all the UpdateLogs before the specified update_log
+ :param update_log: The UpdateLog Doc in context
+ :param consumer: The EventConsumer doc
+ """
+ if update_log.update_type == 'Create':
+ # consumer is obviously up to date
+ return True
+
+ prev_logs = frappe.get_all(
+ 'Event Update Log',
+ filters={
+ 'ref_doctype': update_log.ref_doctype,
+ 'docname': update_log.docname,
+ 'creation': ['<', update_log.creation]
+ },
+ order_by='creation desc',
+ limit_page_length=1
+ )
+
+ if not len(prev_logs):
+ return False
+
+ prev_log_consumers = frappe.get_all(
+ 'Event Update Log Consumer',
+ fields=['consumer'],
+ filters={
+ 'parent': prev_logs[0].name,
+ 'parenttype': 'Event Update Log',
+ 'consumer': consumer.name
+ }
+ )
+
+ return len(prev_log_consumers) > 0
+
+
+def mark_consumer_read(update_log_name, consumer_name):
+ """
+ This function appends the Consumer to the list of Consumers that has 'read' an Update Log
+ """
+ update_log = frappe.get_doc('Event Update Log', update_log_name)
+ if len([x for x in update_log.consumers if x.consumer == consumer_name]):
+ return
+
+ frappe.get_doc(frappe._dict(
+ doctype='Event Update Log Consumer',
+ consumer=consumer_name,
+ parent=update_log_name,
+ parenttype='Event Update Log',
+ parentfield='consumers'
+ )).insert(ignore_permissions=True)
+
+
+def get_unread_update_logs(consumer_name, dt, dn):
+ """
+ Get old logs unread by the consumer on a particular document
+ """
+ already_consumed = [x[0] for x in frappe.db.sql("""
+ SELECT
+ update_log.name
+ FROM `tabEvent Update Log` update_log
+ JOIN `tabEvent Update Log Consumer` consumer ON consumer.parent = update_log.name
+ WHERE
+ consumer.consumer = %(consumer)s
+ AND update_log.ref_doctype = %(dt)s
+ AND update_log.docname = %(dn)s
+ """, {'consumer': consumer_name, "dt": dt, "dn": dn}, as_dict=0)]
+
+ logs = frappe.get_all(
+ 'Event Update Log',
+ fields=['update_type', 'ref_doctype',
+ 'docname', 'data', 'name', 'creation'],
+ filters={
+ 'ref_doctype': dt,
+ 'docname': dn,
+ 'name': ['not in', already_consumed]
+ },
+ order_by='creation'
+ )
+
+ return logs
+
+
+@frappe.whitelist()
+def get_update_logs_for_consumer(event_consumer, doctypes, last_update):
+ """
+ Fetches all the UpdateLogs for the consumer
+ It will inject old un-consumed Update Logs if a doc was just found to be accessible to the Consumer
+ """
+
+ if isinstance(doctypes, str):
+ doctypes = frappe.parse_json(doctypes)
+
+ from frappe.event_streaming.doctype.event_consumer.event_consumer import has_consumer_access
+
+ consumer = frappe.get_doc('Event Consumer', event_consumer)
+ docs = frappe.get_list(
+ doctype='Event Update Log',
+ filters={'ref_doctype': ('in', doctypes),
+ 'creation': ('>', last_update)},
+ fields=['update_type', 'ref_doctype',
+ 'docname', 'data', 'name', 'creation'],
+ order_by='creation desc'
+ )
+
+ result = []
+ to_update_history = []
+ for d in docs:
+ if (d.ref_doctype, d.docname) in to_update_history:
+ # will be notified by background jobs
+ continue
+
+ if not has_consumer_access(consumer=consumer, update_log=d):
+ continue
+
+ if not is_consumer_uptodate(d, consumer):
+ to_update_history.append((d.ref_doctype, d.docname))
+ # get_unread_update_logs will have the current log
+ old_logs = get_unread_update_logs(consumer.name, d.ref_doctype, d.docname)
+ if old_logs:
+ old_logs.reverse()
+ result.extend(old_logs)
+ else:
+ result.append(d)
+
+
+ for d in result:
+ mark_consumer_read(update_log_name=d.name, consumer_name=consumer.name)
+
+ result.reverse()
+ return result
\ No newline at end of file
diff --git a/frappe/integrations/doctype/twilio_number_group/__init__.py b/frappe/event_streaming/doctype/event_update_log_consumer/__init__.py
similarity index 100%
rename from frappe/integrations/doctype/twilio_number_group/__init__.py
rename to frappe/event_streaming/doctype/event_update_log_consumer/__init__.py
diff --git a/frappe/event_streaming/doctype/event_update_log_consumer/event_update_log_consumer.json b/frappe/event_streaming/doctype/event_update_log_consumer/event_update_log_consumer.json
new file mode 100644
index 0000000000..b3484c6481
--- /dev/null
+++ b/frappe/event_streaming/doctype/event_update_log_consumer/event_update_log_consumer.json
@@ -0,0 +1,32 @@
+{
+ "actions": [],
+ "creation": "2020-06-30 10:54:53.301787",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+ "consumer"
+ ],
+ "fields": [
+ {
+ "fieldname": "consumer",
+ "fieldtype": "Link",
+ "in_list_view": 1,
+ "label": "Consumer",
+ "options": "Event Consumer",
+ "reqd": 1
+ }
+ ],
+ "istable": 1,
+ "links": [],
+ "modified": "2020-06-30 10:54:53.301787",
+ "modified_by": "Administrator",
+ "module": "Event Streaming",
+ "name": "Event Update Log Consumer",
+ "owner": "Administrator",
+ "permissions": [],
+ "quick_entry": 1,
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "track_changes": 1
+}
\ No newline at end of file
diff --git a/frappe/integrations/doctype/twilio_number_group/twilio_number_group.py b/frappe/event_streaming/doctype/event_update_log_consumer/event_update_log_consumer.py
similarity index 85%
rename from frappe/integrations/doctype/twilio_number_group/twilio_number_group.py
rename to frappe/event_streaming/doctype/event_update_log_consumer/event_update_log_consumer.py
index 04cb9ae146..ee6d5d8ca9 100644
--- a/frappe/integrations/doctype/twilio_number_group/twilio_number_group.py
+++ b/frappe/event_streaming/doctype/event_update_log_consumer/event_update_log_consumer.py
@@ -6,5 +6,5 @@ from __future__ import unicode_literals
# import frappe
from frappe.model.document import Document
-class TwilioNumberGroup(Document):
+class EventUpdateLogConsumer(Document):
pass
diff --git a/frappe/exceptions.py b/frappe/exceptions.py
index 267f5410af..82fbff7a90 100644
--- a/frappe/exceptions.py
+++ b/frappe/exceptions.py
@@ -106,8 +106,10 @@ class InvalidDates(ValidationError): pass
class DataTooLongException(ValidationError): pass
class FileAlreadyAttachedException(Exception): pass
class DocumentAlreadyRestored(Exception): pass
+class AttachmentLimitReached(Exception): pass
# OAuth exceptions
class InvalidAuthorizationHeader(CSRFTokenError): pass
class InvalidAuthorizationPrefix(CSRFTokenError): pass
class InvalidAuthorizationToken(CSRFTokenError): pass
-class InvalidDatabaseFile(ValidationError): pass
\ No newline at end of file
+class InvalidDatabaseFile(ValidationError): pass
+class ExecutableNotFound(FileNotFoundError): pass
diff --git a/frappe/installer.py b/frappe/installer.py
index 51113beae8..6a77e5e713 100755
--- a/frappe/installer.py
+++ b/frappe/installer.py
@@ -3,8 +3,90 @@
import json
import os
-from frappe.defaults import _clear_cache
+import sys
+
import frappe
+from frappe.defaults import _clear_cache
+
+
+def _new_site(
+ db_name,
+ site,
+ mariadb_root_username=None,
+ mariadb_root_password=None,
+ admin_password=None,
+ verbose=False,
+ install_apps=None,
+ source_sql=None,
+ force=False,
+ no_mariadb_socket=False,
+ reinstall=False,
+ db_password=None,
+ db_type=None,
+ db_host=None,
+ db_port=None,
+ new_site=False,
+):
+ """Install a new Frappe site"""
+
+ if not force and os.path.exists(site):
+ print("Site {0} already exists".format(site))
+ sys.exit(1)
+
+ if no_mariadb_socket and not db_type == "mariadb":
+ print("--no-mariadb-socket requires db_type to be set to mariadb.")
+ sys.exit(1)
+
+ if not db_name:
+ import hashlib
+ db_name = "_" + hashlib.sha1(site.encode()).hexdigest()[:16]
+
+ frappe.init(site=site)
+
+ from frappe.commands.scheduler import _is_scheduler_enabled
+ from frappe.utils import get_site_path, scheduler, touch_file
+
+ try:
+ # enable scheduler post install?
+ enable_scheduler = _is_scheduler_enabled()
+ except Exception:
+ enable_scheduler = False
+
+ make_site_dirs()
+
+ installing = touch_file(get_site_path("locks", "installing.lock"))
+
+ install_db(
+ root_login=mariadb_root_username,
+ root_password=mariadb_root_password,
+ db_name=db_name,
+ admin_password=admin_password,
+ verbose=verbose,
+ source_sql=source_sql,
+ force=force,
+ reinstall=reinstall,
+ db_password=db_password,
+ db_type=db_type,
+ db_host=db_host,
+ db_port=db_port,
+ no_mariadb_socket=no_mariadb_socket,
+ )
+ apps_to_install = (
+ ["frappe"] + (frappe.conf.get("install_apps") or []) + (list(install_apps) or [])
+ )
+
+ for app in apps_to_install:
+ install_app(app, verbose=verbose, set_as_patched=not source_sql)
+
+ os.remove(installing)
+
+ scheduler.toggle_scheduler(enable_scheduler)
+ frappe.db.commit()
+
+ scheduler_status = (
+ "disabled" if frappe.utils.scheduler.is_scheduler_disabled() else "enabled"
+ )
+ print("*** Scheduler is", scheduler_status, "***")
def install_db(root_login="root", root_password=None, db_name=None, source_sql=None,
@@ -36,9 +118,9 @@ def install_db(root_login="root", root_password=None, db_name=None, source_sql=N
def install_app(name, verbose=False, set_as_patched=True):
from frappe.core.doctype.scheduled_job_type.scheduled_job_type import sync_jobs
- from frappe.utils.fixtures import sync_fixtures
from frappe.model.sync import sync_for
from frappe.modules.utils import sync_customizations
+ from frappe.utils.fixtures import sync_fixtures
frappe.flags.in_install = name
frappe.flags.ignore_in_install = False
@@ -122,64 +204,80 @@ def remove_app(app_name, dry_run=False, yes=False, no_backup=False, force=False)
"""Remove app and all linked to the app's module with the app from a site."""
import click
+ site = frappe.local.site
+
# dont allow uninstall app if not installed unless forced
if not force:
if app_name not in frappe.get_installed_apps():
- click.secho("App {0} not installed on Site {1}".format(app_name, frappe.local.site), fg="yellow")
+ click.secho(f"App {app_name} not installed on Site {site}", fg="yellow")
return
- print("Uninstalling App {0} from Site {1}...".format(app_name, frappe.local.site))
+ print(f"Uninstalling App {app_name} from Site {site}...")
if not dry_run and not yes:
- confirm = click.confirm("All doctypes (including custom), modules related to this app will be deleted. Are you sure you want to continue?")
+ confirm = click.confirm(
+ "All doctypes (including custom), modules related to this app will be"
+ " deleted. Are you sure you want to continue?"
+ )
if not confirm:
return
- if not no_backup:
+ if not (dry_run or no_backup):
from frappe.utils.backups import scheduled_backup
+
print("Backing up...")
scheduled_backup(ignore_files=True)
frappe.flags.in_uninstall = True
drop_doctypes = []
- modules = (x.name for x in frappe.get_all("Module Def", filters={"app_name": app_name}))
+ modules = frappe.get_all("Module Def", filters={"app_name": app_name}, pluck="name")
for module_name in modules:
- print("Deleting Module '{0}'".format(module_name))
+ print(f"Deleting Module '{module_name}'")
- for doctype in frappe.get_list("DocType", filters={"module": module_name}, fields=["name", "issingle"]):
- print("* removing DocType '{0}'...".format(doctype.name))
+ for doctype in frappe.get_all(
+ "DocType", filters={"module": module_name}, fields=["name", "issingle"]
+ ):
+ print(f"* removing DocType '{doctype.name}'...")
if not dry_run:
- frappe.delete_doc("DocType", doctype.name)
+ frappe.delete_doc("DocType", doctype.name, ignore_on_trash=True)
if not doctype.issingle:
drop_doctypes.append(doctype.name)
- linked_doctypes = frappe.get_all("DocField", filters={"fieldtype": "Link", "options": "Module Def"}, fields=['parent'])
+ linked_doctypes = frappe.get_all(
+ "DocField", filters={"fieldtype": "Link", "options": "Module Def"}, fields=["parent"]
+ )
ordered_doctypes = ["Desk Page", "Report", "Page", "Web Form"]
- doctypes_with_linked_modules = ordered_doctypes + [doctype.parent for doctype in linked_doctypes if doctype.parent not in ordered_doctypes]
-
+ all_doctypes_with_linked_modules = ordered_doctypes + [
+ doctype.parent
+ for doctype in linked_doctypes
+ if doctype.parent not in ordered_doctypes
+ ]
+ doctypes_with_linked_modules = [
+ x for x in all_doctypes_with_linked_modules if frappe.db.exists("DocType", x)
+ ]
for doctype in doctypes_with_linked_modules:
- for record in frappe.get_list(doctype, filters={"module": module_name}):
- print("* removing {0} '{1}'...".format(doctype, record.name))
+ for record in frappe.get_all(doctype, filters={"module": module_name}, pluck="name"):
+ print(f"* removing {doctype} '{record}'...")
if not dry_run:
- frappe.delete_doc(doctype, record.name)
+ frappe.delete_doc(doctype, record, ignore_on_trash=True)
- print("* removing Module Def '{0}'...".format(module_name))
+ print(f"* removing Module Def '{module_name}'...")
if not dry_run:
- frappe.delete_doc("Module Def", module_name)
+ frappe.delete_doc("Module Def", module_name, ignore_on_trash=True)
+
+ for doctype in set(drop_doctypes):
+ print(f"* dropping Table for '{doctype}'...")
+ if not dry_run:
+ frappe.db.sql_ddl(f"drop table `tab{doctype}`")
if not dry_run:
remove_from_installed_apps(app_name)
-
- for doctype in set(drop_doctypes):
- print("* dropping Table for '{0}'...".format(doctype))
- frappe.db.sql_ddl("drop table `tab{0}`".format(doctype))
-
frappe.db.commit()
- click.secho("Uninstalled App {0} from Site {1}".format(app_name, frappe.local.site), fg="green")
+ click.secho(f"Uninstalled App {app_name} from Site {site}", fg="green")
frappe.flags.in_uninstall = False
@@ -331,6 +429,37 @@ def remove_missing_apps():
frappe.db.set_global("installed_apps", json.dumps(installed_apps))
+def extract_sql_from_archive(sql_file_path):
+ """Return the path of an SQL file if the passed argument is the path of a gzipped
+ SQL file or an SQL file path. The path may be absolute or relative from the bench
+ root directory or the sites sub-directory.
+
+ Args:
+ sql_file_path (str): Path of the SQL file
+
+ Returns:
+ str: Path of the decompressed SQL file
+ """
+ # Extract the gzip file if user has passed *.sql.gz file instead of *.sql file
+ if not os.path.exists(sql_file_path):
+ base_path = '..'
+ sql_file_path = os.path.join(base_path, sql_file_path)
+ if not os.path.exists(sql_file_path):
+ print('Invalid path {0}'.format(sql_file_path[3:]))
+ sys.exit(1)
+ elif sql_file_path.startswith(os.sep):
+ base_path = os.sep
+ else:
+ base_path = '.'
+
+ if sql_file_path.endswith('sql.gz'):
+ decompressed_file_name = extract_sql_gzip(os.path.abspath(sql_file_path))
+ else:
+ decompressed_file_name = sql_file_path
+
+ return decompressed_file_name
+
+
def extract_sql_gzip(sql_gz_path):
import subprocess
@@ -345,9 +474,10 @@ def extract_sql_gzip(sql_gz_path):
return decompressed_file
+
def extract_files(site_name, file_path, folder_name):
- import subprocess
import shutil
+ import subprocess
# Need to do frappe.init to maintain the site locals
frappe.init(site=site_name)
@@ -375,6 +505,12 @@ def extract_files(site_name, file_path, folder_name):
def is_downgrade(sql_file_path, verbose=False):
"""checks if input db backup will get downgraded on current bench"""
+
+ # This function is only tested with mariadb
+ # TODO: Add postgres support
+ if frappe.conf.db_type not in (None, "mariadb"):
+ return False
+
from semantic_version import Version
head = "INSERT INTO `tabInstalled Application` VALUES"
@@ -408,6 +544,37 @@ def is_downgrade(sql_file_path, verbose=False):
return downgrade
+def is_partial(sql_file_path):
+ with open(sql_file_path) as f:
+ header = " ".join([f.readline() for _ in range(5)])
+ if "Partial Backup" in header:
+ return True
+ return False
+
+
+def partial_restore(sql_file_path, verbose=False):
+ sql_file = extract_sql_from_archive(sql_file_path)
+
+ if frappe.conf.db_type in (None, "mariadb"):
+ from frappe.database.mariadb.setup_db import import_db_from_sql
+ elif frappe.conf.db_type == "postgres":
+ from frappe.database.postgres.setup_db import import_db_from_sql
+ import warnings
+ from click import style
+ warn = style(
+ "Delete the tables you want to restore manually before attempting"
+ " partial restore operation for PostreSQL databases",
+ fg="yellow"
+ )
+ warnings.warn(warn)
+
+ import_db_from_sql(source_sql=sql_file, verbose=verbose)
+
+ # Removing temporarily created file
+ if sql_file != sql_file_path:
+ os.remove(sql_file)
+
+
def validate_database_sql(path, _raise=True):
"""Check if file has contents and if DefaultValue table exists
@@ -415,21 +582,29 @@ def validate_database_sql(path, _raise=True):
path (str): Path of the decompressed SQL file
_raise (bool, optional): Raise exception if invalid file. Defaults to True.
"""
- _raise = False
+ empty_file = False
+ missing_table = True
+
error_message = ""
if not os.path.getsize(path):
error_message = f"{path} is an empty file!"
- _raise = True
+ empty_file = True
- if not _raise:
+ # dont bother checking if empty file
+ if not empty_file:
with open(path, "r") as f:
for line in f:
if 'tabDefaultValue' in line:
- error_message = "Table `tabDefaultValue` not found in file."
- _raise = True
+ missing_table = False
+ break
- if error_message and _raise:
+ if missing_table:
+ error_message = "Table `tabDefaultValue` not found in file."
+
+ if error_message:
import click
click.secho(error_message, fg="red")
+
+ if _raise and (missing_table or empty_file):
raise frappe.InvalidDatabaseFile
diff --git a/frappe/integrations/desk_page/integrations/integrations.json b/frappe/integrations/desk_page/integrations/integrations.json
index cbf7c9c085..1acf4e6c4a 100644
--- a/frappe/integrations/desk_page/integrations/integrations.json
+++ b/frappe/integrations/desk_page/integrations/integrations.json
@@ -23,7 +23,7 @@
{
"hidden": 0,
"label": "Settings",
- "links": "[\n {\n \"description\": \"Webhooks calling API requests into web apps\",\n \"label\": \"Webhook\",\n \"name\": \"Webhook\",\n \"type\": \"doctype\"\n },\n {\n \"description\": \"Slack Webhooks for internal integration\",\n \"label\": \"Slack Webhook URL\",\n \"name\": \"Slack Webhook URL\",\n \"type\": \"doctype\"\n },\n {\n \"description\": \"Twilio Settings for WhatsApp integration\",\n \"label\": \"Twilio Settings\",\n \"name\": \"Twilio Settings\",\n \"type\": \"doctype\"\n },\n {\n \"description\": \"SMS Settings for sending sms\",\n \"label\": \"SMS Settings\",\n \"name\": \"SMS Settings\",\n \"type\": \"doctype\"\n }\n]"
+ "links": "[\n {\n \"description\": \"Webhooks calling API requests into web apps\",\n \"label\": \"Webhook\",\n \"name\": \"Webhook\",\n \"type\": \"doctype\"\n },\n {\n \"description\": \"Slack Webhooks for internal integration\",\n \"label\": \"Slack Webhook URL\",\n \"name\": \"Slack Webhook URL\",\n \"type\": \"doctype\"\n },\n {\n \"description\": \"SMS Settings for sending sms\",\n \"label\": \"SMS Settings\",\n \"name\": \"SMS Settings\",\n \"type\": \"doctype\"\n }\n]"
}
],
"category": "Administration",
@@ -38,7 +38,7 @@
"idx": 0,
"is_standard": 1,
"label": "Integrations",
- "modified": "2020-08-20 23:04:04.528572",
+ "modified": "2020-10-28 10:25:54.792363",
"modified_by": "Administrator",
"module": "Integrations",
"name": "Integrations",
diff --git a/frappe/integrations/doctype/dropbox_settings/dropbox_settings.py b/frappe/integrations/doctype/dropbox_settings/dropbox_settings.py
index 6b95a3f5bf..71445b44d7 100644
--- a/frappe/integrations/doctype/dropbox_settings/dropbox_settings.py
+++ b/frappe/integrations/doctype/dropbox_settings/dropbox_settings.py
@@ -9,7 +9,7 @@ import frappe
import os
from frappe import _
from frappe.model.document import Document
-from frappe.integrations.offsite_backup_utils import get_latest_backup_file, send_email, validate_file_size
+from frappe.integrations.offsite_backup_utils import get_latest_backup_file, send_email, validate_file_size, get_chunk_site
from frappe.integrations.utils import make_post_request
from frappe.utils import (cint, get_request_site_address,
get_files_path, get_backups_path, get_url, encode)
@@ -167,8 +167,9 @@ def upload_file_to_dropbox(filename, folder, dropbox_client):
return
create_folder_if_not_exists(folder, dropbox_client)
- chunk_size = 15 * 1024 * 1024
file_size = os.path.getsize(encode(filename))
+ chunk_size = get_chunk_site(file_size)
+
mode = (dropbox.files.WriteMode.overwrite)
f = open(encode(filename), 'rb')
diff --git a/frappe/integrations/doctype/twilio_number_group/twilio_number_group.json b/frappe/integrations/doctype/twilio_number_group/twilio_number_group.json
deleted file mode 100644
index 9d51e4b452..0000000000
--- a/frappe/integrations/doctype/twilio_number_group/twilio_number_group.json
+++ /dev/null
@@ -1,36 +0,0 @@
-{
- "actions": [],
- "autoname": "field:phone_number",
- "creation": "2020-02-24 13:58:58.036914",
- "doctype": "DocType",
- "editable_grid": 1,
- "engine": "InnoDB",
- "field_order": [
- "phone_number"
- ],
- "fields": [
- {
- "fieldname": "phone_number",
- "fieldtype": "Data",
- "in_list_view": 1,
- "label": "Phone Number",
- "options": "Phone",
- "show_days": 1,
- "show_seconds": 1,
- "unique": 1
- }
- ],
- "index_web_pages_for_search": 1,
- "istable": 1,
- "links": [],
- "modified": "2020-08-20 22:48:57.166791",
- "modified_by": "Administrator",
- "module": "Integrations",
- "name": "Twilio Number Group",
- "owner": "Administrator",
- "permissions": [],
- "quick_entry": 1,
- "sort_field": "modified",
- "sort_order": "DESC",
- "track_changes": 1
-}
\ No newline at end of file
diff --git a/frappe/integrations/doctype/twilio_settings/__init__.py b/frappe/integrations/doctype/twilio_settings/__init__.py
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/frappe/integrations/doctype/twilio_settings/test_twilio_settings.py b/frappe/integrations/doctype/twilio_settings/test_twilio_settings.py
deleted file mode 100644
index bcb1368d68..0000000000
--- a/frappe/integrations/doctype/twilio_settings/test_twilio_settings.py
+++ /dev/null
@@ -1,10 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright (c) 2020, Frappe Technologies and Contributors
-# See license.txt
-from __future__ import unicode_literals
-
-# import frappe
-import unittest
-
-class TestTwilioSettings(unittest.TestCase):
- pass
diff --git a/frappe/integrations/doctype/twilio_settings/twilio_settings.js b/frappe/integrations/doctype/twilio_settings/twilio_settings.js
deleted file mode 100644
index 59ebcf2e7d..0000000000
--- a/frappe/integrations/doctype/twilio_settings/twilio_settings.js
+++ /dev/null
@@ -1,8 +0,0 @@
-// Copyright (c) 2020, Frappe Technologies and contributors
-// For license information, please see license.txt
-
-frappe.ui.form.on('Twilio Settings', {
- refresh: function(frm) {
- frm.dashboard.set_headline(__("For more information, {0}.", [`${__('Click here')}`]));
- }
-});
diff --git a/frappe/integrations/doctype/twilio_settings/twilio_settings.json b/frappe/integrations/doctype/twilio_settings/twilio_settings.json
deleted file mode 100644
index 9eb2c0c512..0000000000
--- a/frappe/integrations/doctype/twilio_settings/twilio_settings.json
+++ /dev/null
@@ -1,67 +0,0 @@
-{
- "actions": [],
- "creation": "2020-01-28 15:21:44.457163",
- "doctype": "DocType",
- "editable_grid": 1,
- "engine": "InnoDB",
- "field_order": [
- "enabled",
- "account_sid",
- "auth_token",
- "column_break_2",
- "twilio_number"
- ],
- "fields": [
- {
- "fieldname": "account_sid",
- "fieldtype": "Data",
- "label": "Account SID",
- "mandatory_depends_on": "eval: doc.enabled"
- },
- {
- "fieldname": "auth_token",
- "fieldtype": "Password",
- "label": "Auth Token",
- "mandatory_depends_on": "eval: doc.enabled"
- },
- {
- "fieldname": "column_break_2",
- "fieldtype": "Column Break"
- },
- {
- "fieldname": "twilio_number",
- "fieldtype": "Table",
- "label": "Twilio Number",
- "options": "Twilio Number Group"
- },
- {
- "default": "0",
- "fieldname": "enabled",
- "fieldtype": "Check",
- "label": "Enabled"
- }
- ],
- "index_web_pages_for_search": 1,
- "issingle": 1,
- "links": [],
- "modified": "2020-09-03 10:17:21.318743",
- "modified_by": "Administrator",
- "module": "Integrations",
- "name": "Twilio Settings",
- "owner": "Administrator",
- "permissions": [
- {
- "create": 1,
- "delete": 1,
- "email": 1,
- "print": 1,
- "read": 1,
- "role": "System Manager",
- "share": 1,
- "write": 1
- }
- ],
- "sort_field": "modified",
- "sort_order": "DESC",
- "track_changes": 1
-}
\ No newline at end of file
diff --git a/frappe/integrations/doctype/twilio_settings/twilio_settings.py b/frappe/integrations/doctype/twilio_settings/twilio_settings.py
deleted file mode 100644
index b8f991e829..0000000000
--- a/frappe/integrations/doctype/twilio_settings/twilio_settings.py
+++ /dev/null
@@ -1,63 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright (c) 2020, Frappe Technologies and contributors
-# For license information, please see license.txt
-
-from __future__ import unicode_literals
-import frappe
-from frappe.model.document import Document
-from frappe import _
-from frappe.utils.password import get_decrypted_password
-from twilio.rest import Client
-from six import string_types
-from json import loads
-
-class TwilioSettings(Document):
- def on_update(self):
- if self.enabled:
- self.validate_twilio_credentials()
-
- def validate_twilio_credentials(self):
- try:
- auth_token = get_decrypted_password("Twilio Settings", "Twilio Settings", 'auth_token')
- client = Client(self.account_sid, auth_token)
- client.api.accounts(self.account_sid).fetch()
- except Exception:
- frappe.throw(_("Invalid Account SID or Auth Token."))
-
-def send_whatsapp_message(sender, receiver_list, message):
- twilio_settings = frappe.get_doc("Twilio Settings")
- if not twilio_settings.enabled:
- frappe.throw(_("Please enable twilio settings before sending WhatsApp messages"))
-
- if isinstance(receiver_list, string_types):
- receiver_list = loads(receiver_list)
- if not isinstance(receiver_list, list):
- receiver_list = [receiver_list]
-
- auth_token = get_decrypted_password("Twilio Settings", "Twilio Settings", 'auth_token')
- client = Client(twilio_settings.account_sid, auth_token)
- args = {
- "from_": 'whatsapp:+{}'.format(sender),
- "body": message
- }
-
- failed_delivery = []
-
- for rec in receiver_list:
- args.update({"to": 'whatsapp:{}'.format(rec)})
- resp = _send_whatsapp(args, client)
- if not resp or resp.error_message:
- failed_delivery.append(rec)
-
- if failed_delivery:
- frappe.log_error(_("The message wasn't correctly delivered to: {}".format(", ".join(failed_delivery))), _('Delivery Failed'))
-
-
-def _send_whatsapp(message_dict, client):
- response = frappe._dict()
- try:
- response = client.messages.create(**message_dict)
- except Exception as e:
- frappe.log_error(e, title = _('Twilio WhatsApp Message Error'))
-
- return response
\ No newline at end of file
diff --git a/frappe/integrations/frappe_providers/frappecloud.py b/frappe/integrations/frappe_providers/frappecloud.py
index e09f09a44b..f60344ee8f 100644
--- a/frappe/integrations/frappe_providers/frappecloud.py
+++ b/frappe/integrations/frappe_providers/frappecloud.py
@@ -6,7 +6,7 @@ import frappe
def frappecloud_migrator(local_site):
- print("Retreiving Site Migrator...")
+ print("Retrieving Site Migrator...")
remote_site = frappe.conf.frappecloud_url or "frappecloud.com"
request_url = "https://{}/api/method/press.api.script".format(remote_site)
request = requests.get(request_url)
diff --git a/frappe/integrations/oauth2.py b/frappe/integrations/oauth2.py
index c8dfc52c95..a750c8328c 100644
--- a/frappe/integrations/oauth2.py
+++ b/frappe/integrations/oauth2.py
@@ -1,41 +1,50 @@
from __future__ import unicode_literals
-import frappe, json
-from frappe.oauth import OAuthWebRequestValidator, WebApplicationServer
+
+import hashlib
+import json
+from urllib.parse import quote, urlencode, urlparse
+
+import jwt
from oauthlib.oauth2 import FatalClientError, OAuth2Error
-from werkzeug import url_fix
-from six.moves.urllib.parse import quote, urlencode, urlparse
-from frappe.integrations.doctype.oauth_provider_settings.oauth_provider_settings import get_oauth_settings
+
+import frappe
from frappe import _
+from frappe.oauth import OAuthWebRequestValidator, WebApplicationServer
+from frappe.integrations.doctype.oauth_provider_settings.oauth_provider_settings import get_oauth_settings
def get_oauth_server():
if not getattr(frappe.local, 'oauth_server', None):
oauth_validator = OAuthWebRequestValidator()
- frappe.local.oauth_server = WebApplicationServer(oauth_validator)
+ frappe.local.oauth_server = WebApplicationServer(oauth_validator)
return frappe.local.oauth_server
-def get_urlparams_from_kwargs(param_kwargs):
+def sanitize_kwargs(param_kwargs):
arguments = param_kwargs
- if arguments.get("data"):
- arguments.pop("data")
- if arguments.get("cmd"):
- arguments.pop("cmd")
+ arguments.pop('data', None)
+ arguments.pop('cmd', None)
- return urlencode(arguments)
+ return arguments
@frappe.whitelist()
def approve(*args, **kwargs):
r = frappe.request
- uri = url_fix(r.url.replace("+"," "))
- http_method = r.method
- body = r.get_data()
- headers = r.headers
try:
- scopes, frappe.flags.oauth_credentials = get_oauth_server().validate_authorization_request(uri, http_method, body, headers)
+ scopes, frappe.flags.oauth_credentials = get_oauth_server().validate_authorization_request(
+ r.url,
+ r.method,
+ r.get_data(),
+ r.headers
+ )
- headers, body, status = get_oauth_server().create_authorization_response(uri=frappe.flags.oauth_credentials['redirect_uri'], \
- body=body, headers=headers, scopes=scopes, credentials=frappe.flags.oauth_credentials)
+ headers, body, status = get_oauth_server().create_authorization_response(
+ uri=frappe.flags.oauth_credentials['redirect_uri'],
+ body=r.get_data(),
+ headers=r.headers,
+ scopes=scopes,
+ credentials=frappe.flags.oauth_credentials
+ )
uri = headers.get('Location', None)
frappe.local.response["type"] = "redirect"
@@ -47,34 +56,28 @@ def approve(*args, **kwargs):
return e
@frappe.whitelist(allow_guest=True)
-def authorize(*args, **kwargs):
- #Fetch provider URL from settings
- oauth_settings = get_oauth_settings()
- params = get_urlparams_from_kwargs(kwargs)
- request_url = urlparse(frappe.request.url)
- success_url = request_url.scheme + "://" + request_url.netloc + "/api/method/frappe.integrations.oauth2.approve?" + params
+def authorize(**kwargs):
+ success_url = "/api/method/frappe.integrations.oauth2.approve?" + encode_params(sanitize_kwargs(kwargs))
failure_url = frappe.form_dict["redirect_uri"] + "?error=access_denied"
- if frappe.session['user']=='Guest':
+ if frappe.session.user == 'Guest':
#Force login, redirect to preauth again.
frappe.local.response["type"] = "redirect"
- frappe.local.response["location"] = "/login?redirect-to=/api/method/frappe.integrations.oauth2.authorize?" + quote(params.replace("+"," "))
-
- elif frappe.session['user']!='Guest':
+ frappe.local.response["location"] = "/login?" + encode_params({'redirect-to': frappe.request.url})
+ else:
try:
r = frappe.request
- uri = url_fix(r.url)
- http_method = r.method
- body = r.get_data()
- headers = r.headers
-
- scopes, frappe.flags.oauth_credentials = get_oauth_server().validate_authorization_request(uri, http_method, body, headers)
+ scopes, frappe.flags.oauth_credentials = get_oauth_server().validate_authorization_request(
+ r.url,
+ r.method,
+ r.get_data(),
+ r.headers
+ )
skip_auth = frappe.db.get_value("OAuth Client", frappe.flags.oauth_credentials['client_id'], "skip_authorization")
unrevoked_tokens = frappe.get_all("OAuth Bearer Token", filters={"status":"Active"})
- if skip_auth or (oauth_settings["skip_authorization"] == "Auto" and len(unrevoked_tokens)):
-
+ if skip_auth or (get_oauth_settings().skip_authorization == "Auto" and unrevoked_tokens):
frappe.local.response["type"] = "redirect"
frappe.local.response["location"] = success_url
else:
@@ -87,7 +90,6 @@ def authorize(*args, **kwargs):
})
resp_html = frappe.render_template("templates/includes/oauth_confirmation.html", response_html_params)
frappe.respond_as_web_page("Confirm Access", resp_html)
-
except FatalClientError as e:
return e
except OAuth2Error as e:
@@ -95,20 +97,20 @@ def authorize(*args, **kwargs):
@frappe.whitelist(allow_guest=True)
def get_token(*args, **kwargs):
- r = frappe.request
-
- uri = url_fix(r.url)
- http_method = r.method
- body = r.form
- headers = r.headers
-
#Check whether frappe server URL is set
frappe_server_url = frappe.db.get_value("Social Login Key", "frappe", "base_url") or None
if not frappe_server_url:
frappe.throw(_("Please set Base URL in Social Login Key for Frappe"))
try:
- headers, body, status = get_oauth_server().create_token_response(uri, http_method, body, headers, frappe.flags.oauth_credentials)
+ r = frappe.request
+ headers, body, status = get_oauth_server().create_token_response(
+ r.url,
+ r.method,
+ r.form,
+ r.headers,
+ frappe.flags.oauth_credentials
+ )
out = frappe._dict(json.loads(body))
if not out.error and "openid" in out.scope:
token_user = frappe.db.get_value("OAuth Bearer Token", out.access_token, "user")
@@ -116,7 +118,7 @@ def get_token(*args, **kwargs):
client_secret = frappe.db.get_value("OAuth Client", token_client, "client_secret")
if token_user in ["Guest", "Administrator"]:
frappe.throw(_("Logged in as Guest or Administrator"))
- import hashlib
+
id_token_header = {
"typ":"jwt",
"alg":"HS256"
@@ -128,9 +130,10 @@ def get_token(*args, **kwargs):
"iss": frappe_server_url,
"at_hash": frappe.oauth.calculate_at_hash(out.access_token, hashlib.sha256)
}
- import jwt
+
id_token_encoded = jwt.encode(id_token, client_secret, algorithm='HS256', headers=id_token_header)
- out.update({"id_token":str(id_token_encoded)})
+ out.update({"id_token": str(id_token_encoded)})
+
frappe.local.response = out
except FatalClientError as e:
@@ -140,12 +143,12 @@ def get_token(*args, **kwargs):
@frappe.whitelist(allow_guest=True)
def revoke_token(*args, **kwargs):
r = frappe.request
- uri = url_fix(r.url)
- http_method = r.method
- body = r.form
- headers = r.headers
-
- headers, body, status = get_oauth_server().create_revocation_response(uri, headers=headers, body=body, http_method=http_method)
+ headers, body, status = get_oauth_server().create_revocation_response(
+ r.url,
+ headers=r.headers,
+ body=r.form,
+ http_method=r.method
+ )
frappe.local.response['http_status_code'] = status
if status == 200:
@@ -174,15 +177,22 @@ def openid_profile(*args, **kwargs):
"email": name,
"picture": picture
})
-
+
frappe.local.response = user_profile
def validate_url(url_string):
try:
result = urlparse(url_string)
- if result.scheme and result.scheme in ["http", "https", "ftp", "ftps"]:
- return True
- else:
- return False
+ return result.scheme and result.scheme in ["http", "https", "ftp", "ftps"]
except:
- return False
\ No newline at end of file
+ return False
+
+def encode_params(params):
+ """
+ Encode a dict of params into a query string.
+
+ Use `quote_via=urllib.parse.quote` so that whitespaces will be encoded as
+ `%20` instead of as `+`. This is needed because oauthlib cannot handle `+`
+ as a whitespace.
+ """
+ return urlencode(params, quote_via=quote)
diff --git a/frappe/integrations/offsite_backup_utils.py b/frappe/integrations/offsite_backup_utils.py
index db176538e4..48a2c89107 100644
--- a/frappe/integrations/offsite_backup_utils.py
+++ b/frappe/integrations/offsite_backup_utils.py
@@ -6,8 +6,7 @@ from __future__ import unicode_literals
import frappe
import glob
import os
-from frappe.utils import split_emails, get_backups_path
-
+from frappe.utils import split_emails, cint
def send_email(success, service_name, doctype, email_field, error_status=None):
recipients = get_recipients(doctype, email_field)
@@ -81,6 +80,22 @@ def get_file_size(file_path, unit):
return file_size
+def get_chunk_site(file_size):
+ ''' this function will return chunk size in megabytes based on file size '''
+
+ file_size_in_gb = cint(file_size/1024/1024)
+
+ MB = 1024 * 1024
+ if file_size_in_gb > 5000:
+ return 200 * MB
+ elif file_size_in_gb >= 3000:
+ return 150 * MB
+ elif file_size_in_gb >= 1000:
+ return 100 * MB
+ elif file_size_in_gb >= 500:
+ return 50 * MB
+ else:
+ return 15 * MB
def validate_file_size():
frappe.flags.create_new_backup = True
@@ -98,4 +113,4 @@ def generate_files_backup():
db_type=frappe.conf.db_type, db_port=frappe.conf.db_port)
backup.set_backup_file_name()
- backup.zip_files()
\ No newline at end of file
+ backup.zip_files()
diff --git a/frappe/model/delete_doc.py b/frappe/model/delete_doc.py
index a38470e3f5..862abe375c 100644
--- a/frappe/model/delete_doc.py
+++ b/frappe/model/delete_doc.py
@@ -335,19 +335,25 @@ def clear_timeline_references(link_doctype, link_name):
WHERE `tabCommunication Link`.link_doctype=%s AND `tabCommunication Link`.link_name=%s""", (link_doctype, link_name))
def insert_feed(doc):
- from frappe.utils import get_fullname
-
- if frappe.flags.in_install or frappe.flags.in_import or getattr(doc, "no_feed_on_delete", False):
+ if (
+ frappe.flags.in_install
+ or frappe.flags.in_uninstall
+ or frappe.flags.in_import
+ or getattr(doc, "no_feed_on_delete", False)
+ ):
return
+ from frappe.utils import get_fullname
+
frappe.get_doc({
"doctype": "Comment",
"comment_type": "Deleted",
"reference_doctype": doc.doctype,
"subject": "{0} {1}".format(_(doc.doctype), doc.name),
- "full_name": get_fullname(doc.owner)
+ "full_name": get_fullname(doc.owner),
}).insert(ignore_permissions=True)
+
def delete_controllers(doctype, module):
"""
Delete controller code in the doctype folder
diff --git a/frappe/model/workflow.py b/frappe/model/workflow.py
index 7239b202bd..72ce8c9ce4 100644
--- a/frappe/model/workflow.py
+++ b/frappe/model/workflow.py
@@ -29,6 +29,8 @@ def get_transitions(doc, workflow = None, raise_exception=False):
if doc.is_new():
return []
+ doc.load_from_db()
+
frappe.has_permission(doc, 'read', throw=True)
roles = frappe.get_roles()
diff --git a/frappe/oauth.py b/frappe/oauth.py
index bf225ac118..09af5ad809 100644
--- a/frappe/oauth.py
+++ b/frappe/oauth.py
@@ -148,7 +148,7 @@ class OAuthWebRequestValidator(RequestValidator):
print("Failed body authentication: Application %s does not exist".format(cid=request.client_id))
cookie_dict = get_cookie_dict_from_headers(request)
- user_id = unquote(cookie_dict['user_id']) if 'user_id' in cookie_dict else "Guest"
+ user_id = unquote(cookie_dict.get('user_id').value) if 'user_id' in cookie_dict else "Guest"
return frappe.session.user == user_id
def authenticate_client_id(self, client_id, request, *args, **kwargs):
diff --git a/frappe/public/js/frappe/file_uploader/index.js b/frappe/public/js/frappe/file_uploader/index.js
index 62a7bff822..646f60715a 100644
--- a/frappe/public/js/frappe/file_uploader/index.js
+++ b/frappe/public/js/frappe/file_uploader/index.js
@@ -15,7 +15,11 @@ export default class FileUploader {
allow_multiple,
as_dataurl,
disable_file_browser,
+ frm
} = {}) {
+
+ frm && frm.attachments.max_reached(true);
+
if (!wrapper) {
this.make_dialog();
} else {
diff --git a/frappe/public/js/frappe/form/controls/code.js b/frappe/public/js/frappe/form/controls/code.js
index a547cfcf32..f3c51e0232 100644
--- a/frappe/public/js/frappe/form/controls/code.js
+++ b/frappe/public/js/frappe/form/controls/code.js
@@ -66,6 +66,7 @@ frappe.ui.form.ControlCode = frappe.ui.form.ControlText.extend({
const ace_language_mode = language_map[language] || '';
this.editor.session.setMode(ace_language_mode);
+ this.editor.setKeyboardHandler('ace/keyboard/vscode');
},
parse(value) {
diff --git a/frappe/public/js/frappe/form/controls/comment.js b/frappe/public/js/frappe/form/controls/comment.js
index a64df56bca..d00c915065 100644
--- a/frappe/public/js/frappe/form/controls/comment.js
+++ b/frappe/public/js/frappe/form/controls/comment.js
@@ -60,7 +60,7 @@ frappe.ui.form.ControlComment = frappe.ui.form.ControlTextEditor.extend({
update_state() {
const value = this.get_value();
- if (strip_html(value).trim() != "") {
+ if (strip_html(value).trim() != "" || value.includes('img')) {
this.button.removeClass('btn-default').addClass('btn-primary');
} else {
this.button.addClass('btn-default').removeClass('btn-primary');
diff --git a/frappe/public/js/frappe/form/dashboard.js b/frappe/public/js/frappe/form/dashboard.js
index 3b6ccd9a5c..b6daabc616 100644
--- a/frappe/public/js/frappe/form/dashboard.js
+++ b/frappe/public/js/frappe/form/dashboard.js
@@ -35,6 +35,7 @@ frappe.ui.form.Dashboard = Class.extend({
// clear custom
this.wrapper.find('.custom').remove();
+ this.hide();
},
set_headline: function(html, color) {
this.frm.layout.show_message(html, color);
@@ -171,7 +172,7 @@ frappe.ui.form.Dashboard = Class.extend({
if(this.data.graph) {
this.setup_graph();
- show = true;
+ // show = true;
}
if(show) {
@@ -494,6 +495,9 @@ frappe.ui.form.Dashboard = Class.extend({
callback: function(r) {
if(r.message) {
me.render_graph(r.message);
+ me.show();
+ } else {
+ me.hide();
}
}
});
diff --git a/frappe/public/js/frappe/form/footer/timeline.js b/frappe/public/js/frappe/form/footer/timeline.js
index 84f34d4757..159ab8a61b 100644
--- a/frappe/public/js/frappe/form/footer/timeline.js
+++ b/frappe/public/js/frappe/form/footer/timeline.js
@@ -30,7 +30,7 @@ frappe.ui.form.Timeline = class Timeline {
render_input: true,
only_input: true,
on_submit: (val) => {
- if(strip_html(val).trim() != "") {
+ if (strip_html(val).trim() != "" || val.includes('img')) {
this.insert_comment(val, this.comment_area.button);
}
}
@@ -547,10 +547,7 @@ frappe.ui.form.Timeline = class Timeline {
log.color = 'dark';
log.sender = log.owner;
log.comment_type = 'Milestone';
- log.content = __('{0} changed {1} to {2}', [
- frappe.user.full_name(log.owner).bold(),
- frappe.meta.get_label(this.frm.doctype, log.track_field),
- log.value.bold()]);
+ log.content = __('{0} changed {1} to {2}', [ frappe.user.full_name(log.owner).bold(), frappe.meta.get_label(this.frm.doctype, log.track_field), log.value.bold()]);
return log;
});
return milestones;
@@ -613,11 +610,7 @@ frappe.ui.form.Timeline = class Timeline {
const field_display_status = frappe.perm.get_field_display_status(df, null,
me.frm.perm);
if (field_display_status === 'Read' || field_display_status === 'Write') {
- parts.push(__('{0} from {1} to {2}', [
- __(df.label),
- me.format_content_for_timeline(p[1]),
- me.format_content_for_timeline(p[2])
- ]));
+ parts.push(__('{0} from {1} to {2}', [ __(df.label), me.format_content_for_timeline(p[1]), me.format_content_for_timeline(p[2])]));
}
}
}
@@ -648,13 +641,7 @@ frappe.ui.form.Timeline = class Timeline {
null, me.frm.perm);
if (field_display_status === 'Read' || field_display_status === 'Write') {
- parts.push(__('{0} from {1} to {2} in row #{3}', [
- frappe.meta.get_label(me.frm.fields_dict[row[0]].grid.doctype,
- p[0]),
- me.format_content_for_timeline(p[1]),
- me.format_content_for_timeline(p[2]),
- row[1]
- ]));
+ parts.push(__('{0} from {1} to {2} in row #{3}', [ frappe.meta.get_label( me.frm.fields_dict[row[0]].grid.doctype, p[0]), me.format_content_for_timeline(p[1]), me.format_content_for_timeline(p[2]), row[1] ]));
}
}
return parts.length < 3;
@@ -691,8 +678,7 @@ frappe.ui.form.Timeline = class Timeline {
return p;
});
if (parts.length) {
- out.push(me.get_version_comment(version, __("{0} rows for {1}",
- [__(key), parts.join(', ')])));
+ out.push(me.get_version_comment(version, __("{0} rows for {1}", [__(key), parts.join(', ')])));
}
}
});
diff --git a/frappe/public/js/frappe/form/form.js b/frappe/public/js/frappe/form/form.js
index bb9e8c22d1..90b628f269 100644
--- a/frappe/public/js/frappe/form/form.js
+++ b/frappe/public/js/frappe/form/form.js
@@ -232,14 +232,10 @@ frappe.ui.form.Form = class FrappeForm {
throw "attach error";
}
- if(me.attachments.max_reached()) {
- frappe.msgprint(__("Maximum Attachment Limit for this record reached."));
- throw "attach error";
- }
-
new frappe.ui.FileUploader({
doctype: me.doctype,
docname: me.docname,
+ frm: me,
files: dataTransfer.files,
folder: 'Home/Attachments',
on_success(file_doc) {
diff --git a/frappe/public/js/frappe/form/layout.js b/frappe/public/js/frappe/form/layout.js
index 6ea21e6e63..3505cf4857 100644
--- a/frappe/public/js/frappe/form/layout.js
+++ b/frappe/public/js/frappe/form/layout.js
@@ -113,7 +113,7 @@ frappe.ui.form.Layout = Class.extend({
label: __('Dashboard'),
cssClass: 'form-dashboard',
collapsible: 1,
- hidden: 1
+ // hidden: 1
});
},
diff --git a/frappe/public/js/frappe/form/sidebar/attachments.js b/frappe/public/js/frappe/form/sidebar/attachments.js
index 165527e281..56b484e7c4 100644
--- a/frappe/public/js/frappe/form/sidebar/attachments.js
+++ b/frappe/public/js/frappe/form/sidebar/attachments.js
@@ -16,15 +16,19 @@ frappe.ui.form.Attachments = Class.extend({
this.add_attachment_wrapper = this.parent.find(".add_attachment").parent();
this.attachments_label = this.parent.find(".attachments-label");
},
- max_reached: function() {
- // no of attachments
- var n = Object.keys(this.get_attachments()).length;
-
- // button if the number of attachments is less than max
- if(n < this.frm.meta.max_attachments || !this.frm.meta.max_attachments) {
- return false;
+ max_reached: function(raise_exception=false) {
+ const attachment_count = Object.keys(this.get_attachments()).length;
+ const attachment_limit = this.frm.meta.max_attachments;
+ if (attachment_limit && attachment_count >= attachment_limit) {
+ if (raise_exception) {
+ frappe.throw({
+ title: __("Attachment Limit Reached"),
+ message: __("Maximum attachment limit of {0} has been reached.", [cstr(attachment_limit).bold()]),
+ });
+ }
+ return true;
}
- return true;
+ return false;
},
refresh: function() {
var me = this;
@@ -140,7 +144,6 @@ frappe.ui.form.Attachments = Class.extend({
});
},
new_attachment: function(fieldname) {
- var me = this;
if (this.dialog) {
// remove upload dialog
this.dialog.$wrapper.remove();
@@ -149,6 +152,7 @@ frappe.ui.form.Attachments = Class.extend({
new frappe.ui.FileUploader({
doctype: this.frm.doctype,
docname: this.frm.docname,
+ frm: this.frm,
folder: 'Home/Attachments',
on_success: (file_doc) => {
this.attachment_uploaded(file_doc);
diff --git a/frappe/public/js/frappe/form/templates/form_sidebar.html b/frappe/public/js/frappe/form/templates/form_sidebar.html
index 67e674b1c1..481d0159c3 100644
--- a/frappe/public/js/frappe/form/templates/form_sidebar.html
+++ b/frappe/public/js/frappe/form/templates/form_sidebar.html
@@ -60,7 +60,7 @@