vassili/seitime-frappe
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 4

Merge branch 'develop' of github.com:frappe/frappe into fix-broken-folders

Browse source
This commit is contained in:
Gavin D'souza 2020-11-23 11:17:41 +05:30
commit d8335e7a0e
78 changed files with 1937 additions and 916 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
frappe/automation/doctype/assignment_rule/assignment_rule.js
View file

@ -57,7 +57,8 @@ frappe.ui.form.on('Assignment Rule', {
frm.set_fields_as_options(
'field',
doctype,
(df) => df.fieldtype == 'Link' && df.options == 'User',
(df) => ['Dynamic Link', 'Data'].includes(df.fieldtype)
|| (df.fieldtype == 'Link' && df.options == 'User'),
[{ label: 'Owner', value: 'owner' }]
);
if (doctype) {

7
frappe/automation/doctype/assignment_rule/assignment_rule.py
View file

@ -82,7 +82,7 @@ class AssignmentRule(Document):
elif self.rule == 'Load Balancing':
return self.get_user_load_balancing()
elif self.rule == 'Based on Field':
return doc.get(self.field)
return self.get_user_based_on_field(doc)
def get_user_round_robin(self):
'''
@ -119,6 +119,11 @@ class AssignmentRule(Document):
# pick the first user
return sorted_counts[0].get('user')
def get_user_based_on_field(self, doc):
val = doc.get(self.field)
if frappe.db.exists('User', val):
return val
def safe_eval(self, fieldname, doc):
try:
if self.get(fieldname):

2
frappe/build.py
View file

@ -105,7 +105,7 @@ def download_frappe_assets(verbose=True):
if frappe_head:
try:
url = get_assets_link(frappe_head)
click.secho("Retreiving assets...", fg="yellow")
click.secho("Retrieving assets...", fg="yellow")
prefix = mkdtemp(prefix="frappe-assets-", suffix=frappe_head)
assets_archive = download_file(url, prefix)
print("\n{0} Downloaded Frappe assets from {1}".format(green('✔'), url))

214
frappe/commands/site.py
View file

@ -9,7 +9,7 @@ import click
import frappe
from frappe.commands import get_site, pass_context
from frappe.exceptions import SiteNotSpecifiedError
from frappe.utils import get_site_path, touch_file
from frappe.installer import _new_site
@click.command('new-site')
@ -42,57 +42,6 @@ def new_site(site, mariadb_root_username=None, mariadb_root_password=None, admin
if len(frappe.utils.get_sites()) == 1:
use(site)
def _new_site(db_name, site, mariadb_root_username=None, mariadb_root_password=None,
admin_password=None, verbose=False, install_apps=None, source_sql=None, force=False,
no_mariadb_socket=False, reinstall=False, db_password=None, db_type=None, db_host=None,
db_port=None, new_site=False):
"""Install a new Frappe site"""
if not force and os.path.exists(site):
print('Site {0} already exists'.format(site))
sys.exit(1)
if no_mariadb_socket and not db_type == "mariadb":
print('--no-mariadb-socket requires db_type to be set to mariadb.')
sys.exit(1)
if not db_name:
import hashlib
db_name = '_' + hashlib.sha1(site.encode()).hexdigest()[:16]
from frappe.commands.scheduler import _is_scheduler_enabled
from frappe.installer import install_db, make_site_dirs
from frappe.installer import install_app as _install_app
import frappe.utils.scheduler
frappe.init(site=site)
try:
# enable scheduler post install?
enable_scheduler = _is_scheduler_enabled()
except Exception:
enable_scheduler = False
make_site_dirs()
installing = touch_file(get_site_path('locks', 'installing.lock'))
install_db(root_login=mariadb_root_username, root_password=mariadb_root_password, db_name=db_name,
admin_password=admin_password, verbose=verbose, source_sql=source_sql, force=force, reinstall=reinstall,
db_password=db_password, db_type=db_type, db_host=db_host, db_port=db_port, no_mariadb_socket=no_mariadb_socket)
apps_to_install = ['frappe'] + (frappe.conf.get("install_apps") or []) + (list(install_apps) or [])
for app in apps_to_install:
_install_app(app, verbose=verbose, set_as_patched=not source_sql)
os.remove(installing)
frappe.utils.scheduler.toggle_scheduler(enable_scheduler)
frappe.db.commit()
scheduler_status = "disabled" if frappe.utils.scheduler.is_scheduler_disabled() else "enabled"
print("*** Scheduler is", scheduler_status, "***")
@click.command('restore')
@click.argument('sql-file-path')
@ -103,36 +52,45 @@ def _new_site(db_name, site, mariadb_root_username=None, mariadb_root_password=N
@click.option('--install-app', multiple=True, help='Install app after installation')
@click.option('--with-public-files', help='Restores the public files of the site, given path to its tar file')
@click.option('--with-private-files', help='Restores the private files of the site, given path to its tar file')
@click.option('--force', is_flag=True, default=False, help='Ignore the site downgrade warning, if applicable')
@click.option('--force', is_flag=True, default=False, help='Ignore the validations and downgrade warnings. This action is not recommended')
@pass_context
def restore(context, sql_file_path, mariadb_root_username=None, mariadb_root_password=None, db_name=None, verbose=None, install_app=None, admin_password=None, force=None, with_public_files=None, with_private_files=None):
"Restore site database from an sql file"
from frappe.installer import extract_sql_gzip, extract_files, is_downgrade
from frappe.installer import (
extract_sql_from_archive,
extract_files,
is_downgrade,
is_partial,
validate_database_sql
)
force = context.force or force
decompressed_file_name = extract_sql_from_archive(sql_file_path)
# Extract the gzip file if user has passed *.sql.gz file instead of *.sql file
if not os.path.exists(sql_file_path):
base_path = '..'
sql_file_path = os.path.join(base_path, sql_file_path)
if not os.path.exists(sql_file_path):
print('Invalid path {0}'.format(sql_file_path[3:]))
sys.exit(1)
elif sql_file_path.startswith(os.sep):
base_path = os.sep
else:
base_path = '.'
# check if partial backup
if is_partial(decompressed_file_name):
click.secho(
"Partial Backup file detected. You cannot use a partial file to restore a Frappe Site.",
fg="red"
)
click.secho(
"Use `bench partial-restore` to restore a partial backup to an existing site.",
fg="yellow"
)
sys.exit(1)
if sql_file_path.endswith('sql.gz'):
decompressed_file_name = extract_sql_gzip(os.path.abspath(sql_file_path))
else:
decompressed_file_name = sql_file_path
# check if valid SQL file
validate_database_sql(decompressed_file_name, _raise=not force)
site = get_site(context)
frappe.init(site=site)
# dont allow downgrading to older versions of frappe without force
if not force and is_downgrade(decompressed_file_name, verbose=True):
warn_message = "This is not recommended and may lead to unexpected behaviour. Do you want to continue anyway?"
warn_message = (
"This is not recommended and may lead to unexpected behaviour. "
"Do you want to continue anyway?"
)
click.confirm(warn_message, abort=True)
_new_site(frappe.conf.db_name, site, mariadb_root_username=mariadb_root_username,
@ -155,9 +113,28 @@ def restore(context, sql_file_path, mariadb_root_username=None, mariadb_root_pas
if decompressed_file_name != sql_file_path:
os.remove(decompressed_file_name)
success_message = "Site {0} has been restored{1}".format(site, " with files" if (with_public_files or with_private_files) else "")
success_message = "Site {0} has been restored{1}".format(
site,
" with files" if (with_public_files or with_private_files) else ""
)
click.secho(success_message, fg="green")
@click.command('partial-restore')
@click.argument('sql-file-path')
@click.option("--verbose", "-v", is_flag=True)
@pass_context
def partial_restore(context, sql_file_path, verbose):
from frappe.installer import partial_restore
verbose = context.verbose or verbose
site = get_site(context)
frappe.init(site=site)
frappe.connect(site=site)
partial_restore(sql_file_path, verbose)
frappe.destroy()
@click.command('reinstall')
@click.option('--admin-password', help='Administrator Password for reinstalled site')
@click.option('--mariadb-root-username', help='Root username for MariaDB')
@ -222,15 +199,51 @@ def install_app(context, apps):
sys.exit(exit_code)
@click.command('list-apps')
@click.command("list-apps")
@pass_context
def list_apps(context):
"List apps in site"
site = get_site(context)
frappe.init(site=site)
frappe.connect()
print("\n".join(frappe.get_installed_apps()))
frappe.destroy()
def fix_whitespaces(text):
if site == context.sites[-1]:
text = text.rstrip()
if len(context.sites) == 1:
text = text.lstrip()
return text
for site in context.sites:
frappe.init(site=site)
frappe.connect()
site_title = (
click.style(f"{site}", fg="green") if len(context.sites) > 1 else ""
)
apps = frappe.get_single("Installed Applications").installed_applications
if apps:
name_len, ver_len = [
max([len(x.get(y)) for x in apps])
for y in ["app_name", "app_version"]
]
template = "{{0:{0}}} {{1:{1}}} {{2}}".format(name_len, ver_len)
installed_applications = [
template.format(app.app_name, app.app_version, app.git_branch)
for app in apps
]
applications_summary = "\n".join(installed_applications)
summary = f"{site_title}\n{applications_summary}\n"
else:
applications_summary = "\n".join(frappe.get_installed_apps())
summary = f"{site_title}\n{applications_summary}\n"
summary = fix_whitespaces(summary)
if applications_summary and summary:
print(summary)
frappe.destroy()
@click.command('add-system-manager')
@click.argument('email')
@ -305,15 +318,16 @@ def migrate_to(context, frappe_provider):
@click.command('run-patch')
@click.argument('module')
@click.option('--force', is_flag=True)
@pass_context
def run_patch(context, module):
def run_patch(context, module, force):
"Run a particular patch"
import frappe.modules.patch_handler
for site in context.sites:
frappe.init(site=site)
try:
frappe.connect()
frappe.modules.patch_handler.run_single(module, force=context.force)
frappe.modules.patch_handler.run_single(module, force=force or context.force)
finally:
frappe.destroy()
if not context.sites:
@ -378,16 +392,20 @@ def use(site, sites_path='.'):
@click.command('backup')
@click.option('--with-files', default=False, is_flag=True, help="Take backup with files")
@click.option('--include', '--only', '-i', default="", type=str, help="Specify the DocTypes to backup seperated by commas")
@click.option('--exclude', '-e', default="", type=str, help="Specify the DocTypes to not backup seperated by commas")
@click.option('--backup-path', default=None, help="Set path for saving all the files in this operation")
@click.option('--backup-path-db', default=None, help="Set path for saving database file")
@click.option('--backup-path-files', default=None, help="Set path for saving public file")
@click.option('--backup-path-private-files', default=None, help="Set path for saving private file")
@click.option('--backup-path-conf', default=None, help="Set path for saving config file")
@click.option('--ignore-backup-conf', default=False, is_flag=True, help="Ignore excludes/includes set in config")
@click.option('--verbose', default=False, is_flag=True, help="Add verbosity")
@click.option('--compress', default=False, is_flag=True, help="Compress private and public files")
@pass_context
def backup(context, with_files=False, backup_path=None, backup_path_db=None, backup_path_files=None,
backup_path_private_files=None, backup_path_conf=None, verbose=False, compress=False):
backup_path_private_files=None, backup_path_conf=None, ignore_backup_conf=False, verbose=False,
compress=False, include="", exclude=""):
"Backup"
from frappe.utils.backups import scheduled_backup
verbose = verbose or context.verbose
@ -397,11 +415,27 @@ def backup(context, with_files=False, backup_path=None, backup_path_db=None, bac
try:
frappe.init(site=site)
frappe.connect()
odb = scheduled_backup(ignore_files=not with_files, backup_path=backup_path, backup_path_db=backup_path_db, backup_path_files=backup_path_files, backup_path_private_files=backup_path_private_files, backup_path_conf=backup_path_conf, force=True, verbose=verbose, compress=compress)
odb = scheduled_backup(
ignore_files=not with_files,
backup_path=backup_path,
backup_path_db=backup_path_db,
backup_path_files=backup_path_files,
backup_path_private_files=backup_path_private_files,
backup_path_conf=backup_path_conf,
ignore_conf=ignore_backup_conf,
include_doctypes=include,
exclude_doctypes=exclude,
compress=compress,
verbose=verbose,
force=True
)
except Exception:
click.secho("Backup failed for Site {0}. Database or site_config.json may be corrupted".format(site), fg="red")
if verbose:
print(frappe.get_traceback())
exit_code = 1
continue
odb.print_summary()
click.secho("Backup for Site {0} has been successfully completed{1}".format(site, " with files" if with_files else ""), fg="green")
frappe.destroy()
@ -474,13 +508,14 @@ def _drop_site(site, root_login='root', root_password=None, archived_sites_path=
if force:
pass
else:
click.echo("="*80)
click.echo("Error: The operation has stopped because backup of {s}'s database failed.".format(s=site))
click.echo("Reason: {reason}{sep}".format(reason=str(err), sep="\n"))
click.echo("Fix the issue and try again.")
click.echo(
"Hint: Use 'bench drop-site {s} --force' to force the removal of {s}".format(sep="\n", tab="\t", s=site)
)
messages = [
"=" * 80,
"Error: The operation has stopped because backup of {0}'s database failed.".format(site),
"Reason: {0}\n".format(str(err)),
"Fix the issue and try again.",
"Hint: Use 'bench drop-site {0} --force' to force the removal of {0}".format(site)
]
click.echo("\n".join(messages))
sys.exit(1)
drop_user_and_database(frappe.conf.db_name, root_login, root_password)
@ -696,5 +731,6 @@ commands = [
stop_recording,
add_to_hosts,
start_ngrok,
build_search_index
build_search_index,
partial_restore
]

20
frappe/core/doctype/doctype/doctype.py
View file

@ -56,7 +56,8 @@ class DocType(Document):
- Check fieldnames (duplication etc)
- Clear permission table for child tables
- Add `amended_from` and `amended_by` if Amendable
- Add custom field `auto_repeat` if Repeatable"""
- Add custom field `auto_repeat` if Repeatable
- Check if links point to valid fieldnames"""
self.check_developer_mode()
@ -88,6 +89,7 @@ class DocType(Document):
self.make_repeatable()
self.validate_nestedset()
self.validate_website()
self.validate_links_table_fieldnames()
if not self.is_new():
self.before_update = frappe.get_doc('DocType', self.name)
@ -588,7 +590,8 @@ class DocType(Document):
def make_repeatable(self):
"""If allow_auto_repeat is set, add auto_repeat custom field."""
if self.allow_auto_repeat:
if not frappe.db.exists('Custom Field', {'fieldname': 'auto_repeat', 'dt': self.name}):
if not frappe.db.exists('Custom Field', {'fieldname': 'auto_repeat', 'dt': self.name}) and \
not frappe.db.exists('DocField', {'fieldname': 'auto_repeat', 'parent': self.name}):
insert_after = self.fields[len(self.fields) - 1].fieldname
df = dict(fieldname='auto_repeat', label='Auto Repeat', fieldtype='Link', options='Auto Repeat', insert_after=insert_after, read_only=1, no_copy=1, print_hide=1)
create_custom_field(self.name, df)
@ -674,6 +677,19 @@ class DocType(Document):
if not re.match("^(?![\W])[^\d_\s][\w ]+$", name, **flags):
frappe.throw(_("DocType's name should start with a letter and it can only consist of letters, numbers, spaces and underscores"), frappe.NameError)
def validate_links_table_fieldnames(self):
"""Validate fieldnames in Links table"""
if frappe.flags.in_patch: return
if frappe.flags.in_fixtures: return
if not self.links: return
for index, link in enumerate(self.links):
meta = frappe.get_meta(link.link_doctype)
if not meta.get_field(link.link_fieldname):
message = _("Row #{0}: Could not find field {1} in {2} DocType").format(index+1, frappe.bold(link.link_fieldname), frappe.bold(link.link_doctype))
frappe.throw(message, InvalidFieldNameError, _("Invalid Fieldname"))
def validate_fields_for_doctype(doctype):
doc = frappe.get_doc("DocType", doctype)

27
frappe/core/doctype/doctype/test_doctype.py
View file

@ -451,6 +451,33 @@ class TestDocType(unittest.TestCase):
test_doc_1.delete()
frappe.db.commit()
def test_links_table_fieldname_validation(self):
doc = new_doctype("Test Links Table Validation")
# check valid data
doc.append("links", {
'link_doctype': "User",
'link_fieldname': "first_name"
})
doc.validate_links_table_fieldnames() # no error
doc.links = [] # reset links table
# check invalid doctype
doc.append("links", {
'link_doctype': "User2",
'link_fieldname': "first_name"
})
self.assertRaises(frappe.DoesNotExistError, doc.validate_links_table_fieldnames)
doc.links = [] # reset links table
# check invalid fieldname
doc.append("links", {
'link_doctype': "User",
'link_fieldname': "a_field_that_does_not_exists"
})
self.assertRaises(InvalidFieldNameError, doc.validate_links_table_fieldnames)
def new_doctype(name, unique=0, depends_on='', fields=None):
doc = frappe.get_doc({
"doctype": "DocType",

28
frappe/core/doctype/file/file.py
View file

@ -93,6 +93,7 @@ class File(Document):
self.set_is_private()
self.set_file_name()
self.validate_duplicate_entry()
self.validate_attachment_limit()
self.validate_folder()
if not self.file_url and not self.flags.ignore_file_validate:
@ -140,6 +141,26 @@ class File(Document):
if self.file_url and (self.is_private != self.file_url.startswith('/private')):
frappe.throw(_('Invalid file URL. Please contact System Administrator.'))
def validate_attachment_limit(self):
attachment_limit = 0
if self.attached_to_doctype and self.attached_to_name:
attachment_limit = cint(frappe.get_meta(self.attached_to_doctype).max_attachments)
if attachment_limit:
current_attachment_count = len(frappe.get_all('File', filters={
'attached_to_doctype': self.attached_to_doctype,
'attached_to_name': self.attached_to_name,
}, limit=attachment_limit + 1))
if current_attachment_count >= attachment_limit:
frappe.throw(
_("Maximum Attachment Limit of {0} has been reached for {1} {2}.").format(
frappe.bold(attachment_limit), self.attached_to_doctype, self.attached_to_name
),
exc=frappe.exceptions.AttachmentLimitReached,
title=_('Attachment Limit Reached')
)
def set_folder_name(self):
"""Make parent folders if not exists based on reference doctype and name"""
if self.attached_to_doctype and not self.folder:
@ -612,7 +633,12 @@ def get_extension(filename, extn, content):
return extn
def get_local_image(file_url):
file_path = frappe.get_site_path("public", file_url.lstrip("/"))
if file_url.startswith("/private"):
file_url_path = (file_url.lstrip("/"), )
else:
file_url_path = ("public", file_url.lstrip("/"))
file_path = frappe.get_site_path(*file_url_path)
try:
image = Image.open(file_path)

25
frappe/core/doctype/file/test_file.py
View file

@ -160,6 +160,31 @@ class TestSameContent(unittest.TestCase):
def test_saved_content(self):
self.assertFalse(os.path.exists(get_files_path(self.dup_filename)))
def test_attachment_limit(self):
doctype, docname = make_test_doc()
from frappe.custom.doctype.property_setter.property_setter import make_property_setter
limit_property = make_property_setter('ToDo', None, 'max_attachments', 1, 'int', for_doctype=True)
file1 = frappe.get_doc({
"doctype": "File",
"file_name": 'test-attachment',
"attached_to_doctype": doctype,
"attached_to_name": docname,
"content": 'test'
})
file1.insert()
file2 = frappe.get_doc({
"doctype": "File",
"file_name": 'test-attachment',
"attached_to_doctype": doctype,
"attached_to_name": docname,
"content": 'test2'
})
self.assertRaises(frappe.exceptions.AttachmentLimitReached, file2.insert)
limit_property.delete()
frappe.clear_cache(doctype='ToDo')
def tearDown(self):
# File gets deleted on rollback, so blank

20
frappe/core/doctype/prepared_report/prepared_report.py
View file

@ -89,20 +89,18 @@ def delete_expired_prepared_reports():
'creation': ['<', frappe.utils.add_days(frappe.utils.now(), -expiry_period)]
})
args = {
'reports': prepared_reports_to_delete,
'limit': 50
}
enqueue(method=delete_prepared_reports, job_name="delete_prepared_reports", **args)
batches = frappe.utils.create_batch(prepared_reports_to_delete, 100)
for batch in batches:
args = {
'reports': batch,
}
enqueue(method=delete_prepared_reports, job_name="delete_prepared_reports", **args)
@frappe.whitelist()
def delete_prepared_reports(reports, limit=None):
def delete_prepared_reports(reports):
reports = frappe.parse_json(reports)
for index, doc in enumerate(reports):
if limit and index == limit:
return
frappe.delete_doc('Prepared Report', doc['name'], ignore_permissions=True)
for report in reports:
frappe.delete_doc('Prepared Report', report['name'], ignore_permissions=True)
def create_json_gz_file(data, dt, dn):
# Storing data in CSV file causes information loss

5
frappe/core/doctype/report/report.py
View file

@ -61,8 +61,9 @@ class Report(Document):
def set_doctype_roles(self):
if not self.get('roles') and self.is_standard == 'No':
meta = frappe.get_meta(self.ref_doctype)
roles = [{'role': d.role} for d in meta.permissions if d.permlevel==0]
self.set('roles', roles)
if not meta.istable:
roles = [{'role': d.role} for d in meta.permissions if d.permlevel==0]
self.set('roles', roles)
def is_permitted(self):
"""Returns true if Has Role is not set or the user is allowed."""

4
frappe/core/doctype/role/role.py
View file

@ -37,7 +37,7 @@ class Role(Document):
def get_info_based_on_role(role, field='email'):
''' Get information of all users that have been assigned this role '''
users = frappe.get_list("Has Role", filters={"role": role, "parenttype": "User"},
fields=["parent"])
fields=["parent as user_name"])
return get_user_info(users, field)
@ -45,7 +45,7 @@ def get_user_info(users, field='email'):
''' Fetch details about users for the specified field '''
info_list = []
for user in users:
user_info, enabled = frappe.db.get_value("User", user.parent, [field, "enabled"])
user_info, enabled = frappe.db.get_value("User", user.get("user_name"), [field, "enabled"])
if enabled and user_info not in ["admin@example.com", "guest@example.com"]:
info_list.append(user_info)
return info_list

3
frappe/core/doctype/server_script/server_script.json
View file

@ -31,6 +31,7 @@
"fieldname": "script",
"fieldtype": "Code",
"label": "Script",
"options": "Python",
"reqd": 1
},
{
@ -87,7 +88,7 @@
],
"index_web_pages_for_search": 1,
"links": [],
"modified": "2020-08-24 16:44:41.060350",
"modified": "2020-11-11 12:39:41.391052",
"modified_by": "Administrator",
"module": "Core",
"name": "Server Script",

8
frappe/core/doctype/user/user.py
View file

@ -13,7 +13,7 @@ from frappe.utils.user import get_system_managers
from bs4 import BeautifulSoup
import frappe.permissions
import frappe.share
import frappe.defaults
from frappe.website.utils import is_signup_enabled
from frappe.utils.background_jobs import enqueue
@ -107,6 +107,10 @@ class User(Document):
)
if self.name not in ('Administrator', 'Guest') and not self.user_image:
frappe.enqueue('frappe.core.doctype.user.user.update_gravatar', name=self.name)
# Set user selected timezone
if self.time_zone:
frappe.defaults.set_default("time_zone", self.time_zone, self.name)
def has_website_permission(self, ptype, user, verbose=False):
"""Returns true if current user is the session user"""
@ -1129,4 +1133,4 @@ def check_password_reset_limit(user, rate_limit):
frappe.throw(_("You have reached the hourly limit for generating password reset links. Please try again later."))
def get_generated_link_count(user):
return cint(frappe.cache().hget("password_reset_link_count", user)) or 0
return cint(frappe.cache().hget("password_reset_link_count", user)) or 0

4
frappe/custom/doctype/customize_form/customize_form.py
View file

@ -76,8 +76,8 @@ class CustomizeForm(Document):
def create_auto_repeat_custom_field_if_requried(self, meta):
if self.allow_auto_repeat:
if not frappe.db.exists('Custom Field', {'fieldname': 'auto_repeat',
'dt': self.doc_type}):
if not frappe.db.exists('Custom Field', {'fieldname': 'auto_repeat', 'dt': self.doc_type}) and \
not frappe.db.exists('DocField', {'fieldname': 'auto_repeat', 'parent': self.name}):
insert_after = self.fields[len(self.fields) - 1].fieldname
df = dict(
fieldname='auto_repeat',

11
frappe/database/db_manager.py
View file

@ -3,7 +3,6 @@ import frappe
class DbManager:
def __init__(self, db):
"""
Pass root_conn here for access to all databases.
@ -66,10 +65,10 @@ class DbManager:
esc = make_esc('$ ')
from distutils.spawn import find_executable
pipe = find_executable('pv')
if pipe:
pipe = '{pipe} {source} |'.format(
pipe=pipe,
pv = find_executable('pv')
if pv:
pipe = '{pv} {source} |'.format(
pv=pv,
source=source
)
source = ''
@ -78,7 +77,7 @@ class DbManager:
source = '< {source}'.format(source=source)
if pipe:
print('Creating Database...')
print('Restoring Database file...')
command = '{pipe} mysql -u {user} -p{password} -h{host} ' + ('-P{port}' if frappe.db.port else '') + ' {target} {source}'
command = command.format(

18
frappe/database/mariadb/setup_db.py
View file

@ -1,7 +1,7 @@
from __future__ import unicode_literals
import frappe
import os, sys
import os
from frappe.database.db_manager import DbManager
expected_settings_10_2_earlier = {
@ -86,6 +86,8 @@ def drop_user_and_database(db_name, root_login, root_password):
dbman.drop_database(db_name)
def bootstrap_database(db_name, verbose, source_sql=None):
import sys
frappe.connect(db_name=db_name)
if not check_database_settings():
print('Database settings do not match expected values; stopping database setup.')
@ -94,9 +96,17 @@ def bootstrap_database(db_name, verbose, source_sql=None):
import_db_from_sql(source_sql, verbose)
frappe.connect(db_name=db_name)
if not 'tabDefaultValue' in frappe.db.get_tables():
print('''Database not installed, this can due to lack of permission, or that the database name exists.
Check your mysql root password, or use --force to reinstall''')
if 'tabDefaultValue' not in frappe.db.get_tables():
from click import secho
secho(
"Table 'tabDefaultValue' missing in the restored site. "
"Database not installed correctly, this can due to lack of "
"permission, or that the database name exists. Check your mysql"
" root password, validity of the backup file or use --force to"
" reinstall",
fg="red"
)
sys.exit(1)
def import_db_from_sql(source_sql=None, verbose=False):

65
frappe/database/postgres/setup_db.py
View file

@ -1,5 +1,7 @@
import frappe, subprocess, os
from six.moves import input
import os
import frappe
def setup_database(force, source_sql=None, verbose=False):
root_conn = get_root_connection()
@ -10,24 +12,62 @@ def setup_database(force, source_sql=None, verbose=False):
root_conn.sql("CREATE user {0} password '{1}'".format(frappe.conf.db_name,
frappe.conf.db_password))
root_conn.sql("GRANT ALL PRIVILEGES ON DATABASE `{0}` TO {0}".format(frappe.conf.db_name))
root_conn.close()
bootstrap_database(frappe.conf.db_name, verbose, source_sql=source_sql)
frappe.connect()
def bootstrap_database(db_name, verbose, source_sql=None):
frappe.connect(db_name=db_name)
import_db_from_sql(source_sql, verbose)
frappe.connect(db_name=db_name)
if 'tabDefaultValue' not in frappe.db.get_tables():
import sys
from click import secho
secho(
"Table 'tabDefaultValue' missing in the restored site. "
"This may be due to incorrect permissions or the result of a restore from a bad backup file. "
"Database not installed correctly.",
fg="red"
)
sys.exit(1)
def import_db_from_sql(source_sql=None, verbose=False):
from shutil import which
from subprocess import run, PIPE
# we can't pass psql password in arguments in postgresql as mysql. So
# set password connection parameter in environment variable
subprocess_env = os.environ.copy()
subprocess_env['PGPASSWORD'] = str(frappe.conf.db_password)
# bootstrap db
if not source_sql:
source_sql = os.path.join(os.path.dirname(__file__), 'framework_postgres.sql')
subprocess.check_output([
'psql', frappe.conf.db_name,
'-h', frappe.conf.db_host or 'localhost',
'-p', str(frappe.conf.db_port or '5432'),
'-U', frappe.conf.db_name,
'-f', source_sql
], env=subprocess_env)
pv = which('pv')
frappe.connect()
_command = (
f"psql {frappe.conf.db_name} "
f"-h {frappe.conf.db_host or 'localhost'} -p {str(frappe.conf.db_port or '5432')} "
f"-U {frappe.conf.db_name}"
)
if pv:
command = f"{pv} {source_sql} | " + _command
else:
command = _command + f" -f {source_sql}"
print("Restoring Database file...")
if verbose:
print(command)
restore_proc = run(command, env=subprocess_env, shell=True, stdout=PIPE)
if verbose:
print(f"\nSTDOUT by psql:\n{restore_proc.stdout.decode()}\nImported from Database File: {source_sql}")
def setup_help_database(help_db_name):
root_conn = get_root_connection()
@ -38,19 +78,20 @@ def setup_help_database(help_db_name):
root_conn.sql("GRANT ALL PRIVILEGES ON DATABASE `{0}` TO {0}".format(help_db_name))
def get_root_connection(root_login=None, root_password=None):
import getpass
if not frappe.local.flags.root_connection:
if not root_login:
root_login = frappe.conf.get("root_login") or None
if not root_login:
from six.moves import input
root_login = input("Enter postgres super user: ")
if not root_password:
root_password = frappe.conf.get("root_password") or None
if not root_password:
root_password = getpass.getpass("Postgres super user password: ")
from getpass import getpass
root_password = getpass("Postgres super user password: ")
frappe.local.flags.root_connection = frappe.database.get_db(user=root_login, password=root_password)

15
frappe/email/doctype/notification/notification.js
View file

@ -97,14 +97,7 @@ frappe.notification = {
},
setup_example_message: function(frm) {
let template = '';
if (frm.doc.channel === 'WhatsApp') {
template = `<h5 style='display: inline-block'>Warning:</h5> Only Use Pre-Approved WhatsApp for Business Template
<h5>Message Example</h5>
<pre>
Your appointment is coming up on {{ doc.date }} at {{ doc.time }}
</pre>`;
} else if (frm.doc.channel === 'Email') {
if (frm.doc.channel === 'Email') {
template = `<h5>Message Example</h5>
<pre>&lt;h3&gt;Order Overdue&lt;/h3&gt;
@ -124,7 +117,7 @@ Last comment: {{ comments[-1].comment }} by {{ comments[-1].by }}
&lt;/ul&gt;
</pre>
`;
} else {
} else if (in_list(['Slack', 'System Notification', 'SMS'], frm.doc.channel)) {
template = `<h5>Message Example</h5>
<pre>*Order Overdue*
@ -142,7 +135,9 @@ Last comment: {{ comments[-1].comment }} by {{ comments[-1].by }}
• Amount: {{ doc.grand_total }}
</pre>`;
}
frm.set_df_property('message_examples', 'options', template);
if (template) {
frm.set_df_property('message_examples', 'options', template);
}
}
};

20
frappe/email/doctype/notification/notification.json
View file

@ -10,7 +10,6 @@
"enabled",
"column_break_2",
"channel",
"twilio_number",
"slack_webhook_url",
"filters",
"subject",
@ -61,7 +60,7 @@
"fieldname": "channel",
"fieldtype": "Select",
"label": "Channel",
"options": "Email\nSlack\nSystem Notification\nWhatsApp\nSMS",
"options": "Email\nSlack\nSystem Notification\nSMS",
"reqd": 1,
"set_only_once": 1
},
@ -80,14 +79,14 @@
"label": "Filters"
},
{
"depends_on": "eval: !in_list(['SMS', 'WhatsApp'], doc.channel)",
"depends_on": "eval: in_list(['Email', 'Slack', 'System Notification'], doc.channel)",
"description": "To add dynamic subject, use jinja tags like\n\n<div><pre><code>{{ doc.name }} Delivered</code></pre></div>",
"fieldname": "subject",
"fieldtype": "Data",
"ignore_xss_filter": 1,
"in_list_view": 1,
"label": "Subject",
"mandatory_depends_on": "eval:!in_list(['SMS', 'WhatsApp'], doc.channel)"
"mandatory_depends_on": "eval: in_list(['Email', 'Slack', 'System Notification'], doc.channel)"
},
{
"fieldname": "document_type",
@ -208,7 +207,7 @@
"label": "Value To Be Set"
},
{
"depends_on": "eval:in_list(['Email', 'SMS', 'WhatsApp'], doc.channel)",
"depends_on": "eval:in_list(['Email', 'SMS'], doc.channel)",
"fieldname": "column_break_5",
"fieldtype": "Section Break",
"label": "Recipients"
@ -263,15 +262,6 @@
"label": "Print Format",
"options": "Print Format"
},
{
"depends_on": "eval: doc.channel==='WhatsApp'",
"description": "To use WhatsApp for Business, initialize <a href=\"#Form/Twilio Settings\">Twilio Settings</a>.",
"fieldname": "twilio_number",
"fieldtype": "Link",
"label": "Twilio Number",
"mandatory_depends_on": "eval: doc.channel==='WhatsApp'",
"options": "Twilio Number Group"
},
{
"default": "0",
"depends_on": "eval: doc.channel !== 'System Notification'",
@ -291,7 +281,7 @@
"icon": "fa fa-envelope",
"index_web_pages_for_search": 1,
"links": [],
"modified": "2020-09-03 10:33:23.084590",
"modified": "2020-10-28 11:04:54.955567",
"modified_by": "Administrator",
"module": "Email",
"name": "Notification",

21
frappe/email/doctype/notification/notification.py
View file

@ -14,7 +14,6 @@ from frappe.utils.safe_exec import get_safe_globals
from frappe.modules.utils import export_module_json, get_doc_module
from six import string_types
from frappe.integrations.doctype.slack_webhook_url.slack_webhook_url import send_slack_message
from frappe.integrations.doctype.twilio_settings.twilio_settings import send_whatsapp_message
from frappe.core.doctype.sms_settings.sms_settings import send_sms
from frappe.desk.doctype.notification_log.notification_log import enqueue_create_notification
@ -29,7 +28,7 @@ class Notification(Document):
self.name = self.subject
def validate(self):
if self.channel not in ('WhatsApp', 'SMS'):
if self.channel in ("Email", "Slack", "System Notification"):
validate_template(self.subject)
validate_template(self.message)
@ -43,7 +42,6 @@ class Notification(Document):
self.validate_forbidden_types()
self.validate_condition()
self.validate_standard()
self.validate_twilio_settings()
frappe.cache().hdel('notifications', self.document_type)
def on_update(self):
@ -70,11 +68,6 @@ def get_context(context):
if self.is_standard and not frappe.conf.developer_mode:
frappe.throw(_('Cannot edit Standard Notification. To edit, please disable this and duplicate it'))
def validate_twilio_settings(self):
if self.enabled and self.channel == "WhatsApp" \
and not frappe.db.get_single_value("Twilio Settings", "enabled"):
frappe.throw(_("Please enable Twilio settings to send WhatsApp messages"))
def validate_condition(self):
temp_doc = frappe.new_doc(self.document_type)
if self.condition:
@ -137,9 +130,6 @@ def get_context(context):
if self.channel == 'Slack':
self.send_a_slack_msg(doc, context)
if self.channel == 'WhatsApp':
self.send_whatsapp_msg(doc, context)
if self.channel == 'SMS':
self.send_sms(doc, context)
@ -230,13 +220,6 @@ def get_context(context):
reference_doctype=doc.doctype,
reference_name=doc.name)
def send_whatsapp_msg(self, doc, context):
send_whatsapp_message(
sender=self.twilio_number,
receiver_list=self.get_receiver_list(doc, context),
message=frappe.render_template(self.message, context),
)
def send_sms(self, doc, context):
send_sms(
receiver_list=self.get_receiver_list(doc, context),
@ -302,7 +285,7 @@ def get_context(context):
# For sending messages to the owner's mobile phone number
if recipient.receiver_by_document_field == 'owner':
receiver_list.append(get_user_info(doc.get('owner'), 'mobile_no'))
receiver_list += get_user_info([dict(user_name=doc.get('owner'))], 'mobile_no')
# For sending messages to the number specified in the receiver field
elif recipient.receiver_by_document_field:
receiver_list.append(doc.get(recipient.receiver_by_document_field))

59
frappe/event_streaming/doctype/event_consumer/event_consumer.py
View file

@ -31,10 +31,12 @@ class EventConsumer(Document):
self.update_consumer_status()
else:
frappe.db.set_value(self.doctype, self.name, 'incoming_change', 0)
frappe.cache().delete_value('event_consumer_document_type_map')
def on_trash(self):
for i in frappe.get_all('Event Update Log Consumer', {'consumer': self.name}):
frappe.delete_doc('Event Update Log Consumer', i.name)
frappe.cache().delete_value('event_consumer_document_type_map')
def update_consumer_status(self):
@ -88,8 +90,9 @@ def register_consumer(data):
for entry in consumer_doctypes:
consumer.append('consumer_doctypes', {
'ref_doctype': entry,
'status': 'Pending'
'ref_doctype': entry.get('doctype'),
'status': 'Pending',
'condition': entry.get('condition')
})
consumer.insert()
@ -153,3 +156,53 @@ def notify(consumer):
jobs = get_jobs()
if not jobs or enqueued_method not in jobs[frappe.local.site] and not consumer.flags.notifed:
frappe.enqueue(enqueued_method, queue='long', enqueue_after_commit=True, **{'consumer': consumer})
def has_consumer_access(consumer, update_log):
"""Checks if consumer has completely satisfied all the conditions on the doc"""
if isinstance(consumer, str):
consumer = frappe.get_doc('Event Consumer', consumer)
if not frappe.db.exists(update_log.ref_doctype, update_log.docname):
# Delete Log
# Check if the last Update Log of this document was read by this consumer
last_update_log = frappe.get_all(
'Event Update Log',
filters={
'ref_doctype': update_log.ref_doctype,
'docname': update_log.docname,
'creation': ['<', update_log.creation]
},
order_by='creation desc',
limit_page_length=1
)
if not len(last_update_log):
return False
last_update_log = frappe.get_doc('Event Update Log', last_update_log[0].name)
return len([x for x in last_update_log.consumers if x.consumer == consumer.name])
doc = frappe.get_doc(update_log.ref_doctype, update_log.docname)
try:
for dt_entry in consumer.consumer_doctypes:
if dt_entry.ref_doctype != update_log.ref_doctype:
continue
if not dt_entry.condition:
return True
condition: str = dt_entry.condition
if condition.startswith('cmd:'):
cmd = condition.split('cmd:')[1].strip()
args = {
'consumer': consumer,
'doc': doc,
'update_log': update_log
}
return frappe.call(cmd, **args)
else:
return frappe.safe_eval(condition, frappe._dict(doc=doc))
except Exception as e:
frappe.log_error(title='has_consumer_access error', message=e)
return False

11
frappe/event_streaming/doctype/event_consumer_document_type/event_consumer_document_type.json
View file

@ -7,7 +7,8 @@
"field_order": [
"ref_doctype",
"status",
"unsubscribed"
"unsubscribed",
"condition"
],
"fields": [
{
@ -37,11 +38,17 @@
"in_list_view": 1,
"label": "Unsubscribed",
"read_only": 1
},
{
"fieldname": "condition",
"fieldtype": "Code",
"label": "Condition",
"read_only": 1
}
],
"istable": 1,
"links": [],
"modified": "2020-08-14 12:38:40.918620",
"modified": "2020-11-07 09:26:49.894294",
"modified_by": "Administrator",
"module": "Event Streaming",
"name": "Event Consumer Document Type",

25
frappe/event_streaming/doctype/event_producer/event_producer.py
View file

@ -102,9 +102,13 @@ class EventProducer(Document):
for entry in self.producer_doctypes:
if entry.has_mapping:
# if mapping, subscribe to remote doctype on consumer's site
consumer_doctypes.append(frappe.db.get_value('Document Type Mapping', entry.mapping, 'remote_doctype'))
dt = frappe.db.get_value('Document Type Mapping', entry.mapping, 'remote_doctype')
else:
consumer_doctypes.append(entry.ref_doctype)
dt = entry.ref_doctype
consumer_doctypes.append({
"doctype": dt,
"condition": entry.condition
})
user_key = frappe.db.get_value('User', self.user, 'api_key')
user_secret = get_decrypted_password('User', self.user, 'api_secret')
@ -145,7 +149,8 @@ class EventProducer(Document):
event_consumer.consumer_doctypes.append({
'ref_doctype': ref_doctype,
'status': get_approval_status(config, ref_doctype),
'unsubscribed': entry.unsubscribe
'unsubscribed': entry.unsubscribe,
'condition': entry.condition
})
event_consumer.user = self.user
event_consumer.incoming_change = True
@ -347,13 +352,13 @@ def set_delete(update):
def get_updates(producer_site, last_update, doctypes):
"""Get all updates generated after the last update timestamp"""
docs = producer_site.get_list(
doctype='Event Update Log',
filters={'ref_doctype': ('in', doctypes), 'creation': ('>', last_update)},
fields=['update_type', 'ref_doctype', 'docname', 'data', 'name', 'creation']
)
docs.reverse()
return [frappe._dict(d) for d in docs]
docs = producer_site.post_request({
'cmd': 'frappe.event_streaming.doctype.event_update_log.event_update_log.get_update_logs_for_consumer',
'event_consumer': get_url(),
'doctypes': frappe.as_json(doctypes),
'last_update': last_update
})
return [frappe._dict(d) for d in (docs or [])]
def get_local_doc(update):

79
frappe/event_streaming/doctype/event_producer/test_event_producer.py
View file

@ -152,6 +152,82 @@ class TestEventProducer(unittest.TestCase):
reset_configuration(producer_url)
def test_conditional_events(self):
producer = get_remote_site()
# Add Condition
event_producer = frappe.get_doc('Event Producer', producer_url)
note_producer_entry = [
x for x in event_producer.producer_doctypes if x.ref_doctype == 'Note'
][0]
note_producer_entry.condition = 'doc.public == 1'
event_producer.save()
# Make test doc
producer_note1 = frappe._dict(doctype='Note', public=0, title='test conditional sync')
delete_on_remote_if_exists(producer, 'Note', {'title': producer_note1['title']})
producer_note1 = producer.insert(producer_note1)
# Make Update
producer_note1['content'] = 'Test Conditional Sync Content'
producer_note1 = producer.update(producer_note1)
self.pull_producer_data()
# Check if synced here
self.assertFalse(frappe.db.exists('Note', producer_note1.name))
# Lets satisfy the condition
producer_note1['public'] = 1
producer_note1 = producer.update(producer_note1)
self.pull_producer_data()
# it should sync now
self.assertTrue(frappe.db.exists('Note', producer_note1.name))
local_note = frappe.get_doc('Note', producer_note1.name)
self.assertEqual(local_note.content, producer_note1.content)
reset_configuration(producer_url)
def test_conditional_events_with_cmd(self):
producer = get_remote_site()
# Add Condition
event_producer = frappe.get_doc('Event Producer', producer_url)
note_producer_entry = [
x for x in event_producer.producer_doctypes if x.ref_doctype == 'Note'
][0]
note_producer_entry.condition = 'cmd: frappe.event_streaming.doctype.event_producer.test_event_producer.can_sync_note'
event_producer.save()
# Make test doc
producer_note1 = frappe._dict(doctype='Note', public=0, title='test conditional sync cmd')
delete_on_remote_if_exists(producer, 'Note', {'title': producer_note1['title']})
producer_note1 = producer.insert(producer_note1)
# Make Update
producer_note1['content'] = 'Test Conditional Sync Content'
producer_note1 = producer.update(producer_note1)
self.pull_producer_data()
# Check if synced here
self.assertFalse(frappe.db.exists('Note', producer_note1.name))
# Lets satisfy the condition
producer_note1['public'] = 1
producer_note1 = producer.update(producer_note1)
self.pull_producer_data()
# it should sync now
self.assertTrue(frappe.db.exists('Note', producer_note1.name))
local_note = frappe.get_doc('Note', producer_note1.name)
self.assertEqual(local_note.content, producer_note1.content)
reset_configuration(producer_url)
def test_update_log(self):
producer = get_remote_site()
producer_doc = insert_into_producer(producer, 'test update log')
@ -221,6 +297,8 @@ class TestEventProducer(unittest.TestCase):
reset_configuration(producer_url)
def can_sync_note(consumer, doc, update_log):
return doc.public == 1
def setup_event_producer_for_inner_mapping():
event_producer = frappe.get_doc('Event Producer', producer_url, for_update=True)
@ -322,6 +400,7 @@ def create_event_producer(producer_url):
def reset_configuration(producer_url):
event_producer = frappe.get_doc('Event Producer', producer_url, for_update=True)
event_producer.producer_doctypes = []
event_producer.conditions = []
event_producer.producer_url = producer_url
event_producer.append('producer_doctypes', {
'ref_doctype': 'ToDo',

10
frappe/event_streaming/doctype/event_producer_document_type/event_producer_document_type.json
View file

@ -10,7 +10,8 @@
"use_same_name",
"unsubscribe",
"has_mapping",
"mapping"
"mapping",
"condition"
],
"fields": [
{
@ -63,11 +64,16 @@
"fieldtype": "Check",
"in_list_view": 1,
"label": "Unsubscribe"
},
{
"fieldname": "condition",
"fieldtype": "Code",
"label": "Condition"
}
],
"istable": 1,
"links": [],
"modified": "2020-08-14 11:38:01.278996",
"modified": "2020-11-07 09:26:58.463868",
"modified_by": "Administrator",
"module": "Event Streaming",
"name": "Event Producer Document Type",

15
frappe/event_streaming/doctype/event_update_log/event_update_log.json
View file

@ -1,4 +1,5 @@
{
"actions": [],
"creation": "2019-07-30 15:31:26.352527",
"doctype": "DocType",
"editable_grid": 1,
@ -7,7 +8,8 @@
"update_type",
"ref_doctype",
"docname",
"data"
"data",
"consumers"
],
"fields": [
{
@ -31,7 +33,6 @@
"fieldtype": "Data",
"in_list_view": 1,
"label": "Document Name",
"options": "ref_doctype",
"read_only": 1
},
{
@ -39,10 +40,18 @@
"fieldtype": "Code",
"label": "Data",
"read_only": 1
},
{
"fieldname": "consumers",
"fieldtype": "Table MultiSelect",
"label": "Consumers",
"options": "Event Update Log Consumer",
"read_only": 1
}
],
"in_create": 1,
"modified": "2019-09-24 23:16:07.207707",
"links": [],
"modified": "2020-09-04 07:31:52.599804",
"modified_by": "Administrator",
"module": "Event Streaming",
"name": "Event Update Log",

134
frappe/event_streaming/doctype/event_update_log/event_update_log.py
View file

@ -140,3 +140,137 @@ def check_docstatus(out, old, new, for_child):
if not for_child and old.docstatus != new.docstatus:
out.changed['docstatus'] = new.docstatus
return out
def is_consumer_uptodate(update_log, consumer):
"""
Checks if Consumer has read all the UpdateLogs before the specified update_log
:param update_log: The UpdateLog Doc in context
:param consumer: The EventConsumer doc
"""
if update_log.update_type == 'Create':
# consumer is obviously up to date
return True
prev_logs = frappe.get_all(
'Event Update Log',
filters={
'ref_doctype': update_log.ref_doctype,
'docname': update_log.docname,
'creation': ['<', update_log.creation]
},
order_by='creation desc',
limit_page_length=1
)
if not len(prev_logs):
return False
prev_log_consumers = frappe.get_all(
'Event Update Log Consumer',
fields=['consumer'],
filters={
'parent': prev_logs[0].name,
'parenttype': 'Event Update Log',
'consumer': consumer.name
}
)
return len(prev_log_consumers) > 0
def mark_consumer_read(update_log_name, consumer_name):
"""
This function appends the Consumer to the list of Consumers that has 'read' an Update Log
"""
update_log = frappe.get_doc('Event Update Log', update_log_name)
if len([x for x in update_log.consumers if x.consumer == consumer_name]):
return
frappe.get_doc(frappe._dict(
doctype='Event Update Log Consumer',
consumer=consumer_name,
parent=update_log_name,
parenttype='Event Update Log',
parentfield='consumers'
)).insert(ignore_permissions=True)
def get_unread_update_logs(consumer_name, dt, dn):
"""
Get old logs unread by the consumer on a particular document
"""
already_consumed = [x[0] for x in frappe.db.sql("""
SELECT
update_log.name
FROM `tabEvent Update Log` update_log
JOIN `tabEvent Update Log Consumer` consumer ON consumer.parent = update_log.name
WHERE
consumer.consumer = %(consumer)s
AND update_log.ref_doctype = %(dt)s
AND update_log.docname = %(dn)s
""", {'consumer': consumer_name, "dt": dt, "dn": dn}, as_dict=0)]
logs = frappe.get_all(
'Event Update Log',
fields=['update_type', 'ref_doctype',
'docname', 'data', 'name', 'creation'],
filters={
'ref_doctype': dt,
'docname': dn,
'name': ['not in', already_consumed]
},
order_by='creation'
)
return logs
@frappe.whitelist()
def get_update_logs_for_consumer(event_consumer, doctypes, last_update):
"""
Fetches all the UpdateLogs for the consumer
It will inject old un-consumed Update Logs if a doc was just found to be accessible to the Consumer
"""
if isinstance(doctypes, str):
doctypes = frappe.parse_json(doctypes)
from frappe.event_streaming.doctype.event_consumer.event_consumer import has_consumer_access
consumer = frappe.get_doc('Event Consumer', event_consumer)
docs = frappe.get_list(
doctype='Event Update Log',
filters={'ref_doctype': ('in', doctypes),
'creation': ('>', last_update)},
fields=['update_type', 'ref_doctype',
'docname', 'data', 'name', 'creation'],
order_by='creation desc'
)
result = []
to_update_history = []
for d in docs:
if (d.ref_doctype, d.docname) in to_update_history:
# will be notified by background jobs
continue
if not has_consumer_access(consumer=consumer, update_log=d):
continue
if not is_consumer_uptodate(d, consumer):
to_update_history.append((d.ref_doctype, d.docname))
# get_unread_update_logs will have the current log
old_logs = get_unread_update_logs(consumer.name, d.ref_doctype, d.docname)
if old_logs:
old_logs.reverse()
result.extend(old_logs)
else:
result.append(d)
for d in result:
mark_consumer_read(update_log_name=d.name, consumer_name=consumer.name)
result.reverse()
return result

0
frappe/integrations/doctype/twilio_number_group/__init__.py → frappe/event_streaming/doctype/event_update_log_consumer/__init__.py
View file

32
frappe/event_streaming/doctype/event_update_log_consumer/event_update_log_consumer.json Normal file
View file

@ -0,0 +1,32 @@
{
"actions": [],
"creation": "2020-06-30 10:54:53.301787",
"doctype": "DocType",
"editable_grid": 1,
"engine": "InnoDB",
"field_order": [
"consumer"
],
"fields": [
{
"fieldname": "consumer",
"fieldtype": "Link",
"in_list_view": 1,
"label": "Consumer",
"options": "Event Consumer",
"reqd": 1
}
],
"istable": 1,
"links": [],
"modified": "2020-06-30 10:54:53.301787",
"modified_by": "Administrator",
"module": "Event Streaming",
"name": "Event Update Log Consumer",
"owner": "Administrator",
"permissions": [],
"quick_entry": 1,
"sort_field": "modified",
"sort_order": "DESC",
"track_changes": 1
}

2
frappe/integrations/doctype/twilio_number_group/twilio_number_group.py → frappe/event_streaming/doctype/event_update_log_consumer/event_update_log_consumer.py
View file

@ -6,5 +6,5 @@ from __future__ import unicode_literals
# import frappe
from frappe.model.document import Document
class TwilioNumberGroup(Document):
class EventUpdateLogConsumer(Document):
pass

3
frappe/exceptions.py
View file

@ -106,7 +106,10 @@ class InvalidDates(ValidationError): pass
class DataTooLongException(ValidationError): pass
class FileAlreadyAttachedException(Exception): pass
class DocumentAlreadyRestored(Exception): pass
class AttachmentLimitReached(Exception): pass
# OAuth exceptions
class InvalidAuthorizationHeader(CSRFTokenError): pass
class InvalidAuthorizationPrefix(CSRFTokenError): pass
class InvalidAuthorizationToken(CSRFTokenError): pass
class InvalidDatabaseFile(ValidationError): pass
class ExecutableNotFound(FileNotFoundError): pass

254
frappe/installer.py
View file

@ -3,8 +3,90 @@
import json
import os
from frappe.defaults import _clear_cache
import sys
import frappe
from frappe.defaults import _clear_cache
def _new_site(
db_name,
site,
mariadb_root_username=None,
mariadb_root_password=None,
admin_password=None,
verbose=False,
install_apps=None,
source_sql=None,
force=False,
no_mariadb_socket=False,
reinstall=False,
db_password=None,
db_type=None,
db_host=None,
db_port=None,
new_site=False,
):
"""Install a new Frappe site"""
if not force and os.path.exists(site):
print("Site {0} already exists".format(site))
sys.exit(1)
if no_mariadb_socket and not db_type == "mariadb":
print("--no-mariadb-socket requires db_type to be set to mariadb.")
sys.exit(1)
if not db_name:
import hashlib
db_name = "_" + hashlib.sha1(site.encode()).hexdigest()[:16]
frappe.init(site=site)
from frappe.commands.scheduler import _is_scheduler_enabled
from frappe.utils import get_site_path, scheduler, touch_file
try:
# enable scheduler post install?
enable_scheduler = _is_scheduler_enabled()
except Exception:
enable_scheduler = False
make_site_dirs()
installing = touch_file(get_site_path("locks", "installing.lock"))
install_db(
root_login=mariadb_root_username,
root_password=mariadb_root_password,
db_name=db_name,
admin_password=admin_password,
verbose=verbose,
source_sql=source_sql,
force=force,
reinstall=reinstall,
db_password=db_password,
db_type=db_type,
db_host=db_host,
db_port=db_port,
no_mariadb_socket=no_mariadb_socket,
)
apps_to_install = (
["frappe"] + (frappe.conf.get("install_apps") or []) + (list(install_apps) or [])
)
for app in apps_to_install:
install_app(app, verbose=verbose, set_as_patched=not source_sql)
os.remove(installing)
scheduler.toggle_scheduler(enable_scheduler)
frappe.db.commit()
scheduler_status = (
"disabled" if frappe.utils.scheduler.is_scheduler_disabled() else "enabled"
)
print("*** Scheduler is", scheduler_status, "***")
def install_db(root_login="root", root_password=None, db_name=None, source_sql=None,
@ -36,9 +118,9 @@ def install_db(root_login="root", root_password=None, db_name=None, source_sql=N
def install_app(name, verbose=False, set_as_patched=True):
from frappe.core.doctype.scheduled_job_type.scheduled_job_type import sync_jobs
from frappe.utils.fixtures import sync_fixtures
from frappe.model.sync import sync_for
from frappe.modules.utils import sync_customizations
from frappe.utils.fixtures import sync_fixtures
frappe.flags.in_install = name
frappe.flags.ignore_in_install = False
@ -122,64 +204,80 @@ def remove_app(app_name, dry_run=False, yes=False, no_backup=False, force=False)
"""Remove app and all linked to the app's module with the app from a site."""
import click
site = frappe.local.site
# dont allow uninstall app if not installed unless forced
if not force:
if app_name not in frappe.get_installed_apps():
click.secho("App {0} not installed on Site {1}".format(app_name, frappe.local.site), fg="yellow")
click.secho(f"App {app_name} not installed on Site {site}", fg="yellow")
return
print("Uninstalling App {0} from Site {1}...".format(app_name, frappe.local.site))
print(f"Uninstalling App {app_name} from Site {site}...")
if not dry_run and not yes:
confirm = click.confirm("All doctypes (including custom), modules related to this app will be deleted. Are you sure you want to continue?")
confirm = click.confirm(
"All doctypes (including custom), modules related to this app will be"
" deleted. Are you sure you want to continue?"
)
if not confirm:
return
if not no_backup:
if not (dry_run or no_backup):
from frappe.utils.backups import scheduled_backup
print("Backing up...")
scheduled_backup(ignore_files=True)
frappe.flags.in_uninstall = True
drop_doctypes = []
modules = (x.name for x in frappe.get_all("Module Def", filters={"app_name": app_name}))
modules = frappe.get_all("Module Def", filters={"app_name": app_name}, pluck="name")
for module_name in modules:
print("Deleting Module '{0}'".format(module_name))
print(f"Deleting Module '{module_name}'")
for doctype in frappe.get_list("DocType", filters={"module": module_name}, fields=["name", "issingle"]):
print("* removing DocType '{0}'...".format(doctype.name))
for doctype in frappe.get_all(
"DocType", filters={"module": module_name}, fields=["name", "issingle"]
):
print(f"* removing DocType '{doctype.name}'...")
if not dry_run:
frappe.delete_doc("DocType", doctype.name)
frappe.delete_doc("DocType", doctype.name, ignore_on_trash=True)
if not doctype.issingle:
drop_doctypes.append(doctype.name)
linked_doctypes = frappe.get_all("DocField", filters={"fieldtype": "Link", "options": "Module Def"}, fields=['parent'])
linked_doctypes = frappe.get_all(
"DocField", filters={"fieldtype": "Link", "options": "Module Def"}, fields=["parent"]
)
ordered_doctypes = ["Desk Page", "Report", "Page", "Web Form"]
doctypes_with_linked_modules = ordered_doctypes + [doctype.parent for doctype in linked_doctypes if doctype.parent not in ordered_doctypes]
all_doctypes_with_linked_modules = ordered_doctypes + [
doctype.parent
for doctype in linked_doctypes
if doctype.parent not in ordered_doctypes
]
doctypes_with_linked_modules = [
x for x in all_doctypes_with_linked_modules if frappe.db.exists("DocType", x)
]
for doctype in doctypes_with_linked_modules:
for record in frappe.get_list(doctype, filters={"module": module_name}):
print("* removing {0} '{1}'...".format(doctype, record.name))
for record in frappe.get_all(doctype, filters={"module": module_name}, pluck="name"):
print(f"* removing {doctype} '{record}'...")
if not dry_run:
frappe.delete_doc(doctype, record.name)
frappe.delete_doc(doctype, record, ignore_on_trash=True)
print("* removing Module Def '{0}'...".format(module_name))
print(f"* removing Module Def '{module_name}'...")
if not dry_run:
frappe.delete_doc("Module Def", module_name)
frappe.delete_doc("Module Def", module_name, ignore_on_trash=True)
for doctype in set(drop_doctypes):
print(f"* dropping Table for '{doctype}'...")
if not dry_run:
frappe.db.sql_ddl(f"drop table `tab{doctype}`")
if not dry_run:
remove_from_installed_apps(app_name)
for doctype in set(drop_doctypes):
print("* dropping Table for '{0}'...".format(doctype))
frappe.db.sql_ddl("drop table `tab{0}`".format(doctype))
frappe.db.commit()
click.secho("Uninstalled App {0} from Site {1}".format(app_name, frappe.local.site), fg="green")
click.secho(f"Uninstalled App {app_name} from Site {site}", fg="green")
frappe.flags.in_uninstall = False
@ -331,6 +429,37 @@ def remove_missing_apps():
frappe.db.set_global("installed_apps", json.dumps(installed_apps))
def extract_sql_from_archive(sql_file_path):
"""Return the path of an SQL file if the passed argument is the path of a gzipped
SQL file or an SQL file path. The path may be absolute or relative from the bench
root directory or the sites sub-directory.
Args:
sql_file_path (str): Path of the SQL file
Returns:
str: Path of the decompressed SQL file
"""
# Extract the gzip file if user has passed *.sql.gz file instead of *.sql file
if not os.path.exists(sql_file_path):
base_path = '..'
sql_file_path = os.path.join(base_path, sql_file_path)
if not os.path.exists(sql_file_path):
print('Invalid path {0}'.format(sql_file_path[3:]))
sys.exit(1)
elif sql_file_path.startswith(os.sep):
base_path = os.sep
else:
base_path = '.'
if sql_file_path.endswith('sql.gz'):
decompressed_file_name = extract_sql_gzip(os.path.abspath(sql_file_path))
else:
decompressed_file_name = sql_file_path
return decompressed_file_name
def extract_sql_gzip(sql_gz_path):
import subprocess
@ -345,9 +474,10 @@ def extract_sql_gzip(sql_gz_path):
return decompressed_file
def extract_files(site_name, file_path, folder_name):
import subprocess
import shutil
import subprocess
# Need to do frappe.init to maintain the site locals
frappe.init(site=site_name)
@ -375,6 +505,12 @@ def extract_files(site_name, file_path, folder_name):
def is_downgrade(sql_file_path, verbose=False):
"""checks if input db backup will get downgraded on current bench"""
# This function is only tested with mariadb
# TODO: Add postgres support
if frappe.conf.db_type not in (None, "mariadb"):
return False
from semantic_version import Version
head = "INSERT INTO `tabInstalled Application` VALUES"
@ -406,3 +542,69 @@ def is_downgrade(sql_file_path, verbose=False):
print("Your site will be downgraded from Frappe {0} to {1}".format(current_version, backup_version))
return downgrade
def is_partial(sql_file_path):
with open(sql_file_path) as f:
header = " ".join([f.readline() for _ in range(5)])
if "Partial Backup" in header:
return True
return False
def partial_restore(sql_file_path, verbose=False):
sql_file = extract_sql_from_archive(sql_file_path)
if frappe.conf.db_type in (None, "mariadb"):
from frappe.database.mariadb.setup_db import import_db_from_sql
elif frappe.conf.db_type == "postgres":
from frappe.database.postgres.setup_db import import_db_from_sql
import warnings
from click import style
warn = style(
"Delete the tables you want to restore manually before attempting"
" partial restore operation for PostreSQL databases",
fg="yellow"
)
warnings.warn(warn)
import_db_from_sql(source_sql=sql_file, verbose=verbose)
# Removing temporarily created file
if sql_file != sql_file_path:
os.remove(sql_file)
def validate_database_sql(path, _raise=True):
"""Check if file has contents and if DefaultValue table exists
Args:
path (str): Path of the decompressed SQL file
_raise (bool, optional): Raise exception if invalid file. Defaults to True.
"""
empty_file = False
missing_table = True
error_message = ""
if not os.path.getsize(path):
error_message = f"{path} is an empty file!"
empty_file = True
# dont bother checking if empty file
if not empty_file:
with open(path, "r") as f:
for line in f:
if 'tabDefaultValue' in line:
missing_table = False
break
if missing_table:
error_message = "Table `tabDefaultValue` not found in file."
if error_message:
import click
click.secho(error_message, fg="red")
if _raise and (missing_table or empty_file):
raise frappe.InvalidDatabaseFile

4
frappe/integrations/desk_page/integrations/integrations.json
View file

@ -23,7 +23,7 @@
{
"hidden": 0,
"label": "Settings",
"links": "[\n {\n \"description\": \"Webhooks calling API requests into web apps\",\n \"label\": \"Webhook\",\n \"name\": \"Webhook\",\n \"type\": \"doctype\"\n },\n {\n \"description\": \"Slack Webhooks for internal integration\",\n \"label\": \"Slack Webhook URL\",\n \"name\": \"Slack Webhook URL\",\n \"type\": \"doctype\"\n },\n {\n \"description\": \"Twilio Settings for WhatsApp integration\",\n \"label\": \"Twilio Settings\",\n \"name\": \"Twilio Settings\",\n \"type\": \"doctype\"\n },\n {\n \"description\": \"SMS Settings for sending sms\",\n \"label\": \"SMS Settings\",\n \"name\": \"SMS Settings\",\n \"type\": \"doctype\"\n }\n]"
"links": "[\n {\n \"description\": \"Webhooks calling API requests into web apps\",\n \"label\": \"Webhook\",\n \"name\": \"Webhook\",\n \"type\": \"doctype\"\n },\n {\n \"description\": \"Slack Webhooks for internal integration\",\n \"label\": \"Slack Webhook URL\",\n \"name\": \"Slack Webhook URL\",\n \"type\": \"doctype\"\n },\n {\n \"description\": \"SMS Settings for sending sms\",\n \"label\": \"SMS Settings\",\n \"name\": \"SMS Settings\",\n \"type\": \"doctype\"\n }\n]"
}
],
"category": "Administration",
@ -38,7 +38,7 @@
"idx": 0,
"is_standard": 1,
"label": "Integrations",
"modified": "2020-08-20 23:04:04.528572",
"modified": "2020-10-28 10:25:54.792363",
"modified_by": "Administrator",
"module": "Integrations",
"name": "Integrations",

5
frappe/integrations/doctype/dropbox_settings/dropbox_settings.py
View file

@ -9,7 +9,7 @@ import frappe
import os
from frappe import _
from frappe.model.document import Document
from frappe.integrations.offsite_backup_utils import get_latest_backup_file, send_email, validate_file_size
from frappe.integrations.offsite_backup_utils import get_latest_backup_file, send_email, validate_file_size, get_chunk_site
from frappe.integrations.utils import make_post_request
from frappe.utils import (cint, get_request_site_address,
get_files_path, get_backups_path, get_url, encode)
@ -167,8 +167,9 @@ def upload_file_to_dropbox(filename, folder, dropbox_client):
return
create_folder_if_not_exists(folder, dropbox_client)
chunk_size = 15 * 1024 * 1024
file_size = os.path.getsize(encode(filename))
chunk_size = get_chunk_site(file_size)
mode = (dropbox.files.WriteMode.overwrite)
f = open(encode(filename), 'rb')

36
frappe/integrations/doctype/twilio_number_group/twilio_number_group.json
View file

@ -1,36 +0,0 @@
{
"actions": [],
"autoname": "field:phone_number",
"creation": "2020-02-24 13:58:58.036914",
"doctype": "DocType",
"editable_grid": 1,
"engine": "InnoDB",
"field_order": [
"phone_number"
],
"fields": [
{
"fieldname": "phone_number",
"fieldtype": "Data",
"in_list_view": 1,
"label": "Phone Number",
"options": "Phone",
"show_days": 1,
"show_seconds": 1,
"unique": 1
}
],
"index_web_pages_for_search": 1,
"istable": 1,
"links": [],
"modified": "2020-08-20 22:48:57.166791",
"modified_by": "Administrator",
"module": "Integrations",
"name": "Twilio Number Group",
"owner": "Administrator",
"permissions": [],
"quick_entry": 1,
"sort_field": "modified",
"sort_order": "DESC",
"track_changes": 1
}

0
frappe/integrations/doctype/twilio_settings/__init__.py
View file

10
frappe/integrations/doctype/twilio_settings/test_twilio_settings.py
View file

@ -1,10 +0,0 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2020, Frappe Technologies and Contributors
# See license.txt
from __future__ import unicode_literals
# import frappe
import unittest
class TestTwilioSettings(unittest.TestCase):
pass

8
frappe/integrations/doctype/twilio_settings/twilio_settings.js
View file

@ -1,8 +0,0 @@
// Copyright (c) 2020, Frappe Technologies and contributors
// For license information, please see license.txt
frappe.ui.form.on('Twilio Settings', {
refresh: function(frm) {
frm.dashboard.set_headline(__("For more information, {0}.", [`<a href='https://docs.erpnext.com/docs/user/manual/en/setting-up/notifications'>${__('Click here')}</a>`]));
}
});

67
frappe/integrations/doctype/twilio_settings/twilio_settings.json
View file

@ -1,67 +0,0 @@
{
"actions": [],
"creation": "2020-01-28 15:21:44.457163",
"doctype": "DocType",
"editable_grid": 1,
"engine": "InnoDB",
"field_order": [
"enabled",
"account_sid",
"auth_token",
"column_break_2",
"twilio_number"
],
"fields": [
{
"fieldname": "account_sid",
"fieldtype": "Data",
"label": "Account SID",
"mandatory_depends_on": "eval: doc.enabled"
},
{
"fieldname": "auth_token",
"fieldtype": "Password",
"label": "Auth Token",
"mandatory_depends_on": "eval: doc.enabled"
},
{
"fieldname": "column_break_2",
"fieldtype": "Column Break"
},
{
"fieldname": "twilio_number",
"fieldtype": "Table",
"label": "Twilio Number",
"options": "Twilio Number Group"
},
{
"default": "0",
"fieldname": "enabled",
"fieldtype": "Check",
"label": "Enabled"
}
],
"index_web_pages_for_search": 1,
"issingle": 1,
"links": [],
"modified": "2020-09-03 10:17:21.318743",
"modified_by": "Administrator",
"module": "Integrations",
"name": "Twilio Settings",
"owner": "Administrator",
"permissions": [
{
"create": 1,
"delete": 1,
"email": 1,
"print": 1,
"read": 1,
"role": "System Manager",
"share": 1,
"write": 1
}
],
"sort_field": "modified",
"sort_order": "DESC",
"track_changes": 1
}

63
frappe/integrations/doctype/twilio_settings/twilio_settings.py
View file

@ -1,63 +0,0 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2020, Frappe Technologies and contributors
# For license information, please see license.txt
from __future__ import unicode_literals
import frappe
from frappe.model.document import Document
from frappe import _
from frappe.utils.password import get_decrypted_password
from twilio.rest import Client
from six import string_types
from json import loads
class TwilioSettings(Document):
def on_update(self):
if self.enabled:
self.validate_twilio_credentials()
def validate_twilio_credentials(self):
try:
auth_token = get_decrypted_password("Twilio Settings", "Twilio Settings", 'auth_token')
client = Client(self.account_sid, auth_token)
client.api.accounts(self.account_sid).fetch()
except Exception:
frappe.throw(_("Invalid Account SID or Auth Token."))
def send_whatsapp_message(sender, receiver_list, message):
twilio_settings = frappe.get_doc("Twilio Settings")
if not twilio_settings.enabled:
frappe.throw(_("Please enable twilio settings before sending WhatsApp messages"))
if isinstance(receiver_list, string_types):
receiver_list = loads(receiver_list)
if not isinstance(receiver_list, list):
receiver_list = [receiver_list]
auth_token = get_decrypted_password("Twilio Settings", "Twilio Settings", 'auth_token')
client = Client(twilio_settings.account_sid, auth_token)
args = {
"from_": 'whatsapp:+{}'.format(sender),
"body": message
}
failed_delivery = []
for rec in receiver_list:
args.update({"to": 'whatsapp:{}'.format(rec)})
resp = _send_whatsapp(args, client)
if not resp or resp.error_message:
failed_delivery.append(rec)
if failed_delivery:
frappe.log_error(_("The message wasn't correctly delivered to: {}".format(", ".join(failed_delivery))), _('Delivery Failed'))
def _send_whatsapp(message_dict, client):
response = frappe._dict()
try:
response = client.messages.create(**message_dict)
except Exception as e:
frappe.log_error(e, title = _('Twilio WhatsApp Message Error'))
return response

2
frappe/integrations/frappe_providers/frappecloud.py
View file

@ -6,7 +6,7 @@ import frappe
def frappecloud_migrator(local_site):
print("Retreiving Site Migrator...")
print("Retrieving Site Migrator...")
remote_site = frappe.conf.frappecloud_url or "frappecloud.com"
request_url = "https://{}/api/method/press.api.script".format(remote_site)
request = requests.get(request_url)

132
frappe/integrations/oauth2.py
View file

@ -1,41 +1,50 @@
from __future__ import unicode_literals
import frappe, json
from frappe.oauth import OAuthWebRequestValidator, WebApplicationServer
import hashlib
import json
from urllib.parse import quote, urlencode, urlparse
import jwt
from oauthlib.oauth2 import FatalClientError, OAuth2Error
from werkzeug import url_fix
from six.moves.urllib.parse import quote, urlencode, urlparse
from frappe.integrations.doctype.oauth_provider_settings.oauth_provider_settings import get_oauth_settings
import frappe
from frappe import _
from frappe.oauth import OAuthWebRequestValidator, WebApplicationServer
from frappe.integrations.doctype.oauth_provider_settings.oauth_provider_settings import get_oauth_settings
def get_oauth_server():
if not getattr(frappe.local, 'oauth_server', None):
oauth_validator = OAuthWebRequestValidator()
frappe.local.oauth_server = WebApplicationServer(oauth_validator)
frappe.local.oauth_server = WebApplicationServer(oauth_validator)
return frappe.local.oauth_server
def get_urlparams_from_kwargs(param_kwargs):
def sanitize_kwargs(param_kwargs):
arguments = param_kwargs
if arguments.get("data"):
arguments.pop("data")
if arguments.get("cmd"):
arguments.pop("cmd")
arguments.pop('data', None)
arguments.pop('cmd', None)
return urlencode(arguments)
return arguments
@frappe.whitelist()
def approve(*args, **kwargs):
r = frappe.request
uri = url_fix(r.url.replace("+"," "))
http_method = r.method
body = r.get_data()
headers = r.headers
try:
scopes, frappe.flags.oauth_credentials = get_oauth_server().validate_authorization_request(uri, http_method, body, headers)
scopes, frappe.flags.oauth_credentials = get_oauth_server().validate_authorization_request(
r.url,
r.method,
r.get_data(),
r.headers
)
headers, body, status = get_oauth_server().create_authorization_response(uri=frappe.flags.oauth_credentials['redirect_uri'], \
body=body, headers=headers, scopes=scopes, credentials=frappe.flags.oauth_credentials)
headers, body, status = get_oauth_server().create_authorization_response(
uri=frappe.flags.oauth_credentials['redirect_uri'],
body=r.get_data(),
headers=r.headers,
scopes=scopes,
credentials=frappe.flags.oauth_credentials
)
uri = headers.get('Location', None)
frappe.local.response["type"] = "redirect"
@ -47,34 +56,28 @@ def approve(*args, **kwargs):
return e
@frappe.whitelist(allow_guest=True)
def authorize(*args, **kwargs):
#Fetch provider URL from settings
oauth_settings = get_oauth_settings()
params = get_urlparams_from_kwargs(kwargs)
request_url = urlparse(frappe.request.url)
success_url = request_url.scheme + "://" + request_url.netloc + "/api/method/frappe.integrations.oauth2.approve?" + params
def authorize(**kwargs):
success_url = "/api/method/frappe.integrations.oauth2.approve?" + encode_params(sanitize_kwargs(kwargs))
failure_url = frappe.form_dict["redirect_uri"] + "?error=access_denied"
if frappe.session['user']=='Guest':
if frappe.session.user == 'Guest':
#Force login, redirect to preauth again.
frappe.local.response["type"] = "redirect"
frappe.local.response["location"] = "/login?redirect-to=/api/method/frappe.integrations.oauth2.authorize?" + quote(params.replace("+"," "))
elif frappe.session['user']!='Guest':
frappe.local.response["location"] = "/login?" + encode_params({'redirect-to': frappe.request.url})
else:
try:
r = frappe.request
uri = url_fix(r.url)
http_method = r.method
body = r.get_data()
headers = r.headers
scopes, frappe.flags.oauth_credentials = get_oauth_server().validate_authorization_request(uri, http_method, body, headers)
scopes, frappe.flags.oauth_credentials = get_oauth_server().validate_authorization_request(
r.url,
r.method,
r.get_data(),
r.headers
)
skip_auth = frappe.db.get_value("OAuth Client", frappe.flags.oauth_credentials['client_id'], "skip_authorization")
unrevoked_tokens = frappe.get_all("OAuth Bearer Token", filters={"status":"Active"})
if skip_auth or (oauth_settings["skip_authorization"] == "Auto" and len(unrevoked_tokens)):
if skip_auth or (get_oauth_settings().skip_authorization == "Auto" and unrevoked_tokens):
frappe.local.response["type"] = "redirect"
frappe.local.response["location"] = success_url
else:
@ -87,7 +90,6 @@ def authorize(*args, **kwargs):
})
resp_html = frappe.render_template("templates/includes/oauth_confirmation.html", response_html_params)
frappe.respond_as_web_page("Confirm Access", resp_html)
except FatalClientError as e:
return e
except OAuth2Error as e:
@ -95,20 +97,20 @@ def authorize(*args, **kwargs):
@frappe.whitelist(allow_guest=True)
def get_token(*args, **kwargs):
r = frappe.request
uri = url_fix(r.url)
http_method = r.method
body = r.form
headers = r.headers
#Check whether frappe server URL is set
frappe_server_url = frappe.db.get_value("Social Login Key", "frappe", "base_url") or None
if not frappe_server_url:
frappe.throw(_("Please set Base URL in Social Login Key for Frappe"))
try:
headers, body, status = get_oauth_server().create_token_response(uri, http_method, body, headers, frappe.flags.oauth_credentials)
r = frappe.request
headers, body, status = get_oauth_server().create_token_response(
r.url,
r.method,
r.form,
r.headers,
frappe.flags.oauth_credentials
)
out = frappe._dict(json.loads(body))
if not out.error and "openid" in out.scope:
token_user = frappe.db.get_value("OAuth Bearer Token", out.access_token, "user")
@ -116,7 +118,7 @@ def get_token(*args, **kwargs):
client_secret = frappe.db.get_value("OAuth Client", token_client, "client_secret")
if token_user in ["Guest", "Administrator"]:
frappe.throw(_("Logged in as Guest or Administrator"))
import hashlib
id_token_header = {
"typ":"jwt",
"alg":"HS256"
@ -128,9 +130,10 @@ def get_token(*args, **kwargs):
"iss": frappe_server_url,
"at_hash": frappe.oauth.calculate_at_hash(out.access_token, hashlib.sha256)
}
import jwt
id_token_encoded = jwt.encode(id_token, client_secret, algorithm='HS256', headers=id_token_header)
out.update({"id_token":str(id_token_encoded)})
out.update({"id_token": str(id_token_encoded)})
frappe.local.response = out
except FatalClientError as e:
@ -140,12 +143,12 @@ def get_token(*args, **kwargs):
@frappe.whitelist(allow_guest=True)
def revoke_token(*args, **kwargs):
r = frappe.request
uri = url_fix(r.url)
http_method = r.method
body = r.form
headers = r.headers
headers, body, status = get_oauth_server().create_revocation_response(uri, headers=headers, body=body, http_method=http_method)
headers, body, status = get_oauth_server().create_revocation_response(
r.url,
headers=r.headers,
body=r.form,
http_method=r.method
)
frappe.local.response['http_status_code'] = status
if status == 200:
@ -174,15 +177,22 @@ def openid_profile(*args, **kwargs):
"email": name,
"picture": picture
})
frappe.local.response = user_profile
def validate_url(url_string):
try:
result = urlparse(url_string)
if result.scheme and result.scheme in ["http", "https", "ftp", "ftps"]:
return True
else:
return False
return result.scheme and result.scheme in ["http", "https", "ftp", "ftps"]
except:
return False
return False
def encode_params(params):
"""
Encode a dict of params into a query string.
Use `quote_via=urllib.parse.quote` so that whitespaces will be encoded as
`%20` instead of as `+`. This is needed because oauthlib cannot handle `+`
as a whitespace.
"""
return urlencode(params, quote_via=quote)

21
frappe/integrations/offsite_backup_utils.py
View file

@ -6,8 +6,7 @@ from __future__ import unicode_literals
import frappe
import glob
import os
from frappe.utils import split_emails, get_backups_path
from frappe.utils import split_emails, cint
def send_email(success, service_name, doctype, email_field, error_status=None):
recipients = get_recipients(doctype, email_field)
@ -81,6 +80,22 @@ def get_file_size(file_path, unit):
return file_size
def get_chunk_site(file_size):
''' this function will return chunk size in megabytes based on file size '''
file_size_in_gb = cint(file_size/1024/1024)
MB = 1024 * 1024
if file_size_in_gb > 5000:
return 200 * MB
elif file_size_in_gb >= 3000:
return 150 * MB
elif file_size_in_gb >= 1000:
return 100 * MB
elif file_size_in_gb >= 500:
return 50 * MB
else:
return 15 * MB
def validate_file_size():
frappe.flags.create_new_backup = True
@ -98,4 +113,4 @@ def generate_files_backup():
db_type=frappe.conf.db_type, db_port=frappe.conf.db_port)
backup.set_backup_file_name()
backup.zip_files()
backup.zip_files()

14
frappe/model/delete_doc.py
View file

@ -340,19 +340,25 @@ def clear_timeline_references(link_doctype, link_name):
WHERE `tabCommunication Link`.link_doctype=%s AND `tabCommunication Link`.link_name=%s""", (link_doctype, link_name))
def insert_feed(doc):
from frappe.utils import get_fullname
if frappe.flags.in_install or frappe.flags.in_import or getattr(doc, "no_feed_on_delete", False):
if (
frappe.flags.in_install
or frappe.flags.in_uninstall
or frappe.flags.in_import
or getattr(doc, "no_feed_on_delete", False)
):
return
from frappe.utils import get_fullname
frappe.get_doc({
"doctype": "Comment",
"comment_type": "Deleted",
"reference_doctype": doc.doctype,
"subject": "{0} {1}".format(_(doc.doctype), doc.name),
"full_name": get_fullname(doc.owner)
"full_name": get_fullname(doc.owner),
}).insert(ignore_permissions=True)
def delete_controllers(doctype, module):
"""
Delete controller code in the doctype folder

2
frappe/model/workflow.py
View file

@ -29,6 +29,8 @@ def get_transitions(doc, workflow = None, raise_exception=False):
if doc.is_new():
return []
doc.load_from_db()
frappe.has_permission(doc, 'read', throw=True)
roles = frappe.get_roles()

2
frappe/oauth.py
View file

@ -148,7 +148,7 @@ class OAuthWebRequestValidator(RequestValidator):
print("Failed body authentication: Application %s does not exist".format(cid=request.client_id))
cookie_dict = get_cookie_dict_from_headers(request)
user_id = unquote(cookie_dict['user_id']) if 'user_id' in cookie_dict else "Guest"
user_id = unquote(cookie_dict.get('user_id').value) if 'user_id' in cookie_dict else "Guest"
return frappe.session.user == user_id
def authenticate_client_id(self, client_id, request, *args, **kwargs):

4
frappe/public/build.json
View file

@ -245,7 +245,9 @@
"public/js/frappe/ui/chart.js",
"public/js/frappe/ui/datatable.js",
"public/js/frappe/ui/driver.js",
"public/js/frappe/barcode_scanner/index.js"
"public/js/frappe/barcode_scanner/index.js",
"public/js/frappe/widgets/utils.js"
],
"css/form.min.css": [
"public/less/form_grid.less"

4
frappe/public/js/frappe/file_uploader/index.js
View file

@ -15,7 +15,11 @@ export default class FileUploader {
allow_multiple,
as_dataurl,
disable_file_browser,
frm
} = {}) {
frm && frm.attachments.max_reached(true);
if (!wrapper) {
this.make_dialog();
} else {

1
frappe/public/js/frappe/form/controls/code.js
View file

@ -66,6 +66,7 @@ frappe.ui.form.ControlCode = frappe.ui.form.ControlText.extend({
const ace_language_mode = language_map[language] || '';
this.editor.session.setMode(ace_language_mode);
this.editor.setKeyboardHandler('ace/keyboard/vscode');
},
parse(value) {

2
frappe/public/js/frappe/form/controls/comment.js
View file

@ -60,7 +60,7 @@ frappe.ui.form.ControlComment = frappe.ui.form.ControlTextEditor.extend({
update_state() {
const value = this.get_value();
if (strip_html(value).trim() != "") {
if (strip_html(value).trim() != "" || value.includes('img')) {
this.button.removeClass('btn-default').addClass('btn-primary');
} else {
this.button.addClass('btn-default').removeClass('btn-primary');

6
frappe/public/js/frappe/form/dashboard.js
View file

@ -35,6 +35,7 @@ frappe.ui.form.Dashboard = Class.extend({
// clear custom
this.wrapper.find('.custom').remove();
this.hide();
},
set_headline: function(html, color) {
this.frm.layout.show_message(html, color);
@ -171,7 +172,7 @@ frappe.ui.form.Dashboard = Class.extend({
if(this.data.graph) {
this.setup_graph();
show = true;
// show = true;
}
if(show) {
@ -494,6 +495,9 @@ frappe.ui.form.Dashboard = Class.extend({
callback: function(r) {
if(r.message) {
me.render_graph(r.message);
me.show();
} else {
me.hide();
}
}
});

24
frappe/public/js/frappe/form/footer/timeline.js
View file

@ -30,7 +30,7 @@ frappe.ui.form.Timeline = class Timeline {
render_input: true,
only_input: true,
on_submit: (val) => {
if(strip_html(val).trim() != "") {
if (strip_html(val).trim() != "" || val.includes('img')) {
this.insert_comment(val, this.comment_area.button);
}
}
@ -547,10 +547,7 @@ frappe.ui.form.Timeline = class Timeline {
log.color = 'dark';
log.sender = log.owner;
log.comment_type = 'Milestone';
log.content = __('{0} changed {1} to {2}', [
frappe.user.full_name(log.owner).bold(),
frappe.meta.get_label(this.frm.doctype, log.track_field),
log.value.bold()]);
log.content = __('{0} changed {1} to {2}', [ frappe.user.full_name(log.owner).bold(), frappe.meta.get_label(this.frm.doctype, log.track_field), log.value.bold()]);
return log;
});
return milestones;
@ -613,11 +610,7 @@ frappe.ui.form.Timeline = class Timeline {
const field_display_status = frappe.perm.get_field_display_status(df, null,
me.frm.perm);
if (field_display_status === 'Read' || field_display_status === 'Write') {
parts.push(__('{0} from {1} to {2}', [
__(df.label),
me.format_content_for_timeline(p[1]),
me.format_content_for_timeline(p[2])
]));
parts.push(__('{0} from {1} to {2}', [ __(df.label), me.format_content_for_timeline(p[1]), me.format_content_for_timeline(p[2])]));
}
}
}
@ -648,13 +641,7 @@ frappe.ui.form.Timeline = class Timeline {
null, me.frm.perm);
if (field_display_status === 'Read' || field_display_status === 'Write') {
parts.push(__('{0} from {1} to {2} in row #{3}', [
frappe.meta.get_label(me.frm.fields_dict[row[0]].grid.doctype,
p[0]),
me.format_content_for_timeline(p[1]),
me.format_content_for_timeline(p[2]),
row[1]
]));
parts.push(__('{0} from {1} to {2} in row #{3}', [ frappe.meta.get_label( me.frm.fields_dict[row[0]].grid.doctype, p[0]), me.format_content_for_timeline(p[1]), me.format_content_for_timeline(p[2]), row[1] ]));
}
}
return parts.length < 3;
@ -691,8 +678,7 @@ frappe.ui.form.Timeline = class Timeline {
return p;
});
if (parts.length) {
out.push(me.get_version_comment(version, __("{0} rows for {1}",
[__(key), parts.join(', ')])));
out.push(me.get_version_comment(version, __("{0} rows for {1}", [__(key), parts.join(', ')])));
}
}
});

6
frappe/public/js/frappe/form/form.js
View file

@ -232,14 +232,10 @@ frappe.ui.form.Form = class FrappeForm {
throw "attach error";
}
if(me.attachments.max_reached()) {
frappe.msgprint(__("Maximum Attachment Limit for this record reached."));
throw "attach error";
}
new frappe.ui.FileUploader({
doctype: me.doctype,
docname: me.docname,
frm: me,
files: dataTransfer.files,
folder: 'Home/Attachments',
on_success(file_doc) {

2
frappe/public/js/frappe/form/layout.js
View file

@ -113,7 +113,7 @@ frappe.ui.form.Layout = Class.extend({
label: __('Dashboard'),
cssClass: 'form-dashboard',
collapsible: 1,
//hidden: 1
// hidden: 1
});
},

22
frappe/public/js/frappe/form/sidebar/attachments.js
View file

@ -16,15 +16,19 @@ frappe.ui.form.Attachments = Class.extend({
this.add_attachment_wrapper = this.parent.find(".add_attachment").parent();
this.attachments_label = this.parent.find(".attachments-label");
},
max_reached: function() {
// no of attachments
var n = Object.keys(this.get_attachments()).length;
// button if the number of attachments is less than max
if(n < this.frm.meta.max_attachments || !this.frm.meta.max_attachments) {
return false;
max_reached: function(raise_exception=false) {
const attachment_count = Object.keys(this.get_attachments()).length;
const attachment_limit = this.frm.meta.max_attachments;
if (attachment_limit && attachment_count >= attachment_limit) {
if (raise_exception) {
frappe.throw({
title: __("Attachment Limit Reached"),
message: __("Maximum attachment limit of {0} has been reached.", [cstr(attachment_limit).bold()]),
});
}
return true;
}
return true;
return false;
},
refresh: function() {
var me = this;
@ -140,7 +144,6 @@ frappe.ui.form.Attachments = Class.extend({
});
},
new_attachment: function(fieldname) {
var me = this;
if (this.dialog) {
// remove upload dialog
this.dialog.$wrapper.remove();
@ -149,6 +152,7 @@ frappe.ui.form.Attachments = Class.extend({
new frappe.ui.FileUploader({
doctype: this.frm.doctype,
docname: this.frm.docname,
frm: this.frm,
folder: 'Home/Attachments',
on_success: (file_doc) => {
this.attachment_uploaded(file_doc);

2
frappe/public/js/frappe/form/templates/form_sidebar.html
View file

@ -60,7 +60,7 @@
<ul class="list-unstyled sidebar-menu form-attachments">
<li class="h6 attachments-label">{%= __("Attachments") %}</li>
<li><a class="add-attachment text-muted">{%= __("Attach File") %}
<i class="octicon octicon-plus" style="margin-left: 2px;"></i></li></a>
<i class="octicon octicon-plus" style="margin-left: 2px;"></i></a></li>
</ul>
<ul class="list-unstyled sidebar-menu">
<li class="h6 tags-label">{%= __("Tags") %}</li>

130
frappe/public/js/frappe/utils/utils.js
View file

@ -2,7 +2,8 @@
// MIT License. See license.txt
import deep_equal from "fast-deep-equal";
frappe.provide('frappe.utils');
frappe.provide("frappe.utils");
Object.assign(frappe.utils, {
get_random: function(len) {
@ -897,7 +898,132 @@ Object.assign(frappe.utils, {
hide_seconds: docfield.hide_seconds
};
return duration_options;
}
},
generate_route: function(item) {
const type = item.type.toLowerCase();
if (type === "doctype") {
item.doctype = item.name;
}
let route = "";
if (!item.route) {
if (item.link) {
route = strip(item.link, "#");
} else if (type === "doctype") {
if (frappe.model.is_single(item.doctype)) {
route = "Form/" + item.doctype;
} else {
if (!item.doc_view) {
if (frappe.model.is_tree(item.doctype)) {
item.doc_view = "Tree";
} else {
item.doc_view = "List";
}
}
switch (item.doc_view) {
case "List":
if (item.filters) {
frappe.route_options = item.filters;
}
route = "List/" + item.doctype;
break;
case "Tree":
route = "Tree/" + item.doctype;
break;
case "Report Builder":
route = "List/" + item.doctype + "/Report";
break;
case "Dashboard":
route = "List/" + item.doctype + "/Dashboard";
break;
case "New":
route = "Form/" + item.doctype + "/New " + item.doctype;
break;
case "Calendar":
route = "List/" + item.doctype + "/Calendar/Default";
break;
default:
frappe.throw({ message: __("Not a valid DocType view:") + item.doc_view, title: __("Unknown View") });
route = "";
}
}
} else if (type === "report" && item.is_query_report) {
route = "query-report/" + item.name;
} else if (type === "report") {
route = "List/" + item.doctype + "/Report/" + item.name;
} else if (type === "page") {
route = item.name;
} else if (type === "dashboard") {
route = "dashboard/" + item.name;
}
route = "#" + route;
} else {
route = item.route;
}
if (item.route_options) {
route +=
"?" +
$.map(item.route_options, function (value, key) {
return (
encodeURIComponent(key) + "=" + encodeURIComponent(value)
);
}).join("&");
}
// if(type==="page" || type==="help" || type==="report" ||
// (item.doctype && frappe.model.can_read(item.doctype))) {
// item.shown = true;
// }
return route;
},
shorten_number: function (number, country) {
country = (country == 'India') ? country : '';
const number_system = this.get_number_system(country);
let x = Math.abs(Math.round(number));
for (const map of number_system) {
const condition = map.condition ? map.condition(x) : x >= map.divisor;
if (condition) {
return (number/map.divisor).toFixed(2) + ' ' + map.symbol;
}
}
return number.toFixed();
},
get_number_system: function (country) {
let number_system_map = {
'India':
[{
divisor: 1.0e+7,
symbol: 'Cr'
},
{
divisor: 1.0e+5,
symbol: 'Lakh'
}],
'':
[{
divisor: 1.0e+12,
symbol: 'T'
},
{
divisor: 1.0e+9,
symbol: 'B'
},
{
divisor: 1.0e+6,
symbol: 'M'
},
{
divisor: 1.0e+3,
symbol: 'K',
condition: (num) => num.toFixed().length > 5
}]
};
return number_system_map[country];
},
});
// Array de duplicate

6
frappe/public/js/frappe/views/reports/query_report.js
View file

@ -1,8 +1,8 @@
// Copyright (c) 2018, Frappe Technologies Pvt. Ltd. and Contributors
// MIT License. See license.txt
import DataTable from 'frappe-datatable';
import { build_summary_item } from "../../widgets/utils";
frappe.provide('frappe.widget.utils');
frappe.provide('frappe.views');
frappe.provide('frappe.query_reports');
@ -631,7 +631,7 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
render_summary(data) {
data.forEach((summary) => {
build_summary_item(summary).appendTo(this.$summary);
frappe.widget.utils.build_summary_item(summary).appendTo(this.$summary);
})
this.$summary.show();
@ -813,6 +813,7 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
data.splice(-1, 1);
}
this.$report.show();
if (this.datatable && this.datatable.options
&& (this.datatable.options.showTotalRow ===this.raw_data.add_total_row)) {
this.datatable.options.treeView = this.tree_report;
@ -844,7 +845,6 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
if (this.report_settings.after_datatable_render) {
this.report_settings.after_datatable_render(this.datatable);
}
this.$report.show();
}
get_chart_options(data) {

5
frappe/public/js/frappe/widgets/chart_widget.js
View file

@ -1,5 +1,6 @@
import Widget from "./base_widget.js";
import { build_summary_item } from "./utils";
frappe.provide('frappe.widget.utils');
frappe.provide("frappe.dashboards");
frappe.provide("frappe.dashboards.chart_sources");
@ -80,7 +81,7 @@ export default class ChartWidget extends Widget {
}
this.summary.forEach(summary => {
build_summary_item(summary).appendTo(this.$summary);
frappe.widget.utils.build_summary_item(summary).appendTo(this.$summary);
});
this.summary.length && this.$summary.show();
}

5
frappe/public/js/frappe/widgets/links_widget.js
View file

@ -1,5 +1,6 @@
import Widget from "./base_widget.js";
import { generate_route } from "./utils";
frappe.provide("frappe.utils");
export default class LinksWidget extends Widget {
constructor(opts) {
@ -55,7 +56,7 @@ export default class LinksWidget extends Widget {
return `<span class="link-content help-video-link ellipsis" data-youtubeid="${item.youtube_id}">
${item.label ? item.label : item.name}</span>`;
return `<a data-route="${generate_route(item)}" class="link-content ellipsis">
return `<a data-route="${frappe.utils.generate_route(item)}" class="link-content ellipsis">
${item.label ? item.label : item.name}</a>`;
};

7
frappe/public/js/frappe/widgets/number_card_widget.js
View file

@ -1,5 +1,6 @@
import Widget from "./base_widget.js";
import { generate_route, shorten_number } from "./utils";
frappe.provide("frappe.utils");
export default class NumberCardWidget extends Widget {
constructor(opts) {
@ -74,7 +75,7 @@ export default class NumberCardWidget extends Widget {
set_route() {
const is_document_type = this.card_doc.type !== 'Report';
const name = is_document_type ? this.card_doc.document_type : this.card_doc.report_name;
const route = generate_route({
const route = frappe.utils.generate_route({
name: name,
type: is_document_type ? 'doctype' : 'report',
is_query_report: !is_document_type,
@ -203,7 +204,7 @@ export default class NumberCardWidget extends Widget {
get_formatted_number(df) {
const default_country = frappe.sys_defaults.country;
const shortened_number = shorten_number(this.number, default_country);
const shortened_number = frappe.utils.shorten_number(this.number, default_country);
let number_parts = shortened_number.split(' ');
const symbol = number_parts[1] || '';

5
frappe/public/js/frappe/widgets/onboarding_widget.js
View file

@ -1,5 +1,6 @@
import Widget from "./base_widget.js";
import { generate_route } from "./utils";
frappe.provide("frappe.utils");
export default class OnboardingWidget extends Widget {
constructor(opts) {
@ -92,7 +93,7 @@ export default class OnboardingWidget extends Widget {
}
open_report(step) {
let route = generate_route({
let route = frappe.utils.generate_route({
name: step.reference_report,
type: "report",
is_query_report: ["Query Report", "Script Report"].includes(

5
frappe/public/js/frappe/widgets/shortcut_widget.js
View file

@ -1,5 +1,6 @@
import Widget from "./base_widget.js";
import { generate_route } from "./utils";
frappe.provide("frappe.utils");
export default class ShortcutWidget extends Widget {
constructor(opts) {
@ -25,7 +26,7 @@ export default class ShortcutWidget extends Widget {
this.widget.click(() => {
if (this.in_customize_mode) return;
let route = generate_route({
let route = frappe.utils.generate_route({
route: this.route,
name: this.link_to,
type: this.type,

209
frappe/public/js/frappe/widgets/utils.js
View file

@ -1,193 +1,24 @@
function generate_route(item) {
const type = item.type.toLowerCase()
if (type === "doctype") {
item.doctype = item.name;
}
let route = "";
if (!item.route) {
if (item.link) {
route = strip(item.link, "#");
} else if (type === "doctype") {
if (frappe.model.is_single(item.doctype)) {
route = "Form/" + item.doctype;
} else {
if (!item.doc_view) {
if (frappe.model.is_tree(item.doctype)) {
item.doc_view = "Tree";
} else {
item.doc_view = "List";
}
}
switch (item.doc_view) {
case "List":
if (item.filters) {
frappe.route_options = item.filters;
}
route = "List/" + item.doctype;
break;
case "Tree":
route = "Tree/" + item.doctype;
break;
case "Report Builder":
route = "List/" + item.doctype + "/Report";
break;
case "Dashboard":
route = "List/" + item.doctype + "/Dashboard";
break;
case "New":
route = "Form/" + item.doctype + "/New " + item.doctype;
break;
case "Calendar":
route = "List/" + item.doctype + "/Calendar/Default";
break;
default:
frappe.throw({ message: __("Not a valid DocType view:") + item.doc_view, title: __("Unknown View") });
route = "";
}
}
} else if (type === "report" && item.is_query_report) {
route = "query-report/" + item.name;
} else if (type === "report") {
route = "List/" + item.doctype + "/Report/" + item.name;
} else if (type === "page") {
route = item.name;
} else if (type === "dashboard") {
route = "dashboard/" + item.name;
frappe.provide('frappe.widget.utils');
frappe.widget.utils = {
build_summary_item: function(summary) {
let df = { fieldtype: summary.datatype };
let doc = null;
if (summary.datatype == "Currency") {
df.options = "currency";
doc = { currency: summary.currency };
}
route = "#" + route;
} else {
route = item.route;
}
let value = frappe.format(summary.value, df, null, doc);
let indicator = summary.indicator
? `indicator ${summary.indicator.toLowerCase()}`
: "";
if (item.route_options) {
route +=
"?" +
$.map(item.route_options, function (value, key) {
return (
encodeURIComponent(key) + "=" + encodeURIComponent(value)
);
}).join("&");
}
// if(type==="page" || type==="help" || type==="report" ||
// (item.doctype && frappe.model.can_read(item.doctype))) {
// item.shown = true;
// }
return route;
}
function generate_grid(data) {
function add(a, b) {
return a + b;
}
const grid_max_cols = 6
// Split the data into multiple arrays
// Each array will contain grid elements of one row
let processed = []
let temp = []
let init = 0
data.forEach((data) => {
init = init + data.columns;
if (init > grid_max_cols) {
init = data.columns
processed.push(temp)
temp = []
}
temp.push(data)
})
processed.push(temp)
let grid_template = [];
processed.forEach((data, index) => {
let aa = data.map(dd => {
return Array.apply(null, Array(dd.columns)).map(String.prototype.valueOf, dd.name)
}).flat()
if (aa.length < grid_max_cols) {
let diff = grid_max_cols - aa.length;
for (let ii = 0; ii < diff; ii++) {
aa.push(`grid-${index}-${ii}`)
}
}
grid_template.push(aa.join(" "))
})
let grid_template_area = ""
grid_template.forEach(temp => {
grid_template_area += `"${temp}" `
})
return grid_template_area
}
const build_summary_item = (summary) => {
let df = { fieldtype: summary.datatype };
let doc = null;
if (summary.datatype == "Currency") {
df.options = "currency";
doc = { currency: summary.currency };
}
let value = frappe.format(summary.value, df, null, doc);
let indicator = summary.indicator ? `indicator ${summary.indicator.toLowerCase()}` : '';
return $(`<div class="summary-item">
<span class="summary-label small text-muted ${indicator}">${summary.label}</span>
<h1 class="summary-value">${ value}</h1>
</div>`);
return $(
`<div class="summary-item"><span class="summary-label small text-muted ${indicator}">${
summary.label
}</span><h1 class="summary-value">${value}</h1></div>`
);
},
};
function shorten_number(number, country) {
country = (country == 'India') ? country : '';
const number_system = get_number_system(country);
let x = Math.abs(Math.round(number));
for (const map of number_system) {
const condition = map.condition ? map.condition(x) : x >= map.divisor;
if (condition) {
return (number/map.divisor).toFixed(2) + ' ' + map.symbol;
}
}
return number.toFixed(2);
}
function get_number_system(country) {
let number_system_map = {
'India':
[{
divisor: 1.0e+7,
symbol: 'Cr'
},
{
divisor: 1.0e+5,
symbol: 'Lakh'
}],
'':
[{
divisor: 1.0e+12,
symbol: 'T'
},
{
divisor: 1.0e+9,
symbol: 'B'
},
{
divisor: 1.0e+6,
symbol: 'M'
},
{
divisor: 1.0e+3,
symbol: 'K',
condition: (num) => num.toFixed().length > 5
}]
};
return number_system_map[country];
}
export { generate_route, generate_grid, build_summary_item, shorten_number };

8
frappe/templates/base.html
View file

@ -30,11 +30,11 @@
{%- if theme.name != 'Standard' -%}
<link type="text/css" rel="stylesheet" href="{{ theme.theme_url }}">
{%- else -%}
<link type="text/css" rel="stylesheet" href="/assets/css/frappe-web-b4.css">
<link type="text/css" rel="stylesheet" href="/assets/css/frappe-web-b4.css?ver={{ build_version }}">
{%- endif -%}
{%- for link in web_include_css %}
<link type="text/css" rel="stylesheet" href="{{ link|abs_url }}">
<link type="text/css" rel="stylesheet" href="{{ link|abs_url }}?ver={{ build_version }}">
{%- endfor -%}
{%- endblock -%}
@ -94,12 +94,12 @@
{% block base_scripts %}
<!-- js should be loaded in body! -->
<script type="text/javascript" src="/assets/frappe/js/lib/jquery/jquery.min.js"></script>
<script type="text/javascript" src="/assets/js/frappe-web.min.js"></script>
<script type="text/javascript" src="/assets/js/frappe-web.min.js?ver={{ build_version }}"></script>
<script type="text/javascript" src="/assets/js/bootstrap-4-web.min.js"></script>
{% endblock %}
{%- for link in web_include_js %}
<script type="text/javascript" src="{{ link | abs_url }}"></script>
<script type="text/javascript" src="{{ link | abs_url }}?ver={{ build_version }}"></script>
{%- endfor -%}
{%- block script %}

16
frappe/templates/includes/oauth_confirmation.html
View file

@ -1,7 +1,7 @@
{% if not error %}
<div class="panel panel-default">
<div class="panel-heading">
<h3 class="panel-title">{{ client_id }} wants to access the following details from your account</h3>
<h3 class="panel-title">{{ _("{} wants to access the following details from your account").format(client_id) }}</h3>
</div>
<div class="panel-body">
<ul class="list-group">
@ -11,10 +11,10 @@
</ul>
<ul class="list-inline">
<li>
<button id="allow" class="btn btn-sm btn-primary">Allow</button>
<button id="allow" class="btn btn-sm btn-primary">{{ _("Allow") }}</button>
</li>
<li>
<button id="deny" class="btn btn-sm btn-light">Deny</button>
<button id="deny" class="btn btn-sm btn-light">{{ _("Deny") }}</button>
</li>
</ul>
</div>
@ -22,24 +22,24 @@
<script type="text/javascript">
frappe.ready(function() {
$('#allow').on('click', function(event) {
window.location.replace("{{ success_url|string }}");
window.location.replace("{{ success_url | string }}");
});
$('#deny').on('click', function(event) {
window.location.replace("{{ failure_url|string }}");
window.location.replace("{{ failure_url | string }}");
});
});
</script>
{% else %}
<div class="panel panel-danger">
<div class="panel-heading">
<h3 class="panel-title">Authorization error for {{ client_id }}</h3>
<h3 class="panel-title">{{ _("Authorization error for {}.").format(client_id) }}</h3>
</div>
<div class="panel-body">
<p>An unexpected error occurred while authorizing {{ client_id }}.</p>
<p>{{ _("An unexpected error occurred while authorizing {}.").format(client_id) }}</p>
<h4>{{ error }}</h4>
<ul class="list-inline">
<li>
<button class="btn btn-sm btn-light">OK</button>
<button class="btn btn-sm btn-light">{{ _("OK") }}</button>
</li>
</ul>
</div>

247
frappe/tests/test_commands.py
View file

@ -1,24 +1,88 @@
# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors
# imports - standard imports
import gzip
import json
import os
import shlex
import subprocess
import sys
import unittest
from glob import glob
import glob
# imports - module imports
import frappe
from frappe.utils.backups import fetch_latest_backups
import frappe.recorder
from frappe.installer import add_to_installed_apps
from frappe.utils import add_to_date, now
from frappe.utils.backups import fetch_latest_backups
# TODO: check frappe.cli.coloured_output to set coloured output!
def supports_color():
"""
Returns True if the running system's terminal supports color, and False
otherwise.
"""
plat = sys.platform
supported_platform = plat != 'Pocket PC' and (plat != 'win32' or 'ANSICON' in os.environ)
# isatty is not always implemented, #6223.
is_a_tty = hasattr(sys.stdout, 'isatty') and sys.stdout.isatty()
return supported_platform and is_a_tty
class color(dict):
nc = "\033[0m"
blue = "\033[94m"
green = "\033[92m"
yellow = "\033[93m"
red = "\033[91m"
silver = "\033[90m"
def __getattr__(self, key):
if supports_color():
ret = self.get(key)
else:
ret = ""
return ret
def clean(value):
if isinstance(value, (bytes, str)):
value = value.decode().strip()
"""Strips and converts bytes to str
Args:
value ([type]): [description]
Returns:
[type]: [description]
"""
if isinstance(value, bytes):
value = value.decode()
if isinstance(value, str):
value = value.strip()
return value
def exists_in_backup(doctypes, file):
"""Checks if the list of doctypes exist in the database.sql.gz file supplied
Args:
doctypes (list): List of DocTypes to be checked
file (str): Path of the database file
Returns:
bool: True if all tables exist
"""
predicate = (
'COPY public."tab{}"'
if frappe.conf.db_type == "postgres"
else "CREATE TABLE `tab{}`"
)
with gzip.open(file, "rb") as f:
content = f.read().decode("utf8")
return all([predicate.format(doctype).lower() in content.lower() for doctype in doctypes])
class BaseTestCommands(unittest.TestCase):
def execute(self, command, kwargs=None):
site = {"site": frappe.local.site}
@ -26,13 +90,26 @@ class BaseTestCommands(unittest.TestCase):
kwargs.update(site)
else:
kwargs = site
command = command.replace("\n", " ").format(**kwargs)
command = shlex.split(command)
self.command = " ".join(command.split()).format(**kwargs)
print("{0}$ {1}{2}".format(color.silver, self.command, color.nc))
command = shlex.split(self.command)
self._proc = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
self.stdout = clean(self._proc.stdout)
self.stderr = clean(self._proc.stderr)
self.returncode = clean(self._proc.returncode)
def _formatMessage(self, msg, standardMsg):
output = super(BaseTestCommands, self)._formatMessage(msg, standardMsg)
cmd_execution_summary = "\n".join([
"-" * 70,
"Last Command Execution Summary:",
"Command: {}".format(self.command) if self.command else "",
"Standard Output: {}".format(self.stdout) if self.stdout else "",
"Standard Error: {}".format(self.stderr) if self.stderr else "",
"Return Code: {}".format(self.returncode) if self.returncode else "",
]).strip()
return "{}\n\n{}".format(output, cmd_execution_summary)
class TestCommands(BaseTestCommands):
def test_execute(self):
@ -52,9 +129,24 @@ class TestCommands(BaseTestCommands):
# The returned value has quotes which have been trimmed for the test
self.execute("""bench --site {site} execute frappe.bold --kwargs '{{"text": "DocType"}}'""")
self.assertEquals(self.returncode, 0)
self.assertEquals(self.stdout[1:-1], frappe.bold(text='DocType'))
self.assertEquals(self.stdout[1:-1], frappe.bold(text="DocType"))
def test_backup(self):
backup = {
"includes": {
"includes": [
"ToDo",
"Note",
]
},
"excludes": {
"excludes": [
"Activity Log",
"Access Log",
"Error Log"
]
}
}
home = os.path.expanduser("~")
site_backup_path = frappe.utils.get_site_path("private", "backups")
@ -94,16 +186,19 @@ class TestCommands(BaseTestCommands):
"db_path": "database.sql.gz",
"files_path": "public.tar",
"private_path": "private.tar",
"conf_path": "config.json"
"conf_path": "config.json",
}.items()
}
self.execute("""bench
self.execute(
"""bench
--site {site} backup --with-files
--backup-path-db {db_path}
--backup-path-files {files_path}
--backup-path-private-files {private_path}
--backup-path-conf {conf_path}""", kwargs)
--backup-path-conf {conf_path}""",
kwargs,
)
self.assertEquals(self.returncode, 0)
for path in kwargs.values():
@ -111,16 +206,122 @@ class TestCommands(BaseTestCommands):
# test 5: take a backup with --compress
self.execute("bench --site {site} backup --with-files --compress")
self.assertEquals(self.returncode, 0)
compressed_files = glob(site_backup_path + "/*.tgz")
compressed_files = glob.glob(site_backup_path + "/*.tgz")
self.assertGreater(len(compressed_files), 0)
# test 6: take a backup with --verbose
self.execute("bench --site {site} backup --verbose")
self.assertEquals(self.returncode, 0)
# test 7: take a backup with frappe.conf.backup.includes
self.execute(
"bench --site {site} set-config backup '{includes}' --as-dict",
{"includes": json.dumps(backup["includes"])},
)
self.execute("bench --site {site} backup --verbose")
self.assertEquals(self.returncode, 0)
database = fetch_latest_backups(partial=True)["database"]
self.assertTrue(exists_in_backup(backup["includes"]["includes"], database))
# test 8: take a backup with frappe.conf.backup.excludes
self.execute(
"bench --site {site} set-config backup '{excludes}' --as-dict",
{"excludes": json.dumps(backup["excludes"])},
)
self.execute("bench --site {site} backup --verbose")
self.assertEquals(self.returncode, 0)
database = fetch_latest_backups(partial=True)["database"]
self.assertFalse(exists_in_backup(backup["excludes"]["excludes"], database))
self.assertTrue(exists_in_backup(backup["includes"]["includes"], database))
# test 9: take a backup with --include (with frappe.conf.excludes still set)
self.execute(
"bench --site {site} backup --include '{include}'",
{"include": ",".join(backup["includes"]["includes"])},
)
self.assertEquals(self.returncode, 0)
database = fetch_latest_backups(partial=True)["database"]
self.assertTrue(exists_in_backup(backup["includes"]["includes"], database))
# test 10: take a backup with --exclude
self.execute(
"bench --site {site} backup --exclude '{exclude}'",
{"exclude": ",".join(backup["excludes"]["excludes"])},
)
self.assertEquals(self.returncode, 0)
database = fetch_latest_backups(partial=True)["database"]
self.assertFalse(exists_in_backup(backup["excludes"]["excludes"], database))
# test 11: take a backup with --ignore-backup-conf
self.execute("bench --site {site} backup --ignore-backup-conf")
self.assertEquals(self.returncode, 0)
database = fetch_latest_backups()["database"]
self.assertTrue(exists_in_backup(backup["excludes"]["excludes"], database))
def test_restore(self):
# step 0: create a site to run the test on
global_config = {
"admin_password": frappe.conf.admin_password,
"root_login": frappe.conf.root_login,
"root_password": frappe.conf.root_password,
"db_type": frappe.conf.db_type,
}
site_data = {"another_site": f"{frappe.local.site}-restore.test", **global_config}
for key, value in global_config.items():
if value:
self.execute(f"bench set-config {key} {value} -g")
self.execute(
"bench new-site {another_site} --admin-password {admin_password} --db-type"
" {db_type}",
site_data,
)
# test 1: bench restore from full backup
self.execute("bench --site {another_site} backup --ignore-backup-conf", site_data)
self.execute(
"bench --site {another_site} execute frappe.utils.backups.fetch_latest_backups",
site_data,
)
site_data.update({"database": json.loads(self.stdout)["database"]})
self.execute("bench --site {another_site} restore {database}", site_data)
# test 2: restore from partial backup
self.execute("bench --site {another_site} backup --exclude 'ToDo'", site_data)
site_data.update({"kw": "\"{'partial':True}\""})
self.execute(
"bench --site {another_site} execute"
" frappe.utils.backups.fetch_latest_backups --kwargs {kw}",
site_data,
)
site_data.update({"database": json.loads(self.stdout)["database"]})
self.execute("bench --site {another_site} restore {database}", site_data)
self.assertEquals(self.returncode, 1)
def test_partial_restore(self):
_now = now()
for num in range(10):
frappe.get_doc({
"doctype": "ToDo",
"date": add_to_date(_now, days=num),
"description": frappe.mock("paragraph")
}).insert()
frappe.db.commit()
todo_count = frappe.db.count("ToDo")
# check if todos exist, create a partial backup and see if the state is the same after restore
self.assertIsNot(todo_count, 0)
self.execute("bench --site {site} backup --only 'ToDo'")
db_path = fetch_latest_backups(partial=True)["database"]
self.assertTrue("partial" in db_path)
frappe.db.sql_ddl("DROP TABLE IF EXISTS `tabToDo`")
frappe.db.commit()
self.execute("bench --site {site} partial-restore {path}", {"path": db_path})
self.assertEquals(self.returncode, 0)
self.assertEquals(frappe.db.count("ToDo"), todo_count)
def test_recorder(self):
frappe.recorder.stop()
@ -133,7 +334,6 @@ class TestCommands(BaseTestCommands):
self.assertEqual(frappe.recorder.status(), False)
def test_remove_from_installed_apps(self):
from frappe.installer import add_to_installed_apps
app = "test_remove_app"
add_to_installed_apps(app)
@ -143,5 +343,24 @@ class TestCommands(BaseTestCommands):
# test 1: remove app from installed_apps global default
self.execute("bench --site {site} remove-from-installed-apps {app}", {"app": app})
self.assertEquals(self.returncode, 0)
self.execute("bench --site {site} list-apps")
self.assertNotIn(app, self.stdout)
def test_list_apps(self):
# test 1: sanity check for command
self.execute("bench --site all list-apps")
self.assertEquals(self.returncode, 0)
# test 2: bare functionality for single site
self.execute("bench --site {site} list-apps")
self.assertEquals(self.returncode, 0)
list_apps = set([
_x.split()[0] for _x in self.stdout.split("\n")
])
doctype = frappe.get_single("Installed Applications").installed_applications
if doctype:
installed_apps = set([x.app_name for x in doctype])
else:
installed_apps = set(frappe.get_installed_apps())
self.assertSetEqual(list_apps, installed_apps)

11
frappe/tests/test_oauth20.py
View file

@ -6,6 +6,7 @@ import unittest, frappe, requests, time
from frappe.test_runner import make_test_records
from six.moves.urllib.parse import urlparse, parse_qs, urljoin
from urllib.parse import urlencode, quote
from frappe.integrations.oauth2 import encode_params
class TestOAuth20(unittest.TestCase):
@ -232,13 +233,3 @@ def login(session):
def get_full_url(endpoint):
"""Turn '/endpoint' into 'http://127.0.0.1:8000/endpoint'."""
return urljoin(frappe.utils.get_url(), endpoint)
def encode_params(params):
"""
Encode a dict of params into a query string.
Use `quote_via=urllib.parse.quote` so that whitespaces will be encoded as
`%20` instead of as `+`. This is needed because oauthlib cannot handle `+`
as a whitespace.
"""
return urlencode(params, quote_via=quote)

8
frappe/utils/__init__.py
View file

@ -721,3 +721,11 @@ def get_file_size(path, format=False):
num /= 1024
return "{0:.1f}{1}{2}".format(num, 'Yi', suffix)
def get_build_version():
try:
return str(os.path.getmtime(os.path.join(frappe.local.sites_path, '.build')))
except OSError:
# .build can sometimes not exist
# this is not a major problem so send fallback
return frappe.utils.random_string(8)

555
frappe/utils/backups.py
View file

@ -2,11 +2,12 @@
# MIT License. See license.txt
# imports - standard imports
import json
import gzip
import os
from calendar import timegm
from datetime import datetime
from glob import glob
from shutil import which
# imports - third party imports
import click
@ -14,24 +15,42 @@ import click
# imports - module imports
import frappe
from frappe import _, conf
from frappe.utils import get_url, now, now_datetime, get_file_size
from frappe.utils import get_file_size, get_url, now, now_datetime
# backup variable for backwards compatibility
verbose = False
compress = False
_verbose = verbose
base_tables = ["__Auth", "__global_search", "__UserSettings"]
class BackupGenerator:
"""
This class contains methods to perform On Demand Backup
This class contains methods to perform On Demand Backup
To initialize, specify (db_name, user, password, db_file_name=None, db_host="localhost")
If specifying db_file_name, also append ".sql.gz"
To initialize, specify (db_name, user, password, db_file_name=None, db_host="localhost")
If specifying db_file_name, also append ".sql.gz"
"""
def __init__(self, db_name, user, password, backup_path=None, backup_path_db=None,
backup_path_files=None, backup_path_private_files=None, db_host="localhost", db_port=None,
verbose=False, db_type='mariadb', backup_path_conf=None, compress_files=False):
def __init__(
self,
db_name,
user,
password,
backup_path=None,
backup_path_db=None,
backup_path_files=None,
backup_path_private_files=None,
db_host="localhost",
db_port=None,
db_type="mariadb",
backup_path_conf=None,
ignore_conf=False,
compress_files=False,
include_doctypes="",
exclude_doctypes="",
verbose=False,
):
global _verbose
self.compress_files = compress_files or compress
self.db_host = db_host
@ -45,23 +64,35 @@ class BackupGenerator:
self.backup_path_db = backup_path_db
self.backup_path_files = backup_path_files
self.backup_path_private_files = backup_path_private_files
self.ignore_conf = ignore_conf
self.include_doctypes = include_doctypes
self.exclude_doctypes = exclude_doctypes
self.partial = False
if not self.db_type:
self.db_type = 'mariadb'
self.db_type = "mariadb"
if not self.db_port and self.db_type == 'mariadb':
self.db_port = 3306
elif not self.db_port and self.db_type == 'postgres':
self.db_port = 5432
if not self.db_port:
if self.db_type == "mariadb":
self.db_port = 3306
if self.db_type == "postgres":
self.db_port = 5432
site = frappe.local.site or frappe.generate_hash(length=8)
self.site_slug = site.replace('.', '_')
self.site_slug = site.replace(".", "_")
self.verbose = verbose
self.setup_backup_directory()
self.setup_backup_tables()
_verbose = verbose
def setup_backup_directory(self):
specified = self.backup_path or self.backup_path_db or self.backup_path_files or self.backup_path_private_files or self.backup_path_conf
specified = (
self.backup_path
or self.backup_path_db
or self.backup_path_files
or self.backup_path_private_files
or self.backup_path_conf
)
if not specified:
backups_folder = get_backup_path()
@ -71,32 +102,93 @@ class BackupGenerator:
if self.backup_path:
os.makedirs(self.backup_path, exist_ok=True)
for file_path in set([self.backup_path_files, self.backup_path_db, self.backup_path_private_files, self.backup_path_conf]):
for file_path in set(
[
self.backup_path_files,
self.backup_path_db,
self.backup_path_private_files,
self.backup_path_conf,
]
):
if file_path:
dir = os.path.dirname(file_path)
os.makedirs(dir, exist_ok=True)
def setup_backup_tables(self):
"""Sets self.backup_includes, self.backup_excludes based on passed args"""
existing_doctypes = set([x.name for x in frappe.get_all("DocType")])
def get_tables(doctypes):
tables = []
for doctype in doctypes:
if doctype and doctype in existing_doctypes:
if doctype.startswith("tab"):
tables.append(doctype)
else:
tables.append("tab" + doctype)
return tables
passed_tables = {
"include": get_tables(self.include_doctypes.strip().split(",")),
"exclude": get_tables(self.exclude_doctypes.strip().split(",")),
}
specified_tables = get_tables(frappe.conf.get("backup", {}).get("includes", []))
include_tables = (specified_tables + base_tables) if specified_tables else []
conf_tables = {
"include": include_tables,
"exclude": get_tables(frappe.conf.get("backup", {}).get("excludes", [])),
}
self.backup_includes = passed_tables["include"]
self.backup_excludes = passed_tables["exclude"]
if not (self.backup_includes or self.backup_excludes) and not self.ignore_conf:
self.backup_includes = self.backup_includes or conf_tables["include"]
self.backup_excludes = self.backup_excludes or conf_tables["exclude"]
self.partial = (self.backup_includes or self.backup_excludes) and not self.ignore_conf
@property
def site_config_backup_path(self):
# For backwards compatibility
click.secho("BackupGenerator.site_config_backup_path has been deprecated in favour of BackupGenerator.backup_path_conf", fg="yellow")
click.secho(
"BackupGenerator.site_config_backup_path has been deprecated in favour of"
" BackupGenerator.backup_path_conf",
fg="yellow",
)
return getattr(self, "backup_path_conf", None)
def get_backup(self, older_than=24, ignore_files=False, force=False):
"""
Takes a new dump if existing file is old
and sends the link to the file as email
Takes a new dump if existing file is old
and sends the link to the file as email
"""
#Check if file exists and is less than a day old
#If not Take Dump
# Check if file exists and is less than a day old
# If not Take Dump
if not force:
last_db, last_file, last_private_file, site_config_backup_path = self.get_recent_backup(older_than)
(
last_db,
last_file,
last_private_file,
site_config_backup_path,
) = self.get_recent_backup(older_than)
else:
last_db, last_file, last_private_file, site_config_backup_path = False, False, False, False
last_db, last_file, last_private_file, site_config_backup_path = (
False,
False,
False,
False,
)
self.todays_date = now_datetime().strftime('%Y%m%d_%H%M%S')
self.todays_date = now_datetime().strftime("%Y%m%d_%H%M%S")
if not (self.backup_path_conf and self.backup_path_db and self.backup_path_files and self.backup_path_private_files):
if not (
self.backup_path_conf
and self.backup_path_db
and self.backup_path_files
and self.backup_path_private_files
):
self.set_backup_file_name()
if not (last_db and last_file and last_private_file and site_config_backup_path):
@ -112,13 +204,13 @@ class BackupGenerator:
self.backup_path_conf = site_config_backup_path
def set_backup_file_name(self):
#Generate a random name using today's date and a 8 digit random number
for_conf = self.todays_date + "-" + self.site_slug + "-site_config_backup.json"
for_db = self.todays_date + "-" + self.site_slug + "-database.sql.gz"
partial = "-partial" if self.partial else ""
ext = "tgz" if self.compress_files else "tar"
for_public_files = self.todays_date + "-" + self.site_slug + "-files." + ext
for_private_files = self.todays_date + "-" + self.site_slug + "-private-files." + ext
for_conf = f"{self.todays_date}-{self.site_slug}-site_config_backup.json"
for_db = f"{self.todays_date}-{self.site_slug}{partial}-database.sql.gz"
for_public_files = f"{self.todays_date}-{self.site_slug}-files.{ext}"
for_private_files = f"{self.todays_date}-{self.site_slug}-private-files.{ext}"
backup_path = self.backup_path or get_backup_path()
if not self.backup_path_conf:
@ -130,11 +222,11 @@ class BackupGenerator:
if not self.backup_path_private_files:
self.backup_path_private_files = os.path.join(backup_path, for_private_files)
def get_recent_backup(self, older_than):
def get_recent_backup(self, older_than, partial=False):
backup_path = get_backup_path()
file_type_slugs = {
"database": "*-{}-database.sql.gz",
"database": "*-{{}}-{}database.sql.gz".format('*' if partial else ''),
"public": "*-{}-files.tar",
"private": "*-{}-private-files.tar",
"config": "*-{}-site_config_backup.json",
@ -158,8 +250,7 @@ class BackupGenerator:
return file_path
latest_backups = {
file_type: get_latest(pattern)
for file_type, pattern in file_type_slugs.items()
file_type: get_latest(pattern) for file_type, pattern in file_type_slugs.items()
}
recent_backups = {
@ -175,32 +266,40 @@ class BackupGenerator:
def zip_files(self):
# For backwards compatibility - pre v13
click.secho("BackupGenerator.zip_files has been deprecated in favour of BackupGenerator.backup_files", fg="yellow")
click.secho(
"BackupGenerator.zip_files has been deprecated in favour of"
" BackupGenerator.backup_files",
fg="yellow",
)
return self.backup_files()
def get_summary(self):
summary = {
"config": {
"path": self.backup_path_conf,
"size": get_file_size(self.backup_path_conf, format=True)
"size": get_file_size(self.backup_path_conf, format=True),
},
"database": {
"path": self.backup_path_db,
"size": get_file_size(self.backup_path_db, format=True)
}
"size": get_file_size(self.backup_path_db, format=True),
},
}
if os.path.exists(self.backup_path_files) and os.path.exists(self.backup_path_private_files):
summary.update({
"public": {
"path": self.backup_path_files,
"size": get_file_size(self.backup_path_files, format=True)
},
"private": {
"path": self.backup_path_private_files,
"size": get_file_size(self.backup_path_private_files, format=True)
if os.path.exists(self.backup_path_files) and os.path.exists(
self.backup_path_private_files
):
summary.update(
{
"public": {
"path": self.backup_path_files,
"size": get_file_size(self.backup_path_files, format=True),
},
"private": {
"path": self.backup_path_private_files,
"size": get_file_size(self.backup_path_private_files, format=True),
},
}
})
)
return summary
@ -208,21 +307,29 @@ class BackupGenerator:
backup_summary = self.get_summary()
print("Backup Summary for {0} at {1}".format(frappe.local.site, now()))
title = max([len(x) for x in backup_summary])
path = max([len(x["path"]) for x in backup_summary.values()])
for _type, info in backup_summary.items():
print("{0:8}: {1:85} {2}".format(_type.title(), info["path"], info["size"]))
template = "{{0:{0}}}: {{1:{1}}} {{2}}".format(title, path)
print(template.format(_type.title(), info["path"], info["size"]))
def backup_files(self):
import subprocess
for folder in ("public", "private"):
files_path = frappe.get_site_path(folder, "files")
backup_path = self.backup_path_files if folder=="public" else self.backup_path_private_files
backup_path = (
self.backup_path_files if folder == "public" else self.backup_path_private_files
)
if self.compress_files:
cmd_string = "tar cf - {1} | gzip > {0}"
else:
cmd_string = "tar -cf {0} {1}"
output = subprocess.check_output(cmd_string.format(backup_path, files_path), shell=True)
output = subprocess.check_output(
cmd_string.format(backup_path, files_path), shell=True
)
if self.verbose and output:
print(output.decode("utf8"))
@ -236,34 +343,114 @@ class BackupGenerator:
def take_dump(self):
import frappe.utils
from frappe.utils.change_log import get_app_branch
db_exc = {
"mariadb": ("mysqldump", which("mysqldump")),
"postgres": ("pg_dump", which("pg_dump")),
}[self.db_type]
gzip_exc = which("gzip")
if not (gzip_exc and db_exc[1]):
_exc = "gzip" if not gzip_exc else db_exc[0]
frappe.throw(
f"{_exc} not found in PATH! This is required to take a backup.",
exc=frappe.ExecutableNotFound
)
db_exc = db_exc[0]
database_header_content = [
f"Backup generated by Frappe {frappe.__version__} on branch {get_app_branch('frappe') or 'N/A'}",
"",
]
# escape reserved characters
args = dict([item[0], frappe.utils.esc(str(item[1]), '$ ')]
for item in self.__dict__.copy().items())
args = frappe._dict(
[item[0], frappe.utils.esc(str(item[1]), "$ ")]
for item in self.__dict__.copy().items()
)
cmd_string = """mysqldump --single-transaction --quick --lock-tables=false -u %(user)s -p%(password)s %(db_name)s -h %(db_host)s -P %(db_port)s | gzip > %(backup_path_db)s """ % args
if self.backup_includes:
backup_info = ("Backing Up Tables: ", ", ".join(self.backup_includes))
elif self.backup_excludes:
backup_info = ("Skipping Tables: ", ", ".join(self.backup_excludes))
if self.db_type == 'postgres':
cmd_string = "pg_dump postgres://{user}:{password}@{db_host}:{db_port}/{db_name} | gzip > {backup_path_db}".format(
user=args.get('user'),
password=args.get('password'),
db_host=args.get('db_host'),
db_port=args.get('db_port'),
db_name=args.get('db_name'),
backup_path_db=args.get('backup_path_db')
if self.partial:
print(''.join(backup_info), "\n")
database_header_content.extend([
f"Partial Backup of Frappe Site {frappe.local.site}",
("Backup contains: " if self.backup_includes else "Backup excludes: ") + backup_info[1],
"",
])
generated_header = "\n".join([f"-- {x}" for x in database_header_content]) + "\n"
with gzip.open(args.backup_path_db, "wt") as f:
f.write(generated_header)
if self.db_type == "postgres":
if self.backup_includes:
args["include"] = " ".join(
["--table='public.\"{0}\"'".format(table) for table in self.backup_includes]
)
elif self.backup_excludes:
args["exclude"] = " ".join(
["--exclude-table-data='public.\"{0}\"'".format(table) for table in self.backup_excludes]
)
cmd_string = (
"{db_exc} postgres://{user}:{password}@{db_host}:{db_port}/{db_name}"
" {include} {exclude} | {gzip} >> {backup_path_db}"
)
err, out = frappe.utils.execute_in_shell(cmd_string)
else:
if self.backup_includes:
args["include"] = " ".join(["'{0}'".format(x) for x in self.backup_includes])
elif self.backup_excludes:
args["exclude"] = " ".join(
[
"--ignore-table='{0}.{1}'".format(frappe.conf.db_name, table)
for table in self.backup_excludes
]
)
cmd_string = (
"{db_exc} --single-transaction --quick --lock-tables=false -u {user}"
" -p{password} {db_name} -h {db_host} -P {db_port} {include} {exclude}"
" | {gzip} >> {backup_path_db}"
)
command = cmd_string.format(
user=args.user,
password=args.password,
db_exc=db_exc,
db_host=args.db_host,
db_port=args.db_port,
db_name=args.db_name,
backup_path_db=args.backup_path_db,
exclude=args.get("exclude", ""),
include=args.get("include", ""),
gzip=gzip_exc,
)
if self.verbose:
print(command + "\n")
err, out = frappe.utils.execute_in_shell(command)
def send_email(self):
"""
Sends the link to backup file located at erpnext/backups
Sends the link to backup file located at erpnext/backups
"""
from frappe.email import get_system_managers
recipient_list = get_system_managers()
db_backup_url = get_url(os.path.join('backups', os.path.basename(self.backup_path_db)))
files_backup_url = get_url(os.path.join('backups', os.path.basename(self.backup_path_files)))
db_backup_url = get_url(
os.path.join("backups", os.path.basename(self.backup_path_db))
)
files_backup_url = get_url(
os.path.join("backups", os.path.basename(self.backup_path_files))
)
msg = """Hello,
@ -275,11 +462,13 @@ Your backups are ready to be downloaded.
This link will be valid for 24 hours. A new backup will be available for
download only after 24 hours.""" % {
"db_backup_url": db_backup_url,
"files_backup_url": files_backup_url
"files_backup_url": files_backup_url,
}
datetime_str = datetime.fromtimestamp(os.stat(self.backup_path_db).st_ctime)
subject = datetime_str.strftime("%d/%m/%Y %H:%M:%S") + """ - Backup ready to be downloaded"""
subject = (
datetime_str.strftime("%d/%m/%Y %H:%M:%S") + """ - Backup ready to be downloaded"""
)
frappe.sendmail(recipients=recipient_list, msg=msg, subject=subject)
return recipient_list
@ -288,20 +477,29 @@ download only after 24 hours.""" % {
@frappe.whitelist()
def get_backup():
"""
This function is executed when the user clicks on
Toos > Download Backup
This function is executed when the user clicks on
Toos > Download Backup
"""
delete_temp_backups()
odb = BackupGenerator(frappe.conf.db_name, frappe.conf.db_name,\
frappe.conf.db_password, db_host = frappe.db.host,\
db_type=frappe.conf.db_type, db_port=frappe.conf.db_port)
odb = BackupGenerator(
frappe.conf.db_name,
frappe.conf.db_name,
frappe.conf.db_password,
db_host=frappe.db.host,
db_type=frappe.conf.db_type,
db_port=frappe.conf.db_port,
)
odb.get_backup()
recipient_list = odb.send_email()
frappe.msgprint(_("Download link for your backup will be emailed on the following email address: {0}").format(', '.join(recipient_list)))
frappe.msgprint(
_(
"Download link for your backup will be emailed on the following email address: {0}"
).format(", ".join(recipient_list))
)
@frappe.whitelist()
def fetch_latest_backups():
def fetch_latest_backups(partial=False):
"""Fetches paths of the latest backup taken in the last 30 days
Only for: System Managers
@ -317,43 +515,88 @@ def fetch_latest_backups():
db_type=frappe.conf.db_type,
db_port=frappe.conf.db_port,
)
database, public, private, config = odb.get_recent_backup(older_than=24 * 30)
database, public, private, config = odb.get_recent_backup(older_than=24 * 30, partial=partial)
return {
"database": database,
"public": public,
"private": private,
"config": config
}
return {"database": database, "public": public, "private": private, "config": config}
def scheduled_backup(older_than=6, ignore_files=False, backup_path=None, backup_path_db=None, backup_path_files=None, backup_path_private_files=None, backup_path_conf=None, force=False, verbose=False, compress=False):
def scheduled_backup(
older_than=6,
ignore_files=False,
backup_path=None,
backup_path_db=None,
backup_path_files=None,
backup_path_private_files=None,
backup_path_conf=None,
ignore_conf=False,
include_doctypes="",
exclude_doctypes="",
compress=False,
force=False,
verbose=False,
):
"""this function is called from scheduler
deletes backups older than 7 days
takes backup"""
odb = new_backup(older_than, ignore_files, backup_path=backup_path, backup_path_db=backup_path_db, backup_path_files=backup_path_files, backup_path_private_files=backup_path_private_files, backup_path_conf=backup_path_conf, force=force, verbose=verbose, compress=compress)
deletes backups older than 7 days
takes backup"""
odb = new_backup(
older_than=older_than,
ignore_files=ignore_files,
backup_path=backup_path,
backup_path_db=backup_path_db,
backup_path_files=backup_path_files,
backup_path_private_files=backup_path_private_files,
backup_path_conf=backup_path_conf,
ignore_conf=ignore_conf,
include_doctypes=include_doctypes,
exclude_doctypes=exclude_doctypes,
compress=compress,
force=force,
verbose=verbose,
)
return odb
def new_backup(older_than=6, ignore_files=False, backup_path=None, backup_path_db=None, backup_path_files=None, backup_path_private_files=None, backup_path_conf=None, force=False, verbose=False, compress=False):
delete_temp_backups(older_than = frappe.conf.keep_backups_for_hours or 24)
odb = BackupGenerator(frappe.conf.db_name, frappe.conf.db_name,\
frappe.conf.db_password,
backup_path=backup_path,
backup_path_db=backup_path_db,
backup_path_files=backup_path_files,
backup_path_private_files=backup_path_private_files,
backup_path_conf=backup_path_conf,
db_host = frappe.db.host,
db_port = frappe.db.port,
db_type = frappe.conf.db_type,
verbose=verbose,
compress_files=compress)
def new_backup(
older_than=6,
ignore_files=False,
backup_path=None,
backup_path_db=None,
backup_path_files=None,
backup_path_private_files=None,
backup_path_conf=None,
ignore_conf=False,
include_doctypes="",
exclude_doctypes="",
compress=False,
force=False,
verbose=False,
):
delete_temp_backups(older_than=frappe.conf.keep_backups_for_hours or 24)
odb = BackupGenerator(
frappe.conf.db_name,
frappe.conf.db_name,
frappe.conf.db_password,
db_host=frappe.db.host,
db_port=frappe.db.port,
db_type=frappe.conf.db_type,
backup_path=backup_path,
backup_path_db=backup_path_db,
backup_path_files=backup_path_files,
backup_path_private_files=backup_path_private_files,
backup_path_conf=backup_path_conf,
ignore_conf=ignore_conf,
include_doctypes=include_doctypes,
exclude_doctypes=exclude_doctypes,
verbose=verbose,
compress_files=compress,
)
odb.get_backup(older_than, ignore_files, force=force)
return odb
def delete_temp_backups(older_than=24):
"""
Cleans up the backup_link_path directory by deleting files older than 24 hours
Cleans up the backup_link_path directory by deleting files older than 24 hours
"""
backup_path = get_backup_path()
if os.path.exists(backup_path):
@ -363,54 +606,68 @@ def delete_temp_backups(older_than=24):
if is_file_old(this_file_path, older_than):
os.remove(this_file_path)
def is_file_old(db_file_name, older_than=24):
"""
Checks if file exists and is older than specified hours
Returns ->
True: file does not exist or file is old
False: file is new
"""
if os.path.isfile(db_file_name):
from datetime import timedelta
#Get timestamp of the file
file_datetime = datetime.fromtimestamp\
(os.stat(db_file_name).st_ctime)
if datetime.today() - file_datetime >= timedelta(hours = older_than):
if _verbose:
print("File is old")
return True
else:
if _verbose:
print("File is recent")
return False
def is_file_old(file_path, older_than=24):
"""
Checks if file exists and is older than specified hours
Returns ->
True: file does not exist or file is old
False: file is new
"""
if os.path.isfile(file_path):
from datetime import timedelta
# Get timestamp of the file
file_datetime = datetime.fromtimestamp(os.stat(file_path).st_ctime)
if datetime.today() - file_datetime >= timedelta(hours=older_than):
if _verbose:
print(f"File {file_path} is older than {older_than} hours")
return True
else:
if _verbose:
print("File does not exist")
return True
print(f"File {file_path} is recent")
return False
else:
if _verbose:
print(f"File {file_path} does not exist")
return True
def get_backup_path():
backup_path = frappe.utils.get_site_path(conf.get("backup_path", "private/backups"))
return backup_path
def backup(with_files=False, backup_path_db=None, backup_path_files=None, backup_path_private_files=None, backup_path_conf=None, quiet=False):
def backup(
with_files=False,
backup_path_db=None,
backup_path_files=None,
backup_path_private_files=None,
backup_path_conf=None,
quiet=False,
):
"Backup"
odb = scheduled_backup(ignore_files=not with_files, backup_path_db=backup_path_db, backup_path_files=backup_path_files, backup_path_private_files=backup_path_private_files, backup_path_conf=backup_path_conf, force=True)
odb = scheduled_backup(
ignore_files=not with_files,
backup_path_db=backup_path_db,
backup_path_files=backup_path_files,
backup_path_private_files=backup_path_private_files,
backup_path_conf=backup_path_conf,
force=True,
)
return {
"backup_path_db": odb.backup_path_db,
"backup_path_files": odb.backup_path_files,
"backup_path_private_files": odb.backup_path_private_files
"backup_path_private_files": odb.backup_path_private_files,
}
if __name__ == "__main__":
"""
is_file_old db_name user password db_host db_type db_port
get_backup db_name user password db_host db_type db_port
"""
import sys
cmd = sys.argv[1]
db_type = 'mariadb'
db_type = "mariadb"
try:
db_type = sys.argv[6]
except IndexError:
@ -423,19 +680,47 @@ if __name__ == "__main__":
pass
if cmd == "is_file_old":
odb = BackupGenerator(sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5] or "localhost", db_type=db_type, db_port=db_port)
odb = BackupGenerator(
sys.argv[2],
sys.argv[3],
sys.argv[4],
sys.argv[5] or "localhost",
db_type=db_type,
db_port=db_port,
)
is_file_old(odb.db_file_name)
if cmd == "get_backup":
odb = BackupGenerator(sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5] or "localhost", db_type=db_type, db_port=db_port)
odb = BackupGenerator(
sys.argv[2],
sys.argv[3],
sys.argv[4],
sys.argv[5] or "localhost",
db_type=db_type,
db_port=db_port,
)
odb.get_backup()
if cmd == "take_dump":
odb = BackupGenerator(sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5] or "localhost", db_type=db_type, db_port=db_port)
odb = BackupGenerator(
sys.argv[2],
sys.argv[3],
sys.argv[4],
sys.argv[5] or "localhost",
db_type=db_type,
db_port=db_port,
)
odb.take_dump()
if cmd == "send_email":
odb = BackupGenerator(sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5] or "localhost", db_type=db_type, db_port=db_port)
odb = BackupGenerator(
sys.argv[2],
sys.argv[3],
sys.argv[4],
sys.argv[5] or "localhost",
db_type=db_type,
db_port=db_port,
)
odb.send_email("abc.sql.gz")
if cmd == "delete_temp_backups":

4
frappe/utils/goal.py
View file

@ -57,6 +57,10 @@ def get_monthly_goal_graph_data(title, doctype, docname, goal_value_field, goal_
from frappe.utils.formatters import format_value
import json
# should have atleast read perm
if not frappe.has_permission(goal_doctype):
return None
meta = frappe.get_meta(doctype)
doc = frappe.get_doc(doctype, docname)

2
frappe/utils/html_utils.py
View file

@ -34,7 +34,7 @@ def clean_email_html(html):
'margin', 'margin-top', 'margin-bottom', 'margin-left', 'margin-right',
'padding', 'padding-top', 'padding-bottom', 'padding-left', 'padding-right',
'font-size', 'font-weight', 'font-family', 'text-decoration',
'line-height', 'text-align', 'vertical-align'
'line-height', 'text-align', 'vertical-align', 'display'
],
protocols=['cid', 'http', 'https', 'mailto', 'data'],
strip=True, strip_comments=True)

3
frappe/utils/safe_exec.py
View file

@ -277,5 +277,6 @@ VALID_UTILS = (
"to_markdown",
"md_to_html",
"is_subset",
"generate_hash"
"generate_hash",
"formatdate"
)

2
frappe/website/doctype/web_form/web_form.js
View file

@ -59,6 +59,8 @@ frappe.ui.form.on("Web Form", {
default: field.default,
read_only: field.read_only,
depends_on: field.depends_on,
mandatory_depends_on: field.mandatory_depends_on,
read_only_depends_on: field.read_only_depends_on,
hidden: field.hidden,
description: field.description
});

27
frappe/website/doctype/web_form_field/web_form_field.json
View file

@ -18,6 +18,10 @@
"options",
"max_length",
"max_value",
"property_depends_on_section",
"mandatory_depends_on",
"column_break_16",
"read_only_depends_on",
"section_break_6",
"description",
"column_break_8",
@ -117,11 +121,32 @@
"fieldname": "default",
"fieldtype": "Data",
"label": "Default"
},
{
"fieldname": "property_depends_on_section",
"fieldtype": "Section Break",
"label": "Property Depends On"
},
{
"fieldname": "mandatory_depends_on",
"fieldtype": "Code",
"label": "Mandatory Depends On",
"options": "JS"
},
{
"fieldname": "column_break_16",
"fieldtype": "Column Break"
},
{
"fieldname": "read_only_depends_on",
"fieldtype": "Code",
"label": "Read Only Depends On",
"options": "JS"
}
],
"istable": 1,
"links": [],
"modified": "2020-05-13 13:35:08.454427",
"modified": "2020-11-10 23:20:44.354862",
"modified_by": "Administrator",
"module": "Website",
"name": "Web Form Field",

3
frappe/website/router.py
View file

@ -275,8 +275,7 @@ def get_page_info(path, app, start, basepath=None, app_path=None, fname=None):
# extract properties from controller attributes
load_properties_from_controller(page_info)
# if not page_info.title:
# print('no-title-for', page_info.route)
page_info.build_version = frappe.utils.get_build_version()
return page_info

10
frappe/www/desk.py
View file

@ -36,7 +36,7 @@ def get_context(context):
context.update({
"no_cache": 1,
"build_version": get_build_version(),
"build_version": frappe.utils.get_build_version(),
"include_js": hooks["app_include_js"],
"include_css": hooks["app_include_css"],
"sounds": hooks["sounds"],
@ -82,11 +82,3 @@ def get_desk_assets(build_version):
"boot": data["boot"],
"assets": assets
}
def get_build_version():
try:
return str(os.path.getmtime(os.path.join(frappe.local.sites_path, '.build')))
except OSError:
# .build can sometimes not exist
# this is not a major problem so send fallback
return frappe.utils.random_string(8)

2
requirements.txt
View file

@ -72,7 +72,5 @@ zxcvbn-python==4.4.24
pycryptodome==3.9.8
paytmchecksum==1.7.0
wrapt==1.10.11
twilio==6.44.2
razorpay==1.2.0
rsa>=4.1 # not directly required, pinned by Snyk to avoid a vulnerability