vassili/seitime-frappe
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Merge branch 'develop' of github.com:frappe/frappe into fix-redis-dependency-during-setup-1

Browse source
This commit is contained in:
Gavin D'souza 2020-12-15 11:52:53 +05:30
commit 49b9540f4e
102 changed files with 2345 additions and 1041 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
.editorconfig Normal file
View file

@ -0,0 +1,14 @@
# Root editor config file
root = true
# Common settings
[*]
end_of_line = lf
insert_final_newline = true
trim_trailing_whitespace = true
charset = utf-8
# python, js indentation settings
[{*.py,*.js}]
indent_style = tab
indent_size = 4

5
.github/ISSUE_TEMPLATE/config.yml vendored Normal file
View file

@ -0,0 +1,5 @@
blank_issues_enabled: false
contact_links:
- name: Community Forum
url: https://discuss.erpnext.com/
about: For general QnA, discussions and community help.

4
.github/helper/documentation.py vendored
View file

@ -21,8 +21,8 @@ def docs_link_exists(body):
if word.startswith('http') and uri_validator(word):
parsed_url = urlparse(word)
if parsed_url.netloc == "github.com":
_, org, repo, _type, ref = parsed_url.path.split('/')
if org == "frappe" and repo in docs_repos:
parts = parsed_url.path.split('/')
if len(parts) == 5 and parts[1] == "frappe" and parts[2] in docs_repos:
return True

6
frappe/__init__.py
View file

@ -945,7 +945,11 @@ def get_installed_apps(sort=False, frappe_last=False):
connect()
if not local.all_apps:
local.all_apps = get_all_apps(True)
local.all_apps = cache().get_value('all_apps', get_all_apps)
#cache bench apps
if not cache().get_value('all_apps'):
cache().set_value('all_apps', local.all_apps)
installed = json.loads(db.get_global("installed_apps") or "[]")

14
frappe/automation/doctype/auto_repeat/auto_repeat.js
View file

@ -44,6 +44,20 @@ frappe.ui.form.on('Auto Repeat', {
// auto repeat schedule
frappe.auto_repeat.render_schedule(frm);
frm.trigger('toggle_submit_on_creation');
},
reference_doctype: function(frm) {
frm.trigger('toggle_submit_on_creation');
},
toggle_submit_on_creation: function(frm) {
// submit on creation checkbox
frappe.model.with_doctype(frm.doc.reference_doctype, () => {
let meta = frappe.get_meta(frm.doc.reference_doctype);
frm.toggle_display('submit_on_creation', meta.is_submittable);
});
},
template: function(frm) {

11
frappe/automation/doctype/auto_repeat/auto_repeat.json
View file

@ -1,4 +1,5 @@
{
"actions": [],
"allow_import": 1,
"allow_rename": 1,
"autoname": "format:AUT-AR-{#####}",
@ -12,6 +13,7 @@
"section_break_3",
"reference_doctype",
"reference_document",
"submit_on_creation",
"column_break_5",
"start_date",
"end_date",
@ -186,9 +188,16 @@
"fieldname": "repeat_on_last_day",
"fieldtype": "Check",
"label": "Repeat on Last Day of the Month"
},
{
"default": "0",
"fieldname": "submit_on_creation",
"fieldtype": "Check",
"label": "Submit on Creation"
}
],
"modified": "2019-07-17 11:30:51.412317",
"links": [],
"modified": "2020-12-10 10:43:13.449172",
"modified_by": "Administrator",
"module": "Automation",
"name": "Auto Repeat",

11
frappe/automation/doctype/auto_repeat/auto_repeat.py
View file

@ -21,6 +21,7 @@ class AutoRepeat(Document):
def validate(self):
self.update_status()
self.validate_reference_doctype()
self.validate_submit_on_creation()
self.validate_dates()
self.validate_email_id()
self.set_dates()
@ -60,6 +61,11 @@ class AutoRepeat(Document):
if not frappe.get_meta(self.reference_doctype).allow_auto_repeat:
frappe.throw(_("Enable Allow Auto Repeat for the doctype {0} in Customize Form").format(self.reference_doctype))
def validate_submit_on_creation(self):
if self.submit_on_creation and not frappe.get_meta(self.reference_doctype).is_submittable:
frappe.throw(_('Cannot enable {0} for a non-submittable doctype').format(
frappe.bold('Submit on Creation')))
def validate_dates(self):
if frappe.flags.in_patch:
return
@ -150,6 +156,9 @@ class AutoRepeat(Document):
self.update_doc(new_doc, reference_doc)
new_doc.insert(ignore_permissions = True)
if self.submit_on_creation:
new_doc.submit()
return new_doc
def update_doc(self, new_doc, reference_doc):
@ -160,7 +169,7 @@ class AutoRepeat(Document):
if new_doc.meta.get_field('auto_repeat'):
new_doc.set('auto_repeat', self.name)
for fieldname in ['naming_series', 'ignore_pricing_rule', 'posting_time', 'select_print_heading', 'remarks', 'owner']:
for fieldname in ['naming_series', 'ignore_pricing_rule', 'posting_time', 'select_print_heading', 'user_remark', 'remarks', 'owner']:
if new_doc.meta.get_field(fieldname):
new_doc.set(fieldname, reference_doc.get(fieldname))

51
frappe/automation/doctype/auto_repeat/test_auto_repeat.py
View file

@ -111,6 +111,25 @@ class TestAutoRepeat(unittest.TestCase):
doc = make_auto_repeat(frequency='Daily', reference_document=todo.name, start_date=add_days(today(), -2))
self.assertEqual(getdate(doc.next_schedule_date), current_date)
def test_submit_on_creation(self):
doctype = 'Test Submittable DocType'
create_submittable_doctype(doctype)
current_date = getdate()
submittable_doc = frappe.get_doc(dict(doctype=doctype, test='test submit on creation')).insert()
submittable_doc.submit()
doc = make_auto_repeat(frequency='Daily', reference_doctype=doctype, reference_document=submittable_doc.name,
start_date=add_days(current_date, -1), submit_on_creation=1)
data = get_auto_repeat_entries(current_date)
create_repeated_entries(data)
docnames = frappe.db.get_all(doc.reference_doctype,
filters={'auto_repeat': doc.name},
fields=['docstatus'],
limit=1
)
self.assertEquals(docnames[0].docstatus, 1)
def make_auto_repeat(**args):
args = frappe._dict(args)
@ -118,6 +137,7 @@ def make_auto_repeat(**args):
'doctype': 'Auto Repeat',
'reference_doctype': args.reference_doctype or 'ToDo',
'reference_document': args.reference_document or frappe.db.get_value('ToDo', 'name'),
'submit_on_creation': args.submit_on_creation or 0,
'frequency': args.frequency or 'Daily',
'start_date': args.start_date or add_days(today(), -1),
'end_date': args.end_date or "",
@ -128,3 +148,34 @@ def make_auto_repeat(**args):
}).insert(ignore_permissions=True)
return doc
def create_submittable_doctype(doctype):
if frappe.db.exists('DocType', doctype):
return
else:
doc = frappe.get_doc({
'doctype': 'DocType',
'__newname': doctype,
'module': 'Custom',
'custom': 1,
'is_submittable': 1,
'fields': [{
'fieldname': 'test',
'label': 'Test',
'fieldtype': 'Data'
}],
'permissions': [{
'role': 'System Manager',
'read': 1,
'write': 1,
'create': 1,
'delete': 1,
'submit': 1,
'cancel': 1,
'amend': 1
}]
}).insert()
doc.allow_auto_repeat = 1
doc.save()

2
frappe/build.py
View file

@ -105,7 +105,7 @@ def download_frappe_assets(verbose=True):
if frappe_head:
try:
url = get_assets_link(frappe_head)
click.secho("Retreiving assets...", fg="yellow")
click.secho("Retrieving assets...", fg="yellow")
prefix = mkdtemp(prefix="frappe-assets-", suffix=frappe_head)
assets_archive = download_file(url, prefix)
print("\n{0} Downloaded Frappe assets from {1}".format(green('✔'), url))

30
frappe/chat/util/util.py
View file

@ -1,27 +1,21 @@
from __future__ import unicode_literals
# imports - standard imports
import json
from collections.abc import MutableMapping, MutableSequence, Sequence
# imports - third-party imports
import requests
# imports - compatibility imports
import six
# imports - standard imports
from collections import Sequence, MutableSequence, Mapping, MutableMapping
if six.PY2:
from urlparse import urlparse # PY2
else:
from urllib.parse import urlparse # PY3
import json
from urllib.parse import urlparse
# imports - module imports
from frappe.model.document import Document
from frappe.exceptions import DuplicateEntryError
from frappe import _dict
import frappe
from frappe.exceptions import DuplicateEntryError
from frappe.model.document import Document
session = frappe.session
def get_user_doc(user = None):
if isinstance(user, Document):
return user
@ -38,12 +32,12 @@ def squashify(what):
return what
def safe_json_loads(*args):
results = [ ]
results = []
for arg in args:
try:
arg = json.loads(arg)
except Exception as e:
except Exception:
pass
results.append(arg)
@ -81,7 +75,7 @@ def dictify(arg):
for i, a in enumerate(arg):
arg[i] = dictify(a)
elif isinstance(arg, MutableMapping):
arg = _dict(arg)
arg = frappe._dict(arg)
return arg
@ -113,4 +107,4 @@ def get_emojis():
emojis = resp.json()
redis.hset('frappe_emojis', 'emojis', emojis)
return dictify(emojis)
return dictify(emojis)

166
frappe/commands/site.py
View file

@ -9,7 +9,7 @@ import click
import frappe
from frappe.commands import get_site, pass_context
from frappe.exceptions import SiteNotSpecifiedError
from frappe.utils import get_site_path, touch_file
from frappe.installer import _new_site
@click.command('new-site')
@ -42,57 +42,6 @@ def new_site(site, mariadb_root_username=None, mariadb_root_password=None, admin
if len(frappe.utils.get_sites()) == 1:
use(site)
def _new_site(db_name, site, mariadb_root_username=None, mariadb_root_password=None,
admin_password=None, verbose=False, install_apps=None, source_sql=None, force=False,
no_mariadb_socket=False, reinstall=False, db_password=None, db_type=None, db_host=None,
db_port=None, new_site=False):
"""Install a new Frappe site"""
if not force and os.path.exists(site):
print('Site {0} already exists'.format(site))
sys.exit(1)
if no_mariadb_socket and not db_type == "mariadb":
print('--no-mariadb-socket requires db_type to be set to mariadb.')
sys.exit(1)
if not db_name:
import hashlib
db_name = '_' + hashlib.sha1(site.encode()).hexdigest()[:16]
from frappe.commands.scheduler import _is_scheduler_enabled
from frappe.installer import install_db, make_site_dirs
from frappe.installer import install_app as _install_app
import frappe.utils.scheduler
frappe.init(site=site)
try:
# enable scheduler post install?
enable_scheduler = _is_scheduler_enabled()
except Exception:
enable_scheduler = False
make_site_dirs()
installing = touch_file(get_site_path('locks', 'installing.lock'))
install_db(root_login=mariadb_root_username, root_password=mariadb_root_password, db_name=db_name,
admin_password=admin_password, verbose=verbose, source_sql=source_sql, force=force, reinstall=reinstall,
db_password=db_password, db_type=db_type, db_host=db_host, db_port=db_port, no_mariadb_socket=no_mariadb_socket)
apps_to_install = ['frappe'] + (frappe.conf.get("install_apps") or []) + (list(install_apps) or [])
for app in apps_to_install:
_install_app(app, verbose=verbose, set_as_patched=not source_sql)
os.remove(installing)
frappe.utils.scheduler.toggle_scheduler(enable_scheduler)
frappe.db.commit()
scheduler_status = "disabled" if frappe.utils.scheduler.is_scheduler_disabled() else "enabled"
print("*** Scheduler is", scheduler_status, "***")
@click.command('restore')
@click.argument('sql-file-path')
@ -107,33 +56,41 @@ def _new_site(db_name, site, mariadb_root_username=None, mariadb_root_password=N
@pass_context
def restore(context, sql_file_path, mariadb_root_username=None, mariadb_root_password=None, db_name=None, verbose=None, install_app=None, admin_password=None, force=None, with_public_files=None, with_private_files=None):
"Restore site database from an sql file"
from frappe.installer import extract_sql_gzip, extract_files, is_downgrade, validate_database_sql
from frappe.installer import (
extract_sql_from_archive,
extract_files,
is_downgrade,
is_partial,
validate_database_sql
)
force = context.force or force
decompressed_file_name = extract_sql_from_archive(sql_file_path)
# Extract the gzip file if user has passed *.sql.gz file instead of *.sql file
if not os.path.exists(sql_file_path):
base_path = '..'
sql_file_path = os.path.join(base_path, sql_file_path)
if not os.path.exists(sql_file_path):
print('Invalid path {0}'.format(sql_file_path[3:]))
sys.exit(1)
elif sql_file_path.startswith(os.sep):
base_path = os.sep
else:
base_path = '.'
if sql_file_path.endswith('sql.gz'):
decompressed_file_name = extract_sql_gzip(os.path.abspath(sql_file_path))
else:
decompressed_file_name = sql_file_path
# check if partial backup
if is_partial(decompressed_file_name):
click.secho(
"Partial Backup file detected. You cannot use a partial file to restore a Frappe Site.",
fg="red"
)
click.secho(
"Use `bench partial-restore` to restore a partial backup to an existing site.",
fg="yellow"
)
sys.exit(1)
# check if valid SQL file
validate_database_sql(decompressed_file_name, _raise=not force)
site = get_site(context)
frappe.init(site=site)
# dont allow downgrading to older versions of frappe without force
if not force and is_downgrade(decompressed_file_name, verbose=True):
warn_message = "This is not recommended and may lead to unexpected behaviour. Do you want to continue anyway?"
warn_message = (
"This is not recommended and may lead to unexpected behaviour. "
"Do you want to continue anyway?"
)
click.confirm(warn_message, abort=True)
_new_site(frappe.conf.db_name, site, mariadb_root_username=mariadb_root_username,
@ -143,22 +100,39 @@ def restore(context, sql_file_path, mariadb_root_username=None, mariadb_root_pas
# Extract public and/or private files to the restored site, if user has given the path
if with_public_files:
with_public_files = os.path.join(base_path, with_public_files)
public = extract_files(site, with_public_files, 'public')
public = extract_files(site, with_public_files)
os.remove(public)
if with_private_files:
with_private_files = os.path.join(base_path, with_private_files)
private = extract_files(site, with_private_files, 'private')
private = extract_files(site, with_private_files)
os.remove(private)
# Removing temporarily created file
if decompressed_file_name != sql_file_path:
os.remove(decompressed_file_name)
success_message = "Site {0} has been restored{1}".format(site, " with files" if (with_public_files or with_private_files) else "")
success_message = "Site {0} has been restored{1}".format(
site,
" with files" if (with_public_files or with_private_files) else ""
)
click.secho(success_message, fg="green")
@click.command('partial-restore')
@click.argument('sql-file-path')
@click.option("--verbose", "-v", is_flag=True)
@pass_context
def partial_restore(context, sql_file_path, verbose):
from frappe.installer import partial_restore
verbose = context.verbose or verbose
site = get_site(context)
frappe.init(site=site)
frappe.connect(site=site)
partial_restore(sql_file_path, verbose)
frappe.destroy()
@click.command('reinstall')
@click.option('--admin-password', help='Administrator Password for reinstalled site')
@click.option('--mariadb-root-username', help='Root username for MariaDB')
@ -416,16 +390,20 @@ def use(site, sites_path='.'):
@click.command('backup')
@click.option('--with-files', default=False, is_flag=True, help="Take backup with files")
@click.option('--include', '--only', '-i', default="", type=str, help="Specify the DocTypes to backup seperated by commas")
@click.option('--exclude', '-e', default="", type=str, help="Specify the DocTypes to not backup seperated by commas")
@click.option('--backup-path', default=None, help="Set path for saving all the files in this operation")
@click.option('--backup-path-db', default=None, help="Set path for saving database file")
@click.option('--backup-path-files', default=None, help="Set path for saving public file")
@click.option('--backup-path-private-files', default=None, help="Set path for saving private file")
@click.option('--backup-path-conf', default=None, help="Set path for saving config file")
@click.option('--ignore-backup-conf', default=False, is_flag=True, help="Ignore excludes/includes set in config")
@click.option('--verbose', default=False, is_flag=True, help="Add verbosity")
@click.option('--compress', default=False, is_flag=True, help="Compress private and public files")
@pass_context
def backup(context, with_files=False, backup_path=None, backup_path_db=None, backup_path_files=None,
backup_path_private_files=None, backup_path_conf=None, verbose=False, compress=False):
backup_path_private_files=None, backup_path_conf=None, ignore_backup_conf=False, verbose=False,
compress=False, include="", exclude=""):
"Backup"
from frappe.utils.backups import scheduled_backup
verbose = verbose or context.verbose
@ -435,11 +413,27 @@ def backup(context, with_files=False, backup_path=None, backup_path_db=None, bac
try:
frappe.init(site=site)
frappe.connect()
odb = scheduled_backup(ignore_files=not with_files, backup_path=backup_path, backup_path_db=backup_path_db, backup_path_files=backup_path_files, backup_path_private_files=backup_path_private_files, backup_path_conf=backup_path_conf, force=True, verbose=verbose, compress=compress)
odb = scheduled_backup(
ignore_files=not with_files,
backup_path=backup_path,
backup_path_db=backup_path_db,
backup_path_files=backup_path_files,
backup_path_private_files=backup_path_private_files,
backup_path_conf=backup_path_conf,
ignore_conf=ignore_backup_conf,
include_doctypes=include,
exclude_doctypes=exclude,
compress=compress,
verbose=verbose,
force=True
)
except Exception:
click.secho("Backup failed for Site {0}. Database or site_config.json may be corrupted".format(site), fg="red")
if verbose:
print(frappe.get_traceback())
exit_code = 1
continue
odb.print_summary()
click.secho("Backup for Site {0} has been successfully completed{1}".format(site, " with files" if with_files else ""), fg="green")
frappe.destroy()
@ -512,13 +506,14 @@ def _drop_site(site, root_login='root', root_password=None, archived_sites_path=
if force:
pass
else:
click.echo("="*80)
click.echo("Error: The operation has stopped because backup of {s}'s database failed.".format(s=site))
click.echo("Reason: {reason}{sep}".format(reason=str(err), sep="\n"))
click.echo("Fix the issue and try again.")
click.echo(
"Hint: Use 'bench drop-site {s} --force' to force the removal of {s}".format(sep="\n", tab="\t", s=site)
)
messages = [
"=" * 80,
"Error: The operation has stopped because backup of {0}'s database failed.".format(site),
"Reason: {0}\n".format(str(err)),
"Fix the issue and try again.",
"Hint: Use 'bench drop-site {0} --force' to force the removal of {0}".format(site)
]
click.echo("\n".join(messages))
sys.exit(1)
drop_user_and_database(frappe.conf.db_name, root_login, root_password)
@ -734,5 +729,6 @@ commands = [
stop_recording,
add_to_hosts,
start_ngrok,
build_search_index
build_search_index,
partial_restore
]

8
frappe/core/doctype/doctype/doctype.py
View file

@ -394,7 +394,10 @@ class DocType(Document):
frappe.db.sql("""update tabSingles set value=%s
where doctype=%s and field='name' and value = %s""", (new, new, old))
else:
frappe.db.sql("rename table `tab%s` to `tab%s`" % (old, new))
frappe.db.multisql({
"mariadb": f"RENAME TABLE `tab{old}` TO `tab{new}`",
"postgres": f"ALTER TABLE `tab{old}` RENAME TO `tab{new}`"
})
def rename_files_and_folders(self, old, new):
# move files
@ -572,7 +575,8 @@ class DocType(Document):
def make_repeatable(self):
"""If allow_auto_repeat is set, add auto_repeat custom field."""
if self.allow_auto_repeat:
if not frappe.db.exists('Custom Field', {'fieldname': 'auto_repeat', 'dt': self.name}):
if not frappe.db.exists('Custom Field', {'fieldname': 'auto_repeat', 'dt': self.name}) and \
not frappe.db.exists('DocField', {'fieldname': 'auto_repeat', 'parent': self.name}):
insert_after = self.fields[len(self.fields) - 1].fieldname
df = dict(fieldname='auto_repeat', label='Auto Repeat', fieldtype='Link', options='Auto Repeat', insert_after=insert_after, read_only=1, no_copy=1, print_hide=1)
create_custom_field(self.name, df)

3
frappe/core/doctype/domain_settings/domain_settings.js
View file

@ -18,6 +18,9 @@ frappe.ui.form.on('Domain Settings', {
checked: active_domains.includes(domain)
};
});
},
on_change: () => {
frm.dirty();
}
},
render_input: true

44
frappe/core/doctype/file/file.py
View file

@ -30,7 +30,7 @@ import frappe
from frappe import _, conf
from frappe.model.document import Document
from frappe.utils import call_hook_method, cint, cstr, encode, get_files_path, get_hook_method, random_string, strip
from frappe.utils.image import strip_exif_data
class MaxFileSizeReachedError(frappe.ValidationError):
pass
@ -93,6 +93,7 @@ class File(Document):
self.set_is_private()
self.set_file_name()
self.validate_duplicate_entry()
self.validate_attachment_limit()
self.validate_folder()
if not self.file_url and not self.flags.ignore_file_validate:
@ -140,6 +141,26 @@ class File(Document):
if self.file_url and (self.is_private != self.file_url.startswith('/private')):
frappe.throw(_('Invalid file URL. Please contact System Administrator.'))
def validate_attachment_limit(self):
attachment_limit = 0
if self.attached_to_doctype and self.attached_to_name:
attachment_limit = cint(frappe.get_meta(self.attached_to_doctype).max_attachments)
if attachment_limit:
current_attachment_count = len(frappe.get_all('File', filters={
'attached_to_doctype': self.attached_to_doctype,
'attached_to_name': self.attached_to_name,
}, limit=attachment_limit + 1))
if current_attachment_count >= attachment_limit:
frappe.throw(
_("Maximum Attachment Limit of {0} has been reached for {1} {2}.").format(
frappe.bold(attachment_limit), self.attached_to_doctype, self.attached_to_name
),
exc=frappe.exceptions.AttachmentLimitReached,
title=_('Attachment Limit Reached')
)
def set_folder_name(self):
"""Make parent folders if not exists based on reference doctype and name"""
if self.attached_to_doctype and not self.folder:
@ -435,6 +456,7 @@ class File(Document):
def save_file(self, content=None, decode=False, ignore_existing_file_check=False):
file_exists = False
self.content = content
if decode:
if isinstance(content, text_type):
self.content = content.encode("utf-8")
@ -445,10 +467,19 @@ class File(Document):
if not self.is_private:
self.is_private = 0
self.file_size = self.check_max_file_size()
self.content_hash = get_content_hash(self.content)
self.content_type = mimetypes.guess_type(self.file_name)[0]
self.file_size = self.check_max_file_size()
if (
self.content_type and "image" in self.content_type
and frappe.get_system_settings("strip_exif_metadata_from_uploaded_images")
):
self.content = strip_exif_data(self.content, self.content_type)
self.content_hash = get_content_hash(self.content)
duplicate_file = None
# check if a file exists with the same content hash and is also in the same folder (public or private)
@ -612,7 +643,12 @@ def get_extension(filename, extn, content):
return extn
def get_local_image(file_url):
file_path = frappe.get_site_path("public", file_url.lstrip("/"))
if file_url.startswith("/private"):
file_url_path = (file_url.lstrip("/"), )
else:
file_url_path = ("public", file_url.lstrip("/"))
file_path = frappe.get_site_path(*file_url_path)
try:
image = Image.open(file_path)

25
frappe/core/doctype/file/test_file.py
View file

@ -160,6 +160,31 @@ class TestSameContent(unittest.TestCase):
def test_saved_content(self):
self.assertFalse(os.path.exists(get_files_path(self.dup_filename)))
def test_attachment_limit(self):
doctype, docname = make_test_doc()
from frappe.custom.doctype.property_setter.property_setter import make_property_setter
limit_property = make_property_setter('ToDo', None, 'max_attachments', 1, 'int', for_doctype=True)
file1 = frappe.get_doc({
"doctype": "File",
"file_name": 'test-attachment',
"attached_to_doctype": doctype,
"attached_to_name": docname,
"content": 'test'
})
file1.insert()
file2 = frappe.get_doc({
"doctype": "File",
"file_name": 'test-attachment',
"attached_to_doctype": doctype,
"attached_to_name": docname,
"content": 'test2'
})
self.assertRaises(frappe.exceptions.AttachmentLimitReached, file2.insert)
limit_property.delete()
frappe.clear_cache(doctype='ToDo')
def tearDown(self):
# File gets deleted on rollback, so blank

20
frappe/core/doctype/prepared_report/prepared_report.py
View file

@ -89,20 +89,18 @@ def delete_expired_prepared_reports():
'creation': ['<', frappe.utils.add_days(frappe.utils.now(), -expiry_period)]
})
args = {
'reports': prepared_reports_to_delete,
'limit': 50
}
enqueue(method=delete_prepared_reports, job_name="delete_prepared_reports", **args)
batches = frappe.utils.create_batch(prepared_reports_to_delete, 100)
for batch in batches:
args = {
'reports': batch,
}
enqueue(method=delete_prepared_reports, job_name="delete_prepared_reports", **args)
@frappe.whitelist()
def delete_prepared_reports(reports, limit=None):
def delete_prepared_reports(reports):
reports = frappe.parse_json(reports)
for index, doc in enumerate(reports):
if limit and index == limit:
return
frappe.delete_doc('Prepared Report', doc['name'], ignore_permissions=True)
for report in reports:
frappe.delete_doc('Prepared Report', report['name'], ignore_permissions=True)
def create_json_gz_file(data, dt, dn):
# Storing data in CSV file causes information loss

4
frappe/core/doctype/report_filter/report_filter.json
View file

@ -44,7 +44,7 @@
},
{
"fieldname": "options",
"fieldtype": "Data",
"fieldtype": "Small Text",
"label": "Options"
},
{
@ -58,7 +58,7 @@
"index_web_pages_for_search": 1,
"istable": 1,
"links": [],
"modified": "2020-08-17 16:15:46.937267",
"modified": "2020-12-05 19:20:00.503097",
"modified_by": "Administrator",
"module": "Core",
"name": "Report Filter",

41
frappe/core/doctype/server_script/server_script.js
View file

@ -48,29 +48,33 @@ frappe.ui.form.on('Server Script', {
setup_help(frm) {
frm.get_field('help_html').html(`
<h3>Examples</h3>
<h4>DocType Event</h4>
<pre><code>
<p>Add logic for standard doctype events like Before Insert, After Submit, etc.</p>
<pre>
<code>
# set property
if "test" in doc.description:
doc.status = 'Closed'
doc.status = 'Closed'
# validate
if "validate" in doc.description:
raise frappe.ValidationError
raise frappe.ValidationError
# auto create another document
if doc.allocted_to:
frappe.get_doc(dict(
doctype = 'ToDo'
owner = doc.allocated_to,
description = doc.subject
)).insert()
</code></pre>
if doc.allocated_to:
frappe.get_doc(dict(
doctype = 'ToDo'
owner = doc.allocated_to,
description = doc.subject
)).insert()
</code>
</pre>
<hr>
<h4>API Call</h4>
<p>Respond to <code>/api/method/&lt;method-name&gt;</code> calls, just like whitelisted methods</p>
<pre><code>
# respond to API
@ -79,6 +83,21 @@ if frappe.form_dict.message == "ping":
else:
frappe.response['message'] = "ok"
</code></pre>
<hr>
<h4>Permission Query</h4>
<p>Add conditions to the where clause of list queries.</p>
<pre><code>
# generate dynamic conditions and set it in the conditions variable
tenant_id = frappe.db.get_value(...)
conditions = 'tenant_id = {}'.format(tenant_id)
# resulting select query
select name from \`tabPerson\`
where tenant_id = 2
order by creation desc
</code></pre>
`);
}

6
frappe/core/doctype/server_script/server_script.json
View file

@ -24,7 +24,7 @@
"fieldtype": "Select",
"in_list_view": 1,
"label": "Script Type",
"options": "DocType Event\nScheduler Event\nAPI",
"options": "DocType Event\nScheduler Event\nPermission Query\nAPI",
"reqd": 1
},
{
@ -35,7 +35,7 @@
"reqd": 1
},
{
"depends_on": "eval:doc.script_type==='DocType Event'",
"depends_on": "eval:['DocType Event', 'Permission Query'].includes(doc.script_type)",
"fieldname": "reference_doctype",
"fieldtype": "Link",
"in_list_view": 1,
@ -88,7 +88,7 @@
],
"index_web_pages_for_search": 1,
"links": [],
"modified": "2020-11-11 12:39:41.391052",
"modified": "2020-12-03 22:42:02.708148",
"modified_by": "Administrator",
"module": "Core",
"name": "Server Script",

12
frappe/core/doctype/server_script/server_script.py
View file

@ -4,6 +4,8 @@
from __future__ import unicode_literals
import ast
import frappe
from frappe.model.document import Document
from frappe.utils.safe_exec import safe_exec
@ -11,9 +13,9 @@ from frappe import _
class ServerScript(Document):
@staticmethod
def validate():
def validate(self):
frappe.only_for('Script Manager', True)
ast.parse(self.script)
@staticmethod
def on_update():
@ -41,6 +43,12 @@ class ServerScript(Document):
# wrong report type!
raise frappe.DoesNotExistError
def get_permission_query_conditions(self, user):
locals = {"user": user, "conditions": ""}
safe_exec(self.script, None, locals)
if locals["conditions"]:
return locals["conditions"]
@frappe.whitelist()
def setup_scheduler_events(script_name, frequency):
method = frappe.scrub('{0}-{1}'.format(script_name, frequency))

11
frappe/core/doctype/server_script/server_script_utils.py
View file

@ -50,6 +50,9 @@ def get_server_script_map():
# },
# '_api': {
# '[path]': '[server script]'
# },
# 'permission_query': {
# 'DocType': '[server script]'
# }
# }
if frappe.flags.in_patch and not frappe.db.table_exists('Server Script'):
@ -57,16 +60,20 @@ def get_server_script_map():
script_map = frappe.cache().get_value('server_script_map')
if script_map is None:
script_map = {}
script_map = {
'permission_query': {}
}
enabled_server_scripts = frappe.get_all('Server Script',
fields=('name', 'reference_doctype', 'doctype_event','api_method', 'script_type'),
filters={'disabled': 0})
for script in enabled_server_scripts:
if script.script_type == 'DocType Event':
script_map.setdefault(script.reference_doctype, {}).setdefault(script.doctype_event, []).append(script.name)
elif script.script_type == 'Permission Query':
script_map['permission_query'][script.reference_doctype] = script.name
else:
script_map.setdefault('_api', {})[script.api_method] = script.name
frappe.cache().set_value('server_script_map', script_map)
return script_map
return script_map

25
frappe/core/doctype/server_script/test_server_script.py
View file

@ -45,6 +45,22 @@ frappe.response['message'] = 'hello'
allow_guest = 1,
script = '''
frappe.flags = 'hello'
'''
),
dict(
name='test_permission_query',
script_type = 'Permission Query',
reference_doctype = 'ToDo',
script = '''
conditions = '1 = 1'
'''),
dict(
name='test_invalid_namespace_method',
script_type = 'DocType Event',
doctype_event = 'Before Insert',
reference_doctype = 'Note',
script = '''
frappe.method_that_doesnt_exist("do some magic")
'''
)
]
@ -85,3 +101,12 @@ class TestServerScript(unittest.TestCase):
def test_api_return(self):
self.assertEqual(frappe.get_doc('Server Script', 'test_return_value').execute_method(), 'hello')
def test_permission_query(self):
self.assertTrue('where (1 = 1)' in frappe.db.get_list('ToDo', return_query=1))
self.assertTrue(isinstance(frappe.db.get_list('ToDo'), list))
def test_attribute_error(self):
"""Raise AttributeError if method not found in Namespace"""
note = frappe.get_doc({"doctype": "Note", "title": "Test Note: Server Script"})
self.assertRaises(AttributeError, note.insert)

63
frappe/core/doctype/system_settings/system_settings.js
View file

@ -1,37 +1,36 @@
frappe.ui.form.on("System Settings", "refresh", function(frm) {
frappe.call({
method: "frappe.core.doctype.system_settings.system_settings.load",
callback: function(data) {
frappe.all_timezones = data.message.timezones;
frm.set_df_property("time_zone", "options", frappe.all_timezones);
frappe.ui.form.on("System Settings", {
refresh: function(frm) {
frappe.call({
method: "frappe.core.doctype.system_settings.system_settings.load",
callback: function(data) {
frappe.all_timezones = data.message.timezones;
frm.set_df_property("time_zone", "options", frappe.all_timezones);
$.each(data.message.defaults, function(key, val) {
frm.set_value(key, val);
frappe.sys_defaults[key] = val;
})
$.each(data.message.defaults, function(key, val) {
frm.set_value(key, val);
frappe.sys_defaults[key] = val;
});
}
});
},
enable_password_policy: function(frm) {
if (frm.doc.enable_password_policy == 0) {
frm.set_value("minimum_password_score", "");
} else {
frm.set_value("minimum_password_score", "2");
}
});
});
frappe.ui.form.on("System Settings", "enable_password_policy", function(frm) {
if(frm.doc.enable_password_policy == 0){
frm.set_value("minimum_password_score", "");
} else {
frm.set_value("minimum_password_score", "2");
}
});
frappe.ui.form.on("System Settings", "enable_two_factor_auth", function(frm) {
if(frm.doc.enable_two_factor_auth == 0){
frm.set_value("bypass_2fa_for_retricted_ip_users", 0);
frm.set_value("bypass_restrict_ip_check_if_2fa_enabled", 0);
}
});
frappe.ui.form.on("System Settings", "enable_prepared_report_auto_deletion", function(frm) {
if (frm.doc.enable_prepared_report_auto_deletion) {
if (!frm.doc.prepared_report_expiry_period) {
frm.set_value('prepared_report_expiry_period', 7);
},
enable_two_factor_auth: function(frm) {
if (frm.doc.enable_two_factor_auth == 0) {
frm.set_value("bypass_2fa_for_retricted_ip_users", 0);
frm.set_value("bypass_restrict_ip_check_if_2fa_enabled", 0);
}
},
enable_prepared_report_auto_deletion: function(frm) {
if (frm.doc.enable_prepared_report_auto_deletion) {
if (!frm.doc.prepared_report_expiry_period) {
frm.set_value('prepared_report_expiry_period', 7);
}
}
}
});

9
frappe/core/doctype/system_settings/system_settings.json
View file

@ -37,6 +37,7 @@
"allow_login_using_mobile_number",
"allow_login_using_user_name",
"allow_error_traceback",
"strip_exif_metadata_from_uploaded_images",
"password_settings",
"logout_on_password_reset",
"force_user_to_reset_password",
@ -460,12 +461,18 @@
"fieldname": "prepared_report_section",
"fieldtype": "Section Break",
"label": "Prepared Report"
},
{
"default": "1",
"fieldname": "strip_exif_metadata_from_uploaded_images",
"fieldtype": "Check",
"label": "Strip EXIF tags from uploaded images"
}
],
"icon": "fa fa-cog",
"issingle": 1,
"links": [],
"modified": "2020-08-12 14:35:45.214327",
"modified": "2020-11-30 18:52:22.161391",
"modified_by": "Administrator",
"module": "Core",
"name": "System Settings",

5
frappe/custom/doctype/customize_form/customize_form.js
View file

@ -81,6 +81,11 @@ frappe.ui.form.on("Customize Form", {
} else {
f._sortable = false;
}
if (f.fieldtype == "Table") {
frm.add_custom_button(f.options, function() {
frm.set_value('doc_type', f.options);
}, __('Customize Child Table'));
}
});
frm.fields_dict.fields.grid.refresh();
},

32
frappe/custom/doctype/customize_form/customize_form.py
View file

@ -39,7 +39,7 @@ class CustomizeForm(Document):
translation = self.get_name_translation()
self.label = translation.translated_text if translation else ''
self.create_auto_repeat_custom_field_if_requried(meta)
self.create_auto_repeat_custom_field_if_required(meta)
# NOTE doc (self) is sent to clientside by run_method
@ -74,19 +74,25 @@ class CustomizeForm(Document):
for d in meta.get(fieldname):
self.append(fieldname, d)
def create_auto_repeat_custom_field_if_requried(self, meta):
def create_auto_repeat_custom_field_if_required(self, meta):
'''
Create auto repeat custom field if it's not already present
'''
if self.allow_auto_repeat:
if not frappe.db.exists('Custom Field', {'fieldname': 'auto_repeat',
'dt': self.doc_type}):
insert_after = self.fields[len(self.fields) - 1].fieldname
df = dict(
fieldname='auto_repeat',
label='Auto Repeat',
fieldtype='Link',
options='Auto Repeat',
insert_after=insert_after,
read_only=1, no_copy=1, print_hide=1)
create_custom_field(self.doc_type, df)
all_fields = [df.fieldname for df in meta.fields]
if "auto_repeat" in all_fields:
return
insert_after = self.fields[len(self.fields) - 1].fieldname
create_custom_field(self.doc_type, dict(
fieldname='auto_repeat',
label='Auto Repeat',
fieldtype='Link',
options='Auto Repeat',
insert_after=insert_after,
read_only=1, no_copy=1, print_hide=1
))
def get_name_translation(self):

11
frappe/database/db_manager.py
View file

@ -3,7 +3,6 @@ import frappe
class DbManager:
def __init__(self, db):
"""
Pass root_conn here for access to all databases.
@ -66,10 +65,10 @@ class DbManager:
esc = make_esc('$ ')
from distutils.spawn import find_executable
pipe = find_executable('pv')
if pipe:
pipe = '{pipe} {source} |'.format(
pipe=pipe,
pv = find_executable('pv')
if pv:
pipe = '{pv} {source} |'.format(
pv=pv,
source=source
)
source = ''
@ -78,7 +77,7 @@ class DbManager:
source = '< {source}'.format(source=source)
if pipe:
print('Creating Database...')
print('Restoring Database file...')
command = '{pipe} mysql -u {user} -p{password} -h{host} ' + ('-P{port}' if frappe.db.port else '') + ' {target} {source}'
command = command.format(

2
frappe/database/mariadb/framework_mariadb.sql
View file

@ -233,7 +233,7 @@ CREATE TABLE `tabDocType` (
DROP TABLE IF EXISTS `tabSeries`;
CREATE TABLE `tabSeries` (
`name` varchar(100) DEFAULT NULL,
`name` varchar(100),
`current` int(10) NOT NULL DEFAULT 0,
PRIMARY KEY(`name`)
) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;

18
frappe/database/mariadb/setup_db.py
View file

@ -1,7 +1,7 @@
from __future__ import unicode_literals
import frappe
import os, sys
import os
from frappe.database.db_manager import DbManager
expected_settings_10_2_earlier = {
@ -86,6 +86,8 @@ def drop_user_and_database(db_name, root_login, root_password):
dbman.drop_database(db_name)
def bootstrap_database(db_name, verbose, source_sql=None):
import sys
frappe.connect(db_name=db_name)
if not check_database_settings():
print('Database settings do not match expected values; stopping database setup.')
@ -94,9 +96,17 @@ def bootstrap_database(db_name, verbose, source_sql=None):
import_db_from_sql(source_sql, verbose)
frappe.connect(db_name=db_name)
if not 'tabDefaultValue' in frappe.db.get_tables():
print('''Database not installed, this can due to lack of permission, or that the database name exists.
Check your mysql root password, or use --force to reinstall''')
if 'tabDefaultValue' not in frappe.db.get_tables():
from click import secho
secho(
"Table 'tabDefaultValue' missing in the restored site. "
"Database not installed correctly, this can due to lack of "
"permission, or that the database name exists. Check your mysql"
" root password, validity of the backup file or use --force to"
" reinstall",
fg="red"
)
sys.exit(1)
def import_db_from_sql(source_sql=None, verbose=False):

65
frappe/database/postgres/setup_db.py
View file

@ -1,5 +1,7 @@
import frappe, subprocess, os
from six.moves import input
import os
import frappe
def setup_database(force, source_sql=None, verbose=False):
root_conn = get_root_connection()
@ -10,24 +12,62 @@ def setup_database(force, source_sql=None, verbose=False):
root_conn.sql("CREATE user {0} password '{1}'".format(frappe.conf.db_name,
frappe.conf.db_password))
root_conn.sql("GRANT ALL PRIVILEGES ON DATABASE `{0}` TO {0}".format(frappe.conf.db_name))
root_conn.close()
bootstrap_database(frappe.conf.db_name, verbose, source_sql=source_sql)
frappe.connect()
def bootstrap_database(db_name, verbose, source_sql=None):
frappe.connect(db_name=db_name)
import_db_from_sql(source_sql, verbose)
frappe.connect(db_name=db_name)
if 'tabDefaultValue' not in frappe.db.get_tables():
import sys
from click import secho
secho(
"Table 'tabDefaultValue' missing in the restored site. "
"This may be due to incorrect permissions or the result of a restore from a bad backup file. "
"Database not installed correctly.",
fg="red"
)
sys.exit(1)
def import_db_from_sql(source_sql=None, verbose=False):
from shutil import which
from subprocess import run, PIPE
# we can't pass psql password in arguments in postgresql as mysql. So
# set password connection parameter in environment variable
subprocess_env = os.environ.copy()
subprocess_env['PGPASSWORD'] = str(frappe.conf.db_password)
# bootstrap db
if not source_sql:
source_sql = os.path.join(os.path.dirname(__file__), 'framework_postgres.sql')
subprocess.check_output([
'psql', frappe.conf.db_name,
'-h', frappe.conf.db_host or 'localhost',
'-p', str(frappe.conf.db_port or '5432'),
'-U', frappe.conf.db_name,
'-f', source_sql
], env=subprocess_env)
pv = which('pv')
frappe.connect()
_command = (
f"psql {frappe.conf.db_name} "
f"-h {frappe.conf.db_host or 'localhost'} -p {str(frappe.conf.db_port or '5432')} "
f"-U {frappe.conf.db_name}"
)
if pv:
command = f"{pv} {source_sql} | " + _command
else:
command = _command + f" -f {source_sql}"
print("Restoring Database file...")
if verbose:
print(command)
restore_proc = run(command, env=subprocess_env, shell=True, stdout=PIPE)
if verbose:
print(f"\nSTDOUT by psql:\n{restore_proc.stdout.decode()}\nImported from Database File: {source_sql}")
def setup_help_database(help_db_name):
root_conn = get_root_connection()
@ -38,19 +78,20 @@ def setup_help_database(help_db_name):
root_conn.sql("GRANT ALL PRIVILEGES ON DATABASE `{0}` TO {0}".format(help_db_name))
def get_root_connection(root_login=None, root_password=None):
import getpass
if not frappe.local.flags.root_connection:
if not root_login:
root_login = frappe.conf.get("root_login") or None
if not root_login:
from six.moves import input
root_login = input("Enter postgres super user: ")
if not root_password:
root_password = frappe.conf.get("root_password") or None
if not root_password:
root_password = getpass.getpass("Postgres super user password: ")
from getpass import getpass
root_password = getpass("Postgres super user password: ")
frappe.local.flags.root_connection = frappe.database.get_db(user=root_login, password=root_password)

19
frappe/desk/doctype/dashboard/dashboard.py
View file

@ -5,6 +5,7 @@
from __future__ import unicode_literals
from frappe.model.document import Document
from frappe.modules.export_file import export_to_files
from frappe.config import get_modules_from_all_apps_for_user
import frappe
from frappe import _
import json
@ -42,6 +43,24 @@ class Dashboard(Document):
except ValueError as error:
frappe.throw(_("Invalid json added in the custom options: {0}").format(error))
def get_permission_query_conditions(user):
if not user:
user = frappe.session.user
if user == 'Administrator':
return
roles = frappe.get_roles(user)
if "System Manager" in roles:
return None
allowed_modules = [frappe.db.escape(module.get('module_name')) for module in get_modules_from_all_apps_for_user()]
module_condition = '`tabDashboard`.`module` in ({allowed_modules}) or `tabDashboard`.`module` is NULL'.format(
allowed_modules=','.join(allowed_modules))
return module_condition
@frappe.whitelist()
def get_permitted_charts(dashboard_name):
permitted_charts = []

111
frappe/desk/doctype/dashboard_chart/dashboard_chart.py
View file

@ -7,17 +7,18 @@ import frappe
from frappe import _
import datetime
import json
from frappe.utils.dashboard import cache_source, get_from_date_from_timespan
from frappe.utils import nowdate, add_to_date, getdate, get_last_day, formatdate,\
get_datetime, cint, now_datetime
from frappe.utils.dashboard import cache_source
from frappe.utils import nowdate, getdate, get_datetime, cint, now_datetime
from frappe.utils.dateutils import\
get_period, get_period_beginning, get_from_date_from_timespan, get_dates_from_timegrain
from frappe.model.naming import append_number_if_name_exists
from frappe.boot import get_allowed_reports
from frappe.config import get_modules_from_all_apps_for_user
from frappe.model.document import Document
from frappe.modules.export_file import export_to_files
def get_permission_query_conditions(user):
if not user:
user = frappe.session.user
@ -30,9 +31,11 @@ def get_permission_query_conditions(user):
doctype_condition = False
report_condition = False
module_condition = False
allowed_doctypes = [frappe.db.escape(doctype) for doctype in frappe.permissions.get_doctypes_with_read()]
allowed_reports = [frappe.db.escape(key) if type(key) == str else key.encode('UTF8') for key in get_allowed_reports()]
allowed_modules = [frappe.db.escape(module.get('module_name')) for module in get_modules_from_all_apps_for_user()]
if allowed_doctypes:
doctype_condition = '`tabDashboard Chart`.`document_type` in ({allowed_doctypes})'.format(
@ -40,18 +43,24 @@ def get_permission_query_conditions(user):
if allowed_reports:
report_condition = '`tabDashboard Chart`.`report_name` in ({allowed_reports})'.format(
allowed_reports=','.join(allowed_reports))
if allowed_modules:
module_condition = '''`tabDashboard Chart`.`module` in ({allowed_modules})
or `tabDashboard Chart`.`module` is NULL'''.format(
allowed_modules=','.join(allowed_modules))
return '''
(`tabDashboard Chart`.`chart_type` in ('Count', 'Sum', 'Average')
and {doctype_condition})
or
(`tabDashboard Chart`.`chart_type` = 'Report'
and {report_condition})
'''.format(
doctype_condition=doctype_condition,
report_condition=report_condition
)
((`tabDashboard Chart`.`chart_type` in ('Count', 'Sum', 'Average')
and {doctype_condition})
or
(`tabDashboard Chart`.`chart_type` = 'Report'
and {report_condition}))
and
({module_condition})
'''.format(
doctype_condition=doctype_condition,
report_condition=report_condition,
module_condition=module_condition
)
def has_permission(doc, ptype, user):
roles = frappe.get_roles(user)
@ -156,6 +165,7 @@ def add_chart_to_dashboard(args):
def get_chart_config(chart, filters, timespan, timegrain, from_date, to_date):
if not from_date:
from_date = get_from_date_from_timespan(to_date, timespan)
from_date = get_period_beginning(from_date, timegrain)
if not to_date:
to_date = now_datetime()
@ -185,7 +195,7 @@ def get_chart_config(chart, filters, timespan, timegrain, from_date, to_date):
result = get_result(data, timegrain, from_date, to_date)
chart_config = {
"labels": [formatdate(r[0].strftime('%Y-%m-%d')) for r in result],
"labels": [get_period(r[0], timegrain) for r in result],
"datasets": [{
"name": chart.name,
"values": [r[1] for r in result]
@ -279,16 +289,8 @@ def get_aggregate_function(chart_type):
def get_result(data, timegrain, from_date, to_date):
start_date = getdate(from_date)
end_date = getdate(to_date)
result = [[start_date, 0.0]]
while start_date < end_date:
next_date = get_next_expected_date(start_date, timegrain)
result.append([next_date, 0.0])
start_date = next_date
dates = get_dates_from_timegrain(from_date, to_date, timegrain)
result = [[date, 0] for date in dates]
data_index = 0
if data:
for i, d in enumerate(result):
@ -298,65 +300,6 @@ def get_result(data, timegrain, from_date, to_date):
return result
def get_next_expected_date(date, timegrain):
next_date = None
# given date is always assumed to be the period ending date
next_date = get_period_ending(add_to_date(date, days=1), timegrain)
return getdate(next_date)
def get_period_ending(date, timegrain):
date = getdate(date)
if timegrain == 'Daily':
pass
elif timegrain == 'Weekly':
date = get_week_ending(date)
elif timegrain == 'Monthly':
date = get_month_ending(date)
elif timegrain == 'Quarterly':
date = get_quarter_ending(date)
elif timegrain == 'Yearly':
date = get_year_ending(date)
return getdate(date)
def get_week_ending(date):
# week starts on monday
from datetime import timedelta
start = date - timedelta(days = date.weekday())
end = start + timedelta(days=6)
return end
def get_month_ending(date):
month_of_the_year = int(date.strftime('%m'))
# first day of next month (note month starts from 1)
date = add_to_date('{}-01-01'.format(date.year), months = month_of_the_year)
# last day of this month
return add_to_date(date, days=-1)
def get_quarter_ending(date):
date = getdate(date)
# find the earliest quarter ending date that is after
# the given date
for month in (3, 6, 9, 12):
quarter_end_month = getdate('{}-{}-01'.format(date.year, month))
quarter_end_date = getdate(get_last_day(quarter_end_month))
if date <= quarter_end_date:
date = quarter_end_date
break
return date
def get_year_ending(date):
''' returns year ending of the given date '''
# first day of next year (note year starts from 1)
date = add_to_date('{}-01-01'.format(date.year), months = 12)
# last day of this month
return add_to_date(date, days=-1)
@frappe.whitelist()
@frappe.validate_and_sanitize_search_inputs
def get_charts_for_user(doctype, txt, searchfield, start, page_len, filters):

38
frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py
View file

@ -5,8 +5,8 @@ from __future__ import unicode_literals
import unittest, frappe
from frappe.utils import getdate, formatdate, get_last_day
from frappe.desk.doctype.dashboard_chart.dashboard_chart import (get,
get_period_ending)
from frappe.utils.dateutils import get_period_ending, get_period
from frappe.desk.doctype.dashboard_chart.dashboard_chart import get
from datetime import datetime
from dateutil.relativedelta import relativedelta
@ -53,15 +53,11 @@ class TestDashboardChart(unittest.TestCase):
cur_date = datetime.now() - relativedelta(years=1)
result = get(chart_name='Test Dashboard Chart', refresh=1)
self.assertEqual(result.get('labels')[0], formatdate(cur_date.strftime('%Y-%m-%d')))
if formatdate(cur_date.strftime('%Y-%m-%d')) == formatdate(get_last_day(cur_date).strftime('%Y-%m-%d')):
cur_date += relativedelta(months=1)
for idx in range(1, 13):
for idx in range(13):
month = get_last_day(cur_date)
month = formatdate(month.strftime('%Y-%m-%d'))
self.assertEqual(result.get('labels')[idx], month)
self.assertEqual(result.get('labels')[idx], get_period(month))
cur_date += relativedelta(months=1)
frappe.db.rollback()
@ -87,15 +83,11 @@ class TestDashboardChart(unittest.TestCase):
cur_date = datetime.now() - relativedelta(years=1)
result = get(chart_name ='Test Empty Dashboard Chart', refresh=1)
self.assertEqual(result.get('labels')[0], formatdate(cur_date.strftime('%Y-%m-%d')))
if formatdate(cur_date.strftime('%Y-%m-%d')) == formatdate(get_last_day(cur_date).strftime('%Y-%m-%d')):
cur_date += relativedelta(months=1)
for idx in range(1, 13):
for idx in range(13):
month = get_last_day(cur_date)
month = formatdate(month.strftime('%Y-%m-%d'))
self.assertEqual(result.get('labels')[idx], month)
self.assertEqual(result.get('labels')[idx], get_period(month))
cur_date += relativedelta(months=1)
frappe.db.rollback()
@ -124,15 +116,11 @@ class TestDashboardChart(unittest.TestCase):
cur_date = datetime.now() - relativedelta(years=1)
result = get(chart_name ='Test Empty Dashboard Chart 2', refresh = 1)
self.assertEqual(result.get('labels')[0], formatdate(cur_date.strftime('%Y-%m-%d')))
if formatdate(cur_date.strftime('%Y-%m-%d')) == formatdate(get_last_day(cur_date).strftime('%Y-%m-%d')):
cur_date += relativedelta(months=1)
for idx in range(1, 13):
for idx in range(13):
month = get_last_day(cur_date)
month = formatdate(month.strftime('%Y-%m-%d'))
self.assertEqual(result.get('labels')[idx], month)
self.assertEqual(result.get('labels')[idx], get_period(month))
cur_date += relativedelta(months=1)
# only 1 data point with value
@ -183,13 +171,12 @@ class TestDashboardChart(unittest.TestCase):
timeseries = 1
)).insert()
result = get(chart_name ='Test Daily Dashboard Chart', refresh = 1)
result = get(chart_name = 'Test Daily Dashboard Chart', refresh = 1)
self.assertEqual(result.get('datasets')[0].get('values'), [200.0, 400.0, 300.0, 0.0, 100.0, 0.0])
self.assertEqual(
result.get('labels'),
[formatdate('2019-01-06'), formatdate('2019-01-07'), formatdate('2019-01-08'),\
formatdate('2019-01-09'), formatdate('2019-01-10'), formatdate('2019-01-11')]
['06-01-19', '07-01-19', '08-01-19', '09-01-19', '10-01-19', '11-01-19']
)
frappe.db.rollback()
@ -218,7 +205,10 @@ class TestDashboardChart(unittest.TestCase):
result = get(chart_name ='Test Weekly Dashboard Chart', refresh = 1)
self.assertEqual(result.get('datasets')[0].get('values'), [50.0, 300.0, 800.0, 0.0])
self.assertEqual(result.get('labels'), [formatdate('2018-12-30'), formatdate('2019-01-06'), formatdate('2019-01-13'), formatdate('2019-01-20')])
self.assertEqual(
result.get('labels'),
['30-12-18', '06-01-19', '13-01-19', '20-01-19']
)
frappe.db.rollback()

13
frappe/desk/doctype/number_card/number_card.py
View file

@ -8,6 +8,7 @@ from frappe.model.document import Document
from frappe.utils import cint
from frappe.model.naming import append_number_if_name_exists
from frappe.modules.export_file import export_to_files
from frappe.config import get_modules_from_all_apps_for_user
class NumberCard(Document):
def autoname(self):
@ -33,16 +34,24 @@ def get_permission_query_conditions(user=None):
return None
doctype_condition = False
module_condition = False
allowed_doctypes = [frappe.db.escape(doctype) for doctype in frappe.permissions.get_doctypes_with_read()]
allowed_modules = [frappe.db.escape(module.get('module_name')) for module in get_modules_from_all_apps_for_user()]
if allowed_doctypes:
doctype_condition = '`tabNumber Card`.`document_type` in ({allowed_doctypes})'.format(
allowed_doctypes=','.join(allowed_doctypes))
if allowed_modules:
module_condition = '''`tabNumber Card`.`module` in ({allowed_modules})
or `tabNumber Card`.`module` is NULL'''.format(
allowed_modules=','.join(allowed_modules))
return '''
{doctype_condition}
'''.format(doctype_condition=doctype_condition)
{doctype_condition}
and
{module_condition}
'''.format(doctype_condition=doctype_condition, module_condition=module_condition)
def has_permission(doc, ptype, user):
roles = frappe.get_roles(user)

2
frappe/desk/form/document_follow.py
View file

@ -21,7 +21,7 @@ def follow_document(doctype, doc_name, user, force=False):
avoided for some doctype
follow only if track changes are set to 1
'''
if (doctype in ("Communication", "ToDo", "Email Unsubscribe", "File", "Comment")
if (doctype in ("Communication", "ToDo", "Email Unsubscribe", "File", "Comment", "Email Account", "Email Domain")
or doctype in log_types):
return

22
frappe/email/doctype/email_template/email_template.json
View file

@ -1,4 +1,5 @@
{
"actions": [],
"allow_import": 1,
"allow_rename": 1,
"autoname": "Prompt",
@ -8,6 +9,8 @@
"engine": "InnoDB",
"field_order": [
"subject",
"use_html",
"response_html",
"response",
"owner",
"section_break_4",
@ -22,11 +25,12 @@
"reqd": 1
},
{
"depends_on": "eval:!doc.use_html",
"fieldname": "response",
"fieldtype": "Text Editor",
"in_list_view": 1,
"label": "Response",
"reqd": 1
"mandatory_depends_on": "eval:!doc.use_html"
},
{
"default": "user",
@ -45,10 +49,24 @@
"fieldtype": "HTML",
"label": "Email Reply Help",
"options": "<h4>Email Reply Example</h4>\n\n<pre>Order Overdue\n\nTransaction {{ name }} has exceeded Due Date. Please take necessary action.\n\nDetails\n\n- Customer: {{ customer }}\n- Amount: {{ grand_total }}\n</pre>\n\n<h4>How to get fieldnames</h4>\n\n<p>The fieldnames you can use in your email template are the fields in the document from which you are sending the email. You can find out the fields of any documents via Setup &gt; Customize Form View and selecting the document type (e.g. Sales Invoice)</p>\n\n<h4>Templating</h4>\n\n<p>Templates are compiled using the Jinja Templating Language. To learn more about Jinja, <a class=\"strong\" href=\"http://jinja.pocoo.org/docs/dev/templates/\">read this documentation.</a></p>\n"
},
{
"default": "0",
"fieldname": "use_html",
"fieldtype": "Check",
"label": "Use HTML"
},
{
"depends_on": "eval:doc.use_html",
"fieldname": "response_html",
"fieldtype": "Code",
"label": "Response ",
"options": "HTML"
}
],
"icon": "fa fa-comment",
"modified": "2019-10-30 14:15:00.956347",
"links": [],
"modified": "2020-11-30 14:12:50.321633",
"modified_by": "Administrator",
"module": "Email",
"name": "Email Template",

27
frappe/email/doctype/email_template/email_template.py
View file

@ -9,7 +9,29 @@ from six import string_types
class EmailTemplate(Document):
def validate(self):
validate_template(self.response)
if self.use_html:
validate_template(self.response_html)
else:
validate_template(self.response)
def get_formatted_subject(self, doc):
return frappe.render_template(self.subject, doc)
def get_formatted_response(self, doc):
if self.use_html:
return frappe.render_template(self.response_html, doc)
return frappe.render_template(self.response, doc)
def get_formatted_email(self, doc):
if isinstance(doc, string_types):
doc = json.loads(doc)
return {
"subject" : self.get_formatted_subject(doc),
"message" : self.get_formatted_response(doc)
}
@frappe.whitelist()
def get_email_template(template_name, doc):
@ -18,5 +40,4 @@ def get_email_template(template_name, doc):
doc = json.loads(doc)
email_template = frappe.get_doc("Email Template", template_name)
return {"subject" : frappe.render_template(email_template.subject, doc),
"message" : frappe.render_template(email_template.response, doc)}
return email_template.get_formatted_email(doc)

6
frappe/email/doctype/newsletter/newsletter.py
View file

@ -85,11 +85,11 @@ class Newsletter(WebsiteGenerator):
self.db_set("scheduled_to_send", len(self.recipients))
def get_message(self):
if self.content_type == "HTML":
return frappe.render_template(self.message_html, {"doc": self.as_dict()})
return {
'Rich Text': self.message,
'Markdown': markdown(self.message_md),
'HTML': self.message_html
'Markdown': markdown(self.message_md)
}[self.content_type or 'Rich Text']
def get_recipients(self):

4
frappe/email/doctype/notification/notification.json
View file

@ -207,7 +207,7 @@
"label": "Value To Be Set"
},
{
"depends_on": "eval:in_list(['Email', 'SMS'], doc.channel)",
"depends_on": "eval:doc.channel !=\"Slack\"",
"fieldname": "column_break_5",
"fieldtype": "Section Break",
"label": "Recipients"
@ -281,7 +281,7 @@
"icon": "fa fa-envelope",
"index_web_pages_for_search": 1,
"links": [],
"modified": "2020-10-28 11:04:54.955567",
"modified": "2020-11-24 14:25:43.245677",
"modified_by": "Administrator",
"module": "Email",
"name": "Notification",

1
frappe/email/doctype/notification/notification.py
View file

@ -181,6 +181,7 @@ def get_context(context):
'document_type': doc.doctype,
'document_name': doc.name,
'subject': subject,
'from_user': doc.modified_by or doc.owner,
'email_content': frappe.render_template(self.message, context),
'attached_file': attachments and json.dumps(attachments[0])
}

5
frappe/email/smtp.py
View file

@ -210,10 +210,9 @@ class SMTPServer:
try:
if self.use_ssl:
if not self.port:
self.smtp_port = 465
self.port = 465
self._sess = smtplib.SMTP_SSL((self.server or "").encode('utf-8'),
cint(self.port) or None)
self._sess = smtplib.SMTP_SSL((self.server or ""), cint(self.port))
else:
if self.use_tls and not self.port:
self.port = 587

25
frappe/email/test_smtp.py Normal file
View file

@ -0,0 +1,25 @@
# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors
# License: The MIT License
import unittest
from frappe.email.smtp import SMTPServer
class TestSMTP(unittest.TestCase):
def test_smtp_ssl_session(self):
for port in [None, 0, 465, "465"]:
make_server(port, 1, 0)
def test_smtp_tls_session(self):
for port in [None, 0, 587, "587"]:
make_server(port, 0, 1)
def make_server(port, ssl, tls):
server = SMTPServer(
server = "smtp.gmail.com",
port = port,
use_ssl = ssl,
use_tls = tls
)
server.sess

4
frappe/exceptions.py
View file

@ -106,8 +106,10 @@ class InvalidDates(ValidationError): pass
class DataTooLongException(ValidationError): pass
class FileAlreadyAttachedException(Exception): pass
class DocumentAlreadyRestored(Exception): pass
class AttachmentLimitReached(Exception): pass
# OAuth exceptions
class InvalidAuthorizationHeader(CSRFTokenError): pass
class InvalidAuthorizationPrefix(CSRFTokenError): pass
class InvalidAuthorizationToken(CSRFTokenError): pass
class InvalidDatabaseFile(ValidationError): pass
class InvalidDatabaseFile(ValidationError): pass
class ExecutableNotFound(FileNotFoundError): pass

1
frappe/hooks.py
View file

@ -94,6 +94,7 @@ permission_query_conditions = {
"User": "frappe.core.doctype.user.user.get_permission_query_conditions",
"Dashboard Settings": "frappe.desk.doctype.dashboard_settings.dashboard_settings.get_permission_query_conditions",
"Notification Log": "frappe.desk.doctype.notification_log.notification_log.get_permission_query_conditions",
"Dashboard": "frappe.desk.doctype.dashboard.dashboard.get_permission_query_conditions",
"Dashboard Chart": "frappe.desk.doctype.dashboard_chart.dashboard_chart.get_permission_query_conditions",
"Number Card": "frappe.desk.doctype.number_card.number_card.get_permission_query_conditions",
"Notification Settings": "frappe.desk.doctype.notification_settings.notification_settings.get_permission_query_conditions",

175
frappe/installer.py
View file

@ -3,8 +3,90 @@
import json
import os
from frappe.defaults import _clear_cache
import sys
import frappe
from frappe.defaults import _clear_cache
def _new_site(
db_name,
site,
mariadb_root_username=None,
mariadb_root_password=None,
admin_password=None,
verbose=False,
install_apps=None,
source_sql=None,
force=False,
no_mariadb_socket=False,
reinstall=False,
db_password=None,
db_type=None,
db_host=None,
db_port=None,
new_site=False,
):
"""Install a new Frappe site"""
if not force and os.path.exists(site):
print("Site {0} already exists".format(site))
sys.exit(1)
if no_mariadb_socket and not db_type == "mariadb":
print("--no-mariadb-socket requires db_type to be set to mariadb.")
sys.exit(1)
if not db_name:
import hashlib
db_name = "_" + hashlib.sha1(site.encode()).hexdigest()[:16]
frappe.init(site=site)
from frappe.commands.scheduler import _is_scheduler_enabled
from frappe.utils import get_site_path, scheduler, touch_file
try:
# enable scheduler post install?
enable_scheduler = _is_scheduler_enabled()
except Exception:
enable_scheduler = False
make_site_dirs()
installing = touch_file(get_site_path("locks", "installing.lock"))
install_db(
root_login=mariadb_root_username,
root_password=mariadb_root_password,
db_name=db_name,
admin_password=admin_password,
verbose=verbose,
source_sql=source_sql,
force=force,
reinstall=reinstall,
db_password=db_password,
db_type=db_type,
db_host=db_host,
db_port=db_port,
no_mariadb_socket=no_mariadb_socket,
)
apps_to_install = (
["frappe"] + (frappe.conf.get("install_apps") or []) + (list(install_apps) or [])
)
for app in apps_to_install:
install_app(app, verbose=verbose, set_as_patched=not source_sql)
os.remove(installing)
scheduler.toggle_scheduler(enable_scheduler)
frappe.db.commit()
scheduler_status = (
"disabled" if frappe.utils.scheduler.is_scheduler_disabled() else "enabled"
)
print("*** Scheduler is", scheduler_status, "***")
def install_db(root_login="root", root_password=None, db_name=None, source_sql=None,
@ -36,9 +118,9 @@ def install_db(root_login="root", root_password=None, db_name=None, source_sql=N
def install_app(name, verbose=False, set_as_patched=True):
from frappe.core.doctype.scheduled_job_type.scheduled_job_type import sync_jobs
from frappe.utils.fixtures import sync_fixtures
from frappe.model.sync import sync_for
from frappe.modules.utils import sync_customizations
from frappe.utils.fixtures import sync_fixtures
frappe.flags.in_install = name
frappe.flags.ignore_in_install = False
@ -180,11 +262,11 @@ def remove_app(app_name, dry_run=False, yes=False, no_backup=False, force=False)
for record in frappe.get_all(doctype, filters={"module": module_name}, pluck="name"):
print(f"* removing {doctype} '{record}'...")
if not dry_run:
frappe.delete_doc(doctype, record, ignore_on_trash=True)
frappe.delete_doc(doctype, record, ignore_on_trash=True, force=True)
print(f"* removing Module Def '{module_name}'...")
if not dry_run:
frappe.delete_doc("Module Def", module_name, ignore_on_trash=True)
frappe.delete_doc("Module Def", module_name, ignore_on_trash=True, force=True)
for doctype in set(drop_doctypes):
print(f"* dropping Table for '{doctype}'...")
@ -347,6 +429,28 @@ def remove_missing_apps():
frappe.db.set_global("installed_apps", json.dumps(installed_apps))
def extract_sql_from_archive(sql_file_path):
"""Return the path of an SQL file if the passed argument is the path of a gzipped
SQL file or an SQL file path. The path may be absolute or relative from the bench
root directory or the sites sub-directory.
Args:
sql_file_path (str): Path of the SQL file
Returns:
str: Path of the decompressed SQL file
"""
from frappe.utils import get_bench_relative_path
sql_file_path = get_bench_relative_path(sql_file_path)
# Extract the gzip file if user has passed *.sql.gz file instead of *.sql file
if sql_file_path.endswith('sql.gz'):
decompressed_file_name = extract_sql_gzip(sql_file_path)
else:
decompressed_file_name = sql_file_path
return decompressed_file_name
def extract_sql_gzip(sql_gz_path):
import subprocess
@ -361,9 +465,13 @@ def extract_sql_gzip(sql_gz_path):
return decompressed_file
def extract_files(site_name, file_path, folder_name):
import subprocess
def extract_files(site_name, file_path):
import shutil
import subprocess
from frappe.utils import get_bench_relative_path
file_path = get_bench_relative_path(file_path)
# Need to do frappe.init to maintain the site locals
frappe.init(site=site_name)
@ -391,6 +499,12 @@ def extract_files(site_name, file_path, folder_name):
def is_downgrade(sql_file_path, verbose=False):
"""checks if input db backup will get downgraded on current bench"""
# This function is only tested with mariadb
# TODO: Add postgres support
if frappe.conf.db_type not in (None, "mariadb"):
return False
from semantic_version import Version
head = "INSERT INTO `tabInstalled Application` VALUES"
@ -424,6 +538,37 @@ def is_downgrade(sql_file_path, verbose=False):
return downgrade
def is_partial(sql_file_path):
with open(sql_file_path) as f:
header = " ".join([f.readline() for _ in range(5)])
if "Partial Backup" in header:
return True
return False
def partial_restore(sql_file_path, verbose=False):
sql_file = extract_sql_from_archive(sql_file_path)
if frappe.conf.db_type in (None, "mariadb"):
from frappe.database.mariadb.setup_db import import_db_from_sql
elif frappe.conf.db_type == "postgres":
from frappe.database.postgres.setup_db import import_db_from_sql
import warnings
from click import style
warn = style(
"Delete the tables you want to restore manually before attempting"
" partial restore operation for PostreSQL databases",
fg="yellow"
)
warnings.warn(warn)
import_db_from_sql(source_sql=sql_file, verbose=verbose)
# Removing temporarily created file
if sql_file != sql_file_path:
os.remove(sql_file)
def validate_database_sql(path, _raise=True):
"""Check if file has contents and if DefaultValue table exists
@ -431,23 +576,29 @@ def validate_database_sql(path, _raise=True):
path (str): Path of the decompressed SQL file
_raise (bool, optional): Raise exception if invalid file. Defaults to True.
"""
to_raise = False
empty_file = False
missing_table = True
error_message = ""
if not os.path.getsize(path):
error_message = f"{path} is an empty file!"
to_raise = True
empty_file = True
if not _raise:
# dont bother checking if empty file
if not empty_file:
with open(path, "r") as f:
for line in f:
if 'tabDefaultValue' in line:
error_message = "Table `tabDefaultValue` not found in file."
to_raise = True
missing_table = False
break
if missing_table:
error_message = "Table `tabDefaultValue` not found in file."
if error_message:
import click
click.secho(error_message, fg="red")
if _raise and to_raise:
if _raise and (missing_table or empty_file):
raise frappe.InvalidDatabaseFile

5
frappe/integrations/doctype/dropbox_settings/dropbox_settings.py
View file

@ -9,7 +9,7 @@ import frappe
import os
from frappe import _
from frappe.model.document import Document
from frappe.integrations.offsite_backup_utils import get_latest_backup_file, send_email, validate_file_size
from frappe.integrations.offsite_backup_utils import get_latest_backup_file, send_email, validate_file_size, get_chunk_site
from frappe.integrations.utils import make_post_request
from frappe.utils import (cint, get_request_site_address,
get_files_path, get_backups_path, get_url, encode)
@ -167,8 +167,9 @@ def upload_file_to_dropbox(filename, folder, dropbox_client):
return
create_folder_if_not_exists(folder, dropbox_client)
chunk_size = 15 * 1024 * 1024
file_size = os.path.getsize(encode(filename))
chunk_size = get_chunk_site(file_size)
mode = (dropbox.files.WriteMode.overwrite)
f = open(encode(filename), 'rb')

32
frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.json
View file

@ -18,12 +18,9 @@
"bucket",
"endpoint_url",
"column_break_13",
"region",
"backup_details_section",
"frequency",
"backup_files",
"column_break_18",
"backup_limit"
"backup_files"
],
"fields": [
{
@ -42,7 +39,7 @@
},
{
"default": "1",
"description": "Note: By default emails for failed backups are sent.",
"description": "By default, emails are only sent for failed backups.",
"fieldname": "send_email_for_successful_backup",
"fieldtype": "Check",
"label": "Send Email for Successful Backup"
@ -73,14 +70,7 @@
"reqd": 1
},
{
"default": "us-east-1",
"description": "See https://docs.aws.amazon.com/general/latest/gr/s3.html for details.",
"fieldname": "region",
"fieldtype": "Select",
"label": "Region",
"options": "us-east-1\nus-east-2\nus-west-1\nus-west-2\naf-south-1\nap-east-1\nap-south-1\nap-southeast-1\nap-southeast-2\nap-northeast-1\nap-northeast-2\nap-northeast-3\nca-central-1\ncn-north-1\ncn-northwest-1\neu-central-1\neu-west-1\neu-west-2\neu-west-3\neu-south-1\neu-north-1\nme-south-1\nsa-east-1"
},
{
"default": "https://s3.amazonaws.com",
"fieldname": "endpoint_url",
"fieldtype": "Data",
"label": "Endpoint URL"
@ -92,14 +82,6 @@
"mandatory_depends_on": "enabled",
"reqd": 1
},
{
"description": "Set to 0 for no limit on the number of backups taken",
"fieldname": "backup_limit",
"fieldtype": "Int",
"label": "Backup Limit",
"mandatory_depends_on": "enabled",
"reqd": 1
},
{
"depends_on": "enabled",
"fieldname": "api_access_section",
@ -142,16 +124,12 @@
"fieldname": "backup_files",
"fieldtype": "Check",
"label": "Backup Files"
},
{
"fieldname": "column_break_18",
"fieldtype": "Column Break"
}
],
"hide_toolbar": 1,
"issingle": 1,
"links": [],
"modified": "2020-07-27 17:27:21.400000",
"modified": "2020-12-07 15:30:55.047689",
"modified_by": "Administrator",
"module": "Integrations",
"name": "S3 Backup Settings",
@ -172,4 +150,4 @@
"sort_field": "modified",
"sort_order": "DESC",
"track_changes": 1
}
}

58
frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.py
View file

@ -24,6 +24,7 @@ class S3BackupSettings(Document):
if not self.endpoint_url:
self.endpoint_url = 'https://s3.amazonaws.com'
conn = boto3.client(
's3',
aws_access_key_id=self.access_key_id,
@ -31,25 +32,21 @@ class S3BackupSettings(Document):
endpoint_url=self.endpoint_url
)
bucket_lower = str(self.bucket)
try:
conn.list_buckets()
except ClientError:
frappe.throw(_("Invalid Access Key ID or Secret Access Key."))
try:
# Head_bucket returns a 200 OK if the bucket exists and have access to it.
conn.head_bucket(Bucket=bucket_lower)
# Requires ListBucket permission
conn.head_bucket(Bucket=self.bucket)
except ClientError as e:
error_code = e.response['Error']['Code']
bucket_name = frappe.bold(self.bucket)
if error_code == '403':
frappe.throw(_("Do not have permission to access {0} bucket.").format(bucket_lower))
else: # '400'-Bad request or '404'-Not Found return
# try to create bucket
conn.create_bucket(Bucket=bucket_lower, CreateBucketConfiguration={
'LocationConstraint': self.region})
msg = _("Do not have permission to access bucket {0}.").format(bucket_name)
elif error_code == '404':
msg = _("Bucket {0} not found.").format(bucket_name)
else:
msg = e.args[0]
frappe.throw(msg)
@frappe.whitelist()
@ -70,11 +67,13 @@ def take_backups_weekly():
def take_backups_monthly():
take_backups_if("Monthly")
def take_backups_if(freq):
if cint(frappe.db.get_value("S3 Backup Settings", None, "enabled")):
if frappe.db.get_value("S3 Backup Settings", None, "frequency") == freq:
take_backups_s3()
@frappe.whitelist()
def take_backups_s3(retry_count=0):
try:
@ -146,42 +145,13 @@ def backup_to_s3():
if files_filename:
upload_file_to_s3(files_filename, folder, conn, bucket)
delete_old_backups(doc.backup_limit, bucket)
def upload_file_to_s3(filename, folder, conn, bucket):
destpath = os.path.join(folder, os.path.basename(filename))
try:
print("Uploading file:", filename)
conn.upload_file(filename, bucket, destpath)
conn.upload_file(filename, bucket, destpath) # Requires PutObject permission
except Exception as e:
frappe.log_error()
print("Error uploading: %s" % (e))
def delete_old_backups(limit, bucket):
all_backups = []
doc = frappe.get_single("S3 Backup Settings")
backup_limit = int(limit)
s3 = boto3.resource(
's3',
aws_access_key_id=doc.access_key_id,
aws_secret_access_key=doc.get_password('secret_access_key'),
endpoint_url=doc.endpoint_url or 'https://s3.amazonaws.com'
)
bucket = s3.Bucket(bucket)
objects = bucket.meta.client.list_objects_v2(Bucket=bucket.name, Delimiter='/')
if objects:
for obj in objects.get('CommonPrefixes'):
all_backups.append(obj.get('Prefix'))
oldest_backup = sorted(all_backups)[0] if all_backups else ''
if len(all_backups) > backup_limit:
print("Deleting Backup: {0}".format(oldest_backup))
for obj in bucket.objects.filter(Prefix=oldest_backup):
# delete all keys that are inside the oldest_backup
s3.Object(bucket.name, obj.key).delete()

2
frappe/integrations/frappe_providers/frappecloud.py
View file

@ -6,7 +6,7 @@ import frappe
def frappecloud_migrator(local_site):
print("Retreiving Site Migrator...")
print("Retrieving Site Migrator...")
remote_site = frappe.conf.frappecloud_url or "frappecloud.com"
request_url = "https://{}/api/method/press.api.script".format(remote_site)
request = requests.get(request_url)

132
frappe/integrations/oauth2.py
View file

@ -1,41 +1,50 @@
from __future__ import unicode_literals
import frappe, json
from frappe.oauth import OAuthWebRequestValidator, WebApplicationServer
import hashlib
import json
from urllib.parse import quote, urlencode, urlparse
import jwt
from oauthlib.oauth2 import FatalClientError, OAuth2Error
from werkzeug import url_fix
from six.moves.urllib.parse import quote, urlencode, urlparse
from frappe.integrations.doctype.oauth_provider_settings.oauth_provider_settings import get_oauth_settings
import frappe
from frappe import _
from frappe.oauth import OAuthWebRequestValidator, WebApplicationServer
from frappe.integrations.doctype.oauth_provider_settings.oauth_provider_settings import get_oauth_settings
def get_oauth_server():
if not getattr(frappe.local, 'oauth_server', None):
oauth_validator = OAuthWebRequestValidator()
frappe.local.oauth_server = WebApplicationServer(oauth_validator)
frappe.local.oauth_server = WebApplicationServer(oauth_validator)
return frappe.local.oauth_server
def get_urlparams_from_kwargs(param_kwargs):
def sanitize_kwargs(param_kwargs):
arguments = param_kwargs
if arguments.get("data"):
arguments.pop("data")
if arguments.get("cmd"):
arguments.pop("cmd")
arguments.pop('data', None)
arguments.pop('cmd', None)
return urlencode(arguments)
return arguments
@frappe.whitelist()
def approve(*args, **kwargs):
r = frappe.request
uri = url_fix(r.url.replace("+"," "))
http_method = r.method
body = r.get_data()
headers = r.headers
try:
scopes, frappe.flags.oauth_credentials = get_oauth_server().validate_authorization_request(uri, http_method, body, headers)
scopes, frappe.flags.oauth_credentials = get_oauth_server().validate_authorization_request(
r.url,
r.method,
r.get_data(),
r.headers
)
headers, body, status = get_oauth_server().create_authorization_response(uri=frappe.flags.oauth_credentials['redirect_uri'], \
body=body, headers=headers, scopes=scopes, credentials=frappe.flags.oauth_credentials)
headers, body, status = get_oauth_server().create_authorization_response(
uri=frappe.flags.oauth_credentials['redirect_uri'],
body=r.get_data(),
headers=r.headers,
scopes=scopes,
credentials=frappe.flags.oauth_credentials
)
uri = headers.get('Location', None)
frappe.local.response["type"] = "redirect"
@ -47,34 +56,28 @@ def approve(*args, **kwargs):
return e
@frappe.whitelist(allow_guest=True)
def authorize(*args, **kwargs):
#Fetch provider URL from settings
oauth_settings = get_oauth_settings()
params = get_urlparams_from_kwargs(kwargs)
request_url = urlparse(frappe.request.url)
success_url = request_url.scheme + "://" + request_url.netloc + "/api/method/frappe.integrations.oauth2.approve?" + params
def authorize(**kwargs):
success_url = "/api/method/frappe.integrations.oauth2.approve?" + encode_params(sanitize_kwargs(kwargs))
failure_url = frappe.form_dict["redirect_uri"] + "?error=access_denied"
if frappe.session['user']=='Guest':
if frappe.session.user == 'Guest':
#Force login, redirect to preauth again.
frappe.local.response["type"] = "redirect"
frappe.local.response["location"] = "/login?redirect-to=/api/method/frappe.integrations.oauth2.authorize?" + quote(params.replace("+"," "))
elif frappe.session['user']!='Guest':
frappe.local.response["location"] = "/login?" + encode_params({'redirect-to': frappe.request.url})
else:
try:
r = frappe.request
uri = url_fix(r.url)
http_method = r.method
body = r.get_data()
headers = r.headers
scopes, frappe.flags.oauth_credentials = get_oauth_server().validate_authorization_request(uri, http_method, body, headers)
scopes, frappe.flags.oauth_credentials = get_oauth_server().validate_authorization_request(
r.url,
r.method,
r.get_data(),
r.headers
)
skip_auth = frappe.db.get_value("OAuth Client", frappe.flags.oauth_credentials['client_id'], "skip_authorization")
unrevoked_tokens = frappe.get_all("OAuth Bearer Token", filters={"status":"Active"})
if skip_auth or (oauth_settings["skip_authorization"] == "Auto" and len(unrevoked_tokens)):
if skip_auth or (get_oauth_settings().skip_authorization == "Auto" and unrevoked_tokens):
frappe.local.response["type"] = "redirect"
frappe.local.response["location"] = success_url
else:
@ -87,7 +90,6 @@ def authorize(*args, **kwargs):
})
resp_html = frappe.render_template("templates/includes/oauth_confirmation.html", response_html_params)
frappe.respond_as_web_page("Confirm Access", resp_html)
except FatalClientError as e:
return e
except OAuth2Error as e:
@ -95,20 +97,20 @@ def authorize(*args, **kwargs):
@frappe.whitelist(allow_guest=True)
def get_token(*args, **kwargs):
r = frappe.request
uri = url_fix(r.url)
http_method = r.method
body = r.form
headers = r.headers
#Check whether frappe server URL is set
frappe_server_url = frappe.db.get_value("Social Login Key", "frappe", "base_url") or None
if not frappe_server_url:
frappe.throw(_("Please set Base URL in Social Login Key for Frappe"))
try:
headers, body, status = get_oauth_server().create_token_response(uri, http_method, body, headers, frappe.flags.oauth_credentials)
r = frappe.request
headers, body, status = get_oauth_server().create_token_response(
r.url,
r.method,
r.form,
r.headers,
frappe.flags.oauth_credentials
)
out = frappe._dict(json.loads(body))
if not out.error and "openid" in out.scope:
token_user = frappe.db.get_value("OAuth Bearer Token", out.access_token, "user")
@ -116,7 +118,7 @@ def get_token(*args, **kwargs):
client_secret = frappe.db.get_value("OAuth Client", token_client, "client_secret")
if token_user in ["Guest", "Administrator"]:
frappe.throw(_("Logged in as Guest or Administrator"))
import hashlib
id_token_header = {
"typ":"jwt",
"alg":"HS256"
@ -128,9 +130,10 @@ def get_token(*args, **kwargs):
"iss": frappe_server_url,
"at_hash": frappe.oauth.calculate_at_hash(out.access_token, hashlib.sha256)
}
import jwt
id_token_encoded = jwt.encode(id_token, client_secret, algorithm='HS256', headers=id_token_header)
out.update({"id_token":str(id_token_encoded)})
out.update({"id_token": str(id_token_encoded)})
frappe.local.response = out
except FatalClientError as e:
@ -140,12 +143,12 @@ def get_token(*args, **kwargs):
@frappe.whitelist(allow_guest=True)
def revoke_token(*args, **kwargs):
r = frappe.request
uri = url_fix(r.url)
http_method = r.method
body = r.form
headers = r.headers
headers, body, status = get_oauth_server().create_revocation_response(uri, headers=headers, body=body, http_method=http_method)
headers, body, status = get_oauth_server().create_revocation_response(
r.url,
headers=r.headers,
body=r.form,
http_method=r.method
)
frappe.local.response['http_status_code'] = status
if status == 200:
@ -174,15 +177,22 @@ def openid_profile(*args, **kwargs):
"email": name,
"picture": picture
})
frappe.local.response = user_profile
def validate_url(url_string):
try:
result = urlparse(url_string)
if result.scheme and result.scheme in ["http", "https", "ftp", "ftps"]:
return True
else:
return False
return result.scheme and result.scheme in ["http", "https", "ftp", "ftps"]
except:
return False
return False
def encode_params(params):
"""
Encode a dict of params into a query string.
Use `quote_via=urllib.parse.quote` so that whitespaces will be encoded as
`%20` instead of as `+`. This is needed because oauthlib cannot handle `+`
as a whitespace.
"""
return urlencode(params, quote_via=quote)

21
frappe/integrations/offsite_backup_utils.py
View file

@ -6,8 +6,7 @@ from __future__ import unicode_literals
import frappe
import glob
import os
from frappe.utils import split_emails, get_backups_path
from frappe.utils import split_emails, cint
def send_email(success, service_name, doctype, email_field, error_status=None):
recipients = get_recipients(doctype, email_field)
@ -81,6 +80,22 @@ def get_file_size(file_path, unit):
return file_size
def get_chunk_site(file_size):
''' this function will return chunk size in megabytes based on file size '''
file_size_in_gb = cint(file_size/1024/1024)
MB = 1024 * 1024
if file_size_in_gb > 5000:
return 200 * MB
elif file_size_in_gb >= 3000:
return 150 * MB
elif file_size_in_gb >= 1000:
return 100 * MB
elif file_size_in_gb >= 500:
return 50 * MB
else:
return 15 * MB
def validate_file_size():
frappe.flags.create_new_backup = True
@ -98,4 +113,4 @@ def generate_files_backup():
db_type=frappe.conf.db_type, db_port=frappe.conf.db_port)
backup.set_backup_file_name()
backup.zip_files()
backup.zip_files()

6
frappe/model/base_document.py
View file

@ -802,12 +802,12 @@ class BaseDocument(object):
if translated:
val = _(val)
if absolute_value and isinstance(val, (int, float)):
val = abs(self.get(fieldname))
if not doc:
doc = getattr(self, "parent_doc", None) or self
if (absolute_value or doc.get('absolute_value')) and isinstance(val, (int, float)):
val = abs(self.get(fieldname))
return format_value(val, df=df, doc=doc, currency=currency)
def is_print_hide(self, fieldname, df=None, for_print=True):

13
frappe/model/db_query.py
View file

@ -18,6 +18,7 @@ from frappe.client import check_parent_permission
from frappe.model.utils.user_settings import get_user_settings, update_user_settings
from frappe.utils import flt, cint, get_time, make_filter_tuple, get_filter, add_to_date, cstr, get_timespan_date_range
from frappe.model.meta import get_table_columns
from frappe.core.doctype.server_script.server_script_utils import get_server_script_map
class DatabaseQuery(object):
def __init__(self, doctype, user=None):
@ -683,15 +684,23 @@ class DatabaseQuery(object):
self.match_filters.append(match_filters)
def get_permission_query_conditions(self):
conditions = []
condition_methods = frappe.get_hooks("permission_query_conditions", {}).get(self.doctype, [])
if condition_methods:
conditions = []
for method in condition_methods:
c = frappe.call(frappe.get_attr(method), self.user)
if c:
conditions.append(c)
return " and ".join(conditions) if conditions else None
permision_script_name = get_server_script_map().get("permission_query").get(self.doctype)
if permision_script_name:
script = frappe.get_doc("Server Script", permision_script_name)
condition = script.get_permission_query_conditions(self.user)
if condition:
conditions.append(condition)
return " and ".join(conditions) if conditions else ""
def run_custom_query(self, query):
if '%(key)s' in query:

3
frappe/model/meta.py
View file

@ -209,7 +209,8 @@ class Meta(Document):
'owner': _('Created By'),
'modified_by': _('Modified By'),
'creation': _('Created On'),
'modified': _('Last Modified On')
'modified': _('Last Modified On'),
'_assign': _('Assigned To')
}.get(fieldname) or _('No Label')
return label

121
frappe/model/rename_doc.py
View file

@ -1,14 +1,15 @@
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt
from __future__ import unicode_literals, print_function
from __future__ import print_function, unicode_literals
import frappe
from frappe import _, bold
from frappe.utils import cint
from frappe.model.naming import validate_name
from frappe.model.dynamic_links import get_dynamic_link_map
from frappe.utils.password import rename_password
from frappe.model.naming import validate_name
from frappe.model.utils.user_settings import sync_user_settings, update_user_settings_data
from frappe.utils import cint
from frappe.utils.password import rename_password
@frappe.whitelist()
@ -42,7 +43,6 @@ def rename_doc(doctype, old, new, force=False, merge=False, ignore_permissions=F
force = cint(force)
merge = cint(merge)
meta = frappe.get_meta(doctype)
# call before_rename
@ -249,8 +249,17 @@ def update_link_field_values(link_fields, old, new, doctype):
# or no longer exists
pass
else:
# because the table hasn't been renamed yet!
parent = field['parent'] if field['parent']!=new else old
parent = field['parent']
# Handles the case where one of the link fields belongs to
# the DocType being renamed.
# Here this field could have the current DocType as its value too.
# In this case while updating link field value, the field's parent
# or the current DocType table name hasn't been renamed yet,
# so consider it's old name.
if parent == new and doctype == "DocType":
parent = old
frappe.db.sql("""
update `tab{table_name}` set `{fieldname}`=%s
@ -306,8 +315,7 @@ def get_link_fields(doctype):
def update_options_for_fieldtype(fieldtype, old, new):
if frappe.conf.developer_mode:
for name in frappe.db.sql_list("""select parent from
tabDocField where options=%s""", old):
for name in frappe.get_all("DocField", filters={"options": old}, pluck="parent"):
doctype = frappe.get_doc("DocType", name)
save = False
for f in doctype.fields:
@ -413,20 +421,21 @@ def update_parenttype_values(old, new):
child_doctypes += custom_child_doctypes
fields = [d['fieldname'] for d in child_doctypes]
property_setter_child_doctypes = frappe.db.sql("""\
select value as options from `tabProperty Setter`
where doc_type=%s and property='options' and
field_name in ("%s")""" % ('%s', '", "'.join(fields)),
(new,))
property_setter_child_doctypes = frappe.get_all(
"Property Setter",
filters={
"doc_type": new,
"property": "options",
"field_name": ("in", fields)
},
pluck="value"
)
child_doctypes = list(d['options'] for d in child_doctypes)
child_doctypes += property_setter_child_doctypes
child_doctypes = (d['options'] for d in child_doctypes)
for doctype in child_doctypes:
frappe.db.sql("""\
update `tab%s` set parenttype=%s
where parenttype=%s""" % (doctype, '%s', '%s'),
(new, old))
frappe.db.sql(f"update `tab{doctype}` set parenttype=%s where parenttype=%s", (new, old))
def rename_dynamic_links(doctype, old, new):
for df in get_dynamic_link_map().get(doctype, []):
@ -482,60 +491,30 @@ def bulk_rename(doctype, rows=None, via_console = False):
return rename_log
def update_linked_doctypes(doctype, docname, linked_to, value, ignore_doctypes=None):
"""
linked_doctype_info_list = list formed by get_fetch_fields() function
docname = Master DocType's name in which modification are made
value = Value for the field thats set in other DocType's by fetching from Master DocType
"""
linked_doctype_info_list = get_fetch_fields(doctype, linked_to, ignore_doctypes)
from frappe.model.utils.rename_doc import update_linked_doctypes
show_deprecation_warning("update_linked_doctypes")
return update_linked_doctypes(
doctype=doctype,
docname=docname,
linked_to=linked_to,
value=value,
ignore_doctypes=ignore_doctypes,
)
for d in linked_doctype_info_list:
frappe.db.sql("""
update
`tab{doctype}`
set
{linked_to_fieldname} = "{value}"
where
{master_fieldname} = {docname}
and {linked_to_fieldname} != "{value}"
""".format(
doctype = d['doctype'],
linked_to_fieldname = d['linked_to_fieldname'],
value = value,
master_fieldname = d['master_fieldname'],
docname = frappe.db.escape(docname)
))
def get_fetch_fields(doctype, linked_to, ignore_doctypes=None):
"""
doctype = Master DocType in which the changes are being made
linked_to = DocType name of the field thats being updated in Master
from frappe.model.utils.rename_doc import get_fetch_fields
show_deprecation_warning("get_fetch_fields")
This function fetches list of all DocType where both doctype and linked_to is found
as link fields.
Forms a list of dict in the form -
[{doctype: , master_fieldname: , linked_to_fieldname: ]
where
doctype = DocType where changes need to be made
master_fieldname = Fieldname where options = doctype
linked_to_fieldname = Fieldname where options = linked_to
"""
return get_fetch_fields(
doctype=doctype, linked_to=linked_to, ignore_doctypes=ignore_doctypes
)
master_list = get_link_fields(doctype)
linked_to_list = get_link_fields(linked_to)
out = []
from itertools import product
product_list = product(master_list, linked_to_list)
for d in product_list:
linked_doctype_info = frappe._dict()
if d[0]['parent'] == d[1]['parent'] \
and (not ignore_doctypes or d[0]['parent'] not in ignore_doctypes) \
and not d[1]['issingle']:
linked_doctype_info['doctype'] = d[0]['parent']
linked_doctype_info['master_fieldname'] = d[0]['fieldname']
linked_doctype_info['linked_to_fieldname'] = d[1]['fieldname']
out.append(linked_doctype_info)
return out
def show_deprecation_warning(funct):
from click import secho
message = (
f"Function frappe.model.rename_doc.{funct} has been deprecated and "
"moved to the frappe.model.utils.rename_doc"
)
secho(message, fg="yellow")

58
frappe/model/utils/rename_doc.py Normal file
View file

@ -0,0 +1,58 @@
from itertools import product
import frappe
from frappe.model.rename_doc import get_link_fields
def update_linked_doctypes(doctype, docname, linked_to, value, ignore_doctypes=None):
"""
linked_doctype_info_list = list formed by get_fetch_fields() function
docname = Master DocType's name in which modification are made
value = Value for the field thats set in other DocType's by fetching from Master DocType
"""
linked_doctype_info_list = get_fetch_fields(doctype, linked_to, ignore_doctypes)
for d in linked_doctype_info_list:
frappe.db.set_value(
d.doctype,
{
d.master_fieldname : docname,
d.linked_to_fieldname : ("!=", value),
},
d.linked_to_fieldname,
value,
)
def get_fetch_fields(doctype, linked_to, ignore_doctypes=None):
"""
doctype = Master DocType in which the changes are being made
linked_to = DocType name of the field thats being updated in Master
This function fetches list of all DocType where both doctype and linked_to is found
as link fields.
Forms a list of dict in the form -
[{doctype: , master_fieldname: , linked_to_fieldname: ]
where
doctype = DocType where changes need to be made
master_fieldname = Fieldname where options = doctype
linked_to_fieldname = Fieldname where options = linked_to
"""
out = []
master_list = get_link_fields(doctype)
linked_to_list = get_link_fields(linked_to)
product_list = product(master_list, linked_to_list)
for d in product_list:
linked_doctype_info = frappe._dict()
if (
d[0]["parent"] == d[1]["parent"]
and (not ignore_doctypes or d[0]["parent"] not in ignore_doctypes)
and not d[1]["issingle"]
):
linked_doctype_info.doctype = d[0]["parent"]
linked_doctype_info.master_fieldname = d[0]["fieldname"]
linked_doctype_info.linked_to_fieldname = d[1]["fieldname"]
out.append(linked_doctype_info)
return out

2
frappe/model/workflow.py
View file

@ -29,6 +29,8 @@ def get_transitions(doc, workflow = None, raise_exception=False):
if doc.is_new():
return []
doc.load_from_db()
frappe.has_permission(doc, 'read', throw=True)
roles = frappe.get_roles()

2
frappe/oauth.py
View file

@ -148,7 +148,7 @@ class OAuthWebRequestValidator(RequestValidator):
print("Failed body authentication: Application %s does not exist".format(cid=request.client_id))
cookie_dict = get_cookie_dict_from_headers(request)
user_id = unquote(cookie_dict['user_id']) if 'user_id' in cookie_dict else "Guest"
user_id = unquote(cookie_dict.get('user_id').value) if 'user_id' in cookie_dict else "Guest"
return frappe.session.user == user_id
def authenticate_client_id(self, client_id, request, *args, **kwargs):

1
frappe/patches.txt
View file

@ -21,6 +21,7 @@ execute:frappe.reload_doc('email', 'doctype', 'document_follow')
execute:frappe.reload_doc('core', 'doctype', 'communication_link') #2019-10-02
execute:frappe.reload_doc('core', 'doctype', 'has_role')
execute:frappe.reload_doc('core', 'doctype', 'communication') #2019-10-02
execute:frappe.reload_doc('core', 'doctype', 'server_script')
frappe.patches.v11_0.replicate_old_user_permissions
frappe.patches.v11_0.reload_and_rename_view_log #2019-01-03
frappe.patches.v7_1.rename_scheduler_log_to_error_log

18
frappe/printing/doctype/print_format/print_format.js
View file

@ -19,6 +19,7 @@ frappe.ui.form.on("Print Format", {
}
frm.trigger('render_buttons');
frm.toggle_display('standard', frappe.boot.developer_mode);
frm.trigger('hide_absolute_value_field');
},
render_buttons: function (frm) {
frm.page.clear_inner_toolbar();
@ -58,5 +59,20 @@ frappe.ui.form.on("Print Format", {
frm.set_value('show_section_headings', value);
frm.set_value('line_breaks', value);
frm.trigger('render_buttons');
},
doc_type: function (frm) {
frm.trigger('hide_absolute_value_field');
},
hide_absolute_value_field: function (frm) {
// TODO: make it work with frm.doc.doc_type
// Problem: frm isn't updated in some random cases
const doctype = locals[frm.doc.doctype][frm.doc.name].doc_type;
if (doctype) {
frappe.model.with_doctype(doctype, () => {
const meta = frappe.get_meta(doctype);
const has_int_float_currency_field = meta.fields.filter(df => in_list(['Int', 'Float', 'Currency'], df.fieldtype));
frm.toggle_display('absolute_value', has_int_float_currency_field.length);
});
}
}
})
});

11
frappe/printing/doctype/print_format/print_format.json
View file

@ -22,6 +22,7 @@
"align_labels_right",
"show_section_headings",
"line_breaks",
"absolute_value",
"column_break_11",
"font",
"css_section",
@ -196,13 +197,21 @@
"fieldtype": "Check",
"hidden": 1,
"label": "Print Format Builder"
},
{
"default": "0",
"depends_on": "doc_type",
"description": "If checked, negative numeric values of Currency, Quantity or Count would be shown as positive",
"fieldname": "absolute_value",
"fieldtype": "Check",
"label": "Show Absolute Values"
}
],
"icon": "fa fa-print",
"idx": 1,
"index_web_pages_for_search": 1,
"links": [],
"modified": "2020-10-27 18:27:58.307070",
"modified": "2020-12-14 11:38:49.132061",
"modified_by": "Administrator",
"module": "Printing",
"name": "Print Format",

1
frappe/public/js/frappe/data_import/import_preview.js
View file

@ -101,6 +101,7 @@ frappe.data_import.ImportPreview = class ImportPreview {
.replace('%H', 'HH')
.replace('%M', 'mm')
.replace('%S', 'ss')
.replace('%b', 'Mon')
: null;
let column_title = `<span class="indicator green">

1
frappe/public/js/frappe/desk.js
View file

@ -148,7 +148,6 @@ frappe.Application = Class.extend({
user: frappe.session.user
},
callback: function(r) {
console.log(r);
if(r.message.show_alert){
frappe.show_alert({
indicator: 'red',

4
frappe/public/js/frappe/file_uploader/index.js
View file

@ -15,7 +15,11 @@ export default class FileUploader {
allow_multiple,
as_dataurl,
disable_file_browser,
frm
} = {}) {
frm && frm.attachments.max_reached(true);
if (!wrapper) {
this.make_dialog();
} else {

2
frappe/public/js/frappe/form/controls/comment.js
View file

@ -60,7 +60,7 @@ frappe.ui.form.ControlComment = frappe.ui.form.ControlTextEditor.extend({
update_state() {
const value = this.get_value();
if (strip_html(value).trim() != "") {
if (strip_html(value).trim() != "" || value.includes('img')) {
this.button.removeClass('btn-default').addClass('btn-primary');
} else {
this.button.addClass('btn-default').removeClass('btn-primary');

18
frappe/public/js/frappe/form/controls/data.js
View file

@ -22,27 +22,9 @@ frappe.ui.form.ControlData = frappe.ui.form.ControlInput.extend({
this.has_input = true;
this.bind_change_event();
this.setup_autoname_check();
if (this.df.options == 'Phone') {
this.setup_phone();
}
// somehow this event does not bubble up to document
// after v7, if you can debug, remove this
},
setup_phone() {
if (frappe.phone_call.handler) {
this.$wrapper.find('.control-input')
.append(`
<span class="phone-btn">
<a class="btn-open no-decoration" title="${__('Make a call')}">
<i class="fa fa-phone"></i></a>
</span>
`)
.find('.phone-btn')
.click(() => {
frappe.phone_call.handler(this.get_value(), this.frm);
});
}
},
setup_autoname_check: function() {
if (!this.df.parent) return;
this.meta = frappe.get_meta(this.df.parent);

24
frappe/public/js/frappe/form/footer/timeline.js
View file

@ -30,7 +30,7 @@ frappe.ui.form.Timeline = class Timeline {
render_input: true,
only_input: true,
on_submit: (val) => {
if(strip_html(val).trim() != "") {
if (strip_html(val).trim() != "" || val.includes('img')) {
this.insert_comment(val, this.comment_area.button);
}
}
@ -547,10 +547,7 @@ frappe.ui.form.Timeline = class Timeline {
log.color = 'dark';
log.sender = log.owner;
log.comment_type = 'Milestone';
log.content = __('{0} changed {1} to {2}', [
frappe.user.full_name(log.owner).bold(),
frappe.meta.get_label(this.frm.doctype, log.track_field),
log.value.bold()]);
log.content = __('{0} changed {1} to {2}', [ frappe.user.full_name(log.owner).bold(), frappe.meta.get_label(this.frm.doctype, log.track_field), log.value.bold()]);
return log;
});
return milestones;
@ -613,11 +610,7 @@ frappe.ui.form.Timeline = class Timeline {
const field_display_status = frappe.perm.get_field_display_status(df, null,
me.frm.perm);
if (field_display_status === 'Read' || field_display_status === 'Write') {
parts.push(__('{0} from {1} to {2}', [
__(df.label),
me.format_content_for_timeline(p[1]),
me.format_content_for_timeline(p[2])
]));
parts.push(__('{0} from {1} to {2}', [ __(df.label), me.format_content_for_timeline(p[1]), me.format_content_for_timeline(p[2])]));
}
}
}
@ -648,13 +641,7 @@ frappe.ui.form.Timeline = class Timeline {
null, me.frm.perm);
if (field_display_status === 'Read' || field_display_status === 'Write') {
parts.push(__('{0} from {1} to {2} in row #{3}', [
frappe.meta.get_label(me.frm.fields_dict[row[0]].grid.doctype,
p[0]),
me.format_content_for_timeline(p[1]),
me.format_content_for_timeline(p[2]),
row[1]
]));
parts.push(__('{0} from {1} to {2} in row #{3}', [ frappe.meta.get_label( me.frm.fields_dict[row[0]].grid.doctype, p[0]), me.format_content_for_timeline(p[1]), me.format_content_for_timeline(p[2]), row[1] ]));
}
}
return parts.length < 3;
@ -691,8 +678,7 @@ frappe.ui.form.Timeline = class Timeline {
return p;
});
if (parts.length) {
out.push(me.get_version_comment(version, __("{0} rows for {1}",
[__(key), parts.join(', ')])));
out.push(me.get_version_comment(version, __("{0} rows for {1}", [__(key), parts.join(', ')])));
}
}
});

6
frappe/public/js/frappe/form/form.js
View file

@ -232,14 +232,10 @@ frappe.ui.form.Form = class FrappeForm {
throw "attach error";
}
if(me.attachments.max_reached()) {
frappe.msgprint(__("Maximum Attachment Limit for this record reached."));
throw "attach error";
}
new frappe.ui.FileUploader({
doctype: me.doctype,
docname: me.docname,
frm: me,
files: dataTransfer.files,
folder: 'Home/Attachments',
on_success(file_doc) {

1
frappe/public/js/frappe/form/grid_row.js
View file

@ -373,6 +373,7 @@ export default class GridRow {
// no text editor in grid
if (df.fieldtype=='Text Editor') {
df = Object.assign({}, df);
df.fieldtype = 'Text';
}

22
frappe/public/js/frappe/form/sidebar/attachments.js
View file

@ -16,15 +16,19 @@ frappe.ui.form.Attachments = Class.extend({
this.add_attachment_wrapper = this.parent.find(".add_attachment").parent();
this.attachments_label = this.parent.find(".attachments-label");
},
max_reached: function() {
// no of attachments
var n = Object.keys(this.get_attachments()).length;
// button if the number of attachments is less than max
if(n < this.frm.meta.max_attachments || !this.frm.meta.max_attachments) {
return false;
max_reached: function(raise_exception=false) {
const attachment_count = Object.keys(this.get_attachments()).length;
const attachment_limit = this.frm.meta.max_attachments;
if (attachment_limit && attachment_count >= attachment_limit) {
if (raise_exception) {
frappe.throw({
title: __("Attachment Limit Reached"),
message: __("Maximum attachment limit of {0} has been reached.", [cstr(attachment_limit).bold()]),
});
}
return true;
}
return true;
return false;
},
refresh: function() {
var me = this;
@ -140,7 +144,6 @@ frappe.ui.form.Attachments = Class.extend({
});
},
new_attachment: function(fieldname) {
var me = this;
if (this.dialog) {
// remove upload dialog
this.dialog.$wrapper.remove();
@ -149,6 +152,7 @@ frappe.ui.form.Attachments = Class.extend({
new frappe.ui.FileUploader({
doctype: this.frm.doctype,
docname: this.frm.docname,
frm: this.frm,
folder: 'Home/Attachments',
on_success: (file_doc) => {
this.attachment_uploaded(file_doc);

10
frappe/public/js/frappe/microtemplate.js
View file

@ -89,11 +89,19 @@ frappe.render_template = function(name, data) {
}
frappe.render_grid = function(opts) {
// build context
if(opts.grid) {
if (opts.grid) {
opts.columns = opts.grid.getColumns();
opts.data = opts.grid.getData().getItems();
}
if (
opts.print_settings &&
opts.print_settings.orientation &&
opts.print_settings.orientation.toLowerCase() === "landscape"
) {
opts.landscape = true;
}
// show landscape view if columns more than 10
if (opts.landscape == null) {
if(opts.columns && opts.columns.length > 10) {

1
frappe/public/js/frappe/model/create_new.js
View file

@ -306,6 +306,7 @@ $.extend(frappe.model, {
selected_children: opts.frm ? opts.frm.get_selected() : null
},
freeze: true,
freeze_message: opts.freeze_message || '',
callback: function(r) {
if(!r.exc) {
frappe.model.sync(r.message);

33
frappe/public/js/frappe/model/model.js
View file

@ -103,6 +103,31 @@ $.extend(frappe.model, {
return docfield[0];
},
get_from_localstorage: function(doctype) {
if (localStorage["_doctype:" + doctype]) {
return JSON.parse(localStorage["_doctype:" + doctype]);
}
},
set_in_localstorage: function(doctype, docs) {
try {
localStorage["_doctype:" + doctype] = JSON.stringify(docs);
} catch(e) {
// if quota is exceeded, clear local storage and set item
console.warn("localStorage quota exceeded, clearing doctype cache")
frappe.model.clear_local_storage();
localStorage["_doctype:" + doctype] = JSON.stringify(docs);
}
},
clear_local_storage: function() {
for(var key in localStorage) {
if (key.startsWith("_doctype:")) {
localStorage.removeItem(key);
}
}
},
with_doctype: function(doctype, callback, async) {
if(locals.DocType[doctype]) {
callback && callback();
@ -110,13 +135,15 @@ $.extend(frappe.model, {
let cached_timestamp = null;
let cached_doc = null;
if(localStorage["_doctype:" + doctype]) {
let cached_docs = JSON.parse(localStorage["_doctype:" + doctype]);
let cached_docs = frappe.model.get_from_localstorage(doctype)
if (cached_docs) {
cached_doc = cached_docs.filter(doc => doc.name === doctype)[0];
if(cached_doc) {
cached_timestamp = cached_doc.modified;
}
}
return frappe.call({
method:'frappe.desk.form.load.getdoctype',
type: "GET",
@ -134,7 +161,7 @@ $.extend(frappe.model, {
if(r.message=="use_cache") {
frappe.model.sync(cached_doc);
} else {
localStorage["_doctype:" + doctype] = JSON.stringify(r.docs);
frappe.model.set_in_localstorage(doctype, r.docs)
}
frappe.model.init_doctype(doctype);

10
frappe/public/js/frappe/utils/common.js
View file

@ -234,11 +234,11 @@ frappe.utils.xss_sanitise = function (string, options) {
strategies: ['html', 'js'] // use all strategies.
}
const HTML_ESCAPE_MAP = {
'<': '&lt',
'>': '&gt',
'"': '&quot',
"'": '&#x27',
'/': '&#x2F'
'<': '&lt;',
'>': '&gt;',
'"': '&quot;',
"'": '&#x27;',
'/': '&#x2F;'
};
const REGEX_SCRIPT = /<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi; // used in jQuery 1.7.2 src/ajax.js Line 14
options = Object.assign({ }, DEFAULT_OPTIONS, options); // don't deep copy, immutable beauty.

41
frappe/public/js/frappe/utils/utils.js
View file

@ -979,17 +979,42 @@ Object.assign(frappe.utils, {
return route;
},
shorten_number: function (number, country) {
country = (country == 'India') ? country : '';
shorten_number: function (number, country, min_length=4, max_no_of_decimals=2) {
/* returns the number as an abbreviated string
* PARAMS
* number - number to be shortened
* country - country that determines the numnber system to be used
* min_length - length below which the number will not be shortened
* max_no_of_decimals - max number of decimals of the shortened number
*/
// return number if total digits is lesser than min_length
const len = String(number).match(/\d/g).length;
if (len < min_length) return number.toString();
const number_system = this.get_number_system(country);
let x = Math.abs(Math.round(number));
for (const map of number_system) {
const condition = map.condition ? map.condition(x) : x >= map.divisor;
if (condition) {
return (number/map.divisor).toFixed(2) + ' ' + map.symbol;
if (x >= map.divisor) {
let result = number/map.divisor;
const no_of_decimals = this.get_number_of_decimals(result);
/*
If no_of_decimals is greater than max_no_of_decimals,
round the number to max_no_of_decimals
*/
result = no_of_decimals > max_no_of_decimals
? result.toFixed(max_no_of_decimals)
: result;
return result + ' ' + map.symbol;
}
}
return number.toFixed();
return number.toFixed(max_no_of_decimals);
},
get_number_of_decimals: function (number) {
if (Math.floor(number) === number) return 0;
return number.toString().split(".")[1].length || 0;
},
get_number_system: function (country) {
@ -1019,9 +1044,11 @@ Object.assign(frappe.utils, {
{
divisor: 1.0e+3,
symbol: 'K',
condition: (num) => num.toFixed().length > 5
}]
};
if (!Object.keys(number_system_map).includes(country)) country = '';
return number_system_map[country];
},
});

2
frappe/public/js/frappe/views/kanban/kanban_view.js
View file

@ -30,7 +30,7 @@ frappe.views.KanbanView = class KanbanView extends frappe.views.ListView {
return super.setup_defaults()
.then(() => {
this.board_name = frappe.get_route()[3];
this.page_title = this.board_name;
this.page_title = __(this.board_name);
this.card_meta = this.get_card_meta();
this.menu_items.push({

4
frappe/public/js/frappe/views/reports/query_report.js
View file

@ -1112,7 +1112,9 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
}
get_filter_values(raise) {
const mandatory = this.filters.filter(f => f.df.reqd);
// check for mandatory property for filters added via UI
const mandatory = this.filters.filter(f => (f.df.reqd || f.df.mandatory));
const missing_mandatory = mandatory.filter(f => !f.get_value());
if (raise && missing_mandatory.length > 0) {
let message = __('Please set filters');

4
frappe/public/js/frappe/views/reports/report_utils.js
View file

@ -15,7 +15,9 @@ frappe.report_utils = {
if (raw_data.add_total_row) {
labels = labels.slice(0, -1);
datasets[0].values = datasets[0].values.slice(0, -1);
datasets.forEach(dataset => {
dataset.values = dataset.values.slice(0, -1);
});
}
return {

4
frappe/public/js/frappe/widgets/number_card_widget.js
View file

@ -204,7 +204,7 @@ export default class NumberCardWidget extends Widget {
get_formatted_number(df) {
const default_country = frappe.sys_defaults.country;
const shortened_number = frappe.utils.shorten_number(this.number, default_country);
const shortened_number = frappe.utils.shorten_number(this.number, default_country, 5);
let number_parts = shortened_number.split(' ');
const symbol = number_parts[1] || '';
@ -269,7 +269,7 @@ export default class NumberCardWidget extends Widget {
result: this.number
}).then(res => {
if (res !== undefined) {
this.percentage_stat = shorten_number(res);
this.percentage_stat = frappe.utils.shorten_number(res);
}
});
}

11
frappe/public/scss/website.scss
View file

@ -244,9 +244,18 @@ h5.modal-title {
white-space: nowrap;
text-overflow: ellipsis;
}
.about-section {
padding-top: 1rem;
}
.about-footer {
padding-top: 1rem;
}
}
.logged-in > .nav-link {
max-width: 200px;
@extend .ellipsis;
max-width: 100%;
vertical-align: middle;
}

16
frappe/templates/includes/oauth_confirmation.html
View file

@ -1,7 +1,7 @@
{% if not error %}
<div class="panel panel-default">
<div class="panel-heading">
<h3 class="panel-title">{{ client_id }} wants to access the following details from your account</h3>
<h3 class="panel-title">{{ _("{} wants to access the following details from your account").format(client_id) }}</h3>
</div>
<div class="panel-body">
<ul class="list-group">
@ -11,10 +11,10 @@
</ul>
<ul class="list-inline">
<li>
<button id="allow" class="btn btn-sm btn-primary">Allow</button>
<button id="allow" class="btn btn-sm btn-primary">{{ _("Allow") }}</button>
</li>
<li>
<button id="deny" class="btn btn-sm btn-light">Deny</button>
<button id="deny" class="btn btn-sm btn-light">{{ _("Deny") }}</button>
</li>
</ul>
</div>
@ -22,24 +22,24 @@
<script type="text/javascript">
frappe.ready(function() {
$('#allow').on('click', function(event) {
window.location.replace("{{ success_url|string }}");
window.location.replace("{{ success_url | string }}");
});
$('#deny').on('click', function(event) {
window.location.replace("{{ failure_url|string }}");
window.location.replace("{{ failure_url | string }}");
});
});
</script>
{% else %}
<div class="panel panel-danger">
<div class="panel-heading">
<h3 class="panel-title">Authorization error for {{ client_id }}</h3>
<h3 class="panel-title">{{ _("Authorization error for {}.").format(client_id) }}</h3>
</div>
<div class="panel-body">
<p>An unexpected error occurred while authorizing {{ client_id }}.</p>
<p>{{ _("An unexpected error occurred while authorizing {}.").format(client_id) }}</p>
<h4>{{ error }}</h4>
<ul class="list-inline">
<li>
<button class="btn btn-sm btn-light">OK</button>
<button class="btn btn-sm btn-light">{{ _("OK") }}</button>
</li>
</ul>
</div>

5
frappe/templates/print_formats/standard_macros.html
View file

@ -136,10 +136,9 @@ data-fieldname="{{ df.fieldname }}" data-fieldtype="{{ df.fieldtype }}"
{%- if df.print_width %} style="width: {{ get_width(df) }};"{% endif %}>
{% elif df.fieldtype=="HTML" %}
{{ frappe.render_template(df.options, {"doc":doc}) }}
{% elif df.fieldtype=="Currency" %}
{{ doc.get_formatted(df.fieldname, doc, translated=df.translatable) }}
{% else %}
{{ doc.get_formatted(df.fieldname, parent_doc or doc, translated=df.translatable) }}
{%- set parent = parent_doc or doc -%}
{{ doc.get_formatted(df.fieldname, parent, translated=df.translatable, absolute_value=parent.absolute_value) }}
{% endif %}
{%- endmacro %}

BIN
frappe/tests/data/exif_sample_image.jpg Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 158 KiB

246
frappe/tests/test_commands.py
View file

@ -1,24 +1,88 @@
# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors
# imports - standard imports
import gzip
import json
import os
import shlex
import subprocess
import sys
import unittest
from glob import glob
import glob
# imports - module imports
import frappe
from frappe.utils.backups import fetch_latest_backups
import frappe.recorder
from frappe.installer import add_to_installed_apps
from frappe.utils import add_to_date, get_bench_relative_path, now
from frappe.utils.backups import fetch_latest_backups
# TODO: check frappe.cli.coloured_output to set coloured output!
def supports_color():
"""
Returns True if the running system's terminal supports color, and False
otherwise.
"""
plat = sys.platform
supported_platform = plat != 'Pocket PC' and (plat != 'win32' or 'ANSICON' in os.environ)
# isatty is not always implemented, #6223.
is_a_tty = hasattr(sys.stdout, 'isatty') and sys.stdout.isatty()
return supported_platform and is_a_tty
class color(dict):
nc = "\033[0m"
blue = "\033[94m"
green = "\033[92m"
yellow = "\033[93m"
red = "\033[91m"
silver = "\033[90m"
def __getattr__(self, key):
if supports_color():
ret = self.get(key)
else:
ret = ""
return ret
def clean(value):
if isinstance(value, (bytes, str)):
value = value.decode().strip()
"""Strips and converts bytes to str
Args:
value ([type]): [description]
Returns:
[type]: [description]
"""
if isinstance(value, bytes):
value = value.decode()
if isinstance(value, str):
value = value.strip()
return value
def exists_in_backup(doctypes, file):
"""Checks if the list of doctypes exist in the database.sql.gz file supplied
Args:
doctypes (list): List of DocTypes to be checked
file (str): Path of the database file
Returns:
bool: True if all tables exist
"""
predicate = (
'COPY public."tab{}"'
if frappe.conf.db_type == "postgres"
else "CREATE TABLE `tab{}`"
)
with gzip.open(file, "rb") as f:
content = f.read().decode("utf8")
return all([predicate.format(doctype).lower() in content.lower() for doctype in doctypes])
class BaseTestCommands(unittest.TestCase):
def execute(self, command, kwargs=None):
site = {"site": frappe.local.site}
@ -26,13 +90,26 @@ class BaseTestCommands(unittest.TestCase):
kwargs.update(site)
else:
kwargs = site
command = command.replace("\n", " ").format(**kwargs)
command = shlex.split(command)
self.command = " ".join(command.split()).format(**kwargs)
print("{0}$ {1}{2}".format(color.silver, self.command, color.nc))
command = shlex.split(self.command)
self._proc = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
self.stdout = clean(self._proc.stdout)
self.stderr = clean(self._proc.stderr)
self.returncode = clean(self._proc.returncode)
def _formatMessage(self, msg, standardMsg):
output = super(BaseTestCommands, self)._formatMessage(msg, standardMsg)
cmd_execution_summary = "\n".join([
"-" * 70,
"Last Command Execution Summary:",
"Command: {}".format(self.command) if self.command else "",
"Standard Output: {}".format(self.stdout) if self.stdout else "",
"Standard Error: {}".format(self.stderr) if self.stderr else "",
"Return Code: {}".format(self.returncode) if self.returncode else "",
]).strip()
return "{}\n\n{}".format(output, cmd_execution_summary)
class TestCommands(BaseTestCommands):
def test_execute(self):
@ -52,9 +129,24 @@ class TestCommands(BaseTestCommands):
# The returned value has quotes which have been trimmed for the test
self.execute("""bench --site {site} execute frappe.bold --kwargs '{{"text": "DocType"}}'""")
self.assertEquals(self.returncode, 0)
self.assertEquals(self.stdout[1:-1], frappe.bold(text='DocType'))
self.assertEquals(self.stdout[1:-1], frappe.bold(text="DocType"))
def test_backup(self):
backup = {
"includes": {
"includes": [
"ToDo",
"Note",
]
},
"excludes": {
"excludes": [
"Activity Log",
"Access Log",
"Error Log"
]
}
}
home = os.path.expanduser("~")
site_backup_path = frappe.utils.get_site_path("private", "backups")
@ -94,16 +186,19 @@ class TestCommands(BaseTestCommands):
"db_path": "database.sql.gz",
"files_path": "public.tar",
"private_path": "private.tar",
"conf_path": "config.json"
"conf_path": "config.json",
}.items()
}
self.execute("""bench
self.execute(
"""bench
--site {site} backup --with-files
--backup-path-db {db_path}
--backup-path-files {files_path}
--backup-path-private-files {private_path}
--backup-path-conf {conf_path}""", kwargs)
--backup-path-conf {conf_path}""",
kwargs,
)
self.assertEquals(self.returncode, 0)
for path in kwargs.values():
@ -111,16 +206,122 @@ class TestCommands(BaseTestCommands):
# test 5: take a backup with --compress
self.execute("bench --site {site} backup --with-files --compress")
self.assertEquals(self.returncode, 0)
compressed_files = glob(site_backup_path + "/*.tgz")
compressed_files = glob.glob(site_backup_path + "/*.tgz")
self.assertGreater(len(compressed_files), 0)
# test 6: take a backup with --verbose
self.execute("bench --site {site} backup --verbose")
self.assertEquals(self.returncode, 0)
# test 7: take a backup with frappe.conf.backup.includes
self.execute(
"bench --site {site} set-config backup '{includes}' --as-dict",
{"includes": json.dumps(backup["includes"])},
)
self.execute("bench --site {site} backup --verbose")
self.assertEquals(self.returncode, 0)
database = fetch_latest_backups(partial=True)["database"]
self.assertTrue(exists_in_backup(backup["includes"]["includes"], database))
# test 8: take a backup with frappe.conf.backup.excludes
self.execute(
"bench --site {site} set-config backup '{excludes}' --as-dict",
{"excludes": json.dumps(backup["excludes"])},
)
self.execute("bench --site {site} backup --verbose")
self.assertEquals(self.returncode, 0)
database = fetch_latest_backups(partial=True)["database"]
self.assertFalse(exists_in_backup(backup["excludes"]["excludes"], database))
self.assertTrue(exists_in_backup(backup["includes"]["includes"], database))
# test 9: take a backup with --include (with frappe.conf.excludes still set)
self.execute(
"bench --site {site} backup --include '{include}'",
{"include": ",".join(backup["includes"]["includes"])},
)
self.assertEquals(self.returncode, 0)
database = fetch_latest_backups(partial=True)["database"]
self.assertTrue(exists_in_backup(backup["includes"]["includes"], database))
# test 10: take a backup with --exclude
self.execute(
"bench --site {site} backup --exclude '{exclude}'",
{"exclude": ",".join(backup["excludes"]["excludes"])},
)
self.assertEquals(self.returncode, 0)
database = fetch_latest_backups(partial=True)["database"]
self.assertFalse(exists_in_backup(backup["excludes"]["excludes"], database))
# test 11: take a backup with --ignore-backup-conf
self.execute("bench --site {site} backup --ignore-backup-conf")
self.assertEquals(self.returncode, 0)
database = fetch_latest_backups()["database"]
self.assertTrue(exists_in_backup(backup["excludes"]["excludes"], database))
def test_restore(self):
# step 0: create a site to run the test on
global_config = {
"admin_password": frappe.conf.admin_password,
"root_login": frappe.conf.root_login,
"root_password": frappe.conf.root_password,
"db_type": frappe.conf.db_type,
}
site_data = {"another_site": f"{frappe.local.site}-restore.test", **global_config}
for key, value in global_config.items():
if value:
self.execute(f"bench set-config {key} {value} -g")
self.execute(
"bench new-site {another_site} --admin-password {admin_password} --db-type"
" {db_type}",
site_data,
)
# test 1: bench restore from full backup
self.execute("bench --site {another_site} backup --ignore-backup-conf", site_data)
self.execute(
"bench --site {another_site} execute frappe.utils.backups.fetch_latest_backups",
site_data,
)
site_data.update({"database": json.loads(self.stdout)["database"]})
self.execute("bench --site {another_site} restore {database}", site_data)
# test 2: restore from partial backup
self.execute("bench --site {another_site} backup --exclude 'ToDo'", site_data)
site_data.update({"kw": "\"{'partial':True}\""})
self.execute(
"bench --site {another_site} execute"
" frappe.utils.backups.fetch_latest_backups --kwargs {kw}",
site_data,
)
site_data.update({"database": json.loads(self.stdout)["database"]})
self.execute("bench --site {another_site} restore {database}", site_data)
self.assertEquals(self.returncode, 1)
def test_partial_restore(self):
_now = now()
for num in range(10):
frappe.get_doc({
"doctype": "ToDo",
"date": add_to_date(_now, days=num),
"description": frappe.mock("paragraph")
}).insert()
frappe.db.commit()
todo_count = frappe.db.count("ToDo")
# check if todos exist, create a partial backup and see if the state is the same after restore
self.assertIsNot(todo_count, 0)
self.execute("bench --site {site} backup --only 'ToDo'")
db_path = fetch_latest_backups(partial=True)["database"]
self.assertTrue("partial" in db_path)
frappe.db.sql_ddl("DROP TABLE IF EXISTS `tabToDo`")
frappe.db.commit()
self.execute("bench --site {site} partial-restore {path}", {"path": db_path})
self.assertEquals(self.returncode, 0)
self.assertEquals(frappe.db.count("ToDo"), todo_count)
def test_recorder(self):
frappe.recorder.stop()
@ -133,7 +334,6 @@ class TestCommands(BaseTestCommands):
self.assertEqual(frappe.recorder.status(), False)
def test_remove_from_installed_apps(self):
from frappe.installer import add_to_installed_apps
app = "test_remove_app"
add_to_installed_apps(app)
@ -164,3 +364,21 @@ class TestCommands(BaseTestCommands):
else:
installed_apps = set(frappe.get_installed_apps())
self.assertSetEqual(list_apps, installed_apps)
def test_get_bench_relative_path(self):
bench_path = frappe.utils.get_bench_path()
test1_path = os.path.join(bench_path, "test1.txt")
test2_path = os.path.join(bench_path, "sites", "test2.txt")
with open(test1_path, "w+") as test1:
test1.write("asdf")
with open(test2_path, "w+") as test2:
test2.write("asdf")
self.assertTrue("test1.txt" in get_bench_relative_path("test1.txt"))
self.assertTrue("sites/test2.txt" in get_bench_relative_path("test2.txt"))
with self.assertRaises(SystemExit):
get_bench_relative_path("test3.txt")
os.remove(test1_path)
os.remove(test2_path)

37
frappe/tests/test_document.py
View file

@ -290,6 +290,41 @@ class TestDocument(unittest.TestCase):
for docname in available_documents:
frappe.delete_doc(doctype, docname)
def test_rename_doctype(self):
from frappe.core.doctype.doctype.test_doctype import new_doctype
fields =[{
"label": "Linked To",
"fieldname": "linked_to_doctype",
"fieldtype": "Link",
"options": "DocType",
"unique": 0
}]
if not frappe.db.exists("DocType", "Rename This"):
new_doctype("Rename This", unique=0, fields=fields).insert()
to_rename_record = frappe.get_doc({
"doctype": "Rename This",
"linked_to_doctype": "Rename This"
})
to_rename_record.insert()
# Rename doctype
self.assertEqual("Renamed Doc", frappe.rename_doc("DocType", "Rename This", "Renamed Doc", force=True))
# Test if Doctype value has changed in Link field
renamed_doctype_record = frappe.get_doc("Renamed Doc", to_rename_record.name)
self.assertEqual(renamed_doctype_record.linked_to_doctype, "Renamed Doc")
# Test if there are conflicts between a record and a DocType
# having the same name
old_name = to_rename_record.name
new_name = "ToDo"
self.assertEqual(new_name, frappe.rename_doc("Renamed Doc", old_name, new_name, force=True))
frappe.delete_doc_if_exists("Renamed Doc", "ToDo")
frappe.delete_doc_if_exists("DocType", "Renamed Doc")
def test_non_negative_check(self):
frappe.delete_doc_if_exists("Currency", "Frappe Coin", 1)
@ -305,4 +340,4 @@ class TestDocument(unittest.TestCase):
d.insert()
self.assertEqual(frappe.db.get_value("Currency", d.name), d.name)
frappe.delete_doc_if_exists("Currency", "Frappe Coin", 1)
frappe.delete_doc_if_exists("Currency", "Frappe Coin", 1)

11
frappe/tests/test_oauth20.py
View file

@ -6,6 +6,7 @@ import unittest, frappe, requests, time
from frappe.test_runner import make_test_records
from six.moves.urllib.parse import urlparse, parse_qs, urljoin
from urllib.parse import urlencode, quote
from frappe.integrations.oauth2 import encode_params
class TestOAuth20(unittest.TestCase):
@ -232,13 +233,3 @@ def login(session):
def get_full_url(endpoint):
"""Turn '/endpoint' into 'http://127.0.0.1:8000/endpoint'."""
return urljoin(frappe.utils.get_url(), endpoint)
def encode_params(params):
"""
Encode a dict of params into a query string.
Use `quote_via=urllib.parse.quote` so that whitespaces will be encoded as
`%20` instead of as `+`. This is needed because oauthlib cannot handle `+`
as a whitespace.
"""
return urlencode(params, quote_via=quote)

15
frappe/tests/test_utils.py
View file

@ -7,6 +7,10 @@ import unittest
from frappe.utils import evaluate_filters, money_in_words, scrub_urls, get_url
from frappe.utils import ceil, floor
from PIL import Image
from frappe.utils.image import strip_exif_data
import io
class TestFilters(unittest.TestCase):
def test_simple_dict(self):
self.assertTrue(evaluate_filters({'doctype': 'User', 'status': 'Open'}, {'status': 'Open'}))
@ -122,3 +126,14 @@ class TestHTMLUtils(unittest.TestCase):
clean = clean_email_html(sample)
self.assertTrue('<h1>Hello</h1>' in clean)
self.assertTrue('<a href="http://test.com">text</a>' in clean)
class TestImage(unittest.TestCase):
def test_strip_exif_data(self):
original_image = Image.open("../apps/frappe/frappe/tests/data/exif_sample_image.jpg")
original_image_content = io.open("../apps/frappe/frappe/tests/data/exif_sample_image.jpg", mode='rb').read()
new_image_content = strip_exif_data(original_image_content, "image/jpeg")
new_image = Image.open(io.BytesIO(new_image_content))
self.assertEqual(new_image._getexif(), None)
self.assertNotEqual(original_image._getexif(), new_image._getexif())

33
frappe/utils/__init__.py
View file

@ -66,9 +66,14 @@ def get_email_address(user=None):
def get_formatted_email(user, mail=None):
"""get Email Address of user formatted as: `John Doe <johndoe@example.com>`"""
fullname = get_fullname(user)
if not mail:
mail = get_email_address(user)
return cstr(make_header(decode_header(formataddr((fullname, mail)))))
mail = get_email_address(user) or validate_email_address(user)
if not mail:
return ''
else:
return cstr(make_header(decode_header(formataddr((fullname, mail)))))
def extract_email_id(email):
"""fetch only the email part of the Email Address"""
@ -729,3 +734,27 @@ def get_build_version():
# .build can sometimes not exist
# this is not a major problem so send fallback
return frappe.utils.random_string(8)
def get_bench_relative_path(file_path):
"""Fixes paths relative to the bench root directory if exists and returns the absolute path
Args:
file_path (str, Path): Path of a file that exists on the file system
Returns:
str: Absolute path of the file_path
"""
if not os.path.exists(file_path):
base_path = '..'
elif file_path.startswith(os.sep):
base_path = os.sep
else:
base_path = '.'
file_path = os.path.join(base_path, file_path)
if not os.path.exists(file_path):
print('Invalid path {0}'.format(file_path[3:]))
sys.exit(1)
return os.path.abspath(file_path)

555
frappe/utils/backups.py
View file

@ -2,11 +2,12 @@
# MIT License. See license.txt
# imports - standard imports
import json
import gzip
import os
from calendar import timegm
from datetime import datetime
from glob import glob
from shutil import which
# imports - third party imports
import click
@ -14,24 +15,42 @@ import click
# imports - module imports
import frappe
from frappe import _, conf
from frappe.utils import get_url, now, now_datetime, get_file_size
from frappe.utils import get_file_size, get_url, now, now_datetime
# backup variable for backwards compatibility
verbose = False
compress = False
_verbose = verbose
base_tables = ["__Auth", "__global_search", "__UserSettings"]
class BackupGenerator:
"""
This class contains methods to perform On Demand Backup
This class contains methods to perform On Demand Backup
To initialize, specify (db_name, user, password, db_file_name=None, db_host="localhost")
If specifying db_file_name, also append ".sql.gz"
To initialize, specify (db_name, user, password, db_file_name=None, db_host="localhost")
If specifying db_file_name, also append ".sql.gz"
"""
def __init__(self, db_name, user, password, backup_path=None, backup_path_db=None,
backup_path_files=None, backup_path_private_files=None, db_host="localhost", db_port=None,
verbose=False, db_type='mariadb', backup_path_conf=None, compress_files=False):
def __init__(
self,
db_name,
user,
password,
backup_path=None,
backup_path_db=None,
backup_path_files=None,
backup_path_private_files=None,
db_host="localhost",
db_port=None,
db_type="mariadb",
backup_path_conf=None,
ignore_conf=False,
compress_files=False,
include_doctypes="",
exclude_doctypes="",
verbose=False,
):
global _verbose
self.compress_files = compress_files or compress
self.db_host = db_host
@ -45,23 +64,35 @@ class BackupGenerator:
self.backup_path_db = backup_path_db
self.backup_path_files = backup_path_files
self.backup_path_private_files = backup_path_private_files
self.ignore_conf = ignore_conf
self.include_doctypes = include_doctypes
self.exclude_doctypes = exclude_doctypes
self.partial = False
if not self.db_type:
self.db_type = 'mariadb'
self.db_type = "mariadb"
if not self.db_port and self.db_type == 'mariadb':
self.db_port = 3306
elif not self.db_port and self.db_type == 'postgres':
self.db_port = 5432
if not self.db_port:
if self.db_type == "mariadb":
self.db_port = 3306
if self.db_type == "postgres":
self.db_port = 5432
site = frappe.local.site or frappe.generate_hash(length=8)
self.site_slug = site.replace('.', '_')
self.site_slug = site.replace(".", "_")
self.verbose = verbose
self.setup_backup_directory()
self.setup_backup_tables()
_verbose = verbose
def setup_backup_directory(self):
specified = self.backup_path or self.backup_path_db or self.backup_path_files or self.backup_path_private_files or self.backup_path_conf
specified = (
self.backup_path
or self.backup_path_db
or self.backup_path_files
or self.backup_path_private_files
or self.backup_path_conf
)
if not specified:
backups_folder = get_backup_path()
@ -71,32 +102,93 @@ class BackupGenerator:
if self.backup_path:
os.makedirs(self.backup_path, exist_ok=True)
for file_path in set([self.backup_path_files, self.backup_path_db, self.backup_path_private_files, self.backup_path_conf]):
for file_path in set(
[
self.backup_path_files,
self.backup_path_db,
self.backup_path_private_files,
self.backup_path_conf,
]
):
if file_path:
dir = os.path.dirname(file_path)
os.makedirs(dir, exist_ok=True)
def setup_backup_tables(self):
"""Sets self.backup_includes, self.backup_excludes based on passed args"""
existing_doctypes = set([x.name for x in frappe.get_all("DocType")])
def get_tables(doctypes):
tables = []
for doctype in doctypes:
if doctype and doctype in existing_doctypes:
if doctype.startswith("tab"):
tables.append(doctype)
else:
tables.append("tab" + doctype)
return tables
passed_tables = {
"include": get_tables(self.include_doctypes.strip().split(",")),
"exclude": get_tables(self.exclude_doctypes.strip().split(",")),
}
specified_tables = get_tables(frappe.conf.get("backup", {}).get("includes", []))
include_tables = (specified_tables + base_tables) if specified_tables else []
conf_tables = {
"include": include_tables,
"exclude": get_tables(frappe.conf.get("backup", {}).get("excludes", [])),
}
self.backup_includes = passed_tables["include"]
self.backup_excludes = passed_tables["exclude"]
if not (self.backup_includes or self.backup_excludes) and not self.ignore_conf:
self.backup_includes = self.backup_includes or conf_tables["include"]
self.backup_excludes = self.backup_excludes or conf_tables["exclude"]
self.partial = (self.backup_includes or self.backup_excludes) and not self.ignore_conf
@property
def site_config_backup_path(self):
# For backwards compatibility
click.secho("BackupGenerator.site_config_backup_path has been deprecated in favour of BackupGenerator.backup_path_conf", fg="yellow")
click.secho(
"BackupGenerator.site_config_backup_path has been deprecated in favour of"
" BackupGenerator.backup_path_conf",
fg="yellow",
)
return getattr(self, "backup_path_conf", None)
def get_backup(self, older_than=24, ignore_files=False, force=False):
"""
Takes a new dump if existing file is old
and sends the link to the file as email
Takes a new dump if existing file is old
and sends the link to the file as email
"""
#Check if file exists and is less than a day old
#If not Take Dump
# Check if file exists and is less than a day old
# If not Take Dump
if not force:
last_db, last_file, last_private_file, site_config_backup_path = self.get_recent_backup(older_than)
(
last_db,
last_file,
last_private_file,
site_config_backup_path,
) = self.get_recent_backup(older_than)
else:
last_db, last_file, last_private_file, site_config_backup_path = False, False, False, False
last_db, last_file, last_private_file, site_config_backup_path = (
False,
False,
False,
False,
)
self.todays_date = now_datetime().strftime('%Y%m%d_%H%M%S')
self.todays_date = now_datetime().strftime("%Y%m%d_%H%M%S")
if not (self.backup_path_conf and self.backup_path_db and self.backup_path_files and self.backup_path_private_files):
if not (
self.backup_path_conf
and self.backup_path_db
and self.backup_path_files
and self.backup_path_private_files
):
self.set_backup_file_name()
if not (last_db and last_file and last_private_file and site_config_backup_path):
@ -112,13 +204,13 @@ class BackupGenerator:
self.backup_path_conf = site_config_backup_path
def set_backup_file_name(self):
#Generate a random name using today's date and a 8 digit random number
for_conf = self.todays_date + "-" + self.site_slug + "-site_config_backup.json"
for_db = self.todays_date + "-" + self.site_slug + "-database.sql.gz"
partial = "-partial" if self.partial else ""
ext = "tgz" if self.compress_files else "tar"
for_public_files = self.todays_date + "-" + self.site_slug + "-files." + ext
for_private_files = self.todays_date + "-" + self.site_slug + "-private-files." + ext
for_conf = f"{self.todays_date}-{self.site_slug}-site_config_backup.json"
for_db = f"{self.todays_date}-{self.site_slug}{partial}-database.sql.gz"
for_public_files = f"{self.todays_date}-{self.site_slug}-files.{ext}"
for_private_files = f"{self.todays_date}-{self.site_slug}-private-files.{ext}"
backup_path = self.backup_path or get_backup_path()
if not self.backup_path_conf:
@ -130,11 +222,11 @@ class BackupGenerator:
if not self.backup_path_private_files:
self.backup_path_private_files = os.path.join(backup_path, for_private_files)
def get_recent_backup(self, older_than):
def get_recent_backup(self, older_than, partial=False):
backup_path = get_backup_path()
file_type_slugs = {
"database": "*-{}-database.sql.gz",
"database": "*-{{}}-{}database.sql.gz".format('*' if partial else ''),
"public": "*-{}-files.tar",
"private": "*-{}-private-files.tar",
"config": "*-{}-site_config_backup.json",
@ -158,8 +250,7 @@ class BackupGenerator:
return file_path
latest_backups = {
file_type: get_latest(pattern)
for file_type, pattern in file_type_slugs.items()
file_type: get_latest(pattern) for file_type, pattern in file_type_slugs.items()
}
recent_backups = {
@ -175,32 +266,40 @@ class BackupGenerator:
def zip_files(self):
# For backwards compatibility - pre v13
click.secho("BackupGenerator.zip_files has been deprecated in favour of BackupGenerator.backup_files", fg="yellow")
click.secho(
"BackupGenerator.zip_files has been deprecated in favour of"
" BackupGenerator.backup_files",
fg="yellow",
)
return self.backup_files()
def get_summary(self):
summary = {
"config": {
"path": self.backup_path_conf,
"size": get_file_size(self.backup_path_conf, format=True)
"size": get_file_size(self.backup_path_conf, format=True),
},
"database": {
"path": self.backup_path_db,
"size": get_file_size(self.backup_path_db, format=True)
}
"size": get_file_size(self.backup_path_db, format=True),
},
}
if os.path.exists(self.backup_path_files) and os.path.exists(self.backup_path_private_files):
summary.update({
"public": {
"path": self.backup_path_files,
"size": get_file_size(self.backup_path_files, format=True)
},
"private": {
"path": self.backup_path_private_files,
"size": get_file_size(self.backup_path_private_files, format=True)
if os.path.exists(self.backup_path_files) and os.path.exists(
self.backup_path_private_files
):
summary.update(
{
"public": {
"path": self.backup_path_files,
"size": get_file_size(self.backup_path_files, format=True),
},
"private": {
"path": self.backup_path_private_files,
"size": get_file_size(self.backup_path_private_files, format=True),
},
}
})
)
return summary
@ -208,21 +307,29 @@ class BackupGenerator:
backup_summary = self.get_summary()
print("Backup Summary for {0} at {1}".format(frappe.local.site, now()))
title = max([len(x) for x in backup_summary])
path = max([len(x["path"]) for x in backup_summary.values()])
for _type, info in backup_summary.items():
print("{0:8}: {1:85} {2}".format(_type.title(), info["path"], info["size"]))
template = "{{0:{0}}}: {{1:{1}}} {{2}}".format(title, path)
print(template.format(_type.title(), info["path"], info["size"]))
def backup_files(self):
import subprocess
for folder in ("public", "private"):
files_path = frappe.get_site_path(folder, "files")
backup_path = self.backup_path_files if folder=="public" else self.backup_path_private_files
backup_path = (
self.backup_path_files if folder == "public" else self.backup_path_private_files
)
if self.compress_files:
cmd_string = "tar cf - {1} | gzip > {0}"
else:
cmd_string = "tar -cf {0} {1}"
output = subprocess.check_output(cmd_string.format(backup_path, files_path), shell=True)
output = subprocess.check_output(
cmd_string.format(backup_path, files_path), shell=True
)
if self.verbose and output:
print(output.decode("utf8"))
@ -236,34 +343,114 @@ class BackupGenerator:
def take_dump(self):
import frappe.utils
from frappe.utils.change_log import get_app_branch
db_exc = {
"mariadb": ("mysqldump", which("mysqldump")),
"postgres": ("pg_dump", which("pg_dump")),
}[self.db_type]
gzip_exc = which("gzip")
if not (gzip_exc and db_exc[1]):
_exc = "gzip" if not gzip_exc else db_exc[0]
frappe.throw(
f"{_exc} not found in PATH! This is required to take a backup.",
exc=frappe.ExecutableNotFound
)
db_exc = db_exc[0]
database_header_content = [
f"Backup generated by Frappe {frappe.__version__} on branch {get_app_branch('frappe') or 'N/A'}",
"",
]
# escape reserved characters
args = dict([item[0], frappe.utils.esc(str(item[1]), '$ ')]
for item in self.__dict__.copy().items())
args = frappe._dict(
[item[0], frappe.utils.esc(str(item[1]), "$ ")]
for item in self.__dict__.copy().items()
)
cmd_string = """mysqldump --single-transaction --quick --lock-tables=false -u %(user)s -p%(password)s %(db_name)s -h %(db_host)s -P %(db_port)s | gzip > %(backup_path_db)s """ % args
if self.backup_includes:
backup_info = ("Backing Up Tables: ", ", ".join(self.backup_includes))
elif self.backup_excludes:
backup_info = ("Skipping Tables: ", ", ".join(self.backup_excludes))
if self.db_type == 'postgres':
cmd_string = "pg_dump postgres://{user}:{password}@{db_host}:{db_port}/{db_name} | gzip > {backup_path_db}".format(
user=args.get('user'),
password=args.get('password'),
db_host=args.get('db_host'),
db_port=args.get('db_port'),
db_name=args.get('db_name'),
backup_path_db=args.get('backup_path_db')
if self.partial:
print(''.join(backup_info), "\n")
database_header_content.extend([
f"Partial Backup of Frappe Site {frappe.local.site}",
("Backup contains: " if self.backup_includes else "Backup excludes: ") + backup_info[1],
"",
])
generated_header = "\n".join([f"-- {x}" for x in database_header_content]) + "\n"
with gzip.open(args.backup_path_db, "wt") as f:
f.write(generated_header)
if self.db_type == "postgres":
if self.backup_includes:
args["include"] = " ".join(
["--table='public.\"{0}\"'".format(table) for table in self.backup_includes]
)
elif self.backup_excludes:
args["exclude"] = " ".join(
["--exclude-table-data='public.\"{0}\"'".format(table) for table in self.backup_excludes]
)
cmd_string = (
"{db_exc} postgres://{user}:{password}@{db_host}:{db_port}/{db_name}"
" {include} {exclude} | {gzip} >> {backup_path_db}"
)
err, out = frappe.utils.execute_in_shell(cmd_string)
else:
if self.backup_includes:
args["include"] = " ".join(["'{0}'".format(x) for x in self.backup_includes])
elif self.backup_excludes:
args["exclude"] = " ".join(
[
"--ignore-table='{0}.{1}'".format(frappe.conf.db_name, table)
for table in self.backup_excludes
]
)
cmd_string = (
"{db_exc} --single-transaction --quick --lock-tables=false -u {user}"
" -p{password} {db_name} -h {db_host} -P {db_port} {include} {exclude}"
" | {gzip} >> {backup_path_db}"
)
command = cmd_string.format(
user=args.user,
password=args.password,
db_exc=db_exc,
db_host=args.db_host,
db_port=args.db_port,
db_name=args.db_name,
backup_path_db=args.backup_path_db,
exclude=args.get("exclude", ""),
include=args.get("include", ""),
gzip=gzip_exc,
)
if self.verbose:
print(command + "\n")
err, out = frappe.utils.execute_in_shell(command)
def send_email(self):
"""
Sends the link to backup file located at erpnext/backups
Sends the link to backup file located at erpnext/backups
"""
from frappe.email import get_system_managers
recipient_list = get_system_managers()
db_backup_url = get_url(os.path.join('backups', os.path.basename(self.backup_path_db)))
files_backup_url = get_url(os.path.join('backups', os.path.basename(self.backup_path_files)))
db_backup_url = get_url(
os.path.join("backups", os.path.basename(self.backup_path_db))
)
files_backup_url = get_url(
os.path.join("backups", os.path.basename(self.backup_path_files))
)
msg = """Hello,
@ -275,11 +462,13 @@ Your backups are ready to be downloaded.
This link will be valid for 24 hours. A new backup will be available for
download only after 24 hours.""" % {
"db_backup_url": db_backup_url,
"files_backup_url": files_backup_url
"files_backup_url": files_backup_url,
}
datetime_str = datetime.fromtimestamp(os.stat(self.backup_path_db).st_ctime)
subject = datetime_str.strftime("%d/%m/%Y %H:%M:%S") + """ - Backup ready to be downloaded"""
subject = (
datetime_str.strftime("%d/%m/%Y %H:%M:%S") + """ - Backup ready to be downloaded"""
)
frappe.sendmail(recipients=recipient_list, msg=msg, subject=subject)
return recipient_list
@ -288,20 +477,29 @@ download only after 24 hours.""" % {
@frappe.whitelist()
def get_backup():
"""
This function is executed when the user clicks on
Toos > Download Backup
This function is executed when the user clicks on
Toos > Download Backup
"""
delete_temp_backups()
odb = BackupGenerator(frappe.conf.db_name, frappe.conf.db_name,\
frappe.conf.db_password, db_host = frappe.db.host,\
db_type=frappe.conf.db_type, db_port=frappe.conf.db_port)
odb = BackupGenerator(
frappe.conf.db_name,
frappe.conf.db_name,
frappe.conf.db_password,
db_host=frappe.db.host,
db_type=frappe.conf.db_type,
db_port=frappe.conf.db_port,
)
odb.get_backup()
recipient_list = odb.send_email()
frappe.msgprint(_("Download link for your backup will be emailed on the following email address: {0}").format(', '.join(recipient_list)))
frappe.msgprint(
_(
"Download link for your backup will be emailed on the following email address: {0}"
).format(", ".join(recipient_list))
)
@frappe.whitelist()
def fetch_latest_backups():
def fetch_latest_backups(partial=False):
"""Fetches paths of the latest backup taken in the last 30 days
Only for: System Managers
@ -317,43 +515,88 @@ def fetch_latest_backups():
db_type=frappe.conf.db_type,
db_port=frappe.conf.db_port,
)
database, public, private, config = odb.get_recent_backup(older_than=24 * 30)
database, public, private, config = odb.get_recent_backup(older_than=24 * 30, partial=partial)
return {
"database": database,
"public": public,
"private": private,
"config": config
}
return {"database": database, "public": public, "private": private, "config": config}
def scheduled_backup(older_than=6, ignore_files=False, backup_path=None, backup_path_db=None, backup_path_files=None, backup_path_private_files=None, backup_path_conf=None, force=False, verbose=False, compress=False):
def scheduled_backup(
older_than=6,
ignore_files=False,
backup_path=None,
backup_path_db=None,
backup_path_files=None,
backup_path_private_files=None,
backup_path_conf=None,
ignore_conf=False,
include_doctypes="",
exclude_doctypes="",
compress=False,
force=False,
verbose=False,
):
"""this function is called from scheduler
deletes backups older than 7 days
takes backup"""
odb = new_backup(older_than, ignore_files, backup_path=backup_path, backup_path_db=backup_path_db, backup_path_files=backup_path_files, backup_path_private_files=backup_path_private_files, backup_path_conf=backup_path_conf, force=force, verbose=verbose, compress=compress)
deletes backups older than 7 days
takes backup"""
odb = new_backup(
older_than=older_than,
ignore_files=ignore_files,
backup_path=backup_path,
backup_path_db=backup_path_db,
backup_path_files=backup_path_files,
backup_path_private_files=backup_path_private_files,
backup_path_conf=backup_path_conf,
ignore_conf=ignore_conf,
include_doctypes=include_doctypes,
exclude_doctypes=exclude_doctypes,
compress=compress,
force=force,
verbose=verbose,
)
return odb
def new_backup(older_than=6, ignore_files=False, backup_path=None, backup_path_db=None, backup_path_files=None, backup_path_private_files=None, backup_path_conf=None, force=False, verbose=False, compress=False):
delete_temp_backups(older_than = frappe.conf.keep_backups_for_hours or 24)
odb = BackupGenerator(frappe.conf.db_name, frappe.conf.db_name,\
frappe.conf.db_password,
backup_path=backup_path,
backup_path_db=backup_path_db,
backup_path_files=backup_path_files,
backup_path_private_files=backup_path_private_files,
backup_path_conf=backup_path_conf,
db_host = frappe.db.host,
db_port = frappe.db.port,
db_type = frappe.conf.db_type,
verbose=verbose,
compress_files=compress)
def new_backup(
older_than=6,
ignore_files=False,
backup_path=None,
backup_path_db=None,
backup_path_files=None,
backup_path_private_files=None,
backup_path_conf=None,
ignore_conf=False,
include_doctypes="",
exclude_doctypes="",
compress=False,
force=False,
verbose=False,
):
delete_temp_backups(older_than=frappe.conf.keep_backups_for_hours or 24)
odb = BackupGenerator(
frappe.conf.db_name,
frappe.conf.db_name,
frappe.conf.db_password,
db_host=frappe.db.host,
db_port=frappe.db.port,
db_type=frappe.conf.db_type,
backup_path=backup_path,
backup_path_db=backup_path_db,
backup_path_files=backup_path_files,
backup_path_private_files=backup_path_private_files,
backup_path_conf=backup_path_conf,
ignore_conf=ignore_conf,
include_doctypes=include_doctypes,
exclude_doctypes=exclude_doctypes,
verbose=verbose,
compress_files=compress,
)
odb.get_backup(older_than, ignore_files, force=force)
return odb
def delete_temp_backups(older_than=24):
"""
Cleans up the backup_link_path directory by deleting files older than 24 hours
Cleans up the backup_link_path directory by deleting files older than 24 hours
"""
backup_path = get_backup_path()
if os.path.exists(backup_path):
@ -363,54 +606,68 @@ def delete_temp_backups(older_than=24):
if is_file_old(this_file_path, older_than):
os.remove(this_file_path)
def is_file_old(db_file_name, older_than=24):
"""
Checks if file exists and is older than specified hours
Returns ->
True: file does not exist or file is old
False: file is new
"""
if os.path.isfile(db_file_name):
from datetime import timedelta
#Get timestamp of the file
file_datetime = datetime.fromtimestamp\
(os.stat(db_file_name).st_ctime)
if datetime.today() - file_datetime >= timedelta(hours = older_than):
if _verbose:
print("File is old")
return True
else:
if _verbose:
print("File is recent")
return False
def is_file_old(file_path, older_than=24):
"""
Checks if file exists and is older than specified hours
Returns ->
True: file does not exist or file is old
False: file is new
"""
if os.path.isfile(file_path):
from datetime import timedelta
# Get timestamp of the file
file_datetime = datetime.fromtimestamp(os.stat(file_path).st_ctime)
if datetime.today() - file_datetime >= timedelta(hours=older_than):
if _verbose:
print(f"File {file_path} is older than {older_than} hours")
return True
else:
if _verbose:
print("File does not exist")
return True
print(f"File {file_path} is recent")
return False
else:
if _verbose:
print(f"File {file_path} does not exist")
return True
def get_backup_path():
backup_path = frappe.utils.get_site_path(conf.get("backup_path", "private/backups"))
return backup_path
def backup(with_files=False, backup_path_db=None, backup_path_files=None, backup_path_private_files=None, backup_path_conf=None, quiet=False):
def backup(
with_files=False,
backup_path_db=None,
backup_path_files=None,
backup_path_private_files=None,
backup_path_conf=None,
quiet=False,
):
"Backup"
odb = scheduled_backup(ignore_files=not with_files, backup_path_db=backup_path_db, backup_path_files=backup_path_files, backup_path_private_files=backup_path_private_files, backup_path_conf=backup_path_conf, force=True)
odb = scheduled_backup(
ignore_files=not with_files,
backup_path_db=backup_path_db,
backup_path_files=backup_path_files,
backup_path_private_files=backup_path_private_files,
backup_path_conf=backup_path_conf,
force=True,
)
return {
"backup_path_db": odb.backup_path_db,
"backup_path_files": odb.backup_path_files,
"backup_path_private_files": odb.backup_path_private_files
"backup_path_private_files": odb.backup_path_private_files,
}
if __name__ == "__main__":
"""
is_file_old db_name user password db_host db_type db_port
get_backup db_name user password db_host db_type db_port
"""
import sys
cmd = sys.argv[1]
db_type = 'mariadb'
db_type = "mariadb"
try:
db_type = sys.argv[6]
except IndexError:
@ -423,19 +680,47 @@ if __name__ == "__main__":
pass
if cmd == "is_file_old":
odb = BackupGenerator(sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5] or "localhost", db_type=db_type, db_port=db_port)
odb = BackupGenerator(
sys.argv[2],
sys.argv[3],
sys.argv[4],
sys.argv[5] or "localhost",
db_type=db_type,
db_port=db_port,
)
is_file_old(odb.db_file_name)
if cmd == "get_backup":
odb = BackupGenerator(sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5] or "localhost", db_type=db_type, db_port=db_port)
odb = BackupGenerator(
sys.argv[2],
sys.argv[3],
sys.argv[4],
sys.argv[5] or "localhost",
db_type=db_type,
db_port=db_port,
)
odb.get_backup()
if cmd == "take_dump":
odb = BackupGenerator(sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5] or "localhost", db_type=db_type, db_port=db_port)
odb = BackupGenerator(
sys.argv[2],
sys.argv[3],
sys.argv[4],
sys.argv[5] or "localhost",
db_type=db_type,
db_port=db_port,
)
odb.take_dump()
if cmd == "send_email":
odb = BackupGenerator(sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5] or "localhost", db_type=db_type, db_port=db_port)
odb = BackupGenerator(
sys.argv[2],
sys.argv[3],
sys.argv[4],
sys.argv[5] or "localhost",
db_type=db_type,
db_port=db_port,
)
odb.send_email("abc.sql.gz")
if cmd == "delete_temp_backups":

74
frappe/utils/change_log.py
View file

@ -1,15 +1,17 @@
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt
from __future__ import unicode_literals
from six.moves import range
import json, os
from semantic_version import Version
import frappe
import json
import os
import subprocess # nosec
import requests
import subprocess # nosec
from frappe.utils import cstr
from semantic_version import Version
from six.moves import range
import frappe
from frappe import _, safe_decode
from frappe.utils import cstr
def get_change_log(user=None):
@ -165,9 +167,10 @@ def check_for_update():
add_message_to_redis(updates)
def parse_latest_non_beta_release(response):
"""
Pasrses the response JSON for all the releases and returns the latest non prerelease
Parses the response JSON for all the releases and returns the latest non prerelease
Parameters
response (list): response object returned by github
@ -182,32 +185,51 @@ def parse_latest_non_beta_release(response):
return None
def check_release_on_github(app):
# Check if repo remote is on github
from subprocess import CalledProcessError
def check_release_on_github(app: str):
"""
Check the latest release for a given Frappe application hosted on Github.
Args:
app (str): The name of the Frappe application.
Returns:
tuple(Version, str): The semantic version object of the latest release and the
organization name, if the application exists, otherwise None.
"""
from giturlparse import parse
from giturlparse.parser import ParserError
try:
remote_url = subprocess.check_output("cd ../apps/{} && git ls-remote --get-url".format(app), shell=True).decode()
except CalledProcessError:
# Passing this since some apps may not have git initializaed in them
return None
# Check if repo remote is on github
remote_url = subprocess.check_output("cd ../apps/{} && git ls-remote --get-url".format(app), shell=True)
except subprocess.CalledProcessError:
# Passing this since some apps may not have git initialized in them
return
if isinstance(remote_url, bytes):
remote_url = remote_url.decode()
if "github.com" not in remote_url:
return None
try:
parsed_url = parse(remote_url)
except ParserError:
# Invalid URL
return
# Get latest version from github
if 'https' not in remote_url:
return None
if parsed_url.resource != "github.com":
return
org_name = remote_url.split('/')[3]
r = requests.get('https://api.github.com/repos/{}/{}/releases'.format(org_name, app))
owner = parsed_url.owner
repo = parsed_url.name
# Get latest version from GitHub
r = requests.get(f"https://api.github.com/repos/{owner}/{repo}/releases")
if r.ok:
lastest_non_beta_release = parse_latest_non_beta_release(r.json())
return Version(lastest_non_beta_release), org_name
# In case of an improper response or if there are no releases
return None
latest_non_beta_release = parse_latest_non_beta_release(r.json())
if latest_non_beta_release:
return Version(latest_non_beta_release), owner
def add_message_to_redis(update_json):
# "update-message" will store the update message string

15
frappe/utils/dashboard.py
View file

@ -61,21 +61,6 @@ def generate_and_cache_results(args, function, cache_key, chart):
frappe.db.set_value("Dashboard Chart", args.chart_name, "last_synced_on", frappe.utils.now(), update_modified = False)
return results
def get_from_date_from_timespan(to_date, timespan):
days = months = years = 0
if timespan == "Last Week":
days = -7
if timespan == "Last Month":
months = -1
elif timespan == "Last Quarter":
months = -3
elif timespan == "Last Year":
years = -1
elif timespan == "All Time":
years = -50
return add_to_date(to_date, years=years, months=months, days=days,
as_datetime=True)
def get_dashboards_with_link(docname, doctype):
dashboards = []
links = []

25
frappe/utils/data.py
View file

@ -221,6 +221,27 @@ def get_last_day(dt):
"""
return get_first_day(dt, 0, 1) + datetime.timedelta(-1)
def get_quarter_ending(date):
date = getdate(date)
# find the earliest quarter ending date that is after
# the given date
for month in (3, 6, 9, 12):
quarter_end_month = getdate('{}-{}-01'.format(date.year, month))
quarter_end_date = getdate(get_last_day(quarter_end_month))
if date <= quarter_end_date:
date = quarter_end_date
break
return date
def get_year_ending(date):
''' returns year ending of the given date '''
# first day of next year (note year starts from 1)
date = add_to_date('{}-01-01'.format(date.year), months = 12)
# last day of this month
return add_to_date(date, days=-1)
def get_time(time_str):
if isinstance(time_str, datetime.datetime):
@ -348,6 +369,8 @@ def format_duration(seconds, hide_days=False):
example: converts 12885 to '3h 34m 45s' where 12885 = seconds in float
"""
seconds = cint(seconds)
total_duration = {
'days': math.floor(seconds / (3600 * 24)),
@ -1300,12 +1323,14 @@ def generate_hash(*args, **kwargs):
def guess_date_format(date_string):
DATE_FORMATS = [
r"%d/%b/%y",
r"%d-%m-%Y",
r"%m-%d-%Y",
r"%Y-%m-%d",
r"%d-%m-%y",
r"%m-%d-%y",
r"%y-%m-%d",
r"%y-%b-%d",
r"%d/%m/%Y",
r"%m/%d/%Y",
r"%Y/%m/%d",

57
frappe/utils/dateutils.py
View file

@ -5,10 +5,9 @@ from __future__ import unicode_literals
import frappe
import frappe.defaults
import datetime
from frappe.utils import get_datetime
from frappe.utils import add_to_date, getdate
from frappe.utils.data import get_last_day_of_week
from frappe.desk.doctype.dashboard_chart.dashboard_chart import get_period_ending
from frappe.utils import get_datetime, add_to_date, getdate
from frappe.utils.data import get_first_day, get_first_day_of_week, get_quarter_start, get_year_start,\
get_last_day, get_last_day_of_week, get_quarter_ending, get_year_ending
from six import string_types
# global values -- used for caching
@ -102,4 +101,52 @@ def get_dates_from_timegrain(from_date, to_date, timegrain="Daily"):
else:
date = get_period_ending(add_to_date(dates[-1], years=years, months=months, days=days), timegrain)
dates.append(date)
return dates
return dates
def get_from_date_from_timespan(to_date, timespan):
days = months = years = 0
if timespan == "Last Week":
days = -7
if timespan == "Last Month":
months = -1
elif timespan == "Last Quarter":
months = -3
elif timespan == "Last Year":
years = -1
elif timespan == "All Time":
years = -50
return add_to_date(to_date, years=years, months=months, days=days,
as_datetime=True)
def get_period(date, interval='Monthly'):
date = getdate(date)
months = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]
return {
'Daily': date.strftime('%d-%m-%y'),
'Weekly': date.strftime('%d-%m-%y'),
'Monthly': str(months[date.month - 1]) + ' ' + str(date.year),
'Quarterly': 'Quarter ' + str(((date.month-1)//3)+1) + ' ' + str(date.year),
'Yearly': str(date.year)
}[interval]
def get_period_beginning(date, timegrain, as_str=True):
return getdate({
'Daily': date,
'Weekly': get_first_day_of_week(date),
'Monthly': get_first_day(date),
'Quarterly': get_quarter_start(date),
'Yearly': get_year_start(date)
}[timegrain])
def get_period_ending(date, timegrain):
date = getdate(date)
if timegrain == 'Daily':
return date
else:
return getdate({
'Daily': date,
'Weekly': get_last_day_of_week(date),
'Monthly': get_last_day(date),
'Quarterly': get_quarter_ending(date),
'Yearly': get_year_ending(date)
}[timegrain])

6
frappe/utils/global_search.py
View file

@ -10,6 +10,7 @@ import json
import os
from bs4 import BeautifulSoup
from frappe.utils import cint, strip_html_tags
from frappe.utils.html_utils import unescape_html
from frappe.model.base_document import get_controller
from six import text_type
@ -345,11 +346,8 @@ def get_formatted_value(value, field):
:return:
"""
from six.moves.html_parser import HTMLParser
if getattr(field, 'fieldtype', None) in ["Text", "Text Editor"]:
h = HTMLParser()
value = h.unescape(frappe.safe_decode(value))
value = unescape_html(frappe.safe_decode(value))
value = (re.subn(r'<[\s]*(script|style).*?</\1>(?s)', '', text_type(value))[0])
value = ' '.join(value.split())
return field.label + " : " + strip_html_tags(text_type(value))

7
frappe/utils/html_utils.py
View file

@ -34,7 +34,7 @@ def clean_email_html(html):
'margin', 'margin-top', 'margin-bottom', 'margin-left', 'margin-right',
'padding', 'padding-top', 'padding-bottom', 'padding-left', 'padding-right',
'font-size', 'font-weight', 'font-family', 'text-decoration',
'line-height', 'text-align', 'vertical-align'
'line-height', 'text-align', 'vertical-align', 'display'
],
protocols=['cid', 'http', 'https', 'mailto', 'data'],
strip=True, strip_comments=True)
@ -106,9 +106,8 @@ def get_icon_html(icon, small=False):
return "<i class='{icon}'></i>".format(icon=icon)
def unescape_html(value):
from six.moves.html_parser import HTMLParser
h = HTMLParser()
return h.unescape(value)
from html import unescape
return unescape(value)
# adapted from https://raw.githubusercontent.com/html5lib/html5lib-python/4aa79f113e7486c7ec5d15a6e1777bfe546d3259/html5lib/sanitizer.py
acceptable_elements = [

26
frappe/utils/image.py
View file

@ -5,7 +5,7 @@ from __future__ import unicode_literals, print_function
import os
def resize_images(path, maxdim=700):
import Image
from PIL import Image
size = (maxdim, maxdim)
for basepath, folders, files in os.walk(path):
for fname in files:
@ -17,3 +17,27 @@ def resize_images(path, maxdim=700):
im.save(os.path.join(basepath, fname))
print("resized {0}".format(os.path.join(basepath, fname)))
def strip_exif_data(content, content_type):
""" Strips EXIF from image files which support it.
Works by creating a new Image object which ignores exif by
default and then extracts the binary data back into content.
Returns:
Bytes: Stripped image content
"""
from PIL import Image
import io
original_image = Image.open(io.BytesIO(content))
output = io.BytesIO()
new_image = Image.new(original_image.mode, original_image.size)
new_image.putdata(list(original_image.getdata()))
new_image.save(output, format=content_type.split('/')[1])
content = output.getvalue()
return content

4
frappe/utils/response.py
View file

@ -123,7 +123,7 @@ def make_logs(response = None):
def json_handler(obj):
"""serialize non-serializable data for json"""
# serialize date
import collections
import collections.abc
if isinstance(obj, (datetime.date, datetime.timedelta, datetime.datetime)):
return text_type(obj)
@ -138,7 +138,7 @@ def json_handler(obj):
doc = obj.as_dict(no_nulls=True)
return doc
elif isinstance(obj, collections.Iterable):
elif isinstance(obj, collections.abc.Iterable):
return list(obj)
elif type(obj)==type or isinstance(obj, Exception):

20
frappe/utils/safe_exec.py
View file

@ -13,7 +13,19 @@ from frappe.www.printview import get_visible_columns
import frappe.exceptions
import frappe.integrations.utils
class ServerScriptNotEnabled(frappe.PermissionError): pass
class ServerScriptNotEnabled(frappe.PermissionError):
pass
class NamespaceDict(frappe._dict):
"""Raise AttributeError if function not found in namespace"""
def __getattr__(self, key):
ret = self.get(key)
if (not ret and key.startswith("__")) or (key not in self):
def default_function(*args, **kwargs):
raise AttributeError(f"module has no attribute '{key}'")
return default_function
return ret
def safe_exec(script, _globals=None, _locals=None):
# script reports must be enabled via site_config.json
@ -46,13 +58,13 @@ def get_safe_globals():
user = getattr(frappe.local, "session", None) and frappe.local.session.user or "Guest"
out = frappe._dict(
out = NamespaceDict(
# make available limited methods of frappe
json=json,
dict=dict,
log=frappe.log,
_dict=frappe._dict,
frappe=frappe._dict(
frappe=NamespaceDict(
flags=frappe._dict(),
format=frappe.format_value,
format_value=frappe.format_value,
@ -112,7 +124,7 @@ def get_safe_globals():
out.get_visible_columns = get_visible_columns
out.frappe.date_format = date_format
out.frappe.time_format = time_format
out.frappe.db = frappe._dict(
out.frappe.db = NamespaceDict(
get_list = frappe.get_list,
get_all = frappe.get_all,
get_value = frappe.db.get_value,

1
frappe/www/printview.py
View file

@ -100,6 +100,7 @@ def get_rendered_template(doc, name=None, print_format=None, meta=None,
doc.print_section_headings = print_format.show_section_headings
doc.print_line_breaks = print_format.line_breaks
doc.align_labels_right = print_format.align_labels_right
doc.absolute_value = print_format.absolute_value
def get_template_from_string():
return jenv.from_string(get_print_format(doc.doctype,

4
package.json
View file

@ -32,7 +32,7 @@
"frappe-datatable": "^1.15.3",
"frappe-gantt": "^0.5.0",
"fuse.js": "^3.4.6",
"highlight.js": "^9.18.1",
"highlight.js": "^9.18.2",
"js-sha256": "^0.9.0",
"jsbarcode": "^3.9.0",
"moment": "^2.20.1",
@ -44,7 +44,7 @@
"qz-tray": "^2.0.8",
"redis": "^2.8.0",
"showdown": "^1.9.1",
"snyk": "^1.398.1",
"snyk": "^1.425.4",
"socket.io": "^2.3.0",
"superagent": "^3.8.2",
"touch": "^3.1.0",

Some files were not shown because too many files have changed in this diff Show more