vassili/seitime-frappe
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Merge branch 'develop' into generic_oauth_client

Browse source
This commit is contained in:
Raffael Meyer 2020-11-23 12:25:40 +01:00 • committed by GitHub
commit af0d2085d4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
33 changed files with 1253 additions and 442 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
frappe/build.py
View file

@ -105,7 +105,7 @@ def download_frappe_assets(verbose=True):
if frappe_head:
try:
url = get_assets_link(frappe_head)
click.secho("Retreiving assets...", fg="yellow")
click.secho("Retrieving assets...", fg="yellow")
prefix = mkdtemp(prefix="frappe-assets-", suffix=frappe_head)
assets_archive = download_file(url, prefix)
print("\n{0} Downloaded Frappe assets from {1}".format(green('✔'), url))

160
frappe/commands/site.py
View file

@ -9,7 +9,7 @@ import click
import frappe
from frappe.commands import get_site, pass_context
from frappe.exceptions import SiteNotSpecifiedError
from frappe.utils import get_site_path, touch_file
from frappe.installer import _new_site
@click.command('new-site')
@ -42,57 +42,6 @@ def new_site(site, mariadb_root_username=None, mariadb_root_password=None, admin
if len(frappe.utils.get_sites()) == 1:
use(site)
def _new_site(db_name, site, mariadb_root_username=None, mariadb_root_password=None,
admin_password=None, verbose=False, install_apps=None, source_sql=None, force=False,
no_mariadb_socket=False, reinstall=False, db_password=None, db_type=None, db_host=None,
db_port=None, new_site=False):
"""Install a new Frappe site"""
if not force and os.path.exists(site):
print('Site {0} already exists'.format(site))
sys.exit(1)
if no_mariadb_socket and not db_type == "mariadb":
print('--no-mariadb-socket requires db_type to be set to mariadb.')
sys.exit(1)
if not db_name:
import hashlib
db_name = '_' + hashlib.sha1(site.encode()).hexdigest()[:16]
from frappe.commands.scheduler import _is_scheduler_enabled
from frappe.installer import install_db, make_site_dirs
from frappe.installer import install_app as _install_app
import frappe.utils.scheduler
frappe.init(site=site)
try:
# enable scheduler post install?
enable_scheduler = _is_scheduler_enabled()
except Exception:
enable_scheduler = False
make_site_dirs()
installing = touch_file(get_site_path('locks', 'installing.lock'))
install_db(root_login=mariadb_root_username, root_password=mariadb_root_password, db_name=db_name,
admin_password=admin_password, verbose=verbose, source_sql=source_sql, force=force, reinstall=reinstall,
db_password=db_password, db_type=db_type, db_host=db_host, db_port=db_port, no_mariadb_socket=no_mariadb_socket)
apps_to_install = ['frappe'] + (frappe.conf.get("install_apps") or []) + (list(install_apps) or [])
for app in apps_to_install:
_install_app(app, verbose=verbose, set_as_patched=not source_sql)
os.remove(installing)
frappe.utils.scheduler.toggle_scheduler(enable_scheduler)
frappe.db.commit()
scheduler_status = "disabled" if frappe.utils.scheduler.is_scheduler_disabled() else "enabled"
print("*** Scheduler is", scheduler_status, "***")
@click.command('restore')
@click.argument('sql-file-path')
@ -107,33 +56,41 @@ def _new_site(db_name, site, mariadb_root_username=None, mariadb_root_password=N
@pass_context
def restore(context, sql_file_path, mariadb_root_username=None, mariadb_root_password=None, db_name=None, verbose=None, install_app=None, admin_password=None, force=None, with_public_files=None, with_private_files=None):
"Restore site database from an sql file"
from frappe.installer import extract_sql_gzip, extract_files, is_downgrade, validate_database_sql
from frappe.installer import (
extract_sql_from_archive,
extract_files,
is_downgrade,
is_partial,
validate_database_sql
)
force = context.force or force
decompressed_file_name = extract_sql_from_archive(sql_file_path)
# Extract the gzip file if user has passed *.sql.gz file instead of *.sql file
if not os.path.exists(sql_file_path):
base_path = '..'
sql_file_path = os.path.join(base_path, sql_file_path)
if not os.path.exists(sql_file_path):
print('Invalid path {0}'.format(sql_file_path[3:]))
sys.exit(1)
elif sql_file_path.startswith(os.sep):
base_path = os.sep
else:
base_path = '.'
if sql_file_path.endswith('sql.gz'):
decompressed_file_name = extract_sql_gzip(os.path.abspath(sql_file_path))
else:
decompressed_file_name = sql_file_path
# check if partial backup
if is_partial(decompressed_file_name):
click.secho(
"Partial Backup file detected. You cannot use a partial file to restore a Frappe Site.",
fg="red"
)
click.secho(
"Use `bench partial-restore` to restore a partial backup to an existing site.",
fg="yellow"
)
sys.exit(1)
# check if valid SQL file
validate_database_sql(decompressed_file_name, _raise=not force)
site = get_site(context)
frappe.init(site=site)
# dont allow downgrading to older versions of frappe without force
if not force and is_downgrade(decompressed_file_name, verbose=True):
warn_message = "This is not recommended and may lead to unexpected behaviour. Do you want to continue anyway?"
warn_message = (
"This is not recommended and may lead to unexpected behaviour. "
"Do you want to continue anyway?"
)
click.confirm(warn_message, abort=True)
_new_site(frappe.conf.db_name, site, mariadb_root_username=mariadb_root_username,
@ -156,9 +113,28 @@ def restore(context, sql_file_path, mariadb_root_username=None, mariadb_root_pas
if decompressed_file_name != sql_file_path:
os.remove(decompressed_file_name)
success_message = "Site {0} has been restored{1}".format(site, " with files" if (with_public_files or with_private_files) else "")
success_message = "Site {0} has been restored{1}".format(
site,
" with files" if (with_public_files or with_private_files) else ""
)
click.secho(success_message, fg="green")
@click.command('partial-restore')
@click.argument('sql-file-path')
@click.option("--verbose", "-v", is_flag=True)
@pass_context
def partial_restore(context, sql_file_path, verbose):
from frappe.installer import partial_restore
verbose = context.verbose or verbose
site = get_site(context)
frappe.init(site=site)
frappe.connect(site=site)
partial_restore(sql_file_path, verbose)
frappe.destroy()
@click.command('reinstall')
@click.option('--admin-password', help='Administrator Password for reinstalled site')
@click.option('--mariadb-root-username', help='Root username for MariaDB')
@ -416,16 +392,20 @@ def use(site, sites_path='.'):
@click.command('backup')
@click.option('--with-files', default=False, is_flag=True, help="Take backup with files")
@click.option('--include', '--only', '-i', default="", type=str, help="Specify the DocTypes to backup seperated by commas")
@click.option('--exclude', '-e', default="", type=str, help="Specify the DocTypes to not backup seperated by commas")
@click.option('--backup-path', default=None, help="Set path for saving all the files in this operation")
@click.option('--backup-path-db', default=None, help="Set path for saving database file")
@click.option('--backup-path-files', default=None, help="Set path for saving public file")
@click.option('--backup-path-private-files', default=None, help="Set path for saving private file")
@click.option('--backup-path-conf', default=None, help="Set path for saving config file")
@click.option('--ignore-backup-conf', default=False, is_flag=True, help="Ignore excludes/includes set in config")
@click.option('--verbose', default=False, is_flag=True, help="Add verbosity")
@click.option('--compress', default=False, is_flag=True, help="Compress private and public files")
@pass_context
def backup(context, with_files=False, backup_path=None, backup_path_db=None, backup_path_files=None,
backup_path_private_files=None, backup_path_conf=None, verbose=False, compress=False):
backup_path_private_files=None, backup_path_conf=None, ignore_backup_conf=False, verbose=False,
compress=False, include="", exclude=""):
"Backup"
from frappe.utils.backups import scheduled_backup
verbose = verbose or context.verbose
@ -435,11 +415,27 @@ def backup(context, with_files=False, backup_path=None, backup_path_db=None, bac
try:
frappe.init(site=site)
frappe.connect()
odb = scheduled_backup(ignore_files=not with_files, backup_path=backup_path, backup_path_db=backup_path_db, backup_path_files=backup_path_files, backup_path_private_files=backup_path_private_files, backup_path_conf=backup_path_conf, force=True, verbose=verbose, compress=compress)
odb = scheduled_backup(
ignore_files=not with_files,
backup_path=backup_path,
backup_path_db=backup_path_db,
backup_path_files=backup_path_files,
backup_path_private_files=backup_path_private_files,
backup_path_conf=backup_path_conf,
ignore_conf=ignore_backup_conf,
include_doctypes=include,
exclude_doctypes=exclude,
compress=compress,
verbose=verbose,
force=True
)
except Exception:
click.secho("Backup failed for Site {0}. Database or site_config.json may be corrupted".format(site), fg="red")
if verbose:
print(frappe.get_traceback())
exit_code = 1
continue
odb.print_summary()
click.secho("Backup for Site {0} has been successfully completed{1}".format(site, " with files" if with_files else ""), fg="green")
frappe.destroy()
@ -512,13 +508,14 @@ def _drop_site(site, root_login='root', root_password=None, archived_sites_path=
if force:
pass
else:
click.echo("="*80)
click.echo("Error: The operation has stopped because backup of {s}'s database failed.".format(s=site))
click.echo("Reason: {reason}{sep}".format(reason=str(err), sep="\n"))
click.echo("Fix the issue and try again.")
click.echo(
"Hint: Use 'bench drop-site {s} --force' to force the removal of {s}".format(sep="\n", tab="\t", s=site)
)
messages = [
"=" * 80,
"Error: The operation has stopped because backup of {0}'s database failed.".format(site),
"Reason: {0}\n".format(str(err)),
"Fix the issue and try again.",
"Hint: Use 'bench drop-site {0} --force' to force the removal of {0}".format(site)
]
click.echo("\n".join(messages))
sys.exit(1)
drop_user_and_database(frappe.conf.db_name, root_login, root_password)
@ -734,5 +731,6 @@ commands = [
stop_recording,
add_to_hosts,
start_ngrok,
build_search_index
build_search_index,
partial_restore
]

3
frappe/core/doctype/doctype/doctype.py
View file

@ -572,7 +572,8 @@ class DocType(Document):
def make_repeatable(self):
"""If allow_auto_repeat is set, add auto_repeat custom field."""
if self.allow_auto_repeat:
if not frappe.db.exists('Custom Field', {'fieldname': 'auto_repeat', 'dt': self.name}):
if not frappe.db.exists('Custom Field', {'fieldname': 'auto_repeat', 'dt': self.name}) and \
not frappe.db.exists('DocField', {'fieldname': 'auto_repeat', 'parent': self.name}):
insert_after = self.fields[len(self.fields) - 1].fieldname
df = dict(fieldname='auto_repeat', label='Auto Repeat', fieldtype='Link', options='Auto Repeat', insert_after=insert_after, read_only=1, no_copy=1, print_hide=1)
create_custom_field(self.name, df)

28
frappe/core/doctype/file/file.py
View file

@ -93,6 +93,7 @@ class File(Document):
self.set_is_private()
self.set_file_name()
self.validate_duplicate_entry()
self.validate_attachment_limit()
self.validate_folder()
if not self.file_url and not self.flags.ignore_file_validate:
@ -140,6 +141,26 @@ class File(Document):
if self.file_url and (self.is_private != self.file_url.startswith('/private')):
frappe.throw(_('Invalid file URL. Please contact System Administrator.'))
def validate_attachment_limit(self):
attachment_limit = 0
if self.attached_to_doctype and self.attached_to_name:
attachment_limit = cint(frappe.get_meta(self.attached_to_doctype).max_attachments)
if attachment_limit:
current_attachment_count = len(frappe.get_all('File', filters={
'attached_to_doctype': self.attached_to_doctype,
'attached_to_name': self.attached_to_name,
}, limit=attachment_limit + 1))
if current_attachment_count >= attachment_limit:
frappe.throw(
_("Maximum Attachment Limit of {0} has been reached for {1} {2}.").format(
frappe.bold(attachment_limit), self.attached_to_doctype, self.attached_to_name
),
exc=frappe.exceptions.AttachmentLimitReached,
title=_('Attachment Limit Reached')
)
def set_folder_name(self):
"""Make parent folders if not exists based on reference doctype and name"""
if self.attached_to_doctype and not self.folder:
@ -612,7 +633,12 @@ def get_extension(filename, extn, content):
return extn
def get_local_image(file_url):
file_path = frappe.get_site_path("public", file_url.lstrip("/"))
if file_url.startswith("/private"):
file_url_path = (file_url.lstrip("/"), )
else:
file_url_path = ("public", file_url.lstrip("/"))
file_path = frappe.get_site_path(*file_url_path)
try:
image = Image.open(file_path)

25
frappe/core/doctype/file/test_file.py
View file

@ -160,6 +160,31 @@ class TestSameContent(unittest.TestCase):
def test_saved_content(self):
self.assertFalse(os.path.exists(get_files_path(self.dup_filename)))
def test_attachment_limit(self):
doctype, docname = make_test_doc()
from frappe.custom.doctype.property_setter.property_setter import make_property_setter
limit_property = make_property_setter('ToDo', None, 'max_attachments', 1, 'int', for_doctype=True)
file1 = frappe.get_doc({
"doctype": "File",
"file_name": 'test-attachment',
"attached_to_doctype": doctype,
"attached_to_name": docname,
"content": 'test'
})
file1.insert()
file2 = frappe.get_doc({
"doctype": "File",
"file_name": 'test-attachment',
"attached_to_doctype": doctype,
"attached_to_name": docname,
"content": 'test2'
})
self.assertRaises(frappe.exceptions.AttachmentLimitReached, file2.insert)
limit_property.delete()
frappe.clear_cache(doctype='ToDo')
def tearDown(self):
# File gets deleted on rollback, so blank

20
frappe/core/doctype/prepared_report/prepared_report.py
View file

@ -89,20 +89,18 @@ def delete_expired_prepared_reports():
'creation': ['<', frappe.utils.add_days(frappe.utils.now(), -expiry_period)]
})
args = {
'reports': prepared_reports_to_delete,
'limit': 50
}
enqueue(method=delete_prepared_reports, job_name="delete_prepared_reports", **args)
batches = frappe.utils.create_batch(prepared_reports_to_delete, 100)
for batch in batches:
args = {
'reports': batch,
}
enqueue(method=delete_prepared_reports, job_name="delete_prepared_reports", **args)
@frappe.whitelist()
def delete_prepared_reports(reports, limit=None):
def delete_prepared_reports(reports):
reports = frappe.parse_json(reports)
for index, doc in enumerate(reports):
if limit and index == limit:
return
frappe.delete_doc('Prepared Report', doc['name'], ignore_permissions=True)
for report in reports:
frappe.delete_doc('Prepared Report', report['name'], ignore_permissions=True)
def create_json_gz_file(data, dt, dn):
# Storing data in CSV file causes information loss

5
frappe/core/doctype/report/report.py
View file

@ -61,8 +61,9 @@ class Report(Document):
def set_doctype_roles(self):
if not self.get('roles') and self.is_standard == 'No':
meta = frappe.get_meta(self.ref_doctype)
roles = [{'role': d.role} for d in meta.permissions if d.permlevel==0]
self.set('roles', roles)
if not meta.istable:
roles = [{'role': d.role} for d in meta.permissions if d.permlevel==0]
self.set('roles', roles)
def is_permitted(self):
"""Returns true if Has Role is not set or the user is allowed."""

8
frappe/core/doctype/user/user.py
View file

@ -13,7 +13,7 @@ from frappe.utils.user import get_system_managers
from bs4 import BeautifulSoup
import frappe.permissions
import frappe.share
import frappe.defaults
from frappe.website.utils import is_signup_enabled
from frappe.utils.background_jobs import enqueue
@ -107,6 +107,10 @@ class User(Document):
)
if self.name not in ('Administrator', 'Guest') and not self.user_image:
frappe.enqueue('frappe.core.doctype.user.user.update_gravatar', name=self.name)
# Set user selected timezone
if self.time_zone:
frappe.defaults.set_default("time_zone", self.time_zone, self.name)
def has_website_permission(self, ptype, user, verbose=False):
"""Returns true if current user is the session user"""
@ -1129,4 +1133,4 @@ def check_password_reset_limit(user, rate_limit):
frappe.throw(_("You have reached the hourly limit for generating password reset links. Please try again later."))
def get_generated_link_count(user):
return cint(frappe.cache().hget("password_reset_link_count", user)) or 0
return cint(frappe.cache().hget("password_reset_link_count", user)) or 0

32
frappe/custom/doctype/customize_form/customize_form.py
View file

@ -39,7 +39,7 @@ class CustomizeForm(Document):
translation = self.get_name_translation()
self.label = translation.translated_text if translation else ''
self.create_auto_repeat_custom_field_if_requried(meta)
self.create_auto_repeat_custom_field_if_required(meta)
# NOTE doc (self) is sent to clientside by run_method
@ -74,19 +74,25 @@ class CustomizeForm(Document):
for d in meta.get(fieldname):
self.append(fieldname, d)
def create_auto_repeat_custom_field_if_requried(self, meta):
def create_auto_repeat_custom_field_if_required(self, meta):
'''
Create auto repeat custom field if it's not already present
'''
if self.allow_auto_repeat:
if not frappe.db.exists('Custom Field', {'fieldname': 'auto_repeat',
'dt': self.doc_type}):
insert_after = self.fields[len(self.fields) - 1].fieldname
df = dict(
fieldname='auto_repeat',
label='Auto Repeat',
fieldtype='Link',
options='Auto Repeat',
insert_after=insert_after,
read_only=1, no_copy=1, print_hide=1)
create_custom_field(self.doc_type, df)
all_fields = [df.fieldname for df in meta.fields]
if "auto_repeat" in all_fields:
return
insert_after = self.fields[len(self.fields) - 1].fieldname
create_custom_field(self.doc_type, dict(
fieldname='auto_repeat',
label='Auto Repeat',
fieldtype='Link',
options='Auto Repeat',
insert_after=insert_after,
read_only=1, no_copy=1, print_hide=1
))
def get_name_translation(self):

11
frappe/database/db_manager.py
View file

@ -3,7 +3,6 @@ import frappe
class DbManager:
def __init__(self, db):
"""
Pass root_conn here for access to all databases.
@ -66,10 +65,10 @@ class DbManager:
esc = make_esc('$ ')
from distutils.spawn import find_executable
pipe = find_executable('pv')
if pipe:
pipe = '{pipe} {source} |'.format(
pipe=pipe,
pv = find_executable('pv')
if pv:
pipe = '{pv} {source} |'.format(
pv=pv,
source=source
)
source = ''
@ -78,7 +77,7 @@ class DbManager:
source = '< {source}'.format(source=source)
if pipe:
print('Creating Database...')
print('Restoring Database file...')
command = '{pipe} mysql -u {user} -p{password} -h{host} ' + ('-P{port}' if frappe.db.port else '') + ' {target} {source}'
command = command.format(

18
frappe/database/mariadb/setup_db.py
View file

@ -1,7 +1,7 @@
from __future__ import unicode_literals
import frappe
import os, sys
import os
from frappe.database.db_manager import DbManager
expected_settings_10_2_earlier = {
@ -86,6 +86,8 @@ def drop_user_and_database(db_name, root_login, root_password):
dbman.drop_database(db_name)
def bootstrap_database(db_name, verbose, source_sql=None):
import sys
frappe.connect(db_name=db_name)
if not check_database_settings():
print('Database settings do not match expected values; stopping database setup.')
@ -94,9 +96,17 @@ def bootstrap_database(db_name, verbose, source_sql=None):
import_db_from_sql(source_sql, verbose)
frappe.connect(db_name=db_name)
if not 'tabDefaultValue' in frappe.db.get_tables():
print('''Database not installed, this can due to lack of permission, or that the database name exists.
Check your mysql root password, or use --force to reinstall''')
if 'tabDefaultValue' not in frappe.db.get_tables():
from click import secho
secho(
"Table 'tabDefaultValue' missing in the restored site. "
"Database not installed correctly, this can due to lack of "
"permission, or that the database name exists. Check your mysql"
" root password, validity of the backup file or use --force to"
" reinstall",
fg="red"
)
sys.exit(1)
def import_db_from_sql(source_sql=None, verbose=False):

65
frappe/database/postgres/setup_db.py
View file

@ -1,5 +1,7 @@
import frappe, subprocess, os
from six.moves import input
import os
import frappe
def setup_database(force, source_sql=None, verbose=False):
root_conn = get_root_connection()
@ -10,24 +12,62 @@ def setup_database(force, source_sql=None, verbose=False):
root_conn.sql("CREATE user {0} password '{1}'".format(frappe.conf.db_name,
frappe.conf.db_password))
root_conn.sql("GRANT ALL PRIVILEGES ON DATABASE `{0}` TO {0}".format(frappe.conf.db_name))
root_conn.close()
bootstrap_database(frappe.conf.db_name, verbose, source_sql=source_sql)
frappe.connect()
def bootstrap_database(db_name, verbose, source_sql=None):
frappe.connect(db_name=db_name)
import_db_from_sql(source_sql, verbose)
frappe.connect(db_name=db_name)
if 'tabDefaultValue' not in frappe.db.get_tables():
import sys
from click import secho
secho(
"Table 'tabDefaultValue' missing in the restored site. "
"This may be due to incorrect permissions or the result of a restore from a bad backup file. "
"Database not installed correctly.",
fg="red"
)
sys.exit(1)
def import_db_from_sql(source_sql=None, verbose=False):
from shutil import which
from subprocess import run, PIPE
# we can't pass psql password in arguments in postgresql as mysql. So
# set password connection parameter in environment variable
subprocess_env = os.environ.copy()
subprocess_env['PGPASSWORD'] = str(frappe.conf.db_password)
# bootstrap db
if not source_sql:
source_sql = os.path.join(os.path.dirname(__file__), 'framework_postgres.sql')
subprocess.check_output([
'psql', frappe.conf.db_name,
'-h', frappe.conf.db_host or 'localhost',
'-p', str(frappe.conf.db_port or '5432'),
'-U', frappe.conf.db_name,
'-f', source_sql
], env=subprocess_env)
pv = which('pv')
frappe.connect()
_command = (
f"psql {frappe.conf.db_name} "
f"-h {frappe.conf.db_host or 'localhost'} -p {str(frappe.conf.db_port or '5432')} "
f"-U {frappe.conf.db_name}"
)
if pv:
command = f"{pv} {source_sql} | " + _command
else:
command = _command + f" -f {source_sql}"
print("Restoring Database file...")
if verbose:
print(command)
restore_proc = run(command, env=subprocess_env, shell=True, stdout=PIPE)
if verbose:
print(f"\nSTDOUT by psql:\n{restore_proc.stdout.decode()}\nImported from Database File: {source_sql}")
def setup_help_database(help_db_name):
root_conn = get_root_connection()
@ -38,19 +78,20 @@ def setup_help_database(help_db_name):
root_conn.sql("GRANT ALL PRIVILEGES ON DATABASE `{0}` TO {0}".format(help_db_name))
def get_root_connection(root_login=None, root_password=None):
import getpass
if not frappe.local.flags.root_connection:
if not root_login:
root_login = frappe.conf.get("root_login") or None
if not root_login:
from six.moves import input
root_login = input("Enter postgres super user: ")
if not root_password:
root_password = frappe.conf.get("root_password") or None
if not root_password:
root_password = getpass.getpass("Postgres super user password: ")
from getpass import getpass
root_password = getpass("Postgres super user password: ")
frappe.local.flags.root_connection = frappe.database.get_db(user=root_login, password=root_password)

4
frappe/exceptions.py
View file

@ -106,8 +106,10 @@ class InvalidDates(ValidationError): pass
class DataTooLongException(ValidationError): pass
class FileAlreadyAttachedException(Exception): pass
class DocumentAlreadyRestored(Exception): pass
class AttachmentLimitReached(Exception): pass
# OAuth exceptions
class InvalidAuthorizationHeader(CSRFTokenError): pass
class InvalidAuthorizationPrefix(CSRFTokenError): pass
class InvalidAuthorizationToken(CSRFTokenError): pass
class InvalidDatabaseFile(ValidationError): pass
class InvalidDatabaseFile(ValidationError): pass
class ExecutableNotFound(FileNotFoundError): pass

237
frappe/installer.py
View file

@ -3,8 +3,90 @@
import json
import os
from frappe.defaults import _clear_cache
import sys
import frappe
from frappe.defaults import _clear_cache
def _new_site(
db_name,
site,
mariadb_root_username=None,
mariadb_root_password=None,
admin_password=None,
verbose=False,
install_apps=None,
source_sql=None,
force=False,
no_mariadb_socket=False,
reinstall=False,
db_password=None,
db_type=None,
db_host=None,
db_port=None,
new_site=False,
):
"""Install a new Frappe site"""
if not force and os.path.exists(site):
print("Site {0} already exists".format(site))
sys.exit(1)
if no_mariadb_socket and not db_type == "mariadb":
print("--no-mariadb-socket requires db_type to be set to mariadb.")
sys.exit(1)
if not db_name:
import hashlib
db_name = "_" + hashlib.sha1(site.encode()).hexdigest()[:16]
frappe.init(site=site)
from frappe.commands.scheduler import _is_scheduler_enabled
from frappe.utils import get_site_path, scheduler, touch_file
try:
# enable scheduler post install?
enable_scheduler = _is_scheduler_enabled()
except Exception:
enable_scheduler = False
make_site_dirs()
installing = touch_file(get_site_path("locks", "installing.lock"))
install_db(
root_login=mariadb_root_username,
root_password=mariadb_root_password,
db_name=db_name,
admin_password=admin_password,
verbose=verbose,
source_sql=source_sql,
force=force,
reinstall=reinstall,
db_password=db_password,
db_type=db_type,
db_host=db_host,
db_port=db_port,
no_mariadb_socket=no_mariadb_socket,
)
apps_to_install = (
["frappe"] + (frappe.conf.get("install_apps") or []) + (list(install_apps) or [])
)
for app in apps_to_install:
install_app(app, verbose=verbose, set_as_patched=not source_sql)
os.remove(installing)
scheduler.toggle_scheduler(enable_scheduler)
frappe.db.commit()
scheduler_status = (
"disabled" if frappe.utils.scheduler.is_scheduler_disabled() else "enabled"
)
print("*** Scheduler is", scheduler_status, "***")
def install_db(root_login="root", root_password=None, db_name=None, source_sql=None,
@ -36,9 +118,9 @@ def install_db(root_login="root", root_password=None, db_name=None, source_sql=N
def install_app(name, verbose=False, set_as_patched=True):
from frappe.core.doctype.scheduled_job_type.scheduled_job_type import sync_jobs
from frappe.utils.fixtures import sync_fixtures
from frappe.model.sync import sync_for
from frappe.modules.utils import sync_customizations
from frappe.utils.fixtures import sync_fixtures
frappe.flags.in_install = name
frappe.flags.ignore_in_install = False
@ -122,64 +204,80 @@ def remove_app(app_name, dry_run=False, yes=False, no_backup=False, force=False)
"""Remove app and all linked to the app's module with the app from a site."""
import click
site = frappe.local.site
# dont allow uninstall app if not installed unless forced
if not force:
if app_name not in frappe.get_installed_apps():
click.secho("App {0} not installed on Site {1}".format(app_name, frappe.local.site), fg="yellow")
click.secho(f"App {app_name} not installed on Site {site}", fg="yellow")
return
print("Uninstalling App {0} from Site {1}...".format(app_name, frappe.local.site))
print(f"Uninstalling App {app_name} from Site {site}...")
if not dry_run and not yes:
confirm = click.confirm("All doctypes (including custom), modules related to this app will be deleted. Are you sure you want to continue?")
confirm = click.confirm(
"All doctypes (including custom), modules related to this app will be"
" deleted. Are you sure you want to continue?"
)
if not confirm:
return
if not no_backup:
if not (dry_run or no_backup):
from frappe.utils.backups import scheduled_backup
print("Backing up...")
scheduled_backup(ignore_files=True)
frappe.flags.in_uninstall = True
drop_doctypes = []
modules = (x.name for x in frappe.get_all("Module Def", filters={"app_name": app_name}))
modules = frappe.get_all("Module Def", filters={"app_name": app_name}, pluck="name")
for module_name in modules:
print("Deleting Module '{0}'".format(module_name))
print(f"Deleting Module '{module_name}'")
for doctype in frappe.get_list("DocType", filters={"module": module_name}, fields=["name", "issingle"]):
print("* removing DocType '{0}'...".format(doctype.name))
for doctype in frappe.get_all(
"DocType", filters={"module": module_name}, fields=["name", "issingle"]
):
print(f"* removing DocType '{doctype.name}'...")
if not dry_run:
frappe.delete_doc("DocType", doctype.name)
frappe.delete_doc("DocType", doctype.name, ignore_on_trash=True)
if not doctype.issingle:
drop_doctypes.append(doctype.name)
linked_doctypes = frappe.get_all("DocField", filters={"fieldtype": "Link", "options": "Module Def"}, fields=['parent'])
linked_doctypes = frappe.get_all(
"DocField", filters={"fieldtype": "Link", "options": "Module Def"}, fields=["parent"]
)
ordered_doctypes = ["Desk Page", "Report", "Page", "Web Form"]
doctypes_with_linked_modules = ordered_doctypes + [doctype.parent for doctype in linked_doctypes if doctype.parent not in ordered_doctypes]
all_doctypes_with_linked_modules = ordered_doctypes + [
doctype.parent
for doctype in linked_doctypes
if doctype.parent not in ordered_doctypes
]
doctypes_with_linked_modules = [
x for x in all_doctypes_with_linked_modules if frappe.db.exists("DocType", x)
]
for doctype in doctypes_with_linked_modules:
for record in frappe.get_list(doctype, filters={"module": module_name}):
print("* removing {0} '{1}'...".format(doctype, record.name))
for record in frappe.get_all(doctype, filters={"module": module_name}, pluck="name"):
print(f"* removing {doctype} '{record}'...")
if not dry_run:
frappe.delete_doc(doctype, record.name)
frappe.delete_doc(doctype, record, ignore_on_trash=True)
print("* removing Module Def '{0}'...".format(module_name))
print(f"* removing Module Def '{module_name}'...")
if not dry_run:
frappe.delete_doc("Module Def", module_name)
frappe.delete_doc("Module Def", module_name, ignore_on_trash=True)
for doctype in set(drop_doctypes):
print(f"* dropping Table for '{doctype}'...")
if not dry_run:
frappe.db.sql_ddl(f"drop table `tab{doctype}`")
if not dry_run:
remove_from_installed_apps(app_name)
for doctype in set(drop_doctypes):
print("* dropping Table for '{0}'...".format(doctype))
frappe.db.sql_ddl("drop table `tab{0}`".format(doctype))
frappe.db.commit()
click.secho("Uninstalled App {0} from Site {1}".format(app_name, frappe.local.site), fg="green")
click.secho(f"Uninstalled App {app_name} from Site {site}", fg="green")
frappe.flags.in_uninstall = False
@ -331,6 +429,37 @@ def remove_missing_apps():
frappe.db.set_global("installed_apps", json.dumps(installed_apps))
def extract_sql_from_archive(sql_file_path):
"""Return the path of an SQL file if the passed argument is the path of a gzipped
SQL file or an SQL file path. The path may be absolute or relative from the bench
root directory or the sites sub-directory.
Args:
sql_file_path (str): Path of the SQL file
Returns:
str: Path of the decompressed SQL file
"""
# Extract the gzip file if user has passed *.sql.gz file instead of *.sql file
if not os.path.exists(sql_file_path):
base_path = '..'
sql_file_path = os.path.join(base_path, sql_file_path)
if not os.path.exists(sql_file_path):
print('Invalid path {0}'.format(sql_file_path[3:]))
sys.exit(1)
elif sql_file_path.startswith(os.sep):
base_path = os.sep
else:
base_path = '.'
if sql_file_path.endswith('sql.gz'):
decompressed_file_name = extract_sql_gzip(os.path.abspath(sql_file_path))
else:
decompressed_file_name = sql_file_path
return decompressed_file_name
def extract_sql_gzip(sql_gz_path):
import subprocess
@ -345,9 +474,10 @@ def extract_sql_gzip(sql_gz_path):
return decompressed_file
def extract_files(site_name, file_path, folder_name):
import subprocess
import shutil
import subprocess
# Need to do frappe.init to maintain the site locals
frappe.init(site=site_name)
@ -375,6 +505,12 @@ def extract_files(site_name, file_path, folder_name):
def is_downgrade(sql_file_path, verbose=False):
"""checks if input db backup will get downgraded on current bench"""
# This function is only tested with mariadb
# TODO: Add postgres support
if frappe.conf.db_type not in (None, "mariadb"):
return False
from semantic_version import Version
head = "INSERT INTO `tabInstalled Application` VALUES"
@ -408,6 +544,37 @@ def is_downgrade(sql_file_path, verbose=False):
return downgrade
def is_partial(sql_file_path):
with open(sql_file_path) as f:
header = " ".join([f.readline() for _ in range(5)])
if "Partial Backup" in header:
return True
return False
def partial_restore(sql_file_path, verbose=False):
sql_file = extract_sql_from_archive(sql_file_path)
if frappe.conf.db_type in (None, "mariadb"):
from frappe.database.mariadb.setup_db import import_db_from_sql
elif frappe.conf.db_type == "postgres":
from frappe.database.postgres.setup_db import import_db_from_sql
import warnings
from click import style
warn = style(
"Delete the tables you want to restore manually before attempting"
" partial restore operation for PostreSQL databases",
fg="yellow"
)
warnings.warn(warn)
import_db_from_sql(source_sql=sql_file, verbose=verbose)
# Removing temporarily created file
if sql_file != sql_file_path:
os.remove(sql_file)
def validate_database_sql(path, _raise=True):
"""Check if file has contents and if DefaultValue table exists
@ -415,23 +582,29 @@ def validate_database_sql(path, _raise=True):
path (str): Path of the decompressed SQL file
_raise (bool, optional): Raise exception if invalid file. Defaults to True.
"""
to_raise = False
empty_file = False
missing_table = True
error_message = ""
if not os.path.getsize(path):
error_message = f"{path} is an empty file!"
to_raise = True
empty_file = True
if not _raise:
# dont bother checking if empty file
if not empty_file:
with open(path, "r") as f:
for line in f:
if 'tabDefaultValue' in line:
error_message = "Table `tabDefaultValue` not found in file."
to_raise = True
missing_table = False
break
if missing_table:
error_message = "Table `tabDefaultValue` not found in file."
if error_message:
import click
click.secho(error_message, fg="red")
if _raise and to_raise:
if _raise and (missing_table or empty_file):
raise frappe.InvalidDatabaseFile

5
frappe/integrations/doctype/dropbox_settings/dropbox_settings.py
View file

@ -9,7 +9,7 @@ import frappe
import os
from frappe import _
from frappe.model.document import Document
from frappe.integrations.offsite_backup_utils import get_latest_backup_file, send_email, validate_file_size
from frappe.integrations.offsite_backup_utils import get_latest_backup_file, send_email, validate_file_size, get_chunk_site
from frappe.integrations.utils import make_post_request
from frappe.utils import (cint, get_request_site_address,
get_files_path, get_backups_path, get_url, encode)
@ -167,8 +167,9 @@ def upload_file_to_dropbox(filename, folder, dropbox_client):
return
create_folder_if_not_exists(folder, dropbox_client)
chunk_size = 15 * 1024 * 1024
file_size = os.path.getsize(encode(filename))
chunk_size = get_chunk_site(file_size)
mode = (dropbox.files.WriteMode.overwrite)
f = open(encode(filename), 'rb')

2
frappe/integrations/frappe_providers/frappecloud.py
View file

@ -6,7 +6,7 @@ import frappe
def frappecloud_migrator(local_site):
print("Retreiving Site Migrator...")
print("Retrieving Site Migrator...")
remote_site = frappe.conf.frappecloud_url or "frappecloud.com"
request_url = "https://{}/api/method/press.api.script".format(remote_site)
request = requests.get(request_url)

132
frappe/integrations/oauth2.py
View file

@ -1,42 +1,51 @@
from __future__ import unicode_literals
import frappe, json
from frappe.oauth import OAuthWebRequestValidator, WebApplicationServer
import hashlib
import json
from urllib.parse import quote, urlencode, urlparse
import jwt
from oauthlib.oauth2 import FatalClientError, OAuth2Error
from werkzeug import url_fix
from six.moves.urllib.parse import quote, urlencode, urlparse
from frappe.integrations.doctype.oauth_provider_settings.oauth_provider_settings import get_oauth_settings
import frappe
from frappe import _
from frappe.oauth import OAuthWebRequestValidator, WebApplicationServer
from frappe.integrations.doctype.oauth_provider_settings.oauth_provider_settings import get_oauth_settings
def get_oauth_server():
if not getattr(frappe.local, 'oauth_server', None):
oauth_validator = OAuthWebRequestValidator()
frappe.local.oauth_server = WebApplicationServer(oauth_validator)
frappe.local.oauth_server = WebApplicationServer(oauth_validator)
return frappe.local.oauth_server
def clean_urlparams(param_kwargs):
def sanitize_kwargs(param_kwargs):
"""Remove 'data' and 'cmd' keys, if present."""
arguments = param_kwargs
if arguments.get("data"):
arguments.pop("data")
if arguments.get("cmd"):
arguments.pop("cmd")
arguments.pop('data', None)
arguments.pop('cmd', None)
return urlencode(arguments)
return arguments
@frappe.whitelist()
def approve(*args, **kwargs):
r = frappe.request
uri = url_fix(r.url.replace("+"," "))
http_method = r.method
body = r.get_data()
headers = r.headers
try:
scopes, frappe.flags.oauth_credentials = get_oauth_server().validate_authorization_request(uri, http_method, body, headers)
scopes, frappe.flags.oauth_credentials = get_oauth_server().validate_authorization_request(
r.url,
r.method,
r.get_data(),
r.headers
)
headers, body, status = get_oauth_server().create_authorization_response(uri=frappe.flags.oauth_credentials['redirect_uri'], \
body=body, headers=headers, scopes=scopes, credentials=frappe.flags.oauth_credentials)
headers, body, status = get_oauth_server().create_authorization_response(
uri=frappe.flags.oauth_credentials['redirect_uri'],
body=r.get_data(),
headers=r.headers,
scopes=scopes,
credentials=frappe.flags.oauth_credentials
)
uri = headers.get('Location', None)
frappe.local.response["type"] = "redirect"
@ -48,34 +57,28 @@ def approve(*args, **kwargs):
return e
@frappe.whitelist(allow_guest=True)
def authorize(*args, **kwargs):
#Fetch provider URL from settings
oauth_settings = get_oauth_settings()
params = clean_urlparams(kwargs)
request_url = urlparse(frappe.request.url)
success_url = request_url.scheme + "://" + request_url.netloc + "/api/method/frappe.integrations.oauth2.approve?" + params
def authorize(**kwargs):
success_url = "/api/method/frappe.integrations.oauth2.approve?" + encode_params(sanitize_kwargs(kwargs))
failure_url = frappe.form_dict["redirect_uri"] + "?error=access_denied"
if frappe.session['user']=='Guest':
if frappe.session.user == 'Guest':
#Force login, redirect to preauth again.
frappe.local.response["type"] = "redirect"
frappe.local.response["location"] = "/login?redirect-to=/api/method/frappe.integrations.oauth2.authorize?" + quote(params.replace("+"," "))
elif frappe.session['user']!='Guest':
frappe.local.response["location"] = "/login?" + encode_params({'redirect-to': frappe.request.url})
else:
try:
r = frappe.request
uri = url_fix(r.url)
http_method = r.method
body = r.get_data()
headers = r.headers
scopes, frappe.flags.oauth_credentials = get_oauth_server().validate_authorization_request(uri, http_method, body, headers)
scopes, frappe.flags.oauth_credentials = get_oauth_server().validate_authorization_request(
r.url,
r.method,
r.get_data(),
r.headers
)
skip_auth = frappe.db.get_value("OAuth Client", frappe.flags.oauth_credentials['client_id'], "skip_authorization")
unrevoked_tokens = frappe.get_all("OAuth Bearer Token", filters={"status":"Active"})
if skip_auth or (oauth_settings["skip_authorization"] == "Auto" and len(unrevoked_tokens)):
if skip_auth or (get_oauth_settings().skip_authorization == "Auto" and unrevoked_tokens):
frappe.local.response["type"] = "redirect"
frappe.local.response["location"] = success_url
else:
@ -88,7 +91,6 @@ def authorize(*args, **kwargs):
})
resp_html = frappe.render_template("templates/includes/oauth_confirmation.html", response_html_params)
frappe.respond_as_web_page("Confirm Access", resp_html)
except FatalClientError as e:
return e
except OAuth2Error as e:
@ -96,20 +98,20 @@ def authorize(*args, **kwargs):
@frappe.whitelist(allow_guest=True)
def get_token(*args, **kwargs):
r = frappe.request
uri = url_fix(r.url)
http_method = r.method
body = r.form
headers = r.headers
#Check whether frappe server URL is set
frappe_server_url = frappe.db.get_value("Social Login Key", "frappe", "base_url") or None
if not frappe_server_url:
frappe.throw(_("Please set Base URL in Social Login Key for Frappe"))
try:
headers, body, status = get_oauth_server().create_token_response(uri, http_method, body, headers, frappe.flags.oauth_credentials)
r = frappe.request
headers, body, status = get_oauth_server().create_token_response(
r.url,
r.method,
r.form,
r.headers,
frappe.flags.oauth_credentials
)
out = frappe._dict(json.loads(body))
if not out.error and "openid" in out.scope:
token_user = frappe.db.get_value("OAuth Bearer Token", out.access_token, "user")
@ -117,7 +119,7 @@ def get_token(*args, **kwargs):
client_secret = frappe.db.get_value("OAuth Client", token_client, "client_secret")
if token_user in ["Guest", "Administrator"]:
frappe.throw(_("Logged in as Guest or Administrator"))
import hashlib
id_token_header = {
"typ":"jwt",
"alg":"HS256"
@ -129,9 +131,10 @@ def get_token(*args, **kwargs):
"iss": frappe_server_url,
"at_hash": frappe.oauth.calculate_at_hash(out.access_token, hashlib.sha256)
}
import jwt
id_token_encoded = jwt.encode(id_token, client_secret, algorithm='HS256', headers=id_token_header)
out.update({"id_token":str(id_token_encoded)})
out.update({"id_token": str(id_token_encoded)})
frappe.local.response = out
except FatalClientError as e:
@ -141,12 +144,12 @@ def get_token(*args, **kwargs):
@frappe.whitelist(allow_guest=True)
def revoke_token(*args, **kwargs):
r = frappe.request
uri = url_fix(r.url)
http_method = r.method
body = r.form
headers = r.headers
headers, body, status = get_oauth_server().create_revocation_response(uri, headers=headers, body=body, http_method=http_method)
headers, body, status = get_oauth_server().create_revocation_response(
r.url,
headers=r.headers,
body=r.form,
http_method=r.method
)
frappe.local.response['http_status_code'] = status
if status == 200:
@ -175,15 +178,22 @@ def openid_profile(*args, **kwargs):
"email": name,
"picture": picture
})
frappe.local.response = user_profile
def validate_url(url_string):
try:
result = urlparse(url_string)
if result.scheme and result.scheme in ["http", "https", "ftp", "ftps"]:
return True
else:
return False
return result.scheme and result.scheme in ["http", "https", "ftp", "ftps"]
except:
return False
return False
def encode_params(params):
"""
Encode a dict of params into a query string.
Use `quote_via=urllib.parse.quote` so that whitespaces will be encoded as
`%20` instead of as `+`. This is needed because oauthlib cannot handle `+`
as a whitespace.
"""
return urlencode(params, quote_via=quote)

21
frappe/integrations/offsite_backup_utils.py
View file

@ -6,8 +6,7 @@ from __future__ import unicode_literals
import frappe
import glob
import os
from frappe.utils import split_emails, get_backups_path
from frappe.utils import split_emails, cint
def send_email(success, service_name, doctype, email_field, error_status=None):
recipients = get_recipients(doctype, email_field)
@ -81,6 +80,22 @@ def get_file_size(file_path, unit):
return file_size
def get_chunk_site(file_size):
''' this function will return chunk size in megabytes based on file size '''
file_size_in_gb = cint(file_size/1024/1024)
MB = 1024 * 1024
if file_size_in_gb > 5000:
return 200 * MB
elif file_size_in_gb >= 3000:
return 150 * MB
elif file_size_in_gb >= 1000:
return 100 * MB
elif file_size_in_gb >= 500:
return 50 * MB
else:
return 15 * MB
def validate_file_size():
frappe.flags.create_new_backup = True
@ -98,4 +113,4 @@ def generate_files_backup():
db_type=frappe.conf.db_type, db_port=frappe.conf.db_port)
backup.set_backup_file_name()
backup.zip_files()
backup.zip_files()

14
frappe/model/delete_doc.py
View file

@ -335,19 +335,25 @@ def clear_timeline_references(link_doctype, link_name):
WHERE `tabCommunication Link`.link_doctype=%s AND `tabCommunication Link`.link_name=%s""", (link_doctype, link_name))
def insert_feed(doc):
from frappe.utils import get_fullname
if frappe.flags.in_install or frappe.flags.in_import or getattr(doc, "no_feed_on_delete", False):
if (
frappe.flags.in_install
or frappe.flags.in_uninstall
or frappe.flags.in_import
or getattr(doc, "no_feed_on_delete", False)
):
return
from frappe.utils import get_fullname
frappe.get_doc({
"doctype": "Comment",
"comment_type": "Deleted",
"reference_doctype": doc.doctype,
"subject": "{0} {1}".format(_(doc.doctype), doc.name),
"full_name": get_fullname(doc.owner)
"full_name": get_fullname(doc.owner),
}).insert(ignore_permissions=True)
def delete_controllers(doctype, module):
"""
Delete controller code in the doctype folder

2
frappe/model/workflow.py
View file

@ -29,6 +29,8 @@ def get_transitions(doc, workflow = None, raise_exception=False):
if doc.is_new():
return []
doc.load_from_db()
frappe.has_permission(doc, 'read', throw=True)
roles = frappe.get_roles()

2
frappe/oauth.py
View file

@ -148,7 +148,7 @@ class OAuthWebRequestValidator(RequestValidator):
print("Failed body authentication: Application %s does not exist".format(cid=request.client_id))
cookie_dict = get_cookie_dict_from_headers(request)
user_id = unquote(cookie_dict['user_id']) if 'user_id' in cookie_dict else "Guest"
user_id = unquote(cookie_dict.get('user_id').value) if 'user_id' in cookie_dict else "Guest"
return frappe.session.user == user_id
def authenticate_client_id(self, client_id, request, *args, **kwargs):

4
frappe/public/js/frappe/file_uploader/index.js
View file

@ -15,7 +15,11 @@ export default class FileUploader {
allow_multiple,
as_dataurl,
disable_file_browser,
frm
} = {}) {
frm && frm.attachments.max_reached(true);
if (!wrapper) {
this.make_dialog();
} else {

2
frappe/public/js/frappe/form/controls/comment.js
View file

@ -60,7 +60,7 @@ frappe.ui.form.ControlComment = frappe.ui.form.ControlTextEditor.extend({
update_state() {
const value = this.get_value();
if (strip_html(value).trim() != "") {
if (strip_html(value).trim() != "" || value.includes('img')) {
this.button.removeClass('btn-default').addClass('btn-primary');
} else {
this.button.addClass('btn-default').removeClass('btn-primary');

24
frappe/public/js/frappe/form/footer/timeline.js
View file

@ -30,7 +30,7 @@ frappe.ui.form.Timeline = class Timeline {
render_input: true,
only_input: true,
on_submit: (val) => {
if(strip_html(val).trim() != "") {
if (strip_html(val).trim() != "" || val.includes('img')) {
this.insert_comment(val, this.comment_area.button);
}
}
@ -547,10 +547,7 @@ frappe.ui.form.Timeline = class Timeline {
log.color = 'dark';
log.sender = log.owner;
log.comment_type = 'Milestone';
log.content = __('{0} changed {1} to {2}', [
frappe.user.full_name(log.owner).bold(),
frappe.meta.get_label(this.frm.doctype, log.track_field),
log.value.bold()]);
log.content = __('{0} changed {1} to {2}', [ frappe.user.full_name(log.owner).bold(), frappe.meta.get_label(this.frm.doctype, log.track_field), log.value.bold()]);
return log;
});
return milestones;
@ -613,11 +610,7 @@ frappe.ui.form.Timeline = class Timeline {
const field_display_status = frappe.perm.get_field_display_status(df, null,
me.frm.perm);
if (field_display_status === 'Read' || field_display_status === 'Write') {
parts.push(__('{0} from {1} to {2}', [
__(df.label),
me.format_content_for_timeline(p[1]),
me.format_content_for_timeline(p[2])
]));
parts.push(__('{0} from {1} to {2}', [ __(df.label), me.format_content_for_timeline(p[1]), me.format_content_for_timeline(p[2])]));
}
}
}
@ -648,13 +641,7 @@ frappe.ui.form.Timeline = class Timeline {
null, me.frm.perm);
if (field_display_status === 'Read' || field_display_status === 'Write') {
parts.push(__('{0} from {1} to {2} in row #{3}', [
frappe.meta.get_label(me.frm.fields_dict[row[0]].grid.doctype,
p[0]),
me.format_content_for_timeline(p[1]),
me.format_content_for_timeline(p[2]),
row[1]
]));
parts.push(__('{0} from {1} to {2} in row #{3}', [ frappe.meta.get_label( me.frm.fields_dict[row[0]].grid.doctype, p[0]), me.format_content_for_timeline(p[1]), me.format_content_for_timeline(p[2]), row[1] ]));
}
}
return parts.length < 3;
@ -691,8 +678,7 @@ frappe.ui.form.Timeline = class Timeline {
return p;
});
if (parts.length) {
out.push(me.get_version_comment(version, __("{0} rows for {1}",
[__(key), parts.join(', ')])));
out.push(me.get_version_comment(version, __("{0} rows for {1}", [__(key), parts.join(', ')])));
}
}
});

6
frappe/public/js/frappe/form/form.js
View file

@ -232,14 +232,10 @@ frappe.ui.form.Form = class FrappeForm {
throw "attach error";
}
if(me.attachments.max_reached()) {
frappe.msgprint(__("Maximum Attachment Limit for this record reached."));
throw "attach error";
}
new frappe.ui.FileUploader({
doctype: me.doctype,
docname: me.docname,
frm: me,
files: dataTransfer.files,
folder: 'Home/Attachments',
on_success(file_doc) {

22
frappe/public/js/frappe/form/sidebar/attachments.js
View file

@ -16,15 +16,19 @@ frappe.ui.form.Attachments = Class.extend({
this.add_attachment_wrapper = this.parent.find(".add_attachment").parent();
this.attachments_label = this.parent.find(".attachments-label");
},
max_reached: function() {
// no of attachments
var n = Object.keys(this.get_attachments()).length;
// button if the number of attachments is less than max
if(n < this.frm.meta.max_attachments || !this.frm.meta.max_attachments) {
return false;
max_reached: function(raise_exception=false) {
const attachment_count = Object.keys(this.get_attachments()).length;
const attachment_limit = this.frm.meta.max_attachments;
if (attachment_limit && attachment_count >= attachment_limit) {
if (raise_exception) {
frappe.throw({
title: __("Attachment Limit Reached"),
message: __("Maximum attachment limit of {0} has been reached.", [cstr(attachment_limit).bold()]),
});
}
return true;
}
return true;
return false;
},
refresh: function() {
var me = this;
@ -140,7 +144,6 @@ frappe.ui.form.Attachments = Class.extend({
});
},
new_attachment: function(fieldname) {
var me = this;
if (this.dialog) {
// remove upload dialog
this.dialog.$wrapper.remove();
@ -149,6 +152,7 @@ frappe.ui.form.Attachments = Class.extend({
new frappe.ui.FileUploader({
doctype: this.frm.doctype,
docname: this.frm.docname,
frm: this.frm,
folder: 'Home/Attachments',
on_success: (file_doc) => {
this.attachment_uploaded(file_doc);

16
frappe/templates/includes/oauth_confirmation.html
View file

@ -1,7 +1,7 @@
{% if not error %}
<div class="panel panel-default">
<div class="panel-heading">
<h3 class="panel-title">{{ client_id }} wants to access the following details from your account</h3>
<h3 class="panel-title">{{ _("{} wants to access the following details from your account").format(client_id) }}</h3>
</div>
<div class="panel-body">
<ul class="list-group">
@ -11,10 +11,10 @@
</ul>
<ul class="list-inline">
<li>
<button id="allow" class="btn btn-sm btn-primary">Allow</button>
<button id="allow" class="btn btn-sm btn-primary">{{ _("Allow") }}</button>
</li>
<li>
<button id="deny" class="btn btn-sm btn-light">Deny</button>
<button id="deny" class="btn btn-sm btn-light">{{ _("Deny") }}</button>
</li>
</ul>
</div>
@ -22,24 +22,24 @@
<script type="text/javascript">
frappe.ready(function() {
$('#allow').on('click', function(event) {
window.location.replace("{{ success_url|string }}");
window.location.replace("{{ success_url | string }}");
});
$('#deny').on('click', function(event) {
window.location.replace("{{ failure_url|string }}");
window.location.replace("{{ failure_url | string }}");
});
});
</script>
{% else %}
<div class="panel panel-danger">
<div class="panel-heading">
<h3 class="panel-title">Authorization error for {{ client_id }}</h3>
<h3 class="panel-title">{{ _("Authorization error for {}.").format(client_id) }}</h3>
</div>
<div class="panel-body">
<p>An unexpected error occurred while authorizing {{ client_id }}.</p>
<p>{{ _("An unexpected error occurred while authorizing {}.").format(client_id) }}</p>
<h4>{{ error }}</h4>
<ul class="list-inline">
<li>
<button class="btn btn-sm btn-light">OK</button>
<button class="btn btn-sm btn-light">{{ _("OK") }}</button>
</li>
</ul>
</div>

228
frappe/tests/test_commands.py
View file

@ -1,24 +1,88 @@
# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors
# imports - standard imports
import gzip
import json
import os
import shlex
import subprocess
import sys
import unittest
from glob import glob
import glob
# imports - module imports
import frappe
from frappe.utils.backups import fetch_latest_backups
import frappe.recorder
from frappe.installer import add_to_installed_apps
from frappe.utils import add_to_date, now
from frappe.utils.backups import fetch_latest_backups
# TODO: check frappe.cli.coloured_output to set coloured output!
def supports_color():
"""
Returns True if the running system's terminal supports color, and False
otherwise.
"""
plat = sys.platform
supported_platform = plat != 'Pocket PC' and (plat != 'win32' or 'ANSICON' in os.environ)
# isatty is not always implemented, #6223.
is_a_tty = hasattr(sys.stdout, 'isatty') and sys.stdout.isatty()
return supported_platform and is_a_tty
class color(dict):
nc = "\033[0m"
blue = "\033[94m"
green = "\033[92m"
yellow = "\033[93m"
red = "\033[91m"
silver = "\033[90m"
def __getattr__(self, key):
if supports_color():
ret = self.get(key)
else:
ret = ""
return ret
def clean(value):
if isinstance(value, (bytes, str)):
value = value.decode().strip()
"""Strips and converts bytes to str
Args:
value ([type]): [description]
Returns:
[type]: [description]
"""
if isinstance(value, bytes):
value = value.decode()
if isinstance(value, str):
value = value.strip()
return value
def exists_in_backup(doctypes, file):
"""Checks if the list of doctypes exist in the database.sql.gz file supplied
Args:
doctypes (list): List of DocTypes to be checked
file (str): Path of the database file
Returns:
bool: True if all tables exist
"""
predicate = (
'COPY public."tab{}"'
if frappe.conf.db_type == "postgres"
else "CREATE TABLE `tab{}`"
)
with gzip.open(file, "rb") as f:
content = f.read().decode("utf8")
return all([predicate.format(doctype).lower() in content.lower() for doctype in doctypes])
class BaseTestCommands(unittest.TestCase):
def execute(self, command, kwargs=None):
site = {"site": frappe.local.site}
@ -26,13 +90,26 @@ class BaseTestCommands(unittest.TestCase):
kwargs.update(site)
else:
kwargs = site
command = command.replace("\n", " ").format(**kwargs)
command = shlex.split(command)
self.command = " ".join(command.split()).format(**kwargs)
print("{0}$ {1}{2}".format(color.silver, self.command, color.nc))
command = shlex.split(self.command)
self._proc = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
self.stdout = clean(self._proc.stdout)
self.stderr = clean(self._proc.stderr)
self.returncode = clean(self._proc.returncode)
def _formatMessage(self, msg, standardMsg):
output = super(BaseTestCommands, self)._formatMessage(msg, standardMsg)
cmd_execution_summary = "\n".join([
"-" * 70,
"Last Command Execution Summary:",
"Command: {}".format(self.command) if self.command else "",
"Standard Output: {}".format(self.stdout) if self.stdout else "",
"Standard Error: {}".format(self.stderr) if self.stderr else "",
"Return Code: {}".format(self.returncode) if self.returncode else "",
]).strip()
return "{}\n\n{}".format(output, cmd_execution_summary)
class TestCommands(BaseTestCommands):
def test_execute(self):
@ -52,9 +129,24 @@ class TestCommands(BaseTestCommands):
# The returned value has quotes which have been trimmed for the test
self.execute("""bench --site {site} execute frappe.bold --kwargs '{{"text": "DocType"}}'""")
self.assertEquals(self.returncode, 0)
self.assertEquals(self.stdout[1:-1], frappe.bold(text='DocType'))
self.assertEquals(self.stdout[1:-1], frappe.bold(text="DocType"))
def test_backup(self):
backup = {
"includes": {
"includes": [
"ToDo",
"Note",
]
},
"excludes": {
"excludes": [
"Activity Log",
"Access Log",
"Error Log"
]
}
}
home = os.path.expanduser("~")
site_backup_path = frappe.utils.get_site_path("private", "backups")
@ -94,16 +186,19 @@ class TestCommands(BaseTestCommands):
"db_path": "database.sql.gz",
"files_path": "public.tar",
"private_path": "private.tar",
"conf_path": "config.json"
"conf_path": "config.json",
}.items()
}
self.execute("""bench
self.execute(
"""bench
--site {site} backup --with-files
--backup-path-db {db_path}
--backup-path-files {files_path}
--backup-path-private-files {private_path}
--backup-path-conf {conf_path}""", kwargs)
--backup-path-conf {conf_path}""",
kwargs,
)
self.assertEquals(self.returncode, 0)
for path in kwargs.values():
@ -111,16 +206,122 @@ class TestCommands(BaseTestCommands):
# test 5: take a backup with --compress
self.execute("bench --site {site} backup --with-files --compress")
self.assertEquals(self.returncode, 0)
compressed_files = glob(site_backup_path + "/*.tgz")
compressed_files = glob.glob(site_backup_path + "/*.tgz")
self.assertGreater(len(compressed_files), 0)
# test 6: take a backup with --verbose
self.execute("bench --site {site} backup --verbose")
self.assertEquals(self.returncode, 0)
# test 7: take a backup with frappe.conf.backup.includes
self.execute(
"bench --site {site} set-config backup '{includes}' --as-dict",
{"includes": json.dumps(backup["includes"])},
)
self.execute("bench --site {site} backup --verbose")
self.assertEquals(self.returncode, 0)
database = fetch_latest_backups(partial=True)["database"]
self.assertTrue(exists_in_backup(backup["includes"]["includes"], database))
# test 8: take a backup with frappe.conf.backup.excludes
self.execute(
"bench --site {site} set-config backup '{excludes}' --as-dict",
{"excludes": json.dumps(backup["excludes"])},
)
self.execute("bench --site {site} backup --verbose")
self.assertEquals(self.returncode, 0)
database = fetch_latest_backups(partial=True)["database"]
self.assertFalse(exists_in_backup(backup["excludes"]["excludes"], database))
self.assertTrue(exists_in_backup(backup["includes"]["includes"], database))
# test 9: take a backup with --include (with frappe.conf.excludes still set)
self.execute(
"bench --site {site} backup --include '{include}'",
{"include": ",".join(backup["includes"]["includes"])},
)
self.assertEquals(self.returncode, 0)
database = fetch_latest_backups(partial=True)["database"]
self.assertTrue(exists_in_backup(backup["includes"]["includes"], database))
# test 10: take a backup with --exclude
self.execute(
"bench --site {site} backup --exclude '{exclude}'",
{"exclude": ",".join(backup["excludes"]["excludes"])},
)
self.assertEquals(self.returncode, 0)
database = fetch_latest_backups(partial=True)["database"]
self.assertFalse(exists_in_backup(backup["excludes"]["excludes"], database))
# test 11: take a backup with --ignore-backup-conf
self.execute("bench --site {site} backup --ignore-backup-conf")
self.assertEquals(self.returncode, 0)
database = fetch_latest_backups()["database"]
self.assertTrue(exists_in_backup(backup["excludes"]["excludes"], database))
def test_restore(self):
# step 0: create a site to run the test on
global_config = {
"admin_password": frappe.conf.admin_password,
"root_login": frappe.conf.root_login,
"root_password": frappe.conf.root_password,
"db_type": frappe.conf.db_type,
}
site_data = {"another_site": f"{frappe.local.site}-restore.test", **global_config}
for key, value in global_config.items():
if value:
self.execute(f"bench set-config {key} {value} -g")
self.execute(
"bench new-site {another_site} --admin-password {admin_password} --db-type"
" {db_type}",
site_data,
)
# test 1: bench restore from full backup
self.execute("bench --site {another_site} backup --ignore-backup-conf", site_data)
self.execute(
"bench --site {another_site} execute frappe.utils.backups.fetch_latest_backups",
site_data,
)
site_data.update({"database": json.loads(self.stdout)["database"]})
self.execute("bench --site {another_site} restore {database}", site_data)
# test 2: restore from partial backup
self.execute("bench --site {another_site} backup --exclude 'ToDo'", site_data)
site_data.update({"kw": "\"{'partial':True}\""})
self.execute(
"bench --site {another_site} execute"
" frappe.utils.backups.fetch_latest_backups --kwargs {kw}",
site_data,
)
site_data.update({"database": json.loads(self.stdout)["database"]})
self.execute("bench --site {another_site} restore {database}", site_data)
self.assertEquals(self.returncode, 1)
def test_partial_restore(self):
_now = now()
for num in range(10):
frappe.get_doc({
"doctype": "ToDo",
"date": add_to_date(_now, days=num),
"description": frappe.mock("paragraph")
}).insert()
frappe.db.commit()
todo_count = frappe.db.count("ToDo")
# check if todos exist, create a partial backup and see if the state is the same after restore
self.assertIsNot(todo_count, 0)
self.execute("bench --site {site} backup --only 'ToDo'")
db_path = fetch_latest_backups(partial=True)["database"]
self.assertTrue("partial" in db_path)
frappe.db.sql_ddl("DROP TABLE IF EXISTS `tabToDo`")
frappe.db.commit()
self.execute("bench --site {site} partial-restore {path}", {"path": db_path})
self.assertEquals(self.returncode, 0)
self.assertEquals(frappe.db.count("ToDo"), todo_count)
def test_recorder(self):
frappe.recorder.stop()
@ -133,7 +334,6 @@ class TestCommands(BaseTestCommands):
self.assertEqual(frappe.recorder.status(), False)
def test_remove_from_installed_apps(self):
from frappe.installer import add_to_installed_apps
app = "test_remove_app"
add_to_installed_apps(app)

11
frappe/tests/test_oauth20.py
View file

@ -6,6 +6,7 @@ import unittest, frappe, requests, time
from frappe.test_runner import make_test_records
from six.moves.urllib.parse import urlparse, parse_qs, urljoin
from urllib.parse import urlencode, quote
from frappe.integrations.oauth2 import encode_params
class TestOAuth20(unittest.TestCase):
@ -232,13 +233,3 @@ def login(session):
def get_full_url(endpoint):
"""Turn '/endpoint' into 'http://127.0.0.1:8000/endpoint'."""
return urljoin(frappe.utils.get_url(), endpoint)
def encode_params(params):
"""
Encode a dict of params into a query string.
Use `quote_via=urllib.parse.quote` so that whitespaces will be encoded as
`%20` instead of as `+`. This is needed because oauthlib cannot handle `+`
as a whitespace.
"""
return urlencode(params, quote_via=quote)

555
frappe/utils/backups.py
View file

@ -2,11 +2,12 @@
# MIT License. See license.txt
# imports - standard imports
import json
import gzip
import os
from calendar import timegm
from datetime import datetime
from glob import glob
from shutil import which
# imports - third party imports
import click
@ -14,24 +15,42 @@ import click
# imports - module imports
import frappe
from frappe import _, conf
from frappe.utils import get_url, now, now_datetime, get_file_size
from frappe.utils import get_file_size, get_url, now, now_datetime
# backup variable for backwards compatibility
verbose = False
compress = False
_verbose = verbose
base_tables = ["__Auth", "__global_search", "__UserSettings"]
class BackupGenerator:
"""
This class contains methods to perform On Demand Backup
This class contains methods to perform On Demand Backup
To initialize, specify (db_name, user, password, db_file_name=None, db_host="localhost")
If specifying db_file_name, also append ".sql.gz"
To initialize, specify (db_name, user, password, db_file_name=None, db_host="localhost")
If specifying db_file_name, also append ".sql.gz"
"""
def __init__(self, db_name, user, password, backup_path=None, backup_path_db=None,
backup_path_files=None, backup_path_private_files=None, db_host="localhost", db_port=None,
verbose=False, db_type='mariadb', backup_path_conf=None, compress_files=False):
def __init__(
self,
db_name,
user,
password,
backup_path=None,
backup_path_db=None,
backup_path_files=None,
backup_path_private_files=None,
db_host="localhost",
db_port=None,
db_type="mariadb",
backup_path_conf=None,
ignore_conf=False,
compress_files=False,
include_doctypes="",
exclude_doctypes="",
verbose=False,
):
global _verbose
self.compress_files = compress_files or compress
self.db_host = db_host
@ -45,23 +64,35 @@ class BackupGenerator:
self.backup_path_db = backup_path_db
self.backup_path_files = backup_path_files
self.backup_path_private_files = backup_path_private_files
self.ignore_conf = ignore_conf
self.include_doctypes = include_doctypes
self.exclude_doctypes = exclude_doctypes
self.partial = False
if not self.db_type:
self.db_type = 'mariadb'
self.db_type = "mariadb"
if not self.db_port and self.db_type == 'mariadb':
self.db_port = 3306
elif not self.db_port and self.db_type == 'postgres':
self.db_port = 5432
if not self.db_port:
if self.db_type == "mariadb":
self.db_port = 3306
if self.db_type == "postgres":
self.db_port = 5432
site = frappe.local.site or frappe.generate_hash(length=8)
self.site_slug = site.replace('.', '_')
self.site_slug = site.replace(".", "_")
self.verbose = verbose
self.setup_backup_directory()
self.setup_backup_tables()
_verbose = verbose
def setup_backup_directory(self):
specified = self.backup_path or self.backup_path_db or self.backup_path_files or self.backup_path_private_files or self.backup_path_conf
specified = (
self.backup_path
or self.backup_path_db
or self.backup_path_files
or self.backup_path_private_files
or self.backup_path_conf
)
if not specified:
backups_folder = get_backup_path()
@ -71,32 +102,93 @@ class BackupGenerator:
if self.backup_path:
os.makedirs(self.backup_path, exist_ok=True)
for file_path in set([self.backup_path_files, self.backup_path_db, self.backup_path_private_files, self.backup_path_conf]):
for file_path in set(
[
self.backup_path_files,
self.backup_path_db,
self.backup_path_private_files,
self.backup_path_conf,
]
):
if file_path:
dir = os.path.dirname(file_path)
os.makedirs(dir, exist_ok=True)
def setup_backup_tables(self):
"""Sets self.backup_includes, self.backup_excludes based on passed args"""
existing_doctypes = set([x.name for x in frappe.get_all("DocType")])
def get_tables(doctypes):
tables = []
for doctype in doctypes:
if doctype and doctype in existing_doctypes:
if doctype.startswith("tab"):
tables.append(doctype)
else:
tables.append("tab" + doctype)
return tables
passed_tables = {
"include": get_tables(self.include_doctypes.strip().split(",")),
"exclude": get_tables(self.exclude_doctypes.strip().split(",")),
}
specified_tables = get_tables(frappe.conf.get("backup", {}).get("includes", []))
include_tables = (specified_tables + base_tables) if specified_tables else []
conf_tables = {
"include": include_tables,
"exclude": get_tables(frappe.conf.get("backup", {}).get("excludes", [])),
}
self.backup_includes = passed_tables["include"]
self.backup_excludes = passed_tables["exclude"]
if not (self.backup_includes or self.backup_excludes) and not self.ignore_conf:
self.backup_includes = self.backup_includes or conf_tables["include"]
self.backup_excludes = self.backup_excludes or conf_tables["exclude"]
self.partial = (self.backup_includes or self.backup_excludes) and not self.ignore_conf
@property
def site_config_backup_path(self):
# For backwards compatibility
click.secho("BackupGenerator.site_config_backup_path has been deprecated in favour of BackupGenerator.backup_path_conf", fg="yellow")
click.secho(
"BackupGenerator.site_config_backup_path has been deprecated in favour of"
" BackupGenerator.backup_path_conf",
fg="yellow",
)
return getattr(self, "backup_path_conf", None)
def get_backup(self, older_than=24, ignore_files=False, force=False):
"""
Takes a new dump if existing file is old
and sends the link to the file as email
Takes a new dump if existing file is old
and sends the link to the file as email
"""
#Check if file exists and is less than a day old
#If not Take Dump
# Check if file exists and is less than a day old
# If not Take Dump
if not force:
last_db, last_file, last_private_file, site_config_backup_path = self.get_recent_backup(older_than)
(
last_db,
last_file,
last_private_file,
site_config_backup_path,
) = self.get_recent_backup(older_than)
else:
last_db, last_file, last_private_file, site_config_backup_path = False, False, False, False
last_db, last_file, last_private_file, site_config_backup_path = (
False,
False,
False,
False,
)
self.todays_date = now_datetime().strftime('%Y%m%d_%H%M%S')
self.todays_date = now_datetime().strftime("%Y%m%d_%H%M%S")
if not (self.backup_path_conf and self.backup_path_db and self.backup_path_files and self.backup_path_private_files):
if not (
self.backup_path_conf
and self.backup_path_db
and self.backup_path_files
and self.backup_path_private_files
):
self.set_backup_file_name()
if not (last_db and last_file and last_private_file and site_config_backup_path):
@ -112,13 +204,13 @@ class BackupGenerator:
self.backup_path_conf = site_config_backup_path
def set_backup_file_name(self):
#Generate a random name using today's date and a 8 digit random number
for_conf = self.todays_date + "-" + self.site_slug + "-site_config_backup.json"
for_db = self.todays_date + "-" + self.site_slug + "-database.sql.gz"
partial = "-partial" if self.partial else ""
ext = "tgz" if self.compress_files else "tar"
for_public_files = self.todays_date + "-" + self.site_slug + "-files." + ext
for_private_files = self.todays_date + "-" + self.site_slug + "-private-files." + ext
for_conf = f"{self.todays_date}-{self.site_slug}-site_config_backup.json"
for_db = f"{self.todays_date}-{self.site_slug}{partial}-database.sql.gz"
for_public_files = f"{self.todays_date}-{self.site_slug}-files.{ext}"
for_private_files = f"{self.todays_date}-{self.site_slug}-private-files.{ext}"
backup_path = self.backup_path or get_backup_path()
if not self.backup_path_conf:
@ -130,11 +222,11 @@ class BackupGenerator:
if not self.backup_path_private_files:
self.backup_path_private_files = os.path.join(backup_path, for_private_files)
def get_recent_backup(self, older_than):
def get_recent_backup(self, older_than, partial=False):
backup_path = get_backup_path()
file_type_slugs = {
"database": "*-{}-database.sql.gz",
"database": "*-{{}}-{}database.sql.gz".format('*' if partial else ''),
"public": "*-{}-files.tar",
"private": "*-{}-private-files.tar",
"config": "*-{}-site_config_backup.json",
@ -158,8 +250,7 @@ class BackupGenerator:
return file_path
latest_backups = {
file_type: get_latest(pattern)
for file_type, pattern in file_type_slugs.items()
file_type: get_latest(pattern) for file_type, pattern in file_type_slugs.items()
}
recent_backups = {
@ -175,32 +266,40 @@ class BackupGenerator:
def zip_files(self):
# For backwards compatibility - pre v13
click.secho("BackupGenerator.zip_files has been deprecated in favour of BackupGenerator.backup_files", fg="yellow")
click.secho(
"BackupGenerator.zip_files has been deprecated in favour of"
" BackupGenerator.backup_files",
fg="yellow",
)
return self.backup_files()
def get_summary(self):
summary = {
"config": {
"path": self.backup_path_conf,
"size": get_file_size(self.backup_path_conf, format=True)
"size": get_file_size(self.backup_path_conf, format=True),
},
"database": {
"path": self.backup_path_db,
"size": get_file_size(self.backup_path_db, format=True)
}
"size": get_file_size(self.backup_path_db, format=True),
},
}
if os.path.exists(self.backup_path_files) and os.path.exists(self.backup_path_private_files):
summary.update({
"public": {
"path": self.backup_path_files,
"size": get_file_size(self.backup_path_files, format=True)
},
"private": {
"path": self.backup_path_private_files,
"size": get_file_size(self.backup_path_private_files, format=True)
if os.path.exists(self.backup_path_files) and os.path.exists(
self.backup_path_private_files
):
summary.update(
{
"public": {
"path": self.backup_path_files,
"size": get_file_size(self.backup_path_files, format=True),
},
"private": {
"path": self.backup_path_private_files,
"size": get_file_size(self.backup_path_private_files, format=True),
},
}
})
)
return summary
@ -208,21 +307,29 @@ class BackupGenerator:
backup_summary = self.get_summary()
print("Backup Summary for {0} at {1}".format(frappe.local.site, now()))
title = max([len(x) for x in backup_summary])
path = max([len(x["path"]) for x in backup_summary.values()])
for _type, info in backup_summary.items():
print("{0:8}: {1:85} {2}".format(_type.title(), info["path"], info["size"]))
template = "{{0:{0}}}: {{1:{1}}} {{2}}".format(title, path)
print(template.format(_type.title(), info["path"], info["size"]))
def backup_files(self):
import subprocess
for folder in ("public", "private"):
files_path = frappe.get_site_path(folder, "files")
backup_path = self.backup_path_files if folder=="public" else self.backup_path_private_files
backup_path = (
self.backup_path_files if folder == "public" else self.backup_path_private_files
)
if self.compress_files:
cmd_string = "tar cf - {1} | gzip > {0}"
else:
cmd_string = "tar -cf {0} {1}"
output = subprocess.check_output(cmd_string.format(backup_path, files_path), shell=True)
output = subprocess.check_output(
cmd_string.format(backup_path, files_path), shell=True
)
if self.verbose and output:
print(output.decode("utf8"))
@ -236,34 +343,114 @@ class BackupGenerator:
def take_dump(self):
import frappe.utils
from frappe.utils.change_log import get_app_branch
db_exc = {
"mariadb": ("mysqldump", which("mysqldump")),
"postgres": ("pg_dump", which("pg_dump")),
}[self.db_type]
gzip_exc = which("gzip")
if not (gzip_exc and db_exc[1]):
_exc = "gzip" if not gzip_exc else db_exc[0]
frappe.throw(
f"{_exc} not found in PATH! This is required to take a backup.",
exc=frappe.ExecutableNotFound
)
db_exc = db_exc[0]
database_header_content = [
f"Backup generated by Frappe {frappe.__version__} on branch {get_app_branch('frappe') or 'N/A'}",
"",
]
# escape reserved characters
args = dict([item[0], frappe.utils.esc(str(item[1]), '$ ')]
for item in self.__dict__.copy().items())
args = frappe._dict(
[item[0], frappe.utils.esc(str(item[1]), "$ ")]
for item in self.__dict__.copy().items()
)
cmd_string = """mysqldump --single-transaction --quick --lock-tables=false -u %(user)s -p%(password)s %(db_name)s -h %(db_host)s -P %(db_port)s | gzip > %(backup_path_db)s """ % args
if self.backup_includes:
backup_info = ("Backing Up Tables: ", ", ".join(self.backup_includes))
elif self.backup_excludes:
backup_info = ("Skipping Tables: ", ", ".join(self.backup_excludes))
if self.db_type == 'postgres':
cmd_string = "pg_dump postgres://{user}:{password}@{db_host}:{db_port}/{db_name} | gzip > {backup_path_db}".format(
user=args.get('user'),
password=args.get('password'),
db_host=args.get('db_host'),
db_port=args.get('db_port'),
db_name=args.get('db_name'),
backup_path_db=args.get('backup_path_db')
if self.partial:
print(''.join(backup_info), "\n")
database_header_content.extend([
f"Partial Backup of Frappe Site {frappe.local.site}",
("Backup contains: " if self.backup_includes else "Backup excludes: ") + backup_info[1],
"",
])
generated_header = "\n".join([f"-- {x}" for x in database_header_content]) + "\n"
with gzip.open(args.backup_path_db, "wt") as f:
f.write(generated_header)
if self.db_type == "postgres":
if self.backup_includes:
args["include"] = " ".join(
["--table='public.\"{0}\"'".format(table) for table in self.backup_includes]
)
elif self.backup_excludes:
args["exclude"] = " ".join(
["--exclude-table-data='public.\"{0}\"'".format(table) for table in self.backup_excludes]
)
cmd_string = (
"{db_exc} postgres://{user}:{password}@{db_host}:{db_port}/{db_name}"
" {include} {exclude} | {gzip} >> {backup_path_db}"
)
err, out = frappe.utils.execute_in_shell(cmd_string)
else:
if self.backup_includes:
args["include"] = " ".join(["'{0}'".format(x) for x in self.backup_includes])
elif self.backup_excludes:
args["exclude"] = " ".join(
[
"--ignore-table='{0}.{1}'".format(frappe.conf.db_name, table)
for table in self.backup_excludes
]
)
cmd_string = (
"{db_exc} --single-transaction --quick --lock-tables=false -u {user}"
" -p{password} {db_name} -h {db_host} -P {db_port} {include} {exclude}"
" | {gzip} >> {backup_path_db}"
)
command = cmd_string.format(
user=args.user,
password=args.password,
db_exc=db_exc,
db_host=args.db_host,
db_port=args.db_port,
db_name=args.db_name,
backup_path_db=args.backup_path_db,
exclude=args.get("exclude", ""),
include=args.get("include", ""),
gzip=gzip_exc,
)
if self.verbose:
print(command + "\n")
err, out = frappe.utils.execute_in_shell(command)
def send_email(self):
"""
Sends the link to backup file located at erpnext/backups
Sends the link to backup file located at erpnext/backups
"""
from frappe.email import get_system_managers
recipient_list = get_system_managers()
db_backup_url = get_url(os.path.join('backups', os.path.basename(self.backup_path_db)))
files_backup_url = get_url(os.path.join('backups', os.path.basename(self.backup_path_files)))
db_backup_url = get_url(
os.path.join("backups", os.path.basename(self.backup_path_db))
)
files_backup_url = get_url(
os.path.join("backups", os.path.basename(self.backup_path_files))
)
msg = """Hello,
@ -275,11 +462,13 @@ Your backups are ready to be downloaded.
This link will be valid for 24 hours. A new backup will be available for
download only after 24 hours.""" % {
"db_backup_url": db_backup_url,
"files_backup_url": files_backup_url
"files_backup_url": files_backup_url,
}
datetime_str = datetime.fromtimestamp(os.stat(self.backup_path_db).st_ctime)
subject = datetime_str.strftime("%d/%m/%Y %H:%M:%S") + """ - Backup ready to be downloaded"""
subject = (
datetime_str.strftime("%d/%m/%Y %H:%M:%S") + """ - Backup ready to be downloaded"""
)
frappe.sendmail(recipients=recipient_list, msg=msg, subject=subject)
return recipient_list
@ -288,20 +477,29 @@ download only after 24 hours.""" % {
@frappe.whitelist()
def get_backup():
"""
This function is executed when the user clicks on
Toos > Download Backup
This function is executed when the user clicks on
Toos > Download Backup
"""
delete_temp_backups()
odb = BackupGenerator(frappe.conf.db_name, frappe.conf.db_name,\
frappe.conf.db_password, db_host = frappe.db.host,\
db_type=frappe.conf.db_type, db_port=frappe.conf.db_port)
odb = BackupGenerator(
frappe.conf.db_name,
frappe.conf.db_name,
frappe.conf.db_password,
db_host=frappe.db.host,
db_type=frappe.conf.db_type,
db_port=frappe.conf.db_port,
)
odb.get_backup()
recipient_list = odb.send_email()
frappe.msgprint(_("Download link for your backup will be emailed on the following email address: {0}").format(', '.join(recipient_list)))
frappe.msgprint(
_(
"Download link for your backup will be emailed on the following email address: {0}"
).format(", ".join(recipient_list))
)
@frappe.whitelist()
def fetch_latest_backups():
def fetch_latest_backups(partial=False):
"""Fetches paths of the latest backup taken in the last 30 days
Only for: System Managers
@ -317,43 +515,88 @@ def fetch_latest_backups():
db_type=frappe.conf.db_type,
db_port=frappe.conf.db_port,
)
database, public, private, config = odb.get_recent_backup(older_than=24 * 30)
database, public, private, config = odb.get_recent_backup(older_than=24 * 30, partial=partial)
return {
"database": database,
"public": public,
"private": private,
"config": config
}
return {"database": database, "public": public, "private": private, "config": config}
def scheduled_backup(older_than=6, ignore_files=False, backup_path=None, backup_path_db=None, backup_path_files=None, backup_path_private_files=None, backup_path_conf=None, force=False, verbose=False, compress=False):
def scheduled_backup(
older_than=6,
ignore_files=False,
backup_path=None,
backup_path_db=None,
backup_path_files=None,
backup_path_private_files=None,
backup_path_conf=None,
ignore_conf=False,
include_doctypes="",
exclude_doctypes="",
compress=False,
force=False,
verbose=False,
):
"""this function is called from scheduler
deletes backups older than 7 days
takes backup"""
odb = new_backup(older_than, ignore_files, backup_path=backup_path, backup_path_db=backup_path_db, backup_path_files=backup_path_files, backup_path_private_files=backup_path_private_files, backup_path_conf=backup_path_conf, force=force, verbose=verbose, compress=compress)
deletes backups older than 7 days
takes backup"""
odb = new_backup(
older_than=older_than,
ignore_files=ignore_files,
backup_path=backup_path,
backup_path_db=backup_path_db,
backup_path_files=backup_path_files,
backup_path_private_files=backup_path_private_files,
backup_path_conf=backup_path_conf,
ignore_conf=ignore_conf,
include_doctypes=include_doctypes,
exclude_doctypes=exclude_doctypes,
compress=compress,
force=force,
verbose=verbose,
)
return odb
def new_backup(older_than=6, ignore_files=False, backup_path=None, backup_path_db=None, backup_path_files=None, backup_path_private_files=None, backup_path_conf=None, force=False, verbose=False, compress=False):
delete_temp_backups(older_than = frappe.conf.keep_backups_for_hours or 24)
odb = BackupGenerator(frappe.conf.db_name, frappe.conf.db_name,\
frappe.conf.db_password,
backup_path=backup_path,
backup_path_db=backup_path_db,
backup_path_files=backup_path_files,
backup_path_private_files=backup_path_private_files,
backup_path_conf=backup_path_conf,
db_host = frappe.db.host,
db_port = frappe.db.port,
db_type = frappe.conf.db_type,
verbose=verbose,
compress_files=compress)
def new_backup(
older_than=6,
ignore_files=False,
backup_path=None,
backup_path_db=None,
backup_path_files=None,
backup_path_private_files=None,
backup_path_conf=None,
ignore_conf=False,
include_doctypes="",
exclude_doctypes="",
compress=False,
force=False,
verbose=False,
):
delete_temp_backups(older_than=frappe.conf.keep_backups_for_hours or 24)
odb = BackupGenerator(
frappe.conf.db_name,
frappe.conf.db_name,
frappe.conf.db_password,
db_host=frappe.db.host,
db_port=frappe.db.port,
db_type=frappe.conf.db_type,
backup_path=backup_path,
backup_path_db=backup_path_db,
backup_path_files=backup_path_files,
backup_path_private_files=backup_path_private_files,
backup_path_conf=backup_path_conf,
ignore_conf=ignore_conf,
include_doctypes=include_doctypes,
exclude_doctypes=exclude_doctypes,
verbose=verbose,
compress_files=compress,
)
odb.get_backup(older_than, ignore_files, force=force)
return odb
def delete_temp_backups(older_than=24):
"""
Cleans up the backup_link_path directory by deleting files older than 24 hours
Cleans up the backup_link_path directory by deleting files older than 24 hours
"""
backup_path = get_backup_path()
if os.path.exists(backup_path):
@ -363,54 +606,68 @@ def delete_temp_backups(older_than=24):
if is_file_old(this_file_path, older_than):
os.remove(this_file_path)
def is_file_old(db_file_name, older_than=24):
"""
Checks if file exists and is older than specified hours
Returns ->
True: file does not exist or file is old
False: file is new
"""
if os.path.isfile(db_file_name):
from datetime import timedelta
#Get timestamp of the file
file_datetime = datetime.fromtimestamp\
(os.stat(db_file_name).st_ctime)
if datetime.today() - file_datetime >= timedelta(hours = older_than):
if _verbose:
print("File is old")
return True
else:
if _verbose:
print("File is recent")
return False
def is_file_old(file_path, older_than=24):
"""
Checks if file exists and is older than specified hours
Returns ->
True: file does not exist or file is old
False: file is new
"""
if os.path.isfile(file_path):
from datetime import timedelta
# Get timestamp of the file
file_datetime = datetime.fromtimestamp(os.stat(file_path).st_ctime)
if datetime.today() - file_datetime >= timedelta(hours=older_than):
if _verbose:
print(f"File {file_path} is older than {older_than} hours")
return True
else:
if _verbose:
print("File does not exist")
return True
print(f"File {file_path} is recent")
return False
else:
if _verbose:
print(f"File {file_path} does not exist")
return True
def get_backup_path():
backup_path = frappe.utils.get_site_path(conf.get("backup_path", "private/backups"))
return backup_path
def backup(with_files=False, backup_path_db=None, backup_path_files=None, backup_path_private_files=None, backup_path_conf=None, quiet=False):
def backup(
with_files=False,
backup_path_db=None,
backup_path_files=None,
backup_path_private_files=None,
backup_path_conf=None,
quiet=False,
):
"Backup"
odb = scheduled_backup(ignore_files=not with_files, backup_path_db=backup_path_db, backup_path_files=backup_path_files, backup_path_private_files=backup_path_private_files, backup_path_conf=backup_path_conf, force=True)
odb = scheduled_backup(
ignore_files=not with_files,
backup_path_db=backup_path_db,
backup_path_files=backup_path_files,
backup_path_private_files=backup_path_private_files,
backup_path_conf=backup_path_conf,
force=True,
)
return {
"backup_path_db": odb.backup_path_db,
"backup_path_files": odb.backup_path_files,
"backup_path_private_files": odb.backup_path_private_files
"backup_path_private_files": odb.backup_path_private_files,
}
if __name__ == "__main__":
"""
is_file_old db_name user password db_host db_type db_port
get_backup db_name user password db_host db_type db_port
"""
import sys
cmd = sys.argv[1]
db_type = 'mariadb'
db_type = "mariadb"
try:
db_type = sys.argv[6]
except IndexError:
@ -423,19 +680,47 @@ if __name__ == "__main__":
pass
if cmd == "is_file_old":
odb = BackupGenerator(sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5] or "localhost", db_type=db_type, db_port=db_port)
odb = BackupGenerator(
sys.argv[2],
sys.argv[3],
sys.argv[4],
sys.argv[5] or "localhost",
db_type=db_type,
db_port=db_port,
)
is_file_old(odb.db_file_name)
if cmd == "get_backup":
odb = BackupGenerator(sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5] or "localhost", db_type=db_type, db_port=db_port)
odb = BackupGenerator(
sys.argv[2],
sys.argv[3],
sys.argv[4],
sys.argv[5] or "localhost",
db_type=db_type,
db_port=db_port,
)
odb.get_backup()
if cmd == "take_dump":
odb = BackupGenerator(sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5] or "localhost", db_type=db_type, db_port=db_port)
odb = BackupGenerator(
sys.argv[2],
sys.argv[3],
sys.argv[4],
sys.argv[5] or "localhost",
db_type=db_type,
db_port=db_port,
)
odb.take_dump()
if cmd == "send_email":
odb = BackupGenerator(sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5] or "localhost", db_type=db_type, db_port=db_port)
odb = BackupGenerator(
sys.argv[2],
sys.argv[3],
sys.argv[4],
sys.argv[5] or "localhost",
db_type=db_type,
db_port=db_port,
)
odb.send_email("abc.sql.gz")
if cmd == "delete_temp_backups":

2
frappe/utils/html_utils.py
View file

@ -34,7 +34,7 @@ def clean_email_html(html):
'margin', 'margin-top', 'margin-bottom', 'margin-left', 'margin-right',
'padding', 'padding-top', 'padding-bottom', 'padding-left', 'padding-right',
'font-size', 'font-weight', 'font-family', 'text-decoration',
'line-height', 'text-align', 'vertical-align'
'line-height', 'text-align', 'vertical-align', 'display'
],
protocols=['cid', 'http', 'https', 'mailto', 'data'],
strip=True, strip_comments=True)

2
frappe/website/doctype/web_form/web_form.js
View file

@ -59,6 +59,8 @@ frappe.ui.form.on("Web Form", {
default: field.default,
read_only: field.read_only,
depends_on: field.depends_on,
mandatory_depends_on: field.mandatory_depends_on,
read_only_depends_on: field.read_only_depends_on,
hidden: field.hidden,
description: field.description
});

27
frappe/website/doctype/web_form_field/web_form_field.json
View file

@ -18,6 +18,10 @@
"options",
"max_length",
"max_value",
"property_depends_on_section",
"mandatory_depends_on",
"column_break_16",
"read_only_depends_on",
"section_break_6",
"description",
"column_break_8",
@ -117,11 +121,32 @@
"fieldname": "default",
"fieldtype": "Data",
"label": "Default"
},
{
"fieldname": "property_depends_on_section",
"fieldtype": "Section Break",
"label": "Property Depends On"
},
{
"fieldname": "mandatory_depends_on",
"fieldtype": "Code",
"label": "Mandatory Depends On",
"options": "JS"
},
{
"fieldname": "column_break_16",
"fieldtype": "Column Break"
},
{
"fieldname": "read_only_depends_on",
"fieldtype": "Code",
"label": "Read Only Depends On",
"options": "JS"
}
],
"istable": 1,
"links": [],
"modified": "2020-05-13 13:35:08.454427",
"modified": "2020-11-10 23:20:44.354862",
"modified_by": "Administrator",
"module": "Website",
"name": "Web Form Field",