+Add logic for standard doctype events like Before Insert, After Submit, etc.
+
+
# set property
if "test" in doc.description:
- doc.status = 'Closed'
+ doc.status = 'Closed'
# validate
if "validate" in doc.description:
- raise frappe.ValidationError
+ raise frappe.ValidationError
# auto create another document
-if doc.allocted_to:
- frappe.get_doc(dict(
- doctype = 'ToDo'
- owner = doc.allocated_to,
- description = doc.subject
- )).insert()
-
+if doc.allocated_to:
+ frappe.get_doc(dict(
+ doctype = 'ToDo'
+ owner = doc.allocated_to,
+ description = doc.subject
+ )).insert()
+
+Respond to /api/method/<method-name> calls, just like whitelisted methods
# respond to API
@@ -79,6 +83,21 @@ if frappe.form_dict.message == "ping":
else:
frappe.response['message'] = "ok"
Add conditions to the where clause of list queries.
+
+# generate dynamic conditions and set it in the conditions variable
+tenant_id = frappe.db.get_value(...)
+conditions = 'tenant_id = {}'.format(tenant_id)
+
+# resulting select query
+select name from \`tabPerson\`
+where tenant_id = 2
+order by creation desc
+Order Overdue\n\nTransaction {{ name }} has exceeded Due Date. Please take necessary action.\n\nDetails\n\n- Customer: {{ customer }}\n- Amount: {{ grand_total }}\n\n\nThe fieldnames you can use in your email template are the fields in the document from which you are sending the email. You can find out the fields of any documents via Setup > Customize Form View and selecting the document type (e.g. Sales Invoice)
\n\nTemplates are compiled using the Jinja Templating Language. To learn more about Jinja, read this documentation.
\n" + }, + { + "default": "0", + "fieldname": "use_html", + "fieldtype": "Check", + "label": "Use HTML" + }, + { + "depends_on": "eval:doc.use_html", + "fieldname": "response_html", + "fieldtype": "Code", + "label": "Response ", + "options": "HTML" } ], "icon": "fa fa-comment", - "modified": "2019-10-30 14:15:00.956347", + "links": [], + "modified": "2020-11-30 14:12:50.321633", "modified_by": "Administrator", "module": "Email", "name": "Email Template", diff --git a/frappe/email/doctype/email_template/email_template.py b/frappe/email/doctype/email_template/email_template.py index 2743032331..6708e9dd3f 100644 --- a/frappe/email/doctype/email_template/email_template.py +++ b/frappe/email/doctype/email_template/email_template.py @@ -9,7 +9,29 @@ from six import string_types class EmailTemplate(Document): def validate(self): - validate_template(self.response) + if self.use_html: + validate_template(self.response_html) + else: + validate_template(self.response) + + def get_formatted_subject(self, doc): + return frappe.render_template(self.subject, doc) + + def get_formatted_response(self, doc): + if self.use_html: + return frappe.render_template(self.response_html, doc) + + return frappe.render_template(self.response, doc) + + def get_formatted_email(self, doc): + if isinstance(doc, string_types): + doc = json.loads(doc) + + return { + "subject" : self.get_formatted_subject(doc), + "message" : self.get_formatted_response(doc) + } + @frappe.whitelist() def get_email_template(template_name, doc): @@ -18,5 +40,4 @@ def get_email_template(template_name, doc): doc = json.loads(doc) email_template = frappe.get_doc("Email Template", template_name) - return {"subject" : frappe.render_template(email_template.subject, doc), - "message" : frappe.render_template(email_template.response, doc)} \ No newline at end of file + return email_template.get_formatted_email(doc) \ No newline at end of file diff --git a/frappe/email/doctype/newsletter/newsletter.py b/frappe/email/doctype/newsletter/newsletter.py index a4d60706eb..2791ebb75b 100755 --- a/frappe/email/doctype/newsletter/newsletter.py +++ b/frappe/email/doctype/newsletter/newsletter.py @@ -85,11 +85,11 @@ class Newsletter(WebsiteGenerator): self.db_set("scheduled_to_send", len(self.recipients)) def get_message(self): - + if self.content_type == "HTML": + return frappe.render_template(self.message_html, {"doc": self.as_dict()}) return { 'Rich Text': self.message, - 'Markdown': markdown(self.message_md), - 'HTML': self.message_html + 'Markdown': markdown(self.message_md) }[self.content_type or 'Rich Text'] def get_recipients(self): diff --git a/frappe/email/doctype/notification/notification.json b/frappe/email/doctype/notification/notification.json index 73a84e1d3e..c1c877efd4 100644 --- a/frappe/email/doctype/notification/notification.json +++ b/frappe/email/doctype/notification/notification.json @@ -207,7 +207,7 @@ "label": "Value To Be Set" }, { - "depends_on": "eval:in_list(['Email', 'SMS'], doc.channel)", + "depends_on": "eval:doc.channel !=\"Slack\"", "fieldname": "column_break_5", "fieldtype": "Section Break", "label": "Recipients" @@ -281,7 +281,7 @@ "icon": "fa fa-envelope", "index_web_pages_for_search": 1, "links": [], - "modified": "2020-10-28 11:04:54.955567", + "modified": "2020-11-24 14:25:43.245677", "modified_by": "Administrator", "module": "Email", "name": "Notification", diff --git a/frappe/email/doctype/notification/notification.py b/frappe/email/doctype/notification/notification.py index 75281d427e..2ea7a3785e 100644 --- a/frappe/email/doctype/notification/notification.py +++ b/frappe/email/doctype/notification/notification.py @@ -181,6 +181,7 @@ def get_context(context): 'document_type': doc.doctype, 'document_name': doc.name, 'subject': subject, + 'from_user': doc.modified_by or doc.owner, 'email_content': frappe.render_template(self.message, context), 'attached_file': attachments and json.dumps(attachments[0]) } diff --git a/frappe/email/smtp.py b/frappe/email/smtp.py index f53b835757..9ba81fa146 100644 --- a/frappe/email/smtp.py +++ b/frappe/email/smtp.py @@ -210,10 +210,9 @@ class SMTPServer: try: if self.use_ssl: if not self.port: - self.smtp_port = 465 + self.port = 465 - self._sess = smtplib.SMTP_SSL((self.server or "").encode('utf-8'), - cint(self.port) or None) + self._sess = smtplib.SMTP_SSL((self.server or ""), cint(self.port)) else: if self.use_tls and not self.port: self.port = 587 diff --git a/frappe/email/test_smtp.py b/frappe/email/test_smtp.py new file mode 100644 index 0000000000..869d708430 --- /dev/null +++ b/frappe/email/test_smtp.py @@ -0,0 +1,25 @@ +# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors +# License: The MIT License + +import unittest +from frappe.email.smtp import SMTPServer + +class TestSMTP(unittest.TestCase): + def test_smtp_ssl_session(self): + for port in [None, 0, 465, "465"]: + make_server(port, 1, 0) + + def test_smtp_tls_session(self): + for port in [None, 0, 587, "587"]: + make_server(port, 0, 1) + + +def make_server(port, ssl, tls): + server = SMTPServer( + server = "smtp.gmail.com", + port = port, + use_ssl = ssl, + use_tls = tls + ) + + server.sess \ No newline at end of file diff --git a/frappe/exceptions.py b/frappe/exceptions.py index 267f5410af..82fbff7a90 100644 --- a/frappe/exceptions.py +++ b/frappe/exceptions.py @@ -106,8 +106,10 @@ class InvalidDates(ValidationError): pass class DataTooLongException(ValidationError): pass class FileAlreadyAttachedException(Exception): pass class DocumentAlreadyRestored(Exception): pass +class AttachmentLimitReached(Exception): pass # OAuth exceptions class InvalidAuthorizationHeader(CSRFTokenError): pass class InvalidAuthorizationPrefix(CSRFTokenError): pass class InvalidAuthorizationToken(CSRFTokenError): pass -class InvalidDatabaseFile(ValidationError): pass \ No newline at end of file +class InvalidDatabaseFile(ValidationError): pass +class ExecutableNotFound(FileNotFoundError): pass diff --git a/frappe/geo/utils.py b/frappe/geo/utils.py new file mode 100644 index 0000000000..d94a13ea41 --- /dev/null +++ b/frappe/geo/utils.py @@ -0,0 +1,96 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020, Frappe Technologies and contributors +# For license information, please see license.txt + +from __future__ import unicode_literals + +import frappe + +from pymysql import InternalError + + +@frappe.whitelist() +def get_coords(doctype, filters, type): + '''Get a geojson dict representing a doctype.''' + filters_sql = get_coords_conditions(doctype, filters)[4:] + + coords = None + if type == 'location_field': + coords = return_location(doctype, filters_sql) + elif type == 'coordinates': + coords = return_coordinates(doctype, filters_sql) + + out = convert_to_geojson(type, coords) + return out + +def convert_to_geojson(type, coords): + '''Converts GPS coordinates to geoJSON string.''' + geojson = {"type": "FeatureCollection", "features": None} + + if type == 'location_field': + geojson['features'] = merge_location_features_in_one(coords) + elif type == 'coordinates': + geojson['features'] = create_gps_markers(coords) + + return geojson + + +def merge_location_features_in_one(coords): + '''Merging all features from location field.''' + geojson_dict = [] + for element in coords: + geojson_loc = frappe.parse_json(element['location']) + if not geojson_loc: + continue + for coord in geojson_loc['features']: + coord['properties']['name'] = element['name'] + geojson_dict.append(coord.copy()) + + return geojson_dict + + +def create_gps_markers(coords): + '''Build Marker based on latitude and longitude.''' + geojson_dict = [] + for i in coords: + node = {"type": "Feature", "properties": {}, "geometry": {"type": "Point", "coordinates": None}} + node['properties']['name'] = i.name + node['geometry']['coordinates'] = [i.latitude, i.longitude] + geojson_dict.append(node.copy()) + + return geojson_dict + + +def return_location(doctype, filters_sql): + '''Get name and location fields for Doctype.''' + if filters_sql: + try: + coords = frappe.db.sql('''SELECT name, location FROM `tab{}` WHERE {}'''.format(doctype, filters_sql), as_dict=True) + except InternalError: + frappe.msgprint(frappe._('This Doctype does not contain location fields'), raise_exception=True) + return + else: + coords = frappe.get_all(doctype, fields=['name', 'location']) + return coords + + +def return_coordinates(doctype, filters_sql): + '''Get name, latitude and longitude fields for Doctype.''' + if filters_sql: + try: + coords = frappe.db.sql('''SELECT name, latitude, longitude FROM `tab{}` WHERE {}'''.format(doctype, filters_sql), as_dict=True) + except InternalError: + frappe.msgprint(frappe._('This Doctype does not contain latitude and longitude fields'), raise_exception=True) + return + else: + coords = frappe.get_all(doctype, fields=['name', 'latitude', 'longitude']) + return coords + + +def get_coords_conditions(doctype, filters=None): + '''Returns SQL conditions with user permissions and filters for event queries.''' + from frappe.desk.reportview import get_filters_cond + if not frappe.has_permission(doctype): + frappe.throw(frappe._("Not Permitted"), frappe.PermissionError) + + return get_filters_cond(doctype, filters, [], with_match_conditions=True) diff --git a/frappe/hooks.py b/frappe/hooks.py index d8c8cd841c..ea0a91a639 100644 --- a/frappe/hooks.py +++ b/frappe/hooks.py @@ -18,7 +18,7 @@ app_email = "info@frappe.io" docs_app = "frappe_io" -translator_url = "https://translatev2.erpnext.com" +translator_url = "https://translate.erpnext.com" before_install = "frappe.utils.install.before_install" after_install = "frappe.utils.install.after_install" @@ -94,6 +94,7 @@ permission_query_conditions = { "User": "frappe.core.doctype.user.user.get_permission_query_conditions", "Dashboard Settings": "frappe.desk.doctype.dashboard_settings.dashboard_settings.get_permission_query_conditions", "Notification Log": "frappe.desk.doctype.notification_log.notification_log.get_permission_query_conditions", + "Dashboard": "frappe.desk.doctype.dashboard.dashboard.get_permission_query_conditions", "Dashboard Chart": "frappe.desk.doctype.dashboard_chart.dashboard_chart.get_permission_query_conditions", "Number Card": "frappe.desk.doctype.number_card.number_card.get_permission_query_conditions", "Notification Settings": "frappe.desk.doctype.notification_settings.notification_settings.get_permission_query_conditions", diff --git a/frappe/installer.py b/frappe/installer.py index be9b04d453..a11c8dfbfa 100755 --- a/frappe/installer.py +++ b/frappe/installer.py @@ -3,8 +3,90 @@ import json import os -from frappe.defaults import _clear_cache +import sys + import frappe +from frappe.defaults import _clear_cache + + +def _new_site( + db_name, + site, + mariadb_root_username=None, + mariadb_root_password=None, + admin_password=None, + verbose=False, + install_apps=None, + source_sql=None, + force=False, + no_mariadb_socket=False, + reinstall=False, + db_password=None, + db_type=None, + db_host=None, + db_port=None, + new_site=False, +): + """Install a new Frappe site""" + + if not force and os.path.exists(site): + print("Site {0} already exists".format(site)) + sys.exit(1) + + if no_mariadb_socket and not db_type == "mariadb": + print("--no-mariadb-socket requires db_type to be set to mariadb.") + sys.exit(1) + + if not db_name: + import hashlib + db_name = "_" + hashlib.sha1(site.encode()).hexdigest()[:16] + + frappe.init(site=site) + + from frappe.commands.scheduler import _is_scheduler_enabled + from frappe.utils import get_site_path, scheduler, touch_file + + try: + # enable scheduler post install? + enable_scheduler = _is_scheduler_enabled() + except Exception: + enable_scheduler = False + + make_site_dirs() + + installing = touch_file(get_site_path("locks", "installing.lock")) + + install_db( + root_login=mariadb_root_username, + root_password=mariadb_root_password, + db_name=db_name, + admin_password=admin_password, + verbose=verbose, + source_sql=source_sql, + force=force, + reinstall=reinstall, + db_password=db_password, + db_type=db_type, + db_host=db_host, + db_port=db_port, + no_mariadb_socket=no_mariadb_socket, + ) + apps_to_install = ( + ["frappe"] + (frappe.conf.get("install_apps") or []) + (list(install_apps) or []) + ) + + for app in apps_to_install: + install_app(app, verbose=verbose, set_as_patched=not source_sql) + + os.remove(installing) + + scheduler.toggle_scheduler(enable_scheduler) + frappe.db.commit() + + scheduler_status = ( + "disabled" if frappe.utils.scheduler.is_scheduler_disabled() else "enabled" + ) + print("*** Scheduler is", scheduler_status, "***") def install_db(root_login="root", root_password=None, db_name=None, source_sql=None, @@ -36,9 +118,9 @@ def install_db(root_login="root", root_password=None, db_name=None, source_sql=N def install_app(name, verbose=False, set_as_patched=True): from frappe.core.doctype.scheduled_job_type.scheduled_job_type import sync_jobs - from frappe.utils.fixtures import sync_fixtures from frappe.model.sync import sync_for from frappe.modules.utils import sync_customizations + from frappe.utils.fixtures import sync_fixtures frappe.flags.in_install = name frappe.flags.ignore_in_install = False @@ -180,11 +262,11 @@ def remove_app(app_name, dry_run=False, yes=False, no_backup=False, force=False) for record in frappe.get_all(doctype, filters={"module": module_name}, pluck="name"): print(f"* removing {doctype} '{record}'...") if not dry_run: - frappe.delete_doc(doctype, record, ignore_on_trash=True) + frappe.delete_doc(doctype, record, ignore_on_trash=True, force=True) print(f"* removing Module Def '{module_name}'...") if not dry_run: - frappe.delete_doc("Module Def", module_name, ignore_on_trash=True) + frappe.delete_doc("Module Def", module_name, ignore_on_trash=True, force=True) for doctype in set(drop_doctypes): print(f"* dropping Table for '{doctype}'...") @@ -347,6 +429,28 @@ def remove_missing_apps(): frappe.db.set_global("installed_apps", json.dumps(installed_apps)) +def extract_sql_from_archive(sql_file_path): + """Return the path of an SQL file if the passed argument is the path of a gzipped + SQL file or an SQL file path. The path may be absolute or relative from the bench + root directory or the sites sub-directory. + + Args: + sql_file_path (str): Path of the SQL file + + Returns: + str: Path of the decompressed SQL file + """ + from frappe.utils import get_bench_relative_path + sql_file_path = get_bench_relative_path(sql_file_path) + # Extract the gzip file if user has passed *.sql.gz file instead of *.sql file + if sql_file_path.endswith('sql.gz'): + decompressed_file_name = extract_sql_gzip(sql_file_path) + else: + decompressed_file_name = sql_file_path + + return decompressed_file_name + + def extract_sql_gzip(sql_gz_path): import subprocess @@ -361,9 +465,13 @@ def extract_sql_gzip(sql_gz_path): return decompressed_file -def extract_files(site_name, file_path, folder_name): - import subprocess + +def extract_files(site_name, file_path): import shutil + import subprocess + from frappe.utils import get_bench_relative_path + + file_path = get_bench_relative_path(file_path) # Need to do frappe.init to maintain the site locals frappe.init(site=site_name) @@ -391,6 +499,12 @@ def extract_files(site_name, file_path, folder_name): def is_downgrade(sql_file_path, verbose=False): """checks if input db backup will get downgraded on current bench""" + + # This function is only tested with mariadb + # TODO: Add postgres support + if frappe.conf.db_type not in (None, "mariadb"): + return False + from semantic_version import Version head = "INSERT INTO `tabInstalled Application` VALUES" @@ -424,6 +538,37 @@ def is_downgrade(sql_file_path, verbose=False): return downgrade +def is_partial(sql_file_path): + with open(sql_file_path) as f: + header = " ".join([f.readline() for _ in range(5)]) + if "Partial Backup" in header: + return True + return False + + +def partial_restore(sql_file_path, verbose=False): + sql_file = extract_sql_from_archive(sql_file_path) + + if frappe.conf.db_type in (None, "mariadb"): + from frappe.database.mariadb.setup_db import import_db_from_sql + elif frappe.conf.db_type == "postgres": + from frappe.database.postgres.setup_db import import_db_from_sql + import warnings + from click import style + warn = style( + "Delete the tables you want to restore manually before attempting" + " partial restore operation for PostreSQL databases", + fg="yellow" + ) + warnings.warn(warn) + + import_db_from_sql(source_sql=sql_file, verbose=verbose) + + # Removing temporarily created file + if sql_file != sql_file_path: + os.remove(sql_file) + + def validate_database_sql(path, _raise=True): """Check if file has contents and if DefaultValue table exists @@ -431,23 +576,29 @@ def validate_database_sql(path, _raise=True): path (str): Path of the decompressed SQL file _raise (bool, optional): Raise exception if invalid file. Defaults to True. """ - to_raise = False + empty_file = False + missing_table = True + error_message = "" if not os.path.getsize(path): error_message = f"{path} is an empty file!" - to_raise = True + empty_file = True - if not _raise: + # dont bother checking if empty file + if not empty_file: with open(path, "r") as f: for line in f: if 'tabDefaultValue' in line: - error_message = "Table `tabDefaultValue` not found in file." - to_raise = True + missing_table = False + break + + if missing_table: + error_message = "Table `tabDefaultValue` not found in file." if error_message: import click click.secho(error_message, fg="red") - if _raise and to_raise: + if _raise and (missing_table or empty_file): raise frappe.InvalidDatabaseFile diff --git a/frappe/integrations/doctype/dropbox_settings/dropbox_settings.py b/frappe/integrations/doctype/dropbox_settings/dropbox_settings.py index 6b95a3f5bf..71445b44d7 100644 --- a/frappe/integrations/doctype/dropbox_settings/dropbox_settings.py +++ b/frappe/integrations/doctype/dropbox_settings/dropbox_settings.py @@ -9,7 +9,7 @@ import frappe import os from frappe import _ from frappe.model.document import Document -from frappe.integrations.offsite_backup_utils import get_latest_backup_file, send_email, validate_file_size +from frappe.integrations.offsite_backup_utils import get_latest_backup_file, send_email, validate_file_size, get_chunk_site from frappe.integrations.utils import make_post_request from frappe.utils import (cint, get_request_site_address, get_files_path, get_backups_path, get_url, encode) @@ -167,8 +167,9 @@ def upload_file_to_dropbox(filename, folder, dropbox_client): return create_folder_if_not_exists(folder, dropbox_client) - chunk_size = 15 * 1024 * 1024 file_size = os.path.getsize(encode(filename)) + chunk_size = get_chunk_site(file_size) + mode = (dropbox.files.WriteMode.overwrite) f = open(encode(filename), 'rb') diff --git a/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.json b/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.json index 123bb21e88..2ca1723cb2 100755 --- a/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.json +++ b/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.json @@ -18,12 +18,9 @@ "bucket", "endpoint_url", "column_break_13", - "region", "backup_details_section", "frequency", - "backup_files", - "column_break_18", - "backup_limit" + "backup_files" ], "fields": [ { @@ -42,7 +39,7 @@ }, { "default": "1", - "description": "Note: By default emails for failed backups are sent.", + "description": "By default, emails are only sent for failed backups.", "fieldname": "send_email_for_successful_backup", "fieldtype": "Check", "label": "Send Email for Successful Backup" @@ -73,14 +70,7 @@ "reqd": 1 }, { - "default": "us-east-1", - "description": "See https://docs.aws.amazon.com/general/latest/gr/s3.html for details.", - "fieldname": "region", - "fieldtype": "Select", - "label": "Region", - "options": "us-east-1\nus-east-2\nus-west-1\nus-west-2\naf-south-1\nap-east-1\nap-south-1\nap-southeast-1\nap-southeast-2\nap-northeast-1\nap-northeast-2\nap-northeast-3\nca-central-1\ncn-north-1\ncn-northwest-1\neu-central-1\neu-west-1\neu-west-2\neu-west-3\neu-south-1\neu-north-1\nme-south-1\nsa-east-1" - }, - { + "default": "https://s3.amazonaws.com", "fieldname": "endpoint_url", "fieldtype": "Data", "label": "Endpoint URL" @@ -92,14 +82,6 @@ "mandatory_depends_on": "enabled", "reqd": 1 }, - { - "description": "Set to 0 for no limit on the number of backups taken", - "fieldname": "backup_limit", - "fieldtype": "Int", - "label": "Backup Limit", - "mandatory_depends_on": "enabled", - "reqd": 1 - }, { "depends_on": "enabled", "fieldname": "api_access_section", @@ -142,16 +124,12 @@ "fieldname": "backup_files", "fieldtype": "Check", "label": "Backup Files" - }, - { - "fieldname": "column_break_18", - "fieldtype": "Column Break" } ], "hide_toolbar": 1, "issingle": 1, "links": [], - "modified": "2020-07-27 17:27:21.400000", + "modified": "2020-12-07 15:30:55.047689", "modified_by": "Administrator", "module": "Integrations", "name": "S3 Backup Settings", @@ -172,4 +150,4 @@ "sort_field": "modified", "sort_order": "DESC", "track_changes": 1 -} +} \ No newline at end of file diff --git a/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.py b/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.py index 7c90d37f82..308d34c5c2 100755 --- a/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.py +++ b/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.py @@ -24,6 +24,7 @@ class S3BackupSettings(Document): if not self.endpoint_url: self.endpoint_url = 'https://s3.amazonaws.com' + conn = boto3.client( 's3', aws_access_key_id=self.access_key_id, @@ -31,25 +32,21 @@ class S3BackupSettings(Document): endpoint_url=self.endpoint_url ) - bucket_lower = str(self.bucket) - - try: - conn.list_buckets() - - except ClientError: - frappe.throw(_("Invalid Access Key ID or Secret Access Key.")) - try: # Head_bucket returns a 200 OK if the bucket exists and have access to it. - conn.head_bucket(Bucket=bucket_lower) + # Requires ListBucket permission + conn.head_bucket(Bucket=self.bucket) except ClientError as e: error_code = e.response['Error']['Code'] + bucket_name = frappe.bold(self.bucket) if error_code == '403': - frappe.throw(_("Do not have permission to access {0} bucket.").format(bucket_lower)) - else: # '400'-Bad request or '404'-Not Found return - # try to create bucket - conn.create_bucket(Bucket=bucket_lower, CreateBucketConfiguration={ - 'LocationConstraint': self.region}) + msg = _("Do not have permission to access bucket {0}.").format(bucket_name) + elif error_code == '404': + msg = _("Bucket {0} not found.").format(bucket_name) + else: + msg = e.args[0] + + frappe.throw(msg) @frappe.whitelist() @@ -70,11 +67,13 @@ def take_backups_weekly(): def take_backups_monthly(): take_backups_if("Monthly") + def take_backups_if(freq): if cint(frappe.db.get_value("S3 Backup Settings", None, "enabled")): if frappe.db.get_value("S3 Backup Settings", None, "frequency") == freq: take_backups_s3() + @frappe.whitelist() def take_backups_s3(retry_count=0): try: @@ -146,42 +145,13 @@ def backup_to_s3(): if files_filename: upload_file_to_s3(files_filename, folder, conn, bucket) - delete_old_backups(doc.backup_limit, bucket) - def upload_file_to_s3(filename, folder, conn, bucket): destpath = os.path.join(folder, os.path.basename(filename)) try: print("Uploading file:", filename) - conn.upload_file(filename, bucket, destpath) + conn.upload_file(filename, bucket, destpath) # Requires PutObject permission except Exception as e: frappe.log_error() print("Error uploading: %s" % (e)) - - -def delete_old_backups(limit, bucket): - all_backups = [] - doc = frappe.get_single("S3 Backup Settings") - backup_limit = int(limit) - - s3 = boto3.resource( - 's3', - aws_access_key_id=doc.access_key_id, - aws_secret_access_key=doc.get_password('secret_access_key'), - endpoint_url=doc.endpoint_url or 'https://s3.amazonaws.com' - ) - - bucket = s3.Bucket(bucket) - objects = bucket.meta.client.list_objects_v2(Bucket=bucket.name, Delimiter='/') - if objects: - for obj in objects.get('CommonPrefixes'): - all_backups.append(obj.get('Prefix')) - - oldest_backup = sorted(all_backups)[0] if all_backups else '' - - if len(all_backups) > backup_limit: - print("Deleting Backup: {0}".format(oldest_backup)) - for obj in bucket.objects.filter(Prefix=oldest_backup): - # delete all keys that are inside the oldest_backup - s3.Object(bucket.name, obj.key).delete() diff --git a/frappe/integrations/doctype/webhook/webhook.py b/frappe/integrations/doctype/webhook/webhook.py index f1556aa661..ad64d9f714 100644 --- a/frappe/integrations/doctype/webhook/webhook.py +++ b/frappe/integrations/doctype/webhook/webhook.py @@ -85,7 +85,7 @@ def enqueue_webhook(doc, webhook): for i in range(3): try: - r = requests.post(webhook.request_url, data=json.dumps(data), headers=headers, timeout=5) + r = requests.post(webhook.request_url, data=json.dumps(data, default=str), headers=headers, timeout=5) r.raise_for_status() frappe.logger().debug({"webhook_success": r.text}) break diff --git a/frappe/integrations/frappe_providers/frappecloud.py b/frappe/integrations/frappe_providers/frappecloud.py index e09f09a44b..f60344ee8f 100644 --- a/frappe/integrations/frappe_providers/frappecloud.py +++ b/frappe/integrations/frappe_providers/frappecloud.py @@ -6,7 +6,7 @@ import frappe def frappecloud_migrator(local_site): - print("Retreiving Site Migrator...") + print("Retrieving Site Migrator...") remote_site = frappe.conf.frappecloud_url or "frappecloud.com" request_url = "https://{}/api/method/press.api.script".format(remote_site) request = requests.get(request_url) diff --git a/frappe/integrations/oauth2.py b/frappe/integrations/oauth2.py index c8dfc52c95..a750c8328c 100644 --- a/frappe/integrations/oauth2.py +++ b/frappe/integrations/oauth2.py @@ -1,41 +1,50 @@ from __future__ import unicode_literals -import frappe, json -from frappe.oauth import OAuthWebRequestValidator, WebApplicationServer + +import hashlib +import json +from urllib.parse import quote, urlencode, urlparse + +import jwt from oauthlib.oauth2 import FatalClientError, OAuth2Error -from werkzeug import url_fix -from six.moves.urllib.parse import quote, urlencode, urlparse -from frappe.integrations.doctype.oauth_provider_settings.oauth_provider_settings import get_oauth_settings + +import frappe from frappe import _ +from frappe.oauth import OAuthWebRequestValidator, WebApplicationServer +from frappe.integrations.doctype.oauth_provider_settings.oauth_provider_settings import get_oauth_settings def get_oauth_server(): if not getattr(frappe.local, 'oauth_server', None): oauth_validator = OAuthWebRequestValidator() - frappe.local.oauth_server = WebApplicationServer(oauth_validator) + frappe.local.oauth_server = WebApplicationServer(oauth_validator) return frappe.local.oauth_server -def get_urlparams_from_kwargs(param_kwargs): +def sanitize_kwargs(param_kwargs): arguments = param_kwargs - if arguments.get("data"): - arguments.pop("data") - if arguments.get("cmd"): - arguments.pop("cmd") + arguments.pop('data', None) + arguments.pop('cmd', None) - return urlencode(arguments) + return arguments @frappe.whitelist() def approve(*args, **kwargs): r = frappe.request - uri = url_fix(r.url.replace("+"," ")) - http_method = r.method - body = r.get_data() - headers = r.headers try: - scopes, frappe.flags.oauth_credentials = get_oauth_server().validate_authorization_request(uri, http_method, body, headers) + scopes, frappe.flags.oauth_credentials = get_oauth_server().validate_authorization_request( + r.url, + r.method, + r.get_data(), + r.headers + ) - headers, body, status = get_oauth_server().create_authorization_response(uri=frappe.flags.oauth_credentials['redirect_uri'], \ - body=body, headers=headers, scopes=scopes, credentials=frappe.flags.oauth_credentials) + headers, body, status = get_oauth_server().create_authorization_response( + uri=frappe.flags.oauth_credentials['redirect_uri'], + body=r.get_data(), + headers=r.headers, + scopes=scopes, + credentials=frappe.flags.oauth_credentials + ) uri = headers.get('Location', None) frappe.local.response["type"] = "redirect" @@ -47,34 +56,28 @@ def approve(*args, **kwargs): return e @frappe.whitelist(allow_guest=True) -def authorize(*args, **kwargs): - #Fetch provider URL from settings - oauth_settings = get_oauth_settings() - params = get_urlparams_from_kwargs(kwargs) - request_url = urlparse(frappe.request.url) - success_url = request_url.scheme + "://" + request_url.netloc + "/api/method/frappe.integrations.oauth2.approve?" + params +def authorize(**kwargs): + success_url = "/api/method/frappe.integrations.oauth2.approve?" + encode_params(sanitize_kwargs(kwargs)) failure_url = frappe.form_dict["redirect_uri"] + "?error=access_denied" - if frappe.session['user']=='Guest': + if frappe.session.user == 'Guest': #Force login, redirect to preauth again. frappe.local.response["type"] = "redirect" - frappe.local.response["location"] = "/login?redirect-to=/api/method/frappe.integrations.oauth2.authorize?" + quote(params.replace("+"," ")) - - elif frappe.session['user']!='Guest': + frappe.local.response["location"] = "/login?" + encode_params({'redirect-to': frappe.request.url}) + else: try: r = frappe.request - uri = url_fix(r.url) - http_method = r.method - body = r.get_data() - headers = r.headers - - scopes, frappe.flags.oauth_credentials = get_oauth_server().validate_authorization_request(uri, http_method, body, headers) + scopes, frappe.flags.oauth_credentials = get_oauth_server().validate_authorization_request( + r.url, + r.method, + r.get_data(), + r.headers + ) skip_auth = frappe.db.get_value("OAuth Client", frappe.flags.oauth_credentials['client_id'], "skip_authorization") unrevoked_tokens = frappe.get_all("OAuth Bearer Token", filters={"status":"Active"}) - if skip_auth or (oauth_settings["skip_authorization"] == "Auto" and len(unrevoked_tokens)): - + if skip_auth or (get_oauth_settings().skip_authorization == "Auto" and unrevoked_tokens): frappe.local.response["type"] = "redirect" frappe.local.response["location"] = success_url else: @@ -87,7 +90,6 @@ def authorize(*args, **kwargs): }) resp_html = frappe.render_template("templates/includes/oauth_confirmation.html", response_html_params) frappe.respond_as_web_page("Confirm Access", resp_html) - except FatalClientError as e: return e except OAuth2Error as e: @@ -95,20 +97,20 @@ def authorize(*args, **kwargs): @frappe.whitelist(allow_guest=True) def get_token(*args, **kwargs): - r = frappe.request - - uri = url_fix(r.url) - http_method = r.method - body = r.form - headers = r.headers - #Check whether frappe server URL is set frappe_server_url = frappe.db.get_value("Social Login Key", "frappe", "base_url") or None if not frappe_server_url: frappe.throw(_("Please set Base URL in Social Login Key for Frappe")) try: - headers, body, status = get_oauth_server().create_token_response(uri, http_method, body, headers, frappe.flags.oauth_credentials) + r = frappe.request + headers, body, status = get_oauth_server().create_token_response( + r.url, + r.method, + r.form, + r.headers, + frappe.flags.oauth_credentials + ) out = frappe._dict(json.loads(body)) if not out.error and "openid" in out.scope: token_user = frappe.db.get_value("OAuth Bearer Token", out.access_token, "user") @@ -116,7 +118,7 @@ def get_token(*args, **kwargs): client_secret = frappe.db.get_value("OAuth Client", token_client, "client_secret") if token_user in ["Guest", "Administrator"]: frappe.throw(_("Logged in as Guest or Administrator")) - import hashlib + id_token_header = { "typ":"jwt", "alg":"HS256" @@ -128,9 +130,10 @@ def get_token(*args, **kwargs): "iss": frappe_server_url, "at_hash": frappe.oauth.calculate_at_hash(out.access_token, hashlib.sha256) } - import jwt + id_token_encoded = jwt.encode(id_token, client_secret, algorithm='HS256', headers=id_token_header) - out.update({"id_token":str(id_token_encoded)}) + out.update({"id_token": str(id_token_encoded)}) + frappe.local.response = out except FatalClientError as e: @@ -140,12 +143,12 @@ def get_token(*args, **kwargs): @frappe.whitelist(allow_guest=True) def revoke_token(*args, **kwargs): r = frappe.request - uri = url_fix(r.url) - http_method = r.method - body = r.form - headers = r.headers - - headers, body, status = get_oauth_server().create_revocation_response(uri, headers=headers, body=body, http_method=http_method) + headers, body, status = get_oauth_server().create_revocation_response( + r.url, + headers=r.headers, + body=r.form, + http_method=r.method + ) frappe.local.response['http_status_code'] = status if status == 200: @@ -174,15 +177,22 @@ def openid_profile(*args, **kwargs): "email": name, "picture": picture }) - + frappe.local.response = user_profile def validate_url(url_string): try: result = urlparse(url_string) - if result.scheme and result.scheme in ["http", "https", "ftp", "ftps"]: - return True - else: - return False + return result.scheme and result.scheme in ["http", "https", "ftp", "ftps"] except: - return False \ No newline at end of file + return False + +def encode_params(params): + """ + Encode a dict of params into a query string. + + Use `quote_via=urllib.parse.quote` so that whitespaces will be encoded as + `%20` instead of as `+`. This is needed because oauthlib cannot handle `+` + as a whitespace. + """ + return urlencode(params, quote_via=quote) diff --git a/frappe/integrations/offsite_backup_utils.py b/frappe/integrations/offsite_backup_utils.py index db176538e4..48a2c89107 100644 --- a/frappe/integrations/offsite_backup_utils.py +++ b/frappe/integrations/offsite_backup_utils.py @@ -6,8 +6,7 @@ from __future__ import unicode_literals import frappe import glob import os -from frappe.utils import split_emails, get_backups_path - +from frappe.utils import split_emails, cint def send_email(success, service_name, doctype, email_field, error_status=None): recipients = get_recipients(doctype, email_field) @@ -81,6 +80,22 @@ def get_file_size(file_path, unit): return file_size +def get_chunk_site(file_size): + ''' this function will return chunk size in megabytes based on file size ''' + + file_size_in_gb = cint(file_size/1024/1024) + + MB = 1024 * 1024 + if file_size_in_gb > 5000: + return 200 * MB + elif file_size_in_gb >= 3000: + return 150 * MB + elif file_size_in_gb >= 1000: + return 100 * MB + elif file_size_in_gb >= 500: + return 50 * MB + else: + return 15 * MB def validate_file_size(): frappe.flags.create_new_backup = True @@ -98,4 +113,4 @@ def generate_files_backup(): db_type=frappe.conf.db_type, db_port=frappe.conf.db_port) backup.set_backup_file_name() - backup.zip_files() \ No newline at end of file + backup.zip_files() diff --git a/frappe/model/base_document.py b/frappe/model/base_document.py index 0a219b4253..5d86b3bac8 100644 --- a/frappe/model/base_document.py +++ b/frappe/model/base_document.py @@ -802,12 +802,12 @@ class BaseDocument(object): if translated: val = _(val) - if absolute_value and isinstance(val, (int, float)): - val = abs(self.get(fieldname)) - if not doc: doc = getattr(self, "parent_doc", None) or self + if (absolute_value or doc.get('absolute_value')) and isinstance(val, (int, float)): + val = abs(self.get(fieldname)) + return format_value(val, df=df, doc=doc, currency=currency) def is_print_hide(self, fieldname, df=None, for_print=True): diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py index ace9b04cec..c799586d61 100644 --- a/frappe/model/db_query.py +++ b/frappe/model/db_query.py @@ -18,6 +18,7 @@ from frappe.client import check_parent_permission from frappe.model.utils.user_settings import get_user_settings, update_user_settings from frappe.utils import flt, cint, get_time, make_filter_tuple, get_filter, add_to_date, cstr, get_timespan_date_range from frappe.model.meta import get_table_columns +from frappe.core.doctype.server_script.server_script_utils import get_server_script_map class DatabaseQuery(object): def __init__(self, doctype, user=None): @@ -39,7 +40,10 @@ class DatabaseQuery(object): ignore_ifnull=False, save_user_settings=False, save_user_settings_fields=False, update=None, add_total_row=None, user_settings=None, reference_doctype=None, return_query=False, strict=True, pluck=None, ignore_ddl=False): - if not ignore_permissions and not frappe.has_permission(self.doctype, "read", user=user): + if not ignore_permissions and \ + not frappe.has_permission(self.doctype, "select", user=user) and \ + not frappe.has_permission(self.doctype, "read", user=user): + frappe.flags.error_message = _('Insufficient Permission for {0}').format(frappe.bold(self.doctype)) raise frappe.PermissionError(self.doctype) @@ -314,7 +318,10 @@ class DatabaseQuery(object): def append_table(self, table_name): self.tables.append(table_name) doctype = table_name[4:-1] - if (not self.flags.ignore_permissions) and (not frappe.has_permission(doctype)): + ptype = 'select' if frappe.only_has_select_perm(doctype) else 'read' + + if (not self.flags.ignore_permissions) and\ + (not frappe.has_permission(doctype, ptype=ptype)): frappe.flags.error_message = _('Insufficient Permission for {0}').format(frappe.bold(doctype)) raise frappe.PermissionError(doctype) @@ -575,7 +582,7 @@ class DatabaseQuery(object): self.shared = frappe.share.get_shared(self.doctype, self.user) if (not meta.istable and - not role_permissions.get("read") and + not (role_permissions.get("select") or role_permissions.get("read")) and not self.flags.ignore_permissions and not has_any_user_permission_for_doctype(self.doctype, self.user, self.reference_doctype)): only_if_shared = True @@ -683,15 +690,23 @@ class DatabaseQuery(object): self.match_filters.append(match_filters) def get_permission_query_conditions(self): + conditions = [] condition_methods = frappe.get_hooks("permission_query_conditions", {}).get(self.doctype, []) if condition_methods: - conditions = [] for method in condition_methods: c = frappe.call(frappe.get_attr(method), self.user) if c: conditions.append(c) - return " and ".join(conditions) if conditions else None + permision_script_name = get_server_script_map().get("permission_query").get(self.doctype) + if permision_script_name: + script = frappe.get_doc("Server Script", permision_script_name) + condition = script.get_permission_query_conditions(self.user) + if condition: + conditions.append(condition) + + return " and ".join(conditions) if conditions else "" + def run_custom_query(self, query): if '%(key)s' in query: diff --git a/frappe/model/delete_doc.py b/frappe/model/delete_doc.py index 862abe375c..15de673e4b 100644 --- a/frappe/model/delete_doc.py +++ b/frappe/model/delete_doc.py @@ -76,7 +76,12 @@ def delete_doc(doctype=None, name=None, force=0, ignore_doctypes=None, for_reloa delete_from_table(doctype, name, ignore_doctypes, None) - if not (for_reload or frappe.flags.in_migrate or frappe.flags.in_install or frappe.flags.in_uninstall or frappe.flags.in_test): + if frappe.conf.developer_mode and not doc.custom and not ( + for_reload + or frappe.flags.in_migrate + or frappe.flags.in_install + or frappe.flags.in_uninstall + ): try: delete_controllers(name, doc.module) except (FileNotFoundError, OSError, KeyError): diff --git a/frappe/model/meta.py b/frappe/model/meta.py index 8c17a5b19b..c740d495c1 100644 --- a/frappe/model/meta.py +++ b/frappe/model/meta.py @@ -209,7 +209,8 @@ class Meta(Document): 'owner': _('Created By'), 'modified_by': _('Modified By'), 'creation': _('Created On'), - 'modified': _('Last Modified On') + 'modified': _('Last Modified On'), + '_assign': _('Assigned To') }.get(fieldname) or _('No Label') return label diff --git a/frappe/model/rename_doc.py b/frappe/model/rename_doc.py index 7a2129e76e..2baf0c562c 100644 --- a/frappe/model/rename_doc.py +++ b/frappe/model/rename_doc.py @@ -1,14 +1,15 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals, print_function +from __future__ import print_function, unicode_literals + import frappe from frappe import _, bold -from frappe.utils import cint -from frappe.model.naming import validate_name from frappe.model.dynamic_links import get_dynamic_link_map -from frappe.utils.password import rename_password +from frappe.model.naming import validate_name from frappe.model.utils.user_settings import sync_user_settings, update_user_settings_data +from frappe.utils import cint +from frappe.utils.password import rename_password @frappe.whitelist() @@ -42,16 +43,13 @@ def rename_doc(doctype, old, new, force=False, merge=False, ignore_permissions=F force = cint(force) merge = cint(merge) - meta = frappe.get_meta(doctype) # call before_rename old_doc = frappe.get_doc(doctype, old) out = old_doc.run_method("before_rename", old, new, merge) or {} new = (out.get("new") or new) if isinstance(out, dict) else (out or new) - - if doctype != "DocType": - new = validate_rename(doctype, new, meta, merge, force, ignore_permissions) + new = validate_rename(doctype, new, meta, merge, force, ignore_permissions) if not merge: rename_parent_and_child(doctype, old, new, meta) @@ -249,14 +247,21 @@ def update_link_field_values(link_fields, old, new, doctype): # or no longer exists pass else: - # because the table hasn't been renamed yet! - parent = field['parent'] if field['parent']!=new else old + parent = field['parent'] + docfield = field["fieldname"] + + # Handles the case where one of the link fields belongs to + # the DocType being renamed. + # Here this field could have the current DocType as its value too. + + # In this case while updating link field value, the field's parent + # or the current DocType table name hasn't been renamed yet, + # so consider it's old name. + if parent == new and doctype == "DocType": + parent = old + + frappe.db.set_value(parent, {docfield: old}, docfield, new) - frappe.db.sql(""" - update `tab{table_name}` set `{fieldname}`=%s - where `{fieldname}`=%s""".format( - table_name=parent, - fieldname=field['fieldname']), (new, old)) # update cached link_fields as per new if doctype=='DocType' and field['parent'] == old: field['parent'] = new @@ -306,8 +311,7 @@ def get_link_fields(doctype): def update_options_for_fieldtype(fieldtype, old, new): if frappe.conf.developer_mode: - for name in frappe.db.sql_list("""select parent from - tabDocField where options=%s""", old): + for name in frappe.get_all("DocField", filters={"options": old}, pluck="parent"): doctype = frappe.get_doc("DocType", name) save = False for f in doctype.fields: @@ -413,20 +417,21 @@ def update_parenttype_values(old, new): child_doctypes += custom_child_doctypes fields = [d['fieldname'] for d in child_doctypes] - property_setter_child_doctypes = frappe.db.sql("""\ - select value as options from `tabProperty Setter` - where doc_type=%s and property='options' and - field_name in ("%s")""" % ('%s', '", "'.join(fields)), - (new,)) + property_setter_child_doctypes = frappe.get_all( + "Property Setter", + filters={ + "doc_type": new, + "property": "options", + "field_name": ("in", fields) + }, + pluck="value" + ) + child_doctypes = list(d['options'] for d in child_doctypes) child_doctypes += property_setter_child_doctypes - child_doctypes = (d['options'] for d in child_doctypes) for doctype in child_doctypes: - frappe.db.sql("""\ - update `tab%s` set parenttype=%s - where parenttype=%s""" % (doctype, '%s', '%s'), - (new, old)) + frappe.db.sql(f"update `tab{doctype}` set parenttype=%s where parenttype=%s", (new, old)) def rename_dynamic_links(doctype, old, new): for df in get_dynamic_link_map().get(doctype, []): @@ -482,60 +487,30 @@ def bulk_rename(doctype, rows=None, via_console = False): return rename_log def update_linked_doctypes(doctype, docname, linked_to, value, ignore_doctypes=None): - """ - linked_doctype_info_list = list formed by get_fetch_fields() function - docname = Master DocType's name in which modification are made - value = Value for the field thats set in other DocType's by fetching from Master DocType - """ - linked_doctype_info_list = get_fetch_fields(doctype, linked_to, ignore_doctypes) + from frappe.model.utils.rename_doc import update_linked_doctypes + show_deprecation_warning("update_linked_doctypes") + + return update_linked_doctypes( + doctype=doctype, + docname=docname, + linked_to=linked_to, + value=value, + ignore_doctypes=ignore_doctypes, + ) - for d in linked_doctype_info_list: - frappe.db.sql(""" - update - `tab{doctype}` - set - {linked_to_fieldname} = "{value}" - where - {master_fieldname} = {docname} - and {linked_to_fieldname} != "{value}" - """.format( - doctype = d['doctype'], - linked_to_fieldname = d['linked_to_fieldname'], - value = value, - master_fieldname = d['master_fieldname'], - docname = frappe.db.escape(docname) - )) def get_fetch_fields(doctype, linked_to, ignore_doctypes=None): - """ - doctype = Master DocType in which the changes are being made - linked_to = DocType name of the field thats being updated in Master + from frappe.model.utils.rename_doc import get_fetch_fields + show_deprecation_warning("get_fetch_fields") - This function fetches list of all DocType where both doctype and linked_to is found - as link fields. - Forms a list of dict in the form - - [{doctype: , master_fieldname: , linked_to_fieldname: ] - where - doctype = DocType where changes need to be made - master_fieldname = Fieldname where options = doctype - linked_to_fieldname = Fieldname where options = linked_to - """ + return get_fetch_fields( + doctype=doctype, linked_to=linked_to, ignore_doctypes=ignore_doctypes + ) - master_list = get_link_fields(doctype) - linked_to_list = get_link_fields(linked_to) - out = [] - - from itertools import product - product_list = product(master_list, linked_to_list) - - for d in product_list: - linked_doctype_info = frappe._dict() - if d[0]['parent'] == d[1]['parent'] \ - and (not ignore_doctypes or d[0]['parent'] not in ignore_doctypes) \ - and not d[1]['issingle']: - linked_doctype_info['doctype'] = d[0]['parent'] - linked_doctype_info['master_fieldname'] = d[0]['fieldname'] - linked_doctype_info['linked_to_fieldname'] = d[1]['fieldname'] - out.append(linked_doctype_info) - - return out +def show_deprecation_warning(funct): + from click import secho + message = ( + f"Function frappe.model.rename_doc.{funct} has been deprecated and " + "moved to the frappe.model.utils.rename_doc" + ) + secho(message, fg="yellow") diff --git a/frappe/model/utils/rename_doc.py b/frappe/model/utils/rename_doc.py new file mode 100644 index 0000000000..bf71d36a42 --- /dev/null +++ b/frappe/model/utils/rename_doc.py @@ -0,0 +1,58 @@ +from itertools import product + +import frappe +from frappe.model.rename_doc import get_link_fields + + +def update_linked_doctypes(doctype, docname, linked_to, value, ignore_doctypes=None): + """ + linked_doctype_info_list = list formed by get_fetch_fields() function + docname = Master DocType's name in which modification are made + value = Value for the field thats set in other DocType's by fetching from Master DocType + """ + linked_doctype_info_list = get_fetch_fields(doctype, linked_to, ignore_doctypes) + + for d in linked_doctype_info_list: + frappe.db.set_value( + d.doctype, + { + d.master_fieldname : docname, + d.linked_to_fieldname : ("!=", value), + }, + d.linked_to_fieldname, + value, + ) + + +def get_fetch_fields(doctype, linked_to, ignore_doctypes=None): + """ + doctype = Master DocType in which the changes are being made + linked_to = DocType name of the field thats being updated in Master + This function fetches list of all DocType where both doctype and linked_to is found + as link fields. + Forms a list of dict in the form - + [{doctype: , master_fieldname: , linked_to_fieldname: ] + where + doctype = DocType where changes need to be made + master_fieldname = Fieldname where options = doctype + linked_to_fieldname = Fieldname where options = linked_to + """ + + out = [] + master_list = get_link_fields(doctype) + linked_to_list = get_link_fields(linked_to) + product_list = product(master_list, linked_to_list) + + for d in product_list: + linked_doctype_info = frappe._dict() + if ( + d[0]["parent"] == d[1]["parent"] + and (not ignore_doctypes or d[0]["parent"] not in ignore_doctypes) + and not d[1]["issingle"] + ): + linked_doctype_info.doctype = d[0]["parent"] + linked_doctype_info.master_fieldname = d[0]["fieldname"] + linked_doctype_info.linked_to_fieldname = d[1]["fieldname"] + out.append(linked_doctype_info) + + return out diff --git a/frappe/model/workflow.py b/frappe/model/workflow.py index 7239b202bd..3e8125f9b1 100644 --- a/frappe/model/workflow.py +++ b/frappe/model/workflow.py @@ -29,6 +29,8 @@ def get_transitions(doc, workflow = None, raise_exception=False): if doc.is_new(): return [] + doc.load_from_db() + frappe.has_permission(doc, 'read', throw=True) roles = frappe.get_roles() @@ -51,14 +53,17 @@ def get_transitions(doc, workflow = None, raise_exception=False): return transitions def get_workflow_safe_globals(): - # access to frappe.db.get_value and frappe.db.get_list + # access to frappe.db.get_value, frappe.db.get_list, and date time utils. return dict( frappe=frappe._dict( - db=frappe._dict( - get_value=frappe.db.get_value, - get_list=frappe.db.get_list + db=frappe._dict(get_value=frappe.db.get_value, get_list=frappe.db.get_list), + session=frappe.session, + utils=frappe._dict( + now_datetime=frappe.utils.now_datetime, + add_to_date=frappe.utils.add_to_date, + get_datetime=frappe.utils.get_datetime, + now=frappe.utils.now, ), - session=frappe.session ) ) @@ -115,9 +120,8 @@ def apply_workflow(doc, action): return doc @frappe.whitelist() -def can_cancel_document(doc): - doc = frappe.get_doc(frappe.parse_json(doc)) - workflow = get_workflow(doc.doctype) +def can_cancel_document(doctype): + workflow = get_workflow(doctype) for state_doc in workflow.states: if state_doc.doc_status == '2': for transition in workflow.transitions: diff --git a/frappe/oauth.py b/frappe/oauth.py index bf225ac118..09af5ad809 100644 --- a/frappe/oauth.py +++ b/frappe/oauth.py @@ -148,7 +148,7 @@ class OAuthWebRequestValidator(RequestValidator): print("Failed body authentication: Application %s does not exist".format(cid=request.client_id)) cookie_dict = get_cookie_dict_from_headers(request) - user_id = unquote(cookie_dict['user_id']) if 'user_id' in cookie_dict else "Guest" + user_id = unquote(cookie_dict.get('user_id').value) if 'user_id' in cookie_dict else "Guest" return frappe.session.user == user_id def authenticate_client_id(self, client_id, request, *args, **kwargs): diff --git a/frappe/patches.txt b/frappe/patches.txt index 0daf29e001..1a086303ba 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -21,6 +21,7 @@ execute:frappe.reload_doc('email', 'doctype', 'document_follow') execute:frappe.reload_doc('core', 'doctype', 'communication_link') #2019-10-02 execute:frappe.reload_doc('core', 'doctype', 'has_role') execute:frappe.reload_doc('core', 'doctype', 'communication') #2019-10-02 +execute:frappe.reload_doc('core', 'doctype', 'server_script') frappe.patches.v11_0.replicate_old_user_permissions frappe.patches.v11_0.reload_and_rename_view_log #2019-01-03 frappe.patches.v7_1.rename_scheduler_log_to_error_log @@ -316,3 +317,4 @@ frappe.patches.v13_0.web_template_set_module #2020-10-05 frappe.patches.v13_0.remove_custom_link execute:frappe.delete_doc("DocType", "Footer Item") frappe.patches.v13_0.replace_field_target_with_open_in_new_tab +frappe.patches.v13_0.delete_package_publish_tool diff --git a/frappe/patches/v13_0/delete_package_publish_tool.py b/frappe/patches/v13_0/delete_package_publish_tool.py new file mode 100644 index 0000000000..25024f58dd --- /dev/null +++ b/frappe/patches/v13_0/delete_package_publish_tool.py @@ -0,0 +1,11 @@ +# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors +# MIT License. See license.txt + +from __future__ import unicode_literals +import frappe + + +def execute(): + frappe.delete_doc("DocType", "Package Publish Tool", ignore_missing=True) + frappe.delete_doc("DocType", "Package Document Type", ignore_missing=True) + frappe.delete_doc("DocType", "Package Publish Target", ignore_missing=True) diff --git a/frappe/permissions.py b/frappe/permissions.py index 0d766aec8d..a45fbdcd06 100644 --- a/frappe/permissions.py +++ b/frappe/permissions.py @@ -7,7 +7,7 @@ import frappe, copy, json from frappe import _, msgprint from frappe.utils import cint import frappe.share -rights = ("read", "write", "create", "delete", "submit", "cancel", "amend", +rights = ("select", "read", "write", "create", "delete", "submit", "cancel", "amend", "print", "email", "report", "import", "export", "set_user_permissions", "share") # TODO: @@ -73,6 +73,7 @@ def has_permission(doctype, ptype="read", doc=None, verbose=False, user=None, ra role_permissions = get_role_permissions(meta, user=user) perm = role_permissions.get(ptype) + if not perm: push_perm_check_log(_('User {0} does not have doctype access via role permission for document {1}').format(frappe.bold(user), frappe.bold(doctype))) @@ -192,9 +193,9 @@ def get_role_permissions(doctype_meta, user=None): and ptype != 'create'): perms['if_owner'][ptype] = 1 # has no access if not owner - # only provide read access so that user is able to at-least access list + # only provide select or read access so that user is able to at-least access list # (and the documents will be filtered based on owner sin further checks) - perms[ptype] = 1 if ptype == 'read' else 0 + perms[ptype] = 1 if ptype in ['select', 'read'] else 0 frappe.local.role_permissions[cache_key] = perms diff --git a/frappe/printing/doctype/print_format/print_format.js b/frappe/printing/doctype/print_format/print_format.js index e6599b2496..786f8f97ab 100644 --- a/frappe/printing/doctype/print_format/print_format.js +++ b/frappe/printing/doctype/print_format/print_format.js @@ -19,6 +19,7 @@ frappe.ui.form.on("Print Format", { } frm.trigger('render_buttons'); frm.toggle_display('standard', frappe.boot.developer_mode); + frm.trigger('hide_absolute_value_field'); }, render_buttons: function (frm) { frm.page.clear_inner_toolbar(); @@ -58,5 +59,20 @@ frappe.ui.form.on("Print Format", { frm.set_value('show_section_headings', value); frm.set_value('line_breaks', value); frm.trigger('render_buttons'); + }, + doc_type: function (frm) { + frm.trigger('hide_absolute_value_field'); + }, + hide_absolute_value_field: function (frm) { + // TODO: make it work with frm.doc.doc_type + // Problem: frm isn't updated in some random cases + const doctype = locals[frm.doc.doctype][frm.doc.name].doc_type; + if (doctype) { + frappe.model.with_doctype(doctype, () => { + const meta = frappe.get_meta(doctype); + const has_int_float_currency_field = meta.fields.filter(df => in_list(['Int', 'Float', 'Currency'], df.fieldtype)); + frm.toggle_display('absolute_value', has_int_float_currency_field.length); + }); + } } -}) +}); diff --git a/frappe/printing/doctype/print_format/print_format.json b/frappe/printing/doctype/print_format/print_format.json index 63448ccc39..3a47fb554f 100644 --- a/frappe/printing/doctype/print_format/print_format.json +++ b/frappe/printing/doctype/print_format/print_format.json @@ -22,6 +22,7 @@ "align_labels_right", "show_section_headings", "line_breaks", + "absolute_value", "column_break_11", "font", "css_section", @@ -196,13 +197,21 @@ "fieldtype": "Check", "hidden": 1, "label": "Print Format Builder" + }, + { + "default": "0", + "depends_on": "doc_type", + "description": "If checked, negative numeric values of Currency, Quantity or Count would be shown as positive", + "fieldname": "absolute_value", + "fieldtype": "Check", + "label": "Show Absolute Values" } ], "icon": "fa fa-print", "idx": 1, "index_web_pages_for_search": 1, "links": [], - "modified": "2020-10-27 18:27:58.307070", + "modified": "2020-12-14 11:38:49.132061", "modified_by": "Administrator", "module": "Printing", "name": "Print Format", diff --git a/frappe/public/build.json b/frappe/public/build.json index a3622499d5..d744da98d1 100755 --- a/frappe/public/build.json +++ b/frappe/public/build.json @@ -307,6 +307,7 @@ "public/js/frappe/views/calendar/calendar.js", "public/js/frappe/views/dashboard/dashboard_view.js", "public/js/frappe/views/image/image_view.js", + "public/js/frappe/views/map/map_view.js", "public/js/frappe/views/kanban/kanban_view.js", "public/js/frappe/views/inbox/inbox_view.js", "public/js/frappe/views/file/file_view.js", diff --git a/frappe/public/css/list.css b/frappe/public/css/list.css index 5ae77c73ca..88ad147d33 100644 --- a/frappe/public/css/list.css +++ b/frappe/public/css/list.css @@ -401,6 +401,13 @@ input.list-row-checkbox { .pswp__more-item img { max-height: 100%; } +.map-view-container { + display: flex; + flex-wrap: wrap; + width: 100%; + height: calc(100vh - 284px); + z-index: 0; +} .list-paging-area .gantt-view-mode { margin-left: 15px; margin-right: 15px; diff --git a/frappe/public/js/frappe/data_import/import_preview.js b/frappe/public/js/frappe/data_import/import_preview.js index 6c17cb4351..477cfb0786 100644 --- a/frappe/public/js/frappe/data_import/import_preview.js +++ b/frappe/public/js/frappe/data_import/import_preview.js @@ -101,6 +101,7 @@ frappe.data_import.ImportPreview = class ImportPreview { .replace('%H', 'HH') .replace('%M', 'mm') .replace('%S', 'ss') + .replace('%b', 'Mon') : null; let column_title = ` diff --git a/frappe/public/js/frappe/desk.js b/frappe/public/js/frappe/desk.js index c8ed29fb76..5fa7a9dbcb 100644 --- a/frappe/public/js/frappe/desk.js +++ b/frappe/public/js/frappe/desk.js @@ -148,7 +148,6 @@ frappe.Application = Class.extend({ user: frappe.session.user }, callback: function(r) { - console.log(r); if(r.message.show_alert){ frappe.show_alert({ indicator: 'red', diff --git a/frappe/public/js/frappe/file_uploader/index.js b/frappe/public/js/frappe/file_uploader/index.js index 62a7bff822..646f60715a 100644 --- a/frappe/public/js/frappe/file_uploader/index.js +++ b/frappe/public/js/frappe/file_uploader/index.js @@ -15,7 +15,11 @@ export default class FileUploader { allow_multiple, as_dataurl, disable_file_browser, + frm } = {}) { + + frm && frm.attachments.max_reached(true); + if (!wrapper) { this.make_dialog(); } else { diff --git a/frappe/public/js/frappe/form/controls/base_control.js b/frappe/public/js/frappe/form/controls/base_control.js index 319aa067cc..d7f873bee0 100644 --- a/frappe/public/js/frappe/form/controls/base_control.js +++ b/frappe/public/js/frappe/form/controls/base_control.js @@ -40,23 +40,31 @@ frappe.ui.form.Control = Class.extend({ return this.df.get_status(this); } - if((!this.doctype && !this.docname) || this.df.parenttype === 'Web Form') { + if ((!this.doctype && !this.docname) || this.df.parenttype === 'Web Form' || this.df.is_web_form) { // like in case of a dialog box if (cint(this.df.hidden)) { // eslint-disable-next-line - if(explain) console.log("By Hidden: None"); + if (explain) console.log("By Hidden: None"); // eslint-disable-line no-console return "None"; } else if (cint(this.df.hidden_due_to_dependency)) { // eslint-disable-next-line - if(explain) console.log("By Hidden Dependency: None"); + if(explain) console.log("By Hidden Dependency: None"); // eslint-disable-line no-console return "None"; } else if (cint(this.df.read_only)) { // eslint-disable-next-line - if(explain) console.log("By Read Only: Read"); + if (explain) console.log("By Read Only: Read"); // eslint-disable-line no-console return "Read"; + } else if ((this.grid && + this.grid.display_status == 'Read') || + (this.layout && + this.layout.grid && + this.layout.grid.display_status == 'Read')) { + // parent grid is read + if (explain) console.log("By Parent Grid Read-only: Read"); // eslint-disable-line no-console + return "Read"; } return "Write"; @@ -65,13 +73,22 @@ frappe.ui.form.Control = Class.extend({ var status = frappe.perm.get_field_display_status(this.df, frappe.model.get_doc(this.doctype, this.docname), this.perm || (this.frm && this.frm.perm), explain); + // Match parent grid controls read only status + if (status === 'Write' && (this.grid || (this.layout && this.layout.grid))) { + var grid = this.grid || this.layout.grid; + if (grid.display_status == 'Read') { + status = 'Read'; + if (explain) console.log("By Parent Grid Read-only: Read"); // eslint-disable-line no-console + } + } + // hide if no value if (this.doctype && status==="Read" && !this.only_input && is_null(frappe.model.get_value(this.doctype, this.docname, this.df.fieldname)) && !in_list(["HTML", "Image", "Button"], this.df.fieldtype)) { // eslint-disable-next-line - if(explain) console.log("By Hide Read-only, null fields: None"); + if (explain) console.log("By Hide Read-only, null fields: None"); // eslint-disable-line no-console status = "None"; } diff --git a/frappe/public/js/frappe/form/controls/comment.js b/frappe/public/js/frappe/form/controls/comment.js index a64df56bca..d00c915065 100644 --- a/frappe/public/js/frappe/form/controls/comment.js +++ b/frappe/public/js/frappe/form/controls/comment.js @@ -60,7 +60,7 @@ frappe.ui.form.ControlComment = frappe.ui.form.ControlTextEditor.extend({ update_state() { const value = this.get_value(); - if (strip_html(value).trim() != "") { + if (strip_html(value).trim() != "" || value.includes('img')) { this.button.removeClass('btn-default').addClass('btn-primary'); } else { this.button.addClass('btn-default').removeClass('btn-primary'); diff --git a/frappe/public/js/frappe/form/controls/data.js b/frappe/public/js/frappe/form/controls/data.js index 4db2553bd1..401de2ed5d 100644 --- a/frappe/public/js/frappe/form/controls/data.js +++ b/frappe/public/js/frappe/form/controls/data.js @@ -22,27 +22,9 @@ frappe.ui.form.ControlData = frappe.ui.form.ControlInput.extend({ this.has_input = true; this.bind_change_event(); this.setup_autoname_check(); - if (this.df.options == 'Phone') { - this.setup_phone(); - } // somehow this event does not bubble up to document // after v7, if you can debug, remove this }, - setup_phone() { - if (frappe.phone_call.handler) { - this.$wrapper.find('.control-input') - .append(` - - - - - `) - .find('.phone-btn') - .click(() => { - frappe.phone_call.handler(this.get_value(), this.frm); - }); - } - }, setup_autoname_check: function() { if (!this.df.parent) return; this.meta = frappe.get_meta(this.df.parent); diff --git a/frappe/public/js/frappe/form/controls/geolocation.js b/frappe/public/js/frappe/form/controls/geolocation.js index 9dfad09299..9e4d1d82ec 100644 --- a/frappe/public/js/frappe/form/controls/geolocation.js +++ b/frappe/public/js/frappe/form/controls/geolocation.js @@ -1,3 +1,5 @@ +frappe.provide('frappe.utils.utils'); + frappe.ui.form.ControlGeolocation = frappe.ui.form.ControlData.extend({ horizontal: false, @@ -90,11 +92,11 @@ frappe.ui.form.ControlGeolocation = frappe.ui.form.ControlData.extend({ }); L.Icon.Default.imagePath = '/assets/frappe/images/leaflet/'; - this.map = L.map(this.map_id).setView([19.0800, 72.8961], 13); + this.map = L.map(this.map_id).setView(frappe.utils.map_defaults.center, + frappe.utils.map_defaults.zoom); - L.tileLayer('http://{s}.tile.osm.org/{z}/{x}/{y}.png', { - attribution: '© OpenStreetMap contributors' - }).addTo(this.map); + L.tileLayer(frappe.utils.map_defaults.tiles, + frappe.utils.map_defaults.options).addTo(this.map); }, bind_leaflet_locate_control() { diff --git a/frappe/public/js/frappe/form/controls/link.js b/frappe/public/js/frappe/form/controls/link.js index 56f9430238..111ee7d8f6 100644 --- a/frappe/public/js/frappe/form/controls/link.js +++ b/frappe/public/js/frappe/form/controls/link.js @@ -215,6 +215,7 @@ frappe.ui.form.ControlLink = frappe.ui.form.ControlData.extend({ } me.$input.cache[doctype][term] = r.results; me.awesomplete.list = me.$input.cache[doctype][term]; + me.toggle_href(doctype); } }); }, 500)); @@ -296,6 +297,15 @@ frappe.ui.form.ControlLink = frappe.ui.form.ControlData.extend({ // returns [{value: 'Manufacturer 1', 'description': 'mobile part 1, mobile part 2'}] }, + toggle_href(doctype) { + if (frappe.model.can_select(doctype) && !frappe.model.can_read(doctype)) { + // remove href from link field as user has only select perm + this.$input_area.find(".link-btn").addClass('hide'); + } else { + this.$input_area.find(".link-btn").removeClass('hide'); + } + }, + get_filter_description(filters) { let doctype = this.get_options(); let filter_array = []; diff --git a/frappe/public/js/frappe/form/controls/rating.js b/frappe/public/js/frappe/form/controls/rating.js index 34e890d45c..191db35538 100644 --- a/frappe/public/js/frappe/form/controls/rating.js +++ b/frappe/public/js/frappe/form/controls/rating.js @@ -47,7 +47,7 @@ frappe.ui.form.ControlRating = frappe.ui.form.ControlInt.extend({ }); }, get_value() { - return cint(this.value); + return cint(this.value, null); }, set_formatted_input(value) { let el = $(this.input_area).find('i'); diff --git a/frappe/public/js/frappe/form/controls/table.js b/frappe/public/js/frappe/form/controls/table.js index 14fad1c010..a87a4ad2a6 100644 --- a/frappe/public/js/frappe/form/controls/table.js +++ b/frappe/public/js/frappe/form/controls/table.js @@ -9,7 +9,8 @@ frappe.ui.form.ControlTable = frappe.ui.form.Control.extend({ frm: this.frm, df: this.df, perm: this.perm || (this.frm && this.frm.perm) || this.df.perm, - parent: this.wrapper + parent: this.wrapper, + control: this }); if(this.frm) { this.frm.grids[this.frm.grids.length] = this; diff --git a/frappe/public/js/frappe/form/footer/timeline.js b/frappe/public/js/frappe/form/footer/timeline.js index 84f34d4757..159ab8a61b 100644 --- a/frappe/public/js/frappe/form/footer/timeline.js +++ b/frappe/public/js/frappe/form/footer/timeline.js @@ -30,7 +30,7 @@ frappe.ui.form.Timeline = class Timeline { render_input: true, only_input: true, on_submit: (val) => { - if(strip_html(val).trim() != "") { + if (strip_html(val).trim() != "" || val.includes('img')) { this.insert_comment(val, this.comment_area.button); } } @@ -547,10 +547,7 @@ frappe.ui.form.Timeline = class Timeline { log.color = 'dark'; log.sender = log.owner; log.comment_type = 'Milestone'; - log.content = __('{0} changed {1} to {2}', [ - frappe.user.full_name(log.owner).bold(), - frappe.meta.get_label(this.frm.doctype, log.track_field), - log.value.bold()]); + log.content = __('{0} changed {1} to {2}', [ frappe.user.full_name(log.owner).bold(), frappe.meta.get_label(this.frm.doctype, log.track_field), log.value.bold()]); return log; }); return milestones; @@ -613,11 +610,7 @@ frappe.ui.form.Timeline = class Timeline { const field_display_status = frappe.perm.get_field_display_status(df, null, me.frm.perm); if (field_display_status === 'Read' || field_display_status === 'Write') { - parts.push(__('{0} from {1} to {2}', [ - __(df.label), - me.format_content_for_timeline(p[1]), - me.format_content_for_timeline(p[2]) - ])); + parts.push(__('{0} from {1} to {2}', [ __(df.label), me.format_content_for_timeline(p[1]), me.format_content_for_timeline(p[2])])); } } } @@ -648,13 +641,7 @@ frappe.ui.form.Timeline = class Timeline { null, me.frm.perm); if (field_display_status === 'Read' || field_display_status === 'Write') { - parts.push(__('{0} from {1} to {2} in row #{3}', [ - frappe.meta.get_label(me.frm.fields_dict[row[0]].grid.doctype, - p[0]), - me.format_content_for_timeline(p[1]), - me.format_content_for_timeline(p[2]), - row[1] - ])); + parts.push(__('{0} from {1} to {2} in row #{3}', [ frappe.meta.get_label( me.frm.fields_dict[row[0]].grid.doctype, p[0]), me.format_content_for_timeline(p[1]), me.format_content_for_timeline(p[2]), row[1] ])); } } return parts.length < 3; @@ -691,8 +678,7 @@ frappe.ui.form.Timeline = class Timeline { return p; }); if (parts.length) { - out.push(me.get_version_comment(version, __("{0} rows for {1}", - [__(key), parts.join(', ')]))); + out.push(me.get_version_comment(version, __("{0} rows for {1}", [__(key), parts.join(', ')]))); } } }); diff --git a/frappe/public/js/frappe/form/form.js b/frappe/public/js/frappe/form/form.js index fc348704fa..1e23969afa 100644 --- a/frappe/public/js/frappe/form/form.js +++ b/frappe/public/js/frappe/form/form.js @@ -232,14 +232,10 @@ frappe.ui.form.Form = class FrappeForm { throw "attach error"; } - if(me.attachments.max_reached()) { - frappe.msgprint(__("Maximum Attachment Limit for this record reached.")); - throw "attach error"; - } - new frappe.ui.FileUploader({ doctype: me.doctype, docname: me.docname, + frm: me, files: dataTransfer.files, folder: 'Home/Attachments', on_success(file_doc) { diff --git a/frappe/public/js/frappe/form/formatters.js b/frappe/public/js/frappe/form/formatters.js index 3f422d0a9b..2b8956653b 100644 --- a/frappe/public/js/frappe/form/formatters.js +++ b/frappe/public/js/frappe/form/formatters.js @@ -50,7 +50,15 @@ frappe.form.formatters = { return frappe.form.formatters._right(value==null ? "" : cint(value), options) }, Percent: function(value, docfield, options) { - return frappe.form.formatters._right(flt(value, 2) + "%", options) + const precision = ( + docfield.precision + || cint( + frappe.boot.sysdefaults + && frappe.boot.sysdefaults.float_precision + ) + || 2 + ); + return frappe.form.formatters._right(flt(value, precision) + "%", options); }, Rating: function(value) { return `