vassili/seitime-frappe
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 4

Merge branch 'develop' into new-ui-for-api-key

Browse source
This commit is contained in:
Abdeali Chharchhoda 2025-07-07 12:04:34 +05:30
commit 363c20b2ea
223 changed files with 146027 additions and 110304 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
.git-blame-ignore-revs
View file

@ -58,3 +58,6 @@ e9bbe03354079cfcef65a77b0c33f57b047a7c93
# ruff update
84ef6ec677c8657c3243ac456a1ef794bfb34a50
# replace `frappe.flags.in_test` with `frappe.in_test`
653c80b8483cc41aef25cd7d66b9b6bb188bf5f8

1
.github/helper/documentation.py vendored
View file

@ -11,6 +11,7 @@ WEBSITE_REPOS = [
DOCUMENTATION_DOMAINS = [
"docs.erpnext.com",
"frappeframework.com",
"docs.frappe.io",
]

7
README.md
View file

@ -103,13 +103,10 @@ To setup the repository locally follow the steps mentioned below:
2. In a separate terminal window, run the following commands:
```
# Create a new site
bench new-site frappe.dev
# Map your site to localhost
bench --site frappe.dev add-to-hosts
bench new-site frappe.localhost
```
3. Open the URL `http://frappe.dev:8000/app` in your browser, you should see the app running
3. Open the URL `http://frappe.localhost:8000/app` in your browser, you should see the app running
## Learning and community

7
esbuild/esbuild.js
View file

@ -472,6 +472,11 @@ async function write_assets_json(metafile) {
}
async function update_assets_json_in_cache() {
// Redis won't be present during docker image build
if (process.env.FRAPPE_DOCKER_BUILD) {
return;
}
// update assets_json cache in redis, so that it can be read directly by python
let client = get_redis_subscriber("redis_cache");
// handle error event to avoid printing stack traces
@ -523,7 +528,7 @@ function run_build_command_for_apps(apps) {
log(
`\nInstalling dependencies for ${chalk.bold(app)} (because node_modules not found)`
);
execSync("yarn install", { encoding: "utf8", stdio: "inherit" });
execSync("yarn install --frozen-lockfile", { encoding: "utf8", stdio: "inherit" });
}
log("\nRunning build command for", chalk.bold(app));

43
frappe/__init__.py
View file

@ -34,6 +34,7 @@ from typing import (
)
import click
import orjson
from werkzeug.datastructures import Headers
import frappe
@ -83,6 +84,9 @@ cache: Optional["RedisWrapper"] = None
client_cache: Optional["ClientCache"] = None
STANDARD_USERS = ("Guest", "Administrator")
# this global may be subsequently changed by frappe.tests.utils.toggle_test_mode()
in_test = False
_dev_server = int(sbool(os.environ.get("DEV_SERVER", False)))
if _dev_server:
@ -219,7 +223,7 @@ def init(site: str, sites_path: str = ".", new_site: bool = False, force: bool =
"in_install_db": False,
"in_install_app": False,
"in_import": False,
"in_test": False,
"in_test": in_test,
"mute_messages": False,
"ignore_links": False,
"mute_emails": False,
@ -263,7 +267,7 @@ def init(site: str, sites_path: str = ".", new_site: bool = False, force: bool =
local.form_dict = _dict()
local.preload_assets = {"style": [], "script": [], "icons": []}
local.session = _dict()
local.dev_server = _dev_server
local.dev_server = _dev_server # only for backwards compatibility
local.qb = get_query_builder(local.conf.db_type)
if not cache or not client_cache:
setup_redis_cache_connection()
@ -616,6 +620,16 @@ xss_safe_methods: set[Callable] = set()
allowed_http_methods_for_whitelisted_func: dict[Callable, list[str]] = {}
def _in_request_or_test():
"""
Internal
Used by whitelist to determine whether type hints should be validated or not
"""
return getattr(local, "request", None) or in_test
def whitelist(allow_guest=False, xss_safe=False, methods=None):
"""
Decorator for whitelisting a function and making it accessible via HTTP.
@ -639,17 +653,8 @@ def whitelist(allow_guest=False, xss_safe=False, methods=None):
global whitelisted, guest_methods, xss_safe_methods, allowed_http_methods_for_whitelisted_func
# validate argument types only if request is present
in_request_or_test = lambda: getattr(local, "request", None) or local.flags.in_test # noqa: E731
# get function from the unbound / bound method
# this is needed because functions can be compared, but not methods
method = None
if hasattr(fn, "__func__"):
method = validate_argument_types(fn, apply_condition=in_request_or_test)
fn = method.__func__
else:
fn = validate_argument_types(fn, apply_condition=in_request_or_test)
# validate argument types if request is present or in test context
fn = validate_argument_types(fn, apply_condition=_in_request_or_test)
whitelisted.add(fn)
allowed_http_methods_for_whitelisted_func[fn] = methods
@ -660,7 +665,7 @@ def whitelist(allow_guest=False, xss_safe=False, methods=None):
if xss_safe:
xss_safe_methods.add(fn)
return method or fn
return fn
return innerfn
@ -740,7 +745,7 @@ def only_for(roles: list[str] | tuple[str] | str, message=False):
:param roles: Permitted role(s)
"""
if local.flags.in_test or local.session.user == "Administrator":
if local.session.user == "Administrator":
return
if isinstance(roles, str):
@ -767,7 +772,7 @@ def get_domain_data(module):
else:
return _dict()
except ImportError:
if local.flags.in_test:
if in_test:
return _dict()
else:
raise
@ -1262,7 +1267,7 @@ def get_installed_apps(*, _ensure_on_bench: bool = False) -> list[str]:
if not db:
connect()
installed = json.loads(db.get_global("installed_apps") or "[]")
installed = orjson.loads(db.get_global("installed_apps") or "[]")
if _ensure_on_bench:
all_apps = cache.get_value("all_apps", get_all_apps)
@ -1595,10 +1600,10 @@ def copy_doc(doc: "Document", ignore_no_copy: bool = True) -> "Document":
fields_to_clear = ["name", "owner", "creation", "modified", "modified_by"]
if not local.flags.in_test:
if not in_test:
fields_to_clear.append("docstatus")
if isinstance(doc, BaseDocument) or hasattr(doc, "as_dict"):
if isinstance(doc, BaseDocument):
d = doc.as_dict()
elif isinstance(doc, MappingProxyType): # global test record
d = dict(doc)

2
frappe/api/v1.py
View file

@ -39,8 +39,6 @@ def handle_rpc_call(method: str):
def create_doc(doctype: str):
data = get_request_form_data()
data.pop("doctype", None)
if (name := data.get("name")) and isinstance(name, str):
frappe.flags.api_name_set = True
return frappe.new_doc(doctype, **data).insert()

116
frappe/api/v2.py
View file

@ -15,7 +15,7 @@ from werkzeug.routing import Rule
import frappe
import frappe.client
from frappe import _, get_newargs, is_whitelisted
from frappe import _, cint, cstr, get_newargs, is_whitelisted
from frappe.core.doctype.server_script.server_script_utils import get_server_script_map
from frappe.handler import is_valid_http_method, run_server_script, upload_file
@ -65,17 +65,99 @@ def read_doc(doctype: str, name: str):
doc = frappe.get_doc(doctype, name)
doc.check_permission("read")
doc.apply_fieldlevel_read_permissions()
return doc
_doc = doc.as_dict()
for key in _doc:
df = doc.meta.get_field(key)
if df and df.fieldtype == "Link" and isinstance(_doc.get(key), int):
_doc[key] = cstr(_doc.get(key))
return _doc
def document_list(doctype: str):
if frappe.form_dict.get("fields"):
frappe.form_dict["fields"] = json.loads(frappe.form_dict["fields"])
def document_list(doctype: str) -> list[dict[str, Any]]:
"""
GET /api/v2/document/<doctype>?fields=[...],filters={...},...
# set limit of records for frappe.get_list
frappe.form_dict.limit_page_length = frappe.form_dict.limit or 20
# evaluate frappe.get_list
return frappe.call(frappe.client.get_list, doctype, **frappe.form_dict)
REST API endpoint for fetching doctype records
Args:
doctype: DocType name
Query Parameters (accessible via frappe.form_dict):
fields: JSON string of field names to fetch
filters: JSON string of filters to apply
order_by: Order by field
start: Starting offset for pagination (default: 0)
limit: Maximum number of records to fetch (default: 20)
group_by: Group by field
as_dict: Return results as dictionary (default: True)
Response:
frappe.response["data"]: List of document records as dicts
frappe.response["has_next_page"]: Indicates if more pages are available
Controller Customization:
Doctype controllers can customize queries by implementing a static get_list(query) method
that receives a QueryBuilder object and returns a modified QueryBuilder.
Example:
class Project(Document):
@staticmethod
def get_list(query):
Project = frappe.qb.DocType("Project")
if user_has_role("Project Owner"):
query = query.where(Project.owner == frappe.session.user)
else:
query = query.where(Project.is_private == 0)
return query
"""
from frappe.model.base_document import get_controller
args = frappe.form_dict
fields: list | None = frappe.parse_json(args.get("fields", None))
filters: dict | None = frappe.parse_json(args.get("filters", None))
order_by: str | None = args.get("order_by", None)
start: int = cint(args.get("start", 0))
limit: int = cint(args.get("limit", 20))
group_by: str | None = args.get("group_by", None)
debug: bool = args.get("debug", False)
as_dict: bool = args.get("as_dict", True)
query = frappe.qb.get_query(
table=doctype,
fields=fields,
filters=filters,
order_by=order_by,
offset=start,
limit=limit + 1, # Fetch one extra to check if there's a next page
group_by=group_by,
ignore_permissions=False,
)
# Check if the doctype controller has a static get_list method
controller = get_controller(doctype)
if hasattr(controller, "get_list"):
try:
return_value = controller.get_list(query)
if return_value is not None:
# Validate that the returned value has a run method (is a QueryBuilder-like object)
if not hasattr(return_value, "run"):
frappe.throw(
_(
"Custom get_list method for {0} must return a QueryBuilder object or None, got {1}"
).format(doctype, type(return_value).__name__)
)
query = return_value
except Exception as e:
frappe.throw(_("Error in {0}.get_list: {1}").format(doctype, str(e)))
data = query.run(as_dict=as_dict, debug=debug)
frappe.response["has_next_page"] = len(data) > limit
return data[:limit]
def count(doctype: str) -> int:
@ -89,9 +171,13 @@ def count(doctype: str) -> int:
def create_doc(doctype: str):
data = frappe.form_dict
data.pop("doctype", None)
if (name := data.get("name")) and isinstance(name, str):
frappe.flags.api_name_set = True
return frappe.new_doc(doctype, **data).insert()
doc = frappe.new_doc(doctype, **data)
if (name := data.get("name")) and isinstance(name, str | int):
doc.flags.name_set = True
return doc.insert().as_dict()
def copy_doc(doctype: str, name: str, ignore_no_copy: bool = True):
@ -118,7 +204,7 @@ def update_doc(doctype: str, name: str):
if doc.get("parenttype"):
frappe.get_doc(doc.parenttype, doc.parent).save()
return doc
return doc.as_dict()
def delete_doc(doctype: str, name: str):
@ -144,7 +230,9 @@ def execute_doc_method(doctype: str, name: str, method: str | None = None):
doc.is_whitelisted(method)
doc.check_permission(PERMISSION_MAP[frappe.request.method])
return doc.run_method(method, **frappe.form_dict)
result = doc.run_method(method, **frappe.form_dict)
frappe.response.docs.append(doc.as_dict())
return result
def run_doc_method(method: str, document: dict[str, Any] | str, kwargs=None):

73
frappe/app.py
View file

@ -5,6 +5,7 @@ import functools
import logging
import os
import orjson
from werkzeug.exceptions import HTTPException, NotFound
from werkzeug.middleware.profiler import ProfilerMiddleware
from werkzeug.middleware.proxy_fix import ProxyFix
@ -21,6 +22,7 @@ import frappe.recorder
import frappe.utils.response
from frappe import _
from frappe.auth import SAFE_HTTP_METHODS, UNSAFE_HTTP_METHODS, HTTPRequest, check_request_ip, validate_auth
from frappe.integrations.oauth2 import get_resource_url, handle_wellknown, is_oauth_metadata_enabled
from frappe.middlewares import StaticDataMiddleware
from frappe.permissions import handle_does_not_exist_error
from frappe.utils import CallbackManager, cint, get_site_name
@ -65,6 +67,11 @@ import frappe.website.website_generator # web page doctypes
# end: module pre-loading
# better werkzeug default
# this is necessary because frappe desk sends most requests as form data
# and some of them can exceed werkzeug's default limit of 500kb
Request.max_form_memory_size = None
def after_response_wrapper(app):
"""Wrap a WSGI application to call after_response hooks after we have responded.
@ -92,8 +99,6 @@ def application(request: Request):
response = None
try:
rollback = True
init_request(request)
validate_auth()
@ -121,29 +126,28 @@ def application(request: Request):
elif request.path.startswith("/private/files/"):
response = frappe.utils.response.download_private_file(request.path)
elif request.path.startswith("/.well-known/") and request.method == "GET":
response = handle_wellknown(request.path)
elif request.method in ("GET", "HEAD", "POST"):
response = get_response()
else:
raise NotFound
except HTTPException as e:
return e
except Exception as e:
response = handle_exception(e)
response = e.get_response(request.environ) if isinstance(e, HTTPException) else handle_exception(e)
if db := getattr(frappe.local, "db", None):
db.rollback(chain=True)
else:
rollback = sync_database(rollback)
sync_database()
finally:
# Important note:
# this function *must* always return a response, hence any exception thrown outside of
# try..catch block like this finally block needs to be handled appropriately.
if rollback and request.method in UNSAFE_HTTP_METHODS and frappe.db:
frappe.db.rollback()
try:
run_after_request_hooks(request, response)
except Exception:
@ -177,14 +181,13 @@ def init_request(request):
# site does not exist
raise NotFound
frappe.connect(set_admin_as_user=False)
if frappe.local.conf.maintenance_mode:
frappe.connect()
if frappe.local.conf.allow_reads_during_maintenance:
setup_read_only_mode()
else:
raise frappe.SessionStopped("Session Stopped")
else:
frappe.connect(set_admin_as_user=False)
if request.path.startswith("/api/method/upload_file"):
from frappe.core.api.file import get_max_file_size
@ -256,6 +259,9 @@ def process_response(response: Response):
if hasattr(frappe.local, "conf"):
set_cors_headers(response)
if response.status_code in (401, 403) and is_oauth_metadata_enabled("resource"):
set_authenticate_headers(response)
# Update custom headers added during request processing
response.headers.update(frappe.local.response_headers)
@ -269,10 +275,12 @@ def process_response(response: Response):
def set_cors_headers(response):
allowed_origins = frappe.conf.allow_cors
if hasattr(frappe.local, "allow_cors"):
allowed_origins = frappe.local.allow_cors
if not (
(allowed_origins := frappe.conf.allow_cors)
and (request := frappe.local.request)
and (origin := request.headers.get("Origin"))
allowed_origins and (request := frappe.local.request) and (origin := request.headers.get("Origin"))
):
return
@ -303,12 +311,17 @@ def set_cors_headers(response):
response.headers.update(cors_headers)
def make_form_dict(request: Request):
import json
def set_authenticate_headers(response: Response):
headers = {
"WWW-Authenticate": f'Bearer resource_metadata="{get_resource_url()}/.well-known/oauth-protected-resource"'
}
response.headers.update(headers)
def make_form_dict(request: Request):
request_data = request.get_data(as_text=True)
if request_data and request.is_json:
args = json.loads(request_data)
args = orjson.loads(request_data)
else:
args = {}
args.update(request.args or {})
@ -397,21 +410,21 @@ def handle_exception(e):
return response
def sync_database(rollback: bool) -> bool:
def sync_database():
db = getattr(frappe.local, "db", None)
if not db:
# db isn't initialized, can't commit or rollback
return
# if HTTP method would change server state, commit if necessary
if frappe.db and (frappe.local.flags.commit or frappe.local.request.method in UNSAFE_HTTP_METHODS):
frappe.db.commit()
rollback = False
elif frappe.db:
frappe.db.rollback()
rollback = False
if frappe.local.request.method in UNSAFE_HTTP_METHODS or frappe.local.flags.commit:
db.commit(chain=True)
else:
db.rollback(chain=True)
# update session
if session := getattr(frappe.local, "session_obj", None):
if session.update():
rollback = False
return rollback
frappe.request.after_response.add(session.update)
# Always initialize sentry SDK if the DSN is sent

6
frappe/automation/doctype/auto_repeat/auto_repeat.py
View file

@ -86,7 +86,7 @@ class AutoRepeat(Document):
validate_template(self.message or "")
def before_insert(self):
if not frappe.flags.in_test:
if not frappe.in_test:
start_date = getdate(self.start_date)
today_date = getdate(today())
if start_date <= today_date:
@ -112,7 +112,7 @@ class AutoRepeat(Document):
frappe.db.set_value(self.reference_doctype, self.reference_document, "auto_repeat", "")
def validate_reference_doctype(self):
if frappe.flags.in_test or frappe.flags.in_patch:
if frappe.in_test or frappe.flags.in_patch:
return
if not frappe.get_meta(self.reference_doctype).allow_auto_repeat:
frappe.throw(
@ -229,7 +229,7 @@ class AutoRepeat(Document):
self.disable_auto_repeat()
if self.reference_document and not frappe.flags.in_test:
if self.reference_document and not frappe.in_test:
self.notify_error_to_user(error_log)
def make_new_document(self):

121
frappe/automation/workspace/tools/tools.json
View file

@ -1,6 +1,6 @@
{
"charts": [],
"content": "[{\"id\":\"sR-UFcO7II\",\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"Import Data\",\"col\":3}},{\"id\":\"IkcVmgWb3z\",\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"ToDo\",\"col\":3}},{\"id\":\"6wir-jZFRE\",\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"File\",\"col\":3}},{\"id\":\"45a1jzQkTm\",\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"Assignment Rule\",\"col\":3}},{\"id\":\"EgtURZsoiF\",\"type\":\"spacer\",\"data\":{\"col\":12}},{\"id\":\"0yceBIfhHM\",\"type\":\"card\",\"data\":{\"card_name\":\"Data\",\"col\":4}},{\"id\":\"42WbBA9rpj\",\"type\":\"card\",\"data\":{\"card_name\":\"Tools\",\"col\":4}},{\"id\":\"wE9n7TIrAc\",\"type\":\"card\",\"data\":{\"card_name\":\"Alerts and Notifications\",\"col\":4}},{\"id\":\"7_U7_xCOos\",\"type\":\"card\",\"data\":{\"card_name\":\"Email\",\"col\":4}},{\"id\":\"3imoh2oqsJ\",\"type\":\"card\",\"data\":{\"card_name\":\"Printing\",\"col\":4}},{\"id\":\"SlYKJZj5r3\",\"type\":\"card\",\"data\":{\"card_name\":\"Automation\",\"col\":4}},{\"id\":\"O7jrc2YQTN\",\"type\":\"card\",\"data\":{\"card_name\":\"Newsletter\",\"col\":4}}]",
"content": "[{\"id\":\"sR-UFcO7II\",\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"Import Data\",\"col\":3}},{\"id\":\"IkcVmgWb3z\",\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"ToDo\",\"col\":3}},{\"id\":\"6wir-jZFRE\",\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"File\",\"col\":3}},{\"id\":\"45a1jzQkTm\",\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"Assignment Rule\",\"col\":3}},{\"id\":\"EgtURZsoiF\",\"type\":\"spacer\",\"data\":{\"col\":12}},{\"id\":\"0yceBIfhHM\",\"type\":\"card\",\"data\":{\"card_name\":\"Data\",\"col\":4}},{\"id\":\"42WbBA9rpj\",\"type\":\"card\",\"data\":{\"card_name\":\"Tools\",\"col\":4}},{\"id\":\"wE9n7TIrAc\",\"type\":\"card\",\"data\":{\"card_name\":\"Alerts and Notifications\",\"col\":4}},{\"id\":\"7_U7_xCOos\",\"type\":\"card\",\"data\":{\"card_name\":\"Email\",\"col\":4}},{\"id\":\"3imoh2oqsJ\",\"type\":\"card\",\"data\":{\"card_name\":\"Printing\",\"col\":4}},{\"id\":\"SlYKJZj5r3\",\"type\":\"card\",\"data\":{\"card_name\":\"Automation\",\"col\":4}}]",
"creation": "2020-03-02 14:53:24.980279",
"custom_blocks": [],
"docstatus": 0,
@ -105,74 +105,6 @@
"onboard": 0,
"type": "Link"
},
{
"hidden": 0,
"is_query_report": 0,
"label": "Email",
"link_count": 3,
"link_type": "DocType",
"onboard": 0,
"type": "Card Break"
},
{
"hidden": 0,
"is_query_report": 0,
"label": "Email Account",
"link_count": 0,
"link_to": "Email Account",
"link_type": "DocType",
"onboard": 0,
"type": "Link"
},
{
"hidden": 0,
"is_query_report": 0,
"label": "Email Domain",
"link_count": 0,
"link_to": "Email Domain",
"link_type": "DocType",
"onboard": 0,
"type": "Link"
},
{
"hidden": 0,
"is_query_report": 0,
"label": "Email Template",
"link_count": 0,
"link_to": "Email Template",
"link_type": "DocType",
"onboard": 0,
"type": "Link"
},
{
"hidden": 0,
"is_query_report": 0,
"label": "Newsletter",
"link_count": 2,
"link_type": "DocType",
"onboard": 0,
"type": "Card Break"
},
{
"hidden": 0,
"is_query_report": 0,
"label": "Newsletter",
"link_count": 0,
"link_to": "Newsletter",
"link_type": "DocType",
"onboard": 0,
"type": "Link"
},
{
"hidden": 0,
"is_query_report": 0,
"label": "Email Group",
"link_count": 0,
"link_to": "Email Group",
"link_type": "DocType",
"onboard": 0,
"type": "Link"
},
{
"hidden": 0,
"is_query_report": 0,
@ -320,9 +252,58 @@
"link_type": "DocType",
"onboard": 0,
"type": "Link"
},
{
"hidden": 0,
"is_query_report": 0,
"label": "Email",
"link_count": 4,
"link_type": "DocType",
"onboard": 0,
"type": "Card Break"
},
{
"hidden": 0,
"is_query_report": 0,
"label": "Email Account",
"link_count": 0,
"link_to": "Email Account",
"link_type": "DocType",
"onboard": 0,
"type": "Link"
},
{
"hidden": 0,
"is_query_report": 0,
"label": "Email Domain",
"link_count": 0,
"link_to": "Email Domain",
"link_type": "DocType",
"onboard": 0,
"type": "Link"
},
{
"hidden": 0,
"is_query_report": 0,
"label": "Email Template",
"link_count": 0,
"link_to": "Email Template",
"link_type": "DocType",
"onboard": 0,
"type": "Link"
},
{
"hidden": 0,
"is_query_report": 0,
"label": "Email Group",
"link_count": 0,
"link_to": "Email Group",
"link_type": "DocType",
"onboard": 0,
"type": "Link"
}
],
"modified": "2024-09-03 21:54:05.403066",
"modified": "2025-06-27 11:39:44.392114",
"modified_by": "Administrator",
"module": "Automation",
"name": "Tools",

3
frappe/boot.py
View file

@ -145,7 +145,8 @@ def remove_apps_with_incomplete_dependencies(bootinfo):
remove_apps.add(app)
for app in remove_apps:
bootinfo.setup_wizard_not_required_apps.remove(app)
if app in bootinfo.setup_wizard_not_required_apps:
bootinfo.setup_wizard_not_required_apps.remove(app)
def get_letter_heads():

2
frappe/build.py
View file

@ -52,7 +52,7 @@ def build_missing_files():
folder = os.path.join(sites_path, "assets", "frappe", "dist", type)
current_asset_files.extend(os.listdir(folder))
development = frappe.local.conf.developer_mode or frappe.local.dev_server
development = frappe.local.conf.developer_mode or frappe._dev_server
build_mode = "development" if development else "production"
assets_json = frappe.read_file("assets/assets.json")

5
frappe/commands/testing.py
View file

@ -183,6 +183,7 @@ def main(
def run_tests_in_light_mode(test_params):
from frappe.testing.loader import FrappeTestLoader
from frappe.testing.result import FrappeTestResult
from frappe.tests.utils import toggle_test_mode
# init environment
frappe.init(test_params.site)
@ -196,6 +197,7 @@ def run_tests_in_light_mode(test_params):
frappe.utils.scheduler.disable_scheduler()
frappe.clear_cache()
toggle_test_mode(True)
suite = FrappeTestLoader().discover_tests(test_params)
result = unittest.TextTestRunner(failfast=test_params.failfast, resultclass=FrappeTestResult).run(suite)
if not result.wasSuccessful():
@ -370,6 +372,7 @@ def run_tests(
)
@click.option("--use-orchestrator", is_flag=True, help="Use orchestrator to run parallel tests")
@click.option("--dry-run", is_flag=True, default=False, help="Dont actually run tests")
@click.option("--lightmode", is_flag=True, default=False, help="Skips all before test setup")
@pass_context
def run_parallel_tests(
context: CliCtxObj,
@ -379,6 +382,7 @@ def run_parallel_tests(
with_coverage=False,
use_orchestrator=False,
dry_run=False,
lightmode=False,
):
from traceback_with_variables import activate_by_import
@ -399,6 +403,7 @@ def run_parallel_tests(
build_number=build_number,
total_builds=total_builds,
dry_run=dry_run,
lightmode=lightmode,
)
mode = "Orchestrator" if use_orchestrator else "Parallel"
banner = f"""

2
frappe/commands/utils.py
View file

@ -77,7 +77,7 @@ def build(
skip_frappe = False
# don't minify in developer_mode for faster builds
development = frappe.local.conf.developer_mode or frappe.local.dev_server
development = frappe.local.conf.developer_mode or frappe._dev_server
mode = "development" if development else "production"
if production:
mode = "production"

50
frappe/core/doctype/access_log/access_log.py
View file

@ -38,6 +38,12 @@ class AccessLog(Document):
@frappe.whitelist()
@frappe.write_only()
@retry(
stop=stop_after_attempt(3),
retry=retry_if_exception_type(frappe.DuplicateEntryError),
reraise=True,
)
def make_access_log(
doctype=None,
document=None,
@ -48,41 +54,10 @@ def make_access_log(
page=None,
columns=None,
):
_make_access_log(
doctype,
document,
method,
file_type,
report_name,
filters,
page,
columns,
)
@frappe.write_only()
@retry(
stop=stop_after_attempt(3),
retry=retry_if_exception_type(frappe.DuplicateEntryError),
reraise=True,
)
def _make_access_log(
doctype=None,
document=None,
method=None,
file_type=None,
report_name=None,
filters=None,
page=None,
columns=None,
):
user = frappe.session.user
in_request = frappe.request and frappe.request.method == "GET"
access_log = frappe.get_doc(
{
"doctype": "Access Log",
"user": user,
"user": frappe.session.user,
"export_from": doctype,
"reference_document": document,
"file_type": file_type,
@ -94,14 +69,11 @@ def _make_access_log(
}
)
if frappe.flags.read_only:
if not frappe.in_test:
access_log.deferred_insert()
return
else:
access_log.db_insert()
# `frappe.db.commit` added because insert doesnt `commit` when called in GET requests like `printview`
# dont commit in test mode. It must be tempting to put this block along with the in_request in the
# whitelisted method...yeah, don't do it. That part would be executed possibly on a read only DB conn
if not frappe.flags.in_test or in_request:
frappe.db.commit()
# only for backward compatibility
_make_access_log = make_access_log

6
frappe/core/doctype/communication/communication.py
View file

@ -490,8 +490,8 @@ def get_permission_query_conditions_for_communication(user):
return """`tabCommunication`.communication_medium!='Email'"""
email_accounts = ['"{}"'.format(account.get("email_account")) for account in accounts]
return """`tabCommunication`.email_account in ({email_accounts}) or `tabCommunication`.recipients LIKE '%{user}%' or `tabCommunication`.sender LIKE '%{user}%' or `tabCommunication`.cc LIKE '%{user}%' or `tabCommunication`.bcc LIKE '%{user}%'""".format(
email_accounts=",".join(email_accounts), user=user
return """`tabCommunication`.email_account in ({email_accounts})""".format(
email_accounts=",".join(email_accounts)
)
@ -579,7 +579,7 @@ def parse_email(email_strings):
if not document_parts or len(document_parts) != 2:
continue
doctype = unquote_plus(document_parts[0])
doctype = frappe.unscrub(unquote_plus(document_parts[0]))
docname = unquote_plus(document_parts[1])
yield doctype, docname

2
frappe/core/doctype/data_import/data_import.py
View file

@ -101,7 +101,7 @@ class DataImport(Document):
def start_import(self):
from frappe.utils.scheduler import is_scheduler_inactive
run_now = frappe.flags.in_test or frappe.conf.developer_mode
run_now = frappe.in_test or frappe.conf.developer_mode
if is_scheduler_inactive() and not run_now:
frappe.throw(_("Scheduler is inactive. Cannot import data."), title=_("Scheduler Inactive"))

7
frappe/core/doctype/data_import/importer.py
View file

@ -1035,8 +1035,11 @@ class Column:
if self.df.fieldtype == "Link":
# find all values that dont exist
values = list({cstr(v) for v in self.column_values if v})
exists = [cstr(d.name) for d in frappe.get_all(self.df.options, filters={"name": ("in", values)})]
values = list({cstr(v).lower() for v in self.column_values if v})
exists = [
cstr(d.name).lower()
for d in frappe.get_all(self.df.options, filters={"name": ("in", values)})
]
not_exists = list(set(values) - set(exists))
if not_exists:
missing_values = ", ".join(not_exists)

3
frappe/core/doctype/doctype/doctype.json
View file

@ -288,6 +288,7 @@
},
{
"default": "0",
"depends_on": "eval:!doc.issingle",
"fieldname": "allow_import",
"fieldtype": "Check",
"label": "Allow Import (via Data Import Tool)"
@ -784,7 +785,7 @@
"link_fieldname": "document_type"
}
],
"modified": "2025-05-21 21:58:59.947374",
"modified": "2025-06-24 07:46:34.380662",
"modified_by": "Administrator",
"module": "Core",
"name": "DocType",

4
frappe/core/doctype/doctype/doctype.py
View file

@ -322,7 +322,7 @@ class DocType(Document):
def check_developer_mode(self):
"""Throw exception if not developer mode or via patch"""
if frappe.flags.in_patch or frappe.flags.in_test:
if frappe.flags.in_patch or frappe.in_test:
return
if not frappe.conf.get("developer_mode") and not self.custom:
@ -594,7 +594,7 @@ class DocType(Document):
global_search_fields_after_update.append("name")
if set(global_search_fields_before_update) != set(global_search_fields_after_update):
now = (not frappe.request) or frappe.flags.in_test or frappe.flags.in_install
now = (not frappe.request) or frappe.in_test or frappe.flags.in_install
frappe.enqueue("frappe.utils.global_search.rebuild_for_doctype", now=now, doctype=self.name)
def set_base_class_for_controller(self):

8
frappe/core/doctype/installed_application/installed_application.json
View file

@ -17,8 +17,7 @@
"fieldtype": "Data",
"in_list_view": 1,
"label": "Git Branch",
"read_only": 1,
"reqd": 1
"read_only": 1
},
{
"columns": 2,
@ -35,8 +34,7 @@
"fieldtype": "Data",
"in_list_view": 1,
"label": "Application Version",
"read_only": 1,
"reqd": 1
"read_only": 1
},
{
"columns": 2,
@ -58,7 +56,7 @@
"grid_page_length": 50,
"istable": 1,
"links": [],
"modified": "2025-05-22 12:26:49.523690",
"modified": "2025-05-27 12:26:49.523690",
"modified_by": "Administrator",
"module": "Core",
"name": "Installed Application",

8
frappe/core/doctype/installed_applications/installed_applications.json
View file

@ -18,7 +18,7 @@
],
"issingle": 1,
"links": [],
"modified": "2024-03-23 16:03:27.992755",
"modified": "2025-06-30 21:26:13.462828",
"modified_by": "Administrator",
"module": "Core",
"name": "Installed Applications",
@ -36,8 +36,8 @@
}
],
"quick_entry": 1,
"row_format": "Dynamic",
"sort_field": "creation",
"sort_order": "DESC",
"states": [],
"track_changes": 1
}
"states": []
}

12
frappe/core/doctype/installed_applications/installed_applications.py
View file

@ -26,6 +26,8 @@ class InstalledApplications(Document):
# end: auto-generated types
def update_versions(self):
self.reload_doc_if_required()
app_wise_setup_details = self.get_app_wise_setup_details()
self.delete_key("installed_applications")
@ -52,6 +54,8 @@ class InstalledApplications(Document):
)
self.save()
frappe.clear_cache(doctype="System Settings")
frappe.db.set_single_value("System Settings", "setup_complete", frappe.is_setup_complete())
def get_app_wise_setup_details(self):
"""Get app wise setup details from the Installed Application doctype"""
@ -64,6 +68,14 @@ class InstalledApplications(Document):
)
)
def reload_doc_if_required(self):
if frappe.db.has_column("Installed Application", "is_setup_complete"):
return
frappe.reload_doc("core", "doctype", "installed_application")
frappe.reload_doc("core", "doctype", "installed_applications")
frappe.reload_doc("integrations", "doctype", "webhook")
@frappe.whitelist()
def update_installed_apps_order(new_order: list[str] | str):

2
frappe/core/doctype/prepared_report/prepared_report.py
View file

@ -242,7 +242,7 @@ def create_json_gz_file(data, dt, dn, report_name):
frappe.scrub(report_name), frappe.utils.data.format_datetime(frappe.utils.now(), "Y-m-d-H-M")
)
encoded_content = frappe.safe_encode(frappe.as_json(data, indent=None, separators=(",", ":")))
compressed_content = gzip.compress(encoded_content)
compressed_content = gzip.compress(encoded_content, compresslevel=5)
# Call save() file function to upload and attach the file
_file = frappe.get_doc(

26
frappe/core/doctype/report/test_report.py
View file

@ -175,12 +175,11 @@ class TestReport(IntegrationTestCase):
)
def test_report_permissions(self):
frappe.set_user("test@example.com")
frappe.db.delete("Has Role", {"parent": frappe.session.user, "role": "Test Has Role"})
frappe.db.commit()
# create role "Test Has Role"
if not frappe.db.exists("Role", "Test Has Role"):
frappe.get_doc({"doctype": "Role", "role_name": "Test Has Role"}).insert(ignore_permissions=True)
# create report "Test Report"
if not frappe.db.exists("Report", "Test Report"):
report = frappe.get_doc(
{
@ -195,13 +194,16 @@ class TestReport(IntegrationTestCase):
else:
report = frappe.get_doc("Report", "Test Report")
self.assertNotEqual(report.is_permitted(), True)
frappe.set_user("Administrator")
with self.set_user("test@example.com"):
# remove role "Test Has Role" from user if found
frappe.db.delete("Has Role", {"parent": frappe.session.user, "role": "Test Has Role"})
self.assertNotEqual(report.is_permitted(), True)
def test_report_custom_permissions(self):
frappe.set_user("test@example.com")
# delete custom role if exists
frappe.db.delete("Custom Role", {"report": "Test Custom Role Report"})
frappe.db.commit() # nosemgrep
# create report if not exists
if not frappe.db.exists("Report", "Test Custom Role Report"):
report = frappe.get_doc(
{
@ -216,8 +218,11 @@ class TestReport(IntegrationTestCase):
else:
report = frappe.get_doc("Report", "Test Custom Role Report")
self.assertEqual(report.is_permitted(), True)
# check report is permitted without custom role created
with self.set_user("test@example.com"):
self.assertEqual(report.is_permitted(), True)
# create custom role for report
frappe.get_doc(
{
"doctype": "Custom Role",
@ -227,8 +232,9 @@ class TestReport(IntegrationTestCase):
}
).insert(ignore_permissions=True)
self.assertNotEqual(report.is_permitted(), True)
frappe.set_user("Administrator")
# check report is not permitted with custom role created
with self.set_user("test@example.com"):
self.assertNotEqual(report.is_permitted(), True)
# test for the `_format` method if report data doesn't have sort_by parameter
def test_format_method(self):

2
frappe/core/doctype/role_profile/role_profile.py
View file

@ -29,7 +29,7 @@ class RoleProfile(Document):
self.clear_cache()
self.queue_action(
"update_all_users",
now=frappe.flags.in_test or frappe.flags.in_install,
now=frappe.in_test or frappe.flags.in_install,
enqueue_after_commit=True,
queue="long",
)

10
frappe/core/doctype/rq_job/rq_job.py
View file

@ -112,6 +112,16 @@ class RQJob(Document):
except InvalidJobOperation:
frappe.msgprint(_("Job is not running."), title=_("Invalid Operation"))
@check_permissions
def cancel(self):
if self.status == "queued":
self.job.cancel()
else:
frappe.msgprint(
_("Job is in {0} state and can't be cancelled").format(self.status),
title=_("Invalid Operation"),
)
@staticmethod
def get_count(filters=None) -> int:
return len(RQJob.get_matching_job_ids(filters))

6
frappe/core/doctype/rq_job/test_rq_job.py
View file

@ -26,6 +26,12 @@ def wait_for_completion(job: Job):
class TestRQJob(IntegrationTestCase):
BG_JOB = "frappe.core.doctype.rq_job.test_rq_job.test_func"
def setUp(self) -> None:
# Cleanup all pending jobs
for job in frappe.get_all("RQ Job", {"status": "queued"}):
frappe.get_doc("RQ Job", job.name).cancel()
return super().setUp()
def check_status(self, job: Job, status, wait=True):
if wait:
wait_for_completion(job)

7
frappe/core/doctype/scheduled_job_type/scheduled_job_type.py
View file

@ -4,6 +4,7 @@
import hashlib
import json
from datetime import datetime, timedelta
from functools import lru_cache
import click
from croniter import CroniterBadCronError, croniter
@ -14,6 +15,8 @@ from frappe.model.document import Document
from frappe.utils import get_datetime, now_datetime
from frappe.utils.background_jobs import enqueue, is_job_enqueued
parse_cron = lru_cache(croniter) # Cache parsed cron-expressions
class ScheduledJobType(Document):
# begin: auto-generated types
@ -132,10 +135,10 @@ class ScheduledJobType(Document):
# A dynamic fallback like current time might miss the scheduler interval and job will never start.
last_execution = get_datetime(self.last_execution or self.creation)
next_execution = croniter(self.cron_format, last_execution).get_next(datetime)
next_execution = parse_cron(self.cron_format).get_next(datetime, start_time=last_execution)
if self.frequency in ("Hourly Maintenance", "Daily Maintenance"):
next_execution += timedelta(minutes=maintenance_offset)
return croniter(self.cron_format, last_execution).get_next(datetime)
return parse_cron(self.cron_format).get_next(datetime, start_time=last_execution)
def execute(self):
if frappe.job:

17
frappe/core/doctype/user/test_user.py
View file

@ -25,6 +25,7 @@ from frappe.model.delete_doc import delete_doc
from frappe.tests import IntegrationTestCase
from frappe.tests.classes.context_managers import change_settings
from frappe.tests.test_api import FrappeAPITestCase
from frappe.tests.utils import toggle_test_mode
from frappe.utils import get_url
user_module = frappe.core.doctype.user.user
@ -212,13 +213,15 @@ class TestUser(IntegrationTestCase):
# test password strength while saving user with new password
user = frappe.get_doc("User", "test@example.com")
frappe.flags.in_test = False
user.new_password = "password"
self.assertRaises(frappe.exceptions.ValidationError, user.save)
user.reload()
user.new_password = "Eastern_43A1W"
user.save()
frappe.flags.in_test = True
toggle_test_mode(False)
try:
user.new_password = "password"
self.assertRaises(frappe.exceptions.ValidationError, user.save)
user.reload()
user.new_password = "Eastern_43A1W"
user.save()
finally:
toggle_test_mode(True)
def test_comment_mentions(self):
comment = """

4
frappe/core/doctype/user/user.py
View file

@ -173,7 +173,7 @@ class User(Document):
self.__new_password = self.new_password
self.new_password = ""
if not frappe.flags.in_test:
if not frappe.in_test:
self.password_strength_test()
if self.name not in STANDARD_USERS:
@ -269,7 +269,7 @@ class User(Document):
self.share_with_self()
clear_notifications(user=self.name)
frappe.clear_cache(user=self.name)
now = frappe.flags.in_test or frappe.flags.in_install
now = frappe.in_test or frappe.flags.in_install
self.send_password_notification(self.__new_password)
frappe.enqueue(
"frappe.core.doctype.user.user.create_contact",

13
frappe/core/doctype/user_permission/user_permission.py
View file

@ -105,7 +105,7 @@ def get_user_permissions(user=None):
out = {}
def add_doc_to_perm(perm, doc_name, is_default):
def add_doc_to_perm(perm, doc_name, is_default, hide_descendants):
# group rules for each type
# for example if allow is "Customer", then build all allowed customers
# in a list
@ -114,7 +114,12 @@ def get_user_permissions(user=None):
out[perm.allow].append(
frappe._dict(
{"doc": doc_name, "applicable_for": perm.get("applicable_for"), "is_default": is_default}
{
"doc": doc_name,
"applicable_for": perm.get("applicable_for"),
"is_default": is_default,
"hide_descendants": hide_descendants,
}
)
)
@ -125,12 +130,12 @@ def get_user_permissions(user=None):
filters=dict(user=user),
):
meta = frappe.get_meta(perm.allow)
add_doc_to_perm(perm, perm.for_value, perm.is_default)
add_doc_to_perm(perm, perm.for_value, perm.is_default, perm.hide_descendants)
if meta.is_nested_set() and not perm.hide_descendants:
decendants = frappe.db.get_descendants(perm.allow, perm.for_value)
for doc in decendants:
add_doc_to_perm(perm, doc, False)
add_doc_to_perm(perm, doc, False, False)
out = frappe._dict(out)
frappe.cache.hset("user_permissions", user, out)

2
frappe/database/__init__.py
View file

@ -75,7 +75,7 @@ def get_db(socket=None, host=None, user=None, password=None, port=None, cur_db_n
import frappe.database.sqlite.database
return frappe.database.sqlite.database.SQLiteDatabase(cur_db_name=cur_db_name)
elif conf.use_mysqlclient:
elif conf.get("use_mysqlclient", 1):
import frappe.database.mariadb.mysqlclient
return frappe.database.mariadb.mysqlclient.MariaDBDatabase(

33
frappe/database/database.py
View file

@ -306,12 +306,11 @@ class Database:
):
raise
self.log_query(query, query_type, values, debug)
if debug:
time_end = time()
frappe.log(f"Execution time: {(time_end - time_start) * 1000:.3f} ms")
self.log_query(query, query_type, values, debug)
if auto_commit:
self.commit()
@ -1147,7 +1146,7 @@ class Database:
mode = "READ ONLY" if read_only else ""
self.sql(f"START TRANSACTION {mode}")
def commit(self):
def commit(self, *, chain=False):
"""Commit current transaction. Calls SQL `COMMIT`."""
if self._disable_transaction_control:
warnings.warn(message=TRANSACTION_DISABLED_MSG, stacklevel=2)
@ -1158,12 +1157,15 @@ class Database:
self.before_commit.run()
self.sql("commit")
self.begin() # explicitly start a new transaction
if chain:
self.sql("commit and chain")
else:
self.sql("commit")
self.begin()
self.after_commit.run()
def rollback(self, *, save_point=None):
def rollback(self, *, save_point=None, chain=False):
"""`ROLLBACK` current transaction. Optionally rollback to a known save_point."""
if save_point:
self.sql(f"rollback to savepoint {save_point}")
@ -1173,8 +1175,11 @@ class Database:
self.before_rollback.run()
self.sql("rollback")
self.begin()
if chain:
self.sql("rollback and chain")
else:
self.sql("rollback")
self.begin()
self.after_rollback.run()
else:
@ -1508,6 +1513,18 @@ class Database:
"""
raise NotImplementedError
def get_routines(self):
information_schema = frappe.qb.Schema("information_schema")
return (
frappe.qb.from_(information_schema.routines)
.select(information_schema.routines.routine_name)
.where(
(information_schema.routines.routine_type.isin(["FUNCTION", "PROCEDURE"]))
& (information_schema.routines.routine_schema.eq(frappe.conf.db_name))
)
.run(as_dict=1, pluck="routine_name")
)
@contextmanager
def savepoint(catch: type | tuple[type, ...] = Exception):

4
frappe/database/mariadb/database.py
View file

@ -165,11 +165,11 @@ class MariaDBDatabase(MariaDBConnectionUtil, MariaDBExceptionUtil, Database):
self.db_type = "mariadb"
self.type_map = {
"Currency": ("decimal", "21,9"),
"Int": ("int", None),
"Int": ("int", "11"),
"Long Int": ("bigint", "20"),
"Float": ("decimal", "21,9"),
"Percent": ("decimal", "21,9"),
"Check": ("tinyint", None),
"Check": ("tinyint", 4),
"Small Text": ("text", ""),
"Long Text": ("longtext", ""),
"Code": ("longtext", ""),

4
frappe/database/mariadb/mysqlclient.py
View file

@ -198,11 +198,11 @@ class MariaDBDatabase(MariaDBConnectionUtil, MariaDBExceptionUtil, Database):
self.db_type = "mariadb"
self.type_map = {
"Currency": ("decimal", "21,9"),
"Int": ("int", None),
"Int": ("int", "11"),
"Long Int": ("bigint", "20"),
"Float": ("decimal", "21,9"),
"Percent": ("decimal", "21,9"),
"Check": ("tinyint", None),
"Check": ("tinyint", "4"),
"Small Text": ("text", ""),
"Long Text": ("longtext", ""),
"Code": ("longtext", ""),

1435
frappe/database/query.py
View file

File diff suppressed because it is too large Load diff

59
frappe/database/schema.py
View file

@ -175,6 +175,9 @@ class DBTable:
pass
NOT_NULL_TYPES = ("Check", "Int", "Currency", "Float", "Percent")
class DbColumn:
def __init__(
self,
@ -216,13 +219,14 @@ class DbColumn:
default = None
unique = False
if self.fieldtype in NOT_NULL_TYPES:
null = False
if self.fieldtype in ("Check", "Int"):
default = cint(self.default)
null = False
elif self.fieldtype in ("Currency", "Float", "Percent"):
default = flt(self.default)
null = False
elif (
self.default
@ -271,7 +275,10 @@ class DbColumn:
return
# type
if current_def["type"] != column_type:
if current_def["type"] != column_type and not (
# XXX: MariaDB JSON is same as longtext and information schema still returns longtext
current_def["type"] == "longtext" and column_type == "json" and frappe.db.db_type == "mariadb"
):
self.table.change_type.append(self)
# unique
@ -289,7 +296,11 @@ class DbColumn:
self.table.set_default.append(self)
# nullability
if self.not_nullable is not None and (self.not_nullable != current_def.get("not_nullable")):
if (
self.not_nullable is not None
and (self.not_nullable != current_def.get("not_nullable"))
and self.fieldtype not in NOT_NULL_TYPES
):
self.table.change_nullability.append(self)
# index should be applied or dropped irrespective of type change
@ -310,24 +321,36 @@ class DbColumn:
else:
# Strip quotes from default value
# eg. database returns default value as "'System Manager'"
cur_default = cur_default.lstrip("'").rstrip("'")
cur_default = cur_default.lstrip("'").rstrip("'").replace("\\\\", "\\")
fieldtype = self.fieldtype
db_field_type = frappe.db.type_map.get(fieldtype)
if fieldtype in ["Int", "Check"]:
cur_default = cint(cur_default)
new_default = cint(new_default)
elif fieldtype in ["Currency", "Float", "Percent"]:
cur_default = flt(cur_default)
new_default = flt(new_default)
elif fieldtype == "Time":
return self.default_changed_for_time(cur_default, new_default)
elif db_field_type and db_field_type[0] in ("varchar", "longtext", "text"):
new_default = cstr(new_default)
if not current_def.get("not_nullable"):
cur_default = cstr(cur_default)
return cur_default != new_default
def default_changed_for_decimal(self, current_def):
cur_default = current_def["default"]
if cur_default == "NULL":
cur_default = None
try:
if current_def["default"] in ("", None) and self.default in ("", None):
# both none, empty
if cur_default in ("", None) and self.default in ("", None):
return False
elif current_def["default"] in ("", None):
elif flt(cur_default) == 0.0 and flt(self.default) == 0.0:
return False
elif cur_default in ("", None):
try:
# check if new default value is valid
float(self.default)
@ -341,10 +364,28 @@ class DbColumn:
else:
# NOTE float() raise ValueError when "" or None is passed
return float(current_def["default"]) != float(self.default)
return float(cur_default) != float(self.default)
except TypeError:
return True
def default_changed_for_time(self, cur_default: str, new_default: str):
from datetime import datetime
# Normalize time values to HH:MM:SS.ssssss format, from formats: HH:MM:SS.ssssss, HH:MM:SS, HH:MM
def normalize_time(val):
if not val:
return None
for fmt in ("%H:%M:%S.%f", "%H:%M:%S", "%H:%M"):
try:
return datetime.strptime(val, fmt).time().strftime("%H:%M:%S.%f")
except ValueError:
continue
return val
cur = normalize_time(cur_default)
new = normalize_time(new_default)
return cur != new
def validate_column_name(n):
if special_characters := SPECIAL_CHAR_PATTERN.findall(n):

2
frappe/deprecation_dumpster.py
View file

@ -306,7 +306,7 @@ def read_multi_pdf(output) -> bytes:
@deprecated("frappe.gzip_compress", "unknown", "v17", "Use py3 methods directly (this was compat for py2).")
def gzip_compress(data, compresslevel=9):
def gzip_compress(data, compresslevel=5):
"""Compress data in one shot and return the compressed string.
Optional argument is the compression level, in range of 0-9.
"""

13
frappe/desk/doctype/event/event.json
View file

@ -297,7 +297,7 @@
"icon": "fa fa-calendar",
"idx": 1,
"links": [],
"modified": "2025-04-10 13:08:32.540745",
"modified": "2025-06-17 15:31:01.945146",
"modified_by": "Administrator",
"module": "Desk",
"name": "Event",
@ -310,7 +310,7 @@
"print": 1,
"read": 1,
"report": 1,
"role": "All",
"role": "Desk User",
"share": 1,
"write": 1
},
@ -326,6 +326,15 @@
"role": "System Manager",
"share": 1,
"write": 1
},
{
"email": 1,
"export": 1,
"print": 1,
"read": 1,
"report": 1,
"role": "All",
"share": 1
}
],
"read_only": 1,

6
frappe/desk/doctype/notification_log/notification_log.py
View file

@ -94,8 +94,8 @@ def enqueue_create_notification(users: list[str] | str, doc: dict):
"frappe.desk.doctype.notification_log.notification_log.make_notification_logs",
doc=doc,
users=users,
now=frappe.flags.in_test,
enqueue_after_commit=not frappe.flags.in_test,
now=frappe.in_test,
enqueue_after_commit=not frappe.in_test,
)
@ -141,7 +141,7 @@ def send_notification_email(doc: NotificationLog):
template="new_notification",
args=args,
header=[header, "orange"],
now=frappe.flags.in_test,
now=frappe.in_test,
)

2
frappe/desk/doctype/system_console/system_console.py
View file

@ -49,7 +49,7 @@ class SystemConsole(Document):
frappe.db.commit()
@frappe.whitelist()
@frappe.whitelist(methods=["POST"])
def execute_code(doc):
console = frappe.get_doc(json.loads(doc))
console.run()

2
frappe/desk/doctype/system_health_report/system_health_report.py
View file

@ -57,7 +57,7 @@ def health_check(step: str):
try:
return func(*args, **kwargs)
except Exception as e:
if frappe.flags.in_test:
if frappe.in_test:
raise
frappe.log(frappe.get_traceback())
# nosemgrep

4
frappe/desk/doctype/tag/tag.py
View file

@ -112,9 +112,7 @@ class DocTags:
tl = unique(filter(lambda x: x, tl))
tags = ",".join(tl)
try:
frappe.db.sql(
"update `tab{}` set _user_tags={} where name={}".format(self.dt, "%s", "%s"), (tags, dn)
)
frappe.db.set_value(self.dt, dn, "_user_tags", tags, update_modified=False)
doc = frappe.get_lazy_doc(self.dt, dn)
update_tags(doc, tags)
except Exception as e:

2
frappe/desk/doctype/workspace/workspace.py
View file

@ -235,7 +235,7 @@ def disable_saving_as_public():
frappe.flags.in_install
or frappe.flags.in_uninstall
or frappe.flags.in_patch
or frappe.flags.in_test
or frappe.in_test
or frappe.flags.in_fixtures
or frappe.flags.in_migrate
)

5
frappe/desk/form/load.py
View file

@ -59,7 +59,7 @@ def getdoc(doctype, name):
@frappe.whitelist()
def getdoctype(doctype, with_parent=False, cached_timestamp=None):
def getdoctype(doctype, with_parent=False):
"""load doctype"""
docs = []
@ -75,9 +75,6 @@ def getdoctype(doctype, with_parent=False, cached_timestamp=None):
frappe.response["user_settings"] = get_user_settings(parent_dt or doctype)
if cached_timestamp and docs[0].modified == cached_timestamp:
return "use_cache"
frappe.response.docs.extend(docs)

36
frappe/desk/form/meta.py
View file

@ -16,9 +16,6 @@ ASSET_KEYS = (
"__css",
"__list_js",
"__calendar_js",
"__map_js",
"__linked_with",
"__messages",
"__print_formats",
"__workflow_docs",
"__form_grid_templates",
@ -60,9 +57,6 @@ class FormMeta(Meta):
if self.get("__assets_loaded", False):
return
self.add_search_fields()
self.add_linked_document_type()
if not self.istable:
self.add_code()
self.add_custom_script()
@ -77,15 +71,10 @@ class FormMeta(Meta):
def as_dict(self, no_nulls=False):
d = super().as_dict(no_nulls=no_nulls)
__dict = self.__dict__
for k in ASSET_KEYS:
d[k] = self.get(k)
# d['fields'] = d.get('fields', [])
for i, df in enumerate(d.get("fields") or []):
for k in ("search_fields", "is_custom_field", "linked_document_type"):
df[k] = self.get("fields")[i].get(k)
d[k] = __dict.get(k)
return d
@ -186,19 +175,6 @@ class FormMeta(Meta):
self.set("__custom_js", form_script)
self.set("__custom_list_js", list_script)
def add_search_fields(self):
"""add search fields found in the doctypes indicated by link fields' options"""
for df in self.get("fields", {"fieldtype": "Link", "options": ["!=", "[Select]"]}):
if df.options:
try:
search_fields = frappe.get_meta(df.options).search_fields
except frappe.DoesNotExistError:
self._show_missing_doctype_msg(df)
if search_fields:
search_fields = search_fields.split(",")
df.search_fields = [sf.strip() for sf in search_fields]
def _show_missing_doctype_msg(self, df):
# A link field is referring to non-existing doctype, this usually happens when
# customizations are removed or some custom app is removed but hasn't cleaned
@ -217,14 +193,6 @@ class FormMeta(Meta):
frappe.throw(msg, title=_("Missing DocType"))
def add_linked_document_type(self):
for df in self.get("fields", {"fieldtype": "Link"}):
if df.options:
try:
df.linked_document_type = frappe.get_meta(df.options).document_type
except frappe.DoesNotExistError:
self._show_missing_doctype_msg(df)
def load_print_formats(self):
print_formats = frappe.db.sql(
"""select * FROM `tabPrint Format`

8
frappe/desk/form/save.py
View file

@ -12,7 +12,7 @@ from frappe.utils.scheduler import is_scheduler_inactive
from frappe.utils.telemetry import capture_doc
@frappe.whitelist()
@frappe.whitelist(methods=["POST", "PUT"])
def savedocs(doc, action):
"""save / submit / update doclist"""
doc = frappe.get_doc(json.loads(doc))
@ -51,7 +51,7 @@ def savedocs(doc, action):
frappe.msgprint(frappe._(status_message), indicator="green", alert=True)
@frappe.whitelist()
@frappe.whitelist(methods=["POST", "PUT"])
def cancel(doctype=None, name=None, workflow_state_fieldname=None, workflow_state=None):
"""cancel a doclist"""
doc = frappe.get_doc(doctype, name)
@ -64,7 +64,7 @@ def cancel(doctype=None, name=None, workflow_state_fieldname=None, workflow_stat
frappe.msgprint(frappe._("Cancelled"), indicator="red", alert=True)
@frappe.whitelist()
@frappe.whitelist(methods=["POST", "PUT"])
def discard(doctype: str, name: str | int):
"""discard a draft document"""
doc = frappe.get_doc(doctype, name)
@ -79,7 +79,7 @@ def send_updated_docs(doc):
from .load import get_docinfo
get_docinfo(doc)
doc.apply_fieldlevel_read_permissions()
d = doc.as_dict()
if hasattr(doc, "localname"):
d["localname"] = doc.localname

29
frappe/desk/page/setup_wizard/setup_wizard.py
View file

@ -187,11 +187,32 @@ def run_setup_success(args): # nosemgrep
def get_stages_hooks(args): # nosemgrep
stages = []
for method in frappe.get_hooks("setup_wizard_stages"):
stages += frappe.get_attr(method)(args)
installed_apps = frappe.get_installed_apps(_ensure_on_bench=True)
for app_name in installed_apps:
setup_wizard_stages = frappe.get_hooks(app_name=app_name).get("setup_wizard_stages")
if not setup_wizard_stages:
continue
for method in setup_wizard_stages:
_stages = frappe.get_attr(method)(args)
update_app_details_in_stages(_stages, app_name)
stages += _stages
return stages
def update_app_details_in_stages(_stages, app_name):
for stage in _stages:
for key in stage:
if key != "tasks":
continue
for task in stage[key]:
if task.get("app_name") is None:
task["app_name"] = app_name
def get_setup_complete_hooks(args): # nosemgrep
return [
{
@ -244,7 +265,7 @@ def update_system_settings(args): # nosemgrep
"date_format": frappe.db.get_value("Country", args.get("country"), "date_format"),
"time_format": frappe.db.get_value("Country", args.get("country"), "time_format"),
"number_format": number_format,
"enable_scheduler": 1 if not frappe.flags.in_test else 0,
"enable_scheduler": 1 if not frappe.in_test else 0,
"backup_limit": 3, # Default for downloadable backups
"enable_telemetry": cint(args.get("enable_telemetry")),
}
@ -348,7 +369,9 @@ def _get_default_roles() -> set[str]:
def disable_future_access():
frappe.db.set_default("desktop:home_page", "workspace")
# Enable onboarding after install
frappe.clear_cache(doctype="System Settings")
frappe.db.set_single_value("System Settings", "enable_onboarding", 1)
frappe.db.set_single_value("System Settings", "setup_complete", frappe.is_setup_complete())
@frappe.whitelist()

32
frappe/desk/query_report.py
View file

@ -14,8 +14,9 @@ from frappe.desk.reportview import clean_params, parse_json
from frappe.model.utils import render_include
from frappe.modules import get_module_path, scrub
from frappe.monitor import add_data_to_monitor
from frappe.permissions import get_role_permissions, has_permission
from frappe.permissions import get_role_permissions, get_roles, has_permission
from frappe.utils import cint, cstr, flt, format_duration, get_html_format, sbool
from frappe.utils.caching import request_cache
def get_report_doc(report_name):
@ -706,6 +707,9 @@ def has_match(
match = False
break
if match:
match = has_unrestricted_read_access(doctype=ref_doctype, user=frappe.session.user)
# each doctype could have multiple conflicting user permission doctypes, hence using OR
# so that even if one of the sets allows a match, it is true
matched_for_doctype = matched_for_doctype or match
@ -722,6 +726,32 @@ def has_match(
return resultant_match
@request_cache
def has_unrestricted_read_access(doctype, user):
roles = get_roles(user)
permission_filters = {
"parent": doctype,
"role": ["in", roles],
"permlevel": 0,
"read": 1,
"if_owner": 0,
}
standard_perm_exists = frappe.db.exists(
"DocPerm",
permission_filters,
)
custom_perm_exists = frappe.db.exists(
"Custom DocPerm",
permission_filters,
)
has_perm = bool(custom_perm_exists or standard_perm_exists)
return has_perm
def get_linked_doctypes(columns, data):
linked_doctypes = {}

6
frappe/desk/reportview.py
View file

@ -315,7 +315,7 @@ def compress(data, args=None):
return {"keys": keys, "values": values, "user_info": user_info}
@frappe.whitelist()
@frappe.whitelist(methods=["POST", "PUT"])
def save_report(name, doctype, report_settings):
"""Save reports of type Report Builder from Report View"""
@ -345,7 +345,7 @@ def save_report(name, doctype, report_settings):
return report.name
@frappe.whitelist()
@frappe.whitelist(methods=["POST", "DELETE"])
def delete_report(name):
"""Delete reports of type Report Builder from Report View"""
@ -555,7 +555,7 @@ def parse_field(field: str) -> tuple[str | None, str]:
return None, key.strip("`")
@frappe.whitelist()
@frappe.whitelist(methods=["POST", "DELETE"])
def delete_items():
"""delete selected items"""
import json

8
frappe/email/doctype/email_account/email_account.py
View file

@ -159,7 +159,7 @@ class EmailAccount(Document):
if self.enable_incoming and self.use_imap and len(self.imap_folder) <= 0:
frappe.throw(_("You need to set one IMAP folder for {0}").format(frappe.bold(self.email_id)))
if frappe.local.flags.in_patch or frappe.local.flags.in_test:
if frappe.local.flags.in_patch or frappe.in_test:
return
use_oauth = self.auth_method == "OAuth"
@ -363,9 +363,7 @@ class EmailAccount(Document):
@property
def _password(self):
raise_exception = not (
self.auth_method == "OAuth" or self.no_smtp_authentication or frappe.flags.in_test
)
raise_exception = not (self.auth_method == "OAuth" or self.no_smtp_authentication or frappe.in_test)
return self.get_password(raise_exception=raise_exception)
@property
@ -565,7 +563,7 @@ class EmailAccount(Document):
self.set_failed_attempts_count(self.get_failed_attempts_count() + 1)
def _disable_broken_incoming_account(self, description):
if frappe.flags.in_test:
if frappe.in_test:
return
self.db_set("enable_incoming", 0)

2
frappe/email/doctype/email_domain/email_domain.py
View file

@ -81,7 +81,7 @@ class EmailDomain(Document):
def validate(self):
"""Validate POP3/IMAP and SMTP connections."""
if frappe.local.flags.in_patch or frappe.local.flags.in_test or frappe.local.flags.in_install:
if frappe.local.flags.in_patch or frappe.in_test or frappe.local.flags.in_install:
return
self.validate_incoming_server_conn()

6
frappe/email/doctype/email_queue/email_queue.py
View file

@ -182,7 +182,7 @@ class EmailQueue(Document):
message = ctx.build_message(recipient.recipient)
if method := get_hook_method("override_email_send"):
method(self, self.sender, recipient.recipient, message)
elif not frappe.flags.in_test or frappe.flags.testing_email:
elif not frappe.in_test or frappe.flags.testing_email:
if ctx.email_account_doc.service == "Frappe Mail":
is_newsletter = self.reference_doctype == "Newsletter"
ctx.frappe_mail_client.send_raw(
@ -200,7 +200,7 @@ class EmailQueue(Document):
ctx.update_recipient_status_to_sent(recipient)
if frappe.flags.in_test and not frappe.flags.testing_email:
if frappe.in_test and not frappe.flags.testing_email:
frappe.flags.sent_mail = message
return
@ -773,7 +773,7 @@ class QueueBuilder:
job_name=frappe.utils.get_job_name(
"send_bulk_emails_for", self.reference_doctype, self.reference_name
),
now=frappe.flags.in_test or send_now,
now=frappe.in_test or send_now,
queue="long",
)

16
frappe/email/doctype/newsletter/exceptions.py
View file

@ -1,16 +0,0 @@
# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See LICENSE
from frappe.exceptions import ValidationError
class NewsletterAlreadySentError(ValidationError):
pass
class NoRecipientFoundError(ValidationError):
pass
class NewsletterNotSavedError(ValidationError):
pass

229
frappe/email/doctype/newsletter/newsletter.js
View file

@ -1,229 +0,0 @@
// Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
// License: GNU General Public License v3. See license.txt
frappe.ui.form.on("Newsletter", {
refresh(frm) {
let doc = frm.doc;
let can_write = frappe.boot.user.can_write.includes(doc.doctype);
if (!frm.is_new() && !frm.is_dirty() && !doc.email_sent && can_write) {
frm.add_custom_button(
__("Send a test email"),
() => {
frm.events.send_test_email(frm);
},
__("Preview")
);
frm.add_custom_button(
__("Check broken links"),
() => {
frm.dashboard.set_headline(__("Checking broken links..."));
frm.call("find_broken_links").then((r) => {
frm.dashboard.set_headline("");
let links = r.message;
if (links && links.length) {
let html =
"<ul>" +
links.map((link) => `<li>${link}</li>`).join("") +
"</ul>";
frm.dashboard.set_headline(
__("Following links are broken in the email content: {0}", [html])
);
} else {
frm.dashboard.set_headline(
__("No broken links found in the email content")
);
setTimeout(() => {
frm.dashboard.set_headline("");
}, 3000);
}
});
},
__("Preview")
);
frm.add_custom_button(
__("Send now"),
() => {
if (frm.doc.schedule_send) {
frappe.confirm(
__(
"This newsletter was scheduled to send on a later date. Are you sure you want to send it now?"
),
function () {
frm.events.send_emails(frm);
}
);
return;
}
frappe.confirm(
__("Are you sure you want to send this newsletter now?"),
() => {
frm.events.send_emails(frm);
}
);
},
__("Send")
);
frm.add_custom_button(
__("Schedule sending"),
() => {
frm.events.schedule_send_dialog(frm);
},
__("Send")
);
}
frm.events.update_sending_status(frm);
if (frm.is_new() && !doc.sender_email) {
let { fullname, email } = frappe.user_info(doc.owner);
frm.set_value("sender_email", email);
frm.set_value("sender_name", fullname);
}
frm.trigger("update_schedule_message");
},
send_emails(frm) {
frappe.dom.freeze(__("Queuing emails..."));
frm.call("send_emails").then(() => {
frm.refresh();
frappe.dom.unfreeze();
frappe.show_alert(
__("Queued {0} emails", [frappe.utils.shorten_number(frm.doc.total_recipients)])
);
});
},
schedule_send_dialog(frm) {
let hours = frappe.utils.range(24);
let time_slots = hours.map((hour) => {
return `${(hour + "").padStart(2, "0")}:00`;
});
let d = new frappe.ui.Dialog({
title: __("Schedule Newsletter"),
fields: [
{
label: __("Date"),
fieldname: "date",
fieldtype: "Date",
options: {
minDate: new Date(),
},
reqd: true,
},
{
label: __("Time"),
fieldname: "time",
fieldtype: "Select",
options: time_slots,
reqd: true,
},
],
primary_action_label: __("Schedule"),
primary_action({ date, time }) {
frm.set_value("schedule_sending", 1);
frm.set_value("schedule_send", `${date} ${time}:00`);
d.hide();
frm.save();
},
secondary_action_label: __("Cancel Scheduling"),
secondary_action() {
frm.set_value("schedule_sending", 0);
frm.set_value("schedule_send", "");
d.hide();
frm.save();
},
});
if (frm.doc.schedule_sending) {
let parts = frm.doc.schedule_send.split(" ");
if (parts.length === 2) {
let [date, time] = parts;
d.set_value("date", date);
d.set_value("time", time.slice(0, 5));
}
}
d.show();
},
send_test_email(frm) {
let d = new frappe.ui.Dialog({
title: __("Send Test Email"),
fields: [
{
label: __("Email"),
fieldname: "email",
fieldtype: "Data",
options: "Email",
},
],
primary_action_label: __("Send"),
primary_action({ email }) {
d.get_primary_btn().text(__("Sending...")).prop("disabled", true);
frm.call("send_test_email", { email }).then(() => {
d.get_primary_btn().text(__("Send again")).prop("disabled", false);
});
},
});
d.show();
},
async update_sending_status(frm) {
if (frm.doc.email_sent && frm.$wrapper.is(":visible") && !frm.waiting_for_request) {
frm.waiting_for_request = true;
let res = await frm.call("get_sending_status");
frm.waiting_for_request = false;
let stats = res.message;
stats && frm.events.update_sending_progress(frm, stats);
if (
stats.sent + stats.error >= frm.doc.total_recipients ||
(!stats.total && !stats.emails_queued)
) {
frm.sending_status && clearInterval(frm.sending_status);
frm.sending_status = null;
return;
}
}
if (frm.sending_status) return;
frm.sending_status = setInterval(() => frm.events.update_sending_status(frm), 5000);
},
update_sending_progress(frm, stats) {
if (stats.sent + stats.error >= frm.doc.total_recipients || !frm.doc.email_sent) {
frm.doc.email_sent && frm.page.set_indicator(__("Sent"), "green");
frm.dashboard.hide_progress();
return;
}
if (stats.total) {
frm.page.set_indicator(__("Sending"), "blue");
frm.dashboard.show_progress(
__("Sending emails"),
(stats.sent * 100) / frm.doc.total_recipients,
__("{0} of {1} sent", [stats.sent, frm.doc.total_recipients])
);
} else if (stats.emails_queued) {
frm.page.set_indicator(__("Queued"), "blue");
}
},
on_hide(frm) {
if (frm.sending_status) {
clearInterval(frm.sending_status);
frm.sending_status = null;
}
},
update_schedule_message(frm) {
if (!frm.doc.email_sent && frm.doc.schedule_send) {
let datetime = frappe.datetime.global_date_format(frm.doc.schedule_send);
frm.dashboard.set_headline_alert(
__("This newsletter is scheduled to be sent on {0}", [datetime.bold()])
);
} else {
frm.dashboard.clear_headline();
}
},
});

282
frappe/email/doctype/newsletter/newsletter.json
View file

@ -1,282 +0,0 @@
{
"actions": [],
"allow_guest_to_view": 1,
"allow_rename": 1,
"creation": "2013-01-10 16:34:31",
"description": "Create and send emails to a specific group of subscribers periodically.",
"doctype": "DocType",
"document_type": "Other",
"engine": "InnoDB",
"field_order": [
"status_section",
"email_sent_at",
"column_break_3",
"total_recipients",
"column_break_12",
"total_views",
"email_sent",
"from_section",
"sender_name",
"column_break_5",
"sender_email",
"column_break_7",
"send_from",
"recipients",
"email_group",
"subject_section",
"subject",
"newsletter_content",
"content_type",
"message",
"message_md",
"message_html",
"campaign",
"attachments",
"send_unsubscribe_link",
"send_webview_link",
"schedule_settings_section",
"scheduled_to_send",
"schedule_sending",
"schedule_send",
"publish_as_a_web_page_section",
"published",
"route"
],
"fields": [
{
"fieldname": "email_group",
"fieldtype": "Table",
"in_standard_filter": 1,
"label": "Audience",
"options": "Newsletter Email Group",
"reqd": 1
},
{
"fieldname": "send_from",
"fieldtype": "Data",
"ignore_xss_filter": 1,
"label": "Sender",
"read_only": 1
},
{
"default": "0",
"fieldname": "email_sent",
"fieldtype": "Check",
"hidden": 1,
"label": "Email Sent",
"no_copy": 1,
"read_only": 1
},
{
"fieldname": "newsletter_content",
"fieldtype": "Section Break",
"label": "Content"
},
{
"fieldname": "subject",
"fieldtype": "Small Text",
"in_global_search": 1,
"in_list_view": 1,
"label": "Subject",
"reqd": 1
},
{
"depends_on": "eval: doc.content_type === 'Rich Text'",
"fieldname": "message",
"fieldtype": "Text Editor",
"in_list_view": 1,
"label": "Message",
"mandatory_depends_on": "eval: doc.content_type === 'Rich Text'"
},
{
"default": "1",
"fieldname": "send_unsubscribe_link",
"fieldtype": "Check",
"label": "Send Unsubscribe Link"
},
{
"default": "0",
"fieldname": "published",
"fieldtype": "Check",
"label": "Published"
},
{
"depends_on": "published",
"fieldname": "route",
"fieldtype": "Data",
"label": "Route",
"read_only": 1
},
{
"fieldname": "scheduled_to_send",
"fieldtype": "Int",
"hidden": 1,
"label": "Scheduled To Send"
},
{
"fieldname": "recipients",
"fieldtype": "Section Break",
"label": "To"
},
{
"depends_on": "eval: doc.schedule_sending",
"fieldname": "schedule_send",
"fieldtype": "Datetime",
"label": "Send Email At",
"read_only": 1,
"read_only_depends_on": "eval: doc.email_sent"
},
{
"fieldname": "content_type",
"fieldtype": "Select",
"label": "Content Type",
"options": "Rich Text\nMarkdown\nHTML"
},
{
"depends_on": "eval:doc.content_type === 'Markdown'",
"fieldname": "message_md",
"fieldtype": "Markdown Editor",
"label": "Message (Markdown)",
"mandatory_depends_on": "eval:doc.content_type === 'Markdown'"
},
{
"depends_on": "eval:doc.content_type === 'HTML'",
"fieldname": "message_html",
"fieldtype": "HTML Editor",
"label": "Message (HTML)",
"mandatory_depends_on": "eval:doc.content_type === 'HTML'"
},
{
"default": "0",
"fieldname": "schedule_sending",
"fieldtype": "Check",
"label": "Schedule sending at a later time",
"read_only_depends_on": "eval: doc.email_sent"
},
{
"default": "0",
"fieldname": "send_webview_link",
"fieldtype": "Check",
"label": "Send Web View Link"
},
{
"fieldname": "from_section",
"fieldtype": "Section Break",
"label": "From"
},
{
"fieldname": "sender_name",
"fieldtype": "Data",
"label": "Sender Name"
},
{
"fieldname": "sender_email",
"fieldtype": "Data",
"label": "Sender Email",
"options": "Email",
"reqd": 1
},
{
"fieldname": "column_break_5",
"fieldtype": "Column Break"
},
{
"fieldname": "column_break_7",
"fieldtype": "Column Break"
},
{
"fieldname": "subject_section",
"fieldtype": "Section Break",
"label": "Subject"
},
{
"fieldname": "publish_as_a_web_page_section",
"fieldtype": "Section Break",
"label": "Publish as a web page"
},
{
"depends_on": "schedule_sending",
"fieldname": "schedule_settings_section",
"fieldtype": "Section Break",
"label": "Scheduled Sending"
},
{
"fieldname": "attachments",
"fieldtype": "Table",
"label": "Attachments",
"options": "Newsletter Attachment"
},
{
"fieldname": "email_sent_at",
"fieldtype": "Datetime",
"label": "Email Sent At",
"read_only": 1
},
{
"fieldname": "total_recipients",
"fieldtype": "Int",
"label": "Total Recipients",
"read_only": 1
},
{
"depends_on": "email_sent",
"fieldname": "status_section",
"fieldtype": "Section Break",
"label": "Status"
},
{
"fieldname": "column_break_12",
"fieldtype": "Column Break"
},
{
"fieldname": "column_break_3",
"fieldtype": "Column Break"
},
{
"default": "0",
"fieldname": "total_views",
"fieldtype": "Int",
"label": "Total Views",
"no_copy": 1,
"read_only": 1
},
{
"fieldname": "campaign",
"fieldtype": "Link",
"label": "Campaign",
"options": "UTM Campaign"
}
],
"has_web_view": 1,
"icon": "fa fa-envelope",
"idx": 1,
"index_web_pages_for_search": 1,
"is_published_field": "published",
"links": [],
"make_attachments_public": 1,
"modified": "2024-11-12 12:41:02.569631",
"modified_by": "Administrator",
"module": "Email",
"name": "Newsletter",
"owner": "Administrator",
"permissions": [
{
"create": 1,
"delete": 1,
"email": 1,
"export": 1,
"print": 1,
"read": 1,
"report": 1,
"role": "Newsletter Manager",
"share": 1,
"write": 1
}
],
"route": "newsletters",
"sort_field": "creation",
"sort_order": "ASC",
"states": [],
"title_field": "subject",
"track_changes": 1
}

457
frappe/email/doctype/newsletter/newsletter.py
View file

@ -1,457 +0,0 @@
# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See LICENSE
import frappe
import frappe.utils
from frappe import _
from frappe.email.doctype.email_group.email_group import add_subscribers
from frappe.rate_limiter import rate_limit
from frappe.utils.safe_exec import is_job_queued
from frappe.utils.verified_command import get_signed_params, verify_request
from frappe.website.website_generator import WebsiteGenerator
from .exceptions import NewsletterAlreadySentError, NewsletterNotSavedError, NoRecipientFoundError
class Newsletter(WebsiteGenerator):
# begin: auto-generated types
# This code is auto-generated. Do not modify anything in this block.
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from frappe.email.doctype.newsletter_attachment.newsletter_attachment import NewsletterAttachment
from frappe.email.doctype.newsletter_email_group.newsletter_email_group import NewsletterEmailGroup
from frappe.types import DF
attachments: DF.Table[NewsletterAttachment]
campaign: DF.Link | None
content_type: DF.Literal["Rich Text", "Markdown", "HTML"]
email_group: DF.Table[NewsletterEmailGroup]
email_sent: DF.Check
email_sent_at: DF.Datetime | None
message: DF.TextEditor | None
message_html: DF.HTMLEditor | None
message_md: DF.MarkdownEditor | None
published: DF.Check
route: DF.Data | None
schedule_send: DF.Datetime | None
schedule_sending: DF.Check
scheduled_to_send: DF.Int
send_from: DF.Data | None
send_unsubscribe_link: DF.Check
send_webview_link: DF.Check
sender_email: DF.Data
sender_name: DF.Data | None
subject: DF.SmallText
total_recipients: DF.Int
total_views: DF.Int
# end: auto-generated types
def validate(self):
self.route = f"newsletters/{self.name}"
self.validate_sender_address()
self.validate_publishing()
self.validate_scheduling_date()
@property
def newsletter_recipients(self) -> list[str]:
if getattr(self, "_recipients", None) is None:
self._recipients = self.get_recipients()
return self._recipients
@frappe.whitelist()
def get_sending_status(self):
count_by_status = frappe.get_all(
"Email Queue",
filters={"reference_doctype": self.doctype, "reference_name": self.name},
fields=["status", "count(name) as count"],
group_by="status",
order_by="status",
)
sent = 0
error = 0
total = 0
for row in count_by_status:
if row.status == "Sent":
sent = row.count
elif row.status == "Error":
error = row.count
total += row.count
emails_queued = is_job_queued(
job_name=frappe.utils.get_job_name("send_bulk_emails_for", self.doctype, self.name),
queue="long",
)
return {"sent": sent, "error": error, "total": total, "emails_queued": emails_queued}
@frappe.whitelist()
def send_test_email(self, email):
test_emails = frappe.utils.validate_email_address(email, throw=True)
self.send_newsletter(emails=test_emails, test_email=True)
frappe.msgprint(_("Test email sent to {0}").format(email), alert=True)
@frappe.whitelist()
def find_broken_links(self):
import requests
from bs4 import BeautifulSoup
html = self.get_message()
soup = BeautifulSoup(html, "html.parser")
links = soup.find_all("a")
images = soup.find_all("img")
broken_links = []
for el in links + images:
url = el.attrs.get("href") or el.attrs.get("src")
try:
response = requests.head(url, verify=False, timeout=5)
if response.status_code >= 400:
broken_links.append(url)
except Exception:
broken_links.append(url)
return broken_links
@frappe.whitelist()
def send_emails(self):
"""queue sending emails to recipients"""
self.schedule_sending = False
self.schedule_send = None
self.queue_all()
def validate_send(self):
"""Validate if Newsletter can be sent."""
self.validate_newsletter_status()
self.validate_newsletter_recipients()
def validate_newsletter_status(self):
if self.email_sent:
frappe.throw(_("Newsletter has already been sent"), exc=NewsletterAlreadySentError)
if self.get("__islocal"):
frappe.throw(_("Please save the Newsletter before sending"), exc=NewsletterNotSavedError)
def validate_newsletter_recipients(self):
if not self.newsletter_recipients:
frappe.throw(_("Newsletter should have atleast one recipient"), exc=NoRecipientFoundError)
def validate_sender_address(self):
"""Validate self.send_from is a valid email address or not."""
if self.sender_email:
frappe.utils.validate_email_address(self.sender_email, throw=True)
self.send_from = (
f"{self.sender_name} <{self.sender_email}>" if self.sender_name else self.sender_email
)
def validate_publishing(self):
if self.send_webview_link and not self.published:
frappe.throw(_("Newsletter must be published to send webview link in email"))
def validate_scheduling_date(self):
if getattr(frappe.flags, "is_scheduler_running", False):
return
if (
self.schedule_sending
and frappe.utils.get_datetime(self.schedule_send) < frappe.utils.now_datetime()
):
frappe.throw(_("Past dates are not allowed for Scheduling."))
def get_linked_email_queue(self) -> list[str]:
"""Get list of email queue linked to this newsletter."""
return frappe.get_all(
"Email Queue",
filters={
"reference_doctype": self.doctype,
"reference_name": self.name,
},
pluck="name",
)
def get_queued_recipients(self) -> list[str]:
"""Recipients who have already been queued for receiving the newsletter."""
return frappe.get_all(
"Email Queue Recipient",
filters={
"parent": ("in", self.get_linked_email_queue()),
},
pluck="recipient",
)
def get_pending_recipients(self) -> list[str]:
"""Get list of pending recipients of the newsletter. These
recipients may not have receive the newsletter in the previous iteration.
"""
queued_recipients = set(self.get_queued_recipients())
return [x for x in self.newsletter_recipients if x not in queued_recipients]
def queue_all(self):
"""Queue Newsletter to all the recipients generated from the `Email Group` table"""
self.validate()
self.validate_send()
recipients = self.get_pending_recipients()
self.send_newsletter(emails=recipients)
self.email_sent = True
self.email_sent_at = frappe.utils.now()
self.total_recipients = len(recipients)
self.save()
def get_newsletter_attachments(self) -> list[dict[str, str]]:
"""Get list of attachments on current Newsletter"""
return [{"file_url": row.attachment} for row in self.attachments]
def send_newsletter(self, emails: list[str], test_email: bool = False):
"""Trigger email generation for `emails` and add it in Email Queue."""
attachments = self.get_newsletter_attachments()
sender = self.send_from or frappe.utils.get_formatted_email(self.owner)
args = self.as_dict()
args["message"] = self.get_message(medium="email")
is_auto_commit_set = bool(frappe.db.auto_commit_on_many_writes)
frappe.db.auto_commit_on_many_writes = not frappe.flags.in_test
frappe.sendmail(
subject=self.subject,
sender=sender,
recipients=emails,
attachments=attachments,
template="newsletter",
add_unsubscribe_link=self.send_unsubscribe_link,
unsubscribe_method="/unsubscribe",
reference_doctype=self.doctype,
reference_name=self.name,
queue_separately=True,
send_priority=0,
args=args,
email_read_tracker_url=None
if test_email
else "/api/method/frappe.email.doctype.newsletter.newsletter.newsletter_email_read",
)
frappe.db.auto_commit_on_many_writes = is_auto_commit_set
def get_message(self, medium=None) -> str:
message = self.message
if self.content_type == "Markdown":
message = frappe.utils.md_to_html(self.message_md)
if self.content_type == "HTML":
message = self.message_html
html = frappe.render_template(message, {"doc": self.as_dict()})
return self.add_source(html, medium=medium)
def add_source(self, html: str, medium="None") -> str:
"""Add source to the site links in the newsletter content."""
from bs4 import BeautifulSoup
soup = BeautifulSoup(html, "html.parser")
links = soup.find_all("a")
for link in links:
href = link.get("href")
if href and not href.startswith("#"):
if not frappe.utils.is_site_link(href):
continue
new_href = frappe.utils.add_trackers_to_url(
href, source="Newsletter", campaign=self.campaign, medium=medium
)
link["href"] = new_href
return str(soup)
def get_recipients(self) -> list[str]:
"""Get recipients from Email Group"""
emails = frappe.get_all(
"Email Group Member",
filters={"unsubscribed": 0, "email_group": ("in", self.get_email_groups())},
pluck="email",
)
return list(set(emails))
def get_email_groups(self) -> list[str]:
# wondering why the 'or'? i can't figure out why both aren't equivalent - @gavin
return [x.email_group for x in self.email_group] or frappe.get_all(
"Newsletter Email Group",
filters={"parent": self.name, "parenttype": "Newsletter"},
pluck="email_group",
)
def get_attachments(self) -> list[dict[str, str]]:
return frappe.get_all(
"File",
fields=["name", "file_name", "file_url", "is_private"],
filters={
"attached_to_name": self.name,
"attached_to_doctype": "Newsletter",
"is_private": 0,
},
)
def confirmed_unsubscribe(email, group):
"""unsubscribe the email(user) from the mailing list(email_group)"""
frappe.flags.ignore_permissions = True
doc = frappe.get_doc("Email Group Member", {"email": email, "email_group": group})
if not doc.unsubscribed:
doc.unsubscribed = 1
doc.save(ignore_permissions=True)
@frappe.whitelist(allow_guest=True)
@rate_limit(limit=10, seconds=60 * 60)
def subscribe(email, email_group=None):
"""API endpoint to subscribe an email to a particular email group. Triggers a confirmation email."""
if email_group is None:
email_group = get_default_email_group()
# build subscription confirmation URL
api_endpoint = frappe.utils.get_url(
"/api/method/frappe.email.doctype.newsletter.newsletter.confirm_subscription"
)
signed_params = get_signed_params({"email": email, "email_group": email_group})
confirm_subscription_url = f"{api_endpoint}?{signed_params}"
# fetch custom template if available
email_confirmation_template = frappe.db.get_value(
"Email Group", email_group, "confirmation_email_template"
)
# build email and send
if email_confirmation_template:
args = {"email": email, "confirmation_url": confirm_subscription_url, "email_group": email_group}
email_template = frappe.get_doc("Email Template", email_confirmation_template)
email_subject = email_template.subject
content = frappe.render_template(email_template.response, args)
else:
email_subject = _("Confirm Your Email")
translatable_content = (
_("Thank you for your interest in subscribing to our updates"),
_("Please verify your Email Address"),
confirm_subscription_url,
_("Click here to verify"),
)
content = """
<p>{}. {}.</p>
<p><a href="{}">{}</a></p>
""".format(*translatable_content)
frappe.sendmail(
email,
subject=email_subject,
content=content,
)
@frappe.whitelist(allow_guest=True)
def confirm_subscription(email, email_group=None):
"""API endpoint to confirm email subscription.
This endpoint is called when user clicks on the link sent to their mail.
"""
if not verify_request():
return
if email_group is None:
email_group = get_default_email_group()
try:
group = frappe.get_doc("Email Group", email_group)
except frappe.DoesNotExistError:
group = frappe.get_doc({"doctype": "Email Group", "title": email_group}).insert(
ignore_permissions=True
)
frappe.flags.ignore_permissions = True
add_subscribers(email_group, email)
frappe.db.commit()
welcome_url = group.get_welcome_url(email)
if welcome_url:
frappe.local.response["type"] = "redirect"
frappe.local.response["location"] = welcome_url
else:
frappe.respond_as_web_page(
_("Confirmed"),
_("{0} has been successfully added to the Email Group.").format(email),
indicator_color="green",
)
def get_list_context(context=None):
context.update(
{
"show_search": True,
"no_breadcrumbs": True,
"title": _("Newsletters"),
"filters": {"published": 1},
"row_template": "email/doctype/newsletter/templates/newsletter_row.html",
}
)
def send_scheduled_email():
"""Send scheduled newsletter to the recipients."""
frappe.flags.is_scheduler_running = True
scheduled_newsletter = frappe.get_all(
"Newsletter",
filters={
"schedule_send": ("<=", frappe.utils.now_datetime()),
"email_sent": False,
"schedule_sending": True,
},
ignore_ifnull=True,
pluck="name",
)
for newsletter_name in scheduled_newsletter:
try:
newsletter = frappe.get_doc("Newsletter", newsletter_name)
newsletter.queue_all()
except Exception:
frappe.db.rollback()
# wasn't able to send emails :(
frappe.db.set_value("Newsletter", newsletter_name, "email_sent", 0)
newsletter.log_error("Failed to send newsletter")
if not frappe.flags.in_test:
frappe.db.commit()
frappe.flags.is_scheduler_running = False
@frappe.whitelist(allow_guest=True)
def newsletter_email_read(recipient_email=None, reference_doctype=None, reference_name=None):
if not (recipient_email and reference_name):
return
verify_request()
try:
doc = frappe.get_cached_doc("Newsletter", reference_name)
if doc.add_viewed(recipient_email, force=True, unique_views=True):
newsletter = frappe.qb.DocType("Newsletter")
(
frappe.qb.update(newsletter)
.set(newsletter.total_views, newsletter.total_views + 1)
.where(newsletter.name == doc.name)
).run()
except Exception:
frappe.log_error(
title=f"Unable to mark as viewed for {recipient_email}",
reference_doctype="Newsletter",
reference_name=reference_name,
)
finally:
frappe.response.update(frappe.utils.get_imaginary_pixel_response())
def get_default_email_group():
return _("Website", lang=frappe.db.get_default("language"))

12
frappe/email/doctype/newsletter/newsletter_list.js
View file

@ -1,12 +0,0 @@
frappe.listview_settings["Newsletter"] = {
add_fields: ["subject", "email_sent", "schedule_sending"],
get_indicator: function (doc) {
if (doc.email_sent) {
return [__("Sent"), "green", "email_sent,=,1"];
} else if (doc.schedule_sending) {
return [__("Scheduled"), "purple", "email_sent,=,0|schedule_sending,=,1"];
} else {
return [__("Not Sent"), "gray", "email_sent,=,0"];
}
},
};

65
frappe/email/doctype/newsletter/templates/newsletter.html
View file

@ -1,65 +0,0 @@
{% extends "templates/web.html" %}
{% block title %} {{ doc.subject }} {% endblock %}
{% block page_content %}
<style>
.blog-container {
max-width: 720px;
margin: auto;
}
.blog-header {
font-weight: 700;
font-size: 1.5em;
}
.blog-info {
text-align:center;
margin-top: 30px;
}
.blog-text {
padding-top: 50px;
padding-bottom: 50px;
font-size: 15px;
line-height: 1.5;
}
.blog-text p {
margin-bottom: 30px;
}
</style>
<div class="blog-container">
<article class="blog-content" itemscope>
<div class="blog-info">
<h1 itemprop="headline" class="blog-header">{{ doc.subject }}</h1>
<p class="post-by text-muted">
{{ frappe.format_date(doc.modified) }}
</p>
</div>
<div itemprop="articleBody" class="longform blog-text">
{{ doc.get_message(medium="web_page") }}
</div>
</article>
{% if doc.attachments %}
<div>
<div class="row text-muted">
<div class="col-sm-12 h6 text-uppercase">
{{ _("Attachments") }}
</div>
</div>
<div class="row">
<div class="col-sm-12">
{% for attachment in doc.attachments %}
<p class="small">
<a href="{{ attachment.attachment }}" target="_blank">
{{ attachment.attachment }}
</a>
</p>
{% endfor %}
</div>
</div>
</div>
{% endif %}
</div>
{% endblock %}

15
frappe/email/doctype/newsletter/templates/newsletter_row.html
View file

@ -1,15 +0,0 @@
<div class="web-list-item transaction-list-item">
<a href = "{{ route }}/">
<div class="row">
<div class="col-sm-8 text-left bold">
{{ doc.subject }}
</div>
<div class="col-sm-4">
<div class="text-muted text-right"
title="{{ frappe.utils.format_datetime(doc.modified, "medium") }}">
{{ frappe.utils.pretty_date(doc.modified) }}
</div>
</div>
</div>
</a>
</div>

252
frappe/email/doctype/newsletter/test_newsletter.py
View file

@ -1,252 +0,0 @@
# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See LICENSE
from random import choice
from unittest.mock import MagicMock, PropertyMock, patch
import frappe
from frappe.email.doctype.newsletter.exceptions import (
NewsletterAlreadySentError,
NoRecipientFoundError,
)
from frappe.email.doctype.newsletter.newsletter import (
Newsletter,
confirmed_unsubscribe,
send_scheduled_email,
)
from frappe.email.queue import flush
from frappe.tests import IntegrationTestCase
from frappe.utils import add_days, getdate
emails = [
"test_subscriber1@example.com",
"test_subscriber2@example.com",
"test_subscriber3@example.com",
"test1@example.com",
]
newsletters = []
def get_dotted_path(obj: type) -> str:
klass = obj.__class__
module = klass.__module__
if module == "builtins":
return klass.__qualname__ # avoid outputs like 'builtins.str'
return f"{module}.{klass.__qualname__}"
class TestNewsletterMixin:
def setUp(self):
frappe.set_user("Administrator")
self.setup_email_group()
def tearDown(self):
frappe.set_user("Administrator")
for newsletter in newsletters:
frappe.db.delete(
"Email Queue",
{
"reference_doctype": "Newsletter",
"reference_name": newsletter,
},
)
frappe.delete_doc("Newsletter", newsletter)
frappe.db.delete("Newsletter Email Group", {"parent": newsletter})
newsletters.remove(newsletter)
def setup_email_group(self):
if not frappe.db.exists("Email Group", "_Test Email Group"):
frappe.get_doc({"doctype": "Email Group", "title": "_Test Email Group"}).insert()
for email in emails:
doctype = "Email Group Member"
email_filters = {"email": email, "email_group": "_Test Email Group"}
savepoint = "setup_email_group"
frappe.db.savepoint(savepoint)
try:
frappe.get_doc(
{
"doctype": doctype,
**email_filters,
}
).insert(ignore_if_duplicate=True)
except Exception:
frappe.db.rollback(save_point=savepoint)
frappe.db.set_value(doctype, email_filters, "unsubscribed", 0)
frappe.db.release_savepoint(savepoint)
def send_newsletter(self, published=0, schedule_send=None) -> str | None:
frappe.db.delete("Email Queue")
frappe.db.delete("Email Queue Recipient")
frappe.db.delete("Newsletter")
newsletter_options = {
"published": published,
"schedule_sending": bool(schedule_send),
"schedule_send": schedule_send,
}
newsletter = self.get_newsletter(**newsletter_options)
if schedule_send:
send_scheduled_email()
else:
newsletter.send_emails()
return newsletter.name
return newsletter
@staticmethod
def get_newsletter(**kwargs) -> "Newsletter":
"""Generate and return Newsletter object"""
doctype = "Newsletter"
newsletter_content = {
"subject": "_Test Newsletter",
"sender_name": "Test Sender",
"sender_email": "test_sender@example.com",
"content_type": "Rich Text",
"message": "Testing my news.",
}
similar_newsletters = frappe.get_all(doctype, newsletter_content, pluck="name")
for similar_newsletter in similar_newsletters:
frappe.delete_doc(doctype, similar_newsletter)
newsletter = frappe.get_doc({"doctype": doctype, **newsletter_content, **kwargs})
newsletter.append("email_group", {"email_group": "_Test Email Group"})
newsletter.save(ignore_permissions=True)
newsletter.reload()
newsletters.append(newsletter.name)
attached_files = frappe.get_all(
"File",
{
"attached_to_doctype": newsletter.doctype,
"attached_to_name": newsletter.name,
},
pluck="name",
)
for file in attached_files:
frappe.delete_doc("File", file)
return newsletter
class TestNewsletter(TestNewsletterMixin, IntegrationTestCase):
def test_send(self):
self.send_newsletter()
email_queue_list = [frappe.get_doc("Email Queue", e.name) for e in frappe.get_all("Email Queue")]
self.assertEqual(len(email_queue_list), 4)
recipients = {e.recipients[0].recipient for e in email_queue_list}
self.assertTrue(set(emails).issubset(recipients))
def test_unsubscribe(self):
name = self.send_newsletter()
to_unsubscribe = choice(emails)
group = frappe.get_all("Newsletter Email Group", filters={"parent": name}, fields=["email_group"])
flush()
confirmed_unsubscribe(to_unsubscribe, group[0].email_group)
name = self.send_newsletter()
email_queue_list = [frappe.get_doc("Email Queue", e.name) for e in frappe.get_all("Email Queue")]
self.assertEqual(len(email_queue_list), 3)
recipients = [e.recipients[0].recipient for e in email_queue_list]
for email in emails:
if email != to_unsubscribe:
self.assertTrue(email in recipients)
def test_schedule_send(self):
newsletter = self.send_newsletter(schedule_send=add_days(getdate(), 1))
newsletter.db_set("schedule_send", add_days(getdate(), -1)) # Set date in past
send_scheduled_email()
email_queue_list = [frappe.get_doc("Email Queue", e.name) for e in frappe.get_all("Email Queue")]
self.assertEqual(len(email_queue_list), 4)
recipients = [e.recipients[0].recipient for e in email_queue_list]
for email in emails:
self.assertTrue(email in recipients)
def test_newsletter_send_test_email(self):
"""Test "Send Test Email" functionality of Newsletter"""
newsletter = self.get_newsletter()
test_email = choice(emails)
newsletter.send_test_email(test_email)
self.assertFalse(newsletter.email_sent)
newsletter.save = MagicMock()
self.assertFalse(newsletter.save.called)
# check if the test email is in the queue
email_queue = frappe.get_all(
"Email Queue",
filters=[
["reference_doctype", "=", "Newsletter"],
["reference_name", "=", newsletter.name],
["Email Queue Recipient", "recipient", "=", test_email],
],
)
self.assertTrue(email_queue)
def test_newsletter_status(self):
"""Test for Newsletter's stats on onload event"""
newsletter = self.get_newsletter()
newsletter.email_sent = True
result = newsletter.get_sending_status()
self.assertTrue("total" in result)
self.assertTrue("sent" in result)
def test_already_sent_newsletter(self):
newsletter = self.get_newsletter()
newsletter.send_emails()
with self.assertRaises(NewsletterAlreadySentError):
newsletter.send_emails()
def test_newsletter_with_no_recipient(self):
newsletter = self.get_newsletter()
property_path = f"{get_dotted_path(newsletter)}.newsletter_recipients"
with patch(property_path, new_callable=PropertyMock) as mock_newsletter_recipients:
mock_newsletter_recipients.return_value = []
with self.assertRaises(NoRecipientFoundError):
newsletter.send_emails()
def test_send_scheduled_email_error_handling(self):
newsletter = self.get_newsletter(schedule_send=add_days(getdate(), -1))
job_path = "frappe.email.doctype.newsletter.newsletter.Newsletter.queue_all"
m = MagicMock(side_effect=frappe.OutgoingEmailError)
with self.assertRaises(frappe.OutgoingEmailError):
with patch(job_path, new_callable=m):
send_scheduled_email()
newsletter.reload()
self.assertEqual(newsletter.email_sent, 0)
def test_retry_partially_sent_newsletter(self):
frappe.db.delete("Email Queue")
frappe.db.delete("Email Queue Recipient")
frappe.db.delete("Newsletter")
newsletter = self.get_newsletter()
newsletter.send_emails()
email_queue_list = [frappe.get_doc("Email Queue", e.name) for e in frappe.get_all("Email Queue")]
self.assertEqual(len(email_queue_list), 4)
# delete a queue document to emulate partial send
queue_recipient_name = email_queue_list[0].recipients[0].recipient
email_queue_list[0].delete()
newsletter.email_sent = False
# make sure the pending recipient is only the one which has been deleted
self.assertEqual(newsletter.get_pending_recipients(), [queue_recipient_name])
# retry
newsletter.send_emails()
self.assertEqual(frappe.db.count("Email Queue"), 4)
self.assertTrue(newsletter.email_sent)

0
frappe/email/doctype/newsletter_attachment/__init__.py
View file

32
frappe/email/doctype/newsletter_attachment/newsletter_attachment.json
View file

@ -1,32 +0,0 @@
{
"actions": [],
"allow_rename": 1,
"creation": "2021-12-06 16:37:40.652468",
"doctype": "DocType",
"editable_grid": 1,
"engine": "InnoDB",
"field_order": [
"attachment"
],
"fields": [
{
"fieldname": "attachment",
"fieldtype": "Attach",
"in_list_view": 1,
"label": "Attachment",
"reqd": 1
}
],
"index_web_pages_for_search": 1,
"istable": 1,
"links": [],
"modified": "2024-03-23 16:03:31.101104",
"modified_by": "Administrator",
"module": "Email",
"name": "Newsletter Attachment",
"owner": "Administrator",
"permissions": [],
"sort_field": "creation",
"sort_order": "DESC",
"states": []
}

23
frappe/email/doctype/newsletter_attachment/newsletter_attachment.py
View file

@ -1,23 +0,0 @@
# Copyright (c) 2021, Frappe Technologies and contributors
# For license information, please see license.txt
# import frappe
from frappe.model.document import Document
class NewsletterAttachment(Document):
# begin: auto-generated types
# This code is auto-generated. Do not modify anything in this block.
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from frappe.types import DF
attachment: DF.Attach
parent: DF.Data
parentfield: DF.Data
parenttype: DF.Data
# end: auto-generated types
pass

0
frappe/email/doctype/newsletter_email_group/__init__.py
View file

43
frappe/email/doctype/newsletter_email_group/newsletter_email_group.json
View file

@ -1,43 +0,0 @@
{
"actions": [],
"creation": "2017-02-26 16:20:52.654136",
"doctype": "DocType",
"editable_grid": 1,
"engine": "InnoDB",
"field_order": [
"email_group",
"total_subscribers"
],
"fields": [
{
"columns": 7,
"fieldname": "email_group",
"fieldtype": "Link",
"in_list_view": 1,
"label": "Email Group",
"options": "Email Group",
"reqd": 1
},
{
"columns": 3,
"fetch_from": "email_group.total_subscribers",
"fieldname": "total_subscribers",
"fieldtype": "Read Only",
"in_list_view": 1,
"label": "Total Subscribers"
}
],
"istable": 1,
"links": [],
"modified": "2024-03-23 16:03:31.190219",
"modified_by": "Administrator",
"module": "Email",
"name": "Newsletter Email Group",
"owner": "Administrator",
"permissions": [],
"quick_entry": 1,
"sort_field": "creation",
"sort_order": "DESC",
"states": [],
"track_changes": 1
}

23
frappe/email/doctype/newsletter_email_group/newsletter_email_group.py
View file

@ -1,23 +0,0 @@
# Copyright (c) 2015, Frappe Technologies and contributors
# License: MIT. See LICENSE
from frappe.model.document import Document
class NewsletterEmailGroup(Document):
# begin: auto-generated types
# This code is auto-generated. Do not modify anything in this block.
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from frappe.types import DF
email_group: DF.Link
parent: DF.Data
parentfield: DF.Data
parenttype: DF.Data
total_subscribers: DF.ReadOnly | None
# end: auto-generated types
pass

2
frappe/email/doctype/notification/notification.py
View file

@ -322,7 +322,7 @@ def get_context(context):
"frappe.email.doctype.notification.notification.evaluate_alert",
doc=doc,
alert=self,
now=frappe.flags.in_test,
now=frappe.in_test,
enqueue_after_commit=enqueue_after_commit,
)

2
frappe/email/queue.py
View file

@ -95,7 +95,7 @@ def get_unsubcribed_url(reference_doctype, reference_name, email, unsubscribe_me
@frappe.whitelist(allow_guest=True)
def unsubscribe(doctype, name, email):
# unsubsribe from comments and communications
if not frappe.flags.in_test and not verify_request():
if not frappe.in_test and not verify_request():
return
try:

20
frappe/email/receive.py
View file

@ -162,10 +162,14 @@ class EmailServer:
return res[0] == "OK" # The folder exists TODO: handle other responses too
def logout(self):
if cint(self.settings.use_imap):
self.imap.logout()
else:
self.pop.quit()
try:
if cint(self.settings.use_imap):
self.imap.logout()
else:
self.pop.quit()
except imaplib.IMAP4.abort:
self.connect()
self.logout()
return
def get_messages(self, folder="INBOX"):
@ -219,6 +223,9 @@ class EmailServer:
uidnext = int(self.parse_imap_response("UIDNEXT", message[0]) or "1")
frappe.db.set_value("Email Account", self.settings.email_account, "uidnext", uidnext)
if uid_validity is None:
frappe.flags.initial_sync = True
if not uid_validity or uid_validity != current_uid_validity:
# uidvalidity changed & all email uids are reindexed by server
frappe.db.set_value(
@ -277,8 +284,9 @@ class EmailServer:
except imaplib.IMAP4.abort:
if self.retry_count < self.retry_limit:
self.connect()
self.get_messages(folder)
self.retry_count += 1
self.get_messages(folder)
except Exception as e:
if self.has_login_limit_exceeded(e):
raise LoginLimitExceeded(e) from e
@ -632,7 +640,7 @@ class InboundMail(Email):
def process(self):
"""Create communication record from email."""
if self.is_sender_same_as_receiver() and not self.is_reply():
if frappe.flags.in_test:
if frappe.in_test:
print("WARN: Cannot pull email. Sender same as recipient inbox")
raise SentEmailInInboxError

2
frappe/email/smtp.py
View file

@ -109,7 +109,7 @@ class SMTPServer:
frappe.request.after_response.add(self.quit)
elif frappe.job:
frappe.job.after_job.add(self.quit)
elif not frappe.flags.in_test:
elif not frappe.in_test:
# Console?
import atexit

21
frappe/hooks.py
View file

@ -56,17 +56,12 @@ email_css = ["email.bundle.css"]
website_route_rules = [
{"from_route": "/blog/<category>", "to_route": "Blog Post"},
{"from_route": "/kb/<category>", "to_route": "Help Article"},
{"from_route": "/newsletters", "to_route": "Newsletter"},
{"from_route": "/profile", "to_route": "me"},
{"from_route": "/app/<path:app_path>", "to_route": "app"},
]
website_redirects = [
{"source": r"/desk(.*)", "target": r"/app\1"},
{
"source": "/.well-known/openid-configuration",
"target": "/api/method/frappe.integrations.oauth2.openid_configuration",
},
]
base_template = "templates/base.html"
@ -225,9 +220,7 @@ scheduler_events = {
"frappe.monitor.flush",
"frappe.integrations.doctype.google_calendar.google_calendar.sync",
],
"hourly": [
"frappe.email.doctype.newsletter.newsletter.send_scheduled_email",
],
"hourly": [],
# Maintenance queue happen roughly once an hour but don't align with wall-clock time of *:00
# Use these for when you don't care about when the job runs but just need some guarantee for
# frequency.
@ -248,31 +241,21 @@ scheduler_events = {
],
"daily_long": [],
"daily_maintenance": [
"frappe.integrations.doctype.dropbox_settings.dropbox_settings.take_backups_daily",
"frappe.integrations.doctype.s3_backup_settings.s3_backup_settings.take_backups_daily",
"frappe.integrations.doctype.google_drive.google_drive.daily_backup",
"frappe.email.doctype.auto_email_report.auto_email_report.send_daily",
"frappe.desk.notifications.clear_notifications",
"frappe.sessions.clear_expired_sessions",
"frappe.website.doctype.personal_data_deletion_request.personal_data_deletion_request.remove_unverified_record",
"frappe.integrations.doctype.google_contacts.google_contacts.sync",
"frappe.automation.doctype.auto_repeat.auto_repeat.make_auto_repeat_entry",
"frappe.core.doctype.log_settings.log_settings.run_log_clean_up",
],
"weekly_long": [
"frappe.integrations.doctype.dropbox_settings.dropbox_settings.take_backups_weekly",
"frappe.integrations.doctype.s3_backup_settings.s3_backup_settings.take_backups_weekly",
"frappe.desk.form.document_follow.send_weekly_updates",
"frappe.utils.change_log.check_for_update",
"frappe.integrations.doctype.google_drive.google_drive.weekly_backup",
"frappe.desk.doctype.changelog_feed.changelog_feed.fetch_changelog_feed",
],
"monthly": [
"frappe.email.doctype.auto_email_report.auto_email_report.send_monthly",
],
"monthly_long": [
"frappe.integrations.doctype.s3_backup_settings.s3_backup_settings.take_backups_monthly"
],
}
sounds = [
@ -372,7 +355,6 @@ global_search_doctypes = {
{"doctype": "Dashboard"},
{"doctype": "Country"},
{"doctype": "Currency"},
{"doctype": "Newsletter"},
{"doctype": "Letter Head"},
{"doctype": "Workflow"},
{"doctype": "Web Page"},
@ -431,6 +413,7 @@ before_request = [
"frappe.recorder.record",
"frappe.monitor.start",
"frappe.rate_limiter.apply",
"frappe.integrations.oauth2.set_cors_for_privileged_requests",
]
after_request = [

1
frappe/installer.py
View file

@ -447,6 +447,7 @@ def _delete_modules(modules: list[str], dry_run: bool) -> list[str]:
if not dry_run:
if doctype.issingle:
frappe.delete_doc(doctype.name, doctype.name, ignore_on_trash=True, force=True)
frappe.delete_doc("DocType", doctype.name, ignore_on_trash=True, force=True)
else:
drop_doctypes.append(doctype.name)

70
frappe/integrations/README.md Normal file
View file

@ -0,0 +1,70 @@
# Integrations
## OAuth 2
Frappe Framework uses [`oauthlib`](https://github.com/oauthlib/oauthlib) to manage OAuth2 requirements. A Frappe instance can function as all of these:
1. **Resource Server**: contains resources, for example the data in your DocTypes.
2. **Authorization Server**: server that issues tokens to access some resource.
3. **Client**: app that requires access to some resource on a resource server.
DocTypes pertaining to the above roles:
1. **Common**
- **OAuth Settings**: allows configuring certain OAuth features pertaining to the three roles.
2. **Authorization Server**
- **OAuth Client**: keeps records of _clients_ registered with the frappe instance.
- **OAuth Bearer Token**: tokens given out to registered _clients_ are maintained here.
- **OAuth Authorization Code**: keeps track of OAuth codes a client responds with in exchange for a token.
- **OAuth Provider Settings**: allows skipping authorization. `[DEPRECATED]` use **OAuth Settings** instead.
3. **Client**
- **Connected App**: keeps records of _authorization servers_ against whom this frappe instance is registered as a _client_ so some resource can be accessed. Eg. a users Google Drive account.
- **Social Key Login**: similar to **Connected App**, but for the purpose of logging into the frappe instance. Eg. a users Google account to enable "Login with Google".
- **Token Cache**: tokens received by the Frappe instance when accessing a **Connected App**.
### Features
Additional features over `oauthlib` that have implemented in the Framework:
- **Dynamic Client Registration**: allows a client to register itself without manual configuration by the resource owner. [RFC7591](https://datatracker.ietf.org/doc/html/rfc7591)
- **Authorization Server Metadata Discovery**: allows a client to view the instance's auth server (itself) metadata such as auth end points. [RFC8414](https://datatracker.ietf.org/doc/html/rfc8414)
- **Resource Server Metadata Discovery**: allows a client to view the instance's resource server metadata such as documentation, auth servers, etc. [RFC9728](https://datatracker.ietf.org/doc/html/rfc9728)
### Additional Docs
Documentation of various OAuth2 features:
1. [How to setup OAuth 2?](https://docs.frappe.io/framework/user/en/guides/integration/how_to_set_up_oauth)
2. [OAuth 2](https://docs.frappe.io/framework/user/en/guides/integration/rest_api/oauth-2)
3. [Token Based Authentication](https://docs.frappe.io/framework/user/en/guides/integration/rest_api/token_based_authentication)
4. [Using Frappe as OAuth Service](https://docs.frappe.io/framework/user/en/using_frappe_as_oauth_service)
5. [Social Login Key](https://docs.frappe.io/framework/user/en/guides/integration/social_login_key)
6. [Connected App](https://docs.frappe.io/framework/user/en/guides/app-development/connected-app)
> [!WARNING]
>
> Some of these might be outdated, it is always recommended to check the code
> when in doubt.
### OAuth Settings
A Single doctype that allows configuring OAuth2 related features. It is
recommended to open the DocType page itself as each field and section has a
sufficiently descriptive help text.
The settings allow toggling the following features:
- Authorization check when active token is present using the _Skip Authorization_ field. _**Note**: Keep this unchecked in production._
- **Authorization Server Metadata Discovery**: by toggling the _Show Auth Server Metadata_ field.
- **Dynamic Client Registration**: by toggling the _Enable Dynamic Client Registration_ field.
- **Resource Server Metadata Discovery**: by toggling the _Show Protected Resource Metadata_.
The remaining fields (in the **Resource** section) are used only when responding to requests on `/.well-known/oauth-protected-resource`
> **Regarding Public Clients**
>
> Public clients, for example an SPA, have restricted access by default. This
> restriction is applied by use of CORS.
>
> To side-step this restriction for certain trusted clients, you may add their
> hostnames to the **Allowed Public Client Origins** field.

0
frappe/integrations/doctype/dropbox_settings/__init__.py
View file

47
frappe/integrations/doctype/dropbox_settings/dropbox_settings.js
View file

@ -1,47 +0,0 @@
// Copyright (c) 2016, Frappe Technologies and contributors
// For license information, please see license.txt
frappe.ui.form.on("Dropbox Settings", {
refresh: function (frm) {
frm.toggle_display(
["app_access_key", "app_secret_key"],
!frm.doc.__onload?.dropbox_setup_via_site_config
);
frm.events.take_backup(frm);
},
are_keys_present: function (frm) {
return (
(frm.doc.app_access_key && frm.doc.app_secret_key) ||
frm.doc.__onload?.dropbox_setup_via_site_config
);
},
allow_dropbox_access: function (frm) {
if (!frm.events.are_keys_present(frm)) {
frappe.msgprint(__("App Access Key and/or Secret Key are not present."));
return;
}
frappe.call({
method: "frappe.integrations.doctype.dropbox_settings.dropbox_settings.get_dropbox_authorize_url",
freeze: true,
callback: function (r) {
if (!r.exc) {
window.open(r.message.auth_url);
}
},
});
},
take_backup: function (frm) {
if (frm.doc.enabled && (frm.doc.dropbox_refresh_token || frm.doc.dropbox_access_token)) {
frm.add_custom_button(__("Take Backup Now"), function () {
frappe.call({
method: "frappe.integrations.doctype.dropbox_settings.dropbox_settings.take_backup",
freeze: true,
});
});
}
},
});

126
frappe/integrations/doctype/dropbox_settings/dropbox_settings.json
View file

@ -1,126 +0,0 @@
{
"actions": [],
"creation": "2016-09-21 10:12:57.399174",
"doctype": "DocType",
"document_type": "System",
"editable_grid": 1,
"engine": "InnoDB",
"field_order": [
"enabled",
"send_notifications_to",
"send_email_for_successful_backup",
"backup_frequency",
"limit_no_of_backups",
"no_of_backups",
"file_backup",
"app_access_key",
"app_secret_key",
"allow_dropbox_access",
"dropbox_refresh_token",
"dropbox_access_token"
],
"fields": [
{
"default": "0",
"fieldname": "enabled",
"fieldtype": "Check",
"label": "Enabled"
},
{
"fieldname": "send_notifications_to",
"fieldtype": "Data",
"in_list_view": 1,
"label": "Send Notifications To",
"reqd": 1
},
{
"default": "1",
"description": "Note: By default emails for failed backups are sent.",
"fieldname": "send_email_for_successful_backup",
"fieldtype": "Check",
"label": "Send Email for Successful Backup"
},
{
"fieldname": "backup_frequency",
"fieldtype": "Select",
"in_list_view": 1,
"label": "Backup Frequency",
"options": "\nDaily\nWeekly",
"reqd": 1
},
{
"default": "0",
"fieldname": "limit_no_of_backups",
"fieldtype": "Check",
"label": "Limit Number of DB Backups"
},
{
"default": "5",
"depends_on": "eval:doc.limit_no_of_backups",
"fieldname": "no_of_backups",
"fieldtype": "Int",
"label": "Number of DB Backups"
},
{
"default": "1",
"fieldname": "file_backup",
"fieldtype": "Check",
"label": "File Backup"
},
{
"fieldname": "app_access_key",
"fieldtype": "Data",
"label": "App Access Key"
},
{
"fieldname": "app_secret_key",
"fieldtype": "Password",
"label": "App Secret Key"
},
{
"fieldname": "allow_dropbox_access",
"fieldtype": "Button",
"label": "Allow Dropbox Access"
},
{
"fieldname": "dropbox_refresh_token",
"fieldtype": "Password",
"hidden": 1,
"label": "Dropbox Refresh Token",
"no_copy": 1,
"read_only": 1
},
{
"fieldname": "dropbox_access_token",
"fieldtype": "Password",
"hidden": 1,
"label": "Dropbox Access Token"
}
],
"in_create": 1,
"issingle": 1,
"links": [],
"modified": "2024-03-23 16:03:23.176690",
"modified_by": "Administrator",
"module": "Integrations",
"name": "Dropbox Settings",
"owner": "Administrator",
"permissions": [
{
"create": 1,
"delete": 1,
"email": 1,
"export": 1,
"print": 1,
"read": 1,
"role": "System Manager",
"share": 1,
"write": 1
}
],
"read_only": 1,
"sort_field": "creation",
"sort_order": "DESC",
"states": [],
"track_changes": 1
}

378
frappe/integrations/doctype/dropbox_settings/dropbox_settings.py
View file

@ -1,378 +0,0 @@
# Copyright (c) 2015, Frappe Technologies and contributors
# License: MIT. See LICENSE
import os
from urllib.parse import parse_qs, urlparse
import dropbox
from rq.timeouts import JobTimeoutException
import frappe
from frappe import _
from frappe.integrations.offsite_backup_utils import (
get_chunk_site,
get_latest_backup_file,
send_email,
validate_file_size,
)
from frappe.model.document import Document
from frappe.utils import cint, encode, get_backups_path, get_files_path, get_request_site_address
from frappe.utils.background_jobs import enqueue
from frappe.utils.backups import new_backup
ignore_list = [".DS_Store"]
class DropboxSettings(Document):
# begin: auto-generated types
# This code is auto-generated. Do not modify anything in this block.
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from frappe.types import DF
app_access_key: DF.Data | None
app_secret_key: DF.Password | None
backup_frequency: DF.Literal["", "Daily", "Weekly"]
dropbox_access_token: DF.Password | None
dropbox_refresh_token: DF.Password | None
enabled: DF.Check
file_backup: DF.Check
limit_no_of_backups: DF.Check
no_of_backups: DF.Int
send_email_for_successful_backup: DF.Check
send_notifications_to: DF.Data
# end: auto-generated types
def onload(self):
if not self.app_access_key and frappe.conf.dropbox_access_key:
self.set_onload("dropbox_setup_via_site_config", 1)
def validate(self):
if self.enabled and self.limit_no_of_backups and self.no_of_backups < 1:
frappe.throw(_("Number of DB backups cannot be less than 1"))
@frappe.whitelist()
def take_backup():
"""Enqueue longjob for taking backup to dropbox"""
enqueue(
"frappe.integrations.doctype.dropbox_settings.dropbox_settings.take_backup_to_dropbox",
queue="long",
timeout=1500,
)
frappe.msgprint(_("Queued for backup. It may take a few minutes to an hour."))
def take_backups_daily():
take_backups_if("Daily")
def take_backups_weekly():
take_backups_if("Weekly")
def take_backups_if(freq):
if frappe.db.get_single_value("Dropbox Settings", "backup_frequency") == freq:
take_backup_to_dropbox()
def take_backup_to_dropbox(retry_count=0, upload_db_backup=True):
did_not_upload, error_log = [], []
try:
if cint(frappe.db.get_single_value("Dropbox Settings", "enabled")):
validate_file_size()
did_not_upload, error_log = backup_to_dropbox(upload_db_backup)
if did_not_upload:
raise Exception
if cint(frappe.db.get_single_value("Dropbox Settings", "send_email_for_successful_backup")):
send_email(True, "Dropbox", "Dropbox Settings", "send_notifications_to")
except JobTimeoutException:
if retry_count < 2:
args = {
"retry_count": retry_count + 1,
"upload_db_backup": False, # considering till worker timeout db backup is uploaded
}
enqueue(
"frappe.integrations.doctype.dropbox_settings.dropbox_settings.take_backup_to_dropbox",
queue="long",
timeout=1500,
**args,
)
except Exception:
if isinstance(error_log, str):
error_message = error_log + "\n" + frappe.get_traceback()
else:
file_and_error = [" - ".join(f) for f in zip(did_not_upload, error_log, strict=False)]
error_message = "\n".join(file_and_error) + "\n" + frappe.get_traceback()
send_email(False, "Dropbox", "Dropbox Settings", "send_notifications_to", error_message)
def backup_to_dropbox(upload_db_backup=True):
# upload database
dropbox_settings = get_dropbox_settings()
dropbox_client = get_dropbox_client(dropbox_settings)
if upload_db_backup:
if frappe.flags.create_new_backup:
backup = new_backup(ignore_files=True)
filename = os.path.join(get_backups_path(), os.path.basename(backup.backup_path_db))
site_config = os.path.join(get_backups_path(), os.path.basename(backup.backup_path_conf))
else:
filename, site_config = get_latest_backup_file()
upload_file_to_dropbox(filename, "/database", dropbox_client)
upload_file_to_dropbox(site_config, "/database", dropbox_client)
# delete older databases
if dropbox_settings["no_of_backups"]:
delete_older_backups(dropbox_client, "/database", dropbox_settings["no_of_backups"])
# upload files to files folder
did_not_upload = []
error_log = []
if dropbox_settings["file_backup"]:
upload_from_folder(get_files_path(), 0, "/files", dropbox_client, did_not_upload, error_log)
upload_from_folder(
get_files_path(is_private=1), 1, "/private/files", dropbox_client, did_not_upload, error_log
)
return did_not_upload, list(set(error_log))
def upload_from_folder(path, is_private, dropbox_folder, dropbox_client, did_not_upload, error_log):
if not os.path.exists(path):
return
if is_fresh_upload():
response = get_uploaded_files_meta(dropbox_folder, dropbox_client)
else:
response = frappe._dict({"entries": []})
path = str(path)
for f in frappe.get_all(
"File",
filters={"is_folder": 0, "is_private": is_private, "uploaded_to_dropbox": 0},
fields=["file_url", "name", "file_name"],
):
if not f.file_url:
continue
filename = f.file_url.rsplit("/", 1)[-1]
filepath = os.path.join(path, filename)
if filename in ignore_list:
continue
found = False
for file_metadata in response.entries:
try:
if os.path.basename(filepath) == file_metadata.name and os.stat(
encode(filepath)
).st_size == int(file_metadata.size):
found = True
update_file_dropbox_status(f.name)
break
except Exception:
error_log.append(frappe.get_traceback())
if not found:
try:
upload_file_to_dropbox(filepath, dropbox_folder, dropbox_client)
update_file_dropbox_status(f.name)
except Exception:
did_not_upload.append(filepath)
error_log.append(frappe.get_traceback())
def upload_file_to_dropbox(filename, folder, dropbox_client):
"""upload files with chunk of 15 mb to reduce session append calls"""
if not os.path.exists(filename):
return
create_folder_if_not_exists(folder, dropbox_client)
file_size = os.path.getsize(encode(filename))
chunk_size = get_chunk_site(file_size)
mode = dropbox.files.WriteMode.overwrite
f = open(encode(filename), "rb")
path = f"{folder}/{os.path.basename(filename)}"
try:
if file_size <= chunk_size:
dropbox_client.files_upload(f.read(), path, mode)
else:
upload_session_start_result = dropbox_client.files_upload_session_start(f.read(chunk_size))
cursor = dropbox.files.UploadSessionCursor(
session_id=upload_session_start_result.session_id, offset=f.tell()
)
commit = dropbox.files.CommitInfo(path=path, mode=mode)
while f.tell() < file_size:
if (file_size - f.tell()) <= chunk_size:
dropbox_client.files_upload_session_finish(f.read(chunk_size), cursor, commit)
else:
dropbox_client.files_upload_session_append(
f.read(chunk_size), cursor.session_id, cursor.offset
)
cursor.offset = f.tell()
except dropbox.exceptions.ApiError as e:
if isinstance(e.error, dropbox.files.UploadError):
error = f"File Path: {path}\n"
error += frappe.get_traceback()
frappe.log_error(error)
else:
raise
def create_folder_if_not_exists(folder, dropbox_client):
try:
dropbox_client.files_get_metadata(folder)
except dropbox.exceptions.ApiError as e:
# folder not found
if isinstance(e.error, dropbox.files.GetMetadataError):
dropbox_client.files_create_folder(folder)
else:
raise
def update_file_dropbox_status(file_name):
frappe.db.set_value("File", file_name, "uploaded_to_dropbox", 1, update_modified=False)
def is_fresh_upload():
file_name = frappe.db.get_value("File", {"uploaded_to_dropbox": 1}, "name")
return not file_name
def get_uploaded_files_meta(dropbox_folder, dropbox_client):
try:
return dropbox_client.files_list_folder(dropbox_folder)
except dropbox.exceptions.ApiError as e:
# folder not found
if isinstance(e.error, dropbox.files.ListFolderError):
return frappe._dict({"entries": []})
raise
def get_dropbox_client(dropbox_settings):
dropbox_client = dropbox.Dropbox(
oauth2_access_token=dropbox_settings["access_token"],
oauth2_refresh_token=dropbox_settings["refresh_token"],
app_key=dropbox_settings["app_key"],
app_secret=dropbox_settings["app_secret"],
timeout=None,
)
# checking if the access token has expired
dropbox_client.files_list_folder("")
if dropbox_settings["access_token"] != dropbox_client._oauth2_access_token:
set_dropbox_token(dropbox_client._oauth2_access_token)
return dropbox_client
def get_dropbox_settings(redirect_uri=False):
# NOTE: access token is kept for legacy dropbox apps
settings = frappe.get_doc("Dropbox Settings")
app_details = {
"app_key": settings.app_access_key or frappe.conf.dropbox_access_key,
"app_secret": settings.get_password(fieldname="app_secret_key", raise_exception=False)
if settings.app_secret_key
else frappe.conf.dropbox_secret_key,
"refresh_token": settings.get_password("dropbox_refresh_token", raise_exception=False),
"access_token": settings.get_password("dropbox_access_token", raise_exception=False),
"file_backup": settings.file_backup,
"no_of_backups": settings.no_of_backups if settings.limit_no_of_backups else None,
}
if redirect_uri:
app_details.update(
{
"redirect_uri": get_request_site_address(True)
+ "/api/method/frappe.integrations.doctype.dropbox_settings.dropbox_settings.dropbox_auth_finish"
}
)
if not (app_details["app_key"] and app_details["app_secret"]):
raise Exception(_("Please set Dropbox access keys in site config or doctype"))
return app_details
def delete_older_backups(dropbox_client, folder_path, to_keep):
res = dropbox_client.files_list_folder(path=folder_path)
files = [f for f in res.entries if isinstance(f, dropbox.files.FileMetadata) and "sql" in f.name]
if len(files) <= to_keep:
return
files.sort(key=lambda item: item.client_modified, reverse=True)
for f in files[to_keep:]:
dropbox_client.files_delete(os.path.join(folder_path, f.name))
@frappe.whitelist()
def get_dropbox_authorize_url():
app_details = get_dropbox_settings(redirect_uri=True)
dropbox_oauth_flow = dropbox.DropboxOAuth2Flow(
consumer_key=app_details["app_key"],
redirect_uri=app_details["redirect_uri"],
session={},
csrf_token_session_key="dropbox-auth-csrf-token",
consumer_secret=app_details["app_secret"],
token_access_type="offline",
)
auth_url = dropbox_oauth_flow.start()
return {"auth_url": auth_url, "args": parse_qs(urlparse(auth_url).query)}
@frappe.whitelist()
def dropbox_auth_finish():
app_details = get_dropbox_settings(redirect_uri=True)
callback = frappe.form_dict
close = '<p class="text-muted">' + _("Please close this window") + "</p>"
if not callback.state or not callback.code:
frappe.respond_as_web_page(
_("Dropbox Setup"),
_("Illegal Access Token. Please try again") + close,
indicator_color="red",
http_status_code=frappe.AuthenticationError.http_status_code,
)
return
dropbox_oauth_flow = dropbox.DropboxOAuth2Flow(
consumer_key=app_details["app_key"],
redirect_uri=app_details["redirect_uri"],
session={"dropbox-auth-csrf-token": callback.state},
csrf_token_session_key="dropbox-auth-csrf-token",
consumer_secret=app_details["app_secret"],
)
token = dropbox_oauth_flow.finish({"state": callback.state, "code": callback.code})
set_dropbox_token(token.access_token, token.refresh_token)
frappe.local.response["type"] = "redirect"
frappe.local.response["location"] = "/app/dropbox-settings"
def set_dropbox_token(access_token, refresh_token=None):
# NOTE: used doc object instead of db.set_value so that password field is set properly
dropbox_settings = frappe.get_single("Dropbox Settings")
dropbox_settings.dropbox_access_token = access_token
if refresh_token:
dropbox_settings.dropbox_refresh_token = refresh_token
dropbox_settings.save()
frappe.db.commit()

8
frappe/integrations/doctype/dropbox_settings/test_dropbox_settings.py
View file

@ -1,8 +0,0 @@
# Copyright (c) 2019, Frappe Technologies and Contributors
# License: MIT. See LICENSE
# import frappe
from frappe.tests import IntegrationTestCase
class TestDropboxSettings(IntegrationTestCase):
pass

0
frappe/integrations/doctype/google_drive/__init__.py
View file

71
frappe/integrations/doctype/google_drive/google_drive.js
View file

@ -1,71 +0,0 @@
// Copyright (c) 2019, Frappe Technologies and contributors
// For license information, please see license.txt
frappe.ui.form.on("Google Drive", {
refresh: function (frm) {
if (!frm.doc.enable) {
frm.dashboard.set_headline(
__("To use Google Drive, enable {0}.", [
`<a href='/app/google-settings'>${__("Google Settings")}</a>`,
])
);
}
frappe.realtime.on("upload_to_google_drive", (data) => {
if (data.progress) {
const progress_title = __("Uploading to Google Drive");
frm.dashboard.show_progress(
progress_title,
(data.progress / data.total) * 100,
data.message
);
if (data.progress === data.total) {
frm.dashboard.hide_progress(progress_title);
}
}
});
if (frm.doc.enable && frm.doc.refresh_token) {
let sync_button = frm.add_custom_button(__("Take Backup"), function () {
frappe.show_alert({
indicator: "green",
message: __("Backing up to Google Drive."),
});
frappe
.call({
method: "frappe.integrations.doctype.google_drive.google_drive.take_backup",
btn: sync_button,
})
.then((r) => {
frappe.msgprint(r.message);
});
});
}
if (frm.doc.enable && frm.doc.backup_folder_name && !frm.doc.refresh_token) {
frm.dashboard.set_headline(
__(
"Click on <b>Authorize Google Drive Access</b> to authorize Google Drive Access."
)
);
}
if (frm.doc.enable && frm.doc.refresh_token && frm.doc.authorization_code) {
frm.page.set_indicator("Authorized", "green");
}
},
authorize_google_drive_access: function (frm) {
frappe.call({
method: "frappe.integrations.doctype.google_drive.google_drive.authorize_access",
args: {
reauthorize: frm.doc.authorization_code ? 1 : 0,
},
callback: function (r) {
if (!r.exc) {
frm.save();
window.open(r.message.url);
}
},
});
},
});

126
frappe/integrations/doctype/google_drive/google_drive.json
View file

@ -1,126 +0,0 @@
{
"actions": [],
"creation": "2019-08-13 17:24:05.470876",
"doctype": "DocType",
"engine": "InnoDB",
"field_order": [
"enable",
"google_drive_section",
"backup_folder_name",
"frequency",
"email",
"send_email_for_successful_backup",
"file_backup",
"authorize_google_drive_access",
"column_break_5",
"backup_folder_id",
"last_backup_on",
"refresh_token",
"authorization_code"
],
"fields": [
{
"default": "0",
"fieldname": "enable",
"fieldtype": "Check",
"label": "Enable"
},
{
"fieldname": "backup_folder_name",
"fieldtype": "Data",
"in_list_view": 1,
"label": "Backup Folder Name",
"reqd": 1
},
{
"depends_on": "eval:!doc.__islocal",
"fieldname": "authorize_google_drive_access",
"fieldtype": "Button",
"label": "Authorize Google Drive Access"
},
{
"fieldname": "column_break_5",
"fieldtype": "Column Break"
},
{
"fieldname": "backup_folder_id",
"fieldtype": "Data",
"label": "Backup Folder ID",
"read_only": 1
},
{
"fieldname": "frequency",
"fieldtype": "Select",
"label": "Frequency",
"options": "\nDaily\nWeekly",
"reqd": 1
},
{
"fieldname": "refresh_token",
"fieldtype": "Data",
"hidden": 1,
"label": "Refresh Token"
},
{
"fieldname": "authorization_code",
"fieldtype": "Data",
"hidden": 1,
"label": "Authorization Code"
},
{
"fieldname": "last_backup_on",
"fieldtype": "Datetime",
"label": "Last Backup On",
"read_only": 1
},
{
"default": "0",
"description": "Note: By default emails for failed backups are sent.",
"fieldname": "send_email_for_successful_backup",
"fieldtype": "Check",
"label": "Send Email for Successful backup"
},
{
"default": "0",
"fieldname": "file_backup",
"fieldtype": "Check",
"label": "File Backup"
},
{
"depends_on": "enable",
"fieldname": "google_drive_section",
"fieldtype": "Section Break",
"label": "Google Drive"
},
{
"fieldname": "email",
"fieldtype": "Data",
"label": "Send Notification To",
"options": "Email",
"reqd": 1
}
],
"issingle": 1,
"links": [],
"modified": "2024-03-23 16:03:26.999110",
"modified_by": "Administrator",
"module": "Integrations",
"name": "Google Drive",
"owner": "Administrator",
"permissions": [
{
"create": 1,
"delete": 1,
"email": 1,
"print": 1,
"read": 1,
"role": "System Manager",
"share": 1,
"write": 1
}
],
"sort_field": "creation",
"sort_order": "DESC",
"states": [],
"track_changes": 1
}

229
frappe/integrations/doctype/google_drive/google_drive.py
View file

@ -1,229 +0,0 @@
# Copyright (c) 2019, Frappe Technologies and contributors
# License: MIT. See LICENSE
import os
from urllib.parse import quote
from apiclient.http import MediaFileUpload
from googleapiclient.errors import HttpError
import frappe
from frappe import _
from frappe.integrations.google_oauth import GoogleOAuth
from frappe.integrations.offsite_backup_utils import (
get_latest_backup_file,
send_email,
validate_file_size,
)
from frappe.model.document import Document
from frappe.utils import get_backups_path, get_bench_path
from frappe.utils.background_jobs import enqueue
from frappe.utils.backups import new_backup
class GoogleDrive(Document):
# begin: auto-generated types
# This code is auto-generated. Do not modify anything in this block.
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from frappe.types import DF
authorization_code: DF.Data | None
backup_folder_id: DF.Data | None
backup_folder_name: DF.Data
email: DF.Data
enable: DF.Check
file_backup: DF.Check
frequency: DF.Literal["", "Daily", "Weekly"]
last_backup_on: DF.Datetime | None
refresh_token: DF.Data | None
send_email_for_successful_backup: DF.Check
# end: auto-generated types
def validate(self):
doc_before_save = self.get_doc_before_save()
if doc_before_save and doc_before_save.backup_folder_name != self.backup_folder_name:
self.backup_folder_id = ""
def get_access_token(self):
if not self.refresh_token:
button_label = frappe.bold(_("Allow Google Drive Access"))
raise frappe.ValidationError(_("Click on {0} to generate Refresh Token.").format(button_label))
oauth_obj = GoogleOAuth("drive")
r = oauth_obj.refresh_access_token(
self.get_password(fieldname="refresh_token", raise_exception=False)
)
return r.get("access_token")
@frappe.whitelist(methods=["POST"])
def authorize_access(reauthorize=False, code=None):
"""
If no Authorization code get it from Google and then request for Refresh Token.
Google Contact Name is set to flags to set_value after Authorization Code is obtained.
"""
oauth_code = frappe.db.get_single_value("Google Drive", "authorization_code") if not code else code
oauth_obj = GoogleOAuth("drive")
if not oauth_code or reauthorize:
if reauthorize:
frappe.db.set_single_value("Google Drive", "backup_folder_id", "")
return oauth_obj.get_authentication_url(
{
"redirect": f"/app/Form/{quote('Google Drive')}",
},
)
r = oauth_obj.authorize(oauth_code)
frappe.db.set_single_value(
"Google Drive",
{"authorization_code": oauth_code, "refresh_token": r.get("refresh_token")},
)
def get_google_drive_object():
"""Return an object of Google Drive."""
account = frappe.get_doc("Google Drive")
oauth_obj = GoogleOAuth("drive")
google_drive = oauth_obj.get_google_service_object(
account.get_access_token(),
account.get_password(fieldname="indexing_refresh_token", raise_exception=False),
)
return google_drive, account
def check_for_folder_in_google_drive():
"""Checks if folder exists in Google Drive else create it."""
def _create_folder_in_google_drive(google_drive, account):
file_metadata = {
"name": account.backup_folder_name,
"mimeType": "application/vnd.google-apps.folder",
}
try:
folder = google_drive.files().create(body=file_metadata, fields="id").execute()
frappe.db.set_single_value("Google Drive", "backup_folder_id", folder.get("id"))
frappe.db.commit()
except HttpError as e:
frappe.throw(
_("Google Drive - Could not create folder in Google Drive - Error Code {0}").format(e)
)
google_drive, account = get_google_drive_object()
if account.backup_folder_id:
return
backup_folder_exists = False
try:
google_drive_folders = (
google_drive.files().list(q="mimeType='application/vnd.google-apps.folder'").execute()
)
except HttpError as e:
frappe.throw(_("Google Drive - Could not find folder in Google Drive - Error Code {0}").format(e))
for f in google_drive_folders.get("files"):
if f.get("name") == account.backup_folder_name:
frappe.db.set_single_value("Google Drive", "backup_folder_id", f.get("id"))
frappe.db.commit()
backup_folder_exists = True
break
if not backup_folder_exists:
_create_folder_in_google_drive(google_drive, account)
@frappe.whitelist()
def take_backup():
"""Enqueue longjob for taking backup to Google Drive"""
enqueue(
"frappe.integrations.doctype.google_drive.google_drive.upload_system_backup_to_google_drive",
queue="long",
timeout=1500,
)
frappe.msgprint(_("Queued for backup. It may take a few minutes to an hour."))
def upload_system_backup_to_google_drive():
"""
Upload system backup to Google Drive
"""
# Get Google Drive Object
google_drive, account = get_google_drive_object()
# Check if folder exists in Google Drive
check_for_folder_in_google_drive()
account.load_from_db()
validate_file_size()
if frappe.flags.create_new_backup:
set_progress(1, _("Backing up Data."))
backup = new_backup()
file_urls = []
file_urls.append(backup.backup_path_db)
file_urls.append(backup.backup_path_conf)
if account.file_backup:
file_urls.append(backup.backup_path_files)
file_urls.append(backup.backup_path_private_files)
else:
file_urls = get_latest_backup_file(with_files=account.file_backup)
for fileurl in file_urls:
if not fileurl:
continue
file_metadata = {"name": os.path.basename(fileurl), "parents": [account.backup_folder_id]}
try:
media = MediaFileUpload(
get_absolute_path(filename=fileurl), mimetype="application/gzip", resumable=True
)
except OSError as e:
frappe.throw(_("Google Drive - Could not locate - {0}").format(e))
try:
set_progress(2, _("Uploading backup to Google Drive."))
google_drive.files().create(body=file_metadata, media_body=media, fields="id").execute()
except HttpError as e:
send_email(False, "Google Drive", "Google Drive", "email", error_status=e)
set_progress(3, _("Uploading successful."))
frappe.db.set_single_value("Google Drive", "last_backup_on", frappe.utils.now_datetime())
send_email(True, "Google Drive", "Google Drive", "email")
return _("Google Drive Backup Successful.")
def daily_backup():
drive_settings = frappe.db.get_singles_dict("Google Drive", cast=True)
if drive_settings.enable and drive_settings.frequency == "Daily":
upload_system_backup_to_google_drive()
def weekly_backup():
drive_settings = frappe.db.get_singles_dict("Google Drive", cast=True)
if drive_settings.enable and drive_settings.frequency == "Weekly":
upload_system_backup_to_google_drive()
def get_absolute_path(filename):
file_path = os.path.join(get_backups_path()[2:], os.path.basename(filename))
return f"{get_bench_path()}/sites/{file_path}"
def set_progress(progress, message):
frappe.publish_realtime(
"upload_to_google_drive",
dict(progress=progress, total=3, message=message),
user=frappe.session.user,
)

8
frappe/integrations/doctype/google_drive/test_google_drive.py
View file

@ -1,8 +0,0 @@
# Copyright (c) 2019, Frappe Technologies and Contributors
# License: MIT. See LICENSE
# import frappe
from frappe.tests import IntegrationTestCase
class TestGoogleDrive(IntegrationTestCase):
pass

114
frappe/integrations/doctype/oauth_client/oauth_client.json
View file

@ -7,19 +7,29 @@
"engine": "InnoDB",
"field_order": [
"client_id",
"app_name",
"user",
"allowed_roles",
"cb_1",
"client_secret",
"skip_authorization",
"sb_1",
"scopes",
"cb_3",
"redirect_uris",
"default_redirect_uri",
"skip_authorization",
"client_metadata_section",
"app_name",
"scopes",
"column_break_htfq",
"redirect_uris",
"section_break_ggiv",
"client_uri",
"software_id",
"tos_uri",
"contacts",
"column_break_ziii",
"logo_uri",
"software_version",
"policy_uri",
"sb_advanced",
"grant_type",
"token_endpoint_auth_method",
"cb_2",
"response_type"
],
@ -27,13 +37,13 @@
{
"fieldname": "client_id",
"fieldtype": "Data",
"label": "App Client ID",
"label": "Client ID",
"read_only": 1
},
{
"fieldname": "app_name",
"fieldtype": "Data",
"label": "App Name",
"label": "App Name (Client Name)",
"reqd": 1
},
{
@ -50,7 +60,7 @@
{
"fieldname": "client_secret",
"fieldtype": "Data",
"label": "App Client Secret",
"label": "Client Secret",
"read_only": 1
},
{
@ -60,10 +70,6 @@
"fieldtype": "Check",
"label": "Skip Authorization"
},
{
"fieldname": "sb_1",
"fieldtype": "Section Break"
},
{
"default": "all openid",
"description": "A list of resources which the Client App will have access to after the user allows it.<br> e.g. project",
@ -72,10 +78,6 @@
"label": "Scopes",
"reqd": 1
},
{
"fieldname": "cb_3",
"fieldtype": "Column Break"
},
{
"description": "URIs for receiving authorization code once the user allows access, as well as failure responses. Typically a REST endpoint exposed by the Client App.\n<br>e.g. http://hostname/api/method/frappe.integrations.oauth2_logins.login_via_facebook",
"fieldname": "redirect_uris",
@ -121,10 +123,85 @@
"fieldtype": "Table MultiSelect",
"label": "Allowed Roles",
"options": "OAuth Client Role"
},
{
"fieldname": "client_metadata_section",
"fieldtype": "Section Break",
"label": "Client Metadata"
},
{
"depends_on": "eval: doc.client_uri",
"description": "URL of a web page providing information about the client.",
"fieldname": "client_uri",
"fieldtype": "Data",
"label": "Client URI"
},
{
"fieldname": "column_break_htfq",
"fieldtype": "Column Break"
},
{
"depends_on": "eval: doc.client_uri",
"description": "URL that references a logo for the client.",
"fieldname": "logo_uri",
"fieldtype": "Data",
"label": "Logo URI"
},
{
"fieldname": "section_break_ggiv",
"fieldtype": "Section Break"
},
{
"depends_on": "eval: doc.software_id",
"description": "Unique ID assigned by the client developer used to identify the client software to be dynamically registered.\n<br>\n<b>Should remain same</b> across multiple versions or updates of the software.",
"fieldname": "software_id",
"fieldtype": "Data",
"label": "Software ID"
},
{
"fieldname": "column_break_ziii",
"fieldtype": "Column Break"
},
{
"depends_on": "eval: doc.software_version",
"description": "A version identifier string for the client software.\n<br>\nThe value of the should change on any update of the client software with the same Software ID.",
"fieldname": "software_version",
"fieldtype": "Data",
"label": "Software Version"
},
{
"depends_on": "eval: doc.tos_uri",
"description": "URL that points to a human-readable terms of service document for the client. Should be shown to end-user before authorizing.",
"fieldname": "tos_uri",
"fieldtype": "Data",
"label": "TOS URI"
},
{
"depends_on": "eval: doc.policy_uri",
"description": "URL that points to a human-readable policy document for the client. Should be shown to end-user before authorizing.",
"fieldname": "policy_uri",
"fieldtype": "Data",
"label": "Policy URI"
},
{
"depends_on": "eval: doc.contacts",
"description": "New lines separated list of strings representing ways to contact people responsible for this client, typically email addresses.",
"fieldname": "contacts",
"fieldtype": "Small Text",
"label": "Contacts"
},
{
"default": "Client Secret Basic",
"description": "Value of \"None\" implies a public client. In such a case Client Secret is not given to the client and token exchange makes use of PKCE.",
"fieldname": "token_endpoint_auth_method",
"fieldtype": "Select",
"label": "Token Endpoint Auth Method",
"options": "Client Secret Basic\nClient Secret Post\nNone"
}
],
"grid_page_length": 50,
"links": [],
"modified": "2024-04-29 12:07:07.946980",
"modified": "2025-07-04 14:07:36.146393",
"modified_by": "Administrator",
"module": "Integrations",
"name": "OAuth Client",
@ -143,6 +220,7 @@
"write": 1
}
],
"row_format": "Dynamic",
"sort_field": "creation",
"sort_order": "DESC",
"states": [],

27
frappe/integrations/doctype/oauth_client/oauth_client.py
View file

@ -1,7 +1,11 @@
# Copyright (c) 2015, Frappe Technologies and contributors
# License: MIT. See LICENSE
import datetime
import time
import frappe
import frappe.utils
from frappe import _
from frappe.model.document import Document
from frappe.permissions import SYSTEM_USER_ROLE
@ -21,12 +25,20 @@ class OAuthClient(Document):
app_name: DF.Data
client_id: DF.Data | None
client_secret: DF.Data | None
client_uri: DF.Data | None
contacts: DF.SmallText | None
default_redirect_uri: DF.Data
grant_type: DF.Literal["Authorization Code", "Implicit"]
logo_uri: DF.Data | None
policy_uri: DF.Data | None
redirect_uris: DF.Text | None
response_type: DF.Literal["Code", "Token"]
scopes: DF.Text
skip_authorization: DF.Check
software_id: DF.Data | None
software_version: DF.Data | None
token_endpoint_auth_method: DF.Literal["Client Secret Basic", "Client Secret Post", "None"]
tos_uri: DF.Data | None
user: DF.Link | None
# end: auto-generated types
@ -55,3 +67,18 @@ class OAuthClient(Document):
"""Returns true if session user is allowed to use this client."""
allowed_roles = {d.role for d in self.allowed_roles}
return bool(allowed_roles & set(frappe.get_roles()))
def is_public_client(self) -> bool:
return self.token_endpoint_auth_method == "None"
def client_id_issued_at(self) -> int:
"""Returns UNIX timestamp (seconds since epoch) of the client creation time."""
if isinstance(self.creation, datetime.datetime):
return int(self.creation.timestamp())
try:
d = datetime.datetime.fromisoformat(self.creation)
return int(d.timestamp())
except Exception:
return int(frappe.utils.now_datetime().timestamp())

7
frappe/integrations/doctype/oauth_provider_settings/oauth_provider_settings.py
View file

@ -19,10 +19,3 @@ class OAuthProviderSettings(Document):
# end: auto-generated types
pass
def get_oauth_settings():
"""Return OAuth settings."""
return frappe._dict(
{"skip_authorization": frappe.db.get_single_value("OAuth Provider Settings", "skip_authorization")}
)

0
frappe/email/doctype/newsletter/__init__.py → frappe/integrations/doctype/oauth_settings/__init__.py
View file

8
frappe/integrations/doctype/oauth_settings/oauth_settings.js Normal file
View file

@ -0,0 +1,8 @@
// Copyright (c) 2025, Frappe Technologies and contributors
// For license information, please see license.txt
// frappe.ui.form.on("OAuth Settings", {
// refresh(frm) {
// },
// });

166
frappe/integrations/doctype/oauth_settings/oauth_settings.json Normal file
View file

@ -0,0 +1,166 @@
{
"actions": [],
"allow_rename": 1,
"creation": "2025-07-03 12:04:14.759362",
"description": "A Frappe Framework instance can function as an OAuth Client, Resource, or Authorization server. This DocType contains settings related to all three.",
"doctype": "DocType",
"engine": "InnoDB",
"field_order": [
"authorization_tab",
"authorization_server_section",
"show_auth_server_metadata",
"skip_authorization",
"column_break_ogmd",
"enable_dynamic_client_registration",
"allowed_public_client_origins",
"resource_tab",
"config_section",
"show_protected_resource_metadata",
"column_break_wlfj",
"show_social_login_key_as_authorization_server",
"resource_server_section",
"resource_name",
"resource_policy_uri",
"column_break_zyte",
"resource_documentation",
"resource_tos_uri",
"scopes_supported"
],
"fields": [
{
"description": "These fields are used to provide resource server metadata to clients querying the \"well known protected resource\" end point.",
"fieldname": "resource_server_section",
"fieldtype": "Section Break",
"label": "Metadata"
},
{
"default": "Frappe Framework Application",
"description": "Human-readable name intended for display to the end user.",
"fieldname": "resource_name",
"fieldtype": "Data",
"label": "Resource Name"
},
{
"fieldname": "column_break_zyte",
"fieldtype": "Column Break"
},
{
"default": "https://docs.frappe.io/framework",
"description": "URL of a human-readable page with info that developers might need.",
"fieldname": "resource_documentation",
"fieldtype": "Data",
"label": "Resource Documentation"
},
{
"description": "URL of human-readable page with info on requirements about how the client can use the data.",
"fieldname": "resource_policy_uri",
"fieldtype": "Data",
"label": "Resource Policy URI"
},
{
"description": "URL of human-readable page with info about the protected resource's terms of service.",
"fieldname": "resource_tos_uri",
"fieldtype": "Data",
"label": "Resource TOS URI"
},
{
"fieldname": "authorization_server_section",
"fieldtype": "Section Break"
},
{
"default": "1",
"description": "Allows clients to fetch metadata from the <code>/.well-known/oauth-authorization-server</code> endpoint. Reference: <a href=\"https://datatracker.ietf.org/doc/html/rfc8414\">RFC8414</a>",
"fieldname": "show_auth_server_metadata",
"fieldtype": "Check",
"label": "Show Auth Server Metadata"
},
{
"default": "1",
"description": "Allows clients to fetch metadata from the <code>/.well-known/oauth-protected-resource</code> endpoint. Reference: <a href=\"https://datatracker.ietf.org/doc/html/rfc9728#name-protected-resource-metadata\">RFC9728</a>",
"fieldname": "show_protected_resource_metadata",
"fieldtype": "Check",
"label": "Show Protected Resource Metadata"
},
{
"description": "New line separated list of scope values.",
"fieldname": "scopes_supported",
"fieldtype": "Small Text",
"label": "Scopes Supported"
},
{
"default": "1",
"description": "Allows clients to register themselves without manual intervention. Registration creates a <b>OAuth Client</b> entry. Reference: <a href=\"https://datatracker.ietf.org/doc/html/rfc7591\">RFC7591</a>",
"fieldname": "enable_dynamic_client_registration",
"fieldtype": "Check",
"label": "Enable Dynamic Client Registration"
},
{
"default": "0",
"description": "Allows skipping authorization if a user has active tokens.",
"fieldname": "skip_authorization",
"fieldtype": "Check",
"label": "Skip Authorization"
},
{
"default": "0",
"description": "Allows enabled Social Login Key Base URL to be shown as authorization server.",
"fieldname": "show_social_login_key_as_authorization_server",
"fieldtype": "Check",
"label": "Show Social Login Key as Authorization Server"
},
{
"fieldname": "column_break_ogmd",
"fieldtype": "Column Break"
},
{
"fieldname": "authorization_tab",
"fieldtype": "Tab Break",
"label": "Authorization"
},
{
"fieldname": "resource_tab",
"fieldtype": "Tab Break",
"label": "Resource"
},
{
"fieldname": "config_section",
"fieldtype": "Section Break",
"label": "Config"
},
{
"fieldname": "column_break_wlfj",
"fieldtype": "Column Break"
},
{
"description": "New line separated list of allowed public client URLs (eg <code>https://frappe.io</code>), or <code>*</code> to accept all.\n<br>\nPublic clients are restricted by default.",
"fieldname": "allowed_public_client_origins",
"fieldtype": "Small Text",
"label": "Allowed Public Client Origins"
}
],
"grid_page_length": 50,
"index_web_pages_for_search": 1,
"issingle": 1,
"links": [],
"modified": "2025-07-04 15:01:45.453238",
"modified_by": "Administrator",
"module": "Integrations",
"name": "OAuth Settings",
"owner": "Administrator",
"permissions": [
{
"create": 1,
"delete": 1,
"email": 1,
"print": 1,
"read": 1,
"role": "System Manager",
"share": 1,
"write": 1
}
],
"row_format": "Dynamic",
"sort_field": "creation",
"sort_order": "DESC",
"states": []
}

30
frappe/integrations/doctype/oauth_settings/oauth_settings.py Normal file
View file

@ -0,0 +1,30 @@
# Copyright (c) 2025, Frappe Technologies and contributors
# For license information, please see license.txt
# import frappe
from frappe.model.document import Document
class OAuthSettings(Document):
# begin: auto-generated types
# This code is auto-generated. Do not modify anything in this block.
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from frappe.types import DF
allowed_public_client_origins: DF.SmallText | None
enable_dynamic_client_registration: DF.Check
resource_documentation: DF.Data | None
resource_name: DF.Data | None
resource_policy_uri: DF.Data | None
resource_tos_uri: DF.Data | None
scopes_supported: DF.SmallText | None
show_auth_server_metadata: DF.Check
show_protected_resource_metadata: DF.Check
show_social_login_key_as_authorization_server: DF.Check
skip_authorization: DF.Check
# end: auto-generated types
pass

20
frappe/integrations/doctype/oauth_settings/test_oauth_settings.py Normal file
View file

@ -0,0 +1,20 @@
# Copyright (c) 2025, Frappe Technologies and Contributors
# See license.txt
# import frappe
from frappe.tests import IntegrationTestCase
# On IntegrationTestCase, the doctype test records and all
# link-field test record dependencies are recursively loaded
# Use these module variables to add/remove to/from that list
EXTRA_TEST_RECORD_DEPENDENCIES = [] # eg. ["User"]
IGNORE_TEST_RECORD_DEPENDENCIES = [] # eg. ["User"]
class IntegrationTestOAuthSettings(IntegrationTestCase):
"""
Integration tests for OAuthSettings.
Use this class for testing interactions between multiple components.
"""
pass

0
frappe/integrations/doctype/s3_backup_settings/__init__.py
View file

26
frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.js
View file

@ -1,26 +0,0 @@
// Copyright (c) 2017, Frappe Technologies and contributors
// For license information, please see license.txt
frappe.ui.form.on("S3 Backup Settings", {
refresh: function (frm) {
frm.clear_custom_buttons();
frm.events.take_backup(frm);
},
take_backup: function (frm) {
if (frm.doc.access_key_id && frm.doc.secret_access_key) {
frm.add_custom_button(__("Take Backup Now"), function () {
frm.dashboard.set_headline_alert("S3 Backup Started!");
frappe.call({
method: "frappe.integrations.doctype.s3_backup_settings.s3_backup_settings.take_backups_s3",
callback: function (r) {
if (!r.exc) {
frappe.msgprint(__("S3 Backup complete!"));
frm.dashboard.clear_headline();
}
},
});
}).addClass("btn-primary");
}
},
});

163
frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.json
View file

@ -1,163 +0,0 @@
{
"actions": [],
"creation": "2017-09-04 20:57:20.129205",
"doctype": "DocType",
"editable_grid": 1,
"engine": "InnoDB",
"field_order": [
"enabled",
"api_access_section",
"access_key_id",
"column_break_4",
"secret_access_key",
"notification_section",
"notify_email",
"column_break_8",
"send_email_for_successful_backup",
"s3_bucket_details_section",
"bucket",
"endpoint_url",
"column_break_13",
"backup_path",
"backup_details_section",
"frequency",
"backup_files"
],
"fields": [
{
"default": "0",
"fieldname": "enabled",
"fieldtype": "Check",
"label": "Enable Automatic Backup"
},
{
"fieldname": "notify_email",
"fieldtype": "Data",
"in_list_view": 1,
"label": "Send Notifications To",
"mandatory_depends_on": "enabled",
"reqd": 1
},
{
"default": "1",
"description": "By default, emails are only sent for failed backups.",
"fieldname": "send_email_for_successful_backup",
"fieldtype": "Check",
"label": "Send Email for Successful Backup"
},
{
"fieldname": "frequency",
"fieldtype": "Select",
"in_list_view": 1,
"label": "Backup Frequency",
"mandatory_depends_on": "enabled",
"options": "Daily\nWeekly\nMonthly\nNone",
"reqd": 1
},
{
"fieldname": "access_key_id",
"fieldtype": "Data",
"in_list_view": 1,
"label": "Access Key ID",
"mandatory_depends_on": "enabled",
"reqd": 1
},
{
"fieldname": "secret_access_key",
"fieldtype": "Password",
"in_list_view": 1,
"label": "Access Key Secret",
"mandatory_depends_on": "enabled",
"reqd": 1
},
{
"default": "https://s3.amazonaws.com",
"description": "Only change this if you want to use other S3 compatible object storage backends.",
"fieldname": "endpoint_url",
"fieldtype": "Data",
"label": "Endpoint URL"
},
{
"fieldname": "bucket",
"fieldtype": "Data",
"label": "Bucket Name",
"mandatory_depends_on": "enabled",
"reqd": 1
},
{
"depends_on": "enabled",
"fieldname": "api_access_section",
"fieldtype": "Section Break",
"label": "API Access"
},
{
"fieldname": "column_break_4",
"fieldtype": "Column Break"
},
{
"depends_on": "enabled",
"fieldname": "notification_section",
"fieldtype": "Section Break",
"label": "Notification"
},
{
"fieldname": "column_break_8",
"fieldtype": "Column Break"
},
{
"depends_on": "enabled",
"fieldname": "s3_bucket_details_section",
"fieldtype": "Section Break",
"label": "S3 Bucket Details"
},
{
"fieldname": "column_break_13",
"fieldtype": "Column Break"
},
{
"depends_on": "enabled",
"fieldname": "backup_details_section",
"fieldtype": "Section Break",
"label": "Backup Details"
},
{
"default": "1",
"description": "Backup public and private files along with the database.",
"fieldname": "backup_files",
"fieldtype": "Check",
"label": "Backup Files"
},
{
"description": "If it's empty, it will backup to the root of the bucket.",
"fieldname": "backup_path",
"fieldtype": "Data",
"label": "Backup Path"
}
],
"hide_toolbar": 1,
"issingle": 1,
"links": [],
"modified": "2025-03-15 12:17:49.167012",
"modified_by": "Administrator",
"module": "Integrations",
"name": "S3 Backup Settings",
"owner": "Administrator",
"permissions": [
{
"create": 1,
"delete": 1,
"email": 1,
"print": 1,
"read": 1,
"role": "System Manager",
"share": 1,
"write": 1
}
],
"quick_entry": 1,
"row_format": "Dynamic",
"sort_field": "creation",
"sort_order": "DESC",
"states": [],
"track_changes": 1
}

196
frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.py
View file

@ -1,196 +0,0 @@
# Copyright (c) 2017, Frappe Technologies and contributors
# License: MIT. See LICENSE
import os
import os.path
import boto3
from botocore.exceptions import ClientError
from rq.timeouts import JobTimeoutException
import frappe
from frappe import _
from frappe.integrations.offsite_backup_utils import (
generate_files_backup,
get_latest_backup_file,
send_email,
validate_file_size,
)
from frappe.model.document import Document
from frappe.utils import cint
from frappe.utils.background_jobs import enqueue
class S3BackupSettings(Document):
# begin: auto-generated types
# This code is auto-generated. Do not modify anything in this block.
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from frappe.types import DF
access_key_id: DF.Data
backup_files: DF.Check
backup_path: DF.Data | None
bucket: DF.Data
enabled: DF.Check
endpoint_url: DF.Data | None
frequency: DF.Literal["Daily", "Weekly", "Monthly", "None"]
notify_email: DF.Data
secret_access_key: DF.Password
send_email_for_successful_backup: DF.Check
# end: auto-generated types
def validate(self):
if not self.enabled:
return
if not self.endpoint_url:
self.endpoint_url = "https://s3.amazonaws.com"
if self.backup_path and self.backup_path[-1] != "/":
self.backup_path += "/"
conn = boto3.client(
"s3",
aws_access_key_id=self.access_key_id,
aws_secret_access_key=self.get_password("secret_access_key"),
endpoint_url=self.endpoint_url,
)
try:
# Head_bucket returns a 200 OK if the bucket exists and have access to it.
# Requires ListBucket permission
conn.head_bucket(Bucket=self.bucket)
except ClientError as e:
error_code = e.response["Error"]["Code"]
bucket_name = frappe.bold(self.bucket)
if error_code == "403":
msg = _("Do not have permission to access bucket {0}.").format(bucket_name)
elif error_code == "404":
msg = _("Bucket {0} not found.").format(bucket_name)
else:
msg = e.args[0]
frappe.throw(msg)
@frappe.whitelist()
def take_backup():
"""Enqueue longjob for taking backup to s3"""
enqueue(
"frappe.integrations.doctype.s3_backup_settings.s3_backup_settings.take_backups_s3",
queue="long",
timeout=1500,
)
frappe.msgprint(_("Queued for backup. It may take a few minutes to an hour."))
def take_backups_daily():
take_backups_if("Daily")
def take_backups_weekly():
take_backups_if("Weekly")
def take_backups_monthly():
take_backups_if("Monthly")
def take_backups_if(freq):
if cint(frappe.db.get_single_value("S3 Backup Settings", "enabled")):
if frappe.db.get_single_value("S3 Backup Settings", "frequency") == freq:
take_backups_s3()
@frappe.whitelist()
def take_backups_s3(retry_count=0):
try:
validate_file_size()
backup_to_s3()
send_email(True, "Amazon S3", "S3 Backup Settings", "notify_email")
except JobTimeoutException:
if retry_count < 2:
args = {"retry_count": retry_count + 1}
enqueue(
"frappe.integrations.doctype.s3_backup_settings.s3_backup_settings.take_backups_s3",
queue="long",
timeout=1500,
**args,
)
else:
notify()
except Exception:
notify()
def notify():
error_message = frappe.get_traceback()
send_email(False, "Amazon S3", "S3 Backup Settings", "notify_email", error_message)
def backup_to_s3():
from frappe.utils import get_backups_path
from frappe.utils.backups import new_backup
doc = frappe.get_single("S3 Backup Settings")
bucket = doc.bucket
path = doc.backup_path or ""
backup_files = cint(doc.backup_files)
conn = boto3.client(
"s3",
aws_access_key_id=doc.access_key_id,
aws_secret_access_key=doc.get_password("secret_access_key"),
endpoint_url=doc.endpoint_url or "https://s3.amazonaws.com",
)
if frappe.flags.create_new_backup:
backup = new_backup(
ignore_files=False,
backup_path_db=None,
backup_path_files=None,
backup_path_private_files=None,
force=True,
)
db_filename = os.path.join(get_backups_path(), os.path.basename(backup.backup_path_db))
site_config = os.path.join(get_backups_path(), os.path.basename(backup.backup_path_conf))
if backup_files:
files_filename = os.path.join(get_backups_path(), os.path.basename(backup.backup_path_files))
private_files = os.path.join(
get_backups_path(), os.path.basename(backup.backup_path_private_files)
)
else:
if backup_files:
db_filename, site_config, files_filename, private_files = get_latest_backup_file(
with_files=backup_files
)
if not files_filename or not private_files:
generate_files_backup()
db_filename, site_config, files_filename, private_files = get_latest_backup_file(
with_files=backup_files
)
else:
db_filename, site_config = get_latest_backup_file()
folder = path + os.path.basename(db_filename)[:15] + "/"
# for adding datetime to folder name
upload_file_to_s3(db_filename, folder, conn, bucket)
upload_file_to_s3(site_config, folder, conn, bucket)
if backup_files:
if private_files:
upload_file_to_s3(private_files, folder, conn, bucket)
if files_filename:
upload_file_to_s3(files_filename, folder, conn, bucket)
def upload_file_to_s3(filename, folder, conn, bucket):
destpath = os.path.join(folder, os.path.basename(filename))
print("Uploading file:", filename)
conn.upload_file(filename, bucket, destpath) # Requires PutObject permission

7
frappe/integrations/doctype/s3_backup_settings/test_s3_backup_settings.py
View file

@ -1,7 +0,0 @@
# Copyright (c) 2017, Frappe Technologies and Contributors
# License: MIT. See LICENSE
from frappe.tests import IntegrationTestCase
class TestS3BackupSettings(IntegrationTestCase):
pass

14
frappe/integrations/doctype/social_login_key/social_login_key.json
View file

@ -20,6 +20,7 @@
"base_url",
"configuration_section",
"sign_ups",
"show_in_resource_metadata",
"client_urls",
"authorize_url",
"access_token_url",
@ -172,11 +173,19 @@
"fieldtype": "Select",
"label": "Sign ups",
"options": "\nAllow\nDeny"
},
{
"default": "1",
"description": "Allows clients to view this as an Authorization Server when querying the <code>/.well-known/oauth-protected-resource</code> end point.",
"fieldname": "show_in_resource_metadata",
"fieldtype": "Check",
"label": "Show in Resource Metadata"
}
],
"grid_page_length": 50,
"index_web_pages_for_search": 1,
"links": [],
"modified": "2024-09-06 15:22:46.342392",
"modified": "2025-07-03 12:47:01.696817",
"modified_by": "Administrator",
"module": "Integrations",
"name": "Social Login Key",
@ -195,9 +204,10 @@
"write": 1
}
],
"row_format": "Dynamic",
"sort_field": "creation",
"sort_order": "DESC",
"states": [],
"title_field": "provider_name",
"track_changes": 1
}
}

1
frappe/integrations/doctype/social_login_key/social_login_key.py
View file

@ -54,6 +54,7 @@ class SocialLoginKey(Document):
icon: DF.Data | None
provider_name: DF.Data
redirect_url: DF.Data | None
show_in_resource_metadata: DF.Check
sign_ups: DF.Literal["", "Allow", "Deny"]
social_login_provider: DF.Literal[
"Custom",

Some files were not shown because too many files have changed in this diff Show more