vassili/seitime-frappe
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Merge branch 'develop' into grid-header-rebuild-fix

Browse source
This commit is contained in:
Suraj Shetty 2021-08-03 12:50:21 +05:30 • committed by GitHub
commit d5a29996ca
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
93 changed files with 1165 additions and 584 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
frappe/__init__.py
View file

@ -28,6 +28,8 @@ from .exceptions import *
from .utils.jinja import (get_jenv, get_template, render_template, get_email_from_template, get_jloader)
from .utils.lazy_loader import lazy_import
from frappe.query_builder import get_query_builder
# Lazy imports
faker = lazy_import('faker')
@ -118,6 +120,7 @@ def set_user_lang(user, user_language=None):
# local-globals
db = local("db")
qb = local("qb")
conf = local("conf")
form = form_dict = local("form_dict")
request = local("request")
@ -202,6 +205,7 @@ def init(site, sites_path=None, new_site=False):
local.form_dict = _dict()
local.session = _dict()
local.dev_server = _dev_server
local.qb = get_query_builder(local.conf.db_type or "mariadb")
setup_module_map()

2
frappe/api.py
View file

@ -82,7 +82,7 @@ def handle():
if frappe.local.request.method=="PUT":
data = get_request_form_data()
doc = frappe.get_doc(doctype, name)
doc = frappe.get_doc(doctype, name, for_update=True)
if "flags" in data:
del data["flags"]

18
frappe/cache_manager.py
View file

@ -141,17 +141,13 @@ def build_table_count_cache():
return
_cache = frappe.cache()
data = frappe.db.multisql({
"mariadb": """
SELECT table_name AS name,
table_rows AS count
FROM information_schema.tables""",
"postgres": """
SELECT "relname" AS name,
"n_tup_ins" AS count
FROM "pg_stat_all_tables"
"""
}, as_dict=1)
table_name = frappe.qb.Field("table_name").as_("name")
table_rows = frappe.qb.Field("table_rows").as_("count")
information_schema = frappe.qb.Schema("information_schema")
query = frappe.qb.from_(information_schema.tables).select(table_name, table_rows)
data = frappe.db.sql(query, as_dict=1)
counts = {d.get('name').lstrip('tab'): d.get('count', None) for d in data}
_cache.set_value("information_schema:counts", counts)

4
frappe/commands/__init__.py
View file

@ -102,7 +102,9 @@ def get_commands():
from .site import commands as site_commands
from .translate import commands as translate_commands
from .utils import commands as utils_commands
from .redis import commands as redis_commands
return list(set(scheduler_commands + site_commands + translate_commands + utils_commands))
all_commands = scheduler_commands + site_commands + translate_commands + utils_commands + redis_commands
return list(set(all_commands))
commands = get_commands()

53
frappe/commands/redis.py Normal file
View file

@ -0,0 +1,53 @@
import os
import click
import frappe
from frappe.utils.rq import RedisQueue
from frappe.installer import update_site_config
@click.command('create-rq-users')
@click.option('--set-admin-password', is_flag=True, default=False, help='Set new Redis admin(default user) password')
@click.option('--use-rq-auth', is_flag=True, default=False, help='Enable Redis authentication for sites')
def create_rq_users(set_admin_password=False, use_rq_auth=False):
"""Create Redis Queue users and add to acl and app configs.
acl config file will be used by redis server while starting the server
and app config is used by app while connecting to redis server.
"""
acl_file_path = os.path.abspath('../config/redis_queue.acl')
with frappe.init_site():
acl_list, user_credentials = RedisQueue.gen_acl_list(
set_admin_password=set_admin_password)
with open(acl_file_path, 'w') as f:
f.writelines([acl+'\n' for acl in acl_list])
sites_path = os.getcwd()
common_site_config_path = os.path.join(sites_path, 'common_site_config.json')
update_site_config("rq_username", user_credentials['bench'][0], validate=False,
site_config_path=common_site_config_path)
update_site_config("rq_password", user_credentials['bench'][1], validate=False,
site_config_path=common_site_config_path)
update_site_config("use_rq_auth", use_rq_auth, validate=False,
site_config_path=common_site_config_path)
click.secho('* ACL and site configs are updated with new user credentials. '
'Please restart Redis Queue server to enable namespaces.',
fg='green')
if set_admin_password:
env_key = 'RQ_ADMIN_PASWORD'
click.secho('* Redis admin password is successfully set up. '
'Include below line in .bashrc file for system to use',
fg='green')
click.secho(f"`export {env_key}={user_credentials['default'][1]}`")
click.secho('NOTE: Please save the admin password as you '
'can not access redis server without the password',
fg='yellow')
commands = [
create_rq_users
]

8
frappe/commands/scheduler.py
View file

@ -172,9 +172,13 @@ def start_scheduler():
@click.command('worker')
@click.option('--queue', type=str)
@click.option('--quiet', is_flag = True, default = False, help = 'Hide Log Outputs')
def start_worker(queue, quiet = False):
@click.option('-u', '--rq-username', default=None, help='Redis ACL user')
@click.option('-p', '--rq-password', default=None, help='Redis ACL user password')
def start_worker(queue, quiet = False, rq_username=None, rq_password=None):
"""Site is used to find redis credentals.
"""
from frappe.utils.background_jobs import start_worker
start_worker(queue, quiet = quiet)
start_worker(queue, quiet = quiet, rq_username=rq_username, rq_password=rq_password)
@click.command('ready-for-migration')
@click.option('--site', help='site name')

19
frappe/config/__init__.py
View file

@ -39,18 +39,13 @@ def get_modules_from_app(app):
)
def get_all_empty_tables_by_module():
empty_tables = set(r[0] for r in frappe.db.multisql({
"mariadb": """
SELECT table_name
FROM information_schema.tables
WHERE table_rows = 0 and table_schema = "{}"
""".format(frappe.conf.db_name),
"postgres": """
SELECT "relname" as "table_name"
FROM "pg_stat_all_tables"
WHERE n_tup_ins = 0
"""
}))
table_rows = frappe.qb.Field("table_rows")
table_name = frappe.qb.Field("table_name")
information_schema = frappe.qb.Schema("information_schema")
query = frappe.qb.from_(information_schema.tables).select(table_name).where(table_rows == 0)
empty_tables = {r[0] for r in frappe.db.sql(query)}
results = frappe.get_all("DocType", fields=["name", "module"])
empty_tables_by_module = {}

10
frappe/core/doctype/activity_log/feed.py
View file

@ -29,10 +29,12 @@ def update_feed(doc, method=None):
name = feed.name or doc.name
# delete earlier feed
frappe.db.sql("""delete from `tabActivity Log`
where
reference_doctype=%s and reference_name=%s
and link_doctype=%s""", (doctype, name,feed.link_doctype))
frappe.db.delete("Activity Log", {
"reference_doctype": doctype,
"reference_name": name,
"link_doctype": feed.link_doctype
})
frappe.get_doc({
"doctype": "Activity Log",
"reference_doctype": doctype,

9
frappe/core/doctype/communication/mixins.py
View file

@ -3,6 +3,7 @@ from frappe import _
from frappe.core.utils import get_parent_doc
from frappe.utils import parse_addr, get_formatted_email, get_url
from frappe.email.doctype.email_account.email_account import EmailAccount
from frappe.desk.doctype.todo.todo import ToDo
class CommunicationEmailMixin:
"""Mixin class to handle communication mails.
@ -76,6 +77,7 @@ class CommunicationEmailMixin:
if is_inbound_mail_communcation:
cc.append(self.get_owner())
cc = set(cc) - {self.sender_mailid}
cc.update(self.get_assignees())
cc = set(cc) - set(self.filter_thread_notification_disbled_users(cc))
cc = cc - set(self.mail_recipients(is_inbound_mail_communcation=is_inbound_mail_communcation))
@ -201,6 +203,13 @@ class CommunicationEmailMixin:
self.mail_cc(is_inbound_mail_communcation = is_inbound_mail_communcation, include_sender=include_sender)
return set(all_ids) - set(final_ids)
def get_assignees(self):
"""Get owners of the reference document.
"""
filters = {'status': 'Open', 'reference_name': self.reference_name,
'reference_type': self.reference_doctype}
return ToDo.get_owners(filters)
@staticmethod
def filter_thread_notification_disbled_users(emails):
"""Filter users based on notifications for email threads setting is disabled.

11
frappe/core/doctype/doctype/doctype.json
View file

@ -76,6 +76,7 @@
"index_web_pages_for_search",
"route",
"is_published_field",
"website_search_field",
"advanced",
"engine"
],
@ -547,6 +548,12 @@
{
"fieldname": "column_break_51",
"fieldtype": "Column Break"
},
{
"depends_on": "has_web_view",
"fieldname": "website_search_field",
"fieldtype": "Data",
"label": "Website Search Field"
}
],
"icon": "fa fa-bolt",
@ -628,7 +635,7 @@
"link_fieldname": "reference_doctype"
}
],
"modified": "2021-04-16 12:26:41.031135",
"modified": "2021-06-17 23:31:44.974199",
"modified_by": "Administrator",
"module": "Core",
"name": "DocType",
@ -662,4 +669,4 @@
"sort_field": "modified",
"sort_order": "DESC",
"track_changes": 1
}
}

13
frappe/core/doctype/doctype/doctype.py
View file

@ -396,10 +396,7 @@ class DocType(Document):
frappe.db.sql("""update tabSingles set value=%s
where doctype=%s and field='name' and value = %s""", (new, new, old))
else:
frappe.db.multisql({
"mariadb": f"RENAME TABLE `tab{old}` TO `tab{new}`",
"postgres": f"ALTER TABLE `tab{old}` RENAME TO `tab{new}`"
})
frappe.db.rename_table(old, new)
frappe.db.commit()
# Do not rename and move files and folders for custom doctype
@ -927,6 +924,13 @@ def validate_fields(meta):
if meta.is_published_field not in fieldname_list:
frappe.throw(_("Is Published Field must be a valid fieldname"), InvalidFieldNameError)
def check_website_search_field(meta):
if not meta.website_search_field:
return
if meta.website_search_field not in fieldname_list:
frappe.throw(_("Website Search Field must be a valid fieldname"), InvalidFieldNameError)
def check_timeline_field(meta):
if not meta.timeline_field:
return
@ -1046,6 +1050,7 @@ def validate_fields(meta):
check_title_field(meta)
check_timeline_field(meta)
check_is_published_field(meta)
check_website_search_field(meta)
check_sort_field(meta)
check_image_field(meta)

8
frappe/core/doctype/doctype/test_doctype.py
View file

@ -348,7 +348,6 @@ class TestDocType(unittest.TestCase):
dump_docs = json.dumps(docs.get('docs'))
cancel_all_linked_docs(dump_docs)
data_link_doc.cancel()
data_doc.name = '{}-CAN-0'.format(data_doc.name)
data_doc.load_from_db()
self.assertEqual(data_link_doc.docstatus, 2)
self.assertEqual(data_doc.docstatus, 2)
@ -372,7 +371,7 @@ class TestDocType(unittest.TestCase):
for data in link_doc.get('permissions'):
data.submit = 1
data.cancel = 1
link_doc.insert(ignore_if_duplicate=True)
link_doc.insert()
#create first parent doctype
test_doc_1 = new_doctype('Test Doctype 1')
@ -387,7 +386,7 @@ class TestDocType(unittest.TestCase):
for data in test_doc_1.get('permissions'):
data.submit = 1
data.cancel = 1
test_doc_1.insert(ignore_if_duplicate=True)
test_doc_1.insert()
#crete second parent doctype
doc = new_doctype('Test Doctype 2')
@ -402,7 +401,7 @@ class TestDocType(unittest.TestCase):
for data in link_doc.get('permissions'):
data.submit = 1
data.cancel = 1
doc.insert(ignore_if_duplicate=True)
doc.insert()
# create doctype data
data_link_doc_1 = frappe.new_doc('Test Linked Doctype 1')
@ -433,7 +432,6 @@ class TestDocType(unittest.TestCase):
# checking that doc for Test Doctype 2 is not canceled
self.assertRaises(frappe.LinkExistsError, data_link_doc_1.cancel)
data_doc_2.name = '{}-CAN-0'.format(data_doc_2.name)
data_doc.load_from_db()
data_doc_2.load_from_db()
self.assertEqual(data_link_doc_1.docstatus, 2)

2
frappe/core/doctype/domain_settings/domain_settings.py
View file

@ -34,7 +34,7 @@ class DomainSettings(Document):
all_domains = list((frappe.get_hooks('domains') or {}))
def remove_role(role):
frappe.db.sql('delete from `tabHas Role` where role=%s', role)
frappe.db.delete("Has Role", {"role": role})
frappe.set_value('Role', role, 'disabled', 1)
for domain in all_domains:

2
frappe/core/doctype/error_log/error_log.py
View file

@ -20,4 +20,4 @@ def set_old_logs_as_seen():
def clear_error_logs():
'''Flush all Error Logs'''
frappe.only_for('System Manager')
frappe.db.sql('''DELETE FROM `tabError Log`''')
frappe.db.truncate("Error Log")

8
frappe/core/doctype/report/test_report.py
View file

@ -82,9 +82,11 @@ class TestReport(unittest.TestCase):
def test_report_permissions(self):
frappe.set_user('test@example.com')
frappe.db.sql("""delete from `tabHas Role` where parent = %s
and role = 'Test Has Role'""", frappe.session.user, auto_commit=1)
frappe.db.delete("Has Role", {
"parent": frappe.session.user,
"role": "Test Has Role"
})
frappe.db.commit()
if not frappe.db.exists('Role', 'Test Has Role'):
role = frappe.get_doc({
'doctype': 'Role',

2
frappe/core/doctype/role/role.py
View file

@ -38,7 +38,7 @@ class Role(Document):
self.set(key, 0)
def remove_roles(self):
frappe.db.sql("delete from `tabHas Role` where role = %s", self.name)
frappe.db.delete("Has Role", {"role": self.name})
frappe.clear_cache()
def on_update(self):

5
frappe/core/doctype/scheduled_job_type/scheduled_job_type.py
View file

@ -1,5 +1,4 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2019, Frappe Technologies and contributors
# Copyright (c) 2021, Frappe Technologies and contributors
# For license information, please see license.txt
import json
@ -110,7 +109,7 @@ class ScheduledJobType(Document):
return 'long' if ('Long' in self.frequency) else 'default'
def on_trash(self):
frappe.db.sql('delete from `tabScheduled Job Log` where scheduled_job_type=%s', self.name)
frappe.db.delete("Scheduled Job Log", {"scheduled_job_type": self.name})
@frappe.whitelist()

8
frappe/core/doctype/user/user.py
View file

@ -368,17 +368,15 @@ class User(Document):
frappe.local.login_manager.logout(user=self.name)
# delete todos
frappe.db.sql("""DELETE FROM `tabToDo` WHERE `owner`=%s""", (self.name,))
frappe.db.delete("ToDo", {"owner": self.name})
frappe.db.sql("""UPDATE `tabToDo` SET `assigned_by`=NULL WHERE `assigned_by`=%s""",
(self.name,))
# delete events
frappe.db.sql("""delete from `tabEvent` where owner=%s
and event_type='Private'""", (self.name,))
frappe.db.delete("Event", {"owner": self.name, "event_type": "Private"})
# delete shares
frappe.db.sql("""delete from `tabDocShare` where user=%s""", self.name)
frappe.db.delete("DocShare", {"user": self.name})
# delete messages
frappe.db.sql("""delete from `tabCommunication`
where communication_type in ('Chat', 'Notification')

18
frappe/core/doctype/user_permission/test_user_permission.py
View file

@ -1,6 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2017, Frappe Technologies and Contributors
# See license.txt
# Copyright (c) 2021, Frappe Technologies and Contributors
# See LICENSE
from frappe.core.doctype.user_permission.user_permission import add_user_permissions, remove_applicable
from frappe.permissions import has_user_permission
from frappe.core.doctype.doctype.test_doctype import new_doctype
@ -10,11 +9,14 @@ import unittest
class TestUserPermission(unittest.TestCase):
def setUp(self):
frappe.db.sql("""DELETE FROM `tabUser Permission`
WHERE `user` in (
'test_bulk_creation_update@example.com',
'test_user_perm1@example.com',
'nested_doc_user@example.com')""")
test_users = (
"test_bulk_creation_update@example.com",
"test_user_perm1@example.com",
"nested_doc_user@example.com",
)
frappe.db.delete("User Permission", {
"user": ("in", test_users)
})
frappe.delete_doc_if_exists("DocType", "Person")
frappe.db.sql_ddl("DROP TABLE IF EXISTS `tabPerson`")
frappe.delete_doc_if_exists("DocType", "Doc A")

52
frappe/core/doctype/user_permission/user_permission.py
View file

@ -1,5 +1,4 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2017, Frappe Technologies and contributors
# Copyright (c) 2021, Frappe Technologies and contributors
# For license information, please see license.txt
import frappe, json
@ -179,11 +178,16 @@ def check_applicable_doc_perm(user, doctype, docname):
@frappe.whitelist()
def clear_user_permissions(user, for_doctype):
frappe.only_for('System Manager')
total = frappe.db.count('User Permission', filters = dict(user=user, allow=for_doctype))
frappe.only_for("System Manager")
total = frappe.db.count("User Permission", {"user": user, "allow": for_doctype})
if total:
frappe.db.sql('DELETE FROM `tabUser Permission` WHERE `user`=%s AND `allow`=%s', (user, for_doctype))
frappe.db.delete("User Permission", {
"allow": for_doctype,
"user": user,
})
frappe.clear_cache()
return total
@frappe.whitelist()
@ -225,7 +229,7 @@ def insert_user_perm(user, doctype, docname, is_default=0, hide_descendants=0, a
user_perm.is_default = is_default
user_perm.hide_descendants = hide_descendants
if applicable:
user_perm.applicable_for = applicable
user_perm.applicable_for = applicable
user_perm.apply_to_all_doctypes = 0
else:
user_perm.apply_to_all_doctypes = 1
@ -233,27 +237,27 @@ def insert_user_perm(user, doctype, docname, is_default=0, hide_descendants=0, a
def remove_applicable(perm_applied_docs, user, doctype, docname):
for applicable_for in perm_applied_docs:
frappe.db.sql("""DELETE FROM `tabUser Permission`
WHERE `user`=%s
AND `applicable_for`=%s
AND `allow`=%s
AND `for_value`=%s
""", (user, applicable_for, doctype, docname))
frappe.db.delete("User Permission", {
"applicable_for": applicable_for,
"for_value": docname,
"allow": doctype,
"user": user,
})
def remove_apply_to_all(user, doctype, docname):
frappe.db.sql("""DELETE from `tabUser Permission`
WHERE `user`=%s
AND `apply_to_all_doctypes`=1
AND `allow`=%s
AND `for_value`=%s
""",(user, doctype, docname))
frappe.db.delete("User Permission", {
"apply_to_all_doctypes": 1,
"for_value": docname,
"allow": doctype,
"user": user,
})
def update_applicable(already_applied, to_apply, user, doctype, docname):
for applied in already_applied:
if applied not in to_apply:
frappe.db.sql("""DELETE FROM `tabUser Permission`
WHERE `user`=%s
AND `applicable_for`=%s
AND `allow`=%s
AND `for_value`=%s
""",(user, applied, doctype, docname))
frappe.db.delete("User Permission", {
"applicable_for": applied,
"for_value": docname,
"allow": doctype,
"user": user,
})

8
frappe/core/page/background_jobs/background_jobs.py
View file

@ -4,12 +4,12 @@
import json
from typing import TYPE_CHECKING, Dict, List
from rq import Queue, Worker
from rq import Worker
import frappe
from frappe import _
from frappe.utils import convert_utc_to_user_timezone, format_datetime
from frappe.utils.background_jobs import get_redis_conn
from frappe.utils.background_jobs import get_redis_conn, get_queues
from frappe.utils.scheduler import is_scheduler_inactive
if TYPE_CHECKING:
@ -29,7 +29,7 @@ def get_info(show_failed=False) -> List[Dict]:
show_failed = json.loads(show_failed)
conn = get_redis_conn()
queues = Queue.all(conn)
queues = get_queues()
workers = Worker.all(conn)
jobs = []
@ -75,7 +75,7 @@ def get_info(show_failed=False) -> List[Dict]:
@frappe.whitelist()
def remove_failed_jobs():
conn = get_redis_conn()
queues = Queue.all(conn)
queues = get_queues()
for queue in queues:
fail_registry = queue.failed_job_registry
for job_id in fail_registry.get_job_ids():

17
frappe/core/page/permission_manager/permission_manager.py
View file

@ -92,14 +92,14 @@ def update(doctype, role, permlevel, ptype, value=None):
"""Update role permission params
Args:
doctype (str): Name of the DocType to update params for
role (str): Role to be updated for, eg "Website Manager".
permlevel (int): perm level the provided rule applies to
ptype (str): permission type, example "read", "delete", etc.
value (None, optional): value for ptype, None indicates False
doctype (str): Name of the DocType to update params for
role (str): Role to be updated for, eg "Website Manager".
permlevel (int): perm level the provided rule applies to
ptype (str): permission type, example "read", "delete", etc.
value (None, optional): value for ptype, None indicates False
Returns:
str: Refresh flag is permission is updated successfully
str: Refresh flag is permission is updated successfully
"""
frappe.only_for("System Manager")
out = update_permission_property(doctype, role, permlevel, ptype, value)
@ -110,10 +110,9 @@ def remove(doctype, role, permlevel):
frappe.only_for("System Manager")
setup_custom_perms(doctype)
name = frappe.get_value('Custom DocPerm', dict(parent=doctype, role=role, permlevel=permlevel))
frappe.db.delete("Custom DocPerm", {"parent": doctype, "role": role, "permlevel": permlevel})
frappe.db.sql('delete from `tabCustom DocPerm` where name=%s', name)
if not frappe.get_all('Custom DocPerm', dict(parent=doctype)):
if not frappe.get_all('Custom DocPerm', {"parent": doctype}):
frappe.throw(_('There must be atleast one permission rule.'), title=_('Cannot Remove'))
validate_permissions_for_doctype(doctype, for_remove=True, alert=True)

10
frappe/custom/doctype/custom_field/custom_field.py
View file

@ -85,12 +85,10 @@ class CustomField(Document):
frappe.bold(self.label)))
# delete property setter entries
frappe.db.sql("""\
DELETE FROM `tabProperty Setter`
WHERE doc_type = %s
AND field_name = %s""",
(self.dt, self.fieldname))
frappe.db.delete("Property Setter", {
"doc_type": self.dt,
"field_name": self.fieldname
})
frappe.clear_cache(doctype=self.dt)
def validate_insert_after(self, meta):

9
frappe/custom/doctype/customize_form/customize_form.py
View file

@ -1,5 +1,5 @@
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt
# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See LICENSE
"""
Customize Form is a Single DocType used to mask the Property Setter
@ -18,10 +18,11 @@ from frappe.custom.doctype.property_setter.property_setter import delete_propert
from frappe.model.docfield import supports_translation
from frappe.core.doctype.doctype.doctype import validate_series
class CustomizeForm(Document):
def on_update(self):
frappe.db.sql("delete from tabSingles where doctype='Customize Form'")
frappe.db.sql("delete from `tabCustomize Form Field`")
frappe.db.delete("Singles", {"doctype": "Customize Form"})
frappe.db.delete("Customize Form Field")
@frappe.whitelist()
def fetch_to_customize(self):

48
frappe/database/database.py
View file

@ -6,6 +6,7 @@
import re
import time
from typing import Dict, List, Union
import frappe
import datetime
import frappe.defaults
@ -13,7 +14,7 @@ import frappe.model.meta
from frappe import _
from time import time
from frappe.utils import now, getdate, cast_fieldtype, get_datetime
from frappe.utils import now, getdate, cast_fieldtype, get_datetime, get_table_name
from frappe.model.utils.link_count import flush_local_link_count
@ -103,6 +104,7 @@ class Database(object):
{"name": "a%", "owner":"test@example.com"})
"""
query = str(query)
if re.search(r'ifnull\(', query, flags=re.IGNORECASE):
# replaces ifnull in query with coalesce
query = re.sub(r'ifnull\(', 'coalesce(', query, flags=re.IGNORECASE)
@ -951,15 +953,37 @@ class Database(object):
query = sql_dict.get(current_dialect)
return self.sql(query, values, **kwargs)
def delete(self, doctype, conditions, debug=False):
if conditions:
conditions, values = self.build_conditions(conditions)
return self.sql("DELETE FROM `tab{doctype}` where {conditions}".format(
doctype=doctype,
conditions=conditions
), values, debug=debug)
else:
frappe.throw(_('No conditions provided'))
def delete(self, doctype: str, filters: Union[Dict, List] = None, debug=False, **kwargs):
"""Delete rows from a table in site which match the passed filters. This
does trigger DocType hooks. Simply runs a DELETE query in the database.
Doctype name can be passed directly, it will be pre-pended with `tab`.
"""
values = ()
filters = filters or kwargs.get("conditions")
table = get_table_name(doctype)
query = f"DELETE FROM `{table}`"
if "debug" not in kwargs:
kwargs["debug"] = debug
if filters:
conditions, values = self.build_conditions(filters)
query = f"{query} WHERE {conditions}"
return self.sql(query, values, **kwargs)
def truncate(self, doctype: str):
"""Truncate a table in the database. This runs a DDL command `TRUNCATE TABLE`.
This cannot be rolled back.
Doctype name can be passed directly, it will be pre-pended with `tab`.
"""
table = doctype if doctype.startswith("__") else f"tab{doctype}"
return self.sql_ddl(f"truncate `{table}`")
def clear_table(self, doctype):
return self.truncate(doctype)
def get_last_created(self, doctype):
last_record = self.get_all(doctype, ('creation'), limit=1, order_by='creation desc')
@ -968,9 +992,6 @@ class Database(object):
else:
return None
def clear_table(self, doctype):
self.sql('truncate `tab{}`'.format(doctype))
def log_touched_tables(self, query, values=None):
if values:
query = frappe.safe_decode(self._cursor.mogrify(query, values))
@ -1021,6 +1042,7 @@ class Database(object):
), tuple(insert_list))
insert_list = []
def enqueue_jobs_after_commit():
from frappe.utils.background_jobs import execute_job, get_queue

17
frappe/database/mariadb/database.py
View file

@ -1,3 +1,5 @@
from typing import List, Tuple, Union
import pymysql
from pymysql.constants import ER, FIELD_TYPE
from pymysql.converters import conversions, escape_string
@ -5,7 +7,7 @@ from pymysql.converters import conversions, escape_string
import frappe
from frappe.database.database import Database
from frappe.database.mariadb.schema import MariaDBTable
from frappe.utils import UnicodeWithAttrs, cstr, get_datetime
from frappe.utils import UnicodeWithAttrs, cstr, get_datetime, get_table_name
class MariaDBDatabase(Database):
@ -123,6 +125,19 @@ class MariaDBDatabase(Database):
def is_type_datetime(code):
return code in (pymysql.DATE, pymysql.DATETIME)
def rename_table(self, old_name: str, new_name: str) -> Union[List, Tuple]:
old_name = get_table_name(old_name)
new_name = get_table_name(new_name)
return self.sql(f"RENAME TABLE `{old_name}` TO `{new_name}`")
def describe(self, doctype: str) -> Union[List, Tuple]:
table_name = get_table_name(doctype)
return self.sql(f"DESC `{table_name}`")
def change_column_type(self, table: str, column: str, type: str) -> Union[List, Tuple]:
table_name = get_table_name(table)
return self.sql(f"ALTER TABLE `{table_name}` MODIFY `{column}` {type} NOT NULL")
# exception types
@staticmethod
def is_deadlocked(e):

1
frappe/database/mariadb/framework_mariadb.sql
View file

@ -220,6 +220,7 @@ CREATE TABLE `tabDocType` (
`allow_guest_to_view` int(1) NOT NULL DEFAULT 0,
`route` varchar(255) DEFAULT NULL,
`is_published_field` varchar(255) DEFAULT NULL,
`website_search_field` varchar(255) DEFAULT NULL,
`email_append_to` int(1) NOT NULL DEFAULT 0,
`subject_field` varchar(255) DEFAULT NULL,
`sender_field` varchar(255) DEFAULT NULL,

20
frappe/database/postgres/database.py
View file

@ -1,12 +1,14 @@
import re
import frappe
from typing import List, Tuple, Union
import psycopg2
import psycopg2.extensions
from frappe.utils import cstr
from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
import frappe
from frappe.database.database import Database
from frappe.database.postgres.schema import PostgresTable
from frappe.utils import cstr, get_table_name
# cast decimals as floats
DEC2FLOAT = psycopg2.extensions.new_type(
@ -170,6 +172,19 @@ class PostgresDatabase(Database):
def is_data_too_long(e):
return e.pgcode == '22001'
def rename_table(self, old_name: str, new_name: str) -> Union[List, Tuple]:
old_name = get_table_name(old_name)
new_name = get_table_name(new_name)
return self.sql(f"ALTER TABLE `{old_name}` RENAME TO `{new_name}`")
def describe(self, doctype: str)-> Union[List, Tuple]:
table_name = get_table_name(doctype)
return self.sql(f"SELECT COLUMN_NAME FROM information_schema.COLUMNS WHERE TABLE_NAME = '{table_name}'")
def change_column_type(self, table: str, column: str, type: str) -> Union[List, Tuple]:
table_name = get_table_name(table)
return self.sql(f'ALTER TABLE "{table_name}" ALTER COLUMN "{column}" TYPE {type}')
def create_auth_table(self):
self.sql_ddl("""create table if not exists "__Auth" (
"doctype" VARCHAR(140) NOT NULL,
@ -297,6 +312,7 @@ class PostgresDatabase(Database):
def modify_query(query):
""""Modifies query according to the requirements of postgres"""
# replace ` with " for definitions
query = str(query)
query = query.replace('`', '"')
query = replace_locate_with_strpos(query)
# select from requires ""

1
frappe/database/postgres/framework_postgres.sql
View file

@ -225,6 +225,7 @@ CREATE TABLE "tabDocType" (
"allow_guest_to_view" smallint NOT NULL DEFAULT 0,
"route" varchar(255) DEFAULT NULL,
"is_published_field" varchar(255) DEFAULT NULL,
"website_search_field" varchar(255) DEFAULT NULL,
"email_append_to" smallint NOT NULL DEFAULT 0,
"subject_field" varchar(255) DEFAULT NULL,
"sender_field" varchar(255) DEFAULT NULL,

32
frappe/defaults.py
View file

@ -124,11 +124,10 @@ def set_default(key, value, parent, parenttype="__default"):
where
defkey=%s and parent=%s
for update''', (key, parent)):
frappe.db.sql("""
delete from
`tabDefaultValue`
where
defkey=%s and parent=%s""", (key, parent))
frappe.db.delete("DefaultValue", {
"defkey": key,
"parent": parent
})
if value != None:
add_default(key, value, parent)
else:
@ -155,29 +154,23 @@ def clear_default(key=None, value=None, parent=None, name=None, parenttype=None)
:param name: Default ID.
:param parenttype: Clear defaults table for a particular type e.g. **User**.
"""
conditions = []
values = []
filters = {}
if name:
conditions.append("name=%s")
values.append(name)
filters.update({"name": name})
else:
if key:
conditions.append("defkey=%s")
values.append(key)
filters.update({"defkey": key})
if value:
conditions.append("defvalue=%s")
values.append(value)
filters.update({"defvalue": value})
if parent:
conditions.append("parent=%s")
values.append(parent)
filters.update({"parent": parent})
if parenttype:
conditions.append("parenttype=%s")
values.append(parenttype)
filters.update({"parenttype": parenttype})
if parent:
clear_defaults_cache(parent)
@ -185,11 +178,10 @@ def clear_default(key=None, value=None, parent=None, name=None, parenttype=None)
clear_defaults_cache("__default")
clear_defaults_cache("__global")
if not conditions:
if not filters:
raise Exception("[clear_default] No key specified.")
frappe.db.sql("""delete from tabDefaultValue where {0}""".format(" and ".join(conditions)),
tuple(values))
frappe.db.delete("DefaultValue", filters)
_clear_cache(parent)

4
frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py
View file

@ -64,7 +64,7 @@ class TestDashboardChart(unittest.TestCase):
if frappe.db.exists('Dashboard Chart', 'Test Empty Dashboard Chart'):
frappe.delete_doc('Dashboard Chart', 'Test Empty Dashboard Chart')
frappe.db.sql('delete from `tabError Log`')
frappe.db.delete("Error Log")
frappe.get_doc(dict(
doctype = 'Dashboard Chart',
@ -94,7 +94,7 @@ class TestDashboardChart(unittest.TestCase):
if frappe.db.exists('Dashboard Chart', 'Test Empty Dashboard Chart 2'):
frappe.delete_doc('Dashboard Chart', 'Test Empty Dashboard Chart 2')
frappe.db.sql('delete from `tabError Log`')
frappe.db.delete("Error Log")
# create one data point
frappe.get_doc(dict(doctype = 'Error Log', creation = '2018-06-01 00:00:00')).insert()

2
frappe/desk/doctype/desktop_icon/desktop_icon.py
View file

@ -197,7 +197,7 @@ def set_desktop_icons(visible_list, ignore_duplicate=True):
# clear all custom only if setup is not complete
if not int(frappe.defaults.get_defaults().setup_complete or 0):
frappe.db.sql('delete from `tabDesktop Icon` where standard=0')
frappe.db.delete("Desktop Icon", {"standard": 0})
# set standard as blocked and hidden if setting first active domain
if not frappe.flags.keep_desktop_icons:

5
frappe/desk/doctype/event/event.py
View file

@ -338,9 +338,8 @@ def delete_events(ref_type, ref_name, delete_event=False):
total_participants = frappe.get_all("Event Participants", filters={"parenttype": "Event", "parent": participation.parent})
if len(total_participants) <= 1:
frappe.db.sql("DELETE FROM `tabEvent` WHERE `name` = %(name)s", {'name': participation.parent})
frappe.db.sql("DELETE FROM `tabEvent Participants ` WHERE `name` = %(name)s", {'name': participation.name})
frappe.db.delete("Event", {"name": participation.parent})
frappe.db.delete("Event Participants", {"name": participation.name})
# Close events if ends_on or repeat_till is less than now_datetime
def set_status_of_events():

21
frappe/desk/doctype/route_history/route_history.py
View file

@ -1,5 +1,4 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2018, Frappe Technologies and contributors
# Copyright (c) 2021, Frappe Technologies and contributors
# For license information, please see license.txt
import frappe
@ -8,6 +7,7 @@ from frappe.model.document import Document
class RouteHistory(Document):
pass
def flush_old_route_records():
"""Deletes all route records except last 500 records per user"""
@ -24,19 +24,14 @@ def flush_old_route_records():
for user in users:
user = user[0]
last_record_to_keep = frappe.db.get_all('Route History',
filters={
'user': user,
},
filters={'user': user},
limit=1,
limit_start=500,
fields=['modified'],
order_by='modified desc')
order_by='modified desc'
)
frappe.db.sql('''
DELETE
FROM `tabRoute History`
WHERE `modified` <= %(modified)s and `user`=%(modified)s
''', {
"modified": last_record_to_keep[0].modified,
frappe.db.delete("Route History", {
"modified": ("<=", last_record_to_keep[0].modified),
"user": user
})
})

12
frappe/desk/doctype/tag/tag.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2019, Frappe Technologies and contributors
# For license information, please see license.txt
@ -123,7 +122,10 @@ def delete_tags_for_document(doc):
if not frappe.db.table_exists("Tag Link"):
return
frappe.db.sql("""DELETE FROM `tabTag Link` WHERE `document_type`=%s AND `document_name`=%s""", (doc.doctype, doc.name))
frappe.db.delete("Tag Link", {
"document_type": doc.doctype,
"document_name": doc.name
})
def update_tags(doc, tags):
"""
@ -161,7 +163,11 @@ def get_deleted_tags(new_tags, existing_tags):
return list(set(existing_tags) - set(new_tags))
def delete_tag_for_document(dt, dn, tag):
frappe.db.sql("""DELETE FROM `tabTag Link` WHERE `document_type`=%s AND `document_name`=%s AND tag=%s""", (dt, dn, tag))
frappe.db.delete("Tag Link", {
"document_type": dt,
"document_name": dn,
"tag": tag
})
@frappe.whitelist()
def get_documents_for_tag(tag):

26
frappe/desk/doctype/todo/todo.py
View file

@ -5,11 +5,13 @@ import frappe
import json
from frappe.model.document import Document
from frappe.utils import get_fullname
from frappe.utils import get_fullname, parse_addr
exclude_from_linked_with = True
class ToDo(Document):
DocType = 'ToDo'
def validate(self):
self._assignment = None
if self.is_new():
@ -39,13 +41,7 @@ class ToDo(Document):
self.update_in_reference()
def on_trash(self):
# unlink todo from linked comments
frappe.db.sql("""
delete from `tabCommunication Link`
where link_doctype=%(doctype)s and link_name=%(name)s""", {
"doctype": self.doctype, "name": self.name
})
self.delete_communication_links()
self.update_in_reference()
def add_assign_comment(self, text, comment_type):
@ -54,6 +50,13 @@ class ToDo(Document):
frappe.get_doc(self.reference_type, self.reference_name).add_comment(comment_type, text)
def delete_communication_links(self):
# unlink todo from linked comments
return frappe.db.delete("Communication Link", {
"link_doctype": self.doctype,
"link_name": self.name
})
def update_in_reference(self):
if not (self.reference_type and self.reference_name):
return
@ -84,6 +87,13 @@ class ToDo(Document):
else:
raise
@classmethod
def get_owners(cls, filters=None):
"""Returns list of owners after applying filters on todo's.
"""
rows = frappe.get_all(cls.DocType, filters=filters or {}, fields=['owner'])
return [parse_addr(row.owner)[1] for row in rows if row.owner]
# NOTE: todo is viewable if a user is an owner, or set as assigned_to value, or has any role that is allowed to access ToDo doctype.
def on_doctype_update():
frappe.db.add_index("ToDo", ["reference_type", "reference_name"])

5
frappe/email/doctype/unhandled_email/unhandled_email.py
View file

@ -10,5 +10,6 @@ class UnhandledEmail(Document):
def remove_old_unhandled_emails():
frappe.db.sql("""DELETE FROM `tabUnhandled Email`
WHERE creation < %s""", frappe.utils.add_days(frappe.utils.nowdate(), -30))
frappe.db.delete("Unhandled Email", {
"creation": ("<", frappe.utils.add_days(frappe.utils.nowdate(), -30))
})

9
frappe/email/queue.py
View file

@ -173,13 +173,8 @@ def clear_outbox(days=None):
WHERE `priority`=0 AND `modified` < (NOW() - INTERVAL '{0}' DAY)""".format(days))
if email_queues:
frappe.db.sql("""DELETE FROM `tabEmail Queue` WHERE `name` IN ({0})""".format(
','.join(['%s']*len(email_queues)
)), tuple(email_queues))
frappe.db.sql("""DELETE FROM `tabEmail Queue Recipient` WHERE `parent` IN ({0})""".format(
','.join(['%s']*len(email_queues)
)), tuple(email_queues))
frappe.db.delete("Email Queue", {"name": ("in", email_queues)})
frappe.db.delete("Email Queue Recipient", {"parent": ("in", email_queues)})
def set_expiry_for_email_queue():
''' Mark emails as expire that has not sent for 7 days.

28
frappe/model/__init__.py
View file

@ -152,32 +152,22 @@ def delete_fields(args_dict, delete=0):
if not fields:
continue
frappe.db.sql("""
DELETE FROM `tabDocField`
WHERE parent='%s' AND fieldname IN (%s)
""" % (dt, ", ".join(["'{}'".format(f) for f in fields])))
frappe.db.delete("DocField", {
"parent": dt,
"fieldname": ("in", fields),
})
# Delete the data/column only if delete is specified
if not delete:
continue
if frappe.db.get_value("DocType", dt, "issingle"):
frappe.db.sql("""
DELETE FROM `tabSingles`
WHERE doctype='%s' AND field IN (%s)
""" % (dt, ", ".join("'{}'".format(f) for f in fields)))
frappe.db.delete("Singles", {
"doctype": dt,
"field": ("in", fields),
})
else:
existing_fields = frappe.db.multisql({
"mariadb": "DESC `tab%s`" % dt,
"postgres": """
SELECT
COLUMN_NAME
FROM
information_schema.COLUMNS
WHERE
TABLE_NAME = 'tab%s';
""" % dt,
})
existing_fields = frappe.db.describe(dt)
existing_fields = existing_fields and [e[0] for e in existing_fields] or []
fields_need_to_delete = set(fields) & set(existing_fields)
if not fields_need_to_delete:

23
frappe/model/delete_doc.py
View file

@ -65,12 +65,12 @@ def delete_doc(doctype=None, name=None, force=0, ignore_doctypes=None, for_reloa
update_flags(doc, flags, ignore_permissions)
check_permission_and_not_submitted(doc)
frappe.db.sql("delete from `tabCustom Field` where dt = %s", name)
frappe.db.sql("delete from `tabClient Script` where dt = %s", name)
frappe.db.sql("delete from `tabProperty Setter` where doc_type = %s", name)
frappe.db.sql("delete from `tabReport` where ref_doctype=%s", name)
frappe.db.sql("delete from `tabCustom DocPerm` where parent=%s", name)
frappe.db.sql("delete from `__global_search` where doctype=%s", name)
frappe.db.delete("Custom Field", {"dt": name})
frappe.db.delete("Client Script", {"dt": name})
frappe.db.delete("Property Setter", {"doc_type": name})
frappe.db.delete("Report", {"ref_doctype": name})
frappe.db.delete("Custom DocPerm", {"parent": name})
frappe.db.delete("__global_search", {"doctype": name})
delete_from_table(doctype, name, ignore_doctypes, None)
@ -162,10 +162,9 @@ def update_naming_series(doc):
def delete_from_table(doctype, name, ignore_doctypes, doc):
if doctype!="DocType" and doctype==name:
frappe.db.sql("delete from `tabSingles` where `doctype`=%s", name)
frappe.db.delete("Singles", {"doctype": name})
else:
frappe.db.sql("delete from `tab{0}` where `name`=%s".format(doctype), name)
frappe.db.delete(doctype, {"name": name})
# get child tables
if doc:
tables = [d.options for d in doc.meta.get_table_fields()]
@ -339,8 +338,10 @@ def clear_references(doctype, reference_doctype, reference_name,
(reference_doctype, reference_name))
def clear_timeline_references(link_doctype, link_name):
frappe.db.sql("""DELETE FROM `tabCommunication Link`
WHERE `tabCommunication Link`.link_doctype=%s AND `tabCommunication Link`.link_name=%s""", (link_doctype, link_name))
frappe.db.delete("Communication Link", {
"link_doctype": link_doctype,
"link_name": link_name
})
def insert_feed(doc):
if (

22
frappe/model/document.py
View file

@ -5,7 +5,7 @@ import time
from frappe import _, msgprint, is_whitelisted
from frappe.utils import flt, cstr, now, get_datetime_str, file_lock, date_diff
from frappe.model.base_document import BaseDocument, get_controller
from frappe.model.naming import set_new_name, gen_new_name_for_cancelled_doc
from frappe.model.naming import set_new_name
from werkzeug.exceptions import NotFound, Forbidden
import hashlib, json
from frappe.model import optional_fields, table_fields
@ -390,10 +390,11 @@ class Document(BaseDocument):
else:
# no rows found, delete all rows
frappe.db.sql("""delete from `tab{0}` where parent=%s
and parenttype=%s and parentfield=%s""".format(df.options),
(self.name, self.doctype, fieldname))
frappe.db.delete(df.options, {
"parent": self.name,
"parenttype": self.doctype,
"parentfield": fieldname
})
def get_doc_before_save(self):
return getattr(self, '_doc_before_save', None)
@ -451,7 +452,9 @@ class Document(BaseDocument):
def update_single(self, d):
"""Updates values for Single type Document in `tabSingles`."""
frappe.db.sql("""delete from `tabSingles` where doctype=%s""", self.doctype)
frappe.db.delete("Singles", {
"doctype": self.doctype
})
for field, value in d.items():
if field != "doctype":
frappe.db.sql("""insert into `tabSingles` (doctype, field, value)
@ -705,6 +708,7 @@ class Document(BaseDocument):
else:
tmp = frappe.db.sql("""select modified, docstatus from `tab{0}`
where name = %s for update""".format(self.doctype), self.name, as_dict=True)
if not tmp:
frappe.throw(_("Record does not exist"))
else:
@ -915,12 +919,8 @@ class Document(BaseDocument):
@whitelist.__func__
def _cancel(self):
"""Cancel the document. Sets `docstatus` = 2, then saves.
"""
"""Cancel the document. Sets `docstatus` = 2, then saves."""
self.docstatus = 2
new_name = gen_new_name_for_cancelled_doc(self)
frappe.rename_doc(self.doctype, self.name, new_name, force=True, show_alert=False)
self.name = new_name
self.save()
@whitelist.__func__

83
frappe/model/naming.py
View file

@ -28,7 +28,7 @@ def set_new_name(doc):
doc.name = None
if getattr(doc, "amended_from", None):
doc.name = _get_amended_name(doc)
_set_amended_name(doc)
return
elif getattr(doc.meta, "issingle", False):
@ -221,15 +221,6 @@ def revert_series_if_last(key, name, doc=None):
* prefix = #### and hashes = 2021 (hash doesn't exist)
* will search hash in key then accordingly get prefix = ""
"""
if hasattr(doc, 'amended_from'):
# do not revert if doc is amended, since cancelled docs still exist
if doc.docstatus != 2 and doc.amended_from:
return
# for first cancelled doc
if doc.docstatus == 2 and not doc.amended_from:
name, _ = NameParser.parse_docname(doc.name, sep='-CAN-')
if ".#" in key:
prefix, hashes = key.rsplit(".", 1)
if "#" not in hashes:
@ -312,9 +303,16 @@ def append_number_if_name_exists(doctype, value, fieldname="name", separator="-"
return value
def _get_amended_name(doc):
name, _ = NameParser(doc).parse_amended_from()
return name
def _set_amended_name(doc):
am_id = 1
am_prefix = doc.amended_from
if frappe.db.get_value(doc.doctype, doc.amended_from, "amended_from"):
am_id = cint(doc.amended_from.split("-")[-1]) + 1
am_prefix = "-".join(doc.amended_from.split("-")[:-1]) # except the last hyphen
doc.name = am_prefix + "-" + str(am_id)
return doc.name
def _field_autoname(autoname, doc, skip_slicing=None):
"""
@ -325,6 +323,7 @@ def _field_autoname(autoname, doc, skip_slicing=None):
name = (cstr(doc.get(fieldname)) or "").strip()
return name
def _prompt_autoname(autoname, doc):
"""
Generate a name using Prompt option. This simply means the user will have to set the name manually.
@ -355,61 +354,3 @@ def _format_autoname(autoname, doc):
name = re.sub(r"(\{[\w | #]+\})", get_param_value_for_match, autoname_value)
return name
class NameParser:
"""Parse document name and return all the parts of it.
NOTE: It handles cancellend and amended doc parsing for now. It can be expanded.
"""
def __init__(self, doc):
self.doc = doc
def parse_name(self):
if not hasattr(self.doc, "amended_from"):
return (self.doc.name, None, None)
#If document is cancelled document
if hasattr(self.doc, "amended_from") and self.doc.docstatus == 2:
return self.parse_docname(self.doc.name, sep='-CAN-')
return self.parse_docname(self.doc.name)
def parse_amended_from(self):
if not getattr(self.doc, 'amended_from', None):
return (None, None)
return self.parse_docname(self.doc.amended_from, '-CAN-')
@classmethod
def parse_docname(cls, name, sep='-'):
split_list = name.rsplit(sep, 1)
if len(split_list) == 1:
return (name, None)
return (split_list[0], split_list[1])
def get_cancelled_doc_latest_counter(tname, docname):
"""Get the latest counter used for cancelled docs of given docname.
"""
name_prefix = f'{docname}-CAN-'
rows = frappe.db.sql("""
select
name
from `tab{tname}`
where
name like %(name_prefix)s and docstatus=2
""".format(tname=tname), {'name_prefix': name_prefix+'%'}, as_dict=1)
if not rows:
return -1
return max([int(row.name.replace(name_prefix, '') or -1) for row in rows])
def gen_new_name_for_cancelled_doc(doc):
"""Generate a new name for cancelled document.
"""
if getattr(doc, "amended_from", None):
name, _ = NameParser(doc).parse_amended_from()
else:
name = doc.name
counter = get_cancelled_doc_latest_counter(doc.doctype, name)
return f'{name}-CAN-{counter+1}'

2
frappe/patches.txt
View file

@ -180,4 +180,4 @@ frappe.patches.v12_0.rename_uploaded_files_with_proper_name
frappe.patches.v13_0.queryreport_columns
frappe.patches.v13_0.jinja_hook
frappe.patches.v13_0.update_notification_channel_if_empty
frappe.patches.v13_0.rename_cancelled_docs
frappe.patches.v14_0.drop_data_import_legacy

4
frappe/patches/v11_0/apply_customization_to_custom_doctype.py
View file

@ -28,7 +28,7 @@ def execute():
for prop in property_setters:
property_setter_map[prop.field_name] = prop
frappe.db.sql('DELETE FROM `tabProperty Setter` WHERE `name`=%s', prop.name)
frappe.db.delete("Property Setter", {"name": prop.name})
meta = frappe.get_meta(doctype.name)
@ -50,6 +50,6 @@ def execute():
df = frappe.new_doc('DocField', meta, 'fields')
df.update(cf)
meta.fields.append(df)
frappe.db.sql('DELETE FROM `tabCustom Field` WHERE name=%s', cf.name)
frappe.db.delete("Custom Field", {"name": cf.name})
meta.save()

2
frappe/patches/v11_0/sync_stripe_settings_before_migrate.py
View file

@ -17,4 +17,4 @@ def execute():
settings.secret_key = secret_key
settings.save(ignore_permissions=True)
frappe.db.sql("""DELETE FROM tabSingles WHERE doctype='Stripe Settings'""")
frappe.db.delete("Singles", {"doctype": "Stripe Settings"})

5
frappe/patches/v12_0/delete_feedback_request_if_exists.py
View file

@ -2,7 +2,4 @@
import frappe
def execute():
frappe.db.sql('''
DELETE from `tabDocType`
WHERE name = 'Feedback Request'
''')
frappe.db.delete("DocType", {"name": "Feedback Request"})

7
frappe/patches/v12_0/remove_deprecated_fields_from_doctype.py
View file

@ -8,7 +8,6 @@ def execute():
'DocType': ['hide_heading', 'image_view', 'read_only_onload']
}, delete=1)
frappe.db.sql('''
DELETE from `tabProperty Setter`
WHERE property = 'read_only_onload'
''')
frappe.db.delete("Property Setter", {
"property": "read_only_onload"
})

37
frappe/patches/v12_0/set_correct_assign_value_in_docs.py
View file

@ -1,32 +1,29 @@
import frappe
from frappe.query_builder.functions import GroupConcat, Coalesce
def execute():
frappe.reload_doc('desk', 'doctype', 'todo')
frappe.reload_doc("desk", "doctype", "todo")
query = '''
SELECT
name, reference_type, reference_name, {} as assignees
FROM
`tabToDo`
WHERE
COALESCE(reference_type, '') != '' AND
COALESCE(reference_name, '') != '' AND
status != 'Cancelled'
GROUP BY
reference_type, reference_name
'''
ToDo = frappe.qb.Table("ToDo")
assignees = GroupConcat("owner").distinct().as_("assignees")
assignments = frappe.db.multisql({
'mariadb': query.format('GROUP_CONCAT(DISTINCT `owner`)'),
'postgres': query.format('STRING_AGG(DISTINCT "owner", ",")')
}, as_dict=True)
query = (
frappe.qb.from_(ToDo)
.select(ToDo.name, ToDo.reference_type, assignees)
.where(Coalesce(ToDo.reference_type, "") != "")
.where(Coalesce(ToDo.reference_name, "") != "")
.where(ToDo.status != "Cancelled")
.groupby(ToDo.reference_type, ToDo.reference_name)
)
assignments = frappe.db.sql(query, as_dict=True)
for doc in assignments:
assignments = doc.assignees.split(',')
assignments = doc.assignees.split(",")
frappe.db.set_value(
doc.reference_type,
doc.reference_name,
'_assign',
"_assign",
frappe.as_json(assignments),
update_modified=False
)
)

39
frappe/patches/v12_0/set_primary_key_in_series.py
View file

@ -1,21 +1,24 @@
import frappe
def execute():
#if current = 0, simply delete the key as it'll be recreated on first entry
frappe.db.sql('delete from `tabSeries` where current = 0')
duplicate_keys = frappe.db.sql('''
SELECT name, max(current) as current
from
`tabSeries`
group by
name
having count(name) > 1
''', as_dict=True)
for row in duplicate_keys:
frappe.db.sql('delete from `tabSeries` where name = %(key)s', {
'key': row.name
})
if row.current:
frappe.db.sql('insert into `tabSeries`(`name`, `current`) values (%(name)s, %(current)s)', row)
frappe.db.commit()
frappe.db.sql('ALTER table `tabSeries` ADD PRIMARY KEY IF NOT EXISTS (name)')
#if current = 0, simply delete the key as it'll be recreated on first entry
frappe.db.delete("Series", {"current": 0})
duplicate_keys = frappe.db.sql('''
SELECT name, max(current) as current
from
`tabSeries`
group by
name
having count(name) > 1
''', as_dict=True)
for row in duplicate_keys:
frappe.db.delete("Series", {
"name": row.name
})
if row.current:
frappe.db.sql('insert into `tabSeries`(`name`, `current`) values (%(name)s, %(current)s)', row)
frappe.db.commit()
frappe.db.sql('ALTER table `tabSeries` ADD PRIMARY KEY IF NOT EXISTS (name)')

4
frappe/patches/v12_0/setup_comments_from_communications.py
View file

@ -29,4 +29,6 @@ def execute():
frappe.db.auto_commit_on_many_writes = False
# clean up
frappe.db.sql("delete from `tabCommunication` where communication_type = 'Comment'")
frappe.db.delete("Communication", {
"communication_type": "Comment"
})

5
frappe/patches/v13_0/increase_password_length.py
View file

@ -1,7 +1,4 @@
import frappe
def execute():
frappe.db.multisql({
"mariadb": "ALTER TABLE `__Auth` MODIFY `password` TEXT NOT NULL",
"postgres": 'ALTER TABLE "__Auth" ALTER COLUMN "password" TYPE TEXT'
})
frappe.db.change_column_type(table="__Auth", column="password", type="TEXT")

4
frappe/patches/v13_0/remove_twilio_settings.py
View file

@ -12,7 +12,9 @@ def execute():
frappe.delete_doc_if_exists('DocType', 'Twilio Number Group')
if twilio_settings_doctype_in_integrations():
frappe.delete_doc_if_exists('DocType', 'Twilio Settings')
frappe.db.sql("delete from `tabSingles` where `doctype`=%s", 'Twilio Settings')
frappe.db.delete("Singles", {
"doctype": "Twilio Settings"
})
def twilio_settings_doctype_in_integrations() -> bool:
"""Check Twilio Settings doctype exists in integrations module or not.

27
frappe/patches/v13_0/rename_cancelled_docs.py
View file

@ -1,27 +0,0 @@
import frappe
from frappe.model.naming import NameParser
from frappe.model.rename_doc import rename_doc
def execute():
"""Rename already cancelled documents by adding `CAN-X` postfix instead of `-X`.
"""
for doctype in frappe.db.get_all('DocType'):
doctype = frappe.get_doc('DocType', doctype.name)
if doctype.is_submittable and frappe.db.table_exists(doctype.name):
cancelled_docs = frappe.db.get_all(doctype.name, ['amended_from', 'name'], {'docstatus':2})
for doc in cancelled_docs:
if '-CAN-' in doc.name:
continue
current_name = doc.name
if getattr(doc, "amended_from", None):
orig_name, counter = NameParser.parse_docname(doc.name)
else:
orig_name, counter = doc.name, 0
new_name = f'{orig_name}-CAN-{counter or 0}'
print(f"Renaming {doctype.name} record from {current_name} to {new_name}")
rename_doc(doctype.name, current_name, new_name, ignore_permissions=True, show_alert=False)
frappe.db.commit()

0
frappe/patches/v14_0/__init__.py Normal file
View file

22
frappe/patches/v14_0/drop_data_import_legacy.py Normal file
View file

@ -0,0 +1,22 @@
import frappe
import click
def execute():
doctype = "Data Import Legacy"
table = frappe.utils.get_table_name(doctype)
# delete the doctype record to avoid broken links
frappe.db.delete("DocType", {"name": doctype})
# leaving table in database for manual cleanup
click.secho(
f"`{doctype}` has been deprecated. The DocType is deleted, but the data still"
" exists on the database. If this data is worth recovering, you may export it"
f" using\n\n\tbench --site {frappe.local.site} backup -i '{doctype}'\n\nAfter"
" this, the table will continue to persist in the database, until you choose"
" to remove it yourself. If you want to drop the table, you may run\n\n\tbench"
f" --site {frappe.local.site} execute frappe.db.sql --args \"('DROP TABLE IF"
f" EXISTS `{table}`', )\"\n",
fg="yellow",
)

5
frappe/permissions.py
View file

@ -7,9 +7,11 @@ import frappe.share
from frappe import _, msgprint
from frappe.utils import cint
rights = ("select", "read", "write", "create", "delete", "submit", "cancel", "amend",
"print", "email", "report", "import", "export", "set_user_permissions", "share")
def check_admin_or_system_manager(user=None):
if not user: user = frappe.session.user
@ -516,8 +518,7 @@ def reset_perms(doctype):
"""Reset permissions for given doctype."""
from frappe.desk.notifications import delete_notification_count_for
delete_notification_count_for(doctype)
frappe.db.sql("""delete from `tabCustom DocPerm` where parent=%s""", doctype)
frappe.db.delete("Custom DocPerm", {"parent": doctype})
def get_linked_doctypes(dt):
return list(set([dt] + [d.options for d in

2
frappe/public/html/print_template.html
View file

@ -7,7 +7,7 @@
<meta name="description" content="">
<meta name="author" content="">
<title>{{ title }}</title>
<link href="{{ base_url }}{{ frappe.assets.bundled_asset('print.bundle.css') }}" rel="stylesheet">
<link href="{{ base_url }}{{ frappe.assets.bundled_asset('print.bundle.css', frappe.utils.is_rtl(lang)) }}" rel="stylesheet">
<style>
{{ print_css }}
</style>

2
frappe/public/js/frappe/form/dashboard.js
View file

@ -204,7 +204,7 @@ frappe.ui.form.Dashboard = class FormDashboard {
$(this).removeClass('hidden');
}
});
this.set_open_count();
!this.frm.is_new() && this.set_open_count();
}
init_data() {

1
frappe/public/js/frappe/form/footer/version_timeline_content_builder.js
View file

@ -152,6 +152,7 @@ function get_version_comment(version_doc, text) {
let unlinked_content = "";
try {
text += '</>';
Array.from($(text)).forEach(element => {
if ($(element).is('a')) {
version_comment += unlinked_content ? frappe.utils.get_form_link('Version', version_doc.name, true, unlinked_content) : "";

24
frappe/public/js/frappe/form/form.js
View file

@ -770,36 +770,32 @@ frappe.ui.form.Form = class FrappeForm {
}
_cancel(btn, callback, on_error, skip_confirm) {
const me = this;
const cancel_doc = () => {
frappe.validated = true;
this.script_manager.trigger("before_cancel").then(() => {
me.script_manager.trigger("before_cancel").then(() => {
if (!frappe.validated) {
return this.handle_save_fail(btn, on_error);
return me.handle_save_fail(btn, on_error);
}
const original_name = this.docname;
const after_cancel = (r) => {
var after_cancel = function(r) {
if (r.exc) {
this.handle_save_fail(btn, on_error);
me.handle_save_fail(btn, on_error);
} else {
frappe.utils.play_sound("cancel");
me.refresh();
callback && callback();
this.script_manager.trigger("after_cancel");
frappe.run_serially([
() => this.rename_notify(this.doctype, original_name, r.docs[0].name),
() => frappe.router.clear_re_route(this.doctype, original_name),
() => this.refresh(),
]);
me.script_manager.trigger("after_cancel");
}
};
frappe.ui.form.save(this, "cancel", after_cancel, btn);
frappe.ui.form.save(me, "cancel", after_cancel, btn);
});
}
if (skip_confirm) {
cancel_doc();
} else {
frappe.confirm(__("Permanently Cancel {0}?", [this.docname]), cancel_doc, this.handle_save_fail(btn, on_error));
frappe.confirm(__("Permanently Cancel {0}?", [this.docname]), cancel_doc, me.handle_save_fail(btn, on_error));
}
};
@ -821,7 +817,7 @@ frappe.ui.form.Form = class FrappeForm {
'docname': this.doc.name
}).then(is_amended => {
if (is_amended) {
frappe.throw(__('This document is already amended, you cannot amend it again'));
frappe.throw(__('This document is already amended, you cannot ammend it again'));
}
this.validate_form_action("Amend");
var me = this;

4
frappe/public/js/frappe/microtemplate.js
View file

@ -119,6 +119,10 @@ frappe.render_grid = function(opts) {
// render HTML wrapper page
opts.base_url = frappe.urllib.get_base_url();
opts.print_css = frappe.boot.print_css;
opts.lang = opts.lang || frappe.boot.lang,
opts.layout_direction = opts.layout_direction || frappe.utils.is_rtl() ? "rtl" : "ltr";
var html = frappe.render_template("print_template", opts);
var w = window.open();

9
frappe/public/js/frappe/recorder/RequestDetail.vue
View file

@ -64,7 +64,7 @@
<div class="grid-body">
<div class="rows">
<div class="grid-row" :class="showing == call.index ? 'grid-row-open' : ''" v-for="call in paginated(sorted(grouped(request.calls)))" :key="call.index">
<div class="data-row row" v-if="showing != call.index" style="display: block;" @click="showing = call.index" >
<div class="data-row row" @click="showing = showing == call.index ? null : call.index" >
<div class="row-index col col-xs-1"><span>{{ call.index }}</span></div>
<div class="col grid-static-col col-xs-6" data-fieldtype="Code">
<div class="static-area"><span>{{ call.query }}</span></div>
@ -76,16 +76,13 @@
<div class="static-area ellipsis text-right">{{ call.exact_copies }}</div>
</div>
<div class="col col-xs-1"><a class="close btn-open-row">
<span class="octicon octicon-triangle-down"></span></a>
<span class="octicon" :class="showing == call.index? 'octicon-triangle-up' : 'octicon-triangle-down'"></span></a>
</div>
</div>
<div class="recorder-form-in-grid" v-if="showing == call.index">
<div class="grid-form-heading" @click="showing = null">
<div class="toolbar grid-header-toolbar">
<span class="panel-title">{{ __("SQL Query") }} #<span class="grid-form-row-index">{{ call.index }}</span></span>
<div class="btn btn-default btn-xs pull-right" style="margin-left: 7px;">
<span class="hidden-xs octicon octicon-triangle-up"></span>
</div>
</div>
</div>
<div class="grid-form-body">
@ -116,7 +113,7 @@
</div>
<div class="frappe-control">
<div class="form-group">
<div class="clearfix"><label class="control-label"{{ __("Stack Trace") }}</label></div>
<div class="clearfix"><label class="control-label">{{ __("Stack Trace") }}</label></div>
<div class="control-value like-disabled-input for-description" style="overflow:auto">
<table class="table table-striped">
<thead>

18
frappe/public/js/frappe/router.js
View file

@ -51,7 +51,7 @@ $('body').on('click', 'a', function(e) {
return override('/app');
}
if (href.startsWith('#')) {
if (href && href.startsWith('#')) {
// target startswith "#", this is a v1 style route, so remake it.
return override(e.currentTarget.hash);
}
@ -169,10 +169,8 @@ frappe.router = {
standard_route = ['Tree', doctype_route.doctype];
} else {
standard_route = ['List', doctype_route.doctype, frappe.utils.to_title_case(route[2])];
if (route[3]) {
// calendar / kanban / dashboard / folder name
standard_route.push([...route].splice(3, route.length));
}
// calendar / kanban / dashboard / folder
if (route[3]) standard_route.push(...route.slice(3, route.length));
}
return standard_route;
},
@ -234,12 +232,6 @@ frappe.router = {
}
},
clear_re_route(doctype, docname) {
delete frappe.re_route[
`${encodeURIComponent(frappe.router.slug(doctype))}/${encodeURIComponent(docname)}`
];
},
set_title(sub_path) {
if (frappe.route_titles[sub_path]) {
frappe.utils.set_title(frappe.route_titles[sub_path]);
@ -251,7 +243,7 @@ frappe.router = {
// example 1: frappe.set_route('a', 'b', 'c');
// example 2: frappe.set_route(['a', 'b', 'c']);
// example 3: frappe.set_route('a/b/c');
let route = arguments;
let route = Array.from(arguments);
return new Promise(resolve => {
route = this.get_route_from_arguments(route);
@ -303,7 +295,7 @@ frappe.router = {
new_route = [this.slug(route[1]), 'view', route[2].toLowerCase()];
// calendar / inbox / file folder
if (route[3]) new_route.push([...route].slice(3, route.length));
if (route[3]) new_route.push(...route.slice(3, route.length));
} else {
if ($.isPlainObject(route[2])) {
frappe.route_options = route[2];

4
frappe/public/js/frappe/views/reports/print_grid.html
View file

@ -11,6 +11,7 @@
<!-- heading -->
<thead>
<tr>
<th> # </th>
{% for col in columns %}
{% if col.name && col._id !== "_check" %}
<th
@ -30,6 +31,9 @@
<tbody>
{% for row in data %}
<tr style="height: 30px">
<td {% if row.bold == 1 %} style="font-weight: bold" {% endif %}>
<span> {{ row._index + 1 }} </span>
</td>
{% for col in columns %}
{% if col.name && col._id !== "_check" %}

1
frappe/query_builder/__init__.py Normal file
View file

@ -0,0 +1 @@
from frappe.query_builder.utils import get_query_builder

55
frappe/query_builder/builder.py Normal file
View file

@ -0,0 +1,55 @@
from pypika import MySQLQuery, Order, PostgreSQLQuery, terms
from pypika.queries import Schema, Table
from frappe.utils import get_table_name
class Base:
terms = terms
desc = Order.desc
Schema = Schema
@staticmethod
def Table(table_name: str, *args, **kwargs) -> Table:
table_name = get_table_name(table_name)
return Table(table_name, *args, **kwargs)
class MariaDB(Base, MySQLQuery):
Field = terms.Field
@classmethod
def from_(cls, table, *args, **kwargs):
if isinstance(table, str):
table = cls.Table(table)
return super().from_(table, *args, **kwargs)
class Postgres(Base, PostgreSQLQuery):
field_translation = {"table_name": "relname", "table_rows": "n_tup_ins"}
schema_translation = {"tables": "pg_stat_all_tables"}
# TODO: Find a better way to do this
# These are interdependent query changes that need fixing. These
# translations happen in the same query. But there is no check to see if
# the Fields are changed only when a particular `information_schema` schema
# is used. Replacing them is not straightforward because the "from_"
# function can not see the arguments passed to the "select" function as
# they are two different objects. The quick fix used here is to replace the
# Field names in the "Field" function.
@classmethod
def Field(cls, field_name, *args, **kwargs):
if field_name in cls.field_translation:
field_name = cls.field_translation[field_name]
return terms.Field(field_name, *args, **kwargs)
@classmethod
def from_(cls, table, *args, **kwargs):
if isinstance(table, Table):
if table._schema:
if table._schema._name == "information_schema":
table = cls.schema_translation[table._table_name]
elif isinstance(table, str):
table = cls.Table(table)
return super().from_(table, *args, **kwargs)

83
frappe/query_builder/custom.py Normal file
View file

@ -0,0 +1,83 @@
from typing import Optional
from pypika.functions import DistinctOptionFunction
from pypika.utils import builder
import frappe
class GROUP_CONCAT(DistinctOptionFunction):
def __init__(self, column: str, alias: Optional[str] = None):
"""[ Implements the group concat function read more about it at https://www.geeksforgeeks.org/mysql-group_concat-function ]
Args:
column (str): [ name of the column you want to concat]
alias (Optional[str], optional): [ is this an alias? ]. Defaults to None.
"""
super(GROUP_CONCAT, self).__init__("GROUP_CONCAT", column, alias=alias)
class STRING_AGG(DistinctOptionFunction):
def __init__(self, column: str, separator: str = ",", alias: Optional[str] = None):
"""[ Implements the group concat function read more about it at https://docs.microsoft.com/en-us/sql/t-sql/functions/string-agg-transact-sql?view=sql-server-ver15 ]
Args:
column (str): [ name of the column you want to concat ]
separator (str, optional): [separator to be used]. Defaults to ",".
alias (Optional[str], optional): [description]. Defaults to None.
"""
super(STRING_AGG, self).__init__("STRING_AGG", column, separator, alias=alias)
class MATCH(DistinctOptionFunction):
def __init__(self, column: str, *args, **kwargs):
"""[ Implementation of Match Against read more about it https://dev.mysql.com/doc/refman/8.0/en/fulltext-search.html#function_match ]
Args:
column (str):[ column to search in ]
"""
alias = kwargs.get("alias")
super(MATCH, self).__init__(" MATCH", column, *args, alias=alias)
self._Against = False
def get_function_sql(self, **kwargs):
s = super(DistinctOptionFunction, self).get_function_sql(**kwargs)
if self._Against:
return f"{s} AGAINST ({frappe.db.escape(f'+{self._Against}*')} IN BOOLEAN MODE)"
return s
@builder
def Against(self, text: str):
"""[ Text that has to be searched against ]
Args:
text (str): [ the text string that we match it against ]
"""
self._Against = text
class TO_TSVECTOR(DistinctOptionFunction):
def __init__(self, column: str, *args, **kwargs):
"""[ Implementation of TO_TSVECTOR read more about it https://www.postgresql.org/docs/9.1/textsearch-controls.html]
Args:
column (str): [ column to search in ]
"""
alias = kwargs.get("alias")
super(TO_TSVECTOR, self).__init__("TO_TSVECTOR", column, *args, alias=alias)
self._PLAINTO_TSQUERY = False
def get_function_sql(self, **kwargs):
s = super(DistinctOptionFunction, self).get_function_sql(**kwargs)
if self._PLAINTO_TSQUERY:
return f"{s} @@ PLAINTO_TSQUERY({frappe.db.escape(self._PLAINTO_TSQUERY)})"
return s
@builder
def Against(self, text: str):
"""[ Text that has to be searched against ]
Args:
text (str): [ the text string that we match it against ]
"""
self._PLAINTO_TSQUERY = text

17
frappe/query_builder/functions.py Normal file
View file

@ -0,0 +1,17 @@
from pypika.functions import *
from frappe.query_builder.utils import ImportMapper, db_type_is
from frappe.query_builder.custom import GROUP_CONCAT, STRING_AGG, MATCH, TO_TSVECTOR
GroupConcat = ImportMapper(
{
db_type_is.MARIADB: GROUP_CONCAT,
db_type_is.POSTGRES: STRING_AGG
}
)
Match = ImportMapper(
{
db_type_is.MARIADB: MATCH,
db_type_is.POSTGRES: TO_TSVECTOR
}
)

34
frappe/query_builder/utils.py Normal file
View file

@ -0,0 +1,34 @@
from enum import Enum
from typing import Any, Callable, Dict
from pypika import Query
import frappe
from .builder import MariaDB, Postgres
class db_type_is(Enum):
MARIADB = "mariadb"
POSTGRES = "postgres"
class ImportMapper:
def __init__(self, func_map: Dict[db_type_is, Callable]) -> None:
self.func_map = func_map
def __call__(self, *args: Any, **kwds: Any) -> Callable:
db = db_type_is(frappe.conf.db_type or "mariadb")
return self.func_map[db](*args, **kwds)
def get_query_builder(type_of_db: str) -> Query:
"""[return the query builder object]
Args:
type_of_db (str): [string value of the db used]
Returns:
Query: [Query object]
"""
db = db_type_is(type_of_db)
picks = {db_type_is.MARIADB: MariaDB, db_type_is.POSTGRES: Postgres}
return picks[db]

21
frappe/search/website_search.py
View file

@ -35,10 +35,12 @@ class WebsiteSearch(FullTextSearch):
if getattr(self, "_items_to_index", False):
return self._items_to_index
routes = get_static_pages_from_all_apps() + slugs_with_web_view()
self._items_to_index = []
routes = get_static_pages_from_all_apps() + slugs_with_web_view(self._items_to_index)
for i, route in enumerate(routes):
update_progress_bar("Retrieving Routes", i, len(routes))
self._items_to_index += [self.get_document_to_index(route)]
@ -85,16 +87,23 @@ class WebsiteSearch(FullTextSearch):
)
def slugs_with_web_view():
def slugs_with_web_view(_items_to_index):
all_routes = []
filters = { "has_web_view": 1, "allow_guest_to_view": 1, "index_web_pages_for_search": 1}
fields = ["name", "is_published_field"]
fields = ["name", "is_published_field", 'website_search_field']
doctype_with_web_views = frappe.get_all("DocType", filters=filters, fields=fields)
for doctype in doctype_with_web_views:
if doctype.is_published_field:
routes = frappe.get_all(doctype.name, filters={doctype.is_published_field: 1}, fields="route")
all_routes += [route.route for route in routes]
docs = frappe.get_all(doctype.name, filters={doctype.is_published_field: 1}, fields=["route", doctype.website_search_field, 'title'])
if doctype.website_search_field:
for doc in docs:
content = frappe.utils.md_to_html(getattr(doc, doctype.website_search_field))
soup = BeautifulSoup(content, "html.parser")
text_content = soup.text if soup else ""
_items_to_index += [frappe._dict(title=doc.title, content=text_content, path=doc.route)]
else:
all_routes += [route.route for route in docs]
return all_routes

2
frappe/sessions.py
View file

@ -84,7 +84,7 @@ def delete_session(sid=None, user=None, reason="Session Expired"):
if user_details: user = user_details[0].get("user")
logout_feed(user, reason)
frappe.db.sql("""delete from tabSessions where sid=%s""", sid)
frappe.db.delete("Sessions", {"sid": sid})
frappe.db.commit()
def clear_all_sessions(reason=None):

1
frappe/test_runner.py
View file

@ -56,6 +56,7 @@ def main(app=None, module=None, doctype=None, verbose=False, tests=(),
frappe.clear_cache()
frappe.utils.scheduler.disable_scheduler()
set_test_email_config()
frappe.conf.update({'bench_id': 'test_bench', 'use_rq_auth': False})
if not frappe.flags.skip_before_tests:
if verbose:

6
frappe/tests/test_background_jobs.py
View file

@ -4,7 +4,7 @@ from rq import Queue
import frappe
from frappe.core.page.background_jobs.background_jobs import remove_failed_jobs
from frappe.utils.background_jobs import get_redis_conn
from frappe.utils.background_jobs import get_redis_conn, generate_qname
import time
@ -17,14 +17,14 @@ class TestBackgroundJobs(unittest.TestCase):
queues = Queue.all(conn)
for queue in queues:
if queue.name == "short":
if queue.name == generate_qname("short"):
fail_registry = queue.failed_job_registry
self.assertGreater(fail_registry.count, 0)
remove_failed_jobs()
for queue in queues:
if queue.name == "short":
if queue.name == generate_qname("short"):
fail_registry = queue.failed_job_registry
self.assertEqual(fail_registry.count, 0)

2
frappe/tests/test_commands.py
View file

@ -433,6 +433,6 @@ class TestCommands(BaseTestCommands):
for output in ["legacy", "plain", "table", "json"]:
self.execute(f"bench version -f {output}")
self.assertEqual(self.returncode, 0)
self.execute("bench version -f invalid")
self.assertEqual(self.returncode, 2)

99
frappe/tests/test_db.py
View file

@ -12,6 +12,8 @@ from frappe.custom.doctype.custom_field.custom_field import create_custom_field
from frappe.utils import random_string
from frappe.utils.testutils import clear_custom_fields
from .test_query_builder import run_only_if, db_type_is
class TestDB(unittest.TestCase):
def test_get_value(self):
@ -146,7 +148,7 @@ class TestDB(unittest.TestCase):
# Create documents under that doctype and query them via ORM
for _ in range(10):
docfields = { key.lower(): random_string(10) for key in fields }
docfields = {key.lower(): random_string(10) for key in fields}
doc = frappe.get_doc({"doctype": test_doctype, "description": random_string(20), **docfields})
doc.insert()
created_docs.append(doc.name)
@ -189,3 +191,98 @@ class TestDB(unittest.TestCase):
for doc in created_docs:
frappe.delete_doc(test_doctype, doc)
clear_custom_fields(test_doctype)
@run_only_if(db_type_is.MARIADB)
class TestDDLCommandsMaria(unittest.TestCase):
test_table_name = "TestNotes"
def setUp(self) -> None:
frappe.db.commit()
frappe.db.sql(
f"""
CREATE TABLE `tab{self.test_table_name}` (`id` INT NULL,PRIMARY KEY (`id`));
"""
)
def tearDown(self) -> None:
frappe.db.sql(f"DROP TABLE tab{self.test_table_name};")
self.test_table_name = "TestNotes"
def test_rename(self) -> None:
new_table_name = f"{self.test_table_name}_new"
frappe.db.rename_table(self.test_table_name, new_table_name)
check_exists = frappe.db.sql(
f"""
SELECT * FROM INFORMATION_SCHEMA.TABLES
WHERE TABLE_NAME = N'tab{new_table_name}';
"""
)
self.assertGreater(len(check_exists), 0)
self.assertIn(f"tab{new_table_name}", check_exists[0])
# * so this table is deleted after the rename
self.test_table_name = new_table_name
def test_describe(self) -> None:
self.assertEqual(
(("id", "int(11)", "NO", "PRI", None, ""),),
frappe.db.describe(self.test_table_name),
)
def test_change_type(self) -> None:
frappe.db.change_column_type("TestNotes", "id", "varchar(255)")
test_table_description = frappe.db.sql(f"DESC tab{self.test_table_name};")
self.assertGreater(len(test_table_description), 0)
self.assertIn("varchar(255)", test_table_description[0])
@run_only_if(db_type_is.POSTGRES)
class TestDDLCommandsPost(unittest.TestCase):
test_table_name = "TestNotes"
def setUp(self) -> None:
frappe.db.sql(
f"""
CREATE TABLE "tab{self.test_table_name}" ("id" INT NULL,PRIMARY KEY ("id"))
"""
)
def tearDown(self) -> None:
frappe.db.sql(f'DROP TABLE "tab{self.test_table_name}"')
self.test_table_name = "TestNotes"
def test_rename(self) -> None:
new_table_name = f"{self.test_table_name}_new"
frappe.db.rename_table(self.test_table_name, new_table_name)
check_exists = frappe.db.sql(
f"""
SELECT EXISTS (
SELECT FROM information_schema.tables
WHERE table_name = 'tab{new_table_name}'
);
"""
)
self.assertTrue(check_exists[0][0])
# * so this table is deleted after the rename
self.test_table_name = new_table_name
def test_describe(self) -> None:
self.assertEqual([("id",)], frappe.db.describe(self.test_table_name))
def test_change_type(self) -> None:
frappe.db.change_column_type(self.test_table_name, "id", "varchar(255)")
check_change = frappe.db.sql(
f"""
SELECT
table_name,
column_name,
data_type
FROM
information_schema.columns
WHERE
table_name = 'tab{self.test_table_name}'
"""
)
self.assertGreater(len(check_change), 0)
self.assertIn("character varying", check_change[0])

34
frappe/tests/test_naming.py
View file

@ -116,37 +116,3 @@ class TestNaming(unittest.TestCase):
self.assertEqual(current_index.get('current'), 2)
frappe.db.sql("""delete from `tabSeries` where name = %s""", series)
def test_naming_for_cancelled_and_amended_doc(self):
submittable_doctype = frappe.get_doc({
"doctype": "DocType",
"module": "Core",
"custom": 1,
"is_submittable": 1,
"permissions": [{
"role": "System Manager",
"read": 1
}],
"name": 'Submittable Doctype'
}).insert(ignore_if_duplicate=True)
doc = frappe.new_doc('Submittable Doctype')
doc.save()
original_name = doc.name
doc.submit()
doc.cancel()
cancelled_name = doc.name
self.assertEqual(cancelled_name, "{}-CAN-0".format(original_name))
amended_doc = frappe.copy_doc(doc)
amended_doc.docstatus = 0
amended_doc.amended_from = doc.name
amended_doc.save()
self.assertEqual(amended_doc.name, original_name)
amended_doc.submit()
amended_doc.cancel()
self.assertEqual(amended_doc.name, "{}-CAN-1".format(original_name))
submittable_doctype.delete()

74
frappe/tests/test_query_builder.py Normal file
View file

@ -0,0 +1,74 @@
import unittest
from typing import Callable
import frappe
from frappe.query_builder.functions import GroupConcat, Match
from frappe.query_builder.utils import db_type_is
def run_only_if(dbtype: db_type_is) -> Callable:
return unittest.skipIf(
db_type_is(frappe.conf.db_type) != dbtype, f"Only runs for {dbtype.value}"
)
@run_only_if(db_type_is.MARIADB)
class TestCustomFunctionsMariaDB(unittest.TestCase):
def test_concat(self):
self.assertEqual("GROUP_CONCAT('Notes')", GroupConcat("Notes").get_sql())
def test_match(self):
query = Match("Notes").Against("text")
self.assertEqual(
" MATCH('Notes') AGAINST ('+text*' IN BOOLEAN MODE)", query.get_sql()
)
@run_only_if(db_type_is.POSTGRES)
class TestCustomFunctionsPostgres(unittest.TestCase):
def test_concat(self):
self.assertEqual("STRING_AGG('Notes',',')", GroupConcat("Notes").get_sql())
def test_match(self):
query = Match("Notes").Against("text")
self.assertEqual(
"TO_TSVECTOR('Notes') @@ PLAINTO_TSQUERY('text')", query.get_sql()
)
class TestBuilderBase(object):
def test_adding_tabs(self):
self.assertEqual("tabNotes", frappe.qb.Table("Notes").get_sql())
self.assertEqual("__Auth", frappe.qb.Table("__Auth").get_sql())
@run_only_if(db_type_is.MARIADB)
class TestBuilderMaria(unittest.TestCase, TestBuilderBase):
def test_adding_tabs_in_from(self):
self.assertEqual(
"SELECT * FROM `tabNotes`", frappe.qb.from_("Notes").select("*").get_sql()
)
self.assertEqual(
"SELECT * FROM `__Auth`", frappe.qb.from_("__Auth").select("*").get_sql()
)
@run_only_if(db_type_is.POSTGRES)
class TestBuilderPostgres(unittest.TestCase, TestBuilderBase):
def test_adding_tabs_in_from(self):
self.assertEqual(
'SELECT * FROM "tabNotes"', frappe.qb.from_("Notes").select("*").get_sql()
)
self.assertEqual(
'SELECT * FROM "__Auth"', frappe.qb.from_("__Auth").select("*").get_sql()
)
def test_replace_tables(self):
info_schema = frappe.qb.Schema("information_schema")
self.assertEqual(
'SELECT * FROM "pg_stat_all_tables"',
frappe.qb.from_(info_schema.tables).select("*").get_sql(),
)
def test_replace_fields_post(self):
self.assertEqual("relname", frappe.qb.Field("table_name").get_sql())

70
frappe/tests/test_redis.py Normal file
View file

@ -0,0 +1,70 @@
import unittest
import functools
import redis
import frappe
from frappe.utils import get_bench_id
from frappe.utils.rq import RedisQueue
from frappe.utils.background_jobs import get_redis_conn
def version_tuple(version):
return tuple(map(int, (version.split("."))))
def skip_if_redis_version_lt(version):
def decorator(func):
@functools.wraps(func)
def wrapper(*args, **kwargs):
conn = get_redis_conn()
redis_version = conn.execute_command('info')['redis_version']
if version_tuple(redis_version) < version_tuple(version):
return
return func(*args, **kwargs)
return wrapper
return decorator
class TestRedisAuth(unittest.TestCase):
@skip_if_redis_version_lt('6.0')
def test_rq_gen_acllist(self):
"""Make sure that ACL list is genrated
"""
acl_list = RedisQueue.gen_acl_list()
self.assertEqual(acl_list[1]['bench'][0], get_bench_id())
@skip_if_redis_version_lt('6.0')
def test_adding_redis_user(self):
acl_list = RedisQueue.gen_acl_list()
username, password = acl_list[1]['bench']
conn = get_redis_conn()
conn.acl_deluser(username)
_ = RedisQueue(conn).add_user(username, password)
self.assertTrue(conn.acl_getuser(username))
conn.acl_deluser(username)
@skip_if_redis_version_lt('6.0')
def test_rq_namespace(self):
"""Make sure that user can access only their respective namespace.
"""
# Current bench ID
bench_id = frappe.conf.get('bench_id')
conn = get_redis_conn()
conn.set('rq:queue:test_bench1:abc', 'value')
conn.set(f'rq:queue:{bench_id}:abc', 'value')
# Create new Redis Queue user
tmp_bench_id = 'test_bench1'
username, password = tmp_bench_id, 'password1'
conn.acl_deluser(username)
frappe.conf.update({'bench_id': tmp_bench_id})
_ = RedisQueue(conn).add_user(username, password)
test_bench1_conn = RedisQueue.get_connection(username, password)
self.assertEqual(test_bench1_conn.get('rq:queue:test_bench1:abc'), b'value')
# User should not be able to access queues apart from their bench queues
with self.assertRaises(redis.exceptions.NoPermissionError):
test_bench1_conn.get(f'rq:queue:{bench_id}:abc')
frappe.conf.update({'bench_id': bench_id})
conn.acl_deluser(username)

9
frappe/utils/__init__.py
View file

@ -383,6 +383,12 @@ def get_files_path(*path, **kwargs):
def get_bench_path():
return os.path.realpath(os.path.join(os.path.dirname(frappe.__file__), '..', '..', '..'))
def get_bench_id():
return frappe.get_conf().get('bench_id', get_bench_path().strip('/').replace('/', '-'))
def get_site_id(site=None):
return f"{site or frappe.local.site}@{get_bench_id()}"
def get_backups_path():
return get_site_path("private", "backups")
@ -843,3 +849,6 @@ def groupby_metric(iterable: typing.Dict[str, list], key: str):
for item in items:
records.setdefault(item[key], {}).setdefault(category, []).append(item)
return records
def get_table_name(table_name: str) -> str:
return f"tab{table_name}" if not table_name.startswith("__") else table_name

84
frappe/utils/background_jobs.py
View file

@ -1,13 +1,21 @@
import os
import socket
import time
from uuid import uuid4
from collections import defaultdict
import redis
from typing import List
from rq import Connection, Queue, Worker
from rq.logutils import setup_loghandlers
from frappe.utils import cstr
from collections import defaultdict
import frappe
import os, socket, time
from frappe import _
from uuid import uuid4
import frappe.monitor
from frappe.utils import cstr, get_bench_id
from frappe.utils.rq import RedisQueue
from frappe.utils.commands import log
default_timeout = 300
@ -131,21 +139,22 @@ def execute_job(site, method, event, job_name, kwargs, user=None, is_async=True,
if is_async:
frappe.destroy()
def start_worker(queue=None, quiet = False):
def start_worker(queue=None, quiet = False, rq_username=None, rq_password=None):
'''Wrapper to start rq worker. Connects to redis and monitors these queues.'''
with frappe.init_site():
# empty init is required to get redis_queue from common_site_config.json
redis_connection = get_redis_conn()
redis_connection = get_redis_conn(username=rq_username, password=rq_password)
queues = get_queue_list(queue, build_queue_name=True)
queue_name = queue and generate_qname(queue)
if os.environ.get('CI'):
setup_loghandlers('ERROR')
with Connection(redis_connection):
queues = get_queue_list(queue)
logging_level = "INFO"
if quiet:
logging_level = "WARNING"
Worker(queues, name=get_worker_name(queue)).work(logging_level = logging_level)
Worker(queues, name=get_worker_name(queue_name)).work(logging_level = logging_level)
def get_worker_name(queue):
'''When limiting worker to a specific queue, also append queue name to default worker name'''
@ -186,7 +195,7 @@ def get_jobs(site=None, queue=None, key='method'):
return jobs_per_site
def get_queue_list(queue_list=None):
def get_queue_list(queue_list=None, build_queue_name=False):
'''Defines possible queues. Also wraps a given queue in a list after validating.'''
default_queue_list = list(queue_timeout)
if queue_list:
@ -195,11 +204,9 @@ def get_queue_list(queue_list=None):
for queue in queue_list:
validate_queue(queue, default_queue_list)
return queue_list
else:
return default_queue_list
queue_list = default_queue_list
return [generate_qname(qtype) for qtype in queue_list] if build_queue_name else queue_list
def get_workers(queue):
'''Returns a list of Worker objects tied to a queue object'''
@ -215,10 +222,10 @@ def get_running_jobs_in_queue(queue):
jobs.append(current_job)
return jobs
def get_queue(queue, is_async=True):
def get_queue(qtype, is_async=True):
'''Returns a Queue object tied to a redis connection'''
validate_queue(queue)
return Queue(queue, connection=get_redis_conn(), is_async=is_async)
validate_queue(qtype)
return Queue(generate_qname(qtype), connection=get_redis_conn(), is_async=is_async)
def validate_queue(queue, default_queue_list=None):
if not default_queue_list:
@ -227,7 +234,7 @@ def validate_queue(queue, default_queue_list=None):
if queue not in default_queue_list:
frappe.throw(_("Queue should be one of {0}").format(', '.join(default_queue_list)))
def get_redis_conn():
def get_redis_conn(username=None, password=None):
if not hasattr(frappe.local, 'conf'):
raise Exception('You need to call frappe.init')
@ -236,11 +243,50 @@ def get_redis_conn():
global redis_connection
if not redis_connection:
redis_connection = redis.from_url(frappe.local.conf.redis_queue)
cred = frappe._dict()
if frappe.conf.get('use_rq_auth'):
if username:
cred['username'] = username
cred['password'] = password
else:
cred['username'] = frappe.get_site_config().rq_username or get_bench_id()
cred['password'] = frappe.get_site_config().rq_password
elif os.environ.get('RQ_ADMIN_PASWORD'):
cred['username'] = 'default'
cred['password'] = os.environ.get('RQ_ADMIN_PASWORD')
try:
redis_connection = RedisQueue.get_connection(**cred)
except (redis.exceptions.AuthenticationError, redis.exceptions.ResponseError):
log(f'Wrong credentials used for {cred.username or "default user"}. '
'You can reset credentials using `bench create-rq-users` CLI and restart the server',
colour='red')
raise
except Exception:
log(f'Please make sure that Redis Queue runs @ {frappe.get_conf().redis_queue}', colour='red')
raise
return redis_connection
def get_queues() -> List[Queue]:
"""Get all the queues linked to the current bench.
"""
queues = Queue.all(connection=get_redis_conn())
return [q for q in queues if is_queue_accessible(q)]
def generate_qname(qtype: str) -> str:
"""Generate qname by combining bench ID and queue type.
qnames are useful to define namespaces of customers.
"""
return f"{get_bench_id()}:{qtype}"
def is_queue_accessible(qobj: Queue) -> bool:
"""Checks whether queue is relate to current bench or not.
"""
accessible_queues = [generate_qname(q) for q in list(queue_timeout)]
return qobj.name in accessible_queues
def enqueue_test_job():
enqueue('frappe.utils.background_jobs.test_job', s=100)

12
frappe/utils/backups.py
View file

@ -116,16 +116,16 @@ class BackupGenerator:
def setup_backup_tables(self):
"""Sets self.backup_includes, self.backup_excludes based on passed args"""
existing_doctypes = set([x.name for x in frappe.get_all("DocType")])
existing_tables = frappe.db.get_tables()
def get_tables(doctypes):
tables = []
for doctype in doctypes:
if doctype and doctype in existing_doctypes:
if doctype.startswith("tab"):
tables.append(doctype)
else:
tables.append("tab" + doctype)
if not doctype:
continue
table = frappe.utils.get_table_name(doctype)
if table in existing_tables:
tables.append(table)
return tables
passed_tables = {

1
frappe/utils/error.py
View file

@ -176,6 +176,7 @@ def collect_error_snapshots():
def clear_old_snapshots():
"""Clear snapshots that are older than a month"""
frappe.db.sql("""delete from `tabError Snapshot`
where creation < (NOW() - INTERVAL '1' MONTH)""")

74
frappe/utils/global_search.py
View file

@ -23,8 +23,7 @@ def reset():
Deletes all data in __global_search
:return:
"""
frappe.db.sql('DELETE FROM `__global_search`')
frappe.db.delete("__global_search")
def get_doctypes_with_global_search(with_child_tables=True):
"""
@ -146,10 +145,9 @@ def rebuild_for_doctype(doctype):
def delete_global_search_records_for_doctype(doctype):
frappe.db.sql('''DELETE
FROM `__global_search`
WHERE doctype = %s''', doctype, as_dict=True)
frappe.db.delete("__global_search", {
"doctype": doctype
})
def get_selected_fields(meta, global_search_fields):
fieldnames = [df.fieldname for df in global_search_fields]
@ -399,12 +397,10 @@ def delete_for_document(doc):
been deleted
:param doc: Deleted document
"""
frappe.db.sql('''DELETE
FROM `__global_search`
WHERE doctype = %s
AND name = %s''', (doc.doctype, doc.name), as_dict=True)
frappe.db.delete("__global_search", {
"doctype": doc.doctype,
"name": doc.name
})
@frappe.whitelist()
def search(text, start=0, limit=20, doctype=""):
@ -415,51 +411,41 @@ def search(text, start=0, limit=20, doctype=""):
:param limit: number of results to return, default 20
:return: Array of result objects
"""
from frappe.desk.doctype.global_search_settings.global_search_settings import get_doctypes_for_global_search
from frappe.desk.doctype.global_search_settings.global_search_settings import (
get_doctypes_for_global_search,
)
from frappe.query_builder.functions import Match
results = []
sorted_results = []
allowed_doctypes = get_doctypes_for_global_search()
for text in set(text.split('&')):
for text in set(text.split("&")):
text = text.strip()
if not text:
continue
conditions = '1=1'
offset = ''
mariadb_text = frappe.db.escape('+' + text + '*')
mariadb_fields = '`doctype`, `name`, `content`, MATCH (`content`) AGAINST ({} IN BOOLEAN MODE) AS rank'.format(mariadb_text)
postgres_fields = '`doctype`, `name`, `content`, TO_TSVECTOR("content") @@ PLAINTO_TSQUERY({}) AS rank'.format(frappe.db.escape(text))
values = {}
global_search = frappe.qb.Table("__global_search")
rank = Match(global_search.content).Against(text).as_("rank")
query = (
frappe.qb.from_(global_search)
.select(
global_search.doctype, global_search.name, global_search.content, rank
)
.orderby("rank", order=frappe.qb.desc)
.limit(limit)
)
if doctype:
conditions = '`doctype` = %(doctype)s'
values['doctype'] = doctype
query = query.where(global_search.doctype == doctype)
elif allowed_doctypes:
conditions = '`doctype` IN %(allowed_doctypes)s'
values['allowed_doctypes'] = tuple(allowed_doctypes)
query = query.where(global_search.doctype.isin(allowed_doctypes))
if int(start) > 0:
offset = 'OFFSET {}'.format(start)
if start > 0:
query = query.offset(start)
common_query = """
SELECT {fields}
FROM `__global_search`
WHERE {conditions}
ORDER BY rank DESC
LIMIT {limit}
{offset}
"""
result = frappe.db.multisql({
'mariadb': common_query.format(fields=mariadb_fields, conditions=conditions, limit=limit, offset=offset),
'postgres': common_query.format(fields=postgres_fields, conditions=conditions, limit=limit, offset=offset)
}, values=values, as_dict=True)
result = frappe.db.sql(query, as_dict=True)
results.extend(result)
@ -470,7 +456,9 @@ def search(text, start=0, limit=20, doctype=""):
try:
meta = frappe.get_meta(r.doctype)
if meta.image_field:
r.image = frappe.db.get_value(r.doctype, r.name, meta.image_field)
r.image = frappe.db.get_value(
r.doctype, r.name, meta.image_field
)
except Exception:
frappe.clear_messages()

6
frappe/utils/install.py
View file

@ -111,9 +111,9 @@ def before_tests():
# don't run before tests if any other app is installed
return
frappe.db.sql("delete from `tabCustom Field`")
frappe.db.sql("delete from `tabEvent`")
frappe.db.commit()
frappe.db.truncate("Custom Field")
frappe.db.truncate("Event")
frappe.clear_cache()
# complete setup if missing

6
frappe/utils/nestedset.py
View file

@ -57,7 +57,7 @@ def update_add_node(doc, parent, parent_field):
# get the last sibling of the parent
if parent:
left, right = frappe.db.sql("select lft, rgt from `tab{0}` where name=%s"
left, right = frappe.db.sql("select lft, rgt from `tab{0}` where name=%s for update"
.format(doctype), parent)[0]
validate_loop(doc.doctype, doc.name, left, right)
else: # root
@ -89,7 +89,7 @@ def update_move_node(doc, parent_field):
if parent:
new_parent = frappe.db.sql("""select lft, rgt from `tab{0}`
where name = %s""".format(doc.doctype), parent, as_dict=1)[0]
where name = %s for update""".format(doc.doctype), parent, as_dict=1)[0]
validate_loop(doc.doctype, doc.name, new_parent.lft, new_parent.rgt)
@ -108,7 +108,7 @@ def update_move_node(doc, parent_field):
if parent:
new_parent = frappe.db.sql("""select lft, rgt from `tab%s`
where name = %s""" % (doc.doctype, '%s'), parent, as_dict=1)[0]
where name = %s for update""" % (doc.doctype, '%s'), parent, as_dict=1)[0]
# set parent lft, rgt

16
frappe/utils/password.py
View file

@ -65,11 +65,11 @@ def set_encrypted_password(doctype, name, pwd, fieldname='password'):
def remove_encrypted_password(doctype, name, fieldname='password'):
frappe.db.sql(
'DELETE FROM `__Auth` WHERE doctype = %s and name = %s and fieldname = %s',
values=[doctype, name, fieldname]
)
frappe.db.delete("__Auth", {
"doctype": doctype,
"name": name,
"fieldname": fieldname
})
def check_password(user, pwd, doctype='User', fieldname='password', delete_tracker_cache=True):
'''Checks if user and password are correct, else raises frappe.AuthenticationError'''
@ -131,8 +131,10 @@ def update_password(user, pwd, doctype='User', fieldname='password', logout_all_
def delete_all_passwords_for(doctype, name):
try:
frappe.db.sql("""delete from `__Auth` where `doctype`=%(doctype)s and `name`=%(name)s""",
{ 'doctype': doctype, 'name': name })
frappe.db.delete("__Auth", {
"doctype": doctype,
"name": name
})
except Exception as e:
if not frappe.db.is_missing_column(e):
raise

83
frappe/utils/rq.py Normal file
View file

@ -0,0 +1,83 @@
import redis
import frappe
from frappe.utils import get_bench_id, random_string
class RedisQueue:
def __init__(self, conn):
self.conn = conn
def add_user(self, username, password=None):
"""Create or update the user.
"""
password = password or self.conn.acl_genpass()
user_settings = self.get_new_user_settings(username, password)
is_created = self.conn.acl_setuser(**user_settings)
return frappe._dict(user_settings) if is_created else {}
@classmethod
def get_connection(cls, username=None, password=None):
rq_url = frappe.local.conf.redis_queue
domain = rq_url.split("redis://", 1)[-1]
url = (username and f"redis://{username}:{password or ''}@{domain}") or rq_url
conn = redis.from_url(url)
conn.ping()
return conn
@classmethod
def new(cls, username='default', password=None):
return cls(cls.get_connection(username, password))
@classmethod
def set_admin_password(cls, cur_password=None, new_password=None, reset_passwords=False):
username = 'default'
conn = cls.get_connection(username, cur_password)
password = '+'+(new_password or conn.acl_genpass())
conn.acl_setuser(
username=username, enabled=True, reset_passwords=reset_passwords, passwords=password
)
return password[1:]
@classmethod
def get_new_user_settings(cls, username, password):
d = {}
d['username'] = username
d['passwords'] = '+'+password
d['reset_keys'] = True
d['enabled'] = True
d['keys'] = cls.get_acl_key_rules()
d['commands'] = cls.get_acl_command_rules()
return d
@classmethod
def get_acl_key_rules(cls, include_key_prefix=False):
"""FIXME: Find better way
"""
rules = ['rq:[^q]*', 'rq:queues', f'rq:queue:{get_bench_id()}:*']
if include_key_prefix:
return ['~'+pattern for pattern in rules]
return rules
@classmethod
def get_acl_command_rules(cls):
return ['+@all', '-@admin']
@classmethod
def gen_acl_list(cls, set_admin_password=False):
"""Generate list of ACL users needed for this branch.
This list contains default ACL user and the bench ACL user(used by all sites incase of ACL is enabled).
"""
bench_username = get_bench_id()
bench_user_rules = cls.get_acl_key_rules(include_key_prefix=True) + cls.get_acl_command_rules()
bench_user_rule_str = ' '.join(bench_user_rules).strip()
bench_user_password = random_string(20)
default_username = 'default'
_default_user_password = random_string(20) if set_admin_password else ''
default_user_password = '>'+_default_user_password if _default_user_password else 'nopass'
return [
f'user {default_username} on {default_user_password} ~* &* +@all',
f'user {bench_username} on >{bench_user_password} {bench_user_rule_str}'
], {'bench': (bench_username, bench_user_password), 'default': (default_username, _default_user_password)}

2
frappe/utils/testutils.py
View file

@ -12,5 +12,5 @@ def add_custom_field(doctype, fieldname, fieldtype='Data', options=None):
}).insert()
def clear_custom_fields(doctype):
frappe.db.sql('delete from `tabCustom Field` where dt=%s', doctype)
frappe.db.delete("Custom Field", {"dt": doctype})
frappe.clear_cache(doctype=doctype)

2
frappe/website/doctype/web_form/web_form.py
View file

@ -542,7 +542,7 @@ def get_form_data(doctype, docname=None, web_form_name=None):
# For Table fields, server-side processing for meta
for field in out.web_form.web_form_fields:
if field.fieldtype == "Table":
field.fields = get_in_list_view_fields(field.options)
field.fields = frappe.get_meta(field.options).fields
out.update({field.fieldname: field.fields})
if field.fieldtype == "Link":

45
frappe/website/report/website_analytics/website_analytics.py
View file

@ -1,11 +1,14 @@
# Copyright (c) 2013, Frappe Technologies and contributors
# For license information, please see license.txt
import frappe
from datetime import datetime
import frappe
from frappe.query_builder.functions import Coalesce, Count
from frappe.utils import getdate
from frappe.utils.dateutils import get_dates_from_timegrain
def execute(filters=None):
return WebsiteAnalytics(filters).run()
@ -56,33 +59,21 @@ class WebsiteAnalytics(object):
]
def get_data(self):
pg_query = """
SELECT
path,
COUNT(*) as count,
COUNT(CASE WHEN CAST(is_unique as Integer) = 1 THEN 1 END) as unique_count
FROM `tabWeb Page View`
WHERE coalesce("tabWeb Page View".creation, '0001-01-01') BETWEEN %s AND %s
GROUP BY path
ORDER BY count desc
"""
WebPageView = frappe.qb.Table("Web Page View")
count_all = Count("*").as_("count")
case = frappe.qb.terms.Case().when(WebPageView.is_unique == "1", "1")
count_is_unique = Count(case).as_("unique_count")
mariadb_query = """
SELECT
path,
COUNT(*) as count,
COUNT(CASE WHEN is_unique = 1 THEN 1 END) as unique_count
FROM `tabWeb Page View`
WHERE creation BETWEEN %s AND %s
GROUP BY path
ORDER BY count desc
"""
data = frappe.db.multisql({
"mariadb": mariadb_query,
"postgres": pg_query
}, (self.filters.from_date, self.filters.to_date))
return data
query = (
frappe.qb.from_(WebPageView)
.select("path", count_all, count_is_unique)
.where(
Coalesce(WebPageView.creation, "0001-01-01")[self.filters.from_date:self.filters.to_date]
)
.groupby(WebPageView.path)
.orderby("count", Order=frappe.qb.desc)
)
return frappe.db.sql(query)
def _get_query_for_mariadb(self):
filters_range = self.filters.range

5
frappe/website/utils.py
View file

@ -488,11 +488,12 @@ def set_content_type(response, data, path):
return data
def add_preload_headers(response):
from bs4 import BeautifulSoup
from bs4 import BeautifulSoup, SoupStrainer
try:
preload = []
soup = BeautifulSoup(response.data, "lxml")
strainer = SoupStrainer(re.compile("script|link"))
soup = BeautifulSoup(response.data, "lxml", parse_only=strainer)
for elem in soup.find_all('script', src=re.compile(".*")):
preload.append(("script", elem.get("src")))

18
frappe/workflow/doctype/workflow_action/workflow_action.py
View file

@ -133,9 +133,12 @@ def return_link_expired_page(doc, doc_workflow_state):
def clear_old_workflow_actions(doc, user=None):
user = user if user else frappe.session.user
frappe.db.sql("""DELETE FROM `tabWorkflow Action`
WHERE `reference_doctype`=%s AND `reference_name`=%s AND `user`!=%s AND `status`='Open'""",
(doc.get('doctype'), doc.get('name'), user))
frappe.db.delete("Workflow Action", {
"reference_doctype": doc.get("doctype"),
"reference_name": doc.get("name"),
"user": ("!=", user),
"status": "Open"
})
def update_completed_workflow_actions(doc, user=None):
user = user if user else frappe.session.user
@ -253,11 +256,10 @@ def is_workflow_action_already_created(doc):
def clear_workflow_actions(doctype, name):
if not (doctype and name):
return
frappe.db.sql('''delete from `tabWorkflow Action`
where reference_doctype=%s and reference_name=%s''',
(doctype, name))
frappe.db.delete("Workflow Action", {
"reference_doctype": doctype,
"reference_name": name
})
def get_doc_workflow_state(doc):
workflow_name = get_workflow_name(doc.get('doctype'))
workflow_state_field = get_workflow_state_field(workflow_name)

1
requirements.txt
View file

@ -49,6 +49,7 @@ pyngrok~=5.0.5
pyOpenSSL~=20.0.1
pyotp~=2.6.0
PyPDF2~=1.26.0
PyPika~=0.48.6
pypng~=0.0.20
PyQRCode~=1.2.1
python-dateutil~=2.8.1