vassili/seitime-frappe
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Merge remote-tracking branch 'origin/develop' into fix/error

Browse source
This commit is contained in:
David Arnold 2024-01-16 12:47:51 +01:00
commit dcf0efa123
No known key found for this signature in database
GPG key ID: AB15A6AF1101390D
10 changed files with 211 additions and 41 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
frappe/contacts/doctype/contact/test_contact.py
View file

@ -23,15 +23,15 @@ class TestContact(FrappeTestCase):
def test_check_default_phone_and_mobile(self):
phones = [
{"phone": "+91 0000000000", "is_primary_phone": 0, "is_primary_mobile_no": 0},
{"phone": "+91 0000000001", "is_primary_phone": 0, "is_primary_mobile_no": 0},
{"phone": "+91 0000000002", "is_primary_phone": 1, "is_primary_mobile_no": 0},
{"phone": "+91 0000000003", "is_primary_phone": 0, "is_primary_mobile_no": 1},
{"phone": "+91 0000000010", "is_primary_phone": 0, "is_primary_mobile_no": 0},
{"phone": "+91 0000000011", "is_primary_phone": 0, "is_primary_mobile_no": 0},
{"phone": "+91 0000000012", "is_primary_phone": 1, "is_primary_mobile_no": 0},
{"phone": "+91 0000000013", "is_primary_phone": 0, "is_primary_mobile_no": 1},
]
contact = create_contact("Phone", "Mr", phones=phones)
self.assertEqual(contact.phone, "+91 0000000002")
self.assertEqual(contact.mobile_no, "+91 0000000003")
self.assertEqual(contact.phone, "+91 0000000012")
self.assertEqual(contact.mobile_no, "+91 0000000013")
def test_get_full_name(self):
self.assertEqual(get_full_name(first="John"), "John")

4
frappe/contacts/report/addresses_and_contacts/test_addresses_and_contacts.py
View file

@ -76,7 +76,7 @@ def create_linked_contact(link_list, address):
}
)
contact.add_email("test_contact@example.com", is_primary=True)
contact.add_phone("+91 0000000000", is_primary_phone=True)
contact.add_phone("+91 0000000020", is_primary_phone=True)
for name in link_list:
contact.append("links", {"link_doctype": "Test Custom Doctype", "link_name": name})
@ -105,7 +105,7 @@ class TestAddressesAndContacts(FrappeTestCase):
"_Test First Name",
"_Test Last Name",
"_Test Address-Billing",
"+91 0000000000",
"+91 0000000020",
"",
"test_contact@example.com",
1,

59
frappe/database/database.py
View file

@ -46,6 +46,8 @@ INDEX_PATTERN = re.compile(r"\s*\([^)]+\)\s*")
SINGLE_WORD_PATTERN = re.compile(r'([`"]?)(tab([A-Z]\w+))\1')
MULTI_WORD_PATTERN = re.compile(r'([`"])(tab([A-Z]\w+)( [A-Z]\w+)+)\1')
SQL_ITERATOR_BATCH_SIZE = 100
class Database:
"""
@ -175,6 +177,8 @@ class Database:
:param pluck: Get the plucked field only.
:param explain: Print `EXPLAIN` in error log.
:param as_iterator: Returns iterator over results instead of fetching all results at once.
This should be used with unbuffered cursor as default cursors used by pymysql and postgres
buffer the results internally. See `Database.unbuffered_cursor`.
Examples:
# return customer names as dicts
@ -276,12 +280,10 @@ class Database:
if not self._cursor.description:
return ()
last_result = self._transform_result(self._cursor.fetchall())
if as_iterator:
return self._return_as_iterator(
last_result, pluck=pluck, as_dict=as_dict, as_list=as_list, update=update
)
return self._return_as_iterator(pluck=pluck, as_dict=as_dict, as_list=as_list, update=update)
last_result = self._transform_result(self._cursor.fetchall())
if pluck:
last_result = [r[0] for r in last_result]
self._clean_up()
@ -300,24 +302,25 @@ class Database:
self._clean_up()
return last_result
def _return_as_iterator(self, result, *, pluck, as_dict, as_list, update):
if pluck:
for row in result:
yield row[0]
def _return_as_iterator(self, *, pluck, as_dict, as_list, update):
while result := self._transform_result(self._cursor.fetchmany(SQL_ITERATOR_BATCH_SIZE)):
if pluck:
for row in result:
yield row[0]
elif as_dict:
keys = [column[0] for column in self._cursor.description]
for row in result:
row = frappe._dict(zip(keys, row))
if update:
row.update(update)
yield row
elif as_dict:
keys = [column[0] for column in self._cursor.description]
for row in result:
row = frappe._dict(zip(keys, row))
if update:
row.update(update)
yield row
elif as_list:
for row in result:
yield list(row)
else:
frappe.throw(_("`as_iterator` only works with `as_list=True` or `as_dict=True`"))
elif as_list:
for row in result:
yield list(row)
else:
frappe.throw(_("`as_iterator` only works with `as_list=True` or `as_dict=True`"))
self._clean_up()
@ -1344,6 +1347,22 @@ class Database:
def rename_column(self, doctype: str, old_column_name: str, new_column_name: str):
raise NotImplementedError
@contextmanager
def unbuffered_cursor(self):
"""Context manager to temporarily use unbuffered cursor.
Using this with `as_iterator=True` provides O(1) memory usage while reading large result sets.
NOTE: You MUST do entire result set processing in the context, otherwise underlying cursor
will be switched and you'll not get complete results.
Usage:
with frappe.db.unbuffered_cursor():
for row in frappe.db.sql("query with huge result", as_iterator=True):
continue # Do some processing.
"""
raise NotImplementedError
@contextmanager
def savepoint(catch: type | tuple[type, ...] = Exception):

13
frappe/database/mariadb/database.py
View file

@ -1,4 +1,5 @@
import re
from contextlib import contextmanager
import pymysql
from pymysql.constants import ER, FIELD_TYPE
@ -525,3 +526,15 @@ class MariaDBDatabase(MariaDBConnectionUtil, MariaDBExceptionUtil, Database):
if est_row_size:
return int(est_row_size[0][0])
@contextmanager
def unbuffered_cursor(self):
from pymysql.cursors import SSCursor
try:
original_cursor = self._cursor
new_cursor = self._cursor = self._conn.cursor(SSCursor)
yield
finally:
self._cursor = original_cursor
new_cursor.close()

6
frappe/desk/doctype/event/event.json
View file

@ -123,7 +123,7 @@
"fieldtype": "Select",
"in_global_search": 1,
"label": "Repeat On",
"options": "\nDaily\nWeekly\nMonthly\nYearly"
"options": "\nDaily\nWeekly\nMonthly\nQuarterly\nHalf Yearly\nYearly"
},
{
"depends_on": "repeat_this_event",
@ -295,7 +295,7 @@
"icon": "fa fa-calendar",
"idx": 1,
"links": [],
"modified": "2023-06-23 10:33:15.685368",
"modified": "2024-01-11 07:11:17.467503",
"modified_by": "Administrator",
"module": "Desk",
"name": "Event",
@ -336,4 +336,4 @@
"track_changes": 1,
"track_seen": 1,
"track_views": 1
}
}

59
frappe/desk/doctype/event/event.py
View file

@ -21,6 +21,7 @@ from frappe.utils import (
format_datetime,
get_datetime_str,
getdate,
month_diff,
now_datetime,
nowdate,
)
@ -62,7 +63,7 @@ class Event(Document):
google_meet_link: DF.Data | None
monday: DF.Check
pulled_from_google_calendar: DF.Check
repeat_on: DF.Literal["", "Daily", "Weekly", "Monthly", "Yearly"]
repeat_on: DF.Literal["", "Daily", "Weekly", "Monthly", "Quarterly", "Half Yearly", "Yearly"]
repeat_this_event: DF.Check
repeat_till: DF.Date | None
saturday: DF.Check
@ -392,6 +393,62 @@ def get_events(start, end, user=None, for_reminder=False, filters=None) -> list[
remove_events.append(e)
if e.repeat_on == "Half Yearly":
# creates a string with date (27) and month (07) and year (2019) eg: 2019-07-27
year, month = start.split("-", maxsplit=2)[:2]
date = f"{year}-{month}-" + event_start.split("-", maxsplit=3)[2]
# last day of month issue, start from prev month!
try:
getdate(date)
except Exception:
date = date.split("-")
date = date[0] + "-" + str(cint(date[1]) - 1) + "-" + date[2]
start_from = date
for i in range(int(date_diff(end, start) / 30) + 3):
diff = month_diff(date, event_start) - 1
if diff % 6 != 0:
continue
if (
getdate(date) >= getdate(start)
and getdate(date) <= getdate(end)
and getdate(date) <= getdate(repeat)
and getdate(date) >= getdate(event_start)
):
add_event(e, date)
date = add_months(start_from, i + 1)
remove_events.append(e)
if e.repeat_on == "Quarterly":
# creates a string with date (27) and month (07) and year (2019) eg: 2019-07-27
year, month = start.split("-", maxsplit=2)[:2]
date = f"{year}-{month}-" + event_start.split("-", maxsplit=3)[2]
# last day of month issue, start from prev month!
try:
getdate(date)
except Exception:
date = date.split("-")
date = date[0] + "-" + str(cint(date[1]) - 1) + "-" + date[2]
start_from = date
for i in range(int(date_diff(end, start) / 30) + 3):
diff = month_diff(date, event_start) - 1
if diff % 3 != 0:
continue
if (
getdate(date) >= getdate(start)
and getdate(date) <= getdate(end)
and getdate(date) <= getdate(repeat)
and getdate(date) >= getdate(event_start)
):
add_event(e, date)
date = add_months(start_from, i + 1)
remove_events.append(e)
if e.repeat_on == "Monthly":
# creates a string with date (27) and month (07) and year (2019) eg: 2019-07-27
year, month = start.split("-", maxsplit=2)[:2]

74
frappe/desk/doctype/event/test_event.py
View file

@ -136,3 +136,77 @@ class TestEvent(FrappeTestCase):
ev_list3 = get_events("2015-02-01", "2015-02-01", "Administrator", for_reminder=True)
self.assertTrue(bool(list(filter(lambda e: e.name == ev.name, ev_list3))))
def test_quaterly_repeat(self):
ev = frappe.get_doc(
{
"doctype": "Event",
"subject": "_Test Event",
"starts_on": "2023-02-17",
"repeat_till": "2024-02-17",
"event_type": "Public",
"repeat_this_event": 1,
"repeat_on": "Quarterly",
}
)
ev.insert()
# Test Quaterly months
ev_list = get_events("2023-02-17", "2023-02-17", "Administrator", for_reminder=True)
self.assertTrue(bool(list(filter(lambda e: e.name == ev.name, ev_list))))
ev_list1 = get_events("2023-05-17", "2023-05-17", "Administrator", for_reminder=True)
self.assertTrue(bool(list(filter(lambda e: e.name == ev.name, ev_list1))))
ev_list2 = get_events("2023-08-17", "2023-08-17", "Administrator", for_reminder=True)
self.assertTrue(bool(list(filter(lambda e: e.name == ev.name, ev_list2))))
ev_list3 = get_events("2023-11-17", "2023-11-17", "Administrator", for_reminder=True)
self.assertTrue(bool(list(filter(lambda e: e.name == ev.name, ev_list3))))
# Test before event start date and after event end date
ev_list4 = get_events("2022-11-17", "2022-11-17", "Administrator", for_reminder=True)
self.assertFalse(bool(list(filter(lambda e: e.name == ev.name, ev_list4))))
ev_list4 = get_events("2024-02-17", "2024-02-17", "Administrator", for_reminder=True)
self.assertFalse(bool(list(filter(lambda e: e.name == ev.name, ev_list4))))
# Test months that aren't part of the quarterly cycle
ev_list4 = get_events("2023-12-17", "2023-12-17", "Administrator", for_reminder=True)
self.assertFalse(bool(list(filter(lambda e: e.name == ev.name, ev_list4))))
ev_list4 = get_events("2023-03-17", "2023-03-17", "Administrator", for_reminder=True)
self.assertFalse(bool(list(filter(lambda e: e.name == ev.name, ev_list4))))
def test_half_yearly_repeat(self):
ev = frappe.get_doc(
{
"doctype": "Event",
"subject": "_Test Event",
"starts_on": "2023-02-17",
"repeat_till": "2024-02-17",
"event_type": "Public",
"repeat_this_event": 1,
"repeat_on": "Half Yearly",
}
)
ev.insert()
# Test Half Yearly months
ev_list = get_events("2023-02-17", "2023-02-17", "Administrator", for_reminder=True)
self.assertTrue(bool(list(filter(lambda e: e.name == ev.name, ev_list))))
ev_list1 = get_events("2023-08-17", "2023-08-17", "Administrator", for_reminder=True)
self.assertTrue(bool(list(filter(lambda e: e.name == ev.name, ev_list1))))
# Test before event start date and after event end date
ev_list4 = get_events("2022-08-17", "2022-08-17", "Administrator", for_reminder=True)
self.assertFalse(bool(list(filter(lambda e: e.name == ev.name, ev_list4))))
ev_list4 = get_events("2024-02-17", "2024-02-17", "Administrator", for_reminder=True)
self.assertFalse(bool(list(filter(lambda e: e.name == ev.name, ev_list4))))
# Test months that aren't part of the half yearly cycle
ev_list4 = get_events("2023-12-17", "2023-12-17", "Administrator", for_reminder=True)
self.assertFalse(bool(list(filter(lambda e: e.name == ev.name, ev_list4))))
ev_list4 = get_events("2023-05-17", "2023-05-17", "Administrator", for_reminder=True)
self.assertFalse(bool(list(filter(lambda e: e.name == ev.name, ev_list4))))

4
frappe/public/js/frappe/ui/toolbar/awesome_bar.js
View file

@ -317,7 +317,9 @@ frappe.search.AwesomeBar = class AwesomeBar {
var route = frappe.get_route();
if (route[0] === "List" && txt.indexOf(" in") === -1) {
// search in title field
var meta = frappe.get_meta(frappe.container.page.list_view.doctype);
const doctype = frappe.container.page?.list_view?.doctype;
if (!doctype) return;
var meta = frappe.get_meta(doctype);
var search_field = meta.title_field || "name";
var options = {};
options[search_field] = ["like", "%" + txt + "%"];

7
frappe/tests/test_db.py
View file

@ -11,7 +11,7 @@ import frappe
from frappe.core.utils import find
from frappe.custom.doctype.custom_field.custom_field import create_custom_field
from frappe.database import savepoint
from frappe.database.database import Database, get_query_execution_timeout
from frappe.database.database import get_query_execution_timeout
from frappe.database.utils import FallBackDateTimeStr
from frappe.query_builder import Field
from frappe.query_builder.functions import Concat_ws
@ -1007,3 +1007,8 @@ class TestSqlIterator(FrappeTestCase):
list(frappe.db.sql(query, as_list=True, as_iterator=True)),
msg=f"{query=} results not same as iterator",
)
@run_only_if(db_type_is.MARIADB)
def test_unbuffered_cursor(self):
with frappe.db.unbuffered_cursor():
self.test_db_sql_iterator()

14
frappe/utils/sentry.py
View file

@ -107,13 +107,13 @@ def capture_exception(message: str | None = None) -> None:
return
try:
hub = Hub.current
if frappe.request:
with hub.configure_scope() as scope:
if (
os.getenv("ENABLE_SENTRY_DB_MONITORING") is None
or os.getenv("SENTRY_TRACING_SAMPLE_RATE") is None
):
set_scope(scope)
with hub.configure_scope() as scope:
if (
os.getenv("ENABLE_SENTRY_DB_MONITORING") is None
or os.getenv("SENTRY_TRACING_SAMPLE_RATE") is None
):
set_scope(scope)
if frappe.request:
evt_processor = _make_wsgi_event_processor(frappe.request.environ, False)
scope.add_event_processor(evt_processor)
if frappe.request.is_json: