diff --git a/.github/helper/roulette.py b/.github/helper/roulette.py index 9eec32e863..e3b212fa89 100644 --- a/.github/helper/roulette.py +++ b/.github/helper/roulette.py @@ -4,8 +4,10 @@ import re import shlex import subprocess import sys +import time import urllib.request from functools import lru_cache +from urllib.error import HTTPError @lru_cache(maxsize=None) @@ -15,11 +17,30 @@ def fetch_pr_data(pr_number, repo, endpoint=""): if endpoint: api_url += f"/{endpoint}" - req = urllib.request.Request(api_url) - res = urllib.request.urlopen(req) + res = req(api_url) return json.loads(res.read().decode("utf8")) +def req(url): + "Simple resilient request call to handle rate limits." + headers = None + token = os.environ.get("GITHUB_TOKEN") + if token: + headers = {"authorization": f"Bearer {token}"} + + retries = 0 + while True: + try: + req = urllib.request.Request(url, headers=headers) + return urllib.request.urlopen(req) + except HTTPError as exc: + if exc.code == 403 and retries < 5: + retries += 1 + time.sleep(retries) + continue + raise + + def get_files_list(pr_number, repo="frappe/frappe"): return [change["filename"] for change in fetch_pr_data(pr_number, repo, "files")] diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index 010022b7f6..6dcbccd85e 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -19,7 +19,7 @@ jobs: - name: Setup Node.js uses: actions/setup-node@v3 with: - node-version: 16 + node-version: 18 - name: Setup dependencies run: | npm install @semantic-release/git @semantic-release/exec --no-save diff --git a/.github/workflows/patch-mariadb-tests.yml b/.github/workflows/patch-mariadb-tests.yml index 57f421cc1b..4b487d2aea 100644 --- a/.github/workflows/patch-mariadb-tests.yml +++ b/.github/workflows/patch-mariadb-tests.yml @@ -9,6 +9,7 @@ concurrency: cancel-in-progress: true permissions: + # Do not change this as GITHUB_TOKEN is being used by roulette contents: read jobs: @@ -31,6 +32,7 @@ jobs: TYPE: "server" PR_NUMBER: ${{ github.event.number }} REPO_NAME: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} test: name: Patch diff --git a/.github/workflows/server-tests.yml b/.github/workflows/server-tests.yml index cc310fd8d7..3b76da1973 100644 --- a/.github/workflows/server-tests.yml +++ b/.github/workflows/server-tests.yml @@ -12,6 +12,7 @@ concurrency: permissions: + # Do not change this as GITHUB_TOKEN is being used by roulette contents: read jobs: @@ -34,6 +35,7 @@ jobs: TYPE: "server" PR_NUMBER: ${{ github.event.number }} REPO_NAME: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} test: name: Unit Tests diff --git a/.github/workflows/ui-tests.yml b/.github/workflows/ui-tests.yml index f022f10ea4..1b88bc73ce 100644 --- a/.github/workflows/ui-tests.yml +++ b/.github/workflows/ui-tests.yml @@ -11,6 +11,7 @@ concurrency: cancel-in-progress: true permissions: + # Do not change this as GITHUB_TOKEN is being used by roulette contents: read jobs: @@ -33,6 +34,7 @@ jobs: TYPE: "ui" PR_NUMBER: ${{ github.event.number }} REPO_NAME: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} test: runs-on: ubuntu-latest diff --git a/frappe/custom/doctype/customize_form/customize_form.js b/frappe/custom/doctype/customize_form/customize_form.js index c90da92f19..fed8505147 100644 --- a/frappe/custom/doctype/customize_form/customize_form.js +++ b/frappe/custom/doctype/customize_form/customize_form.js @@ -111,54 +111,59 @@ frappe.ui.form.on("Customize Form", { frm.page.clear_icons(); if (frm.doc.doc_type) { - frm.page.set_title(__("Customize Form - {0}", [frm.doc.doc_type])); - frappe.customize_form.set_primary_action(frm); + frappe.model.with_doctype(frm.doc.doc_type).then(() => { + frm.page.set_title(__("Customize Form - {0}", [frm.doc.doc_type])); + frappe.customize_form.set_primary_action(frm); - if (!frm.is_new()) { - frm.add_custom_button(__("Try new form builder", [__(frm.doc.doc_type)]), () => { - frappe.set_route("form-builder", frm.doc.doc_type, "customize"); - }); - } + if (!frm.is_new()) { + frm.add_custom_button( + __("Try new form builder", [__(frm.doc.doc_type)]), + () => { + frappe.set_route("form-builder", frm.doc.doc_type, "customize"); + } + ); + } - frm.add_custom_button( - __("Go to {0} List", [__(frm.doc.doc_type)]), - function () { - frappe.set_route("List", frm.doc.doc_type); - }, - __("Actions") - ); + frm.add_custom_button( + __("Go to {0} List", [__(frm.doc.doc_type)]), + function () { + frappe.set_route("List", frm.doc.doc_type); + }, + __("Actions") + ); - frm.add_custom_button( - __("Reload"), - function () { - frm.script_manager.trigger("doc_type"); - }, - __("Actions") - ); + frm.add_custom_button( + __("Reload"), + function () { + frm.script_manager.trigger("doc_type"); + }, + __("Actions") + ); - frm.add_custom_button( - __("Reset to defaults"), - function () { - frappe.customize_form.confirm(__("Remove all customizations?"), frm); - }, - __("Actions") - ); + frm.add_custom_button( + __("Reset to defaults"), + function () { + frappe.customize_form.confirm(__("Remove all customizations?"), frm); + }, + __("Actions") + ); - frm.add_custom_button( - __("Set Permissions"), - function () { - frappe.set_route("permission-manager", frm.doc.doc_type); - }, - __("Actions") - ); + frm.add_custom_button( + __("Set Permissions"), + function () { + frappe.set_route("permission-manager", frm.doc.doc_type); + }, + __("Actions") + ); - const is_autoname_autoincrement = frm.doc.autoname === "autoincrement"; - frm.set_df_property("naming_rule", "hidden", is_autoname_autoincrement); - frm.set_df_property("autoname", "read_only", is_autoname_autoincrement); - frm.toggle_display( - ["queue_in_background"], - frappe.get_meta(frm.doc.doc_type).is_submittable || 0 - ); + const is_autoname_autoincrement = frm.doc.autoname === "autoincrement"; + frm.set_df_property("naming_rule", "hidden", is_autoname_autoincrement); + frm.set_df_property("autoname", "read_only", is_autoname_autoincrement); + frm.toggle_display( + ["queue_in_background"], + frappe.get_meta(frm.doc.doc_type).is_submittable || 0 + ); + }); } frm.events.setup_export(frm); diff --git a/frappe/desk/page/user_profile/user_profile.py b/frappe/desk/page/user_profile/user_profile.py index 117ed5f560..3013df54a5 100644 --- a/frappe/desk/page/user_profile/user_profile.py +++ b/frappe/desk/page/user_profile/user_profile.py @@ -1,6 +1,8 @@ from datetime import datetime import frappe +from frappe.query_builder import Interval, Order +from frappe.query_builder.functions import Date, Sum, UnixTimestamp from frappe.utils import getdate @@ -11,21 +13,18 @@ def get_energy_points_heatmap_data(user, date): except Exception: date = getdate() + eps_log = frappe.qb.DocType("Energy Point Log") + return dict( - frappe.db.sql( - """select unix_timestamp(date(creation)), sum(points) - from `tabEnergy Point Log` - where - date(creation) > subdate('{date}', interval 1 year) and - date(creation) < subdate('{date}', interval -1 year) and - user = %s and - type != 'Review' - group by date(creation) - order by creation asc""".format( - date=date - ), - user, - ) + frappe.qb.from_(eps_log) + .select(UnixTimestamp(Date(eps_log.creation)), Sum(eps_log.points)) + .where(eps_log.user == user) + .where(eps_log["type"] != "Review") + .where(Date(eps_log.creation) > Date(date) - Interval(years=1)) + .where(Date(eps_log.creation) < Date(date) + Interval(years=1)) + .groupby(Date(eps_log.creation)) + .orderby(Date(eps_log.creation), order=Order.asc) + .run() ) @@ -51,7 +50,7 @@ def get_user_rank(user): month_start = datetime.today().replace(day=1) monthly_rank = frappe.get_all( "Energy Point Log", - group_by="user", + group_by="`tabEnergy Point Log`.`user`", filters={"creation": [">", month_start], "type": ["!=", "Review"]}, fields=["user", "sum(points)"], order_by="sum(points) desc", @@ -60,7 +59,7 @@ def get_user_rank(user): all_time_rank = frappe.get_all( "Energy Point Log", - group_by="user", + group_by="`tabEnergy Point Log`.`user`", filters={"type": ["!=", "Review"]}, fields=["user", "sum(points)"], order_by="sum(points) desc", diff --git a/frappe/desk/reportview.py b/frappe/desk/reportview.py index 5450d77377..593e6bf0c2 100644 --- a/frappe/desk/reportview.py +++ b/frappe/desk/reportview.py @@ -20,7 +20,7 @@ from frappe.utils import add_user_info, format_duration @frappe.read_only() def get(): args = get_form_params() - # If virtual doctype get data from controller het_list method + # If virtual doctype, get data from controller get_list method if is_virtual_doctype(args.doctype): controller = get_controller(args.doctype) data = compress(controller.get_list(args)) diff --git a/frappe/hooks.py b/frappe/hooks.py index 9ad95e796b..28b2c0dea1 100644 --- a/frappe/hooks.py +++ b/frappe/hooks.py @@ -392,4 +392,5 @@ ignore_links_on_delete = [ "Email Queue", "Document Share Key", "Integration Request", + "Unhandled Email", ] diff --git a/frappe/permissions.py b/frappe/permissions.py index 3f53f12a33..ef33c03875 100644 --- a/frappe/permissions.py +++ b/frappe/permissions.py @@ -27,28 +27,12 @@ rights = ( ) -def check_admin_or_system_manager(user=None): - from frappe.utils.commands import warn - - warn( - "The function check_admin_or_system_manager will be deprecated in version 15." - 'Please use frappe.only_for("System Manager") instead.', - category=PendingDeprecationWarning, - ) - - if not user: - user = frappe.session.user - - if ("System Manager" not in frappe.get_roles(user)) and (user != "Administrator"): - frappe.throw(_("Not permitted"), frappe.PermissionError) - - def print_has_permission_check_logs(func): def inner(*args, **kwargs): frappe.flags["has_permission_check_logs"] = [] result = func(*args, **kwargs) self_perm_check = True if not kwargs.get("user") else kwargs.get("user") == frappe.session.user - raise_exception = False if kwargs.get("raise_exception") is False else True + raise_exception = kwargs.get("raise_exception", True) # print only if access denied # and if user is checking his own permission diff --git a/frappe/query_builder/functions.py b/frappe/query_builder/functions.py index 24e2ee0e5f..88156b1799 100644 --- a/frappe/query_builder/functions.py +++ b/frappe/query_builder/functions.py @@ -74,6 +74,22 @@ DateFormat = ImportMapper( ) +class _PostgresUnixTimestamp(Extract): + # Note: this is just a special case of "Extract" function with "epoch" hardcoded. + # Check super definition to see how it works. + def __init__(self, field, alias=None): + super().__init__("epoch", field=field, alias=alias) + self.field = field + + +UnixTimestamp = ImportMapper( + { + db_type_is.MARIADB: CustomFunction("unix_timestamp", ["date"]), + db_type_is.POSTGRES: _PostgresUnixTimestamp, + } +) + + class Cast_(Function): def __init__(self, value, as_type, alias=None): if frappe.db.db_type == "mariadb" and ( diff --git a/frappe/social/doctype/energy_point_log/test_energy_point_log.py b/frappe/social/doctype/energy_point_log/test_energy_point_log.py index 5043019c19..c97e2a44e4 100644 --- a/frappe/social/doctype/energy_point_log/test_energy_point_log.py +++ b/frappe/social/doctype/energy_point_log/test_energy_point_log.py @@ -2,6 +2,7 @@ # License: MIT. See LICENSE import frappe from frappe.desk.form.assign_to import add as assign_to +from frappe.desk.page.user_profile.user_profile import get_energy_points_heatmap_data from frappe.tests.utils import FrappeTestCase from frappe.utils.testutils import add_custom_field, clear_custom_fields @@ -234,6 +235,10 @@ class TestEnergyPointLog(FrappeTestCase): self.assertEqual(test2_user_after_points, test2_user_before_points + rule.points) + def test_eps_heatmap_query(self): + # Just asserts that query works, not correctness. + self.assertIsInstance(get_energy_points_heatmap_data(user="test@example.com", date=None), dict) + def test_points_on_field_value_change(self): rule = create_energy_point_rule_for_todo( for_doc_event="Value Change", field_to_check="description" diff --git a/frappe/tests/test_query_builder.py b/frappe/tests/test_query_builder.py index d05c8d7d02..a16c2a23ae 100644 --- a/frappe/tests/test_query_builder.py +++ b/frappe/tests/test_query_builder.py @@ -6,7 +6,15 @@ import frappe from frappe.query_builder import Case from frappe.query_builder.builder import Function from frappe.query_builder.custom import ConstantColumn -from frappe.query_builder.functions import Cast_, Coalesce, CombineDatetime, GroupConcat, Match +from frappe.query_builder.functions import ( + Cast_, + Coalesce, + CombineDatetime, + Date, + GroupConcat, + Match, + UnixTimestamp, +) from frappe.query_builder.utils import db_type_is from frappe.tests.utils import FrappeTestCase @@ -75,6 +83,47 @@ class TestCustomFunctionsMariaDB(FrappeTestCase): str(select_query).lower(), ) + def test_unix_ts_mariadb(self): + # Simple Query + note = frappe.qb.DocType("Note") + self.assertEqual( + "unix_timestamp(posting_date)", + UnixTimestamp(note.posting_date).get_sql(), + ) + + # Complex multi table query + todo = frappe.qb.DocType("ToDo") + select_query = ( + frappe.qb.from_(note) + .join(todo) + .on(todo.refernce_name == note.name) + .select(UnixTimestamp(note.posting_date)) + ) + self.assertIn("select unix_timestamp(`tabnote`.`posting_date`)", str(select_query).lower()) + + # Order by + select_query = select_query.orderby(UnixTimestamp(note.posting_date)) + self.assertIn( + "order by unix_timestamp(`tabnote`.`posting_date`)", + str(select_query).lower(), + ) + + # Function comparison + select_query = select_query.where( + UnixTimestamp(note.posting_date) >= UnixTimestamp("2021-01-01") + ) + self.assertIn( + "unix_timestamp(`tabnote`.`posting_date`)>=unix_timestamp('2021-01-01')", + str(select_query).lower(), + ) + + # aliasing + select_query = select_query.select(UnixTimestamp(note.posting_date, alias="unix_ts")) + self.assertIn( + "unix_timestamp(`tabnote`.`posting_date`) `unix_ts`", + str(select_query).lower(), + ) + def test_time(self): note = frappe.qb.DocType("Note") self.assertEqual( @@ -162,6 +211,47 @@ class TestCustomFunctionsPostgres(FrappeTestCase): '"tabnote"."posting_date"+"tabnote"."posting_time" "timestamp"', str(select_query).lower() ) + def test_unix_ts_postgres(self): + # Simple Query + note = frappe.qb.DocType("Note") + self.assertEqual( + "extract(epoch from posting_date)", + UnixTimestamp(note.posting_date).get_sql().lower(), + ) + + # Complex multi table query + todo = frappe.qb.DocType("ToDo") + select_query = ( + frappe.qb.from_(note) + .join(todo) + .on(todo.refernce_name == note.name) + .select(UnixTimestamp(note.posting_date)) + ) + self.assertIn('extract(epoch from "tabnote"."posting_date")', str(select_query).lower()) + + # Order by + select_query = select_query.orderby(UnixTimestamp(note.posting_date)) + self.assertIn( + 'order by extract(epoch from "tabnote"."posting_date")', + str(select_query).lower(), + ) + + # Function comparison + select_query = select_query.where( + UnixTimestamp(note.posting_date) >= UnixTimestamp(Date("2021-01-01")) + ) + self.assertIn( + 'extract(epoch from "tabnote"."posting_date")>=extract(epoch from date(\'2021-01-01\'))', + str(select_query).lower(), + ) + + # aliasing + select_query = select_query.select(UnixTimestamp(note.posting_date, alias="unix_ts")) + self.assertIn( + 'extract(epoch from "tabnote"."posting_date") "unix_ts"', + str(select_query).lower(), + ) + def test_time(self): note = frappe.qb.DocType("Note")