From a7c706672f4eb28bdaecb6109e042070db1da188 Mon Sep 17 00:00:00 2001 From: barredterra <14891507+barredterra@users.noreply.github.com> Date: Wed, 25 May 2022 01:12:48 +0200 Subject: [PATCH 001/205] fix: allow All to select a User --- frappe/core/doctype/user/user.json | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/frappe/core/doctype/user/user.json b/frappe/core/doctype/user/user.json index 42122ebfda..82e3fa71f3 100644 --- a/frappe/core/doctype/user/user.json +++ b/frappe/core/doctype/user/user.json @@ -722,7 +722,7 @@ "link_fieldname": "user" } ], - "modified": "2022-03-09 01:47:56.745069", + "modified": "2022-05-25 01:00:51.345319", "modified_by": "Administrator", "module": "Core", "name": "User", @@ -747,6 +747,10 @@ "read": 1, "role": "System Manager", "write": 1 + }, + { + "role": "All", + "select": 1 } ], "quick_entry": 1, From 4422754624297c1622efd321ca75519530e15eb7 Mon Sep 17 00:00:00 2001 From: Vladislav Date: Tue, 31 May 2022 18:09:38 +0300 Subject: [PATCH 002/205] Update ru.csv --- frappe/translations/ru.csv | 64 +++++++++++++++++++++----------------- 1 file changed, 36 insertions(+), 28 deletions(-) diff --git a/frappe/translations/ru.csv b/frappe/translations/ru.csv index 3fdeab5546..94a87bdcf8 100644 --- a/frappe/translations/ru.csv +++ b/frappe/translations/ru.csv @@ -840,7 +840,7 @@ Default Sending and Inbox,По умолчанию отправка и получ Default Sort Field,Поле сортировки по умолчанию, Default Sort Order,Порядок сортировки по умолчанию, Default Value,Значение по умолчанию, -"Default: ""Contact Us""","По умолчанию: ""Обратная связь""", +"Default: ""Contact Us""","По умолчанию: ""Contact Us""", DefaultValue,DefaultValue, Define workflows for forms.,Определите рабочие процессы для форм., Defines actions on states and the next step and allowed roles.,"Определяет действия на статусах, следующий шаг и роли, обладающие правами перевода статусов.", @@ -849,7 +849,7 @@ Delayed,Задерживается, Delete Data,Удалить данные, Delete comment?,Удалить комментарий?, Delete this record to allow sending to this email address,"Удалить эту запись, чтобы разрешить отправку на этот адрес электронной почты", -Delete {0} items permanently?,Удалить {0} продуктов навсегда?, +Delete {0} items permanently?,Удалить {0} объектов навсегда?, Deleted,Удаленный, Deleted DocType,Удаленный DocType, Deleted Document,Удаленный документ, @@ -914,7 +914,7 @@ Document can't saved.,Документ не может быть сохранен Document {0} has been set to state {1} by {2},Документ {0} установлен в состояние {1} на {2}, Documents,Документы, Documents assigned to you and by you.,"Документы, назначенные вам и вами.", -Domain Settings,Настройки домена, +Domain Settings,Настройка сфер деятельности, Domains HTML,Домены HTML, "Don't HTML Encode HTML tags like <script> or just characters like < or >, as they could be intentionally used in this field","Не HTML Кодировать HTML-теги, такие как <скрипт> или просто символы, такие как <или>, так как они могут быть преднамеренно использованы в этой области", Don't Override Status,Не переопределять статус, @@ -990,7 +990,7 @@ Enable Auto Reply,Включить автоматический ответ, Enable Automatic Backup,Включить автоматическое резервное копирование, Enable Chat,Включить чат, Enable Comments,Включить комментарии, -Enable Incoming,Включение входящей, +Enable Incoming,Включить входящие, Enable Outgoing,Включить исходящие, Enable Password Policy,Включить политику паролей, Enable Print Server,Включить сервер печати, @@ -1000,7 +1000,7 @@ Enable Scheduled Jobs,Включить запланированных задан Enable Social Login,Включить социальный вход, Enable Two Factor Auth,Включить двухфакторный аут, Enabled email inbox for user {0},Включен почтовый ящик для пользователя {0}, -"Encryption key is invalid, Please check site_config.json","Ключ шифрования недействителен, проверьте сайт_config.json", +"Encryption key is invalid, Please check site_config.json","Ключ шифрования недействителен, проверьте site_config.json", End Date Field,Поле конечной даты, End Date cannot be before Start Date!,Дата окончания не может быть до даты начала!, Endpoint URL,URL конечной точки, @@ -1029,8 +1029,8 @@ Error in Notification: {},Ошибка в уведомлении: {}, Error while connecting to email account {0},Ошибка при подключении к учетной записи электронной почты {0}, Error while evaluating Notification {0}. Please fix your template.,Ошибка при оценке уведомления {0}. Исправьте шаблон., Error: Document has been modified after you have opened it,"Ошибка: документ был изменен после того, как вы открыли его", -Error: Value missing for {0}: {1},Ошибка: значение отсутствует для {0}: {1}, -Errors in Background Events,Ошибки в фоновых событий, +Error: Value missing for {0}: {1},Ошибка: отсутствует значение для {0}: {1}, +Errors in Background Events,Ошибки в фоновых событиях, Event Category,Категория события, Event Participants,Участники мероприятия, Event Type,Тип события, @@ -1184,7 +1184,7 @@ Get Contacts,Получить контакты, Get Fields,Получить поля, Get your globally recognized avatar from Gravatar.com,Получить всемирно признанный аватара из Gravatar.com, GitHub,GitHub, -Give Review Points,Дайте очки обзора, +Give Review Points,Дайте баллы обзора, Global Unsubscribe,Глобальная отписка, Go to the document,Перейти к документу, Go to this URL after completing the form (only for Guest users),Перейдите по этому URL-адресу после заполнения формы (только для гостевых пользователей), @@ -1461,8 +1461,8 @@ Letter Head Name,Название заголовка письма, Letter Head in HTML,Заголовок письма в HTML, Level Name,Название уровня, Liked,Понравилось, -Liked By,В избранное К, -Liked by {0},В избранное {0}, +Liked By,Нравится, +Liked by {0},Нравится {0}, Likes,Понравившееся, Limit Number of DB Backups,Ограничение количества резервных копий БД, Line,Линия, @@ -1470,7 +1470,7 @@ Link DocType,Ссылка DocType, Link Expired,Срок действия ссылки, Link Name,Имя ссылки, Link Title,Название ссылки, -"Link that is the website home page. Standard Links (index, login, products, blog, about, contact)","Ссылка, которая является стартовой страницей сайта. Стандартные ссылки (индекс, логин, продукты, блог, о, контакт)", +"Link that is the website home page. Standard Links (index, login, products, blog, about, contact)","Ссылка, которая является стартовой страницей сайта. Стандартные ссылки (index, login, products, blog, about, contact)", Link to the page you want to open. Leave blank if you want to make it a group parent.,"Ссылка на страницу, которую вы хотите открыть. Оставьте пустым, если хотите сделать его родительским элементом группы.", Linked,Связанный, Linked With,Связанные с, @@ -2096,7 +2096,7 @@ Revert Of,Вернуть из, Reverted,Отменено, Review Level,Уровень обзора, Review Levels,Уровни обзора, -Review Points,Очки обзора, +Review Points,Баллы обзора, Reviews,Отзывы, Revoke,Аннулировать, Revoked,Аннулировано, @@ -2141,10 +2141,10 @@ SMS sent to following numbers: {0},SMS отправлено следующим SMTP Server,SMTP-сервер, SMTP Settings for outgoing emails,Настройки SMTP для исходящих писем, "SQL Conditions. Example: status=""Open""",SQL условия. Пример: статус = "Открыть", -SSL/TLS Mode,Режим SSL / TLS, +SSL/TLS Mode,Режим SSL/TLS, Salesforce,Salesforce, Same Field is entered more than once,Одно и то же поле вводится не один раз, -Save API Secret: ,Сохранить API-интерфейс:, +Save API Secret: ,Сохранить API секрет: , Save As,Сохранить как, Save Filter,Сохранить фильтр, Save Report,Сохранить отчет, @@ -2258,7 +2258,7 @@ Set Property After Alert,Задать свойство после оповеще Set Quantity,Установите Количество, Set Role For,Установить роль для, Set User Permissions,Задание разрешений пользователя, -Set Value,Задать значение, +Set Value,Установить значение, Set custom roles for page and report,Набор пользовательских ролей для страницы и отчета, "Set default format, page size, print style etc.","Установить форму, размер страницы, стиль печати и т.д., используюмых по умолчанию", Set non-standard precision for a Float or Currency field,Установите нестандартные точность для поплавка или валютной области, @@ -2337,7 +2337,7 @@ Slideshow like display for the website,"Слайд-шоу, как дисплей Small Text,Маленьикий текст, Smallest Currency Fraction Value,Минимальное дробное значение, Smallest circulating fraction unit (coin). For e.g. 1 cent for USD and it should be entered as 0.01,"Минимальная разменная денежная единица (монета). Например, для доллара — 1 цент, и его нужно ввести как 0,01", -Snapshot View,Снимок Посмотреть, +Snapshot View,Просмотр снимка, Social,Сообщество, Social Login Key,Ключ социального входа, Social Login Provider,Социальный провайдер, @@ -2418,7 +2418,7 @@ Suspend Sending,Приостановить Отправка, Switch To Desk,Переключение на рабочий стол, Symbol,Символ, Sync,Синхронизация, -Sync on Migrate,Синхронизация по Migrate, +Sync on Migrate,Синхронизировать при переносе, Syntax error in template,Синтаксическая ошибка в шаблоне, System,Система, System Page,Страница системы, @@ -2450,7 +2450,7 @@ Thank you for your interest in subscribing to our updates,Спасибо за в Thank you for your message,Спасибо за ваше сообщение, The CSV format is case sensitive,Формат CSV чувствителен к регистру, The Condition '{0}' is invalid,Условие '{0}' является недействительным, -The First User: You,Первый пользователя: Вы, +The First User: You,Первый пользователь: Вы, "The application has been updated to a new version, please refresh this page","Приложение был обновлен до новой версии, пожалуйста, обновите эту страницу", The attachments could not be correctly linked to the new document,Вложения не могут быть правильно связаны с новым документом, The document could not be correctly assigned,Документ не может быть правильно назначен, @@ -2653,7 +2653,7 @@ User Field,Поле пользователя, User ID of a Blogger,ID пользователя-блоггера, User Image,Изображение пользователя, User Name,Имя пользователя, -User Permission,Пользователь Введено, +User Permission,Разрешения пользователя, User Permissions,Разрешения пользователей, User Permissions are used to limit users to specific records.,Пользовательские разрешения используются для ограничения пользователей конкретными записями., User Permissions created sucessfully,Пользовательские разрешения созданы успешно, @@ -3068,8 +3068,8 @@ zoom-out,отдалить, {0} or {1},{0} или {1}, {0} record deleted,{0} запись удалена, {0} records deleted,{0} записей удалено, -{0} reverted your point on {1},{0} вернул вашу точку на {1}, -{0} reverted your points on {1},{0} вернул ваши очки на {1}, +{0} reverted your point on {1},{0} вернул ваш балл на {1}, +{0} reverted your points on {1},{0} вернул ваши баллы на {1}, {0} reverted {1},{0} вернул {1}, {0} room must have atmost one user.,{0} номер должен иметь самого одного пользователя., {0} rows for {1},{0} строк для {1}, @@ -3148,7 +3148,7 @@ Access not allowed from this IP Address,Доступ с этого IP-адрес Action Type,Тип действия, Activity Log by ,Активность Журнал по, Add Fields,Добавить поля, -Administration,Администрация, +Administration,Администрирование, After Cancel,После отмены, After Delete,После удаления, After Save,После сохранения, @@ -3157,7 +3157,7 @@ After Submit,После отправки, Aggregate Function Based On,"Агрегатная функция, основанная на", Aggregate Function field is required to create a dashboard chart,Поле Aggregate Function необходимо для создания диаграммы панели мониторинга., All Records,Все записи, -Allot Points To Assigned Users,Выделить очки назначенным пользователям, +Allot Points To Assigned Users,Выделить баллы назначенным пользователям, Allow Auto Repeat,Разрешить автоматическое повторение, Allow Google Calendar Access,Разрешить доступ к Календарю Google, Allow Google Contacts Access,Разрешить доступ к контактам Google, @@ -3380,7 +3380,7 @@ Invalid field name: {0},Неверное имя поля: {0}, Invalid file URL. Please contact System Administrator.,"Неверный URL файла. Пожалуйста, свяжитесь с системным администратором.", Invalid include path,Неверный путь включения, Invalid username or password,неправильное имя пользователя или пароль, -Is Primary,Первичный, +Is Primary,Основной, Is Primary Mobile,Основной мобильный, Is Primary Phone,Основной телефон, Is Tree,Дерево, @@ -3667,7 +3667,7 @@ via Data Import,через импорт данных, {0} are mandatory fields,{0} обязательные поля, {0} are required,{0} требуется, {0} assigned a new task {1} {2} to you,{0} назначил вам новое задание {1} {2}, -{0} gained {1} point for {2} {3},{0} набрал {1} очко за {2} {3}, +{0} gained {1} point for {2} {3},{0} получил {1} балл за {2} {3}, {0} gained {1} points for {2} {3},{0} набрал {1} баллов за {2} {3}, {0} has no versions tracked.,{0} не отслеживает версии., {0} is not a valid report format. Report format should one of the following {1},{0} не является допустимым форматом отчета. Формат отчета должен быть одним из следующих {1}, @@ -4045,7 +4045,7 @@ No Permitted Charts on this Dashboard,На этой панели инструм No Permitted Charts,Нет разрешенных графиков, Reset Chart,Сбросить график, via {0},через {0}, -{0} is not a valid Phone Number,{0} не является действительным номером телефона, +{0} is not a valid Phone Number,{0} недействительный номер телефона, Failed Transactions,Неудачные транзакции, Value for field {0} is too long in {1}. Length should be lesser than {2} characters,Значение поля {0} слишком длинное в {1}. Длина должна быть меньше {2} симв., Data Too Long,Данные слишком длинные, @@ -4121,8 +4121,8 @@ Using this console may allow attackers to impersonate you and steal your informa {0} w,{0} н, {0} M,{0} М, {0} y,{0} г, -yesterday,вчерашний день, -{0} years ago,{0} лет назад, +yesterday,вчера, +{0} years ago,{0} год назад, New Chart,Новый график, New Shortcut,Новый ярлык, Edit Chart,Изменить диаграмму, @@ -4700,3 +4700,11 @@ Value cannot be negative for {0}: {1},Значение не может быть Negative Value,Отрицательное значение, Authentication failed while receiving emails from Email Account: {0}.,Ошибка аутентификации при получении писем из учетной записи электронной почты: {0}., Message from server: {0},Сообщение с сервера: {0}, +Documentation,Документация, +User Forum,Форум пользователей, +Report an issue,Сообщить об ошибке, +My Profile,Мой профиль, +My Settings,Мои настройки, +Toggle Full Width,Переключить ширину, +Toggle Theme,Переключить тему, +Modules,Модули, From e150d99ff02e1a000d2a5737b07b10b63b2d33b0 Mon Sep 17 00:00:00 2001 From: naveen <172697+naveensrinivasan@users.noreply.github.com> Date: Thu, 9 Jun 2022 01:29:31 +0000 Subject: [PATCH 003/205] chore: Included githubactions in the dependabot config This should help with keeping the GitHub actions updated on new releases. This will also help with keeping it secure. Dependabot helps in keeping the supply chain secure https://docs.github.com/en/code-security/dependabot GitHub actions up to date https://docs.github.com/en/code-security/dependabot/working-with-dependabot/keeping-your-actions-up-to-date-with-dependabot https://github.com/ossf/scorecard/blob/main/docs/checks.md#dependency-update-tool Signed-off-by: naveen <172697+naveensrinivasan@users.noreply.github.com> --- .github/dependabot.yml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000000..5ace4600a1 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" From ac59087f49510e22284a9cb872c8463305bd52be Mon Sep 17 00:00:00 2001 From: s-aga-r Date: Thu, 9 Jun 2022 13:30:55 +0530 Subject: [PATCH 004/205] fix: can't create a new User Type --- frappe/core/doctype/user_type/user_type.js | 9 ++------ frappe/core/doctype/user_type/user_type.json | 24 ++++++++++++-------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/frappe/core/doctype/user_type/user_type.js b/frappe/core/doctype/user_type/user_type.js index c8bd499b58..6b53248fd4 100644 --- a/frappe/core/doctype/user_type/user_type.js +++ b/frappe/core/doctype/user_type/user_type.js @@ -3,13 +3,8 @@ frappe.ui.form.on('User Type', { refresh: function(frm) { - frm.toggle_display('is_standard', frappe.boot.developer_mode); - frm.set_df_property('is_standard', 'read_only', !frappe.boot.developer_mode); - - const fields = ['role', 'apply_user_permission_on', 'user_id_field', - 'user_doctypes', 'user_type_modules']; - - frm.toggle_display(fields, !frm.doc.is_standard); + if (frm.is_new() && !frappe.boot.developer_mode) + frm.set_value('is_standard', 1); frm.set_query('document_type', 'user_doctypes', function() { return { diff --git a/frappe/core/doctype/user_type/user_type.json b/frappe/core/doctype/user_type/user_type.json index 9ea5d5be71..3d6b470af5 100644 --- a/frappe/core/doctype/user_type/user_type.json +++ b/frappe/core/doctype/user_type/user_type.json @@ -22,9 +22,11 @@ "fields": [ { "default": "0", + "depends_on": "eval: frappe.boot.developer_mode", "fieldname": "is_standard", "fieldtype": "Check", - "label": "Is Standard" + "label": "Is Standard", + "read_only_depends_on": "eval: !frappe.boot.developer_mode" }, { "depends_on": "eval: !doc.is_standard", @@ -33,21 +35,21 @@ "label": "Document Types and Permissions" }, { + "depends_on": "eval: !doc.is_standard", "fieldname": "user_doctypes", "fieldtype": "Table", "label": "Document Types", "mandatory_depends_on": "eval: !doc.is_standard", - "options": "User Document Type", - "read_only": 1 + "options": "User Document Type" }, { + "depends_on": "eval: !doc.is_standard", "fieldname": "role", "fieldtype": "Link", "in_list_view": 1, "label": "Role", "mandatory_depends_on": "eval: !doc.is_standard", - "options": "Role", - "read_only": 1 + "options": "Role" }, { "fieldname": "select_doctypes", @@ -62,13 +64,13 @@ "fieldtype": "Column Break" }, { + "depends_on": "eval: !doc.is_standard", "description": "Can only list down the document types which has been linked to the User document type.", "fieldname": "apply_user_permission_on", "fieldtype": "Link", "label": "Apply User Permission On", "mandatory_depends_on": "eval: !doc.is_standard", - "options": "DocType", - "read_only": 1 + "options": "DocType" }, { "depends_on": "eval: !doc.is_standard", @@ -81,8 +83,7 @@ "fieldname": "user_id_field", "fieldtype": "Select", "label": "User Id Field", - "mandatory_depends_on": "eval: !doc.is_standard", - "read_only": 1 + "mandatory_depends_on": "eval: !doc.is_standard" }, { "depends_on": "eval: !doc.is_standard", @@ -93,6 +94,7 @@ { "fieldname": "user_type_modules", "fieldtype": "Table", + "label": "User Type Module", "no_copy": 1, "options": "User Type Module", "print_hide": 1, @@ -107,10 +109,11 @@ ], "index_web_pages_for_search": 1, "links": [], - "modified": "2021-03-12 16:25:18.639050", + "modified": "2022-06-09 14:00:36.820306", "modified_by": "Administrator", "module": "Core", "name": "User Type", + "naming_rule": "Set by user", "owner": "Administrator", "permissions": [ { @@ -137,5 +140,6 @@ ], "sort_field": "modified", "sort_order": "DESC", + "states": [], "track_changes": 1 } \ No newline at end of file From c67ec6c87aeb3b829eecd96c30126bd0d420d326 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 9 Jun 2022 16:31:29 +0530 Subject: [PATCH 005/205] chore(deps): Bump cryptography from 3.4.8 to 37.0.2 changelog: https://cryptography.io/en/latest/changelog/ diff: https://github.com/pyca/cryptography/compare/3.4.8...37.0.x --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index bf798fe747..616f1a2c42 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ braintree~=4.8.0 chardet~=4.0.0 Click~=7.1.2 croniter~=1.0.11 -cryptography~=3.4.7 +cryptography~=37.0.2 dropbox~=11.7.0 email-reply-parser~=0.5.12 git-url-parse~=1.2.2 From 7e346933c73235e3a0f68e4c323083ab5c6f84d9 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 9 Jun 2022 16:43:58 +0530 Subject: [PATCH 006/205] fix: Cleaner error message on invalid encryption_key --- frappe/utils/password.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/frappe/utils/password.py b/frappe/utils/password.py index f2c4b9685a..c539891ac7 100644 --- a/frappe/utils/password.py +++ b/frappe/utils/password.py @@ -213,21 +213,16 @@ def decrypt(txt, encryption_key=None): try: cipher_suite = Fernet(encode(encryption_key or get_encryption_key())) - plain_text = cstr(cipher_suite.decrypt(encode(txt))) - return plain_text + return cstr(cipher_suite.decrypt(encode(txt))) except InvalidToken: # encryption_key in site_config is changed and not valid - frappe.throw( - _("Encryption key is invalid") + "!" - if encryption_key - else _(", please check site_config.json.") - ) + frappe.throw(_("Encryption key is invalid! Please check site_config.json")) def get_encryption_key(): - from frappe.installer import update_site_config - if "encryption_key" not in frappe.local.conf: + from frappe.installer import update_site_config + encryption_key = Fernet.generate_key().decode() update_site_config("encryption_key", encryption_key) frappe.local.conf.encryption_key = encryption_key From eb86a933e0ea22528a560c653d6f6418f6fa7ab5 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 9 Jun 2022 16:49:27 +0530 Subject: [PATCH 007/205] chore(deps): Bump IPython from 7.31.0 to 8.4.0 Major version includes multiple bug, security & performance fixes changelog: https://ipython.readthedocs.io/en/stable/whatsnew/version8.html --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 616f1a2c42..6710dd4004 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,7 +21,7 @@ googlemaps~=4.4.5 gunicorn~=20.1.0 html2text==2020.1.16 html5lib~=1.1 -ipython~=7.31.1 +ipython~=8.4.0 Jinja2~=3.0.1 ldap3~=2.9 markdown2~=2.4.0 From e7f2b7c6287d1f80607803c7e227c617dd2868c5 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 10 Jun 2022 11:31:59 +0530 Subject: [PATCH 008/205] chore(deps): bump faker from 8.1.4 to 13.12.1 Faker is a dev dependency installed for tests - to access the frappe.mock API to generate paragraphs, names & emails. --- dev-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index f4045c6bed..b67e915a16 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,4 +1,4 @@ coverage==5.5 -Faker~=8.1.0 +Faker~=13.12.1 pyngrok~=5.0.5 unittest-xml-reporting~=3.0.4 From 88f27f4af4ce88c0e7de3e0a22115c21867fef45 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 10 Jun 2022 11:36:05 +0530 Subject: [PATCH 009/205] chore: Remove deprecated dependency_links from setup * pdfkit dep is already being fulfilled by requirements.txt * ref: https://stackoverflow.com/questions/12518499/pip-ignores-dependency-links-in-setup-py --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 92ff63baff..ba4034a766 100644 --- a/setup.py +++ b/setup.py @@ -53,7 +53,6 @@ setup( zip_safe=False, include_package_data=True, install_requires=install_requires, - dependency_links=["https://github.com/frappe/python-pdfkit.git#egg=pdfkit"], cmdclass={"clean": CleanCommand}, python_requires=">=3.8", ) From 3aef47200b3e0c1afbbe51e6af89e803118227a8 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 10 Jun 2022 11:53:50 +0530 Subject: [PATCH 010/205] chore(deps): Bumped pdfkit from 0.6.1 to 1.0.0 * Updated from_string API to match latest signature * changelog: https://github.com/JazzCore/python-pdfkit/blob/master/HISTORY.rst --- frappe/utils/pdf.py | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/utils/pdf.py b/frappe/utils/pdf.py index 952717434c..5e8b65a5bd 100644 --- a/frappe/utils/pdf.py +++ b/frappe/utils/pdf.py @@ -35,7 +35,7 @@ def get_pdf(html, options=None, output=None): try: # Set filename property to false, so no file is actually created - filedata = pdfkit.from_string(html, False, options=options or {}) + filedata = pdfkit.from_string(html, options=options or {}, verbose=True) # https://pythonhosted.org/PyPDF2/PdfFileReader.html # create in-memory binary streams from filedata and create a PdfFileReader object diff --git a/requirements.txt b/requirements.txt index 6710dd4004..b69716675e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -32,7 +32,7 @@ openpyxl~=3.0.7 parse~=1.19.0 passlib~=1.7.4 paytmchecksum~=1.7.0 -pdfkit~=0.6.1 +pdfkit~=1.0.0 Pillow~=9.0.0 premailer~=3.8.0 psutil~=5.8.0 From ef0a92c8499051c4ed4cb73740c1c9ed4ebed43d Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 10 Jun 2022 12:08:10 +0530 Subject: [PATCH 011/205] chore(deps): Bump PyPDF2 from 1.26.0 to 2.1.0 * Updated changes in API usages * changelog: https://github.com/py-pdf/PyPDF2/blob/main/CHANGELOG --- frappe/tests/test_pdf.py | 4 ++-- frappe/utils/pdf.py | 20 ++++++++++---------- frappe/utils/print_format.py | 6 +++--- requirements.txt | 2 +- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/frappe/tests/test_pdf.py b/frappe/tests/test_pdf.py index 497546ebd5..8f2a2c1cfa 100644 --- a/frappe/tests/test_pdf.py +++ b/frappe/tests/test_pdf.py @@ -3,7 +3,7 @@ import io import unittest -from PyPDF2 import PdfFileReader +from PyPDF2 import PdfReader import frappe import frappe.utils.pdf as pdfgen @@ -42,7 +42,7 @@ class TestPdf(unittest.TestCase): def test_pdf_encryption(self): password = "qwe" pdf = pdfgen.get_pdf(self.html, options={"password": password}) - reader = PdfFileReader(io.BytesIO(pdf)) + reader = PdfReader(io.BytesIO(pdf)) self.assertTrue(reader.isEncrypted) self.assertTrue(reader.decrypt(password)) diff --git a/frappe/utils/pdf.py b/frappe/utils/pdf.py index 5e8b65a5bd..811a6511fd 100644 --- a/frappe/utils/pdf.py +++ b/frappe/utils/pdf.py @@ -5,10 +5,11 @@ import os import re import subprocess from distutils.version import LooseVersion +from typing import Optional import pdfkit from bs4 import BeautifulSoup -from PyPDF2 import PdfFileReader, PdfFileWriter +from PyPDF2 import PdfReader, PdfWriter import frappe from frappe import _ @@ -23,7 +24,7 @@ PDF_CONTENT_ERRORS = [ ] -def get_pdf(html, options=None, output=None): +def get_pdf(html, options=None, output: Optional[PdfWriter] = None): html = scrub_urls(html) html, options = prepare_options(html, options) @@ -37,9 +38,8 @@ def get_pdf(html, options=None, output=None): # Set filename property to false, so no file is actually created filedata = pdfkit.from_string(html, options=options or {}, verbose=True) - # https://pythonhosted.org/PyPDF2/PdfFileReader.html - # create in-memory binary streams from filedata and create a PdfFileReader object - reader = PdfFileReader(io.BytesIO(filedata)) + # create in-memory binary streams from filedata and create a PdfReader object + reader = PdfReader(io.BytesIO(filedata)) except OSError as e: if any([error in str(e) for error in PDF_CONTENT_ERRORS]): if not filedata: @@ -47,8 +47,8 @@ def get_pdf(html, options=None, output=None): frappe.throw(_("PDF generation failed because of broken image links")) # allow pdfs with missing images if file got created - if output: # output is a PdfFileWriter object - output.appendPagesFromReader(reader) + if output: + output.append_pages_from_reader(reader) else: raise finally: @@ -58,11 +58,11 @@ def get_pdf(html, options=None, output=None): password = options["password"] if output: - output.appendPagesFromReader(reader) + output.append_pages_from_reader(reader) return output - writer = PdfFileWriter() - writer.appendPagesFromReader(reader) + writer = PdfWriter() + writer.append_pages_from_reader(reader) if "password" in options: writer.encrypt(password) diff --git a/frappe/utils/print_format.py b/frappe/utils/print_format.py index 028501f306..a48d7ab84f 100644 --- a/frappe/utils/print_format.py +++ b/frappe/utils/print_format.py @@ -1,6 +1,6 @@ import os -from PyPDF2 import PdfFileWriter +from PyPDF2 import PdfWriter import frappe from frappe import _ @@ -58,7 +58,7 @@ def download_multi_pdf(doctype, name, format=None, no_letterhead=False, options= import json - output = PdfFileWriter() + output = PdfWriter() if isinstance(options, str): options = json.loads(options) @@ -152,7 +152,7 @@ def print_by_server( cups.setServer(print_settings.server_ip) cups.setPort(print_settings.port) conn = cups.Connection() - output = PdfFileWriter() + output = PdfWriter() output = frappe.get_print( doctype, name, print_format, doc=doc, no_letterhead=no_letterhead, as_pdf=True, output=output ) diff --git a/requirements.txt b/requirements.txt index b69716675e..62275b587e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -43,7 +43,7 @@ PyJWT~=2.0.1 PyMySQL~=1.0.2 pyOpenSSL~=20.0.1 pyotp~=2.6.0 -PyPDF2~=1.26.0 +PyPDF2~=2.1.0 PyPika~=0.48.9 pypng~=0.0.20 PyQRCode~=1.2.1 From 8fab5b96b79e134b428cbbda490eb99aa54fbaa0 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 10 Jun 2022 12:15:08 +0530 Subject: [PATCH 012/205] chore(deps): Bump pytz from 2021.1 to 2022.1 Multiple bug fixes & IANA 2022a * changelog: https://github.com/stub42/pytz/blob/master/tz/NEWS * diff: https://github.com/stub42/pytz/compare/release_2021.1...release_2022.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 62275b587e..085b596cea 100644 --- a/requirements.txt +++ b/requirements.txt @@ -48,7 +48,7 @@ PyPika~=0.48.9 pypng~=0.0.20 PyQRCode~=1.2.1 python-dateutil~=2.8.1 -pytz==2021.1 +pytz==2022.1 PyYAML~=5.4.1 rauth~=0.7.3 razorpay~=1.2.0 From 8f2bb4780a98081c841ace8f45e9bdad22d0ae7a Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 10 Jun 2022 12:30:07 +0530 Subject: [PATCH 013/205] chore(deps): Bump rq from 1.8.1 to 1.10.1 Exciting improvements to rq! Cleanup & bugs that made us sccratch heads in prod. changelog: https://github.com/rq/rq/blob/master/CHANGES.md --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 085b596cea..03731d77e5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -56,7 +56,7 @@ redis~=3.5.3 requests-oauthlib~=1.3.0 requests~=2.25.1 RestrictedPython~=5.1 -rq~=1.8.0 +rq~=1.10.1 rsa>=4.1 # not directly required, pinned by Snyk to avoid a vulnerability schedule~=1.1.0 semantic-version~=2.8.5 From d60f228c9029528fd5f8d7e15e4e72f1dced2c00 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 10 Jun 2022 12:43:34 +0530 Subject: [PATCH 014/205] chore(deps): Bump Jinja2 from 3.0.3 to 3.1.2 changelog: https://jinja.palletsprojects.com/en/3.1.x/changes/ --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 03731d77e5..3b99aa06d5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,7 +22,7 @@ gunicorn~=20.1.0 html2text==2020.1.16 html5lib~=1.1 ipython~=8.4.0 -Jinja2~=3.0.1 +Jinja2~=3.1.2 ldap3~=2.9 markdown2~=2.4.0 maxminddb-geolite2==2018.703 From 3d1df3525e78e8c0e8033e48aca2083ae901c12f Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 10 Jun 2022 12:46:59 +0530 Subject: [PATCH 015/205] chore(deps): Bump psutil from 5.8.0 to 5.9.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3b99aa06d5..cc35b594c9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -35,7 +35,7 @@ paytmchecksum~=1.7.0 pdfkit~=1.0.0 Pillow~=9.0.0 premailer~=3.8.0 -psutil~=5.8.0 +psutil~=5.9.1 psycopg2-binary~=2.9.1 pyasn1~=0.4.8 pycryptodome~=3.10.1 From 5680f458c18f4b59bff645231d2bbb8e5c228656 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 10 Jun 2022 12:48:36 +0530 Subject: [PATCH 016/205] chore(deps): Bump Werkzeug from 2.0.3 to 2.1.2 changelog: https://werkzeug.palletsprojects.com/en/2.1.x/changes/ --- requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index cc35b594c9..2e3ec3434a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -65,7 +65,7 @@ stripe~=2.56.0 terminaltables~=3.1.0 traceback-with-variables~=2.0.4 urllib3~=1.26.4 -Werkzeug~=2.0.3 +Werkzeug~=2.1.2 Whoosh~=2.7.4 xlrd~=2.0.1 zxcvbn-python~=4.4.24 @@ -73,4 +73,3 @@ tenacity~=8.0.1 cairocffi==1.2.0 WeasyPrint==52.5 phonenumbers==8.12.40 - From 65b011f14147f1e10670888f2424654096972192 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Sun, 5 Jun 2022 12:54:03 +0530 Subject: [PATCH 017/205] chore(deps): Bump semantic-version from 2.8.5 to 2.10.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2e3ec3434a..6c16a5be85 100644 --- a/requirements.txt +++ b/requirements.txt @@ -59,7 +59,7 @@ RestrictedPython~=5.1 rq~=1.10.1 rsa>=4.1 # not directly required, pinned by Snyk to avoid a vulnerability schedule~=1.1.0 -semantic-version~=2.8.5 +semantic-version~=2.10.0 sqlparse~=0.4.1 stripe~=2.56.0 terminaltables~=3.1.0 From 9bec09e14e20b8482baa867c1c7b793a42471202 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Sun, 5 Jun 2022 12:57:04 +0530 Subject: [PATCH 018/205] chore(deps): Bump croniter from 1.0.15 to 1.3.5 changelog: https://pypi.org/project/croniter/#changelog --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6c16a5be85..53839d9e18 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ boto3~=1.17.53 braintree~=4.8.0 chardet~=4.0.0 Click~=7.1.2 -croniter~=1.0.11 +croniter~=1.3.5 cryptography~=37.0.2 dropbox~=11.7.0 email-reply-parser~=0.5.12 From 111ad3cc82de1a7d6d89f5121e5e1dcdb98dd5fe Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 2 Jun 2022 12:58:40 +0530 Subject: [PATCH 019/205] chore(deps): Bump requests from 2.25.1 to 2.27.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 53839d9e18..66a75c99ca 100644 --- a/requirements.txt +++ b/requirements.txt @@ -54,7 +54,7 @@ rauth~=0.7.3 razorpay~=1.2.0 redis~=3.5.3 requests-oauthlib~=1.3.0 -requests~=2.25.1 +requests~=2.27.1 RestrictedPython~=5.1 rq~=1.10.1 rsa>=4.1 # not directly required, pinned by Snyk to avoid a vulnerability From fc422f4a97bd12ceadb05ffbad582e2a921c44f3 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 2 Jun 2022 13:03:57 +0530 Subject: [PATCH 020/205] chore(deps): Bump Pillow from 9.0.1 to 9.1.1 Includes security fixes, API & constants deprecations Changelog: https://pillow.readthedocs.io/en/stable/releasenotes/index.html --- frappe/core/doctype/file/file.py | 4 ++-- frappe/utils/data.py | 2 +- frappe/utils/image.py | 6 ++---- requirements.txt | 2 +- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/frappe/core/doctype/file/file.py b/frappe/core/doctype/file/file.py index 1bcbaf161a..e8b8da76ab 100755 --- a/frappe/core/doctype/file/file.py +++ b/frappe/core/doctype/file/file.py @@ -341,9 +341,9 @@ class File(Document): size = width, height if crop: - image = ImageOps.fit(image, size, Image.ANTIALIAS) + image = ImageOps.fit(image, size, Image.Resampling.LANCZOS) else: - image.thumbnail(size, Image.ANTIALIAS) + image.thumbnail(size, Image.Resampling.LANCZOS) thumbnail_url = f"{filename}_{suffix}.{extn}" path = os.path.abspath(frappe.get_site_path("public", thumbnail_url.lstrip("/"))) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index 60770ef6a9..49f9ead437 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -1349,7 +1349,7 @@ def get_thumbnail_base64_for_image(src): original_size = image.size size = 50, 50 - image.thumbnail(size, Image.ANTIALIAS) + image.thumbnail(size, Image.Resampling.LANCZOS) base64_string = image_to_base64(image, extn) return { diff --git a/frappe/utils/image.py b/frappe/utils/image.py index 0cbc02fb31..8823ea3dfe 100644 --- a/frappe/utils/image.py +++ b/frappe/utils/image.py @@ -7,8 +7,6 @@ from PIL import Image def resize_images(path, maxdim=700): - from PIL import Image - size = (maxdim, maxdim) for basepath, folders, files in os.walk(path): for fname in files: @@ -16,7 +14,7 @@ def resize_images(path, maxdim=700): if extn in ("jpg", "jpeg", "png", "gif"): im = Image.open(os.path.join(basepath, fname)) if im.size[0] > size[0] or im.size[1] > size[1]: - im.thumbnail(size, Image.ANTIALIAS) + im.thumbnail(size, Image.Resampling.LANCZOS) im.save(os.path.join(basepath, fname)) print("resized {0}".format(os.path.join(basepath, fname))) @@ -56,7 +54,7 @@ def optimize_image( image = Image.open(io.BytesIO(content)) image_format = content_type.split("/")[1] size = max_width, max_height - image.thumbnail(size, Image.LANCZOS) + image.thumbnail(size, Image.Resampling.LANCZOS) output = io.BytesIO() image.save( diff --git a/requirements.txt b/requirements.txt index 66a75c99ca..1a6b6120a3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -33,7 +33,7 @@ parse~=1.19.0 passlib~=1.7.4 paytmchecksum~=1.7.0 pdfkit~=1.0.0 -Pillow~=9.0.0 +Pillow~=9.1.1 premailer~=3.8.0 psutil~=5.9.1 psycopg2-binary~=2.9.1 From 70188934bc143d84c3367a1f3a90628e77cc12b7 Mon Sep 17 00:00:00 2001 From: Jannat Patel <31363128+pateljannat@users.noreply.github.com> Date: Fri, 10 Jun 2022 15:00:32 +0530 Subject: [PATCH 021/205] fix: pick the last signup template from hook (#17118) --- frappe/www/login.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frappe/www/login.py b/frappe/www/login.py index fbb34e43e7..1b9a8c239a 100644 --- a/frappe/www/login.py +++ b/frappe/www/login.py @@ -55,10 +55,10 @@ def get_context(context): ) signup_form_template = frappe.get_hooks("signup_form_template") - if signup_form_template and len(signup_form_template) and signup_form_template[0]: - path = signup_form_template[0] + if signup_form_template and len(signup_form_template): + path = signup_form_template[-1] if not guess_is_path(path): - path = frappe.get_attr(signup_form_template[0])() + path = frappe.get_attr(signup_form_template[-1])() else: path = "frappe/templates/signup.html" if path: From e740ad180d8cdd4804f4bc568412c857f9f2ced9 Mon Sep 17 00:00:00 2001 From: Shariq Ansari <30859809+shariquerik@users.noreply.github.com> Date: Fri, 10 Jun 2022 15:46:50 +0530 Subject: [PATCH 022/205] test: fix for failing Data Control UI test (#17133) --- cypress/support/commands.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cypress/support/commands.js b/cypress/support/commands.js index c64f0bf469..c168b0c201 100644 --- a/cypress/support/commands.js +++ b/cypress/support/commands.js @@ -271,10 +271,9 @@ Cypress.Commands.add('save', () => { cy.get(`button[data-label="Save"]:visible`).click({scrollBehavior: false, force: true}); cy.wait('@api'); }); - Cypress.Commands.add('hide_dialog', () => { - cy.wait(400); - cy.get('.btn-modal-close:visible').click({force: true}); + cy.wait(300); + cy.get_open_dialog().focus().find('.btn-modal-close').click(); cy.get('.modal:visible').should('not.exist'); }); From 076816a7d7d7541b66cc7c3d6a977256b77642dd Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Fri, 10 Jun 2022 15:55:34 +0530 Subject: [PATCH 023/205] fix: Allow to set empty values in Bulk Edit - minor UX fixes --- .../public/js/frappe/list/bulk_operations.js | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/frappe/public/js/frappe/list/bulk_operations.js b/frappe/public/js/frappe/list/bulk_operations.js index 94ec9d4e67..7c8c515643 100644 --- a/frappe/public/js/frappe/list/bulk_operations.js +++ b/frappe/public/js/frappe/list/bulk_operations.js @@ -208,7 +208,7 @@ export default class BulkOperations { const default_field = field_options.find(value => status_regex.test(value)); const dialog = new frappe.ui.Dialog({ - title: __('Edit'), + title: __('Bulk Edit'), fields: [ { 'fieldtype': 'Select', @@ -225,7 +225,9 @@ export default class BulkOperations { 'fieldtype': 'Data', 'label': __('Value'), 'fieldname': 'value', - 'reqd': 1 + onchange() { + show_help_text(); + } } ], primary_action: ({ value }) => { @@ -239,7 +241,7 @@ export default class BulkOperations { docnames: docnames, action: 'update', data: { - [fieldname]: value + [fieldname]: value || null } } }).then(r => { @@ -254,10 +256,11 @@ export default class BulkOperations { frappe.show_alert(__('Updated successfully')); }); }, - primary_action_label: __('Update') + primary_action_label: __('Update {0} records', [docnames.length]), }); if (default_field) set_value_field(dialog); // to set `Value` df based on default `Field` + show_help_text(); function set_value_field (dialogObj) { const new_df = Object.assign({}, @@ -275,9 +278,20 @@ export default class BulkOperations { new_df.default = options[0] || options[1]; } new_df.label = __('Value'); - new_df.reqd = 1; + new_df.onchange = show_help_text; + delete new_df.depends_on; dialogObj.replace_field('value', new_df); + show_help_text(); + } + + function show_help_text() { + let value = dialog.get_value('value'); + if (value == null || value === '') { + dialog.set_df_property('value', 'description', __('You have not entered a value. The field will be set to empty.')); + } else { + dialog.set_df_property('value', 'description', ''); + } } dialog.refresh(); From 5f45897471be85bc96f64f0809a0c9d6bb447565 Mon Sep 17 00:00:00 2001 From: Vladislav Date: Fri, 10 Jun 2022 13:31:53 +0300 Subject: [PATCH 024/205] Update ru.csv --- frappe/translations/ru.csv | 42 +++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/frappe/translations/ru.csv b/frappe/translations/ru.csv index 94a87bdcf8..5a23409ae8 100644 --- a/frappe/translations/ru.csv +++ b/frappe/translations/ru.csv @@ -68,7 +68,7 @@ Document Status,Статус документа, Document Type,Тип документа, Domain,Сфера деятельности, Domains,Сферы деятельности, -Draft,Проект, +Draft,Черновик, Edit,Редактировать, Email Account,Электронная почта, Email Address,Адрес электронной почты, @@ -415,7 +415,7 @@ Allowed In Mentions,Разрешено в упоминаниях, "Allowing DocType, DocType. Be careful!","Разрешение DocType, DocType. Будьте осторожны!", Already Registered,Уже регистрировались, Also adding the dependent currency field {0},Также добавление зависимого валютного поля {0}, -"Always add ""Draft"" Heading for printing draft documents",Всегда добавляйте "Черновик" Заголовок для печати проектов документов, +"Always add ""Draft"" Heading for printing draft documents","Всегда добавляйте заголовок "Черновик" для печати черновых документов", Always use Account's Email Address as Sender,Всегда использовать почту учетной записи в качестве отправителя, Always use Account's Name as Sender's Name,Всегда использовать имя учетной записи в качестве имени отправителя, Amend,Изменен, @@ -456,7 +456,7 @@ Archive,Архив, Archived,Архивные, Archived Columns,Архивные столбцы, Are you sure you want to delete the attachment?,"Вы уверены, что хотите удалить вложение?", -Are you sure you want to relink this communication to {0}?,"Вы уверены, что хотите перелинковать эту куммуникацию на {0}?", +Are you sure you want to relink this communication to {0}?,"Вы уверены, что хотите перелинковать эту коммуникацию на {0}?", Are you sure?,Вы уверены?, Arial,Arial, "As a best practice, do not assign the same set of permission rule to different Roles. Instead, set multiple Roles to the same User.","В качестве лучшего решения, не присваивайте одинаковый набор правил доступа для различных ролей. Вместо этого, установите несколько ролей одному и тому же пользователю.", @@ -795,7 +795,7 @@ Customize...,Пользовательские настройки..., Customized HTML Templates for printing transactions.,Индивидуальные шаблоны HTML для печатных операций., Cut,Вырезать, DESC,DESC, -Daily Event Digest is sent for Calendar Events where reminders are set.,"Ежедневно событие Дайджест направляется на Календарь событий, где установлены напоминания.", +Daily Event Digest is sent for Calendar Events where reminders are set.,"Ежедневный дайджест событий направляется в календарь событий, где установлены напоминания.", Danger,Опасность, Dark Color,Темный цвет, Dashboard Chart,Диаграмма приборной панели, @@ -3159,7 +3159,7 @@ Aggregate Function field is required to create a dashboard chart,Поле Aggreg All Records,Все записи, Allot Points To Assigned Users,Выделить баллы назначенным пользователям, Allow Auto Repeat,Разрешить автоматическое повторение, -Allow Google Calendar Access,Разрешить доступ к Календарю Google, +Allow Google Calendar Access,Разрешить доступ к календарю Google, Allow Google Contacts Access,Разрешить доступ к контактам Google, Allow Google Drive Access,Разрешить доступ Google Drive, Allow Guest,Разрешить гость, @@ -3181,7 +3181,7 @@ Assignment Days,Дни назначения, Assignment Rule Day,День Правил Назначения, Assignments,Назначения, Attach a web link,Прикрепите веб-ссылку, -Authorize Google Calendar Access,Авторизовать доступ к Календарю Google, +Authorize Google Calendar Access,Авторизовать доступ к календарю Google, Authorize Google Contacts Access,Авторизовать доступ к контактам Google, Authorize Google Drive Access,Авторизовать Google Drive Access, Auto Repeat Document Creation Failed,Ошибка автоматического создания документа, @@ -3325,12 +3325,12 @@ Google API Settings.,Настройки Google API., Google Calendar,Календарь Google, "Google Calendar - Could not create Calendar for {0}, error code {1}.","Календарь Google. Не удалось создать календарь для {0}, код ошибки {1}.", "Google Calendar - Could not delete Event {0} from Google Calendar, error code {1}.","Календарь Google - не удалось удалить событие {0} из календаря Google, код ошибки {1}.", -"Google Calendar - Could not fetch event from Google Calendar, error code {0}.","Календарь Google - не удалось получить событие из Календаря Google, код ошибки {0}.", +"Google Calendar - Could not fetch event from Google Calendar, error code {0}.","Календарь Google - не удалось получить событие из календаря Google, код ошибки {0}.", "Google Calendar - Could not insert contact in Google Contacts {0}, error code {1}.","Календарь Google. Не удалось вставить контакт в контакты Google {0}, код ошибки {1}.", -"Google Calendar - Could not insert event in Google Calendar {0}, error code {1}.","Календарь Google. Не удалось вставить событие в Календарь Google {0}, код ошибки {1}.", -"Google Calendar - Could not update Event {0} in Google Calendar, error code {1}.","Календарь Google - не удалось обновить событие {0} в Календаре Google, код ошибки {1}.", -Google Calendar Event ID,Идентификатор события Календаря Google, -Google Calendar Integration.,Интеграция Календаря Google., +"Google Calendar - Could not insert event in Google Calendar {0}, error code {1}.","Календарь Google. Не удалось вставить событие в календарь Google {0}, код ошибки {1}.", +"Google Calendar - Could not update Event {0} in Google Calendar, error code {1}.","Календарь Google - не удалось обновить событие {0} в календаре Google, код ошибки {1}.", +Google Calendar Event ID,Идентификатор события календаря Google, +Google Calendar Integration.,Интеграция календаря Google., Google Calendar has been configured.,Календарь Google был настроен., Google Contacts,Контакты Google, "Google Contacts - Could not sync contacts from Google Contacts {0}, error code {1}.","Контакты Google - не удалось синхронизировать контакты из контактов Google {0}, код ошибки {1}.", @@ -3434,7 +3434,7 @@ New Notification,Новое уведомление, New {0}: {1},Новый {0}: {1}, Newsletter should have atleast one recipient,Бюллетень должен иметь как минимум одного получателя, No Events Today,Сегодня нет событий, -No Google Calendar Event to sync.,Нет события Календаря Google для синхронизации., +No Google Calendar Event to sync.,Нет события в календаря Google для синхронизации., No More Activity,Нет больше активности, No Name Specified for {0},Имя не указано для {0}, No Upcoming Events,Нет предстоящих событий, @@ -3484,11 +3484,11 @@ Print Settings...,Настройки печати..., Producer Document Name,Название документа производителя, Producer URL,URL производителя, Property Depends On,Недвижимость зависит от, -Pull from Google Calendar,Вытащить из календаря Google, -Pull from Google Contacts,Вытащить из контактов Google, -Pulled from Google Calendar,Вытащил из Календаря Google, -Pulled from Google Contacts,Вытащено из контактов Google, -Push to Google Calendar,Нажмите на Google Calendar, +Pull from Google Calendar,Загрузить из календаря Google, +Pull from Google Contacts,Загрузить из контактов Google, +Pulled from Google Calendar,Загружено из календаря Google, +Pulled from Google Contacts,Загружено из контактов Google, +Push to Google Calendar,Нажмите на Google календарь, Push to Google Contacts,Нажмите на контакты Google, Queue / Worker,Очередь / Рабочий, RAW Information Log,Необработанный информационный журнал, @@ -3526,7 +3526,7 @@ Select Date Range,Выберите диапазон дат, Select Field,Выберите поле, Select Field...,Выберите поле..., Select Filters,Выберите фильтры, -Select Google Calendar to which event should be synced.,"Выберите Календарь Google, к которому нужно синхронизировать событие.", +Select Google Calendar to which event should be synced.,"Выберите календарь Google, к которому нужно синхронизировать событие.", Select Google Contacts to which contact should be synced.,"Выберите Google Контакты, с которыми контакт должен быть синхронизирован.", Select Group By...,Выбрать группу по..., Select Mandatory,Выберите Обязательный, @@ -3593,7 +3593,7 @@ Time series based on is required to create a dashboard chart,Временной Time {0} must be in format: {1},Время {0} должно быть в формате: {1}, "To configure Auto Repeat, enable ""Allow Auto Repeat"" from {0}.","Чтобы настроить автоматический повтор, включите «Разрешить автоматический повтор» из {0}.", To enable it follow the instructions in the following link: {0},"Чтобы включить его, следуйте инструкциям по следующей ссылке: {0}", -"To use Google Calendar, enable {0}.","Чтобы использовать Календарь Google, включите {0}.", +"To use Google Calendar, enable {0}.","Чтобы использовать календарь Google, включите {0}.", "To use Google Contacts, enable {0}.","Чтобы использовать контакты Google, включите {0}.", "To use Google Drive, enable {0}.","Чтобы использовать Google Диск, включите {0}.", Today's Events,Сегодняшние события, @@ -3662,7 +3662,7 @@ submitted this document {0},отправил этот документ {0}, "tag name..., e.g. #tag","имя тега ..., например #tag", uploaded file,загруженный файл, via Data Import,через импорт данных, -{0} Google Calendar Events synced.,{0} События Календаря Google синхронизированы., +{0} Google Calendar Events synced.,{0} События календаря Google синхронизированы., {0} Google Contacts synced.,{0} Google Контакты синхронизированы., {0} are mandatory fields,{0} обязательные поля, {0} are required,{0} требуется, @@ -3697,7 +3697,7 @@ Cancelled,отменен, Chart,Диаграмма, Close,Закрыть, Communication,Коммуникация, -Compact Item Print,Компактный товара печати, +Compact Item Print,Компактное отоборажение товара при печати, Company,Организация, Complete,Завершен, Completed,Завершено, From 6734379848c507f368781df8aba1b1fb38bd7e7d Mon Sep 17 00:00:00 2001 From: Vladislav Date: Fri, 10 Jun 2022 16:17:15 +0300 Subject: [PATCH 025/205] Update ru.csv --- frappe/translations/ru.csv | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/frappe/translations/ru.csv b/frappe/translations/ru.csv index 5a23409ae8..0295ab88e9 100644 --- a/frappe/translations/ru.csv +++ b/frappe/translations/ru.csv @@ -10,7 +10,7 @@ Actions,Действия, Active,Активен, Add,Добавить, Add Comment,Добавить комментарий, -Add Row,Добавить ряд, +Add Row,Добавить строку, Address,Адрес, Address Line 2,Адрес (2-я строка), Address Title,Название адреса, @@ -1357,7 +1357,7 @@ Invalid Outgoing Mail Server or Port,Неверный сервер исходя Invalid Output Format,Неверный формат выходного, Invalid Password,Неверный пароль, Invalid Password:,Неверный пароль:, -Invalid Request,Неверная заявка, +Invalid Request,Неверный запрос, Invalid Search Field {0},Неверное поле поиска {0}, Invalid Subscription,Недействительная подписка, Invalid Token,Недопустимый токен, @@ -1972,7 +1972,7 @@ Pull Insert,Вставить вкладыш, Pull Update,Pull Update, Push,От себя, Push Delete,Нажмите Удалить, -Push Failed,Ошибка нажата, +Push Failed,Ошибка отправки, Push Insert,Push Insert, Push Update,Push Update, Python Module,Модуль Python, @@ -2122,9 +2122,9 @@ Row No,Строка №, Row Status,Статус строки, Row Values Changed,Значения строк Измененные, Row {0}: Not allowed to disable Mandatory for standard fields,Строка {0}: не разрешено отключать обязательные для стандартных полей, -Row {0}: Not allowed to enable Allow on Submit for standard fields,Ряд {0}: Не разрешается включать Разрешить проведение для стандартных полей, -Rows Added,Ряды Добавлено, -Rows Removed,Ряды Удалены, +Row {0}: Not allowed to enable Allow on Submit for standard fields,Строка {0}: Не разрешается включать Разрешить проведение для стандартных полей, +Rows Added,Строки добавлены, +Rows Removed,Строки удалены, Rule,Правило, Rule Name,Название правила, Rules defining transition of state in the workflow.,"Правила, определяющие переход этапов в потоке.", @@ -2194,8 +2194,8 @@ Select File Type,Выберите тип файла, Select Language...,Выберите язык..., Select Languages,Выберите языки, Select Module,Выбор модуля, -Select Print Format,Выберите Печатную форму, -Select Print Format to Edit,Выберите Печатную форму для Редактирование, +Select Print Format,Выберите бланк для печати, +Select Print Format to Edit,Выберите печатный бланк для редактирования, Select Role,Выберите роль, Select Table Columns for {0},Выберите столбцы таблицы для {0}, Select Your Region,Выберите регион, @@ -3814,7 +3814,7 @@ and,и, {0} Name,{0} Имя, {0} is required,{0} требуется, ALL,ВСЕ, -Attach File,Прикрепить файл, +Attach File,Прикрепить файл, Barcode,Штрих-код, Beginning with,Начиная с, Bold,Жирный, @@ -4011,11 +4011,11 @@ Setup > User,Настройка> Пользователь, Setup > Customize Form,Настройка> Настройка формы, Setup > User Permissions,Настройка> Полномочия пользователя, "Error connecting to QZ Tray Application...

You need to have QZ Tray application installed and running, to use the Raw Print feature.

Click here to Download and install QZ Tray.
Click here to learn more about Raw Printing.","Ошибка подключения к приложению QZ Tray ...

Вам необходимо установить и запустить приложение QZ Tray, чтобы использовать функцию Raw Print.

Нажмите здесь, чтобы загрузить и установить QZ Tray .
Нажмите здесь, чтобы узнать больше о Raw Printing .", -No email account associated with the User. Please add an account under User > Email Inbox.,"Нет учетной записи электронной почты, связанной с пользователем. Пожалуйста, добавьте учетную запись под User> Email Inbox.", +No email account associated with the User. Please add an account under User > Email Inbox.,"Нет учетной записи электронной почты, связанной с пользователем. Пожалуйста, добавьте электронную почту в настройках пользователя", "For comparison, use >5, <10 or =324. For ranges, use 5:10 (for values between 5 & 10).","Для сравнения используйте> 5, <10 или = 324. Для диапазонов используйте 5:10 (для значений от 5 до 10).", -No default Address Template found. Please create a new one from Setup > Printing and Branding > Address Template.,"Шаблон адреса по умолчанию не найден. Создайте новый, выбрав «Настройка»> «Печать и брендинг»> «Шаблон адреса».", -Please setup default Email Account from Setup > Email > Email Account,"Пожалуйста, настройте учетную запись электронной почты по умолчанию из меню «Настройка»> «Электронная почта»>", -Email Account not setup. Please create a new Email Account from Setup > Email > Email Account,"Учетная запись электронной почты не настроена. Пожалуйста, создайте новую учетную запись электронной почты от Настройка> Электронная почта> Учетная запись электронной почты", +No default Address Template found. Please create a new one from Setup > Printing and Branding > Address Template.,"Шаблон адреса по умолчанию не найден. Создайте новый, выбрав «Настройка» > «Печать и брендинг» > «Шаблон адреса».", +Please setup default Email Account from Setup > Email > Email Account,"Пожалуйста, настройте учетную запись электронной почты по умолчанию из меню «Настройка» > «Электронная почта»>", +Email Account not setup. Please create a new Email Account from Setup > Email > Email Account,"Учетная запись электронной почты не настроена. Пожалуйста, создайте новую учетную запись электронной почты в «Настройки» > «Электронная почта» > «Учетная запись электронной почты»", Attach file,Прикрепить файл, Contribution Status,Статус вклада, Contribution Document Name,Название документа о взносе, From 718ad100f5613def280f2c33893193c7c3a90fb3 Mon Sep 17 00:00:00 2001 From: Vladislav Date: Fri, 10 Jun 2022 16:32:18 +0300 Subject: [PATCH 026/205] Update ru.csv --- frappe/translations/ru.csv | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/frappe/translations/ru.csv b/frappe/translations/ru.csv index 0295ab88e9..2999f9fa23 100644 --- a/frappe/translations/ru.csv +++ b/frappe/translations/ru.csv @@ -181,7 +181,7 @@ Portal,Портал, Portal Settings,Настройки портала, Preview,Просмотр, Primary,Основной, -Print Format,Формат печати, +Print Format,Печатный бланк, Print Settings,Настройки печати, Print taxes with zero amount,Печать налогов с нулевой суммой, Private,Личные, @@ -745,7 +745,7 @@ Create Chart,Создать диаграмму, Create New,Создать, Create Post,Создать пост, Create User Email,Создать электронной почты пользователя, -Create a New Format,Создать новый формат, +Create a New Format,Создать новый бланк, Create a new record,Создать новую запись, Create a new {0},Создать {0}, Create and Send Newsletters,Создание и отправка рассылки, @@ -928,7 +928,7 @@ Download Report,Скачать отчет, Download Your Data,Скачать ваши данные, Download link for your backup will be emailed on the following email address: {0},Ссылка для скачивания вашей резервной копии будет отправлена на следующий электронный адрес: {0}, Download with Data,Скачать с данными, -Drag and Drop tool to build and customize Print Formats.,Конструктор для сборки и настройки форматов печати., +Drag and Drop tool to build and customize Print Formats.,Инструмент для сборки и настройки бланков для печати., Drag elements from the sidebar to add. Drag them back to trash.,"Перетащите элементы с боковой панели, чтобы добавить. Перетащите их обратно чтобы удалить.", Dropbox Access Key,Dropbox ключ доступа, Dropbox Access Secret,Dropbox секретный ключ, @@ -1605,7 +1605,7 @@ New Activity,Новая активность, New Chat,Новый чат, New Comment on {0}: {1},Новый комментарий к {0}: {1}, New Connection,Новое соединение, -New Custom Print Format,Новый пользовательский Формат печати, +New Custom Print Format,Новый пользовательский печатный бланк, New Email,Новая электронная почта, New Email Account,Новый аккаунт электронной почты, New Event,Новое событие, @@ -1616,7 +1616,7 @@ New Name,Новое имя, New Newsletter,Новый бюллетень, New Password,Новый пароль, New Password Required.,Требуется новый пароль., -New Print Format Name,Название нового формата печати, +New Print Format Name,Название нового печатного бланка, New Report name,Новое название отчёта, New Value,Новое значение, New data will be inserted.,Новые данные будут вставлены., @@ -1932,7 +1932,7 @@ Previous,Предыдущая, Previous Hash,Предыдущий хэш, Primary Color,Основной цвет, Print Documents,Печать документов, -Print Format Builder,Конструктор печатных форм, +Print Format Builder,Конструктор бланков для печати, Print Format Help,Помощь по печатным формам, Print Format Type,Тип печатной формы, Print Format {0} is disabled,Печатная форма {0} отключена, @@ -2200,7 +2200,7 @@ Select Role,Выберите роль, Select Table Columns for {0},Выберите столбцы таблицы для {0}, Select Your Region,Выберите регион, Select a Brand Image first.,Выберите бренд изображение в первую очередь., -Select a DocType to make a new format,"Выберите DOCTYPE, чтобы сделать новый формат", +Select a DocType to make a new format,"Выберите DOCTYPE, чтобы сделать новый бланк", Select a chat to start messaging.,"Выберите чат, чтобы начать обмен сообщениями.", Select a group node first.,Выберите узел группы в первую очередь., Select an existing format to edit or start a new format.,Выберите существующий формат для редактирования или начать новый формат., @@ -2360,7 +2360,7 @@ Spam,Спам, SparkPost,SparkPost, Special Characters are not allowed,Спецсимволы не допустимы, "Standard DocType cannot have default print format, use Customize Form","Стандартный DocType не может иметь формат печати по умолчанию, используйте Настроить форму", -Standard Print Format cannot be updated,Стандартный Формат печати не может быть обновлен, +Standard Print Format cannot be updated,Стандартный печатный бланк не может быть обновлен, Standard Print Style cannot be changed. Please duplicate to edit.,Стандартный стиль печати не может быть изменен. Повторите попытку для редактирования., Standard Reports,Стандартные отчеты, Standard Sidebar Menu,Стандартное боковое меню, From d3a935af604f15932b465fda7e0beb61640360dd Mon Sep 17 00:00:00 2001 From: Vladislav Date: Fri, 10 Jun 2022 17:24:30 +0300 Subject: [PATCH 027/205] Update ru.csv --- frappe/translations/ru.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/translations/ru.csv b/frappe/translations/ru.csv index 2999f9fa23..81b8bdecd2 100644 --- a/frappe/translations/ru.csv +++ b/frappe/translations/ru.csv @@ -2962,7 +2962,7 @@ question-sign,Вопрос-знак, remove-circle,удалить-круг, remove-sign,удалить-знак, removed,удален, -renamed from {0} to {1},переименован из {0} до {1}, +renamed from {0} to {1},переименован из {0} в {1}, repeat,повторение, resize-full,изменить размер-полный, resize-horizontal,изменить размер горизонтальной, From cdc850f12eb52214fb7d4d6d37b3f5fbc75ee08c Mon Sep 17 00:00:00 2001 From: barredterra <14891507+barredterra@users.noreply.github.com> Date: Fri, 10 Jun 2022 16:30:20 +0200 Subject: [PATCH 028/205] test: user permissions affecting User --- frappe/tests/test_permissions.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/frappe/tests/test_permissions.py b/frappe/tests/test_permissions.py index 4164b0be36..26d5c714ef 100644 --- a/frappe/tests/test_permissions.py +++ b/frappe/tests/test_permissions.py @@ -672,3 +672,31 @@ class TestPermissions(FrappeTestCase): doctype="Has Role", parent_doctype="Has Role", ) + + def test_select_user(self): + """If test3@example.com is restricted by a User Permission to see only + users linked to a certain doctype (in this case: Gender "Female"), he + should not be able to query other users (Gender "Male"). + """ + # ensure required genders exist + for gender in ("Male", "Female"): + if frappe.db.exists("Gender", gender): + continue + + frappe.get_doc({"doctype": "Gender", "gender": gender}).insert() + + # asssign gender to test users + frappe.db.set_value("User", "test1@example.com", "gender", "Male") + frappe.db.set_value("User", "test2@example.com", "gender", "Female") + frappe.db.set_value("User", "test3@example.com", "gender", "Female") + + # restrict test3@example.com to see only female users + add_user_permission("Gender", "Female", "test3@example.com") + + # become user test3@example.com and see what users he can query + frappe.set_user("test3@example.com") + users = frappe.get_list("User", pluck="name") + + self.assertNotIn("test1@example.com", users) + self.assertIn("test2@example.com", users) + self.assertIn("test3@example.com", users) From c2b9197624efbb66a2f0a7743c96d7ed163cd768 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Fri, 10 Jun 2022 20:01:33 +0530 Subject: [PATCH 029/205] fix: default tab fieldname conflict --- frappe/public/js/frappe/form/layout.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/form/layout.js b/frappe/public/js/frappe/form/layout.js index 403abf0981..c1c9967507 100644 --- a/frappe/public/js/frappe/form/layout.js +++ b/frappe/public/js/frappe/form/layout.js @@ -122,7 +122,8 @@ frappe.ui.form.Layout = class Layout { } if (this.is_tabbed_layout()) { - let default_tab = {label: __('Details'), fieldname: 'details', fieldtype: "Tab Break"}; + // add a tab without `fieldname` to avoid conflicts + let default_tab = {label: __('Details'), fieldtype: "Tab Break"}; let first_tab = this.fields[1].fieldtype === "Tab Break" ? this.fields[1] : null; if (!first_tab) { this.fields.splice(1, 0, default_tab); From c157564281a786fba0835fd58310947074e56772 Mon Sep 17 00:00:00 2001 From: Ritwik Puri Date: Fri, 10 Jun 2022 22:19:38 +0530 Subject: [PATCH 030/205] fix: use doctype property instead of name for Customize Form in validate_autoincrement_autoname (#17139) --- frappe/core/doctype/doctype/doctype.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/frappe/core/doctype/doctype/doctype.py b/frappe/core/doctype/doctype/doctype.py index 3e58146ae7..e834b698d5 100644 --- a/frappe/core/doctype/doctype/doctype.py +++ b/frappe/core/doctype/doctype/doctype.py @@ -8,6 +8,7 @@ import os # imports - standard imports import re import shutil +from typing import TYPE_CHECKING, Union # imports - module imports import frappe @@ -35,6 +36,9 @@ from frappe.query_builder.functions import Concat from frappe.utils import cint from frappe.website.utils import clear_cache +if TYPE_CHECKING: + from frappe.custom.doctype.customize_form.customize_form import CustomizeForm + DEPENDS_ON_PATTERN = re.compile(r'[\w\.:_]+\s*={1}\s*[\w\.@\'"]+') ILLEGAL_FIELDNAME_PATTERN = re.compile("""['",./%@()<>{}]""") WHITESPACE_PADDING_PATTERN = re.compile(r"^[ \t\n\r]+|[ \t\n\r]+$", flags=re.ASCII) @@ -916,11 +920,11 @@ def validate_series(dt, autoname=None, name=None): frappe.throw(_("Series {0} already used in {1}").format(prefix, used_in[0][0])) -def validate_autoincrement_autoname(dt: DocType) -> bool: +def validate_autoincrement_autoname(dt: Union[DocType, "CustomizeForm"]) -> bool: """Checks if can doctype can change to/from autoincrement autoname""" - def get_autoname_before_save(dt: DocType) -> str: - if dt.name == "Customize Form": + def get_autoname_before_save(dt: Union[DocType, "CustomizeForm"]) -> str: + if dt.doctype == "Customize Form": property_value = frappe.db.get_value( "Property Setter", {"doc_type": dt.doc_type, "property": "autoname"}, "value" ) @@ -943,10 +947,10 @@ def validate_autoincrement_autoname(dt: DocType) -> bool: or (not is_autoname_autoincrement and autoname_before_save == "autoincrement") ): - if frappe.get_meta(dt.name).issingle: - if dt.name == "Customize Form": - frappe.throw(_("Cannot change to/from autoincrement autoname in Customize Form")) + if dt.doctype == "Customize Form": + frappe.throw(_("Cannot change to/from autoincrement autoname in Customize Form")) + if frappe.get_meta(dt.name).issingle: return False if not frappe.get_all(dt.name, limit=1): From 536f88aafc6892250a524140ee773295cfa8c67d Mon Sep 17 00:00:00 2001 From: phot0n Date: Sun, 12 Jun 2022 00:05:32 +0530 Subject: [PATCH 031/205] fix: format date as per system settings for dashboard charts --- .../desk/doctype/dashboard_chart/dashboard_chart.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/frappe/desk/doctype/dashboard_chart/dashboard_chart.py b/frappe/desk/doctype/dashboard_chart/dashboard_chart.py index 246c9ad4cd..ca29bad33b 100644 --- a/frappe/desk/doctype/dashboard_chart/dashboard_chart.py +++ b/frappe/desk/doctype/dashboard_chart/dashboard_chart.py @@ -14,6 +14,7 @@ from frappe.model.naming import append_number_if_name_exists from frappe.modules.export_file import export_to_files from frappe.utils import cint, get_datetime, getdate, now_datetime, nowdate from frappe.utils.dashboard import cache_source +from frappe.utils.data import format_date from frappe.utils.dateutils import ( get_dates_from_timegrain, get_from_date_from_timespan, @@ -221,13 +222,16 @@ def get_chart_config(chart, filters, timespan, timegrain, from_date, to_date): result = get_result(data, timegrain, from_date, to_date, chart.chart_type) - chart_config = { - "labels": [get_period(r[0], timegrain) for r in result], + return { + "labels": [ + format_date(get_period(r[0], timegrain)) + if timegrain in ("Daily", "Weekly") + else get_period(r[0], timegrain) + for r in result + ], "datasets": [{"name": chart.name, "values": [r[1] for r in result]}], } - return chart_config - def get_heatmap_chart_config(chart, filters, heatmap_year): aggregate_function = get_aggregate_function(chart.chart_type) From 89889753e4dadab618bdbf8101823e8187cc995d Mon Sep 17 00:00:00 2001 From: Florian HENRY Date: Sun, 12 Jun 2022 12:11:25 +0200 Subject: [PATCH 032/205] fix: JS error in page.js when the is a quote in button translation --- frappe/public/js/frappe/ui/page.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/ui/page.js b/frappe/public/js/frappe/ui/page.js index 70e0862ba5..eded1aefc5 100644 --- a/frappe/public/js/frappe/ui/page.js +++ b/frappe/public/js/frappe/ui/page.js @@ -510,7 +510,7 @@ frappe.ui.Page = class Page { if (!label || !parent) return false; - const item_selector = `${selector}[data-label='${encodeURIComponent(label)}']`; + const item_selector = `${selector}[data-label="${encodeURIComponent(label)}"]`; const existing_items = $(parent).find(item_selector); return existing_items?.length > 0 && existing_items; From 03f48580ef1baa91b3d1bd7fdb90e20fa3f9adb4 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sun, 12 Jun 2022 16:24:39 +0530 Subject: [PATCH 033/205] test: button with single quote in label --- cypress/integration/custom_buttons.js | 1 + 1 file changed, 1 insertion(+) diff --git a/cypress/integration/custom_buttons.js b/cypress/integration/custom_buttons.js index e2f02668e9..6045d009c2 100644 --- a/cypress/integration/custom_buttons.js +++ b/cypress/integration/custom_buttons.js @@ -4,6 +4,7 @@ const test_button_names = [ "Porcupine Tree (the GOAT)", "AC / DC", `Electronic Dance "music"`, + "l'imperatrice", ]; const add_button = (label, group = "TestGroup") => { From 21442f5cbaa5c7b1a5a0aa1c8a376ddbc6d04ad6 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sun, 12 Jun 2022 17:19:13 +0530 Subject: [PATCH 034/205] perf: disable creating version for new docs Each new doc inserts a version, this contains nothing but creator and creation time.. which is already immutable information on the original document. This was added for cases like data import to track from where document got created, ref: https://github.com/frappe/frappe/commit/b7dfe7969dd6b2851844b1e9c090cbb6447748de Fix: only add a version on creation IF creation info is present on flags --- frappe/core/doctype/version/test_version.py | 13 +++++++++++++ frappe/core/doctype/version/version.py | 19 ++++++++++++++++--- frappe/model/document.py | 7 +++---- frappe/tests/test_form_load.py | 2 +- 4 files changed, 33 insertions(+), 8 deletions(-) diff --git a/frappe/core/doctype/version/test_version.py b/frappe/core/doctype/version/test_version.py index c35430b17b..3e82f30f06 100644 --- a/frappe/core/doctype/version/test_version.py +++ b/frappe/core/doctype/version/test_version.py @@ -32,6 +32,19 @@ class TestVersion(unittest.TestCase): self.assertEqual(get_old_values(diff)[1], "01-01-2014 00:00:00") self.assertEqual(get_new_values(diff)[1], "07-20-2017 00:00:00") + def test_no_version_on_new_doc(self): + from frappe.desk.form.load import get_versions + + t = frappe.get_doc(doctype="ToDo", description="something") + t.save(ignore_version=False) + + self.assertFalse(get_versions(t)) + + t = frappe.get_doc(t.doctype, t.name) + t.description = "changed" + t.save(ignore_version=False) + self.assertTrue(get_versions(t)) + def get_fieldnames(change_array): return [d[0] for d in change_array] diff --git a/frappe/core/doctype/version/version.py b/frappe/core/doctype/version/version.py index 863885e85c..fa6ba0a9cf 100644 --- a/frappe/core/doctype/version/version.py +++ b/frappe/core/doctype/version/version.py @@ -2,6 +2,7 @@ # License: MIT. See LICENSE import json +from typing import Optional import frappe from frappe.model import no_value_fields, table_fields @@ -9,7 +10,15 @@ from frappe.model.document import Document class Version(Document): - def set_diff(self, old, new): + def update_version_info(self, old: Optional[Document], new: Document) -> bool: + """Update changed info and return true if change contains useful data.""" + if not old: + # Check if doc has some information about creation source like data import + return self.for_insert(new) + else: + return self.set_diff(old, new) + + def set_diff(self, old: Document, new: Document) -> bool: """Set the data property with the diff of the docs if present""" diff = get_diff(old, new) if diff: @@ -20,8 +29,11 @@ class Version(Document): else: return False - def for_insert(self, doc): + def for_insert(self, doc: Document) -> bool: updater_reference = doc.flags.updater_reference + if not updater_reference: + return False + data = { "creation": doc.creation, "updater_reference": updater_reference, @@ -29,7 +41,8 @@ class Version(Document): } self.ref_doctype = doc.doctype self.docname = doc.name - self.data = frappe.as_json(data) + self.data = frappe.as_json(data, indent=None, separators=(",", ":")) + return True def get_data(self): return json.loads(self.data) diff --git a/frappe/model/document.py b/frappe/model/document.py index fa1f423d11..a98d49e58c 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -1198,11 +1198,10 @@ class Document(BaseDocument): return version = frappe.new_doc("Version") - if not self._doc_before_save: - version.for_insert(self) - version.insert(ignore_permissions=True) - elif version.set_diff(self._doc_before_save, self): + + if is_useful_diff := version.update_version_info(self._doc_before_save, self): version.insert(ignore_permissions=True) + if not frappe.flags.in_migrate: # follow since you made a change? if frappe.get_cached_value("User", frappe.session.user, "follow_created_documents"): diff --git a/frappe/tests/test_form_load.py b/frappe/tests/test_form_load.py index e92b8c3ff2..22db56eeef 100644 --- a/frappe/tests/test_form_load.py +++ b/frappe/tests/test_form_load.py @@ -181,7 +181,7 @@ class TestFormLoad(unittest.TestCase): self.assertEqual(len(docinfo.comments), 1) self.assertIn("test", docinfo.comments[0].content) - self.assertGreaterEqual(len(docinfo.versions), 2) + self.assertGreaterEqual(len(docinfo.versions), 1) self.assertEqual(set(docinfo.tags.split(",")), {"more_tag", "test_tag"}) From a4f2912fdf5e0945d89fe588f5cb6b02cc3c2582 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Mon, 13 Jun 2022 11:11:47 +0530 Subject: [PATCH 035/205] fix: Handle case where document title can be NONE (cherry picked from commit 9b67fc24bc290789158f37a8f2ce10b505878792) --- frappe/www/printview.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/www/printview.py b/frappe/www/printview.py index 85ffc671fa..c8595a6f2c 100644 --- a/frappe/www/printview.py +++ b/frappe/www/printview.py @@ -63,7 +63,7 @@ def get_context(context): "body": body, "print_style": print_style, "comment": frappe.session.user, - "title": frappe.utils.strip_html(doc.get_title()), + "title": frappe.utils.strip_html(doc.get_title() or doc.name), "lang": frappe.local.lang, "layout_direction": "rtl" if is_rtl() else "ltr", "doctype": frappe.form_dict.doctype, From 53a079f1017e883bb07c745135db9506705b3841 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Mon, 13 Jun 2022 11:12:53 +0530 Subject: [PATCH 036/205] fix: `doc.get_title` should return empty string if title is not set (cherry picked from commit b244c9148192362de43aa483b328709745f3be82) --- frappe/model/document.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/model/document.py b/frappe/model/document.py index a98d49e58c..22514df75a 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -438,7 +438,7 @@ class Document(BaseDocument): def get_title(self): """Get the document title based on title_field or `title` or `name`""" - return self.get(self.meta.get_title_field()) + return self.get(self.meta.get_title_field()) or "" def set_title_field(self): """Set title field based on template""" From 63a5db94cfa728b6116e2a806ae8f2fc7988e4d8 Mon Sep 17 00:00:00 2001 From: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> Date: Mon, 13 Jun 2022 12:03:39 +0530 Subject: [PATCH 037/205] chore: Add mergify[bot] to exception list https://user-images.githubusercontent.com/13928957/173293263-4987c494-4524-46dd-996c-36f1ca760c68.png According to this mergify's login id is `mergify[bot]` so am guessing this should work. --- .mergify.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.mergify.yml b/.mergify.yml index f1333362a8..97df91a927 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -7,6 +7,7 @@ pull_request_rules: - author!=gavindsouza - author!=deepeshgarg007 - author!=ankush + - author!=mergify[bot] - or: - base=version-13 - base=version-12 From eea5cc52da8d246629fa4c7793745faf35f15276 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Mon, 13 Jun 2022 16:24:00 +0530 Subject: [PATCH 038/205] refactor!: Remove Data Migration Tool (#17134) * refactor: Remove Data Migration Tool * chore: unnecessary imports --- frappe/data_migration/__init__.py | 0 frappe/data_migration/doctype/__init__.py | 0 .../data_migration_connector/__init__.py | 0 .../connectors/__init__.py | 0 .../connectors/base.py | 24 - .../connectors/frappe_connection.py | 32 - .../data_migration_connector.js | 47 - .../data_migration_connector.json | 307 ------- .../data_migration_connector.py | 107 --- .../test_data_migration_connector.py | 8 - .../data_migration_mapping/__init__.py | 0 .../data_migration_mapping.js | 8 - .../data_migration_mapping.json | 456 ---------- .../data_migration_mapping.py | 83 -- .../test_data_migration_mapping.py | 8 - .../data_migration_mapping_detail/__init__.py | 0 .../data_migration_mapping_detail.json | 163 ---- .../data_migration_mapping_detail.py | 9 - .../doctype/data_migration_plan/__init__.py | 0 .../data_migration_plan.js | 10 - .../data_migration_plan.json | 224 ----- .../data_migration_plan.py | 78 -- .../test_data_migration_plan.py | 8 - .../data_migration_plan_mapping/__init__.py | 0 .../data_migration_plan_mapping.json | 103 --- .../data_migration_plan_mapping.py | 9 - .../doctype/data_migration_run/__init__.py | 0 .../data_migration_run/data_migration_run.js | 14 - .../data_migration_run.json | 838 ------------------ .../data_migration_run/data_migration_run.py | 514 ----------- .../test_data_migration_run.py | 128 --- frappe/model/sync.py | 18 - frappe/modules.txt | 1 - frappe/patches.txt | 1 + .../v14_0/delete_data_migration_tool.py | 12 + 35 files changed, 13 insertions(+), 3197 deletions(-) delete mode 100644 frappe/data_migration/__init__.py delete mode 100644 frappe/data_migration/doctype/__init__.py delete mode 100644 frappe/data_migration/doctype/data_migration_connector/__init__.py delete mode 100644 frappe/data_migration/doctype/data_migration_connector/connectors/__init__.py delete mode 100644 frappe/data_migration/doctype/data_migration_connector/connectors/base.py delete mode 100644 frappe/data_migration/doctype/data_migration_connector/connectors/frappe_connection.py delete mode 100644 frappe/data_migration/doctype/data_migration_connector/data_migration_connector.js delete mode 100644 frappe/data_migration/doctype/data_migration_connector/data_migration_connector.json delete mode 100644 frappe/data_migration/doctype/data_migration_connector/data_migration_connector.py delete mode 100644 frappe/data_migration/doctype/data_migration_connector/test_data_migration_connector.py delete mode 100644 frappe/data_migration/doctype/data_migration_mapping/__init__.py delete mode 100644 frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.js delete mode 100644 frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.json delete mode 100644 frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.py delete mode 100644 frappe/data_migration/doctype/data_migration_mapping/test_data_migration_mapping.py delete mode 100644 frappe/data_migration/doctype/data_migration_mapping_detail/__init__.py delete mode 100644 frappe/data_migration/doctype/data_migration_mapping_detail/data_migration_mapping_detail.json delete mode 100644 frappe/data_migration/doctype/data_migration_mapping_detail/data_migration_mapping_detail.py delete mode 100644 frappe/data_migration/doctype/data_migration_plan/__init__.py delete mode 100644 frappe/data_migration/doctype/data_migration_plan/data_migration_plan.js delete mode 100644 frappe/data_migration/doctype/data_migration_plan/data_migration_plan.json delete mode 100644 frappe/data_migration/doctype/data_migration_plan/data_migration_plan.py delete mode 100644 frappe/data_migration/doctype/data_migration_plan/test_data_migration_plan.py delete mode 100644 frappe/data_migration/doctype/data_migration_plan_mapping/__init__.py delete mode 100644 frappe/data_migration/doctype/data_migration_plan_mapping/data_migration_plan_mapping.json delete mode 100644 frappe/data_migration/doctype/data_migration_plan_mapping/data_migration_plan_mapping.py delete mode 100644 frappe/data_migration/doctype/data_migration_run/__init__.py delete mode 100644 frappe/data_migration/doctype/data_migration_run/data_migration_run.js delete mode 100644 frappe/data_migration/doctype/data_migration_run/data_migration_run.json delete mode 100644 frappe/data_migration/doctype/data_migration_run/data_migration_run.py delete mode 100644 frappe/data_migration/doctype/data_migration_run/test_data_migration_run.py create mode 100644 frappe/patches/v14_0/delete_data_migration_tool.py diff --git a/frappe/data_migration/__init__.py b/frappe/data_migration/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/data_migration/doctype/__init__.py b/frappe/data_migration/doctype/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/data_migration/doctype/data_migration_connector/__init__.py b/frappe/data_migration/doctype/data_migration_connector/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/data_migration/doctype/data_migration_connector/connectors/__init__.py b/frappe/data_migration/doctype/data_migration_connector/connectors/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/data_migration/doctype/data_migration_connector/connectors/base.py b/frappe/data_migration/doctype/data_migration_connector/connectors/base.py deleted file mode 100644 index 7d2b320c59..0000000000 --- a/frappe/data_migration/doctype/data_migration_connector/connectors/base.py +++ /dev/null @@ -1,24 +0,0 @@ -from abc import ABCMeta, abstractmethod - -from frappe.utils.password import get_decrypted_password - - -class BaseConnection(metaclass=ABCMeta): - @abstractmethod - def get(self, remote_objectname, fields=None, filters=None, start=0, page_length=10): - pass - - @abstractmethod - def insert(self, doctype, doc): - pass - - @abstractmethod - def update(self, doctype, doc, migration_id): - pass - - @abstractmethod - def delete(self, doctype, migration_id): - pass - - def get_password(self): - return get_decrypted_password("Data Migration Connector", self.connector.name) diff --git a/frappe/data_migration/doctype/data_migration_connector/connectors/frappe_connection.py b/frappe/data_migration/doctype/data_migration_connector/connectors/frappe_connection.py deleted file mode 100644 index 8228529562..0000000000 --- a/frappe/data_migration/doctype/data_migration_connector/connectors/frappe_connection.py +++ /dev/null @@ -1,32 +0,0 @@ -import frappe -from frappe.frappeclient import FrappeClient - -from .base import BaseConnection - - -class FrappeConnection(BaseConnection): - def __init__(self, connector): - self.connector = connector - self.connection = FrappeClient( - self.connector.hostname, self.connector.username, self.get_password() - ) - self.name_field = "name" - - def insert(self, doctype, doc): - doc = frappe._dict(doc) - doc.doctype = doctype - return self.connection.insert(doc) - - def update(self, doctype, doc, migration_id): - doc = frappe._dict(doc) - doc.doctype = doctype - doc.name = migration_id - return self.connection.update(doc) - - def delete(self, doctype, migration_id): - return self.connection.delete(doctype, migration_id) - - def get(self, doctype, fields='"*"', filters=None, start=0, page_length=20): - return self.connection.get_list( - doctype, fields=fields, filters=filters, limit_start=start, limit_page_length=page_length - ) diff --git a/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.js b/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.js deleted file mode 100644 index 0898fcf4e7..0000000000 --- a/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.js +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright (c) 2017, Frappe Technologies and contributors -// For license information, please see license.txt - -frappe.ui.form.on('Data Migration Connector', { - onload(frm) { - if(frappe.boot.developer_mode) { - frm.add_custom_button(__('New Connection'), () => frm.events.new_connection(frm)); - } - }, - new_connection(frm) { - const d = new frappe.ui.Dialog({ - title: __('New Connection'), - fields: [ - { label: __('Module'), fieldtype: 'Link', options: 'Module Def', reqd: 1 }, - { label: __('Connection Name'), fieldtype: 'Data', description: 'For e.g: Shopify Connection', reqd: 1 }, - ], - primary_action_label: __('Create'), - primary_action: (values) => { - let { module, connection_name } = values; - - frm.events.create_new_connection(module, connection_name) - .then(r => { - if (r.message) { - const connector_name = connection_name - .replace('connection', 'Connector') - .replace('Connection', 'Connector') - .trim(); - - frm.set_value('connector_name', connector_name); - frm.set_value('connector_type', 'Custom'); - frm.set_value('python_module', r.message); - frm.save(); - frappe.show_alert(__("New module created {0}", [r.message])); - d.hide(); - } - }); - } - }); - - d.show(); - }, - create_new_connection(module, connection_name) { - return frappe.call('frappe.data_migration.doctype.data_migration_connector.data_migration_connector.create_new_connection', { - module, connection_name - }); - } -}); diff --git a/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.json b/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.json deleted file mode 100644 index 338d59aadd..0000000000 --- a/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.json +++ /dev/null @@ -1,307 +0,0 @@ -{ - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 1, - "autoname": "field:connector_name", - "beta": 1, - "creation": "2017-08-11 05:03:27.091416", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", - "fields": [ - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "connector_name", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Connector Name", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "depends_on": "eval:!doc.is_custom", - "fieldname": "connector_type", - "fieldtype": "Select", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Connector Type", - "length": 0, - "no_copy": 0, - "options": "\nFrappe\nCustom", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "depends_on": "eval:doc.connector_type == 'Custom'", - "fieldname": "python_module", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Python Module", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "authentication_credentials", - "fieldtype": "Section Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Authentication Credentials", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "default": "", - "fieldname": "hostname", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Hostname", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "database_name", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Database Name", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "username", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Username", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "password", - "fieldtype": "Password", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Password", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2017-12-01 13:38:55.992499", - "modified_by": "Administrator", - "module": "Data Migration", - "name": "Data Migration Connector", - "name_case": "", - "owner": "Administrator", - "permissions": [ - { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "set_user_permissions": 0, - "share": 1, - "submit": 0, - "write": 1 - } - ], - "quick_entry": 0, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1, - "track_seen": 0 -} \ No newline at end of file diff --git a/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.py b/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.py deleted file mode 100644 index 9db7fc2445..0000000000 --- a/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.py +++ /dev/null @@ -1,107 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017, Frappe Technologies and contributors -# License: MIT. See LICENSE - -import os - -import frappe -from frappe import _ -from frappe.model.document import Document -from frappe.modules.export_file import create_init_py - -from .connectors.base import BaseConnection -from .connectors.frappe_connection import FrappeConnection - - -class DataMigrationConnector(Document): - def validate(self): - if not (self.python_module or self.connector_type): - frappe.throw(_("Enter python module or select connector type")) - - if self.python_module: - try: - get_connection_class(self.python_module) - except: - frappe.throw(frappe._("Invalid module path")) - - def get_connection(self): - if self.python_module: - _class = get_connection_class(self.python_module) - return _class(self) - else: - self.connection = FrappeConnection(self) - - return self.connection - - -@frappe.whitelist() -def create_new_connection(module, connection_name): - if not frappe.conf.get("developer_mode"): - frappe.msgprint(_("Please enable developer mode to create new connection")) - return - # create folder - module_path = frappe.get_module_path(module) - connectors_folder = os.path.join(module_path, "connectors") - frappe.create_folder(connectors_folder) - - # create init py - create_init_py(module_path, "connectors", "") - - connection_class = connection_name.replace(" ", "") - file_name = frappe.scrub(connection_name) + ".py" - file_path = os.path.join(module_path, "connectors", file_name) - - # create boilerplate file - with open(file_path, "w") as f: - f.write(connection_boilerplate.format(connection_class=connection_class)) - - # get python module string from file_path - app_name = frappe.db.get_value("Module Def", module, "app_name") - python_module = os.path.relpath(file_path, "../apps/{0}".format(app_name)).replace( - os.path.sep, "." - )[:-3] - - return python_module - - -def get_connection_class(python_module): - filename = python_module.rsplit(".", 1)[-1] - classname = frappe.unscrub(filename).replace(" ", "") - module = frappe.get_module(python_module) - - raise_error = False - if hasattr(module, classname): - _class = getattr(module, classname) - if not issubclass(_class, BaseConnection): - raise_error = True - else: - raise_error = True - - if raise_error: - raise ImportError(filename) - - return _class - - -connection_boilerplate = """from frappe.data_migration.doctype.data_migration_connector.connectors.base import BaseConnection - -class {connection_class}(BaseConnection): - def __init__(self, connector): - # self.connector = connector - # self.connection = YourModule(self.connector.username, self.get_password()) - # self.name_field = 'id' - pass - - def get(self, remote_objectname, fields=None, filters=None, start=0, page_length=10): - pass - - def insert(self, doctype, doc): - pass - - def update(self, doctype, doc, migration_id): - pass - - def delete(self, doctype, migration_id): - pass - -""" diff --git a/frappe/data_migration/doctype/data_migration_connector/test_data_migration_connector.py b/frappe/data_migration/doctype/data_migration_connector/test_data_migration_connector.py deleted file mode 100644 index c4090796ab..0000000000 --- a/frappe/data_migration/doctype/data_migration_connector/test_data_migration_connector.py +++ /dev/null @@ -1,8 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017, Frappe Technologies and Contributors -# License: MIT. See LICENSE -import unittest - - -class TestDataMigrationConnector(unittest.TestCase): - pass diff --git a/frappe/data_migration/doctype/data_migration_mapping/__init__.py b/frappe/data_migration/doctype/data_migration_mapping/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.js b/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.js deleted file mode 100644 index 6c99b9a54d..0000000000 --- a/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.js +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright (c) 2017, Frappe Technologies and contributors -// For license information, please see license.txt - -frappe.ui.form.on('Data Migration Mapping', { - refresh: function() { - - } -}); diff --git a/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.json b/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.json deleted file mode 100644 index 998abdf6ca..0000000000 --- a/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.json +++ /dev/null @@ -1,456 +0,0 @@ -{ - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 1, - "autoname": "field:mapping_name", - "beta": 1, - "creation": "2017-08-11 05:11:49.975801", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", - "fields": [ - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "mapping_name", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Mapping Name", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "remote_objectname", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Remote Objectname", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "remote_primary_key", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Remote Primary Key", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "local_doctype", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Local DocType", - "length": 0, - "no_copy": 0, - "options": "DocType", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "local_primary_key", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Local Primary Key", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "column_break_5", - "fieldtype": "Column Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "mapping_type", - "fieldtype": "Select", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Mapping Type", - "length": 0, - "no_copy": 0, - "options": "Push\nPull\nSync", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "default": "10", - "fieldname": "page_length", - "fieldtype": "Int", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Page Length", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "migration_id_field", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Migration ID Field", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "mapping", - "fieldtype": "Section Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Mapping", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "fields", - "fieldtype": "Table", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Field Maps", - "length": 0, - "no_copy": 0, - "options": "Data Migration Mapping Detail", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 1, - "columns": 0, - "fieldname": "condition_detail", - "fieldtype": "Section Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Condition Detail", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "condition", - "fieldtype": "Code", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Condition", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2017-09-27 18:06:43.275207", - "modified_by": "Administrator", - "module": "Data Migration", - "name": "Data Migration Mapping", - "name_case": "", - "owner": "Administrator", - "permissions": [ - { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "set_user_permissions": 0, - "share": 1, - "submit": 0, - "write": 1 - } - ], - "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1, - "track_seen": 0 -} \ No newline at end of file diff --git a/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.py b/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.py deleted file mode 100644 index 49af65e99b..0000000000 --- a/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.py +++ /dev/null @@ -1,83 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017, Frappe Technologies and contributors -# License: MIT. See LICENSE - -import frappe -from frappe.model.document import Document -from frappe.utils.safe_exec import get_safe_globals - - -class DataMigrationMapping(Document): - def get_filters(self): - if self.condition: - return frappe.safe_eval(self.condition, get_safe_globals()) - - def get_fields(self): - fields = [] - for f in self.fields: - if not (f.local_fieldname[0] in ('"', "'") or f.local_fieldname.startswith("eval:")): - fields.append(f.local_fieldname) - - if frappe.db.has_column(self.local_doctype, self.migration_id_field): - fields.append(self.migration_id_field) - - if "name" not in fields: - fields.append("name") - - return fields - - def get_mapped_record(self, doc): - """Build a mapped record using information from the fields table""" - mapped = frappe._dict() - - key_fieldname = "remote_fieldname" - value_fieldname = "local_fieldname" - - if self.mapping_type == "Pull": - key_fieldname, value_fieldname = value_fieldname, key_fieldname - - for field_map in self.fields: - key = get_source_value(field_map, key_fieldname) - - if not field_map.is_child_table: - # field to field mapping - value = get_value_from_fieldname(field_map, value_fieldname, doc) - else: - # child table mapping - mapping_name = field_map.child_table_mapping - value = get_mapped_child_records( - mapping_name, doc.get(get_source_value(field_map, value_fieldname)) - ) - - mapped[key] = value - - return mapped - - -def get_mapped_child_records(mapping_name, child_docs): - mapped_child_docs = [] - mapping = frappe.get_doc("Data Migration Mapping", mapping_name) - for child_doc in child_docs: - mapped_child_docs.append(mapping.get_mapped_record(child_doc)) - - return mapped_child_docs - - -def get_value_from_fieldname(field_map, fieldname_field, doc): - field_name = get_source_value(field_map, fieldname_field) - - if field_name.startswith("eval:"): - value = frappe.safe_eval(field_name[5:], get_safe_globals()) - elif field_name[0] in ('"', "'"): - value = field_name[1:-1] - else: - value = get_source_value(doc, field_name) - return value - - -def get_source_value(source, key): - """Get value from source (object or dict) based on key""" - if isinstance(source, dict): - return source.get(key) - else: - return getattr(source, key) diff --git a/frappe/data_migration/doctype/data_migration_mapping/test_data_migration_mapping.py b/frappe/data_migration/doctype/data_migration_mapping/test_data_migration_mapping.py deleted file mode 100644 index 30d2a6bcfe..0000000000 --- a/frappe/data_migration/doctype/data_migration_mapping/test_data_migration_mapping.py +++ /dev/null @@ -1,8 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017, Frappe Technologies and Contributors -# License: MIT. See LICENSE -import unittest - - -class TestDataMigrationMapping(unittest.TestCase): - pass diff --git a/frappe/data_migration/doctype/data_migration_mapping_detail/__init__.py b/frappe/data_migration/doctype/data_migration_mapping_detail/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/data_migration/doctype/data_migration_mapping_detail/data_migration_mapping_detail.json b/frappe/data_migration/doctype/data_migration_mapping_detail/data_migration_mapping_detail.json deleted file mode 100644 index ede9213f14..0000000000 --- a/frappe/data_migration/doctype/data_migration_mapping_detail/data_migration_mapping_detail.json +++ /dev/null @@ -1,163 +0,0 @@ -{ - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 0, - "creation": "2017-08-11 05:09:10.900237", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", - "fields": [ - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "remote_fieldname", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Remote Fieldname", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "local_fieldname", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Local Fieldname", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "is_child_table", - "fieldtype": "Check", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Is Child Table", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "depends_on": "is_child_table", - "fieldname": "child_table_mapping", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Child Table Mapping", - "length": 0, - "no_copy": 0, - "options": "Data Migration Mapping", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 1, - "max_attachments": 0, - "modified": "2017-09-28 17:13:31.337005", - "modified_by": "Administrator", - "module": "Data Migration", - "name": "Data Migration Mapping Detail", - "name_case": "", - "owner": "Administrator", - "permissions": [], - "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1, - "track_seen": 0 -} \ No newline at end of file diff --git a/frappe/data_migration/doctype/data_migration_mapping_detail/data_migration_mapping_detail.py b/frappe/data_migration/doctype/data_migration_mapping_detail/data_migration_mapping_detail.py deleted file mode 100644 index abd6348a26..0000000000 --- a/frappe/data_migration/doctype/data_migration_mapping_detail/data_migration_mapping_detail.py +++ /dev/null @@ -1,9 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017, Frappe Technologies and contributors -# License: MIT. See LICENSE - -from frappe.model.document import Document - - -class DataMigrationMappingDetail(Document): - pass diff --git a/frappe/data_migration/doctype/data_migration_plan/__init__.py b/frappe/data_migration/doctype/data_migration_plan/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.js b/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.js deleted file mode 100644 index 357ef2972f..0000000000 --- a/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.js +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright (c) 2017, Frappe Technologies and contributors -// For license information, please see license.txt - -frappe.ui.form.on('Data Migration Plan', { - onload(frm) { - frm.add_custom_button(__('Run'), () => frappe.new_doc('Data Migration Run', { - data_migration_plan: frm.doc.name - })); - } -}); diff --git a/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.json b/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.json deleted file mode 100644 index 2cfc2e3bd7..0000000000 --- a/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.json +++ /dev/null @@ -1,224 +0,0 @@ -{ - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "autoname": "field:plan_name", - "beta": 0, - "creation": "2017-08-11 05:15:51.482165", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", - "fields": [ - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "plan_name", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Plan Name", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 1 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "module", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Module", - "length": 0, - "no_copy": 0, - "options": "Module Def", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "mappings", - "fieldtype": "Table", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Mappings", - "length": 0, - "no_copy": 0, - "options": "Data Migration Plan Mapping", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "preprocess_method", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Preprocess Method", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "postprocess_method", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Postprocess Method", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2020-09-18 17:26:09.703215", - "modified_by": "Administrator", - "module": "Data Migration", - "name": "Data Migration Plan", - "name_case": "", - "owner": "Administrator", - "permissions": [ - { - "amend": 0, - "cancel": 0, - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "set_user_permissions": 0, - "share": 1, - "submit": 0, - "write": 1 - } - ], - "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1, - "track_seen": 0 -} \ No newline at end of file diff --git a/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.py b/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.py deleted file mode 100644 index 4118e8e7fe..0000000000 --- a/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.py +++ /dev/null @@ -1,78 +0,0 @@ -# Copyright (c) 2021, Frappe Technologies and contributors -# License: MIT. See LICENSE - -import frappe -from frappe.custom.doctype.custom_field.custom_field import create_custom_field -from frappe.model.document import Document -from frappe.modules import get_module_path, scrub_dt_dn -from frappe.modules.export_file import create_init_py, export_to_files - - -def get_mapping_module(module, mapping_name): - app_name = frappe.db.get_value("Module Def", module, "app_name") - mapping_name = frappe.scrub(mapping_name) - module = frappe.scrub(module) - - try: - return frappe.get_module(f"{app_name}.{module}.data_migration_mapping.{mapping_name}") - except ImportError: - return None - - -class DataMigrationPlan(Document): - def on_update(self): - # update custom fields in mappings - self.make_custom_fields_for_mappings() - - if frappe.flags.in_import or frappe.flags.in_test: - return - - if frappe.local.conf.get("developer_mode"): - record_list = [["Data Migration Plan", self.name]] - - for m in self.mappings: - record_list.append(["Data Migration Mapping", m.mapping]) - - export_to_files(record_list=record_list, record_module=self.module) - - for m in self.mappings: - dt, dn = scrub_dt_dn("Data Migration Mapping", m.mapping) - create_init_py(get_module_path(self.module), dt, dn) - - def make_custom_fields_for_mappings(self): - frappe.flags.ignore_in_install = True - label = self.name + " ID" - fieldname = frappe.scrub(label) - - df = { - "label": label, - "fieldname": fieldname, - "fieldtype": "Data", - "hidden": 1, - "read_only": 1, - "unique": 1, - "no_copy": 1, - } - - for m in self.mappings: - mapping = frappe.get_doc("Data Migration Mapping", m.mapping) - create_custom_field(mapping.local_doctype, df) - mapping.migration_id_field = fieldname - mapping.save() - - # Create custom field in Deleted Document - create_custom_field("Deleted Document", df) - frappe.flags.ignore_in_install = False - - def pre_process_doc(self, mapping_name, doc): - module = get_mapping_module(self.module, mapping_name) - - if module and hasattr(module, "pre_process"): - return module.pre_process(doc) - return doc - - def post_process_doc(self, mapping_name, local_doc=None, remote_doc=None): - module = get_mapping_module(self.module, mapping_name) - - if module and hasattr(module, "post_process"): - return module.post_process(local_doc=local_doc, remote_doc=remote_doc) diff --git a/frappe/data_migration/doctype/data_migration_plan/test_data_migration_plan.py b/frappe/data_migration/doctype/data_migration_plan/test_data_migration_plan.py deleted file mode 100644 index ef3bfa3a70..0000000000 --- a/frappe/data_migration/doctype/data_migration_plan/test_data_migration_plan.py +++ /dev/null @@ -1,8 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017, Frappe Technologies and Contributors -# License: MIT. See LICENSE -import unittest - - -class TestDataMigrationPlan(unittest.TestCase): - pass diff --git a/frappe/data_migration/doctype/data_migration_plan_mapping/__init__.py b/frappe/data_migration/doctype/data_migration_plan_mapping/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/data_migration/doctype/data_migration_plan_mapping/data_migration_plan_mapping.json b/frappe/data_migration/doctype/data_migration_plan_mapping/data_migration_plan_mapping.json deleted file mode 100644 index 5acf014715..0000000000 --- a/frappe/data_migration/doctype/data_migration_plan_mapping/data_migration_plan_mapping.json +++ /dev/null @@ -1,103 +0,0 @@ -{ - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 1, - "creation": "2017-08-11 05:15:38.390831", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", - "fields": [ - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "mapping", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Mapping", - "length": 0, - "no_copy": 0, - "options": "Data Migration Mapping", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "default": "1", - "fieldname": "enabled", - "fieldtype": "Check", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Enabled", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 1, - "max_attachments": 0, - "modified": "2017-09-20 21:43:04.908650", - "modified_by": "Administrator", - "module": "Data Migration", - "name": "Data Migration Plan Mapping", - "name_case": "", - "owner": "Administrator", - "permissions": [], - "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1, - "track_seen": 0 -} \ No newline at end of file diff --git a/frappe/data_migration/doctype/data_migration_plan_mapping/data_migration_plan_mapping.py b/frappe/data_migration/doctype/data_migration_plan_mapping/data_migration_plan_mapping.py deleted file mode 100644 index 0650f4b2c7..0000000000 --- a/frappe/data_migration/doctype/data_migration_plan_mapping/data_migration_plan_mapping.py +++ /dev/null @@ -1,9 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017, Frappe Technologies and contributors -# License: MIT. See LICENSE - -from frappe.model.document import Document - - -class DataMigrationPlanMapping(Document): - pass diff --git a/frappe/data_migration/doctype/data_migration_run/__init__.py b/frappe/data_migration/doctype/data_migration_run/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/data_migration/doctype/data_migration_run/data_migration_run.js b/frappe/data_migration/doctype/data_migration_run/data_migration_run.js deleted file mode 100644 index 82323c62f1..0000000000 --- a/frappe/data_migration/doctype/data_migration_run/data_migration_run.js +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright (c) 2017, Frappe Technologies and contributors -// For license information, please see license.txt - -frappe.ui.form.on('Data Migration Run', { - refresh: function(frm) { - if (frm.doc.status !== 'Success') { - frm.add_custom_button(__('Run'), () => frm.call('run')); - } - if (frm.doc.status === 'Started') { - frm.dashboard.add_progress(__('Percent Complete'), frm.doc.percent_complete, - __('Currently updating {0}', [frm.doc.current_mapping])); - } - } -}); diff --git a/frappe/data_migration/doctype/data_migration_run/data_migration_run.json b/frappe/data_migration/doctype/data_migration_run/data_migration_run.json deleted file mode 100644 index db77997928..0000000000 --- a/frappe/data_migration/doctype/data_migration_run/data_migration_run.json +++ /dev/null @@ -1,838 +0,0 @@ -{ - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 0, - "creation": "2017-09-11 12:55:27.597728", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", - "fields": [ - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "data_migration_plan", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Data Migration Plan", - "length": 0, - "no_copy": 0, - "options": "Data Migration Plan", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "data_migration_connector", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Data Migration Connector", - "length": 0, - "no_copy": 0, - "options": "Data Migration Connector", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "default": "Pending", - "fieldname": "status", - "fieldtype": "Select", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Status", - "length": 0, - "no_copy": 1, - "options": "Pending\nStarted\nPartial Success\nSuccess\nFail\nError", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "start_time", - "fieldtype": "Datetime", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Start Time", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "end_time", - "fieldtype": "Datetime", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "End Time", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "remote_id", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Remote ID", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "current_mapping", - "fieldtype": "Data", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Current Mapping", - "length": 0, - "no_copy": 1, - "options": "", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "current_mapping_start", - "fieldtype": "Int", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Current Mapping Start", - "length": 0, - "no_copy": 1, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "current_mapping_delete_start", - "fieldtype": "Int", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Current Mapping Delete Start", - "length": 0, - "no_copy": 1, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "current_mapping_type", - "fieldtype": "Select", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Current Mapping Type", - "length": 0, - "no_copy": 0, - "options": "Push\nPull", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "depends_on": "eval:(doc.status !== 'Pending')", - "fieldname": "current_mapping_action", - "fieldtype": "Select", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Current Mapping Action", - "length": 0, - "no_copy": 1, - "options": "Insert\nDelete", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "total_pages", - "fieldtype": "Int", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Total Pages", - "length": 0, - "no_copy": 1, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "percent_complete", - "fieldtype": "Percent", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Percent Complete", - "length": 0, - "no_copy": 1, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "trigger_name", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Trigger Name", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "depends_on": "eval:(doc.status !== 'Pending')", - "fieldname": "logs_sb", - "fieldtype": "Section Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Logs", - "length": 0, - "no_copy": 1, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "push_insert", - "fieldtype": "Int", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Push Insert", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "push_update", - "fieldtype": "Int", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Push Update", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "push_delete", - "fieldtype": "Int", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Push Delete", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "push_failed", - "fieldtype": "Code", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Push Failed", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "column_break_16", - "fieldtype": "Column Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "pull_insert", - "fieldtype": "Int", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Pull Insert", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "pull_update", - "fieldtype": "Int", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Pull Update", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "pull_failed", - "fieldtype": "Code", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Pull Failed", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "depends_on": "eval:doc.failed_log !== '[]'", - "fieldname": "log", - "fieldtype": "Code", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Log", - "length": 0, - "no_copy": 1, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2020-09-18 17:26:09.703215", - "modified_by": "Administrator", - "module": "Data Migration", - "name": "Data Migration Run", - "name_case": "", - "owner": "Administrator", - "permissions": [ - { - "amend": 0, - "cancel": 0, - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "set_user_permissions": 0, - "share": 1, - "submit": 0, - "write": 1 - } - ], - "quick_entry": 0, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1, - "track_seen": 0 -} \ No newline at end of file diff --git a/frappe/data_migration/doctype/data_migration_run/data_migration_run.py b/frappe/data_migration/doctype/data_migration_run/data_migration_run.py deleted file mode 100644 index c734cb105b..0000000000 --- a/frappe/data_migration/doctype/data_migration_run/data_migration_run.py +++ /dev/null @@ -1,514 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017, Frappe Technologies and contributors -# License: MIT. See LICENSE - -import json -import math - -import frappe -from frappe import _ -from frappe.data_migration.doctype.data_migration_mapping.data_migration_mapping import ( - get_source_value, -) -from frappe.model.document import Document -from frappe.utils import cstr - - -class DataMigrationRun(Document): - @frappe.whitelist() - def run(self): - self.begin() - if self.total_pages > 0: - self.enqueue_next_mapping() - else: - self.complete() - - def enqueue_next_mapping(self): - next_mapping_name = self.get_next_mapping_name() - if next_mapping_name: - next_mapping = self.get_mapping(next_mapping_name) - self.db_set( - dict( - current_mapping=next_mapping.name, - current_mapping_start=0, - current_mapping_delete_start=0, - current_mapping_action="Insert", - ), - notify=True, - commit=True, - ) - frappe.enqueue_doc(self.doctype, self.name, "run_current_mapping", now=frappe.flags.in_test) - else: - self.complete() - - def enqueue_next_page(self): - mapping = self.get_mapping(self.current_mapping) - percent_complete = self.percent_complete + (100.0 / self.total_pages) - fields = dict(percent_complete=percent_complete) - if self.current_mapping_action == "Insert": - start = self.current_mapping_start + mapping.page_length - fields["current_mapping_start"] = start - elif self.current_mapping_action == "Delete": - delete_start = self.current_mapping_delete_start + mapping.page_length - fields["current_mapping_delete_start"] = delete_start - - self.db_set(fields, notify=True, commit=True) - - if percent_complete < 100: - frappe.publish_realtime( - self.trigger_name, {"progress_percent": percent_complete}, user=frappe.session.user - ) - - frappe.enqueue_doc(self.doctype, self.name, "run_current_mapping", now=frappe.flags.in_test) - - def run_current_mapping(self): - try: - mapping = self.get_mapping(self.current_mapping) - - if mapping.mapping_type == "Push": - done = self.push() - elif mapping.mapping_type == "Pull": - done = self.pull() - - if done: - self.enqueue_next_mapping() - else: - self.enqueue_next_page() - - except Exception as e: - self.db_set("status", "Error", notify=True, commit=True) - print("Data Migration Run failed") - print(frappe.get_traceback()) - self.execute_postprocess("Error") - raise e - - def get_last_modified_condition(self): - last_run_timestamp = frappe.db.get_value( - "Data Migration Run", - dict( - data_migration_plan=self.data_migration_plan, - data_migration_connector=self.data_migration_connector, - name=("!=", self.name), - ), - "modified", - ) - if last_run_timestamp: - condition = dict(modified=(">", last_run_timestamp)) - else: - condition = {} - return condition - - def begin(self): - plan_active_mappings = [m for m in self.get_plan().mappings if m.enabled] - self.mappings = [ - frappe.get_doc("Data Migration Mapping", m.mapping) for m in plan_active_mappings - ] - - total_pages = 0 - for m in [mapping for mapping in self.mappings]: - if m.mapping_type == "Push": - count = float(self.get_count(m)) - page_count = math.ceil(count / m.page_length) - total_pages += page_count - if m.mapping_type == "Pull": - total_pages += 10 - - self.db_set( - dict( - status="Started", - current_mapping=None, - current_mapping_start=0, - current_mapping_delete_start=0, - percent_complete=0, - current_mapping_action="Insert", - total_pages=total_pages, - ), - notify=True, - commit=True, - ) - - def complete(self): - fields = dict() - - push_failed = self.get_log("push_failed", []) - pull_failed = self.get_log("pull_failed", []) - - status = "Partial Success" - - if not push_failed and not pull_failed: - status = "Success" - fields["percent_complete"] = 100 - - fields["status"] = status - - self.db_set(fields, notify=True, commit=True) - - self.execute_postprocess(status) - - frappe.publish_realtime(self.trigger_name, {"progress_percent": 100}, user=frappe.session.user) - - def execute_postprocess(self, status): - # Execute post process - postprocess_method_path = self.get_plan().postprocess_method - - if postprocess_method_path: - frappe.get_attr(postprocess_method_path)( - { - "status": status, - "stats": { - "push_insert": self.push_insert, - "push_update": self.push_update, - "push_delete": self.push_delete, - "pull_insert": self.pull_insert, - "pull_update": self.pull_update, - }, - } - ) - - def get_plan(self): - if not hasattr(self, "plan"): - self.plan = frappe.get_doc("Data Migration Plan", self.data_migration_plan) - return self.plan - - def get_mapping(self, mapping_name): - if hasattr(self, "mappings"): - for m in self.mappings: - if m.name == mapping_name: - return m - return frappe.get_doc("Data Migration Mapping", mapping_name) - - def get_next_mapping_name(self): - mappings = [m for m in self.get_plan().mappings if m.enabled] - if not self.current_mapping: - # first - return mappings[0].mapping - for i, d in enumerate(mappings): - if i == len(mappings) - 1: - # last - return None - if d.mapping == self.current_mapping: - return mappings[i + 1].mapping - - raise frappe.ValidationError("Mapping Broken") - - def get_data(self, filters): - mapping = self.get_mapping(self.current_mapping) - or_filters = self.get_or_filters(mapping) - start = self.current_mapping_start - - data = [] - doclist = frappe.get_all( - mapping.local_doctype, - filters=filters, - or_filters=or_filters, - start=start, - page_length=mapping.page_length, - ) - - for d in doclist: - doc = frappe.get_doc(mapping.local_doctype, d["name"]) - data.append(doc) - return data - - def get_new_local_data(self): - """Fetch newly inserted local data using `frappe.get_all`. Used during Push""" - mapping = self.get_mapping(self.current_mapping) - filters = mapping.get_filters() or {} - - # new docs dont have migration field set - filters.update({mapping.migration_id_field: ""}) - - return self.get_data(filters) - - def get_updated_local_data(self): - """Fetch local updated data using `frappe.get_all`. Used during Push""" - mapping = self.get_mapping(self.current_mapping) - filters = mapping.get_filters() or {} - - # existing docs must have migration field set - filters.update({mapping.migration_id_field: ("!=", "")}) - - return self.get_data(filters) - - def get_deleted_local_data(self): - """Fetch local deleted data using `frappe.get_all`. Used during Push""" - mapping = self.get_mapping(self.current_mapping) - filters = self.get_last_modified_condition() - filters.update({"deleted_doctype": mapping.local_doctype}) - - data = frappe.get_all("Deleted Document", fields=["name", "data"], filters=filters) - - _data = [] - for d in data: - doc = json.loads(d.data) - if doc.get(mapping.migration_id_field): - doc["_deleted_document_name"] = d["name"] - _data.append(doc) - - return _data - - def get_remote_data(self): - """Fetch data from remote using `connection.get`. Used during Pull""" - mapping = self.get_mapping(self.current_mapping) - start = self.current_mapping_start - filters = mapping.get_filters() or {} - connection = self.get_connection() - - return connection.get( - mapping.remote_objectname, - fields=["*"], - filters=filters, - start=start, - page_length=mapping.page_length, - ) - - def get_count(self, mapping): - filters = mapping.get_filters() or {} - or_filters = self.get_or_filters(mapping) - - to_insert = frappe.get_all( - mapping.local_doctype, ["count(name) as total"], filters=filters, or_filters=or_filters - )[0].total - - to_delete = frappe.get_all( - "Deleted Document", - ["count(name) as total"], - filters={"deleted_doctype": mapping.local_doctype}, - or_filters=or_filters, - )[0].total - - return to_insert + to_delete - - def get_or_filters(self, mapping): - or_filters = self.get_last_modified_condition() - - # docs whose migration_id_field is not set - # failed in the previous run, include those too - or_filters.update({mapping.migration_id_field: ("=", "")}) - - return or_filters - - def get_connection(self): - if not hasattr(self, "connection"): - self.connection = frappe.get_doc( - "Data Migration Connector", self.data_migration_connector - ).get_connection() - - return self.connection - - def push(self): - self.db_set("current_mapping_type", "Push") - done = True - - if self.current_mapping_action == "Insert": - done = self._push_insert() - - elif self.current_mapping_action == "Update": - done = self._push_update() - - elif self.current_mapping_action == "Delete": - done = self._push_delete() - - return done - - def _push_insert(self): - """Inserts new local docs on remote""" - mapping = self.get_mapping(self.current_mapping) - connection = self.get_connection() - data = self.get_new_local_data() - - for d in data: - # pre process before insert - doc = self.pre_process_doc(d) - doc = mapping.get_mapped_record(doc) - - try: - response_doc = connection.insert(mapping.remote_objectname, doc) - frappe.db.set_value( - mapping.local_doctype, - d.name, - mapping.migration_id_field, - response_doc[connection.name_field], - update_modified=False, - ) - frappe.db.commit() - self.update_log("push_insert", 1) - # post process after insert - self.post_process_doc(local_doc=d, remote_doc=response_doc) - except Exception as e: - self.update_log("push_failed", {d.name: cstr(e)}) - - # update page_start - self.db_set("current_mapping_start", self.current_mapping_start + mapping.page_length) - - if len(data) < mapping.page_length: - # done, no more new data to insert - self.db_set({"current_mapping_action": "Update", "current_mapping_start": 0}) - # not done with this mapping - return False - - def _push_update(self): - """Updates local modified docs on remote""" - mapping = self.get_mapping(self.current_mapping) - connection = self.get_connection() - data = self.get_updated_local_data() - - for d in data: - migration_id_value = d.get(mapping.migration_id_field) - # pre process before update - doc = self.pre_process_doc(d) - doc = mapping.get_mapped_record(doc) - try: - response_doc = connection.update(mapping.remote_objectname, doc, migration_id_value) - self.update_log("push_update", 1) - # post process after update - self.post_process_doc(local_doc=d, remote_doc=response_doc) - except Exception as e: - self.update_log("push_failed", {d.name: cstr(e)}) - - # update page_start - self.db_set("current_mapping_start", self.current_mapping_start + mapping.page_length) - - if len(data) < mapping.page_length: - # done, no more data to update - self.db_set({"current_mapping_action": "Delete", "current_mapping_start": 0}) - # not done with this mapping - return False - - def _push_delete(self): - """Deletes docs deleted from local on remote""" - mapping = self.get_mapping(self.current_mapping) - connection = self.get_connection() - data = self.get_deleted_local_data() - - for d in data: - # Deleted Document also has a custom field for migration_id - migration_id_value = d.get(mapping.migration_id_field) - # pre process before update - self.pre_process_doc(d) - try: - response_doc = connection.delete(mapping.remote_objectname, migration_id_value) - self.update_log("push_delete", 1) - # post process only when action is success - self.post_process_doc(local_doc=d, remote_doc=response_doc) - except Exception as e: - self.update_log("push_failed", {d.name: cstr(e)}) - - # update page_start - self.db_set("current_mapping_start", self.current_mapping_start + mapping.page_length) - - if len(data) < mapping.page_length: - # done, no more new data to delete - # done with this mapping - return True - - def pull(self): - self.db_set("current_mapping_type", "Pull") - - connection = self.get_connection() - mapping = self.get_mapping(self.current_mapping) - data = self.get_remote_data() - - for d in data: - migration_id_value = get_source_value(d, connection.name_field) - doc = self.pre_process_doc(d) - doc = mapping.get_mapped_record(doc) - - if migration_id_value: - try: - if not local_doc_exists(mapping, migration_id_value): - # insert new local doc - local_doc = insert_local_doc(mapping, doc) - - self.update_log("pull_insert", 1) - # set migration id - frappe.db.set_value( - mapping.local_doctype, - local_doc.name, - mapping.migration_id_field, - migration_id_value, - update_modified=False, - ) - frappe.db.commit() - else: - # update doc - local_doc = update_local_doc(mapping, doc, migration_id_value) - self.update_log("pull_update", 1) - # post process doc after success - self.post_process_doc(remote_doc=d, local_doc=local_doc) - except Exception as e: - # failed, append to log - self.update_log("pull_failed", {migration_id_value: cstr(e)}) - - if len(data) < mapping.page_length: - # last page, done with pull - return True - - def pre_process_doc(self, doc): - plan = self.get_plan() - doc = plan.pre_process_doc(self.current_mapping, doc) - return doc - - def post_process_doc(self, local_doc=None, remote_doc=None): - plan = self.get_plan() - doc = plan.post_process_doc(self.current_mapping, local_doc=local_doc, remote_doc=remote_doc) - return doc - - def set_log(self, key, value): - value = json.dumps(value) if "_failed" in key else value - self.db_set(key, value) - - def update_log(self, key, value=None): - """ - Helper for updating logs, - push_failed and pull_failed are stored as json, - other keys are stored as int - """ - if "_failed" in key: - # json - self.set_log(key, self.get_log(key, []) + [value]) - else: - # int - self.set_log(key, self.get_log(key, 0) + (value or 1)) - - def get_log(self, key, default=None): - value = self.db_get(key) - if "_failed" in key: - if not value: - value = json.dumps(default) - value = json.loads(value) - return value or default - - -def insert_local_doc(mapping, doc): - try: - # insert new doc - if not doc.doctype: - doc.doctype = mapping.local_doctype - doc = frappe.get_doc(doc).insert() - return doc - except Exception: - print("Data Migration Run failed: Error in Pull insert") - print(frappe.get_traceback()) - return None - - -def update_local_doc(mapping, remote_doc, migration_id_value): - try: - # migration id value is set in migration_id_field in mapping.local_doctype - docname = frappe.db.get_value( - mapping.local_doctype, filters={mapping.migration_id_field: migration_id_value} - ) - - doc = frappe.get_doc(mapping.local_doctype, docname) - doc.update(remote_doc) - doc.save() - return doc - except Exception: - print("Data Migration Run failed: Error in Pull update") - print(frappe.get_traceback()) - return None - - -def local_doc_exists(mapping, migration_id_value): - return frappe.db.exists(mapping.local_doctype, {mapping.migration_id_field: migration_id_value}) diff --git a/frappe/data_migration/doctype/data_migration_run/test_data_migration_run.py b/frappe/data_migration/doctype/data_migration_run/test_data_migration_run.py deleted file mode 100644 index 0357b1e0f5..0000000000 --- a/frappe/data_migration/doctype/data_migration_run/test_data_migration_run.py +++ /dev/null @@ -1,128 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017, Frappe Technologies and Contributors -# License: MIT. See LICENSE -import unittest - -import frappe - - -class TestDataMigrationRun(unittest.TestCase): - def test_run(self): - create_plan() - - description = "data migration todo" - new_todo = frappe.get_doc({"doctype": "ToDo", "description": description}).insert() - - event_subject = "data migration event" - frappe.get_doc( - dict( - doctype="Event", - subject=event_subject, - repeat_on="Monthly", - starts_on=frappe.utils.now_datetime(), - ) - ).insert() - - run = frappe.get_doc( - { - "doctype": "Data Migration Run", - "data_migration_plan": "ToDo Sync", - "data_migration_connector": "Local Connector", - } - ).insert() - - run.run() - self.assertEqual(run.db_get("status"), "Success") - - self.assertEqual(run.db_get("push_insert"), 1) - self.assertEqual(run.db_get("pull_insert"), 1) - - todo = frappe.get_doc("ToDo", new_todo.name) - self.assertTrue(todo.todo_sync_id) - - # Pushed Event - event = frappe.get_doc("Event", todo.todo_sync_id) - self.assertEqual(event.subject, description) - - # Pulled ToDo - created_todo = frappe.get_doc("ToDo", {"description": event_subject}) - self.assertEqual(created_todo.description, event_subject) - - todo_list = frappe.get_list( - "ToDo", filters={"description": "data migration todo"}, fields=["name"] - ) - todo_name = todo_list[0].name - - todo = frappe.get_doc("ToDo", todo_name) - todo.description = "data migration todo updated" - todo.save() - - run = frappe.get_doc( - { - "doctype": "Data Migration Run", - "data_migration_plan": "ToDo Sync", - "data_migration_connector": "Local Connector", - } - ).insert() - - run.run() - - # Update - self.assertEqual(run.db_get("status"), "Success") - self.assertEqual(run.db_get("pull_update"), 1) - - -def create_plan(): - frappe.get_doc( - { - "doctype": "Data Migration Mapping", - "mapping_name": "Todo to Event", - "remote_objectname": "Event", - "remote_primary_key": "name", - "mapping_type": "Push", - "local_doctype": "ToDo", - "fields": [ - {"remote_fieldname": "subject", "local_fieldname": "description"}, - { - "remote_fieldname": "starts_on", - "local_fieldname": "eval:frappe.utils.get_datetime_str(frappe.utils.get_datetime())", - }, - ], - "condition": '{"description": "data migration todo" }', - } - ).insert(ignore_if_duplicate=True) - - frappe.get_doc( - { - "doctype": "Data Migration Mapping", - "mapping_name": "Event to ToDo", - "remote_objectname": "Event", - "remote_primary_key": "name", - "local_doctype": "ToDo", - "local_primary_key": "name", - "mapping_type": "Pull", - "condition": '{"subject": "data migration event" }', - "fields": [{"remote_fieldname": "subject", "local_fieldname": "description"}], - } - ).insert(ignore_if_duplicate=True) - - frappe.get_doc( - { - "doctype": "Data Migration Plan", - "plan_name": "ToDo Sync", - "module": "Core", - "mappings": [{"mapping": "Todo to Event"}, {"mapping": "Event to ToDo"}], - } - ).insert(ignore_if_duplicate=True) - - frappe.get_doc( - { - "doctype": "Data Migration Connector", - "connector_name": "Local Connector", - "connector_type": "Frappe", - # connect to same host. - "hostname": frappe.conf.host_name or frappe.utils.get_site_url(frappe.local.site), - "username": "Administrator", - "password": frappe.conf.get("admin_password") or "admin", - } - ).insert(ignore_if_duplicate=True) diff --git a/frappe/model/sync.py b/frappe/model/sync.py index 4c535b2811..93b883cda6 100644 --- a/frappe/model/sync.py +++ b/frappe/model/sync.py @@ -53,22 +53,6 @@ def sync_for(app_name, force=0, reset_permissions=False): os.path.join(FRAPPE_PATH, "website", "doctype", website_module, f"{website_module}.json") ) - for data_migration_module in [ - "data_migration_mapping_detail", - "data_migration_mapping", - "data_migration_plan_mapping", - "data_migration_plan", - ]: - files.append( - os.path.join( - FRAPPE_PATH, - "data_migration", - "doctype", - data_migration_module, - f"{data_migration_module}.json", - ) - ) - for desk_module in [ "number_card", "dashboard_chart", @@ -124,8 +108,6 @@ def get_doc_files(files, start_path): "web_template", "notification", "print_style", - "data_migration_mapping", - "data_migration_plan", "workspace", "onboarding_step", "module_onboarding", diff --git a/frappe/modules.txt b/frappe/modules.txt index a707ca853e..fb7817f6ba 100644 --- a/frappe/modules.txt +++ b/frappe/modules.txt @@ -8,7 +8,6 @@ Desk Integrations Printing Contacts -Data Migration Social Automation Event Streaming \ No newline at end of file diff --git a/frappe/patches.txt b/frappe/patches.txt index 6c46d5dcd9..b771485c3c 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -201,3 +201,4 @@ frappe.patches.v14_0.update_color_names_in_kanban_board_column frappe.patches.v14_0.update_is_system_generated_flag frappe.patches.v14_0.update_auto_account_deletion_duration frappe.patches.v14_0.set_document_expiry_default +frappe.patches.v14_0.delete_data_migration_tool \ No newline at end of file diff --git a/frappe/patches/v14_0/delete_data_migration_tool.py b/frappe/patches/v14_0/delete_data_migration_tool.py new file mode 100644 index 0000000000..d0416cb1e7 --- /dev/null +++ b/frappe/patches/v14_0/delete_data_migration_tool.py @@ -0,0 +1,12 @@ +# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors +# MIT License. See license.txt + +import frappe + + +def execute(): + doctypes = frappe.db.get_all("DocType", {"module": "Data Migration", "custom": 0}, pluck="name") + for doctype in doctypes: + frappe.delete_doc("DocType", doctype, ignore_missing=True) + + frappe.delete_doc("Module Def", "Data Migration", ignore_missing=True, force=True) From 792c1451e7eac6653c06d67f377a00cfd52ca686 Mon Sep 17 00:00:00 2001 From: Deepesh Garg Date: Mon, 13 Jun 2022 16:39:15 +0530 Subject: [PATCH 039/205] fix: Force system admin role only if active --- frappe/core/doctype/user/user.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py index 9c127d9eca..531fd316b1 100644 --- a/frappe/core/doctype/user/user.py +++ b/frappe/core/doctype/user/user.py @@ -163,6 +163,9 @@ class User(Document): toggle_notifications(self.name, enable=cint(self.enabled)) def add_system_manager_role(self): + if self.is_system_manager_disabled(): + return + # if adding system manager, do nothing if not cint(self.enabled) or ( "System Manager" in [user_role.role for user_role in self.get("roles")] @@ -189,6 +192,9 @@ class User(Document): ], ) + def is_system_manager_disabled(self): + return frappe.db.get_value("Role", {"name": "System Manager"}, ["disabled"]) + def email_new_password(self, new_password=None): if new_password and not self.flags.in_insert: _update_password(user=self.name, pwd=new_password, logout_all_sessions=self.logout_all_sessions) @@ -372,6 +378,9 @@ class User(Document): ) def a_system_manager_should_exist(self): + if self.is_system_manager_disabled(): + return + if not self.get_other_system_managers(): throw(_("There should remain at least one System Manager")) From 8c8249894beb0b6707954485e2b3947192e8025b Mon Sep 17 00:00:00 2001 From: Abhirup Pal Date: Mon, 13 Jun 2022 16:41:43 +0530 Subject: [PATCH 040/205] fix: Update label for editing in full page and add custom logic for rendering it. (#17149) --- cypress/integration/timeline_email.js | 2 +- frappe/public/js/frappe/form/quick_entry.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cypress/integration/timeline_email.js b/cypress/integration/timeline_email.js index f2a239401d..993847bcb8 100644 --- a/cypress/integration/timeline_email.js +++ b/cypress/integration/timeline_email.js @@ -7,7 +7,7 @@ context('Timeline Email', () => { it('Adding new ToDo', () => { cy.click_listview_primary_button('Add ToDo'); - cy.get('.custom-actions:visible > .btn').contains("Edit in full page").click({delay: 500}); + cy.get('.custom-actions:visible > .btn').contains("Edit Full Form").click({delay: 500}); cy.fill_field("description", "Test ToDo", "Text Editor"); cy.wait(500); cy.get('.primary-action').contains('Save').click({force: true}); diff --git a/frappe/public/js/frappe/form/quick_entry.js b/frappe/public/js/frappe/form/quick_entry.js index 86523d7088..f55f7139c9 100644 --- a/frappe/public/js/frappe/form/quick_entry.js +++ b/frappe/public/js/frappe/form/quick_entry.js @@ -267,7 +267,7 @@ frappe.ui.form.QuickEntryForm = class QuickEntryForm { render_edit_in_full_page_link() { var me = this; this.dialog.add_custom_action( - `${__("Edit in full page")}`, + `${__("Edit Full Form")}`, () => me.open_doc(true) ); } From cfc4d966c7113683b1e96cd08df801194312bc05 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 13 Jun 2022 15:03:14 +0530 Subject: [PATCH 041/205] fix(ux): patch title field and make patch read only --- frappe/core/doctype/patch_log/patch_log.json | 115 ++++++------------- 1 file changed, 36 insertions(+), 79 deletions(-) diff --git a/frappe/core/doctype/patch_log/patch_log.json b/frappe/core/doctype/patch_log/patch_log.json index aa054f1360..9750c51279 100644 --- a/frappe/core/doctype/patch_log/patch_log.json +++ b/frappe/core/doctype/patch_log/patch_log.json @@ -1,87 +1,44 @@ { - "allow_copy": 0, - "allow_import": 0, - "allow_rename": 0, - "autoname": "PATCHLOG.#####", - "beta": 0, - "creation": "2013-01-17 11:36:45", - "custom": 0, - "description": "List of patches executed", - "docstatus": 0, - "doctype": "DocType", - "document_type": "System", - "editable_grid": 0, + "actions": [], + "autoname": "PATCHLOG.#####", + "creation": "2013-01-17 11:36:45", + "description": "List of patches executed", + "doctype": "DocType", + "document_type": "System", + "engine": "InnoDB", + "field_order": [ + "patch" + ], "fields": [ { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "patch", - "fieldtype": "Code", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Patch", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "fieldname": "patch", + "fieldtype": "Code", + "label": "Patch", + "read_only": 1 } - ], - "hide_heading": 0, - "hide_toolbar": 0, - "icon": "fa fa-cog", - "idx": 1, - "image_view": 0, - "in_create": 0, - - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2016-12-29 14:40:35.048570", - "modified_by": "Administrator", - "module": "Core", - "name": "Patch Log", - "owner": "Administrator", + ], + "icon": "fa fa-cog", + "idx": 1, + "links": [], + "modified": "2022-06-13 05:34:37.845368", + "modified_by": "Administrator", + "module": "Core", + "name": "Patch Log", + "naming_rule": "Expression (old style)", + "owner": "Administrator", "permissions": [ { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 0, - "delete": 0, - "email": 1, - "export": 0, - "if_owner": 0, - "import": 0, - "is_custom": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 1, - "role": "Administrator", - "set_user_permissions": 0, - "share": 0, - "submit": 0, - "write": 0 + "email": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "Administrator" } - ], - "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "track_changes": 1, - "track_seen": 0 + ], + "quick_entry": 1, + "sort_field": "modified", + "sort_order": "DESC", + "states": [], + "title_field": "patch", + "track_changes": 1 } \ No newline at end of file From 602e8d0a7863608d0c5d11267c0bec72cdc76230 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 13 Jun 2022 15:11:35 +0530 Subject: [PATCH 042/205] fix(ux): show title field on scheduled job log --- .../core/doctype/scheduled_job_log/scheduled_job_log.json | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/frappe/core/doctype/scheduled_job_log/scheduled_job_log.json b/frappe/core/doctype/scheduled_job_log/scheduled_job_log.json index 396b32bdf9..451c4108a0 100644 --- a/frappe/core/doctype/scheduled_job_log/scheduled_job_log.json +++ b/frappe/core/doctype/scheduled_job_log/scheduled_job_log.json @@ -38,7 +38,7 @@ } ], "links": [], - "modified": "2021-10-25 00:00:00.000000", + "modified": "2022-06-13 05:41:21.090972", "modified_by": "Administrator", "module": "Core", "name": "Scheduled Job Log", @@ -59,5 +59,7 @@ ], "quick_entry": 1, "sort_field": "modified", - "sort_order": "DESC" -} + "sort_order": "DESC", + "states": [], + "title_field": "scheduled_job_type" +} \ No newline at end of file From 3730e20618a26887d5c1a4f7819c6b27c0d0bcfa Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 13 Jun 2022 15:15:30 +0530 Subject: [PATCH 043/205] fix(ux): better layout on error log - plus show button to see relevant error logs --- frappe/core/doctype/error_log/error_log.js | 15 +++++++++++---- frappe/core/doctype/error_log/error_log.json | 18 ++++++++++++++---- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/frappe/core/doctype/error_log/error_log.js b/frappe/core/doctype/error_log/error_log.js index 4fe8fde5d6..277969edd9 100644 --- a/frappe/core/doctype/error_log/error_log.js +++ b/frappe/core/doctype/error_log/error_log.js @@ -1,8 +1,15 @@ -// Copyright (c) 2016, Frappe Technologies and contributors +// Copyright (c) 2022, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Error Log', { +frappe.ui.form.on("Error Log", { refresh: function(frm) { - - } + if (frm.doc.reference_doctype && frm.doc.reference_name) { + frm.add_custom_button(__("Show Errors for This Document"), function() { + frappe.set_route("List", "Error Log", { + reference_doctype: frm.doc.reference_doctype, + reference_name: frm.doc.reference_name, + }); + }); + } + }, }); diff --git a/frappe/core/doctype/error_log/error_log.json b/frappe/core/doctype/error_log/error_log.json index e0ce109595..a93161c891 100644 --- a/frappe/core/doctype/error_log/error_log.json +++ b/frappe/core/doctype/error_log/error_log.json @@ -6,10 +6,12 @@ "engine": "MyISAM", "field_order": [ "seen", - "method", - "error", "reference_doctype", - "reference_name" + "column_break_3", + "reference_name", + "section_break_5", + "method", + "error" ], "fields": [ { @@ -47,12 +49,20 @@ "fieldtype": "Data", "label": "Reference Name", "read_only": 1 + }, + { + "fieldname": "column_break_3", + "fieldtype": "Column Break" + }, + { + "fieldname": "section_break_5", + "fieldtype": "Section Break" } ], "icon": "fa fa-warning-sign", "idx": 1, "links": [], - "modified": "2022-05-19 05:32:16.026684", + "modified": "2022-06-13 05:43:30.969901", "modified_by": "Administrator", "module": "Core", "name": "Error Log", From 1de9c6ac68ea5f29e4144b7a879f6667ff668e8a Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 13 Jun 2022 15:18:25 +0530 Subject: [PATCH 044/205] fix(ux): add user as filter on route history --- .../doctype/route_history/route_history.json | 100 +++--------------- 1 file changed, 13 insertions(+), 87 deletions(-) diff --git a/frappe/desk/doctype/route_history/route_history.json b/frappe/desk/doctype/route_history/route_history.json index 09db2320ca..a5d73fc360 100644 --- a/frappe/desk/doctype/route_history/route_history.json +++ b/frappe/desk/doctype/route_history/route_history.json @@ -1,126 +1,52 @@ { - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 0, + "actions": [], "creation": "2018-10-05 11:26:04.601113", - "custom": 0, - "docstatus": 0, "doctype": "DocType", - "document_type": "", "editable_grid": 1, "engine": "InnoDB", + "field_order": [ + "route", + "user" + ], "fields": [ { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "route", "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, "in_list_view": 1, - "in_standard_filter": 0, - "label": "Route", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "label": "Route" }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "user", "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, "in_list_view": 1, - "in_standard_filter": 0, + "in_standard_filter": 1, "label": "User", - "length": 0, - "no_copy": 0, - "options": "User", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "options": "User" } ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2021-10-25 13:26:03.106050", + "links": [], + "modified": "2022-06-13 05:48:56.967244", "modified_by": "Administrator", "module": "Desk", "name": "Route History", - "name_case": "", "owner": "Administrator", "permissions": [ { - "amend": 0, - "cancel": 0, "create": 1, "delete": 1, "email": 1, "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 0, "print": 1, "read": 1, "report": 1, "role": "System Manager", - "set_user_permissions": 0, "share": 1, - "submit": 0, "write": 1 } ], "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, "sort_field": "modified", "sort_order": "DESC", - "track_seen": 0, - "track_views": 0 -} + "states": [], + "title_field": "route" +} \ No newline at end of file From 149594448ead89608eb46971cbca3d9fbbf639db Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 13 Jun 2022 15:21:53 +0530 Subject: [PATCH 045/205] fix(ux): make deleted doctype read only - also add filter for doctype in list view --- .../deleted_document/deleted_document.json | 309 ++++-------------- 1 file changed, 67 insertions(+), 242 deletions(-) diff --git a/frappe/core/doctype/deleted_document/deleted_document.json b/frappe/core/doctype/deleted_document/deleted_document.json index 1a612c7411..6b95a523c1 100644 --- a/frappe/core/doctype/deleted_document/deleted_document.json +++ b/frappe/core/doctype/deleted_document/deleted_document.json @@ -1,256 +1,81 @@ { - "allow_copy": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 0, - "creation": "2016-12-29 12:59:48.638970", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", + "actions": [], + "creation": "2016-12-29 12:59:48.638970", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "deleted_name", + "deleted_doctype", + "column_break_3", + "restored", + "new_name", + "section_break_6", + "data" + ], "fields": [ { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "deleted_name", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Deleted Name", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "deleted_name", + "fieldtype": "Data", + "in_list_view": 1, + "in_standard_filter": 1, + "label": "Deleted Name", + "read_only": 1 + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "deleted_doctype", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Deleted DocType", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "deleted_doctype", + "fieldtype": "Data", + "in_list_view": 1, + "in_standard_filter": 1, + "label": "Deleted DocType", + "read_only": 1 + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "column_break_3", - "fieldtype": "Column Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "column_break_3", + "fieldtype": "Column Break" + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "restored", - "fieldtype": "Check", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Restored", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "default": "0", + "fieldname": "restored", + "fieldtype": "Check", + "in_list_view": 1, + "label": "Restored", + "read_only": 1 + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "new_name", - "fieldtype": "Read Only", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "New Name", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "new_name", + "fieldtype": "Read Only", + "label": "New Name" + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "section_break_6", - "fieldtype": "Section Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "section_break_6", + "fieldtype": "Section Break" + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "data", - "fieldtype": "Code", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Data", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "fieldname": "data", + "fieldtype": "Code", + "label": "Data", + "read_only": 1 } - ], - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 1, - - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2016-12-29 14:39:45.724494", - "modified_by": "Administrator", - "module": "Core", - "name": "Deleted Document", - "name_case": "", - "owner": "Administrator", + ], + "in_create": 1, + "links": [], + "modified": "2022-06-13 05:50:58.314908", + "modified_by": "Administrator", + "module": "Core", + "name": "Deleted Document", + "owner": "Administrator", "permissions": [ { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 0, - "delete": 1, - "email": 0, - "export": 1, - "if_owner": 0, - "import": 0, - "is_custom": 0, - "permlevel": 0, - "print": 0, - "read": 1, - "report": 0, - "role": "System Manager", - "set_user_permissions": 0, - "share": 0, - "submit": 0, - "write": 0 + "delete": 1, + "export": 1, + "read": 1, + "role": "System Manager" } - ], - "quick_entry": 0, - "read_only": 0, - "read_only_onload": 0, - "sort_field": "modified", - "sort_order": "DESC", - "title_field": "deleted_name", - "track_changes": 1, - "track_seen": 0 + ], + "sort_field": "modified", + "sort_order": "DESC", + "states": [], + "title_field": "deleted_name", + "track_changes": 1 } \ No newline at end of file From 37086c035c4f1130ceba591a3b6a27778e673947 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 13 Jun 2022 15:29:46 +0530 Subject: [PATCH 046/205] fix: show user and docname filter on access log --- frappe/core/doctype/access_log/access_log.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frappe/core/doctype/access_log/access_log.json b/frappe/core/doctype/access_log/access_log.json index c5f1030266..69803ef05a 100644 --- a/frappe/core/doctype/access_log/access_log.json +++ b/frappe/core/doctype/access_log/access_log.json @@ -36,6 +36,7 @@ "fieldname": "user", "fieldtype": "Link", "in_list_view": 1, + "in_standard_filter": 1, "label": "User ", "options": "User", "read_only": 1 @@ -51,6 +52,7 @@ "fieldname": "reference_document", "fieldtype": "Data", "in_list_view": 1, + "in_standard_filter": 1, "label": "Reference Document", "read_only": 1 }, @@ -129,7 +131,7 @@ } ], "links": [], - "modified": "2022-05-03 09:34:19.337551", + "modified": "2022-06-13 05:59:26.866004", "modified_by": "Administrator", "module": "Core", "name": "Access Log", From ec860cacea07a1ecb6b9042769c2a8a9425b76bf Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 13 Jun 2022 15:34:39 +0530 Subject: [PATCH 047/205] fix(ux): list view filters for server script --- frappe/core/doctype/server_script/server_script.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frappe/core/doctype/server_script/server_script.json b/frappe/core/doctype/server_script/server_script.json index 9312ae178b..5446cc1a39 100644 --- a/frappe/core/doctype/server_script/server_script.json +++ b/frappe/core/doctype/server_script/server_script.json @@ -25,6 +25,7 @@ "fieldname": "script_type", "fieldtype": "Select", "in_list_view": 1, + "in_standard_filter": 1, "label": "Script Type", "options": "DocType Event\nScheduler Event\nPermission Query\nAPI", "reqd": 1 @@ -41,6 +42,7 @@ "fieldname": "reference_doctype", "fieldtype": "Link", "in_list_view": 1, + "in_standard_filter": 1, "label": "Reference Document Type", "options": "DocType" }, @@ -109,7 +111,7 @@ "link_fieldname": "server_script" } ], - "modified": "2022-04-27 11:42:52.032963", + "modified": "2022-06-13 06:04:20.937969", "modified_by": "Administrator", "module": "Core", "name": "Server Script", From 77d03d239387e562c9f09c46efc4cc37d95400b4 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 13 Jun 2022 15:44:51 +0530 Subject: [PATCH 048/205] fix: add list view filters for prepared report --- frappe/core/doctype/prepared_report/prepared_report.json | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/frappe/core/doctype/prepared_report/prepared_report.json b/frappe/core/doctype/prepared_report/prepared_report.json index 4663dcb463..114b5f5b13 100644 --- a/frappe/core/doctype/prepared_report/prepared_report.json +++ b/frappe/core/doctype/prepared_report/prepared_report.json @@ -31,6 +31,7 @@ "fieldname": "ref_report_doctype", "fieldtype": "Link", "hidden": 1, + "in_standard_filter": 1, "label": "Ref Report DocType", "options": "Report", "read_only": 1 @@ -41,6 +42,7 @@ "fieldtype": "Select", "hidden": 1, "in_list_view": 1, + "in_standard_filter": 1, "label": "Status", "options": "Error\nQueued\nCompleted", "read_only": 1 @@ -103,10 +105,11 @@ ], "in_create": 1, "links": [], - "modified": "2020-03-05 10:52:56.598365", + "modified": "2022-06-13 06:14:30.144230", "modified_by": "Administrator", "module": "Core", "name": "Prepared Report", + "naming_rule": "Expression (old style)", "owner": "Administrator", "permissions": [ { @@ -134,6 +137,7 @@ "quick_entry": 1, "sort_field": "modified", "sort_order": "DESC", + "states": [], "title_field": "report_name", "track_changes": 1 } \ No newline at end of file From 9b7b32e60417bcfd13749aac51825082ccbb10c8 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 13 Jun 2022 15:52:07 +0530 Subject: [PATCH 049/205] fix(ux): report and status filter for prepared_report --- frappe/core/doctype/prepared_report/prepared_report.json | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/frappe/core/doctype/prepared_report/prepared_report.json b/frappe/core/doctype/prepared_report/prepared_report.json index 114b5f5b13..cafe323519 100644 --- a/frappe/core/doctype/prepared_report/prepared_report.json +++ b/frappe/core/doctype/prepared_report/prepared_report.json @@ -23,16 +23,14 @@ { "fieldname": "report_name", "fieldtype": "Data", - "hidden": 1, "label": "Report Name", "read_only": 1 }, { "fieldname": "ref_report_doctype", "fieldtype": "Link", - "hidden": 1, "in_standard_filter": 1, - "label": "Ref Report DocType", + "label": "Report Type", "options": "Report", "read_only": 1 }, @@ -105,7 +103,7 @@ ], "in_create": 1, "links": [], - "modified": "2022-06-13 06:14:30.144230", + "modified": "2022-06-13 06:20:34.496412", "modified_by": "Administrator", "module": "Core", "name": "Prepared Report", @@ -134,10 +132,9 @@ "share": 1 } ], - "quick_entry": 1, "sort_field": "modified", "sort_order": "DESC", "states": [], - "title_field": "report_name", + "title_field": "ref_report_doctype", "track_changes": 1 } \ No newline at end of file From cc0672537ea8815326b49d4802c31a445f8b0697 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 13 Jun 2022 16:05:28 +0530 Subject: [PATCH 050/205] fix: make error/patch log read only --- frappe/core/doctype/error_log/error_log.js | 4 +++- frappe/core/doctype/error_log/error_log.json | 4 ++-- frappe/core/doctype/patch_log/patch_log.js | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/frappe/core/doctype/error_log/error_log.js b/frappe/core/doctype/error_log/error_log.js index 277969edd9..1262002b04 100644 --- a/frappe/core/doctype/error_log/error_log.js +++ b/frappe/core/doctype/error_log/error_log.js @@ -3,8 +3,10 @@ frappe.ui.form.on("Error Log", { refresh: function(frm) { + frm.disable_save(); + if (frm.doc.reference_doctype && frm.doc.reference_name) { - frm.add_custom_button(__("Show Errors for This Document"), function() { + frm.add_custom_button(__("Show Related Errors"), function() { frappe.set_route("List", "Error Log", { reference_doctype: frm.doc.reference_doctype, reference_name: frm.doc.reference_name, diff --git a/frappe/core/doctype/error_log/error_log.json b/frappe/core/doctype/error_log/error_log.json index a93161c891..2ee86bd118 100644 --- a/frappe/core/doctype/error_log/error_log.json +++ b/frappe/core/doctype/error_log/error_log.json @@ -61,8 +61,9 @@ ], "icon": "fa fa-warning-sign", "idx": 1, + "in_create": 1, "links": [], - "modified": "2022-06-13 05:43:30.969901", + "modified": "2022-06-13 06:34:05.158606", "modified_by": "Administrator", "module": "Core", "name": "Error Log", @@ -80,7 +81,6 @@ "write": 1 } ], - "quick_entry": 1, "sort_field": "modified", "sort_order": "ASC", "states": [], diff --git a/frappe/core/doctype/patch_log/patch_log.js b/frappe/core/doctype/patch_log/patch_log.js index 0080584a29..b52876ac97 100644 --- a/frappe/core/doctype/patch_log/patch_log.js +++ b/frappe/core/doctype/patch_log/patch_log.js @@ -3,6 +3,6 @@ frappe.ui.form.on('Patch Log', { refresh: function(frm) { - + frm.disable_save(); } }); From 6dfbf7b19ea735d45a61c3e3a3e70be540827b60 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 13 Jun 2022 16:10:36 +0530 Subject: [PATCH 051/205] fix: show doctype and fieldname filters --- frappe/custom/doctype/custom_field/custom_field.json | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/frappe/custom/doctype/custom_field/custom_field.json b/frappe/custom/doctype/custom_field/custom_field.json index 045a0981f3..63be70c644 100644 --- a/frappe/custom/doctype/custom_field/custom_field.json +++ b/frappe/custom/doctype/custom_field/custom_field.json @@ -67,7 +67,8 @@ "fieldtype": "Link", "in_filter": 1, "in_list_view": 1, - "label": "Document", + "in_standard_filter": 1, + "label": "DocType", "oldfieldname": "dt", "oldfieldtype": "Link", "options": "DocType", @@ -94,6 +95,7 @@ "fieldname": "fieldname", "fieldtype": "Data", "in_list_view": 1, + "in_standard_filter": 1, "label": "Fieldname", "no_copy": 1, "oldfieldname": "fieldname", @@ -439,7 +441,7 @@ "idx": 1, "index_web_pages_for_search": 1, "links": [], - "modified": "2022-04-14 09:46:58.849765", + "modified": "2022-06-13 06:39:03.319667", "modified_by": "Administrator", "module": "Custom", "name": "Custom Field", From 963648667d5a03cb80160577e565814a050565b4 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 3 Jun 2022 15:41:55 +0530 Subject: [PATCH 052/205] refactor: filter_dynamic_link_doctypes API * Added typing, better variable naming * Remove unnecessary re-iterations * Optimize queries and membership processing --- frappe/contacts/address_and_contact.py | 40 +++++++++++++++----------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/frappe/contacts/address_and_contact.py b/frappe/contacts/address_and_contact.py index 036594926e..a0f742c55a 100644 --- a/frappe/contacts/address_and_contact.py +++ b/frappe/contacts/address_and_contact.py @@ -3,6 +3,7 @@ import functools import re +from typing import Dict, List import frappe from frappe import _ @@ -169,29 +170,34 @@ def delete_contact_and_address(doctype, docname): @frappe.whitelist() @frappe.validate_and_sanitize_search_inputs -def filter_dynamic_link_doctypes(doctype, txt, searchfield, start, page_len, filters): - if not txt: - txt = "" +def filter_dynamic_link_doctypes(txt: str, filters: Dict) -> List[List[str]]: + from frappe.permissions import get_doctypes_with_read - doctypes = frappe.db.get_all( - "DocField", filters=filters, fields=["parent"], distinct=True, as_list=True + txt = txt or "" + filters = filters or {} + TXT_PATTERN = re.compile(f"{txt}.*") + + _doctypes_from_df = frappe.get_all( + "DocField", + filters=filters, + pluck="parent", + distinct=True, + order_by=None, ) + doctypes_from_df = {d for d in _doctypes_from_df if TXT_PATTERN.search(_(d), re.IGNORECASE)} - doctypes = tuple(d for d in doctypes if re.search(txt + ".*", _(d[0]), re.IGNORECASE)) + filters.update({"dt": ("not in", doctypes_from_df)}) + _doctypes_from_cdf = frappe.get_all( + "Custom Field", filters=filters, pluck="dt", distinct=True, order_by=None + ) + doctypes_from_cdf = {d for d in _doctypes_from_cdf if TXT_PATTERN.search(_(d), re.IGNORECASE)} - filters.update({"dt": ("not in", [d[0] for d in doctypes])}) + all_doctypes = doctypes_from_df.union(doctypes_from_cdf) + allowed_doctypes = set(get_doctypes_with_read()) - _doctypes = frappe.db.get_all("Custom Field", filters=filters, fields=["dt"], as_list=True) + valid_doctypes = sorted(all_doctypes.intersection(allowed_doctypes)) - _doctypes = tuple([d for d in _doctypes if re.search(txt + ".*", _(d[0]), re.IGNORECASE)]) - - all_doctypes = [d[0] for d in doctypes + _doctypes] - allowed_doctypes = frappe.permissions.get_doctypes_with_read() - - valid_doctypes = sorted(set(all_doctypes).intersection(set(allowed_doctypes))) - valid_doctypes = [[doctype] for doctype in valid_doctypes] - - return valid_doctypes + return [[doctype] for doctype in valid_doctypes] def set_link_title(doc): From 13cf4964a6b921ed7c0ffe2d3daf163c6e3cd430 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 3 Jun 2022 16:06:12 +0530 Subject: [PATCH 053/205] perf: Check query type via is_query_type --- frappe/database/database.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index 1de22af037..7aafa3b7f0 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -29,6 +29,10 @@ SINGLE_WORD_PATTERN = re.compile(r'([`"]?)(tab([A-Z]\w+))\1') MULTI_WORD_PATTERN = re.compile(r'([`"])(tab([A-Z]\w+)( [A-Z]\w+)+)\1') +def is_query_type(query: str, query_type: Union[str, Tuple[str]]) -> bool: + return query.lstrip().split(maxsplit=1)[0].lower().startswith(query_type) + + class Database(object): """ Open a database connection with the given parmeters, if use_default is True, use the @@ -248,7 +252,7 @@ class Database(object): # debug if debug: - if explain and query.strip().lower().startswith("select"): + if explain and is_query_type(query, "select"): self.explain_query(query, values) frappe.errprint(self.mogrify(query, values)) @@ -305,7 +309,7 @@ class Database(object): could cause the system to hang.""" self.check_implicit_commit(query) - if query and query.strip().lower() in ("commit", "rollback"): + if query and is_query_type(query, ("commit", "rollback")): self.transaction_writes = 0 if query[:6].lower() in ("update", "insert", "delete"): @@ -322,8 +326,7 @@ class Database(object): if ( self.transaction_writes and query - and query.strip().split()[0].lower() - in ["start", "alter", "drop", "create", "begin", "truncate"] + and is_query_type(query, ("start", "alter", "drop", "create", "begin", "truncate")) ): raise Exception("This statement can cause implicit commit") @@ -346,7 +349,7 @@ class Database(object): @staticmethod def clear_db_table_cache(query): - if query and query.strip().split()[0].lower() in {"drop", "create"}: + if query and is_query_type(query, ("drop", "create")): frappe.cache().delete_key("db_tables") @staticmethod @@ -1207,7 +1210,7 @@ class Database(object): def log_touched_tables(self, query, values=None): if values: query = frappe.safe_decode(self._cursor.mogrify(query, values)) - if query.strip().lower().split()[0] in ("insert", "delete", "update", "alter", "drop", "rename"): + if is_query_type(query, ("insert", "delete", "update", "alter", "drop", "rename")): # single_word_regex is designed to match following patterns # `tabXxx`, tabXxx and "tabXxx" From a1691784a87b239f4efdc2e0ef62013cadca16e9 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 3 Jun 2022 21:20:39 +0530 Subject: [PATCH 054/205] chore: Drop duplicate get_frontmatter definition --- frappe/website/router.py | 22 +--------------------- 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/frappe/website/router.py b/frappe/website/router.py index e9f0d0f09c..8c21501a4e 100644 --- a/frappe/website/router.py +++ b/frappe/website/router.py @@ -8,7 +8,7 @@ import re from werkzeug.routing import Map, NotFound, Rule import frappe -from frappe.website.utils import extract_title +from frappe.website.utils import extract_title, get_frontmatter def get_page_info_from_web_page_with_dynamic_routes(path): @@ -161,26 +161,6 @@ def get_page_info(path, app, start, basepath=None, app_path=None, fname=None): return page_info -def get_frontmatter(string): - """ - Reference: https://github.com/jonbeebe/frontmatter - """ - import yaml - - fmatter = "" - body = "" - result = re.compile(r"^\s*(?:---|\+\+\+)(.*?)(?:---|\+\+\+)\s*(.+)$", re.S | re.M).search(string) - - if result: - fmatter = result.group(1) - body = result.group(2) - - return { - "attributes": yaml.safe_load(fmatter), - "body": body, - } - - def setup_source(page_info): """Get the HTML source of the template""" jenv = frappe.get_jenv() From 7e25cc4568073ff4696b75858d0d9d39adbb566c Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 3 Jun 2022 22:06:46 +0530 Subject: [PATCH 055/205] perf: Login Page Improves performance 3x - from 0.047s to 0.017s * Use frappe.get_*_settings to query table once * Use cached LDAP Settings' document via get_ldap_client_settings * Use single get_all to query all Social Login providers and related data * Skip provider if client_secret doesn't exist --- .../doctype/ldap_settings/ldap_settings.py | 2 +- frappe/www/login.py | 55 +++++++++---------- 2 files changed, 26 insertions(+), 31 deletions(-) diff --git a/frappe/integrations/doctype/ldap_settings/ldap_settings.py b/frappe/integrations/doctype/ldap_settings/ldap_settings.py index a14124234f..96007ee918 100644 --- a/frappe/integrations/doctype/ldap_settings/ldap_settings.py +++ b/frappe/integrations/doctype/ldap_settings/ldap_settings.py @@ -120,7 +120,7 @@ class LDAPSettings(Document): def get_ldap_client_settings(): # return the settings to be used on the client side. result = {"enabled": False} - ldap = frappe.get_doc("LDAP Settings") + ldap = frappe.get_cached_doc("LDAP Settings") if ldap.enabled: result["enabled"] = True result["method"] = "frappe.integrations.doctype.ldap_settings.ldap_settings.login" diff --git a/frappe/www/login.py b/frappe/www/login.py index 1b9a8c239a..119dfefcd7 100644 --- a/frappe/www/login.py +++ b/frappe/www/login.py @@ -36,22 +36,14 @@ def get_context(context): frappe.local.flags.redirect_location = redirect_to raise frappe.Redirect - # get settings from site config context.no_header = True context.for_test = "login.html" context["title"] = "Login" context["provider_logins"] = [] - context["disable_signup"] = frappe.utils.cint( - frappe.db.get_single_value("Website Settings", "disable_signup") - ) - context["logo"] = ( - frappe.db.get_single_value("Website Settings", "app_logo") - or frappe.get_hooks("app_logo_url")[-1] - ) + context["disable_signup"] = frappe.utils.cint(frappe.get_website_settings("disable_signup")) + context["logo"] = frappe.get_website_settings("app_logo") or frappe.get_hooks("app_logo_url")[-1] context["app_name"] = ( - frappe.db.get_single_value("Website Settings", "app_name") - or frappe.get_system_settings("app_name") - or _("Frappe") + frappe.get_website_settings("app_name") or frappe.get_system_settings("app_name") or _("Frappe") ) signup_form_template = frappe.get_hooks("signup_form_template") @@ -61,38 +53,41 @@ def get_context(context): path = frappe.get_attr(signup_form_template[-1])() else: path = "frappe/templates/signup.html" + if path: context["signup_form_template"] = frappe.get_template(path).render() - providers = [ - i.name - for i in frappe.get_all("Social Login Key", filters={"enable_social_login": 1}, order_by="name") - ] + providers = frappe.get_all( + "Social Login Key", + filters={"enable_social_login": 1}, + fields=["name", "client_id", "base_url", "provider_name", "icon"], + order_by="name", + ) + for provider in providers: - client_id, base_url = frappe.get_value("Social Login Key", provider, ["client_id", "base_url"]) - client_secret = get_decrypted_password("Social Login Key", provider, "client_secret") - provider_name = frappe.get_value("Social Login Key", provider, "provider_name") + client_secret = get_decrypted_password("Social Login Key", provider.name, "client_secret") + if not client_secret: + continue icon = None - icon_url = frappe.get_value("Social Login Key", provider, "icon") - if icon_url: - if provider_name != "Custom": - icon = "{1}".format(icon_url, provider_name) + if provider.icon: + if provider.provider_name == "Custom": + icon = get_icon_html(provider.icon, small=True) else: - icon = get_icon_html(icon_url, small=True) + icon = f"{provider.provider_name}" - if get_oauth_keys(provider) and client_secret and client_id and base_url: + if provider.client_id and provider.base_url and get_oauth_keys(provider.name): context.provider_logins.append( { - "name": provider, - "provider_name": provider_name, - "auth_url": get_oauth2_authorize_url(provider, redirect_to), + "name": provider.name, + "provider_name": provider.provider_name, + "auth_url": get_oauth2_authorize_url(provider.name, redirect_to), "icon": icon, } ) context["social_login"] = True - ldap_settings = LDAPSettings.get_ldap_client_settings() - context["ldap_settings"] = ldap_settings + + context["ldap_settings"] = LDAPSettings.get_ldap_client_settings() login_label = [_("Email")] @@ -102,7 +97,7 @@ def get_context(context): if frappe.utils.cint(frappe.get_system_settings("allow_login_using_user_name")): login_label.append(_("Username")) - context["login_label"] = " {0} ".format(_("or")).join(login_label) + context["login_label"] = f" {_('or')} ".join(login_label) return context From ce38587a4e14df9a89ec5c0a055060b464e1b6d9 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 3 Jun 2022 22:22:21 +0530 Subject: [PATCH 056/205] perf: About Us Settings Use cached document for building /about page --- frappe/www/about.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/www/about.py b/frappe/www/about.py index fd2331d9f6..a233bfd311 100644 --- a/frappe/www/about.py +++ b/frappe/www/about.py @@ -7,6 +7,6 @@ sitemap = 1 def get_context(context): - context.doc = frappe.get_doc("About Us Settings", "About Us Settings") + context.doc = frappe.get_cached_doc("About Us Settings") return context From 3871fe6cd042587abbd3d722f97860a6c006e0a2 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 3 Jun 2022 22:29:05 +0530 Subject: [PATCH 057/205] perf: App Page Reduced time taken for get_context to execute from 0.035s to 0.02s (75% reduction) --- frappe/sessions.py | 2 +- frappe/www/app.py | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/frappe/sessions.py b/frappe/sessions.py index 6e0ce73732..67b58e1d89 100644 --- a/frappe/sessions.py +++ b/frappe/sessions.py @@ -187,7 +187,7 @@ def get(): bootinfo["translated_search_doctypes"] = frappe.get_hooks("translated_search_doctypes") bootinfo["disable_async"] = frappe.conf.disable_async - bootinfo["setup_complete"] = cint(frappe.db.get_single_value("System Settings", "setup_complete")) + bootinfo["setup_complete"] = cint(frappe.get_system_settings("setup_complete")) bootinfo["desk_theme"] = frappe.db.get_value("User", frappe.session.user, "desk_theme") or "Light" diff --git a/frappe/www/app.py b/frappe/www/app.py index f1b62a0899..9a8c80d6b1 100644 --- a/frappe/www/app.py +++ b/frappe/www/app.py @@ -32,8 +32,6 @@ def get_context(context): frappe.db.commit() - desk_theme = frappe.db.get_value("User", frappe.session.user, "desk_theme") - boot_json = frappe.as_json(boot) # remove script tags from boot @@ -52,7 +50,7 @@ def get_context(context): "lang": frappe.local.lang, "sounds": hooks["sounds"], "boot": boot if context.get("for_mobile") else boot_json, - "desk_theme": desk_theme or "Light", + "desk_theme": boot.get("desk_theme") or "Light", "csrf_token": csrf_token, "google_analytics_id": frappe.conf.get("google_analytics_id"), "google_analytics_anonymize_ip": frappe.conf.get("google_analytics_anonymize_ip"), From 64e52737646d1661e0d295868fc85aa0da47b1f2 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 10 Jun 2022 17:42:47 +0530 Subject: [PATCH 058/205] perf: Patch qb only once - not on every init --- frappe/__init__.py | 7 +++++-- frappe/query_builder/utils.py | 5 ++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/frappe/__init__.py b/frappe/__init__.py index 51c6ba3a74..37a03e412c 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -44,6 +44,7 @@ local = Local() STANDARD_USERS = ("Guest", "Administrator") _dev_server = int(sbool(os.environ.get("DEV_SERVER", False))) +_qb_patched = False if _dev_server: warnings.simplefilter("always", DeprecationWarning) @@ -236,8 +237,10 @@ def init(site, sites_path=None, new_site=False): local.qb = get_query_builder(local.conf.db_type or "mariadb") setup_module_map() - patch_query_execute() - patch_query_aggregation() + + if not _qb_patched: + patch_query_execute() + patch_query_aggregation() local.initialised = True diff --git a/frappe/query_builder/utils.py b/frappe/query_builder/utils.py index 69aee9b350..c75e380d49 100644 --- a/frappe/query_builder/utils.py +++ b/frappe/query_builder/utils.py @@ -64,7 +64,6 @@ def patch_query_execute(): This excludes the use of `frappe.db.sql` method while executing the query object """ - from frappe.utils.safe_exec import check_safe_sql_query def execute_query(query, *args, **kwargs): query, params = prepare_query(query) @@ -73,6 +72,8 @@ def patch_query_execute(): def prepare_query(query): import inspect + from frappe.utils.safe_exec import check_safe_sql_query + param_collector = NamedParameterWrapper() query = query.get_sql(param_wrapper=param_collector) if frappe.flags.in_safe_exec and not check_safe_sql_query(query, throw=False): @@ -103,6 +104,7 @@ def patch_query_execute(): builder_class.run = execute_query builder_class.walk = prepare_query + frappe._qb_patched = True def patch_query_aggregation(): @@ -113,3 +115,4 @@ def patch_query_aggregation(): frappe.qb.min = _min frappe.qb.avg = _avg frappe.qb.sum = _sum + frappe._qb_patched = True From e681233e982c7eb44f4b2df86ed49ba763066c03 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 3 Jun 2022 22:05:33 +0530 Subject: [PATCH 059/205] perf: Fetch and cache entire settings' dicts --- frappe/__init__.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/frappe/__init__.py b/frappe/__init__.py index 37a03e412c..7c71bc8815 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -871,6 +871,10 @@ def clear_cache(user=None, doctype=None): local.role_permissions = {} if hasattr(local, "request_cache"): local.request_cache.clear() + if hasattr(local, "system_settings"): + del local.system_settings + if hasattr(local, "website_settings"): + del local.website_settings def only_has_select_perm(doctype, user=None, ignore_permissions=False): @@ -1094,6 +1098,10 @@ def clear_document_cache(doctype, name): if key in local.document_cache: del local.document_cache[key] cache().hdel("document_cache", key) + if doctype == "System Settings" and hasattr(local, "system_settings"): + delattr(local, "system_settings") + if doctype == "Website Settings" and hasattr(local, "website_settings"): + delattr(local, "website_settings") def get_cached_value(doctype, name, fieldname="name", as_dict=False): @@ -2206,8 +2214,18 @@ def safe_eval(code, eval_globals=None, eval_locals=None): return eval(code, eval_globals, eval_locals) +def get_website_settings(key): + if not hasattr(local, "website_settings"): + local.website_settings = db.get_singles_dict("Website Settings") + + return local.website_settings[key] + + def get_system_settings(key): - return db.get_single_value("System Settings", key, cache=True) + if not hasattr(local, "system_settings"): + local.system_settings = db.get_singles_dict("System Settings") + + return local.system_settings[key] def get_active_domains(): From f74dc5023d5ab1598e80a586b656b34e18a5ec0c Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 13 Jun 2022 11:52:36 +0530 Subject: [PATCH 060/205] refactor!: frappe.db.get_singles_dict * Cast single's values as their fieldtypes before returning * Support previously dead debug parameter * Consider single with no meta as non-existent; skip query Decided to go ahead with the breaking change given the nature of the existing usages of get_singles_dict :crie: --- frappe/database/database.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index 7aafa3b7f0..39ca846904 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -1,4 +1,4 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors +# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE # Database Module @@ -18,6 +18,7 @@ import frappe import frappe.defaults import frappe.model.meta from frappe import _ +from frappe.exceptions import DoesNotExistError from frappe.model.utils.link_count import flush_local_link_count from frappe.query_builder.functions import Count from frappe.query_builder.utils import DocType @@ -628,14 +629,28 @@ class Database(object): # Get coulmn and value of the single doctype Accounts Settings account_settings = frappe.db.get_singles_dict("Accounts Settings") """ - result = self.query.get_sql( + return_value = frappe._dict() + + try: + meta = frappe.get_meta(doctype) + except DoesNotExistError: + return return_value + + queried_result = self.query.get_sql( "Singles", filters={"doctype": doctype}, fields=["field", "value"], for_update=for_update, - ).run() + ).run(debug=debug) - return frappe._dict(result) + for fieldname, value in queried_result: + if df := meta.get_field(fieldname): + casted_value = cast(df.fieldtype, value) + else: + casted_value = value + return_value[fieldname] = casted_value + + return return_value @staticmethod def get_all(*args, **kwargs): From 601217a4a233ce666eff5388159b7d72da75f231 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 13 Jun 2022 19:14:48 +0530 Subject: [PATCH 061/205] ci: Run tests bypassing roulette with labels "Run UI Tests", "Run Server Tests" --- .github/helper/roulette.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/.github/helper/roulette.py b/.github/helper/roulette.py index f68ef5046f..9165198012 100644 --- a/.github/helper/roulette.py +++ b/.github/helper/roulette.py @@ -5,8 +5,10 @@ import shlex import subprocess import sys import urllib.request +from functools import cache +@cache def fetch_pr_data(pr_number, repo, endpoint): api_url = f"https://api.github.com/repos/{repo}/pulls/{pr_number}" @@ -26,7 +28,16 @@ def get_output(command, shell=True): return subprocess.check_output(command, shell=shell, encoding="utf8").strip() def has_skip_ci_label(pr_number, repo="frappe/frappe"): - return any([label["name"] for label in fetch_pr_data(pr_number, repo, "")["labels"] if label["name"] == "Skip CI"]) + return has_label(pr_number, "Skip CI", repo) + +def has_run_server_tests_label(pr_number, repo="frappe/frappe"): + return has_label(pr_number, "Run Server Tests", repo) + +def has_run_ui_tests_label(pr_number, repo="frappe/frappe"): + return has_label(pr_number, "Run UI Tests", repo) + +def has_label(pr_number, label, repo="frappe/frappe"): + return any([label["name"] for label in fetch_pr_data(pr_number, repo, "")["labels"] if label["name"] == label]) def is_py(file): return file.endswith("py") @@ -77,11 +88,11 @@ if __name__ == "__main__": print("Only docs were updated, stopping build process.") sys.exit(0) - elif only_frontend_code_changed and build_type == "server": + elif only_frontend_code_changed and build_type == "server" and not has_run_server_tests_label(pr_number, repo): print("Only Frontend code was updated; Stopping Python build process.") sys.exit(0) - elif build_type == "ui" and only_py_changed: + elif build_type == "ui" and only_py_changed and not has_run_ui_tests_label(pr_number, repo): print("Only Python code was updated, stopping Cypress build process.") sys.exit(0) From 2058b0cbeab88aef8116ca44d85b52436fa1525e Mon Sep 17 00:00:00 2001 From: Rushabh Mehta Date: Fri, 10 Jun 2022 15:44:05 +0530 Subject: [PATCH 062/205] fix(ui): tab refresh was not implemented --- frappe/public/js/frappe/form/form.js | 7 +------ frappe/public/js/frappe/form/layout.js | 27 +++++++++++--------------- frappe/public/js/frappe/form/tab.js | 18 +++++++++++++++++ 3 files changed, 30 insertions(+), 22 deletions(-) diff --git a/frappe/public/js/frappe/form/form.js b/frappe/public/js/frappe/form/form.js index b8fa42fa94..eefc629b4d 100644 --- a/frappe/public/js/frappe/form/form.js +++ b/frappe/public/js/frappe/form/form.js @@ -575,7 +575,7 @@ frappe.ui.form.Form = class FrappeForm { this.$wrapper.trigger('render_complete'); - this.cscript.is_onload && this.set_first_tab_as_active(); + this.layout.set_first_tab_as_active(switched || this.cscript.is_onload); if(!this.hidden) { this.layout.show_empty_form_message(); @@ -592,11 +592,6 @@ frappe.ui.form.Form = class FrappeForm { this.setup_image_autocompletions_in_markdown(); } - set_first_tab_as_active() { - this.layout.tabs[0] - && this.layout.tabs[0].set_active(); - } - focus_on_first_input() { let first = this.form_wrapper.find('.form-layout :input:visible:first'); if (!in_list(["Date", "Datetime"], first.attr("data-fieldtype"))) { diff --git a/frappe/public/js/frappe/form/layout.js b/frappe/public/js/frappe/form/layout.js index c1c9967507..6d000c99f9 100644 --- a/frappe/public/js/frappe/form/layout.js +++ b/frappe/public/js/frappe/form/layout.js @@ -294,7 +294,7 @@ frappe.ui.form.Layout = class Layout { this.refresh_sections(); // refresh tabs - this.tabbed_layout && this.refresh_tabs(); + this.is_tabbed_layout() && this.refresh_tabs(); if (this.frm) { // collapse sections @@ -328,21 +328,9 @@ frappe.ui.form.Layout = class Layout { } refresh_tabs() { - this.tabs.forEach(tab => { - if (!tab.wrapper.hasClass('hide') || !tab.parent.hasClass('hide')) { - tab.parent.removeClass('show hide'); - tab.wrapper.removeClass('show hide'); - if ( - tab.wrapper.find( - ".form-section:not(.hide-control, .empty-section), .form-dashboard-section:not(.hide-control, .empty-section)" - ).length - ) { - tab.toggle(true); - } else { - tab.toggle(false); - } - } - }); + for (let tab of this.tabs) { + tab.refresh(); + } const visible_tabs = this.tabs.filter(tab => !tab.hidden); if (visible_tabs && visible_tabs.length == 1) { @@ -350,6 +338,13 @@ frappe.ui.form.Layout = class Layout { } } + set_first_tab_as_active(switched) { + if (this.tabs.length && (switched || !this.frm.active_tab)) { + // set first tab as active when opening for first time, or new doc + this.tabs[0].set_active(); + } + } + refresh_fields(fields) { let fieldnames = fields.map((field) => { if (field.fieldname) return field.fieldname; diff --git a/frappe/public/js/frappe/form/tab.js b/frappe/public/js/frappe/form/tab.js index 3fad807f06..fd23595d80 100644 --- a/frappe/public/js/frappe/form/tab.js +++ b/frappe/public/js/frappe/form/tab.js @@ -36,10 +36,27 @@ export default class Tab { // hide if explicitly hidden let hide = this.df.hidden || this.df.hidden_due_to_dependency; + + // hide if dashboard and not saved + if (!hide && this.df.show_dashboard && this.frm.is_new() && !this.fields_list.length) { + hide = true; + } + + // hide if no read permission if (!hide && this.frm && !this.frm.get_perm(this.df.permlevel || 0, "read")) { hide = true; } + if (!hide && !this.df.show_dashboard) { + // show only if there is at least one visibe section or control + hide = true; + if (this.wrapper.find( + ".form-section:not(.hide-control, .empty-section), .form-dashboard-section:not(.hide-control, .empty-section)" + ).length) { + hide = false; + } + } + this.toggle(!hide); } @@ -62,6 +79,7 @@ export default class Tab { set_active() { this.parent.find('.nav-link').tab('show'); this.wrapper.addClass('show'); + this.frm.active_tab = this; } is_active() { From 52359a0ad9af91413896074eb726b29a0e67bc29 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 14 Jun 2022 12:22:12 +0530 Subject: [PATCH 063/205] test: Scheduler tests cleanup --- frappe/tests/test_scheduler.py | 43 +++++++++++---------------------- frappe/utils/background_jobs.py | 7 ++++-- 2 files changed, 19 insertions(+), 31 deletions(-) diff --git a/frappe/tests/test_scheduler.py b/frappe/tests/test_scheduler.py index 5161e1e80f..c6b381e487 100644 --- a/frappe/tests/test_scheduler.py +++ b/frappe/tests/test_scheduler.py @@ -1,18 +1,16 @@ +import os import time from unittest import TestCase +from unittest.mock import patch import frappe -from frappe.core.doctype.scheduled_job_type.scheduled_job_type import sync_jobs +from frappe.core.doctype.scheduled_job_type.scheduled_job_type import ScheduledJobType, sync_jobs from frappe.utils import add_days, get_datetime from frappe.utils.background_jobs import enqueue from frappe.utils.doctor import purge_pending_jobs from frappe.utils.scheduler import enqueue_events, is_dormant, schedule_jobs_based_on_activity -def test_timeout(): - time.sleep(100) - - def test_timeout_10(): time.sleep(10) @@ -23,6 +21,11 @@ def test_method(): class TestScheduler(TestCase): def setUp(self): + frappe.db.rollback() + + if not os.environ.get("CI"): + return + purge_pending_jobs() if not frappe.get_all("Scheduled Job Type", limit=1): sync_jobs() @@ -44,15 +47,9 @@ class TestScheduler(TestCase): def test_queue_peeking(self): job = get_test_job() - self.assertTrue(job.enqueue()) - job.db_set("last_execution", "2010-01-01 00:00:00") - frappe.db.commit() - - time.sleep(0.5) - - # 1st job is in the queue (or running), don't enqueue it again - self.assertFalse(job.enqueue()) - frappe.db.delete("Scheduled Job Log", {"scheduled_job_type": job.name}) + with patch.object(job, "is_job_in_queue", return_value=True): + # 1st job is in the queue (or running), don't enqueue it again + self.assertFalse(job.enqueue()) def test_is_dormant(self): self.assertTrue(is_dormant(check_time=get_datetime("2100-01-01 00:00:00"))) @@ -88,22 +85,10 @@ class TestScheduler(TestCase): ) ) - frappe.db.rollback() - def test_job_timeout(self): - return - job = enqueue(test_timeout, timeout=10) - count = 5 - while count > 0: - count -= 1 - time.sleep(5) - if job.get_status() == "failed": - break - - self.assertTrue(job.is_failed) - - -def get_test_job(method="frappe.tests.test_scheduler.test_timeout_10", frequency="All"): +def get_test_job( + method="frappe.tests.test_scheduler.test_timeout_10", frequency="All" +) -> ScheduledJobType: if not frappe.db.exists("Scheduled Job Type", dict(method=method)): job = frappe.get_doc( dict( diff --git a/frappe/utils/background_jobs.py b/frappe/utils/background_jobs.py index ce8e44665a..f49c641673 100755 --- a/frappe/utils/background_jobs.py +++ b/frappe/utils/background_jobs.py @@ -3,7 +3,7 @@ import socket import time from collections import defaultdict from functools import lru_cache -from typing import List +from typing import TYPE_CHECKING, List from uuid import uuid4 import redis @@ -19,6 +19,9 @@ from frappe.utils import cstr, get_bench_id from frappe.utils.commands import log from frappe.utils.redis_queue import RedisQueue +if TYPE_CHECKING: + from rq.job import Job + @lru_cache() def get_queues_timeout(): @@ -52,7 +55,7 @@ def enqueue( *, at_front=False, **kwargs, -): +) -> "Job": """ Enqueue method to be executed using a background worker From 80334698a79f0b948cd07de0c557cb9ac97ea61d Mon Sep 17 00:00:00 2001 From: Rushabh Mehta Date: Tue, 14 Jun 2022 12:54:10 +0530 Subject: [PATCH 064/205] fix(minor): Onboarding: add option to view list view in create action --- .../js/frappe/widgets/onboarding_widget.js | 23 ++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/frappe/public/js/frappe/widgets/onboarding_widget.js b/frappe/public/js/frappe/widgets/onboarding_widget.js index 128d7e691a..3da102759f 100644 --- a/frappe/public/js/frappe/widgets/onboarding_widget.js +++ b/frappe/public/js/frappe/widgets/onboarding_widget.js @@ -99,12 +99,7 @@ export default class OnboardingWidget extends Widget { const toggle_content = () => { this.step_body.empty(); this.step_footer.empty(); - - this.step_body.html( - step.description ? - frappe.markdown(step.description) - : `

${step.title}

` - ); + set_description(); if (step.intro_video_url) { $(``) @@ -117,6 +112,22 @@ export default class OnboardingWidget extends Widget { } }; + const set_description = () => { + let content = step.description ? + frappe.markdown(step.description) + : `

${step.title}

` + + if (step.action === 'Create Entry') { + // add a secondary action to view list + content += `

+ + ${ __('Show {0} List', [step.reference_document])} +

` + } + + this.step_body.html(content); + } + const toggle_video = () => { this.step_body.empty(); this.step_footer.empty(); From 602d4376ba8e4db592d28292a39b2f87ce7a9e26 Mon Sep 17 00:00:00 2001 From: Rushabh Mehta Date: Tue, 14 Jun 2022 13:07:13 +0530 Subject: [PATCH 065/205] fix(minor): js lint --- frappe/public/js/frappe/widgets/onboarding_widget.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/frappe/public/js/frappe/widgets/onboarding_widget.js b/frappe/public/js/frappe/widgets/onboarding_widget.js index 3da102759f..e560551f79 100644 --- a/frappe/public/js/frappe/widgets/onboarding_widget.js +++ b/frappe/public/js/frappe/widgets/onboarding_widget.js @@ -114,19 +114,18 @@ export default class OnboardingWidget extends Widget { const set_description = () => { let content = step.description ? - frappe.markdown(step.description) - : `

${step.title}

` + frappe.markdown(step.description) : `

${step.title}

`; if (step.action === 'Create Entry') { // add a secondary action to view list content += `

${ __('Show {0} List', [step.reference_document])} -

` +

`; } this.step_body.html(content); - } + }; const toggle_video = () => { this.step_body.empty(); From 9f941e93121040afd7c90073498e2c8f120dd9dc Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 14 Jun 2022 13:15:56 +0530 Subject: [PATCH 066/205] build: Declarative builds via pyproject.toml There was an attempt to switch to declarative setup, which was to use setup.cfg. However, we've made more progress in the "world of packaging" since. Also, given we're using a pyproject file already to govern certain aspects of our project maintenance, it's probably a good time to bring these things together. bye-bye to clutter and confusion in maintaining multiple files and formats. We've used setuptools to package frappe for the longest time. Maybe since the first editable installation. Howver setuptools doesn't support PEP 660 yet which supports editable installs via pyproject file. So why now? Primarily because I wanted to start tracking system dependencies for Frappe. Doing so through pyproject file made the most sense. This change seemed to be the obvious pre-requisite for the upcoming changes to our packaging systems. Sys deps tracking: https://github.com/frappe/frappe/issues/13811 Flit docs: https://flit.pypa.io/en/latest/pyproject_toml.html Previous attempt: https://github.com/frappe/frappe/pull/10773 Which build tool: https://stackoverflow.com/a/69711730/10309266 PEP 660: https://peps.python.org/pep-0660/ --- pyproject.toml | 94 ++++++++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 75 -------------------------------------- setup.py | 58 ------------------------------ 3 files changed, 94 insertions(+), 133 deletions(-) delete mode 100644 requirements.txt delete mode 100644 setup.py diff --git a/pyproject.toml b/pyproject.toml index 8043dd9906..81be41c480 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,97 @@ +[project] +name = "frappe" +authors = [ + { name = "Frappe Technologies Pvt Ltd", email = "developers@frappe.io"} +] +description = "Metadata driven, full-stack low code web framework" +requires-python = ">=3.8" +readme = "README.md" +dynamic = ["version"] +dependencies = [ + # core dependencies + "Babel~=2.9.0", + "Click~=7.1.2", + "GitPython~=3.1.14", + "Jinja2~=3.1.2", + "Pillow~=9.1.1", + "PyJWT~=2.0.1", + "PyMySQL~=1.0.2", + "PyPDF2~=2.1.0", + "PyPika~=0.48.9", + "PyQRCode~=1.2.1", + "PyYAML~=5.4.1", + "RestrictedPython~=5.1", + "WeasyPrint==52.5", + "Werkzeug~=2.1.2", + "Whoosh~=2.7.4", + "beautifulsoup4~=4.9.3", + "bleach-allowlist~=1.0.3", + "bleach~=3.3.0", + "cairocffi==1.2.0", + "chardet~=4.0.0", + "croniter~=1.3.5", + "cryptography~=37.0.2", + "email-reply-parser~=0.5.12", + "git-url-parse~=1.2.2", + "gitdb~=4.0.7", + "gunicorn~=20.1.0", + "html2text==2020.1.16", + "html5lib~=1.1", + "ipython~=8.4.0", + "ldap3~=2.9", + "markdown2~=2.4.0", + "maxminddb-geolite2==2018.703", + "num2words~=0.5.10", + "oauthlib~=3.1.0", + "openpyxl~=3.0.7", + "parse~=1.19.0", + "passlib~=1.7.4", + "pdfkit~=1.0.0", + "phonenumbers==8.12.40", + "premailer~=3.8.0", + "psutil~=5.9.1", + "psycopg2-binary~=2.9.1", + "pyOpenSSL~=20.0.1", + "pyasn1~=0.4.8", + "pycryptodome~=3.10.1", + "pyotp~=2.6.0", + "pypng~=0.0.20", + "python-dateutil~=2.8.1", + "pytz==2022.1", + "rauth~=0.7.3", + "redis~=3.5.3", + "requests-oauthlib~=1.3.0", + "requests~=2.27.1", + "rq~=1.10.1", + "rsa>=4.1", + "schedule~=1.1.0", + "semantic-version~=2.10.0", + "sqlparse~=0.4.1", + "tenacity~=8.0.1", + "terminaltables~=3.1.0", + "traceback-with-variables~=2.0.4", + "urllib3~=1.26.4", + "xlrd~=2.0.1", + "zxcvbn-python~=4.4.24", + + # integration dependencies + "boto3~=1.17.53", + "braintree~=4.8.0", + "dropbox~=11.7.0", + "google-api-python-client~=2.2.0", + "google-auth-httplib2~=0.1.0", + "google-auth-oauthlib~=0.4.4", + "google-auth~=1.29.0", + "googlemaps~=4.4.5", + "paytmchecksum~=1.7.0", + "razorpay~=1.2.0", + "stripe~=2.56.0", +] + +[build-system] +requires = ["flit_core >=3.4,<4"] +build-backend = "flit_core.buildapi" + [tool.black] line-length = 99 diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 1a6b6120a3..0000000000 --- a/requirements.txt +++ /dev/null @@ -1,75 +0,0 @@ -Babel~=2.9.0 -beautifulsoup4~=4.9.3 -bleach-allowlist~=1.0.3 -bleach~=3.3.0 -boto3~=1.17.53 -braintree~=4.8.0 -chardet~=4.0.0 -Click~=7.1.2 -croniter~=1.3.5 -cryptography~=37.0.2 -dropbox~=11.7.0 -email-reply-parser~=0.5.12 -git-url-parse~=1.2.2 -gitdb~=4.0.7 -GitPython~=3.1.14 -google-api-python-client~=2.2.0 -google-auth-httplib2~=0.1.0 -google-auth-oauthlib~=0.4.4 -google-auth~=1.29.0 -googlemaps~=4.4.5 -gunicorn~=20.1.0 -html2text==2020.1.16 -html5lib~=1.1 -ipython~=8.4.0 -Jinja2~=3.1.2 -ldap3~=2.9 -markdown2~=2.4.0 -maxminddb-geolite2==2018.703 -num2words~=0.5.10 -oauthlib~=3.1.0 -openpyxl~=3.0.7 -parse~=1.19.0 -passlib~=1.7.4 -paytmchecksum~=1.7.0 -pdfkit~=1.0.0 -Pillow~=9.1.1 -premailer~=3.8.0 -psutil~=5.9.1 -psycopg2-binary~=2.9.1 -pyasn1~=0.4.8 -pycryptodome~=3.10.1 -PyJWT~=2.0.1 -PyMySQL~=1.0.2 -pyOpenSSL~=20.0.1 -pyotp~=2.6.0 -PyPDF2~=2.1.0 -PyPika~=0.48.9 -pypng~=0.0.20 -PyQRCode~=1.2.1 -python-dateutil~=2.8.1 -pytz==2022.1 -PyYAML~=5.4.1 -rauth~=0.7.3 -razorpay~=1.2.0 -redis~=3.5.3 -requests-oauthlib~=1.3.0 -requests~=2.27.1 -RestrictedPython~=5.1 -rq~=1.10.1 -rsa>=4.1 # not directly required, pinned by Snyk to avoid a vulnerability -schedule~=1.1.0 -semantic-version~=2.10.0 -sqlparse~=0.4.1 -stripe~=2.56.0 -terminaltables~=3.1.0 -traceback-with-variables~=2.0.4 -urllib3~=1.26.4 -Werkzeug~=2.1.2 -Whoosh~=2.7.4 -xlrd~=2.0.1 -zxcvbn-python~=4.4.24 -tenacity~=8.0.1 -cairocffi==1.2.0 -WeasyPrint==52.5 -phonenumbers==8.12.40 diff --git a/setup.py b/setup.py deleted file mode 100644 index ba4034a766..0000000000 --- a/setup.py +++ /dev/null @@ -1,58 +0,0 @@ -# imports - standard imports -import ast -import os -import re -import shutil -from distutils.command.clean import clean as Clean - -from setuptools import find_packages, setup - -# get version from __version__ variable in frappe/__init__.py -_version_re = re.compile(r"__version__\s+=\s+(.*)") - -with open("requirements.txt") as f: - install_requires = f.read().strip().split("\n") - -with open("frappe/__init__.py", "rb") as f: - version = str(ast.literal_eval(_version_re.search(f.read().decode("utf-8")).group(1))) - - -class CleanCommand(Clean): - def run(self): - Clean.run(self) - - basedir = os.path.abspath(os.path.dirname(__file__)) - - for relpath in ["build", ".cache", ".coverage", "dist", "frappe.egg-info"]: - abspath = os.path.join(basedir, relpath) - if os.path.exists(abspath): - if os.path.isfile(abspath): - os.remove(abspath) - else: - shutil.rmtree(abspath) - - for dirpath, dirnames, filenames in os.walk(basedir): - for filename in filenames: - _, extension = os.path.splitext(filename) - if extension in [".pyc"]: - abspath = os.path.join(dirpath, filename) - os.remove(abspath) - for dirname in dirnames: - if dirname in ["__pycache__"]: - abspath = os.path.join(dirpath, dirname) - shutil.rmtree(abspath) - - -setup( - name="frappe", - version=version, - description="Metadata driven, full-stack web framework", - author="Frappe Technologies", - author_email="info@frappe.io", - packages=find_packages(), - zip_safe=False, - include_package_data=True, - install_requires=install_requires, - cmdclass={"clean": CleanCommand}, - python_requires=">=3.8", -) From 4c965d54df61b02127e6f736b28d132594e88033 Mon Sep 17 00:00:00 2001 From: Rucha Mahabal Date: Tue, 14 Jun 2022 14:12:43 +0530 Subject: [PATCH 067/205] fix(Event Streaming): set remote docname and site name before inserting mapped doc - needed to differentiate a remote doc at the time of insertion --- .../doctype/event_producer/event_producer.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/frappe/event_streaming/doctype/event_producer/event_producer.py b/frappe/event_streaming/doctype/event_producer/event_producer.py index f639e48b50..adbb706c3d 100644 --- a/frappe/event_streaming/doctype/event_producer/event_producer.py +++ b/frappe/event_streaming/doctype/event_producer/event_producer.py @@ -315,8 +315,9 @@ def set_insert(update, producer_site, event_producer): else: # if event consumer is not saving documents with the same name as the producer # store the remote docname in a custom field for future updates - local_doc = doc.insert(set_child_names=False) - set_custom_fields(local_doc, update.docname, event_producer) + doc.remote_docname = update.docname + doc.remote_site_name = event_producer + doc.insert(set_child_names=False) def set_update(update, producer_site): @@ -567,9 +568,3 @@ def resync(update): update = get_mapped_update(update, producer_site) update.data = json.loads(update.data) return sync(update, producer_site, event_producer, in_retry=True) - - -def set_custom_fields(local_doc, remote_docname, remote_site_name): - """sets custom field in doc for storing remote docname""" - frappe.db.set_value(local_doc.doctype, local_doc.name, "remote_docname", remote_docname) - frappe.db.set_value(local_doc.doctype, local_doc.name, "remote_site_name", remote_site_name) From a5c04561d0f573efa619d2f108bc1123bb01b883 Mon Sep 17 00:00:00 2001 From: Rucha Mahabal Date: Tue, 14 Jun 2022 14:12:50 +0530 Subject: [PATCH 068/205] fix: use the correct dt name while populating data in mapping table --- .../doctype/document_type_mapping/document_type_mapping.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.js b/frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.js index 4653bf4d03..22b7f2ef4c 100644 --- a/frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.js +++ b/frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.js @@ -7,7 +7,7 @@ frappe.ui.form.on('Document Type Mapping', { frappe.model.clear_table(frm.doc, 'field_mapping'); let fields = frm.events.get_fields(frm); $.each(fields, function(i, data) { - let row = frappe.model.add_child(frm.doc, 'Document Type Mapping', 'field_mapping'); + let row = frappe.model.add_child(frm.doc, 'Document Type Field Mapping', 'field_mapping'); row.local_fieldname = data; }); refresh_field('field_mapping'); From c1df1b1f1892bd580cb5ea5441407fc4b994adc8 Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Tue, 14 Jun 2022 14:44:55 +0530 Subject: [PATCH 069/205] fix: min height for disabled field moved to common scss --- frappe/public/scss/common/css_variables.scss | 6 ++++++ frappe/public/scss/common/form.scss | 1 + frappe/public/scss/desk/css_variables.scss | 6 ------ 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/frappe/public/scss/common/css_variables.scss b/frappe/public/scss/common/css_variables.scss index e047747f5c..ab52c10e45 100644 --- a/frappe/public/scss/common/css_variables.scss +++ b/frappe/public/scss/common/css_variables.scss @@ -1,3 +1,5 @@ +$input-height: 28px !default; + :root, [data-theme="light"] { --brand-color: #0089FF; @@ -250,6 +252,10 @@ --primary-color: #2490EF; --btn-height: 30px; + // input + --input-height: #{$input-height}; + --input-disabled-bg: var(--gray-200); + // Checkbox --checkbox-right-margin: var(--margin-xs); --checkbox-size: 14px; diff --git a/frappe/public/scss/common/form.scss b/frappe/public/scss/common/form.scss index fcea603994..a01ff54c28 100644 --- a/frappe/public/scss/common/form.scss +++ b/frappe/public/scss/common/form.scss @@ -13,6 +13,7 @@ font-weight: normal; font-size: var(--text-sm); } + min-height: var(--input-height); border-radius: $border-radius; font-weight: 400; padding: 6px 12px; diff --git a/frappe/public/scss/desk/css_variables.scss b/frappe/public/scss/desk/css_variables.scss index a06ba3e9b0..aceaa3e1e6 100644 --- a/frappe/public/scss/desk/css_variables.scss +++ b/frappe/public/scss/desk/css_variables.scss @@ -1,7 +1,5 @@ @import '../common/css_variables.scss'; -$input-height: 28px !default; - :root, [data-theme="light"] { // breakpoints @@ -31,10 +29,6 @@ $input-height: 28px !default; --page-head-height: 75px; --page-bottom-margin: 60px; - // input - --input-height: #{$input-height}; - --input-disabled-bg: var(--gray-200); - // checkbox --checkbox-right-margin: var(--margin-xs); --checkbox-size: 14px; From 0d6891afbf08229bcd223c021eb9c3a70e76aeae Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Tue, 14 Jun 2022 14:46:14 +0530 Subject: [PATCH 070/205] fix: parse value before setting display status --- .../js/frappe/form/controls/base_control.js | 20 ++++++++++++++----- frappe/public/js/frappe/form/controls/date.js | 3 +++ .../js/frappe/form/controls/datetime.js | 3 +++ .../js/frappe/form/controls/duration.js | 6 ++++++ frappe/public/js/frappe/form/controls/time.js | 3 +++ 5 files changed, 30 insertions(+), 5 deletions(-) diff --git a/frappe/public/js/frappe/form/controls/base_control.js b/frappe/public/js/frappe/form/controls/base_control.js index e22235f60f..f491fb1427 100644 --- a/frappe/public/js/frappe/form/controls/base_control.js +++ b/frappe/public/js/frappe/form/controls/base_control.js @@ -72,9 +72,12 @@ frappe.ui.form.Control = class BaseControl { status = "Read"; } + let value = this.value || this.get_model_value(); + value = this.get_parsed_value(value); + if ( status === "Read" && - is_null(this.value) && + is_null(value) && !in_list(["HTML", "Image", "Button"], this.df.fieldtype) ) status = "None"; @@ -93,9 +96,12 @@ frappe.ui.form.Control = class BaseControl { } } + let value = frappe.model.get_value(this.doctype, this.docname, this.df.fieldname); + value = this.get_parsed_value(value); + // hide if no value if (this.doctype && status==="Read" && !this.only_input - && is_null(frappe.model.get_value(this.doctype, this.docname, this.df.fieldname)) + && is_null(value) && !in_list(["HTML", "Image", "Button"], this.df.fieldtype)) { // eslint-disable-next-line @@ -159,14 +165,18 @@ frappe.ui.form.Control = class BaseControl { return this.doc[this.df.fieldname]; } } + get_parsed_value(value) { + if (this.parse) { + value = this.parse(value); + } + return value; + } set_value(value, force_set_value=false) { return this.validate_and_set_in_model(value, null, force_set_value); } parse_validate_and_set_in_model(value, e) { - if(this.parse) { - value = this.parse(value); - } + value = this.get_parsed_value(value); return this.validate_and_set_in_model(value, e); } validate_and_set_in_model(value, e, force_set_value=false) { diff --git a/frappe/public/js/frappe/form/controls/date.js b/frappe/public/js/frappe/form/controls/date.js index a8b82604c9..09517e742b 100644 --- a/frappe/public/js/frappe/form/controls/date.js +++ b/frappe/public/js/frappe/form/controls/date.js @@ -140,6 +140,9 @@ frappe.ui.form.ControlDate = class ControlDate extends frappe.ui.form.ControlDat } parse(value) { if (value) { + if (value == "Invalid date") { + return ""; + } return frappe.datetime.user_to_str(value, false, true); } } diff --git a/frappe/public/js/frappe/form/controls/datetime.js b/frappe/public/js/frappe/form/controls/datetime.js index 9b10465d7b..11dff265f2 100644 --- a/frappe/public/js/frappe/form/controls/datetime.js +++ b/frappe/public/js/frappe/form/controls/datetime.js @@ -40,6 +40,9 @@ frappe.ui.form.ControlDatetime = class ControlDatetime extends frappe.ui.form.Co value = frappe.datetime.convert_to_system_tz(value, true); } + if (value == "Invalid date") { + value = ""; + } return value; } } diff --git a/frappe/public/js/frappe/form/controls/duration.js b/frappe/public/js/frappe/form/controls/duration.js index 361d10982e..35c691a0ca 100644 --- a/frappe/public/js/frappe/form/controls/duration.js +++ b/frappe/public/js/frappe/form/controls/duration.js @@ -109,6 +109,12 @@ frappe.ui.form.ControlDuration = class ControlDuration extends frappe.ui.form.Co return cint(this.value); } + parse(value) { + if (!value) { + return ""; + } + } + refresh_input() { super.refresh_input(); this.set_duration_options(); diff --git a/frappe/public/js/frappe/form/controls/time.js b/frappe/public/js/frappe/form/controls/time.js index f7fcc4c618..bbdec69cf8 100644 --- a/frappe/public/js/frappe/form/controls/time.js +++ b/frappe/public/js/frappe/form/controls/time.js @@ -82,6 +82,9 @@ frappe.ui.form.ControlTime = class ControlTime extends frappe.ui.form.ControlDat } parse(value) { if (value) { + if (value == "Invalid date") { + value = ""; + } return frappe.datetime.user_to_str(value, true); } } From 60ec324956164eaa95709dde48226a19b2b4035b Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 14 Jun 2022 16:40:41 +0530 Subject: [PATCH 071/205] fix: Remove unwanted blacklist over fields A field like 'count(`tabBOM Update Log`.name) as total_count' split by whitespace loses it's meaning. Tried splitting it meaningfully but didn't get the point of this tbh and stopped. I'm not sure what the code before was trying to do and with what set of inputs. Imagine the following fields: [count(`tabBOM Update Log`.name) as total_count, `tabBOM Update Log`.name as update_name, `tabBOM Update Log`.name, `tabBOM Update as Log`.name, tabBOM.name, name], I couldn't see what the previous check was trying to protect - hence, didn't add any equivalent functionality. --- frappe/model/db_query.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py index fe52818235..91c70f5b97 100644 --- a/frappe/model/db_query.py +++ b/frappe/model/db_query.py @@ -378,9 +378,6 @@ class DatabaseQuery(object): for field in self.fields: if sub_query_regex.match(field): - if any(keyword in field.lower().split() for keyword in blacklisted_keywords): - _raise_exception() - if any(f"({keyword}" in field.lower() for keyword in blacklisted_keywords): _raise_exception() From 678eebe4fb8531413b88865f3e1bea162758910e Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 14 Jun 2022 17:16:34 +0530 Subject: [PATCH 072/205] perf: Limit re internal cache to avoid caching patterns twice --- frappe/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/frappe/__init__.py b/frappe/__init__.py index 7c71bc8815..c4f4b2a690 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -15,6 +15,7 @@ import importlib import inspect import json import os +import re import warnings from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union @@ -45,6 +46,10 @@ STANDARD_USERS = ("Guest", "Administrator") _dev_server = int(sbool(os.environ.get("DEV_SERVER", False))) _qb_patched = False +re._MAXCACHE = ( + 50 # reduced from default 512 given we are already maintaining this on parent worker +) + if _dev_server: warnings.simplefilter("always", DeprecationWarning) From 9b4db43b84ead21a135fa0ce29e810306c01f9b5 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 14 Jun 2022 17:17:22 +0530 Subject: [PATCH 073/205] perf(db_query): Maintain compiled pattern globally --- frappe/model/db_query.py | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py index 91c70f5b97..82913db98d 100644 --- a/frappe/model/db_query.py +++ b/frappe/model/db_query.py @@ -40,6 +40,16 @@ CAST_VARCHAR_PATTERN = re.compile( r"([`\"]?tab[\w`\" -]+\.[`\"]?name[`\"]?)(?!\w)", flags=re.IGNORECASE ) ORDER_BY_PATTERN = re.compile(r"\ order\ by\ |\ asc|\ ASC|\ desc|\ DESC", flags=re.IGNORECASE) +SUB_QUERY_PATTERN = re.compile("^.*[,();@].*") +IS_QUERY_PATTERN = re.compile(r"^(select|delete|update|drop|create)\s") +IS_QUERY_PREDICATE_PATTERN = re.compile( + r"\s*[0-9a-zA-z]*\s*( from | group by | order by | where | join )" +) +FIELD_QUOTE_PATTERN = re.compile(r"[0-9a-zA-Z]+\s*'") +FIELD_COMMA_PATTERN = re.compile(r"[0-9a-zA-Z]+\s*,") +STRICT_FIELD_PATTERN = re.compile(r".*/\*.*") +STRICT_UNION_PATTERN = re.compile(r".*\s(union).*\s") +ORDER_GROUP_PATTERN = re.compile(r".*[^a-z0-9-_ ,`'\"\.\(\)].*") class DatabaseQuery(object): @@ -343,8 +353,6 @@ class DatabaseQuery(object): As field contains `,` and mysql function `version()`, with the help of regex the system will filter out this field. """ - - sub_query_regex = re.compile("^.*[,();@].*") blacklisted_keywords = ["select", "create", "insert", "delete", "drop", "update", "case", "show"] blacklisted_functions = [ "concat", @@ -368,16 +376,14 @@ class DatabaseQuery(object): frappe.throw(_("Use of sub-query or function is restricted"), frappe.DataError) def _is_query(field): - if re.compile(r"^(select|delete|update|drop|create)\s").match(field): + if IS_QUERY_PATTERN.match(field): _raise_exception() - elif re.compile(r"\s*[0-9a-zA-z]*\s*( from | group by | order by | where | join )").match( - field - ): + elif IS_QUERY_PREDICATE_PATTERN.match(field): _raise_exception() for field in self.fields: - if sub_query_regex.match(field): + if SUB_QUERY_PATTERN.match(field): if any(f"({keyword}" in field.lower() for keyword in blacklisted_keywords): _raise_exception() @@ -388,19 +394,19 @@ class DatabaseQuery(object): # prevent access to global variables _raise_exception() - if re.compile(r"[0-9a-zA-Z]+\s*'").match(field): + if FIELD_QUOTE_PATTERN.match(field): _raise_exception() - if re.compile(r"[0-9a-zA-Z]+\s*,").match(field): + if FIELD_COMMA_PATTERN.match(field): _raise_exception() _is_query(field) if self.strict: - if re.compile(r".*/\*.*").match(field): + if STRICT_FIELD_PATTERN.match(field): frappe.throw(_("Illegal SQL Query")) - if re.compile(r".*\s(union).*\s").match(field.lower()): + if STRICT_UNION_PATTERN.match(field.lower()): frappe.throw(_("Illegal SQL Query")) def extract_tables(self): @@ -907,7 +913,7 @@ class DatabaseQuery(object): if "select" in _lower and "from" in _lower: frappe.throw(_("Cannot use sub-query in order by")) - if re.compile(r".*[^a-z0-9-_ ,`'\"\.\(\)].*").match(_lower): + if ORDER_GROUP_PATTERN.match(_lower): frappe.throw(_("Illegal SQL Query")) for field in parameters.split(","): From ec2aadbf5a685591dd8f90bb8812d34fc4167c66 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Tue, 14 Jun 2022 18:15:48 +0530 Subject: [PATCH 074/205] chore: typo --- frappe/public/js/frappe/ui/toolbar/about.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/ui/toolbar/about.js b/frappe/public/js/frappe/ui/toolbar/about.js index 47dc7ee851..464c7c4787 100644 --- a/frappe/public/js/frappe/ui/toolbar/about.js +++ b/frappe/public/js/frappe/ui/toolbar/about.js @@ -19,7 +19,7 @@ frappe.ui.misc.about = function() {

Installed Apps

\
Loading versions...
\
\ -

© Frappe Technologies Pvt. Ltd and contributors

\ +

© Frappe Technologies Pvt. Ltd. and contributors

\ ", frappe.app)); frappe.ui.misc.about_dialog = d; From e5bc96d4d80286a9eda34b97d649b29d1f3a5a66 Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Tue, 14 Jun 2022 18:19:11 +0530 Subject: [PATCH 075/205] fix: added validation in parse method for table multiselect and multiselect pills --- frappe/public/js/frappe/form/controls/multiselect_pills.js | 4 ++++ frappe/public/js/frappe/form/controls/table_multiselect.js | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/frappe/public/js/frappe/form/controls/multiselect_pills.js b/frappe/public/js/frappe/form/controls/multiselect_pills.js index bf93ac0dd8..3f3e204039 100644 --- a/frappe/public/js/frappe/form/controls/multiselect_pills.js +++ b/frappe/public/js/frappe/form/controls/multiselect_pills.js @@ -38,6 +38,10 @@ frappe.ui.form.ControlMultiSelectPills = class ControlMultiSelectPills extends f } parse(value) { + if (typeof value == "object" || !this.rows) { + return value; + } + if (value) { this.rows.push(value); } diff --git a/frappe/public/js/frappe/form/controls/table_multiselect.js b/frappe/public/js/frappe/form/controls/table_multiselect.js index e106d8eed6..14bc0e297c 100644 --- a/frappe/public/js/frappe/form/controls/table_multiselect.js +++ b/frappe/public/js/frappe/form/controls/table_multiselect.js @@ -50,6 +50,10 @@ frappe.ui.form.ControlTableMultiSelect = class ControlTableMultiSelect extends f this.$input_area.find('.link-btn').remove(); } parse(value, label) { + if (typeof value == "object" || !this.rows) { + return value; + } + const link_field = this.get_link_field(); if (value) { From cb28af838e63dfb38761b9f593365ec57e291978 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 14 Jun 2022 18:15:12 +0530 Subject: [PATCH 076/205] fix: Add temporary backwards compatible setup.py setup.py is maintained so that there is no hard dependency on bench v5.11.0 and people get enough time to update their CLI. Hope the TODO adds a subtle reminder for the future ;) --- setup.py | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 setup.py diff --git a/setup.py b/setup.py new file mode 100644 index 0000000000..7a90eed81a --- /dev/null +++ b/setup.py @@ -0,0 +1,6 @@ +# TODO: Remove this file when bench >=v5.11.0 is adopted / v15.0.0 is released +from setuptools import setup + +name = "frappe" + +setup() From 4bc544bb0b32085abf5d08ee90e0a5f0b8d0a1c2 Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Tue, 14 Jun 2022 18:40:36 +0530 Subject: [PATCH 077/205] fix: validation in date range while parsing --- frappe/public/js/frappe/form/controls/date_range.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/form/controls/date_range.js b/frappe/public/js/frappe/form/controls/date_range.js index 170404f575..02cac1cf28 100644 --- a/frappe/public/js/frappe/form/controls/date_range.js +++ b/frappe/public/js/frappe/form/controls/date_range.js @@ -41,9 +41,10 @@ frappe.ui.form.ControlDateRange = class ControlDateRange extends frappe.ui.form. this.set_mandatory && this.set_mandatory(value); } parse(value) { + if (!value || (value && !value.includes('to'))) return value; // replace the separator (which can be in user language) with comma const to = __('{0} to {1}').replace('{0}', '').replace('{1}', ''); - value = value.replace(to, ','); + value = value && value.replace(to, ','); if(value && value.includes(',')) { var vals = value.split(','); From c77f36600a7d35ca0adad7f44dd070124761b225 Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Tue, 14 Jun 2022 18:58:28 +0530 Subject: [PATCH 078/205] chore: code cleanup --- frappe/public/js/frappe/form/controls/datetime.js | 2 +- frappe/public/js/frappe/form/controls/duration.js | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/frappe/public/js/frappe/form/controls/datetime.js b/frappe/public/js/frappe/form/controls/datetime.js index 11dff265f2..43873b3b1e 100644 --- a/frappe/public/js/frappe/form/controls/datetime.js +++ b/frappe/public/js/frappe/form/controls/datetime.js @@ -43,8 +43,8 @@ frappe.ui.form.ControlDatetime = class ControlDatetime extends frappe.ui.form.Co if (value == "Invalid date") { value = ""; } - return value; } + return value; } format_for_input(value) { if (!value) return ""; diff --git a/frappe/public/js/frappe/form/controls/duration.js b/frappe/public/js/frappe/form/controls/duration.js index 35c691a0ca..940ad9d58a 100644 --- a/frappe/public/js/frappe/form/controls/duration.js +++ b/frappe/public/js/frappe/form/controls/duration.js @@ -110,9 +110,7 @@ frappe.ui.form.ControlDuration = class ControlDuration extends frappe.ui.form.Co } parse(value) { - if (!value) { - return ""; - } + return !value ? "" : value; } refresh_input() { From 562499609c27c1b42a50b4197eef0f789ab3ecb2 Mon Sep 17 00:00:00 2001 From: Rushabh Mehta Date: Tue, 14 Jun 2022 19:00:01 +0530 Subject: [PATCH 079/205] fix(minor): refresh tabs with sections (#17182) --- frappe/public/js/frappe/form/layout.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frappe/public/js/frappe/form/layout.js b/frappe/public/js/frappe/form/layout.js index 6d000c99f9..080f8e0180 100644 --- a/frappe/public/js/frappe/form/layout.js +++ b/frappe/public/js/frappe/form/layout.js @@ -293,9 +293,6 @@ frappe.ui.form.Layout = class Layout { // refresh sections this.refresh_sections(); - // refresh tabs - this.is_tabbed_layout() && this.refresh_tabs(); - if (this.frm) { // collapse sections this.refresh_section_collapse(); @@ -325,6 +322,9 @@ frappe.ui.form.Layout = class Layout { section.addClass("empty-section"); } }); + + // refresh tabs + this.is_tabbed_layout() && this.refresh_tabs(); } refresh_tabs() { From bbc90e6578097ede9bff42849150a8fdc8a4bb03 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 15 Jun 2022 10:33:06 +0530 Subject: [PATCH 080/205] refactor!: frappe.db.get_singles_dict * Don't cast values by default - only if cast kwarg is set * Reverts breaking behaviour added via https://github.com/frappe/frappe/commit/f74dc5023d5ab1598e80a586b656b34e18a5ec0c --- frappe/database/database.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index 39ca846904..44256f58b0 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -619,23 +619,19 @@ class Database(object): else: return r and [[i[1] for i in r]] or [] - def get_singles_dict(self, doctype, debug=False, *, for_update=False): + def get_singles_dict(self, doctype, debug=False, *, for_update=False, cast=False): """Get Single DocType as dict. :param doctype: DocType of the single object whose value is requested + :param debug: Execute query in debug mode - print to STDOUT + :param for_update: Take `FOR UPDATE` lock on the records + :param cast: Cast values to Python data types based on field type Example: # Get coulmn and value of the single doctype Accounts Settings account_settings = frappe.db.get_singles_dict("Accounts Settings") """ - return_value = frappe._dict() - - try: - meta = frappe.get_meta(doctype) - except DoesNotExistError: - return return_value - queried_result = self.query.get_sql( "Singles", filters={"doctype": doctype}, @@ -643,6 +639,16 @@ class Database(object): for_update=for_update, ).run(debug=debug) + if not cast: + return frappe._dict(queried_result) + + try: + meta = frappe.get_meta(doctype) + except DoesNotExistError: + return frappe._dict(queried_result) + + return_value = frappe._dict() + for fieldname, value in queried_result: if df := meta.get_field(fieldname): casted_value = cast(df.fieldtype, value) From 80b19a6031f94a83961dbdf15fb58a747fd85b23 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 15 Jun 2022 10:40:26 +0530 Subject: [PATCH 081/205] fix: Cast singles_dict' values This is to adapt with the changes made in frappe.db.get_singles_dict in the previous commit --- frappe/__init__.py | 4 ++-- frappe/integrations/doctype/google_drive/google_drive.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/frappe/__init__.py b/frappe/__init__.py index c4f4b2a690..054643903d 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -2221,14 +2221,14 @@ def safe_eval(code, eval_globals=None, eval_locals=None): def get_website_settings(key): if not hasattr(local, "website_settings"): - local.website_settings = db.get_singles_dict("Website Settings") + local.website_settings = db.get_singles_dict("Website Settings", cast=True) return local.website_settings[key] def get_system_settings(key): if not hasattr(local, "system_settings"): - local.system_settings = db.get_singles_dict("System Settings") + local.system_settings = db.get_singles_dict("System Settings", cast=True) return local.system_settings[key] diff --git a/frappe/integrations/doctype/google_drive/google_drive.py b/frappe/integrations/doctype/google_drive/google_drive.py index bbb1e8485e..347488ee44 100644 --- a/frappe/integrations/doctype/google_drive/google_drive.py +++ b/frappe/integrations/doctype/google_drive/google_drive.py @@ -259,13 +259,13 @@ def upload_system_backup_to_google_drive(): def daily_backup(): - drive_settings = frappe.db.get_singles_dict("Google Drive") + drive_settings = frappe.db.get_singles_dict("Google Drive", cast=True) if drive_settings.enable and drive_settings.frequency == "Daily": upload_system_backup_to_google_drive() def weekly_backup(): - drive_settings = frappe.db.get_singles_dict("Google Drive") + drive_settings = frappe.db.get_singles_dict("Google Drive", cast=True) if drive_settings.enable and drive_settings.frequency == "Weekly": upload_system_backup_to_google_drive() From 57bd1b6031980085c3bdca9a66586471b2a82715 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 15 Jun 2022 10:54:48 +0530 Subject: [PATCH 082/205] fix(db): Import cast as cast_fieldtype to manevour ambiguity --- frappe/database/database.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index 44256f58b0..c68368ba38 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -22,7 +22,8 @@ from frappe.exceptions import DoesNotExistError from frappe.model.utils.link_count import flush_local_link_count from frappe.query_builder.functions import Count from frappe.query_builder.utils import DocType -from frappe.utils import cast, get_datetime, get_table_name, getdate, now, sbool +from frappe.utils import cast as cast_fieldtype +from frappe.utils import get_datetime, get_table_name, getdate, now, sbool IFNULL_PATTERN = re.compile(r"ifnull\(", flags=re.IGNORECASE) INDEX_PATTERN = re.compile(r"\s*\([^)]+\)\s*") @@ -651,7 +652,7 @@ class Database(object): for fieldname, value in queried_result: if df := meta.get_field(fieldname): - casted_value = cast(df.fieldtype, value) + casted_value = cast_fieldtype(df.fieldtype, value) else: casted_value = value return_value[fieldname] = casted_value @@ -719,7 +720,7 @@ class Database(object): _("Invalid field name: {0}").format(frappe.bold(fieldname)), self.InvalidColumnName ) - val = cast(df.fieldtype, val) + val = cast_fieldtype(df.fieldtype, val) self.value_cache[doctype][fieldname] = val From ef1200d94b235fd2dd6e58e6177f229354bcf88e Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sat, 11 Jun 2022 11:42:16 +0530 Subject: [PATCH 083/205] fix!: dont auto set old logs as "seen" Not sure why this is ever required, also not configurable. --- frappe/core/doctype/error_log/error_log.py | 8 -------- frappe/hooks.py | 1 - 2 files changed, 9 deletions(-) diff --git a/frappe/core/doctype/error_log/error_log.py b/frappe/core/doctype/error_log/error_log.py index d93029179c..569e6d047e 100644 --- a/frappe/core/doctype/error_log/error_log.py +++ b/frappe/core/doctype/error_log/error_log.py @@ -13,14 +13,6 @@ class ErrorLog(Document): frappe.db.commit() -def set_old_logs_as_seen(): - # set logs as seen - frappe.db.sql( - """UPDATE `tabError Log` SET `seen`=1 - WHERE `seen`=0 AND `creation` < (NOW() - INTERVAL '7' DAY)""" - ) - - @frappe.whitelist() def clear_error_logs(): """Flush all Error Logs""" diff --git a/frappe/hooks.py b/frappe/hooks.py index ca84b83663..ae2abcec68 100644 --- a/frappe/hooks.py +++ b/frappe/hooks.py @@ -219,7 +219,6 @@ scheduler_events = { "daily": [ "frappe.email.queue.set_expiry_for_email_queue", "frappe.desk.notifications.clear_notifications", - "frappe.core.doctype.error_log.error_log.set_old_logs_as_seen", "frappe.desk.doctype.event.event.send_event_digest", "frappe.sessions.clear_expired_sessions", "frappe.email.doctype.notification.notification.trigger_daily_alerts", From ea416f9d6bd5312e48ca030ad915f01ad8434d40 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sat, 11 Jun 2022 12:26:09 +0530 Subject: [PATCH 084/205] feat: log settings with "interface" We have hardcoded "Log settings" to only apply on 3 doctypes, there are few more logging doctypes in core which are not cleared right now, on top of that it's not easy for user to configure all logging behaviour from one place. This change adds a table on log settings where logging doctypes that support the interface required by log settings can auto-register and show up in settings. Currently only supported configuration is "number of days" to keep. --- frappe/__init__.py | 12 +- .../core/doctype/activity_log/activity_log.py | 16 +-- frappe/core/doctype/error_log/error_log.py | 7 + .../core/doctype/log_settings/log_settings.js | 16 ++- .../doctype/log_settings/log_settings.json | 56 ++------ .../core/doctype/log_settings/log_settings.py | 122 ++++++++++++------ .../doctype/log_settings/test_log_settings.py | 19 ++- frappe/core/doctype/logs_to_clear/__init__.py | 0 .../doctype/logs_to_clear/logs_to_clear.json | 42 ++++++ .../doctype/logs_to_clear/logs_to_clear.py | 9 ++ .../email/doctype/email_queue/email_queue.py | 28 +++- .../doctype/email_queue/test_email_queue.py | 5 +- frappe/email/queue.py | 25 ---- frappe/patches.txt | 1 + .../patches/v14_0/log_settings_migration.py | 26 ++++ 15 files changed, 255 insertions(+), 129 deletions(-) create mode 100644 frappe/core/doctype/logs_to_clear/__init__.py create mode 100644 frappe/core/doctype/logs_to_clear/logs_to_clear.json create mode 100644 frappe/core/doctype/logs_to_clear/logs_to_clear.py create mode 100644 frappe/patches/v14_0/log_settings_migration.py diff --git a/frappe/__init__.py b/frappe/__init__.py index 054643903d..4b64732245 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -17,7 +17,7 @@ import json import os import re import warnings -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union import click from werkzeug.local import Local, release_local @@ -1551,7 +1551,15 @@ def call(fn, *args, **kwargs): return fn(*args, **newargs) -def get_newargs(fn, kwargs): +def get_newargs(fn: Callable, kwargs: Dict[str, Any]) -> Dict[str, Any]: + """Remove any kwargs that are not supported by the function. + + Example: + >>> def fn(a=1, b=2): pass + + >>> get_newargs(fn, {"a": 2, "c": 1}) + {"a": 2} + """ # if function has any **kwargs parameter that capture arbitrary keyword arguments # Ref: https://docs.python.org/3/library/inspect.html#inspect.Parameter.kind diff --git a/frappe/core/doctype/activity_log/activity_log.py b/frappe/core/doctype/activity_log/activity_log.py index 61dedd7bc0..468b7f4473 100644 --- a/frappe/core/doctype/activity_log/activity_log.py +++ b/frappe/core/doctype/activity_log/activity_log.py @@ -25,6 +25,13 @@ class ActivityLog(Document): if self.reference_doctype and self.reference_name: self.status = "Linked" + @staticmethod + def clear_old_logs(days=None): + if not days: + days = 90 + doctype = DocType("Activity Log") + frappe.db.delete(doctype, filters=(doctype.modified < (Now() - Interval(days=days)))) + def on_doctype_update(): """Add indexes in `tabActivity Log`""" @@ -43,12 +50,3 @@ def add_authentication_log(subject, user, operation="Login", status="Success"): "operation": operation, } ).insert(ignore_permissions=True, ignore_links=True) - - -def clear_activity_logs(days=None): - """clear 90 day old authentication logs or configured in log settings""" - - if not days: - days = 90 - doctype = DocType("Activity Log") - frappe.db.delete(doctype, filters=(doctype.creation < (Now() - Interval(days=days)))) diff --git a/frappe/core/doctype/error_log/error_log.py b/frappe/core/doctype/error_log/error_log.py index 569e6d047e..57c519bd2e 100644 --- a/frappe/core/doctype/error_log/error_log.py +++ b/frappe/core/doctype/error_log/error_log.py @@ -4,6 +4,8 @@ import frappe from frappe.model.document import Document +from frappe.query_builder import Interval +from frappe.query_builder.functions import Now class ErrorLog(Document): @@ -12,6 +14,11 @@ class ErrorLog(Document): self.db_set("seen", 1, update_modified=0) frappe.db.commit() + @staticmethod + def clear_old_logs(days=30): + table = frappe.qb.DocType("Error Log") + frappe.db.delete(table, filters=(table.creation < (Now() - Interval(days=days)))) + @frappe.whitelist() def clear_error_logs(): diff --git a/frappe/core/doctype/log_settings/log_settings.js b/frappe/core/doctype/log_settings/log_settings.js index 09a2086a1d..dc7cc7eac2 100644 --- a/frappe/core/doctype/log_settings/log_settings.js +++ b/frappe/core/doctype/log_settings/log_settings.js @@ -1,8 +1,16 @@ // Copyright (c) 2020, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Log Settings', { - // refresh: function(frm) { - - // } +frappe.ui.form.on("Log Settings", { + refresh: (frm) => { + frm.set_query("ref_doctype", "logs_to_clear", () => { + const added_doctypes = frm.doc.logs_to_clear.map((r) => r.ref_doctype); + return { + query: "frappe.core.doctype.log_settings.log_settings.get_log_doctypes", + filters: [ + ["name", "not in", added_doctypes], + ], + }; + }); + }, }); diff --git a/frappe/core/doctype/log_settings/log_settings.json b/frappe/core/doctype/log_settings/log_settings.json index f06d14f16b..5a9dd159cc 100644 --- a/frappe/core/doctype/log_settings/log_settings.json +++ b/frappe/core/doctype/log_settings/log_settings.json @@ -5,61 +5,20 @@ "editable_grid": 1, "engine": "InnoDB", "field_order": [ - "error_log_notification_section", - "users_to_notify", - "log_cleanup_section", - "clear_error_log_after", - "clear_activity_log_after", - "column_break_4", - "clear_email_queue_after" + "logs_to_clear" ], "fields": [ { - "fieldname": "log_cleanup_section", - "fieldtype": "Section Break", - "label": "Log Cleanup" - }, - { - "fieldname": "column_break_4", - "fieldtype": "Column Break" - }, - { - "fieldname": "error_log_notification_section", - "fieldtype": "Section Break", - "label": "Error Log Notification" - }, - { - "fieldname": "users_to_notify", - "fieldtype": "Table MultiSelect", - "label": "Users To Notify", - "options": "Log Setting User" - }, - { - "default": "90", - "description": "In Days", - "fieldname": "clear_error_log_after", - "fieldtype": "Int", - "label": "Clear Error log After" - }, - { - "default": "90", - "description": "In Days", - "fieldname": "clear_activity_log_after", - "fieldtype": "Int", - "label": "Clear Activity Log After" - }, - { - "default": "30", - "description": "In Days", - "fieldname": "clear_email_queue_after", - "fieldtype": "Int", - "label": "Clear Email Queue After" + "fieldname": "logs_to_clear", + "fieldtype": "Table", + "label": "Logs to Clear", + "options": "Logs To Clear" } ], "index_web_pages_for_search": 1, "issingle": 1, "links": [], - "modified": "2020-10-13 12:18:48.649038", + "modified": "2022-06-11 02:17:30.803721", "modified_by": "Administrator", "module": "Core", "name": "Log Settings", @@ -79,5 +38,6 @@ "quick_entry": 1, "sort_field": "modified", "sort_order": "DESC", + "states": [], "track_changes": 1 -} +} \ No newline at end of file diff --git a/frappe/core/doctype/log_settings/log_settings.py b/frappe/core/doctype/log_settings/log_settings.py index 5632f05a36..4f8ea549b9 100644 --- a/frappe/core/doctype/log_settings/log_settings.py +++ b/frappe/core/doctype/log_settings/log_settings.py @@ -2,49 +2,88 @@ # Copyright (c) 2020, Frappe Technologies and contributors # License: MIT. See LICENSE +from typing import Protocol, runtime_checkable + import frappe from frappe import _ +from frappe.model.base_document import get_controller from frappe.model.document import Document -from frappe.query_builder import DocType, Interval -from frappe.query_builder.functions import Now +from frappe.utils import cint +from frappe.utils.caching import site_cache + + +@runtime_checkable +class LogType(Protocol): + """Interface requirement for doctypes that can be cleared using log settings.""" + + @staticmethod + def clear_old_logs(days: int) -> None: + ... + + +@site_cache +def _supports_log_clearing(doctype: str) -> bool: + try: + controller = get_controller(doctype) + return issubclass(controller, LogType) + except Exception: + return False class LogSettings(Document): - def clear_logs(self, commit=False): - self.clear_email_queue() - if commit: - # Since since deleting many logs can take significant amount of time, commit is required to relase locks. - # Error log table doesn't require commit - myisam - # activity logs are deleted last so background job finishes and commits. + def validate(self): + self.validate_supported_doctypes() + self.validate_duplicates() + + def validate_supported_doctypes(self): + for entry in self.logs_to_clear: + if _supports_log_clearing(entry.ref_doctype): + continue + + msg = _("{} does not support automated log clearing.").format(frappe.bold(entry.ref_doctype)) + if frappe.conf.developer_mode: + msg += "
" + _("Implement `clear_old_logs` method to enable auto error clearing.") + frappe.throw(msg, title=_("DocType not supported by Log Settings.")) + + def validate_duplicates(self): + seen = set() + for entry in self.logs_to_clear: + if entry.ref_doctype in seen: + frappe.throw( + _("{} appears more than once in configured log doctypes.").format(entry.ref_doctype) + ) + seen.add(entry.ref_doctype) + + def clear_logs(self): + """ + Log settings can clear any log type that's registered to it and provides a method to delete old logs. + + Check `LogDoctype` above for interface that doctypes need to implement. + """ + + for entry in self.logs_to_clear: + controller: LogType = get_controller(entry.ref_doctype) + func = controller.clear_old_logs + + # Only pass what the method can handle, this is considering any + # future addition that might happen to the required interface. + kwargs = frappe.get_newargs(func, {"days": entry.days}) + func(**kwargs) frappe.db.commit() - self.clear_error_logs() - self.clear_activity_logs() - def clear_error_logs(self): - table = DocType("Error Log") - frappe.db.delete( - table, filters=(table.creation < (Now() - Interval(days=self.clear_error_log_after))) - ) - - def clear_activity_logs(self): - from frappe.core.doctype.activity_log.activity_log import clear_activity_logs - - clear_activity_logs(days=self.clear_activity_log_after) - - def clear_email_queue(self): - from frappe.email.queue import clear_outbox - - clear_outbox(days=self.clear_email_queue_after) + def register_doctype(self, doctype: str, days=30): + if doctype not in {d.ref_doctype for d in self.logs_to_clear}: + self.append("logs_to_clear", {"ref_doctype": doctype, "days": cint(days)}) def run_log_clean_up(): doc = frappe.get_doc("Log Settings") - doc.clear_logs(commit=True) + doc.clear_logs() @frappe.whitelist() -def has_unseen_error_log(user): - def _get_response(show_alert=True): +def has_unseen_error_log(): + if frappe.get_all("Error Log", filters={"seen": 0}, limit=1): return { "show_alert": True, "message": _("You have unseen {0}").format( @@ -52,13 +91,22 @@ def has_unseen_error_log(user): ), } - if frappe.get_all("Error Log", filters={"seen": 0}, limit=1): - log_settings = frappe.get_cached_doc("Log Settings") - if log_settings.users_to_notify: - if user in [u.user for u in log_settings.users_to_notify]: - return _get_response() - else: - return _get_response(show_alert=False) - else: - return _get_response() +@frappe.whitelist() +@frappe.validate_and_sanitize_search_inputs +def get_log_doctypes(doctype, txt, searchfield, start, page_len, filters): + + filters = filters or {} + + filters.extend( + [ + ["istable", "=", 0], + ["issingle", "=", 0], + ["name", "like", f"%%{txt}%%"], + ] + ) + doctypes = frappe.get_list("DocType", filters=filters, pluck="name") + + supported_doctypes = [(d,) for d in doctypes if _supports_log_clearing(d)] + + return supported_doctypes[start:page_len] diff --git a/frappe/core/doctype/log_settings/test_log_settings.py b/frappe/core/doctype/log_settings/test_log_settings.py index 1b78745103..d7f43a181d 100644 --- a/frappe/core/doctype/log_settings/test_log_settings.py +++ b/frappe/core/doctype/log_settings/test_log_settings.py @@ -4,7 +4,7 @@ from datetime import datetime import frappe -from frappe.core.doctype.log_settings.log_settings import run_log_clean_up +from frappe.core.doctype.log_settings.log_settings import _supports_log_clearing, run_log_clean_up from frappe.tests.utils import FrappeTestCase from frappe.utils import add_to_date, now_datetime @@ -56,6 +56,23 @@ class TestLogSettings(FrappeTestCase): self.assertEqual(error_log_count, 0) self.assertEqual(email_queue_count, 0) + def test_logtype_identification(self): + supported_types = [ + "Error Log", + "Activity Log", + "Email Queue", + "Route History", + "Error Snapshot", + "Scheduled Job Log", + ] + + for lt in supported_types: + self.assertTrue(_supports_log_clearing(lt), f"{lt} should be recognized as log type") + + unsupported_types = ["DocType", "User", "Non Existing dt"] + for dt in unsupported_types: + self.assertFalse(_supports_log_clearing(dt), f"{dt} shouldn't be recognized as log type") + def setup_test_logs(past: datetime) -> None: activity_log = frappe.get_doc( diff --git a/frappe/core/doctype/logs_to_clear/__init__.py b/frappe/core/doctype/logs_to_clear/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/frappe/core/doctype/logs_to_clear/logs_to_clear.json b/frappe/core/doctype/logs_to_clear/logs_to_clear.json new file mode 100644 index 0000000000..212390adac --- /dev/null +++ b/frappe/core/doctype/logs_to_clear/logs_to_clear.json @@ -0,0 +1,42 @@ +{ + "actions": [], + "autoname": "autoincrement", + "creation": "2022-06-11 02:02:39.472511", + "doctype": "DocType", + "engine": "InnoDB", + "field_order": [ + "ref_doctype", + "days" + ], + "fields": [ + { + "fieldname": "ref_doctype", + "fieldtype": "Link", + "in_list_view": 1, + "label": "Log DocType", + "options": "DocType", + "reqd": 1 + }, + { + "fieldname": "days", + "fieldtype": "Int", + "in_list_view": 1, + "label": "Clear Logs After (days)", + "non_negative": 1, + "reqd": 1 + } + ], + "index_web_pages_for_search": 1, + "istable": 1, + "links": [], + "modified": "2022-06-11 03:20:47.721188", + "modified_by": "Administrator", + "module": "Core", + "name": "Logs To Clear", + "naming_rule": "Autoincrement", + "owner": "Administrator", + "permissions": [], + "sort_field": "modified", + "sort_order": "DESC", + "states": [] +} \ No newline at end of file diff --git a/frappe/core/doctype/logs_to_clear/logs_to_clear.py b/frappe/core/doctype/logs_to_clear/logs_to_clear.py new file mode 100644 index 0000000000..3fb4f8e72a --- /dev/null +++ b/frappe/core/doctype/logs_to_clear/logs_to_clear.py @@ -0,0 +1,9 @@ +# Copyright (c) 2022, Frappe Technologies and contributors +# For license information, please see license.txt + +# import frappe +from frappe.model.document import Document + + +class LogsToClear(Document): + pass diff --git a/frappe/email/doctype/email_queue/email_queue.py b/frappe/email/doctype/email_queue/email_queue.py index 662ba1b2ed..c3002607b4 100644 --- a/frappe/email/doctype/email_queue/email_queue.py +++ b/frappe/email/doctype/email_queue/email_queue.py @@ -18,7 +18,8 @@ from frappe.email.doctype.email_account.email_account import EmailAccount from frappe.email.email_body import add_attachment, get_email, get_formatted_html from frappe.email.queue import get_unsubcribed_url, get_unsubscribe_message from frappe.model.document import Document -from frappe.query_builder.utils import DocType +from frappe.query_builder import DocType, Interval +from frappe.query_builder.functions import Now from frappe.utils import ( add_days, cint, @@ -144,6 +145,31 @@ class EmailQueue(Document): if ctx.email_account_doc.append_emails_to_sent_folder and ctx.sent_to: ctx.email_account_doc.append_email_to_sent_folder(message) + @staticmethod + def clear_old_logs(days=30): + """Remove low priority older than 31 days in Outbox or configured in Log Settings. + Note: Used separate query to avoid deadlock + """ + days = days or 31 + email_queue = frappe.qb.DocType("Email Queue") + email_recipient = frappe.qb.DocType("Email Queue Recipient") + + # Delete queue table + ( + frappe.qb.from_(email_queue) + .delete() + .where((email_queue.modified < (Now() - Interval(days=days)))) + ).run() + + # delete child tables, note that this has potential to leave some orphan + # child table behind if modified time was later than parent doc (rare). + # But it's safe since child table doesn't contain links. + ( + frappe.qb.from_(email_recipient) + .delete() + .where((email_recipient.modified < (Now() - Interval(days=days)))) + ).run() + @task(queue="short") def send_mail(email_queue_name, is_background_task=False): diff --git a/frappe/email/doctype/email_queue/test_email_queue.py b/frappe/email/doctype/email_queue/test_email_queue.py index 96c566a041..435e4e691f 100644 --- a/frappe/email/doctype/email_queue/test_email_queue.py +++ b/frappe/email/doctype/email_queue/test_email_queue.py @@ -3,12 +3,13 @@ # License: MIT. See LICENSE import frappe -from frappe.email.queue import clear_outbox from frappe.tests.utils import FrappeTestCase class TestEmailQueue(FrappeTestCase): def test_email_queue_deletion_based_on_modified_date(self): + from frappe.email.doctype.email_queue.email_queue import EmailQueue + old_record = frappe.get_doc( { "doctype": "Email Queue", @@ -32,7 +33,7 @@ class TestEmailQueue(FrappeTestCase): new_record = frappe.copy_doc(old_record) new_record.insert() - clear_outbox() + EmailQueue.clear_old_logs() self.assertFalse(frappe.db.exists("Email Queue", old_record.name)) self.assertFalse(frappe.db.exists("Email Queue Recipient", {"parent": old_record.name})) diff --git a/frappe/email/queue.py b/frappe/email/queue.py index 07731417d8..1519c26841 100755 --- a/frappe/email/queue.py +++ b/frappe/email/queue.py @@ -190,31 +190,6 @@ def get_queue(): ) -def clear_outbox(days: int = None) -> None: - """Remove low priority older than 31 days in Outbox or configured in Log Settings. - Note: Used separate query to avoid deadlock - """ - days = days or 31 - email_queue = frappe.qb.DocType("Email Queue") - email_recipient = frappe.qb.DocType("Email Queue Recipient") - - # Delete queue table - ( - frappe.qb.from_(email_queue) - .delete() - .where((email_queue.modified < (Now() - Interval(days=days)))) - ).run() - - # delete child tables, note that this has potential to leave some orphan - # child table behind if modified time was later than parent doc (rare). - # But it's safe since child table doesn't contain links. - ( - frappe.qb.from_(email_recipient) - .delete() - .where((email_recipient.modified < (Now() - Interval(days=days)))) - ).run() - - def set_expiry_for_email_queue(): """Mark emails as expire that has not sent for 7 days. Called daily via scheduler. diff --git a/frappe/patches.txt b/frappe/patches.txt index b771485c3c..853f02ff4c 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -191,6 +191,7 @@ frappe.patches.v14_0.remove_post_and_post_comment frappe.patches.v14_0.reset_creation_datetime frappe.patches.v14_0.remove_is_first_startup frappe.patches.v14_0.reload_workspace_child_tables +frappe.patches.v14_0.log_settings_migration [post_model_sync] frappe.patches.v14_0.drop_data_import_legacy diff --git a/frappe/patches/v14_0/log_settings_migration.py b/frappe/patches/v14_0/log_settings_migration.py new file mode 100644 index 0000000000..87f6f6e082 --- /dev/null +++ b/frappe/patches/v14_0/log_settings_migration.py @@ -0,0 +1,26 @@ +import frappe + + +def execute(): + logging_doctypes = { + "Error Log": get_current_setting("clear_error_log_after") or 30, + "Activity Log": get_current_setting("clear_activity_log_after") or 90, + "Email Queue": get_current_setting("clear_email_queue_after") or 30, + } + + frappe.reload_doc("core", "doctype", "Logs To Clear") + frappe.reload_doc("core", "doctype", "Log Settings") + + log_settings = frappe.get_doc("Log Settings") + + for doctype, days in logging_doctypes.items(): + log_settings.register_doctype(doctype, days) + + log_settings.save() + + +def get_current_setting(fieldname): + try: + return frappe.db.get_single_value("Log Settings", fieldname) + except Exception: + pass From 95eb6cd0851bbad438f9cabf7957e167739c101e Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sat, 11 Jun 2022 14:14:45 +0530 Subject: [PATCH 085/205] feat: support log clearing for more doctypes - Route History - Error Snapshot - Scheduled Job Log (only completed ones) --- frappe/core/doctype/error_log/error_log.py | 2 +- frappe/core/doctype/error_snapshot/error_snapshot.py | 7 +++++++ frappe/core/doctype/log_settings/log_settings.py | 3 ++- frappe/core/doctype/logs_to_clear/logs_to_clear.json | 3 ++- .../core/doctype/scheduled_job_log/scheduled_job_log.py | 9 +++++++-- frappe/desk/doctype/route_history/route_history.py | 9 ++++++--- frappe/patches/v14_0/log_settings_migration.py | 3 +++ 7 files changed, 28 insertions(+), 8 deletions(-) diff --git a/frappe/core/doctype/error_log/error_log.py b/frappe/core/doctype/error_log/error_log.py index 57c519bd2e..224a5673a7 100644 --- a/frappe/core/doctype/error_log/error_log.py +++ b/frappe/core/doctype/error_log/error_log.py @@ -17,7 +17,7 @@ class ErrorLog(Document): @staticmethod def clear_old_logs(days=30): table = frappe.qb.DocType("Error Log") - frappe.db.delete(table, filters=(table.creation < (Now() - Interval(days=days)))) + frappe.db.delete(table, filters=(table.modified < (Now() - Interval(days=days)))) @frappe.whitelist() diff --git a/frappe/core/doctype/error_snapshot/error_snapshot.py b/frappe/core/doctype/error_snapshot/error_snapshot.py index 82f189217f..6e13b7a654 100644 --- a/frappe/core/doctype/error_snapshot/error_snapshot.py +++ b/frappe/core/doctype/error_snapshot/error_snapshot.py @@ -4,6 +4,8 @@ import frappe from frappe.model.document import Document +from frappe.query_builder import Interval +from frappe.query_builder.functions import Now class ErrorSnapshot(Document): @@ -32,3 +34,8 @@ class ErrorSnapshot(Document): frappe.db.set_value("Error Snapshot", parent["name"], "relapses", parent["relapses"] + 1) if parent["seen"]: frappe.db.set_value("Error Snapshot", parent["name"], "seen", False) + + @staticmethod + def clear_old_logs(days=30): + table = frappe.qb.DocType("Error Snapshot") + frappe.db.delete(table, filters=(table.modified < (Now() - Interval(days=days)))) diff --git a/frappe/core/doctype/log_settings/log_settings.py b/frappe/core/doctype/log_settings/log_settings.py index 4f8ea549b9..82d6afb385 100644 --- a/frappe/core/doctype/log_settings/log_settings.py +++ b/frappe/core/doctype/log_settings/log_settings.py @@ -72,7 +72,8 @@ class LogSettings(Document): frappe.db.commit() def register_doctype(self, doctype: str, days=30): - if doctype not in {d.ref_doctype for d in self.logs_to_clear}: + existing_logtypes = {d.ref_doctype for d in self.logs_to_clear} + if doctype not in existing_logtypes and _supports_log_clearing(doctype): self.append("logs_to_clear", {"ref_doctype": doctype, "days": cint(days)}) diff --git a/frappe/core/doctype/logs_to_clear/logs_to_clear.json b/frappe/core/doctype/logs_to_clear/logs_to_clear.json index 212390adac..5e242bc12f 100644 --- a/frappe/core/doctype/logs_to_clear/logs_to_clear.json +++ b/frappe/core/doctype/logs_to_clear/logs_to_clear.json @@ -18,6 +18,7 @@ "reqd": 1 }, { + "default": "30", "fieldname": "days", "fieldtype": "Int", "in_list_view": 1, @@ -29,7 +30,7 @@ "index_web_pages_for_search": 1, "istable": 1, "links": [], - "modified": "2022-06-11 03:20:47.721188", + "modified": "2022-06-11 04:35:54.706775", "modified_by": "Administrator", "module": "Core", "name": "Logs To Clear", diff --git a/frappe/core/doctype/scheduled_job_log/scheduled_job_log.py b/frappe/core/doctype/scheduled_job_log/scheduled_job_log.py index bead463ba5..68541a36a0 100644 --- a/frappe/core/doctype/scheduled_job_log/scheduled_job_log.py +++ b/frappe/core/doctype/scheduled_job_log/scheduled_job_log.py @@ -2,9 +2,14 @@ # Copyright (c) 2019, Frappe Technologies and contributors # License: MIT. See LICENSE -# import frappe +import frappe from frappe.model.document import Document +from frappe.query_builder import Interval +from frappe.query_builder.functions import Now class ScheduledJobLog(Document): - pass + @staticmethod + def clear_old_logs(days=90): + table = frappe.qb.DocType("Scheduled Job Log") + frappe.db.delete(table, filters=(table.modified < (Now() - Interval(days=days)))) diff --git a/frappe/desk/doctype/route_history/route_history.py b/frappe/desk/doctype/route_history/route_history.py index e712a5bb11..c62311ae02 100644 --- a/frappe/desk/doctype/route_history/route_history.py +++ b/frappe/desk/doctype/route_history/route_history.py @@ -4,12 +4,15 @@ import frappe from frappe.deferred_insert import deferred_insert as _deferred_insert from frappe.model.document import Document -from frappe.query_builder import DocType -from frappe.query_builder.functions import Count +from frappe.query_builder import DocType, Interval +from frappe.query_builder.functions import Count, Now class RouteHistory(Document): - pass + @staticmethod + def clear_old_logs(days=30): + table = frappe.qb.DocType("Route History") + frappe.db.delete(table, filters=(table.modified < (Now() - Interval(days=days)))) def flush_old_route_records(): diff --git a/frappe/patches/v14_0/log_settings_migration.py b/frappe/patches/v14_0/log_settings_migration.py index 87f6f6e082..b855cac355 100644 --- a/frappe/patches/v14_0/log_settings_migration.py +++ b/frappe/patches/v14_0/log_settings_migration.py @@ -6,6 +6,9 @@ def execute(): "Error Log": get_current_setting("clear_error_log_after") or 30, "Activity Log": get_current_setting("clear_activity_log_after") or 90, "Email Queue": get_current_setting("clear_email_queue_after") or 30, + "Route History": 90, + "Error Snapshot": 30, + "Scheduled Job Log": 90, } frappe.reload_doc("core", "doctype", "Logs To Clear") From 8b8552b0e49f1353dc937a86882ccb1030d3e065 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sat, 11 Jun 2022 16:04:43 +0530 Subject: [PATCH 086/205] feat: clear-log-table to clear large log tables --- frappe/commands/site.py | 76 +++++++++++++++++++++++++++++++++++ frappe/tests/test_commands.py | 14 +++++++ 2 files changed, 90 insertions(+) diff --git a/frappe/commands/site.py b/frappe/commands/site.py index 70b48e1f0d..2477de859f 100644 --- a/frappe/commands/site.py +++ b/frappe/commands/site.py @@ -1088,6 +1088,81 @@ def build_search_index(context): frappe.destroy() +LOG_DOCTYPES = [ + "Scheduled Job Log", + "Activity Log", + "Route History", + "Email Queue", + "Error Snapshot", + "Error Log", +] + + +@click.command("clear-log-table") +@click.option("--doctype", default="text", type=click.Choice(LOG_DOCTYPES), help="Log DocType") +@click.option("--days", type=int, help="Keep records for days") +@click.option("--no-backup", is_flag=True, default=False, help="Do not backup the table") +@pass_context +def clear_log_table(context, doctype, days, no_backup): + """If any logtype table grows too large then clearing it with DELETE query + is not feasible in reasonable time. This command copies recent data to new + table and replaces current table with new smaller table. + + + ref: https://mariadb.com/kb/en/big-deletes/#deleting-more-than-half-a-table + """ + from frappe.utils.backups import scheduled_backup + + if not context.sites: + raise SiteNotSpecifiedError + + if doctype not in LOG_DOCTYPES: + raise frappe.ValidationError(f"Unsupported logging DocType: {doctype}") + + for site in context.sites: + frappe.init(site=site) + frappe.connect() + + if frappe.db.db_type != "mariadb": + click.echo("Postgres database isn't supported by this command") + sys.exit(1) + + if not no_backup: + scheduled_backup( + ignore_conf=False, + include_doctypes=doctype, + ignore_files=True, + force=True, + ) + click.echo(f"Backed up {doctype}") + + original = f"`tab{doctype}`" + temporary = f"`tab{doctype} temp_table`" + backup = f"`tab{doctype} backup_table`" + + try: + frappe.db.sql(f"CREATE TABLE {temporary} LIKE {original}") + + click.echo(f"Copying {doctype} records from last {days} days to temporary table.") + # Copy all recent data to new table + frappe.db.sql( + f"""INSERT INTO {temporary} + SELECT * FROM {original} + WHERE {original}.`modified` > NOW() - INTERVAL '{days}' DAY""" + ) + frappe.db.sql(f"RENAME TABLE {original} TO {backup}, {temporary} TO {original}") + except Exception as e: + # Discard created tables + frappe.db.rollback() + frappe.db.sql(f"DROP TABLE IF EXISTS {temporary}") + click.echo(f"Log cleanup for {doctype} failed:\n{e}") + sys.exit(1) + else: + frappe.db.commit() + frappe.db.sql(f"DROP TABLE {backup}") + click.secho(f"Cleared {doctype} records older than {days} days", fg="green") + + @click.command("trim-database") @click.option("--dry-run", is_flag=True, default=False, help="Show what would be deleted") @click.option( @@ -1260,4 +1335,5 @@ commands = [ partial_restore, trim_tables, trim_database, + clear_log_table, ] diff --git a/frappe/tests/test_commands.py b/frappe/tests/test_commands.py index be968511a8..e2a94a24dd 100644 --- a/frappe/tests/test_commands.py +++ b/frappe/tests/test_commands.py @@ -27,6 +27,8 @@ import frappe.commands.site import frappe.commands.utils import frappe.recorder from frappe.installer import add_to_installed_apps, remove_app +from frappe.query_builder.utils import db_type_is +from frappe.tests.test_query_builder import run_only_if from frappe.utils import add_to_date, get_bench_path, get_bench_relative_path, now from frappe.utils.backups import fetch_latest_backups @@ -518,6 +520,18 @@ class TestBackups(BaseTestCommands): self.assertIsNotNone(after_backup["public"]) self.assertIsNotNone(after_backup["private"]) + @run_only_if(db_type_is.MARIADB) + def test_clear_log_table(self): + d = frappe.get_doc(doctype="Error Log", title="Something").insert() + d.db_set("modified", "2010-01-01", update_modified=False) + frappe.db.commit() + + self.execute("bench --site {site} clear-log-table --days=30 --doctype='Error Log'") + self.assertEqual(self.returncode, 0) + frappe.db.commit() + + self.assertFalse(frappe.db.exists("Error Log", d.name)) + def test_backup_with_custom_path(self): """Backup to a custom path (--backup-path)""" backup_path = os.path.join(self.home, "backups") From aeb5034a9660020235c5204232177b47b6bbc165 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sun, 12 Jun 2022 21:59:54 +0530 Subject: [PATCH 087/205] refactor: slightly resilient patch and defaults --- .../core/doctype/log_settings/log_settings.py | 31 +++++++++++++++++++ .../patches/v14_0/log_settings_migration.py | 17 +++++----- 2 files changed, 39 insertions(+), 9 deletions(-) diff --git a/frappe/core/doctype/log_settings/log_settings.py b/frappe/core/doctype/log_settings/log_settings.py index 82d6afb385..550ee1a353 100644 --- a/frappe/core/doctype/log_settings/log_settings.py +++ b/frappe/core/doctype/log_settings/log_settings.py @@ -11,6 +11,15 @@ from frappe.model.document import Document from frappe.utils import cint from frappe.utils.caching import site_cache +DEFAULT_LOGTYPES = { + "Error Log": 30, + "Activity Log": 90, + "Email Queue": 30, + "Route History": 90, + "Error Snapshot": 30, + "Scheduled Job Log": 90, +} + @runtime_checkable class LogType(Protocol): @@ -34,6 +43,7 @@ class LogSettings(Document): def validate(self): self.validate_supported_doctypes() self.validate_duplicates() + self.add_default_logtypes() def validate_supported_doctypes(self): for entry in self.logs_to_clear: @@ -54,6 +64,19 @@ class LogSettings(Document): ) seen.add(entry.ref_doctype) + def add_default_logtypes(self): + existing_logtypes = {d.ref_doctype for d in self.logs_to_clear} + added_logtypes = set() + for logtype, frequency in DEFAULT_LOGTYPES.items(): + if logtype not in existing_logtypes and _supports_log_clearing(logtype): + self.append("logs_to_clear", {"ref_doctype": logtype, "days": cint(frequency)}) + added_logtypes.add(logtype) + + if added_logtypes: + frappe.msgprint( + _("Added default log doctypes: {}").format(",".join(added_logtypes)), alert=True + ) + def clear_logs(self): """ Log settings can clear any log type that's registered to it and provides a method to delete old logs. @@ -73,12 +96,20 @@ class LogSettings(Document): def register_doctype(self, doctype: str, days=30): existing_logtypes = {d.ref_doctype for d in self.logs_to_clear} + if doctype not in existing_logtypes and _supports_log_clearing(doctype): self.append("logs_to_clear", {"ref_doctype": doctype, "days": cint(days)}) + else: + for entry in self.logs_to_clear: + if entry.ref_doctype == doctype: + entry.days = days + break def run_log_clean_up(): doc = frappe.get_doc("Log Settings") + doc.add_default_logtypes() + doc.save() doc.clear_logs() diff --git a/frappe/patches/v14_0/log_settings_migration.py b/frappe/patches/v14_0/log_settings_migration.py index b855cac355..04a55a84d8 100644 --- a/frappe/patches/v14_0/log_settings_migration.py +++ b/frappe/patches/v14_0/log_settings_migration.py @@ -2,22 +2,21 @@ import frappe def execute(): - logging_doctypes = { - "Error Log": get_current_setting("clear_error_log_after") or 30, - "Activity Log": get_current_setting("clear_activity_log_after") or 90, - "Email Queue": get_current_setting("clear_email_queue_after") or 30, - "Route History": 90, - "Error Snapshot": 30, - "Scheduled Job Log": 90, + old_settings = { + "Error Log": get_current_setting("clear_error_log_after"), + "Activity Log": get_current_setting("clear_activity_log_after"), + "Email Queue": get_current_setting("clear_email_queue_after"), } frappe.reload_doc("core", "doctype", "Logs To Clear") frappe.reload_doc("core", "doctype", "Log Settings") log_settings = frappe.get_doc("Log Settings") + log_settings.add_default_logtypes() - for doctype, days in logging_doctypes.items(): - log_settings.register_doctype(doctype, days) + for doctype, days in old_settings.items(): + if days: + log_settings.register_doctype(doctype, days) log_settings.save() From d3bb434e3c7099b167a3b11385494e28d9d0b723 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 13 Jun 2022 15:01:37 +0530 Subject: [PATCH 088/205] fix(ux): make log setting grid editable --- frappe/core/doctype/logs_to_clear/logs_to_clear.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frappe/core/doctype/logs_to_clear/logs_to_clear.json b/frappe/core/doctype/logs_to_clear/logs_to_clear.json index 5e242bc12f..6c2a3f879a 100644 --- a/frappe/core/doctype/logs_to_clear/logs_to_clear.json +++ b/frappe/core/doctype/logs_to_clear/logs_to_clear.json @@ -3,6 +3,7 @@ "autoname": "autoincrement", "creation": "2022-06-11 02:02:39.472511", "doctype": "DocType", + "editable_grid": 1, "engine": "InnoDB", "field_order": [ "ref_doctype", @@ -30,7 +31,7 @@ "index_web_pages_for_search": 1, "istable": 1, "links": [], - "modified": "2022-06-11 04:35:54.706775", + "modified": "2022-06-13 02:51:36.857786", "modified_by": "Administrator", "module": "Core", "name": "Logs To Clear", From edbb44925e6db2a2b7b0215b7b37c1cb31533458 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 13 Jun 2022 17:01:23 +0530 Subject: [PATCH 089/205] refactor: better varname and use get_table_name Co-Authored-By: gavin --- frappe/commands/site.py | 22 +++++++++---------- .../core/doctype/log_settings/log_settings.py | 4 ++-- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/frappe/commands/site.py b/frappe/commands/site.py index 2477de859f..a23a3d2a6e 100644 --- a/frappe/commands/site.py +++ b/frappe/commands/site.py @@ -1111,6 +1111,7 @@ def clear_log_table(context, doctype, days, no_backup): ref: https://mariadb.com/kb/en/big-deletes/#deleting-more-than-half-a-table """ + from frappe.utils import get_table_name from frappe.utils.backups import scheduled_backup if not context.sites: @@ -1136,30 +1137,29 @@ def clear_log_table(context, doctype, days, no_backup): ) click.echo(f"Backed up {doctype}") - original = f"`tab{doctype}`" - temporary = f"`tab{doctype} temp_table`" - backup = f"`tab{doctype} backup_table`" + original = get_table_name(doctype) + temporary = f"{original} temp_table" + backup = f"{original} backup_table" try: - frappe.db.sql(f"CREATE TABLE {temporary} LIKE {original}") + frappe.db.sql_ddl(f"CREATE TABLE `{temporary}` LIKE `{original}`") click.echo(f"Copying {doctype} records from last {days} days to temporary table.") # Copy all recent data to new table frappe.db.sql( - f"""INSERT INTO {temporary} - SELECT * FROM {original} - WHERE {original}.`modified` > NOW() - INTERVAL '{days}' DAY""" + f"""INSERT INTO `{temporary}` + SELECT * FROM `{original}` + WHERE `{original}`.`modified` > NOW() - INTERVAL '{days}' DAY""" ) - frappe.db.sql(f"RENAME TABLE {original} TO {backup}, {temporary} TO {original}") + frappe.db.sql_ddl(f"RENAME TABLE `{original}` TO `{backup}`, `{temporary}` TO `{original}`") except Exception as e: - # Discard created tables frappe.db.rollback() - frappe.db.sql(f"DROP TABLE IF EXISTS {temporary}") + frappe.db.sql_list(f"DROP TABLE IF EXISTS `{temporary}`") click.echo(f"Log cleanup for {doctype} failed:\n{e}") sys.exit(1) else: frappe.db.commit() - frappe.db.sql(f"DROP TABLE {backup}") + frappe.db.sql_ddl(f"DROP TABLE `{backup}`") click.secho(f"Cleared {doctype} records older than {days} days", fg="green") diff --git a/frappe/core/doctype/log_settings/log_settings.py b/frappe/core/doctype/log_settings/log_settings.py index 550ee1a353..c3b67bc389 100644 --- a/frappe/core/doctype/log_settings/log_settings.py +++ b/frappe/core/doctype/log_settings/log_settings.py @@ -11,7 +11,7 @@ from frappe.model.document import Document from frappe.utils import cint from frappe.utils.caching import site_cache -DEFAULT_LOGTYPES = { +DEFAULT_LOGTYPES_RETENTION = { "Error Log": 30, "Activity Log": 90, "Email Queue": 30, @@ -67,7 +67,7 @@ class LogSettings(Document): def add_default_logtypes(self): existing_logtypes = {d.ref_doctype for d in self.logs_to_clear} added_logtypes = set() - for logtype, frequency in DEFAULT_LOGTYPES.items(): + for logtype, frequency in DEFAULT_LOGTYPES_RETENTION.items(): if logtype not in existing_logtypes and _supports_log_clearing(logtype): self.append("logs_to_clear", {"ref_doctype": logtype, "days": cint(frequency)}) added_logtypes.add(logtype) From f1271b78def0b548704204a01ae2a3529fbba6cf Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 13 Jun 2022 17:08:33 +0530 Subject: [PATCH 090/205] test: ensure temp tables are removed exist --- frappe/tests/test_commands.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/frappe/tests/test_commands.py b/frappe/tests/test_commands.py index e2a94a24dd..aeb7f364bc 100644 --- a/frappe/tests/test_commands.py +++ b/frappe/tests/test_commands.py @@ -526,11 +526,16 @@ class TestBackups(BaseTestCommands): d.db_set("modified", "2010-01-01", update_modified=False) frappe.db.commit() + tables_before = frappe.db.get_tables(cached=False) + self.execute("bench --site {site} clear-log-table --days=30 --doctype='Error Log'") self.assertEqual(self.returncode, 0) frappe.db.commit() self.assertFalse(frappe.db.exists("Error Log", d.name)) + tables_after = frappe.db.get_tables(cached=False) + + self.assertEqual(set(tables_before), set(tables_after)) def test_backup_with_custom_path(self): """Backup to a custom path (--backup-path)""" From a1fa2353c1f06fd4b47c893331c2888827cd28ec Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 13 Jun 2022 17:28:33 +0530 Subject: [PATCH 091/205] fix: clear pending logs using big-deletes If some sites have long pending old data that doesn't get cleared automcatically then this patch now attempts to discard old data by using "big-delete" code instead of typical delete query. --- frappe/commands/site.py | 35 ++----------- .../core/doctype/log_settings/log_settings.py | 49 ++++++++++++++++++- frappe/patches.txt | 3 +- .../v14_0/clear_long_pending_stale_logs.py | 42 ++++++++++++++++ .../patches/v14_0/log_settings_migration.py | 7 +-- 5 files changed, 98 insertions(+), 38 deletions(-) create mode 100644 frappe/patches/v14_0/clear_long_pending_stale_logs.py diff --git a/frappe/commands/site.py b/frappe/commands/site.py index a23a3d2a6e..a8667d6595 100644 --- a/frappe/commands/site.py +++ b/frappe/commands/site.py @@ -9,6 +9,7 @@ import click # imports - module imports import frappe from frappe.commands import get_site, pass_context +from frappe.core.doctype.log_settings.log_settings import LOG_DOCTYPES from frappe.exceptions import SiteNotSpecifiedError @@ -1088,16 +1089,6 @@ def build_search_index(context): frappe.destroy() -LOG_DOCTYPES = [ - "Scheduled Job Log", - "Activity Log", - "Route History", - "Email Queue", - "Error Snapshot", - "Error Log", -] - - @click.command("clear-log-table") @click.option("--doctype", default="text", type=click.Choice(LOG_DOCTYPES), help="Log DocType") @click.option("--days", type=int, help="Keep records for days") @@ -1111,7 +1102,7 @@ def clear_log_table(context, doctype, days, no_backup): ref: https://mariadb.com/kb/en/big-deletes/#deleting-more-than-half-a-table """ - from frappe.utils import get_table_name + from frappe.core.doctype.log_settings.log_settings import clear_log_table as clear_logs from frappe.utils.backups import scheduled_backup if not context.sites: @@ -1124,10 +1115,6 @@ def clear_log_table(context, doctype, days, no_backup): frappe.init(site=site) frappe.connect() - if frappe.db.db_type != "mariadb": - click.echo("Postgres database isn't supported by this command") - sys.exit(1) - if not no_backup: scheduled_backup( ignore_conf=False, @@ -1137,29 +1124,13 @@ def clear_log_table(context, doctype, days, no_backup): ) click.echo(f"Backed up {doctype}") - original = get_table_name(doctype) - temporary = f"{original} temp_table" - backup = f"{original} backup_table" - try: - frappe.db.sql_ddl(f"CREATE TABLE `{temporary}` LIKE `{original}`") - click.echo(f"Copying {doctype} records from last {days} days to temporary table.") - # Copy all recent data to new table - frappe.db.sql( - f"""INSERT INTO `{temporary}` - SELECT * FROM `{original}` - WHERE `{original}`.`modified` > NOW() - INTERVAL '{days}' DAY""" - ) - frappe.db.sql_ddl(f"RENAME TABLE `{original}` TO `{backup}`, `{temporary}` TO `{original}`") + clear_logs(doctype, days=days) except Exception as e: - frappe.db.rollback() - frappe.db.sql_list(f"DROP TABLE IF EXISTS `{temporary}`") click.echo(f"Log cleanup for {doctype} failed:\n{e}") sys.exit(1) else: - frappe.db.commit() - frappe.db.sql_ddl(f"DROP TABLE `{backup}`") click.secho(f"Cleared {doctype} records older than {days} days", fg="green") diff --git a/frappe/core/doctype/log_settings/log_settings.py b/frappe/core/doctype/log_settings/log_settings.py index c3b67bc389..b691c61397 100644 --- a/frappe/core/doctype/log_settings/log_settings.py +++ b/frappe/core/doctype/log_settings/log_settings.py @@ -67,9 +67,9 @@ class LogSettings(Document): def add_default_logtypes(self): existing_logtypes = {d.ref_doctype for d in self.logs_to_clear} added_logtypes = set() - for logtype, frequency in DEFAULT_LOGTYPES_RETENTION.items(): + for logtype, retention in DEFAULT_LOGTYPES_RETENTION.items(): if logtype not in existing_logtypes and _supports_log_clearing(logtype): - self.append("logs_to_clear", {"ref_doctype": logtype, "days": cint(frequency)}) + self.append("logs_to_clear", {"ref_doctype": logtype, "days": cint(retention)}) added_logtypes.add(logtype) if added_logtypes: @@ -142,3 +142,48 @@ def get_log_doctypes(doctype, txt, searchfield, start, page_len, filters): supported_doctypes = [(d,) for d in doctypes if _supports_log_clearing(d)] return supported_doctypes[start:page_len] + + +LOG_DOCTYPES = [ + "Scheduled Job Log", + "Activity Log", + "Route History", + "Email Queue", + "Email Queue Recipient", + "Error Snapshot", + "Error Log", +] + + +def clear_log_table(doctype, days=90): + """If any logtype table grows too large then clearing it with DELETE query + is not feasible in reasonable time. This command copies recent data to new + table and replaces current table with new smaller table. + + ref: https://mariadb.com/kb/en/big-deletes/#deleting-more-than-half-a-table + """ + from frappe.utils import get_table_name + + if doctype not in LOG_DOCTYPES: + raise frappe.ValidationError(f"Unsupported logging DocType: {doctype}") + + original = get_table_name(doctype) + temporary = f"{original} temp_table" + backup = f"{original} backup_table" + + try: + frappe.db.sql_ddl(f"CREATE TABLE `{temporary}` LIKE `{original}`") + + # Copy all recent data to new table + frappe.db.sql( + f"""INSERT INTO `{temporary}` + SELECT * FROM `{original}` + WHERE `{original}`.`modified` > NOW() - INTERVAL '{days}' DAY""" + ) + frappe.db.sql_ddl(f"RENAME TABLE `{original}` TO `{backup}`, `{temporary}` TO `{original}`") + except Exception: + frappe.db.rollback() + frappe.db.sql_ddl(f"DROP TABLE IF EXISTS `{temporary}`") + raise + else: + frappe.db.sql_ddl(f"DROP TABLE `{backup}`") diff --git a/frappe/patches.txt b/frappe/patches.txt index 853f02ff4c..d46d40655e 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -191,6 +191,7 @@ frappe.patches.v14_0.remove_post_and_post_comment frappe.patches.v14_0.reset_creation_datetime frappe.patches.v14_0.remove_is_first_startup frappe.patches.v14_0.reload_workspace_child_tables +frappe.patches.v14_0.clear_long_pending_stale_logs frappe.patches.v14_0.log_settings_migration [post_model_sync] @@ -202,4 +203,4 @@ frappe.patches.v14_0.update_color_names_in_kanban_board_column frappe.patches.v14_0.update_is_system_generated_flag frappe.patches.v14_0.update_auto_account_deletion_duration frappe.patches.v14_0.set_document_expiry_default -frappe.patches.v14_0.delete_data_migration_tool \ No newline at end of file +frappe.patches.v14_0.delete_data_migration_tool diff --git a/frappe/patches/v14_0/clear_long_pending_stale_logs.py b/frappe/patches/v14_0/clear_long_pending_stale_logs.py new file mode 100644 index 0000000000..7b6719c69f --- /dev/null +++ b/frappe/patches/v14_0/clear_long_pending_stale_logs.py @@ -0,0 +1,42 @@ +import frappe +from frappe.core.doctype.log_settings.log_settings import clear_log_table +from frappe.utils import add_to_date, today + + +def execute(): + """Due to large size of log tables on old sites some table cleanups never finished during daily log clean up. This patch discards such data by using "big delete" code. + + ref: https://github.com/frappe/frappe/issues/16971 + """ + + DOCTYPE_RETENTION_MAP = { + "Error Log": get_current_setting("clear_error_log_after") or 90, + "Activity Log": get_current_setting("clear_activity_log_after") or 90, + "Email Queue": get_current_setting("clear_email_queue_after") or 30, + # child table on email queue + "Email Queue Recipient": get_current_setting("clear_email_queue_after") or 30, + "Error Snapshot": get_current_setting("clear_error_log_after") or 90, + # newly added + "Scheduled Job Log": 90, + "Route History": 90, + } + + for doctype, retention in DOCTYPE_RETENTION_MAP.items(): + if is_log_cleanup_stuck(doctype, retention): + print(f"Clearing old {doctype} records") + clear_log_table(doctype, retention) + + +def is_log_cleanup_stuck(doctype: str, retention: int) -> bool: + """Check if doctype has data significantly older than configured cleanup period""" + threshold = add_to_date(today(), days=retention * -2) + + return bool(frappe.db.exists(doctype, {"modified": ("<", threshold)})) + + +def get_current_setting(fieldname): + try: + return frappe.db.get_single_value("Log Settings", fieldname) + except Exception: + # Field might be gone if patch is reattempted + pass diff --git a/frappe/patches/v14_0/log_settings_migration.py b/frappe/patches/v14_0/log_settings_migration.py index 04a55a84d8..203405e69b 100644 --- a/frappe/patches/v14_0/log_settings_migration.py +++ b/frappe/patches/v14_0/log_settings_migration.py @@ -14,9 +14,9 @@ def execute(): log_settings = frappe.get_doc("Log Settings") log_settings.add_default_logtypes() - for doctype, days in old_settings.items(): - if days: - log_settings.register_doctype(doctype, days) + for doctype, retention in old_settings.items(): + if retention: + log_settings.register_doctype(doctype, retention) log_settings.save() @@ -25,4 +25,5 @@ def get_current_setting(fieldname): try: return frappe.db.get_single_value("Log Settings", fieldname) except Exception: + # Field might be gone if patch is reattempted pass From c380d4a1b86f2fa84be8edc611931dea59133c39 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Tue, 14 Jun 2022 11:34:17 +0530 Subject: [PATCH 092/205] refactor: dont delete route history by default It can still be added to log settings manually. --- frappe/core/doctype/log_settings/log_settings.py | 1 - frappe/patches/v14_0/clear_long_pending_stale_logs.py | 1 - 2 files changed, 2 deletions(-) diff --git a/frappe/core/doctype/log_settings/log_settings.py b/frappe/core/doctype/log_settings/log_settings.py index b691c61397..1a7ce532cd 100644 --- a/frappe/core/doctype/log_settings/log_settings.py +++ b/frappe/core/doctype/log_settings/log_settings.py @@ -15,7 +15,6 @@ DEFAULT_LOGTYPES_RETENTION = { "Error Log": 30, "Activity Log": 90, "Email Queue": 30, - "Route History": 90, "Error Snapshot": 30, "Scheduled Job Log": 90, } diff --git a/frappe/patches/v14_0/clear_long_pending_stale_logs.py b/frappe/patches/v14_0/clear_long_pending_stale_logs.py index 7b6719c69f..53127cb197 100644 --- a/frappe/patches/v14_0/clear_long_pending_stale_logs.py +++ b/frappe/patches/v14_0/clear_long_pending_stale_logs.py @@ -18,7 +18,6 @@ def execute(): "Error Snapshot": get_current_setting("clear_error_log_after") or 90, # newly added "Scheduled Job Log": 90, - "Route History": 90, } for doctype, retention in DOCTYPE_RETENTION_MAP.items(): From 8a7458ac5719b192b82fadb2b599191d5449449e Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Tue, 14 Jun 2022 13:05:32 +0530 Subject: [PATCH 093/205] feat(ux): show log retention policy in sidebar for discoverablity --- .../doctype/activity_log/activity_log_list.js | 9 ++++-- .../core/doctype/error_log/error_log_list.js | 14 +++++---- .../error_snapshot/error_snapshot_list.js | 7 ++++- .../scheduled_job_log_list.js | 7 +++++ .../route_history/route_history_list.js | 7 +++++ .../doctype/email_queue/email_queue_list.js | 6 ++++ frappe/public/js/frappe/logtypes.js | 30 +++++++++++++++++++ frappe/public/js/logtypes.bundle.js | 1 + 8 files changed, 73 insertions(+), 8 deletions(-) create mode 100644 frappe/core/doctype/scheduled_job_log/scheduled_job_log_list.js create mode 100644 frappe/desk/doctype/route_history/route_history_list.js create mode 100644 frappe/public/js/frappe/logtypes.js create mode 100644 frappe/public/js/logtypes.bundle.js diff --git a/frappe/core/doctype/activity_log/activity_log_list.js b/frappe/core/doctype/activity_log/activity_log_list.js index 111a230827..e3a75a1941 100644 --- a/frappe/core/doctype/activity_log/activity_log_list.js +++ b/frappe/core/doctype/activity_log/activity_log_list.js @@ -4,5 +4,10 @@ frappe.listview_settings['Activity Log'] = { return [__(doc.status), "green"]; else if(doc.operation == "Login" && doc.status == "Failed") return [__(doc.status), "red"]; - } -}; \ No newline at end of file + }, + onload: function(listview) { + frappe.require("logtypes.bundle.js", () => { + frappe.utils.logtypes.show_log_retention_message(cur_list.doctype); + }) + }, +}; diff --git a/frappe/core/doctype/error_log/error_log_list.js b/frappe/core/doctype/error_log/error_log_list.js index 91e69452ff..e92773a9de 100644 --- a/frappe/core/doctype/error_log/error_log_list.js +++ b/frappe/core/doctype/error_log/error_log_list.js @@ -1,7 +1,7 @@ -frappe.listview_settings['Error Log'] = { +frappe.listview_settings["Error Log"] = { add_fields: ["seen"], get_indicator: function(doc) { - if(cint(doc.seen)) { + if (cint(doc.seen)) { return [__("Seen"), "green", "seen,=,1"]; } else { return [__("Not Seen"), "red", "seen,=,0"]; @@ -11,11 +11,15 @@ frappe.listview_settings['Error Log'] = { onload: function(listview) { listview.page.add_menu_item(__("Clear Error Logs"), function() { frappe.call({ - method:'frappe.core.doctype.error_log.error_log.clear_error_logs', + method: "frappe.core.doctype.error_log.error_log.clear_error_logs", callback: function() { listview.refresh(); - } + }, }); }); - } + + frappe.require("logtypes.bundle.js", () => { + frappe.utils.logtypes.show_log_retention_message(cur_list.doctype); + }) + }, }; diff --git a/frappe/core/doctype/error_snapshot/error_snapshot_list.js b/frappe/core/doctype/error_snapshot/error_snapshot_list.js index 1ba3e344ae..553495beb1 100644 --- a/frappe/core/doctype/error_snapshot/error_snapshot_list.js +++ b/frappe/core/doctype/error_snapshot/error_snapshot_list.js @@ -10,5 +10,10 @@ frappe.listview_settings["Error Snapshot"] = { } else { return [__("First Level"), !doc.seen ? "red" : "green", "parent_error_snapshot,=,"]; } - } + }, + onload: function(listview) { + frappe.require("logtypes.bundle.js", () => { + frappe.utils.logtypes.show_log_retention_message(cur_list.doctype); + }) + }, } diff --git a/frappe/core/doctype/scheduled_job_log/scheduled_job_log_list.js b/frappe/core/doctype/scheduled_job_log/scheduled_job_log_list.js new file mode 100644 index 0000000000..5ddccb5d44 --- /dev/null +++ b/frappe/core/doctype/scheduled_job_log/scheduled_job_log_list.js @@ -0,0 +1,7 @@ +frappe.listview_settings["Scheduled Job Log"] = { + onload: function(listview) { + frappe.require("logtypes.bundle.js", () => { + frappe.utils.logtypes.show_log_retention_message(cur_list.doctype); + }) + }, +}; diff --git a/frappe/desk/doctype/route_history/route_history_list.js b/frappe/desk/doctype/route_history/route_history_list.js new file mode 100644 index 0000000000..84a441852c --- /dev/null +++ b/frappe/desk/doctype/route_history/route_history_list.js @@ -0,0 +1,7 @@ +frappe.listview_settings["Route History"] = { + onload: function(listview) { + frappe.require("logtypes.bundle.js", () => { + frappe.utils.logtypes.show_log_retention_message(cur_list.doctype); + }) + }, +}; diff --git a/frappe/email/doctype/email_queue/email_queue_list.js b/frappe/email/doctype/email_queue/email_queue_list.js index 0445a3ca19..edc6250714 100644 --- a/frappe/email/doctype/email_queue/email_queue_list.js +++ b/frappe/email/doctype/email_queue/email_queue_list.js @@ -19,5 +19,11 @@ frappe.listview_settings['Email Queue'] = { }) } } + }, + + onload: function(listview) { + frappe.require("logtypes.bundle.js", () => { + frappe.utils.logtypes.show_log_retention_message(cur_list.doctype); + }) } } diff --git a/frappe/public/js/frappe/logtypes.js b/frappe/public/js/frappe/logtypes.js new file mode 100644 index 0000000000..d6e386c4d2 --- /dev/null +++ b/frappe/public/js/frappe/logtypes.js @@ -0,0 +1,30 @@ +// Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors +// MIT License. See license.txt + +// Common utility functions for logging doctypes. + +frappe.provide("frappe.utils.logtypes"); + +frappe.utils.logtypes.show_log_retention_message = (doctype) => { + if (!frappe.model.can_write("Log Settings")) { + return; + } + + const add_sidebar_message = (message) => { + let sidebar_entry = $('
    ').appendTo(cur_list.page.sidebar); + $(`
    ${message}
    `).appendTo(sidebar_entry); + }; + + const log_settings_link = `${__('Log Settings')}`; + const cta = __("You can change the retention policy from {0}.", [log_settings_link,]); + let message = __("{0} records are not automatically deleted.", [__(doctype),]); + + frappe.db + .get_value("Logs To Clear", { ref_doctype: doctype }, "days", null, "Log Settings") + .then((r) => { + if (!r.exc && r.message && r.message.days) { + message = __("{0} records are retained for {1} days.", [__(doctype), r.message.days,]); + } + add_sidebar_message(`${message} ${cta}`); + }); +}; diff --git a/frappe/public/js/logtypes.bundle.js b/frappe/public/js/logtypes.bundle.js new file mode 100644 index 0000000000..775ac730ad --- /dev/null +++ b/frappe/public/js/logtypes.bundle.js @@ -0,0 +1 @@ +import "./frappe/logtypes" From 57a4d590d0e80dd5a4a0e1df05fe72dfc84bfe64 Mon Sep 17 00:00:00 2001 From: phot0n Date: Wed, 15 Jun 2022 12:49:29 +0530 Subject: [PATCH 094/205] test: test_user_date_label_dashboard_chart --- .../dashboard_chart/test_dashboard_chart.py | 48 +++++++++++++------ 1 file changed, 33 insertions(+), 15 deletions(-) diff --git a/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py b/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py index 94ea1af35c..51ad5ca78a 100644 --- a/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py +++ b/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # License: MIT. See LICENSE -import unittest + from datetime import datetime from unittest.mock import patch @@ -9,11 +9,12 @@ from dateutil.relativedelta import relativedelta import frappe from frappe.desk.doctype.dashboard_chart.dashboard_chart import get +from frappe.tests.utils import FrappeTestCase from frappe.utils import formatdate, get_last_day, getdate from frappe.utils.dateutils import get_period, get_period_ending -class TestDashboardChart(unittest.TestCase): +class TestDashboardChart(FrappeTestCase): def test_period_ending(self): self.assertEqual(get_period_ending("2019-04-10", "Daily"), getdate("2019-04-10")) @@ -57,8 +58,6 @@ class TestDashboardChart(unittest.TestCase): self.assertEqual(result.get("labels")[idx], get_period(month)) cur_date += relativedelta(months=1) - frappe.db.rollback() - def test_empty_dashboard_chart(self): if frappe.db.exists("Dashboard Chart", "Test Empty Dashboard Chart"): frappe.delete_doc("Dashboard Chart", "Test Empty Dashboard Chart") @@ -89,8 +88,6 @@ class TestDashboardChart(unittest.TestCase): self.assertEqual(result.get("labels")[idx], get_period(month)) cur_date += relativedelta(months=1) - frappe.db.rollback() - def test_chart_wih_one_value(self): if frappe.db.exists("Dashboard Chart", "Test Empty Dashboard Chart 2"): frappe.delete_doc("Dashboard Chart", "Test Empty Dashboard Chart 2") @@ -127,8 +124,6 @@ class TestDashboardChart(unittest.TestCase): # only 1 data point with value self.assertEqual(result.get("datasets")[0].get("values")[2], 0) - frappe.db.rollback() - def test_group_by_chart_type(self): if frappe.db.exists("Dashboard Chart", "Test Group By Dashboard Chart"): frappe.delete_doc("Dashboard Chart", "Test Group By Dashboard Chart") @@ -151,8 +146,6 @@ class TestDashboardChart(unittest.TestCase): self.assertEqual(result.get("datasets")[0].get("values")[0], todo_status_count) - frappe.db.rollback() - def test_daily_dashboard_chart(self): insert_test_records() @@ -183,8 +176,6 @@ class TestDashboardChart(unittest.TestCase): result.get("labels"), ["06-01-19", "07-01-19", "08-01-19", "09-01-19", "10-01-19", "11-01-19"] ) - frappe.db.rollback() - def test_weekly_dashboard_chart(self): insert_test_records() @@ -214,8 +205,6 @@ class TestDashboardChart(unittest.TestCase): self.assertEqual(result.get("datasets")[0].get("values"), [50.0, 300.0, 800.0, 0.0]) self.assertEqual(result.get("labels"), ["30-12-18", "06-01-19", "13-01-19", "20-01-19"]) - frappe.db.rollback() - def test_avg_dashboard_chart(self): insert_test_records() @@ -244,7 +233,36 @@ class TestDashboardChart(unittest.TestCase): self.assertEqual(result.get("labels"), ["30-12-18", "06-01-19", "13-01-19", "20-01-19"]) self.assertEqual(result.get("datasets")[0].get("values"), [50.0, 150.0, 266.6666666666667, 0.0]) - frappe.db.rollback() + def test_user_date_label_dashboard_chart(self): + frappe.delete_doc_if_exists("Dashboard Chart", "Test Dashboard Chart Date Label") + + frappe.get_doc( + dict( + doctype="Dashboard Chart", + chart_name="Test Dashboard Chart Date Label", + chart_type="Count", + document_type="DocType", + based_on="creation", + timespan="Select Date Range", + time_interval="Weekly", + from_date=datetime(2018, 12, 30), + to_date=datetime(2019, 1, 15), + filters_json="[]", + timeseries=1, + ) + ).insert() + + with patch.object(frappe.utils.data, "get_user_date_format", return_value="dd.mm.yyyy"): + result = get(chart_name="Test Dashboard Chart Date Label") + self.assertEqual( + sorted(result.get("labels")), sorted(["01.05.2019", "01.12.2019", "19.01.2019"]) + ) + + with patch.object(frappe.utils.data, "get_user_date_format", return_value="mm-dd-yyyy"): + result = get(chart_name="Test Dashboard Chart Date Label") + self.assertEqual( + sorted(result.get("labels")), sorted(["01-19-2019", "05-01-2019", "12-01-2019"]) + ) def insert_test_records(): From 4937b3d17f73656c1ae6235f8e0ae5bd55fa16c0 Mon Sep 17 00:00:00 2001 From: RJPvT <48353029+RJPvT@users.noreply.github.com> Date: Wed, 8 Jun 2022 15:23:14 +0200 Subject: [PATCH 095/205] fix: no add/change image-field if user is not allowed no add/change image-field (dropdown) if user is not allowed to change (cherry picked from commit 4af5006470739a11ddcf3141c95ae6fb157d6825) --- .../js/frappe/form/sidebar/user_image.js | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/frappe/public/js/frappe/form/sidebar/user_image.js b/frappe/public/js/frappe/form/sidebar/user_image.js index f657cbca02..bd6487fd52 100644 --- a/frappe/public/js/frappe/form/sidebar/user_image.js +++ b/frappe/public/js/frappe/form/sidebar/user_image.js @@ -65,15 +65,17 @@ frappe.ui.form.setup_user_image_event = function(frm) { }); } - frm.sidebar.image_wrapper.on('click', ':not(.sidebar-image-actions)', (e) => { - let $target = $(e.currentTarget); - if ($target.is('a.dropdown-toggle, .dropdown')) { - return; - } - let dropdown = frm.sidebar.image_wrapper.find('.sidebar-image-actions .dropdown'); - dropdown.toggleClass('open'); - e.stopPropagation(); - }); + if (frm.fields_dict[frm.meta.image_field].df.read_only == 0) { + frm.sidebar.image_wrapper.on('click', ':not(.sidebar-image-actions)', (e) => { + let $target = $(e.currentTarget); + if ($target.is('a.dropdown-toggle, .dropdown')) { + return; + } + let dropdown = frm.sidebar.image_wrapper.find('.sidebar-image-actions .dropdown'); + dropdown.toggleClass('open'); + e.stopPropagation(); + }); + } // bind click on image_wrapper frm.sidebar.image_wrapper.on('click', '.sidebar-image-change, .sidebar-image-remove', function(e) { From 70c4305b534d0291f083afaa5a6ac627c29fe410 Mon Sep 17 00:00:00 2001 From: RJPvT <48353029+RJPvT@users.noreply.github.com> Date: Wed, 8 Jun 2022 21:22:31 +0200 Subject: [PATCH 096/205] Update user_image.js (cherry picked from commit d9d3b1421e5516fa8471e5afedebed99ed3bd095) --- frappe/public/js/frappe/form/sidebar/user_image.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/form/sidebar/user_image.js b/frappe/public/js/frappe/form/sidebar/user_image.js index bd6487fd52..e07a426b75 100644 --- a/frappe/public/js/frappe/form/sidebar/user_image.js +++ b/frappe/public/js/frappe/form/sidebar/user_image.js @@ -66,7 +66,7 @@ frappe.ui.form.setup_user_image_event = function(frm) { } if (frm.fields_dict[frm.meta.image_field].df.read_only == 0) { - frm.sidebar.image_wrapper.on('click', ':not(.sidebar-image-actions)', (e) => { + frm.sidebar.image_wrapper.on('click', ':not(.sidebar-image-actions)', (e) => { let $target = $(e.currentTarget); if ($target.is('a.dropdown-toggle, .dropdown')) { return; From dcf7252fba3ba7c0b1cf640622e7cd386b5f4b8e Mon Sep 17 00:00:00 2001 From: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> Date: Wed, 15 Jun 2022 12:52:03 +0530 Subject: [PATCH 097/205] fix: Add image_field check (cherry picked from commit e48f6620646e213526b7ba5614779558e729c638) --- frappe/public/js/frappe/form/sidebar/user_image.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/form/sidebar/user_image.js b/frappe/public/js/frappe/form/sidebar/user_image.js index e07a426b75..d08e17bced 100644 --- a/frappe/public/js/frappe/form/sidebar/user_image.js +++ b/frappe/public/js/frappe/form/sidebar/user_image.js @@ -65,7 +65,7 @@ frappe.ui.form.setup_user_image_event = function(frm) { }); } - if (frm.fields_dict[frm.meta.image_field].df.read_only == 0) { + if (frm.meta.image_field && !frm.fields_dict[frm.meta.image_field].df.read_only) { frm.sidebar.image_wrapper.on('click', ':not(.sidebar-image-actions)', (e) => { let $target = $(e.currentTarget); if ($target.is('a.dropdown-toggle, .dropdown')) { From c2ce0ed8c1f47bfb50f77cc5e181a14f9a60f2e8 Mon Sep 17 00:00:00 2001 From: Nabin Hait Date: Mon, 6 Jun 2022 12:19:18 +0530 Subject: [PATCH 098/205] fix: set first tab active after routing --- frappe/public/js/frappe/form/tab.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/form/tab.js b/frappe/public/js/frappe/form/tab.js index fd23595d80..69c573186b 100644 --- a/frappe/public/js/frappe/form/tab.js +++ b/frappe/public/js/frappe/form/tab.js @@ -78,7 +78,7 @@ export default class Tab { set_active() { this.parent.find('.nav-link').tab('show'); - this.wrapper.addClass('show'); + this.wrapper.addClass('active'); this.frm.active_tab = this; } From b8a1d06363470262fb7918a842c6617d7fdc7d8b Mon Sep 17 00:00:00 2001 From: Nabin Hait Date: Mon, 6 Jun 2022 12:20:30 +0530 Subject: [PATCH 099/205] fix: Global datetime format: show AM/PM in capital letter --- frappe/public/js/frappe/utils/datetime.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/utils/datetime.js b/frappe/public/js/frappe/utils/datetime.js index 2bf64fd88e..12cefb4fd9 100644 --- a/frappe/public/js/frappe/utils/datetime.js +++ b/frappe/public/js/frappe/utils/datetime.js @@ -191,7 +191,7 @@ $.extend(frappe.datetime, { global_date_format: function(d) { var m = moment(d); if(m._f && m._f.indexOf("HH")!== -1) { - return m.format("Do MMMM YYYY, h:mma") + return m.format("Do MMMM YYYY, hh:mm A"); } else { return m.format('Do MMMM YYYY'); } From b36bf64261a44b078f77e94b550ab2ceaf8d1305 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Wed, 15 Jun 2022 13:35:21 +0530 Subject: [PATCH 100/205] chore: revert naming to default (#17194) --- frappe/core/doctype/logs_to_clear/logs_to_clear.json | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/frappe/core/doctype/logs_to_clear/logs_to_clear.json b/frappe/core/doctype/logs_to_clear/logs_to_clear.json index 6c2a3f879a..df19ccd9e7 100644 --- a/frappe/core/doctype/logs_to_clear/logs_to_clear.json +++ b/frappe/core/doctype/logs_to_clear/logs_to_clear.json @@ -1,6 +1,6 @@ { "actions": [], - "autoname": "autoincrement", + "autoname": "hash", "creation": "2022-06-11 02:02:39.472511", "doctype": "DocType", "editable_grid": 1, @@ -35,10 +35,9 @@ "modified_by": "Administrator", "module": "Core", "name": "Logs To Clear", - "naming_rule": "Autoincrement", "owner": "Administrator", "permissions": [], "sort_field": "modified", "sort_order": "DESC", "states": [] -} \ No newline at end of file +} From 5affa2248d3e964f79e3bf84e14084b9017304ba Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Wed, 15 Jun 2022 14:59:25 +0530 Subject: [PATCH 101/205] fix: pass parent doctype on client.get_value (#17196) --- frappe/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/client.py b/frappe/client.py index 3b7e0c397d..4afe0898bc 100644 --- a/frappe/client.py +++ b/frappe/client.py @@ -100,7 +100,7 @@ def get_value(doctype, fieldname, filters=None, as_dict=True, debug=False, paren if frappe.is_table(doctype): check_parent_permission(parent, doctype) - if not frappe.has_permission(doctype): + if not frappe.has_permission(doctype, parent_doctype=parent): frappe.throw(_("No permission for {0}").format(doctype), frappe.PermissionError) filters = get_safe_filters(filters) From 59ab6531c9f6c1ff4a5e1447bf7c0b1c30d4eb3d Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 15 Jun 2022 14:39:48 +0530 Subject: [PATCH 102/205] ci: Invalidate GHA pip cache on pyproject or setup changes - Cache invalidated if any of the files change: *requirements.txt, pyproject.toml, setup.py or setup.cfg - Updated boilerplate for new apps too --- .github/workflows/patch-mariadb-tests.yml | 2 +- .github/workflows/server-mariadb-tests.yml | 2 +- .github/workflows/server-postgres-tests.yml | 2 +- .github/workflows/ui-tests.yml | 2 +- frappe/utils/boilerplate.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/patch-mariadb-tests.yml b/.github/workflows/patch-mariadb-tests.yml index 224e380925..9eb08be94e 100644 --- a/.github/workflows/patch-mariadb-tests.yml +++ b/.github/workflows/patch-mariadb-tests.yml @@ -59,7 +59,7 @@ jobs: uses: actions/cache@v2 with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + key: ${{ runner.os }}-pip-${{ hashFiles('**/*requirements.txt', '**/pyproject.toml', '**/setup.py') }} restore-keys: | ${{ runner.os }}-pip- ${{ runner.os }}- diff --git a/.github/workflows/server-mariadb-tests.yml b/.github/workflows/server-mariadb-tests.yml index 48104b8f16..33fc221e80 100644 --- a/.github/workflows/server-mariadb-tests.yml +++ b/.github/workflows/server-mariadb-tests.yml @@ -70,7 +70,7 @@ jobs: uses: actions/cache@v2 with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + key: ${{ runner.os }}-pip-${{ hashFiles('**/*requirements.txt', '**/pyproject.toml', '**/setup.py') }} restore-keys: | ${{ runner.os }}-pip- ${{ runner.os }}- diff --git a/.github/workflows/server-postgres-tests.yml b/.github/workflows/server-postgres-tests.yml index 241b7ddf96..2b4a2edae0 100644 --- a/.github/workflows/server-postgres-tests.yml +++ b/.github/workflows/server-postgres-tests.yml @@ -73,7 +73,7 @@ jobs: uses: actions/cache@v2 with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + key: ${{ runner.os }}-pip-${{ hashFiles('**/*requirements.txt', '**/pyproject.toml', '**/setup.py') }} restore-keys: | ${{ runner.os }}-pip- ${{ runner.os }}- diff --git a/.github/workflows/ui-tests.yml b/.github/workflows/ui-tests.yml index 06ad921a6a..08bf3584f5 100644 --- a/.github/workflows/ui-tests.yml +++ b/.github/workflows/ui-tests.yml @@ -69,7 +69,7 @@ jobs: uses: actions/cache@v2 with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + key: ${{ runner.os }}-pip-${{ hashFiles('**/*requirements.txt', '**/pyproject.toml', '**/setup.py') }} restore-keys: | ${{ runner.os }}-pip- ${{ runner.os }}- diff --git a/frappe/utils/boilerplate.py b/frappe/utils/boilerplate.py index 28eefd0085..edb742feb4 100644 --- a/frappe/utils/boilerplate.py +++ b/frappe/utils/boilerplate.py @@ -490,7 +490,7 @@ jobs: uses: actions/cache@v2 with: path: ~/.cache/pip - key: ${{{{ runner.os }}}}-pip-${{{{ hashFiles('**/*requirements.txt') }}}} + key: ${{{{ runner.os }}}}-pip-${{{{ hashFiles('**/*requirements.txt', '**/pyproject.toml', '**/setup.py', '**/setup.cfg') }}}} restore-keys: | ${{{{ runner.os }}}}-pip- ${{{{ runner.os }}}}- From 21bad02b72764642926933d7d2f45c33cbdd9a1f Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 15 Jun 2022 14:40:29 +0530 Subject: [PATCH 103/205] ci: Remove dependency on requirements.txt for builds --- .github/workflows/patch-mariadb-tests.yml | 2 +- CODEOWNERS | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/patch-mariadb-tests.yml b/.github/workflows/patch-mariadb-tests.yml index 9eb08be94e..7dffc30dc0 100644 --- a/.github/workflows/patch-mariadb-tests.yml +++ b/.github/workflows/patch-mariadb-tests.yml @@ -124,7 +124,7 @@ jobs: git fetch --depth 1 upstream $branch_name:$branch_name git checkout -q -f $branch_name - pip install -q -r requirements.txt + bench setup requirements --python bench --site test_site migrate done diff --git a/CODEOWNERS b/CODEOWNERS index 170334a4b4..59832e8636 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -12,7 +12,7 @@ data_import* @netchampfaris core/ @surajshetty3416 database @gavindsouza model @gavindsouza -requirements.txt @gavindsouza +pyproject.toml @gavindsouza query_builder/ @gavindsouza commands/ @gavindsouza workspace @shariquerik From 36fa2122a3fca266c38e141665a8ce720c0bfc48 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 15 Jun 2022 15:50:09 +0530 Subject: [PATCH 104/205] fix: Patch qb for different schemas in same process You would want to switch schemas in the same process. Eversince the change https://github.com/frappe/frappe/commit/64e52737646d1661e0d295868fc85aa0da47b1f2 we stopped patching on every frappe.init call which meant, if a MariaDB site was initialized first, frappe._qb_patched would be set to True and if a Postgres site was initialized after, _qb_patched would be lying as the PG engine isn't patched yet. Sooooo we need a Dict instead to maintain this record of patching. This issue caused weird errors lol - Traceback: File "/home/frappe/Desktop/frappe-bench-dev/env/lib/python3.10/site-packages/click/decorators.py", line 21, in new_func return f(get_current_context(), *args, **kwargs) File "/home/frappe/Desktop/frappe-bench-dev/apps/frappe/frappe/commands/__init__.py", line 29, in _func ret = f(frappe._dict(ctx.obj), *args, **kwargs) File "/home/frappe/Desktop/frappe-bench-dev/apps/frappe/frappe/commands/site.py", line 524, in migrate SiteMigration( File "/home/frappe/Desktop/frappe-bench-dev/apps/frappe/frappe/migrate.py", line 169, in run self.setUp() File "/home/frappe/Desktop/frappe-bench-dev/apps/frappe/frappe/migrate.py", line 73, in setUp clear_global_cache() File "/home/frappe/Desktop/frappe-bench-dev/apps/frappe/frappe/cache_manager.py", line 102, in clear_global_cache clear_website_cache() File "/home/frappe/Desktop/frappe-bench-dev/apps/frappe/frappe/website/utils.py", line 374, in clear_website_cache clear_cache(path) File "/home/frappe/Desktop/frappe-bench-dev/apps/frappe/frappe/website/utils.py", line 369, in clear_cache for method in frappe.get_hooks("website_clear_cache"): File "/home/frappe/Desktop/frappe-bench-dev/apps/frappe/frappe/__init__.py", line 1440, in get_hooks hooks = _dict(_load_app_hooks()) File "/home/frappe/Desktop/frappe-bench-dev/apps/frappe/frappe/utils/caching.py", line 57, in wrapper return_val = func(*args, **kwargs) File "/home/frappe/Desktop/frappe-bench-dev/apps/frappe/frappe/__init__.py", line 1407, in _load_app_hooks apps = [app_name] if app_name else get_installed_apps(sort=True) File "/home/frappe/Desktop/frappe-bench-dev/apps/frappe/frappe/utils/caching.py", line 57, in wrapper return_val = func(*args, **kwargs) File "/home/frappe/Desktop/frappe-bench-dev/apps/frappe/frappe/__init__.py", line 1374, in get_installed_apps installed = json.loads(db.get_global("installed_apps") or "[]") File "/home/frappe/Desktop/frappe-bench-dev/apps/frappe/frappe/database/database.py", line 917, in get_global return self.get_default(key, user) File "/home/frappe/Desktop/frappe-bench-dev/apps/frappe/frappe/database/database.py", line 921, in get_default d = self.get_defaults(key, parent) File "/home/frappe/Desktop/frappe-bench-dev/apps/frappe/frappe/database/database.py", line 938, in get_defaults defaults = frappe.defaults.get_defaults(parent) File "/home/frappe/Desktop/frappe-bench-dev/apps/frappe/frappe/defaults.py", line 88, in get_defaults globald = get_defaults_for() File "/home/frappe/Desktop/frappe-bench-dev/apps/frappe/frappe/defaults.py", line 218, in get_defaults_for frappe.qb.from_(table) TypeError: 'Field' object is not callable --- frappe/__init__.py | 4 ++-- frappe/query_builder/utils.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/frappe/__init__.py b/frappe/__init__.py index 4b64732245..76888c1824 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -45,7 +45,7 @@ local = Local() STANDARD_USERS = ("Guest", "Administrator") _dev_server = int(sbool(os.environ.get("DEV_SERVER", False))) -_qb_patched = False +_qb_patched = {} re._MAXCACHE = ( 50 # reduced from default 512 given we are already maintaining this on parent worker ) @@ -243,7 +243,7 @@ def init(site, sites_path=None, new_site=False): setup_module_map() - if not _qb_patched: + if not _qb_patched.get(local.conf.db_type): patch_query_execute() patch_query_aggregation() diff --git a/frappe/query_builder/utils.py b/frappe/query_builder/utils.py index c75e380d49..10bab38a63 100644 --- a/frappe/query_builder/utils.py +++ b/frappe/query_builder/utils.py @@ -104,7 +104,7 @@ def patch_query_execute(): builder_class.run = execute_query builder_class.walk = prepare_query - frappe._qb_patched = True + frappe._qb_patched[frappe.conf.db_type] = True def patch_query_aggregation(): @@ -115,4 +115,4 @@ def patch_query_aggregation(): frappe.qb.min = _min frappe.qb.avg = _avg frappe.qb.sum = _sum - frappe._qb_patched = True + frappe._qb_patched[frappe.conf.db_type] = True From 77a6c103ec6c55b5964835a85fad158ed3d2be4b Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Wed, 15 Jun 2022 16:11:55 +0530 Subject: [PATCH 105/205] test: Fix dashboard chart test cases --- .../desk/doctype/dashboard_chart/test_dashboard_chart.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py b/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py index 51ad5ca78a..ca84b2c301 100644 --- a/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py +++ b/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py @@ -173,7 +173,8 @@ class TestDashboardChart(FrappeTestCase): self.assertEqual(result.get("datasets")[0].get("values"), [200.0, 400.0, 300.0, 0.0, 100.0, 0.0]) self.assertEqual( - result.get("labels"), ["06-01-19", "07-01-19", "08-01-19", "09-01-19", "10-01-19", "11-01-19"] + result.get("labels"), + ["06-01-2019", "07-01-2019", "08-01-2019", "09-01-2019", "10-01-2019", "11-01-2019"], ) def test_weekly_dashboard_chart(self): @@ -203,7 +204,7 @@ class TestDashboardChart(FrappeTestCase): result = get(chart_name="Test Weekly Dashboard Chart", refresh=1) self.assertEqual(result.get("datasets")[0].get("values"), [50.0, 300.0, 800.0, 0.0]) - self.assertEqual(result.get("labels"), ["30-12-18", "06-01-19", "13-01-19", "20-01-19"]) + self.assertEqual(result.get("labels"), ["12-30-2018", "06-01-2019", "01-13-2019", "01-20-2019"]) def test_avg_dashboard_chart(self): insert_test_records() @@ -230,7 +231,7 @@ class TestDashboardChart(FrappeTestCase): with patch.object(frappe.utils.data, "get_first_day_of_the_week", return_value="Monday"): result = get(chart_name="Test Average Dashboard Chart", refresh=1) - self.assertEqual(result.get("labels"), ["30-12-18", "06-01-19", "13-01-19", "20-01-19"]) + self.assertEqual(result.get("labels"), ["12-30-2018", "06-01-2019", "01-13-2019", "01-20-2019"]) self.assertEqual(result.get("datasets")[0].get("values"), [50.0, 150.0, 266.6666666666667, 0.0]) def test_user_date_label_dashboard_chart(self): From f7e00633b935951829d05f48cd831a135b66716d Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Wed, 15 Jun 2022 17:27:04 +0530 Subject: [PATCH 106/205] chore: useless flake8 warnings --- .flake8 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.flake8 b/.flake8 index 56c9b9a369..4b852abd7c 100644 --- a/.flake8 +++ b/.flake8 @@ -28,6 +28,10 @@ ignore = B007, B950, W191, + E124, # closing bracket, irritating while writing QB code + E131, # continuation line unaligned for hanging indent + E123, # closing bracket does not match indentation of opening bracket's line + E101, # ensured by use of black max-line-length = 200 exclude=.github/helper/semgrep_rules From 42f3013699d5cf1cb61021937a11d2b14ed365e6 Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Thu, 16 Jun 2022 11:35:58 +0530 Subject: [PATCH 107/205] fix: daterange value not updating --- frappe/public/js/frappe/form/controls/date_range.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/form/controls/date_range.js b/frappe/public/js/frappe/form/controls/date_range.js index 02cac1cf28..a1e796604f 100644 --- a/frappe/public/js/frappe/form/controls/date_range.js +++ b/frappe/public/js/frappe/form/controls/date_range.js @@ -41,7 +41,8 @@ frappe.ui.form.ControlDateRange = class ControlDateRange extends frappe.ui.form. this.set_mandatory && this.set_mandatory(value); } parse(value) { - if (!value || (value && !value.includes('to'))) return value; + if (value == undefined || typeof value == 'object') return value; + // replace the separator (which can be in user language) with comma const to = __('{0} to {1}').replace('{0}', '').replace('{1}', ''); value = value && value.replace(to, ','); From 1298b39bb3877c72b5dbbcf30041ad724bac56a9 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Thu, 16 Jun 2022 12:07:19 +0530 Subject: [PATCH 108/205] ci: Add one more instance for faster UI tests execution --- .github/workflows/ui-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ui-tests.yml b/.github/workflows/ui-tests.yml index 06ad921a6a..3850f33627 100644 --- a/.github/workflows/ui-tests.yml +++ b/.github/workflows/ui-tests.yml @@ -21,7 +21,7 @@ jobs: strategy: fail-fast: false matrix: - containers: [1, 2] + containers: [1, 2, 3] name: UI Tests (Cypress) From 6cebacaf6fad3a534484884ae3e35192b42d29ad Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Thu, 16 Jun 2022 12:07:55 +0530 Subject: [PATCH 109/205] ci: Update mergify.yml --- .mergify.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.mergify.yml b/.mergify.yml index 97df91a927..d9896df921 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -21,13 +21,13 @@ pull_request_rules: - name: Automatic merge on CI success and review conditions: - status-success=Sider - - status-success=Semantic Pull Request - status-success=Python Unit Tests (MariaDB) (1) - status-success=Python Unit Tests (MariaDB) (2) - status-success=Python Unit Tests (Postgres) (1) - status-success=Python Unit Tests (Postgres) (2) - status-success=UI Tests (Cypress) (1) - status-success=UI Tests (Cypress) (2) + - status-success=UI Tests (Cypress) (3) - status-success=security/snyk (frappe) - label!=dont-merge - label!=squash @@ -44,6 +44,7 @@ pull_request_rules: - status-success=Python Unit Tests (Postgres) (2) - status-success=UI Tests (Cypress) (1) - status-success=UI Tests (Cypress) (2) + - status-success=UI Tests (Cypress) (3) - status-success=security/snyk (frappe) - label!=dont-merge - label=squash From d9e848ef882721427466601b8768f1a2bbb295fe Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Thu, 16 Jun 2022 13:00:34 +0530 Subject: [PATCH 110/205] test: added UI test for date range control --- cypress/integration/control_date_range.js | 42 +++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 cypress/integration/control_date_range.js diff --git a/cypress/integration/control_date_range.js b/cypress/integration/control_date_range.js new file mode 100644 index 0000000000..6f26b35f84 --- /dev/null +++ b/cypress/integration/control_date_range.js @@ -0,0 +1,42 @@ +context('Date Range Control', () => { + before(() => { + cy.login(); + cy.visit('/app'); + }); + + function get_dialog() { + return cy.dialog({ + title: 'Date Range', + fields: [{ + "label": "Date Range", + "fieldname": "date_range", + "fieldtype": "Date Range", + }] + }); + } + + it('Selecting a date range from the datepicker', () => { + cy.clear_dialogs(); + cy.clear_datepickers(); + + get_dialog().as('dialog'); + cy.get_field('date_range', 'Date Range').click(); + cy.get('.datepicker--nav-title').click(); + cy.get('.datepicker--nav-title').click({force: true}); + + //Inputing date range values in the date range field + cy.get('.datepicker--years > .datepicker--cells > .datepicker--cell[data-year=2020]').click(); + cy.get('.datepicker--months > .datepicker--cells > .datepicker--cell[data-month=0]').click(); + cy.get('.datepicker--cell[data-date=1]:first').click({force: true}); + cy.get('.datepicker--cell[data-date=15]:first').click({force: true}); + + // Verify if the selected date range values is set in the date range field + cy.window() + .its('cur_dialog') + .then(dialog => { + let date_range = dialog.get_value("date_range"); + expect(date_range[0]).to.equal('2020-01-01'); + expect(date_range[1]).to.equal('2020-01-15'); + }); + }); +}); \ No newline at end of file From 214be6a7e861a94d99af91bbdf9fded5e4785bc6 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Wed, 15 Jun 2022 17:18:03 +0530 Subject: [PATCH 111/205] chore: discard stale patches, add relevant ones to patches.txt --- frappe/patches.txt | 1 + ..._user_permission_doctype_before_migrate.py | 7 -- .../patches/v12_0/delete_gsuite_if_exists.py | 9 --- frappe/patches/v12_0/init_desk_settings.py | 11 --- .../patches/v12_0/remove_gcalendar_gmaps.py | 11 --- ...webpage_migrate_description_to_meta_tag.py | 12 --- .../patches/v12_0/website_meta_tag_parent.py | 12 --- frappe/patches/v13_0/cleanup_desk_cards.py | 75 ------------------- 8 files changed, 1 insertion(+), 137 deletions(-) delete mode 100644 frappe/patches/v11_0/sync_user_permission_doctype_before_migrate.py delete mode 100644 frappe/patches/v12_0/delete_gsuite_if_exists.py delete mode 100644 frappe/patches/v12_0/init_desk_settings.py delete mode 100644 frappe/patches/v12_0/remove_gcalendar_gmaps.py delete mode 100644 frappe/patches/v12_0/webpage_migrate_description_to_meta_tag.py delete mode 100644 frappe/patches/v12_0/website_meta_tag_parent.py delete mode 100644 frappe/patches/v13_0/cleanup_desk_cards.py diff --git a/frappe/patches.txt b/frappe/patches.txt index d46d40655e..66422c7db0 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -202,5 +202,6 @@ frappe.patches.v14_0.remove_db_aggregation frappe.patches.v14_0.update_color_names_in_kanban_board_column frappe.patches.v14_0.update_is_system_generated_flag frappe.patches.v14_0.update_auto_account_deletion_duration +frappe.patches.v14_0.update_integration_request frappe.patches.v14_0.set_document_expiry_default frappe.patches.v14_0.delete_data_migration_tool diff --git a/frappe/patches/v11_0/sync_user_permission_doctype_before_migrate.py b/frappe/patches/v11_0/sync_user_permission_doctype_before_migrate.py deleted file mode 100644 index 6b7a7695f6..0000000000 --- a/frappe/patches/v11_0/sync_user_permission_doctype_before_migrate.py +++ /dev/null @@ -1,7 +0,0 @@ -import frappe - - -def execute(): - frappe.flags.in_patch = True - frappe.reload_doc("core", "doctype", "user_permission") - frappe.db.commit() diff --git a/frappe/patches/v12_0/delete_gsuite_if_exists.py b/frappe/patches/v12_0/delete_gsuite_if_exists.py deleted file mode 100644 index 1fb3a8c2d0..0000000000 --- a/frappe/patches/v12_0/delete_gsuite_if_exists.py +++ /dev/null @@ -1,9 +0,0 @@ -import frappe - - -def execute(): - """ - Remove GSuite Template and GSuite Settings - """ - frappe.delete_doc_if_exists("DocType", "GSuite Settings") - frappe.delete_doc_if_exists("DocType", "GSuite Templates") diff --git a/frappe/patches/v12_0/init_desk_settings.py b/frappe/patches/v12_0/init_desk_settings.py deleted file mode 100644 index 5ec9764e8f..0000000000 --- a/frappe/patches/v12_0/init_desk_settings.py +++ /dev/null @@ -1,11 +0,0 @@ -import json - -import frappe -from frappe.config import get_modules_from_all_apps_for_user -from frappe.desk.moduleview import get_onboard_items - - -def execute(): - """Reset the initial customizations for desk, with modules, indices and links.""" - frappe.reload_doc("core", "doctype", "user") - frappe.db.sql("""update tabUser set home_settings = ''""") diff --git a/frappe/patches/v12_0/remove_gcalendar_gmaps.py b/frappe/patches/v12_0/remove_gcalendar_gmaps.py deleted file mode 100644 index 1177441130..0000000000 --- a/frappe/patches/v12_0/remove_gcalendar_gmaps.py +++ /dev/null @@ -1,11 +0,0 @@ -import frappe - - -def execute(): - """ - Remove GCalendar and GCalendar Settings - Remove Google Maps Settings as its been merged with Delivery Trips - """ - frappe.delete_doc_if_exists("DocType", "GCalendar Account") - frappe.delete_doc_if_exists("DocType", "GCalendar Settings") - frappe.delete_doc_if_exists("DocType", "Google Maps Settings") diff --git a/frappe/patches/v12_0/webpage_migrate_description_to_meta_tag.py b/frappe/patches/v12_0/webpage_migrate_description_to_meta_tag.py deleted file mode 100644 index 32473481b8..0000000000 --- a/frappe/patches/v12_0/webpage_migrate_description_to_meta_tag.py +++ /dev/null @@ -1,12 +0,0 @@ -import frappe - - -def execute(): - web_pages = frappe.get_all("Web Page", ["name", "description"]) - - for web_page in web_pages: - if web_page.description and web_page.route: - doc = frappe.new_doc("Website Route Meta") - doc.name = web_page.route - doc.append("meta_tags", {"key": "description", "value": web_page.description}) - doc.save() diff --git a/frappe/patches/v12_0/website_meta_tag_parent.py b/frappe/patches/v12_0/website_meta_tag_parent.py deleted file mode 100644 index 8920189826..0000000000 --- a/frappe/patches/v12_0/website_meta_tag_parent.py +++ /dev/null @@ -1,12 +0,0 @@ -import frappe - - -def execute(): - # convert all /path to path - frappe.db.sql( - """ - UPDATE `tabWebsite Meta Tag` - SET parent = SUBSTR(parent, 2) - WHERE parent like '/%' - """ - ) diff --git a/frappe/patches/v13_0/cleanup_desk_cards.py b/frappe/patches/v13_0/cleanup_desk_cards.py deleted file mode 100644 index 988e98a647..0000000000 --- a/frappe/patches/v13_0/cleanup_desk_cards.py +++ /dev/null @@ -1,75 +0,0 @@ -from json import loads - -import frappe -from frappe.desk.doctype.workspace.workspace import get_link_type, get_report_type - - -def execute(): - frappe.reload_doc("desk", "doctype", "workspace") - - pages = frappe.db.sql("Select `name` from `tabDesk Page`") - # pages = frappe.get_all("Workspace", filters={"is_standard": 0}, pluck="name") - - for page in pages: - rebuild_links(page[0]) - - frappe.delete_doc("DocType", "Desk Card") - - -def rebuild_links(page): - # Empty links table - - try: - doc = frappe.get_doc("Workspace", page) - except frappe.DoesNotExistError: - db_doc = get_doc_from_db(page) - - doc = frappe.get_doc(db_doc) - doc.insert(ignore_permissions=True) - - doc.links = [] - - for card in get_all_cards(page): - if isinstance(card.links, str): - links = loads(card.links) - else: - links = card.links - - doc.append( - "links", - {"label": card.label, "type": "Card Break", "icon": card.icon, "hidden": card.hidden or False}, - ) - - for link in links: - if not frappe.db.exists(get_link_type(link.get("type")), link.get("name")): - continue - - doc.append( - "links", - { - "label": link.get("label") or link.get("name"), - "type": "Link", - "link_type": get_link_type(link.get("type")), - "link_to": link.get("name"), - "onboard": link.get("onboard"), - "dependencies": ", ".join(link.get("dependencies", [])), - "is_query_report": get_report_type(link.get("name")) - if link.get("type").lower() == "report" - else 0, - }, - ) - - try: - doc.save(ignore_permissions=True) - except frappe.LinkValidationError: - print(doc.as_dict()) - - -def get_doc_from_db(page): - result = frappe.db.sql("SELECT * FROM `tabDesk Page` WHERE name=%s", [page], as_dict=True) - if result: - return result[0].update({"doctype": "Workspace"}) - - -def get_all_cards(page): - return frappe.db.get_all("Desk Card", filters={"parent": page}, fields=["*"], order_by="idx") From 1c385826ef1c5e848627446c23c237bbdfdb0ab1 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Wed, 15 Jun 2022 17:18:23 +0530 Subject: [PATCH 112/205] test: make sure all patches are in patch.txt --- frappe/tests/test_patches.py | 40 ++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/frappe/tests/test_patches.py b/frappe/tests/test_patches.py index 6ddd8e01d3..aefd6da4e5 100644 --- a/frappe/tests/test_patches.py +++ b/frappe/tests/test_patches.py @@ -1,8 +1,10 @@ import unittest +from pathlib import Path from unittest.mock import mock_open, patch import frappe from frappe.modules import patch_handler +from frappe.utils import get_bench_path EMTPY_FILE = "" EMTPY_SECTION = """ @@ -135,3 +137,41 @@ class TestPatchReader(unittest.TestCase): def test_ignore_comments(self, _file): all, pre, post = self.get_patches() self.assertEqual(pre, ["app.module.patch1", "app.module.patch3"]) + + def test_verify_patch_txt(self): + """Make sure all patches/**.py files are part of patches.txt""" + check_patch_files("frappe") + + +# Do not remove/rename this function, other apps depend on it to test their patches +def check_patch_files(app): + """Make sure all patches/**.py files are part of patches.txt""" + + patch_dir = Path(frappe.get_app_path(app)) / "patches" + + app_patches = [p.split()[0] for p in patch_handler.get_patches_from_app(app)] + + missing_patches = [] + + for file in patch_dir.glob("**/*.py"): + module = _get_dotted_path(file, app) + try: + patch_module = frappe.get_module(module) + if hasattr(patch_module, "execute"): + if module not in app_patches: + missing_patches.append(module) + except Exception: + # patch so bad it doesn't even import :shrug: + missing_patches.append(module) + + if missing_patches: + raise Exception(f"Patches missing in patch.txt: \n" + "\n".join(missing_patches)) + + +def _get_dotted_path(file: Path, app) -> str: + app_path = Path(get_bench_path()) / "apps" / app + + *path, filename = file.relative_to(app_path).parts + base_filename = Path(filename).stem + + return ".".join(path + [base_filename]) From a7f9c2a6729c35c81d2732eacfe6d2f22de36fce Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Thu, 16 Jun 2022 11:48:57 +0530 Subject: [PATCH 113/205] chore: remove migrate --- frappe/hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/hooks.py b/frappe/hooks.py index ae2abcec68..a5f532ad4d 100644 --- a/frappe/hooks.py +++ b/frappe/hooks.py @@ -278,7 +278,7 @@ setup_wizard_exception = [ "frappe.desk.page.setup_wizard.setup_wizard.log_setup_wizard_exception", ] -before_migrate = ["frappe.patches.v11_0.sync_user_permission_doctype_before_migrate.execute"] +before_migrate = [] after_migrate = ["frappe.website.doctype.website_theme.website_theme.after_migrate"] otp_methods = ["OTP App", "Email", "SMS"] From 782fb801ecf63bef43b886968c131463a9ca0f1c Mon Sep 17 00:00:00 2001 From: phot0n Date: Thu, 16 Jun 2022 14:28:04 +0530 Subject: [PATCH 114/205] fix: ignore if integration_type column doesn't exist (for direct v14 installations) for integration request patch --- frappe/patches/v14_0/update_integration_request.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/frappe/patches/v14_0/update_integration_request.py b/frappe/patches/v14_0/update_integration_request.py index 7d491461e3..d067411166 100644 --- a/frappe/patches/v14_0/update_integration_request.py +++ b/frappe/patches/v14_0/update_integration_request.py @@ -3,6 +3,10 @@ import frappe def execute(): doctype = "Integration Request" + + if not frappe.db.has_column(doctype, "integration_type"): + return + frappe.db.set_value( doctype, {"integration_type": "Remote", "integration_request_service": ("!=", "PayPal")}, From 27b0b3669dd6e786a4eb1d026a811bdf04720b3e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 16 Jun 2022 10:24:59 +0000 Subject: [PATCH 115/205] build(deps): bump bruceadams/get-release from 1.2.0 to 1.2.3 Bumps [bruceadams/get-release](https://github.com/bruceadams/get-release) from 1.2.0 to 1.2.3. - [Release notes](https://github.com/bruceadams/get-release/releases) - [Commits](https://github.com/bruceadams/get-release/compare/v1.2.0...v1.2.3) --- updated-dependencies: - dependency-name: bruceadams/get-release dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/publish-assets-releases.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-assets-releases.yml b/.github/workflows/publish-assets-releases.yml index 2582632fa0..faf41ed23e 100644 --- a/.github/workflows/publish-assets-releases.yml +++ b/.github/workflows/publish-assets-releases.yml @@ -36,7 +36,7 @@ jobs: - name: Get release id: get_release - uses: bruceadams/get-release@v1.2.0 + uses: bruceadams/get-release@v1.2.3 - name: Upload built Assets to Release uses: actions/upload-release-asset@v1.0.2 From 86f94654325874aa2ed22d09ec208cb74fb4b403 Mon Sep 17 00:00:00 2001 From: Michelle Alva <50285544+michellealva@users.noreply.github.com> Date: Thu, 16 Jun 2022 16:16:00 +0530 Subject: [PATCH 116/205] chore: typo --- frappe/core/page/permission_manager/permission_manager.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/core/page/permission_manager/permission_manager.js b/frappe/core/page/permission_manager/permission_manager.js index cb218b2eae..8a06a9aac5 100644 --- a/frappe/core/page/permission_manager/permission_manager.js +++ b/frappe/core/page/permission_manager/permission_manager.js @@ -284,7 +284,7 @@ frappe.PermissionEngine = class PermissionEngine { } setup_if_owner(d, role_cell) { - this.add_check(role_cell, d, "if_owner", "Only If Creator") + this.add_check(role_cell, d, "if_owner", "Only if Creator") .removeClass("col-md-4") .css({ "margin-top": "15px" }); } From 335edc14f3b66cd34c97deab0e0f807d71afe723 Mon Sep 17 00:00:00 2001 From: Ritwik Puri Date: Thu, 16 Jun 2022 20:25:01 +0530 Subject: [PATCH 117/205] fix: number card ui inconsistencies (#17160) --- frappe/desk/doctype/number_card/number_card.json | 5 +++-- frappe/desk/doctype/number_card/number_card.py | 9 ++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/frappe/desk/doctype/number_card/number_card.json b/frappe/desk/doctype/number_card/number_card.json index 7975d878ba..ab33715d12 100644 --- a/frappe/desk/doctype/number_card/number_card.json +++ b/frappe/desk/doctype/number_card/number_card.json @@ -51,7 +51,7 @@ "options": "Count\nSum\nAverage\nMinimum\nMaximum" }, { - "depends_on": "eval: doc.function !== 'Count'", + "depends_on": "eval: doc.type === 'Document Type' && doc.function !== 'Count'", "fieldname": "aggregate_function_based_on", "fieldtype": "Select", "label": "Aggregate Function Based On", @@ -192,6 +192,7 @@ }, { "description": "The document type selected is a child table, so the parent document type is required.", + "depends_on": "eval: doc.type === 'Document Type'", "fieldname": "parent_document_type", "fieldtype": "Link", "label": "Parent Document Type", @@ -199,7 +200,7 @@ } ], "links": [], - "modified": "2022-03-10 15:34:38.210910", + "modified": "2022-06-12 15:34:38.210910", "modified_by": "Administrator", "module": "Desk", "name": "Number Card", diff --git a/frappe/desk/doctype/number_card/number_card.py b/frappe/desk/doctype/number_card/number_card.py index d6d4f00b69..2290e49656 100644 --- a/frappe/desk/doctype/number_card/number_card.py +++ b/frappe/desk/doctype/number_card/number_card.py @@ -26,11 +26,10 @@ class NumberCard(Document): if not (self.document_type and self.function): frappe.throw(_("Document Type and Function are required to create a number card")) - if ( - self.document_type - and frappe.get_meta(self.document_type).istable - and not self.parent_document_type - ): + if self.function != "Count" and not self.aggregate_function_based_on: + frappe.throw(_("Aggregate Field is required to create a number card")) + + if frappe.get_meta(self.document_type).istable and not self.parent_document_type: frappe.throw(_("Parent Document Type is required to create a number card")) elif self.type == "Report": From 2ea87532e7e4b511295714d719a1591288a26303 Mon Sep 17 00:00:00 2001 From: Vladislav Date: Thu, 16 Jun 2022 23:15:51 +0300 Subject: [PATCH 118/205] Update ru.csv Fix logic!!! --- frappe/translations/ru.csv | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/translations/ru.csv b/frappe/translations/ru.csv index 81b8bdecd2..31f94dfcf6 100644 --- a/frappe/translations/ru.csv +++ b/frappe/translations/ru.csv @@ -3825,14 +3825,14 @@ Clear,Отчистить, Comment,Комментарий, Comments,Комментарии, DRAFT,ЧЕРНОВИК, -Dashboard,панель инструментов, +Dashboard,Панель инструментов, DocType,DocType, Download,Скачать, EMail,Эл. адрес, Edit in Full Page,Редактировать на полной странице, Email Inbox,Email Inbox, File,Файл, -Forward,Дальше, +Forward,Переслать, Icon,Икона, In,В, Inbox,Входящие, From e1154314fe245a39b4dcacfef5053ffd6b847c4d Mon Sep 17 00:00:00 2001 From: Vladislav Date: Fri, 17 Jun 2022 00:29:16 +0300 Subject: [PATCH 119/205] Update ru.csv --- frappe/translations/ru.csv | 50 +++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/frappe/translations/ru.csv b/frappe/translations/ru.csv index 31f94dfcf6..96b48ea99c 100644 --- a/frappe/translations/ru.csv +++ b/frappe/translations/ru.csv @@ -43,7 +43,7 @@ Client Secret,Секрет клиента, Closed,Закрыт, Code,Код, Collapse All,Свернуть все, -Color,цвет, +Color,Цвет, Company Name,Название компании, Condition,Условия, Contact,Контакты, @@ -118,13 +118,13 @@ Is Active,Активен, Is Completed,Выполнен, Is Default,По умолчанию, Kanban Board,Канбан-доска, -Label,Имя поля, +Label,Имя метки, Language Name,Название языка, Last Name,Фамилия, Leaderboard,Доска почёта, Letter Head,Печатный бланк, Level,Уровень, -Limit,Предел, +Limit,Лимит, Log,Запись в журнале, Logs,Журналы, Low,Низкий, @@ -302,8 +302,8 @@ old_parent,old_parent, 1 month ago,1 месяц назад, 1 year ago,1 год назад, ; not allowed in condition,; не допускается в состоянии, -"

    Default Template

    \n

    Uses Jinja Templating and all the fields of Address (including Custom Fields if any) will be available

    \n
    {{ address_line1 }}<br>\n{% if address_line2 %}{{ address_line2 }}<br>{% endif -%}\n{{ city }}<br>\n{% if state %}{{ state }}<br>{% endif -%}\n{% if pincode %} PIN:  {{ pincode }}<br>{% endif -%}\n{{ country }}<br>\n{% if phone %}Phone: {{ phone }}<br>{% endif -%}\n{% if fax %}Fax: {{ fax }}<br>{% endif -%}\n{% if email_id %}Email: {{ email_id }}<br>{% endif -%}\n
    ","

    Шаблон по умолчанию \n <р> Использование дзиндзя Templating и все поля адрес ( в том числе Пользовательские поля если таковые имеются) будут доступны \n 
     <код> {{address_line1}} & LT; BR & GT; \n {%, если address_line2%} {{address_line2}} & лт; бр & GT; { % ENDIF -%} \n {{город}} & Lt; BR & GT; \n {%, если государство%} {{состояние}} & Lt; BR & GT; {% ENDIF -%} \n {%, если пин-код%} PIN: {{пин-код}} & Lt; BR & GT; {% ENDIF -%} \n {{страна}} & Lt; BR & GT; \n {%, если телефон%} Телефон: {{телефон}} & Lt; BR & GT; { % ENDIF -%} \n {%, если факс%} Факс: {{FAX}} & Lt; BR & GT; {% ENDIF -%} \n {%, если email_id%} E-mail: {{email_id}} & Lt; BR & GT ; {% ENDIF -%} \n </ код>

    ", -A Lead with this Email Address should exist,Обращение с этим адресом электронной почты должно существовать, +"

    Default Template

    \n

    Uses Jinja Templating and all the fields of Address (including Custom Fields if any) will be available

    \n
    {{ address_line1 }}<br>\n{% if address_line2 %}{{ address_line2 }}<br>{% endif -%}\n{{ city }}<br>\n{% if state %}{{ state }}<br>{% endif -%}\n{% if pincode %} PIN:  {{ pincode }}<br>{% endif -%}\n{{ country }}<br>\n{% if phone %}Phone: {{ phone }}<br>{% endif -%}\n{% if fax %}Fax: {{ fax }}<br>{% endif -%}\n{% if email_id %}Email: {{ email_id }}<br>{% endif -%}\n
    ","

    Шаблон по умолчанию

    \n

    Используя шаблонизатор Jinja и все поля адреса (включая настраиваемые поля, если таковые имеются) будет доступно

    \n
    {{ address_line1 }}<br>\n{% if address_line2 %}{{ address_line2 }}<br>{% endif -%}\n{{ city }}<br>\n{% if state %}{{ state }}<br>{% endif -%}\n{% if pincode %} PIN:  {{ pincode }}<br>{% endif -%}\n{{ country }}<br>\n{% if phone %}Телефон: {{ phone }}<br>{% endif -%}\n{% if fax %}Факс: {{ fax }}<br>{% endif -%}\n{% if email_id %}Email: {{ email_id }}<br>{% endif -%}\n
    ", +A Lead with this Email Address should exist,Лид с этим адресом электронной почты должно существовать, A list of resources which the Client App will have access to after the user allows it.
    e.g. project,"Список ресурсов, к которым клиентское приложение будет иметь доступ после того, как пользователь разрешит это.
    например. проект", A log of request errors,Журнал ошибок запроса, A new account has been created for you at {0},Новая учетная запись была создана для вас в {0}, @@ -373,12 +373,12 @@ Administrator accessed {0} on {1} via IP Address {2}.,Администратор Advanced,Продвинутый, Advanced Control,Расширенный контроль, Advanced Search,Расширенный поиск, -Align Labels to the Right,Выровнять метки справа, -Align Value,Align значение, +Align Labels to the Right,Выровнять метки по правому краю, +Align Value,Значение выравнивания, All Images attached to Website Slideshow should be public,"Все изображения, прикрепленные к Слайд-шоу веб-сайта, должны быть общедоступными", All customizations will be removed. Please confirm.,"Все настройки будут удалены. Пожалуйста, подтвердите.", -"All possible Workflow States and roles of the workflow. Docstatus Options: 0 is""Saved"", 1 is ""Submitted"" and 2 is ""Cancelled""","Все возможные статусы и роли бизнес-процесса. Варианты статуса документа: 0 — ""Сохранён (черновик)"", 1 — ""Проведён/ Утвержден"" и 2 — ""Отменен""", -All-uppercase is almost as easy to guess as all-lowercase.,"All-прописные почти так же легко догадаться, как все-строчными буквами.", +"All possible Workflow States and roles of the workflow. Docstatus Options: 0 is""Saved"", 1 is ""Submitted"" and 2 is ""Cancelled""","Все возможные статусы и роли бизнес-процесса. Варианты статуса документа: 0 — ""Сохранён"", 1 — ""Проведён/Утвержден"" и 2 — ""Отменен""", +All-uppercase is almost as easy to guess as all-lowercase.,"Все заглавные почти так же легко угадать, как и все строчные.", Allocated To,Выделяемых на, Allow,Разрешить, Allow Bulk Edit,Разрешить массовое редактирование, @@ -415,7 +415,7 @@ Allowed In Mentions,Разрешено в упоминаниях, "Allowing DocType, DocType. Be careful!","Разрешение DocType, DocType. Будьте осторожны!", Already Registered,Уже регистрировались, Also adding the dependent currency field {0},Также добавление зависимого валютного поля {0}, -"Always add ""Draft"" Heading for printing draft documents","Всегда добавляйте заголовок "Черновик" для печати черновых документов", +"Always add ""Draft"" Heading for printing draft documents","Всегда добавлять в заголовок "Черновик" при печати черновых документов", Always use Account's Email Address as Sender,Всегда использовать почту учетной записи в качестве отправителя, Always use Account's Name as Sender's Name,Всегда использовать имя учетной записи в качестве имени отправителя, Amend,Изменен, @@ -470,7 +470,7 @@ Assigned By,Назначаются, Assigned By Full Name,Присваиваемый Полное имя, Assigned By Me,Назначенные мной, Assigned To,Назначено для, -Assigned To/Owner,Назначено / Владельца, +Assigned To/Owner,Назначено для/Владелец, Assignment,Назначение, Assignment Complete,Задание выполнено, Assignment Completed,Задание выполнено, @@ -481,9 +481,9 @@ Assignment closed by {0},Назначение закрыт {0}, Assignment for {0} {1},Назначение для {0} {1}, Atleast one field of Parent Document Type is mandatory,По крайней мере одно поле родительского документа является обязательным, Attach,Прикрепить, -Attach Document Print,Прикрепите документе печать, +Attach Document Print,Прикрепить печатный документ, Attach Image,Прикрепить изображение, -Attach Print,Прикрепите Печать, +Attach Print,Прикрепите печать, Attach Your Picture,Прикрепите свою фотографию, Attach file for Import,Прикрепить файл для импорта, Attach files / urls and add in table.,Прикрепить файлы / URL-адреса и добавить в таблицу., @@ -494,7 +494,7 @@ Attachment Limit (MB),Лимит вложения (MB), Attachment Removed,Вложение удалено, Attempting Connection to QZ Tray...,Попытка подключения к QZ Tray..., Attempting to launch QZ Tray...,Попытка запустить QZ Tray..., -Auth URL Data,Данные URL-адреса Auth, +Auth URL Data,Данные URL-адреса аутентификации, Authenticating...,Проверка подлинности..., Authentication,Аутентификация, Authentication Apps you can use are: ,"Приложения аутентификации, которые вы можете использовать:", @@ -572,7 +572,7 @@ Bulk Update,Массовое обновление, Busy,Занятый, Button,Кнопка, Button Help,Кнопка помощь, -Button Label,Кнопка ярлык, +Button Label,Кнопка метка, Bypass Two Factor Auth for users who login from restricted IP Address,"Обход двух факторов для пользователей, которые подключаются с ограниченным IP-адресом", Bypass restricted IP Address check If Two Factor Auth Enabled,Обход ограниченного IP-адреса. Если включен параметр Two Factor Auth, CC,Копия, @@ -630,7 +630,7 @@ Cent,Цент, "Certain documents, like an Invoice, should not be changed once final. The final state for such documents is called Submitted. You can restrict which roles can Submit.","Некоторые документы, как, например, счета-фактуры, не подлежат изменению, если являются финальными. Конечное состояние для таких документов называется Проведенный. Вы можете ограничить, какие роли имеют право проводить документы.", Chain Integrity,Целостность цепей, Chaining Hash,Цепочный хэш, -Change Label (via Custom Translation),Изменение этикетки (с помощью пользовательского перевода), +Change Label (via Custom Translation),Изменение метки (с помощью пользовательского перевода), Change Password,Изменить пароль, "Change field properties (hide, readonly, permission etc.)","Изменение свойств поля (скрыть, только для чтения, доступ и т.д.)", Channel,Канал, @@ -684,7 +684,7 @@ Collapsible Depends On,Складные Зависит от, Column,Колонка, Column {0} already exist.,Столбец {0} уже существует., Column Break,Разрыв столбца, -Column Labels:,Колонка ярлыки:, +Column Labels:,Колонка меток:, Column Name,Имя столбца, Column Name cannot be empty,Имя столбца не может быть пустым, Columns,Колонки, @@ -789,7 +789,7 @@ Customizations Reset,Сброс настроек, Customizations for {0} exported to:
    {1},Настройки для {0} экспортированы в:
    {1}, Customize Form,Настроить форму, Customize Form Field,Настроить поля формы, -"Customize Label, Print Hide, Default etc.","Настроить Label, распечатать спрятать, Default т.д.", +"Customize Label, Print Hide, Default etc.","Пользовательские метки, скрыть печать, по умолчанию и т.д.", Customize...,Пользовательские настройки..., "Customized Formats for Printing, Email","Индивидуальные форматы для печати, электронной почты", Customized HTML Templates for printing transactions.,Индивидуальные шаблоны HTML для печатных операций., @@ -1421,9 +1421,9 @@ LDAP Security,LDAP Security, LDAP Server Url,URL cервера LDAP, LDAP Username Field,LDAP Имя пользователя Поле, LDAP is not enabled.,LDAP не включен., -Label Help,Ярлык Помощь, -Label and Type,Ярлык и Тип, -Label is mandatory,Ярлык является обязательным, +Label Help,Метка Помощь, +Label and Type,Метка и Тип, +Label is mandatory,Метка является обязательной, Landing Page,Страница входа, Language,Язык, Language Code,Языковой код, @@ -1796,7 +1796,7 @@ Page to show on the website\n,Страница для показа на сайт Pages in Desk (place holders),Страницы-заглушки, Parent,Родитель, Parent Error Snapshot,Родитель снимка ошибки, -Parent Label,Родительская этикетка, +Parent Label,Родительская метка, Parent Table,Родитель Таблица, Parent is required to get child table data,Родитель обязан получать данные дочерней таблицы, Parent is the name of the document to which the data will get added to.,"Родитель - это имя документа, к которому будут добавлены данные.", @@ -4300,7 +4300,7 @@ Application Version,Версия приложения, Git Branch,Git Branch, Installed Applications,Установленные приложения, Navbar Item,Элемент навигационной панели, -Item Label,Этикетка товара, +Item Label,Метка товара, Item Type,Тип объекта, Separator,Разделитель, Navbar Settings,Настройки навигационной панели, @@ -4434,7 +4434,7 @@ Meta Title,Мета-заголовок, Enable Social Sharing,Включить обмен в социальных сетях, Show CTA in Blog,Показать CTA в блоге, CTA,CTA, -CTA Label,Ярлык CTA, +CTA Label,Метка CTA, CTA URL,CTA URL, Default Portal Home,Главная страница портала по умолчанию, "Example: ""/desk""",Пример: "/ стол", @@ -4521,7 +4521,7 @@ Select Fields,Выбрать поля, Warning: Unable to find {0} in any table related to {1},"Предупреждение: невозможно найти {0} ни в одной таблице, связанной с {1}", Tree view is not available for {0},Просмотр в виде дерева недоступен для {0}, Create Card,Создать карту, -Card Label,Этикетка карты, +Card Label,Метка карты, Reports already in Queue,Отчеты уже в очереди, Proceed Anyway,Все равно продолжайте, Delete and Generate New,Удалить и создать новое, From 2a75b2b48099bdd5a62e51c3567b5c25d3125215 Mon Sep 17 00:00:00 2001 From: Vladislav Date: Fri, 17 Jun 2022 01:04:46 +0300 Subject: [PATCH 120/205] Update ru.csv Fix logic!!! --- frappe/translations/ru.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/translations/ru.csv b/frappe/translations/ru.csv index 96b48ea99c..60e140c2a5 100644 --- a/frappe/translations/ru.csv +++ b/frappe/translations/ru.csv @@ -2023,7 +2023,7 @@ Register OAuth Client App,Регистрация OAuth Client App, Registered but disabled,Зарегистрированный но отключен, Relapsed,Повторный, Relapses,Повторные, -Relink,Перередактируйте, +Relink,Связь, Relink Communication,Повторно связать коммуникации, Relinked,Связать повторно, Reload,Обновить страницу, From 6d6a67e9e322abd831d9669dc93c54c0b14dd833 Mon Sep 17 00:00:00 2001 From: Himanshu Date: Fri, 17 Jun 2022 12:07:07 +0530 Subject: [PATCH 121/205] feat: set image as footer in letter head (#17119) * feat: set image as footer in letter head * fix: do not hide footer section * fix: hide footer source * style: black * fix: reduce code for setting image as html * fix: Force system admin role only if active * fix(ui): tab refresh was not implemented * fix(minor): Onboarding: add option to view list view in create action * fix(minor): js lint * fix: allow All to select a User * test: user permissions affecting User * refactor: filter_dynamic_link_doctypes API * Added typing, better variable naming * Remove unnecessary re-iterations * Optimize queries and membership processing * perf: Check query type via is_query_type * chore: Drop duplicate get_frontmatter definition * perf: Login Page Improves performance 3x - from 0.047s to 0.017s * Use frappe.get_*_settings to query table once * Use cached LDAP Settings' document via get_ldap_client_settings * Use single get_all to query all Social Login providers and related data * Skip provider if client_secret doesn't exist * perf: About Us Settings Use cached document for building /about page * perf: App Page Reduced time taken for get_context to execute from 0.035s to 0.02s (75% reduction) * perf: Patch qb only once - not on every init * perf: Fetch and cache entire settings' dicts * refactor!: frappe.db.get_singles_dict * Cast single's values as their fieldtypes before returning * Support previously dead debug parameter * Consider single with no meta as non-existent; skip query Decided to go ahead with the breaking change given the nature of the existing usages of get_singles_dict :crie: * ci: Run tests bypassing roulette with labels "Run UI Tests", "Run Server Tests" * test: Scheduler tests cleanup * chore: linter changes * chore: linter changes * chore: linter changes * fix: set HTML as default to keep existing formats working * refactor: no **kwargs Co-authored-by: Raffael Meyer <14891507+barredterra@users.noreply.github.com> Co-authored-by: Deepesh Garg Co-authored-by: Rushabh Mehta Co-authored-by: Gavin D'souza --- .../doctype/letter_head/letter_head.json | 48 +++++++++++++- .../doctype/letter_head/letter_head.py | 62 ++++++++++++++----- 2 files changed, 92 insertions(+), 18 deletions(-) diff --git a/frappe/printing/doctype/letter_head/letter_head.json b/frappe/printing/doctype/letter_head/letter_head.json index f723a6b489..d49b65ab36 100644 --- a/frappe/printing/doctype/letter_head/letter_head.json +++ b/frappe/printing/doctype/letter_head/letter_head.json @@ -9,6 +9,7 @@ "field_order": [ "letter_head_name", "source", + "footer_source", "column_break_3", "disabled", "is_default", @@ -20,7 +21,12 @@ "header_section", "content", "footer_section", - "footer" + "footer", + "footer_image_section", + "footer_image", + "footer_image_height", + "footer_image_width", + "footer_align" ], "fields": [ { @@ -93,7 +99,7 @@ "oldfieldtype": "Text Editor" }, { - "collapsible": 1, + "depends_on": "eval:doc.footer_source==='HTML' && doc.letter_head_name", "fieldname": "footer_section", "fieldtype": "Section Break", "label": "Footer" @@ -121,13 +127,48 @@ "fieldname": "image_width", "fieldtype": "Float", "label": "Image Width" + }, + { + "depends_on": "eval:doc.footer_source==='Image' && doc.letter_head_name", + "fieldname": "footer_image_section", + "fieldtype": "Section Break", + "label": "Footer Image" + }, + { + "fieldname": "footer_image", + "fieldtype": "Attach Image", + "label": "Image" + }, + { + "fieldname": "footer_image_height", + "fieldtype": "Float", + "label": "Image Height" + }, + { + "fieldname": "footer_image_width", + "fieldtype": "Float", + "label": "Image Width" + }, + { + "fieldname": "footer_align", + "fieldtype": "Select", + "label": "Align", + "options": "Left\nRight\nCenter" + }, + { + "default": "HTML", + "depends_on": "letter_head_name", + "fieldname": "footer_source", + "fieldtype": "Select", + "label": "Footer Based On", + "options": "Image\nHTML" } ], "icon": "fa fa-font", "idx": 1, "links": [], "max_attachments": 3, - "modified": "2021-10-03 14:37:58.314696", + "modified": "2022-06-16 23:10:46.852116", "modified_by": "Administrator", "module": "Printing", "name": "Letter Head", @@ -152,5 +193,6 @@ ], "sort_field": "modified", "sort_order": "ASC", + "states": [], "track_changes": 1 } \ No newline at end of file diff --git a/frappe/printing/doctype/letter_head/letter_head.py b/frappe/printing/doctype/letter_head/letter_head.py index 98c2fc7c2b..9edd84a425 100644 --- a/frappe/printing/doctype/letter_head/letter_head.py +++ b/frappe/printing/doctype/letter_head/letter_head.py @@ -26,21 +26,53 @@ class LetterHead(Document): def set_image(self): if self.source == "Image": - if self.image and is_image(self.image): - self.image_width = flt(self.image_width) - self.image_height = flt(self.image_height) - dimension = "width" if self.image_width > self.image_height else "height" - dimension_value = self.get("image_" + dimension) - self.content = f""" -
    - {self.name} -
    - """ - frappe.msgprint(frappe._("Header HTML set from attachment {0}").format(self.image), alert=True) - else: - frappe.msgprint( - frappe._("Please attach an image file to set HTML"), alert=True, indicator="orange" - ) + self.set_image_as_html( + field="image", + width="image_width", + height="image_height", + align="align", + html_field="content", + dimension_prefix="image_", + success_msg=_("Header HTML set from attachment {0}").format(self.image), + failure_msg=_("Please attach an image file to set HTML for Letter Head."), + ) + + if self.footer_source == "Image": + self.set_image_as_html( + field="footer_image", + width="footer_image_width", + height="footer_image_height", + align="footer_align", + html_field="footer", + dimension_prefix="footer_image_", + success_msg=_("Footer HTML set from attachment {0}").format(self.footer_image), + failure_msg=_("Please attach an image file to set HTML for Footer."), + ) + + def set_image_as_html( + self, field, width, height, dimension_prefix, align, html_field, success_msg, failure_msg + ): + if not self.get(field) or not is_image(self.get(field)): + frappe.msgprint(failure_msg, alert=True, indicator="orange") + return + + self.set(width, flt(self.get(width))) + self.set(height, flt(self.get(height))) + + # To preserve the aspect ratio of the image, apply constraints only on + # the greater dimension and allow the other to scale accordingly + dimension = "width" if width > height else "height" + dimension_value = self.get(f"{dimension_prefix}{dimension}") + + self.set( + html_field, + f"""
    +{self.get( +
    """, + ) + + frappe.msgprint(success_msg, alert=True) def on_update(self): self.set_as_default() From ea18460cc29ed159cf3d6731605e28333babc837 Mon Sep 17 00:00:00 2001 From: vishdha Date: Fri, 10 Jun 2022 19:19:05 +0530 Subject: [PATCH 122/205] fix: date_field not able to fetch in _format_naming autoname --- frappe/model/naming.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frappe/model/naming.py b/frappe/model/naming.py index f6a3846699..9fffa4f493 100644 --- a/frappe/model/naming.py +++ b/frappe/model/naming.py @@ -213,7 +213,7 @@ def set_name_from_naming_options(autoname, doc): doc.name = _format_autoname(autoname, doc) elif "#" in autoname: doc.name = make_autoname(autoname, doc=doc) - + return doc.name def set_naming_from_document_naming_rule(doc): """ @@ -317,9 +317,9 @@ def parse_naming_series( part = frappe.defaults.get_user_default("fiscal_year") elif e.startswith("{") and doc: e = e.replace("{", "").replace("}", "") - part = doc.get(e) + part = (cstr(doc.get(e)) or "").strip() elif doc and doc.get(e): - part = doc.get(e) + part = (cstr(doc.get(e)) or "").strip() else: part = e From 27f2cdf0169ca8217093aec01375bcc45f413c23 Mon Sep 17 00:00:00 2001 From: vishdha Date: Fri, 10 Jun 2022 20:33:33 +0530 Subject: [PATCH 123/205] chore: remove unused code --- frappe/model/naming.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/model/naming.py b/frappe/model/naming.py index 9fffa4f493..b674b0cd81 100644 --- a/frappe/model/naming.py +++ b/frappe/model/naming.py @@ -213,7 +213,7 @@ def set_name_from_naming_options(autoname, doc): doc.name = _format_autoname(autoname, doc) elif "#" in autoname: doc.name = make_autoname(autoname, doc=doc) - return doc.name + def set_naming_from_document_naming_rule(doc): """ From a9e222a7066a40586aaff66b7c9aba5ab01a99d5 Mon Sep 17 00:00:00 2001 From: vishdha Date: Fri, 17 Jun 2022 13:59:00 +0530 Subject: [PATCH 124/205] chore: test case added for _auto_name format for date field --- frappe/tests/test_naming.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/frappe/tests/test_naming.py b/frappe/tests/test_naming.py index d966fd5ce8..1a60b41b2b 100644 --- a/frappe/tests/test_naming.py +++ b/frappe/tests/test_naming.py @@ -12,7 +12,7 @@ from frappe.model.naming import ( revert_series_if_last, ) from frappe.tests.utils import FrappeTestCase -from frappe.utils import now_datetime +from frappe.utils import now_datetime, nowdate class TestNaming(FrappeTestCase): @@ -100,6 +100,23 @@ class TestNaming(FrappeTestCase): self.assertEqual(doc.name, f"TODO-{now_datetime().strftime('%m')}-{description}-{series:02}") + def test_format_autoname_for_date_field(self): + """ + Test if braced params are replaced in format autoname for date field + """ + doctype = new_doctype(autoname="format:TODO-{date}-{##}").insert() + + date = nowdate() + + doc = frappe.new_doc(doctype.name) + doc.date = date + doc.insert() + + series = getseries("", 2) + series = int(series) - 1 + + self.assertEqual(doc.name, f"TODO-{date}-{series:02}") + def test_format_autoname_for_consecutive_week_number(self): """ Test if braced params are replaced for consecutive week number in format autoname From 754de9a56b7bc97993c4a3336ecb718115e75880 Mon Sep 17 00:00:00 2001 From: vishdha Date: Fri, 17 Jun 2022 14:03:29 +0530 Subject: [PATCH 125/205] fix: minor change added --- frappe/tests/test_naming.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/tests/test_naming.py b/frappe/tests/test_naming.py index 1a60b41b2b..cb95cd60b3 100644 --- a/frappe/tests/test_naming.py +++ b/frappe/tests/test_naming.py @@ -106,7 +106,7 @@ class TestNaming(FrappeTestCase): """ doctype = new_doctype(autoname="format:TODO-{date}-{##}").insert() - date = nowdate() + date = nowdate() doc = frappe.new_doc(doctype.name) doc.date = date From 8c6d266fad4a99103a45125987ff21d260457a3a Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Fri, 17 Jun 2022 14:14:44 +0530 Subject: [PATCH 126/205] fix: can't select dynamic link on address doctype --- frappe/contacts/address_and_contact.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frappe/contacts/address_and_contact.py b/frappe/contacts/address_and_contact.py index a0f742c55a..1461690378 100644 --- a/frappe/contacts/address_and_contact.py +++ b/frappe/contacts/address_and_contact.py @@ -170,7 +170,9 @@ def delete_contact_and_address(doctype, docname): @frappe.whitelist() @frappe.validate_and_sanitize_search_inputs -def filter_dynamic_link_doctypes(txt: str, filters: Dict) -> List[List[str]]: +def filter_dynamic_link_doctypes( + doctype, txt: str, searchfield, start, page_len, filters: Dict +) -> List[List[str]]: from frappe.permissions import get_doctypes_with_read txt = txt or "" From 05ea37c779c9b9e80bc6b2fc0a5a480b7b595ae8 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Fri, 17 Jun 2022 16:16:58 +0530 Subject: [PATCH 127/205] fix: Set first visible tab as active --- frappe/public/js/frappe/form/form.js | 2 -- frappe/public/js/frappe/form/layout.js | 10 ++++++---- frappe/public/js/frappe/form/tab.js | 3 +-- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/frappe/public/js/frappe/form/form.js b/frappe/public/js/frappe/form/form.js index eefc629b4d..98a02850c1 100644 --- a/frappe/public/js/frappe/form/form.js +++ b/frappe/public/js/frappe/form/form.js @@ -575,8 +575,6 @@ frappe.ui.form.Form = class FrappeForm { this.$wrapper.trigger('render_complete'); - this.layout.set_first_tab_as_active(switched || this.cscript.is_onload); - if(!this.hidden) { this.layout.show_empty_form_message(); } diff --git a/frappe/public/js/frappe/form/layout.js b/frappe/public/js/frappe/form/layout.js index 080f8e0180..b808bdc7dc 100644 --- a/frappe/public/js/frappe/form/layout.js +++ b/frappe/public/js/frappe/form/layout.js @@ -123,7 +123,7 @@ frappe.ui.form.Layout = class Layout { if (this.is_tabbed_layout()) { // add a tab without `fieldname` to avoid conflicts - let default_tab = {label: __('Details'), fieldtype: "Tab Break"}; + let default_tab = {label: __('Details'), fieldtype: "Tab Break", fieldname: "__details"}; let first_tab = this.fields[1].fieldtype === "Tab Break" ? this.fields[1] : null; if (!first_tab) { this.fields.splice(1, 0, default_tab); @@ -336,12 +336,14 @@ frappe.ui.form.Layout = class Layout { if (visible_tabs && visible_tabs.length == 1) { visible_tabs[0].parent.toggleClass('hide show'); } + this.set_first_tab_as_active(); } - set_first_tab_as_active(switched) { - if (this.tabs.length && (switched || !this.frm.active_tab)) { + set_first_tab_as_active() { + if (this.tabs.length && !this.frm.active_tab) { // set first tab as active when opening for first time, or new doc - this.tabs[0].set_active(); + let first_visible_tab = this.tabs.find(tab => !tab.is_hidden()); + first_visible_tab.set_active(); } } diff --git a/frappe/public/js/frappe/form/tab.js b/frappe/public/js/frappe/form/tab.js index 69c573186b..5c95fad582 100644 --- a/frappe/public/js/frappe/form/tab.js +++ b/frappe/public/js/frappe/form/tab.js @@ -87,7 +87,6 @@ export default class Tab { } is_hidden() { - this.wrapper.hasClass('hide') - && this.parent.hasClass('hide'); + return this.wrapper.hasClass('hide'); } } From 3f2901d837d6aabbb28aafc8c9ea4f121848f9ca Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Fri, 17 Jun 2022 16:56:11 +0530 Subject: [PATCH 128/205] fix: Check if visible tab exists --- frappe/public/js/frappe/form/layout.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/form/layout.js b/frappe/public/js/frappe/form/layout.js index b808bdc7dc..f46f70ca48 100644 --- a/frappe/public/js/frappe/form/layout.js +++ b/frappe/public/js/frappe/form/layout.js @@ -343,7 +343,7 @@ frappe.ui.form.Layout = class Layout { if (this.tabs.length && !this.frm.active_tab) { // set first tab as active when opening for first time, or new doc let first_visible_tab = this.tabs.find(tab => !tab.is_hidden()); - first_visible_tab.set_active(); + first_visible_tab && first_visible_tab.set_active(); } } From ee7cb22cd64e2614b05ed5a83db6756ad71e982c Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Fri, 17 Jun 2022 18:17:54 +0530 Subject: [PATCH 129/205] fix: Remember active tab for a document in a browsing session --- frappe/public/js/frappe/form/form.js | 9 +++++++++ frappe/public/js/frappe/form/layout.js | 9 ++++++--- frappe/public/js/frappe/form/tab.js | 8 +++++++- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/frappe/public/js/frappe/form/form.js b/frappe/public/js/frappe/form/form.js index 98a02850c1..13d61d689b 100644 --- a/frappe/public/js/frappe/form/form.js +++ b/frappe/public/js/frappe/form/form.js @@ -1840,6 +1840,15 @@ frappe.ui.form.Form = class FrappeForm { }); }); } + set_active_tab(tab) { + if (!this.active_tab_map) { + this.active_tab_map = {}; + } + this.active_tab_map[this.docname] = tab; + } + get_active_tab() { + return this.active_tab_map && this.active_tab_map[this.docname]; + } }; frappe.validated = 0; diff --git a/frappe/public/js/frappe/form/layout.js b/frappe/public/js/frappe/form/layout.js index f46f70ca48..add9ce7b8b 100644 --- a/frappe/public/js/frappe/form/layout.js +++ b/frappe/public/js/frappe/form/layout.js @@ -336,11 +336,14 @@ frappe.ui.form.Layout = class Layout { if (visible_tabs && visible_tabs.length == 1) { visible_tabs[0].parent.toggleClass('hide show'); } - this.set_first_tab_as_active(); + this.set_tab_as_active(); } - set_first_tab_as_active() { - if (this.tabs.length && !this.frm.active_tab) { + set_tab_as_active() { + let frm_active_tab = this?.frm.get_active_tab?.(); + if (frm_active_tab) { + frm_active_tab.set_active(); + } else if (this.tabs.length) { // set first tab as active when opening for first time, or new doc let first_visible_tab = this.tabs.find(tab => !tab.is_hidden()); first_visible_tab && first_visible_tab.set_active(); diff --git a/frappe/public/js/frappe/form/tab.js b/frappe/public/js/frappe/form/tab.js index 5c95fad582..324d0c50c8 100644 --- a/frappe/public/js/frappe/form/tab.js +++ b/frappe/public/js/frappe/form/tab.js @@ -10,6 +10,7 @@ export default class Tab { this.fields_list = []; this.fields_dict = {}; this.make(); + this.setup_listeners(); this.refresh(); } @@ -79,7 +80,6 @@ export default class Tab { set_active() { this.parent.find('.nav-link').tab('show'); this.wrapper.addClass('active'); - this.frm.active_tab = this; } is_active() { @@ -89,4 +89,10 @@ export default class Tab { is_hidden() { return this.wrapper.hasClass('hide'); } + + setup_listeners() { + this.parent.find('.nav-link').on('shown.bs.tab', () => { + this?.frm.set_active_tab?.(this); + }); + } } From 2ce6597a692d60b1dd47c22c73499e5876a6d2f3 Mon Sep 17 00:00:00 2001 From: phot0n Date: Fri, 17 Jun 2022 18:47:06 +0530 Subject: [PATCH 130/205] fix: cast list to tuple when sending parameterized query for postgres --- frappe/database/postgres/database.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/frappe/database/postgres/database.py b/frappe/database/postgres/database.py index 14872b2b16..b977184e18 100644 --- a/frappe/database/postgres/database.py +++ b/frappe/database/postgres/database.py @@ -418,10 +418,16 @@ def modify_values(values): if isinstance(values, dict): for k, v in values.items(): + if isinstance(v, list): + v = tuple(v) + values[k] = stringify_value(v) elif isinstance(values, (tuple, list)): new_values = [] for val in values: + if isinstance(val, list): + val = tuple(val) + new_values.append(stringify_value(val)) values = new_values else: From 23bd9749f670dcc26692faa9dd31e8fe870cb82e Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sun, 5 Jun 2022 19:36:38 +0530 Subject: [PATCH 131/205] feat(UX): refresh URL when updating filters --- frappe/public/js/frappe/list/list_view.js | 34 ++++------------------- 1 file changed, 5 insertions(+), 29 deletions(-) diff --git a/frappe/public/js/frappe/list/list_view.js b/frappe/public/js/frappe/list/list_view.js index d57c57a0b6..1ca37e1138 100644 --- a/frappe/public/js/frappe/list/list_view.js +++ b/frappe/public/js/frappe/list/list_view.js @@ -1454,7 +1454,11 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList { on_update() {} - get_share_url() { + on_filter_change() { + window.history.replaceState(null, null, this.get_url_with_filters()); + } + + get_url_with_filters() { const query_params = this.get_filters_for_args() .map((filter) => { filter[3] = encodeURIComponent(filter[3]); @@ -1476,27 +1480,6 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList { return full_url; } - share_url() { - const d = new frappe.ui.Dialog({ - title: __("Share URL"), - fields: [ - { - fieldtype: "Code", - fieldname: "url", - label: "URL", - default: this.get_share_url(), - read_only: 1, - }, - ], - primary_action_label: __("Copy to clipboard"), - primary_action: () => { - frappe.utils.copy_to_clipboard(this.get_share_url()); - d.hide(); - }, - }); - d.show(); - } - get_menu_items() { const doctype = this.doctype; const items = []; @@ -1561,13 +1544,6 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList { shortcut: "Ctrl+K", }); - items.push({ - label: __("Share URL", null, "Button in list view menu"), - action: () => this.share_url(), - standard: true, - shortcut: "Ctrl+L", - }); - if ( frappe.user.has_role("System Manager") && frappe.boot.developer_mode === 1 From e0e0c6f219c057119ba0afd3abb25caa9c0f96f7 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sun, 5 Jun 2022 19:41:48 +0530 Subject: [PATCH 132/205] fix: remove query params from existing URL --- frappe/public/js/frappe/list/list_view.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/list/list_view.js b/frappe/public/js/frappe/list/list_view.js index 1ca37e1138..68d4b9417b 100644 --- a/frappe/public/js/frappe/list/list_view.js +++ b/frappe/public/js/frappe/list/list_view.js @@ -1473,7 +1473,7 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList { }) .join("&"); - let full_url = window.location.href; + let full_url = window.location.href.replace(window.location.search, ""); if (query_params) { full_url += "?" + query_params; } From 813793a70074c59e81e102d449114071484c3192 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sun, 5 Jun 2022 20:29:06 +0530 Subject: [PATCH 133/205] fix: view routing from URL query paramters --- frappe/public/js/frappe/router.js | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/frappe/public/js/frappe/router.js b/frappe/public/js/frappe/router.js index 39e6555654..a6bff4d72f 100644 --- a/frappe/public/js/frappe/router.js +++ b/frappe/public/js/frappe/router.js @@ -121,7 +121,7 @@ frappe.router = { route = this.get_sub_path_string(route).split('/'); if (!route) return []; route = $.map(route, this.decode_component); - this.set_route_options_from_url(route); + this.set_route_options_from_url(); return this.convert_to_standard_route(route); }, @@ -410,18 +410,17 @@ frappe.router = { return route; }, - set_route_options_from_url(route) { + set_route_options_from_url() { // set query parameters as frappe.route_options - var last_part = route[route.length - 1]; - if (last_part.indexOf("?") < last_part.indexOf("=")) { - // has ? followed by = - let parts = last_part.split("?"); + let query_string = window.location.search; - // route should not contain string after ? - route[route.length - 1] = parts[0]; + if (!frappe.route_options) { + frappe.route_options = {}; + } - let query_params = frappe.utils.get_query_params(parts[1]); - frappe.route_options = $.extend(frappe.route_options || {}, query_params); + let params = new URLSearchParams(query_string); + for (const [key, value] of params) { + frappe.route_options[key] = value; } }, From f4e8dee669e7922d27618e3c66fa10836e49326a Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sun, 5 Jun 2022 22:19:20 +0530 Subject: [PATCH 134/205] fix: double urlencoding of values This was breaking URLs for complex filters --- frappe/public/js/frappe/list/list_view.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/frappe/public/js/frappe/list/list_view.js b/frappe/public/js/frappe/list/list_view.js index 68d4b9417b..da6d2a78f6 100644 --- a/frappe/public/js/frappe/list/list_view.js +++ b/frappe/public/js/frappe/list/list_view.js @@ -1461,9 +1461,8 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList { get_url_with_filters() { const query_params = this.get_filters_for_args() .map((filter) => { - filter[3] = encodeURIComponent(filter[3]); if (filter[2] === "=") { - return `${filter[1]}=${filter[3]}`; + return `${filter[1]}=${encodeURIComponent(filter[3])}`; } return [ filter[1], From b3b382991293e683301b8180462e7d9b95d36c35 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sun, 5 Jun 2022 23:04:47 +0530 Subject: [PATCH 135/205] fix: don't forget 'like' filters Currently like filters get reset if field is part of standard filters --- frappe/public/js/frappe/list/base_list.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/list/base_list.js b/frappe/public/js/frappe/list/base_list.js index b91693257a..bbee90048b 100644 --- a/frappe/public/js/frappe/list/base_list.js +++ b/frappe/public/js/frappe/list/base_list.js @@ -621,6 +621,8 @@ class FilterArea { filters = filters.filter(f => !this.exists(f)); + // standard filters = filters visible on list view + // non-standard filters = filters set by filter button const { non_standard_filters, promise } = this.set_standard_filter( filters ); @@ -680,9 +682,14 @@ class FilterArea { out.promise = out.promise || Promise.resolve(); out.non_standard_filters = out.non_standard_filters || []; + // set in list view area if filters are present + // don't set like filter on link fields (gets reset) if ( fields_dict[fieldname] && - (condition === "=" || condition === "like") + ( + condition === "=" || + (condition === "like" && fields_dict[fieldname]?.df?.fieldtype != "Link") + ) ) { // standard filter out.promise = out.promise.then(() => From 69b7c72be86d5eca72fea269d1308804f22bca7e Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sun, 5 Jun 2022 23:34:15 +0530 Subject: [PATCH 136/205] fix: kanban board from shared links - Apply filter from URL - Update URL in browser when filters change --- frappe/public/js/frappe/views/kanban/kanban_view.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/frappe/public/js/frappe/views/kanban/kanban_view.js b/frappe/public/js/frappe/views/kanban/kanban_view.js index b546365fd9..02acfef278 100644 --- a/frappe/public/js/frappe/views/kanban/kanban_view.js +++ b/frappe/public/js/frappe/views/kanban/kanban_view.js @@ -64,10 +64,6 @@ frappe.views.KanbanView = class KanbanView extends frappe.views.ListView { }); } - before_refresh() { - - } - setup_page() { this.hide_sidebar = true; this.hide_page_form = true; @@ -105,6 +101,7 @@ frappe.views.KanbanView = class KanbanView extends frappe.views.ListView { } else { this.page.clear_indicator(); } + super.on_filter_change(); } save_kanban_board_filters() { @@ -154,7 +151,10 @@ frappe.views.KanbanView = class KanbanView extends frappe.views.ListView { get_card_meta() { var meta = frappe.get_meta(this.doctype); + // preserve route options erased by new doc + let route_options = {...frappe.route_options}; var doc = frappe.model.get_new_doc(this.doctype); + frappe.route_options = route_options; var title_field = null; var quick_entry = false; From 37999ea17e773fe07326310888ab4491727f3c5a Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 6 Jun 2022 01:12:05 +0530 Subject: [PATCH 137/205] fix: handle multiselect report options --- frappe/public/js/frappe/views/reports/query_report.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/views/reports/query_report.js b/frappe/public/js/frappe/views/reports/query_report.js index 7d00f3ed13..a4da6c84c3 100644 --- a/frappe/public/js/frappe/views/reports/query_report.js +++ b/frappe/public/js/frappe/views/reports/query_report.js @@ -538,7 +538,11 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList { const promises = filters_to_set.map(f => { return () => { - const value = route_options[f.df.fieldname]; + let value = route_options[f.df.fieldname]; + if (typeof value === 'string' && value[0] === '[') { + // multiselect array + value = JSON.parse(value); + } f.set_value(value); }; }); From 90bcbe46f12d1d78eb70f88f735d26f2b2147e05 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 6 Jun 2022 01:12:27 +0530 Subject: [PATCH 138/205] feat(ux): Always sharable query report URLs --- .../js/frappe/views/reports/query_report.js | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/frappe/public/js/frappe/views/reports/query_report.js b/frappe/public/js/frappe/views/reports/query_report.js index a4da6c84c3..8f1cecb965 100644 --- a/frappe/public/js/frappe/views/reports/query_report.js +++ b/frappe/public/js/frappe/views/reports/query_report.js @@ -57,6 +57,30 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList { this.menu_items = []; } + on_filter_change() { + window.history.replaceState(null, null, this.get_url_with_filters()); + } + + get_url_with_filters() { + const query_params = Object.entries(this.get_filter_values()) + .map(([field, value], _idx) => { + // multiselects + if (Array.isArray(value)) { + if (!value.length) return ''; + value = JSON.stringify(value); + } + return `${field}=${encodeURIComponent(value)}`; + }) + .filter(Boolean) + .join("&"); + + let full_url = window.location.href.replace(window.location.search, ""); + if (query_params) { + full_url += "?" + query_params; + } + return full_url; + } + set_default_secondary_action() { this.refresh_button && this.refresh_button.remove(); this.refresh_button = this.page.add_action_icon("refresh", () => { @@ -482,6 +506,7 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList { if (df.on_change) f.on_change = df.on_change; df.onchange = () => { + this.on_filter_change(); this.refresh_filters_dependency(); let current_filters = this.get_filter_values(); From 61c895c2fc9488abdbccbe1f17ac73705ade24b2 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Thu, 16 Jun 2022 15:17:48 +0530 Subject: [PATCH 139/205] refactor: update listview URL on all refresh Previous changes only made it refresh on change of filters, this change now updates URL ALWAYS. Co-Auhtored-By: Suraj Shetty --- frappe/public/js/frappe/list/list_view.js | 3 ++- frappe/public/js/frappe/views/kanban/kanban_view.js | 1 - frappe/public/js/frappe/views/reports/query_report.js | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/frappe/public/js/frappe/list/list_view.js b/frappe/public/js/frappe/list/list_view.js index da6d2a78f6..3a39a949c9 100644 --- a/frappe/public/js/frappe/list/list_view.js +++ b/frappe/public/js/frappe/list/list_view.js @@ -291,6 +291,7 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList { super.refresh().then(() => { this.render_header(refresh_header); this.update_checkbox(); + this.update_url_with_filters(); }); } @@ -1454,7 +1455,7 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList { on_update() {} - on_filter_change() { + update_url_with_filters() { window.history.replaceState(null, null, this.get_url_with_filters()); } diff --git a/frappe/public/js/frappe/views/kanban/kanban_view.js b/frappe/public/js/frappe/views/kanban/kanban_view.js index 02acfef278..129db13b07 100644 --- a/frappe/public/js/frappe/views/kanban/kanban_view.js +++ b/frappe/public/js/frappe/views/kanban/kanban_view.js @@ -101,7 +101,6 @@ frappe.views.KanbanView = class KanbanView extends frappe.views.ListView { } else { this.page.clear_indicator(); } - super.on_filter_change(); } save_kanban_board_filters() { diff --git a/frappe/public/js/frappe/views/reports/query_report.js b/frappe/public/js/frappe/views/reports/query_report.js index 8f1cecb965..ac7f8327ab 100644 --- a/frappe/public/js/frappe/views/reports/query_report.js +++ b/frappe/public/js/frappe/views/reports/query_report.js @@ -57,7 +57,7 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList { this.menu_items = []; } - on_filter_change() { + update_url_with_filters() { window.history.replaceState(null, null, this.get_url_with_filters()); } @@ -506,7 +506,6 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList { if (df.on_change) f.on_change = df.on_change; df.onchange = () => { - this.on_filter_change(); this.refresh_filters_dependency(); let current_filters = this.get_filter_values(); @@ -681,6 +680,7 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList { frappe.hide_progress(); }).finally(() => { this.hide_loading_screen(); + this.update_url_with_filters(); }); } From 80de4349848be66cbbde9d4a78d72e52205db09c Mon Sep 17 00:00:00 2001 From: vishdha Date: Fri, 17 Jun 2022 20:07:14 +0530 Subject: [PATCH 140/205] chore: Test case added for datetime, date and time field --- frappe/tests/test_naming.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/frappe/tests/test_naming.py b/frappe/tests/test_naming.py index cb95cd60b3..f4cc28daef 100644 --- a/frappe/tests/test_naming.py +++ b/frappe/tests/test_naming.py @@ -12,7 +12,7 @@ from frappe.model.naming import ( revert_series_if_last, ) from frappe.tests.utils import FrappeTestCase -from frappe.utils import now_datetime, nowdate +from frappe.utils import now_datetime, nowdate, nowtime class TestNaming(FrappeTestCase): @@ -100,22 +100,26 @@ class TestNaming(FrappeTestCase): self.assertEqual(doc.name, f"TODO-{now_datetime().strftime('%m')}-{description}-{series:02}") - def test_format_autoname_for_date_field(self): + def test_format_autoname_for_datetime_field(self): """ - Test if braced params are replaced in format autoname for date field + Test if braced params are replaced in format autoname for datetime, date and time field """ - doctype = new_doctype(autoname="format:TODO-{date}-{##}").insert() + datetime = now_datetime() date = nowdate() + time = nowtime() - doc = frappe.new_doc(doctype.name) - doc.date = date - doc.insert() + for field in [datetime, date, time]: + doctype = new_doctype(autoname="format:TODO-{field}-{##}").insert() - series = getseries("", 2) - series = int(series) - 1 + doc = frappe.new_doc(doctype.name) + doc.field = field + doc.insert() - self.assertEqual(doc.name, f"TODO-{date}-{series:02}") + series = getseries("", 2) + series = int(series) - 1 + + self.assertEqual(doc.name, f"TODO-{field}-{series:02}") def test_format_autoname_for_consecutive_week_number(self): """ From 54824071c2f9b80c0d75b056dc39bba7cf95bea9 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Fri, 17 Jun 2022 20:47:55 +0530 Subject: [PATCH 141/205] test: simple test for URL routing with params --- cypress/integration/routing.js | 36 ++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 cypress/integration/routing.js diff --git a/cypress/integration/routing.js b/cypress/integration/routing.js new file mode 100644 index 0000000000..ad07754010 --- /dev/null +++ b/cypress/integration/routing.js @@ -0,0 +1,36 @@ +const list_view = "/app/todo"; + +// test round trip with filter types + +const test_queries = [ + "?status=Open", + `?date=%5B"Between"%2C%5B"2022-06-01"%2C"2022-06-30"%5D%5D`, + `?date=%5B">"%2C"2022-06-01"%5D`, + `?name=%5B"like"%2C"%2542%25"%5D`, + `?status=%5B"not%20in"%2C%5B"Open"%2C"Closed"%5D%5D`, +]; + +describe("SPA Routing", { scrollBehavior: false }, () => { + before(() => { + cy.login(); + cy.go_to_list("ToDo"); + }); + + it("should apply filter on list view from route", () => { + test_queries.forEach((query) => { + const full_url = `${list_view}${query}`; + cy.visit(full_url); + cy.findByTitle("To Do").should("exist"); + + const expected = new URLSearchParams(query); + cy.location().then((loc) => { + const actual = new URLSearchParams(loc.search); + // This might appear like a dumb test checking visited URL to itself + // but it's actually doing a round trip + // URL with params -> parsed filters -> new URL + // if it's same that means everything worked in between. + expect(actual.toString()).to.eq(expected.toString()); + }); + }); + }); +}); From f6c1eb10dd3e0c7cd8a5c11292d4564e4cb46e92 Mon Sep 17 00:00:00 2001 From: phot0n Date: Sat, 18 Jun 2022 10:11:58 +0530 Subject: [PATCH 142/205] refactor(minor): modify_values * don't truncate float(s) --- frappe/database/postgres/database.py | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/frappe/database/postgres/database.py b/frappe/database/postgres/database.py index b977184e18..f9a4723d72 100644 --- a/frappe/database/postgres/database.py +++ b/frappe/database/postgres/database.py @@ -403,13 +403,12 @@ def modify_query(query): def modify_values(values): - def stringify_value(value): - if isinstance(value, int): + def modify_value(value): + if isinstance(value, (list, tuple)): + value = tuple(modify_values(value)) + + elif isinstance(value, int): value = str(value) - elif isinstance(value, float): - truncated_float = int(value) - if value == truncated_float: - value = str(truncated_float) return value @@ -418,20 +417,15 @@ def modify_values(values): if isinstance(values, dict): for k, v in values.items(): - if isinstance(v, list): - v = tuple(v) - - values[k] = stringify_value(v) + values[k] = modify_value(v) elif isinstance(values, (tuple, list)): new_values = [] for val in values: - if isinstance(val, list): - val = tuple(val) + new_values.append(modify_value(val)) - new_values.append(stringify_value(val)) values = new_values else: - values = stringify_value(values) + values = modify_value(values) return values From 215507a742b5cbc0626324a416a5b18ede804dbf Mon Sep 17 00:00:00 2001 From: phot0n Date: Sat, 18 Jun 2022 10:45:40 +0530 Subject: [PATCH 143/205] test: add list value in test_modify_values --- frappe/tests/test_db.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/frappe/tests/test_db.py b/frappe/tests/test_db.py index 73b5446404..31fccd0132 100644 --- a/frappe/tests/test_db.py +++ b/frappe/tests/test_db.py @@ -834,10 +834,13 @@ class TestDDLCommandsPost(unittest.TestCase): from frappe.database.postgres.database import modify_values self.assertEqual( - {"abcd": "23", "efgh": "23", "ijkl": 23.0345, "mnop": "wow"}, - modify_values({"abcd": 23, "efgh": 23.0, "ijkl": 23.0345, "mnop": "wow"}), + {"a": "23", "b": 23.0, "c": 23.0345, "d": "wow", "e": ("1", "2", "3", "abc")}, + modify_values({"a": 23, "b": 23.0, "c": 23.0345, "d": "wow", "e": [1, 2, 3, "abc"]}), + ) + self.assertEqual( + ["23", 23.0, 23.00004345, "wow", ("1", "2", "3", "abc")], + modify_values((23, 23.0, 23.00004345, "wow", [1, 2, 3, "abc"])), ) - self.assertEqual(["23", "23", 23.00004345, "wow"], modify_values((23, 23.0, 23.00004345, "wow"))) def test_sequence_table_creation(self): from frappe.core.doctype.doctype.test_doctype import new_doctype From 5e1007940c8dea64823ebe9a69b8c0ffc37e4ac9 Mon Sep 17 00:00:00 2001 From: phot0n Date: Sat, 18 Jun 2022 10:53:43 +0530 Subject: [PATCH 144/205] refactor(minor): move test_modify_query and test_modify_values to TestDB class --- frappe/tests/test_db.py | 60 +++++++++++++++++++++-------------------- 1 file changed, 31 insertions(+), 29 deletions(-) diff --git a/frappe/tests/test_db.py b/frappe/tests/test_db.py index 31fccd0132..69f7a7d5aa 100644 --- a/frappe/tests/test_db.py +++ b/frappe/tests/test_db.py @@ -511,6 +511,37 @@ class TestDB(unittest.TestCase): frappe.db.rollback() + @run_only_if(db_type_is.POSTGRES) + def test_modify_query(self): + from frappe.database.postgres.database import modify_query + + query = "select * from `tabtree b` where lft > 13 and rgt <= 16 and name =1.0 and parent = 4134qrsdc and isgroup = 1.00045" + self.assertEqual( + "select * from \"tabtree b\" where lft > '13' and rgt <= '16' and name = '1' and parent = 4134qrsdc and isgroup = 1.00045", + modify_query(query), + ) + + query = ( + 'select locate(".io", "frappe.io"), locate("3", cast(3 as varchar)), locate("3", 3::varchar)' + ) + self.assertEqual( + 'select strpos( "frappe.io", ".io"), strpos( cast(3 as varchar), "3"), strpos( 3::varchar, "3")', + modify_query(query), + ) + + @run_only_if(db_type_is.POSTGRES) + def test_modify_values(self): + from frappe.database.postgres.database import modify_values + + self.assertEqual( + {"a": "23", "b": 23.0, "c": 23.0345, "d": "wow", "e": ("1", "2", "3", "abc")}, + modify_values({"a": 23, "b": 23.0, "c": 23.0345, "d": "wow", "e": [1, 2, 3, "abc"]}), + ) + self.assertEqual( + ["23", 23.0, 23.00004345, "wow", ("1", "2", "3", "abc")], + modify_values((23, 23.0, 23.00004345, "wow", [1, 2, 3, "abc"])), + ) + @run_only_if(db_type_is.MARIADB) class TestDDLCommandsMaria(unittest.TestCase): @@ -813,35 +844,6 @@ class TestDDLCommandsPost(unittest.TestCase): ) self.assertEqual(len(indexs_in_table), 1) - def test_modify_query(self): - from frappe.database.postgres.database import modify_query - - query = "select * from `tabtree b` where lft > 13 and rgt <= 16 and name =1.0 and parent = 4134qrsdc and isgroup = 1.00045" - self.assertEqual( - "select * from \"tabtree b\" where lft > '13' and rgt <= '16' and name = '1' and parent = 4134qrsdc and isgroup = 1.00045", - modify_query(query), - ) - - query = ( - 'select locate(".io", "frappe.io"), locate("3", cast(3 as varchar)), locate("3", 3::varchar)' - ) - self.assertEqual( - 'select strpos( "frappe.io", ".io"), strpos( cast(3 as varchar), "3"), strpos( 3::varchar, "3")', - modify_query(query), - ) - - def test_modify_values(self): - from frappe.database.postgres.database import modify_values - - self.assertEqual( - {"a": "23", "b": 23.0, "c": 23.0345, "d": "wow", "e": ("1", "2", "3", "abc")}, - modify_values({"a": 23, "b": 23.0, "c": 23.0345, "d": "wow", "e": [1, 2, 3, "abc"]}), - ) - self.assertEqual( - ["23", 23.0, 23.00004345, "wow", ("1", "2", "3", "abc")], - modify_values((23, 23.0, 23.00004345, "wow", [1, 2, 3, "abc"])), - ) - def test_sequence_table_creation(self): from frappe.core.doctype.doctype.test_doctype import new_doctype From 7b67e1f847121f255c3262811bc54493a2563342 Mon Sep 17 00:00:00 2001 From: Raffael Meyer <14891507+barredterra@users.noreply.github.com> Date: Mon, 20 Jun 2022 07:05:57 +0200 Subject: [PATCH 145/205] fix: translate doctype in error messages (#17239) --- frappe/__init__.py | 2 +- frappe/client.py | 4 ++-- frappe/model/db_query.py | 2 +- frappe/share.py | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/frappe/__init__.py b/frappe/__init__.py index 76888c1824..1cf3526b45 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -926,7 +926,7 @@ def has_permission( if throw and not out: # mimics frappe.throw - document_label = f"{doc.doctype} {doc.name}" if doc else doctype + document_label = f"{_(doc.doctype)} {doc.name}" if doc else _(doctype) msgprint( _("No permission for {0}").format(document_label), raise_exception=ValidationError, diff --git a/frappe/client.py b/frappe/client.py index 4afe0898bc..154fcf31e2 100644 --- a/frappe/client.py +++ b/frappe/client.py @@ -101,7 +101,7 @@ def get_value(doctype, fieldname, filters=None, as_dict=True, debug=False, paren check_parent_permission(parent, doctype) if not frappe.has_permission(doctype, parent_doctype=parent): - frappe.throw(_("No permission for {0}").format(doctype), frappe.PermissionError) + frappe.throw(_("No permission for {0}").format(_(doctype)), frappe.PermissionError) filters = get_safe_filters(filters) if isinstance(filters, str): @@ -143,7 +143,7 @@ def get_value(doctype, fieldname, filters=None, as_dict=True, debug=False, paren @frappe.whitelist() def get_single_value(doctype, field): if not frappe.has_permission(doctype): - frappe.throw(_("No permission for {0}").format(doctype), frappe.PermissionError) + frappe.throw(_("No permission for {0}").format(_(doctype)), frappe.PermissionError) value = frappe.db.get_single_value(doctype, field) return value diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py index 82913db98d..7fb38848e2 100644 --- a/frappe/model/db_query.py +++ b/frappe/model/db_query.py @@ -743,7 +743,7 @@ class DatabaseQuery(object): ): only_if_shared = True if not self.shared: - frappe.throw(_("No permission to read {0}").format(self.doctype), frappe.PermissionError) + frappe.throw(_("No permission to read {0}").format(_(self.doctype)), frappe.PermissionError) else: self.conditions.append(self.get_share_condition()) diff --git a/frappe/share.py b/frappe/share.py index 3edcb1be38..dfb4836995 100644 --- a/frappe/share.py +++ b/frappe/share.py @@ -175,7 +175,7 @@ def check_share_permission(doctype, name): """Check if the user can share with other users""" if not frappe.has_permission(doctype, ptype="share", doc=name): frappe.throw( - _("No permission to {0} {1} {2}").format("share", doctype, name), frappe.PermissionError + _("No permission to {0} {1} {2}").format("share", _(doctype), name), frappe.PermissionError ) @@ -190,7 +190,7 @@ def notify_assignment(shared_by, doctype, doc_name, everyone, notify=0): reference_user = get_fullname(frappe.session.user) notification_message = _("{0} shared a document {1} {2} with you").format( - frappe.bold(reference_user), frappe.bold(doctype), get_title_html(title) + frappe.bold(reference_user), frappe.bold(_(doctype)), get_title_html(title) ) notification_doc = { From e0e452bdcc1c8dd03e4261e24a48abc8bdf095ad Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 20 Jun 2022 11:11:10 +0530 Subject: [PATCH 146/205] fix: incorrect filtering on address doctype (#17241) [skip ci] --- frappe/contacts/address_and_contact.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/frappe/contacts/address_and_contact.py b/frappe/contacts/address_and_contact.py index 1461690378..8d078196e0 100644 --- a/frappe/contacts/address_and_contact.py +++ b/frappe/contacts/address_and_contact.py @@ -177,7 +177,6 @@ def filter_dynamic_link_doctypes( txt = txt or "" filters = filters or {} - TXT_PATTERN = re.compile(f"{txt}.*") _doctypes_from_df = frappe.get_all( "DocField", @@ -186,13 +185,13 @@ def filter_dynamic_link_doctypes( distinct=True, order_by=None, ) - doctypes_from_df = {d for d in _doctypes_from_df if TXT_PATTERN.search(_(d), re.IGNORECASE)} + doctypes_from_df = {d for d in _doctypes_from_df if txt.lower() in d.lower()} filters.update({"dt": ("not in", doctypes_from_df)}) _doctypes_from_cdf = frappe.get_all( "Custom Field", filters=filters, pluck="dt", distinct=True, order_by=None ) - doctypes_from_cdf = {d for d in _doctypes_from_cdf if TXT_PATTERN.search(_(d), re.IGNORECASE)} + doctypes_from_cdf = {d for d in _doctypes_from_cdf if txt.lower() in d.lower()} all_doctypes = doctypes_from_df.union(doctypes_from_cdf) allowed_doctypes = set(get_doctypes_with_read()) From 28fd355f247d169fe656c7610b034e69777c3543 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 20 Jun 2022 11:16:24 +0530 Subject: [PATCH 147/205] chore: translate before searching doctype --- frappe/contacts/address_and_contact.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/contacts/address_and_contact.py b/frappe/contacts/address_and_contact.py index 8d078196e0..1c5803ffea 100644 --- a/frappe/contacts/address_and_contact.py +++ b/frappe/contacts/address_and_contact.py @@ -185,13 +185,13 @@ def filter_dynamic_link_doctypes( distinct=True, order_by=None, ) - doctypes_from_df = {d for d in _doctypes_from_df if txt.lower() in d.lower()} + doctypes_from_df = {d for d in _doctypes_from_df if txt.lower() in _(d).lower()} filters.update({"dt": ("not in", doctypes_from_df)}) _doctypes_from_cdf = frappe.get_all( "Custom Field", filters=filters, pluck="dt", distinct=True, order_by=None ) - doctypes_from_cdf = {d for d in _doctypes_from_cdf if txt.lower() in d.lower()} + doctypes_from_cdf = {d for d in _doctypes_from_cdf if txt.lower() in _(d).lower()} all_doctypes = doctypes_from_df.union(doctypes_from_cdf) allowed_doctypes = set(get_doctypes_with_read()) From dd8aabcc2f7225be6f055b16df304936c2805efa Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Jun 2022 11:48:06 +0530 Subject: [PATCH 148/205] build(deps): bump actions/setup-python from 2 to 4 (#17218) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 2 to 4. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v2...v4) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/docs-checker.yml | 2 +- .github/workflows/linters.yml | 2 +- .github/workflows/patch-mariadb-tests.yml | 2 +- .github/workflows/publish-assets-develop.yml | 2 +- .github/workflows/publish-assets-releases.yml | 2 +- .github/workflows/server-mariadb-tests.yml | 2 +- .github/workflows/server-postgres-tests.yml | 2 +- .github/workflows/ui-tests.yml | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/docs-checker.yml b/.github/workflows/docs-checker.yml index a0f77b43fd..80b2469c0c 100644 --- a/.github/workflows/docs-checker.yml +++ b/.github/workflows/docs-checker.yml @@ -13,7 +13,7 @@ jobs: steps: - name: 'Setup Environment' - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: 3.8 diff --git a/.github/workflows/linters.yml b/.github/workflows/linters.yml index 443ee45bf7..7a17648ff0 100644 --- a/.github/workflows/linters.yml +++ b/.github/workflows/linters.yml @@ -12,7 +12,7 @@ jobs: - uses: actions/checkout@v2 - name: Set up Python 3.8 - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: 3.8 diff --git a/.github/workflows/patch-mariadb-tests.yml b/.github/workflows/patch-mariadb-tests.yml index 7dffc30dc0..8f787a145b 100644 --- a/.github/workflows/patch-mariadb-tests.yml +++ b/.github/workflows/patch-mariadb-tests.yml @@ -31,7 +31,7 @@ jobs: uses: actions/checkout@v2 - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: '3.9' diff --git a/.github/workflows/publish-assets-develop.yml b/.github/workflows/publish-assets-develop.yml index f56d1460b5..161f410d87 100644 --- a/.github/workflows/publish-assets-develop.yml +++ b/.github/workflows/publish-assets-develop.yml @@ -16,7 +16,7 @@ jobs: - uses: actions/setup-node@v1 with: node-version: 14 - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v4 with: python-version: '3.9' - name: Set up bench and build assets diff --git a/.github/workflows/publish-assets-releases.yml b/.github/workflows/publish-assets-releases.yml index faf41ed23e..a18b70e717 100644 --- a/.github/workflows/publish-assets-releases.yml +++ b/.github/workflows/publish-assets-releases.yml @@ -19,7 +19,7 @@ jobs: - uses: actions/setup-node@v1 with: python-version: '12.x' - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v4 with: python-version: '3.9' - name: Set up bench and build assets diff --git a/.github/workflows/server-mariadb-tests.yml b/.github/workflows/server-mariadb-tests.yml index 33fc221e80..a231808120 100644 --- a/.github/workflows/server-mariadb-tests.yml +++ b/.github/workflows/server-mariadb-tests.yml @@ -40,7 +40,7 @@ jobs: uses: actions/checkout@v2 - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: '3.9' diff --git a/.github/workflows/server-postgres-tests.yml b/.github/workflows/server-postgres-tests.yml index 2b4a2edae0..169630da70 100644 --- a/.github/workflows/server-postgres-tests.yml +++ b/.github/workflows/server-postgres-tests.yml @@ -43,7 +43,7 @@ jobs: uses: actions/checkout@v2 - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: '3.9' diff --git a/.github/workflows/ui-tests.yml b/.github/workflows/ui-tests.yml index 3518608f33..4a18cda02d 100644 --- a/.github/workflows/ui-tests.yml +++ b/.github/workflows/ui-tests.yml @@ -39,7 +39,7 @@ jobs: uses: actions/checkout@v2 - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: '3.9' From 7570c0e2a158b7923e6f826b674a080ae4526813 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Jun 2022 11:48:32 +0530 Subject: [PATCH 149/205] build(deps): bump actions/setup-node from 2 to 3 (#17222) Bumps [actions/setup-node](https://github.com/actions/setup-node) from 2 to 3. - [Release notes](https://github.com/actions/setup-node/releases) - [Commits](https://github.com/actions/setup-node/compare/v2...v3) --- updated-dependencies: - dependency-name: actions/setup-node dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/patch-mariadb-tests.yml | 2 +- .github/workflows/publish-assets-develop.yml | 2 +- .github/workflows/publish-assets-releases.yml | 2 +- .github/workflows/release.yml | 2 +- .github/workflows/server-mariadb-tests.yml | 2 +- .github/workflows/server-postgres-tests.yml | 2 +- .github/workflows/ui-tests.yml | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/patch-mariadb-tests.yml b/.github/workflows/patch-mariadb-tests.yml index 8f787a145b..982577a085 100644 --- a/.github/workflows/patch-mariadb-tests.yml +++ b/.github/workflows/patch-mariadb-tests.yml @@ -36,7 +36,7 @@ jobs: python-version: '3.9' - name: Setup Node - uses: actions/setup-node@v2 + uses: actions/setup-node@v3 with: node-version: 14 check-latest: true diff --git a/.github/workflows/publish-assets-develop.yml b/.github/workflows/publish-assets-develop.yml index 161f410d87..6cd3af78f2 100644 --- a/.github/workflows/publish-assets-develop.yml +++ b/.github/workflows/publish-assets-develop.yml @@ -13,7 +13,7 @@ jobs: - uses: actions/checkout@v2 with: path: 'frappe' - - uses: actions/setup-node@v1 + - uses: actions/setup-node@v3 with: node-version: 14 - uses: actions/setup-python@v4 diff --git a/.github/workflows/publish-assets-releases.yml b/.github/workflows/publish-assets-releases.yml index a18b70e717..f242a363b4 100644 --- a/.github/workflows/publish-assets-releases.yml +++ b/.github/workflows/publish-assets-releases.yml @@ -16,7 +16,7 @@ jobs: - uses: actions/checkout@v2 with: path: 'frappe' - - uses: actions/setup-node@v1 + - uses: actions/setup-node@v3 with: python-version: '12.x' - uses: actions/setup-python@v4 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e9936482b0..32c5d53831 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -17,7 +17,7 @@ jobs: fetch-depth: 0 persist-credentials: false - name: Setup Node.js v14 - uses: actions/setup-node@v2 + uses: actions/setup-node@v3 with: node-version: 14 - name: Setup dependencies diff --git a/.github/workflows/server-mariadb-tests.yml b/.github/workflows/server-mariadb-tests.yml index a231808120..b9c79db1e6 100644 --- a/.github/workflows/server-mariadb-tests.yml +++ b/.github/workflows/server-mariadb-tests.yml @@ -53,7 +53,7 @@ jobs: PR_NUMBER: ${{ github.event.number }} REPO_NAME: ${{ github.repository }} - - uses: actions/setup-node@v2 + - uses: actions/setup-node@v3 if: ${{ steps.check-build.outputs.build == 'strawberry' }} with: node-version: 14 diff --git a/.github/workflows/server-postgres-tests.yml b/.github/workflows/server-postgres-tests.yml index 169630da70..f4aa756181 100644 --- a/.github/workflows/server-postgres-tests.yml +++ b/.github/workflows/server-postgres-tests.yml @@ -56,7 +56,7 @@ jobs: PR_NUMBER: ${{ github.event.number }} REPO_NAME: ${{ github.repository }} - - uses: actions/setup-node@v2 + - uses: actions/setup-node@v3 if: ${{ steps.check-build.outputs.build == 'strawberry' }} with: node-version: '14' diff --git a/.github/workflows/ui-tests.yml b/.github/workflows/ui-tests.yml index 4a18cda02d..49426f5ad5 100644 --- a/.github/workflows/ui-tests.yml +++ b/.github/workflows/ui-tests.yml @@ -52,7 +52,7 @@ jobs: PR_NUMBER: ${{ github.event.number }} REPO_NAME: ${{ github.repository }} - - uses: actions/setup-node@v2 + - uses: actions/setup-node@v3 if: ${{ steps.check-build.outputs.build == 'strawberry' }} with: node-version: 14 From dd5a9e0d9c0ad390b4c87327954320b3600a5193 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Jun 2022 11:49:09 +0530 Subject: [PATCH 150/205] build(deps): bump actions/cache from 2 to 3 (#17220) Bumps [actions/cache](https://github.com/actions/cache) from 2 to 3. - [Release notes](https://github.com/actions/cache/releases) - [Changelog](https://github.com/actions/cache/blob/main/RELEASES.md) - [Commits](https://github.com/actions/cache/compare/v2...v3) --- updated-dependencies: - dependency-name: actions/cache dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/patch-mariadb-tests.yml | 6 +++--- .github/workflows/server-mariadb-tests.yml | 6 +++--- .github/workflows/server-postgres-tests.yml | 6 +++--- .github/workflows/ui-tests.yml | 8 ++++---- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/patch-mariadb-tests.yml b/.github/workflows/patch-mariadb-tests.yml index 982577a085..ae5802a8e1 100644 --- a/.github/workflows/patch-mariadb-tests.yml +++ b/.github/workflows/patch-mariadb-tests.yml @@ -56,7 +56,7 @@ jobs: - name: Cache pip if: ${{ steps.check-build.outputs.build == 'strawberry' }} - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ~/.cache/pip key: ${{ runner.os }}-pip-${{ hashFiles('**/*requirements.txt', '**/pyproject.toml', '**/setup.py') }} @@ -66,7 +66,7 @@ jobs: - name: Cache node modules if: ${{ steps.check-build.outputs.build == 'strawberry' }} - uses: actions/cache@v2 + uses: actions/cache@v3 env: cache-name: cache-node-modules with: @@ -82,7 +82,7 @@ jobs: id: yarn-cache-dir-path run: echo "::set-output name=dir::$(yarn cache dir)" - - uses: actions/cache@v2 + - uses: actions/cache@v3 if: ${{ steps.check-build.outputs.build == 'strawberry' }} id: yarn-cache with: diff --git a/.github/workflows/server-mariadb-tests.yml b/.github/workflows/server-mariadb-tests.yml index b9c79db1e6..b14f4bb0b4 100644 --- a/.github/workflows/server-mariadb-tests.yml +++ b/.github/workflows/server-mariadb-tests.yml @@ -67,7 +67,7 @@ jobs: - name: Cache pip if: ${{ steps.check-build.outputs.build == 'strawberry' }} - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ~/.cache/pip key: ${{ runner.os }}-pip-${{ hashFiles('**/*requirements.txt', '**/pyproject.toml', '**/setup.py') }} @@ -77,7 +77,7 @@ jobs: - name: Cache node modules if: ${{ steps.check-build.outputs.build == 'strawberry' }} - uses: actions/cache@v2 + uses: actions/cache@v3 env: cache-name: cache-node-modules with: @@ -93,7 +93,7 @@ jobs: id: yarn-cache-dir-path run: echo "::set-output name=dir::$(yarn cache dir)" - - uses: actions/cache@v2 + - uses: actions/cache@v3 if: ${{ steps.check-build.outputs.build == 'strawberry' }} id: yarn-cache with: diff --git a/.github/workflows/server-postgres-tests.yml b/.github/workflows/server-postgres-tests.yml index f4aa756181..e7e614f3e8 100644 --- a/.github/workflows/server-postgres-tests.yml +++ b/.github/workflows/server-postgres-tests.yml @@ -70,7 +70,7 @@ jobs: - name: Cache pip if: ${{ steps.check-build.outputs.build == 'strawberry' }} - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ~/.cache/pip key: ${{ runner.os }}-pip-${{ hashFiles('**/*requirements.txt', '**/pyproject.toml', '**/setup.py') }} @@ -80,7 +80,7 @@ jobs: - name: Cache node modules if: ${{ steps.check-build.outputs.build == 'strawberry' }} - uses: actions/cache@v2 + uses: actions/cache@v3 env: cache-name: cache-node-modules with: @@ -96,7 +96,7 @@ jobs: id: yarn-cache-dir-path run: echo "::set-output name=dir::$(yarn cache dir)" - - uses: actions/cache@v2 + - uses: actions/cache@v3 if: ${{ steps.check-build.outputs.build == 'strawberry' }} id: yarn-cache with: diff --git a/.github/workflows/ui-tests.yml b/.github/workflows/ui-tests.yml index 49426f5ad5..816085aa64 100644 --- a/.github/workflows/ui-tests.yml +++ b/.github/workflows/ui-tests.yml @@ -66,7 +66,7 @@ jobs: - name: Cache pip if: ${{ steps.check-build.outputs.build == 'strawberry' }} - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ~/.cache/pip key: ${{ runner.os }}-pip-${{ hashFiles('**/*requirements.txt', '**/pyproject.toml', '**/setup.py') }} @@ -76,7 +76,7 @@ jobs: - name: Cache node modules if: ${{ steps.check-build.outputs.build == 'strawberry' }} - uses: actions/cache@v2 + uses: actions/cache@v3 env: cache-name: cache-node-modules with: @@ -92,7 +92,7 @@ jobs: id: yarn-cache-dir-path run: echo "::set-output name=dir::$(yarn cache dir)" - - uses: actions/cache@v2 + - uses: actions/cache@v3 if: ${{ steps.check-build.outputs.build == 'strawberry' }} id: yarn-cache with: @@ -103,7 +103,7 @@ jobs: - name: Cache cypress binary if: ${{ steps.check-build.outputs.build == 'strawberry' }} - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ~/.cache key: ${{ runner.os }}-cypress- From b04bffe439213ae6e6406a58377c269456153653 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Jun 2022 11:50:10 +0530 Subject: [PATCH 151/205] build(deps): bump codecov/codecov-action from 2 to 3 (#17219) Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 2 to 3. - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/master/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/v2...v3) --- updated-dependencies: - dependency-name: codecov/codecov-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/server-mariadb-tests.yml | 2 +- .github/workflows/server-postgres-tests.yml | 2 +- .github/workflows/ui-tests.yml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/server-mariadb-tests.yml b/.github/workflows/server-mariadb-tests.yml index b14f4bb0b4..f7f3aae8d0 100644 --- a/.github/workflows/server-mariadb-tests.yml +++ b/.github/workflows/server-mariadb-tests.yml @@ -126,7 +126,7 @@ jobs: - name: Upload coverage data if: ${{ steps.check-build.outputs.build == 'strawberry' }} - uses: codecov/codecov-action@v2 + uses: codecov/codecov-action@v3 with: name: MariaDB fail_ci_if_error: true diff --git a/.github/workflows/server-postgres-tests.yml b/.github/workflows/server-postgres-tests.yml index e7e614f3e8..5116d4198e 100644 --- a/.github/workflows/server-postgres-tests.yml +++ b/.github/workflows/server-postgres-tests.yml @@ -129,7 +129,7 @@ jobs: - name: Upload coverage data if: ${{ steps.check-build.outputs.build == 'strawberry' }} - uses: codecov/codecov-action@v2 + uses: codecov/codecov-action@v3 with: name: Postgres fail_ci_if_error: true diff --git a/.github/workflows/ui-tests.yml b/.github/workflows/ui-tests.yml index 816085aa64..72fe3e67bb 100644 --- a/.github/workflows/ui-tests.yml +++ b/.github/workflows/ui-tests.yml @@ -158,7 +158,7 @@ jobs: - name: Upload Coverage Data if: ${{ steps.check-build.outputs.build == 'strawberry' && steps.check_coverage.outputs.files_exists == 'true' }} - uses: codecov/codecov-action@v2 + uses: codecov/codecov-action@v3 with: name: Cypress fail_ci_if_error: true @@ -168,7 +168,7 @@ jobs: - name: Upload Server Coverage Data if: ${{ steps.check-build.outputs.build-server == 'strawberry' }} - uses: codecov/codecov-action@v2 + uses: codecov/codecov-action@v3 with: name: MariaDB fail_ci_if_error: true From 6350406305f9445513b6db1d4c55e53a2dd251b2 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 20 Jun 2022 14:28:38 +0530 Subject: [PATCH 152/205] perf: remove duplicate control assets (#17237) * test: flaky tests due to uncleared filters * perf: remove duplicate code in assets Remove controls which are separately bundled too, saves ~1.1MB of network transfer. Co-authored-by: Suraj Shetty --- cypress/integration/awesome_bar.js | 1 + cypress/integration/routing.js | 4 ++++ frappe/public/js/form.bundle.js | 1 - 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/cypress/integration/awesome_bar.js b/cypress/integration/awesome_bar.js index e62ba6bec5..938034a34a 100644 --- a/cypress/integration/awesome_bar.js +++ b/cypress/integration/awesome_bar.js @@ -28,6 +28,7 @@ context('Awesome Bar', () => { cy.findByPlaceholderText('ID') .should('have.value', '%test%'); + cy.clear_filters(); }); it('navigates to new form', () => { diff --git a/cypress/integration/routing.js b/cypress/integration/routing.js index ad07754010..0822dd9b7d 100644 --- a/cypress/integration/routing.js +++ b/cypress/integration/routing.js @@ -16,6 +16,10 @@ describe("SPA Routing", { scrollBehavior: false }, () => { cy.go_to_list("ToDo"); }); + after(() => { + cy.clear_filters(); // avoid flake in future tests + }); + it("should apply filter on list view from route", () => { test_queries.forEach((query) => { const full_url = `${list_view}${query}`; diff --git a/frappe/public/js/form.bundle.js b/frappe/public/js/form.bundle.js index 2719535599..38d76f1f26 100644 --- a/frappe/public/js/form.bundle.js +++ b/frappe/public/js/form.bundle.js @@ -10,7 +10,6 @@ import "./frappe/form/templates/set_sharing.html"; import "./frappe/form/templates/timeline_message_box.html"; import "./frappe/form/templates/users_in_sidebar.html"; -import "./frappe/form/controls/control.js"; import "./frappe/views/formview.js"; import "./frappe/form/form.js"; import "./frappe/meta_tag.js"; From f40c82d4d8aeeb75ae042cd2c57acf7676ace2a7 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 20 Jun 2022 15:34:11 +0530 Subject: [PATCH 153/205] test(meta): control JS bundle sizes (#17244) Any drastic increase in production bundle sizes should fail in CI. --- frappe/tests/test_commands.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/frappe/tests/test_commands.py b/frappe/tests/test_commands.py index aeb7f364bc..16fea9155d 100644 --- a/frappe/tests/test_commands.py +++ b/frappe/tests/test_commands.py @@ -12,6 +12,7 @@ import unittest from contextlib import contextmanager from functools import wraps from glob import glob +from pathlib import Path from typing import List, Optional from unittest.case import skipIf from unittest.mock import patch @@ -31,6 +32,7 @@ from frappe.query_builder.utils import db_type_is from frappe.tests.test_query_builder import run_only_if from frappe.utils import add_to_date, get_bench_path, get_bench_relative_path, now from frappe.utils.backups import fetch_latest_backups +from frappe.utils.jinja_globals import bundled_asset _result: Optional[Result] = None TEST_SITE = "commands-site-O4PN2QKA.test" # added random string tag to avoid collisions @@ -692,7 +694,25 @@ class TestSiteMigration(BaseTestCommands): class TestBenchBuild(BaseTestCommands): - def test_build_assets(self): - with cli(frappe.commands.utils.build) as result: + def test_build_assets_size_check(self): + with cli(frappe.commands.utils.build, "--force --production") as result: self.assertEqual(result.exit_code, 0) self.assertEqual(result.exception, None) + + CURRENT_SIZE = 3.7 # MB + JS_ASSET_THRESHOLD = 0.1 + + hooks = frappe.get_hooks() + default_bundle = hooks["app_include_js"] + + default_bundle_size = 0.0 + + for chunk in default_bundle: + abs_path = Path.cwd() / frappe.local.sites_path / bundled_asset(chunk)[1:] + default_bundle_size += abs_path.stat().st_size + + self.assertLessEqual( + default_bundle_size / (1024 * 1024), + CURRENT_SIZE * (1 + JS_ASSET_THRESHOLD), + f"Default JS bundle size increased by {JS_ASSET_THRESHOLD:.2%} or more", + ) From 69cb144b06f11edcc7fbb32208d13816c4e6c378 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 20 Jun 2022 20:05:25 +0530 Subject: [PATCH 154/205] refactor!: move plyr from default bundle --- frappe/public/js/desk.bundle.js | 1 - frappe/public/js/frappe/utils/help.js | 13 ++++++++----- frappe/public/js/frappe/utils/utils.js | 4 ++++ .../public/js/frappe/widgets/onboarding_widget.js | 2 +- .../{frappe/ui/plyr.js => video_player.bundle.js} | 2 +- frappe/tests/test_commands.py | 2 +- 6 files changed, 15 insertions(+), 9 deletions(-) rename frappe/public/js/{frappe/ui/plyr.js => video_player.bundle.js} (70%) diff --git a/frappe/public/js/desk.bundle.js b/frappe/public/js/desk.bundle.js index e056a34be2..89270bad3f 100644 --- a/frappe/public/js/desk.bundle.js +++ b/frappe/public/js/desk.bundle.js @@ -107,6 +107,5 @@ import "./frappe/utils/dashboard_utils.js"; import "./frappe/ui/chart.js"; import "./frappe/ui/datatable.js"; import "./frappe/ui/driver.js"; -import "./frappe/ui/plyr.js"; import "./frappe/barcode_scanner/index.js"; import "./frappe/scanner"; diff --git a/frappe/public/js/frappe/utils/help.js b/frappe/public/js/frappe/utils/help.js index 9b8149e20e..56109c7701 100644 --- a/frappe/public/js/frappe/utils/help.js +++ b/frappe/public/js/frappe/utils/help.js @@ -33,13 +33,16 @@ frappe.help.show_video = function (youtube_id, title) { dialog.show(); dialog.$wrapper.addClass("video-modal"); - let plyr = new frappe.Plyr(video[0], { - hideControls: true, - resetOnEnd: true, - }); + let plyr; + frappe.utils.load_video_player().then(() => { + plyr = new frappe.Plyr(video[0], { + hideControls: true, + resetOnEnd: true, + }); + }) dialog.onhide = () => { - plyr.destroy(); + plyr?.destroy(); }; } diff --git a/frappe/public/js/frappe/utils/utils.js b/frappe/public/js/frappe/utils/utils.js index aa305a9ce7..74c89aa01a 100644 --- a/frappe/public/js/frappe/utils/utils.js +++ b/frappe/public/js/frappe/utils/utils.js @@ -1528,5 +1528,9 @@ Object.assign(frappe.utils, { return [doctype, filter, val[0], val[1], false]; }); } + }, + + load_video_player() { + return frappe.require("video_player.bundle.js"); } }); diff --git a/frappe/public/js/frappe/widgets/onboarding_widget.js b/frappe/public/js/frappe/widgets/onboarding_widget.js index e560551f79..5c3b996bb1 100644 --- a/frappe/public/js/frappe/widgets/onboarding_widget.js +++ b/frappe/public/js/frappe/widgets/onboarding_widget.js @@ -5,6 +5,7 @@ frappe.provide("frappe.utils"); export default class OnboardingWidget extends Widget { async refresh() { + frappe.utils.load_video_player(); this.new && await this.get_onboarding_data(); this.set_title(); this.set_actions(); @@ -156,7 +157,6 @@ export default class OnboardingWidget extends Widget { }; toggle_content(); - // toggle_video(); } go_to_page(step) { diff --git a/frappe/public/js/frappe/ui/plyr.js b/frappe/public/js/video_player.bundle.js similarity index 70% rename from frappe/public/js/frappe/ui/plyr.js rename to frappe/public/js/video_player.bundle.js index 97b36837f9..14e2d754bd 100644 --- a/frappe/public/js/frappe/ui/plyr.js +++ b/frappe/public/js/video_player.bundle.js @@ -1,3 +1,3 @@ import Plyr from "plyr/dist/plyr.polyfilled"; -frappe.Plyr = Plyr; \ No newline at end of file +frappe.Plyr = Plyr; diff --git a/frappe/tests/test_commands.py b/frappe/tests/test_commands.py index 16fea9155d..aa00a884e1 100644 --- a/frappe/tests/test_commands.py +++ b/frappe/tests/test_commands.py @@ -699,7 +699,7 @@ class TestBenchBuild(BaseTestCommands): self.assertEqual(result.exit_code, 0) self.assertEqual(result.exception, None) - CURRENT_SIZE = 3.7 # MB + CURRENT_SIZE = 3.5 # MB JS_ASSET_THRESHOLD = 0.1 hooks = frappe.get_hooks() From fd324db67e62adfab91e03c855a20e968b4130b4 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 20 Jun 2022 21:04:30 +0530 Subject: [PATCH 155/205] chore!: remove deprecated quaggajs based barcode scanner (#17248) --- frappe/public/js/barcode_scanner.bundle.js | 1 - frappe/public/js/desk.bundle.js | 1 - .../public/js/frappe/barcode_scanner/index.js | 23 --- .../js/frappe/barcode_scanner/quagga.js | 94 ------------- package.json | 1 - yarn.lock | 133 +----------------- 6 files changed, 3 insertions(+), 250 deletions(-) delete mode 100644 frappe/public/js/barcode_scanner.bundle.js delete mode 100644 frappe/public/js/frappe/barcode_scanner/index.js delete mode 100644 frappe/public/js/frappe/barcode_scanner/quagga.js diff --git a/frappe/public/js/barcode_scanner.bundle.js b/frappe/public/js/barcode_scanner.bundle.js deleted file mode 100644 index 294f20c08f..0000000000 --- a/frappe/public/js/barcode_scanner.bundle.js +++ /dev/null @@ -1 +0,0 @@ -import "./frappe/barcode_scanner/quagga"; diff --git a/frappe/public/js/desk.bundle.js b/frappe/public/js/desk.bundle.js index e056a34be2..735860635e 100644 --- a/frappe/public/js/desk.bundle.js +++ b/frappe/public/js/desk.bundle.js @@ -108,5 +108,4 @@ import "./frappe/ui/chart.js"; import "./frappe/ui/datatable.js"; import "./frappe/ui/driver.js"; import "./frappe/ui/plyr.js"; -import "./frappe/barcode_scanner/index.js"; import "./frappe/scanner"; diff --git a/frappe/public/js/frappe/barcode_scanner/index.js b/frappe/public/js/frappe/barcode_scanner/index.js deleted file mode 100644 index fa3975b578..0000000000 --- a/frappe/public/js/frappe/barcode_scanner/index.js +++ /dev/null @@ -1,23 +0,0 @@ -frappe.provide('frappe.barcode'); - -frappe.barcode.scan_barcode = function() { - return new Promise((resolve, reject) => { - if ( - window.cordova && - window.cordova.plugins && - window.cordova.plugins.barcodeScanner - ) { - window.cordova.plugins.barcodeScanner.scan(result => { - if (!result.cancelled) { - resolve(result.text); - } - }, reject); - } else { - frappe.require('barcode_scanner.bundle.js', () => { - frappe.barcode.get_barcode().then(barcode => { - resolve(barcode); - }); - }); - } - }); -}; diff --git a/frappe/public/js/frappe/barcode_scanner/quagga.js b/frappe/public/js/frappe/barcode_scanner/quagga.js deleted file mode 100644 index fcab3b4dbe..0000000000 --- a/frappe/public/js/frappe/barcode_scanner/quagga.js +++ /dev/null @@ -1,94 +0,0 @@ -import Quagga from 'quagga/dist/quagga'; -frappe.provide('frappe.barcode'); - -Quagga.onProcessed(function(result) { - let drawingCtx = Quagga.canvas.ctx.overlay, - drawingCanvas = Quagga.canvas.dom.overlay; - - if (result) { - if (result.boxes) { - drawingCtx.clearRect( - 0, - 0, - parseInt(drawingCanvas.getAttribute('width')), - parseInt(drawingCanvas.getAttribute('height')) - ); - result.boxes - .filter(function(box) { - return box !== result.box; - }) - .forEach(function(box) { - Quagga.ImageDebug.drawPath(box, { x: 0, y: 1 }, drawingCtx, { - color: 'green', - lineWidth: 2 - }); - }); - } - - if (result.box) { - Quagga.ImageDebug.drawPath(result.box, { x: 0, y: 1 }, drawingCtx, { - color: '#00F', - lineWidth: 2 - }); - } - - if (result.codeResult && result.codeResult.code) { - Quagga.ImageDebug.drawPath(result.line, { x: 'x', y: 'y' }, drawingCtx, { - color: 'red', - lineWidth: 3 - }); - } - } -}); - -frappe.barcode.get_barcode = function() { - return new Promise(resolve => { - let d = new frappe.ui.Dialog({ - title: __('Scan Barcode'), - fields: [ - { - fieldtype: 'HTML', - fieldname: 'scan_area' - } - ], - on_page_show() { - let $scan_area = d.get_field('scan_area').$wrapper; - $scan_area.addClass('barcode-scanner'); - - Quagga.init( - { - inputStream: { - name: 'Live', - type: 'LiveStream', - target: $scan_area.get(0) - }, - decoder: { - readers: ['code_128_reader'] - } - }, - function(err) { - if (err) { - // eslint-disable-next-line - console.log(err); - return; - } - // eslint-disable-next-line - console.log('Initialization finished. Ready to start'); - Quagga.start(); - } - ); - - Quagga.onDetected(function(result) { - let code = result.codeResult.code; - if (code) { - Quagga.stop(); - d.hide(); - resolve(code); - } - }); - } - }); - - d.show(); - }); -}; diff --git a/package.json b/package.json index e144c76e89..f4dc2cb3f0 100644 --- a/package.json +++ b/package.json @@ -49,7 +49,6 @@ "node-sass": "^7.0.0", "plyr": "^3.7.2", "popper.js": "^1.16.0", - "quagga": "^0.12.1", "quill": "2.0.0-dev.4", "quill-image-resize": "^3.0.9", "quill-magic-url": "^3.0.0", diff --git a/yarn.lock b/yarn.lock index ec908cbb11..bd14584723 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1059,13 +1059,6 @@ cwd@^0.10.0: find-pkg "^0.1.2" fs-exists-sync "^0.1.0" -cwise-compiler@^1.1.2: - version "1.1.3" - resolved "https://registry.yarnpkg.com/cwise-compiler/-/cwise-compiler-1.1.3.tgz#f4d667410e850d3a313a7d2db7b1e505bb034cc5" - integrity sha1-9NZnQQ6FDToxOn0tt7HlBbsDTMU= - dependencies: - uniq "^1.0.0" - dashdash@^1.12.0: version "1.14.1" resolved "https://registry.yarnpkg.com/dashdash/-/dashdash-1.14.1.tgz#853cfa0f7cbe2fed5de20326b8dd581035f6e2f0" @@ -1073,11 +1066,6 @@ dashdash@^1.12.0: dependencies: assert-plus "^1.0.0" -data-uri-to-buffer@0.0.3: - version "0.0.3" - resolved "https://registry.yarnpkg.com/data-uri-to-buffer/-/data-uri-to-buffer-0.0.3.tgz#18ae979a6a0ca994b0625853916d2662bbae0b1a" - integrity sha1-GK6XmmoMqZSwYlhTkW0mYruuCxo= - de-indent@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/de-indent/-/de-indent-1.0.2.tgz#b2038e846dc33baa5796128d0804b455b8c1e21d" @@ -1905,23 +1893,6 @@ get-intrinsic@^1.0.2: has "^1.0.3" has-symbols "^1.0.1" -get-pixels@^3.2.3: - version "3.3.2" - resolved "https://registry.yarnpkg.com/get-pixels/-/get-pixels-3.3.2.tgz#3f62fb8811932c69f262bba07cba72b692b4ff03" - integrity sha512-6ar+8yPxRd1pskEcl2GSEu1La0+xYRjjnkby6AYiRDDwZ0tJbPQmHnSeH9fGLskT8kvR0OukVgtZLcsENF9YKQ== - dependencies: - data-uri-to-buffer "0.0.3" - jpeg-js "^0.3.2" - mime-types "^2.0.1" - ndarray "^1.0.13" - ndarray-pack "^1.1.1" - node-bitmap "0.0.1" - omggif "^1.0.5" - parse-data-uri "^0.2.0" - pngjs "^3.3.3" - request "^2.44.0" - through "^2.3.4" - get-stdin@^4.0.1: version "4.0.1" resolved "https://registry.yarnpkg.com/get-stdin/-/get-stdin-4.0.1.tgz#b968c6b0a04384324902e8bf1a5df32579a450fe" @@ -1934,21 +1905,6 @@ getpass@^0.1.1: dependencies: assert-plus "^1.0.0" -gl-mat2@^1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/gl-mat2/-/gl-mat2-1.0.1.tgz#142505730a5c2fe1e9f25d9ece3d0d6cc2710a30" - integrity sha1-FCUFcwpcL+Hp8l2ezj0NbMJxCjA= - -gl-vec2@^1.0.0: - version "1.3.0" - resolved "https://registry.yarnpkg.com/gl-vec2/-/gl-vec2-1.3.0.tgz#83d472ed46034de8e09cbc857123fb6c81c51199" - integrity sha512-YiqaAuNsheWmUV0Sa8k94kBB0D6RWjwZztyO+trEYS8KzJ6OQB/4686gdrf59wld4hHFIvaxynO3nRxpk1Ij/A== - -gl-vec3@^1.0.3: - version "1.1.3" - resolved "https://registry.yarnpkg.com/gl-vec3/-/gl-vec3-1.1.3.tgz#a47c62f918774a06cbed1b65bcd0288ecbb03826" - integrity sha512-jduKUqT0SGH02l8Yl+mV1yVsDfYgQAJyXGxkJQGyxPLHRiW25DwVIRPt6uvhrEMHftJfqhqKthRcyZqNEl9Xdw== - glob-parent@^5.1.0, glob-parent@~5.1.0: version "5.1.2" resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-5.1.2.tgz#869832c58034fe68a4093c17dc15e8340d8401c4" @@ -2278,11 +2234,6 @@ ini@^1.3.4: resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.8.tgz#a29da425b48806f34767a4efce397269af28432c" integrity sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew== -iota-array@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/iota-array/-/iota-array-1.0.0.tgz#81ef57fe5d05814cd58c2483632a99c30a0e8087" - integrity sha1-ge9X/l0FgUzVjCSDYyqZwwoOgIc= - ipaddr.js@1.9.1: version "1.9.1" resolved "https://registry.yarnpkg.com/ipaddr.js/-/ipaddr.js-1.9.1.tgz#bff38543eeb8984825079ff3a2a8e6cbd46781b3" @@ -2320,7 +2271,7 @@ is-boolean-object@^1.0.0: resolved "https://registry.yarnpkg.com/is-boolean-object/-/is-boolean-object-1.0.1.tgz#10edc0900dd127697a92f6f9807c7617d68ac48e" integrity sha512-TqZuVwa/sppcrhUCAYkGBk7w0yxfQQnxq28fjkO53tnK9FQXmdwz2JS5+GjsWQ6RByES1K40nI+yDic5c9/aAQ== -is-buffer@^1.0.2, is-buffer@^1.1.5, is-buffer@~1.1.6: +is-buffer@^1.1.5, is-buffer@~1.1.6: version "1.1.6" resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be" integrity sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w== @@ -2535,11 +2486,6 @@ isstream@~0.1.2: resolved "https://registry.yarnpkg.com/isstream/-/isstream-0.1.2.tgz#47e63f7af55afa6f92e1500e690eb8b8529c099a" integrity sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo= -jpeg-js@^0.3.2: - version "0.3.6" - resolved "https://registry.yarnpkg.com/jpeg-js/-/jpeg-js-0.3.6.tgz#c40382aac9506e7d1f2d856eb02f6c7b2a98b37c" - integrity sha512-MUj2XlMB8kpe+8DJUGH/3UJm4XpI8XEgZQ+CiHDeyrGoKPdW/8FJv6ku+3UiYm5Fz3CWaL+iXmD8Q4Ap6aC1Jw== - jquery@3.6.0: version "3.6.0" resolved "https://registry.yarnpkg.com/jquery/-/jquery-3.6.0.tgz#c72a09f15c1bdce142f49dbf1170bdf8adac2470" @@ -2904,11 +2850,6 @@ micromatch@^4.0.2: braces "^3.0.1" picomatch "^2.0.5" -mime-db@1.40.0: - version "1.40.0" - resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.40.0.tgz#a65057e998db090f732a68f6c276d387d4126c32" - integrity sha512-jYdeOMPy9vnxEqFRRo6ZvTZ8d9oPb+k18PKoYNYUe2stVEBPPwsln/qWzdbmaIvnhZ9v2P+CuecK+fpUfsV2mA== - mime-db@1.43.0: version "1.43.0" resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.43.0.tgz#0a12e0502650e473d735535050e7c8f4eb4fae58" @@ -2919,13 +2860,6 @@ mime-db@1.44.0: resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.44.0.tgz#fa11c5eb0aca1334b4233cb4d52f10c5a6272f92" integrity sha512-/NOTfLrsPBVeH7YtFPgsVWveuL+4SjjYxaQ1xtM1KMFj7HdxlBlxeyNLzhyJVx7r4rZGJAZ/6lkKCitSc/Nmpg== -mime-types@^2.0.1: - version "2.1.24" - resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.24.tgz#b6f8d0b3e951efb77dedeca194cff6d16f676f81" - integrity sha512-WaFHS3MCl5fapm3oLxU4eYDw77IQM2ACcxQ9RIxfaC3ooc6PFuBMGZZsYpvoXS5D5QTWPieo1jjLdAm3TBP3cQ== - dependencies: - mime-db "1.40.0" - mime-types@^2.1.12, mime-types@~2.1.19: version "2.1.27" resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.27.tgz#47949f98e279ea53119f5722e0f34e529bec009f" @@ -3038,27 +2972,6 @@ native-request@^1.0.5: resolved "https://registry.yarnpkg.com/native-request/-/native-request-1.0.8.tgz#8f66bf606e0f7ea27c0e5995eb2f5d03e33ae6fb" integrity sha512-vU2JojJVelUGp6jRcLwToPoWGxSx23z/0iX+I77J3Ht17rf2INGjrhOoQnjVo60nQd8wVsgzKkPfRXBiVdD2ag== -ndarray-linear-interpolate@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/ndarray-linear-interpolate/-/ndarray-linear-interpolate-1.0.0.tgz#78bc92b85b9abc15b6e67ee65828f9e2137ae72b" - integrity sha1-eLySuFuavBW25n7mWCj54hN65ys= - -ndarray-pack@^1.1.1: - version "1.2.1" - resolved "https://registry.yarnpkg.com/ndarray-pack/-/ndarray-pack-1.2.1.tgz#8caebeaaa24d5ecf70ff86020637977da8ee585a" - integrity sha1-jK6+qqJNXs9w/4YCBjeXfajuWFo= - dependencies: - cwise-compiler "^1.1.2" - ndarray "^1.0.13" - -ndarray@^1.0.13, ndarray@^1.0.18: - version "1.0.18" - resolved "https://registry.yarnpkg.com/ndarray/-/ndarray-1.0.18.tgz#b60d3a73224ec555d0faa79711e502448fd3f793" - integrity sha1-tg06cyJOxVXQ+qeXEeUCRI/T95M= - dependencies: - iota-array "^1.0.0" - is-buffer "^1.0.2" - needle@^2.5.2: version "2.6.0" resolved "https://registry.yarnpkg.com/needle/-/needle-2.6.0.tgz#24dbb55f2509e2324b4a99d61f413982013ccdbe" @@ -3086,11 +2999,6 @@ node-addon-api@^3.0.0: resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-3.1.0.tgz#98b21931557466c6729e51cb77cd39c965f42239" integrity sha512-flmrDNB06LIl5lywUz7YlNGZH/5p0M7W28k8hzd9Lshtdh1wshD2Y+U4h9LD6KObOy1f+fEVdgprPrEymjM5uw== -node-bitmap@0.0.1: - version "0.0.1" - resolved "https://registry.yarnpkg.com/node-bitmap/-/node-bitmap-0.0.1.tgz#180eac7003e0c707618ef31368f62f84b2a69091" - integrity sha1-GA6scAPgxwdhjvMTaPYvhLKmkJE= - node-gyp-build@^4.2.2: version "4.2.3" resolved "https://registry.yarnpkg.com/node-gyp-build/-/node-gyp-build-4.2.3.tgz#ce6277f853835f718829efb47db20f3e4d9c4739" @@ -3267,11 +3175,6 @@ object.assign@^4.1.0: has-symbols "^1.0.0" object-keys "^1.0.11" -omggif@^1.0.5: - version "1.0.10" - resolved "https://registry.yarnpkg.com/omggif/-/omggif-1.0.10.tgz#ddaaf90d4a42f532e9e7cb3a95ecdd47f17c7b19" - integrity sha512-LMJTtvgc/nugXj0Vcrrs68Mn2D1r0zf630VNtqtpI1FEO7e+O9FP4gqs9AcnBaSEeoHIPm28u6qgPR0oyEpGSw== - on-finished@~2.3.0: version "2.3.0" resolved "https://registry.yarnpkg.com/on-finished/-/on-finished-2.3.0.tgz#20f1336481b083cd75337992a16971aa2d906947" @@ -3353,13 +3256,6 @@ parent-module@^1.0.0: dependencies: callsites "^3.0.0" -parse-data-uri@^0.2.0: - version "0.2.0" - resolved "https://registry.yarnpkg.com/parse-data-uri/-/parse-data-uri-0.2.0.tgz#bf04d851dd5c87b0ab238e5d01ace494b604b4c9" - integrity sha1-vwTYUd1ch7CrI45dAazklLYEtMk= - dependencies: - data-uri-to-buffer "0.0.3" - parse-json@^5.0.0: version "5.2.0" resolved "https://registry.yarnpkg.com/parse-json/-/parse-json-5.2.0.tgz#c76fc66dee54231c962b22bcc8a72cf2f99753cd" @@ -3477,11 +3373,6 @@ plyr@^3.7.2: rangetouch "^2.0.1" url-polyfill "^1.1.12" -pngjs@^3.3.3: - version "3.4.0" - resolved "https://registry.yarnpkg.com/pngjs/-/pngjs-3.4.0.tgz#99ca7d725965fb655814eaf65f38f12bbdbf555f" - integrity sha512-NCrCHhWmnQklfH4MtJMRjZ2a8c80qXeMlQMv2uVp9ISJMTt562SbGd6n2oq0PaPgKm7Z6pL9E2UlLIhC+SHL3w== - popper.js@^1.16.0: version "1.16.1" resolved "https://registry.yarnpkg.com/popper.js/-/popper.js-1.16.1.tgz#2a223cb3dc7b6213d740e40372be40de43e65b1b" @@ -3998,19 +3889,6 @@ qs@~6.5.2: resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.2.tgz#cb3ae806e8740444584ef154ce8ee98d403f3e36" integrity sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA== -quagga@^0.12.1: - version "0.12.1" - resolved "https://registry.yarnpkg.com/quagga/-/quagga-0.12.1.tgz#6f48c56ed992dc5fdeb90dbee7069c2e1cdde8b7" - integrity sha1-b0jFbtmS3F/euQ2+5wacLhzd6Lc= - dependencies: - get-pixels "^3.2.3" - gl-mat2 "^1.0.0" - gl-vec2 "^1.0.0" - gl-vec3 "^1.0.3" - lodash "^4.17.4" - ndarray "^1.0.18" - ndarray-linear-interpolate "^1.0.0" - queue-microtask@^1.2.2: version "1.2.3" resolved "https://registry.yarnpkg.com/queue-microtask/-/queue-microtask-1.2.3.tgz#4929228bbc724dfac43e0efb058caf7b6cfb6243" @@ -4237,7 +4115,7 @@ repeat-string@^1.5.2: resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" integrity sha1-jcrkcOHIirwtYA//Sndihtp15jc= -request@^2.44.0, request@^2.88.0, request@^2.88.2: +request@^2.88.0, request@^2.88.2: version "2.88.2" resolved "https://registry.yarnpkg.com/request/-/request-2.88.2.tgz#d73c918731cb5a87da047e207234146f664d12b3" integrity sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw== @@ -4942,11 +4820,6 @@ tar@^6.0.2: mkdirp "^1.0.3" yallist "^4.0.0" -through@^2.3.4: - version "2.3.8" - resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" - integrity sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU= - timsort@^0.3.0: version "0.3.0" resolved "https://registry.yarnpkg.com/timsort/-/timsort-0.3.0.tgz#405411a8e7e6339fe64db9a234de11dc31e02bd4" @@ -5075,7 +4948,7 @@ uglify-to-browserify@~1.0.0: resolved "https://registry.yarnpkg.com/uglify-to-browserify/-/uglify-to-browserify-1.0.2.tgz#6e0924d6bda6b5afe349e39a6d632850a0f882b7" integrity sha1-bgkk1r2mta/jSeOabWMoUKD4grc= -uniq@^1.0.0, uniq@^1.0.1: +uniq@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/uniq/-/uniq-1.0.1.tgz#b31c5ae8254844a3a8281541ce2b04b865a734ff" integrity sha1-sxxa6CVIRKOoKBVBzisEuGWnNP8= From 4811c51ecc4795f3aeba1597cf7a64de7a465f91 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Jun 2022 18:40:31 +0000 Subject: [PATCH 156/205] build(deps): bump pre-commit/action from 2.0.3 to 3.0.0 Bumps [pre-commit/action](https://github.com/pre-commit/action) from 2.0.3 to 3.0.0. - [Release notes](https://github.com/pre-commit/action/releases) - [Commits](https://github.com/pre-commit/action/compare/v2.0.3...v3.0.0) --- updated-dependencies: - dependency-name: pre-commit/action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/linters.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linters.yml b/.github/workflows/linters.yml index 7a17648ff0..ce7ffb7de7 100644 --- a/.github/workflows/linters.yml +++ b/.github/workflows/linters.yml @@ -17,7 +17,7 @@ jobs: python-version: 3.8 - name: Install and Run Pre-commit - uses: pre-commit/action@v2.0.3 + uses: pre-commit/action@v3.0.0 - name: Download Semgrep rules run: git clone --depth 1 https://github.com/frappe/semgrep-rules.git frappe-semgrep-rules From e12d37a9433c9370e15273b5b2ff843e17eb5dbe Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Jun 2022 18:40:35 +0000 Subject: [PATCH 157/205] build(deps): bump actions/checkout from 2 to 3 Bumps [actions/checkout](https://github.com/actions/checkout) from 2 to 3. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v2...v3) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/docs-checker.yml | 2 +- .github/workflows/linters.yml | 2 +- .github/workflows/patch-mariadb-tests.yml | 2 +- .github/workflows/publish-assets-develop.yml | 2 +- .github/workflows/publish-assets-releases.yml | 2 +- .github/workflows/release.yml | 2 +- .github/workflows/server-mariadb-tests.yml | 2 +- .github/workflows/server-postgres-tests.yml | 2 +- .github/workflows/ui-tests.yml | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/docs-checker.yml b/.github/workflows/docs-checker.yml index 80b2469c0c..dade810b64 100644 --- a/.github/workflows/docs-checker.yml +++ b/.github/workflows/docs-checker.yml @@ -18,7 +18,7 @@ jobs: python-version: 3.8 - name: 'Clone repo' - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Validate Docs env: diff --git a/.github/workflows/linters.yml b/.github/workflows/linters.yml index 7a17648ff0..477195b53a 100644 --- a/.github/workflows/linters.yml +++ b/.github/workflows/linters.yml @@ -9,7 +9,7 @@ jobs: name: Frappe Linter runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python 3.8 uses: actions/setup-python@v4 diff --git a/.github/workflows/patch-mariadb-tests.yml b/.github/workflows/patch-mariadb-tests.yml index ae5802a8e1..60d0ce60af 100644 --- a/.github/workflows/patch-mariadb-tests.yml +++ b/.github/workflows/patch-mariadb-tests.yml @@ -28,7 +28,7 @@ jobs: steps: - name: Clone - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Setup Python uses: actions/setup-python@v4 diff --git a/.github/workflows/publish-assets-develop.yml b/.github/workflows/publish-assets-develop.yml index 6cd3af78f2..037c8c26eb 100644 --- a/.github/workflows/publish-assets-develop.yml +++ b/.github/workflows/publish-assets-develop.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: path: 'frappe' - uses: actions/setup-node@v3 diff --git a/.github/workflows/publish-assets-releases.yml b/.github/workflows/publish-assets-releases.yml index f242a363b4..594229a156 100644 --- a/.github/workflows/publish-assets-releases.yml +++ b/.github/workflows/publish-assets-releases.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: path: 'frappe' - uses: actions/setup-node@v3 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 32c5d53831..f73bed09c7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout Entire Repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 persist-credentials: false diff --git a/.github/workflows/server-mariadb-tests.yml b/.github/workflows/server-mariadb-tests.yml index f7f3aae8d0..51d379d5db 100644 --- a/.github/workflows/server-mariadb-tests.yml +++ b/.github/workflows/server-mariadb-tests.yml @@ -37,7 +37,7 @@ jobs: steps: - name: Clone - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Setup Python uses: actions/setup-python@v4 diff --git a/.github/workflows/server-postgres-tests.yml b/.github/workflows/server-postgres-tests.yml index 5116d4198e..d93edad0ec 100644 --- a/.github/workflows/server-postgres-tests.yml +++ b/.github/workflows/server-postgres-tests.yml @@ -40,7 +40,7 @@ jobs: steps: - name: Clone - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Setup Python uses: actions/setup-python@v4 diff --git a/.github/workflows/ui-tests.yml b/.github/workflows/ui-tests.yml index 72fe3e67bb..09b2a3caf8 100644 --- a/.github/workflows/ui-tests.yml +++ b/.github/workflows/ui-tests.yml @@ -36,7 +36,7 @@ jobs: steps: - name: Clone - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Setup Python uses: actions/setup-python@v4 From d7fe7ae39cae03c1c073897f92c92d5f33e1b95a Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Tue, 21 Jun 2022 07:02:58 +0530 Subject: [PATCH 158/205] perf(UX): Load splash screen immediately after login --- frappe/templates/includes/login/login.js | 1 + frappe/templates/includes/splash_screen.html | 4 ++++ frappe/www/app.html | 5 +---- 3 files changed, 6 insertions(+), 4 deletions(-) create mode 100644 frappe/templates/includes/splash_screen.html diff --git a/frappe/templates/includes/login/login.js b/frappe/templates/includes/login/login.js index 3af48b4b53..8365947e6c 100644 --- a/frappe/templates/includes/login/login.js +++ b/frappe/templates/includes/login/login.js @@ -195,6 +195,7 @@ login.login_handlers = (function () { 200: function (data) { if (data.message == 'Logged In') { login.set_status('{{ _("Success") }}', 'green'); + document.body.innerHTML = `{% include "templates/includes/splash_screen.html" %}`; window.location.href = frappe.utils.sanitise_redirect(frappe.utils.get_url_arg("redirect-to")) || data.home_page; } else if (data.message == 'Password Reset') { window.location.href = frappe.utils.sanitise_redirect(data.redirect_to); diff --git a/frappe/templates/includes/splash_screen.html b/frappe/templates/includes/splash_screen.html new file mode 100644 index 0000000000..0cbd118dcd --- /dev/null +++ b/frappe/templates/includes/splash_screen.html @@ -0,0 +1,4 @@ +
    + +
    \ No newline at end of file diff --git a/frappe/www/app.html b/frappe/www/app.html index 4402dbb481..f4de074d32 100644 --- a/frappe/www/app.html +++ b/frappe/www/app.html @@ -26,10 +26,7 @@ {% include "public/icons/timeless/symbol-defs.svg" %} -
    - -
    + {% include "templates/includes/splash_screen.html" %}
    From d6f2d34bf4b9e214c8dff273ba05302766ff9a10 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sat, 18 Jun 2022 22:20:49 +0530 Subject: [PATCH 159/205] perf: send boot string instead of JSON literal changes: - compact boot info in /app HTML /app size went from 451kb to 393kb - ~13% less Verified that regex applied on this JSON aren't affecting perf, infact found them to be faster with compact JSON. - Send json string instead of placing JSON literal in code using Jinja. JS takes more time to pass object literal than parsing a plain JSON string. Overall content transfer size remains roughly same (albeit slightly lower) since double escaping ends up adding extra `\` around quotes. Co-authored-by: Suraj Shetty --- frappe/www/app.html | 2 +- frappe/www/app.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/frappe/www/app.html b/frappe/www/app.html index f4de074d32..9189570674 100644 --- a/frappe/www/app.html +++ b/frappe/www/app.html @@ -42,7 +42,7 @@ if (!window.frappe) window.frappe = {}; - frappe.boot = {{ boot }}; + frappe.boot = JSON.parse({{ boot }}); frappe._messages = frappe.boot["__messages"]; frappe.csrf_token = "{{ csrf_token }}"; diff --git a/frappe/www/app.py b/frappe/www/app.py index 9a8c80d6b1..0447d00f89 100644 --- a/frappe/www/app.py +++ b/frappe/www/app.py @@ -2,6 +2,7 @@ # License: MIT. See LICENSE no_cache = 1 +import json import os import re @@ -32,13 +33,14 @@ def get_context(context): frappe.db.commit() - boot_json = frappe.as_json(boot) + boot_json = frappe.as_json(boot, indent=None, separators=(",", ":")) # remove script tags from boot boot_json = SCRIPT_TAG_PATTERN.sub("", boot_json) # TODO: Find better fix boot_json = CLOSING_SCRIPT_TAG_PATTERN.sub("", boot_json) + boot_json = json.dumps(boot_json) context.update( { From 64cc07227eaac493865a851bde90ca5ca0480b7d Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Mon, 20 Jun 2022 07:53:41 +0530 Subject: [PATCH 160/205] refactor: Replace usage of deprecated attribute --- .../print_format_builder_column_selector.html | 2 +- .../print_format_builder_field.html | 6 +++--- .../print_format_builder_layout.html | 4 ++-- .../print_format_builder_section.html | 4 ++-- .../print_format_builder_sidebar.html | 2 +- frappe/public/icons/social/fair.svg | 4 ++-- frappe/public/icons/social/office_365.svg | 4 ++-- frappe/public/images/leaflet/spritesheet.svg | 4 ++-- frappe/public/js/frappe/file_uploader/FileUploader.vue | 2 +- .../public/js/frappe/form/templates/print_layout.html | 2 +- .../js/frappe/form/templates/timeline_message_box.html | 2 +- frappe/public/js/frappe/list/list_sidebar.html | 4 ++-- frappe/public/js/frappe/ui/group_by/group_by.html | 2 +- frappe/public/js/frappe/ui/page.html | 10 +++++----- frappe/public/js/frappe/ui/sort_selector.html | 2 +- frappe/public/js/frappe/ui/toolbar/navbar.html | 2 +- .../public/js/frappe/views/kanban/kanban_column.html | 2 +- frappe/public/js/frappe/widgets/chart_widget.js | 2 +- .../js/print_format_builder/ConfigureColumns.vue | 4 ++-- frappe/public/js/print_format_builder/Field.vue | 4 ++-- .../js/print_format_builder/PrintFormatSection.vue | 2 +- frappe/templates/discussions/reply_card.html | 2 +- .../section_with_cta/section_with_cta.html | 2 +- .../section_with_small_cta/section_with_small_cta.html | 2 +- frappe/www/me.html | 8 ++++---- 25 files changed, 42 insertions(+), 42 deletions(-) diff --git a/frappe/printing/page/print_format_builder/print_format_builder_column_selector.html b/frappe/printing/page/print_format_builder/print_format_builder_column_selector.html index 15d029704b..5292531e4c 100644 --- a/frappe/printing/page/print_format_builder/print_format_builder_column_selector.html +++ b/frappe/printing/page/print_format_builder/print_format_builder_column_selector.html @@ -13,7 +13,7 @@
    - +
    @@ -23,7 +23,7 @@
    - + {%= __("Add a new section") %}
    diff --git a/frappe/printing/page/print_format_builder/print_format_builder_section.html b/frappe/printing/page/print_format_builder/print_format_builder_section.html index 70a97a15f0..108e5ff01c 100644 --- a/frappe/printing/page/print_format_builder/print_format_builder_section.html +++ b/frappe/printing/page/print_format_builder/print_format_builder_section.html @@ -1,10 +1,10 @@
    - +
    - + {{ section.label || "" }}
    diff --git a/frappe/printing/page/print_format_builder/print_format_builder_sidebar.html b/frappe/printing/page/print_format_builder/print_format_builder_sidebar.html index c608eecbbd..2b5b040bca 100644 --- a/frappe/printing/page/print_format_builder/print_format_builder_sidebar.html +++ b/frappe/printing/page/print_format_builder/print_format_builder_sidebar.html @@ -11,7 +11,7 @@ {%= (f.fieldtype==="Custom HTML") ? "sidebar-custom-field" : "" %}" style="display: inline-block"> - + {%= __(f.label) || __(f.fieldname) %}
    diff --git a/frappe/public/icons/social/fair.svg b/frappe/public/icons/social/fair.svg index 4cfc77a71f..60c00dc7af 100644 --- a/frappe/public/icons/social/fair.svg +++ b/frappe/public/icons/social/fair.svg @@ -4,11 +4,11 @@ - + - + diff --git a/frappe/public/icons/social/office_365.svg b/frappe/public/icons/social/office_365.svg index d93167741d..77c3559408 100644 --- a/frappe/public/icons/social/office_365.svg +++ b/frappe/public/icons/social/office_365.svg @@ -35,11 +35,11 @@ - + - + diff --git a/frappe/public/images/leaflet/spritesheet.svg b/frappe/public/images/leaflet/spritesheet.svg index 3c00f30314..05feda62df 100644 --- a/frappe/public/images/leaflet/spritesheet.svg +++ b/frappe/public/images/leaflet/spritesheet.svg @@ -134,14 +134,14 @@ transform="translate(120,0)" style="fill:#bbbbbb"> diff --git a/frappe/public/js/frappe/form/templates/print_layout.html b/frappe/public/js/frappe/form/templates/print_layout.html index 366a771218..f65eb86bd4 100644 --- a/frappe/public/js/frappe/form/templates/print_layout.html +++ b/frappe/public/js/frappe/form/templates/print_layout.html @@ -25,7 +25,7 @@ {%= __("Settings") %} - + diff --git a/frappe/public/js/frappe/form/templates/timeline_message_box.html b/frappe/public/js/frappe/form/templates/timeline_message_box.html index a38e36c7b5..ad7ddc37e5 100644 --- a/frappe/public/js/frappe/form/templates/timeline_message_box.html +++ b/frappe/public/js/frappe/form/templates/timeline_message_box.html @@ -68,7 +68,7 @@
      diff --git a/frappe/public/js/frappe/list/list_sidebar.html b/frappe/public/js/frappe/list/list_sidebar.html index 0428f32e60..3c45f6ab47 100644 --- a/frappe/public/js/frappe/list/list_sidebar.html +++ b/frappe/public/js/frappe/list/list_sidebar.html @@ -17,7 +17,7 @@ - + @@ -50,7 +50,7 @@ {{ __("Tags") }} - + diff --git a/frappe/public/js/frappe/ui/group_by/group_by.html b/frappe/public/js/frappe/ui/group_by/group_by.html index 17ef0e4e3d..8522b46da8 100644 --- a/frappe/public/js/frappe/ui/group_by/group_by.html +++ b/frappe/public/js/frappe/ui/group_by/group_by.html @@ -45,7 +45,7 @@ diff --git a/frappe/public/js/frappe/ui/page.html b/frappe/public/js/frappe/ui/page.html index 6d7f4e460c..06d1baec69 100644 --- a/frappe/public/js/frappe/ui/page.html +++ b/frappe/public/js/frappe/ui/page.html @@ -6,11 +6,11 @@ - + - + @@ -25,7 +25,7 @@
    @@ -41,7 +41,7 @@ - + @@ -55,7 +55,7 @@ diff --git a/frappe/public/js/frappe/ui/sort_selector.html b/frappe/public/js/frappe/ui/sort_selector.html index 81288f305e..6f79f986a1 100644 --- a/frappe/public/js/frappe/ui/sort_selector.html +++ b/frappe/public/js/frappe/ui/sort_selector.html @@ -5,7 +5,7 @@ title="{{ sort_order==="desc" ? "descending" : "ascending" }}"> - + diff --git a/frappe/public/js/frappe/ui/toolbar/navbar.html b/frappe/public/js/frappe/ui/toolbar/navbar.html index 73767d2019..3276ca302a 100644 --- a/frappe/public/js/frappe/ui/toolbar/navbar.html +++ b/frappe/public/js/frappe/ui/toolbar/navbar.html @@ -15,7 +15,7 @@ aria-haspopup="true" > - +
    diff --git a/frappe/public/js/frappe/views/kanban/kanban_column.html b/frappe/public/js/frappe/views/kanban/kanban_column.html index f2b537aca4..9c9f1eede0 100644 --- a/frappe/public/js/frappe/views/kanban/kanban_column.html +++ b/frappe/public/js/frappe/views/kanban/kanban_column.html @@ -7,7 +7,7 @@
      diff --git a/frappe/public/js/frappe/widgets/chart_widget.js b/frappe/public/js/frappe/widgets/chart_widget.js index 24b80d8ad0..1785e0b138 100644 --- a/frappe/public/js/frappe/widgets/chart_widget.js +++ b/frappe/public/js/frappe/widgets/chart_widget.js @@ -481,7 +481,7 @@ export default class ChartWidget extends Widget { class="btn btn-xs btn-secondary chart-menu" > - + diff --git a/frappe/public/js/print_format_builder/ConfigureColumns.vue b/frappe/public/js/print_format_builder/ConfigureColumns.vue index da10f99e40..a236a119b9 100644 --- a/frappe/public/js/print_format_builder/ConfigureColumns.vue +++ b/frappe/public/js/print_format_builder/ConfigureColumns.vue @@ -26,7 +26,7 @@
      - +
      @@ -54,7 +54,7 @@ @click="remove_column(column)" > - +
      diff --git a/frappe/public/js/print_format_builder/Field.vue b/frappe/public/js/print_format_builder/Field.vue index ca53402083..b35603f00a 100644 --- a/frappe/public/js/print_format_builder/Field.vue +++ b/frappe/public/js/print_format_builder/Field.vue @@ -35,7 +35,7 @@ @click="edit_html" > - +
      diff --git a/frappe/public/js/print_format_builder/PrintFormatSection.vue b/frappe/public/js/print_format_builder/PrintFormatSection.vue index 9a065e5e26..80050a3cbb 100644 --- a/frappe/public/js/print_format_builder/PrintFormatSection.vue +++ b/frappe/public/js/print_format_builder/PrintFormatSection.vue @@ -25,7 +25,7 @@ data-toggle="dropdown" > - +
      • diff --git a/frappe/website/web_template/section_with_cta/section_with_cta.html b/frappe/website/web_template/section_with_cta/section_with_cta.html index ffc0cd79fb..f59dbc5e66 100644 --- a/frappe/website/web_template/section_with_cta/section_with_cta.html +++ b/frappe/website/web_template/section_with_cta/section_with_cta.html @@ -7,7 +7,7 @@

        {{ _(cta_label) }} - +

        diff --git a/frappe/website/web_template/section_with_small_cta/section_with_small_cta.html b/frappe/website/web_template/section_with_small_cta/section_with_small_cta.html index b898ebdcbe..4465fcc153 100644 --- a/frappe/website/web_template/section_with_small_cta/section_with_small_cta.html +++ b/frappe/website/web_template/section_with_small_cta/section_with_small_cta.html @@ -10,7 +10,7 @@ {%- if cta_label and cta_url -%}

        {{ cta_label }} - +

        {%- endif -%} diff --git a/frappe/www/me.html b/frappe/www/me.html index 8b706afd6a..b0fbf1f899 100644 --- a/frappe/www/me.html +++ b/frappe/www/me.html @@ -20,7 +20,7 @@
      • - {{ _(item.label) }}
        • {% else %} - {{ _(item.label) }} diff --git a/frappe/templates/includes/web_block.html b/frappe/templates/includes/web_block.html index c8b22b3551..1215df63cb 100644 --- a/frappe/templates/includes/web_block.html +++ b/frappe/templates/includes/web_block.html @@ -11,7 +11,9 @@ {%- if web_template_type == 'Section' -%} {%- if not web_block.hide_block -%} -
        Date: Tue, 21 Jun 2022 20:07:59 +0530 Subject: [PATCH 166/205] fix: App logo not set in website settings (#17261) * fix: App logo not set in website settings * chore: potential error in system settings --- frappe/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/__init__.py b/frappe/__init__.py index 1cf3526b45..ee199c8aed 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -2231,14 +2231,14 @@ def get_website_settings(key): if not hasattr(local, "website_settings"): local.website_settings = db.get_singles_dict("Website Settings", cast=True) - return local.website_settings[key] + return local.website_settings.get(key) def get_system_settings(key): if not hasattr(local, "system_settings"): local.system_settings = db.get_singles_dict("System Settings", cast=True) - return local.system_settings[key] + return local.system_settings.get(key) def get_active_domains(): From e63e26890ce5cfa10acaf4dba20c2001a0842d96 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Tue, 21 Jun 2022 19:52:06 +0530 Subject: [PATCH 167/205] fix: show "doc modifed after" load message --- cypress/integration/form.js | 16 +++++++++++++++ cypress/support/commands.js | 28 ++++++++++++++++++++++++++ frappe/public/js/frappe/form/form.js | 6 ++---- frappe/public/js/frappe/model/model.js | 2 +- 4 files changed, 47 insertions(+), 5 deletions(-) diff --git a/cypress/integration/form.js b/cypress/integration/form.js index 99a4336bcb..4d50a5f66a 100644 --- a/cypress/integration/form.js +++ b/cypress/integration/form.js @@ -78,4 +78,20 @@ context('Form', () => { cy.get('@row2').click(); cy.get('@email_input2').should('not.have.class', 'invalid'); }); + + it('Shows version conflict warning', { scrollBehavior: false }, () => { + cy.visit('/app/todo'); + + cy.insert_doc("ToDo", {"description": "old"}).then(doc => { + cy.visit(`/app/todo/${doc.name}`); + // make form dirty + cy.fill_field("status", "Cancelled", "Select"); + + // update doc using api - simulating parallel change by another user + cy.update_doc("ToDo", doc.name, {"status": "Closed"}).then(() => { + cy.findByRole("button", {name: "Refresh"}).click(); + cy.get_field("status", "Select").should("have.value", "Closed"); + }) + }) + }); }); diff --git a/cypress/support/commands.js b/cypress/support/commands.js index b3708a1dba..5ee26348e2 100644 --- a/cypress/support/commands.js +++ b/cypress/support/commands.js @@ -291,7 +291,11 @@ Cypress.Commands.add('clear_datepickers', () => { cy.get('.datepicker').should('not.exist'); }); + Cypress.Commands.add('insert_doc', (doctype, args, ignore_duplicate) => { + if (!args.doctype) { + args.doctype = doctype; + } return cy .window() .its('frappe.csrf_token') @@ -324,6 +328,30 @@ Cypress.Commands.add('insert_doc', (doctype, args, ignore_duplicate) => { }); }); +Cypress.Commands.add('update_doc', (doctype, docname, args) => { + return cy + .window() + .its('frappe.csrf_token') + .then(csrf_token => { + return cy + .request({ + method: 'PUT', + url: `/api/resource/${doctype}/${docname}`, + body: args, + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + 'X-Frappe-CSRF-Token': csrf_token + }, + }) + .then(res => { + expect(res.status).to.eq(200); + return res.body.data; + }); + }); +}); + + Cypress.Commands.add('open_list_filter', () => { cy.get('.filter-section .filter-button').click(); cy.wait(300); diff --git a/frappe/public/js/frappe/form/form.js b/frappe/public/js/frappe/form/form.js index 13d61d689b..148ec7ca86 100644 --- a/frappe/public/js/frappe/form/form.js +++ b/frappe/public/js/frappe/form/form.js @@ -506,8 +506,6 @@ frappe.ui.form.Form = class FrappeForm { } initialize_new_doc() { - // moved this call to refresh function - // this.check_doctype_conflict(docname); var me = this; // hide any open grid @@ -1024,8 +1022,8 @@ frappe.ui.form.Form = class FrappeForm { if(this.doc.__unsaved) { this.dashboard.clear_headline(); this.dashboard.set_headline_alert(__("This form has been modified after you have loaded it") - + '' - + __("Refresh") + '', "alert-warning"); + + '', "alert-warning"); } else { this.reload_doc(); } diff --git a/frappe/public/js/frappe/model/model.js b/frappe/public/js/frappe/model/model.js index 4b6ee31b71..32ba1d89c2 100644 --- a/frappe/public/js/frappe/model/model.js +++ b/frappe/public/js/frappe/model/model.js @@ -54,7 +54,7 @@ $.extend(frappe.model, { if(frappe.get_route()[0]==="Form" && cur_frm.doc.doctype===doc.doctype && cur_frm.doc.name===doc.name) { if(!frappe.ui.form.is_saving && data.modified!=cur_frm.doc.modified) { doc.__needs_refresh = true; - cur_frm.check_doctype_conflict(); + cur_frm.show_conflict_message(); } } else { if(!doc.__unsaved) { From 8814041b3316754dc58f0c6dcc251d576cb38a04 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Tue, 21 Jun 2022 20:42:14 +0530 Subject: [PATCH 168/205] fix: set web-template attr in script and style tags from web templates Also, deduplication of styles and scripts when web_blocks jinja method is used --- frappe/utils/jinja_globals.py | 18 ++++++++-- .../doctype/web_page/templates/web_page.html | 23 +++++++++---- frappe/website/doctype/web_page/web_page.py | 34 +++++++++++++------ 3 files changed, 55 insertions(+), 20 deletions(-) diff --git a/frappe/utils/jinja_globals.py b/frappe/utils/jinja_globals.py index ae0523b82c..649a4c296f 100644 --- a/frappe/utils/jinja_globals.py +++ b/frappe/utils/jinja_globals.py @@ -36,6 +36,7 @@ def web_block(template, values=None, **kwargs): def web_blocks(blocks): + import frappe from frappe import _, _dict, throw from frappe.website.doctype.web_page.web_page import get_web_blocks_html @@ -62,8 +63,21 @@ def web_blocks(blocks): out = get_web_blocks_html(web_blocks) html = out.html - for script in out.scripts: - html += "".format(script) + for template, scripts in out.scripts.items(): + # deduplication of scripts when web_blocks methods are used in web pages + # see render_dynamic method web_page.py + if template not in frappe.flags.web_block_scripts: + for script in scripts: + html += f"" + frappe.flags.web_block_scripts[template] = True + + for template, styles in out.styles.items(): + # deduplication of styles when web_blocks methods are used in web pages + # see render_dynamic method web_page.py + if template not in frappe.flags.web_block_styles: + for style in styles: + html += f"" + frappe.flags.web_block_styles[template] = True return html diff --git a/frappe/website/doctype/web_page/templates/web_page.html b/frappe/website/doctype/web_page/templates/web_page.html index 384eb1f91a..c8679345c1 100644 --- a/frappe/website/doctype/web_page/templates/web_page.html +++ b/frappe/website/doctype/web_page/templates/web_page.html @@ -33,13 +33,18 @@ {% endblock %} {% block style %} - - -{%- for style in page_builder_styles -%} +{%- if style -%} +{%- endif -%} + +{%- for web_template, styles in (page_builder_styles or {}).items() -%} +{%- if styles -%} +{%- for style in styles -%} + {%- endfor -%} +{%- endif -%} +{%- endfor -%} + {% endblock %} {% block script %} @@ -47,7 +52,11 @@ {%- endif -%} - {%- for script in page_builder_scripts -%} - + {%- for web_template, scripts in (page_builder_scripts or {}).items() -%} + {%- if scripts -%} + {%- for script in scripts -%} + + {%- endfor -%} + {%- endif -%} {%- endfor -%} {% endblock %} diff --git a/frappe/website/doctype/web_page/web_page.py b/frappe/website/doctype/web_page/web_page.py index f74af1d8c7..98ecd163e6 100644 --- a/frappe/website/doctype/web_page/web_page.py +++ b/frappe/website/doctype/web_page/web_page.py @@ -77,15 +77,23 @@ class WebPage(WebsiteGenerator): def render_dynamic(self, context): # dynamic - is_jinja = context.dynamic_template or "" in context.main_section - if is_jinja or ("{{" in context.main_section): + is_jinja = ( + context.dynamic_template + or "" in context.main_section + or ("{{" in context.main_section) + ) + if is_jinja: + frappe.flags.web_block_scripts = {} + frappe.flags.web_block_styles = {} try: context["main_section"] = render_template(context.main_section, context) if not "" in context.main_section: context["no_cache"] = 1 except TemplateSyntaxError: - if is_jinja: - raise + raise + finally: + frappe.flags.web_block_scripts = {} + frappe.flags.web_block_styles = {} def set_breadcrumbs(self, context): """Build breadcrumbs template""" @@ -191,9 +199,9 @@ def check_publish_status(): def get_web_blocks_html(blocks): """Converts a list of blocks into Raw HTML and extracts out their scripts for deduplication""" - out = frappe._dict(html="", scripts=[], styles=[]) - extracted_scripts = [] - extracted_styles = [] + out = frappe._dict(html="", scripts={}, styles={}) + extracted_scripts = {} + extracted_styles = {} for block in blocks: web_template = frappe.get_cached_doc("Web Template", block.web_template) rendered_html = frappe.render_template( @@ -207,11 +215,15 @@ def get_web_blocks_html(blocks): html, scripts, styles = extract_script_and_style_tags(rendered_html) out.html += html if block.web_template not in extracted_scripts: - out.scripts += scripts - extracted_scripts.append(block.web_template) + extracted_scripts.setdefault(block.web_template, []) + extracted_scripts[block.web_template] += scripts + if block.web_template not in extracted_styles: - out.styles += styles - extracted_styles.append(block.web_template) + extracted_styles.setdefault(block.web_template, []) + extracted_styles[block.web_template] += styles + + out.scripts = extracted_scripts + out.styles = extracted_styles return out From 23cad5480203015af3f5f4fc442c16e8bf802969 Mon Sep 17 00:00:00 2001 From: Aditya Hase Date: Tue, 21 Jun 2022 21:23:39 +0530 Subject: [PATCH 169/205] test(oauth): Send id_token of the authorized user instead of Guest This only affects OAuth clients that use `id_token` obtained from `frappe.integrations.oauth2.get_token`. Doesn't affect OAuth clients that ignore id_token and explicitly use `frappe.integrations.oauth2.openid_profile` endpoint for getting user details. e.g. Frappe OAuth client. A simple way to replicate this is to setup Frappe-Frappe OAuth client-server pair and use `login_via_oauth2_id_token` instead of `login_via_oauth2` in `login_via_frappe`. --- frappe/tests/test_oauth20.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/frappe/tests/test_oauth20.py b/frappe/tests/test_oauth20.py index b809630204..a634ace62a 100644 --- a/frappe/tests/test_oauth20.py +++ b/frappe/tests/test_oauth20.py @@ -16,7 +16,9 @@ class TestOAuth20(unittest.TestCase): def setUp(self): make_test_records("OAuth Client") make_test_records("User") - self.client_id = frappe.get_all("OAuth Client", fields=["*"])[0].get("client_id") + client = frappe.get_all("OAuth Client", fields=["*"])[0] + self.client_id = client.get("client_id") + self.client_secret = client.get("client_secret") self.form_header = {"content-type": "application/x-www-form-urlencoded"} self.scope = "all openid" self.redirect_uri = "http://localhost" @@ -90,6 +92,9 @@ class TestOAuth20(unittest.TestCase): self.assertTrue(bearer_token.get("token_type") == "Bearer") self.assertTrue(check_valid_openid_response(bearer_token.get("access_token"))) + decoded_token = self.decode_id_token(bearer_token.get("id_token")) + self.assertEqual(decoded_token["email"], "test@example.com") + def test_login_using_authorization_code_with_pkce(self): update_client_for_auth_code_grant(self.client_id) @@ -142,6 +147,9 @@ class TestOAuth20(unittest.TestCase): self.assertTrue(bearer_token.get("access_token")) self.assertTrue(bearer_token.get("id_token")) + decoded_token = self.decode_id_token(bearer_token.get("id_token")) + self.assertEqual(decoded_token["email"], "test@example.com") + def test_revoke_token(self): client = frappe.get_doc("OAuth Client", self.client_id) client.grant_type = "Authorization Code" @@ -316,16 +324,19 @@ class TestOAuth20(unittest.TestCase): # Parse bearer token json bearer_token = token_response.json() - id_token = bearer_token.get("id_token") - payload = jwt.decode( - id_token, - audience=client.client_id, - key=client.client_secret, - algorithms=["HS256"], - ) + payload = self.decode_id_token(bearer_token.get("id_token")) + self.assertEqual(payload["email"], "test@example.com") self.assertTrue(payload.get("nonce") == nonce) + def decode_id_token(self, id_token): + return jwt.decode( + id_token, + audience=self.client_id, + key=self.client_secret, + algorithms=["HS256"], + ) + def check_valid_openid_response(access_token=None): """Return True for valid response.""" From 93320ce6e7f2f40550418c67b47953260cea8a5c Mon Sep 17 00:00:00 2001 From: Aditya Hase Date: Tue, 21 Jun 2022 21:52:22 +0530 Subject: [PATCH 170/205] fix(oauth): Send id_token of the authorized user instead of Guest This only affects OAuth clients that use `id_token` obtained from `frappe.integrations.oauth2.get_token`. Doesn't affect OAuth clients that ignore id_token and explicitly use `frappe.integrations.oauth2.openid_profile` endpoint for getting user details. e.g. Frappe OAuth client. A simple way to replicate this is to setup Frappe-Frappe OAuth client-server pair and use `login_via_oauth2_id_token` instead of `login_via_oauth2` in `login_via_frappe`. --- frappe/oauth.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/frappe/oauth.py b/frappe/oauth.py index e7fa101bfd..68e21ac88b 100644 --- a/frappe/oauth.py +++ b/frappe/oauth.py @@ -323,10 +323,7 @@ class OAuthWebRequestValidator(RequestValidator): # Check whether frappe server URL is set id_token_header = {"typ": "jwt", "alg": "HS256"} - user = frappe.get_doc( - "User", - frappe.session.user, - ) + user = frappe.get_doc("User", request.user) if request.nonce: id_token["nonce"] = request.nonce From d49f9bfb8fc2651fd6bf6a4c3e93018988d68499 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Wed, 22 Jun 2022 12:04:57 +0530 Subject: [PATCH 171/205] fix: handle case where web_block is invoked from places other than web page --- frappe/utils/jinja_globals.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/frappe/utils/jinja_globals.py b/frappe/utils/jinja_globals.py index 649a4c296f..44835be352 100644 --- a/frappe/utils/jinja_globals.py +++ b/frappe/utils/jinja_globals.py @@ -63,6 +63,11 @@ def web_blocks(blocks): out = get_web_blocks_html(web_blocks) html = out.html + + if not frappe.flags.web_block_scripts: + frappe.flags.web_block_scripts = {} + frappe.flags.web_block_styles = {} + for template, scripts in out.scripts.items(): # deduplication of scripts when web_blocks methods are used in web pages # see render_dynamic method web_page.py From c6d8f3bc7f1772e8d8eb3767436882ab172def91 Mon Sep 17 00:00:00 2001 From: Sagar Vora Date: Wed, 22 Jun 2022 06:43:35 +0000 Subject: [PATCH 172/205] fix: dont clear `_meta` when caching doc (#17115) fix: dont clear meta when caching doc --- frappe/model/base_document.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/frappe/model/base_document.py b/frappe/model/base_document.py index d3cc662d52..8e417bb45c 100644 --- a/frappe/model/base_document.py +++ b/frappe/model/base_document.py @@ -123,8 +123,23 @@ class BaseDocument(object): return meta def __getstate__(self): - self._meta = None - return self.__dict__ + """ + Called when pickling. + Returns a copy of `__dict__` excluding unpicklable values like `_meta`. + + More info: https://docs.python.org/3/library/pickle.html#handling-stateful-objects + """ + + # Always use the dict.copy() method to avoid modifying the original state + state = self.__dict__.copy() + self.remove_unpicklable_values(state) + + return state + + def remove_unpicklable_values(self, state): + """Remove unpicklable values before pickling""" + + state.pop("_meta", None) def update(self, d): """Update multiple fields of a doctype using a dictionary of key-value pairs. From fd4a7031b976cb4d40b63b4c096d5a3798b144b5 Mon Sep 17 00:00:00 2001 From: Jannat Patel <31363128+pateljannat@users.noreply.github.com> Date: Wed, 22 Jun 2022 13:23:15 +0530 Subject: [PATCH 173/205] fix: added mailto to email displayed on contact us page (#17269) --- frappe/www/contact.html | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/frappe/www/contact.html b/frappe/www/contact.html index 91f14985b2..a6672d63d2 100644 --- a/frappe/www/contact.html +++ b/frappe/www/contact.html @@ -67,7 +67,11 @@ {{ phone }}
        {% endif %} {% if email_id %} - {{ email_id }}
        + + + {{ email_id }} + +
        {% endif %} {% if skype %} {{ skype }}
        From 593fd0a1780dafbf06c20ee845937ce9cdc1a38b Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 20 Jun 2022 17:00:30 +0530 Subject: [PATCH 174/205] ci: Add audit for python dependencies via pip-audit --- .github/workflows/deps-checker.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .github/workflows/deps-checker.yml diff --git a/.github/workflows/deps-checker.yml b/.github/workflows/deps-checker.yml new file mode 100644 index 0000000000..3f81b5c633 --- /dev/null +++ b/.github/workflows/deps-checker.yml @@ -0,0 +1,22 @@ +name: 'Python Dependency Check' +on: + pull_request: + workflow_dispatch: + push: + branches: [ develop ] + +permissions: + contents: read + +jobs: + deps-vulnerable-check: + name: 'Vulnerable Dependency' + runs-on: ubuntu-latest + + steps: + - uses: actions/setup-python@v4 + with: + python-version: 3.8 + - uses: actions/checkout@v3 + - run: pip install pip-audit + - run: pip-audit ${GITHUB_WORKSPACE} From 71531bd2906d34b53973ae6db6667f0b470fb0c4 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 21 Jun 2022 11:33:07 +0530 Subject: [PATCH 175/205] chore(deps): Bump PyJWT from 2.0.1 to 2.4.0 Consists of security & bug fixes, and seemingly no breaking changes Ref: https://pyjwt.readthedocs.io/en/stable/changelog.html --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 81be41c480..89be81eaf0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ dependencies = [ "GitPython~=3.1.14", "Jinja2~=3.1.2", "Pillow~=9.1.1", - "PyJWT~=2.0.1", + "PyJWT~=2.4.0", "PyMySQL~=1.0.2", "PyPDF2~=2.1.0", "PyPika~=0.48.9", From 9737bb39fa25a7ba5f15d2f824ee66afd21381a3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 22 Jun 2022 10:45:05 +0000 Subject: [PATCH 176/205] build(deps): bump shell-quote from 1.7.2 to 1.7.3 Bumps [shell-quote](https://github.com/substack/node-shell-quote) from 1.7.2 to 1.7.3. - [Release notes](https://github.com/substack/node-shell-quote/releases) - [Changelog](https://github.com/substack/node-shell-quote/blob/master/CHANGELOG.md) - [Commits](https://github.com/substack/node-shell-quote/compare/v1.7.2...1.7.3) --- updated-dependencies: - dependency-name: shell-quote dependency-type: indirect ... Signed-off-by: dependabot[bot] --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index bd14584723..5934bc00c3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4374,9 +4374,9 @@ shebang-regex@^3.0.0: integrity sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A== shell-quote@^1.6.1: - version "1.7.2" - resolved "https://registry.yarnpkg.com/shell-quote/-/shell-quote-1.7.2.tgz#67a7d02c76c9da24f99d20808fcaded0e0e04be2" - integrity sha512-mRz/m/JVscCrkMyPqHc/bczi3OQHkLTqXHEFu0zDhK/qfv3UcOA4SVmRCLmos4bhjr9ekVQubj/R7waKapmiQg== + version "1.7.3" + resolved "https://registry.yarnpkg.com/shell-quote/-/shell-quote-1.7.3.tgz#aa40edac170445b9a431e17bb62c0b881b9c4123" + integrity sha512-Vpfqwm4EnqGdlsBFNmHhxhElJYrdfcxPThu+ryKS5J8L/fhAwLazFZtq+S+TWZ9ANj2piSQLGj6NQg+lKPmxrw== showdown@^1.9.1: version "1.9.1" From b076ec1b91bd83ad7e523262dec5e105319f4a3c Mon Sep 17 00:00:00 2001 From: Ernesto Ruiz Date: Wed, 22 Jun 2022 09:41:48 -0600 Subject: [PATCH 177/205] fix: Make OTP link for get QR Code clickleable (#17275) --- frappe/twofactor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/twofactor.py b/frappe/twofactor.py index 9e9a2c5d76..6d01331d7d 100644 --- a/frappe/twofactor.py +++ b/frappe/twofactor.py @@ -279,7 +279,7 @@ def get_email_body_for_qr_code(kwargs_dict): """Get QRCode email body.""" body_template = _( "Please click on the following link and follow the instructions on the page. {0}" - ).format("

        {{qrcode_link}}") + ).format("

        {{qrcode_link}}") body = frappe.render_template(body_template, kwargs_dict) return body From 20c94a5d32582fd5b682e7ffc0b2b27761de1da0 Mon Sep 17 00:00:00 2001 From: Jannat Patel Date: Thu, 23 Jun 2022 12:26:31 +0530 Subject: [PATCH 178/205] fix: navbar child redirection issue --- frappe/templates/includes/navbar/navbar_items.html | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frappe/templates/includes/navbar/navbar_items.html b/frappe/templates/includes/navbar/navbar_items.html index 309027be09..9fbb22eeaa 100644 --- a/frappe/templates/includes/navbar/navbar_items.html +++ b/frappe/templates/includes/navbar/navbar_items.html @@ -32,9 +32,10 @@ {% else %} -{% if parent %} {% set url = item.url or '' %} {% set url = url if url.startswith('#') else url | abs_url %} + +{% if parent %}
      • From efed50811c68d3f11eea7cd487042fe5aef8c291 Mon Sep 17 00:00:00 2001 From: Vladislav Date: Thu, 23 Jun 2022 11:52:14 +0300 Subject: [PATCH 179/205] Update frappe/translations/ru.csv Yes, this translation is more correct Co-authored-by: Shariq Ansari <30859809+shariquerik@users.noreply.github.com> --- frappe/translations/ru.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/translations/ru.csv b/frappe/translations/ru.csv index 60e140c2a5..5e333c8e9d 100644 --- a/frappe/translations/ru.csv +++ b/frappe/translations/ru.csv @@ -4011,7 +4011,7 @@ Setup > User,Настройка> Пользователь, Setup > Customize Form,Настройка> Настройка формы, Setup > User Permissions,Настройка> Полномочия пользователя, "Error connecting to QZ Tray Application...

        You need to have QZ Tray application installed and running, to use the Raw Print feature.

        Click here to Download and install QZ Tray.
        Click here to learn more about Raw Printing.","Ошибка подключения к приложению QZ Tray ...

        Вам необходимо установить и запустить приложение QZ Tray, чтобы использовать функцию Raw Print.

        Нажмите здесь, чтобы загрузить и установить QZ Tray .
        Нажмите здесь, чтобы узнать больше о Raw Printing .", -No email account associated with the User. Please add an account under User > Email Inbox.,"Нет учетной записи электронной почты, связанной с пользователем. Пожалуйста, добавьте электронную почту в настройках пользователя", +No email account associated with the User. Please add an account under User > Email Inbox.,"Нет учетной записи электронной почты, связанной с пользователем. Пожалуйста, добавьте учетную запись в разделе Пользователь > Электронная почта Входящие", "For comparison, use >5, <10 or =324. For ranges, use 5:10 (for values between 5 & 10).","Для сравнения используйте> 5, <10 или = 324. Для диапазонов используйте 5:10 (для значений от 5 до 10).", No default Address Template found. Please create a new one from Setup > Printing and Branding > Address Template.,"Шаблон адреса по умолчанию не найден. Создайте новый, выбрав «Настройка» > «Печать и брендинг» > «Шаблон адреса».", Please setup default Email Account from Setup > Email > Email Account,"Пожалуйста, настройте учетную запись электронной почты по умолчанию из меню «Настройка» > «Электронная почта»>", From 1349a73e145ac631f146d6b332d85668172217b0 Mon Sep 17 00:00:00 2001 From: gruener Date: Thu, 23 Jun 2022 13:38:29 +0200 Subject: [PATCH 180/205] fix: Virtual DocTypes currently breaking Parent-DocType Update and Deletion (#16977) When you create a Virtual DocType as a Child Table (which is possible without any warning), then it will lead to several errors when updating or deleting of the parent document. This is because the following files just execute a SQL Statement for the doctype (which doesnt have a DB Table, as this is the nature of a virtual doctype ;-) **apps/frappe/frappe/model/document.py** ```py frappe.db.sql("""delete from `tab{0}` where parent=%s and parenttype=%s and parentfield=%s""".format(df.options), (self.name, self.doctype, fieldname)) ``` **apps/frappe/frappe/model/delete_doc.py** ```py frappe.db.sql( "delete from `tab%s` where parenttype=%s and parent = %s" % (t, "%s", "%s"), (doctype, name) ) ``` So at these points, I added a check to not perform any sql command for virtual doctypes. With these changes, my affected situation is solved. Perhaps there are other situations, I didn't encounter yet. As an additional feature, those virtual doctype models should also get an information about the parent is deleted, to propagate the deletion to the remote data pools; but for now I hope this bugfix can be approved. --- frappe/core/doctype/doctype/doctype.py | 2 +- frappe/core/doctype/doctype/test_doctype.py | 40 +++++++++++++++++++++ frappe/model/delete_doc.py | 5 ++- frappe/model/document.py | 16 ++++++++- 4 files changed, 60 insertions(+), 3 deletions(-) diff --git a/frappe/core/doctype/doctype/doctype.py b/frappe/core/doctype/doctype/doctype.py index e834b698d5..b4cbbd6233 100644 --- a/frappe/core/doctype/doctype/doctype.py +++ b/frappe/core/doctype/doctype/doctype.py @@ -818,7 +818,7 @@ class DocType(Document): self.nsm_parent_field = parent_field_name def validate_child_table(self): - if not self.get("istable") or self.is_new(): + if not self.get("istable") or self.is_new() or self.get("is_virtual"): # if the doctype is not a child table then return # if the doctype is a new doctype and also a child table then # don't move forward as it will be handled via schema diff --git a/frappe/core/doctype/doctype/test_doctype.py b/frappe/core/doctype/doctype/test_doctype.py index 0bcd972c68..569cf9af2f 100644 --- a/frappe/core/doctype/doctype/test_doctype.py +++ b/frappe/core/doctype/doctype/test_doctype.py @@ -564,6 +564,46 @@ class TestDocType(unittest.TestCase): self.assertEqual(doc.is_virtual, 1) self.assertFalse(frappe.db.table_exists("Test Virtual Doctype")) + def test_create_virtual_doctype_as_child_table(self): + """Test virtual DocType as Child Table below a normal DocType.""" + frappe.delete_doc_if_exists("DocType", "Test Parent Virtual DocType", force=1) + frappe.delete_doc_if_exists("DocType", "Test Virtual DocType as Child Table", force=1) + + virtual_doc = new_doctype("Test Virtual DocType as Child Table") + virtual_doc.is_virtual = 1 + virtual_doc.istable = 1 + virtual_doc.insert(ignore_permissions=True) + + doc = frappe.get_doc("DocType", "Test Virtual DocType as Child Table") + + self.assertEqual(doc.is_virtual, 1) + self.assertEqual(doc.istable, 1) + self.assertFalse(frappe.db.table_exists("Test Virtual DocType as Child Table")) + + parent_doc = new_doctype("Test Parent Virtual DocType") + parent_doc.append( + "fields", + { + "fieldname": "virtual_child_table", + "fieldtype": "Table", + "options": "Test Virtual DocType as Child Table", + }, + ) + parent_doc.insert(ignore_permissions=True) + + # create entry for parent doctype + parent_doc_entry = frappe.get_doc( + {"doctype": "Test Parent Virtual DocType", "some_fieldname": "Test"} + ) + parent_doc_entry.insert(ignore_permissions=True) + + # update the parent doc (should not abort because of any DB query to a virtual child table, as there is none) + parent_doc_entry.some_fieldname = "Test update" + parent_doc_entry.save(ignore_permissions=True) + + # delete the parent doc (should not abort because of any DB query to a virtual child table, as there is none) + parent_doc_entry.delete() + def test_default_fieldname(self): fields = [ {"label": "title", "fieldname": "title", "fieldtype": "Data", "default": "{some_fieldname}"} diff --git a/frappe/model/delete_doc.py b/frappe/model/delete_doc.py index 985cc53682..20dac14458 100644 --- a/frappe/model/delete_doc.py +++ b/frappe/model/delete_doc.py @@ -196,7 +196,10 @@ def delete_from_table(doctype: str, name: str, ignore_doctypes: List[str], doc): else: frappe.db.delete(doctype, {"name": name}) if doc: - child_doctypes = [d.options for d in doc.meta.get_table_fields()] + child_doctypes = [ + d.options for d in doc.meta.get_table_fields() if frappe.get_meta(d.options).is_virtual == 0 + ] + else: child_doctypes = frappe.get_all( "DocField", diff --git a/frappe/model/document.py b/frappe/model/document.py index 22514df75a..898c40861c 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -152,6 +152,17 @@ class Document(BaseDocument): super(Document, self).__init__(d) for df in self._get_table_fields(): + # Make sure not to query the DB for a child table, if it is a virtual one. + # During frappe is installed, the property "is_virtual" is not available in tabDocType, so + # we need to filter those cases for the access to frappe.db.get_value() as it would crash otherwise. + if ( + hasattr(self, "doctype") + and not hasattr(self, "module") + and frappe.db.get_value("DocType", df.options, "is_virtual", cache=True) + ): + self.set(df.fieldname, []) + continue + children = ( frappe.db.get_values( df.options, @@ -379,7 +390,10 @@ class Document(BaseDocument): d.db_update() rows.append(d.name) - if df.options in (self.flags.ignore_children_type or []): + if ( + df.options in (self.flags.ignore_children_type or []) + or frappe.get_meta(df.options).is_virtual == 1 + ): # do not delete rows for this because of flags # hack for docperm :( return From 3bc636ce30ba702a77c4d1e77a0fb46ffeed2e83 Mon Sep 17 00:00:00 2001 From: Samuel Danieli <23150094+scdanieli@users.noreply.github.com> Date: Fri, 24 Jun 2022 11:27:30 +0200 Subject: [PATCH 181/205] fix(Image View): translate info field --- frappe/public/js/frappe/views/image/image_view.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/public/js/frappe/views/image/image_view.js b/frappe/public/js/frappe/views/image/image_view.js index 57647642a3..94494e72e9 100644 --- a/frappe/public/js/frappe/views/image/image_view.js +++ b/frappe/public/js/frappe/views/image/image_view.js @@ -75,8 +75,8 @@ frappe.views.ImageView = class ImageView extends frappe.views.ListView { let set = false; info_fields.forEach((field, index) => { if (item[field] && !set) { - if (index == 0) info_html += `
      • ${item[field]}
        • `; - else info_html += `
      • ${item[field]}
        • `; + if (index == 0) info_html += `
      • ${__(item[field])}
        • `; + else info_html += `
      • ${__(item[field])}
        • `; set = true; } }); From 47483f833fc132d32a9c6166b911882859ffe35f Mon Sep 17 00:00:00 2001 From: Danny <66078599+mmdanny89@users.noreply.github.com> Date: Fri, 24 Jun 2022 07:34:49 -0400 Subject: [PATCH 182/205] feat(translations-cli): Fetch translations for a particular app (#17276) * translataions for particular app get-untranslated/update-translations string for particular app frappe#17268 (feature request) * fix: linting, making stuff DRY-er Co-authored-by: gavin --- frappe/commands/translate.py | 10 ++++++---- frappe/translate.py | 24 ++++++++++++++++++------ 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/frappe/commands/translate.py b/frappe/commands/translate.py index 0b14e03002..69970d8d97 100644 --- a/frappe/commands/translate.py +++ b/frappe/commands/translate.py @@ -48,11 +48,12 @@ def new_language(context, lang_code, app): @click.command("get-untranslated") +@click.option("--app", default="_ALL_APPS") @click.argument("lang") @click.argument("untranslated_file") @click.option("--all", default=False, is_flag=True, help="Get all message strings") @pass_context -def get_untranslated(context, lang, untranslated_file, all=None): +def get_untranslated(context, lang, untranslated_file, app="_ALL_APPS", all=None): "Get untranslated strings for language" import frappe.translate @@ -60,17 +61,18 @@ def get_untranslated(context, lang, untranslated_file, all=None): try: frappe.init(site=site) frappe.connect() - frappe.translate.get_untranslated(lang, untranslated_file, get_all=all) + frappe.translate.get_untranslated(lang, untranslated_file, get_all=all, app=app) finally: frappe.destroy() @click.command("update-translations") +@click.option("--app", default="_ALL_APPS") @click.argument("lang") @click.argument("untranslated_file") @click.argument("translated-file") @pass_context -def update_translations(context, lang, untranslated_file, translated_file): +def update_translations(context, lang, untranslated_file, translated_file, app="_ALL_APPS"): "Update translated strings" import frappe.translate @@ -78,7 +80,7 @@ def update_translations(context, lang, untranslated_file, translated_file): try: frappe.init(site=site) frappe.connect() - frappe.translate.update_translations(lang, untranslated_file, translated_file) + frappe.translate.update_translations(lang, untranslated_file, translated_file, app=app) finally: frappe.destroy() diff --git a/frappe/translate.py b/frappe/translate.py index eb26124ba8..460c7db9b0 100644 --- a/frappe/translate.py +++ b/frappe/translate.py @@ -808,7 +808,7 @@ def write_csv_file(path, app_messages, lang_dict): w.writerow([message, translated_string, context]) -def get_untranslated(lang, untranslated_file, get_all=False): +def get_untranslated(lang, untranslated_file, get_all=False, app="_ALL_APPS"): """Returns all untranslated strings for a language and writes in a file :param lang: Language code. @@ -816,11 +816,16 @@ def get_untranslated(lang, untranslated_file, get_all=False): :param get_all: Return all strings, translated or not.""" clear_cache() apps = frappe.get_all_apps(True) + if app != "_ALL_APPS": + if app not in apps: + print(f"Application {app} not found!") + return + apps = [app] messages = [] untranslated = [] - for app in apps: - messages.extend(get_messages_for_app(app)) + for app_name in apps: + messages.extend(get_messages_for_app(app_name)) messages = deduplicate_messages(messages) @@ -850,7 +855,7 @@ def get_untranslated(lang, untranslated_file, get_all=False): print("all translated!") -def update_translations(lang, untranslated_file, translated_file): +def update_translations(lang, untranslated_file, translated_file, app="_ALL_APPS"): """Update translations from a source and target file for a given language. :param lang: Language code (e.g. `en`). @@ -879,9 +884,16 @@ def update_translations(lang, untranslated_file, translated_file): translation_dict[restore_newlines(key)] = restore_newlines(value) full_dict.update(translation_dict) + apps = frappe.get_all_apps(True) - for app in frappe.get_all_apps(True): - write_translations_file(app, lang, full_dict) + if app != "_ALL_APPS": + if app not in apps: + print(f"Application {app} not found!") + return + apps = [app] + + for app_name in apps: + write_translations_file(app_name, lang, full_dict) def import_translations(lang, path): From ac842872666e5ff92a9a58183efd2b6ef12c2e58 Mon Sep 17 00:00:00 2001 From: Sagar Vora Date: Fri, 24 Jun 2022 18:31:43 +0530 Subject: [PATCH 183/205] fix: improve SVG for expenses icon --- frappe/public/icons/timeless/icons.svg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/public/icons/timeless/icons.svg b/frappe/public/icons/timeless/icons.svg index fbd72d6fb5..af63d0c8ff 100644 --- a/frappe/public/icons/timeless/icons.svg +++ b/frappe/public/icons/timeless/icons.svg @@ -613,8 +613,8 @@ - + From 3c617e3b06e128192dc2bbab5abccf198145d55c Mon Sep 17 00:00:00 2001 From: Sagar Vora Date: Sun, 26 Jun 2022 11:51:52 +0530 Subject: [PATCH 184/205] fix!: dont delete custom permissions when doctype is deleted --- frappe/model/delete_doc.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/frappe/model/delete_doc.py b/frappe/model/delete_doc.py index 20dac14458..04e25fe198 100644 --- a/frappe/model/delete_doc.py +++ b/frappe/model/delete_doc.py @@ -89,8 +89,6 @@ def delete_doc( update_flags(doc, flags, ignore_permissions) check_permission_and_not_submitted(doc) - - frappe.db.delete("Custom DocPerm", {"parent": name}) frappe.db.delete("__global_search", {"doctype": name}) delete_from_table(doctype, name, ignore_doctypes, None) From b6963b1dec8a9c2e8abea61df48e3c905fa02a63 Mon Sep 17 00:00:00 2001 From: Smit Vora Date: Sun, 26 Jun 2022 12:53:38 +0530 Subject: [PATCH 185/205] fix: ignore integration request when deleting doc --- frappe/model/delete_doc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/frappe/model/delete_doc.py b/frappe/model/delete_doc.py index 20dac14458..a71cfa6053 100644 --- a/frappe/model/delete_doc.py +++ b/frappe/model/delete_doc.py @@ -31,6 +31,7 @@ doctypes_to_skip = ( "Notification Log", "Email Queue", "Document Share Key", + "Integration Request", ) From 217644ac07aca387da760217686c42635c76804e Mon Sep 17 00:00:00 2001 From: Sagar Vora Date: Sun, 26 Jun 2022 19:31:03 +0000 Subject: [PATCH 186/205] fix!: remove unnecessary `get_default` API (#17295) --- frappe/client.py | 6 ------ .../js/print_format_builder/LetterHeadEditor.vue | 10 ++-------- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/frappe/client.py b/frappe/client.py index 154fcf31e2..d5dc890f56 100644 --- a/frappe/client.py +++ b/frappe/client.py @@ -281,12 +281,6 @@ def set_default(key, value, parent=None): frappe.clear_cache(user=frappe.session.user) -@frappe.whitelist() -def get_default(key, parent=None): - """set a user default value""" - return frappe.db.get_default(key, parent) - - @frappe.whitelist(methods=["POST", "PUT"]) def make_width_property_setter(doc): """Set width Property Setter diff --git a/frappe/public/js/print_format_builder/LetterHeadEditor.vue b/frappe/public/js/print_format_builder/LetterHeadEditor.vue index 1eae56f81a..5dd0ee3575 100644 --- a/frappe/public/js/print_format_builder/LetterHeadEditor.vue +++ b/frappe/public/js/print_format_builder/LetterHeadEditor.vue @@ -150,14 +150,8 @@ export default { } }, mounted() { - if (!this.letterhead) { - frappe - .call("frappe.client.get_default", { key: "letter_head" }) - .then(r => { - if (r.message) { - this.set_letterhead(r.message); - } - }); + if (!this.letterhead && frappe.boot.sysdefaults.letter_head) { + this.set_letterhead(frappe.boot.sysdefaults.letter_head); } this.$watch( From c99dd7c5efb8467dcf686807980ef6169c14dd71 Mon Sep 17 00:00:00 2001 From: Abhirup Date: Thu, 16 Jun 2022 12:57:06 +0530 Subject: [PATCH 187/205] Add data titles to sidebar workspace fields for easier selection through CSS. --- frappe/public/js/frappe/views/workspace/workspace.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/views/workspace/workspace.js b/frappe/public/js/frappe/views/workspace/workspace.js index d95925eea6..6c719d5dc1 100644 --- a/frappe/public/js/frappe/views/workspace/workspace.js +++ b/frappe/public/js/frappe/views/workspace/workspace.js @@ -118,7 +118,7 @@ frappe.views.Workspace = class Workspace { } build_sidebar_section(title, root_pages) { - let sidebar_section = $(`
        `); + let sidebar_section = $(`
        `); let $title = $(`
        ${frappe.utils.icon("small-down", "xs")} From 99388652c9966b5826fafffdb2c0301a5704e467 Mon Sep 17 00:00:00 2001 From: Sagar Vora Date: Mon, 27 Jun 2022 01:56:58 +0530 Subject: [PATCH 188/205] chore(redis): remove unnecessary exception handling --- frappe/utils/redis_wrapper.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/frappe/utils/redis_wrapper.py b/frappe/utils/redis_wrapper.py index 06480f0b7b..178c3d4416 100644 --- a/frappe/utils/redis_wrapper.py +++ b/frappe/utils/redis_wrapper.py @@ -105,10 +105,7 @@ class RedisWrapper(redis.Redis): def delete_keys(self, key): """Delete keys with wildcard `*`.""" - try: - self.delete_value(self.get_keys(key), make_keys=False) - except redis.exceptions.ConnectionError: - pass + self.delete_value(self.get_keys(key), make_keys=False) def delete_key(self, *args, **kwargs): self.delete_value(*args, **kwargs) @@ -202,10 +199,7 @@ class RedisWrapper(redis.Redis): frappe.local.cache[_name][key] = value elif generator: value = generator() - try: - self.hset(name, key, value) - except redis.exceptions.ConnectionError: - pass + self.hset(name, key, value) return value def hdel(self, name, key, shared=False): From f89dc40b94e30e3a00b1c54434df2f3a05d4c610 Mon Sep 17 00:00:00 2001 From: Shariq Ansari <30859809+shariquerik@users.noreply.github.com> Date: Mon, 27 Jun 2022 14:46:06 +0530 Subject: [PATCH 189/205] fix: Allow permitted number card of type report (#17316) --- frappe/desk/doctype/number_card/number_card.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/frappe/desk/doctype/number_card/number_card.py b/frappe/desk/doctype/number_card/number_card.py index 2290e49656..74c8e9eb99 100644 --- a/frappe/desk/doctype/number_card/number_card.py +++ b/frappe/desk/doctype/number_card/number_card.py @@ -4,6 +4,7 @@ import frappe from frappe import _ +from frappe.boot import get_allowed_reports from frappe.config import get_modules_from_all_apps_for_user from frappe.model.document import Document from frappe.model.naming import append_number_if_name_exists @@ -90,9 +91,16 @@ def has_permission(doc, ptype, user): if "System Manager" in roles: return True - allowed_doctypes = tuple(frappe.permissions.get_doctypes_with_read()) - if doc.document_type in allowed_doctypes: - return True + if doc.type == "Report": + allowed_reports = [ + key if type(key) == str else key.encode("UTF8") for key in get_allowed_reports() + ] + if doc.report_name in allowed_reports: + return True + else: + allowed_doctypes = tuple(frappe.permissions.get_doctypes_with_read()) + if doc.document_type in allowed_doctypes: + return True return False From 084a1e6c31520de09d639dbb5db129fb7e757d4b Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 27 Jun 2022 15:18:06 +0530 Subject: [PATCH 190/205] refactor: get_permissions * Show page even if dangling Custom DocPerm records encountered * Add typing hints * Cleanup APIs --- .../permission_manager/permission_manager.py | 25 +++++++++++++------ frappe/permissions.py | 23 ++++++++--------- 2 files changed, 28 insertions(+), 20 deletions(-) diff --git a/frappe/core/page/permission_manager/permission_manager.py b/frappe/core/page/permission_manager/permission_manager.py index ad12e0fd4c..e2d08488c0 100644 --- a/frappe/core/page/permission_manager/permission_manager.py +++ b/frappe/core/page/permission_manager/permission_manager.py @@ -1,6 +1,8 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +from typing import Optional + import frappe import frappe.defaults from frappe import _ @@ -8,6 +10,7 @@ from frappe.core.doctype.doctype.doctype import ( clear_permissions_cache, validate_permissions_for_doctype, ) +from frappe.exceptions import DoesNotExistError from frappe.modules.import_file import get_file_path, read_doc_from_file from frappe.permissions import ( add_permission, @@ -68,17 +71,19 @@ def get_roles_and_doctypes(): @frappe.whitelist() -def get_permissions(doctype=None, role=None): +def get_permissions(doctype: Optional[str] = None, role: Optional[str] = None): frappe.only_for("System Manager") + if role: out = get_all_perms(role) if doctype: out = [p for p in out if p.parent == doctype] + else: - filters = dict(parent=doctype) + filters = {"parent": doctype} if frappe.session.user != "Administrator": - custom_roles = frappe.get_all("Role", filters={"is_custom": 1}) - filters["role"] = ["not in", [row.name for row in custom_roles]] + custom_roles = frappe.get_all("Role", filters={"is_custom": 1}, pluck="name") + filters["role"] = ["not in", custom_roles] out = frappe.get_all("Custom DocPerm", fields="*", filters=filters, order_by="permlevel") if not out: @@ -86,11 +91,15 @@ def get_permissions(doctype=None, role=None): linked_doctypes = {} for d in out: - if not d.parent in linked_doctypes: - linked_doctypes[d.parent] = get_linked_doctypes(d.parent) + if d.parent not in linked_doctypes: + try: + linked_doctypes[d.parent] = get_linked_doctypes(d.parent) + except DoesNotExistError: + # exclude & continue if linked doctype is not found + frappe.clear_last_message() + continue d.linked_doctypes = linked_doctypes[d.parent] - meta = frappe.get_meta(d.parent) - if meta: + if meta := frappe.get_meta(d.parent): d.is_submittable = meta.is_submittable d.in_create = meta.in_create diff --git a/frappe/permissions.py b/frappe/permissions.py index 8980d2e63e..55a7c0e5f3 100644 --- a/frappe/permissions.py +++ b/frappe/permissions.py @@ -1,6 +1,7 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE import copy +from typing import List import frappe import frappe.share @@ -605,19 +606,17 @@ def reset_perms(doctype): frappe.db.delete("Custom DocPerm", {"parent": doctype}) -def get_linked_doctypes(dt): - return list( - set( - [dt] - + [ - d.options - for d in frappe.get_meta(dt).get( - "fields", - {"fieldtype": "Link", "ignore_user_permissions": ("!=", 1), "options": ("!=", "[Select]")}, - ) - ] +def get_linked_doctypes(dt: str) -> List: + meta = frappe.get_meta(dt) + linked_doctypes = [dt] + [ + d.options + for d in meta.get( + "fields", + {"fieldtype": "Link", "ignore_user_permissions": ("!=", 1), "options": ("!=", "[Select]")}, ) - ) + ] + + return list(set(linked_doctypes)) def get_doc_name(doc): From fba75c35951982631322a17b3af1778ed2d129aa Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 27 Jun 2022 15:55:54 +0530 Subject: [PATCH 191/205] fix: use console.error for logging errors (#17318) Currently if uncaught exception occurs on server side, there's no simple way to find it out. Converting these console.log to console.error simplifies this for UI tests. Also converted `console.trace to` `console.error`; Both are practically same and give stack trace too, `trace` expands tracebacks by default. This is done to make spying on window.console.error easier. --- frappe/public/js/frappe/request.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/frappe/public/js/frappe/request.js b/frappe/public/js/frappe/request.js index 87ab06ce18..dd4c352f41 100644 --- a/frappe/public/js/frappe/request.js +++ b/frappe/public/js/frappe/request.js @@ -280,7 +280,7 @@ frappe.request.call = function(opts) { } } catch(e) { console.log("Unable to handle success response", data); // eslint-disable-line - console.trace(e); // eslint-disable-line + console.error(e); // eslint-disable-line } }) @@ -332,7 +332,7 @@ frappe.request.call = function(opts) { opts.error_callback && opts.error_callback(xhr); } catch(e) { console.log("Unable to handle failed response"); // eslint-disable-line - console.trace(e); // eslint-disable-line + console.error(e); // eslint-disable-line } }); } @@ -433,13 +433,13 @@ frappe.request.cleanup = function(opts, r) { if(r.exc) { r.exc = JSON.parse(r.exc); if(r.exc instanceof Array) { - $.each(r.exc, function(i, v) { - if(v) { - console.log(v); + r.exc.forEach(exc => { + if(exc) { + console.error(exc); } - }) + }); } else { - console.log(r.exc); + console.error(r.exc); } } From 112f11359847cd286752177b42a357d4b6910cc3 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 27 Jun 2022 17:04:34 +0530 Subject: [PATCH 192/205] fix!: remove dangerous "rollback_on_exception" flag (#17321) --- frappe/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/frappe/__init__.py b/frappe/__init__.py index ee199c8aed..9104cd5109 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -432,9 +432,6 @@ def msgprint( def _raise_exception(): if raise_exception: - if flags.rollback_on_exception: - db.rollback() - if inspect.isclass(raise_exception) and issubclass(raise_exception, Exception): raise raise_exception(msg) else: From b7514a05ca9039bf9336ece3938b69bdeecb6585 Mon Sep 17 00:00:00 2001 From: Sagar Vora Date: Tue, 28 Jun 2022 00:37:31 +0530 Subject: [PATCH 193/205] fix(redis): pass shared param when setting value based on generator --- frappe/utils/redis_wrapper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/utils/redis_wrapper.py b/frappe/utils/redis_wrapper.py index 178c3d4416..c29836af32 100644 --- a/frappe/utils/redis_wrapper.py +++ b/frappe/utils/redis_wrapper.py @@ -199,7 +199,7 @@ class RedisWrapper(redis.Redis): frappe.local.cache[_name][key] = value elif generator: value = generator() - self.hset(name, key, value) + self.hset(name, key, value, shared=shared) return value def hdel(self, name, key, shared=False): From 5604b090c0b2e891a19d1b122ea2e30ff23b543d Mon Sep 17 00:00:00 2001 From: Samuel Danieli <23150094+scdanieli@users.noreply.github.com> Date: Mon, 27 Jun 2022 22:15:23 +0200 Subject: [PATCH 194/205] feat: enable further translations --- frappe/public/js/frappe/ui/sort_selector.html | 2 +- frappe/public/js/frappe/ui/sort_selector.js | 2 +- frappe/public/js/frappe/ui/toolbar/toolbar.js | 2 +- frappe/public/js/frappe/utils/dashboard_utils.js | 4 +++- frappe/public/js/frappe/views/kanban/kanban_column.html | 2 +- 5 files changed, 7 insertions(+), 5 deletions(-) diff --git a/frappe/public/js/frappe/ui/sort_selector.html b/frappe/public/js/frappe/ui/sort_selector.html index 6f79f986a1..4e1945c46c 100644 --- a/frappe/public/js/frappe/ui/sort_selector.html +++ b/frappe/public/js/frappe/ui/sort_selector.html @@ -2,7 +2,7 @@
        `; @@ -24,10 +24,12 @@ frappe.dashboard_utils = { if (filter.fieldnames) { options_html = filter.options.map((option, i) => + // TODO: Make option translatable - be careful, since the text of the a tag is later used to perform some action `
      • ${option}
        • `).join(''); } else { + // TODO: Make option translatable - be careful, since the text of the a tag is later used to perform some action options_html = filter.options.map( option => `
      • ${option}
        • `).join(''); } diff --git a/frappe/public/js/frappe/views/kanban/kanban_column.html b/frappe/public/js/frappe/views/kanban/kanban_column.html index 9c9f1eede0..5e6b6e1e7b 100644 --- a/frappe/public/js/frappe/views/kanban/kanban_column.html +++ b/frappe/public/js/frappe/views/kanban/kanban_column.html @@ -17,7 +17,7 @@
        - + {{ __("Add " + doctype) }} + + {{ __("Add {0}", [__(doctype)]) }}
        From 3263bf8324d645e57b93289d29ca1795a11247ce Mon Sep 17 00:00:00 2001 From: Samuel Danieli <23150094+scdanieli@users.noreply.github.com> Date: Mon, 27 Jun 2022 22:19:05 +0200 Subject: [PATCH 195/205] style: indent --- frappe/public/js/frappe/views/kanban/kanban_column.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/views/kanban/kanban_column.html b/frappe/public/js/frappe/views/kanban/kanban_column.html index 5e6b6e1e7b..8606cdc69d 100644 --- a/frappe/public/js/frappe/views/kanban/kanban_column.html +++ b/frappe/public/js/frappe/views/kanban/kanban_column.html @@ -17,7 +17,7 @@
        - + {{ __("Add {0}", [__(doctype)]) }} + + {{ __("Add {0}", [__(doctype)]) }}
        From 19a13931525b15427f34836e0e922e952de7c7b7 Mon Sep 17 00:00:00 2001 From: Samuel Danieli <23150094+scdanieli@users.noreply.github.com> Date: Mon, 27 Jun 2022 23:32:55 +0200 Subject: [PATCH 196/205] feat: make field label translatable --- .../print_format_builder_column_selector.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/printing/page/print_format_builder/print_format_builder_column_selector.html b/frappe/printing/page/print_format_builder/print_format_builder_column_selector.html index 5292531e4c..495b837f48 100644 --- a/frappe/printing/page/print_format_builder/print_format_builder_column_selector.html +++ b/frappe/printing/page/print_format_builder/print_format_builder_column_selector.html @@ -20,7 +20,7 @@ - {{ f.label }} + {{ __(f.label) }}
        From c6fa8ab090fdc0d07dbd295b5a262ed2a5acf456 Mon Sep 17 00:00:00 2001 From: Shariq Ansari <30859809+shariquerik@users.noreply.github.com> Date: Tue, 28 Jun 2022 11:50:57 +0530 Subject: [PATCH 197/205] fix: email not sent if contain file with current site url (#17250) --- frappe/core/doctype/file/file.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/frappe/core/doctype/file/file.py b/frappe/core/doctype/file/file.py index e8b8da76ab..bb4b441680 100755 --- a/frappe/core/doctype/file/file.py +++ b/frappe/core/doctype/file/file.py @@ -16,7 +16,7 @@ from requests.exceptions import HTTPError, SSLError import frappe from frappe import _ from frappe.model.document import Document -from frappe.utils import call_hook_method, cint, get_files_path, get_hook_method +from frappe.utils import call_hook_method, cint, get_files_path, get_hook_method, get_url from frappe.utils.file_manager import is_safe_path from frappe.utils.image import optimize_image, strip_exif_data @@ -61,7 +61,12 @@ class File(Document): self.set_file_name() self.validate_attachment_limit() - if not self.is_folder and not self.is_remote_file: + if self.is_folder: + return + + if self.is_remote_file: + self.validate_remote_file() + else: self.save_file(content=self.get_content()) self.flags.new_file = True frappe.local.rollback_observers.append(self) @@ -255,6 +260,12 @@ class File(Document): title=_("Attachment Limit Reached"), ) + def validate_remote_file(self): + """Validates if file uploaded using URL already exist""" + site_url = get_url() + if "/files/" in self.file_url and self.file_url.startswith(site_url): + self.file_url = self.file_url.split(site_url, 1)[1] + def set_folder_name(self): """Make parent folders if not exists based on reference doctype and name""" if self.folder: @@ -445,6 +456,10 @@ class File(Document): file_path = self.file_url or self.file_name + site_url = get_url() + if "/files/" in file_path and file_path.startswith(site_url): + file_path = file_path.split(site_url, 1)[1] + if "/" not in file_path: if self.is_private: file_path = f"/private/files/{file_path}" From 9bd753551b6af9cb25a445526969f563bb738e5a Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Tue, 28 Jun 2022 12:07:34 +0530 Subject: [PATCH 198/205] ci: temp fix for semgrep (#17228) * ci: respekt my authoritah * ci: use pip semgrep --- .github/helper/roulette.py | 8 ++++---- .github/workflows/linters.yml | 16 +++++++--------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/.github/helper/roulette.py b/.github/helper/roulette.py index 9165198012..b859b87047 100644 --- a/.github/helper/roulette.py +++ b/.github/helper/roulette.py @@ -77,13 +77,13 @@ if __name__ == "__main__": updated_py_file_count = len(list(filter(is_py, files_list))) only_py_changed = updated_py_file_count == len(files_list) - if ci_files_changed: - print("CI related files were updated, running all build processes.") - - elif has_skip_ci_label(pr_number, repo): + if has_skip_ci_label(pr_number, repo): print("Found `Skip CI` label on pr, stopping build process.") sys.exit(0) + elif ci_files_changed: + print("CI related files were updated, running all build processes.") + elif only_docs_changed: print("Only docs were updated, stopping build process.") sys.exit(0) diff --git a/.github/workflows/linters.yml b/.github/workflows/linters.yml index 4c845dba8c..6d1029d51d 100644 --- a/.github/workflows/linters.yml +++ b/.github/workflows/linters.yml @@ -11,10 +11,10 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Set up Python 3.8 + - name: Set up Python uses: actions/setup-python@v4 with: - python-version: 3.8 + python-version: '3.10' - name: Install and Run Pre-commit uses: pre-commit/action@v3.0.0 @@ -22,10 +22,8 @@ jobs: - name: Download Semgrep rules run: git clone --depth 1 https://github.com/frappe/semgrep-rules.git frappe-semgrep-rules - - uses: returntocorp/semgrep-action@v1 - env: - SEMGREP_TIMEOUT: 120 - with: - config: >- - r/python.lang.correctness - ./frappe-semgrep-rules/rules + - name: Download semgrep + run: pip install semgrep==0.97.0 + + - name: Run Semgrep rules + run: semgrep ci --config ./frappe-semgrep-rules/rules --config r/python.lang.correctness From 07d7b34fd9d3b60eb4c03b61069f6718cff84b6d Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 28 Jun 2022 11:31:17 +0530 Subject: [PATCH 199/205] refactor: get_unsubscribe_message * Add fallback label "Unsubscribe" instead of printing None in the email * Add typing hints, f-stringify & make code DRY-er for better readability --- frappe/__init__.py | 2 +- .../email/doctype/email_queue/email_queue.py | 16 ++++---- frappe/email/email_body.py | 3 +- frappe/email/queue.py | 37 ++++++------------- 4 files changed, 23 insertions(+), 35 deletions(-) diff --git a/frappe/__init__.py b/frappe/__init__.py index 9104cd5109..063a62c84b 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -78,7 +78,7 @@ class _dict(dict): return _dict(self) -def _(msg, lang=None, context=None): +def _(msg, lang=None, context=None) -> str: """Returns translated string in current lang, if exists. Usage: _('Change') diff --git a/frappe/email/doctype/email_queue/email_queue.py b/frappe/email/doctype/email_queue/email_queue.py index c3002607b4..330dab9e7f 100644 --- a/frappe/email/doctype/email_queue/email_queue.py +++ b/frappe/email/doctype/email_queue/email_queue.py @@ -183,7 +183,7 @@ def send_mail(email_queue_name, is_background_task=False): class SendMailContext: def __init__(self, queue_doc: Document, is_background_task: bool = False): - self.queue_doc = queue_doc + self.queue_doc: EmailQueue = queue_doc self.is_background_task = is_background_task self.email_account_doc = queue_doc.get_email_account() self.smtp_server = self.email_account_doc.get_smtp_server() @@ -287,16 +287,16 @@ class SendMailContext: ).decode() return message - def get_unsubscribe_str(self, recipient_email): + def get_unsubscribe_str(self, recipient_email: str) -> str: unsubscribe_url = "" + if self.queue_doc.add_unsubscribe_link and self.queue_doc.reference_doctype: - doctype, doc_name = self.queue_doc.reference_doctype, self.queue_doc.reference_name unsubscribe_url = get_unsubcribed_url( - doctype, - doc_name, - recipient_email, - self.queue_doc.unsubscribe_method, - self.queue_doc.unsubscribe_param, + reference_doctype=self.queue_doc.reference_doctype, + reference_name=self.queue_doc.reference_name, + email=recipient_email, + unsubscribe_method=self.queue_doc.unsubscribe_method, + unsubscribe_params=self.queue_doc.unsubscribe_param, ) return quopri.encodestring(unsubscribe_url.encode()).decode() diff --git a/frappe/email/email_body.py b/frappe/email/email_body.py index 50c66e1ad2..3a952e1487 100755 --- a/frappe/email/email_body.py +++ b/frappe/email/email_body.py @@ -7,6 +7,7 @@ import re from email import policy from email.header import Header from email.mime.multipart import MIMEMultipart +from typing import Optional import frappe from frappe.email.doctype.email_account.email_account import EmailAccount @@ -353,7 +354,7 @@ def get_formatted_html( print_html=None, email_account=None, header=None, - unsubscribe_link=None, + unsubscribe_link: Optional[frappe._dict] = None, sender=None, with_container=False, ): diff --git a/frappe/email/queue.py b/frappe/email/queue.py index 1519c26841..45abe0374a 100755 --- a/frappe/email/queue.py +++ b/frappe/email/queue.py @@ -67,37 +67,24 @@ def get_emails_sent_today(email_account=None): return frappe.db.sql(q, q_args)[0][0] -def get_unsubscribe_message(unsubscribe_message, expose_recipients): - if unsubscribe_message: - unsubscribe_html = """{0}""".format( - unsubscribe_message - ) - else: - unsubscribe_link = """{0}""".format( - _("Unsubscribe") - ) - unsubscribe_html = _("{0} to stop receiving emails of this type").format(unsubscribe_link) - - html = """""" + text = f"\n\n{unsubscribe_message}: \n" if expose_recipients == "footer": - text = "\n" - else: - text = "" - text += "\n\n{unsubscribe_message}: \n".format( - unsubscribe_message=unsubscribe_message - ) + text = f"\n{text}" - return frappe._dict({"html": html, "text": text}) + return frappe._dict(html=html, text=text) def get_unsubcribed_url( From 4bf7cd8210d92e89e9ad818fc9c72e317688d8d7 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Tue, 28 Jun 2022 12:19:52 +0530 Subject: [PATCH 200/205] fix: ignore virtual docfield property conflicts --- frappe/core/doctype/doctype/doctype.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/core/doctype/doctype/doctype.py b/frappe/core/doctype/doctype/doctype.py index b4cbbd6233..e56803acb7 100644 --- a/frappe/core/doctype/doctype/doctype.py +++ b/frappe/core/doctype/doctype/doctype.py @@ -171,7 +171,7 @@ class DocType(Document): if docfield.fieldname in method_set: conflict_type = "controller method" - if docfield.fieldname in property_set: + if docfield.fieldname in property_set and not docfield.is_virtual: conflict_type = "class property" if conflict_type: From 600186488866709b2099630fc3a402b6d0e5bb30 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Tue, 28 Jun 2022 11:38:40 +0530 Subject: [PATCH 201/205] fix(UX): show next execution time on scheduled job --- .../scheduled_job_type.json | 25 +++++++++++++++++-- .../scheduled_job_type/scheduled_job_type.py | 4 +++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/frappe/core/doctype/scheduled_job_type/scheduled_job_type.json b/frappe/core/doctype/scheduled_job_type/scheduled_job_type.json index d4d79b21fb..cc2a0e870a 100644 --- a/frappe/core/doctype/scheduled_job_type/scheduled_job_type.json +++ b/frappe/core/doctype/scheduled_job_type/scheduled_job_type.json @@ -16,8 +16,11 @@ "server_script", "frequency", "cron_format", + "create_log", + "status_section", "last_execution", - "create_log" + "column_break_9", + "next_execution" ], "fields": [ { @@ -72,6 +75,22 @@ "options": "Server Script", "read_only": 1, "search_index": 1 + }, + { + "fieldname": "next_execution", + "fieldtype": "Datetime", + "is_virtual": 1, + "label": "Next Execution", + "read_only": 1 + }, + { + "fieldname": "status_section", + "fieldtype": "Section Break", + "label": "Status" + }, + { + "fieldname": "column_break_9", + "fieldtype": "Column Break" } ], "in_create": 1, @@ -81,7 +100,7 @@ "link_fieldname": "scheduled_job_type" } ], - "modified": "2020-10-07 10:39:24.519460", + "modified": "2022-06-28 02:55:12.470915", "modified_by": "Administrator", "module": "Core", "name": "Scheduled Job Type", @@ -103,5 +122,7 @@ "quick_entry": 1, "sort_field": "modified", "sort_order": "DESC", + "states": [], + "title_field": "method", "track_changes": 1 } \ No newline at end of file diff --git a/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py b/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py index 318b156dcd..673805ae8b 100644 --- a/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py +++ b/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py @@ -50,6 +50,10 @@ class ScheduledJobType(Document): queued_jobs = get_jobs(site=frappe.local.site, key="job_type")[frappe.local.site] return self.method in queued_jobs + @property + def next_execution(self): + return self.get_next_execution() + def get_next_execution(self): CRON_MAP = { "Yearly": "0 0 1 1 *", From 971b8160a33fab9ce939e9c751fa45da7499e387 Mon Sep 17 00:00:00 2001 From: Shariq Ansari <30859809+shariquerik@users.noreply.github.com> Date: Tue, 28 Jun 2022 12:39:45 +0530 Subject: [PATCH 202/205] fix: extra column in excel after exporting report with group by (#17126) Co-authored-by: gavin --- frappe/desk/reportview.py | 22 ++++++++++++++++------ frappe/model/db_query.py | 5 +++++ frappe/tests/test_db_query.py | 4 +--- 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/frappe/desk/reportview.py b/frappe/desk/reportview.py index d6dce68399..893c2a6606 100644 --- a/frappe/desk/reportview.py +++ b/frappe/desk/reportview.py @@ -156,8 +156,6 @@ def setup_group_by(data): **data ) ) - if data.aggregate_on_field: - data.fields.append(f"`tab{data.aggregate_on_doctype}`.`{data.aggregate_on_field}`") else: raise_invalid_field(data.aggregate_on_field) @@ -435,11 +433,20 @@ def append_totals_row(data): def get_labels(fields, doctype): """get column labels based on column names""" labels = [] + doctype = doctype.lower() for key in fields: - key = key.split(" as ")[0] + aggregate_function = "" + + key = key.casefold().split(" as ", maxsplit=1)[0] if key.startswith(("count(", "sum(", "avg(")): - continue + # Get aggregate function and _aggregate_column + # key = 'sum(`tabDocType`.`fieldname`)' + if not key.rstrip().endswith(")"): + continue + _agg_fn, _key = key.split("(", maxsplit=1) + aggregate_function = _agg_fn.lower() # aggregate_function = 'sum' + key = _key[:-1] # key = `tabDocType`.`fieldname` if "." in key: parenttype, fieldname = key.split(".")[0][4:-1], key.split(".")[1].strip("`") @@ -455,7 +462,10 @@ def get_labels(fields, doctype): if parenttype != doctype: # If the column is from a child table, append the child doctype. # For example, "Item Code (Sales Invoice Item)". - label += f" ({ _(parenttype) })" + label += f" ({ _(parenttype.title()) })" + + if aggregate_function: + label = _("{0} of {1}").format(aggregate_function.capitalize(), label) labels.append(label) @@ -464,7 +474,7 @@ def get_labels(fields, doctype): def handle_duration_fieldtype_values(doctype, data, fields): for field in fields: - key = field.split(" as ")[0] + key = field.casefold().split(" as ", maxsplit=1)[0] if key.startswith(("count(", "sum(", "avg(")): continue diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py index 7fb38848e2..0b8f790246 100644 --- a/frappe/model/db_query.py +++ b/frappe/model/db_query.py @@ -417,6 +417,8 @@ class DatabaseQuery(object): "extract(", "locate(", "strpos(", + ] + aggregate_functions = [ "count(", "sum(", "avg(", @@ -427,6 +429,9 @@ class DatabaseQuery(object): if not ("tab" in field and "." in field) or any(x for x in sql_functions if x in field): continue + if any(x for x in aggregate_functions if x in field): + field = field.split("(", 1)[1][:-1] + table_name = field.split(".")[0] if table_name.lower().startswith("group_concat("): diff --git a/frappe/tests/test_db_query.py b/frappe/tests/test_db_query.py index c1b2e05266..2dd1743767 100644 --- a/frappe/tests/test_db_query.py +++ b/frappe/tests/test_db_query.py @@ -643,9 +643,7 @@ class TestReportview(unittest.TestCase): ) response = execute_cmd("frappe.desk.reportview.get") - self.assertListEqual( - response["keys"], ["field_label", "field_name", "_aggregate_column", "columns"] - ) + self.assertListEqual(response["keys"], ["field_label", "field_name", "_aggregate_column"]) def test_cast_name(self): from frappe.core.doctype.doctype.test_doctype import new_doctype From 4dff0d25a36942560bf65ace9e58bbb2da73d351 Mon Sep 17 00:00:00 2001 From: Shariq Ansari <30859809+shariquerik@users.noreply.github.com> Date: Tue, 28 Jun 2022 16:17:26 +0530 Subject: [PATCH 203/205] feat: add email retry limit in system settings (#17259) **MAX_RETRY_COUNT** for email was hard coded as 3. Added **Email Retry Limit** field in **Email** section in **System Settings** image >no-docs --- .../doctype/system_settings/system_settings.json | 13 ++++++++++--- frappe/email/doctype/email_queue/email_queue.py | 8 +++++--- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/frappe/core/doctype/system_settings/system_settings.json b/frappe/core/doctype/system_settings/system_settings.json index c954e41202..a444062b5a 100644 --- a/frappe/core/doctype/system_settings/system_settings.json +++ b/frappe/core/doctype/system_settings/system_settings.json @@ -63,6 +63,7 @@ "otp_issuer_name", "email", "email_footer_address", + "email_retry_limit", "column_break_18", "disable_standard_email_footer", "hide_footer_in_auto_email_reports", @@ -495,8 +496,8 @@ "fieldname": "allow_older_web_view_links", "fieldtype": "Check", "label": "Allow Older Web View Links (Insecure)" - }, - { + }, + { "fieldname": "column_break_64", "fieldtype": "Column Break" }, @@ -518,12 +519,18 @@ "fieldtype": "Duration", "label": "Reset Password Link Expiry Duration", "non_negative": 1 + }, + { + "default": "3", + "fieldname": "email_retry_limit", + "fieldtype": "Int", + "label": "Email Retry Limit" } ], "icon": "fa fa-cog", "issingle": 1, "links": [], - "modified": "2022-05-19 00:00:18.095269", + "modified": "2022-06-21 13:55:04.796152", "modified_by": "Administrator", "module": "Core", "name": "System Settings", diff --git a/frappe/email/doctype/email_queue/email_queue.py b/frappe/email/doctype/email_queue/email_queue.py index c3002607b4..4fc828eb62 100644 --- a/frappe/email/doctype/email_queue/email_queue.py +++ b/frappe/email/doctype/email_queue/email_queue.py @@ -30,8 +30,6 @@ from frappe.utils import ( split_emails, ) -MAX_RETRY_COUNT = 3 - class EmailQueue(Document): DOCTYPE = "Email Queue" @@ -210,7 +208,7 @@ class SendMailContext: email_status = (self.sent_to and "Partially Sent") or "Not Sent" self.queue_doc.update_status(status=email_status, commit=True) elif exc_type: - if self.queue_doc.retry < MAX_RETRY_COUNT: + if self.queue_doc.retry < get_email_retry_limit(): update_fields = {"status": "Not Sent", "retry": self.queue_doc.retry + 1} else: update_fields = {"status": (self.sent_to and "Partially Errored") or "Error"} @@ -372,6 +370,10 @@ def on_doctype_update(): ) +def get_email_retry_limit(): + return cint(frappe.db.get_system_setting("email_retry_limit")) or 3 + + class QueueBuilder: """Builds Email Queue from the given data""" From d58f2ed8e7cf1ef33df8196174193c760f2d2daf Mon Sep 17 00:00:00 2001 From: Samuel Danieli <23150094+scdanieli@users.noreply.github.com> Date: Tue, 28 Jun 2022 13:58:27 +0200 Subject: [PATCH 204/205] fix: german translations (#17324) --- frappe/translations/de.csv | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/frappe/translations/de.csv b/frappe/translations/de.csv index d00548458c..b22befd321 100644 --- a/frappe/translations/de.csv +++ b/frappe/translations/de.csv @@ -1993,7 +1993,7 @@ QR Code,QR-Code, QR Code for Login Verification,QR Code für Login-Bestätigung, QZ Tray Connection Active!,QZ-Tray-Verbindung aktiv!, QZ Tray Failed: ,QZ-Fach fehlgeschlagen:, -Quarter Day,Quartalstag, +Quarter Day,Viertel-Tag, Query,Abfrage, Query Report,Abfragebericht, Query must be a SELECT,Abfrage muss ein SELECT sein, @@ -4793,3 +4793,17 @@ Reset to default,Auf Standard zurücksetzen, Column Width,Spaltenbreite, Choose Kanban Board,Kanban-Tafel auswählen, Create New Board,Neue Tafel erstellen, +Only If Creator,Nur wenn Ersteller, +Rebuild Tree,Baum neu aufbauen, +Customize Dashboard,Dashboard anpassen, +Reset Dashboard Customizations,Dashboard-Anpassungen zurücksetzen, +Add {0},{0} hinzufügen, +descending,absteigend, +ascending,aufsteigend, +Next Document,Nächstes Dokument, +Previous Document,Vorheriges Dokument, +Mark all as read,Alle als gelesen markieren, +See all Activity,Alle Aktivitäten anzeigen, +Go to Notification Settings List,Gehe zur Listenansicht Benachrichtigungseinstellungen, +Load more,Mehr laden, +Edit Full Form,Vollständiges Formular bearbeiten, From c8ac3f922e88336f6398219063de2f2a368683d2 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Tue, 28 Jun 2022 18:40:27 +0530 Subject: [PATCH 205/205] Revert "fix: extra column in excel after exporting report with group by (#17126)" This reverts commit 971b8160a33fab9ce939e9c751fa45da7499e387. --- frappe/desk/reportview.py | 22 ++++++---------------- frappe/model/db_query.py | 5 ----- frappe/tests/test_db_query.py | 4 +++- 3 files changed, 9 insertions(+), 22 deletions(-) diff --git a/frappe/desk/reportview.py b/frappe/desk/reportview.py index 893c2a6606..d6dce68399 100644 --- a/frappe/desk/reportview.py +++ b/frappe/desk/reportview.py @@ -156,6 +156,8 @@ def setup_group_by(data): **data ) ) + if data.aggregate_on_field: + data.fields.append(f"`tab{data.aggregate_on_doctype}`.`{data.aggregate_on_field}`") else: raise_invalid_field(data.aggregate_on_field) @@ -433,20 +435,11 @@ def append_totals_row(data): def get_labels(fields, doctype): """get column labels based on column names""" labels = [] - doctype = doctype.lower() for key in fields: - aggregate_function = "" - - key = key.casefold().split(" as ", maxsplit=1)[0] + key = key.split(" as ")[0] if key.startswith(("count(", "sum(", "avg(")): - # Get aggregate function and _aggregate_column - # key = 'sum(`tabDocType`.`fieldname`)' - if not key.rstrip().endswith(")"): - continue - _agg_fn, _key = key.split("(", maxsplit=1) - aggregate_function = _agg_fn.lower() # aggregate_function = 'sum' - key = _key[:-1] # key = `tabDocType`.`fieldname` + continue if "." in key: parenttype, fieldname = key.split(".")[0][4:-1], key.split(".")[1].strip("`") @@ -462,10 +455,7 @@ def get_labels(fields, doctype): if parenttype != doctype: # If the column is from a child table, append the child doctype. # For example, "Item Code (Sales Invoice Item)". - label += f" ({ _(parenttype.title()) })" - - if aggregate_function: - label = _("{0} of {1}").format(aggregate_function.capitalize(), label) + label += f" ({ _(parenttype) })" labels.append(label) @@ -474,7 +464,7 @@ def get_labels(fields, doctype): def handle_duration_fieldtype_values(doctype, data, fields): for field in fields: - key = field.casefold().split(" as ", maxsplit=1)[0] + key = field.split(" as ")[0] if key.startswith(("count(", "sum(", "avg(")): continue diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py index 0b8f790246..7fb38848e2 100644 --- a/frappe/model/db_query.py +++ b/frappe/model/db_query.py @@ -417,8 +417,6 @@ class DatabaseQuery(object): "extract(", "locate(", "strpos(", - ] - aggregate_functions = [ "count(", "sum(", "avg(", @@ -429,9 +427,6 @@ class DatabaseQuery(object): if not ("tab" in field and "." in field) or any(x for x in sql_functions if x in field): continue - if any(x for x in aggregate_functions if x in field): - field = field.split("(", 1)[1][:-1] - table_name = field.split(".")[0] if table_name.lower().startswith("group_concat("): diff --git a/frappe/tests/test_db_query.py b/frappe/tests/test_db_query.py index 2dd1743767..c1b2e05266 100644 --- a/frappe/tests/test_db_query.py +++ b/frappe/tests/test_db_query.py @@ -643,7 +643,9 @@ class TestReportview(unittest.TestCase): ) response = execute_cmd("frappe.desk.reportview.get") - self.assertListEqual(response["keys"], ["field_label", "field_name", "_aggregate_column"]) + self.assertListEqual( + response["keys"], ["field_label", "field_name", "_aggregate_column", "columns"] + ) def test_cast_name(self): from frappe.core.doctype.doctype.test_doctype import new_doctype