diff --git a/.github/helper/roulette.py b/.github/helper/roulette.py index d00c47d8d7..9831df7f30 100644 --- a/.github/helper/roulette.py +++ b/.github/helper/roulette.py @@ -25,7 +25,7 @@ def is_ci(file): return ".github" in file def is_frontend_code(file): - return file.endswith((".css", ".scss", ".less", ".sass", ".styl", ".js", ".ts")) + return file.lower().endswith((".css", ".scss", ".less", ".sass", ".styl", ".js", ".ts", ".vue")) def is_docs(file): regex = re.compile(r'\.(md|png|jpg|jpeg|csv)$|^.github|LICENSE') @@ -38,8 +38,12 @@ if __name__ == "__main__": pr_number = os.environ.get("PR_NUMBER") repo = os.environ.get("REPO_NAME") - if not files_list and pr_number: - files_list = get_files_list(pr_number=pr_number, repo=repo) + # this is a push build, run all builds + if not pr_number: + os.system('echo "::set-output name=build::strawberry"') + sys.exit(0) + + files_list = files_list or get_files_list(pr_number=pr_number, repo=repo) if not files_list: print("No files' changes detected. Build is shutting") diff --git a/.github/workflows/patch-mariadb-tests.yml b/.github/workflows/patch-mariadb-tests.yml index 6ccc059afb..0dd4cd51d8 100644 --- a/.github/workflows/patch-mariadb-tests.yml +++ b/.github/workflows/patch-mariadb-tests.yml @@ -2,6 +2,11 @@ name: Patch on: [pull_request, workflow_dispatch] + +concurrency: + group: patch-mariadb-develop-${{ github.event.number }} + cancel-in-progress: true + jobs: test: runs-on: ubuntu-18.04 diff --git a/.github/workflows/server-mariadb-tests.yml b/.github/workflows/server-mariadb-tests.yml index 65b6666678..fb6e56037c 100644 --- a/.github/workflows/server-mariadb-tests.yml +++ b/.github/workflows/server-mariadb-tests.yml @@ -6,6 +6,11 @@ on: push: branches: [ develop ] +concurrency: + group: server-mariadb-develop-${{ github.event.number }} + cancel-in-progress: true + + jobs: test: runs-on: ubuntu-18.04 @@ -131,17 +136,29 @@ jobs: COVERALLS_SERVICE_NAME: ${{ github.event_name == 'pull_request' && 'github' || 'github-actions' }} COVERALLS_PARALLEL: true + - run: echo ${{ steps.check-build.outputs.build }} > guess-the-fruit.txt + - uses: actions/upload-artifact@v1 + with: + name: fruit + path: guess-the-fruit.txt + coveralls: name: Coverage Wrap Up needs: test - if: ${{ needs.test.steps.check-build.build == 'strawberry' }} container: python:3-slim runs-on: ubuntu-18.04 steps: + - uses: actions/download-artifact@v1 + with: + name: fruit + - run: echo "WILDCARD=$(cat fruit/guess-the-fruit.txt)" >> $GITHUB_ENV + - name: Clone + if: ${{ env.WILDCARD == 'strawberry' }} uses: actions/checkout@v2 - name: Coveralls Finished + if: ${{ env.WILDCARD == 'strawberry' }} run: | cd ${GITHUB_WORKSPACE} pip3 install coverage==5.5 diff --git a/.github/workflows/server-postgres-tests.yml b/.github/workflows/server-postgres-tests.yml index 17a0f6f94f..1539e8c2d5 100644 --- a/.github/workflows/server-postgres-tests.yml +++ b/.github/workflows/server-postgres-tests.yml @@ -4,6 +4,10 @@ on: pull_request: workflow_dispatch: +concurrency: + group: server-postgres-develop-${{ github.event.number }} + cancel-in-progress: true + jobs: test: runs-on: ubuntu-18.04 diff --git a/.github/workflows/ui-tests.yml b/.github/workflows/ui-tests.yml index d56433c216..2a55546ec4 100644 --- a/.github/workflows/ui-tests.yml +++ b/.github/workflows/ui-tests.yml @@ -6,6 +6,10 @@ on: push: branches: [ develop ] +concurrency: + group: ui-develop-${{ github.event.number }} + cancel-in-progress: true + jobs: test: runs-on: ubuntu-18.04 diff --git a/CODEOWNERS b/CODEOWNERS index 2dff157294..30cdb4d64d 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -7,10 +7,13 @@ templates/ @surajshetty3416 www/ @surajshetty3416 integrations/ @leela -patches/ @surajshetty3416 +patches/ @surajshetty3416 @gavindsouza email/ @leela event_streaming/ @ruchamahabal data_import* @netchampfaris core/ @surajshetty3416 +database @gavindsouza +model @gavindsouza requirements.txt @gavindsouza commands/ @gavindsouza +workspace @shariquerik diff --git a/cypress/integration/datetime_field_form_validation.js b/cypress/integration/datetime_field_form_validation.js new file mode 100644 index 0000000000..66fdde6863 --- /dev/null +++ b/cypress/integration/datetime_field_form_validation.js @@ -0,0 +1,19 @@ +context('Datetime Field Validation', () => { + before(() => { + cy.login(); + cy.visit('/app/communication'); + cy.window().its('frappe').then(frappe => { + frappe.call("frappe.tests.ui_test_helpers.create_communication_records"); + }); + }); + + // validating datetime field value when value is set from backend and get validated on form load. + it('datetime field form validation', () => { + cy.visit('/app/communication'); + cy.get('a[title="Test Form Communication 1"]').invoke('attr', 'data-name') + .then((name) => { + cy.visit(`/app/communication/${name}`); + cy.get('.indicator-pill').should('contain', 'Open').should('have.class', 'red'); + }); + }); +}); \ No newline at end of file diff --git a/cypress/integration/folder_navigation.js b/cypress/integration/folder_navigation.js new file mode 100644 index 0000000000..1b7c02d98c --- /dev/null +++ b/cypress/integration/folder_navigation.js @@ -0,0 +1,79 @@ +context('Folder Navigation', () => { + before(() => { + cy.visit('/login'); + cy.login(); + cy.visit('/app/file'); + }); + + it('Adding Folders', () => { + //Adding filter to go into the home folder + cy.get('.filter-selector > .btn').findByText('1 filter').click(); + cy.findByRole('button', {name: 'Clear Filters'}).click(); + cy.get('.filter-action-buttons > .text-muted').findByText('+ Add a Filter').click(); + cy.get('.fieldname-select-area > .awesomplete > .form-control').type('Fol{enter}'); + cy.get('.filter-field > .form-group > .link-field > .awesomplete > .input-with-feedback').type('Home{enter}'); + cy.get('.filter-action-buttons > div > .btn-primary').findByText('Apply Filters').click(); + + //Adding folder (Test Folder) + cy.get('.menu-btn-group > .btn').click(); + cy.get('.menu-btn-group [data-label="New Folder"]').click(); + cy.get('form > [data-fieldname="value"]').type('Test Folder'); + cy.findByRole('button', {name: 'Create'}).click(); + }); + + it('Navigating the nested folders, checking if the URL formed is correct, checking if the added content in the child folder is correct', () => { + //Navigating inside the Attachments folder + cy.get('[title="Attachments"] > span').click(); + + //To check if the URL formed after visiting the attachments folder is correct + cy.location('pathname').should('eq', '/app/file/view/home/Attachments'); + cy.visit('/app/file/view/home/Attachments'); + + //Adding folder inside the attachments folder + cy.get('.menu-btn-group > .btn').click(); + cy.get('.menu-btn-group [data-label="New Folder"]').click(); + cy.get('form > [data-fieldname="value"]').type('Test Folder'); + cy.findByRole('button', {name: 'Create'}).click(); + + //Navigating inside the added folder in the Attachments folder + cy.get('[title="Test Folder"] > span').click(); + + //To check if the URL is correct after visiting the Test Folder + cy.location('pathname').should('eq', '/app/file/view/home/Attachments/Test%20Folder'); + cy.visit('/app/file/view/home/Attachments/Test%20Folder'); + + //Adding a file inside the Test Folder + cy.findByRole('button', {name: 'Add File'}).eq(0).click({force: true}); + cy.get('.file-uploader').findByText('Link').click(); + cy.get('.input-group > .form-control').type('https://wallpaperplay.com/walls/full/8/2/b/72402.jpg'); + cy.findByRole('button', {name: 'Upload'}).click(); + + //To check if the added file is present in the Test Folder + cy.get('span.level-item > span').should('contain', 'Test Folder'); + cy.get('.list-row-container').eq(0).should('contain.text', '72402.jpg'); + cy.get('.list-row-checkbox').eq(0).click(); + + //Deleting the added file from the Test folder + cy.findByRole('button', {name: 'Actions'}).click(); + cy.get('.actions-btn-group [data-label="Delete"]').click(); + cy.wait(700); + cy.findByRole('button', {name: 'Yes'}).click(); + cy.wait(700); + + //Deleting the Test Folder + cy.visit('/app/file/view/home/Attachments'); + cy.get('.list-row-checkbox').eq(0).click(); + cy.findByRole('button', {name: 'Actions'}).click(); + cy.get('.actions-btn-group [data-label="Delete"]').click(); + cy.findByRole('button', {name: 'Yes'}).click(); + }); + + it('Deleting Test Folder from the home', () => { + //Deleting the Test Folder added in the home directory + cy.visit('/app/file/view/home'); + cy.get('.level-left > .list-subject > .list-row-checkbox').eq(0).click({force: true, delay: 500}); + cy.findByRole('button', {name: 'Actions'}).click(); + cy.get('.actions-btn-group [data-label="Delete"]').click(); + cy.findByRole('button', {name: 'Yes'}).click(); + }); +}); diff --git a/frappe/core/doctype/file/file.js b/frappe/core/doctype/file/file.js index bc0cc17553..d40328d3cd 100644 --- a/frappe/core/doctype/file/file.js +++ b/frappe/core/doctype/file/file.js @@ -29,15 +29,8 @@ frappe.ui.form.on("File", "refresh", function(frm) { if (is_optimizable) { frm.add_custom_button(__("Optimize"), function() { frappe.show_alert(__("Optimizing image...")); - frappe.call({ - method: "frappe.core.doctype.file.file.optimize_saved_image", - args: { - doc_name: frm.doc.name, - }, - callback: function() { - frappe.show_alert(__("Image optimized")); - frappe.set_route("List", "File"); - } + frm.call("optimize_file").then(() => { + frappe.show_alert(__("Image optimized")); }); }); } diff --git a/frappe/core/doctype/file/file.py b/frappe/core/doctype/file/file.py index b8ea134db5..36ff67ce7c 100755 --- a/frappe/core/doctype/file/file.py +++ b/frappe/core/doctype/file/file.py @@ -313,8 +313,16 @@ class File(Document): self.delete_file_data_content(only_thumbnail=True) def on_rollback(self): - self.flags.on_rollback = True - self.on_trash() + # if original_content flag is set, this rollback should revert the file to its original state + if self.flags.original_content: + file_path = self.get_full_path() + with open(file_path, "wb+") as f: + f.write(self.flags.original_content) + + # following condition is only executed when an insert has been rolledback + else: + self.flags.on_rollback = True + self.on_trash() def unzip(self): '''Unzip current file and replace it by its children''' @@ -531,6 +539,35 @@ class File(Document): if self.file_url: self.is_private = cint(self.file_url.startswith('/private')) + @frappe.whitelist() + def optimize_file(self): + if self.is_folder: + raise TypeError('Folders cannot be optimized') + + content_type = mimetypes.guess_type(self.file_name)[0] + is_local_image = content_type.startswith('image/') and self.file_size > 0 + is_svg = content_type == 'image/svg+xml' + + if not is_local_image: + raise NotImplementedError('Only local image files can be optimized') + + if is_svg: + raise TypeError('Optimization of SVG images is not supported') + + content = self.get_content() + file_path = self.get_full_path() + optimized_content = optimize_image(content, content_type) + + with open(file_path, 'wb+') as f: + f.write(optimized_content) + + self.file_size = len(optimized_content) + self.content_hash = get_content_hash(optimized_content) + # if rolledback, revert back to original + self.flags.original_content = content + frappe.local.rollback_observers.append(self) + self.save() + def on_doctype_update(): frappe.db.add_index("File", ["attached_to_doctype", "attached_to_name"]) @@ -838,22 +875,6 @@ def unzip_file(name): files = file_obj.unzip() return files -@frappe.whitelist() -def optimize_saved_image(doc_name): - file_doc = frappe.get_doc('File', doc_name) - content = file_doc.get_content() - content_type = mimetypes.guess_type(file_doc.file_name)[0] - - optimized_content = optimize_image(content, content_type) - - file_path = get_files_path(is_private=file_doc.is_private) - file_path = os.path.join(file_path.encode('utf-8'), file_doc.file_name.encode('utf-8')) - with open(file_path, 'wb+') as f: - f.write(optimized_content) - - file_doc.file_size = len(optimized_content) - file_doc.content_hash = get_content_hash(optimized_content) - file_doc.save() @frappe.whitelist() def get_attached_images(doctype, names): diff --git a/frappe/core/doctype/file/test_file.py b/frappe/core/doctype/file/test_file.py index 5478d7ab85..18ad95eeba 100644 --- a/frappe/core/doctype/file/test_file.py +++ b/frappe/core/doctype/file/test_file.py @@ -440,6 +440,7 @@ class TestFile(unittest.TestCase): }).insert(ignore_permissions=True) self.assertRaisesRegex(frappe.exceptions.ValidationError, 'not a zip file', test_file.unzip) + class TestAttachment(unittest.TestCase): test_doctype = 'Test For Attachment' @@ -569,3 +570,68 @@ class TestFileUtils(unittest.TestCase): from frappe.core.doctype.file.file import create_new_folder folder = create_new_folder('test_folder', 'Home') self.assertTrue(folder.is_folder) + + +class TestFileOptimization(unittest.TestCase): + def test_optimize_file(self): + file_path = frappe.get_app_path("frappe", "tests/data/sample_image_for_optimization.jpg") + with open(file_path, "rb") as f: + file_content = f.read() + test_file = frappe.get_doc({ + "doctype": "File", + "file_name": "sample_image_for_optimization.jpg", + "content": file_content + }).insert() + original_size = test_file.file_size + original_content_hash = test_file.content_hash + + test_file.optimize_file() + optimized_size = test_file.file_size + updated_content_hash = test_file.content_hash + + self.assertLess(optimized_size, original_size) + self.assertNotEqual(original_content_hash, updated_content_hash) + test_file.delete() + + def test_optimize_svg(self): + file_path = frappe.get_app_path("frappe", "tests/data/sample_svg.svg") + with open(file_path, "rb") as f: + file_content = f.read() + test_file = frappe.get_doc({ + "doctype": "File", + "file_name": "sample_svg.svg", + "content": file_content + }).insert() + self.assertRaises(TypeError, test_file.optimize_file) + test_file.delete() + + def test_optimize_textfile(self): + test_file = frappe.get_doc({ + "doctype": "File", + "file_name": "sample_text.txt", + "content": "Text files cannot be optimized" + }).insert() + self.assertRaises(NotImplementedError, test_file.optimize_file) + test_file.delete() + + def test_optimize_folder(self): + test_folder = frappe.get_doc("File", "Home/Attachments") + self.assertRaises(TypeError, test_folder.optimize_file) + + def test_revert_optimized_file_on_rollback(self): + file_path = frappe.get_app_path("frappe", "tests/data/sample_image_for_optimization.jpg") + with open(file_path, "rb") as f: + file_content = f.read() + test_file = frappe.get_doc({ + "doctype": "File", + "file_name": "sample_image_for_optimization.jpg", + "content": file_content + }).insert() + image_path = test_file.get_full_path() + size_before_optimization = os.stat(image_path).st_size + + test_file.optimize_file() + frappe.db.rollback() + size_after_rollback = os.stat(image_path).st_size + self.assertEqual(size_before_optimization, size_after_rollback) + test_file.delete() \ No newline at end of file diff --git a/frappe/custom/doctype/customize_form/customize_form.py b/frappe/custom/doctype/customize_form/customize_form.py index 8de194fb00..94f25a41aa 100644 --- a/frappe/custom/doctype/customize_form/customize_form.py +++ b/frappe/custom/doctype/customize_form/customize_form.py @@ -193,6 +193,16 @@ class CustomizeForm(Document): if prop == "fieldtype": self.validate_fieldtype_change(df, meta_df[0].get(prop), df.get(prop)) + elif prop == "length": + old_value_length = cint(meta_df[0].get(prop)) + new_value_length = cint(df.get(prop)) + + if new_value_length and (old_value_length > new_value_length): + self.check_length_for_fieldtypes.append({'df': df, 'old_value': meta_df[0].get(prop)}) + self.validate_fieldtype_length() + else: + self.flags.update_db = True + elif prop == "allow_on_submit" and df.get(prop): if not frappe.db.get_value("DocField", {"parent": self.doc_type, "fieldname": df.fieldname}, "allow_on_submit"): diff --git a/frappe/custom/doctype/customize_form/test_customize_form.py b/frappe/custom/doctype/customize_form/test_customize_form.py index aef95cd676..266ece8a1e 100644 --- a/frappe/custom/doctype/customize_form/test_customize_form.py +++ b/frappe/custom/doctype/customize_form/test_customize_form.py @@ -188,6 +188,26 @@ class TestCustomizeForm(unittest.TestCase): def test_core_doctype_customization(self): self.assertRaises(frappe.ValidationError, self.get_customize_form, 'User') + def test_save_customization_length_field_property(self): + # Using Notification Log doctype as it doesn't have any other custom fields + d = self.get_customize_form("Notification Log") + + document_name = d.get("fields", {"fieldname": "document_name"})[0] + document_name.length = 255 + d.run_method("save_customization") + + self.assertEqual(frappe.db.get_value("Property Setter", + {"doc_type": "Notification Log", "property": "length", "field_name": "document_name"}, "value"), '255') + + self.assertTrue(d.flags.update_db) + + length = frappe.db.sql("""SELECT character_maximum_length + FROM information_schema.columns + WHERE table_name = 'tabNotification Log' + AND column_name = 'document_name'""")[0][0] + + self.assertEqual(length, 255) + def test_custom_link(self): try: # create a dummy doctype linked to Event diff --git a/frappe/desk/query_report.py b/frappe/desk/query_report.py index b42c9c89a0..610eaf466a 100644 --- a/frappe/desk/query_report.py +++ b/frappe/desk/query_report.py @@ -391,7 +391,7 @@ def handle_duration_fieldtype_values(result, columns): return result -def build_xlsx_data(columns, data, visible_idx, include_indentation): +def build_xlsx_data(columns, data, visible_idx, include_indentation, ignore_visible_idx=False): result = [[]] column_widths = [] @@ -407,7 +407,7 @@ def build_xlsx_data(columns, data, visible_idx, include_indentation): # build table from result for row_idx, row in enumerate(data.result): # only pick up rows that are visible in the report - if row_idx in visible_idx: + if ignore_visible_idx or row_idx in visible_idx: row_data = [] if isinstance(row, dict): for col_idx, column in enumerate(data.columns): diff --git a/frappe/email/doctype/auto_email_report/auto_email_report.py b/frappe/email/doctype/auto_email_report/auto_email_report.py index f30279e308..ccfff594b7 100644 --- a/frappe/email/doctype/auto_email_report/auto_email_report.py +++ b/frappe/email/doctype/auto_email_report/auto_email_report.py @@ -13,6 +13,7 @@ from frappe.utils import (format_time, get_link_to_form, get_url_to_report, from frappe.model.naming import append_number_if_name_exists from frappe.utils.csvutils import to_csv from frappe.utils.xlsxutils import make_xlsx +from frappe.desk.query_report import build_xlsx_data max_reports_per_user = frappe.local.conf.max_reports_per_user or 3 @@ -99,13 +100,21 @@ class AutoEmailReport(Document): return self.get_html_table(columns, data) elif self.format == 'XLSX': - spreadsheet_data = self.get_spreadsheet_data(columns, data) - xlsx_file = make_xlsx(spreadsheet_data, "Auto Email Report") + report_data = frappe._dict() + report_data['columns'] = columns + report_data['result'] = data + + xlsx_data, column_widths = build_xlsx_data(columns, report_data, [], 1, ignore_visible_idx=True) + xlsx_file = make_xlsx(xlsx_data, "Auto Email Report", column_widths=column_widths) return xlsx_file.getvalue() elif self.format == 'CSV': - spreadsheet_data = self.get_spreadsheet_data(columns, data) - return to_csv(spreadsheet_data) + report_data = frappe._dict() + report_data['columns'] = columns + report_data['result'] = data + + xlsx_data, column_widths = build_xlsx_data(columns, report_data, [], 1, ignore_visible_idx=True) + return to_csv(xlsx_data) else: frappe.throw(_('Invalid Output Format')) @@ -126,18 +135,6 @@ class AutoEmailReport(Document): 'edit_report_settings': get_link_to_form('Auto Email Report', self.name) }) - @staticmethod - def get_spreadsheet_data(columns, data): - out = [[_(df.label) for df in columns], ] - for row in data: - new_row = [] - out.append(new_row) - for df in columns: - if df.fieldname not in row: continue - new_row.append(frappe.format(row[df.fieldname], df, row)) - - return out - def get_file_name(self): return "{0}.{1}".format(self.report.replace(" ", "-").replace("/", "-"), self.format.lower()) diff --git a/frappe/model/base_document.py b/frappe/model/base_document.py index 752543f46a..5603b2daae 100644 --- a/frappe/model/base_document.py +++ b/frappe/model/base_document.py @@ -874,7 +874,7 @@ class BaseDocument(object): return self._precision[cache_key][fieldname] - def get_formatted(self, fieldname, doc=None, currency=None, absolute_value=False, translated=False): + def get_formatted(self, fieldname, doc=None, currency=None, absolute_value=False, translated=False, format=None): from frappe.utils.formatters import format_value df = self.meta.get_field(fieldname) @@ -898,7 +898,7 @@ class BaseDocument(object): if (absolute_value or doc.get('absolute_value')) and isinstance(val, (int, float)): val = abs(self.get(fieldname)) - return format_value(val, df=df, doc=doc, currency=currency) + return format_value(val, df=df, doc=doc, currency=currency, format=format) def is_print_hide(self, fieldname, df=None, for_print=True): """Returns true if fieldname is to be hidden for print. diff --git a/frappe/public/icons/timeless/symbol-defs.svg b/frappe/public/icons/timeless/symbol-defs.svg index f216374526..b2f1428967 100644 --- a/frappe/public/icons/timeless/symbol-defs.svg +++ b/frappe/public/icons/timeless/symbol-defs.svg @@ -567,7 +567,7 @@ - diff --git a/frappe/public/js/frappe/data_import/data_exporter.js b/frappe/public/js/frappe/data_import/data_exporter.js index 03e6288856..8fa5a08945 100644 --- a/frappe/public/js/frappe/data_import/data_exporter.js +++ b/frappe/public/js/frappe/data_import/data_exporter.js @@ -13,6 +13,13 @@ frappe.data_import.DataExporter = class DataExporter { this.dialog = new frappe.ui.Dialog({ title: __('Export Data'), fields: [ + { + fieldtype: 'Select', + fieldname: 'file_type', + label: __('File Type'), + options: ['Excel', 'CSV'], + default: 'CSV' + }, { fieldtype: 'Select', fieldname: 'export_records', @@ -45,13 +52,6 @@ frappe.data_import.DataExporter = class DataExporter { fieldname: 'filter_area', depends_on: doc => doc.export_records === 'by_filter' }, - { - fieldtype: 'Select', - fieldname: 'file_type', - label: __('File Type'), - options: ['Excel', 'CSV'], - default: 'CSV' - }, { fieldtype: 'Section Break' }, @@ -141,7 +141,7 @@ frappe.data_import.DataExporter = class DataExporter { let for_insert = this.exporting_for === 'Insert New Records'; let section_title = for_insert ? __('Select Fields To Insert') : __('Select Fields To Update'); let $select_all_buttons = $(` -
+
${section_title}