Merge pull request #14269 from mituldavid/coverage

.github/workflows/server-mariadb-tests.yml

@@ -127,4 +127,5 @@ jobs:
           name: MariaDB
           fail_ci_if_error: true
           files: /home/runner/frappe-bench/sites/coverage.xml
-          verbose: true
+          verbose: true
+          flags: server

.github/workflows/server-postgres-tests.yml

@@ -131,3 +131,4 @@ jobs:
           fail_ci_if_error: true
           files: /home/runner/frappe-bench/sites/coverage.xml
           verbose: true
+          flags: server

.github/workflows/ui-tests.yml

@@ -122,12 +122,36 @@ jobs:
           DB: mariadb
           TYPE: ui
 
+      - name: Instrument Source Code
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
+        run: cd ~/frappe-bench/apps/frappe/ && npx nyc instrument -x 'frappe/public/dist/**' -x 'frappe/public/js/lib/**' -x '**/*.bundle.js' --compact=false --in-place frappe
+
+      - name: Build
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
+        run: cd ~/frappe-bench/ && bench build --apps frappe
+
       - name: Site Setup
         if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         run: cd ~/frappe-bench/ && bench --site test_site execute frappe.utils.install.complete_setup_wizard
 
       - name: UI Tests
         if: ${{ steps.check-build.outputs.build == 'strawberry' }}
-        run: cd ~/frappe-bench/ && bench --site test_site run-ui-tests frappe --headless --parallel --ci-build-id $GITHUB_RUN_ID
+        run: cd ~/frappe-bench/ && bench --site test_site run-ui-tests frappe --with-coverage --headless --parallel --ci-build-id $GITHUB_RUN_ID
         env:
           CYPRESS_RECORD_KEY: 4a48f41c-11b3-425b-aa88-c58048fa69eb
+
+      - name: Check If Coverage Report Exists
+        id: check_coverage
+        uses: andstor/file-existence-action@v1
+        with:
+          files: "/home/runner/frappe-bench/apps/frappe/.cypress-coverage/clover.xml"
+
+      - name: Upload Coverage Data
+        if: ${{ steps.check-build.outputs.build == 'strawberry' && steps.check_coverage.outputs.files_exists == 'true' }}
+        uses: codecov/codecov-action@v2
+        with:
+          name: Cypress
+          fail_ci_if_error: true
+          directory: /home/runner/frappe-bench/apps/frappe/.cypress-coverage/
+          verbose: true
+          flags: ui-tests

.gitignore

@@ -67,6 +67,7 @@ coverage.xml
 *.cover
 .hypothesis/
 .pytest_cache/
+.cypress-coverage
 
 # Translations
 *.mo

codecov.yml

@@ -4,10 +4,28 @@ codecov:
 coverage:
   status:
     project:
-      default:
+      default: false
+      server:
         target: auto
         threshold: 0.5%
+        flags:
+          - server
+      ui-tests:
+        target: auto
+        threshold: 0.5%
+        flags:
+          - ui-tests
 
 comment:
-  layout: "diff"
+  layout: "diff, flags"
   require_changes: true
+
+flags:
+  server:
+    paths:
+      - ".*\\.py"
+    carryforward: true
+  ui-tests:
+    paths:
+      - ".*\\.js"
+    carryforward: true

cypress/integration/dashboard_links.js

@@ -54,13 +54,12 @@ context('Dashboard links', () => {
 				cur_frm.dashboard.data.reports = [
 					{
 						'label': 'Reports',
-						'items': ['Permitted Documents For User']
+						'items': ['Website Analytics']
 					}
 				];
 				cur_frm.dashboard.render_report_links();
-				cy.get('[data-report="Permitted Documents For User"]').contains('Permitted Documents For User').click();
-				cy.findByText('Permitted Documents For User');
-				cy.findByPlaceholderText('User').should("have.value", "Administrator");
+				cy.get('[data-report="Website Analytics"]').contains('Website Analytics').click();
+				cy.findByText('Website Analytics');
 			});
 	});
 });

cypress/plugins/index.js

@@ -11,7 +11,7 @@
 // This function is called when a project is opened or re-opened (e.g. due to
 // the project's config changing)
 
-module.exports = () => {
-	// `on` is used to hook into various events Cypress emits
-	// `config` is the resolved Cypress config
-};
+module.exports = (on, config) => {
+	require('@cypress/code-coverage/task')(on, config);
+	return config;
+};

cypress/support/index.js

@@ -15,6 +15,7 @@
 
 // Import commands.js using ES2015 syntax:
 import './commands';
+import '@cypress/code-coverage/support';
 
 
 // Alternatively you can use CommonJS syntax:

frappe/commands/utils.py

@@ -679,9 +679,10 @@ def run_parallel_tests(context, app, build_number, total_builds, with_coverage=F
 @click.argument('app')
 @click.option('--headless', is_flag=True, help="Run UI Test in headless mode")
 @click.option('--parallel', is_flag=True, help="Run UI Test in parallel mode")
+@click.option('--with-coverage', is_flag=True, help="Generate coverage report")
 @click.option('--ci-build-id')
 @pass_context
-def run_ui_tests(context, app, headless=False, parallel=True, ci_build_id=None):
+def run_ui_tests(context, app, headless=False, parallel=True, with_coverage=False, ci_build_id=None):
 	"Run UI tests"
 	site = get_site(context)
 	app_base_path = os.path.abspath(os.path.join(frappe.get_app_path(app), '..'))
@@ -691,6 +692,7 @@ def run_ui_tests(context, app, headless=False, parallel=True, ci_build_id=None):
 	# override baseUrl using env variable
 	site_env = f'CYPRESS_baseUrl={site_url}'
 	password_env = f'CYPRESS_adminPassword={admin_password}' if admin_password else ''
+	coverage_env = f'CYPRESS_coverage={str(with_coverage).lower()}'
 
 	os.chdir(app_base_path)
 
@@ -698,22 +700,23 @@ def run_ui_tests(context, app, headless=False, parallel=True, ci_build_id=None):
 	cypress_path = f"{node_bin}/cypress"
 	plugin_path = f"{node_bin}/../cypress-file-upload"
 	testing_library_path = f"{node_bin}/../@testing-library"
+	coverage_plugin_path = f"{node_bin}/../@cypress/code-coverage"
 
 	# check if cypress in path...if not, install it.
 	if not (
 		os.path.exists(cypress_path)
 		and os.path.exists(plugin_path)
 		and os.path.exists(testing_library_path)
+		and os.path.exists(coverage_plugin_path)
 		and cint(subprocess.getoutput("npm view cypress version")[:1]) >= 6
 	):
 		# install cypress
 		click.secho("Installing Cypress...", fg="yellow")
-		frappe.commands.popen("yarn add cypress@^6 cypress-file-upload@^5 @testing-library/cypress@^8 --no-lockfile")
+		frappe.commands.popen("yarn add cypress@^6 cypress-file-upload@^5 @testing-library/cypress@^8 @cypress/code-coverage@^3 --no-lockfile")
 
 	# run for headless mode
 	run_or_open = 'run --browser firefox --record' if headless else 'open'
-	command = '{site_env} {password_env} {cypress} {run_or_open}'
-	formatted_command = command.format(site_env=site_env, password_env=password_env, cypress=cypress_path, run_or_open=run_or_open)
+	formatted_command = f'{site_env} {password_env} {coverage_env} {cypress_path} {run_or_open}'
 
 	if parallel:
 		formatted_command += ' --parallel'

package.json

@@ -4,7 +4,8 @@
     "build": "node esbuild",
     "production": "node esbuild --production",
     "watch": "node esbuild --watch",
-    "snyk-protect": "snyk protect"
+    "snyk-protect": "snyk protect",
+    "coverage:report": "npx nyc report --reporter=clover"
   },
   "repository": {
     "type": "git",
@@ -74,5 +75,8 @@
     "rtlcss": "^3.2.1",
     "yargs": "^16.2.0"
   },
-  "snyk": true
+  "snyk": true,
+  "nyc": {
+    "report-dir": ".cypress-coverage"
+  }
 }