Merge pull request #17827 from ankush/stable_coverage_submission

ci: stablilize coverage tracking

.github/helper/roulette.py

@@ -62,7 +62,6 @@ if __name__ == "__main__":
 	# this is a push build, run all builds
 	if not pr_number:
 		os.system('echo "::set-output name=build::strawberry"')
-		os.system('echo "::set-output name=build-server::strawberry"')
 		sys.exit(0)
 
 	files_list = files_list or get_files_list(pr_number=pr_number, repo=repo)

.github/workflows/server-mariadb-tests.yml

@@ -129,10 +129,26 @@ jobs:
 
       - name: Upload coverage data
         if: ${{ steps.check-build.outputs.build == 'strawberry' }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: coverage-${{ matrix.container }}
+          path: /home/runner/frappe-bench/sites/coverage.xml
+
+  coverage:
+    name: Coverage Wrap Up
+    needs: test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Clone
+        uses: actions/checkout@v3
+
+      - name: Download artifacts
+        uses: actions/download-artifact@v3
+
+      - name: Upload coverage data
         uses: codecov/codecov-action@v3
         with:
           name: MariaDB
           fail_ci_if_error: true
-          files: /home/runner/frappe-bench/sites/coverage.xml
           verbose: true
-          flags: server
+          flags: server-mariadb

.github/workflows/server-postgres-tests.yml

@@ -132,10 +132,26 @@ jobs:
 
       - name: Upload coverage data
         if: ${{ steps.check-build.outputs.build == 'strawberry' }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: coverage-${{ matrix.container }}
+          path: /home/runner/frappe-bench/sites/coverage.xml
+
+  coverage:
+    name: Coverage Wrap Up
+    needs: test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Clone
+        uses: actions/checkout@v3
+
+      - name: Download artifacts
+        uses: actions/download-artifact@v3
+
+      - name: Upload coverage data
         uses: codecov/codecov-action@v3
         with:
           name: Postgres
           fail_ci_if_error: true
-          files: /home/runner/frappe-bench/sites/coverage.xml
           verbose: true
-          flags: server
+          flags: server-postgres

.github/workflows/ui-tests.yml

@@ -21,7 +21,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-       containers: [1, 2, 3]
+       # Make sure you modify coverage submission file list if changing this
+       container: [1, 2, 3]
 
     name: UI Tests (Cypress)
 
@@ -148,38 +149,57 @@ jobs:
         env:
           CYPRESS_RECORD_KEY: 4a48f41c-11b3-425b-aa88-c58048fa69eb
 
-      - name: Stop server
-        if: ${{ steps.check-build.outputs.build-server == 'strawberry' }}
+      - name: Stop server and wait for coverage file
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         run: |
-          ps -ef | grep "frappe serve" | awk '{print $2}' | xargs kill -s SIGINT 2> /dev/null || true
+          ps -ef | grep "[f]rappe serve" | awk '{print $2}' | xargs kill -s SIGINT
           sleep 5
+          ( tail -f /home/runner/frappe-bench/sites/coverage.xml & ) | grep -q "\/coverage"
 
-      - name: Check If Coverage Report Exists
-        id: check_coverage
-        uses: andstor/file-existence-action@v1
+      - name: Upload JS coverage data
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
+        uses: actions/upload-artifact@v3
         with:
-          files: "/home/runner/frappe-bench/apps/frappe/.cypress-coverage/clover.xml"
+          name: coverage-js-${{ matrix.container }}
+          path: /home/runner/frappe-bench/apps/frappe/.cypress-coverage/clover.xml
 
-      - name: Upload Coverage Data
-        if: ${{ steps.check-build.outputs.build == 'strawberry' && steps.check_coverage.outputs.files_exists == 'true' }}
-        uses: codecov/codecov-action@v3
+      - name: Upload python coverage data
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
+        uses: actions/upload-artifact@v3
         with:
-          name: Cypress
-          fail_ci_if_error: true
-          directory: /home/runner/frappe-bench/apps/frappe/.cypress-coverage/
-          verbose: true
-          flags: ui-tests
+          name: coverage-py-${{ matrix.container }}
+          path: /home/runner/frappe-bench/sites/coverage.xml
 
-      - name: Upload Server Coverage Data
-        if: ${{ steps.check-build.outputs.build-server == 'strawberry' }}
+      - name: Show bench output
+        if: ${{ always() }}
+        run: cat ~/frappe-bench/bench_start.log || true
+
+
+  coverage:
+    name: Coverage Wrap Up
+    needs: test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Clone
+        uses: actions/checkout@v2
+
+      - name: Download artifacts
+        uses: actions/download-artifact@v3
+
+      - name: Upload python coverage data
         uses: codecov/codecov-action@v3
         with:
           name: MariaDB
           fail_ci_if_error: true
-          files: /home/runner/frappe-bench/sites/coverage.xml
           verbose: true
-          flags: server
+          files: ./coverage-py-1/coverage.xml,./coverage-py-2/coverage.xml,./coverage-py-3/coverage.xml
+          flags: server-ui
 
-      - name: Show bench console if tests failed
-        if: ${{ failure() }}
-        run: cat ~/frappe-bench/bench_start.log
+      - name: Upload JS coverage data
+        uses: codecov/codecov-action@v3
+        with:
+          name: Cypress
+          fail_ci_if_error: true
+          files: ./coverage-js-1/clover.xml,./coverage-js-2/clover.xml,./coverage-js-3/clover.xml
+          verbose: true
+          flags: ui-tests

codecov.yml

@@ -5,31 +5,39 @@ coverage:
   status:
     project:
       default: false
-      server:
+      server-mariadb:
         target: auto
         threshold: 0.5%
         flags:
-          - server
+          - server-mariadb
     patch:
       default: false
-      server:
+      server-mariadb:
         target: 85%
         threshold: 0%
         only_pulls: true
         if_ci_failed: ignore
         flags:
-          - server
+          - server-mariadb
 
 comment:
   layout: "diff, flags"
   require_changes: true
 
 flags:
-  server:
+  server-mariadb:
     paths:
-      - ".*\\.py"
+      - "**/*.py"
+    carryforward: true
+  server-postgres:
+    paths:
+      - "**/*.py"
     carryforward: true
   ui-tests:
     paths:
-      - ".*\\.js"
+      - "**/*.js"
+    carryforward: true
+  server-ui:
+    paths:
+      - "**/*.py"
     carryforward: true

frappe/coverage.py

@@ -73,3 +73,4 @@ class CodeCoverage:
 			self.coverage.stop()
 			self.coverage.save()
 			self.coverage.xml_report()
+			print("Saved Coverage")