From eab85fc291403e316d079037841719d0c70a3cb3 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sat, 13 Aug 2022 18:18:11 +0530 Subject: [PATCH 1/4] ci: stablilize coverage tracking --- .github/workflows/server-mariadb-tests.yml | 20 +++++++- .github/workflows/server-postgres-tests.yml | 20 +++++++- .github/workflows/ui-tests.yml | 57 +++++++++++++++------ codecov.yml | 14 +++-- 4 files changed, 88 insertions(+), 23 deletions(-) diff --git a/.github/workflows/server-mariadb-tests.yml b/.github/workflows/server-mariadb-tests.yml index 051b657b0e..f5b87178ee 100644 --- a/.github/workflows/server-mariadb-tests.yml +++ b/.github/workflows/server-mariadb-tests.yml @@ -129,10 +129,26 @@ jobs: - name: Upload coverage data if: ${{ steps.check-build.outputs.build == 'strawberry' }} + uses: actions/upload-artifact@v3 + with: + name: coverage-${{ matrix.container }} + path: /home/runner/frappe-bench/sites/coverage.xml + + coverage: + name: Coverage Wrap Up + needs: test + runs-on: ubuntu-latest + steps: + - name: Clone + uses: actions/checkout@v3 + + - name: Download artifacts + uses: actions/download-artifact@v3 + + - name: Upload coverage data uses: codecov/codecov-action@v3 with: name: MariaDB fail_ci_if_error: true - files: /home/runner/frappe-bench/sites/coverage.xml verbose: true - flags: server + flags: server-mariadb diff --git a/.github/workflows/server-postgres-tests.yml b/.github/workflows/server-postgres-tests.yml index 1d81f55802..d8f9a4bf99 100644 --- a/.github/workflows/server-postgres-tests.yml +++ b/.github/workflows/server-postgres-tests.yml @@ -132,10 +132,26 @@ jobs: - name: Upload coverage data if: ${{ steps.check-build.outputs.build == 'strawberry' }} + uses: actions/upload-artifact@v3 + with: + name: coverage-${{ matrix.container }} + path: /home/runner/frappe-bench/sites/coverage.xml + + coverage: + name: Coverage Wrap Up + needs: test + runs-on: ubuntu-latest + steps: + - name: Clone + uses: actions/checkout@v3 + + - name: Download artifacts + uses: actions/download-artifact@v3 + + - name: Upload coverage data uses: codecov/codecov-action@v3 with: name: Postgres fail_ci_if_error: true - files: /home/runner/frappe-bench/sites/coverage.xml verbose: true - flags: server + flags: server-postgres diff --git a/.github/workflows/ui-tests.yml b/.github/workflows/ui-tests.yml index 5bf59f087e..877ad67f6c 100644 --- a/.github/workflows/ui-tests.yml +++ b/.github/workflows/ui-tests.yml @@ -21,7 +21,8 @@ jobs: strategy: fail-fast: false matrix: - containers: [1, 2, 3] + # Make sure you modify coverage submission file list if changing this + container: [1, 2, 3] name: UI Tests (Cypress) @@ -160,26 +161,50 @@ jobs: with: files: "/home/runner/frappe-bench/apps/frappe/.cypress-coverage/clover.xml" - - name: Upload Coverage Data + - name: Upload JS coverage data if: ${{ steps.check-build.outputs.build == 'strawberry' && steps.check_coverage.outputs.files_exists == 'true' }} - uses: codecov/codecov-action@v3 + uses: actions/upload-artifact@v3 with: - name: Cypress - fail_ci_if_error: true - directory: /home/runner/frappe-bench/apps/frappe/.cypress-coverage/ - verbose: true - flags: ui-tests + name: coverage-js-${{ matrix.container }} + path: /home/runner/frappe-bench/apps/frappe/.cypress-coverage/clover.xml - - name: Upload Server Coverage Data - if: ${{ steps.check-build.outputs.build-server == 'strawberry' }} - uses: codecov/codecov-action@v3 + - name: Upload python coverage data + if: ${{ steps.check-build.outputs.build == 'strawberry' }} + uses: actions/upload-artifact@v3 with: - name: MariaDB - fail_ci_if_error: true - files: /home/runner/frappe-bench/sites/coverage.xml - verbose: true - flags: server + name: coverage-py-${{ matrix.container }} + path: /home/runner/frappe-bench/sites/coverage.xml - name: Show bench console if tests failed if: ${{ failure() }} run: cat ~/frappe-bench/bench_start.log + + + coverage: + name: Coverage Wrap Up + needs: test + runs-on: ubuntu-latest + steps: + - name: Clone + uses: actions/checkout@v2 + + - name: Download artifacts + uses: actions/download-artifact@v3 + + - name: Upload python coverage data + uses: codecov/codecov-action@v3 + with: + name: MariaDB + fail_ci_if_error: true + verbose: true + files: ./coverage-py-1/coverage.xml,./coverage-py-2/coverage.xml,./coverage-py-3/coverage.xml + flags: server-ui + + - name: Upload JS coverage data + uses: codecov/codecov-action@v3 + with: + name: Cypress + fail_ci_if_error: true + files: ./coverage-js-1/clover.xml,./coverage-js-2/clover.xml,./coverage-js-3/clover.xml + verbose: true + flags: ui-tests diff --git a/codecov.yml b/codecov.yml index 1326403cfe..bd3aa72c57 100644 --- a/codecov.yml +++ b/codecov.yml @@ -25,11 +25,19 @@ comment: require_changes: true flags: - server: + server-mariadb: paths: - - ".*\\.py" + - "**/*.py" + carryforward: true + server-postgres: + paths: + - "**/*.py" carryforward: true ui-tests: paths: - - ".*\\.js" + - "**/*.js" + carryforward: true + server-ui: + paths: + - "**/*.py" carryforward: true From 936c073d232bd21e5389e9f46af98e22ce9b1f7f Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sat, 13 Aug 2022 20:06:14 +0530 Subject: [PATCH 2/4] Revert "ci: Only upload server-side coverage report if .py files are changed" partially reverts this PR https://github.com/frappe/frappe/pull/16048 This is causing random fluctations. --- .github/helper/roulette.py | 1 - .github/workflows/ui-tests.yml | 11 +++-------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/.github/helper/roulette.py b/.github/helper/roulette.py index 554f4ae5f5..13f843eb3e 100644 --- a/.github/helper/roulette.py +++ b/.github/helper/roulette.py @@ -62,7 +62,6 @@ if __name__ == "__main__": # this is a push build, run all builds if not pr_number: os.system('echo "::set-output name=build::strawberry"') - os.system('echo "::set-output name=build-server::strawberry"') sys.exit(0) files_list = files_list or get_files_list(pr_number=pr_number, repo=repo) diff --git a/.github/workflows/ui-tests.yml b/.github/workflows/ui-tests.yml index 877ad67f6c..2db81bd2cc 100644 --- a/.github/workflows/ui-tests.yml +++ b/.github/workflows/ui-tests.yml @@ -149,20 +149,15 @@ jobs: env: CYPRESS_RECORD_KEY: 4a48f41c-11b3-425b-aa88-c58048fa69eb + # Required to flush python coverage files - name: Stop server - if: ${{ steps.check-build.outputs.build-server == 'strawberry' }} + if: ${{ steps.check-build.outputs.build == 'strawberry' }} run: | ps -ef | grep "frappe serve" | awk '{print $2}' | xargs kill -s SIGINT 2> /dev/null || true sleep 5 - - name: Check If Coverage Report Exists - id: check_coverage - uses: andstor/file-existence-action@v1 - with: - files: "/home/runner/frappe-bench/apps/frappe/.cypress-coverage/clover.xml" - - name: Upload JS coverage data - if: ${{ steps.check-build.outputs.build == 'strawberry' && steps.check_coverage.outputs.files_exists == 'true' }} + if: ${{ steps.check-build.outputs.build == 'strawberry' }} uses: actions/upload-artifact@v3 with: name: coverage-js-${{ matrix.container }} From c5a514055e5f16bfa423ad124eb09ad62685a5d5 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sat, 13 Aug 2022 20:58:58 +0530 Subject: [PATCH 3/4] ci: dont fail silently and wait for coverage --- .github/workflows/ui-tests.yml | 12 ++++++------ frappe/coverage.py | 1 + 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ui-tests.yml b/.github/workflows/ui-tests.yml index 2db81bd2cc..4d5ec5c1db 100644 --- a/.github/workflows/ui-tests.yml +++ b/.github/workflows/ui-tests.yml @@ -149,12 +149,12 @@ jobs: env: CYPRESS_RECORD_KEY: 4a48f41c-11b3-425b-aa88-c58048fa69eb - # Required to flush python coverage files - - name: Stop server + - name: Stop server and wait for coverage file if: ${{ steps.check-build.outputs.build == 'strawberry' }} run: | - ps -ef | grep "frappe serve" | awk '{print $2}' | xargs kill -s SIGINT 2> /dev/null || true + ps -ef | grep "[f]rappe serve" | awk '{print $2}' | xargs kill -s SIGINT sleep 5 + ( tail -f /home/runner/frappe-bench/sites/coverage.xml & ) | grep -q "\/coverage" - name: Upload JS coverage data if: ${{ steps.check-build.outputs.build == 'strawberry' }} @@ -170,9 +170,9 @@ jobs: name: coverage-py-${{ matrix.container }} path: /home/runner/frappe-bench/sites/coverage.xml - - name: Show bench console if tests failed - if: ${{ failure() }} - run: cat ~/frappe-bench/bench_start.log + - name: Show bench output + if: ${{ always() }} + run: cat ~/frappe-bench/bench_start.log || true coverage: diff --git a/frappe/coverage.py b/frappe/coverage.py index 8519b04c55..a1f3e26585 100644 --- a/frappe/coverage.py +++ b/frappe/coverage.py @@ -73,3 +73,4 @@ class CodeCoverage: self.coverage.stop() self.coverage.save() self.coverage.xml_report() + print("Saved Coverage") From fc56c827452cbac4452e31c490be3689eb44b2c1 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sat, 13 Aug 2022 23:49:23 +0530 Subject: [PATCH 4/4] ci: use new tags for coverage checks --- codecov.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/codecov.yml b/codecov.yml index bd3aa72c57..ad60b8a6d9 100644 --- a/codecov.yml +++ b/codecov.yml @@ -5,20 +5,20 @@ coverage: status: project: default: false - server: + server-mariadb: target: auto threshold: 0.5% flags: - - server + - server-mariadb patch: default: false - server: + server-mariadb: target: 85% threshold: 0% only_pulls: true if_ci_failed: ignore flags: - - server + - server-mariadb comment: layout: "diff, flags"