diff --git a/.github/workflows/downstream.yml b/.github/workflows/downstream.yml deleted file mode 100644 index 1d10c1275e..0000000000 --- a/.github/workflows/downstream.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: Downstream - -on: - workflow_dispatch: - -jobs: - dispatch: - runs-on: "ubuntu-latest" - strategy: - matrix: - repo: - - frappe/erpnext - - frappe/lending - - frappe/hrms - steps: - - name: Dispatch Downstream CI (if supported) - uses: peter-evans/repository-dispatch@v3 - with: - token: ${{ secrets.CI_PAT }} - repository: ${{ matrix.repo }} - event-type: frappe-framework-change - client-payload: '{"ref": "${{ github.ref }}", "sha": "${{ github.sha }}"}' diff --git a/.github/workflows/patch-mariadb-tests.yml b/.github/workflows/patch-mariadb-tests.yml deleted file mode 100644 index 1822e61be9..0000000000 --- a/.github/workflows/patch-mariadb-tests.yml +++ /dev/null @@ -1,179 +0,0 @@ -name: Patch (MariaDB) - -on: - pull_request: - workflow_dispatch: - -concurrency: - group: patch-mariadb-develop-${{ github.event_name }}-${{ github.event.number }} - cancel-in-progress: true - -permissions: - # Do not change this as GITHUB_TOKEN is being used by roulette - contents: read - -jobs: - checkrun: - name: Build Check - runs-on: ubuntu-latest - - outputs: - build: ${{ steps.check-build.outputs.build }} - - steps: - - name: Clone - uses: actions/checkout@v4 - - - name: Check if build should be run - id: check-build - run: | - python "${GITHUB_WORKSPACE}/.github/helper/roulette.py" - env: - TYPE: "server" - PR_NUMBER: ${{ github.event.number }} - REPO_NAME: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - test: - name: Patch - runs-on: ubuntu-latest - needs: checkrun - if: ${{ needs.checkrun.outputs.build == 'strawberry' }} - timeout-minutes: 60 - - services: - mariadb: - image: mariadb:11.3 - env: - MARIADB_ROOT_PASSWORD: db_root - ports: - - 3306:3306 - options: --health-cmd="healthcheck.sh --connect --innodb_initialized" --health-interval=5s --health-timeout=2s --health-retries=3 - - steps: - - name: Clone - uses: actions/checkout@v4 - - - name: Check for Merge Conflicts - run: | - if grep -lr --exclude-dir=node_modules "^<<<<<<< " "${GITHUB_WORKSPACE}" - then echo "Found merge conflicts" - exit 1 - fi - - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - - name: Setup Node - uses: actions/setup-node@v4 - with: - node-version: 20 - check-latest: true - - - name: Add to Hosts - run: echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts - - - name: Cache pip - uses: actions/cache@v4 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/*requirements.txt', '**/pyproject.toml', '**/setup.py') }} - restore-keys: | - ${{ runner.os }}-pip- - ${{ runner.os }}- - - - name: Get yarn cache directory path - id: yarn-cache-dir-path - run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT - - - uses: actions/cache@v4 - id: yarn-cache - with: - path: ${{ steps.yarn-cache-dir-path.outputs.dir }} - key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} - restore-keys: | - ${{ runner.os }}-yarn- - - - name: Install Dependencies - run: | - bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh - - - name: Init Bench - run: | - pip install frappe-bench - bash ${GITHUB_WORKSPACE}/.github/helper/install_bench.sh - env: - TYPE: server - - - name: Init Test Site - run: | - bash ${GITHUB_WORKSPACE}/.github/helper/install_site.sh - env: - BEFORE: ${{ env.GITHUB_EVENT_PATH.before }} - AFTER: ${{ env.GITHUB_EVENT_PATH.after }} - TYPE: server - DB: mariadb - - - name: Run Patch Tests - run: | - cd ~/frappe-bench/ - sed -i 's/^worker:/# worker:/g' Procfile - wget https://frappeframework.com/files/v13-frappe.sql.gz - bench --site test_site --force restore ~/frappe-bench/v13-frappe.sql.gz - - source env/bin/activate - cd apps/frappe/ - git remote set-url upstream https://github.com/frappe/frappe.git - - function update_to_version() { - version=$1 - - branch_name="version-$version-hotfix" - echo "Updating to v$version" - git fetch --depth 1 upstream $branch_name:$branch_name - git checkout -q -f $branch_name - - pgrep honcho | xargs kill - sleep 3 - rm -rf ~/frappe-bench/env - bench -v setup env - bench start &>> ~/frappe-bench/bench_start.log & - - bench --site test_site migrate - } - - update_to_version 14 - update_to_version 15 - - echo "Updating to last commit" - pgrep honcho | xargs kill - sleep 3 - rm -rf ~/frappe-bench/env - git checkout -q -f "$GITHUB_SHA" - bench -v setup env - bench start &>> ~/frappe-bench/bench_start.log & - bench --site test_site migrate - bench --site test_site execute frappe.tests.utils.check_orpahned_doctypes - - - name: Show bench output - if: ${{ always() }} - run: | - cd ~/frappe-bench - cat bench_start.log || true - cd logs - for f in ./*.log*; do - echo "Printing log: $f"; - cat $f - done - - faux-test: - name: Patch - runs-on: ubuntu-latest - needs: checkrun - if: ${{ needs.checkrun.outputs.build != 'strawberry' }} - - steps: - - name: Pass skipped tests unconditionally - run: "echo Skipped" diff --git a/.github/workflows/server-tests.yml b/.github/workflows/server-tests.yml index e1c88b0124..20979d1220 100644 --- a/.github/workflows/server-tests.yml +++ b/.github/workflows/server-tests.yml @@ -1,6 +1,8 @@ name: Server on: + repository_dispatch: + types: [frappe-framework-change] pull_request: workflow_dispatch: schedule: @@ -8,7 +10,7 @@ on: - cron: "0 0 * * *" concurrency: - group: server-develop-${{ github.event_name }}-${{ github.event.number }} + group: server-${{ github.event_name }}-${{ github.event.number }} cancel-in-progress: true permissions: @@ -17,89 +19,18 @@ permissions: jobs: typecheck: - name: Type Check - runs-on: ubuntu-latest - steps: - - run: npm install toml - - name: Get pyproject.toml - uses: actions/github-script@v7 - id: get-pyproject - with: - github-token: ${{secrets.GITHUB_TOKEN}} - script: | - const { data: pyprojectContent } = await github.rest.repos.getContent({ - owner: context.repo.owner, - repo: context.repo.repo, - path: 'pyproject.toml', - ref: context.payload.pull_request.head.sha - }); - const content = Buffer.from(pyprojectContent.content, 'base64').toString(); - const toml = require('toml'); - const parsed = toml.parse(content); - const mypyFiles = parsed.tool.mypy.files; - return { mypyFiles, content }; - - - name: Check for changes in mypy files - uses: actions/github-script@v7 - id: check-changes - with: - github-token: ${{secrets.GITHUB_TOKEN}} - script: | - const { mypyFiles } = ${{ steps.get-pyproject.outputs.result }}; - const { data: changedFiles } = await github.rest.pulls.listFiles({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: context.payload.pull_request.number - }); - const changedMypyFiles = changedFiles - .filter(file => mypyFiles.includes(file.filename)) - .map(file => file.filename); - return changedMypyFiles.length > 0; - - - name: Set up Python - if: steps.check-changes.outputs.result == 'true' - uses: actions/setup-python@v5 - with: - python-version: '3.12.6' - - - uses: actions/checkout@v4 - if: steps.check-changes.outputs.result == 'true' - - - name: Cache pip - uses: actions/cache@v4 - if: steps.check-changes.outputs.result == 'true' - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-typecheck-${{ hashFiles('**/pyproject.toml') }} - restore-keys: | - ${{ runner.os }}-pip-typecheck- - ${{ runner.os }}-pip- - ${{ runner.os }}- - - - name: Install dependencies - if: steps.check-changes.outputs.result == 'true' - run: | - python -m pip install --upgrade pip - pip install -e .[dev,test] - - - name: Run mypy - if: steps.check-changes.outputs.result == 'true' - run: | - mypy --version - mypy + name: Types + uses: frappe/frappe/.github/workflows/type-check-base.yml@develop checkrun: name: Plan Tests runs-on: ubuntu-latest needs: typecheck - outputs: build: ${{ steps.check-build.outputs.build }} - steps: - name: Clone uses: actions/checkout@v4 - - name: Check if unit tests should be run id: check-build run: | @@ -109,193 +40,36 @@ jobs: PR_NUMBER: ${{ github.event.number }} REPO_NAME: ${{ github.repository }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - test: - name: Unit Tests - runs-on: ubuntu-latest + name: Tests + uses: frappe/frappe/.github/workflows/tests-base.yml@develop + with: + enable-postgres: true # This will test against both MariaDB and PostgreSQL + parallel-runs: 2 + enable-coverage: ${{ github.event_name != 'pull_request' }} + fake-success: ${{ needs.checkrun.outputs.build != 'strawberry' }} needs: checkrun - if: ${{ needs.checkrun.outputs.build == 'strawberry' }} - timeout-minutes: 30 - env: - NODE_ENV: "production" - PYTHONOPTIMIZE: 2 - # noisy 3rd party library warnings - PYTHONWARNINGS: "module,ignore:::babel.messages.extract" + secrets: inherit - strategy: - fail-fast: false - matrix: - db: ["mariadb", "postgres"] - container: [1, 2] - - services: - mariadb: - image: mariadb:11.3 - env: - MARIADB_ROOT_PASSWORD: db_root - ports: - - 3306:3306 - options: --health-cmd="healthcheck.sh --connect --innodb_initialized" --health-interval=5s --health-timeout=2s --health-retries=3 - - postgres: - image: postgres:12.4 - env: - POSTGRES_PASSWORD: db_root - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 5432:5432 - - smtp_server: - image: rnwood/smtp4dev - ports: - - 2525:25 - - 3000:80 - - steps: - - name: Has pyproject.toml changed? - id: changed-pyproject - uses: tj-actions/changed-files@v45 - with: - files: pyproject.toml - - - name: Clone - uses: actions/checkout@v4 - - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - - name: Check for valid Python & Merge Conflicts - run: | - python -m compileall -q -f "${GITHUB_WORKSPACE}" - if grep -lr --exclude-dir=node_modules "^<<<<<<< " "${GITHUB_WORKSPACE}" - then echo "Found merge conflicts" - exit 1 - fi - - - uses: actions/setup-node@v4 - with: - node-version: 20 - check-latest: true - - - name: Add to Hosts - run: | - echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts - - - name: Cache pip - uses: actions/cache@v4 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/*requirements.txt', '**/pyproject.toml', '**/setup.py') }} - restore-keys: | - ${{ runner.os }}-pip- - ${{ runner.os }}- - - - name: Get yarn cache directory path - id: yarn-cache-dir-path - run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT - - - uses: actions/cache@v4 - id: yarn-cache - with: - path: ${{ steps.yarn-cache-dir-path.outputs.dir }} - key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} - restore-keys: | - ${{ runner.os }}-yarn- - - - name: Install Dependencies - run: | - bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh - - - name: Init Bench - run: | - bash ${GITHUB_WORKSPACE}/.github/helper/install_bench.sh - env: - TYPE: server - BENCH_VERBOSITY_FLAG: ${{ steps.changed-pyproject.outputs.any_modified == 'true' && '-v' || ''}} - - - name: Init Test Site - run: | - bash ${GITHUB_WORKSPACE}/.github/helper/install_site.sh - env: - BEFORE: ${{ env.GITHUB_EVENT_PATH.before }} - AFTER: ${{ env.GITHUB_EVENT_PATH.after }} - FRAPPE_SENTRY_DSN: ${{ secrets.SENTRY_DSN }} - TYPE: server - DB: ${{ matrix.db }} - - - name: Run Tests - run: | - bench --site test_site \ - run-parallel-tests --app frappe \ - --total-builds 2 --build-number ${{ matrix.container }} - working-directory: /home/runner/frappe-bench/sites - env: - SITE: test_site - CI_BUILD_ID: ${{ github.run_id }} - BUILD_NUMBER: ${{ matrix.container }} - WITH_COVERAGE: ${{ github.event_name != 'pull_request' }} - FRAPPE_SENTRY_DSN: ${{ secrets.SENTRY_DSN }} - TOTAL_BUILDS: 2 - COVERAGE_RCFILE: /home/runner/frappe-bench/apps/frappe/.coveragerc - - name: Setup tmate session - - uses: mxschmitt/action-tmate@v3 - if: ${{ failure() && contains( github.event.pull_request.labels.*.name, 'debug-gha') }} - - - name: Show bench output - if: ${{ always() }} - run: | - cd ~/frappe-bench - cat bench_start.log || true - cd logs - for f in ./*.log*; do - echo "Printing log: $f"; - cat $f - done - - - - name: Upload coverage data - uses: actions/upload-artifact@v3 - if: github.event_name != 'pull_request' - with: - name: coverage-${{ matrix.db }}-${{ matrix.container }} - path: /home/runner/frappe-bench/sites/coverage.xml - - # This is required because github still doesn't understand knowingly skipped tests - faux-test: - name: Unit Tests - runs-on: ubuntu-latest + migrate: + name: Migration needs: checkrun - if: ${{ needs.checkrun.outputs.build != 'strawberry' }} - - strategy: - matrix: - db: ["mariadb", "postgres"] - container: [1, 2] - - steps: - - name: Pass skipped tests unconditionally - run: "echo Skipped" + uses: frappe/frappe/.github/workflows/patch-base.yml@develop + with: + python-version: '3.10' + node-version: 20 + fake-success: ${{ needs.checkrun.outputs.build != 'strawberry' }} coverage: name: Coverage Wrap Up needs: [test, checkrun] runs-on: ubuntu-latest - if: ${{ needs.checkrun.outputs.build == 'strawberry' && github.event_name != 'pull_request' }} + if: ${{ github.event_name != 'pull_request' }} steps: - name: Clone uses: actions/checkout@v4 - - name: Download artifacts uses: actions/download-artifact@v4.1.8 - - name: Upload coverage data uses: codecov/codecov-action@v5 with: @@ -304,3 +78,22 @@ jobs: fail_ci_if_error: true verbose: true flags: server + + dispatch: + runs-on: "ubuntu-latest" + needs: [test, migrate] + if: ${{ contains( github.event.pull_request.labels.*.name, 'trigger-downstream-ci') }} + strategy: + matrix: + repo: + - frappe/erpnext + - frappe/lending + - frappe/hrms + steps: + - name: Dispatch Downstream CI (if supported) + uses: peter-evans/repository-dispatch@v3 + with: + token: ${{ secrets.CI_PAT }} + repository: ${{ matrix.repo }} + event-type: frappe-framework-change + client-payload: '{"frappe_sha": "${{ github.sha }}"}' diff --git a/.github/workflows/ui-tests.yml b/.github/workflows/ui-tests.yml index 7ff383618c..c5e6d141ff 100644 --- a/.github/workflows/ui-tests.yml +++ b/.github/workflows/ui-tests.yml @@ -2,6 +2,8 @@ name: UI on: pull_request: + repository_dispatch: + types: [frappe-framework-change] workflow_dispatch: schedule: # Run everday at midnight UTC / 5:30 IST @@ -17,7 +19,7 @@ permissions: jobs: checkrun: - name: Build Check + name: Plan Tests runs-on: ubuntu-latest outputs: @@ -38,182 +40,24 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} test: - runs-on: ubuntu-latest + name: Tests (Cypress) + uses: frappe/frappe/.github/workflows/tests-ui-base.yml@develop + with: + parallel-runs: 3 + enable-coverage: ${{ github.event_name != 'pull_request' }} + fake-success: ${{ needs.checkrun.outputs.build == 'strawberry' }} needs: checkrun - if: ${{ needs.checkrun.outputs.build == 'strawberry' && github.repository_owner == 'frappe' }} - timeout-minutes: 60 - env: - NODE_ENV: "production" - - strategy: - fail-fast: false - matrix: - # Make sure you modify coverage submission file list if changing this - container: [1, 2, 3] - - name: UI Tests (Cypress) - - services: - mariadb: - image: mariadb:11.3 - env: - MARIADB_ROOT_PASSWORD: db_root - ports: - - 3306:3306 - options: --health-cmd="healthcheck.sh --connect --innodb_initialized" --health-interval=5s --health-timeout=2s --health-retries=3 - - steps: - - name: Clone - uses: actions/checkout@v4 - - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - - name: Check for valid Python & Merge Conflicts - run: | - python -m compileall -q -f "${GITHUB_WORKSPACE}" - if grep -lr --exclude-dir=node_modules "^<<<<<<< " "${GITHUB_WORKSPACE}" - then echo "Found merge conflicts" - exit 1 - fi - - - uses: actions/setup-node@v4 - with: - node-version: 20 - check-latest: true - - - name: Add to Hosts - run: | - echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts - - - name: Cache pip - uses: actions/cache@v4 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/*requirements.txt', '**/pyproject.toml', '**/setup.py') }} - restore-keys: | - ${{ runner.os }}-pip- - ${{ runner.os }}- - - - name: Get yarn cache directory path - id: yarn-cache-dir-path - run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT - - - uses: actions/cache@v4 - id: yarn-cache - with: - path: ${{ steps.yarn-cache-dir-path.outputs.dir }} - key: ${{ runner.os }}-yarn-ui-${{ hashFiles('**/yarn.lock') }} - restore-keys: | - ${{ runner.os }}-yarn-ui- - - - name: Cache cypress binary - uses: actions/cache@v4 - with: - path: ~/.cache/Cypress - key: ${{ runner.os }}-cypress - - - name: Install Dependencies - run: | - bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh - - - name: Init Bench - run: | - bash ${GITHUB_WORKSPACE}/.github/helper/install_bench.sh - env: - TYPE: ui - - - name: Init Test Site - run: | - bash ${GITHUB_WORKSPACE}/.github/helper/install_site.sh - env: - BEFORE: ${{ env.GITHUB_EVENT_PATH.before }} - AFTER: ${{ env.GITHUB_EVENT_PATH.after }} - FRAPPE_SENTRY_DSN: ${{ secrets.SENTRY_DSN }} - TYPE: ui - DB: mariadb - - - name: Verify yarn.lock - run: | - cd ~/frappe-bench/apps/frappe - git diff --exit-code yarn.lock - - - name: Instrument Source Code - run: cd ~/frappe-bench/apps/frappe/ && npx nyc instrument -x 'frappe/public/dist/**' -x 'frappe/public/js/lib/**' -x '**/*.bundle.js' --compact=false --in-place frappe - - - name: Build - run: cd ~/frappe-bench/ && bench build --apps frappe - - - name: Site Setup - run: | - cd ~/frappe-bench/ - bench --site test_site execute frappe.utils.install.complete_setup_wizard - bench --site test_site execute frappe.tests.ui_test_helpers.create_test_user - - - name: UI Tests - run: cd ~/frappe-bench/ && bench --site test_site run-ui-tests frappe --with-coverage --headless --parallel --ci-build-id $GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT - env: - CYPRESS_RECORD_KEY: 4a48f41c-11b3-425b-aa88-c58048fa69eb - - - name: Stop server and wait for coverage file - if: github.event_name != 'pull_request' - run: | - ps -ef | grep "[f]rappe serve" | awk '{print $2}' | xargs kill -s SIGINT - sleep 5 - ( tail -f /home/runner/frappe-bench/sites/coverage.xml & ) | grep -q "\/coverage" - - - name: Upload JS coverage data - uses: actions/upload-artifact@v3 - if: github.event_name != 'pull_request' - with: - name: coverage-js-${{ matrix.container }} - path: /home/runner/frappe-bench/apps/frappe/.cypress-coverage/clover.xml - - - name: Upload python coverage data - uses: actions/upload-artifact@v3 - if: github.event_name != 'pull_request' - with: - name: coverage-py-${{ matrix.container }} - path: /home/runner/frappe-bench/sites/coverage.xml - - - name: Show bench output - if: ${{ always() }} - run: | - cd ~/frappe-bench - cat bench_start.log || true - cd logs - for f in ./*.log*; do - echo "Printing log: $f"; - cat $f - done - - faux-test: - runs-on: ubuntu-latest - needs: checkrun - if: ${{ needs.checkrun.outputs.build != 'strawberry' && github.repository_owner == 'frappe' }} - name: UI Tests (Cypress) - strategy: - matrix: - container: [1, 2, 3] - - steps: - - name: Pass skipped tests unconditionally - run: "echo Skipped" coverage: name: Coverage Wrap Up needs: [test, checkrun] - if: ${{ needs.checkrun.outputs.build == 'strawberry' && github.event_name != 'pull_request' }} + if: ${{ github.event_name != 'pull_request' }} runs-on: ubuntu-latest steps: - name: Clone uses: actions/checkout@v4 - - name: Download artifacts uses: actions/download-artifact@v4.1.8 - - name: Upload python coverage data uses: codecov/codecov-action@v5 with: @@ -221,15 +65,14 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} fail_ci_if_error: true verbose: true - files: ./coverage-py-1/coverage.xml,./coverage-py-2/coverage.xml,./coverage-py-3/coverage.xml + exclude: coverage-js* flags: server-ui - - name: Upload JS coverage data uses: codecov/codecov-action@v5 with: name: Cypress token: ${{ secrets.CODECOV_TOKEN }} fail_ci_if_error: true - files: ./coverage-js-1/clover.xml,./coverage-js-2/clover.xml,./coverage-js-3/clover.xml + exclude: coverage-py* verbose: true flags: ui-tests