From d23705f60ccf142e40448d3a4f267daccd0b3387 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 25 Jul 2022 12:50:49 +0530 Subject: [PATCH 1/7] ci: Cleanup install.sh --- .github/helper/install.sh | 66 ++++++++++++++++++++++----------------- 1 file changed, 37 insertions(+), 29 deletions(-) diff --git a/.github/helper/install.sh b/.github/helper/install.sh index 21d4a7b972..1a2c62c973 100644 --- a/.github/helper/install.sh +++ b/.github/helper/install.sh @@ -5,55 +5,63 @@ cd ~ || exit echo "Setting Up Bench..." pip install frappe-bench - bench -v init frappe-bench --skip-assets --python "$(which python)" --frappe-path "${GITHUB_WORKSPACE}" +cd ./frappe-bench || exit + +bench -v setup requirements --dev +if [ "$TYPE" == "ui" ]; then + bench -v setup requirements --node; +fi + +echo "Setting Up Sites & Database..." mkdir ~/frappe-bench/sites/test_site cp "${GITHUB_WORKSPACE}/.github/helper/consumer_db/$DB.json" ~/frappe-bench/sites/test_site/site_config.json if [ "$TYPE" == "server" ]; then - mkdir ~/frappe-bench/sites/test_site_producer; - cp "${GITHUB_WORKSPACE}/.github/helper/producer_db/$DB.json" ~/frappe-bench/sites/test_site_producer/site_config.json; + mkdir ~/frappe-bench/sites/test_site_producer; + cp "${GITHUB_WORKSPACE}/.github/helper/producer_db/$DB.json" ~/frappe-bench/sites/test_site_producer/site_config.json; fi - if [ "$DB" == "mariadb" ];then - mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "SET GLOBAL character_set_server = 'utf8mb4'"; - mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "SET GLOBAL collation_server = 'utf8mb4_unicode_ci'"; + mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "SET GLOBAL character_set_server = 'utf8mb4'"; + mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "SET GLOBAL collation_server = 'utf8mb4_unicode_ci'"; - mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "CREATE DATABASE test_frappe_consumer"; - mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "CREATE USER 'test_frappe_consumer'@'localhost' IDENTIFIED BY 'test_frappe_consumer'"; - mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "GRANT ALL PRIVILEGES ON \`test_frappe_consumer\`.* TO 'test_frappe_consumer'@'localhost'"; + mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "CREATE DATABASE test_frappe_consumer"; + mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "CREATE USER 'test_frappe_consumer'@'localhost' IDENTIFIED BY 'test_frappe_consumer'"; + mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "GRANT ALL PRIVILEGES ON \`test_frappe_consumer\`.* TO 'test_frappe_consumer'@'localhost'"; - mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "CREATE DATABASE test_frappe_producer"; - mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "CREATE USER 'test_frappe_producer'@'localhost' IDENTIFIED BY 'test_frappe_producer'"; - mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "GRANT ALL PRIVILEGES ON \`test_frappe_producer\`.* TO 'test_frappe_producer'@'localhost'"; - - mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "FLUSH PRIVILEGES"; - fi + mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "CREATE DATABASE test_frappe_producer"; + mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "CREATE USER 'test_frappe_producer'@'localhost' IDENTIFIED BY 'test_frappe_producer'"; + mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "GRANT ALL PRIVILEGES ON \`test_frappe_producer\`.* TO 'test_frappe_producer'@'localhost'"; + mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "FLUSH PRIVILEGES"; +fi if [ "$DB" == "postgres" ];then - echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE DATABASE test_frappe_consumer" -U postgres; - echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE USER test_frappe_consumer WITH PASSWORD 'test_frappe'" -U postgres; + echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE DATABASE test_frappe_consumer" -U postgres; + echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE USER test_frappe_consumer WITH PASSWORD 'test_frappe'" -U postgres; - echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE DATABASE test_frappe_producer" -U postgres; - echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE USER test_frappe_producer WITH PASSWORD 'test_frappe'" -U postgres; + echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE DATABASE test_frappe_producer" -U postgres; + echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE USER test_frappe_producer WITH PASSWORD 'test_frappe'" -U postgres; fi -cd ./frappe-bench || exit +echo "Setting Up Procfile..." sed -i 's/^watch:/# watch:/g' Procfile sed -i 's/^schedule:/# schedule:/g' Procfile +if [ "$TYPE" == "server" ]; then + sed -i 's/^socketio:/# socketio:/g' Procfile; + sed -i 's/^redis_socketio:/# redis_socketio:/g' Procfile; +fi +if [ "$TYPE" == "ui" ]; then + sed -i 's/^web: bench serve/web: bench serve --with-coverage/g' Procfile; +fi -if [ "$TYPE" == "server" ]; then sed -i 's/^socketio:/# socketio:/g' Procfile; fi -if [ "$TYPE" == "server" ]; then sed -i 's/^redis_socketio:/# redis_socketio:/g' Procfile; fi - -if [ "$TYPE" == "ui" ]; then bench -v setup requirements --node; fi -bench -v setup requirements --dev - -if [ "$TYPE" == "ui" ]; then sed -i 's/^web: bench serve/web: bench serve --with-coverage/g' Procfile; fi +echo "Starting Bench..." bench start &> bench_start.log & bench --site test_site reinstall --yes -if [ "$TYPE" == "server" ]; then bench --site test_site_producer reinstall --yes; fi -if [ "$TYPE" == "server" ]; then CI=Yes bench build --app frappe; fi +if [ "$TYPE" == "server" ]; then + bench --site test_site_producer reinstall --yes; + CI=Yes bench build --app frappe; +fi From 31658e5241b7822bd42a73fdd393c91531f47619 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 25 Jul 2022 13:07:53 +0530 Subject: [PATCH 2/7] ci: Install wkhtmltopdf in the background --- .github/helper/install_dependencies.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/helper/install_dependencies.sh b/.github/helper/install_dependencies.sh index f16bd61a53..694b0a9504 100644 --- a/.github/helper/install_dependencies.sh +++ b/.github/helper/install_dependencies.sh @@ -3,8 +3,11 @@ set -e echo "Setting Up System Dependencies..." -wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.focal_amd64.deb -sudo apt install ./wkhtmltox_0.12.6-1.focal_amd64.deb +install_wkhtmltopdf() { + wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.focal_amd64.deb + sudo apt install ./wkhtmltox_0.12.6-1.focal_amd64.deb +} +install_wkhtmltopdf & curl -LsS -O https://downloads.mariadb.com/MariaDB/mariadb_repo_setup sudo bash mariadb_repo_setup --mariadb-server-version=10.6 From 4289e7c7ea24757b51a966a98d6753af00c1acff Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 25 Jul 2022 14:33:30 +0530 Subject: [PATCH 3/7] ci: Merge PR linter checks * Combine jobs to be triggered at pull_request events: commit-lint, docs-required, linter * PY310-ize helper scripts --- .github/helper/documentation.py | 2 +- .github/helper/translation.py | 15 ++----- .github/workflows/docs-checker.yml | 28 ------------- .github/workflows/linters.yml | 57 ++++++++++++++++++++++---- .github/workflows/semantic-commits.yml | 30 -------------- 5 files changed, 55 insertions(+), 77 deletions(-) delete mode 100644 .github/workflows/docs-checker.yml delete mode 100644 .github/workflows/semantic-commits.yml diff --git a/.github/helper/documentation.py b/.github/helper/documentation.py index aece5f543b..eb0b373acd 100644 --- a/.github/helper/documentation.py +++ b/.github/helper/documentation.py @@ -30,7 +30,7 @@ def docs_link_exists(body): if __name__ == "__main__": pr = sys.argv[1] - response = requests.get("https://api.github.com/repos/frappe/frappe/pulls/{}".format(pr)) + response = requests.get(f"https://api.github.com/repos/frappe/frappe/pulls/{pr}") if response.ok: payload = response.json() diff --git a/.github/helper/translation.py b/.github/helper/translation.py index 9146b3b32b..72f661d3e1 100644 --- a/.github/helper/translation.py +++ b/.github/helper/translation.py @@ -20,19 +20,12 @@ for _file in files_to_scan: if 'frappe-lint: disable-translate' in line: continue - start_matches = start_pattern.search(line) - if start_matches: - starts_with_f = starts_with_f_pattern.search(line) - - if starts_with_f: - has_f_string = f_string_pattern.search(line) - if has_f_string: + if start_matches := start_pattern.search(line): + if starts_with_f := starts_with_f_pattern.search(line): + if has_f_string := f_string_pattern.search(line): errors_encounter += 1 print(f'\nF-strings are not supported for translations at line number {line_number}\n{line.strip()[:100]}') - continue - else: - continue - + continue match = pattern.search(line) error_found = False diff --git a/.github/workflows/docs-checker.yml b/.github/workflows/docs-checker.yml deleted file mode 100644 index e61ee6355a..0000000000 --- a/.github/workflows/docs-checker.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: 'Documentation Check' -on: - pull_request: - types: [ opened, synchronize, reopened, edited ] - -permissions: - contents: read - -jobs: - docs-required: - name: 'Documentation Required' - runs-on: ubuntu-latest - - steps: - - name: 'Setup Environment' - uses: actions/setup-python@v4 - with: - python-version: '3.10' - - - name: 'Clone repo' - uses: actions/checkout@v3 - - - name: Validate Docs - env: - PR_NUMBER: ${{ github.event.number }} - run: | - pip install requests --quiet - python $GITHUB_WORKSPACE/.github/helper/documentation.py $PR_NUMBER diff --git a/.github/workflows/linters.yml b/.github/workflows/linters.yml index 6d1029d51d..e3f71455cd 100644 --- a/.github/workflows/linters.yml +++ b/.github/workflows/linters.yml @@ -1,12 +1,56 @@ name: Linters on: - pull_request: { } + pull_request: + +permissions: + contents: read + +concurrency: + group: commitcheck-frappe-${{ github.event.number }} + cancel-in-progress: true jobs: + commit-lint: + name: 'Semantic Commits' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 200 - linters: - name: Frappe Linter + - uses: actions/setup-node@v3 + with: + node-version: 16 + check-latest: true + + - name: Check commit titles + run: | + npm install @commitlint/cli @commitlint/config-conventional + npx commitlint --verbose --from ${{ github.event.pull_request.base.sha }} --to ${{ github.event.pull_request.head.sha }} + + docs-required: + name: 'Documentation Required' + runs-on: ubuntu-latest + + steps: + - name: 'Setup Environment' + uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: 'Clone repo' + uses: actions/checkout@v3 + + - name: Validate Docs + env: + PR_NUMBER: ${{ github.event.number }} + run: | + pip install requests --quiet + python $GITHUB_WORKSPACE/.github/helper/documentation.py $PR_NUMBER + + linter: + name: 'Frappe Linter' runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -22,8 +66,7 @@ jobs: - name: Download Semgrep rules run: git clone --depth 1 https://github.com/frappe/semgrep-rules.git frappe-semgrep-rules - - name: Download semgrep - run: pip install semgrep==0.97.0 - - name: Run Semgrep rules - run: semgrep ci --config ./frappe-semgrep-rules/rules --config r/python.lang.correctness + run: | + pip install semgrep==0.97.0 + semgrep ci --config ./frappe-semgrep-rules/rules --config r/python.lang.correctness diff --git a/.github/workflows/semantic-commits.yml b/.github/workflows/semantic-commits.yml deleted file mode 100644 index 7afa02d1b9..0000000000 --- a/.github/workflows/semantic-commits.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: Semantic Commits - -on: - pull_request: {} - -permissions: - contents: read - -concurrency: - group: commitcheck-frappe-${{ github.event.number }} - cancel-in-progress: true - -jobs: - commitlint: - name: Check Commit Titles - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 200 - - - uses: actions/setup-node@v3 - with: - node-version: 16 - check-latest: true - - - name: Check commit titles - run: | - npm install @commitlint/cli @commitlint/config-conventional - npx commitlint --verbose --from ${{ github.event.pull_request.base.sha }} --to ${{ github.event.pull_request.head.sha }} From a7ab78bde1439672fe49fe6a544a5baf372a3dcb Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 25 Jul 2022 14:35:16 +0530 Subject: [PATCH 4/7] ci(minor): Actions * Rename CI actions to add database_type * Add workflow_dispatch to assets build action on develop * Rename unit test jobs for better labelling on PR check titles * Rename Patch action for consistency --- .github/workflows/patch-mariadb-tests.yml | 10 +++++----- .github/workflows/publish-assets-develop.yml | 1 + .github/workflows/server-mariadb-tests.yml | 5 ++--- .github/workflows/server-postgres-tests.yml | 5 ++--- 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/.github/workflows/patch-mariadb-tests.yml b/.github/workflows/patch-mariadb-tests.yml index e18cbf53ba..3412fe7503 100644 --- a/.github/workflows/patch-mariadb-tests.yml +++ b/.github/workflows/patch-mariadb-tests.yml @@ -1,7 +1,8 @@ -name: Patch - -on: [pull_request, workflow_dispatch] +name: Server (MariaDB) +on: + pull_request: + workflow_dispatch: concurrency: group: patch-mariadb-develop-${{ github.event.number }} @@ -12,11 +13,10 @@ permissions: jobs: test: + name: Patch runs-on: ubuntu-latest timeout-minutes: 60 - name: Patch Test - services: mariadb: image: mariadb:10.6 diff --git a/.github/workflows/publish-assets-develop.yml b/.github/workflows/publish-assets-develop.yml index 467922e766..12bf9eca55 100644 --- a/.github/workflows/publish-assets-develop.yml +++ b/.github/workflows/publish-assets-develop.yml @@ -1,6 +1,7 @@ name: 'Frappe Assets' on: + workflow_dispatch: push: branches: [ develop ] diff --git a/.github/workflows/server-mariadb-tests.yml b/.github/workflows/server-mariadb-tests.yml index 9e7cffba5d..c8ccfa7862 100644 --- a/.github/workflows/server-mariadb-tests.yml +++ b/.github/workflows/server-mariadb-tests.yml @@ -1,4 +1,4 @@ -name: Server +name: Server (MariaDB) on: pull_request: @@ -16,6 +16,7 @@ permissions: jobs: test: + name: Unit Tests runs-on: ubuntu-latest timeout-minutes: 60 @@ -24,8 +25,6 @@ jobs: matrix: container: [1, 2] - name: Python Unit Tests (MariaDB) - services: mariadb: image: mariadb:10.6 diff --git a/.github/workflows/server-postgres-tests.yml b/.github/workflows/server-postgres-tests.yml index 1741752e6b..9760067197 100644 --- a/.github/workflows/server-postgres-tests.yml +++ b/.github/workflows/server-postgres-tests.yml @@ -1,4 +1,4 @@ -name: Server +name: Server (Postgres) on: pull_request: @@ -15,6 +15,7 @@ permissions: jobs: test: + name: Unit Tests runs-on: ubuntu-latest timeout-minutes: 60 @@ -23,8 +24,6 @@ jobs: matrix: container: [1, 2] - name: Python Unit Tests (Postgres) - services: postgres: image: postgres:12.4 From de97eaf603aea9e25fbed365c5091a30e0719034 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 25 Jul 2022 15:00:16 +0530 Subject: [PATCH 5/7] ci: Merge vulnerable dependency check to linter action --- .github/workflows/deps-checker.yml | 22 -------------------- .github/workflows/linters.yml | 32 ++++++++++++++++++++---------- 2 files changed, 22 insertions(+), 32 deletions(-) delete mode 100644 .github/workflows/deps-checker.yml diff --git a/.github/workflows/deps-checker.yml b/.github/workflows/deps-checker.yml deleted file mode 100644 index d3fa8c80fb..0000000000 --- a/.github/workflows/deps-checker.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: 'Python Dependency Check' -on: - pull_request: - workflow_dispatch: - push: - branches: [ develop ] - -permissions: - contents: read - -jobs: - deps-vulnerable-check: - name: 'Vulnerable Dependency' - runs-on: ubuntu-latest - - steps: - - uses: actions/setup-python@v4 - with: - python-version: '3.10' - - uses: actions/checkout@v3 - - run: pip install pip-audit - - run: pip-audit ${GITHUB_WORKSPACE} diff --git a/.github/workflows/linters.yml b/.github/workflows/linters.yml index e3f71455cd..c0c44ffe56 100644 --- a/.github/workflows/linters.yml +++ b/.github/workflows/linters.yml @@ -2,6 +2,9 @@ name: Linters on: pull_request: + workflow_dispatch: + push: + branches: [ develop ] permissions: contents: read @@ -14,11 +17,12 @@ jobs: commit-lint: name: 'Semantic Commits' runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + steps: - uses: actions/checkout@v3 with: fetch-depth: 200 - - uses: actions/setup-node@v3 with: node-version: 16 @@ -32,15 +36,14 @@ jobs: docs-required: name: 'Documentation Required' runs-on: ubuntu-latest + if: github.event_name == 'pull_request' steps: - name: 'Setup Environment' uses: actions/setup-python@v4 with: python-version: '3.10' - - - name: 'Clone repo' - uses: actions/checkout@v3 + - uses: actions/checkout@v3 - name: Validate Docs env: @@ -54,14 +57,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4 + - uses: actions/setup-python@v4 with: python-version: '3.10' - - - name: Install and Run Pre-commit - uses: pre-commit/action@v3.0.0 + - uses: pre-commit/action@v3.0.0 - name: Download Semgrep rules run: git clone --depth 1 https://github.com/frappe/semgrep-rules.git frappe-semgrep-rules @@ -70,3 +69,16 @@ jobs: run: | pip install semgrep==0.97.0 semgrep ci --config ./frappe-semgrep-rules/rules --config r/python.lang.correctness + + deps-vulnerable-check: + name: 'Vulnerable Dependency Check' + runs-on: ubuntu-latest + + steps: + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + - uses: actions/checkout@v3 + - run: | + pip install pip-audit + pip-audit ${GITHUB_WORKSPACE} From 1e40b32ebf3ddec6aac755e2fc1ef6b096b01349 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 25 Jul 2022 15:01:13 +0530 Subject: [PATCH 6/7] ci: Combine actions to be triggered 'on release' * Trigger static asset building & updating release assets * Trigger Docker release build --- .github/workflows/docker-release.yml | 20 ------------------- ...ish-assets-releases.yml => on_release.yml} | 20 +++++++++++++++++-- 2 files changed, 18 insertions(+), 22 deletions(-) delete mode 100644 .github/workflows/docker-release.yml rename .github/workflows/{publish-assets-releases.yml => on_release.yml} (71%) diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml deleted file mode 100644 index 988c2dcc6c..0000000000 --- a/.github/workflows/docker-release.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: 'Trigger Docker build on release' -on: - release: - types: [released] -permissions: - contents: read - -jobs: - curl: - permissions: - contents: none - name: 'Trigger Docker build on release' - runs-on: ubuntu-latest - container: - image: alpine:latest - steps: - - name: curl - run: | - apk add curl bash - curl -X POST -H "Accept: application/vnd.github.v3+json" -H "Authorization: Bearer ${{ secrets.CI_PAT }}" https://api.github.com/repos/frappe/frappe_docker/actions/workflows/build_stable.yml/dispatches -d '{"ref":"main"}' diff --git a/.github/workflows/publish-assets-releases.yml b/.github/workflows/on_release.yml similarity index 71% rename from .github/workflows/publish-assets-releases.yml rename to .github/workflows/on_release.yml index ff1656e55d..59e14a8c4d 100644 --- a/.github/workflows/publish-assets-releases.yml +++ b/.github/workflows/on_release.yml @@ -1,8 +1,11 @@ -name: 'Frappe Assets' +name: 'Release' on: release: - types: [ created ] + types: [released] + +permissions: + contents: read env: GITHUB_TOKEN: ${{ github.token }} @@ -47,3 +50,16 @@ jobs: asset_path: build/assets.tar.gz asset_name: assets.tar.gz asset_content_type: application/octet-stream + + docker-release: + name: 'Trigger Docker build on release' + runs-on: ubuntu-latest + permissions: + contents: none + container: + image: alpine:latest + steps: + - name: curl + run: | + apk add curl bash + curl -X POST -H "Accept: application/vnd.github.v3+json" -H "Authorization: Bearer ${{ secrets.CI_PAT }}" https://api.github.com/repos/frappe/frappe_docker/actions/workflows/build_stable.yml/dispatches -d '{"ref":"main"}' From 590e983a7f68a1fc131590d0c3f37bb7d129ffa9 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 25 Jul 2022 15:03:06 +0530 Subject: [PATCH 7/7] ci: Rename release action release.yml was a bit ambiguous given we have separate action configs related to release event or intent. --- .github/workflows/{release.yml => create-release.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{release.yml => create-release.yml} (100%) diff --git a/.github/workflows/release.yml b/.github/workflows/create-release.yml similarity index 100% rename from .github/workflows/release.yml rename to .github/workflows/create-release.yml