ci: prepare shared action for concrete downstream use (#28572)

* ci: cleanup setup action

* ci: make migration & setup downstream-ready

* ci: replace obscure bench remove-app with an equivalent rm -rf
This commit is contained in:
David Arnold 2024-11-26 18:11:08 +01:00 committed by GitHub
parent 2b43e13691
commit 894c7d871c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 255 additions and 179 deletions

View file

@ -28,9 +28,6 @@ inputs:
disable-socketio:
required: false
default: false
disable-redis-socketio:
required: false
default: false
db:
required: false
default: mariadb
@ -40,18 +37,17 @@ inputs:
runs:
using: "composite"
steps:
- shell: bash -e {0}
run: |
# Add 'test_site' to /etc/hosts & setup git config
echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts
git config --global init.defaultBranch main
git config --global advice.detachedHead false
- name: Clone
uses: actions/checkout@v4
with:
path: ${{ github.event.repository.name }}
- name: Checkout Frappe
uses: actions/checkout@v4
with:
repository: ${{ env.FRAPPE_GH_ORG || github.repository_owner }}/frappe
ref: ${{ github.event.client_payload.frappe_sha || github.base_ref || github.ref_name }}
path: frappe
if: github.event.repository.name != 'frappe'
path: apps/${{ github.event.repository.name }}
- name: Setup Python
uses: actions/setup-python@v5
@ -67,18 +63,64 @@ runs:
exit 1
fi
- name: Fetch tombl util
shell: bash -e {0}
run: |
# Fetch tombl util
pushd $RUNNER_TEMP
curl -LO https://github.com/snyball/tombl/releases/download/v0.2.3/tombl-v0.2.3.tar.gz
curl -LO https://github.com/snyball/tombl/releases/download/v0.2.3/tombl-v0.2.3.sha256sum
sha256sum -c tombl-v0.2.3.sha256sum
tar -xzf tombl-v0.2.3.tar.gz
chmod +x tombl-v0.2.3/tombl
popd
echo "$RUNNER_TEMP/tombl-v0.2.3" >> $GITHUB_PATH
- name: Checkout Frappe
uses: actions/checkout@v4
with:
repository: ${{ env.FRAPPE_GH_ORG || github.repository_owner }}/frappe
ref: ${{ github.event.client_payload.frappe_sha || github.base_ref || github.ref_name }}
path: apps/frappe
if: github.event.repository.name != 'frappe'
- name: Maybe clone additional apps
shell: bash -e {0}
env:
org: ${{ env.FRAPPE_GH_ORG || github.repository_owner }}
ref: ${{ github.event.client_payload.frappe_sha || github.base_ref || github.ref_name }}
run: |
# Maybe clone additional apps
eval "$(tombl -e CHECKOUTS=tool.frappe-ci.setup.app-checkouts ${{ github.event.repository.name }}/pyproject.toml)" || exit 0
start_time=$(date +%s)
for spec in "${CHECKOUTS[@]}"; do
spec_basename=$(basename "$spec")
if [[ "$spec" == *"/"* ]] || [[ "$spec" == http* ]]; then
remote_url="$spec"
else
remote_url="$org/$spec_basename"
fi
if [[ "$remote_url" != http* ]]; then
remote_url="https://github.com/$remote_url"
fi
mkdir "apps/$spec_basename"
pushd "apps/$spec_basename"
git init
git remote add origin "$remote_url"
git fetch origin "$ref" --depth 1
git checkout FETCH_HEAD --quiet
popd
done
end_time=$(date +%s)
echo -e "\033[33mClone additional apps: $((end_time - start_time)) seconds\033[0m"
- uses: actions/setup-node@v4
with:
node-version: ${{ inputs.node-version }}
check-latest: true
- shell: bash -e {0}
run: |
# Add 'test_site' to /etc/hosts
echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts
git config --global init.defaultBranch main
git config --global advice.detachedHead false
- name: Cache pip
uses: actions/cache@v4
with:
@ -120,68 +162,48 @@ runs:
echo -e "\033[33mInstall System Dependencies: $((end_time - start_time)) seconds\033[0m"
- shell: bash -e {0}
env:
DB: ${{ inputs.db }}
run: |
# Install Bench
# Init Bench & test_site
start_time=$(date +%s)
mkdir ${GITHUB_WORKSPACE}/{sites,config,logs,config/pids,sites/test_site}
python -m venv ${GITHUB_WORKSPACE}/env
source ${GITHUB_WORKSPACE}/env/bin/activate
pip install --quiet --upgrade pip
cd ~ || exit
pip install frappe-bench
# pip install --quiet frappe-bench
# revert after merge: https://github.com/frappe/bench/pull/1600
pip install --quiet git+https://github.com/blaggacao/bench.git@feat/add-direct-config-module-calling
end_time=$(date +%s)
echo -e "\033[33mInstall Bench: $((end_time - start_time)) seconds\033[0m"
- shell: bash -e {0}
run: |
# Init Bench
start_time=$(date +%s)
cd ~ || exit
verbosity="${BENCH_INIT_VERBOSITY_FLAG:-}"
bench $verbosity init frappe-bench --skip-assets --python "$(which python)" --frappe-path "${GITHUB_WORKSPACE}/frappe"
end_time=$(date +%s)
echo -e "\033[33mInit Bench: $((end_time - start_time)) seconds\033[0m"
- shell: bash -e {0}
run: |
# Install App(s)
start_time=$(date +%s)
cd ~/frappe-bench || exit
verbosity="${BENCH_INIT_VERBOSITY_FLAG:-}"
for app in ${GITHUB_WORKSPACE}/*/; do
if [ -f "${app}setup.py" ] || [ -f "${app}pyproject.toml" ]; then
start_time=$(date +%s)
echo "Installing app in ${app}"
pip install --upgrade -e "${app}[dev,test]"
end_time=$(date +%s)
echo "Time taken to Install ${app} requirements: $((end_time - start_time)) seconds"
fi
done
# collect old style tools.bench.dev-dependencies
bench $verbosity setup requirements --dev;
if [ "$TYPE" == "ui" ]
then
bench $verbosity setup requirements --node;
python -m bench.config.common_site_config ${GITHUB_WORKSPACE}
python -m bench.config.redis ${GITHUB_WORKSPACE}
args=()
if [[ "${{ inputs.enable-watch }}" != "true" ]]; then
args+=("--skip-watch")
fi
if [[ "${{ inputs.enable-schedule }}" != "true" ]]; then
args+=("--skip-schedule")
fi
if [[ "${{ inputs.disable-socketio }}" == "true" ]]; then
args+=("--skip-socketio")
fi
if [[ "${{ inputs.disable-web }}" == "true" ]]; then
args+=("--skip-web")
fi
if [[ "${{ inputs.enable-coverage }}" == "true" ]]; then
args+=("--with-coverage")
fi
python -m bench.config.procfile ${GITHUB_WORKSPACE} "${args[@]}"
end_time=$(date +%s)
echo -e "\033[33mInstall App(s): $((end_time - start_time)) seconds\033[0m"
env:
TYPE: server
- shell: bash -e {0}
run: |
# Setup Test Site
start_time=$(date +%s)
cd ~/frappe-bench || exit
mkdir ~/frappe-bench/sites/test_site
echo -e "\033[33mInit Bench: $((end_time - start_time)) seconds\033[0m"
cat ${GITHUB_WORKSPACE}/Procfile | awk '{print "\033[0;34m" $0 "\033[0m"}'
# Attempt to copy the configuration file
if cp "${GITHUB_WORKSPACE}/${{ github.event.repository.name }}/.github/helper/db/$DB.json" ~/frappe-bench/sites/test_site/site_config.json; then
if cp "${GITHUB_WORKSPACE}/apps/${{ github.event.repository.name }}/.github/helper/db/$DB.json" ${GITHUB_WORKSPACE}/sites/test_site/site_config.json; then
echo "Successfully copied ${DB}.json to site_config.json."
else
echo "Error: The configuration file ${GITHUB_WORKSPACE}/${{ github.event.repository.name }}/.github/helper/db/$DB.json does not exist."
@ -203,45 +225,43 @@ runs:
echo "${{ inputs.db-root-password }}" | psql -h 127.0.0.1 -p 5432 -c "CREATE USER test_frappe WITH PASSWORD 'test_frappe'" -U postgres;
fi
end_time=$(date +%s)
echo -e "\033[33mSetup Test Site: $((end_time - start_time)) seconds\033[0m"
- shell: bash -e {0}
run: |
# Install App(s)
step_start_time=$(date +%s)
source ${GITHUB_WORKSPACE}/env/bin/activate
for app in ${GITHUB_WORKSPACE}/apps/*/; do
app_name="$(basename $app)"
if [ -f "${app}setup.py" ] || [ -f "${app}pyproject.toml" ]; then
start_time=$(date +%s)
echo -e "\033[36mInstalling python app from ${app}\033[0m"
pip install --upgrade -e "${app}[dev,test]"
end_time=$(date +%s)
echo -e "\033[36mTime taken to Install python ${app}: $((end_time - start_time)) seconds\033[0m"
fi
if [ "${{ inputs.build-assets }}" == "true" ] && [ -f "${app}package.json" ]; then
start_time=$(date +%s)
echo -e "\033[36mInstalling js app dependencies from ${app}\033[0m"
pushd "$app"
yarn --check-files
popd
end_time=$(date +%s)
echo -e "\033[36mTime taken to Install js ${app}: $((end_time - start_time)) seconds\033[0m"
fi
echo "$app_name" >> sites/apps.txt
echo -e "\033[32mAdded $app_name to $PWD/sites/apps.txt\033[0m"
done
step_end_time=$(date +%s)
echo -e "\033[33mInstall App(s): $((step_end_time - step_start_time)) seconds\033[0m"
env:
DB: ${{ inputs.db }}
- shell: bash -e {0}
run: |
# Modify Procfile
cd ~/frappe-bench || exit
if ${{ inputs.enable-watch != 'true' }}; then
sed -i 's/^watch:/# watch:/g' Procfile
fi
if ${{ inputs.enable-schedule != 'true'}}; then
sed -i 's/^schedule:/# schedule:/g' Procfile
fi
if ${{ inputs.disable-socketio }}; then
sed -i 's/^socketio:/# socketio:/g' Procfile
fi
if ${{ inputs.disable-redis-socketio }}; then
sed -i 's/^redis_socketio:/# redis_socketio:/g' Procfile
fi
if ${{ inputs.enable-coverage }}; then
sed -i 's/^web: bench serve/web: bench serve --with-coverage/g' Procfile
fi
if ${{ inputs.disable-web }}; then
sed -i 's/^web:/# web:/g' Procfile
fi
- shell: bash -e {0}
run: |
# Display modified Procfile
cd ~/frappe-bench || exit
cat Procfile | awk '{print "\033[0;34m" $0 "\033[0m"}'
TYPE: server
- shell: bash -e {0}
run: |
# Start Bench
cd ~/frappe-bench || exit
bench start &> ~/frappe-bench/bench_start.log &
source ${GITHUB_WORKSPACE}/env/bin/activate
bench start &> ${GITHUB_WORKSPACE}/bench_start.log &
- shell: bash -e {0}
if: ${{ inputs.build-assets == 'true' }}
@ -249,7 +269,7 @@ runs:
# Build Assets
start_time=$(date +%s)
cd ~/frappe-bench || exit
source ${GITHUB_WORKSPACE}/env/bin/activate
CI=Yes bench build
end_time=$(date +%s)
@ -260,7 +280,7 @@ runs:
# Reinstall Test Site
start_time=$(date +%s)
cd ~/frappe-bench || exit
source ${GITHUB_WORKSPACE}/env/bin/activate
bench --site test_site reinstall --yes
end_time=$(date +%s)

View file

@ -44,79 +44,135 @@ jobs:
disable-web: true
db-root-password: ${{ env.DB_ROOT_PASSWORD }}
- name: Recover v13 database artifact
- name: Download database artifact
env:
current-base-ref: ${{ github.base_ref || github.ref_name }}
current-head-ref: ${{ github.head_ref || github.ref_name }}
run: |
cd ~/frappe-bench/
wget https://frappeframework.com/files/v13-frappe.sql.gz
bench --site test_site --force restore ~/frappe-bench/v13-frappe.sql.gz
eval "$(tombl -e MIGRATION_DB=tool.frappe-ci.setup.migration-db ${GITHUB_WORKSPACE}/apps/${{ github.event.repository.name }}/pyproject.toml)" || exit 0
source ${GITHUB_WORKSPACE}/env/bin/activate
wget "$MIGRATION_DB"
bench --site test_site --force restore ${GITHUB_WORKSPACE}/$(basename "$MIGRATION_DB")
source env/bin/activate
cd apps/frappe/
git remote set-url upstream https://github.com/frappe/frappe.git
function update_to_version() {
eval "$(tombl -e FRAPPE_DEPENDENCIES=tool.bench.frappe-dependencies ${GITHUB_WORKSPACE}/apps/${{ github.event.repository.name }}/pyproject.toml)" || true
version="$1"
if [ -z "$version" ]; then
base_ref="${{ env.current-base-ref }}"
base_ref="${{ env.current-head-ref }}"
else
base_ref="version-$version-hotfix"
head_ref="version-$version-hotfix"
fi
source ${GITHUB_WORKSPACE}/env/bin/activate
echo "Updating to version ${version:-$base_ref}"
# Fetch and checkout branches
for app in ${GITHUB_WORKSPACE}/apps/*/; do
app_name=$(basename "$app")
echo "Processing app: $app_name"
if [[ ! " ${FRAPPE_DEPENDENCIES[@]} " =~ " $app_name " && "$app_name" != "${{ github.event.repository.name }}" ]]; then
rm -rf $app
echo "Removed $app_name as it's not part of tool.bench.frappe-dependencies"
else
if [[ "$app_name" == "${{ github.event.repository.name }}" ]]; then
git -C "$app" fetch --depth 1 origin $head_ref:$head_ref
if git -C "$app" checkout --quiet --force $head_ref; then
echo "Checked out $head_ref successfully for $app"
else
echo "Failed to checkout $ref for $app" >&2
return 1
fi
else
git -C "$app" fetch --depth 1 origin $base_ref:$base_ref
if git -C "$app" checkout --quiet --force $base_ref; then
echo "Checked out $base_ref successfully for $app"
else
echo "Failed to checkout $base_ref for $app" >&2
return 1
fi
fi
fi
done
# Resetup env and install apps
if pgrep honcho > /dev/null; then
echo "Stopping honcho process..."
pgrep honcho | xargs kill
sleep 3
fi
echo "Setting up environment..."
if rm -rf ${GITHUB_WORKSPACE}/env && python -m venv ${GITHUB_WORKSPACE}/env; then
source ${GITHUB_WORKSPACE}/env/bin/activate
pip install --quiet --upgrade pip
# pip install --quiet frappe-bench
# revert after merge: https://github.com/frappe/bench/pull/1600
pip install --quiet git+https://github.com/blaggacao/bench.git@feat/add-direct-config-module-calling
echo "Environment setup completed."
else
echo "Environment setup failed." >&2
return 1
fi
echo "Installing apps..."
for app in ${GITHUB_WORKSPACE}/apps/*/; do
if pip install --upgrade -e "$app"; then
echo "Installed $app successfully."
else
echo "Failed to install $app." >&2
return 1
fi
done
echo "Starting bench..."
bench start &>> ${GITHUB_WORKSPACE}/bench_start.log &
echo "Running migrations on test_site..."
if bench --site test_site migrate; then
echo "Migration completed successfully."
else
echo "Migration failed." >&2
return 1
fi
echo "Update to version ${version:-$base_ref} completed."
}
# Save this script into a file for later use.
declare -f update_to_version > "$RUNNER_TEMP/migrate"
- name: Update to v14
run: |
cd ~/frappe-bench/apps/frappe/
function update_to_version() {
version=$1
branch_name="version-$version-hotfix"
echo "Updating to v$version"
git fetch --depth 1 upstream $branch_name:$branch_name
git checkout -q -f $branch_name
pgrep honcho | xargs kill
sleep 3
rm -rf ~/frappe-bench/env
bench -v setup env
bench start &>> ~/frappe-bench/bench_start.log &
bench --site test_site migrate
}
source $RUNNER_TEMP/migrate
update_to_version 14
exit $?
- name: Update to v15
run: |
cd ~/frappe-bench/apps/frappe/
function update_to_version() {
version=$1
branch_name="version-$version-hotfix"
echo "Updating to v$version"
git fetch --depth 1 upstream $branch_name:$branch_name
git checkout -q -f $branch_name
pgrep honcho | xargs kill
sleep 3
rm -rf ~/frappe-bench/env
bench -v setup env
bench start &>> ~/frappe-bench/bench_start.log &
bench --site test_site migrate
}
source $RUNNER_TEMP/migrate
update_to_version 15
exit $?
- name: Update to last commit
run: |
cd ~/frappe-bench/apps/frappe/
echo "Updating to last commit"
pgrep honcho | xargs kill
sleep 3
rm -rf ~/frappe-bench/env
git checkout -q -f "$GITHUB_SHA"
bench -v setup env
bench start &>> ~/frappe-bench/bench_start.log &
bench --site test_site migrate
source $RUNNER_TEMP/migrate
update_to_version
exit $?
bench --site test_site execute frappe.tests.utils.check_orpahned_doctypes
- name: Setup tmate session
uses: mxschmitt/action-tmate@v3
if: ${{ failure() && contains( github.event.pull_request.labels.*.name, 'debug-gha') }}
- name: Show bench output
if: ${{ always() }}
run: |
cd ~/frappe-bench
cat bench_start.log || true
cd logs
for f in ./*.log*; do
for f in ${GITHUB_WORKSPACE}/*.log*; do
echo "Printing log: $f";
cat $f
done

View file

@ -59,7 +59,7 @@ jobs:
# noisy 3rd party library warnings
PYTHONWARNINGS: "module,ignore:::babel.messages.extract"
DB_ROOT_PASSWORD: db_root
COVERAGE_RCFILE: ~/frappe-bench/apps/frappe/.coveragerc
COVERAGE_RCFILE: ./apps/frappe/.coveragerc
strategy:
fail-fast: false
@ -97,7 +97,6 @@ jobs:
python-version: ${{ inputs.python-version }}
node-version: ${{ inputs.node-version }}
disable-socketio: true
disable-redis-socketio: true
db-root-password: ${{ env.DB_ROOT_PASSWORD }}
db: ${{ matrix.db }}
env:
@ -105,7 +104,7 @@ jobs:
- name: Run Tests
run: |
cd ~/frappe-bench || exit
source ${GITHUB_WORKSPACE}/env/bin/activate
bench --site test_site \
run-parallel-tests \
--app "${{ github.event.repository.name }}" \
@ -142,7 +141,7 @@ jobs:
if: inputs.enable-coverage
with:
name: coverage-${{ matrix.db }}-${{ matrix.index }}
path: ~/frappe-bench/sites/*-coverage*.xml
path: ./sites/*-coverage*.xml
- name: Setup tmate session
uses: mxschmitt/action-tmate@v3
@ -151,10 +150,9 @@ jobs:
- name: Show bench output
if: ${{ always() }}
run: |
cd ~/frappe-bench || exit
cat bench_start.log || true
cd logs
for f in ./*.log*; do
for f in ${GITHUB_WORKSPACE}/*.log*; do
echo "Printing log: $f";
cat $f
done

View file

@ -48,7 +48,7 @@ jobs:
# noisy 3rd party library warnings
PYTHONWARNINGS: "ignore"
DB_ROOT_PASSWORD: db_root
COVERAGE_RCFILE: ~/frappe-bench/apps/frappe/.coveragerc
COVERAGE_RCFILE: ./apps/frappe/.coveragerc
strategy:
fail-fast: false
@ -76,7 +76,7 @@ jobs:
- name: Verify yarn.lock
run: |
cd ~/frappe-bench/apps/${{ github.event.repository.name }}
cd ${GITHUB_WORKSPACE}/apps/${{ github.event.repository.name }}
git diff --exit-code yarn.lock
- name: Cache cypress binary
@ -87,7 +87,7 @@ jobs:
- name: Instrument Source Code
run: |
cd ~/frappe-bench/apps/${{ github.event.repository.name }}
cd ${GITHUB_WORKSPACE}/apps/${{ github.event.repository.name }}
npx nyc instrument \
-x '${{ github.event.repository.name }}/public/dist/**' \
-x '${{ github.event.repository.name }}/public/js/lib/**' \
@ -95,18 +95,18 @@ jobs:
- name: Build
run: |
cd ~/frappe-bench/
source ${GITHUB_WORKSPACE}/env/bin/activate
bench build --apps ${{ github.event.repository.name }}
- name: Site Setup
run: |
cd ~/frappe-bench/
source ${GITHUB_WORKSPACE}/env/bin/activate
bench --site test_site execute frappe.utils.install.complete_setup_wizard
bench --site test_site execute frappe.tests.ui_test_helpers.create_test_user
- name: Run Tests
run: |
cd ~/frappe-bench/
source ${GITHUB_WORKSPACE}/env/bin/activate
bench --site test_site \
run-ui-tests ${{ github.event.repository.name }} \
--with-coverage \
@ -121,29 +121,28 @@ jobs:
run: |
ps -ef | grep "[f]rappe serve" | awk '{print $2}' | xargs kill -s SIGINT
sleep 5
( tail -f ~/frappe-bench/sites/*-coverage*.xml & ) | grep -q "\/coverage"
( tail -f ${GITHUB_WORKSPACE}/sites/*-coverage*.xml & ) | grep -q "\/coverage"
- name: Upload JS coverage data
uses: actions/upload-artifact@v3
if: inputs.enable-coverage
with:
name: coverage-js-${{ matrix.index }}
path: ~/frappe-bench/apps/${{ github.event.repository.name }}/.cypress-coverage/clover.xml
path: ./apps/${{ github.event.repository.name }}/.cypress-coverage/clover.xml
- name: Upload coverage data
uses: actions/upload-artifact@v3
if: inputs.enable-coverage
with:
name: coverage-py-${{ matrix.index }}
path: ~/frappe-bench/sites/*-coverage*.xml
path: ./sites/*-coverage*.xml
- name: Show bench output
if: ${{ always() }}
run: |
cd ~/frappe-bench || exit
cat bench_start.log || true
cd logs
for f in ./*.log*; do
for f in ${GITHUB_WORKSPACE}/*.log*; do
echo "Printing log: $f";
cat $f
done

View file

@ -149,6 +149,9 @@ responses = "==0.23.1"
freezegun = "~=1.2.2"
pdbpp = "~=0.10.3"
[tool.frappe-ci.setup]
migration-db = "https://frappeframework.com/files/v13-frappe.sql.gz"
[tool.ruff]
line-length = 110
target-version = "py310"