refactor: ci

- Use bench commands directly
- Drop some redundant checks

Signed-off-by: Akhil Narang <me@akhilnarang.dev>

.github/actions/setup/action.yml

@@ -11,17 +11,11 @@ inputs:
     default: '24'
   build-assets:
     required: false
-    description: 'Wether to build assets'
+    description: 'Whether to build assets'
     default: true
   enable-coverage:
     required: false
     default: false
-  enable-watch:
-    required: false
-    default: false
-  enable-schedule:
-    required: false
-    default: false
   disable-web:
     required: false
     default: false
@@ -47,7 +41,7 @@ runs:
   - name: Clone
     uses: actions/checkout@v6
     with:
-      path: apps/${{ github.event.repository.name }}
+      path: frappe-src
 
   - name: Setup Python
     uses: actions/setup-python@v6
@@ -57,25 +51,19 @@ runs:
   - shell: bash -e {0}
     run: |
       # Check for valid Python & Merge Conflicts
-      python -m compileall -q -f "${GITHUB_WORKSPACE}/apps/${{ github.event.repository.name }}"
+      python -m compileall -q -f "${GITHUB_WORKSPACE}/frappe-src"
-      if grep -lr --exclude-dir=node_modules "^<<<<<<< " "${GITHUB_WORKSPACE}/apps/${{ github.event.repository.name }}"
+      if grep -lr --exclude-dir=node_modules "^<<<<<<< " "${GITHUB_WORKSPACE}/frappe-src"
           then echo "Found merge conflicts"
           exit 1
       fi
 
-  - name: Checkout Frappe
-    uses: actions/checkout@v6
-    with:
-      repository: ${{ env.FRAPPE_GH_ORG || github.repository_owner }}/frappe
-      ref: ${{ github.event.client_payload.frappe_sha || github.base_ref || github.ref_name }}
-      path: apps/frappe
-    if: github.event.repository.name != 'frappe'
-
   - uses: actions/setup-node@v6
     with:
       node-version: ${{ inputs.node-version }}
       check-latest: true
 
+  - uses: astral-sh/setup-uv@v6
+
   - name: Cache pip
     uses: actions/cache@v4
     with:
@@ -118,107 +106,92 @@ runs:
       echo -e "\033[33mInstall System Dependencies: $((end_time - start_time)) seconds\033[0m"
 
   - shell: bash -e {0}
-    env:
-      DB: ${{ inputs.db }}
     run: |
-      # Init Bench & test_site
+      # Init Bench
       start_time=$(date +%s)
-      mkdir ${GITHUB_WORKSPACE}/{sites,config,logs,config/pids,sites/test_site}
+      uv tool install frappe-bench
-      python -m venv ${GITHUB_WORKSPACE}/env
+      bench init ${GITHUB_WORKSPACE} \
-      source ${GITHUB_WORKSPACE}/env/bin/activate
+        --ignore-exist \
-      pip install --quiet --upgrade pip
+        --frappe-path "${GITHUB_WORKSPACE}/frappe-src" \
-      pip cache remove mysqlclient
+        --skip-assets \
+        --no-backups \
+        --python "$(which python)"
 
-      pip install --quiet frappe-bench
+      # bench init sets origin to the local checkout path, fix it to point at GitHub
+      git -C apps/frappe remote set-url upstream "https://github.com/${{ github.repository }}"
 
-      python <<EOF
+      # Trim Procfile for CI
-      from bench.config.common_site_config import setup_config
+      sed -i '/^watch:/d' Procfile
-      from bench.config.redis import generate_config
+      sed -i '/^schedule:/d' Procfile
-      from bench.config.procfile import setup_procfile
+      if [ "${{ inputs.disable-web }}" == "true" ]; then
+        sed -i '/^web:/d' Procfile
+      elif [ "${{ inputs.enable-coverage }}" == "true" ]; then
+        sed -i 's|^web: bench serve|web: bench serve --with-coverage|' Procfile
+      fi
+      if [ "${{ inputs.disable-socketio }}" == "true" ]; then
+        sed -i '/^socketio:/d' Procfile
+      fi
 
-      bench_path = "${{ github.workspace }}"
-      is_true = lambda str: True if str == "true" else False
-      is_not_true = lambda str: True if str != "true" else False
-
-      setup_config(bench_path)
-      generate_config(bench_path)
-      setup_procfile(
-        bench_path,
-        skip_redis=False,
-        skip_web=is_true("${{ inputs.disable-web }}"),
-        skip_watch=is_not_true("${{ inputs.enable-watch }}"),
-        skip_socketio=is_true("${{ inputs.disable-socketio }}"),
-        skip_schedule=is_not_true("${{ inputs.enable-schedule }}"),
-        with_coverage=is_true("${{ inputs.enable-coverage }}"),
-      )
-      EOF
       end_time=$(date +%s)
       echo -e "\033[33mInit Bench: $((end_time - start_time)) seconds\033[0m"
-      cat ${GITHUB_WORKSPACE}/Procfile | awk '{print "\033[0;34m" $0 "\033[0m"}'
-      # Attempt to copy the configuration file
-      if cp "${GITHUB_WORKSPACE}/apps/${{ github.event.repository.name }}/.github/helper/db/$DB.json" ${GITHUB_WORKSPACE}/sites/test_site/site_config.json; then
-          echo "Successfully copied ${DB}.json to site_config.json."
-      else
-          echo "Error: The configuration file ${GITHUB_WORKSPACE}/apps/${{ github.event.repository.name }}/.github/helper/db/$DB.json does not exist."
-          echo "Please ensure that the database JSON file is correctly named and located in the helper/db directory."
-          exit 1  # Exit with a non-zero status to indicate failure
-      fi
-
-      if [ "$DB" == "mariadb" ]; then
-        mariadb --host 127.0.0.1 --port 3306 -u root -p${{ inputs.db-root-password }} -e "SET GLOBAL character_set_server = 'utf8mb4'";
-        mariadb --host 127.0.0.1 --port 3306 -u root -p${{ inputs.db-root-password }} -e "SET GLOBAL collation_server = 'utf8mb4_unicode_ci'";
-
-        mariadb --host 127.0.0.1 --port 3306 -u root -p${{ inputs.db-root-password }} -e "CREATE DATABASE test_frappe";
-        mariadb --host 127.0.0.1 --port 3306 -u root -p${{ inputs.db-root-password }} -e "CREATE USER 'test_frappe'@'localhost' IDENTIFIED BY 'test_frappe'";
-        mariadb --host 127.0.0.1 --port 3306 -u root -p${{ inputs.db-root-password }} -e "GRANT ALL PRIVILEGES ON \`test_frappe\`.* TO 'test_frappe'@'localhost'";
-
-        mariadb --host 127.0.0.1 --port 3306 -u root -p${{ inputs.db-root-password }} -e "FLUSH PRIVILEGES";
-      fi
-
-      if [ "$DB" == "postgres" ]; then
-        export PGPASSWORD='travis'
-        psql -h 127.0.0.1 -p 5432 -c "CREATE DATABASE test_frappe" -U postgres
-        psql -h 127.0.0.1 -p 5432 -c "CREATE USER test_frappe WITH PASSWORD 'test_frappe'" -U postgres
-        psql -h 127.0.0.1 -p 5432 -U postgres -c "GRANT ALL PRIVILEGES ON DATABASE test_frappe TO test_frappe;"
-        unset PGPASSWORD
-      fi
 
   - shell: bash -e {0}
     run: |
-      # Install App(s)
+      # Install dev/test dependencies
-      step_start_time=$(date +%s)
+      start_time=$(date +%s)
-      source ${GITHUB_WORKSPACE}/env/bin/activate
+      bench setup requirements --dev
+      if [ "${{ inputs.build-assets }}" == "true" ]; then
+        bench setup requirements --node
+      fi
 
-      for app in ${GITHUB_WORKSPACE}/apps/*/; do
-        app_name="$(basename $app)"
-        if [ -f "${app}setup.py" ] || [ -f "${app}pyproject.toml" ]; then
-          start_time=$(date +%s)
-          echo -e "\033[36mInstalling python app from ${app}\033[0m"
-          pip install --upgrade -e "${app}[dev,test]"
       end_time=$(date +%s)
-          echo -e "\033[36mTime taken to Install python ${app}: $((end_time - start_time)) seconds\033[0m"
+      echo -e "\033[33mInstall dev/test deps: $((end_time - start_time)) seconds\033[0m"
-        fi
+
-        if  [ "${{ inputs.build-assets }}" == "true" ] && [ -f "${app}package.json" ]; then
+  - shell: bash -e {0}
-          start_time=$(date +%s)
-          echo -e "\033[36mInstalling js app dependencies from ${app}\033[0m"
-          pushd "$app"
-            yarn --check-files
-          popd
-          end_time=$(date +%s)
-          echo -e "\033[36mTime taken to Install js ${app}: $((end_time - start_time)) seconds\033[0m"
-        fi
-        echo "$app_name" >> sites/apps.txt
-        echo -e "\033[32mAdded $app_name to $PWD/sites/apps.txt\033[0m"
-      done
-      step_end_time=$(date +%s)
-      echo -e "\033[33mInstall App(s): $((step_end_time - step_start_time)) seconds\033[0m"
     env:
-      TYPE: server
+      DB: ${{ inputs.db }}
+    run: |
+      # Create Site
+      start_time=$(date +%s)
+      if [ "$DB" == "postgres" ]; then
+        DB_ROOT_USER="postgres"
+        DB_ROOT_PWD="travis"
+      else
+        DB_ROOT_USER="root"
+        DB_ROOT_PWD="${{ inputs.db-root-password }}"
+      fi
+
+      bench set-config -g root_login "$DB_ROOT_USER"
+      bench set-config -g root_password "$DB_ROOT_PWD"
+      bench set-config -g admin_password admin
+
+      bench new-site test_site \
+        --db-type "$DB" \
+        --db-host 127.0.0.1 \
+        --db-name test_frappe \
+        --db-password test_frappe \
+        --verbose
+      bench --site test_site set-config allow_tests 1 --parse
+      bench --site test_site set-config server_script_enabled 1 --parse
+      bench --site test_site set-config host_name "http://test_site:8000"
+      bench --site test_site set-config auto_email_id "test@example.com"
+      bench --site test_site set-config mail_server localhost
+      bench --site test_site set-config mail_port 2525 --parse
+      bench --site test_site set-config mail_login "test@example.com"
+      bench --site test_site set-config mail_password test
+      bench --site test_site set-config disable_mail_smtp_authentication 1 --parse
+
+      if [ "$DB" == "mariadb" ]; then
+        bench --site test_site set-config monitor 1 --parse
+        bench --site test_site set-config use_mysqlclient 1 --parse
+      fi
+
+      end_time=$(date +%s)
+      echo -e "\033[33mCreate Site: $((end_time - start_time)) seconds\033[0m"
 
   - shell: bash -e {0}
     run: |
       # Start Bench
-      source ${GITHUB_WORKSPACE}/env/bin/activate
       bench start &> ${GITHUB_WORKSPACE}/bench_start.log &
 
   - shell: bash -e {0}
@@ -226,12 +199,7 @@ runs:
     run: |
       # Build Assets
       start_time=$(date +%s)
-
+      CI=Yes bench build --force --production
-      source ${GITHUB_WORKSPACE}/env/bin/activate
-      CI=Yes bench build --force --production &
-      build_pid=$!
-      bench --site test_site reinstall --yes
-      wait $build_pid
 
       end_time=$(date +%s)
-      echo -e "\033[33mBuild Assets and reinstall site: $((end_time - start_time)) seconds\033[0m"
+      echo -e "\033[33mBuild Assets: $((end_time - start_time)) seconds\033[0m"

.github/helper/db/mariadb.json

@@ -1,20 +0,0 @@
-{
- "db_host": "127.0.0.1",
- "db_port": 3306,
- "db_name": "test_frappe",
- "db_password": "test_frappe",
- "allow_tests": true,
- "db_type": "mariadb",
- "auto_email_id": "test@example.com",
- "mail_server": "localhost",
- "mail_port": 2525,
- "mail_login": "test@example.com",
- "mail_password": "test",
- "admin_password": "admin",
- "root_login": "root",
- "root_password": "db_root",
- "host_name": "http://test_site:8000",
- "use_mysqlclient": 1,
- "monitor": 1,
- "server_script_enabled": true
-}

.github/helper/db/postgres.json

@@ -1,18 +0,0 @@
-{
-    "db_host": "127.0.0.1",
-    "db_port": 5432,
-    "db_name": "test_frappe",
-    "db_password": "test_frappe",
-    "db_type": "postgres",
-    "allow_tests": true,
-    "auto_email_id": "test@example.com",
-    "mail_server": "localhost",
-    "mail_port": 2525,
-    "mail_login": "test@example.com",
-    "mail_password": "test",
-    "admin_password": "admin",
-    "root_login": "postgres",
-    "root_password": "travis",
-    "host_name": "http://test_site:8000",
-    "server_script_enabled": true
-}

.github/helper/db/sqlite.json

@@ -1,13 +0,0 @@
-{
- "db_name": "test_frappe",
- "db_type": "sqlite",
- "allow_tests": true,
- "auto_email_id": "test@example.com",
- "mail_server": "localhost",
- "mail_port": 2525,
- "mail_login": "test@example.com",
- "mail_password": "test",
- "admin_password": "admin",
- "host_name": "http://test_site:8000",
- "server_script_enabled": true
-}

.github/workflows/_base-migration.yml

@@ -68,92 +68,36 @@ jobs:
       env:
         DB_ARTIFACT_URL: ${{ inputs.db-artifact-url }}
       run: |
-        source ${GITHUB_WORKSPACE}/env/bin/activate
         wget "$DB_ARTIFACT_URL"
         bench --site test_site --force restore ${GITHUB_WORKSPACE}/$(basename "$DB_ARTIFACT_URL")
 
         function update_to_version() {
             version="$1"
+            python_version="${2:-${{ inputs.python-version }}}"
+
             if [ -z "$version" ]; then
-                base_ref="${{ github.base_ref || github.ref_name }}"
+                ref="${{ github.sha }}"
-                head_ref="${{ github.sha }}"
             else
-                base_ref="version-$version-hotfix"
+                ref="version-$version-hotfix"
-                head_ref="version-$version-hotfix"
             fi
 
-            source ${GITHUB_WORKSPACE}/env/bin/activate
+            echo "Updating to $ref"
-            echo "Updating to version ${version:-$head_ref}"
 
-            # Fetch and checkout branches
+            git -C apps/frappe fetch --depth 1 upstream "$ref":"$ref"
-            for app in ${GITHUB_WORKSPACE}/apps/*/; do
+            git -C apps/frappe checkout --quiet --force "$ref"
-                app_name=$(basename "$app")
-                echo "Processing app: $app_name"
 
-                if [[ "$app_name" == "${{ github.event.repository.name }}" ]]; then
-                  git -C "$app" fetch --depth 1 origin $head_ref:$head_ref
-                  if git -C "$app" checkout --quiet --force $head_ref; then
-                      echo "Checked out $head_ref successfully at $app"
-                  else
-                      echo "Failed to checkout $ref at $app" >&2
-                      return 1
-                  fi
-                else
-                  git -C "$app" fetch --depth 1 origin $base_ref:$base_ref
-                    if git -C "$app" checkout --quiet --force $base_ref; then
-                        echo "Checked out $base_ref successfully at $app"
-                    else
-                        echo "Failed to checkout $base_ref at $app" >&2
-                        return 1
-                    fi
-                fi
-            done
-
-            # Resetup env and install apps
             if pgrep honcho > /dev/null; then
-                echo "Stopping honcho process..."
                 pgrep honcho | xargs kill
                 sleep 10
             fi
 
-            echo "Setting up environment..."
+            bench migrate-env "python$python_version"
 
-            # Last python version in the array is the "default", so the 2nd parameter here is optional
+            bench setup requirements
-            if rm -rf ${GITHUB_WORKSPACE}/env && python"$2" -m venv ${GITHUB_WORKSPACE}/env; then
-                source ${GITHUB_WORKSPACE}/env/bin/activate
-                pip install --quiet --upgrade pip
-                pip install --quiet frappe-bench
-                echo "Environment setup completed."
-            else
-                echo "Environment setup failed." >&2
-                return 1
-            fi
-
-            echo "Installing apps..."
-            for app in ${GITHUB_WORKSPACE}/apps/*/; do
-                if pip install --upgrade -e "$app"; then
-                    echo "Installed $app successfully."
-                else
-                    echo "Failed to install $app." >&2
-                    return 1
-                fi
-            done
-
-            echo "Starting bench..."
             bench start &>> ${GITHUB_WORKSPACE}/bench_start.log &
-
+            bench --site test_site migrate
-            echo "Running migrations on test_site..."
-            if bench --site test_site migrate; then
-                echo "Migration completed successfully."
-            else
-                echo "Migration failed." >&2
-                return 1
-            fi
-
-            echo "Update to version ${version:-$base_ref} completed."
         }
 
-        # Save this script into a file for later use.
         declare -f update_to_version > "$RUNNER_TEMP/migrate"
 
     - name: Update to v15

.github/workflows/_base-server-tests.yml

@@ -108,7 +108,6 @@ jobs:
 
       - name: Run Tests
         run: |
-          source ${GITHUB_WORKSPACE}/env/bin/activate
           bench --site test_site \
             run-parallel-tests \
             --app "${{ github.event.repository.name }}" \

.github/workflows/_base-ui-tests.yml

@@ -94,7 +94,6 @@ jobs:
 
       - name: Site Setup
         run: |
-          source ${GITHUB_WORKSPACE}/env/bin/activate
           bench --site test_site execute frappe.utils.install.complete_setup_wizard
           bench --site test_site execute frappe.tests.ui_test_helpers.create_test_user
 
@@ -105,7 +104,6 @@ jobs:
       - name: Run Tests
         id: ui-tests
         run: |
-          source ${GITHUB_WORKSPACE}/env/bin/activate
           bench --site test_site \
             run-ui-tests ${{ github.event.repository.name }} \
               --with-coverage \