ci: Roulette for GHA

.github/helper/install_dependencies.sh

@@ -2,11 +2,6 @@
 
 set -e
 
-# python "${GITHUB_WORKSPACE}/.github/helper/roulette.py"
-# if [[ $? != 2 ]];then
-#   exit;
-# fi
-
  # install wkhtmltopdf
 wget -O /tmp/wkhtmltox.tar.xz https://github.com/frappe/wkhtmltopdf/raw/master/wkhtmltox-0.12.3_linux-generic-amd64.tar.xz
 tar -xf /tmp/wkhtmltox.tar.xz -C /tmp

.github/helper/roulette.py

@@ -1,4 +1,3 @@
-# if the script ends with exit code 0, then no tests are run further, else all tests are run
 import os
 import re
 import shlex
@@ -7,50 +6,50 @@ import sys
 
 
 def get_output(command, shell=True):
-    print(command)
-    command = shlex.split(command)
-    return subprocess.check_output(command, shell=shell, encoding="utf8").strip()
+	print(command)
+	command = shlex.split(command)
+	return subprocess.check_output(command, shell=shell, encoding="utf8").strip()
 
 def is_py(file):
-    return file.endswith("py")
+	return file.endswith("py")
+
+def is_ci(file):
+	return file.endswith("yml") or ".github" in file
 
 def is_js(file):
-    return file.endswith("js")
+	return file.endswith("js")
 
 def is_docs(file):
-    regex = re.compile(r'\.(md|png|jpg|jpeg)$|^.github|LICENSE')
-    return bool(regex.search(file))
+	regex = re.compile(r'\.(md|png|jpg|jpeg)$|^.github|LICENSE')
+	return bool(regex.search(file))
 
 
 if __name__ == "__main__":
-    build_type = os.environ.get("TYPE")
-    before = os.environ.get("BEFORE")
-    after = os.environ.get("AFTER")
-    commit_range = before + '...' + after
-    print("Build Type: {}".format(build_type))
-    print("Commit Range: {}".format(commit_range))
+	files_list = sys.argv[1:]
+	build_type = os.environ.get("TYPE")
 
-    try:
-        files_changed = get_output("git diff --name-only {}".format(commit_range), shell=False)
-    except Exception:
-        sys.exit(2)
+	if not files_list:
+		print("No files' changes detected. Build is shutting")
+		sys.exit(0)
 
-    if "fatal" not in files_changed:
-        files_list = files_changed.split()
-        only_docs_changed = len(list(filter(is_docs, files_list))) == len(files_list)
-        only_js_changed = len(list(filter(is_js, files_list))) == len(files_list)
-        only_py_changed = len(list(filter(is_py, files_list))) == len(files_list)
+	ci_files_changed = any(f for f in files_list if is_ci(f))
+	only_docs_changed = len(list(filter(is_docs, files_list))) == len(files_list)
+	only_js_changed = len(list(filter(is_js, files_list))) == len(files_list)
+	only_py_changed = len(list(filter(is_py, files_list))) == len(files_list)
 
-        if only_docs_changed:
-            print("Only docs were updated, stopping build process.")
-            sys.exit(0)
+	if ci_files_changed:
+		print("CI related files were updated, running all build processes.")
 
-        if only_js_changed and build_type == "server":
-            print("Only JavaScript code was updated; Stopping Python build process.")
-            sys.exit(0)
+	if only_docs_changed:
+		print("Only docs were updated, stopping build process.")
+		sys.exit(0)
 
-        if only_py_changed and build_type == "ui":
-            print("Only Python code was updated, stopping Cypress build process.")
-            sys.exit(0)
+	if only_js_changed and build_type == "server":
+		print("Only JavaScript code was updated; Stopping Python build process.")
+		sys.exit(0)
 
-    sys.exit(2)
+	if only_py_changed and build_type == "ui":
+		print("Only Python code was updated, stopping Cypress build process.")
+		sys.exit(0)
+
+	os.system('echo "::set-output name=build::strawberry"')

.github/workflows/patch-mariadb-tests.yml

@@ -26,10 +26,22 @@ jobs:
         with:
           python-version: 3.7
 
+      - uses: jitterbit/get-changed-files@v1
+        id: files
+
+      - name: Check if build should be run
+        id: check-build
+        run: |
+          python "${GITHUB_WORKSPACE}/.github/helper/roulette.py" ${{ steps.files.outputs.all }}
+        env:
+          TYPE: "server"
+
       - name: Add to Hosts
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         run: echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts
 
       - name: Cache pip
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         uses: actions/cache@v2
         with:
           path: ~/.cache/pip
@@ -39,6 +51,7 @@ jobs:
             ${{ runner.os }}-
 
       - name: Cache node modules
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         uses: actions/cache@v2
         env:
           cache-name: cache-node-modules
@@ -51,10 +64,12 @@ jobs:
             ${{ runner.os }}-
 
       - name: Get yarn cache directory path
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         id: yarn-cache-dir-path
         run: echo "::set-output name=dir::$(yarn cache dir)"
 
       - uses: actions/cache@v2
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         id: yarn-cache
         with:
           path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
@@ -63,6 +78,7 @@ jobs:
             ${{ runner.os }}-yarn-
 
       - name: Install Dependencies
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh
         env:
           BEFORE: ${{ env.GITHUB_EVENT_PATH.before }}
@@ -70,12 +86,14 @@ jobs:
           TYPE: server
 
       - name: Install
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh
         env:
           DB: mariadb
           TYPE: server
 
       - name: Run Patch Tests
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         run: |
           cd ~/frappe-bench/
           wget https://frappeframework.com/files/v10-frappe.sql.gz

.github/workflows/server-mariadb-tests.yml

@@ -35,17 +35,30 @@ jobs:
         with:
           python-version: 3.7
 
+      - uses: jitterbit/get-changed-files@v1
+        id: files
+
+      - name: Check if build should be run
+        id: check-build
+        run: |
+          python "${GITHUB_WORKSPACE}/.github/helper/roulette.py" ${{ steps.files.outputs.all }}
+        env:
+          TYPE: "server"
+
       - uses: actions/setup-node@v2
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         with:
           node-version: 14
           check-latest: true
 
       - name: Add to Hosts
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         run: |
           echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts
           echo "127.0.0.1 test_site_producer" | sudo tee -a /etc/hosts
 
       - name: Cache pip
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         uses: actions/cache@v2
         with:
           path: ~/.cache/pip
@@ -55,6 +68,7 @@ jobs:
             ${{ runner.os }}-
 
       - name: Cache node modules
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         uses: actions/cache@v2
         env:
           cache-name: cache-node-modules
@@ -67,10 +81,12 @@ jobs:
             ${{ runner.os }}-
 
       - name: Get yarn cache directory path
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         id: yarn-cache-dir-path
         run: echo "::set-output name=dir::$(yarn cache dir)"
 
       - uses: actions/cache@v2
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         id: yarn-cache
         with:
           path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
@@ -79,6 +95,7 @@ jobs:
             ${{ runner.os }}-yarn-
 
       - name: Install Dependencies
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh
         env:
           BEFORE: ${{ env.GITHUB_EVENT_PATH.before }}
@@ -86,18 +103,22 @@ jobs:
           TYPE: server
 
       - name: Install
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh
         env:
           DB: mariadb
           TYPE: server
 
       - name: Run Tests
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         run: cd ~/frappe-bench/ && bench --site test_site run-parallel-tests --use-orchestrator --with-coverage
         env:
           CI_BUILD_ID: ${{ github.run_id }}
           ORCHESTRATOR_URL: http://test-orchestrator.frappe.io
 
       - name: Upload Coverage Data
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
+        id: upload-coverage-data
         run: |
           cp ~/frappe-bench/sites/.coverage ${GITHUB_WORKSPACE}
           cd ${GITHUB_WORKSPACE}
@@ -114,6 +135,7 @@ jobs:
   coveralls:
     name: Coverage Wrap Up
     needs: test
+    if: ${{ needs.test.steps.check-build.build == 'strawberry' }}
     container: python:3-slim
     runs-on: ubuntu-18.04
     steps:

.github/workflows/server-postgres-tests.yml

@@ -37,17 +37,30 @@ jobs:
         with:
           python-version: 3.7
 
+      - uses: jitterbit/get-changed-files@v1
+        id: files
+
+      - name: Check if build should be run
+        id: check-build
+        run: |
+          python "${GITHUB_WORKSPACE}/.github/helper/roulette.py" ${{ steps.files.outputs.all }}
+        env:
+          TYPE: "server"
+
       - uses: actions/setup-node@v2
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         with:
           node-version: '14'
           check-latest: true
 
       - name: Add to Hosts
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         run: |
           echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts
           echo "127.0.0.1 test_site_producer" | sudo tee -a /etc/hosts
 
       - name: Cache pip
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         uses: actions/cache@v2
         with:
           path: ~/.cache/pip
@@ -57,6 +70,7 @@ jobs:
             ${{ runner.os }}-
 
       - name: Cache node modules
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         uses: actions/cache@v2
         env:
           cache-name: cache-node-modules
@@ -69,10 +83,12 @@ jobs:
             ${{ runner.os }}-
 
       - name: Get yarn cache directory path
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         id: yarn-cache-dir-path
         run: echo "::set-output name=dir::$(yarn cache dir)"
 
       - uses: actions/cache@v2
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         id: yarn-cache
         with:
           path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
@@ -81,6 +97,7 @@ jobs:
             ${{ runner.os }}-yarn-
 
       - name: Install Dependencies
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh
         env:
           BEFORE: ${{ env.GITHUB_EVENT_PATH.before }}
@@ -88,12 +105,14 @@ jobs:
           TYPE: server
 
       - name: Install
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh
         env:
           DB: postgres
           TYPE: server
 
       - name: Run Tests
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         run: cd ~/frappe-bench/ && bench --site test_site run-parallel-tests --use-orchestrator
         env:
           CI_BUILD_ID: ${{ github.run_id }}

.github/workflows/ui-tests.yml

@@ -35,17 +35,30 @@ jobs:
         with:
           python-version: 3.7
 
+      - uses: jitterbit/get-changed-files@v1
+        id: files
+
+      - name: Check if build should be run
+        id: check-build
+        run: |
+          python "${GITHUB_WORKSPACE}/.github/helper/roulette.py" ${{ steps.files.outputs.all }}
+        env:
+          TYPE: "ui"
+
       - uses: actions/setup-node@v2
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         with:
           node-version: 14
           check-latest: true
 
       - name: Add to Hosts
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         run: |
           echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts
           echo "127.0.0.1 test_site_producer" | sudo tee -a /etc/hosts
 
       - name: Cache pip
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         uses: actions/cache@v2
         with:
           path: ~/.cache/pip
@@ -55,6 +68,7 @@ jobs:
             ${{ runner.os }}-
 
       - name: Cache node modules
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         uses: actions/cache@v2
         env:
           cache-name: cache-node-modules
@@ -67,10 +81,12 @@ jobs:
             ${{ runner.os }}-
 
       - name: Get yarn cache directory path
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         id: yarn-cache-dir-path
         run: echo "::set-output name=dir::$(yarn cache dir)"
 
       - uses: actions/cache@v2
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         id: yarn-cache
         with:
           path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
@@ -79,6 +95,7 @@ jobs:
             ${{ runner.os }}-yarn-
 
       - name: Cache cypress binary
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         uses: actions/cache@v2
         with:
           path: ~/.cache
@@ -88,6 +105,7 @@ jobs:
             ${{ runner.os }}-
 
       - name: Install Dependencies
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh
         env:
           BEFORE: ${{ env.GITHUB_EVENT_PATH.before }}
@@ -95,15 +113,18 @@ jobs:
           TYPE: ui
 
       - name: Install
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh
         env:
           DB: mariadb
           TYPE: ui
 
       - name: Site Setup
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         run: cd ~/frappe-bench/ && bench --site test_site execute frappe.utils.install.complete_setup_wizard
 
       - name: UI Tests
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         run: cd ~/frappe-bench/ && bench --site test_site run-ui-tests frappe --headless --parallel --ci-build-id $GITHUB_RUN_ID
         env:
           CYPRESS_RECORD_KEY: 4a48f41c-11b3-425b-aa88-c58048fa69eb