diff --git a/.github/workflows/publish-assets-develop.yml b/.github/workflows/publish-assets-develop.yml new file mode 100644 index 0000000000..522cfc815c --- /dev/null +++ b/.github/workflows/publish-assets-develop.yml @@ -0,0 +1,43 @@ +name: Build and Publish Assets for Development + +on: + push: + branches: [ develop ] + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + with: + path: 'frappe' + - uses: actions/setup-node@v1 + with: + python-version: '12.x' + - uses: actions/setup-python@v2 + with: + python-version: '3.6' + - name: Set up bench for current push + run: | + npm install -g yarn + pip3 install -U frappe-bench + bench init frappe-bench --no-procfile --no-backups --skip-assets --skip-redis-config-generation --python $(which python) --frappe-path $GITHUB_WORKSPACE/frappe + cd frappe-bench && bench build + + - name: Package assets + run: | + mkdir -p $GITHUB_WORKSPACE/build + tar -cvpzf $GITHUB_WORKSPACE/build/$GITHUB_SHA.tar.gz ./frappe-bench/sites/assets/js ./frappe-bench/sites/assets/css + + - name: Publish assets to S3 + uses: jakejarvis/s3-sync-action@master + with: + args: --acl public-read + env: + AWS_S3_BUCKET: 'frappe-assets' + AWS_ACCESS_KEY_ID: ${{ secrets.S3_ASSETS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_ASSETS_SECRET_ACCESS_KEY }} + AWS_S3_ENDPOINT: 'http://assets.frappeframework.com' + AWS_REGION: 'fr-par' + SOURCE_DIR: '$GITHUB_WORKSPACE/build' diff --git a/.github/workflows/publish-assets-releases.yml b/.github/workflows/publish-assets-releases.yml new file mode 100644 index 0000000000..5c412ea1b0 --- /dev/null +++ b/.github/workflows/publish-assets-releases.yml @@ -0,0 +1,47 @@ +name: Build and Publish Assets built for Releases + +on: + release: + types: [ created ] + +env: + GITHUB_TOKEN: ${{ github.token }} + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + with: + path: 'frappe' + - uses: actions/setup-node@v1 + with: + python-version: '12.x' + - uses: actions/setup-python@v2 + with: + python-version: '3.6' + - name: Set up bench for current push + run: | + npm install -g yarn + pip3 install -U frappe-bench + bench init frappe-bench --no-procfile --no-backups --skip-assets --skip-redis-config-generation --python $(which python) --frappe-path $GITHUB_WORKSPACE/frappe + cd frappe-bench && bench build + + - name: Package assets + run: | + mkdir -p $GITHUB_WORKSPACE/build + tar -cvpzf $GITHUB_WORKSPACE/build/assets.tar.gz ./frappe-bench/sites/assets/js ./frappe-bench/sites/assets/css + + - name: Get release + id: get_release + uses: bruceadams/get-release@v1.2.0 + + - name: Upload built Assets to Release + uses: actions/upload-release-asset@v1.0.2 + with: + upload_url: ${{ steps.get_release.outputs.upload_url }} + asset_path: build/assets.tar.gz + asset_name: assets.tar.gz + asset_content_type: application/octet-stream + diff --git a/frappe/build.py b/frappe/build.py index 761541f7a9..767217a9b9 100644 --- a/frappe/build.py +++ b/frappe/build.py @@ -11,24 +11,141 @@ import warnings import tempfile from distutils.spawn import find_executable -from six import iteritems, text_type - import frappe from frappe.utils.minify import JavascriptMinify +import click +from requests import get +from six import iteritems, text_type +from six.moves.urllib.parse import urlparse + timestamps = {} app_paths = None +sites_path = os.path.abspath(os.getcwd()) + + +def download_file(url, prefix): + filename = urlparse(url).path.split("/")[-1] + local_filename = os.path.join(prefix, filename) + with get(url, stream=True, allow_redirects=True) as r: + r.raise_for_status() + with open(local_filename, "wb") as f: + for chunk in r.iter_content(chunk_size=8192): + f.write(chunk) + return local_filename + + +def build_missing_files(): + # check which files dont exist yet from the build.json and tell build.js to build only those! + missing_assets = [] + current_asset_files = [] + + for type in ["css", "js"]: + current_asset_files.extend( + [ + "{0}/{1}".format(type, name) + for name in os.listdir(os.path.join(sites_path, "assets", type)) + ] + ) + + with open(os.path.join(sites_path, "assets", "frappe", "build.json")) as f: + all_asset_files = json.load(f).keys() + + for asset in all_asset_files: + if asset.replace("concat:", "") not in current_asset_files: + missing_assets.append(asset) + + if missing_assets: + from subprocess import check_call + from shlex import split + + click.secho("\nBuilding missing assets...\n", fg="yellow") + command = split( + "node rollup/build.js --files {0} --no-concat".format(",".join(missing_assets)) + ) + check_call(command, cwd=os.path.join("..", "apps", "frappe")) + + +def get_assets_link(frappe_head): + from subprocess import getoutput + from requests import head + + tag = getoutput( + "cd ../apps/frappe && git show-ref --tags -d | grep %s | sed -e 's,.*" + " refs/tags/,,' -e 's/\^{}//'" + % frappe_head + ) + + if tag: + # if tag exists, download assets from github release + url = "https://github.com/frappe/frappe/releases/download/{0}/assets.tar.gz".format(tag) + else: + url = "http://assets.frappeframework.com/{0}.tar.gz".format(frappe_head) + + if not head(url): + raise ValueError("URL {0} doesn't exist".format(url)) + + return url + + +def download_frappe_assets(verbose=True): + """Downloads and sets up Frappe assets if they exist based on the current + commit HEAD. + Returns True if correctly setup else returns False. + """ + from simple_chalk import green + from subprocess import getoutput + from tempfile import mkdtemp + + assets_setup = False + frappe_head = getoutput("cd ../apps/frappe && git rev-parse HEAD") + + if frappe_head: + try: + url = get_assets_link(frappe_head) + click.secho("Retreiving assets...", fg="yellow") + prefix = mkdtemp(prefix="frappe-assets-", suffix=frappe_head) + assets_archive = download_file(url, prefix) + print("\n{0} Downloaded Frappe assets from {1}".format(green('✔'), url)) + + if assets_archive: + import tarfile + + click.secho("\nExtracting assets...\n", fg="yellow") + with tarfile.open(assets_archive) as tar: + for file in tar: + if not file.isdir(): + dest = "." + file.name.replace("./frappe-bench/sites", "") + show = dest.replace("./assets/", "") + tar.makefile(file, dest) + print("{0} Restored {1}".format(green('✔'), show)) + + build_missing_files() + return True + else: + raise + except Exception: + # TODO: log traceback in bench.log + click.secho("An Error occurred while downloading assets...", fg="red") + assets_setup = False + finally: + try: + shutil.rmtree(os.path.dirname(assets_archive)) + except Exception: + pass + + return assets_setup def symlink(target, link_name, overwrite=False): - ''' + """ Create a symbolic link named link_name pointing to target. If link_name exists then FileExistsError is raised, unless overwrite=True. When trying to overwrite a directory, IsADirectoryError is raised. Source: https://stackoverflow.com/a/55742015/10309266 - ''' + """ if not overwrite: return os.symlink(target, link_name) @@ -76,27 +193,28 @@ def setup(): def get_node_pacman(): - pacmans = ['yarn', 'npm'] - for exec_ in pacmans: - exec_ = find_executable(exec_) - if exec_: - return exec_ - raise ValueError('No Node.js Package Manager found.') + exec_ = find_executable("yarn") + if exec_: + return exec_ + raise ValueError("Yarn not found") -def bundle(no_compress, app=None, make_copy=False, restore=False, verbose=False): +def bundle(no_compress, app=None, make_copy=False, restore=False, verbose=False, skip_frappe=False): """concat / minify js files""" setup() make_asset_dirs(make_copy=make_copy, restore=restore) pacman = get_node_pacman() - mode = 'build' if no_compress else 'production' - command = '{pacman} run {mode}'.format(pacman=pacman, mode=mode) + mode = "build" if no_compress else "production" + command = "{pacman} run {mode}".format(pacman=pacman, mode=mode) if app: - command += ' --app {app}'.format(app=app) + command += " --app {app}".format(app=app) - frappe_app_path = os.path.abspath(os.path.join(app_paths[0], '..')) + if skip_frappe: + command += " --skip_frappe" + + frappe_app_path = os.path.abspath(os.path.join(app_paths[0], "..")) check_yarn() frappe.commands.popen(command, cwd=frappe_app_path) @@ -107,22 +225,22 @@ def watch(no_compress): pacman = get_node_pacman() - frappe_app_path = os.path.abspath(os.path.join(app_paths[0], '..')) + frappe_app_path = os.path.abspath(os.path.join(app_paths[0], "..")) check_yarn() - frappe_app_path = frappe.get_app_path('frappe', '..') - frappe.commands.popen('{pacman} run watch'.format(pacman=pacman), cwd=frappe_app_path) + frappe_app_path = frappe.get_app_path("frappe", "..") + frappe.commands.popen("{pacman} run watch".format(pacman=pacman), cwd=frappe_app_path) def check_yarn(): - if not find_executable('yarn'): - print('Please install yarn using below command and try again.\nnpm install -g yarn') + if not find_executable("yarn"): + print("Please install yarn using below command and try again.\nnpm install -g yarn") def make_asset_dirs(make_copy=False, restore=False): # don't even think of making assets_path absolute - rm -rf ahead. assets_path = os.path.join(frappe.local.sites_path, "assets") - for dir_path in [os.path.join(assets_path, 'js'), os.path.join(assets_path, 'css')]: + for dir_path in [os.path.join(assets_path, "js"), os.path.join(assets_path, "css")]: if not os.path.exists(dir_path): os.makedirs(dir_path) @@ -131,24 +249,27 @@ def make_asset_dirs(make_copy=False, restore=False): app_base_path = os.path.abspath(os.path.dirname(pymodule.__file__)) symlinks = [] - app_public_path = os.path.join(app_base_path, 'public') + app_public_path = os.path.join(app_base_path, "public") # app/public > assets/app symlinks.append([app_public_path, os.path.join(assets_path, app_name)]) # app/node_modules > assets/app/node_modules if os.path.exists(os.path.abspath(app_public_path)): - symlinks.append([os.path.join(app_base_path, '..', 'node_modules'), os.path.join( - assets_path, app_name, 'node_modules')]) + symlinks.append( + [ + os.path.join(app_base_path, "..", "node_modules"), + os.path.join(assets_path, app_name, "node_modules"), + ] + ) app_doc_path = None - if os.path.isdir(os.path.join(app_base_path, 'docs')): - app_doc_path = os.path.join(app_base_path, 'docs') + if os.path.isdir(os.path.join(app_base_path, "docs")): + app_doc_path = os.path.join(app_base_path, "docs") - elif os.path.isdir(os.path.join(app_base_path, 'www', 'docs')): - app_doc_path = os.path.join(app_base_path, 'www', 'docs') + elif os.path.isdir(os.path.join(app_base_path, "www", "docs")): + app_doc_path = os.path.join(app_base_path, "www", "docs") if app_doc_path: - symlinks.append([app_doc_path, os.path.join( - assets_path, app_name + '_docs')]) + symlinks.append([app_doc_path, os.path.join(assets_path, app_name + "_docs")]) for source, target in symlinks: source = os.path.abspath(source) @@ -162,7 +283,7 @@ def make_asset_dirs(make_copy=False, restore=False): shutil.copytree(source, target) elif make_copy: if os.path.exists(target): - warnings.warn('Target {target} already exists.'.format(target=target)) + warnings.warn("Target {target} already exists.".format(target=target)) else: shutil.copytree(source, target) else: @@ -174,7 +295,7 @@ def make_asset_dirs(make_copy=False, restore=False): try: symlink(source, target, overwrite=True) except OSError: - print('Cannot link {} to {}'.format(source, target)) + print("Cannot link {} to {}".format(source, target)) else: # warnings.warn('Source {source} does not exist.'.format(source = source)) pass @@ -193,7 +314,7 @@ def get_build_maps(): build_maps = {} for app_path in app_paths: - path = os.path.join(app_path, 'public', 'build.json') + path = os.path.join(app_path, "public", "build.json") if os.path.exists(path): with open(path) as f: try: @@ -202,8 +323,7 @@ def get_build_maps(): source_paths = [] for source in sources: if isinstance(source, list): - s = frappe.get_pymodule_path( - source[0], *source[1].split("/")) + s = frappe.get_pymodule_path(source[0], *source[1].split("/")) else: s = os.path.join(app_path, source) source_paths.append(s) @@ -211,36 +331,42 @@ def get_build_maps(): build_maps[target] = source_paths except ValueError as e: print(path) - print('JSON syntax error {0}'.format(str(e))) + print("JSON syntax error {0}".format(str(e))) return build_maps def pack(target, sources, no_compress, verbose): from six import StringIO - outtype, outtxt = target.split(".")[-1], '' + outtype, outtxt = target.split(".")[-1], "" jsm = JavascriptMinify() for f in sources: suffix = None - if ':' in f: - f, suffix = f.split(':') + if ":" in f: + f, suffix = f.split(":") if not os.path.exists(f) or os.path.isdir(f): print("did not find " + f) continue timestamps[f] = os.path.getmtime(f) try: - with open(f, 'r') as sourcefile: - data = text_type(sourcefile.read(), 'utf-8', errors='ignore') + with open(f, "r") as sourcefile: + data = text_type(sourcefile.read(), "utf-8", errors="ignore") extn = f.rsplit(".", 1)[1] - if outtype == "js" and extn == "js" and (not no_compress) and suffix != "concat" and (".min." not in f): - tmpin, tmpout = StringIO(data.encode('utf-8')), StringIO() + if ( + outtype == "js" + and extn == "js" + and (not no_compress) + and suffix != "concat" + and (".min." not in f) + ): + tmpin, tmpout = StringIO(data.encode("utf-8")), StringIO() jsm.minify(tmpin, tmpout) minified = tmpout.getvalue() if minified: - outtxt += text_type(minified or '', 'utf-8').strip('\n') + ';' + outtxt += text_type(minified or "", "utf-8").strip("\n") + ";" if verbose: print("{0}: {1}k".format(f, int(len(minified) / 1024))) @@ -248,27 +374,27 @@ def pack(target, sources, no_compress, verbose): # add to frappe.templates outtxt += html_to_js_template(f, data) else: - outtxt += ('\n/*\n *\t%s\n */' % f) - outtxt += '\n' + data + '\n' + outtxt += "\n/*\n *\t%s\n */" % f + outtxt += "\n" + data + "\n" except Exception: print("--Error in:" + f + "--") print(frappe.get_traceback()) - with open(target, 'w') as f: + with open(target, "w") as f: f.write(outtxt.encode("utf-8")) - print("Wrote %s - %sk" % (target, str(int(os.path.getsize(target)/1024)))) + print("Wrote %s - %sk" % (target, str(int(os.path.getsize(target) / 1024)))) def html_to_js_template(path, content): - '''returns HTML template content as Javascript code, adding it to `frappe.templates`''' + """returns HTML template content as Javascript code, adding it to `frappe.templates`""" return """frappe.templates["{key}"] = '{content}';\n""".format( key=path.rsplit("/", 1)[-1][:-5], content=scrub_html_template(content)) def scrub_html_template(content): - '''Returns HTML content with removed whitespace and comments''' + """Returns HTML content with removed whitespace and comments""" # remove whitespace to a single space content = re.sub("\s+", " ", content) @@ -281,12 +407,12 @@ def scrub_html_template(content): def files_dirty(): for target, sources in iteritems(get_build_maps()): for f in sources: - if ':' in f: - f, suffix = f.split(':') + if ":" in f: + f, suffix = f.split(":") if not os.path.exists(f) or os.path.isdir(f): continue if os.path.getmtime(f) != timestamps.get(f): - print(f + ' dirty') + print(f + " dirty") return True else: return False diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py index acd25eb166..cf99cc914e 100644 --- a/frappe/commands/utils.py +++ b/frappe/commands/utils.py @@ -1,17 +1,17 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals, absolute_import, print_function -import click -import json, os, sys, subprocess +import json +import os +import subprocess +import sys from distutils.spawn import find_executable + +import click + import frappe -from frappe.commands import pass_context, get_site +from frappe.commands import get_site, pass_context from frappe.exceptions import SiteNotSpecifiedError -from frappe.utils import update_progress_bar, get_bench_path -from frappe.utils.response import json_handler -from coverage import Coverage -import cProfile, pstats -from six import StringIO +from frappe.utils import get_bench_path, update_progress_bar @click.command('build') @@ -19,14 +19,22 @@ from six import StringIO @click.option('--make-copy', is_flag=True, default=False, help='Copy the files instead of symlinking') @click.option('--restore', is_flag=True, default=False, help='Copy the files instead of symlinking with force') @click.option('--verbose', is_flag=True, default=False, help='Verbose') -def build(app=None, make_copy=False, restore = False, verbose=False): +@click.option('--force', is_flag=True, default=False, help='Force build assets instead of downloading available') +def build(app=None, make_copy=False, restore=False, verbose=False, force=False): "Minify + concatenate JS and CSS files, build translations" import frappe.build - import frappe frappe.init('') # don't minify in developer_mode for faster builds no_compress = frappe.local.conf.developer_mode or False - frappe.build.bundle(no_compress, app=app, make_copy=make_copy, restore = restore, verbose=verbose) + + # dont try downloading assets if force used, app specified or running via CI + if not (force or app or os.environ.get('CI')): + # skip building frappe if assets exist remotely + skip_frappe = frappe.build.download_frappe_assets(verbose=verbose) + else: + skip_frappe = False + + frappe.build.bundle(no_compress, app=app, make_copy=make_copy, restore=restore, verbose=verbose, skip_frappe=skip_frappe) @click.command('watch') @@ -152,6 +160,7 @@ def execute(context, method, args=None, kwargs=None, profile=False): kwargs = {} if profile: + import cProfile pr = cProfile.Profile() pr.enable() @@ -161,6 +170,9 @@ def execute(context, method, args=None, kwargs=None, profile=False): ret = frappe.safe_eval(method + "(*args, **kwargs)", eval_globals=globals(), eval_locals=locals()) if profile: + import pstats + from six import StringIO + pr.disable() s = StringIO() pstats.Stats(pr, stream=s).sort_stats('cumulative').print_stats(.5) @@ -171,6 +183,7 @@ def execute(context, method, args=None, kwargs=None, profile=False): finally: frappe.destroy() if ret: + from frappe.utils.response import json_handler print(json.dumps(ret, default=json_handler)) if not context.sites: @@ -496,6 +509,8 @@ def run_tests(context, app=None, module=None, doctype=None, test=(), frappe.flags.skip_test_records = skip_test_records if coverage: + from coverage import Coverage + # Generate coverage report only for app that is being tested source_path = os.path.join(get_bench_path(), 'apps', app or 'frappe') cov = Coverage(source=[source_path], omit=[ diff --git a/frappe/utils/change_log.py b/frappe/utils/change_log.py index c75b3289db..29fee2bac0 100644 --- a/frappe/utils/change_log.py +++ b/frappe/utils/change_log.py @@ -9,7 +9,6 @@ import frappe import requests import subprocess # nosec from frappe.utils import cstr -from frappe.utils.gitutils import get_app_branch from frappe import _, safe_decode diff --git a/frappe/utils/gitutils.py b/frappe/utils/gitutils.py deleted file mode 100644 index 10268a6581..0000000000 --- a/frappe/utils/gitutils.py +++ /dev/null @@ -1,24 +0,0 @@ -from __future__ import unicode_literals - -import subprocess - -def get_app_branch(app): - '''Returns branch of an app''' - try: - branch = subprocess.check_output('cd ../apps/{0} && git rev-parse --abbrev-ref HEAD'.format(app), - shell=True) - branch = branch.decode('utf-8') - branch = branch.strip() - return branch - except Exception: - return '' - -def get_app_last_commit_ref(app): - try: - commit_id = subprocess.check_output('cd ../apps/{0} && git rev-parse HEAD'.format(app), - shell=True) - commit_id = commit_id.decode('utf-8') - commit_id = commit_id.strip()[:7] - return commit_id - except Exception: - return '' diff --git a/requirements.txt b/requirements.txt index 67952d54b8..dab8d1214b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -57,6 +57,7 @@ RestrictedPython==5.0 rq>=1.1.0 schedule==0.6.0 semantic-version==2.8.4 +simple-chalk==0.1.0 six==1.14.0 sqlparse==0.2.4 stripe==2.40.0 diff --git a/rollup/build.js b/rollup/build.js index ea1ac54c09..3375b2d1cd 100644 --- a/rollup/build.js +++ b/rollup/build.js @@ -15,17 +15,35 @@ const { } = require('./rollup.utils'); const { - get_options_for + get_options_for, + get_options } = require('./config'); -const build_for_app = process.argv[2] === '--app' ? process.argv[3] : null; +const skip_frappe = process.argv.includes("--skip_frappe") -show_production_message(); +if (skip_frappe) { + let idx = apps_list.indexOf("frappe"); + if (idx > -1) { + apps_list.splice(idx, 1); + } +} + +const exists = (flag) => process.argv.indexOf(flag) != -1 +const value = (flag) => (process.argv.indexOf(flag) != -1) ? process.argv[process.argv.indexOf(flag) + 1] : null; + +const files = exists("--files") ? value("--files").split(",") : false; +const build_for_app = exists("--app") ? value("--app") : null; +const concat = !exists("--no-concat"); + +if (!files) show_production_message(); ensure_js_css_dirs(); -concatenate_files(); +if (concat) concatenate_files(); create_build_file(); -if (build_for_app) { + +if (files) { + build_files(files); +} else if (build_for_app) { build_assets_for_app(build_for_app) .then(() => { run_build_command_for_app(build_for_app); @@ -48,11 +66,7 @@ function build_assets_for_app(app) { return build_assets(app) } -function build_assets(app) { - const options = get_options_for(app); - if (!options.length) return Promise.resolve(); - log(chalk.yellow(`\nBuilding ${app} assets...\n`)); - +function build_from_(options) { const promises = options.map(({ inputOptions, outputOptions, output_file}) => { return build(inputOptions, outputOptions) .then(() => { @@ -68,6 +82,23 @@ function build_assets(app) { }); } +function build_assets(app) { + const options = get_options_for(app); + if (!options.length) return Promise.resolve(); + log(chalk.yellow(`\nBuilding ${app} assets...\n`)); + return build_from_(options); +} + +function build_files(files, app="frappe") { + let ret; + for (let file of files) { + let options = get_options(file, app); + if (!options.length) return Promise.resolve(); + ret += build_from_(options); + } + return ret; +} + function build(inputOptions, outputOptions) { return rollup.rollup(inputOptions) .then(bundle => bundle.write(outputOptions)) diff --git a/rollup/config.js b/rollup/config.js index b1816cb4c6..cdeb8eb952 100644 --- a/rollup/config.js +++ b/rollup/config.js @@ -165,6 +165,31 @@ function get_rollup_options_for_css(output_file, input_files) { }; } +function get_options(file, app="frappe") { + const build_json = get_build_json(app); + if (!build_json) return []; + + return Object.keys(build_json) + .map(output_file => { + if (output_file === file) { + if (output_file.startsWith('concat:')) return null; + const input_files = build_json[output_file] + .map(input_file => { + let prefix = get_app_path(app); + if (input_file.startsWith('node_modules/')) { + prefix = path.resolve(get_app_path(app), '..'); + } + return path.resolve(prefix, input_file); + }); + return Object.assign( + get_rollup_options(output_file, input_files), { + output_file + }); + } + }) + .filter(Boolean); +} + function get_options_for(app) { const build_json = get_build_json(app); if (!build_json) return []; @@ -205,5 +230,6 @@ function ignore_css() { }; module.exports = { - get_options_for + get_options_for, + get_options };