diff --git a/frappe/build.py b/frappe/build.py index 761541f7a9..7bfffb3c8c 100644 --- a/frappe/build.py +++ b/frappe/build.py @@ -11,24 +11,129 @@ import warnings import tempfile from distutils.spawn import find_executable -from six import iteritems, text_type - import frappe from frappe.utils.minify import JavascriptMinify +import click +from requests import get +from six import iteritems, text_type +from six.moves.urllib.parse import urlparse + timestamps = {} app_paths = None +sites_path = os.path.abspath(os.getcwd()) + + +def download_file(url, prefix): + filename = urlparse(url).path.split("/")[-1] + local_filename = os.path.join(prefix, filename) + with get(url, stream=True, allow_redirects=True) as r: + r.raise_for_status() + with open(local_filename, "wb") as f: + for chunk in r.iter_content(chunk_size=8192): + f.write(chunk) + return local_filename + + +def exists(url): + from requests import head + return head(url, allow_redirects=True) + + +def build_missing_files(): + # check which files dont exist yet from the build.json and tell build.js to build only those! + missing_assets = [] + current_asset_files = [ + "js/{0}".format(x) for x in os.listdir(os.path.join(sites_path, "assets", "js")) + ] + [ + "css/{0}".format(x) for x in os.listdir(os.path.join(sites_path, "assets", "css")) + ] + + with open(os.path.join(sites_path, "assets", "frappe", "build.json")) as f: + all_asset_files = json.load(f).keys() + + for asset in all_asset_files: + if asset.replace("concat:", "") not in current_asset_files: + missing_assets.append(asset) + + if missing_assets: + from subprocess import check_call + from shlex import split + + click.secho("Building Missing Assets...", fg="yellow") + command = split( + "node rollup/build.js --files {0} --no-concat".format(",".join(missing_assets)) + ) + check_call(command, cwd=os.path.join("..", "apps", "frappe")) + + +def download_frappe_assets(): + """Downloads and sets up Frappe assets if they exist based on the current + commit HEAD. + Returns True if correctly setup else returns False. + """ + from subprocess import getoutput + + frappe_head = getoutput("cd ../apps/frappe && git rev-parse HEAD") + exc = False + + if frappe_head: + from tempfile import mkdtemp + + tag = getoutput( + "cd ../apps/frappe && git show-ref --tags -d | grep %s | sed -e 's,.*" + " refs/tags/,,' -e 's/\^{}//'" + % frappe_head + ) + + if tag: + # if tag exists, download assets from github release + url = "https://github.com/frappe/frappe/releases/{0}/assets.tar.gz".format(tag) + else: + url = "http://assets.frappeframework.com/{0}.tar.gz".format(frappe_head) + + try: + click.secho("Retreiving Assets...", fg="yellow") + + if not exists(url): + return False + + prefix = mkdtemp(prefix="frappe-assets-", suffix=frappe_head) + assets_archive = download_file(url, prefix) + + if assets_archive: + import subprocess + + click.secho("Extracting Assets...", fg="yellow") + subprocess.check_output( + ["tar", "xf", assets_archive, "--strip", "3"], cwd=sites_path + ) + build_missing_files() + return True + else: + raise + except Exception: + exc = True + click.secho("No Assets Found...Building...", fg="yellow") + print(frappe.get_traceback()) + finally: + try: + shutil.rmtree(os.path.dirname(assets_archive)) + except Exception: + pass + + return not exc def symlink(target, link_name, overwrite=False): - ''' + """ Create a symbolic link named link_name pointing to target. If link_name exists then FileExistsError is raised, unless overwrite=True. When trying to overwrite a directory, IsADirectoryError is raised. Source: https://stackoverflow.com/a/55742015/10309266 - ''' + """ if not overwrite: return os.symlink(target, link_name) @@ -76,27 +181,28 @@ def setup(): def get_node_pacman(): - pacmans = ['yarn', 'npm'] - for exec_ in pacmans: - exec_ = find_executable(exec_) - if exec_: - return exec_ - raise ValueError('No Node.js Package Manager found.') + exec_ = find_executable("yarn") + if exec_: + return exec_ + raise ValueError("Yarn not found") -def bundle(no_compress, app=None, make_copy=False, restore=False, verbose=False): +def bundle(no_compress, app=None, make_copy=False, restore=False, verbose=False, skip_frappe=False): """concat / minify js files""" setup() make_asset_dirs(make_copy=make_copy, restore=restore) pacman = get_node_pacman() - mode = 'build' if no_compress else 'production' - command = '{pacman} run {mode}'.format(pacman=pacman, mode=mode) + mode = "build" if no_compress else "production" + command = "{pacman} run {mode}".format(pacman=pacman, mode=mode) if app: - command += ' --app {app}'.format(app=app) + command += " --app {app}".format(app=app) - frappe_app_path = os.path.abspath(os.path.join(app_paths[0], '..')) + if skip_frappe: + command += " --skip_frappe" + + frappe_app_path = os.path.abspath(os.path.join(app_paths[0], "..")) check_yarn() frappe.commands.popen(command, cwd=frappe_app_path) diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py index 721376016c..d688d1ab0c 100644 --- a/frappe/commands/utils.py +++ b/frappe/commands/utils.py @@ -19,14 +19,22 @@ from six import StringIO @click.option('--make-copy', is_flag=True, default=False, help='Copy the files instead of symlinking') @click.option('--restore', is_flag=True, default=False, help='Copy the files instead of symlinking with force') @click.option('--verbose', is_flag=True, default=False, help='Verbose') -def build(app=None, make_copy=False, restore = False, verbose=False): +@click.option('--force', is_flag=True, default=False, help='Force build assets instead of downloading available') +def build(app=None, make_copy=False, restore=False, verbose=False, force=False): "Minify + concatenate JS and CSS files, build translations" import frappe.build import frappe frappe.init('') # don't minify in developer_mode for faster builds no_compress = frappe.local.conf.developer_mode or False - frappe.build.bundle(no_compress, app=app, make_copy=make_copy, restore = restore, verbose=verbose) + + if not (force or app): + # skip building frappe if assets exist remotely + skip_frappe = frappe.build.download_frappe_assets() + else: + skip_frappe = False + + frappe.build.bundle(no_compress, app=app, make_copy=make_copy, restore = restore, verbose=verbose, skip_frappe=skip_frappe) @click.command('watch') diff --git a/rollup/build.js b/rollup/build.js index ea1ac54c09..5fdfe80790 100644 --- a/rollup/build.js +++ b/rollup/build.js @@ -15,17 +15,35 @@ const { } = require('./rollup.utils'); const { - get_options_for + get_options_for, + get_options } = require('./config'); -const build_for_app = process.argv[2] === '--app' ? process.argv[3] : null; +const skip_frappe = process.argv.includes("--skip_frappe") + +if (skip_frappe) { + let idx = apps_list.indexOf("frappe"); + if (idx > -1) { + apps_list.splice(idx, 1); + } +} + +const exists = (flag) => process.argv.indexOf(flag) != -1 +const value = (flag) => (process.argv.indexOf(flag) != -1) ? process.argv[process.argv.indexOf(flag) + 1] : null; + +const files = exists("--files") ? value("--files").split(",") : false; +const build_for_app = exists("--app") ? value("--app") : null; +const concat = !exists("--no-concat"); show_production_message(); ensure_js_css_dirs(); -concatenate_files(); +if (concat) concatenate_files(); create_build_file(); -if (build_for_app) { + +if (files) { + build_files(files); +} else if (build_for_app) { build_assets_for_app(build_for_app) .then(() => { run_build_command_for_app(build_for_app); @@ -68,6 +86,28 @@ function build_assets(app) { }); } +function build_files(files, app="frappe") { + for (let file of files) { + let options = get_options(file, app); + if (!options.length) return Promise.resolve(); + log(chalk.yellow(`\nBuilding ${app} assets...\n`)); + + let promises = options.map(({ inputOptions, outputOptions, output_file}) => { + return build(inputOptions, outputOptions) + .then(() => { + log(`${chalk.green('✔')} Built ${output_file}`); + }); + }); + + let start = Date.now(); + return Promise.all(promises) + .then(() => { + let time = Date.now() - start; + log(chalk.green(`✨ Done in ${time / 1000}s`)); + }); + } +} + function build(inputOptions, outputOptions) { return rollup.rollup(inputOptions) .then(bundle => bundle.write(outputOptions)) diff --git a/rollup/config.js b/rollup/config.js index b1816cb4c6..cdeb8eb952 100644 --- a/rollup/config.js +++ b/rollup/config.js @@ -165,6 +165,31 @@ function get_rollup_options_for_css(output_file, input_files) { }; } +function get_options(file, app="frappe") { + const build_json = get_build_json(app); + if (!build_json) return []; + + return Object.keys(build_json) + .map(output_file => { + if (output_file === file) { + if (output_file.startsWith('concat:')) return null; + const input_files = build_json[output_file] + .map(input_file => { + let prefix = get_app_path(app); + if (input_file.startsWith('node_modules/')) { + prefix = path.resolve(get_app_path(app), '..'); + } + return path.resolve(prefix, input_file); + }); + return Object.assign( + get_rollup_options(output_file, input_files), { + output_file + }); + } + }) + .filter(Boolean); +} + function get_options_for(app) { const build_json = get_build_json(app); if (!build_json) return []; @@ -205,5 +230,6 @@ function ignore_css() { }; module.exports = { - get_options_for + get_options_for, + get_options };