vassili/seitime-frappe
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Merge pull request #11436 from gavindsouza/build-assets

Browse source 
feat: Ship built assets
This commit is contained in:
gavin 2020-09-15 13:01:38 +05:30 • committed by GitHub
commit f710ad095f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 365 additions and 101 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

43
.github/workflows/publish-assets-develop.yml vendored Normal file
View file

@ -0,0 +1,43 @@
name: Build and Publish Assets for Development
on:
push:
branches: [ develop ]
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
with:
path: 'frappe'
- uses: actions/setup-node@v1
with:
python-version: '12.x'
- uses: actions/setup-python@v2
with:
python-version: '3.6'
- name: Set up bench for current push
run: |
npm install -g yarn
pip3 install -U frappe-bench
bench init frappe-bench --no-procfile --no-backups --skip-assets --skip-redis-config-generation --python $(which python) --frappe-path $GITHUB_WORKSPACE/frappe
cd frappe-bench && bench build
- name: Package assets
run: |
mkdir -p $GITHUB_WORKSPACE/build
tar -cvpzf $GITHUB_WORKSPACE/build/$GITHUB_SHA.tar.gz ./frappe-bench/sites/assets/js ./frappe-bench/sites/assets/css
- name: Publish assets to S3
uses: jakejarvis/s3-sync-action@master
with:
args: --acl public-read
env:
AWS_S3_BUCKET: 'frappe-assets'
AWS_ACCESS_KEY_ID: ${{ secrets.S3_ASSETS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_ASSETS_SECRET_ACCESS_KEY }}
AWS_S3_ENDPOINT: 'http://assets.frappeframework.com'
AWS_REGION: 'fr-par'
SOURCE_DIR: '$GITHUB_WORKSPACE/build'

47
.github/workflows/publish-assets-releases.yml vendored Normal file
View file

@ -0,0 +1,47 @@
name: Build and Publish Assets built for Releases
on:
release:
types: [ created ]
env:
GITHUB_TOKEN: ${{ github.token }}
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
with:
path: 'frappe'
- uses: actions/setup-node@v1
with:
python-version: '12.x'
- uses: actions/setup-python@v2
with:
python-version: '3.6'
- name: Set up bench for current push
run: |
npm install -g yarn
pip3 install -U frappe-bench
bench init frappe-bench --no-procfile --no-backups --skip-assets --skip-redis-config-generation --python $(which python) --frappe-path $GITHUB_WORKSPACE/frappe
cd frappe-bench && bench build
- name: Package assets
run: |
mkdir -p $GITHUB_WORKSPACE/build
tar -cvpzf $GITHUB_WORKSPACE/build/assets.tar.gz ./frappe-bench/sites/assets/js ./frappe-bench/sites/assets/css
- name: Get release
id: get_release
uses: bruceadams/get-release@v1.2.0
- name: Upload built Assets to Release
uses: actions/upload-release-asset@v1.0.2
with:
upload_url: ${{ steps.get_release.outputs.upload_url }}
asset_path: build/assets.tar.gz
asset_name: assets.tar.gz
asset_content_type: application/octet-stream

232
frappe/build.py
View file

@ -11,24 +11,141 @@ import warnings
import tempfile
from distutils.spawn import find_executable
from six import iteritems, text_type
import frappe
from frappe.utils.minify import JavascriptMinify
import click
from requests import get
from six import iteritems, text_type
from six.moves.urllib.parse import urlparse
timestamps = {}
app_paths = None
sites_path = os.path.abspath(os.getcwd())
def download_file(url, prefix):
filename = urlparse(url).path.split("/")[-1]
local_filename = os.path.join(prefix, filename)
with get(url, stream=True, allow_redirects=True) as r:
r.raise_for_status()
with open(local_filename, "wb") as f:
for chunk in r.iter_content(chunk_size=8192):
f.write(chunk)
return local_filename
def build_missing_files():
# check which files dont exist yet from the build.json and tell build.js to build only those!
missing_assets = []
current_asset_files = []
for type in ["css", "js"]:
current_asset_files.extend(
[
"{0}/{1}".format(type, name)
for name in os.listdir(os.path.join(sites_path, "assets", type))
]
)
with open(os.path.join(sites_path, "assets", "frappe", "build.json")) as f:
all_asset_files = json.load(f).keys()
for asset in all_asset_files:
if asset.replace("concat:", "") not in current_asset_files:
missing_assets.append(asset)
if missing_assets:
from subprocess import check_call
from shlex import split
click.secho("\nBuilding missing assets...\n", fg="yellow")
command = split(
"node rollup/build.js --files {0} --no-concat".format(",".join(missing_assets))
)
check_call(command, cwd=os.path.join("..", "apps", "frappe"))
def get_assets_link(frappe_head):
from subprocess import getoutput
from requests import head
tag = getoutput(
"cd ../apps/frappe && git show-ref --tags -d | grep %s | sed -e 's,.*"
" refs/tags/,,' -e 's/\^{}//'"
% frappe_head
)
if tag:
# if tag exists, download assets from github release
url = "https://github.com/frappe/frappe/releases/download/{0}/assets.tar.gz".format(tag)
else:
url = "http://assets.frappeframework.com/{0}.tar.gz".format(frappe_head)
if not head(url):
raise ValueError("URL {0} doesn't exist".format(url))
return url
def download_frappe_assets(verbose=True):
"""Downloads and sets up Frappe assets if they exist based on the current
commit HEAD.
Returns True if correctly setup else returns False.
"""
from simple_chalk import green
from subprocess import getoutput
from tempfile import mkdtemp
assets_setup = False
frappe_head = getoutput("cd ../apps/frappe && git rev-parse HEAD")
if frappe_head:
try:
url = get_assets_link(frappe_head)
click.secho("Retreiving assets...", fg="yellow")
prefix = mkdtemp(prefix="frappe-assets-", suffix=frappe_head)
assets_archive = download_file(url, prefix)
print("\n{0} Downloaded Frappe assets from {1}".format(green('✔'), url))
if assets_archive:
import tarfile
click.secho("\nExtracting assets...\n", fg="yellow")
with tarfile.open(assets_archive) as tar:
for file in tar:
if not file.isdir():
dest = "." + file.name.replace("./frappe-bench/sites", "")
show = dest.replace("./assets/", "")
tar.makefile(file, dest)
print("{0} Restored {1}".format(green('✔'), show))
build_missing_files()
return True
else:
raise
except Exception:
# TODO: log traceback in bench.log
click.secho("An Error occurred while downloading assets...", fg="red")
assets_setup = False
finally:
try:
shutil.rmtree(os.path.dirname(assets_archive))
except Exception:
pass
return assets_setup
def symlink(target, link_name, overwrite=False):
'''
"""
Create a symbolic link named link_name pointing to target.
If link_name exists then FileExistsError is raised, unless overwrite=True.
When trying to overwrite a directory, IsADirectoryError is raised.
Source: https://stackoverflow.com/a/55742015/10309266
'''
"""
if not overwrite:
return os.symlink(target, link_name)
@ -76,27 +193,28 @@ def setup():
def get_node_pacman():
pacmans = ['yarn', 'npm']
for exec_ in pacmans:
exec_ = find_executable(exec_)
if exec_:
return exec_
raise ValueError('No Node.js Package Manager found.')
exec_ = find_executable("yarn")
if exec_:
return exec_
raise ValueError("Yarn not found")
def bundle(no_compress, app=None, make_copy=False, restore=False, verbose=False):
def bundle(no_compress, app=None, make_copy=False, restore=False, verbose=False, skip_frappe=False):
"""concat / minify js files"""
setup()
make_asset_dirs(make_copy=make_copy, restore=restore)
pacman = get_node_pacman()
mode = 'build' if no_compress else 'production'
command = '{pacman} run {mode}'.format(pacman=pacman, mode=mode)
mode = "build" if no_compress else "production"
command = "{pacman} run {mode}".format(pacman=pacman, mode=mode)
if app:
command += ' --app {app}'.format(app=app)
command += " --app {app}".format(app=app)
frappe_app_path = os.path.abspath(os.path.join(app_paths[0], '..'))
if skip_frappe:
command += " --skip_frappe"
frappe_app_path = os.path.abspath(os.path.join(app_paths[0], ".."))
check_yarn()
frappe.commands.popen(command, cwd=frappe_app_path)
@ -107,22 +225,22 @@ def watch(no_compress):
pacman = get_node_pacman()
frappe_app_path = os.path.abspath(os.path.join(app_paths[0], '..'))
frappe_app_path = os.path.abspath(os.path.join(app_paths[0], ".."))
check_yarn()
frappe_app_path = frappe.get_app_path('frappe', '..')
frappe.commands.popen('{pacman} run watch'.format(pacman=pacman), cwd=frappe_app_path)
frappe_app_path = frappe.get_app_path("frappe", "..")
frappe.commands.popen("{pacman} run watch".format(pacman=pacman), cwd=frappe_app_path)
def check_yarn():
if not find_executable('yarn'):
print('Please install yarn using below command and try again.\nnpm install -g yarn')
if not find_executable("yarn"):
print("Please install yarn using below command and try again.\nnpm install -g yarn")
def make_asset_dirs(make_copy=False, restore=False):
# don't even think of making assets_path absolute - rm -rf ahead.
assets_path = os.path.join(frappe.local.sites_path, "assets")
for dir_path in [os.path.join(assets_path, 'js'), os.path.join(assets_path, 'css')]:
for dir_path in [os.path.join(assets_path, "js"), os.path.join(assets_path, "css")]:
if not os.path.exists(dir_path):
os.makedirs(dir_path)
@ -131,24 +249,27 @@ def make_asset_dirs(make_copy=False, restore=False):
app_base_path = os.path.abspath(os.path.dirname(pymodule.__file__))
symlinks = []
app_public_path = os.path.join(app_base_path, 'public')
app_public_path = os.path.join(app_base_path, "public")
# app/public > assets/app
symlinks.append([app_public_path, os.path.join(assets_path, app_name)])
# app/node_modules > assets/app/node_modules
if os.path.exists(os.path.abspath(app_public_path)):
symlinks.append([os.path.join(app_base_path, '..', 'node_modules'), os.path.join(
assets_path, app_name, 'node_modules')])
symlinks.append(
[
os.path.join(app_base_path, "..", "node_modules"),
os.path.join(assets_path, app_name, "node_modules"),
]
)
app_doc_path = None
if os.path.isdir(os.path.join(app_base_path, 'docs')):
app_doc_path = os.path.join(app_base_path, 'docs')
if os.path.isdir(os.path.join(app_base_path, "docs")):
app_doc_path = os.path.join(app_base_path, "docs")
elif os.path.isdir(os.path.join(app_base_path, 'www', 'docs')):
app_doc_path = os.path.join(app_base_path, 'www', 'docs')
elif os.path.isdir(os.path.join(app_base_path, "www", "docs")):
app_doc_path = os.path.join(app_base_path, "www", "docs")
if app_doc_path:
symlinks.append([app_doc_path, os.path.join(
assets_path, app_name + '_docs')])
symlinks.append([app_doc_path, os.path.join(assets_path, app_name + "_docs")])
for source, target in symlinks:
source = os.path.abspath(source)
@ -162,7 +283,7 @@ def make_asset_dirs(make_copy=False, restore=False):
shutil.copytree(source, target)
elif make_copy:
if os.path.exists(target):
warnings.warn('Target {target} already exists.'.format(target=target))
warnings.warn("Target {target} already exists.".format(target=target))
else:
shutil.copytree(source, target)
else:
@ -174,7 +295,7 @@ def make_asset_dirs(make_copy=False, restore=False):
try:
symlink(source, target, overwrite=True)
except OSError:
print('Cannot link {} to {}'.format(source, target))
print("Cannot link {} to {}".format(source, target))
else:
# warnings.warn('Source {source} does not exist.'.format(source = source))
pass
@ -193,7 +314,7 @@ def get_build_maps():
build_maps = {}
for app_path in app_paths:
path = os.path.join(app_path, 'public', 'build.json')
path = os.path.join(app_path, "public", "build.json")
if os.path.exists(path):
with open(path) as f:
try:
@ -202,8 +323,7 @@ def get_build_maps():
source_paths = []
for source in sources:
if isinstance(source, list):
s = frappe.get_pymodule_path(
source[0], *source[1].split("/"))
s = frappe.get_pymodule_path(source[0], *source[1].split("/"))
else:
s = os.path.join(app_path, source)
source_paths.append(s)
@ -211,36 +331,42 @@ def get_build_maps():
build_maps[target] = source_paths
except ValueError as e:
print(path)
print('JSON syntax error {0}'.format(str(e)))
print("JSON syntax error {0}".format(str(e)))
return build_maps
def pack(target, sources, no_compress, verbose):
from six import StringIO
outtype, outtxt = target.split(".")[-1], ''
outtype, outtxt = target.split(".")[-1], ""
jsm = JavascriptMinify()
for f in sources:
suffix = None
if ':' in f:
f, suffix = f.split(':')
if ":" in f:
f, suffix = f.split(":")
if not os.path.exists(f) or os.path.isdir(f):
print("did not find " + f)
continue
timestamps[f] = os.path.getmtime(f)
try:
with open(f, 'r') as sourcefile:
data = text_type(sourcefile.read(), 'utf-8', errors='ignore')
with open(f, "r") as sourcefile:
data = text_type(sourcefile.read(), "utf-8", errors="ignore")
extn = f.rsplit(".", 1)[1]
if outtype == "js" and extn == "js" and (not no_compress) and suffix != "concat" and (".min." not in f):
tmpin, tmpout = StringIO(data.encode('utf-8')), StringIO()
if (
outtype == "js"
and extn == "js"
and (not no_compress)
and suffix != "concat"
and (".min." not in f)
):
tmpin, tmpout = StringIO(data.encode("utf-8")), StringIO()
jsm.minify(tmpin, tmpout)
minified = tmpout.getvalue()
if minified:
outtxt += text_type(minified or '', 'utf-8').strip('\n') + ';'
outtxt += text_type(minified or "", "utf-8").strip("\n") + ";"
if verbose:
print("{0}: {1}k".format(f, int(len(minified) / 1024)))
@ -248,27 +374,27 @@ def pack(target, sources, no_compress, verbose):
# add to frappe.templates
outtxt += html_to_js_template(f, data)
else:
outtxt += ('\n/*\n *\t%s\n */' % f)
outtxt += '\n' + data + '\n'
outtxt += "\n/*\n *\t%s\n */" % f
outtxt += "\n" + data + "\n"
except Exception:
print("--Error in:" + f + "--")
print(frappe.get_traceback())
with open(target, 'w') as f:
with open(target, "w") as f:
f.write(outtxt.encode("utf-8"))
print("Wrote %s - %sk" % (target, str(int(os.path.getsize(target)/1024))))
print("Wrote %s - %sk" % (target, str(int(os.path.getsize(target) / 1024))))
def html_to_js_template(path, content):
'''returns HTML template content as Javascript code, adding it to `frappe.templates`'''
"""returns HTML template content as Javascript code, adding it to `frappe.templates`"""
return """frappe.templates["{key}"] = '{content}';\n""".format(
key=path.rsplit("/", 1)[-1][:-5], content=scrub_html_template(content))
def scrub_html_template(content):
'''Returns HTML content with removed whitespace and comments'''
"""Returns HTML content with removed whitespace and comments"""
# remove whitespace to a single space
content = re.sub("\s+", " ", content)
@ -281,12 +407,12 @@ def scrub_html_template(content):
def files_dirty():
for target, sources in iteritems(get_build_maps()):
for f in sources:
if ':' in f:
f, suffix = f.split(':')
if ":" in f:
f, suffix = f.split(":")
if not os.path.exists(f) or os.path.isdir(f):
continue
if os.path.getmtime(f) != timestamps.get(f):
print(f + ' dirty')
print(f + " dirty")
return True
else:
return False

39
frappe/commands/utils.py
View file

@ -1,17 +1,17 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals, absolute_import, print_function
import click
import json, os, sys, subprocess
import json
import os
import subprocess
import sys
from distutils.spawn import find_executable
import click
import frappe
from frappe.commands import pass_context, get_site
from frappe.commands import get_site, pass_context
from frappe.exceptions import SiteNotSpecifiedError
from frappe.utils import update_progress_bar, get_bench_path
from frappe.utils.response import json_handler
from coverage import Coverage
import cProfile, pstats
from six import StringIO
from frappe.utils import get_bench_path, update_progress_bar
@click.command('build')
@ -19,14 +19,22 @@ from six import StringIO
@click.option('--make-copy', is_flag=True, default=False, help='Copy the files instead of symlinking')
@click.option('--restore', is_flag=True, default=False, help='Copy the files instead of symlinking with force')
@click.option('--verbose', is_flag=True, default=False, help='Verbose')
def build(app=None, make_copy=False, restore = False, verbose=False):
@click.option('--force', is_flag=True, default=False, help='Force build assets instead of downloading available')
def build(app=None, make_copy=False, restore=False, verbose=False, force=False):
"Minify + concatenate JS and CSS files, build translations"
import frappe.build
import frappe
frappe.init('')
# don't minify in developer_mode for faster builds
no_compress = frappe.local.conf.developer_mode or False
frappe.build.bundle(no_compress, app=app, make_copy=make_copy, restore = restore, verbose=verbose)
# dont try downloading assets if force used, app specified or running via CI
if not (force or app or os.environ.get('CI')):
# skip building frappe if assets exist remotely
skip_frappe = frappe.build.download_frappe_assets(verbose=verbose)
else:
skip_frappe = False
frappe.build.bundle(no_compress, app=app, make_copy=make_copy, restore=restore, verbose=verbose, skip_frappe=skip_frappe)
@click.command('watch')
@ -152,6 +160,7 @@ def execute(context, method, args=None, kwargs=None, profile=False):
kwargs = {}
if profile:
import cProfile
pr = cProfile.Profile()
pr.enable()
@ -161,6 +170,9 @@ def execute(context, method, args=None, kwargs=None, profile=False):
ret = frappe.safe_eval(method + "(*args, **kwargs)", eval_globals=globals(), eval_locals=locals())
if profile:
import pstats
from six import StringIO
pr.disable()
s = StringIO()
pstats.Stats(pr, stream=s).sort_stats('cumulative').print_stats(.5)
@ -171,6 +183,7 @@ def execute(context, method, args=None, kwargs=None, profile=False):
finally:
frappe.destroy()
if ret:
from frappe.utils.response import json_handler
print(json.dumps(ret, default=json_handler))
if not context.sites:
@ -496,6 +509,8 @@ def run_tests(context, app=None, module=None, doctype=None, test=(),
frappe.flags.skip_test_records = skip_test_records
if coverage:
from coverage import Coverage
# Generate coverage report only for app that is being tested
source_path = os.path.join(get_bench_path(), 'apps', app or 'frappe')
cov = Coverage(source=[source_path], omit=[

1
frappe/utils/change_log.py
View file

@ -9,7 +9,6 @@ import frappe
import requests
import subprocess # nosec
from frappe.utils import cstr
from frappe.utils.gitutils import get_app_branch
from frappe import _, safe_decode

24
frappe/utils/gitutils.py
View file

@ -1,24 +0,0 @@
from __future__ import unicode_literals
import subprocess
def get_app_branch(app):
'''Returns branch of an app'''
try:
branch = subprocess.check_output('cd ../apps/{0} && git rev-parse --abbrev-ref HEAD'.format(app),
shell=True)
branch = branch.decode('utf-8')
branch = branch.strip()
return branch
except Exception:
return ''
def get_app_last_commit_ref(app):
try:
commit_id = subprocess.check_output('cd ../apps/{0} && git rev-parse HEAD'.format(app),
shell=True)
commit_id = commit_id.decode('utf-8')
commit_id = commit_id.strip()[:7]
return commit_id
except Exception:
return ''

1
requirements.txt
View file

@ -57,6 +57,7 @@ RestrictedPython==5.0
rq>=1.1.0
schedule==0.6.0
semantic-version==2.8.4
simple-chalk==0.1.0
six==1.14.0
sqlparse==0.2.4
stripe==2.40.0

51
rollup/build.js
View file

@ -15,17 +15,35 @@ const {
} = require('./rollup.utils');
const {
get_options_for
get_options_for,
get_options
} = require('./config');
const build_for_app = process.argv[2] === '--app' ? process.argv[3] : null;
const skip_frappe = process.argv.includes("--skip_frappe")
show_production_message();
if (skip_frappe) {
let idx = apps_list.indexOf("frappe");
if (idx > -1) {
apps_list.splice(idx, 1);
}
}
const exists = (flag) => process.argv.indexOf(flag) != -1
const value = (flag) => (process.argv.indexOf(flag) != -1) ? process.argv[process.argv.indexOf(flag) + 1] : null;
const files = exists("--files") ? value("--files").split(",") : false;
const build_for_app = exists("--app") ? value("--app") : null;
const concat = !exists("--no-concat");
if (!files) show_production_message();
ensure_js_css_dirs();
concatenate_files();
if (concat) concatenate_files();
create_build_file();
if (build_for_app) {
if (files) {
build_files(files);
} else if (build_for_app) {
build_assets_for_app(build_for_app)
.then(() => {
run_build_command_for_app(build_for_app);
@ -48,11 +66,7 @@ function build_assets_for_app(app) {
return build_assets(app)
}
function build_assets(app) {
const options = get_options_for(app);
if (!options.length) return Promise.resolve();
log(chalk.yellow(`\nBuilding ${app} assets...\n`));
function build_from_(options) {
const promises = options.map(({ inputOptions, outputOptions, output_file}) => {
return build(inputOptions, outputOptions)
.then(() => {
@ -68,6 +82,23 @@ function build_assets(app) {
});
}
function build_assets(app) {
const options = get_options_for(app);
if (!options.length) return Promise.resolve();
log(chalk.yellow(`\nBuilding ${app} assets...\n`));
return build_from_(options);
}
function build_files(files, app="frappe") {
let ret;
for (let file of files) {
let options = get_options(file, app);
if (!options.length) return Promise.resolve();
ret += build_from_(options);
}
return ret;
}
function build(inputOptions, outputOptions) {
return rollup.rollup(inputOptions)
.then(bundle => bundle.write(outputOptions))

28
rollup/config.js
View file

@ -165,6 +165,31 @@ function get_rollup_options_for_css(output_file, input_files) {
};
}
function get_options(file, app="frappe") {
const build_json = get_build_json(app);
if (!build_json) return [];
return Object.keys(build_json)
.map(output_file => {
if (output_file === file) {
if (output_file.startsWith('concat:')) return null;
const input_files = build_json[output_file]
.map(input_file => {
let prefix = get_app_path(app);
if (input_file.startsWith('node_modules/')) {
prefix = path.resolve(get_app_path(app), '..');
}
return path.resolve(prefix, input_file);
});
return Object.assign(
get_rollup_options(output_file, input_files), {
output_file
});
}
})
.filter(Boolean);
}
function get_options_for(app) {
const build_json = get_build_json(app);
if (!build_json) return [];
@ -205,5 +230,6 @@ function ignore_css() {
};
module.exports = {
get_options_for
get_options_for,
get_options
};