Commit ce7f4793 authored by Hunter Loftis's avatar Hunter Loftis

bring caching logic to the forefront

parent 97f67177
#!/usr/bin/env bash
# bin/compile <build-dir> <cache-dir> <env-dir>
####### Configure environment
### Configure environment
set -o errexit # always exit on error
set -o errtrace # trap errors in functions as well
set -o pipefail # don't ignore exit codes when piping output
set -o posix # more strict failures in subshells
# set -x # enable debugging
set -o nounset # fail on unset variables
unset GIT_DIR # Avoid GIT_DIR leak from previous build steps
# Configure directories
build_dir=$1
cache_dir=$2
env_dir=$3
bp_dir=$(cd $(dirname $0); cd ..; pwd)
heroku_dir=$build_dir/.heroku
mkdir -p $heroku_dir/node
warnings=$(mktemp)
### Configure directories
# Load dependencies
source $bp_dir/lib/common.sh
source $bp_dir/lib/build.sh
source $bp_dir/lib/warnings.sh
BUILD_DIR=${1:-}
CACHE_DIR=${2:-}
ENV_DIR=${3:-}
BP_DIR=$(cd $(dirname ${0:-}); cd ..; pwd)
# Avoid GIT_DIR leak from previous build steps
unset GIT_DIR
mkdir -p "$BUILD_DIR/.heroku/node/"
cd $BUILD_DIR
export PATH="$BUILD_DIR/.heroku/node/bin":$PATH
# Provide hook to deal with errors
trap build_failed ERR
### Load dependencies
####### Determine current state
source $BP_DIR/lib/output.sh
source $BP_DIR/lib/json.sh
source $BP_DIR/lib/failure.sh
source $BP_DIR/lib/environment.sh
source $BP_DIR/lib/binaries.sh
source $BP_DIR/lib/cache.sh
source $BP_DIR/lib/dependencies.sh
head "Reading application state"
read_current_state
show_current_state
### Handle errors
if [ "$iojs_engine" == "" ]; then
warn_node_engine "$node_engine"
else
warn_node_engine "$iojs_engine"
fi
warn_node_modules "$modules_source"
handle_failure() {
header "Build failed"
failure_message | indent
}
trap 'handle_failure' ERR
### Check initial state
####### Vendor in binaries
[ -e "$BUILD_DIR/node_modules" ] && PREBUILD=true || PREBUILD=false
head "Installing binaries"
if [ "$iojs_engine" == "" ]; then
install_node "$node_engine"
else
install_iojs "$iojs_engine"
fi
install_npm
### Failures that should be caught immediately
####### Build the project's dependencies
fail_invalid_package_json "$BUILD_DIR"
warn_prebuilt_modules "$BUILD_DIR"
warn_missing_package_json "$BUILD_DIR"
head "Building dependencies"
cd $build_dir
build_dependencies
### Compile
####### Create a Procfile if possible
create_env() {
write_profile "$BP_DIR" "$BUILD_DIR"
write_export "$BP_DIR" "$BUILD_DIR"
export_env_dir "$ENV_DIR"
create_default_env
}
head "Checking startup method"
ensure_procfile "$start_method" "$build_dir"
warn_start "$start_method"
header "Creating runtime environment"
create_env # can't indent the whole thing because piping causes subshells; no exporting possible
list_node_config | indent
####### Finalize the build
install_bins() {
local node_engine=$(read_json "$BUILD_DIR/package.json" ".engines.node")
local iojs_engine=$(read_json "$BUILD_DIR/package.json" ".engines.iojs")
local npm_engine=$(read_json "$BUILD_DIR/package.json" ".engines.npm")
head "Finalizing build"
write_profile
write_export
clean_npm
clean_cache
create_cache
build_succeeded
if [ -n "$iojs_engine" ]; then
echo "engines.iojs (package.json): $iojs_engine (iojs)"
else
echo "engines.node (package.json): ${node_engine:-unspecified}"
fi
echo "engines.npm (package.json): ${npm_engine:-unspecified (use default)}"
echo ""
if [ -n "$iojs_engine" ]; then
warn_node_engine "$iojs_engine"
install_iojs "$iojs_engine" "$BUILD_DIR/.heroku/node"
else
warn_node_engine "$node_engine"
install_nodejs "$node_engine" "$BUILD_DIR/.heroku/node"
fi
install_npm "$npm_engine" "$BUILD_DIR/.heroku/node"
warn_old_npm
}
header "Installing binaries"
install_bins | indent
restore_cache() {
local cache_status=$(get_cache_status)
if [ "$cache_status" == "disabled" ]; then
echo "Skipping (cache disabled)"
elif [ "$cache_status" == "invalidated" ]; then
echo "Skipping (cache invalidated)"
else
local cache_directories=$(get_cache_directories)
if [ "$cache_directories" == "" ]; then
echo "Loading 1 from cacheDirectories (default):"
restore_cache_directories "$BUILD_DIR" "$CACHE_DIR" "node_modules"
else
echo "Loading $(echo $cache_directories | wc -w | xargs) from cacheDirectories (package.json):"
restore_cache_directories "$BUILD_DIR" "$CACHE_DIR" $cache_directories
fi
fi
}
header "Restoring cache"
restore_cache | indent
build_dependencies() {
if $PREBUILD; then
echo "Prebuild detected (node_modules already exists)"
rebuild_node_modules "$BUILD_DIR"
else
install_node_modules "$BUILD_DIR"
fi
}
header "Building dependencies"
build_dependencies | indent
cache_build() {
local cache_directories=$(get_cache_directories)
echo "Clearing previous node cache"
clear_cache
if [ "$cache_directories" == "" ]; then
echo "Saving 1 cacheDirectories (default):"
save_cache_directories "$BUILD_DIR" "$CACHE_DIR" "node_modules"
else
echo "Saving $(echo $cache_directories | wc -w | xargs) cacheDirectories (package.json):"
save_cache_directories "$BUILD_DIR" "$CACHE_DIR" $cache_directories
fi
}
header "Caching build"
cache_build | indent
summarize_build() {
cd $BUILD_DIR
(npm ls --depth=0 | tail -n +2 || true) 2>/dev/null
}
header "Build succeeded!"
summarize_build | indent
......@@ -2,9 +2,8 @@
# bin/detect <build-dir>
if [ -f $1/package.json ]; then
echo "Node.js" && exit 0
elif [ -f $1/server.js ]; then
echo "Node.js" && exit 0
else
echo "no" && exit 1
echo "Node.js"
exit 0
fi
exit 1
#!/usr/bin/env bash
# bin/release <build-dir>
cat << EOF
addons: []
default_process_types: {}
default_process_types:
web: npm start
EOF
needs_resolution() {
local semver=$1
if ! [[ "$semver" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
return 0
else
return 1
fi
}
install_nodejs() {
local version="$1"
local dir="$2"
if needs_resolution "$version"; then
echo "Resolving node version ${version:-(latest stable)} via semver.io..."
local version=$(curl --silent --get --data-urlencode "range=${version}" https://semver.herokuapp.com/node/resolve)
fi
echo "Downloading and installing node $version..."
local download_url="http://s3pository.heroku.com/node/v$version/node-v$version-$os-$cpu.tar.gz"
curl "$download_url" -s -o - | tar xzf - -C /tmp
mv /tmp/node-v$version-$os-$cpu/* $dir
chmod +x $dir/bin/*
}
install_iojs() {
local version="$1"
local dir="$2"
if needs_resolution "$version"; then
echo "Resolving iojs version ${version:-(latest stable)} via semver.io..."
version=$(curl --silent --get --data-urlencode "range=${version}" https://semver.herokuapp.com/iojs/resolve)
fi
echo "Downloading and installing iojs $version..."
local download_url="https://iojs.org/dist/v$version/iojs-v$version-$os-$cpu.tar.gz"
curl $download_url -s -o - | tar xzf - -C /tmp
mv /tmp/iojs-v$version-$os-$cpu/* $dir
chmod +x $dir/bin/*
}
install_npm() {
local version="$1"
if [ "$version" == "" ]; then
echo "Using default npm version: `npm --version`"
else
if needs_resolution "$version"; then
echo "Resolving npm version ${version} via semver.io..."
version=$(curl --silent --get --data-urlencode "range=${version}" https://semver.herokuapp.com/npm/resolve)
fi
if [[ `npm --version` == "$version" ]]; then
echo "npm `npm --version` already installed with node"
else
echo "Downloading and installing npm $version (replacing version `npm --version`)..."
npm install --unsafe-perm --quiet -g npm@$version 2>&1 >/dev/null
fi
fi
}
This diff is collapsed.
source $BP_DIR/lib/binaries.sh
create_signature() {
echo "$(node --version); $(npm --version)"
}
save_signature() {
echo "$(get_signature)" > $CACHE_DIR/node/signature
}
load_signature() {
if test -f $CACHE_DIR/node/signature; then
cat $CACHE_DIR/node/signature
else
echo ""
fi
}
signature_changed() {
if ! [ "$(create_signature)" == "$(load_signature)" ]; then
return 1
else
return 0
fi
}
get_cache_status() {
if ! ${NODE_MODULES_CACHE:-true}; then
echo "disabled"
elif signature_changed; then
echo "invalidated"
else
echo "valid"
fi
}
get_cache_directories() {
local dirs1=$(read_json "$BUILD_DIR/package.json" ".cacheDirectories | .[]?")
local dirs2=$(read_json "$BUILD_DIR/package.json" ".cache_directories | .[]?")
if [ -n "$dirs1" ]; then
echo "$dirs1"
else
echo "$dirs2"
fi
}
restore_cache_directories() {
local build_dir=${1:-}
local cache_dir=${2:-}
for cachepath in ${@:3}; do
if [ -e "$build_dir/$cachepath" ]; then
echo "- $cachepath (exists - skipping)"
else
if [ -e "$cache_dir/node/$cachepath" ]; then
echo "- $cachepath"
mkdir -p "$build_dir/$cachepath"
cp -a "$cache_dir/node/$cachepath" $(dirname "$build_dir/$cachepath")
else
echo "- $cachepath (not cached - skipping)"
fi
fi
done
}
clear_cache() {
rm -rf $CACHE_DIR/node
}
save_cache_directories() {
local build_dir=${1:-}
local cache_dir=${2:-}
mkdir -p $cache_dir/node
for cachepath in ${@:3}; do
if [ -e "$build_dir/$cachepath" ]; then
echo "- $cachepath"
mkdir -p "$cache_dir/node/$cachepath"
cp -a "$build_dir/$cachepath" $(dirname "$cache_dir/node/$cachepath")
else
echo "- $cachepath (nothing to cache)"
fi
done
}
error() {
echo " ! $*" >&2
echo ""
return 1
}
head() {
echo ""
echo "-----> $*"
}
info() {
#echo "`date +\"%M:%S\"` $*"
echo " $*"
}
warning() {
local tip=$1
local url=$2
echo "- $tip" >> $warnings
echo " ${url:-https://devcenter.heroku.com/articles/nodejs-support}" >> $warnings
echo "" >> $warnings
}
achievement() {
local msg=$1
echo " ACHIEVEMENT UNLOCKED: $msg :)"
echo ""
}
assert_json() {
local file=$1
if test -f $file; then
if ! cat $file | $bp_dir/vendor/jq '.' > /dev/null; then
error "Unable to parse $file as JSON"
fi
fi
}
file_contents() {
if test -f $1; then
echo "$(cat $1)"
else
echo ""
fi
}
read_json() {
local file=$1
local node=$2
if test -f $file; then
cat $file | $bp_dir/vendor/jq --raw-output "$node // \"\"" || return 1
else
echo ""
fi
}
# sed -l basically makes sed replace and buffer through stdin to stdout
# so you get updates while the command runs and dont wait for the end
# e.g. npm install | indent
indent() {
c='s/^/ /'
case $(uname) in
Darwin) sed -l "$c";; # mac/bsd sed: -l buffers on line boundaries
*) sed -u "$c";; # unix/gnu sed: -u unbuffered (arbitrary) chunks of data
esac
}
cat_npm_debug_log() {
test -f $build_dir/npm-debug.log && cat $build_dir/npm-debug.log
}
export_env_dir() {
env_dir=$1
if [ -d "$env_dir" ]; then
whitelist_regex=${2:-''}
blacklist_regex=${3:-'^(PATH|GIT_DIR|CPATH|CPPATH|LD_PRELOAD|LIBRARY_PATH)$'}
if [ -d "$env_dir" ]; then
for e in $(ls $env_dir); do
echo "$e" | grep -E "$whitelist_regex" | grep -qvE "$blacklist_regex" &&
export "$e=$(cat $env_dir/$e)"
:
done
fi
fi
}
install_node_modules() {
local build_dir=${1:-}
if [ -e $build_dir/package.json ]; then
cd $build_dir
echo "Pruning any extraneous modules"
npm prune --unsafe-perm --userconfig $build_dir/.npmrc 2>&1
if [ -e $build_dir/npm-shrinkwrap.json ]; then
echo "Installing node modules (package.json + shrinkwrap)"
else
echo "Installing node modules (package.json)"
fi
npm install --unsafe-perm --userconfig $build_dir/.npmrc 2>&1
else
echo "Skipping (no package.json)"
fi
}
rebuild_node_modules() {
local build_dir=${1:-}
if [ -e $build_dir/package.json ]; then
cd $build_dir
echo "Pruning any extraneous modules"
npm prune --unsafe-perm --userconfig $build_dir/.npmrc 2>&1
echo "Rebuilding any native modules"
npm rebuild 2>&1
if [ -e $build_dir/npm-shrinkwrap.json ]; then
echo "Installing any new modules (package.json + shrinkwrap)"
else
echo "Installing any new modules (package.json)"
fi
npm install --unsafe-perm --userconfig $build_dir/.npmrc 2>&1
else
echo "Skipping (no package.json)"
fi
}
create_default_env() {
export NPM_CONFIG_PRODUCTION=${NPM_CONFIG_PRODUCTION:-true}
export NPM_CONFIG_LOGLEVEL=${NPM_CONFIG_LOGLEVEL:-error}
export NODE_MODULES_CACHE=${NODE_MODULES_CACHE:-true}
}
list_node_config() {
echo ""
printenv | grep ^NPM_CONFIG_ || true
printenv | grep ^NODE_ || true
}
export_env_dir() {
local env_dir=$1
if [ -d "$env_dir" ]; then
local whitelist_regex=${2:-''}
local blacklist_regex=${3:-'^(PATH|GIT_DIR|CPATH|CPPATH|LD_PRELOAD|LIBRARY_PATH)$'}
if [ -d "$env_dir" ]; then
for e in $(ls $env_dir); do
echo "$e" | grep -E "$whitelist_regex" | grep -qvE "$blacklist_regex" &&
export "$e=$(cat $env_dir/$e)"
:
done
fi
fi
}
write_profile() {
local bp_dir="$1"
local build_dir="$2"
mkdir -p $build_dir/.profile.d
cp $bp_dir/profile/* $build_dir/.profile.d/
}
write_export() {
local bp_dir="$1"
local build_dir="$2"
echo "export PATH=\"$build_dir/.heroku/node/bin:$build_dir/node_modules/.bin:\$PATH\"" > $bp_dir/export
echo "export NODE_HOME=\"$build_dir/.heroku/node\"" >> $bp_dir/export
}
warnings=$(mktemp -t heroku-buildpack-nodejs-XXXX)
failure_message() {
local warn="$(cat $warnings)"
echo ""
echo "We're sorry this build is failing! You can troubleshoot common issues here:"
echo "https://devcenter.heroku.com/articles/troubleshooting-node-deploys"
echo ""
if [ "$warn" != "" ]; then
echo "Some possible problems:"
echo ""
echo "$warn"
else
echo "If you're stuck, please submit a ticket so we can help:"
echo "https://help.heroku.com/"
fi
echo ""
echo "Love,"
echo "Heroku"
echo ""
}
fail_invalid_package_json() {
if ! cat ${1:-}/package.json | $JQ "." 1>/dev/null; then
error "Unable to parse package.json"
return 1
fi
}
warning() {
local tip=${1:-}
local url=${2:-https://devcenter.heroku.com/articles/nodejs-support}
echo "- $tip" >> $warnings
echo " $url" >> $warnings
echo "" >> $warnings
}
warn_node_engine() {
local node_engine=$1
local node_engine=${1:-}
if [ "$node_engine" == "" ]; then
warning "Node version not specified in package.json" "https://devcenter.heroku.com/articles/nodejs-support#specifying-a-node-js-version"
elif [ "$node_engine" == "*" ]; then
......@@ -9,26 +46,24 @@ warn_node_engine() {
fi
}
warn_node_modules() {
local modules_source=$1
if [ "$modules_source" == "prebuilt" ]; then
warn_prebuilt_modules() {
local build_dir=${1:-}
if [ -e "$build_dir/node_modules" ]; then
warning "node_modules checked into source control" "https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git-"
elif [ "$modules_source" == "" ]; then
warning "No package.json found"
fi
}
warn_start() {
local start_method=$1
if [ "$start_method" == "" ]; then
warning "No Procfile, package.json start script, or server.js file found" "https://devcenter.heroku.com/articles/nodejs-support#runtime-behavior"
warn_missing_package_json() {
local build_dir=${1:-}
if ! [ -e "$build_dir/package.json" ]; then
warning "No package.json found"
fi
}
warn_old_npm() {
local npm_version=$1
local npm_version="$(npm --version)"
if [ "${npm_version:0:1}" -lt "2" ]; then
local latest_npm=$(curl --silent --get https://semver.herokuapp.com/npm/stable)
local latest_npm="$(curl --silent --get https://semver.herokuapp.com/npm/stable)"
warning "This version of npm ($npm_version) has several known issues - consider upgrading to the latest release ($latest_npm)" "https://devcenter.heroku.com/articles/nodejs-support#specifying-an-npm-version"
fi
}
get_os() {
uname | tr A-Z a-z
}
get_cpu() {
if [[ "$(uname -p)" = "i686" ]]; then
echo "x86"
else
echo "x64"
fi
}
os=$(get_os)
cpu=$(get_cpu)
export JQ="$BP_DIR/vendor/jq-$os"
read_json() {
local file=$1
local key=$2
if test -f $file; then
cat $file | $JQ --raw-output "$key // \"\"" || return 1
else
echo ""
fi
}
info() {
echo " $*"
}
# sed -l basically makes sed replace and buffer through stdin to stdout
# so you get updates while the command runs and dont wait for the end
# e.g. npm install | indent
indent() {
c='s/^/ /'
case $(uname) in
Darwin) sed -l "$c";; # mac/bsd sed: -l buffers on line boundaries
*) sed -u "$c";; # unix/gnu sed: -u unbuffered (arbitrary) chunks of data
esac
}
header() {
echo ""
echo "-----> $*"
}
error() {
echo " ! $*" >&2
echo ""
}
......@@ -8,3 +8,8 @@ test-cedar-14:
test-cedar-10:
@echo "Running tests in docker (cedar)..."
@docker run -v $(shell pwd):/buildpack:ro --rm -it fabiokung/cedar bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run;'
shell:
@echo "Opening cedar-14 shell..."
@docker run -v $(shell pwd):/buildpack:ro --rm -it heroku/cedar:14 bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; bash'
@echo ""
......@@ -27,6 +27,9 @@ detect_memory() {
esac
}
export PATH="$HOME/.heroku/node/bin:$HOME/bin:$HOME/node_modules/.bin:$PATH"
export NODE_HOME="$HOME/.heroku/node"
calculate_concurrency
log_concurrency
......
{
"name": "client",
"version": "1.0.0",
"description": "",
"main": "index.js",
"repository": {
"type": "git",
"url": "http://github.com/example/example.git"
},
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"author": "",
"license": "ISC",
"dependencies": {
"underscore": "^1.8.3"
}
}
......@@ -7,13 +7,11 @@
"url" : "http://github.com/example/example.git"
},
"engines": {
"node": "0.10.18"
},
"dependencies": {
"bower": "1.3.12"
"node": "0.12.4"
},
"dependencies": {},
"scripts": {
"postinstall": "bower install --allow-root"
"postinstall": "npm install --prefix=server && npm install --prefix=client"
},
"cacheDirectories": ["bower_components", "node_modules"]
"cacheDirectories": ["server/node_modules", "client/node_modules", "non/existent"]
}
{
"name": "server",
"version": "1.0.0",
"description": "",
"main": "index.js",
"repository": {
"type": "git",
"url": "http://github.com/example/example.git"
},
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"author": "",
"license": "ISC",
"dependencies": {
"lodash": "^3.9.3"
}
}
......@@ -6,8 +6,8 @@
"type" : "git",
"url" : "http://github.com/example/example.git"
},
"dependencies": {
"forcefailure": "*"
"scripts": {
"postinstall": "exit 1"
},
"engines": {
"node": "*"
......
......@@ -10,7 +10,7 @@
"hashish": "*"
},
"engines": {
"node": "0.10.x"
"node": "0.10.38"
},
"scripts": {
"postinstall": "exit 1"
......
......@@ -7,7 +7,7 @@
"url" : "http://github.com/example/example.git"
},
"engines": {
"node": "~0.10.20"
"node": "0.12.4"
},
"dependencies": {
"some-crazy-dep-that-doesnt-exist": "*"
......
......@@ -10,6 +10,6 @@
"lodash": "1.0.0"
},
"engines": {
"node": "~0.10.0"
"node": "0.12.4"
}
}
......@@ -10,6 +10,6 @@
"lodash": "^1.0.0"
},
"engines": {
"node": "~0.10.0"
"node": "0.12.4"
}
}
A fake README, to keep npm from polluting stderr.
\ No newline at end of file
{
"name": "node-buildpack-test-app",
"version": "0.0.1",
"description": "node buildpack integration test app",
"repository" : {
"type" : "git",
"url" : "http://github.com/example/example.git"
},
"dependencies": {
"hashish": "*"
},
"engines": {
"node": "~0.10.0"
}
}
A fake README, to keep npm from polluting stderr.
\ No newline at end of file
{
"name": "node-buildpack-test-app",
"version": "0.0.1",
"description": "node buildpack integration test app",
"repository" : {
"type" : "git",
"url" : "http://github.com/example/example.git"
},
"dependencies": {
"hashish": "*"
},
"engines": {
"node": "~0.10.0"
},
"scripts": {
"start": "echo foo"
}
}
A fake README, to keep npm from polluting stderr.
\ No newline at end of file
{
"name": "node-buildpack-test-app",
"version": "0.0.1",
"description": "node buildpack integration test app",
"repository" : {
"type" : "git",
"url" : "http://github.com/example/example.git"
},
"dependencies": {
"hashish": "*"
},
"engines": {
"node": "~0.10.0"
}
}
A fake README, to keep npm from polluting stderr.
\ No newline at end of file
A fake README, to keep npm from polluting stderr.
\ No newline at end of file
#!/usr/bin/env bash
mktmpdir() {
local dir=$(mktemp -t testXXXXX)
rm -rf $dir
mkdir $dir
echo $dir
}
compile() {
local fixture=$1
local bp_dir=$(mktmpdir)
local build_dir=$(mktmpdir)
local cache_dir=$(mktmpdir)
local env_dir=$(mktmpdir)
echo "Compiling $fixture"
echo "in $build_dir"
echo "(caching in $cache_dir)"
cp -a $(pwd)/* ${bp_dir}
cp -a ${bp_dir}/test/fixtures/$fixture/. ${build_dir}
"$bp_dir/bin/compile" "$build_dir" "$cache_dir"
}
fixture=${1:-'stable-node'}
compile "$fixture"
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment