Skip to content

H
heroku-buildpack-nodejs
Commit 06f23f2e authored by Hunter Loftis's avatar Hunter Loftis
Browse files
Options

better caching for yarn

parent 0387c211
Hide whitespace changes
Inline Side-by-side
Showing with 74 additions and 77 deletions
CHANGELOG.md
View file @ 06f23f2e
# Node.js Buildpack Changelog
## Master
- Use cache directories instead of node_modules
- Use yarn, if available, as part of the cache signature
- Warn about yarn's youth and evolution on build failures
- Link to opt-out of yarn instructions
- Use `yarn list` instead of `yarn ls`
- Hide final dep tree listings under a `NODE_VERBOSE` flag
## v94 (2016-12-16)
- Warn on yarn NODE_ENV and NPM_CONFIG incompatibility
......
bin/compile
View file @ 06f23f2e
......@@ -10,7 +10,7 @@ unset GIT_DIR # Avoid GIT_DIR leak from previous build steps
### Constants
DEFAULT_CACHE="node_modules bower_components"
DEFAULT_CACHE=".npm .cache/yarn bower_components"
### Configure directories
......@@ -44,6 +44,7 @@ handle_failure() {
warn_angular_resolution "$LOG_FILE"
warn_missing_devdeps "$LOG_FILE"
warn_econnreset "$LOG_FILE"
warn_young_yarn "$LOG_FILE"
failure_message | output "$LOG_FILE"
}
trap 'handle_failure' ERR
......@@ -117,7 +118,7 @@ restore_cache() {
if [ "$cache_status" == "valid" ]; then
local cache_directories=$(get_cache_directories)
if [ "$cache_directories" == "" ]; then
echo "Loading 2 from cacheDirectories (default):"
echo "Loading 3 from cacheDirectories (default):"
restore_cache_directories "$BUILD_DIR" "$CACHE_DIR" "$DEFAULT_CACHE"
else
echo "Loading $(echo $cache_directories | wc -w | xargs) from cacheDirectories (package.json):"
......@@ -156,7 +157,7 @@ cache_build() {
if ! ${NODE_MODULES_CACHE:-true}; then
echo "Skipping cache save (disabled by config)"
elif [ "$cache_directories" == "" ]; then
echo "Saving 2 cacheDirectories (default):"
echo "Saving 3 cacheDirectories (default):"
save_cache_directories "$BUILD_DIR" "$CACHE_DIR" "$DEFAULT_CACHE"
else
echo "Saving $(echo $cache_directories | wc -w | xargs) cacheDirectories (package.json):"
......@@ -169,7 +170,9 @@ header "Caching build"
cache_build | output "$LOG_FILE"
summarize_build() {
list_dependencies "$BUILD_DIR"
if $NODE_VERBOSE; then
list_dependencies "$BUILD_DIR"
fi
}
header "Build succeeded!"
......
bin/test-compile
View file @ 06f23f2e
#!/usr/bin/env bash
NPM_CONFIG_PRODUCTION=false "$(dirname ${0:-})/compile" "$1" "$2" "$3"
NPM_CONFIG_PRODUCTION=false YARN_PRODUCTION=false "$(dirname ${0:-})/compile" "$1" "$2" "$3"
lib/cache.sh
View file @ 06f23f2e
source $BP_DIR/lib/binaries.sh
create_signature() {
echo "$(node --version); $(npm --version)"
echo "$(node --version); $(npm --version); $(yarn --version 2>/dev/null || true)"
}
save_signature() {
......
lib/dependencies.sh
View file @ 06f23f2e
......@@ -4,7 +4,7 @@ list_dependencies() {
cd "$build_dir"
if $YARN; then
echo ""
(yarn ls || true) 2>/dev/null
(yarn list --depth=0 || true) 2>/dev/null
echo ""
else
(npm ls --depth=0 | tail -n +2 || true) 2>/dev/null
......@@ -27,20 +27,17 @@ run_if_present() {
yarn_node_modules() {
local build_dir=${1:-}
echo "Installing node modules (yarn)"
echo "Installing node modules (yarn.lock)"
cd "$build_dir"
yarn install --pure-lockfile --ignore-engines --cache-folder $build_dir/.cache/yarn 2>&1
# according to docs: "Verifies that versions of the package dependencies in the current project’s package.json matches that of yarn’s lock file."
# however, appears to also check for the presence of deps in node_modules
# yarn check 1>/dev/null
if [ "$NODE_ENV" == "production" ] && [ "$NPM_CONFIG_PRODUCTION" == "false" ]; then
echo ""
echo "Warning: when NODE_ENV=production, yarn will NOT install any devDependencies"
echo " (even if NPM_CONFIG_PRODUCTION is false)"
echo " https://yarnpkg.com/en/docs/cli/install#toc-yarn-install-production"
echo ""
# however, appears to also check for the presence of deps in node_modules, so must be run after install
if $(yarn check 1>/dev/null); then
echo "yarn.lock and package.json match"
else
error "yarn.lock is outdated. run \`yarn install\`, commit the updated \`yarn.lock\`, and redeploy"
return 1
fi
yarn install --pure-lockfile --ignore-engines 2>&1
}
npm_node_modules() {
......@@ -54,7 +51,7 @@ npm_node_modules() {
else
echo "Installing node modules (package.json)"
fi
npm install --unsafe-perm --userconfig $build_dir/.npmrc 2>&1
npm install --unsafe-perm --userconfig $build_dir/.npmrc --cache $build_dir/.npm 2>&1
else
echo "Skipping (no package.json)"
fi
......
lib/environment.sh
View file @ 06f23f2e
......@@ -3,11 +3,13 @@ create_default_env() {
export NPM_CONFIG_LOGLEVEL=${NPM_CONFIG_LOGLEVEL:-error}
export NODE_MODULES_CACHE=${NODE_MODULES_CACHE:-true}
export NODE_ENV=${NODE_ENV:-production}
export NODE_VERBOSE=${NODE_VERBOSE:-false}
}
list_node_config() {
echo ""
printenv | grep ^NPM_CONFIG_ || true
printenv | grep ^YARN_ || true
printenv | grep ^NODE_ || true
if [ "$NPM_CONFIG_PRODUCTION" = "true" ] && [ "$NODE_ENV" != "production" ]; then
......
lib/failure.sh
View file @ 06f23f2e
......@@ -76,6 +76,12 @@ warn_old_npm() {
fi
}
warn_young_yarn() {
if $YARN; then
warning "This project was built with yarn, which is new and under development. Some projects can still be built more reliably with npm" "https://devcenter.heroku.com/articles/nodejs-support#build-behavior"
fi
}
warn_untracked_dependencies() {
local log_file="$1"
if grep -qi 'gulp: not found' "$log_file" || grep -qi 'gulp: command not found' "$log_file"; then
......
test/run
View file @ 06f23f2e
#!/usr/bin/env bash
# See README.md for info on running these tests.
testNodeModulesCached() {
cache=$(mktmpdir)
compile "caching" $cache
assertCaptured "Saving 3 cacheDirectories (default)"
assertCaptured "- .npm"
assertCaptured "- .cache/yarn (nothing to cache)"
assertCaptured "- bower_components (nothing to cache)"
assertEquals "1" "$(ls -1 $cache/node/.npm | grep express | wc -l | tr -d ' ')"
assertCapturedSuccess
}
testYarn() {
compile "yarn"
assertCaptured "installing yarn"
assertCaptured "Installing node modules (yarn.lock)"
assertNotCaptured "Installing node modules (package.json"
assertCapturedSuccess
}
testBuildWithCache() {
cache=$(mktmpdir)
compile "stable-node" $cache
assertCaptured "Skipping cache restore (new runtime"
assertEquals "1" "$(ls -1 $cache/node/.npm | grep hashish | wc -l | tr -d ' ')"
assertCapturedSuccess
compile "stable-node" $cache
assertNotCaptured "- .npm (not cached - skipping)"
assertCapturedSuccess
rm -rf "$cache/node/.npm"
compile "stable-node" $cache
assertCaptured "- .npm (not cached - skipping)"
assertCapturedSuccess
}
testYarnSemver() {
compile "yarn-semver"
assertCaptured "Resolving yarn version ~0.17"
......@@ -21,14 +59,6 @@ testYarnEngine() {
assertCapturedSuccess
}
testYarn() {
compile "yarn"
assertCaptured "installing yarn"
assertCaptured "Installing node modules (yarn)"
assertNotCaptured "Installing node modules (package.json"
assertCapturedSuccess
}
testWarnUnmetDep() {
compile "unmet-dep"
assertCaptured "may cause runtime issues"
......@@ -116,27 +146,6 @@ testMultipleRuns() {
assertCapturedSuccess
}
testDisableCache() {
cache=$(mktmpdir)
env_dir=$(mktmpdir)
compile "node-modules-cache-1" $cache
assertCaptured "lodash@1.0.0"
assertEquals "1" "$(ls -1 $cache/node/node_modules | grep lodash | wc -l | tr -d ' ')"
assertCapturedSuccess
compile "node-modules-cache-2" $cache
assertCaptured "lodash@1.0.0"
assertCaptured "Saving 2 cacheDirectories"
assertCapturedSuccess
echo "false" > $env_dir/NODE_MODULES_CACHE
compile "node-modules-cache-2" $cache $env_dir
assertCaptured "lodash@1.3.1"
assertNotCaptured "Saving 2 cacheDirectories"
assertCapturedSuccess
}
testBowerAngularResolution() {
compile "bower-angular-resolution"
assertCaptured "Bower may need a resolution hint for angular"
......@@ -157,17 +166,6 @@ testBadJson() {
assertCapturedError 1 "Unable to parse"
}
testNodeModulesCached() {
cache=$(mktmpdir)
compile "caching" $cache
assertCaptured "Saving 2 cacheDirectories (default)"
assertCaptured "- node_modules"
assertCaptured "- bower_components (nothing to cache)"
assertEquals "1" "$(ls -1 $cache/node | grep node_modules | wc -l | tr -d ' ')"
assertCapturedSuccess
}
testBuildWithUserCacheDirectoriesCamel() {
cache=$(mktmpdir)
......@@ -234,24 +232,6 @@ testInvalidIo() {
assertCapturedError
}
testBuildWithCache() {
cache=$(mktmpdir)
compile "stable-node" $cache
assertCaptured "Skipping cache restore (new runtime"
assertEquals "1" "$(ls -1 $cache/node | grep node_modules | wc -l | tr -d ' ')"
assertCapturedSuccess
compile "stable-node" $cache
assertNotCaptured "- node_modules (not cached - skipping)"
assertCapturedSuccess
rm -rf "$cache/node/node_modules"
compile "stable-node" $cache
assertCaptured "- node_modules (not cached - skipping)"
assertCapturedSuccess
}
testSignatureInvalidation() {
cache=$(mktmpdir)
env_dir=$(mktmpdir)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment