Commit 06f23f2e authored by Hunter Loftis's avatar Hunter Loftis

better caching for yarn

parent 0387c211
# Node.js Buildpack Changelog # Node.js Buildpack Changelog
## Master
- Use cache directories instead of node_modules
- Use yarn, if available, as part of the cache signature
- Warn about yarn's youth and evolution on build failures
- Link to opt-out of yarn instructions
- Use `yarn list` instead of `yarn ls`
- Hide final dep tree listings under a `NODE_VERBOSE` flag
## v94 (2016-12-16) ## v94 (2016-12-16)
- Warn on yarn NODE_ENV and NPM_CONFIG incompatibility - Warn on yarn NODE_ENV and NPM_CONFIG incompatibility
......
...@@ -10,7 +10,7 @@ unset GIT_DIR # Avoid GIT_DIR leak from previous build steps ...@@ -10,7 +10,7 @@ unset GIT_DIR # Avoid GIT_DIR leak from previous build steps
### Constants ### Constants
DEFAULT_CACHE="node_modules bower_components" DEFAULT_CACHE=".npm .cache/yarn bower_components"
### Configure directories ### Configure directories
...@@ -44,6 +44,7 @@ handle_failure() { ...@@ -44,6 +44,7 @@ handle_failure() {
warn_angular_resolution "$LOG_FILE" warn_angular_resolution "$LOG_FILE"
warn_missing_devdeps "$LOG_FILE" warn_missing_devdeps "$LOG_FILE"
warn_econnreset "$LOG_FILE" warn_econnreset "$LOG_FILE"
warn_young_yarn "$LOG_FILE"
failure_message | output "$LOG_FILE" failure_message | output "$LOG_FILE"
} }
trap 'handle_failure' ERR trap 'handle_failure' ERR
...@@ -117,7 +118,7 @@ restore_cache() { ...@@ -117,7 +118,7 @@ restore_cache() {
if [ "$cache_status" == "valid" ]; then if [ "$cache_status" == "valid" ]; then
local cache_directories=$(get_cache_directories) local cache_directories=$(get_cache_directories)
if [ "$cache_directories" == "" ]; then if [ "$cache_directories" == "" ]; then
echo "Loading 2 from cacheDirectories (default):" echo "Loading 3 from cacheDirectories (default):"
restore_cache_directories "$BUILD_DIR" "$CACHE_DIR" "$DEFAULT_CACHE" restore_cache_directories "$BUILD_DIR" "$CACHE_DIR" "$DEFAULT_CACHE"
else else
echo "Loading $(echo $cache_directories | wc -w | xargs) from cacheDirectories (package.json):" echo "Loading $(echo $cache_directories | wc -w | xargs) from cacheDirectories (package.json):"
...@@ -156,7 +157,7 @@ cache_build() { ...@@ -156,7 +157,7 @@ cache_build() {
if ! ${NODE_MODULES_CACHE:-true}; then if ! ${NODE_MODULES_CACHE:-true}; then
echo "Skipping cache save (disabled by config)" echo "Skipping cache save (disabled by config)"
elif [ "$cache_directories" == "" ]; then elif [ "$cache_directories" == "" ]; then
echo "Saving 2 cacheDirectories (default):" echo "Saving 3 cacheDirectories (default):"
save_cache_directories "$BUILD_DIR" "$CACHE_DIR" "$DEFAULT_CACHE" save_cache_directories "$BUILD_DIR" "$CACHE_DIR" "$DEFAULT_CACHE"
else else
echo "Saving $(echo $cache_directories | wc -w | xargs) cacheDirectories (package.json):" echo "Saving $(echo $cache_directories | wc -w | xargs) cacheDirectories (package.json):"
...@@ -169,7 +170,9 @@ header "Caching build" ...@@ -169,7 +170,9 @@ header "Caching build"
cache_build | output "$LOG_FILE" cache_build | output "$LOG_FILE"
summarize_build() { summarize_build() {
list_dependencies "$BUILD_DIR" if $NODE_VERBOSE; then
list_dependencies "$BUILD_DIR"
fi
} }
header "Build succeeded!" header "Build succeeded!"
......
#!/usr/bin/env bash #!/usr/bin/env bash
NPM_CONFIG_PRODUCTION=false "$(dirname ${0:-})/compile" "$1" "$2" "$3" NPM_CONFIG_PRODUCTION=false YARN_PRODUCTION=false "$(dirname ${0:-})/compile" "$1" "$2" "$3"
source $BP_DIR/lib/binaries.sh source $BP_DIR/lib/binaries.sh
create_signature() { create_signature() {
echo "$(node --version); $(npm --version)" echo "$(node --version); $(npm --version); $(yarn --version 2>/dev/null || true)"
} }
save_signature() { save_signature() {
......
...@@ -4,7 +4,7 @@ list_dependencies() { ...@@ -4,7 +4,7 @@ list_dependencies() {
cd "$build_dir" cd "$build_dir"
if $YARN; then if $YARN; then
echo "" echo ""
(yarn ls || true) 2>/dev/null (yarn list --depth=0 || true) 2>/dev/null
echo "" echo ""
else else
(npm ls --depth=0 | tail -n +2 || true) 2>/dev/null (npm ls --depth=0 | tail -n +2 || true) 2>/dev/null
...@@ -27,20 +27,17 @@ run_if_present() { ...@@ -27,20 +27,17 @@ run_if_present() {
yarn_node_modules() { yarn_node_modules() {
local build_dir=${1:-} local build_dir=${1:-}
echo "Installing node modules (yarn.lock)"
echo "Installing node modules (yarn)"
cd "$build_dir" cd "$build_dir"
yarn install --pure-lockfile --ignore-engines --cache-folder $build_dir/.cache/yarn 2>&1
# according to docs: "Verifies that versions of the package dependencies in the current project’s package.json matches that of yarn’s lock file." # according to docs: "Verifies that versions of the package dependencies in the current project’s package.json matches that of yarn’s lock file."
# however, appears to also check for the presence of deps in node_modules # however, appears to also check for the presence of deps in node_modules, so must be run after install
# yarn check 1>/dev/null if $(yarn check 1>/dev/null); then
if [ "$NODE_ENV" == "production" ] && [ "$NPM_CONFIG_PRODUCTION" == "false" ]; then echo "yarn.lock and package.json match"
echo "" else
echo "Warning: when NODE_ENV=production, yarn will NOT install any devDependencies" error "yarn.lock is outdated. run \`yarn install\`, commit the updated \`yarn.lock\`, and redeploy"
echo " (even if NPM_CONFIG_PRODUCTION is false)" return 1
echo " https://yarnpkg.com/en/docs/cli/install#toc-yarn-install-production"
echo ""
fi fi
yarn install --pure-lockfile --ignore-engines 2>&1
} }
npm_node_modules() { npm_node_modules() {
...@@ -54,7 +51,7 @@ npm_node_modules() { ...@@ -54,7 +51,7 @@ npm_node_modules() {
else else
echo "Installing node modules (package.json)" echo "Installing node modules (package.json)"
fi fi
npm install --unsafe-perm --userconfig $build_dir/.npmrc 2>&1 npm install --unsafe-perm --userconfig $build_dir/.npmrc --cache $build_dir/.npm 2>&1
else else
echo "Skipping (no package.json)" echo "Skipping (no package.json)"
fi fi
......
...@@ -3,11 +3,13 @@ create_default_env() { ...@@ -3,11 +3,13 @@ create_default_env() {
export NPM_CONFIG_LOGLEVEL=${NPM_CONFIG_LOGLEVEL:-error} export NPM_CONFIG_LOGLEVEL=${NPM_CONFIG_LOGLEVEL:-error}
export NODE_MODULES_CACHE=${NODE_MODULES_CACHE:-true} export NODE_MODULES_CACHE=${NODE_MODULES_CACHE:-true}
export NODE_ENV=${NODE_ENV:-production} export NODE_ENV=${NODE_ENV:-production}
export NODE_VERBOSE=${NODE_VERBOSE:-false}
} }
list_node_config() { list_node_config() {
echo "" echo ""
printenv | grep ^NPM_CONFIG_ || true printenv | grep ^NPM_CONFIG_ || true
printenv | grep ^YARN_ || true
printenv | grep ^NODE_ || true printenv | grep ^NODE_ || true
if [ "$NPM_CONFIG_PRODUCTION" = "true" ] && [ "$NODE_ENV" != "production" ]; then if [ "$NPM_CONFIG_PRODUCTION" = "true" ] && [ "$NODE_ENV" != "production" ]; then
......
...@@ -76,6 +76,12 @@ warn_old_npm() { ...@@ -76,6 +76,12 @@ warn_old_npm() {
fi fi
} }
warn_young_yarn() {
if $YARN; then
warning "This project was built with yarn, which is new and under development. Some projects can still be built more reliably with npm" "https://devcenter.heroku.com/articles/nodejs-support#build-behavior"
fi
}
warn_untracked_dependencies() { warn_untracked_dependencies() {
local log_file="$1" local log_file="$1"
if grep -qi 'gulp: not found' "$log_file" || grep -qi 'gulp: command not found' "$log_file"; then if grep -qi 'gulp: not found' "$log_file" || grep -qi 'gulp: command not found' "$log_file"; then
......
#!/usr/bin/env bash #!/usr/bin/env bash
# See README.md for info on running these tests. # See README.md for info on running these tests.
testNodeModulesCached() {
cache=$(mktmpdir)
compile "caching" $cache
assertCaptured "Saving 3 cacheDirectories (default)"
assertCaptured "- .npm"
assertCaptured "- .cache/yarn (nothing to cache)"
assertCaptured "- bower_components (nothing to cache)"
assertEquals "1" "$(ls -1 $cache/node/.npm | grep express | wc -l | tr -d ' ')"
assertCapturedSuccess
}
testYarn() {
compile "yarn"
assertCaptured "installing yarn"
assertCaptured "Installing node modules (yarn.lock)"
assertNotCaptured "Installing node modules (package.json"
assertCapturedSuccess
}
testBuildWithCache() {
cache=$(mktmpdir)
compile "stable-node" $cache
assertCaptured "Skipping cache restore (new runtime"
assertEquals "1" "$(ls -1 $cache/node/.npm | grep hashish | wc -l | tr -d ' ')"
assertCapturedSuccess
compile "stable-node" $cache
assertNotCaptured "- .npm (not cached - skipping)"
assertCapturedSuccess
rm -rf "$cache/node/.npm"
compile "stable-node" $cache
assertCaptured "- .npm (not cached - skipping)"
assertCapturedSuccess
}
testYarnSemver() { testYarnSemver() {
compile "yarn-semver" compile "yarn-semver"
assertCaptured "Resolving yarn version ~0.17" assertCaptured "Resolving yarn version ~0.17"
...@@ -21,14 +59,6 @@ testYarnEngine() { ...@@ -21,14 +59,6 @@ testYarnEngine() {
assertCapturedSuccess assertCapturedSuccess
} }
testYarn() {
compile "yarn"
assertCaptured "installing yarn"
assertCaptured "Installing node modules (yarn)"
assertNotCaptured "Installing node modules (package.json"
assertCapturedSuccess
}
testWarnUnmetDep() { testWarnUnmetDep() {
compile "unmet-dep" compile "unmet-dep"
assertCaptured "may cause runtime issues" assertCaptured "may cause runtime issues"
...@@ -116,27 +146,6 @@ testMultipleRuns() { ...@@ -116,27 +146,6 @@ testMultipleRuns() {
assertCapturedSuccess assertCapturedSuccess
} }
testDisableCache() {
cache=$(mktmpdir)
env_dir=$(mktmpdir)
compile "node-modules-cache-1" $cache
assertCaptured "lodash@1.0.0"
assertEquals "1" "$(ls -1 $cache/node/node_modules | grep lodash | wc -l | tr -d ' ')"
assertCapturedSuccess
compile "node-modules-cache-2" $cache
assertCaptured "lodash@1.0.0"
assertCaptured "Saving 2 cacheDirectories"
assertCapturedSuccess
echo "false" > $env_dir/NODE_MODULES_CACHE
compile "node-modules-cache-2" $cache $env_dir
assertCaptured "lodash@1.3.1"
assertNotCaptured "Saving 2 cacheDirectories"
assertCapturedSuccess
}
testBowerAngularResolution() { testBowerAngularResolution() {
compile "bower-angular-resolution" compile "bower-angular-resolution"
assertCaptured "Bower may need a resolution hint for angular" assertCaptured "Bower may need a resolution hint for angular"
...@@ -157,17 +166,6 @@ testBadJson() { ...@@ -157,17 +166,6 @@ testBadJson() {
assertCapturedError 1 "Unable to parse" assertCapturedError 1 "Unable to parse"
} }
testNodeModulesCached() {
cache=$(mktmpdir)
compile "caching" $cache
assertCaptured "Saving 2 cacheDirectories (default)"
assertCaptured "- node_modules"
assertCaptured "- bower_components (nothing to cache)"
assertEquals "1" "$(ls -1 $cache/node | grep node_modules | wc -l | tr -d ' ')"
assertCapturedSuccess
}
testBuildWithUserCacheDirectoriesCamel() { testBuildWithUserCacheDirectoriesCamel() {
cache=$(mktmpdir) cache=$(mktmpdir)
...@@ -234,24 +232,6 @@ testInvalidIo() { ...@@ -234,24 +232,6 @@ testInvalidIo() {
assertCapturedError assertCapturedError
} }
testBuildWithCache() {
cache=$(mktmpdir)
compile "stable-node" $cache
assertCaptured "Skipping cache restore (new runtime"
assertEquals "1" "$(ls -1 $cache/node | grep node_modules | wc -l | tr -d ' ')"
assertCapturedSuccess
compile "stable-node" $cache
assertNotCaptured "- node_modules (not cached - skipping)"
assertCapturedSuccess
rm -rf "$cache/node/node_modules"
compile "stable-node" $cache
assertCaptured "- node_modules (not cached - skipping)"
assertCapturedSuccess
}
testSignatureInvalidation() { testSignatureInvalidation() {
cache=$(mktmpdir) cache=$(mktmpdir)
env_dir=$(mktmpdir) env_dir=$(mktmpdir)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment