better caching for yarn

CHANGELOG.md

 # Node.js Buildpack Changelog
 
+## Master
+
+- Use cache directories instead of node_modules
+- Use yarn, if available, as part of the cache signature
+- Warn about yarn's youth and evolution on build failures
+- Link to opt-out of yarn instructions
+- Use `yarn list` instead of `yarn ls`
+- Hide final dep tree listings under a `NODE_VERBOSE` flag
+
 ## v94 (2016-12-16)
 
 - Warn on yarn NODE_ENV and NPM_CONFIG incompatibility

bin/compile

@@ -10,7 +10,7 @@ unset GIT_DIR     # Avoid GIT_DIR leak from previous build steps
 
 ### Constants
 
-DEFAULT_CACHE="node_modules bower_components"
+DEFAULT_CACHE=".npm .cache/yarn bower_components"
 
 ### Configure directories
 
@@ -44,6 +44,7 @@ handle_failure() {
   warn_angular_resolution "$LOG_FILE"
   warn_missing_devdeps "$LOG_FILE"
   warn_econnreset "$LOG_FILE"
+  warn_young_yarn "$LOG_FILE"
   failure_message | output "$LOG_FILE"
 }
 trap 'handle_failure' ERR
@@ -117,7 +118,7 @@ restore_cache() {
   if [ "$cache_status" == "valid" ]; then
     local cache_directories=$(get_cache_directories)
     if [ "$cache_directories" == "" ]; then
-      echo "Loading 2 from cacheDirectories (default):"
+      echo "Loading 3 from cacheDirectories (default):"
       restore_cache_directories "$BUILD_DIR" "$CACHE_DIR" "$DEFAULT_CACHE"
     else
       echo "Loading $(echo $cache_directories | wc -w | xargs) from cacheDirectories (package.json):"
@@ -156,7 +157,7 @@ cache_build() {
   if ! ${NODE_MODULES_CACHE:-true}; then
     echo "Skipping cache save (disabled by config)"
   elif [ "$cache_directories" == "" ]; then
-    echo "Saving 2 cacheDirectories (default):"
+    echo "Saving 3 cacheDirectories (default):"
     save_cache_directories "$BUILD_DIR" "$CACHE_DIR" "$DEFAULT_CACHE"
   else
     echo "Saving $(echo $cache_directories | wc -w | xargs) cacheDirectories (package.json):"
@@ -169,7 +170,9 @@ header "Caching build"
 cache_build | output "$LOG_FILE"
 
 summarize_build() {
-  list_dependencies "$BUILD_DIR"
+  if $NODE_VERBOSE; then
+    list_dependencies "$BUILD_DIR"
+  fi
 }
 
 header "Build succeeded!"

bin/test-compile

 #!/usr/bin/env bash
 
-NPM_CONFIG_PRODUCTION=false "$(dirname ${0:-})/compile" "$1" "$2" "$3"
+NPM_CONFIG_PRODUCTION=false YARN_PRODUCTION=false "$(dirname ${0:-})/compile" "$1" "$2" "$3"

lib/cache.sh

 source $BP_DIR/lib/binaries.sh
 
 create_signature() {
-  echo "$(node --version); $(npm --version)"
+  echo "$(node --version); $(npm --version); $(yarn --version 2>/dev/null || true)"
 }
 
 save_signature() {

lib/dependencies.sh

@@ -4,7 +4,7 @@ list_dependencies() {
   cd "$build_dir"
   if $YARN; then
     echo ""
-    (yarn ls || true) 2>/dev/null
+    (yarn list --depth=0 || true) 2>/dev/null
     echo ""
   else
     (npm ls --depth=0 | tail -n +2 || true) 2>/dev/null
@@ -27,20 +27,17 @@ run_if_present() {
 
 yarn_node_modules() {
   local build_dir=${1:-}
-
-  echo "Installing node modules (yarn)"
+  echo "Installing node modules (yarn.lock)"
   cd "$build_dir"
+  yarn install --pure-lockfile --ignore-engines --cache-folder $build_dir/.cache/yarn 2>&1
   # according to docs: "Verifies that versions of the package dependencies in the current project’s package.json matches that of yarn’s lock file."
-  # however, appears to also check for the presence of deps in node_modules
-  # yarn check 1>/dev/null
-  if [ "$NODE_ENV" == "production" ] && [ "$NPM_CONFIG_PRODUCTION" == "false" ]; then
-    echo ""
-    echo "Warning: when NODE_ENV=production, yarn will NOT install any devDependencies"
-    echo "  (even if NPM_CONFIG_PRODUCTION is false)"
-    echo "  https://yarnpkg.com/en/docs/cli/install#toc-yarn-install-production"
-    echo ""
+  # however, appears to also check for the presence of deps in node_modules, so must be run after install
+  if $(yarn check 1>/dev/null); then
+    echo "yarn.lock and package.json match"
+  else
+    error "yarn.lock is outdated. run \`yarn install\`, commit the updated \`yarn.lock\`, and redeploy"
+    return 1
   fi
-  yarn install --pure-lockfile --ignore-engines 2>&1
 }
 
 npm_node_modules() {
@@ -54,7 +51,7 @@ npm_node_modules() {
     else
       echo "Installing node modules (package.json)"
     fi
-    npm install --unsafe-perm --userconfig $build_dir/.npmrc 2>&1
+    npm install --unsafe-perm --userconfig $build_dir/.npmrc --cache $build_dir/.npm 2>&1
   else
     echo "Skipping (no package.json)"
   fi

lib/environment.sh

@@ -3,11 +3,13 @@ create_default_env() {
   export NPM_CONFIG_LOGLEVEL=${NPM_CONFIG_LOGLEVEL:-error}
   export NODE_MODULES_CACHE=${NODE_MODULES_CACHE:-true}
   export NODE_ENV=${NODE_ENV:-production}
+  export NODE_VERBOSE=${NODE_VERBOSE:-false}
 }
 
 list_node_config() {
   echo ""
   printenv | grep ^NPM_CONFIG_ || true
+  printenv | grep ^YARN_ || true
   printenv | grep ^NODE_ || true
 
   if [ "$NPM_CONFIG_PRODUCTION" = "true" ] && [ "$NODE_ENV" != "production" ]; then

lib/failure.sh

@@ -76,6 +76,12 @@ warn_old_npm() {
   fi
 }
 
+warn_young_yarn() {
+  if $YARN; then
+    warning "This project was built with yarn, which is new and under development. Some projects can still be built more reliably with npm" "https://devcenter.heroku.com/articles/nodejs-support#build-behavior"
+  fi
+}
+
 warn_untracked_dependencies() {
   local log_file="$1"
   if grep -qi 'gulp: not found' "$log_file" || grep -qi 'gulp: command not found' "$log_file"; then

test/run

 #!/usr/bin/env bash
 # See README.md for info on running these tests.
 
+testNodeModulesCached() {
+  cache=$(mktmpdir)
+
+  compile "caching" $cache
+  assertCaptured "Saving 3 cacheDirectories (default)"
+  assertCaptured "- .npm"
+  assertCaptured "- .cache/yarn (nothing to cache)"
+  assertCaptured "- bower_components (nothing to cache)"
+  assertEquals "1" "$(ls -1 $cache/node/.npm | grep express | wc -l | tr -d ' ')"
+  assertCapturedSuccess
+}
+
+testYarn() {
+  compile "yarn"
+  assertCaptured "installing yarn"
+  assertCaptured "Installing node modules (yarn.lock)"
+  assertNotCaptured "Installing node modules (package.json"
+  assertCapturedSuccess
+}
+
+testBuildWithCache() {
+  cache=$(mktmpdir)
+
+  compile "stable-node" $cache
+  assertCaptured "Skipping cache restore (new runtime"
+  assertEquals "1" "$(ls -1 $cache/node/.npm | grep hashish | wc -l | tr -d ' ')"
+  assertCapturedSuccess
+
+  compile "stable-node" $cache
+  assertNotCaptured "- .npm (not cached - skipping)"
+  assertCapturedSuccess
+
+  rm -rf "$cache/node/.npm"
+  compile "stable-node" $cache
+  assertCaptured "- .npm (not cached - skipping)"
+  assertCapturedSuccess
+}
+
 testYarnSemver() {
   compile "yarn-semver"
   assertCaptured "Resolving yarn version ~0.17"
@@ -21,14 +59,6 @@ testYarnEngine() {
   assertCapturedSuccess
 }
 
-testYarn() {
-  compile "yarn"
-  assertCaptured "installing yarn"
-  assertCaptured "Installing node modules (yarn)"
-  assertNotCaptured "Installing node modules (package.json"
-  assertCapturedSuccess
-}
-
 testWarnUnmetDep() {
   compile "unmet-dep"
   assertCaptured "may cause runtime issues"
@@ -116,27 +146,6 @@ testMultipleRuns() {
   assertCapturedSuccess
 }
 
-testDisableCache() {
-  cache=$(mktmpdir)
-  env_dir=$(mktmpdir)
-
-  compile "node-modules-cache-1" $cache
-  assertCaptured "lodash@1.0.0"
-  assertEquals "1" "$(ls -1 $cache/node/node_modules | grep lodash | wc -l | tr -d ' ')"
-  assertCapturedSuccess
-
-  compile "node-modules-cache-2" $cache
-  assertCaptured "lodash@1.0.0"
-  assertCaptured "Saving 2 cacheDirectories"
-  assertCapturedSuccess
-
-  echo "false" > $env_dir/NODE_MODULES_CACHE
-  compile "node-modules-cache-2" $cache $env_dir
-  assertCaptured "lodash@1.3.1"
-  assertNotCaptured "Saving 2 cacheDirectories"
-  assertCapturedSuccess
-}
-
 testBowerAngularResolution() {
   compile "bower-angular-resolution"
   assertCaptured "Bower may need a resolution hint for angular"
@@ -157,17 +166,6 @@ testBadJson() {
   assertCapturedError 1 "Unable to parse"
 }
 
-testNodeModulesCached() {
-  cache=$(mktmpdir)
-
-  compile "caching" $cache
-  assertCaptured "Saving 2 cacheDirectories (default)"
-  assertCaptured "- node_modules"
-  assertCaptured "- bower_components (nothing to cache)"
-  assertEquals "1" "$(ls -1 $cache/node | grep node_modules | wc -l | tr -d ' ')"
-  assertCapturedSuccess
-}
-
 testBuildWithUserCacheDirectoriesCamel() {
   cache=$(mktmpdir)
 
@@ -234,24 +232,6 @@ testInvalidIo() {
   assertCapturedError
 }
 
-testBuildWithCache() {
-  cache=$(mktmpdir)
-
-  compile "stable-node" $cache
-  assertCaptured "Skipping cache restore (new runtime"
-  assertEquals "1" "$(ls -1 $cache/node | grep node_modules | wc -l | tr -d ' ')"
-  assertCapturedSuccess
-
-  compile "stable-node" $cache
-  assertNotCaptured "- node_modules (not cached - skipping)"
-  assertCapturedSuccess
-
-  rm -rf "$cache/node/node_modules"
-  compile "stable-node" $cache
-  assertCaptured "- node_modules (not cached - skipping)"
-  assertCapturedSuccess
-}
-
 testSignatureInvalidation() {
   cache=$(mktmpdir)
   env_dir=$(mktmpdir)