Make sure that failing builds in CI actually fail (#613)

Make sure that failing builds in CI actually fail

bin/test-compile

 #!/usr/bin/env bash
+# bin/test-compile <build-dir> <cache-dir> <env-dir>
+
+### Configure environment
+
+set -o errexit    # always exit on error
+set -o pipefail   # don't ignore exit codes when piping output
+
+### Configure directories
 
 BP_DIR=$(cd "$(dirname "${0:-}")" || exit; cd ..; pwd)
 
+### Load dependencies
 # shellcheck source=lib/environment.sh
 source "$BP_DIR/lib/environment.sh"
 
+### Set up test Node environment
+
 export NPM_CONFIG_PRODUCTION=${NPM_CONFIG_PRODUCTION:-false}
 export NODE_ENV=${NODE_ENV:-test}
-"$BP_DIR/bin/compile" "$1" "$2" "$3"
 
+### Compile the app
+
+"$BP_DIR/bin/compile" "$1" "$2" "$3"
 write_ci_profile "$BP_DIR" "$1"

test/fixtures/failing-build/package.json

@@ -10,9 +10,9 @@
     "hashish": "*"
   },
   "engines": {
-    "node": "0.10.38"
+    "node": "8.x"
   },
   "scripts": {
-    "postinstall": "exit 1"
+    "heroku-postbuild": "exit 1"
   }
 }

test/run

@@ -562,11 +562,6 @@ testOldNpm() {
   assertCapturedError
 }
 
-testOldNpm2() {
-  compile "failing-build"
-  assertCaptured "This version of npm (1.4.28) has several known issues"
-}
-
 testNonexistentNpm() {
   compile "nonexistent-npm"
   assertCaptured "Unable to install npm 1.1.65"
@@ -874,6 +869,12 @@ testCIEnvVars() {
   assertCapturedSuccess
 }
 
+# If compile fails, test-compile should also fail
+testCICompileFails() {
+  testCompile "failing-build"
+  assertCapturedError
+}
+
 testCIEnvVarsOverride() {
   env_dir=$(mktmpdir)
   echo "banana" > $env_dir/NODE_ENV
@@ -1143,6 +1144,15 @@ compile() {
   capture ${bp_dir}/bin/compile ${compile_dir} ${2:-$(mktmpdir)} $3
 }
 
+testCompile() {
+  default_process_types_cleanup
+  bp_dir=$(mktmpdir)
+  compile_dir=$(mktmpdir)
+  cp -a "$(pwd)"/* ${bp_dir}
+  cp -a ${bp_dir}/test/fixtures/$1/. ${compile_dir}
+  capture ${bp_dir}/bin/test-compile ${compile_dir} ${2:-$(mktmpdir)} $3
+}
+
 # This is meant to be run after `compile`. `cleanupStartup` must be run
 # after this function is called before other tests are executed
 executeStartup() {