Rename build-data module to metadata and update the callsites (#629)

bin/compile

@@ -50,8 +50,8 @@ source "$BP_DIR/lib/uuid.sh"
 source "$BP_DIR/lib/kvstore.sh"
 # shellcheck source=lib/metadata.sh
 source "$BP_DIR/lib/metadata.sh"
-# shellcheck source=lib/build-data.sh
-source "$BP_DIR/lib/build-data.sh"
+# shellcheck source=lib/builddata.sh
+source "$BP_DIR/lib/builddata.sh"
 
 export PATH="$BUILD_DIR/.heroku/node/bin:$BUILD_DIR/.heroku/yarn/bin":$PATH
 
@@ -86,7 +86,7 @@ handle_failure() {
 trap 'handle_failure' ERR
 
 ### Initalize metadata store
-bd_create "$CACHE_DIR"
+meta_create "$CACHE_DIR"
 
 ### Check initial state
 
@@ -137,13 +137,13 @@ install_bins() {
   npm_engine=$(read_json "$BUILD_DIR/package.json" ".engines.npm")
   yarn_engine=$(read_json "$BUILD_DIR/package.json" ".engines.yarn")
 
-  bd_set "node-version-request" "$node_engine"
-  bd_set "npm-version-request" "$npm_engine"
-  bd_set "yarn-version-request" "$yarn_engine"
+  meta_set "node-version-request" "$node_engine"
+  meta_set "npm-version-request" "$npm_engine"
+  meta_set "yarn-version-request" "$yarn_engine"
 
-  bd_set "node-version-request" "$node_engine"
-  bd_set "npm-version-request" "$npm_engine"
-  bd_set "yarn-version-request" "$yarn_engine"
+  meta_set "node-version-request" "$node_engine"
+  meta_set "npm-version-request" "$npm_engine"
+  meta_set "yarn-version-request" "$yarn_engine"
 
   if [ -n "$iojs_engine" ]; then
     echo "engines.iojs (package.json):  $iojs_engine (iojs)"
@@ -169,7 +169,7 @@ install_bins() {
     monitor "install-npm-binary" install_npm "$npm_engine" "$BUILD_DIR/.heroku/node" $NPM_LOCK
     node_version="$(node --version)"
     mcount "version.node.$node_version"
-    bd_set "node-version" "$node_version"
+    meta_set "node-version" "$node_version"
   fi
 
   # Download yarn if there is a yarn.lock file or if the user
@@ -181,10 +181,10 @@ install_bins() {
 
   if $YARN; then
     mcount "version.yarn.$(yarn --version)"
-    bd_set "yarn-version" "$(yarn --version)"
+    meta_set "yarn-version" "$(yarn --version)"
   else
     mcount "version.npm.$(npm --version)"
-    bd_set "npm-version" "$(npm --version)"
+    meta_set "npm-version" "$(npm --version)"
   fi
 
   warn_old_npm
@@ -235,7 +235,7 @@ restore_cache() {
   fi
 
   mcount "cache.$cache_status"
-  bd_set "cache-status" "$cache_status"
+  meta_set "cache-status" "$cache_status"
 }
 
 restore_cache | output "$LOG_FILE"
@@ -305,7 +305,7 @@ summarize_build() {
   fi
 
   mmeasure 'modules.size' "$(measure_size)"
-  bd_set "node-modules-size" "$(measure_size)"
+  meta_set "node-modules-size" "$(measure_size)"
 }
 
 install_plugin "$BP_DIR" "$BUILD_DIR"
@@ -313,8 +313,8 @@ install_plugin "$BP_DIR" "$BUILD_DIR"
 header "Build succeeded!" | output "$LOG_FILE"
 mcount "compile"
 summarize_build | output "$LOG_FILE"
-bd_set "node-build-success" "true"
-bd_time "build-time" "$build_start_time"
+meta_set "node-build-success" "true"
+meta_time "build-time" "$build_start_time"
 
 warn_no_start "$BUILD_DIR"
 warn_unmet_dep "$LOG_FILE"
@@ -327,4 +327,4 @@ echo ""
 echo "       Read more: https://devcenter.heroku.com/changelog-items/1573"
 echo ""
 
-log_build_data >> "$BUILDPACK_LOG_FILE"
+log_meta_data >> "$BUILDPACK_LOG_FILE"

lib/build-data.sh

-#!/usr/bin/env bash
-
-# variable shared by this whole module
-BUILD_DATA_FILE=""
-PREVIOUS_BUILD_DATA_FILE=""
-
-bd_create() {
-  local cache_dir="$1"
-  BUILD_DATA_FILE="$cache_dir/build-data/nodejs"
-  PREVIOUS_BUILD_DATA_FILE="$cache_dir/build-data/nodejs-prev"
-
-  # if the file already exists because it's from the last build, save it
-  if [[ -f "$BUILD_DATA_FILE" ]]; then
-    cp "$BUILD_DATA_FILE" "$PREVIOUS_BUILD_DATA_FILE"
-  fi
-
-  kv_create "$BUILD_DATA_FILE"
-  # make sure this doesnt grow over time
-  kv_clear "$BUILD_DATA_FILE"
-}
-
-bd_get() {
-  kv_get "$BUILD_DATA_FILE" "$1"
-}
-
-bd_set() {
-  kv_set "$BUILD_DATA_FILE" "$1" "$2"
-}
-
-# similar to mtime from stdlib
-bd_time() {
-  local key="$1"
-  local start="$2"
-  local end="${3:-$(nowms)}"
-  local time
-  time="$(echo "${start}" "${end}" | awk '{ printf "%.3f", ($2 - $1)/1000 }')"
-  kv_set "$BUILD_DATA_FILE" "$key" "$time"
-}
-
-# similar to mtime from stdlib
-bd_time() {
-  local key="$1"
-  local start="$2"
-  local end="${3:-$(nowms)}"
-  local time
-  time="$(echo "$start" "$end" | awk '{ printf "%.3f", ($2 - $1)/1000 }')"
-  kv_set "$BUILD_DATA_FILE" "$1" "$time"
-}
-
-# Retrieve a value from a previous build if it exists
-# This is useful to give the user context about what changed if the 
-# build has failed. Ex:
-#   - changed stacks
-#   - deployed with a new major version of Node
-#   - etc
-bd_prev_get() {
-  kv_get "$PREVIOUS_BUILD_DATA_FILE" "$1"
-}
-
-log_build_data() {
-  # print all values on one line in logfmt format
-  # https://brandur.org/logfmt
-  # the echo call ensures that all values are printed on a single line
-  # shellcheck disable=SC2005 disable=SC2046
-  echo $(kv_list "$BUILD_DATA_FILE")
-}

lib/builddata.sh

+#!/usr/bin/env bash
+
+log_initial_state() {
+  if "$YARN"; then
+    meta_set "node-package-manager" "yarn"
+    meta_set "has-node-lock-file" "true"
+  else
+    meta_set "node-package-manager" "npm"
+    meta_set "has-node-lock-file" "$NPM_LOCK"
+  fi
+
+  meta_set "stack" "$STACK"
+}
+
+generate_uuids() {
+  # generate a unique id for each build
+  meta_set "build-uuid" "$(uuid)"
+
+  # propagate an app-uuid forward unless the cache is cleared
+  if [[ -n "$(meta_prev_get "app-uuid")" ]]; then
+    meta_set "app-uuid" "$(meta_prev_get "app-uuid")"
+  else
+    meta_set "app-uuid" "$(uuid)"
+  fi
+}
+
+log_build_script_opt_in() {
+  local opted_in="$1"
+  local build_dir="$2"
+  local has_build_script has_heroku_build_script
+
+  has_build_script=$(read_json "$build_dir/package.json" ".scripts.build")
+  has_heroku_build_script=$(read_json "$build_dir/package.json" ".scripts[\"heroku-postbuild\"]")
+
+  # if this app will be affected by the change
+  if [[ -z "$has_heroku_build_script" ]] && [[ -n "$has_build_script" ]]; then
+    mcount "affected-by-build-change"
+
+    if [[ "$opted_in" = "true" ]]; then
+      mcount "affected-by-build-change-opted-in"
+      meta_set "affected-but-opted-in" "true"
+    else
+      meta_set "affected-but-opted-in" "false"
+    fi
+
+  fi
+
+  if [[ "$opted_in" = true ]]; then
+    meta_set "build-script-opt-in" "true"
+  else
+    meta_set "build-script-opt-in" "false"
+  fi
+}
\ No newline at end of file

lib/cache.sh

@@ -114,13 +114,13 @@ save_default_cache_directories() {
   # bower_components
   if [[ -e "$build_dir/bower_components" ]]; then
     mcount "cache.saved-bower-components"
-    bd_set "cached-bower-components" "true"
+    meta_set "cached-bower-components" "true"
     echo "- bower_components"
     mkdir -p "$cache_dir/node/cache/bower_components"
     cp -a "$build_dir/bower_components" "$(dirname "$cache_dir/node/cache/bower_components")"
   fi
 
-  bd_set "node-custom-cache-dirs" "false"
+  meta_set "node-custom-cache-dirs" "false"
 }
 
 save_custom_cache_directories() {
@@ -142,5 +142,5 @@ save_custom_cache_directories() {
     fi
   done
 
-  bd_set "node-custom-cache-dirs" "true"
+  meta_set "node-custom-cache-dirs" "true"
 }

lib/dependencies.sh

@@ -69,15 +69,15 @@ log_build_scripts() {
   heroku_postbuild=$(read_json "$build_dir/package.json" ".scripts[\"heroku-postbuild\"]")
   postinstall=$(read_json "$build_dir/package.json" ".scripts[\"heroku-postbuild\"]")
 
-  bd_set "build-script" "$build"
-  bd_set "postinstall-script" "$postinstall"
-  bd_set "heroku-prebuild-script" "$heroku_prebuild"
-  bd_set "heroku-postbuild-script" "$heroku_prebuild"
+  meta_set "build-script" "$build"
+  meta_set "postinstall-script" "$postinstall"
+  meta_set "heroku-prebuild-script" "$heroku_prebuild"
+  meta_set "heroku-postbuild-script" "$heroku_prebuild"
 
-  bd_set "build-script" "$build"
-  bd_set "postinstall-script" "$postinstall"
-  bd_set "heroku-prebuild-script" "$heroku_prebuild"
-  bd_set "heroku-postbuild-script" "$heroku_prebuild"
+  meta_set "build-script" "$build"
+  meta_set "postinstall-script" "$postinstall"
+  meta_set "heroku-prebuild-script" "$heroku_prebuild"
+  meta_set "heroku-postbuild-script" "$heroku_prebuild"
 
   if [ -n "$build" ]; then
     mcount "scripts.build"
@@ -143,20 +143,20 @@ yarn_prune_devdependencies() {
 
   if [ "$NODE_ENV" == "test" ]; then
     echo "Skipping because NODE_ENV is 'test'"
-    bd_set "skipped-prune" "true"
+    meta_set "skipped-prune" "true"
     return 0
   elif [ "$NODE_ENV" != "production" ]; then
     echo "Skipping because NODE_ENV is not 'production'"
-    bd_set "skipped-prune" "true"
+    meta_set "skipped-prune" "true"
     return 0
   elif [ -n "$YARN_PRODUCTION" ]; then
     echo "Skipping because YARN_PRODUCTION is '$YARN_PRODUCTION'"
-    bd_set "skipped-prune" "true"
+    meta_set "skipped-prune" "true"
     return 0
   else 
     cd "$build_dir" || return
     monitor "yarn-prune" yarn install --frozen-lockfile --ignore-engines --ignore-scripts --prefer-offline 2>&1
-    bd_set "skipped-prune" "false"
+    meta_set "skipped-prune" "false"
   fi
 }
 
@@ -207,15 +207,15 @@ npm_prune_devdependencies() {
 
   if [ "$NODE_ENV" == "test" ]; then
     echo "Skipping because NODE_ENV is 'test'"
-    bd_set "skipped-prune" "true"
+    meta_set "skipped-prune" "true"
     return 0
   elif [ "$NODE_ENV" != "production" ]; then
     echo "Skipping because NODE_ENV is not 'production'"
-    bd_set "skipped-prune" "true"
+    meta_set "skipped-prune" "true"
     return 0
   elif [ -n "$NPM_CONFIG_PRODUCTION" ]; then
     echo "Skipping because NPM_CONFIG_PRODUCTION is '$NPM_CONFIG_PRODUCTION'"
-    bd_set "skipped-prune" "true"
+    meta_set "skipped-prune" "true"
     return 0
   elif [ "$npm_version" == "5.3.0" ]; then
     mcount "skip-prune-issue-npm-5.3.0"
@@ -224,7 +224,7 @@ npm_prune_devdependencies() {
     echo ""
     echo "You can silence this warning by updating to at least npm 5.7.1 in your package.json"
     echo "https://devcenter.heroku.com/articles/nodejs-support#specifying-an-npm-version"
-    bd_set "skipped-prune" "true"
+    meta_set "skipped-prune" "true"
     return 0
   elif [ "$npm_version" == "5.6.0" ] ||
        [ "$npm_version" == "5.5.1" ] ||
@@ -239,11 +239,11 @@ npm_prune_devdependencies() {
     echo ""
     echo "You can silence this warning by updating to at least npm 5.7.1 in your package.json"
     echo "https://devcenter.heroku.com/articles/nodejs-support#specifying-an-npm-version"
-    bd_set "skipped-prune" "true"
+    meta_set "skipped-prune" "true"
     return 0
   else
     cd "$build_dir" || return
     monitor "npm-prune" npm prune --userconfig "$build_dir/.npmrc" 2>&1
-    bd_set "skipped-prune" "false"
+    meta_set "skipped-prune" "false"
   fi
 }

lib/failure.sh

@@ -513,9 +513,9 @@ warn_prebuilt_modules() {
   if [ -e "$build_dir/node_modules" ]; then
     warning "node_modules checked into source control" "https://blog.heroku.com/node-habits-2016#9-only-git-the-important-bits"
     mcount 'warnings.modules.prebuilt'
-    bd_set "checked-in-node-modules" "true"
+    meta_set "checked-in-node-modules" "true"
   else
-    bd_set "checked-in-node-modules" "false"
+    meta_set "checked-in-node-modules" "false"
   fi
 }
 

lib/metadata.sh

 #!/usr/bin/env bash
 
-log_initial_state() {
-  if "$YARN"; then
-    bd_set "node-package-manager" "yarn"
-    bd_set "has-node-lock-file" "true"
-  else
-    bd_set "node-package-manager" "npm"
-    bd_set "has-node-lock-file" "$NPM_LOCK"
+# variable shared by this whole module
+BUILD_DATA_FILE=""
+PREVIOUS_BUILD_DATA_FILE=""
+
+meta_create() {
+  local cache_dir="$1"
+  BUILD_DATA_FILE="$cache_dir/build-data/nodejs"
+  PREVIOUS_BUILD_DATA_FILE="$cache_dir/build-data/nodejs-prev"
+
+  # if the file already exists because it's from the last build, save it
+  if [[ -f "$BUILD_DATA_FILE" ]]; then
+    cp "$BUILD_DATA_FILE" "$PREVIOUS_BUILD_DATA_FILE"
   fi
 
-  bd_set "stack" "$STACK"
+  kv_create "$BUILD_DATA_FILE"
+  # make sure this doesnt grow over time
+  kv_clear "$BUILD_DATA_FILE"
+}
+
+meta_get() {
+  kv_get "$BUILD_DATA_FILE" "$1"
 }
 
-generate_uuids() {
-  # generate a unique id for each build
-  bd_set "build-uuid" "$(uuid)"
+meta_set() {
+  kv_set "$BUILD_DATA_FILE" "$1" "$2"
+}
 
-  # propagate an app-uuid forward unless the cache is cleared
-  if [[ -n "$(bd_prev_get "app-uuid")" ]]; then
-    bd_set "app-uuid" "$(bd_prev_get "app-uuid")"
-  else
-    bd_set "app-uuid" "$(uuid)"
-  fi
+# similar to mtime from stdlib
+meta_time() {
+  local key="$1"
+  local start="$2"
+  local end="${3:-$(nowms)}"
+  local time
+  time="$(echo "${start}" "${end}" | awk '{ printf "%.3f", ($2 - $1)/1000 }')"
+  kv_set "$BUILD_DATA_FILE" "$key" "$time"
+}
+
+# similar to mtime from stdlib
+meta_time() {
+  local key="$1"
+  local start="$2"
+  local end="${3:-$(nowms)}"
+  local time
+  time="$(echo "$start" "$end" | awk '{ printf "%.3f", ($2 - $1)/1000 }')"
+  kv_set "$BUILD_DATA_FILE" "$1" "$time"
+}
+
+# Retrieve a value from a previous build if it exists
+# This is useful to give the user context about what changed if the 
+# build has failed. Ex:
+#   - changed stacks
+#   - deployed with a new major version of Node
+#   - etc
+meta_prev_get() {
+  kv_get "$PREVIOUS_BUILD_DATA_FILE" "$1"
+}
+
+log_meta_data() {
+  # print all values on one line in logfmt format
+  # https://brandur.org/logfmt
+  # the echo call ensures that all values are printed on a single line
+  # shellcheck disable=SC2005 disable=SC2046
+  echo $(kv_list "$BUILD_DATA_FILE")
 }

lib/monitor.sh

@@ -59,6 +59,6 @@ monitor() {
   mtime "exec.$command_name.time" "${start}"
   mmeasure "exec.$command_name.memory" "$(cat "$peak_mem_output")"
 
-  bd_time "$command_name-time" "$start"
-  bd_set "$command_name-memory" "$(cat "$peak_mem_output")"
+  meta_time "$command_name-time" "$start"
+  meta_set "$command_name-memory" "$(cat "$peak_mem_output")"
 }

test/unit

@@ -171,71 +171,71 @@ testKeyValueNoFile() {
 testBuildData() {
   local cache_dir=$(mktemp -d)
 
-  bd_create $cache_dir
+  meta_create $cache_dir
 
-  bd_set "test" "foo"
-  assertEquals "test=foo" "$(log_build_data)"
+  meta_set "test" "foo"
+  assertEquals "test=foo" "$(log_meta_data)"
 
-  bd_set "test" "different-foo"
-  assertEquals "test=different-foo" "$(log_build_data)"
+  meta_set "test" "different-foo"
+  assertEquals "test=different-foo" "$(log_meta_data)"
 
-  bd_set "foo" "value with spaces"
-  assertEquals "foo=\"value with spaces\" test=different-foo" "$(log_build_data)"
+  meta_set "foo" "value with spaces"
+  assertEquals "foo=\"value with spaces\" test=different-foo" "$(log_meta_data)"
 
   # values are printed with the keys sorted alphabetically
   # this isn't required, and this test serves as documentation
-  bd_set "a" "this should come first"
-  assertEquals "a=\"this should come first\" foo=\"value with spaces\" test=different-foo" "$(log_build_data)"
+  meta_set "a" "this should come first"
+  assertEquals "a=\"this should come first\" foo=\"value with spaces\" test=different-foo" "$(log_meta_data)"
 
   # dates generated by running `nowms; sleep 10; nowms`
-  bd_time "time" "1545178120033" "1545178130043"
-  assertEquals "10.010" "$(bd_get time)"
+  meta_time "time" "1545178120033" "1545178130043"
+  assertEquals "10.010" "$(meta_get time)"
 
   # dates generated by running `nowms; sleep 1; nowms`
-  bd_time "time" "1545178503025" "1545178504027"
-  assertEquals "1.002" "$(bd_get time)"
+  meta_time "time" "1545178503025" "1545178504027"
+  assertEquals "1.002" "$(meta_get time)"
 
   # dates generated by running `nowms; sleep 30; nowms`
-  bd_time "time" "1545178521204" "1545178551206"
-  assertEquals "30.002" "$(bd_get time)"
+  meta_time "time" "1545178521204" "1545178551206"
+  assertEquals "30.002" "$(meta_get time)"
 }
 
 testBuildDataPreviousBuild() {
   local cache_dir=$(mktemp -d)
 
   # the first time, there will be no previous build file
-  bd_create "$cache_dir"
+  meta_create "$cache_dir"
   assertContains "nodejs" "$BUILD_DATA_FILE"
   assertContains "nodejs-prev" "$PREVIOUS_BUILD_DATA_FILE"
   assertFileExists "$BUILD_DATA_FILE"
 
   # set a value in the build data file
-  bd_set "test" "foo"
+  meta_set "test" "foo"
   assertFileContains "test=foo" "$BUILD_DATA_FILE"
   assertFileDoesNotExist "$PREVIOUS_BUILD_DATA_FILE"
 
-  assertEquals "$(bd_get test)" "foo"
-  assertEquals "$(bd_prev_get test)" ""
+  assertEquals "$(meta_get test)" "foo"
+  assertEquals "$(meta_prev_get test)" ""
 
   # the second time this is called (cache restored)
   # there will be a previous build file
-  bd_create "$cache_dir"
+  meta_create "$cache_dir"
   assertFileExists "$BUILD_DATA_FILE"
   assertFileExists "$PREVIOUS_BUILD_DATA_FILE"
 
   # the data stored in the previous build should now be in the second file
   assertFileNotContains "test=foo" "$BUILD_DATA_FILE"
   assertFileContains "test=foo" "$PREVIOUS_BUILD_DATA_FILE"
-  assertEquals "$(bd_get test)" ""
-  assertEquals "$(bd_prev_get test)" "foo"
-  bd_set "test" "bar"
+  assertEquals "$(meta_get test)" ""
+  assertEquals "$(meta_prev_get test)" "foo"
+  meta_set "test" "bar"
 
   # doing it once more does not result in an error
-  bd_create "$cache_dir"
+  meta_create "$cache_dir"
   assertFileExists "$BUILD_DATA_FILE"
   assertFileExists "$PREVIOUS_BUILD_DATA_FILE" 
-  assertEquals "$(bd_prev_get test)" "bar"
-  assertEquals "$(bd_get test)" ""
+  assertEquals "$(meta_prev_get test)" "bar"
+  assertEquals "$(meta_get test)" ""
 }
 
 testWebConcurrencyProfileScript() {
@@ -371,8 +371,8 @@ source "$(pwd)"/lib/json.sh
 source "$(pwd)"/lib/monitor.sh
 source "$(pwd)"/lib/output.sh
 source "$(pwd)"/lib/kvstore.sh
-source "$(pwd)"/lib/build-data.sh
 source "$(pwd)"/lib/experiments.sh
+source "$(pwd)"/lib/metadata.sh
 source "$(pwd)"/profile/WEB_CONCURRENCY.sh
 
 # testing utils