Add hatchet tests to be called from nodebin service (#557)

These changes move the hatchet ci specs from specs/ to specs/ci/ and creates a separate directory with some new specs that mimic the ones for CI, but are designed so that you can specify the Node version via ENV var so that they can be called from nodebin to test new Node releases before making them public

etc/hatchet.sh

@@ -17,7 +17,11 @@ if [ "$TRAVIS" == "true" ] && [ "$TRAVIS_PULL_REQUEST" != "false" ]; then
 fi
 
 if [ -z "$HEROKU_API_KEY" ]; then
+  echo ""
   echo "ERROR: Missing \$HEROKU_API_KEY."
+  echo ""
+  echo "NOTE: You can create token this by running: heroku authorizations:create --description \"For Travis\""
+  echo ""
   exit 1
 fi
 

makefile

@@ -16,9 +16,15 @@ cedar-14:
 	@echo ""
 
 hatchet:
-	@echo "Running hatchet integration tests"
-	bash etc/ci-setup.sh
-	bash etc/hatchet.sh
+	@echo "Running hatchet integration tests..."
+	@bash etc/ci-setup.sh
+	@bash etc/hatchet.sh spec/ci/
+	@echo ""
+
+nodebin-test:
+	@echo "Running test for Node v${TEST_NODE_VERSION}..."
+	@bash etc/ci-setup.sh
+	@bash etc/hatchet.sh spec/nodebin/
 	@echo ""
 
 shell:

spec/node_10_metrics_spec.rb → spec/ci/node_10_metrics_spec.rb

-require_relative 'spec_helper'
+require_relative '../spec_helper'
 
 describe "Node Metrics for v10.x" do
   context "test metrics for Node v10.x app" do

spec/node_10_spec.rb → spec/ci/node_10_spec.rb

-require_relative 'spec_helper'
+require_relative '../spec_helper'
 
 describe "Hello World for Node v10.x" do
   context "a single-process Node v10.x app" do

spec/node_6_spec.rb → spec/ci/node_6_spec.rb

-require_relative 'spec_helper'
+require_relative '../spec_helper'
 
 describe "Hello World for Node v6.x" do
   context "a single-process Node v6.x app" do

spec/node_8_metrics_spec.rb → spec/ci/node_8_metrics_spec.rb

-require_relative 'spec_helper'
+require_relative '../spec_helper'
 
 describe "Node Metrics for v8.x" do
   context "test metrics for Node v8.x app" do

spec/node_8_spec.rb → spec/ci/node_8_spec.rb

-require_relative 'spec_helper'
+require_relative '../spec_helper'
 
 describe "Hello World for Node v8.x" do
   context "a single-process Node v8.x app" do

spec/node_9_metrics_spec.rb → spec/ci/node_9_metrics_spec.rb

-require_relative 'spec_helper'
+require_relative '../spec_helper'
 
 describe "Node Metrics for v9.x" do
   context "test metrics for Node v9.x app" do

spec/node_9_spec.rb → spec/ci/node_9_spec.rb

-require_relative 'spec_helper'
+require_relative '../spec_helper'
 
 describe "Hello World for Node v9.x" do
   context "a single-process Node v9.x app" do

spec/nodebin/node_vX_metrics_spec.rb

+require_relative '../spec_helper'
+
+versions = get_test_versions
+
+versions.select { |version| version_supports_metrics(version) }.each do |version|
+  describe "Node Metrics for v#{version}" do
+    context "test metrics for Node v#{version} app" do
+      let(:app) {
+        Hatchet::Runner.new(
+          "spec/fixtures/repos/node-10-metrics",
+          before_deploy: -> { set_node_version(version) },
+          config: {
+            "HEROKU_METRICS_URL" => "http://localhost:3000",
+            "METRICS_INTERVAL_OVERRIDE" => "10000"
+          }
+        )
+      }
+
+      it "should deploy" do
+        app.deploy do |app|
+          expect(app.output).to include("Downloading and installing node #{version}...")
+          data = successful_json_body(app)
+          expect(data["gauges"]["node.eventloop.delay.ms.max"]).to be >= 2000
+          expect(data["counters"]["node.gc.collections"]).to be >= 0
+          expect(data["counters"]["node.gc.young.collections"]).to be >= 0
+          expect(data["counters"]["node.gc.old.collections"]).to be >= 0
+        end
+      end
+    end
+  end
+end

spec/nodebin/node_vX_spec.rb

+require_relative '../spec_helper'
+
+versions = get_test_versions
+
+versions.each do |version|
+  describe "Hello World for Node v#{version}" do
+    context "a single-process Node v#{version} app" do
+      let(:app) {
+        Hatchet::Runner.new(
+          "spec/fixtures/repos/node-10",
+          before_deploy: -> { set_node_version(version) }
+        )
+      }
+
+      it "should deploy successfully" do
+        app.deploy do |app|
+          expect(app.output).to include("Downloading and installing node #{version}...")
+          expect(successful_body(app).strip).to eq("Hello, world!")
+        end
+      end
+    end
+  end
+end

spec/spec_helper.rb

@@ -32,3 +32,47 @@ def successful_json_body(app, options = {})
   body = successful_body(app, options)
   JSON.parse(body)
 end
+
+def set_node_version(version)
+  package_json = File.read('package.json')
+  package = JSON.parse(package_json)
+  package["engines"]["node"] = version
+  File.open('package.json', 'w') do |f|
+    f.puts JSON.dump(package)
+  end
+end
+
+def resolve_node_version(requirements, options = {})
+  # use nodebin to get latest node versions
+  requirements.map do |requirement|
+    retry_limit = options[:retry_limit] || 50
+    body = Excon.get("https://nodebin.herokai.com/v1/node/linux-x64/latest?range=#{requirement}", :idempotent => true, :expects => 200, :retry_limit => retry_limit).body
+    JSON.parse(body)['number']
+  end
+end
+
+def resolve_all_supported_node_versions(options = {})
+  retry_limit = options[:retry_limit] || 50 
+  body = Excon.get("https://nodebin.herokai.com/v1/node/linux-x64/", :idempotent => true, :expects => 200, :retry_limit => retry_limit).body
+  list = JSON.parse(body).map { |n| n['number'] }
+
+  list.select do |n|
+    SemVersion.new(n).satisfies?('>= 6.0.0')
+  end
+end
+
+def version_supports_metrics(version)
+  SemVersion.new(version).satisfies?('>= 8.0.0')
+end
+
+def get_test_versions
+  if ENV['TEST_NODE_VERSION']
+    versions = [ENV['TEST_NODE_VERSION']]
+  elsif ENV['TEST_ALL_NODE_VERSIONS'] == 'true'
+    versions = resolve_all_supported_node_versions()
+  else
+    versions = resolve_node_version(['6.x', '8.x', '9.x', '10.x'])
+  end
+  puts("Running tests for Node versions: #{versions.join(', ')}")
+  versions
+end