Merge pull request #63 from heroku/proxy_resolver

Fixes #44. Respect DNS TTL for proxies.

scripts/config/lib/nginx_config.rb

@@ -8,7 +8,8 @@ class NginxConfig
     encoding: "UTF-8",
     clean_urls: false,
     https_only: false,
-    worker_connections: 512
+    worker_connections: 512,
+    resolver: "8.8.8.8"
   }
 
   def initialize(json_file)
@@ -18,20 +19,23 @@ class NginxConfig
     json["port"] ||= ENV["PORT"] || 5000
     json["root"] ||= DEFAULT[:root]
     json["encoding"] ||= DEFAULT[:encoding]
+
+    index = 0
     json["proxies"] ||= {}
     json["proxies"].each do |loc, hash|
       evaled_origin = NginxConfigUtil.interpolate(hash['origin'], ENV)
-      if evaled_origin != "/"
-        json["proxies"][loc].merge!("origin" => evaled_origin + "/")
-      end
+      uri           = URI(evaled_origin)
 
-      uri = URI(evaled_origin)
-      json["proxies"][loc]["path"] = uri.path
-      uri.path = ""
-      json["proxies"][loc]["host"] = uri.to_s
-      redirect_scheme = uri.scheme == "https" ? "http" : "https"
-      json["proxies"][loc]["redirect"] = uri.dup.tap {|u| u.scheme = redirect_scheme }.to_s
-      json["proxies"][loc]["redirect"] += "/" if !uri.to_s.end_with?("/")
+      json["proxies"][loc]["name"] = "upstream_endpoint_#{index}"
+      cleaned_path = uri.path
+      cleaned_path.chop! if cleaned_path.end_with?("/")
+      json["proxies"][loc]["path"] = cleaned_path
+      json["proxies"][loc]["host"] = uri.dup.tap {|u| u.path = '' }.to_s
+      %w(http https).each do |scheme|
+        json["proxies"][loc]["redirect_#{scheme}"] = uri.dup.tap {|u| u.scheme = scheme }.to_s
+        json["proxies"][loc]["redirect_#{scheme}"] += "/" if !uri.to_s.end_with?("/")
+      end
+      index += 1
     end
 
     json["clean_urls"] ||= DEFAULT[:clean_urls]
@@ -47,6 +51,17 @@ class NginxConfig
 
     json["error_page"] ||= nil
     json["debug"] ||= ENV['STATIC_DEBUG']
+
+    nameservers = []
+    if File.exist?("/etc/resolv.conf")
+      File.open("/etc/resolv.conf", "r").each do |line|
+        next unless md = line.match(/^nameserver\s*(\S*)/)
+        nameservers << md[1]
+      end
+    end
+    nameservers << [DEFAULT[:resolver]] unless nameservers.empty?
+    json["resolver"] = nameservers.join(" ")
+
     json.each do |key, value|
       self.class.send(:define_method, key) { value }
     end

scripts/config/lib/nginx_config_util.rb

@@ -28,11 +28,16 @@ module NginxConfigUtil
   def self.match_proxies(proxies, uri)
     return false unless proxies
 
-    proxies.each do |proxy|
-      return proxy if Regexp.compile("^#{proxy}") =~ uri
+    matched = proxies.select do |proxy|
+      Regexp.compile("^#{proxy}") =~ uri
     end
 
-    false
+    # return the longest matched proxy
+    if matched.any?
+      matched.max_by {|proxy| proxy.size }
+    else
+      false
+    end
   end
 
   def self.match_redirects(redirects, uri)

scripts/config/templates/nginx.conf.erb

@@ -41,6 +41,9 @@ http {
   <% if error_page %>
     error_page 404 500 /<%= error_page %>;
   <% end %>
+  <% if proxies.any? %>
+    resolver <%= resolver %>;
+  <% end %>
 
     location / {
       mruby_post_read_handler /app/bin/config/lib/ngx_mruby/headers.rb cache;
@@ -78,11 +81,15 @@ http {
   <% end %>
 
   <% proxies.each do |location, hash| %>
+    set $<%= hash['name'] %> <%= hash['host'] %>;
     location <%= location %> {
-      proxy_pass <%= hash['origin'] %>;
+      rewrite ^<%= location %>/?(.*) <%= hash['path'] %>/$1 break;
+      proxy_pass $<%= hash['name'] %>;
       proxy_ssl_server_name on;
-      proxy_redirect default;
-      proxy_redirect <%= hash["redirect"] %>  <%= location %>;
+      # handle Location rewrites from the proxy properly
+      <% %w(http https).each do |scheme| %>
+      proxy_redirect <%= hash["redirect_#{scheme}"] %> <%= location %>;
+      <% end %>
     }
   <% end %>
 
@@ -94,11 +101,13 @@ http {
   # fallback proxy named match
   <% proxies.each do |location, hash| %>
     location @<%= location %> {
-      rewrite ^<%= location %>(.*)$ <%= hash['path'] %>/$1 break;
-      proxy_pass <%= hash['host'] %>;
+      rewrite ^<%= location %>/?(.*)$ <%= hash['path'] %>/$1 break;
+      # can reuse variable set above
+      proxy_pass $<%= hash['name'] %>;
       proxy_ssl_server_name on;
-      proxy_redirect default;
-      proxy_redirect <%= hash["redirect"] %>  <%= location %>;
+      <% %w(http https).each do |scheme| %>
+      proxy_redirect <%= hash["redirect_#{scheme}"] %> <%= location %>;
+      <% end %>
     }
   <% end %>
 

spec/simple_spec.rb

@@ -294,6 +294,321 @@ STATIC_JSON
         expect(response.body.chomp).to eq("api")
       end
     end
+
+    context "proxy to a pathed URI" do
+      let(:proxy) do
+        <<PROXY
+get "/foo/hello" do
+  "hello"
+end
+
+get "/foo/http_redirect/" do
+  uri = URI("http://\#{request.host}/foo/redirect")
+  redirect URI(uri), 307
+end
+
+get "/foo/https_redirect/" do
+  uri = URI("https://\#{request.host}/foo/redirect")
+  redirect URI(uri), 307
+end
+PROXY
+      end
+
+      let(:proxy_scheme) { "http" }
+      let(:setup_static_json) do
+        Proc.new do |path|
+          File.open(static_json_path, "w") do |file|
+            file.puts <<STATIC_JSON
+{
+  "proxies": {
+    "/api/": {
+      "origin": "#{proxy_scheme}://#{@proxy_ip_address}#{path}"
+    },
+    "/api_no_slash": {
+      "origin": "#{proxy_scheme}://#{@proxy_ip_address}#{path}"
+    },
+    "/api_origin_no_slash/": {
+      "origin": "#{proxy_scheme}://#{@proxy_ip_address}#{path.chop}"
+    },
+    "/api_no_slash_origin_no_slash": {
+      "origin": "#{proxy_scheme}://#{@proxy_ip_address}#{path.chop}"
+    }
+  }
+}
+STATIC_JSON
+          end
+        end
+      end
+
+      before do
+        setup_static_json.call("/foo/")
+      end
+
+      it "proxies properly" do
+        response = app.get("/api/hello")
+        expect(response.code).to eq("200")
+        expect(response.body.chomp).to eq("hello")
+
+        response = app.get("/api_no_slash/hello")
+        expect(response.code).to eq("200")
+        expect(response.body.chomp).to eq("hello")
+
+        response = app.get("/api_origin_no_slash/hello")
+        expect(response.code).to eq("200")
+        expect(response.body.chomp).to eq("hello")
+
+        response = app.get("/api_no_slash_origin_no_slash/hello")
+        expect(response.code).to eq("200")
+        expect(response.body.chomp).to eq("hello")
+      end
+
+      it "should handle redirects regardless of scheme" do
+        app.run do
+          response = app.get("/api/http_redirect/")
+          expect(response.code).to eq("307")
+          expect(response["Location"]).not_to include(@proxy_ip_address)
+
+          response = app.get("/api/https_redirect/")
+          expect(response.code).to eq("307")
+          expect(response["Location"]).not_to include(@proxy_ip_address)
+
+          response = app.get("/api_no_slash/http_redirect/")
+          expect(response.code).to eq("307")
+          expect(response["Location"]).not_to include(@proxy_ip_address)
+
+          response = app.get("/api_no_slash/https_redirect/")
+          expect(response.code).to eq("307")
+          expect(response["Location"]).not_to include(@proxy_ip_address)
+
+          response = app.get("/api_origin_no_slash/http_redirect/")
+          expect(response.code).to eq("307")
+          expect(response["Location"]).not_to include(@proxy_ip_address)
+
+          response = app.get("/api_origin_no_slash/https_redirect/")
+          expect(response.code).to eq("307")
+          expect(response["Location"]).not_to include(@proxy_ip_address)
+
+          response = app.get("/api_no_slash_origin_no_slash/http_redirect/")
+          expect(response.code).to eq("307")
+          expect(response["Location"]).not_to include(@proxy_ip_address)
+
+          response = app.get("/api_no_slash_origin_no_slash/https_redirect/")
+          expect(response.code).to eq("307")
+          expect(response["Location"]).not_to include(@proxy_ip_address)
+        end
+      end
+    end
+
+    context "proxy to a domain" do
+      let(:proxy) do
+          <<PROXY
+get "/hello" do
+  "hello"
+end
+
+get "/http_redirect/" do
+  uri = URI("http://\#{request.host}/foo/redirect")
+  redirect URI(uri), 307
+end
+
+get "/https_redirect/" do
+  uri = URI("https://\#{request.host}/foo/redirect")
+  redirect URI(uri), 307
+end
+PROXY
+      end
+
+      let(:proxy_scheme) { "http" }
+      let(:setup_static_json) do
+        Proc.new do |path|
+          File.open(static_json_path, "w") do |file|
+            file.puts <<STATIC_JSON
+{
+  "proxies": {
+    "/api/": {
+      "origin": "#{proxy_scheme}://#{@proxy_ip_address}#{path}"
+    },
+    "/api_no_slash": {
+      "origin": "#{proxy_scheme}://#{@proxy_ip_address}#{path}"
+    },
+    "/api_origin_no_slash/": {
+      "origin": "#{proxy_scheme}://#{@proxy_ip_address}#{path.chop}"
+    },
+    "/api_no_slash_origin_no_slash": {
+      "origin": "#{proxy_scheme}://#{@proxy_ip_address}#{path.chop}"
+    }
+  }
+}
+STATIC_JSON
+          end
+        end
+      end
+
+      before do
+        setup_static_json.call("/")
+      end
+
+      it "proxies properly" do
+        response = app.get("/api/hello")
+        expect(response.code).to eq("200")
+        expect(response.body.chomp).to eq("hello")
+
+        response = app.get("/api_no_slash/hello")
+        expect(response.code).to eq("200")
+        expect(response.body.chomp).to eq("hello")
+
+        response = app.get("/api_origin_no_slash/hello")
+        expect(response.code).to eq("200")
+        expect(response.body.chomp).to eq("hello")
+
+        response = app.get("/api_no_slash_origin_no_slash/hello")
+        expect(response.code).to eq("200")
+        expect(response.body.chomp).to eq("hello")
+      end
+
+      it "should handle redirects regardless of scheme" do
+        app.run do
+          response = app.get("/api/http_redirect/")
+          expect(response.code).to eq("307")
+          expect(response["Location"]).not_to include(@proxy_ip_address)
+
+          response = app.get("/api/https_redirect/")
+          expect(response.code).to eq("307")
+          expect(response["Location"]).not_to include(@proxy_ip_address)
+
+          response = app.get("/api_no_slash/http_redirect/")
+          expect(response.code).to eq("307")
+          expect(response["Location"]).not_to include(@proxy_ip_address)
+
+          response = app.get("/api_no_slash/https_redirect/")
+          expect(response.code).to eq("307")
+          expect(response["Location"]).not_to include(@proxy_ip_address)
+
+          response = app.get("/api_origin_no_slash/http_redirect/")
+          expect(response.code).to eq("307")
+          expect(response["Location"]).not_to include(@proxy_ip_address)
+
+          response = app.get("/api_origin_no_slash/https_redirect/")
+          expect(response.code).to eq("307")
+          expect(response["Location"]).not_to include(@proxy_ip_address)
+
+          response = app.get("/api_no_slash_origin_no_slash/http_redirect/")
+          expect(response.code).to eq("307")
+          expect(response["Location"]).not_to include(@proxy_ip_address)
+
+          response = app.get("/api_no_slash_origin_no_slash/https_redirect/")
+          expect(response.code).to eq("307")
+          expect(response["Location"]).not_to include(@proxy_ip_address)
+        end
+      end
+    end
+
+    context "fallback" do
+      let(:proxy) do
+          <<PROXY
+get "/hello" do
+  "hello"
+end
+
+get "/http_redirect/" do
+  uri = URI("http://\#{request.host}/foo/redirect")
+  redirect URI(uri), 307
+end
+
+get "/https_redirect/" do
+  uri = URI("https://\#{request.host}/foo/redirect")
+  redirect URI(uri), 307
+end
+PROXY
+      end
+
+      let(:proxy_scheme) { "http" }
+      let(:setup_static_json) do
+        Proc.new do |path|
+          File.open(static_json_path, "w") do |file|
+            file.puts <<STATIC_JSON
+{
+  "proxies": {
+    "/api/": {
+      "origin": "#{proxy_scheme}://#{@proxy_ip_address}#{path}"
+    },
+    "/api_no_slash": {
+      "origin": "#{proxy_scheme}://#{@proxy_ip_address}#{path}"
+    },
+    "/api_origin_no_slash/": {
+      "origin": "#{proxy_scheme}://#{@proxy_ip_address}#{path.chop}"
+    },
+    "/api_no_slash_origin_no_slash": {
+      "origin": "#{proxy_scheme}://#{@proxy_ip_address}#{path.chop}"
+    }
+  },
+  "routes": {
+    "/**": "index.html"
+  }
+}
+STATIC_JSON
+          end
+        end
+      end
+
+      before do
+        setup_static_json.call("/")
+      end
+
+      it "should proxy properly" do
+        response = app.get("/api/hello")
+        expect(response.code).to eq("200")
+        expect(response.body.chomp).to eq("hello")
+
+        response = app.get("/api_no_slash/hello")
+        expect(response.code).to eq("200")
+        expect(response.body.chomp).to eq("hello")
+
+        response = app.get("/api_origin_no_slash/hello")
+        expect(response.code).to eq("200")
+        expect(response.body.chomp).to eq("hello")
+
+        response = app.get("/api_no_slash_origin_no_slash/hello")
+        expect(response.code).to eq("200")
+        expect(response.body.chomp).to eq("hello")
+      end
+
+      it "should handle redirects regardless of scheme" do
+        app.run do
+          response = app.get("/api/http_redirect/")
+          expect(response.code).to eq("307")
+          expect(response["Location"]).not_to include(@proxy_ip_address)
+
+          response = app.get("/api/https_redirect/")
+          expect(response.code).to eq("307")
+          expect(response["Location"]).not_to include(@proxy_ip_address)
+
+          response = app.get("/api_no_slash/http_redirect/")
+          expect(response.code).to eq("307")
+          expect(response["Location"]).not_to include(@proxy_ip_address)
+
+          response = app.get("/api_no_slash/https_redirect/")
+          expect(response.code).to eq("307")
+          expect(response["Location"]).not_to include(@proxy_ip_address)
+
+          response = app.get("/api_origin_no_slash/http_redirect/")
+          expect(response.code).to eq("307")
+          expect(response["Location"]).not_to include(@proxy_ip_address)
+
+          response = app.get("/api_origin_no_slash/https_redirect/")
+          expect(response.code).to eq("307")
+          expect(response["Location"]).not_to include(@proxy_ip_address)
+
+          response = app.get("/api_no_slash_origin_no_slash/http_redirect/")
+          expect(response.code).to eq("307")
+          expect(response["Location"]).not_to include(@proxy_ip_address)
+
+          response = app.get("/api_no_slash_origin_no_slash/https_redirect/")
+          expect(response.code).to eq("307")
+          expect(response["Location"]).not_to include(@proxy_ip_address)
+        end
+      end
+    end
   end
 
   describe "custom headers" do
@@ -411,16 +726,6 @@ end
 get "/foo/baz/" do
   "baz"
 end
-
-get "/foo/http_redirect/" do
-  uri = URI("http://\#{request.host}/foo/redirect")
-  redirect URI(uri), 307
-end
-
-get "/foo/https_redirect/" do
-  uri = URI("https://\#{request.host}/foo/redirect")
-  redirect URI(uri), 307
-end
 PROXY
       end
       let(:setup_static_json) do
@@ -467,18 +772,6 @@ STATIC_JSON
           expect(response["X-Header"]).to be_nil
         end
       end
-
-      it "should hanadle redirects regardless of scheme" do
-        app.run do
-          response = app.get("/api/http_redirect/")
-          expect(response.code).to eq("307")
-          expect(response["Location"]).not_to include(@proxy_ip_address)
-
-          response = app.get("/api/https_redirect/")
-          expect(response.code).to eq("307")
-          expect(response["Location"]).not_to include(@proxy_ip_address)
-        end
-      end
     end
   end