Add helper function for fetching S3 objects (#651)

* Add logic for fetching S3 objects

cmd/resolve-version/main.go

 package main
 
 import (
+	"encoding/xml"
 	"errors"
 	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
 	"regexp"
+	"time"
 
 	"github.com/Masterminds/semver"
 )
 
+type result struct {
+	Name                  string     `xml:"Name"`
+	KeyCount              int        `xml:"KeyCount"`
+	MaxKeys               int        `xml:"MaxKeys"`
+	IsTruncated           bool       `xml:"IsTruncated"`
+	ContinuationToken     string     `xml:"ContinuationToken"`
+	NextContinuationToken string     `xml:"NextContinuationToken"`
+	Prefix                string     `xml:"Prefix"`
+	Contents              []s3Object `xml:"Contents"`
+}
+
+type s3Object struct {
+	Key          string    `xml:"Key"`
+	LastModified time.Time `xml:"LastModified"`
+	ETag         string    `xml:"ETag"`
+	Size         int       `xml:"Size"`
+	StorageClass string    `xml:"StorageClass"`
+}
+
 type release struct {
 	binary   string
 	stage    string
@@ -55,6 +79,54 @@ func parseObject(key string) (release, error) {
 	return release{}, fmt.Errorf("Failed to parse key: %s", key)
 }
 
+// Wrapper around the S3 API for listing objects
+// This maps directly to the API and parses the XML response but will not handle
+// paging and offsets automaticaly
+func fetchS3Result(bucketName string, options map[string]string) (result, error) {
+	var result result
+	v := url.Values{}
+	v.Set("list-type", "2")
+	for key, val := range options {
+		v.Set(key, val)
+	}
+	url := fmt.Sprintf("https://%s.s3.amazonaws.com?%s", bucketName, v.Encode())
+	resp, err := http.Get(url)
+	if err != nil {
+		return result, err
+	}
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return result, err
+	}
+
+	return result, xml.Unmarshal(body, &result)
+}
+
+// Query the S3 API for a list of all the objects in an S3 bucket with a
+// given prefix. This will handle the inherent 1000 item limit and paging
+// for you
+func listS3Objects(bucketName string, prefix string) ([]s3Object, error) {
+	var out = []s3Object{}
+	var options = map[string]string{"prefix": prefix}
+
+	for {
+		result, err := fetchS3Result(bucketName, options)
+		if err != nil {
+			return nil, err
+		}
+
+		out = append(out, result.Contents...)
+		if !result.IsTruncated {
+			break
+		}
+
+		options["continuation-token"] = result.NextContinuationToken
+	}
+
+	return out, nil
+}
+
 func main() {
 	fmt.Println("hello world")
 }

cmd/resolve-version/main_integration_test.go

+// +build integration
+
+package main
+
+import (
+	"regexp"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestListS3Objects(t *testing.T) {
+	// Node
+	objects, err := listS3Objects("heroku-nodebin", "node")
+	assert.Nil(t, err)
+	assert.NotEmpty(t, objects)
+
+	// every returned result started with "node"
+	for _, obj := range objects {
+		assert.Regexp(t, regexp.MustCompile("^node"), obj.Key)
+	}
+
+	// every node object must parse as a valid release
+	for _, obj := range objects {
+		release, err := parseObject(obj.Key)
+		assert.Nil(t, err)
+		assert.Regexp(t, regexp.MustCompile("https:\\/\\/s3.amazonaws.com\\/heroku-nodebin"), release.url)
+		assert.Regexp(t, regexp.MustCompile("[0-9]+.[0-9]+.[0-9]+"), release.version.String())
+	}
+
+	// Yarn
+	objects, err = listS3Objects("heroku-nodebin", "yarn")
+	assert.Nil(t, err)
+	assert.NotEmpty(t, objects)
+
+	// every returned result started with "yarn"
+	for _, obj := range objects {
+		assert.Regexp(t, regexp.MustCompile("^yarn"), obj.Key)
+	}
+
+	// every yarn object must parse as a valid release
+	for _, obj := range objects {
+		release, err := parseObject(obj.Key)
+		assert.Nil(t, err)
+		assert.Regexp(t, regexp.MustCompile("https:\\/\\/s3.amazonaws.com\\/heroku-nodebin"), release.url)
+		assert.Regexp(t, regexp.MustCompile("[0-9]+.[0-9]+.[0-9]+"), release.version.String())
+	}
+}

makefile

@@ -5,7 +5,7 @@ build:
 	@GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -v -o ./vendor/resolve-version-linux ./cmd/resolve-version
 
 test-binary:
-	go test -v ./cmd/...
+	go test -v ./cmd/... -tags=integration
 
 shellcheck:
 	@shellcheck -x bin/compile bin/detect bin/release bin/test bin/test-compile