Implement local plan storage. Refactor S3 storage (runatlantis#33)

* added new flags `plan-backend`, `plan-s3-bucket`, `plan-s3-prefix` and deleted `s3-bucket`
* interface `plan.Backend` that is implemented by `file` and `s3`
* simplified s3 code
* didn't end up following my suggestions in runatlantis#30 since storing stuff in metadata requires you to `Get` the object *first* and then use the metadata. By parsing the `Key` to get repo, pull, path, and env, it skips an initial `Get` step, and I can download directly to the correct directory
* allow users to specify `application/json` or `application/x-www-form-urlencoded` for the webhook delivery type
* remove sending of special header for pull request api (fixes runatlantis#11)

Closes runatlantis#30 and runatlantis#17 and runatlantis#11

Author: lkysow

circle.yml

@@ -30,7 +30,7 @@ test:
   post:
     - cd "${WORKDIR}" && ./scripts/e2e-deps.sh
     # Start atlantis server
-    - cd "${WORKDIR}/e2e" && ./atlantis server --gh-user="$GITHUB_USERNAME" --gh-password="$GITHUB_PASSWORD" --data-dir="/tmp" --require-approval=false --s3-bucket="$ATLANTIS_S3_BUCKET_NAME" --log-level="debug" &> /tmp/atlantis-server.log:
+    - cd "${WORKDIR}/e2e" && ./atlantis server --gh-user="$GITHUB_USERNAME" --gh-password="$GITHUB_PASSWORD" --data-dir="/tmp" --require-approval=false --plan-backend="file" --log-level="debug" &> /tmp/atlantis-server.log:
           background: true
     - sleep 2
     - cd "${WORKDIR}/e2e" && ./ngrok http 4141:

cmd/root.go

@@ -7,7 +7,7 @@ import (
 
 var RootCmd = &cobra.Command{
 	Use:   "atlantis",
-	Short: "Terraform collaboration tool",
+	Short: "Terraform collaboration tool", // todo: decide on name #opensource
 }
 
 func Execute() {

cmd/server.go

@@ -25,9 +25,11 @@ const (
 	lockingBackendFlag  = "locking-backend"
 	lockingTableFlag    = "locking-dynamodb-table"
 	logLevelFlag        = "log-level"
+	planBackendFlag     = "plan-backend"
+	planS3BucketFlag    = "plan-s3-bucket"
+	planS3PrefixFlag    = "plan-s3-prefix"
 	portFlag            = "port"
 	requireApprovalFlag = "require-approval"
-	s3BucketFlag        = "s3-bucket"
 	scratchDirFlag      = "scratch-dir"
 	sshKeyFlag          = "ssh-key"
 )
@@ -77,7 +79,7 @@ var stringFlags = []stringFlag{
 	},
 	{
 		name:        lockingTableFlag,
-		description: "Name of table in DynamoDB to use for locking. Only read if locking-backend is set to dynamodb.",
+		description: "Name of table in DynamoDB to use for locking. Only read if " + lockingBackendFlag + " is set to dynamodb.",
 		value:       "atlantis-locks",
 	},
 	{
@@ -86,10 +88,20 @@ var stringFlags = []stringFlag{
 		value:       "warn",
 	},
 	{
-		name:        s3BucketFlag,
-		description: "The S3 bucket name to store atlantis data (terraform plans, terraform state, etc).",
+		name:        planS3BucketFlag,
+		description: "S3 bucket for storing plan files. Only read if " + planBackendFlag + " is set to s3",
 		value:       "atlantis",
 	},
+	{
+		name:        planS3PrefixFlag,
+		description: "Prefix of plan file names stored in S3. Only read if " + planBackendFlag + " is set to s3",
+		value:       "",
+	},
+	{
+		name:        planBackendFlag,
+		description: "How to store plan files: file or s3. If set to file, will store plan files on disk in the directory specified by data-dir.",
+		value:       "file",
+	},
 	{
 		name:        scratchDirFlag,
 		description: "Path to directory to use as a temporary workspace for checking out repos.",
@@ -147,7 +159,7 @@ Config values are overridden by environment variables which in turn are overridd
 		if configFile != "" {
 			viper.SetConfigFile(configFile)
 			if err := viper.ReadInConfig(); err != nil {
-				return fmt.Errorf("invalid config: reading %s: %s", configFile, err)
+				return errors.Wrapf(err, "invalid config: reading %s", configFile)
 			}
 		}
 		return nil
@@ -212,6 +224,9 @@ func validate(config server.ServerConfig) error {
 	if config.LockingBackend != server.LockingFileBackend && config.LockingBackend != server.LockingDynamoDBBackend {
 		return fmt.Errorf("unsupported locking backend %q: not one of %q or %q", config.LockingBackend, server.LockingFileBackend, server.LockingDynamoDBBackend)
 	}
+	if config.PlanBackend != server.PlanFileBackend && config.PlanBackend != server.PlanS3Backend {
+		return fmt.Errorf("unsupported plan backend %q: not one of %q or %q", config.PlanBackend, server.PlanFileBackend, server.PlanS3Backend)
+	}
 	return nil
 }
 

cmd/version.go

@@ -8,7 +8,7 @@ import (
 
 var versionCmd = &cobra.Command{
 	Use:   "version",
-	Short: "Print the current atlantis version",
+	Short: "Print the current Atlantis version",
 	Run: func(cmd *cobra.Command, args []string) {
 		fmt.Printf("atlantis %s\n", viper.Get("version"))
 	},

e2e/main.go

@@ -14,8 +14,8 @@ import (
 
 var defaultAtlantisURL = "http://localhost:4141"
 var projectTypes = []Project{
-	Project{"standalone", "run plan", "run apply"},
-	Project{"standalone-with-env", "run plan staging", "run apply staging"},
+	{"standalone", "run plan", "run apply"},
+	{"standalone-with-env", "run plan staging", "run apply staging"},
 }
 
 type Project struct {

e2e/secrets-envs

@@ -0,0 +1,16 @@
+package plan
+
+import (
+	"github.com/hootsuite/atlantis/models"
+)
+
+type Backend interface {
+	SavePlan(path string, project models.Project, env string, pullNum int) error
+	CopyPlans(dstRepoPath string, repoFullName string, env string, pullNum int) ([]Plan, error)
+}
+
+type Plan struct {
+	Project models.Project
+	// LocalPath is the path to the plan on disk
+	LocalPath string
+}

plan/backend.go

@@ -0,0 +1,93 @@
+package file
+
+import (
+	"github.com/hootsuite/atlantis/models"
+	"github.com/hootsuite/atlantis/plan"
+	"github.com/pkg/errors"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strconv"
+)
+
+type Backend struct {
+	// baseDir is the root at which all plans will be stored
+	baseDir string
+}
+
+func New(baseDir string) (*Backend, error) {
+	baseDir = filepath.Clean(baseDir)
+	if err := os.MkdirAll(baseDir, 0755); err != nil {
+		return nil, err
+	}
+	return &Backend{baseDir}, nil
+}
+
+// save plans to baseDir/owner/repo/pullNum/path/env.tfplan
+func (b *Backend) SavePlan(path string, project models.Project, env string, pullNum int) error {
+	savePath := b.path(project, pullNum)
+	if err := os.MkdirAll(savePath, 0755); err != nil {
+		return errors.Wrap(err, "creating save directory")
+	}
+	if err := b.copy(path, filepath.Join(savePath, env+".tfplan")); err != nil {
+		return errors.Wrap(err, "saving plan")
+	}
+	return nil
+}
+
+func (b *Backend) CopyPlans(dstRepo string, repoFullName string, env string, pullNum int) ([]plan.Plan, error) {
+	// Look in the directory for this repo/pull and get plans for all projects.
+	// Then filter to the plans for this environment
+	var toCopy []string // will contain paths to the plan files relative to repo root
+	root := filepath.Join(b.baseDir, repoFullName, strconv.Itoa(pullNum))
+	err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		// if the plan is for the right env,
+		if info.Name() == env+".tfplan" {
+			rel, err := filepath.Rel(root, path)
+			if err == nil {
+				toCopy = append(toCopy, rel)
+			}
+		}
+		return nil
+	})
+
+	var plans []plan.Plan
+	if err != nil {
+		return plans, errors.Wrap(err, "listing plans")
+	}
+
+	// copy the plans to the destination repo
+	for _, file := range toCopy {
+		dst := filepath.Join(dstRepo, file)
+		if err := b.copy(filepath.Join(root, file), dst); err != nil {
+			return plans, errors.Wrap(err, "copying plan")
+		}
+		plans = append(plans, plan.Plan{
+			Project: models.Project{
+				Path:         filepath.Dir(file),
+				RepoFullName: repoFullName,
+			},
+			LocalPath: dst,
+		})
+	}
+	return plans, nil
+}
+
+func (b *Backend) copy(src string, dst string) error {
+	data, err := ioutil.ReadFile(src)
+	if err != nil {
+		return errors.Wrapf(err, "reading %s", src)
+	}
+
+	if err = ioutil.WriteFile(dst, data, 0644); err != nil {
+		return errors.Wrapf(err, "writing %s", dst)
+	}
+	return nil
+}
+
+func (b *Backend) path(p models.Project, pullNum int) string {
+	return filepath.Join(b.baseDir, p.RepoFullName, strconv.Itoa(pullNum), p.Path)
+}

plan/file/file.go

@@ -0,0 +1,106 @@
+package s3
+
+import (
+	"os"
+	pathutil "path"
+	"path/filepath"
+	"strconv"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/client"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+	"github.com/hootsuite/atlantis/models"
+	"github.com/hootsuite/atlantis/plan"
+	"github.com/pkg/errors"
+)
+
+type Backend struct {
+	s3         *s3.S3
+	uploader   *s3manager.Uploader
+	downloader *s3manager.Downloader
+	bucket     string
+	keyPrefix  string
+}
+
+func New(p client.ConfigProvider, bucket string, keyPrefix string) *Backend {
+	return &Backend{
+		s3:         s3.New(p),
+		uploader:   s3manager.NewUploader(p),
+		downloader: s3manager.NewDownloader(p),
+		bucket:     bucket,
+		keyPrefix:  keyPrefix,
+	}
+}
+
+func (b *Backend) CopyPlans(repoDir string, repoFullName string, env string, pullNum int) ([]plan.Plan, error) {
+	// first list the plans with the correct prefix
+	prefix := pathutil.Join(b.keyPrefix, repoFullName, strconv.Itoa(pullNum))
+	list, err := b.s3.ListObjects(&s3.ListObjectsInput{Bucket: aws.String(b.bucket), Prefix: &prefix})
+	if err != nil {
+		return nil, errors.Wrap(err, "listing plans")
+	}
+
+	var plans []plan.Plan
+	for _, obj := range list.Contents {
+		planName := pathutil.Base(*obj.Key)
+
+		// only get plans from the correct env
+		if planName != env+".tfplan" {
+			continue
+		}
+
+		// determine the path relative to the repo
+		relPath, err := filepath.Rel(prefix, *obj.Key)
+		if err != nil {
+			continue
+		}
+		downloadPath := filepath.Join(repoDir, relPath)
+		file, err := os.Create(downloadPath)
+		if err != nil {
+			return nil, errors.Wrapf(err, "creating file %s to download plan to", downloadPath)
+		}
+		defer file.Close()
+
+		_, err = b.downloader.Download(file,
+			&s3.GetObjectInput{
+				Bucket: aws.String(b.bucket),
+				Key:    obj.Key,
+			})
+		if err != nil {
+			return nil, errors.Wrapf(err, "downloading file at %s", *obj.Key)
+		}
+		plans = append(plans, plan.Plan{
+			Project: models.Project{
+				Path:         pathutil.Dir(relPath),
+				RepoFullName: repoFullName,
+			},
+			LocalPath: downloadPath,
+		})
+	}
+	return plans, nil
+}
+
+func (b *Backend) SavePlan(path string, project models.Project, env string, pullNum int) error {
+	f, err := os.Open(path)
+	if err != nil {
+		return errors.Wrapf(err, "opening plan at %s", path)
+	}
+
+	key := pathutil.Join(b.keyPrefix, project.RepoFullName, strconv.Itoa(pullNum), project.Path, env+".tfplan")
+	_, err = b.uploader.Upload(&s3manager.UploadInput{
+		Bucket: aws.String(b.bucket),
+		Key:    &key,
+		Body:   f,
+		Metadata: map[string]*string{
+			"repoFullName": aws.String(project.RepoFullName),
+			"path":         aws.String(project.Path),
+			"env":          aws.String(env),
+			"pullNum":      aws.String(strconv.Itoa(pullNum)),
+		},
+	})
+	if err != nil {
+		return errors.Wrap(err, "uploading plan to s3")
+	}
+	return nil
+}