Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add --tf-download-url configuration option #787

Closed
wants to merge 3 commits into from
Closed

Add --tf-download-url configuration option #787

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ac008b1
Add `--tf-download-url` configuration option
cullenmcdermott Aug 25, 2019
5daec3e
Apply suggestions from code review
cullenmcdermott Nov 3, 2019
3b21f31
Address comments
cullenmcdermott Nov 4, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions cmd/server.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ const (
SlackTokenFlag = "slack-token"
SSLCertFileFlag = "ssl-cert-file"
SSLKeyFileFlag = "ssl-key-file"
TFDownloadURLFlag = "tf-download-url"
TFEHostnameFlag = "tfe-hostname"
TFETokenFlag = "tfe-token"
WriteGitCredsFlag = "write-git-creds"
Expand All @@ -86,6 +87,7 @@ const (
DefaultGitlabHostname = "gitlab.com"
DefaultLogLevel = "info"
DefaultPort = 4141
DefaultTFDownloadURL = "https://releases.hashicorp.com"
DefaultTFEHostname = "app.terraform.io"
)

Expand Down Expand Up @@ -202,6 +204,10 @@ var stringFlags = map[string]stringFlag{
SSLKeyFileFlag: {
description: fmt.Sprintf("File containing x509 private key matching --%s.", SSLCertFileFlag),
},
TFDownloadURLFlag: {
description: "URL to download Terraform from.",
defaultValue: DefaultTFDownloadURL,
},
TFEHostnameFlag: {
description: "Hostname of your Terraform Enterprise installation. If using Terraform Cloud no need to set.",
defaultValue: DefaultTFEHostname,
Expand Down Expand Up @@ -454,6 +460,9 @@ func (s *ServerCmd) setDefaults(c *server.UserConfig) {
if c.Port == 0 {
c.Port = DefaultPort
}
if c.TFDownloadURL == "" {
c.TFDownloadURL = DefaultTFDownloadURL
}
if c.TFEHostname == "" {
c.TFEHostname = DefaultTFEHostname
}
Expand Down
14 changes: 14 additions & 0 deletions cmd/server_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -392,6 +392,7 @@ func TestExecute_Defaults(t *testing.T) {
Equals(t, "", passedConfig.SlackToken)
Equals(t, "", passedConfig.SSLCertFile)
Equals(t, "", passedConfig.SSLKeyFile)
Equals(t, "https://releases.hashicorp.com", passedConfig.TFDownloadURL)
Equals(t, "app.terraform.io", passedConfig.TFEHostname)
Equals(t, "", passedConfig.TFEToken)
Equals(t, false, passedConfig.WriteGitCreds)
Expand Down Expand Up @@ -515,6 +516,7 @@ func TestExecute_Flags(t *testing.T) {
cmd.SlackTokenFlag: "slack-token",
cmd.SSLCertFileFlag: "cert-file",
cmd.SSLKeyFileFlag: "key-file",
cmd.TFDownloadURLFlag: "https://my-hostname.com",
cmd.TFEHostnameFlag: "my-hostname",
cmd.TFETokenFlag: "my-token",
cmd.WriteGitCredsFlag: true,
Expand Down Expand Up @@ -554,6 +556,7 @@ func TestExecute_Flags(t *testing.T) {
Equals(t, "slack-token", passedConfig.SlackToken)
Equals(t, "cert-file", passedConfig.SSLCertFile)
Equals(t, "key-file", passedConfig.SSLKeyFile)
Equals(t, "https://my-hostname.com", passedConfig.TFDownloadURL)
Equals(t, "my-hostname", passedConfig.TFEHostname)
Equals(t, "my-token", passedConfig.TFEToken)
Equals(t, true, passedConfig.WriteGitCreds)
Expand Down Expand Up @@ -594,6 +597,7 @@ require-mergeable: true
slack-token: slack-token
ssl-cert-file: cert-file
ssl-key-file: key-file
tf-download-url: https://my-hostname.com
tfe-hostname: my-hostname
tfe-token: my-token
write-git-creds: true
Expand Down Expand Up @@ -637,6 +641,7 @@ write-git-creds: true
Equals(t, "slack-token", passedConfig.SlackToken)
Equals(t, "cert-file", passedConfig.SSLCertFile)
Equals(t, "key-file", passedConfig.SSLKeyFile)
Equals(t, "https://my-hostname.com", passedConfig.TFDownloadURL)
Equals(t, "my-hostname", passedConfig.TFEHostname)
Equals(t, "my-token", passedConfig.TFEToken)
Equals(t, true, passedConfig.WriteGitCreds)
Expand Down Expand Up @@ -676,6 +681,7 @@ require-approval: true
slack-token: slack-token
ssl-cert-file: cert-file
ssl-key-file: key-file
tf-download-url: https://my-hostname.com
tfe-hostname: my-hostname
tfe-token: my-token
write-git-creds: true
Expand Down Expand Up @@ -716,6 +722,7 @@ write-git-creds: true
"SLACK_TOKEN": "override-slack-token",
"SSL_CERT_FILE": "override-cert-file",
"SSL_KEY_FILE": "override-key-file",
"TF_DOWNLOAD_URL": "https://override-my-hostname.com",
"TFE_HOSTNAME": "override-my-hostname",
"TFE_TOKEN": "override-my-token",
"WRITE_GIT_CREDS": "false",
Expand Down Expand Up @@ -759,6 +766,7 @@ write-git-creds: true
Equals(t, "override-slack-token", passedConfig.SlackToken)
Equals(t, "override-cert-file", passedConfig.SSLCertFile)
Equals(t, "override-key-file", passedConfig.SSLKeyFile)
Equals(t, "https://override-my-hostname.com", passedConfig.TFDownloadURL)
Equals(t, "override-my-hostname", passedConfig.TFEHostname)
Equals(t, "override-my-token", passedConfig.TFEToken)
Equals(t, false, passedConfig.WriteGitCreds)
Expand Down Expand Up @@ -799,6 +807,7 @@ require-mergeable: true
slack-token: slack-token
ssl-cert-file: cert-file
ssl-key-file: key-file
tf-download-url: https://my-hostname.com
tfe-hostname: my-hostname
tfe-token: my-token
write-git-creds: true
Expand Down Expand Up @@ -838,6 +847,7 @@ write-git-creds: true
cmd.SlackTokenFlag: "override-slack-token",
cmd.SSLCertFileFlag: "override-cert-file",
cmd.SSLKeyFileFlag: "override-key-file",
cmd.TFDownloadURLFlag: "https://override-my-hostname.com",
cmd.TFEHostnameFlag: "override-my-hostname",
cmd.TFETokenFlag: "override-my-token",
cmd.WriteGitCredsFlag: false,
Expand Down Expand Up @@ -875,6 +885,7 @@ write-git-creds: true
Equals(t, "override-slack-token", passedConfig.SlackToken)
Equals(t, "override-cert-file", passedConfig.SSLCertFile)
Equals(t, "override-key-file", passedConfig.SSLKeyFile)
Equals(t, "https://override-my-hostname.com", passedConfig.TFDownloadURL)
Equals(t, "override-my-hostname", passedConfig.TFEHostname)
Equals(t, "override-my-token", passedConfig.TFEToken)
Equals(t, false, passedConfig.WriteGitCreds)
Expand Down Expand Up @@ -917,6 +928,7 @@ func TestExecute_FlagEnvVarOverride(t *testing.T) {
"SLACK_TOKEN": "slack-token",
"SSL_CERT_FILE": "cert-file",
"SSL_KEY_FILE": "key-file",
"TF_DOWNLOAD_URL": "https://my-hostname.com",
"TFE_HOSTNAME": "my-hostname",
"TFE_TOKEN": "my-token",
"WRITE_GIT_CREDS": "true",
Expand Down Expand Up @@ -964,6 +976,7 @@ func TestExecute_FlagEnvVarOverride(t *testing.T) {
cmd.SlackTokenFlag: "override-slack-token",
cmd.SSLCertFileFlag: "override-cert-file",
cmd.SSLKeyFileFlag: "override-key-file",
cmd.TFDownloadURLFlag: "https://override-my-hostname.com",
cmd.TFEHostnameFlag: "override-my-hostname",
cmd.TFETokenFlag: "override-my-token",
cmd.WriteGitCredsFlag: false,
Expand Down Expand Up @@ -1003,6 +1016,7 @@ func TestExecute_FlagEnvVarOverride(t *testing.T) {
Equals(t, "override-slack-token", passedConfig.SlackToken)
Equals(t, "override-cert-file", passedConfig.SSLCertFile)
Equals(t, "override-key-file", passedConfig.SSLKeyFile)
Equals(t, "https://override-my-hostname.com", passedConfig.TFDownloadURL)
Equals(t, "override-my-hostname", passedConfig.TFEHostname)
Equals(t, "override-my-token", passedConfig.TFEToken)
Equals(t, false, passedConfig.WriteGitCreds)
Expand Down
10 changes: 9 additions & 1 deletion runatlantis.io/docs/server-configuration.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -421,7 +421,15 @@ Values are chosen in this order:
atlantis server --ssl-cert-file="/etc/ssl/private/my-cert.key"
```
File containing x509 private key matching `--ssl-cert-file`.


* ### `--tf-download-url`
```bash
atlantis server --tf-download-url="https://releases.company.com"
```
An alternative URL to download Terraform versions if they are missing. Useful in an airgapped
environment where releases.hashicorp.com is not available. Directory structure of the custom
endpoint should match that of releases.hashicorp.com.

* ### `--tfe-hostname`
```bash
atlantis server --tfe-hostname="my-terraform-enterprise.company.com"
Expand Down
20 changes: 10 additions & 10 deletions server/events/terraform/terraform_client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,8 @@ type DefaultClient struct {
// with another binary, ex. echo.
overrideTF string
// downloader downloads terraform versions.
downloader Downloader
downloader Downloader
downloadBaseURL string
// versions maps from the string representation of a tf version (ex. 0.11.10)
// to the absolute path of that binary on disk (if it exists).
// Use versionsLock to control access.
Expand All @@ -80,8 +81,6 @@ const (
// binDirName is the name of the directory inside our data dir where
// we download terraform binaries.
binDirName = "bin"
// releasesURL is the base url to download terraform from.
releasesURL = "https://releases.hashicorp.com"
)

// versionRegex extracts the version from `terraform version` output.
Expand All @@ -107,6 +106,7 @@ func NewClient(
tfeHostname string,
defaultVersionStr string,
defaultVersionFlagName string,
tfDownloadURL string,
tfDownloader Downloader) (*DefaultClient, error) {
var finalDefaultVersion *version.Version
var localVersion *version.Version
Expand Down Expand Up @@ -145,7 +145,7 @@ func NewClient(
// Since ensureVersion might end up downloading terraform,
// we call it asynchronously so as to not delay server startup.
versionsLock.Lock()
_, err := ensureVersion(log, tfDownloader, versions, defaultVersion, binDir)
_, err := ensureVersion(log, tfDownloader, versions, defaultVersion, binDir, tfDownloadURL)
versionsLock.Unlock()
if err != nil {
log.Err("could not download terraform %s", defaultVersion.String())
Expand Down Expand Up @@ -176,6 +176,7 @@ func NewClient(
terraformPluginCacheDir: cacheDir,
binDir: binDir,
downloader: tfDownloader,
downloadBaseURL: tfDownloadURL,
versionsLock: &versionsLock,
versions: versions,
}, nil
Expand All @@ -200,7 +201,7 @@ func (c *DefaultClient) EnsureVersion(log *logging.SimpleLogger, v *version.Vers

var err error
c.versionsLock.Lock()
_, err = ensureVersion(log, c.downloader, c.versions, v, c.binDir)
_, err = ensureVersion(log, c.downloader, c.versions, v, c.binDir, c.downloadBaseURL)
c.versionsLock.Unlock()
if err != nil {
return err
Expand Down Expand Up @@ -245,7 +246,7 @@ func (c *DefaultClient) prepCmd(log *logging.SimpleLogger, v *version.Version, w
} else {
var err error
c.versionsLock.Lock()
binPath, err = ensureVersion(log, c.downloader, c.versions, v, c.binDir)
binPath, err = ensureVersion(log, c.downloader, c.versions, v, c.binDir, c.downloadBaseURL)
c.versionsLock.Unlock()
if err != nil {
return "", nil, err
Expand Down Expand Up @@ -390,7 +391,7 @@ func MustConstraint(v string) version.Constraints {

// ensureVersion returns the path to a terraform binary of version v.
// It will download this version if we don't have it.
func ensureVersion(log *logging.SimpleLogger, dl Downloader, versions map[string]string, v *version.Version, binDir string) (string, error) {
func ensureVersion(log *logging.SimpleLogger, dl Downloader, versions map[string]string, v *version.Version, binDir string, downloadURL string) (string, error) {
if binPath, ok := versions[v.String()]; ok {
return binPath, nil
}
Expand All @@ -411,9 +412,8 @@ func ensureVersion(log *logging.SimpleLogger, dl Downloader, versions map[string
versions[v.String()] = dest
return dest, nil
}

log.Info("could not find terraform version %s in PATH or %s, downloading from %s", v.String(), binDir, releasesURL)
urlPrefix := fmt.Sprintf("%s/terraform/%s/terraform_%s", releasesURL, v.String(), v.String())
log.Info("could not find terraform version %s in PATH or %s, downloading from %s", v.String(), binDir, downloadURL)
urlPrefix := fmt.Sprintf("%s/terraform/%s/terraform_%s", downloadURL, v.String(), v.String())
binURL := fmt.Sprintf("%s_%s_%s.zip", urlPrefix, runtime.GOOS, runtime.GOARCH)
checksumURL := fmt.Sprintf("%s_SHA256SUMS", urlPrefix)
if err := dl.GetFile(dest, fmt.Sprintf("%s?checksum=file:%s", binURL, checksumURL)); err != nil {
Expand Down
24 changes: 12 additions & 12 deletions server/events/terraform/terraform_client_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ is 0.11.13. You can update by downloading from www.terraform.io/downloads.html
Ok(t, err)
defer tempSetEnv(t, "PATH", fmt.Sprintf("%s:%s", tmp, os.Getenv("PATH")))()

c, err := terraform.NewClient(nil, tmp, "", "", "", cmd.DefaultTFVersionFlag, nil)
c, err := terraform.NewClient(nil, tmp, "", "", "", cmd.DefaultTFVersionFlag, "https://releases.hashicorp.com", nil)
Ok(t, err)

Ok(t, err)
Expand Down Expand Up @@ -96,7 +96,7 @@ is 0.11.13. You can update by downloading from www.terraform.io/downloads.html
Ok(t, err)
defer tempSetEnv(t, "PATH", fmt.Sprintf("%s:%s", tmp, os.Getenv("PATH")))()

c, err := terraform.NewClient(nil, tmp, "", "", "0.11.10", cmd.DefaultTFVersionFlag, nil)
c, err := terraform.NewClient(nil, tmp, "", "", "0.11.10", cmd.DefaultTFVersionFlag, cmd.TFDownloadURLFlag, nil)
Ok(t, err)

Ok(t, err)
Expand All @@ -116,7 +116,7 @@ func TestNewClient_NoTF(t *testing.T) {
// Set PATH to only include our empty directory.
defer tempSetEnv(t, "PATH", tmp)()

_, err := terraform.NewClient(nil, tmp, "", "", "", cmd.DefaultTFVersionFlag, nil)
_, err := terraform.NewClient(nil, tmp, "", "", "", cmd.DefaultTFVersionFlag, cmd.TFDownloadURLFlag, nil)
ErrEquals(t, "terraform not found in $PATH. Set --default-tf-version or download terraform from https://www.terraform.io/downloads.html", err)
}

Expand All @@ -133,7 +133,7 @@ func TestNewClient_DefaultTFFlagInPath(t *testing.T) {
Ok(t, err)
defer tempSetEnv(t, "PATH", fmt.Sprintf("%s:%s", tmp, os.Getenv("PATH")))()

c, err := terraform.NewClient(nil, tmp, "", "", "0.11.10", cmd.DefaultTFVersionFlag, nil)
c, err := terraform.NewClient(nil, tmp, "", "", "0.11.10", cmd.DefaultTFVersionFlag, cmd.TFDownloadURLFlag, nil)
Ok(t, err)

Ok(t, err)
Expand All @@ -157,7 +157,7 @@ func TestNewClient_DefaultTFFlagInBinDir(t *testing.T) {
Ok(t, err)
defer tempSetEnv(t, "PATH", fmt.Sprintf("%s:%s", tmp, os.Getenv("PATH")))()

c, err := terraform.NewClient(logging.NewNoopLogger(), tmp, "", "", "0.11.10", cmd.DefaultTFVersionFlag, nil)
c, err := terraform.NewClient(logging.NewNoopLogger(), tmp, "", "", "0.11.10", cmd.DefaultTFVersionFlag, cmd.TFDownloadURLFlag, nil)
Ok(t, err)

Ok(t, err)
Expand All @@ -183,12 +183,12 @@ func TestNewClient_DefaultTFFlagDownload(t *testing.T) {
err := ioutil.WriteFile(params[0].(string), []byte("#!/bin/sh\necho '\nTerraform v0.11.10\n'"), 0755)
return []pegomock.ReturnValue{err}
})
c, err := terraform.NewClient(nil, tmp, "", "", "0.11.10", cmd.DefaultTFVersionFlag, mockDownloader)
c, err := terraform.NewClient(nil, tmp, "", "", "0.11.10", cmd.DefaultTFVersionFlag, "https://my-mirror.releases.mycompany.com", mockDownloader)
Ok(t, err)

Ok(t, err)
Equals(t, "0.11.10", c.DefaultVersion().String())
baseURL := "https://releases.hashicorp.com/terraform/0.11.10"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe set this to https://my-mirror.releases.mycompany.com in this test to make it clear it's being tested

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think I adjusted this to match what you are asking, can you double check?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes this looks good

baseURL := "https://my-mirror.releases.mycompany.com/terraform/0.11.10"
expURL := fmt.Sprintf("%s/terraform_0.11.10_%s_%s.zip?checksum=file:%s/terraform_0.11.10_SHA256SUMS",
baseURL,
runtime.GOOS,
Expand All @@ -207,7 +207,7 @@ func TestNewClient_DefaultTFFlagDownload(t *testing.T) {
func TestNewClient_BadVersion(t *testing.T) {
tmp, cleanup := TempDir(t)
defer cleanup()
_, err := terraform.NewClient(nil, tmp, "", "", "malformed", cmd.DefaultTFVersionFlag, nil)
_, err := terraform.NewClient(nil, tmp, "", "", "malformed", cmd.DefaultTFVersionFlag, cmd.TFDownloadURLFlag, nil)
ErrEquals(t, "Malformed version: malformed", err)
}

Expand All @@ -219,7 +219,7 @@ func TestRunCommandWithVersion_DLsTF(t *testing.T) {

mockDownloader := mocks.NewMockDownloader()
// Set up our mock downloader to write a fake tf binary when it's called.
baseURL := "https://releases.hashicorp.com/terraform/99.99.99"
baseURL := fmt.Sprintf("%s/terraform/99.99.99", cmd.TFDownloadURLFlag)
expURL := fmt.Sprintf("%s/terraform_99.99.99_%s_%s.zip?checksum=file:%s/terraform_99.99.99_SHA256SUMS",
baseURL,
runtime.GOOS,
Expand All @@ -230,7 +230,7 @@ func TestRunCommandWithVersion_DLsTF(t *testing.T) {
return []pegomock.ReturnValue{err}
})

c, err := terraform.NewClient(nil, tmp, "", "", "0.11.10", cmd.DefaultTFVersionFlag, mockDownloader)
c, err := terraform.NewClient(nil, tmp, "", "", "0.11.10", cmd.DefaultTFVersionFlag, cmd.TFDownloadURLFlag, mockDownloader)
Ok(t, err)
Equals(t, "0.11.10", c.DefaultVersion().String())

Expand All @@ -249,7 +249,7 @@ func TestEnsureVersion_downloaded(t *testing.T) {

mockDownloader := mocks.NewMockDownloader()

c, err := terraform.NewClient(nil, tmp, "", "", "0.11.10", cmd.DefaultTFVersionFlag, mockDownloader)
c, err := terraform.NewClient(nil, tmp, "", "", "0.11.10", cmd.DefaultTFVersionFlag, cmd.TFDownloadURLFlag, mockDownloader)
Ok(t, err)

Equals(t, "0.11.10", c.DefaultVersion().String())
Expand All @@ -261,7 +261,7 @@ func TestEnsureVersion_downloaded(t *testing.T) {

Ok(t, err)

baseURL := "https://releases.hashicorp.com/terraform/99.99.99"
baseURL := fmt.Sprintf("%s/terraform/99.99.99", cmd.TFDownloadURLFlag)
expURL := fmt.Sprintf("%s/terraform_99.99.99_%s_%s.zip?checksum=file:%s/terraform_99.99.99_SHA256SUMS",
baseURL,
runtime.GOOS,
Expand Down
2 changes: 1 addition & 1 deletion server/events_controller_e2e_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -400,7 +400,7 @@ func setupE2E(t *testing.T, repoDir string) (server.EventsController, *vcsmocks.
GithubUser: "github-user",
GitlabUser: "gitlab-user",
}
terraformClient, err := terraform.NewClient(logger, dataDir, "", "", "", "default-tf-version", &NoopTFDownloader{})
terraformClient, err := terraform.NewClient(logger, dataDir, "", "", "", "tfdownloadurl", "default-tf-version", &NoopTFDownloader{})
Ok(t, err)
boltdb, err := db.New(dataDir)
Ok(t, err)
Expand Down
1 change: 1 addition & 0 deletions server/server.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -211,6 +211,7 @@ func NewServer(userConfig UserConfig, config Config) (*Server, error) {
userConfig.DataDir,
userConfig.TFEToken,
userConfig.TFEHostname,
userConfig.TFDownloadURL,
userConfig.DefaultTFVersion,
config.DefaultTFVersionFlag,
&terraform.DefaultDownloader{})
Expand Down
1 change: 1 addition & 0 deletions server/user_config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ type UserConfig struct {
SlackToken string `mapstructure:"slack-token"`
SSLCertFile string `mapstructure:"ssl-cert-file"`
SSLKeyFile string `mapstructure:"ssl-key-file"`
TFDownloadURL string `mapstructure:"tf-download-url"`
TFEHostname string `mapstructure:"tfe-hostname"`
TFEToken string `mapstructure:"tfe-token"`
DefaultTFVersion string `mapstructure:"default-tf-version"`
Expand Down