rkrtechnology/litestream
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Configuration file environment variable expansion

Browse Source 
This commit adds simple variable expansion using either `$FOO`
or `${FOO}` when evaluating the config file. This can be disabled
by any command by using the `-no-expand-env` flag.
This commit is contained in:
Ben Johnson
2021-04-16 09:28:01 -06:00
parent cefbcb0460
commit 04ae010378
8 changed files with 100 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
cmd/litestream/databases.go
View File

@@ -15,7 +15,7 @@ type DatabasesCommand struct{}
// Run executes the command.
func (c *DatabasesCommand) Run(ctx context.Context, args []string) (err error) {
fs := flag.NewFlagSet("litestream-databases", flag.ContinueOnError)
configPath := registerConfigFlag(fs)
configPath, noExpandEnv := registerConfigFlag(fs)
fs.Usage = c.Usage
if err := fs.Parse(args); err != nil {
return err
@@ -27,7 +27,7 @@ func (c *DatabasesCommand) Run(ctx context.Context, args []string) (err error) {
if *configPath == "" {
*configPath = DefaultConfigPath()
}
config, err := ReadConfigFile(*configPath)
config, err := ReadConfigFile(*configPath, !*noExpandEnv)
if err != nil {
return err
}
@@ -72,6 +72,9 @@ Arguments:
Specifies the configuration file.
Defaults to %s
-no-expand-env
Disables environment variable expansion in configuration file.
`[1:],
DefaultConfigPath(),
)

7
cmd/litestream/generations.go
View File

@@ -18,7 +18,7 @@ type GenerationsCommand struct{}
// Run executes the command.
func (c *GenerationsCommand) Run(ctx context.Context, args []string) (err error) {
fs := flag.NewFlagSet("litestream-generations", flag.ContinueOnError)
configPath := registerConfigFlag(fs)
configPath, noExpandEnv := registerConfigFlag(fs)
replicaName := fs.String("replica", "", "replica name")
fs.Usage = c.Usage
if err := fs.Parse(args); err != nil {
@@ -45,7 +45,7 @@ func (c *GenerationsCommand) Run(ctx context.Context, args []string) (err error)
}
// Load configuration.
config, err := ReadConfigFile(*configPath)
config, err := ReadConfigFile(*configPath, !*noExpandEnv)
if err != nil {
return err
}
@@ -131,6 +131,9 @@ Arguments:
Specifies the configuration file.
Defaults to %s
-no-expand-env
Disables environment variable expansion in configuration file.
-replica NAME
Optional, filters by replica.

22
cmd/litestream/main.go
View File

@@ -179,7 +179,8 @@ func (c *Config) DBConfig(path string) *DBConfig {
}
// ReadConfigFile unmarshals config from filename. Expands path if needed.
func ReadConfigFile(filename string) (_ Config, err error) {
// If expandEnv is true then environment variables are expanded in the config.
func ReadConfigFile(filename string, expandEnv bool) (_ Config, err error) {
config := DefaultConfig()
// Expand filename, if necessary.
@@ -188,12 +189,20 @@ func ReadConfigFile(filename string) (_ Config, err error) {
return config, err
}
// Read & deserialize configuration.
if buf, err := ioutil.ReadFile(filename); os.IsNotExist(err) {
// Read configuration.
buf, err := ioutil.ReadFile(filename)
if os.IsNotExist(err) {
return config, fmt.Errorf("config file not found: %s", filename)
} else if err != nil {
return config, err
} else if err := yaml.Unmarshal(buf, &config); err != nil {
}
// Expand environment variables, if enabled.
if expandEnv {
buf = []byte(os.ExpandEnv(string(buf)))
}
if err := yaml.Unmarshal(buf, &config); err != nil {
return config, err
}
@@ -461,8 +470,9 @@ func DefaultConfigPath() string {
return defaultConfigPath
}
func registerConfigFlag(fs *flag.FlagSet) *string {
return fs.String("config", "", "config path")
func registerConfigFlag(fs *flag.FlagSet) (configPath *string, noExpandEnv *bool) {
return fs.String("config", "", "config path"),
fs.Bool("no-expand-env", false, "do not expand env vars in config")
}
// expand returns an absolute path for s.

53
cmd/litestream/main_test.go
View File

@@ -2,6 +2,7 @@ package main_test
import (
"io/ioutil"
"os"
"path/filepath"
"testing"
@@ -26,7 +27,7 @@ dbs:
t.Fatal(err)
}
config, err := main.ReadConfigFile(filename)
config, err := main.ReadConfigFile(filename, true)
if err != nil {
t.Fatal(err)
} else if got, want := config.AccessKeyID, `XXX`; got != want {
@@ -39,6 +40,56 @@ dbs:
t.Fatalf("Replica.SecretAccessKey=%v, want %v", got, want)
}
})
// Ensure environment variables are expanded.
t.Run("ExpandEnv", func(t *testing.T) {
os.Setenv("LITESTREAM_TEST_0129380", "/path/to/db")
os.Setenv("LITESTREAM_TEST_1872363", "s3://foo/bar")
filename := filepath.Join(t.TempDir(), "litestream.yml")
if err := ioutil.WriteFile(filename, []byte(`
dbs:
- path: $LITESTREAM_TEST_0129380
replicas:
- url: ${LITESTREAM_TEST_1872363}
- url: ${LITESTREAM_TEST_NO_SUCH_ENV}
`[1:]), 0666); err != nil {
t.Fatal(err)
}
config, err := main.ReadConfigFile(filename, true)
if err != nil {
t.Fatal(err)
} else if got, want := config.DBs[0].Path, `/path/to/db`; got != want {
t.Fatalf("DB.Path=%v, want %v", got, want)
} else if got, want := config.DBs[0].Replicas[0].URL, `s3://foo/bar`; got != want {
t.Fatalf("Replica[0].URL=%v, want %v", got, want)
} else if got, want := config.DBs[0].Replicas[1].URL, ``; got != want {
t.Fatalf("Replica[1].URL=%v, want %v", got, want)
}
})
// Ensure environment variables are not expanded.
t.Run("NoExpandEnv", func(t *testing.T) {
os.Setenv("LITESTREAM_TEST_9847533", "s3://foo/bar")
filename := filepath.Join(t.TempDir(), "litestream.yml")
if err := ioutil.WriteFile(filename, []byte(`
dbs:
- path: /path/to/db
replicas:
- url: ${LITESTREAM_TEST_9847533}
`[1:]), 0666); err != nil {
t.Fatal(err)
}
config, err := main.ReadConfigFile(filename, false)
if err != nil {
t.Fatal(err)
} else if got, want := config.DBs[0].Replicas[0].URL, `${LITESTREAM_TEST_9847533}`; got != want {
t.Fatalf("Replica.URL=%v, want %v", got, want)
}
})
}
func TestNewFileReplicaFromConfig(t *testing.T) {

7
cmd/litestream/replicate.go
View File

@@ -32,7 +32,7 @@ func NewReplicateCommand() *ReplicateCommand {
func (c *ReplicateCommand) ParseFlags(ctx context.Context, args []string) (err error) {
fs := flag.NewFlagSet("litestream-replicate", flag.ContinueOnError)
tracePath := fs.String("trace", "", "trace path")
configPath := registerConfigFlag(fs)
configPath, noExpandEnv := registerConfigFlag(fs)
fs.Usage = c.Usage
if err := fs.Parse(args); err != nil {
return err
@@ -58,7 +58,7 @@ func (c *ReplicateCommand) ParseFlags(ctx context.Context, args []string) (err e
if *configPath == "" {
*configPath = DefaultConfigPath()
}
if c.Config, err = ReadConfigFile(*configPath); err != nil {
if c.Config, err = ReadConfigFile(*configPath, !*noExpandEnv); err != nil {
return err
}
}
@@ -168,6 +168,9 @@ Arguments:
Specifies the configuration file.
Defaults to %s
-no-expand-env
Disables environment variable expansion in configuration file.
-trace PATH
Write verbose trace logging to PATH.

11
cmd/litestream/restore.go
View File

@@ -21,7 +21,7 @@ func (c *RestoreCommand) Run(ctx context.Context, args []string) (err error) {
opt.Verbose = true
fs := flag.NewFlagSet("litestream-restore", flag.ContinueOnError)
configPath := registerConfigFlag(fs)
configPath, noExpandEnv := registerConfigFlag(fs)
fs.StringVar(&opt.OutputPath, "o", "", "output path")
fs.StringVar(&opt.ReplicaName, "replica", "", "replica name")
fs.StringVar(&opt.Generation, "generation", "", "generation name")
@@ -69,7 +69,7 @@ func (c *RestoreCommand) Run(ctx context.Context, args []string) (err error) {
if *configPath == "" {
*configPath = DefaultConfigPath()
}
if r, err = c.loadFromConfig(ctx, fs.Arg(0), *configPath, &opt); err != nil {
if r, err = c.loadFromConfig(ctx, fs.Arg(0), *configPath, !*noExpandEnv, &opt); err != nil {
return err
}
}
@@ -98,9 +98,9 @@ func (c *RestoreCommand) loadFromURL(ctx context.Context, replicaURL string, opt
}
// loadFromConfig returns a replica & updates the restore options from a DB reference.
func (c *RestoreCommand) loadFromConfig(ctx context.Context, dbPath, configPath string, opt *litestream.RestoreOptions) (litestream.Replica, error) {
func (c *RestoreCommand) loadFromConfig(ctx context.Context, dbPath, configPath string, expandEnv bool, opt *litestream.RestoreOptions) (litestream.Replica, error) {
// Load configuration.
config, err := ReadConfigFile(configPath)
config, err := ReadConfigFile(configPath, expandEnv)
if err != nil {
return nil, err
}
@@ -150,6 +150,9 @@ Arguments:
Specifies the configuration file.
Defaults to %s
-no-expand-env
Disables environment variable expansion in configuration file.
-replica NAME
Restore from a specific replica.
Defaults to replica with latest data.

7
cmd/litestream/snapshots.go
View File

@@ -17,7 +17,7 @@ type SnapshotsCommand struct{}
// Run executes the command.
func (c *SnapshotsCommand) Run(ctx context.Context, args []string) (err error) {
fs := flag.NewFlagSet("litestream-snapshots", flag.ContinueOnError)
configPath := registerConfigFlag(fs)
configPath, noExpandEnv := registerConfigFlag(fs)
replicaName := fs.String("replica", "", "replica name")
fs.Usage = c.Usage
if err := fs.Parse(args); err != nil {
@@ -43,7 +43,7 @@ func (c *SnapshotsCommand) Run(ctx context.Context, args []string) (err error) {
}
// Load configuration.
config, err := ReadConfigFile(*configPath)
config, err := ReadConfigFile(*configPath, !*noExpandEnv)
if err != nil {
return err
}
@@ -112,6 +112,9 @@ Arguments:
Specifies the configuration file.
Defaults to %s
-no-expand-env
Disables environment variable expansion in configuration file.
-replica NAME
Optional, filter by a specific replica.

7
cmd/litestream/wal.go
View File

@@ -17,7 +17,7 @@ type WALCommand struct{}
// Run executes the command.
func (c *WALCommand) Run(ctx context.Context, args []string) (err error) {
fs := flag.NewFlagSet("litestream-wal", flag.ContinueOnError)
configPath := registerConfigFlag(fs)
configPath, noExpandEnv := registerConfigFlag(fs)
replicaName := fs.String("replica", "", "replica name")
generation := fs.String("generation", "", "generation name")
fs.Usage = c.Usage
@@ -44,7 +44,7 @@ func (c *WALCommand) Run(ctx context.Context, args []string) (err error) {
}
// Load configuration.
config, err := ReadConfigFile(*configPath)
config, err := ReadConfigFile(*configPath, !*noExpandEnv)
if err != nil {
return err
}
@@ -118,6 +118,9 @@ Arguments:
Specifies the configuration file.
Defaults to %s
-no-expand-env
Disables environment variable expansion in configuration file.
-replica NAME
Optional, filter by a specific replica.