CLI test coverage

This commit is contained in:
Ben Johnson
2022-01-11 13:05:14 -07:00
parent 3f0ec9fa9f
commit f308e0b154
154 changed files with 1619 additions and 312 deletions

View File

@@ -1,9 +1,10 @@
.PHONY: default
default: testdata
default:
.PHONY: testdata
testdata:
make -C testdata
make -C cmd/litestream testdata
docker:
docker build -t litestream .

6
cmd/litestream/Makefile Normal file
View File

@@ -0,0 +1,6 @@
.PHONY: default
default:
.PHONY: testdata
testdata:
make -C testdata

View File

@@ -4,17 +4,30 @@ import (
"context"
"flag"
"fmt"
"os"
"io"
"strings"
"text/tabwriter"
)
// DatabasesCommand is a command for listing managed databases.
type DatabasesCommand struct {
stdin io.Reader
stdout io.Writer
stderr io.Writer
configPath string
noExpandEnv bool
}
// NewDatabasesCommand returns a new instance of DatabasesCommand.
func NewDatabasesCommand(stdin io.Reader, stdout, stderr io.Writer) *DatabasesCommand {
return &DatabasesCommand{
stdin: stdin,
stdout: stdout,
stderr: stderr,
}
}
// Run executes the command.
func (c *DatabasesCommand) Run(ctx context.Context, args []string) (err error) {
fs := flag.NewFlagSet("litestream-databases", flag.ContinueOnError)
@@ -27,16 +40,16 @@ func (c *DatabasesCommand) Run(ctx context.Context, args []string) (err error) {
}
// Load configuration.
if c.configPath == "" {
c.configPath = DefaultConfigPath()
}
config, err := ReadConfigFile(c.configPath, !c.noExpandEnv)
if err != nil {
return err
} else if len(config.DBs) == 0 {
fmt.Fprintln(c.stdout, "No databases found in config file.")
return nil
}
// List all databases.
w := tabwriter.NewWriter(os.Stdout, 0, 8, 2, ' ', 0)
w := tabwriter.NewWriter(c.stdout, 0, 8, 2, ' ', 0)
defer w.Flush()
fmt.Fprintln(w, "path\treplicas")
@@ -62,7 +75,7 @@ func (c *DatabasesCommand) Run(ctx context.Context, args []string) (err error) {
// Usage prints the help screen to STDOUT.
func (c *DatabasesCommand) Usage() {
fmt.Printf(`
fmt.Fprintf(c.stdout, `
The databases command lists all databases in the configuration file.
Usage:

View File

@@ -0,0 +1,66 @@
package main_test
import (
"context"
"flag"
"path/filepath"
"strings"
"testing"
"github.com/benbjohnson/litestream/internal/testingutil"
)
func TestDatabasesCommand(t *testing.T) {
t.Run("OK", func(t *testing.T) {
testDir := filepath.Join("testdata", "databases", "ok")
m, _, stdout, _ := newMain()
if err := m.Run(context.Background(), []string{"databases", "-config", filepath.Join(testDir, "litestream.yml")}); err != nil {
t.Fatal(err)
} else if got, want := stdout.String(), string(testingutil.MustReadFile(t, filepath.Join(testDir, "stdout"))); got != want {
t.Fatalf("stdout=%q, want %q", got, want)
}
})
t.Run("NoDatabases", func(t *testing.T) {
testDir := filepath.Join("testdata", "databases", "no-databases")
m, _, stdout, _ := newMain()
if err := m.Run(context.Background(), []string{"databases", "-config", filepath.Join(testDir, "litestream.yml")}); err != nil {
t.Fatal(err)
} else if got, want := stdout.String(), string(testingutil.MustReadFile(t, filepath.Join(testDir, "stdout"))); got != want {
t.Fatalf("stdout=%q, want %q", got, want)
}
})
t.Run("ErrConfigNotFound", func(t *testing.T) {
testDir := filepath.Join("testdata", "databases", "no-config")
m, _, _, _ := newMain()
err := m.Run(context.Background(), []string{"databases", "-config", filepath.Join(testDir, "litestream.yml")})
if err == nil || !strings.Contains(err.Error(), `config file not found:`) {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrInvalidConfig", func(t *testing.T) {
testDir := filepath.Join("testdata", "databases", "invalid-config")
m, _, _, _ := newMain()
err := m.Run(context.Background(), []string{"databases", "-config", filepath.Join(testDir, "litestream.yml")})
if err == nil || !strings.Contains(err.Error(), `replica path cannot be a url`) {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrTooManyArguments", func(t *testing.T) {
m, _, _, _ := newMain()
err := m.Run(context.Background(), []string{"databases", "xyz"})
if err == nil || err.Error() != `too many arguments` {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("Usage", func(t *testing.T) {
m, _, _, _ := newMain()
if err := m.Run(context.Background(), []string{"databases", "-h"}); err != flag.ErrHelp {
t.Fatalf("unexpected error: %s", err)
}
})
}

View File

@@ -4,93 +4,80 @@ import (
"context"
"flag"
"fmt"
"log"
"io"
"os"
"text/tabwriter"
"time"
"github.com/benbjohnson/litestream"
"github.com/benbjohnson/litestream/internal"
)
// GenerationsCommand represents a command to list all generations for a database.
type GenerationsCommand struct {
stdin io.Reader
stdout io.Writer
stderr io.Writer
configPath string
noExpandEnv bool
replicaName string
}
// NewGenerationsCommand returns a new instance of GenerationsCommand.
func NewGenerationsCommand(stdin io.Reader, stdout, stderr io.Writer) *GenerationsCommand {
return &GenerationsCommand{
stdin: stdin,
stdout: stdout,
stderr: stderr,
}
}
// Run executes the command.
func (c *GenerationsCommand) Run(ctx context.Context, args []string) (err error) {
func (c *GenerationsCommand) Run(ctx context.Context, args []string) (ret error) {
fs := flag.NewFlagSet("litestream-generations", flag.ContinueOnError)
registerConfigFlag(fs, &c.configPath, &c.noExpandEnv)
replicaName := fs.String("replica", "", "replica name")
fs.StringVar(&c.replicaName, "replica", "", "replica name")
fs.Usage = c.Usage
if err := fs.Parse(args); err != nil {
return err
} else if fs.NArg() == 0 || fs.Arg(0) == "" {
} else if fs.Arg(0) == "" {
return fmt.Errorf("database path or replica URL required")
} else if fs.NArg() > 1 {
return fmt.Errorf("too many arguments")
}
var db *litestream.DB
var r *litestream.Replica
dbUpdatedAt := time.Now()
if isURL(fs.Arg(0)) {
if c.configPath != "" {
return fmt.Errorf("cannot specify a replica URL and the -config flag")
}
if r, err = NewReplicaFromConfig(&ReplicaConfig{URL: fs.Arg(0)}, nil); err != nil {
return err
}
} else {
if c.configPath == "" {
c.configPath = DefaultConfigPath()
}
// Load configuration.
config, err := ReadConfigFile(c.configPath, !c.noExpandEnv)
if err != nil {
return err
}
// Lookup database from configuration file by path.
if path, err := expand(fs.Arg(0)); err != nil {
return err
} else if dbc := config.DBConfig(path); dbc == nil {
return fmt.Errorf("database not found in config: %s", path)
} else if db, err = NewDBFromConfig(dbc); err != nil {
return err
}
// Filter by replica, if specified.
if *replicaName != "" {
if r = db.Replica(*replicaName); r == nil {
return fmt.Errorf("replica %q not found for database %q", *replicaName, db.Path())
}
}
// Determine last time database or WAL was updated.
if dbUpdatedAt, err = db.UpdatedAt(); err != nil {
return err
}
// Load configuration.
config, err := ReadConfigFile(c.configPath, !c.noExpandEnv)
if err != nil {
return err
}
var replicas []*litestream.Replica
if r != nil {
replicas = []*litestream.Replica{r}
} else {
replicas = db.Replicas
replicas, db, err := loadReplicas(ctx, config, fs.Arg(0), c.replicaName)
if err != nil {
return err
}
// Determine last time database or WAL was updated.
var dbUpdatedAt time.Time
if db != nil {
if dbUpdatedAt, err = db.UpdatedAt(); err != nil && !os.IsNotExist(err) {
return err
}
}
// List each generation.
w := tabwriter.NewWriter(os.Stdout, 0, 8, 2, ' ', 0)
w := tabwriter.NewWriter(c.stdout, 0, 8, 2, ' ', 0)
defer w.Flush()
fmt.Fprintln(w, "name\tgeneration\tlag\tstart\tend")
for _, r := range replicas {
generations, err := r.Client.Generations(ctx)
if err != nil {
log.Printf("%s: cannot list generations: %s", r.Name(), err)
fmt.Fprintf(c.stderr, "%s: cannot list generations: %s", r.Name(), err)
ret = errExit // signal error return without printing message
continue
}
@@ -98,26 +85,35 @@ func (c *GenerationsCommand) Run(ctx context.Context, args []string) (err error)
for _, generation := range generations {
createdAt, updatedAt, err := litestream.GenerationTimeBounds(ctx, r.Client, generation)
if err != nil {
log.Printf("%s: cannot determine generation time bounds: %s", r.Name(), err)
fmt.Fprintf(c.stderr, "%s: cannot determine generation time bounds: %s", r.Name(), err)
ret = errExit // signal error return without printing message
continue
}
// Calculate lag from database mod time to the replica mod time.
// This is ignored if the database mod time is unavailable such as
// when specifying the replica URL or if the database file is missing.
lag := "-"
if !dbUpdatedAt.IsZero() {
lag = internal.TruncateDuration(dbUpdatedAt.Sub(updatedAt)).String()
}
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n",
r.Name(),
generation,
truncateDuration(dbUpdatedAt.Sub(updatedAt)).String(),
lag,
createdAt.Format(time.RFC3339),
updatedAt.Format(time.RFC3339),
)
}
}
return nil
return ret
}
// Usage prints the help message to STDOUT.
func (c *GenerationsCommand) Usage() {
fmt.Printf(`
fmt.Fprintf(c.stdout, `
The generations command lists all generations for a database or replica. It also
lists stats about their lag behind the primary database and the time range they
cover.
@@ -144,29 +140,3 @@ Arguments:
DefaultConfigPath(),
)
}
func truncateDuration(d time.Duration) time.Duration {
if d < 0 {
if d < -10*time.Second {
return d.Truncate(time.Second)
} else if d < -time.Second {
return d.Truncate(time.Second / 10)
} else if d < -time.Millisecond {
return d.Truncate(time.Millisecond)
} else if d < -time.Microsecond {
return d.Truncate(time.Microsecond)
}
return d
}
if d > 10*time.Second {
return d.Truncate(time.Second)
} else if d > time.Second {
return d.Truncate(time.Second / 10)
} else if d > time.Millisecond {
return d.Truncate(time.Millisecond)
} else if d > time.Microsecond {
return d.Truncate(time.Microsecond)
}
return d
}

View File

@@ -0,0 +1,140 @@
package main_test
import (
"context"
"flag"
"path/filepath"
"strings"
"testing"
"github.com/benbjohnson/litestream/internal/testingutil"
)
func TestGenerationsCommand(t *testing.T) {
t.Run("OK", func(t *testing.T) {
testDir := filepath.Join("testdata", "generations", "ok")
defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
m, _, stdout, _ := newMain()
if err := m.Run(context.Background(), []string{"generations", "-config", filepath.Join(testDir, "litestream.yml"), filepath.Join(testDir, "db")}); err != nil {
t.Fatal(err)
} else if got, want := stdout.String(), string(testingutil.MustReadFile(t, filepath.Join(testDir, "stdout"))); got != want {
t.Fatalf("stdout=%q, want %q", got, want)
}
})
t.Run("ReplicaName", func(t *testing.T) {
testDir := filepath.Join("testdata", "generations", "replica-name")
defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
m, _, stdout, _ := newMain()
if err := m.Run(context.Background(), []string{"generations", "-config", filepath.Join(testDir, "litestream.yml"), "-replica", "replica1", filepath.Join(testDir, "db")}); err != nil {
t.Fatal(err)
} else if got, want := stdout.String(), string(testingutil.MustReadFile(t, filepath.Join(testDir, "stdout"))); got != want {
t.Fatalf("stdout=%q, want %q", got, want)
}
})
t.Run("ReplicaURL", func(t *testing.T) {
testDir := filepath.Join(testingutil.Getwd(t), "testdata", "generations", "replica-url")
defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
replicaURL := "file://" + filepath.ToSlash(testDir) + "/replica"
m, _, stdout, _ := newMain()
if err := m.Run(context.Background(), []string{"generations", replicaURL}); err != nil {
t.Fatal(err)
} else if got, want := stdout.String(), string(testingutil.MustReadFile(t, filepath.Join(testDir, "stdout"))); got != want {
t.Fatalf("stdout=%q, want %q", got, want)
}
})
t.Run("NoDatabase", func(t *testing.T) {
testDir := filepath.Join("testdata", "generations", "no-database")
defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
m, _, stdout, _ := newMain()
if err := m.Run(context.Background(), []string{"generations", "-config", filepath.Join(testDir, "litestream.yml"), filepath.Join(testDir, "db")}); err != nil {
t.Fatal(err)
} else if got, want := stdout.String(), string(testingutil.MustReadFile(t, filepath.Join(testDir, "stdout"))); got != want {
t.Fatalf("stdout=%q, want %q", got, want)
}
})
t.Run("ErrDatabaseOrReplicaRequired", func(t *testing.T) {
m, _, _, _ := newMain()
err := m.Run(context.Background(), []string{"generations"})
if err == nil || err.Error() != `database path or replica URL required` {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrTooManyArguments", func(t *testing.T) {
m, _, _, _ := newMain()
err := m.Run(context.Background(), []string{"generations", "abc", "123"})
if err == nil || err.Error() != `too many arguments` {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrInvalidFlags", func(t *testing.T) {
m, _, _, _ := newMain()
err := m.Run(context.Background(), []string{"generations", "-no-such-flag"})
if err == nil || err.Error() != `flag provided but not defined: -no-such-flag` {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrConfigFileNotFound", func(t *testing.T) {
m, _, _, _ := newMain()
err := m.Run(context.Background(), []string{"generations", "-config", "/no/such/file", "/var/lib/db"})
if err == nil || err.Error() != `config file not found: /no/such/file` {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrInvalidConfig", func(t *testing.T) {
testDir := filepath.Join("testdata", "generations", "invalid-config")
m, _, _, _ := newMain()
err := m.Run(context.Background(), []string{"generations", "-config", filepath.Join(testDir, "litestream.yml"), "/var/lib/db"})
if err == nil || !strings.Contains(err.Error(), `replica path cannot be a url`) {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrDatabaseNotFound", func(t *testing.T) {
testDir := filepath.Join("testdata", "generations", "database-not-found")
defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
m, _, _, _ := newMain()
err := m.Run(context.Background(), []string{"generations", "-config", filepath.Join(testDir, "litestream.yml"), "/no/such/db"})
if err == nil || err.Error() != `database not found in config: /no/such/db` {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrReplicaNotFound", func(t *testing.T) {
testDir := filepath.Join(testingutil.Getwd(t), "testdata", "generations", "replica-not-found")
defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
m, _, _, _ := newMain()
err := m.Run(context.Background(), []string{"generations", "-config", filepath.Join(testDir, "litestream.yml"), "-replica", "no_such_replica", filepath.Join(testDir, "db")})
if err == nil || err.Error() != `replica "no_such_replica" not found for database "`+filepath.Join(testDir, "db")+`"` {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrInvalidReplicaURL", func(t *testing.T) {
m, _, _, _ := newMain()
err := m.Run(context.Background(), []string{"generations", "xyz://xyz"})
if err == nil || !strings.Contains(err.Error(), `unknown replica type in config: "xyz"`) {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("Usage", func(t *testing.T) {
m, _, _, _ := newMain()
if err := m.Run(context.Background(), []string{"generations", "-h"}); err != flag.ErrHelp {
t.Fatalf("unexpected error: %s", err)
}
})
}

View File

@@ -5,6 +5,7 @@ import (
"errors"
"flag"
"fmt"
"io"
"io/ioutil"
"log"
"net/url"
@@ -32,14 +33,14 @@ var (
Version = "(development build)"
)
// errStop is a terminal error for indicating program should quit.
var errStop = errors.New("stop")
// errExit is a terminal error for indicating program should quit.
var errExit = errors.New("exit")
func main() {
log.SetFlags(0)
m := NewMain()
if err := m.Run(context.Background(), os.Args[1:]); err == flag.ErrHelp || err == errStop {
m := NewMain(os.Stdin, os.Stdout, os.Stderr)
if err := m.Run(context.Background(), os.Args[1:]); err == flag.ErrHelp || err == errExit {
os.Exit(1)
} else if err != nil {
log.Println(err)
@@ -48,11 +49,19 @@ func main() {
}
// Main represents the main program execution.
type Main struct{}
type Main struct {
stdin io.Reader
stdout io.Writer
stderr io.Writer
}
// NewMain returns a new instance of Main.
func NewMain() *Main {
return &Main{}
func NewMain(stdin io.Reader, stdout, stderr io.Writer) *Main {
return &Main{
stdin: stdin,
stdout: stdout,
stderr: stderr,
}
}
// Run executes the program.
@@ -75,11 +84,11 @@ func (m *Main) Run(ctx context.Context, args []string) (err error) {
switch cmd {
case "databases":
return (&DatabasesCommand{}).Run(ctx, args)
return NewDatabasesCommand(m.stdin, m.stdout, m.stderr).Run(ctx, args)
case "generations":
return (&GenerationsCommand{}).Run(ctx, args)
return NewGenerationsCommand(m.stdin, m.stdout, m.stderr).Run(ctx, args)
case "replicate":
c := NewReplicateCommand()
c := NewReplicateCommand(m.stdin, m.stdout, m.stderr)
if err := c.ParseFlags(ctx, args); err != nil {
return err
}
@@ -96,21 +105,21 @@ func (m *Main) Run(ctx context.Context, args []string) (err error) {
// Wait for signal to stop program.
select {
case <-ctx.Done():
fmt.Println("context done, litestream shutting down")
fmt.Fprintln(m.stdout, "context done, litestream shutting down")
case err = <-c.execCh:
cancel()
fmt.Println("subprocess exited, litestream shutting down")
fmt.Fprintln(m.stdout, "subprocess exited, litestream shutting down")
case sig := <-signalCh:
cancel()
fmt.Println("signal received, litestream shutting down")
fmt.Fprintln(m.stdout, "signal received, litestream shutting down")
if c.cmd != nil {
fmt.Println("sending signal to exec process")
fmt.Fprintln(m.stdout, "sending signal to exec process")
if err := c.cmd.Process.Signal(sig); err != nil {
return fmt.Errorf("cannot signal exec process: %w", err)
}
fmt.Println("waiting for exec process to close")
fmt.Fprintln(m.stdout, "waiting for exec process to close")
if err := <-c.execCh; err != nil && !strings.HasPrefix(err.Error(), "signal:") {
return fmt.Errorf("cannot wait for exec process: %w", err)
}
@@ -121,17 +130,17 @@ func (m *Main) Run(ctx context.Context, args []string) (err error) {
if e := c.Close(); e != nil && err == nil {
err = e
}
fmt.Println("litestream shut down")
fmt.Fprintln(m.stdout, "litestream shut down")
return err
case "restore":
return NewRestoreCommand().Run(ctx, args)
return NewRestoreCommand(m.stdin, m.stdout, m.stderr).Run(ctx, args)
case "snapshots":
return (&SnapshotsCommand{}).Run(ctx, args)
return NewSnapshotsCommand(m.stdin, m.stdout, m.stderr).Run(ctx, args)
case "version":
return (&VersionCommand{}).Run(ctx, args)
return NewVersionCommand(m.stdin, m.stdout, m.stderr).Run(ctx, args)
case "wal":
return (&WALCommand{}).Run(ctx, args)
return NewWALCommand(m.stdin, m.stdout, m.stderr).Run(ctx, args)
default:
if cmd == "" || cmd == "help" || strings.HasPrefix(cmd, "-") {
m.Usage()
@@ -143,7 +152,7 @@ func (m *Main) Run(ctx context.Context, args []string) (err error) {
// Usage prints the help screen to STDOUT.
func (m *Main) Usage() {
fmt.Println(`
fmt.Fprintln(m.stdout, `
litestream is a tool for replicating SQLite databases.
Usage:
@@ -210,9 +219,15 @@ func (c *Config) DBConfig(path string) *DBConfig {
// ReadConfigFile unmarshals config from filename. Expands path if needed.
// If expandEnv is true then environment variables are expanded in the config.
// If filename is blank then the default config path is used.
func ReadConfigFile(filename string, expandEnv bool) (_ Config, err error) {
config := DefaultConfig()
useDefaultPath := filename == ""
if useDefaultPath {
filename = DefaultConfigPath()
}
// Expand filename, if necessary.
filename, err = expand(filename)
if err != nil {
@@ -220,8 +235,12 @@ func ReadConfigFile(filename string, expandEnv bool) (_ Config, err error) {
}
// Read configuration.
// Do not return an error if using default path and file is missing.
buf, err := ioutil.ReadFile(filename)
if os.IsNotExist(err) {
if useDefaultPath {
return config, nil
}
return config, fmt.Errorf("config file not found: %s", filename)
} else if err != nil {
return config, err
@@ -354,7 +373,7 @@ func NewReplicaFromConfig(c *ReplicaConfig, db *litestream.DB) (_ *litestream.Re
}
// Build and set client on replica.
switch c.ReplicaType() {
switch typ := c.ReplicaType(); typ {
case "file":
if r.Client, err = newFileReplicaClientFromConfig(c, r); err != nil {
return nil, err
@@ -376,7 +395,7 @@ func NewReplicaFromConfig(c *ReplicaConfig, db *litestream.DB) (_ *litestream.Re
return nil, err
}
default:
return nil, fmt.Errorf("unknown replica type in config: %q", c.Type)
return nil, fmt.Errorf("unknown replica type in config: %q", typ)
}
return r, nil
@@ -714,3 +733,45 @@ func (v *indexVar) Set(s string) error {
*v = indexVar(i)
return nil
}
// loadReplicas returns a list of replicas to use based on CLI flags. Filters
// by replicaName, if not blank. The DB is returned if pathOrURL is not a replica URL.
func loadReplicas(ctx context.Context, config Config, pathOrURL, replicaName string) ([]*litestream.Replica, *litestream.DB, error) {
// Build a replica based on URL, if specified.
if isURL(pathOrURL) {
r, err := NewReplicaFromConfig(&ReplicaConfig{
URL: pathOrURL,
AccessKeyID: config.AccessKeyID,
SecretAccessKey: config.SecretAccessKey,
}, nil)
if err != nil {
return nil, nil, err
}
return []*litestream.Replica{r}, nil, nil
}
// Otherwise use replicas from the database configuration file.
path, err := expand(pathOrURL)
if err != nil {
return nil, nil, err
}
dbc := config.DBConfig(path)
if dbc == nil {
return nil, nil, fmt.Errorf("database not found in config: %s", path)
}
db, err := NewDBFromConfig(dbc)
if err != nil {
return nil, nil, err
}
// Filter by replica, if specified.
if replicaName != "" {
r := db.Replica(replicaName)
if r == nil {
return nil, nil, fmt.Errorf("replica %q not found for database %q", replicaName, db.Path())
}
return []*litestream.Replica{r}, db, nil
}
return db.Replicas, db, nil
}

View File

@@ -1,6 +1,8 @@
package main_test
import (
"bytes"
"io"
"io/ioutil"
"log"
"os"
@@ -180,3 +182,17 @@ func TestNewGCSReplicaFromConfig(t *testing.T) {
t.Fatalf("Path=%s, want %s", got, want)
}
}
// newMain returns a new instance of Main and associated buffers.
func newMain() (m *main.Main, stdin, stdout, stderr *bytes.Buffer) {
stdin, stdout, stderr = &bytes.Buffer{}, &bytes.Buffer{}, &bytes.Buffer{}
// Split stdout/stderr to terminal if verbose flag set.
out, err := io.Writer(stdout), io.Writer(stderr)
if testing.Verbose() {
out = io.MultiWriter(out, os.Stdout)
err = io.MultiWriter(err, os.Stderr)
}
return main.NewMain(stdin, out, err), stdin, stdout, stderr
}

View File

@@ -1,3 +1,4 @@
//go:build windows
// +build windows
package main
@@ -41,7 +42,7 @@ func runWindowsService(ctx context.Context) error {
log.Print("Litestream service starting")
if err := svc.Run(serviceName, &windowsService{ctx: ctx}); err != nil {
return errStop
return errExit
}
log.Print("Litestream service stopped")

View File

@@ -4,6 +4,7 @@ import (
"context"
"flag"
"fmt"
"io"
"log"
"net"
"net/http"
@@ -22,6 +23,10 @@ import (
// ReplicateCommand represents a command that continuously replicates SQLite databases.
type ReplicateCommand struct {
stdin io.Reader
stdout io.Writer
stderr io.Writer
configPath string
noExpandEnv bool
@@ -34,8 +39,13 @@ type ReplicateCommand struct {
DBs []*litestream.DB
}
func NewReplicateCommand() *ReplicateCommand {
// NewReplicateCommand returns a new instance of ReplicateCommand.
func NewReplicateCommand(stdin io.Reader, stdout, stderr io.Writer) *ReplicateCommand {
return &ReplicateCommand{
stdin: stdin,
stdout: stdout,
stderr: stderr,
execCh: make(chan error),
}
}
@@ -181,7 +191,7 @@ func (c *ReplicateCommand) Close() (err error) {
// Usage prints the help screen to STDOUT.
func (c *ReplicateCommand) Usage() {
fmt.Printf(`
fmt.Fprintf(c.stdout, `
The replicate command starts a server to monitor & replicate databases.
You can specify your database & replicas in a configuration file or you can
replicate a single database file by specifying its path and its replicas in the

View File

@@ -13,7 +13,6 @@ import (
"testing"
"time"
main "github.com/benbjohnson/litestream/cmd/litestream"
"golang.org/x/sync/errgroup"
)
@@ -82,7 +81,8 @@ dbs:
// Replicate database unless the context is canceled.
g.Go(func() error {
return main.NewMain().Run(mainctx, []string{"replicate", "-config", configPath})
m, _, _, _ := newMain()
return m.Run(mainctx, []string{"replicate", "-config", configPath})
})
if err := g.Wait(); err != nil {
@@ -94,7 +94,8 @@ dbs:
chksum0 := mustChecksum(t, dbPath)
// Restore to another path.
if err := main.NewMain().Run(context.Background(), []string{"restore", "-config", configPath, "-o", restorePath, dbPath}); err != nil && !errors.Is(err, context.Canceled) {
m, _, _, _ := newMain()
if err := m.Run(context.Background(), []string{"restore", "-config", configPath, "-o", restorePath, dbPath}); err != nil && !errors.Is(err, context.Canceled) {
t.Fatal(err)
}

View File

@@ -5,6 +5,7 @@ import (
"errors"
"flag"
"fmt"
"io"
"log"
"os"
"path/filepath"
@@ -15,6 +16,10 @@ import (
// RestoreCommand represents a command to restore a database from a backup.
type RestoreCommand struct {
stdin io.Reader
stdout io.Writer
stderr io.Writer
snapshotIndex int // index of snapshot to start from
// CLI options
@@ -29,8 +34,13 @@ type RestoreCommand struct {
opt litestream.RestoreOptions
}
func NewRestoreCommand() *RestoreCommand {
// NewRestoreCommand returns a new instance of RestoreCommand.
func NewRestoreCommand(stdin io.Reader, stdout, stderr io.Writer) *RestoreCommand {
return &RestoreCommand{
stdin: stdin,
stdout: stdout,
stderr: stderr,
targetIndex: -1,
opt: litestream.NewRestoreOptions(),
}
@@ -55,31 +65,39 @@ func (c *RestoreCommand) Run(ctx context.Context, args []string) (err error) {
} else if fs.NArg() > 1 {
return fmt.Errorf("too many arguments")
}
arg := fs.Arg(0)
pathOrURL := fs.Arg(0)
// Ensure a generation is specified if target index is specified.
if c.targetIndex != -1 && c.generation == "" {
return fmt.Errorf("must specify -generation when using -index flag")
return fmt.Errorf("must specify -generation flag when using -index flag")
}
// Default to original database path if output path not specified.
if !isURL(arg) && c.outputPath == "" {
c.outputPath = arg
if !isURL(pathOrURL) && c.outputPath == "" {
c.outputPath = pathOrURL
}
// Exit successfully if the output file already exists and flag is set.
if _, err := os.Stat(c.outputPath); !os.IsNotExist(err) && c.ifDBNotExists {
fmt.Println("database already exists, skipping")
return nil
if _, err := os.Stat(c.outputPath); os.IsNotExist(err) {
// file doesn't exist, continue
} else if err != nil {
return err
} else if err == nil {
if c.ifDBNotExists {
fmt.Fprintln(c.stdout, "database already exists, skipping")
return nil
}
return fmt.Errorf("output file already exists: %s", c.outputPath)
}
// Create parent directory if it doesn't already exist.
if err := os.MkdirAll(filepath.Dir(c.outputPath), 0700); err != nil {
return fmt.Errorf("cannot create parent directory: %w", err)
// Load configuration.
config, err := ReadConfigFile(c.configPath, !c.noExpandEnv)
if err != nil {
return err
}
// Build replica from either a URL or config.
r, err := c.loadReplica(ctx, arg)
r, err := c.loadReplica(ctx, config, pathOrURL)
if err != nil {
return err
}
@@ -90,7 +108,7 @@ func (c *RestoreCommand) Run(ctx context.Context, args []string) (err error) {
// Return an error if no matching targets found.
// If optional flag set, return success. Useful for automated recovery.
if c.ifReplicaExists {
fmt.Println("no matching backups found")
fmt.Fprintln(c.stdout, "no matching backups found, skipping")
return nil
}
return fmt.Errorf("no matching backups found")
@@ -112,47 +130,42 @@ func (c *RestoreCommand) Run(ctx context.Context, args []string) (err error) {
return fmt.Errorf("cannot find snapshot index: %w", err)
}
c.opt.Logger = log.New(os.Stderr, "", log.LstdFlags|log.Lmicroseconds)
// Create parent directory if it doesn't already exist.
if err := os.MkdirAll(filepath.Dir(c.outputPath), 0700); err != nil {
return fmt.Errorf("cannot create parent directory: %w", err)
}
c.opt.Logger = log.New(c.stdout, "", log.LstdFlags|log.Lmicroseconds)
return litestream.Restore(ctx, r.Client, c.outputPath, c.generation, c.snapshotIndex, c.targetIndex, c.opt)
}
func (c *RestoreCommand) loadReplica(ctx context.Context, arg string) (*litestream.Replica, error) {
func (c *RestoreCommand) loadReplica(ctx context.Context, config Config, arg string) (*litestream.Replica, error) {
if isURL(arg) {
return c.loadReplicaFromURL(ctx, arg)
return c.loadReplicaFromURL(ctx, config, arg)
}
return c.loadReplicaFromConfig(ctx, arg)
return c.loadReplicaFromConfig(ctx, config, arg)
}
// loadReplicaFromURL creates a replica & updates the restore options from a replica URL.
func (c *RestoreCommand) loadReplicaFromURL(ctx context.Context, replicaURL string) (*litestream.Replica, error) {
if c.configPath != "" {
return nil, fmt.Errorf("cannot specify a replica URL and the -config flag")
} else if c.replicaName != "" {
return nil, fmt.Errorf("cannot specify a replica URL and the -replica flag")
func (c *RestoreCommand) loadReplicaFromURL(ctx context.Context, config Config, replicaURL string) (*litestream.Replica, error) {
if c.replicaName != "" {
return nil, fmt.Errorf("cannot specify both the replica URL and the -replica flag")
} else if c.outputPath == "" {
return nil, fmt.Errorf("output path required")
return nil, fmt.Errorf("output path required when using a replica URL")
}
syncInterval := litestream.DefaultSyncInterval
return NewReplicaFromConfig(&ReplicaConfig{
URL: replicaURL,
SyncInterval: &syncInterval,
URL: replicaURL,
AccessKeyID: config.AccessKeyID,
SecretAccessKey: config.SecretAccessKey,
SyncInterval: &syncInterval,
}, nil)
}
// loadReplicaFromConfig returns replicas based on the specific config path.
func (c *RestoreCommand) loadReplicaFromConfig(ctx context.Context, dbPath string) (*litestream.Replica, error) {
if c.configPath == "" {
c.configPath = DefaultConfigPath()
}
// Load configuration.
config, err := ReadConfigFile(c.configPath, !c.noExpandEnv)
if err != nil {
return nil, err
}
func (c *RestoreCommand) loadReplicaFromConfig(ctx context.Context, config Config, dbPath string) (_ *litestream.Replica, err error) {
// Lookup database from configuration file by path.
if dbPath, err = expand(dbPath); err != nil {
return nil, err
@@ -184,7 +197,7 @@ func (c *RestoreCommand) loadReplicaFromConfig(ctx context.Context, dbPath strin
// A replica must be specified when restoring a specific generation with multiple replicas.
if c.generation != "" {
return nil, fmt.Errorf("must specify -replica when restoring from a specific generation")
return nil, fmt.Errorf("must specify -replica flag when restoring from a specific generation")
}
// Determine latest replica to restore from.
@@ -197,7 +210,7 @@ func (c *RestoreCommand) loadReplicaFromConfig(ctx context.Context, dbPath strin
// Usage prints the help screen to STDOUT.
func (c *RestoreCommand) Usage() {
fmt.Printf(`
fmt.Fprintf(c.stdout, `
The restore command recovers a database from a previous snapshot and WAL.
Usage:

View File

@@ -0,0 +1,330 @@
package main_test
import (
"context"
"flag"
"os"
"path/filepath"
"strings"
"testing"
"github.com/benbjohnson/litestream/internal/testingutil"
)
func TestRestoreCommand(t *testing.T) {
t.Run("OK", func(t *testing.T) {
testDir := filepath.Join("testdata", "restore", "ok")
defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
tempDir := t.TempDir()
m, _, stdout, stderr := newMain()
if err := m.Run(context.Background(), []string{"restore", "-config", filepath.Join(testDir, "litestream.yml"), "-o", filepath.Join(tempDir, "db"), filepath.Join(testDir, "db")}); err != nil {
t.Fatal(err)
} else if got, want := stderr.String(), ""; got != want {
t.Fatalf("stderr=%q, want %q", got, want)
}
// STDOUT has timing info so we need to grep per line.
lines := strings.Split(stdout.String(), "\n")
for i, substr := range []string{
`restoring snapshot 0000000000000000/00000000 to ` + filepath.Join(tempDir, "db.tmp"),
`applied wal 0000000000000000/00000000 elapsed=`,
`applied wal 0000000000000000/00000001 elapsed=`,
`applied wal 0000000000000000/00000002 elapsed=`,
`renaming database from temporary location`,
} {
if !strings.Contains(lines[i], substr) {
t.Fatalf("stdout: unexpected line %d:\n%s", i+1, stdout)
}
}
})
t.Run("ReplicaName", func(t *testing.T) {
testDir := filepath.Join("testdata", "restore", "replica-name")
defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
tempDir := t.TempDir()
m, _, stdout, stderr := newMain()
if err := m.Run(context.Background(), []string{"restore", "-config", filepath.Join(testDir, "litestream.yml"), "-o", filepath.Join(tempDir, "db"), "-replica", "replica1", filepath.Join(testDir, "db")}); err != nil {
t.Fatal(err)
} else if got, want := stderr.String(), ""; got != want {
t.Fatalf("stderr=%q, want %q", got, want)
}
// STDOUT has timing info so we need to grep per line.
lines := strings.Split(stdout.String(), "\n")
for i, substr := range []string{
`restoring snapshot 0000000000000001/00000001 to ` + filepath.Join(tempDir, "db.tmp"),
`no wal files found, snapshot only`,
`renaming database from temporary location`,
} {
if !strings.Contains(lines[i], substr) {
t.Fatalf("stdout: unexpected line %d:\n%s", i+1, stdout)
}
}
})
t.Run("ReplicaURL", func(t *testing.T) {
testDir := filepath.Join(testingutil.Getwd(t), "testdata", "restore", "replica-url")
tempDir := t.TempDir()
replicaURL := "file://" + filepath.ToSlash(testDir) + "/replica"
m, _, stdout, stderr := newMain()
if err := m.Run(context.Background(), []string{"restore", "-o", filepath.Join(tempDir, "db"), replicaURL}); err != nil {
t.Fatal(err)
} else if got, want := stderr.String(), ""; got != want {
t.Fatalf("stderr=%q, want %q", got, want)
}
lines := strings.Split(stdout.String(), "\n")
for i, substr := range []string{
`restoring snapshot 0000000000000000/00000000 to ` + filepath.Join(tempDir, "db.tmp"),
`no wal files found, snapshot only`,
`renaming database from temporary location`,
} {
if !strings.Contains(lines[i], substr) {
t.Fatalf("stdout: unexpected line %d:\n%s", i+1, stdout)
}
}
})
t.Run("LatestReplica", func(t *testing.T) {
testDir := filepath.Join("testdata", "restore", "latest-replica")
defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
tempDir := t.TempDir()
m, _, stdout, stderr := newMain()
if err := m.Run(context.Background(), []string{"restore", "-config", filepath.Join(testDir, "litestream.yml"), "-o", filepath.Join(tempDir, "db"), filepath.Join(testDir, "db")}); err != nil {
t.Fatal(err)
} else if got, want := stderr.String(), ""; got != want {
t.Fatalf("stderr=%q, want %q", got, want)
}
lines := strings.Split(stdout.String(), "\n")
for i, substr := range []string{
`restoring snapshot 0000000000000001/00000000 to ` + filepath.Join(tempDir, "db.tmp"),
`no wal files found, snapshot only`,
`renaming database from temporary location`,
} {
if !strings.Contains(lines[i], substr) {
t.Fatalf("stdout: unexpected line %d:\n%s", i+1, stdout)
}
}
})
t.Run("IfDBNotExistsFlag", func(t *testing.T) {
testDir := filepath.Join("testdata", "restore", "if-db-not-exists-flag")
defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
m, _, stdout, _ := newMain()
err := m.Run(context.Background(), []string{"restore", "-config", filepath.Join(testDir, "litestream.yml"), "-if-db-not-exists", filepath.Join(testDir, "db")})
if err != nil {
t.Fatal(err)
} else if got, want := stdout.String(), string(testingutil.MustReadFile(t, filepath.Join(testDir, "stdout"))); got != want {
t.Fatalf("stdout=%q, want %q", got, want)
}
})
t.Run("IfReplicaExists", func(t *testing.T) {
testDir := filepath.Join("testdata", "restore", "if-replica-exists-flag")
defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
m, _, stdout, _ := newMain()
err := m.Run(context.Background(), []string{"restore", "-config", filepath.Join(testDir, "litestream.yml"), "-if-replica-exists", filepath.Join(testDir, "db")})
if err != nil {
t.Fatal(err)
} else if got, want := stdout.String(), string(testingutil.MustReadFile(t, filepath.Join(testDir, "stdout"))); got != want {
t.Fatalf("stdout=%q, want %q", got, want)
}
})
t.Run("ErrNoBackups", func(t *testing.T) {
testDir := filepath.Join("testdata", "restore", "no-backups")
defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
tempDir := t.TempDir()
m, _, stdout, stderr := newMain()
err := m.Run(context.Background(), []string{"restore", "-config", filepath.Join(testDir, "litestream.yml"), "-o", filepath.Join(tempDir, "db"), filepath.Join(testDir, "db")})
if err == nil || err.Error() != `no matching backups found` {
t.Fatalf("unexpected error: %s", err)
} else if got, want := stdout.String(), string(testingutil.MustReadFile(t, filepath.Join(testDir, "stdout"))); got != want {
t.Fatalf("stdout=%q, want %q", got, want)
} else if got, want := stderr.String(), string(testingutil.MustReadFile(t, filepath.Join(testDir, "stderr"))); got != want {
t.Fatalf("stderr=%q, want %q", got, want)
}
})
t.Run("ErrNoGeneration", func(t *testing.T) {
testDir := filepath.Join("testdata", "restore", "no-generation")
defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
m, _, _, _ := newMain()
err := m.Run(context.Background(), []string{"restore", "-config", filepath.Join(testDir, "litestream.yml"), filepath.Join(testDir, "db")})
if err == nil || err.Error() != `no matching backups found` {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrOutputPathExists", func(t *testing.T) {
testDir := filepath.Join("testdata", "restore", "output-path-exists")
defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
m, _, _, _ := newMain()
err := m.Run(context.Background(), []string{"restore", "-config", filepath.Join(testDir, "litestream.yml"), filepath.Join(testDir, "db")})
if err == nil || err.Error() != `output file already exists: `+filepath.Join(testDir, "db") {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrDatabaseOrReplicaRequired", func(t *testing.T) {
m, _, _, _ := newMain()
err := m.Run(context.Background(), []string{"restore"})
if err == nil || err.Error() != `database path or replica URL required` {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrTooManyArguments", func(t *testing.T) {
m, _, _, _ := newMain()
err := m.Run(context.Background(), []string{"restore", "abc", "123"})
if err == nil || err.Error() != `too many arguments` {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrInvalidFlags", func(t *testing.T) {
m, _, _, _ := newMain()
err := m.Run(context.Background(), []string{"restore", "-no-such-flag"})
if err == nil || err.Error() != `flag provided but not defined: -no-such-flag` {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrIndexFlagOnly", func(t *testing.T) {
m, _, _, _ := newMain()
err := m.Run(context.Background(), []string{"restore", "-index", "0", "/var/lib/db"})
if err == nil || err.Error() != `must specify -generation flag when using -index flag` {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrConfigFileNotFound", func(t *testing.T) {
m, _, _, _ := newMain()
err := m.Run(context.Background(), []string{"restore", "-config", "/no/such/file", "/var/lib/db"})
if err == nil || err.Error() != `config file not found: /no/such/file` {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrInvalidConfig", func(t *testing.T) {
testDir := filepath.Join("testdata", "restore", "invalid-config")
m, _, _, _ := newMain()
err := m.Run(context.Background(), []string{"restore", "-config", filepath.Join(testDir, "litestream.yml"), "/var/lib/db"})
if err == nil || !strings.Contains(err.Error(), `replica path cannot be a url`) {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrMkdir", func(t *testing.T) {
tempDir := t.TempDir()
if err := os.Mkdir(filepath.Join(tempDir, "noperm"), 0000); err != nil {
t.Fatal(err)
}
m, _, _, _ := newMain()
err := m.Run(context.Background(), []string{"restore", "-o", filepath.Join(tempDir, "noperm", "subdir", "db"), "/var/lib/db"})
if err == nil || !strings.Contains(err.Error(), `permission denied`) {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrNoOutputPathWithReplicaURL", func(t *testing.T) {
m, _, _, _ := newMain()
err := m.Run(context.Background(), []string{"restore", "file://path/to/replica"})
if err == nil || err.Error() != `output path required when using a replica URL` {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrReplicaNameWithReplicaURL", func(t *testing.T) {
m, _, _, _ := newMain()
err := m.Run(context.Background(), []string{"restore", "-replica", "replica0", "file://path/to/replica"})
if err == nil || err.Error() != `cannot specify both the replica URL and the -replica flag` {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrInvalidReplicaURL", func(t *testing.T) {
m, _, _, _ := newMain()
err := m.Run(context.Background(), []string{"restore", "-o", "/tmp/db", "xyz://xyz"})
if err == nil || err.Error() != `unknown replica type in config: "xyz"` {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrDatabaseNotFound", func(t *testing.T) {
testDir := filepath.Join("testdata", "restore", "database-not-found")
defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
m, _, _, _ := newMain()
err := m.Run(context.Background(), []string{"restore", "-config", filepath.Join(testDir, "litestream.yml"), "/no/such/db"})
if err == nil || err.Error() != `database not found in config: /no/such/db` {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrNoReplicas", func(t *testing.T) {
testDir := filepath.Join("testdata", "restore", "no-replicas")
defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
tempDir := t.TempDir()
m, _, _, _ := newMain()
err := m.Run(context.Background(), []string{"restore", "-config", filepath.Join(testDir, "litestream.yml"), "-o", filepath.Join(tempDir, "db"), filepath.Join(testDir, "db")})
if err == nil || err.Error() != `database has no replicas: `+filepath.Join(testingutil.Getwd(t), testDir, "db") {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrReplicaNotFound", func(t *testing.T) {
testDir := filepath.Join("testdata", "restore", "replica-not-found")
defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
tempDir := t.TempDir()
m, _, _, _ := newMain()
err := m.Run(context.Background(), []string{"restore", "-config", filepath.Join(testDir, "litestream.yml"), "-o", filepath.Join(tempDir, "db"), "-replica", "no_such_replica", filepath.Join(testDir, "db")})
if err == nil || err.Error() != `replica "no_such_replica" not found` {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrGenerationWithNoReplicaName", func(t *testing.T) {
testDir := filepath.Join("testdata", "restore", "generation-with-no-replica")
defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
tempDir := t.TempDir()
m, _, _, _ := newMain()
err := m.Run(context.Background(), []string{"restore", "-config", filepath.Join(testDir, "litestream.yml"), "-o", filepath.Join(tempDir, "db"), "-generation", "0000000000000000", filepath.Join(testDir, "db")})
if err == nil || err.Error() != `must specify -replica flag when restoring from a specific generation` {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrNoSnapshotsAvailable", func(t *testing.T) {
testDir := filepath.Join("testdata", "restore", "no-snapshots")
defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
tempDir := t.TempDir()
m, _, _, _ := newMain()
err := m.Run(context.Background(), []string{"restore", "-config", filepath.Join(testDir, "litestream.yml"), "-o", filepath.Join(tempDir, "db"), "-generation", "0000000000000000", filepath.Join(testDir, "db")})
if err == nil || err.Error() != `cannot determine latest index in generation "0000000000000000": no snapshots available` {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("Usage", func(t *testing.T) {
m, _, _, _ := newMain()
if err := m.Run(context.Background(), []string{"restore", "-h"}); err != flag.ErrHelp {
t.Fatalf("unexpected error: %s", err)
}
})
}

View File

@@ -4,8 +4,8 @@ import (
"context"
"flag"
"fmt"
"io"
"log"
"os"
"sort"
"text/tabwriter"
"time"
@@ -15,99 +15,89 @@ import (
// SnapshotsCommand represents a command to list snapshots for a command.
type SnapshotsCommand struct {
stdin io.Reader
stdout io.Writer
stderr io.Writer
configPath string
noExpandEnv bool
replicaName string
}
// NewSnapshotsCommand returns a new instance of SnapshotsCommand.
func NewSnapshotsCommand(stdin io.Reader, stdout, stderr io.Writer) *SnapshotsCommand {
return &SnapshotsCommand{
stdin: stdin,
stdout: stdout,
stderr: stderr,
}
}
// Run executes the command.
func (c *SnapshotsCommand) Run(ctx context.Context, args []string) (err error) {
func (c *SnapshotsCommand) Run(ctx context.Context, args []string) (ret error) {
fs := flag.NewFlagSet("litestream-snapshots", flag.ContinueOnError)
registerConfigFlag(fs, &c.configPath, &c.noExpandEnv)
replicaName := fs.String("replica", "", "replica name")
fs.StringVar(&c.replicaName, "replica", "", "replica name")
fs.Usage = c.Usage
if err := fs.Parse(args); err != nil {
return err
} else if fs.NArg() == 0 || fs.Arg(0) == "" {
return fmt.Errorf("database path required")
return fmt.Errorf("database path or replica URL required")
} else if fs.NArg() > 1 {
return fmt.Errorf("too many arguments")
}
var db *litestream.DB
var r *litestream.Replica
if isURL(fs.Arg(0)) {
if c.configPath != "" {
return fmt.Errorf("cannot specify a replica URL and the -config flag")
}
if r, err = NewReplicaFromConfig(&ReplicaConfig{URL: fs.Arg(0)}, nil); err != nil {
return err
}
} else {
if c.configPath == "" {
c.configPath = DefaultConfigPath()
}
// Load configuration.
config, err := ReadConfigFile(c.configPath, !c.noExpandEnv)
if err != nil {
return err
}
// Load configuration.
config, err := ReadConfigFile(c.configPath, !c.noExpandEnv)
// Determine list of replicas to pull snapshots from.
replicas, _, err := loadReplicas(ctx, config, fs.Arg(0), c.replicaName)
if err != nil {
return err
}
// Build list of snapshot metadata with associated replica.
var infos []replicaSnapshotInfo
for _, r := range replicas {
a, err := r.Snapshots(ctx)
if err != nil {
return err
log.Printf("cannot determine snapshots: %s", err)
ret = errExit // signal error return without printing message
continue
}
// Lookup database from configuration file by path.
if path, err := expand(fs.Arg(0)); err != nil {
return err
} else if dbc := config.DBConfig(path); dbc == nil {
return fmt.Errorf("database not found in config: %s", path)
} else if db, err = NewDBFromConfig(dbc); err != nil {
return err
}
// Filter by replica, if specified.
if *replicaName != "" {
if r = db.Replica(*replicaName); r == nil {
return fmt.Errorf("replica %q not found for database %q", *replicaName, db.Path())
}
for i := range a {
infos = append(infos, replicaSnapshotInfo{SnapshotInfo: a[i], replicaName: r.Name()})
}
}
// Find snapshots by db or replica.
var replicas []*litestream.Replica
if r != nil {
replicas = []*litestream.Replica{r}
} else {
replicas = db.Replicas
}
// Sort snapshots by creation time from newest to oldest.
sort.Slice(infos, func(i, j int) bool { return infos[i].CreatedAt.After(infos[j].CreatedAt) })
// List all snapshots.
w := tabwriter.NewWriter(os.Stdout, 0, 8, 2, ' ', 0)
w := tabwriter.NewWriter(c.stdout, 0, 8, 2, ' ', 0)
defer w.Flush()
fmt.Fprintln(w, "replica\tgeneration\tindex\tsize\tcreated")
for _, r := range replicas {
infos, err := r.Snapshots(ctx)
if err != nil {
log.Printf("cannot determine snapshots: %s", err)
continue
}
// Sort snapshots by creation time from newest to oldest.
sort.Slice(infos, func(i, j int) bool { return infos[i].CreatedAt.After(infos[j].CreatedAt) })
for _, info := range infos {
fmt.Fprintf(w, "%s\t%s\t%d\t%d\t%s\n",
r.Name(),
info.Generation,
info.Index,
info.Size,
info.CreatedAt.Format(time.RFC3339),
)
}
for _, info := range infos {
fmt.Fprintf(w, "%s\t%s\t%08x\t%d\t%s\n",
info.replicaName,
info.Generation,
info.Index,
info.Size,
info.CreatedAt.Format(time.RFC3339),
)
}
return nil
return ret
}
// Usage prints the help screen to STDOUT.
func (c *SnapshotsCommand) Usage() {
fmt.Printf(`
fmt.Fprintf(c.stdout, `
The snapshots command lists all snapshots available for a database or replica.
Usage:
@@ -143,3 +133,9 @@ Examples:
DefaultConfigPath(),
)
}
// replicaSnapshotInfo represents snapshot metadata with associated replica name.
type replicaSnapshotInfo struct {
litestream.SnapshotInfo
replicaName string
}

View File

@@ -0,0 +1,128 @@
package main_test
import (
"context"
"flag"
"path/filepath"
"strings"
"testing"
"github.com/benbjohnson/litestream/internal/testingutil"
)
func TestSnapshotsCommand(t *testing.T) {
t.Run("OK", func(t *testing.T) {
testDir := filepath.Join("testdata", "snapshots", "ok")
defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
m, _, stdout, _ := newMain()
if err := m.Run(context.Background(), []string{"snapshots", "-config", filepath.Join(testDir, "litestream.yml"), filepath.Join(testDir, "db")}); err != nil {
t.Fatal(err)
} else if got, want := stdout.String(), string(testingutil.MustReadFile(t, filepath.Join(testDir, "stdout"))); got != want {
t.Fatalf("stdout=%q, want %q", got, want)
}
})
t.Run("ReplicaName", func(t *testing.T) {
testDir := filepath.Join("testdata", "snapshots", "replica-name")
defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
m, _, stdout, _ := newMain()
if err := m.Run(context.Background(), []string{"snapshots", "-config", filepath.Join(testDir, "litestream.yml"), "-replica", "replica1", filepath.Join(testDir, "db")}); err != nil {
t.Fatal(err)
} else if got, want := stdout.String(), string(testingutil.MustReadFile(t, filepath.Join(testDir, "stdout"))); got != want {
t.Fatalf("stdout=%q, want %q", got, want)
}
})
t.Run("ReplicaURL", func(t *testing.T) {
testDir := filepath.Join(testingutil.Getwd(t), "testdata", "snapshots", "replica-url")
defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
replicaURL := "file://" + filepath.ToSlash(testDir) + "/replica"
m, _, stdout, _ := newMain()
if err := m.Run(context.Background(), []string{"snapshots", replicaURL}); err != nil {
t.Fatal(err)
} else if got, want := stdout.String(), string(testingutil.MustReadFile(t, filepath.Join(testDir, "stdout"))); got != want {
t.Fatalf("stdout=%q, want %q", got, want)
}
})
t.Run("ErrDatabaseOrReplicaRequired", func(t *testing.T) {
m, _, _, _ := newMain()
err := m.Run(context.Background(), []string{"snapshots"})
if err == nil || err.Error() != `database path or replica URL required` {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrTooManyArguments", func(t *testing.T) {
m, _, _, _ := newMain()
err := m.Run(context.Background(), []string{"snapshots", "abc", "123"})
if err == nil || err.Error() != `too many arguments` {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrInvalidFlags", func(t *testing.T) {
m, _, _, _ := newMain()
err := m.Run(context.Background(), []string{"snapshots", "-no-such-flag"})
if err == nil || err.Error() != `flag provided but not defined: -no-such-flag` {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrConfigFileNotFound", func(t *testing.T) {
m, _, _, _ := newMain()
err := m.Run(context.Background(), []string{"snapshots", "-config", "/no/such/file", "/var/lib/db"})
if err == nil || err.Error() != `config file not found: /no/such/file` {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrInvalidConfig", func(t *testing.T) {
testDir := filepath.Join("testdata", "snapshots", "invalid-config")
m, _, _, _ := newMain()
err := m.Run(context.Background(), []string{"snapshots", "-config", filepath.Join(testDir, "litestream.yml"), "/var/lib/db"})
if err == nil || !strings.Contains(err.Error(), `replica path cannot be a url`) {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrDatabaseNotFound", func(t *testing.T) {
testDir := filepath.Join("testdata", "snapshots", "database-not-found")
defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
m, _, _, _ := newMain()
err := m.Run(context.Background(), []string{"snapshots", "-config", filepath.Join(testDir, "litestream.yml"), "/no/such/db"})
if err == nil || err.Error() != `database not found in config: /no/such/db` {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrReplicaNotFound", func(t *testing.T) {
testDir := filepath.Join(testingutil.Getwd(t), "testdata", "snapshots", "replica-not-found")
defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
m, _, _, _ := newMain()
err := m.Run(context.Background(), []string{"snapshots", "-config", filepath.Join(testDir, "litestream.yml"), "-replica", "no_such_replica", filepath.Join(testDir, "db")})
if err == nil || err.Error() != `replica "no_such_replica" not found for database "`+filepath.Join(testDir, "db")+`"` {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrInvalidReplicaURL", func(t *testing.T) {
m, _, _, _ := newMain()
err := m.Run(context.Background(), []string{"snapshots", "xyz://xyz"})
if err == nil || !strings.Contains(err.Error(), `unknown replica type in config: "xyz"`) {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("Usage", func(t *testing.T) {
m, _, _, _ := newMain()
if err := m.Run(context.Background(), []string{"snapshots", "-h"}); err != flag.ErrHelp {
t.Fatalf("unexpected error: %s", err)
}
})
}

13
cmd/litestream/testdata/Makefile vendored Normal file
View File

@@ -0,0 +1,13 @@
.PHONY: default
default:
make -C generations/ok
make -C generations/no-database
make -C generations/replica-name
make -C generations/replica-url
make -C restore/latest-replica
make -C snapshots/ok
make -C snapshots/replica-name
make -C snapshots/replica-url
make -C wal/ok
make -C wal/replica-name
make -C wal/replica-url

View File

@@ -0,0 +1,4 @@
dbs:
- path: /var/lib/db
replicas:
- path: s3://bkt/db

View File

View File

@@ -0,0 +1 @@
dbs:

View File

@@ -0,0 +1 @@
No databases found in config file.

View File

@@ -0,0 +1,7 @@
dbs:
- path: /var/lib/db
replicas:
- path: /var/lib/replica
- url: s3://mybkt/db
- path: /my/other/db

View File

@@ -0,0 +1,3 @@
path replicas
/var/lib/db file,s3
/my/other/db

View File

@@ -0,0 +1,2 @@
dbs:
- path: $LITESTREAM_TESTDIR/db

View File

@@ -0,0 +1,4 @@
dbs:
- path: /var/lib/db
replicas:
- path: s3://bkt/db

View File

@@ -0,0 +1,4 @@
.PHONY: default
default:
TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/snapshots/00000000.snapshot.lz4
TZ=UTC touch -ct 200001020000 replica/generations/0000000000000001/snapshots/00000000.snapshot.lz4

View File

@@ -0,0 +1,4 @@
dbs:
- path: $LITESTREAM_TESTDIR/db
replicas:
- path: $LITESTREAM_TESTDIR/replica

View File

@@ -0,0 +1,3 @@
name generation lag start end
file 0000000000000000 - 2000-01-01T00:00:00Z 2000-01-01T00:00:00Z
file 0000000000000001 - 2000-01-02T00:00:00Z 2000-01-02T00:00:00Z

View File

@@ -0,0 +1,9 @@
.PHONY: default
default:
TZ=UTC touch -ct 200001030000 db
TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/snapshots/00000000.snapshot.lz4
TZ=UTC touch -ct 200001020000 replica/generations/0000000000000000/snapshots/00000001.snapshot.lz4
TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/wal/00000000/00000000.wal.lz4
TZ=UTC touch -ct 200001020000 replica/generations/0000000000000000/wal/00000000/00000001.wal.lz4
TZ=UTC touch -ct 200001030000 replica/generations/0000000000000000/wal/00000001/00000000.wal.lz4
TZ=UTC touch -ct 200001010000 replica/generations/0000000000000001/snapshots/00000000.snapshot.lz4

View File

View File

@@ -0,0 +1,4 @@
dbs:
- path: $LITESTREAM_TESTDIR/db
replicas:
- path: $LITESTREAM_TESTDIR/replica

View File

View File

@@ -0,0 +1,3 @@
name generation lag start end
file 0000000000000000 0s 2000-01-01T00:00:00Z 2000-01-03T00:00:00Z
file 0000000000000001 48h0m0s 2000-01-01T00:00:00Z 2000-01-01T00:00:00Z

View File

@@ -0,0 +1,5 @@
.PHONY: default
default:
TZ=UTC touch -ct 200001030000 db
TZ=UTC touch -ct 200001010000 replica0/generations/0000000000000000/snapshots/00000000.snapshot.lz4
TZ=UTC touch -ct 200001020000 replica1/generations/0000000000000001/snapshots/00000000.snapshot.lz4

View File

View File

@@ -0,0 +1,7 @@
dbs:
- path: $LITESTREAM_TESTDIR/db
replicas:
- name: replica0
path: $LITESTREAM_TESTDIR/replica0
- name: replica1
path: $LITESTREAM_TESTDIR/replica1

View File

@@ -0,0 +1,2 @@
name generation lag start end
replica1 0000000000000001 24h0m0s 2000-01-02T00:00:00Z 2000-01-02T00:00:00Z

View File

@@ -0,0 +1,4 @@
dbs:
- path: $LITESTREAM_TESTDIR/db
replicas:
- url: s3://bkt/db

View File

@@ -0,0 +1,9 @@
.PHONY: default
default:
TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/snapshots/00000000.snapshot.lz4
TZ=UTC touch -ct 200001020000 replica/generations/0000000000000000/snapshots/00000001.snapshot.lz4
TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/wal/00000000/00000000.wal.lz4
TZ=UTC touch -ct 200001020000 replica/generations/0000000000000000/wal/00000000/00000001.wal.lz4
TZ=UTC touch -ct 200001030000 replica/generations/0000000000000000/wal/00000001/00000000.wal.lz4
TZ=UTC touch -ct 200001020000 replica/generations/0000000000000001/snapshots/00000000.snapshot.lz4

View File

@@ -0,0 +1,4 @@
dbs:
- path: $LITESTREAM_TESTDIR/db
replicas:
- path: $LITESTREAM_TESTDIR/replica

View File

@@ -0,0 +1,3 @@
name generation lag start end
file 0000000000000000 - 2000-01-01T00:00:00Z 2000-01-03T00:00:00Z
file 0000000000000001 - 2000-01-02T00:00:00Z 2000-01-02T00:00:00Z

View File

@@ -0,0 +1,4 @@
dbs:
- path: $LITESTREAM_TESTDIR/db
replicas:
- path: $LITESTREAM_TESTDIR/replica

View File

@@ -0,0 +1,5 @@
dbs:
- path: $LITESTREAM_TESTDIR/db
replicas:
- path: $LITESTREAM_TESTDIR/replica0
- path: $LITESTREAM_TESTDIR/replica1

Binary file not shown.

View File

@@ -0,0 +1,4 @@
dbs:
- path: $LITESTREAM_TESTDIR/db
replicas:
- path: $LITESTREAM_TESTDIR/replica

View File

@@ -0,0 +1 @@
database already exists, skipping

View File

@@ -0,0 +1,4 @@
dbs:
- path: $LITESTREAM_TESTDIR/db
replicas:
- path: $LITESTREAM_TESTDIR/replica

View File

@@ -0,0 +1 @@
no matching backups found, skipping

View File

@@ -0,0 +1,4 @@
dbs:
- path: /var/lib/db
replicas:
- path: s3://bkt/db

View File

@@ -0,0 +1,6 @@
.PHONY: default
default:
TZ=UTC touch -ct 200001010000 replica0/generations/0000000000000000/snapshots/00000000.snapshot.lz4
TZ=UTC touch -ct 200001020000 replica1/generations/0000000000000002/snapshots/00000000.snapshot.lz4
TZ=UTC touch -ct 200001030000 replica0/generations/0000000000000001/snapshots/00000000.snapshot.lz4

View File

@@ -0,0 +1,7 @@
dbs:
- path: $LITESTREAM_TESTDIR/db
replicas:
- name: replica0
path: $LITESTREAM_TESTDIR/replica0
- name: replica1
path: $LITESTREAM_TESTDIR/replica1

View File

@@ -0,0 +1,4 @@
dbs:
- path: $LITESTREAM_TESTDIR/db
replicas:
- path: $LITESTREAM_TESTDIR/replica

View File

View File

View File

@@ -0,0 +1,4 @@
dbs:
- path: $LITESTREAM_TESTDIR/db
replicas:
- path: $LITESTREAM_TESTDIR/replica

View File

@@ -0,0 +1,2 @@
dbs:
- path: $LITESTREAM_TESTDIR/db

View File

@@ -0,0 +1,4 @@
dbs:
- path: $LITESTREAM_TESTDIR/db
replicas:
- path: $LITESTREAM_TESTDIR/replica

Binary file not shown.

View File

@@ -0,0 +1,36 @@
To reproduce this testdata, run sqlite3 and execute:
PRAGMA journal_mode = WAL;
CREATE TABLE t (x);
INSERT INTO t (x) VALUES (1);
INSERT INTO t (x) VALUES (2);
sl3 split -o generations/0000000000000000/wal/00000000 db-wal
cp db generations/0000000000000000/snapshots/00000000.snapshot
lz4 -c --rm generations/0000000000000000/snapshots/00000000.snapshot
Then execute:
PRAGMA wal_checkpoint(TRUNCATE);
INSERT INTO t (x) VALUES (3);
sl3 split -o generations/0000000000000000/wal/00000001 db-wal
Then execute:
PRAGMA wal_checkpoint(TRUNCATE);
INSERT INTO t (x) VALUES (4);
INSERT INTO t (x) VALUES (5);
sl3 split -o generations/0000000000000000/wal/00000002 db-wal
Finally, obtain the final snapshot:
PRAGMA wal_checkpoint(TRUNCATE);
cp db 00000002.db
rm db*

View File

@@ -0,0 +1,4 @@
dbs:
- path: $LITESTREAM_TESTDIR/db
replicas:
- path: $LITESTREAM_TESTDIR/replica

Binary file not shown.

View File

@@ -0,0 +1,4 @@
dbs:
- path: $LITESTREAM_TESTDIR/db
replicas:
- path: $LITESTREAM_TESTDIR/replica

View File

@@ -0,0 +1,7 @@
dbs:
- path: $LITESTREAM_TESTDIR/db
replicas:
- name: replica0
path: $LITESTREAM_TESTDIR/replica0
- name: replica1
path: $LITESTREAM_TESTDIR/replica1

View File

@@ -0,0 +1,5 @@
dbs:
- path: $LITESTREAM_TESTDIR/db
replicas:
- name: replica0
path: $LITESTREAM_TESTDIR/replica

View File

@@ -0,0 +1,2 @@
dbs:
- path: $LITESTREAM_TESTDIR/db

View File

@@ -0,0 +1,4 @@
dbs:
- path: /var/lib/db
replicas:
- path: s3://bkt/db

Some files were not shown because too many files have changed in this diff Show More