diff --git a/Makefile b/Makefile index 70d3709..598eddd 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,10 @@ .PHONY: default -default: testdata +default: .PHONY: testdata testdata: make -C testdata + make -C cmd/litestream testdata docker: docker build -t litestream . diff --git a/cmd/litestream/Makefile b/cmd/litestream/Makefile new file mode 100644 index 0000000..4073858 --- /dev/null +++ b/cmd/litestream/Makefile @@ -0,0 +1,6 @@ +.PHONY: default +default: + +.PHONY: testdata +testdata: + make -C testdata diff --git a/cmd/litestream/databases.go b/cmd/litestream/databases.go index dd7747c..a9e99ae 100644 --- a/cmd/litestream/databases.go +++ b/cmd/litestream/databases.go @@ -4,17 +4,30 @@ import ( "context" "flag" "fmt" - "os" + "io" "strings" "text/tabwriter" ) // DatabasesCommand is a command for listing managed databases. type DatabasesCommand struct { + stdin io.Reader + stdout io.Writer + stderr io.Writer + configPath string noExpandEnv bool } +// NewDatabasesCommand returns a new instance of DatabasesCommand. +func NewDatabasesCommand(stdin io.Reader, stdout, stderr io.Writer) *DatabasesCommand { + return &DatabasesCommand{ + stdin: stdin, + stdout: stdout, + stderr: stderr, + } +} + // Run executes the command. func (c *DatabasesCommand) Run(ctx context.Context, args []string) (err error) { fs := flag.NewFlagSet("litestream-databases", flag.ContinueOnError) @@ -27,16 +40,16 @@ func (c *DatabasesCommand) Run(ctx context.Context, args []string) (err error) { } // Load configuration. - if c.configPath == "" { - c.configPath = DefaultConfigPath() - } config, err := ReadConfigFile(c.configPath, !c.noExpandEnv) if err != nil { return err + } else if len(config.DBs) == 0 { + fmt.Fprintln(c.stdout, "No databases found in config file.") + return nil } // List all databases. - w := tabwriter.NewWriter(os.Stdout, 0, 8, 2, ' ', 0) + w := tabwriter.NewWriter(c.stdout, 0, 8, 2, ' ', 0) defer w.Flush() fmt.Fprintln(w, "path\treplicas") @@ -62,7 +75,7 @@ func (c *DatabasesCommand) Run(ctx context.Context, args []string) (err error) { // Usage prints the help screen to STDOUT. func (c *DatabasesCommand) Usage() { - fmt.Printf(` + fmt.Fprintf(c.stdout, ` The databases command lists all databases in the configuration file. Usage: diff --git a/cmd/litestream/databases_test.go b/cmd/litestream/databases_test.go new file mode 100644 index 0000000..9499dc6 --- /dev/null +++ b/cmd/litestream/databases_test.go @@ -0,0 +1,66 @@ +package main_test + +import ( + "context" + "flag" + "path/filepath" + "strings" + "testing" + + "github.com/benbjohnson/litestream/internal/testingutil" +) + +func TestDatabasesCommand(t *testing.T) { + t.Run("OK", func(t *testing.T) { + testDir := filepath.Join("testdata", "databases", "ok") + m, _, stdout, _ := newMain() + if err := m.Run(context.Background(), []string{"databases", "-config", filepath.Join(testDir, "litestream.yml")}); err != nil { + t.Fatal(err) + } else if got, want := stdout.String(), string(testingutil.MustReadFile(t, filepath.Join(testDir, "stdout"))); got != want { + t.Fatalf("stdout=%q, want %q", got, want) + } + }) + + t.Run("NoDatabases", func(t *testing.T) { + testDir := filepath.Join("testdata", "databases", "no-databases") + m, _, stdout, _ := newMain() + if err := m.Run(context.Background(), []string{"databases", "-config", filepath.Join(testDir, "litestream.yml")}); err != nil { + t.Fatal(err) + } else if got, want := stdout.String(), string(testingutil.MustReadFile(t, filepath.Join(testDir, "stdout"))); got != want { + t.Fatalf("stdout=%q, want %q", got, want) + } + }) + + t.Run("ErrConfigNotFound", func(t *testing.T) { + testDir := filepath.Join("testdata", "databases", "no-config") + m, _, _, _ := newMain() + err := m.Run(context.Background(), []string{"databases", "-config", filepath.Join(testDir, "litestream.yml")}) + if err == nil || !strings.Contains(err.Error(), `config file not found:`) { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrInvalidConfig", func(t *testing.T) { + testDir := filepath.Join("testdata", "databases", "invalid-config") + m, _, _, _ := newMain() + err := m.Run(context.Background(), []string{"databases", "-config", filepath.Join(testDir, "litestream.yml")}) + if err == nil || !strings.Contains(err.Error(), `replica path cannot be a url`) { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrTooManyArguments", func(t *testing.T) { + m, _, _, _ := newMain() + err := m.Run(context.Background(), []string{"databases", "xyz"}) + if err == nil || err.Error() != `too many arguments` { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("Usage", func(t *testing.T) { + m, _, _, _ := newMain() + if err := m.Run(context.Background(), []string{"databases", "-h"}); err != flag.ErrHelp { + t.Fatalf("unexpected error: %s", err) + } + }) +} diff --git a/cmd/litestream/generations.go b/cmd/litestream/generations.go index e4f9faf..da74099 100644 --- a/cmd/litestream/generations.go +++ b/cmd/litestream/generations.go @@ -4,93 +4,80 @@ import ( "context" "flag" "fmt" - "log" + "io" "os" "text/tabwriter" "time" "github.com/benbjohnson/litestream" + "github.com/benbjohnson/litestream/internal" ) // GenerationsCommand represents a command to list all generations for a database. type GenerationsCommand struct { + stdin io.Reader + stdout io.Writer + stderr io.Writer + configPath string noExpandEnv bool + + replicaName string +} + +// NewGenerationsCommand returns a new instance of GenerationsCommand. +func NewGenerationsCommand(stdin io.Reader, stdout, stderr io.Writer) *GenerationsCommand { + return &GenerationsCommand{ + stdin: stdin, + stdout: stdout, + stderr: stderr, + } } // Run executes the command. -func (c *GenerationsCommand) Run(ctx context.Context, args []string) (err error) { +func (c *GenerationsCommand) Run(ctx context.Context, args []string) (ret error) { fs := flag.NewFlagSet("litestream-generations", flag.ContinueOnError) registerConfigFlag(fs, &c.configPath, &c.noExpandEnv) - replicaName := fs.String("replica", "", "replica name") + fs.StringVar(&c.replicaName, "replica", "", "replica name") fs.Usage = c.Usage if err := fs.Parse(args); err != nil { return err - } else if fs.NArg() == 0 || fs.Arg(0) == "" { + } else if fs.Arg(0) == "" { return fmt.Errorf("database path or replica URL required") } else if fs.NArg() > 1 { return fmt.Errorf("too many arguments") } - var db *litestream.DB - var r *litestream.Replica - dbUpdatedAt := time.Now() - if isURL(fs.Arg(0)) { - if c.configPath != "" { - return fmt.Errorf("cannot specify a replica URL and the -config flag") - } - if r, err = NewReplicaFromConfig(&ReplicaConfig{URL: fs.Arg(0)}, nil); err != nil { - return err - } - } else { - if c.configPath == "" { - c.configPath = DefaultConfigPath() - } - - // Load configuration. - config, err := ReadConfigFile(c.configPath, !c.noExpandEnv) - if err != nil { - return err - } - - // Lookup database from configuration file by path. - if path, err := expand(fs.Arg(0)); err != nil { - return err - } else if dbc := config.DBConfig(path); dbc == nil { - return fmt.Errorf("database not found in config: %s", path) - } else if db, err = NewDBFromConfig(dbc); err != nil { - return err - } - - // Filter by replica, if specified. - if *replicaName != "" { - if r = db.Replica(*replicaName); r == nil { - return fmt.Errorf("replica %q not found for database %q", *replicaName, db.Path()) - } - } - - // Determine last time database or WAL was updated. - if dbUpdatedAt, err = db.UpdatedAt(); err != nil { - return err - } + // Load configuration. + config, err := ReadConfigFile(c.configPath, !c.noExpandEnv) + if err != nil { + return err } - var replicas []*litestream.Replica - if r != nil { - replicas = []*litestream.Replica{r} - } else { - replicas = db.Replicas + replicas, db, err := loadReplicas(ctx, config, fs.Arg(0), c.replicaName) + if err != nil { + return err + } + + // Determine last time database or WAL was updated. + var dbUpdatedAt time.Time + if db != nil { + if dbUpdatedAt, err = db.UpdatedAt(); err != nil && !os.IsNotExist(err) { + return err + } } // List each generation. - w := tabwriter.NewWriter(os.Stdout, 0, 8, 2, ' ', 0) + w := tabwriter.NewWriter(c.stdout, 0, 8, 2, ' ', 0) defer w.Flush() fmt.Fprintln(w, "name\tgeneration\tlag\tstart\tend") + for _, r := range replicas { generations, err := r.Client.Generations(ctx) if err != nil { - log.Printf("%s: cannot list generations: %s", r.Name(), err) + fmt.Fprintf(c.stderr, "%s: cannot list generations: %s", r.Name(), err) + ret = errExit // signal error return without printing message continue } @@ -98,26 +85,35 @@ func (c *GenerationsCommand) Run(ctx context.Context, args []string) (err error) for _, generation := range generations { createdAt, updatedAt, err := litestream.GenerationTimeBounds(ctx, r.Client, generation) if err != nil { - log.Printf("%s: cannot determine generation time bounds: %s", r.Name(), err) + fmt.Fprintf(c.stderr, "%s: cannot determine generation time bounds: %s", r.Name(), err) + ret = errExit // signal error return without printing message continue } + // Calculate lag from database mod time to the replica mod time. + // This is ignored if the database mod time is unavailable such as + // when specifying the replica URL or if the database file is missing. + lag := "-" + if !dbUpdatedAt.IsZero() { + lag = internal.TruncateDuration(dbUpdatedAt.Sub(updatedAt)).String() + } + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n", r.Name(), generation, - truncateDuration(dbUpdatedAt.Sub(updatedAt)).String(), + lag, createdAt.Format(time.RFC3339), updatedAt.Format(time.RFC3339), ) } } - return nil + return ret } // Usage prints the help message to STDOUT. func (c *GenerationsCommand) Usage() { - fmt.Printf(` + fmt.Fprintf(c.stdout, ` The generations command lists all generations for a database or replica. It also lists stats about their lag behind the primary database and the time range they cover. @@ -144,29 +140,3 @@ Arguments: DefaultConfigPath(), ) } - -func truncateDuration(d time.Duration) time.Duration { - if d < 0 { - if d < -10*time.Second { - return d.Truncate(time.Second) - } else if d < -time.Second { - return d.Truncate(time.Second / 10) - } else if d < -time.Millisecond { - return d.Truncate(time.Millisecond) - } else if d < -time.Microsecond { - return d.Truncate(time.Microsecond) - } - return d - } - - if d > 10*time.Second { - return d.Truncate(time.Second) - } else if d > time.Second { - return d.Truncate(time.Second / 10) - } else if d > time.Millisecond { - return d.Truncate(time.Millisecond) - } else if d > time.Microsecond { - return d.Truncate(time.Microsecond) - } - return d -} diff --git a/cmd/litestream/generations_test.go b/cmd/litestream/generations_test.go new file mode 100644 index 0000000..097bd35 --- /dev/null +++ b/cmd/litestream/generations_test.go @@ -0,0 +1,140 @@ +package main_test + +import ( + "context" + "flag" + "path/filepath" + "strings" + "testing" + + "github.com/benbjohnson/litestream/internal/testingutil" +) + +func TestGenerationsCommand(t *testing.T) { + t.Run("OK", func(t *testing.T) { + testDir := filepath.Join("testdata", "generations", "ok") + defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)() + + m, _, stdout, _ := newMain() + if err := m.Run(context.Background(), []string{"generations", "-config", filepath.Join(testDir, "litestream.yml"), filepath.Join(testDir, "db")}); err != nil { + t.Fatal(err) + } else if got, want := stdout.String(), string(testingutil.MustReadFile(t, filepath.Join(testDir, "stdout"))); got != want { + t.Fatalf("stdout=%q, want %q", got, want) + } + }) + + t.Run("ReplicaName", func(t *testing.T) { + testDir := filepath.Join("testdata", "generations", "replica-name") + defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)() + + m, _, stdout, _ := newMain() + if err := m.Run(context.Background(), []string{"generations", "-config", filepath.Join(testDir, "litestream.yml"), "-replica", "replica1", filepath.Join(testDir, "db")}); err != nil { + t.Fatal(err) + } else if got, want := stdout.String(), string(testingutil.MustReadFile(t, filepath.Join(testDir, "stdout"))); got != want { + t.Fatalf("stdout=%q, want %q", got, want) + } + }) + + t.Run("ReplicaURL", func(t *testing.T) { + testDir := filepath.Join(testingutil.Getwd(t), "testdata", "generations", "replica-url") + defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)() + replicaURL := "file://" + filepath.ToSlash(testDir) + "/replica" + + m, _, stdout, _ := newMain() + if err := m.Run(context.Background(), []string{"generations", replicaURL}); err != nil { + t.Fatal(err) + } else if got, want := stdout.String(), string(testingutil.MustReadFile(t, filepath.Join(testDir, "stdout"))); got != want { + t.Fatalf("stdout=%q, want %q", got, want) + } + }) + + t.Run("NoDatabase", func(t *testing.T) { + testDir := filepath.Join("testdata", "generations", "no-database") + defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)() + + m, _, stdout, _ := newMain() + if err := m.Run(context.Background(), []string{"generations", "-config", filepath.Join(testDir, "litestream.yml"), filepath.Join(testDir, "db")}); err != nil { + t.Fatal(err) + } else if got, want := stdout.String(), string(testingutil.MustReadFile(t, filepath.Join(testDir, "stdout"))); got != want { + t.Fatalf("stdout=%q, want %q", got, want) + } + }) + + t.Run("ErrDatabaseOrReplicaRequired", func(t *testing.T) { + m, _, _, _ := newMain() + err := m.Run(context.Background(), []string{"generations"}) + if err == nil || err.Error() != `database path or replica URL required` { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrTooManyArguments", func(t *testing.T) { + m, _, _, _ := newMain() + err := m.Run(context.Background(), []string{"generations", "abc", "123"}) + if err == nil || err.Error() != `too many arguments` { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrInvalidFlags", func(t *testing.T) { + m, _, _, _ := newMain() + err := m.Run(context.Background(), []string{"generations", "-no-such-flag"}) + if err == nil || err.Error() != `flag provided but not defined: -no-such-flag` { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrConfigFileNotFound", func(t *testing.T) { + m, _, _, _ := newMain() + err := m.Run(context.Background(), []string{"generations", "-config", "/no/such/file", "/var/lib/db"}) + if err == nil || err.Error() != `config file not found: /no/such/file` { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrInvalidConfig", func(t *testing.T) { + testDir := filepath.Join("testdata", "generations", "invalid-config") + m, _, _, _ := newMain() + err := m.Run(context.Background(), []string{"generations", "-config", filepath.Join(testDir, "litestream.yml"), "/var/lib/db"}) + if err == nil || !strings.Contains(err.Error(), `replica path cannot be a url`) { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrDatabaseNotFound", func(t *testing.T) { + testDir := filepath.Join("testdata", "generations", "database-not-found") + defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)() + + m, _, _, _ := newMain() + err := m.Run(context.Background(), []string{"generations", "-config", filepath.Join(testDir, "litestream.yml"), "/no/such/db"}) + if err == nil || err.Error() != `database not found in config: /no/such/db` { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrReplicaNotFound", func(t *testing.T) { + testDir := filepath.Join(testingutil.Getwd(t), "testdata", "generations", "replica-not-found") + defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)() + + m, _, _, _ := newMain() + err := m.Run(context.Background(), []string{"generations", "-config", filepath.Join(testDir, "litestream.yml"), "-replica", "no_such_replica", filepath.Join(testDir, "db")}) + if err == nil || err.Error() != `replica "no_such_replica" not found for database "`+filepath.Join(testDir, "db")+`"` { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrInvalidReplicaURL", func(t *testing.T) { + m, _, _, _ := newMain() + err := m.Run(context.Background(), []string{"generations", "xyz://xyz"}) + if err == nil || !strings.Contains(err.Error(), `unknown replica type in config: "xyz"`) { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("Usage", func(t *testing.T) { + m, _, _, _ := newMain() + if err := m.Run(context.Background(), []string{"generations", "-h"}); err != flag.ErrHelp { + t.Fatalf("unexpected error: %s", err) + } + }) +} diff --git a/cmd/litestream/main.go b/cmd/litestream/main.go index 7f6f101..176ec99 100644 --- a/cmd/litestream/main.go +++ b/cmd/litestream/main.go @@ -5,6 +5,7 @@ import ( "errors" "flag" "fmt" + "io" "io/ioutil" "log" "net/url" @@ -32,14 +33,14 @@ var ( Version = "(development build)" ) -// errStop is a terminal error for indicating program should quit. -var errStop = errors.New("stop") +// errExit is a terminal error for indicating program should quit. +var errExit = errors.New("exit") func main() { log.SetFlags(0) - m := NewMain() - if err := m.Run(context.Background(), os.Args[1:]); err == flag.ErrHelp || err == errStop { + m := NewMain(os.Stdin, os.Stdout, os.Stderr) + if err := m.Run(context.Background(), os.Args[1:]); err == flag.ErrHelp || err == errExit { os.Exit(1) } else if err != nil { log.Println(err) @@ -48,11 +49,19 @@ func main() { } // Main represents the main program execution. -type Main struct{} +type Main struct { + stdin io.Reader + stdout io.Writer + stderr io.Writer +} // NewMain returns a new instance of Main. -func NewMain() *Main { - return &Main{} +func NewMain(stdin io.Reader, stdout, stderr io.Writer) *Main { + return &Main{ + stdin: stdin, + stdout: stdout, + stderr: stderr, + } } // Run executes the program. @@ -75,11 +84,11 @@ func (m *Main) Run(ctx context.Context, args []string) (err error) { switch cmd { case "databases": - return (&DatabasesCommand{}).Run(ctx, args) + return NewDatabasesCommand(m.stdin, m.stdout, m.stderr).Run(ctx, args) case "generations": - return (&GenerationsCommand{}).Run(ctx, args) + return NewGenerationsCommand(m.stdin, m.stdout, m.stderr).Run(ctx, args) case "replicate": - c := NewReplicateCommand() + c := NewReplicateCommand(m.stdin, m.stdout, m.stderr) if err := c.ParseFlags(ctx, args); err != nil { return err } @@ -96,21 +105,21 @@ func (m *Main) Run(ctx context.Context, args []string) (err error) { // Wait for signal to stop program. select { case <-ctx.Done(): - fmt.Println("context done, litestream shutting down") + fmt.Fprintln(m.stdout, "context done, litestream shutting down") case err = <-c.execCh: cancel() - fmt.Println("subprocess exited, litestream shutting down") + fmt.Fprintln(m.stdout, "subprocess exited, litestream shutting down") case sig := <-signalCh: cancel() - fmt.Println("signal received, litestream shutting down") + fmt.Fprintln(m.stdout, "signal received, litestream shutting down") if c.cmd != nil { - fmt.Println("sending signal to exec process") + fmt.Fprintln(m.stdout, "sending signal to exec process") if err := c.cmd.Process.Signal(sig); err != nil { return fmt.Errorf("cannot signal exec process: %w", err) } - fmt.Println("waiting for exec process to close") + fmt.Fprintln(m.stdout, "waiting for exec process to close") if err := <-c.execCh; err != nil && !strings.HasPrefix(err.Error(), "signal:") { return fmt.Errorf("cannot wait for exec process: %w", err) } @@ -121,17 +130,17 @@ func (m *Main) Run(ctx context.Context, args []string) (err error) { if e := c.Close(); e != nil && err == nil { err = e } - fmt.Println("litestream shut down") + fmt.Fprintln(m.stdout, "litestream shut down") return err case "restore": - return NewRestoreCommand().Run(ctx, args) + return NewRestoreCommand(m.stdin, m.stdout, m.stderr).Run(ctx, args) case "snapshots": - return (&SnapshotsCommand{}).Run(ctx, args) + return NewSnapshotsCommand(m.stdin, m.stdout, m.stderr).Run(ctx, args) case "version": - return (&VersionCommand{}).Run(ctx, args) + return NewVersionCommand(m.stdin, m.stdout, m.stderr).Run(ctx, args) case "wal": - return (&WALCommand{}).Run(ctx, args) + return NewWALCommand(m.stdin, m.stdout, m.stderr).Run(ctx, args) default: if cmd == "" || cmd == "help" || strings.HasPrefix(cmd, "-") { m.Usage() @@ -143,7 +152,7 @@ func (m *Main) Run(ctx context.Context, args []string) (err error) { // Usage prints the help screen to STDOUT. func (m *Main) Usage() { - fmt.Println(` + fmt.Fprintln(m.stdout, ` litestream is a tool for replicating SQLite databases. Usage: @@ -210,9 +219,15 @@ func (c *Config) DBConfig(path string) *DBConfig { // ReadConfigFile unmarshals config from filename. Expands path if needed. // If expandEnv is true then environment variables are expanded in the config. +// If filename is blank then the default config path is used. func ReadConfigFile(filename string, expandEnv bool) (_ Config, err error) { config := DefaultConfig() + useDefaultPath := filename == "" + if useDefaultPath { + filename = DefaultConfigPath() + } + // Expand filename, if necessary. filename, err = expand(filename) if err != nil { @@ -220,8 +235,12 @@ func ReadConfigFile(filename string, expandEnv bool) (_ Config, err error) { } // Read configuration. + // Do not return an error if using default path and file is missing. buf, err := ioutil.ReadFile(filename) if os.IsNotExist(err) { + if useDefaultPath { + return config, nil + } return config, fmt.Errorf("config file not found: %s", filename) } else if err != nil { return config, err @@ -354,7 +373,7 @@ func NewReplicaFromConfig(c *ReplicaConfig, db *litestream.DB) (_ *litestream.Re } // Build and set client on replica. - switch c.ReplicaType() { + switch typ := c.ReplicaType(); typ { case "file": if r.Client, err = newFileReplicaClientFromConfig(c, r); err != nil { return nil, err @@ -376,7 +395,7 @@ func NewReplicaFromConfig(c *ReplicaConfig, db *litestream.DB) (_ *litestream.Re return nil, err } default: - return nil, fmt.Errorf("unknown replica type in config: %q", c.Type) + return nil, fmt.Errorf("unknown replica type in config: %q", typ) } return r, nil @@ -714,3 +733,45 @@ func (v *indexVar) Set(s string) error { *v = indexVar(i) return nil } + +// loadReplicas returns a list of replicas to use based on CLI flags. Filters +// by replicaName, if not blank. The DB is returned if pathOrURL is not a replica URL. +func loadReplicas(ctx context.Context, config Config, pathOrURL, replicaName string) ([]*litestream.Replica, *litestream.DB, error) { + // Build a replica based on URL, if specified. + if isURL(pathOrURL) { + r, err := NewReplicaFromConfig(&ReplicaConfig{ + URL: pathOrURL, + AccessKeyID: config.AccessKeyID, + SecretAccessKey: config.SecretAccessKey, + }, nil) + if err != nil { + return nil, nil, err + } + return []*litestream.Replica{r}, nil, nil + } + + // Otherwise use replicas from the database configuration file. + path, err := expand(pathOrURL) + if err != nil { + return nil, nil, err + } + dbc := config.DBConfig(path) + if dbc == nil { + return nil, nil, fmt.Errorf("database not found in config: %s", path) + } + db, err := NewDBFromConfig(dbc) + if err != nil { + return nil, nil, err + } + + // Filter by replica, if specified. + if replicaName != "" { + r := db.Replica(replicaName) + if r == nil { + return nil, nil, fmt.Errorf("replica %q not found for database %q", replicaName, db.Path()) + } + return []*litestream.Replica{r}, db, nil + } + + return db.Replicas, db, nil +} diff --git a/cmd/litestream/main_test.go b/cmd/litestream/main_test.go index 3886095..f3e9fb1 100644 --- a/cmd/litestream/main_test.go +++ b/cmd/litestream/main_test.go @@ -1,6 +1,8 @@ package main_test import ( + "bytes" + "io" "io/ioutil" "log" "os" @@ -180,3 +182,17 @@ func TestNewGCSReplicaFromConfig(t *testing.T) { t.Fatalf("Path=%s, want %s", got, want) } } + +// newMain returns a new instance of Main and associated buffers. +func newMain() (m *main.Main, stdin, stdout, stderr *bytes.Buffer) { + stdin, stdout, stderr = &bytes.Buffer{}, &bytes.Buffer{}, &bytes.Buffer{} + + // Split stdout/stderr to terminal if verbose flag set. + out, err := io.Writer(stdout), io.Writer(stderr) + if testing.Verbose() { + out = io.MultiWriter(out, os.Stdout) + err = io.MultiWriter(err, os.Stderr) + } + + return main.NewMain(stdin, out, err), stdin, stdout, stderr +} diff --git a/cmd/litestream/main_windows.go b/cmd/litestream/main_windows.go index 512ab26..d437c1b 100644 --- a/cmd/litestream/main_windows.go +++ b/cmd/litestream/main_windows.go @@ -1,3 +1,4 @@ +//go:build windows // +build windows package main @@ -41,7 +42,7 @@ func runWindowsService(ctx context.Context) error { log.Print("Litestream service starting") if err := svc.Run(serviceName, &windowsService{ctx: ctx}); err != nil { - return errStop + return errExit } log.Print("Litestream service stopped") diff --git a/cmd/litestream/replicate.go b/cmd/litestream/replicate.go index 3da238f..5d32db7 100644 --- a/cmd/litestream/replicate.go +++ b/cmd/litestream/replicate.go @@ -4,6 +4,7 @@ import ( "context" "flag" "fmt" + "io" "log" "net" "net/http" @@ -22,6 +23,10 @@ import ( // ReplicateCommand represents a command that continuously replicates SQLite databases. type ReplicateCommand struct { + stdin io.Reader + stdout io.Writer + stderr io.Writer + configPath string noExpandEnv bool @@ -34,8 +39,13 @@ type ReplicateCommand struct { DBs []*litestream.DB } -func NewReplicateCommand() *ReplicateCommand { +// NewReplicateCommand returns a new instance of ReplicateCommand. +func NewReplicateCommand(stdin io.Reader, stdout, stderr io.Writer) *ReplicateCommand { return &ReplicateCommand{ + stdin: stdin, + stdout: stdout, + stderr: stderr, + execCh: make(chan error), } } @@ -181,7 +191,7 @@ func (c *ReplicateCommand) Close() (err error) { // Usage prints the help screen to STDOUT. func (c *ReplicateCommand) Usage() { - fmt.Printf(` + fmt.Fprintf(c.stdout, ` The replicate command starts a server to monitor & replicate databases. You can specify your database & replicas in a configuration file or you can replicate a single database file by specifying its path and its replicas in the diff --git a/cmd/litestream/replicate_test.go b/cmd/litestream/replicate_test.go index 4708580..7d85b04 100644 --- a/cmd/litestream/replicate_test.go +++ b/cmd/litestream/replicate_test.go @@ -13,7 +13,6 @@ import ( "testing" "time" - main "github.com/benbjohnson/litestream/cmd/litestream" "golang.org/x/sync/errgroup" ) @@ -82,7 +81,8 @@ dbs: // Replicate database unless the context is canceled. g.Go(func() error { - return main.NewMain().Run(mainctx, []string{"replicate", "-config", configPath}) + m, _, _, _ := newMain() + return m.Run(mainctx, []string{"replicate", "-config", configPath}) }) if err := g.Wait(); err != nil { @@ -94,7 +94,8 @@ dbs: chksum0 := mustChecksum(t, dbPath) // Restore to another path. - if err := main.NewMain().Run(context.Background(), []string{"restore", "-config", configPath, "-o", restorePath, dbPath}); err != nil && !errors.Is(err, context.Canceled) { + m, _, _, _ := newMain() + if err := m.Run(context.Background(), []string{"restore", "-config", configPath, "-o", restorePath, dbPath}); err != nil && !errors.Is(err, context.Canceled) { t.Fatal(err) } diff --git a/cmd/litestream/restore.go b/cmd/litestream/restore.go index 9e3dca1..1a0f5fd 100644 --- a/cmd/litestream/restore.go +++ b/cmd/litestream/restore.go @@ -5,6 +5,7 @@ import ( "errors" "flag" "fmt" + "io" "log" "os" "path/filepath" @@ -15,6 +16,10 @@ import ( // RestoreCommand represents a command to restore a database from a backup. type RestoreCommand struct { + stdin io.Reader + stdout io.Writer + stderr io.Writer + snapshotIndex int // index of snapshot to start from // CLI options @@ -29,8 +34,13 @@ type RestoreCommand struct { opt litestream.RestoreOptions } -func NewRestoreCommand() *RestoreCommand { +// NewRestoreCommand returns a new instance of RestoreCommand. +func NewRestoreCommand(stdin io.Reader, stdout, stderr io.Writer) *RestoreCommand { return &RestoreCommand{ + stdin: stdin, + stdout: stdout, + stderr: stderr, + targetIndex: -1, opt: litestream.NewRestoreOptions(), } @@ -55,31 +65,39 @@ func (c *RestoreCommand) Run(ctx context.Context, args []string) (err error) { } else if fs.NArg() > 1 { return fmt.Errorf("too many arguments") } - arg := fs.Arg(0) + pathOrURL := fs.Arg(0) // Ensure a generation is specified if target index is specified. if c.targetIndex != -1 && c.generation == "" { - return fmt.Errorf("must specify -generation when using -index flag") + return fmt.Errorf("must specify -generation flag when using -index flag") } // Default to original database path if output path not specified. - if !isURL(arg) && c.outputPath == "" { - c.outputPath = arg + if !isURL(pathOrURL) && c.outputPath == "" { + c.outputPath = pathOrURL } // Exit successfully if the output file already exists and flag is set. - if _, err := os.Stat(c.outputPath); !os.IsNotExist(err) && c.ifDBNotExists { - fmt.Println("database already exists, skipping") - return nil + if _, err := os.Stat(c.outputPath); os.IsNotExist(err) { + // file doesn't exist, continue + } else if err != nil { + return err + } else if err == nil { + if c.ifDBNotExists { + fmt.Fprintln(c.stdout, "database already exists, skipping") + return nil + } + return fmt.Errorf("output file already exists: %s", c.outputPath) } - // Create parent directory if it doesn't already exist. - if err := os.MkdirAll(filepath.Dir(c.outputPath), 0700); err != nil { - return fmt.Errorf("cannot create parent directory: %w", err) + // Load configuration. + config, err := ReadConfigFile(c.configPath, !c.noExpandEnv) + if err != nil { + return err } // Build replica from either a URL or config. - r, err := c.loadReplica(ctx, arg) + r, err := c.loadReplica(ctx, config, pathOrURL) if err != nil { return err } @@ -90,7 +108,7 @@ func (c *RestoreCommand) Run(ctx context.Context, args []string) (err error) { // Return an error if no matching targets found. // If optional flag set, return success. Useful for automated recovery. if c.ifReplicaExists { - fmt.Println("no matching backups found") + fmt.Fprintln(c.stdout, "no matching backups found, skipping") return nil } return fmt.Errorf("no matching backups found") @@ -112,47 +130,42 @@ func (c *RestoreCommand) Run(ctx context.Context, args []string) (err error) { return fmt.Errorf("cannot find snapshot index: %w", err) } - c.opt.Logger = log.New(os.Stderr, "", log.LstdFlags|log.Lmicroseconds) + // Create parent directory if it doesn't already exist. + if err := os.MkdirAll(filepath.Dir(c.outputPath), 0700); err != nil { + return fmt.Errorf("cannot create parent directory: %w", err) + } + + c.opt.Logger = log.New(c.stdout, "", log.LstdFlags|log.Lmicroseconds) return litestream.Restore(ctx, r.Client, c.outputPath, c.generation, c.snapshotIndex, c.targetIndex, c.opt) } -func (c *RestoreCommand) loadReplica(ctx context.Context, arg string) (*litestream.Replica, error) { +func (c *RestoreCommand) loadReplica(ctx context.Context, config Config, arg string) (*litestream.Replica, error) { if isURL(arg) { - return c.loadReplicaFromURL(ctx, arg) + return c.loadReplicaFromURL(ctx, config, arg) } - return c.loadReplicaFromConfig(ctx, arg) + return c.loadReplicaFromConfig(ctx, config, arg) } // loadReplicaFromURL creates a replica & updates the restore options from a replica URL. -func (c *RestoreCommand) loadReplicaFromURL(ctx context.Context, replicaURL string) (*litestream.Replica, error) { - if c.configPath != "" { - return nil, fmt.Errorf("cannot specify a replica URL and the -config flag") - } else if c.replicaName != "" { - return nil, fmt.Errorf("cannot specify a replica URL and the -replica flag") +func (c *RestoreCommand) loadReplicaFromURL(ctx context.Context, config Config, replicaURL string) (*litestream.Replica, error) { + if c.replicaName != "" { + return nil, fmt.Errorf("cannot specify both the replica URL and the -replica flag") } else if c.outputPath == "" { - return nil, fmt.Errorf("output path required") + return nil, fmt.Errorf("output path required when using a replica URL") } syncInterval := litestream.DefaultSyncInterval return NewReplicaFromConfig(&ReplicaConfig{ - URL: replicaURL, - SyncInterval: &syncInterval, + URL: replicaURL, + AccessKeyID: config.AccessKeyID, + SecretAccessKey: config.SecretAccessKey, + SyncInterval: &syncInterval, }, nil) } // loadReplicaFromConfig returns replicas based on the specific config path. -func (c *RestoreCommand) loadReplicaFromConfig(ctx context.Context, dbPath string) (*litestream.Replica, error) { - if c.configPath == "" { - c.configPath = DefaultConfigPath() - } - - // Load configuration. - config, err := ReadConfigFile(c.configPath, !c.noExpandEnv) - if err != nil { - return nil, err - } - +func (c *RestoreCommand) loadReplicaFromConfig(ctx context.Context, config Config, dbPath string) (_ *litestream.Replica, err error) { // Lookup database from configuration file by path. if dbPath, err = expand(dbPath); err != nil { return nil, err @@ -184,7 +197,7 @@ func (c *RestoreCommand) loadReplicaFromConfig(ctx context.Context, dbPath strin // A replica must be specified when restoring a specific generation with multiple replicas. if c.generation != "" { - return nil, fmt.Errorf("must specify -replica when restoring from a specific generation") + return nil, fmt.Errorf("must specify -replica flag when restoring from a specific generation") } // Determine latest replica to restore from. @@ -197,7 +210,7 @@ func (c *RestoreCommand) loadReplicaFromConfig(ctx context.Context, dbPath strin // Usage prints the help screen to STDOUT. func (c *RestoreCommand) Usage() { - fmt.Printf(` + fmt.Fprintf(c.stdout, ` The restore command recovers a database from a previous snapshot and WAL. Usage: diff --git a/cmd/litestream/restore_test.go b/cmd/litestream/restore_test.go new file mode 100644 index 0000000..4d0770c --- /dev/null +++ b/cmd/litestream/restore_test.go @@ -0,0 +1,330 @@ +package main_test + +import ( + "context" + "flag" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/benbjohnson/litestream/internal/testingutil" +) + +func TestRestoreCommand(t *testing.T) { + t.Run("OK", func(t *testing.T) { + testDir := filepath.Join("testdata", "restore", "ok") + defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)() + tempDir := t.TempDir() + + m, _, stdout, stderr := newMain() + if err := m.Run(context.Background(), []string{"restore", "-config", filepath.Join(testDir, "litestream.yml"), "-o", filepath.Join(tempDir, "db"), filepath.Join(testDir, "db")}); err != nil { + t.Fatal(err) + } else if got, want := stderr.String(), ""; got != want { + t.Fatalf("stderr=%q, want %q", got, want) + } + + // STDOUT has timing info so we need to grep per line. + lines := strings.Split(stdout.String(), "\n") + for i, substr := range []string{ + `restoring snapshot 0000000000000000/00000000 to ` + filepath.Join(tempDir, "db.tmp"), + `applied wal 0000000000000000/00000000 elapsed=`, + `applied wal 0000000000000000/00000001 elapsed=`, + `applied wal 0000000000000000/00000002 elapsed=`, + `renaming database from temporary location`, + } { + if !strings.Contains(lines[i], substr) { + t.Fatalf("stdout: unexpected line %d:\n%s", i+1, stdout) + } + } + }) + + t.Run("ReplicaName", func(t *testing.T) { + testDir := filepath.Join("testdata", "restore", "replica-name") + defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)() + tempDir := t.TempDir() + + m, _, stdout, stderr := newMain() + if err := m.Run(context.Background(), []string{"restore", "-config", filepath.Join(testDir, "litestream.yml"), "-o", filepath.Join(tempDir, "db"), "-replica", "replica1", filepath.Join(testDir, "db")}); err != nil { + t.Fatal(err) + } else if got, want := stderr.String(), ""; got != want { + t.Fatalf("stderr=%q, want %q", got, want) + } + + // STDOUT has timing info so we need to grep per line. + lines := strings.Split(stdout.String(), "\n") + for i, substr := range []string{ + `restoring snapshot 0000000000000001/00000001 to ` + filepath.Join(tempDir, "db.tmp"), + `no wal files found, snapshot only`, + `renaming database from temporary location`, + } { + if !strings.Contains(lines[i], substr) { + t.Fatalf("stdout: unexpected line %d:\n%s", i+1, stdout) + } + } + }) + + t.Run("ReplicaURL", func(t *testing.T) { + testDir := filepath.Join(testingutil.Getwd(t), "testdata", "restore", "replica-url") + tempDir := t.TempDir() + replicaURL := "file://" + filepath.ToSlash(testDir) + "/replica" + + m, _, stdout, stderr := newMain() + if err := m.Run(context.Background(), []string{"restore", "-o", filepath.Join(tempDir, "db"), replicaURL}); err != nil { + t.Fatal(err) + } else if got, want := stderr.String(), ""; got != want { + t.Fatalf("stderr=%q, want %q", got, want) + } + + lines := strings.Split(stdout.String(), "\n") + for i, substr := range []string{ + `restoring snapshot 0000000000000000/00000000 to ` + filepath.Join(tempDir, "db.tmp"), + `no wal files found, snapshot only`, + `renaming database from temporary location`, + } { + if !strings.Contains(lines[i], substr) { + t.Fatalf("stdout: unexpected line %d:\n%s", i+1, stdout) + } + } + }) + + t.Run("LatestReplica", func(t *testing.T) { + testDir := filepath.Join("testdata", "restore", "latest-replica") + defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)() + tempDir := t.TempDir() + + m, _, stdout, stderr := newMain() + if err := m.Run(context.Background(), []string{"restore", "-config", filepath.Join(testDir, "litestream.yml"), "-o", filepath.Join(tempDir, "db"), filepath.Join(testDir, "db")}); err != nil { + t.Fatal(err) + } else if got, want := stderr.String(), ""; got != want { + t.Fatalf("stderr=%q, want %q", got, want) + } + + lines := strings.Split(stdout.String(), "\n") + for i, substr := range []string{ + `restoring snapshot 0000000000000001/00000000 to ` + filepath.Join(tempDir, "db.tmp"), + `no wal files found, snapshot only`, + `renaming database from temporary location`, + } { + if !strings.Contains(lines[i], substr) { + t.Fatalf("stdout: unexpected line %d:\n%s", i+1, stdout) + } + } + }) + + t.Run("IfDBNotExistsFlag", func(t *testing.T) { + testDir := filepath.Join("testdata", "restore", "if-db-not-exists-flag") + defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)() + + m, _, stdout, _ := newMain() + err := m.Run(context.Background(), []string{"restore", "-config", filepath.Join(testDir, "litestream.yml"), "-if-db-not-exists", filepath.Join(testDir, "db")}) + if err != nil { + t.Fatal(err) + } else if got, want := stdout.String(), string(testingutil.MustReadFile(t, filepath.Join(testDir, "stdout"))); got != want { + t.Fatalf("stdout=%q, want %q", got, want) + } + }) + + t.Run("IfReplicaExists", func(t *testing.T) { + testDir := filepath.Join("testdata", "restore", "if-replica-exists-flag") + defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)() + + m, _, stdout, _ := newMain() + err := m.Run(context.Background(), []string{"restore", "-config", filepath.Join(testDir, "litestream.yml"), "-if-replica-exists", filepath.Join(testDir, "db")}) + if err != nil { + t.Fatal(err) + } else if got, want := stdout.String(), string(testingutil.MustReadFile(t, filepath.Join(testDir, "stdout"))); got != want { + t.Fatalf("stdout=%q, want %q", got, want) + } + }) + + t.Run("ErrNoBackups", func(t *testing.T) { + testDir := filepath.Join("testdata", "restore", "no-backups") + defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)() + tempDir := t.TempDir() + + m, _, stdout, stderr := newMain() + err := m.Run(context.Background(), []string{"restore", "-config", filepath.Join(testDir, "litestream.yml"), "-o", filepath.Join(tempDir, "db"), filepath.Join(testDir, "db")}) + if err == nil || err.Error() != `no matching backups found` { + t.Fatalf("unexpected error: %s", err) + } else if got, want := stdout.String(), string(testingutil.MustReadFile(t, filepath.Join(testDir, "stdout"))); got != want { + t.Fatalf("stdout=%q, want %q", got, want) + } else if got, want := stderr.String(), string(testingutil.MustReadFile(t, filepath.Join(testDir, "stderr"))); got != want { + t.Fatalf("stderr=%q, want %q", got, want) + } + }) + + t.Run("ErrNoGeneration", func(t *testing.T) { + testDir := filepath.Join("testdata", "restore", "no-generation") + defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)() + + m, _, _, _ := newMain() + err := m.Run(context.Background(), []string{"restore", "-config", filepath.Join(testDir, "litestream.yml"), filepath.Join(testDir, "db")}) + if err == nil || err.Error() != `no matching backups found` { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrOutputPathExists", func(t *testing.T) { + testDir := filepath.Join("testdata", "restore", "output-path-exists") + defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)() + + m, _, _, _ := newMain() + err := m.Run(context.Background(), []string{"restore", "-config", filepath.Join(testDir, "litestream.yml"), filepath.Join(testDir, "db")}) + if err == nil || err.Error() != `output file already exists: `+filepath.Join(testDir, "db") { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrDatabaseOrReplicaRequired", func(t *testing.T) { + m, _, _, _ := newMain() + err := m.Run(context.Background(), []string{"restore"}) + if err == nil || err.Error() != `database path or replica URL required` { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrTooManyArguments", func(t *testing.T) { + m, _, _, _ := newMain() + err := m.Run(context.Background(), []string{"restore", "abc", "123"}) + if err == nil || err.Error() != `too many arguments` { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrInvalidFlags", func(t *testing.T) { + m, _, _, _ := newMain() + err := m.Run(context.Background(), []string{"restore", "-no-such-flag"}) + if err == nil || err.Error() != `flag provided but not defined: -no-such-flag` { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrIndexFlagOnly", func(t *testing.T) { + m, _, _, _ := newMain() + err := m.Run(context.Background(), []string{"restore", "-index", "0", "/var/lib/db"}) + if err == nil || err.Error() != `must specify -generation flag when using -index flag` { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrConfigFileNotFound", func(t *testing.T) { + m, _, _, _ := newMain() + err := m.Run(context.Background(), []string{"restore", "-config", "/no/such/file", "/var/lib/db"}) + if err == nil || err.Error() != `config file not found: /no/such/file` { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrInvalidConfig", func(t *testing.T) { + testDir := filepath.Join("testdata", "restore", "invalid-config") + m, _, _, _ := newMain() + err := m.Run(context.Background(), []string{"restore", "-config", filepath.Join(testDir, "litestream.yml"), "/var/lib/db"}) + if err == nil || !strings.Contains(err.Error(), `replica path cannot be a url`) { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrMkdir", func(t *testing.T) { + tempDir := t.TempDir() + if err := os.Mkdir(filepath.Join(tempDir, "noperm"), 0000); err != nil { + t.Fatal(err) + } + + m, _, _, _ := newMain() + err := m.Run(context.Background(), []string{"restore", "-o", filepath.Join(tempDir, "noperm", "subdir", "db"), "/var/lib/db"}) + if err == nil || !strings.Contains(err.Error(), `permission denied`) { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrNoOutputPathWithReplicaURL", func(t *testing.T) { + m, _, _, _ := newMain() + err := m.Run(context.Background(), []string{"restore", "file://path/to/replica"}) + if err == nil || err.Error() != `output path required when using a replica URL` { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrReplicaNameWithReplicaURL", func(t *testing.T) { + m, _, _, _ := newMain() + err := m.Run(context.Background(), []string{"restore", "-replica", "replica0", "file://path/to/replica"}) + if err == nil || err.Error() != `cannot specify both the replica URL and the -replica flag` { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrInvalidReplicaURL", func(t *testing.T) { + m, _, _, _ := newMain() + err := m.Run(context.Background(), []string{"restore", "-o", "/tmp/db", "xyz://xyz"}) + if err == nil || err.Error() != `unknown replica type in config: "xyz"` { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrDatabaseNotFound", func(t *testing.T) { + testDir := filepath.Join("testdata", "restore", "database-not-found") + defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)() + + m, _, _, _ := newMain() + err := m.Run(context.Background(), []string{"restore", "-config", filepath.Join(testDir, "litestream.yml"), "/no/such/db"}) + if err == nil || err.Error() != `database not found in config: /no/such/db` { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrNoReplicas", func(t *testing.T) { + testDir := filepath.Join("testdata", "restore", "no-replicas") + defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)() + tempDir := t.TempDir() + + m, _, _, _ := newMain() + err := m.Run(context.Background(), []string{"restore", "-config", filepath.Join(testDir, "litestream.yml"), "-o", filepath.Join(tempDir, "db"), filepath.Join(testDir, "db")}) + if err == nil || err.Error() != `database has no replicas: `+filepath.Join(testingutil.Getwd(t), testDir, "db") { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrReplicaNotFound", func(t *testing.T) { + testDir := filepath.Join("testdata", "restore", "replica-not-found") + defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)() + tempDir := t.TempDir() + + m, _, _, _ := newMain() + err := m.Run(context.Background(), []string{"restore", "-config", filepath.Join(testDir, "litestream.yml"), "-o", filepath.Join(tempDir, "db"), "-replica", "no_such_replica", filepath.Join(testDir, "db")}) + if err == nil || err.Error() != `replica "no_such_replica" not found` { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrGenerationWithNoReplicaName", func(t *testing.T) { + testDir := filepath.Join("testdata", "restore", "generation-with-no-replica") + defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)() + tempDir := t.TempDir() + + m, _, _, _ := newMain() + err := m.Run(context.Background(), []string{"restore", "-config", filepath.Join(testDir, "litestream.yml"), "-o", filepath.Join(tempDir, "db"), "-generation", "0000000000000000", filepath.Join(testDir, "db")}) + if err == nil || err.Error() != `must specify -replica flag when restoring from a specific generation` { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrNoSnapshotsAvailable", func(t *testing.T) { + testDir := filepath.Join("testdata", "restore", "no-snapshots") + defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)() + tempDir := t.TempDir() + + m, _, _, _ := newMain() + err := m.Run(context.Background(), []string{"restore", "-config", filepath.Join(testDir, "litestream.yml"), "-o", filepath.Join(tempDir, "db"), "-generation", "0000000000000000", filepath.Join(testDir, "db")}) + if err == nil || err.Error() != `cannot determine latest index in generation "0000000000000000": no snapshots available` { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("Usage", func(t *testing.T) { + m, _, _, _ := newMain() + if err := m.Run(context.Background(), []string{"restore", "-h"}); err != flag.ErrHelp { + t.Fatalf("unexpected error: %s", err) + } + }) +} diff --git a/cmd/litestream/snapshots.go b/cmd/litestream/snapshots.go index d8f84fa..c274f3e 100644 --- a/cmd/litestream/snapshots.go +++ b/cmd/litestream/snapshots.go @@ -4,8 +4,8 @@ import ( "context" "flag" "fmt" + "io" "log" - "os" "sort" "text/tabwriter" "time" @@ -15,99 +15,89 @@ import ( // SnapshotsCommand represents a command to list snapshots for a command. type SnapshotsCommand struct { + stdin io.Reader + stdout io.Writer + stderr io.Writer + configPath string noExpandEnv bool + + replicaName string +} + +// NewSnapshotsCommand returns a new instance of SnapshotsCommand. +func NewSnapshotsCommand(stdin io.Reader, stdout, stderr io.Writer) *SnapshotsCommand { + return &SnapshotsCommand{ + stdin: stdin, + stdout: stdout, + stderr: stderr, + } } // Run executes the command. -func (c *SnapshotsCommand) Run(ctx context.Context, args []string) (err error) { +func (c *SnapshotsCommand) Run(ctx context.Context, args []string) (ret error) { fs := flag.NewFlagSet("litestream-snapshots", flag.ContinueOnError) registerConfigFlag(fs, &c.configPath, &c.noExpandEnv) - replicaName := fs.String("replica", "", "replica name") + fs.StringVar(&c.replicaName, "replica", "", "replica name") fs.Usage = c.Usage if err := fs.Parse(args); err != nil { return err } else if fs.NArg() == 0 || fs.Arg(0) == "" { - return fmt.Errorf("database path required") + return fmt.Errorf("database path or replica URL required") } else if fs.NArg() > 1 { return fmt.Errorf("too many arguments") } - var db *litestream.DB - var r *litestream.Replica - if isURL(fs.Arg(0)) { - if c.configPath != "" { - return fmt.Errorf("cannot specify a replica URL and the -config flag") - } - if r, err = NewReplicaFromConfig(&ReplicaConfig{URL: fs.Arg(0)}, nil); err != nil { - return err - } - } else { - if c.configPath == "" { - c.configPath = DefaultConfigPath() - } + // Load configuration. + config, err := ReadConfigFile(c.configPath, !c.noExpandEnv) + if err != nil { + return err + } - // Load configuration. - config, err := ReadConfigFile(c.configPath, !c.noExpandEnv) + // Determine list of replicas to pull snapshots from. + replicas, _, err := loadReplicas(ctx, config, fs.Arg(0), c.replicaName) + if err != nil { + return err + } + + // Build list of snapshot metadata with associated replica. + var infos []replicaSnapshotInfo + for _, r := range replicas { + a, err := r.Snapshots(ctx) if err != nil { - return err + log.Printf("cannot determine snapshots: %s", err) + ret = errExit // signal error return without printing message + continue } - - // Lookup database from configuration file by path. - if path, err := expand(fs.Arg(0)); err != nil { - return err - } else if dbc := config.DBConfig(path); dbc == nil { - return fmt.Errorf("database not found in config: %s", path) - } else if db, err = NewDBFromConfig(dbc); err != nil { - return err - } - - // Filter by replica, if specified. - if *replicaName != "" { - if r = db.Replica(*replicaName); r == nil { - return fmt.Errorf("replica %q not found for database %q", *replicaName, db.Path()) - } + for i := range a { + infos = append(infos, replicaSnapshotInfo{SnapshotInfo: a[i], replicaName: r.Name()}) } } - // Find snapshots by db or replica. - var replicas []*litestream.Replica - if r != nil { - replicas = []*litestream.Replica{r} - } else { - replicas = db.Replicas - } + // Sort snapshots by creation time from newest to oldest. + sort.Slice(infos, func(i, j int) bool { return infos[i].CreatedAt.After(infos[j].CreatedAt) }) // List all snapshots. - w := tabwriter.NewWriter(os.Stdout, 0, 8, 2, ' ', 0) + w := tabwriter.NewWriter(c.stdout, 0, 8, 2, ' ', 0) defer w.Flush() fmt.Fprintln(w, "replica\tgeneration\tindex\tsize\tcreated") - for _, r := range replicas { - infos, err := r.Snapshots(ctx) - if err != nil { - log.Printf("cannot determine snapshots: %s", err) - continue - } - // Sort snapshots by creation time from newest to oldest. - sort.Slice(infos, func(i, j int) bool { return infos[i].CreatedAt.After(infos[j].CreatedAt) }) - for _, info := range infos { - fmt.Fprintf(w, "%s\t%s\t%d\t%d\t%s\n", - r.Name(), - info.Generation, - info.Index, - info.Size, - info.CreatedAt.Format(time.RFC3339), - ) - } + for _, info := range infos { + fmt.Fprintf(w, "%s\t%s\t%08x\t%d\t%s\n", + info.replicaName, + info.Generation, + info.Index, + info.Size, + info.CreatedAt.Format(time.RFC3339), + ) } - return nil + return ret } // Usage prints the help screen to STDOUT. func (c *SnapshotsCommand) Usage() { - fmt.Printf(` + fmt.Fprintf(c.stdout, ` The snapshots command lists all snapshots available for a database or replica. Usage: @@ -143,3 +133,9 @@ Examples: DefaultConfigPath(), ) } + +// replicaSnapshotInfo represents snapshot metadata with associated replica name. +type replicaSnapshotInfo struct { + litestream.SnapshotInfo + replicaName string +} diff --git a/cmd/litestream/snapshots_test.go b/cmd/litestream/snapshots_test.go new file mode 100644 index 0000000..3dc9288 --- /dev/null +++ b/cmd/litestream/snapshots_test.go @@ -0,0 +1,128 @@ +package main_test + +import ( + "context" + "flag" + "path/filepath" + "strings" + "testing" + + "github.com/benbjohnson/litestream/internal/testingutil" +) + +func TestSnapshotsCommand(t *testing.T) { + t.Run("OK", func(t *testing.T) { + testDir := filepath.Join("testdata", "snapshots", "ok") + defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)() + + m, _, stdout, _ := newMain() + if err := m.Run(context.Background(), []string{"snapshots", "-config", filepath.Join(testDir, "litestream.yml"), filepath.Join(testDir, "db")}); err != nil { + t.Fatal(err) + } else if got, want := stdout.String(), string(testingutil.MustReadFile(t, filepath.Join(testDir, "stdout"))); got != want { + t.Fatalf("stdout=%q, want %q", got, want) + } + }) + + t.Run("ReplicaName", func(t *testing.T) { + testDir := filepath.Join("testdata", "snapshots", "replica-name") + defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)() + + m, _, stdout, _ := newMain() + if err := m.Run(context.Background(), []string{"snapshots", "-config", filepath.Join(testDir, "litestream.yml"), "-replica", "replica1", filepath.Join(testDir, "db")}); err != nil { + t.Fatal(err) + } else if got, want := stdout.String(), string(testingutil.MustReadFile(t, filepath.Join(testDir, "stdout"))); got != want { + t.Fatalf("stdout=%q, want %q", got, want) + } + }) + + t.Run("ReplicaURL", func(t *testing.T) { + testDir := filepath.Join(testingutil.Getwd(t), "testdata", "snapshots", "replica-url") + defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)() + replicaURL := "file://" + filepath.ToSlash(testDir) + "/replica" + + m, _, stdout, _ := newMain() + if err := m.Run(context.Background(), []string{"snapshots", replicaURL}); err != nil { + t.Fatal(err) + } else if got, want := stdout.String(), string(testingutil.MustReadFile(t, filepath.Join(testDir, "stdout"))); got != want { + t.Fatalf("stdout=%q, want %q", got, want) + } + }) + + t.Run("ErrDatabaseOrReplicaRequired", func(t *testing.T) { + m, _, _, _ := newMain() + err := m.Run(context.Background(), []string{"snapshots"}) + if err == nil || err.Error() != `database path or replica URL required` { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrTooManyArguments", func(t *testing.T) { + m, _, _, _ := newMain() + err := m.Run(context.Background(), []string{"snapshots", "abc", "123"}) + if err == nil || err.Error() != `too many arguments` { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrInvalidFlags", func(t *testing.T) { + m, _, _, _ := newMain() + err := m.Run(context.Background(), []string{"snapshots", "-no-such-flag"}) + if err == nil || err.Error() != `flag provided but not defined: -no-such-flag` { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrConfigFileNotFound", func(t *testing.T) { + m, _, _, _ := newMain() + err := m.Run(context.Background(), []string{"snapshots", "-config", "/no/such/file", "/var/lib/db"}) + if err == nil || err.Error() != `config file not found: /no/such/file` { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrInvalidConfig", func(t *testing.T) { + testDir := filepath.Join("testdata", "snapshots", "invalid-config") + m, _, _, _ := newMain() + err := m.Run(context.Background(), []string{"snapshots", "-config", filepath.Join(testDir, "litestream.yml"), "/var/lib/db"}) + if err == nil || !strings.Contains(err.Error(), `replica path cannot be a url`) { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrDatabaseNotFound", func(t *testing.T) { + testDir := filepath.Join("testdata", "snapshots", "database-not-found") + defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)() + + m, _, _, _ := newMain() + err := m.Run(context.Background(), []string{"snapshots", "-config", filepath.Join(testDir, "litestream.yml"), "/no/such/db"}) + if err == nil || err.Error() != `database not found in config: /no/such/db` { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrReplicaNotFound", func(t *testing.T) { + testDir := filepath.Join(testingutil.Getwd(t), "testdata", "snapshots", "replica-not-found") + defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)() + + m, _, _, _ := newMain() + err := m.Run(context.Background(), []string{"snapshots", "-config", filepath.Join(testDir, "litestream.yml"), "-replica", "no_such_replica", filepath.Join(testDir, "db")}) + if err == nil || err.Error() != `replica "no_such_replica" not found for database "`+filepath.Join(testDir, "db")+`"` { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrInvalidReplicaURL", func(t *testing.T) { + m, _, _, _ := newMain() + err := m.Run(context.Background(), []string{"snapshots", "xyz://xyz"}) + if err == nil || !strings.Contains(err.Error(), `unknown replica type in config: "xyz"`) { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("Usage", func(t *testing.T) { + m, _, _, _ := newMain() + if err := m.Run(context.Background(), []string{"snapshots", "-h"}); err != flag.ErrHelp { + t.Fatalf("unexpected error: %s", err) + } + }) +} diff --git a/cmd/litestream/testdata/Makefile b/cmd/litestream/testdata/Makefile new file mode 100644 index 0000000..14e2085 --- /dev/null +++ b/cmd/litestream/testdata/Makefile @@ -0,0 +1,13 @@ +.PHONY: default +default: + make -C generations/ok + make -C generations/no-database + make -C generations/replica-name + make -C generations/replica-url + make -C restore/latest-replica + make -C snapshots/ok + make -C snapshots/replica-name + make -C snapshots/replica-url + make -C wal/ok + make -C wal/replica-name + make -C wal/replica-url diff --git a/cmd/litestream/testdata/databases/invalid-config/litestream.yml b/cmd/litestream/testdata/databases/invalid-config/litestream.yml new file mode 100644 index 0000000..26eb1ff --- /dev/null +++ b/cmd/litestream/testdata/databases/invalid-config/litestream.yml @@ -0,0 +1,4 @@ +dbs: + - path: /var/lib/db + replicas: + - path: s3://bkt/db diff --git a/cmd/litestream/testdata/databases/no-config/.gitignore b/cmd/litestream/testdata/databases/no-config/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/cmd/litestream/testdata/databases/no-databases/litestream.yml b/cmd/litestream/testdata/databases/no-databases/litestream.yml new file mode 100644 index 0000000..f6fff35 --- /dev/null +++ b/cmd/litestream/testdata/databases/no-databases/litestream.yml @@ -0,0 +1 @@ +dbs: diff --git a/cmd/litestream/testdata/databases/no-databases/stdout b/cmd/litestream/testdata/databases/no-databases/stdout new file mode 100644 index 0000000..9f9c245 --- /dev/null +++ b/cmd/litestream/testdata/databases/no-databases/stdout @@ -0,0 +1 @@ +No databases found in config file. diff --git a/cmd/litestream/testdata/databases/ok/litestream.yml b/cmd/litestream/testdata/databases/ok/litestream.yml new file mode 100644 index 0000000..14788e4 --- /dev/null +++ b/cmd/litestream/testdata/databases/ok/litestream.yml @@ -0,0 +1,7 @@ +dbs: + - path: /var/lib/db + replicas: + - path: /var/lib/replica + - url: s3://mybkt/db + + - path: /my/other/db \ No newline at end of file diff --git a/cmd/litestream/testdata/databases/ok/stdout b/cmd/litestream/testdata/databases/ok/stdout new file mode 100644 index 0000000..58fcd65 --- /dev/null +++ b/cmd/litestream/testdata/databases/ok/stdout @@ -0,0 +1,3 @@ +path replicas +/var/lib/db file,s3 +/my/other/db diff --git a/cmd/litestream/testdata/generations/database-not-found/litestream.yml b/cmd/litestream/testdata/generations/database-not-found/litestream.yml new file mode 100644 index 0000000..266721e --- /dev/null +++ b/cmd/litestream/testdata/generations/database-not-found/litestream.yml @@ -0,0 +1,2 @@ +dbs: + - path: $LITESTREAM_TESTDIR/db diff --git a/cmd/litestream/testdata/generations/invalid-config/litestream.yml b/cmd/litestream/testdata/generations/invalid-config/litestream.yml new file mode 100644 index 0000000..26eb1ff --- /dev/null +++ b/cmd/litestream/testdata/generations/invalid-config/litestream.yml @@ -0,0 +1,4 @@ +dbs: + - path: /var/lib/db + replicas: + - path: s3://bkt/db diff --git a/cmd/litestream/testdata/generations/no-database/Makefile b/cmd/litestream/testdata/generations/no-database/Makefile new file mode 100644 index 0000000..793e5cd --- /dev/null +++ b/cmd/litestream/testdata/generations/no-database/Makefile @@ -0,0 +1,4 @@ +.PHONY: default +default: + TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/snapshots/00000000.snapshot.lz4 + TZ=UTC touch -ct 200001020000 replica/generations/0000000000000001/snapshots/00000000.snapshot.lz4 diff --git a/cmd/litestream/testdata/generations/no-database/litestream.yml b/cmd/litestream/testdata/generations/no-database/litestream.yml new file mode 100644 index 0000000..544b74f --- /dev/null +++ b/cmd/litestream/testdata/generations/no-database/litestream.yml @@ -0,0 +1,4 @@ +dbs: + - path: $LITESTREAM_TESTDIR/db + replicas: + - path: $LITESTREAM_TESTDIR/replica diff --git a/cmd/litestream/testdata/generations/no-database/replica/generations/0000000000000000/snapshots/00000000.snapshot.lz4 b/cmd/litestream/testdata/generations/no-database/replica/generations/0000000000000000/snapshots/00000000.snapshot.lz4 new file mode 100644 index 0000000..7536340 Binary files /dev/null and b/cmd/litestream/testdata/generations/no-database/replica/generations/0000000000000000/snapshots/00000000.snapshot.lz4 differ diff --git a/cmd/litestream/testdata/generations/no-database/replica/generations/0000000000000001/snapshots/00000000.snapshot.lz4 b/cmd/litestream/testdata/generations/no-database/replica/generations/0000000000000001/snapshots/00000000.snapshot.lz4 new file mode 100644 index 0000000..7536340 Binary files /dev/null and b/cmd/litestream/testdata/generations/no-database/replica/generations/0000000000000001/snapshots/00000000.snapshot.lz4 differ diff --git a/cmd/litestream/testdata/generations/no-database/stdout b/cmd/litestream/testdata/generations/no-database/stdout new file mode 100644 index 0000000..774650c --- /dev/null +++ b/cmd/litestream/testdata/generations/no-database/stdout @@ -0,0 +1,3 @@ +name generation lag start end +file 0000000000000000 - 2000-01-01T00:00:00Z 2000-01-01T00:00:00Z +file 0000000000000001 - 2000-01-02T00:00:00Z 2000-01-02T00:00:00Z diff --git a/cmd/litestream/testdata/generations/ok/Makefile b/cmd/litestream/testdata/generations/ok/Makefile new file mode 100644 index 0000000..51f5394 --- /dev/null +++ b/cmd/litestream/testdata/generations/ok/Makefile @@ -0,0 +1,9 @@ +.PHONY: default +default: + TZ=UTC touch -ct 200001030000 db + TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/snapshots/00000000.snapshot.lz4 + TZ=UTC touch -ct 200001020000 replica/generations/0000000000000000/snapshots/00000001.snapshot.lz4 + TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/wal/00000000/00000000.wal.lz4 + TZ=UTC touch -ct 200001020000 replica/generations/0000000000000000/wal/00000000/00000001.wal.lz4 + TZ=UTC touch -ct 200001030000 replica/generations/0000000000000000/wal/00000001/00000000.wal.lz4 + TZ=UTC touch -ct 200001010000 replica/generations/0000000000000001/snapshots/00000000.snapshot.lz4 diff --git a/cmd/litestream/testdata/generations/ok/db b/cmd/litestream/testdata/generations/ok/db new file mode 100644 index 0000000..e69de29 diff --git a/cmd/litestream/testdata/generations/ok/litestream.yml b/cmd/litestream/testdata/generations/ok/litestream.yml new file mode 100644 index 0000000..544b74f --- /dev/null +++ b/cmd/litestream/testdata/generations/ok/litestream.yml @@ -0,0 +1,4 @@ +dbs: + - path: $LITESTREAM_TESTDIR/db + replicas: + - path: $LITESTREAM_TESTDIR/replica diff --git a/cmd/litestream/testdata/generations/ok/replica/db b/cmd/litestream/testdata/generations/ok/replica/db new file mode 100644 index 0000000..e69de29 diff --git a/cmd/litestream/testdata/generations/ok/replica/generations/0000000000000000/snapshots/00000000.snapshot.lz4 b/cmd/litestream/testdata/generations/ok/replica/generations/0000000000000000/snapshots/00000000.snapshot.lz4 new file mode 100644 index 0000000..7536340 Binary files /dev/null and b/cmd/litestream/testdata/generations/ok/replica/generations/0000000000000000/snapshots/00000000.snapshot.lz4 differ diff --git a/cmd/litestream/testdata/generations/ok/replica/generations/0000000000000000/snapshots/00000001.snapshot.lz4 b/cmd/litestream/testdata/generations/ok/replica/generations/0000000000000000/snapshots/00000001.snapshot.lz4 new file mode 100644 index 0000000..7536340 Binary files /dev/null and b/cmd/litestream/testdata/generations/ok/replica/generations/0000000000000000/snapshots/00000001.snapshot.lz4 differ diff --git a/cmd/litestream/testdata/generations/ok/replica/generations/0000000000000000/wal/00000000/00000000.wal.lz4 b/cmd/litestream/testdata/generations/ok/replica/generations/0000000000000000/wal/00000000/00000000.wal.lz4 new file mode 100644 index 0000000..7536340 Binary files /dev/null and b/cmd/litestream/testdata/generations/ok/replica/generations/0000000000000000/wal/00000000/00000000.wal.lz4 differ diff --git a/cmd/litestream/testdata/generations/ok/replica/generations/0000000000000000/wal/00000000/00000001.wal.lz4 b/cmd/litestream/testdata/generations/ok/replica/generations/0000000000000000/wal/00000000/00000001.wal.lz4 new file mode 100644 index 0000000..7536340 Binary files /dev/null and b/cmd/litestream/testdata/generations/ok/replica/generations/0000000000000000/wal/00000000/00000001.wal.lz4 differ diff --git a/cmd/litestream/testdata/generations/ok/replica/generations/0000000000000000/wal/00000001/00000000.wal.lz4 b/cmd/litestream/testdata/generations/ok/replica/generations/0000000000000000/wal/00000001/00000000.wal.lz4 new file mode 100644 index 0000000..7536340 Binary files /dev/null and b/cmd/litestream/testdata/generations/ok/replica/generations/0000000000000000/wal/00000001/00000000.wal.lz4 differ diff --git a/cmd/litestream/testdata/generations/ok/replica/generations/0000000000000001/snapshots/00000000.snapshot.lz4 b/cmd/litestream/testdata/generations/ok/replica/generations/0000000000000001/snapshots/00000000.snapshot.lz4 new file mode 100644 index 0000000..7536340 Binary files /dev/null and b/cmd/litestream/testdata/generations/ok/replica/generations/0000000000000001/snapshots/00000000.snapshot.lz4 differ diff --git a/cmd/litestream/testdata/generations/ok/stdout b/cmd/litestream/testdata/generations/ok/stdout new file mode 100644 index 0000000..23d7795 --- /dev/null +++ b/cmd/litestream/testdata/generations/ok/stdout @@ -0,0 +1,3 @@ +name generation lag start end +file 0000000000000000 0s 2000-01-01T00:00:00Z 2000-01-03T00:00:00Z +file 0000000000000001 48h0m0s 2000-01-01T00:00:00Z 2000-01-01T00:00:00Z diff --git a/cmd/litestream/testdata/generations/replica-name/Makefile b/cmd/litestream/testdata/generations/replica-name/Makefile new file mode 100644 index 0000000..f6a5eae --- /dev/null +++ b/cmd/litestream/testdata/generations/replica-name/Makefile @@ -0,0 +1,5 @@ +.PHONY: default +default: + TZ=UTC touch -ct 200001030000 db + TZ=UTC touch -ct 200001010000 replica0/generations/0000000000000000/snapshots/00000000.snapshot.lz4 + TZ=UTC touch -ct 200001020000 replica1/generations/0000000000000001/snapshots/00000000.snapshot.lz4 diff --git a/cmd/litestream/testdata/generations/replica-name/db b/cmd/litestream/testdata/generations/replica-name/db new file mode 100644 index 0000000..e69de29 diff --git a/cmd/litestream/testdata/generations/replica-name/litestream.yml b/cmd/litestream/testdata/generations/replica-name/litestream.yml new file mode 100644 index 0000000..8511213 --- /dev/null +++ b/cmd/litestream/testdata/generations/replica-name/litestream.yml @@ -0,0 +1,7 @@ +dbs: + - path: $LITESTREAM_TESTDIR/db + replicas: + - name: replica0 + path: $LITESTREAM_TESTDIR/replica0 + - name: replica1 + path: $LITESTREAM_TESTDIR/replica1 diff --git a/cmd/litestream/testdata/generations/replica-name/replica0/db b/cmd/litestream/testdata/generations/replica-name/replica0/db new file mode 100644 index 0000000..e69de29 diff --git a/cmd/litestream/testdata/generations/replica-name/replica0/generations/0000000000000000/snapshots/00000000.snapshot.lz4 b/cmd/litestream/testdata/generations/replica-name/replica0/generations/0000000000000000/snapshots/00000000.snapshot.lz4 new file mode 100644 index 0000000..7536340 Binary files /dev/null and b/cmd/litestream/testdata/generations/replica-name/replica0/generations/0000000000000000/snapshots/00000000.snapshot.lz4 differ diff --git a/cmd/litestream/testdata/generations/replica-name/replica0/generations/0000000000000000/snapshots/00000001.snapshot.lz4 b/cmd/litestream/testdata/generations/replica-name/replica0/generations/0000000000000000/snapshots/00000001.snapshot.lz4 new file mode 100644 index 0000000..7536340 Binary files /dev/null and b/cmd/litestream/testdata/generations/replica-name/replica0/generations/0000000000000000/snapshots/00000001.snapshot.lz4 differ diff --git a/cmd/litestream/testdata/generations/replica-name/replica0/generations/0000000000000000/wal/00000000/00000000.wal.lz4 b/cmd/litestream/testdata/generations/replica-name/replica0/generations/0000000000000000/wal/00000000/00000000.wal.lz4 new file mode 100644 index 0000000..7536340 Binary files /dev/null and b/cmd/litestream/testdata/generations/replica-name/replica0/generations/0000000000000000/wal/00000000/00000000.wal.lz4 differ diff --git a/cmd/litestream/testdata/generations/replica-name/replica0/generations/0000000000000000/wal/00000000/00000001.wal.lz4 b/cmd/litestream/testdata/generations/replica-name/replica0/generations/0000000000000000/wal/00000000/00000001.wal.lz4 new file mode 100644 index 0000000..7536340 Binary files /dev/null and b/cmd/litestream/testdata/generations/replica-name/replica0/generations/0000000000000000/wal/00000000/00000001.wal.lz4 differ diff --git a/cmd/litestream/testdata/generations/replica-name/replica0/generations/0000000000000000/wal/00000001/00000000.wal.lz4 b/cmd/litestream/testdata/generations/replica-name/replica0/generations/0000000000000000/wal/00000001/00000000.wal.lz4 new file mode 100644 index 0000000..7536340 Binary files /dev/null and b/cmd/litestream/testdata/generations/replica-name/replica0/generations/0000000000000000/wal/00000001/00000000.wal.lz4 differ diff --git a/cmd/litestream/testdata/generations/replica-name/replica1/db b/cmd/litestream/testdata/generations/replica-name/replica1/db new file mode 100644 index 0000000..e69de29 diff --git a/cmd/litestream/testdata/generations/replica-name/replica1/generations/0000000000000001/snapshots/00000000.snapshot.lz4 b/cmd/litestream/testdata/generations/replica-name/replica1/generations/0000000000000001/snapshots/00000000.snapshot.lz4 new file mode 100644 index 0000000..7536340 Binary files /dev/null and b/cmd/litestream/testdata/generations/replica-name/replica1/generations/0000000000000001/snapshots/00000000.snapshot.lz4 differ diff --git a/cmd/litestream/testdata/generations/replica-name/stdout b/cmd/litestream/testdata/generations/replica-name/stdout new file mode 100644 index 0000000..111a6b2 --- /dev/null +++ b/cmd/litestream/testdata/generations/replica-name/stdout @@ -0,0 +1,2 @@ +name generation lag start end +replica1 0000000000000001 24h0m0s 2000-01-02T00:00:00Z 2000-01-02T00:00:00Z diff --git a/cmd/litestream/testdata/generations/replica-not-found/litestream.yml b/cmd/litestream/testdata/generations/replica-not-found/litestream.yml new file mode 100644 index 0000000..5d911bd --- /dev/null +++ b/cmd/litestream/testdata/generations/replica-not-found/litestream.yml @@ -0,0 +1,4 @@ +dbs: + - path: $LITESTREAM_TESTDIR/db + replicas: + - url: s3://bkt/db diff --git a/cmd/litestream/testdata/generations/replica-url/Makefile b/cmd/litestream/testdata/generations/replica-url/Makefile new file mode 100644 index 0000000..3125ed2 --- /dev/null +++ b/cmd/litestream/testdata/generations/replica-url/Makefile @@ -0,0 +1,9 @@ +.PHONY: default +default: + TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/snapshots/00000000.snapshot.lz4 + TZ=UTC touch -ct 200001020000 replica/generations/0000000000000000/snapshots/00000001.snapshot.lz4 + TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/wal/00000000/00000000.wal.lz4 + TZ=UTC touch -ct 200001020000 replica/generations/0000000000000000/wal/00000000/00000001.wal.lz4 + TZ=UTC touch -ct 200001030000 replica/generations/0000000000000000/wal/00000001/00000000.wal.lz4 + TZ=UTC touch -ct 200001020000 replica/generations/0000000000000001/snapshots/00000000.snapshot.lz4 + diff --git a/cmd/litestream/testdata/generations/replica-url/litestream.yml b/cmd/litestream/testdata/generations/replica-url/litestream.yml new file mode 100644 index 0000000..544b74f --- /dev/null +++ b/cmd/litestream/testdata/generations/replica-url/litestream.yml @@ -0,0 +1,4 @@ +dbs: + - path: $LITESTREAM_TESTDIR/db + replicas: + - path: $LITESTREAM_TESTDIR/replica diff --git a/cmd/litestream/testdata/generations/replica-url/replica/generations/0000000000000000/snapshots/00000000.snapshot.lz4 b/cmd/litestream/testdata/generations/replica-url/replica/generations/0000000000000000/snapshots/00000000.snapshot.lz4 new file mode 100644 index 0000000..7536340 Binary files /dev/null and b/cmd/litestream/testdata/generations/replica-url/replica/generations/0000000000000000/snapshots/00000000.snapshot.lz4 differ diff --git a/cmd/litestream/testdata/generations/replica-url/replica/generations/0000000000000000/snapshots/00000001.snapshot.lz4 b/cmd/litestream/testdata/generations/replica-url/replica/generations/0000000000000000/snapshots/00000001.snapshot.lz4 new file mode 100644 index 0000000..7536340 Binary files /dev/null and b/cmd/litestream/testdata/generations/replica-url/replica/generations/0000000000000000/snapshots/00000001.snapshot.lz4 differ diff --git a/cmd/litestream/testdata/generations/replica-url/replica/generations/0000000000000000/wal/00000000/00000000.wal.lz4 b/cmd/litestream/testdata/generations/replica-url/replica/generations/0000000000000000/wal/00000000/00000000.wal.lz4 new file mode 100644 index 0000000..7536340 Binary files /dev/null and b/cmd/litestream/testdata/generations/replica-url/replica/generations/0000000000000000/wal/00000000/00000000.wal.lz4 differ diff --git a/cmd/litestream/testdata/generations/replica-url/replica/generations/0000000000000000/wal/00000000/00000001.wal.lz4 b/cmd/litestream/testdata/generations/replica-url/replica/generations/0000000000000000/wal/00000000/00000001.wal.lz4 new file mode 100644 index 0000000..7536340 Binary files /dev/null and b/cmd/litestream/testdata/generations/replica-url/replica/generations/0000000000000000/wal/00000000/00000001.wal.lz4 differ diff --git a/cmd/litestream/testdata/generations/replica-url/replica/generations/0000000000000000/wal/00000001/00000000.wal.lz4 b/cmd/litestream/testdata/generations/replica-url/replica/generations/0000000000000000/wal/00000001/00000000.wal.lz4 new file mode 100644 index 0000000..7536340 Binary files /dev/null and b/cmd/litestream/testdata/generations/replica-url/replica/generations/0000000000000000/wal/00000001/00000000.wal.lz4 differ diff --git a/cmd/litestream/testdata/generations/replica-url/replica/generations/0000000000000001/snapshots/00000000.snapshot.lz4 b/cmd/litestream/testdata/generations/replica-url/replica/generations/0000000000000001/snapshots/00000000.snapshot.lz4 new file mode 100644 index 0000000..7536340 Binary files /dev/null and b/cmd/litestream/testdata/generations/replica-url/replica/generations/0000000000000001/snapshots/00000000.snapshot.lz4 differ diff --git a/cmd/litestream/testdata/generations/replica-url/stdout b/cmd/litestream/testdata/generations/replica-url/stdout new file mode 100644 index 0000000..e099c74 --- /dev/null +++ b/cmd/litestream/testdata/generations/replica-url/stdout @@ -0,0 +1,3 @@ +name generation lag start end +file 0000000000000000 - 2000-01-01T00:00:00Z 2000-01-03T00:00:00Z +file 0000000000000001 - 2000-01-02T00:00:00Z 2000-01-02T00:00:00Z diff --git a/cmd/litestream/testdata/restore/database-not-found/litestream.yml b/cmd/litestream/testdata/restore/database-not-found/litestream.yml new file mode 100644 index 0000000..544b74f --- /dev/null +++ b/cmd/litestream/testdata/restore/database-not-found/litestream.yml @@ -0,0 +1,4 @@ +dbs: + - path: $LITESTREAM_TESTDIR/db + replicas: + - path: $LITESTREAM_TESTDIR/replica diff --git a/cmd/litestream/testdata/restore/generation-with-no-replica/litestream.yml b/cmd/litestream/testdata/restore/generation-with-no-replica/litestream.yml new file mode 100644 index 0000000..8696dbe --- /dev/null +++ b/cmd/litestream/testdata/restore/generation-with-no-replica/litestream.yml @@ -0,0 +1,5 @@ +dbs: + - path: $LITESTREAM_TESTDIR/db + replicas: + - path: $LITESTREAM_TESTDIR/replica0 + - path: $LITESTREAM_TESTDIR/replica1 diff --git a/cmd/litestream/testdata/restore/if-db-not-exists-flag/db b/cmd/litestream/testdata/restore/if-db-not-exists-flag/db new file mode 100644 index 0000000..cfd2b8d Binary files /dev/null and b/cmd/litestream/testdata/restore/if-db-not-exists-flag/db differ diff --git a/cmd/litestream/testdata/restore/if-db-not-exists-flag/litestream.yml b/cmd/litestream/testdata/restore/if-db-not-exists-flag/litestream.yml new file mode 100644 index 0000000..544b74f --- /dev/null +++ b/cmd/litestream/testdata/restore/if-db-not-exists-flag/litestream.yml @@ -0,0 +1,4 @@ +dbs: + - path: $LITESTREAM_TESTDIR/db + replicas: + - path: $LITESTREAM_TESTDIR/replica diff --git a/cmd/litestream/testdata/restore/if-db-not-exists-flag/stdout b/cmd/litestream/testdata/restore/if-db-not-exists-flag/stdout new file mode 100644 index 0000000..2e1bdc7 --- /dev/null +++ b/cmd/litestream/testdata/restore/if-db-not-exists-flag/stdout @@ -0,0 +1 @@ +database already exists, skipping diff --git a/cmd/litestream/testdata/restore/if-replica-exists-flag/litestream.yml b/cmd/litestream/testdata/restore/if-replica-exists-flag/litestream.yml new file mode 100644 index 0000000..544b74f --- /dev/null +++ b/cmd/litestream/testdata/restore/if-replica-exists-flag/litestream.yml @@ -0,0 +1,4 @@ +dbs: + - path: $LITESTREAM_TESTDIR/db + replicas: + - path: $LITESTREAM_TESTDIR/replica diff --git a/cmd/litestream/testdata/restore/if-replica-exists-flag/stdout b/cmd/litestream/testdata/restore/if-replica-exists-flag/stdout new file mode 100644 index 0000000..bb156b9 --- /dev/null +++ b/cmd/litestream/testdata/restore/if-replica-exists-flag/stdout @@ -0,0 +1 @@ +no matching backups found, skipping diff --git a/cmd/litestream/testdata/restore/invalid-config/litestream.yml b/cmd/litestream/testdata/restore/invalid-config/litestream.yml new file mode 100644 index 0000000..26eb1ff --- /dev/null +++ b/cmd/litestream/testdata/restore/invalid-config/litestream.yml @@ -0,0 +1,4 @@ +dbs: + - path: /var/lib/db + replicas: + - path: s3://bkt/db diff --git a/cmd/litestream/testdata/restore/latest-replica/Makefile b/cmd/litestream/testdata/restore/latest-replica/Makefile new file mode 100644 index 0000000..a8a9885 --- /dev/null +++ b/cmd/litestream/testdata/restore/latest-replica/Makefile @@ -0,0 +1,6 @@ +.PHONY: default +default: + TZ=UTC touch -ct 200001010000 replica0/generations/0000000000000000/snapshots/00000000.snapshot.lz4 + TZ=UTC touch -ct 200001020000 replica1/generations/0000000000000002/snapshots/00000000.snapshot.lz4 + TZ=UTC touch -ct 200001030000 replica0/generations/0000000000000001/snapshots/00000000.snapshot.lz4 + diff --git a/cmd/litestream/testdata/restore/latest-replica/litestream.yml b/cmd/litestream/testdata/restore/latest-replica/litestream.yml new file mode 100644 index 0000000..8511213 --- /dev/null +++ b/cmd/litestream/testdata/restore/latest-replica/litestream.yml @@ -0,0 +1,7 @@ +dbs: + - path: $LITESTREAM_TESTDIR/db + replicas: + - name: replica0 + path: $LITESTREAM_TESTDIR/replica0 + - name: replica1 + path: $LITESTREAM_TESTDIR/replica1 diff --git a/cmd/litestream/testdata/restore/latest-replica/replica0/generations/0000000000000000/snapshots/00000000.snapshot.lz4 b/cmd/litestream/testdata/restore/latest-replica/replica0/generations/0000000000000000/snapshots/00000000.snapshot.lz4 new file mode 100644 index 0000000..7536340 Binary files /dev/null and b/cmd/litestream/testdata/restore/latest-replica/replica0/generations/0000000000000000/snapshots/00000000.snapshot.lz4 differ diff --git a/cmd/litestream/testdata/restore/latest-replica/replica0/generations/0000000000000001/snapshots/00000000.snapshot.lz4 b/cmd/litestream/testdata/restore/latest-replica/replica0/generations/0000000000000001/snapshots/00000000.snapshot.lz4 new file mode 100644 index 0000000..7536340 Binary files /dev/null and b/cmd/litestream/testdata/restore/latest-replica/replica0/generations/0000000000000001/snapshots/00000000.snapshot.lz4 differ diff --git a/cmd/litestream/testdata/restore/latest-replica/replica1/generations/0000000000000002/snapshots/00000000.snapshot.lz4 b/cmd/litestream/testdata/restore/latest-replica/replica1/generations/0000000000000002/snapshots/00000000.snapshot.lz4 new file mode 100644 index 0000000..7536340 Binary files /dev/null and b/cmd/litestream/testdata/restore/latest-replica/replica1/generations/0000000000000002/snapshots/00000000.snapshot.lz4 differ diff --git a/cmd/litestream/testdata/restore/no-backups/litestream.yml b/cmd/litestream/testdata/restore/no-backups/litestream.yml new file mode 100644 index 0000000..544b74f --- /dev/null +++ b/cmd/litestream/testdata/restore/no-backups/litestream.yml @@ -0,0 +1,4 @@ +dbs: + - path: $LITESTREAM_TESTDIR/db + replicas: + - path: $LITESTREAM_TESTDIR/replica diff --git a/cmd/litestream/testdata/restore/no-backups/stderr b/cmd/litestream/testdata/restore/no-backups/stderr new file mode 100644 index 0000000..e69de29 diff --git a/cmd/litestream/testdata/restore/no-backups/stdout b/cmd/litestream/testdata/restore/no-backups/stdout new file mode 100644 index 0000000..e69de29 diff --git a/cmd/litestream/testdata/restore/no-generation/litestream.yml b/cmd/litestream/testdata/restore/no-generation/litestream.yml new file mode 100644 index 0000000..544b74f --- /dev/null +++ b/cmd/litestream/testdata/restore/no-generation/litestream.yml @@ -0,0 +1,4 @@ +dbs: + - path: $LITESTREAM_TESTDIR/db + replicas: + - path: $LITESTREAM_TESTDIR/replica diff --git a/cmd/litestream/testdata/restore/no-replicas/litestream.yml b/cmd/litestream/testdata/restore/no-replicas/litestream.yml new file mode 100644 index 0000000..266721e --- /dev/null +++ b/cmd/litestream/testdata/restore/no-replicas/litestream.yml @@ -0,0 +1,2 @@ +dbs: + - path: $LITESTREAM_TESTDIR/db diff --git a/cmd/litestream/testdata/restore/no-snapshots/litestream.yml b/cmd/litestream/testdata/restore/no-snapshots/litestream.yml new file mode 100644 index 0000000..544b74f --- /dev/null +++ b/cmd/litestream/testdata/restore/no-snapshots/litestream.yml @@ -0,0 +1,4 @@ +dbs: + - path: $LITESTREAM_TESTDIR/db + replicas: + - path: $LITESTREAM_TESTDIR/replica diff --git a/cmd/litestream/testdata/restore/ok/00000002.db b/cmd/litestream/testdata/restore/ok/00000002.db new file mode 100644 index 0000000..cfd2b8d Binary files /dev/null and b/cmd/litestream/testdata/restore/ok/00000002.db differ diff --git a/cmd/litestream/testdata/restore/ok/README b/cmd/litestream/testdata/restore/ok/README new file mode 100644 index 0000000..9450f45 --- /dev/null +++ b/cmd/litestream/testdata/restore/ok/README @@ -0,0 +1,36 @@ +To reproduce this testdata, run sqlite3 and execute: + + PRAGMA journal_mode = WAL; + CREATE TABLE t (x); + INSERT INTO t (x) VALUES (1); + INSERT INTO t (x) VALUES (2); + + sl3 split -o generations/0000000000000000/wal/00000000 db-wal + cp db generations/0000000000000000/snapshots/00000000.snapshot + lz4 -c --rm generations/0000000000000000/snapshots/00000000.snapshot + + +Then execute: + + PRAGMA wal_checkpoint(TRUNCATE); + INSERT INTO t (x) VALUES (3); + + sl3 split -o generations/0000000000000000/wal/00000001 db-wal + + +Then execute: + + PRAGMA wal_checkpoint(TRUNCATE); + INSERT INTO t (x) VALUES (4); + INSERT INTO t (x) VALUES (5); + + sl3 split -o generations/0000000000000000/wal/00000002 db-wal + + +Finally, obtain the final snapshot: + + PRAGMA wal_checkpoint(TRUNCATE); + + cp db 00000002.db + rm db* + diff --git a/cmd/litestream/testdata/restore/ok/litestream.yml b/cmd/litestream/testdata/restore/ok/litestream.yml new file mode 100644 index 0000000..544b74f --- /dev/null +++ b/cmd/litestream/testdata/restore/ok/litestream.yml @@ -0,0 +1,4 @@ +dbs: + - path: $LITESTREAM_TESTDIR/db + replicas: + - path: $LITESTREAM_TESTDIR/replica diff --git a/cmd/litestream/testdata/restore/ok/replica/generations/0000000000000000/snapshots/00000000.snapshot.lz4 b/cmd/litestream/testdata/restore/ok/replica/generations/0000000000000000/snapshots/00000000.snapshot.lz4 new file mode 100644 index 0000000..7536340 Binary files /dev/null and b/cmd/litestream/testdata/restore/ok/replica/generations/0000000000000000/snapshots/00000000.snapshot.lz4 differ diff --git a/cmd/litestream/testdata/restore/ok/replica/generations/0000000000000000/wal/00000000/00000000.wal.lz4 b/cmd/litestream/testdata/restore/ok/replica/generations/0000000000000000/wal/00000000/00000000.wal.lz4 new file mode 100644 index 0000000..37e1dcf Binary files /dev/null and b/cmd/litestream/testdata/restore/ok/replica/generations/0000000000000000/wal/00000000/00000000.wal.lz4 differ diff --git a/cmd/litestream/testdata/restore/ok/replica/generations/0000000000000000/wal/00000000/00002050.wal.lz4 b/cmd/litestream/testdata/restore/ok/replica/generations/0000000000000000/wal/00000000/00002050.wal.lz4 new file mode 100644 index 0000000..3bd7ab7 Binary files /dev/null and b/cmd/litestream/testdata/restore/ok/replica/generations/0000000000000000/wal/00000000/00002050.wal.lz4 differ diff --git a/cmd/litestream/testdata/restore/ok/replica/generations/0000000000000000/wal/00000000/00003068.wal.lz4 b/cmd/litestream/testdata/restore/ok/replica/generations/0000000000000000/wal/00000000/00003068.wal.lz4 new file mode 100644 index 0000000..c73bf2c Binary files /dev/null and b/cmd/litestream/testdata/restore/ok/replica/generations/0000000000000000/wal/00000000/00003068.wal.lz4 differ diff --git a/cmd/litestream/testdata/restore/ok/replica/generations/0000000000000000/wal/00000001/00000000.wal.lz4 b/cmd/litestream/testdata/restore/ok/replica/generations/0000000000000000/wal/00000001/00000000.wal.lz4 new file mode 100644 index 0000000..64a4899 Binary files /dev/null and b/cmd/litestream/testdata/restore/ok/replica/generations/0000000000000000/wal/00000001/00000000.wal.lz4 differ diff --git a/cmd/litestream/testdata/restore/ok/replica/generations/0000000000000000/wal/00000002/00000000.wal.lz4 b/cmd/litestream/testdata/restore/ok/replica/generations/0000000000000000/wal/00000002/00000000.wal.lz4 new file mode 100644 index 0000000..2265d0e Binary files /dev/null and b/cmd/litestream/testdata/restore/ok/replica/generations/0000000000000000/wal/00000002/00000000.wal.lz4 differ diff --git a/cmd/litestream/testdata/restore/ok/replica/generations/0000000000000000/wal/00000002/00001038.wal.lz4 b/cmd/litestream/testdata/restore/ok/replica/generations/0000000000000000/wal/00000002/00001038.wal.lz4 new file mode 100644 index 0000000..c7dc94f Binary files /dev/null and b/cmd/litestream/testdata/restore/ok/replica/generations/0000000000000000/wal/00000002/00001038.wal.lz4 differ diff --git a/cmd/litestream/testdata/restore/output-path-exists/db b/cmd/litestream/testdata/restore/output-path-exists/db new file mode 100644 index 0000000..cfd2b8d Binary files /dev/null and b/cmd/litestream/testdata/restore/output-path-exists/db differ diff --git a/cmd/litestream/testdata/restore/output-path-exists/litestream.yml b/cmd/litestream/testdata/restore/output-path-exists/litestream.yml new file mode 100644 index 0000000..544b74f --- /dev/null +++ b/cmd/litestream/testdata/restore/output-path-exists/litestream.yml @@ -0,0 +1,4 @@ +dbs: + - path: $LITESTREAM_TESTDIR/db + replicas: + - path: $LITESTREAM_TESTDIR/replica diff --git a/cmd/litestream/testdata/restore/replica-name/litestream.yml b/cmd/litestream/testdata/restore/replica-name/litestream.yml new file mode 100644 index 0000000..8511213 --- /dev/null +++ b/cmd/litestream/testdata/restore/replica-name/litestream.yml @@ -0,0 +1,7 @@ +dbs: + - path: $LITESTREAM_TESTDIR/db + replicas: + - name: replica0 + path: $LITESTREAM_TESTDIR/replica0 + - name: replica1 + path: $LITESTREAM_TESTDIR/replica1 diff --git a/cmd/litestream/testdata/restore/replica-name/replica0/generations/0000000000000000/snapshots/00000000.snapshot.lz4 b/cmd/litestream/testdata/restore/replica-name/replica0/generations/0000000000000000/snapshots/00000000.snapshot.lz4 new file mode 100644 index 0000000..7536340 Binary files /dev/null and b/cmd/litestream/testdata/restore/replica-name/replica0/generations/0000000000000000/snapshots/00000000.snapshot.lz4 differ diff --git a/cmd/litestream/testdata/restore/replica-name/replica1/generations/0000000000000001/snapshots/00000001.snapshot.lz4 b/cmd/litestream/testdata/restore/replica-name/replica1/generations/0000000000000001/snapshots/00000001.snapshot.lz4 new file mode 100644 index 0000000..7536340 Binary files /dev/null and b/cmd/litestream/testdata/restore/replica-name/replica1/generations/0000000000000001/snapshots/00000001.snapshot.lz4 differ diff --git a/cmd/litestream/testdata/restore/replica-not-found/litestream.yml b/cmd/litestream/testdata/restore/replica-not-found/litestream.yml new file mode 100644 index 0000000..b2a5e14 --- /dev/null +++ b/cmd/litestream/testdata/restore/replica-not-found/litestream.yml @@ -0,0 +1,5 @@ +dbs: + - path: $LITESTREAM_TESTDIR/db + replicas: + - name: replica0 + path: $LITESTREAM_TESTDIR/replica diff --git a/cmd/litestream/testdata/restore/replica-url/replica/generations/0000000000000000/snapshots/00000000.snapshot.lz4 b/cmd/litestream/testdata/restore/replica-url/replica/generations/0000000000000000/snapshots/00000000.snapshot.lz4 new file mode 100644 index 0000000..7536340 Binary files /dev/null and b/cmd/litestream/testdata/restore/replica-url/replica/generations/0000000000000000/snapshots/00000000.snapshot.lz4 differ diff --git a/cmd/litestream/testdata/snapshots/database-not-found/litestream.yml b/cmd/litestream/testdata/snapshots/database-not-found/litestream.yml new file mode 100644 index 0000000..266721e --- /dev/null +++ b/cmd/litestream/testdata/snapshots/database-not-found/litestream.yml @@ -0,0 +1,2 @@ +dbs: + - path: $LITESTREAM_TESTDIR/db diff --git a/cmd/litestream/testdata/snapshots/invalid-config/litestream.yml b/cmd/litestream/testdata/snapshots/invalid-config/litestream.yml new file mode 100644 index 0000000..26eb1ff --- /dev/null +++ b/cmd/litestream/testdata/snapshots/invalid-config/litestream.yml @@ -0,0 +1,4 @@ +dbs: + - path: /var/lib/db + replicas: + - path: s3://bkt/db diff --git a/cmd/litestream/testdata/snapshots/ok/Makefile b/cmd/litestream/testdata/snapshots/ok/Makefile new file mode 100644 index 0000000..866903e --- /dev/null +++ b/cmd/litestream/testdata/snapshots/ok/Makefile @@ -0,0 +1,6 @@ +.PHONY: default +default: + TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/snapshots/00000000.snapshot.lz4 + TZ=UTC touch -ct 200001020000 replica/generations/0000000000000000/snapshots/00000001.snapshot.lz4 + TZ=UTC touch -ct 200001030000 replica/generations/0000000000000001/snapshots/00000000.snapshot.lz4 + diff --git a/cmd/litestream/testdata/snapshots/ok/litestream.yml b/cmd/litestream/testdata/snapshots/ok/litestream.yml new file mode 100644 index 0000000..544b74f --- /dev/null +++ b/cmd/litestream/testdata/snapshots/ok/litestream.yml @@ -0,0 +1,4 @@ +dbs: + - path: $LITESTREAM_TESTDIR/db + replicas: + - path: $LITESTREAM_TESTDIR/replica diff --git a/cmd/litestream/testdata/snapshots/ok/replica/generations/0000000000000000/snapshots/00000000.snapshot.lz4 b/cmd/litestream/testdata/snapshots/ok/replica/generations/0000000000000000/snapshots/00000000.snapshot.lz4 new file mode 100644 index 0000000..7536340 Binary files /dev/null and b/cmd/litestream/testdata/snapshots/ok/replica/generations/0000000000000000/snapshots/00000000.snapshot.lz4 differ diff --git a/cmd/litestream/testdata/snapshots/ok/replica/generations/0000000000000000/snapshots/00000001.snapshot.lz4 b/cmd/litestream/testdata/snapshots/ok/replica/generations/0000000000000000/snapshots/00000001.snapshot.lz4 new file mode 100644 index 0000000..7536340 Binary files /dev/null and b/cmd/litestream/testdata/snapshots/ok/replica/generations/0000000000000000/snapshots/00000001.snapshot.lz4 differ diff --git a/cmd/litestream/testdata/snapshots/ok/replica/generations/0000000000000001/snapshots/00000000.snapshot.lz4 b/cmd/litestream/testdata/snapshots/ok/replica/generations/0000000000000001/snapshots/00000000.snapshot.lz4 new file mode 100644 index 0000000..7536340 Binary files /dev/null and b/cmd/litestream/testdata/snapshots/ok/replica/generations/0000000000000001/snapshots/00000000.snapshot.lz4 differ diff --git a/cmd/litestream/testdata/snapshots/ok/stdout b/cmd/litestream/testdata/snapshots/ok/stdout new file mode 100644 index 0000000..270a252 --- /dev/null +++ b/cmd/litestream/testdata/snapshots/ok/stdout @@ -0,0 +1,4 @@ +replica generation index size created +file 0000000000000001 00000000 93 2000-01-03T00:00:00Z +file 0000000000000000 00000001 93 2000-01-02T00:00:00Z +file 0000000000000000 00000000 93 2000-01-01T00:00:00Z diff --git a/cmd/litestream/testdata/snapshots/replica-name/Makefile b/cmd/litestream/testdata/snapshots/replica-name/Makefile new file mode 100644 index 0000000..050a241 --- /dev/null +++ b/cmd/litestream/testdata/snapshots/replica-name/Makefile @@ -0,0 +1,4 @@ +.PHONY: default +default: + TZ=UTC touch -ct 200001010000 replica0/generations/0000000000000000/snapshots/00000000.snapshot.lz4 + TZ=UTC touch -ct 200001020000 replica1/generations/0000000000000001/snapshots/00000000.snapshot.lz4 diff --git a/cmd/litestream/testdata/snapshots/replica-name/litestream.yml b/cmd/litestream/testdata/snapshots/replica-name/litestream.yml new file mode 100644 index 0000000..8511213 --- /dev/null +++ b/cmd/litestream/testdata/snapshots/replica-name/litestream.yml @@ -0,0 +1,7 @@ +dbs: + - path: $LITESTREAM_TESTDIR/db + replicas: + - name: replica0 + path: $LITESTREAM_TESTDIR/replica0 + - name: replica1 + path: $LITESTREAM_TESTDIR/replica1 diff --git a/cmd/litestream/testdata/snapshots/replica-name/replica0/generations/0000000000000000/snapshots/00000000.snapshot.lz4 b/cmd/litestream/testdata/snapshots/replica-name/replica0/generations/0000000000000000/snapshots/00000000.snapshot.lz4 new file mode 100644 index 0000000..7536340 Binary files /dev/null and b/cmd/litestream/testdata/snapshots/replica-name/replica0/generations/0000000000000000/snapshots/00000000.snapshot.lz4 differ diff --git a/cmd/litestream/testdata/snapshots/replica-name/replica0/generations/0000000000000000/snapshots/00000001.snapshot.lz4 b/cmd/litestream/testdata/snapshots/replica-name/replica0/generations/0000000000000000/snapshots/00000001.snapshot.lz4 new file mode 100644 index 0000000..7536340 Binary files /dev/null and b/cmd/litestream/testdata/snapshots/replica-name/replica0/generations/0000000000000000/snapshots/00000001.snapshot.lz4 differ diff --git a/cmd/litestream/testdata/snapshots/replica-name/replica1/generations/0000000000000001/snapshots/00000000.snapshot.lz4 b/cmd/litestream/testdata/snapshots/replica-name/replica1/generations/0000000000000001/snapshots/00000000.snapshot.lz4 new file mode 100644 index 0000000..7536340 Binary files /dev/null and b/cmd/litestream/testdata/snapshots/replica-name/replica1/generations/0000000000000001/snapshots/00000000.snapshot.lz4 differ diff --git a/cmd/litestream/testdata/snapshots/replica-name/stdout b/cmd/litestream/testdata/snapshots/replica-name/stdout new file mode 100644 index 0000000..42c074e --- /dev/null +++ b/cmd/litestream/testdata/snapshots/replica-name/stdout @@ -0,0 +1,2 @@ +replica generation index size created +replica1 0000000000000001 00000000 93 2000-01-02T00:00:00Z diff --git a/cmd/litestream/testdata/snapshots/replica-not-found/litestream.yml b/cmd/litestream/testdata/snapshots/replica-not-found/litestream.yml new file mode 100644 index 0000000..5d911bd --- /dev/null +++ b/cmd/litestream/testdata/snapshots/replica-not-found/litestream.yml @@ -0,0 +1,4 @@ +dbs: + - path: $LITESTREAM_TESTDIR/db + replicas: + - url: s3://bkt/db diff --git a/cmd/litestream/testdata/snapshots/replica-url/Makefile b/cmd/litestream/testdata/snapshots/replica-url/Makefile new file mode 100644 index 0000000..f300c83 --- /dev/null +++ b/cmd/litestream/testdata/snapshots/replica-url/Makefile @@ -0,0 +1,5 @@ +.PHONY: default +default: + TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/snapshots/00000000.snapshot.lz4 + TZ=UTC touch -ct 200001020000 replica/generations/0000000000000000/snapshots/00000001.snapshot.lz4 + TZ=UTC touch -ct 200001030000 replica/generations/0000000000000001/snapshots/00000000.snapshot.lz4 diff --git a/cmd/litestream/testdata/snapshots/replica-url/replica/generations/0000000000000000/snapshots/00000000.snapshot.lz4 b/cmd/litestream/testdata/snapshots/replica-url/replica/generations/0000000000000000/snapshots/00000000.snapshot.lz4 new file mode 100644 index 0000000..7536340 Binary files /dev/null and b/cmd/litestream/testdata/snapshots/replica-url/replica/generations/0000000000000000/snapshots/00000000.snapshot.lz4 differ diff --git a/cmd/litestream/testdata/snapshots/replica-url/replica/generations/0000000000000000/snapshots/00000001.snapshot.lz4 b/cmd/litestream/testdata/snapshots/replica-url/replica/generations/0000000000000000/snapshots/00000001.snapshot.lz4 new file mode 100644 index 0000000..7536340 Binary files /dev/null and b/cmd/litestream/testdata/snapshots/replica-url/replica/generations/0000000000000000/snapshots/00000001.snapshot.lz4 differ diff --git a/cmd/litestream/testdata/snapshots/replica-url/replica/generations/0000000000000001/snapshots/00000000.snapshot.lz4 b/cmd/litestream/testdata/snapshots/replica-url/replica/generations/0000000000000001/snapshots/00000000.snapshot.lz4 new file mode 100644 index 0000000..7536340 Binary files /dev/null and b/cmd/litestream/testdata/snapshots/replica-url/replica/generations/0000000000000001/snapshots/00000000.snapshot.lz4 differ diff --git a/cmd/litestream/testdata/snapshots/replica-url/stdout b/cmd/litestream/testdata/snapshots/replica-url/stdout new file mode 100644 index 0000000..270a252 --- /dev/null +++ b/cmd/litestream/testdata/snapshots/replica-url/stdout @@ -0,0 +1,4 @@ +replica generation index size created +file 0000000000000001 00000000 93 2000-01-03T00:00:00Z +file 0000000000000000 00000001 93 2000-01-02T00:00:00Z +file 0000000000000000 00000000 93 2000-01-01T00:00:00Z diff --git a/cmd/litestream/testdata/wal/database-not-found/litestream.yml b/cmd/litestream/testdata/wal/database-not-found/litestream.yml new file mode 100644 index 0000000..266721e --- /dev/null +++ b/cmd/litestream/testdata/wal/database-not-found/litestream.yml @@ -0,0 +1,2 @@ +dbs: + - path: $LITESTREAM_TESTDIR/db diff --git a/cmd/litestream/testdata/wal/invalid-config/litestream.yml b/cmd/litestream/testdata/wal/invalid-config/litestream.yml new file mode 100644 index 0000000..26eb1ff --- /dev/null +++ b/cmd/litestream/testdata/wal/invalid-config/litestream.yml @@ -0,0 +1,4 @@ +dbs: + - path: /var/lib/db + replicas: + - path: s3://bkt/db diff --git a/cmd/litestream/testdata/wal/ok/Makefile b/cmd/litestream/testdata/wal/ok/Makefile new file mode 100644 index 0000000..2bb5a8e --- /dev/null +++ b/cmd/litestream/testdata/wal/ok/Makefile @@ -0,0 +1,7 @@ +.PHONY: default +default: + TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/wal/00000000/00000000.wal.lz4 + TZ=UTC touch -ct 200001020000 replica/generations/0000000000000000/wal/00000000/00000001.wal.lz4 + TZ=UTC touch -ct 200001030000 replica/generations/0000000000000000/wal/00000001/00000000.wal.lz4 + TZ=UTC touch -ct 200001040000 replica/generations/0000000000000001/wal/00000000/00000000.wal.lz4 + diff --git a/cmd/litestream/testdata/wal/ok/litestream.yml b/cmd/litestream/testdata/wal/ok/litestream.yml new file mode 100644 index 0000000..544b74f --- /dev/null +++ b/cmd/litestream/testdata/wal/ok/litestream.yml @@ -0,0 +1,4 @@ +dbs: + - path: $LITESTREAM_TESTDIR/db + replicas: + - path: $LITESTREAM_TESTDIR/replica diff --git a/cmd/litestream/testdata/wal/ok/replica/generations/0000000000000000/wal/00000000/00000000.wal.lz4 b/cmd/litestream/testdata/wal/ok/replica/generations/0000000000000000/wal/00000000/00000000.wal.lz4 new file mode 100644 index 0000000..7536340 Binary files /dev/null and b/cmd/litestream/testdata/wal/ok/replica/generations/0000000000000000/wal/00000000/00000000.wal.lz4 differ diff --git a/cmd/litestream/testdata/wal/ok/replica/generations/0000000000000000/wal/00000000/00000001.wal.lz4 b/cmd/litestream/testdata/wal/ok/replica/generations/0000000000000000/wal/00000000/00000001.wal.lz4 new file mode 100644 index 0000000..7536340 Binary files /dev/null and b/cmd/litestream/testdata/wal/ok/replica/generations/0000000000000000/wal/00000000/00000001.wal.lz4 differ diff --git a/cmd/litestream/testdata/wal/ok/replica/generations/0000000000000000/wal/00000001/00000000.wal.lz4 b/cmd/litestream/testdata/wal/ok/replica/generations/0000000000000000/wal/00000001/00000000.wal.lz4 new file mode 100644 index 0000000..7536340 Binary files /dev/null and b/cmd/litestream/testdata/wal/ok/replica/generations/0000000000000000/wal/00000001/00000000.wal.lz4 differ diff --git a/cmd/litestream/testdata/wal/ok/replica/generations/0000000000000001/wal/00000000/00000000.wal.lz4 b/cmd/litestream/testdata/wal/ok/replica/generations/0000000000000001/wal/00000000/00000000.wal.lz4 new file mode 100644 index 0000000..7536340 Binary files /dev/null and b/cmd/litestream/testdata/wal/ok/replica/generations/0000000000000001/wal/00000000/00000000.wal.lz4 differ diff --git a/cmd/litestream/testdata/wal/ok/stdout b/cmd/litestream/testdata/wal/ok/stdout new file mode 100644 index 0000000..90a58f5 --- /dev/null +++ b/cmd/litestream/testdata/wal/ok/stdout @@ -0,0 +1,5 @@ +replica generation index offset size created +file 0000000000000001 00000000 00000000 93 2000-01-04T00:00:00Z +file 0000000000000000 00000001 00000000 93 2000-01-03T00:00:00Z +file 0000000000000000 00000000 00000001 93 2000-01-02T00:00:00Z +file 0000000000000000 00000000 00000000 93 2000-01-01T00:00:00Z diff --git a/cmd/litestream/testdata/wal/replica-name/Makefile b/cmd/litestream/testdata/wal/replica-name/Makefile new file mode 100644 index 0000000..5556bc8 --- /dev/null +++ b/cmd/litestream/testdata/wal/replica-name/Makefile @@ -0,0 +1,6 @@ +.PHONY: default +default: + TZ=UTC touch -ct 200001010000 replica0/generations/0000000000000000/wal/00000000/00000000.wal.lz4 + TZ=UTC touch -ct 200001020000 replica1/generations/0000000000000000/wal/00000000/00000001.wal.lz4 + TZ=UTC touch -ct 200001030000 replica1/generations/0000000000000000/wal/00000001/00000000.wal.lz4 + TZ=UTC touch -ct 200001040000 replica1/generations/0000000000000001/wal/00000000/00000000.wal.lz4 diff --git a/cmd/litestream/testdata/wal/replica-name/litestream.yml b/cmd/litestream/testdata/wal/replica-name/litestream.yml new file mode 100644 index 0000000..8511213 --- /dev/null +++ b/cmd/litestream/testdata/wal/replica-name/litestream.yml @@ -0,0 +1,7 @@ +dbs: + - path: $LITESTREAM_TESTDIR/db + replicas: + - name: replica0 + path: $LITESTREAM_TESTDIR/replica0 + - name: replica1 + path: $LITESTREAM_TESTDIR/replica1 diff --git a/cmd/litestream/testdata/wal/replica-name/replica0/generations/0000000000000000/wal/00000000/00000000.wal.lz4 b/cmd/litestream/testdata/wal/replica-name/replica0/generations/0000000000000000/wal/00000000/00000000.wal.lz4 new file mode 100644 index 0000000..7536340 Binary files /dev/null and b/cmd/litestream/testdata/wal/replica-name/replica0/generations/0000000000000000/wal/00000000/00000000.wal.lz4 differ diff --git a/cmd/litestream/testdata/wal/replica-name/replica0/generations/0000000000000000/wal/00000000/00000001.wal.lz4 b/cmd/litestream/testdata/wal/replica-name/replica0/generations/0000000000000000/wal/00000000/00000001.wal.lz4 new file mode 100644 index 0000000..7536340 Binary files /dev/null and b/cmd/litestream/testdata/wal/replica-name/replica0/generations/0000000000000000/wal/00000000/00000001.wal.lz4 differ diff --git a/cmd/litestream/testdata/wal/replica-name/replica0/generations/0000000000000000/wal/00000001/00000000.wal.lz4 b/cmd/litestream/testdata/wal/replica-name/replica0/generations/0000000000000000/wal/00000001/00000000.wal.lz4 new file mode 100644 index 0000000..7536340 Binary files /dev/null and b/cmd/litestream/testdata/wal/replica-name/replica0/generations/0000000000000000/wal/00000001/00000000.wal.lz4 differ diff --git a/cmd/litestream/testdata/wal/replica-name/replica1/generations/0000000000000001/wal/00000000/00000000.wal.lz4 b/cmd/litestream/testdata/wal/replica-name/replica1/generations/0000000000000001/wal/00000000/00000000.wal.lz4 new file mode 100644 index 0000000..7536340 Binary files /dev/null and b/cmd/litestream/testdata/wal/replica-name/replica1/generations/0000000000000001/wal/00000000/00000000.wal.lz4 differ diff --git a/cmd/litestream/testdata/wal/replica-name/stdout b/cmd/litestream/testdata/wal/replica-name/stdout new file mode 100644 index 0000000..2e9f9d9 --- /dev/null +++ b/cmd/litestream/testdata/wal/replica-name/stdout @@ -0,0 +1,2 @@ +replica generation index offset size created +replica1 0000000000000001 00000000 00000000 93 2000-01-04T00:00:00Z diff --git a/cmd/litestream/testdata/wal/replica-not-found/litestream.yml b/cmd/litestream/testdata/wal/replica-not-found/litestream.yml new file mode 100644 index 0000000..5d911bd --- /dev/null +++ b/cmd/litestream/testdata/wal/replica-not-found/litestream.yml @@ -0,0 +1,4 @@ +dbs: + - path: $LITESTREAM_TESTDIR/db + replicas: + - url: s3://bkt/db diff --git a/cmd/litestream/testdata/wal/replica-url/Makefile b/cmd/litestream/testdata/wal/replica-url/Makefile new file mode 100644 index 0000000..2bb5a8e --- /dev/null +++ b/cmd/litestream/testdata/wal/replica-url/Makefile @@ -0,0 +1,7 @@ +.PHONY: default +default: + TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/wal/00000000/00000000.wal.lz4 + TZ=UTC touch -ct 200001020000 replica/generations/0000000000000000/wal/00000000/00000001.wal.lz4 + TZ=UTC touch -ct 200001030000 replica/generations/0000000000000000/wal/00000001/00000000.wal.lz4 + TZ=UTC touch -ct 200001040000 replica/generations/0000000000000001/wal/00000000/00000000.wal.lz4 + diff --git a/cmd/litestream/testdata/wal/replica-url/replica/generations/0000000000000000/wal/00000000/00000000.wal.lz4 b/cmd/litestream/testdata/wal/replica-url/replica/generations/0000000000000000/wal/00000000/00000000.wal.lz4 new file mode 100644 index 0000000..7536340 Binary files /dev/null and b/cmd/litestream/testdata/wal/replica-url/replica/generations/0000000000000000/wal/00000000/00000000.wal.lz4 differ diff --git a/cmd/litestream/testdata/wal/replica-url/replica/generations/0000000000000000/wal/00000000/00000001.wal.lz4 b/cmd/litestream/testdata/wal/replica-url/replica/generations/0000000000000000/wal/00000000/00000001.wal.lz4 new file mode 100644 index 0000000..7536340 Binary files /dev/null and b/cmd/litestream/testdata/wal/replica-url/replica/generations/0000000000000000/wal/00000000/00000001.wal.lz4 differ diff --git a/cmd/litestream/testdata/wal/replica-url/replica/generations/0000000000000000/wal/00000001/00000000.wal.lz4 b/cmd/litestream/testdata/wal/replica-url/replica/generations/0000000000000000/wal/00000001/00000000.wal.lz4 new file mode 100644 index 0000000..7536340 Binary files /dev/null and b/cmd/litestream/testdata/wal/replica-url/replica/generations/0000000000000000/wal/00000001/00000000.wal.lz4 differ diff --git a/cmd/litestream/testdata/wal/replica-url/replica/generations/0000000000000001/wal/00000000/00000000.wal.lz4 b/cmd/litestream/testdata/wal/replica-url/replica/generations/0000000000000001/wal/00000000/00000000.wal.lz4 new file mode 100644 index 0000000..7536340 Binary files /dev/null and b/cmd/litestream/testdata/wal/replica-url/replica/generations/0000000000000001/wal/00000000/00000000.wal.lz4 differ diff --git a/cmd/litestream/testdata/wal/replica-url/stdout b/cmd/litestream/testdata/wal/replica-url/stdout new file mode 100644 index 0000000..90a58f5 --- /dev/null +++ b/cmd/litestream/testdata/wal/replica-url/stdout @@ -0,0 +1,5 @@ +replica generation index offset size created +file 0000000000000001 00000000 00000000 93 2000-01-04T00:00:00Z +file 0000000000000000 00000001 00000000 93 2000-01-03T00:00:00Z +file 0000000000000000 00000000 00000001 93 2000-01-02T00:00:00Z +file 0000000000000000 00000000 00000000 93 2000-01-01T00:00:00Z diff --git a/cmd/litestream/version.go b/cmd/litestream/version.go index 4669861..ccfae6d 100644 --- a/cmd/litestream/version.go +++ b/cmd/litestream/version.go @@ -4,10 +4,24 @@ import ( "context" "flag" "fmt" + "io" ) // VersionCommand represents a command to print the current version. -type VersionCommand struct{} +type VersionCommand struct { + stdin io.Reader + stdout io.Writer + stderr io.Writer +} + +// NewVersionCommand returns a new instance of VersionCommand. +func NewVersionCommand(stdin io.Reader, stdout, stderr io.Writer) *VersionCommand { + return &VersionCommand{ + stdin: stdin, + stdout: stdout, + stderr: stderr, + } +} // Run executes the command. func (c *VersionCommand) Run(ctx context.Context, args []string) (err error) { @@ -17,14 +31,14 @@ func (c *VersionCommand) Run(ctx context.Context, args []string) (err error) { return err } - fmt.Println(Version) + fmt.Fprintln(c.stdout, Version) return nil } // Usage prints the help screen to STDOUT. func (c *VersionCommand) Usage() { - fmt.Println(` + fmt.Fprintln(c.stdout, ` Prints the version. Usage: diff --git a/cmd/litestream/wal.go b/cmd/litestream/wal.go index d3cc681..fa28107 100644 --- a/cmd/litestream/wal.go +++ b/cmd/litestream/wal.go @@ -4,8 +4,9 @@ import ( "context" "flag" "fmt" + "io" "log" - "os" + "sort" "text/tabwriter" "time" @@ -14,82 +15,63 @@ import ( // WALCommand represents a command to list WAL files for a database. type WALCommand struct { + stdin io.Reader + stdout io.Writer + stderr io.Writer + configPath string noExpandEnv bool + + replicaName string + generation string +} + +// NewWALCommand returns a new instance of WALCommand. +func NewWALCommand(stdin io.Reader, stdout, stderr io.Writer) *WALCommand { + return &WALCommand{ + stdin: stdin, + stdout: stdout, + stderr: stderr, + } } // Run executes the command. -func (c *WALCommand) Run(ctx context.Context, args []string) (err error) { +func (c *WALCommand) Run(ctx context.Context, args []string) (ret error) { fs := flag.NewFlagSet("litestream-wal", flag.ContinueOnError) registerConfigFlag(fs, &c.configPath, &c.noExpandEnv) - replicaName := fs.String("replica", "", "replica name") - generation := fs.String("generation", "", "generation name") + fs.StringVar(&c.replicaName, "replica", "", "replica name") + fs.StringVar(&c.generation, "generation", "", "generation name") fs.Usage = c.Usage if err := fs.Parse(args); err != nil { return err } else if fs.NArg() == 0 || fs.Arg(0) == "" { - return fmt.Errorf("database path required") + return fmt.Errorf("database path or replica URL required") } else if fs.NArg() > 1 { return fmt.Errorf("too many arguments") } - var db *litestream.DB - var r *litestream.Replica - if isURL(fs.Arg(0)) { - if c.configPath != "" { - return fmt.Errorf("cannot specify a replica URL and the -config flag") - } - if r, err = NewReplicaFromConfig(&ReplicaConfig{URL: fs.Arg(0)}, nil); err != nil { - return err - } - } else { - if c.configPath == "" { - c.configPath = DefaultConfigPath() - } - - // Load configuration. - config, err := ReadConfigFile(c.configPath, !c.noExpandEnv) - if err != nil { - return err - } - - // Lookup database from configuration file by path. - if path, err := expand(fs.Arg(0)); err != nil { - return err - } else if dbc := config.DBConfig(path); dbc == nil { - return fmt.Errorf("database not found in config: %s", path) - } else if db, err = NewDBFromConfig(dbc); err != nil { - return err - } - - // Filter by replica, if specified. - if *replicaName != "" { - if r = db.Replica(*replicaName); r == nil { - return fmt.Errorf("replica %q not found for database %q", *replicaName, db.Path()) - } - } + // Load configuration. + config, err := ReadConfigFile(c.configPath, !c.noExpandEnv) + if err != nil { + return err } - // Find WAL files by db or replica. - var replicas []*litestream.Replica - if r != nil { - replicas = []*litestream.Replica{r} - } else { - replicas = db.Replicas + // Build list of replicas from CLI flags. + replicas, _, err := loadReplicas(ctx, config, fs.Arg(0), c.replicaName) + if err != nil { + return err } - // List all WAL files. - w := tabwriter.NewWriter(os.Stdout, 0, 8, 2, ' ', 0) - defer w.Flush() - - fmt.Fprintln(w, "replica\tgeneration\tindex\toffset\tsize\tcreated") + // Build list of WAL metadata with associated replica. + var infos []replicaWALSegmentInfo for _, r := range replicas { var generations []string - if *generation != "" { - generations = []string{*generation} + if c.generation != "" { + generations = []string{c.generation} } else { if generations, err = r.Client.Generations(ctx); err != nil { log.Printf("%s: cannot determine generations: %s", r.Name(), err) + ret = errExit // signal error return without printing message continue } } @@ -103,31 +85,45 @@ func (c *WALCommand) Run(ctx context.Context, args []string) (err error) { defer itr.Close() for itr.Next() { - info := itr.WALSegment() - - fmt.Fprintf(w, "%s\t%s\t%d\t%d\t%d\t%s\n", - r.Name(), - info.Generation, - info.Index, - info.Offset, - info.Size, - info.CreatedAt.Format(time.RFC3339), - ) + infos = append(infos, replicaWALSegmentInfo{ + WALSegmentInfo: itr.WALSegment(), + replicaName: r.Name(), + }) } return itr.Close() }(); err != nil { log.Printf("%s: cannot fetch wal segments: %s", r.Name(), err) + ret = errExit // signal error return without printing message continue } } } - return nil + // Sort WAL segments by creation time from newest to oldest. + sort.Slice(infos, func(i, j int) bool { return infos[i].CreatedAt.After(infos[j].CreatedAt) }) + + // List all WAL files. + w := tabwriter.NewWriter(c.stdout, 0, 8, 2, ' ', 0) + defer w.Flush() + + fmt.Fprintln(w, "replica\tgeneration\tindex\toffset\tsize\tcreated") + for _, info := range infos { + fmt.Fprintf(w, "%s\t%s\t%08x\t%08x\t%d\t%s\n", + info.replicaName, + info.Generation, + info.Index, + info.Offset, + info.Size, + info.CreatedAt.Format(time.RFC3339), + ) + } + + return ret } // Usage prints the help screen to STDOUT. func (c *WALCommand) Usage() { - fmt.Printf(` + fmt.Fprintf(c.stdout, ` The wal command lists all wal segments available for a database. Usage: @@ -166,3 +162,9 @@ Examples: DefaultConfigPath(), ) } + +// replicaWALSegmentInfo represents WAL segment metadata with associated replica name. +type replicaWALSegmentInfo struct { + litestream.WALSegmentInfo + replicaName string +} diff --git a/cmd/litestream/wal_test.go b/cmd/litestream/wal_test.go new file mode 100644 index 0000000..f313e01 --- /dev/null +++ b/cmd/litestream/wal_test.go @@ -0,0 +1,128 @@ +package main_test + +import ( + "context" + "flag" + "path/filepath" + "strings" + "testing" + + "github.com/benbjohnson/litestream/internal/testingutil" +) + +func TestWALCommand(t *testing.T) { + t.Run("OK", func(t *testing.T) { + testDir := filepath.Join("testdata", "wal", "ok") + defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)() + + m, _, stdout, _ := newMain() + if err := m.Run(context.Background(), []string{"wal", "-config", filepath.Join(testDir, "litestream.yml"), filepath.Join(testDir, "db")}); err != nil { + t.Fatal(err) + } else if got, want := stdout.String(), string(testingutil.MustReadFile(t, filepath.Join(testDir, "stdout"))); got != want { + t.Fatalf("stdout=%q, want %q", got, want) + } + }) + + t.Run("ReplicaName", func(t *testing.T) { + testDir := filepath.Join("testdata", "wal", "replica-name") + defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)() + + m, _, stdout, _ := newMain() + if err := m.Run(context.Background(), []string{"wal", "-config", filepath.Join(testDir, "litestream.yml"), "-replica", "replica1", filepath.Join(testDir, "db")}); err != nil { + t.Fatal(err) + } else if got, want := stdout.String(), string(testingutil.MustReadFile(t, filepath.Join(testDir, "stdout"))); got != want { + t.Fatalf("stdout=%q, want %q", got, want) + } + }) + + t.Run("ReplicaURL", func(t *testing.T) { + testDir := filepath.Join(testingutil.Getwd(t), "testdata", "wal", "replica-url") + defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)() + replicaURL := "file://" + filepath.ToSlash(testDir) + "/replica" + + m, _, stdout, _ := newMain() + if err := m.Run(context.Background(), []string{"wal", replicaURL}); err != nil { + t.Fatal(err) + } else if got, want := stdout.String(), string(testingutil.MustReadFile(t, filepath.Join(testDir, "stdout"))); got != want { + t.Fatalf("stdout=%q, want %q", got, want) + } + }) + + t.Run("ErrDatabaseOrReplicaRequired", func(t *testing.T) { + m, _, _, _ := newMain() + err := m.Run(context.Background(), []string{"wal"}) + if err == nil || err.Error() != `database path or replica URL required` { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrTooManyArguments", func(t *testing.T) { + m, _, _, _ := newMain() + err := m.Run(context.Background(), []string{"wal", "abc", "123"}) + if err == nil || err.Error() != `too many arguments` { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrInvalidFlags", func(t *testing.T) { + m, _, _, _ := newMain() + err := m.Run(context.Background(), []string{"wal", "-no-such-flag"}) + if err == nil || err.Error() != `flag provided but not defined: -no-such-flag` { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrConfigFileNotFound", func(t *testing.T) { + m, _, _, _ := newMain() + err := m.Run(context.Background(), []string{"wal", "-config", "/no/such/file", "/var/lib/db"}) + if err == nil || err.Error() != `config file not found: /no/such/file` { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrInvalidConfig", func(t *testing.T) { + testDir := filepath.Join("testdata", "wal", "invalid-config") + m, _, _, _ := newMain() + err := m.Run(context.Background(), []string{"wal", "-config", filepath.Join(testDir, "litestream.yml"), "/var/lib/db"}) + if err == nil || !strings.Contains(err.Error(), `replica path cannot be a url`) { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrDatabaseNotFound", func(t *testing.T) { + testDir := filepath.Join("testdata", "wal", "database-not-found") + defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)() + + m, _, _, _ := newMain() + err := m.Run(context.Background(), []string{"wal", "-config", filepath.Join(testDir, "litestream.yml"), "/no/such/db"}) + if err == nil || err.Error() != `database not found in config: /no/such/db` { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrReplicaNotFound", func(t *testing.T) { + testDir := filepath.Join(testingutil.Getwd(t), "testdata", "wal", "replica-not-found") + defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)() + + m, _, _, _ := newMain() + err := m.Run(context.Background(), []string{"wal", "-config", filepath.Join(testDir, "litestream.yml"), "-replica", "no_such_replica", filepath.Join(testDir, "db")}) + if err == nil || err.Error() != `replica "no_such_replica" not found for database "`+filepath.Join(testDir, "db")+`"` { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrInvalidReplicaURL", func(t *testing.T) { + m, _, _, _ := newMain() + err := m.Run(context.Background(), []string{"wal", "xyz://xyz"}) + if err == nil || !strings.Contains(err.Error(), `unknown replica type in config: "xyz"`) { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("Usage", func(t *testing.T) { + m, _, _, _ := newMain() + if err := m.Run(context.Background(), []string{"wal", "-h"}); err != flag.ErrHelp { + t.Fatalf("unexpected error: %s", err) + } + }) +} diff --git a/internal/internal.go b/internal/internal.go index f8e5c60..36598d4 100644 --- a/internal/internal.go +++ b/internal/internal.go @@ -7,6 +7,7 @@ import ( "regexp" "strconv" "syscall" + "time" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" @@ -190,3 +191,30 @@ var ( Help: "The number of bytes used by replica operations", }, []string{"replica_type", "operation"}) ) + +// TruncateDuration truncates d to the nearest major unit (s, ms, µs, ns). +func TruncateDuration(d time.Duration) time.Duration { + if d < 0 { + if d < -10*time.Second { + return d.Truncate(time.Second) + } else if d < -time.Second { + return d.Truncate(time.Second / 10) + } else if d < -time.Millisecond { + return d.Truncate(time.Millisecond) + } else if d < -time.Microsecond { + return d.Truncate(time.Microsecond) + } + return d + } + + if d > 10*time.Second { + return d.Truncate(time.Second) + } else if d > time.Second { + return d.Truncate(time.Second / 10) + } else if d > time.Millisecond { + return d.Truncate(time.Millisecond) + } else if d > time.Microsecond { + return d.Truncate(time.Microsecond) + } + return d +} diff --git a/internal/internal_test.go b/internal/internal_test.go index a8eda5d..5b661de 100644 --- a/internal/internal_test.go +++ b/internal/internal_test.go @@ -4,6 +4,7 @@ import ( "fmt" "reflect" "testing" + "time" "github.com/benbjohnson/litestream/internal" ) @@ -59,3 +60,41 @@ func TestParseWALSegmentPath(t *testing.T) { }) } } + +func TestTruncateDuration(t *testing.T) { + for _, tt := range []struct { + input, output time.Duration + }{ + {0, 0 * time.Nanosecond}, + + {1, 1 * time.Nanosecond}, + {12, 12 * time.Nanosecond}, + {123, 123 * time.Nanosecond}, + {1234, 1 * time.Microsecond}, + {12345, 12 * time.Microsecond}, + {123456, 123 * time.Microsecond}, + {1234567, 1 * time.Millisecond}, + {12345678, 12 * time.Millisecond}, + {123456789, 123 * time.Millisecond}, + {1234567890, 1200 * time.Millisecond}, + {12345678900, 12 * time.Second}, + + {-1, -1 * time.Nanosecond}, + {-12, -12 * time.Nanosecond}, + {-123, -123 * time.Nanosecond}, + {-1234, -1 * time.Microsecond}, + {-12345, -12 * time.Microsecond}, + {-123456, -123 * time.Microsecond}, + {-1234567, -1 * time.Millisecond}, + {-12345678, -12 * time.Millisecond}, + {-123456789, -123 * time.Millisecond}, + {-1234567890, -1200 * time.Millisecond}, + {-12345678900, -12 * time.Second}, + } { + t.Run(fmt.Sprint(int(tt.input)), func(t *testing.T) { + if got, want := internal.TruncateDuration(tt.input), tt.output; got != want { + t.Fatalf("duration=%s, want %s", got, want) + } + }) + } +} diff --git a/internal/testingutil/testingutil.go b/internal/testingutil/testingutil.go new file mode 100644 index 0000000..99bc47f --- /dev/null +++ b/internal/testingutil/testingutil.go @@ -0,0 +1,43 @@ +package testingutil + +import ( + "os" + "testing" +) + +// MustReadFile reads all data from filename. Fail on error. +func MustReadFile(tb testing.TB, filename string) []byte { + tb.Helper() + b, err := os.ReadFile(filename) + if err != nil { + tb.Fatal(err) + } + return b +} + +// Getpwd returns the working directory. Fail on error. +func Getwd(tb testing.TB) string { + tb.Helper() + + dir, err := os.Getwd() + if err != nil { + tb.Fatal(err) + } + return dir +} + +// Setenv sets the environment variable key to value. The returned function reverts it. +func Setenv(tb testing.TB, key, value string) func() { + tb.Helper() + + prevValue := os.Getenv(key) + if err := os.Setenv(key, value); err != nil { + tb.Fatal(err) + } + + return func() { + if err := os.Setenv(key, prevValue); err != nil { + tb.Fatal(tb) + } + } +} diff --git a/testdata/find-latest-generation/ok/Makefile b/testdata/find-latest-generation/ok/Makefile index 847b844..c71ce14 100644 --- a/testdata/find-latest-generation/ok/Makefile +++ b/testdata/find-latest-generation/ok/Makefile @@ -1,7 +1,7 @@ .PHONY: default default: - TZ=UTC touch -t 200001010000 generations/0000000000000000/snapshots/00000000.snapshot.lz4 - TZ=UTC touch -t 200001020000 generations/0000000000000000/snapshots/00000001.snapshot.lz4 - TZ=UTC touch -t 200001010000 generations/0000000000000001/snapshots/00000000.snapshot.lz4 - TZ=UTC touch -t 200001030000 generations/0000000000000001/snapshots/00000001.snapshot.lz4 - TZ=UTC touch -t 200001010000 generations/0000000000000002/snapshots/00000000.snapshot.lz4 + TZ=UTC touch -ct 200001010000 generations/0000000000000000/snapshots/00000000.snapshot.lz4 + TZ=UTC touch -ct 200001020000 generations/0000000000000000/snapshots/00000001.snapshot.lz4 + TZ=UTC touch -ct 200001010000 generations/0000000000000001/snapshots/00000000.snapshot.lz4 + TZ=UTC touch -ct 200001030000 generations/0000000000000001/snapshots/00000001.snapshot.lz4 + TZ=UTC touch -ct 200001010000 generations/0000000000000002/snapshots/00000000.snapshot.lz4 diff --git a/testdata/generation-time-bounds/ok/Makefile b/testdata/generation-time-bounds/ok/Makefile index 06d5044..e29f9e4 100644 --- a/testdata/generation-time-bounds/ok/Makefile +++ b/testdata/generation-time-bounds/ok/Makefile @@ -1,8 +1,8 @@ .PHONY: default default: - TZ=UTC touch -t 200001010000 generations/0000000000000000/snapshots/00000000.snapshot.lz4 - TZ=UTC touch -t 200001020000 generations/0000000000000000/snapshots/00000001.snapshot.lz4 - TZ=UTC touch -t 200001010000 generations/0000000000000000/wal/00000000/00000000.wal.lz4 - TZ=UTC touch -t 200001020000 generations/0000000000000000/wal/00000000/00000001.wal.lz4 - TZ=UTC touch -t 200001030000 generations/0000000000000000/wal/00000001/00000000.wal.lz4 + TZ=UTC touch -ct 200001010000 generations/0000000000000000/snapshots/00000000.snapshot.lz4 + TZ=UTC touch -ct 200001020000 generations/0000000000000000/snapshots/00000001.snapshot.lz4 + TZ=UTC touch -ct 200001010000 generations/0000000000000000/wal/00000000/00000000.wal.lz4 + TZ=UTC touch -ct 200001020000 generations/0000000000000000/wal/00000000/00000001.wal.lz4 + TZ=UTC touch -ct 200001030000 generations/0000000000000000/wal/00000001/00000000.wal.lz4 diff --git a/testdata/generation-time-bounds/snapshots-only/Makefile b/testdata/generation-time-bounds/snapshots-only/Makefile index 18b382a..6405068 100644 --- a/testdata/generation-time-bounds/snapshots-only/Makefile +++ b/testdata/generation-time-bounds/snapshots-only/Makefile @@ -1,5 +1,5 @@ .PHONY: default default: - TZ=UTC touch -t 200001010000 generations/0000000000000000/snapshots/00000000.snapshot.lz4 - TZ=UTC touch -t 200001020000 generations/0000000000000000/snapshots/00000001.snapshot.lz4 + TZ=UTC touch -ct 200001010000 generations/0000000000000000/snapshots/00000000.snapshot.lz4 + TZ=UTC touch -ct 200001020000 generations/0000000000000000/snapshots/00000001.snapshot.lz4 diff --git a/testdata/max-snapshot-index/ok/Makefile b/testdata/max-snapshot-index/ok/Makefile index 3d808b7..d7b4d6c 100644 --- a/testdata/max-snapshot-index/ok/Makefile +++ b/testdata/max-snapshot-index/ok/Makefile @@ -1,6 +1,6 @@ .PHONY: default default: - TZ=UTC touch -t 200001020000 generations/0000000000000000/snapshots/00000000.snapshot.lz4 - TZ=UTC touch -t 200001010000 generations/0000000000000001/snapshots/00000000.snapshot.lz4 - TZ=UTC touch -t 200001030000 generations/0000000000000001/snapshots/00000001.snapshot.lz4 - TZ=UTC touch -t 200001010000 generations/0000000000000002/snapshots/00000000.snapshot.lz4 + TZ=UTC touch -ct 200001020000 generations/0000000000000000/snapshots/00000000.snapshot.lz4 + TZ=UTC touch -ct 200001010000 generations/0000000000000001/snapshots/00000000.snapshot.lz4 + TZ=UTC touch -ct 200001030000 generations/0000000000000001/snapshots/00000001.snapshot.lz4 + TZ=UTC touch -ct 200001010000 generations/0000000000000002/snapshots/00000000.snapshot.lz4 diff --git a/testdata/replica-client-time-bounds/ok/Makefile b/testdata/replica-client-time-bounds/ok/Makefile index 3d808b7..d7b4d6c 100644 --- a/testdata/replica-client-time-bounds/ok/Makefile +++ b/testdata/replica-client-time-bounds/ok/Makefile @@ -1,6 +1,6 @@ .PHONY: default default: - TZ=UTC touch -t 200001020000 generations/0000000000000000/snapshots/00000000.snapshot.lz4 - TZ=UTC touch -t 200001010000 generations/0000000000000001/snapshots/00000000.snapshot.lz4 - TZ=UTC touch -t 200001030000 generations/0000000000000001/snapshots/00000001.snapshot.lz4 - TZ=UTC touch -t 200001010000 generations/0000000000000002/snapshots/00000000.snapshot.lz4 + TZ=UTC touch -ct 200001020000 generations/0000000000000000/snapshots/00000000.snapshot.lz4 + TZ=UTC touch -ct 200001010000 generations/0000000000000001/snapshots/00000000.snapshot.lz4 + TZ=UTC touch -ct 200001030000 generations/0000000000000001/snapshots/00000001.snapshot.lz4 + TZ=UTC touch -ct 200001010000 generations/0000000000000002/snapshots/00000000.snapshot.lz4 diff --git a/testdata/snapshot-time-bounds/ok/Makefile b/testdata/snapshot-time-bounds/ok/Makefile index 0a3ea13..6c7e69a 100644 --- a/testdata/snapshot-time-bounds/ok/Makefile +++ b/testdata/snapshot-time-bounds/ok/Makefile @@ -1,6 +1,6 @@ .PHONY: default default: - TZ=UTC touch -t 200001010000 generations/0000000000000000/snapshots/00000000.snapshot.lz4 - TZ=UTC touch -t 200001020000 generations/0000000000000000/snapshots/00000001.snapshot.lz4 - TZ=UTC touch -t 200001030000 generations/0000000000000000/snapshots/00000002.snapshot.lz4 + TZ=UTC touch -ct 200001010000 generations/0000000000000000/snapshots/00000000.snapshot.lz4 + TZ=UTC touch -ct 200001020000 generations/0000000000000000/snapshots/00000001.snapshot.lz4 + TZ=UTC touch -ct 200001030000 generations/0000000000000000/snapshots/00000002.snapshot.lz4 diff --git a/testdata/wal-time-bounds/ok/Makefile b/testdata/wal-time-bounds/ok/Makefile index 875381c..fa7ab33 100644 --- a/testdata/wal-time-bounds/ok/Makefile +++ b/testdata/wal-time-bounds/ok/Makefile @@ -1,6 +1,6 @@ .PHONY: default default: - TZ=UTC touch -t 200001010000 generations/0000000000000000/wal/00000000/00000000.wal.lz4 - TZ=UTC touch -t 200001020000 generations/0000000000000000/wal/00000000/00000001.wal.lz4 - TZ=UTC touch -t 200001030000 generations/0000000000000000/wal/00000001/00000000.wal.lz4 + TZ=UTC touch -ct 200001010000 generations/0000000000000000/wal/00000000/00000000.wal.lz4 + TZ=UTC touch -ct 200001020000 generations/0000000000000000/wal/00000000/00000001.wal.lz4 + TZ=UTC touch -ct 200001030000 generations/0000000000000000/wal/00000001/00000000.wal.lz4