Add db metrics

This was originally meant to add a TRUNCATE checkpoint before starting
a new generation, however, there is a write lock that blocks the
checkpoint and it's more complicated to roll it back and attempt the
truncation.

cmd/litestream/databases.go

@@ -0,0 +1,76 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"os"
+	"strings"
+	"text/tabwriter"
+)
+
+type DatabasesCommand struct{}
+
+func (c *DatabasesCommand) Run(ctx context.Context, args []string) (err error) {
+	var configPath string
+	fs := flag.NewFlagSet("litestream-databases", flag.ContinueOnError)
+	registerConfigFlag(fs, &configPath)
+	fs.Usage = c.Usage
+	if err := fs.Parse(args); err != nil {
+		return err
+	} else if fs.NArg() != 0 {
+		return fmt.Errorf("too many argument")
+	}
+
+	// Load configuration.
+	if configPath == "" {
+		return errors.New("-config required")
+	}
+	config, err := ReadConfigFile(configPath)
+	if err != nil {
+		return err
+	}
+
+	// List all databases.
+	w := tabwriter.NewWriter(os.Stdout, 0, 8, 1, '\t', 0)
+	fmt.Fprintln(w, "path\treplicas")
+	for _, dbConfig := range config.DBs {
+		db, err := newDBFromConfig(dbConfig)
+		if err != nil {
+			return err
+		}
+
+		var replicaNames []string
+		for _, r := range db.Replicas {
+			replicaNames = append(replicaNames, r.Name())
+		}
+
+		fmt.Fprintf(w, "%s\t%s\n",
+			db.Path(),
+			strings.Join(replicaNames, ","),
+		)
+	}
+	w.Flush()
+
+	return nil
+}
+
+func (c *DatabasesCommand) Usage() {
+	fmt.Printf(`
+The databases command lists all databases in the configuration file.
+
+Usage:
+
+	litestream databases [arguments]
+
+Arguments:
+
+	-config PATH
+	    Specifies the configuration file.
+	    Defaults to %s
+
+`[1:],
+		DefaultConfigPath,
+	)
+}

cmd/litestream/generations.go

@@ -12,53 +12,42 @@ import (
 	"time"
 )
 
-type GenerationsCommand struct {
+type GenerationsCommand struct{}
-	ConfigPath string
-	Config     Config
-
-	DBPath string
-}
-
-func NewGenerationsCommand() *GenerationsCommand {
-	return &GenerationsCommand{}
-}
 
 func (c *GenerationsCommand) Run(ctx context.Context, args []string) (err error) {
+	var configPath string
 	fs := flag.NewFlagSet("litestream-generations", flag.ContinueOnError)
-	registerConfigFlag(fs, &c.ConfigPath)
+	registerConfigFlag(fs, &configPath)
+	replicaName := fs.String("replica", "", "replica name")
 	fs.Usage = c.Usage
 	if err := fs.Parse(args); err != nil {
 		return err
+	} else if fs.NArg() == 0 || fs.Arg(0) == "" {
+		return fmt.Errorf("database path required")
 	} else if fs.NArg() > 1 {
 		return fmt.Errorf("too many arguments")
 	}
 
 	// Load configuration.
-	if c.ConfigPath == "" {
+	if configPath == "" {
 		return errors.New("-config required")
 	}
-	config, err := ReadConfigFile(c.ConfigPath)
+	config, err := ReadConfigFile(configPath)
 	if err != nil {
 		return err
 	}
 
-	// Determine absolute path for database, if specified.
+	// Determine absolute path for database.
-	if c.DBPath = fs.Arg(0); c.DBPath != "" {
+	dbPath, err := filepath.Abs(fs.Arg(0))
-		if c.DBPath, err = filepath.Abs(c.DBPath); err != nil {
+	if err != nil {
 		return err
 	}
-	}
-
-	// List each generation.
-	w := tabwriter.NewWriter(os.Stdout, 0, 8, 1, '\t', 0)
-	fmt.Fprintln(w, "db\tname\tgeneration\tlag\tstart\tend")
-	for _, dbConfig := range config.DBs {
-		// Filter database, if specified in the arguments.
-		if c.DBPath != "" && dbConfig.Path != c.DBPath {
-			continue
-		}
 
 	// Instantiate DB from from configuration.
+	dbConfig := config.DBConfig(dbPath)
+	if dbConfig == nil {
+		return fmt.Errorf("database not found in config: %s", dbPath)
+	}
 	db, err := newDBFromConfig(dbConfig)
 	if err != nil {
 		return err
@@ -70,8 +59,14 @@ func (c *GenerationsCommand) Run(ctx context.Context, args []string) (err error)
 		return err
 	}
 
-		// Iterate over each replica in the database.
+	// List each generation.
+	w := tabwriter.NewWriter(os.Stdout, 0, 8, 1, '\t', 0)
+	fmt.Fprintln(w, "name\tgeneration\tlag\tstart\tend")
 	for _, r := range db.Replicas {
+		if *replicaName != "" && r.Name() != *replicaName {
+			continue
+		}
+
 		generations, err := r.Generations(ctx)
 		if err != nil {
 			log.Printf("%s: cannot list generations: %s", r.Name(), err)
@@ -86,8 +81,7 @@ func (c *GenerationsCommand) Run(ctx context.Context, args []string) (err error)
 				continue
 			}
 
-				fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\n",
+			fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n",
-					db.Path(),
 				r.Name(),
 				generation,
 				truncateDuration(stats.UpdatedAt.Sub(updatedAt)).String(),
@@ -96,7 +90,6 @@ func (c *GenerationsCommand) Run(ctx context.Context, args []string) (err error)
 			)
 		}
 	}
-	}
 	w.Flush()
 
 	return nil
@@ -104,7 +97,7 @@ func (c *GenerationsCommand) Run(ctx context.Context, args []string) (err error)
 
 func (c *GenerationsCommand) Usage() {
 	fmt.Printf(`
-The generations command lists all generations across all replicas along with
+The generations command lists all generations for a database. It also lists
 stats about their lag behind the primary database and the time range they cover.
 
 Usage:
@@ -116,6 +109,9 @@ Arguments:
 	-config PATH
 	    Specifies the configuration file. Defaults to %s
 
+	-replica NAME
+	    Optional, filters by replica.
+
 `[1:],
 		DefaultConfigPath,
 	)

cmd/litestream/main.go

@@ -48,14 +48,22 @@ func (m *Main) Run(ctx context.Context, args []string) (err error) {
 	}
 
 	switch cmd {
+	case "databases":
+		return (&DatabasesCommand{}).Run(ctx, args)
 	case "generations":
 		return (&GenerationsCommand{}).Run(ctx, args)
 	case "replicate":
 		return (&ReplicateCommand{}).Run(ctx, args)
 	case "restore":
 		return (&RestoreCommand{}).Run(ctx, args)
+	case "snapshots":
+		return (&SnapshotsCommand{}).Run(ctx, args)
+	case "validate":
+		return (&ValidateCommand{}).Run(ctx, args)
 	case "version":
 		return (&VersionCommand{}).Run(ctx, args)
+	case "wal":
+		return (&WALCommand{}).Run(ctx, args)
 	default:
 		if cmd == "" || cmd == "help" || strings.HasPrefix(cmd, "-") {
 			m.Usage()
@@ -75,21 +83,35 @@ Usage:
 
 The commands are:
 
-	generations  list available generations across all dbs & replicas
+	generations  list available generations for a database
 	replicate    runs a server to replicate databases
 	restore      recovers database backup from a replica
+	snapshots    list available snapshots for a database
+	validate     checks replica to ensure a consistent state with primary
 	version      prints the version
+	wal          list available WAL files for a database
 `[1:])
 }
 
+// Default configuration settings.
+const (
+	DefaultAddr = ":9090"
+)
+
 // Config represents a configuration file for the litestream daemon.
 type Config struct {
+	// Bind address for serving metrics.
+	Addr string `yaml:"addr"`
+
+	// List of databases to manage.
 	DBs []*DBConfig `yaml:"databases"`
 }
 
 // DefaultConfig returns a new instance of Config with defaults set.
 func DefaultConfig() Config {
-	return Config{}
+	return Config{
+		Addr: DefaultAddr,
+	}
 }
 
 func (c *Config) DBConfig(path string) *DBConfig {

cmd/litestream/replicate.go

@@ -5,10 +5,13 @@ import (
 	"errors"
 	"flag"
 	"fmt"
+	"net"
+	"net/http"
 	"os"
 	"os/signal"
 
 	"github.com/benbjohnson/litestream"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
 type ReplicateCommand struct {
@@ -19,10 +22,6 @@ type ReplicateCommand struct {
 	DBs []*litestream.DB
 }
 
-func NewReplicateCommand() *ReplicateCommand {
-	return &ReplicateCommand{}
-}
-
 // Run loads all databases specified in the configuration.
 func (c *ReplicateCommand) Run(ctx context.Context, args []string) (err error) {
 	fs := flag.NewFlagSet("litestream-replicate", flag.ContinueOnError)
@@ -42,7 +41,7 @@ func (c *ReplicateCommand) Run(ctx context.Context, args []string) (err error) {
 	}
 
 	// Setup signal handler.
-	ctx, cancel := context.WithCancel(context.Background())
+	ctx, cancel := context.WithCancel(ctx)
 	ch := make(chan os.Signal, 1)
 	signal.Notify(ch, os.Interrupt)
 	go func() { <-ch; cancel() }()
@@ -70,6 +69,16 @@ func (c *ReplicateCommand) Run(ctx context.Context, args []string) (err error) {
 	// Notify user that initialization is done.
 	fmt.Printf("Initialized with %d databases.\n", len(c.DBs))
 
+	// Serve metrics over HTTP if enabled.
+	if config.Addr != "" {
+		_, port, _ := net.SplitHostPort(config.Addr)
+		fmt.Printf("Serving metrics on http://localhost:%s/metrics\n", port)
+		go func() {
+			http.Handle("/metrics", promhttp.Handler())
+			http.ListenAndServe(config.Addr, nil)
+		}()
+	}
+
 	// Wait for signal to stop program.
 	<-ctx.Done()
 	signal.Reset()

cmd/litestream/restore.go

@@ -14,21 +14,17 @@ import (
 )
 
 type RestoreCommand struct {
-	DBPath string
-}
-
-func NewRestoreCommand() *RestoreCommand {
-	return &RestoreCommand{}
 }
 
 func (c *RestoreCommand) Run(ctx context.Context, args []string) (err error) {
 	var configPath string
-	var opt litestream.RestoreOptions
+	opt := litestream.NewRestoreOptions()
 	fs := flag.NewFlagSet("litestream-restore", flag.ContinueOnError)
 	registerConfigFlag(fs, &configPath)
 	fs.StringVar(&opt.OutputPath, "o", "", "output path")
 	fs.StringVar(&opt.ReplicaName, "replica", "", "replica name")
 	fs.StringVar(&opt.Generation, "generation", "", "generation name")
+	fs.IntVar(&opt.Index, "index", opt.Index, "wal index")
 	fs.BoolVar(&opt.DryRun, "dry-run", false, "dry run")
 	timestampStr := fs.String("timestamp", "", "timestamp")
 	verbose := fs.Bool("v", false, "verbose output")
@@ -67,15 +63,16 @@ func (c *RestoreCommand) Run(ctx context.Context, args []string) (err error) {
 		opt.Logger = log.New(os.Stderr, "", log.LstdFlags)
 	}
 
-	// Determine absolute path for database, if specified.
+	// Determine absolute path for database.
-	if c.DBPath, err = filepath.Abs(fs.Arg(0)); err != nil {
+	dbPath, err := filepath.Abs(fs.Arg(0))
+	if err != nil {
 		return err
 	}
 
 	// Instantiate DB.
-	dbConfig := config.DBConfig(c.DBPath)
+	dbConfig := config.DBConfig(dbPath)
 	if dbConfig == nil {
-		return fmt.Errorf("database not found in config: %s", c.DBPath)
+		return fmt.Errorf("database not found in config: %s", dbPath)
 	}
 	db, err := newDBFromConfig(dbConfig)
 	if err != nil {
@@ -107,6 +104,10 @@ Arguments:
 	    Restore from a specific generation.
 	    Defaults to generation with latest data.
 
+	-index NUM
+	    Restore up to a specific WAL index (inclusive).
+	    Defaults to use the highest available index.
+
 	-timestamp TIMESTAMP
 	    Restore to a specific point-in-time.
 	    Defaults to use the latest available backup.

cmd/litestream/snapshots.go

@@ -0,0 +1,117 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"os"
+	"path/filepath"
+	"text/tabwriter"
+	"time"
+
+	"github.com/benbjohnson/litestream"
+)
+
+type SnapshotsCommand struct{}
+
+func (c *SnapshotsCommand) Run(ctx context.Context, args []string) (err error) {
+	var configPath string
+	fs := flag.NewFlagSet("litestream-snapshots", flag.ContinueOnError)
+	registerConfigFlag(fs, &configPath)
+	replicaName := fs.String("replica", "", "replica name")
+	fs.Usage = c.Usage
+	if err := fs.Parse(args); err != nil {
+		return err
+	} else if fs.NArg() == 0 || fs.Arg(0) == "" {
+		return fmt.Errorf("database path required")
+	} else if fs.NArg() > 1 {
+		return fmt.Errorf("too many arguments")
+	}
+
+	// Load configuration.
+	if configPath == "" {
+		return errors.New("-config required")
+	}
+	config, err := ReadConfigFile(configPath)
+	if err != nil {
+		return err
+	}
+
+	// Determine absolute path for database.
+	dbPath, err := filepath.Abs(fs.Arg(0))
+	if err != nil {
+		return err
+	}
+
+	// Instantiate DB.
+	dbConfig := config.DBConfig(dbPath)
+	if dbConfig == nil {
+		return fmt.Errorf("database not found in config: %s", dbPath)
+	}
+	db, err := newDBFromConfig(dbConfig)
+	if err != nil {
+		return err
+	}
+
+	// Find snapshots by db or replica.
+	var infos []*litestream.SnapshotInfo
+	if *replicaName != "" {
+		if r := db.Replica(*replicaName); r == nil {
+			return fmt.Errorf("replica %q not found for database %q", *replicaName, dbPath)
+		} else if infos, err = r.Snapshots(ctx); err != nil {
+			return err
+		}
+	} else {
+		if infos, err = db.Snapshots(ctx); err != nil {
+			return err
+		}
+	}
+
+	// List all snapshots.
+	w := tabwriter.NewWriter(os.Stdout, 0, 8, 1, '\t', 0)
+	fmt.Fprintln(w, "replica\tgeneration\tindex\tsize\tcreated")
+	for _, info := range infos {
+		fmt.Fprintf(w, "%s\t%s\t%d\t%d\t%s\n",
+			info.Replica,
+			info.Generation,
+			info.Index,
+			info.Size,
+			info.CreatedAt.Format(time.RFC3339),
+		)
+	}
+	w.Flush()
+
+	return nil
+}
+
+func (c *SnapshotsCommand) Usage() {
+	fmt.Printf(`
+The snapshots command lists all snapshots available for a database.
+
+Usage:
+
+	litestream snapshots [arguments] DB
+
+Arguments:
+
+	-config PATH
+	    Specifies the configuration file.
+	    Defaults to %s
+
+	-replica NAME
+	    Optional, filter by a specific replica.
+
+
+Examples:
+
+	# List all snapshots for a database.
+	$ litestream snapshots /path/to/db
+
+	# List all snapshots on S3.
+	$ litestream snapshots -replica s3 /path/to/db
+
+`[1:],
+		DefaultConfigPath,
+	)
+}

cmd/litestream/validate.go

@@ -0,0 +1,136 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+
+	"github.com/benbjohnson/litestream"
+)
+
+type ValidateCommand struct{}
+
+func (c *ValidateCommand) Run(ctx context.Context, args []string) (err error) {
+	var configPath string
+	opt := litestream.NewRestoreOptions()
+	fs := flag.NewFlagSet("litestream-validate", flag.ContinueOnError)
+	registerConfigFlag(fs, &configPath)
+	fs.StringVar(&opt.ReplicaName, "replica", "", "replica name")
+	fs.BoolVar(&opt.DryRun, "dry-run", false, "dry run")
+	verbose := fs.Bool("v", false, "verbose output")
+	fs.Usage = c.Usage
+	if err := fs.Parse(args); err != nil {
+		return err
+	} else if fs.NArg() == 0 || fs.Arg(0) == "" {
+		return fmt.Errorf("database path required")
+	} else if fs.NArg() > 1 {
+		return fmt.Errorf("too many arguments")
+	}
+
+	// Load configuration.
+	if configPath == "" {
+		return errors.New("-config required")
+	}
+	config, err := ReadConfigFile(configPath)
+	if err != nil {
+		return err
+	}
+
+	// Verbose output is automatically enabled if dry run is specified.
+	if opt.DryRun {
+		*verbose = true
+	}
+
+	// Instantiate logger if verbose output is enabled.
+	if *verbose {
+		opt.Logger = log.New(os.Stderr, "", log.LstdFlags)
+	}
+
+	// Determine absolute path for database.
+	dbPath, err := filepath.Abs(fs.Arg(0))
+	if err != nil {
+		return err
+	}
+
+	// Instantiate DB.
+	dbConfig := config.DBConfig(dbPath)
+	if dbConfig == nil {
+		return fmt.Errorf("database not found in config: %s", dbPath)
+	}
+	db, err := newDBFromConfig(dbConfig)
+	if err != nil {
+		return err
+	}
+
+	// Ensure replica exists, if specified.
+	if opt.ReplicaName != "" && db.Replica(opt.ReplicaName) == nil {
+		return fmt.Errorf("replica not found: %s", opt.ReplicaName)
+	}
+
+	// Validate all matching replicas.
+	var hasInvalidReplica bool
+	for _, r := range db.Replicas {
+		if opt.ReplicaName != "" && opt.ReplicaName != r.Name() {
+			continue
+		}
+
+		if err := db.Validate(ctx, r.Name(), opt); err != nil {
+			fmt.Printf("%s: replica invalid: %s\n", r.Name(), err)
+		}
+	}
+
+	if hasInvalidReplica {
+		return fmt.Errorf("one or more invalid replicas found")
+	}
+
+	fmt.Println("ok")
+	return nil
+}
+
+func (c *ValidateCommand) Usage() {
+	fmt.Printf(`
+The validate command compares a checksum of the primary database with a
+checksum of the replica at the same point in time. Returns an error if the
+databases are not equal.
+
+The restored database must be written to a temporary file so you must ensure
+you have enough disk space before performing this operation.
+
+Usage:
+
+	litestream validate [arguments] DB
+
+Arguments:
+
+	-config PATH
+	    Specifies the configuration file.
+	    Defaults to %s
+
+	-replica NAME
+	    Validate a specific replica.
+	    Defaults to validating all replicas.
+
+	-dry-run
+	    Prints all log output as if it were running but does
+	    not perform actual validation.
+
+	-v
+	    Verbose output.
+
+
+Examples:
+
+	# Validate all replicas for the given database.
+	$ litestream validate /path/to/db
+
+	# Validate only the S3 replica.
+	$ litestream restore -replica s3 /path/to/db
+
+`[1:],
+		DefaultConfigPath,
+	)
+}

cmd/litestream/wal.go

@@ -0,0 +1,126 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"os"
+	"path/filepath"
+	"text/tabwriter"
+	"time"
+
+	"github.com/benbjohnson/litestream"
+)
+
+type WALCommand struct{}
+
+func (c *WALCommand) Run(ctx context.Context, args []string) (err error) {
+	var configPath string
+	fs := flag.NewFlagSet("litestream-wal", flag.ContinueOnError)
+	registerConfigFlag(fs, &configPath)
+	replicaName := fs.String("replica", "", "replica name")
+	generation := fs.String("generation", "", "generation name")
+	fs.Usage = c.Usage
+	if err := fs.Parse(args); err != nil {
+		return err
+	} else if fs.NArg() == 0 || fs.Arg(0) == "" {
+		return fmt.Errorf("database path required")
+	} else if fs.NArg() > 1 {
+		return fmt.Errorf("too many arguments")
+	}
+
+	// Load configuration.
+	if configPath == "" {
+		return errors.New("-config required")
+	}
+	config, err := ReadConfigFile(configPath)
+	if err != nil {
+		return err
+	}
+
+	// Determine absolute path for database.
+	dbPath, err := filepath.Abs(fs.Arg(0))
+	if err != nil {
+		return err
+	}
+
+	// Instantiate DB.
+	dbConfig := config.DBConfig(dbPath)
+	if dbConfig == nil {
+		return fmt.Errorf("database not found in config: %s", dbPath)
+	}
+	db, err := newDBFromConfig(dbConfig)
+	if err != nil {
+		return err
+	}
+
+	// Find snapshots by db or replica.
+	var infos []*litestream.WALInfo
+	if *replicaName != "" {
+		if r := db.Replica(*replicaName); r == nil {
+			return fmt.Errorf("replica %q not found for database %q", *replicaName, dbPath)
+		} else if infos, err = r.WALs(ctx); err != nil {
+			return err
+		}
+	} else {
+		if infos, err = db.WALs(ctx); err != nil {
+			return err
+		}
+	}
+
+	// List all WAL files.
+	w := tabwriter.NewWriter(os.Stdout, 0, 8, 1, '\t', 0)
+	fmt.Fprintln(w, "replica\tgeneration\tindex\toffset\tsize\tcreated")
+	for _, info := range infos {
+		if *generation != "" && info.Generation != *generation {
+			continue
+		}
+
+		fmt.Fprintf(w, "%s\t%s\t%d\t%d\t%d\t%s\n",
+			info.Replica,
+			info.Generation,
+			info.Index,
+			info.Offset,
+			info.Size,
+			info.CreatedAt.Format(time.RFC3339),
+		)
+	}
+	w.Flush()
+
+	return nil
+}
+
+func (c *WALCommand) Usage() {
+	fmt.Printf(`
+The wal command lists all wal files available for a database.
+
+Usage:
+
+	litestream wal [arguments] DB
+
+Arguments:
+
+	-config PATH
+	    Specifies the configuration file.
+	    Defaults to %s
+
+	-replica NAME
+	    Optional, filter by a specific replica.
+
+	-generation NAME
+	    Optional, filter by a specific generation.
+
+
+Examples:
+
+	# List all WAL files for a database.
+	$ litestream wal /path/to/db
+
+	# List all WAL files on S3 for a specific generation.
+	$ litestream snapshots -replica s3 -generation xxxxxxxx /path/to/db
+
+`[1:],
+		DefaultConfigPath,
+	)
+}

db.go

@@ -8,20 +8,27 @@ import (
 	"encoding/hex"
 	"errors"
 	"fmt"
+	"hash/crc32"
 	"io"
 	"io/ioutil"
 	"log"
+	"math"
 	"math/rand"
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
 	"sync"
 	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 )
 
 // Default DB settings.
 const (
 	DefaultMonitorInterval    = 1 * time.Second
+	DefaultCheckpointInterval = 1 * time.Minute
 	DefaultMinCheckpointPageN = 1000
 )
 
@@ -32,13 +39,26 @@ type DB struct {
 	db               *sql.DB       // target database
 	rtx              *sql.Tx       // long running read transaction
 	pageSize         int           // page size, in bytes
-
 	notify           chan struct{} // closes on WAL change
+	lastCheckpointAt time.Time     // last checkpoint time
 
 	ctx    context.Context
 	cancel func()
 	wg     sync.WaitGroup
 
+	// Metrics
+	dbSizeGauge                 prometheus.Gauge
+	walSizeGauge                prometheus.Gauge
+	totalWALBytesCounter        prometheus.Counter
+	shadowWALIndexGauge         prometheus.Gauge
+	shadowWALSizeGauge          prometheus.Gauge
+	syncNCounter                prometheus.Counter
+	syncErrorNCounter           prometheus.Counter
+	syncSecondsCounter          prometheus.Counter
+	checkpointNCounterVec       *prometheus.CounterVec
+	checkpointErrorNCounterVec  *prometheus.CounterVec
+	checkpointSecondsCounterVec *prometheus.CounterVec
+
 	// Minimum threshold of WAL size, in pages, before a passive checkpoint.
 	// A passive checkpoint will attempt a checkpoint but fail if there are
 	// active transactions occurring at the same time.
@@ -52,6 +72,11 @@ type DB struct {
 	// unbounded if there are always read transactions occurring.
 	MaxCheckpointPageN int
 
+	// Time between automatic checkpoints in the WAL. This is done to allow
+	// more fine-grained WAL files so that restores can be performed with
+	// better precision.
+	CheckpointInterval time.Duration
+
 	// List of replicas for the database.
 	// Must be set before calling Open().
 	Replicas []Replica
@@ -67,9 +92,24 @@ func NewDB(path string) *DB {
 		notify: make(chan struct{}),
 
 		MinCheckpointPageN: DefaultMinCheckpointPageN,
+		CheckpointInterval: DefaultCheckpointInterval,
 		MonitorInterval:    DefaultMonitorInterval,
 	}
+
+	db.dbSizeGauge = dbSizeGaugeVec.WithLabelValues(db.path)
+	db.walSizeGauge = walSizeGaugeVec.WithLabelValues(db.path)
+	db.totalWALBytesCounter = totalWALBytesCounterVec.WithLabelValues(db.path)
+	db.shadowWALIndexGauge = shadowWALIndexGaugeVec.WithLabelValues(db.path)
+	db.shadowWALSizeGauge = shadowWALSizeGaugeVec.WithLabelValues(db.path)
+	db.syncNCounter = syncNCounterVec.WithLabelValues(db.path)
+	db.syncErrorNCounter = syncErrorNCounterVec.WithLabelValues(db.path)
+	db.syncSecondsCounter = syncSecondsCounterVec.WithLabelValues(db.path)
+	db.checkpointNCounterVec = checkpointNCounterVec.MustCurryWith(prometheus.Labels{"db": db.path})
+	db.checkpointErrorNCounterVec = checkpointErrorNCounterVec.MustCurryWith(prometheus.Labels{"db": db.path})
+	db.checkpointSecondsCounterVec = checkpointSecondsCounterVec.MustCurryWith(prometheus.Labels{"db": db.path})
+
 	db.ctx, db.cancel = context.WithCancel(context.Background())
+
 	return db
 }
 
@@ -99,43 +139,83 @@ func (db *DB) GenerationPath(generation string) string {
 	return filepath.Join(db.MetaPath(), "generations", generation)
 }
 
+// ShadowWALDir returns the path of the shadow wal directory.
+func (db *DB) ShadowWALDir(generation string) string {
+	return filepath.Join(db.GenerationPath(generation), "wal")
+}
+
 // ShadowWALPath returns the path of a single shadow WAL file.
 func (db *DB) ShadowWALPath(generation string, index int) string {
 	assert(index >= 0, "shadow wal index cannot be negative")
-	return filepath.Join(db.GenerationPath(generation), "wal", fmt.Sprintf("%016x", index)+WALExt)
+	return filepath.Join(db.ShadowWALDir(generation), FormatWALPath(index))
 }
 
 // CurrentShadowWALPath returns the path to the last shadow WAL in a generation.
 func (db *DB) CurrentShadowWALPath(generation string) (string, error) {
-	index, err := db.CurrentShadowWALIndex(generation)
+	index, _, err := db.CurrentShadowWALIndex(generation)
 	if err != nil {
 		return "", err
 	}
 	return db.ShadowWALPath(generation, index), nil
 }
 
-// CurrentShadowWALIndex returns the current WAL index for a given generation.
+// CurrentShadowWALIndex returns the current WAL index & total size.
-func (db *DB) CurrentShadowWALIndex(generation string) (int, error) {
+func (db *DB) CurrentShadowWALIndex(generation string) (index int, size int64, err error) {
 	fis, err := ioutil.ReadDir(filepath.Join(db.GenerationPath(generation), "wal"))
 	if os.IsNotExist(err) {
-		return 0, nil // no wal files written for generation
+		return 0, 0, nil // no wal files written for generation
 	} else if err != nil {
-		return 0, err
+		return 0, 0, err
 	}
 
 	// Find highest wal index.
-	var index int
 	for _, fi := range fis {
 		if !strings.HasSuffix(fi.Name(), WALExt) {
 			continue
 		}
-		if v, err := ParseWALFilename(filepath.Base(fi.Name())); err != nil {
+		if v, _, _, err := ParseWALPath(fi.Name()); err != nil {
 			continue // invalid wal filename
 		} else if v > index {
 			index = v
 		}
+
+		size += fi.Size()
 	}
-	return index, nil
+	return index, size, nil
+}
+
+// Replica returns a replica by name.
+func (db *DB) Replica(name string) Replica {
+	for _, r := range db.Replicas {
+		if r.Name() == name {
+			return r
+		}
+	}
+	return nil
+}
+
+// Pos returns the current position of the database.
+func (db *DB) Pos() (Pos, error) {
+	generation, err := db.CurrentGeneration()
+	if err != nil {
+		return Pos{}, err
+	} else if generation == "" {
+		return Pos{}, nil
+	}
+
+	index, _, err := db.CurrentShadowWALIndex(generation)
+	if err != nil {
+		return Pos{}, err
+	}
+
+	fi, err := os.Stat(db.ShadowWALPath(generation, index))
+	if os.IsNotExist(err) {
+		return Pos{Generation: generation, Index: index}, nil
+	} else if err != nil {
+		return Pos{}, err
+	}
+
+	return Pos{Generation: generation, Index: index, Offset: fi.Size()}, nil
 }
 
 // Notify returns a channel that closes when the shadow WAL changes.
@@ -210,6 +290,42 @@ func (db *DB) UpdatedAt() (time.Time, error) {
 	return t, nil
 }
 
+// Snapshots returns a list of all snapshots across all replicas.
+func (db *DB) Snapshots(ctx context.Context) ([]*SnapshotInfo, error) {
+	var infos []*SnapshotInfo
+	for _, r := range db.Replicas {
+		a, err := r.Snapshots(ctx)
+		if err != nil {
+			return nil, err
+		}
+		infos = append(infos, a...)
+	}
+
+	// Sort in order by time.
+	sort.Slice(infos, func(i, j int) bool {
+		return infos[i].CreatedAt.Before(infos[j].CreatedAt)
+	})
+	return infos, nil
+}
+
+// WALs returns a list of all WAL files across all replicas.
+func (db *DB) WALs(ctx context.Context) ([]*WALInfo, error) {
+	var infos []*WALInfo
+	for _, r := range db.Replicas {
+		a, err := r.WALs(ctx)
+		if err != nil {
+			return nil, err
+		}
+		infos = append(infos, a...)
+	}
+
+	// Sort in order by time.
+	sort.Slice(infos, func(i, j int) bool {
+		return infos[i].CreatedAt.Before(infos[j].CreatedAt)
+	})
+	return infos, nil
+}
+
 // Init initializes the connection to the database.
 // Skipped if already initialized or if the database file does not exist.
 func (db *DB) Init() (err error) {
@@ -283,8 +399,16 @@ func (db *DB) Init() (err error) {
 	return nil
 }
 
-// clean removes old generations.
+// clean removes old generations & WAL files.
 func (db *DB) clean() error {
+	if err := db.cleanGenerations(); err != nil {
+		return err
+	}
+	return db.cleanWAL()
+}
+
+// cleanGenerations removes old generations.
+func (db *DB) cleanGenerations() error {
 	generation, err := db.CurrentGeneration()
 	if err != nil {
 		return err
@@ -311,6 +435,50 @@ func (db *DB) clean() error {
 	return nil
 }
 
+// cleanWAL removes WAL files that have been replicated.
+func (db *DB) cleanWAL() error {
+	generation, err := db.CurrentGeneration()
+	if err != nil {
+		return err
+	}
+
+	// Determine lowest index that's been replicated to all replicas.
+	min := -1
+	for _, r := range db.Replicas {
+		pos := r.LastPos()
+		if pos.Generation != generation {
+			pos = Pos{} // different generation, reset index to zero
+		}
+		if min == -1 || pos.Index < min {
+			min = pos.Index
+		}
+	}
+
+	// Skip if our lowest index is too small.
+	if min <= 0 {
+		return nil
+	}
+	min-- // Keep an extra WAL file.
+
+	// Remove all WAL files for the generation before the lowest index.
+	dir := db.ShadowWALDir(generation)
+	fis, err := ioutil.ReadDir(dir)
+	if os.IsNotExist(err) {
+		return nil
+	} else if err != nil {
+		return err
+	}
+	for _, fi := range fis {
+		if idx, _, _, err := ParseWALPath(fi.Name()); err != nil || idx >= min {
+			continue
+		}
+		if err := os.Remove(filepath.Join(dir, fi.Name())); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // SoftClose closes everything but the underlying db connection. This method
 // is available because the binary needs to avoid closing the database on exit
 // to prevent autocheckpointing.
@@ -432,6 +600,16 @@ func (db *DB) Sync() (err error) {
 		return nil
 	}
 
+	// Track total sync metrics.
+	t := time.Now()
+	defer func() {
+		db.syncNCounter.Inc()
+		if err != nil {
+			db.syncErrorNCounter.Inc()
+		}
+		db.syncSecondsCounter.Add(float64(time.Since(t).Seconds()))
+	}()
+
 	// TODO: Force "-wal" file if it doesn't exist.
 
 	// Ensure WAL has at least one frame in it.
@@ -495,11 +673,14 @@ func (db *DB) Sync() (err error) {
 
 	// If WAL size is great than max threshold, force checkpoint.
 	// If WAL size is greater than min threshold, attempt checkpoint.
-	var checkpoint, forceCheckpoint bool
+	var checkpoint bool
+	checkpointMode := CheckpointModePassive
 	if newWALSize >= calcWALSize(db.pageSize, db.MinCheckpointPageN) {
-		checkpoint, forceCheckpoint = true, false
+		checkpoint = true
 	} else if db.MaxCheckpointPageN > 0 && newWALSize >= calcWALSize(db.pageSize, db.MaxCheckpointPageN) {
-		checkpoint, forceCheckpoint = true, true
+		checkpoint, checkpointMode = true, CheckpointModeRestart
+	} else if db.CheckpointInterval > 0 && time.Since(db.lastCheckpointAt) > db.CheckpointInterval && newWALSize > calcWALSize(db.pageSize, 1) {
+		checkpoint = true
 	}
 
 	// Release write lock before checkpointing & exiting.
@@ -511,11 +692,22 @@ func (db *DB) Sync() (err error) {
 	if checkpoint {
 		changed = true
 
-		if err := db.checkpoint(info, forceCheckpoint); err != nil {
+		if err := db.checkpointAndInit(info, checkpointMode); err != nil {
-			return fmt.Errorf("checkpoint: force=%v err=%w", forceCheckpoint, err)
+			return fmt.Errorf("checkpoint: mode=%v err=%w", checkpointMode, err)
 		}
 	}
 
+	// Clean up any old files.
+	if err := db.clean(); err != nil {
+		return fmt.Errorf("cannot clean: %w", err)
+	}
+
+	// Compute current index and total shadow WAL size.
+	// This is only for metrics so we ignore any errors that occur.
+	index, size, _ := db.CurrentShadowWALIndex(info.generation)
+	db.shadowWALIndexGauge.Set(float64(index))
+	db.shadowWALSizeGauge.Set(float64(size))
+
 	// Notify replicas of WAL changes.
 	if changed {
 		close(db.notify)
@@ -551,12 +743,21 @@ func (db *DB) verify() (info syncInfo, err error) {
 	}
 	info.generation = generation
 
+	// Determine total bytes of real DB for metrics.
+	if fi, err := os.Stat(db.Path()); err != nil {
+		return info, err
+	} else {
+		db.dbSizeGauge.Set(float64(fi.Size()))
+	}
+
 	// Determine total bytes of real WAL.
 	fi, err := os.Stat(db.WALPath())
 	if err != nil {
 		return info, err
 	}
 	info.walSize = fi.Size()
+	info.walModTime = fi.ModTime()
+	db.walSizeGauge.Set(float64(fi.Size()))
 
 	// Open shadow WAL to copy append to.
 	info.shadowWALPath, err = db.CurrentShadowWALPath(info.generation)
@@ -622,6 +823,7 @@ func (db *DB) verify() (info syncInfo, err error) {
 type syncInfo struct {
 	generation    string    // generation name
 	walSize       int64     // size of real WAL file
+	walModTime    time.Time // last modified date of real WAL file
 	shadowWALPath string    // name of last shadow WAL file
 	shadowWALSize int64     // size of last shadow WAL file
 	restart       bool      // if true, real WAL header does not match shadow WAL
@@ -640,13 +842,13 @@ func (db *DB) syncWAL(info syncInfo) (newSize int64, err error) {
 
 	// Parse index of current shadow WAL file.
 	dir, base := filepath.Split(info.shadowWALPath)
-	index, err := ParseWALFilename(base)
+	index, _, _, err := ParseWALPath(base)
 	if err != nil {
 		return 0, fmt.Errorf("cannot parse shadow wal filename: %s", base)
 	}
 
 	// Start a new shadow WAL file with next index.
-	newShadowWALPath := filepath.Join(dir, FormatWALFilename(index+1))
+	newShadowWALPath := filepath.Join(dir, FormatWALPath(index+1))
 	if err := db.initShadowWALFile(newShadowWALPath); err != nil {
 		return 0, fmt.Errorf("cannot init shadow wal file: name=%s err=%w", newShadowWALPath, err)
 	}
@@ -732,7 +934,8 @@ func (db *DB) copyToShadowWAL(filename string) (newSize int64, err error) {
 	// TODO: Optimize to use bufio on reader & writer to minimize syscalls.
 
 	// Loop over each page, verify checksum, & copy to writer.
-	newSize = fi.Size()
+	origSize := fi.Size()
+	newSize = origSize
 	buf := make([]byte, db.pageSize+WALFrameHeaderSize)
 	for {
 		// Read next page from WAL file.
@@ -774,6 +977,9 @@ func (db *DB) copyToShadowWAL(filename string) (newSize int64, err error) {
 		return newSize, err
 	}
 
+	// Track total number of bytes written to WAL.
+	db.totalWALBytesCounter.Add(float64(newSize - origSize))
+
 	return newSize, nil
 }
 
@@ -909,12 +1115,17 @@ func readLastChecksumFrom(f *os.File, pageSize int) (uint32, uint32, error) {
 }
 
 // checkpoint performs a checkpoint on the WAL file.
-func (db *DB) checkpoint(info syncInfo, force bool) error {
+func (db *DB) checkpoint(mode string) (err error) {
-	// Read WAL header before checkpoint to check if it has been restarted.
+	// Track checkpoint metrics.
-	hdr, err := readWALHeader(db.WALPath())
+	t := time.Now()
+	defer func() {
+		labels := prometheus.Labels{"mode": mode}
+		db.checkpointNCounterVec.With(labels).Inc()
 		if err != nil {
-		return err
+			db.checkpointErrorNCounterVec.With(labels).Inc()
 		}
+		db.checkpointSecondsCounterVec.With(labels).Add(float64(time.Since(t).Seconds()))
+	}()
 
 	// Ensure the read lock has been removed before issuing a checkpoint.
 	// We defer the re-acquire to ensure it occurs even on an early return.
@@ -929,23 +1140,38 @@ func (db *DB) checkpoint(info syncInfo, force bool) error {
 	// forcing the checkpoint and restarting the WAL.
 	//
 	// See: https://www.sqlite.org/pragma.html#pragma_wal_checkpoint
-	rawsql := `PRAGMA wal_checkpoint;`
+	rawsql := `PRAGMA wal_checkpoint(` + mode + `);`
-	if force {
-		rawsql = `PRAGMA wal_checkpoint(RESTART);`
-	}
 
 	var row [3]int
 	if err := db.db.QueryRow(rawsql).Scan(&row[0], &row[1], &row[2]); err != nil {
 		return err
 	}
-	log.Printf("%s: checkpoint: force=%v (%d,%d,%d)", db.path, force, row[0], row[1], row[2])
+	log.Printf("%s: checkpoint: mode=%v (%d,%d,%d)", db.path, mode, row[0], row[1], row[2])
 
 	// Reacquire the read lock immediately after the checkpoint.
 	if err := db.acquireReadLock(); err != nil {
 		return fmt.Errorf("release read lock: %w", err)
 	}
 
-	if _, err = db.db.Exec(`INSERT INTO _litestream_seq (id, seq) VALUES (1, 1) ON CONFLICT (id) DO UPDATE SET seq = seq + 1`); err != nil {
+	db.lastCheckpointAt = time.Now()
+
+	return nil
+}
+
+// checkpointAndInit performs a checkpoint on the WAL file and initializes a
+// new shadow WAL file.
+func (db *DB) checkpointAndInit(info syncInfo, mode string) error {
+	// Read WAL header before checkpoint to check if it has been restarted.
+	hdr, err := readWALHeader(db.WALPath())
+	if err != nil {
+		return err
+	}
+
+	// Execute checkpoint and immediately issue a write to the WAL to ensure
+	// a new page is written.
+	if err := db.checkpoint(mode); err != nil {
+		return err
+	} else if _, err = db.db.Exec(`INSERT INTO _litestream_seq (id, seq) VALUES (1, 1) ON CONFLICT (id) DO UPDATE SET seq = seq + 1`); err != nil {
 		return err
 	}
 
@@ -962,14 +1188,13 @@ func (db *DB) checkpoint(info syncInfo, force bool) error {
 	}
 
 	// Parse index of current shadow WAL file.
-	dir, base := filepath.Split(info.shadowWALPath)
+	index, _, _, err := ParseWALPath(info.shadowWALPath)
-	index, err := ParseWALFilename(base)
 	if err != nil {
-		return fmt.Errorf("cannot parse shadow wal filename: %s", base)
+		return fmt.Errorf("cannot parse shadow wal filename: %s", info.shadowWALPath)
 	}
 
 	// Start a new shadow WAL file with next index.
-	newShadowWALPath := filepath.Join(dir, FormatWALFilename(index+1))
+	newShadowWALPath := filepath.Join(filepath.Dir(info.shadowWALPath), FormatWALPath(index+1))
 	if err := db.initShadowWALFile(newShadowWALPath); err != nil {
 		return fmt.Errorf("cannot init shadow wal file: name=%s err=%w", newShadowWALPath, err)
 	}
@@ -1010,6 +1235,13 @@ func (db *DB) monitor() {
 // a timestamp can be specified to restore the database to a specific
 // point-in-time.
 func (db *DB) Restore(ctx context.Context, opt RestoreOptions) error {
+	// Validate options.
+	if opt.Generation == "" && opt.Index != math.MaxInt64 {
+		return fmt.Errorf("must specify generation when restoring to index")
+	} else if opt.Index != math.MaxInt64 && !opt.Timestamp.IsZero() {
+		return fmt.Errorf("cannot specify index & timestamp to restore")
+	}
+
 	// Ensure logger exists.
 	logger := opt.Logger
 	if logger == nil {
@@ -1044,7 +1276,7 @@ func (db *DB) Restore(ctx context.Context, opt RestoreOptions) error {
 	}
 
 	// Find the maximum WAL index that occurs before timestamp.
-	maxWALIndex, err := r.WALIndexAt(ctx, generation, opt.Timestamp)
+	maxWALIndex, err := r.WALIndexAt(ctx, generation, opt.Index, opt.Timestamp)
 	if err != nil {
 		return fmt.Errorf("cannot find max wal index for restore: %w", err)
 	}
@@ -1171,6 +1403,13 @@ func (db *DB) restoreSnapshot(ctx context.Context, r Replica, generation string,
 
 // restoreWAL copies a WAL file from the replica to the local WAL and forces checkpoint.
 func (db *DB) restoreWAL(ctx context.Context, r Replica, generation string, index int, dbPath string) error {
+	// Open WAL file from replica.
+	rd, err := r.WALReader(ctx, generation, index)
+	if err != nil {
+		return err
+	}
+	defer rd.Close()
+
 	// Open handle to destination WAL path.
 	f, err := os.Create(dbPath + "-wal")
 	if err != nil {
@@ -1178,13 +1417,7 @@ func (db *DB) restoreWAL(ctx context.Context, r Replica, generation string, inde
 	}
 	defer f.Close()
 
-	rd, err := r.WALReader(ctx, generation, index)
+	// Copy WAL to target path.
-	if err != nil {
-		return err
-	}
-	defer rd.Close()
-
-	//
 	if _, err := io.Copy(f, rd); err != nil {
 		return err
 	} else if err := f.Close(); err != nil {
@@ -1207,6 +1440,167 @@ func (db *DB) restoreWAL(ctx context.Context, r Replica, generation string, inde
 	return nil
 }
 
+// Validate restores the most recent data from a replica and validates
+// that the resulting database matches the current database.
+func (db *DB) Validate(ctx context.Context, replicaName string, opt RestoreOptions) error {
+	if replicaName == "" {
+		return fmt.Errorf("replica name required")
+	}
+
+	// Look up replica by name.
+	r := db.Replica(replicaName)
+	if r == nil {
+		return fmt.Errorf("replica not found: %q", replicaName)
+	}
+
+	// Ensure logger exists.
+	logger := opt.Logger
+	if logger == nil {
+		logger = log.New(ioutil.Discard, "", 0)
+	}
+
+	logger.Printf("computing primary checksum")
+
+	// Compute checksum of primary database under read lock. This prevents a
+	// sync from occurring and the database will not be written.
+	chksum0, pos, err := db.CRC32C()
+	if err != nil {
+		return fmt.Errorf("cannot compute checksum: %w", err)
+	}
+	logger.Printf("primary checksum computed: %08x", chksum0)
+
+	// Wait until replica catches up to position.
+	logger.Printf("waiting for replica")
+	if err := db.waitForReplica(ctx, r, pos, logger); err != nil {
+		return fmt.Errorf("cannot wait for replica: %w", err)
+	}
+	logger.Printf("replica ready, restoring")
+
+	// Restore replica to a temporary directory.
+	tmpdir, err := ioutil.TempDir("", "*-litestream")
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(tmpdir)
+
+	restorePath := filepath.Join(tmpdir, "db")
+	if err := db.Restore(ctx, RestoreOptions{
+		OutputPath:  restorePath,
+		ReplicaName: replicaName,
+		Generation:  pos.Generation,
+		Index:       pos.Index - 1,
+		DryRun:      opt.DryRun,
+		Logger:      opt.Logger,
+	}); err != nil {
+		return fmt.Errorf("cannot restore: %w", err)
+	}
+
+	// Skip remaining validation if this is just a dry run.
+	if opt.DryRun {
+		return fmt.Errorf("validation stopped, dry run only")
+	}
+
+	logger.Printf("restore complete, computing checksum")
+
+	// Open file handle for restored database.
+	f, err := os.Open(db.Path())
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	// Compute checksum.
+	h := crc32.New(crc32.MakeTable(crc32.Castagnoli))
+	if _, err := io.Copy(h, f); err != nil {
+		return err
+	}
+	chksum1 := h.Sum32()
+
+	logger.Printf("replica checksum computed: %08x", chksum1)
+
+	// Validate checksums match.
+	if chksum0 != chksum1 {
+		return ErrChecksumMismatch
+	}
+
+	return nil
+}
+
+// waitForReplica blocks until replica reaches at least the given position.
+func (db *DB) waitForReplica(ctx context.Context, r Replica, pos Pos, logger *log.Logger) error {
+	ticker := time.NewTicker(500 * time.Millisecond)
+	defer ticker.Stop()
+
+	once := make(chan struct{}, 1)
+	once <- struct{}{}
+
+	for {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-ticker.C:
+		case <-once: // immediate on first check
+		}
+
+		// Obtain current position of replica, check if past target position.
+		curr, err := r.CalcPos(pos.Generation)
+		if err != nil {
+			logger.Printf("cannot obtain replica position: %s", err)
+			continue
+		}
+
+		ready := true
+		if curr.Generation != pos.Generation {
+			ready = false
+		} else if curr.Index < pos.Index {
+			ready = false
+		} else if curr.Index == pos.Index && curr.Offset < pos.Offset {
+			ready = false
+		}
+
+		// If not ready, restart loop.
+		if !ready {
+			logger.Printf("replica at %s, waiting for %s", curr, pos)
+			continue
+		}
+
+		// Current position at or after target position.
+		return nil
+	}
+}
+
+// CRC32C returns a CRC-32C checksum of the database and its current position.
+//
+// This function obtains a read lock so it prevents syncs from occuring until
+// the operation is complete. The database will still be usable but it will be
+// unable to checkpoint during this time.
+func (db *DB) CRC32C() (uint32, Pos, error) {
+	db.mu.RLock()
+	defer db.mu.RUnlock()
+
+	// Obtain current position. Clear the offset since we are only reading the
+	// DB and not applying the current WAL.
+	pos, err := db.Pos()
+	if err != nil {
+		return 0, pos, err
+	}
+	pos.Offset = 0
+
+	// Open file handle for database.
+	f, err := os.Open(db.Path())
+	if err != nil {
+		return 0, pos, err
+	}
+	defer f.Close()
+
+	// Compute checksum.
+	h := crc32.New(crc32.MakeTable(crc32.Castagnoli))
+	if _, err := io.Copy(h, f); err != nil {
+		return 0, pos, err
+	}
+	return h.Sum32(), pos, nil
+}
+
 // RestoreOptions represents options for DB.Restore().
 type RestoreOptions struct {
 	// Target path to restore into.
@@ -1221,6 +1615,10 @@ type RestoreOptions struct {
 	// If blank, all generations considered.
 	Generation string
 
+	// Specific index to restore from.
+	// Set to math.MaxInt64 to ignore index.
+	Index int
+
 	// Point-in-time to restore database.
 	// If zero, database restore to most recent state available.
 	Timestamp time.Time
@@ -1233,6 +1631,92 @@ type RestoreOptions struct {
 	Logger *log.Logger
 }
 
+func NewRestoreOptions() RestoreOptions {
+	return RestoreOptions{
+		Index: math.MaxInt64,
+	}
+}
+
+// Database metrics.
+var (
+	dbSizeGaugeVec = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: "litestream",
+		Subsystem: "db",
+		Name:      "db_size",
+		Help:      "The current size of the real DB",
+	}, []string{"db"})
+
+	walSizeGaugeVec = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: "litestream",
+		Subsystem: "db",
+		Name:      "wal_size",
+		Help:      "The current size of the real WAL",
+	}, []string{"db"})
+
+	totalWALBytesCounterVec = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: "litestream",
+		Subsystem: "db",
+		Name:      "total_wal_bytes",
+		Help:      "Total number of bytes written to shadow WAL",
+	}, []string{"db"})
+
+	shadowWALIndexGaugeVec = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: "litestream",
+		Subsystem: "db",
+		Name:      "shadow_wal_index",
+		Help:      "The current index of the shadow WAL",
+	}, []string{"db"})
+
+	shadowWALSizeGaugeVec = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: "litestream",
+		Subsystem: "db",
+		Name:      "shadow_wal_size",
+		Help:      "Current size of shadow WAL, in bytes",
+	}, []string{"db"})
+
+	syncNCounterVec = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: "litestream",
+		Subsystem: "db",
+		Name:      "sync_count",
+		Help:      "Number of sync operations performed",
+	}, []string{"db"})
+
+	syncErrorNCounterVec = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: "litestream",
+		Subsystem: "db",
+		Name:      "sync_error_count",
+		Help:      "Number of sync errors that have occurred",
+	}, []string{"db"})
+
+	syncSecondsCounterVec = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: "litestream",
+		Subsystem: "db",
+		Name:      "sync_seconds",
+		Help:      "Time spent syncing shadow WAL, in seconds",
+	}, []string{"db"})
+
+	checkpointNCounterVec = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: "litestream",
+		Subsystem: "db",
+		Name:      "checkpoint_count",
+		Help:      "Number of checkpoint operations performed",
+	}, []string{"db", "mode"})
+
+	checkpointErrorNCounterVec = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: "litestream",
+		Subsystem: "db",
+		Name:      "checkpoint_error_count",
+		Help:      "Number of checkpoint errors that have occurred",
+	}, []string{"db", "mode"})
+
+	checkpointSecondsCounterVec = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: "litestream",
+		Subsystem: "db",
+		Name:      "checkpoint_seconds",
+		Help:      "Time spent checkpointing WAL, in seconds",
+	}, []string{"db", "mode"})
+)
+
 func headerByteOrder(hdr []byte) (binary.ByteOrder, error) {
 	magic := binary.BigEndian.Uint32(hdr[0:])
 	switch magic {

go.mod

@@ -4,6 +4,6 @@ go 1.15
 
 require (
 	github.com/mattn/go-sqlite3 v1.14.5
-	github.com/pelletier/go-toml v1.8.1
+	github.com/prometheus/client_golang v1.9.0
 	gopkg.in/yaml.v2 v2.4.0
 )

go.sum

@@ -1,38 +1,403 @@
-github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74=
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
+github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
+github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
+github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g=
+github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c=
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
+github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
+github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
+github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
+github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
+github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
+github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A=
+github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU=
+github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
+github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
+github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
+github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
+github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
+github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
+github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
-github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4=
+github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
+github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
+github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
+github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
+github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4=
+github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o=
+github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
+github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
+github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
+github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
+github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
+github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
+github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
+github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE=
+github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
+github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
+github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
+github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
+github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
+github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
+github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
+github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
+github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
+github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
+github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
+github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg=
+github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
+github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
+github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
+github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
+github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=
+github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
+github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
+github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/mattn/go-sqlite3 v1.14.5 h1:1IdxlwTNazvbKJQSxoJ5/9ECbEeaTTyeU7sEAZ5KKTQ=
 github.com/mattn/go-sqlite3 v1.14.5/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI=
-github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNCRM=
+github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
-github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc=
+github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
+github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
+github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
+github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
+github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
+github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg=
+github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU=
+github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k=
+github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w=
+github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w=
+github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w=
+github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
+github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs=
+github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA=
+github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
+github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis=
+github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74=
+github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
+github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
+github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxSfWAKL3wpBW7V8scJMt8N8gnaMCS9E/cA=
+github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=
+github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4=
+github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4=
+github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM=
+github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
+github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
+github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac=
+github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc=
+github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/robertkrimen/godocdown v0.0.0-20130622164427-0bfa04905481/go.mod h1:C9WhFzY47SzYBIvzFqSvHIR6ROgDo4TtdTuRaOMjF/s=
+github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
-github.com/stephens2424/writerset v1.0.2/go.mod h1:aS2JhsMn6eA7e82oNmW4rfsgAOp9COBTTl8mzkwADnc=
+github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs=
+github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
+github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og=
+github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
+github.com/prometheus/client_golang v1.9.0 h1:Rrch9mh17XcxvEu9D9DEpb4isxjGBtcevQjKvxPRQIU=
+github.com/prometheus/client_golang v1.9.0/go.mod h1:FqZLKOZnGdFAhOK4nqGHa7D66IdsO+O441Eve7ptJDU=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
+github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA=
+github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
+github.com/prometheus/common v0.15.0 h1:4fgOnadei3EZvgRwxJ7RMpG1k1pOZth5Pc13tyspaKM=
+github.com/prometheus/common v0.15.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s=
+github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
+github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
+github.com/prometheus/procfs v0.2.0 h1:wH4vA7pcjKuZzjF7lM8awk4fnuJO6idemZXoKnULUx4=
+github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
+github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
+github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
+github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E=
+github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
+github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
+github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY=
+github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
+github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
+github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
+github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM=
+github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
-github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
+github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
+github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
+go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
+go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
+go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
+go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
+go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
+go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
+go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
+go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191210023423-ac6580df4449 h1:gSbV7h1NRL2G1xTg/owz62CST1oJBmxy4QpMMregXVQ=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e h1:AyodaIpKjppX+cBfTASF2E1US3H2JFBj920Ot3rtDjs=
+golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
-golang.org/x/tools v0.0.0-20200423201157-2723c5de0d66/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM=
+google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
+google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o=
+gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
+gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
+sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU=

litestream.go

@@ -13,6 +13,7 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
+	"time"
 
 	_ "github.com/mattn/go-sqlite3"
 )
@@ -27,11 +28,41 @@ const (
 	GenerationNameLen = 16
 )
 
+// SQLite checkpoint modes.
+const (
+	CheckpointModePassive  = "PASSIVE"
+	CheckpointModeFull     = "FULL"
+	CheckpointModeRestart  = "RESTART"
+	CheckpointModeTruncate = "TRUNCATE"
+)
+
 // Litestream errors.
 var (
 	ErrNoSnapshots      = errors.New("no snapshots available")
+	ErrChecksumMismatch = errors.New("invalid replica, checksum mismatch")
 )
 
+// SnapshotInfo represents file information about a snapshot.
+type SnapshotInfo struct {
+	Name       string
+	Replica    string
+	Generation string
+	Index      int
+	Size       int64
+	CreatedAt  time.Time
+}
+
+// WALInfo represents file information about a WAL file.
+type WALInfo struct {
+	Name       string
+	Replica    string
+	Generation string
+	Index      int
+	Offset     int64
+	Size       int64
+	CreatedAt  time.Time
+}
+
 // Pos is a position in the WAL for a generation.
 type Pos struct {
 	Generation string // generation name
@@ -119,19 +150,6 @@ func readFileAt(filename string, offset, n int64) ([]byte, error) {
 	return buf, nil
 }
 
-func ParseWALFilename(name string) (index int, err error) {
-	v, err := strconv.ParseInt(strings.TrimSuffix(name, WALExt), 16, 64)
-	if err != nil {
-		return 0, fmt.Errorf("invalid wal filename: %q", name)
-	}
-	return int(v), nil
-}
-
-func FormatWALFilename(index int) string {
-	assert(index >= 0, "wal index must be non-negative")
-	return fmt.Sprintf("%016x%s", index, WALExt)
-}
-
 // removeTmpFiles recursively finds and removes .tmp files.
 func removeTmpFiles(root string) error {
 	return filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
@@ -167,6 +185,8 @@ func IsSnapshotPath(s string) bool {
 // ParseSnapshotPath returns the index for the snapshot.
 // Returns an error if the path is not a valid snapshot path.
 func ParseSnapshotPath(s string) (index int, typ, ext string, err error) {
+	s = filepath.Base(s)
+
 	a := snapshotPathRegex.FindStringSubmatch(s)
 	if a == nil {
 		return 0, "", "", fmt.Errorf("invalid snapshot path: %s", s)
@@ -186,6 +206,8 @@ func IsWALPath(s string) bool {
 // ParseWALPath returns the index & offset for the WAL file.
 // Returns an error if the path is not a valid snapshot path.
 func ParseWALPath(s string) (index int, offset int64, ext string, err error) {
+	s = filepath.Base(s)
+
 	a := walPathRegex.FindStringSubmatch(s)
 	if a == nil {
 		return 0, 0, "", fmt.Errorf("invalid wal path: %s", s)
@@ -196,6 +218,19 @@ func ParseWALPath(s string) (index int, offset int64, ext string, err error) {
 	return int(i64), int64(off64), a[3], nil
 }
 
+// FormatWALPath formats a WAL filename with a given index.
+func FormatWALPath(index int) string {
+	assert(index >= 0, "wal index must be non-negative")
+	return fmt.Sprintf("%016x%s", index, WALExt)
+}
+
+// FormatWALPathWithOffset formats a WAL filename with a given index & offset.
+func FormatWALPathWithOffset(index int, offset int64) string {
+	assert(index >= 0, "wal index must be non-negative")
+	assert(offset >= 0, "wal offset must be non-negative")
+	return fmt.Sprintf("%016x_%016x%s", index, offset, WALExt)
+}
+
 var walPathRegex = regexp.MustCompile(`^([0-9a-f]{16})(?:_([0-9a-f]{16}))?(.wal(?:.gz)?)$`)
 
 // isHexChar returns true if ch is a lowercase hex character.

replica.go

@@ -7,10 +7,11 @@ import (
 	"io"
 	"io/ioutil"
 	"log"
+	"math"
 	"os"
 	"path/filepath"
 	"sort"
-	"strings"
+	"strconv"
 	"sync"
 	"time"
 )
@@ -29,6 +30,12 @@ type Replica interface {
 	// Stops all replication processing. Blocks until processing stopped.
 	Stop()
 
+	// Returns the last replication position.
+	LastPos() Pos
+
+	// Returns the computed position of the replica for a given generation.
+	CalcPos(generation string) (Pos, error)
+
 	// Returns a list of generation names for the replica.
 	Generations(ctx context.Context) ([]string, error)
 
@@ -36,13 +43,19 @@ type Replica interface {
 	// snapshot & WAL files as well as the time range covered.
 	GenerationStats(ctx context.Context, generation string) (GenerationStats, error)
 
+	// Returns a list of available snapshots in the replica.
+	Snapshots(ctx context.Context) ([]*SnapshotInfo, error)
+
+	// Returns a list of available WAL files in the replica.
+	WALs(ctx context.Context) ([]*WALInfo, error)
+
 	// Returns the highest index for a snapshot within a generation that occurs
 	// before timestamp. If timestamp is zero, returns the latest snapshot.
 	SnapshotIndexAt(ctx context.Context, generation string, timestamp time.Time) (int, error)
 
 	// Returns the highest index for a WAL file that occurs before timestamp.
 	// If timestamp is zero, returns the highest WAL index.
-	WALIndexAt(ctx context.Context, generation string, timestamp time.Time) (int, error)
+	WALIndexAt(ctx context.Context, generation string, maxIndex int, timestamp time.Time) (int, error)
 
 	// Returns a reader for snapshot data at the given generation/index.
 	SnapshotReader(ctx context.Context, generation string, index int) (io.ReadCloser, error)
@@ -51,6 +64,17 @@ type Replica interface {
 	WALReader(ctx context.Context, generation string, index int) (io.ReadCloser, error)
 }
 
+// GenerationStats represents high level stats for a single generation.
+type GenerationStats struct {
+	// Count of snapshot & WAL files.
+	SnapshotN int
+	WALN      int
+
+	// Time range for the earliest snapshot & latest WAL file update.
+	CreatedAt time.Time
+	UpdatedAt time.Time
+}
+
 var _ Replica = (*FileReplica)(nil)
 
 // FileReplica is a replica that replicates a DB to a local file path.
@@ -59,9 +83,10 @@ type FileReplica struct {
 	name string // replica name, optional
 	dst  string // destination path
 
-	// mu sync.RWMutex
+	mu  sync.RWMutex
-	wg sync.WaitGroup
+	pos Pos // last position
 
+	wg     sync.WaitGroup
 	ctx    context.Context
 	cancel func()
 }
@@ -89,6 +114,13 @@ func (r *FileReplica) Type() string {
 	return "file"
 }
 
+// LastPos returns the last successfully replicated position.
+func (r *FileReplica) LastPos() Pos {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+	return r.pos
+}
+
 // SnapshotDir returns the path to a generation's snapshot directory.
 func (r *FileReplica) SnapshotDir(generation string) string {
 	return filepath.Join(r.dst, "generations", generation, "snapshots")
@@ -99,14 +131,78 @@ func (r *FileReplica) SnapshotPath(generation string, index int) string {
 	return filepath.Join(r.SnapshotDir(generation), fmt.Sprintf("%016x.snapshot.gz", index))
 }
 
+// MaxSnapshotIndex returns the highest index for the snapshots.
+func (r *FileReplica) MaxSnapshotIndex(generation string) (int, error) {
+	fis, err := ioutil.ReadDir(r.SnapshotDir(generation))
+	if err != nil {
+		return 0, err
+	}
+
+	index := -1
+	for _, fi := range fis {
+		if idx, _, _, err := ParseSnapshotPath(fi.Name()); err != nil {
+			continue
+		} else if index == -1 || idx > index {
+			index = idx
+		}
+	}
+	if index == -1 {
+		return 0, fmt.Errorf("no snapshots found")
+	}
+	return index, nil
+}
+
 // WALDir returns the path to a generation's WAL directory
 func (r *FileReplica) WALDir(generation string) string {
 	return filepath.Join(r.dst, "generations", generation, "wal")
 }
 
+// WALSubdir returns the directory used for grouping WAL files.
+func (r *FileReplica) WALSubdir(generation string, index int) string {
+	return filepath.Join(r.WALDir(generation), fmt.Sprintf("%016x", uint64(index)&walDirMask))
+}
+
+// WALSubdirNames returns a list of all WAL subdirectory group names.
+func (r *FileReplica) WALSubdirNames(generation string) ([]string, error) {
+	fis, err := ioutil.ReadDir(r.WALDir(generation))
+	if err != nil && !os.IsNotExist(err) {
+		return nil, err
+	}
+
+	var names []string
+	for _, fi := range fis {
+		if _, err := strconv.ParseUint(fi.Name(), 16, 64); err != nil {
+			continue
+		}
+		names = append(names, fi.Name())
+	}
+	return names, nil
+}
+
+// MaxWALSubdirName returns the highest WAL subdirectory group name.
+func (r *FileReplica) MaxWALSubdirName(generation string) (string, error) {
+	fis, err := ioutil.ReadDir(r.WALDir(generation))
+	if err != nil {
+		return "", err
+	}
+
+	var name string
+	for _, fi := range fis {
+		if _, err := strconv.ParseUint(fi.Name(), 16, 64); err != nil {
+			continue
+		} else if name == "" || fi.Name() > name {
+			name = fi.Name()
+		}
+	}
+	if name == "" {
+		return "", os.ErrNotExist
+	}
+	return name, nil
+}
+
 // WALPath returns the path to a WAL file.
 func (r *FileReplica) WALPath(generation string, index int) string {
-	return filepath.Join(r.WALDir(generation), fmt.Sprintf("%016x.wal", index))
+	return filepath.Join(r.WALSubdir(generation, index), fmt.Sprintf("%016x.wal", index))
 }
 
 // Generations returns a list of available generation names.
@@ -184,7 +280,13 @@ func (r *FileReplica) snapshotStats(generation string) (n int, min, max time.Tim
 }
 
 func (r *FileReplica) walStats(generation string) (n int, min, max time.Time, err error) {
-	fis, err := ioutil.ReadDir(r.WALDir(generation))
+	names, err := r.WALSubdirNames(generation)
+	if err != nil {
+		return n, min, max, err
+	}
+
+	for _, name := range names {
+		fis, err := ioutil.ReadDir(filepath.Join(r.WALDir(generation), name))
 		if os.IsNotExist(err) {
 			return n, min, max, nil
 		} else if err != nil {
@@ -205,14 +307,101 @@ func (r *FileReplica) walStats(generation string) (n int, min, max time.Time, er
 				max = modTime
 			}
 		}
+	}
 	return n, min, max, nil
 }
 
-type GenerationStats struct {
+// Snapshots returns a list of available snapshots in the replica.
-	SnapshotN int
+func (r *FileReplica) Snapshots(ctx context.Context) ([]*SnapshotInfo, error) {
-	WALN      int
+	generations, err := r.Generations(ctx)
-	CreatedAt time.Time
+	if err != nil {
-	UpdatedAt time.Time
+		return nil, err
+	}
+
+	var infos []*SnapshotInfo
+	for _, generation := range generations {
+		fis, err := ioutil.ReadDir(r.SnapshotDir(generation))
+		if os.IsNotExist(err) {
+			continue
+		} else if err != nil {
+			return nil, err
+		}
+
+		for _, fi := range fis {
+			index, _, _, err := ParseSnapshotPath(fi.Name())
+			if err != nil {
+				continue
+			}
+
+			// TODO: Add schedule name to snapshot info.
+
+			infos = append(infos, &SnapshotInfo{
+				Name:       fi.Name(),
+				Replica:    r.Name(),
+				Generation: generation,
+				Index:      index,
+				Size:       fi.Size(),
+				CreatedAt:  fi.ModTime().UTC(),
+			})
+		}
+	}
+
+	return infos, nil
+}
+
+// WALs returns a list of available WAL files in the replica.
+func (r *FileReplica) WALs(ctx context.Context) ([]*WALInfo, error) {
+	generations, err := r.Generations(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	var infos []*WALInfo
+	for _, generation := range generations {
+		// Find a list of all directory groups.
+		dir := r.WALDir(generation)
+		subfis, err := ioutil.ReadDir(dir)
+		if os.IsNotExist(err) {
+			continue
+		} else if err != nil {
+			return nil, err
+		}
+
+		// Iterate over WAL group subdirectories.
+		for _, subfi := range subfis {
+			if !subfi.IsDir() {
+				continue
+			}
+
+			// Find a list of all WAL files in the group.
+			fis, err := ioutil.ReadDir(filepath.Join(dir, subfi.Name()))
+			if os.IsNotExist(err) {
+				continue
+			} else if err != nil {
+				return nil, err
+			}
+
+			// Iterate over each WAL file.
+			for _, fi := range fis {
+				index, offset, _, err := ParseWALPath(fi.Name())
+				if err != nil {
+					continue
+				}
+
+				infos = append(infos, &WALInfo{
+					Name:       fi.Name(),
+					Replica:    r.Name(),
+					Generation: generation,
+					Index:      index,
+					Offset:     offset,
+					Size:       fi.Size(),
+					CreatedAt:  fi.ModTime().UTC(),
+				})
+			}
+		}
+	}
+
+	return infos, nil
 }
 
 // Start starts replication for a given generation.
@@ -246,8 +435,6 @@ func (r *FileReplica) monitor(ctx context.Context) {
 	close(ch)
 	var notify <-chan struct{} = ch
 
-	var pos Pos
-	var err error
 	for {
 		select {
 		case <-ctx.Done():
@@ -258,86 +445,57 @@ func (r *FileReplica) monitor(ctx context.Context) {
 		// Fetch new notify channel before replicating data.
 		notify = r.db.Notify()
 
-		// Determine position, if necessary.
-		if pos.IsZero() {
-			if pos, err = r.pos(); err != nil {
-				log.Printf("%s(%s): cannot determine position: %s", r.db.Path(), r.Name(), err)
-				continue
-			} else if pos.IsZero() {
-				log.Printf("%s(%s): no generation, waiting for data", r.db.Path(), r.Name())
-				continue
-			}
-		}
-
-		// If we have no replicated WALs, start from last index in shadow WAL.
-		if pos.Index == 0 && pos.Offset == 0 {
-			if pos.Index, err = r.db.CurrentShadowWALIndex(pos.Generation); err != nil {
-				log.Printf("%s(%s): cannot determine latest shadow wal index: %s", r.db.Path(), r.Name(), err)
-				continue
-			}
-		}
-
 		// Synchronize the shadow wal into the replication directory.
-		if pos, err = r.sync(ctx, pos); err != nil {
+		if err := r.Sync(ctx); err != nil {
 			log.Printf("%s(%s): sync error: %s", r.db.Path(), r.Name(), err)
 			continue
 		}
-
-		// Gzip any old WAL files.
-		if pos.Generation != "" {
-			if err := r.compress(ctx, pos.Generation); err != nil {
-				log.Printf("%s(%s): compress error: %s", r.db.Path(), r.Name(), err)
-				continue
-			}
-		}
 	}
 }
 
-// pos returns the position for the replica for the current generation.
+// CalcPos returns the position for the replica for the current generation.
 // Returns a zero value if there is no active generation.
-func (r *FileReplica) pos() (pos Pos, err error) {
+func (r *FileReplica) CalcPos(generation string) (pos Pos, err error) {
-	// Find the current generation from the DB. Return zero pos if no generation.
-	generation, err := r.db.CurrentGeneration()
-	if err != nil {
-		return pos, err
-	} else if generation == "" {
-		return pos, nil // empty position
-	}
 	pos.Generation = generation
 
-	// Find the max WAL file.
+	// Find maximum snapshot index.
-	dir := r.WALDir(generation)
+	if pos.Index, err = r.MaxSnapshotIndex(generation); err != nil {
-	fis, err := ioutil.ReadDir(dir)
+		return Pos{}, err
+	}
+
+	// Find highest WAL subdirectory group.
+	subdir, err := r.MaxWALSubdirName(generation)
 	if os.IsNotExist(err) {
-		return pos, nil // no replicated wal, start at beginning of generation
+		return pos, nil // no replicated wal, start at snapshot index
 	} else if err != nil {
-		return pos, err
+		return Pos{}, err
+	}
+
+	// Find the max WAL file within WAL group.
+	fis, err := ioutil.ReadDir(filepath.Join(r.WALDir(generation), subdir))
+	if os.IsNotExist(err) {
+		return pos, nil // no replicated wal, start at snapshot index.
+	} else if err != nil {
+		return Pos{}, err
 	}
 
 	index := -1
 	for _, fi := range fis {
-		name := fi.Name()
+		if idx, _, _, err := ParseWALPath(fi.Name()); err != nil {
-		name = strings.TrimSuffix(name, ".gz")
-
-		if !strings.HasSuffix(name, WALExt) {
-			continue
-		}
-
-		if v, err := ParseWALFilename(filepath.Base(name)); err != nil {
 			continue // invalid wal filename
-		} else if index == -1 || v > index {
+		} else if index == -1 || idx > index {
-			index = v
+			index = idx
 		}
 	}
 	if index == -1 {
-		return pos, nil // wal directory exists but no wal files, return beginning pos
+		return pos, nil // wal directory exists but no wal files, return snapshot position
 	}
 	pos.Index = index
 
 	// Determine current offset.
-	fi, err := os.Stat(filepath.Join(dir, FormatWALFilename(pos.Index)))
+	fi, err := os.Stat(r.WALPath(pos.Generation, pos.Index))
 	if err != nil {
-		return pos, err
+		return Pos{}, err
 	}
 	pos.Offset = fi.Size()
 
@@ -387,67 +545,100 @@ func (r *FileReplica) snapshotN(generation string) (int, error) {
 	return n, nil
 }
 
-func (r *FileReplica) sync(ctx context.Context, pos Pos) (_ Pos, err error) {
+func (r *FileReplica) Sync(ctx context.Context) (err error) {
+	// Find current position of database.
+	dpos, err := r.db.Pos()
+	if err != nil {
+		return fmt.Errorf("cannot determine current generation: %w", err)
+	} else if dpos.IsZero() {
+		return fmt.Errorf("no generation, waiting for data")
+	}
+	generation := dpos.Generation
+
+	// Create snapshot if no snapshots exist for generation.
+	if n, err := r.snapshotN(generation); err != nil {
+		return err
+	} else if n == 0 {
+		if err := r.snapshot(ctx, generation, dpos.Index); err != nil {
+			return err
+		}
+	}
+
+	// Determine position, if necessary.
+	if r.LastPos().IsZero() {
+		pos, err := r.CalcPos(generation)
+		if err != nil {
+			return fmt.Errorf("cannot determine replica position: %s", err)
+		}
+
+		r.mu.Lock()
+		r.pos = pos
+		r.mu.Unlock()
+	}
+
 	// Read all WAL files since the last position.
 	for {
-		if pos, err = r.syncNext(ctx, pos); err == io.EOF {
+		if err = r.syncWAL(ctx); err == io.EOF {
-			return pos, nil
+			break
 		} else if err != nil {
-			return pos, err
+			return err
-		}
 		}
 	}
 
-func (r *FileReplica) syncNext(ctx context.Context, pos Pos) (_ Pos, err error) {
+	// Gzip any old WAL files.
-	rd, err := r.db.ShadowWALReader(pos)
+	if generation != "" {
+		if err := r.compress(ctx, generation); err != nil {
+			return fmt.Errorf("cannot compress: %s", err)
+		}
+	}
+
+	return nil
+}
+
+func (r *FileReplica) syncWAL(ctx context.Context) (err error) {
+	rd, err := r.db.ShadowWALReader(r.LastPos())
 	if err == io.EOF {
-		return pos, err
+		return err
 	} else if err != nil {
-		return pos, fmt.Errorf("wal reader: %w", err)
+		return fmt.Errorf("wal reader: %w", err)
 	}
 	defer rd.Close()
 
-	// Create snapshot if no snapshots exist.
-	if n, err := r.snapshotN(rd.Pos().Generation); err != nil {
-		return pos, err
-	} else if n == 0 {
-		if err := r.snapshot(ctx, rd.Pos().Generation, rd.Pos().Index); err != nil {
-			return pos, err
-		}
-	}
-
 	// Ensure parent directory exists for WAL file.
 	filename := r.WALPath(rd.Pos().Generation, rd.Pos().Index)
 	if err := os.MkdirAll(filepath.Dir(filename), 0700); err != nil {
-		return pos, err
+		return err
 	}
 
 	// Create a temporary file to write into so we don't have partial writes.
 	w, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE, 0600)
 	if err != nil {
-		return pos, err
+		return err
 	}
 	defer w.Close()
 
 	// Seek, copy & sync WAL contents.
 	if _, err := w.Seek(rd.Pos().Offset, io.SeekStart); err != nil {
-		return pos, err
+		return err
 	} else if _, err := io.Copy(w, rd); err != nil {
-		return pos, err
+		return err
 	} else if err := w.Sync(); err != nil {
-		return pos, err
+		return err
 	} else if err := w.Close(); err != nil {
-		return pos, err
+		return err
 	}
 
-	// Return ending position of the reader.
+	// Save last replicated position.
-	return rd.Pos(), nil
+	r.mu.Lock()
+	r.pos = rd.Pos()
+	r.mu.Unlock()
+
+	return nil
 }
 
 // compress gzips all WAL files before the current one.
 func (r *FileReplica) compress(ctx context.Context, generation string) error {
-	dir := r.WALDir(generation)
+	filenames, err := filepath.Glob(filepath.Join(r.WALDir(generation), "**/*.wal"))
-	filenames, err := filepath.Glob(filepath.Join(dir, "*.wal"))
 	if err != nil {
 		return err
 	} else if len(filenames) <= 1 {
@@ -510,17 +701,26 @@ func (r *FileReplica) SnapshotIndexAt(ctx context.Context, generation string, ti
 	return index, nil
 }
 
-// Returns the highest index for a WAL file that occurs before timestamp.
+// Returns the highest index for a WAL file that occurs before maxIndex & timestamp.
 // If timestamp is zero, returns the highest WAL index.
-func (r *FileReplica) WALIndexAt(ctx context.Context, generation string, timestamp time.Time) (int, error) {
+func (r *FileReplica) WALIndexAt(ctx context.Context, generation string, maxIndex int, timestamp time.Time) (int, error) {
-	fis, err := ioutil.ReadDir(r.WALDir(generation))
+	names, err := r.WALSubdirNames(generation)
+	if err != nil {
+		return 0, err
+	}
+
+	// TODO: Optimize to only read the last group if no timestamp specified.
+	// TODO: Perform binary search to find correct timestamp.
+
+	var index int
+	for _, name := range names {
+		fis, err := ioutil.ReadDir(filepath.Join(r.WALDir(generation), name))
 		if os.IsNotExist(err) {
 			return 0, nil
 		} else if err != nil {
 			return 0, err
 		}
 
-	index := -1
 		for _, fi := range fis {
 			// Read index from snapshot filename.
 			idx, _, _, err := ParseWALPath(fi.Name())
@@ -528,16 +728,21 @@ func (r *FileReplica) WALIndexAt(ctx context.Context, generation string, timesta
 				continue // not a snapshot, skip
 			} else if !timestamp.IsZero() && fi.ModTime().After(timestamp) {
 				continue // after timestamp, skip
+			} else if idx > maxIndex {
+				continue // after timestamp, skip
 			} else if idx < index {
 				continue // earlier index, skip
 			}
 
 			index = idx
 		}
-
-	if index == -1 {
-		return 0, nil
 	}
+
+	// If max index is specified but not found, return an error.
+	if maxIndex != math.MaxInt64 && index != maxIndex {
+		return index, fmt.Errorf("unable to locate index %d in generation %q, highest index was %d", maxIndex, generation, index)
+	}
+
 	return index, nil
 }
 
@@ -638,3 +843,9 @@ func compressFile(src, dst string) error {
 	// Move compressed file to final location.
 	return os.Rename(dst+".tmp", dst)
 }
+
+// walDirMask is a mask used to group 64K wal files into a directory.
+const (
+	walDirFileN = 0x10000
+	walDirMask  = uint64(0xFFFFFFFFFFFFFFFF ^ (walDirFileN - 1))
+)