Group replica wal segments by index

This commit changes the replica path format to group segments within
a single index in the same directory. This is to eventually add the
ability to seek to a record on file-based systems without having
to iterate over the records. The DB shadow WAL will also be changed
to this same format to support live replicas.

.github/workflows/release.linux.yml

@@ -6,7 +6,7 @@ on:
 name: release (linux)
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-18.04
     strategy:
       matrix:
         include:
@@ -30,6 +30,8 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-go@v2
+        with:
+          go-version: '1.16'
 
       - id: release
         uses: bruceadams/get-release@v1.2.2

.github/workflows/release.linux_static.yml

@@ -6,7 +6,7 @@ on:
 name: release (linux/static)
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-18.04
     strategy:
       matrix:
         include:
@@ -30,6 +30,8 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-go@v2
+        with:
+          go-version: '1.16'
 
       - id: release
         uses: bruceadams/get-release@v1.2.2

.github/workflows/test.yml

@@ -2,11 +2,11 @@ on: push
 name: test
 jobs:
   test:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-18.04
     steps:
       - uses: actions/setup-go@v2
         with:
-          go-version: '1.15'
+          go-version: '1.16'
 
       - uses: actions/checkout@v2
 
@@ -17,5 +17,46 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-go-
 
+      - name: Extract GCP credentials
+        run: 'echo "$GOOGLE_APPLICATION_CREDENTIALS" > /opt/gcp.json'
+        shell: bash
+        env:
+          GOOGLE_APPLICATION_CREDENTIALS: ${{secrets.GOOGLE_APPLICATION_CREDENTIALS}}
+
+      - name: Extract SSH key
+        run: 'echo "$LITESTREAM_SFTP_KEY" > /opt/id_ed25519'
+        shell: bash
+        env:
+          LITESTREAM_SFTP_KEY: ${{secrets.LITESTREAM_SFTP_KEY}}
+
       - name: Run unit tests
         run: go test -v ./...
+
+      - name: Run aws s3 tests
+        run: go test -v -run=TestReplicaClient . -integration s3
+        env:
+          LITESTREAM_S3_ACCESS_KEY_ID:     ${{ secrets.LITESTREAM_S3_ACCESS_KEY_ID }}
+          LITESTREAM_S3_SECRET_ACCESS_KEY: ${{ secrets.LITESTREAM_S3_SECRET_ACCESS_KEY }}
+          LITESTREAM_S3_REGION:            ${{ secrets.LITESTREAM_S3_REGION }}
+          LITESTREAM_S3_BUCKET:            ${{ secrets.LITESTREAM_S3_BUCKET }}
+
+      - name: Run google cloud storage (gcs) tests
+        run: go test -v -run=TestReplicaClient . -integration gcs
+        env:
+          GOOGLE_APPLICATION_CREDENTIALS:  /opt/gcp.json
+          LITESTREAM_GCS_BUCKET:           ${{ secrets.LITESTREAM_GCS_BUCKET }}
+
+      - name: Run azure blob storage (abs) tests
+        run: go test -v -run=TestReplicaClient . -integration abs
+        env:
+          LITESTREAM_ABS_ACCOUNT_NAME:     ${{ secrets.LITESTREAM_ABS_ACCOUNT_NAME }}
+          LITESTREAM_ABS_ACCOUNT_KEY:      ${{ secrets.LITESTREAM_ABS_ACCOUNT_KEY }}
+          LITESTREAM_ABS_BUCKET:           ${{ secrets.LITESTREAM_ABS_BUCKET }}
+
+      - name: Run sftp tests
+        run: go test -v -run=TestReplicaClient . -integration sftp
+        env:
+          LITESTREAM_SFTP_HOST:     ${{ secrets.LITESTREAM_SFTP_HOST }}
+          LITESTREAM_SFTP_USER:     ${{ secrets.LITESTREAM_SFTP_USER }}
+          LITESTREAM_SFTP_KEY_PATH: /opt/id_ed25519
+          LITESTREAM_SFTP_PATH:     ${{ secrets.LITESTREAM_SFTP_PATH }}

Dockerfile

@@ -1,9 +1,10 @@
 FROM golang:1.16 as builder
 WORKDIR /src/litestream
 COPY . .
+ARG LITESTREAM_VERSION=latest
 RUN --mount=type=cache,target=/root/.cache/go-build \
 	--mount=type=cache,target=/go/pkg \
-	go build -ldflags '-s -w -extldflags "-static"' -tags osusergo,netgo,sqlite_omit_load_extension -o /usr/local/bin/litestream ./cmd/litestream
+	go build -ldflags "-s -w -X 'main.Version=${LITESTREAM_VERSION}' -extldflags '-static'" -tags osusergo,netgo,sqlite_omit_load_extension -o /usr/local/bin/litestream ./cmd/litestream
 
 FROM alpine
 COPY --from=builder /usr/local/bin/litestream /usr/local/bin/litestream

README.md

@@ -29,9 +29,16 @@ of the most valuable contributions are in the forms of testing, feedback, and
 documentation. These help harden software and streamline usage for other users.
 
 I want to give special thanks to individuals who invest much of their time and 
-energy into the project to help make it better. Shout out to [Michael
+energy into the project to help make it better:
-Lynch](https://github.com/mtlynch) for digging into issues and contributing to
+
-the documentation.
+- Thanks to [Cory LaNou](https://twitter.com/corylanou) for giving early feedback and testing when Litestream was still pre-release.
+- Thanks to [Michael Lynch](https://github.com/mtlynch) for digging into issues and contributing to the documentation.
+- Thanks to [Kurt Mackey](https://twitter.com/mrkurt) for feedback and testing. Also, thanks to fly.io for providing testing resources.
+- Thanks to [Sam Weston](https://twitter.com/cablespaghetti) for figuring out how to run Litestream on Kubernetes and writing up the docs for it.
+- Thanks to [Rafael](https://github.com/netstx) & [Jungle Boogie](https://github.com/jungle-boogie) for helping to get OpenBSD release builds working.
+- Thanks to [Simon Gottschlag](https://github.com/simongottschlag), [Marin](https://github.com/supermarin),[Victor Björklund](https://github.com/victorbjorklund), [Jonathan Beri](https://twitter.com/beriberikix) [Yuri](https://github.com/yurivish), [Nathan Probst](https://github.com/nprbst), [Yann Coleuu](https://github.com/yanc0), and [Nicholas Grilly](https://twitter.com/ngrilly) for frequent feedback, testing, & support.
+
+
 
 
 ## Open-source, not open-contribution

abs/replica_client.go

@@ -0,0 +1,560 @@
+package abs
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/url"
+	"os"
+	"path"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/Azure/azure-storage-blob-go/azblob"
+	"github.com/benbjohnson/litestream"
+	"github.com/benbjohnson/litestream/internal"
+	"golang.org/x/sync/errgroup"
+)
+
+// ReplicaClientType is the client type for this package.
+const ReplicaClientType = "abs"
+
+var _ litestream.ReplicaClient = (*ReplicaClient)(nil)
+
+// ReplicaClient is a client for writing snapshots & WAL segments to disk.
+type ReplicaClient struct {
+	mu           sync.Mutex
+	containerURL *azblob.ContainerURL
+
+	// Azure credentials
+	AccountName string
+	AccountKey  string
+	Endpoint    string
+
+	// Azure Blob Storage container information
+	Bucket string
+	Path   string
+}
+
+// NewReplicaClient returns a new instance of ReplicaClient.
+func NewReplicaClient() *ReplicaClient {
+	return &ReplicaClient{}
+}
+
+// Type returns "abs" as the client type.
+func (c *ReplicaClient) Type() string {
+	return ReplicaClientType
+}
+
+// Init initializes the connection to Azure. No-op if already initialized.
+func (c *ReplicaClient) Init(ctx context.Context) (err error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if c.containerURL != nil {
+		return nil
+	}
+
+	// Read account key from environment, if available.
+	accountKey := c.AccountKey
+	if accountKey == "" {
+		accountKey = os.Getenv("LITESTREAM_AZURE_ACCOUNT_KEY")
+	}
+
+	// Authenticate to ACS.
+	credential, err := azblob.NewSharedKeyCredential(c.AccountName, accountKey)
+	if err != nil {
+		return err
+	}
+
+	// Construct & parse endpoint unless already set.
+	endpoint := c.Endpoint
+	if endpoint == "" {
+		endpoint = fmt.Sprintf("https://%s.blob.core.windows.net", c.AccountName)
+	}
+	endpointURL, err := url.Parse(endpoint)
+	if err != nil {
+		return fmt.Errorf("cannot parse azure endpoint: %w", err)
+	}
+
+	// Build pipeline and reference to container.
+	pipeline := azblob.NewPipeline(credential, azblob.PipelineOptions{
+		Retry: azblob.RetryOptions{
+			TryTimeout: 24 * time.Hour,
+		},
+	})
+	containerURL := azblob.NewServiceURL(*endpointURL, pipeline).NewContainerURL(c.Bucket)
+	c.containerURL = &containerURL
+
+	return nil
+}
+
+// Generations returns a list of available generation names.
+func (c *ReplicaClient) Generations(ctx context.Context) ([]string, error) {
+	if err := c.Init(ctx); err != nil {
+		return nil, err
+	}
+
+	var generations []string
+	var marker azblob.Marker
+	for marker.NotDone() {
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "LIST").Inc()
+
+		resp, err := c.containerURL.ListBlobsHierarchySegment(ctx, marker, "/", azblob.ListBlobsSegmentOptions{
+			Prefix: path.Join(c.Path, "generations") + "/",
+		})
+		if err != nil {
+			return nil, err
+		}
+		marker = resp.NextMarker
+
+		for _, prefix := range resp.Segment.BlobPrefixes {
+			name := path.Base(strings.TrimSuffix(prefix.Name, "/"))
+			if !litestream.IsGenerationName(name) {
+				continue
+			}
+			generations = append(generations, name)
+		}
+	}
+
+	return generations, nil
+}
+
+// DeleteGeneration deletes all snapshots & WAL segments within a generation.
+func (c *ReplicaClient) DeleteGeneration(ctx context.Context, generation string) error {
+	if err := c.Init(ctx); err != nil {
+		return err
+	} else if generation == "" {
+		return fmt.Errorf("generation required")
+	}
+
+	prefix := path.Join(c.Path, "generations", generation) + "/"
+
+	var marker azblob.Marker
+	for marker.NotDone() {
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "LIST").Inc()
+
+		resp, err := c.containerURL.ListBlobsFlatSegment(ctx, marker, azblob.ListBlobsSegmentOptions{Prefix: prefix})
+		if err != nil {
+			return err
+		}
+		marker = resp.NextMarker
+
+		for _, item := range resp.Segment.BlobItems {
+			internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "DELETE").Inc()
+
+			blobURL := c.containerURL.NewBlobURL(item.Name)
+			if _, err := blobURL.Delete(ctx, azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{}); isNotExists(err) {
+				continue
+			} else if err != nil {
+				return err
+			}
+		}
+	}
+
+	// log.Printf("%s(%s): retainer: deleting generation: %s", r.db.Path(), r.Name(), generation)
+
+	return nil
+}
+
+// Snapshots returns an iterator over all available snapshots for a generation.
+func (c *ReplicaClient) Snapshots(ctx context.Context, generation string) (litestream.SnapshotIterator, error) {
+	if err := c.Init(ctx); err != nil {
+		return nil, err
+	}
+	return newSnapshotIterator(ctx, generation, c), nil
+}
+
+// WriteSnapshot writes LZ4 compressed data from rd to the object storage.
+func (c *ReplicaClient) WriteSnapshot(ctx context.Context, generation string, index int, rd io.Reader) (info litestream.SnapshotInfo, err error) {
+	if err := c.Init(ctx); err != nil {
+		return info, err
+	} else if generation == "" {
+		return info, fmt.Errorf("generation required")
+	}
+
+	key := path.Join(c.Path, "generations", generation, "snapshots", litestream.FormatIndex(index)+".snapshot.lz4")
+	startTime := time.Now()
+
+	rc := internal.NewReadCounter(rd)
+
+	blobURL := c.containerURL.NewBlockBlobURL(key)
+	if _, err := azblob.UploadStreamToBlockBlob(ctx, rc, blobURL, azblob.UploadStreamToBlockBlobOptions{
+		BlobHTTPHeaders: azblob.BlobHTTPHeaders{ContentType: "application/octet-stream"},
+		BlobAccessTier:  azblob.DefaultAccessTier,
+	}); err != nil {
+		return info, err
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "PUT").Inc()
+	internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "PUT").Add(float64(rc.N()))
+
+	// log.Printf("%s(%s): snapshot: creating %s/%08x t=%s", r.db.Path(), r.Name(), generation, index, time.Since(startTime).Truncate(time.Millisecond))
+
+	return litestream.SnapshotInfo{
+		Generation: generation,
+		Index:      index,
+		Size:       rc.N(),
+		CreatedAt:  startTime.UTC(),
+	}, nil
+}
+
+// SnapshotReader returns a reader for snapshot data at the given generation/index.
+func (c *ReplicaClient) SnapshotReader(ctx context.Context, generation string, index int) (io.ReadCloser, error) {
+	if err := c.Init(ctx); err != nil {
+		return nil, err
+	} else if generation == "" {
+		return nil, fmt.Errorf("generation required")
+	}
+
+	key := path.Join(c.Path, "generations", generation, "snapshots", litestream.FormatIndex(index)+".snapshot.lz4")
+
+	blobURL := c.containerURL.NewBlobURL(key)
+	resp, err := blobURL.Download(ctx, 0, 0, azblob.BlobAccessConditions{}, false, azblob.ClientProvidedKeyOptions{})
+	if isNotExists(err) {
+		return nil, os.ErrNotExist
+	} else if err != nil {
+		return nil, fmt.Errorf("cannot start new reader for %q: %w", key, err)
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "GET").Inc()
+	internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "GET").Add(float64(resp.ContentLength()))
+
+	return resp.Body(azblob.RetryReaderOptions{}), nil
+}
+
+// DeleteSnapshot deletes a snapshot with the given generation & index.
+func (c *ReplicaClient) DeleteSnapshot(ctx context.Context, generation string, index int) error {
+	if err := c.Init(ctx); err != nil {
+		return err
+	} else if generation == "" {
+		return fmt.Errorf("generation required")
+	}
+
+	key := path.Join(c.Path, "generations", generation, "snapshots", litestream.FormatIndex(index)+".snapshot.lz4")
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "DELETE").Inc()
+
+	blobURL := c.containerURL.NewBlobURL(key)
+	if _, err := blobURL.Delete(ctx, azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{}); isNotExists(err) {
+		return nil
+	} else if err != nil {
+		return fmt.Errorf("cannot delete snapshot %q: %w", key, err)
+	}
+	return nil
+}
+
+// WALSegments returns an iterator over all available WAL files for a generation.
+func (c *ReplicaClient) WALSegments(ctx context.Context, generation string) (litestream.WALSegmentIterator, error) {
+	if err := c.Init(ctx); err != nil {
+		return nil, err
+	}
+	return newWALSegmentIterator(ctx, generation, c), nil
+}
+
+// WriteWALSegment writes LZ4 compressed data from rd into a file on disk.
+func (c *ReplicaClient) WriteWALSegment(ctx context.Context, pos litestream.Pos, rd io.Reader) (info litestream.WALSegmentInfo, err error) {
+	if err := c.Init(ctx); err != nil {
+		return info, err
+	} else if pos.Generation == "" {
+		return info, fmt.Errorf("generation required")
+	}
+
+	key := path.Join(c.Path, "generations", pos.Generation, "wal", litestream.FormatIndex(pos.Index), litestream.FormatOffset(pos.Offset)+".wal.lz4")
+	startTime := time.Now()
+
+	rc := internal.NewReadCounter(rd)
+
+	blobURL := c.containerURL.NewBlockBlobURL(key)
+	if _, err := azblob.UploadStreamToBlockBlob(ctx, rc, blobURL, azblob.UploadStreamToBlockBlobOptions{
+		BlobHTTPHeaders: azblob.BlobHTTPHeaders{ContentType: "application/octet-stream"},
+		BlobAccessTier:  azblob.DefaultAccessTier,
+	}); err != nil {
+		return info, err
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "PUT").Inc()
+	internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "PUT").Add(float64(rc.N()))
+
+	return litestream.WALSegmentInfo{
+		Generation: pos.Generation,
+		Index:      pos.Index,
+		Offset:     pos.Offset,
+		Size:       rc.N(),
+		CreatedAt:  startTime.UTC(),
+	}, nil
+}
+
+// WALSegmentReader returns a reader for a section of WAL data at the given index.
+// Returns os.ErrNotExist if no matching index/offset is found.
+func (c *ReplicaClient) WALSegmentReader(ctx context.Context, pos litestream.Pos) (io.ReadCloser, error) {
+	if err := c.Init(ctx); err != nil {
+		return nil, err
+	} else if pos.Generation == "" {
+		return nil, fmt.Errorf("generation required")
+	}
+
+	key := path.Join(c.Path, "generations", pos.Generation, "wal", litestream.FormatIndex(pos.Index), litestream.FormatOffset(pos.Offset)+".wal.lz4")
+
+	blobURL := c.containerURL.NewBlobURL(key)
+	resp, err := blobURL.Download(ctx, 0, 0, azblob.BlobAccessConditions{}, false, azblob.ClientProvidedKeyOptions{})
+	if isNotExists(err) {
+		return nil, os.ErrNotExist
+	} else if err != nil {
+		return nil, fmt.Errorf("cannot start new reader for %q: %w", key, err)
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "GET").Inc()
+	internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "GET").Add(float64(resp.ContentLength()))
+
+	return resp.Body(azblob.RetryReaderOptions{}), nil
+}
+
+// DeleteWALSegments deletes WAL segments with at the given positions.
+func (c *ReplicaClient) DeleteWALSegments(ctx context.Context, a []litestream.Pos) error {
+	if err := c.Init(ctx); err != nil {
+		return err
+	}
+
+	for _, pos := range a {
+		if pos.Generation == "" {
+			return fmt.Errorf("generation required")
+		}
+
+		key := path.Join(c.Path, "generations", pos.Generation, "wal", litestream.FormatIndex(pos.Index), litestream.FormatOffset(pos.Offset)+".wal.lz4")
+
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "DELETE").Inc()
+
+		blobURL := c.containerURL.NewBlobURL(key)
+		if _, err := blobURL.Delete(ctx, azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{}); isNotExists(err) {
+			continue
+		} else if err != nil {
+			return fmt.Errorf("cannot delete wal segment %q: %w", key, err)
+		}
+	}
+
+	return nil
+}
+
+type snapshotIterator struct {
+	client     *ReplicaClient
+	generation string
+
+	ch     chan litestream.SnapshotInfo
+	g      errgroup.Group
+	ctx    context.Context
+	cancel func()
+
+	info litestream.SnapshotInfo
+	err  error
+}
+
+func newSnapshotIterator(ctx context.Context, generation string, client *ReplicaClient) *snapshotIterator {
+	itr := &snapshotIterator{
+		client:     client,
+		generation: generation,
+		ch:         make(chan litestream.SnapshotInfo),
+	}
+
+	itr.ctx, itr.cancel = context.WithCancel(ctx)
+	itr.g.Go(itr.fetch)
+
+	return itr
+}
+
+// fetch runs in a separate goroutine to fetch pages of objects and stream them to a channel.
+func (itr *snapshotIterator) fetch() error {
+	defer close(itr.ch)
+
+	if itr.generation == "" {
+		return fmt.Errorf("generation required")
+	}
+
+	prefix := path.Join(itr.client.Path, "generations", itr.generation) + "/"
+
+	var marker azblob.Marker
+	for marker.NotDone() {
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "LIST").Inc()
+
+		resp, err := itr.client.containerURL.ListBlobsFlatSegment(itr.ctx, marker, azblob.ListBlobsSegmentOptions{Prefix: prefix})
+		if err != nil {
+			return err
+		}
+		marker = resp.NextMarker
+
+		for _, item := range resp.Segment.BlobItems {
+			index, err := internal.ParseSnapshotPath(path.Base(item.Name))
+			if err != nil {
+				continue
+			}
+
+			info := litestream.SnapshotInfo{
+				Generation: itr.generation,
+				Index:      index,
+				Size:       *item.Properties.ContentLength,
+				CreatedAt:  item.Properties.CreationTime.UTC(),
+			}
+
+			select {
+			case <-itr.ctx.Done():
+			case itr.ch <- info:
+			}
+		}
+	}
+	return nil
+}
+
+func (itr *snapshotIterator) Close() (err error) {
+	err = itr.err
+
+	// Cancel context and wait for error group to finish.
+	itr.cancel()
+	if e := itr.g.Wait(); e != nil && err == nil {
+		err = e
+	}
+
+	return err
+}
+
+func (itr *snapshotIterator) Next() bool {
+	// Exit if an error has already occurred.
+	if itr.err != nil {
+		return false
+	}
+
+	// Return false if context was canceled or if there are no more snapshots.
+	// Otherwise fetch the next snapshot and store it on the iterator.
+	select {
+	case <-itr.ctx.Done():
+		return false
+	case info, ok := <-itr.ch:
+		if !ok {
+			return false
+		}
+		itr.info = info
+		return true
+	}
+}
+
+func (itr *snapshotIterator) Err() error { return itr.err }
+
+func (itr *snapshotIterator) Snapshot() litestream.SnapshotInfo {
+	return itr.info
+}
+
+type walSegmentIterator struct {
+	client     *ReplicaClient
+	generation string
+
+	ch     chan litestream.WALSegmentInfo
+	g      errgroup.Group
+	ctx    context.Context
+	cancel func()
+
+	info litestream.WALSegmentInfo
+	err  error
+}
+
+func newWALSegmentIterator(ctx context.Context, generation string, client *ReplicaClient) *walSegmentIterator {
+	itr := &walSegmentIterator{
+		client:     client,
+		generation: generation,
+		ch:         make(chan litestream.WALSegmentInfo),
+	}
+
+	itr.ctx, itr.cancel = context.WithCancel(ctx)
+	itr.g.Go(itr.fetch)
+
+	return itr
+}
+
+// fetch runs in a separate goroutine to fetch pages of objects and stream them to a channel.
+func (itr *walSegmentIterator) fetch() error {
+	defer close(itr.ch)
+
+	if itr.generation == "" {
+		return fmt.Errorf("generation required")
+	}
+	prefix := path.Join(itr.client.Path, "generations", itr.generation, "wal")
+
+	var marker azblob.Marker
+	for marker.NotDone() {
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "LIST").Inc()
+
+		resp, err := itr.client.containerURL.ListBlobsFlatSegment(itr.ctx, marker, azblob.ListBlobsSegmentOptions{Prefix: prefix})
+		if err != nil {
+			return err
+		}
+		marker = resp.NextMarker
+
+		for _, item := range resp.Segment.BlobItems {
+			key := strings.TrimPrefix(item.Name, prefix+"/")
+			index, offset, err := internal.ParseWALSegmentPath(key)
+			if err != nil {
+				continue
+			}
+
+			info := litestream.WALSegmentInfo{
+				Generation: itr.generation,
+				Index:      index,
+				Offset:     offset,
+				Size:       *item.Properties.ContentLength,
+				CreatedAt:  item.Properties.CreationTime.UTC(),
+			}
+
+			select {
+			case <-itr.ctx.Done():
+			case itr.ch <- info:
+			}
+		}
+	}
+	return nil
+}
+
+func (itr *walSegmentIterator) Close() (err error) {
+	err = itr.err
+
+	// Cancel context and wait for error group to finish.
+	itr.cancel()
+	if e := itr.g.Wait(); e != nil && err == nil {
+		err = e
+	}
+
+	return err
+}
+
+func (itr *walSegmentIterator) Next() bool {
+	// Exit if an error has already occurred.
+	if itr.err != nil {
+		return false
+	}
+
+	// Return false if context was canceled or if there are no more segments.
+	// Otherwise fetch the next segment and store it on the iterator.
+	select {
+	case <-itr.ctx.Done():
+		return false
+	case info, ok := <-itr.ch:
+		if !ok {
+			return false
+		}
+		itr.info = info
+		return true
+	}
+}
+
+func (itr *walSegmentIterator) Err() error { return itr.err }
+
+func (itr *walSegmentIterator) WALSegment() litestream.WALSegmentInfo {
+	return itr.info
+}
+
+func isNotExists(err error) bool {
+	switch err := err.(type) {
+	case azblob.StorageError:
+		return err.ServiceCode() == azblob.ServiceCodeBlobNotFound
+	default:
+		return false
+	}
+}

cmd/litestream/generations.go

@@ -30,8 +30,8 @@ func (c *GenerationsCommand) Run(ctx context.Context, args []string) (err error)
 	}
 
 	var db *litestream.DB
-	var r litestream.Replica
+	var r *litestream.Replica
-	updatedAt := time.Now()
+	dbUpdatedAt := time.Now()
 	if isURL(fs.Arg(0)) {
 		if *configPath != "" {
 			return fmt.Errorf("cannot specify a replica URL and the -config flag")
@@ -67,14 +67,14 @@ func (c *GenerationsCommand) Run(ctx context.Context, args []string) (err error)
 		}
 
 		// Determine last time database or WAL was updated.
-		if updatedAt, err = db.UpdatedAt(); err != nil {
+		if dbUpdatedAt, err = db.UpdatedAt(); err != nil {
 			return err
 		}
 	}
 
-	var replicas []litestream.Replica
+	var replicas []*litestream.Replica
 	if r != nil {
-		replicas = []litestream.Replica{r}
+		replicas = []*litestream.Replica{r}
 	} else {
 		replicas = db.Replicas
 	}
@@ -85,7 +85,7 @@ func (c *GenerationsCommand) Run(ctx context.Context, args []string) (err error)
 
 	fmt.Fprintln(w, "name\tgeneration\tlag\tstart\tend")
 	for _, r := range replicas {
-		generations, err := r.Generations(ctx)
+		generations, err := r.Client.Generations(ctx)
 		if err != nil {
 			log.Printf("%s: cannot list generations: %s", r.Name(), err)
 			continue
@@ -93,18 +93,18 @@ func (c *GenerationsCommand) Run(ctx context.Context, args []string) (err error)
 
 		// Iterate over each generation for the replica.
 		for _, generation := range generations {
-			stats, err := r.GenerationStats(ctx, generation)
+			createdAt, updatedAt, err := r.GenerationTimeBounds(ctx, generation)
 			if err != nil {
-				log.Printf("%s: cannot find generation stats: %s", r.Name(), err)
+				log.Printf("%s: cannot determine generation time bounds: %s", r.Name(), err)
 				continue
 			}
 
 			fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n",
 				r.Name(),
 				generation,
-				truncateDuration(updatedAt.Sub(stats.UpdatedAt)).String(),
+				truncateDuration(dbUpdatedAt.Sub(updatedAt)).String(),
-				stats.CreatedAt.Format(time.RFC3339),
+				createdAt.Format(time.RFC3339),
-				stats.UpdatedAt.Format(time.RFC3339),
+				updatedAt.Format(time.RFC3339),
 			)
 		}
 	}

cmd/litestream/main.go

@@ -9,7 +9,6 @@ import (
 	"log"
 	"net/url"
 	"os"
-	"os/signal"
 	"os/user"
 	"path"
 	"path/filepath"
@@ -19,7 +18,11 @@ import (
 	"time"
 
 	"github.com/benbjohnson/litestream"
+	"github.com/benbjohnson/litestream/abs"
+	"github.com/benbjohnson/litestream/file"
+	"github.com/benbjohnson/litestream/gcs"
 	"github.com/benbjohnson/litestream/s3"
+	"github.com/benbjohnson/litestream/sftp"
 	_ "github.com/mattn/go-sqlite3"
 	"gopkg.in/yaml.v2"
 )
@@ -83,24 +86,40 @@ func (m *Main) Run(ctx context.Context, args []string) (err error) {
 
 		// Setup signal handler.
 		ctx, cancel := context.WithCancel(ctx)
-		ch := signalChan()
+		signalCh := signalChan()
-		go func() { <-ch; cancel() }()
 
 		if err := c.Run(ctx); err != nil {
 			return err
 		}
 
 		// Wait for signal to stop program.
-		<-ctx.Done()
+		select {
-		signal.Reset()
+		case err = <-c.execCh:
+			cancel()
+			fmt.Println("subprocess exited, litestream shutting down")
+		case sig := <-signalCh:
+			cancel()
 			fmt.Println("signal received, litestream shutting down")
 
+			if c.cmd != nil {
+				fmt.Println("sending signal to exec process")
+				if err := c.cmd.Process.Signal(sig); err != nil {
+					return fmt.Errorf("cannot signal exec process: %w", err)
+				}
+
+				fmt.Println("waiting for exec process to close")
+				if err := <-c.execCh; err != nil && !strings.HasPrefix(err.Error(), "signal:") {
+					return fmt.Errorf("cannot wait for exec process: %w", err)
+				}
+			}
+		}
+
 		// Gracefully close.
-		if err := c.Close(); err != nil {
+		if e := c.Close(); e != nil && err == nil {
-			return err
+			err = e
 		}
 		fmt.Println("litestream shut down")
-		return nil
+		return err
 
 	case "restore":
 		return (&RestoreCommand{}).Run(ctx, args)
@@ -148,6 +167,10 @@ type Config struct {
 	// List of databases to manage.
 	DBs []*DBConfig `yaml:"dbs"`
 
+	// Subcommand to execute during replication.
+	// Litestream will shutdown when subcommand exits.
+	Exec string `yaml:"exec"`
+
 	// Global S3 settings
 	AccessKeyID     string `yaml:"access-key-id"`
 	SecretAccessKey string `yaml:"secret-access-key"`
@@ -276,11 +299,11 @@ type ReplicaConfig struct {
 	Name                   string         `yaml:"name"` // name of replica, optional.
 	Path                   string         `yaml:"path"`
 	URL                    string         `yaml:"url"`
-	Retention              time.Duration `yaml:"retention"`
+	Retention              *time.Duration `yaml:"retention"`
-	RetentionCheckInterval time.Duration `yaml:"retention-check-interval"`
+	RetentionCheckInterval *time.Duration `yaml:"retention-check-interval"`
-	SyncInterval           time.Duration `yaml:"sync-interval"` // s3 only
+	SyncInterval           *time.Duration `yaml:"sync-interval"`
-	SnapshotInterval       time.Duration `yaml:"snapshot-interval"`
+	SnapshotInterval       *time.Duration `yaml:"snapshot-interval"`
-	ValidationInterval     time.Duration `yaml:"validation-interval"`
+	ValidationInterval     *time.Duration `yaml:"validation-interval"`
 
 	// S3 settings
 	AccessKeyID     string `yaml:"access-key-id"`
@@ -290,27 +313,74 @@ type ReplicaConfig struct {
 	Endpoint        string `yaml:"endpoint"`
 	ForcePathStyle  *bool  `yaml:"force-path-style"`
 	SkipVerify      bool   `yaml:"skip-verify"`
+
+	// ABS settings
+	AccountName string `yaml:"account-name"`
+	AccountKey  string `yaml:"account-key"`
+
+	// SFTP settings
+	Host     string `yaml:"host"`
+	User     string `yaml:"user"`
+	Password string `yaml:"password"`
+	KeyPath  string `yaml:"key-path"`
 }
 
 // NewReplicaFromConfig instantiates a replica for a DB based on a config.
-func NewReplicaFromConfig(c *ReplicaConfig, db *litestream.DB) (litestream.Replica, error) {
+func NewReplicaFromConfig(c *ReplicaConfig, db *litestream.DB) (_ *litestream.Replica, err error) {
 	// Ensure user did not specify URL in path.
 	if isURL(c.Path) {
 		return nil, fmt.Errorf("replica path cannot be a url, please use the 'url' field instead: %s", c.Path)
 	}
 
+	// Build replica.
+	r := litestream.NewReplica(db, c.Name)
+	if v := c.Retention; v != nil {
+		r.Retention = *v
+	}
+	if v := c.RetentionCheckInterval; v != nil {
+		r.RetentionCheckInterval = *v
+	}
+	if v := c.SyncInterval; v != nil {
+		r.SyncInterval = *v
+	}
+	if v := c.SnapshotInterval; v != nil {
+		r.SnapshotInterval = *v
+	}
+	if v := c.ValidationInterval; v != nil {
+		r.ValidationInterval = *v
+	}
+
+	// Build and set client on replica.
 	switch c.ReplicaType() {
 	case "file":
-		return newFileReplicaFromConfig(c, db)
+		if r.Client, err = newFileReplicaClientFromConfig(c, r); err != nil {
+			return nil, err
+		}
 	case "s3":
-		return newS3ReplicaFromConfig(c, db)
+		if r.Client, err = newS3ReplicaClientFromConfig(c, r); err != nil {
+			return nil, err
+		}
+	case "gcs":
+		if r.Client, err = newGCSReplicaClientFromConfig(c, r); err != nil {
+			return nil, err
+		}
+	case "abs":
+		if r.Client, err = newABSReplicaClientFromConfig(c, r); err != nil {
+			return nil, err
+		}
+	case "sftp":
+		if r.Client, err = newSFTPReplicaClientFromConfig(c, r); err != nil {
+			return nil, err
+		}
 	default:
 		return nil, fmt.Errorf("unknown replica type in config: %q", c.Type)
 	}
+
+	return r, nil
 }
 
-// newFileReplicaFromConfig returns a new instance of FileReplica build from config.
+// newFileReplicaClientFromConfig returns a new instance of file.ReplicaClient built from config.
-func newFileReplicaFromConfig(c *ReplicaConfig, db *litestream.DB) (_ *litestream.FileReplica, err error) {
+func newFileReplicaClientFromConfig(c *ReplicaConfig, r *litestream.Replica) (_ *file.ReplicaClient, err error) {
 	// Ensure URL & path are not both specified.
 	if c.URL != "" && c.Path != "" {
 		return nil, fmt.Errorf("cannot specify url & path for file replica")
@@ -335,24 +405,13 @@ func newFileReplicaFromConfig(c *ReplicaConfig, db *litestream.DB) (_ *litestrea
 	}
 
 	// Instantiate replica and apply time fields, if set.
-	r := litestream.NewFileReplica(db, c.Name, path)
+	client := file.NewReplicaClient(path)
-	if v := c.Retention; v > 0 {
+	client.Replica = r
-		r.Retention = v
+	return client, nil
-	}
-	if v := c.RetentionCheckInterval; v > 0 {
-		r.RetentionCheckInterval = v
-	}
-	if v := c.SnapshotInterval; v > 0 {
-		r.SnapshotInterval = v
-	}
-	if v := c.ValidationInterval; v > 0 {
-		r.ValidationInterval = v
-	}
-	return r, nil
 }
 
-// newS3ReplicaFromConfig returns a new instance of S3Replica build from config.
+// newS3ReplicaClientFromConfig returns a new instance of s3.ReplicaClient built from config.
-func newS3ReplicaFromConfig(c *ReplicaConfig, db *litestream.DB) (_ *s3.Replica, err error) {
+func newS3ReplicaClientFromConfig(c *ReplicaConfig, r *litestream.Replica) (_ *s3.ReplicaClient, err error) {
 	// Ensure URL & constituent parts are not both specified.
 	if c.URL != "" && c.Path != "" {
 		return nil, fmt.Errorf("cannot specify url & path for s3 replica")
@@ -402,32 +461,148 @@ func newS3ReplicaFromConfig(c *ReplicaConfig, db *litestream.DB) (_ *s3.Replica,
 	}
 
 	// Build replica.
-	r := s3.NewReplica(db, c.Name)
+	client := s3.NewReplicaClient()
-	r.AccessKeyID = c.AccessKeyID
+	client.AccessKeyID = c.AccessKeyID
-	r.SecretAccessKey = c.SecretAccessKey
+	client.SecretAccessKey = c.SecretAccessKey
-	r.Bucket = bucket
+	client.Bucket = bucket
-	r.Path = path
+	client.Path = path
-	r.Region = region
+	client.Region = region
-	r.Endpoint = endpoint
+	client.Endpoint = endpoint
-	r.ForcePathStyle = forcePathStyle
+	client.ForcePathStyle = forcePathStyle
-	r.SkipVerify = skipVerify
+	client.SkipVerify = skipVerify
+	return client, nil
+}
 
-	if v := c.Retention; v > 0 {
+// newGCSReplicaClientFromConfig returns a new instance of gcs.ReplicaClient built from config.
-		r.Retention = v
+func newGCSReplicaClientFromConfig(c *ReplicaConfig, r *litestream.Replica) (_ *gcs.ReplicaClient, err error) {
+	// Ensure URL & constituent parts are not both specified.
+	if c.URL != "" && c.Path != "" {
+		return nil, fmt.Errorf("cannot specify url & path for gcs replica")
+	} else if c.URL != "" && c.Bucket != "" {
+		return nil, fmt.Errorf("cannot specify url & bucket for gcs replica")
 	}
-	if v := c.RetentionCheckInterval; v > 0 {
+
-		r.RetentionCheckInterval = v
+	bucket, path := c.Bucket, c.Path
+
+	// Apply settings from URL, if specified.
+	if c.URL != "" {
+		_, uhost, upath, err := ParseReplicaURL(c.URL)
+		if err != nil {
+			return nil, err
 		}
-	if v := c.SyncInterval; v > 0 {
+
-		r.SyncInterval = v
+		// Only apply URL parts to field that have not been overridden.
+		if path == "" {
+			path = upath
 		}
-	if v := c.SnapshotInterval; v > 0 {
+		if bucket == "" {
-		r.SnapshotInterval = v
+			bucket = uhost
 		}
-	if v := c.ValidationInterval; v > 0 {
-		r.ValidationInterval = v
 	}
-	return r, nil
+
+	// Ensure required settings are set.
+	if bucket == "" {
+		return nil, fmt.Errorf("bucket required for gcs replica")
+	}
+
+	// Build replica.
+	client := gcs.NewReplicaClient()
+	client.Bucket = bucket
+	client.Path = path
+	return client, nil
+}
+
+// newABSReplicaClientFromConfig returns a new instance of abs.ReplicaClient built from config.
+func newABSReplicaClientFromConfig(c *ReplicaConfig, r *litestream.Replica) (_ *abs.ReplicaClient, err error) {
+	// Ensure URL & constituent parts are not both specified.
+	if c.URL != "" && c.Path != "" {
+		return nil, fmt.Errorf("cannot specify url & path for abs replica")
+	} else if c.URL != "" && c.Bucket != "" {
+		return nil, fmt.Errorf("cannot specify url & bucket for abs replica")
+	}
+
+	// Build replica.
+	client := abs.NewReplicaClient()
+	client.AccountName = c.AccountName
+	client.AccountKey = c.AccountKey
+	client.Bucket = c.Bucket
+	client.Path = c.Path
+	client.Endpoint = c.Endpoint
+
+	// Apply settings from URL, if specified.
+	if c.URL != "" {
+		u, err := url.Parse(c.URL)
+		if err != nil {
+			return nil, err
+		}
+
+		if client.AccountName == "" && u.User != nil {
+			client.AccountName = u.User.Username()
+		}
+		if client.Bucket == "" {
+			client.Bucket = u.Host
+		}
+		if client.Path == "" {
+			client.Path = strings.TrimPrefix(path.Clean(u.Path), "/")
+		}
+	}
+
+	// Ensure required settings are set.
+	if client.Bucket == "" {
+		return nil, fmt.Errorf("bucket required for abs replica")
+	}
+
+	return client, nil
+}
+
+// newSFTPReplicaClientFromConfig returns a new instance of sftp.ReplicaClient built from config.
+func newSFTPReplicaClientFromConfig(c *ReplicaConfig, r *litestream.Replica) (_ *sftp.ReplicaClient, err error) {
+	// Ensure URL & constituent parts are not both specified.
+	if c.URL != "" && c.Path != "" {
+		return nil, fmt.Errorf("cannot specify url & path for sftp replica")
+	} else if c.URL != "" && c.Host != "" {
+		return nil, fmt.Errorf("cannot specify url & host for sftp replica")
+	}
+
+	host, user, password, path := c.Host, c.User, c.Password, c.Path
+
+	// Apply settings from URL, if specified.
+	if c.URL != "" {
+		u, err := url.Parse(c.URL)
+		if err != nil {
+			return nil, err
+		}
+
+		// Only apply URL parts to field that have not been overridden.
+		if host == "" {
+			host = u.Host
+		}
+		if user == "" && u.User != nil {
+			user = u.User.Username()
+		}
+		if password == "" && u.User != nil {
+			password, _ = u.User.Password()
+		}
+		if path == "" {
+			path = u.Path
+		}
+	}
+
+	// Ensure required settings are set.
+	if host == "" {
+		return nil, fmt.Errorf("host required for sftp replica")
+	} else if user == "" {
+		return nil, fmt.Errorf("user required for sftp replica")
+	}
+
+	// Build replica.
+	client := sftp.NewReplicaClient()
+	client.Host = host
+	client.User = user
+	client.Password = password
+	client.Path = path
+	client.KeyPath = c.KeyPath
+	return client, nil
 }
 
 // applyLitestreamEnv copies "LITESTREAM" prefixed environment variables to

cmd/litestream/main_notwindows.go

@@ -20,7 +20,7 @@ func runWindowsService(ctx context.Context) error {
 }
 
 func signalChan() <-chan os.Signal {
-	ch := make(chan os.Signal, 1)
+	ch := make(chan os.Signal, 2)
 	signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM)
 	return ch
 }

cmd/litestream/main_test.go

@@ -6,8 +6,9 @@ import (
 	"path/filepath"
 	"testing"
 
-	"github.com/benbjohnson/litestream"
 	main "github.com/benbjohnson/litestream/cmd/litestream"
+	"github.com/benbjohnson/litestream/file"
+	"github.com/benbjohnson/litestream/gcs"
 	"github.com/benbjohnson/litestream/s3"
 )
 
@@ -96,9 +97,9 @@ func TestNewFileReplicaFromConfig(t *testing.T) {
 	r, err := main.NewReplicaFromConfig(&main.ReplicaConfig{Path: "/foo"}, nil)
 	if err != nil {
 		t.Fatal(err)
-	} else if r, ok := r.(*litestream.FileReplica); !ok {
+	} else if client, ok := r.Client.(*file.ReplicaClient); !ok {
 		t.Fatal("unexpected replica type")
-	} else if got, want := r.Path(), "/foo"; got != want {
+	} else if got, want := client.Path(), "/foo"; got != want {
 		t.Fatalf("Path=%s, want %s", got, want)
 	}
 }
@@ -108,17 +109,17 @@ func TestNewS3ReplicaFromConfig(t *testing.T) {
 		r, err := main.NewReplicaFromConfig(&main.ReplicaConfig{URL: "s3://foo/bar"}, nil)
 		if err != nil {
 			t.Fatal(err)
-		} else if r, ok := r.(*s3.Replica); !ok {
+		} else if client, ok := r.Client.(*s3.ReplicaClient); !ok {
 			t.Fatal("unexpected replica type")
-		} else if got, want := r.Bucket, "foo"; got != want {
+		} else if got, want := client.Bucket, "foo"; got != want {
 			t.Fatalf("Bucket=%s, want %s", got, want)
-		} else if got, want := r.Path, "bar"; got != want {
+		} else if got, want := client.Path, "bar"; got != want {
 			t.Fatalf("Path=%s, want %s", got, want)
-		} else if got, want := r.Region, ""; got != want {
+		} else if got, want := client.Region, ""; got != want {
 			t.Fatalf("Region=%s, want %s", got, want)
-		} else if got, want := r.Endpoint, ""; got != want {
+		} else if got, want := client.Endpoint, ""; got != want {
 			t.Fatalf("Endpoint=%s, want %s", got, want)
-		} else if got, want := r.ForcePathStyle, false; got != want {
+		} else if got, want := client.ForcePathStyle, false; got != want {
 			t.Fatalf("ForcePathStyle=%v, want %v", got, want)
 		}
 	})
@@ -127,17 +128,17 @@ func TestNewS3ReplicaFromConfig(t *testing.T) {
 		r, err := main.NewReplicaFromConfig(&main.ReplicaConfig{URL: "s3://foo.localhost:9000/bar"}, nil)
 		if err != nil {
 			t.Fatal(err)
-		} else if r, ok := r.(*s3.Replica); !ok {
+		} else if client, ok := r.Client.(*s3.ReplicaClient); !ok {
 			t.Fatal("unexpected replica type")
-		} else if got, want := r.Bucket, "foo"; got != want {
+		} else if got, want := client.Bucket, "foo"; got != want {
 			t.Fatalf("Bucket=%s, want %s", got, want)
-		} else if got, want := r.Path, "bar"; got != want {
+		} else if got, want := client.Path, "bar"; got != want {
 			t.Fatalf("Path=%s, want %s", got, want)
-		} else if got, want := r.Region, "us-east-1"; got != want {
+		} else if got, want := client.Region, "us-east-1"; got != want {
 			t.Fatalf("Region=%s, want %s", got, want)
-		} else if got, want := r.Endpoint, "http://localhost:9000"; got != want {
+		} else if got, want := client.Endpoint, "http://localhost:9000"; got != want {
 			t.Fatalf("Endpoint=%s, want %s", got, want)
-		} else if got, want := r.ForcePathStyle, true; got != want {
+		} else if got, want := client.ForcePathStyle, true; got != want {
 			t.Fatalf("ForcePathStyle=%v, want %v", got, want)
 		}
 	})
@@ -146,37 +147,31 @@ func TestNewS3ReplicaFromConfig(t *testing.T) {
 		r, err := main.NewReplicaFromConfig(&main.ReplicaConfig{URL: "s3://foo.s3.us-west-000.backblazeb2.com/bar"}, nil)
 		if err != nil {
 			t.Fatal(err)
-		} else if r, ok := r.(*s3.Replica); !ok {
+		} else if client, ok := r.Client.(*s3.ReplicaClient); !ok {
 			t.Fatal("unexpected replica type")
-		} else if got, want := r.Bucket, "foo"; got != want {
+		} else if got, want := client.Bucket, "foo"; got != want {
 			t.Fatalf("Bucket=%s, want %s", got, want)
-		} else if got, want := r.Path, "bar"; got != want {
+		} else if got, want := client.Path, "bar"; got != want {
 			t.Fatalf("Path=%s, want %s", got, want)
-		} else if got, want := r.Region, "us-west-000"; got != want {
+		} else if got, want := client.Region, "us-west-000"; got != want {
 			t.Fatalf("Region=%s, want %s", got, want)
-		} else if got, want := r.Endpoint, "https://s3.us-west-000.backblazeb2.com"; got != want {
+		} else if got, want := client.Endpoint, "https://s3.us-west-000.backblazeb2.com"; got != want {
 			t.Fatalf("Endpoint=%s, want %s", got, want)
-		} else if got, want := r.ForcePathStyle, true; got != want {
+		} else if got, want := client.ForcePathStyle, true; got != want {
 			t.Fatalf("ForcePathStyle=%v, want %v", got, want)
 		}
 	})
+}
 
-	t.Run("GCS", func(t *testing.T) {
+func TestNewGCSReplicaFromConfig(t *testing.T) {
-		r, err := main.NewReplicaFromConfig(&main.ReplicaConfig{URL: "s3://foo.storage.googleapis.com/bar"}, nil)
+	r, err := main.NewReplicaFromConfig(&main.ReplicaConfig{URL: "gcs://foo/bar"}, nil)
 	if err != nil {
 		t.Fatal(err)
-		} else if r, ok := r.(*s3.Replica); !ok {
+	} else if client, ok := r.Client.(*gcs.ReplicaClient); !ok {
 		t.Fatal("unexpected replica type")
-		} else if got, want := r.Bucket, "foo"; got != want {
+	} else if got, want := client.Bucket, "foo"; got != want {
 		t.Fatalf("Bucket=%s, want %s", got, want)
-		} else if got, want := r.Path, "bar"; got != want {
+	} else if got, want := client.Path, "bar"; got != want {
 		t.Fatalf("Path=%s, want %s", got, want)
-		} else if got, want := r.Region, "us-east-1"; got != want {
-			t.Fatalf("Region=%s, want %s", got, want)
-		} else if got, want := r.Endpoint, "https://storage.googleapis.com"; got != want {
-			t.Fatalf("Endpoint=%s, want %s", got, want)
-		} else if got, want := r.ForcePathStyle, true; got != want {
-			t.Fatalf("ForcePathStyle=%v, want %v", got, want)
 	}
-	})
 }

cmd/litestream/main_windows.go

@@ -62,7 +62,7 @@ func (s *windowsService) Execute(args []string, r <-chan svc.ChangeRequest, stat
 
 	// Instantiate replication command and load configuration.
 	c := NewReplicateCommand()
-	if c.Config, err = ReadConfigFile(DefaultConfigPath()); err != nil {
+	if c.Config, err = ReadConfigFile(DefaultConfigPath(), true); err != nil {
 		log.Printf("cannot load configuration: %s", err)
 		return true, 1
 	}

cmd/litestream/replicate.go

@@ -9,15 +9,23 @@ import (
 	"net/http"
 	_ "net/http/pprof"
 	"os"
-	"time"
+	"os/exec"
 
 	"github.com/benbjohnson/litestream"
+	"github.com/benbjohnson/litestream/abs"
+	"github.com/benbjohnson/litestream/file"
+	"github.com/benbjohnson/litestream/gcs"
 	"github.com/benbjohnson/litestream/s3"
+	"github.com/benbjohnson/litestream/sftp"
+	"github.com/mattn/go-shellwords"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
 // ReplicateCommand represents a command that continuously replicates SQLite databases.
 type ReplicateCommand struct {
+	cmd    *exec.Cmd  // subcommand
+	execCh chan error // subcommand error channel
+
 	Config Config
 
 	// List of managed databases specified in the config.
@@ -25,12 +33,15 @@ type ReplicateCommand struct {
 }
 
 func NewReplicateCommand() *ReplicateCommand {
-	return &ReplicateCommand{}
+	return &ReplicateCommand{
+		execCh: make(chan error),
+	}
 }
 
 // ParseFlags parses the CLI flags and loads the configuration file.
 func (c *ReplicateCommand) ParseFlags(ctx context.Context, args []string) (err error) {
 	fs := flag.NewFlagSet("litestream-replicate", flag.ContinueOnError)
+	execFlag := fs.String("exec", "", "execute subcommand")
 	tracePath := fs.String("trace", "", "trace path")
 	configPath, noExpandEnv := registerConfigFlag(fs)
 	fs.Usage = c.Usage
@@ -48,9 +59,10 @@ func (c *ReplicateCommand) ParseFlags(ctx context.Context, args []string) (err e
 
 		dbConfig := &DBConfig{Path: fs.Arg(0)}
 		for _, u := range fs.Args()[1:] {
+			syncInterval := litestream.DefaultSyncInterval
 			dbConfig.Replicas = append(dbConfig.Replicas, &ReplicaConfig{
 				URL:          u,
-				SyncInterval: 1 * time.Second,
+				SyncInterval: &syncInterval,
 			})
 		}
 		c.Config.DBs = []*DBConfig{dbConfig}
@@ -63,6 +75,11 @@ func (c *ReplicateCommand) ParseFlags(ctx context.Context, args []string) (err e
 		}
 	}
 
+	// Override config exec command, if specified.
+	if *execFlag != "" {
+		c.Config.Exec = *execFlag
+	}
+
 	// Enable trace logging.
 	if *tracePath != "" {
 		f, err := os.Create(*tracePath)
@@ -81,6 +98,7 @@ func (c *ReplicateCommand) Run(ctx context.Context) (err error) {
 	// Display version information.
 	log.Printf("litestream %s", Version)
 
+	// Setup databases.
 	if len(c.Config.DBs) == 0 {
 		log.Println("no databases specified in configuration")
 	}
@@ -102,13 +120,19 @@ func (c *ReplicateCommand) Run(ctx context.Context) (err error) {
 	for _, db := range c.DBs {
 		log.Printf("initialized db: %s", db.Path())
 		for _, r := range db.Replicas {
-			switch r := r.(type) {
+			switch client := r.Client.(type) {
-			case *litestream.FileReplica:
+			case *file.ReplicaClient:
-				log.Printf("replicating to: name=%q type=%q path=%q", r.Name(), r.Type(), r.Path())
+				log.Printf("replicating to: name=%q type=%q path=%q", r.Name(), client.Type(), client.Path())
-			case *s3.Replica:
+			case *s3.ReplicaClient:
-				log.Printf("replicating to: name=%q type=%q bucket=%q path=%q region=%q endpoint=%q sync-interval=%s", r.Name(), r.Type(), r.Bucket, r.Path, r.Region, r.Endpoint, r.SyncInterval)
+				log.Printf("replicating to: name=%q type=%q bucket=%q path=%q region=%q endpoint=%q sync-interval=%s", r.Name(), client.Type(), client.Bucket, client.Path, client.Region, client.Endpoint, r.SyncInterval)
+			case *gcs.ReplicaClient:
+				log.Printf("replicating to: name=%q type=%q bucket=%q path=%q sync-interval=%s", r.Name(), client.Type(), client.Bucket, client.Path, r.SyncInterval)
+			case *abs.ReplicaClient:
+				log.Printf("replicating to: name=%q type=%q bucket=%q path=%q endpoint=%q sync-interval=%s", r.Name(), client.Type(), client.Bucket, client.Path, client.Endpoint, r.SyncInterval)
+			case *sftp.ReplicaClient:
+				log.Printf("replicating to: name=%q type=%q host=%q user=%q path=%q sync-interval=%s", r.Name(), client.Type(), client.Host, client.User, client.Path, r.SyncInterval)
 			default:
-				log.Printf("replicating to: name=%q type=%q", r.Name(), r.Type())
+				log.Printf("replicating to: name=%q type=%q", r.Name(), client.Type())
 			}
 		}
 	}
@@ -131,6 +155,23 @@ func (c *ReplicateCommand) Run(ctx context.Context) (err error) {
 		}()
 	}
 
+	// Parse exec commands args & start subprocess.
+	if c.Config.Exec != "" {
+		execArgs, err := shellwords.Parse(c.Config.Exec)
+		if err != nil {
+			return fmt.Errorf("cannot parse exec command: %w", err)
+		}
+
+		c.cmd = exec.CommandContext(ctx, execArgs[0], execArgs[1:]...)
+		c.cmd.Env = os.Environ()
+		c.cmd.Stdout = os.Stdout
+		c.cmd.Stderr = os.Stderr
+		if err := c.cmd.Start(); err != nil {
+			return fmt.Errorf("cannot start exec command: %w", err)
+		}
+		go func() { c.execCh <- c.cmd.Wait() }()
+	}
+
 	return nil
 }
 
@@ -144,7 +185,6 @@ func (c *ReplicateCommand) Close() (err error) {
 			}
 		}
 	}
-	// TODO(windows): Clear DBs
 	return err
 }
 
@@ -168,6 +208,10 @@ Arguments:
 	    Specifies the configuration file.
 	    Defaults to %s
 
+	-exec CMD
+	    Executes a subcommand. Litestream will exit when the child
+	    process exits. Useful for simple process management.
+
 	-no-expand-env
 	    Disables environment variable expansion in configuration file.
 

cmd/litestream/restore.go

@@ -28,6 +28,7 @@ func (c *RestoreCommand) Run(ctx context.Context, args []string) (err error) {
 	fs.StringVar(&opt.Generation, "generation", "", "generation name")
 	fs.Var((*indexVar)(&opt.Index), "index", "wal index")
 	fs.IntVar(&opt.Parallelism, "parallelism", opt.Parallelism, "parallelism")
+	ifDBNotExists := fs.Bool("if-db-not-exists", false, "")
 	ifReplicaExists := fs.Bool("if-replica-exists", false, "")
 	timestampStr := fs.String("timestamp", "", "timestamp")
 	verbose := fs.Bool("v", false, "verbose output")
@@ -53,19 +54,25 @@ func (c *RestoreCommand) Run(ctx context.Context, args []string) (err error) {
 	}
 
 	// Determine replica & generation to restore from.
-	var r litestream.Replica
+	var r *litestream.Replica
 	if isURL(fs.Arg(0)) {
 		if *configPath != "" {
 			return fmt.Errorf("cannot specify a replica URL and the -config flag")
 		}
-		if r, err = c.loadFromURL(ctx, fs.Arg(0), &opt); err != nil {
+		if r, err = c.loadFromURL(ctx, fs.Arg(0), *ifDBNotExists, &opt); err == errSkipDBExists {
+			fmt.Println("database already exists, skipping")
+			return nil
+		} else if err != nil {
 			return err
 		}
 	} else {
 		if *configPath == "" {
 			*configPath = DefaultConfigPath()
 		}
-		if r, err = c.loadFromConfig(ctx, fs.Arg(0), *configPath, !*noExpandEnv, &opt); err != nil {
+		if r, err = c.loadFromConfig(ctx, fs.Arg(0), *configPath, !*noExpandEnv, *ifDBNotExists, &opt); err == errSkipDBExists {
+			fmt.Println("database already exists, skipping")
+			return nil
+		} else if err != nil {
 			return err
 		}
 	}
@@ -80,21 +87,34 @@ func (c *RestoreCommand) Run(ctx context.Context, args []string) (err error) {
 		return fmt.Errorf("no matching backups found")
 	}
 
-	return litestream.RestoreReplica(ctx, r, opt)
+	return r.Restore(ctx, opt)
 }
 
 // loadFromURL creates a replica & updates the restore options from a replica URL.
-func (c *RestoreCommand) loadFromURL(ctx context.Context, replicaURL string, opt *litestream.RestoreOptions) (litestream.Replica, error) {
+func (c *RestoreCommand) loadFromURL(ctx context.Context, replicaURL string, ifDBNotExists bool, opt *litestream.RestoreOptions) (*litestream.Replica, error) {
-	r, err := NewReplicaFromConfig(&ReplicaConfig{URL: replicaURL}, nil)
+	if opt.OutputPath == "" {
+		return nil, fmt.Errorf("output path required")
+	}
+
+	// Exit successfully if the output file already exists.
+	if _, err := os.Stat(opt.OutputPath); !os.IsNotExist(err) && ifDBNotExists {
+		return nil, errSkipDBExists
+	}
+
+	syncInterval := litestream.DefaultSyncInterval
+	r, err := NewReplicaFromConfig(&ReplicaConfig{
+		URL:          replicaURL,
+		SyncInterval: &syncInterval,
+	}, nil)
 	if err != nil {
 		return nil, err
 	}
-	opt.Generation, _, err = litestream.CalcReplicaRestoreTarget(ctx, r, *opt)
+	opt.Generation, _, err = r.CalcRestoreTarget(ctx, *opt)
 	return r, err
 }
 
 // loadFromConfig returns a replica & updates the restore options from a DB reference.
-func (c *RestoreCommand) loadFromConfig(ctx context.Context, dbPath, configPath string, expandEnv bool, opt *litestream.RestoreOptions) (litestream.Replica, error) {
+func (c *RestoreCommand) loadFromConfig(ctx context.Context, dbPath, configPath string, expandEnv, ifDBNotExists bool, opt *litestream.RestoreOptions) (*litestream.Replica, error) {
 	// Load configuration.
 	config, err := ReadConfigFile(configPath, expandEnv)
 	if err != nil {
@@ -119,6 +139,11 @@ func (c *RestoreCommand) loadFromConfig(ctx context.Context, dbPath, configPath
 		opt.OutputPath = dbPath
 	}
 
+	// Exit successfully if the output file already exists.
+	if _, err := os.Stat(opt.OutputPath); !os.IsNotExist(err) && ifDBNotExists {
+		return nil, errSkipDBExists
+	}
+
 	// Determine the appropriate replica & generation to restore from,
 	r, generation, err := db.CalcRestoreTarget(ctx, *opt)
 	if err != nil {
@@ -169,6 +194,9 @@ Arguments:
 	    Output path of the restored database.
 	    Defaults to original DB path.
 
+	-if-db-not-exists
+	    Returns exit code of 0 if the database already exists.
+
 	-if-replica-exists
 	    Returns exit code of 0 if no backups found.
 
@@ -201,3 +229,5 @@ Examples:
 		DefaultConfigPath(),
 	)
 }
+
+var errSkipDBExists = errors.New("database already exists, skipping")

cmd/litestream/snapshots.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"log"
 	"os"
 	"text/tabwriter"
 	"time"
@@ -29,7 +30,7 @@ func (c *SnapshotsCommand) Run(ctx context.Context, args []string) (err error) {
 	}
 
 	var db *litestream.DB
-	var r litestream.Replica
+	var r *litestream.Replica
 	if isURL(fs.Arg(0)) {
 		if *configPath != "" {
 			return fmt.Errorf("cannot specify a replica URL and the -config flag")
@@ -66,15 +67,11 @@ func (c *SnapshotsCommand) Run(ctx context.Context, args []string) (err error) {
 	}
 
 	// Find snapshots by db or replica.
-	var infos []*litestream.SnapshotInfo
+	var replicas []*litestream.Replica
 	if r != nil {
-		if infos, err = r.Snapshots(ctx); err != nil {
+		replicas = []*litestream.Replica{r}
-			return err
-		}
 	} else {
-		if infos, err = db.Snapshots(ctx); err != nil {
+		replicas = db.Replicas
-			return err
-		}
 	}
 
 	// List all snapshots.
@@ -82,15 +79,22 @@ func (c *SnapshotsCommand) Run(ctx context.Context, args []string) (err error) {
 	defer w.Flush()
 
 	fmt.Fprintln(w, "replica\tgeneration\tindex\tsize\tcreated")
+	for _, r := range replicas {
+		infos, err := r.Snapshots(ctx)
+		if err != nil {
+			log.Printf("cannot determine snapshots: %s", err)
+			continue
+		}
 		for _, info := range infos {
 			fmt.Fprintf(w, "%s\t%s\t%d\t%d\t%s\n",
-			info.Replica,
+				r.Name(),
 				info.Generation,
 				info.Index,
 				info.Size,
 				info.CreatedAt.Format(time.RFC3339),
 			)
 		}
+	}
 
 	return nil
 }

cmd/litestream/wal.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"log"
 	"os"
 	"text/tabwriter"
 	"time"
@@ -30,7 +31,7 @@ func (c *WALCommand) Run(ctx context.Context, args []string) (err error) {
 	}
 
 	var db *litestream.DB
-	var r litestream.Replica
+	var r *litestream.Replica
 	if isURL(fs.Arg(0)) {
 		if *configPath != "" {
 			return fmt.Errorf("cannot specify a replica URL and the -config flag")
@@ -67,15 +68,11 @@ func (c *WALCommand) Run(ctx context.Context, args []string) (err error) {
 	}
 
 	// Find WAL files by db or replica.
-	var infos []*litestream.WALInfo
+	var replicas []*litestream.Replica
 	if r != nil {
-		if infos, err = r.WALs(ctx); err != nil {
+		replicas = []*litestream.Replica{r}
-			return err
-		}
 	} else {
-		if infos, err = db.WALs(ctx); err != nil {
+		replicas = db.Replicas
-			return err
-		}
 	}
 
 	// List all WAL files.
@@ -83,13 +80,30 @@ func (c *WALCommand) Run(ctx context.Context, args []string) (err error) {
 	defer w.Flush()
 
 	fmt.Fprintln(w, "replica\tgeneration\tindex\toffset\tsize\tcreated")
-	for _, info := range infos {
+	for _, r := range replicas {
-		if *generation != "" && info.Generation != *generation {
+		var generations []string
+		if *generation != "" {
+			generations = []string{*generation}
+		} else {
+			if generations, err = r.Client.Generations(ctx); err != nil {
+				log.Printf("%s: cannot determine generations: %s", r.Name(), err)
 				continue
 			}
+		}
+
+		for _, generation := range generations {
+			if err := func() error {
+				itr, err := r.Client.WALSegments(ctx, generation)
+				if err != nil {
+					return err
+				}
+				defer itr.Close()
+
+				for itr.Next() {
+					info := itr.WALSegment()
 
 					fmt.Fprintf(w, "%s\t%s\t%d\t%d\t%d\t%s\n",
-			info.Replica,
+						r.Name(),
 						info.Generation,
 						info.Index,
 						info.Offset,
@@ -97,6 +111,13 @@ func (c *WALCommand) Run(ctx context.Context, args []string) (err error) {
 						info.CreatedAt.Format(time.RFC3339),
 					)
 				}
+				return itr.Close()
+			}(); err != nil {
+				log.Printf("%s: cannot fetch wal segments: %s", r.Name(), err)
+				continue
+			}
+		}
+	}
 
 	return nil
 }
@@ -104,7 +125,7 @@ func (c *WALCommand) Run(ctx context.Context, args []string) (err error) {
 // Usage prints the help screen to STDOUT.
 func (c *WALCommand) Usage() {
 	fmt.Printf(`
-The wal command lists all wal files available for a database.
+The wal command lists all wal segments available for a database.
 
 Usage:
 
@@ -129,13 +150,13 @@ Arguments:
 
 Examples:
 
-	# List all WAL files for a database.
+	# List all WAL segments for a database.
 	$ litestream wal /path/to/db
 
-	# List all WAL files on S3 for a specific generation.
+	# List all WAL segments on S3 for a specific generation.
 	$ litestream wal -replica s3 -generation xxxxxxxx /path/to/db
 
-	# List all WAL files for replica URL.
+	# List all WAL segments for replica URL.
 	$ litestream wal s3://mybkt/db
 
 `[1:],

db.go

@@ -16,14 +16,15 @@ import (
 	"math/rand"
 	"os"
 	"path/filepath"
-	"sort"
+	"regexp"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
 
+	"github.com/benbjohnson/litestream/internal"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
-	"golang.org/x/sync/errgroup"
 )
 
 // Default DB settings.
@@ -51,10 +52,8 @@ type DB struct {
 	pageSize int           // page size, in bytes
 	notify   chan struct{} // closes on WAL change
 
-	uid, gid       int // db user/group obtained on init
+	fileInfo os.FileInfo // db info cached during init
-	mode           os.FileMode
+	dirInfo  os.FileInfo // parent dir info cached during init
-	diruid, dirgid int // db parent user/group obtained on init
-	dirmode        os.FileMode
 
 	ctx    context.Context
 	cancel func()
@@ -96,7 +95,7 @@ type DB struct {
 
 	// List of replicas for the database.
 	// Must be set before calling Open().
-	Replicas []Replica
+	Replicas []*Replica
 }
 
 // NewDB returns a new instance of DB for a given path.
@@ -104,8 +103,6 @@ func NewDB(path string) *DB {
 	db := &DB{
 		path:   path,
 		notify: make(chan struct{}),
-		uid:    -1, gid: -1, mode: 0600,
-		diruid: -1, dirgid: -1, dirmode: 0700,
 
 		MinCheckpointPageN: DefaultMinCheckpointPageN,
 		MaxCheckpointPageN: DefaultMaxCheckpointPageN,
@@ -173,7 +170,7 @@ func (db *DB) ShadowWALDir(generation string) string {
 // Panics if generation is blank or index is negative.
 func (db *DB) ShadowWALPath(generation string, index int) string {
 	assert(index >= 0, "shadow wal index cannot be negative")
-	return filepath.Join(db.ShadowWALDir(generation), FormatWALPath(index))
+	return filepath.Join(db.ShadowWALDir(generation), FormatIndex(index)+".wal")
 }
 
 // CurrentShadowWALPath returns the path to the last shadow WAL in a generation.
@@ -196,11 +193,8 @@ func (db *DB) CurrentShadowWALIndex(generation string) (index int, size int64, e
 
 	// Find highest wal index.
 	for _, fi := range fis {
-		if !strings.HasSuffix(fi.Name(), WALExt) {
+		if v, err := parseWALPath(fi.Name()); err != nil {
-			continue
+			continue // invalid filename
-		}
-		if v, _, _, err := ParseWALPath(fi.Name()); err != nil {
-			continue // invalid wal filename
 		} else if v > index {
 			index = v
 		}
@@ -210,8 +204,18 @@ func (db *DB) CurrentShadowWALIndex(generation string) (index int, size int64, e
 	return index, size, nil
 }
 
+// FileInfo returns the cached file stats for the database file when it was initialized.
+func (db *DB) FileInfo() os.FileInfo {
+	return db.fileInfo
+}
+
+// DirInfo returns the cached file stats for the parent directory of the database file when it was initialized.
+func (db *DB) DirInfo() os.FileInfo {
+	return db.dirInfo
+}
+
 // Replica returns a replica by name.
-func (db *DB) Replica(name string) Replica {
+func (db *DB) Replica(name string) *Replica {
 	for _, r := range db.Replicas {
 		if r.Name() == name {
 			return r
@@ -364,42 +368,6 @@ func (db *DB) UpdatedAt() (time.Time, error) {
 	return t, nil
 }
 
-// Snapshots returns a list of all snapshots across all replicas.
-func (db *DB) Snapshots(ctx context.Context) ([]*SnapshotInfo, error) {
-	var infos []*SnapshotInfo
-	for _, r := range db.Replicas {
-		a, err := r.Snapshots(ctx)
-		if err != nil {
-			return nil, err
-		}
-		infos = append(infos, a...)
-	}
-
-	// Sort in order by time.
-	sort.Slice(infos, func(i, j int) bool {
-		return infos[i].CreatedAt.Before(infos[j].CreatedAt)
-	})
-	return infos, nil
-}
-
-// WALs returns a list of all WAL files across all replicas.
-func (db *DB) WALs(ctx context.Context) ([]*WALInfo, error) {
-	var infos []*WALInfo
-	for _, r := range db.Replicas {
-		a, err := r.WALs(ctx)
-		if err != nil {
-			return nil, err
-		}
-		infos = append(infos, a...)
-	}
-
-	// Sort in order by time.
-	sort.Slice(infos, func(i, j int) bool {
-		return infos[i].CreatedAt.Before(infos[j].CreatedAt)
-	})
-	return infos, nil
-}
-
 // init initializes the connection to the database.
 // Skipped if already initialized or if the database file does not exist.
 func (db *DB) init() (err error) {
@@ -415,15 +383,13 @@ func (db *DB) init() (err error) {
 	} else if err != nil {
 		return err
 	}
-	db.uid, db.gid = fileinfo(fi)
+	db.fileInfo = fi
-	db.mode = fi.Mode()
 
 	// Obtain permissions for parent directory.
 	if fi, err = os.Stat(filepath.Dir(db.path)); err != nil {
 		return err
 	}
-	db.diruid, db.dirgid = fileinfo(fi)
+	db.dirInfo = fi
-	db.dirmode = fi.Mode()
 
 	dsn := db.path
 	dsn += fmt.Sprintf("?_busy_timeout=%d", BusyTimeout.Milliseconds())
@@ -489,7 +455,7 @@ func (db *DB) init() (err error) {
 	}
 
 	// Ensure meta directory structure exists.
-	if err := mkdirAll(db.MetaPath(), db.dirmode, db.diruid, db.dirgid); err != nil {
+	if err := internal.MkdirAll(db.MetaPath(), db.dirInfo); err != nil {
 		return err
 	}
 
@@ -596,7 +562,7 @@ func (db *DB) cleanWAL() error {
 	// Determine lowest index that's been replicated to all replicas.
 	min := -1
 	for _, r := range db.Replicas {
-		pos := r.LastPos()
+		pos := r.Pos()
 		if pos.Generation != generation {
 			pos = Pos{} // different generation, reset index to zero
 		}
@@ -620,7 +586,7 @@ func (db *DB) cleanWAL() error {
 		return err
 	}
 	for _, fi := range fis {
-		if idx, _, _, err := ParseWALPath(fi.Name()); err != nil || idx >= min {
+		if idx, err := parseWALPath(fi.Name()); err != nil || idx >= min {
 			continue
 		}
 		if err := os.Remove(filepath.Join(dir, fi.Name())); err != nil {
@@ -696,7 +662,7 @@ func (db *DB) createGeneration() (string, error) {
 
 	// Generate new directory.
 	dir := filepath.Join(db.MetaPath(), "generations", generation)
-	if err := mkdirAll(dir, db.dirmode, db.diruid, db.dirgid); err != nil {
+	if err := internal.MkdirAll(dir, db.dirInfo); err != nil {
 		return "", err
 	}
 
@@ -707,10 +673,15 @@ func (db *DB) createGeneration() (string, error) {
 
 	// Atomically write generation name as current generation.
 	generationNamePath := db.GenerationNamePath()
-	if err := ioutil.WriteFile(generationNamePath+".tmp", []byte(generation+"\n"), db.mode); err != nil {
+	mode := os.FileMode(0600)
+	if db.fileInfo != nil {
+		mode = db.fileInfo.Mode()
+	}
+	if err := ioutil.WriteFile(generationNamePath+".tmp", []byte(generation+"\n"), mode); err != nil {
 		return "", fmt.Errorf("write generation temp file: %w", err)
 	}
-	_ = os.Chown(generationNamePath+".tmp", db.uid, db.gid)
+	uid, gid := internal.Fileinfo(db.fileInfo)
+	_ = os.Chown(generationNamePath+".tmp", uid, gid)
 	if err := os.Rename(generationNamePath+".tmp", generationNamePath); err != nil {
 		return "", fmt.Errorf("rename generation file: %w", err)
 	}
@@ -801,7 +772,7 @@ func (db *DB) Sync(ctx context.Context) (err error) {
 	if checkpoint {
 		changed = true
 
-		if err := db.checkpointAndInit(ctx, info.generation, checkpointMode); err != nil {
+		if err := db.checkpoint(ctx, info.generation, checkpointMode); err != nil {
 			return fmt.Errorf("checkpoint: mode=%v err=%w", checkpointMode, err)
 		}
 	}
@@ -959,13 +930,13 @@ func (db *DB) syncWAL(info syncInfo) (newSize int64, err error) {
 
 	// Parse index of current shadow WAL file.
 	dir, base := filepath.Split(info.shadowWALPath)
-	index, _, _, err := ParseWALPath(base)
+	index, err := parseWALPath(base)
 	if err != nil {
 		return 0, fmt.Errorf("cannot parse shadow wal filename: %s", base)
 	}
 
 	// Start a new shadow WAL file with next index.
-	newShadowWALPath := filepath.Join(dir, FormatWALPath(index+1))
+	newShadowWALPath := filepath.Join(dir, formatWALPath(index+1))
 	newSize, err = db.initShadowWALFile(newShadowWALPath)
 	if err != nil {
 		return 0, fmt.Errorf("cannot init shadow wal file: name=%s err=%w", newShadowWALPath, err)
@@ -993,12 +964,17 @@ func (db *DB) initShadowWALFile(filename string) (int64, error) {
 	}
 
 	// Write header to new WAL shadow file.
-	if err := mkdirAll(filepath.Dir(filename), db.dirmode, db.diruid, db.dirgid); err != nil {
+	mode := os.FileMode(0600)
+	if fi := db.fileInfo; fi != nil {
+		mode = fi.Mode()
+	}
+	if err := internal.MkdirAll(filepath.Dir(filename), db.dirInfo); err != nil {
 		return 0, err
-	} else if err := ioutil.WriteFile(filename, hdr, db.mode); err != nil {
+	} else if err := ioutil.WriteFile(filename, hdr, mode); err != nil {
 		return 0, err
 	}
-	_ = os.Chown(filename, db.uid, db.gid)
+	uid, gid := internal.Fileinfo(db.fileInfo)
+	_ = os.Chown(filename, uid, gid)
 
 	// Copy as much shadow WAL as available.
 	newSize, err := db.copyToShadowWAL(filename)
@@ -1084,7 +1060,7 @@ func (db *DB) copyToShadowWAL(filename string) (newSize int64, err error) {
 		chksum0, chksum1 = Checksum(bo, chksum0, chksum1, frame[:8])  // frame header
 		chksum0, chksum1 = Checksum(bo, chksum0, chksum1, frame[24:]) // frame data
 		if chksum0 != fchksum0 || chksum1 != fchksum1 {
-			log.Printf("copy shadow: checksum mismatch, skipping: offset=%d (%x,%x) != (%x,%x)", offset, chksum0, chksum1, fchksum0, fchksum1)
+			Tracef("%s: copy shadow: checksum mismatch, skipping: offset=%d (%x,%x) != (%x,%x)", db.path, offset, chksum0, chksum1, fchksum0, fchksum1)
 			break
 		}
 
@@ -1155,7 +1131,7 @@ func (db *DB) shadowWALReader(pos Pos) (r *ShadowWALReader, err error) {
 	// Ensure file is closed if any error occurs.
 	defer func() {
 		if err != nil {
-			r.Close()
+			f.Close()
 		}
 	}()
 
@@ -1258,13 +1234,91 @@ func readLastChecksumFrom(f *os.File, pageSize int) (uint32, uint32, error) {
 }
 
 // Checkpoint performs a checkpoint on the WAL file.
-func (db *DB) Checkpoint(mode string) (err error) {
+func (db *DB) Checkpoint(ctx context.Context, mode string) (err error) {
 	db.mu.Lock()
 	defer db.mu.Unlock()
-	return db.checkpoint(mode)
+
+	generation, err := db.CurrentGeneration()
+	if err != nil {
+		return fmt.Errorf("cannot determine generation: %w", err)
+	}
+	return db.checkpoint(ctx, generation, mode)
 }
 
-func (db *DB) checkpoint(mode string) (err error) {
+// checkpointAndInit performs a checkpoint on the WAL file and initializes a
+// new shadow WAL file.
+func (db *DB) checkpoint(ctx context.Context, generation, mode string) error {
+	shadowWALPath, err := db.CurrentShadowWALPath(generation)
+	if err != nil {
+		return err
+	}
+
+	// Read WAL header before checkpoint to check if it has been restarted.
+	hdr, err := readWALHeader(db.WALPath())
+	if err != nil {
+		return err
+	}
+
+	// Copy shadow WAL before checkpoint to copy as much as possible.
+	if _, err := db.copyToShadowWAL(shadowWALPath); err != nil {
+		return fmt.Errorf("cannot copy to end of shadow wal before checkpoint: %w", err)
+	}
+
+	// Execute checkpoint and immediately issue a write to the WAL to ensure
+	// a new page is written.
+	if err := db.execCheckpoint(mode); err != nil {
+		return err
+	} else if _, err = db.db.Exec(`INSERT INTO _litestream_seq (id, seq) VALUES (1, 1) ON CONFLICT (id) DO UPDATE SET seq = seq + 1`); err != nil {
+		return err
+	}
+
+	// If WAL hasn't been restarted, exit.
+	if other, err := readWALHeader(db.WALPath()); err != nil {
+		return err
+	} else if bytes.Equal(hdr, other) {
+		return nil
+	}
+
+	// Start a transaction. This will be promoted immediately after.
+	tx, err := db.db.Begin()
+	if err != nil {
+		return fmt.Errorf("begin: %w", err)
+	}
+	defer func() { _ = rollback(tx) }()
+
+	// Insert into the lock table to promote to a write tx. The lock table
+	// insert will never actually occur because our tx will be rolled back,
+	// however, it will ensure our tx grabs the write lock. Unfortunately,
+	// we can't call "BEGIN IMMEDIATE" as we are already in a transaction.
+	if _, err := tx.ExecContext(ctx, `INSERT INTO _litestream_lock (id) VALUES (1);`); err != nil {
+		return fmt.Errorf("_litestream_lock: %w", err)
+	}
+
+	// Copy the end of the previous WAL before starting a new shadow WAL.
+	if _, err := db.copyToShadowWAL(shadowWALPath); err != nil {
+		return fmt.Errorf("cannot copy to end of shadow wal: %w", err)
+	}
+
+	// Parse index of current shadow WAL file.
+	index, err := parseWALPath(shadowWALPath)
+	if err != nil {
+		return fmt.Errorf("cannot parse shadow wal filename: %s", shadowWALPath)
+	}
+
+	// Start a new shadow WAL file with next index.
+	newShadowWALPath := filepath.Join(filepath.Dir(shadowWALPath), formatWALPath(index+1))
+	if _, err := db.initShadowWALFile(newShadowWALPath); err != nil {
+		return fmt.Errorf("cannot init shadow wal file: name=%s err=%w", newShadowWALPath, err)
+	}
+
+	// Release write lock before checkpointing & exiting.
+	if err := tx.Rollback(); err != nil {
+		return fmt.Errorf("rollback post-checkpoint tx: %w", err)
+	}
+	return nil
+}
+
+func (db *DB) execCheckpoint(mode string) (err error) {
 	// Ignore if there is no underlying database.
 	if db.db == nil {
 		return nil
@@ -1310,79 +1364,6 @@ func (db *DB) checkpoint(mode string) (err error) {
 	return nil
 }
 
-// checkpointAndInit performs a checkpoint on the WAL file and initializes a
-// new shadow WAL file.
-func (db *DB) checkpointAndInit(ctx context.Context, generation, mode string) error {
-	shadowWALPath, err := db.CurrentShadowWALPath(generation)
-	if err != nil {
-		return err
-	}
-
-	// Read WAL header before checkpoint to check if it has been restarted.
-	hdr, err := readWALHeader(db.WALPath())
-	if err != nil {
-		return err
-	}
-
-	// Copy shadow WAL before checkpoint to copy as much as possible.
-	if _, err := db.copyToShadowWAL(shadowWALPath); err != nil {
-		return fmt.Errorf("cannot copy to end of shadow wal before checkpoint: %w", err)
-	}
-
-	// Execute checkpoint and immediately issue a write to the WAL to ensure
-	// a new page is written.
-	if err := db.checkpoint(mode); err != nil {
-		return err
-	} else if _, err = db.db.Exec(`INSERT INTO _litestream_seq (id, seq) VALUES (1, 1) ON CONFLICT (id) DO UPDATE SET seq = seq + 1`); err != nil {
-		return err
-	}
-
-	// If WAL hasn't been restarted, exit.
-	if other, err := readWALHeader(db.WALPath()); err != nil {
-		return err
-	} else if bytes.Equal(hdr, other) {
-		return nil
-	}
-
-	// Start a transaction. This will be promoted immediately after.
-	tx, err := db.db.Begin()
-	if err != nil {
-		return fmt.Errorf("begin: %w", err)
-	}
-	defer func() { _ = rollback(tx) }()
-
-	// Insert into the lock table to promote to a write tx. The lock table
-	// insert will never actually occur because our tx will be rolled back,
-	// however, it will ensure our tx grabs the write lock. Unfortunately,
-	// we can't call "BEGIN IMMEDIATE" as we are already in a transaction.
-	if _, err := tx.ExecContext(ctx, `INSERT INTO _litestream_lock (id) VALUES (1);`); err != nil {
-		return fmt.Errorf("_litestream_lock: %w", err)
-	}
-
-	// Copy the end of the previous WAL before starting a new shadow WAL.
-	if _, err := db.copyToShadowWAL(shadowWALPath); err != nil {
-		return fmt.Errorf("cannot copy to end of shadow wal: %w", err)
-	}
-
-	// Parse index of current shadow WAL file.
-	index, _, _, err := ParseWALPath(shadowWALPath)
-	if err != nil {
-		return fmt.Errorf("cannot parse shadow wal filename: %s", shadowWALPath)
-	}
-
-	// Start a new shadow WAL file with next index.
-	newShadowWALPath := filepath.Join(filepath.Dir(shadowWALPath), FormatWALPath(index+1))
-	if _, err := db.initShadowWALFile(newShadowWALPath); err != nil {
-		return fmt.Errorf("cannot init shadow wal file: name=%s err=%w", newShadowWALPath, err)
-	}
-
-	// Release write lock before checkpointing & exiting.
-	if err := tx.Rollback(); err != nil {
-		return fmt.Errorf("rollback post-checkpoint tx: %w", err)
-	}
-	return nil
-}
-
 // monitor runs in a separate goroutine and monitors the database & WAL.
 func (db *DB) monitor() {
 	ticker := time.NewTicker(db.MonitorInterval)
@@ -1403,189 +1384,12 @@ func (db *DB) monitor() {
 	}
 }
 
-// RestoreReplica restores the database from a replica based on the options given.
-// This method will restore into opt.OutputPath, if specified, or into the
-// DB's original database path. It can optionally restore from a specific
-// replica or generation or it will automatically choose the best one. Finally,
-// a timestamp can be specified to restore the database to a specific
-// point-in-time.
-func RestoreReplica(ctx context.Context, r Replica, opt RestoreOptions) (err error) {
-	// Validate options.
-	if opt.OutputPath == "" {
-		return fmt.Errorf("output path required")
-	} else if opt.Generation == "" && opt.Index != math.MaxInt32 {
-		return fmt.Errorf("must specify generation when restoring to index")
-	} else if opt.Index != math.MaxInt32 && !opt.Timestamp.IsZero() {
-		return fmt.Errorf("cannot specify index & timestamp to restore")
-	}
-
-	// Ensure logger exists.
-	logger := opt.Logger
-	if logger == nil {
-		logger = log.New(ioutil.Discard, "", 0)
-	}
-
-	logPrefix := r.Name()
-	if db := r.DB(); db != nil {
-		logPrefix = fmt.Sprintf("%s(%s)", db.Path(), r.Name())
-	}
-
-	// Ensure output path does not already exist.
-	if _, err := os.Stat(opt.OutputPath); err == nil {
-		return fmt.Errorf("cannot restore, output path already exists: %s", opt.OutputPath)
-	} else if err != nil && !os.IsNotExist(err) {
-		return err
-	}
-
-	// Find lastest snapshot that occurs before timestamp or index.
-	var minWALIndex int
-	if opt.Index < math.MaxInt32 {
-		if minWALIndex, err = SnapshotIndexByIndex(ctx, r, opt.Generation, opt.Index); err != nil {
-			return fmt.Errorf("cannot find snapshot index: %w", err)
-		}
-	} else {
-		if minWALIndex, err = SnapshotIndexAt(ctx, r, opt.Generation, opt.Timestamp); err != nil {
-			return fmt.Errorf("cannot find snapshot index by timestamp: %w", err)
-		}
-	}
-
-	// Find the maximum WAL index that occurs before timestamp.
-	maxWALIndex, err := WALIndexAt(ctx, r, opt.Generation, opt.Index, opt.Timestamp)
-	if err != nil {
-		return fmt.Errorf("cannot find max wal index for restore: %w", err)
-	}
-	snapshotOnly := maxWALIndex == -1
-
-	// Initialize starting position.
-	pos := Pos{Generation: opt.Generation, Index: minWALIndex}
-	tmpPath := opt.OutputPath + ".tmp"
-
-	// Copy snapshot to output path.
-	logger.Printf("%s: restoring snapshot %s/%08x to %s", logPrefix, opt.Generation, minWALIndex, tmpPath)
-	if err := restoreSnapshot(ctx, r, pos.Generation, pos.Index, tmpPath); err != nil {
-		return fmt.Errorf("cannot restore snapshot: %w", err)
-	}
-
-	// If no WAL files available, move snapshot to final path & exit early.
-	if snapshotOnly {
-		logger.Printf("%s: snapshot only, finalizing database", logPrefix)
-		return os.Rename(tmpPath, opt.OutputPath)
-	}
-
-	// Begin processing WAL files.
-	logger.Printf("%s: restoring wal files: generation=%s index=[%08x,%08x]", logPrefix, opt.Generation, minWALIndex, maxWALIndex)
-
-	// Fill input channel with all WAL indexes to be loaded in order.
-	ch := make(chan int, maxWALIndex-minWALIndex+1)
-	for index := minWALIndex; index <= maxWALIndex; index++ {
-		ch <- index
-	}
-	close(ch)
-
-	// Track load state for each WAL.
-	var mu sync.Mutex
-	cond := sync.NewCond(&mu)
-	walStates := make([]walRestoreState, maxWALIndex-minWALIndex+1)
-
-	parallelism := opt.Parallelism
-	if parallelism < 1 {
-		parallelism = 1
-	}
-
-	// Download WAL files to disk in parallel.
-	g, ctx := errgroup.WithContext(ctx)
-	for i := 0; i < parallelism; i++ {
-		g.Go(func() error {
-			for {
-				select {
-				case <-ctx.Done():
-					cond.Broadcast()
-					return err
-				case index, ok := <-ch:
-					if !ok {
-						cond.Broadcast()
-						return nil
-					}
-
-					startTime := time.Now()
-
-					err := downloadWAL(ctx, r, opt.Generation, index, tmpPath)
-					if err != nil {
-						err = fmt.Errorf("cannot download wal %s/%08x: %w", opt.Generation, index, err)
-					}
-
-					// Mark index as ready-to-apply and notify applying code.
-					mu.Lock()
-					walStates[index-minWALIndex] = walRestoreState{ready: true, err: err}
-					mu.Unlock()
-					cond.Broadcast()
-
-					// Returning the error here will cancel the other goroutines.
-					if err != nil {
-						return err
-					}
-
-					logger.Printf("%s: downloaded wal %s/%08x elapsed=%s",
-						logPrefix, opt.Generation, index,
-						time.Since(startTime).String(),
-					)
-				}
-			}
-		})
-	}
-
-	// Apply WAL files in order as they are ready.
-	for index := minWALIndex; index <= maxWALIndex; index++ {
-		// Wait until next WAL file is ready to apply.
-		mu.Lock()
-		for !walStates[index-minWALIndex].ready {
-			if err := ctx.Err(); err != nil {
-				return err
-			}
-			cond.Wait()
-		}
-		if err := walStates[index-minWALIndex].err; err != nil {
-			return err
-		}
-		mu.Unlock()
-
-		// Apply WAL to database file.
-		startTime := time.Now()
-		if err = applyWAL(ctx, index, tmpPath); err != nil {
-			return fmt.Errorf("cannot apply wal: %w", err)
-		}
-		logger.Printf("%s: applied wal %s/%08x elapsed=%s",
-			logPrefix, opt.Generation, index,
-			time.Since(startTime).String(),
-		)
-	}
-
-	// Ensure all goroutines finish. All errors should have been handled during
-	// the processing of WAL files but this ensures that all processing is done.
-	if err := g.Wait(); err != nil {
-		return err
-	}
-
-	// Copy file to final location.
-	logger.Printf("%s: renaming database from temporary location", logPrefix)
-	if err := os.Rename(tmpPath, opt.OutputPath); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-type walRestoreState struct {
-	ready bool
-	err   error
-}
-
 // CalcRestoreTarget returns a replica & generation to restore from based on opt criteria.
-func (db *DB) CalcRestoreTarget(ctx context.Context, opt RestoreOptions) (Replica, string, error) {
+func (db *DB) CalcRestoreTarget(ctx context.Context, opt RestoreOptions) (*Replica, string, error) {
 	var target struct {
-		replica    Replica
+		replica    *Replica
 		generation string
-		stats      GenerationStats
+		updatedAt  time.Time
 	}
 
 	for _, r := range db.Replicas {
@@ -1594,134 +1398,21 @@ func (db *DB) CalcRestoreTarget(ctx context.Context, opt RestoreOptions) (Replic
 			continue
 		}
 
-		generation, stats, err := CalcReplicaRestoreTarget(ctx, r, opt)
+		generation, updatedAt, err := r.CalcRestoreTarget(ctx, opt)
 		if err != nil {
 			return nil, "", err
 		}
 
 		// Use the latest replica if we have multiple candidates.
-		if !stats.UpdatedAt.After(target.stats.UpdatedAt) {
+		if !updatedAt.After(target.updatedAt) {
 			continue
 		}
 
-		target.replica, target.generation, target.stats = r, generation, stats
+		target.replica, target.generation, target.updatedAt = r, generation, updatedAt
 	}
 	return target.replica, target.generation, nil
 }
 
-// CalcReplicaRestoreTarget returns a generation to restore from.
-func CalcReplicaRestoreTarget(ctx context.Context, r Replica, opt RestoreOptions) (generation string, stats GenerationStats, err error) {
-	var target struct {
-		generation string
-		stats      GenerationStats
-	}
-
-	generations, err := r.Generations(ctx)
-	if err != nil {
-		return "", stats, fmt.Errorf("cannot fetch generations: %w", err)
-	}
-
-	// Search generations for one that contains the requested timestamp.
-	for _, generation := range generations {
-		// Skip generation if it does not match filter.
-		if opt.Generation != "" && generation != opt.Generation {
-			continue
-		}
-
-		// Fetch stats for generation.
-		stats, err := r.GenerationStats(ctx, generation)
-		if err != nil {
-			return "", stats, fmt.Errorf("cannot determine stats for generation (%s/%s): %s", r.Name(), generation, err)
-		}
-
-		// Skip if it does not contain timestamp.
-		if !opt.Timestamp.IsZero() {
-			if opt.Timestamp.Before(stats.CreatedAt) || opt.Timestamp.After(stats.UpdatedAt) {
-				continue
-			}
-		}
-
-		// Use the latest replica if we have multiple candidates.
-		if !stats.UpdatedAt.After(target.stats.UpdatedAt) {
-			continue
-		}
-
-		target.generation = generation
-		target.stats = stats
-	}
-
-	return target.generation, target.stats, nil
-}
-
-// restoreSnapshot copies a snapshot from the replica to a file.
-func restoreSnapshot(ctx context.Context, r Replica, generation string, index int, filename string) error {
-	// Determine the user/group & mode based on the DB, if available.
-	uid, gid, mode := -1, -1, os.FileMode(0600)
-	diruid, dirgid, dirmode := -1, -1, os.FileMode(0700)
-	if db := r.DB(); db != nil {
-		uid, gid, mode = db.uid, db.gid, db.mode
-		diruid, dirgid, dirmode = db.diruid, db.dirgid, db.dirmode
-	}
-
-	if err := mkdirAll(filepath.Dir(filename), dirmode, diruid, dirgid); err != nil {
-		return err
-	}
-
-	f, err := createFile(filename, mode, uid, gid)
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-
-	rd, err := r.SnapshotReader(ctx, generation, index)
-	if err != nil {
-		return err
-	}
-	defer rd.Close()
-
-	if _, err := io.Copy(f, rd); err != nil {
-		return err
-	}
-
-	if err := f.Sync(); err != nil {
-		return err
-	}
-	return f.Close()
-}
-
-// downloadWAL copies a WAL file from the replica to a local copy next to the DB.
-// The WAL is later applied by applyWAL(). This function can be run in parallel
-// to download multiple WAL files simultaneously.
-func downloadWAL(ctx context.Context, r Replica, generation string, index int, dbPath string) error {
-	// Determine the user/group & mode based on the DB, if available.
-	uid, gid, mode := -1, -1, os.FileMode(0600)
-	if db := r.DB(); db != nil {
-		uid, gid, mode = db.uid, db.gid, db.mode
-	}
-
-	// Open WAL file from replica.
-	rd, err := r.WALReader(ctx, generation, index)
-	if err != nil {
-		return err
-	}
-	defer rd.Close()
-
-	// Open handle to destination WAL path.
-	f, err := createFile(fmt.Sprintf("%s-%08x-wal", dbPath, index), mode, uid, gid)
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-
-	// Copy WAL to target path.
-	if _, err := io.Copy(f, rd); err != nil {
-		return err
-	} else if err := f.Close(); err != nil {
-		return err
-	}
-	return nil
-}
-
 // applyWAL performs a truncating checkpoint on the given database.
 func applyWAL(ctx context.Context, index int, dbPath string) error {
 	// Copy WAL file from it's staging path to the correct "-wal" location.
@@ -1770,7 +1461,7 @@ func (db *DB) CRC64(ctx context.Context) (uint64, Pos, error) {
 	}
 
 	// Force a RESTART checkpoint to ensure the database is at the start of the WAL.
-	if err := db.checkpointAndInit(ctx, generation, CheckpointModeRestart); err != nil {
+	if err := db.checkpoint(ctx, generation, CheckpointModeRestart); err != nil {
 		return 0, Pos{}, err
 	}
 
@@ -1792,6 +1483,28 @@ func (db *DB) CRC64(ctx context.Context) (uint64, Pos, error) {
 	return h.Sum64(), pos, nil
 }
 
+// parseWALPath returns the index for the WAL file.
+// Returns an error if the path is not a valid WAL path.
+func parseWALPath(s string) (index int, err error) {
+	s = filepath.Base(s)
+
+	a := walPathRegex.FindStringSubmatch(s)
+	if a == nil {
+		return 0, fmt.Errorf("invalid wal path: %s", s)
+	}
+
+	i64, _ := strconv.ParseUint(a[1], 16, 64)
+	return int(i64), nil
+}
+
+// formatWALPath formats a WAL filename with a given index.
+func formatWALPath(index int) string {
+	assert(index >= 0, "wal index must be non-negative")
+	return FormatIndex(index) + ".wal"
+}
+
+var walPathRegex = regexp.MustCompile(`^([0-9a-f]{8})\.wal$`)
+
 // DefaultRestoreParallelism is the default parallelism when downloading WAL files.
 const DefaultRestoreParallelism = 8
 
@@ -1836,79 +1549,57 @@ func NewRestoreOptions() RestoreOptions {
 // Database metrics.
 var (
 	dbSizeGaugeVec = promauto.NewGaugeVec(prometheus.GaugeOpts{
-		Namespace: "litestream",
+		Name: "litestream_db_size",
-		Subsystem: "db",
-		Name:      "db_size",
 		Help: "The current size of the real DB",
 	}, []string{"db"})
 
 	walSizeGaugeVec = promauto.NewGaugeVec(prometheus.GaugeOpts{
-		Namespace: "litestream",
+		Name: "litestream_wal_size",
-		Subsystem: "db",
-		Name:      "wal_size",
 		Help: "The current size of the real WAL",
 	}, []string{"db"})
 
 	totalWALBytesCounterVec = promauto.NewCounterVec(prometheus.CounterOpts{
-		Namespace: "litestream",
+		Name: "litestream_total_wal_bytes",
-		Subsystem: "db",
-		Name:      "total_wal_bytes",
 		Help: "Total number of bytes written to shadow WAL",
 	}, []string{"db"})
 
 	shadowWALIndexGaugeVec = promauto.NewGaugeVec(prometheus.GaugeOpts{
-		Namespace: "litestream",
+		Name: "litestream_shadow_wal_index",
-		Subsystem: "db",
-		Name:      "shadow_wal_index",
 		Help: "The current index of the shadow WAL",
 	}, []string{"db"})
 
 	shadowWALSizeGaugeVec = promauto.NewGaugeVec(prometheus.GaugeOpts{
-		Namespace: "litestream",
+		Name: "litestream_shadow_wal_size",
-		Subsystem: "db",
-		Name:      "shadow_wal_size",
 		Help: "Current size of shadow WAL, in bytes",
 	}, []string{"db"})
 
 	syncNCounterVec = promauto.NewCounterVec(prometheus.CounterOpts{
-		Namespace: "litestream",
+		Name: "litestream_sync_count",
-		Subsystem: "db",
-		Name:      "sync_count",
 		Help: "Number of sync operations performed",
 	}, []string{"db"})
 
 	syncErrorNCounterVec = promauto.NewCounterVec(prometheus.CounterOpts{
-		Namespace: "litestream",
+		Name: "litestream_sync_error_count",
-		Subsystem: "db",
-		Name:      "sync_error_count",
 		Help: "Number of sync errors that have occurred",
 	}, []string{"db"})
 
 	syncSecondsCounterVec = promauto.NewCounterVec(prometheus.CounterOpts{
-		Namespace: "litestream",
+		Name: "litestream_sync_seconds",
-		Subsystem: "db",
-		Name:      "sync_seconds",
 		Help: "Time spent syncing shadow WAL, in seconds",
 	}, []string{"db"})
 
 	checkpointNCounterVec = promauto.NewCounterVec(prometheus.CounterOpts{
-		Namespace: "litestream",
+		Name: "litestream_checkpoint_count",
-		Subsystem: "db",
-		Name:      "checkpoint_count",
 		Help: "Number of checkpoint operations performed",
 	}, []string{"db", "mode"})
 
 	checkpointErrorNCounterVec = promauto.NewCounterVec(prometheus.CounterOpts{
-		Namespace: "litestream",
+		Name: "litestream_checkpoint_error_count",
-		Subsystem: "db",
-		Name:      "checkpoint_error_count",
 		Help: "Number of checkpoint errors that have occurred",
 	}, []string{"db", "mode"})
 
 	checkpointSecondsCounterVec = promauto.NewCounterVec(prometheus.CounterOpts{
-		Namespace: "litestream",
+		Name: "litestream_checkpoint_seconds",
-		Subsystem: "db",
-		Name:      "checkpoint_seconds",
 		Help: "Time spent checkpointing WAL, in seconds",
 	}, []string{"db", "mode"})
 )

db_test.go

@@ -151,7 +151,7 @@ func TestDB_CRC64(t *testing.T) {
 		}
 
 		// Checkpoint change into database. Checksum should change.
-		if _, err := sqldb.Exec(`PRAGMA wal_checkpoint(TRUNCATE);`); err != nil {
+		if err := db.Checkpoint(context.Background(), litestream.CheckpointModeTruncate); err != nil {
 			t.Fatal(err)
 		}
 
@@ -262,7 +262,7 @@ func TestDB_Sync(t *testing.T) {
 		}
 
 		// Checkpoint & fully close which should close WAL file.
-		if err := db.Checkpoint(litestream.CheckpointModeTruncate); err != nil {
+		if err := db.Checkpoint(context.Background(), litestream.CheckpointModeTruncate); err != nil {
 			t.Fatal(err)
 		} else if err := db.Close(); err != nil {
 			t.Fatal(err)

file/replica_client.go

@@ -0,0 +1,468 @@
+package file
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"github.com/benbjohnson/litestream"
+	"github.com/benbjohnson/litestream/internal"
+)
+
+// ReplicaClientType is the client type for this package.
+const ReplicaClientType = "file"
+
+var _ litestream.ReplicaClient = (*ReplicaClient)(nil)
+
+// ReplicaClient is a client for writing snapshots & WAL segments to disk.
+type ReplicaClient struct {
+	path string // destination path
+
+	Replica *litestream.Replica
+}
+
+// NewReplicaClient returns a new instance of ReplicaClient.
+func NewReplicaClient(path string) *ReplicaClient {
+	return &ReplicaClient{
+		path: path,
+	}
+}
+
+// db returns the database, if available.
+func (c *ReplicaClient) db() *litestream.DB {
+	if c.Replica == nil {
+		return nil
+	}
+	return c.Replica.DB()
+}
+
+// Type returns "file" as the client type.
+func (c *ReplicaClient) Type() string {
+	return ReplicaClientType
+}
+
+// Path returns the destination path to replicate the database to.
+func (c *ReplicaClient) Path() string {
+	return c.path
+}
+
+// GenerationsDir returns the path to a generation root directory.
+func (c *ReplicaClient) GenerationsDir() (string, error) {
+	if c.path == "" {
+		return "", fmt.Errorf("file replica path required")
+	}
+	return filepath.Join(c.path, "generations"), nil
+}
+
+// GenerationDir returns the path to a generation's root directory.
+func (c *ReplicaClient) GenerationDir(generation string) (string, error) {
+	dir, err := c.GenerationsDir()
+	if err != nil {
+		return "", err
+	} else if generation == "" {
+		return "", fmt.Errorf("generation required")
+	}
+	return filepath.Join(dir, generation), nil
+}
+
+// SnapshotsDir returns the path to a generation's snapshot directory.
+func (c *ReplicaClient) SnapshotsDir(generation string) (string, error) {
+	dir, err := c.GenerationDir(generation)
+	if err != nil {
+		return "", err
+	}
+	return filepath.Join(dir, "snapshots"), nil
+}
+
+// SnapshotPath returns the path to an uncompressed snapshot file.
+func (c *ReplicaClient) SnapshotPath(generation string, index int) (string, error) {
+	dir, err := c.SnapshotsDir(generation)
+	if err != nil {
+		return "", err
+	}
+	return filepath.Join(dir, litestream.FormatIndex(index)+".snapshot.lz4"), nil
+}
+
+// WALDir returns the path to a generation's WAL directory
+func (c *ReplicaClient) WALDir(generation string) (string, error) {
+	dir, err := c.GenerationDir(generation)
+	if err != nil {
+		return "", err
+	}
+	return filepath.Join(dir, "wal"), nil
+}
+
+// WALSegmentPath returns the path to a WAL segment file.
+func (c *ReplicaClient) WALSegmentPath(generation string, index int, offset int64) (string, error) {
+	dir, err := c.WALDir(generation)
+	if err != nil {
+		return "", err
+	}
+	return filepath.Join(dir, litestream.FormatIndex(index), fmt.Sprintf("%08x.wal.lz4", offset)), nil
+}
+
+// Generations returns a list of available generation names.
+func (c *ReplicaClient) Generations(ctx context.Context) ([]string, error) {
+	root, err := c.GenerationsDir()
+	if err != nil {
+		return nil, fmt.Errorf("cannot determine generations path: %w", err)
+	}
+
+	fis, err := ioutil.ReadDir(root)
+	if os.IsNotExist(err) {
+		return nil, nil
+	} else if err != nil {
+		return nil, err
+	}
+
+	var generations []string
+	for _, fi := range fis {
+		if !litestream.IsGenerationName(fi.Name()) {
+			continue
+		} else if !fi.IsDir() {
+			continue
+		}
+		generations = append(generations, fi.Name())
+	}
+	return generations, nil
+}
+
+// DeleteGeneration deletes all snapshots & WAL segments within a generation.
+func (c *ReplicaClient) DeleteGeneration(ctx context.Context, generation string) error {
+	dir, err := c.GenerationDir(generation)
+	if err != nil {
+		return fmt.Errorf("cannot determine generation path: %w", err)
+	}
+
+	if err := os.RemoveAll(dir); err != nil && !os.IsNotExist(err) {
+		return err
+	}
+	return nil
+}
+
+// Snapshots returns an iterator over all available snapshots for a generation.
+func (c *ReplicaClient) Snapshots(ctx context.Context, generation string) (litestream.SnapshotIterator, error) {
+	dir, err := c.SnapshotsDir(generation)
+	if err != nil {
+		return nil, fmt.Errorf("cannot determine snapshots path: %w", err)
+	}
+
+	f, err := os.Open(dir)
+	if os.IsNotExist(err) {
+		return litestream.NewSnapshotInfoSliceIterator(nil), nil
+	} else if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	fis, err := f.Readdir(-1)
+	if err != nil {
+		return nil, err
+	}
+
+	// Iterate over every file and convert to metadata.
+	infos := make([]litestream.SnapshotInfo, 0, len(fis))
+	for _, fi := range fis {
+		// Parse index from filename.
+		index, err := internal.ParseSnapshotPath(filepath.Base(fi.Name()))
+		if err != nil {
+			continue
+		}
+
+		infos = append(infos, litestream.SnapshotInfo{
+			Generation: generation,
+			Index:      index,
+			Size:       fi.Size(),
+			CreatedAt:  fi.ModTime().UTC(),
+		})
+	}
+
+	sort.Sort(litestream.SnapshotInfoSlice(infos))
+
+	return litestream.NewSnapshotInfoSliceIterator(infos), nil
+}
+
+// WriteSnapshot writes LZ4 compressed data from rd into a file on disk.
+func (c *ReplicaClient) WriteSnapshot(ctx context.Context, generation string, index int, rd io.Reader) (info litestream.SnapshotInfo, err error) {
+	filename, err := c.SnapshotPath(generation, index)
+	if err != nil {
+		return info, fmt.Errorf("cannot determine snapshot path: %w", err)
+	}
+
+	var fileInfo, dirInfo os.FileInfo
+	if db := c.db(); db != nil {
+		fileInfo, dirInfo = db.FileInfo(), db.DirInfo()
+	}
+
+	// Ensure parent directory exists.
+	if err := internal.MkdirAll(filepath.Dir(filename), dirInfo); err != nil {
+		return info, err
+	}
+
+	// Write snapshot to temporary file next to destination path.
+	f, err := internal.CreateFile(filename+".tmp", fileInfo)
+	if err != nil {
+		return info, err
+	}
+	defer f.Close()
+
+	if _, err := io.Copy(f, rd); err != nil {
+		return info, err
+	} else if err := f.Sync(); err != nil {
+		return info, err
+	} else if err := f.Close(); err != nil {
+		return info, err
+	}
+
+	// Build metadata.
+	fi, err := os.Stat(filename + ".tmp")
+	if err != nil {
+		return info, err
+	}
+	info = litestream.SnapshotInfo{
+		Generation: generation,
+		Index:      index,
+		Size:       fi.Size(),
+		CreatedAt:  fi.ModTime().UTC(),
+	}
+
+	// Move snapshot to final path when it has been fully written & synced to disk.
+	if err := os.Rename(filename+".tmp", filename); err != nil {
+		return info, err
+	}
+
+	return info, nil
+}
+
+// SnapshotReader returns a reader for snapshot data at the given generation/index.
+// Returns os.ErrNotExist if no matching index is found.
+func (c *ReplicaClient) SnapshotReader(ctx context.Context, generation string, index int) (io.ReadCloser, error) {
+	filename, err := c.SnapshotPath(generation, index)
+	if err != nil {
+		return nil, fmt.Errorf("cannot determine snapshot path: %w", err)
+	}
+	return os.Open(filename)
+}
+
+// DeleteSnapshot deletes a snapshot with the given generation & index.
+func (c *ReplicaClient) DeleteSnapshot(ctx context.Context, generation string, index int) error {
+	filename, err := c.SnapshotPath(generation, index)
+	if err != nil {
+		return fmt.Errorf("cannot determine snapshot path: %w", err)
+	}
+	if err := os.Remove(filename); err != nil && !os.IsNotExist(err) {
+		return err
+	}
+	return nil
+}
+
+// WALSegments returns an iterator over all available WAL files for a generation.
+func (c *ReplicaClient) WALSegments(ctx context.Context, generation string) (litestream.WALSegmentIterator, error) {
+	dir, err := c.WALDir(generation)
+	if err != nil {
+		return nil, fmt.Errorf("cannot determine wal path: %w", err)
+	}
+
+	f, err := os.Open(dir)
+	if os.IsNotExist(err) {
+		return litestream.NewWALSegmentInfoSliceIterator(nil), nil
+	} else if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	fis, err := f.Readdir(-1)
+	if err != nil {
+		return nil, err
+	}
+
+	// Iterate over every file and convert to metadata.
+	indexes := make([]int, 0, len(fis))
+	for _, fi := range fis {
+		index, err := litestream.ParseIndex(fi.Name())
+		if err != nil || !fi.IsDir() {
+			continue
+		}
+		indexes = append(indexes, index)
+	}
+
+	sort.Ints(indexes)
+
+	return newWALSegmentIterator(dir, generation, indexes), nil
+}
+
+// WriteWALSegment writes LZ4 compressed data from rd into a file on disk.
+func (c *ReplicaClient) WriteWALSegment(ctx context.Context, pos litestream.Pos, rd io.Reader) (info litestream.WALSegmentInfo, err error) {
+	filename, err := c.WALSegmentPath(pos.Generation, pos.Index, pos.Offset)
+	if err != nil {
+		return info, fmt.Errorf("cannot determine wal segment path: %w", err)
+	}
+
+	var fileInfo, dirInfo os.FileInfo
+	if db := c.db(); db != nil {
+		fileInfo, dirInfo = db.FileInfo(), db.DirInfo()
+	}
+
+	// Ensure parent directory exists.
+	if err := internal.MkdirAll(filepath.Dir(filename), dirInfo); err != nil {
+		return info, err
+	}
+
+	// Write WAL segment to temporary file next to destination path.
+	f, err := internal.CreateFile(filename+".tmp", fileInfo)
+	if err != nil {
+		return info, err
+	}
+	defer f.Close()
+
+	if _, err := io.Copy(f, rd); err != nil {
+		return info, err
+	} else if err := f.Sync(); err != nil {
+		return info, err
+	} else if err := f.Close(); err != nil {
+		return info, err
+	}
+
+	// Build metadata.
+	fi, err := os.Stat(filename + ".tmp")
+	if err != nil {
+		return info, err
+	}
+	info = litestream.WALSegmentInfo{
+		Generation: pos.Generation,
+		Index:      pos.Index,
+		Offset:     pos.Offset,
+		Size:       fi.Size(),
+		CreatedAt:  fi.ModTime().UTC(),
+	}
+
+	// Move WAL segment to final path when it has been written & synced to disk.
+	if err := os.Rename(filename+".tmp", filename); err != nil {
+		return info, err
+	}
+
+	return info, nil
+}
+
+// WALSegmentReader returns a reader for a section of WAL data at the given position.
+// Returns os.ErrNotExist if no matching index/offset is found.
+func (c *ReplicaClient) WALSegmentReader(ctx context.Context, pos litestream.Pos) (io.ReadCloser, error) {
+	filename, err := c.WALSegmentPath(pos.Generation, pos.Index, pos.Offset)
+	if err != nil {
+		return nil, fmt.Errorf("cannot determine wal segment path: %w", err)
+	}
+	return os.Open(filename)
+}
+
+// DeleteWALSegments deletes WAL segments at the given positions.
+func (c *ReplicaClient) DeleteWALSegments(ctx context.Context, a []litestream.Pos) error {
+	for _, pos := range a {
+		filename, err := c.WALSegmentPath(pos.Generation, pos.Index, pos.Offset)
+		if err != nil {
+			return fmt.Errorf("cannot determine wal segment path: %w", err)
+		}
+		if err := os.Remove(filename); err != nil && !os.IsNotExist(err) {
+			return err
+		}
+	}
+	return nil
+}
+
+type walSegmentIterator struct {
+	dir        string
+	generation string
+	indexes    []int
+
+	infos []litestream.WALSegmentInfo
+	err   error
+}
+
+func newWALSegmentIterator(dir, generation string, indexes []int) *walSegmentIterator {
+	return &walSegmentIterator{
+		dir:        dir,
+		generation: generation,
+		indexes:    indexes,
+	}
+}
+
+func (itr *walSegmentIterator) Close() (err error) {
+	return itr.err
+}
+
+func (itr *walSegmentIterator) Next() bool {
+	// Exit if an error has already occurred.
+	if itr.err != nil {
+		return false
+	}
+
+	for {
+		// Move to the next segment in cache, if available.
+		if len(itr.infos) > 1 {
+			itr.infos = itr.infos[1:]
+			return true
+		}
+		itr.infos = itr.infos[:0] // otherwise clear infos
+
+		// Move to the next index unless this is the first time initializing.
+		if itr.infos != nil && len(itr.indexes) > 0 {
+			itr.indexes = itr.indexes[1:]
+		}
+
+		// If no indexes remain, stop iteration.
+		if len(itr.indexes) == 0 {
+			return false
+		}
+
+		// Read segments into a cache for the current index.
+		index := itr.indexes[0]
+		f, err := os.Open(filepath.Join(itr.dir, litestream.FormatIndex(index)))
+		if err != nil {
+			itr.err = err
+			return false
+		}
+		defer f.Close()
+
+		fis, err := f.Readdir(-1)
+		if err != nil {
+			itr.err = err
+			return false
+		}
+		for _, fi := range fis {
+			filename := filepath.Base(fi.Name())
+			if fi.IsDir() {
+				continue
+			}
+
+			offset, err := litestream.ParseOffset(strings.TrimSuffix(filename, ".wal.lz4"))
+			if err != nil {
+				continue
+			}
+
+			itr.infos = append(itr.infos, litestream.WALSegmentInfo{
+				Generation: itr.generation,
+				Index:      index,
+				Offset:     offset,
+				Size:       fi.Size(),
+				CreatedAt:  fi.ModTime().UTC(),
+			})
+		}
+
+		if len(itr.infos) > 0 {
+			return true
+		}
+	}
+}
+
+func (itr *walSegmentIterator) Err() error { return itr.err }
+
+func (itr *walSegmentIterator) WALSegment() litestream.WALSegmentInfo {
+	if len(itr.infos) == 0 {
+		return litestream.WALSegmentInfo{}
+	}
+	return itr.infos[0]
+}

file/replica_client_test.go

@@ -0,0 +1,223 @@
+package file_test
+
+import (
+	"testing"
+
+	"github.com/benbjohnson/litestream/file"
+)
+
+func TestReplicaClient_Path(t *testing.T) {
+	c := file.NewReplicaClient("/foo/bar")
+	if got, want := c.Path(), "/foo/bar"; got != want {
+		t.Fatalf("Path()=%v, want %v", got, want)
+	}
+}
+
+func TestReplicaClient_Type(t *testing.T) {
+	if got, want := file.NewReplicaClient("").Type(), "file"; got != want {
+		t.Fatalf("Type()=%v, want %v", got, want)
+	}
+}
+
+func TestReplicaClient_GenerationsDir(t *testing.T) {
+	t.Run("OK", func(t *testing.T) {
+		if got, err := file.NewReplicaClient("/foo").GenerationsDir(); err != nil {
+			t.Fatal(err)
+		} else if want := "/foo/generations"; got != want {
+			t.Fatalf("GenerationsDir()=%v, want %v", got, want)
+		}
+	})
+	t.Run("ErrNoPath", func(t *testing.T) {
+		if _, err := file.NewReplicaClient("").GenerationsDir(); err == nil || err.Error() != `file replica path required` {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	})
+}
+
+func TestReplicaClient_GenerationDir(t *testing.T) {
+	t.Run("OK", func(t *testing.T) {
+		if got, err := file.NewReplicaClient("/foo").GenerationDir("0123456701234567"); err != nil {
+			t.Fatal(err)
+		} else if want := "/foo/generations/0123456701234567"; got != want {
+			t.Fatalf("GenerationDir()=%v, want %v", got, want)
+		}
+	})
+	t.Run("ErrNoPath", func(t *testing.T) {
+		if _, err := file.NewReplicaClient("").GenerationDir("0123456701234567"); err == nil || err.Error() != `file replica path required` {
+			t.Fatalf("expected error: %v", err)
+		}
+	})
+	t.Run("ErrNoGeneration", func(t *testing.T) {
+		if _, err := file.NewReplicaClient("/foo").GenerationDir(""); err == nil || err.Error() != `generation required` {
+			t.Fatalf("expected error: %v", err)
+		}
+	})
+}
+
+func TestReplicaClient_SnapshotsDir(t *testing.T) {
+	t.Run("OK", func(t *testing.T) {
+		if got, err := file.NewReplicaClient("/foo").SnapshotsDir("0123456701234567"); err != nil {
+			t.Fatal(err)
+		} else if want := "/foo/generations/0123456701234567/snapshots"; got != want {
+			t.Fatalf("SnapshotsDir()=%v, want %v", got, want)
+		}
+	})
+	t.Run("ErrNoPath", func(t *testing.T) {
+		if _, err := file.NewReplicaClient("").SnapshotsDir("0123456701234567"); err == nil || err.Error() != `file replica path required` {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	})
+	t.Run("ErrNoGeneration", func(t *testing.T) {
+		if _, err := file.NewReplicaClient("/foo").SnapshotsDir(""); err == nil || err.Error() != `generation required` {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	})
+}
+
+func TestReplicaClient_SnapshotPath(t *testing.T) {
+	t.Run("OK", func(t *testing.T) {
+		if got, err := file.NewReplicaClient("/foo").SnapshotPath("0123456701234567", 1000); err != nil {
+			t.Fatal(err)
+		} else if want := "/foo/generations/0123456701234567/snapshots/000003e8.snapshot.lz4"; got != want {
+			t.Fatalf("SnapshotPath()=%v, want %v", got, want)
+		}
+	})
+	t.Run("ErrNoPath", func(t *testing.T) {
+		if _, err := file.NewReplicaClient("").SnapshotPath("0123456701234567", 1000); err == nil || err.Error() != `file replica path required` {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	})
+	t.Run("ErrNoGeneration", func(t *testing.T) {
+		if _, err := file.NewReplicaClient("/foo").SnapshotPath("", 1000); err == nil || err.Error() != `generation required` {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	})
+}
+
+func TestReplicaClient_WALDir(t *testing.T) {
+	t.Run("OK", func(t *testing.T) {
+		if got, err := file.NewReplicaClient("/foo").WALDir("0123456701234567"); err != nil {
+			t.Fatal(err)
+		} else if want := "/foo/generations/0123456701234567/wal"; got != want {
+			t.Fatalf("WALDir()=%v, want %v", got, want)
+		}
+	})
+	t.Run("ErrNoPath", func(t *testing.T) {
+		if _, err := file.NewReplicaClient("").WALDir("0123456701234567"); err == nil || err.Error() != `file replica path required` {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	})
+	t.Run("ErrNoGeneration", func(t *testing.T) {
+		if _, err := file.NewReplicaClient("/foo").WALDir(""); err == nil || err.Error() != `generation required` {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	})
+}
+
+func TestReplicaClient_WALSegmentPath(t *testing.T) {
+	t.Run("OK", func(t *testing.T) {
+		if got, err := file.NewReplicaClient("/foo").WALSegmentPath("0123456701234567", 1000, 1001); err != nil {
+			t.Fatal(err)
+		} else if want := "/foo/generations/0123456701234567/wal/000003e8/000003e9.wal.lz4"; got != want {
+			t.Fatalf("WALPath()=%v, want %v", got, want)
+		}
+	})
+	t.Run("ErrNoPath", func(t *testing.T) {
+		if _, err := file.NewReplicaClient("").WALSegmentPath("0123456701234567", 1000, 0); err == nil || err.Error() != `file replica path required` {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	})
+	t.Run("ErrNoGeneration", func(t *testing.T) {
+		if _, err := file.NewReplicaClient("/foo").WALSegmentPath("", 1000, 0); err == nil || err.Error() != `generation required` {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	})
+}
+
+/*
+func TestReplica_Sync(t *testing.T) {
+	// Ensure replica can successfully sync after DB has sync'd.
+	t.Run("InitialSync", func(t *testing.T) {
+		db, sqldb := MustOpenDBs(t)
+		defer MustCloseDBs(t, db, sqldb)
+
+		r := litestream.NewReplica(db, "", file.NewReplicaClient(t.TempDir()))
+		r.MonitorEnabled = false
+		db.Replicas = []*litestream.Replica{r}
+
+		// Sync database & then sync replica.
+		if err := db.Sync(context.Background()); err != nil {
+			t.Fatal(err)
+		} else if err := r.Sync(context.Background()); err != nil {
+			t.Fatal(err)
+		}
+
+		// Ensure posistions match.
+		if want, err := db.Pos(); err != nil {
+			t.Fatal(err)
+		} else if got, err := r.Pos(context.Background()); err != nil {
+			t.Fatal(err)
+		} else if got != want {
+			t.Fatalf("Pos()=%v, want %v", got, want)
+		}
+	})
+
+	// Ensure replica can successfully sync multiple times.
+	t.Run("MultiSync", func(t *testing.T) {
+		db, sqldb := MustOpenDBs(t)
+		defer MustCloseDBs(t, db, sqldb)
+
+		r := litestream.NewReplica(db, "", file.NewReplicaClient(t.TempDir()))
+		r.MonitorEnabled = false
+		db.Replicas = []*litestream.Replica{r}
+
+		if _, err := sqldb.Exec(`CREATE TABLE foo (bar TEXT);`); err != nil {
+			t.Fatal(err)
+		}
+
+		// Write to the database multiple times and sync after each write.
+		for i, n := 0, db.MinCheckpointPageN*2; i < n; i++ {
+			if _, err := sqldb.Exec(`INSERT INTO foo (bar) VALUES ('baz')`); err != nil {
+				t.Fatal(err)
+			}
+
+			// Sync periodically.
+			if i%100 == 0 || i == n-1 {
+				if err := db.Sync(context.Background()); err != nil {
+					t.Fatal(err)
+				} else if err := r.Sync(context.Background()); err != nil {
+					t.Fatal(err)
+				}
+			}
+		}
+
+		// Ensure posistions match.
+		pos, err := db.Pos()
+		if err != nil {
+			t.Fatal(err)
+		} else if got, want := pos.Index, 2; got != want {
+			t.Fatalf("Index=%v, want %v", got, want)
+		}
+
+		if want, err := r.Pos(context.Background()); err != nil {
+			t.Fatal(err)
+		} else if got := pos; got != want {
+			t.Fatalf("Pos()=%v, want %v", got, want)
+		}
+	})
+
+	// Ensure replica returns an error if there is no generation available from the DB.
+	t.Run("ErrNoGeneration", func(t *testing.T) {
+		db, sqldb := MustOpenDBs(t)
+		defer MustCloseDBs(t, db, sqldb)
+
+		r := litestream.NewReplica(db, "", file.NewReplicaClient(t.TempDir()))
+		r.MonitorEnabled = false
+		db.Replicas = []*litestream.Replica{r}
+
+		if err := r.Sync(context.Background()); err == nil || err.Error() != `no generation, waiting for data` {
+			t.Fatal(err)
+		}
+	})
+}
+*/

gcs/replica_client.go

@@ -0,0 +1,422 @@
+package gcs
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"strings"
+	"sync"
+	"time"
+
+	"cloud.google.com/go/storage"
+	"github.com/benbjohnson/litestream"
+	"github.com/benbjohnson/litestream/internal"
+	"google.golang.org/api/iterator"
+)
+
+// ReplicaClientType is the client type for this package.
+const ReplicaClientType = "gcs"
+
+var _ litestream.ReplicaClient = (*ReplicaClient)(nil)
+
+// ReplicaClient is a client for writing snapshots & WAL segments to disk.
+type ReplicaClient struct {
+	mu     sync.Mutex
+	client *storage.Client       // gcs client
+	bkt    *storage.BucketHandle // gcs bucket handle
+
+	// GCS bucket information
+	Bucket string
+	Path   string
+}
+
+// NewReplicaClient returns a new instance of ReplicaClient.
+func NewReplicaClient() *ReplicaClient {
+	return &ReplicaClient{}
+}
+
+// Type returns "gcs" as the client type.
+func (c *ReplicaClient) Type() string {
+	return ReplicaClientType
+}
+
+// Init initializes the connection to GCS. No-op if already initialized.
+func (c *ReplicaClient) Init(ctx context.Context) (err error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if c.client != nil {
+		return nil
+	}
+
+	if c.client, err = storage.NewClient(ctx); err != nil {
+		return err
+	}
+	c.bkt = c.client.Bucket(c.Bucket)
+
+	return nil
+}
+
+// Generations returns a list of available generation names.
+func (c *ReplicaClient) Generations(ctx context.Context) ([]string, error) {
+	if err := c.Init(ctx); err != nil {
+		return nil, err
+	}
+
+	// Construct query to only pull generation directory names.
+	query := &storage.Query{
+		Delimiter: "/",
+		Prefix:    path.Join(c.Path, "generations") + "/",
+	}
+
+	// Loop over results and only build list of generation-formatted names.
+	it := c.bkt.Objects(ctx, query)
+	var generations []string
+	for {
+		attrs, err := it.Next()
+		if err == iterator.Done {
+			break
+		} else if err != nil {
+			return nil, err
+		}
+
+		name := path.Base(strings.TrimSuffix(attrs.Prefix, "/"))
+		if !litestream.IsGenerationName(name) {
+			continue
+		}
+		generations = append(generations, name)
+	}
+
+	return generations, nil
+}
+
+// DeleteGeneration deletes all snapshots & WAL segments within a generation.
+func (c *ReplicaClient) DeleteGeneration(ctx context.Context, generation string) error {
+	if err := c.Init(ctx); err != nil {
+		return err
+	} else if generation == "" {
+		return fmt.Errorf("generation required")
+	}
+
+	prefix := path.Join(c.Path, "generations", generation) + "/"
+
+	// Iterate over every object in generation and delete it.
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "LIST").Inc()
+	for it := c.bkt.Objects(ctx, &storage.Query{Prefix: prefix}); ; {
+		attrs, err := it.Next()
+		if err == iterator.Done {
+			break
+		} else if err != nil {
+			return err
+		}
+
+		if err := c.bkt.Object(attrs.Name).Delete(ctx); isNotExists(err) {
+			continue
+		} else if err != nil {
+			return fmt.Errorf("cannot delete object %q: %w", attrs.Name, err)
+		}
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "DELETE").Inc()
+	}
+
+	// log.Printf("%s(%s): retainer: deleting generation: %s", r.db.Path(), r.Name(), generation)
+
+	return nil
+}
+
+// Snapshots returns an iterator over all available snapshots for a generation.
+func (c *ReplicaClient) Snapshots(ctx context.Context, generation string) (litestream.SnapshotIterator, error) {
+	if err := c.Init(ctx); err != nil {
+		return nil, err
+	} else if generation == "" {
+		return nil, fmt.Errorf("generation required")
+	}
+	prefix := path.Join(c.Path, "generations", generation) + "/"
+	return newSnapshotIterator(generation, c.bkt.Objects(ctx, &storage.Query{Prefix: prefix})), nil
+}
+
+// WriteSnapshot writes LZ4 compressed data from rd to the object storage.
+func (c *ReplicaClient) WriteSnapshot(ctx context.Context, generation string, index int, rd io.Reader) (info litestream.SnapshotInfo, err error) {
+	if err := c.Init(ctx); err != nil {
+		return info, err
+	} else if generation == "" {
+		return info, fmt.Errorf("generation required")
+	}
+
+	key := path.Join(c.Path, "generations", generation, "snapshots", litestream.FormatIndex(index)+".snapshot.lz4")
+	startTime := time.Now()
+
+	w := c.bkt.Object(key).NewWriter(ctx)
+	defer w.Close()
+
+	n, err := io.Copy(w, rd)
+	if err != nil {
+		return info, err
+	} else if err := w.Close(); err != nil {
+		return info, err
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "PUT").Inc()
+	internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "PUT").Add(float64(n))
+
+	// log.Printf("%s(%s): snapshot: creating %s/%08x t=%s", r.db.Path(), r.Name(), generation, index, time.Since(startTime).Truncate(time.Millisecond))
+
+	return litestream.SnapshotInfo{
+		Generation: generation,
+		Index:      index,
+		Size:       n,
+		CreatedAt:  startTime.UTC(),
+	}, nil
+}
+
+// SnapshotReader returns a reader for snapshot data at the given generation/index.
+func (c *ReplicaClient) SnapshotReader(ctx context.Context, generation string, index int) (io.ReadCloser, error) {
+	if err := c.Init(ctx); err != nil {
+		return nil, err
+	} else if generation == "" {
+		return nil, fmt.Errorf("generation required")
+	}
+
+	key := path.Join(c.Path, "generations", generation, "snapshots", litestream.FormatIndex(index)+".snapshot.lz4")
+
+	r, err := c.bkt.Object(key).NewReader(ctx)
+	if isNotExists(err) {
+		return nil, os.ErrNotExist
+	} else if err != nil {
+		return nil, fmt.Errorf("cannot start new reader for %q: %w", key, err)
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "GET").Inc()
+	internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "GET").Add(float64(r.Attrs.Size))
+
+	return r, nil
+}
+
+// DeleteSnapshot deletes a snapshot with the given generation & index.
+func (c *ReplicaClient) DeleteSnapshot(ctx context.Context, generation string, index int) error {
+	if err := c.Init(ctx); err != nil {
+		return err
+	} else if generation == "" {
+		return fmt.Errorf("generation required")
+	}
+
+	key := path.Join(c.Path, "generations", generation, "snapshots", litestream.FormatIndex(index), ".snapshot.lz4")
+
+	if err := c.bkt.Object(key).Delete(ctx); err != nil && !isNotExists(err) {
+		return fmt.Errorf("cannot delete snapshot %q: %w", key, err)
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "DELETE").Inc()
+	return nil
+}
+
+// WALSegments returns an iterator over all available WAL files for a generation.
+func (c *ReplicaClient) WALSegments(ctx context.Context, generation string) (litestream.WALSegmentIterator, error) {
+	if err := c.Init(ctx); err != nil {
+		return nil, err
+	} else if generation == "" {
+		return nil, fmt.Errorf("generation required")
+	}
+	prefix := path.Join(c.Path, "generations", generation, "wal") + "/"
+	return newWALSegmentIterator(generation, prefix, c.bkt.Objects(ctx, &storage.Query{Prefix: prefix})), nil
+}
+
+// WriteWALSegment writes LZ4 compressed data from rd into a file on disk.
+func (c *ReplicaClient) WriteWALSegment(ctx context.Context, pos litestream.Pos, rd io.Reader) (info litestream.WALSegmentInfo, err error) {
+	if err := c.Init(ctx); err != nil {
+		return info, err
+	} else if pos.Generation == "" {
+		return info, fmt.Errorf("generation required")
+	}
+
+	key := path.Join(c.Path, "generations", pos.Generation, "wal", litestream.FormatIndex(pos.Index), litestream.FormatOffset(pos.Offset)+".wal.lz4")
+	startTime := time.Now()
+
+	w := c.bkt.Object(key).NewWriter(ctx)
+	defer w.Close()
+
+	n, err := io.Copy(w, rd)
+	if err != nil {
+		return info, err
+	} else if err := w.Close(); err != nil {
+		return info, err
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "PUT").Inc()
+	internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "PUT").Add(float64(n))
+
+	return litestream.WALSegmentInfo{
+		Generation: pos.Generation,
+		Index:      pos.Index,
+		Offset:     pos.Offset,
+		Size:       n,
+		CreatedAt:  startTime.UTC(),
+	}, nil
+}
+
+// WALSegmentReader returns a reader for a section of WAL data at the given index.
+// Returns os.ErrNotExist if no matching index/offset is found.
+func (c *ReplicaClient) WALSegmentReader(ctx context.Context, pos litestream.Pos) (io.ReadCloser, error) {
+	if err := c.Init(ctx); err != nil {
+		return nil, err
+	} else if pos.Generation == "" {
+		return nil, fmt.Errorf("generation required")
+	}
+
+	key := path.Join(c.Path, "generations", pos.Generation, "wal", litestream.FormatIndex(pos.Index), litestream.FormatOffset(pos.Offset)+".wal.lz4")
+
+	r, err := c.bkt.Object(key).NewReader(ctx)
+	if isNotExists(err) {
+		return nil, os.ErrNotExist
+	} else if err != nil {
+		return nil, err
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "GET").Inc()
+	internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "GET").Add(float64(r.Attrs.Size))
+
+	return r, nil
+}
+
+// DeleteWALSegments deletes WAL segments with at the given positions.
+func (c *ReplicaClient) DeleteWALSegments(ctx context.Context, a []litestream.Pos) error {
+	if err := c.Init(ctx); err != nil {
+		return err
+	}
+
+	for _, pos := range a {
+		if pos.Generation == "" {
+			return fmt.Errorf("generation required")
+		}
+
+		key := path.Join(c.Path, "generations", pos.Generation, "wal", litestream.FormatIndex(pos.Index), litestream.FormatOffset(pos.Offset)+".wal.lz4")
+		if err := c.bkt.Object(key).Delete(ctx); err != nil && !isNotExists(err) {
+			return fmt.Errorf("cannot delete wal segment %q: %w", key, err)
+		}
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "DELETE").Inc()
+	}
+
+	return nil
+}
+
+type snapshotIterator struct {
+	generation string
+
+	it   *storage.ObjectIterator
+	info litestream.SnapshotInfo
+	err  error
+}
+
+func newSnapshotIterator(generation string, it *storage.ObjectIterator) *snapshotIterator {
+	return &snapshotIterator{
+		generation: generation,
+		it:         it,
+	}
+}
+
+func (itr *snapshotIterator) Close() (err error) {
+	return itr.err
+}
+
+func (itr *snapshotIterator) Next() bool {
+	// Exit if an error has already occurred.
+	if itr.err != nil {
+		return false
+	}
+
+	for {
+		// Fetch next object.
+		attrs, err := itr.it.Next()
+		if err == iterator.Done {
+			return false
+		} else if err != nil {
+			itr.err = err
+			return false
+		}
+
+		// Parse index, otherwise skip to the next object.
+		index, err := internal.ParseSnapshotPath(path.Base(attrs.Name))
+		if err != nil {
+			continue
+		}
+
+		// Store current snapshot and return.
+		itr.info = litestream.SnapshotInfo{
+			Generation: itr.generation,
+			Index:      index,
+			Size:       attrs.Size,
+			CreatedAt:  attrs.Created.UTC(),
+		}
+		return true
+	}
+}
+
+func (itr *snapshotIterator) Err() error { return itr.err }
+
+func (itr *snapshotIterator) Snapshot() litestream.SnapshotInfo { return itr.info }
+
+type walSegmentIterator struct {
+	generation string
+	prefix     string
+
+	it   *storage.ObjectIterator
+	info litestream.WALSegmentInfo
+	err  error
+}
+
+func newWALSegmentIterator(generation, prefix string, it *storage.ObjectIterator) *walSegmentIterator {
+	return &walSegmentIterator{
+		generation: generation,
+		prefix:     prefix,
+		it:         it,
+	}
+}
+
+func (itr *walSegmentIterator) Close() (err error) {
+	return itr.err
+}
+
+func (itr *walSegmentIterator) Next() bool {
+	// Exit if an error has already occurred.
+	if itr.err != nil {
+		return false
+	}
+
+	for {
+		// Fetch next object.
+		attrs, err := itr.it.Next()
+		if err == iterator.Done {
+			return false
+		} else if err != nil {
+			itr.err = err
+			return false
+		}
+
+		// Parse index & offset, otherwise skip to the next object.
+		index, offset, err := internal.ParseWALSegmentPath(strings.TrimPrefix(attrs.Name, itr.prefix))
+		if err != nil {
+			continue
+		}
+
+		// Store current snapshot and return.
+		itr.info = litestream.WALSegmentInfo{
+			Generation: itr.generation,
+			Index:      index,
+			Offset:     offset,
+			Size:       attrs.Size,
+			CreatedAt:  attrs.Created.UTC(),
+		}
+		return true
+	}
+}
+
+func (itr *walSegmentIterator) Err() error { return itr.err }
+
+func (itr *walSegmentIterator) WALSegment() litestream.WALSegmentInfo {
+	return itr.info
+}
+
+func isNotExists(err error) bool {
+	return err == storage.ErrObjectNotExist
+}

go.mod

@@ -1,14 +1,21 @@
 module github.com/benbjohnson/litestream
 
-go 1.15
+go 1.16
 
 require (
+	cloud.google.com/go/storage v1.15.0
+	github.com/Azure/azure-storage-blob-go v0.13.0 // indirect
+	github.com/Azure/go-autorest/autorest v0.9.0 // indirect
 	github.com/aws/aws-sdk-go v1.27.0
 	github.com/davecgh/go-spew v1.1.1
+	github.com/mattn/go-shellwords v1.0.11 // indirect
 	github.com/mattn/go-sqlite3 v1.14.5
 	github.com/pierrec/lz4/v4 v4.1.3
+	github.com/pkg/sftp v1.13.0 // indirect
 	github.com/prometheus/client_golang v1.9.0
-	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
+	golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a // indirect
-	golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
+	golang.org/x/sys v0.0.0-20210412220455-f1c623a9e750
+	google.golang.org/api v0.45.0
 	gopkg.in/yaml.v2 v2.4.0
 )

go.sum

@@ -1,6 +1,62 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
+cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
+cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
+cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
+cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
+cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
+cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
+cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
+cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
+cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
+cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
+cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
+cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI=
+cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk=
+cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg=
+cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8=
+cloud.google.com/go v0.81.0 h1:at8Tk2zUz63cLPR0JPWm5vp77pEZmzxEQBEfRKn1VV8=
+cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0=
+cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
+cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
+cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
+cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
+cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
+cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
+cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
+cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
+cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
+cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
+cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
+cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
+cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
+cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
+cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
+cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
+cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
+cloud.google.com/go/storage v1.15.0 h1:Ljj+ZXVEhCr/1+4ZhvtteN1ND7UUsNTlduGclLh8GO0=
+cloud.google.com/go/storage v1.15.0/go.mod h1:mjjQMoxxyGH7Jr8K5qrx6N2O0AHsczI61sMNn03GIZI=
+dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U=
+github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k=
+github.com/Azure/azure-storage-blob-go v0.13.0 h1:lgWHvFh+UYBNVQLFHXkvul2f6yOPA9PIH82RTG2cSwc=
+github.com/Azure/azure-storage-blob-go v0.13.0/go.mod h1:pA9kNqtjUeQF2zOSu4s//nUdBD+e64lEuc4sVnuOfNs=
+github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
+github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI=
+github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0=
+github.com/Azure/go-autorest/autorest/adal v0.9.2/go.mod h1:/3SMAM86bP6wC9Ev35peQDUeqFZBMH07vvUOmg4z/fE=
+github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA=
+github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74=
+github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
+github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
+github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
+github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc=
+github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk=
+github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
 github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
 github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
@@ -31,8 +87,14 @@ github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QH
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
@@ -50,13 +112,21 @@ github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1
 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
 github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
 github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4=
 github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o=
@@ -72,18 +142,36 @@ github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zV
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
+github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
+github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
 github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
 github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
 github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
 github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
 github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
 github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM=
+github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
+github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
@@ -91,9 +179,36 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
+github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM=
+github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
 github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
@@ -124,6 +239,8 @@ github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2p
 github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg=
+github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
@@ -134,6 +251,9 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV
 github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o=
+github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
@@ -141,6 +261,8 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
+github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -149,9 +271,13 @@ github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-b
 github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
 github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-ieproxy v0.0.1 h1:qiyop7gCflfhwCzGyeT0gro3sF9AIg9HU98JORTkqfI=
+github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
+github.com/mattn/go-shellwords v1.0.11 h1:vCoR9VPpsk/TZFW2JwK5I9S0xdrtUq2bph6/YjEPnaw=
+github.com/mattn/go-shellwords v1.0.11/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y=
 github.com/mattn/go-sqlite3 v1.14.5 h1:1IdxlwTNazvbKJQSxoJ5/9ECbEeaTTyeU7sEAZ5KKTQ=
 github.com/mattn/go-sqlite3 v1.14.5/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
@@ -177,6 +303,7 @@ github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzE
 github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w=
 github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w=
 github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs=
 github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA=
 github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
@@ -204,8 +331,11 @@ github.com/pierrec/lz4/v4 v4.1.3 h1:/dvQpkb0o1pVlSgKNQqfkavlnXaIK+hJ0LXsKRUN9D4=
 github.com/pierrec/lz4/v4 v4.1.3/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA=
+github.com/pkg/sftp v1.13.0 h1:Riw6pgOKK41foc1I1Uu03CjvbLZDXeGpInycM4shXoI=
+github.com/pkg/sftp v1.13.0/go.mod h1:41g+FIPlQUTDCveupEmEA65IoiQFrtgCeDopC4ajGIM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
@@ -260,15 +390,28 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
+github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
 go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
 go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
+go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M=
+go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
@@ -280,17 +423,49 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf
 golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad h1:DN0cp81fZ3njFcrLCytUHRSUkqBjfTo4Tx9RJTWs0EY=
+golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
+golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a h1:kr2P4QFmQr29mSLA43kwrOcgcReGTfbE9N577tCTuBc=
+golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
+golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
+golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
+golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
+golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
+golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 h1:2M3HP5CCK1Si9FQhwnzYhXdG6DXeebvUHFpre8QvbyI=
+golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
+golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.1 h1:Kvvh58BN8Y9/lBi7hTekvtMpm07eUZ0ck5pRHpsMWrY=
+golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -303,19 +478,60 @@ golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4 h1:b0LrWgu8+q7z4J+0Y3Umo5q1dL7NXBkKBWkaVkAq17E=
+golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210413134643-5e61552d6c78 h1:rPRtHfUb0UKZeZ6GH4K4Nt4YRbE9V1u+QZX5upZXqJQ=
+golang.org/x/oauth2 v0.0.0-20210413134643-5e61552d6c78/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -327,21 +543,65 @@ golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e h1:AyodaIpKjppX+cBfTASF2E1US3H2JFBj920Ot3rtDjs=
 golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210412220455-f1c623a9e750 h1:ZBu6861dZq7xBnG1bn5SRU0vA8nx42at4+kP07FMTog=
+golang.org/x/sys v0.0.0-20210412220455-f1c623a9e750/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ=
+golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -349,46 +609,179 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
+golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE=
+golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY=
+golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
+google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
+google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
+google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg=
+google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE=
+google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8=
+google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU=
+google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94=
+google.golang.org/api v0.45.0 h1:pqMffJFLBVUDIoYsHcqtxgQVTsmxMDpYLOc5MT4Jrww=
+google.golang.org/api v0.45.0/go.mod h1:ISLIJCedJolbZvDfAk+Ctuq5hf+aJ33WgtUsfyFoLXA=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
+google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
+google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s=
+google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
+google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
+google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
+google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A=
+google.golang.org/genproto v0.0.0-20210413151531-c14fb6ef47c3/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A=
+google.golang.org/genproto v0.0.0-20210420162539-3c870d7478d2 h1:g2sJMUGCpeHZqTx8p3wsAWRS64nFq20i4dvJWcKGqvY=
+google.golang.org/genproto v0.0.0-20210420162539-3c870d7478d2/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A=
 google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
 google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
+google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
+google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
+google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8=
+google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
+google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
+google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
+google.golang.org/grpc v1.37.0 h1:uSZWeQJX5j11bIQ4AJoj+McDBo29cY1MCoC1wO3ts+c=
+google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
 google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
 google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM=
 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
@@ -404,9 +797,17 @@ gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
+rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
+rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
 sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU=

internal/internal.go

@@ -1,7 +1,15 @@
 package internal
 
 import (
+	"fmt"
 	"io"
+	"os"
+	"regexp"
+	"strconv"
+	"syscall"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 )
 
 // ReadCloser wraps a reader to also attach a separate closer.
@@ -30,3 +38,134 @@ func (r *ReadCloser) Close() error {
 	}
 	return r.c.Close()
 }
+
+// ReadCounter wraps an io.Reader and counts the total number of bytes read.
+type ReadCounter struct {
+	r io.Reader
+	n int64
+}
+
+// NewReadCounter returns a new instance of ReadCounter that wraps r.
+func NewReadCounter(r io.Reader) *ReadCounter {
+	return &ReadCounter{r: r}
+}
+
+// Read reads from the underlying reader into p and adds the bytes read to the counter.
+func (r *ReadCounter) Read(p []byte) (int, error) {
+	n, err := r.r.Read(p)
+	r.n += int64(n)
+	return n, err
+}
+
+// N returns the total number of bytes read.
+func (r *ReadCounter) N() int64 { return r.n }
+
+// CreateFile creates the file and matches the mode & uid/gid of fi.
+func CreateFile(filename string, fi os.FileInfo) (*os.File, error) {
+	mode := os.FileMode(0600)
+	if fi != nil {
+		mode = fi.Mode()
+	}
+
+	f, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode)
+	if err != nil {
+		return nil, err
+	}
+
+	uid, gid := Fileinfo(fi)
+	_ = f.Chown(uid, gid)
+	return f, nil
+}
+
+// MkdirAll is a copy of os.MkdirAll() except that it attempts to set the
+// mode/uid/gid to match fi for each created directory.
+func MkdirAll(path string, fi os.FileInfo) error {
+	uid, gid := Fileinfo(fi)
+
+	// Fast path: if we can tell whether path is a directory or file, stop with success or error.
+	dir, err := os.Stat(path)
+	if err == nil {
+		if dir.IsDir() {
+			return nil
+		}
+		return &os.PathError{Op: "mkdir", Path: path, Err: syscall.ENOTDIR}
+	}
+
+	// Slow path: make sure parent exists and then call Mkdir for path.
+	i := len(path)
+	for i > 0 && os.IsPathSeparator(path[i-1]) { // Skip trailing path separator.
+		i--
+	}
+
+	j := i
+	for j > 0 && !os.IsPathSeparator(path[j-1]) { // Scan backward over element.
+		j--
+	}
+
+	if j > 1 {
+		// Create parent.
+		err = MkdirAll(fixRootDirectory(path[:j-1]), fi)
+		if err != nil {
+			return err
+		}
+	}
+
+	// Parent now exists; invoke Mkdir and use its result.
+	mode := os.FileMode(0700)
+	if fi != nil {
+		mode = fi.Mode()
+	}
+	err = os.Mkdir(path, mode)
+	if err != nil {
+		// Handle arguments like "foo/." by
+		// double-checking that directory doesn't exist.
+		dir, err1 := os.Lstat(path)
+		if err1 == nil && dir.IsDir() {
+			_ = os.Chown(path, uid, gid)
+			return nil
+		}
+		return err
+	}
+	_ = os.Chown(path, uid, gid)
+	return nil
+}
+
+// ParseSnapshotPath parses the index from a snapshot filename. Used by path-based replicas.
+func ParseSnapshotPath(s string) (index int, err error) {
+	a := snapshotPathRegex.FindStringSubmatch(s)
+	if a == nil {
+		return 0, fmt.Errorf("invalid snapshot path")
+	}
+
+	i64, _ := strconv.ParseUint(a[1], 16, 64)
+	return int(i64), nil
+}
+
+var snapshotPathRegex = regexp.MustCompile(`^([0-9a-f]{8})\.snapshot\.lz4$`)
+
+// ParseWALSegmentPath parses the index/offset from a segment filename. Used by path-based replicas.
+func ParseWALSegmentPath(s string) (index int, offset int64, err error) {
+	a := walSegmentPathRegex.FindStringSubmatch(s)
+	if a == nil {
+		return 0, 0, fmt.Errorf("invalid wal segment path")
+	}
+
+	i64, _ := strconv.ParseUint(a[1], 16, 64)
+	off64, _ := strconv.ParseUint(a[2], 16, 64)
+	return int(i64), int64(off64), nil
+}
+
+var walSegmentPathRegex = regexp.MustCompile(`^([0-9a-f]{8})\/([0-9a-f]{8})\.wal\.lz4$`)
+
+// Shared replica metrics.
+var (
+	OperationTotalCounterVec = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: "litestream_replica_operation_total",
+		Help: "The number of replica operations performed",
+	}, []string{"replica_type", "operation"})
+
+	OperationBytesCounterVec = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: "litestream_replica_operation_bytes",
+		Help: "The number of bytes used by replica operations",
+	}, []string{"replica_type", "operation"})
+)

internal/internal_test.go

@@ -0,0 +1,61 @@
+package internal_test
+
+import (
+	"fmt"
+	"reflect"
+	"testing"
+
+	"github.com/benbjohnson/litestream/internal"
+)
+
+func TestParseSnapshotPath(t *testing.T) {
+	for _, tt := range []struct {
+		s     string
+		index int
+		err   error
+	}{
+		{"00bc614e.snapshot.lz4", 12345678, nil},
+		{"xxxxxxxx.snapshot.lz4", 0, fmt.Errorf("invalid snapshot path")},
+		{"00bc614.snapshot.lz4", 0, fmt.Errorf("invalid snapshot path")},
+		{"00bc614e.snapshot.lz", 0, fmt.Errorf("invalid snapshot path")},
+		{"00bc614e.snapshot", 0, fmt.Errorf("invalid snapshot path")},
+		{"00bc614e", 0, fmt.Errorf("invalid snapshot path")},
+		{"", 0, fmt.Errorf("invalid snapshot path")},
+	} {
+		t.Run("", func(t *testing.T) {
+			index, err := internal.ParseSnapshotPath(tt.s)
+			if got, want := index, tt.index; got != want {
+				t.Errorf("index=%#v, want %#v", got, want)
+			} else if got, want := err, tt.err; !reflect.DeepEqual(got, want) {
+				t.Errorf("err=%#v, want %#v", got, want)
+			}
+		})
+	}
+}
+
+func TestParseWALSegmentPath(t *testing.T) {
+	for _, tt := range []struct {
+		s      string
+		index  int
+		offset int64
+		err    error
+	}{
+		{"00bc614e/000003e8.wal.lz4", 12345678, 1000, nil},
+		{"00000000/00000000.wal", 0, 0, fmt.Errorf("invalid wal segment path")},
+		{"00000000/00000000", 0, 0, fmt.Errorf("invalid wal segment path")},
+		{"00000000/", 0, 0, fmt.Errorf("invalid wal segment path")},
+		{"00000000", 0, 0, fmt.Errorf("invalid wal segment path")},
+		{"", 0, 0, fmt.Errorf("invalid wal segment path")},
+	} {
+		t.Run("", func(t *testing.T) {
+			index, offset, err := internal.ParseWALSegmentPath(tt.s)
+			if got, want := index, tt.index; got != want {
+				t.Errorf("index=%#v, want %#v", got, want)
+			} else if got, want := offset, tt.offset; got != want {
+				t.Errorf("offset=%#v, want %#v", got, want)
+			} else if got, want := err, tt.err; !reflect.DeepEqual(got, want) {
+				t.Errorf("err=%#v, want %#v", got, want)
+			}
+		})
+	}
+}

internal/internal_unix.go

@@ -1,14 +1,17 @@
 // +build aix darwin dragonfly freebsd linux netbsd openbsd solaris
 
-package litestream
+package internal
 
 import (
 	"os"
 	"syscall"
 )
 
-// fileinfo returns syscall fields from a FileInfo object.
+// Fileinfo returns syscall fields from a FileInfo object.
-func fileinfo(fi os.FileInfo) (uid, gid int) {
+func Fileinfo(fi os.FileInfo) (uid, gid int) {
+	if fi == nil {
+		return -1, -1
+	}
 	stat := fi.Sys().(*syscall.Stat_t)
 	return int(stat.Uid), int(stat.Gid)
 }

internal/internal_windows.go

@@ -1,13 +1,13 @@
 // +build windows
 
-package litestream
+package internal
 
 import (
 	"os"
 )
 
-// fileinfo returns syscall fields from a FileInfo object.
+// Fileinfo returns syscall fields from a FileInfo object.
-func fileinfo(fi os.FileInfo) (uid, gid int) {
+func Fileinfo(fi os.FileInfo) (uid, gid int) {
 	return -1, -1
 }
 

internal/metrics.go

@@ -1,44 +0,0 @@
-package internal
-
-import (
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
-)
-
-// Shared replica metrics.
-var (
-	ReplicaSnapshotTotalGaugeVec = promauto.NewGaugeVec(prometheus.GaugeOpts{
-		Namespace: "litestream",
-		Subsystem: "replica",
-		Name:      "snapshot_total",
-		Help:      "The current number of snapshots",
-	}, []string{"db", "name"})
-
-	ReplicaWALBytesCounterVec = promauto.NewCounterVec(prometheus.CounterOpts{
-		Namespace: "litestream",
-		Subsystem: "replica",
-		Name:      "wal_bytes",
-		Help:      "The number wal bytes written",
-	}, []string{"db", "name"})
-
-	ReplicaWALIndexGaugeVec = promauto.NewGaugeVec(prometheus.GaugeOpts{
-		Namespace: "litestream",
-		Subsystem: "replica",
-		Name:      "wal_index",
-		Help:      "The current WAL index",
-	}, []string{"db", "name"})
-
-	ReplicaWALOffsetGaugeVec = promauto.NewGaugeVec(prometheus.GaugeOpts{
-		Namespace: "litestream",
-		Subsystem: "replica",
-		Name:      "wal_offset",
-		Help:      "The current WAL offset",
-	}, []string{"db", "name"})
-
-	ReplicaValidationTotalCounterVec = promauto.NewCounterVec(prometheus.CounterOpts{
-		Namespace: "litestream",
-		Subsystem: "replica",
-		Name:      "validation_total",
-		Help:      "The number of validations performed",
-	}, []string{"db", "name", "status"})
-)

litestream.go

@@ -8,10 +8,8 @@ import (
 	"io"
 	"os"
 	"path/filepath"
-	"regexp"
 	"strconv"
 	"strings"
-	"syscall"
 	"time"
 )
 
@@ -21,7 +19,8 @@ const (
 
 	WALDirName    = "wal"
 	WALExt        = ".wal"
-	SnapshotExt = ".snapshot"
+	WALSegmentExt = ".wal.lz4"
+	SnapshotExt   = ".snapshot.lz4"
 
 	GenerationNameLen = 16
 )
@@ -41,19 +40,160 @@ var (
 	ErrChecksumMismatch = errors.New("invalid replica, checksum mismatch")
 )
 
+// SnapshotIterator represents an iterator over a collection of snapshot metadata.
+type SnapshotIterator interface {
+	io.Closer
+
+	// Prepares the the next snapshot for reading with the Snapshot() method.
+	// Returns true if another snapshot is available. Returns false if no more
+	// snapshots are available or if an error occured.
+	Next() bool
+
+	// Returns an error that occurred during iteration.
+	Err() error
+
+	// Returns metadata for the currently positioned snapshot.
+	Snapshot() SnapshotInfo
+}
+
+// SliceSnapshotIterator returns all snapshots from an iterator as a slice.
+func SliceSnapshotIterator(itr SnapshotIterator) ([]SnapshotInfo, error) {
+	var a []SnapshotInfo
+	for itr.Next() {
+		a = append(a, itr.Snapshot())
+	}
+	return a, itr.Close()
+}
+
+var _ SnapshotIterator = (*SnapshotInfoSliceIterator)(nil)
+
+// SnapshotInfoSliceIterator represents an iterator for iterating over a slice of snapshots.
+type SnapshotInfoSliceIterator struct {
+	init bool
+	a    []SnapshotInfo
+}
+
+// NewSnapshotInfoSliceIterator returns a new instance of SnapshotInfoSliceIterator.
+func NewSnapshotInfoSliceIterator(a []SnapshotInfo) *SnapshotInfoSliceIterator {
+	return &SnapshotInfoSliceIterator{a: a}
+}
+
+// Close always returns nil.
+func (itr *SnapshotInfoSliceIterator) Close() error { return nil }
+
+// Next moves to the next snapshot. Returns true if another snapshot is available.
+func (itr *SnapshotInfoSliceIterator) Next() bool {
+	if !itr.init {
+		itr.init = true
+		return len(itr.a) > 0
+	}
+	itr.a = itr.a[1:]
+	return len(itr.a) > 0
+}
+
+// Err always returns nil.
+func (itr *SnapshotInfoSliceIterator) Err() error { return nil }
+
+// Snapshot returns the metadata from the currently positioned snapshot.
+func (itr *SnapshotInfoSliceIterator) Snapshot() SnapshotInfo {
+	if len(itr.a) == 0 {
+		return SnapshotInfo{}
+	}
+	return itr.a[0]
+}
+
+// WALSegmentIterator represents an iterator over a collection of WAL segments.
+type WALSegmentIterator interface {
+	io.Closer
+
+	// Prepares the the next WAL for reading with the WAL() method.
+	// Returns true if another WAL is available. Returns false if no more
+	// WAL files are available or if an error occured.
+	Next() bool
+
+	// Returns an error that occurred during iteration.
+	Err() error
+
+	// Returns metadata for the currently positioned WAL segment file.
+	WALSegment() WALSegmentInfo
+}
+
+// SliceWALSegmentIterator returns all WAL segment files from an iterator as a slice.
+func SliceWALSegmentIterator(itr WALSegmentIterator) ([]WALSegmentInfo, error) {
+	var a []WALSegmentInfo
+	for itr.Next() {
+		a = append(a, itr.WALSegment())
+	}
+	return a, itr.Close()
+}
+
+var _ WALSegmentIterator = (*WALSegmentInfoSliceIterator)(nil)
+
+// WALSegmentInfoSliceIterator represents an iterator for iterating over a slice of wal segments.
+type WALSegmentInfoSliceIterator struct {
+	init bool
+	a    []WALSegmentInfo
+}
+
+// NewWALSegmentInfoSliceIterator returns a new instance of WALSegmentInfoSliceIterator.
+func NewWALSegmentInfoSliceIterator(a []WALSegmentInfo) *WALSegmentInfoSliceIterator {
+	return &WALSegmentInfoSliceIterator{a: a}
+}
+
+// Close always returns nil.
+func (itr *WALSegmentInfoSliceIterator) Close() error { return nil }
+
+// Next moves to the next wal segment. Returns true if another segment is available.
+func (itr *WALSegmentInfoSliceIterator) Next() bool {
+	if !itr.init {
+		itr.init = true
+		return len(itr.a) > 0
+	}
+	itr.a = itr.a[1:]
+	return len(itr.a) > 0
+}
+
+// Err always returns nil.
+func (itr *WALSegmentInfoSliceIterator) Err() error { return nil }
+
+// WALSegment returns the metadata from the currently positioned wal segment.
+func (itr *WALSegmentInfoSliceIterator) WALSegment() WALSegmentInfo {
+	if len(itr.a) == 0 {
+		return WALSegmentInfo{}
+	}
+	return itr.a[0]
+}
+
 // SnapshotInfo represents file information about a snapshot.
 type SnapshotInfo struct {
-	Name       string
-	Replica    string
 	Generation string
 	Index      int
 	Size       int64
 	CreatedAt  time.Time
 }
 
+// Pos returns the WAL position when the snapshot was made.
+func (info *SnapshotInfo) Pos() Pos {
+	return Pos{Generation: info.Generation, Index: info.Index}
+}
+
+// SnapshotInfoSlice represents a slice of snapshot metadata.
+type SnapshotInfoSlice []SnapshotInfo
+
+func (a SnapshotInfoSlice) Len() int { return len(a) }
+
+func (a SnapshotInfoSlice) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+
+func (a SnapshotInfoSlice) Less(i, j int) bool {
+	if a[i].Generation != a[j].Generation {
+		return a[i].Generation < a[j].Generation
+	}
+	return a[i].Index < a[j].Index
+}
+
 // FilterSnapshotsAfter returns all snapshots that were created on or after t.
-func FilterSnapshotsAfter(a []*SnapshotInfo, t time.Time) []*SnapshotInfo {
+func FilterSnapshotsAfter(a []SnapshotInfo, t time.Time) []SnapshotInfo {
-	other := make([]*SnapshotInfo, 0, len(a))
+	other := make([]SnapshotInfo, 0, len(a))
 	for _, snapshot := range a {
 		if !snapshot.CreatedAt.Before(t) {
 			other = append(other, snapshot)
@@ -63,13 +203,13 @@ func FilterSnapshotsAfter(a []*SnapshotInfo, t time.Time) []*SnapshotInfo {
 }
 
 // FindMinSnapshotByGeneration finds the snapshot with the lowest index in a generation.
-func FindMinSnapshotByGeneration(a []*SnapshotInfo, generation string) *SnapshotInfo {
+func FindMinSnapshotByGeneration(a []SnapshotInfo, generation string) *SnapshotInfo {
 	var min *SnapshotInfo
 	for _, snapshot := range a {
 		if snapshot.Generation != generation {
 			continue
 		} else if min == nil || snapshot.Index < min.Index {
-			min = snapshot
+			min = &snapshot
 		}
 	}
 	return min
@@ -77,8 +217,27 @@ func FindMinSnapshotByGeneration(a []*SnapshotInfo, generation string) *Snapshot
 
 // WALInfo represents file information about a WAL file.
 type WALInfo struct {
-	Name       string
+	Generation string
-	Replica    string
+	Index      int
+	CreatedAt  time.Time
+}
+
+// WALInfoSlice represents a slice of WAL metadata.
+type WALInfoSlice []WALInfo
+
+func (a WALInfoSlice) Len() int { return len(a) }
+
+func (a WALInfoSlice) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+
+func (a WALInfoSlice) Less(i, j int) bool {
+	if a[i].Generation != a[j].Generation {
+		return a[i].Generation < a[j].Generation
+	}
+	return a[i].Index < a[j].Index
+}
+
+// WALSegmentInfo represents file information about a WAL segment file.
+type WALSegmentInfo struct {
 	Generation string
 	Index      int
 	Offset     int64
@@ -86,6 +245,27 @@ type WALInfo struct {
 	CreatedAt  time.Time
 }
 
+// Pos returns the WAL position when the segment was made.
+func (info *WALSegmentInfo) Pos() Pos {
+	return Pos{Generation: info.Generation, Index: info.Index, Offset: info.Offset}
+}
+
+// WALSegmentInfoSlice represents a slice of WAL segment metadata.
+type WALSegmentInfoSlice []WALSegmentInfo
+
+func (a WALSegmentInfoSlice) Len() int { return len(a) }
+
+func (a WALSegmentInfoSlice) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+
+func (a WALSegmentInfoSlice) Less(i, j int) bool {
+	if a[i].Generation != a[j].Generation {
+		return a[i].Generation < a[j].Generation
+	} else if a[i].Index != a[j].Index {
+		return a[i].Index < a[j].Index
+	}
+	return a[i].Offset < a[j].Offset
+}
+
 // Pos is a position in the WAL for a generation.
 type Pos struct {
 	Generation string // generation name
@@ -106,6 +286,11 @@ func (p Pos) IsZero() bool {
 	return p == (Pos{})
 }
 
+// Truncate returns p with the offset truncated to zero.
+func (p Pos) Truncate() Pos {
+	return Pos{Generation: p.Generation, Index: p.Index}
+}
+
 // Checksum computes a running SQLite checksum over a byte slice.
 func Checksum(bo binary.ByteOrder, s0, s1 uint32, b []byte) (uint32, uint32) {
 	assert(len(b)%8 == 0, "misaligned checksum byte slice")
@@ -197,124 +382,39 @@ func IsGenerationName(s string) bool {
 	return true
 }
 
-// IsSnapshotPath returns true if s is a path to a snapshot file.
+// FormatIndex formats an index as an 8-character hex value.
-func IsSnapshotPath(s string) bool {
+func FormatIndex(index int) string {
-	return snapshotPathRegex.MatchString(s)
+	return fmt.Sprintf("%08x", index)
 }
 
-// ParseSnapshotPath returns the index for the snapshot.
+// ParseIndex parses a hex-formatted index into an integer.
-// Returns an error if the path is not a valid snapshot path.
+func ParseIndex(s string) (int, error) {
-func ParseSnapshotPath(s string) (index int, ext string, err error) {
+	v, err := strconv.ParseUint(s, 16, 32)
-	s = filepath.Base(s)
+	if err != nil {
-
+		return -1, fmt.Errorf("cannot parse index: %q", s)
-	a := snapshotPathRegex.FindStringSubmatch(s)
+	}
-	if a == nil {
+	return int(v), nil
-		return 0, "", fmt.Errorf("invalid snapshot path: %s", s)
 }
 
-	i64, _ := strconv.ParseUint(a[1], 16, 64)
+// FormatOffset formats an offset as an 8-character hex value.
-	return int(i64), a[2], nil
+func FormatOffset(offset int64) string {
+	return fmt.Sprintf("%08x", offset)
 }
 
-var snapshotPathRegex = regexp.MustCompile(`^([0-9a-f]{8})(.snapshot(?:.lz4)?)$`)
+// ParseOffset parses a hex-formatted offset into an integer.
-
+func ParseOffset(s string) (int64, error) {
-// IsWALPath returns true if s is a path to a WAL file.
+	v, err := strconv.ParseInt(s, 16, 32)
-func IsWALPath(s string) bool {
+	if err != nil {
-	return walPathRegex.MatchString(s)
+		return -1, fmt.Errorf("cannot parse index: %q", s)
 	}
-
+	return v, nil
-// ParseWALPath returns the index & offset for the WAL file.
-// Returns an error if the path is not a valid snapshot path.
-func ParseWALPath(s string) (index int, offset int64, ext string, err error) {
-	s = filepath.Base(s)
-
-	a := walPathRegex.FindStringSubmatch(s)
-	if a == nil {
-		return 0, 0, "", fmt.Errorf("invalid wal path: %s", s)
 }
 
-	i64, _ := strconv.ParseUint(a[1], 16, 64)
-	off64, _ := strconv.ParseUint(a[2], 16, 64)
-	return int(i64), int64(off64), a[3], nil
-}
-
-// FormatWALPath formats a WAL filename with a given index.
-func FormatWALPath(index int) string {
-	assert(index >= 0, "wal index must be non-negative")
-	return fmt.Sprintf("%08x%s", index, WALExt)
-}
-
-// FormatWALPathWithOffset formats a WAL filename with a given index & offset.
-func FormatWALPathWithOffset(index int, offset int64) string {
-	assert(index >= 0, "wal index must be non-negative")
-	assert(offset >= 0, "wal offset must be non-negative")
-	return fmt.Sprintf("%08x_%08x%s", index, offset, WALExt)
-}
-
-var walPathRegex = regexp.MustCompile(`^([0-9a-f]{8})(?:_([0-9a-f]{8}))?(.wal(?:.lz4)?)$`)
-
 // isHexChar returns true if ch is a lowercase hex character.
 func isHexChar(ch rune) bool {
 	return (ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f')
 }
 
-// createFile creates the file and attempts to set the UID/GID.
-func createFile(filename string, perm os.FileMode, uid, gid int) (*os.File, error) {
-	f, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE|os.O_TRUNC, perm)
-	if err != nil {
-		return nil, err
-	}
-	_ = f.Chown(uid, gid)
-	return f, nil
-}
-
-// mkdirAll is a copy of os.MkdirAll() except that it attempts to set the
-// uid/gid for each created directory.
-func mkdirAll(path string, perm os.FileMode, uid, gid int) error {
-	// Fast path: if we can tell whether path is a directory or file, stop with success or error.
-	dir, err := os.Stat(path)
-	if err == nil {
-		if dir.IsDir() {
-			return nil
-		}
-		return &os.PathError{Op: "mkdir", Path: path, Err: syscall.ENOTDIR}
-	}
-
-	// Slow path: make sure parent exists and then call Mkdir for path.
-	i := len(path)
-	for i > 0 && os.IsPathSeparator(path[i-1]) { // Skip trailing path separator.
-		i--
-	}
-
-	j := i
-	for j > 0 && !os.IsPathSeparator(path[j-1]) { // Scan backward over element.
-		j--
-	}
-
-	if j > 1 {
-		// Create parent.
-		err = mkdirAll(fixRootDirectory(path[:j-1]), perm, uid, gid)
-		if err != nil {
-			return err
-		}
-	}
-
-	// Parent now exists; invoke Mkdir and use its result.
-	err = os.Mkdir(path, perm)
-	if err != nil {
-		// Handle arguments like "foo/." by
-		// double-checking that directory doesn't exist.
-		dir, err1 := os.Lstat(path)
-		if err1 == nil && dir.IsDir() {
-			_ = os.Chown(path, uid, gid)
-			return nil
-		}
-		return err
-	}
-	_ = os.Chown(path, uid, gid)
-	return nil
-}
-
 // Tracef is used for low-level tracing.
 var Tracef = func(format string, a ...interface{}) {}
 

mock/replica_client.go

@@ -0,0 +1,65 @@
+package mock
+
+import (
+	"context"
+	"io"
+
+	"github.com/benbjohnson/litestream"
+)
+
+var _ litestream.ReplicaClient = (*ReplicaClient)(nil)
+
+type ReplicaClient struct {
+	GenerationsFunc       func(ctx context.Context) ([]string, error)
+	DeleteGenerationFunc  func(ctx context.Context, generation string) error
+	SnapshotsFunc         func(ctx context.Context, generation string) (litestream.SnapshotIterator, error)
+	WriteSnapshotFunc     func(ctx context.Context, generation string, index int, r io.Reader) (litestream.SnapshotInfo, error)
+	DeleteSnapshotFunc    func(ctx context.Context, generation string, index int) error
+	SnapshotReaderFunc    func(ctx context.Context, generation string, index int) (io.ReadCloser, error)
+	WALSegmentsFunc       func(ctx context.Context, generation string) (litestream.WALSegmentIterator, error)
+	WriteWALSegmentFunc   func(ctx context.Context, pos litestream.Pos, r io.Reader) (litestream.WALSegmentInfo, error)
+	DeleteWALSegmentsFunc func(ctx context.Context, a []litestream.Pos) error
+	WALSegmentReaderFunc  func(ctx context.Context, pos litestream.Pos) (io.ReadCloser, error)
+}
+
+func (c *ReplicaClient) Type() string { return "mock" }
+
+func (c *ReplicaClient) Generations(ctx context.Context) ([]string, error) {
+	return c.GenerationsFunc(ctx)
+}
+
+func (c *ReplicaClient) DeleteGeneration(ctx context.Context, generation string) error {
+	return c.DeleteGenerationFunc(ctx, generation)
+}
+
+func (c *ReplicaClient) Snapshots(ctx context.Context, generation string) (litestream.SnapshotIterator, error) {
+	return c.SnapshotsFunc(ctx, generation)
+}
+
+func (c *ReplicaClient) WriteSnapshot(ctx context.Context, generation string, index int, r io.Reader) (litestream.SnapshotInfo, error) {
+	return c.WriteSnapshotFunc(ctx, generation, index, r)
+}
+
+func (c *ReplicaClient) DeleteSnapshot(ctx context.Context, generation string, index int) error {
+	return c.DeleteSnapshotFunc(ctx, generation, index)
+}
+
+func (c *ReplicaClient) SnapshotReader(ctx context.Context, generation string, index int) (io.ReadCloser, error) {
+	return c.SnapshotReaderFunc(ctx, generation, index)
+}
+
+func (c *ReplicaClient) WALSegments(ctx context.Context, generation string) (litestream.WALSegmentIterator, error) {
+	return c.WALSegmentsFunc(ctx, generation)
+}
+
+func (c *ReplicaClient) WriteWALSegment(ctx context.Context, pos litestream.Pos, r io.Reader) (litestream.WALSegmentInfo, error) {
+	return c.WriteWALSegmentFunc(ctx, pos, r)
+}
+
+func (c *ReplicaClient) DeleteWALSegments(ctx context.Context, a []litestream.Pos) error {
+	return c.DeleteWALSegmentsFunc(ctx, a)
+}
+
+func (c *ReplicaClient) WALSegmentReader(ctx context.Context, pos litestream.Pos) (io.ReadCloser, error) {
+	return c.WALSegmentReaderFunc(ctx, pos)
+}

replica_client.go

@@ -0,0 +1,48 @@
+package litestream
+
+import (
+	"context"
+	"io"
+)
+
+// ReplicaClient represents client to connect to a Replica.
+type ReplicaClient interface {
+	// Returns the type of client.
+	Type() string
+
+	// Returns a list of available generations.
+	Generations(ctx context.Context) ([]string, error)
+
+	// Deletes all snapshots & WAL segments within a generation.
+	DeleteGeneration(ctx context.Context, generation string) error
+
+	// Returns an iterator of all snapshots within a generation on the replica.
+	Snapshots(ctx context.Context, generation string) (SnapshotIterator, error)
+
+	// Writes LZ4 compressed snapshot data to the replica at a given index
+	// within a generation. Returns metadata for the snapshot.
+	WriteSnapshot(ctx context.Context, generation string, index int, r io.Reader) (SnapshotInfo, error)
+
+	// Deletes a snapshot with the given generation & index.
+	DeleteSnapshot(ctx context.Context, generation string, index int) error
+
+	// Returns a reader that contains LZ4 compressed snapshot data for a
+	// given index within a generation. Returns an os.ErrNotFound error if
+	// the snapshot does not exist.
+	SnapshotReader(ctx context.Context, generation string, index int) (io.ReadCloser, error)
+
+	// Returns an iterator of all WAL segments within a generation on the replica.
+	WALSegments(ctx context.Context, generation string) (WALSegmentIterator, error)
+
+	// Writes an LZ4 compressed WAL segment at a given position.
+	// Returns metadata for the written segment.
+	WriteWALSegment(ctx context.Context, pos Pos, r io.Reader) (WALSegmentInfo, error)
+
+	// Deletes one or more WAL segments at the given positions.
+	DeleteWALSegments(ctx context.Context, a []Pos) error
+
+	// Returns a reader that contains an LZ4 compressed WAL segment at a given
+	// index/offset within a generation. Returns an os.ErrNotFound error if the
+	// WAL segment does not exist.
+	WALSegmentReader(ctx context.Context, pos Pos) (io.ReadCloser, error)
+}

replica_client_test.go

@@ -0,0 +1,573 @@
+package litestream_test
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"path"
+	"reflect"
+	"sort"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/benbjohnson/litestream"
+	"github.com/benbjohnson/litestream/abs"
+	"github.com/benbjohnson/litestream/file"
+	"github.com/benbjohnson/litestream/gcs"
+	"github.com/benbjohnson/litestream/s3"
+	"github.com/benbjohnson/litestream/sftp"
+)
+
+func init() {
+	rand.Seed(time.Now().UnixNano())
+}
+
+var (
+	// Enables integration tests.
+	integration = flag.String("integration", "file", "")
+)
+
+// S3 settings
+var (
+	// Replica client settings
+	s3AccessKeyID     = flag.String("s3-access-key-id", os.Getenv("LITESTREAM_S3_ACCESS_KEY_ID"), "")
+	s3SecretAccessKey = flag.String("s3-secret-access-key", os.Getenv("LITESTREAM_S3_SECRET_ACCESS_KEY"), "")
+	s3Region          = flag.String("s3-region", os.Getenv("LITESTREAM_S3_REGION"), "")
+	s3Bucket          = flag.String("s3-bucket", os.Getenv("LITESTREAM_S3_BUCKET"), "")
+	s3Path            = flag.String("s3-path", os.Getenv("LITESTREAM_S3_PATH"), "")
+	s3Endpoint        = flag.String("s3-endpoint", os.Getenv("LITESTREAM_S3_ENDPOINT"), "")
+	s3ForcePathStyle  = flag.Bool("s3-force-path-style", os.Getenv("LITESTREAM_S3_FORCE_PATH_STYLE") == "true", "")
+	s3SkipVerify      = flag.Bool("s3-skip-verify", os.Getenv("LITESTREAM_S3_SKIP_VERIFY") == "true", "")
+)
+
+// Google cloud storage settings
+var (
+	gcsBucket = flag.String("gcs-bucket", os.Getenv("LITESTREAM_GCS_BUCKET"), "")
+	gcsPath   = flag.String("gcs-path", os.Getenv("LITESTREAM_GCS_PATH"), "")
+)
+
+// Azure blob storage settings
+var (
+	absAccountName = flag.String("abs-account-name", os.Getenv("LITESTREAM_ABS_ACCOUNT_NAME"), "")
+	absAccountKey  = flag.String("abs-account-key", os.Getenv("LITESTREAM_ABS_ACCOUNT_KEY"), "")
+	absBucket      = flag.String("abs-bucket", os.Getenv("LITESTREAM_ABS_BUCKET"), "")
+	absPath        = flag.String("abs-path", os.Getenv("LITESTREAM_ABS_PATH"), "")
+)
+
+// SFTP settings
+var (
+	sftpHost     = flag.String("sftp-host", os.Getenv("LITESTREAM_SFTP_HOST"), "")
+	sftpUser     = flag.String("sftp-user", os.Getenv("LITESTREAM_SFTP_USER"), "")
+	sftpPassword = flag.String("sftp-password", os.Getenv("LITESTREAM_SFTP_PASSWORD"), "")
+	sftpKeyPath  = flag.String("sftp-key-path", os.Getenv("LITESTREAM_SFTP_KEY_PATH"), "")
+	sftpPath     = flag.String("sftp-path", os.Getenv("LITESTREAM_SFTP_PATH"), "")
+)
+
+func TestReplicaClient_Generations(t *testing.T) {
+	RunWithReplicaClient(t, "OK", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+
+		// Write snapshots.
+		if _, err := c.WriteSnapshot(context.Background(), "5efbd8d042012dca", 0, strings.NewReader(`foo`)); err != nil {
+			t.Fatal(err)
+		} else if _, err := c.WriteSnapshot(context.Background(), "b16ddcf5c697540f", 0, strings.NewReader(`bar`)); err != nil {
+			t.Fatal(err)
+		} else if _, err := c.WriteSnapshot(context.Background(), "155fe292f8333c72", 0, strings.NewReader(`baz`)); err != nil {
+			t.Fatal(err)
+		}
+
+		// Verify returned generations.
+		if got, err := c.Generations(context.Background()); err != nil {
+			t.Fatal(err)
+		} else if want := []string{"155fe292f8333c72", "5efbd8d042012dca", "b16ddcf5c697540f"}; !reflect.DeepEqual(got, want) {
+			t.Fatalf("Generations()=%v, want %v", got, want)
+		}
+	})
+
+	RunWithReplicaClient(t, "NoGenerationsDir", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+
+		if generations, err := c.Generations(context.Background()); err != nil {
+			t.Fatal(err)
+		} else if got, want := len(generations), 0; got != want {
+			t.Fatalf("len(Generations())=%v, want %v", got, want)
+		}
+	})
+}
+
+func TestReplicaClient_Snapshots(t *testing.T) {
+	RunWithReplicaClient(t, "OK", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+
+		// Write snapshots.
+		if _, err := c.WriteSnapshot(context.Background(), "5efbd8d042012dca", 1, strings.NewReader(``)); err != nil {
+			t.Fatal(err)
+		} else if _, err := c.WriteSnapshot(context.Background(), "b16ddcf5c697540f", 5, strings.NewReader(`x`)); err != nil {
+			t.Fatal(err)
+		} else if _, err := c.WriteSnapshot(context.Background(), "b16ddcf5c697540f", 10, strings.NewReader(`xyz`)); err != nil {
+			t.Fatal(err)
+		}
+
+		// Fetch all snapshots by generation.
+		itr, err := c.Snapshots(context.Background(), "b16ddcf5c697540f")
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer itr.Close()
+
+		// Read all snapshots into a slice so they can be sorted.
+		a, err := litestream.SliceSnapshotIterator(itr)
+		if err != nil {
+			t.Fatal(err)
+		} else if got, want := len(a), 2; got != want {
+			t.Fatalf("len=%v, want %v", got, want)
+		}
+		sort.Sort(litestream.SnapshotInfoSlice(a))
+
+		// Verify first snapshot metadata.
+		if got, want := a[0].Generation, "b16ddcf5c697540f"; got != want {
+			t.Fatalf("Generation=%v, want %v", got, want)
+		} else if got, want := a[0].Index, 5; got != want {
+			t.Fatalf("Index=%v, want %v", got, want)
+		} else if got, want := a[0].Size, int64(1); got != want {
+			t.Fatalf("Size=%v, want %v", got, want)
+		} else if a[0].CreatedAt.IsZero() {
+			t.Fatalf("expected CreatedAt")
+		}
+
+		// Verify second snapshot metadata.
+		if got, want := a[1].Generation, "b16ddcf5c697540f"; got != want {
+			t.Fatalf("Generation=%v, want %v", got, want)
+		} else if got, want := a[1].Index, 0xA; got != want {
+			t.Fatalf("Index=%v, want %v", got, want)
+		} else if got, want := a[1].Size, int64(3); got != want {
+			t.Fatalf("Size=%v, want %v", got, want)
+		} else if a[1].CreatedAt.IsZero() {
+			t.Fatalf("expected CreatedAt")
+		}
+
+		// Ensure close is clean.
+		if err := itr.Close(); err != nil {
+			t.Fatal(err)
+		}
+	})
+
+	RunWithReplicaClient(t, "NoGenerationDir", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+
+		itr, err := c.Snapshots(context.Background(), "5efbd8d042012dca")
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer itr.Close()
+
+		if itr.Next() {
+			t.Fatal("expected no snapshots")
+		}
+	})
+
+	RunWithReplicaClient(t, "ErrNoGeneration", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+
+		itr, err := c.Snapshots(context.Background(), "")
+		if err == nil {
+			err = itr.Close()
+		}
+		if err == nil || err.Error() != `generation required` {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	})
+}
+
+func TestReplicaClient_WriteSnapshot(t *testing.T) {
+	RunWithReplicaClient(t, "OK", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+
+		if _, err := c.WriteSnapshot(context.Background(), "b16ddcf5c697540f", 1000, strings.NewReader(`foobar`)); err != nil {
+			t.Fatal(err)
+		}
+
+		if r, err := c.SnapshotReader(context.Background(), "b16ddcf5c697540f", 1000); err != nil {
+			t.Fatal(err)
+		} else if buf, err := ioutil.ReadAll(r); err != nil {
+			t.Fatal(err)
+		} else if err := r.Close(); err != nil {
+			t.Fatal(err)
+		} else if got, want := string(buf), `foobar`; got != want {
+			t.Fatalf("data=%q, want %q", got, want)
+		}
+	})
+
+	RunWithReplicaClient(t, "ErrNoGeneration", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+		if _, err := c.WriteSnapshot(context.Background(), "", 0, nil); err == nil || err.Error() != `generation required` {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	})
+}
+
+func TestReplicaClient_SnapshotReader(t *testing.T) {
+	RunWithReplicaClient(t, "OK", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+
+		if _, err := c.WriteSnapshot(context.Background(), "5efbd8d042012dca", 10, strings.NewReader(`foo`)); err != nil {
+			t.Fatal(err)
+		}
+
+		r, err := c.SnapshotReader(context.Background(), "5efbd8d042012dca", 10)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer r.Close()
+
+		if buf, err := ioutil.ReadAll(r); err != nil {
+			t.Fatal(err)
+		} else if got, want := string(buf), "foo"; got != want {
+			t.Fatalf("ReadAll=%v, want %v", got, want)
+		}
+	})
+
+	RunWithReplicaClient(t, "ErrNotFound", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+
+		if _, err := c.SnapshotReader(context.Background(), "5efbd8d042012dca", 1); !os.IsNotExist(err) {
+			t.Fatalf("expected not exist, got %#v", err)
+		}
+	})
+
+	RunWithReplicaClient(t, "ErrNoGeneration", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+
+		if _, err := c.SnapshotReader(context.Background(), "", 1); err == nil || err.Error() != `generation required` {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	})
+}
+
+func TestReplicaClient_WALSegments(t *testing.T) {
+	RunWithReplicaClient(t, "OK", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+
+		if _, err := c.WriteWALSegment(context.Background(), litestream.Pos{Generation: "5efbd8d042012dca", Index: 1, Offset: 0}, strings.NewReader(``)); err != nil {
+			t.Fatal(err)
+		}
+		if _, err := c.WriteWALSegment(context.Background(), litestream.Pos{Generation: "b16ddcf5c697540f", Index: 2, Offset: 0}, strings.NewReader(`12345`)); err != nil {
+			t.Fatal(err)
+		} else if _, err := c.WriteWALSegment(context.Background(), litestream.Pos{Generation: "b16ddcf5c697540f", Index: 2, Offset: 5}, strings.NewReader(`67`)); err != nil {
+			t.Fatal(err)
+		} else if _, err := c.WriteWALSegment(context.Background(), litestream.Pos{Generation: "b16ddcf5c697540f", Index: 3, Offset: 0}, strings.NewReader(`xyz`)); err != nil {
+			t.Fatal(err)
+		}
+
+		itr, err := c.WALSegments(context.Background(), "b16ddcf5c697540f")
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer itr.Close()
+
+		// Read all WAL segment files into a slice so they can be sorted.
+		a, err := litestream.SliceWALSegmentIterator(itr)
+		if err != nil {
+			t.Fatal(err)
+		} else if got, want := len(a), 3; got != want {
+			t.Fatalf("len=%v, want %v", got, want)
+		}
+		sort.Sort(litestream.WALSegmentInfoSlice(a))
+
+		// Verify first WAL segment metadata.
+		if got, want := a[0].Generation, "b16ddcf5c697540f"; got != want {
+			t.Fatalf("Generation=%v, want %v", got, want)
+		} else if got, want := a[0].Index, 2; got != want {
+			t.Fatalf("Index=%v, want %v", got, want)
+		} else if got, want := a[0].Offset, int64(0); got != want {
+			t.Fatalf("Offset=%v, want %v", got, want)
+		} else if got, want := a[0].Size, int64(5); got != want {
+			t.Fatalf("Size=%v, want %v", got, want)
+		} else if a[0].CreatedAt.IsZero() {
+			t.Fatalf("expected CreatedAt")
+		}
+
+		// Verify first WAL segment metadata.
+		if got, want := a[1].Generation, "b16ddcf5c697540f"; got != want {
+			t.Fatalf("Generation=%v, want %v", got, want)
+		} else if got, want := a[1].Index, 2; got != want {
+			t.Fatalf("Index=%v, want %v", got, want)
+		} else if got, want := a[1].Offset, int64(5); got != want {
+			t.Fatalf("Offset=%v, want %v", got, want)
+		} else if got, want := a[1].Size, int64(2); got != want {
+			t.Fatalf("Size=%v, want %v", got, want)
+		} else if a[1].CreatedAt.IsZero() {
+			t.Fatalf("expected CreatedAt")
+		}
+
+		// Verify third WAL segment metadata.
+		if got, want := a[2].Generation, "b16ddcf5c697540f"; got != want {
+			t.Fatalf("Generation=%v, want %v", got, want)
+		} else if got, want := a[2].Index, 3; got != want {
+			t.Fatalf("Index=%v, want %v", got, want)
+		} else if got, want := a[2].Offset, int64(0); got != want {
+			t.Fatalf("Offset=%v, want %v", got, want)
+		} else if got, want := a[2].Size, int64(3); got != want {
+			t.Fatalf("Size=%v, want %v", got, want)
+		} else if a[1].CreatedAt.IsZero() {
+			t.Fatalf("expected CreatedAt")
+		}
+
+		// Ensure close is clean.
+		if err := itr.Close(); err != nil {
+			t.Fatal(err)
+		}
+	})
+
+	RunWithReplicaClient(t, "NoGenerationDir", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+
+		itr, err := c.WALSegments(context.Background(), "5efbd8d042012dca")
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer itr.Close()
+
+		if itr.Next() {
+			t.Fatal("expected no wal files")
+		}
+	})
+
+	RunWithReplicaClient(t, "NoWALs", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+
+		if _, err := c.WriteSnapshot(context.Background(), "5efbd8d042012dca", 0, strings.NewReader(`foo`)); err != nil {
+			t.Fatal(err)
+		}
+
+		itr, err := c.WALSegments(context.Background(), "5efbd8d042012dca")
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer itr.Close()
+
+		if itr.Next() {
+			t.Fatal("expected no wal files")
+		}
+	})
+
+	RunWithReplicaClient(t, "ErrNoGeneration", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+
+		itr, err := c.WALSegments(context.Background(), "")
+		if err == nil {
+			err = itr.Close()
+		}
+		if err == nil || err.Error() != `generation required` {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	})
+}
+
+func TestReplicaClient_WriteWALSegment(t *testing.T) {
+	RunWithReplicaClient(t, "OK", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+
+		if _, err := c.WriteWALSegment(context.Background(), litestream.Pos{Generation: "b16ddcf5c697540f", Index: 1000, Offset: 2000}, strings.NewReader(`foobar`)); err != nil {
+			t.Fatal(err)
+		}
+
+		if r, err := c.WALSegmentReader(context.Background(), litestream.Pos{Generation: "b16ddcf5c697540f", Index: 1000, Offset: 2000}); err != nil {
+			t.Fatal(err)
+		} else if buf, err := ioutil.ReadAll(r); err != nil {
+			t.Fatal(err)
+		} else if err := r.Close(); err != nil {
+			t.Fatal(err)
+		} else if got, want := string(buf), `foobar`; got != want {
+			t.Fatalf("data=%q, want %q", got, want)
+		}
+	})
+
+	RunWithReplicaClient(t, "ErrNoGeneration", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+		if _, err := c.WriteWALSegment(context.Background(), litestream.Pos{Generation: "", Index: 0, Offset: 0}, nil); err == nil || err.Error() != `generation required` {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	})
+}
+
+func TestReplicaClient_WALSegmentReader(t *testing.T) {
+
+	RunWithReplicaClient(t, "OK", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+		if _, err := c.WriteWALSegment(context.Background(), litestream.Pos{Generation: "5efbd8d042012dca", Index: 10, Offset: 5}, strings.NewReader(`foobar`)); err != nil {
+			t.Fatal(err)
+		}
+
+		r, err := c.WALSegmentReader(context.Background(), litestream.Pos{Generation: "5efbd8d042012dca", Index: 10, Offset: 5})
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer r.Close()
+
+		if buf, err := ioutil.ReadAll(r); err != nil {
+			t.Fatal(err)
+		} else if got, want := string(buf), "foobar"; got != want {
+			t.Fatalf("ReadAll=%v, want %v", got, want)
+		}
+	})
+
+	RunWithReplicaClient(t, "ErrNotFound", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+
+		if _, err := c.WALSegmentReader(context.Background(), litestream.Pos{Generation: "5efbd8d042012dca", Index: 1, Offset: 0}); !os.IsNotExist(err) {
+			t.Fatalf("expected not exist, got %#v", err)
+		}
+	})
+}
+
+func TestReplicaClient_DeleteWALSegments(t *testing.T) {
+	RunWithReplicaClient(t, "OK", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+
+		if _, err := c.WriteWALSegment(context.Background(), litestream.Pos{Generation: "b16ddcf5c697540f", Index: 1, Offset: 2}, strings.NewReader(`foo`)); err != nil {
+			t.Fatal(err)
+		} else if _, err := c.WriteWALSegment(context.Background(), litestream.Pos{Generation: "5efbd8d042012dca", Index: 3, Offset: 4}, strings.NewReader(`bar`)); err != nil {
+			t.Fatal(err)
+		}
+
+		if err := c.DeleteWALSegments(context.Background(), []litestream.Pos{
+			{Generation: "b16ddcf5c697540f", Index: 1, Offset: 2},
+			{Generation: "5efbd8d042012dca", Index: 3, Offset: 4},
+		}); err != nil {
+			t.Fatal(err)
+		}
+
+		if _, err := c.WALSegmentReader(context.Background(), litestream.Pos{Generation: "b16ddcf5c697540f", Index: 1, Offset: 2}); !os.IsNotExist(err) {
+			t.Fatalf("expected not exist, got %#v", err)
+		} else if _, err := c.WALSegmentReader(context.Background(), litestream.Pos{Generation: "5efbd8d042012dca", Index: 3, Offset: 4}); !os.IsNotExist(err) {
+			t.Fatalf("expected not exist, got %#v", err)
+		}
+	})
+
+	RunWithReplicaClient(t, "ErrNoGeneration", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+		if err := c.DeleteWALSegments(context.Background(), []litestream.Pos{{}}); err == nil || err.Error() != `generation required` {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	})
+}
+
+// RunWithReplicaClient executes fn with each replica specified by the -integration flag
+func RunWithReplicaClient(t *testing.T, name string, fn func(*testing.T, litestream.ReplicaClient)) {
+	t.Run(name, func(t *testing.T) {
+		for _, typ := range strings.Split(*integration, ",") {
+			t.Run(typ, func(t *testing.T) {
+				c := NewReplicaClient(t, typ)
+				defer MustDeleteAll(t, c)
+
+				fn(t, c)
+			})
+		}
+	})
+}
+
+// NewReplicaClient returns a new client for integration testing by type name.
+func NewReplicaClient(tb testing.TB, typ string) litestream.ReplicaClient {
+	tb.Helper()
+
+	switch typ {
+	case file.ReplicaClientType:
+		return NewFileReplicaClient(tb)
+	case s3.ReplicaClientType:
+		return NewS3ReplicaClient(tb)
+	case gcs.ReplicaClientType:
+		return NewGCSReplicaClient(tb)
+	case abs.ReplicaClientType:
+		return NewABSReplicaClient(tb)
+	case sftp.ReplicaClientType:
+		return NewSFTPReplicaClient(tb)
+	default:
+		tb.Fatalf("invalid replica client type: %q", typ)
+		return nil
+	}
+}
+
+// NewFileReplicaClient returns a new client for integration testing.
+func NewFileReplicaClient(tb testing.TB) *file.ReplicaClient {
+	tb.Helper()
+	return file.NewReplicaClient(tb.TempDir())
+}
+
+// NewS3ReplicaClient returns a new client for integration testing.
+func NewS3ReplicaClient(tb testing.TB) *s3.ReplicaClient {
+	tb.Helper()
+
+	c := s3.NewReplicaClient()
+	c.AccessKeyID = *s3AccessKeyID
+	c.SecretAccessKey = *s3SecretAccessKey
+	c.Region = *s3Region
+	c.Bucket = *s3Bucket
+	c.Path = path.Join(*s3Path, fmt.Sprintf("%016x", rand.Uint64()))
+	c.Endpoint = *s3Endpoint
+	c.ForcePathStyle = *s3ForcePathStyle
+	c.SkipVerify = *s3SkipVerify
+	return c
+}
+
+// NewGCSReplicaClient returns a new client for integration testing.
+func NewGCSReplicaClient(tb testing.TB) *gcs.ReplicaClient {
+	tb.Helper()
+
+	c := gcs.NewReplicaClient()
+	c.Bucket = *gcsBucket
+	c.Path = path.Join(*gcsPath, fmt.Sprintf("%016x", rand.Uint64()))
+	return c
+}
+
+// NewABSReplicaClient returns a new client for integration testing.
+func NewABSReplicaClient(tb testing.TB) *abs.ReplicaClient {
+	tb.Helper()
+
+	c := abs.NewReplicaClient()
+	c.AccountName = *absAccountName
+	c.AccountKey = *absAccountKey
+	c.Bucket = *absBucket
+	c.Path = path.Join(*absPath, fmt.Sprintf("%016x", rand.Uint64()))
+	return c
+}
+
+// NewSFTPReplicaClient returns a new client for integration testing.
+func NewSFTPReplicaClient(tb testing.TB) *sftp.ReplicaClient {
+	tb.Helper()
+
+	c := sftp.NewReplicaClient()
+	c.Host = *sftpHost
+	c.User = *sftpUser
+	c.Password = *sftpPassword
+	c.KeyPath = *sftpKeyPath
+	c.Path = path.Join(*sftpPath, fmt.Sprintf("%016x", rand.Uint64()))
+	return c
+}
+
+// MustDeleteAll deletes all objects under the client's path.
+func MustDeleteAll(tb testing.TB, c litestream.ReplicaClient) {
+	tb.Helper()
+
+	generations, err := c.Generations(context.Background())
+	if err != nil {
+		tb.Fatalf("cannot list generations for deletion: %s", err)
+	}
+
+	for _, generation := range generations {
+		if err := c.DeleteGeneration(context.Background(), generation); err != nil {
+			tb.Fatalf("cannot delete generation: %s", err)
+		}
+	}
+
+	switch c := c.(type) {
+	case *sftp.ReplicaClient:
+		if err := c.Cleanup(context.Background()); err != nil {
+			tb.Fatalf("cannot cleanup sftp: %s", err)
+		}
+	}
+}

replica_test.go

@@ -1,90 +1,144 @@
 package litestream_test
 
 import (
+	"bytes"
 	"context"
+	"io"
+	"os"
 	"testing"
 
 	"github.com/benbjohnson/litestream"
+	"github.com/benbjohnson/litestream/file"
+	"github.com/benbjohnson/litestream/mock"
+	"github.com/pierrec/lz4/v4"
 )
 
-func TestFileReplica_Sync(t *testing.T) {
+func TestReplica_Name(t *testing.T) {
-	// Ensure replica can successfully sync after DB has sync'd.
+	t.Run("WithName", func(t *testing.T) {
-	t.Run("InitialSync", func(t *testing.T) {
+		if got, want := litestream.NewReplica(nil, "NAME").Name(), "NAME"; got != want {
-		db, sqldb := MustOpenDBs(t)
+			t.Fatalf("Name()=%v, want %v", got, want)
-		defer MustCloseDBs(t, db, sqldb)
-		r := NewTestFileReplica(t, db)
-
-		// Sync database & then sync replica.
-		if err := db.Sync(context.Background()); err != nil {
-			t.Fatal(err)
-		} else if err := r.Sync(context.Background()); err != nil {
-			t.Fatal(err)
-		}
-
-		// Ensure posistions match.
-		if pos, err := db.Pos(); err != nil {
-			t.Fatal(err)
-		} else if got, want := r.LastPos(), pos; got != want {
-			t.Fatalf("LastPos()=%v, want %v", got, want)
 		}
 	})
+	t.Run("WithoutName", func(t *testing.T) {
+		r := litestream.NewReplica(nil, "")
+		r.Client = &mock.ReplicaClient{}
+		if got, want := r.Name(), "mock"; got != want {
+			t.Fatalf("Name()=%v, want %v", got, want)
+		}
+	})
+}
 
-	// Ensure replica can successfully sync multiple times.
+func TestReplica_Sync(t *testing.T) {
-	t.Run("MultiSync", func(t *testing.T) {
 	db, sqldb := MustOpenDBs(t)
 	defer MustCloseDBs(t, db, sqldb)
-		r := NewTestFileReplica(t, db)
 
+	// Execute a query to force a write to the WAL.
 	if _, err := sqldb.Exec(`CREATE TABLE foo (bar TEXT);`); err != nil {
 		t.Fatal(err)
 	}
 
-		// Write to the database multiple times and sync after each write.
+	// Issue initial database sync to setup generation.
-		for i, n := 0, db.MinCheckpointPageN*2; i < n; i++ {
+	if err := db.Sync(context.Background()); err != nil {
-			if _, err := sqldb.Exec(`INSERT INTO foo (bar) VALUES ('baz')`); err != nil {
 		t.Fatal(err)
 	}
 
-			// Sync periodically.
+	// Fetch current database position.
-			if i%100 == 0 || i == n-1 {
+	dpos, err := db.Pos()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	c := file.NewReplicaClient(t.TempDir())
+	r := litestream.NewReplica(db, "")
+	c.Replica, r.Client = r, c
+
+	if err := r.Sync(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify client generation matches database.
+	generations, err := c.Generations(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	} else if got, want := len(generations), 1; got != want {
+		t.Fatalf("len(generations)=%v, want %v", got, want)
+	} else if got, want := generations[0], dpos.Generation; got != want {
+		t.Fatalf("generations[0]=%v, want %v", got, want)
+	}
+
+	// Verify WAL matches replica WAL.
+	if b0, err := os.ReadFile(db.Path() + "-wal"); err != nil {
+		t.Fatal(err)
+	} else if r, err := c.WALSegmentReader(context.Background(), litestream.Pos{Generation: generations[0], Index: 0, Offset: 0}); err != nil {
+		t.Fatal(err)
+	} else if b1, err := io.ReadAll(lz4.NewReader(r)); err != nil {
+		t.Fatal(err)
+	} else if err := r.Close(); err != nil {
+		t.Fatal(err)
+	} else if !bytes.Equal(b0, b1) {
+		t.Fatalf("wal mismatch: len(%d), len(%d)", len(b0), len(b1))
+	}
+}
+
+func TestReplica_Snapshot(t *testing.T) {
+	db, sqldb := MustOpenDBs(t)
+	defer MustCloseDBs(t, db, sqldb)
+
+	c := file.NewReplicaClient(t.TempDir())
+	r := litestream.NewReplica(db, "")
+	r.Client = c
+
+	// Execute a query to force a write to the WAL.
+	if _, err := sqldb.Exec(`CREATE TABLE foo (bar TEXT);`); err != nil {
+		t.Fatal(err)
+	} else if err := db.Sync(context.Background()); err != nil {
+		t.Fatal(err)
+	} else if err := r.Sync(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+
+	// Fetch current database position & snapshot.
+	pos0, err := db.Pos()
+	if err != nil {
+		t.Fatal(err)
+	} else if info, err := r.Snapshot(context.Background()); err != nil {
+		t.Fatal(err)
+	} else if got, want := info.Pos(), pos0.Truncate(); got != want {
+		t.Fatalf("pos=%s, want %s", got, want)
+	}
+
+	// Sync database and then replica.
 	if err := db.Sync(context.Background()); err != nil {
 		t.Fatal(err)
 	} else if err := r.Sync(context.Background()); err != nil {
 		t.Fatal(err)
 	}
-			}
-		}
 
-		// Ensure posistions match.
+	// Execute a query to force a write to the WAL & truncate to start new index.
-		if pos, err := db.Pos(); err != nil {
+	if _, err := sqldb.Exec(`INSERT INTO foo (bar) VALUES ('baz');`); err != nil {
 		t.Fatal(err)
-		} else if got, want := pos.Index, 2; got != want {
+	} else if err := db.Checkpoint(context.Background(), litestream.CheckpointModeTruncate); err != nil {
-			t.Fatalf("Index=%v, want %v", got, want)
-		} else if calcPos, err := r.CalcPos(context.Background(), pos.Generation); err != nil {
-			t.Fatal(err)
-		} else if got, want := calcPos, pos; got != want {
-			t.Fatalf("CalcPos()=%v, want %v", got, want)
-		} else if got, want := r.LastPos(), pos; got != want {
-			t.Fatalf("LastPos()=%v, want %v", got, want)
-		}
-	})
-
-	// Ensure replica returns an error if there is no generation available from the DB.
-	t.Run("ErrNoGeneration", func(t *testing.T) {
-		db, sqldb := MustOpenDBs(t)
-		defer MustCloseDBs(t, db, sqldb)
-		r := NewTestFileReplica(t, db)
-
-		if err := r.Sync(context.Background()); err == nil || err.Error() != `no generation, waiting for data` {
 		t.Fatal(err)
 	}
-	})
+
+	// Fetch current database position & snapshot.
+	pos1, err := db.Pos()
+	if err != nil {
+		t.Fatal(err)
+	} else if info, err := r.Snapshot(context.Background()); err != nil {
+		t.Fatal(err)
+	} else if got, want := info.Pos(), pos1.Truncate(); got != want {
+		t.Fatalf("pos=%v, want %v", got, want)
 	}
 
-// NewTestFileReplica returns a new replica using a temp directory & with monitoring disabled.
+	// Verify two snapshots exist.
-func NewTestFileReplica(tb testing.TB, db *litestream.DB) *litestream.FileReplica {
+	if infos, err := r.Snapshots(context.Background()); err != nil {
-	r := litestream.NewFileReplica(db, "", tb.TempDir())
+		t.Fatal(err)
-	r.MonitorEnabled = false
+	} else if got, want := len(infos), 2; got != want {
-	db.Replicas = []litestream.Replica{r}
+		t.Fatalf("len=%v, want %v", got, want)
-	return r
+	} else if got, want := infos[0].Pos(), pos0.Truncate(); got != want {
+		t.Fatalf("info[0]=%s, want %s", got, want)
+	} else if got, want := infos[1].Pos(), pos1.Truncate(); got != want {
+		t.Fatalf("info[1]=%s, want %s", got, want)
+	}
 }

s3/replica_client.go

@@ -0,0 +1,731 @@
+package s3
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"os"
+	"path"
+	"regexp"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/defaults"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+	"github.com/benbjohnson/litestream"
+	"github.com/benbjohnson/litestream/internal"
+	"golang.org/x/sync/errgroup"
+)
+
+// ReplicaClientType is the client type for this package.
+const ReplicaClientType = "s3"
+
+// MaxKeys is the number of keys S3 can operate on per batch.
+const MaxKeys = 1000
+
+// DefaultRegion is the region used if one is not specified.
+const DefaultRegion = "us-east-1"
+
+var _ litestream.ReplicaClient = (*ReplicaClient)(nil)
+
+// ReplicaClient is a client for writing snapshots & WAL segments to disk.
+type ReplicaClient struct {
+	mu       sync.Mutex
+	s3       *s3.S3 // s3 service
+	uploader *s3manager.Uploader
+
+	// AWS authentication keys.
+	AccessKeyID     string
+	SecretAccessKey string
+
+	// S3 bucket information
+	Region         string
+	Bucket         string
+	Path           string
+	Endpoint       string
+	ForcePathStyle bool
+	SkipVerify     bool
+}
+
+// NewReplicaClient returns a new instance of ReplicaClient.
+func NewReplicaClient() *ReplicaClient {
+	return &ReplicaClient{}
+}
+
+// Type returns "s3" as the client type.
+func (c *ReplicaClient) Type() string {
+	return ReplicaClientType
+}
+
+// Init initializes the connection to S3. No-op if already initialized.
+func (c *ReplicaClient) Init(ctx context.Context) (err error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if c.s3 != nil {
+		return nil
+	}
+
+	// Look up region if not specified and no endpoint is used.
+	// Endpoints are typically used for non-S3 object stores and do not
+	// necessarily require a region.
+	region := c.Region
+	if region == "" {
+		if c.Endpoint == "" {
+			if region, err = c.findBucketRegion(ctx, c.Bucket); err != nil {
+				return fmt.Errorf("cannot lookup bucket region: %w", err)
+			}
+		} else {
+			region = DefaultRegion // default for non-S3 object stores
+		}
+	}
+
+	// Create new AWS session.
+	config := c.config()
+	if region != "" {
+		config.Region = aws.String(region)
+	}
+	sess, err := session.NewSession(config)
+	if err != nil {
+		return fmt.Errorf("cannot create aws session: %w", err)
+	}
+	c.s3 = s3.New(sess)
+	c.uploader = s3manager.NewUploader(sess)
+	return nil
+}
+
+// config returns the AWS configuration. Uses the default credential chain
+// unless a key/secret are explicitly set.
+func (c *ReplicaClient) config() *aws.Config {
+	config := defaults.Get().Config
+	if c.AccessKeyID != "" || c.SecretAccessKey != "" {
+		config.Credentials = credentials.NewStaticCredentials(c.AccessKeyID, c.SecretAccessKey, "")
+	}
+	if c.Endpoint != "" {
+		config.Endpoint = aws.String(c.Endpoint)
+	}
+	if c.ForcePathStyle {
+		config.S3ForcePathStyle = aws.Bool(c.ForcePathStyle)
+	}
+	if c.SkipVerify {
+		config.HTTPClient = &http.Client{Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+		}}
+	}
+
+	return config
+}
+
+func (c *ReplicaClient) findBucketRegion(ctx context.Context, bucket string) (string, error) {
+	// Connect to US standard region to fetch info.
+	config := c.config()
+	config.Region = aws.String(DefaultRegion)
+	sess, err := session.NewSession(config)
+	if err != nil {
+		return "", err
+	}
+
+	// Fetch bucket location, if possible. Must be bucket owner.
+	// This call can return a nil location which means it's in us-east-1.
+	if out, err := s3.New(sess).GetBucketLocation(&s3.GetBucketLocationInput{
+		Bucket: aws.String(bucket),
+	}); err != nil {
+		return "", err
+	} else if out.LocationConstraint != nil {
+		return *out.LocationConstraint, nil
+	}
+	return DefaultRegion, nil
+}
+
+// Generations returns a list of available generation names.
+func (c *ReplicaClient) Generations(ctx context.Context) ([]string, error) {
+	if err := c.Init(ctx); err != nil {
+		return nil, err
+	}
+
+	var generations []string
+	if err := c.s3.ListObjectsPagesWithContext(ctx, &s3.ListObjectsInput{
+		Bucket:    aws.String(c.Bucket),
+		Prefix:    aws.String(path.Join(c.Path, "generations") + "/"),
+		Delimiter: aws.String("/"),
+	}, func(page *s3.ListObjectsOutput, lastPage bool) bool {
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "LIST").Inc()
+
+		for _, prefix := range page.CommonPrefixes {
+			name := path.Base(*prefix.Prefix)
+			if !litestream.IsGenerationName(name) {
+				continue
+			}
+			generations = append(generations, name)
+		}
+		return true
+	}); err != nil {
+		return nil, err
+	}
+
+	return generations, nil
+}
+
+// DeleteGeneration deletes all snapshots & WAL segments within a generation.
+func (c *ReplicaClient) DeleteGeneration(ctx context.Context, generation string) error {
+	if err := c.Init(ctx); err != nil {
+		return err
+	} else if generation == "" {
+		return fmt.Errorf("generation required")
+	}
+
+	// Collect all files for the generation.
+	var objIDs []*s3.ObjectIdentifier
+	if err := c.s3.ListObjectsPagesWithContext(ctx, &s3.ListObjectsInput{
+		Bucket: aws.String(c.Bucket),
+		Prefix: aws.String(path.Join(c.Path, "generations", generation)),
+	}, func(page *s3.ListObjectsOutput, lastPage bool) bool {
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "LIST").Inc()
+
+		for _, obj := range page.Contents {
+			objIDs = append(objIDs, &s3.ObjectIdentifier{Key: obj.Key})
+		}
+		return true
+	}); err != nil {
+		return err
+	}
+
+	// Delete all files in batches.
+	for len(objIDs) > 0 {
+		n := MaxKeys
+		if len(objIDs) < n {
+			n = len(objIDs)
+		}
+
+		if _, err := c.s3.DeleteObjectsWithContext(ctx, &s3.DeleteObjectsInput{
+			Bucket: aws.String(c.Bucket),
+			Delete: &s3.Delete{Objects: objIDs[:n], Quiet: aws.Bool(true)},
+		}); err != nil {
+			return err
+		}
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "DELETE").Inc()
+
+		objIDs = objIDs[n:]
+	}
+
+	// log.Printf("%s(%s): retainer: deleting generation: %s", r.db.Path(), r.Name(), generation)
+
+	return nil
+}
+
+// Snapshots returns an iterator over all available snapshots for a generation.
+func (c *ReplicaClient) Snapshots(ctx context.Context, generation string) (litestream.SnapshotIterator, error) {
+	if err := c.Init(ctx); err != nil {
+		return nil, err
+	}
+	return newSnapshotIterator(ctx, c, generation), nil
+}
+
+// WriteSnapshot writes LZ4 compressed data from rd into a file on disk.
+func (c *ReplicaClient) WriteSnapshot(ctx context.Context, generation string, index int, rd io.Reader) (info litestream.SnapshotInfo, err error) {
+	if err := c.Init(ctx); err != nil {
+		return info, err
+	} else if generation == "" {
+		return info, fmt.Errorf("generation required")
+	}
+
+	key := path.Join(c.Path, "generations", generation, "snapshots", litestream.FormatIndex(index)+".snapshot.lz4")
+	startTime := time.Now()
+
+	rc := internal.NewReadCounter(rd)
+	if _, err := c.uploader.UploadWithContext(ctx, &s3manager.UploadInput{
+		Bucket: aws.String(c.Bucket),
+		Key:    aws.String(key),
+		Body:   rc,
+	}); err != nil {
+		return info, err
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "PUT").Inc()
+	internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "PUT").Add(float64(rc.N()))
+
+	// log.Printf("%s(%s): snapshot: creating %s/%08x t=%s", r.db.Path(), r.Name(), generation, index, time.Since(startTime).Truncate(time.Millisecond))
+
+	return litestream.SnapshotInfo{
+		Generation: generation,
+		Index:      index,
+		Size:       rc.N(),
+		CreatedAt:  startTime.UTC(),
+	}, nil
+}
+
+// SnapshotReader returns a reader for snapshot data at the given generation/index.
+func (c *ReplicaClient) SnapshotReader(ctx context.Context, generation string, index int) (io.ReadCloser, error) {
+	if err := c.Init(ctx); err != nil {
+		return nil, err
+	} else if generation == "" {
+		return nil, fmt.Errorf("generation required")
+	}
+
+	key := path.Join(c.Path, "generations", generation, "snapshots", litestream.FormatIndex(index)+".snapshot.lz4")
+
+	out, err := c.s3.GetObjectWithContext(ctx, &s3.GetObjectInput{
+		Bucket: aws.String(c.Bucket),
+		Key:    aws.String(key),
+	})
+	if isNotExists(err) {
+		return nil, os.ErrNotExist
+	} else if err != nil {
+		return nil, err
+	}
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "GET").Inc()
+	internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "GET").Add(float64(*out.ContentLength))
+
+	return out.Body, nil
+}
+
+// DeleteSnapshot deletes a snapshot with the given generation & index.
+func (c *ReplicaClient) DeleteSnapshot(ctx context.Context, generation string, index int) error {
+	if err := c.Init(ctx); err != nil {
+		return err
+	} else if generation == "" {
+		return fmt.Errorf("generation required")
+	}
+
+	key := path.Join(c.Path, "generations", generation, "snapshots", litestream.FormatIndex(index)+".snapshot.lz4")
+
+	if _, err := c.s3.DeleteObjectsWithContext(ctx, &s3.DeleteObjectsInput{
+		Bucket: aws.String(c.Bucket),
+		Delete: &s3.Delete{Objects: []*s3.ObjectIdentifier{{Key: &key}}, Quiet: aws.Bool(true)},
+	}); err != nil {
+		return err
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "DELETE").Inc()
+	return nil
+}
+
+// WALSegments returns an iterator over all available WAL files for a generation.
+func (c *ReplicaClient) WALSegments(ctx context.Context, generation string) (litestream.WALSegmentIterator, error) {
+	if err := c.Init(ctx); err != nil {
+		return nil, err
+	}
+	return newWALSegmentIterator(ctx, c, generation), nil
+}
+
+// WriteWALSegment writes LZ4 compressed data from rd into a file on disk.
+func (c *ReplicaClient) WriteWALSegment(ctx context.Context, pos litestream.Pos, rd io.Reader) (info litestream.WALSegmentInfo, err error) {
+	if err := c.Init(ctx); err != nil {
+		return info, err
+	} else if pos.Generation == "" {
+		return info, fmt.Errorf("generation required")
+	}
+
+	key := path.Join(c.Path, "generations", pos.Generation, "wal", litestream.FormatIndex(pos.Index), litestream.FormatOffset(pos.Offset)+".wal.lz4")
+	startTime := time.Now()
+
+	rc := internal.NewReadCounter(rd)
+	if _, err := c.uploader.UploadWithContext(ctx, &s3manager.UploadInput{
+		Bucket: aws.String(c.Bucket),
+		Key:    aws.String(key),
+		Body:   rc,
+	}); err != nil {
+		return info, err
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "PUT").Inc()
+	internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "PUT").Add(float64(rc.N()))
+
+	return litestream.WALSegmentInfo{
+		Generation: pos.Generation,
+		Index:      pos.Index,
+		Offset:     pos.Offset,
+		Size:       rc.N(),
+		CreatedAt:  startTime.UTC(),
+	}, nil
+}
+
+// WALSegmentReader returns a reader for a section of WAL data at the given index.
+// Returns os.ErrNotExist if no matching index/offset is found.
+func (c *ReplicaClient) WALSegmentReader(ctx context.Context, pos litestream.Pos) (io.ReadCloser, error) {
+	if err := c.Init(ctx); err != nil {
+		return nil, err
+	} else if pos.Generation == "" {
+		return nil, fmt.Errorf("generation required")
+	}
+
+	key := path.Join(c.Path, "generations", pos.Generation, "wal", litestream.FormatIndex(pos.Index), litestream.FormatOffset(pos.Offset)+".wal.lz4")
+
+	out, err := c.s3.GetObjectWithContext(ctx, &s3.GetObjectInput{
+		Bucket: aws.String(c.Bucket),
+		Key:    aws.String(key),
+	})
+	if isNotExists(err) {
+		return nil, os.ErrNotExist
+	} else if err != nil {
+		return nil, err
+	}
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "GET").Inc()
+	internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "GET").Add(float64(*out.ContentLength))
+
+	return out.Body, nil
+}
+
+// DeleteWALSegments deletes WAL segments with at the given positions.
+func (c *ReplicaClient) DeleteWALSegments(ctx context.Context, a []litestream.Pos) error {
+	if err := c.Init(ctx); err != nil {
+		return err
+	}
+
+	objIDs := make([]*s3.ObjectIdentifier, MaxKeys)
+	for len(a) > 0 {
+		n := MaxKeys
+		if len(a) < n {
+			n = len(a)
+		}
+
+		// Generate a batch of object IDs for deleting the WAL segments.
+		for i, pos := range a[:n] {
+			if pos.Generation == "" {
+				return fmt.Errorf("generation required")
+			}
+			key := path.Join(c.Path, "generations", pos.Generation, "wal", litestream.FormatIndex(pos.Index), litestream.FormatOffset(pos.Offset)+".wal.lz4")
+			objIDs[i] = &s3.ObjectIdentifier{Key: &key}
+		}
+
+		// Delete S3 objects in bulk.
+		if _, err := c.s3.DeleteObjectsWithContext(ctx, &s3.DeleteObjectsInput{
+			Bucket: aws.String(c.Bucket),
+			Delete: &s3.Delete{Objects: objIDs[:n], Quiet: aws.Bool(true)},
+		}); err != nil {
+			return err
+		}
+
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "DELETE").Inc()
+
+		a = a[n:]
+	}
+
+	return nil
+}
+
+// DeleteAll deletes everything on the remote path. Mainly used for testing.
+func (c *ReplicaClient) DeleteAll(ctx context.Context) error {
+	if err := c.Init(ctx); err != nil {
+		return err
+	}
+
+	prefix := c.Path
+	if prefix != "" {
+		prefix += "/"
+	}
+
+	// Collect all files for the generation.
+	var objIDs []*s3.ObjectIdentifier
+	if err := c.s3.ListObjectsPagesWithContext(ctx, &s3.ListObjectsInput{
+		Bucket: aws.String(c.Bucket),
+		Prefix: aws.String(prefix),
+	}, func(page *s3.ListObjectsOutput, lastPage bool) bool {
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "LIST").Inc()
+
+		for _, obj := range page.Contents {
+			objIDs = append(objIDs, &s3.ObjectIdentifier{Key: obj.Key})
+		}
+		return true
+	}); err != nil {
+		return err
+	}
+
+	// Delete all files in batches.
+	for len(objIDs) > 0 {
+		n := MaxKeys
+		if len(objIDs) < n {
+			n = len(objIDs)
+		}
+
+		if _, err := c.s3.DeleteObjectsWithContext(ctx, &s3.DeleteObjectsInput{
+			Bucket: aws.String(c.Bucket),
+			Delete: &s3.Delete{Objects: objIDs[:n], Quiet: aws.Bool(true)},
+		}); err != nil {
+			return err
+		}
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "DELETE").Inc()
+
+		objIDs = objIDs[n:]
+	}
+
+	return nil
+}
+
+type snapshotIterator struct {
+	client     *ReplicaClient
+	generation string
+
+	ch     chan litestream.SnapshotInfo
+	g      errgroup.Group
+	ctx    context.Context
+	cancel func()
+
+	info litestream.SnapshotInfo
+	err  error
+}
+
+func newSnapshotIterator(ctx context.Context, client *ReplicaClient, generation string) *snapshotIterator {
+	itr := &snapshotIterator{
+		client:     client,
+		generation: generation,
+		ch:         make(chan litestream.SnapshotInfo),
+	}
+
+	itr.ctx, itr.cancel = context.WithCancel(ctx)
+	itr.g.Go(itr.fetch)
+
+	return itr
+}
+
+// fetch runs in a separate goroutine to fetch pages of objects and stream them to a channel.
+func (itr *snapshotIterator) fetch() error {
+	defer close(itr.ch)
+
+	if itr.generation == "" {
+		return fmt.Errorf("generation required")
+	}
+
+	dir := path.Join(itr.client.Path, "generations", itr.generation, "snapshots")
+
+	return itr.client.s3.ListObjectsPagesWithContext(itr.ctx, &s3.ListObjectsInput{
+		Bucket:    aws.String(itr.client.Bucket),
+		Prefix:    aws.String(dir + "/"),
+		Delimiter: aws.String("/"),
+	}, func(page *s3.ListObjectsOutput, lastPage bool) bool {
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "LIST").Inc()
+
+		for _, obj := range page.Contents {
+			index, err := internal.ParseSnapshotPath(path.Base(*obj.Key))
+			if err != nil {
+				continue
+			}
+
+			info := litestream.SnapshotInfo{
+				Generation: itr.generation,
+				Index:      index,
+				Size:       *obj.Size,
+				CreatedAt:  obj.LastModified.UTC(),
+			}
+
+			select {
+			case <-itr.ctx.Done():
+			case itr.ch <- info:
+			}
+		}
+		return true
+	})
+}
+
+func (itr *snapshotIterator) Close() (err error) {
+	err = itr.err
+
+	// Cancel context and wait for error group to finish.
+	itr.cancel()
+	if e := itr.g.Wait(); e != nil && err == nil {
+		err = e
+	}
+
+	return err
+}
+
+func (itr *snapshotIterator) Next() bool {
+	// Exit if an error has already occurred.
+	if itr.err != nil {
+		return false
+	}
+
+	// Return false if context was canceled or if there are no more snapshots.
+	// Otherwise fetch the next snapshot and store it on the iterator.
+	select {
+	case <-itr.ctx.Done():
+		return false
+	case info, ok := <-itr.ch:
+		if !ok {
+			return false
+		}
+		itr.info = info
+		return true
+	}
+}
+
+func (itr *snapshotIterator) Err() error { return itr.err }
+
+func (itr *snapshotIterator) Snapshot() litestream.SnapshotInfo {
+	return itr.info
+}
+
+type walSegmentIterator struct {
+	client     *ReplicaClient
+	generation string
+
+	ch     chan litestream.WALSegmentInfo
+	g      errgroup.Group
+	ctx    context.Context
+	cancel func()
+
+	info litestream.WALSegmentInfo
+	err  error
+}
+
+func newWALSegmentIterator(ctx context.Context, client *ReplicaClient, generation string) *walSegmentIterator {
+	itr := &walSegmentIterator{
+		client:     client,
+		generation: generation,
+		ch:         make(chan litestream.WALSegmentInfo),
+	}
+
+	itr.ctx, itr.cancel = context.WithCancel(ctx)
+	itr.g.Go(itr.fetch)
+
+	return itr
+}
+
+// fetch runs in a separate goroutine to fetch pages of objects and stream them to a channel.
+func (itr *walSegmentIterator) fetch() error {
+	defer close(itr.ch)
+
+	if itr.generation == "" {
+		return fmt.Errorf("generation required")
+	}
+
+	prefix := path.Join(itr.client.Path, "generations", itr.generation, "wal") + "/"
+
+	return itr.client.s3.ListObjectsPagesWithContext(itr.ctx, &s3.ListObjectsInput{
+		Bucket: aws.String(itr.client.Bucket),
+		Prefix: aws.String(prefix),
+	}, func(page *s3.ListObjectsOutput, lastPage bool) bool {
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "LIST").Inc()
+
+		for _, obj := range page.Contents {
+			index, offset, err := internal.ParseWALSegmentPath(strings.TrimPrefix(*obj.Key, prefix))
+			if err != nil {
+				continue
+			}
+
+			info := litestream.WALSegmentInfo{
+				Generation: itr.generation,
+				Index:      index,
+				Offset:     offset,
+				Size:       *obj.Size,
+				CreatedAt:  obj.LastModified.UTC(),
+			}
+
+			select {
+			case <-itr.ctx.Done():
+				return false
+			case itr.ch <- info:
+			}
+		}
+		return true
+	})
+}
+
+func (itr *walSegmentIterator) Close() (err error) {
+	err = itr.err
+
+	// Cancel context and wait for error group to finish.
+	itr.cancel()
+	if e := itr.g.Wait(); e != nil && err == nil {
+		err = e
+	}
+
+	return err
+}
+
+func (itr *walSegmentIterator) Next() bool {
+	// Exit if an error has already occurred.
+	if itr.err != nil {
+		return false
+	}
+
+	// Return false if context was canceled or if there are no more segments.
+	// Otherwise fetch the next segment and store it on the iterator.
+	select {
+	case <-itr.ctx.Done():
+		return false
+	case info, ok := <-itr.ch:
+		if !ok {
+			return false
+		}
+		itr.info = info
+		return true
+	}
+}
+
+func (itr *walSegmentIterator) Err() error { return itr.err }
+
+func (itr *walSegmentIterator) WALSegment() litestream.WALSegmentInfo {
+	return itr.info
+}
+
+// ParseHost extracts data from a hostname depending on the service provider.
+func ParseHost(s string) (bucket, region, endpoint string, forcePathStyle bool) {
+	// Extract port if one is specified.
+	host, port, err := net.SplitHostPort(s)
+	if err != nil {
+		host = s
+	}
+
+	// Default to path-based URLs, except for with AWS S3 itself.
+	forcePathStyle = true
+
+	// Extract fields from provider-specific host formats.
+	scheme := "https"
+	if a := localhostRegex.FindStringSubmatch(host); a != nil {
+		bucket, region = a[1], "us-east-1"
+		scheme, endpoint = "http", "localhost"
+	} else if a := backblazeRegex.FindStringSubmatch(host); a != nil {
+		bucket, region = a[1], a[2]
+		endpoint = fmt.Sprintf("s3.%s.backblazeb2.com", region)
+	} else if a := filebaseRegex.FindStringSubmatch(host); a != nil {
+		bucket, endpoint = a[1], "s3.filebase.com"
+	} else if a := digitalOceanRegex.FindStringSubmatch(host); a != nil {
+		bucket, region = a[1], a[2]
+		endpoint = fmt.Sprintf("%s.digitaloceanspaces.com", region)
+	} else if a := linodeRegex.FindStringSubmatch(host); a != nil {
+		bucket, region = a[1], a[2]
+		endpoint = fmt.Sprintf("%s.linodeobjects.com", region)
+	} else {
+		bucket = host
+		forcePathStyle = false
+	}
+
+	// Add port back to endpoint, if available.
+	if endpoint != "" && port != "" {
+		endpoint = net.JoinHostPort(endpoint, port)
+	}
+
+	// Prepend scheme to endpoint.
+	if endpoint != "" {
+		endpoint = scheme + "://" + endpoint
+	}
+
+	return bucket, region, endpoint, forcePathStyle
+}
+
+var (
+	localhostRegex    = regexp.MustCompile(`^(?:(.+)\.)?localhost$`)
+	backblazeRegex    = regexp.MustCompile(`^(?:(.+)\.)?s3.([^.]+)\.backblazeb2.com$`)
+	filebaseRegex     = regexp.MustCompile(`^(?:(.+)\.)?s3.filebase.com$`)
+	digitalOceanRegex = regexp.MustCompile(`^(?:(.+)\.)?([^.]+)\.digitaloceanspaces.com$`)
+	linodeRegex       = regexp.MustCompile(`^(?:(.+)\.)?([^.]+)\.linodeobjects.com$`)
+)
+
+func isNotExists(err error) bool {
+	switch err := err.(type) {
+	case awserr.Error:
+		return err.Code() == `NoSuchKey`
+	default:
+		return false
+	}
+}

s3/s3_test.go

@@ -1,80 +0,0 @@
-package s3_test
-
-import (
-	"testing"
-
-	"github.com/benbjohnson/litestream/s3"
-)
-
-func TestParseHost(t *testing.T) {
-	// Ensure non-specific hosts return as buckets.
-	t.Run("S3", func(t *testing.T) {
-		bucket, region, endpoint, forcePathStyle := s3.ParseHost(`test.litestream.io`)
-		if got, want := bucket, `test.litestream.io`; got != want {
-			t.Fatalf("bucket=%q, want %q", got, want)
-		} else if got, want := region, ``; got != want {
-			t.Fatalf("region=%q, want %q", got, want)
-		} else if got, want := endpoint, ``; got != want {
-			t.Fatalf("endpoint=%q, want %q", got, want)
-		} else if got, want := forcePathStyle, false; got != want {
-			t.Fatalf("forcePathStyle=%v, want %v", got, want)
-		}
-	})
-
-	// Ensure localhosts use an HTTP endpoint and extract the bucket name.
-	t.Run("Localhost", func(t *testing.T) {
-		t.Run("WithPort", func(t *testing.T) {
-			bucket, region, endpoint, forcePathStyle := s3.ParseHost(`test.localhost:9000`)
-			if got, want := bucket, `test`; got != want {
-				t.Fatalf("bucket=%q, want %q", got, want)
-			} else if got, want := region, `us-east-1`; got != want {
-				t.Fatalf("region=%q, want %q", got, want)
-			} else if got, want := endpoint, `http://localhost:9000`; got != want {
-				t.Fatalf("endpoint=%q, want %q", got, want)
-			} else if got, want := forcePathStyle, true; got != want {
-				t.Fatalf("forcePathStyle=%v, want %v", got, want)
-			}
-		})
-
-		t.Run("WithoutPort", func(t *testing.T) {
-			bucket, region, endpoint, forcePathStyle := s3.ParseHost(`test.localhost`)
-			if got, want := bucket, `test`; got != want {
-				t.Fatalf("bucket=%q, want %q", got, want)
-			} else if got, want := region, `us-east-1`; got != want {
-				t.Fatalf("region=%q, want %q", got, want)
-			} else if got, want := endpoint, `http://localhost`; got != want {
-				t.Fatalf("endpoint=%q, want %q", got, want)
-			} else if got, want := forcePathStyle, true; got != want {
-				t.Fatalf("forcePathStyle=%v, want %v", got, want)
-			}
-		})
-	})
-
-	// Ensure backblaze B2 URLs extract bucket, region, & endpoint from host.
-	t.Run("Backblaze", func(t *testing.T) {
-		bucket, region, endpoint, forcePathStyle := s3.ParseHost(`test-123.s3.us-west-000.backblazeb2.com`)
-		if got, want := bucket, `test-123`; got != want {
-			t.Fatalf("bucket=%q, want %q", got, want)
-		} else if got, want := region, `us-west-000`; got != want {
-			t.Fatalf("region=%q, want %q", got, want)
-		} else if got, want := endpoint, `https://s3.us-west-000.backblazeb2.com`; got != want {
-			t.Fatalf("endpoint=%q, want %q", got, want)
-		} else if got, want := forcePathStyle, true; got != want {
-			t.Fatalf("forcePathStyle=%v, want %v", got, want)
-		}
-	})
-
-	// Ensure GCS URLs extract bucket & endpoint from host.
-	t.Run("GCS", func(t *testing.T) {
-		bucket, region, endpoint, forcePathStyle := s3.ParseHost(`litestream.io.storage.googleapis.com`)
-		if got, want := bucket, `litestream.io`; got != want {
-			t.Fatalf("bucket=%q, want %q", got, want)
-		} else if got, want := region, `us-east-1`; got != want {
-			t.Fatalf("region=%q, want %q", got, want)
-		} else if got, want := endpoint, `https://storage.googleapis.com`; got != want {
-			t.Fatalf("endpoint=%q, want %q", got, want)
-		} else if got, want := forcePathStyle, true; got != want {
-			t.Fatalf("forcePathStyle=%v, want %v", got, want)
-		}
-	})
-}

sftp/replica_client.go

@@ -0,0 +1,580 @@
+package sftp
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"os"
+	"path"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/benbjohnson/litestream"
+	"github.com/benbjohnson/litestream/internal"
+	"github.com/pkg/sftp"
+	"golang.org/x/crypto/ssh"
+)
+
+// ReplicaClientType is the client type for this package.
+const ReplicaClientType = "sftp"
+
+// Default settings for replica client.
+const (
+	DefaultDialTimeout = 30 * time.Second
+)
+
+var _ litestream.ReplicaClient = (*ReplicaClient)(nil)
+
+// ReplicaClient is a client for writing snapshots & WAL segments to disk.
+type ReplicaClient struct {
+	mu         sync.Mutex
+	sshClient  *ssh.Client
+	sftpClient *sftp.Client
+
+	// SFTP connection info
+	Host        string
+	User        string
+	Password    string
+	Path        string
+	KeyPath     string
+	DialTimeout time.Duration
+}
+
+// NewReplicaClient returns a new instance of ReplicaClient.
+func NewReplicaClient() *ReplicaClient {
+	return &ReplicaClient{
+		DialTimeout: DefaultDialTimeout,
+	}
+}
+
+// Type returns "gcs" as the client type.
+func (c *ReplicaClient) Type() string {
+	return ReplicaClientType
+}
+
+// Init initializes the connection to GCS. No-op if already initialized.
+func (c *ReplicaClient) Init(ctx context.Context) (_ *sftp.Client, err error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if c.sftpClient != nil {
+		return c.sftpClient, nil
+	}
+
+	if c.User == "" {
+		return nil, fmt.Errorf("sftp user required")
+	}
+
+	// Build SSH configuration & auth methods
+	config := &ssh.ClientConfig{
+		User:            c.User,
+		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
+		BannerCallback:  ssh.BannerDisplayStderr(),
+	}
+	if c.Password != "" {
+		config.Auth = append(config.Auth, ssh.Password(c.Password))
+	}
+
+	if c.KeyPath != "" {
+		buf, err := os.ReadFile(c.KeyPath)
+		if err != nil {
+			return nil, fmt.Errorf("cannot read sftp key path: %w", err)
+		}
+
+		signer, err := ssh.ParsePrivateKey(buf)
+		if err != nil {
+			return nil, fmt.Errorf("cannot parse sftp key path: %w", err)
+		}
+		config.Auth = append(config.Auth, ssh.PublicKeys(signer))
+	}
+
+	// Append standard port, if necessary.
+	host := c.Host
+	if _, _, err := net.SplitHostPort(c.Host); err != nil {
+		host = net.JoinHostPort(c.Host, "22")
+	}
+
+	// Connect via SSH.
+	if c.sshClient, err = ssh.Dial("tcp", host, config); err != nil {
+		return nil, err
+	}
+
+	// Wrap connection with an SFTP client.
+	if c.sftpClient, err = sftp.NewClient(c.sshClient); err != nil {
+		c.sshClient.Close()
+		c.sshClient = nil
+		return nil, err
+	}
+
+	return c.sftpClient, nil
+}
+
+// Generations returns a list of available generation names.
+func (c *ReplicaClient) Generations(ctx context.Context) (_ []string, err error) {
+	defer func() { c.resetOnConnError(err) }()
+
+	sftpClient, err := c.Init(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	fis, err := sftpClient.ReadDir(path.Join(c.Path, "generations"))
+	if os.IsNotExist(err) {
+		return nil, nil
+	} else if err != nil {
+		return nil, err
+	}
+
+	var generations []string
+	for _, fi := range fis {
+		if !fi.IsDir() {
+			continue
+		}
+
+		name := path.Base(fi.Name())
+		if !litestream.IsGenerationName(name) {
+			continue
+		}
+		generations = append(generations, name)
+	}
+
+	sort.Strings(generations)
+
+	return generations, nil
+}
+
+// DeleteGeneration deletes all snapshots & WAL segments within a generation.
+func (c *ReplicaClient) DeleteGeneration(ctx context.Context, generation string) (err error) {
+	defer func() { c.resetOnConnError(err) }()
+
+	sftpClient, err := c.Init(ctx)
+	if err != nil {
+		return err
+	} else if generation == "" {
+		return fmt.Errorf("generation required")
+	}
+
+	dir := path.Join(c.Path, "generations", generation)
+
+	var dirs []string
+	walker := sftpClient.Walk(dir)
+	for walker.Step() {
+		if err := walker.Err(); err != nil {
+			return fmt.Errorf("cannot walk path %q: %w", walker.Path(), err)
+		}
+		if walker.Stat().IsDir() {
+			dirs = append(dirs, walker.Path())
+			continue
+		}
+
+		if err := sftpClient.Remove(walker.Path()); err != nil && !os.IsNotExist(err) {
+			return fmt.Errorf("cannot delete file %q: %w", walker.Path(), err)
+		}
+
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "DELETE").Inc()
+	}
+
+	// Remove directories in reverse order after they have been emptied.
+	for i := len(dirs) - 1; i >= 0; i-- {
+		filename := dirs[i]
+		if err := sftpClient.RemoveDirectory(filename); err != nil && !os.IsNotExist(err) {
+			return fmt.Errorf("cannot delete directory %q: %w", filename, err)
+		}
+	}
+
+	// log.Printf("%s(%s): retainer: deleting generation: %s", r.db.Path(), r.Name(), generation)
+
+	return nil
+}
+
+// Snapshots returns an iterator over all available snapshots for a generation.
+func (c *ReplicaClient) Snapshots(ctx context.Context, generation string) (_ litestream.SnapshotIterator, err error) {
+	defer func() { c.resetOnConnError(err) }()
+
+	sftpClient, err := c.Init(ctx)
+	if err != nil {
+		return nil, err
+	} else if generation == "" {
+		return nil, fmt.Errorf("generation required")
+	}
+
+	dir := path.Join(c.Path, "generations", generation, "snapshots")
+
+	fis, err := sftpClient.ReadDir(dir)
+	if os.IsNotExist(err) {
+		return litestream.NewSnapshotInfoSliceIterator(nil), nil
+	} else if err != nil {
+		return nil, err
+	}
+
+	// Iterate over every file and convert to metadata.
+	infos := make([]litestream.SnapshotInfo, 0, len(fis))
+	for _, fi := range fis {
+		// Parse index from filename.
+		index, err := internal.ParseSnapshotPath(path.Base(fi.Name()))
+		if err != nil {
+			continue
+		}
+
+		infos = append(infos, litestream.SnapshotInfo{
+			Generation: generation,
+			Index:      index,
+			Size:       fi.Size(),
+			CreatedAt:  fi.ModTime().UTC(),
+		})
+	}
+
+	sort.Sort(litestream.SnapshotInfoSlice(infos))
+
+	return litestream.NewSnapshotInfoSliceIterator(infos), nil
+}
+
+// WriteSnapshot writes LZ4 compressed data from rd to the object storage.
+func (c *ReplicaClient) WriteSnapshot(ctx context.Context, generation string, index int, rd io.Reader) (info litestream.SnapshotInfo, err error) {
+	defer func() { c.resetOnConnError(err) }()
+
+	sftpClient, err := c.Init(ctx)
+	if err != nil {
+		return info, err
+	} else if generation == "" {
+		return info, fmt.Errorf("generation required")
+	}
+
+	filename := path.Join(c.Path, "generations", generation, "snapshots", litestream.FormatIndex(index)+".snapshot.lz4")
+	startTime := time.Now()
+
+	if err := sftpClient.MkdirAll(path.Dir(filename)); err != nil {
+		return info, fmt.Errorf("cannot make parent wal segment directory %q: %w", path.Dir(filename), err)
+	}
+
+	f, err := sftpClient.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC)
+	if err != nil {
+		return info, fmt.Errorf("cannot open snapshot file for writing: %w", err)
+	}
+	defer f.Close()
+
+	n, err := io.Copy(f, rd)
+	if err != nil {
+		return info, err
+	} else if err := f.Close(); err != nil {
+		return info, err
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "PUT").Inc()
+	internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "PUT").Add(float64(n))
+
+	// log.Printf("%s(%s): snapshot: creating %s/%08x t=%s", r.db.Path(), r.Name(), generation, index, time.Since(startTime).Truncate(time.Millisecond))
+
+	return litestream.SnapshotInfo{
+		Generation: generation,
+		Index:      index,
+		Size:       n,
+		CreatedAt:  startTime.UTC(),
+	}, nil
+}
+
+// SnapshotReader returns a reader for snapshot data at the given generation/index.
+func (c *ReplicaClient) SnapshotReader(ctx context.Context, generation string, index int) (_ io.ReadCloser, err error) {
+	defer func() { c.resetOnConnError(err) }()
+
+	sftpClient, err := c.Init(ctx)
+	if err != nil {
+		return nil, err
+	} else if generation == "" {
+		return nil, fmt.Errorf("generation required")
+	}
+
+	filename := path.Join(c.Path, "generations", generation, "snapshots", litestream.FormatIndex(index)+".snapshot.lz4")
+
+	f, err := sftpClient.Open(filename)
+	if err != nil {
+		return nil, err
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "GET").Inc()
+
+	return f, nil
+}
+
+// DeleteSnapshot deletes a snapshot with the given generation & index.
+func (c *ReplicaClient) DeleteSnapshot(ctx context.Context, generation string, index int) (err error) {
+	defer func() { c.resetOnConnError(err) }()
+
+	sftpClient, err := c.Init(ctx)
+	if err != nil {
+		return err
+	} else if generation == "" {
+		return fmt.Errorf("generation required")
+	}
+
+	filename := path.Join(c.Path, "generations", generation, "snapshots", litestream.FormatIndex(index)+".snapshot.lz4")
+
+	if err := sftpClient.Remove(filename); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("cannot delete snapshot %q: %w", filename, err)
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "DELETE").Inc()
+	return nil
+}
+
+// WALSegments returns an iterator over all available WAL files for a generation.
+func (c *ReplicaClient) WALSegments(ctx context.Context, generation string) (_ litestream.WALSegmentIterator, err error) {
+	defer func() { c.resetOnConnError(err) }()
+
+	sftpClient, err := c.Init(ctx)
+	if err != nil {
+		return nil, err
+	} else if generation == "" {
+		return nil, fmt.Errorf("generation required")
+	}
+
+	dir := path.Join(c.Path, "generations", generation, "wal")
+
+	fis, err := sftpClient.ReadDir(dir)
+	if os.IsNotExist(err) {
+		return litestream.NewWALSegmentInfoSliceIterator(nil), nil
+	} else if err != nil {
+		return nil, err
+	}
+
+	// Iterate over every file and convert to metadata.
+	indexes := make([]int, 0, len(fis))
+	for _, fi := range fis {
+		index, err := litestream.ParseIndex(fi.Name())
+		if err != nil || !fi.IsDir() {
+			continue
+		}
+		indexes = append(indexes, index)
+	}
+
+	sort.Ints(indexes)
+
+	return newWALSegmentIterator(ctx, c, dir, generation, indexes), nil
+}
+
+// WriteWALSegment writes LZ4 compressed data from rd into a file on disk.
+func (c *ReplicaClient) WriteWALSegment(ctx context.Context, pos litestream.Pos, rd io.Reader) (info litestream.WALSegmentInfo, err error) {
+	defer func() { c.resetOnConnError(err) }()
+
+	sftpClient, err := c.Init(ctx)
+	if err != nil {
+		return info, err
+	} else if pos.Generation == "" {
+		return info, fmt.Errorf("generation required")
+	}
+
+	filename := path.Join(c.Path, "generations", pos.Generation, "wal", litestream.FormatIndex(pos.Index), litestream.FormatOffset(pos.Offset)+".wal.lz4")
+	startTime := time.Now()
+
+	if err := sftpClient.MkdirAll(path.Dir(filename)); err != nil {
+		return info, fmt.Errorf("cannot make parent snapshot directory %q: %w", path.Dir(filename), err)
+	}
+
+	f, err := sftpClient.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC)
+	if err != nil {
+		return info, fmt.Errorf("cannot open snapshot file for writing: %w", err)
+	}
+	defer f.Close()
+
+	n, err := io.Copy(f, rd)
+	if err != nil {
+		return info, err
+	} else if err := f.Close(); err != nil {
+		return info, err
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "PUT").Inc()
+	internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "PUT").Add(float64(n))
+
+	return litestream.WALSegmentInfo{
+		Generation: pos.Generation,
+		Index:      pos.Index,
+		Offset:     pos.Offset,
+		Size:       n,
+		CreatedAt:  startTime.UTC(),
+	}, nil
+}
+
+// WALSegmentReader returns a reader for a section of WAL data at the given index.
+// Returns os.ErrNotExist if no matching index/offset is found.
+func (c *ReplicaClient) WALSegmentReader(ctx context.Context, pos litestream.Pos) (_ io.ReadCloser, err error) {
+	defer func() { c.resetOnConnError(err) }()
+
+	sftpClient, err := c.Init(ctx)
+	if err != nil {
+		return nil, err
+	} else if pos.Generation == "" {
+		return nil, fmt.Errorf("generation required")
+	}
+
+	filename := path.Join(c.Path, "generations", pos.Generation, "wal", litestream.FormatIndex(pos.Index), litestream.FormatOffset(pos.Offset)+".wal.lz4")
+
+	f, err := sftpClient.Open(filename)
+	if err != nil {
+		return nil, err
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "GET").Inc()
+
+	return f, nil
+}
+
+// DeleteWALSegments deletes WAL segments with at the given positions.
+func (c *ReplicaClient) DeleteWALSegments(ctx context.Context, a []litestream.Pos) (err error) {
+	defer func() { c.resetOnConnError(err) }()
+
+	sftpClient, err := c.Init(ctx)
+	if err != nil {
+		return err
+	}
+
+	for _, pos := range a {
+		if pos.Generation == "" {
+			return fmt.Errorf("generation required")
+		}
+
+		filename := path.Join(c.Path, "generations", pos.Generation, "wal", litestream.FormatIndex(pos.Index), litestream.FormatOffset(pos.Offset)+".wal.lz4")
+
+		if err := sftpClient.Remove(filename); err != nil && !os.IsNotExist(err) {
+			return fmt.Errorf("cannot delete wal segment %q: %w", filename, err)
+		}
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "DELETE").Inc()
+	}
+
+	return nil
+}
+
+// Cleanup deletes path & generations directories after empty.
+func (c *ReplicaClient) Cleanup(ctx context.Context) (err error) {
+	defer func() { c.resetOnConnError(err) }()
+
+	sftpClient, err := c.Init(ctx)
+	if err != nil {
+		return err
+	}
+
+	if err := sftpClient.RemoveDirectory(path.Join(c.Path, "generations")); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("cannot delete generations path: %w", err)
+	} else if err := sftpClient.RemoveDirectory(c.Path); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("cannot delete path: %w", err)
+	}
+	return nil
+}
+
+// resetOnConnError closes & clears the client if a connection error occurs.
+func (c *ReplicaClient) resetOnConnError(err error) {
+	if !errors.Is(err, sftp.ErrSSHFxConnectionLost) {
+		return
+	}
+
+	if c.sftpClient != nil {
+		c.sftpClient.Close()
+		c.sftpClient = nil
+	}
+	if c.sshClient != nil {
+		c.sshClient.Close()
+		c.sshClient = nil
+	}
+}
+
+type walSegmentIterator struct {
+	ctx        context.Context
+	client     *ReplicaClient
+	dir        string
+	generation string
+	indexes    []int
+
+	infos []litestream.WALSegmentInfo
+	err   error
+}
+
+func newWALSegmentIterator(ctx context.Context, client *ReplicaClient, dir, generation string, indexes []int) *walSegmentIterator {
+	return &walSegmentIterator{
+		ctx:        ctx,
+		client:     client,
+		dir:        dir,
+		generation: generation,
+		indexes:    indexes,
+	}
+}
+
+func (itr *walSegmentIterator) Close() (err error) {
+	return itr.err
+}
+
+func (itr *walSegmentIterator) Next() bool {
+	sftpClient, err := itr.client.Init(itr.ctx)
+	if err != nil {
+		itr.err = err
+		return false
+	}
+
+	// Exit if an error has already occurred.
+	if itr.err != nil {
+		return false
+	}
+
+	for {
+		// Move to the next segment in cache, if available.
+		if len(itr.infos) > 1 {
+			itr.infos = itr.infos[1:]
+			return true
+		}
+		itr.infos = itr.infos[:0] // otherwise clear infos
+
+		// Move to the next index unless this is the first time initializing.
+		if itr.infos != nil && len(itr.indexes) > 0 {
+			itr.indexes = itr.indexes[1:]
+		}
+
+		// If no indexes remain, stop iteration.
+		if len(itr.indexes) == 0 {
+			return false
+		}
+
+		// Read segments into a cache for the current index.
+		index := itr.indexes[0]
+		fis, err := sftpClient.ReadDir(path.Join(itr.dir, litestream.FormatIndex(index)))
+		if err != nil {
+			itr.err = err
+			return false
+		}
+
+		for _, fi := range fis {
+			filename := path.Base(fi.Name())
+			if fi.IsDir() {
+				continue
+			}
+
+			offset, err := litestream.ParseOffset(strings.TrimSuffix(filename, ".wal.lz4"))
+			if err != nil {
+				continue
+			}
+
+			itr.infos = append(itr.infos, litestream.WALSegmentInfo{
+				Generation: itr.generation,
+				Index:      index,
+				Offset:     offset,
+				Size:       fi.Size(),
+				CreatedAt:  fi.ModTime().UTC(),
+			})
+		}
+
+		if len(itr.infos) > 0 {
+			return true
+		}
+	}
+}
+
+func (itr *walSegmentIterator) Err() error { return itr.err }
+
+func (itr *walSegmentIterator) WALSegment() litestream.WALSegmentInfo {
+	if len(itr.infos) == 0 {
+		return litestream.WALSegmentInfo{}
+	}
+	return itr.infos[0]
+}