Allow DB.MetaPath to be configured (#485)

- Update Go module to v1.19 format
- Docker builder pinned to Go v1.20.1
- Alpine image pinned to 3.17.2 (rather than `latest`)

.github/CONTRIBUTING.md

@@ -1,17 +1,18 @@
-## Open-source, not open-contribution
+## Contribution Policy
 
-[Similar to SQLite](https://www.sqlite.org/copyright.html), Litestream is open
-source but closed to contributions. This keeps the code base free of proprietary
-or licensed code but it also helps me continue to maintain and build Litestream.
+Initially, Litestream was closed to outside contributions. The goal was to
+reduce burnout by limiting the maintenance overhead of reviewing and validating
+third-party code. However, this policy is overly broad and has prevented small,
+easily testable patches from being contributed.
 
-As the author of [BoltDB](https://github.com/boltdb/bolt), I found that
-accepting and maintaining third party patches contributed to my burn out and
-I eventually archived the project. Writing databases & low-level replication
-tools involves nuance and simple one line changes can have profound and
-unexpected changes in correctness and performance. Small contributions
-typically required hours of my time to properly test and validate them.
+Litestream is now open to code contributions for bug fixes only. Features carry
+a long-term maintenance burden so they will not be accepted at this time.
+Please [submit an issue][new-issue] if you have a feature you'd like to
+request.
+
+If you find mistakes in the documentation, please submit a fix to the
+[documentation repository][docs].
+
+[new-issue]: https://github.com/benbjohnson/litestream/issues/new
+[docs]: https://github.com/benbjohnson/litestream.io
 
-I am grateful for community involvement, bug reports, & feature requests. I do
-not wish to come off as anything but welcoming, however, I've
-made the decision to keep this project closed to contributions for my own
-mental health and long term viability of the project.

.github/pull_request_template.md

@@ -1,7 +0,0 @@
-Litestream is not accepting code contributions at this time. You can find a summary of why on the project's GitHub README:
-
-https://github.com/benbjohnson/litestream#open-source-not-open-contribution
-
-Web site & Documentation changes, however, are welcome. You can find that repository here:
-
-https://github.com/benbjohnson/litestream.io

.github/workflows/commit.yml

@@ -0,0 +1,121 @@
+on: push
+
+jobs:
+  build:
+    name: Build & Unit Test
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '1.20'
+      - uses: actions/cache@v2
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ inputs.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: ${{ inputs.os }}-go-
+
+      - run: go env
+
+      - run: go install ./cmd/litestream
+
+      - run: go test -v ./...
+
+#      - name: Build integration test
+#        run: go test -c ./integration
+#
+#      - uses: actions/upload-artifact@v2
+#        with:
+#          name: integration.test
+#          path: integration.test
+#          if-no-files-found: error
+
+#  long-running-test:
+#    name: Run Long Running Unit Test
+#    runs-on: ubuntu-22.04
+#    steps:
+#      - uses: actions/checkout@v2
+#      - uses: actions/setup-go@v2
+#        with: 
+#          go-version: '1.20'
+#      - uses: actions/cache@v2
+#        with:
+#          path: ~/go/pkg/mod
+#          key: ${{ inputs.os }}-go-${{ hashFiles('**/go.sum') }}
+#          restore-keys: ${{ inputs.os }}-go-
+#
+#      - run: go install ./cmd/litestream
+#      - run: go test -v -run=TestCmd_Replicate_LongRunning ./integration -long-running-duration 1m
+
+#  s3-integration-test:
+#    name: Run S3 Integration Tests
+#    runs-on: ubuntu-18.04
+#    needs: build
+#    steps:
+#      - uses: actions/download-artifact@v2
+#        with:
+#          name: integration.test
+#      - run: chmod +x integration.test
+#
+#      - run: ./integration.test -test.v -test.run=TestReplicaClient -replica-type s3
+#        env:
+#          LITESTREAM_S3_ACCESS_KEY_ID: ${{ secrets.LITESTREAM_S3_ACCESS_KEY_ID }}
+#          LITESTREAM_S3_SECRET_ACCESS_KEY: ${{ secrets.LITESTREAM_S3_SECRET_ACCESS_KEY }}
+#          LITESTREAM_S3_REGION: us-east-1
+#          LITESTREAM_S3_BUCKET: integration.litestream.io
+
+#  gcp-integration-test:
+#    name: Run GCP Integration Tests
+#    runs-on: ubuntu-18.04
+#    needs: build
+#    steps:
+#      - name: Extract GCP credentials
+#        run: 'echo "$GOOGLE_APPLICATION_CREDENTIALS" > /opt/gcp.json'
+#        shell: bash
+#        env:
+#          GOOGLE_APPLICATION_CREDENTIALS: ${{secrets.GOOGLE_APPLICATION_CREDENTIALS}}
+#
+#      - uses: actions/download-artifact@v2
+#        with:
+#          name: integration.test
+#      - run: chmod +x integration.test
+#
+#      - run: ./integration.test -test.v -test.run=TestReplicaClient -replica-type gcs
+#        env:
+#          GOOGLE_APPLICATION_CREDENTIALS:  /opt/gcp.json
+#          LITESTREAM_GCS_BUCKET:           integration.litestream.io
+
+#  abs-integration-test:
+#    name: Run Azure Blob Store Integration Tests
+#    runs-on: ubuntu-18.04
+#    needs: build
+#    steps:
+#      - uses: actions/download-artifact@v2
+#        with:
+#          name: integration.test
+#      - run: chmod +x integration.test
+#
+#      - run: ./integration.test -test.v -test.run=TestReplicaClient -replica-type abs
+#        env:
+#          LITESTREAM_ABS_ACCOUNT_NAME:     ${{ secrets.LITESTREAM_ABS_ACCOUNT_NAME }}
+#          LITESTREAM_ABS_ACCOUNT_KEY:      ${{ secrets.LITESTREAM_ABS_ACCOUNT_KEY }}
+#          LITESTREAM_ABS_BUCKET:           integration
+
+#  sftp-integration-test:
+#    name: Run SFTP Integration Tests
+#    runs-on: ubuntu-18.04
+#    needs: build
+#    steps:
+#      - name: Extract SSH key
+#        run: 'echo "$LITESTREAM_SFTP_KEY" > /opt/id_ed25519'
+#        shell: bash
+#        env:
+#          LITESTREAM_SFTP_KEY: ${{secrets.LITESTREAM_SFTP_KEY}}
+#
+#      - name: Run sftp tests
+#        run: go test -v -run=TestReplicaClient ./integration -replica-type sftp
+#        env:
+#          LITESTREAM_SFTP_HOST:     ${{ secrets.LITESTREAM_SFTP_HOST }}
+#          LITESTREAM_SFTP_USER:     ${{ secrets.LITESTREAM_SFTP_USER }}
+#          LITESTREAM_SFTP_KEY_PATH: /opt/id_ed25519
+#          LITESTREAM_SFTP_PATH:     ${{ secrets.LITESTREAM_SFTP_PATH }}

.github/workflows/release.docker.yml

@@ -0,0 +1,51 @@
+on:
+  release:
+    types:
+      - published
+  pull_request:
+    types:
+      - opened
+      - synchronize
+      - reopened
+    branches-ignore:
+      - "dependabot/**"
+
+name: Release (Docker)
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    env:
+      PLATFORMS: "linux/amd64,linux/arm64,linux/arm/v7"
+      VERSION: "${{ github.event_name == 'release' && github.event.release.name || github.sha }}"
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: docker/setup-qemu-action@v1
+      - uses: docker/setup-buildx-action@v1
+
+      - uses: docker/login-action@v1
+        with:
+          username: benbjohnson
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      
+      - id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: litestream/litestream
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=sha
+            type=sha,format=long
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+      
+      - uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          platforms: ${{ env.PLATFORMS }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          build-args: |
+            LITESTREAM_VERSION=${{ env.VERSION }}

.github/workflows/release.linux.yml

@@ -6,7 +6,7 @@ on:
 name: release (linux)
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-18.04
     strategy:
       matrix:
         include:
@@ -30,6 +30,8 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-go@v2
+        with:
+          go-version: '1.16'
 
       - id: release
         uses: bruceadams/get-release@v1.2.2

.github/workflows/release.linux_static.yml

@@ -6,7 +6,7 @@ on:
 name: release (linux/static)
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-18.04
     strategy:
       matrix:
         include:
@@ -30,6 +30,8 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-go@v2
+        with:
+          go-version: '1.16'
 
       - id: release
         uses: bruceadams/get-release@v1.2.2

.github/workflows/test.yml

@@ -1,21 +0,0 @@
-on: push
-name: test
-jobs:
-  test:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/setup-go@v2
-        with:
-          go-version: '1.15'
-
-      - uses: actions/checkout@v2
-
-      - uses: actions/cache@v2
-        with:
-          path: ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go-
-
-      - name: Run unit tests
-        run: go test -v ./...

Dockerfile

@@ -1,11 +1,16 @@
-FROM golang:1.16 as builder
+FROM golang:1.20.1 as builder
+
 WORKDIR /src/litestream
 COPY . .
+
+ARG LITESTREAM_VERSION=latest
+
 RUN --mount=type=cache,target=/root/.cache/go-build \
 	--mount=type=cache,target=/go/pkg \
-	go build -ldflags '-s -w -extldflags "-static"' -tags osusergo,netgo,sqlite_omit_load_extension -o /usr/local/bin/litestream ./cmd/litestream
+	go build -ldflags "-s -w -X 'main.Version=${LITESTREAM_VERSION}' -extldflags '-static'" -tags osusergo,netgo,sqlite_omit_load_extension -o /usr/local/bin/litestream ./cmd/litestream
 
-FROM alpine
+
+FROM alpine:3.17.2
 COPY --from=builder /usr/local/bin/litestream /usr/local/bin/litestream
 ENTRYPOINT ["/usr/local/bin/litestream"]
 CMD []

README.md

@@ -6,7 +6,7 @@ Litestream
 ![test](https://github.com/benbjohnson/litestream/workflows/test/badge.svg)
 ==========
 
-Litestream is a standalone streaming replication tool for SQLite. It runs as a
+Litestream is a standalone disaster recovery tool for SQLite. It runs as a
 background process and safely replicates changes incrementally to another file
 or S3. Litestream only communicates with SQLite through the SQLite API so it
 will not corrupt your database.
@@ -29,32 +29,33 @@ of the most valuable contributions are in the forms of testing, feedback, and
 documentation. These help harden software and streamline usage for other users.
 
 I want to give special thanks to individuals who invest much of their time and 
-energy into the project to help make it better. Shout out to [Michael
-Lynch](https://github.com/mtlynch) for digging into issues and contributing to
-the documentation.
+energy into the project to help make it better:
+
+- Thanks to [Cory LaNou](https://twitter.com/corylanou) for giving early feedback and testing when Litestream was still pre-release.
+- Thanks to [Michael Lynch](https://github.com/mtlynch) for digging into issues and contributing to the documentation.
+- Thanks to [Kurt Mackey](https://twitter.com/mrkurt) for feedback and testing.
+- Thanks to [Sam Weston](https://twitter.com/cablespaghetti) for figuring out how to run Litestream on Kubernetes and writing up the docs for it.
+- Thanks to [Rafael](https://github.com/netstx) & [Jungle Boogie](https://github.com/jungle-boogie) for helping to get OpenBSD release builds working.
+- Thanks to [Simon Gottschlag](https://github.com/simongottschlag), [Marin](https://github.com/supermarin),[Victor Björklund](https://github.com/victorbjorklund), [Jonathan Beri](https://twitter.com/beriberikix) [Yuri](https://github.com/yurivish), [Nathan Probst](https://github.com/nprbst), [Yann Coleu](https://github.com/yanc0), and [Nicholas Grilly](https://twitter.com/ngrilly) for frequent feedback, testing, & support.
+
+Huge thanks to fly.io for their support and for contributing credits for testing and development!
 
 
-## Open-source, not open-contribution
+## Contribution Policy
 
-[Similar to SQLite](https://www.sqlite.org/copyright.html), Litestream is open
-source but closed to code contributions. This keeps the code base free of
-proprietary or licensed code but it also helps me continue to maintain and build
-Litestream.
+Initially, Litestream was closed to outside contributions. The goal was to
+reduce burnout by limiting the maintenance overhead of reviewing and validating
+third-party code. However, this policy is overly broad and has prevented small,
+easily testable patches from being contributed.
 
-As the author of [BoltDB](https://github.com/boltdb/bolt), I found that
-accepting and maintaining third party patches contributed to my burn out and
-I eventually archived the project. Writing databases & low-level replication
-tools involves nuance and simple one line changes can have profound and
-unexpected changes in correctness and performance. Small contributions
-typically required hours of my time to properly test and validate them.
+Litestream is now open to code contributions for bug fixes only. Features carry
+a long-term maintenance burden so they will not be accepted at this time.
+Please [submit an issue][new-issue] if you have a feature you'd like to
+request.
 
-I am grateful for community involvement, bug reports, & feature requests. I do
-not wish to come off as anything but welcoming, however, I've
-made the decision to keep this project closed to contributions for my own
-mental health and long term viability of the project.
+If you find mistakes in the documentation, please submit a fix to the
+[documentation repository][docs].
 
-The [documentation repository][docs] is MIT licensed and pull requests are welcome there.
-
-[releases]: https://github.com/benbjohnson/litestream/releases
+[new-issue]: https://github.com/benbjohnson/litestream/issues/new
 [docs]: https://github.com/benbjohnson/litestream.io
 

abs/replica_client.go

@@ -0,0 +1,565 @@
+package abs
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/url"
+	"os"
+	"path"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/Azure/azure-storage-blob-go/azblob"
+	"github.com/benbjohnson/litestream"
+	"github.com/benbjohnson/litestream/internal"
+	"golang.org/x/sync/errgroup"
+)
+
+// ReplicaClientType is the client type for this package.
+const ReplicaClientType = "abs"
+
+var _ litestream.ReplicaClient = (*ReplicaClient)(nil)
+
+// ReplicaClient is a client for writing snapshots & WAL segments to disk.
+type ReplicaClient struct {
+	mu           sync.Mutex
+	containerURL *azblob.ContainerURL
+
+	// Azure credentials
+	AccountName string
+	AccountKey  string
+	Endpoint    string
+
+	// Azure Blob Storage container information
+	Bucket string
+	Path   string
+}
+
+// NewReplicaClient returns a new instance of ReplicaClient.
+func NewReplicaClient() *ReplicaClient {
+	return &ReplicaClient{}
+}
+
+// Type returns "abs" as the client type.
+func (c *ReplicaClient) Type() string {
+	return ReplicaClientType
+}
+
+// Init initializes the connection to Azure. No-op if already initialized.
+func (c *ReplicaClient) Init(ctx context.Context) (err error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if c.containerURL != nil {
+		return nil
+	}
+
+	// Read account key from environment, if available.
+	accountKey := c.AccountKey
+	if accountKey == "" {
+		accountKey = os.Getenv("LITESTREAM_AZURE_ACCOUNT_KEY")
+	}
+
+	// Authenticate to ACS.
+	credential, err := azblob.NewSharedKeyCredential(c.AccountName, accountKey)
+	if err != nil {
+		return err
+	}
+
+	// Construct & parse endpoint unless already set.
+	endpoint := c.Endpoint
+	if endpoint == "" {
+		endpoint = fmt.Sprintf("https://%s.blob.core.windows.net", c.AccountName)
+	}
+	endpointURL, err := url.Parse(endpoint)
+	if err != nil {
+		return fmt.Errorf("cannot parse azure endpoint: %w", err)
+	}
+
+	// Build pipeline and reference to container.
+	pipeline := azblob.NewPipeline(credential, azblob.PipelineOptions{
+		Retry: azblob.RetryOptions{
+			TryTimeout: 24 * time.Hour,
+		},
+	})
+	containerURL := azblob.NewServiceURL(*endpointURL, pipeline).NewContainerURL(c.Bucket)
+	c.containerURL = &containerURL
+
+	return nil
+}
+
+// Generations returns a list of available generation names.
+func (c *ReplicaClient) Generations(ctx context.Context) ([]string, error) {
+	if err := c.Init(ctx); err != nil {
+		return nil, err
+	}
+
+	var generations []string
+	var marker azblob.Marker
+	for marker.NotDone() {
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "LIST").Inc()
+
+		resp, err := c.containerURL.ListBlobsHierarchySegment(ctx, marker, "/", azblob.ListBlobsSegmentOptions{
+			Prefix: litestream.GenerationsPath(c.Path) + "/",
+		})
+		if err != nil {
+			return nil, err
+		}
+		marker = resp.NextMarker
+
+		for _, prefix := range resp.Segment.BlobPrefixes {
+			name := path.Base(strings.TrimSuffix(prefix.Name, "/"))
+			if !litestream.IsGenerationName(name) {
+				continue
+			}
+			generations = append(generations, name)
+		}
+	}
+
+	return generations, nil
+}
+
+// DeleteGeneration deletes all snapshots & WAL segments within a generation.
+func (c *ReplicaClient) DeleteGeneration(ctx context.Context, generation string) error {
+	if err := c.Init(ctx); err != nil {
+		return err
+	}
+
+	dir, err := litestream.GenerationPath(c.Path, generation)
+	if err != nil {
+		return fmt.Errorf("cannot determine generation path: %w", err)
+	}
+
+	var marker azblob.Marker
+	for marker.NotDone() {
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "LIST").Inc()
+
+		resp, err := c.containerURL.ListBlobsFlatSegment(ctx, marker, azblob.ListBlobsSegmentOptions{Prefix: dir + "/"})
+		if err != nil {
+			return err
+		}
+		marker = resp.NextMarker
+
+		for _, item := range resp.Segment.BlobItems {
+			internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "DELETE").Inc()
+
+			blobURL := c.containerURL.NewBlobURL(item.Name)
+			if _, err := blobURL.Delete(ctx, azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{}); isNotExists(err) {
+				continue
+			} else if err != nil {
+				return err
+			}
+		}
+	}
+
+	// log.Printf("%s(%s): retainer: deleting generation: %s", r.db.Path(), r.Name(), generation)
+
+	return nil
+}
+
+// Snapshots returns an iterator over all available snapshots for a generation.
+func (c *ReplicaClient) Snapshots(ctx context.Context, generation string) (litestream.SnapshotIterator, error) {
+	if err := c.Init(ctx); err != nil {
+		return nil, err
+	}
+	return newSnapshotIterator(ctx, generation, c), nil
+}
+
+// WriteSnapshot writes LZ4 compressed data from rd to the object storage.
+func (c *ReplicaClient) WriteSnapshot(ctx context.Context, generation string, index int, rd io.Reader) (info litestream.SnapshotInfo, err error) {
+	if err := c.Init(ctx); err != nil {
+		return info, err
+	}
+
+	key, err := litestream.SnapshotPath(c.Path, generation, index)
+	if err != nil {
+		return info, fmt.Errorf("cannot determine snapshot path: %w", err)
+	}
+	startTime := time.Now()
+
+	rc := internal.NewReadCounter(rd)
+
+	blobURL := c.containerURL.NewBlockBlobURL(key)
+	if _, err := azblob.UploadStreamToBlockBlob(ctx, rc, blobURL, azblob.UploadStreamToBlockBlobOptions{
+		BlobHTTPHeaders: azblob.BlobHTTPHeaders{ContentType: "application/octet-stream"},
+		BlobAccessTier:  azblob.DefaultAccessTier,
+	}); err != nil {
+		return info, err
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "PUT").Inc()
+	internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "PUT").Add(float64(rc.N()))
+
+	// log.Printf("%s(%s): snapshot: creating %s/%08x t=%s", r.db.Path(), r.Name(), generation, index, time.Since(startTime).Truncate(time.Millisecond))
+
+	return litestream.SnapshotInfo{
+		Generation: generation,
+		Index:      index,
+		Size:       rc.N(),
+		CreatedAt:  startTime.UTC(),
+	}, nil
+}
+
+// SnapshotReader returns a reader for snapshot data at the given generation/index.
+func (c *ReplicaClient) SnapshotReader(ctx context.Context, generation string, index int) (io.ReadCloser, error) {
+	if err := c.Init(ctx); err != nil {
+		return nil, err
+	}
+
+	key, err := litestream.SnapshotPath(c.Path, generation, index)
+	if err != nil {
+		return nil, fmt.Errorf("cannot determine snapshot path: %w", err)
+	}
+
+	blobURL := c.containerURL.NewBlobURL(key)
+	resp, err := blobURL.Download(ctx, 0, 0, azblob.BlobAccessConditions{}, false, azblob.ClientProvidedKeyOptions{})
+	if isNotExists(err) {
+		return nil, os.ErrNotExist
+	} else if err != nil {
+		return nil, fmt.Errorf("cannot start new reader for %q: %w", key, err)
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "GET").Inc()
+	internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "GET").Add(float64(resp.ContentLength()))
+
+	return resp.Body(azblob.RetryReaderOptions{}), nil
+}
+
+// DeleteSnapshot deletes a snapshot with the given generation & index.
+func (c *ReplicaClient) DeleteSnapshot(ctx context.Context, generation string, index int) error {
+	if err := c.Init(ctx); err != nil {
+		return err
+	}
+
+	key, err := litestream.SnapshotPath(c.Path, generation, index)
+	if err != nil {
+		return fmt.Errorf("cannot determine snapshot path: %w", err)
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "DELETE").Inc()
+
+	blobURL := c.containerURL.NewBlobURL(key)
+	if _, err := blobURL.Delete(ctx, azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{}); isNotExists(err) {
+		return nil
+	} else if err != nil {
+		return fmt.Errorf("cannot delete snapshot %q: %w", key, err)
+	}
+	return nil
+}
+
+// WALSegments returns an iterator over all available WAL files for a generation.
+func (c *ReplicaClient) WALSegments(ctx context.Context, generation string) (litestream.WALSegmentIterator, error) {
+	if err := c.Init(ctx); err != nil {
+		return nil, err
+	}
+	return newWALSegmentIterator(ctx, generation, c), nil
+}
+
+// WriteWALSegment writes LZ4 compressed data from rd into a file on disk.
+func (c *ReplicaClient) WriteWALSegment(ctx context.Context, pos litestream.Pos, rd io.Reader) (info litestream.WALSegmentInfo, err error) {
+	if err := c.Init(ctx); err != nil {
+		return info, err
+	}
+
+	key, err := litestream.WALSegmentPath(c.Path, pos.Generation, pos.Index, pos.Offset)
+	if err != nil {
+		return info, fmt.Errorf("cannot determine wal segment path: %w", err)
+	}
+	startTime := time.Now()
+
+	rc := internal.NewReadCounter(rd)
+
+	blobURL := c.containerURL.NewBlockBlobURL(key)
+	if _, err := azblob.UploadStreamToBlockBlob(ctx, rc, blobURL, azblob.UploadStreamToBlockBlobOptions{
+		BlobHTTPHeaders: azblob.BlobHTTPHeaders{ContentType: "application/octet-stream"},
+		BlobAccessTier:  azblob.DefaultAccessTier,
+	}); err != nil {
+		return info, err
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "PUT").Inc()
+	internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "PUT").Add(float64(rc.N()))
+
+	return litestream.WALSegmentInfo{
+		Generation: pos.Generation,
+		Index:      pos.Index,
+		Offset:     pos.Offset,
+		Size:       rc.N(),
+		CreatedAt:  startTime.UTC(),
+	}, nil
+}
+
+// WALSegmentReader returns a reader for a section of WAL data at the given index.
+// Returns os.ErrNotExist if no matching index/offset is found.
+func (c *ReplicaClient) WALSegmentReader(ctx context.Context, pos litestream.Pos) (io.ReadCloser, error) {
+	if err := c.Init(ctx); err != nil {
+		return nil, err
+	}
+
+	key, err := litestream.WALSegmentPath(c.Path, pos.Generation, pos.Index, pos.Offset)
+	if err != nil {
+		return nil, fmt.Errorf("cannot determine wal segment path: %w", err)
+	}
+
+	blobURL := c.containerURL.NewBlobURL(key)
+	resp, err := blobURL.Download(ctx, 0, 0, azblob.BlobAccessConditions{}, false, azblob.ClientProvidedKeyOptions{})
+	if isNotExists(err) {
+		return nil, os.ErrNotExist
+	} else if err != nil {
+		return nil, fmt.Errorf("cannot start new reader for %q: %w", key, err)
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "GET").Inc()
+	internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "GET").Add(float64(resp.ContentLength()))
+
+	return resp.Body(azblob.RetryReaderOptions{}), nil
+}
+
+// DeleteWALSegments deletes WAL segments with at the given positions.
+func (c *ReplicaClient) DeleteWALSegments(ctx context.Context, a []litestream.Pos) error {
+	if err := c.Init(ctx); err != nil {
+		return err
+	}
+
+	for _, pos := range a {
+		key, err := litestream.WALSegmentPath(c.Path, pos.Generation, pos.Index, pos.Offset)
+		if err != nil {
+			return fmt.Errorf("cannot determine wal segment path: %w", err)
+		}
+
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "DELETE").Inc()
+
+		blobURL := c.containerURL.NewBlobURL(key)
+		if _, err := blobURL.Delete(ctx, azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{}); isNotExists(err) {
+			continue
+		} else if err != nil {
+			return fmt.Errorf("cannot delete wal segment %q: %w", key, err)
+		}
+	}
+
+	return nil
+}
+
+type snapshotIterator struct {
+	client     *ReplicaClient
+	generation string
+
+	ch     chan litestream.SnapshotInfo
+	g      errgroup.Group
+	ctx    context.Context
+	cancel func()
+
+	info litestream.SnapshotInfo
+	err  error
+}
+
+func newSnapshotIterator(ctx context.Context, generation string, client *ReplicaClient) *snapshotIterator {
+	itr := &snapshotIterator{
+		client:     client,
+		generation: generation,
+		ch:         make(chan litestream.SnapshotInfo),
+	}
+
+	itr.ctx, itr.cancel = context.WithCancel(ctx)
+	itr.g.Go(itr.fetch)
+
+	return itr
+}
+
+// fetch runs in a separate goroutine to fetch pages of objects and stream them to a channel.
+func (itr *snapshotIterator) fetch() error {
+	defer close(itr.ch)
+
+	dir, err := litestream.SnapshotsPath(itr.client.Path, itr.generation)
+	if err != nil {
+		return fmt.Errorf("cannot determine snapshots path: %w", err)
+	}
+
+	var marker azblob.Marker
+	for marker.NotDone() {
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "LIST").Inc()
+
+		resp, err := itr.client.containerURL.ListBlobsFlatSegment(itr.ctx, marker, azblob.ListBlobsSegmentOptions{Prefix: dir + "/"})
+		if err != nil {
+			return err
+		}
+		marker = resp.NextMarker
+
+		for _, item := range resp.Segment.BlobItems {
+			key := path.Base(item.Name)
+			index, err := litestream.ParseSnapshotPath(key)
+			if err != nil {
+				continue
+			}
+
+			info := litestream.SnapshotInfo{
+				Generation: itr.generation,
+				Index:      index,
+				Size:       *item.Properties.ContentLength,
+				CreatedAt:  item.Properties.CreationTime.UTC(),
+			}
+
+			select {
+			case <-itr.ctx.Done():
+			case itr.ch <- info:
+			}
+		}
+	}
+	return nil
+}
+
+func (itr *snapshotIterator) Close() (err error) {
+	err = itr.err
+
+	// Cancel context and wait for error group to finish.
+	itr.cancel()
+	if e := itr.g.Wait(); e != nil && err == nil {
+		err = e
+	}
+
+	return err
+}
+
+func (itr *snapshotIterator) Next() bool {
+	// Exit if an error has already occurred.
+	if itr.err != nil {
+		return false
+	}
+
+	// Return false if context was canceled or if there are no more snapshots.
+	// Otherwise fetch the next snapshot and store it on the iterator.
+	select {
+	case <-itr.ctx.Done():
+		return false
+	case info, ok := <-itr.ch:
+		if !ok {
+			return false
+		}
+		itr.info = info
+		return true
+	}
+}
+
+func (itr *snapshotIterator) Err() error { return itr.err }
+
+func (itr *snapshotIterator) Snapshot() litestream.SnapshotInfo {
+	return itr.info
+}
+
+type walSegmentIterator struct {
+	client     *ReplicaClient
+	generation string
+
+	ch     chan litestream.WALSegmentInfo
+	g      errgroup.Group
+	ctx    context.Context
+	cancel func()
+
+	info litestream.WALSegmentInfo
+	err  error
+}
+
+func newWALSegmentIterator(ctx context.Context, generation string, client *ReplicaClient) *walSegmentIterator {
+	itr := &walSegmentIterator{
+		client:     client,
+		generation: generation,
+		ch:         make(chan litestream.WALSegmentInfo),
+	}
+
+	itr.ctx, itr.cancel = context.WithCancel(ctx)
+	itr.g.Go(itr.fetch)
+
+	return itr
+}
+
+// fetch runs in a separate goroutine to fetch pages of objects and stream them to a channel.
+func (itr *walSegmentIterator) fetch() error {
+	defer close(itr.ch)
+
+	dir, err := litestream.WALPath(itr.client.Path, itr.generation)
+	if err != nil {
+		return fmt.Errorf("cannot determine wal path: %w", err)
+	}
+
+	var marker azblob.Marker
+	for marker.NotDone() {
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "LIST").Inc()
+
+		resp, err := itr.client.containerURL.ListBlobsFlatSegment(itr.ctx, marker, azblob.ListBlobsSegmentOptions{Prefix: dir + "/"})
+		if err != nil {
+			return err
+		}
+		marker = resp.NextMarker
+
+		for _, item := range resp.Segment.BlobItems {
+			key := path.Base(item.Name)
+			index, offset, err := litestream.ParseWALSegmentPath(key)
+			if err != nil {
+				continue
+			}
+
+			info := litestream.WALSegmentInfo{
+				Generation: itr.generation,
+				Index:      index,
+				Offset:     offset,
+				Size:       *item.Properties.ContentLength,
+				CreatedAt:  item.Properties.CreationTime.UTC(),
+			}
+
+			select {
+			case <-itr.ctx.Done():
+			case itr.ch <- info:
+			}
+		}
+	}
+	return nil
+}
+
+func (itr *walSegmentIterator) Close() (err error) {
+	err = itr.err
+
+	// Cancel context and wait for error group to finish.
+	itr.cancel()
+	if e := itr.g.Wait(); e != nil && err == nil {
+		err = e
+	}
+
+	return err
+}
+
+func (itr *walSegmentIterator) Next() bool {
+	// Exit if an error has already occurred.
+	if itr.err != nil {
+		return false
+	}
+
+	// Return false if context was canceled or if there are no more segments.
+	// Otherwise fetch the next segment and store it on the iterator.
+	select {
+	case <-itr.ctx.Done():
+		return false
+	case info, ok := <-itr.ch:
+		if !ok {
+			return false
+		}
+		itr.info = info
+		return true
+	}
+}
+
+func (itr *walSegmentIterator) Err() error { return itr.err }
+
+func (itr *walSegmentIterator) WALSegment() litestream.WALSegmentInfo {
+	return itr.info
+}
+
+func isNotExists(err error) bool {
+	switch err := err.(type) {
+	case azblob.StorageError:
+		return err.ServiceCode() == azblob.ServiceCodeBlobNotFound
+	default:
+		return false
+	}
+}

cmd/litestream/generations.go

@@ -30,8 +30,8 @@ func (c *GenerationsCommand) Run(ctx context.Context, args []string) (err error)
 	}
 
 	var db *litestream.DB
-	var r litestream.Replica
-	updatedAt := time.Now()
+	var r *litestream.Replica
+	dbUpdatedAt := time.Now()
 	if isURL(fs.Arg(0)) {
 		if *configPath != "" {
 			return fmt.Errorf("cannot specify a replica URL and the -config flag")
@@ -67,14 +67,14 @@ func (c *GenerationsCommand) Run(ctx context.Context, args []string) (err error)
 		}
 
 		// Determine last time database or WAL was updated.
-		if updatedAt, err = db.UpdatedAt(); err != nil {
+		if dbUpdatedAt, err = db.UpdatedAt(); err != nil {
 			return err
 		}
 	}
 
-	var replicas []litestream.Replica
+	var replicas []*litestream.Replica
 	if r != nil {
-		replicas = []litestream.Replica{r}
+		replicas = []*litestream.Replica{r}
 	} else {
 		replicas = db.Replicas
 	}
@@ -85,7 +85,7 @@ func (c *GenerationsCommand) Run(ctx context.Context, args []string) (err error)
 
 	fmt.Fprintln(w, "name\tgeneration\tlag\tstart\tend")
 	for _, r := range replicas {
-		generations, err := r.Generations(ctx)
+		generations, err := r.Client.Generations(ctx)
 		if err != nil {
 			log.Printf("%s: cannot list generations: %s", r.Name(), err)
 			continue
@@ -93,18 +93,18 @@ func (c *GenerationsCommand) Run(ctx context.Context, args []string) (err error)
 
 		// Iterate over each generation for the replica.
 		for _, generation := range generations {
-			stats, err := r.GenerationStats(ctx, generation)
+			createdAt, updatedAt, err := r.GenerationTimeBounds(ctx, generation)
 			if err != nil {
-				log.Printf("%s: cannot find generation stats: %s", r.Name(), err)
+				log.Printf("%s: cannot determine generation time bounds: %s", r.Name(), err)
 				continue
 			}
 
 			fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n",
 				r.Name(),
 				generation,
-				truncateDuration(updatedAt.Sub(stats.UpdatedAt)).String(),
-				stats.CreatedAt.Format(time.RFC3339),
-				stats.UpdatedAt.Format(time.RFC3339),
+				truncateDuration(dbUpdatedAt.Sub(updatedAt)).String(),
+				createdAt.Format(time.RFC3339),
+				updatedAt.Format(time.RFC3339),
 			)
 		}
 	}

cmd/litestream/main.go

@@ -9,7 +9,6 @@ import (
 	"log"
 	"net/url"
 	"os"
-	"os/signal"
 	"os/user"
 	"path"
 	"path/filepath"
@@ -18,8 +17,13 @@ import (
 	"strings"
 	"time"
 
+	"filippo.io/age"
 	"github.com/benbjohnson/litestream"
+	"github.com/benbjohnson/litestream/abs"
+	"github.com/benbjohnson/litestream/file"
+	"github.com/benbjohnson/litestream/gcs"
 	"github.com/benbjohnson/litestream/s3"
+	"github.com/benbjohnson/litestream/sftp"
 	_ "github.com/mattn/go-sqlite3"
 	"gopkg.in/yaml.v2"
 )
@@ -82,25 +86,38 @@ func (m *Main) Run(ctx context.Context, args []string) (err error) {
 		}
 
 		// Setup signal handler.
-		ctx, cancel := context.WithCancel(ctx)
-		ch := signalChan()
-		go func() { <-ch; cancel() }()
+		signalCh := signalChan()
 
-		if err := c.Run(ctx); err != nil {
+		if err := c.Run(); err != nil {
 			return err
 		}
 
 		// Wait for signal to stop program.
-		<-ctx.Done()
-		signal.Reset()
-		fmt.Println("signal received, litestream shutting down")
+		select {
+		case err = <-c.execCh:
+			fmt.Println("subprocess exited, litestream shutting down")
+		case sig := <-signalCh:
+			fmt.Println("signal received, litestream shutting down")
+
+			if c.cmd != nil {
+				fmt.Println("sending signal to exec process")
+				if err := c.cmd.Process.Signal(sig); err != nil {
+					return fmt.Errorf("cannot signal exec process: %w", err)
+				}
+
+				fmt.Println("waiting for exec process to close")
+				if err := <-c.execCh; err != nil && !strings.HasPrefix(err.Error(), "signal:") {
+					return fmt.Errorf("cannot wait for exec process: %w", err)
+				}
+			}
+		}
 
 		// Gracefully close.
-		if err := c.Close(); err != nil {
-			return err
+		if e := c.Close(); e != nil && err == nil {
+			err = e
 		}
 		fmt.Println("litestream shut down")
-		return nil
+		return err
 
 	case "restore":
 		return (&RestoreCommand{}).Run(ctx, args)
@@ -148,6 +165,10 @@ type Config struct {
 	// List of databases to manage.
 	DBs []*DBConfig `yaml:"dbs"`
 
+	// Subcommand to execute during replication.
+	// Litestream will shutdown when subcommand exits.
+	Exec string `yaml:"exec"`
+
 	// Global S3 settings
 	AccessKeyID     string `yaml:"access-key-id"`
 	SecretAccessKey string `yaml:"secret-access-key"`
@@ -226,6 +247,7 @@ func ReadConfigFile(filename string, expandEnv bool) (_ Config, err error) {
 // DBConfig represents the configuration for a single database.
 type DBConfig struct {
 	Path               string         `yaml:"path"`
+	MetaPath           *string        `yaml:"meta-path"`
 	MonitorInterval    *time.Duration `yaml:"monitor-interval"`
 	CheckpointInterval *time.Duration `yaml:"checkpoint-interval"`
 	MinCheckpointPageN *int           `yaml:"min-checkpoint-page-count"`
@@ -245,6 +267,9 @@ func NewDBFromConfig(dbc *DBConfig) (*litestream.DB, error) {
 	db := litestream.NewDB(path)
 
 	// Override default database settings if specified in configuration.
+	if dbc.MetaPath != nil {
+		db.SetMetaPath(*dbc.MetaPath)
+	}
 	if dbc.MonitorInterval != nil {
 		db.MonitorInterval = *dbc.MonitorInterval
 	}
@@ -272,15 +297,15 @@ func NewDBFromConfig(dbc *DBConfig) (*litestream.DB, error) {
 
 // ReplicaConfig represents the configuration for a single replica in a database.
 type ReplicaConfig struct {
-	Type                   string        `yaml:"type"` // "file", "s3"
-	Name                   string        `yaml:"name"` // name of replica, optional.
-	Path                   string        `yaml:"path"`
-	URL                    string        `yaml:"url"`
-	Retention              time.Duration `yaml:"retention"`
-	RetentionCheckInterval time.Duration `yaml:"retention-check-interval"`
-	SyncInterval           time.Duration `yaml:"sync-interval"` // s3 only
-	SnapshotInterval       time.Duration `yaml:"snapshot-interval"`
-	ValidationInterval     time.Duration `yaml:"validation-interval"`
+	Type                   string         `yaml:"type"` // "file", "s3"
+	Name                   string         `yaml:"name"` // name of replica, optional.
+	Path                   string         `yaml:"path"`
+	URL                    string         `yaml:"url"`
+	Retention              *time.Duration `yaml:"retention"`
+	RetentionCheckInterval *time.Duration `yaml:"retention-check-interval"`
+	SyncInterval           *time.Duration `yaml:"sync-interval"`
+	SnapshotInterval       *time.Duration `yaml:"snapshot-interval"`
+	ValidationInterval     *time.Duration `yaml:"validation-interval"`
 
 	// S3 settings
 	AccessKeyID     string `yaml:"access-key-id"`
@@ -290,27 +315,96 @@ type ReplicaConfig struct {
 	Endpoint        string `yaml:"endpoint"`
 	ForcePathStyle  *bool  `yaml:"force-path-style"`
 	SkipVerify      bool   `yaml:"skip-verify"`
+
+	// ABS settings
+	AccountName string `yaml:"account-name"`
+	AccountKey  string `yaml:"account-key"`
+
+	// SFTP settings
+	Host     string `yaml:"host"`
+	User     string `yaml:"user"`
+	Password string `yaml:"password"`
+	KeyPath  string `yaml:"key-path"`
+
+	// Encryption identities and recipients
+	Age struct {
+		Identities []string `yaml:"identities"`
+		Recipients []string `yaml:"recipients"`
+	} `yaml:"age"`
 }
 
 // NewReplicaFromConfig instantiates a replica for a DB based on a config.
-func NewReplicaFromConfig(c *ReplicaConfig, db *litestream.DB) (litestream.Replica, error) {
+func NewReplicaFromConfig(c *ReplicaConfig, db *litestream.DB) (_ *litestream.Replica, err error) {
 	// Ensure user did not specify URL in path.
 	if isURL(c.Path) {
 		return nil, fmt.Errorf("replica path cannot be a url, please use the 'url' field instead: %s", c.Path)
 	}
 
+	// Build replica.
+	r := litestream.NewReplica(db, c.Name)
+	if v := c.Retention; v != nil {
+		r.Retention = *v
+	}
+	if v := c.RetentionCheckInterval; v != nil {
+		r.RetentionCheckInterval = *v
+	}
+	if v := c.SyncInterval; v != nil {
+		r.SyncInterval = *v
+	}
+	if v := c.SnapshotInterval; v != nil {
+		r.SnapshotInterval = *v
+	}
+	if v := c.ValidationInterval; v != nil {
+		r.ValidationInterval = *v
+	}
+	for _, str := range c.Age.Identities {
+		identities, err := age.ParseIdentities(strings.NewReader(str))
+		if err != nil {
+			return nil, err
+		}
+
+		r.AgeIdentities = append(r.AgeIdentities, identities...)
+	}
+	for _, str := range c.Age.Recipients {
+		recipients, err := age.ParseRecipients(strings.NewReader(str))
+		if err != nil {
+			return nil, err
+		}
+
+		r.AgeRecipients = append(r.AgeRecipients, recipients...)
+	}
+
+	// Build and set client on replica.
 	switch c.ReplicaType() {
 	case "file":
-		return newFileReplicaFromConfig(c, db)
+		if r.Client, err = newFileReplicaClientFromConfig(c, r); err != nil {
+			return nil, err
+		}
 	case "s3":
-		return newS3ReplicaFromConfig(c, db)
+		if r.Client, err = newS3ReplicaClientFromConfig(c, r); err != nil {
+			return nil, err
+		}
+	case "gcs":
+		if r.Client, err = newGCSReplicaClientFromConfig(c, r); err != nil {
+			return nil, err
+		}
+	case "abs":
+		if r.Client, err = newABSReplicaClientFromConfig(c, r); err != nil {
+			return nil, err
+		}
+	case "sftp":
+		if r.Client, err = newSFTPReplicaClientFromConfig(c, r); err != nil {
+			return nil, err
+		}
 	default:
 		return nil, fmt.Errorf("unknown replica type in config: %q", c.Type)
 	}
+
+	return r, nil
 }
 
-// newFileReplicaFromConfig returns a new instance of FileReplica build from config.
-func newFileReplicaFromConfig(c *ReplicaConfig, db *litestream.DB) (_ *litestream.FileReplica, err error) {
+// newFileReplicaClientFromConfig returns a new instance of file.ReplicaClient built from config.
+func newFileReplicaClientFromConfig(c *ReplicaConfig, r *litestream.Replica) (_ *file.ReplicaClient, err error) {
 	// Ensure URL & path are not both specified.
 	if c.URL != "" && c.Path != "" {
 		return nil, fmt.Errorf("cannot specify url & path for file replica")
@@ -335,24 +429,13 @@ func newFileReplicaFromConfig(c *ReplicaConfig, db *litestream.DB) (_ *litestrea
 	}
 
 	// Instantiate replica and apply time fields, if set.
-	r := litestream.NewFileReplica(db, c.Name, path)
-	if v := c.Retention; v > 0 {
-		r.Retention = v
-	}
-	if v := c.RetentionCheckInterval; v > 0 {
-		r.RetentionCheckInterval = v
-	}
-	if v := c.SnapshotInterval; v > 0 {
-		r.SnapshotInterval = v
-	}
-	if v := c.ValidationInterval; v > 0 {
-		r.ValidationInterval = v
-	}
-	return r, nil
+	client := file.NewReplicaClient(path)
+	client.Replica = r
+	return client, nil
 }
 
-// newS3ReplicaFromConfig returns a new instance of S3Replica build from config.
-func newS3ReplicaFromConfig(c *ReplicaConfig, db *litestream.DB) (_ *s3.Replica, err error) {
+// newS3ReplicaClientFromConfig returns a new instance of s3.ReplicaClient built from config.
+func newS3ReplicaClientFromConfig(c *ReplicaConfig, r *litestream.Replica) (_ *s3.ReplicaClient, err error) {
 	// Ensure URL & constituent parts are not both specified.
 	if c.URL != "" && c.Path != "" {
 		return nil, fmt.Errorf("cannot specify url & path for s3 replica")
@@ -402,32 +485,148 @@ func newS3ReplicaFromConfig(c *ReplicaConfig, db *litestream.DB) (_ *s3.Replica,
 	}
 
 	// Build replica.
-	r := s3.NewReplica(db, c.Name)
-	r.AccessKeyID = c.AccessKeyID
-	r.SecretAccessKey = c.SecretAccessKey
-	r.Bucket = bucket
-	r.Path = path
-	r.Region = region
-	r.Endpoint = endpoint
-	r.ForcePathStyle = forcePathStyle
-	r.SkipVerify = skipVerify
+	client := s3.NewReplicaClient()
+	client.AccessKeyID = c.AccessKeyID
+	client.SecretAccessKey = c.SecretAccessKey
+	client.Bucket = bucket
+	client.Path = path
+	client.Region = region
+	client.Endpoint = endpoint
+	client.ForcePathStyle = forcePathStyle
+	client.SkipVerify = skipVerify
+	return client, nil
+}
 
-	if v := c.Retention; v > 0 {
-		r.Retention = v
+// newGCSReplicaClientFromConfig returns a new instance of gcs.ReplicaClient built from config.
+func newGCSReplicaClientFromConfig(c *ReplicaConfig, r *litestream.Replica) (_ *gcs.ReplicaClient, err error) {
+	// Ensure URL & constituent parts are not both specified.
+	if c.URL != "" && c.Path != "" {
+		return nil, fmt.Errorf("cannot specify url & path for gcs replica")
+	} else if c.URL != "" && c.Bucket != "" {
+		return nil, fmt.Errorf("cannot specify url & bucket for gcs replica")
 	}
-	if v := c.RetentionCheckInterval; v > 0 {
-		r.RetentionCheckInterval = v
+
+	bucket, path := c.Bucket, c.Path
+
+	// Apply settings from URL, if specified.
+	if c.URL != "" {
+		_, uhost, upath, err := ParseReplicaURL(c.URL)
+		if err != nil {
+			return nil, err
+		}
+
+		// Only apply URL parts to field that have not been overridden.
+		if path == "" {
+			path = upath
+		}
+		if bucket == "" {
+			bucket = uhost
+		}
 	}
-	if v := c.SyncInterval; v > 0 {
-		r.SyncInterval = v
+
+	// Ensure required settings are set.
+	if bucket == "" {
+		return nil, fmt.Errorf("bucket required for gcs replica")
 	}
-	if v := c.SnapshotInterval; v > 0 {
-		r.SnapshotInterval = v
+
+	// Build replica.
+	client := gcs.NewReplicaClient()
+	client.Bucket = bucket
+	client.Path = path
+	return client, nil
+}
+
+// newABSReplicaClientFromConfig returns a new instance of abs.ReplicaClient built from config.
+func newABSReplicaClientFromConfig(c *ReplicaConfig, r *litestream.Replica) (_ *abs.ReplicaClient, err error) {
+	// Ensure URL & constituent parts are not both specified.
+	if c.URL != "" && c.Path != "" {
+		return nil, fmt.Errorf("cannot specify url & path for abs replica")
+	} else if c.URL != "" && c.Bucket != "" {
+		return nil, fmt.Errorf("cannot specify url & bucket for abs replica")
 	}
-	if v := c.ValidationInterval; v > 0 {
-		r.ValidationInterval = v
+
+	// Build replica.
+	client := abs.NewReplicaClient()
+	client.AccountName = c.AccountName
+	client.AccountKey = c.AccountKey
+	client.Bucket = c.Bucket
+	client.Path = c.Path
+	client.Endpoint = c.Endpoint
+
+	// Apply settings from URL, if specified.
+	if c.URL != "" {
+		u, err := url.Parse(c.URL)
+		if err != nil {
+			return nil, err
+		}
+
+		if client.AccountName == "" && u.User != nil {
+			client.AccountName = u.User.Username()
+		}
+		if client.Bucket == "" {
+			client.Bucket = u.Host
+		}
+		if client.Path == "" {
+			client.Path = strings.TrimPrefix(path.Clean(u.Path), "/")
+		}
 	}
-	return r, nil
+
+	// Ensure required settings are set.
+	if client.Bucket == "" {
+		return nil, fmt.Errorf("bucket required for abs replica")
+	}
+
+	return client, nil
+}
+
+// newSFTPReplicaClientFromConfig returns a new instance of sftp.ReplicaClient built from config.
+func newSFTPReplicaClientFromConfig(c *ReplicaConfig, r *litestream.Replica) (_ *sftp.ReplicaClient, err error) {
+	// Ensure URL & constituent parts are not both specified.
+	if c.URL != "" && c.Path != "" {
+		return nil, fmt.Errorf("cannot specify url & path for sftp replica")
+	} else if c.URL != "" && c.Host != "" {
+		return nil, fmt.Errorf("cannot specify url & host for sftp replica")
+	}
+
+	host, user, password, path := c.Host, c.User, c.Password, c.Path
+
+	// Apply settings from URL, if specified.
+	if c.URL != "" {
+		u, err := url.Parse(c.URL)
+		if err != nil {
+			return nil, err
+		}
+
+		// Only apply URL parts to field that have not been overridden.
+		if host == "" {
+			host = u.Host
+		}
+		if user == "" && u.User != nil {
+			user = u.User.Username()
+		}
+		if password == "" && u.User != nil {
+			password, _ = u.User.Password()
+		}
+		if path == "" {
+			path = u.Path
+		}
+	}
+
+	// Ensure required settings are set.
+	if host == "" {
+		return nil, fmt.Errorf("host required for sftp replica")
+	} else if user == "" {
+		return nil, fmt.Errorf("user required for sftp replica")
+	}
+
+	// Build replica.
+	client := sftp.NewReplicaClient()
+	client.Host = host
+	client.User = user
+	client.Password = password
+	client.Path = path
+	client.KeyPath = c.KeyPath
+	return client, nil
 }
 
 // applyLitestreamEnv copies "LITESTREAM" prefixed environment variables to

cmd/litestream/main_notwindows.go

@@ -20,7 +20,7 @@ func runWindowsService(ctx context.Context) error {
 }
 
 func signalChan() <-chan os.Signal {
-	ch := make(chan os.Signal, 1)
+	ch := make(chan os.Signal, 2)
 	signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM)
 	return ch
 }

cmd/litestream/main_test.go

@@ -6,8 +6,9 @@ import (
 	"path/filepath"
 	"testing"
 
-	"github.com/benbjohnson/litestream"
 	main "github.com/benbjohnson/litestream/cmd/litestream"
+	"github.com/benbjohnson/litestream/file"
+	"github.com/benbjohnson/litestream/gcs"
 	"github.com/benbjohnson/litestream/s3"
 )
 
@@ -96,9 +97,9 @@ func TestNewFileReplicaFromConfig(t *testing.T) {
 	r, err := main.NewReplicaFromConfig(&main.ReplicaConfig{Path: "/foo"}, nil)
 	if err != nil {
 		t.Fatal(err)
-	} else if r, ok := r.(*litestream.FileReplica); !ok {
+	} else if client, ok := r.Client.(*file.ReplicaClient); !ok {
 		t.Fatal("unexpected replica type")
-	} else if got, want := r.Path(), "/foo"; got != want {
+	} else if got, want := client.Path(), "/foo"; got != want {
 		t.Fatalf("Path=%s, want %s", got, want)
 	}
 }
@@ -108,17 +109,17 @@ func TestNewS3ReplicaFromConfig(t *testing.T) {
 		r, err := main.NewReplicaFromConfig(&main.ReplicaConfig{URL: "s3://foo/bar"}, nil)
 		if err != nil {
 			t.Fatal(err)
-		} else if r, ok := r.(*s3.Replica); !ok {
+		} else if client, ok := r.Client.(*s3.ReplicaClient); !ok {
 			t.Fatal("unexpected replica type")
-		} else if got, want := r.Bucket, "foo"; got != want {
+		} else if got, want := client.Bucket, "foo"; got != want {
 			t.Fatalf("Bucket=%s, want %s", got, want)
-		} else if got, want := r.Path, "bar"; got != want {
+		} else if got, want := client.Path, "bar"; got != want {
 			t.Fatalf("Path=%s, want %s", got, want)
-		} else if got, want := r.Region, ""; got != want {
+		} else if got, want := client.Region, ""; got != want {
 			t.Fatalf("Region=%s, want %s", got, want)
-		} else if got, want := r.Endpoint, ""; got != want {
+		} else if got, want := client.Endpoint, ""; got != want {
 			t.Fatalf("Endpoint=%s, want %s", got, want)
-		} else if got, want := r.ForcePathStyle, false; got != want {
+		} else if got, want := client.ForcePathStyle, false; got != want {
 			t.Fatalf("ForcePathStyle=%v, want %v", got, want)
 		}
 	})
@@ -127,17 +128,17 @@ func TestNewS3ReplicaFromConfig(t *testing.T) {
 		r, err := main.NewReplicaFromConfig(&main.ReplicaConfig{URL: "s3://foo.localhost:9000/bar"}, nil)
 		if err != nil {
 			t.Fatal(err)
-		} else if r, ok := r.(*s3.Replica); !ok {
+		} else if client, ok := r.Client.(*s3.ReplicaClient); !ok {
 			t.Fatal("unexpected replica type")
-		} else if got, want := r.Bucket, "foo"; got != want {
+		} else if got, want := client.Bucket, "foo"; got != want {
 			t.Fatalf("Bucket=%s, want %s", got, want)
-		} else if got, want := r.Path, "bar"; got != want {
+		} else if got, want := client.Path, "bar"; got != want {
 			t.Fatalf("Path=%s, want %s", got, want)
-		} else if got, want := r.Region, "us-east-1"; got != want {
+		} else if got, want := client.Region, "us-east-1"; got != want {
 			t.Fatalf("Region=%s, want %s", got, want)
-		} else if got, want := r.Endpoint, "http://localhost:9000"; got != want {
+		} else if got, want := client.Endpoint, "http://localhost:9000"; got != want {
 			t.Fatalf("Endpoint=%s, want %s", got, want)
-		} else if got, want := r.ForcePathStyle, true; got != want {
+		} else if got, want := client.ForcePathStyle, true; got != want {
 			t.Fatalf("ForcePathStyle=%v, want %v", got, want)
 		}
 	})
@@ -146,37 +147,31 @@ func TestNewS3ReplicaFromConfig(t *testing.T) {
 		r, err := main.NewReplicaFromConfig(&main.ReplicaConfig{URL: "s3://foo.s3.us-west-000.backblazeb2.com/bar"}, nil)
 		if err != nil {
 			t.Fatal(err)
-		} else if r, ok := r.(*s3.Replica); !ok {
+		} else if client, ok := r.Client.(*s3.ReplicaClient); !ok {
 			t.Fatal("unexpected replica type")
-		} else if got, want := r.Bucket, "foo"; got != want {
+		} else if got, want := client.Bucket, "foo"; got != want {
 			t.Fatalf("Bucket=%s, want %s", got, want)
-		} else if got, want := r.Path, "bar"; got != want {
+		} else if got, want := client.Path, "bar"; got != want {
 			t.Fatalf("Path=%s, want %s", got, want)
-		} else if got, want := r.Region, "us-west-000"; got != want {
+		} else if got, want := client.Region, "us-west-000"; got != want {
 			t.Fatalf("Region=%s, want %s", got, want)
-		} else if got, want := r.Endpoint, "https://s3.us-west-000.backblazeb2.com"; got != want {
+		} else if got, want := client.Endpoint, "https://s3.us-west-000.backblazeb2.com"; got != want {
 			t.Fatalf("Endpoint=%s, want %s", got, want)
-		} else if got, want := r.ForcePathStyle, true; got != want {
-			t.Fatalf("ForcePathStyle=%v, want %v", got, want)
-		}
-	})
-
-	t.Run("GCS", func(t *testing.T) {
-		r, err := main.NewReplicaFromConfig(&main.ReplicaConfig{URL: "s3://foo.storage.googleapis.com/bar"}, nil)
-		if err != nil {
-			t.Fatal(err)
-		} else if r, ok := r.(*s3.Replica); !ok {
-			t.Fatal("unexpected replica type")
-		} else if got, want := r.Bucket, "foo"; got != want {
-			t.Fatalf("Bucket=%s, want %s", got, want)
-		} else if got, want := r.Path, "bar"; got != want {
-			t.Fatalf("Path=%s, want %s", got, want)
-		} else if got, want := r.Region, "us-east-1"; got != want {
-			t.Fatalf("Region=%s, want %s", got, want)
-		} else if got, want := r.Endpoint, "https://storage.googleapis.com"; got != want {
-			t.Fatalf("Endpoint=%s, want %s", got, want)
-		} else if got, want := r.ForcePathStyle, true; got != want {
+		} else if got, want := client.ForcePathStyle, true; got != want {
 			t.Fatalf("ForcePathStyle=%v, want %v", got, want)
 		}
 	})
 }
+
+func TestNewGCSReplicaFromConfig(t *testing.T) {
+	r, err := main.NewReplicaFromConfig(&main.ReplicaConfig{URL: "gcs://foo/bar"}, nil)
+	if err != nil {
+		t.Fatal(err)
+	} else if client, ok := r.Client.(*gcs.ReplicaClient); !ok {
+		t.Fatal("unexpected replica type")
+	} else if got, want := client.Bucket, "foo"; got != want {
+		t.Fatalf("Bucket=%s, want %s", got, want)
+	} else if got, want := client.Path, "bar"; got != want {
+		t.Fatalf("Path=%s, want %s", got, want)
+	}
+}

cmd/litestream/main_windows.go

@@ -62,7 +62,7 @@ func (s *windowsService) Execute(args []string, r <-chan svc.ChangeRequest, stat
 
 	// Instantiate replication command and load configuration.
 	c := NewReplicateCommand()
-	if c.Config, err = ReadConfigFile(DefaultConfigPath()); err != nil {
+	if c.Config, err = ReadConfigFile(DefaultConfigPath(), true); err != nil {
 		log.Printf("cannot load configuration: %s", err)
 		return true, 1
 	}

cmd/litestream/replicate.go

@@ -9,15 +9,23 @@ import (
 	"net/http"
 	_ "net/http/pprof"
 	"os"
-	"time"
+	"os/exec"
 
 	"github.com/benbjohnson/litestream"
+	"github.com/benbjohnson/litestream/abs"
+	"github.com/benbjohnson/litestream/file"
+	"github.com/benbjohnson/litestream/gcs"
 	"github.com/benbjohnson/litestream/s3"
+	"github.com/benbjohnson/litestream/sftp"
+	"github.com/mattn/go-shellwords"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
 // ReplicateCommand represents a command that continuously replicates SQLite databases.
 type ReplicateCommand struct {
+	cmd    *exec.Cmd  // subcommand
+	execCh chan error // subcommand error channel
+
 	Config Config
 
 	// List of managed databases specified in the config.
@@ -25,12 +33,15 @@ type ReplicateCommand struct {
 }
 
 func NewReplicateCommand() *ReplicateCommand {
-	return &ReplicateCommand{}
+	return &ReplicateCommand{
+		execCh: make(chan error),
+	}
 }
 
 // ParseFlags parses the CLI flags and loads the configuration file.
 func (c *ReplicateCommand) ParseFlags(ctx context.Context, args []string) (err error) {
 	fs := flag.NewFlagSet("litestream-replicate", flag.ContinueOnError)
+	execFlag := fs.String("exec", "", "execute subcommand")
 	tracePath := fs.String("trace", "", "trace path")
 	configPath, noExpandEnv := registerConfigFlag(fs)
 	fs.Usage = c.Usage
@@ -48,9 +59,10 @@ func (c *ReplicateCommand) ParseFlags(ctx context.Context, args []string) (err e
 
 		dbConfig := &DBConfig{Path: fs.Arg(0)}
 		for _, u := range fs.Args()[1:] {
+			syncInterval := litestream.DefaultSyncInterval
 			dbConfig.Replicas = append(dbConfig.Replicas, &ReplicaConfig{
 				URL:          u,
-				SyncInterval: 1 * time.Second,
+				SyncInterval: &syncInterval,
 			})
 		}
 		c.Config.DBs = []*DBConfig{dbConfig}
@@ -63,6 +75,11 @@ func (c *ReplicateCommand) ParseFlags(ctx context.Context, args []string) (err e
 		}
 	}
 
+	// Override config exec command, if specified.
+	if *execFlag != "" {
+		c.Config.Exec = *execFlag
+	}
+
 	// Enable trace logging.
 	if *tracePath != "" {
 		f, err := os.Create(*tracePath)
@@ -77,10 +94,11 @@ func (c *ReplicateCommand) ParseFlags(ctx context.Context, args []string) (err e
 }
 
 // Run loads all databases specified in the configuration.
-func (c *ReplicateCommand) Run(ctx context.Context) (err error) {
+func (c *ReplicateCommand) Run() (err error) {
 	// Display version information.
 	log.Printf("litestream %s", Version)
 
+	// Setup databases.
 	if len(c.Config.DBs) == 0 {
 		log.Println("no databases specified in configuration")
 	}
@@ -102,13 +120,19 @@ func (c *ReplicateCommand) Run(ctx context.Context) (err error) {
 	for _, db := range c.DBs {
 		log.Printf("initialized db: %s", db.Path())
 		for _, r := range db.Replicas {
-			switch r := r.(type) {
-			case *litestream.FileReplica:
-				log.Printf("replicating to: name=%q type=%q path=%q", r.Name(), r.Type(), r.Path())
-			case *s3.Replica:
-				log.Printf("replicating to: name=%q type=%q bucket=%q path=%q region=%q endpoint=%q sync-interval=%s", r.Name(), r.Type(), r.Bucket, r.Path, r.Region, r.Endpoint, r.SyncInterval)
+			switch client := r.Client.(type) {
+			case *file.ReplicaClient:
+				log.Printf("replicating to: name=%q type=%q path=%q", r.Name(), client.Type(), client.Path())
+			case *s3.ReplicaClient:
+				log.Printf("replicating to: name=%q type=%q bucket=%q path=%q region=%q endpoint=%q sync-interval=%s", r.Name(), client.Type(), client.Bucket, client.Path, client.Region, client.Endpoint, r.SyncInterval)
+			case *gcs.ReplicaClient:
+				log.Printf("replicating to: name=%q type=%q bucket=%q path=%q sync-interval=%s", r.Name(), client.Type(), client.Bucket, client.Path, r.SyncInterval)
+			case *abs.ReplicaClient:
+				log.Printf("replicating to: name=%q type=%q bucket=%q path=%q endpoint=%q sync-interval=%s", r.Name(), client.Type(), client.Bucket, client.Path, client.Endpoint, r.SyncInterval)
+			case *sftp.ReplicaClient:
+				log.Printf("replicating to: name=%q type=%q host=%q user=%q path=%q sync-interval=%s", r.Name(), client.Type(), client.Host, client.User, client.Path, r.SyncInterval)
 			default:
-				log.Printf("replicating to: name=%q type=%q", r.Name(), r.Type())
+				log.Printf("replicating to: name=%q type=%q", r.Name(), client.Type())
 			}
 		}
 	}
@@ -131,20 +155,36 @@ func (c *ReplicateCommand) Run(ctx context.Context) (err error) {
 		}()
 	}
 
+	// Parse exec commands args & start subprocess.
+	if c.Config.Exec != "" {
+		execArgs, err := shellwords.Parse(c.Config.Exec)
+		if err != nil {
+			return fmt.Errorf("cannot parse exec command: %w", err)
+		}
+
+		c.cmd = exec.Command(execArgs[0], execArgs[1:]...)
+		c.cmd.Env = os.Environ()
+		c.cmd.Stdout = os.Stdout
+		c.cmd.Stderr = os.Stderr
+		if err := c.cmd.Start(); err != nil {
+			return fmt.Errorf("cannot start exec command: %w", err)
+		}
+		go func() { c.execCh <- c.cmd.Wait() }()
+	}
+
 	return nil
 }
 
 // Close closes all open databases.
 func (c *ReplicateCommand) Close() (err error) {
 	for _, db := range c.DBs {
-		if e := db.SoftClose(); e != nil {
+		if e := db.Close(); e != nil {
 			log.Printf("error closing db: path=%s err=%s", db.Path(), e)
 			if err == nil {
 				err = e
 			}
 		}
 	}
-	// TODO(windows): Clear DBs
 	return err
 }
 
@@ -168,6 +208,10 @@ Arguments:
 	    Specifies the configuration file.
 	    Defaults to %s
 
+	-exec CMD
+	    Executes a subcommand. Litestream will exit when the child
+	    process exits. Useful for simple process management.
+
 	-no-expand-env
 	    Disables environment variable expansion in configuration file.
 

cmd/litestream/restore.go

@@ -28,6 +28,7 @@ func (c *RestoreCommand) Run(ctx context.Context, args []string) (err error) {
 	fs.StringVar(&opt.Generation, "generation", "", "generation name")
 	fs.Var((*indexVar)(&opt.Index), "index", "wal index")
 	fs.IntVar(&opt.Parallelism, "parallelism", opt.Parallelism, "parallelism")
+	ifDBNotExists := fs.Bool("if-db-not-exists", false, "")
 	ifReplicaExists := fs.Bool("if-replica-exists", false, "")
 	timestampStr := fs.String("timestamp", "", "timestamp")
 	verbose := fs.Bool("v", false, "verbose output")
@@ -53,19 +54,25 @@ func (c *RestoreCommand) Run(ctx context.Context, args []string) (err error) {
 	}
 
 	// Determine replica & generation to restore from.
-	var r litestream.Replica
+	var r *litestream.Replica
 	if isURL(fs.Arg(0)) {
 		if *configPath != "" {
 			return fmt.Errorf("cannot specify a replica URL and the -config flag")
 		}
-		if r, err = c.loadFromURL(ctx, fs.Arg(0), &opt); err != nil {
+		if r, err = c.loadFromURL(ctx, fs.Arg(0), *ifDBNotExists, &opt); err == errSkipDBExists {
+			fmt.Println("database already exists, skipping")
+			return nil
+		} else if err != nil {
 			return err
 		}
 	} else {
 		if *configPath == "" {
 			*configPath = DefaultConfigPath()
 		}
-		if r, err = c.loadFromConfig(ctx, fs.Arg(0), *configPath, !*noExpandEnv, &opt); err != nil {
+		if r, err = c.loadFromConfig(ctx, fs.Arg(0), *configPath, !*noExpandEnv, *ifDBNotExists, &opt); err == errSkipDBExists {
+			fmt.Println("database already exists, skipping")
+			return nil
+		} else if err != nil {
 			return err
 		}
 	}
@@ -80,21 +87,34 @@ func (c *RestoreCommand) Run(ctx context.Context, args []string) (err error) {
 		return fmt.Errorf("no matching backups found")
 	}
 
-	return litestream.RestoreReplica(ctx, r, opt)
+	return r.Restore(ctx, opt)
 }
 
 // loadFromURL creates a replica & updates the restore options from a replica URL.
-func (c *RestoreCommand) loadFromURL(ctx context.Context, replicaURL string, opt *litestream.RestoreOptions) (litestream.Replica, error) {
-	r, err := NewReplicaFromConfig(&ReplicaConfig{URL: replicaURL}, nil)
+func (c *RestoreCommand) loadFromURL(ctx context.Context, replicaURL string, ifDBNotExists bool, opt *litestream.RestoreOptions) (*litestream.Replica, error) {
+	if opt.OutputPath == "" {
+		return nil, fmt.Errorf("output path required")
+	}
+
+	// Exit successfully if the output file already exists.
+	if _, err := os.Stat(opt.OutputPath); !os.IsNotExist(err) && ifDBNotExists {
+		return nil, errSkipDBExists
+	}
+
+	syncInterval := litestream.DefaultSyncInterval
+	r, err := NewReplicaFromConfig(&ReplicaConfig{
+		URL:          replicaURL,
+		SyncInterval: &syncInterval,
+	}, nil)
 	if err != nil {
 		return nil, err
 	}
-	opt.Generation, _, err = litestream.CalcReplicaRestoreTarget(ctx, r, *opt)
+	opt.Generation, _, err = r.CalcRestoreTarget(ctx, *opt)
 	return r, err
 }
 
 // loadFromConfig returns a replica & updates the restore options from a DB reference.
-func (c *RestoreCommand) loadFromConfig(ctx context.Context, dbPath, configPath string, expandEnv bool, opt *litestream.RestoreOptions) (litestream.Replica, error) {
+func (c *RestoreCommand) loadFromConfig(ctx context.Context, dbPath, configPath string, expandEnv, ifDBNotExists bool, opt *litestream.RestoreOptions) (*litestream.Replica, error) {
 	// Load configuration.
 	config, err := ReadConfigFile(configPath, expandEnv)
 	if err != nil {
@@ -119,6 +139,11 @@ func (c *RestoreCommand) loadFromConfig(ctx context.Context, dbPath, configPath
 		opt.OutputPath = dbPath
 	}
 
+	// Exit successfully if the output file already exists.
+	if _, err := os.Stat(opt.OutputPath); !os.IsNotExist(err) && ifDBNotExists {
+		return nil, errSkipDBExists
+	}
+
 	// Determine the appropriate replica & generation to restore from,
 	r, generation, err := db.CalcRestoreTarget(ctx, *opt)
 	if err != nil {
@@ -169,6 +194,9 @@ Arguments:
 	    Output path of the restored database.
 	    Defaults to original DB path.
 
+	-if-db-not-exists
+	    Returns exit code of 0 if the database already exists.
+
 	-if-replica-exists
 	    Returns exit code of 0 if no backups found.
 
@@ -201,3 +229,5 @@ Examples:
 		DefaultConfigPath(),
 	)
 }
+
+var errSkipDBExists = errors.New("database already exists, skipping")

cmd/litestream/snapshots.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"log"
 	"os"
 	"text/tabwriter"
 	"time"
@@ -29,7 +30,7 @@ func (c *SnapshotsCommand) Run(ctx context.Context, args []string) (err error) {
 	}
 
 	var db *litestream.DB
-	var r litestream.Replica
+	var r *litestream.Replica
 	if isURL(fs.Arg(0)) {
 		if *configPath != "" {
 			return fmt.Errorf("cannot specify a replica URL and the -config flag")
@@ -66,15 +67,11 @@ func (c *SnapshotsCommand) Run(ctx context.Context, args []string) (err error) {
 	}
 
 	// Find snapshots by db or replica.
-	var infos []*litestream.SnapshotInfo
+	var replicas []*litestream.Replica
 	if r != nil {
-		if infos, err = r.Snapshots(ctx); err != nil {
-			return err
-		}
+		replicas = []*litestream.Replica{r}
 	} else {
-		if infos, err = db.Snapshots(ctx); err != nil {
-			return err
-		}
+		replicas = db.Replicas
 	}
 
 	// List all snapshots.
@@ -82,14 +79,21 @@ func (c *SnapshotsCommand) Run(ctx context.Context, args []string) (err error) {
 	defer w.Flush()
 
 	fmt.Fprintln(w, "replica\tgeneration\tindex\tsize\tcreated")
-	for _, info := range infos {
-		fmt.Fprintf(w, "%s\t%s\t%d\t%d\t%s\n",
-			info.Replica,
-			info.Generation,
-			info.Index,
-			info.Size,
-			info.CreatedAt.Format(time.RFC3339),
-		)
+	for _, r := range replicas {
+		infos, err := r.Snapshots(ctx)
+		if err != nil {
+			log.Printf("cannot determine snapshots: %s", err)
+			continue
+		}
+		for _, info := range infos {
+			fmt.Fprintf(w, "%s\t%s\t%d\t%d\t%s\n",
+				r.Name(),
+				info.Generation,
+				info.Index,
+				info.Size,
+				info.CreatedAt.Format(time.RFC3339),
+			)
+		}
 	}
 
 	return nil

cmd/litestream/wal.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"log"
 	"os"
 	"text/tabwriter"
 	"time"
@@ -30,7 +31,7 @@ func (c *WALCommand) Run(ctx context.Context, args []string) (err error) {
 	}
 
 	var db *litestream.DB
-	var r litestream.Replica
+	var r *litestream.Replica
 	if isURL(fs.Arg(0)) {
 		if *configPath != "" {
 			return fmt.Errorf("cannot specify a replica URL and the -config flag")
@@ -67,15 +68,11 @@ func (c *WALCommand) Run(ctx context.Context, args []string) (err error) {
 	}
 
 	// Find WAL files by db or replica.
-	var infos []*litestream.WALInfo
+	var replicas []*litestream.Replica
 	if r != nil {
-		if infos, err = r.WALs(ctx); err != nil {
-			return err
-		}
+		replicas = []*litestream.Replica{r}
 	} else {
-		if infos, err = db.WALs(ctx); err != nil {
-			return err
-		}
+		replicas = db.Replicas
 	}
 
 	// List all WAL files.
@@ -83,19 +80,43 @@ func (c *WALCommand) Run(ctx context.Context, args []string) (err error) {
 	defer w.Flush()
 
 	fmt.Fprintln(w, "replica\tgeneration\tindex\toffset\tsize\tcreated")
-	for _, info := range infos {
-		if *generation != "" && info.Generation != *generation {
-			continue
+	for _, r := range replicas {
+		var generations []string
+		if *generation != "" {
+			generations = []string{*generation}
+		} else {
+			if generations, err = r.Client.Generations(ctx); err != nil {
+				log.Printf("%s: cannot determine generations: %s", r.Name(), err)
+				continue
+			}
 		}
 
-		fmt.Fprintf(w, "%s\t%s\t%d\t%d\t%d\t%s\n",
-			info.Replica,
-			info.Generation,
-			info.Index,
-			info.Offset,
-			info.Size,
-			info.CreatedAt.Format(time.RFC3339),
-		)
+		for _, generation := range generations {
+			if err := func() error {
+				itr, err := r.Client.WALSegments(ctx, generation)
+				if err != nil {
+					return err
+				}
+				defer itr.Close()
+
+				for itr.Next() {
+					info := itr.WALSegment()
+
+					fmt.Fprintf(w, "%s\t%s\t%d\t%d\t%d\t%s\n",
+						r.Name(),
+						info.Generation,
+						info.Index,
+						info.Offset,
+						info.Size,
+						info.CreatedAt.Format(time.RFC3339),
+					)
+				}
+				return itr.Close()
+			}(); err != nil {
+				log.Printf("%s: cannot fetch wal segments: %s", r.Name(), err)
+				continue
+			}
+		}
 	}
 
 	return nil
@@ -104,7 +125,7 @@ func (c *WALCommand) Run(ctx context.Context, args []string) (err error) {
 // Usage prints the help screen to STDOUT.
 func (c *WALCommand) Usage() {
 	fmt.Printf(`
-The wal command lists all wal files available for a database.
+The wal command lists all wal segments available for a database.
 
 Usage:
 
@@ -129,13 +150,13 @@ Arguments:
 
 Examples:
 
-	# List all WAL files for a database.
+	# List all WAL segments for a database.
 	$ litestream wal /path/to/db
 
-	# List all WAL files on S3 for a specific generation.
+	# List all WAL segments on S3 for a specific generation.
 	$ litestream wal -replica s3 -generation xxxxxxxx /path/to/db
 
-	# List all WAL files for replica URL.
+	# List all WAL segments for replica URL.
 	$ litestream wal s3://mybkt/db
 
 `[1:],

db.go

@@ -16,14 +16,13 @@ import (
 	"math/rand"
 	"os"
 	"path/filepath"
-	"sort"
 	"strings"
 	"sync"
 	"time"
 
+	"github.com/benbjohnson/litestream/internal"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
-	"golang.org/x/sync/errgroup"
 )
 
 // Default DB settings.
@@ -45,16 +44,15 @@ const BusyTimeout = 1 * time.Second
 type DB struct {
 	mu       sync.RWMutex
 	path     string        // part to database
+	metaPath string        // Path to the database metadata.
 	db       *sql.DB       // target database
 	f        *os.File      // long-running db file descriptor
 	rtx      *sql.Tx       // long running read transaction
 	pageSize int           // page size, in bytes
 	notify   chan struct{} // closes on WAL change
 
-	uid, gid       int // db user/group obtained on init
-	mode           os.FileMode
-	diruid, dirgid int // db parent user/group obtained on init
-	dirmode        os.FileMode
+	fileInfo os.FileInfo // db info cached during init
+	dirInfo  os.FileInfo // parent dir info cached during init
 
 	ctx    context.Context
 	cancel func()
@@ -96,21 +94,27 @@ type DB struct {
 
 	// List of replicas for the database.
 	// Must be set before calling Open().
-	Replicas []Replica
+	Replicas []*Replica
+
+	// Where to send log messages, defaults to log.Default()
+	Logger *log.Logger
 }
 
 // NewDB returns a new instance of DB for a given path.
 func NewDB(path string) *DB {
+	dir, file := filepath.Split(path)
+
 	db := &DB{
-		path:   path,
-		notify: make(chan struct{}),
-		uid:    -1, gid: -1, mode: 0600,
-		diruid: -1, dirgid: -1, dirmode: 0700,
+		path:     path,
+		metaPath: filepath.Join(dir, "."+file+MetaDirSuffix),
+		notify:   make(chan struct{}),
 
 		MinCheckpointPageN: DefaultMinCheckpointPageN,
 		MaxCheckpointPageN: DefaultMaxCheckpointPageN,
 		CheckpointInterval: DefaultCheckpointInterval,
 		MonitorInterval:    DefaultMonitorInterval,
+
+		Logger: log.Default(),
 	}
 
 	db.dbSizeGauge = dbSizeGaugeVec.WithLabelValues(db.path)
@@ -147,20 +151,24 @@ func (db *DB) WALPath() string {
 
 // MetaPath returns the path to the database metadata.
 func (db *DB) MetaPath() string {
-	dir, file := filepath.Split(db.path)
-	return filepath.Join(dir, "."+file+MetaDirSuffix)
+	return db.metaPath
+}
+
+// SetMetaPath sets the path to database metadata.
+func (db *DB) SetMetaPath(mp string) {
+	db.metaPath = mp
 }
 
 // GenerationNamePath returns the path of the name of the current generation.
 func (db *DB) GenerationNamePath() string {
-	return filepath.Join(db.MetaPath(), "generation")
+	return filepath.Join(db.metaPath, "generation")
 }
 
 // GenerationPath returns the path of a single generation.
 // Panics if generation is blank.
 func (db *DB) GenerationPath(generation string) string {
 	assert(generation != "", "generation name required")
-	return filepath.Join(db.MetaPath(), "generations", generation)
+	return filepath.Join(db.metaPath, "generations", generation)
 }
 
 // ShadowWALDir returns the path of the shadow wal directory.
@@ -196,10 +204,7 @@ func (db *DB) CurrentShadowWALIndex(generation string) (index int, size int64, e
 
 	// Find highest wal index.
 	for _, fi := range fis {
-		if !strings.HasSuffix(fi.Name(), WALExt) {
-			continue
-		}
-		if v, _, _, err := ParseWALPath(fi.Name()); err != nil {
+		if v, err := ParseWALPath(fi.Name()); err != nil {
 			continue // invalid wal filename
 		} else if v > index {
 			index = v
@@ -210,8 +215,18 @@ func (db *DB) CurrentShadowWALIndex(generation string) (index int, size int64, e
 	return index, size, nil
 }
 
+// FileInfo returns the cached file stats for the database file when it was initialized.
+func (db *DB) FileInfo() os.FileInfo {
+	return db.fileInfo
+}
+
+// DirInfo returns the cached file stats for the parent directory of the database file when it was initialized.
+func (db *DB) DirInfo() os.FileInfo {
+	return db.dirInfo
+}
+
 // Replica returns a replica by name.
-func (db *DB) Replica(name string) Replica {
+func (db *DB) Replica(name string) *Replica {
 	for _, r := range db.Replicas {
 		if r.Name() == name {
 			return r
@@ -276,7 +291,7 @@ func (db *DB) Open() (err error) {
 	}
 
 	// Clear old temporary files that my have been left from a crash.
-	if err := removeTmpFiles(db.MetaPath()); err != nil {
+	if err := removeTmpFiles(db.metaPath); err != nil {
 		return fmt.Errorf("cannot remove tmp files: %w", err)
 	}
 
@@ -289,20 +304,9 @@ func (db *DB) Open() (err error) {
 	return nil
 }
 
-// Close releases the read lock & closes the database. This method should only
-// be called by tests as it causes the underlying database to be checkpointed.
+// Close flushes outstanding WAL writes to replicas, releases the read lock,
+// and closes the database.
 func (db *DB) Close() (err error) {
-	return db.close(false)
-}
-
-// SoftClose closes everything but the underlying db connection. This method
-// is available because the binary needs to avoid closing the database on exit
-// to prevent autocheckpointing.
-func (db *DB) SoftClose() (err error) {
-	return db.close(true)
-}
-
-func (db *DB) close(soft bool) (err error) {
 	db.cancel()
 	db.wg.Wait()
 
@@ -323,7 +327,7 @@ func (db *DB) close(soft bool) (err error) {
 				err = e
 			}
 		}
-		r.Stop(!soft)
+		r.Stop(true)
 	}
 
 	// Release the read lock to allow other applications to handle checkpointing.
@@ -333,9 +337,7 @@ func (db *DB) close(soft bool) (err error) {
 		}
 	}
 
-	// Only perform full close if this is not a soft close.
-	// This closes the underlying database connection which can clean up the WAL.
-	if !soft && db.db != nil {
+	if db.db != nil {
 		if e := db.db.Close(); e != nil && err == nil {
 			err = e
 		}
@@ -364,42 +366,6 @@ func (db *DB) UpdatedAt() (time.Time, error) {
 	return t, nil
 }
 
-// Snapshots returns a list of all snapshots across all replicas.
-func (db *DB) Snapshots(ctx context.Context) ([]*SnapshotInfo, error) {
-	var infos []*SnapshotInfo
-	for _, r := range db.Replicas {
-		a, err := r.Snapshots(ctx)
-		if err != nil {
-			return nil, err
-		}
-		infos = append(infos, a...)
-	}
-
-	// Sort in order by time.
-	sort.Slice(infos, func(i, j int) bool {
-		return infos[i].CreatedAt.Before(infos[j].CreatedAt)
-	})
-	return infos, nil
-}
-
-// WALs returns a list of all WAL files across all replicas.
-func (db *DB) WALs(ctx context.Context) ([]*WALInfo, error) {
-	var infos []*WALInfo
-	for _, r := range db.Replicas {
-		a, err := r.WALs(ctx)
-		if err != nil {
-			return nil, err
-		}
-		infos = append(infos, a...)
-	}
-
-	// Sort in order by time.
-	sort.Slice(infos, func(i, j int) bool {
-		return infos[i].CreatedAt.Before(infos[j].CreatedAt)
-	})
-	return infos, nil
-}
-
 // init initializes the connection to the database.
 // Skipped if already initialized or if the database file does not exist.
 func (db *DB) init() (err error) {
@@ -415,21 +381,20 @@ func (db *DB) init() (err error) {
 	} else if err != nil {
 		return err
 	}
-	db.uid, db.gid = fileinfo(fi)
-	db.mode = fi.Mode()
+	db.fileInfo = fi
 
 	// Obtain permissions for parent directory.
 	if fi, err = os.Stat(filepath.Dir(db.path)); err != nil {
 		return err
 	}
-	db.diruid, db.dirgid = fileinfo(fi)
-	db.dirmode = fi.Mode()
+	db.dirInfo = fi
 
 	dsn := db.path
 	dsn += fmt.Sprintf("?_busy_timeout=%d", BusyTimeout.Milliseconds())
 
-	// Connect to SQLite database.
-	if db.db, err = sql.Open("sqlite3", dsn); err != nil {
+	// Connect to SQLite database. Use the driver registered with a hook to
+	// prevent WAL files from being removed.
+	if db.db, err = sql.Open("litestream-sqlite3", dsn); err != nil {
 		return err
 	}
 
@@ -489,13 +454,13 @@ func (db *DB) init() (err error) {
 	}
 
 	// Ensure meta directory structure exists.
-	if err := mkdirAll(db.MetaPath(), db.dirmode, db.diruid, db.dirgid); err != nil {
+	if err := internal.MkdirAll(db.metaPath, db.dirInfo); err != nil {
 		return err
 	}
 
 	// If we have an existing shadow WAL, ensure the headers match.
 	if err := db.verifyHeadersMatch(); err != nil {
-		log.Printf("%s: init: cannot determine last wal position, clearing generation; %s", db.path, err)
+		db.Logger.Printf("%s: init: cannot determine last wal position, clearing generation; %s", db.path, err)
 		if err := os.Remove(db.GenerationNamePath()); err != nil && !os.IsNotExist(err) {
 			return fmt.Errorf("remove generation name: %w", err)
 		}
@@ -565,7 +530,7 @@ func (db *DB) cleanGenerations() error {
 		return err
 	}
 
-	dir := filepath.Join(db.MetaPath(), "generations")
+	dir := filepath.Join(db.metaPath, "generations")
 	fis, err := ioutil.ReadDir(dir)
 	if os.IsNotExist(err) {
 		return nil
@@ -596,7 +561,7 @@ func (db *DB) cleanWAL() error {
 	// Determine lowest index that's been replicated to all replicas.
 	min := -1
 	for _, r := range db.Replicas {
-		pos := r.LastPos()
+		pos := r.Pos()
 		if pos.Generation != generation {
 			pos = Pos{} // different generation, reset index to zero
 		}
@@ -620,7 +585,7 @@ func (db *DB) cleanWAL() error {
 		return err
 	}
 	for _, fi := range fis {
-		if idx, _, _, err := ParseWALPath(fi.Name()); err != nil || idx >= min {
+		if idx, err := ParseWALPath(fi.Name()); err != nil || idx >= min {
 			continue
 		}
 		if err := os.Remove(filepath.Join(dir, fi.Name())); err != nil {
@@ -695,8 +660,8 @@ func (db *DB) createGeneration() (string, error) {
 	generation := hex.EncodeToString(buf)
 
 	// Generate new directory.
-	dir := filepath.Join(db.MetaPath(), "generations", generation)
-	if err := mkdirAll(dir, db.dirmode, db.diruid, db.dirgid); err != nil {
+	dir := filepath.Join(db.metaPath, "generations", generation)
+	if err := internal.MkdirAll(dir, db.dirInfo); err != nil {
 		return "", err
 	}
 
@@ -707,10 +672,15 @@ func (db *DB) createGeneration() (string, error) {
 
 	// Atomically write generation name as current generation.
 	generationNamePath := db.GenerationNamePath()
-	if err := ioutil.WriteFile(generationNamePath+".tmp", []byte(generation+"\n"), db.mode); err != nil {
+	mode := os.FileMode(0600)
+	if db.fileInfo != nil {
+		mode = db.fileInfo.Mode()
+	}
+	if err := ioutil.WriteFile(generationNamePath+".tmp", []byte(generation+"\n"), mode); err != nil {
 		return "", fmt.Errorf("write generation temp file: %w", err)
 	}
-	_ = os.Chown(generationNamePath+".tmp", db.uid, db.gid)
+	uid, gid := internal.Fileinfo(db.fileInfo)
+	_ = os.Chown(generationNamePath+".tmp", uid, gid)
 	if err := os.Rename(generationNamePath+".tmp", generationNamePath); err != nil {
 		return "", fmt.Errorf("rename generation file: %w", err)
 	}
@@ -769,7 +739,7 @@ func (db *DB) Sync(ctx context.Context) (err error) {
 		if info.generation, err = db.createGeneration(); err != nil {
 			return fmt.Errorf("create generation: %w", err)
 		}
-		log.Printf("%s: sync: new generation %q, %s", db.path, info.generation, info.reason)
+		db.Logger.Printf("%s: sync: new generation %q, %s", db.path, info.generation, info.reason)
 
 		// Clear shadow wal info.
 		info.shadowWALPath = db.ShadowWALPath(info.generation, 0)
@@ -801,7 +771,7 @@ func (db *DB) Sync(ctx context.Context) (err error) {
 	if checkpoint {
 		changed = true
 
-		if err := db.checkpointAndInit(ctx, info.generation, checkpointMode); err != nil {
+		if err := db.checkpoint(ctx, info.generation, checkpointMode); err != nil {
 			return fmt.Errorf("checkpoint: mode=%v err=%w", checkpointMode, err)
 		}
 	}
@@ -959,7 +929,7 @@ func (db *DB) syncWAL(info syncInfo) (newSize int64, err error) {
 
 	// Parse index of current shadow WAL file.
 	dir, base := filepath.Split(info.shadowWALPath)
-	index, _, _, err := ParseWALPath(base)
+	index, err := ParseWALPath(base)
 	if err != nil {
 		return 0, fmt.Errorf("cannot parse shadow wal filename: %s", base)
 	}
@@ -993,12 +963,17 @@ func (db *DB) initShadowWALFile(filename string) (int64, error) {
 	}
 
 	// Write header to new WAL shadow file.
-	if err := mkdirAll(filepath.Dir(filename), db.dirmode, db.diruid, db.dirgid); err != nil {
+	mode := os.FileMode(0600)
+	if fi := db.fileInfo; fi != nil {
+		mode = fi.Mode()
+	}
+	if err := internal.MkdirAll(filepath.Dir(filename), db.dirInfo); err != nil {
 		return 0, err
-	} else if err := ioutil.WriteFile(filename, hdr, db.mode); err != nil {
+	} else if err := ioutil.WriteFile(filename, hdr, mode); err != nil {
 		return 0, err
 	}
-	_ = os.Chown(filename, db.uid, db.gid)
+	uid, gid := internal.Fileinfo(db.fileInfo)
+	_ = os.Chown(filename, uid, gid)
 
 	// Copy as much shadow WAL as available.
 	newSize, err := db.copyToShadowWAL(filename)
@@ -1084,7 +1059,7 @@ func (db *DB) copyToShadowWAL(filename string) (newSize int64, err error) {
 		chksum0, chksum1 = Checksum(bo, chksum0, chksum1, frame[:8])  // frame header
 		chksum0, chksum1 = Checksum(bo, chksum0, chksum1, frame[24:]) // frame data
 		if chksum0 != fchksum0 || chksum1 != fchksum1 {
-			log.Printf("copy shadow: checksum mismatch, skipping: offset=%d (%x,%x) != (%x,%x)", offset, chksum0, chksum1, fchksum0, fchksum1)
+			Tracef("%s: copy shadow: checksum mismatch, skipping: offset=%d (%x,%x) != (%x,%x)", db.path, offset, chksum0, chksum1, fchksum0, fchksum1)
 			break
 		}
 
@@ -1155,7 +1130,7 @@ func (db *DB) shadowWALReader(pos Pos) (r *ShadowWALReader, err error) {
 	// Ensure file is closed if any error occurs.
 	defer func() {
 		if err != nil {
-			r.Close()
+			f.Close()
 		}
 	}()
 
@@ -1258,13 +1233,91 @@ func readLastChecksumFrom(f *os.File, pageSize int) (uint32, uint32, error) {
 }
 
 // Checkpoint performs a checkpoint on the WAL file.
-func (db *DB) Checkpoint(mode string) (err error) {
+func (db *DB) Checkpoint(ctx context.Context, mode string) (err error) {
 	db.mu.Lock()
 	defer db.mu.Unlock()
-	return db.checkpoint(mode)
+
+	generation, err := db.CurrentGeneration()
+	if err != nil {
+		return fmt.Errorf("cannot determine generation: %w", err)
+	}
+	return db.checkpoint(ctx, generation, mode)
 }
 
-func (db *DB) checkpoint(mode string) (err error) {
+// checkpointAndInit performs a checkpoint on the WAL file and initializes a
+// new shadow WAL file.
+func (db *DB) checkpoint(ctx context.Context, generation, mode string) error {
+	shadowWALPath, err := db.CurrentShadowWALPath(generation)
+	if err != nil {
+		return err
+	}
+
+	// Read WAL header before checkpoint to check if it has been restarted.
+	hdr, err := readWALHeader(db.WALPath())
+	if err != nil {
+		return err
+	}
+
+	// Copy shadow WAL before checkpoint to copy as much as possible.
+	if _, err := db.copyToShadowWAL(shadowWALPath); err != nil {
+		return fmt.Errorf("cannot copy to end of shadow wal before checkpoint: %w", err)
+	}
+
+	// Execute checkpoint and immediately issue a write to the WAL to ensure
+	// a new page is written.
+	if err := db.execCheckpoint(mode); err != nil {
+		return err
+	} else if _, err = db.db.Exec(`INSERT INTO _litestream_seq (id, seq) VALUES (1, 1) ON CONFLICT (id) DO UPDATE SET seq = seq + 1`); err != nil {
+		return err
+	}
+
+	// If WAL hasn't been restarted, exit.
+	if other, err := readWALHeader(db.WALPath()); err != nil {
+		return err
+	} else if bytes.Equal(hdr, other) {
+		return nil
+	}
+
+	// Start a transaction. This will be promoted immediately after.
+	tx, err := db.db.Begin()
+	if err != nil {
+		return fmt.Errorf("begin: %w", err)
+	}
+	defer func() { _ = rollback(tx) }()
+
+	// Insert into the lock table to promote to a write tx. The lock table
+	// insert will never actually occur because our tx will be rolled back,
+	// however, it will ensure our tx grabs the write lock. Unfortunately,
+	// we can't call "BEGIN IMMEDIATE" as we are already in a transaction.
+	if _, err := tx.ExecContext(ctx, `INSERT INTO _litestream_lock (id) VALUES (1);`); err != nil {
+		return fmt.Errorf("_litestream_lock: %w", err)
+	}
+
+	// Copy the end of the previous WAL before starting a new shadow WAL.
+	if _, err := db.copyToShadowWAL(shadowWALPath); err != nil {
+		return fmt.Errorf("cannot copy to end of shadow wal: %w", err)
+	}
+
+	// Parse index of current shadow WAL file.
+	index, err := ParseWALPath(shadowWALPath)
+	if err != nil {
+		return fmt.Errorf("cannot parse shadow wal filename: %s", shadowWALPath)
+	}
+
+	// Start a new shadow WAL file with next index.
+	newShadowWALPath := filepath.Join(filepath.Dir(shadowWALPath), FormatWALPath(index+1))
+	if _, err := db.initShadowWALFile(newShadowWALPath); err != nil {
+		return fmt.Errorf("cannot init shadow wal file: name=%s err=%w", newShadowWALPath, err)
+	}
+
+	// Release write lock before checkpointing & exiting.
+	if err := tx.Rollback(); err != nil {
+		return fmt.Errorf("rollback post-checkpoint tx: %w", err)
+	}
+	return nil
+}
+
+func (db *DB) execCheckpoint(mode string) (err error) {
 	// Ignore if there is no underlying database.
 	if db.db == nil {
 		return nil
@@ -1310,79 +1363,6 @@ func (db *DB) checkpoint(mode string) (err error) {
 	return nil
 }
 
-// checkpointAndInit performs a checkpoint on the WAL file and initializes a
-// new shadow WAL file.
-func (db *DB) checkpointAndInit(ctx context.Context, generation, mode string) error {
-	shadowWALPath, err := db.CurrentShadowWALPath(generation)
-	if err != nil {
-		return err
-	}
-
-	// Read WAL header before checkpoint to check if it has been restarted.
-	hdr, err := readWALHeader(db.WALPath())
-	if err != nil {
-		return err
-	}
-
-	// Copy shadow WAL before checkpoint to copy as much as possible.
-	if _, err := db.copyToShadowWAL(shadowWALPath); err != nil {
-		return fmt.Errorf("cannot copy to end of shadow wal before checkpoint: %w", err)
-	}
-
-	// Execute checkpoint and immediately issue a write to the WAL to ensure
-	// a new page is written.
-	if err := db.checkpoint(mode); err != nil {
-		return err
-	} else if _, err = db.db.Exec(`INSERT INTO _litestream_seq (id, seq) VALUES (1, 1) ON CONFLICT (id) DO UPDATE SET seq = seq + 1`); err != nil {
-		return err
-	}
-
-	// If WAL hasn't been restarted, exit.
-	if other, err := readWALHeader(db.WALPath()); err != nil {
-		return err
-	} else if bytes.Equal(hdr, other) {
-		return nil
-	}
-
-	// Start a transaction. This will be promoted immediately after.
-	tx, err := db.db.Begin()
-	if err != nil {
-		return fmt.Errorf("begin: %w", err)
-	}
-	defer func() { _ = rollback(tx) }()
-
-	// Insert into the lock table to promote to a write tx. The lock table
-	// insert will never actually occur because our tx will be rolled back,
-	// however, it will ensure our tx grabs the write lock. Unfortunately,
-	// we can't call "BEGIN IMMEDIATE" as we are already in a transaction.
-	if _, err := tx.ExecContext(ctx, `INSERT INTO _litestream_lock (id) VALUES (1);`); err != nil {
-		return fmt.Errorf("_litestream_lock: %w", err)
-	}
-
-	// Copy the end of the previous WAL before starting a new shadow WAL.
-	if _, err := db.copyToShadowWAL(shadowWALPath); err != nil {
-		return fmt.Errorf("cannot copy to end of shadow wal: %w", err)
-	}
-
-	// Parse index of current shadow WAL file.
-	index, _, _, err := ParseWALPath(shadowWALPath)
-	if err != nil {
-		return fmt.Errorf("cannot parse shadow wal filename: %s", shadowWALPath)
-	}
-
-	// Start a new shadow WAL file with next index.
-	newShadowWALPath := filepath.Join(filepath.Dir(shadowWALPath), FormatWALPath(index+1))
-	if _, err := db.initShadowWALFile(newShadowWALPath); err != nil {
-		return fmt.Errorf("cannot init shadow wal file: name=%s err=%w", newShadowWALPath, err)
-	}
-
-	// Release write lock before checkpointing & exiting.
-	if err := tx.Rollback(); err != nil {
-		return fmt.Errorf("rollback post-checkpoint tx: %w", err)
-	}
-	return nil
-}
-
 // monitor runs in a separate goroutine and monitors the database & WAL.
 func (db *DB) monitor() {
 	ticker := time.NewTicker(db.MonitorInterval)
@@ -1398,194 +1378,17 @@ func (db *DB) monitor() {
 
 		// Sync the database to the shadow WAL.
 		if err := db.Sync(db.ctx); err != nil && !errors.Is(err, context.Canceled) {
-			log.Printf("%s: sync error: %s", db.path, err)
+			db.Logger.Printf("%s: sync error: %s", db.path, err)
 		}
 	}
 }
 
-// RestoreReplica restores the database from a replica based on the options given.
-// This method will restore into opt.OutputPath, if specified, or into the
-// DB's original database path. It can optionally restore from a specific
-// replica or generation or it will automatically choose the best one. Finally,
-// a timestamp can be specified to restore the database to a specific
-// point-in-time.
-func RestoreReplica(ctx context.Context, r Replica, opt RestoreOptions) (err error) {
-	// Validate options.
-	if opt.OutputPath == "" {
-		return fmt.Errorf("output path required")
-	} else if opt.Generation == "" && opt.Index != math.MaxInt32 {
-		return fmt.Errorf("must specify generation when restoring to index")
-	} else if opt.Index != math.MaxInt32 && !opt.Timestamp.IsZero() {
-		return fmt.Errorf("cannot specify index & timestamp to restore")
-	}
-
-	// Ensure logger exists.
-	logger := opt.Logger
-	if logger == nil {
-		logger = log.New(ioutil.Discard, "", 0)
-	}
-
-	logPrefix := r.Name()
-	if db := r.DB(); db != nil {
-		logPrefix = fmt.Sprintf("%s(%s)", db.Path(), r.Name())
-	}
-
-	// Ensure output path does not already exist.
-	if _, err := os.Stat(opt.OutputPath); err == nil {
-		return fmt.Errorf("cannot restore, output path already exists: %s", opt.OutputPath)
-	} else if err != nil && !os.IsNotExist(err) {
-		return err
-	}
-
-	// Find lastest snapshot that occurs before timestamp or index.
-	var minWALIndex int
-	if opt.Index < math.MaxInt32 {
-		if minWALIndex, err = SnapshotIndexByIndex(ctx, r, opt.Generation, opt.Index); err != nil {
-			return fmt.Errorf("cannot find snapshot index: %w", err)
-		}
-	} else {
-		if minWALIndex, err = SnapshotIndexAt(ctx, r, opt.Generation, opt.Timestamp); err != nil {
-			return fmt.Errorf("cannot find snapshot index by timestamp: %w", err)
-		}
-	}
-
-	// Find the maximum WAL index that occurs before timestamp.
-	maxWALIndex, err := WALIndexAt(ctx, r, opt.Generation, opt.Index, opt.Timestamp)
-	if err != nil {
-		return fmt.Errorf("cannot find max wal index for restore: %w", err)
-	}
-	snapshotOnly := maxWALIndex == -1
-
-	// Initialize starting position.
-	pos := Pos{Generation: opt.Generation, Index: minWALIndex}
-	tmpPath := opt.OutputPath + ".tmp"
-
-	// Copy snapshot to output path.
-	logger.Printf("%s: restoring snapshot %s/%08x to %s", logPrefix, opt.Generation, minWALIndex, tmpPath)
-	if err := restoreSnapshot(ctx, r, pos.Generation, pos.Index, tmpPath); err != nil {
-		return fmt.Errorf("cannot restore snapshot: %w", err)
-	}
-
-	// If no WAL files available, move snapshot to final path & exit early.
-	if snapshotOnly {
-		logger.Printf("%s: snapshot only, finalizing database", logPrefix)
-		return os.Rename(tmpPath, opt.OutputPath)
-	}
-
-	// Begin processing WAL files.
-	logger.Printf("%s: restoring wal files: generation=%s index=[%08x,%08x]", logPrefix, opt.Generation, minWALIndex, maxWALIndex)
-
-	// Fill input channel with all WAL indexes to be loaded in order.
-	ch := make(chan int, maxWALIndex-minWALIndex+1)
-	for index := minWALIndex; index <= maxWALIndex; index++ {
-		ch <- index
-	}
-	close(ch)
-
-	// Track load state for each WAL.
-	var mu sync.Mutex
-	cond := sync.NewCond(&mu)
-	walStates := make([]walRestoreState, maxWALIndex-minWALIndex+1)
-
-	parallelism := opt.Parallelism
-	if parallelism < 1 {
-		parallelism = 1
-	}
-
-	// Download WAL files to disk in parallel.
-	g, ctx := errgroup.WithContext(ctx)
-	for i := 0; i < parallelism; i++ {
-		g.Go(func() error {
-			for {
-				select {
-				case <-ctx.Done():
-					cond.Broadcast()
-					return err
-				case index, ok := <-ch:
-					if !ok {
-						cond.Broadcast()
-						return nil
-					}
-
-					startTime := time.Now()
-
-					err := downloadWAL(ctx, r, opt.Generation, index, tmpPath)
-					if err != nil {
-						err = fmt.Errorf("cannot download wal %s/%08x: %w", opt.Generation, index, err)
-					}
-
-					// Mark index as ready-to-apply and notify applying code.
-					mu.Lock()
-					walStates[index-minWALIndex] = walRestoreState{ready: true, err: err}
-					mu.Unlock()
-					cond.Broadcast()
-
-					// Returning the error here will cancel the other goroutines.
-					if err != nil {
-						return err
-					}
-
-					logger.Printf("%s: downloaded wal %s/%08x elapsed=%s",
-						logPrefix, opt.Generation, index,
-						time.Since(startTime).String(),
-					)
-				}
-			}
-		})
-	}
-
-	// Apply WAL files in order as they are ready.
-	for index := minWALIndex; index <= maxWALIndex; index++ {
-		// Wait until next WAL file is ready to apply.
-		mu.Lock()
-		for !walStates[index-minWALIndex].ready {
-			if err := ctx.Err(); err != nil {
-				return err
-			}
-			cond.Wait()
-		}
-		if err := walStates[index-minWALIndex].err; err != nil {
-			return err
-		}
-		mu.Unlock()
-
-		// Apply WAL to database file.
-		startTime := time.Now()
-		if err = applyWAL(ctx, index, tmpPath); err != nil {
-			return fmt.Errorf("cannot apply wal: %w", err)
-		}
-		logger.Printf("%s: applied wal %s/%08x elapsed=%s",
-			logPrefix, opt.Generation, index,
-			time.Since(startTime).String(),
-		)
-	}
-
-	// Ensure all goroutines finish. All errors should have been handled during
-	// the processing of WAL files but this ensures that all processing is done.
-	if err := g.Wait(); err != nil {
-		return err
-	}
-
-	// Copy file to final location.
-	logger.Printf("%s: renaming database from temporary location", logPrefix)
-	if err := os.Rename(tmpPath, opt.OutputPath); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-type walRestoreState struct {
-	ready bool
-	err   error
-}
-
 // CalcRestoreTarget returns a replica & generation to restore from based on opt criteria.
-func (db *DB) CalcRestoreTarget(ctx context.Context, opt RestoreOptions) (Replica, string, error) {
+func (db *DB) CalcRestoreTarget(ctx context.Context, opt RestoreOptions) (*Replica, string, error) {
 	var target struct {
-		replica    Replica
+		replica    *Replica
 		generation string
-		stats      GenerationStats
+		updatedAt  time.Time
 	}
 
 	for _, r := range db.Replicas {
@@ -1594,134 +1397,21 @@ func (db *DB) CalcRestoreTarget(ctx context.Context, opt RestoreOptions) (Replic
 			continue
 		}
 
-		generation, stats, err := CalcReplicaRestoreTarget(ctx, r, opt)
+		generation, updatedAt, err := r.CalcRestoreTarget(ctx, opt)
 		if err != nil {
 			return nil, "", err
 		}
 
 		// Use the latest replica if we have multiple candidates.
-		if !stats.UpdatedAt.After(target.stats.UpdatedAt) {
+		if !updatedAt.After(target.updatedAt) {
 			continue
 		}
 
-		target.replica, target.generation, target.stats = r, generation, stats
+		target.replica, target.generation, target.updatedAt = r, generation, updatedAt
 	}
 	return target.replica, target.generation, nil
 }
 
-// CalcReplicaRestoreTarget returns a generation to restore from.
-func CalcReplicaRestoreTarget(ctx context.Context, r Replica, opt RestoreOptions) (generation string, stats GenerationStats, err error) {
-	var target struct {
-		generation string
-		stats      GenerationStats
-	}
-
-	generations, err := r.Generations(ctx)
-	if err != nil {
-		return "", stats, fmt.Errorf("cannot fetch generations: %w", err)
-	}
-
-	// Search generations for one that contains the requested timestamp.
-	for _, generation := range generations {
-		// Skip generation if it does not match filter.
-		if opt.Generation != "" && generation != opt.Generation {
-			continue
-		}
-
-		// Fetch stats for generation.
-		stats, err := r.GenerationStats(ctx, generation)
-		if err != nil {
-			return "", stats, fmt.Errorf("cannot determine stats for generation (%s/%s): %s", r.Name(), generation, err)
-		}
-
-		// Skip if it does not contain timestamp.
-		if !opt.Timestamp.IsZero() {
-			if opt.Timestamp.Before(stats.CreatedAt) || opt.Timestamp.After(stats.UpdatedAt) {
-				continue
-			}
-		}
-
-		// Use the latest replica if we have multiple candidates.
-		if !stats.UpdatedAt.After(target.stats.UpdatedAt) {
-			continue
-		}
-
-		target.generation = generation
-		target.stats = stats
-	}
-
-	return target.generation, target.stats, nil
-}
-
-// restoreSnapshot copies a snapshot from the replica to a file.
-func restoreSnapshot(ctx context.Context, r Replica, generation string, index int, filename string) error {
-	// Determine the user/group & mode based on the DB, if available.
-	uid, gid, mode := -1, -1, os.FileMode(0600)
-	diruid, dirgid, dirmode := -1, -1, os.FileMode(0700)
-	if db := r.DB(); db != nil {
-		uid, gid, mode = db.uid, db.gid, db.mode
-		diruid, dirgid, dirmode = db.diruid, db.dirgid, db.dirmode
-	}
-
-	if err := mkdirAll(filepath.Dir(filename), dirmode, diruid, dirgid); err != nil {
-		return err
-	}
-
-	f, err := createFile(filename, mode, uid, gid)
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-
-	rd, err := r.SnapshotReader(ctx, generation, index)
-	if err != nil {
-		return err
-	}
-	defer rd.Close()
-
-	if _, err := io.Copy(f, rd); err != nil {
-		return err
-	}
-
-	if err := f.Sync(); err != nil {
-		return err
-	}
-	return f.Close()
-}
-
-// downloadWAL copies a WAL file from the replica to a local copy next to the DB.
-// The WAL is later applied by applyWAL(). This function can be run in parallel
-// to download multiple WAL files simultaneously.
-func downloadWAL(ctx context.Context, r Replica, generation string, index int, dbPath string) error {
-	// Determine the user/group & mode based on the DB, if available.
-	uid, gid, mode := -1, -1, os.FileMode(0600)
-	if db := r.DB(); db != nil {
-		uid, gid, mode = db.uid, db.gid, db.mode
-	}
-
-	// Open WAL file from replica.
-	rd, err := r.WALReader(ctx, generation, index)
-	if err != nil {
-		return err
-	}
-	defer rd.Close()
-
-	// Open handle to destination WAL path.
-	f, err := createFile(fmt.Sprintf("%s-%08x-wal", dbPath, index), mode, uid, gid)
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-
-	// Copy WAL to target path.
-	if _, err := io.Copy(f, rd); err != nil {
-		return err
-	} else if err := f.Close(); err != nil {
-		return err
-	}
-	return nil
-}
-
 // applyWAL performs a truncating checkpoint on the given database.
 func applyWAL(ctx context.Context, index int, dbPath string) error {
 	// Copy WAL file from it's staging path to the correct "-wal" location.
@@ -1730,7 +1420,7 @@ func applyWAL(ctx context.Context, index int, dbPath string) error {
 	}
 
 	// Open SQLite database and force a truncating checkpoint.
-	d, err := sql.Open("sqlite3", dbPath)
+	d, err := sql.Open("litestream-sqlite3", dbPath)
 	if err != nil {
 		return err
 	}
@@ -1770,7 +1460,7 @@ func (db *DB) CRC64(ctx context.Context) (uint64, Pos, error) {
 	}
 
 	// Force a RESTART checkpoint to ensure the database is at the start of the WAL.
-	if err := db.checkpointAndInit(ctx, generation, CheckpointModeRestart); err != nil {
+	if err := db.checkpoint(ctx, generation, CheckpointModeRestart); err != nil {
 		return 0, Pos{}, err
 	}
 
@@ -1836,80 +1526,58 @@ func NewRestoreOptions() RestoreOptions {
 // Database metrics.
 var (
 	dbSizeGaugeVec = promauto.NewGaugeVec(prometheus.GaugeOpts{
-		Namespace: "litestream",
-		Subsystem: "db",
-		Name:      "db_size",
-		Help:      "The current size of the real DB",
+		Name: "litestream_db_size",
+		Help: "The current size of the real DB",
 	}, []string{"db"})
 
 	walSizeGaugeVec = promauto.NewGaugeVec(prometheus.GaugeOpts{
-		Namespace: "litestream",
-		Subsystem: "db",
-		Name:      "wal_size",
-		Help:      "The current size of the real WAL",
+		Name: "litestream_wal_size",
+		Help: "The current size of the real WAL",
 	}, []string{"db"})
 
 	totalWALBytesCounterVec = promauto.NewCounterVec(prometheus.CounterOpts{
-		Namespace: "litestream",
-		Subsystem: "db",
-		Name:      "total_wal_bytes",
-		Help:      "Total number of bytes written to shadow WAL",
+		Name: "litestream_total_wal_bytes",
+		Help: "Total number of bytes written to shadow WAL",
 	}, []string{"db"})
 
 	shadowWALIndexGaugeVec = promauto.NewGaugeVec(prometheus.GaugeOpts{
-		Namespace: "litestream",
-		Subsystem: "db",
-		Name:      "shadow_wal_index",
-		Help:      "The current index of the shadow WAL",
+		Name: "litestream_shadow_wal_index",
+		Help: "The current index of the shadow WAL",
 	}, []string{"db"})
 
 	shadowWALSizeGaugeVec = promauto.NewGaugeVec(prometheus.GaugeOpts{
-		Namespace: "litestream",
-		Subsystem: "db",
-		Name:      "shadow_wal_size",
-		Help:      "Current size of shadow WAL, in bytes",
+		Name: "litestream_shadow_wal_size",
+		Help: "Current size of shadow WAL, in bytes",
 	}, []string{"db"})
 
 	syncNCounterVec = promauto.NewCounterVec(prometheus.CounterOpts{
-		Namespace: "litestream",
-		Subsystem: "db",
-		Name:      "sync_count",
-		Help:      "Number of sync operations performed",
+		Name: "litestream_sync_count",
+		Help: "Number of sync operations performed",
 	}, []string{"db"})
 
 	syncErrorNCounterVec = promauto.NewCounterVec(prometheus.CounterOpts{
-		Namespace: "litestream",
-		Subsystem: "db",
-		Name:      "sync_error_count",
-		Help:      "Number of sync errors that have occurred",
+		Name: "litestream_sync_error_count",
+		Help: "Number of sync errors that have occurred",
 	}, []string{"db"})
 
 	syncSecondsCounterVec = promauto.NewCounterVec(prometheus.CounterOpts{
-		Namespace: "litestream",
-		Subsystem: "db",
-		Name:      "sync_seconds",
-		Help:      "Time spent syncing shadow WAL, in seconds",
+		Name: "litestream_sync_seconds",
+		Help: "Time spent syncing shadow WAL, in seconds",
 	}, []string{"db"})
 
 	checkpointNCounterVec = promauto.NewCounterVec(prometheus.CounterOpts{
-		Namespace: "litestream",
-		Subsystem: "db",
-		Name:      "checkpoint_count",
-		Help:      "Number of checkpoint operations performed",
+		Name: "litestream_checkpoint_count",
+		Help: "Number of checkpoint operations performed",
 	}, []string{"db", "mode"})
 
 	checkpointErrorNCounterVec = promauto.NewCounterVec(prometheus.CounterOpts{
-		Namespace: "litestream",
-		Subsystem: "db",
-		Name:      "checkpoint_error_count",
-		Help:      "Number of checkpoint errors that have occurred",
+		Name: "litestream_checkpoint_error_count",
+		Help: "Number of checkpoint errors that have occurred",
 	}, []string{"db", "mode"})
 
 	checkpointSecondsCounterVec = promauto.NewCounterVec(prometheus.CounterOpts{
-		Namespace: "litestream",
-		Subsystem: "db",
-		Name:      "checkpoint_seconds",
-		Help:      "Time spent checkpointing WAL, in seconds",
+		Name: "litestream_checkpoint_seconds",
+		Help: "Time spent checkpointing WAL, in seconds",
 	}, []string{"db", "mode"})
 )
 

db_test.go

@@ -151,7 +151,7 @@ func TestDB_CRC64(t *testing.T) {
 		}
 
 		// Checkpoint change into database. Checksum should change.
-		if _, err := sqldb.Exec(`PRAGMA wal_checkpoint(TRUNCATE);`); err != nil {
+		if err := db.Checkpoint(context.Background(), litestream.CheckpointModeTruncate); err != nil {
 			t.Fatal(err)
 		}
 
@@ -262,7 +262,7 @@ func TestDB_Sync(t *testing.T) {
 		}
 
 		// Checkpoint & fully close which should close WAL file.
-		if err := db.Checkpoint(litestream.CheckpointModeTruncate); err != nil {
+		if err := db.Checkpoint(context.Background(), litestream.CheckpointModeTruncate); err != nil {
 			t.Fatal(err)
 		} else if err := db.Close(); err != nil {
 			t.Fatal(err)
@@ -270,8 +270,8 @@ func TestDB_Sync(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		// Verify WAL does not exist.
-		if _, err := os.Stat(db.WALPath()); !os.IsNotExist(err) {
+		// Remove WAL file.
+		if err := os.Remove(db.WALPath()); err != nil {
 			t.Fatal(err)
 		}
 

file/replica_client.go

@@ -0,0 +1,381 @@
+package file
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"sort"
+
+	"github.com/benbjohnson/litestream"
+	"github.com/benbjohnson/litestream/internal"
+)
+
+// ReplicaClientType is the client type for this package.
+const ReplicaClientType = "file"
+
+var _ litestream.ReplicaClient = (*ReplicaClient)(nil)
+
+// ReplicaClient is a client for writing snapshots & WAL segments to disk.
+type ReplicaClient struct {
+	path string // destination path
+
+	Replica *litestream.Replica
+}
+
+// NewReplicaClient returns a new instance of ReplicaClient.
+func NewReplicaClient(path string) *ReplicaClient {
+	return &ReplicaClient{
+		path: path,
+	}
+}
+
+// db returns the database, if available.
+func (c *ReplicaClient) db() *litestream.DB {
+	if c.Replica == nil {
+		return nil
+	}
+	return c.Replica.DB()
+}
+
+// Type returns "file" as the client type.
+func (c *ReplicaClient) Type() string {
+	return ReplicaClientType
+}
+
+// Path returns the destination path to replicate the database to.
+func (c *ReplicaClient) Path() string {
+	return c.path
+}
+
+// GenerationsDir returns the path to a generation root directory.
+func (c *ReplicaClient) GenerationsDir() (string, error) {
+	if c.path == "" {
+		return "", fmt.Errorf("file replica path required")
+	}
+	return filepath.Join(c.path, "generations"), nil
+}
+
+// GenerationDir returns the path to a generation's root directory.
+func (c *ReplicaClient) GenerationDir(generation string) (string, error) {
+	dir, err := c.GenerationsDir()
+	if err != nil {
+		return "", err
+	} else if generation == "" {
+		return "", fmt.Errorf("generation required")
+	}
+	return filepath.Join(dir, generation), nil
+}
+
+// SnapshotsDir returns the path to a generation's snapshot directory.
+func (c *ReplicaClient) SnapshotsDir(generation string) (string, error) {
+	dir, err := c.GenerationDir(generation)
+	if err != nil {
+		return "", err
+	}
+	return filepath.Join(dir, "snapshots"), nil
+}
+
+// SnapshotPath returns the path to an uncompressed snapshot file.
+func (c *ReplicaClient) SnapshotPath(generation string, index int) (string, error) {
+	dir, err := c.SnapshotsDir(generation)
+	if err != nil {
+		return "", err
+	}
+	return filepath.Join(dir, litestream.FormatSnapshotPath(index)), nil
+}
+
+// WALDir returns the path to a generation's WAL directory
+func (c *ReplicaClient) WALDir(generation string) (string, error) {
+	dir, err := c.GenerationDir(generation)
+	if err != nil {
+		return "", err
+	}
+	return filepath.Join(dir, "wal"), nil
+}
+
+// WALSegmentPath returns the path to a WAL segment file.
+func (c *ReplicaClient) WALSegmentPath(generation string, index int, offset int64) (string, error) {
+	dir, err := c.WALDir(generation)
+	if err != nil {
+		return "", err
+	}
+	return filepath.Join(dir, litestream.FormatWALSegmentPath(index, offset)), nil
+}
+
+// Generations returns a list of available generation names.
+func (c *ReplicaClient) Generations(ctx context.Context) ([]string, error) {
+	root, err := c.GenerationsDir()
+	if err != nil {
+		return nil, fmt.Errorf("cannot determine generations path: %w", err)
+	}
+
+	fis, err := ioutil.ReadDir(root)
+	if os.IsNotExist(err) {
+		return nil, nil
+	} else if err != nil {
+		return nil, err
+	}
+
+	var generations []string
+	for _, fi := range fis {
+		if !litestream.IsGenerationName(fi.Name()) {
+			continue
+		} else if !fi.IsDir() {
+			continue
+		}
+		generations = append(generations, fi.Name())
+	}
+	return generations, nil
+}
+
+// DeleteGeneration deletes all snapshots & WAL segments within a generation.
+func (c *ReplicaClient) DeleteGeneration(ctx context.Context, generation string) error {
+	dir, err := c.GenerationDir(generation)
+	if err != nil {
+		return fmt.Errorf("cannot determine generation path: %w", err)
+	}
+
+	if err := os.RemoveAll(dir); err != nil && !os.IsNotExist(err) {
+		return err
+	}
+	return nil
+}
+
+// Snapshots returns an iterator over all available snapshots for a generation.
+func (c *ReplicaClient) Snapshots(ctx context.Context, generation string) (litestream.SnapshotIterator, error) {
+	dir, err := c.SnapshotsDir(generation)
+	if err != nil {
+		return nil, fmt.Errorf("cannot determine snapshots path: %w", err)
+	}
+
+	f, err := os.Open(dir)
+	if os.IsNotExist(err) {
+		return litestream.NewSnapshotInfoSliceIterator(nil), nil
+	} else if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	fis, err := f.Readdir(-1)
+	if err != nil {
+		return nil, err
+	}
+
+	// Iterate over every file and convert to metadata.
+	infos := make([]litestream.SnapshotInfo, 0, len(fis))
+	for _, fi := range fis {
+		// Parse index from filename.
+		index, err := litestream.ParseSnapshotPath(fi.Name())
+		if err != nil {
+			continue
+		}
+
+		infos = append(infos, litestream.SnapshotInfo{
+			Generation: generation,
+			Index:      index,
+			Size:       fi.Size(),
+			CreatedAt:  fi.ModTime().UTC(),
+		})
+	}
+
+	sort.Sort(litestream.SnapshotInfoSlice(infos))
+
+	return litestream.NewSnapshotInfoSliceIterator(infos), nil
+}
+
+// WriteSnapshot writes LZ4 compressed data from rd into a file on disk.
+func (c *ReplicaClient) WriteSnapshot(ctx context.Context, generation string, index int, rd io.Reader) (info litestream.SnapshotInfo, err error) {
+	filename, err := c.SnapshotPath(generation, index)
+	if err != nil {
+		return info, fmt.Errorf("cannot determine snapshot path: %w", err)
+	}
+
+	var fileInfo, dirInfo os.FileInfo
+	if db := c.db(); db != nil {
+		fileInfo, dirInfo = db.FileInfo(), db.DirInfo()
+	}
+
+	// Ensure parent directory exists.
+	if err := internal.MkdirAll(filepath.Dir(filename), dirInfo); err != nil {
+		return info, err
+	}
+
+	// Write snapshot to temporary file next to destination path.
+	f, err := internal.CreateFile(filename+".tmp", fileInfo)
+	if err != nil {
+		return info, err
+	}
+	defer f.Close()
+
+	if _, err := io.Copy(f, rd); err != nil {
+		return info, err
+	} else if err := f.Sync(); err != nil {
+		return info, err
+	} else if err := f.Close(); err != nil {
+		return info, err
+	}
+
+	// Build metadata.
+	fi, err := os.Stat(filename + ".tmp")
+	if err != nil {
+		return info, err
+	}
+	info = litestream.SnapshotInfo{
+		Generation: generation,
+		Index:      index,
+		Size:       fi.Size(),
+		CreatedAt:  fi.ModTime().UTC(),
+	}
+
+	// Move snapshot to final path when it has been fully written & synced to disk.
+	if err := os.Rename(filename+".tmp", filename); err != nil {
+		return info, err
+	}
+
+	return info, nil
+}
+
+// SnapshotReader returns a reader for snapshot data at the given generation/index.
+// Returns os.ErrNotExist if no matching index is found.
+func (c *ReplicaClient) SnapshotReader(ctx context.Context, generation string, index int) (io.ReadCloser, error) {
+	filename, err := c.SnapshotPath(generation, index)
+	if err != nil {
+		return nil, fmt.Errorf("cannot determine snapshot path: %w", err)
+	}
+	return os.Open(filename)
+}
+
+// DeleteSnapshot deletes a snapshot with the given generation & index.
+func (c *ReplicaClient) DeleteSnapshot(ctx context.Context, generation string, index int) error {
+	filename, err := c.SnapshotPath(generation, index)
+	if err != nil {
+		return fmt.Errorf("cannot determine snapshot path: %w", err)
+	}
+	if err := os.Remove(filename); err != nil && !os.IsNotExist(err) {
+		return err
+	}
+	return nil
+}
+
+// WALSegments returns an iterator over all available WAL files for a generation.
+func (c *ReplicaClient) WALSegments(ctx context.Context, generation string) (litestream.WALSegmentIterator, error) {
+	dir, err := c.WALDir(generation)
+	if err != nil {
+		return nil, fmt.Errorf("cannot determine wal path: %w", err)
+	}
+
+	f, err := os.Open(dir)
+	if os.IsNotExist(err) {
+		return litestream.NewWALSegmentInfoSliceIterator(nil), nil
+	} else if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	fis, err := f.Readdir(-1)
+	if err != nil {
+		return nil, err
+	}
+
+	// Iterate over every file and convert to metadata.
+	infos := make([]litestream.WALSegmentInfo, 0, len(fis))
+	for _, fi := range fis {
+		// Parse index from filename.
+		index, offset, err := litestream.ParseWALSegmentPath(fi.Name())
+		if err != nil {
+			continue
+		}
+
+		infos = append(infos, litestream.WALSegmentInfo{
+			Generation: generation,
+			Index:      index,
+			Offset:     offset,
+			Size:       fi.Size(),
+			CreatedAt:  fi.ModTime().UTC(),
+		})
+	}
+
+	sort.Sort(litestream.WALSegmentInfoSlice(infos))
+
+	return litestream.NewWALSegmentInfoSliceIterator(infos), nil
+}
+
+// WriteWALSegment writes LZ4 compressed data from rd into a file on disk.
+func (c *ReplicaClient) WriteWALSegment(ctx context.Context, pos litestream.Pos, rd io.Reader) (info litestream.WALSegmentInfo, err error) {
+	filename, err := c.WALSegmentPath(pos.Generation, pos.Index, pos.Offset)
+	if err != nil {
+		return info, fmt.Errorf("cannot determine wal segment path: %w", err)
+	}
+
+	var fileInfo, dirInfo os.FileInfo
+	if db := c.db(); db != nil {
+		fileInfo, dirInfo = db.FileInfo(), db.DirInfo()
+	}
+
+	// Ensure parent directory exists.
+	if err := internal.MkdirAll(filepath.Dir(filename), dirInfo); err != nil {
+		return info, err
+	}
+
+	// Write WAL segment to temporary file next to destination path.
+	f, err := internal.CreateFile(filename+".tmp", fileInfo)
+	if err != nil {
+		return info, err
+	}
+	defer f.Close()
+
+	if _, err := io.Copy(f, rd); err != nil {
+		return info, err
+	} else if err := f.Sync(); err != nil {
+		return info, err
+	} else if err := f.Close(); err != nil {
+		return info, err
+	}
+
+	// Build metadata.
+	fi, err := os.Stat(filename + ".tmp")
+	if err != nil {
+		return info, err
+	}
+	info = litestream.WALSegmentInfo{
+		Generation: pos.Generation,
+		Index:      pos.Index,
+		Offset:     pos.Offset,
+		Size:       fi.Size(),
+		CreatedAt:  fi.ModTime().UTC(),
+	}
+
+	// Move WAL segment to final path when it has been written & synced to disk.
+	if err := os.Rename(filename+".tmp", filename); err != nil {
+		return info, err
+	}
+
+	return info, nil
+}
+
+// WALSegmentReader returns a reader for a section of WAL data at the given position.
+// Returns os.ErrNotExist if no matching index/offset is found.
+func (c *ReplicaClient) WALSegmentReader(ctx context.Context, pos litestream.Pos) (io.ReadCloser, error) {
+	filename, err := c.WALSegmentPath(pos.Generation, pos.Index, pos.Offset)
+	if err != nil {
+		return nil, fmt.Errorf("cannot determine wal segment path: %w", err)
+	}
+	return os.Open(filename)
+}
+
+// DeleteWALSegments deletes WAL segments at the given positions.
+func (c *ReplicaClient) DeleteWALSegments(ctx context.Context, a []litestream.Pos) error {
+	for _, pos := range a {
+		filename, err := c.WALSegmentPath(pos.Generation, pos.Index, pos.Offset)
+		if err != nil {
+			return fmt.Errorf("cannot determine wal segment path: %w", err)
+		}
+		if err := os.Remove(filename); err != nil && !os.IsNotExist(err) {
+			return err
+		}
+	}
+	return nil
+}

file/replica_client_test.go

@@ -0,0 +1,223 @@
+package file_test
+
+import (
+	"testing"
+
+	"github.com/benbjohnson/litestream/file"
+)
+
+func TestReplicaClient_Path(t *testing.T) {
+	c := file.NewReplicaClient("/foo/bar")
+	if got, want := c.Path(), "/foo/bar"; got != want {
+		t.Fatalf("Path()=%v, want %v", got, want)
+	}
+}
+
+func TestReplicaClient_Type(t *testing.T) {
+	if got, want := file.NewReplicaClient("").Type(), "file"; got != want {
+		t.Fatalf("Type()=%v, want %v", got, want)
+	}
+}
+
+func TestReplicaClient_GenerationsDir(t *testing.T) {
+	t.Run("OK", func(t *testing.T) {
+		if got, err := file.NewReplicaClient("/foo").GenerationsDir(); err != nil {
+			t.Fatal(err)
+		} else if want := "/foo/generations"; got != want {
+			t.Fatalf("GenerationsDir()=%v, want %v", got, want)
+		}
+	})
+	t.Run("ErrNoPath", func(t *testing.T) {
+		if _, err := file.NewReplicaClient("").GenerationsDir(); err == nil || err.Error() != `file replica path required` {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	})
+}
+
+func TestReplicaClient_GenerationDir(t *testing.T) {
+	t.Run("OK", func(t *testing.T) {
+		if got, err := file.NewReplicaClient("/foo").GenerationDir("0123456701234567"); err != nil {
+			t.Fatal(err)
+		} else if want := "/foo/generations/0123456701234567"; got != want {
+			t.Fatalf("GenerationDir()=%v, want %v", got, want)
+		}
+	})
+	t.Run("ErrNoPath", func(t *testing.T) {
+		if _, err := file.NewReplicaClient("").GenerationDir("0123456701234567"); err == nil || err.Error() != `file replica path required` {
+			t.Fatalf("expected error: %v", err)
+		}
+	})
+	t.Run("ErrNoGeneration", func(t *testing.T) {
+		if _, err := file.NewReplicaClient("/foo").GenerationDir(""); err == nil || err.Error() != `generation required` {
+			t.Fatalf("expected error: %v", err)
+		}
+	})
+}
+
+func TestReplicaClient_SnapshotsDir(t *testing.T) {
+	t.Run("OK", func(t *testing.T) {
+		if got, err := file.NewReplicaClient("/foo").SnapshotsDir("0123456701234567"); err != nil {
+			t.Fatal(err)
+		} else if want := "/foo/generations/0123456701234567/snapshots"; got != want {
+			t.Fatalf("SnapshotsDir()=%v, want %v", got, want)
+		}
+	})
+	t.Run("ErrNoPath", func(t *testing.T) {
+		if _, err := file.NewReplicaClient("").SnapshotsDir("0123456701234567"); err == nil || err.Error() != `file replica path required` {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	})
+	t.Run("ErrNoGeneration", func(t *testing.T) {
+		if _, err := file.NewReplicaClient("/foo").SnapshotsDir(""); err == nil || err.Error() != `generation required` {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	})
+}
+
+func TestReplicaClient_SnapshotPath(t *testing.T) {
+	t.Run("OK", func(t *testing.T) {
+		if got, err := file.NewReplicaClient("/foo").SnapshotPath("0123456701234567", 1000); err != nil {
+			t.Fatal(err)
+		} else if want := "/foo/generations/0123456701234567/snapshots/000003e8.snapshot.lz4"; got != want {
+			t.Fatalf("SnapshotPath()=%v, want %v", got, want)
+		}
+	})
+	t.Run("ErrNoPath", func(t *testing.T) {
+		if _, err := file.NewReplicaClient("").SnapshotPath("0123456701234567", 1000); err == nil || err.Error() != `file replica path required` {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	})
+	t.Run("ErrNoGeneration", func(t *testing.T) {
+		if _, err := file.NewReplicaClient("/foo").SnapshotPath("", 1000); err == nil || err.Error() != `generation required` {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	})
+}
+
+func TestReplicaClient_WALDir(t *testing.T) {
+	t.Run("OK", func(t *testing.T) {
+		if got, err := file.NewReplicaClient("/foo").WALDir("0123456701234567"); err != nil {
+			t.Fatal(err)
+		} else if want := "/foo/generations/0123456701234567/wal"; got != want {
+			t.Fatalf("WALDir()=%v, want %v", got, want)
+		}
+	})
+	t.Run("ErrNoPath", func(t *testing.T) {
+		if _, err := file.NewReplicaClient("").WALDir("0123456701234567"); err == nil || err.Error() != `file replica path required` {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	})
+	t.Run("ErrNoGeneration", func(t *testing.T) {
+		if _, err := file.NewReplicaClient("/foo").WALDir(""); err == nil || err.Error() != `generation required` {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	})
+}
+
+func TestReplicaClient_WALSegmentPath(t *testing.T) {
+	t.Run("OK", func(t *testing.T) {
+		if got, err := file.NewReplicaClient("/foo").WALSegmentPath("0123456701234567", 1000, 1001); err != nil {
+			t.Fatal(err)
+		} else if want := "/foo/generations/0123456701234567/wal/000003e8_000003e9.wal.lz4"; got != want {
+			t.Fatalf("WALPath()=%v, want %v", got, want)
+		}
+	})
+	t.Run("ErrNoPath", func(t *testing.T) {
+		if _, err := file.NewReplicaClient("").WALSegmentPath("0123456701234567", 1000, 0); err == nil || err.Error() != `file replica path required` {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	})
+	t.Run("ErrNoGeneration", func(t *testing.T) {
+		if _, err := file.NewReplicaClient("/foo").WALSegmentPath("", 1000, 0); err == nil || err.Error() != `generation required` {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	})
+}
+
+/*
+func TestReplica_Sync(t *testing.T) {
+	// Ensure replica can successfully sync after DB has sync'd.
+	t.Run("InitialSync", func(t *testing.T) {
+		db, sqldb := MustOpenDBs(t)
+		defer MustCloseDBs(t, db, sqldb)
+
+		r := litestream.NewReplica(db, "", file.NewReplicaClient(t.TempDir()))
+		r.MonitorEnabled = false
+		db.Replicas = []*litestream.Replica{r}
+
+		// Sync database & then sync replica.
+		if err := db.Sync(context.Background()); err != nil {
+			t.Fatal(err)
+		} else if err := r.Sync(context.Background()); err != nil {
+			t.Fatal(err)
+		}
+
+		// Ensure posistions match.
+		if want, err := db.Pos(); err != nil {
+			t.Fatal(err)
+		} else if got, err := r.Pos(context.Background()); err != nil {
+			t.Fatal(err)
+		} else if got != want {
+			t.Fatalf("Pos()=%v, want %v", got, want)
+		}
+	})
+
+	// Ensure replica can successfully sync multiple times.
+	t.Run("MultiSync", func(t *testing.T) {
+		db, sqldb := MustOpenDBs(t)
+		defer MustCloseDBs(t, db, sqldb)
+
+		r := litestream.NewReplica(db, "", file.NewReplicaClient(t.TempDir()))
+		r.MonitorEnabled = false
+		db.Replicas = []*litestream.Replica{r}
+
+		if _, err := sqldb.Exec(`CREATE TABLE foo (bar TEXT);`); err != nil {
+			t.Fatal(err)
+		}
+
+		// Write to the database multiple times and sync after each write.
+		for i, n := 0, db.MinCheckpointPageN*2; i < n; i++ {
+			if _, err := sqldb.Exec(`INSERT INTO foo (bar) VALUES ('baz')`); err != nil {
+				t.Fatal(err)
+			}
+
+			// Sync periodically.
+			if i%100 == 0 || i == n-1 {
+				if err := db.Sync(context.Background()); err != nil {
+					t.Fatal(err)
+				} else if err := r.Sync(context.Background()); err != nil {
+					t.Fatal(err)
+				}
+			}
+		}
+
+		// Ensure posistions match.
+		pos, err := db.Pos()
+		if err != nil {
+			t.Fatal(err)
+		} else if got, want := pos.Index, 2; got != want {
+			t.Fatalf("Index=%v, want %v", got, want)
+		}
+
+		if want, err := r.Pos(context.Background()); err != nil {
+			t.Fatal(err)
+		} else if got := pos; got != want {
+			t.Fatalf("Pos()=%v, want %v", got, want)
+		}
+	})
+
+	// Ensure replica returns an error if there is no generation available from the DB.
+	t.Run("ErrNoGeneration", func(t *testing.T) {
+		db, sqldb := MustOpenDBs(t)
+		defer MustCloseDBs(t, db, sqldb)
+
+		r := litestream.NewReplica(db, "", file.NewReplicaClient(t.TempDir()))
+		r.MonitorEnabled = false
+		db.Replicas = []*litestream.Replica{r}
+
+		if err := r.Sync(context.Background()); err == nil || err.Error() != `no generation, waiting for data` {
+			t.Fatal(err)
+		}
+	})
+}
+*/

gcs/replica_client.go

@@ -0,0 +1,428 @@
+package gcs
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"strings"
+	"sync"
+	"time"
+
+	"cloud.google.com/go/storage"
+	"github.com/benbjohnson/litestream"
+	"github.com/benbjohnson/litestream/internal"
+	"google.golang.org/api/iterator"
+)
+
+// ReplicaClientType is the client type for this package.
+const ReplicaClientType = "gcs"
+
+var _ litestream.ReplicaClient = (*ReplicaClient)(nil)
+
+// ReplicaClient is a client for writing snapshots & WAL segments to disk.
+type ReplicaClient struct {
+	mu     sync.Mutex
+	client *storage.Client       // gcs client
+	bkt    *storage.BucketHandle // gcs bucket handle
+
+	// GCS bucket information
+	Bucket string
+	Path   string
+}
+
+// NewReplicaClient returns a new instance of ReplicaClient.
+func NewReplicaClient() *ReplicaClient {
+	return &ReplicaClient{}
+}
+
+// Type returns "gcs" as the client type.
+func (c *ReplicaClient) Type() string {
+	return ReplicaClientType
+}
+
+// Init initializes the connection to GCS. No-op if already initialized.
+func (c *ReplicaClient) Init(ctx context.Context) (err error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if c.client != nil {
+		return nil
+	}
+
+	if c.client, err = storage.NewClient(ctx); err != nil {
+		return err
+	}
+	c.bkt = c.client.Bucket(c.Bucket)
+
+	return nil
+}
+
+// Generations returns a list of available generation names.
+func (c *ReplicaClient) Generations(ctx context.Context) ([]string, error) {
+	if err := c.Init(ctx); err != nil {
+		return nil, err
+	}
+
+	// Construct query to only pull generation directory names.
+	query := &storage.Query{
+		Delimiter: "/",
+		Prefix:    litestream.GenerationsPath(c.Path) + "/",
+	}
+
+	// Loop over results and only build list of generation-formatted names.
+	it := c.bkt.Objects(ctx, query)
+	var generations []string
+	for {
+		attrs, err := it.Next()
+		if err == iterator.Done {
+			break
+		} else if err != nil {
+			return nil, err
+		}
+
+		name := path.Base(strings.TrimSuffix(attrs.Prefix, "/"))
+		if !litestream.IsGenerationName(name) {
+			continue
+		}
+		generations = append(generations, name)
+	}
+
+	return generations, nil
+}
+
+// DeleteGeneration deletes all snapshots & WAL segments within a generation.
+func (c *ReplicaClient) DeleteGeneration(ctx context.Context, generation string) error {
+	if err := c.Init(ctx); err != nil {
+		return err
+	}
+
+	dir, err := litestream.GenerationPath(c.Path, generation)
+	if err != nil {
+		return fmt.Errorf("cannot determine generation path: %w", err)
+	}
+
+	// Iterate over every object in generation and delete it.
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "LIST").Inc()
+	for it := c.bkt.Objects(ctx, &storage.Query{Prefix: dir + "/"}); ; {
+		attrs, err := it.Next()
+		if err == iterator.Done {
+			break
+		} else if err != nil {
+			return err
+		}
+
+		if err := c.bkt.Object(attrs.Name).Delete(ctx); isNotExists(err) {
+			continue
+		} else if err != nil {
+			return fmt.Errorf("cannot delete object %q: %w", attrs.Name, err)
+		}
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "DELETE").Inc()
+	}
+
+	// log.Printf("%s(%s): retainer: deleting generation: %s", r.db.Path(), r.Name(), generation)
+
+	return nil
+}
+
+// Snapshots returns an iterator over all available snapshots for a generation.
+func (c *ReplicaClient) Snapshots(ctx context.Context, generation string) (litestream.SnapshotIterator, error) {
+	if err := c.Init(ctx); err != nil {
+		return nil, err
+	}
+	dir, err := litestream.SnapshotsPath(c.Path, generation)
+	if err != nil {
+		return nil, fmt.Errorf("cannot determine snapshots path: %w", err)
+	}
+	return newSnapshotIterator(generation, c.bkt.Objects(ctx, &storage.Query{Prefix: dir + "/"})), nil
+}
+
+// WriteSnapshot writes LZ4 compressed data from rd to the object storage.
+func (c *ReplicaClient) WriteSnapshot(ctx context.Context, generation string, index int, rd io.Reader) (info litestream.SnapshotInfo, err error) {
+	if err := c.Init(ctx); err != nil {
+		return info, err
+	}
+
+	key, err := litestream.SnapshotPath(c.Path, generation, index)
+	if err != nil {
+		return info, fmt.Errorf("cannot determine snapshot path: %w", err)
+	}
+	startTime := time.Now()
+
+	w := c.bkt.Object(key).NewWriter(ctx)
+	defer w.Close()
+
+	n, err := io.Copy(w, rd)
+	if err != nil {
+		return info, err
+	} else if err := w.Close(); err != nil {
+		return info, err
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "PUT").Inc()
+	internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "PUT").Add(float64(n))
+
+	// log.Printf("%s(%s): snapshot: creating %s/%08x t=%s", r.db.Path(), r.Name(), generation, index, time.Since(startTime).Truncate(time.Millisecond))
+
+	return litestream.SnapshotInfo{
+		Generation: generation,
+		Index:      index,
+		Size:       n,
+		CreatedAt:  startTime.UTC(),
+	}, nil
+}
+
+// SnapshotReader returns a reader for snapshot data at the given generation/index.
+func (c *ReplicaClient) SnapshotReader(ctx context.Context, generation string, index int) (io.ReadCloser, error) {
+	if err := c.Init(ctx); err != nil {
+		return nil, err
+	}
+
+	key, err := litestream.SnapshotPath(c.Path, generation, index)
+	if err != nil {
+		return nil, fmt.Errorf("cannot determine snapshot path: %w", err)
+	}
+
+	r, err := c.bkt.Object(key).NewReader(ctx)
+	if isNotExists(err) {
+		return nil, os.ErrNotExist
+	} else if err != nil {
+		return nil, fmt.Errorf("cannot start new reader for %q: %w", key, err)
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "GET").Inc()
+	internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "GET").Add(float64(r.Attrs.Size))
+
+	return r, nil
+}
+
+// DeleteSnapshot deletes a snapshot with the given generation & index.
+func (c *ReplicaClient) DeleteSnapshot(ctx context.Context, generation string, index int) error {
+	if err := c.Init(ctx); err != nil {
+		return err
+	}
+
+	key, err := litestream.SnapshotPath(c.Path, generation, index)
+	if err != nil {
+		return fmt.Errorf("cannot determine snapshot path: %w", err)
+	}
+
+	if err := c.bkt.Object(key).Delete(ctx); err != nil && !isNotExists(err) {
+		return fmt.Errorf("cannot delete snapshot %q: %w", key, err)
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "DELETE").Inc()
+	return nil
+}
+
+// WALSegments returns an iterator over all available WAL files for a generation.
+func (c *ReplicaClient) WALSegments(ctx context.Context, generation string) (litestream.WALSegmentIterator, error) {
+	if err := c.Init(ctx); err != nil {
+		return nil, err
+	}
+	dir, err := litestream.WALPath(c.Path, generation)
+	if err != nil {
+		return nil, fmt.Errorf("cannot determine wal path: %w", err)
+	}
+	return newWALSegmentIterator(generation, c.bkt.Objects(ctx, &storage.Query{Prefix: dir + "/"})), nil
+}
+
+// WriteWALSegment writes LZ4 compressed data from rd into a file on disk.
+func (c *ReplicaClient) WriteWALSegment(ctx context.Context, pos litestream.Pos, rd io.Reader) (info litestream.WALSegmentInfo, err error) {
+	if err := c.Init(ctx); err != nil {
+		return info, err
+	}
+
+	key, err := litestream.WALSegmentPath(c.Path, pos.Generation, pos.Index, pos.Offset)
+	if err != nil {
+		return info, fmt.Errorf("cannot determine wal segment path: %w", err)
+	}
+	startTime := time.Now()
+
+	w := c.bkt.Object(key).NewWriter(ctx)
+	defer w.Close()
+
+	n, err := io.Copy(w, rd)
+	if err != nil {
+		return info, err
+	} else if err := w.Close(); err != nil {
+		return info, err
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "PUT").Inc()
+	internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "PUT").Add(float64(n))
+
+	return litestream.WALSegmentInfo{
+		Generation: pos.Generation,
+		Index:      pos.Index,
+		Offset:     pos.Offset,
+		Size:       n,
+		CreatedAt:  startTime.UTC(),
+	}, nil
+}
+
+// WALSegmentReader returns a reader for a section of WAL data at the given index.
+// Returns os.ErrNotExist if no matching index/offset is found.
+func (c *ReplicaClient) WALSegmentReader(ctx context.Context, pos litestream.Pos) (io.ReadCloser, error) {
+	if err := c.Init(ctx); err != nil {
+		return nil, err
+	}
+
+	key, err := litestream.WALSegmentPath(c.Path, pos.Generation, pos.Index, pos.Offset)
+	if err != nil {
+		return nil, fmt.Errorf("cannot determine wal segment path: %w", err)
+	}
+
+	r, err := c.bkt.Object(key).NewReader(ctx)
+	if isNotExists(err) {
+		return nil, os.ErrNotExist
+	} else if err != nil {
+		return nil, err
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "GET").Inc()
+	internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "GET").Add(float64(r.Attrs.Size))
+
+	return r, nil
+}
+
+// DeleteWALSegments deletes WAL segments with at the given positions.
+func (c *ReplicaClient) DeleteWALSegments(ctx context.Context, a []litestream.Pos) error {
+	if err := c.Init(ctx); err != nil {
+		return err
+	}
+
+	for _, pos := range a {
+		key, err := litestream.WALSegmentPath(c.Path, pos.Generation, pos.Index, pos.Offset)
+		if err != nil {
+			return fmt.Errorf("cannot determine wal segment path: %w", err)
+		}
+
+		if err := c.bkt.Object(key).Delete(ctx); err != nil && !isNotExists(err) {
+			return fmt.Errorf("cannot delete wal segment %q: %w", key, err)
+		}
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "DELETE").Inc()
+	}
+
+	return nil
+}
+
+type snapshotIterator struct {
+	generation string
+
+	it   *storage.ObjectIterator
+	info litestream.SnapshotInfo
+	err  error
+}
+
+func newSnapshotIterator(generation string, it *storage.ObjectIterator) *snapshotIterator {
+	return &snapshotIterator{
+		generation: generation,
+		it:         it,
+	}
+}
+
+func (itr *snapshotIterator) Close() (err error) {
+	return itr.err
+}
+
+func (itr *snapshotIterator) Next() bool {
+	// Exit if an error has already occurred.
+	if itr.err != nil {
+		return false
+	}
+
+	for {
+		// Fetch next object.
+		attrs, err := itr.it.Next()
+		if err == iterator.Done {
+			return false
+		} else if err != nil {
+			itr.err = err
+			return false
+		}
+
+		// Parse index, otherwise skip to the next object.
+		index, err := litestream.ParseSnapshotPath(path.Base(attrs.Name))
+		if err != nil {
+			continue
+		}
+
+		// Store current snapshot and return.
+		itr.info = litestream.SnapshotInfo{
+			Generation: itr.generation,
+			Index:      index,
+			Size:       attrs.Size,
+			CreatedAt:  attrs.Created.UTC(),
+		}
+		return true
+	}
+}
+
+func (itr *snapshotIterator) Err() error { return itr.err }
+
+func (itr *snapshotIterator) Snapshot() litestream.SnapshotInfo { return itr.info }
+
+type walSegmentIterator struct {
+	generation string
+
+	it   *storage.ObjectIterator
+	info litestream.WALSegmentInfo
+	err  error
+}
+
+func newWALSegmentIterator(generation string, it *storage.ObjectIterator) *walSegmentIterator {
+	return &walSegmentIterator{
+		generation: generation,
+		it:         it,
+	}
+}
+
+func (itr *walSegmentIterator) Close() (err error) {
+	return itr.err
+}
+
+func (itr *walSegmentIterator) Next() bool {
+	// Exit if an error has already occurred.
+	if itr.err != nil {
+		return false
+	}
+
+	for {
+		// Fetch next object.
+		attrs, err := itr.it.Next()
+		if err == iterator.Done {
+			return false
+		} else if err != nil {
+			itr.err = err
+			return false
+		}
+
+		// Parse index & offset, otherwise skip to the next object.
+		index, offset, err := litestream.ParseWALSegmentPath(path.Base(attrs.Name))
+		if err != nil {
+			continue
+		}
+
+		// Store current snapshot and return.
+		itr.info = litestream.WALSegmentInfo{
+			Generation: itr.generation,
+			Index:      index,
+			Offset:     offset,
+			Size:       attrs.Size,
+			CreatedAt:  attrs.Created.UTC(),
+		}
+		return true
+	}
+}
+
+func (itr *walSegmentIterator) Err() error { return itr.err }
+
+func (itr *walSegmentIterator) WALSegment() litestream.WALSegmentInfo {
+	return itr.info
+}
+
+func isNotExists(err error) bool {
+	return err == storage.ErrObjectNotExist
+}

go.mod

@@ -1,14 +1,51 @@
 module github.com/benbjohnson/litestream
 
-go 1.15
+go 1.19
 
 require (
-	github.com/aws/aws-sdk-go v1.27.0
-	github.com/davecgh/go-spew v1.1.1
-	github.com/mattn/go-sqlite3 v1.14.5
-	github.com/pierrec/lz4/v4 v4.1.3
-	github.com/prometheus/client_golang v1.9.0
-	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
-	golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e
+	cloud.google.com/go/storage v1.24.0
+	filippo.io/age v1.1.1
+	github.com/Azure/azure-storage-blob-go v0.15.0
+	github.com/aws/aws-sdk-go v1.44.71
+	github.com/mattn/go-shellwords v1.0.12
+	github.com/mattn/go-sqlite3 v1.14.14
+	github.com/pierrec/lz4/v4 v4.1.15
+	github.com/pkg/sftp v1.13.5
+	github.com/prometheus/client_golang v1.13.0
+	golang.org/x/crypto v0.4.0
+	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4
+	golang.org/x/sys v0.3.0
+	google.golang.org/api v0.91.0
 	gopkg.in/yaml.v2 v2.4.0
 )
+
+require (
+	cloud.google.com/go v0.103.0 // indirect
+	cloud.google.com/go/compute v1.7.0 // indirect
+	cloud.google.com/go/iam v0.3.0 // indirect
+	github.com/Azure/azure-pipeline-go v0.2.3 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/go-cmp v0.5.8 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.1.0 // indirect
+	github.com/googleapis/gax-go/v2 v2.5.1 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/kr/fs v0.1.0 // indirect
+	github.com/mattn/go-ieproxy v0.0.7 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
+	github.com/prometheus/client_model v0.2.0 // indirect
+	github.com/prometheus/common v0.37.0 // indirect
+	github.com/prometheus/procfs v0.8.0 // indirect
+	go.opencensus.io v0.23.0 // indirect
+	golang.org/x/net v0.3.0 // indirect
+	golang.org/x/oauth2 v0.0.0-20220808172628-8227340efae7 // indirect
+	golang.org/x/text v0.5.0 // indirect
+	golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/genproto v0.0.0-20220808204814-fd01256a5276 // indirect
+	google.golang.org/grpc v1.48.0 // indirect
+	google.golang.org/protobuf v1.28.1 // indirect
+)

internal/internal.go

@@ -2,6 +2,11 @@ package internal
 
 import (
 	"io"
+	"os"
+	"syscall"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 )
 
 // ReadCloser wraps a reader to also attach a separate closer.
@@ -30,3 +35,107 @@ func (r *ReadCloser) Close() error {
 	}
 	return r.c.Close()
 }
+
+// ReadCounter wraps an io.Reader and counts the total number of bytes read.
+type ReadCounter struct {
+	r io.Reader
+	n int64
+}
+
+// NewReadCounter returns a new instance of ReadCounter that wraps r.
+func NewReadCounter(r io.Reader) *ReadCounter {
+	return &ReadCounter{r: r}
+}
+
+// Read reads from the underlying reader into p and adds the bytes read to the counter.
+func (r *ReadCounter) Read(p []byte) (int, error) {
+	n, err := r.r.Read(p)
+	r.n += int64(n)
+	return n, err
+}
+
+// N returns the total number of bytes read.
+func (r *ReadCounter) N() int64 { return r.n }
+
+// CreateFile creates the file and matches the mode & uid/gid of fi.
+func CreateFile(filename string, fi os.FileInfo) (*os.File, error) {
+	mode := os.FileMode(0600)
+	if fi != nil {
+		mode = fi.Mode()
+	}
+
+	f, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode)
+	if err != nil {
+		return nil, err
+	}
+
+	uid, gid := Fileinfo(fi)
+	_ = f.Chown(uid, gid)
+	return f, nil
+}
+
+// MkdirAll is a copy of os.MkdirAll() except that it attempts to set the
+// mode/uid/gid to match fi for each created directory.
+func MkdirAll(path string, fi os.FileInfo) error {
+	uid, gid := Fileinfo(fi)
+
+	// Fast path: if we can tell whether path is a directory or file, stop with success or error.
+	dir, err := os.Stat(path)
+	if err == nil {
+		if dir.IsDir() {
+			return nil
+		}
+		return &os.PathError{Op: "mkdir", Path: path, Err: syscall.ENOTDIR}
+	}
+
+	// Slow path: make sure parent exists and then call Mkdir for path.
+	i := len(path)
+	for i > 0 && os.IsPathSeparator(path[i-1]) { // Skip trailing path separator.
+		i--
+	}
+
+	j := i
+	for j > 0 && !os.IsPathSeparator(path[j-1]) { // Scan backward over element.
+		j--
+	}
+
+	if j > 1 {
+		// Create parent.
+		err = MkdirAll(fixRootDirectory(path[:j-1]), fi)
+		if err != nil {
+			return err
+		}
+	}
+
+	// Parent now exists; invoke Mkdir and use its result.
+	mode := os.FileMode(0700)
+	if fi != nil {
+		mode = fi.Mode()
+	}
+	err = os.Mkdir(path, mode)
+	if err != nil {
+		// Handle arguments like "foo/." by
+		// double-checking that directory doesn't exist.
+		dir, err1 := os.Lstat(path)
+		if err1 == nil && dir.IsDir() {
+			_ = os.Chown(path, uid, gid)
+			return nil
+		}
+		return err
+	}
+	_ = os.Chown(path, uid, gid)
+	return nil
+}
+
+// Shared replica metrics.
+var (
+	OperationTotalCounterVec = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: "litestream_replica_operation_total",
+		Help: "The number of replica operations performed",
+	}, []string{"replica_type", "operation"})
+
+	OperationBytesCounterVec = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: "litestream_replica_operation_bytes",
+		Help: "The number of bytes used by replica operations",
+	}, []string{"replica_type", "operation"})
+)

internal/internal_unix.go

@@ -1,14 +1,17 @@
 // +build aix darwin dragonfly freebsd linux netbsd openbsd solaris
 
-package litestream
+package internal
 
 import (
 	"os"
 	"syscall"
 )
 
-// fileinfo returns syscall fields from a FileInfo object.
-func fileinfo(fi os.FileInfo) (uid, gid int) {
+// Fileinfo returns syscall fields from a FileInfo object.
+func Fileinfo(fi os.FileInfo) (uid, gid int) {
+	if fi == nil {
+		return -1, -1
+	}
 	stat := fi.Sys().(*syscall.Stat_t)
 	return int(stat.Uid), int(stat.Gid)
 }

internal/internal_windows.go

@@ -1,13 +1,13 @@
 // +build windows
 
-package litestream
+package internal
 
 import (
 	"os"
 )
 
-// fileinfo returns syscall fields from a FileInfo object.
-func fileinfo(fi os.FileInfo) (uid, gid int) {
+// Fileinfo returns syscall fields from a FileInfo object.
+func Fileinfo(fi os.FileInfo) (uid, gid int) {
 	return -1, -1
 }
 

internal/metrics.go

@@ -1,44 +0,0 @@
-package internal
-
-import (
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
-)
-
-// Shared replica metrics.
-var (
-	ReplicaSnapshotTotalGaugeVec = promauto.NewGaugeVec(prometheus.GaugeOpts{
-		Namespace: "litestream",
-		Subsystem: "replica",
-		Name:      "snapshot_total",
-		Help:      "The current number of snapshots",
-	}, []string{"db", "name"})
-
-	ReplicaWALBytesCounterVec = promauto.NewCounterVec(prometheus.CounterOpts{
-		Namespace: "litestream",
-		Subsystem: "replica",
-		Name:      "wal_bytes",
-		Help:      "The number wal bytes written",
-	}, []string{"db", "name"})
-
-	ReplicaWALIndexGaugeVec = promauto.NewGaugeVec(prometheus.GaugeOpts{
-		Namespace: "litestream",
-		Subsystem: "replica",
-		Name:      "wal_index",
-		Help:      "The current WAL index",
-	}, []string{"db", "name"})
-
-	ReplicaWALOffsetGaugeVec = promauto.NewGaugeVec(prometheus.GaugeOpts{
-		Namespace: "litestream",
-		Subsystem: "replica",
-		Name:      "wal_offset",
-		Help:      "The current WAL offset",
-	}, []string{"db", "name"})
-
-	ReplicaValidationTotalCounterVec = promauto.NewCounterVec(prometheus.CounterOpts{
-		Namespace: "litestream",
-		Subsystem: "replica",
-		Name:      "validation_total",
-		Help:      "The number of validations performed",
-	}, []string{"db", "name", "status"})
-)

litestream.go

@@ -7,21 +7,24 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path"
 	"path/filepath"
 	"regexp"
 	"strconv"
 	"strings"
-	"syscall"
 	"time"
+
+	"github.com/mattn/go-sqlite3"
 )
 
 // Naming constants.
 const (
 	MetaDirSuffix = "-litestream"
 
-	WALDirName  = "wal"
-	WALExt      = ".wal"
-	SnapshotExt = ".snapshot"
+	WALDirName    = "wal"
+	WALExt        = ".wal"
+	WALSegmentExt = ".wal.lz4"
+	SnapshotExt   = ".snapshot.lz4"
 
 	GenerationNameLen = 16
 )
@@ -41,19 +44,179 @@ var (
 	ErrChecksumMismatch = errors.New("invalid replica, checksum mismatch")
 )
 
+var (
+	// LogWriter is the destination writer for all logging.
+	LogWriter = os.Stdout
+
+	// LogFlags are the flags passed to log.New().
+	LogFlags = 0
+)
+
+func init() {
+	sql.Register("litestream-sqlite3", &sqlite3.SQLiteDriver{
+		ConnectHook: func(conn *sqlite3.SQLiteConn) error {
+			if err := conn.SetFileControlInt("main", sqlite3.SQLITE_FCNTL_PERSIST_WAL, 1); err != nil {
+				return fmt.Errorf("cannot set file control: %w", err)
+			}
+			return nil
+		},
+	})
+}
+
+// SnapshotIterator represents an iterator over a collection of snapshot metadata.
+type SnapshotIterator interface {
+	io.Closer
+
+	// Prepares the the next snapshot for reading with the Snapshot() method.
+	// Returns true if another snapshot is available. Returns false if no more
+	// snapshots are available or if an error occured.
+	Next() bool
+
+	// Returns an error that occurred during iteration.
+	Err() error
+
+	// Returns metadata for the currently positioned snapshot.
+	Snapshot() SnapshotInfo
+}
+
+// SliceSnapshotIterator returns all snapshots from an iterator as a slice.
+func SliceSnapshotIterator(itr SnapshotIterator) ([]SnapshotInfo, error) {
+	var a []SnapshotInfo
+	for itr.Next() {
+		a = append(a, itr.Snapshot())
+	}
+	return a, itr.Close()
+}
+
+var _ SnapshotIterator = (*SnapshotInfoSliceIterator)(nil)
+
+// SnapshotInfoSliceIterator represents an iterator for iterating over a slice of snapshots.
+type SnapshotInfoSliceIterator struct {
+	init bool
+	a    []SnapshotInfo
+}
+
+// NewSnapshotInfoSliceIterator returns a new instance of SnapshotInfoSliceIterator.
+func NewSnapshotInfoSliceIterator(a []SnapshotInfo) *SnapshotInfoSliceIterator {
+	return &SnapshotInfoSliceIterator{a: a}
+}
+
+// Close always returns nil.
+func (itr *SnapshotInfoSliceIterator) Close() error { return nil }
+
+// Next moves to the next snapshot. Returns true if another snapshot is available.
+func (itr *SnapshotInfoSliceIterator) Next() bool {
+	if !itr.init {
+		itr.init = true
+		return len(itr.a) > 0
+	}
+	itr.a = itr.a[1:]
+	return len(itr.a) > 0
+}
+
+// Err always returns nil.
+func (itr *SnapshotInfoSliceIterator) Err() error { return nil }
+
+// Snapshot returns the metadata from the currently positioned snapshot.
+func (itr *SnapshotInfoSliceIterator) Snapshot() SnapshotInfo {
+	if len(itr.a) == 0 {
+		return SnapshotInfo{}
+	}
+	return itr.a[0]
+}
+
+// WALSegmentIterator represents an iterator over a collection of WAL segments.
+type WALSegmentIterator interface {
+	io.Closer
+
+	// Prepares the the next WAL for reading with the WAL() method.
+	// Returns true if another WAL is available. Returns false if no more
+	// WAL files are available or if an error occured.
+	Next() bool
+
+	// Returns an error that occurred during iteration.
+	Err() error
+
+	// Returns metadata for the currently positioned WAL segment file.
+	WALSegment() WALSegmentInfo
+}
+
+// SliceWALSegmentIterator returns all WAL segment files from an iterator as a slice.
+func SliceWALSegmentIterator(itr WALSegmentIterator) ([]WALSegmentInfo, error) {
+	var a []WALSegmentInfo
+	for itr.Next() {
+		a = append(a, itr.WALSegment())
+	}
+	return a, itr.Close()
+}
+
+var _ WALSegmentIterator = (*WALSegmentInfoSliceIterator)(nil)
+
+// WALSegmentInfoSliceIterator represents an iterator for iterating over a slice of wal segments.
+type WALSegmentInfoSliceIterator struct {
+	init bool
+	a    []WALSegmentInfo
+}
+
+// NewWALSegmentInfoSliceIterator returns a new instance of WALSegmentInfoSliceIterator.
+func NewWALSegmentInfoSliceIterator(a []WALSegmentInfo) *WALSegmentInfoSliceIterator {
+	return &WALSegmentInfoSliceIterator{a: a}
+}
+
+// Close always returns nil.
+func (itr *WALSegmentInfoSliceIterator) Close() error { return nil }
+
+// Next moves to the next wal segment. Returns true if another segment is available.
+func (itr *WALSegmentInfoSliceIterator) Next() bool {
+	if !itr.init {
+		itr.init = true
+		return len(itr.a) > 0
+	}
+	itr.a = itr.a[1:]
+	return len(itr.a) > 0
+}
+
+// Err always returns nil.
+func (itr *WALSegmentInfoSliceIterator) Err() error { return nil }
+
+// WALSegment returns the metadata from the currently positioned wal segment.
+func (itr *WALSegmentInfoSliceIterator) WALSegment() WALSegmentInfo {
+	if len(itr.a) == 0 {
+		return WALSegmentInfo{}
+	}
+	return itr.a[0]
+}
+
 // SnapshotInfo represents file information about a snapshot.
 type SnapshotInfo struct {
-	Name       string
-	Replica    string
 	Generation string
 	Index      int
 	Size       int64
 	CreatedAt  time.Time
 }
 
+// Pos returns the WAL position when the snapshot was made.
+func (info *SnapshotInfo) Pos() Pos {
+	return Pos{Generation: info.Generation, Index: info.Index}
+}
+
+// SnapshotInfoSlice represents a slice of snapshot metadata.
+type SnapshotInfoSlice []SnapshotInfo
+
+func (a SnapshotInfoSlice) Len() int { return len(a) }
+
+func (a SnapshotInfoSlice) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+
+func (a SnapshotInfoSlice) Less(i, j int) bool {
+	if a[i].Generation != a[j].Generation {
+		return a[i].Generation < a[j].Generation
+	}
+	return a[i].Index < a[j].Index
+}
+
 // FilterSnapshotsAfter returns all snapshots that were created on or after t.
-func FilterSnapshotsAfter(a []*SnapshotInfo, t time.Time) []*SnapshotInfo {
-	other := make([]*SnapshotInfo, 0, len(a))
+func FilterSnapshotsAfter(a []SnapshotInfo, t time.Time) []SnapshotInfo {
+	other := make([]SnapshotInfo, 0, len(a))
 	for _, snapshot := range a {
 		if !snapshot.CreatedAt.Before(t) {
 			other = append(other, snapshot)
@@ -63,9 +226,11 @@ func FilterSnapshotsAfter(a []*SnapshotInfo, t time.Time) []*SnapshotInfo {
 }
 
 // FindMinSnapshotByGeneration finds the snapshot with the lowest index in a generation.
-func FindMinSnapshotByGeneration(a []*SnapshotInfo, generation string) *SnapshotInfo {
+func FindMinSnapshotByGeneration(a []SnapshotInfo, generation string) *SnapshotInfo {
 	var min *SnapshotInfo
-	for _, snapshot := range a {
+	for i := range a {
+		snapshot := &a[i]
+
 		if snapshot.Generation != generation {
 			continue
 		} else if min == nil || snapshot.Index < min.Index {
@@ -77,8 +242,27 @@ func FindMinSnapshotByGeneration(a []*SnapshotInfo, generation string) *Snapshot
 
 // WALInfo represents file information about a WAL file.
 type WALInfo struct {
-	Name       string
-	Replica    string
+	Generation string
+	Index      int
+	CreatedAt  time.Time
+}
+
+// WALInfoSlice represents a slice of WAL metadata.
+type WALInfoSlice []WALInfo
+
+func (a WALInfoSlice) Len() int { return len(a) }
+
+func (a WALInfoSlice) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+
+func (a WALInfoSlice) Less(i, j int) bool {
+	if a[i].Generation != a[j].Generation {
+		return a[i].Generation < a[j].Generation
+	}
+	return a[i].Index < a[j].Index
+}
+
+// WALSegmentInfo represents file information about a WAL segment file.
+type WALSegmentInfo struct {
 	Generation string
 	Index      int
 	Offset     int64
@@ -86,6 +270,27 @@ type WALInfo struct {
 	CreatedAt  time.Time
 }
 
+// Pos returns the WAL position when the segment was made.
+func (info *WALSegmentInfo) Pos() Pos {
+	return Pos{Generation: info.Generation, Index: info.Index, Offset: info.Offset}
+}
+
+// WALSegmentInfoSlice represents a slice of WAL segment metadata.
+type WALSegmentInfoSlice []WALSegmentInfo
+
+func (a WALSegmentInfoSlice) Len() int { return len(a) }
+
+func (a WALSegmentInfoSlice) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+
+func (a WALSegmentInfoSlice) Less(i, j int) bool {
+	if a[i].Generation != a[j].Generation {
+		return a[i].Generation < a[j].Generation
+	} else if a[i].Index != a[j].Index {
+		return a[i].Index < a[j].Index
+	}
+	return a[i].Offset < a[j].Offset
+}
+
 // Pos is a position in the WAL for a generation.
 type Pos struct {
 	Generation string // generation name
@@ -106,6 +311,11 @@ func (p Pos) IsZero() bool {
 	return p == (Pos{})
 }
 
+// Truncate returns p with the offset truncated to zero.
+func (p Pos) Truncate() Pos {
+	return Pos{Generation: p.Generation, Index: p.Index}
+}
+
 // Checksum computes a running SQLite checksum over a byte slice.
 func Checksum(bo binary.ByteOrder, s0, s1 uint32, b []byte) (uint32, uint32) {
 	assert(len(b)%8 == 0, "misaligned checksum byte slice")
@@ -197,6 +407,56 @@ func IsGenerationName(s string) bool {
 	return true
 }
 
+// GenerationsPath returns the path to a generation root directory.
+func GenerationsPath(root string) string {
+	return path.Join(root, "generations")
+}
+
+// GenerationPath returns the path to a generation's root directory.
+func GenerationPath(root, generation string) (string, error) {
+	dir := GenerationsPath(root)
+	if generation == "" {
+		return "", fmt.Errorf("generation required")
+	}
+	return path.Join(dir, generation), nil
+}
+
+// SnapshotsPath returns the path to a generation's snapshot directory.
+func SnapshotsPath(root, generation string) (string, error) {
+	dir, err := GenerationPath(root, generation)
+	if err != nil {
+		return "", err
+	}
+	return path.Join(dir, "snapshots"), nil
+}
+
+// SnapshotPath returns the path to an uncompressed snapshot file.
+func SnapshotPath(root, generation string, index int) (string, error) {
+	dir, err := SnapshotsPath(root, generation)
+	if err != nil {
+		return "", err
+	}
+	return path.Join(dir, FormatSnapshotPath(index)), nil
+}
+
+// WALPath returns the path to a generation's WAL directory
+func WALPath(root, generation string) (string, error) {
+	dir, err := GenerationPath(root, generation)
+	if err != nil {
+		return "", err
+	}
+	return path.Join(dir, "wal"), nil
+}
+
+// WALSegmentPath returns the path to a WAL segment file.
+func WALSegmentPath(root, generation string, index int, offset int64) (string, error) {
+	dir, err := WALPath(root, generation)
+	if err != nil {
+		return "", err
+	}
+	return path.Join(dir, FormatWALSegmentPath(index, offset)), nil
+}
+
 // IsSnapshotPath returns true if s is a path to a snapshot file.
 func IsSnapshotPath(s string) bool {
 	return snapshotPathRegex.MatchString(s)
@@ -204,38 +464,43 @@ func IsSnapshotPath(s string) bool {
 
 // ParseSnapshotPath returns the index for the snapshot.
 // Returns an error if the path is not a valid snapshot path.
-func ParseSnapshotPath(s string) (index int, ext string, err error) {
+func ParseSnapshotPath(s string) (index int, err error) {
 	s = filepath.Base(s)
 
 	a := snapshotPathRegex.FindStringSubmatch(s)
 	if a == nil {
-		return 0, "", fmt.Errorf("invalid snapshot path: %s", s)
+		return 0, fmt.Errorf("invalid snapshot path: %s", s)
 	}
 
 	i64, _ := strconv.ParseUint(a[1], 16, 64)
-	return int(i64), a[2], nil
+	return int(i64), nil
 }
 
-var snapshotPathRegex = regexp.MustCompile(`^([0-9a-f]{8})(.snapshot(?:.lz4)?)$`)
+// FormatSnapshotPath formats a snapshot filename with a given index.
+func FormatSnapshotPath(index int) string {
+	assert(index >= 0, "snapshot index must be non-negative")
+	return fmt.Sprintf("%08x%s", index, SnapshotExt)
+}
+
+var snapshotPathRegex = regexp.MustCompile(`^([0-9a-f]{8})\.snapshot\.lz4$`)
 
 // IsWALPath returns true if s is a path to a WAL file.
 func IsWALPath(s string) bool {
 	return walPathRegex.MatchString(s)
 }
 
-// ParseWALPath returns the index & offset for the WAL file.
-// Returns an error if the path is not a valid snapshot path.
-func ParseWALPath(s string) (index int, offset int64, ext string, err error) {
+// ParseWALPath returns the index for the WAL file.
+// Returns an error if the path is not a valid WAL path.
+func ParseWALPath(s string) (index int, err error) {
 	s = filepath.Base(s)
 
 	a := walPathRegex.FindStringSubmatch(s)
 	if a == nil {
-		return 0, 0, "", fmt.Errorf("invalid wal path: %s", s)
+		return 0, fmt.Errorf("invalid wal path: %s", s)
 	}
 
 	i64, _ := strconv.ParseUint(a[1], 16, 64)
-	off64, _ := strconv.ParseUint(a[2], 16, 64)
-	return int(i64), int64(off64), a[3], nil
+	return int(i64), nil
 }
 
 // FormatWALPath formats a WAL filename with a given index.
@@ -244,77 +509,37 @@ func FormatWALPath(index int) string {
 	return fmt.Sprintf("%08x%s", index, WALExt)
 }
 
-// FormatWALPathWithOffset formats a WAL filename with a given index & offset.
-func FormatWALPathWithOffset(index int, offset int64) string {
-	assert(index >= 0, "wal index must be non-negative")
-	assert(offset >= 0, "wal offset must be non-negative")
-	return fmt.Sprintf("%08x_%08x%s", index, offset, WALExt)
+var walPathRegex = regexp.MustCompile(`^([0-9a-f]{8})\.wal$`)
+
+// ParseWALSegmentPath returns the index & offset for the WAL segment file.
+// Returns an error if the path is not a valid wal segment path.
+func ParseWALSegmentPath(s string) (index int, offset int64, err error) {
+	s = filepath.Base(s)
+
+	a := walSegmentPathRegex.FindStringSubmatch(s)
+	if a == nil {
+		return 0, 0, fmt.Errorf("invalid wal segment path: %s", s)
+	}
+
+	i64, _ := strconv.ParseUint(a[1], 16, 64)
+	off64, _ := strconv.ParseUint(a[2], 16, 64)
+	return int(i64), int64(off64), nil
 }
 
-var walPathRegex = regexp.MustCompile(`^([0-9a-f]{8})(?:_([0-9a-f]{8}))?(.wal(?:.lz4)?)$`)
+// FormatWALSegmentPath formats a WAL segment filename with a given index & offset.
+func FormatWALSegmentPath(index int, offset int64) string {
+	assert(index >= 0, "wal index must be non-negative")
+	assert(offset >= 0, "wal offset must be non-negative")
+	return fmt.Sprintf("%08x_%08x%s", index, offset, WALSegmentExt)
+}
+
+var walSegmentPathRegex = regexp.MustCompile(`^([0-9a-f]{8})(?:_([0-9a-f]{8}))\.wal\.lz4$`)
 
 // isHexChar returns true if ch is a lowercase hex character.
 func isHexChar(ch rune) bool {
 	return (ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f')
 }
 
-// createFile creates the file and attempts to set the UID/GID.
-func createFile(filename string, perm os.FileMode, uid, gid int) (*os.File, error) {
-	f, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE|os.O_TRUNC, perm)
-	if err != nil {
-		return nil, err
-	}
-	_ = f.Chown(uid, gid)
-	return f, nil
-}
-
-// mkdirAll is a copy of os.MkdirAll() except that it attempts to set the
-// uid/gid for each created directory.
-func mkdirAll(path string, perm os.FileMode, uid, gid int) error {
-	// Fast path: if we can tell whether path is a directory or file, stop with success or error.
-	dir, err := os.Stat(path)
-	if err == nil {
-		if dir.IsDir() {
-			return nil
-		}
-		return &os.PathError{Op: "mkdir", Path: path, Err: syscall.ENOTDIR}
-	}
-
-	// Slow path: make sure parent exists and then call Mkdir for path.
-	i := len(path)
-	for i > 0 && os.IsPathSeparator(path[i-1]) { // Skip trailing path separator.
-		i--
-	}
-
-	j := i
-	for j > 0 && !os.IsPathSeparator(path[j-1]) { // Scan backward over element.
-		j--
-	}
-
-	if j > 1 {
-		// Create parent.
-		err = mkdirAll(fixRootDirectory(path[:j-1]), perm, uid, gid)
-		if err != nil {
-			return err
-		}
-	}
-
-	// Parent now exists; invoke Mkdir and use its result.
-	err = os.Mkdir(path, perm)
-	if err != nil {
-		// Handle arguments like "foo/." by
-		// double-checking that directory doesn't exist.
-		dir, err1 := os.Lstat(path)
-		if err1 == nil && dir.IsDir() {
-			_ = os.Chown(path, uid, gid)
-			return nil
-		}
-		return err
-	}
-	_ = os.Chown(path, uid, gid)
-	return nil
-}
-
 // Tracef is used for low-level tracing.
 var Tracef = func(format string, a ...interface{}) {}
 

litestream_test.go

@@ -40,6 +40,104 @@ func TestChecksum(t *testing.T) {
 	})
 }
 
+func TestGenerationsPath(t *testing.T) {
+	t.Run("OK", func(t *testing.T) {
+		if got, want := litestream.GenerationsPath("foo"), "foo/generations"; got != want {
+			t.Fatalf("GenerationsPath()=%v, want %v", got, want)
+		}
+	})
+	t.Run("NoPath", func(t *testing.T) {
+		if got, want := litestream.GenerationsPath(""), "generations"; got != want {
+			t.Fatalf("GenerationsPath()=%v, want %v", got, want)
+		}
+	})
+}
+
+func TestGenerationPath(t *testing.T) {
+	t.Run("OK", func(t *testing.T) {
+		if got, err := litestream.GenerationPath("foo", "0123456701234567"); err != nil {
+			t.Fatal(err)
+		} else if want := "foo/generations/0123456701234567"; got != want {
+			t.Fatalf("GenerationPath()=%v, want %v", got, want)
+		}
+	})
+	t.Run("ErrNoGeneration", func(t *testing.T) {
+		if _, err := litestream.GenerationPath("foo", ""); err == nil || err.Error() != `generation required` {
+			t.Fatalf("expected error: %v", err)
+		}
+	})
+}
+
+func TestSnapshotsPath(t *testing.T) {
+	t.Run("OK", func(t *testing.T) {
+		if got, err := litestream.SnapshotsPath("foo", "0123456701234567"); err != nil {
+			t.Fatal(err)
+		} else if want := "foo/generations/0123456701234567/snapshots"; got != want {
+			t.Fatalf("SnapshotsPath()=%v, want %v", got, want)
+		}
+	})
+	t.Run("ErrNoGeneration", func(t *testing.T) {
+		if _, err := litestream.SnapshotsPath("foo", ""); err == nil || err.Error() != `generation required` {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	})
+}
+
+func TestSnapshotPath(t *testing.T) {
+	t.Run("OK", func(t *testing.T) {
+		if got, err := litestream.SnapshotPath("foo", "0123456701234567", 1000); err != nil {
+			t.Fatal(err)
+		} else if want := "foo/generations/0123456701234567/snapshots/000003e8.snapshot.lz4"; got != want {
+			t.Fatalf("SnapshotPath()=%v, want %v", got, want)
+		}
+	})
+	t.Run("ErrNoGeneration", func(t *testing.T) {
+		if _, err := litestream.SnapshotPath("foo", "", 1000); err == nil || err.Error() != `generation required` {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	})
+}
+
+func TestWALPath(t *testing.T) {
+	t.Run("OK", func(t *testing.T) {
+		if got, err := litestream.WALPath("foo", "0123456701234567"); err != nil {
+			t.Fatal(err)
+		} else if want := "foo/generations/0123456701234567/wal"; got != want {
+			t.Fatalf("WALPath()=%v, want %v", got, want)
+		}
+	})
+	t.Run("ErrNoGeneration", func(t *testing.T) {
+		if _, err := litestream.WALPath("foo", ""); err == nil || err.Error() != `generation required` {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	})
+}
+
+func TestWALSegmentPath(t *testing.T) {
+	t.Run("OK", func(t *testing.T) {
+		if got, err := litestream.WALSegmentPath("foo", "0123456701234567", 1000, 1001); err != nil {
+			t.Fatal(err)
+		} else if want := "foo/generations/0123456701234567/wal/000003e8_000003e9.wal.lz4"; got != want {
+			t.Fatalf("WALPath()=%v, want %v", got, want)
+		}
+	})
+	t.Run("ErrNoGeneration", func(t *testing.T) {
+		if _, err := litestream.WALSegmentPath("foo", "", 1000, 0); err == nil || err.Error() != `generation required` {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	})
+}
+
+func TestFindMinSnapshotByGeneration(t *testing.T) {
+	infos := []litestream.SnapshotInfo{
+		{Generation: "29cf4bced74e92ab", Index: 0},
+		{Generation: "5dfeb4aa03232553", Index: 24},
+	}
+	if got, want := litestream.FindMinSnapshotByGeneration(infos, "29cf4bced74e92ab"), &infos[0]; got != want {
+		t.Fatalf("info=%#v, want %#v", got, want)
+	}
+}
+
 func MustDecodeHexString(s string) []byte {
 	b, err := hex.DecodeString(s)
 	if err != nil {

mock/replica_client.go

@@ -0,0 +1,65 @@
+package mock
+
+import (
+	"context"
+	"io"
+
+	"github.com/benbjohnson/litestream"
+)
+
+var _ litestream.ReplicaClient = (*ReplicaClient)(nil)
+
+type ReplicaClient struct {
+	GenerationsFunc       func(ctx context.Context) ([]string, error)
+	DeleteGenerationFunc  func(ctx context.Context, generation string) error
+	SnapshotsFunc         func(ctx context.Context, generation string) (litestream.SnapshotIterator, error)
+	WriteSnapshotFunc     func(ctx context.Context, generation string, index int, r io.Reader) (litestream.SnapshotInfo, error)
+	DeleteSnapshotFunc    func(ctx context.Context, generation string, index int) error
+	SnapshotReaderFunc    func(ctx context.Context, generation string, index int) (io.ReadCloser, error)
+	WALSegmentsFunc       func(ctx context.Context, generation string) (litestream.WALSegmentIterator, error)
+	WriteWALSegmentFunc   func(ctx context.Context, pos litestream.Pos, r io.Reader) (litestream.WALSegmentInfo, error)
+	DeleteWALSegmentsFunc func(ctx context.Context, a []litestream.Pos) error
+	WALSegmentReaderFunc  func(ctx context.Context, pos litestream.Pos) (io.ReadCloser, error)
+}
+
+func (c *ReplicaClient) Type() string { return "mock" }
+
+func (c *ReplicaClient) Generations(ctx context.Context) ([]string, error) {
+	return c.GenerationsFunc(ctx)
+}
+
+func (c *ReplicaClient) DeleteGeneration(ctx context.Context, generation string) error {
+	return c.DeleteGenerationFunc(ctx, generation)
+}
+
+func (c *ReplicaClient) Snapshots(ctx context.Context, generation string) (litestream.SnapshotIterator, error) {
+	return c.SnapshotsFunc(ctx, generation)
+}
+
+func (c *ReplicaClient) WriteSnapshot(ctx context.Context, generation string, index int, r io.Reader) (litestream.SnapshotInfo, error) {
+	return c.WriteSnapshotFunc(ctx, generation, index, r)
+}
+
+func (c *ReplicaClient) DeleteSnapshot(ctx context.Context, generation string, index int) error {
+	return c.DeleteSnapshotFunc(ctx, generation, index)
+}
+
+func (c *ReplicaClient) SnapshotReader(ctx context.Context, generation string, index int) (io.ReadCloser, error) {
+	return c.SnapshotReaderFunc(ctx, generation, index)
+}
+
+func (c *ReplicaClient) WALSegments(ctx context.Context, generation string) (litestream.WALSegmentIterator, error) {
+	return c.WALSegmentsFunc(ctx, generation)
+}
+
+func (c *ReplicaClient) WriteWALSegment(ctx context.Context, pos litestream.Pos, r io.Reader) (litestream.WALSegmentInfo, error) {
+	return c.WriteWALSegmentFunc(ctx, pos, r)
+}
+
+func (c *ReplicaClient) DeleteWALSegments(ctx context.Context, a []litestream.Pos) error {
+	return c.DeleteWALSegmentsFunc(ctx, a)
+}
+
+func (c *ReplicaClient) WALSegmentReader(ctx context.Context, pos litestream.Pos) (io.ReadCloser, error) {
+	return c.WALSegmentReaderFunc(ctx, pos)
+}

replica_client.go

@@ -0,0 +1,48 @@
+package litestream
+
+import (
+	"context"
+	"io"
+)
+
+// ReplicaClient represents client to connect to a Replica.
+type ReplicaClient interface {
+	// Returns the type of client.
+	Type() string
+
+	// Returns a list of available generations.
+	Generations(ctx context.Context) ([]string, error)
+
+	// Deletes all snapshots & WAL segments within a generation.
+	DeleteGeneration(ctx context.Context, generation string) error
+
+	// Returns an iterator of all snapshots within a generation on the replica.
+	Snapshots(ctx context.Context, generation string) (SnapshotIterator, error)
+
+	// Writes LZ4 compressed snapshot data to the replica at a given index
+	// within a generation. Returns metadata for the snapshot.
+	WriteSnapshot(ctx context.Context, generation string, index int, r io.Reader) (SnapshotInfo, error)
+
+	// Deletes a snapshot with the given generation & index.
+	DeleteSnapshot(ctx context.Context, generation string, index int) error
+
+	// Returns a reader that contains LZ4 compressed snapshot data for a
+	// given index within a generation. Returns an os.ErrNotFound error if
+	// the snapshot does not exist.
+	SnapshotReader(ctx context.Context, generation string, index int) (io.ReadCloser, error)
+
+	// Returns an iterator of all WAL segments within a generation on the replica.
+	WALSegments(ctx context.Context, generation string) (WALSegmentIterator, error)
+
+	// Writes an LZ4 compressed WAL segment at a given position.
+	// Returns metadata for the written segment.
+	WriteWALSegment(ctx context.Context, pos Pos, r io.Reader) (WALSegmentInfo, error)
+
+	// Deletes one or more WAL segments at the given positions.
+	DeleteWALSegments(ctx context.Context, a []Pos) error
+
+	// Returns a reader that contains an LZ4 compressed WAL segment at a given
+	// index/offset within a generation. Returns an os.ErrNotFound error if the
+	// WAL segment does not exist.
+	WALSegmentReader(ctx context.Context, pos Pos) (io.ReadCloser, error)
+}

replica_client_test.go

@@ -0,0 +1,573 @@
+package litestream_test
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"path"
+	"reflect"
+	"sort"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/benbjohnson/litestream"
+	"github.com/benbjohnson/litestream/abs"
+	"github.com/benbjohnson/litestream/file"
+	"github.com/benbjohnson/litestream/gcs"
+	"github.com/benbjohnson/litestream/s3"
+	"github.com/benbjohnson/litestream/sftp"
+)
+
+func init() {
+	rand.Seed(time.Now().UnixNano())
+}
+
+var (
+	// Enables integration tests.
+	integration = flag.String("integration", "file", "")
+)
+
+// S3 settings
+var (
+	// Replica client settings
+	s3AccessKeyID     = flag.String("s3-access-key-id", os.Getenv("LITESTREAM_S3_ACCESS_KEY_ID"), "")
+	s3SecretAccessKey = flag.String("s3-secret-access-key", os.Getenv("LITESTREAM_S3_SECRET_ACCESS_KEY"), "")
+	s3Region          = flag.String("s3-region", os.Getenv("LITESTREAM_S3_REGION"), "")
+	s3Bucket          = flag.String("s3-bucket", os.Getenv("LITESTREAM_S3_BUCKET"), "")
+	s3Path            = flag.String("s3-path", os.Getenv("LITESTREAM_S3_PATH"), "")
+	s3Endpoint        = flag.String("s3-endpoint", os.Getenv("LITESTREAM_S3_ENDPOINT"), "")
+	s3ForcePathStyle  = flag.Bool("s3-force-path-style", os.Getenv("LITESTREAM_S3_FORCE_PATH_STYLE") == "true", "")
+	s3SkipVerify      = flag.Bool("s3-skip-verify", os.Getenv("LITESTREAM_S3_SKIP_VERIFY") == "true", "")
+)
+
+// Google cloud storage settings
+var (
+	gcsBucket = flag.String("gcs-bucket", os.Getenv("LITESTREAM_GCS_BUCKET"), "")
+	gcsPath   = flag.String("gcs-path", os.Getenv("LITESTREAM_GCS_PATH"), "")
+)
+
+// Azure blob storage settings
+var (
+	absAccountName = flag.String("abs-account-name", os.Getenv("LITESTREAM_ABS_ACCOUNT_NAME"), "")
+	absAccountKey  = flag.String("abs-account-key", os.Getenv("LITESTREAM_ABS_ACCOUNT_KEY"), "")
+	absBucket      = flag.String("abs-bucket", os.Getenv("LITESTREAM_ABS_BUCKET"), "")
+	absPath        = flag.String("abs-path", os.Getenv("LITESTREAM_ABS_PATH"), "")
+)
+
+// SFTP settings
+var (
+	sftpHost     = flag.String("sftp-host", os.Getenv("LITESTREAM_SFTP_HOST"), "")
+	sftpUser     = flag.String("sftp-user", os.Getenv("LITESTREAM_SFTP_USER"), "")
+	sftpPassword = flag.String("sftp-password", os.Getenv("LITESTREAM_SFTP_PASSWORD"), "")
+	sftpKeyPath  = flag.String("sftp-key-path", os.Getenv("LITESTREAM_SFTP_KEY_PATH"), "")
+	sftpPath     = flag.String("sftp-path", os.Getenv("LITESTREAM_SFTP_PATH"), "")
+)
+
+func TestReplicaClient_Generations(t *testing.T) {
+	RunWithReplicaClient(t, "OK", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+
+		// Write snapshots.
+		if _, err := c.WriteSnapshot(context.Background(), "5efbd8d042012dca", 0, strings.NewReader(`foo`)); err != nil {
+			t.Fatal(err)
+		} else if _, err := c.WriteSnapshot(context.Background(), "b16ddcf5c697540f", 0, strings.NewReader(`bar`)); err != nil {
+			t.Fatal(err)
+		} else if _, err := c.WriteSnapshot(context.Background(), "155fe292f8333c72", 0, strings.NewReader(`baz`)); err != nil {
+			t.Fatal(err)
+		}
+
+		// Verify returned generations.
+		if got, err := c.Generations(context.Background()); err != nil {
+			t.Fatal(err)
+		} else if want := []string{"155fe292f8333c72", "5efbd8d042012dca", "b16ddcf5c697540f"}; !reflect.DeepEqual(got, want) {
+			t.Fatalf("Generations()=%v, want %v", got, want)
+		}
+	})
+
+	RunWithReplicaClient(t, "NoGenerationsDir", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+
+		if generations, err := c.Generations(context.Background()); err != nil {
+			t.Fatal(err)
+		} else if got, want := len(generations), 0; got != want {
+			t.Fatalf("len(Generations())=%v, want %v", got, want)
+		}
+	})
+}
+
+func TestReplicaClient_Snapshots(t *testing.T) {
+	RunWithReplicaClient(t, "OK", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+
+		// Write snapshots.
+		if _, err := c.WriteSnapshot(context.Background(), "5efbd8d042012dca", 1, strings.NewReader(``)); err != nil {
+			t.Fatal(err)
+		} else if _, err := c.WriteSnapshot(context.Background(), "b16ddcf5c697540f", 5, strings.NewReader(`x`)); err != nil {
+			t.Fatal(err)
+		} else if _, err := c.WriteSnapshot(context.Background(), "b16ddcf5c697540f", 10, strings.NewReader(`xyz`)); err != nil {
+			t.Fatal(err)
+		}
+
+		// Fetch all snapshots by generation.
+		itr, err := c.Snapshots(context.Background(), "b16ddcf5c697540f")
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer itr.Close()
+
+		// Read all snapshots into a slice so they can be sorted.
+		a, err := litestream.SliceSnapshotIterator(itr)
+		if err != nil {
+			t.Fatal(err)
+		} else if got, want := len(a), 2; got != want {
+			t.Fatalf("len=%v, want %v", got, want)
+		}
+		sort.Sort(litestream.SnapshotInfoSlice(a))
+
+		// Verify first snapshot metadata.
+		if got, want := a[0].Generation, "b16ddcf5c697540f"; got != want {
+			t.Fatalf("Generation=%v, want %v", got, want)
+		} else if got, want := a[0].Index, 5; got != want {
+			t.Fatalf("Index=%v, want %v", got, want)
+		} else if got, want := a[0].Size, int64(1); got != want {
+			t.Fatalf("Size=%v, want %v", got, want)
+		} else if a[0].CreatedAt.IsZero() {
+			t.Fatalf("expected CreatedAt")
+		}
+
+		// Verify second snapshot metadata.
+		if got, want := a[1].Generation, "b16ddcf5c697540f"; got != want {
+			t.Fatalf("Generation=%v, want %v", got, want)
+		} else if got, want := a[1].Index, 0xA; got != want {
+			t.Fatalf("Index=%v, want %v", got, want)
+		} else if got, want := a[1].Size, int64(3); got != want {
+			t.Fatalf("Size=%v, want %v", got, want)
+		} else if a[1].CreatedAt.IsZero() {
+			t.Fatalf("expected CreatedAt")
+		}
+
+		// Ensure close is clean.
+		if err := itr.Close(); err != nil {
+			t.Fatal(err)
+		}
+	})
+
+	RunWithReplicaClient(t, "NoGenerationDir", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+
+		itr, err := c.Snapshots(context.Background(), "5efbd8d042012dca")
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer itr.Close()
+
+		if itr.Next() {
+			t.Fatal("expected no snapshots")
+		}
+	})
+
+	RunWithReplicaClient(t, "ErrNoGeneration", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+
+		itr, err := c.Snapshots(context.Background(), "")
+		if err == nil {
+			err = itr.Close()
+		}
+		if err == nil || err.Error() != `cannot determine snapshots path: generation required` {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	})
+}
+
+func TestReplicaClient_WriteSnapshot(t *testing.T) {
+	RunWithReplicaClient(t, "OK", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+
+		if _, err := c.WriteSnapshot(context.Background(), "b16ddcf5c697540f", 1000, strings.NewReader(`foobar`)); err != nil {
+			t.Fatal(err)
+		}
+
+		if r, err := c.SnapshotReader(context.Background(), "b16ddcf5c697540f", 1000); err != nil {
+			t.Fatal(err)
+		} else if buf, err := ioutil.ReadAll(r); err != nil {
+			t.Fatal(err)
+		} else if err := r.Close(); err != nil {
+			t.Fatal(err)
+		} else if got, want := string(buf), `foobar`; got != want {
+			t.Fatalf("data=%q, want %q", got, want)
+		}
+	})
+
+	RunWithReplicaClient(t, "ErrNoGeneration", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+		if _, err := c.WriteSnapshot(context.Background(), "", 0, nil); err == nil || err.Error() != `cannot determine snapshot path: generation required` {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	})
+}
+
+func TestReplicaClient_SnapshotReader(t *testing.T) {
+	RunWithReplicaClient(t, "OK", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+
+		if _, err := c.WriteSnapshot(context.Background(), "5efbd8d042012dca", 10, strings.NewReader(`foo`)); err != nil {
+			t.Fatal(err)
+		}
+
+		r, err := c.SnapshotReader(context.Background(), "5efbd8d042012dca", 10)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer r.Close()
+
+		if buf, err := ioutil.ReadAll(r); err != nil {
+			t.Fatal(err)
+		} else if got, want := string(buf), "foo"; got != want {
+			t.Fatalf("ReadAll=%v, want %v", got, want)
+		}
+	})
+
+	RunWithReplicaClient(t, "ErrNotFound", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+
+		if _, err := c.SnapshotReader(context.Background(), "5efbd8d042012dca", 1); !os.IsNotExist(err) {
+			t.Fatalf("expected not exist, got %#v", err)
+		}
+	})
+
+	RunWithReplicaClient(t, "ErrNoGeneration", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+
+		if _, err := c.SnapshotReader(context.Background(), "", 1); err == nil || err.Error() != `cannot determine snapshot path: generation required` {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	})
+}
+
+func TestReplicaClient_WALs(t *testing.T) {
+	RunWithReplicaClient(t, "OK", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+
+		if _, err := c.WriteWALSegment(context.Background(), litestream.Pos{Generation: "5efbd8d042012dca", Index: 1, Offset: 0}, strings.NewReader(``)); err != nil {
+			t.Fatal(err)
+		}
+		if _, err := c.WriteWALSegment(context.Background(), litestream.Pos{Generation: "b16ddcf5c697540f", Index: 2, Offset: 0}, strings.NewReader(`12345`)); err != nil {
+			t.Fatal(err)
+		} else if _, err := c.WriteWALSegment(context.Background(), litestream.Pos{Generation: "b16ddcf5c697540f", Index: 2, Offset: 5}, strings.NewReader(`67`)); err != nil {
+			t.Fatal(err)
+		} else if _, err := c.WriteWALSegment(context.Background(), litestream.Pos{Generation: "b16ddcf5c697540f", Index: 3, Offset: 0}, strings.NewReader(`xyz`)); err != nil {
+			t.Fatal(err)
+		}
+
+		itr, err := c.WALSegments(context.Background(), "b16ddcf5c697540f")
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer itr.Close()
+
+		// Read all WAL segment files into a slice so they can be sorted.
+		a, err := litestream.SliceWALSegmentIterator(itr)
+		if err != nil {
+			t.Fatal(err)
+		} else if got, want := len(a), 3; got != want {
+			t.Fatalf("len=%v, want %v", got, want)
+		}
+		sort.Sort(litestream.WALSegmentInfoSlice(a))
+
+		// Verify first WAL segment metadata.
+		if got, want := a[0].Generation, "b16ddcf5c697540f"; got != want {
+			t.Fatalf("Generation=%v, want %v", got, want)
+		} else if got, want := a[0].Index, 2; got != want {
+			t.Fatalf("Index=%v, want %v", got, want)
+		} else if got, want := a[0].Offset, int64(0); got != want {
+			t.Fatalf("Offset=%v, want %v", got, want)
+		} else if got, want := a[0].Size, int64(5); got != want {
+			t.Fatalf("Size=%v, want %v", got, want)
+		} else if a[0].CreatedAt.IsZero() {
+			t.Fatalf("expected CreatedAt")
+		}
+
+		// Verify first WAL segment metadata.
+		if got, want := a[1].Generation, "b16ddcf5c697540f"; got != want {
+			t.Fatalf("Generation=%v, want %v", got, want)
+		} else if got, want := a[1].Index, 2; got != want {
+			t.Fatalf("Index=%v, want %v", got, want)
+		} else if got, want := a[1].Offset, int64(5); got != want {
+			t.Fatalf("Offset=%v, want %v", got, want)
+		} else if got, want := a[1].Size, int64(2); got != want {
+			t.Fatalf("Size=%v, want %v", got, want)
+		} else if a[1].CreatedAt.IsZero() {
+			t.Fatalf("expected CreatedAt")
+		}
+
+		// Verify third WAL segment metadata.
+		if got, want := a[2].Generation, "b16ddcf5c697540f"; got != want {
+			t.Fatalf("Generation=%v, want %v", got, want)
+		} else if got, want := a[2].Index, 3; got != want {
+			t.Fatalf("Index=%v, want %v", got, want)
+		} else if got, want := a[2].Offset, int64(0); got != want {
+			t.Fatalf("Offset=%v, want %v", got, want)
+		} else if got, want := a[2].Size, int64(3); got != want {
+			t.Fatalf("Size=%v, want %v", got, want)
+		} else if a[1].CreatedAt.IsZero() {
+			t.Fatalf("expected CreatedAt")
+		}
+
+		// Ensure close is clean.
+		if err := itr.Close(); err != nil {
+			t.Fatal(err)
+		}
+	})
+
+	RunWithReplicaClient(t, "NoGenerationDir", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+
+		itr, err := c.WALSegments(context.Background(), "5efbd8d042012dca")
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer itr.Close()
+
+		if itr.Next() {
+			t.Fatal("expected no wal files")
+		}
+	})
+
+	RunWithReplicaClient(t, "NoWALs", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+
+		if _, err := c.WriteSnapshot(context.Background(), "5efbd8d042012dca", 0, strings.NewReader(`foo`)); err != nil {
+			t.Fatal(err)
+		}
+
+		itr, err := c.WALSegments(context.Background(), "5efbd8d042012dca")
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer itr.Close()
+
+		if itr.Next() {
+			t.Fatal("expected no wal files")
+		}
+	})
+
+	RunWithReplicaClient(t, "ErrNoGeneration", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+
+		itr, err := c.WALSegments(context.Background(), "")
+		if err == nil {
+			err = itr.Close()
+		}
+		if err == nil || err.Error() != `cannot determine wal path: generation required` {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	})
+}
+
+func TestReplicaClient_WriteWALSegment(t *testing.T) {
+	RunWithReplicaClient(t, "OK", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+
+		if _, err := c.WriteWALSegment(context.Background(), litestream.Pos{Generation: "b16ddcf5c697540f", Index: 1000, Offset: 2000}, strings.NewReader(`foobar`)); err != nil {
+			t.Fatal(err)
+		}
+
+		if r, err := c.WALSegmentReader(context.Background(), litestream.Pos{Generation: "b16ddcf5c697540f", Index: 1000, Offset: 2000}); err != nil {
+			t.Fatal(err)
+		} else if buf, err := ioutil.ReadAll(r); err != nil {
+			t.Fatal(err)
+		} else if err := r.Close(); err != nil {
+			t.Fatal(err)
+		} else if got, want := string(buf), `foobar`; got != want {
+			t.Fatalf("data=%q, want %q", got, want)
+		}
+	})
+
+	RunWithReplicaClient(t, "ErrNoGeneration", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+		if _, err := c.WriteWALSegment(context.Background(), litestream.Pos{Generation: "", Index: 0, Offset: 0}, nil); err == nil || err.Error() != `cannot determine wal segment path: generation required` {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	})
+}
+
+func TestReplicaClient_WALReader(t *testing.T) {
+
+	RunWithReplicaClient(t, "OK", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+		if _, err := c.WriteWALSegment(context.Background(), litestream.Pos{Generation: "5efbd8d042012dca", Index: 10, Offset: 5}, strings.NewReader(`foobar`)); err != nil {
+			t.Fatal(err)
+		}
+
+		r, err := c.WALSegmentReader(context.Background(), litestream.Pos{Generation: "5efbd8d042012dca", Index: 10, Offset: 5})
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer r.Close()
+
+		if buf, err := ioutil.ReadAll(r); err != nil {
+			t.Fatal(err)
+		} else if got, want := string(buf), "foobar"; got != want {
+			t.Fatalf("ReadAll=%v, want %v", got, want)
+		}
+	})
+
+	RunWithReplicaClient(t, "ErrNotFound", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+
+		if _, err := c.WALSegmentReader(context.Background(), litestream.Pos{Generation: "5efbd8d042012dca", Index: 1, Offset: 0}); !os.IsNotExist(err) {
+			t.Fatalf("expected not exist, got %#v", err)
+		}
+	})
+}
+
+func TestReplicaClient_DeleteWALSegments(t *testing.T) {
+	RunWithReplicaClient(t, "OK", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+
+		if _, err := c.WriteWALSegment(context.Background(), litestream.Pos{Generation: "b16ddcf5c697540f", Index: 1, Offset: 2}, strings.NewReader(`foo`)); err != nil {
+			t.Fatal(err)
+		} else if _, err := c.WriteWALSegment(context.Background(), litestream.Pos{Generation: "5efbd8d042012dca", Index: 3, Offset: 4}, strings.NewReader(`bar`)); err != nil {
+			t.Fatal(err)
+		}
+
+		if err := c.DeleteWALSegments(context.Background(), []litestream.Pos{
+			{Generation: "b16ddcf5c697540f", Index: 1, Offset: 2},
+			{Generation: "5efbd8d042012dca", Index: 3, Offset: 4},
+		}); err != nil {
+			t.Fatal(err)
+		}
+
+		if _, err := c.WALSegmentReader(context.Background(), litestream.Pos{Generation: "b16ddcf5c697540f", Index: 1, Offset: 2}); !os.IsNotExist(err) {
+			t.Fatalf("expected not exist, got %#v", err)
+		} else if _, err := c.WALSegmentReader(context.Background(), litestream.Pos{Generation: "5efbd8d042012dca", Index: 3, Offset: 4}); !os.IsNotExist(err) {
+			t.Fatalf("expected not exist, got %#v", err)
+		}
+	})
+
+	RunWithReplicaClient(t, "ErrNoGeneration", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+		if err := c.DeleteWALSegments(context.Background(), []litestream.Pos{{}}); err == nil || err.Error() != `cannot determine wal segment path: generation required` {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	})
+}
+
+// RunWithReplicaClient executes fn with each replica specified by the -integration flag
+func RunWithReplicaClient(t *testing.T, name string, fn func(*testing.T, litestream.ReplicaClient)) {
+	t.Run(name, func(t *testing.T) {
+		for _, typ := range strings.Split(*integration, ",") {
+			t.Run(typ, func(t *testing.T) {
+				c := NewReplicaClient(t, typ)
+				defer MustDeleteAll(t, c)
+
+				fn(t, c)
+			})
+		}
+	})
+}
+
+// NewReplicaClient returns a new client for integration testing by type name.
+func NewReplicaClient(tb testing.TB, typ string) litestream.ReplicaClient {
+	tb.Helper()
+
+	switch typ {
+	case file.ReplicaClientType:
+		return NewFileReplicaClient(tb)
+	case s3.ReplicaClientType:
+		return NewS3ReplicaClient(tb)
+	case gcs.ReplicaClientType:
+		return NewGCSReplicaClient(tb)
+	case abs.ReplicaClientType:
+		return NewABSReplicaClient(tb)
+	case sftp.ReplicaClientType:
+		return NewSFTPReplicaClient(tb)
+	default:
+		tb.Fatalf("invalid replica client type: %q", typ)
+		return nil
+	}
+}
+
+// NewFileReplicaClient returns a new client for integration testing.
+func NewFileReplicaClient(tb testing.TB) *file.ReplicaClient {
+	tb.Helper()
+	return file.NewReplicaClient(tb.TempDir())
+}
+
+// NewS3ReplicaClient returns a new client for integration testing.
+func NewS3ReplicaClient(tb testing.TB) *s3.ReplicaClient {
+	tb.Helper()
+
+	c := s3.NewReplicaClient()
+	c.AccessKeyID = *s3AccessKeyID
+	c.SecretAccessKey = *s3SecretAccessKey
+	c.Region = *s3Region
+	c.Bucket = *s3Bucket
+	c.Path = path.Join(*s3Path, fmt.Sprintf("%016x", rand.Uint64()))
+	c.Endpoint = *s3Endpoint
+	c.ForcePathStyle = *s3ForcePathStyle
+	c.SkipVerify = *s3SkipVerify
+	return c
+}
+
+// NewGCSReplicaClient returns a new client for integration testing.
+func NewGCSReplicaClient(tb testing.TB) *gcs.ReplicaClient {
+	tb.Helper()
+
+	c := gcs.NewReplicaClient()
+	c.Bucket = *gcsBucket
+	c.Path = path.Join(*gcsPath, fmt.Sprintf("%016x", rand.Uint64()))
+	return c
+}
+
+// NewABSReplicaClient returns a new client for integration testing.
+func NewABSReplicaClient(tb testing.TB) *abs.ReplicaClient {
+	tb.Helper()
+
+	c := abs.NewReplicaClient()
+	c.AccountName = *absAccountName
+	c.AccountKey = *absAccountKey
+	c.Bucket = *absBucket
+	c.Path = path.Join(*absPath, fmt.Sprintf("%016x", rand.Uint64()))
+	return c
+}
+
+// NewSFTPReplicaClient returns a new client for integration testing.
+func NewSFTPReplicaClient(tb testing.TB) *sftp.ReplicaClient {
+	tb.Helper()
+
+	c := sftp.NewReplicaClient()
+	c.Host = *sftpHost
+	c.User = *sftpUser
+	c.Password = *sftpPassword
+	c.KeyPath = *sftpKeyPath
+	c.Path = path.Join(*sftpPath, fmt.Sprintf("%016x", rand.Uint64()))
+	return c
+}
+
+// MustDeleteAll deletes all objects under the client's path.
+func MustDeleteAll(tb testing.TB, c litestream.ReplicaClient) {
+	tb.Helper()
+
+	generations, err := c.Generations(context.Background())
+	if err != nil {
+		tb.Fatalf("cannot list generations for deletion: %s", err)
+	}
+
+	for _, generation := range generations {
+		if err := c.DeleteGeneration(context.Background(), generation); err != nil {
+			tb.Fatalf("cannot delete generation: %s", err)
+		}
+	}
+
+	switch c := c.(type) {
+	case *sftp.ReplicaClient:
+		if err := c.Cleanup(context.Background()); err != nil {
+			tb.Fatalf("cannot cleanup sftp: %s", err)
+		}
+	}
+}

replica_test.go

@@ -1,90 +1,144 @@
 package litestream_test
 
 import (
+	"bytes"
 	"context"
+	"io"
+	"os"
 	"testing"
 
 	"github.com/benbjohnson/litestream"
+	"github.com/benbjohnson/litestream/file"
+	"github.com/benbjohnson/litestream/mock"
+	"github.com/pierrec/lz4/v4"
 )
 
-func TestFileReplica_Sync(t *testing.T) {
-	// Ensure replica can successfully sync after DB has sync'd.
-	t.Run("InitialSync", func(t *testing.T) {
-		db, sqldb := MustOpenDBs(t)
-		defer MustCloseDBs(t, db, sqldb)
-		r := NewTestFileReplica(t, db)
-
-		// Sync database & then sync replica.
-		if err := db.Sync(context.Background()); err != nil {
-			t.Fatal(err)
-		} else if err := r.Sync(context.Background()); err != nil {
-			t.Fatal(err)
-		}
-
-		// Ensure posistions match.
-		if pos, err := db.Pos(); err != nil {
-			t.Fatal(err)
-		} else if got, want := r.LastPos(), pos; got != want {
-			t.Fatalf("LastPos()=%v, want %v", got, want)
+func TestReplica_Name(t *testing.T) {
+	t.Run("WithName", func(t *testing.T) {
+		if got, want := litestream.NewReplica(nil, "NAME").Name(), "NAME"; got != want {
+			t.Fatalf("Name()=%v, want %v", got, want)
 		}
 	})
-
-	// Ensure replica can successfully sync multiple times.
-	t.Run("MultiSync", func(t *testing.T) {
-		db, sqldb := MustOpenDBs(t)
-		defer MustCloseDBs(t, db, sqldb)
-		r := NewTestFileReplica(t, db)
-
-		if _, err := sqldb.Exec(`CREATE TABLE foo (bar TEXT);`); err != nil {
-			t.Fatal(err)
-		}
-
-		// Write to the database multiple times and sync after each write.
-		for i, n := 0, db.MinCheckpointPageN*2; i < n; i++ {
-			if _, err := sqldb.Exec(`INSERT INTO foo (bar) VALUES ('baz')`); err != nil {
-				t.Fatal(err)
-			}
-
-			// Sync periodically.
-			if i%100 == 0 || i == n-1 {
-				if err := db.Sync(context.Background()); err != nil {
-					t.Fatal(err)
-				} else if err := r.Sync(context.Background()); err != nil {
-					t.Fatal(err)
-				}
-			}
-		}
-
-		// Ensure posistions match.
-		if pos, err := db.Pos(); err != nil {
-			t.Fatal(err)
-		} else if got, want := pos.Index, 2; got != want {
-			t.Fatalf("Index=%v, want %v", got, want)
-		} else if calcPos, err := r.CalcPos(context.Background(), pos.Generation); err != nil {
-			t.Fatal(err)
-		} else if got, want := calcPos, pos; got != want {
-			t.Fatalf("CalcPos()=%v, want %v", got, want)
-		} else if got, want := r.LastPos(), pos; got != want {
-			t.Fatalf("LastPos()=%v, want %v", got, want)
-		}
-	})
-
-	// Ensure replica returns an error if there is no generation available from the DB.
-	t.Run("ErrNoGeneration", func(t *testing.T) {
-		db, sqldb := MustOpenDBs(t)
-		defer MustCloseDBs(t, db, sqldb)
-		r := NewTestFileReplica(t, db)
-
-		if err := r.Sync(context.Background()); err == nil || err.Error() != `no generation, waiting for data` {
-			t.Fatal(err)
+	t.Run("WithoutName", func(t *testing.T) {
+		r := litestream.NewReplica(nil, "")
+		r.Client = &mock.ReplicaClient{}
+		if got, want := r.Name(), "mock"; got != want {
+			t.Fatalf("Name()=%v, want %v", got, want)
 		}
 	})
 }
 
-// NewTestFileReplica returns a new replica using a temp directory & with monitoring disabled.
-func NewTestFileReplica(tb testing.TB, db *litestream.DB) *litestream.FileReplica {
-	r := litestream.NewFileReplica(db, "", tb.TempDir())
-	r.MonitorEnabled = false
-	db.Replicas = []litestream.Replica{r}
-	return r
+func TestReplica_Sync(t *testing.T) {
+	db, sqldb := MustOpenDBs(t)
+	defer MustCloseDBs(t, db, sqldb)
+
+	// Execute a query to force a write to the WAL.
+	if _, err := sqldb.Exec(`CREATE TABLE foo (bar TEXT);`); err != nil {
+		t.Fatal(err)
+	}
+
+	// Issue initial database sync to setup generation.
+	if err := db.Sync(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+
+	// Fetch current database position.
+	dpos, err := db.Pos()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	c := file.NewReplicaClient(t.TempDir())
+	r := litestream.NewReplica(db, "")
+	c.Replica, r.Client = r, c
+
+	if err := r.Sync(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify client generation matches database.
+	generations, err := c.Generations(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	} else if got, want := len(generations), 1; got != want {
+		t.Fatalf("len(generations)=%v, want %v", got, want)
+	} else if got, want := generations[0], dpos.Generation; got != want {
+		t.Fatalf("generations[0]=%v, want %v", got, want)
+	}
+
+	// Verify WAL matches replica WAL.
+	if b0, err := os.ReadFile(db.Path() + "-wal"); err != nil {
+		t.Fatal(err)
+	} else if r, err := c.WALSegmentReader(context.Background(), litestream.Pos{Generation: generations[0], Index: 0, Offset: 0}); err != nil {
+		t.Fatal(err)
+	} else if b1, err := io.ReadAll(lz4.NewReader(r)); err != nil {
+		t.Fatal(err)
+	} else if err := r.Close(); err != nil {
+		t.Fatal(err)
+	} else if !bytes.Equal(b0, b1) {
+		t.Fatalf("wal mismatch: len(%d), len(%d)", len(b0), len(b1))
+	}
+}
+
+func TestReplica_Snapshot(t *testing.T) {
+	db, sqldb := MustOpenDBs(t)
+	defer MustCloseDBs(t, db, sqldb)
+
+	c := file.NewReplicaClient(t.TempDir())
+	r := litestream.NewReplica(db, "")
+	r.Client = c
+
+	// Execute a query to force a write to the WAL.
+	if _, err := sqldb.Exec(`CREATE TABLE foo (bar TEXT);`); err != nil {
+		t.Fatal(err)
+	} else if err := db.Sync(context.Background()); err != nil {
+		t.Fatal(err)
+	} else if err := r.Sync(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+
+	// Fetch current database position & snapshot.
+	pos0, err := db.Pos()
+	if err != nil {
+		t.Fatal(err)
+	} else if info, err := r.Snapshot(context.Background()); err != nil {
+		t.Fatal(err)
+	} else if got, want := info.Pos(), pos0.Truncate(); got != want {
+		t.Fatalf("pos=%s, want %s", got, want)
+	}
+
+	// Sync database and then replica.
+	if err := db.Sync(context.Background()); err != nil {
+		t.Fatal(err)
+	} else if err := r.Sync(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+
+	// Execute a query to force a write to the WAL & truncate to start new index.
+	if _, err := sqldb.Exec(`INSERT INTO foo (bar) VALUES ('baz');`); err != nil {
+		t.Fatal(err)
+	} else if err := db.Checkpoint(context.Background(), litestream.CheckpointModeTruncate); err != nil {
+		t.Fatal(err)
+	}
+
+	// Fetch current database position & snapshot.
+	pos1, err := db.Pos()
+	if err != nil {
+		t.Fatal(err)
+	} else if info, err := r.Snapshot(context.Background()); err != nil {
+		t.Fatal(err)
+	} else if got, want := info.Pos(), pos1.Truncate(); got != want {
+		t.Fatalf("pos=%v, want %v", got, want)
+	}
+
+	// Verify two snapshots exist.
+	if infos, err := r.Snapshots(context.Background()); err != nil {
+		t.Fatal(err)
+	} else if got, want := len(infos), 2; got != want {
+		t.Fatalf("len=%v, want %v", got, want)
+	} else if got, want := infos[0].Pos(), pos0.Truncate(); got != want {
+		t.Fatalf("info[0]=%s, want %s", got, want)
+	} else if got, want := infos[1].Pos(), pos1.Truncate(); got != want {
+		t.Fatalf("info[1]=%s, want %s", got, want)
+	}
 }

s3/replica_client.go

@@ -0,0 +1,767 @@
+package s3
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"os"
+	"path"
+	"regexp"
+	"sync"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+	"github.com/benbjohnson/litestream"
+	"github.com/benbjohnson/litestream/internal"
+	"golang.org/x/sync/errgroup"
+)
+
+// ReplicaClientType is the client type for this package.
+const ReplicaClientType = "s3"
+
+// MaxKeys is the number of keys S3 can operate on per batch.
+const MaxKeys = 1000
+
+// DefaultRegion is the region used if one is not specified.
+const DefaultRegion = "us-east-1"
+
+var _ litestream.ReplicaClient = (*ReplicaClient)(nil)
+
+// ReplicaClient is a client for writing snapshots & WAL segments to disk.
+type ReplicaClient struct {
+	mu       sync.Mutex
+	s3       *s3.S3 // s3 service
+	uploader *s3manager.Uploader
+
+	// AWS authentication keys.
+	AccessKeyID     string
+	SecretAccessKey string
+
+	// S3 bucket information
+	Region         string
+	Bucket         string
+	Path           string
+	Endpoint       string
+	ForcePathStyle bool
+	SkipVerify     bool
+}
+
+// NewReplicaClient returns a new instance of ReplicaClient.
+func NewReplicaClient() *ReplicaClient {
+	return &ReplicaClient{}
+}
+
+// Type returns "s3" as the client type.
+func (c *ReplicaClient) Type() string {
+	return ReplicaClientType
+}
+
+// Init initializes the connection to S3. No-op if already initialized.
+func (c *ReplicaClient) Init(ctx context.Context) (err error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if c.s3 != nil {
+		return nil
+	}
+
+	// Look up region if not specified and no endpoint is used.
+	// Endpoints are typically used for non-S3 object stores and do not
+	// necessarily require a region.
+	region := c.Region
+	if region == "" {
+		if c.Endpoint == "" {
+			if region, err = c.findBucketRegion(ctx, c.Bucket); err != nil {
+				return fmt.Errorf("cannot lookup bucket region: %w", err)
+			}
+		} else {
+			region = DefaultRegion // default for non-S3 object stores
+		}
+	}
+
+	// Create new AWS session.
+	config := c.config()
+	if region != "" {
+		config.Region = aws.String(region)
+	}
+
+	sess, err := session.NewSession(config)
+	if err != nil {
+		return fmt.Errorf("cannot create aws session: %w", err)
+	}
+	c.s3 = s3.New(sess)
+	c.uploader = s3manager.NewUploader(sess)
+	return nil
+}
+
+// config returns the AWS configuration. Uses the default credential chain
+// unless a key/secret are explicitly set.
+func (c *ReplicaClient) config() *aws.Config {
+	config := &aws.Config{}
+
+	if c.AccessKeyID != "" || c.SecretAccessKey != "" {
+		config.Credentials = credentials.NewStaticCredentials(c.AccessKeyID, c.SecretAccessKey, "")
+	}
+	if c.Endpoint != "" {
+		config.Endpoint = aws.String(c.Endpoint)
+	}
+	if c.ForcePathStyle {
+		config.S3ForcePathStyle = aws.Bool(c.ForcePathStyle)
+	}
+	if c.SkipVerify {
+		config.HTTPClient = &http.Client{Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+		}}
+	}
+
+	return config
+}
+
+func (c *ReplicaClient) findBucketRegion(ctx context.Context, bucket string) (string, error) {
+	// Connect to US standard region to fetch info.
+	config := c.config()
+	config.Region = aws.String(DefaultRegion)
+	sess, err := session.NewSession(config)
+	if err != nil {
+		return "", err
+	}
+
+	// Fetch bucket location, if possible. Must be bucket owner.
+	// This call can return a nil location which means it's in us-east-1.
+	if out, err := s3.New(sess).GetBucketLocation(&s3.GetBucketLocationInput{
+		Bucket: aws.String(bucket),
+	}); err != nil {
+		return "", err
+	} else if out.LocationConstraint != nil {
+		return *out.LocationConstraint, nil
+	}
+	return DefaultRegion, nil
+}
+
+// Generations returns a list of available generation names.
+func (c *ReplicaClient) Generations(ctx context.Context) ([]string, error) {
+	if err := c.Init(ctx); err != nil {
+		return nil, err
+	}
+
+	var generations []string
+	if err := c.s3.ListObjectsPagesWithContext(ctx, &s3.ListObjectsInput{
+		Bucket:    aws.String(c.Bucket),
+		Prefix:    aws.String(litestream.GenerationsPath(c.Path) + "/"),
+		Delimiter: aws.String("/"),
+	}, func(page *s3.ListObjectsOutput, lastPage bool) bool {
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "LIST").Inc()
+
+		for _, prefix := range page.CommonPrefixes {
+			name := path.Base(*prefix.Prefix)
+			if !litestream.IsGenerationName(name) {
+				continue
+			}
+			generations = append(generations, name)
+		}
+		return true
+	}); err != nil {
+		return nil, err
+	}
+
+	return generations, nil
+}
+
+// DeleteGeneration deletes all snapshots & WAL segments within a generation.
+func (c *ReplicaClient) DeleteGeneration(ctx context.Context, generation string) error {
+	if err := c.Init(ctx); err != nil {
+		return err
+	}
+
+	dir, err := litestream.GenerationPath(c.Path, generation)
+	if err != nil {
+		return fmt.Errorf("cannot determine generation path: %w", err)
+	}
+
+	// Collect all files for the generation.
+	var objIDs []*s3.ObjectIdentifier
+	if err := c.s3.ListObjectsPagesWithContext(ctx, &s3.ListObjectsInput{
+		Bucket: aws.String(c.Bucket),
+		Prefix: aws.String(dir),
+	}, func(page *s3.ListObjectsOutput, lastPage bool) bool {
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "LIST").Inc()
+
+		for _, obj := range page.Contents {
+			objIDs = append(objIDs, &s3.ObjectIdentifier{Key: obj.Key})
+		}
+		return true
+	}); err != nil {
+		return err
+	}
+
+	// Delete all files in batches.
+	for len(objIDs) > 0 {
+		n := MaxKeys
+		if len(objIDs) < n {
+			n = len(objIDs)
+		}
+
+		out, err := c.s3.DeleteObjectsWithContext(ctx, &s3.DeleteObjectsInput{
+			Bucket: aws.String(c.Bucket),
+			Delete: &s3.Delete{Objects: objIDs[:n], Quiet: aws.Bool(true)},
+		})
+		if err != nil {
+			return err
+		}
+		if err := deleteOutputError(out); err != nil {
+			return err
+		}
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "DELETE").Inc()
+
+		objIDs = objIDs[n:]
+	}
+
+	// log.Printf("%s(%s): retainer: deleting generation: %s", r.db.Path(), r.Name(), generation)
+
+	return nil
+}
+
+// Snapshots returns an iterator over all available snapshots for a generation.
+func (c *ReplicaClient) Snapshots(ctx context.Context, generation string) (litestream.SnapshotIterator, error) {
+	if err := c.Init(ctx); err != nil {
+		return nil, err
+	}
+	return newSnapshotIterator(ctx, c, generation), nil
+}
+
+// WriteSnapshot writes LZ4 compressed data from rd into a file on disk.
+func (c *ReplicaClient) WriteSnapshot(ctx context.Context, generation string, index int, rd io.Reader) (info litestream.SnapshotInfo, err error) {
+	if err := c.Init(ctx); err != nil {
+		return info, err
+	}
+
+	key, err := litestream.SnapshotPath(c.Path, generation, index)
+	if err != nil {
+		return info, fmt.Errorf("cannot determine snapshot path: %w", err)
+	}
+	startTime := time.Now()
+
+	rc := internal.NewReadCounter(rd)
+	if _, err := c.uploader.UploadWithContext(ctx, &s3manager.UploadInput{
+		Bucket: aws.String(c.Bucket),
+		Key:    aws.String(key),
+		Body:   rc,
+	}); err != nil {
+		return info, err
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "PUT").Inc()
+	internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "PUT").Add(float64(rc.N()))
+
+	// log.Printf("%s(%s): snapshot: creating %s/%08x t=%s", r.db.Path(), r.Name(), generation, index, time.Since(startTime).Truncate(time.Millisecond))
+
+	return litestream.SnapshotInfo{
+		Generation: generation,
+		Index:      index,
+		Size:       rc.N(),
+		CreatedAt:  startTime.UTC(),
+	}, nil
+}
+
+// SnapshotReader returns a reader for snapshot data at the given generation/index.
+func (c *ReplicaClient) SnapshotReader(ctx context.Context, generation string, index int) (io.ReadCloser, error) {
+	if err := c.Init(ctx); err != nil {
+		return nil, err
+	}
+
+	key, err := litestream.SnapshotPath(c.Path, generation, index)
+	if err != nil {
+		return nil, fmt.Errorf("cannot determine snapshot path: %w", err)
+	}
+
+	out, err := c.s3.GetObjectWithContext(ctx, &s3.GetObjectInput{
+		Bucket: aws.String(c.Bucket),
+		Key:    aws.String(key),
+	})
+	if isNotExists(err) {
+		return nil, os.ErrNotExist
+	} else if err != nil {
+		return nil, err
+	}
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "GET").Inc()
+	internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "GET").Add(float64(*out.ContentLength))
+
+	return out.Body, nil
+}
+
+// DeleteSnapshot deletes a snapshot with the given generation & index.
+func (c *ReplicaClient) DeleteSnapshot(ctx context.Context, generation string, index int) error {
+	if err := c.Init(ctx); err != nil {
+		return err
+	}
+
+	key, err := litestream.SnapshotPath(c.Path, generation, index)
+	if err != nil {
+		return fmt.Errorf("cannot determine snapshot path: %w", err)
+	}
+
+	out, err := c.s3.DeleteObjectsWithContext(ctx, &s3.DeleteObjectsInput{
+		Bucket: aws.String(c.Bucket),
+		Delete: &s3.Delete{Objects: []*s3.ObjectIdentifier{{Key: &key}}, Quiet: aws.Bool(true)},
+	})
+	if err != nil {
+		return err
+	}
+	if err := deleteOutputError(out); err != nil {
+		return err
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "DELETE").Inc()
+	return nil
+}
+
+// WALSegments returns an iterator over all available WAL files for a generation.
+func (c *ReplicaClient) WALSegments(ctx context.Context, generation string) (litestream.WALSegmentIterator, error) {
+	if err := c.Init(ctx); err != nil {
+		return nil, err
+	}
+	return newWALSegmentIterator(ctx, c, generation), nil
+}
+
+// WriteWALSegment writes LZ4 compressed data from rd into a file on disk.
+func (c *ReplicaClient) WriteWALSegment(ctx context.Context, pos litestream.Pos, rd io.Reader) (info litestream.WALSegmentInfo, err error) {
+	if err := c.Init(ctx); err != nil {
+		return info, err
+	}
+
+	key, err := litestream.WALSegmentPath(c.Path, pos.Generation, pos.Index, pos.Offset)
+	if err != nil {
+		return info, fmt.Errorf("cannot determine wal segment path: %w", err)
+	}
+	startTime := time.Now()
+
+	rc := internal.NewReadCounter(rd)
+	if _, err := c.uploader.UploadWithContext(ctx, &s3manager.UploadInput{
+		Bucket: aws.String(c.Bucket),
+		Key:    aws.String(key),
+		Body:   rc,
+	}); err != nil {
+		return info, err
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "PUT").Inc()
+	internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "PUT").Add(float64(rc.N()))
+
+	return litestream.WALSegmentInfo{
+		Generation: pos.Generation,
+		Index:      pos.Index,
+		Offset:     pos.Offset,
+		Size:       rc.N(),
+		CreatedAt:  startTime.UTC(),
+	}, nil
+}
+
+// WALSegmentReader returns a reader for a section of WAL data at the given index.
+// Returns os.ErrNotExist if no matching index/offset is found.
+func (c *ReplicaClient) WALSegmentReader(ctx context.Context, pos litestream.Pos) (io.ReadCloser, error) {
+	if err := c.Init(ctx); err != nil {
+		return nil, err
+	}
+
+	key, err := litestream.WALSegmentPath(c.Path, pos.Generation, pos.Index, pos.Offset)
+	if err != nil {
+		return nil, fmt.Errorf("cannot determine wal segment path: %w", err)
+	}
+
+	out, err := c.s3.GetObjectWithContext(ctx, &s3.GetObjectInput{
+		Bucket: aws.String(c.Bucket),
+		Key:    aws.String(key),
+	})
+	if isNotExists(err) {
+		return nil, os.ErrNotExist
+	} else if err != nil {
+		return nil, err
+	}
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "GET").Inc()
+	internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "GET").Add(float64(*out.ContentLength))
+
+	return out.Body, nil
+}
+
+// DeleteWALSegments deletes WAL segments with at the given positions.
+func (c *ReplicaClient) DeleteWALSegments(ctx context.Context, a []litestream.Pos) error {
+	if err := c.Init(ctx); err != nil {
+		return err
+	}
+
+	objIDs := make([]*s3.ObjectIdentifier, MaxKeys)
+	for len(a) > 0 {
+		n := MaxKeys
+		if len(a) < n {
+			n = len(a)
+		}
+
+		// Generate a batch of object IDs for deleting the WAL segments.
+		for i, pos := range a[:n] {
+			key, err := litestream.WALSegmentPath(c.Path, pos.Generation, pos.Index, pos.Offset)
+			if err != nil {
+				return fmt.Errorf("cannot determine wal segment path: %w", err)
+			}
+			objIDs[i] = &s3.ObjectIdentifier{Key: &key}
+		}
+
+		// Delete S3 objects in bulk.
+		out, err := c.s3.DeleteObjectsWithContext(ctx, &s3.DeleteObjectsInput{
+			Bucket: aws.String(c.Bucket),
+			Delete: &s3.Delete{Objects: objIDs[:n], Quiet: aws.Bool(true)},
+		})
+		if err != nil {
+			return err
+		}
+		if err := deleteOutputError(out); err != nil {
+			return err
+		}
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "DELETE").Inc()
+
+		a = a[n:]
+	}
+
+	return nil
+}
+
+// DeleteAll deletes everything on the remote path. Mainly used for testing.
+func (c *ReplicaClient) DeleteAll(ctx context.Context) error {
+	if err := c.Init(ctx); err != nil {
+		return err
+	}
+
+	prefix := c.Path
+	if prefix != "" {
+		prefix += "/"
+	}
+
+	// Collect all files for the generation.
+	var objIDs []*s3.ObjectIdentifier
+	if err := c.s3.ListObjectsPagesWithContext(ctx, &s3.ListObjectsInput{
+		Bucket: aws.String(c.Bucket),
+		Prefix: aws.String(prefix),
+	}, func(page *s3.ListObjectsOutput, lastPage bool) bool {
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "LIST").Inc()
+
+		for _, obj := range page.Contents {
+			objIDs = append(objIDs, &s3.ObjectIdentifier{Key: obj.Key})
+		}
+		return true
+	}); err != nil {
+		return err
+	}
+
+	// Delete all files in batches.
+	for len(objIDs) > 0 {
+		n := MaxKeys
+		if len(objIDs) < n {
+			n = len(objIDs)
+		}
+
+		out, err := c.s3.DeleteObjectsWithContext(ctx, &s3.DeleteObjectsInput{
+			Bucket: aws.String(c.Bucket),
+			Delete: &s3.Delete{Objects: objIDs[:n], Quiet: aws.Bool(true)},
+		})
+		if err != nil {
+			return err
+		}
+		if err := deleteOutputError(out); err != nil {
+			return err
+		}
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "DELETE").Inc()
+
+		objIDs = objIDs[n:]
+	}
+
+	return nil
+}
+
+type snapshotIterator struct {
+	client     *ReplicaClient
+	generation string
+
+	ch     chan litestream.SnapshotInfo
+	g      errgroup.Group
+	ctx    context.Context
+	cancel func()
+
+	info litestream.SnapshotInfo
+	err  error
+}
+
+func newSnapshotIterator(ctx context.Context, client *ReplicaClient, generation string) *snapshotIterator {
+	itr := &snapshotIterator{
+		client:     client,
+		generation: generation,
+		ch:         make(chan litestream.SnapshotInfo),
+	}
+
+	itr.ctx, itr.cancel = context.WithCancel(ctx)
+	itr.g.Go(itr.fetch)
+
+	return itr
+}
+
+// fetch runs in a separate goroutine to fetch pages of objects and stream them to a channel.
+func (itr *snapshotIterator) fetch() error {
+	defer close(itr.ch)
+
+	dir, err := litestream.SnapshotsPath(itr.client.Path, itr.generation)
+	if err != nil {
+		return fmt.Errorf("cannot determine snapshots path: %w", err)
+	}
+
+	return itr.client.s3.ListObjectsPagesWithContext(itr.ctx, &s3.ListObjectsInput{
+		Bucket:    aws.String(itr.client.Bucket),
+		Prefix:    aws.String(dir + "/"),
+		Delimiter: aws.String("/"),
+	}, func(page *s3.ListObjectsOutput, lastPage bool) bool {
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "LIST").Inc()
+
+		for _, obj := range page.Contents {
+			key := path.Base(*obj.Key)
+			index, err := litestream.ParseSnapshotPath(key)
+			if err != nil {
+				continue
+			}
+
+			info := litestream.SnapshotInfo{
+				Generation: itr.generation,
+				Index:      index,
+				Size:       *obj.Size,
+				CreatedAt:  obj.LastModified.UTC(),
+			}
+
+			select {
+			case <-itr.ctx.Done():
+			case itr.ch <- info:
+			}
+		}
+		return true
+	})
+}
+
+func (itr *snapshotIterator) Close() (err error) {
+	err = itr.err
+
+	// Cancel context and wait for error group to finish.
+	itr.cancel()
+	if e := itr.g.Wait(); e != nil && err == nil {
+		err = e
+	}
+
+	return err
+}
+
+func (itr *snapshotIterator) Next() bool {
+	// Exit if an error has already occurred.
+	if itr.err != nil {
+		return false
+	}
+
+	// Return false if context was canceled or if there are no more snapshots.
+	// Otherwise fetch the next snapshot and store it on the iterator.
+	select {
+	case <-itr.ctx.Done():
+		return false
+	case info, ok := <-itr.ch:
+		if !ok {
+			return false
+		}
+		itr.info = info
+		return true
+	}
+}
+
+func (itr *snapshotIterator) Err() error { return itr.err }
+
+func (itr *snapshotIterator) Snapshot() litestream.SnapshotInfo {
+	return itr.info
+}
+
+type walSegmentIterator struct {
+	client     *ReplicaClient
+	generation string
+
+	ch     chan litestream.WALSegmentInfo
+	g      errgroup.Group
+	ctx    context.Context
+	cancel func()
+
+	info litestream.WALSegmentInfo
+	err  error
+}
+
+func newWALSegmentIterator(ctx context.Context, client *ReplicaClient, generation string) *walSegmentIterator {
+	itr := &walSegmentIterator{
+		client:     client,
+		generation: generation,
+		ch:         make(chan litestream.WALSegmentInfo),
+	}
+
+	itr.ctx, itr.cancel = context.WithCancel(ctx)
+	itr.g.Go(itr.fetch)
+
+	return itr
+}
+
+// fetch runs in a separate goroutine to fetch pages of objects and stream them to a channel.
+func (itr *walSegmentIterator) fetch() error {
+	defer close(itr.ch)
+
+	dir, err := litestream.WALPath(itr.client.Path, itr.generation)
+	if err != nil {
+		return fmt.Errorf("cannot determine wal path: %w", err)
+	}
+
+	return itr.client.s3.ListObjectsPagesWithContext(itr.ctx, &s3.ListObjectsInput{
+		Bucket:    aws.String(itr.client.Bucket),
+		Prefix:    aws.String(dir + "/"),
+		Delimiter: aws.String("/"),
+	}, func(page *s3.ListObjectsOutput, lastPage bool) bool {
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "LIST").Inc()
+
+		for _, obj := range page.Contents {
+			key := path.Base(*obj.Key)
+			index, offset, err := litestream.ParseWALSegmentPath(key)
+			if err != nil {
+				continue
+			}
+
+			info := litestream.WALSegmentInfo{
+				Generation: itr.generation,
+				Index:      index,
+				Offset:     offset,
+				Size:       *obj.Size,
+				CreatedAt:  obj.LastModified.UTC(),
+			}
+
+			select {
+			case <-itr.ctx.Done():
+				return false
+			case itr.ch <- info:
+			}
+		}
+		return true
+	})
+}
+
+func (itr *walSegmentIterator) Close() (err error) {
+	err = itr.err
+
+	// Cancel context and wait for error group to finish.
+	itr.cancel()
+	if e := itr.g.Wait(); e != nil && err == nil {
+		err = e
+	}
+
+	return err
+}
+
+func (itr *walSegmentIterator) Next() bool {
+	// Exit if an error has already occurred.
+	if itr.err != nil {
+		return false
+	}
+
+	// Return false if context was canceled or if there are no more segments.
+	// Otherwise fetch the next segment and store it on the iterator.
+	select {
+	case <-itr.ctx.Done():
+		return false
+	case info, ok := <-itr.ch:
+		if !ok {
+			return false
+		}
+		itr.info = info
+		return true
+	}
+}
+
+func (itr *walSegmentIterator) Err() error { return itr.err }
+
+func (itr *walSegmentIterator) WALSegment() litestream.WALSegmentInfo {
+	return itr.info
+}
+
+// ParseHost extracts data from a hostname depending on the service provider.
+func ParseHost(s string) (bucket, region, endpoint string, forcePathStyle bool) {
+	// Extract port if one is specified.
+	host, port, err := net.SplitHostPort(s)
+	if err != nil {
+		host = s
+	}
+
+	// Default to path-based URLs, except for with AWS S3 itself.
+	forcePathStyle = true
+
+	// Extract fields from provider-specific host formats.
+	scheme := "https"
+	if a := localhostRegex.FindStringSubmatch(host); a != nil {
+		bucket, region = a[1], "us-east-1"
+		scheme, endpoint = "http", "localhost"
+	} else if a := backblazeRegex.FindStringSubmatch(host); a != nil {
+		bucket, region = a[1], a[2]
+		endpoint = fmt.Sprintf("s3.%s.backblazeb2.com", region)
+	} else if a := filebaseRegex.FindStringSubmatch(host); a != nil {
+		bucket, endpoint = a[1], "s3.filebase.com"
+	} else if a := digitalOceanRegex.FindStringSubmatch(host); a != nil {
+		bucket, region = a[1], a[2]
+		endpoint = fmt.Sprintf("%s.digitaloceanspaces.com", region)
+	} else if a := linodeRegex.FindStringSubmatch(host); a != nil {
+		bucket, region = a[1], a[2]
+		endpoint = fmt.Sprintf("%s.linodeobjects.com", region)
+	} else {
+		bucket = host
+		forcePathStyle = false
+	}
+
+	// Add port back to endpoint, if available.
+	if endpoint != "" && port != "" {
+		endpoint = net.JoinHostPort(endpoint, port)
+	}
+
+	// Prepend scheme to endpoint.
+	if endpoint != "" {
+		endpoint = scheme + "://" + endpoint
+	}
+
+	return bucket, region, endpoint, forcePathStyle
+}
+
+var (
+	localhostRegex    = regexp.MustCompile(`^(?:(.+)\.)?localhost$`)
+	backblazeRegex    = regexp.MustCompile(`^(?:(.+)\.)?s3.([^.]+)\.backblazeb2.com$`)
+	filebaseRegex     = regexp.MustCompile(`^(?:(.+)\.)?s3.filebase.com$`)
+	digitalOceanRegex = regexp.MustCompile(`^(?:(.+)\.)?([^.]+)\.digitaloceanspaces.com$`)
+	linodeRegex       = regexp.MustCompile(`^(?:(.+)\.)?([^.]+)\.linodeobjects.com$`)
+)
+
+func isNotExists(err error) bool {
+	switch err := err.(type) {
+	case awserr.Error:
+		return err.Code() == `NoSuchKey`
+	default:
+		return false
+	}
+}
+
+func deleteOutputError(out *s3.DeleteObjectsOutput) error {
+	switch len(out.Errors) {
+	case 0:
+		return nil
+	case 1:
+		return fmt.Errorf("deleting object %s: %s - %s", *out.Errors[0].Key, *out.Errors[0].Code, *out.Errors[0].Message)
+	default:
+		return fmt.Errorf("%d errors occured deleting objects, %s: %s - (%s (and %d others)",
+			len(out.Errors), *out.Errors[0].Key, *out.Errors[0].Code, *out.Errors[0].Message, len(out.Errors)-1)
+	}
+}

s3/s3_test.go

@@ -1,80 +0,0 @@
-package s3_test
-
-import (
-	"testing"
-
-	"github.com/benbjohnson/litestream/s3"
-)
-
-func TestParseHost(t *testing.T) {
-	// Ensure non-specific hosts return as buckets.
-	t.Run("S3", func(t *testing.T) {
-		bucket, region, endpoint, forcePathStyle := s3.ParseHost(`test.litestream.io`)
-		if got, want := bucket, `test.litestream.io`; got != want {
-			t.Fatalf("bucket=%q, want %q", got, want)
-		} else if got, want := region, ``; got != want {
-			t.Fatalf("region=%q, want %q", got, want)
-		} else if got, want := endpoint, ``; got != want {
-			t.Fatalf("endpoint=%q, want %q", got, want)
-		} else if got, want := forcePathStyle, false; got != want {
-			t.Fatalf("forcePathStyle=%v, want %v", got, want)
-		}
-	})
-
-	// Ensure localhosts use an HTTP endpoint and extract the bucket name.
-	t.Run("Localhost", func(t *testing.T) {
-		t.Run("WithPort", func(t *testing.T) {
-			bucket, region, endpoint, forcePathStyle := s3.ParseHost(`test.localhost:9000`)
-			if got, want := bucket, `test`; got != want {
-				t.Fatalf("bucket=%q, want %q", got, want)
-			} else if got, want := region, `us-east-1`; got != want {
-				t.Fatalf("region=%q, want %q", got, want)
-			} else if got, want := endpoint, `http://localhost:9000`; got != want {
-				t.Fatalf("endpoint=%q, want %q", got, want)
-			} else if got, want := forcePathStyle, true; got != want {
-				t.Fatalf("forcePathStyle=%v, want %v", got, want)
-			}
-		})
-
-		t.Run("WithoutPort", func(t *testing.T) {
-			bucket, region, endpoint, forcePathStyle := s3.ParseHost(`test.localhost`)
-			if got, want := bucket, `test`; got != want {
-				t.Fatalf("bucket=%q, want %q", got, want)
-			} else if got, want := region, `us-east-1`; got != want {
-				t.Fatalf("region=%q, want %q", got, want)
-			} else if got, want := endpoint, `http://localhost`; got != want {
-				t.Fatalf("endpoint=%q, want %q", got, want)
-			} else if got, want := forcePathStyle, true; got != want {
-				t.Fatalf("forcePathStyle=%v, want %v", got, want)
-			}
-		})
-	})
-
-	// Ensure backblaze B2 URLs extract bucket, region, & endpoint from host.
-	t.Run("Backblaze", func(t *testing.T) {
-		bucket, region, endpoint, forcePathStyle := s3.ParseHost(`test-123.s3.us-west-000.backblazeb2.com`)
-		if got, want := bucket, `test-123`; got != want {
-			t.Fatalf("bucket=%q, want %q", got, want)
-		} else if got, want := region, `us-west-000`; got != want {
-			t.Fatalf("region=%q, want %q", got, want)
-		} else if got, want := endpoint, `https://s3.us-west-000.backblazeb2.com`; got != want {
-			t.Fatalf("endpoint=%q, want %q", got, want)
-		} else if got, want := forcePathStyle, true; got != want {
-			t.Fatalf("forcePathStyle=%v, want %v", got, want)
-		}
-	})
-
-	// Ensure GCS URLs extract bucket & endpoint from host.
-	t.Run("GCS", func(t *testing.T) {
-		bucket, region, endpoint, forcePathStyle := s3.ParseHost(`litestream.io.storage.googleapis.com`)
-		if got, want := bucket, `litestream.io`; got != want {
-			t.Fatalf("bucket=%q, want %q", got, want)
-		} else if got, want := region, `us-east-1`; got != want {
-			t.Fatalf("region=%q, want %q", got, want)
-		} else if got, want := endpoint, `https://storage.googleapis.com`; got != want {
-			t.Fatalf("endpoint=%q, want %q", got, want)
-		} else if got, want := forcePathStyle, true; got != want {
-			t.Fatalf("forcePathStyle=%v, want %v", got, want)
-		}
-	})
-}

sftp/replica_client.go

@@ -0,0 +1,495 @@
+package sftp
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"os"
+	"path"
+	"sort"
+	"sync"
+	"time"
+
+	"github.com/benbjohnson/litestream"
+	"github.com/benbjohnson/litestream/internal"
+	"github.com/pkg/sftp"
+	"golang.org/x/crypto/ssh"
+)
+
+// ReplicaClientType is the client type for this package.
+const ReplicaClientType = "sftp"
+
+// Default settings for replica client.
+const (
+	DefaultDialTimeout = 30 * time.Second
+)
+
+var _ litestream.ReplicaClient = (*ReplicaClient)(nil)
+
+// ReplicaClient is a client for writing snapshots & WAL segments to disk.
+type ReplicaClient struct {
+	mu         sync.Mutex
+	sshClient  *ssh.Client
+	sftpClient *sftp.Client
+
+	// SFTP connection info
+	Host        string
+	User        string
+	Password    string
+	Path        string
+	KeyPath     string
+	DialTimeout time.Duration
+}
+
+// NewReplicaClient returns a new instance of ReplicaClient.
+func NewReplicaClient() *ReplicaClient {
+	return &ReplicaClient{
+		DialTimeout: DefaultDialTimeout,
+	}
+}
+
+// Type returns "gcs" as the client type.
+func (c *ReplicaClient) Type() string {
+	return ReplicaClientType
+}
+
+// Init initializes the connection to GCS. No-op if already initialized.
+func (c *ReplicaClient) Init(ctx context.Context) (_ *sftp.Client, err error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if c.sftpClient != nil {
+		return c.sftpClient, nil
+	}
+
+	if c.User == "" {
+		return nil, fmt.Errorf("sftp user required")
+	}
+
+	// Build SSH configuration & auth methods
+	config := &ssh.ClientConfig{
+		User:            c.User,
+		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
+		BannerCallback:  ssh.BannerDisplayStderr(),
+	}
+	if c.Password != "" {
+		config.Auth = append(config.Auth, ssh.Password(c.Password))
+	}
+
+	if c.KeyPath != "" {
+		buf, err := os.ReadFile(c.KeyPath)
+		if err != nil {
+			return nil, fmt.Errorf("cannot read sftp key path: %w", err)
+		}
+
+		signer, err := ssh.ParsePrivateKey(buf)
+		if err != nil {
+			return nil, fmt.Errorf("cannot parse sftp key path: %w", err)
+		}
+		config.Auth = append(config.Auth, ssh.PublicKeys(signer))
+	}
+
+	// Append standard port, if necessary.
+	host := c.Host
+	if _, _, err := net.SplitHostPort(c.Host); err != nil {
+		host = net.JoinHostPort(c.Host, "22")
+	}
+
+	// Connect via SSH.
+	if c.sshClient, err = ssh.Dial("tcp", host, config); err != nil {
+		return nil, err
+	}
+
+	// Wrap connection with an SFTP client.
+	if c.sftpClient, err = sftp.NewClient(c.sshClient); err != nil {
+		c.sshClient.Close()
+		c.sshClient = nil
+		return nil, err
+	}
+
+	return c.sftpClient, nil
+}
+
+// Generations returns a list of available generation names.
+func (c *ReplicaClient) Generations(ctx context.Context) (_ []string, err error) {
+	defer func() { c.resetOnConnError(err) }()
+
+	sftpClient, err := c.Init(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	fis, err := sftpClient.ReadDir(litestream.GenerationsPath(c.Path))
+	if os.IsNotExist(err) {
+		return nil, nil
+	} else if err != nil {
+		return nil, err
+	}
+
+	var generations []string
+	for _, fi := range fis {
+		if !fi.IsDir() {
+			continue
+		}
+
+		name := path.Base(fi.Name())
+		if !litestream.IsGenerationName(name) {
+			continue
+		}
+		generations = append(generations, name)
+	}
+
+	sort.Strings(generations)
+
+	return generations, nil
+}
+
+// DeleteGeneration deletes all snapshots & WAL segments within a generation.
+func (c *ReplicaClient) DeleteGeneration(ctx context.Context, generation string) (err error) {
+	defer func() { c.resetOnConnError(err) }()
+
+	sftpClient, err := c.Init(ctx)
+	if err != nil {
+		return err
+	}
+
+	dir, err := litestream.GenerationPath(c.Path, generation)
+	if err != nil {
+		return fmt.Errorf("cannot determine generation path: %w", err)
+	}
+
+	var dirs []string
+	walker := sftpClient.Walk(dir)
+	for walker.Step() {
+		if err := walker.Err(); err != nil {
+			return fmt.Errorf("cannot walk path %q: %w", walker.Path(), err)
+		}
+		if walker.Stat().IsDir() {
+			dirs = append(dirs, walker.Path())
+			continue
+		}
+
+		if err := sftpClient.Remove(walker.Path()); err != nil && !os.IsNotExist(err) {
+			return fmt.Errorf("cannot delete file %q: %w", walker.Path(), err)
+		}
+
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "DELETE").Inc()
+	}
+
+	// Remove directories in reverse order after they have been emptied.
+	for i := len(dirs) - 1; i >= 0; i-- {
+		filename := dirs[i]
+		if err := sftpClient.RemoveDirectory(filename); err != nil && !os.IsNotExist(err) {
+			return fmt.Errorf("cannot delete directory %q: %w", filename, err)
+		}
+	}
+
+	// log.Printf("%s(%s): retainer: deleting generation: %s", r.db.Path(), r.Name(), generation)
+
+	return nil
+}
+
+// Snapshots returns an iterator over all available snapshots for a generation.
+func (c *ReplicaClient) Snapshots(ctx context.Context, generation string) (_ litestream.SnapshotIterator, err error) {
+	defer func() { c.resetOnConnError(err) }()
+
+	sftpClient, err := c.Init(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	dir, err := litestream.SnapshotsPath(c.Path, generation)
+	if err != nil {
+		return nil, fmt.Errorf("cannot determine snapshots path: %w", err)
+	}
+
+	fis, err := sftpClient.ReadDir(dir)
+	if os.IsNotExist(err) {
+		return litestream.NewSnapshotInfoSliceIterator(nil), nil
+	} else if err != nil {
+		return nil, err
+	}
+
+	// Iterate over every file and convert to metadata.
+	infos := make([]litestream.SnapshotInfo, 0, len(fis))
+	for _, fi := range fis {
+		// Parse index from filename.
+		index, err := litestream.ParseSnapshotPath(path.Base(fi.Name()))
+		if err != nil {
+			continue
+		}
+
+		infos = append(infos, litestream.SnapshotInfo{
+			Generation: generation,
+			Index:      index,
+			Size:       fi.Size(),
+			CreatedAt:  fi.ModTime().UTC(),
+		})
+	}
+
+	sort.Sort(litestream.SnapshotInfoSlice(infos))
+
+	return litestream.NewSnapshotInfoSliceIterator(infos), nil
+}
+
+// WriteSnapshot writes LZ4 compressed data from rd to the object storage.
+func (c *ReplicaClient) WriteSnapshot(ctx context.Context, generation string, index int, rd io.Reader) (info litestream.SnapshotInfo, err error) {
+	defer func() { c.resetOnConnError(err) }()
+
+	sftpClient, err := c.Init(ctx)
+	if err != nil {
+		return info, err
+	}
+
+	filename, err := litestream.SnapshotPath(c.Path, generation, index)
+	if err != nil {
+		return info, fmt.Errorf("cannot determine snapshot path: %w", err)
+	}
+	startTime := time.Now()
+
+	if err := sftpClient.MkdirAll(path.Dir(filename)); err != nil {
+		return info, fmt.Errorf("cannot make parent wal segment directory %q: %w", path.Dir(filename), err)
+	}
+
+	f, err := sftpClient.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC)
+	if err != nil {
+		return info, fmt.Errorf("cannot open snapshot file for writing: %w", err)
+	}
+	defer f.Close()
+
+	n, err := io.Copy(f, rd)
+	if err != nil {
+		return info, err
+	} else if err := f.Close(); err != nil {
+		return info, err
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "PUT").Inc()
+	internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "PUT").Add(float64(n))
+
+	// log.Printf("%s(%s): snapshot: creating %s/%08x t=%s", r.db.Path(), r.Name(), generation, index, time.Since(startTime).Truncate(time.Millisecond))
+
+	return litestream.SnapshotInfo{
+		Generation: generation,
+		Index:      index,
+		Size:       n,
+		CreatedAt:  startTime.UTC(),
+	}, nil
+}
+
+// SnapshotReader returns a reader for snapshot data at the given generation/index.
+func (c *ReplicaClient) SnapshotReader(ctx context.Context, generation string, index int) (_ io.ReadCloser, err error) {
+	defer func() { c.resetOnConnError(err) }()
+
+	sftpClient, err := c.Init(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	filename, err := litestream.SnapshotPath(c.Path, generation, index)
+	if err != nil {
+		return nil, fmt.Errorf("cannot determine snapshot path: %w", err)
+	}
+
+	f, err := sftpClient.Open(filename)
+	if err != nil {
+		return nil, err
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "GET").Inc()
+
+	return f, nil
+}
+
+// DeleteSnapshot deletes a snapshot with the given generation & index.
+func (c *ReplicaClient) DeleteSnapshot(ctx context.Context, generation string, index int) (err error) {
+	defer func() { c.resetOnConnError(err) }()
+
+	sftpClient, err := c.Init(ctx)
+	if err != nil {
+		return err
+	}
+
+	filename, err := litestream.SnapshotPath(c.Path, generation, index)
+	if err != nil {
+		return fmt.Errorf("cannot determine snapshot path: %w", err)
+	}
+
+	if err := sftpClient.Remove(filename); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("cannot delete snapshot %q: %w", filename, err)
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "DELETE").Inc()
+	return nil
+}
+
+// WALSegments returns an iterator over all available WAL files for a generation.
+func (c *ReplicaClient) WALSegments(ctx context.Context, generation string) (_ litestream.WALSegmentIterator, err error) {
+	defer func() { c.resetOnConnError(err) }()
+
+	sftpClient, err := c.Init(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	dir, err := litestream.WALPath(c.Path, generation)
+	if err != nil {
+		return nil, fmt.Errorf("cannot determine wal path: %w", err)
+	}
+
+	fis, err := sftpClient.ReadDir(dir)
+	if os.IsNotExist(err) {
+		return litestream.NewWALSegmentInfoSliceIterator(nil), nil
+	} else if err != nil {
+		return nil, err
+	}
+
+	// Iterate over every file and convert to metadata.
+	infos := make([]litestream.WALSegmentInfo, 0, len(fis))
+	for _, fi := range fis {
+		index, offset, err := litestream.ParseWALSegmentPath(path.Base(fi.Name()))
+		if err != nil {
+			continue
+		}
+
+		infos = append(infos, litestream.WALSegmentInfo{
+			Generation: generation,
+			Index:      index,
+			Offset:     offset,
+			Size:       fi.Size(),
+			CreatedAt:  fi.ModTime().UTC(),
+		})
+	}
+
+	sort.Sort(litestream.WALSegmentInfoSlice(infos))
+
+	return litestream.NewWALSegmentInfoSliceIterator(infos), nil
+}
+
+// WriteWALSegment writes LZ4 compressed data from rd into a file on disk.
+func (c *ReplicaClient) WriteWALSegment(ctx context.Context, pos litestream.Pos, rd io.Reader) (info litestream.WALSegmentInfo, err error) {
+	defer func() { c.resetOnConnError(err) }()
+
+	sftpClient, err := c.Init(ctx)
+	if err != nil {
+		return info, err
+	}
+
+	filename, err := litestream.WALSegmentPath(c.Path, pos.Generation, pos.Index, pos.Offset)
+	if err != nil {
+		return info, fmt.Errorf("cannot determine wal segment path: %w", err)
+	}
+	startTime := time.Now()
+
+	if err := sftpClient.MkdirAll(path.Dir(filename)); err != nil {
+		return info, fmt.Errorf("cannot make parent snapshot directory %q: %w", path.Dir(filename), err)
+	}
+
+	f, err := sftpClient.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC)
+	if err != nil {
+		return info, fmt.Errorf("cannot open snapshot file for writing: %w", err)
+	}
+	defer f.Close()
+
+	n, err := io.Copy(f, rd)
+	if err != nil {
+		return info, err
+	} else if err := f.Close(); err != nil {
+		return info, err
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "PUT").Inc()
+	internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "PUT").Add(float64(n))
+
+	return litestream.WALSegmentInfo{
+		Generation: pos.Generation,
+		Index:      pos.Index,
+		Offset:     pos.Offset,
+		Size:       n,
+		CreatedAt:  startTime.UTC(),
+	}, nil
+}
+
+// WALSegmentReader returns a reader for a section of WAL data at the given index.
+// Returns os.ErrNotExist if no matching index/offset is found.
+func (c *ReplicaClient) WALSegmentReader(ctx context.Context, pos litestream.Pos) (_ io.ReadCloser, err error) {
+	defer func() { c.resetOnConnError(err) }()
+
+	sftpClient, err := c.Init(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	filename, err := litestream.WALSegmentPath(c.Path, pos.Generation, pos.Index, pos.Offset)
+	if err != nil {
+		return nil, fmt.Errorf("cannot determine wal segment path: %w", err)
+	}
+
+	f, err := sftpClient.Open(filename)
+	if err != nil {
+		return nil, err
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "GET").Inc()
+
+	return f, nil
+}
+
+// DeleteWALSegments deletes WAL segments with at the given positions.
+func (c *ReplicaClient) DeleteWALSegments(ctx context.Context, a []litestream.Pos) (err error) {
+	defer func() { c.resetOnConnError(err) }()
+
+	sftpClient, err := c.Init(ctx)
+	if err != nil {
+		return err
+	}
+
+	for _, pos := range a {
+		filename, err := litestream.WALSegmentPath(c.Path, pos.Generation, pos.Index, pos.Offset)
+		if err != nil {
+			return fmt.Errorf("cannot determine wal segment path: %w", err)
+		}
+
+		if err := sftpClient.Remove(filename); err != nil && !os.IsNotExist(err) {
+			return fmt.Errorf("cannot delete wal segment %q: %w", filename, err)
+		}
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "DELETE").Inc()
+	}
+
+	return nil
+}
+
+// Cleanup deletes path & generations directories after empty.
+func (c *ReplicaClient) Cleanup(ctx context.Context) (err error) {
+	defer func() { c.resetOnConnError(err) }()
+
+	sftpClient, err := c.Init(ctx)
+	if err != nil {
+		return err
+	}
+
+	if err := sftpClient.RemoveDirectory(litestream.GenerationsPath(c.Path)); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("cannot delete generations path: %w", err)
+	} else if err := sftpClient.RemoveDirectory(c.Path); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("cannot delete path: %w", err)
+	}
+	return nil
+}
+
+// resetOnConnError closes & clears the client if a connection error occurs.
+func (c *ReplicaClient) resetOnConnError(err error) {
+	if !errors.Is(err, sftp.ErrSSHFxConnectionLost) {
+		return
+	}
+
+	if c.sftpClient != nil {
+		c.sftpClient.Close()
+		c.sftpClient = nil
+	}
+	if c.sshClient != nil {
+		c.sshClient.Close()
+		c.sshClient = nil
+	}
+}