Don't kill exec process immediately (#484)

- Update Go module to v1.19 format
- Docker builder pinned to Go v1.20.1
- Alpine image pinned to 3.17.2 (rather than `latest`)

.github/CONTRIBUTING.md

@@ -1,17 +1,18 @@
-## Open-source, not open-contribution
+## Contribution Policy
 
-[Similar to SQLite](https://www.sqlite.org/copyright.html), Litestream is open
+Initially, Litestream was closed to outside contributions. The goal was to
-source but closed to contributions. This keeps the code base free of proprietary
+reduce burnout by limiting the maintenance overhead of reviewing and validating
-or licensed code but it also helps me continue to maintain and build Litestream.
+third-party code. However, this policy is overly broad and has prevented small,
+easily testable patches from being contributed.
 
-As the author of [BoltDB](https://github.com/boltdb/bolt), I found that
+Litestream is now open to code contributions for bug fixes only. Features carry
-accepting and maintaining third party patches contributed to my burn out and
+a long-term maintenance burden so they will not be accepted at this time.
-I eventually archived the project. Writing databases & low-level replication
+Please [submit an issue][new-issue] if you have a feature you'd like to
-tools involves nuance and simple one line changes can have profound and
+request.
-unexpected changes in correctness and performance. Small contributions
+
-typically required hours of my time to properly test and validate them.
+If you find mistakes in the documentation, please submit a fix to the
+[documentation repository][docs].
+
+[new-issue]: https://github.com/benbjohnson/litestream/issues/new
+[docs]: https://github.com/benbjohnson/litestream.io
 
-I am grateful for community involvement, bug reports, & feature requests. I do
-not wish to come off as anything but welcoming, however, I've
-made the decision to keep this project closed to contributions for my own
-mental health and long term viability of the project.

.github/workflows/commit.yml

@@ -0,0 +1,121 @@
+on: push
+
+jobs:
+  build:
+    name: Build & Unit Test
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '1.20'
+      - uses: actions/cache@v2
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ inputs.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: ${{ inputs.os }}-go-
+
+      - run: go env
+
+      - run: go install ./cmd/litestream
+
+      - run: go test -v ./...
+
+#      - name: Build integration test
+#        run: go test -c ./integration
+#
+#      - uses: actions/upload-artifact@v2
+#        with:
+#          name: integration.test
+#          path: integration.test
+#          if-no-files-found: error
+
+#  long-running-test:
+#    name: Run Long Running Unit Test
+#    runs-on: ubuntu-22.04
+#    steps:
+#      - uses: actions/checkout@v2
+#      - uses: actions/setup-go@v2
+#        with: 
+#          go-version: '1.20'
+#      - uses: actions/cache@v2
+#        with:
+#          path: ~/go/pkg/mod
+#          key: ${{ inputs.os }}-go-${{ hashFiles('**/go.sum') }}
+#          restore-keys: ${{ inputs.os }}-go-
+#
+#      - run: go install ./cmd/litestream
+#      - run: go test -v -run=TestCmd_Replicate_LongRunning ./integration -long-running-duration 1m
+
+#  s3-integration-test:
+#    name: Run S3 Integration Tests
+#    runs-on: ubuntu-18.04
+#    needs: build
+#    steps:
+#      - uses: actions/download-artifact@v2
+#        with:
+#          name: integration.test
+#      - run: chmod +x integration.test
+#
+#      - run: ./integration.test -test.v -test.run=TestReplicaClient -replica-type s3
+#        env:
+#          LITESTREAM_S3_ACCESS_KEY_ID: ${{ secrets.LITESTREAM_S3_ACCESS_KEY_ID }}
+#          LITESTREAM_S3_SECRET_ACCESS_KEY: ${{ secrets.LITESTREAM_S3_SECRET_ACCESS_KEY }}
+#          LITESTREAM_S3_REGION: us-east-1
+#          LITESTREAM_S3_BUCKET: integration.litestream.io
+
+#  gcp-integration-test:
+#    name: Run GCP Integration Tests
+#    runs-on: ubuntu-18.04
+#    needs: build
+#    steps:
+#      - name: Extract GCP credentials
+#        run: 'echo "$GOOGLE_APPLICATION_CREDENTIALS" > /opt/gcp.json'
+#        shell: bash
+#        env:
+#          GOOGLE_APPLICATION_CREDENTIALS: ${{secrets.GOOGLE_APPLICATION_CREDENTIALS}}
+#
+#      - uses: actions/download-artifact@v2
+#        with:
+#          name: integration.test
+#      - run: chmod +x integration.test
+#
+#      - run: ./integration.test -test.v -test.run=TestReplicaClient -replica-type gcs
+#        env:
+#          GOOGLE_APPLICATION_CREDENTIALS:  /opt/gcp.json
+#          LITESTREAM_GCS_BUCKET:           integration.litestream.io
+
+#  abs-integration-test:
+#    name: Run Azure Blob Store Integration Tests
+#    runs-on: ubuntu-18.04
+#    needs: build
+#    steps:
+#      - uses: actions/download-artifact@v2
+#        with:
+#          name: integration.test
+#      - run: chmod +x integration.test
+#
+#      - run: ./integration.test -test.v -test.run=TestReplicaClient -replica-type abs
+#        env:
+#          LITESTREAM_ABS_ACCOUNT_NAME:     ${{ secrets.LITESTREAM_ABS_ACCOUNT_NAME }}
+#          LITESTREAM_ABS_ACCOUNT_KEY:      ${{ secrets.LITESTREAM_ABS_ACCOUNT_KEY }}
+#          LITESTREAM_ABS_BUCKET:           integration
+
+#  sftp-integration-test:
+#    name: Run SFTP Integration Tests
+#    runs-on: ubuntu-18.04
+#    needs: build
+#    steps:
+#      - name: Extract SSH key
+#        run: 'echo "$LITESTREAM_SFTP_KEY" > /opt/id_ed25519'
+#        shell: bash
+#        env:
+#          LITESTREAM_SFTP_KEY: ${{secrets.LITESTREAM_SFTP_KEY}}
+#
+#      - name: Run sftp tests
+#        run: go test -v -run=TestReplicaClient ./integration -replica-type sftp
+#        env:
+#          LITESTREAM_SFTP_HOST:     ${{ secrets.LITESTREAM_SFTP_HOST }}
+#          LITESTREAM_SFTP_USER:     ${{ secrets.LITESTREAM_SFTP_USER }}
+#          LITESTREAM_SFTP_KEY_PATH: /opt/id_ed25519
+#          LITESTREAM_SFTP_PATH:     ${{ secrets.LITESTREAM_SFTP_PATH }}

.github/workflows/release.docker.yml

@@ -0,0 +1,51 @@
+on:
+  release:
+    types:
+      - published
+  pull_request:
+    types:
+      - opened
+      - synchronize
+      - reopened
+    branches-ignore:
+      - "dependabot/**"
+
+name: Release (Docker)
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    env:
+      PLATFORMS: "linux/amd64,linux/arm64,linux/arm/v7"
+      VERSION: "${{ github.event_name == 'release' && github.event.release.name || github.sha }}"
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: docker/setup-qemu-action@v1
+      - uses: docker/setup-buildx-action@v1
+
+      - uses: docker/login-action@v1
+        with:
+          username: benbjohnson
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      
+      - id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: litestream/litestream
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=sha
+            type=sha,format=long
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+      
+      - uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          platforms: ${{ env.PLATFORMS }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          build-args: |
+            LITESTREAM_VERSION=${{ env.VERSION }}

.github/workflows/release.linux.yml

@@ -3,19 +3,46 @@ on:
     types:
       - created
 
-name: release
+name: release (linux)
 jobs:
-  linux:
+  build:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-18.04
+    strategy:
+      matrix:
+        include:
+          - arch: amd64
+            cc:   gcc
+          - arch: arm64
+            cc:   aarch64-linux-gnu-gcc
+          - arch: arm
+            arm:  6
+            cc:   arm-linux-gnueabi-gcc
+          - arch: arm
+            arm:  7
+            cc:   arm-linux-gnueabihf-gcc
+
+    env:
+      GOOS:   linux
+      GOARCH: ${{ matrix.arch }}
+      GOARM:  ${{ matrix.arm }}
+      CC:     ${{ matrix.cc }}
+
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-go@v2
+        with:
+          go-version: '1.16'
 
       - id: release
         uses: bruceadams/get-release@v1.2.2
         env:
           GITHUB_TOKEN: ${{ github.token }}
 
+      - name: Install cross-compilers
+        run: |
+          sudo apt-get update 
+          sudo apt-get install -y gcc-aarch64-linux-gnu gcc-arm-linux-gnueabihf gcc-arm-linux-gnueabi
+
       - name: Install nfpm
         run: |
           wget https://github.com/goreleaser/nfpm/releases/download/v2.2.3/nfpm_2.2.3_Linux_x86_64.tar.gz
@@ -23,23 +50,24 @@ jobs:
 
       - name: Build litestream
         run: |
+          rm -rf dist
           mkdir -p dist
           cp etc/litestream.yml etc/litestream.service dist
           cat etc/nfpm.yml | LITESTREAM_VERSION=${{ steps.release.outputs.tag_name }} envsubst > dist/nfpm.yml
-          go build -ldflags "-X 'main.Version=${{ steps.release.outputs.tag_name }}'" -o dist/litestream ./cmd/litestream
+          CGO_ENABLED=1 go build -ldflags "-s -w -X 'main.Version=${{ steps.release.outputs.tag_name }}'" -o dist/litestream ./cmd/litestream
           
           cd dist
-          tar -czvf litestream-${{ steps.release.outputs.tag_name }}-linux-amd64.tar.gz litestream
+          tar -czvf litestream-${{ steps.release.outputs.tag_name }}-${{ env.GOOS }}-${{ env.GOARCH }}${{ env.GOARM }}.tar.gz litestream
-          ../nfpm pkg --config nfpm.yml --packager deb --target litestream-${{ steps.release.outputs.tag_name }}-linux-amd64.deb 
+          ../nfpm pkg --config nfpm.yml --packager deb --target litestream-${{ steps.release.outputs.tag_name }}-${{ env.GOOS }}-${{ env.GOARCH }}${{ env.GOARM }}.deb 
 
-      - name: Upload release binary
+      - name: Upload release tarball
         uses: actions/upload-release-asset@v1.0.2
         env:
           GITHUB_TOKEN: ${{ github.token }}
         with:
           upload_url: ${{ steps.release.outputs.upload_url }}
-          asset_path: ./dist/litestream-${{ steps.release.outputs.tag_name }}-linux-amd64.tar.gz
+          asset_path: ./dist/litestream-${{ steps.release.outputs.tag_name }}-${{ env.GOOS }}-${{ env.GOARCH }}${{ env.GOARM }}.tar.gz
-          asset_name: litestream-${{ steps.release.outputs.tag_name }}-linux-amd64.tar.gz
+          asset_name: litestream-${{ steps.release.outputs.tag_name }}-${{ env.GOOS }}-${{ env.GOARCH }}${{ env.GOARM }}.tar.gz
           asset_content_type: application/gzip
 
       - name: Upload debian package
@@ -48,6 +76,6 @@ jobs:
           GITHUB_TOKEN: ${{ github.token }}
         with:
           upload_url: ${{ steps.release.outputs.upload_url }}
-          asset_path: ./dist/litestream-${{ steps.release.outputs.tag_name }}-linux-amd64.deb
+          asset_path: ./dist/litestream-${{ steps.release.outputs.tag_name }}-${{ env.GOOS }}-${{ env.GOARCH }}${{ env.GOARM }}.deb
-          asset_name: litestream-${{ steps.release.outputs.tag_name }}-linux-amd64.deb
+          asset_name: litestream-${{ steps.release.outputs.tag_name }}-${{ env.GOOS }}-${{ env.GOARCH }}${{ env.GOARM }}.deb
           asset_content_type: application/octet-stream

.github/workflows/release.linux_static.yml

@@ -0,0 +1,62 @@
+on:
+  release:
+    types:
+      - created
+
+name: release (linux/static)
+jobs:
+  build:
+    runs-on: ubuntu-18.04
+    strategy:
+      matrix:
+        include:
+          - arch: amd64
+            cc:   gcc
+          - arch: arm64
+            cc:   aarch64-linux-gnu-gcc
+          - arch: arm
+            arm:  6
+            cc:   arm-linux-gnueabi-gcc
+          - arch: arm
+            arm:  7
+            cc:   arm-linux-gnueabihf-gcc
+
+    env:
+      GOOS:   linux
+      GOARCH: ${{ matrix.arch }}
+      GOARM:  ${{ matrix.arm }}
+      CC:     ${{ matrix.cc }}
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '1.16'
+
+      - id: release
+        uses: bruceadams/get-release@v1.2.2
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+
+      - name: Install cross-compilers
+        run: |
+          sudo apt-get update 
+          sudo apt-get install -y gcc-aarch64-linux-gnu gcc-arm-linux-gnueabihf gcc-arm-linux-gnueabi
+
+      - name: Build litestream
+        run: |
+          rm -rf dist
+          mkdir -p dist
+          CGO_ENABLED=1 go build -ldflags "-s -w -extldflags "-static" -X 'main.Version=${{ steps.release.outputs.tag_name }}'" -tags osusergo,netgo,sqlite_omit_load_extension -o dist/litestream ./cmd/litestream
+          cd dist
+          tar -czvf litestream-${{ steps.release.outputs.tag_name }}-${{ env.GOOS }}-${{ env.GOARCH }}${{ env.GOARM }}-static.tar.gz litestream
+
+      - name: Upload release tarball
+        uses: actions/upload-release-asset@v1.0.2
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        with:
+          upload_url: ${{ steps.release.outputs.upload_url }}
+          asset_path: ./dist/litestream-${{ steps.release.outputs.tag_name }}-${{ env.GOOS }}-${{ env.GOARCH }}${{ env.GOARM }}-static.tar.gz
+          asset_name: litestream-${{ steps.release.outputs.tag_name }}-${{ env.GOOS }}-${{ env.GOARCH }}${{ env.GOARM }}-static.tar.gz
+          asset_content_type: application/gzip

.github/workflows/test.yml

@@ -1,21 +0,0 @@
-on: push
-name: test
-jobs:
-  test:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/setup-go@v2
-        with:
-          go-version: '1.15'
-
-      - uses: actions/checkout@v2
-
-      - uses: actions/cache@v2
-        with:
-          path: ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go-
-
-      - name: Run unit tests
-        run: go test -v ./...

Dockerfile

@@ -0,0 +1,16 @@
+FROM golang:1.20.1 as builder
+
+WORKDIR /src/litestream
+COPY . .
+
+ARG LITESTREAM_VERSION=latest
+
+RUN --mount=type=cache,target=/root/.cache/go-build \
+	--mount=type=cache,target=/go/pkg \
+	go build -ldflags "-s -w -X 'main.Version=${LITESTREAM_VERSION}' -extldflags '-static'" -tags osusergo,netgo,sqlite_omit_load_extension -o /usr/local/bin/litestream ./cmd/litestream
+
+
+FROM alpine:3.17.2
+COPY --from=builder /usr/local/bin/litestream /usr/local/bin/litestream
+ENTRYPOINT ["/usr/local/bin/litestream"]
+CMD []

LICENSE

@@ -1,674 +1,202 @@
-                    GNU GENERAL PUBLIC LICENSE
+
-                       Version 3, 29 June 2007
+                                 Apache License
-
+                           Version 2.0, January 2004
- Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
+                        http://www.apache.org/licenses/
- Everyone is permitted to copy and distribute verbatim copies
+
- of this license document, but changing it is not allowed.
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
+
-                            Preamble
+   1. Definitions.
-
+
-  The GNU General Public License is a free, copyleft license for
+      "License" shall mean the terms and conditions for use, reproduction,
-software and other kinds of works.
+      and distribution as defined by Sections 1 through 9 of this document.
-
+
-  The licenses for most software and other practical works are designed
+      "Licensor" shall mean the copyright owner or entity authorized by
-to take away your freedom to share and change the works.  By contrast,
+      the copyright owner that is granting the License.
-the GNU General Public License is intended to guarantee your freedom to
+
-share and change all versions of a program--to make sure it remains free
+      "Legal Entity" shall mean the union of the acting entity and all
-software for all its users.  We, the Free Software Foundation, use the
+      other entities that control, are controlled by, or are under common
-GNU General Public License for most of our software; it applies also to
+      control with that entity. For the purposes of this definition,
-any other work released this way by its authors.  You can apply it to
+      "control" means (i) the power, direct or indirect, to cause the
-your programs, too.
+      direction or management of such entity, whether by contract or
-
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-  When we speak of free software, we are referring to freedom, not
+      outstanding shares, or (iii) beneficial ownership of such entity.
-price.  Our General Public Licenses are designed to make sure that you
+
-have the freedom to distribute copies of free software (and charge for
+      "You" (or "Your") shall mean an individual or Legal Entity
-them if you wish), that you receive source code or can get it if you
+      exercising permissions granted by this License.
-want it, that you can change the software or use pieces of it in new
+
-free programs, and that you know you can do these things.
+      "Source" form shall mean the preferred form for making modifications,
-
+      including but not limited to software source code, documentation
-  To protect your rights, we need to prevent others from denying you
+      source, and configuration files.
-these rights or asking you to surrender the rights.  Therefore, you have
+
-certain responsibilities if you distribute copies of the software, or if
+      "Object" form shall mean any form resulting from mechanical
-you modify it: responsibilities to respect the freedom of others.
+      transformation or translation of a Source form, including but
-
+      not limited to compiled object code, generated documentation,
-  For example, if you distribute copies of such a program, whether
+      and conversions to other media types.
-gratis or for a fee, you must pass on to the recipients the same
+
-freedoms that you received.  You must make sure that they, too, receive
+      "Work" shall mean the work of authorship, whether in Source or
-or can get the source code.  And you must show them these terms so they
+      Object form, made available under the License, as indicated by a
-know their rights.
+      copyright notice that is included in or attached to the work
-
+      (an example is provided in the Appendix below).
-  Developers that use the GNU GPL protect your rights with two steps:
+
-(1) assert copyright on the software, and (2) offer you this License
+      "Derivative Works" shall mean any work, whether in Source or Object
-giving you legal permission to copy, distribute and/or modify it.
+      form, that is based on (or derived from) the Work and for which the
-
+      editorial revisions, annotations, elaborations, or other modifications
-  For the developers' and authors' protection, the GPL clearly explains
+      represent, as a whole, an original work of authorship. For the purposes
-that there is no warranty for this free software.  For both users' and
+      of this License, Derivative Works shall not include works that remain
-authors' sake, the GPL requires that modified versions be marked as
+      separable from, or merely link (or bind by name) to the interfaces of,
-changed, so that their problems will not be attributed erroneously to
+      the Work and Derivative Works thereof.
-authors of previous versions.
+
-
+      "Contribution" shall mean any work of authorship, including
-  Some devices are designed to deny users access to install or run
+      the original version of the Work and any modifications or additions
-modified versions of the software inside them, although the manufacturer
+      to that Work or Derivative Works thereof, that is intentionally
-can do so.  This is fundamentally incompatible with the aim of
+      submitted to Licensor for inclusion in the Work by the copyright owner
-protecting users' freedom to change the software.  The systematic
+      or by an individual or Legal Entity authorized to submit on behalf of
-pattern of such abuse occurs in the area of products for individuals to
+      the copyright owner. For the purposes of this definition, "submitted"
-use, which is precisely where it is most unacceptable.  Therefore, we
+      means any form of electronic, verbal, or written communication sent
-have designed this version of the GPL to prohibit the practice for those
+      to the Licensor or its representatives, including but not limited to
-products.  If such problems arise substantially in other domains, we
+      communication on electronic mailing lists, source code control systems,
-stand ready to extend this provision to those domains in future versions
+      and issue tracking systems that are managed by, or on behalf of, the
-of the GPL, as needed to protect the freedom of users.
+      Licensor for the purpose of discussing and improving the Work, but
-
+      excluding communication that is conspicuously marked or otherwise
-  Finally, every program is threatened constantly by software patents.
+      designated in writing by the copyright owner as "Not a Contribution."
-States should not allow patents to restrict development and use of
+
-software on general-purpose computers, but in those that do, we wish to
+      "Contributor" shall mean Licensor and any individual or Legal Entity
-avoid the special danger that patents applied to a free program could
+      on behalf of whom a Contribution has been received by Licensor and
-make it effectively proprietary.  To prevent this, the GPL assures that
+      subsequently incorporated within the Work.
-patents cannot be used to render the program non-free.
+
-
+   2. Grant of Copyright License. Subject to the terms and conditions of
-  The precise terms and conditions for copying, distribution and
+      this License, each Contributor hereby grants to You a perpetual,
-modification follow.
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-
+      copyright license to reproduce, prepare Derivative Works of,
-                       TERMS AND CONDITIONS
+      publicly display, publicly perform, sublicense, and distribute the
-
+      Work and such Derivative Works in Source or Object form.
-  0. Definitions.
+
-
+   3. Grant of Patent License. Subject to the terms and conditions of
-  "This License" refers to version 3 of the GNU General Public License.
+      this License, each Contributor hereby grants to You a perpetual,
-
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-  "Copyright" also means copyright-like laws that apply to other kinds of
+      (except as stated in this section) patent license to make, have made,
-works, such as semiconductor masks.
+      use, offer to sell, sell, import, and otherwise transfer the Work,
-
+      where such license applies only to those patent claims licensable
-  "The Program" refers to any copyrightable work licensed under this
+      by such Contributor that are necessarily infringed by their
-License.  Each licensee is addressed as "you".  "Licensees" and
+      Contribution(s) alone or by combination of their Contribution(s)
-"recipients" may be individuals or organizations.
+      with the Work to which such Contribution(s) was submitted. If You
-
+      institute patent litigation against any entity (including a
-  To "modify" a work means to copy from or adapt all or part of the work
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
-in a fashion requiring copyright permission, other than the making of an
+      or a Contribution incorporated within the Work constitutes direct
-exact copy.  The resulting work is called a "modified version" of the
+      or contributory patent infringement, then any patent licenses
-earlier work or a work "based on" the earlier work.
+      granted to You under this License for that Work shall terminate
-
+      as of the date such litigation is filed.
-  A "covered work" means either the unmodified Program or a work based
+
-on the Program.
+   4. Redistribution. You may reproduce and distribute copies of the
-
+      Work or Derivative Works thereof in any medium, with or without
-  To "propagate" a work means to do anything with it that, without
+      modifications, and in Source or Object form, provided that You
-permission, would make you directly or secondarily liable for
+      meet the following conditions:
-infringement under applicable copyright law, except executing it on a
+
-computer or modifying a private copy.  Propagation includes copying,
+      (a) You must give any other recipients of the Work or
-distribution (with or without modification), making available to the
+          Derivative Works a copy of this License; and
-public, and in some countries other activities as well.
+
-
+      (b) You must cause any modified files to carry prominent notices
-  To "convey" a work means any kind of propagation that enables other
+          stating that You changed the files; and
-parties to make or receive copies.  Mere interaction with a user through
+
-a computer network, with no transfer of a copy, is not conveying.
+      (c) You must retain, in the Source form of any Derivative Works
-
+          that You distribute, all copyright, patent, trademark, and
-  An interactive user interface displays "Appropriate Legal Notices"
+          attribution notices from the Source form of the Work,
-to the extent that it includes a convenient and prominently visible
+          excluding those notices that do not pertain to any part of
-feature that (1) displays an appropriate copyright notice, and (2)
+          the Derivative Works; and
-tells the user that there is no warranty for the work (except to the
+
-extent that warranties are provided), that licensees may convey the
+      (d) If the Work includes a "NOTICE" text file as part of its
-work under this License, and how to view a copy of this License.  If
+          distribution, then any Derivative Works that You distribute must
-the interface presents a list of user commands or options, such as a
+          include a readable copy of the attribution notices contained
-menu, a prominent item in the list meets this criterion.
+          within such NOTICE file, excluding those notices that do not
-
+          pertain to any part of the Derivative Works, in at least one
-  1. Source Code.
+          of the following places: within a NOTICE text file distributed
-
+          as part of the Derivative Works; within the Source form or
-  The "source code" for a work means the preferred form of the work
+          documentation, if provided along with the Derivative Works; or,
-for making modifications to it.  "Object code" means any non-source
+          within a display generated by the Derivative Works, if and
-form of a work.
+          wherever such third-party notices normally appear. The contents
-
+          of the NOTICE file are for informational purposes only and
-  A "Standard Interface" means an interface that either is an official
+          do not modify the License. You may add Your own attribution
-standard defined by a recognized standards body, or, in the case of
+          notices within Derivative Works that You distribute, alongside
-interfaces specified for a particular programming language, one that
+          or as an addendum to the NOTICE text from the Work, provided
-is widely used among developers working in that language.
+          that such additional attribution notices cannot be construed
-
+          as modifying the License.
-  The "System Libraries" of an executable work include anything, other
+
-than the work as a whole, that (a) is included in the normal form of
+      You may add Your own copyright statement to Your modifications and
-packaging a Major Component, but which is not part of that Major
+      may provide additional or different license terms and conditions
-Component, and (b) serves only to enable use of the work with that
+      for use, reproduction, or distribution of Your modifications, or
-Major Component, or to implement a Standard Interface for which an
+      for any such Derivative Works as a whole, provided Your use,
-implementation is available to the public in source code form.  A
+      reproduction, and distribution of the Work otherwise complies with
-"Major Component", in this context, means a major essential component
+      the conditions stated in this License.
-(kernel, window system, and so on) of the specific operating system
+
-(if any) on which the executable work runs, or a compiler used to
+   5. Submission of Contributions. Unless You explicitly state otherwise,
-produce the work, or an object code interpreter used to run it.
+      any Contribution intentionally submitted for inclusion in the Work
-
+      by You to the Licensor shall be under the terms and conditions of
-  The "Corresponding Source" for a work in object code form means all
+      this License, without any additional terms or conditions.
-the source code needed to generate, install, and (for an executable
+      Notwithstanding the above, nothing herein shall supersede or modify
-work) run the object code and to modify the work, including scripts to
+      the terms of any separate license agreement you may have executed
-control those activities.  However, it does not include the work's
+      with Licensor regarding such Contributions.
-System Libraries, or general-purpose tools or generally available free
+
-programs which are used unmodified in performing those activities but
+   6. Trademarks. This License does not grant permission to use the trade
-which are not part of the work.  For example, Corresponding Source
+      names, trademarks, service marks, or product names of the Licensor,
-includes interface definition files associated with source files for
+      except as required for reasonable and customary use in describing the
-the work, and the source code for shared libraries and dynamically
+      origin of the Work and reproducing the content of the NOTICE file.
-linked subprograms that the work is specifically designed to require,
+
-such as by intimate data communication or control flow between those
+   7. Disclaimer of Warranty. Unless required by applicable law or
-subprograms and other parts of the work.
+      agreed to in writing, Licensor provides the Work (and each
-
+      Contributor provides its Contributions) on an "AS IS" BASIS,
-  The Corresponding Source need not include anything that users
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-can regenerate automatically from other parts of the Corresponding
+      implied, including, without limitation, any warranties or conditions
-Source.
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-
+      PARTICULAR PURPOSE. You are solely responsible for determining the
-  The Corresponding Source for a work in source code form is that
+      appropriateness of using or redistributing the Work and assume any
-same work.
+      risks associated with Your exercise of permissions under this License.
-
+
-  2. Basic Permissions.
+   8. Limitation of Liability. In no event and under no legal theory,
-
+      whether in tort (including negligence), contract, or otherwise,
-  All rights granted under this License are granted for the term of
+      unless required by applicable law (such as deliberate and grossly
-copyright on the Program, and are irrevocable provided the stated
+      negligent acts) or agreed to in writing, shall any Contributor be
-conditions are met.  This License explicitly affirms your unlimited
+      liable to You for damages, including any direct, indirect, special,
-permission to run the unmodified Program.  The output from running a
+      incidental, or consequential damages of any character arising as a
-covered work is covered by this License only if the output, given its
+      result of this License or out of the use or inability to use the
-content, constitutes a covered work.  This License acknowledges your
+      Work (including but not limited to damages for loss of goodwill,
-rights of fair use or other equivalent, as provided by copyright law.
+      work stoppage, computer failure or malfunction, or any and all
-
+      other commercial damages or losses), even if such Contributor
-  You may make, run and propagate covered works that you do not
+      has been advised of the possibility of such damages.
-convey, without conditions so long as your license otherwise remains
+
-in force.  You may convey covered works to others for the sole purpose
+   9. Accepting Warranty or Additional Liability. While redistributing
-of having them make modifications exclusively for you, or provide you
+      the Work or Derivative Works thereof, You may choose to offer,
-with facilities for running those works, provided that you comply with
+      and charge a fee for, acceptance of support, warranty, indemnity,
-the terms of this License in conveying all material for which you do
+      or other liability obligations and/or rights consistent with this
-not control copyright.  Those thus making or running the covered works
+      License. However, in accepting such obligations, You may act only
-for you must do so exclusively on your behalf, under your direction
+      on Your own behalf and on Your sole responsibility, not on behalf
-and control, on terms that prohibit them from making any copies of
+      of any other Contributor, and only if You agree to indemnify,
-your copyrighted material outside their relationship with you.
+      defend, and hold each Contributor harmless for any liability
-
+      incurred by, or claims asserted against, such Contributor by reason
-  Conveying under any other circumstances is permitted solely under
+      of your accepting any such warranty or additional liability.
-the conditions stated below.  Sublicensing is not allowed; section 10
+
-makes it unnecessary.
+   END OF TERMS AND CONDITIONS
-
+
-  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+   APPENDIX: How to apply the Apache License to your work.
-
+
-  No covered work shall be deemed part of an effective technological
+      To apply the Apache License to your work, attach the following
-measure under any applicable law fulfilling obligations under article
+      boilerplate notice, with the fields enclosed by brackets "[]"
-11 of the WIPO copyright treaty adopted on 20 December 1996, or
+      replaced with your own identifying information. (Don't include
-similar laws prohibiting or restricting circumvention of such
+      the brackets!)  The text should be enclosed in the appropriate
-measures.
+      comment syntax for the file format. We also recommend that a
-
+      file or class name and description of purpose be included on the
-  When you convey a covered work, you waive any legal power to forbid
+      same "printed page" as the copyright notice for easier
-circumvention of technological measures to the extent such circumvention
+      identification within third-party archives.
-is effected by exercising rights under this License with respect to
+
-the covered work, and you disclaim any intention to limit operation or
+   Copyright [yyyy] [name of copyright owner]
-modification of the work as a means of enforcing, against the work's
+
-users, your or third parties' legal rights to forbid circumvention of
+   Licensed under the Apache License, Version 2.0 (the "License");
-technological measures.
+   you may not use this file except in compliance with the License.
-
+   You may obtain a copy of the License at
-  4. Conveying Verbatim Copies.
+
-
+       http://www.apache.org/licenses/LICENSE-2.0
-  You may convey verbatim copies of the Program's source code as you
+
-receive it, in any medium, provided that you conspicuously and
+   Unless required by applicable law or agreed to in writing, software
-appropriately publish on each copy an appropriate copyright notice;
+   distributed under the License is distributed on an "AS IS" BASIS,
-keep intact all notices stating that this License and any
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-non-permissive terms added in accord with section 7 apply to the code;
+   See the License for the specific language governing permissions and
-keep intact all notices of the absence of any warranty; and give all
+   limitations under the License.
-recipients a copy of this License along with the Program.
-
-  You may charge any price or no price for each copy that you convey,
-and you may offer support or warranty protection for a fee.
-
-  5. Conveying Modified Source Versions.
-
-  You may convey a work based on the Program, or the modifications to
-produce it from the Program, in the form of source code under the
-terms of section 4, provided that you also meet all of these conditions:
-
-    a) The work must carry prominent notices stating that you modified
-    it, and giving a relevant date.
-
-    b) The work must carry prominent notices stating that it is
-    released under this License and any conditions added under section
-    7.  This requirement modifies the requirement in section 4 to
-    "keep intact all notices".
-
-    c) You must license the entire work, as a whole, under this
-    License to anyone who comes into possession of a copy.  This
-    License will therefore apply, along with any applicable section 7
-    additional terms, to the whole of the work, and all its parts,
-    regardless of how they are packaged.  This License gives no
-    permission to license the work in any other way, but it does not
-    invalidate such permission if you have separately received it.
-
-    d) If the work has interactive user interfaces, each must display
-    Appropriate Legal Notices; however, if the Program has interactive
-    interfaces that do not display Appropriate Legal Notices, your
-    work need not make them do so.
-
-  A compilation of a covered work with other separate and independent
-works, which are not by their nature extensions of the covered work,
-and which are not combined with it such as to form a larger program,
-in or on a volume of a storage or distribution medium, is called an
-"aggregate" if the compilation and its resulting copyright are not
-used to limit the access or legal rights of the compilation's users
-beyond what the individual works permit.  Inclusion of a covered work
-in an aggregate does not cause this License to apply to the other
-parts of the aggregate.
-
-  6. Conveying Non-Source Forms.
-
-  You may convey a covered work in object code form under the terms
-of sections 4 and 5, provided that you also convey the
-machine-readable Corresponding Source under the terms of this License,
-in one of these ways:
-
-    a) Convey the object code in, or embodied in, a physical product
-    (including a physical distribution medium), accompanied by the
-    Corresponding Source fixed on a durable physical medium
-    customarily used for software interchange.
-
-    b) Convey the object code in, or embodied in, a physical product
-    (including a physical distribution medium), accompanied by a
-    written offer, valid for at least three years and valid for as
-    long as you offer spare parts or customer support for that product
-    model, to give anyone who possesses the object code either (1) a
-    copy of the Corresponding Source for all the software in the
-    product that is covered by this License, on a durable physical
-    medium customarily used for software interchange, for a price no
-    more than your reasonable cost of physically performing this
-    conveying of source, or (2) access to copy the
-    Corresponding Source from a network server at no charge.
-
-    c) Convey individual copies of the object code with a copy of the
-    written offer to provide the Corresponding Source.  This
-    alternative is allowed only occasionally and noncommercially, and
-    only if you received the object code with such an offer, in accord
-    with subsection 6b.
-
-    d) Convey the object code by offering access from a designated
-    place (gratis or for a charge), and offer equivalent access to the
-    Corresponding Source in the same way through the same place at no
-    further charge.  You need not require recipients to copy the
-    Corresponding Source along with the object code.  If the place to
-    copy the object code is a network server, the Corresponding Source
-    may be on a different server (operated by you or a third party)
-    that supports equivalent copying facilities, provided you maintain
-    clear directions next to the object code saying where to find the
-    Corresponding Source.  Regardless of what server hosts the
-    Corresponding Source, you remain obligated to ensure that it is
-    available for as long as needed to satisfy these requirements.
-
-    e) Convey the object code using peer-to-peer transmission, provided
-    you inform other peers where the object code and Corresponding
-    Source of the work are being offered to the general public at no
-    charge under subsection 6d.
-
-  A separable portion of the object code, whose source code is excluded
-from the Corresponding Source as a System Library, need not be
-included in conveying the object code work.
-
-  A "User Product" is either (1) a "consumer product", which means any
-tangible personal property which is normally used for personal, family,
-or household purposes, or (2) anything designed or sold for incorporation
-into a dwelling.  In determining whether a product is a consumer product,
-doubtful cases shall be resolved in favor of coverage.  For a particular
-product received by a particular user, "normally used" refers to a
-typical or common use of that class of product, regardless of the status
-of the particular user or of the way in which the particular user
-actually uses, or expects or is expected to use, the product.  A product
-is a consumer product regardless of whether the product has substantial
-commercial, industrial or non-consumer uses, unless such uses represent
-the only significant mode of use of the product.
-
-  "Installation Information" for a User Product means any methods,
-procedures, authorization keys, or other information required to install
-and execute modified versions of a covered work in that User Product from
-a modified version of its Corresponding Source.  The information must
-suffice to ensure that the continued functioning of the modified object
-code is in no case prevented or interfered with solely because
-modification has been made.
-
-  If you convey an object code work under this section in, or with, or
-specifically for use in, a User Product, and the conveying occurs as
-part of a transaction in which the right of possession and use of the
-User Product is transferred to the recipient in perpetuity or for a
-fixed term (regardless of how the transaction is characterized), the
-Corresponding Source conveyed under this section must be accompanied
-by the Installation Information.  But this requirement does not apply
-if neither you nor any third party retains the ability to install
-modified object code on the User Product (for example, the work has
-been installed in ROM).
-
-  The requirement to provide Installation Information does not include a
-requirement to continue to provide support service, warranty, or updates
-for a work that has been modified or installed by the recipient, or for
-the User Product in which it has been modified or installed.  Access to a
-network may be denied when the modification itself materially and
-adversely affects the operation of the network or violates the rules and
-protocols for communication across the network.
-
-  Corresponding Source conveyed, and Installation Information provided,
-in accord with this section must be in a format that is publicly
-documented (and with an implementation available to the public in
-source code form), and must require no special password or key for
-unpacking, reading or copying.
-
-  7. Additional Terms.
-
-  "Additional permissions" are terms that supplement the terms of this
-License by making exceptions from one or more of its conditions.
-Additional permissions that are applicable to the entire Program shall
-be treated as though they were included in this License, to the extent
-that they are valid under applicable law.  If additional permissions
-apply only to part of the Program, that part may be used separately
-under those permissions, but the entire Program remains governed by
-this License without regard to the additional permissions.
-
-  When you convey a copy of a covered work, you may at your option
-remove any additional permissions from that copy, or from any part of
-it.  (Additional permissions may be written to require their own
-removal in certain cases when you modify the work.)  You may place
-additional permissions on material, added by you to a covered work,
-for which you have or can give appropriate copyright permission.
-
-  Notwithstanding any other provision of this License, for material you
-add to a covered work, you may (if authorized by the copyright holders of
-that material) supplement the terms of this License with terms:
-
-    a) Disclaiming warranty or limiting liability differently from the
-    terms of sections 15 and 16 of this License; or
-
-    b) Requiring preservation of specified reasonable legal notices or
-    author attributions in that material or in the Appropriate Legal
-    Notices displayed by works containing it; or
-
-    c) Prohibiting misrepresentation of the origin of that material, or
-    requiring that modified versions of such material be marked in
-    reasonable ways as different from the original version; or
-
-    d) Limiting the use for publicity purposes of names of licensors or
-    authors of the material; or
-
-    e) Declining to grant rights under trademark law for use of some
-    trade names, trademarks, or service marks; or
-
-    f) Requiring indemnification of licensors and authors of that
-    material by anyone who conveys the material (or modified versions of
-    it) with contractual assumptions of liability to the recipient, for
-    any liability that these contractual assumptions directly impose on
-    those licensors and authors.
-
-  All other non-permissive additional terms are considered "further
-restrictions" within the meaning of section 10.  If the Program as you
-received it, or any part of it, contains a notice stating that it is
-governed by this License along with a term that is a further
-restriction, you may remove that term.  If a license document contains
-a further restriction but permits relicensing or conveying under this
-License, you may add to a covered work material governed by the terms
-of that license document, provided that the further restriction does
-not survive such relicensing or conveying.
-
-  If you add terms to a covered work in accord with this section, you
-must place, in the relevant source files, a statement of the
-additional terms that apply to those files, or a notice indicating
-where to find the applicable terms.
-
-  Additional terms, permissive or non-permissive, may be stated in the
-form of a separately written license, or stated as exceptions;
-the above requirements apply either way.
-
-  8. Termination.
-
-  You may not propagate or modify a covered work except as expressly
-provided under this License.  Any attempt otherwise to propagate or
-modify it is void, and will automatically terminate your rights under
-this License (including any patent licenses granted under the third
-paragraph of section 11).
-
-  However, if you cease all violation of this License, then your
-license from a particular copyright holder is reinstated (a)
-provisionally, unless and until the copyright holder explicitly and
-finally terminates your license, and (b) permanently, if the copyright
-holder fails to notify you of the violation by some reasonable means
-prior to 60 days after the cessation.
-
-  Moreover, your license from a particular copyright holder is
-reinstated permanently if the copyright holder notifies you of the
-violation by some reasonable means, this is the first time you have
-received notice of violation of this License (for any work) from that
-copyright holder, and you cure the violation prior to 30 days after
-your receipt of the notice.
-
-  Termination of your rights under this section does not terminate the
-licenses of parties who have received copies or rights from you under
-this License.  If your rights have been terminated and not permanently
-reinstated, you do not qualify to receive new licenses for the same
-material under section 10.
-
-  9. Acceptance Not Required for Having Copies.
-
-  You are not required to accept this License in order to receive or
-run a copy of the Program.  Ancillary propagation of a covered work
-occurring solely as a consequence of using peer-to-peer transmission
-to receive a copy likewise does not require acceptance.  However,
-nothing other than this License grants you permission to propagate or
-modify any covered work.  These actions infringe copyright if you do
-not accept this License.  Therefore, by modifying or propagating a
-covered work, you indicate your acceptance of this License to do so.
-
-  10. Automatic Licensing of Downstream Recipients.
-
-  Each time you convey a covered work, the recipient automatically
-receives a license from the original licensors, to run, modify and
-propagate that work, subject to this License.  You are not responsible
-for enforcing compliance by third parties with this License.
-
-  An "entity transaction" is a transaction transferring control of an
-organization, or substantially all assets of one, or subdividing an
-organization, or merging organizations.  If propagation of a covered
-work results from an entity transaction, each party to that
-transaction who receives a copy of the work also receives whatever
-licenses to the work the party's predecessor in interest had or could
-give under the previous paragraph, plus a right to possession of the
-Corresponding Source of the work from the predecessor in interest, if
-the predecessor has it or can get it with reasonable efforts.
-
-  You may not impose any further restrictions on the exercise of the
-rights granted or affirmed under this License.  For example, you may
-not impose a license fee, royalty, or other charge for exercise of
-rights granted under this License, and you may not initiate litigation
-(including a cross-claim or counterclaim in a lawsuit) alleging that
-any patent claim is infringed by making, using, selling, offering for
-sale, or importing the Program or any portion of it.
-
-  11. Patents.
-
-  A "contributor" is a copyright holder who authorizes use under this
-License of the Program or a work on which the Program is based.  The
-work thus licensed is called the contributor's "contributor version".
-
-  A contributor's "essential patent claims" are all patent claims
-owned or controlled by the contributor, whether already acquired or
-hereafter acquired, that would be infringed by some manner, permitted
-by this License, of making, using, or selling its contributor version,
-but do not include claims that would be infringed only as a
-consequence of further modification of the contributor version.  For
-purposes of this definition, "control" includes the right to grant
-patent sublicenses in a manner consistent with the requirements of
-this License.
-
-  Each contributor grants you a non-exclusive, worldwide, royalty-free
-patent license under the contributor's essential patent claims, to
-make, use, sell, offer for sale, import and otherwise run, modify and
-propagate the contents of its contributor version.
-
-  In the following three paragraphs, a "patent license" is any express
-agreement or commitment, however denominated, not to enforce a patent
-(such as an express permission to practice a patent or covenant not to
-sue for patent infringement).  To "grant" such a patent license to a
-party means to make such an agreement or commitment not to enforce a
-patent against the party.
-
-  If you convey a covered work, knowingly relying on a patent license,
-and the Corresponding Source of the work is not available for anyone
-to copy, free of charge and under the terms of this License, through a
-publicly available network server or other readily accessible means,
-then you must either (1) cause the Corresponding Source to be so
-available, or (2) arrange to deprive yourself of the benefit of the
-patent license for this particular work, or (3) arrange, in a manner
-consistent with the requirements of this License, to extend the patent
-license to downstream recipients.  "Knowingly relying" means you have
-actual knowledge that, but for the patent license, your conveying the
-covered work in a country, or your recipient's use of the covered work
-in a country, would infringe one or more identifiable patents in that
-country that you have reason to believe are valid.
-
-  If, pursuant to or in connection with a single transaction or
-arrangement, you convey, or propagate by procuring conveyance of, a
-covered work, and grant a patent license to some of the parties
-receiving the covered work authorizing them to use, propagate, modify
-or convey a specific copy of the covered work, then the patent license
-you grant is automatically extended to all recipients of the covered
-work and works based on it.
-
-  A patent license is "discriminatory" if it does not include within
-the scope of its coverage, prohibits the exercise of, or is
-conditioned on the non-exercise of one or more of the rights that are
-specifically granted under this License.  You may not convey a covered
-work if you are a party to an arrangement with a third party that is
-in the business of distributing software, under which you make payment
-to the third party based on the extent of your activity of conveying
-the work, and under which the third party grants, to any of the
-parties who would receive the covered work from you, a discriminatory
-patent license (a) in connection with copies of the covered work
-conveyed by you (or copies made from those copies), or (b) primarily
-for and in connection with specific products or compilations that
-contain the covered work, unless you entered into that arrangement,
-or that patent license was granted, prior to 28 March 2007.
-
-  Nothing in this License shall be construed as excluding or limiting
-any implied license or other defenses to infringement that may
-otherwise be available to you under applicable patent law.
-
-  12. No Surrender of Others' Freedom.
-
-  If conditions are imposed on you (whether by court order, agreement or
-otherwise) that contradict the conditions of this License, they do not
-excuse you from the conditions of this License.  If you cannot convey a
-covered work so as to satisfy simultaneously your obligations under this
-License and any other pertinent obligations, then as a consequence you may
-not convey it at all.  For example, if you agree to terms that obligate you
-to collect a royalty for further conveying from those to whom you convey
-the Program, the only way you could satisfy both those terms and this
-License would be to refrain entirely from conveying the Program.
-
-  13. Use with the GNU Affero General Public License.
-
-  Notwithstanding any other provision of this License, you have
-permission to link or combine any covered work with a work licensed
-under version 3 of the GNU Affero General Public License into a single
-combined work, and to convey the resulting work.  The terms of this
-License will continue to apply to the part which is the covered work,
-but the special requirements of the GNU Affero General Public License,
-section 13, concerning interaction through a network will apply to the
-combination as such.
-
-  14. Revised Versions of this License.
-
-  The Free Software Foundation may publish revised and/or new versions of
-the GNU General Public License from time to time.  Such new versions will
-be similar in spirit to the present version, but may differ in detail to
-address new problems or concerns.
-
-  Each version is given a distinguishing version number.  If the
-Program specifies that a certain numbered version of the GNU General
-Public License "or any later version" applies to it, you have the
-option of following the terms and conditions either of that numbered
-version or of any later version published by the Free Software
-Foundation.  If the Program does not specify a version number of the
-GNU General Public License, you may choose any version ever published
-by the Free Software Foundation.
-
-  If the Program specifies that a proxy can decide which future
-versions of the GNU General Public License can be used, that proxy's
-public statement of acceptance of a version permanently authorizes you
-to choose that version for the Program.
-
-  Later license versions may give you additional or different
-permissions.  However, no additional obligations are imposed on any
-author or copyright holder as a result of your choosing to follow a
-later version.
-
-  15. Disclaimer of Warranty.
-
-  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
-APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
-HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
-OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
-THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
-IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
-ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
-
-  16. Limitation of Liability.
-
-  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
-WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
-THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
-GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
-USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
-DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
-PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
-EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
-SUCH DAMAGES.
-
-  17. Interpretation of Sections 15 and 16.
-
-  If the disclaimer of warranty and limitation of liability provided
-above cannot be given local legal effect according to their terms,
-reviewing courts shall apply local law that most closely approximates
-an absolute waiver of all civil liability in connection with the
-Program, unless a warranty or assumption of liability accompanies a
-copy of the Program in return for a fee.
-
-                     END OF TERMS AND CONDITIONS
-
-            How to Apply These Terms to Your New Programs
-
-  If you develop a new program, and you want it to be of the greatest
-possible use to the public, the best way to achieve this is to make it
-free software which everyone can redistribute and change under these terms.
-
-  To do so, attach the following notices to the program.  It is safest
-to attach them to the start of each source file to most effectively
-state the exclusion of warranty; and each file should have at least
-the "copyright" line and a pointer to where the full notice is found.
-
-    <one line to give the program's name and a brief idea of what it does.>
-    Copyright (C) <year>  <name of author>
-
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-Also add information on how to contact you by electronic and paper mail.
-
-  If the program does terminal interaction, make it output a short
-notice like this when it starts in an interactive mode:
-
-    <program>  Copyright (C) <year>  <name of author>
-    This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
-    This is free software, and you are welcome to redistribute it
-    under certain conditions; type `show c' for details.
-
-The hypothetical commands `show w' and `show c' should show the appropriate
-parts of the General Public License.  Of course, your program's commands
-might be different; for a GUI interface, you would use an "about box".
-
-  You should also get your employer (if you work as a programmer) or school,
-if any, to sign a "copyright disclaimer" for the program, if necessary.
-For more information on this, and how to apply and follow the GNU GPL, see
-<https://www.gnu.org/licenses/>.
-
-  The GNU General Public License does not permit incorporating your program
-into proprietary programs.  If your program is a subroutine library, you
-may consider it more useful to permit linking proprietary applications with
-the library.  If this is what you want to do, use the GNU Lesser General
-Public License instead of this License.  But first, please read
-<https://www.gnu.org/licenses/why-not-lgpl.html>.

Makefile

@@ -1,17 +1,26 @@
 default:
 
+docker:
+	docker build -t litestream .
+
 dist-linux:
 	mkdir -p dist
 	cp etc/litestream.yml dist/litestream.yml
-	docker run --rm -v "${PWD}":/usr/src/litestream -w /usr/src/litestream -e GOOS=linux -e GOARCH=amd64 golang:1.15 go build -v -o dist/litestream ./cmd/litestream
+	docker run --rm -v "${PWD}":/usr/src/litestream -w /usr/src/litestream -e GOOS=linux -e GOARCH=amd64 golang:1.16 go build -v -ldflags "-s -w" -o dist/litestream ./cmd/litestream
 	tar -cz -f dist/litestream-linux-amd64.tar.gz -C dist litestream
 
+dist-linux-arm:
+	docker run --rm -v "${PWD}":/usr/src/litestream -w /usr/src/litestream -e CGO_ENABLED=1 -e CC=arm-linux-gnueabihf-gcc -e GOOS=linux -e GOARCH=arm golang-xc:1.16 go build -v -o dist/litestream-linux-arm ./cmd/litestream
+
+dist-linux-arm64:
+	docker run --rm -v "${PWD}":/usr/src/litestream -w /usr/src/litestream -e CGO_ENABLED=1 -e CC=aarch64-linux-gnu-gcc -e GOOS=linux -e GOARCH=arm64 golang-xc:1.16 go build -v -o dist/litestream-linux-arm64 ./cmd/litestream
+
 dist-macos:
 ifndef LITESTREAM_VERSION
 	$(error LITESTREAM_VERSION is undefined)
 endif
 	mkdir -p dist
-	go build -v -ldflags "-X 'main.Version=${LITESTREAM_VERSION}'"  -o dist/litestream ./cmd/litestream
+	go build -v -ldflags "-s -w -X 'main.Version=${LITESTREAM_VERSION}'"  -o dist/litestream ./cmd/litestream
 	gon etc/gon.hcl
 	mv dist/litestream.zip dist/litestream-${LITESTREAM_VERSION}-darwin-amd64.zip
 	openssl dgst -sha256 dist/litestream-${LITESTREAM_VERSION}-darwin-amd64.zip

README.md

@@ -2,10 +2,11 @@ Litestream
 ![GitHub release (latest by date)](https://img.shields.io/github/v/release/benbjohnson/litestream)
 ![Status](https://img.shields.io/badge/status-beta-blue)
 ![GitHub](https://img.shields.io/github/license/benbjohnson/litestream)
+[![Docker Pulls](https://img.shields.io/docker/pulls/litestream/litestream.svg?maxAge=604800)](https://hub.docker.com/r/litestream/litestream/)
 ![test](https://github.com/benbjohnson/litestream/workflows/test/badge.svg)
 ==========
 
-Litestream is a standalone streaming replication tool for SQLite. It runs as a
+Litestream is a standalone disaster recovery tool for SQLite. It runs as a
 background process and safely replicates changes incrementally to another file
 or S3. Litestream only communicates with SQLite through the SQLite API so it
 will not corrupt your database.
@@ -28,32 +29,33 @@ of the most valuable contributions are in the forms of testing, feedback, and
 documentation. These help harden software and streamline usage for other users.
 
 I want to give special thanks to individuals who invest much of their time and 
-energy into the project to help make it better. Shout out to [Michael
+energy into the project to help make it better:
-Lynch](https://github.com/mtlynch) for digging into issues and contributing to
+
-the documentation.
+- Thanks to [Cory LaNou](https://twitter.com/corylanou) for giving early feedback and testing when Litestream was still pre-release.
+- Thanks to [Michael Lynch](https://github.com/mtlynch) for digging into issues and contributing to the documentation.
+- Thanks to [Kurt Mackey](https://twitter.com/mrkurt) for feedback and testing.
+- Thanks to [Sam Weston](https://twitter.com/cablespaghetti) for figuring out how to run Litestream on Kubernetes and writing up the docs for it.
+- Thanks to [Rafael](https://github.com/netstx) & [Jungle Boogie](https://github.com/jungle-boogie) for helping to get OpenBSD release builds working.
+- Thanks to [Simon Gottschlag](https://github.com/simongottschlag), [Marin](https://github.com/supermarin),[Victor Björklund](https://github.com/victorbjorklund), [Jonathan Beri](https://twitter.com/beriberikix) [Yuri](https://github.com/yurivish), [Nathan Probst](https://github.com/nprbst), [Yann Coleu](https://github.com/yanc0), and [Nicholas Grilly](https://twitter.com/ngrilly) for frequent feedback, testing, & support.
+
+Huge thanks to fly.io for their support and for contributing credits for testing and development!
 
 
-## Open-source, not open-contribution
+## Contribution Policy
 
-[Similar to SQLite](https://www.sqlite.org/copyright.html), Litestream is open
+Initially, Litestream was closed to outside contributions. The goal was to
-source but closed to code contributions. This keeps the code base free of
+reduce burnout by limiting the maintenance overhead of reviewing and validating
-proprietary or licensed code but it also helps me continue to maintain and build
+third-party code. However, this policy is overly broad and has prevented small,
-Litestream.
+easily testable patches from being contributed.
 
-As the author of [BoltDB](https://github.com/boltdb/bolt), I found that
+Litestream is now open to code contributions for bug fixes only. Features carry
-accepting and maintaining third party patches contributed to my burn out and
+a long-term maintenance burden so they will not be accepted at this time.
-I eventually archived the project. Writing databases & low-level replication
+Please [submit an issue][new-issue] if you have a feature you'd like to
-tools involves nuance and simple one line changes can have profound and
+request.
-unexpected changes in correctness and performance. Small contributions
-typically required hours of my time to properly test and validate them.
 
-I am grateful for community involvement, bug reports, & feature requests. I do
+If you find mistakes in the documentation, please submit a fix to the
-not wish to come off as anything but welcoming, however, I've
+[documentation repository][docs].
-made the decision to keep this project closed to contributions for my own
-mental health and long term viability of the project.
 
-The [documentation repository][docs] is MIT licensed and pull requests are welcome there.
+[new-issue]: https://github.com/benbjohnson/litestream/issues/new
-
-[releases]: https://github.com/benbjohnson/litestream/releases
 [docs]: https://github.com/benbjohnson/litestream.io
 

abs/replica_client.go

@@ -0,0 +1,565 @@
+package abs
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/url"
+	"os"
+	"path"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/Azure/azure-storage-blob-go/azblob"
+	"github.com/benbjohnson/litestream"
+	"github.com/benbjohnson/litestream/internal"
+	"golang.org/x/sync/errgroup"
+)
+
+// ReplicaClientType is the client type for this package.
+const ReplicaClientType = "abs"
+
+var _ litestream.ReplicaClient = (*ReplicaClient)(nil)
+
+// ReplicaClient is a client for writing snapshots & WAL segments to disk.
+type ReplicaClient struct {
+	mu           sync.Mutex
+	containerURL *azblob.ContainerURL
+
+	// Azure credentials
+	AccountName string
+	AccountKey  string
+	Endpoint    string
+
+	// Azure Blob Storage container information
+	Bucket string
+	Path   string
+}
+
+// NewReplicaClient returns a new instance of ReplicaClient.
+func NewReplicaClient() *ReplicaClient {
+	return &ReplicaClient{}
+}
+
+// Type returns "abs" as the client type.
+func (c *ReplicaClient) Type() string {
+	return ReplicaClientType
+}
+
+// Init initializes the connection to Azure. No-op if already initialized.
+func (c *ReplicaClient) Init(ctx context.Context) (err error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if c.containerURL != nil {
+		return nil
+	}
+
+	// Read account key from environment, if available.
+	accountKey := c.AccountKey
+	if accountKey == "" {
+		accountKey = os.Getenv("LITESTREAM_AZURE_ACCOUNT_KEY")
+	}
+
+	// Authenticate to ACS.
+	credential, err := azblob.NewSharedKeyCredential(c.AccountName, accountKey)
+	if err != nil {
+		return err
+	}
+
+	// Construct & parse endpoint unless already set.
+	endpoint := c.Endpoint
+	if endpoint == "" {
+		endpoint = fmt.Sprintf("https://%s.blob.core.windows.net", c.AccountName)
+	}
+	endpointURL, err := url.Parse(endpoint)
+	if err != nil {
+		return fmt.Errorf("cannot parse azure endpoint: %w", err)
+	}
+
+	// Build pipeline and reference to container.
+	pipeline := azblob.NewPipeline(credential, azblob.PipelineOptions{
+		Retry: azblob.RetryOptions{
+			TryTimeout: 24 * time.Hour,
+		},
+	})
+	containerURL := azblob.NewServiceURL(*endpointURL, pipeline).NewContainerURL(c.Bucket)
+	c.containerURL = &containerURL
+
+	return nil
+}
+
+// Generations returns a list of available generation names.
+func (c *ReplicaClient) Generations(ctx context.Context) ([]string, error) {
+	if err := c.Init(ctx); err != nil {
+		return nil, err
+	}
+
+	var generations []string
+	var marker azblob.Marker
+	for marker.NotDone() {
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "LIST").Inc()
+
+		resp, err := c.containerURL.ListBlobsHierarchySegment(ctx, marker, "/", azblob.ListBlobsSegmentOptions{
+			Prefix: litestream.GenerationsPath(c.Path) + "/",
+		})
+		if err != nil {
+			return nil, err
+		}
+		marker = resp.NextMarker
+
+		for _, prefix := range resp.Segment.BlobPrefixes {
+			name := path.Base(strings.TrimSuffix(prefix.Name, "/"))
+			if !litestream.IsGenerationName(name) {
+				continue
+			}
+			generations = append(generations, name)
+		}
+	}
+
+	return generations, nil
+}
+
+// DeleteGeneration deletes all snapshots & WAL segments within a generation.
+func (c *ReplicaClient) DeleteGeneration(ctx context.Context, generation string) error {
+	if err := c.Init(ctx); err != nil {
+		return err
+	}
+
+	dir, err := litestream.GenerationPath(c.Path, generation)
+	if err != nil {
+		return fmt.Errorf("cannot determine generation path: %w", err)
+	}
+
+	var marker azblob.Marker
+	for marker.NotDone() {
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "LIST").Inc()
+
+		resp, err := c.containerURL.ListBlobsFlatSegment(ctx, marker, azblob.ListBlobsSegmentOptions{Prefix: dir + "/"})
+		if err != nil {
+			return err
+		}
+		marker = resp.NextMarker
+
+		for _, item := range resp.Segment.BlobItems {
+			internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "DELETE").Inc()
+
+			blobURL := c.containerURL.NewBlobURL(item.Name)
+			if _, err := blobURL.Delete(ctx, azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{}); isNotExists(err) {
+				continue
+			} else if err != nil {
+				return err
+			}
+		}
+	}
+
+	// log.Printf("%s(%s): retainer: deleting generation: %s", r.db.Path(), r.Name(), generation)
+
+	return nil
+}
+
+// Snapshots returns an iterator over all available snapshots for a generation.
+func (c *ReplicaClient) Snapshots(ctx context.Context, generation string) (litestream.SnapshotIterator, error) {
+	if err := c.Init(ctx); err != nil {
+		return nil, err
+	}
+	return newSnapshotIterator(ctx, generation, c), nil
+}
+
+// WriteSnapshot writes LZ4 compressed data from rd to the object storage.
+func (c *ReplicaClient) WriteSnapshot(ctx context.Context, generation string, index int, rd io.Reader) (info litestream.SnapshotInfo, err error) {
+	if err := c.Init(ctx); err != nil {
+		return info, err
+	}
+
+	key, err := litestream.SnapshotPath(c.Path, generation, index)
+	if err != nil {
+		return info, fmt.Errorf("cannot determine snapshot path: %w", err)
+	}
+	startTime := time.Now()
+
+	rc := internal.NewReadCounter(rd)
+
+	blobURL := c.containerURL.NewBlockBlobURL(key)
+	if _, err := azblob.UploadStreamToBlockBlob(ctx, rc, blobURL, azblob.UploadStreamToBlockBlobOptions{
+		BlobHTTPHeaders: azblob.BlobHTTPHeaders{ContentType: "application/octet-stream"},
+		BlobAccessTier:  azblob.DefaultAccessTier,
+	}); err != nil {
+		return info, err
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "PUT").Inc()
+	internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "PUT").Add(float64(rc.N()))
+
+	// log.Printf("%s(%s): snapshot: creating %s/%08x t=%s", r.db.Path(), r.Name(), generation, index, time.Since(startTime).Truncate(time.Millisecond))
+
+	return litestream.SnapshotInfo{
+		Generation: generation,
+		Index:      index,
+		Size:       rc.N(),
+		CreatedAt:  startTime.UTC(),
+	}, nil
+}
+
+// SnapshotReader returns a reader for snapshot data at the given generation/index.
+func (c *ReplicaClient) SnapshotReader(ctx context.Context, generation string, index int) (io.ReadCloser, error) {
+	if err := c.Init(ctx); err != nil {
+		return nil, err
+	}
+
+	key, err := litestream.SnapshotPath(c.Path, generation, index)
+	if err != nil {
+		return nil, fmt.Errorf("cannot determine snapshot path: %w", err)
+	}
+
+	blobURL := c.containerURL.NewBlobURL(key)
+	resp, err := blobURL.Download(ctx, 0, 0, azblob.BlobAccessConditions{}, false, azblob.ClientProvidedKeyOptions{})
+	if isNotExists(err) {
+		return nil, os.ErrNotExist
+	} else if err != nil {
+		return nil, fmt.Errorf("cannot start new reader for %q: %w", key, err)
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "GET").Inc()
+	internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "GET").Add(float64(resp.ContentLength()))
+
+	return resp.Body(azblob.RetryReaderOptions{}), nil
+}
+
+// DeleteSnapshot deletes a snapshot with the given generation & index.
+func (c *ReplicaClient) DeleteSnapshot(ctx context.Context, generation string, index int) error {
+	if err := c.Init(ctx); err != nil {
+		return err
+	}
+
+	key, err := litestream.SnapshotPath(c.Path, generation, index)
+	if err != nil {
+		return fmt.Errorf("cannot determine snapshot path: %w", err)
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "DELETE").Inc()
+
+	blobURL := c.containerURL.NewBlobURL(key)
+	if _, err := blobURL.Delete(ctx, azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{}); isNotExists(err) {
+		return nil
+	} else if err != nil {
+		return fmt.Errorf("cannot delete snapshot %q: %w", key, err)
+	}
+	return nil
+}
+
+// WALSegments returns an iterator over all available WAL files for a generation.
+func (c *ReplicaClient) WALSegments(ctx context.Context, generation string) (litestream.WALSegmentIterator, error) {
+	if err := c.Init(ctx); err != nil {
+		return nil, err
+	}
+	return newWALSegmentIterator(ctx, generation, c), nil
+}
+
+// WriteWALSegment writes LZ4 compressed data from rd into a file on disk.
+func (c *ReplicaClient) WriteWALSegment(ctx context.Context, pos litestream.Pos, rd io.Reader) (info litestream.WALSegmentInfo, err error) {
+	if err := c.Init(ctx); err != nil {
+		return info, err
+	}
+
+	key, err := litestream.WALSegmentPath(c.Path, pos.Generation, pos.Index, pos.Offset)
+	if err != nil {
+		return info, fmt.Errorf("cannot determine wal segment path: %w", err)
+	}
+	startTime := time.Now()
+
+	rc := internal.NewReadCounter(rd)
+
+	blobURL := c.containerURL.NewBlockBlobURL(key)
+	if _, err := azblob.UploadStreamToBlockBlob(ctx, rc, blobURL, azblob.UploadStreamToBlockBlobOptions{
+		BlobHTTPHeaders: azblob.BlobHTTPHeaders{ContentType: "application/octet-stream"},
+		BlobAccessTier:  azblob.DefaultAccessTier,
+	}); err != nil {
+		return info, err
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "PUT").Inc()
+	internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "PUT").Add(float64(rc.N()))
+
+	return litestream.WALSegmentInfo{
+		Generation: pos.Generation,
+		Index:      pos.Index,
+		Offset:     pos.Offset,
+		Size:       rc.N(),
+		CreatedAt:  startTime.UTC(),
+	}, nil
+}
+
+// WALSegmentReader returns a reader for a section of WAL data at the given index.
+// Returns os.ErrNotExist if no matching index/offset is found.
+func (c *ReplicaClient) WALSegmentReader(ctx context.Context, pos litestream.Pos) (io.ReadCloser, error) {
+	if err := c.Init(ctx); err != nil {
+		return nil, err
+	}
+
+	key, err := litestream.WALSegmentPath(c.Path, pos.Generation, pos.Index, pos.Offset)
+	if err != nil {
+		return nil, fmt.Errorf("cannot determine wal segment path: %w", err)
+	}
+
+	blobURL := c.containerURL.NewBlobURL(key)
+	resp, err := blobURL.Download(ctx, 0, 0, azblob.BlobAccessConditions{}, false, azblob.ClientProvidedKeyOptions{})
+	if isNotExists(err) {
+		return nil, os.ErrNotExist
+	} else if err != nil {
+		return nil, fmt.Errorf("cannot start new reader for %q: %w", key, err)
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "GET").Inc()
+	internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "GET").Add(float64(resp.ContentLength()))
+
+	return resp.Body(azblob.RetryReaderOptions{}), nil
+}
+
+// DeleteWALSegments deletes WAL segments with at the given positions.
+func (c *ReplicaClient) DeleteWALSegments(ctx context.Context, a []litestream.Pos) error {
+	if err := c.Init(ctx); err != nil {
+		return err
+	}
+
+	for _, pos := range a {
+		key, err := litestream.WALSegmentPath(c.Path, pos.Generation, pos.Index, pos.Offset)
+		if err != nil {
+			return fmt.Errorf("cannot determine wal segment path: %w", err)
+		}
+
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "DELETE").Inc()
+
+		blobURL := c.containerURL.NewBlobURL(key)
+		if _, err := blobURL.Delete(ctx, azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{}); isNotExists(err) {
+			continue
+		} else if err != nil {
+			return fmt.Errorf("cannot delete wal segment %q: %w", key, err)
+		}
+	}
+
+	return nil
+}
+
+type snapshotIterator struct {
+	client     *ReplicaClient
+	generation string
+
+	ch     chan litestream.SnapshotInfo
+	g      errgroup.Group
+	ctx    context.Context
+	cancel func()
+
+	info litestream.SnapshotInfo
+	err  error
+}
+
+func newSnapshotIterator(ctx context.Context, generation string, client *ReplicaClient) *snapshotIterator {
+	itr := &snapshotIterator{
+		client:     client,
+		generation: generation,
+		ch:         make(chan litestream.SnapshotInfo),
+	}
+
+	itr.ctx, itr.cancel = context.WithCancel(ctx)
+	itr.g.Go(itr.fetch)
+
+	return itr
+}
+
+// fetch runs in a separate goroutine to fetch pages of objects and stream them to a channel.
+func (itr *snapshotIterator) fetch() error {
+	defer close(itr.ch)
+
+	dir, err := litestream.SnapshotsPath(itr.client.Path, itr.generation)
+	if err != nil {
+		return fmt.Errorf("cannot determine snapshots path: %w", err)
+	}
+
+	var marker azblob.Marker
+	for marker.NotDone() {
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "LIST").Inc()
+
+		resp, err := itr.client.containerURL.ListBlobsFlatSegment(itr.ctx, marker, azblob.ListBlobsSegmentOptions{Prefix: dir + "/"})
+		if err != nil {
+			return err
+		}
+		marker = resp.NextMarker
+
+		for _, item := range resp.Segment.BlobItems {
+			key := path.Base(item.Name)
+			index, err := litestream.ParseSnapshotPath(key)
+			if err != nil {
+				continue
+			}
+
+			info := litestream.SnapshotInfo{
+				Generation: itr.generation,
+				Index:      index,
+				Size:       *item.Properties.ContentLength,
+				CreatedAt:  item.Properties.CreationTime.UTC(),
+			}
+
+			select {
+			case <-itr.ctx.Done():
+			case itr.ch <- info:
+			}
+		}
+	}
+	return nil
+}
+
+func (itr *snapshotIterator) Close() (err error) {
+	err = itr.err
+
+	// Cancel context and wait for error group to finish.
+	itr.cancel()
+	if e := itr.g.Wait(); e != nil && err == nil {
+		err = e
+	}
+
+	return err
+}
+
+func (itr *snapshotIterator) Next() bool {
+	// Exit if an error has already occurred.
+	if itr.err != nil {
+		return false
+	}
+
+	// Return false if context was canceled or if there are no more snapshots.
+	// Otherwise fetch the next snapshot and store it on the iterator.
+	select {
+	case <-itr.ctx.Done():
+		return false
+	case info, ok := <-itr.ch:
+		if !ok {
+			return false
+		}
+		itr.info = info
+		return true
+	}
+}
+
+func (itr *snapshotIterator) Err() error { return itr.err }
+
+func (itr *snapshotIterator) Snapshot() litestream.SnapshotInfo {
+	return itr.info
+}
+
+type walSegmentIterator struct {
+	client     *ReplicaClient
+	generation string
+
+	ch     chan litestream.WALSegmentInfo
+	g      errgroup.Group
+	ctx    context.Context
+	cancel func()
+
+	info litestream.WALSegmentInfo
+	err  error
+}
+
+func newWALSegmentIterator(ctx context.Context, generation string, client *ReplicaClient) *walSegmentIterator {
+	itr := &walSegmentIterator{
+		client:     client,
+		generation: generation,
+		ch:         make(chan litestream.WALSegmentInfo),
+	}
+
+	itr.ctx, itr.cancel = context.WithCancel(ctx)
+	itr.g.Go(itr.fetch)
+
+	return itr
+}
+
+// fetch runs in a separate goroutine to fetch pages of objects and stream them to a channel.
+func (itr *walSegmentIterator) fetch() error {
+	defer close(itr.ch)
+
+	dir, err := litestream.WALPath(itr.client.Path, itr.generation)
+	if err != nil {
+		return fmt.Errorf("cannot determine wal path: %w", err)
+	}
+
+	var marker azblob.Marker
+	for marker.NotDone() {
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "LIST").Inc()
+
+		resp, err := itr.client.containerURL.ListBlobsFlatSegment(itr.ctx, marker, azblob.ListBlobsSegmentOptions{Prefix: dir + "/"})
+		if err != nil {
+			return err
+		}
+		marker = resp.NextMarker
+
+		for _, item := range resp.Segment.BlobItems {
+			key := path.Base(item.Name)
+			index, offset, err := litestream.ParseWALSegmentPath(key)
+			if err != nil {
+				continue
+			}
+
+			info := litestream.WALSegmentInfo{
+				Generation: itr.generation,
+				Index:      index,
+				Offset:     offset,
+				Size:       *item.Properties.ContentLength,
+				CreatedAt:  item.Properties.CreationTime.UTC(),
+			}
+
+			select {
+			case <-itr.ctx.Done():
+			case itr.ch <- info:
+			}
+		}
+	}
+	return nil
+}
+
+func (itr *walSegmentIterator) Close() (err error) {
+	err = itr.err
+
+	// Cancel context and wait for error group to finish.
+	itr.cancel()
+	if e := itr.g.Wait(); e != nil && err == nil {
+		err = e
+	}
+
+	return err
+}
+
+func (itr *walSegmentIterator) Next() bool {
+	// Exit if an error has already occurred.
+	if itr.err != nil {
+		return false
+	}
+
+	// Return false if context was canceled or if there are no more segments.
+	// Otherwise fetch the next segment and store it on the iterator.
+	select {
+	case <-itr.ctx.Done():
+		return false
+	case info, ok := <-itr.ch:
+		if !ok {
+			return false
+		}
+		itr.info = info
+		return true
+	}
+}
+
+func (itr *walSegmentIterator) Err() error { return itr.err }
+
+func (itr *walSegmentIterator) WALSegment() litestream.WALSegmentInfo {
+	return itr.info
+}
+
+func isNotExists(err error) bool {
+	switch err := err.(type) {
+	case azblob.StorageError:
+		return err.ServiceCode() == azblob.ServiceCodeBlobNotFound
+	default:
+		return false
+	}
+}

cmd/litestream/databases.go

@@ -15,7 +15,7 @@ type DatabasesCommand struct{}
 // Run executes the command.
 func (c *DatabasesCommand) Run(ctx context.Context, args []string) (err error) {
 	fs := flag.NewFlagSet("litestream-databases", flag.ContinueOnError)
-	configPath := registerConfigFlag(fs)
+	configPath, noExpandEnv := registerConfigFlag(fs)
 	fs.Usage = c.Usage
 	if err := fs.Parse(args); err != nil {
 		return err
@@ -27,7 +27,7 @@ func (c *DatabasesCommand) Run(ctx context.Context, args []string) (err error) {
 	if *configPath == "" {
 		*configPath = DefaultConfigPath()
 	}
-	config, err := ReadConfigFile(*configPath)
+	config, err := ReadConfigFile(*configPath, !*noExpandEnv)
 	if err != nil {
 		return err
 	}
@@ -72,6 +72,9 @@ Arguments:
 	    Specifies the configuration file.
 	    Defaults to %s
 
+	-no-expand-env
+	    Disables environment variable expansion in configuration file.
+
 `[1:],
 		DefaultConfigPath(),
 	)

cmd/litestream/generations.go

@@ -18,7 +18,7 @@ type GenerationsCommand struct{}
 // Run executes the command.
 func (c *GenerationsCommand) Run(ctx context.Context, args []string) (err error) {
 	fs := flag.NewFlagSet("litestream-generations", flag.ContinueOnError)
-	configPath := registerConfigFlag(fs)
+	configPath, noExpandEnv := registerConfigFlag(fs)
 	replicaName := fs.String("replica", "", "replica name")
 	fs.Usage = c.Usage
 	if err := fs.Parse(args); err != nil {
@@ -30,8 +30,8 @@ func (c *GenerationsCommand) Run(ctx context.Context, args []string) (err error)
 	}
 
 	var db *litestream.DB
-	var r litestream.Replica
+	var r *litestream.Replica
-	updatedAt := time.Now()
+	dbUpdatedAt := time.Now()
 	if isURL(fs.Arg(0)) {
 		if *configPath != "" {
 			return fmt.Errorf("cannot specify a replica URL and the -config flag")
@@ -45,7 +45,7 @@ func (c *GenerationsCommand) Run(ctx context.Context, args []string) (err error)
 		}
 
 		// Load configuration.
-		config, err := ReadConfigFile(*configPath)
+		config, err := ReadConfigFile(*configPath, !*noExpandEnv)
 		if err != nil {
 			return err
 		}
@@ -67,14 +67,14 @@ func (c *GenerationsCommand) Run(ctx context.Context, args []string) (err error)
 		}
 
 		// Determine last time database or WAL was updated.
-		if updatedAt, err = db.UpdatedAt(); err != nil {
+		if dbUpdatedAt, err = db.UpdatedAt(); err != nil {
 			return err
 		}
 	}
 
-	var replicas []litestream.Replica
+	var replicas []*litestream.Replica
 	if r != nil {
-		replicas = []litestream.Replica{r}
+		replicas = []*litestream.Replica{r}
 	} else {
 		replicas = db.Replicas
 	}
@@ -85,7 +85,7 @@ func (c *GenerationsCommand) Run(ctx context.Context, args []string) (err error)
 
 	fmt.Fprintln(w, "name\tgeneration\tlag\tstart\tend")
 	for _, r := range replicas {
-		generations, err := r.Generations(ctx)
+		generations, err := r.Client.Generations(ctx)
 		if err != nil {
 			log.Printf("%s: cannot list generations: %s", r.Name(), err)
 			continue
@@ -93,18 +93,18 @@ func (c *GenerationsCommand) Run(ctx context.Context, args []string) (err error)
 
 		// Iterate over each generation for the replica.
 		for _, generation := range generations {
-			stats, err := r.GenerationStats(ctx, generation)
+			createdAt, updatedAt, err := r.GenerationTimeBounds(ctx, generation)
 			if err != nil {
-				log.Printf("%s: cannot find generation stats: %s", r.Name(), err)
+				log.Printf("%s: cannot determine generation time bounds: %s", r.Name(), err)
 				continue
 			}
 
 			fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n",
 				r.Name(),
 				generation,
-				truncateDuration(updatedAt.Sub(stats.UpdatedAt)).String(),
+				truncateDuration(dbUpdatedAt.Sub(updatedAt)).String(),
-				stats.CreatedAt.Format(time.RFC3339),
+				createdAt.Format(time.RFC3339),
-				stats.UpdatedAt.Format(time.RFC3339),
+				updatedAt.Format(time.RFC3339),
 			)
 		}
 	}
@@ -131,6 +131,9 @@ Arguments:
 	    Specifies the configuration file.
 	    Defaults to %s
 
+	-no-expand-env
+	    Disables environment variable expansion in configuration file.
+
 	-replica NAME
 	    Optional, filters by replica.
 

cmd/litestream/main.go

@@ -9,16 +9,21 @@ import (
 	"log"
 	"net/url"
 	"os"
-	"os/signal"
 	"os/user"
 	"path"
 	"path/filepath"
 	"regexp"
+	"strconv"
 	"strings"
 	"time"
 
+	"filippo.io/age"
 	"github.com/benbjohnson/litestream"
+	"github.com/benbjohnson/litestream/abs"
+	"github.com/benbjohnson/litestream/file"
+	"github.com/benbjohnson/litestream/gcs"
 	"github.com/benbjohnson/litestream/s3"
+	"github.com/benbjohnson/litestream/sftp"
 	_ "github.com/mattn/go-sqlite3"
 	"gopkg.in/yaml.v2"
 )
@@ -60,6 +65,9 @@ func (m *Main) Run(ctx context.Context, args []string) (err error) {
 		return runWindowsService(ctx)
 	}
 
+	// Copy "LITESTEAM" environment credentials.
+	applyLitestreamEnv()
+
 	// Extract command name.
 	var cmd string
 	if len(args) > 0 {
@@ -78,21 +86,38 @@ func (m *Main) Run(ctx context.Context, args []string) (err error) {
 		}
 
 		// Setup signal handler.
-		ctx, cancel := context.WithCancel(ctx)
+		signalCh := signalChan()
-		ch := make(chan os.Signal, 1)
-		signal.Notify(ch, os.Interrupt)
-		go func() { <-ch; cancel() }()
 
-		if err := c.Run(ctx); err != nil {
+		if err := c.Run(); err != nil {
 			return err
 		}
 
 		// Wait for signal to stop program.
-		<-ctx.Done()
+		select {
-		signal.Reset()
+		case err = <-c.execCh:
+			fmt.Println("subprocess exited, litestream shutting down")
+		case sig := <-signalCh:
+			fmt.Println("signal received, litestream shutting down")
+
+			if c.cmd != nil {
+				fmt.Println("sending signal to exec process")
+				if err := c.cmd.Process.Signal(sig); err != nil {
+					return fmt.Errorf("cannot signal exec process: %w", err)
+				}
+
+				fmt.Println("waiting for exec process to close")
+				if err := <-c.execCh; err != nil && !strings.HasPrefix(err.Error(), "signal:") {
+					return fmt.Errorf("cannot wait for exec process: %w", err)
+				}
+			}
+		}
 
 		// Gracefully close.
-		return c.Close()
+		if e := c.Close(); e != nil && err == nil {
+			err = e
+		}
+		fmt.Println("litestream shut down")
+		return err
 
 	case "restore":
 		return (&RestoreCommand{}).Run(ctx, args)
@@ -140,6 +165,10 @@ type Config struct {
 	// List of databases to manage.
 	DBs []*DBConfig `yaml:"dbs"`
 
+	// Subcommand to execute during replication.
+	// Litestream will shutdown when subcommand exits.
+	Exec string `yaml:"exec"`
+
 	// Global S3 settings
 	AccessKeyID     string `yaml:"access-key-id"`
 	SecretAccessKey string `yaml:"secret-access-key"`
@@ -175,7 +204,8 @@ func (c *Config) DBConfig(path string) *DBConfig {
 }
 
 // ReadConfigFile unmarshals config from filename. Expands path if needed.
-func ReadConfigFile(filename string) (_ Config, err error) {
+// If expandEnv is true then environment variables are expanded in the config.
+func ReadConfigFile(filename string, expandEnv bool) (_ Config, err error) {
 	config := DefaultConfig()
 
 	// Expand filename, if necessary.
@@ -184,12 +214,20 @@ func ReadConfigFile(filename string) (_ Config, err error) {
 		return config, err
 	}
 
-	// Read & deserialize configuration.
+	// Read configuration.
-	if buf, err := ioutil.ReadFile(filename); os.IsNotExist(err) {
+	buf, err := ioutil.ReadFile(filename)
+	if os.IsNotExist(err) {
 		return config, fmt.Errorf("config file not found: %s", filename)
 	} else if err != nil {
 		return config, err
-	} else if err := yaml.Unmarshal(buf, &config); err != nil {
+	}
+
+	// Expand environment variables, if enabled.
+	if expandEnv {
+		buf = []byte(os.ExpandEnv(string(buf)))
+	}
+
+	if err := yaml.Unmarshal(buf, &config); err != nil {
 		return config, err
 	}
 
@@ -255,15 +293,15 @@ func NewDBFromConfig(dbc *DBConfig) (*litestream.DB, error) {
 
 // ReplicaConfig represents the configuration for a single replica in a database.
 type ReplicaConfig struct {
-	Type                   string        `yaml:"type"` // "file", "s3"
+	Type                   string         `yaml:"type"` // "file", "s3"
-	Name                   string        `yaml:"name"` // name of replica, optional.
+	Name                   string         `yaml:"name"` // name of replica, optional.
-	Path                   string        `yaml:"path"`
+	Path                   string         `yaml:"path"`
-	URL                    string        `yaml:"url"`
+	URL                    string         `yaml:"url"`
-	Retention              time.Duration `yaml:"retention"`
+	Retention              *time.Duration `yaml:"retention"`
-	RetentionCheckInterval time.Duration `yaml:"retention-check-interval"`
+	RetentionCheckInterval *time.Duration `yaml:"retention-check-interval"`
-	SyncInterval           time.Duration `yaml:"sync-interval"` // s3 only
+	SyncInterval           *time.Duration `yaml:"sync-interval"`
-	SnapshotInterval       time.Duration `yaml:"snapshot-interval"`
+	SnapshotInterval       *time.Duration `yaml:"snapshot-interval"`
-	ValidationInterval     time.Duration `yaml:"validation-interval"`
+	ValidationInterval     *time.Duration `yaml:"validation-interval"`
 
 	// S3 settings
 	AccessKeyID     string `yaml:"access-key-id"`
@@ -272,27 +310,97 @@ type ReplicaConfig struct {
 	Bucket          string `yaml:"bucket"`
 	Endpoint        string `yaml:"endpoint"`
 	ForcePathStyle  *bool  `yaml:"force-path-style"`
+	SkipVerify      bool   `yaml:"skip-verify"`
+
+	// ABS settings
+	AccountName string `yaml:"account-name"`
+	AccountKey  string `yaml:"account-key"`
+
+	// SFTP settings
+	Host     string `yaml:"host"`
+	User     string `yaml:"user"`
+	Password string `yaml:"password"`
+	KeyPath  string `yaml:"key-path"`
+
+	// Encryption identities and recipients
+	Age struct {
+		Identities []string `yaml:"identities"`
+		Recipients []string `yaml:"recipients"`
+	} `yaml:"age"`
 }
 
 // NewReplicaFromConfig instantiates a replica for a DB based on a config.
-func NewReplicaFromConfig(c *ReplicaConfig, db *litestream.DB) (litestream.Replica, error) {
+func NewReplicaFromConfig(c *ReplicaConfig, db *litestream.DB) (_ *litestream.Replica, err error) {
 	// Ensure user did not specify URL in path.
 	if isURL(c.Path) {
 		return nil, fmt.Errorf("replica path cannot be a url, please use the 'url' field instead: %s", c.Path)
 	}
 
+	// Build replica.
+	r := litestream.NewReplica(db, c.Name)
+	if v := c.Retention; v != nil {
+		r.Retention = *v
+	}
+	if v := c.RetentionCheckInterval; v != nil {
+		r.RetentionCheckInterval = *v
+	}
+	if v := c.SyncInterval; v != nil {
+		r.SyncInterval = *v
+	}
+	if v := c.SnapshotInterval; v != nil {
+		r.SnapshotInterval = *v
+	}
+	if v := c.ValidationInterval; v != nil {
+		r.ValidationInterval = *v
+	}
+	for _, str := range c.Age.Identities {
+		identities, err := age.ParseIdentities(strings.NewReader(str))
+		if err != nil {
+			return nil, err
+		}
+
+		r.AgeIdentities = append(r.AgeIdentities, identities...)
+	}
+	for _, str := range c.Age.Recipients {
+		recipients, err := age.ParseRecipients(strings.NewReader(str))
+		if err != nil {
+			return nil, err
+		}
+
+		r.AgeRecipients = append(r.AgeRecipients, recipients...)
+	}
+
+	// Build and set client on replica.
 	switch c.ReplicaType() {
 	case "file":
-		return newFileReplicaFromConfig(c, db)
+		if r.Client, err = newFileReplicaClientFromConfig(c, r); err != nil {
+			return nil, err
+		}
 	case "s3":
-		return newS3ReplicaFromConfig(c, db)
+		if r.Client, err = newS3ReplicaClientFromConfig(c, r); err != nil {
+			return nil, err
+		}
+	case "gcs":
+		if r.Client, err = newGCSReplicaClientFromConfig(c, r); err != nil {
+			return nil, err
+		}
+	case "abs":
+		if r.Client, err = newABSReplicaClientFromConfig(c, r); err != nil {
+			return nil, err
+		}
+	case "sftp":
+		if r.Client, err = newSFTPReplicaClientFromConfig(c, r); err != nil {
+			return nil, err
+		}
 	default:
 		return nil, fmt.Errorf("unknown replica type in config: %q", c.Type)
 	}
+
+	return r, nil
 }
 
-// newFileReplicaFromConfig returns a new instance of FileReplica build from config.
+// newFileReplicaClientFromConfig returns a new instance of file.ReplicaClient built from config.
-func newFileReplicaFromConfig(c *ReplicaConfig, db *litestream.DB) (_ *litestream.FileReplica, err error) {
+func newFileReplicaClientFromConfig(c *ReplicaConfig, r *litestream.Replica) (_ *file.ReplicaClient, err error) {
 	// Ensure URL & path are not both specified.
 	if c.URL != "" && c.Path != "" {
 		return nil, fmt.Errorf("cannot specify url & path for file replica")
@@ -317,24 +425,13 @@ func newFileReplicaFromConfig(c *ReplicaConfig, db *litestream.DB) (_ *litestrea
 	}
 
 	// Instantiate replica and apply time fields, if set.
-	r := litestream.NewFileReplica(db, c.Name, path)
+	client := file.NewReplicaClient(path)
-	if v := c.Retention; v > 0 {
+	client.Replica = r
-		r.Retention = v
+	return client, nil
-	}
-	if v := c.RetentionCheckInterval; v > 0 {
-		r.RetentionCheckInterval = v
-	}
-	if v := c.SnapshotInterval; v > 0 {
-		r.SnapshotInterval = v
-	}
-	if v := c.ValidationInterval; v > 0 {
-		r.ValidationInterval = v
-	}
-	return r, nil
 }
 
-// newS3ReplicaFromConfig returns a new instance of S3Replica build from config.
+// newS3ReplicaClientFromConfig returns a new instance of s3.ReplicaClient built from config.
-func newS3ReplicaFromConfig(c *ReplicaConfig, db *litestream.DB) (_ *s3.Replica, err error) {
+func newS3ReplicaClientFromConfig(c *ReplicaConfig, r *litestream.Replica) (_ *s3.ReplicaClient, err error) {
 	// Ensure URL & constituent parts are not both specified.
 	if c.URL != "" && c.Path != "" {
 		return nil, fmt.Errorf("cannot specify url & path for s3 replica")
@@ -343,7 +440,7 @@ func newS3ReplicaFromConfig(c *ReplicaConfig, db *litestream.DB) (_ *s3.Replica,
 	}
 
 	bucket, path := c.Bucket, c.Path
-	region, endpoint := c.Region, c.Endpoint
+	region, endpoint, skipVerify := c.Region, c.Endpoint, c.SkipVerify
 
 	// Use path style if an endpoint is explicitly set. This works because the
 	// only service to not use path style is AWS which does not use an endpoint.
@@ -384,31 +481,164 @@ func newS3ReplicaFromConfig(c *ReplicaConfig, db *litestream.DB) (_ *s3.Replica,
 	}
 
 	// Build replica.
-	r := s3.NewReplica(db, c.Name)
+	client := s3.NewReplicaClient()
-	r.AccessKeyID = c.AccessKeyID
+	client.AccessKeyID = c.AccessKeyID
-	r.SecretAccessKey = c.SecretAccessKey
+	client.SecretAccessKey = c.SecretAccessKey
-	r.Bucket = bucket
+	client.Bucket = bucket
-	r.Path = path
+	client.Path = path
-	r.Region = region
+	client.Region = region
-	r.Endpoint = endpoint
+	client.Endpoint = endpoint
-	r.ForcePathStyle = forcePathStyle
+	client.ForcePathStyle = forcePathStyle
+	client.SkipVerify = skipVerify
+	return client, nil
+}
 
-	if v := c.Retention; v > 0 {
+// newGCSReplicaClientFromConfig returns a new instance of gcs.ReplicaClient built from config.
-		r.Retention = v
+func newGCSReplicaClientFromConfig(c *ReplicaConfig, r *litestream.Replica) (_ *gcs.ReplicaClient, err error) {
+	// Ensure URL & constituent parts are not both specified.
+	if c.URL != "" && c.Path != "" {
+		return nil, fmt.Errorf("cannot specify url & path for gcs replica")
+	} else if c.URL != "" && c.Bucket != "" {
+		return nil, fmt.Errorf("cannot specify url & bucket for gcs replica")
 	}
-	if v := c.RetentionCheckInterval; v > 0 {
+
-		r.RetentionCheckInterval = v
+	bucket, path := c.Bucket, c.Path
+
+	// Apply settings from URL, if specified.
+	if c.URL != "" {
+		_, uhost, upath, err := ParseReplicaURL(c.URL)
+		if err != nil {
+			return nil, err
+		}
+
+		// Only apply URL parts to field that have not been overridden.
+		if path == "" {
+			path = upath
+		}
+		if bucket == "" {
+			bucket = uhost
+		}
 	}
-	if v := c.SyncInterval; v > 0 {
+
-		r.SyncInterval = v
+	// Ensure required settings are set.
+	if bucket == "" {
+		return nil, fmt.Errorf("bucket required for gcs replica")
 	}
-	if v := c.SnapshotInterval; v > 0 {
+
-		r.SnapshotInterval = v
+	// Build replica.
+	client := gcs.NewReplicaClient()
+	client.Bucket = bucket
+	client.Path = path
+	return client, nil
+}
+
+// newABSReplicaClientFromConfig returns a new instance of abs.ReplicaClient built from config.
+func newABSReplicaClientFromConfig(c *ReplicaConfig, r *litestream.Replica) (_ *abs.ReplicaClient, err error) {
+	// Ensure URL & constituent parts are not both specified.
+	if c.URL != "" && c.Path != "" {
+		return nil, fmt.Errorf("cannot specify url & path for abs replica")
+	} else if c.URL != "" && c.Bucket != "" {
+		return nil, fmt.Errorf("cannot specify url & bucket for abs replica")
 	}
-	if v := c.ValidationInterval; v > 0 {
+
-		r.ValidationInterval = v
+	// Build replica.
+	client := abs.NewReplicaClient()
+	client.AccountName = c.AccountName
+	client.AccountKey = c.AccountKey
+	client.Bucket = c.Bucket
+	client.Path = c.Path
+	client.Endpoint = c.Endpoint
+
+	// Apply settings from URL, if specified.
+	if c.URL != "" {
+		u, err := url.Parse(c.URL)
+		if err != nil {
+			return nil, err
+		}
+
+		if client.AccountName == "" && u.User != nil {
+			client.AccountName = u.User.Username()
+		}
+		if client.Bucket == "" {
+			client.Bucket = u.Host
+		}
+		if client.Path == "" {
+			client.Path = strings.TrimPrefix(path.Clean(u.Path), "/")
+		}
+	}
+
+	// Ensure required settings are set.
+	if client.Bucket == "" {
+		return nil, fmt.Errorf("bucket required for abs replica")
+	}
+
+	return client, nil
+}
+
+// newSFTPReplicaClientFromConfig returns a new instance of sftp.ReplicaClient built from config.
+func newSFTPReplicaClientFromConfig(c *ReplicaConfig, r *litestream.Replica) (_ *sftp.ReplicaClient, err error) {
+	// Ensure URL & constituent parts are not both specified.
+	if c.URL != "" && c.Path != "" {
+		return nil, fmt.Errorf("cannot specify url & path for sftp replica")
+	} else if c.URL != "" && c.Host != "" {
+		return nil, fmt.Errorf("cannot specify url & host for sftp replica")
+	}
+
+	host, user, password, path := c.Host, c.User, c.Password, c.Path
+
+	// Apply settings from URL, if specified.
+	if c.URL != "" {
+		u, err := url.Parse(c.URL)
+		if err != nil {
+			return nil, err
+		}
+
+		// Only apply URL parts to field that have not been overridden.
+		if host == "" {
+			host = u.Host
+		}
+		if user == "" && u.User != nil {
+			user = u.User.Username()
+		}
+		if password == "" && u.User != nil {
+			password, _ = u.User.Password()
+		}
+		if path == "" {
+			path = u.Path
+		}
+	}
+
+	// Ensure required settings are set.
+	if host == "" {
+		return nil, fmt.Errorf("host required for sftp replica")
+	} else if user == "" {
+		return nil, fmt.Errorf("user required for sftp replica")
+	}
+
+	// Build replica.
+	client := sftp.NewReplicaClient()
+	client.Host = host
+	client.User = user
+	client.Password = password
+	client.Path = path
+	client.KeyPath = c.KeyPath
+	return client, nil
+}
+
+// applyLitestreamEnv copies "LITESTREAM" prefixed environment variables to
+// their AWS counterparts as the "AWS" prefix can be confusing when using a
+// non-AWS S3-compatible service.
+func applyLitestreamEnv() {
+	if v, ok := os.LookupEnv("LITESTREAM_ACCESS_KEY_ID"); ok {
+		if _, ok := os.LookupEnv("AWS_ACCESS_KEY_ID"); !ok {
+			os.Setenv("AWS_ACCESS_KEY_ID", v)
+		}
+	}
+	if v, ok := os.LookupEnv("LITESTREAM_SECRET_ACCESS_KEY"); ok {
+		if _, ok := os.LookupEnv("AWS_SECRET_ACCESS_KEY"); !ok {
+			os.Setenv("AWS_SECRET_ACCESS_KEY", v)
+		}
 	}
-	return r, nil
 }
 
 // ParseReplicaURL parses a replica URL.
@@ -455,8 +685,9 @@ func DefaultConfigPath() string {
 	return defaultConfigPath
 }
 
-func registerConfigFlag(fs *flag.FlagSet) *string {
+func registerConfigFlag(fs *flag.FlagSet) (configPath *string, noExpandEnv *bool) {
-	return fs.String("config", "", "config path")
+	return fs.String("config", "", "config path"),
+		fs.Bool("no-expand-env", false, "do not expand env vars in config")
 }
 
 // expand returns an absolute path for s.
@@ -481,3 +712,24 @@ func expand(s string) (string, error) {
 	}
 	return filepath.Join(u.HomeDir, strings.TrimPrefix(s, prefix)), nil
 }
+
+// indexVar allows the flag package to parse index flags as 4-byte hexadecimal values.
+type indexVar int
+
+// Ensure type implements interface.
+var _ flag.Value = (*indexVar)(nil)
+
+// String returns an 8-character hexadecimal value.
+func (v *indexVar) String() string {
+	return fmt.Sprintf("%08x", int(*v))
+}
+
+// Set parses s into an integer from a hexadecimal value.
+func (v *indexVar) Set(s string) error {
+	i, err := strconv.ParseInt(s, 16, 32)
+	if err != nil {
+		return fmt.Errorf("invalid hexadecimal format")
+	}
+	*v = indexVar(i)
+	return nil
+}

cmd/litestream/main_notwindows.go

@@ -4,6 +4,9 @@ package main
 
 import (
 	"context"
+	"os"
+	"os/signal"
+	"syscall"
 )
 
 const defaultConfigPath = "/etc/litestream.yml"
@@ -15,3 +18,9 @@ func isWindowsService() (bool, error) {
 func runWindowsService(ctx context.Context) error {
 	panic("cannot run windows service as unix process")
 }
+
+func signalChan() <-chan os.Signal {
+	ch := make(chan os.Signal, 2)
+	signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM)
+	return ch
+}

cmd/litestream/main_test.go

@@ -2,11 +2,13 @@ package main_test
 
 import (
 	"io/ioutil"
+	"os"
 	"path/filepath"
 	"testing"
 
-	"github.com/benbjohnson/litestream"
 	main "github.com/benbjohnson/litestream/cmd/litestream"
+	"github.com/benbjohnson/litestream/file"
+	"github.com/benbjohnson/litestream/gcs"
 	"github.com/benbjohnson/litestream/s3"
 )
 
@@ -26,7 +28,7 @@ dbs:
 			t.Fatal(err)
 		}
 
-		config, err := main.ReadConfigFile(filename)
+		config, err := main.ReadConfigFile(filename, true)
 		if err != nil {
 			t.Fatal(err)
 		} else if got, want := config.AccessKeyID, `XXX`; got != want {
@@ -39,15 +41,65 @@ dbs:
 			t.Fatalf("Replica.SecretAccessKey=%v, want %v", got, want)
 		}
 	})
+
+	// Ensure environment variables are expanded.
+	t.Run("ExpandEnv", func(t *testing.T) {
+		os.Setenv("LITESTREAM_TEST_0129380", "/path/to/db")
+		os.Setenv("LITESTREAM_TEST_1872363", "s3://foo/bar")
+
+		filename := filepath.Join(t.TempDir(), "litestream.yml")
+		if err := ioutil.WriteFile(filename, []byte(`
+dbs:
+  - path: $LITESTREAM_TEST_0129380
+    replicas:
+      - url: ${LITESTREAM_TEST_1872363}
+      - url: ${LITESTREAM_TEST_NO_SUCH_ENV}
+`[1:]), 0666); err != nil {
+			t.Fatal(err)
+		}
+
+		config, err := main.ReadConfigFile(filename, true)
+		if err != nil {
+			t.Fatal(err)
+		} else if got, want := config.DBs[0].Path, `/path/to/db`; got != want {
+			t.Fatalf("DB.Path=%v, want %v", got, want)
+		} else if got, want := config.DBs[0].Replicas[0].URL, `s3://foo/bar`; got != want {
+			t.Fatalf("Replica[0].URL=%v, want %v", got, want)
+		} else if got, want := config.DBs[0].Replicas[1].URL, ``; got != want {
+			t.Fatalf("Replica[1].URL=%v, want %v", got, want)
+		}
+	})
+
+	// Ensure environment variables are not expanded.
+	t.Run("NoExpandEnv", func(t *testing.T) {
+		os.Setenv("LITESTREAM_TEST_9847533", "s3://foo/bar")
+
+		filename := filepath.Join(t.TempDir(), "litestream.yml")
+		if err := ioutil.WriteFile(filename, []byte(`
+dbs:
+  - path: /path/to/db
+    replicas:
+      - url: ${LITESTREAM_TEST_9847533}
+`[1:]), 0666); err != nil {
+			t.Fatal(err)
+		}
+
+		config, err := main.ReadConfigFile(filename, false)
+		if err != nil {
+			t.Fatal(err)
+		} else if got, want := config.DBs[0].Replicas[0].URL, `${LITESTREAM_TEST_9847533}`; got != want {
+			t.Fatalf("Replica.URL=%v, want %v", got, want)
+		}
+	})
 }
 
 func TestNewFileReplicaFromConfig(t *testing.T) {
 	r, err := main.NewReplicaFromConfig(&main.ReplicaConfig{Path: "/foo"}, nil)
 	if err != nil {
 		t.Fatal(err)
-	} else if r, ok := r.(*litestream.FileReplica); !ok {
+	} else if client, ok := r.Client.(*file.ReplicaClient); !ok {
 		t.Fatal("unexpected replica type")
-	} else if got, want := r.Path(), "/foo"; got != want {
+	} else if got, want := client.Path(), "/foo"; got != want {
 		t.Fatalf("Path=%s, want %s", got, want)
 	}
 }
@@ -57,17 +109,17 @@ func TestNewS3ReplicaFromConfig(t *testing.T) {
 		r, err := main.NewReplicaFromConfig(&main.ReplicaConfig{URL: "s3://foo/bar"}, nil)
 		if err != nil {
 			t.Fatal(err)
-		} else if r, ok := r.(*s3.Replica); !ok {
+		} else if client, ok := r.Client.(*s3.ReplicaClient); !ok {
 			t.Fatal("unexpected replica type")
-		} else if got, want := r.Bucket, "foo"; got != want {
+		} else if got, want := client.Bucket, "foo"; got != want {
 			t.Fatalf("Bucket=%s, want %s", got, want)
-		} else if got, want := r.Path, "bar"; got != want {
+		} else if got, want := client.Path, "bar"; got != want {
 			t.Fatalf("Path=%s, want %s", got, want)
-		} else if got, want := r.Region, ""; got != want {
+		} else if got, want := client.Region, ""; got != want {
 			t.Fatalf("Region=%s, want %s", got, want)
-		} else if got, want := r.Endpoint, ""; got != want {
+		} else if got, want := client.Endpoint, ""; got != want {
 			t.Fatalf("Endpoint=%s, want %s", got, want)
-		} else if got, want := r.ForcePathStyle, false; got != want {
+		} else if got, want := client.ForcePathStyle, false; got != want {
 			t.Fatalf("ForcePathStyle=%v, want %v", got, want)
 		}
 	})
@@ -76,17 +128,17 @@ func TestNewS3ReplicaFromConfig(t *testing.T) {
 		r, err := main.NewReplicaFromConfig(&main.ReplicaConfig{URL: "s3://foo.localhost:9000/bar"}, nil)
 		if err != nil {
 			t.Fatal(err)
-		} else if r, ok := r.(*s3.Replica); !ok {
+		} else if client, ok := r.Client.(*s3.ReplicaClient); !ok {
 			t.Fatal("unexpected replica type")
-		} else if got, want := r.Bucket, "foo"; got != want {
+		} else if got, want := client.Bucket, "foo"; got != want {
 			t.Fatalf("Bucket=%s, want %s", got, want)
-		} else if got, want := r.Path, "bar"; got != want {
+		} else if got, want := client.Path, "bar"; got != want {
 			t.Fatalf("Path=%s, want %s", got, want)
-		} else if got, want := r.Region, "us-east-1"; got != want {
+		} else if got, want := client.Region, "us-east-1"; got != want {
 			t.Fatalf("Region=%s, want %s", got, want)
-		} else if got, want := r.Endpoint, "http://localhost:9000"; got != want {
+		} else if got, want := client.Endpoint, "http://localhost:9000"; got != want {
 			t.Fatalf("Endpoint=%s, want %s", got, want)
-		} else if got, want := r.ForcePathStyle, true; got != want {
+		} else if got, want := client.ForcePathStyle, true; got != want {
 			t.Fatalf("ForcePathStyle=%v, want %v", got, want)
 		}
 	})
@@ -95,37 +147,31 @@ func TestNewS3ReplicaFromConfig(t *testing.T) {
 		r, err := main.NewReplicaFromConfig(&main.ReplicaConfig{URL: "s3://foo.s3.us-west-000.backblazeb2.com/bar"}, nil)
 		if err != nil {
 			t.Fatal(err)
-		} else if r, ok := r.(*s3.Replica); !ok {
+		} else if client, ok := r.Client.(*s3.ReplicaClient); !ok {
 			t.Fatal("unexpected replica type")
-		} else if got, want := r.Bucket, "foo"; got != want {
+		} else if got, want := client.Bucket, "foo"; got != want {
 			t.Fatalf("Bucket=%s, want %s", got, want)
-		} else if got, want := r.Path, "bar"; got != want {
+		} else if got, want := client.Path, "bar"; got != want {
 			t.Fatalf("Path=%s, want %s", got, want)
-		} else if got, want := r.Region, "us-west-000"; got != want {
+		} else if got, want := client.Region, "us-west-000"; got != want {
 			t.Fatalf("Region=%s, want %s", got, want)
-		} else if got, want := r.Endpoint, "https://s3.us-west-000.backblazeb2.com"; got != want {
+		} else if got, want := client.Endpoint, "https://s3.us-west-000.backblazeb2.com"; got != want {
 			t.Fatalf("Endpoint=%s, want %s", got, want)
-		} else if got, want := r.ForcePathStyle, true; got != want {
+		} else if got, want := client.ForcePathStyle, true; got != want {
-			t.Fatalf("ForcePathStyle=%v, want %v", got, want)
-		}
-	})
-
-	t.Run("GCS", func(t *testing.T) {
-		r, err := main.NewReplicaFromConfig(&main.ReplicaConfig{URL: "s3://foo.storage.googleapis.com/bar"}, nil)
-		if err != nil {
-			t.Fatal(err)
-		} else if r, ok := r.(*s3.Replica); !ok {
-			t.Fatal("unexpected replica type")
-		} else if got, want := r.Bucket, "foo"; got != want {
-			t.Fatalf("Bucket=%s, want %s", got, want)
-		} else if got, want := r.Path, "bar"; got != want {
-			t.Fatalf("Path=%s, want %s", got, want)
-		} else if got, want := r.Region, "us-east-1"; got != want {
-			t.Fatalf("Region=%s, want %s", got, want)
-		} else if got, want := r.Endpoint, "https://storage.googleapis.com"; got != want {
-			t.Fatalf("Endpoint=%s, want %s", got, want)
-		} else if got, want := r.ForcePathStyle, true; got != want {
 			t.Fatalf("ForcePathStyle=%v, want %v", got, want)
 		}
 	})
 }
+
+func TestNewGCSReplicaFromConfig(t *testing.T) {
+	r, err := main.NewReplicaFromConfig(&main.ReplicaConfig{URL: "gcs://foo/bar"}, nil)
+	if err != nil {
+		t.Fatal(err)
+	} else if client, ok := r.Client.(*gcs.ReplicaClient); !ok {
+		t.Fatal("unexpected replica type")
+	} else if got, want := client.Bucket, "foo"; got != want {
+		t.Fatalf("Bucket=%s, want %s", got, want)
+	} else if got, want := client.Path, "bar"; got != want {
+		t.Fatalf("Path=%s, want %s", got, want)
+	}
+}

cmd/litestream/main_windows.go

@@ -7,6 +7,7 @@ import (
 	"io"
 	"log"
 	"os"
+	"os/signal"
 
 	"golang.org/x/sys/windows"
 	"golang.org/x/sys/windows/svc"
@@ -61,7 +62,7 @@ func (s *windowsService) Execute(args []string, r <-chan svc.ChangeRequest, stat
 
 	// Instantiate replication command and load configuration.
 	c := NewReplicateCommand()
-	if c.Config, err = ReadConfigFile(DefaultConfigPath()); err != nil {
+	if c.Config, err = ReadConfigFile(DefaultConfigPath(), true); err != nil {
 		log.Printf("cannot load configuration: %s", err)
 		return true, 1
 	}
@@ -103,3 +104,9 @@ func (w *eventlogWriter) Write(p []byte) (n int, err error) {
 	elog := (*eventlog.Log)(w)
 	return 0, elog.Info(1, string(p))
 }
+
+func signalChan() <-chan os.Signal {
+	ch := make(chan os.Signal, 1)
+	signal.Notify(ch, os.Interrupt)
+	return ch
+}

cmd/litestream/replicate.go

@@ -9,15 +9,23 @@ import (
 	"net/http"
 	_ "net/http/pprof"
 	"os"
-	"time"
+	"os/exec"
 
 	"github.com/benbjohnson/litestream"
+	"github.com/benbjohnson/litestream/abs"
+	"github.com/benbjohnson/litestream/file"
+	"github.com/benbjohnson/litestream/gcs"
 	"github.com/benbjohnson/litestream/s3"
+	"github.com/benbjohnson/litestream/sftp"
+	"github.com/mattn/go-shellwords"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
 // ReplicateCommand represents a command that continuously replicates SQLite databases.
 type ReplicateCommand struct {
+	cmd    *exec.Cmd  // subcommand
+	execCh chan error // subcommand error channel
+
 	Config Config
 
 	// List of managed databases specified in the config.
@@ -25,14 +33,17 @@ type ReplicateCommand struct {
 }
 
 func NewReplicateCommand() *ReplicateCommand {
-	return &ReplicateCommand{}
+	return &ReplicateCommand{
+		execCh: make(chan error),
+	}
 }
 
 // ParseFlags parses the CLI flags and loads the configuration file.
 func (c *ReplicateCommand) ParseFlags(ctx context.Context, args []string) (err error) {
 	fs := flag.NewFlagSet("litestream-replicate", flag.ContinueOnError)
+	execFlag := fs.String("exec", "", "execute subcommand")
 	tracePath := fs.String("trace", "", "trace path")
-	configPath := registerConfigFlag(fs)
+	configPath, noExpandEnv := registerConfigFlag(fs)
 	fs.Usage = c.Usage
 	if err := fs.Parse(args); err != nil {
 		return err
@@ -48,9 +59,10 @@ func (c *ReplicateCommand) ParseFlags(ctx context.Context, args []string) (err e
 
 		dbConfig := &DBConfig{Path: fs.Arg(0)}
 		for _, u := range fs.Args()[1:] {
+			syncInterval := litestream.DefaultSyncInterval
 			dbConfig.Replicas = append(dbConfig.Replicas, &ReplicaConfig{
 				URL:          u,
-				SyncInterval: 1 * time.Second,
+				SyncInterval: &syncInterval,
 			})
 		}
 		c.Config.DBs = []*DBConfig{dbConfig}
@@ -58,11 +70,16 @@ func (c *ReplicateCommand) ParseFlags(ctx context.Context, args []string) (err e
 		if *configPath == "" {
 			*configPath = DefaultConfigPath()
 		}
-		if c.Config, err = ReadConfigFile(*configPath); err != nil {
+		if c.Config, err = ReadConfigFile(*configPath, !*noExpandEnv); err != nil {
 			return err
 		}
 	}
 
+	// Override config exec command, if specified.
+	if *execFlag != "" {
+		c.Config.Exec = *execFlag
+	}
+
 	// Enable trace logging.
 	if *tracePath != "" {
 		f, err := os.Create(*tracePath)
@@ -70,17 +87,18 @@ func (c *ReplicateCommand) ParseFlags(ctx context.Context, args []string) (err e
 			return err
 		}
 		defer f.Close()
-		litestream.Tracef = log.New(f, "", log.LstdFlags|log.LUTC|log.Lshortfile).Printf
+		litestream.Tracef = log.New(f, "", log.LstdFlags|log.Lmicroseconds|log.LUTC|log.Lshortfile).Printf
 	}
 
 	return nil
 }
 
 // Run loads all databases specified in the configuration.
-func (c *ReplicateCommand) Run(ctx context.Context) (err error) {
+func (c *ReplicateCommand) Run() (err error) {
 	// Display version information.
 	log.Printf("litestream %s", Version)
 
+	// Setup databases.
 	if len(c.Config.DBs) == 0 {
 		log.Println("no databases specified in configuration")
 	}
@@ -102,13 +120,19 @@ func (c *ReplicateCommand) Run(ctx context.Context) (err error) {
 	for _, db := range c.DBs {
 		log.Printf("initialized db: %s", db.Path())
 		for _, r := range db.Replicas {
-			switch r := r.(type) {
+			switch client := r.Client.(type) {
-			case *litestream.FileReplica:
+			case *file.ReplicaClient:
-				log.Printf("replicating to: name=%q type=%q path=%q", r.Name(), r.Type(), r.Path())
+				log.Printf("replicating to: name=%q type=%q path=%q", r.Name(), client.Type(), client.Path())
-			case *s3.Replica:
+			case *s3.ReplicaClient:
-				log.Printf("replicating to: name=%q type=%q bucket=%q path=%q region=%q endpoint=%q sync-interval=%s", r.Name(), r.Type(), r.Bucket, r.Path, r.Region, r.Endpoint, r.SyncInterval)
+				log.Printf("replicating to: name=%q type=%q bucket=%q path=%q region=%q endpoint=%q sync-interval=%s", r.Name(), client.Type(), client.Bucket, client.Path, client.Region, client.Endpoint, r.SyncInterval)
+			case *gcs.ReplicaClient:
+				log.Printf("replicating to: name=%q type=%q bucket=%q path=%q sync-interval=%s", r.Name(), client.Type(), client.Bucket, client.Path, r.SyncInterval)
+			case *abs.ReplicaClient:
+				log.Printf("replicating to: name=%q type=%q bucket=%q path=%q endpoint=%q sync-interval=%s", r.Name(), client.Type(), client.Bucket, client.Path, client.Endpoint, r.SyncInterval)
+			case *sftp.ReplicaClient:
+				log.Printf("replicating to: name=%q type=%q host=%q user=%q path=%q sync-interval=%s", r.Name(), client.Type(), client.Host, client.User, client.Path, r.SyncInterval)
 			default:
-				log.Printf("replicating to: name=%q type=%q", r.Name(), r.Type())
+				log.Printf("replicating to: name=%q type=%q", r.Name(), client.Type())
 			}
 		}
 	}
@@ -131,20 +155,36 @@ func (c *ReplicateCommand) Run(ctx context.Context) (err error) {
 		}()
 	}
 
+	// Parse exec commands args & start subprocess.
+	if c.Config.Exec != "" {
+		execArgs, err := shellwords.Parse(c.Config.Exec)
+		if err != nil {
+			return fmt.Errorf("cannot parse exec command: %w", err)
+		}
+
+		c.cmd = exec.Command(execArgs[0], execArgs[1:]...)
+		c.cmd.Env = os.Environ()
+		c.cmd.Stdout = os.Stdout
+		c.cmd.Stderr = os.Stderr
+		if err := c.cmd.Start(); err != nil {
+			return fmt.Errorf("cannot start exec command: %w", err)
+		}
+		go func() { c.execCh <- c.cmd.Wait() }()
+	}
+
 	return nil
 }
 
 // Close closes all open databases.
 func (c *ReplicateCommand) Close() (err error) {
 	for _, db := range c.DBs {
-		if e := db.SoftClose(); e != nil {
+		if e := db.Close(); e != nil {
 			log.Printf("error closing db: path=%s err=%s", db.Path(), e)
 			if err == nil {
 				err = e
 			}
 		}
 	}
-	// TODO(windows): Clear DBs
 	return err
 }
 
@@ -168,6 +208,13 @@ Arguments:
 	    Specifies the configuration file.
 	    Defaults to %s
 
+	-exec CMD
+	    Executes a subcommand. Litestream will exit when the child
+	    process exits. Useful for simple process management.
+
+	-no-expand-env
+	    Disables environment variable expansion in configuration file.
+
 	-trace PATH
 	    Write verbose trace logging to PATH.
 

cmd/litestream/restore.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"log"
 	"os"
+	"strconv"
 	"time"
 
 	"github.com/benbjohnson/litestream"
@@ -21,12 +22,14 @@ func (c *RestoreCommand) Run(ctx context.Context, args []string) (err error) {
 	opt.Verbose = true
 
 	fs := flag.NewFlagSet("litestream-restore", flag.ContinueOnError)
-	configPath := registerConfigFlag(fs)
+	configPath, noExpandEnv := registerConfigFlag(fs)
 	fs.StringVar(&opt.OutputPath, "o", "", "output path")
 	fs.StringVar(&opt.ReplicaName, "replica", "", "replica name")
 	fs.StringVar(&opt.Generation, "generation", "", "generation name")
-	fs.IntVar(&opt.Index, "index", opt.Index, "wal index")
+	fs.Var((*indexVar)(&opt.Index), "index", "wal index")
-	fs.BoolVar(&opt.DryRun, "dry-run", false, "dry run")
+	fs.IntVar(&opt.Parallelism, "parallelism", opt.Parallelism, "parallelism")
+	ifDBNotExists := fs.Bool("if-db-not-exists", false, "")
+	ifReplicaExists := fs.Bool("if-replica-exists", false, "")
 	timestampStr := fs.String("timestamp", "", "timestamp")
 	verbose := fs.Bool("v", false, "verbose output")
 	fs.Usage = c.Usage
@@ -45,56 +48,75 @@ func (c *RestoreCommand) Run(ctx context.Context, args []string) (err error) {
 		}
 	}
 
-	// Verbose output is automatically enabled if dry run is specified.
-	if opt.DryRun {
-		*verbose = true
-	}
-
 	// Instantiate logger if verbose output is enabled.
 	if *verbose {
-		opt.Logger = log.New(os.Stderr, "", log.LstdFlags)
+		opt.Logger = log.New(os.Stderr, "", log.LstdFlags|log.Lmicroseconds)
 	}
 
 	// Determine replica & generation to restore from.
-	var r litestream.Replica
+	var r *litestream.Replica
 	if isURL(fs.Arg(0)) {
 		if *configPath != "" {
 			return fmt.Errorf("cannot specify a replica URL and the -config flag")
 		}
-		if r, err = c.loadFromURL(ctx, fs.Arg(0), &opt); err != nil {
+		if r, err = c.loadFromURL(ctx, fs.Arg(0), *ifDBNotExists, &opt); err == errSkipDBExists {
+			fmt.Println("database already exists, skipping")
+			return nil
+		} else if err != nil {
 			return err
 		}
 	} else {
 		if *configPath == "" {
 			*configPath = DefaultConfigPath()
 		}
-		if r, err = c.loadFromConfig(ctx, fs.Arg(0), *configPath, &opt); err != nil {
+		if r, err = c.loadFromConfig(ctx, fs.Arg(0), *configPath, !*noExpandEnv, *ifDBNotExists, &opt); err == errSkipDBExists {
+			fmt.Println("database already exists, skipping")
+			return nil
+		} else if err != nil {
 			return err
 		}
 	}
 
 	// Return an error if no matching targets found.
+	// If optional flag set, return success. Useful for automated recovery.
 	if opt.Generation == "" {
+		if *ifReplicaExists {
+			fmt.Println("no matching backups found")
+			return nil
+		}
 		return fmt.Errorf("no matching backups found")
 	}
 
-	return litestream.RestoreReplica(ctx, r, opt)
+	return r.Restore(ctx, opt)
 }
 
 // loadFromURL creates a replica & updates the restore options from a replica URL.
-func (c *RestoreCommand) loadFromURL(ctx context.Context, replicaURL string, opt *litestream.RestoreOptions) (litestream.Replica, error) {
+func (c *RestoreCommand) loadFromURL(ctx context.Context, replicaURL string, ifDBNotExists bool, opt *litestream.RestoreOptions) (*litestream.Replica, error) {
-	r, err := NewReplicaFromConfig(&ReplicaConfig{URL: replicaURL}, nil)
+	if opt.OutputPath == "" {
+		return nil, fmt.Errorf("output path required")
+	}
+
+	// Exit successfully if the output file already exists.
+	if _, err := os.Stat(opt.OutputPath); !os.IsNotExist(err) && ifDBNotExists {
+		return nil, errSkipDBExists
+	}
+
+	syncInterval := litestream.DefaultSyncInterval
+	r, err := NewReplicaFromConfig(&ReplicaConfig{
+		URL:          replicaURL,
+		SyncInterval: &syncInterval,
+	}, nil)
 	if err != nil {
 		return nil, err
 	}
-	opt.Generation, _, err = litestream.CalcReplicaRestoreTarget(ctx, r, *opt)
+	opt.Generation, _, err = r.CalcRestoreTarget(ctx, *opt)
 	return r, err
 }
 
 // loadFromConfig returns a replica & updates the restore options from a DB reference.
-func (c *RestoreCommand) loadFromConfig(ctx context.Context, dbPath, configPath string, opt *litestream.RestoreOptions) (litestream.Replica, error) {
+func (c *RestoreCommand) loadFromConfig(ctx context.Context, dbPath, configPath string, expandEnv, ifDBNotExists bool, opt *litestream.RestoreOptions) (*litestream.Replica, error) {
 	// Load configuration.
-	config, err := ReadConfigFile(configPath)
+	config, err := ReadConfigFile(configPath, expandEnv)
 	if err != nil {
 		return nil, err
 	}
@@ -117,6 +139,11 @@ func (c *RestoreCommand) loadFromConfig(ctx context.Context, dbPath, configPath
 		opt.OutputPath = dbPath
 	}
 
+	// Exit successfully if the output file already exists.
+	if _, err := os.Stat(opt.OutputPath); !os.IsNotExist(err) && ifDBNotExists {
+		return nil, errSkipDBExists
+	}
+
 	// Determine the appropriate replica & generation to restore from,
 	r, generation, err := db.CalcRestoreTarget(ctx, *opt)
 	if err != nil {
@@ -144,6 +171,9 @@ Arguments:
 	    Specifies the configuration file.
 	    Defaults to %s
 
+	-no-expand-env
+	    Disables environment variable expansion in configuration file.
+
 	-replica NAME
 	    Restore from a specific replica.
 	    Defaults to replica with latest data.
@@ -153,7 +183,7 @@ Arguments:
 	    Defaults to generation with latest data.
 
 	-index NUM
-	    Restore up to a specific WAL index (inclusive).
+	    Restore up to a specific hex-encoded WAL index (inclusive).
 	    Defaults to use the highest available index.
 
 	-timestamp TIMESTAMP
@@ -164,9 +194,15 @@ Arguments:
 	    Output path of the restored database.
 	    Defaults to original DB path.
 
-	-dry-run
+	-if-db-not-exists
-	    Prints all log output as if it were running but does
+	    Returns exit code of 0 if the database already exists.
-	    not perform actual restore.
+
+	-if-replica-exists
+	    Returns exit code of 0 if no backups found.
+
+	-parallelism NUM
+	    Determines the number of WAL files downloaded in parallel.
+	    Defaults to `+strconv.Itoa(litestream.DefaultRestoreParallelism)+`.
 
 	-v
 	    Verbose output.
@@ -193,3 +229,5 @@ Examples:
 		DefaultConfigPath(),
 	)
 }
+
+var errSkipDBExists = errors.New("database already exists, skipping")

cmd/litestream/snapshots.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"log"
 	"os"
 	"text/tabwriter"
 	"time"
@@ -17,7 +18,7 @@ type SnapshotsCommand struct{}
 // Run executes the command.
 func (c *SnapshotsCommand) Run(ctx context.Context, args []string) (err error) {
 	fs := flag.NewFlagSet("litestream-snapshots", flag.ContinueOnError)
-	configPath := registerConfigFlag(fs)
+	configPath, noExpandEnv := registerConfigFlag(fs)
 	replicaName := fs.String("replica", "", "replica name")
 	fs.Usage = c.Usage
 	if err := fs.Parse(args); err != nil {
@@ -29,7 +30,7 @@ func (c *SnapshotsCommand) Run(ctx context.Context, args []string) (err error) {
 	}
 
 	var db *litestream.DB
-	var r litestream.Replica
+	var r *litestream.Replica
 	if isURL(fs.Arg(0)) {
 		if *configPath != "" {
 			return fmt.Errorf("cannot specify a replica URL and the -config flag")
@@ -43,7 +44,7 @@ func (c *SnapshotsCommand) Run(ctx context.Context, args []string) (err error) {
 		}
 
 		// Load configuration.
-		config, err := ReadConfigFile(*configPath)
+		config, err := ReadConfigFile(*configPath, !*noExpandEnv)
 		if err != nil {
 			return err
 		}
@@ -66,15 +67,11 @@ func (c *SnapshotsCommand) Run(ctx context.Context, args []string) (err error) {
 	}
 
 	// Find snapshots by db or replica.
-	var infos []*litestream.SnapshotInfo
+	var replicas []*litestream.Replica
 	if r != nil {
-		if infos, err = r.Snapshots(ctx); err != nil {
+		replicas = []*litestream.Replica{r}
-			return err
-		}
 	} else {
-		if infos, err = db.Snapshots(ctx); err != nil {
+		replicas = db.Replicas
-			return err
-		}
 	}
 
 	// List all snapshots.
@@ -82,14 +79,21 @@ func (c *SnapshotsCommand) Run(ctx context.Context, args []string) (err error) {
 	defer w.Flush()
 
 	fmt.Fprintln(w, "replica\tgeneration\tindex\tsize\tcreated")
-	for _, info := range infos {
+	for _, r := range replicas {
-		fmt.Fprintf(w, "%s\t%s\t%d\t%d\t%s\n",
+		infos, err := r.Snapshots(ctx)
-			info.Replica,
+		if err != nil {
-			info.Generation,
+			log.Printf("cannot determine snapshots: %s", err)
-			info.Index,
+			continue
-			info.Size,
+		}
-			info.CreatedAt.Format(time.RFC3339),
+		for _, info := range infos {
-		)
+			fmt.Fprintf(w, "%s\t%s\t%d\t%d\t%s\n",
+				r.Name(),
+				info.Generation,
+				info.Index,
+				info.Size,
+				info.CreatedAt.Format(time.RFC3339),
+			)
+		}
 	}
 
 	return nil
@@ -112,6 +116,9 @@ Arguments:
 	    Specifies the configuration file.
 	    Defaults to %s
 
+	-no-expand-env
+	    Disables environment variable expansion in configuration file.
+
 	-replica NAME
 	    Optional, filter by a specific replica.
 

cmd/litestream/wal.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"log"
 	"os"
 	"text/tabwriter"
 	"time"
@@ -17,7 +18,7 @@ type WALCommand struct{}
 // Run executes the command.
 func (c *WALCommand) Run(ctx context.Context, args []string) (err error) {
 	fs := flag.NewFlagSet("litestream-wal", flag.ContinueOnError)
-	configPath := registerConfigFlag(fs)
+	configPath, noExpandEnv := registerConfigFlag(fs)
 	replicaName := fs.String("replica", "", "replica name")
 	generation := fs.String("generation", "", "generation name")
 	fs.Usage = c.Usage
@@ -30,7 +31,7 @@ func (c *WALCommand) Run(ctx context.Context, args []string) (err error) {
 	}
 
 	var db *litestream.DB
-	var r litestream.Replica
+	var r *litestream.Replica
 	if isURL(fs.Arg(0)) {
 		if *configPath != "" {
 			return fmt.Errorf("cannot specify a replica URL and the -config flag")
@@ -44,7 +45,7 @@ func (c *WALCommand) Run(ctx context.Context, args []string) (err error) {
 		}
 
 		// Load configuration.
-		config, err := ReadConfigFile(*configPath)
+		config, err := ReadConfigFile(*configPath, !*noExpandEnv)
 		if err != nil {
 			return err
 		}
@@ -67,15 +68,11 @@ func (c *WALCommand) Run(ctx context.Context, args []string) (err error) {
 	}
 
 	// Find WAL files by db or replica.
-	var infos []*litestream.WALInfo
+	var replicas []*litestream.Replica
 	if r != nil {
-		if infos, err = r.WALs(ctx); err != nil {
+		replicas = []*litestream.Replica{r}
-			return err
-		}
 	} else {
-		if infos, err = db.WALs(ctx); err != nil {
+		replicas = db.Replicas
-			return err
-		}
 	}
 
 	// List all WAL files.
@@ -83,19 +80,43 @@ func (c *WALCommand) Run(ctx context.Context, args []string) (err error) {
 	defer w.Flush()
 
 	fmt.Fprintln(w, "replica\tgeneration\tindex\toffset\tsize\tcreated")
-	for _, info := range infos {
+	for _, r := range replicas {
-		if *generation != "" && info.Generation != *generation {
+		var generations []string
-			continue
+		if *generation != "" {
+			generations = []string{*generation}
+		} else {
+			if generations, err = r.Client.Generations(ctx); err != nil {
+				log.Printf("%s: cannot determine generations: %s", r.Name(), err)
+				continue
+			}
 		}
 
-		fmt.Fprintf(w, "%s\t%s\t%d\t%d\t%d\t%s\n",
+		for _, generation := range generations {
-			info.Replica,
+			if err := func() error {
-			info.Generation,
+				itr, err := r.Client.WALSegments(ctx, generation)
-			info.Index,
+				if err != nil {
-			info.Offset,
+					return err
-			info.Size,
+				}
-			info.CreatedAt.Format(time.RFC3339),
+				defer itr.Close()
-		)
+
+				for itr.Next() {
+					info := itr.WALSegment()
+
+					fmt.Fprintf(w, "%s\t%s\t%d\t%d\t%d\t%s\n",
+						r.Name(),
+						info.Generation,
+						info.Index,
+						info.Offset,
+						info.Size,
+						info.CreatedAt.Format(time.RFC3339),
+					)
+				}
+				return itr.Close()
+			}(); err != nil {
+				log.Printf("%s: cannot fetch wal segments: %s", r.Name(), err)
+				continue
+			}
+		}
 	}
 
 	return nil
@@ -104,7 +125,7 @@ func (c *WALCommand) Run(ctx context.Context, args []string) (err error) {
 // Usage prints the help screen to STDOUT.
 func (c *WALCommand) Usage() {
 	fmt.Printf(`
-The wal command lists all wal files available for a database.
+The wal command lists all wal segments available for a database.
 
 Usage:
 
@@ -118,6 +139,9 @@ Arguments:
 	    Specifies the configuration file.
 	    Defaults to %s
 
+	-no-expand-env
+	    Disables environment variable expansion in configuration file.
+
 	-replica NAME
 	    Optional, filter by a specific replica.
 
@@ -126,13 +150,13 @@ Arguments:
 
 Examples:
 
-	# List all WAL files for a database.
+	# List all WAL segments for a database.
 	$ litestream wal /path/to/db
 
-	# List all WAL files on S3 for a specific generation.
+	# List all WAL segments on S3 for a specific generation.
 	$ litestream wal -replica s3 -generation xxxxxxxx /path/to/db
 
-	# List all WAL files for replica URL.
+	# List all WAL segments for replica URL.
 	$ litestream wal s3://mybkt/db
 
 `[1:],

db.go

@@ -16,11 +16,11 @@ import (
 	"math/rand"
 	"os"
 	"path/filepath"
-	"sort"
 	"strings"
 	"sync"
 	"time"
 
+	"github.com/benbjohnson/litestream/internal"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 )
@@ -50,10 +50,8 @@ type DB struct {
 	pageSize int           // page size, in bytes
 	notify   chan struct{} // closes on WAL change
 
-	uid, gid       int // db user/group obtained on init
+	fileInfo os.FileInfo // db info cached during init
-	mode           os.FileMode
+	dirInfo  os.FileInfo // parent dir info cached during init
-	diruid, dirgid int // db parent user/group obtained on init
-	dirmode        os.FileMode
 
 	ctx    context.Context
 	cancel func()
@@ -95,7 +93,10 @@ type DB struct {
 
 	// List of replicas for the database.
 	// Must be set before calling Open().
-	Replicas []Replica
+	Replicas []*Replica
+
+	// Where to send log messages, defaults to log.Default()
+	Logger *log.Logger
 }
 
 // NewDB returns a new instance of DB for a given path.
@@ -103,13 +104,13 @@ func NewDB(path string) *DB {
 	db := &DB{
 		path:   path,
 		notify: make(chan struct{}),
-		uid:    -1, gid: -1, mode: 0600,
-		diruid: -1, dirgid: -1, dirmode: 0700,
 
 		MinCheckpointPageN: DefaultMinCheckpointPageN,
 		MaxCheckpointPageN: DefaultMaxCheckpointPageN,
 		CheckpointInterval: DefaultCheckpointInterval,
 		MonitorInterval:    DefaultMonitorInterval,
+
+		Logger: log.Default(),
 	}
 
 	db.dbSizeGauge = dbSizeGaugeVec.WithLabelValues(db.path)
@@ -195,10 +196,7 @@ func (db *DB) CurrentShadowWALIndex(generation string) (index int, size int64, e
 
 	// Find highest wal index.
 	for _, fi := range fis {
-		if !strings.HasSuffix(fi.Name(), WALExt) {
+		if v, err := ParseWALPath(fi.Name()); err != nil {
-			continue
-		}
-		if v, _, _, err := ParseWALPath(fi.Name()); err != nil {
 			continue // invalid wal filename
 		} else if v > index {
 			index = v
@@ -209,8 +207,18 @@ func (db *DB) CurrentShadowWALIndex(generation string) (index int, size int64, e
 	return index, size, nil
 }
 
+// FileInfo returns the cached file stats for the database file when it was initialized.
+func (db *DB) FileInfo() os.FileInfo {
+	return db.fileInfo
+}
+
+// DirInfo returns the cached file stats for the parent directory of the database file when it was initialized.
+func (db *DB) DirInfo() os.FileInfo {
+	return db.dirInfo
+}
+
 // Replica returns a replica by name.
-func (db *DB) Replica(name string) Replica {
+func (db *DB) Replica(name string) *Replica {
 	for _, r := range db.Replicas {
 		if r.Name() == name {
 			return r
@@ -288,14 +296,33 @@ func (db *DB) Open() (err error) {
 	return nil
 }
 
-// Close releases the read lock & closes the database. This method should only
+// Close flushes outstanding WAL writes to replicas, releases the read lock,
-// be called by tests as it causes the underlying database to be checkpointed.
+// and closes the database.
 func (db *DB) Close() (err error) {
-	// Ensure replicas all stop replicating.
+	db.cancel()
+	db.wg.Wait()
+
+	// Start a new context for shutdown since we canceled the DB context.
+	ctx := context.Background()
+
+	// Perform a final db sync, if initialized.
+	if db.db != nil {
+		if e := db.Sync(ctx); e != nil && err == nil {
+			err = e
+		}
+	}
+
+	// Ensure replicas perform a final sync and stop replicating.
 	for _, r := range db.Replicas {
+		if db.db != nil {
+			if e := r.Sync(ctx); e != nil && err == nil {
+				err = e
+			}
+		}
 		r.Stop(true)
 	}
 
+	// Release the read lock to allow other applications to handle checkpointing.
 	if db.rtx != nil {
 		if e := db.releaseReadLock(); e != nil && err == nil {
 			err = e
@@ -331,42 +358,6 @@ func (db *DB) UpdatedAt() (time.Time, error) {
 	return t, nil
 }
 
-// Snapshots returns a list of all snapshots across all replicas.
-func (db *DB) Snapshots(ctx context.Context) ([]*SnapshotInfo, error) {
-	var infos []*SnapshotInfo
-	for _, r := range db.Replicas {
-		a, err := r.Snapshots(ctx)
-		if err != nil {
-			return nil, err
-		}
-		infos = append(infos, a...)
-	}
-
-	// Sort in order by time.
-	sort.Slice(infos, func(i, j int) bool {
-		return infos[i].CreatedAt.Before(infos[j].CreatedAt)
-	})
-	return infos, nil
-}
-
-// WALs returns a list of all WAL files across all replicas.
-func (db *DB) WALs(ctx context.Context) ([]*WALInfo, error) {
-	var infos []*WALInfo
-	for _, r := range db.Replicas {
-		a, err := r.WALs(ctx)
-		if err != nil {
-			return nil, err
-		}
-		infos = append(infos, a...)
-	}
-
-	// Sort in order by time.
-	sort.Slice(infos, func(i, j int) bool {
-		return infos[i].CreatedAt.Before(infos[j].CreatedAt)
-	})
-	return infos, nil
-}
-
 // init initializes the connection to the database.
 // Skipped if already initialized or if the database file does not exist.
 func (db *DB) init() (err error) {
@@ -382,21 +373,20 @@ func (db *DB) init() (err error) {
 	} else if err != nil {
 		return err
 	}
-	db.uid, db.gid = fileinfo(fi)
+	db.fileInfo = fi
-	db.mode = fi.Mode()
 
 	// Obtain permissions for parent directory.
 	if fi, err = os.Stat(filepath.Dir(db.path)); err != nil {
 		return err
 	}
-	db.diruid, db.dirgid = fileinfo(fi)
+	db.dirInfo = fi
-	db.dirmode = fi.Mode()
 
 	dsn := db.path
 	dsn += fmt.Sprintf("?_busy_timeout=%d", BusyTimeout.Milliseconds())
 
-	// Connect to SQLite database.
+	// Connect to SQLite database. Use the driver registered with a hook to
-	if db.db, err = sql.Open("sqlite3", dsn); err != nil {
+	// prevent WAL files from being removed.
+	if db.db, err = sql.Open("litestream-sqlite3", dsn); err != nil {
 		return err
 	}
 
@@ -420,7 +410,7 @@ func (db *DB) init() (err error) {
 	// https://www.sqlite.org/pragma.html#pragma_journal_mode
 	var mode string
 	if err := db.db.QueryRow(`PRAGMA journal_mode = wal;`).Scan(&mode); err != nil {
-		return fmt.Errorf("enable wal: %w", err)
+		return err
 	} else if mode != "wal" {
 		return fmt.Errorf("enable wal failed, mode=%q", mode)
 	}
@@ -456,13 +446,13 @@ func (db *DB) init() (err error) {
 	}
 
 	// Ensure meta directory structure exists.
-	if err := mkdirAll(db.MetaPath(), db.dirmode, db.diruid, db.dirgid); err != nil {
+	if err := internal.MkdirAll(db.MetaPath(), db.dirInfo); err != nil {
 		return err
 	}
 
 	// If we have an existing shadow WAL, ensure the headers match.
 	if err := db.verifyHeadersMatch(); err != nil {
-		log.Printf("%s: init: cannot determine last wal position, clearing generation; %s", db.path, err)
+		db.Logger.Printf("%s: init: cannot determine last wal position, clearing generation; %s", db.path, err)
 		if err := os.Remove(db.GenerationNamePath()); err != nil && !os.IsNotExist(err) {
 			return fmt.Errorf("remove generation name: %w", err)
 		}
@@ -563,7 +553,7 @@ func (db *DB) cleanWAL() error {
 	// Determine lowest index that's been replicated to all replicas.
 	min := -1
 	for _, r := range db.Replicas {
-		pos := r.LastPos()
+		pos := r.Pos()
 		if pos.Generation != generation {
 			pos = Pos{} // different generation, reset index to zero
 		}
@@ -587,7 +577,7 @@ func (db *DB) cleanWAL() error {
 		return err
 	}
 	for _, fi := range fis {
-		if idx, _, _, err := ParseWALPath(fi.Name()); err != nil || idx >= min {
+		if idx, err := ParseWALPath(fi.Name()); err != nil || idx >= min {
 			continue
 		}
 		if err := os.Remove(filepath.Join(dir, fi.Name())); err != nil {
@@ -597,26 +587,6 @@ func (db *DB) cleanWAL() error {
 	return nil
 }
 
-// SoftClose closes everything but the underlying db connection. This method
-// is available because the binary needs to avoid closing the database on exit
-// to prevent autocheckpointing.
-func (db *DB) SoftClose() (err error) {
-	db.cancel()
-	db.wg.Wait()
-
-	// Ensure replicas all stop replicating.
-	for _, r := range db.Replicas {
-		r.Stop(false)
-	}
-
-	if db.rtx != nil {
-		if e := db.releaseReadLock(); e != nil && err == nil {
-			err = e
-		}
-	}
-	return err
-}
-
 // acquireReadLock begins a read transaction on the database to prevent checkpointing.
 func (db *DB) acquireReadLock() error {
 	if db.rtx != nil {
@@ -683,7 +653,7 @@ func (db *DB) createGeneration() (string, error) {
 
 	// Generate new directory.
 	dir := filepath.Join(db.MetaPath(), "generations", generation)
-	if err := mkdirAll(dir, db.dirmode, db.diruid, db.dirgid); err != nil {
+	if err := internal.MkdirAll(dir, db.dirInfo); err != nil {
 		return "", err
 	}
 
@@ -694,10 +664,15 @@ func (db *DB) createGeneration() (string, error) {
 
 	// Atomically write generation name as current generation.
 	generationNamePath := db.GenerationNamePath()
-	if err := ioutil.WriteFile(generationNamePath+".tmp", []byte(generation+"\n"), db.mode); err != nil {
+	mode := os.FileMode(0600)
+	if db.fileInfo != nil {
+		mode = db.fileInfo.Mode()
+	}
+	if err := ioutil.WriteFile(generationNamePath+".tmp", []byte(generation+"\n"), mode); err != nil {
 		return "", fmt.Errorf("write generation temp file: %w", err)
 	}
-	_ = os.Chown(generationNamePath+".tmp", db.uid, db.gid)
+	uid, gid := internal.Fileinfo(db.fileInfo)
+	_ = os.Chown(generationNamePath+".tmp", uid, gid)
 	if err := os.Rename(generationNamePath+".tmp", generationNamePath); err != nil {
 		return "", fmt.Errorf("rename generation file: %w", err)
 	}
@@ -711,7 +686,7 @@ func (db *DB) createGeneration() (string, error) {
 }
 
 // Sync copies pending data from the WAL to the shadow WAL.
-func (db *DB) Sync() (err error) {
+func (db *DB) Sync(ctx context.Context) (err error) {
 	db.mu.Lock()
 	defer db.mu.Unlock()
 
@@ -738,27 +713,6 @@ func (db *DB) Sync() (err error) {
 		return fmt.Errorf("ensure wal exists: %w", err)
 	}
 
-	// Start a transaction. This will be promoted immediately after.
-	tx, err := db.db.Begin()
-	if err != nil {
-		return fmt.Errorf("begin: %w", err)
-	}
-
-	// Ensure write transaction rolls back before returning.
-	defer func() {
-		if e := rollback(tx); e != nil && err == nil {
-			err = e
-		}
-	}()
-
-	// Insert into the lock table to promote to a write tx. The lock table
-	// insert will never actually occur because our tx will be rolled back,
-	// however, it will ensure our tx grabs the write lock. Unfortunately,
-	// we can't call "BEGIN IMMEDIATE" as we are already in a transaction.
-	if _, err := tx.ExecContext(db.ctx, `INSERT INTO _litestream_lock (id) VALUES (1);`); err != nil {
-		return fmt.Errorf("_litestream_lock: %w", err)
-	}
-
 	// Verify our last sync matches the current state of the WAL.
 	// This ensures that we have an existing generation & that the last sync
 	// position of the real WAL hasn't been overwritten by another process.
@@ -777,7 +731,7 @@ func (db *DB) Sync() (err error) {
 		if info.generation, err = db.createGeneration(); err != nil {
 			return fmt.Errorf("create generation: %w", err)
 		}
-		log.Printf("%s: sync: new generation %q, %s", db.path, info.generation, info.reason)
+		db.Logger.Printf("%s: sync: new generation %q, %s", db.path, info.generation, info.reason)
 
 		// Clear shadow wal info.
 		info.shadowWALPath = db.ShadowWALPath(info.generation, 0)
@@ -805,16 +759,11 @@ func (db *DB) Sync() (err error) {
 		checkpoint = true
 	}
 
-	// Release write lock before checkpointing & exiting.
-	if err := tx.Rollback(); err != nil {
-		return fmt.Errorf("rollback write tx: %w", err)
-	}
-
 	// Issue the checkpoint.
 	if checkpoint {
 		changed = true
 
-		if err := db.checkpointAndInit(info.generation, checkpointMode); err != nil {
+		if err := db.checkpoint(ctx, info.generation, checkpointMode); err != nil {
 			return fmt.Errorf("checkpoint: mode=%v err=%w", checkpointMode, err)
 		}
 	}
@@ -972,7 +921,7 @@ func (db *DB) syncWAL(info syncInfo) (newSize int64, err error) {
 
 	// Parse index of current shadow WAL file.
 	dir, base := filepath.Split(info.shadowWALPath)
-	index, _, _, err := ParseWALPath(base)
+	index, err := ParseWALPath(base)
 	if err != nil {
 		return 0, fmt.Errorf("cannot parse shadow wal filename: %s", base)
 	}
@@ -1006,12 +955,17 @@ func (db *DB) initShadowWALFile(filename string) (int64, error) {
 	}
 
 	// Write header to new WAL shadow file.
-	if err := mkdirAll(filepath.Dir(filename), db.dirmode, db.diruid, db.dirgid); err != nil {
+	mode := os.FileMode(0600)
+	if fi := db.fileInfo; fi != nil {
+		mode = fi.Mode()
+	}
+	if err := internal.MkdirAll(filepath.Dir(filename), db.dirInfo); err != nil {
 		return 0, err
-	} else if err := ioutil.WriteFile(filename, hdr, db.mode); err != nil {
+	} else if err := ioutil.WriteFile(filename, hdr, mode); err != nil {
 		return 0, err
 	}
-	_ = os.Chown(filename, db.uid, db.gid)
+	uid, gid := internal.Fileinfo(db.fileInfo)
+	_ = os.Chown(filename, uid, gid)
 
 	// Copy as much shadow WAL as available.
 	newSize, err := db.copyToShadowWAL(filename)
@@ -1097,7 +1051,7 @@ func (db *DB) copyToShadowWAL(filename string) (newSize int64, err error) {
 		chksum0, chksum1 = Checksum(bo, chksum0, chksum1, frame[:8])  // frame header
 		chksum0, chksum1 = Checksum(bo, chksum0, chksum1, frame[24:]) // frame data
 		if chksum0 != fchksum0 || chksum1 != fchksum1 {
-			log.Printf("copy shadow: checksum mismatch, skipping: offset=%d (%x,%x) != (%x,%x)", offset, chksum0, chksum1, fchksum0, fchksum1)
+			Tracef("%s: copy shadow: checksum mismatch, skipping: offset=%d (%x,%x) != (%x,%x)", db.path, offset, chksum0, chksum1, fchksum0, fchksum1)
 			break
 		}
 
@@ -1168,7 +1122,7 @@ func (db *DB) shadowWALReader(pos Pos) (r *ShadowWALReader, err error) {
 	// Ensure file is closed if any error occurs.
 	defer func() {
 		if err != nil {
-			r.Close()
+			f.Close()
 		}
 	}()
 
@@ -1271,13 +1225,91 @@ func readLastChecksumFrom(f *os.File, pageSize int) (uint32, uint32, error) {
 }
 
 // Checkpoint performs a checkpoint on the WAL file.
-func (db *DB) Checkpoint(mode string) (err error) {
+func (db *DB) Checkpoint(ctx context.Context, mode string) (err error) {
 	db.mu.Lock()
 	defer db.mu.Unlock()
-	return db.checkpoint(mode)
+
+	generation, err := db.CurrentGeneration()
+	if err != nil {
+		return fmt.Errorf("cannot determine generation: %w", err)
+	}
+	return db.checkpoint(ctx, generation, mode)
 }
 
-func (db *DB) checkpoint(mode string) (err error) {
+// checkpointAndInit performs a checkpoint on the WAL file and initializes a
+// new shadow WAL file.
+func (db *DB) checkpoint(ctx context.Context, generation, mode string) error {
+	shadowWALPath, err := db.CurrentShadowWALPath(generation)
+	if err != nil {
+		return err
+	}
+
+	// Read WAL header before checkpoint to check if it has been restarted.
+	hdr, err := readWALHeader(db.WALPath())
+	if err != nil {
+		return err
+	}
+
+	// Copy shadow WAL before checkpoint to copy as much as possible.
+	if _, err := db.copyToShadowWAL(shadowWALPath); err != nil {
+		return fmt.Errorf("cannot copy to end of shadow wal before checkpoint: %w", err)
+	}
+
+	// Execute checkpoint and immediately issue a write to the WAL to ensure
+	// a new page is written.
+	if err := db.execCheckpoint(mode); err != nil {
+		return err
+	} else if _, err = db.db.Exec(`INSERT INTO _litestream_seq (id, seq) VALUES (1, 1) ON CONFLICT (id) DO UPDATE SET seq = seq + 1`); err != nil {
+		return err
+	}
+
+	// If WAL hasn't been restarted, exit.
+	if other, err := readWALHeader(db.WALPath()); err != nil {
+		return err
+	} else if bytes.Equal(hdr, other) {
+		return nil
+	}
+
+	// Start a transaction. This will be promoted immediately after.
+	tx, err := db.db.Begin()
+	if err != nil {
+		return fmt.Errorf("begin: %w", err)
+	}
+	defer func() { _ = rollback(tx) }()
+
+	// Insert into the lock table to promote to a write tx. The lock table
+	// insert will never actually occur because our tx will be rolled back,
+	// however, it will ensure our tx grabs the write lock. Unfortunately,
+	// we can't call "BEGIN IMMEDIATE" as we are already in a transaction.
+	if _, err := tx.ExecContext(ctx, `INSERT INTO _litestream_lock (id) VALUES (1);`); err != nil {
+		return fmt.Errorf("_litestream_lock: %w", err)
+	}
+
+	// Copy the end of the previous WAL before starting a new shadow WAL.
+	if _, err := db.copyToShadowWAL(shadowWALPath); err != nil {
+		return fmt.Errorf("cannot copy to end of shadow wal: %w", err)
+	}
+
+	// Parse index of current shadow WAL file.
+	index, err := ParseWALPath(shadowWALPath)
+	if err != nil {
+		return fmt.Errorf("cannot parse shadow wal filename: %s", shadowWALPath)
+	}
+
+	// Start a new shadow WAL file with next index.
+	newShadowWALPath := filepath.Join(filepath.Dir(shadowWALPath), FormatWALPath(index+1))
+	if _, err := db.initShadowWALFile(newShadowWALPath); err != nil {
+		return fmt.Errorf("cannot init shadow wal file: name=%s err=%w", newShadowWALPath, err)
+	}
+
+	// Release write lock before checkpointing & exiting.
+	if err := tx.Rollback(); err != nil {
+		return fmt.Errorf("rollback post-checkpoint tx: %w", err)
+	}
+	return nil
+}
+
+func (db *DB) execCheckpoint(mode string) (err error) {
 	// Ignore if there is no underlying database.
 	if db.db == nil {
 		return nil
@@ -1323,79 +1355,6 @@ func (db *DB) checkpoint(mode string) (err error) {
 	return nil
 }
 
-// checkpointAndInit performs a checkpoint on the WAL file and initializes a
-// new shadow WAL file.
-func (db *DB) checkpointAndInit(generation, mode string) error {
-	shadowWALPath, err := db.CurrentShadowWALPath(generation)
-	if err != nil {
-		return err
-	}
-
-	// Read WAL header before checkpoint to check if it has been restarted.
-	hdr, err := readWALHeader(db.WALPath())
-	if err != nil {
-		return err
-	}
-
-	// Copy shadow WAL before checkpoint to copy as much as possible.
-	if _, err := db.copyToShadowWAL(shadowWALPath); err != nil {
-		return fmt.Errorf("cannot copy to end of shadow wal before checkpoint: %w", err)
-	}
-
-	// Execute checkpoint and immediately issue a write to the WAL to ensure
-	// a new page is written.
-	if err := db.checkpoint(mode); err != nil {
-		return err
-	} else if _, err = db.db.Exec(`INSERT INTO _litestream_seq (id, seq) VALUES (1, 1) ON CONFLICT (id) DO UPDATE SET seq = seq + 1`); err != nil {
-		return err
-	}
-
-	// If WAL hasn't been restarted, exit.
-	if other, err := readWALHeader(db.WALPath()); err != nil {
-		return err
-	} else if bytes.Equal(hdr, other) {
-		return nil
-	}
-
-	// Start a transaction. This will be promoted immediately after.
-	tx, err := db.db.Begin()
-	if err != nil {
-		return fmt.Errorf("begin: %w", err)
-	}
-	defer func() { _ = rollback(tx) }()
-
-	// Insert into the lock table to promote to a write tx. The lock table
-	// insert will never actually occur because our tx will be rolled back,
-	// however, it will ensure our tx grabs the write lock. Unfortunately,
-	// we can't call "BEGIN IMMEDIATE" as we are already in a transaction.
-	if _, err := tx.ExecContext(db.ctx, `INSERT INTO _litestream_lock (id) VALUES (1);`); err != nil {
-		return fmt.Errorf("_litestream_lock: %w", err)
-	}
-
-	// Copy the end of the previous WAL before starting a new shadow WAL.
-	if _, err := db.copyToShadowWAL(shadowWALPath); err != nil {
-		return fmt.Errorf("cannot copy to end of shadow wal: %w", err)
-	}
-
-	// Parse index of current shadow WAL file.
-	index, _, _, err := ParseWALPath(shadowWALPath)
-	if err != nil {
-		return fmt.Errorf("cannot parse shadow wal filename: %s", shadowWALPath)
-	}
-
-	// Start a new shadow WAL file with next index.
-	newShadowWALPath := filepath.Join(filepath.Dir(shadowWALPath), FormatWALPath(index+1))
-	if _, err := db.initShadowWALFile(newShadowWALPath); err != nil {
-		return fmt.Errorf("cannot init shadow wal file: name=%s err=%w", newShadowWALPath, err)
-	}
-
-	// Release write lock before checkpointing & exiting.
-	if err := tx.Rollback(); err != nil {
-		return fmt.Errorf("rollback post-checkpoint tx: %w", err)
-	}
-	return nil
-}
-
 // monitor runs in a separate goroutine and monitors the database & WAL.
 func (db *DB) monitor() {
 	ticker := time.NewTicker(db.MonitorInterval)
@@ -1410,106 +1369,18 @@ func (db *DB) monitor() {
 		}
 
 		// Sync the database to the shadow WAL.
-		if err := db.Sync(); err != nil && !errors.Is(err, context.Canceled) {
+		if err := db.Sync(db.ctx); err != nil && !errors.Is(err, context.Canceled) {
-			log.Printf("%s: sync error: %s", db.path, err)
+			db.Logger.Printf("%s: sync error: %s", db.path, err)
 		}
 	}
 }
 
-// RestoreReplica restores the database from a replica based on the options given.
-// This method will restore into opt.OutputPath, if specified, or into the
-// DB's original database path. It can optionally restore from a specific
-// replica or generation or it will automatically choose the best one. Finally,
-// a timestamp can be specified to restore the database to a specific
-// point-in-time.
-func RestoreReplica(ctx context.Context, r Replica, opt RestoreOptions) error {
-	// Validate options.
-	if opt.OutputPath == "" {
-		return fmt.Errorf("output path required")
-	} else if opt.Generation == "" && opt.Index != math.MaxInt64 {
-		return fmt.Errorf("must specify generation when restoring to index")
-	} else if opt.Index != math.MaxInt64 && !opt.Timestamp.IsZero() {
-		return fmt.Errorf("cannot specify index & timestamp to restore")
-	}
-
-	// Ensure logger exists.
-	logger := opt.Logger
-	if logger == nil {
-		logger = log.New(ioutil.Discard, "", 0)
-	}
-
-	logPrefix := r.Name()
-	if db := r.DB(); db != nil {
-		logPrefix = fmt.Sprintf("%s(%s)", db.Path(), r.Name())
-	}
-
-	// Ensure output path does not already exist (unless this is a dry run).
-	if !opt.DryRun {
-		if _, err := os.Stat(opt.OutputPath); err == nil {
-			return fmt.Errorf("cannot restore, output path already exists: %s", opt.OutputPath)
-		} else if err != nil && !os.IsNotExist(err) {
-			return err
-		}
-	}
-
-	// Find lastest snapshot that occurs before timestamp.
-	minWALIndex, err := SnapshotIndexAt(ctx, r, opt.Generation, opt.Timestamp)
-	if err != nil {
-		return fmt.Errorf("cannot find snapshot index for restore: %w", err)
-	}
-
-	// Find the maximum WAL index that occurs before timestamp.
-	maxWALIndex, err := WALIndexAt(ctx, r, opt.Generation, opt.Index, opt.Timestamp)
-	if err != nil {
-		return fmt.Errorf("cannot find max wal index for restore: %w", err)
-	}
-	logger.Printf("%s: starting restore: generation %s, index %08x-%08x", logPrefix, opt.Generation, minWALIndex, maxWALIndex)
-
-	// Initialize starting position.
-	pos := Pos{Generation: opt.Generation, Index: minWALIndex}
-	tmpPath := opt.OutputPath + ".tmp"
-
-	// Copy snapshot to output path.
-	logger.Printf("%s: restoring snapshot %s/%08x to %s", logPrefix, opt.Generation, minWALIndex, tmpPath)
-	if !opt.DryRun {
-		if err := restoreSnapshot(ctx, r, pos.Generation, pos.Index, tmpPath); err != nil {
-			return fmt.Errorf("cannot restore snapshot: %w", err)
-		}
-	}
-
-	// Restore each WAL file until we reach our maximum index.
-	for index := minWALIndex; index <= maxWALIndex; index++ {
-		if !opt.DryRun {
-			if err = restoreWAL(ctx, r, opt.Generation, index, tmpPath); os.IsNotExist(err) && index == minWALIndex && index == maxWALIndex {
-				logger.Printf("%s: no wal available, snapshot only", logPrefix)
-				break // snapshot file only, ignore error
-			} else if err != nil {
-				return fmt.Errorf("cannot restore wal: %w", err)
-			}
-		}
-
-		if opt.Verbose {
-			logger.Printf("%s: restored wal %s/%08x", logPrefix, opt.Generation, index)
-		}
-	}
-
-	// Copy file to final location.
-	logger.Printf("%s: renaming database from temporary location", logPrefix)
-	if !opt.DryRun {
-		if err := os.Rename(tmpPath, opt.OutputPath); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
 // CalcRestoreTarget returns a replica & generation to restore from based on opt criteria.
-func (db *DB) CalcRestoreTarget(ctx context.Context, opt RestoreOptions) (Replica, string, error) {
+func (db *DB) CalcRestoreTarget(ctx context.Context, opt RestoreOptions) (*Replica, string, error) {
 	var target struct {
-		replica    Replica
+		replica    *Replica
 		generation string
-		stats      GenerationStats
+		updatedAt  time.Time
 	}
 
 	for _, r := range db.Replicas {
@@ -1518,132 +1389,30 @@ func (db *DB) CalcRestoreTarget(ctx context.Context, opt RestoreOptions) (Replic
 			continue
 		}
 
-		generation, stats, err := CalcReplicaRestoreTarget(ctx, r, opt)
+		generation, updatedAt, err := r.CalcRestoreTarget(ctx, opt)
 		if err != nil {
 			return nil, "", err
 		}
 
 		// Use the latest replica if we have multiple candidates.
-		if !stats.UpdatedAt.After(target.stats.UpdatedAt) {
+		if !updatedAt.After(target.updatedAt) {
 			continue
 		}
 
-		target.replica, target.generation, target.stats = r, generation, stats
+		target.replica, target.generation, target.updatedAt = r, generation, updatedAt
 	}
 	return target.replica, target.generation, nil
 }
 
-// CalcReplicaRestoreTarget returns a generation to restore from.
+// applyWAL performs a truncating checkpoint on the given database.
-func CalcReplicaRestoreTarget(ctx context.Context, r Replica, opt RestoreOptions) (generation string, stats GenerationStats, err error) {
+func applyWAL(ctx context.Context, index int, dbPath string) error {
-	var target struct {
+	// Copy WAL file from it's staging path to the correct "-wal" location.
-		generation string
+	if err := os.Rename(fmt.Sprintf("%s-%08x-wal", dbPath, index), dbPath+"-wal"); err != nil {
-		stats      GenerationStats
-	}
-
-	generations, err := r.Generations(ctx)
-	if err != nil {
-		return "", stats, fmt.Errorf("cannot fetch generations: %w", err)
-	}
-
-	// Search generations for one that contains the requested timestamp.
-	for _, generation := range generations {
-		// Skip generation if it does not match filter.
-		if opt.Generation != "" && generation != opt.Generation {
-			continue
-		}
-
-		// Fetch stats for generation.
-		stats, err := r.GenerationStats(ctx, generation)
-		if err != nil {
-			return "", stats, fmt.Errorf("cannot determine stats for generation (%s/%s): %s", r.Name(), generation, err)
-		}
-
-		// Skip if it does not contain timestamp.
-		if !opt.Timestamp.IsZero() {
-			if opt.Timestamp.Before(stats.CreatedAt) || opt.Timestamp.After(stats.UpdatedAt) {
-				continue
-			}
-		}
-
-		// Use the latest replica if we have multiple candidates.
-		if !stats.UpdatedAt.After(target.stats.UpdatedAt) {
-			continue
-		}
-
-		target.generation = generation
-		target.stats = stats
-	}
-
-	return target.generation, target.stats, nil
-}
-
-// restoreSnapshot copies a snapshot from the replica to a file.
-func restoreSnapshot(ctx context.Context, r Replica, generation string, index int, filename string) error {
-	// Determine the user/group & mode based on the DB, if available.
-	uid, gid, mode := -1, -1, os.FileMode(0600)
-	diruid, dirgid, dirmode := -1, -1, os.FileMode(0700)
-	if db := r.DB(); db != nil {
-		uid, gid, mode = db.uid, db.gid, db.mode
-		diruid, dirgid, dirmode = db.diruid, db.dirgid, db.dirmode
-	}
-
-	if err := mkdirAll(filepath.Dir(filename), dirmode, diruid, dirgid); err != nil {
-		return err
-	}
-
-	f, err := createFile(filename, mode, uid, gid)
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-
-	rd, err := r.SnapshotReader(ctx, generation, index)
-	if err != nil {
-		return err
-	}
-	defer rd.Close()
-
-	if _, err := io.Copy(f, rd); err != nil {
-		return err
-	}
-
-	if err := f.Sync(); err != nil {
-		return err
-	}
-	return f.Close()
-}
-
-// restoreWAL copies a WAL file from the replica to the local WAL and forces checkpoint.
-func restoreWAL(ctx context.Context, r Replica, generation string, index int, dbPath string) error {
-	// Determine the user/group & mode based on the DB, if available.
-	uid, gid, mode := -1, -1, os.FileMode(0600)
-	if db := r.DB(); db != nil {
-		uid, gid, mode = db.uid, db.gid, db.mode
-	}
-
-	// Open WAL file from replica.
-	rd, err := r.WALReader(ctx, generation, index)
-	if err != nil {
-		return err
-	}
-	defer rd.Close()
-
-	// Open handle to destination WAL path.
-	f, err := createFile(dbPath+"-wal", mode, uid, gid)
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-
-	// Copy WAL to target path.
-	if _, err := io.Copy(f, rd); err != nil {
-		return err
-	} else if err := f.Close(); err != nil {
 		return err
 	}
 
 	// Open SQLite database and force a truncating checkpoint.
-	d, err := sql.Open("sqlite3", dbPath)
+	d, err := sql.Open("litestream-sqlite3", dbPath)
 	if err != nil {
 		return err
 	}
@@ -1655,7 +1424,6 @@ func restoreWAL(ctx context.Context, r Replica, generation string, index int, db
 	} else if row[0] != 0 {
 		return fmt.Errorf("truncation checkpoint failed during restore (%d,%d,%d)", row[0], row[1], row[2])
 	}
-
 	return d.Close()
 }
 
@@ -1666,7 +1434,7 @@ func restoreWAL(ctx context.Context, r Replica, generation string, index int, db
 // unable to checkpoint during this time.
 //
 // If dst is set, the database file is copied to that location before checksum.
-func (db *DB) CRC64() (uint64, Pos, error) {
+func (db *DB) CRC64(ctx context.Context) (uint64, Pos, error) {
 	db.mu.Lock()
 	defer db.mu.Unlock()
 
@@ -1684,7 +1452,7 @@ func (db *DB) CRC64() (uint64, Pos, error) {
 	}
 
 	// Force a RESTART checkpoint to ensure the database is at the start of the WAL.
-	if err := db.checkpointAndInit(generation, CheckpointModeRestart); err != nil {
+	if err := db.checkpoint(ctx, generation, CheckpointModeRestart); err != nil {
 		return 0, Pos{}, err
 	}
 
@@ -1706,6 +1474,9 @@ func (db *DB) CRC64() (uint64, Pos, error) {
 	return h.Sum64(), pos, nil
 }
 
+// DefaultRestoreParallelism is the default parallelism when downloading WAL files.
+const DefaultRestoreParallelism = 8
+
 // RestoreOptions represents options for DB.Restore().
 type RestoreOptions struct {
 	// Target path to restore into.
@@ -1721,16 +1492,15 @@ type RestoreOptions struct {
 	Generation string
 
 	// Specific index to restore from.
-	// Set to math.MaxInt64 to ignore index.
+	// Set to math.MaxInt32 to ignore index.
 	Index int
 
 	// Point-in-time to restore database.
 	// If zero, database restore to most recent state available.
 	Timestamp time.Time
 
-	// If true, no actual restore is performed.
+	// Specifies how many WAL files are downloaded in parallel during restore.
-	// Only equivalent log output for a regular restore.
+	Parallelism int
-	DryRun bool
 
 	// Logging settings.
 	Logger  *log.Logger
@@ -1740,87 +1510,66 @@ type RestoreOptions struct {
 // NewRestoreOptions returns a new instance of RestoreOptions with defaults.
 func NewRestoreOptions() RestoreOptions {
 	return RestoreOptions{
-		Index: math.MaxInt64,
+		Index:       math.MaxInt32,
+		Parallelism: DefaultRestoreParallelism,
 	}
 }
 
 // Database metrics.
 var (
 	dbSizeGaugeVec = promauto.NewGaugeVec(prometheus.GaugeOpts{
-		Namespace: "litestream",
+		Name: "litestream_db_size",
-		Subsystem: "db",
+		Help: "The current size of the real DB",
-		Name:      "db_size",
-		Help:      "The current size of the real DB",
 	}, []string{"db"})
 
 	walSizeGaugeVec = promauto.NewGaugeVec(prometheus.GaugeOpts{
-		Namespace: "litestream",
+		Name: "litestream_wal_size",
-		Subsystem: "db",
+		Help: "The current size of the real WAL",
-		Name:      "wal_size",
-		Help:      "The current size of the real WAL",
 	}, []string{"db"})
 
 	totalWALBytesCounterVec = promauto.NewCounterVec(prometheus.CounterOpts{
-		Namespace: "litestream",
+		Name: "litestream_total_wal_bytes",
-		Subsystem: "db",
+		Help: "Total number of bytes written to shadow WAL",
-		Name:      "total_wal_bytes",
-		Help:      "Total number of bytes written to shadow WAL",
 	}, []string{"db"})
 
 	shadowWALIndexGaugeVec = promauto.NewGaugeVec(prometheus.GaugeOpts{
-		Namespace: "litestream",
+		Name: "litestream_shadow_wal_index",
-		Subsystem: "db",
+		Help: "The current index of the shadow WAL",
-		Name:      "shadow_wal_index",
-		Help:      "The current index of the shadow WAL",
 	}, []string{"db"})
 
 	shadowWALSizeGaugeVec = promauto.NewGaugeVec(prometheus.GaugeOpts{
-		Namespace: "litestream",
+		Name: "litestream_shadow_wal_size",
-		Subsystem: "db",
+		Help: "Current size of shadow WAL, in bytes",
-		Name:      "shadow_wal_size",
-		Help:      "Current size of shadow WAL, in bytes",
 	}, []string{"db"})
 
 	syncNCounterVec = promauto.NewCounterVec(prometheus.CounterOpts{
-		Namespace: "litestream",
+		Name: "litestream_sync_count",
-		Subsystem: "db",
+		Help: "Number of sync operations performed",
-		Name:      "sync_count",
-		Help:      "Number of sync operations performed",
 	}, []string{"db"})
 
 	syncErrorNCounterVec = promauto.NewCounterVec(prometheus.CounterOpts{
-		Namespace: "litestream",
+		Name: "litestream_sync_error_count",
-		Subsystem: "db",
+		Help: "Number of sync errors that have occurred",
-		Name:      "sync_error_count",
-		Help:      "Number of sync errors that have occurred",
 	}, []string{"db"})
 
 	syncSecondsCounterVec = promauto.NewCounterVec(prometheus.CounterOpts{
-		Namespace: "litestream",
+		Name: "litestream_sync_seconds",
-		Subsystem: "db",
+		Help: "Time spent syncing shadow WAL, in seconds",
-		Name:      "sync_seconds",
-		Help:      "Time spent syncing shadow WAL, in seconds",
 	}, []string{"db"})
 
 	checkpointNCounterVec = promauto.NewCounterVec(prometheus.CounterOpts{
-		Namespace: "litestream",
+		Name: "litestream_checkpoint_count",
-		Subsystem: "db",
+		Help: "Number of checkpoint operations performed",
-		Name:      "checkpoint_count",
-		Help:      "Number of checkpoint operations performed",
 	}, []string{"db", "mode"})
 
 	checkpointErrorNCounterVec = promauto.NewCounterVec(prometheus.CounterOpts{
-		Namespace: "litestream",
+		Name: "litestream_checkpoint_error_count",
-		Subsystem: "db",
+		Help: "Number of checkpoint errors that have occurred",
-		Name:      "checkpoint_error_count",
-		Help:      "Number of checkpoint errors that have occurred",
 	}, []string{"db", "mode"})
 
 	checkpointSecondsCounterVec = promauto.NewCounterVec(prometheus.CounterOpts{
-		Namespace: "litestream",
+		Name: "litestream_checkpoint_seconds",
-		Subsystem: "db",
+		Help: "Time spent checkpointing WAL, in seconds",
-		Name:      "checkpoint_seconds",
-		Help:      "Time spent checkpointing WAL, in seconds",
 	}, []string{"db", "mode"})
 )
 

db_test.go

@@ -1,10 +1,12 @@
 package litestream_test
 
 import (
+	"context"
 	"database/sql"
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 	"time"
 
@@ -98,9 +100,12 @@ func TestDB_UpdatedAt(t *testing.T) {
 			t.Fatal(err)
 		}
 
+		sleepTime := 100 * time.Millisecond
 		if os.Getenv("CI") != "" {
-			time.Sleep(1 * time.Second)
+			sleepTime = 1 * time.Second
 		}
+		time.Sleep(sleepTime)
+
 		if _, err := sqldb.Exec(`CREATE TABLE t (id INT);`); err != nil {
 			t.Fatal(err)
 		}
@@ -118,7 +123,7 @@ func TestDB_CRC64(t *testing.T) {
 	t.Run("ErrNotExist", func(t *testing.T) {
 		db := MustOpenDB(t)
 		defer MustCloseDB(t, db)
-		if _, _, err := db.CRC64(); !os.IsNotExist(err) {
+		if _, _, err := db.CRC64(context.Background()); !os.IsNotExist(err) {
 			t.Fatalf("unexpected error: %#v", err)
 		}
 	})
@@ -127,11 +132,11 @@ func TestDB_CRC64(t *testing.T) {
 		db, sqldb := MustOpenDBs(t)
 		defer MustCloseDBs(t, db, sqldb)
 
-		if err := db.Sync(); err != nil {
+		if err := db.Sync(context.Background()); err != nil {
 			t.Fatal(err)
 		}
 
-		chksum0, _, err := db.CRC64()
+		chksum0, _, err := db.CRC64(context.Background())
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -139,18 +144,18 @@ func TestDB_CRC64(t *testing.T) {
 		// Issue change that is applied to the WAL. Checksum should not change.
 		if _, err := sqldb.Exec(`CREATE TABLE t (id INT);`); err != nil {
 			t.Fatal(err)
-		} else if chksum1, _, err := db.CRC64(); err != nil {
+		} else if chksum1, _, err := db.CRC64(context.Background()); err != nil {
 			t.Fatal(err)
 		} else if chksum0 == chksum1 {
 			t.Fatal("expected different checksum event after WAL change")
 		}
 
 		// Checkpoint change into database. Checksum should change.
-		if _, err := sqldb.Exec(`PRAGMA wal_checkpoint(TRUNCATE);`); err != nil {
+		if err := db.Checkpoint(context.Background(), litestream.CheckpointModeTruncate); err != nil {
 			t.Fatal(err)
 		}
 
-		if chksum2, _, err := db.CRC64(); err != nil {
+		if chksum2, _, err := db.CRC64(context.Background()); err != nil {
 			t.Fatal(err)
 		} else if chksum0 == chksum2 {
 			t.Fatal("expected different checksums after checkpoint")
@@ -164,7 +169,7 @@ func TestDB_Sync(t *testing.T) {
 	t.Run("NoDB", func(t *testing.T) {
 		db := MustOpenDB(t)
 		defer MustCloseDB(t, db)
-		if err := db.Sync(); err != nil {
+		if err := db.Sync(context.Background()); err != nil {
 			t.Fatal(err)
 		}
 	})
@@ -174,7 +179,7 @@ func TestDB_Sync(t *testing.T) {
 		db, sqldb := MustOpenDBs(t)
 		defer MustCloseDBs(t, db, sqldb)
 
-		if err := db.Sync(); err != nil {
+		if err := db.Sync(context.Background()); err != nil {
 			t.Fatal(err)
 		}
 
@@ -212,7 +217,7 @@ func TestDB_Sync(t *testing.T) {
 		}
 
 		// Perform initial sync & grab initial position.
-		if err := db.Sync(); err != nil {
+		if err := db.Sync(context.Background()); err != nil {
 			t.Fatal(err)
 		}
 
@@ -227,7 +232,7 @@ func TestDB_Sync(t *testing.T) {
 		}
 
 		// Sync to ensure position moves forward one page.
-		if err := db.Sync(); err != nil {
+		if err := db.Sync(context.Background()); err != nil {
 			t.Fatal(err)
 		} else if pos1, err := db.Pos(); err != nil {
 			t.Fatal(err)
@@ -246,7 +251,7 @@ func TestDB_Sync(t *testing.T) {
 		defer MustCloseDBs(t, db, sqldb)
 
 		// Issue initial sync and truncate WAL.
-		if err := db.Sync(); err != nil {
+		if err := db.Sync(context.Background()); err != nil {
 			t.Fatal(err)
 		}
 
@@ -257,7 +262,7 @@ func TestDB_Sync(t *testing.T) {
 		}
 
 		// Checkpoint & fully close which should close WAL file.
-		if err := db.Checkpoint(litestream.CheckpointModeTruncate); err != nil {
+		if err := db.Checkpoint(context.Background(), litestream.CheckpointModeTruncate); err != nil {
 			t.Fatal(err)
 		} else if err := db.Close(); err != nil {
 			t.Fatal(err)
@@ -265,8 +270,8 @@ func TestDB_Sync(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		// Verify WAL does not exist.
+		// Remove WAL file.
-		if _, err := os.Stat(db.WALPath()); !os.IsNotExist(err) {
+		if err := os.Remove(db.WALPath()); err != nil {
 			t.Fatal(err)
 		}
 
@@ -275,7 +280,7 @@ func TestDB_Sync(t *testing.T) {
 		defer MustCloseDB(t, db)
 
 		// Re-sync and ensure new generation has been created.
-		if err := db.Sync(); err != nil {
+		if err := db.Sync(context.Background()); err != nil {
 			t.Fatal(err)
 		}
 
@@ -298,7 +303,7 @@ func TestDB_Sync(t *testing.T) {
 		}
 
 		// Issue initial sync and truncate WAL.
-		if err := db.Sync(); err != nil {
+		if err := db.Sync(context.Background()); err != nil {
 			t.Fatal(err)
 		}
 
@@ -334,7 +339,7 @@ func TestDB_Sync(t *testing.T) {
 		defer MustCloseDB(t, db)
 
 		// Re-sync and ensure new generation has been created.
-		if err := db.Sync(); err != nil {
+		if err := db.Sync(context.Background()); err != nil {
 			t.Fatal(err)
 		}
 
@@ -354,7 +359,7 @@ func TestDB_Sync(t *testing.T) {
 		// Execute a query to force a write to the WAL and then sync.
 		if _, err := sqldb.Exec(`CREATE TABLE foo (bar TEXT);`); err != nil {
 			t.Fatal(err)
-		} else if err := db.Sync(); err != nil {
+		} else if err := db.Sync(context.Background()); err != nil {
 			t.Fatal(err)
 		}
 
@@ -378,7 +383,7 @@ func TestDB_Sync(t *testing.T) {
 		// Reopen managed database & ensure sync will still work.
 		db = MustOpenDBAt(t, db.Path())
 		defer MustCloseDB(t, db)
-		if err := db.Sync(); err != nil {
+		if err := db.Sync(context.Background()); err != nil {
 			t.Fatal(err)
 		}
 
@@ -398,7 +403,7 @@ func TestDB_Sync(t *testing.T) {
 		// Execute a query to force a write to the WAL and then sync.
 		if _, err := sqldb.Exec(`CREATE TABLE foo (bar TEXT);`); err != nil {
 			t.Fatal(err)
-		} else if err := db.Sync(); err != nil {
+		} else if err := db.Sync(context.Background()); err != nil {
 			t.Fatal(err)
 		}
 
@@ -417,7 +422,7 @@ func TestDB_Sync(t *testing.T) {
 		// Reopen managed database & ensure sync will still work.
 		db = MustOpenDBAt(t, db.Path())
 		defer MustCloseDB(t, db)
-		if err := db.Sync(); err != nil {
+		if err := db.Sync(context.Background()); err != nil {
 			t.Fatal(err)
 		}
 
@@ -437,7 +442,7 @@ func TestDB_Sync(t *testing.T) {
 		// Execute a query to force a write to the WAL and then sync.
 		if _, err := sqldb.Exec(`CREATE TABLE foo (bar TEXT);`); err != nil {
 			t.Fatal(err)
-		} else if err := db.Sync(); err != nil {
+		} else if err := db.Sync(context.Background()); err != nil {
 			t.Fatal(err)
 		}
 
@@ -462,7 +467,7 @@ func TestDB_Sync(t *testing.T) {
 		// Reopen managed database & ensure sync will still work.
 		db = MustOpenDBAt(t, db.Path())
 		defer MustCloseDB(t, db)
-		if err := db.Sync(); err != nil {
+		if err := db.Sync(context.Background()); err != nil {
 			t.Fatal(err)
 		}
 
@@ -489,7 +494,7 @@ func TestDB_Sync(t *testing.T) {
 		// Execute a query to force a write to the WAL and then sync.
 		if _, err := sqldb.Exec(`CREATE TABLE foo (bar TEXT);`); err != nil {
 			t.Fatal(err)
-		} else if err := db.Sync(); err != nil {
+		} else if err := db.Sync(context.Background()); err != nil {
 			t.Fatal(err)
 		}
 
@@ -508,7 +513,7 @@ func TestDB_Sync(t *testing.T) {
 		// Reopen managed database & ensure sync will still work.
 		db = MustOpenDBAt(t, db.Path())
 		defer MustCloseDB(t, db)
-		if err := db.Sync(); err != nil {
+		if err := db.Sync(context.Background()); err != nil {
 			t.Fatal(err)
 		}
 
@@ -532,7 +537,7 @@ func TestDB_Sync(t *testing.T) {
 		// Execute a query to force a write to the WAL and then sync.
 		if _, err := sqldb.Exec(`CREATE TABLE foo (bar TEXT);`); err != nil {
 			t.Fatal(err)
-		} else if err := db.Sync(); err != nil {
+		} else if err := db.Sync(context.Background()); err != nil {
 			t.Fatal(err)
 		}
 
@@ -544,7 +549,7 @@ func TestDB_Sync(t *testing.T) {
 		}
 
 		// Sync to shadow WAL.
-		if err := db.Sync(); err != nil {
+		if err := db.Sync(context.Background()); err != nil {
 			t.Fatal(err)
 		}
 
@@ -564,7 +569,7 @@ func TestDB_Sync(t *testing.T) {
 		// Execute a query to force a write to the WAL and then sync.
 		if _, err := sqldb.Exec(`CREATE TABLE foo (bar TEXT);`); err != nil {
 			t.Fatal(err)
-		} else if err := db.Sync(); err != nil {
+		} else if err := db.Sync(context.Background()); err != nil {
 			t.Fatal(err)
 		}
 
@@ -574,7 +579,7 @@ func TestDB_Sync(t *testing.T) {
 		// Write to WAL & sync.
 		if _, err := sqldb.Exec(`INSERT INTO foo (bar) VALUES ('baz');`); err != nil {
 			t.Fatal(err)
-		} else if err := db.Sync(); err != nil {
+		} else if err := db.Sync(context.Background()); err != nil {
 			t.Fatal(err)
 		}
 
@@ -589,12 +594,14 @@ func TestDB_Sync(t *testing.T) {
 
 // MustOpenDBs returns a new instance of a DB & associated SQL DB.
 func MustOpenDBs(tb testing.TB) (*litestream.DB, *sql.DB) {
+	tb.Helper()
 	db := MustOpenDB(tb)
 	return db, MustOpenSQLDB(tb, db.Path())
 }
 
 // MustCloseDBs closes db & sqldb and removes the parent directory.
 func MustCloseDBs(tb testing.TB, db *litestream.DB, sqldb *sql.DB) {
+	tb.Helper()
 	MustCloseDB(tb, db)
 	MustCloseSQLDB(tb, sqldb)
 }
@@ -619,7 +626,7 @@ func MustOpenDBAt(tb testing.TB, path string) *litestream.DB {
 // MustCloseDB closes db and removes its parent directory.
 func MustCloseDB(tb testing.TB, db *litestream.DB) {
 	tb.Helper()
-	if err := db.Close(); err != nil {
+	if err := db.Close(); err != nil && !strings.Contains(err.Error(), `database is closed`) {
 		tb.Fatal(err)
 	} else if err := os.RemoveAll(filepath.Dir(db.Path())); err != nil {
 		tb.Fatal(err)

etc/nfpm.yml

@@ -1,13 +1,13 @@
 name: litestream
-arch: amd64
+arch: "${GOARCH}"
-platform: linux
+platform: "${GOOS}"
 version: "${LITESTREAM_VERSION}"
 section: "default"
 priority: "extra"
 maintainer: "Ben Johnson <benbjohnson@yahoo.com>"
 description: Litestream is a tool for real-time replication of SQLite databases.
 homepage: "https://github.com/benbjohnson/litestream"
-license: "GPLv3"
+license: "Apache 2"
 contents:
 - src: ./litestream
   dst: /usr/bin/litestream

file/replica_client.go

@@ -0,0 +1,381 @@
+package file
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"sort"
+
+	"github.com/benbjohnson/litestream"
+	"github.com/benbjohnson/litestream/internal"
+)
+
+// ReplicaClientType is the client type for this package.
+const ReplicaClientType = "file"
+
+var _ litestream.ReplicaClient = (*ReplicaClient)(nil)
+
+// ReplicaClient is a client for writing snapshots & WAL segments to disk.
+type ReplicaClient struct {
+	path string // destination path
+
+	Replica *litestream.Replica
+}
+
+// NewReplicaClient returns a new instance of ReplicaClient.
+func NewReplicaClient(path string) *ReplicaClient {
+	return &ReplicaClient{
+		path: path,
+	}
+}
+
+// db returns the database, if available.
+func (c *ReplicaClient) db() *litestream.DB {
+	if c.Replica == nil {
+		return nil
+	}
+	return c.Replica.DB()
+}
+
+// Type returns "file" as the client type.
+func (c *ReplicaClient) Type() string {
+	return ReplicaClientType
+}
+
+// Path returns the destination path to replicate the database to.
+func (c *ReplicaClient) Path() string {
+	return c.path
+}
+
+// GenerationsDir returns the path to a generation root directory.
+func (c *ReplicaClient) GenerationsDir() (string, error) {
+	if c.path == "" {
+		return "", fmt.Errorf("file replica path required")
+	}
+	return filepath.Join(c.path, "generations"), nil
+}
+
+// GenerationDir returns the path to a generation's root directory.
+func (c *ReplicaClient) GenerationDir(generation string) (string, error) {
+	dir, err := c.GenerationsDir()
+	if err != nil {
+		return "", err
+	} else if generation == "" {
+		return "", fmt.Errorf("generation required")
+	}
+	return filepath.Join(dir, generation), nil
+}
+
+// SnapshotsDir returns the path to a generation's snapshot directory.
+func (c *ReplicaClient) SnapshotsDir(generation string) (string, error) {
+	dir, err := c.GenerationDir(generation)
+	if err != nil {
+		return "", err
+	}
+	return filepath.Join(dir, "snapshots"), nil
+}
+
+// SnapshotPath returns the path to an uncompressed snapshot file.
+func (c *ReplicaClient) SnapshotPath(generation string, index int) (string, error) {
+	dir, err := c.SnapshotsDir(generation)
+	if err != nil {
+		return "", err
+	}
+	return filepath.Join(dir, litestream.FormatSnapshotPath(index)), nil
+}
+
+// WALDir returns the path to a generation's WAL directory
+func (c *ReplicaClient) WALDir(generation string) (string, error) {
+	dir, err := c.GenerationDir(generation)
+	if err != nil {
+		return "", err
+	}
+	return filepath.Join(dir, "wal"), nil
+}
+
+// WALSegmentPath returns the path to a WAL segment file.
+func (c *ReplicaClient) WALSegmentPath(generation string, index int, offset int64) (string, error) {
+	dir, err := c.WALDir(generation)
+	if err != nil {
+		return "", err
+	}
+	return filepath.Join(dir, litestream.FormatWALSegmentPath(index, offset)), nil
+}
+
+// Generations returns a list of available generation names.
+func (c *ReplicaClient) Generations(ctx context.Context) ([]string, error) {
+	root, err := c.GenerationsDir()
+	if err != nil {
+		return nil, fmt.Errorf("cannot determine generations path: %w", err)
+	}
+
+	fis, err := ioutil.ReadDir(root)
+	if os.IsNotExist(err) {
+		return nil, nil
+	} else if err != nil {
+		return nil, err
+	}
+
+	var generations []string
+	for _, fi := range fis {
+		if !litestream.IsGenerationName(fi.Name()) {
+			continue
+		} else if !fi.IsDir() {
+			continue
+		}
+		generations = append(generations, fi.Name())
+	}
+	return generations, nil
+}
+
+// DeleteGeneration deletes all snapshots & WAL segments within a generation.
+func (c *ReplicaClient) DeleteGeneration(ctx context.Context, generation string) error {
+	dir, err := c.GenerationDir(generation)
+	if err != nil {
+		return fmt.Errorf("cannot determine generation path: %w", err)
+	}
+
+	if err := os.RemoveAll(dir); err != nil && !os.IsNotExist(err) {
+		return err
+	}
+	return nil
+}
+
+// Snapshots returns an iterator over all available snapshots for a generation.
+func (c *ReplicaClient) Snapshots(ctx context.Context, generation string) (litestream.SnapshotIterator, error) {
+	dir, err := c.SnapshotsDir(generation)
+	if err != nil {
+		return nil, fmt.Errorf("cannot determine snapshots path: %w", err)
+	}
+
+	f, err := os.Open(dir)
+	if os.IsNotExist(err) {
+		return litestream.NewSnapshotInfoSliceIterator(nil), nil
+	} else if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	fis, err := f.Readdir(-1)
+	if err != nil {
+		return nil, err
+	}
+
+	// Iterate over every file and convert to metadata.
+	infos := make([]litestream.SnapshotInfo, 0, len(fis))
+	for _, fi := range fis {
+		// Parse index from filename.
+		index, err := litestream.ParseSnapshotPath(fi.Name())
+		if err != nil {
+			continue
+		}
+
+		infos = append(infos, litestream.SnapshotInfo{
+			Generation: generation,
+			Index:      index,
+			Size:       fi.Size(),
+			CreatedAt:  fi.ModTime().UTC(),
+		})
+	}
+
+	sort.Sort(litestream.SnapshotInfoSlice(infos))
+
+	return litestream.NewSnapshotInfoSliceIterator(infos), nil
+}
+
+// WriteSnapshot writes LZ4 compressed data from rd into a file on disk.
+func (c *ReplicaClient) WriteSnapshot(ctx context.Context, generation string, index int, rd io.Reader) (info litestream.SnapshotInfo, err error) {
+	filename, err := c.SnapshotPath(generation, index)
+	if err != nil {
+		return info, fmt.Errorf("cannot determine snapshot path: %w", err)
+	}
+
+	var fileInfo, dirInfo os.FileInfo
+	if db := c.db(); db != nil {
+		fileInfo, dirInfo = db.FileInfo(), db.DirInfo()
+	}
+
+	// Ensure parent directory exists.
+	if err := internal.MkdirAll(filepath.Dir(filename), dirInfo); err != nil {
+		return info, err
+	}
+
+	// Write snapshot to temporary file next to destination path.
+	f, err := internal.CreateFile(filename+".tmp", fileInfo)
+	if err != nil {
+		return info, err
+	}
+	defer f.Close()
+
+	if _, err := io.Copy(f, rd); err != nil {
+		return info, err
+	} else if err := f.Sync(); err != nil {
+		return info, err
+	} else if err := f.Close(); err != nil {
+		return info, err
+	}
+
+	// Build metadata.
+	fi, err := os.Stat(filename + ".tmp")
+	if err != nil {
+		return info, err
+	}
+	info = litestream.SnapshotInfo{
+		Generation: generation,
+		Index:      index,
+		Size:       fi.Size(),
+		CreatedAt:  fi.ModTime().UTC(),
+	}
+
+	// Move snapshot to final path when it has been fully written & synced to disk.
+	if err := os.Rename(filename+".tmp", filename); err != nil {
+		return info, err
+	}
+
+	return info, nil
+}
+
+// SnapshotReader returns a reader for snapshot data at the given generation/index.
+// Returns os.ErrNotExist if no matching index is found.
+func (c *ReplicaClient) SnapshotReader(ctx context.Context, generation string, index int) (io.ReadCloser, error) {
+	filename, err := c.SnapshotPath(generation, index)
+	if err != nil {
+		return nil, fmt.Errorf("cannot determine snapshot path: %w", err)
+	}
+	return os.Open(filename)
+}
+
+// DeleteSnapshot deletes a snapshot with the given generation & index.
+func (c *ReplicaClient) DeleteSnapshot(ctx context.Context, generation string, index int) error {
+	filename, err := c.SnapshotPath(generation, index)
+	if err != nil {
+		return fmt.Errorf("cannot determine snapshot path: %w", err)
+	}
+	if err := os.Remove(filename); err != nil && !os.IsNotExist(err) {
+		return err
+	}
+	return nil
+}
+
+// WALSegments returns an iterator over all available WAL files for a generation.
+func (c *ReplicaClient) WALSegments(ctx context.Context, generation string) (litestream.WALSegmentIterator, error) {
+	dir, err := c.WALDir(generation)
+	if err != nil {
+		return nil, fmt.Errorf("cannot determine wal path: %w", err)
+	}
+
+	f, err := os.Open(dir)
+	if os.IsNotExist(err) {
+		return litestream.NewWALSegmentInfoSliceIterator(nil), nil
+	} else if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	fis, err := f.Readdir(-1)
+	if err != nil {
+		return nil, err
+	}
+
+	// Iterate over every file and convert to metadata.
+	infos := make([]litestream.WALSegmentInfo, 0, len(fis))
+	for _, fi := range fis {
+		// Parse index from filename.
+		index, offset, err := litestream.ParseWALSegmentPath(fi.Name())
+		if err != nil {
+			continue
+		}
+
+		infos = append(infos, litestream.WALSegmentInfo{
+			Generation: generation,
+			Index:      index,
+			Offset:     offset,
+			Size:       fi.Size(),
+			CreatedAt:  fi.ModTime().UTC(),
+		})
+	}
+
+	sort.Sort(litestream.WALSegmentInfoSlice(infos))
+
+	return litestream.NewWALSegmentInfoSliceIterator(infos), nil
+}
+
+// WriteWALSegment writes LZ4 compressed data from rd into a file on disk.
+func (c *ReplicaClient) WriteWALSegment(ctx context.Context, pos litestream.Pos, rd io.Reader) (info litestream.WALSegmentInfo, err error) {
+	filename, err := c.WALSegmentPath(pos.Generation, pos.Index, pos.Offset)
+	if err != nil {
+		return info, fmt.Errorf("cannot determine wal segment path: %w", err)
+	}
+
+	var fileInfo, dirInfo os.FileInfo
+	if db := c.db(); db != nil {
+		fileInfo, dirInfo = db.FileInfo(), db.DirInfo()
+	}
+
+	// Ensure parent directory exists.
+	if err := internal.MkdirAll(filepath.Dir(filename), dirInfo); err != nil {
+		return info, err
+	}
+
+	// Write WAL segment to temporary file next to destination path.
+	f, err := internal.CreateFile(filename+".tmp", fileInfo)
+	if err != nil {
+		return info, err
+	}
+	defer f.Close()
+
+	if _, err := io.Copy(f, rd); err != nil {
+		return info, err
+	} else if err := f.Sync(); err != nil {
+		return info, err
+	} else if err := f.Close(); err != nil {
+		return info, err
+	}
+
+	// Build metadata.
+	fi, err := os.Stat(filename + ".tmp")
+	if err != nil {
+		return info, err
+	}
+	info = litestream.WALSegmentInfo{
+		Generation: pos.Generation,
+		Index:      pos.Index,
+		Offset:     pos.Offset,
+		Size:       fi.Size(),
+		CreatedAt:  fi.ModTime().UTC(),
+	}
+
+	// Move WAL segment to final path when it has been written & synced to disk.
+	if err := os.Rename(filename+".tmp", filename); err != nil {
+		return info, err
+	}
+
+	return info, nil
+}
+
+// WALSegmentReader returns a reader for a section of WAL data at the given position.
+// Returns os.ErrNotExist if no matching index/offset is found.
+func (c *ReplicaClient) WALSegmentReader(ctx context.Context, pos litestream.Pos) (io.ReadCloser, error) {
+	filename, err := c.WALSegmentPath(pos.Generation, pos.Index, pos.Offset)
+	if err != nil {
+		return nil, fmt.Errorf("cannot determine wal segment path: %w", err)
+	}
+	return os.Open(filename)
+}
+
+// DeleteWALSegments deletes WAL segments at the given positions.
+func (c *ReplicaClient) DeleteWALSegments(ctx context.Context, a []litestream.Pos) error {
+	for _, pos := range a {
+		filename, err := c.WALSegmentPath(pos.Generation, pos.Index, pos.Offset)
+		if err != nil {
+			return fmt.Errorf("cannot determine wal segment path: %w", err)
+		}
+		if err := os.Remove(filename); err != nil && !os.IsNotExist(err) {
+			return err
+		}
+	}
+	return nil
+}

file/replica_client_test.go

@@ -0,0 +1,223 @@
+package file_test
+
+import (
+	"testing"
+
+	"github.com/benbjohnson/litestream/file"
+)
+
+func TestReplicaClient_Path(t *testing.T) {
+	c := file.NewReplicaClient("/foo/bar")
+	if got, want := c.Path(), "/foo/bar"; got != want {
+		t.Fatalf("Path()=%v, want %v", got, want)
+	}
+}
+
+func TestReplicaClient_Type(t *testing.T) {
+	if got, want := file.NewReplicaClient("").Type(), "file"; got != want {
+		t.Fatalf("Type()=%v, want %v", got, want)
+	}
+}
+
+func TestReplicaClient_GenerationsDir(t *testing.T) {
+	t.Run("OK", func(t *testing.T) {
+		if got, err := file.NewReplicaClient("/foo").GenerationsDir(); err != nil {
+			t.Fatal(err)
+		} else if want := "/foo/generations"; got != want {
+			t.Fatalf("GenerationsDir()=%v, want %v", got, want)
+		}
+	})
+	t.Run("ErrNoPath", func(t *testing.T) {
+		if _, err := file.NewReplicaClient("").GenerationsDir(); err == nil || err.Error() != `file replica path required` {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	})
+}
+
+func TestReplicaClient_GenerationDir(t *testing.T) {
+	t.Run("OK", func(t *testing.T) {
+		if got, err := file.NewReplicaClient("/foo").GenerationDir("0123456701234567"); err != nil {
+			t.Fatal(err)
+		} else if want := "/foo/generations/0123456701234567"; got != want {
+			t.Fatalf("GenerationDir()=%v, want %v", got, want)
+		}
+	})
+	t.Run("ErrNoPath", func(t *testing.T) {
+		if _, err := file.NewReplicaClient("").GenerationDir("0123456701234567"); err == nil || err.Error() != `file replica path required` {
+			t.Fatalf("expected error: %v", err)
+		}
+	})
+	t.Run("ErrNoGeneration", func(t *testing.T) {
+		if _, err := file.NewReplicaClient("/foo").GenerationDir(""); err == nil || err.Error() != `generation required` {
+			t.Fatalf("expected error: %v", err)
+		}
+	})
+}
+
+func TestReplicaClient_SnapshotsDir(t *testing.T) {
+	t.Run("OK", func(t *testing.T) {
+		if got, err := file.NewReplicaClient("/foo").SnapshotsDir("0123456701234567"); err != nil {
+			t.Fatal(err)
+		} else if want := "/foo/generations/0123456701234567/snapshots"; got != want {
+			t.Fatalf("SnapshotsDir()=%v, want %v", got, want)
+		}
+	})
+	t.Run("ErrNoPath", func(t *testing.T) {
+		if _, err := file.NewReplicaClient("").SnapshotsDir("0123456701234567"); err == nil || err.Error() != `file replica path required` {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	})
+	t.Run("ErrNoGeneration", func(t *testing.T) {
+		if _, err := file.NewReplicaClient("/foo").SnapshotsDir(""); err == nil || err.Error() != `generation required` {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	})
+}
+
+func TestReplicaClient_SnapshotPath(t *testing.T) {
+	t.Run("OK", func(t *testing.T) {
+		if got, err := file.NewReplicaClient("/foo").SnapshotPath("0123456701234567", 1000); err != nil {
+			t.Fatal(err)
+		} else if want := "/foo/generations/0123456701234567/snapshots/000003e8.snapshot.lz4"; got != want {
+			t.Fatalf("SnapshotPath()=%v, want %v", got, want)
+		}
+	})
+	t.Run("ErrNoPath", func(t *testing.T) {
+		if _, err := file.NewReplicaClient("").SnapshotPath("0123456701234567", 1000); err == nil || err.Error() != `file replica path required` {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	})
+	t.Run("ErrNoGeneration", func(t *testing.T) {
+		if _, err := file.NewReplicaClient("/foo").SnapshotPath("", 1000); err == nil || err.Error() != `generation required` {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	})
+}
+
+func TestReplicaClient_WALDir(t *testing.T) {
+	t.Run("OK", func(t *testing.T) {
+		if got, err := file.NewReplicaClient("/foo").WALDir("0123456701234567"); err != nil {
+			t.Fatal(err)
+		} else if want := "/foo/generations/0123456701234567/wal"; got != want {
+			t.Fatalf("WALDir()=%v, want %v", got, want)
+		}
+	})
+	t.Run("ErrNoPath", func(t *testing.T) {
+		if _, err := file.NewReplicaClient("").WALDir("0123456701234567"); err == nil || err.Error() != `file replica path required` {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	})
+	t.Run("ErrNoGeneration", func(t *testing.T) {
+		if _, err := file.NewReplicaClient("/foo").WALDir(""); err == nil || err.Error() != `generation required` {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	})
+}
+
+func TestReplicaClient_WALSegmentPath(t *testing.T) {
+	t.Run("OK", func(t *testing.T) {
+		if got, err := file.NewReplicaClient("/foo").WALSegmentPath("0123456701234567", 1000, 1001); err != nil {
+			t.Fatal(err)
+		} else if want := "/foo/generations/0123456701234567/wal/000003e8_000003e9.wal.lz4"; got != want {
+			t.Fatalf("WALPath()=%v, want %v", got, want)
+		}
+	})
+	t.Run("ErrNoPath", func(t *testing.T) {
+		if _, err := file.NewReplicaClient("").WALSegmentPath("0123456701234567", 1000, 0); err == nil || err.Error() != `file replica path required` {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	})
+	t.Run("ErrNoGeneration", func(t *testing.T) {
+		if _, err := file.NewReplicaClient("/foo").WALSegmentPath("", 1000, 0); err == nil || err.Error() != `generation required` {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	})
+}
+
+/*
+func TestReplica_Sync(t *testing.T) {
+	// Ensure replica can successfully sync after DB has sync'd.
+	t.Run("InitialSync", func(t *testing.T) {
+		db, sqldb := MustOpenDBs(t)
+		defer MustCloseDBs(t, db, sqldb)
+
+		r := litestream.NewReplica(db, "", file.NewReplicaClient(t.TempDir()))
+		r.MonitorEnabled = false
+		db.Replicas = []*litestream.Replica{r}
+
+		// Sync database & then sync replica.
+		if err := db.Sync(context.Background()); err != nil {
+			t.Fatal(err)
+		} else if err := r.Sync(context.Background()); err != nil {
+			t.Fatal(err)
+		}
+
+		// Ensure posistions match.
+		if want, err := db.Pos(); err != nil {
+			t.Fatal(err)
+		} else if got, err := r.Pos(context.Background()); err != nil {
+			t.Fatal(err)
+		} else if got != want {
+			t.Fatalf("Pos()=%v, want %v", got, want)
+		}
+	})
+
+	// Ensure replica can successfully sync multiple times.
+	t.Run("MultiSync", func(t *testing.T) {
+		db, sqldb := MustOpenDBs(t)
+		defer MustCloseDBs(t, db, sqldb)
+
+		r := litestream.NewReplica(db, "", file.NewReplicaClient(t.TempDir()))
+		r.MonitorEnabled = false
+		db.Replicas = []*litestream.Replica{r}
+
+		if _, err := sqldb.Exec(`CREATE TABLE foo (bar TEXT);`); err != nil {
+			t.Fatal(err)
+		}
+
+		// Write to the database multiple times and sync after each write.
+		for i, n := 0, db.MinCheckpointPageN*2; i < n; i++ {
+			if _, err := sqldb.Exec(`INSERT INTO foo (bar) VALUES ('baz')`); err != nil {
+				t.Fatal(err)
+			}
+
+			// Sync periodically.
+			if i%100 == 0 || i == n-1 {
+				if err := db.Sync(context.Background()); err != nil {
+					t.Fatal(err)
+				} else if err := r.Sync(context.Background()); err != nil {
+					t.Fatal(err)
+				}
+			}
+		}
+
+		// Ensure posistions match.
+		pos, err := db.Pos()
+		if err != nil {
+			t.Fatal(err)
+		} else if got, want := pos.Index, 2; got != want {
+			t.Fatalf("Index=%v, want %v", got, want)
+		}
+
+		if want, err := r.Pos(context.Background()); err != nil {
+			t.Fatal(err)
+		} else if got := pos; got != want {
+			t.Fatalf("Pos()=%v, want %v", got, want)
+		}
+	})
+
+	// Ensure replica returns an error if there is no generation available from the DB.
+	t.Run("ErrNoGeneration", func(t *testing.T) {
+		db, sqldb := MustOpenDBs(t)
+		defer MustCloseDBs(t, db, sqldb)
+
+		r := litestream.NewReplica(db, "", file.NewReplicaClient(t.TempDir()))
+		r.MonitorEnabled = false
+		db.Replicas = []*litestream.Replica{r}
+
+		if err := r.Sync(context.Background()); err == nil || err.Error() != `no generation, waiting for data` {
+			t.Fatal(err)
+		}
+	})
+}
+*/

gcs/replica_client.go

@@ -0,0 +1,428 @@
+package gcs
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"strings"
+	"sync"
+	"time"
+
+	"cloud.google.com/go/storage"
+	"github.com/benbjohnson/litestream"
+	"github.com/benbjohnson/litestream/internal"
+	"google.golang.org/api/iterator"
+)
+
+// ReplicaClientType is the client type for this package.
+const ReplicaClientType = "gcs"
+
+var _ litestream.ReplicaClient = (*ReplicaClient)(nil)
+
+// ReplicaClient is a client for writing snapshots & WAL segments to disk.
+type ReplicaClient struct {
+	mu     sync.Mutex
+	client *storage.Client       // gcs client
+	bkt    *storage.BucketHandle // gcs bucket handle
+
+	// GCS bucket information
+	Bucket string
+	Path   string
+}
+
+// NewReplicaClient returns a new instance of ReplicaClient.
+func NewReplicaClient() *ReplicaClient {
+	return &ReplicaClient{}
+}
+
+// Type returns "gcs" as the client type.
+func (c *ReplicaClient) Type() string {
+	return ReplicaClientType
+}
+
+// Init initializes the connection to GCS. No-op if already initialized.
+func (c *ReplicaClient) Init(ctx context.Context) (err error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if c.client != nil {
+		return nil
+	}
+
+	if c.client, err = storage.NewClient(ctx); err != nil {
+		return err
+	}
+	c.bkt = c.client.Bucket(c.Bucket)
+
+	return nil
+}
+
+// Generations returns a list of available generation names.
+func (c *ReplicaClient) Generations(ctx context.Context) ([]string, error) {
+	if err := c.Init(ctx); err != nil {
+		return nil, err
+	}
+
+	// Construct query to only pull generation directory names.
+	query := &storage.Query{
+		Delimiter: "/",
+		Prefix:    litestream.GenerationsPath(c.Path) + "/",
+	}
+
+	// Loop over results and only build list of generation-formatted names.
+	it := c.bkt.Objects(ctx, query)
+	var generations []string
+	for {
+		attrs, err := it.Next()
+		if err == iterator.Done {
+			break
+		} else if err != nil {
+			return nil, err
+		}
+
+		name := path.Base(strings.TrimSuffix(attrs.Prefix, "/"))
+		if !litestream.IsGenerationName(name) {
+			continue
+		}
+		generations = append(generations, name)
+	}
+
+	return generations, nil
+}
+
+// DeleteGeneration deletes all snapshots & WAL segments within a generation.
+func (c *ReplicaClient) DeleteGeneration(ctx context.Context, generation string) error {
+	if err := c.Init(ctx); err != nil {
+		return err
+	}
+
+	dir, err := litestream.GenerationPath(c.Path, generation)
+	if err != nil {
+		return fmt.Errorf("cannot determine generation path: %w", err)
+	}
+
+	// Iterate over every object in generation and delete it.
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "LIST").Inc()
+	for it := c.bkt.Objects(ctx, &storage.Query{Prefix: dir + "/"}); ; {
+		attrs, err := it.Next()
+		if err == iterator.Done {
+			break
+		} else if err != nil {
+			return err
+		}
+
+		if err := c.bkt.Object(attrs.Name).Delete(ctx); isNotExists(err) {
+			continue
+		} else if err != nil {
+			return fmt.Errorf("cannot delete object %q: %w", attrs.Name, err)
+		}
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "DELETE").Inc()
+	}
+
+	// log.Printf("%s(%s): retainer: deleting generation: %s", r.db.Path(), r.Name(), generation)
+
+	return nil
+}
+
+// Snapshots returns an iterator over all available snapshots for a generation.
+func (c *ReplicaClient) Snapshots(ctx context.Context, generation string) (litestream.SnapshotIterator, error) {
+	if err := c.Init(ctx); err != nil {
+		return nil, err
+	}
+	dir, err := litestream.SnapshotsPath(c.Path, generation)
+	if err != nil {
+		return nil, fmt.Errorf("cannot determine snapshots path: %w", err)
+	}
+	return newSnapshotIterator(generation, c.bkt.Objects(ctx, &storage.Query{Prefix: dir + "/"})), nil
+}
+
+// WriteSnapshot writes LZ4 compressed data from rd to the object storage.
+func (c *ReplicaClient) WriteSnapshot(ctx context.Context, generation string, index int, rd io.Reader) (info litestream.SnapshotInfo, err error) {
+	if err := c.Init(ctx); err != nil {
+		return info, err
+	}
+
+	key, err := litestream.SnapshotPath(c.Path, generation, index)
+	if err != nil {
+		return info, fmt.Errorf("cannot determine snapshot path: %w", err)
+	}
+	startTime := time.Now()
+
+	w := c.bkt.Object(key).NewWriter(ctx)
+	defer w.Close()
+
+	n, err := io.Copy(w, rd)
+	if err != nil {
+		return info, err
+	} else if err := w.Close(); err != nil {
+		return info, err
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "PUT").Inc()
+	internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "PUT").Add(float64(n))
+
+	// log.Printf("%s(%s): snapshot: creating %s/%08x t=%s", r.db.Path(), r.Name(), generation, index, time.Since(startTime).Truncate(time.Millisecond))
+
+	return litestream.SnapshotInfo{
+		Generation: generation,
+		Index:      index,
+		Size:       n,
+		CreatedAt:  startTime.UTC(),
+	}, nil
+}
+
+// SnapshotReader returns a reader for snapshot data at the given generation/index.
+func (c *ReplicaClient) SnapshotReader(ctx context.Context, generation string, index int) (io.ReadCloser, error) {
+	if err := c.Init(ctx); err != nil {
+		return nil, err
+	}
+
+	key, err := litestream.SnapshotPath(c.Path, generation, index)
+	if err != nil {
+		return nil, fmt.Errorf("cannot determine snapshot path: %w", err)
+	}
+
+	r, err := c.bkt.Object(key).NewReader(ctx)
+	if isNotExists(err) {
+		return nil, os.ErrNotExist
+	} else if err != nil {
+		return nil, fmt.Errorf("cannot start new reader for %q: %w", key, err)
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "GET").Inc()
+	internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "GET").Add(float64(r.Attrs.Size))
+
+	return r, nil
+}
+
+// DeleteSnapshot deletes a snapshot with the given generation & index.
+func (c *ReplicaClient) DeleteSnapshot(ctx context.Context, generation string, index int) error {
+	if err := c.Init(ctx); err != nil {
+		return err
+	}
+
+	key, err := litestream.SnapshotPath(c.Path, generation, index)
+	if err != nil {
+		return fmt.Errorf("cannot determine snapshot path: %w", err)
+	}
+
+	if err := c.bkt.Object(key).Delete(ctx); err != nil && !isNotExists(err) {
+		return fmt.Errorf("cannot delete snapshot %q: %w", key, err)
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "DELETE").Inc()
+	return nil
+}
+
+// WALSegments returns an iterator over all available WAL files for a generation.
+func (c *ReplicaClient) WALSegments(ctx context.Context, generation string) (litestream.WALSegmentIterator, error) {
+	if err := c.Init(ctx); err != nil {
+		return nil, err
+	}
+	dir, err := litestream.WALPath(c.Path, generation)
+	if err != nil {
+		return nil, fmt.Errorf("cannot determine wal path: %w", err)
+	}
+	return newWALSegmentIterator(generation, c.bkt.Objects(ctx, &storage.Query{Prefix: dir + "/"})), nil
+}
+
+// WriteWALSegment writes LZ4 compressed data from rd into a file on disk.
+func (c *ReplicaClient) WriteWALSegment(ctx context.Context, pos litestream.Pos, rd io.Reader) (info litestream.WALSegmentInfo, err error) {
+	if err := c.Init(ctx); err != nil {
+		return info, err
+	}
+
+	key, err := litestream.WALSegmentPath(c.Path, pos.Generation, pos.Index, pos.Offset)
+	if err != nil {
+		return info, fmt.Errorf("cannot determine wal segment path: %w", err)
+	}
+	startTime := time.Now()
+
+	w := c.bkt.Object(key).NewWriter(ctx)
+	defer w.Close()
+
+	n, err := io.Copy(w, rd)
+	if err != nil {
+		return info, err
+	} else if err := w.Close(); err != nil {
+		return info, err
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "PUT").Inc()
+	internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "PUT").Add(float64(n))
+
+	return litestream.WALSegmentInfo{
+		Generation: pos.Generation,
+		Index:      pos.Index,
+		Offset:     pos.Offset,
+		Size:       n,
+		CreatedAt:  startTime.UTC(),
+	}, nil
+}
+
+// WALSegmentReader returns a reader for a section of WAL data at the given index.
+// Returns os.ErrNotExist if no matching index/offset is found.
+func (c *ReplicaClient) WALSegmentReader(ctx context.Context, pos litestream.Pos) (io.ReadCloser, error) {
+	if err := c.Init(ctx); err != nil {
+		return nil, err
+	}
+
+	key, err := litestream.WALSegmentPath(c.Path, pos.Generation, pos.Index, pos.Offset)
+	if err != nil {
+		return nil, fmt.Errorf("cannot determine wal segment path: %w", err)
+	}
+
+	r, err := c.bkt.Object(key).NewReader(ctx)
+	if isNotExists(err) {
+		return nil, os.ErrNotExist
+	} else if err != nil {
+		return nil, err
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "GET").Inc()
+	internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "GET").Add(float64(r.Attrs.Size))
+
+	return r, nil
+}
+
+// DeleteWALSegments deletes WAL segments with at the given positions.
+func (c *ReplicaClient) DeleteWALSegments(ctx context.Context, a []litestream.Pos) error {
+	if err := c.Init(ctx); err != nil {
+		return err
+	}
+
+	for _, pos := range a {
+		key, err := litestream.WALSegmentPath(c.Path, pos.Generation, pos.Index, pos.Offset)
+		if err != nil {
+			return fmt.Errorf("cannot determine wal segment path: %w", err)
+		}
+
+		if err := c.bkt.Object(key).Delete(ctx); err != nil && !isNotExists(err) {
+			return fmt.Errorf("cannot delete wal segment %q: %w", key, err)
+		}
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "DELETE").Inc()
+	}
+
+	return nil
+}
+
+type snapshotIterator struct {
+	generation string
+
+	it   *storage.ObjectIterator
+	info litestream.SnapshotInfo
+	err  error
+}
+
+func newSnapshotIterator(generation string, it *storage.ObjectIterator) *snapshotIterator {
+	return &snapshotIterator{
+		generation: generation,
+		it:         it,
+	}
+}
+
+func (itr *snapshotIterator) Close() (err error) {
+	return itr.err
+}
+
+func (itr *snapshotIterator) Next() bool {
+	// Exit if an error has already occurred.
+	if itr.err != nil {
+		return false
+	}
+
+	for {
+		// Fetch next object.
+		attrs, err := itr.it.Next()
+		if err == iterator.Done {
+			return false
+		} else if err != nil {
+			itr.err = err
+			return false
+		}
+
+		// Parse index, otherwise skip to the next object.
+		index, err := litestream.ParseSnapshotPath(path.Base(attrs.Name))
+		if err != nil {
+			continue
+		}
+
+		// Store current snapshot and return.
+		itr.info = litestream.SnapshotInfo{
+			Generation: itr.generation,
+			Index:      index,
+			Size:       attrs.Size,
+			CreatedAt:  attrs.Created.UTC(),
+		}
+		return true
+	}
+}
+
+func (itr *snapshotIterator) Err() error { return itr.err }
+
+func (itr *snapshotIterator) Snapshot() litestream.SnapshotInfo { return itr.info }
+
+type walSegmentIterator struct {
+	generation string
+
+	it   *storage.ObjectIterator
+	info litestream.WALSegmentInfo
+	err  error
+}
+
+func newWALSegmentIterator(generation string, it *storage.ObjectIterator) *walSegmentIterator {
+	return &walSegmentIterator{
+		generation: generation,
+		it:         it,
+	}
+}
+
+func (itr *walSegmentIterator) Close() (err error) {
+	return itr.err
+}
+
+func (itr *walSegmentIterator) Next() bool {
+	// Exit if an error has already occurred.
+	if itr.err != nil {
+		return false
+	}
+
+	for {
+		// Fetch next object.
+		attrs, err := itr.it.Next()
+		if err == iterator.Done {
+			return false
+		} else if err != nil {
+			itr.err = err
+			return false
+		}
+
+		// Parse index & offset, otherwise skip to the next object.
+		index, offset, err := litestream.ParseWALSegmentPath(path.Base(attrs.Name))
+		if err != nil {
+			continue
+		}
+
+		// Store current snapshot and return.
+		itr.info = litestream.WALSegmentInfo{
+			Generation: itr.generation,
+			Index:      index,
+			Offset:     offset,
+			Size:       attrs.Size,
+			CreatedAt:  attrs.Created.UTC(),
+		}
+		return true
+	}
+}
+
+func (itr *walSegmentIterator) Err() error { return itr.err }
+
+func (itr *walSegmentIterator) WALSegment() litestream.WALSegmentInfo {
+	return itr.info
+}
+
+func isNotExists(err error) bool {
+	return err == storage.ErrObjectNotExist
+}

go.mod

@@ -1,13 +1,51 @@
 module github.com/benbjohnson/litestream
 
-go 1.15
+go 1.19
 
 require (
-	github.com/aws/aws-sdk-go v1.27.0
+	cloud.google.com/go/storage v1.24.0
-	github.com/davecgh/go-spew v1.1.1
+	filippo.io/age v1.1.1
-	github.com/mattn/go-sqlite3 v1.14.5
+	github.com/Azure/azure-storage-blob-go v0.15.0
-	github.com/pierrec/lz4/v4 v4.1.3
+	github.com/aws/aws-sdk-go v1.44.71
-	github.com/prometheus/client_golang v1.9.0
+	github.com/mattn/go-shellwords v1.0.12
-	golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e
+	github.com/mattn/go-sqlite3 v1.14.14
+	github.com/pierrec/lz4/v4 v4.1.15
+	github.com/pkg/sftp v1.13.5
+	github.com/prometheus/client_golang v1.13.0
+	golang.org/x/crypto v0.4.0
+	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4
+	golang.org/x/sys v0.3.0
+	google.golang.org/api v0.91.0
 	gopkg.in/yaml.v2 v2.4.0
 )
+
+require (
+	cloud.google.com/go v0.103.0 // indirect
+	cloud.google.com/go/compute v1.7.0 // indirect
+	cloud.google.com/go/iam v0.3.0 // indirect
+	github.com/Azure/azure-pipeline-go v0.2.3 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/go-cmp v0.5.8 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.1.0 // indirect
+	github.com/googleapis/gax-go/v2 v2.5.1 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/kr/fs v0.1.0 // indirect
+	github.com/mattn/go-ieproxy v0.0.7 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
+	github.com/prometheus/client_model v0.2.0 // indirect
+	github.com/prometheus/common v0.37.0 // indirect
+	github.com/prometheus/procfs v0.8.0 // indirect
+	go.opencensus.io v0.23.0 // indirect
+	golang.org/x/net v0.3.0 // indirect
+	golang.org/x/oauth2 v0.0.0-20220808172628-8227340efae7 // indirect
+	golang.org/x/text v0.5.0 // indirect
+	golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/genproto v0.0.0-20220808204814-fd01256a5276 // indirect
+	google.golang.org/grpc v1.48.0 // indirect
+	google.golang.org/protobuf v1.28.1 // indirect
+)

internal/internal.go

@@ -2,6 +2,11 @@ package internal
 
 import (
 	"io"
+	"os"
+	"syscall"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 )
 
 // ReadCloser wraps a reader to also attach a separate closer.
@@ -30,3 +35,107 @@ func (r *ReadCloser) Close() error {
 	}
 	return r.c.Close()
 }
+
+// ReadCounter wraps an io.Reader and counts the total number of bytes read.
+type ReadCounter struct {
+	r io.Reader
+	n int64
+}
+
+// NewReadCounter returns a new instance of ReadCounter that wraps r.
+func NewReadCounter(r io.Reader) *ReadCounter {
+	return &ReadCounter{r: r}
+}
+
+// Read reads from the underlying reader into p and adds the bytes read to the counter.
+func (r *ReadCounter) Read(p []byte) (int, error) {
+	n, err := r.r.Read(p)
+	r.n += int64(n)
+	return n, err
+}
+
+// N returns the total number of bytes read.
+func (r *ReadCounter) N() int64 { return r.n }
+
+// CreateFile creates the file and matches the mode & uid/gid of fi.
+func CreateFile(filename string, fi os.FileInfo) (*os.File, error) {
+	mode := os.FileMode(0600)
+	if fi != nil {
+		mode = fi.Mode()
+	}
+
+	f, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode)
+	if err != nil {
+		return nil, err
+	}
+
+	uid, gid := Fileinfo(fi)
+	_ = f.Chown(uid, gid)
+	return f, nil
+}
+
+// MkdirAll is a copy of os.MkdirAll() except that it attempts to set the
+// mode/uid/gid to match fi for each created directory.
+func MkdirAll(path string, fi os.FileInfo) error {
+	uid, gid := Fileinfo(fi)
+
+	// Fast path: if we can tell whether path is a directory or file, stop with success or error.
+	dir, err := os.Stat(path)
+	if err == nil {
+		if dir.IsDir() {
+			return nil
+		}
+		return &os.PathError{Op: "mkdir", Path: path, Err: syscall.ENOTDIR}
+	}
+
+	// Slow path: make sure parent exists and then call Mkdir for path.
+	i := len(path)
+	for i > 0 && os.IsPathSeparator(path[i-1]) { // Skip trailing path separator.
+		i--
+	}
+
+	j := i
+	for j > 0 && !os.IsPathSeparator(path[j-1]) { // Scan backward over element.
+		j--
+	}
+
+	if j > 1 {
+		// Create parent.
+		err = MkdirAll(fixRootDirectory(path[:j-1]), fi)
+		if err != nil {
+			return err
+		}
+	}
+
+	// Parent now exists; invoke Mkdir and use its result.
+	mode := os.FileMode(0700)
+	if fi != nil {
+		mode = fi.Mode()
+	}
+	err = os.Mkdir(path, mode)
+	if err != nil {
+		// Handle arguments like "foo/." by
+		// double-checking that directory doesn't exist.
+		dir, err1 := os.Lstat(path)
+		if err1 == nil && dir.IsDir() {
+			_ = os.Chown(path, uid, gid)
+			return nil
+		}
+		return err
+	}
+	_ = os.Chown(path, uid, gid)
+	return nil
+}
+
+// Shared replica metrics.
+var (
+	OperationTotalCounterVec = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: "litestream_replica_operation_total",
+		Help: "The number of replica operations performed",
+	}, []string{"replica_type", "operation"})
+
+	OperationBytesCounterVec = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: "litestream_replica_operation_bytes",
+		Help: "The number of bytes used by replica operations",
+	}, []string{"replica_type", "operation"})
+)

internal/internal_unix.go

@@ -1,14 +1,17 @@
 // +build aix darwin dragonfly freebsd linux netbsd openbsd solaris
 
-package litestream
+package internal
 
 import (
 	"os"
 	"syscall"
 )
 
-// fileinfo returns syscall fields from a FileInfo object.
+// Fileinfo returns syscall fields from a FileInfo object.
-func fileinfo(fi os.FileInfo) (uid, gid int) {
+func Fileinfo(fi os.FileInfo) (uid, gid int) {
+	if fi == nil {
+		return -1, -1
+	}
 	stat := fi.Sys().(*syscall.Stat_t)
 	return int(stat.Uid), int(stat.Gid)
 }

internal/internal_windows.go

@@ -1,13 +1,13 @@
 // +build windows
 
-package litestream
+package internal
 
 import (
 	"os"
 )
 
-// fileinfo returns syscall fields from a FileInfo object.
+// Fileinfo returns syscall fields from a FileInfo object.
-func fileinfo(fi os.FileInfo) (uid, gid int) {
+func Fileinfo(fi os.FileInfo) (uid, gid int) {
 	return -1, -1
 }
 

internal/metrics.go

@@ -1,44 +0,0 @@
-package internal
-
-import (
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
-)
-
-// Shared replica metrics.
-var (
-	ReplicaSnapshotTotalGaugeVec = promauto.NewGaugeVec(prometheus.GaugeOpts{
-		Namespace: "litestream",
-		Subsystem: "replica",
-		Name:      "snapshot_total",
-		Help:      "The current number of snapshots",
-	}, []string{"db", "name"})
-
-	ReplicaWALBytesCounterVec = promauto.NewCounterVec(prometheus.CounterOpts{
-		Namespace: "litestream",
-		Subsystem: "replica",
-		Name:      "wal_bytes",
-		Help:      "The number wal bytes written",
-	}, []string{"db", "name"})
-
-	ReplicaWALIndexGaugeVec = promauto.NewGaugeVec(prometheus.GaugeOpts{
-		Namespace: "litestream",
-		Subsystem: "replica",
-		Name:      "wal_index",
-		Help:      "The current WAL index",
-	}, []string{"db", "name"})
-
-	ReplicaWALOffsetGaugeVec = promauto.NewGaugeVec(prometheus.GaugeOpts{
-		Namespace: "litestream",
-		Subsystem: "replica",
-		Name:      "wal_offset",
-		Help:      "The current WAL offset",
-	}, []string{"db", "name"})
-
-	ReplicaValidationTotalCounterVec = promauto.NewCounterVec(prometheus.CounterOpts{
-		Namespace: "litestream",
-		Subsystem: "replica",
-		Name:      "validation_total",
-		Help:      "The number of validations performed",
-	}, []string{"db", "name", "status"})
-)

litestream.go

@@ -7,21 +7,24 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path"
 	"path/filepath"
 	"regexp"
 	"strconv"
 	"strings"
-	"syscall"
 	"time"
+
+	"github.com/mattn/go-sqlite3"
 )
 
 // Naming constants.
 const (
 	MetaDirSuffix = "-litestream"
 
-	WALDirName  = "wal"
+	WALDirName    = "wal"
-	WALExt      = ".wal"
+	WALExt        = ".wal"
-	SnapshotExt = ".snapshot"
+	WALSegmentExt = ".wal.lz4"
+	SnapshotExt   = ".snapshot.lz4"
 
 	GenerationNameLen = 16
 )
@@ -41,19 +44,179 @@ var (
 	ErrChecksumMismatch = errors.New("invalid replica, checksum mismatch")
 )
 
+var (
+	// LogWriter is the destination writer for all logging.
+	LogWriter = os.Stdout
+
+	// LogFlags are the flags passed to log.New().
+	LogFlags = 0
+)
+
+func init() {
+	sql.Register("litestream-sqlite3", &sqlite3.SQLiteDriver{
+		ConnectHook: func(conn *sqlite3.SQLiteConn) error {
+			if err := conn.SetFileControlInt("main", sqlite3.SQLITE_FCNTL_PERSIST_WAL, 1); err != nil {
+				return fmt.Errorf("cannot set file control: %w", err)
+			}
+			return nil
+		},
+	})
+}
+
+// SnapshotIterator represents an iterator over a collection of snapshot metadata.
+type SnapshotIterator interface {
+	io.Closer
+
+	// Prepares the the next snapshot for reading with the Snapshot() method.
+	// Returns true if another snapshot is available. Returns false if no more
+	// snapshots are available or if an error occured.
+	Next() bool
+
+	// Returns an error that occurred during iteration.
+	Err() error
+
+	// Returns metadata for the currently positioned snapshot.
+	Snapshot() SnapshotInfo
+}
+
+// SliceSnapshotIterator returns all snapshots from an iterator as a slice.
+func SliceSnapshotIterator(itr SnapshotIterator) ([]SnapshotInfo, error) {
+	var a []SnapshotInfo
+	for itr.Next() {
+		a = append(a, itr.Snapshot())
+	}
+	return a, itr.Close()
+}
+
+var _ SnapshotIterator = (*SnapshotInfoSliceIterator)(nil)
+
+// SnapshotInfoSliceIterator represents an iterator for iterating over a slice of snapshots.
+type SnapshotInfoSliceIterator struct {
+	init bool
+	a    []SnapshotInfo
+}
+
+// NewSnapshotInfoSliceIterator returns a new instance of SnapshotInfoSliceIterator.
+func NewSnapshotInfoSliceIterator(a []SnapshotInfo) *SnapshotInfoSliceIterator {
+	return &SnapshotInfoSliceIterator{a: a}
+}
+
+// Close always returns nil.
+func (itr *SnapshotInfoSliceIterator) Close() error { return nil }
+
+// Next moves to the next snapshot. Returns true if another snapshot is available.
+func (itr *SnapshotInfoSliceIterator) Next() bool {
+	if !itr.init {
+		itr.init = true
+		return len(itr.a) > 0
+	}
+	itr.a = itr.a[1:]
+	return len(itr.a) > 0
+}
+
+// Err always returns nil.
+func (itr *SnapshotInfoSliceIterator) Err() error { return nil }
+
+// Snapshot returns the metadata from the currently positioned snapshot.
+func (itr *SnapshotInfoSliceIterator) Snapshot() SnapshotInfo {
+	if len(itr.a) == 0 {
+		return SnapshotInfo{}
+	}
+	return itr.a[0]
+}
+
+// WALSegmentIterator represents an iterator over a collection of WAL segments.
+type WALSegmentIterator interface {
+	io.Closer
+
+	// Prepares the the next WAL for reading with the WAL() method.
+	// Returns true if another WAL is available. Returns false if no more
+	// WAL files are available or if an error occured.
+	Next() bool
+
+	// Returns an error that occurred during iteration.
+	Err() error
+
+	// Returns metadata for the currently positioned WAL segment file.
+	WALSegment() WALSegmentInfo
+}
+
+// SliceWALSegmentIterator returns all WAL segment files from an iterator as a slice.
+func SliceWALSegmentIterator(itr WALSegmentIterator) ([]WALSegmentInfo, error) {
+	var a []WALSegmentInfo
+	for itr.Next() {
+		a = append(a, itr.WALSegment())
+	}
+	return a, itr.Close()
+}
+
+var _ WALSegmentIterator = (*WALSegmentInfoSliceIterator)(nil)
+
+// WALSegmentInfoSliceIterator represents an iterator for iterating over a slice of wal segments.
+type WALSegmentInfoSliceIterator struct {
+	init bool
+	a    []WALSegmentInfo
+}
+
+// NewWALSegmentInfoSliceIterator returns a new instance of WALSegmentInfoSliceIterator.
+func NewWALSegmentInfoSliceIterator(a []WALSegmentInfo) *WALSegmentInfoSliceIterator {
+	return &WALSegmentInfoSliceIterator{a: a}
+}
+
+// Close always returns nil.
+func (itr *WALSegmentInfoSliceIterator) Close() error { return nil }
+
+// Next moves to the next wal segment. Returns true if another segment is available.
+func (itr *WALSegmentInfoSliceIterator) Next() bool {
+	if !itr.init {
+		itr.init = true
+		return len(itr.a) > 0
+	}
+	itr.a = itr.a[1:]
+	return len(itr.a) > 0
+}
+
+// Err always returns nil.
+func (itr *WALSegmentInfoSliceIterator) Err() error { return nil }
+
+// WALSegment returns the metadata from the currently positioned wal segment.
+func (itr *WALSegmentInfoSliceIterator) WALSegment() WALSegmentInfo {
+	if len(itr.a) == 0 {
+		return WALSegmentInfo{}
+	}
+	return itr.a[0]
+}
+
 // SnapshotInfo represents file information about a snapshot.
 type SnapshotInfo struct {
-	Name       string
-	Replica    string
 	Generation string
 	Index      int
 	Size       int64
 	CreatedAt  time.Time
 }
 
+// Pos returns the WAL position when the snapshot was made.
+func (info *SnapshotInfo) Pos() Pos {
+	return Pos{Generation: info.Generation, Index: info.Index}
+}
+
+// SnapshotInfoSlice represents a slice of snapshot metadata.
+type SnapshotInfoSlice []SnapshotInfo
+
+func (a SnapshotInfoSlice) Len() int { return len(a) }
+
+func (a SnapshotInfoSlice) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+
+func (a SnapshotInfoSlice) Less(i, j int) bool {
+	if a[i].Generation != a[j].Generation {
+		return a[i].Generation < a[j].Generation
+	}
+	return a[i].Index < a[j].Index
+}
+
 // FilterSnapshotsAfter returns all snapshots that were created on or after t.
-func FilterSnapshotsAfter(a []*SnapshotInfo, t time.Time) []*SnapshotInfo {
+func FilterSnapshotsAfter(a []SnapshotInfo, t time.Time) []SnapshotInfo {
-	other := make([]*SnapshotInfo, 0, len(a))
+	other := make([]SnapshotInfo, 0, len(a))
 	for _, snapshot := range a {
 		if !snapshot.CreatedAt.Before(t) {
 			other = append(other, snapshot)
@@ -63,9 +226,11 @@ func FilterSnapshotsAfter(a []*SnapshotInfo, t time.Time) []*SnapshotInfo {
 }
 
 // FindMinSnapshotByGeneration finds the snapshot with the lowest index in a generation.
-func FindMinSnapshotByGeneration(a []*SnapshotInfo, generation string) *SnapshotInfo {
+func FindMinSnapshotByGeneration(a []SnapshotInfo, generation string) *SnapshotInfo {
 	var min *SnapshotInfo
-	for _, snapshot := range a {
+	for i := range a {
+		snapshot := &a[i]
+
 		if snapshot.Generation != generation {
 			continue
 		} else if min == nil || snapshot.Index < min.Index {
@@ -77,8 +242,27 @@ func FindMinSnapshotByGeneration(a []*SnapshotInfo, generation string) *Snapshot
 
 // WALInfo represents file information about a WAL file.
 type WALInfo struct {
-	Name       string
+	Generation string
-	Replica    string
+	Index      int
+	CreatedAt  time.Time
+}
+
+// WALInfoSlice represents a slice of WAL metadata.
+type WALInfoSlice []WALInfo
+
+func (a WALInfoSlice) Len() int { return len(a) }
+
+func (a WALInfoSlice) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+
+func (a WALInfoSlice) Less(i, j int) bool {
+	if a[i].Generation != a[j].Generation {
+		return a[i].Generation < a[j].Generation
+	}
+	return a[i].Index < a[j].Index
+}
+
+// WALSegmentInfo represents file information about a WAL segment file.
+type WALSegmentInfo struct {
 	Generation string
 	Index      int
 	Offset     int64
@@ -86,6 +270,27 @@ type WALInfo struct {
 	CreatedAt  time.Time
 }
 
+// Pos returns the WAL position when the segment was made.
+func (info *WALSegmentInfo) Pos() Pos {
+	return Pos{Generation: info.Generation, Index: info.Index, Offset: info.Offset}
+}
+
+// WALSegmentInfoSlice represents a slice of WAL segment metadata.
+type WALSegmentInfoSlice []WALSegmentInfo
+
+func (a WALSegmentInfoSlice) Len() int { return len(a) }
+
+func (a WALSegmentInfoSlice) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+
+func (a WALSegmentInfoSlice) Less(i, j int) bool {
+	if a[i].Generation != a[j].Generation {
+		return a[i].Generation < a[j].Generation
+	} else if a[i].Index != a[j].Index {
+		return a[i].Index < a[j].Index
+	}
+	return a[i].Offset < a[j].Offset
+}
+
 // Pos is a position in the WAL for a generation.
 type Pos struct {
 	Generation string // generation name
@@ -106,6 +311,11 @@ func (p Pos) IsZero() bool {
 	return p == (Pos{})
 }
 
+// Truncate returns p with the offset truncated to zero.
+func (p Pos) Truncate() Pos {
+	return Pos{Generation: p.Generation, Index: p.Index}
+}
+
 // Checksum computes a running SQLite checksum over a byte slice.
 func Checksum(bo binary.ByteOrder, s0, s1 uint32, b []byte) (uint32, uint32) {
 	assert(len(b)%8 == 0, "misaligned checksum byte slice")
@@ -197,6 +407,56 @@ func IsGenerationName(s string) bool {
 	return true
 }
 
+// GenerationsPath returns the path to a generation root directory.
+func GenerationsPath(root string) string {
+	return path.Join(root, "generations")
+}
+
+// GenerationPath returns the path to a generation's root directory.
+func GenerationPath(root, generation string) (string, error) {
+	dir := GenerationsPath(root)
+	if generation == "" {
+		return "", fmt.Errorf("generation required")
+	}
+	return path.Join(dir, generation), nil
+}
+
+// SnapshotsPath returns the path to a generation's snapshot directory.
+func SnapshotsPath(root, generation string) (string, error) {
+	dir, err := GenerationPath(root, generation)
+	if err != nil {
+		return "", err
+	}
+	return path.Join(dir, "snapshots"), nil
+}
+
+// SnapshotPath returns the path to an uncompressed snapshot file.
+func SnapshotPath(root, generation string, index int) (string, error) {
+	dir, err := SnapshotsPath(root, generation)
+	if err != nil {
+		return "", err
+	}
+	return path.Join(dir, FormatSnapshotPath(index)), nil
+}
+
+// WALPath returns the path to a generation's WAL directory
+func WALPath(root, generation string) (string, error) {
+	dir, err := GenerationPath(root, generation)
+	if err != nil {
+		return "", err
+	}
+	return path.Join(dir, "wal"), nil
+}
+
+// WALSegmentPath returns the path to a WAL segment file.
+func WALSegmentPath(root, generation string, index int, offset int64) (string, error) {
+	dir, err := WALPath(root, generation)
+	if err != nil {
+		return "", err
+	}
+	return path.Join(dir, FormatWALSegmentPath(index, offset)), nil
+}
+
 // IsSnapshotPath returns true if s is a path to a snapshot file.
 func IsSnapshotPath(s string) bool {
 	return snapshotPathRegex.MatchString(s)
@@ -204,38 +464,43 @@ func IsSnapshotPath(s string) bool {
 
 // ParseSnapshotPath returns the index for the snapshot.
 // Returns an error if the path is not a valid snapshot path.
-func ParseSnapshotPath(s string) (index int, ext string, err error) {
+func ParseSnapshotPath(s string) (index int, err error) {
 	s = filepath.Base(s)
 
 	a := snapshotPathRegex.FindStringSubmatch(s)
 	if a == nil {
-		return 0, "", fmt.Errorf("invalid snapshot path: %s", s)
+		return 0, fmt.Errorf("invalid snapshot path: %s", s)
 	}
 
 	i64, _ := strconv.ParseUint(a[1], 16, 64)
-	return int(i64), a[2], nil
+	return int(i64), nil
 }
 
-var snapshotPathRegex = regexp.MustCompile(`^([0-9a-f]{8})(.snapshot(?:.lz4)?)$`)
+// FormatSnapshotPath formats a snapshot filename with a given index.
+func FormatSnapshotPath(index int) string {
+	assert(index >= 0, "snapshot index must be non-negative")
+	return fmt.Sprintf("%08x%s", index, SnapshotExt)
+}
+
+var snapshotPathRegex = regexp.MustCompile(`^([0-9a-f]{8})\.snapshot\.lz4$`)
 
 // IsWALPath returns true if s is a path to a WAL file.
 func IsWALPath(s string) bool {
 	return walPathRegex.MatchString(s)
 }
 
-// ParseWALPath returns the index & offset for the WAL file.
+// ParseWALPath returns the index for the WAL file.
-// Returns an error if the path is not a valid snapshot path.
+// Returns an error if the path is not a valid WAL path.
-func ParseWALPath(s string) (index int, offset int64, ext string, err error) {
+func ParseWALPath(s string) (index int, err error) {
 	s = filepath.Base(s)
 
 	a := walPathRegex.FindStringSubmatch(s)
 	if a == nil {
-		return 0, 0, "", fmt.Errorf("invalid wal path: %s", s)
+		return 0, fmt.Errorf("invalid wal path: %s", s)
 	}
 
 	i64, _ := strconv.ParseUint(a[1], 16, 64)
-	off64, _ := strconv.ParseUint(a[2], 16, 64)
+	return int(i64), nil
-	return int(i64), int64(off64), a[3], nil
 }
 
 // FormatWALPath formats a WAL filename with a given index.
@@ -244,77 +509,37 @@ func FormatWALPath(index int) string {
 	return fmt.Sprintf("%08x%s", index, WALExt)
 }
 
-// FormatWALPathWithOffset formats a WAL filename with a given index & offset.
+var walPathRegex = regexp.MustCompile(`^([0-9a-f]{8})\.wal$`)
-func FormatWALPathWithOffset(index int, offset int64) string {
+
-	assert(index >= 0, "wal index must be non-negative")
+// ParseWALSegmentPath returns the index & offset for the WAL segment file.
-	assert(offset >= 0, "wal offset must be non-negative")
+// Returns an error if the path is not a valid wal segment path.
-	return fmt.Sprintf("%08x_%08x%s", index, offset, WALExt)
+func ParseWALSegmentPath(s string) (index int, offset int64, err error) {
+	s = filepath.Base(s)
+
+	a := walSegmentPathRegex.FindStringSubmatch(s)
+	if a == nil {
+		return 0, 0, fmt.Errorf("invalid wal segment path: %s", s)
+	}
+
+	i64, _ := strconv.ParseUint(a[1], 16, 64)
+	off64, _ := strconv.ParseUint(a[2], 16, 64)
+	return int(i64), int64(off64), nil
 }
 
-var walPathRegex = regexp.MustCompile(`^([0-9a-f]{8})(?:_([0-9a-f]{8}))?(.wal(?:.lz4)?)$`)
+// FormatWALSegmentPath formats a WAL segment filename with a given index & offset.
+func FormatWALSegmentPath(index int, offset int64) string {
+	assert(index >= 0, "wal index must be non-negative")
+	assert(offset >= 0, "wal offset must be non-negative")
+	return fmt.Sprintf("%08x_%08x%s", index, offset, WALSegmentExt)
+}
+
+var walSegmentPathRegex = regexp.MustCompile(`^([0-9a-f]{8})(?:_([0-9a-f]{8}))\.wal\.lz4$`)
 
 // isHexChar returns true if ch is a lowercase hex character.
 func isHexChar(ch rune) bool {
 	return (ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f')
 }
 
-// createFile creates the file and attempts to set the UID/GID.
-func createFile(filename string, perm os.FileMode, uid, gid int) (*os.File, error) {
-	f, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE|os.O_TRUNC, perm)
-	if err != nil {
-		return nil, err
-	}
-	_ = f.Chown(uid, gid)
-	return f, nil
-}
-
-// mkdirAll is a copy of os.MkdirAll() except that it attempts to set the
-// uid/gid for each created directory.
-func mkdirAll(path string, perm os.FileMode, uid, gid int) error {
-	// Fast path: if we can tell whether path is a directory or file, stop with success or error.
-	dir, err := os.Stat(path)
-	if err == nil {
-		if dir.IsDir() {
-			return nil
-		}
-		return &os.PathError{Op: "mkdir", Path: path, Err: syscall.ENOTDIR}
-	}
-
-	// Slow path: make sure parent exists and then call Mkdir for path.
-	i := len(path)
-	for i > 0 && os.IsPathSeparator(path[i-1]) { // Skip trailing path separator.
-		i--
-	}
-
-	j := i
-	for j > 0 && !os.IsPathSeparator(path[j-1]) { // Scan backward over element.
-		j--
-	}
-
-	if j > 1 {
-		// Create parent.
-		err = mkdirAll(fixRootDirectory(path[:j-1]), perm, uid, gid)
-		if err != nil {
-			return err
-		}
-	}
-
-	// Parent now exists; invoke Mkdir and use its result.
-	err = os.Mkdir(path, perm)
-	if err != nil {
-		// Handle arguments like "foo/." by
-		// double-checking that directory doesn't exist.
-		dir, err1 := os.Lstat(path)
-		if err1 == nil && dir.IsDir() {
-			_ = os.Chown(path, uid, gid)
-			return nil
-		}
-		return err
-	}
-	_ = os.Chown(path, uid, gid)
-	return nil
-}
-
 // Tracef is used for low-level tracing.
 var Tracef = func(format string, a ...interface{}) {}
 

litestream_test.go

@@ -40,6 +40,104 @@ func TestChecksum(t *testing.T) {
 	})
 }
 
+func TestGenerationsPath(t *testing.T) {
+	t.Run("OK", func(t *testing.T) {
+		if got, want := litestream.GenerationsPath("foo"), "foo/generations"; got != want {
+			t.Fatalf("GenerationsPath()=%v, want %v", got, want)
+		}
+	})
+	t.Run("NoPath", func(t *testing.T) {
+		if got, want := litestream.GenerationsPath(""), "generations"; got != want {
+			t.Fatalf("GenerationsPath()=%v, want %v", got, want)
+		}
+	})
+}
+
+func TestGenerationPath(t *testing.T) {
+	t.Run("OK", func(t *testing.T) {
+		if got, err := litestream.GenerationPath("foo", "0123456701234567"); err != nil {
+			t.Fatal(err)
+		} else if want := "foo/generations/0123456701234567"; got != want {
+			t.Fatalf("GenerationPath()=%v, want %v", got, want)
+		}
+	})
+	t.Run("ErrNoGeneration", func(t *testing.T) {
+		if _, err := litestream.GenerationPath("foo", ""); err == nil || err.Error() != `generation required` {
+			t.Fatalf("expected error: %v", err)
+		}
+	})
+}
+
+func TestSnapshotsPath(t *testing.T) {
+	t.Run("OK", func(t *testing.T) {
+		if got, err := litestream.SnapshotsPath("foo", "0123456701234567"); err != nil {
+			t.Fatal(err)
+		} else if want := "foo/generations/0123456701234567/snapshots"; got != want {
+			t.Fatalf("SnapshotsPath()=%v, want %v", got, want)
+		}
+	})
+	t.Run("ErrNoGeneration", func(t *testing.T) {
+		if _, err := litestream.SnapshotsPath("foo", ""); err == nil || err.Error() != `generation required` {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	})
+}
+
+func TestSnapshotPath(t *testing.T) {
+	t.Run("OK", func(t *testing.T) {
+		if got, err := litestream.SnapshotPath("foo", "0123456701234567", 1000); err != nil {
+			t.Fatal(err)
+		} else if want := "foo/generations/0123456701234567/snapshots/000003e8.snapshot.lz4"; got != want {
+			t.Fatalf("SnapshotPath()=%v, want %v", got, want)
+		}
+	})
+	t.Run("ErrNoGeneration", func(t *testing.T) {
+		if _, err := litestream.SnapshotPath("foo", "", 1000); err == nil || err.Error() != `generation required` {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	})
+}
+
+func TestWALPath(t *testing.T) {
+	t.Run("OK", func(t *testing.T) {
+		if got, err := litestream.WALPath("foo", "0123456701234567"); err != nil {
+			t.Fatal(err)
+		} else if want := "foo/generations/0123456701234567/wal"; got != want {
+			t.Fatalf("WALPath()=%v, want %v", got, want)
+		}
+	})
+	t.Run("ErrNoGeneration", func(t *testing.T) {
+		if _, err := litestream.WALPath("foo", ""); err == nil || err.Error() != `generation required` {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	})
+}
+
+func TestWALSegmentPath(t *testing.T) {
+	t.Run("OK", func(t *testing.T) {
+		if got, err := litestream.WALSegmentPath("foo", "0123456701234567", 1000, 1001); err != nil {
+			t.Fatal(err)
+		} else if want := "foo/generations/0123456701234567/wal/000003e8_000003e9.wal.lz4"; got != want {
+			t.Fatalf("WALPath()=%v, want %v", got, want)
+		}
+	})
+	t.Run("ErrNoGeneration", func(t *testing.T) {
+		if _, err := litestream.WALSegmentPath("foo", "", 1000, 0); err == nil || err.Error() != `generation required` {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	})
+}
+
+func TestFindMinSnapshotByGeneration(t *testing.T) {
+	infos := []litestream.SnapshotInfo{
+		{Generation: "29cf4bced74e92ab", Index: 0},
+		{Generation: "5dfeb4aa03232553", Index: 24},
+	}
+	if got, want := litestream.FindMinSnapshotByGeneration(infos, "29cf4bced74e92ab"), &infos[0]; got != want {
+		t.Fatalf("info=%#v, want %#v", got, want)
+	}
+}
+
 func MustDecodeHexString(s string) []byte {
 	b, err := hex.DecodeString(s)
 	if err != nil {

mock/replica_client.go

@@ -0,0 +1,65 @@
+package mock
+
+import (
+	"context"
+	"io"
+
+	"github.com/benbjohnson/litestream"
+)
+
+var _ litestream.ReplicaClient = (*ReplicaClient)(nil)
+
+type ReplicaClient struct {
+	GenerationsFunc       func(ctx context.Context) ([]string, error)
+	DeleteGenerationFunc  func(ctx context.Context, generation string) error
+	SnapshotsFunc         func(ctx context.Context, generation string) (litestream.SnapshotIterator, error)
+	WriteSnapshotFunc     func(ctx context.Context, generation string, index int, r io.Reader) (litestream.SnapshotInfo, error)
+	DeleteSnapshotFunc    func(ctx context.Context, generation string, index int) error
+	SnapshotReaderFunc    func(ctx context.Context, generation string, index int) (io.ReadCloser, error)
+	WALSegmentsFunc       func(ctx context.Context, generation string) (litestream.WALSegmentIterator, error)
+	WriteWALSegmentFunc   func(ctx context.Context, pos litestream.Pos, r io.Reader) (litestream.WALSegmentInfo, error)
+	DeleteWALSegmentsFunc func(ctx context.Context, a []litestream.Pos) error
+	WALSegmentReaderFunc  func(ctx context.Context, pos litestream.Pos) (io.ReadCloser, error)
+}
+
+func (c *ReplicaClient) Type() string { return "mock" }
+
+func (c *ReplicaClient) Generations(ctx context.Context) ([]string, error) {
+	return c.GenerationsFunc(ctx)
+}
+
+func (c *ReplicaClient) DeleteGeneration(ctx context.Context, generation string) error {
+	return c.DeleteGenerationFunc(ctx, generation)
+}
+
+func (c *ReplicaClient) Snapshots(ctx context.Context, generation string) (litestream.SnapshotIterator, error) {
+	return c.SnapshotsFunc(ctx, generation)
+}
+
+func (c *ReplicaClient) WriteSnapshot(ctx context.Context, generation string, index int, r io.Reader) (litestream.SnapshotInfo, error) {
+	return c.WriteSnapshotFunc(ctx, generation, index, r)
+}
+
+func (c *ReplicaClient) DeleteSnapshot(ctx context.Context, generation string, index int) error {
+	return c.DeleteSnapshotFunc(ctx, generation, index)
+}
+
+func (c *ReplicaClient) SnapshotReader(ctx context.Context, generation string, index int) (io.ReadCloser, error) {
+	return c.SnapshotReaderFunc(ctx, generation, index)
+}
+
+func (c *ReplicaClient) WALSegments(ctx context.Context, generation string) (litestream.WALSegmentIterator, error) {
+	return c.WALSegmentsFunc(ctx, generation)
+}
+
+func (c *ReplicaClient) WriteWALSegment(ctx context.Context, pos litestream.Pos, r io.Reader) (litestream.WALSegmentInfo, error) {
+	return c.WriteWALSegmentFunc(ctx, pos, r)
+}
+
+func (c *ReplicaClient) DeleteWALSegments(ctx context.Context, a []litestream.Pos) error {
+	return c.DeleteWALSegmentsFunc(ctx, a)
+}
+
+func (c *ReplicaClient) WALSegmentReader(ctx context.Context, pos litestream.Pos) (io.ReadCloser, error) {
+	return c.WALSegmentReaderFunc(ctx, pos)
+}

replica_client.go

@@ -0,0 +1,48 @@
+package litestream
+
+import (
+	"context"
+	"io"
+)
+
+// ReplicaClient represents client to connect to a Replica.
+type ReplicaClient interface {
+	// Returns the type of client.
+	Type() string
+
+	// Returns a list of available generations.
+	Generations(ctx context.Context) ([]string, error)
+
+	// Deletes all snapshots & WAL segments within a generation.
+	DeleteGeneration(ctx context.Context, generation string) error
+
+	// Returns an iterator of all snapshots within a generation on the replica.
+	Snapshots(ctx context.Context, generation string) (SnapshotIterator, error)
+
+	// Writes LZ4 compressed snapshot data to the replica at a given index
+	// within a generation. Returns metadata for the snapshot.
+	WriteSnapshot(ctx context.Context, generation string, index int, r io.Reader) (SnapshotInfo, error)
+
+	// Deletes a snapshot with the given generation & index.
+	DeleteSnapshot(ctx context.Context, generation string, index int) error
+
+	// Returns a reader that contains LZ4 compressed snapshot data for a
+	// given index within a generation. Returns an os.ErrNotFound error if
+	// the snapshot does not exist.
+	SnapshotReader(ctx context.Context, generation string, index int) (io.ReadCloser, error)
+
+	// Returns an iterator of all WAL segments within a generation on the replica.
+	WALSegments(ctx context.Context, generation string) (WALSegmentIterator, error)
+
+	// Writes an LZ4 compressed WAL segment at a given position.
+	// Returns metadata for the written segment.
+	WriteWALSegment(ctx context.Context, pos Pos, r io.Reader) (WALSegmentInfo, error)
+
+	// Deletes one or more WAL segments at the given positions.
+	DeleteWALSegments(ctx context.Context, a []Pos) error
+
+	// Returns a reader that contains an LZ4 compressed WAL segment at a given
+	// index/offset within a generation. Returns an os.ErrNotFound error if the
+	// WAL segment does not exist.
+	WALSegmentReader(ctx context.Context, pos Pos) (io.ReadCloser, error)
+}

replica_client_test.go

@@ -0,0 +1,573 @@
+package litestream_test
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"path"
+	"reflect"
+	"sort"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/benbjohnson/litestream"
+	"github.com/benbjohnson/litestream/abs"
+	"github.com/benbjohnson/litestream/file"
+	"github.com/benbjohnson/litestream/gcs"
+	"github.com/benbjohnson/litestream/s3"
+	"github.com/benbjohnson/litestream/sftp"
+)
+
+func init() {
+	rand.Seed(time.Now().UnixNano())
+}
+
+var (
+	// Enables integration tests.
+	integration = flag.String("integration", "file", "")
+)
+
+// S3 settings
+var (
+	// Replica client settings
+	s3AccessKeyID     = flag.String("s3-access-key-id", os.Getenv("LITESTREAM_S3_ACCESS_KEY_ID"), "")
+	s3SecretAccessKey = flag.String("s3-secret-access-key", os.Getenv("LITESTREAM_S3_SECRET_ACCESS_KEY"), "")
+	s3Region          = flag.String("s3-region", os.Getenv("LITESTREAM_S3_REGION"), "")
+	s3Bucket          = flag.String("s3-bucket", os.Getenv("LITESTREAM_S3_BUCKET"), "")
+	s3Path            = flag.String("s3-path", os.Getenv("LITESTREAM_S3_PATH"), "")
+	s3Endpoint        = flag.String("s3-endpoint", os.Getenv("LITESTREAM_S3_ENDPOINT"), "")
+	s3ForcePathStyle  = flag.Bool("s3-force-path-style", os.Getenv("LITESTREAM_S3_FORCE_PATH_STYLE") == "true", "")
+	s3SkipVerify      = flag.Bool("s3-skip-verify", os.Getenv("LITESTREAM_S3_SKIP_VERIFY") == "true", "")
+)
+
+// Google cloud storage settings
+var (
+	gcsBucket = flag.String("gcs-bucket", os.Getenv("LITESTREAM_GCS_BUCKET"), "")
+	gcsPath   = flag.String("gcs-path", os.Getenv("LITESTREAM_GCS_PATH"), "")
+)
+
+// Azure blob storage settings
+var (
+	absAccountName = flag.String("abs-account-name", os.Getenv("LITESTREAM_ABS_ACCOUNT_NAME"), "")
+	absAccountKey  = flag.String("abs-account-key", os.Getenv("LITESTREAM_ABS_ACCOUNT_KEY"), "")
+	absBucket      = flag.String("abs-bucket", os.Getenv("LITESTREAM_ABS_BUCKET"), "")
+	absPath        = flag.String("abs-path", os.Getenv("LITESTREAM_ABS_PATH"), "")
+)
+
+// SFTP settings
+var (
+	sftpHost     = flag.String("sftp-host", os.Getenv("LITESTREAM_SFTP_HOST"), "")
+	sftpUser     = flag.String("sftp-user", os.Getenv("LITESTREAM_SFTP_USER"), "")
+	sftpPassword = flag.String("sftp-password", os.Getenv("LITESTREAM_SFTP_PASSWORD"), "")
+	sftpKeyPath  = flag.String("sftp-key-path", os.Getenv("LITESTREAM_SFTP_KEY_PATH"), "")
+	sftpPath     = flag.String("sftp-path", os.Getenv("LITESTREAM_SFTP_PATH"), "")
+)
+
+func TestReplicaClient_Generations(t *testing.T) {
+	RunWithReplicaClient(t, "OK", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+
+		// Write snapshots.
+		if _, err := c.WriteSnapshot(context.Background(), "5efbd8d042012dca", 0, strings.NewReader(`foo`)); err != nil {
+			t.Fatal(err)
+		} else if _, err := c.WriteSnapshot(context.Background(), "b16ddcf5c697540f", 0, strings.NewReader(`bar`)); err != nil {
+			t.Fatal(err)
+		} else if _, err := c.WriteSnapshot(context.Background(), "155fe292f8333c72", 0, strings.NewReader(`baz`)); err != nil {
+			t.Fatal(err)
+		}
+
+		// Verify returned generations.
+		if got, err := c.Generations(context.Background()); err != nil {
+			t.Fatal(err)
+		} else if want := []string{"155fe292f8333c72", "5efbd8d042012dca", "b16ddcf5c697540f"}; !reflect.DeepEqual(got, want) {
+			t.Fatalf("Generations()=%v, want %v", got, want)
+		}
+	})
+
+	RunWithReplicaClient(t, "NoGenerationsDir", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+
+		if generations, err := c.Generations(context.Background()); err != nil {
+			t.Fatal(err)
+		} else if got, want := len(generations), 0; got != want {
+			t.Fatalf("len(Generations())=%v, want %v", got, want)
+		}
+	})
+}
+
+func TestReplicaClient_Snapshots(t *testing.T) {
+	RunWithReplicaClient(t, "OK", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+
+		// Write snapshots.
+		if _, err := c.WriteSnapshot(context.Background(), "5efbd8d042012dca", 1, strings.NewReader(``)); err != nil {
+			t.Fatal(err)
+		} else if _, err := c.WriteSnapshot(context.Background(), "b16ddcf5c697540f", 5, strings.NewReader(`x`)); err != nil {
+			t.Fatal(err)
+		} else if _, err := c.WriteSnapshot(context.Background(), "b16ddcf5c697540f", 10, strings.NewReader(`xyz`)); err != nil {
+			t.Fatal(err)
+		}
+
+		// Fetch all snapshots by generation.
+		itr, err := c.Snapshots(context.Background(), "b16ddcf5c697540f")
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer itr.Close()
+
+		// Read all snapshots into a slice so they can be sorted.
+		a, err := litestream.SliceSnapshotIterator(itr)
+		if err != nil {
+			t.Fatal(err)
+		} else if got, want := len(a), 2; got != want {
+			t.Fatalf("len=%v, want %v", got, want)
+		}
+		sort.Sort(litestream.SnapshotInfoSlice(a))
+
+		// Verify first snapshot metadata.
+		if got, want := a[0].Generation, "b16ddcf5c697540f"; got != want {
+			t.Fatalf("Generation=%v, want %v", got, want)
+		} else if got, want := a[0].Index, 5; got != want {
+			t.Fatalf("Index=%v, want %v", got, want)
+		} else if got, want := a[0].Size, int64(1); got != want {
+			t.Fatalf("Size=%v, want %v", got, want)
+		} else if a[0].CreatedAt.IsZero() {
+			t.Fatalf("expected CreatedAt")
+		}
+
+		// Verify second snapshot metadata.
+		if got, want := a[1].Generation, "b16ddcf5c697540f"; got != want {
+			t.Fatalf("Generation=%v, want %v", got, want)
+		} else if got, want := a[1].Index, 0xA; got != want {
+			t.Fatalf("Index=%v, want %v", got, want)
+		} else if got, want := a[1].Size, int64(3); got != want {
+			t.Fatalf("Size=%v, want %v", got, want)
+		} else if a[1].CreatedAt.IsZero() {
+			t.Fatalf("expected CreatedAt")
+		}
+
+		// Ensure close is clean.
+		if err := itr.Close(); err != nil {
+			t.Fatal(err)
+		}
+	})
+
+	RunWithReplicaClient(t, "NoGenerationDir", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+
+		itr, err := c.Snapshots(context.Background(), "5efbd8d042012dca")
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer itr.Close()
+
+		if itr.Next() {
+			t.Fatal("expected no snapshots")
+		}
+	})
+
+	RunWithReplicaClient(t, "ErrNoGeneration", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+
+		itr, err := c.Snapshots(context.Background(), "")
+		if err == nil {
+			err = itr.Close()
+		}
+		if err == nil || err.Error() != `cannot determine snapshots path: generation required` {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	})
+}
+
+func TestReplicaClient_WriteSnapshot(t *testing.T) {
+	RunWithReplicaClient(t, "OK", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+
+		if _, err := c.WriteSnapshot(context.Background(), "b16ddcf5c697540f", 1000, strings.NewReader(`foobar`)); err != nil {
+			t.Fatal(err)
+		}
+
+		if r, err := c.SnapshotReader(context.Background(), "b16ddcf5c697540f", 1000); err != nil {
+			t.Fatal(err)
+		} else if buf, err := ioutil.ReadAll(r); err != nil {
+			t.Fatal(err)
+		} else if err := r.Close(); err != nil {
+			t.Fatal(err)
+		} else if got, want := string(buf), `foobar`; got != want {
+			t.Fatalf("data=%q, want %q", got, want)
+		}
+	})
+
+	RunWithReplicaClient(t, "ErrNoGeneration", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+		if _, err := c.WriteSnapshot(context.Background(), "", 0, nil); err == nil || err.Error() != `cannot determine snapshot path: generation required` {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	})
+}
+
+func TestReplicaClient_SnapshotReader(t *testing.T) {
+	RunWithReplicaClient(t, "OK", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+
+		if _, err := c.WriteSnapshot(context.Background(), "5efbd8d042012dca", 10, strings.NewReader(`foo`)); err != nil {
+			t.Fatal(err)
+		}
+
+		r, err := c.SnapshotReader(context.Background(), "5efbd8d042012dca", 10)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer r.Close()
+
+		if buf, err := ioutil.ReadAll(r); err != nil {
+			t.Fatal(err)
+		} else if got, want := string(buf), "foo"; got != want {
+			t.Fatalf("ReadAll=%v, want %v", got, want)
+		}
+	})
+
+	RunWithReplicaClient(t, "ErrNotFound", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+
+		if _, err := c.SnapshotReader(context.Background(), "5efbd8d042012dca", 1); !os.IsNotExist(err) {
+			t.Fatalf("expected not exist, got %#v", err)
+		}
+	})
+
+	RunWithReplicaClient(t, "ErrNoGeneration", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+
+		if _, err := c.SnapshotReader(context.Background(), "", 1); err == nil || err.Error() != `cannot determine snapshot path: generation required` {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	})
+}
+
+func TestReplicaClient_WALs(t *testing.T) {
+	RunWithReplicaClient(t, "OK", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+
+		if _, err := c.WriteWALSegment(context.Background(), litestream.Pos{Generation: "5efbd8d042012dca", Index: 1, Offset: 0}, strings.NewReader(``)); err != nil {
+			t.Fatal(err)
+		}
+		if _, err := c.WriteWALSegment(context.Background(), litestream.Pos{Generation: "b16ddcf5c697540f", Index: 2, Offset: 0}, strings.NewReader(`12345`)); err != nil {
+			t.Fatal(err)
+		} else if _, err := c.WriteWALSegment(context.Background(), litestream.Pos{Generation: "b16ddcf5c697540f", Index: 2, Offset: 5}, strings.NewReader(`67`)); err != nil {
+			t.Fatal(err)
+		} else if _, err := c.WriteWALSegment(context.Background(), litestream.Pos{Generation: "b16ddcf5c697540f", Index: 3, Offset: 0}, strings.NewReader(`xyz`)); err != nil {
+			t.Fatal(err)
+		}
+
+		itr, err := c.WALSegments(context.Background(), "b16ddcf5c697540f")
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer itr.Close()
+
+		// Read all WAL segment files into a slice so they can be sorted.
+		a, err := litestream.SliceWALSegmentIterator(itr)
+		if err != nil {
+			t.Fatal(err)
+		} else if got, want := len(a), 3; got != want {
+			t.Fatalf("len=%v, want %v", got, want)
+		}
+		sort.Sort(litestream.WALSegmentInfoSlice(a))
+
+		// Verify first WAL segment metadata.
+		if got, want := a[0].Generation, "b16ddcf5c697540f"; got != want {
+			t.Fatalf("Generation=%v, want %v", got, want)
+		} else if got, want := a[0].Index, 2; got != want {
+			t.Fatalf("Index=%v, want %v", got, want)
+		} else if got, want := a[0].Offset, int64(0); got != want {
+			t.Fatalf("Offset=%v, want %v", got, want)
+		} else if got, want := a[0].Size, int64(5); got != want {
+			t.Fatalf("Size=%v, want %v", got, want)
+		} else if a[0].CreatedAt.IsZero() {
+			t.Fatalf("expected CreatedAt")
+		}
+
+		// Verify first WAL segment metadata.
+		if got, want := a[1].Generation, "b16ddcf5c697540f"; got != want {
+			t.Fatalf("Generation=%v, want %v", got, want)
+		} else if got, want := a[1].Index, 2; got != want {
+			t.Fatalf("Index=%v, want %v", got, want)
+		} else if got, want := a[1].Offset, int64(5); got != want {
+			t.Fatalf("Offset=%v, want %v", got, want)
+		} else if got, want := a[1].Size, int64(2); got != want {
+			t.Fatalf("Size=%v, want %v", got, want)
+		} else if a[1].CreatedAt.IsZero() {
+			t.Fatalf("expected CreatedAt")
+		}
+
+		// Verify third WAL segment metadata.
+		if got, want := a[2].Generation, "b16ddcf5c697540f"; got != want {
+			t.Fatalf("Generation=%v, want %v", got, want)
+		} else if got, want := a[2].Index, 3; got != want {
+			t.Fatalf("Index=%v, want %v", got, want)
+		} else if got, want := a[2].Offset, int64(0); got != want {
+			t.Fatalf("Offset=%v, want %v", got, want)
+		} else if got, want := a[2].Size, int64(3); got != want {
+			t.Fatalf("Size=%v, want %v", got, want)
+		} else if a[1].CreatedAt.IsZero() {
+			t.Fatalf("expected CreatedAt")
+		}
+
+		// Ensure close is clean.
+		if err := itr.Close(); err != nil {
+			t.Fatal(err)
+		}
+	})
+
+	RunWithReplicaClient(t, "NoGenerationDir", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+
+		itr, err := c.WALSegments(context.Background(), "5efbd8d042012dca")
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer itr.Close()
+
+		if itr.Next() {
+			t.Fatal("expected no wal files")
+		}
+	})
+
+	RunWithReplicaClient(t, "NoWALs", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+
+		if _, err := c.WriteSnapshot(context.Background(), "5efbd8d042012dca", 0, strings.NewReader(`foo`)); err != nil {
+			t.Fatal(err)
+		}
+
+		itr, err := c.WALSegments(context.Background(), "5efbd8d042012dca")
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer itr.Close()
+
+		if itr.Next() {
+			t.Fatal("expected no wal files")
+		}
+	})
+
+	RunWithReplicaClient(t, "ErrNoGeneration", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+
+		itr, err := c.WALSegments(context.Background(), "")
+		if err == nil {
+			err = itr.Close()
+		}
+		if err == nil || err.Error() != `cannot determine wal path: generation required` {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	})
+}
+
+func TestReplicaClient_WriteWALSegment(t *testing.T) {
+	RunWithReplicaClient(t, "OK", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+
+		if _, err := c.WriteWALSegment(context.Background(), litestream.Pos{Generation: "b16ddcf5c697540f", Index: 1000, Offset: 2000}, strings.NewReader(`foobar`)); err != nil {
+			t.Fatal(err)
+		}
+
+		if r, err := c.WALSegmentReader(context.Background(), litestream.Pos{Generation: "b16ddcf5c697540f", Index: 1000, Offset: 2000}); err != nil {
+			t.Fatal(err)
+		} else if buf, err := ioutil.ReadAll(r); err != nil {
+			t.Fatal(err)
+		} else if err := r.Close(); err != nil {
+			t.Fatal(err)
+		} else if got, want := string(buf), `foobar`; got != want {
+			t.Fatalf("data=%q, want %q", got, want)
+		}
+	})
+
+	RunWithReplicaClient(t, "ErrNoGeneration", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+		if _, err := c.WriteWALSegment(context.Background(), litestream.Pos{Generation: "", Index: 0, Offset: 0}, nil); err == nil || err.Error() != `cannot determine wal segment path: generation required` {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	})
+}
+
+func TestReplicaClient_WALReader(t *testing.T) {
+
+	RunWithReplicaClient(t, "OK", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+		if _, err := c.WriteWALSegment(context.Background(), litestream.Pos{Generation: "5efbd8d042012dca", Index: 10, Offset: 5}, strings.NewReader(`foobar`)); err != nil {
+			t.Fatal(err)
+		}
+
+		r, err := c.WALSegmentReader(context.Background(), litestream.Pos{Generation: "5efbd8d042012dca", Index: 10, Offset: 5})
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer r.Close()
+
+		if buf, err := ioutil.ReadAll(r); err != nil {
+			t.Fatal(err)
+		} else if got, want := string(buf), "foobar"; got != want {
+			t.Fatalf("ReadAll=%v, want %v", got, want)
+		}
+	})
+
+	RunWithReplicaClient(t, "ErrNotFound", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+
+		if _, err := c.WALSegmentReader(context.Background(), litestream.Pos{Generation: "5efbd8d042012dca", Index: 1, Offset: 0}); !os.IsNotExist(err) {
+			t.Fatalf("expected not exist, got %#v", err)
+		}
+	})
+}
+
+func TestReplicaClient_DeleteWALSegments(t *testing.T) {
+	RunWithReplicaClient(t, "OK", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+
+		if _, err := c.WriteWALSegment(context.Background(), litestream.Pos{Generation: "b16ddcf5c697540f", Index: 1, Offset: 2}, strings.NewReader(`foo`)); err != nil {
+			t.Fatal(err)
+		} else if _, err := c.WriteWALSegment(context.Background(), litestream.Pos{Generation: "5efbd8d042012dca", Index: 3, Offset: 4}, strings.NewReader(`bar`)); err != nil {
+			t.Fatal(err)
+		}
+
+		if err := c.DeleteWALSegments(context.Background(), []litestream.Pos{
+			{Generation: "b16ddcf5c697540f", Index: 1, Offset: 2},
+			{Generation: "5efbd8d042012dca", Index: 3, Offset: 4},
+		}); err != nil {
+			t.Fatal(err)
+		}
+
+		if _, err := c.WALSegmentReader(context.Background(), litestream.Pos{Generation: "b16ddcf5c697540f", Index: 1, Offset: 2}); !os.IsNotExist(err) {
+			t.Fatalf("expected not exist, got %#v", err)
+		} else if _, err := c.WALSegmentReader(context.Background(), litestream.Pos{Generation: "5efbd8d042012dca", Index: 3, Offset: 4}); !os.IsNotExist(err) {
+			t.Fatalf("expected not exist, got %#v", err)
+		}
+	})
+
+	RunWithReplicaClient(t, "ErrNoGeneration", func(t *testing.T, c litestream.ReplicaClient) {
+		t.Parallel()
+		if err := c.DeleteWALSegments(context.Background(), []litestream.Pos{{}}); err == nil || err.Error() != `cannot determine wal segment path: generation required` {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	})
+}
+
+// RunWithReplicaClient executes fn with each replica specified by the -integration flag
+func RunWithReplicaClient(t *testing.T, name string, fn func(*testing.T, litestream.ReplicaClient)) {
+	t.Run(name, func(t *testing.T) {
+		for _, typ := range strings.Split(*integration, ",") {
+			t.Run(typ, func(t *testing.T) {
+				c := NewReplicaClient(t, typ)
+				defer MustDeleteAll(t, c)
+
+				fn(t, c)
+			})
+		}
+	})
+}
+
+// NewReplicaClient returns a new client for integration testing by type name.
+func NewReplicaClient(tb testing.TB, typ string) litestream.ReplicaClient {
+	tb.Helper()
+
+	switch typ {
+	case file.ReplicaClientType:
+		return NewFileReplicaClient(tb)
+	case s3.ReplicaClientType:
+		return NewS3ReplicaClient(tb)
+	case gcs.ReplicaClientType:
+		return NewGCSReplicaClient(tb)
+	case abs.ReplicaClientType:
+		return NewABSReplicaClient(tb)
+	case sftp.ReplicaClientType:
+		return NewSFTPReplicaClient(tb)
+	default:
+		tb.Fatalf("invalid replica client type: %q", typ)
+		return nil
+	}
+}
+
+// NewFileReplicaClient returns a new client for integration testing.
+func NewFileReplicaClient(tb testing.TB) *file.ReplicaClient {
+	tb.Helper()
+	return file.NewReplicaClient(tb.TempDir())
+}
+
+// NewS3ReplicaClient returns a new client for integration testing.
+func NewS3ReplicaClient(tb testing.TB) *s3.ReplicaClient {
+	tb.Helper()
+
+	c := s3.NewReplicaClient()
+	c.AccessKeyID = *s3AccessKeyID
+	c.SecretAccessKey = *s3SecretAccessKey
+	c.Region = *s3Region
+	c.Bucket = *s3Bucket
+	c.Path = path.Join(*s3Path, fmt.Sprintf("%016x", rand.Uint64()))
+	c.Endpoint = *s3Endpoint
+	c.ForcePathStyle = *s3ForcePathStyle
+	c.SkipVerify = *s3SkipVerify
+	return c
+}
+
+// NewGCSReplicaClient returns a new client for integration testing.
+func NewGCSReplicaClient(tb testing.TB) *gcs.ReplicaClient {
+	tb.Helper()
+
+	c := gcs.NewReplicaClient()
+	c.Bucket = *gcsBucket
+	c.Path = path.Join(*gcsPath, fmt.Sprintf("%016x", rand.Uint64()))
+	return c
+}
+
+// NewABSReplicaClient returns a new client for integration testing.
+func NewABSReplicaClient(tb testing.TB) *abs.ReplicaClient {
+	tb.Helper()
+
+	c := abs.NewReplicaClient()
+	c.AccountName = *absAccountName
+	c.AccountKey = *absAccountKey
+	c.Bucket = *absBucket
+	c.Path = path.Join(*absPath, fmt.Sprintf("%016x", rand.Uint64()))
+	return c
+}
+
+// NewSFTPReplicaClient returns a new client for integration testing.
+func NewSFTPReplicaClient(tb testing.TB) *sftp.ReplicaClient {
+	tb.Helper()
+
+	c := sftp.NewReplicaClient()
+	c.Host = *sftpHost
+	c.User = *sftpUser
+	c.Password = *sftpPassword
+	c.KeyPath = *sftpKeyPath
+	c.Path = path.Join(*sftpPath, fmt.Sprintf("%016x", rand.Uint64()))
+	return c
+}
+
+// MustDeleteAll deletes all objects under the client's path.
+func MustDeleteAll(tb testing.TB, c litestream.ReplicaClient) {
+	tb.Helper()
+
+	generations, err := c.Generations(context.Background())
+	if err != nil {
+		tb.Fatalf("cannot list generations for deletion: %s", err)
+	}
+
+	for _, generation := range generations {
+		if err := c.DeleteGeneration(context.Background(), generation); err != nil {
+			tb.Fatalf("cannot delete generation: %s", err)
+		}
+	}
+
+	switch c := c.(type) {
+	case *sftp.ReplicaClient:
+		if err := c.Cleanup(context.Background()); err != nil {
+			tb.Fatalf("cannot cleanup sftp: %s", err)
+		}
+	}
+}

replica_test.go

@@ -1,90 +1,144 @@
 package litestream_test
 
 import (
+	"bytes"
 	"context"
+	"io"
+	"os"
 	"testing"
 
 	"github.com/benbjohnson/litestream"
+	"github.com/benbjohnson/litestream/file"
+	"github.com/benbjohnson/litestream/mock"
+	"github.com/pierrec/lz4/v4"
 )
 
-func TestFileReplica_Sync(t *testing.T) {
+func TestReplica_Name(t *testing.T) {
-	// Ensure replica can successfully sync after DB has sync'd.
+	t.Run("WithName", func(t *testing.T) {
-	t.Run("InitialSync", func(t *testing.T) {
+		if got, want := litestream.NewReplica(nil, "NAME").Name(), "NAME"; got != want {
-		db, sqldb := MustOpenDBs(t)
+			t.Fatalf("Name()=%v, want %v", got, want)
-		defer MustCloseDBs(t, db, sqldb)
-		r := NewTestFileReplica(t, db)
-
-		// Sync database & then sync replica.
-		if err := db.Sync(); err != nil {
-			t.Fatal(err)
-		} else if err := r.Sync(context.Background()); err != nil {
-			t.Fatal(err)
-		}
-
-		// Ensure posistions match.
-		if pos, err := db.Pos(); err != nil {
-			t.Fatal(err)
-		} else if got, want := r.LastPos(), pos; got != want {
-			t.Fatalf("LastPos()=%v, want %v", got, want)
 		}
 	})
-
+	t.Run("WithoutName", func(t *testing.T) {
-	// Ensure replica can successfully sync multiple times.
+		r := litestream.NewReplica(nil, "")
-	t.Run("MultiSync", func(t *testing.T) {
+		r.Client = &mock.ReplicaClient{}
-		db, sqldb := MustOpenDBs(t)
+		if got, want := r.Name(), "mock"; got != want {
-		defer MustCloseDBs(t, db, sqldb)
+			t.Fatalf("Name()=%v, want %v", got, want)
-		r := NewTestFileReplica(t, db)
-
-		if _, err := sqldb.Exec(`CREATE TABLE foo (bar TEXT);`); err != nil {
-			t.Fatal(err)
-		}
-
-		// Write to the database multiple times and sync after each write.
-		for i, n := 0, db.MinCheckpointPageN*2; i < n; i++ {
-			if _, err := sqldb.Exec(`INSERT INTO foo (bar) VALUES ('baz')`); err != nil {
-				t.Fatal(err)
-			}
-
-			// Sync periodically.
-			if i%100 == 0 || i == n-1 {
-				if err := db.Sync(); err != nil {
-					t.Fatal(err)
-				} else if err := r.Sync(context.Background()); err != nil {
-					t.Fatal(err)
-				}
-			}
-		}
-
-		// Ensure posistions match.
-		if pos, err := db.Pos(); err != nil {
-			t.Fatal(err)
-		} else if got, want := pos.Index, 2; got != want {
-			t.Fatalf("Index=%v, want %v", got, want)
-		} else if calcPos, err := r.CalcPos(context.Background(), pos.Generation); err != nil {
-			t.Fatal(err)
-		} else if got, want := calcPos, pos; got != want {
-			t.Fatalf("CalcPos()=%v, want %v", got, want)
-		} else if got, want := r.LastPos(), pos; got != want {
-			t.Fatalf("LastPos()=%v, want %v", got, want)
-		}
-	})
-
-	// Ensure replica returns an error if there is no generation available from the DB.
-	t.Run("ErrNoGeneration", func(t *testing.T) {
-		db, sqldb := MustOpenDBs(t)
-		defer MustCloseDBs(t, db, sqldb)
-		r := NewTestFileReplica(t, db)
-
-		if err := r.Sync(context.Background()); err == nil || err.Error() != `no generation, waiting for data` {
-			t.Fatal(err)
 		}
 	})
 }
 
-// NewTestFileReplica returns a new replica using a temp directory & with monitoring disabled.
+func TestReplica_Sync(t *testing.T) {
-func NewTestFileReplica(tb testing.TB, db *litestream.DB) *litestream.FileReplica {
+	db, sqldb := MustOpenDBs(t)
-	r := litestream.NewFileReplica(db, "", tb.TempDir())
+	defer MustCloseDBs(t, db, sqldb)
-	r.MonitorEnabled = false
+
-	db.Replicas = []litestream.Replica{r}
+	// Execute a query to force a write to the WAL.
-	return r
+	if _, err := sqldb.Exec(`CREATE TABLE foo (bar TEXT);`); err != nil {
+		t.Fatal(err)
+	}
+
+	// Issue initial database sync to setup generation.
+	if err := db.Sync(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+
+	// Fetch current database position.
+	dpos, err := db.Pos()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	c := file.NewReplicaClient(t.TempDir())
+	r := litestream.NewReplica(db, "")
+	c.Replica, r.Client = r, c
+
+	if err := r.Sync(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify client generation matches database.
+	generations, err := c.Generations(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	} else if got, want := len(generations), 1; got != want {
+		t.Fatalf("len(generations)=%v, want %v", got, want)
+	} else if got, want := generations[0], dpos.Generation; got != want {
+		t.Fatalf("generations[0]=%v, want %v", got, want)
+	}
+
+	// Verify WAL matches replica WAL.
+	if b0, err := os.ReadFile(db.Path() + "-wal"); err != nil {
+		t.Fatal(err)
+	} else if r, err := c.WALSegmentReader(context.Background(), litestream.Pos{Generation: generations[0], Index: 0, Offset: 0}); err != nil {
+		t.Fatal(err)
+	} else if b1, err := io.ReadAll(lz4.NewReader(r)); err != nil {
+		t.Fatal(err)
+	} else if err := r.Close(); err != nil {
+		t.Fatal(err)
+	} else if !bytes.Equal(b0, b1) {
+		t.Fatalf("wal mismatch: len(%d), len(%d)", len(b0), len(b1))
+	}
+}
+
+func TestReplica_Snapshot(t *testing.T) {
+	db, sqldb := MustOpenDBs(t)
+	defer MustCloseDBs(t, db, sqldb)
+
+	c := file.NewReplicaClient(t.TempDir())
+	r := litestream.NewReplica(db, "")
+	r.Client = c
+
+	// Execute a query to force a write to the WAL.
+	if _, err := sqldb.Exec(`CREATE TABLE foo (bar TEXT);`); err != nil {
+		t.Fatal(err)
+	} else if err := db.Sync(context.Background()); err != nil {
+		t.Fatal(err)
+	} else if err := r.Sync(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+
+	// Fetch current database position & snapshot.
+	pos0, err := db.Pos()
+	if err != nil {
+		t.Fatal(err)
+	} else if info, err := r.Snapshot(context.Background()); err != nil {
+		t.Fatal(err)
+	} else if got, want := info.Pos(), pos0.Truncate(); got != want {
+		t.Fatalf("pos=%s, want %s", got, want)
+	}
+
+	// Sync database and then replica.
+	if err := db.Sync(context.Background()); err != nil {
+		t.Fatal(err)
+	} else if err := r.Sync(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+
+	// Execute a query to force a write to the WAL & truncate to start new index.
+	if _, err := sqldb.Exec(`INSERT INTO foo (bar) VALUES ('baz');`); err != nil {
+		t.Fatal(err)
+	} else if err := db.Checkpoint(context.Background(), litestream.CheckpointModeTruncate); err != nil {
+		t.Fatal(err)
+	}
+
+	// Fetch current database position & snapshot.
+	pos1, err := db.Pos()
+	if err != nil {
+		t.Fatal(err)
+	} else if info, err := r.Snapshot(context.Background()); err != nil {
+		t.Fatal(err)
+	} else if got, want := info.Pos(), pos1.Truncate(); got != want {
+		t.Fatalf("pos=%v, want %v", got, want)
+	}
+
+	// Verify two snapshots exist.
+	if infos, err := r.Snapshots(context.Background()); err != nil {
+		t.Fatal(err)
+	} else if got, want := len(infos), 2; got != want {
+		t.Fatalf("len=%v, want %v", got, want)
+	} else if got, want := infos[0].Pos(), pos0.Truncate(); got != want {
+		t.Fatalf("info[0]=%s, want %s", got, want)
+	} else if got, want := infos[1].Pos(), pos1.Truncate(); got != want {
+		t.Fatalf("info[1]=%s, want %s", got, want)
+	}
 }

s3/replica_client.go

@@ -0,0 +1,767 @@
+package s3
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"os"
+	"path"
+	"regexp"
+	"sync"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+	"github.com/benbjohnson/litestream"
+	"github.com/benbjohnson/litestream/internal"
+	"golang.org/x/sync/errgroup"
+)
+
+// ReplicaClientType is the client type for this package.
+const ReplicaClientType = "s3"
+
+// MaxKeys is the number of keys S3 can operate on per batch.
+const MaxKeys = 1000
+
+// DefaultRegion is the region used if one is not specified.
+const DefaultRegion = "us-east-1"
+
+var _ litestream.ReplicaClient = (*ReplicaClient)(nil)
+
+// ReplicaClient is a client for writing snapshots & WAL segments to disk.
+type ReplicaClient struct {
+	mu       sync.Mutex
+	s3       *s3.S3 // s3 service
+	uploader *s3manager.Uploader
+
+	// AWS authentication keys.
+	AccessKeyID     string
+	SecretAccessKey string
+
+	// S3 bucket information
+	Region         string
+	Bucket         string
+	Path           string
+	Endpoint       string
+	ForcePathStyle bool
+	SkipVerify     bool
+}
+
+// NewReplicaClient returns a new instance of ReplicaClient.
+func NewReplicaClient() *ReplicaClient {
+	return &ReplicaClient{}
+}
+
+// Type returns "s3" as the client type.
+func (c *ReplicaClient) Type() string {
+	return ReplicaClientType
+}
+
+// Init initializes the connection to S3. No-op if already initialized.
+func (c *ReplicaClient) Init(ctx context.Context) (err error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if c.s3 != nil {
+		return nil
+	}
+
+	// Look up region if not specified and no endpoint is used.
+	// Endpoints are typically used for non-S3 object stores and do not
+	// necessarily require a region.
+	region := c.Region
+	if region == "" {
+		if c.Endpoint == "" {
+			if region, err = c.findBucketRegion(ctx, c.Bucket); err != nil {
+				return fmt.Errorf("cannot lookup bucket region: %w", err)
+			}
+		} else {
+			region = DefaultRegion // default for non-S3 object stores
+		}
+	}
+
+	// Create new AWS session.
+	config := c.config()
+	if region != "" {
+		config.Region = aws.String(region)
+	}
+
+	sess, err := session.NewSession(config)
+	if err != nil {
+		return fmt.Errorf("cannot create aws session: %w", err)
+	}
+	c.s3 = s3.New(sess)
+	c.uploader = s3manager.NewUploader(sess)
+	return nil
+}
+
+// config returns the AWS configuration. Uses the default credential chain
+// unless a key/secret are explicitly set.
+func (c *ReplicaClient) config() *aws.Config {
+	config := &aws.Config{}
+
+	if c.AccessKeyID != "" || c.SecretAccessKey != "" {
+		config.Credentials = credentials.NewStaticCredentials(c.AccessKeyID, c.SecretAccessKey, "")
+	}
+	if c.Endpoint != "" {
+		config.Endpoint = aws.String(c.Endpoint)
+	}
+	if c.ForcePathStyle {
+		config.S3ForcePathStyle = aws.Bool(c.ForcePathStyle)
+	}
+	if c.SkipVerify {
+		config.HTTPClient = &http.Client{Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+		}}
+	}
+
+	return config
+}
+
+func (c *ReplicaClient) findBucketRegion(ctx context.Context, bucket string) (string, error) {
+	// Connect to US standard region to fetch info.
+	config := c.config()
+	config.Region = aws.String(DefaultRegion)
+	sess, err := session.NewSession(config)
+	if err != nil {
+		return "", err
+	}
+
+	// Fetch bucket location, if possible. Must be bucket owner.
+	// This call can return a nil location which means it's in us-east-1.
+	if out, err := s3.New(sess).GetBucketLocation(&s3.GetBucketLocationInput{
+		Bucket: aws.String(bucket),
+	}); err != nil {
+		return "", err
+	} else if out.LocationConstraint != nil {
+		return *out.LocationConstraint, nil
+	}
+	return DefaultRegion, nil
+}
+
+// Generations returns a list of available generation names.
+func (c *ReplicaClient) Generations(ctx context.Context) ([]string, error) {
+	if err := c.Init(ctx); err != nil {
+		return nil, err
+	}
+
+	var generations []string
+	if err := c.s3.ListObjectsPagesWithContext(ctx, &s3.ListObjectsInput{
+		Bucket:    aws.String(c.Bucket),
+		Prefix:    aws.String(litestream.GenerationsPath(c.Path) + "/"),
+		Delimiter: aws.String("/"),
+	}, func(page *s3.ListObjectsOutput, lastPage bool) bool {
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "LIST").Inc()
+
+		for _, prefix := range page.CommonPrefixes {
+			name := path.Base(*prefix.Prefix)
+			if !litestream.IsGenerationName(name) {
+				continue
+			}
+			generations = append(generations, name)
+		}
+		return true
+	}); err != nil {
+		return nil, err
+	}
+
+	return generations, nil
+}
+
+// DeleteGeneration deletes all snapshots & WAL segments within a generation.
+func (c *ReplicaClient) DeleteGeneration(ctx context.Context, generation string) error {
+	if err := c.Init(ctx); err != nil {
+		return err
+	}
+
+	dir, err := litestream.GenerationPath(c.Path, generation)
+	if err != nil {
+		return fmt.Errorf("cannot determine generation path: %w", err)
+	}
+
+	// Collect all files for the generation.
+	var objIDs []*s3.ObjectIdentifier
+	if err := c.s3.ListObjectsPagesWithContext(ctx, &s3.ListObjectsInput{
+		Bucket: aws.String(c.Bucket),
+		Prefix: aws.String(dir),
+	}, func(page *s3.ListObjectsOutput, lastPage bool) bool {
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "LIST").Inc()
+
+		for _, obj := range page.Contents {
+			objIDs = append(objIDs, &s3.ObjectIdentifier{Key: obj.Key})
+		}
+		return true
+	}); err != nil {
+		return err
+	}
+
+	// Delete all files in batches.
+	for len(objIDs) > 0 {
+		n := MaxKeys
+		if len(objIDs) < n {
+			n = len(objIDs)
+		}
+
+		out, err := c.s3.DeleteObjectsWithContext(ctx, &s3.DeleteObjectsInput{
+			Bucket: aws.String(c.Bucket),
+			Delete: &s3.Delete{Objects: objIDs[:n], Quiet: aws.Bool(true)},
+		})
+		if err != nil {
+			return err
+		}
+		if err := deleteOutputError(out); err != nil {
+			return err
+		}
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "DELETE").Inc()
+
+		objIDs = objIDs[n:]
+	}
+
+	// log.Printf("%s(%s): retainer: deleting generation: %s", r.db.Path(), r.Name(), generation)
+
+	return nil
+}
+
+// Snapshots returns an iterator over all available snapshots for a generation.
+func (c *ReplicaClient) Snapshots(ctx context.Context, generation string) (litestream.SnapshotIterator, error) {
+	if err := c.Init(ctx); err != nil {
+		return nil, err
+	}
+	return newSnapshotIterator(ctx, c, generation), nil
+}
+
+// WriteSnapshot writes LZ4 compressed data from rd into a file on disk.
+func (c *ReplicaClient) WriteSnapshot(ctx context.Context, generation string, index int, rd io.Reader) (info litestream.SnapshotInfo, err error) {
+	if err := c.Init(ctx); err != nil {
+		return info, err
+	}
+
+	key, err := litestream.SnapshotPath(c.Path, generation, index)
+	if err != nil {
+		return info, fmt.Errorf("cannot determine snapshot path: %w", err)
+	}
+	startTime := time.Now()
+
+	rc := internal.NewReadCounter(rd)
+	if _, err := c.uploader.UploadWithContext(ctx, &s3manager.UploadInput{
+		Bucket: aws.String(c.Bucket),
+		Key:    aws.String(key),
+		Body:   rc,
+	}); err != nil {
+		return info, err
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "PUT").Inc()
+	internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "PUT").Add(float64(rc.N()))
+
+	// log.Printf("%s(%s): snapshot: creating %s/%08x t=%s", r.db.Path(), r.Name(), generation, index, time.Since(startTime).Truncate(time.Millisecond))
+
+	return litestream.SnapshotInfo{
+		Generation: generation,
+		Index:      index,
+		Size:       rc.N(),
+		CreatedAt:  startTime.UTC(),
+	}, nil
+}
+
+// SnapshotReader returns a reader for snapshot data at the given generation/index.
+func (c *ReplicaClient) SnapshotReader(ctx context.Context, generation string, index int) (io.ReadCloser, error) {
+	if err := c.Init(ctx); err != nil {
+		return nil, err
+	}
+
+	key, err := litestream.SnapshotPath(c.Path, generation, index)
+	if err != nil {
+		return nil, fmt.Errorf("cannot determine snapshot path: %w", err)
+	}
+
+	out, err := c.s3.GetObjectWithContext(ctx, &s3.GetObjectInput{
+		Bucket: aws.String(c.Bucket),
+		Key:    aws.String(key),
+	})
+	if isNotExists(err) {
+		return nil, os.ErrNotExist
+	} else if err != nil {
+		return nil, err
+	}
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "GET").Inc()
+	internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "GET").Add(float64(*out.ContentLength))
+
+	return out.Body, nil
+}
+
+// DeleteSnapshot deletes a snapshot with the given generation & index.
+func (c *ReplicaClient) DeleteSnapshot(ctx context.Context, generation string, index int) error {
+	if err := c.Init(ctx); err != nil {
+		return err
+	}
+
+	key, err := litestream.SnapshotPath(c.Path, generation, index)
+	if err != nil {
+		return fmt.Errorf("cannot determine snapshot path: %w", err)
+	}
+
+	out, err := c.s3.DeleteObjectsWithContext(ctx, &s3.DeleteObjectsInput{
+		Bucket: aws.String(c.Bucket),
+		Delete: &s3.Delete{Objects: []*s3.ObjectIdentifier{{Key: &key}}, Quiet: aws.Bool(true)},
+	})
+	if err != nil {
+		return err
+	}
+	if err := deleteOutputError(out); err != nil {
+		return err
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "DELETE").Inc()
+	return nil
+}
+
+// WALSegments returns an iterator over all available WAL files for a generation.
+func (c *ReplicaClient) WALSegments(ctx context.Context, generation string) (litestream.WALSegmentIterator, error) {
+	if err := c.Init(ctx); err != nil {
+		return nil, err
+	}
+	return newWALSegmentIterator(ctx, c, generation), nil
+}
+
+// WriteWALSegment writes LZ4 compressed data from rd into a file on disk.
+func (c *ReplicaClient) WriteWALSegment(ctx context.Context, pos litestream.Pos, rd io.Reader) (info litestream.WALSegmentInfo, err error) {
+	if err := c.Init(ctx); err != nil {
+		return info, err
+	}
+
+	key, err := litestream.WALSegmentPath(c.Path, pos.Generation, pos.Index, pos.Offset)
+	if err != nil {
+		return info, fmt.Errorf("cannot determine wal segment path: %w", err)
+	}
+	startTime := time.Now()
+
+	rc := internal.NewReadCounter(rd)
+	if _, err := c.uploader.UploadWithContext(ctx, &s3manager.UploadInput{
+		Bucket: aws.String(c.Bucket),
+		Key:    aws.String(key),
+		Body:   rc,
+	}); err != nil {
+		return info, err
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "PUT").Inc()
+	internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "PUT").Add(float64(rc.N()))
+
+	return litestream.WALSegmentInfo{
+		Generation: pos.Generation,
+		Index:      pos.Index,
+		Offset:     pos.Offset,
+		Size:       rc.N(),
+		CreatedAt:  startTime.UTC(),
+	}, nil
+}
+
+// WALSegmentReader returns a reader for a section of WAL data at the given index.
+// Returns os.ErrNotExist if no matching index/offset is found.
+func (c *ReplicaClient) WALSegmentReader(ctx context.Context, pos litestream.Pos) (io.ReadCloser, error) {
+	if err := c.Init(ctx); err != nil {
+		return nil, err
+	}
+
+	key, err := litestream.WALSegmentPath(c.Path, pos.Generation, pos.Index, pos.Offset)
+	if err != nil {
+		return nil, fmt.Errorf("cannot determine wal segment path: %w", err)
+	}
+
+	out, err := c.s3.GetObjectWithContext(ctx, &s3.GetObjectInput{
+		Bucket: aws.String(c.Bucket),
+		Key:    aws.String(key),
+	})
+	if isNotExists(err) {
+		return nil, os.ErrNotExist
+	} else if err != nil {
+		return nil, err
+	}
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "GET").Inc()
+	internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "GET").Add(float64(*out.ContentLength))
+
+	return out.Body, nil
+}
+
+// DeleteWALSegments deletes WAL segments with at the given positions.
+func (c *ReplicaClient) DeleteWALSegments(ctx context.Context, a []litestream.Pos) error {
+	if err := c.Init(ctx); err != nil {
+		return err
+	}
+
+	objIDs := make([]*s3.ObjectIdentifier, MaxKeys)
+	for len(a) > 0 {
+		n := MaxKeys
+		if len(a) < n {
+			n = len(a)
+		}
+
+		// Generate a batch of object IDs for deleting the WAL segments.
+		for i, pos := range a[:n] {
+			key, err := litestream.WALSegmentPath(c.Path, pos.Generation, pos.Index, pos.Offset)
+			if err != nil {
+				return fmt.Errorf("cannot determine wal segment path: %w", err)
+			}
+			objIDs[i] = &s3.ObjectIdentifier{Key: &key}
+		}
+
+		// Delete S3 objects in bulk.
+		out, err := c.s3.DeleteObjectsWithContext(ctx, &s3.DeleteObjectsInput{
+			Bucket: aws.String(c.Bucket),
+			Delete: &s3.Delete{Objects: objIDs[:n], Quiet: aws.Bool(true)},
+		})
+		if err != nil {
+			return err
+		}
+		if err := deleteOutputError(out); err != nil {
+			return err
+		}
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "DELETE").Inc()
+
+		a = a[n:]
+	}
+
+	return nil
+}
+
+// DeleteAll deletes everything on the remote path. Mainly used for testing.
+func (c *ReplicaClient) DeleteAll(ctx context.Context) error {
+	if err := c.Init(ctx); err != nil {
+		return err
+	}
+
+	prefix := c.Path
+	if prefix != "" {
+		prefix += "/"
+	}
+
+	// Collect all files for the generation.
+	var objIDs []*s3.ObjectIdentifier
+	if err := c.s3.ListObjectsPagesWithContext(ctx, &s3.ListObjectsInput{
+		Bucket: aws.String(c.Bucket),
+		Prefix: aws.String(prefix),
+	}, func(page *s3.ListObjectsOutput, lastPage bool) bool {
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "LIST").Inc()
+
+		for _, obj := range page.Contents {
+			objIDs = append(objIDs, &s3.ObjectIdentifier{Key: obj.Key})
+		}
+		return true
+	}); err != nil {
+		return err
+	}
+
+	// Delete all files in batches.
+	for len(objIDs) > 0 {
+		n := MaxKeys
+		if len(objIDs) < n {
+			n = len(objIDs)
+		}
+
+		out, err := c.s3.DeleteObjectsWithContext(ctx, &s3.DeleteObjectsInput{
+			Bucket: aws.String(c.Bucket),
+			Delete: &s3.Delete{Objects: objIDs[:n], Quiet: aws.Bool(true)},
+		})
+		if err != nil {
+			return err
+		}
+		if err := deleteOutputError(out); err != nil {
+			return err
+		}
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "DELETE").Inc()
+
+		objIDs = objIDs[n:]
+	}
+
+	return nil
+}
+
+type snapshotIterator struct {
+	client     *ReplicaClient
+	generation string
+
+	ch     chan litestream.SnapshotInfo
+	g      errgroup.Group
+	ctx    context.Context
+	cancel func()
+
+	info litestream.SnapshotInfo
+	err  error
+}
+
+func newSnapshotIterator(ctx context.Context, client *ReplicaClient, generation string) *snapshotIterator {
+	itr := &snapshotIterator{
+		client:     client,
+		generation: generation,
+		ch:         make(chan litestream.SnapshotInfo),
+	}
+
+	itr.ctx, itr.cancel = context.WithCancel(ctx)
+	itr.g.Go(itr.fetch)
+
+	return itr
+}
+
+// fetch runs in a separate goroutine to fetch pages of objects and stream them to a channel.
+func (itr *snapshotIterator) fetch() error {
+	defer close(itr.ch)
+
+	dir, err := litestream.SnapshotsPath(itr.client.Path, itr.generation)
+	if err != nil {
+		return fmt.Errorf("cannot determine snapshots path: %w", err)
+	}
+
+	return itr.client.s3.ListObjectsPagesWithContext(itr.ctx, &s3.ListObjectsInput{
+		Bucket:    aws.String(itr.client.Bucket),
+		Prefix:    aws.String(dir + "/"),
+		Delimiter: aws.String("/"),
+	}, func(page *s3.ListObjectsOutput, lastPage bool) bool {
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "LIST").Inc()
+
+		for _, obj := range page.Contents {
+			key := path.Base(*obj.Key)
+			index, err := litestream.ParseSnapshotPath(key)
+			if err != nil {
+				continue
+			}
+
+			info := litestream.SnapshotInfo{
+				Generation: itr.generation,
+				Index:      index,
+				Size:       *obj.Size,
+				CreatedAt:  obj.LastModified.UTC(),
+			}
+
+			select {
+			case <-itr.ctx.Done():
+			case itr.ch <- info:
+			}
+		}
+		return true
+	})
+}
+
+func (itr *snapshotIterator) Close() (err error) {
+	err = itr.err
+
+	// Cancel context and wait for error group to finish.
+	itr.cancel()
+	if e := itr.g.Wait(); e != nil && err == nil {
+		err = e
+	}
+
+	return err
+}
+
+func (itr *snapshotIterator) Next() bool {
+	// Exit if an error has already occurred.
+	if itr.err != nil {
+		return false
+	}
+
+	// Return false if context was canceled or if there are no more snapshots.
+	// Otherwise fetch the next snapshot and store it on the iterator.
+	select {
+	case <-itr.ctx.Done():
+		return false
+	case info, ok := <-itr.ch:
+		if !ok {
+			return false
+		}
+		itr.info = info
+		return true
+	}
+}
+
+func (itr *snapshotIterator) Err() error { return itr.err }
+
+func (itr *snapshotIterator) Snapshot() litestream.SnapshotInfo {
+	return itr.info
+}
+
+type walSegmentIterator struct {
+	client     *ReplicaClient
+	generation string
+
+	ch     chan litestream.WALSegmentInfo
+	g      errgroup.Group
+	ctx    context.Context
+	cancel func()
+
+	info litestream.WALSegmentInfo
+	err  error
+}
+
+func newWALSegmentIterator(ctx context.Context, client *ReplicaClient, generation string) *walSegmentIterator {
+	itr := &walSegmentIterator{
+		client:     client,
+		generation: generation,
+		ch:         make(chan litestream.WALSegmentInfo),
+	}
+
+	itr.ctx, itr.cancel = context.WithCancel(ctx)
+	itr.g.Go(itr.fetch)
+
+	return itr
+}
+
+// fetch runs in a separate goroutine to fetch pages of objects and stream them to a channel.
+func (itr *walSegmentIterator) fetch() error {
+	defer close(itr.ch)
+
+	dir, err := litestream.WALPath(itr.client.Path, itr.generation)
+	if err != nil {
+		return fmt.Errorf("cannot determine wal path: %w", err)
+	}
+
+	return itr.client.s3.ListObjectsPagesWithContext(itr.ctx, &s3.ListObjectsInput{
+		Bucket:    aws.String(itr.client.Bucket),
+		Prefix:    aws.String(dir + "/"),
+		Delimiter: aws.String("/"),
+	}, func(page *s3.ListObjectsOutput, lastPage bool) bool {
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "LIST").Inc()
+
+		for _, obj := range page.Contents {
+			key := path.Base(*obj.Key)
+			index, offset, err := litestream.ParseWALSegmentPath(key)
+			if err != nil {
+				continue
+			}
+
+			info := litestream.WALSegmentInfo{
+				Generation: itr.generation,
+				Index:      index,
+				Offset:     offset,
+				Size:       *obj.Size,
+				CreatedAt:  obj.LastModified.UTC(),
+			}
+
+			select {
+			case <-itr.ctx.Done():
+				return false
+			case itr.ch <- info:
+			}
+		}
+		return true
+	})
+}
+
+func (itr *walSegmentIterator) Close() (err error) {
+	err = itr.err
+
+	// Cancel context and wait for error group to finish.
+	itr.cancel()
+	if e := itr.g.Wait(); e != nil && err == nil {
+		err = e
+	}
+
+	return err
+}
+
+func (itr *walSegmentIterator) Next() bool {
+	// Exit if an error has already occurred.
+	if itr.err != nil {
+		return false
+	}
+
+	// Return false if context was canceled or if there are no more segments.
+	// Otherwise fetch the next segment and store it on the iterator.
+	select {
+	case <-itr.ctx.Done():
+		return false
+	case info, ok := <-itr.ch:
+		if !ok {
+			return false
+		}
+		itr.info = info
+		return true
+	}
+}
+
+func (itr *walSegmentIterator) Err() error { return itr.err }
+
+func (itr *walSegmentIterator) WALSegment() litestream.WALSegmentInfo {
+	return itr.info
+}
+
+// ParseHost extracts data from a hostname depending on the service provider.
+func ParseHost(s string) (bucket, region, endpoint string, forcePathStyle bool) {
+	// Extract port if one is specified.
+	host, port, err := net.SplitHostPort(s)
+	if err != nil {
+		host = s
+	}
+
+	// Default to path-based URLs, except for with AWS S3 itself.
+	forcePathStyle = true
+
+	// Extract fields from provider-specific host formats.
+	scheme := "https"
+	if a := localhostRegex.FindStringSubmatch(host); a != nil {
+		bucket, region = a[1], "us-east-1"
+		scheme, endpoint = "http", "localhost"
+	} else if a := backblazeRegex.FindStringSubmatch(host); a != nil {
+		bucket, region = a[1], a[2]
+		endpoint = fmt.Sprintf("s3.%s.backblazeb2.com", region)
+	} else if a := filebaseRegex.FindStringSubmatch(host); a != nil {
+		bucket, endpoint = a[1], "s3.filebase.com"
+	} else if a := digitalOceanRegex.FindStringSubmatch(host); a != nil {
+		bucket, region = a[1], a[2]
+		endpoint = fmt.Sprintf("%s.digitaloceanspaces.com", region)
+	} else if a := linodeRegex.FindStringSubmatch(host); a != nil {
+		bucket, region = a[1], a[2]
+		endpoint = fmt.Sprintf("%s.linodeobjects.com", region)
+	} else {
+		bucket = host
+		forcePathStyle = false
+	}
+
+	// Add port back to endpoint, if available.
+	if endpoint != "" && port != "" {
+		endpoint = net.JoinHostPort(endpoint, port)
+	}
+
+	// Prepend scheme to endpoint.
+	if endpoint != "" {
+		endpoint = scheme + "://" + endpoint
+	}
+
+	return bucket, region, endpoint, forcePathStyle
+}
+
+var (
+	localhostRegex    = regexp.MustCompile(`^(?:(.+)\.)?localhost$`)
+	backblazeRegex    = regexp.MustCompile(`^(?:(.+)\.)?s3.([^.]+)\.backblazeb2.com$`)
+	filebaseRegex     = regexp.MustCompile(`^(?:(.+)\.)?s3.filebase.com$`)
+	digitalOceanRegex = regexp.MustCompile(`^(?:(.+)\.)?([^.]+)\.digitaloceanspaces.com$`)
+	linodeRegex       = regexp.MustCompile(`^(?:(.+)\.)?([^.]+)\.linodeobjects.com$`)
+)
+
+func isNotExists(err error) bool {
+	switch err := err.(type) {
+	case awserr.Error:
+		return err.Code() == `NoSuchKey`
+	default:
+		return false
+	}
+}
+
+func deleteOutputError(out *s3.DeleteObjectsOutput) error {
+	switch len(out.Errors) {
+	case 0:
+		return nil
+	case 1:
+		return fmt.Errorf("deleting object %s: %s - %s", *out.Errors[0].Key, *out.Errors[0].Code, *out.Errors[0].Message)
+	default:
+		return fmt.Errorf("%d errors occured deleting objects, %s: %s - (%s (and %d others)",
+			len(out.Errors), *out.Errors[0].Key, *out.Errors[0].Code, *out.Errors[0].Message, len(out.Errors)-1)
+	}
+}

s3/s3_test.go

@@ -1,80 +0,0 @@
-package s3_test
-
-import (
-	"testing"
-
-	"github.com/benbjohnson/litestream/s3"
-)
-
-func TestParseHost(t *testing.T) {
-	// Ensure non-specific hosts return as buckets.
-	t.Run("S3", func(t *testing.T) {
-		bucket, region, endpoint, forcePathStyle := s3.ParseHost(`test.litestream.io`)
-		if got, want := bucket, `test.litestream.io`; got != want {
-			t.Fatalf("bucket=%q, want %q", got, want)
-		} else if got, want := region, ``; got != want {
-			t.Fatalf("region=%q, want %q", got, want)
-		} else if got, want := endpoint, ``; got != want {
-			t.Fatalf("endpoint=%q, want %q", got, want)
-		} else if got, want := forcePathStyle, false; got != want {
-			t.Fatalf("forcePathStyle=%v, want %v", got, want)
-		}
-	})
-
-	// Ensure localhosts use an HTTP endpoint and extract the bucket name.
-	t.Run("Localhost", func(t *testing.T) {
-		t.Run("WithPort", func(t *testing.T) {
-			bucket, region, endpoint, forcePathStyle := s3.ParseHost(`test.localhost:9000`)
-			if got, want := bucket, `test`; got != want {
-				t.Fatalf("bucket=%q, want %q", got, want)
-			} else if got, want := region, `us-east-1`; got != want {
-				t.Fatalf("region=%q, want %q", got, want)
-			} else if got, want := endpoint, `http://localhost:9000`; got != want {
-				t.Fatalf("endpoint=%q, want %q", got, want)
-			} else if got, want := forcePathStyle, true; got != want {
-				t.Fatalf("forcePathStyle=%v, want %v", got, want)
-			}
-		})
-
-		t.Run("WithoutPort", func(t *testing.T) {
-			bucket, region, endpoint, forcePathStyle := s3.ParseHost(`test.localhost`)
-			if got, want := bucket, `test`; got != want {
-				t.Fatalf("bucket=%q, want %q", got, want)
-			} else if got, want := region, `us-east-1`; got != want {
-				t.Fatalf("region=%q, want %q", got, want)
-			} else if got, want := endpoint, `http://localhost`; got != want {
-				t.Fatalf("endpoint=%q, want %q", got, want)
-			} else if got, want := forcePathStyle, true; got != want {
-				t.Fatalf("forcePathStyle=%v, want %v", got, want)
-			}
-		})
-	})
-
-	// Ensure backblaze B2 URLs extract bucket, region, & endpoint from host.
-	t.Run("Backblaze", func(t *testing.T) {
-		bucket, region, endpoint, forcePathStyle := s3.ParseHost(`test-123.s3.us-west-000.backblazeb2.com`)
-		if got, want := bucket, `test-123`; got != want {
-			t.Fatalf("bucket=%q, want %q", got, want)
-		} else if got, want := region, `us-west-000`; got != want {
-			t.Fatalf("region=%q, want %q", got, want)
-		} else if got, want := endpoint, `https://s3.us-west-000.backblazeb2.com`; got != want {
-			t.Fatalf("endpoint=%q, want %q", got, want)
-		} else if got, want := forcePathStyle, true; got != want {
-			t.Fatalf("forcePathStyle=%v, want %v", got, want)
-		}
-	})
-
-	// Ensure GCS URLs extract bucket & endpoint from host.
-	t.Run("GCS", func(t *testing.T) {
-		bucket, region, endpoint, forcePathStyle := s3.ParseHost(`litestream.io.storage.googleapis.com`)
-		if got, want := bucket, `litestream.io`; got != want {
-			t.Fatalf("bucket=%q, want %q", got, want)
-		} else if got, want := region, `us-east-1`; got != want {
-			t.Fatalf("region=%q, want %q", got, want)
-		} else if got, want := endpoint, `https://storage.googleapis.com`; got != want {
-			t.Fatalf("endpoint=%q, want %q", got, want)
-		} else if got, want := forcePathStyle, true; got != want {
-			t.Fatalf("forcePathStyle=%v, want %v", got, want)
-		}
-	})
-}

sftp/replica_client.go

@@ -0,0 +1,495 @@
+package sftp
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"os"
+	"path"
+	"sort"
+	"sync"
+	"time"
+
+	"github.com/benbjohnson/litestream"
+	"github.com/benbjohnson/litestream/internal"
+	"github.com/pkg/sftp"
+	"golang.org/x/crypto/ssh"
+)
+
+// ReplicaClientType is the client type for this package.
+const ReplicaClientType = "sftp"
+
+// Default settings for replica client.
+const (
+	DefaultDialTimeout = 30 * time.Second
+)
+
+var _ litestream.ReplicaClient = (*ReplicaClient)(nil)
+
+// ReplicaClient is a client for writing snapshots & WAL segments to disk.
+type ReplicaClient struct {
+	mu         sync.Mutex
+	sshClient  *ssh.Client
+	sftpClient *sftp.Client
+
+	// SFTP connection info
+	Host        string
+	User        string
+	Password    string
+	Path        string
+	KeyPath     string
+	DialTimeout time.Duration
+}
+
+// NewReplicaClient returns a new instance of ReplicaClient.
+func NewReplicaClient() *ReplicaClient {
+	return &ReplicaClient{
+		DialTimeout: DefaultDialTimeout,
+	}
+}
+
+// Type returns "gcs" as the client type.
+func (c *ReplicaClient) Type() string {
+	return ReplicaClientType
+}
+
+// Init initializes the connection to GCS. No-op if already initialized.
+func (c *ReplicaClient) Init(ctx context.Context) (_ *sftp.Client, err error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if c.sftpClient != nil {
+		return c.sftpClient, nil
+	}
+
+	if c.User == "" {
+		return nil, fmt.Errorf("sftp user required")
+	}
+
+	// Build SSH configuration & auth methods
+	config := &ssh.ClientConfig{
+		User:            c.User,
+		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
+		BannerCallback:  ssh.BannerDisplayStderr(),
+	}
+	if c.Password != "" {
+		config.Auth = append(config.Auth, ssh.Password(c.Password))
+	}
+
+	if c.KeyPath != "" {
+		buf, err := os.ReadFile(c.KeyPath)
+		if err != nil {
+			return nil, fmt.Errorf("cannot read sftp key path: %w", err)
+		}
+
+		signer, err := ssh.ParsePrivateKey(buf)
+		if err != nil {
+			return nil, fmt.Errorf("cannot parse sftp key path: %w", err)
+		}
+		config.Auth = append(config.Auth, ssh.PublicKeys(signer))
+	}
+
+	// Append standard port, if necessary.
+	host := c.Host
+	if _, _, err := net.SplitHostPort(c.Host); err != nil {
+		host = net.JoinHostPort(c.Host, "22")
+	}
+
+	// Connect via SSH.
+	if c.sshClient, err = ssh.Dial("tcp", host, config); err != nil {
+		return nil, err
+	}
+
+	// Wrap connection with an SFTP client.
+	if c.sftpClient, err = sftp.NewClient(c.sshClient); err != nil {
+		c.sshClient.Close()
+		c.sshClient = nil
+		return nil, err
+	}
+
+	return c.sftpClient, nil
+}
+
+// Generations returns a list of available generation names.
+func (c *ReplicaClient) Generations(ctx context.Context) (_ []string, err error) {
+	defer func() { c.resetOnConnError(err) }()
+
+	sftpClient, err := c.Init(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	fis, err := sftpClient.ReadDir(litestream.GenerationsPath(c.Path))
+	if os.IsNotExist(err) {
+		return nil, nil
+	} else if err != nil {
+		return nil, err
+	}
+
+	var generations []string
+	for _, fi := range fis {
+		if !fi.IsDir() {
+			continue
+		}
+
+		name := path.Base(fi.Name())
+		if !litestream.IsGenerationName(name) {
+			continue
+		}
+		generations = append(generations, name)
+	}
+
+	sort.Strings(generations)
+
+	return generations, nil
+}
+
+// DeleteGeneration deletes all snapshots & WAL segments within a generation.
+func (c *ReplicaClient) DeleteGeneration(ctx context.Context, generation string) (err error) {
+	defer func() { c.resetOnConnError(err) }()
+
+	sftpClient, err := c.Init(ctx)
+	if err != nil {
+		return err
+	}
+
+	dir, err := litestream.GenerationPath(c.Path, generation)
+	if err != nil {
+		return fmt.Errorf("cannot determine generation path: %w", err)
+	}
+
+	var dirs []string
+	walker := sftpClient.Walk(dir)
+	for walker.Step() {
+		if err := walker.Err(); err != nil {
+			return fmt.Errorf("cannot walk path %q: %w", walker.Path(), err)
+		}
+		if walker.Stat().IsDir() {
+			dirs = append(dirs, walker.Path())
+			continue
+		}
+
+		if err := sftpClient.Remove(walker.Path()); err != nil && !os.IsNotExist(err) {
+			return fmt.Errorf("cannot delete file %q: %w", walker.Path(), err)
+		}
+
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "DELETE").Inc()
+	}
+
+	// Remove directories in reverse order after they have been emptied.
+	for i := len(dirs) - 1; i >= 0; i-- {
+		filename := dirs[i]
+		if err := sftpClient.RemoveDirectory(filename); err != nil && !os.IsNotExist(err) {
+			return fmt.Errorf("cannot delete directory %q: %w", filename, err)
+		}
+	}
+
+	// log.Printf("%s(%s): retainer: deleting generation: %s", r.db.Path(), r.Name(), generation)
+
+	return nil
+}
+
+// Snapshots returns an iterator over all available snapshots for a generation.
+func (c *ReplicaClient) Snapshots(ctx context.Context, generation string) (_ litestream.SnapshotIterator, err error) {
+	defer func() { c.resetOnConnError(err) }()
+
+	sftpClient, err := c.Init(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	dir, err := litestream.SnapshotsPath(c.Path, generation)
+	if err != nil {
+		return nil, fmt.Errorf("cannot determine snapshots path: %w", err)
+	}
+
+	fis, err := sftpClient.ReadDir(dir)
+	if os.IsNotExist(err) {
+		return litestream.NewSnapshotInfoSliceIterator(nil), nil
+	} else if err != nil {
+		return nil, err
+	}
+
+	// Iterate over every file and convert to metadata.
+	infos := make([]litestream.SnapshotInfo, 0, len(fis))
+	for _, fi := range fis {
+		// Parse index from filename.
+		index, err := litestream.ParseSnapshotPath(path.Base(fi.Name()))
+		if err != nil {
+			continue
+		}
+
+		infos = append(infos, litestream.SnapshotInfo{
+			Generation: generation,
+			Index:      index,
+			Size:       fi.Size(),
+			CreatedAt:  fi.ModTime().UTC(),
+		})
+	}
+
+	sort.Sort(litestream.SnapshotInfoSlice(infos))
+
+	return litestream.NewSnapshotInfoSliceIterator(infos), nil
+}
+
+// WriteSnapshot writes LZ4 compressed data from rd to the object storage.
+func (c *ReplicaClient) WriteSnapshot(ctx context.Context, generation string, index int, rd io.Reader) (info litestream.SnapshotInfo, err error) {
+	defer func() { c.resetOnConnError(err) }()
+
+	sftpClient, err := c.Init(ctx)
+	if err != nil {
+		return info, err
+	}
+
+	filename, err := litestream.SnapshotPath(c.Path, generation, index)
+	if err != nil {
+		return info, fmt.Errorf("cannot determine snapshot path: %w", err)
+	}
+	startTime := time.Now()
+
+	if err := sftpClient.MkdirAll(path.Dir(filename)); err != nil {
+		return info, fmt.Errorf("cannot make parent wal segment directory %q: %w", path.Dir(filename), err)
+	}
+
+	f, err := sftpClient.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC)
+	if err != nil {
+		return info, fmt.Errorf("cannot open snapshot file for writing: %w", err)
+	}
+	defer f.Close()
+
+	n, err := io.Copy(f, rd)
+	if err != nil {
+		return info, err
+	} else if err := f.Close(); err != nil {
+		return info, err
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "PUT").Inc()
+	internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "PUT").Add(float64(n))
+
+	// log.Printf("%s(%s): snapshot: creating %s/%08x t=%s", r.db.Path(), r.Name(), generation, index, time.Since(startTime).Truncate(time.Millisecond))
+
+	return litestream.SnapshotInfo{
+		Generation: generation,
+		Index:      index,
+		Size:       n,
+		CreatedAt:  startTime.UTC(),
+	}, nil
+}
+
+// SnapshotReader returns a reader for snapshot data at the given generation/index.
+func (c *ReplicaClient) SnapshotReader(ctx context.Context, generation string, index int) (_ io.ReadCloser, err error) {
+	defer func() { c.resetOnConnError(err) }()
+
+	sftpClient, err := c.Init(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	filename, err := litestream.SnapshotPath(c.Path, generation, index)
+	if err != nil {
+		return nil, fmt.Errorf("cannot determine snapshot path: %w", err)
+	}
+
+	f, err := sftpClient.Open(filename)
+	if err != nil {
+		return nil, err
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "GET").Inc()
+
+	return f, nil
+}
+
+// DeleteSnapshot deletes a snapshot with the given generation & index.
+func (c *ReplicaClient) DeleteSnapshot(ctx context.Context, generation string, index int) (err error) {
+	defer func() { c.resetOnConnError(err) }()
+
+	sftpClient, err := c.Init(ctx)
+	if err != nil {
+		return err
+	}
+
+	filename, err := litestream.SnapshotPath(c.Path, generation, index)
+	if err != nil {
+		return fmt.Errorf("cannot determine snapshot path: %w", err)
+	}
+
+	if err := sftpClient.Remove(filename); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("cannot delete snapshot %q: %w", filename, err)
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "DELETE").Inc()
+	return nil
+}
+
+// WALSegments returns an iterator over all available WAL files for a generation.
+func (c *ReplicaClient) WALSegments(ctx context.Context, generation string) (_ litestream.WALSegmentIterator, err error) {
+	defer func() { c.resetOnConnError(err) }()
+
+	sftpClient, err := c.Init(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	dir, err := litestream.WALPath(c.Path, generation)
+	if err != nil {
+		return nil, fmt.Errorf("cannot determine wal path: %w", err)
+	}
+
+	fis, err := sftpClient.ReadDir(dir)
+	if os.IsNotExist(err) {
+		return litestream.NewWALSegmentInfoSliceIterator(nil), nil
+	} else if err != nil {
+		return nil, err
+	}
+
+	// Iterate over every file and convert to metadata.
+	infos := make([]litestream.WALSegmentInfo, 0, len(fis))
+	for _, fi := range fis {
+		index, offset, err := litestream.ParseWALSegmentPath(path.Base(fi.Name()))
+		if err != nil {
+			continue
+		}
+
+		infos = append(infos, litestream.WALSegmentInfo{
+			Generation: generation,
+			Index:      index,
+			Offset:     offset,
+			Size:       fi.Size(),
+			CreatedAt:  fi.ModTime().UTC(),
+		})
+	}
+
+	sort.Sort(litestream.WALSegmentInfoSlice(infos))
+
+	return litestream.NewWALSegmentInfoSliceIterator(infos), nil
+}
+
+// WriteWALSegment writes LZ4 compressed data from rd into a file on disk.
+func (c *ReplicaClient) WriteWALSegment(ctx context.Context, pos litestream.Pos, rd io.Reader) (info litestream.WALSegmentInfo, err error) {
+	defer func() { c.resetOnConnError(err) }()
+
+	sftpClient, err := c.Init(ctx)
+	if err != nil {
+		return info, err
+	}
+
+	filename, err := litestream.WALSegmentPath(c.Path, pos.Generation, pos.Index, pos.Offset)
+	if err != nil {
+		return info, fmt.Errorf("cannot determine wal segment path: %w", err)
+	}
+	startTime := time.Now()
+
+	if err := sftpClient.MkdirAll(path.Dir(filename)); err != nil {
+		return info, fmt.Errorf("cannot make parent snapshot directory %q: %w", path.Dir(filename), err)
+	}
+
+	f, err := sftpClient.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC)
+	if err != nil {
+		return info, fmt.Errorf("cannot open snapshot file for writing: %w", err)
+	}
+	defer f.Close()
+
+	n, err := io.Copy(f, rd)
+	if err != nil {
+		return info, err
+	} else if err := f.Close(); err != nil {
+		return info, err
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "PUT").Inc()
+	internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "PUT").Add(float64(n))
+
+	return litestream.WALSegmentInfo{
+		Generation: pos.Generation,
+		Index:      pos.Index,
+		Offset:     pos.Offset,
+		Size:       n,
+		CreatedAt:  startTime.UTC(),
+	}, nil
+}
+
+// WALSegmentReader returns a reader for a section of WAL data at the given index.
+// Returns os.ErrNotExist if no matching index/offset is found.
+func (c *ReplicaClient) WALSegmentReader(ctx context.Context, pos litestream.Pos) (_ io.ReadCloser, err error) {
+	defer func() { c.resetOnConnError(err) }()
+
+	sftpClient, err := c.Init(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	filename, err := litestream.WALSegmentPath(c.Path, pos.Generation, pos.Index, pos.Offset)
+	if err != nil {
+		return nil, fmt.Errorf("cannot determine wal segment path: %w", err)
+	}
+
+	f, err := sftpClient.Open(filename)
+	if err != nil {
+		return nil, err
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "GET").Inc()
+
+	return f, nil
+}
+
+// DeleteWALSegments deletes WAL segments with at the given positions.
+func (c *ReplicaClient) DeleteWALSegments(ctx context.Context, a []litestream.Pos) (err error) {
+	defer func() { c.resetOnConnError(err) }()
+
+	sftpClient, err := c.Init(ctx)
+	if err != nil {
+		return err
+	}
+
+	for _, pos := range a {
+		filename, err := litestream.WALSegmentPath(c.Path, pos.Generation, pos.Index, pos.Offset)
+		if err != nil {
+			return fmt.Errorf("cannot determine wal segment path: %w", err)
+		}
+
+		if err := sftpClient.Remove(filename); err != nil && !os.IsNotExist(err) {
+			return fmt.Errorf("cannot delete wal segment %q: %w", filename, err)
+		}
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "DELETE").Inc()
+	}
+
+	return nil
+}
+
+// Cleanup deletes path & generations directories after empty.
+func (c *ReplicaClient) Cleanup(ctx context.Context) (err error) {
+	defer func() { c.resetOnConnError(err) }()
+
+	sftpClient, err := c.Init(ctx)
+	if err != nil {
+		return err
+	}
+
+	if err := sftpClient.RemoveDirectory(litestream.GenerationsPath(c.Path)); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("cannot delete generations path: %w", err)
+	} else if err := sftpClient.RemoveDirectory(c.Path); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("cannot delete path: %w", err)
+	}
+	return nil
+}
+
+// resetOnConnError closes & clears the client if a connection error occurs.
+func (c *ReplicaClient) resetOnConnError(err error) {
+	if !errors.Is(err, sftp.ErrSSHFxConnectionLost) {
+		return
+	}
+
+	if c.sftpClient != nil {
+		c.sftpClient.Close()
+		c.sftpClient = nil
+	}
+	if c.sshClient != nil {
+		c.sshClient.Close()
+		c.sshClient = nil
+	}
+}