Fix read replication stream restart position

export LITESTREAM_ACCESS_KEY_ID=your_key_id
export LITESTREAM_SECRET_ACCESS_KEY=your_access_key
export LITESTREAM_ENDPOINT=your_endpoint
export LITESTREAM_REGION=your_region
litestream replicate fruits.db s3://mybkt/fruits.db

.github/CONTRIBUTING.md

@@ -1,17 +1,18 @@
-## Open-source, not open-contribution
+## Contribution Policy
 
-[Similar to SQLite](https://www.sqlite.org/copyright.html), Litestream is open
-source but closed to contributions. This keeps the code base free of proprietary
-or licensed code but it also helps me continue to maintain and build Litestream.
+Initially, Litestream was closed to outside contributions. The goal was to
+reduce burnout by limiting the maintenance overhead of reviewing and validating
+third-party code. However, this policy is overly broad and has prevented small,
+easily testable patches from being contributed.
 
-As the author of [BoltDB](https://github.com/boltdb/bolt), I found that
-accepting and maintaining third party patches contributed to my burn out and
-I eventually archived the project. Writing databases & low-level replication
-tools involves nuance and simple one line changes can have profound and
-unexpected changes in correctness and performance. Small contributions
-typically required hours of my time to properly test and validate them.
+Litestream is now open to code contributions for bug fixes only. Features carry
+a long-term maintenance burden so they will not be accepted at this time.
+Please [submit an issue][new-issue] if you have a feature you'd like to
+request.
+
+If you find mistakes in the documentation, please submit a fix to the
+[documentation repository][docs].
+
+[new-issue]: https://github.com/benbjohnson/litestream/issues/new
+[docs]: https://github.com/benbjohnson/litestream.io
 
-I am grateful for community involvement, bug reports, & feature requests. I do
-not wish to come off as anything but welcoming, however, I've
-made the decision to keep this project closed to contributions for my own
-mental health and long term viability of the project.

.github/workflows/build_and_test.yml

@@ -0,0 +1,30 @@
+name: "Build and Unit Test"
+on: pull_request
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '1.17'
+
+      - uses: actions/cache@v2
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ inputs.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: ${{ inputs.os }}-go-
+
+      - name: Build binary
+        run: go install ./cmd/litestream
+
+      - name: Run unit tests
+        run: make testdata && go test -v --coverprofile=.coverage.out ./... && go tool cover -html .coverage.out -o .coverage.html
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: code-coverage
+          path: .coverage.html

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,38 @@
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+  schedule:
+    - cron: '20 16 * * 4'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'go' ]
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.github/workflows/golangci-lint.yml

@@ -0,0 +1,18 @@
+name: golangci-lint
+on:
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  golangci:
+    name: lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      
+      - uses: golangci/golangci-lint-action@v2
+        with:
+          version: latest
+          args: --timeout=10m

.github/workflows/integration_test.yml

@@ -0,0 +1,138 @@
+name: Integration Tests
+on: 
+  pull_request:
+    branches-ignore:
+      - "dependabot/**"
+
+jobs:
+  s3-integration-test:
+    name: Run S3 Integration Tests
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '1.17'
+
+      - uses: actions/cache@v2
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ inputs.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: ${{ inputs.os }}-go-
+
+      - run: go install ./cmd/litestream
+
+      - run: go test -v -run=TestReplicaClient ./integration -replica-type s3
+        env:
+          LITESTREAM_S3_ACCESS_KEY_ID: ${{ secrets.LITESTREAM_S3_ACCESS_KEY_ID }}
+          LITESTREAM_S3_SECRET_ACCESS_KEY: ${{ secrets.LITESTREAM_S3_SECRET_ACCESS_KEY }}
+          LITESTREAM_S3_REGION: us-east-1
+          LITESTREAM_S3_BUCKET: integration.litestream.io
+
+  gcp-integration-test:
+    name: Run GCP Integration Tests
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '1.17'
+
+      - uses: actions/cache@v2
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ inputs.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: ${{ inputs.os }}-go-
+
+      - name: Extract GCP credentials
+        run: 'echo "$GOOGLE_APPLICATION_CREDENTIALS" > /opt/gcp.json'
+        shell: bash
+        env:
+          GOOGLE_APPLICATION_CREDENTIALS: ${{secrets.GOOGLE_APPLICATION_CREDENTIALS}}
+
+      - run: go test -v -run=TestReplicaClient ./integration -replica-type gs
+        env:
+          GOOGLE_APPLICATION_CREDENTIALS:  /opt/gcp.json
+          LITESTREAM_GS_BUCKET:            integration.litestream.io
+
+  abs-integration-test:
+    name: Run Azure Blob Store Integration Tests
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '1.17'
+
+      - uses: actions/cache@v2
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ inputs.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: ${{ inputs.os }}-go-
+
+      - run: go test -v -run=TestReplicaClient ./integration -replica-type abs
+        env:
+          LITESTREAM_ABS_ACCOUNT_NAME:     ${{ secrets.LITESTREAM_ABS_ACCOUNT_NAME }}
+          LITESTREAM_ABS_ACCOUNT_KEY:      ${{ secrets.LITESTREAM_ABS_ACCOUNT_KEY }}
+          LITESTREAM_ABS_BUCKET:           integration
+
+  sftp-integration-test:
+    name: Run SFTP Integration Tests
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '1.17'
+
+      - uses: actions/cache@v2
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ inputs.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: ${{ inputs.os }}-go-
+
+      - name: Extract SSH key
+        run: 'echo "$LITESTREAM_SFTP_KEY" > /opt/id_ed25519'
+        shell: bash
+        env:
+          LITESTREAM_SFTP_KEY: ${{secrets.LITESTREAM_SFTP_KEY}}
+
+      - name: Run sftp tests w/ key
+        run: go test -v -run=TestReplicaClient ./integration -replica-type sftp
+        env:
+          LITESTREAM_SFTP_HOST: litestream-test-sftp.fly.dev:2222
+          LITESTREAM_SFTP_USER: litestream
+          LITESTREAM_SFTP_PATH: /litestream
+          LITESTREAM_SFTP_KEY_PATH:      /opt/id_ed25519
+
+      - name: Run sftp tests w/ password
+        run: go test -v -run=TestReplicaClient ./integration -replica-type sftp
+        env:
+          LITESTREAM_SFTP_HOST:     litestream-test-sftp.fly.dev:2222
+          LITESTREAM_SFTP_USER:     litestream
+          LITESTREAM_SFTP_PASSWORD: ${{ secrets.LITESTREAM_SFTP_PASSWORD }}
+          LITESTREAM_SFTP_PATH:     /litestream
+
+  long-running-test:
+    name: Run Long-Running Test
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-go@v2
+        with: 
+          go-version: '1.17'
+
+      - uses: actions/cache@v2
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ inputs.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: ${{ inputs.os }}-go-
+
+      - run: go install ./cmd/litestream
+
+      - run: go test -v -run=TestCmd_Replicate_LongRunning ./integration -long-running-duration 1m

.github/workflows/release.docker.yml

@@ -0,0 +1,51 @@
+on:
+  release:
+    types:
+      - published
+  pull_request:
+    types:
+      - opened
+      - synchronize
+      - reopened
+    branches-ignore:
+      - "dependabot/**"
+
+name: Release (Docker)
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    env:
+      PLATFORMS: "${{ github.event_name == 'release' && 'linux/amd64,linux/arm64,linux/arm/v7' || 'linux/amd64' }}"
+      VERSION: "${{ github.event_name == 'release' && github.event.release.name || github.sha }}"
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: docker/setup-qemu-action@v1
+      - uses: docker/setup-buildx-action@v1
+
+      - uses: docker/login-action@v1
+        with:
+          username: benbjohnson
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      
+      - id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: litestream/litestream
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=sha
+            type=sha,format=long
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+      
+      - uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          platforms: ${{ env.PLATFORMS }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          build-args: |
+            LITESTREAM_VERSION=${{ env.VERSION }}

.github/workflows/release.linux.yml

@@ -0,0 +1,134 @@
+on:
+  release:
+    types:
+      - published
+  pull_request:
+    types:
+      - opened
+      - synchronize
+      - reopened
+    branches-ignore:
+      - "dependabot/**"
+
+name: Release (Linux)
+jobs:
+  build:
+    runs-on: ubuntu-18.04
+    strategy:
+      matrix:
+        include:
+          - arch:   amd64
+            cc:     gcc
+
+          - arch:   amd64
+            cc:     gcc
+            static: true
+
+          - arch: arm64
+            cc:   aarch64-linux-gnu-gcc
+
+          - arch: arm64
+            cc:   aarch64-linux-gnu-gcc
+            static: true
+
+          - arch: arm
+            arm:  6
+            cc:   arm-linux-gnueabi-gcc
+
+          - arch: arm
+            arm:  6
+            cc:   arm-linux-gnueabi-gcc
+            static: true
+
+          - arch: arm
+            arm:  7
+            cc:   arm-linux-gnueabihf-gcc
+
+          - arch: arm
+            arm:  7
+            cc:   arm-linux-gnueabihf-gcc
+            static: true
+
+    env:
+      GOOS:   linux
+      GOARCH: ${{ matrix.arch }}
+      GOARM:  ${{ matrix.arm }}
+      CC:     ${{ matrix.cc }}
+      LDFLAGS: ${{ matrix.static && '-extldflags "-static"' || '' }}
+      TAGS: ${{ matrix.static && 'osusergo,netgo,sqlite_omit_load_extension' || '' }}
+      SUFFIX: "${{ matrix.static && '-static' || ''}}"
+      VERSION: "${{ github.event_name == 'release' && github.event.release.name || github.sha }}"
+
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '1.17'
+
+      - name: Install cross-compilers
+        run: |
+          sudo apt-get update 
+          sudo apt-get install -y gcc-aarch64-linux-gnu gcc-arm-linux-gnueabihf gcc-arm-linux-gnueabi
+
+      - name: Install nfpm
+        run: |
+          wget https://github.com/goreleaser/nfpm/releases/download/v2.2.3/nfpm_2.2.3_Linux_x86_64.tar.gz
+          tar zxvf nfpm_2.2.3_Linux_x86_64.tar.gz
+
+      - name: Build litestream
+        run: |
+          rm -rf dist && mkdir -p dist
+
+          cp etc/litestream.yml etc/litestream.service dist
+          cat etc/nfpm.yml | LITESTREAM_VERSION=${{ env.VERSION }} envsubst > dist/nfpm.yml
+
+          CGO_ENABLED=1 go build -ldflags "-s -w ${{ env.LDFLAGS }} -X 'main.Version=${{ env.VERSION }}'" -tags "${{ env.TAGS }}" -o dist/litestream ./cmd/litestream
+
+          cd dist
+          tar -czvf litestream-${{ env.VERSION }}-${{ env.GOOS }}-${{ env.GOARCH }}${{ env.GOARM }}${{ env.SUFFIX }}.tar.gz litestream
+          ../nfpm pkg --config nfpm.yml --packager deb --target litestream-${{ env.VERSION }}-${{ env.GOOS }}-${{ env.GOARCH }}${{ env.GOARM }}${{ env.SUFFIX }}.deb 
+
+      - name: Upload binary artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: litestream-${{ env.VERSION }}-${{ env.GOOS }}-${{ env.GOARCH }}${{ env.GOARM }}${{ env.SUFFIX }}.tar.gz
+          path: dist/litestream-${{ env.VERSION }}-${{ env.GOOS }}-${{ env.GOARCH }}${{ env.GOARM }}${{ env.SUFFIX }}.tar.gz
+          if-no-files-found: error
+
+      - name: Upload debian artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: litestream-${{ env.VERSION }}-${{ env.GOOS }}-${{ env.GOARCH }}${{ env.GOARM }}${{ env.SUFFIX }}.deb
+          path: dist/litestream-${{ env.VERSION }}-${{ env.GOOS }}-${{ env.GOARCH }}${{ env.GOARM }}${{ env.SUFFIX }}.deb
+          if-no-files-found: error
+
+      - name: Get release
+        id: release
+        uses: bruceadams/get-release@v1.2.3
+        if: github.event_name == 'release'
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+
+      - name: Upload release tarball
+        uses: actions/upload-release-asset@v1.0.2
+        if: github.event_name == 'release'
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        with:
+          upload_url: ${{ steps.release.outputs.upload_url }}
+          asset_path: ./dist/litestream-${{ env.VERSION }}-${{ env.GOOS }}-${{ env.GOARCH }}${{ env.GOARM }}${{ env.SUFFIX }}.tar.gz
+          asset_name: litestream-${{ env.VERSION }}-${{ env.GOOS }}-${{ env.GOARCH }}${{ env.GOARM }}${{ env.SUFFIX }}.tar.gz
+          asset_content_type: application/gzip
+
+      - name: Upload debian package
+        uses: actions/upload-release-asset@v1.0.2
+        if: github.event_name == 'release'
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        with:
+          upload_url: ${{ steps.release.outputs.upload_url }}
+          asset_path: ./dist/litestream-${{ env.VERSION }}-${{ env.GOOS }}-${{ env.GOARCH }}${{ env.GOARM }}${{ env.SUFFIX }}.deb
+          asset_name: litestream-${{ env.VERSION }}-${{ env.GOOS }}-${{ env.GOARCH }}${{ env.GOARM }}${{ env.SUFFIX }}.deb
+          asset_content_type: application/octet-stream

.github/workflows/release.yml

@@ -1,53 +0,0 @@
-on:
-  release:
-    types:
-      - created
-
-name: release
-jobs:
-  linux:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-go@v2
-
-      - id: release
-        uses: bruceadams/get-release@v1.2.2
-        env:
-          GITHUB_TOKEN: ${{ github.token }}
-
-      - name: Install nfpm
-        run: |
-          wget https://github.com/goreleaser/nfpm/releases/download/v2.2.3/nfpm_2.2.3_Linux_x86_64.tar.gz
-          tar zxvf nfpm_2.2.3_Linux_x86_64.tar.gz
-
-      - name: Build litestream
-        run: |
-          mkdir -p dist
-          cp etc/litestream.yml etc/litestream.service dist
-          cat etc/nfpm.yml | LITESTREAM_VERSION=${{ steps.release.outputs.tag_name }} envsubst > dist/nfpm.yml
-          go build -ldflags "-X 'main.Version=${{ steps.release.outputs.tag_name }}'" -o dist/litestream ./cmd/litestream
-          
-          cd dist
-          tar -czvf litestream-${{ steps.release.outputs.tag_name }}-linux-amd64.tar.gz litestream
-          ../nfpm pkg --config nfpm.yml --packager deb --target litestream-${{ steps.release.outputs.tag_name }}-linux-amd64.deb 
-
-      - name: Upload release binary
-        uses: actions/upload-release-asset@v1.0.2
-        env:
-          GITHUB_TOKEN: ${{ github.token }}
-        with:
-          upload_url: ${{ steps.release.outputs.upload_url }}
-          asset_path: ./dist/litestream-${{ steps.release.outputs.tag_name }}-linux-amd64.tar.gz
-          asset_name: litestream-${{ steps.release.outputs.tag_name }}-linux-amd64.tar.gz
-          asset_content_type: application/gzip
-
-      - name: Upload debian package
-        uses: actions/upload-release-asset@v1.0.2
-        env:
-          GITHUB_TOKEN: ${{ github.token }}
-        with:
-          upload_url: ${{ steps.release.outputs.upload_url }}
-          asset_path: ./dist/litestream-${{ steps.release.outputs.tag_name }}-linux-amd64.deb
-          asset_name: litestream-${{ steps.release.outputs.tag_name }}-linux-amd64.deb
-          asset_content_type: application/octet-stream

.github/workflows/test.yml

@@ -1,21 +0,0 @@
-on: push
-name: test
-jobs:
-  test:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/setup-go@v2
-        with:
-          go-version: '1.15'
-
-      - uses: actions/checkout@v2
-
-      - uses: actions/cache@v2
-        with:
-          path: ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go-
-
-      - name: Run unit tests
-        run: go test -v ./...

.gitignore

@@ -1,2 +1,3 @@
+.coverage.*
 .DS_Store
 /dist

Dockerfile

@@ -0,0 +1,16 @@
+FROM golang:1.17 as builder
+
+WORKDIR /src/litestream
+COPY . .
+
+ARG LITESTREAM_VERSION=latest
+
+RUN --mount=type=cache,target=/root/.cache/go-build \
+	--mount=type=cache,target=/go/pkg \
+	go build -ldflags "-s -w -X 'main.Version=${LITESTREAM_VERSION}' -extldflags '-static'" -tags osusergo,netgo,sqlite_omit_load_extension -o /usr/local/bin/litestream ./cmd/litestream
+
+
+FROM alpine
+COPY --from=builder /usr/local/bin/litestream /usr/local/bin/litestream
+ENTRYPOINT ["/usr/local/bin/litestream"]
+CMD []

LICENSE

@@ -1,674 +1,202 @@
-                    GNU GENERAL PUBLIC LICENSE
-                       Version 3, 29 June 2007
 
- Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
- Everyone is permitted to copy and distribute verbatim copies
- of this license document, but changing it is not allowed.
-
-                            Preamble
-
-  The GNU General Public License is a free, copyleft license for
-software and other kinds of works.
-
-  The licenses for most software and other practical works are designed
-to take away your freedom to share and change the works.  By contrast,
-the GNU General Public License is intended to guarantee your freedom to
-share and change all versions of a program--to make sure it remains free
-software for all its users.  We, the Free Software Foundation, use the
-GNU General Public License for most of our software; it applies also to
-any other work released this way by its authors.  You can apply it to
-your programs, too.
-
-  When we speak of free software, we are referring to freedom, not
-price.  Our General Public Licenses are designed to make sure that you
-have the freedom to distribute copies of free software (and charge for
-them if you wish), that you receive source code or can get it if you
-want it, that you can change the software or use pieces of it in new
-free programs, and that you know you can do these things.
-
-  To protect your rights, we need to prevent others from denying you
-these rights or asking you to surrender the rights.  Therefore, you have
-certain responsibilities if you distribute copies of the software, or if
-you modify it: responsibilities to respect the freedom of others.
-
-  For example, if you distribute copies of such a program, whether
-gratis or for a fee, you must pass on to the recipients the same
-freedoms that you received.  You must make sure that they, too, receive
-or can get the source code.  And you must show them these terms so they
-know their rights.
-
-  Developers that use the GNU GPL protect your rights with two steps:
-(1) assert copyright on the software, and (2) offer you this License
-giving you legal permission to copy, distribute and/or modify it.
-
-  For the developers' and authors' protection, the GPL clearly explains
-that there is no warranty for this free software.  For both users' and
-authors' sake, the GPL requires that modified versions be marked as
-changed, so that their problems will not be attributed erroneously to
-authors of previous versions.
-
-  Some devices are designed to deny users access to install or run
-modified versions of the software inside them, although the manufacturer
-can do so.  This is fundamentally incompatible with the aim of
-protecting users' freedom to change the software.  The systematic
-pattern of such abuse occurs in the area of products for individuals to
-use, which is precisely where it is most unacceptable.  Therefore, we
-have designed this version of the GPL to prohibit the practice for those
-products.  If such problems arise substantially in other domains, we
-stand ready to extend this provision to those domains in future versions
-of the GPL, as needed to protect the freedom of users.
-
-  Finally, every program is threatened constantly by software patents.
-States should not allow patents to restrict development and use of
-software on general-purpose computers, but in those that do, we wish to
-avoid the special danger that patents applied to a free program could
-make it effectively proprietary.  To prevent this, the GPL assures that
-patents cannot be used to render the program non-free.
-
-  The precise terms and conditions for copying, distribution and
-modification follow.
-
-                       TERMS AND CONDITIONS
-
-  0. Definitions.
-
-  "This License" refers to version 3 of the GNU General Public License.
-
-  "Copyright" also means copyright-like laws that apply to other kinds of
-works, such as semiconductor masks.
-
-  "The Program" refers to any copyrightable work licensed under this
-License.  Each licensee is addressed as "you".  "Licensees" and
-"recipients" may be individuals or organizations.
-
-  To "modify" a work means to copy from or adapt all or part of the work
-in a fashion requiring copyright permission, other than the making of an
-exact copy.  The resulting work is called a "modified version" of the
-earlier work or a work "based on" the earlier work.
-
-  A "covered work" means either the unmodified Program or a work based
-on the Program.
-
-  To "propagate" a work means to do anything with it that, without
-permission, would make you directly or secondarily liable for
-infringement under applicable copyright law, except executing it on a
-computer or modifying a private copy.  Propagation includes copying,
-distribution (with or without modification), making available to the
-public, and in some countries other activities as well.
-
-  To "convey" a work means any kind of propagation that enables other
-parties to make or receive copies.  Mere interaction with a user through
-a computer network, with no transfer of a copy, is not conveying.
-
-  An interactive user interface displays "Appropriate Legal Notices"
-to the extent that it includes a convenient and prominently visible
-feature that (1) displays an appropriate copyright notice, and (2)
-tells the user that there is no warranty for the work (except to the
-extent that warranties are provided), that licensees may convey the
-work under this License, and how to view a copy of this License.  If
-the interface presents a list of user commands or options, such as a
-menu, a prominent item in the list meets this criterion.
-
-  1. Source Code.
-
-  The "source code" for a work means the preferred form of the work
-for making modifications to it.  "Object code" means any non-source
-form of a work.
-
-  A "Standard Interface" means an interface that either is an official
-standard defined by a recognized standards body, or, in the case of
-interfaces specified for a particular programming language, one that
-is widely used among developers working in that language.
-
-  The "System Libraries" of an executable work include anything, other
-than the work as a whole, that (a) is included in the normal form of
-packaging a Major Component, but which is not part of that Major
-Component, and (b) serves only to enable use of the work with that
-Major Component, or to implement a Standard Interface for which an
-implementation is available to the public in source code form.  A
-"Major Component", in this context, means a major essential component
-(kernel, window system, and so on) of the specific operating system
-(if any) on which the executable work runs, or a compiler used to
-produce the work, or an object code interpreter used to run it.
-
-  The "Corresponding Source" for a work in object code form means all
-the source code needed to generate, install, and (for an executable
-work) run the object code and to modify the work, including scripts to
-control those activities.  However, it does not include the work's
-System Libraries, or general-purpose tools or generally available free
-programs which are used unmodified in performing those activities but
-which are not part of the work.  For example, Corresponding Source
-includes interface definition files associated with source files for
-the work, and the source code for shared libraries and dynamically
-linked subprograms that the work is specifically designed to require,
-such as by intimate data communication or control flow between those
-subprograms and other parts of the work.
-
-  The Corresponding Source need not include anything that users
-can regenerate automatically from other parts of the Corresponding
-Source.
-
-  The Corresponding Source for a work in source code form is that
-same work.
-
-  2. Basic Permissions.
-
-  All rights granted under this License are granted for the term of
-copyright on the Program, and are irrevocable provided the stated
-conditions are met.  This License explicitly affirms your unlimited
-permission to run the unmodified Program.  The output from running a
-covered work is covered by this License only if the output, given its
-content, constitutes a covered work.  This License acknowledges your
-rights of fair use or other equivalent, as provided by copyright law.
-
-  You may make, run and propagate covered works that you do not
-convey, without conditions so long as your license otherwise remains
-in force.  You may convey covered works to others for the sole purpose
-of having them make modifications exclusively for you, or provide you
-with facilities for running those works, provided that you comply with
-the terms of this License in conveying all material for which you do
-not control copyright.  Those thus making or running the covered works
-for you must do so exclusively on your behalf, under your direction
-and control, on terms that prohibit them from making any copies of
-your copyrighted material outside their relationship with you.
-
-  Conveying under any other circumstances is permitted solely under
-the conditions stated below.  Sublicensing is not allowed; section 10
-makes it unnecessary.
-
-  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
-
-  No covered work shall be deemed part of an effective technological
-measure under any applicable law fulfilling obligations under article
-11 of the WIPO copyright treaty adopted on 20 December 1996, or
-similar laws prohibiting or restricting circumvention of such
-measures.
-
-  When you convey a covered work, you waive any legal power to forbid
-circumvention of technological measures to the extent such circumvention
-is effected by exercising rights under this License with respect to
-the covered work, and you disclaim any intention to limit operation or
-modification of the work as a means of enforcing, against the work's
-users, your or third parties' legal rights to forbid circumvention of
-technological measures.
-
-  4. Conveying Verbatim Copies.
-
-  You may convey verbatim copies of the Program's source code as you
-receive it, in any medium, provided that you conspicuously and
-appropriately publish on each copy an appropriate copyright notice;
-keep intact all notices stating that this License and any
-non-permissive terms added in accord with section 7 apply to the code;
-keep intact all notices of the absence of any warranty; and give all
-recipients a copy of this License along with the Program.
-
-  You may charge any price or no price for each copy that you convey,
-and you may offer support or warranty protection for a fee.
-
-  5. Conveying Modified Source Versions.
-
-  You may convey a work based on the Program, or the modifications to
-produce it from the Program, in the form of source code under the
-terms of section 4, provided that you also meet all of these conditions:
-
-    a) The work must carry prominent notices stating that you modified
-    it, and giving a relevant date.
-
-    b) The work must carry prominent notices stating that it is
-    released under this License and any conditions added under section
-    7.  This requirement modifies the requirement in section 4 to
-    "keep intact all notices".
-
-    c) You must license the entire work, as a whole, under this
-    License to anyone who comes into possession of a copy.  This
-    License will therefore apply, along with any applicable section 7
-    additional terms, to the whole of the work, and all its parts,
-    regardless of how they are packaged.  This License gives no
-    permission to license the work in any other way, but it does not
-    invalidate such permission if you have separately received it.
-
-    d) If the work has interactive user interfaces, each must display
-    Appropriate Legal Notices; however, if the Program has interactive
-    interfaces that do not display Appropriate Legal Notices, your
-    work need not make them do so.
-
-  A compilation of a covered work with other separate and independent
-works, which are not by their nature extensions of the covered work,
-and which are not combined with it such as to form a larger program,
-in or on a volume of a storage or distribution medium, is called an
-"aggregate" if the compilation and its resulting copyright are not
-used to limit the access or legal rights of the compilation's users
-beyond what the individual works permit.  Inclusion of a covered work
-in an aggregate does not cause this License to apply to the other
-parts of the aggregate.
-
-  6. Conveying Non-Source Forms.
-
-  You may convey a covered work in object code form under the terms
-of sections 4 and 5, provided that you also convey the
-machine-readable Corresponding Source under the terms of this License,
-in one of these ways:
-
-    a) Convey the object code in, or embodied in, a physical product
-    (including a physical distribution medium), accompanied by the
-    Corresponding Source fixed on a durable physical medium
-    customarily used for software interchange.
-
-    b) Convey the object code in, or embodied in, a physical product
-    (including a physical distribution medium), accompanied by a
-    written offer, valid for at least three years and valid for as
-    long as you offer spare parts or customer support for that product
-    model, to give anyone who possesses the object code either (1) a
-    copy of the Corresponding Source for all the software in the
-    product that is covered by this License, on a durable physical
-    medium customarily used for software interchange, for a price no
-    more than your reasonable cost of physically performing this
-    conveying of source, or (2) access to copy the
-    Corresponding Source from a network server at no charge.
-
-    c) Convey individual copies of the object code with a copy of the
-    written offer to provide the Corresponding Source.  This
-    alternative is allowed only occasionally and noncommercially, and
-    only if you received the object code with such an offer, in accord
-    with subsection 6b.
-
-    d) Convey the object code by offering access from a designated
-    place (gratis or for a charge), and offer equivalent access to the
-    Corresponding Source in the same way through the same place at no
-    further charge.  You need not require recipients to copy the
-    Corresponding Source along with the object code.  If the place to
-    copy the object code is a network server, the Corresponding Source
-    may be on a different server (operated by you or a third party)
-    that supports equivalent copying facilities, provided you maintain
-    clear directions next to the object code saying where to find the
-    Corresponding Source.  Regardless of what server hosts the
-    Corresponding Source, you remain obligated to ensure that it is
-    available for as long as needed to satisfy these requirements.
-
-    e) Convey the object code using peer-to-peer transmission, provided
-    you inform other peers where the object code and Corresponding
-    Source of the work are being offered to the general public at no
-    charge under subsection 6d.
-
-  A separable portion of the object code, whose source code is excluded
-from the Corresponding Source as a System Library, need not be
-included in conveying the object code work.
-
-  A "User Product" is either (1) a "consumer product", which means any
-tangible personal property which is normally used for personal, family,
-or household purposes, or (2) anything designed or sold for incorporation
-into a dwelling.  In determining whether a product is a consumer product,
-doubtful cases shall be resolved in favor of coverage.  For a particular
-product received by a particular user, "normally used" refers to a
-typical or common use of that class of product, regardless of the status
-of the particular user or of the way in which the particular user
-actually uses, or expects or is expected to use, the product.  A product
-is a consumer product regardless of whether the product has substantial
-commercial, industrial or non-consumer uses, unless such uses represent
-the only significant mode of use of the product.
-
-  "Installation Information" for a User Product means any methods,
-procedures, authorization keys, or other information required to install
-and execute modified versions of a covered work in that User Product from
-a modified version of its Corresponding Source.  The information must
-suffice to ensure that the continued functioning of the modified object
-code is in no case prevented or interfered with solely because
-modification has been made.
-
-  If you convey an object code work under this section in, or with, or
-specifically for use in, a User Product, and the conveying occurs as
-part of a transaction in which the right of possession and use of the
-User Product is transferred to the recipient in perpetuity or for a
-fixed term (regardless of how the transaction is characterized), the
-Corresponding Source conveyed under this section must be accompanied
-by the Installation Information.  But this requirement does not apply
-if neither you nor any third party retains the ability to install
-modified object code on the User Product (for example, the work has
-been installed in ROM).
-
-  The requirement to provide Installation Information does not include a
-requirement to continue to provide support service, warranty, or updates
-for a work that has been modified or installed by the recipient, or for
-the User Product in which it has been modified or installed.  Access to a
-network may be denied when the modification itself materially and
-adversely affects the operation of the network or violates the rules and
-protocols for communication across the network.
-
-  Corresponding Source conveyed, and Installation Information provided,
-in accord with this section must be in a format that is publicly
-documented (and with an implementation available to the public in
-source code form), and must require no special password or key for
-unpacking, reading or copying.
-
-  7. Additional Terms.
-
-  "Additional permissions" are terms that supplement the terms of this
-License by making exceptions from one or more of its conditions.
-Additional permissions that are applicable to the entire Program shall
-be treated as though they were included in this License, to the extent
-that they are valid under applicable law.  If additional permissions
-apply only to part of the Program, that part may be used separately
-under those permissions, but the entire Program remains governed by
-this License without regard to the additional permissions.
-
-  When you convey a copy of a covered work, you may at your option
-remove any additional permissions from that copy, or from any part of
-it.  (Additional permissions may be written to require their own
-removal in certain cases when you modify the work.)  You may place
-additional permissions on material, added by you to a covered work,
-for which you have or can give appropriate copyright permission.
-
-  Notwithstanding any other provision of this License, for material you
-add to a covered work, you may (if authorized by the copyright holders of
-that material) supplement the terms of this License with terms:
-
-    a) Disclaiming warranty or limiting liability differently from the
-    terms of sections 15 and 16 of this License; or
-
-    b) Requiring preservation of specified reasonable legal notices or
-    author attributions in that material or in the Appropriate Legal
-    Notices displayed by works containing it; or
-
-    c) Prohibiting misrepresentation of the origin of that material, or
-    requiring that modified versions of such material be marked in
-    reasonable ways as different from the original version; or
-
-    d) Limiting the use for publicity purposes of names of licensors or
-    authors of the material; or
-
-    e) Declining to grant rights under trademark law for use of some
-    trade names, trademarks, or service marks; or
-
-    f) Requiring indemnification of licensors and authors of that
-    material by anyone who conveys the material (or modified versions of
-    it) with contractual assumptions of liability to the recipient, for
-    any liability that these contractual assumptions directly impose on
-    those licensors and authors.
-
-  All other non-permissive additional terms are considered "further
-restrictions" within the meaning of section 10.  If the Program as you
-received it, or any part of it, contains a notice stating that it is
-governed by this License along with a term that is a further
-restriction, you may remove that term.  If a license document contains
-a further restriction but permits relicensing or conveying under this
-License, you may add to a covered work material governed by the terms
-of that license document, provided that the further restriction does
-not survive such relicensing or conveying.
-
-  If you add terms to a covered work in accord with this section, you
-must place, in the relevant source files, a statement of the
-additional terms that apply to those files, or a notice indicating
-where to find the applicable terms.
-
-  Additional terms, permissive or non-permissive, may be stated in the
-form of a separately written license, or stated as exceptions;
-the above requirements apply either way.
-
-  8. Termination.
-
-  You may not propagate or modify a covered work except as expressly
-provided under this License.  Any attempt otherwise to propagate or
-modify it is void, and will automatically terminate your rights under
-this License (including any patent licenses granted under the third
-paragraph of section 11).
-
-  However, if you cease all violation of this License, then your
-license from a particular copyright holder is reinstated (a)
-provisionally, unless and until the copyright holder explicitly and
-finally terminates your license, and (b) permanently, if the copyright
-holder fails to notify you of the violation by some reasonable means
-prior to 60 days after the cessation.
-
-  Moreover, your license from a particular copyright holder is
-reinstated permanently if the copyright holder notifies you of the
-violation by some reasonable means, this is the first time you have
-received notice of violation of this License (for any work) from that
-copyright holder, and you cure the violation prior to 30 days after
-your receipt of the notice.
-
-  Termination of your rights under this section does not terminate the
-licenses of parties who have received copies or rights from you under
-this License.  If your rights have been terminated and not permanently
-reinstated, you do not qualify to receive new licenses for the same
-material under section 10.
-
-  9. Acceptance Not Required for Having Copies.
-
-  You are not required to accept this License in order to receive or
-run a copy of the Program.  Ancillary propagation of a covered work
-occurring solely as a consequence of using peer-to-peer transmission
-to receive a copy likewise does not require acceptance.  However,
-nothing other than this License grants you permission to propagate or
-modify any covered work.  These actions infringe copyright if you do
-not accept this License.  Therefore, by modifying or propagating a
-covered work, you indicate your acceptance of this License to do so.
-
-  10. Automatic Licensing of Downstream Recipients.
-
-  Each time you convey a covered work, the recipient automatically
-receives a license from the original licensors, to run, modify and
-propagate that work, subject to this License.  You are not responsible
-for enforcing compliance by third parties with this License.
-
-  An "entity transaction" is a transaction transferring control of an
-organization, or substantially all assets of one, or subdividing an
-organization, or merging organizations.  If propagation of a covered
-work results from an entity transaction, each party to that
-transaction who receives a copy of the work also receives whatever
-licenses to the work the party's predecessor in interest had or could
-give under the previous paragraph, plus a right to possession of the
-Corresponding Source of the work from the predecessor in interest, if
-the predecessor has it or can get it with reasonable efforts.
-
-  You may not impose any further restrictions on the exercise of the
-rights granted or affirmed under this License.  For example, you may
-not impose a license fee, royalty, or other charge for exercise of
-rights granted under this License, and you may not initiate litigation
-(including a cross-claim or counterclaim in a lawsuit) alleging that
-any patent claim is infringed by making, using, selling, offering for
-sale, or importing the Program or any portion of it.
-
-  11. Patents.
-
-  A "contributor" is a copyright holder who authorizes use under this
-License of the Program or a work on which the Program is based.  The
-work thus licensed is called the contributor's "contributor version".
-
-  A contributor's "essential patent claims" are all patent claims
-owned or controlled by the contributor, whether already acquired or
-hereafter acquired, that would be infringed by some manner, permitted
-by this License, of making, using, or selling its contributor version,
-but do not include claims that would be infringed only as a
-consequence of further modification of the contributor version.  For
-purposes of this definition, "control" includes the right to grant
-patent sublicenses in a manner consistent with the requirements of
-this License.
-
-  Each contributor grants you a non-exclusive, worldwide, royalty-free
-patent license under the contributor's essential patent claims, to
-make, use, sell, offer for sale, import and otherwise run, modify and
-propagate the contents of its contributor version.
-
-  In the following three paragraphs, a "patent license" is any express
-agreement or commitment, however denominated, not to enforce a patent
-(such as an express permission to practice a patent or covenant not to
-sue for patent infringement).  To "grant" such a patent license to a
-party means to make such an agreement or commitment not to enforce a
-patent against the party.
-
-  If you convey a covered work, knowingly relying on a patent license,
-and the Corresponding Source of the work is not available for anyone
-to copy, free of charge and under the terms of this License, through a
-publicly available network server or other readily accessible means,
-then you must either (1) cause the Corresponding Source to be so
-available, or (2) arrange to deprive yourself of the benefit of the
-patent license for this particular work, or (3) arrange, in a manner
-consistent with the requirements of this License, to extend the patent
-license to downstream recipients.  "Knowingly relying" means you have
-actual knowledge that, but for the patent license, your conveying the
-covered work in a country, or your recipient's use of the covered work
-in a country, would infringe one or more identifiable patents in that
-country that you have reason to believe are valid.
-
-  If, pursuant to or in connection with a single transaction or
-arrangement, you convey, or propagate by procuring conveyance of, a
-covered work, and grant a patent license to some of the parties
-receiving the covered work authorizing them to use, propagate, modify
-or convey a specific copy of the covered work, then the patent license
-you grant is automatically extended to all recipients of the covered
-work and works based on it.
-
-  A patent license is "discriminatory" if it does not include within
-the scope of its coverage, prohibits the exercise of, or is
-conditioned on the non-exercise of one or more of the rights that are
-specifically granted under this License.  You may not convey a covered
-work if you are a party to an arrangement with a third party that is
-in the business of distributing software, under which you make payment
-to the third party based on the extent of your activity of conveying
-the work, and under which the third party grants, to any of the
-parties who would receive the covered work from you, a discriminatory
-patent license (a) in connection with copies of the covered work
-conveyed by you (or copies made from those copies), or (b) primarily
-for and in connection with specific products or compilations that
-contain the covered work, unless you entered into that arrangement,
-or that patent license was granted, prior to 28 March 2007.
-
-  Nothing in this License shall be construed as excluding or limiting
-any implied license or other defenses to infringement that may
-otherwise be available to you under applicable patent law.
-
-  12. No Surrender of Others' Freedom.
-
-  If conditions are imposed on you (whether by court order, agreement or
-otherwise) that contradict the conditions of this License, they do not
-excuse you from the conditions of this License.  If you cannot convey a
-covered work so as to satisfy simultaneously your obligations under this
-License and any other pertinent obligations, then as a consequence you may
-not convey it at all.  For example, if you agree to terms that obligate you
-to collect a royalty for further conveying from those to whom you convey
-the Program, the only way you could satisfy both those terms and this
-License would be to refrain entirely from conveying the Program.
-
-  13. Use with the GNU Affero General Public License.
-
-  Notwithstanding any other provision of this License, you have
-permission to link or combine any covered work with a work licensed
-under version 3 of the GNU Affero General Public License into a single
-combined work, and to convey the resulting work.  The terms of this
-License will continue to apply to the part which is the covered work,
-but the special requirements of the GNU Affero General Public License,
-section 13, concerning interaction through a network will apply to the
-combination as such.
-
-  14. Revised Versions of this License.
-
-  The Free Software Foundation may publish revised and/or new versions of
-the GNU General Public License from time to time.  Such new versions will
-be similar in spirit to the present version, but may differ in detail to
-address new problems or concerns.
-
-  Each version is given a distinguishing version number.  If the
-Program specifies that a certain numbered version of the GNU General
-Public License "or any later version" applies to it, you have the
-option of following the terms and conditions either of that numbered
-version or of any later version published by the Free Software
-Foundation.  If the Program does not specify a version number of the
-GNU General Public License, you may choose any version ever published
-by the Free Software Foundation.
-
-  If the Program specifies that a proxy can decide which future
-versions of the GNU General Public License can be used, that proxy's
-public statement of acceptance of a version permanently authorizes you
-to choose that version for the Program.
-
-  Later license versions may give you additional or different
-permissions.  However, no additional obligations are imposed on any
-author or copyright holder as a result of your choosing to follow a
-later version.
-
-  15. Disclaimer of Warranty.
-
-  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
-APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
-HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
-OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
-THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
-IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
-ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
-
-  16. Limitation of Liability.
-
-  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
-WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
-THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
-GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
-USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
-DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
-PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
-EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
-SUCH DAMAGES.
-
-  17. Interpretation of Sections 15 and 16.
-
-  If the disclaimer of warranty and limitation of liability provided
-above cannot be given local legal effect according to their terms,
-reviewing courts shall apply local law that most closely approximates
-an absolute waiver of all civil liability in connection with the
-Program, unless a warranty or assumption of liability accompanies a
-copy of the Program in return for a fee.
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
 
    END OF TERMS AND CONDITIONS
 
-            How to Apply These Terms to Your New Programs
+   APPENDIX: How to apply the Apache License to your work.
 
-  If you develop a new program, and you want it to be of the greatest
-possible use to the public, the best way to achieve this is to make it
-free software which everyone can redistribute and change under these terms.
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
 
-  To do so, attach the following notices to the program.  It is safest
-to attach them to the start of each source file to most effectively
-state the exclusion of warranty; and each file should have at least
-the "copyright" line and a pointer to where the full notice is found.
+   Copyright [yyyy] [name of copyright owner]
 
-    <one line to give the program's name and a brief idea of what it does.>
-    Copyright (C) <year>  <name of author>
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
 
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
+       http://www.apache.org/licenses/LICENSE-2.0
 
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-Also add information on how to contact you by electronic and paper mail.
-
-  If the program does terminal interaction, make it output a short
-notice like this when it starts in an interactive mode:
-
-    <program>  Copyright (C) <year>  <name of author>
-    This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
-    This is free software, and you are welcome to redistribute it
-    under certain conditions; type `show c' for details.
-
-The hypothetical commands `show w' and `show c' should show the appropriate
-parts of the General Public License.  Of course, your program's commands
-might be different; for a GUI interface, you would use an "about box".
-
-  You should also get your employer (if you work as a programmer) or school,
-if any, to sign a "copyright disclaimer" for the program, if necessary.
-For more information on this, and how to apply and follow the GNU GPL, see
-<https://www.gnu.org/licenses/>.
-
-  The GNU General Public License does not permit incorporating your program
-into proprietary programs.  If your program is a subroutine library, you
-may consider it more useful to permit linking proprietary applications with
-the library.  If this is what you want to do, use the GNU Lesser General
-Public License instead of this License.  But first, please read
-<https://www.gnu.org/licenses/why-not-lgpl.html>.
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

Makefile

@@ -1,17 +1,32 @@
+.PHONY: default
 default:
 
+.PHONY: testdata
+testdata:
+	make -C testdata
+	make -C cmd/litestream testdata
+
+docker:
+	docker build -t litestream .
+
 dist-linux:
 	mkdir -p dist
 	cp etc/litestream.yml dist/litestream.yml
-	docker run --rm -v "${PWD}":/usr/src/litestream -w /usr/src/litestream -e GOOS=linux -e GOARCH=amd64 golang:1.15 go build -v -o dist/litestream ./cmd/litestream
+	docker run --rm -v "${PWD}":/usr/src/litestream -w /usr/src/litestream -e GOOS=linux -e GOARCH=amd64 golang:1.16 go build -v -ldflags "-s -w" -o dist/litestream ./cmd/litestream
 	tar -cz -f dist/litestream-linux-amd64.tar.gz -C dist litestream
 
+dist-linux-arm:
+	docker run --rm -v "${PWD}":/usr/src/litestream -w /usr/src/litestream -e CGO_ENABLED=1 -e CC=arm-linux-gnueabihf-gcc -e GOOS=linux -e GOARCH=arm golang-xc:1.16 go build -v -o dist/litestream-linux-arm ./cmd/litestream
+
+dist-linux-arm64:
+	docker run --rm -v "${PWD}":/usr/src/litestream -w /usr/src/litestream -e CGO_ENABLED=1 -e CC=aarch64-linux-gnu-gcc -e GOOS=linux -e GOARCH=arm64 golang-xc:1.16 go build -v -o dist/litestream-linux-arm64 ./cmd/litestream
+
 dist-macos:
 ifndef LITESTREAM_VERSION
 	$(error LITESTREAM_VERSION is undefined)
 endif
 	mkdir -p dist
-	go build -v -ldflags "-X 'main.Version=${LITESTREAM_VERSION}'"  -o dist/litestream ./cmd/litestream
+	go build -v -ldflags "-s -w -X 'main.Version=${LITESTREAM_VERSION}'"  -o dist/litestream ./cmd/litestream
 	gon etc/gon.hcl
 	mv dist/litestream.zip dist/litestream-${LITESTREAM_VERSION}-darwin-amd64.zip
 	openssl dgst -sha256 dist/litestream-${LITESTREAM_VERSION}-darwin-amd64.zip

README.md

@@ -2,6 +2,7 @@ Litestream
 ![GitHub release (latest by date)](https://img.shields.io/github/v/release/benbjohnson/litestream)
 ![Status](https://img.shields.io/badge/status-beta-blue)
 ![GitHub](https://img.shields.io/github/license/benbjohnson/litestream)
+[![Docker Pulls](https://img.shields.io/docker/pulls/litestream/litestream.svg?maxAge=604800)](https://hub.docker.com/r/litestream/litestream/)
 ![test](https://github.com/benbjohnson/litestream/workflows/test/badge.svg)
 ==========
 
@@ -28,32 +29,33 @@ of the most valuable contributions are in the forms of testing, feedback, and
 documentation. These help harden software and streamline usage for other users.
 
 I want to give special thanks to individuals who invest much of their time and 
-energy into the project to help make it better. Shout out to [Michael
-Lynch](https://github.com/mtlynch) for digging into issues and contributing to
-the documentation.
+energy into the project to help make it better:
+
+- Thanks to [Cory LaNou](https://twitter.com/corylanou) for giving early feedback and testing when Litestream was still pre-release.
+- Thanks to [Michael Lynch](https://github.com/mtlynch) for digging into issues and contributing to the documentation.
+- Thanks to [Kurt Mackey](https://twitter.com/mrkurt) for feedback and testing.
+- Thanks to [Sam Weston](https://twitter.com/cablespaghetti) for figuring out how to run Litestream on Kubernetes and writing up the docs for it.
+- Thanks to [Rafael](https://github.com/netstx) & [Jungle Boogie](https://github.com/jungle-boogie) for helping to get OpenBSD release builds working.
+- Thanks to [Simon Gottschlag](https://github.com/simongottschlag), [Marin](https://github.com/supermarin),[Victor Björklund](https://github.com/victorbjorklund), [Jonathan Beri](https://twitter.com/beriberikix) [Yuri](https://github.com/yurivish), [Nathan Probst](https://github.com/nprbst), [Yann Coleu](https://github.com/yanc0), and [Nicholas Grilly](https://twitter.com/ngrilly) for frequent feedback, testing, & support.
+
+Huge thanks to fly.io for their support and for contributing credits for testing and development!
 
 
-## Open-source, not open-contribution
+## Contribution Policy
 
-[Similar to SQLite](https://www.sqlite.org/copyright.html), Litestream is open
-source but closed to code contributions. This keeps the code base free of
-proprietary or licensed code but it also helps me continue to maintain and build
-Litestream.
+Initially, Litestream was closed to outside contributions. The goal was to
+reduce burnout by limiting the maintenance overhead of reviewing and validating
+third-party code. However, this policy is overly broad and has prevented small,
+easily testable patches from being contributed.
 
-As the author of [BoltDB](https://github.com/boltdb/bolt), I found that
-accepting and maintaining third party patches contributed to my burn out and
-I eventually archived the project. Writing databases & low-level replication
-tools involves nuance and simple one line changes can have profound and
-unexpected changes in correctness and performance. Small contributions
-typically required hours of my time to properly test and validate them.
+Litestream is now open to code contributions for bug fixes only. Features carry
+a long-term maintenance burden so they will not be accepted at this time.
+Please [submit an issue][new-issue] if you have a feature you'd like to
+request.
 
-I am grateful for community involvement, bug reports, & feature requests. I do
-not wish to come off as anything but welcoming, however, I've
-made the decision to keep this project closed to contributions for my own
-mental health and long term viability of the project.
+If you find mistakes in the documentation, please submit a fix to the
+[documentation repository][docs].
 
-The [documentation repository][docs] is MIT licensed and pull requests are welcome there.
-
-[releases]: https://github.com/benbjohnson/litestream/releases
+[new-issue]: https://github.com/benbjohnson/litestream/issues/new
 [docs]: https://github.com/benbjohnson/litestream.io
 

abs/replica_client.go

@@ -0,0 +1,558 @@
+package abs
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/url"
+	"os"
+	"path"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/Azure/azure-storage-blob-go/azblob"
+	"github.com/benbjohnson/litestream"
+	"github.com/benbjohnson/litestream/internal"
+	"golang.org/x/sync/errgroup"
+)
+
+// ReplicaClientType is the client type for this package.
+const ReplicaClientType = "abs"
+
+var _ litestream.ReplicaClient = (*ReplicaClient)(nil)
+
+// ReplicaClient is a client for writing snapshots & WAL segments to disk.
+type ReplicaClient struct {
+	mu           sync.Mutex
+	containerURL *azblob.ContainerURL
+
+	// Azure credentials
+	AccountName string
+	AccountKey  string
+	Endpoint    string
+
+	// Azure Blob Storage container information
+	Bucket string
+	Path   string
+}
+
+// NewReplicaClient returns a new instance of ReplicaClient.
+func NewReplicaClient() *ReplicaClient {
+	return &ReplicaClient{}
+}
+
+// Type returns "abs" as the client type.
+func (c *ReplicaClient) Type() string {
+	return ReplicaClientType
+}
+
+// Init initializes the connection to Azure. No-op if already initialized.
+func (c *ReplicaClient) Init(ctx context.Context) (err error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if c.containerURL != nil {
+		return nil
+	}
+
+	// Read account key from environment, if available.
+	accountKey := c.AccountKey
+	if accountKey == "" {
+		accountKey = os.Getenv("LITESTREAM_AZURE_ACCOUNT_KEY")
+	}
+
+	// Authenticate to ACS.
+	credential, err := azblob.NewSharedKeyCredential(c.AccountName, accountKey)
+	if err != nil {
+		return err
+	}
+
+	// Construct & parse endpoint unless already set.
+	endpoint := c.Endpoint
+	if endpoint == "" {
+		endpoint = fmt.Sprintf("https://%s.blob.core.windows.net", c.AccountName)
+	}
+	endpointURL, err := url.Parse(endpoint)
+	if err != nil {
+		return fmt.Errorf("cannot parse azure endpoint: %w", err)
+	}
+
+	// Build pipeline and reference to container.
+	pipeline := azblob.NewPipeline(credential, azblob.PipelineOptions{
+		Retry: azblob.RetryOptions{
+			TryTimeout: 24 * time.Hour,
+		},
+	})
+	containerURL := azblob.NewServiceURL(*endpointURL, pipeline).NewContainerURL(c.Bucket)
+	c.containerURL = &containerURL
+
+	return nil
+}
+
+// Generations returns a list of available generation names.
+func (c *ReplicaClient) Generations(ctx context.Context) ([]string, error) {
+	if err := c.Init(ctx); err != nil {
+		return nil, err
+	}
+
+	var generations []string
+	var marker azblob.Marker
+	for marker.NotDone() {
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "LIST").Inc()
+
+		resp, err := c.containerURL.ListBlobsHierarchySegment(ctx, marker, "/", azblob.ListBlobsSegmentOptions{
+			Prefix: path.Join(c.Path, "generations") + "/",
+		})
+		if err != nil {
+			return nil, err
+		}
+		marker = resp.NextMarker
+
+		for _, prefix := range resp.Segment.BlobPrefixes {
+			name := path.Base(strings.TrimSuffix(prefix.Name, "/"))
+			if !litestream.IsGenerationName(name) {
+				continue
+			}
+			generations = append(generations, name)
+		}
+	}
+
+	return generations, nil
+}
+
+// DeleteGeneration deletes all snapshots & WAL segments within a generation.
+func (c *ReplicaClient) DeleteGeneration(ctx context.Context, generation string) error {
+	if err := c.Init(ctx); err != nil {
+		return err
+	} else if generation == "" {
+		return fmt.Errorf("generation required")
+	}
+
+	prefix := path.Join(c.Path, "generations", generation) + "/"
+
+	var marker azblob.Marker
+	for marker.NotDone() {
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "LIST").Inc()
+
+		resp, err := c.containerURL.ListBlobsFlatSegment(ctx, marker, azblob.ListBlobsSegmentOptions{Prefix: prefix})
+		if err != nil {
+			return err
+		}
+		marker = resp.NextMarker
+
+		for _, item := range resp.Segment.BlobItems {
+			internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "DELETE").Inc()
+
+			blobURL := c.containerURL.NewBlobURL(item.Name)
+			if _, err := blobURL.Delete(ctx, azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{}); isNotExists(err) {
+				continue
+			} else if err != nil {
+				return err
+			}
+		}
+	}
+
+	// log.Printf("%s(%s): retainer: deleting generation: %s", r.db.Path(), r.Name(), generation)
+
+	return nil
+}
+
+// Snapshots returns an iterator over all available snapshots for a generation.
+func (c *ReplicaClient) Snapshots(ctx context.Context, generation string) (litestream.SnapshotIterator, error) {
+	if err := c.Init(ctx); err != nil {
+		return nil, err
+	}
+	return newSnapshotIterator(ctx, generation, c), nil
+}
+
+// WriteSnapshot writes LZ4 compressed data from rd to the object storage.
+func (c *ReplicaClient) WriteSnapshot(ctx context.Context, generation string, index int, rd io.Reader) (info litestream.SnapshotInfo, err error) {
+	if err := c.Init(ctx); err != nil {
+		return info, err
+	} else if generation == "" {
+		return info, fmt.Errorf("generation required")
+	}
+
+	key := path.Join(c.Path, "generations", generation, "snapshots", litestream.FormatIndex(index)+".snapshot.lz4")
+	startTime := time.Now()
+
+	rc := internal.NewReadCounter(rd)
+
+	blobURL := c.containerURL.NewBlockBlobURL(key)
+	if _, err := azblob.UploadStreamToBlockBlob(ctx, rc, blobURL, azblob.UploadStreamToBlockBlobOptions{
+		BlobHTTPHeaders: azblob.BlobHTTPHeaders{ContentType: "application/octet-stream"},
+		BlobAccessTier:  azblob.DefaultAccessTier,
+	}); err != nil {
+		return info, err
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "PUT").Inc()
+	internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "PUT").Add(float64(rc.N()))
+
+	return litestream.SnapshotInfo{
+		Generation: generation,
+		Index:      index,
+		Size:       rc.N(),
+		CreatedAt:  startTime.UTC(),
+	}, nil
+}
+
+// SnapshotReader returns a reader for snapshot data at the given generation/index.
+func (c *ReplicaClient) SnapshotReader(ctx context.Context, generation string, index int) (io.ReadCloser, error) {
+	if err := c.Init(ctx); err != nil {
+		return nil, err
+	} else if generation == "" {
+		return nil, fmt.Errorf("generation required")
+	}
+
+	key := path.Join(c.Path, "generations", generation, "snapshots", litestream.FormatIndex(index)+".snapshot.lz4")
+
+	blobURL := c.containerURL.NewBlobURL(key)
+	resp, err := blobURL.Download(ctx, 0, 0, azblob.BlobAccessConditions{}, false, azblob.ClientProvidedKeyOptions{})
+	if isNotExists(err) {
+		return nil, os.ErrNotExist
+	} else if err != nil {
+		return nil, fmt.Errorf("cannot start new reader for %q: %w", key, err)
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "GET").Inc()
+	internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "GET").Add(float64(resp.ContentLength()))
+
+	return resp.Body(azblob.RetryReaderOptions{}), nil
+}
+
+// DeleteSnapshot deletes a snapshot with the given generation & index.
+func (c *ReplicaClient) DeleteSnapshot(ctx context.Context, generation string, index int) error {
+	if err := c.Init(ctx); err != nil {
+		return err
+	} else if generation == "" {
+		return fmt.Errorf("generation required")
+	}
+
+	key := path.Join(c.Path, "generations", generation, "snapshots", litestream.FormatIndex(index)+".snapshot.lz4")
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "DELETE").Inc()
+
+	blobURL := c.containerURL.NewBlobURL(key)
+	if _, err := blobURL.Delete(ctx, azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{}); isNotExists(err) {
+		return nil
+	} else if err != nil {
+		return fmt.Errorf("cannot delete snapshot %q: %w", key, err)
+	}
+	return nil
+}
+
+// WALSegments returns an iterator over all available WAL files for a generation.
+func (c *ReplicaClient) WALSegments(ctx context.Context, generation string) (litestream.WALSegmentIterator, error) {
+	if err := c.Init(ctx); err != nil {
+		return nil, err
+	}
+	return newWALSegmentIterator(ctx, generation, c), nil
+}
+
+// WriteWALSegment writes LZ4 compressed data from rd into a file on disk.
+func (c *ReplicaClient) WriteWALSegment(ctx context.Context, pos litestream.Pos, rd io.Reader) (info litestream.WALSegmentInfo, err error) {
+	if err := c.Init(ctx); err != nil {
+		return info, err
+	} else if pos.Generation == "" {
+		return info, fmt.Errorf("generation required")
+	}
+
+	key := path.Join(c.Path, "generations", pos.Generation, "wal", litestream.FormatIndex(pos.Index), litestream.FormatOffset(pos.Offset)+".wal.lz4")
+	startTime := time.Now()
+
+	rc := internal.NewReadCounter(rd)
+
+	blobURL := c.containerURL.NewBlockBlobURL(key)
+	if _, err := azblob.UploadStreamToBlockBlob(ctx, rc, blobURL, azblob.UploadStreamToBlockBlobOptions{
+		BlobHTTPHeaders: azblob.BlobHTTPHeaders{ContentType: "application/octet-stream"},
+		BlobAccessTier:  azblob.DefaultAccessTier,
+	}); err != nil {
+		return info, err
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "PUT").Inc()
+	internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "PUT").Add(float64(rc.N()))
+
+	return litestream.WALSegmentInfo{
+		Generation: pos.Generation,
+		Index:      pos.Index,
+		Offset:     pos.Offset,
+		Size:       rc.N(),
+		CreatedAt:  startTime.UTC(),
+	}, nil
+}
+
+// WALSegmentReader returns a reader for a section of WAL data at the given index.
+// Returns os.ErrNotExist if no matching index/offset is found.
+func (c *ReplicaClient) WALSegmentReader(ctx context.Context, pos litestream.Pos) (io.ReadCloser, error) {
+	if err := c.Init(ctx); err != nil {
+		return nil, err
+	} else if pos.Generation == "" {
+		return nil, fmt.Errorf("generation required")
+	}
+
+	key := path.Join(c.Path, "generations", pos.Generation, "wal", litestream.FormatIndex(pos.Index), litestream.FormatOffset(pos.Offset)+".wal.lz4")
+
+	blobURL := c.containerURL.NewBlobURL(key)
+	resp, err := blobURL.Download(ctx, 0, 0, azblob.BlobAccessConditions{}, false, azblob.ClientProvidedKeyOptions{})
+	if isNotExists(err) {
+		return nil, os.ErrNotExist
+	} else if err != nil {
+		return nil, fmt.Errorf("cannot start new reader for %q: %w", key, err)
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "GET").Inc()
+	internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "GET").Add(float64(resp.ContentLength()))
+
+	return resp.Body(azblob.RetryReaderOptions{}), nil
+}
+
+// DeleteWALSegments deletes WAL segments with at the given positions.
+func (c *ReplicaClient) DeleteWALSegments(ctx context.Context, a []litestream.Pos) error {
+	if err := c.Init(ctx); err != nil {
+		return err
+	}
+
+	for _, pos := range a {
+		if pos.Generation == "" {
+			return fmt.Errorf("generation required")
+		}
+
+		key := path.Join(c.Path, "generations", pos.Generation, "wal", litestream.FormatIndex(pos.Index), litestream.FormatOffset(pos.Offset)+".wal.lz4")
+
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "DELETE").Inc()
+
+		blobURL := c.containerURL.NewBlobURL(key)
+		if _, err := blobURL.Delete(ctx, azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{}); isNotExists(err) {
+			continue
+		} else if err != nil {
+			return fmt.Errorf("cannot delete wal segment %q: %w", key, err)
+		}
+	}
+
+	return nil
+}
+
+type snapshotIterator struct {
+	client     *ReplicaClient
+	generation string
+
+	ch     chan litestream.SnapshotInfo
+	g      errgroup.Group
+	ctx    context.Context
+	cancel func()
+
+	info litestream.SnapshotInfo
+	err  error
+}
+
+func newSnapshotIterator(ctx context.Context, generation string, client *ReplicaClient) *snapshotIterator {
+	itr := &snapshotIterator{
+		client:     client,
+		generation: generation,
+		ch:         make(chan litestream.SnapshotInfo),
+	}
+
+	itr.ctx, itr.cancel = context.WithCancel(ctx)
+	itr.g.Go(itr.fetch)
+
+	return itr
+}
+
+// fetch runs in a separate goroutine to fetch pages of objects and stream them to a channel.
+func (itr *snapshotIterator) fetch() error {
+	defer close(itr.ch)
+
+	if itr.generation == "" {
+		return fmt.Errorf("generation required")
+	}
+
+	prefix := path.Join(itr.client.Path, "generations", itr.generation) + "/"
+
+	var marker azblob.Marker
+	for marker.NotDone() {
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "LIST").Inc()
+
+		resp, err := itr.client.containerURL.ListBlobsFlatSegment(itr.ctx, marker, azblob.ListBlobsSegmentOptions{Prefix: prefix})
+		if err != nil {
+			return err
+		}
+		marker = resp.NextMarker
+
+		for _, item := range resp.Segment.BlobItems {
+			index, err := internal.ParseSnapshotPath(path.Base(item.Name))
+			if err != nil {
+				continue
+			}
+
+			info := litestream.SnapshotInfo{
+				Generation: itr.generation,
+				Index:      index,
+				Size:       *item.Properties.ContentLength,
+				CreatedAt:  item.Properties.CreationTime.UTC(),
+			}
+
+			select {
+			case <-itr.ctx.Done():
+			case itr.ch <- info:
+			}
+		}
+	}
+	return nil
+}
+
+func (itr *snapshotIterator) Close() (err error) {
+	err = itr.err
+
+	// Cancel context and wait for error group to finish.
+	itr.cancel()
+	if e := itr.g.Wait(); e != nil && err == nil {
+		err = e
+	}
+
+	return err
+}
+
+func (itr *snapshotIterator) Next() bool {
+	// Exit if an error has already occurred.
+	if itr.err != nil {
+		return false
+	}
+
+	// Return false if context was canceled or if there are no more snapshots.
+	// Otherwise fetch the next snapshot and store it on the iterator.
+	select {
+	case <-itr.ctx.Done():
+		return false
+	case info, ok := <-itr.ch:
+		if !ok {
+			return false
+		}
+		itr.info = info
+		return true
+	}
+}
+
+func (itr *snapshotIterator) Err() error { return itr.err }
+
+func (itr *snapshotIterator) Snapshot() litestream.SnapshotInfo {
+	return itr.info
+}
+
+type walSegmentIterator struct {
+	client     *ReplicaClient
+	generation string
+
+	ch     chan litestream.WALSegmentInfo
+	g      errgroup.Group
+	ctx    context.Context
+	cancel func()
+
+	info litestream.WALSegmentInfo
+	err  error
+}
+
+func newWALSegmentIterator(ctx context.Context, generation string, client *ReplicaClient) *walSegmentIterator {
+	itr := &walSegmentIterator{
+		client:     client,
+		generation: generation,
+		ch:         make(chan litestream.WALSegmentInfo),
+	}
+
+	itr.ctx, itr.cancel = context.WithCancel(ctx)
+	itr.g.Go(itr.fetch)
+
+	return itr
+}
+
+// fetch runs in a separate goroutine to fetch pages of objects and stream them to a channel.
+func (itr *walSegmentIterator) fetch() error {
+	defer close(itr.ch)
+
+	if itr.generation == "" {
+		return fmt.Errorf("generation required")
+	}
+	prefix := path.Join(itr.client.Path, "generations", itr.generation, "wal")
+
+	var marker azblob.Marker
+	for marker.NotDone() {
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "LIST").Inc()
+
+		resp, err := itr.client.containerURL.ListBlobsFlatSegment(itr.ctx, marker, azblob.ListBlobsSegmentOptions{Prefix: prefix})
+		if err != nil {
+			return err
+		}
+		marker = resp.NextMarker
+
+		for _, item := range resp.Segment.BlobItems {
+			key := strings.TrimPrefix(item.Name, prefix+"/")
+			index, offset, err := internal.ParseWALSegmentPath(key)
+			if err != nil {
+				continue
+			}
+
+			info := litestream.WALSegmentInfo{
+				Generation: itr.generation,
+				Index:      index,
+				Offset:     offset,
+				Size:       *item.Properties.ContentLength,
+				CreatedAt:  item.Properties.CreationTime.UTC(),
+			}
+
+			select {
+			case <-itr.ctx.Done():
+			case itr.ch <- info:
+			}
+		}
+	}
+	return nil
+}
+
+func (itr *walSegmentIterator) Close() (err error) {
+	err = itr.err
+
+	// Cancel context and wait for error group to finish.
+	itr.cancel()
+	if e := itr.g.Wait(); e != nil && err == nil {
+		err = e
+	}
+
+	return err
+}
+
+func (itr *walSegmentIterator) Next() bool {
+	// Exit if an error has already occurred.
+	if itr.err != nil {
+		return false
+	}
+
+	// Return false if context was canceled or if there are no more segments.
+	// Otherwise fetch the next segment and store it on the iterator.
+	select {
+	case <-itr.ctx.Done():
+		return false
+	case info, ok := <-itr.ch:
+		if !ok {
+			return false
+		}
+		itr.info = info
+		return true
+	}
+}
+
+func (itr *walSegmentIterator) Err() error { return itr.err }
+
+func (itr *walSegmentIterator) WALSegment() litestream.WALSegmentInfo {
+	return itr.info
+}
+
+func isNotExists(err error) bool {
+	switch err := err.(type) {
+	case azblob.StorageError:
+		return err.ServiceCode() == azblob.ServiceCodeBlobNotFound
+	default:
+		return false
+	}
+}

cmd/litestream/Makefile

@@ -0,0 +1,6 @@
+.PHONY: default
+default:
+
+.PHONY: testdata
+testdata:
+	make -C testdata

cmd/litestream/databases.go

@@ -4,18 +4,34 @@ import (
 	"context"
 	"flag"
 	"fmt"
-	"os"
+	"io"
 	"strings"
 	"text/tabwriter"
 )
 
 // DatabasesCommand is a command for listing managed databases.
-type DatabasesCommand struct{}
+type DatabasesCommand struct {
+	stdin  io.Reader
+	stdout io.Writer
+	stderr io.Writer
+
+	configPath  string
+	noExpandEnv bool
+}
+
+// NewDatabasesCommand returns a new instance of DatabasesCommand.
+func NewDatabasesCommand(stdin io.Reader, stdout, stderr io.Writer) *DatabasesCommand {
+	return &DatabasesCommand{
+		stdin:  stdin,
+		stdout: stdout,
+		stderr: stderr,
+	}
+}
 
 // Run executes the command.
 func (c *DatabasesCommand) Run(ctx context.Context, args []string) (err error) {
 	fs := flag.NewFlagSet("litestream-databases", flag.ContinueOnError)
-	configPath := registerConfigFlag(fs)
+	registerConfigFlag(fs, &c.configPath, &c.noExpandEnv)
 	fs.Usage = c.Usage
 	if err := fs.Parse(args); err != nil {
 		return err
@@ -24,16 +40,16 @@ func (c *DatabasesCommand) Run(ctx context.Context, args []string) (err error) {
 	}
 
 	// Load configuration.
-	if *configPath == "" {
-		*configPath = DefaultConfigPath()
-	}
-	config, err := ReadConfigFile(*configPath)
+	config, err := ReadConfigFile(c.configPath, !c.noExpandEnv)
 	if err != nil {
 		return err
+	} else if len(config.DBs) == 0 {
+		fmt.Fprintln(c.stdout, "No databases found in config file.")
+		return nil
 	}
 
 	// List all databases.
-	w := tabwriter.NewWriter(os.Stdout, 0, 8, 2, ' ', 0)
+	w := tabwriter.NewWriter(c.stdout, 0, 8, 2, ' ', 0)
 	defer w.Flush()
 
 	fmt.Fprintln(w, "path\treplicas")
@@ -59,7 +75,7 @@ func (c *DatabasesCommand) Run(ctx context.Context, args []string) (err error) {
 
 // Usage prints the help screen to STDOUT.
 func (c *DatabasesCommand) Usage() {
-	fmt.Printf(`
+	fmt.Fprintf(c.stdout, `
 The databases command lists all databases in the configuration file.
 
 Usage:
@@ -72,6 +88,9 @@ Arguments:
 	    Specifies the configuration file.
 	    Defaults to %s
 
+	-no-expand-env
+	    Disables environment variable expansion in configuration file.
+
 `[1:],
 		DefaultConfigPath(),
 	)

cmd/litestream/databases_test.go

@@ -0,0 +1,66 @@
+package main_test
+
+import (
+	"context"
+	"flag"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/benbjohnson/litestream/internal/testingutil"
+)
+
+func TestDatabasesCommand(t *testing.T) {
+	t.Run("OK", func(t *testing.T) {
+		testDir := filepath.Join("testdata", "databases", "ok")
+		m, _, stdout, _ := newMain()
+		if err := m.Run(context.Background(), []string{"databases", "-config", filepath.Join(testDir, "litestream.yml")}); err != nil {
+			t.Fatal(err)
+		} else if got, want := stdout.String(), string(testingutil.ReadFile(t, filepath.Join(testDir, "stdout"))); got != want {
+			t.Fatalf("stdout=%q, want %q", got, want)
+		}
+	})
+
+	t.Run("NoDatabases", func(t *testing.T) {
+		testDir := filepath.Join("testdata", "databases", "no-databases")
+		m, _, stdout, _ := newMain()
+		if err := m.Run(context.Background(), []string{"databases", "-config", filepath.Join(testDir, "litestream.yml")}); err != nil {
+			t.Fatal(err)
+		} else if got, want := stdout.String(), string(testingutil.ReadFile(t, filepath.Join(testDir, "stdout"))); got != want {
+			t.Fatalf("stdout=%q, want %q", got, want)
+		}
+	})
+
+	t.Run("ErrConfigNotFound", func(t *testing.T) {
+		testDir := filepath.Join("testdata", "databases", "no-config")
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"databases", "-config", filepath.Join(testDir, "litestream.yml")})
+		if err == nil || !strings.Contains(err.Error(), `config file not found:`) {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrInvalidConfig", func(t *testing.T) {
+		testDir := filepath.Join("testdata", "databases", "invalid-config")
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"databases", "-config", filepath.Join(testDir, "litestream.yml")})
+		if err == nil || !strings.Contains(err.Error(), `replica path cannot be a url`) {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrTooManyArguments", func(t *testing.T) {
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"databases", "xyz"})
+		if err == nil || err.Error() != `too many arguments` {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("Usage", func(t *testing.T) {
+		m, _, _, _ := newMain()
+		if err := m.Run(context.Background(), []string{"databases", "-h"}); err != flag.ErrHelp {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+}

cmd/litestream/generations.go

@@ -4,117 +4,116 @@ import (
 	"context"
 	"flag"
 	"fmt"
-	"log"
+	"io"
 	"os"
 	"text/tabwriter"
 	"time"
 
 	"github.com/benbjohnson/litestream"
+	"github.com/benbjohnson/litestream/internal"
 )
 
 // GenerationsCommand represents a command to list all generations for a database.
-type GenerationsCommand struct{}
+type GenerationsCommand struct {
+	stdin  io.Reader
+	stdout io.Writer
+	stderr io.Writer
+
+	configPath  string
+	noExpandEnv bool
+
+	replicaName string
+}
+
+// NewGenerationsCommand returns a new instance of GenerationsCommand.
+func NewGenerationsCommand(stdin io.Reader, stdout, stderr io.Writer) *GenerationsCommand {
+	return &GenerationsCommand{
+		stdin:  stdin,
+		stdout: stdout,
+		stderr: stderr,
+	}
+}
 
 // Run executes the command.
-func (c *GenerationsCommand) Run(ctx context.Context, args []string) (err error) {
+func (c *GenerationsCommand) Run(ctx context.Context, args []string) (ret error) {
 	fs := flag.NewFlagSet("litestream-generations", flag.ContinueOnError)
-	configPath := registerConfigFlag(fs)
-	replicaName := fs.String("replica", "", "replica name")
+	registerConfigFlag(fs, &c.configPath, &c.noExpandEnv)
+	fs.StringVar(&c.replicaName, "replica", "", "replica name")
 	fs.Usage = c.Usage
 	if err := fs.Parse(args); err != nil {
 		return err
-	} else if fs.NArg() == 0 || fs.Arg(0) == "" {
+	} else if fs.Arg(0) == "" {
 		return fmt.Errorf("database path or replica URL required")
 	} else if fs.NArg() > 1 {
 		return fmt.Errorf("too many arguments")
 	}
 
-	var db *litestream.DB
-	var r litestream.Replica
-	updatedAt := time.Now()
-	if isURL(fs.Arg(0)) {
-		if *configPath != "" {
-			return fmt.Errorf("cannot specify a replica URL and the -config flag")
-		}
-		if r, err = NewReplicaFromConfig(&ReplicaConfig{URL: fs.Arg(0)}, nil); err != nil {
-			return err
-		}
-	} else {
-		if *configPath == "" {
-			*configPath = DefaultConfigPath()
-		}
-
 	// Load configuration.
-		config, err := ReadConfigFile(*configPath)
+	config, err := ReadConfigFile(c.configPath, !c.noExpandEnv)
 	if err != nil {
 		return err
 	}
 
-		// Lookup database from configuration file by path.
-		if path, err := expand(fs.Arg(0)); err != nil {
+	replicas, db, err := loadReplicas(ctx, config, fs.Arg(0), c.replicaName)
+	if err != nil {
 		return err
-		} else if dbc := config.DBConfig(path); dbc == nil {
-			return fmt.Errorf("database not found in config: %s", path)
-		} else if db, err = NewDBFromConfig(dbc); err != nil {
-			return err
-		}
-
-		// Filter by replica, if specified.
-		if *replicaName != "" {
-			if r = db.Replica(*replicaName); r == nil {
-				return fmt.Errorf("replica %q not found for database %q", *replicaName, db.Path())
-			}
 	}
 
 	// Determine last time database or WAL was updated.
-		if updatedAt, err = db.UpdatedAt(); err != nil {
+	var dbUpdatedAt time.Time
+	if db != nil {
+		if dbUpdatedAt, err = db.UpdatedAt(); err != nil && !os.IsNotExist(err) {
 			return err
 		}
 	}
 
-	var replicas []litestream.Replica
-	if r != nil {
-		replicas = []litestream.Replica{r}
-	} else {
-		replicas = db.Replicas
-	}
-
 	// List each generation.
-	w := tabwriter.NewWriter(os.Stdout, 0, 8, 2, ' ', 0)
+	w := tabwriter.NewWriter(c.stdout, 0, 8, 2, ' ', 0)
 	defer w.Flush()
 
 	fmt.Fprintln(w, "name\tgeneration\tlag\tstart\tend")
+
 	for _, r := range replicas {
-		generations, err := r.Generations(ctx)
+		generations, err := r.Client().Generations(ctx)
 		if err != nil {
-			log.Printf("%s: cannot list generations: %s", r.Name(), err)
+			fmt.Fprintf(c.stderr, "%s: cannot list generations: %s", r.Name(), err)
+			ret = errExit // signal error return without printing message
 			continue
 		}
 
 		// Iterate over each generation for the replica.
 		for _, generation := range generations {
-			stats, err := r.GenerationStats(ctx, generation)
+			createdAt, updatedAt, err := litestream.GenerationTimeBounds(ctx, r.Client(), generation)
 			if err != nil {
-				log.Printf("%s: cannot find generation stats: %s", r.Name(), err)
+				fmt.Fprintf(c.stderr, "%s: cannot determine generation time bounds: %s", r.Name(), err)
+				ret = errExit // signal error return without printing message
 				continue
 			}
 
+			// Calculate lag from database mod time to the replica mod time.
+			// This is ignored if the database mod time is unavailable such as
+			// when specifying the replica URL or if the database file is missing.
+			lag := "-"
+			if !dbUpdatedAt.IsZero() {
+				lag = internal.TruncateDuration(dbUpdatedAt.Sub(updatedAt)).String()
+			}
+
 			fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n",
 				r.Name(),
 				generation,
-				truncateDuration(updatedAt.Sub(stats.UpdatedAt)).String(),
-				stats.CreatedAt.Format(time.RFC3339),
-				stats.UpdatedAt.Format(time.RFC3339),
+				lag,
+				createdAt.Format(time.RFC3339),
+				updatedAt.Format(time.RFC3339),
 			)
 		}
 	}
 
-	return nil
+	return ret
 }
 
 // Usage prints the help message to STDOUT.
 func (c *GenerationsCommand) Usage() {
-	fmt.Printf(`
+	fmt.Fprintf(c.stdout, `
 The generations command lists all generations for a database or replica. It also
 lists stats about their lag behind the primary database and the time range they
 cover.
@@ -131,6 +130,9 @@ Arguments:
 	    Specifies the configuration file.
 	    Defaults to %s
 
+	-no-expand-env
+	    Disables environment variable expansion in configuration file.
+
 	-replica NAME
 	    Optional, filters by replica.
 
@@ -138,29 +140,3 @@ Arguments:
 		DefaultConfigPath(),
 	)
 }
-
-func truncateDuration(d time.Duration) time.Duration {
-	if d < 0 {
-		if d < -10*time.Second {
-			return d.Truncate(time.Second)
-		} else if d < -time.Second {
-			return d.Truncate(time.Second / 10)
-		} else if d < -time.Millisecond {
-			return d.Truncate(time.Millisecond)
-		} else if d < -time.Microsecond {
-			return d.Truncate(time.Microsecond)
-		}
-		return d
-	}
-
-	if d > 10*time.Second {
-		return d.Truncate(time.Second)
-	} else if d > time.Second {
-		return d.Truncate(time.Second / 10)
-	} else if d > time.Millisecond {
-		return d.Truncate(time.Millisecond)
-	} else if d > time.Microsecond {
-		return d.Truncate(time.Microsecond)
-	}
-	return d
-}

cmd/litestream/generations_test.go

@@ -0,0 +1,140 @@
+package main_test
+
+import (
+	"context"
+	"flag"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/benbjohnson/litestream/internal/testingutil"
+)
+
+func TestGenerationsCommand(t *testing.T) {
+	t.Run("OK", func(t *testing.T) {
+		testDir := filepath.Join("testdata", "generations", "ok")
+		defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
+
+		m, _, stdout, _ := newMain()
+		if err := m.Run(context.Background(), []string{"generations", "-config", filepath.Join(testDir, "litestream.yml"), filepath.Join(testDir, "db")}); err != nil {
+			t.Fatal(err)
+		} else if got, want := stdout.String(), string(testingutil.ReadFile(t, filepath.Join(testDir, "stdout"))); got != want {
+			t.Fatalf("stdout=%q, want %q", got, want)
+		}
+	})
+
+	t.Run("ReplicaName", func(t *testing.T) {
+		testDir := filepath.Join("testdata", "generations", "replica-name")
+		defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
+
+		m, _, stdout, _ := newMain()
+		if err := m.Run(context.Background(), []string{"generations", "-config", filepath.Join(testDir, "litestream.yml"), "-replica", "replica1", filepath.Join(testDir, "db")}); err != nil {
+			t.Fatal(err)
+		} else if got, want := stdout.String(), string(testingutil.ReadFile(t, filepath.Join(testDir, "stdout"))); got != want {
+			t.Fatalf("stdout=%q, want %q", got, want)
+		}
+	})
+
+	t.Run("ReplicaURL", func(t *testing.T) {
+		testDir := filepath.Join(testingutil.Getwd(t), "testdata", "generations", "replica-url")
+		defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
+		replicaURL := "file://" + filepath.ToSlash(testDir) + "/replica"
+
+		m, _, stdout, _ := newMain()
+		if err := m.Run(context.Background(), []string{"generations", replicaURL}); err != nil {
+			t.Fatal(err)
+		} else if got, want := stdout.String(), string(testingutil.ReadFile(t, filepath.Join(testDir, "stdout"))); got != want {
+			t.Fatalf("stdout=%q, want %q", got, want)
+		}
+	})
+
+	t.Run("NoDatabase", func(t *testing.T) {
+		testDir := filepath.Join("testdata", "generations", "no-database")
+		defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
+
+		m, _, stdout, _ := newMain()
+		if err := m.Run(context.Background(), []string{"generations", "-config", filepath.Join(testDir, "litestream.yml"), filepath.Join(testDir, "db")}); err != nil {
+			t.Fatal(err)
+		} else if got, want := stdout.String(), string(testingutil.ReadFile(t, filepath.Join(testDir, "stdout"))); got != want {
+			t.Fatalf("stdout=%q, want %q", got, want)
+		}
+	})
+
+	t.Run("ErrDatabaseOrReplicaRequired", func(t *testing.T) {
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"generations"})
+		if err == nil || err.Error() != `database path or replica URL required` {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrTooManyArguments", func(t *testing.T) {
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"generations", "abc", "123"})
+		if err == nil || err.Error() != `too many arguments` {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrInvalidFlags", func(t *testing.T) {
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"generations", "-no-such-flag"})
+		if err == nil || err.Error() != `flag provided but not defined: -no-such-flag` {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrConfigFileNotFound", func(t *testing.T) {
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"generations", "-config", "/no/such/file", "/var/lib/db"})
+		if err == nil || err.Error() != `config file not found: /no/such/file` {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrInvalidConfig", func(t *testing.T) {
+		testDir := filepath.Join("testdata", "generations", "invalid-config")
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"generations", "-config", filepath.Join(testDir, "litestream.yml"), "/var/lib/db"})
+		if err == nil || !strings.Contains(err.Error(), `replica path cannot be a url`) {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrDatabaseNotFound", func(t *testing.T) {
+		testDir := filepath.Join("testdata", "generations", "database-not-found")
+		defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
+
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"generations", "-config", filepath.Join(testDir, "litestream.yml"), "/no/such/db"})
+		if err == nil || err.Error() != `database not found in config: /no/such/db` {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrReplicaNotFound", func(t *testing.T) {
+		testDir := filepath.Join(testingutil.Getwd(t), "testdata", "generations", "replica-not-found")
+		defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
+
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"generations", "-config", filepath.Join(testDir, "litestream.yml"), "-replica", "no_such_replica", filepath.Join(testDir, "db")})
+		if err == nil || err.Error() != `replica "no_such_replica" not found for database "`+filepath.Join(testDir, "db")+`"` {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrInvalidReplicaURL", func(t *testing.T) {
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"generations", "xyz://xyz"})
+		if err == nil || !strings.Contains(err.Error(), `unknown replica type in config: "xyz"`) {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("Usage", func(t *testing.T) {
+		m, _, _, _ := newMain()
+		if err := m.Run(context.Background(), []string{"generations", "-h"}); err != flag.ErrHelp {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+}

cmd/litestream/main.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"flag"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"log"
 	"net/url"
@@ -14,11 +15,17 @@ import (
 	"path"
 	"path/filepath"
 	"regexp"
+	"strconv"
 	"strings"
+	"syscall"
 	"time"
 
 	"github.com/benbjohnson/litestream"
+	"github.com/benbjohnson/litestream/abs"
+	"github.com/benbjohnson/litestream/gs"
+	"github.com/benbjohnson/litestream/http"
 	"github.com/benbjohnson/litestream/s3"
+	"github.com/benbjohnson/litestream/sftp"
 	_ "github.com/mattn/go-sqlite3"
 	"gopkg.in/yaml.v2"
 )
@@ -28,14 +35,15 @@ var (
 	Version = "(development build)"
 )
 
-// errStop is a terminal error for indicating program should quit.
-var errStop = errors.New("stop")
+// errExit is a terminal error for indicating program should quit.
+var errExit = errors.New("exit")
 
 func main() {
 	log.SetFlags(0)
+	log.SetOutput(os.Stdout)
 
-	m := NewMain()
-	if err := m.Run(context.Background(), os.Args[1:]); err == flag.ErrHelp || err == errStop {
+	m := NewMain(os.Stdin, os.Stdout, os.Stderr)
+	if err := m.Run(context.Background(), os.Args[1:]); err == flag.ErrHelp || err == errExit {
 		os.Exit(1)
 	} else if err != nil {
 		log.Println(err)
@@ -44,21 +52,25 @@ func main() {
 }
 
 // Main represents the main program execution.
-type Main struct{}
+type Main struct {
+	stdin  io.Reader
+	stdout io.Writer
+	stderr io.Writer
+}
 
 // NewMain returns a new instance of Main.
-func NewMain() *Main {
-	return &Main{}
+func NewMain(stdin io.Reader, stdout, stderr io.Writer) *Main {
+	return &Main{
+		stdin:  stdin,
+		stdout: stdout,
+		stderr: stderr,
+	}
 }
 
 // Run executes the program.
 func (m *Main) Run(ctx context.Context, args []string) (err error) {
-	// Execute replication command if running as a Windows service.
-	if isService, err := isWindowsService(); err != nil {
-		return err
-	} else if isService {
-		return runWindowsService(ctx)
-	}
+	// Copy "LITESTEAM" environment credentials.
+	applyLitestreamEnv()
 
 	// Extract command name.
 	var cmd string
@@ -68,40 +80,64 @@ func (m *Main) Run(ctx context.Context, args []string) (err error) {
 
 	switch cmd {
 	case "databases":
-		return (&DatabasesCommand{}).Run(ctx, args)
+		return NewDatabasesCommand(m.stdin, m.stdout, m.stderr).Run(ctx, args)
 	case "generations":
-		return (&GenerationsCommand{}).Run(ctx, args)
+		return NewGenerationsCommand(m.stdin, m.stdout, m.stderr).Run(ctx, args)
 	case "replicate":
-		c := NewReplicateCommand()
+		c := NewReplicateCommand(m.stdin, m.stdout, m.stderr)
 		if err := c.ParseFlags(ctx, args); err != nil {
 			return err
 		}
 
 		// Setup signal handler.
 		ctx, cancel := context.WithCancel(ctx)
-		ch := make(chan os.Signal, 1)
-		signal.Notify(ch, os.Interrupt)
-		go func() { <-ch; cancel() }()
+		defer cancel()
+		signalCh := make(chan os.Signal, 1)
+		signal.Notify(signalCh, syscall.SIGINT, syscall.SIGTERM)
 
 		if err := c.Run(ctx); err != nil {
 			return err
 		}
 
 		// Wait for signal to stop program.
-		<-ctx.Done()
-		signal.Reset()
+		select {
+		case <-ctx.Done():
+			fmt.Fprintln(m.stdout, "context done, litestream shutting down")
+		case err = <-c.execCh:
+			cancel()
+			fmt.Fprintln(m.stdout, "subprocess exited, litestream shutting down")
+		case sig := <-signalCh:
+			cancel()
+			fmt.Fprintln(m.stdout, "signal received, litestream shutting down")
+
+			if c.cmd != nil {
+				fmt.Fprintln(m.stdout, "sending signal to exec process")
+				if err := c.cmd.Process.Signal(sig); err != nil {
+					return fmt.Errorf("cannot signal exec process: %w", err)
+				}
+
+				fmt.Fprintln(m.stdout, "waiting for exec process to close")
+				if err := <-c.execCh; err != nil && !strings.HasPrefix(err.Error(), "signal:") {
+					return fmt.Errorf("cannot wait for exec process: %w", err)
+				}
+			}
+		}
 
 		// Gracefully close.
-		return c.Close()
+		if e := c.Close(); e != nil && err == nil {
+			err = e
+		}
+		fmt.Fprintln(m.stdout, "litestream shut down")
+		return err
 
 	case "restore":
-		return (&RestoreCommand{}).Run(ctx, args)
+		return NewRestoreCommand(m.stdin, m.stdout, m.stderr).Run(ctx, args)
 	case "snapshots":
-		return (&SnapshotsCommand{}).Run(ctx, args)
+		return NewSnapshotsCommand(m.stdin, m.stdout, m.stderr).Run(ctx, args)
 	case "version":
-		return (&VersionCommand{}).Run(ctx, args)
+		return NewVersionCommand(m.stdin, m.stdout, m.stderr).Run(ctx, args)
 	case "wal":
-		return (&WALCommand{}).Run(ctx, args)
+		return NewWALCommand(m.stdin, m.stdout, m.stderr).Run(ctx, args)
 	default:
 		if cmd == "" || cmd == "help" || strings.HasPrefix(cmd, "-") {
 			m.Usage()
@@ -113,7 +149,7 @@ func (m *Main) Run(ctx context.Context, args []string) (err error) {
 
 // Usage prints the help screen to STDOUT.
 func (m *Main) Usage() {
-	fmt.Println(`
+	fmt.Fprintln(m.stdout, `
 litestream is a tool for replicating SQLite databases.
 
 Usage:
@@ -140,6 +176,10 @@ type Config struct {
 	// List of databases to manage.
 	DBs []*DBConfig `yaml:"dbs"`
 
+	// Subcommand to execute during replication.
+	// Litestream will shutdown when subcommand exits.
+	Exec string `yaml:"exec"`
+
 	// Global S3 settings
 	AccessKeyID     string `yaml:"access-key-id"`
 	SecretAccessKey string `yaml:"secret-access-key"`
@@ -175,7 +215,35 @@ func (c *Config) DBConfig(path string) *DBConfig {
 }
 
 // ReadConfigFile unmarshals config from filename. Expands path if needed.
-func ReadConfigFile(filename string) (_ Config, err error) {
+// If expandEnv is true then environment variables are expanded in the config.
+// If filename is blank then the default config path is used.
+func ReadConfigFile(filename string, expandEnv bool) (config Config, err error) {
+	var filenames []string
+	if filename != "" {
+		filenames = append(filenames, filename)
+	}
+	filenames = append(filenames, "./litestream.yml")
+	filenames = append(filenames, DefaultConfigPath())
+
+	for _, name := range filenames {
+		isDefaultPath := name != filename
+
+		if config, err = readConfigFile(name, expandEnv); os.IsNotExist(err) {
+			if isDefaultPath {
+				continue
+			}
+			return config, fmt.Errorf("config file not found: %s", filename)
+		} else if err != nil {
+			return config, err
+		}
+		break
+	}
+	return config, nil
+}
+
+// readConfigFile unmarshals config from filename. Expands path if needed.
+// If expandEnv is true then environment variables are expanded in the config.
+func readConfigFile(filename string, expandEnv bool) (_ Config, err error) {
 	config := DefaultConfig()
 
 	// Expand filename, if necessary.
@@ -184,12 +252,19 @@ func ReadConfigFile(filename string) (_ Config, err error) {
 		return config, err
 	}
 
-	// Read & deserialize configuration.
-	if buf, err := ioutil.ReadFile(filename); os.IsNotExist(err) {
-		return config, fmt.Errorf("config file not found: %s", filename)
-	} else if err != nil {
+	// Read configuration.
+	// Do not return an error if using default path and file is missing.
+	buf, err := ioutil.ReadFile(filename)
+	if err != nil {
 		return config, err
-	} else if err := yaml.Unmarshal(buf, &config); err != nil {
+	}
+
+	// Expand environment variables, if enabled.
+	if expandEnv {
+		buf = []byte(os.ExpandEnv(string(buf)))
+	}
+
+	if err := yaml.Unmarshal(buf, &config); err != nil {
 		return config, err
 	}
 
@@ -209,10 +284,12 @@ func ReadConfigFile(filename string) (_ Config, err error) {
 // DBConfig represents the configuration for a single database.
 type DBConfig struct {
 	Path                 string         `yaml:"path"`
-	MonitorInterval    *time.Duration `yaml:"monitor-interval"`
+	Upstream             UpstreamConfig `yaml:"upstream"`
+	MonitorDelayInterval *time.Duration `yaml:"monitor-delay-interval"`
 	CheckpointInterval   *time.Duration `yaml:"checkpoint-interval"`
 	MinCheckpointPageN   *int           `yaml:"min-checkpoint-page-count"`
 	MaxCheckpointPageN   *int           `yaml:"max-checkpoint-page-count"`
+	ShadowRetentionN     *int           `yaml:"shadow-retention-count"`
 
 	Replicas []*ReplicaConfig `yaml:"replicas"`
 }
@@ -223,13 +300,27 @@ func NewDBFromConfig(dbc *DBConfig) (*litestream.DB, error) {
 	if err != nil {
 		return nil, err
 	}
+	return NewDBFromConfigWithPath(dbc, path)
+}
 
+// NewDBFromConfigWithPath instantiates a DB based on a configuration and using a given path.
+func NewDBFromConfigWithPath(dbc *DBConfig, path string) (*litestream.DB, error) {
 	// Initialize database with given path.
 	db := litestream.NewDB(path)
 
+	// Attach upstream HTTP client if specified.
+	if upstreamURL := dbc.Upstream.URL; upstreamURL != "" {
+		// Use local database path if upstream path is not specified.
+		upstreamPath := dbc.Upstream.Path
+		if upstreamPath == "" {
+			upstreamPath = db.Path()
+		}
+		db.StreamClient = http.NewClient(upstreamURL, upstreamPath)
+	}
+
 	// Override default database settings if specified in configuration.
-	if dbc.MonitorInterval != nil {
-		db.MonitorInterval = *dbc.MonitorInterval
+	if dbc.MonitorDelayInterval != nil {
+		db.MonitorDelayInterval = *dbc.MonitorDelayInterval
 	}
 	if dbc.CheckpointInterval != nil {
 		db.CheckpointInterval = *dbc.CheckpointInterval
@@ -240,6 +331,9 @@ func NewDBFromConfig(dbc *DBConfig) (*litestream.DB, error) {
 	if dbc.MaxCheckpointPageN != nil {
 		db.MaxCheckpointPageN = *dbc.MaxCheckpointPageN
 	}
+	if dbc.ShadowRetentionN != nil {
+		db.ShadowRetentionN = *dbc.ShadowRetentionN
+	}
 
 	// Instantiate and attach replicas.
 	for _, rc := range dbc.Replicas {
@@ -253,17 +347,22 @@ func NewDBFromConfig(dbc *DBConfig) (*litestream.DB, error) {
 	return db, nil
 }
 
+type UpstreamConfig struct {
+	URL  string `yaml:"url"`
+	Path string `yaml:"path"`
+}
+
 // ReplicaConfig represents the configuration for a single replica in a database.
 type ReplicaConfig struct {
 	Type                   string         `yaml:"type"` // "file", "s3"
 	Name                   string         `yaml:"name"` // name of replica, optional.
 	Path                   string         `yaml:"path"`
 	URL                    string         `yaml:"url"`
-	Retention              time.Duration `yaml:"retention"`
-	RetentionCheckInterval time.Duration `yaml:"retention-check-interval"`
-	SyncInterval           time.Duration `yaml:"sync-interval"` // s3 only
-	SnapshotInterval       time.Duration `yaml:"snapshot-interval"`
-	ValidationInterval     time.Duration `yaml:"validation-interval"`
+	Retention              *time.Duration `yaml:"retention"`
+	RetentionCheckInterval *time.Duration `yaml:"retention-check-interval"`
+	SyncInterval           *time.Duration `yaml:"sync-interval"`
+	SnapshotInterval       *time.Duration `yaml:"snapshot-interval"`
+	ValidationInterval     *time.Duration `yaml:"validation-interval"`
 
 	// S3 settings
 	AccessKeyID     string `yaml:"access-key-id"`
@@ -272,27 +371,76 @@ type ReplicaConfig struct {
 	Bucket          string `yaml:"bucket"`
 	Endpoint        string `yaml:"endpoint"`
 	ForcePathStyle  *bool  `yaml:"force-path-style"`
+	SkipVerify      bool   `yaml:"skip-verify"`
+
+	// ABS settings
+	AccountName string `yaml:"account-name"`
+	AccountKey  string `yaml:"account-key"`
+
+	// SFTP settings
+	Host     string `yaml:"host"`
+	User     string `yaml:"user"`
+	Password string `yaml:"password"`
+	KeyPath  string `yaml:"key-path"`
 }
 
 // NewReplicaFromConfig instantiates a replica for a DB based on a config.
-func NewReplicaFromConfig(c *ReplicaConfig, db *litestream.DB) (litestream.Replica, error) {
+func NewReplicaFromConfig(c *ReplicaConfig, db *litestream.DB) (_ *litestream.Replica, err error) {
 	// Ensure user did not specify URL in path.
 	if isURL(c.Path) {
 		return nil, fmt.Errorf("replica path cannot be a url, please use the 'url' field instead: %s", c.Path)
 	}
 
-	switch c.ReplicaType() {
+	// Build and set client on replica.
+	var client litestream.ReplicaClient
+	switch typ := c.ReplicaType(); typ {
 	case "file":
-		return newFileReplicaFromConfig(c, db)
-	case "s3":
-		return newS3ReplicaFromConfig(c, db)
-	default:
-		return nil, fmt.Errorf("unknown replica type in config: %q", c.Type)
+		if client, err = newFileReplicaClientFromConfig(c); err != nil {
+			return nil, err
 		}
+	case "s3":
+		if client, err = newS3ReplicaClientFromConfig(c); err != nil {
+			return nil, err
+		}
+	case "gs":
+		if client, err = newGSReplicaClientFromConfig(c); err != nil {
+			return nil, err
+		}
+	case "abs":
+		if client, err = newABSReplicaClientFromConfig(c); err != nil {
+			return nil, err
+		}
+	case "sftp":
+		if client, err = newSFTPReplicaClientFromConfig(c); err != nil {
+			return nil, err
+		}
+	default:
+		return nil, fmt.Errorf("unknown replica type in config: %q", typ)
 	}
 
-// newFileReplicaFromConfig returns a new instance of FileReplica build from config.
-func newFileReplicaFromConfig(c *ReplicaConfig, db *litestream.DB) (_ *litestream.FileReplica, err error) {
+	// Build replica.
+	r := litestream.NewReplica(db, c.Name, client)
+	if v := c.Retention; v != nil {
+		r.Retention = *v
+	}
+	if v := c.RetentionCheckInterval; v != nil {
+		r.RetentionCheckInterval = *v
+	}
+	if v := c.SyncInterval; v != nil {
+		r.SyncInterval = *v
+	}
+	if v := c.SnapshotInterval; v != nil {
+		r.SnapshotInterval = *v
+	}
+	if v := c.ValidationInterval; v != nil {
+		r.ValidationInterval = *v
+	}
+
+	return r, nil
+}
+
+// newFileReplicaClientFromConfig returns a new instance of FileReplicaClient built from config.
+func newFileReplicaClientFromConfig(c *ReplicaConfig) (_ *litestream.FileReplicaClient, err error) {
 	// Ensure URL & path are not both specified.
 	if c.URL != "" && c.Path != "" {
 		return nil, fmt.Errorf("cannot specify url & path for file replica")
@@ -317,24 +465,11 @@ func newFileReplicaFromConfig(c *ReplicaConfig, db *litestream.DB) (_ *litestrea
 	}
 
 	// Instantiate replica and apply time fields, if set.
-	r := litestream.NewFileReplica(db, c.Name, path)
-	if v := c.Retention; v > 0 {
-		r.Retention = v
-	}
-	if v := c.RetentionCheckInterval; v > 0 {
-		r.RetentionCheckInterval = v
-	}
-	if v := c.SnapshotInterval; v > 0 {
-		r.SnapshotInterval = v
-	}
-	if v := c.ValidationInterval; v > 0 {
-		r.ValidationInterval = v
-	}
-	return r, nil
+	return litestream.NewFileReplicaClient(path), nil
 }
 
-// newS3ReplicaFromConfig returns a new instance of S3Replica build from config.
-func newS3ReplicaFromConfig(c *ReplicaConfig, db *litestream.DB) (_ *s3.Replica, err error) {
+// newS3ReplicaClientFromConfig returns a new instance of s3.ReplicaClient built from config.
+func newS3ReplicaClientFromConfig(c *ReplicaConfig) (_ *s3.ReplicaClient, err error) {
 	// Ensure URL & constituent parts are not both specified.
 	if c.URL != "" && c.Path != "" {
 		return nil, fmt.Errorf("cannot specify url & path for s3 replica")
@@ -343,7 +478,7 @@ func newS3ReplicaFromConfig(c *ReplicaConfig, db *litestream.DB) (_ *s3.Replica,
 	}
 
 	bucket, path := c.Bucket, c.Path
-	region, endpoint := c.Region, c.Endpoint
+	region, endpoint, skipVerify := c.Region, c.Endpoint, c.SkipVerify
 
 	// Use path style if an endpoint is explicitly set. This works because the
 	// only service to not use path style is AWS which does not use an endpoint.
@@ -384,31 +519,164 @@ func newS3ReplicaFromConfig(c *ReplicaConfig, db *litestream.DB) (_ *s3.Replica,
 	}
 
 	// Build replica.
-	r := s3.NewReplica(db, c.Name)
-	r.AccessKeyID = c.AccessKeyID
-	r.SecretAccessKey = c.SecretAccessKey
-	r.Bucket = bucket
-	r.Path = path
-	r.Region = region
-	r.Endpoint = endpoint
-	r.ForcePathStyle = forcePathStyle
+	client := s3.NewReplicaClient()
+	client.AccessKeyID = c.AccessKeyID
+	client.SecretAccessKey = c.SecretAccessKey
+	client.Bucket = bucket
+	client.Path = path
+	client.Region = region
+	client.Endpoint = endpoint
+	client.ForcePathStyle = forcePathStyle
+	client.SkipVerify = skipVerify
+	return client, nil
+}
 
-	if v := c.Retention; v > 0 {
-		r.Retention = v
+// newGSReplicaClientFromConfig returns a new instance of gs.ReplicaClient built from config.
+func newGSReplicaClientFromConfig(c *ReplicaConfig) (_ *gs.ReplicaClient, err error) {
+	// Ensure URL & constituent parts are not both specified.
+	if c.URL != "" && c.Path != "" {
+		return nil, fmt.Errorf("cannot specify url & path for gs replica")
+	} else if c.URL != "" && c.Bucket != "" {
+		return nil, fmt.Errorf("cannot specify url & bucket for gs replica")
 	}
-	if v := c.RetentionCheckInterval; v > 0 {
-		r.RetentionCheckInterval = v
+
+	bucket, path := c.Bucket, c.Path
+
+	// Apply settings from URL, if specified.
+	if c.URL != "" {
+		_, uhost, upath, err := ParseReplicaURL(c.URL)
+		if err != nil {
+			return nil, err
 		}
-	if v := c.SyncInterval; v > 0 {
-		r.SyncInterval = v
+
+		// Only apply URL parts to field that have not been overridden.
+		if path == "" {
+			path = upath
 		}
-	if v := c.SnapshotInterval; v > 0 {
-		r.SnapshotInterval = v
+		if bucket == "" {
+			bucket = uhost
+		}
+	}
+
+	// Ensure required settings are set.
+	if bucket == "" {
+		return nil, fmt.Errorf("bucket required for gs replica")
+	}
+
+	// Build replica.
+	client := gs.NewReplicaClient()
+	client.Bucket = bucket
+	client.Path = path
+	return client, nil
+}
+
+// newABSReplicaClientFromConfig returns a new instance of abs.ReplicaClient built from config.
+func newABSReplicaClientFromConfig(c *ReplicaConfig) (_ *abs.ReplicaClient, err error) {
+	// Ensure URL & constituent parts are not both specified.
+	if c.URL != "" && c.Path != "" {
+		return nil, fmt.Errorf("cannot specify url & path for abs replica")
+	} else if c.URL != "" && c.Bucket != "" {
+		return nil, fmt.Errorf("cannot specify url & bucket for abs replica")
+	}
+
+	// Build replica.
+	client := abs.NewReplicaClient()
+	client.AccountName = c.AccountName
+	client.AccountKey = c.AccountKey
+	client.Bucket = c.Bucket
+	client.Path = c.Path
+	client.Endpoint = c.Endpoint
+
+	// Apply settings from URL, if specified.
+	if c.URL != "" {
+		u, err := url.Parse(c.URL)
+		if err != nil {
+			return nil, err
+		}
+
+		if client.AccountName == "" && u.User != nil {
+			client.AccountName = u.User.Username()
+		}
+		if client.Bucket == "" {
+			client.Bucket = u.Host
+		}
+		if client.Path == "" {
+			client.Path = strings.TrimPrefix(path.Clean(u.Path), "/")
+		}
+	}
+
+	// Ensure required settings are set.
+	if client.Bucket == "" {
+		return nil, fmt.Errorf("bucket required for abs replica")
+	}
+
+	return client, nil
+}
+
+// newSFTPReplicaClientFromConfig returns a new instance of sftp.ReplicaClient built from config.
+func newSFTPReplicaClientFromConfig(c *ReplicaConfig) (_ *sftp.ReplicaClient, err error) {
+	// Ensure URL & constituent parts are not both specified.
+	if c.URL != "" && c.Path != "" {
+		return nil, fmt.Errorf("cannot specify url & path for sftp replica")
+	} else if c.URL != "" && c.Host != "" {
+		return nil, fmt.Errorf("cannot specify url & host for sftp replica")
+	}
+
+	host, user, password, path := c.Host, c.User, c.Password, c.Path
+
+	// Apply settings from URL, if specified.
+	if c.URL != "" {
+		u, err := url.Parse(c.URL)
+		if err != nil {
+			return nil, err
+		}
+
+		// Only apply URL parts to field that have not been overridden.
+		if host == "" {
+			host = u.Host
+		}
+		if user == "" && u.User != nil {
+			user = u.User.Username()
+		}
+		if password == "" && u.User != nil {
+			password, _ = u.User.Password()
+		}
+		if path == "" {
+			path = u.Path
+		}
+	}
+
+	// Ensure required settings are set.
+	if host == "" {
+		return nil, fmt.Errorf("host required for sftp replica")
+	} else if user == "" {
+		return nil, fmt.Errorf("user required for sftp replica")
+	}
+
+	// Build replica.
+	client := sftp.NewReplicaClient()
+	client.Host = host
+	client.User = user
+	client.Password = password
+	client.Path = path
+	client.KeyPath = c.KeyPath
+	return client, nil
+}
+
+// applyLitestreamEnv copies "LITESTREAM" prefixed environment variables to
+// their AWS counterparts as the "AWS" prefix can be confusing when using a
+// non-AWS S3-compatible service.
+func applyLitestreamEnv() {
+	if v, ok := os.LookupEnv("LITESTREAM_ACCESS_KEY_ID"); ok {
+		if _, ok := os.LookupEnv("AWS_ACCESS_KEY_ID"); !ok {
+			os.Setenv("AWS_ACCESS_KEY_ID", v)
+		}
+	}
+	if v, ok := os.LookupEnv("LITESTREAM_SECRET_ACCESS_KEY"); ok {
+		if _, ok := os.LookupEnv("AWS_SECRET_ACCESS_KEY"); !ok {
+			os.Setenv("AWS_SECRET_ACCESS_KEY", v)
 		}
-	if v := c.ValidationInterval; v > 0 {
-		r.ValidationInterval = v
 	}
-	return r, nil
 }
 
 // ParseReplicaURL parses a replica URL.
@@ -452,11 +720,12 @@ func DefaultConfigPath() string {
 	if v := os.Getenv("LITESTREAM_CONFIG"); v != "" {
 		return v
 	}
-	return defaultConfigPath
+	return "/etc/litestream.yml"
 }
 
-func registerConfigFlag(fs *flag.FlagSet) *string {
-	return fs.String("config", "", "config path")
+func registerConfigFlag(fs *flag.FlagSet, configPath *string, noExpandEnv *bool) {
+	fs.StringVar(configPath, "config", "", "config path")
+	fs.BoolVar(noExpandEnv, "no-expand-env", false, "do not expand env vars in config")
 }
 
 // expand returns an absolute path for s.
@@ -481,3 +750,66 @@ func expand(s string) (string, error) {
 	}
 	return filepath.Join(u.HomeDir, strings.TrimPrefix(s, prefix)), nil
 }
+
+// indexVar allows the flag package to parse index flags as 4-byte hexadecimal values.
+type indexVar int
+
+// Ensure type implements interface.
+var _ flag.Value = (*indexVar)(nil)
+
+// String returns an 8-character hexadecimal value.
+func (v *indexVar) String() string {
+	return litestream.FormatIndex(int(*v))
+}
+
+// Set parses s into an integer from a hexadecimal value.
+func (v *indexVar) Set(s string) error {
+	i, err := strconv.ParseInt(s, 16, 32)
+	if err != nil {
+		return fmt.Errorf("invalid hexadecimal format")
+	}
+	*v = indexVar(i)
+	return nil
+}
+
+// loadReplicas returns a list of replicas to use based on CLI flags. Filters
+// by replicaName, if not blank. The DB is returned if pathOrURL is not a replica URL.
+func loadReplicas(ctx context.Context, config Config, pathOrURL, replicaName string) ([]*litestream.Replica, *litestream.DB, error) {
+	// Build a replica based on URL, if specified.
+	if isURL(pathOrURL) {
+		r, err := NewReplicaFromConfig(&ReplicaConfig{
+			URL:             pathOrURL,
+			AccessKeyID:     config.AccessKeyID,
+			SecretAccessKey: config.SecretAccessKey,
+		}, nil)
+		if err != nil {
+			return nil, nil, err
+		}
+		return []*litestream.Replica{r}, nil, nil
+	}
+
+	// Otherwise use replicas from the database configuration file.
+	path, err := expand(pathOrURL)
+	if err != nil {
+		return nil, nil, err
+	}
+	dbc := config.DBConfig(path)
+	if dbc == nil {
+		return nil, nil, fmt.Errorf("database not found in config: %s", path)
+	}
+	db, err := NewDBFromConfig(dbc)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// Filter by replica, if specified.
+	if replicaName != "" {
+		r := db.Replica(replicaName)
+		if r == nil {
+			return nil, nil, fmt.Errorf("replica %q not found for database %q", replicaName, db.Path())
+		}
+		return []*litestream.Replica{r}, db, nil
+	}
+
+	return db.Replicas, db, nil
+}

cmd/litestream/main_notwindows.go

@@ -1,17 +0,0 @@
-// +build !windows
-
-package main
-
-import (
-	"context"
-)
-
-const defaultConfigPath = "/etc/litestream.yml"
-
-func isWindowsService() (bool, error) {
-	return false, nil
-}
-
-func runWindowsService(ctx context.Context) error {
-	panic("cannot run windows service as unix process")
-}

cmd/litestream/main_test.go

@@ -1,15 +1,24 @@
 package main_test
 
 import (
+	"bytes"
+	"io"
 	"io/ioutil"
+	"log"
+	"os"
 	"path/filepath"
 	"testing"
 
 	"github.com/benbjohnson/litestream"
 	main "github.com/benbjohnson/litestream/cmd/litestream"
+	"github.com/benbjohnson/litestream/gs"
 	"github.com/benbjohnson/litestream/s3"
 )
 
+func init() {
+	litestream.LogFlags = log.Lmsgprefix | log.Ldate | log.Ltime | log.Lmicroseconds | log.LUTC | log.Lshortfile
+}
+
 func TestReadConfigFile(t *testing.T) {
 	// Ensure global AWS settings are propagated down to replica configurations.
 	t.Run("PropagateGlobalSettings", func(t *testing.T) {
@@ -26,7 +35,7 @@ dbs:
 			t.Fatal(err)
 		}
 
-		config, err := main.ReadConfigFile(filename)
+		config, err := main.ReadConfigFile(filename, true)
 		if err != nil {
 			t.Fatal(err)
 		} else if got, want := config.AccessKeyID, `XXX`; got != want {
@@ -39,15 +48,65 @@ dbs:
 			t.Fatalf("Replica.SecretAccessKey=%v, want %v", got, want)
 		}
 	})
+
+	// Ensure environment variables are expanded.
+	t.Run("ExpandEnv", func(t *testing.T) {
+		os.Setenv("LITESTREAM_TEST_0129380", "/path/to/db")
+		os.Setenv("LITESTREAM_TEST_1872363", "s3://foo/bar")
+
+		filename := filepath.Join(t.TempDir(), "litestream.yml")
+		if err := ioutil.WriteFile(filename, []byte(`
+dbs:
+  - path: $LITESTREAM_TEST_0129380
+    replicas:
+      - url: ${LITESTREAM_TEST_1872363}
+      - url: ${LITESTREAM_TEST_NO_SUCH_ENV}
+`[1:]), 0666); err != nil {
+			t.Fatal(err)
+		}
+
+		config, err := main.ReadConfigFile(filename, true)
+		if err != nil {
+			t.Fatal(err)
+		} else if got, want := config.DBs[0].Path, `/path/to/db`; got != want {
+			t.Fatalf("DB.Path=%v, want %v", got, want)
+		} else if got, want := config.DBs[0].Replicas[0].URL, `s3://foo/bar`; got != want {
+			t.Fatalf("Replica[0].URL=%v, want %v", got, want)
+		} else if got, want := config.DBs[0].Replicas[1].URL, ``; got != want {
+			t.Fatalf("Replica[1].URL=%v, want %v", got, want)
+		}
+	})
+
+	// Ensure environment variables are not expanded.
+	t.Run("NoExpandEnv", func(t *testing.T) {
+		os.Setenv("LITESTREAM_TEST_9847533", "s3://foo/bar")
+
+		filename := filepath.Join(t.TempDir(), "litestream.yml")
+		if err := ioutil.WriteFile(filename, []byte(`
+dbs:
+  - path: /path/to/db
+    replicas:
+      - url: ${LITESTREAM_TEST_9847533}
+`[1:]), 0666); err != nil {
+			t.Fatal(err)
+		}
+
+		config, err := main.ReadConfigFile(filename, false)
+		if err != nil {
+			t.Fatal(err)
+		} else if got, want := config.DBs[0].Replicas[0].URL, `${LITESTREAM_TEST_9847533}`; got != want {
+			t.Fatalf("Replica.URL=%v, want %v", got, want)
+		}
+	})
 }
 
 func TestNewFileReplicaFromConfig(t *testing.T) {
 	r, err := main.NewReplicaFromConfig(&main.ReplicaConfig{Path: "/foo"}, nil)
 	if err != nil {
 		t.Fatal(err)
-	} else if r, ok := r.(*litestream.FileReplica); !ok {
+	} else if client, ok := r.Client().(*litestream.FileReplicaClient); !ok {
 		t.Fatal("unexpected replica type")
-	} else if got, want := r.Path(), "/foo"; got != want {
+	} else if got, want := client.Path(), "/foo"; got != want {
 		t.Fatalf("Path=%s, want %s", got, want)
 	}
 }
@@ -57,17 +116,17 @@ func TestNewS3ReplicaFromConfig(t *testing.T) {
 		r, err := main.NewReplicaFromConfig(&main.ReplicaConfig{URL: "s3://foo/bar"}, nil)
 		if err != nil {
 			t.Fatal(err)
-		} else if r, ok := r.(*s3.Replica); !ok {
+		} else if client, ok := r.Client().(*s3.ReplicaClient); !ok {
 			t.Fatal("unexpected replica type")
-		} else if got, want := r.Bucket, "foo"; got != want {
+		} else if got, want := client.Bucket, "foo"; got != want {
 			t.Fatalf("Bucket=%s, want %s", got, want)
-		} else if got, want := r.Path, "bar"; got != want {
+		} else if got, want := client.Path, "bar"; got != want {
 			t.Fatalf("Path=%s, want %s", got, want)
-		} else if got, want := r.Region, ""; got != want {
+		} else if got, want := client.Region, ""; got != want {
 			t.Fatalf("Region=%s, want %s", got, want)
-		} else if got, want := r.Endpoint, ""; got != want {
+		} else if got, want := client.Endpoint, ""; got != want {
 			t.Fatalf("Endpoint=%s, want %s", got, want)
-		} else if got, want := r.ForcePathStyle, false; got != want {
+		} else if got, want := client.ForcePathStyle, false; got != want {
 			t.Fatalf("ForcePathStyle=%v, want %v", got, want)
 		}
 	})
@@ -76,17 +135,17 @@ func TestNewS3ReplicaFromConfig(t *testing.T) {
 		r, err := main.NewReplicaFromConfig(&main.ReplicaConfig{URL: "s3://foo.localhost:9000/bar"}, nil)
 		if err != nil {
 			t.Fatal(err)
-		} else if r, ok := r.(*s3.Replica); !ok {
+		} else if client, ok := r.Client().(*s3.ReplicaClient); !ok {
 			t.Fatal("unexpected replica type")
-		} else if got, want := r.Bucket, "foo"; got != want {
+		} else if got, want := client.Bucket, "foo"; got != want {
 			t.Fatalf("Bucket=%s, want %s", got, want)
-		} else if got, want := r.Path, "bar"; got != want {
+		} else if got, want := client.Path, "bar"; got != want {
 			t.Fatalf("Path=%s, want %s", got, want)
-		} else if got, want := r.Region, "us-east-1"; got != want {
+		} else if got, want := client.Region, "us-east-1"; got != want {
 			t.Fatalf("Region=%s, want %s", got, want)
-		} else if got, want := r.Endpoint, "http://localhost:9000"; got != want {
+		} else if got, want := client.Endpoint, "http://localhost:9000"; got != want {
 			t.Fatalf("Endpoint=%s, want %s", got, want)
-		} else if got, want := r.ForcePathStyle, true; got != want {
+		} else if got, want := client.ForcePathStyle, true; got != want {
 			t.Fatalf("ForcePathStyle=%v, want %v", got, want)
 		}
 	})
@@ -95,37 +154,45 @@ func TestNewS3ReplicaFromConfig(t *testing.T) {
 		r, err := main.NewReplicaFromConfig(&main.ReplicaConfig{URL: "s3://foo.s3.us-west-000.backblazeb2.com/bar"}, nil)
 		if err != nil {
 			t.Fatal(err)
-		} else if r, ok := r.(*s3.Replica); !ok {
+		} else if client, ok := r.Client().(*s3.ReplicaClient); !ok {
 			t.Fatal("unexpected replica type")
-		} else if got, want := r.Bucket, "foo"; got != want {
+		} else if got, want := client.Bucket, "foo"; got != want {
 			t.Fatalf("Bucket=%s, want %s", got, want)
-		} else if got, want := r.Path, "bar"; got != want {
+		} else if got, want := client.Path, "bar"; got != want {
 			t.Fatalf("Path=%s, want %s", got, want)
-		} else if got, want := r.Region, "us-west-000"; got != want {
+		} else if got, want := client.Region, "us-west-000"; got != want {
 			t.Fatalf("Region=%s, want %s", got, want)
-		} else if got, want := r.Endpoint, "https://s3.us-west-000.backblazeb2.com"; got != want {
+		} else if got, want := client.Endpoint, "https://s3.us-west-000.backblazeb2.com"; got != want {
 			t.Fatalf("Endpoint=%s, want %s", got, want)
-		} else if got, want := r.ForcePathStyle, true; got != want {
+		} else if got, want := client.ForcePathStyle, true; got != want {
 			t.Fatalf("ForcePathStyle=%v, want %v", got, want)
 		}
 	})
+}
 
-	t.Run("GCS", func(t *testing.T) {
-		r, err := main.NewReplicaFromConfig(&main.ReplicaConfig{URL: "s3://foo.storage.googleapis.com/bar"}, nil)
+func TestNewGSReplicaFromConfig(t *testing.T) {
+	r, err := main.NewReplicaFromConfig(&main.ReplicaConfig{URL: "gs://foo/bar"}, nil)
 	if err != nil {
 		t.Fatal(err)
-		} else if r, ok := r.(*s3.Replica); !ok {
+	} else if client, ok := r.Client().(*gs.ReplicaClient); !ok {
 		t.Fatal("unexpected replica type")
-		} else if got, want := r.Bucket, "foo"; got != want {
+	} else if got, want := client.Bucket, "foo"; got != want {
 		t.Fatalf("Bucket=%s, want %s", got, want)
-		} else if got, want := r.Path, "bar"; got != want {
+	} else if got, want := client.Path, "bar"; got != want {
 		t.Fatalf("Path=%s, want %s", got, want)
-		} else if got, want := r.Region, "us-east-1"; got != want {
-			t.Fatalf("Region=%s, want %s", got, want)
-		} else if got, want := r.Endpoint, "https://storage.googleapis.com"; got != want {
-			t.Fatalf("Endpoint=%s, want %s", got, want)
-		} else if got, want := r.ForcePathStyle, true; got != want {
-			t.Fatalf("ForcePathStyle=%v, want %v", got, want)
 	}
-	})
+}
+
+// newMain returns a new instance of Main and associated buffers.
+func newMain() (m *main.Main, stdin, stdout, stderr *bytes.Buffer) {
+	stdin, stdout, stderr = &bytes.Buffer{}, &bytes.Buffer{}, &bytes.Buffer{}
+
+	// Split stdout/stderr to terminal if verbose flag set.
+	out, err := io.Writer(stdout), io.Writer(stderr)
+	if testing.Verbose() {
+		out = io.MultiWriter(out, os.Stdout)
+		err = io.MultiWriter(err, os.Stderr)
+	}
+
+	return main.NewMain(stdin, out, err), stdin, stdout, stderr
 }

cmd/litestream/main_windows.go

@@ -1,105 +0,0 @@
-// +build windows
-
-package main
-
-import (
-	"context"
-	"io"
-	"log"
-	"os"
-
-	"golang.org/x/sys/windows"
-	"golang.org/x/sys/windows/svc"
-	"golang.org/x/sys/windows/svc/eventlog"
-)
-
-const defaultConfigPath = `C:\Litestream\litestream.yml`
-
-// serviceName is the Windows Service name.
-const serviceName = "Litestream"
-
-// isWindowsService returns true if currently executing within a Windows service.
-func isWindowsService() (bool, error) {
-	return svc.IsWindowsService()
-}
-
-func runWindowsService(ctx context.Context) error {
-	// Attempt to install new log service. This will fail if already installed.
-	// We don't log the error because we don't have anywhere to log until we open the log.
-	_ = eventlog.InstallAsEventCreate(serviceName, eventlog.Error|eventlog.Warning|eventlog.Info)
-
-	elog, err := eventlog.Open(serviceName)
-	if err != nil {
-		return err
-	}
-	defer elog.Close()
-
-	// Set eventlog as log writer while running.
-	log.SetOutput((*eventlogWriter)(elog))
-	defer log.SetOutput(os.Stderr)
-
-	log.Print("Litestream service starting")
-
-	if err := svc.Run(serviceName, &windowsService{ctx: ctx}); err != nil {
-		return errStop
-	}
-
-	log.Print("Litestream service stopped")
-	return nil
-}
-
-// windowsService is an interface adapter for svc.Handler.
-type windowsService struct {
-	ctx context.Context
-}
-
-func (s *windowsService) Execute(args []string, r <-chan svc.ChangeRequest, statusCh chan<- svc.Status) (svcSpecificEC bool, exitCode uint32) {
-	var err error
-
-	// Notify Windows that the service is starting up.
-	statusCh <- svc.Status{State: svc.StartPending}
-
-	// Instantiate replication command and load configuration.
-	c := NewReplicateCommand()
-	if c.Config, err = ReadConfigFile(DefaultConfigPath()); err != nil {
-		log.Printf("cannot load configuration: %s", err)
-		return true, 1
-	}
-
-	// Execute replication command.
-	if err := c.Run(s.ctx); err != nil {
-		log.Printf("cannot replicate: %s", err)
-		statusCh <- svc.Status{State: svc.StopPending}
-		return true, 2
-	}
-
-	// Notify Windows that the service is now running.
-	statusCh <- svc.Status{State: svc.Running, Accepts: svc.AcceptStop}
-
-	for {
-		select {
-		case req := <-r:
-			switch req.Cmd {
-			case svc.Stop:
-				c.Close()
-				statusCh <- svc.Status{State: svc.StopPending}
-				return false, windows.NO_ERROR
-			case svc.Interrogate:
-				statusCh <- req.CurrentStatus
-			default:
-				log.Printf("Litestream service received unexpected change request cmd: %d", req.Cmd)
-			}
-		}
-	}
-}
-
-// Ensure implementation implements io.Writer interface.
-var _ io.Writer = (*eventlogWriter)(nil)
-
-// eventlogWriter is an adapter for using eventlog.Log as an io.Writer.
-type eventlogWriter eventlog.Log
-
-func (w *eventlogWriter) Write(p []byte) (n int, err error) {
-	elog := (*eventlog.Log)(w)
-	return 0, elog.Info(1, string(p))
-}

cmd/litestream/replicate.go

@@ -4,35 +4,55 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"io"
 	"log"
-	"net"
-	"net/http"
-	_ "net/http/pprof"
 	"os"
-	"time"
+	"os/exec"
 
 	"github.com/benbjohnson/litestream"
+	"github.com/benbjohnson/litestream/abs"
+	"github.com/benbjohnson/litestream/gs"
+	"github.com/benbjohnson/litestream/http"
 	"github.com/benbjohnson/litestream/s3"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/benbjohnson/litestream/sftp"
+	"github.com/mattn/go-shellwords"
 )
 
 // ReplicateCommand represents a command that continuously replicates SQLite databases.
 type ReplicateCommand struct {
+	stdin  io.Reader
+	stdout io.Writer
+	stderr io.Writer
+
+	configPath  string
+	noExpandEnv bool
+
+	cmd    *exec.Cmd  // subcommand
+	execCh chan error // subcommand error channel
+
 	Config Config
 
-	// List of managed databases specified in the config.
-	DBs []*litestream.DB
+	server     *litestream.Server
+	httpServer *http.Server
 }
 
-func NewReplicateCommand() *ReplicateCommand {
-	return &ReplicateCommand{}
+// NewReplicateCommand returns a new instance of ReplicateCommand.
+func NewReplicateCommand(stdin io.Reader, stdout, stderr io.Writer) *ReplicateCommand {
+	return &ReplicateCommand{
+		stdin:  stdin,
+		stdout: stdout,
+		stderr: stderr,
+
+		execCh: make(chan error),
+	}
 }
 
 // ParseFlags parses the CLI flags and loads the configuration file.
 func (c *ReplicateCommand) ParseFlags(ctx context.Context, args []string) (err error) {
 	fs := flag.NewFlagSet("litestream-replicate", flag.ContinueOnError)
-	tracePath := fs.String("trace", "", "trace path")
-	configPath := registerConfigFlag(fs)
+	execFlag := fs.String("exec", "", "execute subcommand")
+	addr := fs.String("addr", "", "HTTP bind address (host:port)")
+	registerConfigFlag(fs, &c.configPath, &c.noExpandEnv)
 	fs.Usage = c.Usage
 	if err := fs.Parse(args); err != nil {
 		return err
@@ -42,35 +62,34 @@ func (c *ReplicateCommand) ParseFlags(ctx context.Context, args []string) (err e
 	if fs.NArg() == 1 {
 		return fmt.Errorf("must specify at least one replica URL for %s", fs.Arg(0))
 	} else if fs.NArg() > 1 {
-		if *configPath != "" {
+		if c.configPath != "" {
 			return fmt.Errorf("cannot specify a replica URL and the -config flag")
 		}
 
 		dbConfig := &DBConfig{Path: fs.Arg(0)}
 		for _, u := range fs.Args()[1:] {
+			syncInterval := litestream.DefaultSyncInterval
 			dbConfig.Replicas = append(dbConfig.Replicas, &ReplicaConfig{
 				URL:          u,
-				SyncInterval: 1 * time.Second,
+				SyncInterval: &syncInterval,
 			})
 		}
 		c.Config.DBs = []*DBConfig{dbConfig}
 	} else {
-		if *configPath == "" {
-			*configPath = DefaultConfigPath()
+		if c.configPath == "" {
+			c.configPath = DefaultConfigPath()
 		}
-		if c.Config, err = ReadConfigFile(*configPath); err != nil {
+		if c.Config, err = ReadConfigFile(c.configPath, !c.noExpandEnv); err != nil {
 			return err
 		}
 	}
 
-	// Enable trace logging.
-	if *tracePath != "" {
-		f, err := os.Create(*tracePath)
-		if err != nil {
-			return err
+	// Override config with flags, if specified.
+	if *addr != "" {
+		c.Config.Addr = *addr
 	}
-		defer f.Close()
-		litestream.Tracef = log.New(f, "", log.LstdFlags|log.LUTC|log.Lshortfile).Printf
+	if *execFlag != "" {
+		c.Config.Exec = *execFlag
 	}
 
 	return nil
@@ -81,76 +100,106 @@ func (c *ReplicateCommand) Run(ctx context.Context) (err error) {
 	// Display version information.
 	log.Printf("litestream %s", Version)
 
+	// Setup databases.
 	if len(c.Config.DBs) == 0 {
 		log.Println("no databases specified in configuration")
 	}
 
+	c.server = litestream.NewServer()
+	if err := c.server.Open(); err != nil {
+		return fmt.Errorf("open server: %w", err)
+	}
+
+	// Add databases to the server.
 	for _, dbConfig := range c.Config.DBs {
-		db, err := NewDBFromConfig(dbConfig)
+		path, err := expand(dbConfig.Path)
 		if err != nil {
 			return err
 		}
 
-		// Open database & attach to program.
-		if err := db.Open(); err != nil {
+		if err := c.server.Watch(path, func(path string) (*litestream.DB, error) {
+			return NewDBFromConfigWithPath(dbConfig, path)
+		}); err != nil {
 			return err
 		}
-		c.DBs = append(c.DBs, db)
 	}
 
 	// Notify user that initialization is done.
-	for _, db := range c.DBs {
+	for _, db := range c.server.DBs() {
 		log.Printf("initialized db: %s", db.Path())
 		for _, r := range db.Replicas {
-			switch r := r.(type) {
-			case *litestream.FileReplica:
-				log.Printf("replicating to: name=%q type=%q path=%q", r.Name(), r.Type(), r.Path())
-			case *s3.Replica:
-				log.Printf("replicating to: name=%q type=%q bucket=%q path=%q region=%q endpoint=%q sync-interval=%s", r.Name(), r.Type(), r.Bucket, r.Path, r.Region, r.Endpoint, r.SyncInterval)
+			switch client := r.Client().(type) {
+			case *litestream.FileReplicaClient:
+				log.Printf("replicating to: name=%q type=%q path=%q", r.Name(), client.Type(), client.Path())
+			case *s3.ReplicaClient:
+				log.Printf("replicating to: name=%q type=%q bucket=%q path=%q region=%q endpoint=%q sync-interval=%s", r.Name(), client.Type(), client.Bucket, client.Path, client.Region, client.Endpoint, r.SyncInterval)
+			case *gs.ReplicaClient:
+				log.Printf("replicating to: name=%q type=%q bucket=%q path=%q sync-interval=%s", r.Name(), client.Type(), client.Bucket, client.Path, r.SyncInterval)
+			case *abs.ReplicaClient:
+				log.Printf("replicating to: name=%q type=%q bucket=%q path=%q endpoint=%q sync-interval=%s", r.Name(), client.Type(), client.Bucket, client.Path, client.Endpoint, r.SyncInterval)
+			case *sftp.ReplicaClient:
+				log.Printf("replicating to: name=%q type=%q host=%q user=%q path=%q sync-interval=%s", r.Name(), client.Type(), client.Host, client.User, client.Path, r.SyncInterval)
 			default:
-				log.Printf("replicating to: name=%q type=%q", r.Name(), r.Type())
+				log.Printf("replicating to: name=%q type=%q", r.Name(), client.Type())
 			}
 		}
 	}
 
-	// Serve metrics over HTTP if enabled.
+	// Serve HTTP if enabled.
 	if c.Config.Addr != "" {
-		hostport := c.Config.Addr
-		if host, port, _ := net.SplitHostPort(c.Config.Addr); port == "" {
-			return fmt.Errorf("must specify port for bind address: %q", c.Config.Addr)
-		} else if host == "" {
-			hostport = net.JoinHostPort("localhost", port)
+		c.httpServer = http.NewServer(c.server, c.Config.Addr)
+		if err := c.httpServer.Open(); err != nil {
+			return fmt.Errorf("cannot start http server: %w", err)
+		}
+		log.Printf("http server running at %s", c.httpServer.URL())
 	}
 
-		log.Printf("serving metrics on http://%s/metrics", hostport)
-		go func() {
-			http.Handle("/metrics", promhttp.Handler())
-			if err := http.ListenAndServe(c.Config.Addr, nil); err != nil {
-				log.Printf("cannot start metrics server: %s", err)
+	// Parse exec commands args & start subprocess.
+	if c.Config.Exec != "" {
+		execArgs, err := shellwords.Parse(c.Config.Exec)
+		if err != nil {
+			return fmt.Errorf("cannot parse exec command: %w", err)
 		}
-		}()
+
+		// Pass first database path to child process.
+		env := os.Environ()
+		if dbs := c.server.DBs(); len(dbs) > 0 {
+			env = append(env, fmt.Sprintf("LITESTREAM_DB_PATH=%s", dbs[0].Path()))
 		}
 
+		c.cmd = exec.CommandContext(ctx, execArgs[0], execArgs[1:]...)
+		c.cmd.Env = env
+		c.cmd.Stdout = os.Stdout
+		c.cmd.Stderr = os.Stderr
+		if err := c.cmd.Start(); err != nil {
+			return fmt.Errorf("cannot start exec command: %w", err)
+		}
+		go func() { c.execCh <- c.cmd.Wait() }()
+	}
+
+	log.Printf("litestream initialization complete")
+
 	return nil
 }
 
-// Close closes all open databases.
+// Close closes the HTTP server & all open databases.
 func (c *ReplicateCommand) Close() (err error) {
-	for _, db := range c.DBs {
-		if e := db.SoftClose(); e != nil {
-			log.Printf("error closing db: path=%s err=%s", db.Path(), e)
-			if err == nil {
+	if c.httpServer != nil {
+		if e := c.httpServer.Close(); e != nil && err == nil {
 			err = e
 		}
 	}
+	if c.server != nil {
+		if e := c.server.Close(); e != nil && err == nil {
+			err = e
+		}
 	}
-	// TODO(windows): Clear DBs
 	return err
 }
 
 // Usage prints the help screen to STDOUT.
 func (c *ReplicateCommand) Usage() {
-	fmt.Printf(`
+	fmt.Fprintf(c.stdout, `
 The replicate command starts a server to monitor & replicate databases. 
 You can specify your database & replicas in a configuration file or you can
 replicate a single database file by specifying its path and its replicas in the
@@ -168,8 +217,16 @@ Arguments:
 	    Specifies the configuration file.
 	    Defaults to %s
 
-	-trace PATH
-	    Write verbose trace logging to PATH.
+	-exec CMD
+	    Executes a subcommand. Litestream will exit when the child
+	    process exits. Useful for simple process management.
+
+	-addr BIND_ADDR
+	    Starts an HTTP server that reports prometheus metrics and provides
+	    an endpoint for live read replication. (e.g. ":9090")
+
+	-no-expand-env
+	    Disables environment variable expansion in configuration file.
 
 `[1:], DefaultConfigPath())
 }

cmd/litestream/replicate_test.go

@@ -0,0 +1,136 @@
+package main_test
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"hash/crc64"
+	"io"
+	"os"
+	"path/filepath"
+	"runtime"
+	"testing"
+	"time"
+
+	"golang.org/x/sync/errgroup"
+)
+
+func TestReplicateCommand(t *testing.T) {
+	if testing.Short() {
+		t.Skip("long running test, skipping")
+	} else if runtime.GOOS != "linux" {
+		t.Skip("must run system tests on Linux, skipping")
+	}
+
+	const writeTime = 10 * time.Second
+
+	dir := t.TempDir()
+	configPath := filepath.Join(dir, "litestream.yml")
+	dbPath := filepath.Join(dir, "db")
+	restorePath := filepath.Join(dir, "restored")
+	replicaPath := filepath.Join(dir, "replica")
+
+	if err := os.WriteFile(configPath, []byte(`
+dbs:
+  - path: `+dbPath+`
+    replicas:
+      - path: `+replicaPath+`
+`), 0666); err != nil {
+		t.Fatal(err)
+	}
+
+	// Generate data into SQLite database from separate goroutine.
+	g, ctx := errgroup.WithContext(context.Background())
+	mainctx, cancel := context.WithCancel(ctx)
+	g.Go(func() error {
+		defer cancel()
+
+		db, err := sql.Open("sqlite3", dbPath)
+		if err != nil {
+			return err
+		}
+		defer db.Close()
+
+		if _, err := db.ExecContext(ctx, `PRAGMA journal_mode = WAL`); err != nil {
+			return fmt.Errorf("cannot enable wal: %w", err)
+		} else if _, err := db.ExecContext(ctx, `PRAGMA synchronous = NORMAL`); err != nil {
+			return fmt.Errorf("cannot enable wal: %w", err)
+		} else if _, err := db.ExecContext(ctx, `CREATE TABLE t (id INTEGER PRIMARY KEY)`); err != nil {
+			return fmt.Errorf("cannot create table: %w", err)
+		}
+
+		ticker := time.NewTicker(1 * time.Millisecond)
+		defer ticker.Stop()
+		timer := time.NewTimer(writeTime)
+		defer timer.Stop()
+
+		for i := 0; ; i++ {
+			select {
+			case <-ctx.Done():
+				return ctx.Err()
+			case <-timer.C:
+				return nil
+			case <-ticker.C:
+				if _, err := db.ExecContext(ctx, `INSERT INTO t (id) VALUES (?);`, i); err != nil {
+					return fmt.Errorf("cannot insert: i=%d err=%w", i, err)
+				}
+			}
+		}
+	})
+
+	// Replicate database unless the context is canceled.
+	g.Go(func() error {
+		m, _, _, _ := newMain()
+		return m.Run(mainctx, []string{"replicate", "-config", configPath})
+	})
+
+	if err := g.Wait(); err != nil {
+		t.Fatal(err)
+	}
+
+	// Checkpoint database.
+	mustCheckpoint(t, dbPath)
+	chksum0 := mustChecksum(t, dbPath)
+
+	// Restore to another path.
+	m, _, _, _ := newMain()
+	if err := m.Run(context.Background(), []string{"restore", "-config", configPath, "-o", restorePath, dbPath}); err != nil && !errors.Is(err, context.Canceled) {
+		t.Fatal(err)
+	}
+
+	// Verify contents match.
+	if chksum1 := mustChecksum(t, restorePath); chksum0 != chksum1 {
+		t.Fatal("restore mismatch")
+	}
+}
+
+func mustCheckpoint(tb testing.TB, path string) {
+	tb.Helper()
+
+	db, err := sql.Open("sqlite3", path)
+	if err != nil {
+		tb.Fatal(err)
+	}
+	defer db.Close()
+
+	if _, err := db.Exec(`PRAGMA wal_checkpoint(TRUNCATE)`); err != nil {
+		tb.Fatal(err)
+	}
+}
+
+func mustChecksum(tb testing.TB, path string) uint64 {
+	tb.Helper()
+
+	f, err := os.Open(path)
+	if err != nil {
+		tb.Fatal(err)
+	}
+	defer f.Close()
+
+	h := crc64.New(crc64.MakeTable(crc64.ISO))
+	if _, err := io.Copy(h, f); err != nil {
+		tb.Fatal(err)
+	}
+	return h.Sum64()
+}

cmd/litestream/restore.go

@@ -2,33 +2,63 @@ package main
 
 import (
 	"context"
-	"errors"
 	"flag"
 	"fmt"
+	"io"
 	"log"
 	"os"
+	"path/filepath"
+	"strconv"
 	"time"
 
 	"github.com/benbjohnson/litestream"
 )
 
 // RestoreCommand represents a command to restore a database from a backup.
-type RestoreCommand struct{}
+type RestoreCommand struct {
+	stdin  io.Reader
+	stdout io.Writer
+	stderr io.Writer
+
+	snapshotIndex int // index of snapshot to start from
+
+	// CLI options
+	configPath      string    // path to config file
+	noExpandEnv     bool      // if true, do not expand env variables in config
+	outputPath      string    // path to restore database to
+	replicaName     string    // optional, name of replica to restore from
+	generation      string    // optional, generation to restore
+	targetIndex     int       // optional, last WAL index to replay
+	timestamp       time.Time // optional, restore to point-in-time (ISO 8601)
+	ifDBNotExists   bool      // if true, skips restore if output path already exists
+	ifReplicaExists bool      // if true, skips if no backups exist
+	opt             litestream.RestoreOptions
+}
+
+// NewRestoreCommand returns a new instance of RestoreCommand.
+func NewRestoreCommand(stdin io.Reader, stdout, stderr io.Writer) *RestoreCommand {
+	return &RestoreCommand{
+		stdin:  stdin,
+		stdout: stdout,
+		stderr: stderr,
+
+		targetIndex: -1,
+		opt:         litestream.NewRestoreOptions(),
+	}
+}
 
 // Run executes the command.
 func (c *RestoreCommand) Run(ctx context.Context, args []string) (err error) {
-	opt := litestream.NewRestoreOptions()
-	opt.Verbose = true
-
 	fs := flag.NewFlagSet("litestream-restore", flag.ContinueOnError)
-	configPath := registerConfigFlag(fs)
-	fs.StringVar(&opt.OutputPath, "o", "", "output path")
-	fs.StringVar(&opt.ReplicaName, "replica", "", "replica name")
-	fs.StringVar(&opt.Generation, "generation", "", "generation name")
-	fs.IntVar(&opt.Index, "index", opt.Index, "wal index")
-	fs.BoolVar(&opt.DryRun, "dry-run", false, "dry run")
-	timestampStr := fs.String("timestamp", "", "timestamp")
-	verbose := fs.Bool("v", false, "verbose output")
+	registerConfigFlag(fs, &c.configPath, &c.noExpandEnv)
+	fs.StringVar(&c.outputPath, "o", "", "output path")
+	fs.StringVar(&c.replicaName, "replica", "", "replica name")
+	fs.StringVar(&c.generation, "generation", "", "generation name")
+	fs.Var((*indexVar)(&c.targetIndex), "index", "wal index")
+	timestampStr := fs.String("timestamp", "", "point-in-time restore (ISO 8601)")
+	fs.IntVar(&c.opt.Parallelism, "parallelism", c.opt.Parallelism, "parallelism")
+	fs.BoolVar(&c.ifDBNotExists, "if-db-not-exists", false, "")
+	fs.BoolVar(&c.ifReplicaExists, "if-replica-exists", false, "")
 	fs.Usage = c.Usage
 	if err := fs.Parse(args); err != nil {
 		return err
@@ -37,68 +67,122 @@ func (c *RestoreCommand) Run(ctx context.Context, args []string) (err error) {
 	} else if fs.NArg() > 1 {
 		return fmt.Errorf("too many arguments")
 	}
+	pathOrURL := fs.Arg(0)
 
-	// Parse timestamp, if specified.
+	// Parse timestamp.
 	if *timestampStr != "" {
-		if opt.Timestamp, err = time.Parse(time.RFC3339, *timestampStr); err != nil {
-			return errors.New("invalid -timestamp, must specify in ISO 8601 format (e.g. 2000-01-01T00:00:00Z)")
+		if c.timestamp, err = time.Parse(time.RFC3339Nano, *timestampStr); err != nil {
+			return fmt.Errorf("invalid -timestamp, expected ISO 8601: %w", err)
 		}
 	}
 
-	// Verbose output is automatically enabled if dry run is specified.
-	if opt.DryRun {
-		*verbose = true
+	// Ensure a generation is specified if target index is specified.
+	if c.targetIndex != -1 && !c.timestamp.IsZero() {
+		return fmt.Errorf("cannot specify both -index flag and -timestamp flag")
+	} else if c.targetIndex != -1 && c.generation == "" {
+		return fmt.Errorf("must specify -generation flag when using -index flag")
+	} else if !c.timestamp.IsZero() && c.generation == "" {
+		return fmt.Errorf("must specify -generation flag when using -timestamp flag")
 	}
 
-	// Instantiate logger if verbose output is enabled.
-	if *verbose {
-		opt.Logger = log.New(os.Stderr, "", log.LstdFlags)
+	// Default to original database path if output path not specified.
+	if !isURL(pathOrURL) && c.outputPath == "" {
+		c.outputPath = pathOrURL
 	}
 
-	// Determine replica & generation to restore from.
-	var r litestream.Replica
-	if isURL(fs.Arg(0)) {
-		if *configPath != "" {
-			return fmt.Errorf("cannot specify a replica URL and the -config flag")
-		}
-		if r, err = c.loadFromURL(ctx, fs.Arg(0), &opt); err != nil {
-			return err
-		}
-	} else {
-		if *configPath == "" {
-			*configPath = DefaultConfigPath()
-		}
-		if r, err = c.loadFromConfig(ctx, fs.Arg(0), *configPath, &opt); err != nil {
+	// Exit successfully if the output file already exists and flag is set.
+	if _, err := os.Stat(c.outputPath); os.IsNotExist(err) {
+		// file doesn't exist, continue
+	} else if err != nil {
 		return err
+	} else if err == nil {
+		if c.ifDBNotExists {
+			fmt.Fprintln(c.stdout, "database already exists, skipping")
+			return nil
 		}
+		return fmt.Errorf("output file already exists: %s", c.outputPath)
 	}
 
-	// Return an error if no matching targets found.
-	if opt.Generation == "" {
-		return fmt.Errorf("no matching backups found")
-	}
-
-	return litestream.RestoreReplica(ctx, r, opt)
-}
-
-// loadFromURL creates a replica & updates the restore options from a replica URL.
-func (c *RestoreCommand) loadFromURL(ctx context.Context, replicaURL string, opt *litestream.RestoreOptions) (litestream.Replica, error) {
-	r, err := NewReplicaFromConfig(&ReplicaConfig{URL: replicaURL}, nil)
-	if err != nil {
-		return nil, err
-	}
-	opt.Generation, _, err = litestream.CalcReplicaRestoreTarget(ctx, r, *opt)
-	return r, err
-}
-
-// loadFromConfig returns a replica & updates the restore options from a DB reference.
-func (c *RestoreCommand) loadFromConfig(ctx context.Context, dbPath, configPath string, opt *litestream.RestoreOptions) (litestream.Replica, error) {
 	// Load configuration.
-	config, err := ReadConfigFile(configPath)
+	config, err := ReadConfigFile(c.configPath, !c.noExpandEnv)
 	if err != nil {
-		return nil, err
+		return err
 	}
 
+	// Build replica from either a URL or config.
+	r, err := c.loadReplica(ctx, config, pathOrURL)
+	if err != nil {
+		return err
+	}
+
+	// Determine latest generation if one is not specified.
+	if c.generation == "" {
+		if c.generation, err = litestream.FindLatestGeneration(ctx, r.Client()); err == litestream.ErrNoGeneration {
+			// Return an error if no matching targets found.
+			// If optional flag set, return success. Useful for automated recovery.
+			if c.ifReplicaExists {
+				fmt.Fprintln(c.stdout, "no matching backups found, skipping")
+				return nil
+			}
+			return fmt.Errorf("no matching backups found")
+		} else if err != nil {
+			return fmt.Errorf("cannot determine latest generation: %w", err)
+		}
+	}
+
+	// Determine the maximum available index for the generation if one is not specified.
+	if !c.timestamp.IsZero() {
+		if c.targetIndex, err = litestream.FindIndexByTimestamp(ctx, r.Client(), c.generation, c.timestamp); err != nil {
+			return fmt.Errorf("cannot find index for timestamp in generation %q: %w", c.generation, err)
+		}
+	} else if c.targetIndex == -1 {
+		if c.targetIndex, err = litestream.FindMaxIndexByGeneration(ctx, r.Client(), c.generation); err != nil {
+			return fmt.Errorf("cannot determine latest index in generation %q: %w", c.generation, err)
+		}
+	}
+
+	// Find lastest snapshot that occurs before the index.
+	// TODO: Optionally allow -snapshot-index
+	if c.snapshotIndex, err = litestream.FindSnapshotForIndex(ctx, r.Client(), c.generation, c.targetIndex); err != nil {
+		return fmt.Errorf("cannot find snapshot index: %w", err)
+	}
+
+	// Create parent directory if it doesn't already exist.
+	if err := os.MkdirAll(filepath.Dir(c.outputPath), 0700); err != nil {
+		return fmt.Errorf("cannot create parent directory: %w", err)
+	}
+
+	c.opt.Logger = log.New(c.stdout, "", log.LstdFlags|log.Lmicroseconds)
+
+	return litestream.Restore(ctx, r.Client(), c.outputPath, c.generation, c.snapshotIndex, c.targetIndex, c.opt)
+}
+
+func (c *RestoreCommand) loadReplica(ctx context.Context, config Config, arg string) (*litestream.Replica, error) {
+	if isURL(arg) {
+		return c.loadReplicaFromURL(ctx, config, arg)
+	}
+	return c.loadReplicaFromConfig(ctx, config, arg)
+}
+
+// loadReplicaFromURL creates a replica & updates the restore options from a replica URL.
+func (c *RestoreCommand) loadReplicaFromURL(ctx context.Context, config Config, replicaURL string) (*litestream.Replica, error) {
+	if c.replicaName != "" {
+		return nil, fmt.Errorf("cannot specify both the replica URL and the -replica flag")
+	} else if c.outputPath == "" {
+		return nil, fmt.Errorf("output path required when using a replica URL")
+	}
+
+	syncInterval := litestream.DefaultSyncInterval
+	return NewReplicaFromConfig(&ReplicaConfig{
+		URL:             replicaURL,
+		AccessKeyID:     config.AccessKeyID,
+		SecretAccessKey: config.SecretAccessKey,
+		SyncInterval:    &syncInterval,
+	}, nil)
+}
+
+// loadReplicaFromConfig returns replicas based on the specific config path.
+func (c *RestoreCommand) loadReplicaFromConfig(ctx context.Context, config Config, dbPath string) (_ *litestream.Replica, err error) {
 	// Lookup database from configuration file by path.
 	if dbPath, err = expand(dbPath); err != nil {
 		return nil, err
@@ -110,26 +194,40 @@ func (c *RestoreCommand) loadFromConfig(ctx context.Context, dbPath, configPath
 	db, err := NewDBFromConfig(dbConfig)
 	if err != nil {
 		return nil, err
+	} else if len(db.Replicas) == 0 {
+		return nil, fmt.Errorf("database has no replicas: %s", dbPath)
 	}
 
-	// Restore into original database path if not specified.
-	if opt.OutputPath == "" {
-		opt.OutputPath = dbPath
+	// Filter by replica name if specified.
+	if c.replicaName != "" {
+		r := db.Replica(c.replicaName)
+		if r == nil {
+			return nil, fmt.Errorf("replica %q not found", c.replicaName)
+		}
+		return r, nil
 	}
 
-	// Determine the appropriate replica & generation to restore from,
-	r, generation, err := db.CalcRestoreTarget(ctx, *opt)
+	// Choose only replica if only one available and no name is specified.
+	if len(db.Replicas) == 1 {
+		return db.Replicas[0], nil
+	}
+
+	// A replica must be specified when restoring a specific generation with multiple replicas.
+	if c.generation != "" {
+		return nil, fmt.Errorf("must specify -replica flag when restoring from a specific generation")
+	}
+
+	// Determine latest replica to restore from.
+	r, err := litestream.LatestReplica(ctx, db.Replicas)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("cannot determine latest replica: %w", err)
 	}
-	opt.Generation = generation
-
 	return r, nil
 }
 
 // Usage prints the help screen to STDOUT.
 func (c *RestoreCommand) Usage() {
-	fmt.Printf(`
+	fmt.Fprintf(c.stdout, `
 The restore command recovers a database from a previous snapshot and WAL.
 
 Usage:
@@ -144,6 +242,9 @@ Arguments:
 	    Specifies the configuration file.
 	    Defaults to %s
 
+	-no-expand-env
+	    Disables environment variable expansion in configuration file.
+
 	-replica NAME
 	    Restore from a specific replica.
 	    Defaults to replica with latest data.
@@ -153,23 +254,26 @@ Arguments:
 	    Defaults to generation with latest data.
 
 	-index NUM
-	    Restore up to a specific WAL index (inclusive).
+	    Restore up to a specific hex-encoded WAL index (inclusive).
 	    Defaults to use the highest available index.
 
-	-timestamp TIMESTAMP
-	    Restore to a specific point-in-time.
-	    Defaults to use the latest available backup.
+	-timestamp DATETIME
+	    Restore up to a specific point-in-time. Must be ISO 8601.
+	    Cannot be specified with -index flag.
 
 	-o PATH
 	    Output path of the restored database.
 	    Defaults to original DB path.
 
-	-dry-run
-	    Prints all log output as if it were running but does
-	    not perform actual restore.
+	-if-db-not-exists
+	    Returns exit code of 0 if the database already exists.
 
-	-v
-	    Verbose output.
+	-if-replica-exists
+	    Returns exit code of 0 if no backups found.
+
+	-parallelism NUM
+	    Determines the number of WAL files downloaded in parallel.
+	    Defaults to `+strconv.Itoa(litestream.DefaultRestoreParallelism)+`.
 
 
 Examples:
@@ -177,9 +281,6 @@ Examples:
 	# Restore latest replica for database to original location.
 	$ litestream restore /path/to/db
 
-	# Restore replica for database to a given point in time.
-	$ litestream restore -timestamp 2020-01-01T00:00:00Z /path/to/db
-
 	# Restore latest replica for database to new /tmp directory
 	$ litestream restore -o /tmp/db /path/to/db
 
@@ -189,6 +290,9 @@ Examples:
 	# Restore database from specific generation on S3.
 	$ litestream restore -replica s3 -generation xxxxxxxx /path/to/db
 
+	# Restore database to a specific point in time.
+	$ litestream restore -generation xxxxxxxx -timestamp 2000-01-01T00:00:00Z /path/to/db
+
 `[1:],
 		DefaultConfigPath(),
 	)

cmd/litestream/restore_test.go

@@ -0,0 +1,330 @@
+package main_test
+
+import (
+	"context"
+	"flag"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/benbjohnson/litestream/internal/testingutil"
+)
+
+func TestRestoreCommand(t *testing.T) {
+	t.Run("OK", func(t *testing.T) {
+		testDir := filepath.Join("testdata", "restore", "ok")
+		defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
+		tempDir := t.TempDir()
+
+		m, _, stdout, stderr := newMain()
+		if err := m.Run(context.Background(), []string{"restore", "-config", filepath.Join(testDir, "litestream.yml"), "-o", filepath.Join(tempDir, "db"), filepath.Join(testDir, "db")}); err != nil {
+			t.Fatal(err)
+		} else if got, want := stderr.String(), ""; got != want {
+			t.Fatalf("stderr=%q, want %q", got, want)
+		}
+
+		// STDOUT has timing info so we need to grep per line.
+		lines := strings.Split(stdout.String(), "\n")
+		for i, substr := range []string{
+			`restoring snapshot 0000000000000000/0000000000000000 to ` + filepath.Join(tempDir, "db.tmp"),
+			`applied wal 0000000000000000/0000000000000000 elapsed=`,
+			`applied wal 0000000000000000/0000000000000001 elapsed=`,
+			`applied wal 0000000000000000/0000000000000002 elapsed=`,
+			`renaming database from temporary location`,
+		} {
+			if !strings.Contains(lines[i], substr) {
+				t.Fatalf("stdout: unexpected line %d:\n%s", i+1, stdout)
+			}
+		}
+	})
+
+	t.Run("ReplicaName", func(t *testing.T) {
+		testDir := filepath.Join("testdata", "restore", "replica-name")
+		defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
+		tempDir := t.TempDir()
+
+		m, _, stdout, stderr := newMain()
+		if err := m.Run(context.Background(), []string{"restore", "-config", filepath.Join(testDir, "litestream.yml"), "-o", filepath.Join(tempDir, "db"), "-replica", "replica1", filepath.Join(testDir, "db")}); err != nil {
+			t.Fatal(err)
+		} else if got, want := stderr.String(), ""; got != want {
+			t.Fatalf("stderr=%q, want %q", got, want)
+		}
+
+		// STDOUT has timing info so we need to grep per line.
+		lines := strings.Split(stdout.String(), "\n")
+		for i, substr := range []string{
+			`restoring snapshot 0000000000000001/0000000000000001 to ` + filepath.Join(tempDir, "db.tmp"),
+			`no wal files found, snapshot only`,
+			`renaming database from temporary location`,
+		} {
+			if !strings.Contains(lines[i], substr) {
+				t.Fatalf("stdout: unexpected line %d:\n%s", i+1, stdout)
+			}
+		}
+	})
+
+	t.Run("ReplicaURL", func(t *testing.T) {
+		testDir := filepath.Join(testingutil.Getwd(t), "testdata", "restore", "replica-url")
+		tempDir := t.TempDir()
+		replicaURL := "file://" + filepath.ToSlash(testDir) + "/replica"
+
+		m, _, stdout, stderr := newMain()
+		if err := m.Run(context.Background(), []string{"restore", "-o", filepath.Join(tempDir, "db"), replicaURL}); err != nil {
+			t.Fatal(err)
+		} else if got, want := stderr.String(), ""; got != want {
+			t.Fatalf("stderr=%q, want %q", got, want)
+		}
+
+		lines := strings.Split(stdout.String(), "\n")
+		for i, substr := range []string{
+			`restoring snapshot 0000000000000000/0000000000000000 to ` + filepath.Join(tempDir, "db.tmp"),
+			`no wal files found, snapshot only`,
+			`renaming database from temporary location`,
+		} {
+			if !strings.Contains(lines[i], substr) {
+				t.Fatalf("stdout: unexpected line %d:\n%s", i+1, stdout)
+			}
+		}
+	})
+
+	t.Run("LatestReplica", func(t *testing.T) {
+		testDir := filepath.Join("testdata", "restore", "latest-replica")
+		defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
+		tempDir := t.TempDir()
+
+		m, _, stdout, stderr := newMain()
+		if err := m.Run(context.Background(), []string{"restore", "-config", filepath.Join(testDir, "litestream.yml"), "-o", filepath.Join(tempDir, "db"), filepath.Join(testDir, "db")}); err != nil {
+			t.Fatal(err)
+		} else if got, want := stderr.String(), ""; got != want {
+			t.Fatalf("stderr=%q, want %q", got, want)
+		}
+
+		lines := strings.Split(stdout.String(), "\n")
+		for i, substr := range []string{
+			`restoring snapshot 0000000000000001/0000000000000000 to ` + filepath.Join(tempDir, "db.tmp"),
+			`no wal files found, snapshot only`,
+			`renaming database from temporary location`,
+		} {
+			if !strings.Contains(lines[i], substr) {
+				t.Fatalf("stdout: unexpected line %d:\n%s", i+1, stdout)
+			}
+		}
+	})
+
+	t.Run("IfDBNotExistsFlag", func(t *testing.T) {
+		testDir := filepath.Join("testdata", "restore", "if-db-not-exists-flag")
+		defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
+
+		m, _, stdout, _ := newMain()
+		err := m.Run(context.Background(), []string{"restore", "-config", filepath.Join(testDir, "litestream.yml"), "-if-db-not-exists", filepath.Join(testDir, "db")})
+		if err != nil {
+			t.Fatal(err)
+		} else if got, want := stdout.String(), string(testingutil.ReadFile(t, filepath.Join(testDir, "stdout"))); got != want {
+			t.Fatalf("stdout=%q, want %q", got, want)
+		}
+	})
+
+	t.Run("IfReplicaExists", func(t *testing.T) {
+		testDir := filepath.Join("testdata", "restore", "if-replica-exists-flag")
+		defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
+
+		m, _, stdout, _ := newMain()
+		err := m.Run(context.Background(), []string{"restore", "-config", filepath.Join(testDir, "litestream.yml"), "-if-replica-exists", filepath.Join(testDir, "db")})
+		if err != nil {
+			t.Fatal(err)
+		} else if got, want := stdout.String(), string(testingutil.ReadFile(t, filepath.Join(testDir, "stdout"))); got != want {
+			t.Fatalf("stdout=%q, want %q", got, want)
+		}
+	})
+
+	t.Run("ErrNoBackups", func(t *testing.T) {
+		testDir := filepath.Join("testdata", "restore", "no-backups")
+		defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
+		tempDir := t.TempDir()
+
+		m, _, stdout, stderr := newMain()
+		err := m.Run(context.Background(), []string{"restore", "-config", filepath.Join(testDir, "litestream.yml"), "-o", filepath.Join(tempDir, "db"), filepath.Join(testDir, "db")})
+		if err == nil || err.Error() != `no matching backups found` {
+			t.Fatalf("unexpected error: %s", err)
+		} else if got, want := stdout.String(), string(testingutil.ReadFile(t, filepath.Join(testDir, "stdout"))); got != want {
+			t.Fatalf("stdout=%q, want %q", got, want)
+		} else if got, want := stderr.String(), string(testingutil.ReadFile(t, filepath.Join(testDir, "stderr"))); got != want {
+			t.Fatalf("stderr=%q, want %q", got, want)
+		}
+	})
+
+	t.Run("ErrNoGeneration", func(t *testing.T) {
+		testDir := filepath.Join("testdata", "restore", "no-generation")
+		defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
+
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"restore", "-config", filepath.Join(testDir, "litestream.yml"), filepath.Join(testDir, "db")})
+		if err == nil || err.Error() != `no matching backups found` {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrOutputPathExists", func(t *testing.T) {
+		testDir := filepath.Join("testdata", "restore", "output-path-exists")
+		defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
+
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"restore", "-config", filepath.Join(testDir, "litestream.yml"), filepath.Join(testDir, "db")})
+		if err == nil || err.Error() != `output file already exists: `+filepath.Join(testDir, "db") {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrDatabaseOrReplicaRequired", func(t *testing.T) {
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"restore"})
+		if err == nil || err.Error() != `database path or replica URL required` {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrTooManyArguments", func(t *testing.T) {
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"restore", "abc", "123"})
+		if err == nil || err.Error() != `too many arguments` {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrInvalidFlags", func(t *testing.T) {
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"restore", "-no-such-flag"})
+		if err == nil || err.Error() != `flag provided but not defined: -no-such-flag` {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrIndexFlagOnly", func(t *testing.T) {
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"restore", "-index", "0", "/var/lib/db"})
+		if err == nil || err.Error() != `must specify -generation flag when using -index flag` {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrConfigFileNotFound", func(t *testing.T) {
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"restore", "-config", "/no/such/file", "/var/lib/db"})
+		if err == nil || err.Error() != `config file not found: /no/such/file` {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrInvalidConfig", func(t *testing.T) {
+		testDir := filepath.Join("testdata", "restore", "invalid-config")
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"restore", "-config", filepath.Join(testDir, "litestream.yml"), "/var/lib/db"})
+		if err == nil || !strings.Contains(err.Error(), `replica path cannot be a url`) {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrMkdir", func(t *testing.T) {
+		tempDir := t.TempDir()
+		if err := os.Mkdir(filepath.Join(tempDir, "noperm"), 0000); err != nil {
+			t.Fatal(err)
+		}
+
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"restore", "-o", filepath.Join(tempDir, "noperm", "subdir", "db"), "/var/lib/db"})
+		if err == nil || !strings.Contains(err.Error(), `permission denied`) {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrNoOutputPathWithReplicaURL", func(t *testing.T) {
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"restore", "file://path/to/replica"})
+		if err == nil || err.Error() != `output path required when using a replica URL` {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrReplicaNameWithReplicaURL", func(t *testing.T) {
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"restore", "-replica", "replica0", "file://path/to/replica"})
+		if err == nil || err.Error() != `cannot specify both the replica URL and the -replica flag` {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrInvalidReplicaURL", func(t *testing.T) {
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"restore", "-o", filepath.Join(t.TempDir(), "db"), "xyz://xyz"})
+		if err == nil || err.Error() != `unknown replica type in config: "xyz"` {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrDatabaseNotFound", func(t *testing.T) {
+		testDir := filepath.Join("testdata", "restore", "database-not-found")
+		defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
+
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"restore", "-config", filepath.Join(testDir, "litestream.yml"), "/no/such/db"})
+		if err == nil || err.Error() != `database not found in config: /no/such/db` {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrNoReplicas", func(t *testing.T) {
+		testDir := filepath.Join("testdata", "restore", "no-replicas")
+		defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
+		tempDir := t.TempDir()
+
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"restore", "-config", filepath.Join(testDir, "litestream.yml"), "-o", filepath.Join(tempDir, "db"), filepath.Join(testDir, "db")})
+		if err == nil || err.Error() != `database has no replicas: `+filepath.Join(testingutil.Getwd(t), testDir, "db") {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrReplicaNotFound", func(t *testing.T) {
+		testDir := filepath.Join("testdata", "restore", "replica-not-found")
+		defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
+		tempDir := t.TempDir()
+
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"restore", "-config", filepath.Join(testDir, "litestream.yml"), "-o", filepath.Join(tempDir, "db"), "-replica", "no_such_replica", filepath.Join(testDir, "db")})
+		if err == nil || err.Error() != `replica "no_such_replica" not found` {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrGenerationWithNoReplicaName", func(t *testing.T) {
+		testDir := filepath.Join("testdata", "restore", "generation-with-no-replica")
+		defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
+		tempDir := t.TempDir()
+
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"restore", "-config", filepath.Join(testDir, "litestream.yml"), "-o", filepath.Join(tempDir, "db"), "-generation", "0000000000000000", filepath.Join(testDir, "db")})
+		if err == nil || err.Error() != `must specify -replica flag when restoring from a specific generation` {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrNoSnapshotsAvailable", func(t *testing.T) {
+		testDir := filepath.Join("testdata", "restore", "no-snapshots")
+		defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
+		tempDir := t.TempDir()
+
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"restore", "-config", filepath.Join(testDir, "litestream.yml"), "-o", filepath.Join(tempDir, "db"), "-generation", "0000000000000000", filepath.Join(testDir, "db")})
+		if err == nil || err.Error() != `cannot determine latest index in generation "0000000000000000": no snapshots available` {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("Usage", func(t *testing.T) {
+		m, _, _, _ := newMain()
+		if err := m.Run(context.Background(), []string{"restore", "-h"}); err != flag.ErrHelp {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+}

cmd/litestream/snapshots.go

@@ -4,7 +4,9 @@ import (
 	"context"
 	"flag"
 	"fmt"
-	"os"
+	"io"
+	"log"
+	"sort"
 	"text/tabwriter"
 	"time"
 
@@ -12,92 +14,90 @@ import (
 )
 
 // SnapshotsCommand represents a command to list snapshots for a command.
-type SnapshotsCommand struct{}
+type SnapshotsCommand struct {
+	stdin  io.Reader
+	stdout io.Writer
+	stderr io.Writer
+
+	configPath  string
+	noExpandEnv bool
+
+	replicaName string
+}
+
+// NewSnapshotsCommand returns a new instance of SnapshotsCommand.
+func NewSnapshotsCommand(stdin io.Reader, stdout, stderr io.Writer) *SnapshotsCommand {
+	return &SnapshotsCommand{
+		stdin:  stdin,
+		stdout: stdout,
+		stderr: stderr,
+	}
+}
 
 // Run executes the command.
-func (c *SnapshotsCommand) Run(ctx context.Context, args []string) (err error) {
+func (c *SnapshotsCommand) Run(ctx context.Context, args []string) (ret error) {
 	fs := flag.NewFlagSet("litestream-snapshots", flag.ContinueOnError)
-	configPath := registerConfigFlag(fs)
-	replicaName := fs.String("replica", "", "replica name")
+	registerConfigFlag(fs, &c.configPath, &c.noExpandEnv)
+	fs.StringVar(&c.replicaName, "replica", "", "replica name")
 	fs.Usage = c.Usage
 	if err := fs.Parse(args); err != nil {
 		return err
 	} else if fs.NArg() == 0 || fs.Arg(0) == "" {
-		return fmt.Errorf("database path required")
+		return fmt.Errorf("database path or replica URL required")
 	} else if fs.NArg() > 1 {
 		return fmt.Errorf("too many arguments")
 	}
 
-	var db *litestream.DB
-	var r litestream.Replica
-	if isURL(fs.Arg(0)) {
-		if *configPath != "" {
-			return fmt.Errorf("cannot specify a replica URL and the -config flag")
-		}
-		if r, err = NewReplicaFromConfig(&ReplicaConfig{URL: fs.Arg(0)}, nil); err != nil {
-			return err
-		}
-	} else {
-		if *configPath == "" {
-			*configPath = DefaultConfigPath()
-		}
-
 	// Load configuration.
-		config, err := ReadConfigFile(*configPath)
+	config, err := ReadConfigFile(c.configPath, !c.noExpandEnv)
 	if err != nil {
 		return err
 	}
 
-		// Lookup database from configuration file by path.
-		if path, err := expand(fs.Arg(0)); err != nil {
-			return err
-		} else if dbc := config.DBConfig(path); dbc == nil {
-			return fmt.Errorf("database not found in config: %s", path)
-		} else if db, err = NewDBFromConfig(dbc); err != nil {
+	// Determine list of replicas to pull snapshots from.
+	replicas, _, err := loadReplicas(ctx, config, fs.Arg(0), c.replicaName)
+	if err != nil {
 		return err
 	}
 
-		// Filter by replica, if specified.
-		if *replicaName != "" {
-			if r = db.Replica(*replicaName); r == nil {
-				return fmt.Errorf("replica %q not found for database %q", *replicaName, db.Path())
+	// Build list of snapshot metadata with associated replica.
+	var infos []replicaSnapshotInfo
+	for _, r := range replicas {
+		a, err := r.Snapshots(ctx)
+		if err != nil {
+			log.Printf("cannot determine snapshots: %s", err)
+			ret = errExit // signal error return without printing message
+			continue
 		}
+		for i := range a {
+			infos = append(infos, replicaSnapshotInfo{SnapshotInfo: a[i], replicaName: r.Name()})
 		}
 	}
 
-	// Find snapshots by db or replica.
-	var infos []*litestream.SnapshotInfo
-	if r != nil {
-		if infos, err = r.Snapshots(ctx); err != nil {
-			return err
-		}
-	} else {
-		if infos, err = db.Snapshots(ctx); err != nil {
-			return err
-		}
-	}
+	// Sort snapshots by creation time from newest to oldest.
+	sort.Slice(infos, func(i, j int) bool { return infos[i].CreatedAt.After(infos[j].CreatedAt) })
 
 	// List all snapshots.
-	w := tabwriter.NewWriter(os.Stdout, 0, 8, 2, ' ', 0)
+	w := tabwriter.NewWriter(c.stdout, 0, 8, 2, ' ', 0)
 	defer w.Flush()
 
 	fmt.Fprintln(w, "replica\tgeneration\tindex\tsize\tcreated")
 	for _, info := range infos {
-		fmt.Fprintf(w, "%s\t%s\t%d\t%d\t%s\n",
-			info.Replica,
+		fmt.Fprintf(w, "%s\t%s\t%s\t%d\t%s\n",
+			info.replicaName,
 			info.Generation,
-			info.Index,
+			litestream.FormatIndex(info.Index),
 			info.Size,
 			info.CreatedAt.Format(time.RFC3339),
 		)
 	}
 
-	return nil
+	return ret
 }
 
 // Usage prints the help screen to STDOUT.
 func (c *SnapshotsCommand) Usage() {
-	fmt.Printf(`
+	fmt.Fprintf(c.stdout, `
 The snapshots command lists all snapshots available for a database or replica.
 
 Usage:
@@ -112,6 +112,9 @@ Arguments:
 	    Specifies the configuration file.
 	    Defaults to %s
 
+	-no-expand-env
+	    Disables environment variable expansion in configuration file.
+
 	-replica NAME
 	    Optional, filter by a specific replica.
 
@@ -130,3 +133,9 @@ Examples:
 		DefaultConfigPath(),
 	)
 }
+
+// replicaSnapshotInfo represents snapshot metadata with associated replica name.
+type replicaSnapshotInfo struct {
+	litestream.SnapshotInfo
+	replicaName string
+}

cmd/litestream/snapshots_test.go

@@ -0,0 +1,128 @@
+package main_test
+
+import (
+	"context"
+	"flag"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/benbjohnson/litestream/internal/testingutil"
+)
+
+func TestSnapshotsCommand(t *testing.T) {
+	t.Run("OK", func(t *testing.T) {
+		testDir := filepath.Join("testdata", "snapshots", "ok")
+		defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
+
+		m, _, stdout, _ := newMain()
+		if err := m.Run(context.Background(), []string{"snapshots", "-config", filepath.Join(testDir, "litestream.yml"), filepath.Join(testDir, "db")}); err != nil {
+			t.Fatal(err)
+		} else if got, want := stdout.String(), string(testingutil.ReadFile(t, filepath.Join(testDir, "stdout"))); got != want {
+			t.Fatalf("stdout=%q, want %q", got, want)
+		}
+	})
+
+	t.Run("ReplicaName", func(t *testing.T) {
+		testDir := filepath.Join("testdata", "snapshots", "replica-name")
+		defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
+
+		m, _, stdout, _ := newMain()
+		if err := m.Run(context.Background(), []string{"snapshots", "-config", filepath.Join(testDir, "litestream.yml"), "-replica", "replica1", filepath.Join(testDir, "db")}); err != nil {
+			t.Fatal(err)
+		} else if got, want := stdout.String(), string(testingutil.ReadFile(t, filepath.Join(testDir, "stdout"))); got != want {
+			t.Fatalf("stdout=%q, want %q", got, want)
+		}
+	})
+
+	t.Run("ReplicaURL", func(t *testing.T) {
+		testDir := filepath.Join(testingutil.Getwd(t), "testdata", "snapshots", "replica-url")
+		defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
+		replicaURL := "file://" + filepath.ToSlash(testDir) + "/replica"
+
+		m, _, stdout, _ := newMain()
+		if err := m.Run(context.Background(), []string{"snapshots", replicaURL}); err != nil {
+			t.Fatal(err)
+		} else if got, want := stdout.String(), string(testingutil.ReadFile(t, filepath.Join(testDir, "stdout"))); got != want {
+			t.Fatalf("stdout=%q, want %q", got, want)
+		}
+	})
+
+	t.Run("ErrDatabaseOrReplicaRequired", func(t *testing.T) {
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"snapshots"})
+		if err == nil || err.Error() != `database path or replica URL required` {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrTooManyArguments", func(t *testing.T) {
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"snapshots", "abc", "123"})
+		if err == nil || err.Error() != `too many arguments` {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrInvalidFlags", func(t *testing.T) {
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"snapshots", "-no-such-flag"})
+		if err == nil || err.Error() != `flag provided but not defined: -no-such-flag` {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrConfigFileNotFound", func(t *testing.T) {
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"snapshots", "-config", "/no/such/file", "/var/lib/db"})
+		if err == nil || err.Error() != `config file not found: /no/such/file` {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrInvalidConfig", func(t *testing.T) {
+		testDir := filepath.Join("testdata", "snapshots", "invalid-config")
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"snapshots", "-config", filepath.Join(testDir, "litestream.yml"), "/var/lib/db"})
+		if err == nil || !strings.Contains(err.Error(), `replica path cannot be a url`) {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrDatabaseNotFound", func(t *testing.T) {
+		testDir := filepath.Join("testdata", "snapshots", "database-not-found")
+		defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
+
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"snapshots", "-config", filepath.Join(testDir, "litestream.yml"), "/no/such/db"})
+		if err == nil || err.Error() != `database not found in config: /no/such/db` {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrReplicaNotFound", func(t *testing.T) {
+		testDir := filepath.Join(testingutil.Getwd(t), "testdata", "snapshots", "replica-not-found")
+		defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
+
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"snapshots", "-config", filepath.Join(testDir, "litestream.yml"), "-replica", "no_such_replica", filepath.Join(testDir, "db")})
+		if err == nil || err.Error() != `replica "no_such_replica" not found for database "`+filepath.Join(testDir, "db")+`"` {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrInvalidReplicaURL", func(t *testing.T) {
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"snapshots", "xyz://xyz"})
+		if err == nil || !strings.Contains(err.Error(), `unknown replica type in config: "xyz"`) {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("Usage", func(t *testing.T) {
+		m, _, _, _ := newMain()
+		if err := m.Run(context.Background(), []string{"snapshots", "-h"}); err != flag.ErrHelp {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+}

cmd/litestream/testdata/Makefile

@@ -0,0 +1,13 @@
+.PHONY: default
+default:
+	make -C generations/ok
+	make -C generations/no-database
+	make -C generations/replica-name
+	make -C generations/replica-url
+	make -C restore/latest-replica
+	make -C snapshots/ok
+	make -C snapshots/replica-name
+	make -C snapshots/replica-url
+	make -C wal/ok
+	make -C wal/replica-name
+	make -C wal/replica-url

cmd/litestream/testdata/databases/invalid-config/litestream.yml

@@ -0,0 +1,4 @@
+dbs:
+  - path: /var/lib/db
+    replicas:
+      - path: s3://bkt/db

cmd/litestream/testdata/databases/no-databases/litestream.yml

@@ -0,0 +1 @@
+dbs:

cmd/litestream/testdata/databases/no-databases/stdout

@@ -0,0 +1 @@
+No databases found in config file.

cmd/litestream/testdata/databases/ok/litestream.yml

@@ -0,0 +1,7 @@
+dbs:
+  - path: /var/lib/db
+    replicas:
+      - path: /var/lib/replica
+      - url:  s3://mybkt/db
+
+  - path: /my/other/db

cmd/litestream/testdata/databases/ok/stdout

@@ -0,0 +1,3 @@
+path          replicas
+/var/lib/db   file,s3
+/my/other/db  

cmd/litestream/testdata/generations/database-not-found/litestream.yml

@@ -0,0 +1,2 @@
+dbs:
+  - path: $LITESTREAM_TESTDIR/db

cmd/litestream/testdata/generations/invalid-config/litestream.yml

@@ -0,0 +1,4 @@
+dbs:
+  - path: /var/lib/db
+    replicas:
+      - path: s3://bkt/db

cmd/litestream/testdata/generations/no-database/Makefile

@@ -0,0 +1,4 @@
+.PHONY: default
+default:
+	TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/snapshots/0000000000000000.snapshot.lz4
+	TZ=UTC touch -ct 200001020000 replica/generations/0000000000000001/snapshots/0000000000000000.snapshot.lz4

cmd/litestream/testdata/generations/no-database/litestream.yml

@@ -0,0 +1,4 @@
+dbs:
+  - path: $LITESTREAM_TESTDIR/db
+    replicas:
+      - path: $LITESTREAM_TESTDIR/replica

cmd/litestream/testdata/generations/no-database/stdout

@@ -0,0 +1,3 @@
+name  generation        lag  start                 end
+file  0000000000000000  -    2000-01-01T00:00:00Z  2000-01-01T00:00:00Z
+file  0000000000000001  -    2000-01-02T00:00:00Z  2000-01-02T00:00:00Z

cmd/litestream/testdata/generations/ok/Makefile

@@ -0,0 +1,9 @@
+.PHONY: default
+default:
+	TZ=UTC touch -ct 200001030000 db
+	TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/snapshots/0000000000000000.snapshot.lz4
+	TZ=UTC touch -ct 200001020000 replica/generations/0000000000000000/snapshots/0000000000000001.snapshot.lz4
+	TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/wal/0000000000000000/0000000000000000.wal.lz4
+	TZ=UTC touch -ct 200001020000 replica/generations/0000000000000000/wal/0000000000000000/0000000000000001.wal.lz4
+	TZ=UTC touch -ct 200001030000 replica/generations/0000000000000000/wal/0000000000000001/0000000000000000.wal.lz4
+	TZ=UTC touch -ct 200001010000 replica/generations/0000000000000001/snapshots/0000000000000000.snapshot.lz4

cmd/litestream/testdata/generations/ok/litestream.yml

@@ -0,0 +1,4 @@
+dbs:
+  - path: $LITESTREAM_TESTDIR/db
+    replicas:
+      - path: $LITESTREAM_TESTDIR/replica

cmd/litestream/testdata/generations/ok/stdout

@@ -0,0 +1,3 @@
+name  generation        lag      start                 end
+file  0000000000000000  0s       2000-01-01T00:00:00Z  2000-01-03T00:00:00Z
+file  0000000000000001  48h0m0s  2000-01-01T00:00:00Z  2000-01-01T00:00:00Z

cmd/litestream/testdata/generations/replica-name/Makefile

@@ -0,0 +1,5 @@
+.PHONY: default
+default:
+	TZ=UTC touch -ct 200001030000 db
+	TZ=UTC touch -ct 200001010000 replica0/generations/0000000000000000/snapshots/0000000000000000.snapshot.lz4
+	TZ=UTC touch -ct 200001020000 replica1/generations/0000000000000001/snapshots/0000000000000000.snapshot.lz4

cmd/litestream/testdata/generations/replica-name/litestream.yml

@@ -0,0 +1,7 @@
+dbs:
+  - path: $LITESTREAM_TESTDIR/db
+    replicas:
+      - name: replica0
+        path: $LITESTREAM_TESTDIR/replica0
+      - name: replica1
+        path: $LITESTREAM_TESTDIR/replica1

cmd/litestream/testdata/generations/replica-name/stdout

@@ -0,0 +1,2 @@
+name      generation        lag      start                 end
+replica1  0000000000000001  24h0m0s  2000-01-02T00:00:00Z  2000-01-02T00:00:00Z

cmd/litestream/testdata/generations/replica-not-found/litestream.yml

@@ -0,0 +1,4 @@
+dbs:
+  - path: $LITESTREAM_TESTDIR/db
+    replicas:
+      - url: s3://bkt/db

cmd/litestream/testdata/generations/replica-url/Makefile

@@ -0,0 +1,9 @@
+.PHONY: default
+default:
+	TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/snapshots/0000000000000000.snapshot.lz4
+	TZ=UTC touch -ct 200001020000 replica/generations/0000000000000000/snapshots/0000000000000001.snapshot.lz4
+	TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/wal/0000000000000000/0000000000000000.wal.lz4
+	TZ=UTC touch -ct 200001020000 replica/generations/0000000000000000/wal/0000000000000000/0000000000000001.wal.lz4
+	TZ=UTC touch -ct 200001030000 replica/generations/0000000000000000/wal/0000000000000001/0000000000000000.wal.lz4
+	TZ=UTC touch -ct 200001020000 replica/generations/0000000000000001/snapshots/0000000000000000.snapshot.lz4
+

cmd/litestream/testdata/generations/replica-url/litestream.yml

@@ -0,0 +1,4 @@
+dbs:
+  - path: $LITESTREAM_TESTDIR/db
+    replicas:
+      - path: $LITESTREAM_TESTDIR/replica

cmd/litestream/testdata/generations/replica-url/stdout

@@ -0,0 +1,3 @@
+name  generation        lag  start                 end
+file  0000000000000000  -    2000-01-01T00:00:00Z  2000-01-03T00:00:00Z
+file  0000000000000001  -    2000-01-02T00:00:00Z  2000-01-02T00:00:00Z

cmd/litestream/testdata/restore/database-not-found/litestream.yml

@@ -0,0 +1,4 @@
+dbs:
+  - path: $LITESTREAM_TESTDIR/db
+    replicas:
+      - path: $LITESTREAM_TESTDIR/replica

cmd/litestream/testdata/restore/generation-with-no-replica/litestream.yml

@@ -0,0 +1,5 @@
+dbs:
+  - path: $LITESTREAM_TESTDIR/db
+    replicas:
+      - path: $LITESTREAM_TESTDIR/replica0
+      - path: $LITESTREAM_TESTDIR/replica1

cmd/litestream/testdata/restore/if-db-not-exists-flag/litestream.yml

@@ -0,0 +1,4 @@
+dbs:
+  - path: $LITESTREAM_TESTDIR/db
+    replicas:
+      - path: $LITESTREAM_TESTDIR/replica

cmd/litestream/testdata/restore/if-db-not-exists-flag/stdout

@@ -0,0 +1 @@
+database already exists, skipping

cmd/litestream/testdata/restore/if-replica-exists-flag/litestream.yml

@@ -0,0 +1,4 @@
+dbs:
+  - path: $LITESTREAM_TESTDIR/db
+    replicas:
+      - path: $LITESTREAM_TESTDIR/replica

cmd/litestream/testdata/restore/if-replica-exists-flag/stdout

@@ -0,0 +1 @@
+no matching backups found, skipping

cmd/litestream/testdata/restore/invalid-config/litestream.yml

@@ -0,0 +1,4 @@
+dbs:
+  - path: /var/lib/db
+    replicas:
+      - path: s3://bkt/db

cmd/litestream/testdata/restore/latest-replica/Makefile

@@ -0,0 +1,6 @@
+.PHONY: default
+default:
+	TZ=UTC touch -ct 200001010000 replica0/generations/0000000000000000/snapshots/0000000000000000.snapshot.lz4
+	TZ=UTC touch -ct 200001020000 replica1/generations/0000000000000002/snapshots/0000000000000000.snapshot.lz4
+	TZ=UTC touch -ct 200001030000 replica0/generations/0000000000000001/snapshots/0000000000000000.snapshot.lz4
+

cmd/litestream/testdata/restore/latest-replica/litestream.yml

@@ -0,0 +1,7 @@
+dbs:
+  - path: $LITESTREAM_TESTDIR/db
+    replicas:
+      - name: replica0
+        path: $LITESTREAM_TESTDIR/replica0
+      - name: replica1
+        path: $LITESTREAM_TESTDIR/replica1

cmd/litestream/testdata/restore/no-backups/litestream.yml

@@ -0,0 +1,4 @@
+dbs:
+  - path: $LITESTREAM_TESTDIR/db
+    replicas:
+      - path: $LITESTREAM_TESTDIR/replica

cmd/litestream/testdata/restore/no-generation/litestream.yml

@@ -0,0 +1,4 @@
+dbs:
+  - path: $LITESTREAM_TESTDIR/db
+    replicas:
+      - path: $LITESTREAM_TESTDIR/replica

cmd/litestream/testdata/restore/no-replicas/litestream.yml

@@ -0,0 +1,2 @@
+dbs:
+  - path: $LITESTREAM_TESTDIR/db

cmd/litestream/testdata/restore/no-snapshots/litestream.yml

@@ -0,0 +1,4 @@
+dbs:
+  - path: $LITESTREAM_TESTDIR/db
+    replicas:
+      - path: $LITESTREAM_TESTDIR/replica

cmd/litestream/testdata/restore/ok/README

@@ -0,0 +1,36 @@
+To reproduce this testdata, run sqlite3 and execute:
+
+	PRAGMA journal_mode = WAL;
+	CREATE TABLE t (x);
+	INSERT INTO t (x) VALUES (1);
+	INSERT INTO t (x) VALUES (2);
+
+	sl3 split -o generations/0000000000000000/wal/0000000000000000 db-wal
+	cp db generations/0000000000000000/snapshots/0000000000000000.snapshot
+	lz4 -c --rm generations/0000000000000000/snapshots/0000000000000000.snapshot
+
+
+Then execute:
+
+	PRAGMA wal_checkpoint(TRUNCATE);
+	INSERT INTO t (x) VALUES (3);
+
+	sl3 split -o generations/0000000000000000/wal/0000000000000001 db-wal
+
+
+Then execute:
+
+	PRAGMA wal_checkpoint(TRUNCATE);
+	INSERT INTO t (x) VALUES (4);
+	INSERT INTO t (x) VALUES (5);
+
+	sl3 split -o generations/0000000000000000/wal/0000000000000002 db-wal
+
+
+Finally, obtain the final snapshot:
+
+	PRAGMA wal_checkpoint(TRUNCATE);
+
+	cp db 0000000000000002.db
+	rm db*
+

cmd/litestream/testdata/restore/ok/litestream.yml

@@ -0,0 +1,4 @@
+dbs:
+  - path: $LITESTREAM_TESTDIR/db
+    replicas:
+      - path: $LITESTREAM_TESTDIR/replica