Fix read replication stream restart position

export LITESTREAM_ACCESS_KEY_ID=your_key_id
export LITESTREAM_SECRET_ACCESS_KEY=your_access_key
export LITESTREAM_ENDPOINT=your_endpoint
export LITESTREAM_REGION=your_region
litestream replicate fruits.db s3://mybkt/fruits.db

.github/CONTRIBUTING.md

@@ -1,17 +1,18 @@
-## Open-source, not open-contribution
+## Contribution Policy
 
-[Similar to SQLite](https://www.sqlite.org/copyright.html), Litestream is open
+Initially, Litestream was closed to outside contributions. The goal was to
-source but closed to contributions. This keeps the code base free of proprietary
+reduce burnout by limiting the maintenance overhead of reviewing and validating
-or licensed code but it also helps me continue to maintain and build Litestream.
+third-party code. However, this policy is overly broad and has prevented small,
+easily testable patches from being contributed.
 
-As the author of [BoltDB](https://github.com/boltdb/bolt), I found that
+Litestream is now open to code contributions for bug fixes only. Features carry
-accepting and maintaining third party patches contributed to my burn out and
+a long-term maintenance burden so they will not be accepted at this time.
-I eventually archived the project. Writing databases & low-level replication
+Please [submit an issue][new-issue] if you have a feature you'd like to
-tools involves nuance and simple one line changes can have profound and
+request.
-unexpected changes in correctness and performance. Small contributions
+
-typically required hours of my time to properly test and validate them.
+If you find mistakes in the documentation, please submit a fix to the
+[documentation repository][docs].
+
+[new-issue]: https://github.com/benbjohnson/litestream/issues/new
+[docs]: https://github.com/benbjohnson/litestream.io
 
-I am grateful for community involvement, bug reports, & feature requests. I do
-not wish to come off as anything but welcoming, however, I've
-made the decision to keep this project closed to contributions for my own
-mental health and long term viability of the project.

.github/workflows/build_and_test.yml

@@ -0,0 +1,30 @@
+name: "Build and Unit Test"
+on: pull_request
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '1.17'
+
+      - uses: actions/cache@v2
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ inputs.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: ${{ inputs.os }}-go-
+
+      - name: Build binary
+        run: go install ./cmd/litestream
+
+      - name: Run unit tests
+        run: make testdata && go test -v --coverprofile=.coverage.out ./... && go tool cover -html .coverage.out -o .coverage.html
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: code-coverage
+          path: .coverage.html

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,38 @@
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+  schedule:
+    - cron: '20 16 * * 4'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'go' ]
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.github/workflows/golangci-lint.yml

@@ -0,0 +1,18 @@
+name: golangci-lint
+on:
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  golangci:
+    name: lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      
+      - uses: golangci/golangci-lint-action@v2
+        with:
+          version: latest
+          args: --timeout=10m

.github/workflows/integration_test.yml

@@ -0,0 +1,138 @@
+name: Integration Tests
+on: 
+  pull_request:
+    branches-ignore:
+      - "dependabot/**"
+
+jobs:
+  s3-integration-test:
+    name: Run S3 Integration Tests
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '1.17'
+
+      - uses: actions/cache@v2
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ inputs.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: ${{ inputs.os }}-go-
+
+      - run: go install ./cmd/litestream
+
+      - run: go test -v -run=TestReplicaClient ./integration -replica-type s3
+        env:
+          LITESTREAM_S3_ACCESS_KEY_ID: ${{ secrets.LITESTREAM_S3_ACCESS_KEY_ID }}
+          LITESTREAM_S3_SECRET_ACCESS_KEY: ${{ secrets.LITESTREAM_S3_SECRET_ACCESS_KEY }}
+          LITESTREAM_S3_REGION: us-east-1
+          LITESTREAM_S3_BUCKET: integration.litestream.io
+
+  gcp-integration-test:
+    name: Run GCP Integration Tests
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '1.17'
+
+      - uses: actions/cache@v2
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ inputs.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: ${{ inputs.os }}-go-
+
+      - name: Extract GCP credentials
+        run: 'echo "$GOOGLE_APPLICATION_CREDENTIALS" > /opt/gcp.json'
+        shell: bash
+        env:
+          GOOGLE_APPLICATION_CREDENTIALS: ${{secrets.GOOGLE_APPLICATION_CREDENTIALS}}
+
+      - run: go test -v -run=TestReplicaClient ./integration -replica-type gs
+        env:
+          GOOGLE_APPLICATION_CREDENTIALS:  /opt/gcp.json
+          LITESTREAM_GS_BUCKET:            integration.litestream.io
+
+  abs-integration-test:
+    name: Run Azure Blob Store Integration Tests
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '1.17'
+
+      - uses: actions/cache@v2
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ inputs.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: ${{ inputs.os }}-go-
+
+      - run: go test -v -run=TestReplicaClient ./integration -replica-type abs
+        env:
+          LITESTREAM_ABS_ACCOUNT_NAME:     ${{ secrets.LITESTREAM_ABS_ACCOUNT_NAME }}
+          LITESTREAM_ABS_ACCOUNT_KEY:      ${{ secrets.LITESTREAM_ABS_ACCOUNT_KEY }}
+          LITESTREAM_ABS_BUCKET:           integration
+
+  sftp-integration-test:
+    name: Run SFTP Integration Tests
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '1.17'
+
+      - uses: actions/cache@v2
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ inputs.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: ${{ inputs.os }}-go-
+
+      - name: Extract SSH key
+        run: 'echo "$LITESTREAM_SFTP_KEY" > /opt/id_ed25519'
+        shell: bash
+        env:
+          LITESTREAM_SFTP_KEY: ${{secrets.LITESTREAM_SFTP_KEY}}
+
+      - name: Run sftp tests w/ key
+        run: go test -v -run=TestReplicaClient ./integration -replica-type sftp
+        env:
+          LITESTREAM_SFTP_HOST: litestream-test-sftp.fly.dev:2222
+          LITESTREAM_SFTP_USER: litestream
+          LITESTREAM_SFTP_PATH: /litestream
+          LITESTREAM_SFTP_KEY_PATH:      /opt/id_ed25519
+
+      - name: Run sftp tests w/ password
+        run: go test -v -run=TestReplicaClient ./integration -replica-type sftp
+        env:
+          LITESTREAM_SFTP_HOST:     litestream-test-sftp.fly.dev:2222
+          LITESTREAM_SFTP_USER:     litestream
+          LITESTREAM_SFTP_PASSWORD: ${{ secrets.LITESTREAM_SFTP_PASSWORD }}
+          LITESTREAM_SFTP_PATH:     /litestream
+
+  long-running-test:
+    name: Run Long-Running Test
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-go@v2
+        with: 
+          go-version: '1.17'
+
+      - uses: actions/cache@v2
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ inputs.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: ${{ inputs.os }}-go-
+
+      - run: go install ./cmd/litestream
+
+      - run: go test -v -run=TestCmd_Replicate_LongRunning ./integration -long-running-duration 1m

.github/workflows/release.docker.yml

@@ -0,0 +1,51 @@
+on:
+  release:
+    types:
+      - published
+  pull_request:
+    types:
+      - opened
+      - synchronize
+      - reopened
+    branches-ignore:
+      - "dependabot/**"
+
+name: Release (Docker)
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    env:
+      PLATFORMS: "${{ github.event_name == 'release' && 'linux/amd64,linux/arm64,linux/arm/v7' || 'linux/amd64' }}"
+      VERSION: "${{ github.event_name == 'release' && github.event.release.name || github.sha }}"
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: docker/setup-qemu-action@v1
+      - uses: docker/setup-buildx-action@v1
+
+      - uses: docker/login-action@v1
+        with:
+          username: benbjohnson
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      
+      - id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: litestream/litestream
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=sha
+            type=sha,format=long
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+      
+      - uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          platforms: ${{ env.PLATFORMS }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          build-args: |
+            LITESTREAM_VERSION=${{ env.VERSION }}

.github/workflows/release.linux.yml

@@ -0,0 +1,134 @@
+on:
+  release:
+    types:
+      - published
+  pull_request:
+    types:
+      - opened
+      - synchronize
+      - reopened
+    branches-ignore:
+      - "dependabot/**"
+
+name: Release (Linux)
+jobs:
+  build:
+    runs-on: ubuntu-18.04
+    strategy:
+      matrix:
+        include:
+          - arch:   amd64
+            cc:     gcc
+
+          - arch:   amd64
+            cc:     gcc
+            static: true
+
+          - arch: arm64
+            cc:   aarch64-linux-gnu-gcc
+
+          - arch: arm64
+            cc:   aarch64-linux-gnu-gcc
+            static: true
+
+          - arch: arm
+            arm:  6
+            cc:   arm-linux-gnueabi-gcc
+
+          - arch: arm
+            arm:  6
+            cc:   arm-linux-gnueabi-gcc
+            static: true
+
+          - arch: arm
+            arm:  7
+            cc:   arm-linux-gnueabihf-gcc
+
+          - arch: arm
+            arm:  7
+            cc:   arm-linux-gnueabihf-gcc
+            static: true
+
+    env:
+      GOOS:   linux
+      GOARCH: ${{ matrix.arch }}
+      GOARM:  ${{ matrix.arm }}
+      CC:     ${{ matrix.cc }}
+      LDFLAGS: ${{ matrix.static && '-extldflags "-static"' || '' }}
+      TAGS: ${{ matrix.static && 'osusergo,netgo,sqlite_omit_load_extension' || '' }}
+      SUFFIX: "${{ matrix.static && '-static' || ''}}"
+      VERSION: "${{ github.event_name == 'release' && github.event.release.name || github.sha }}"
+
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '1.17'
+
+      - name: Install cross-compilers
+        run: |
+          sudo apt-get update 
+          sudo apt-get install -y gcc-aarch64-linux-gnu gcc-arm-linux-gnueabihf gcc-arm-linux-gnueabi
+
+      - name: Install nfpm
+        run: |
+          wget https://github.com/goreleaser/nfpm/releases/download/v2.2.3/nfpm_2.2.3_Linux_x86_64.tar.gz
+          tar zxvf nfpm_2.2.3_Linux_x86_64.tar.gz
+
+      - name: Build litestream
+        run: |
+          rm -rf dist && mkdir -p dist
+
+          cp etc/litestream.yml etc/litestream.service dist
+          cat etc/nfpm.yml | LITESTREAM_VERSION=${{ env.VERSION }} envsubst > dist/nfpm.yml
+
+          CGO_ENABLED=1 go build -ldflags "-s -w ${{ env.LDFLAGS }} -X 'main.Version=${{ env.VERSION }}'" -tags "${{ env.TAGS }}" -o dist/litestream ./cmd/litestream
+
+          cd dist
+          tar -czvf litestream-${{ env.VERSION }}-${{ env.GOOS }}-${{ env.GOARCH }}${{ env.GOARM }}${{ env.SUFFIX }}.tar.gz litestream
+          ../nfpm pkg --config nfpm.yml --packager deb --target litestream-${{ env.VERSION }}-${{ env.GOOS }}-${{ env.GOARCH }}${{ env.GOARM }}${{ env.SUFFIX }}.deb 
+
+      - name: Upload binary artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: litestream-${{ env.VERSION }}-${{ env.GOOS }}-${{ env.GOARCH }}${{ env.GOARM }}${{ env.SUFFIX }}.tar.gz
+          path: dist/litestream-${{ env.VERSION }}-${{ env.GOOS }}-${{ env.GOARCH }}${{ env.GOARM }}${{ env.SUFFIX }}.tar.gz
+          if-no-files-found: error
+
+      - name: Upload debian artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: litestream-${{ env.VERSION }}-${{ env.GOOS }}-${{ env.GOARCH }}${{ env.GOARM }}${{ env.SUFFIX }}.deb
+          path: dist/litestream-${{ env.VERSION }}-${{ env.GOOS }}-${{ env.GOARCH }}${{ env.GOARM }}${{ env.SUFFIX }}.deb
+          if-no-files-found: error
+
+      - name: Get release
+        id: release
+        uses: bruceadams/get-release@v1.2.3
+        if: github.event_name == 'release'
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+
+      - name: Upload release tarball
+        uses: actions/upload-release-asset@v1.0.2
+        if: github.event_name == 'release'
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        with:
+          upload_url: ${{ steps.release.outputs.upload_url }}
+          asset_path: ./dist/litestream-${{ env.VERSION }}-${{ env.GOOS }}-${{ env.GOARCH }}${{ env.GOARM }}${{ env.SUFFIX }}.tar.gz
+          asset_name: litestream-${{ env.VERSION }}-${{ env.GOOS }}-${{ env.GOARCH }}${{ env.GOARM }}${{ env.SUFFIX }}.tar.gz
+          asset_content_type: application/gzip
+
+      - name: Upload debian package
+        uses: actions/upload-release-asset@v1.0.2
+        if: github.event_name == 'release'
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        with:
+          upload_url: ${{ steps.release.outputs.upload_url }}
+          asset_path: ./dist/litestream-${{ env.VERSION }}-${{ env.GOOS }}-${{ env.GOARCH }}${{ env.GOARM }}${{ env.SUFFIX }}.deb
+          asset_name: litestream-${{ env.VERSION }}-${{ env.GOOS }}-${{ env.GOARCH }}${{ env.GOARM }}${{ env.SUFFIX }}.deb
+          asset_content_type: application/octet-stream

.github/workflows/release.yml

@@ -1,53 +0,0 @@
-on:
-  release:
-    types:
-      - created
-
-name: release
-jobs:
-  linux:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-go@v2
-
-      - id: release
-        uses: bruceadams/get-release@v1.2.2
-        env:
-          GITHUB_TOKEN: ${{ github.token }}
-
-      - name: Install nfpm
-        run: |
-          wget https://github.com/goreleaser/nfpm/releases/download/v2.2.3/nfpm_2.2.3_Linux_x86_64.tar.gz
-          tar zxvf nfpm_2.2.3_Linux_x86_64.tar.gz
-
-      - name: Build litestream
-        run: |
-          mkdir -p dist
-          cp etc/litestream.yml etc/litestream.service dist
-          cat etc/nfpm.yml | LITESTREAM_VERSION=${{ steps.release.outputs.tag_name }} envsubst > dist/nfpm.yml
-          go build -ldflags "-X 'main.Version=${{ steps.release.outputs.tag_name }}'" -o dist/litestream ./cmd/litestream
-          
-          cd dist
-          tar -czvf litestream-${{ steps.release.outputs.tag_name }}-linux-amd64.tar.gz litestream
-          ../nfpm pkg --config nfpm.yml --packager deb --target litestream-${{ steps.release.outputs.tag_name }}-linux-amd64.deb 
-
-      - name: Upload release binary
-        uses: actions/upload-release-asset@v1.0.2
-        env:
-          GITHUB_TOKEN: ${{ github.token }}
-        with:
-          upload_url: ${{ steps.release.outputs.upload_url }}
-          asset_path: ./dist/litestream-${{ steps.release.outputs.tag_name }}-linux-amd64.tar.gz
-          asset_name: litestream-${{ steps.release.outputs.tag_name }}-linux-amd64.tar.gz
-          asset_content_type: application/gzip
-
-      - name: Upload debian package
-        uses: actions/upload-release-asset@v1.0.2
-        env:
-          GITHUB_TOKEN: ${{ github.token }}
-        with:
-          upload_url: ${{ steps.release.outputs.upload_url }}
-          asset_path: ./dist/litestream-${{ steps.release.outputs.tag_name }}-linux-amd64.deb
-          asset_name: litestream-${{ steps.release.outputs.tag_name }}-linux-amd64.deb
-          asset_content_type: application/octet-stream

.github/workflows/test.yml

@@ -1,21 +0,0 @@
-on: push
-name: test
-jobs:
-  test:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/setup-go@v2
-        with:
-          go-version: '1.15'
-
-      - uses: actions/checkout@v2
-
-      - uses: actions/cache@v2
-        with:
-          path: ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go-
-
-      - name: Run unit tests
-        run: go test -v ./...

.gitignore

@@ -1,2 +1,3 @@
+.coverage.*
 .DS_Store
 /dist

Dockerfile

@@ -0,0 +1,16 @@
+FROM golang:1.17 as builder
+
+WORKDIR /src/litestream
+COPY . .
+
+ARG LITESTREAM_VERSION=latest
+
+RUN --mount=type=cache,target=/root/.cache/go-build \
+	--mount=type=cache,target=/go/pkg \
+	go build -ldflags "-s -w -X 'main.Version=${LITESTREAM_VERSION}' -extldflags '-static'" -tags osusergo,netgo,sqlite_omit_load_extension -o /usr/local/bin/litestream ./cmd/litestream
+
+
+FROM alpine
+COPY --from=builder /usr/local/bin/litestream /usr/local/bin/litestream
+ENTRYPOINT ["/usr/local/bin/litestream"]
+CMD []

LICENSE

@@ -1,674 +1,202 @@
-                    GNU GENERAL PUBLIC LICENSE
+
-                       Version 3, 29 June 2007
+                                 Apache License
-
+                           Version 2.0, January 2004
- Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
+                        http://www.apache.org/licenses/
- Everyone is permitted to copy and distribute verbatim copies
+
- of this license document, but changing it is not allowed.
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
+
-                            Preamble
+   1. Definitions.
-
+
-  The GNU General Public License is a free, copyleft license for
+      "License" shall mean the terms and conditions for use, reproduction,
-software and other kinds of works.
+      and distribution as defined by Sections 1 through 9 of this document.
-
+
-  The licenses for most software and other practical works are designed
+      "Licensor" shall mean the copyright owner or entity authorized by
-to take away your freedom to share and change the works.  By contrast,
+      the copyright owner that is granting the License.
-the GNU General Public License is intended to guarantee your freedom to
+
-share and change all versions of a program--to make sure it remains free
+      "Legal Entity" shall mean the union of the acting entity and all
-software for all its users.  We, the Free Software Foundation, use the
+      other entities that control, are controlled by, or are under common
-GNU General Public License for most of our software; it applies also to
+      control with that entity. For the purposes of this definition,
-any other work released this way by its authors.  You can apply it to
+      "control" means (i) the power, direct or indirect, to cause the
-your programs, too.
+      direction or management of such entity, whether by contract or
-
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-  When we speak of free software, we are referring to freedom, not
+      outstanding shares, or (iii) beneficial ownership of such entity.
-price.  Our General Public Licenses are designed to make sure that you
+
-have the freedom to distribute copies of free software (and charge for
+      "You" (or "Your") shall mean an individual or Legal Entity
-them if you wish), that you receive source code or can get it if you
+      exercising permissions granted by this License.
-want it, that you can change the software or use pieces of it in new
+
-free programs, and that you know you can do these things.
+      "Source" form shall mean the preferred form for making modifications,
-
+      including but not limited to software source code, documentation
-  To protect your rights, we need to prevent others from denying you
+      source, and configuration files.
-these rights or asking you to surrender the rights.  Therefore, you have
+
-certain responsibilities if you distribute copies of the software, or if
+      "Object" form shall mean any form resulting from mechanical
-you modify it: responsibilities to respect the freedom of others.
+      transformation or translation of a Source form, including but
-
+      not limited to compiled object code, generated documentation,
-  For example, if you distribute copies of such a program, whether
+      and conversions to other media types.
-gratis or for a fee, you must pass on to the recipients the same
+
-freedoms that you received.  You must make sure that they, too, receive
+      "Work" shall mean the work of authorship, whether in Source or
-or can get the source code.  And you must show them these terms so they
+      Object form, made available under the License, as indicated by a
-know their rights.
+      copyright notice that is included in or attached to the work
-
+      (an example is provided in the Appendix below).
-  Developers that use the GNU GPL protect your rights with two steps:
+
-(1) assert copyright on the software, and (2) offer you this License
+      "Derivative Works" shall mean any work, whether in Source or Object
-giving you legal permission to copy, distribute and/or modify it.
+      form, that is based on (or derived from) the Work and for which the
-
+      editorial revisions, annotations, elaborations, or other modifications
-  For the developers' and authors' protection, the GPL clearly explains
+      represent, as a whole, an original work of authorship. For the purposes
-that there is no warranty for this free software.  For both users' and
+      of this License, Derivative Works shall not include works that remain
-authors' sake, the GPL requires that modified versions be marked as
+      separable from, or merely link (or bind by name) to the interfaces of,
-changed, so that their problems will not be attributed erroneously to
+      the Work and Derivative Works thereof.
-authors of previous versions.
+
-
+      "Contribution" shall mean any work of authorship, including
-  Some devices are designed to deny users access to install or run
+      the original version of the Work and any modifications or additions
-modified versions of the software inside them, although the manufacturer
+      to that Work or Derivative Works thereof, that is intentionally
-can do so.  This is fundamentally incompatible with the aim of
+      submitted to Licensor for inclusion in the Work by the copyright owner
-protecting users' freedom to change the software.  The systematic
+      or by an individual or Legal Entity authorized to submit on behalf of
-pattern of such abuse occurs in the area of products for individuals to
+      the copyright owner. For the purposes of this definition, "submitted"
-use, which is precisely where it is most unacceptable.  Therefore, we
+      means any form of electronic, verbal, or written communication sent
-have designed this version of the GPL to prohibit the practice for those
+      to the Licensor or its representatives, including but not limited to
-products.  If such problems arise substantially in other domains, we
+      communication on electronic mailing lists, source code control systems,
-stand ready to extend this provision to those domains in future versions
+      and issue tracking systems that are managed by, or on behalf of, the
-of the GPL, as needed to protect the freedom of users.
+      Licensor for the purpose of discussing and improving the Work, but
-
+      excluding communication that is conspicuously marked or otherwise
-  Finally, every program is threatened constantly by software patents.
+      designated in writing by the copyright owner as "Not a Contribution."
-States should not allow patents to restrict development and use of
+
-software on general-purpose computers, but in those that do, we wish to
+      "Contributor" shall mean Licensor and any individual or Legal Entity
-avoid the special danger that patents applied to a free program could
+      on behalf of whom a Contribution has been received by Licensor and
-make it effectively proprietary.  To prevent this, the GPL assures that
+      subsequently incorporated within the Work.
-patents cannot be used to render the program non-free.
+
-
+   2. Grant of Copyright License. Subject to the terms and conditions of
-  The precise terms and conditions for copying, distribution and
+      this License, each Contributor hereby grants to You a perpetual,
-modification follow.
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-
+      copyright license to reproduce, prepare Derivative Works of,
-                       TERMS AND CONDITIONS
+      publicly display, publicly perform, sublicense, and distribute the
-
+      Work and such Derivative Works in Source or Object form.
-  0. Definitions.
+
-
+   3. Grant of Patent License. Subject to the terms and conditions of
-  "This License" refers to version 3 of the GNU General Public License.
+      this License, each Contributor hereby grants to You a perpetual,
-
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-  "Copyright" also means copyright-like laws that apply to other kinds of
+      (except as stated in this section) patent license to make, have made,
-works, such as semiconductor masks.
+      use, offer to sell, sell, import, and otherwise transfer the Work,
-
+      where such license applies only to those patent claims licensable
-  "The Program" refers to any copyrightable work licensed under this
+      by such Contributor that are necessarily infringed by their
-License.  Each licensee is addressed as "you".  "Licensees" and
+      Contribution(s) alone or by combination of their Contribution(s)
-"recipients" may be individuals or organizations.
+      with the Work to which such Contribution(s) was submitted. If You
-
+      institute patent litigation against any entity (including a
-  To "modify" a work means to copy from or adapt all or part of the work
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
-in a fashion requiring copyright permission, other than the making of an
+      or a Contribution incorporated within the Work constitutes direct
-exact copy.  The resulting work is called a "modified version" of the
+      or contributory patent infringement, then any patent licenses
-earlier work or a work "based on" the earlier work.
+      granted to You under this License for that Work shall terminate
-
+      as of the date such litigation is filed.
-  A "covered work" means either the unmodified Program or a work based
+
-on the Program.
+   4. Redistribution. You may reproduce and distribute copies of the
-
+      Work or Derivative Works thereof in any medium, with or without
-  To "propagate" a work means to do anything with it that, without
+      modifications, and in Source or Object form, provided that You
-permission, would make you directly or secondarily liable for
+      meet the following conditions:
-infringement under applicable copyright law, except executing it on a
+
-computer or modifying a private copy.  Propagation includes copying,
+      (a) You must give any other recipients of the Work or
-distribution (with or without modification), making available to the
+          Derivative Works a copy of this License; and
-public, and in some countries other activities as well.
+
-
+      (b) You must cause any modified files to carry prominent notices
-  To "convey" a work means any kind of propagation that enables other
+          stating that You changed the files; and
-parties to make or receive copies.  Mere interaction with a user through
+
-a computer network, with no transfer of a copy, is not conveying.
+      (c) You must retain, in the Source form of any Derivative Works
-
+          that You distribute, all copyright, patent, trademark, and
-  An interactive user interface displays "Appropriate Legal Notices"
+          attribution notices from the Source form of the Work,
-to the extent that it includes a convenient and prominently visible
+          excluding those notices that do not pertain to any part of
-feature that (1) displays an appropriate copyright notice, and (2)
+          the Derivative Works; and
-tells the user that there is no warranty for the work (except to the
+
-extent that warranties are provided), that licensees may convey the
+      (d) If the Work includes a "NOTICE" text file as part of its
-work under this License, and how to view a copy of this License.  If
+          distribution, then any Derivative Works that You distribute must
-the interface presents a list of user commands or options, such as a
+          include a readable copy of the attribution notices contained
-menu, a prominent item in the list meets this criterion.
+          within such NOTICE file, excluding those notices that do not
-
+          pertain to any part of the Derivative Works, in at least one
-  1. Source Code.
+          of the following places: within a NOTICE text file distributed
-
+          as part of the Derivative Works; within the Source form or
-  The "source code" for a work means the preferred form of the work
+          documentation, if provided along with the Derivative Works; or,
-for making modifications to it.  "Object code" means any non-source
+          within a display generated by the Derivative Works, if and
-form of a work.
+          wherever such third-party notices normally appear. The contents
-
+          of the NOTICE file are for informational purposes only and
-  A "Standard Interface" means an interface that either is an official
+          do not modify the License. You may add Your own attribution
-standard defined by a recognized standards body, or, in the case of
+          notices within Derivative Works that You distribute, alongside
-interfaces specified for a particular programming language, one that
+          or as an addendum to the NOTICE text from the Work, provided
-is widely used among developers working in that language.
+          that such additional attribution notices cannot be construed
-
+          as modifying the License.
-  The "System Libraries" of an executable work include anything, other
+
-than the work as a whole, that (a) is included in the normal form of
+      You may add Your own copyright statement to Your modifications and
-packaging a Major Component, but which is not part of that Major
+      may provide additional or different license terms and conditions
-Component, and (b) serves only to enable use of the work with that
+      for use, reproduction, or distribution of Your modifications, or
-Major Component, or to implement a Standard Interface for which an
+      for any such Derivative Works as a whole, provided Your use,
-implementation is available to the public in source code form.  A
+      reproduction, and distribution of the Work otherwise complies with
-"Major Component", in this context, means a major essential component
+      the conditions stated in this License.
-(kernel, window system, and so on) of the specific operating system
+
-(if any) on which the executable work runs, or a compiler used to
+   5. Submission of Contributions. Unless You explicitly state otherwise,
-produce the work, or an object code interpreter used to run it.
+      any Contribution intentionally submitted for inclusion in the Work
-
+      by You to the Licensor shall be under the terms and conditions of
-  The "Corresponding Source" for a work in object code form means all
+      this License, without any additional terms or conditions.
-the source code needed to generate, install, and (for an executable
+      Notwithstanding the above, nothing herein shall supersede or modify
-work) run the object code and to modify the work, including scripts to
+      the terms of any separate license agreement you may have executed
-control those activities.  However, it does not include the work's
+      with Licensor regarding such Contributions.
-System Libraries, or general-purpose tools or generally available free
+
-programs which are used unmodified in performing those activities but
+   6. Trademarks. This License does not grant permission to use the trade
-which are not part of the work.  For example, Corresponding Source
+      names, trademarks, service marks, or product names of the Licensor,
-includes interface definition files associated with source files for
+      except as required for reasonable and customary use in describing the
-the work, and the source code for shared libraries and dynamically
+      origin of the Work and reproducing the content of the NOTICE file.
-linked subprograms that the work is specifically designed to require,
+
-such as by intimate data communication or control flow between those
+   7. Disclaimer of Warranty. Unless required by applicable law or
-subprograms and other parts of the work.
+      agreed to in writing, Licensor provides the Work (and each
-
+      Contributor provides its Contributions) on an "AS IS" BASIS,
-  The Corresponding Source need not include anything that users
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-can regenerate automatically from other parts of the Corresponding
+      implied, including, without limitation, any warranties or conditions
-Source.
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-
+      PARTICULAR PURPOSE. You are solely responsible for determining the
-  The Corresponding Source for a work in source code form is that
+      appropriateness of using or redistributing the Work and assume any
-same work.
+      risks associated with Your exercise of permissions under this License.
-
+
-  2. Basic Permissions.
+   8. Limitation of Liability. In no event and under no legal theory,
-
+      whether in tort (including negligence), contract, or otherwise,
-  All rights granted under this License are granted for the term of
+      unless required by applicable law (such as deliberate and grossly
-copyright on the Program, and are irrevocable provided the stated
+      negligent acts) or agreed to in writing, shall any Contributor be
-conditions are met.  This License explicitly affirms your unlimited
+      liable to You for damages, including any direct, indirect, special,
-permission to run the unmodified Program.  The output from running a
+      incidental, or consequential damages of any character arising as a
-covered work is covered by this License only if the output, given its
+      result of this License or out of the use or inability to use the
-content, constitutes a covered work.  This License acknowledges your
+      Work (including but not limited to damages for loss of goodwill,
-rights of fair use or other equivalent, as provided by copyright law.
+      work stoppage, computer failure or malfunction, or any and all
-
+      other commercial damages or losses), even if such Contributor
-  You may make, run and propagate covered works that you do not
+      has been advised of the possibility of such damages.
-convey, without conditions so long as your license otherwise remains
+
-in force.  You may convey covered works to others for the sole purpose
+   9. Accepting Warranty or Additional Liability. While redistributing
-of having them make modifications exclusively for you, or provide you
+      the Work or Derivative Works thereof, You may choose to offer,
-with facilities for running those works, provided that you comply with
+      and charge a fee for, acceptance of support, warranty, indemnity,
-the terms of this License in conveying all material for which you do
+      or other liability obligations and/or rights consistent with this
-not control copyright.  Those thus making or running the covered works
+      License. However, in accepting such obligations, You may act only
-for you must do so exclusively on your behalf, under your direction
+      on Your own behalf and on Your sole responsibility, not on behalf
-and control, on terms that prohibit them from making any copies of
+      of any other Contributor, and only if You agree to indemnify,
-your copyrighted material outside their relationship with you.
+      defend, and hold each Contributor harmless for any liability
-
+      incurred by, or claims asserted against, such Contributor by reason
-  Conveying under any other circumstances is permitted solely under
+      of your accepting any such warranty or additional liability.
-the conditions stated below.  Sublicensing is not allowed; section 10
+
-makes it unnecessary.
+   END OF TERMS AND CONDITIONS
-
+
-  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+   APPENDIX: How to apply the Apache License to your work.
-
+
-  No covered work shall be deemed part of an effective technological
+      To apply the Apache License to your work, attach the following
-measure under any applicable law fulfilling obligations under article
+      boilerplate notice, with the fields enclosed by brackets "[]"
-11 of the WIPO copyright treaty adopted on 20 December 1996, or
+      replaced with your own identifying information. (Don't include
-similar laws prohibiting or restricting circumvention of such
+      the brackets!)  The text should be enclosed in the appropriate
-measures.
+      comment syntax for the file format. We also recommend that a
-
+      file or class name and description of purpose be included on the
-  When you convey a covered work, you waive any legal power to forbid
+      same "printed page" as the copyright notice for easier
-circumvention of technological measures to the extent such circumvention
+      identification within third-party archives.
-is effected by exercising rights under this License with respect to
+
-the covered work, and you disclaim any intention to limit operation or
+   Copyright [yyyy] [name of copyright owner]
-modification of the work as a means of enforcing, against the work's
+
-users, your or third parties' legal rights to forbid circumvention of
+   Licensed under the Apache License, Version 2.0 (the "License");
-technological measures.
+   you may not use this file except in compliance with the License.
-
+   You may obtain a copy of the License at
-  4. Conveying Verbatim Copies.
+
-
+       http://www.apache.org/licenses/LICENSE-2.0
-  You may convey verbatim copies of the Program's source code as you
+
-receive it, in any medium, provided that you conspicuously and
+   Unless required by applicable law or agreed to in writing, software
-appropriately publish on each copy an appropriate copyright notice;
+   distributed under the License is distributed on an "AS IS" BASIS,
-keep intact all notices stating that this License and any
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-non-permissive terms added in accord with section 7 apply to the code;
+   See the License for the specific language governing permissions and
-keep intact all notices of the absence of any warranty; and give all
+   limitations under the License.
-recipients a copy of this License along with the Program.
-
-  You may charge any price or no price for each copy that you convey,
-and you may offer support or warranty protection for a fee.
-
-  5. Conveying Modified Source Versions.
-
-  You may convey a work based on the Program, or the modifications to
-produce it from the Program, in the form of source code under the
-terms of section 4, provided that you also meet all of these conditions:
-
-    a) The work must carry prominent notices stating that you modified
-    it, and giving a relevant date.
-
-    b) The work must carry prominent notices stating that it is
-    released under this License and any conditions added under section
-    7.  This requirement modifies the requirement in section 4 to
-    "keep intact all notices".
-
-    c) You must license the entire work, as a whole, under this
-    License to anyone who comes into possession of a copy.  This
-    License will therefore apply, along with any applicable section 7
-    additional terms, to the whole of the work, and all its parts,
-    regardless of how they are packaged.  This License gives no
-    permission to license the work in any other way, but it does not
-    invalidate such permission if you have separately received it.
-
-    d) If the work has interactive user interfaces, each must display
-    Appropriate Legal Notices; however, if the Program has interactive
-    interfaces that do not display Appropriate Legal Notices, your
-    work need not make them do so.
-
-  A compilation of a covered work with other separate and independent
-works, which are not by their nature extensions of the covered work,
-and which are not combined with it such as to form a larger program,
-in or on a volume of a storage or distribution medium, is called an
-"aggregate" if the compilation and its resulting copyright are not
-used to limit the access or legal rights of the compilation's users
-beyond what the individual works permit.  Inclusion of a covered work
-in an aggregate does not cause this License to apply to the other
-parts of the aggregate.
-
-  6. Conveying Non-Source Forms.
-
-  You may convey a covered work in object code form under the terms
-of sections 4 and 5, provided that you also convey the
-machine-readable Corresponding Source under the terms of this License,
-in one of these ways:
-
-    a) Convey the object code in, or embodied in, a physical product
-    (including a physical distribution medium), accompanied by the
-    Corresponding Source fixed on a durable physical medium
-    customarily used for software interchange.
-
-    b) Convey the object code in, or embodied in, a physical product
-    (including a physical distribution medium), accompanied by a
-    written offer, valid for at least three years and valid for as
-    long as you offer spare parts or customer support for that product
-    model, to give anyone who possesses the object code either (1) a
-    copy of the Corresponding Source for all the software in the
-    product that is covered by this License, on a durable physical
-    medium customarily used for software interchange, for a price no
-    more than your reasonable cost of physically performing this
-    conveying of source, or (2) access to copy the
-    Corresponding Source from a network server at no charge.
-
-    c) Convey individual copies of the object code with a copy of the
-    written offer to provide the Corresponding Source.  This
-    alternative is allowed only occasionally and noncommercially, and
-    only if you received the object code with such an offer, in accord
-    with subsection 6b.
-
-    d) Convey the object code by offering access from a designated
-    place (gratis or for a charge), and offer equivalent access to the
-    Corresponding Source in the same way through the same place at no
-    further charge.  You need not require recipients to copy the
-    Corresponding Source along with the object code.  If the place to
-    copy the object code is a network server, the Corresponding Source
-    may be on a different server (operated by you or a third party)
-    that supports equivalent copying facilities, provided you maintain
-    clear directions next to the object code saying where to find the
-    Corresponding Source.  Regardless of what server hosts the
-    Corresponding Source, you remain obligated to ensure that it is
-    available for as long as needed to satisfy these requirements.
-
-    e) Convey the object code using peer-to-peer transmission, provided
-    you inform other peers where the object code and Corresponding
-    Source of the work are being offered to the general public at no
-    charge under subsection 6d.
-
-  A separable portion of the object code, whose source code is excluded
-from the Corresponding Source as a System Library, need not be
-included in conveying the object code work.
-
-  A "User Product" is either (1) a "consumer product", which means any
-tangible personal property which is normally used for personal, family,
-or household purposes, or (2) anything designed or sold for incorporation
-into a dwelling.  In determining whether a product is a consumer product,
-doubtful cases shall be resolved in favor of coverage.  For a particular
-product received by a particular user, "normally used" refers to a
-typical or common use of that class of product, regardless of the status
-of the particular user or of the way in which the particular user
-actually uses, or expects or is expected to use, the product.  A product
-is a consumer product regardless of whether the product has substantial
-commercial, industrial or non-consumer uses, unless such uses represent
-the only significant mode of use of the product.
-
-  "Installation Information" for a User Product means any methods,
-procedures, authorization keys, or other information required to install
-and execute modified versions of a covered work in that User Product from
-a modified version of its Corresponding Source.  The information must
-suffice to ensure that the continued functioning of the modified object
-code is in no case prevented or interfered with solely because
-modification has been made.
-
-  If you convey an object code work under this section in, or with, or
-specifically for use in, a User Product, and the conveying occurs as
-part of a transaction in which the right of possession and use of the
-User Product is transferred to the recipient in perpetuity or for a
-fixed term (regardless of how the transaction is characterized), the
-Corresponding Source conveyed under this section must be accompanied
-by the Installation Information.  But this requirement does not apply
-if neither you nor any third party retains the ability to install
-modified object code on the User Product (for example, the work has
-been installed in ROM).
-
-  The requirement to provide Installation Information does not include a
-requirement to continue to provide support service, warranty, or updates
-for a work that has been modified or installed by the recipient, or for
-the User Product in which it has been modified or installed.  Access to a
-network may be denied when the modification itself materially and
-adversely affects the operation of the network or violates the rules and
-protocols for communication across the network.
-
-  Corresponding Source conveyed, and Installation Information provided,
-in accord with this section must be in a format that is publicly
-documented (and with an implementation available to the public in
-source code form), and must require no special password or key for
-unpacking, reading or copying.
-
-  7. Additional Terms.
-
-  "Additional permissions" are terms that supplement the terms of this
-License by making exceptions from one or more of its conditions.
-Additional permissions that are applicable to the entire Program shall
-be treated as though they were included in this License, to the extent
-that they are valid under applicable law.  If additional permissions
-apply only to part of the Program, that part may be used separately
-under those permissions, but the entire Program remains governed by
-this License without regard to the additional permissions.
-
-  When you convey a copy of a covered work, you may at your option
-remove any additional permissions from that copy, or from any part of
-it.  (Additional permissions may be written to require their own
-removal in certain cases when you modify the work.)  You may place
-additional permissions on material, added by you to a covered work,
-for which you have or can give appropriate copyright permission.
-
-  Notwithstanding any other provision of this License, for material you
-add to a covered work, you may (if authorized by the copyright holders of
-that material) supplement the terms of this License with terms:
-
-    a) Disclaiming warranty or limiting liability differently from the
-    terms of sections 15 and 16 of this License; or
-
-    b) Requiring preservation of specified reasonable legal notices or
-    author attributions in that material or in the Appropriate Legal
-    Notices displayed by works containing it; or
-
-    c) Prohibiting misrepresentation of the origin of that material, or
-    requiring that modified versions of such material be marked in
-    reasonable ways as different from the original version; or
-
-    d) Limiting the use for publicity purposes of names of licensors or
-    authors of the material; or
-
-    e) Declining to grant rights under trademark law for use of some
-    trade names, trademarks, or service marks; or
-
-    f) Requiring indemnification of licensors and authors of that
-    material by anyone who conveys the material (or modified versions of
-    it) with contractual assumptions of liability to the recipient, for
-    any liability that these contractual assumptions directly impose on
-    those licensors and authors.
-
-  All other non-permissive additional terms are considered "further
-restrictions" within the meaning of section 10.  If the Program as you
-received it, or any part of it, contains a notice stating that it is
-governed by this License along with a term that is a further
-restriction, you may remove that term.  If a license document contains
-a further restriction but permits relicensing or conveying under this
-License, you may add to a covered work material governed by the terms
-of that license document, provided that the further restriction does
-not survive such relicensing or conveying.
-
-  If you add terms to a covered work in accord with this section, you
-must place, in the relevant source files, a statement of the
-additional terms that apply to those files, or a notice indicating
-where to find the applicable terms.
-
-  Additional terms, permissive or non-permissive, may be stated in the
-form of a separately written license, or stated as exceptions;
-the above requirements apply either way.
-
-  8. Termination.
-
-  You may not propagate or modify a covered work except as expressly
-provided under this License.  Any attempt otherwise to propagate or
-modify it is void, and will automatically terminate your rights under
-this License (including any patent licenses granted under the third
-paragraph of section 11).
-
-  However, if you cease all violation of this License, then your
-license from a particular copyright holder is reinstated (a)
-provisionally, unless and until the copyright holder explicitly and
-finally terminates your license, and (b) permanently, if the copyright
-holder fails to notify you of the violation by some reasonable means
-prior to 60 days after the cessation.
-
-  Moreover, your license from a particular copyright holder is
-reinstated permanently if the copyright holder notifies you of the
-violation by some reasonable means, this is the first time you have
-received notice of violation of this License (for any work) from that
-copyright holder, and you cure the violation prior to 30 days after
-your receipt of the notice.
-
-  Termination of your rights under this section does not terminate the
-licenses of parties who have received copies or rights from you under
-this License.  If your rights have been terminated and not permanently
-reinstated, you do not qualify to receive new licenses for the same
-material under section 10.
-
-  9. Acceptance Not Required for Having Copies.
-
-  You are not required to accept this License in order to receive or
-run a copy of the Program.  Ancillary propagation of a covered work
-occurring solely as a consequence of using peer-to-peer transmission
-to receive a copy likewise does not require acceptance.  However,
-nothing other than this License grants you permission to propagate or
-modify any covered work.  These actions infringe copyright if you do
-not accept this License.  Therefore, by modifying or propagating a
-covered work, you indicate your acceptance of this License to do so.
-
-  10. Automatic Licensing of Downstream Recipients.
-
-  Each time you convey a covered work, the recipient automatically
-receives a license from the original licensors, to run, modify and
-propagate that work, subject to this License.  You are not responsible
-for enforcing compliance by third parties with this License.
-
-  An "entity transaction" is a transaction transferring control of an
-organization, or substantially all assets of one, or subdividing an
-organization, or merging organizations.  If propagation of a covered
-work results from an entity transaction, each party to that
-transaction who receives a copy of the work also receives whatever
-licenses to the work the party's predecessor in interest had or could
-give under the previous paragraph, plus a right to possession of the
-Corresponding Source of the work from the predecessor in interest, if
-the predecessor has it or can get it with reasonable efforts.
-
-  You may not impose any further restrictions on the exercise of the
-rights granted or affirmed under this License.  For example, you may
-not impose a license fee, royalty, or other charge for exercise of
-rights granted under this License, and you may not initiate litigation
-(including a cross-claim or counterclaim in a lawsuit) alleging that
-any patent claim is infringed by making, using, selling, offering for
-sale, or importing the Program or any portion of it.
-
-  11. Patents.
-
-  A "contributor" is a copyright holder who authorizes use under this
-License of the Program or a work on which the Program is based.  The
-work thus licensed is called the contributor's "contributor version".
-
-  A contributor's "essential patent claims" are all patent claims
-owned or controlled by the contributor, whether already acquired or
-hereafter acquired, that would be infringed by some manner, permitted
-by this License, of making, using, or selling its contributor version,
-but do not include claims that would be infringed only as a
-consequence of further modification of the contributor version.  For
-purposes of this definition, "control" includes the right to grant
-patent sublicenses in a manner consistent with the requirements of
-this License.
-
-  Each contributor grants you a non-exclusive, worldwide, royalty-free
-patent license under the contributor's essential patent claims, to
-make, use, sell, offer for sale, import and otherwise run, modify and
-propagate the contents of its contributor version.
-
-  In the following three paragraphs, a "patent license" is any express
-agreement or commitment, however denominated, not to enforce a patent
-(such as an express permission to practice a patent or covenant not to
-sue for patent infringement).  To "grant" such a patent license to a
-party means to make such an agreement or commitment not to enforce a
-patent against the party.
-
-  If you convey a covered work, knowingly relying on a patent license,
-and the Corresponding Source of the work is not available for anyone
-to copy, free of charge and under the terms of this License, through a
-publicly available network server or other readily accessible means,
-then you must either (1) cause the Corresponding Source to be so
-available, or (2) arrange to deprive yourself of the benefit of the
-patent license for this particular work, or (3) arrange, in a manner
-consistent with the requirements of this License, to extend the patent
-license to downstream recipients.  "Knowingly relying" means you have
-actual knowledge that, but for the patent license, your conveying the
-covered work in a country, or your recipient's use of the covered work
-in a country, would infringe one or more identifiable patents in that
-country that you have reason to believe are valid.
-
-  If, pursuant to or in connection with a single transaction or
-arrangement, you convey, or propagate by procuring conveyance of, a
-covered work, and grant a patent license to some of the parties
-receiving the covered work authorizing them to use, propagate, modify
-or convey a specific copy of the covered work, then the patent license
-you grant is automatically extended to all recipients of the covered
-work and works based on it.
-
-  A patent license is "discriminatory" if it does not include within
-the scope of its coverage, prohibits the exercise of, or is
-conditioned on the non-exercise of one or more of the rights that are
-specifically granted under this License.  You may not convey a covered
-work if you are a party to an arrangement with a third party that is
-in the business of distributing software, under which you make payment
-to the third party based on the extent of your activity of conveying
-the work, and under which the third party grants, to any of the
-parties who would receive the covered work from you, a discriminatory
-patent license (a) in connection with copies of the covered work
-conveyed by you (or copies made from those copies), or (b) primarily
-for and in connection with specific products or compilations that
-contain the covered work, unless you entered into that arrangement,
-or that patent license was granted, prior to 28 March 2007.
-
-  Nothing in this License shall be construed as excluding or limiting
-any implied license or other defenses to infringement that may
-otherwise be available to you under applicable patent law.
-
-  12. No Surrender of Others' Freedom.
-
-  If conditions are imposed on you (whether by court order, agreement or
-otherwise) that contradict the conditions of this License, they do not
-excuse you from the conditions of this License.  If you cannot convey a
-covered work so as to satisfy simultaneously your obligations under this
-License and any other pertinent obligations, then as a consequence you may
-not convey it at all.  For example, if you agree to terms that obligate you
-to collect a royalty for further conveying from those to whom you convey
-the Program, the only way you could satisfy both those terms and this
-License would be to refrain entirely from conveying the Program.
-
-  13. Use with the GNU Affero General Public License.
-
-  Notwithstanding any other provision of this License, you have
-permission to link or combine any covered work with a work licensed
-under version 3 of the GNU Affero General Public License into a single
-combined work, and to convey the resulting work.  The terms of this
-License will continue to apply to the part which is the covered work,
-but the special requirements of the GNU Affero General Public License,
-section 13, concerning interaction through a network will apply to the
-combination as such.
-
-  14. Revised Versions of this License.
-
-  The Free Software Foundation may publish revised and/or new versions of
-the GNU General Public License from time to time.  Such new versions will
-be similar in spirit to the present version, but may differ in detail to
-address new problems or concerns.
-
-  Each version is given a distinguishing version number.  If the
-Program specifies that a certain numbered version of the GNU General
-Public License "or any later version" applies to it, you have the
-option of following the terms and conditions either of that numbered
-version or of any later version published by the Free Software
-Foundation.  If the Program does not specify a version number of the
-GNU General Public License, you may choose any version ever published
-by the Free Software Foundation.
-
-  If the Program specifies that a proxy can decide which future
-versions of the GNU General Public License can be used, that proxy's
-public statement of acceptance of a version permanently authorizes you
-to choose that version for the Program.
-
-  Later license versions may give you additional or different
-permissions.  However, no additional obligations are imposed on any
-author or copyright holder as a result of your choosing to follow a
-later version.
-
-  15. Disclaimer of Warranty.
-
-  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
-APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
-HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
-OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
-THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
-IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
-ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
-
-  16. Limitation of Liability.
-
-  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
-WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
-THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
-GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
-USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
-DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
-PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
-EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
-SUCH DAMAGES.
-
-  17. Interpretation of Sections 15 and 16.
-
-  If the disclaimer of warranty and limitation of liability provided
-above cannot be given local legal effect according to their terms,
-reviewing courts shall apply local law that most closely approximates
-an absolute waiver of all civil liability in connection with the
-Program, unless a warranty or assumption of liability accompanies a
-copy of the Program in return for a fee.
-
-                     END OF TERMS AND CONDITIONS
-
-            How to Apply These Terms to Your New Programs
-
-  If you develop a new program, and you want it to be of the greatest
-possible use to the public, the best way to achieve this is to make it
-free software which everyone can redistribute and change under these terms.
-
-  To do so, attach the following notices to the program.  It is safest
-to attach them to the start of each source file to most effectively
-state the exclusion of warranty; and each file should have at least
-the "copyright" line and a pointer to where the full notice is found.
-
-    <one line to give the program's name and a brief idea of what it does.>
-    Copyright (C) <year>  <name of author>
-
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-Also add information on how to contact you by electronic and paper mail.
-
-  If the program does terminal interaction, make it output a short
-notice like this when it starts in an interactive mode:
-
-    <program>  Copyright (C) <year>  <name of author>
-    This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
-    This is free software, and you are welcome to redistribute it
-    under certain conditions; type `show c' for details.
-
-The hypothetical commands `show w' and `show c' should show the appropriate
-parts of the General Public License.  Of course, your program's commands
-might be different; for a GUI interface, you would use an "about box".
-
-  You should also get your employer (if you work as a programmer) or school,
-if any, to sign a "copyright disclaimer" for the program, if necessary.
-For more information on this, and how to apply and follow the GNU GPL, see
-<https://www.gnu.org/licenses/>.
-
-  The GNU General Public License does not permit incorporating your program
-into proprietary programs.  If your program is a subroutine library, you
-may consider it more useful to permit linking proprietary applications with
-the library.  If this is what you want to do, use the GNU Lesser General
-Public License instead of this License.  But first, please read
-<https://www.gnu.org/licenses/why-not-lgpl.html>.

Makefile

@@ -1,17 +1,32 @@
+.PHONY: default
 default:
 
+.PHONY: testdata
+testdata:
+	make -C testdata
+	make -C cmd/litestream testdata
+
+docker:
+	docker build -t litestream .
+
 dist-linux:
 	mkdir -p dist
 	cp etc/litestream.yml dist/litestream.yml
-	docker run --rm -v "${PWD}":/usr/src/litestream -w /usr/src/litestream -e GOOS=linux -e GOARCH=amd64 golang:1.15 go build -v -o dist/litestream ./cmd/litestream
+	docker run --rm -v "${PWD}":/usr/src/litestream -w /usr/src/litestream -e GOOS=linux -e GOARCH=amd64 golang:1.16 go build -v -ldflags "-s -w" -o dist/litestream ./cmd/litestream
 	tar -cz -f dist/litestream-linux-amd64.tar.gz -C dist litestream
 
+dist-linux-arm:
+	docker run --rm -v "${PWD}":/usr/src/litestream -w /usr/src/litestream -e CGO_ENABLED=1 -e CC=arm-linux-gnueabihf-gcc -e GOOS=linux -e GOARCH=arm golang-xc:1.16 go build -v -o dist/litestream-linux-arm ./cmd/litestream
+
+dist-linux-arm64:
+	docker run --rm -v "${PWD}":/usr/src/litestream -w /usr/src/litestream -e CGO_ENABLED=1 -e CC=aarch64-linux-gnu-gcc -e GOOS=linux -e GOARCH=arm64 golang-xc:1.16 go build -v -o dist/litestream-linux-arm64 ./cmd/litestream
+
 dist-macos:
 ifndef LITESTREAM_VERSION
 	$(error LITESTREAM_VERSION is undefined)
 endif
 	mkdir -p dist
-	go build -v -ldflags "-X 'main.Version=${LITESTREAM_VERSION}'"  -o dist/litestream ./cmd/litestream
+	go build -v -ldflags "-s -w -X 'main.Version=${LITESTREAM_VERSION}'"  -o dist/litestream ./cmd/litestream
 	gon etc/gon.hcl
 	mv dist/litestream.zip dist/litestream-${LITESTREAM_VERSION}-darwin-amd64.zip
 	openssl dgst -sha256 dist/litestream-${LITESTREAM_VERSION}-darwin-amd64.zip

README.md

@@ -2,6 +2,7 @@ Litestream
 ![GitHub release (latest by date)](https://img.shields.io/github/v/release/benbjohnson/litestream)
 ![Status](https://img.shields.io/badge/status-beta-blue)
 ![GitHub](https://img.shields.io/github/license/benbjohnson/litestream)
+[![Docker Pulls](https://img.shields.io/docker/pulls/litestream/litestream.svg?maxAge=604800)](https://hub.docker.com/r/litestream/litestream/)
 ![test](https://github.com/benbjohnson/litestream/workflows/test/badge.svg)
 ==========
 
@@ -10,287 +11,51 @@ background process and safely replicates changes incrementally to another file
 or S3. Litestream only communicates with SQLite through the SQLite API so it
 will not corrupt your database.
 
-If you need support or have ideas for improving Litestream, please visit the
+If you need support or have ideas for improving Litestream, please join the
-[GitHub Discussions](https://github.com/benbjohnson/litestream/discussions) to
+[Litestream Slack][slack] or visit the [GitHub Discussions](https://github.com/benbjohnson/litestream/discussions).
-chat. 
+Please visit the [Litestream web site](https://litestream.io) for installation
+instructions and documentation.
 
 If you find this project interesting, please consider starring the project on
 GitHub.
 
+[slack]: https://join.slack.com/t/litestream/shared_invite/zt-n0j4s3ci-lx1JziR3bV6L2NMF723H3Q
 
-## Installation
 
-### Mac OS (Homebrew)
+## Acknowledgements
 
-To install from homebrew, run the following command:
+While the Litestream project does not accept external code patches, many
+of the most valuable contributions are in the forms of testing, feedback, and
+documentation. These help harden software and streamline usage for other users.
 
-```sh
+I want to give special thanks to individuals who invest much of their time and 
-$ brew install benbjohnson/litestream/litestream
+energy into the project to help make it better:
-```
 
+- Thanks to [Cory LaNou](https://twitter.com/corylanou) for giving early feedback and testing when Litestream was still pre-release.
+- Thanks to [Michael Lynch](https://github.com/mtlynch) for digging into issues and contributing to the documentation.
+- Thanks to [Kurt Mackey](https://twitter.com/mrkurt) for feedback and testing.
+- Thanks to [Sam Weston](https://twitter.com/cablespaghetti) for figuring out how to run Litestream on Kubernetes and writing up the docs for it.
+- Thanks to [Rafael](https://github.com/netstx) & [Jungle Boogie](https://github.com/jungle-boogie) for helping to get OpenBSD release builds working.
+- Thanks to [Simon Gottschlag](https://github.com/simongottschlag), [Marin](https://github.com/supermarin),[Victor Björklund](https://github.com/victorbjorklund), [Jonathan Beri](https://twitter.com/beriberikix) [Yuri](https://github.com/yurivish), [Nathan Probst](https://github.com/nprbst), [Yann Coleu](https://github.com/yanc0), and [Nicholas Grilly](https://twitter.com/ngrilly) for frequent feedback, testing, & support.
 
-### Linux (Debian)
+Huge thanks to fly.io for their support and for contributing credits for testing and development!
 
-You can download the `.deb` file from the [Releases page][releases] page and
-then run the following:
 
-```sh
+## Contribution Policy
-$ sudo dpkg -i litestream-v0.3.0-linux-amd64.deb
-```
 
-Once installed, you'll need to enable & start the service:
+Initially, Litestream was closed to outside contributions. The goal was to
+reduce burnout by limiting the maintenance overhead of reviewing and validating
+third-party code. However, this policy is overly broad and has prevented small,
+easily testable patches from being contributed.
 
-```sh
+Litestream is now open to code contributions for bug fixes only. Features carry
-$ sudo systemctl enable litestream
+a long-term maintenance burden so they will not be accepted at this time.
-$ sudo systemctl start litestream
+Please [submit an issue][new-issue] if you have a feature you'd like to
-```
+request.
 
+If you find mistakes in the documentation, please submit a fix to the
+[documentation repository][docs].
 
-### Release binaries
+[new-issue]: https://github.com/benbjohnson/litestream/issues/new
+[docs]: https://github.com/benbjohnson/litestream.io
 
-You can also download the release binary for your system from the
-[releases page][releases] and run it as a standalone application.
-
-
-### Building from source
-
-Download and install the [Go toolchain](https://golang.org/) and then run:
-
-```sh
-$ go install ./cmd/litestream
-```
-
-The `litestream` binary should be in your `$GOPATH/bin` folder.
-
-
-## Quick Start
-
-Litestream provides a configuration file that can be used for production
-deployments but you can also specify a single database and replica on the
-command line when trying it out.
-
-First, you'll need to create an S3 bucket that we'll call `"mybkt"` in this
-example. You'll also need to set your AWS credentials:
-
-```sh
-$ export AWS_ACCESS_KEY_ID=AKIAxxxxxxxxxxxxxxxx
-$ export AWS_SECRET_ACCESS_KEY=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx/xxxxxxxxx
-```
-
-Next, you can run the `litestream replicate` command with the path to the
-database you want to backup and the URL of your replica destination:
-
-```sh
-$ litestream replicate /path/to/db s3://mybkt/db
-```
-
-If you make changes to your local database, those changes will be replicated
-to S3 every 10 seconds. From another terminal window, you can restore your
-database from your S3 replica:
-
-```
-$ litestream restore -o /path/to/restored/db s3://mybkt/db
-```
-
-Voila! 🎉
-
-Your database should be restored to the last replicated state that
-was sent to S3. You can adjust your replication frequency and other options by
-using a configuration-based approach specified below.
-
-
-## Configuration
-
-A configuration-based install gives you more replication options. By default,
-the config file lives at `/etc/litestream.yml` but you can pass in a different
-path to any `litestream` command using the `-config PATH` flag. You can also 
-set the `LITESTREAM_CONFIG` environment variable to specify a new path.
-
-The configuration specifies one or more `dbs` and a list of one or more replica
-locations for each db. Below are some common configurations:
-
-### Replicate to S3
-
-This will replicate the database at `/path/to/db` to the `"/db"` path inside
-the S3 bucket named `"mybkt"`.
-
-```yaml
-access-key-id:     AKIAxxxxxxxxxxxxxxxx
-secret-access-key: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx/xxxxxxxxx
-
-dbs:
-  - path: /path/to/db
-    replicas:
-      - url: s3://mybkt/db
-```
-
-### Replicate to another file path
-
-This will replicate the database at `/path/to/db` to a directory named
-`/path/to/replica`.
-
-```yaml
-dbs:
-  - path: /path/to/db
-    replicas:
-      - path: /path/to/replica
-```
-
-### Retention period
-
-By default, replicas will retain a snapshot & subsequent WAL changes for 24
-hours. When the snapshot age exceeds the retention threshold, a new snapshot
-is taken and uploaded and the previous snapshot and WAL files are removed.
-
-You can configure this setting per-replica. Times are parsed using [Go's
-duration](https://golang.org/pkg/time/#ParseDuration) so time units of hours
-(`h`), minutes (`m`), and seconds (`s`) are allowed but days, weeks, months, and
-years are not.
-
-
-```yaml
-db:
-  - path: /path/to/db
-    replicas:
-      - url: s3://mybkt/db
-        retention: 1h          # 1 hour retention
-```
-
-
-### Monitoring replication
-
-You can also enable a Prometheus metrics endpoint to monitor replication by
-specifying a bind address with the `addr` field:
-
-```yml
-addr: ":9090"
-```
-
-This will make metrics available at: http://localhost:9090/metrics
-
-
-### Other configuration options
-
-These are some additional configuration options available on replicas:
-
-- `type`—Specify the type of replica (`"file"` or `"s3"`). Derived from `"path"`.
-- `name`—Specify an optional name for the replica if you are using multiple replicas.
-- `path`—File path to the replica location.
-- `url`—URL to the replica location.
-- `retention-check-interval`—Time between retention enforcement checks. Defaults to `1h`.
-- `validation-interval`—Interval between periodic checks to ensure restored backup matches current database. Disabled by default.
-
-These replica options are only available for S3 replicas:
-
-- `bucket`—S3 bucket name. Derived from `"path"`.
-- `region`—S3 bucket region. Looked up on startup if unspecified.
-- `sync-interval`—Replication sync frequency.
-
-
-
-## Usage
-
-### Replication
-
-Once your configuration is saved, you'll need to begin replication. If you
-installed the `.deb` file then run:
-
-```sh
-$ sudo systemctl restart litestream
-```
-
-To run litestream on its own, run:
-
-```sh
-# Replicate using the /etc/litestream.yml configuration.
-$ litestream replicate
-
-# Replicate using a different configuration path.
-$ litestream replicate -config /path/to/litestream.yml
-```
-
-The `litestream` command will initialize and then wait indefinitely for changes.
-You should see your destination replica path is now populated with a
-`generations` directory. Inside there should be a 16-character hex generation
-directory and inside there should be snapshots & WAL files. As you make changes
-to your source database, changes will be copied over to your replica incrementally.
-
-
-### Restoring a backup
-
-Litestream can restore a previous snapshot and replay all replicated WAL files.
-By default, it will restore up to the latest WAL file but you can also perform
-point-in-time restores.
-
-A database can only be restored to a path that does not exist so you don't need
-to worry about accidentally overwriting your current database.
-
-```sh
-# Restore database to original path.
-$ litestream restore /path/to/db
-
-# Restore database to a new location.
-$ litestream restore -o /path/to/restored/db /path/to/db
-
-# Restore from a replica URL.
-$ litestream restore -o /path/to/restored/db s3://mybkt/db
-
-# Restore database to a specific point-in-time.
-$ litestream restore -timestamp 2020-01-01T00:00:00Z /path/to/db
-```
-
-Point-in-time restores only have the resolution of the timestamp of the WAL file
-itself. By default, Litestream will start a new WAL file every minute so
-point-in-time restores are only accurate to the minute.
-
-
-
-## How it works
-
-SQLite provides a WAL (write-ahead log) journaling mode which writes pages to
-a `-wal` file before eventually being copied over to the original database file.
-This copying process is known as checkpointing. The WAL file works as a circular
-buffer so when the WAL reaches a certain size then it restarts from the beginning.
-
-Litestream works by taking over the checkpointing process and controlling when
-it is restarted to ensure that it copies every new page. Checkpointing is only
-allowed when there are no read transactions so Litestream maintains a
-long-running read transaction against each database until it is ready to
-checkpoint.
-
-The SQLite WAL file is copied to a separate location called the shadow WAL which
-ensures that it will not be overwritten by SQLite. This shadow WAL acts as a
-temporary buffer so that replicas can replicate to their destination (e.g.
-another file path or to S3). The shadow WAL files are removed once they have
-been fully replicated. You can find the shadow directory as a hidden directory
-next to your database file. If you database file is named `/var/lib/my.db` then
-the shadow directory will be `/var/lib/.my.db-litestream`.
-
-Litestream groups a snapshot and all subsequent WAL changes into "generations".
-A generation is started on initial replication of a database and a new
-generation will be started if litestream detects that the WAL replication is
-no longer contiguous. This can occur if the `litestream` process is stopped and
-another process is allowed to checkpoint the WAL.
-
-
-
-## Open-source, not open-contribution
-
-[Similar to SQLite](https://www.sqlite.org/copyright.html), Litestream is open
-source but closed to contributions. This keeps the code base free of proprietary
-or licensed code but it also helps me continue to maintain and build Litestream.
-
-As the author of [BoltDB](https://github.com/boltdb/bolt), I found that
-accepting and maintaining third party patches contributed to my burn out and
-I eventually archived the project. Writing databases & low-level replication
-tools involves nuance and simple one line changes can have profound and
-unexpected changes in correctness and performance. Small contributions
-typically required hours of my time to properly test and validate them.
-
-I am grateful for community involvement, bug reports, & feature requests. I do
-not wish to come off as anything but welcoming, however, I've
-made the decision to keep this project closed to contributions for my own
-mental health and long term viability of the project.
-
-
-[releases]: https://github.com/benbjohnson/litestream/releases

abs/replica_client.go

@@ -0,0 +1,558 @@
+package abs
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/url"
+	"os"
+	"path"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/Azure/azure-storage-blob-go/azblob"
+	"github.com/benbjohnson/litestream"
+	"github.com/benbjohnson/litestream/internal"
+	"golang.org/x/sync/errgroup"
+)
+
+// ReplicaClientType is the client type for this package.
+const ReplicaClientType = "abs"
+
+var _ litestream.ReplicaClient = (*ReplicaClient)(nil)
+
+// ReplicaClient is a client for writing snapshots & WAL segments to disk.
+type ReplicaClient struct {
+	mu           sync.Mutex
+	containerURL *azblob.ContainerURL
+
+	// Azure credentials
+	AccountName string
+	AccountKey  string
+	Endpoint    string
+
+	// Azure Blob Storage container information
+	Bucket string
+	Path   string
+}
+
+// NewReplicaClient returns a new instance of ReplicaClient.
+func NewReplicaClient() *ReplicaClient {
+	return &ReplicaClient{}
+}
+
+// Type returns "abs" as the client type.
+func (c *ReplicaClient) Type() string {
+	return ReplicaClientType
+}
+
+// Init initializes the connection to Azure. No-op if already initialized.
+func (c *ReplicaClient) Init(ctx context.Context) (err error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if c.containerURL != nil {
+		return nil
+	}
+
+	// Read account key from environment, if available.
+	accountKey := c.AccountKey
+	if accountKey == "" {
+		accountKey = os.Getenv("LITESTREAM_AZURE_ACCOUNT_KEY")
+	}
+
+	// Authenticate to ACS.
+	credential, err := azblob.NewSharedKeyCredential(c.AccountName, accountKey)
+	if err != nil {
+		return err
+	}
+
+	// Construct & parse endpoint unless already set.
+	endpoint := c.Endpoint
+	if endpoint == "" {
+		endpoint = fmt.Sprintf("https://%s.blob.core.windows.net", c.AccountName)
+	}
+	endpointURL, err := url.Parse(endpoint)
+	if err != nil {
+		return fmt.Errorf("cannot parse azure endpoint: %w", err)
+	}
+
+	// Build pipeline and reference to container.
+	pipeline := azblob.NewPipeline(credential, azblob.PipelineOptions{
+		Retry: azblob.RetryOptions{
+			TryTimeout: 24 * time.Hour,
+		},
+	})
+	containerURL := azblob.NewServiceURL(*endpointURL, pipeline).NewContainerURL(c.Bucket)
+	c.containerURL = &containerURL
+
+	return nil
+}
+
+// Generations returns a list of available generation names.
+func (c *ReplicaClient) Generations(ctx context.Context) ([]string, error) {
+	if err := c.Init(ctx); err != nil {
+		return nil, err
+	}
+
+	var generations []string
+	var marker azblob.Marker
+	for marker.NotDone() {
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "LIST").Inc()
+
+		resp, err := c.containerURL.ListBlobsHierarchySegment(ctx, marker, "/", azblob.ListBlobsSegmentOptions{
+			Prefix: path.Join(c.Path, "generations") + "/",
+		})
+		if err != nil {
+			return nil, err
+		}
+		marker = resp.NextMarker
+
+		for _, prefix := range resp.Segment.BlobPrefixes {
+			name := path.Base(strings.TrimSuffix(prefix.Name, "/"))
+			if !litestream.IsGenerationName(name) {
+				continue
+			}
+			generations = append(generations, name)
+		}
+	}
+
+	return generations, nil
+}
+
+// DeleteGeneration deletes all snapshots & WAL segments within a generation.
+func (c *ReplicaClient) DeleteGeneration(ctx context.Context, generation string) error {
+	if err := c.Init(ctx); err != nil {
+		return err
+	} else if generation == "" {
+		return fmt.Errorf("generation required")
+	}
+
+	prefix := path.Join(c.Path, "generations", generation) + "/"
+
+	var marker azblob.Marker
+	for marker.NotDone() {
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "LIST").Inc()
+
+		resp, err := c.containerURL.ListBlobsFlatSegment(ctx, marker, azblob.ListBlobsSegmentOptions{Prefix: prefix})
+		if err != nil {
+			return err
+		}
+		marker = resp.NextMarker
+
+		for _, item := range resp.Segment.BlobItems {
+			internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "DELETE").Inc()
+
+			blobURL := c.containerURL.NewBlobURL(item.Name)
+			if _, err := blobURL.Delete(ctx, azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{}); isNotExists(err) {
+				continue
+			} else if err != nil {
+				return err
+			}
+		}
+	}
+
+	// log.Printf("%s(%s): retainer: deleting generation: %s", r.db.Path(), r.Name(), generation)
+
+	return nil
+}
+
+// Snapshots returns an iterator over all available snapshots for a generation.
+func (c *ReplicaClient) Snapshots(ctx context.Context, generation string) (litestream.SnapshotIterator, error) {
+	if err := c.Init(ctx); err != nil {
+		return nil, err
+	}
+	return newSnapshotIterator(ctx, generation, c), nil
+}
+
+// WriteSnapshot writes LZ4 compressed data from rd to the object storage.
+func (c *ReplicaClient) WriteSnapshot(ctx context.Context, generation string, index int, rd io.Reader) (info litestream.SnapshotInfo, err error) {
+	if err := c.Init(ctx); err != nil {
+		return info, err
+	} else if generation == "" {
+		return info, fmt.Errorf("generation required")
+	}
+
+	key := path.Join(c.Path, "generations", generation, "snapshots", litestream.FormatIndex(index)+".snapshot.lz4")
+	startTime := time.Now()
+
+	rc := internal.NewReadCounter(rd)
+
+	blobURL := c.containerURL.NewBlockBlobURL(key)
+	if _, err := azblob.UploadStreamToBlockBlob(ctx, rc, blobURL, azblob.UploadStreamToBlockBlobOptions{
+		BlobHTTPHeaders: azblob.BlobHTTPHeaders{ContentType: "application/octet-stream"},
+		BlobAccessTier:  azblob.DefaultAccessTier,
+	}); err != nil {
+		return info, err
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "PUT").Inc()
+	internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "PUT").Add(float64(rc.N()))
+
+	return litestream.SnapshotInfo{
+		Generation: generation,
+		Index:      index,
+		Size:       rc.N(),
+		CreatedAt:  startTime.UTC(),
+	}, nil
+}
+
+// SnapshotReader returns a reader for snapshot data at the given generation/index.
+func (c *ReplicaClient) SnapshotReader(ctx context.Context, generation string, index int) (io.ReadCloser, error) {
+	if err := c.Init(ctx); err != nil {
+		return nil, err
+	} else if generation == "" {
+		return nil, fmt.Errorf("generation required")
+	}
+
+	key := path.Join(c.Path, "generations", generation, "snapshots", litestream.FormatIndex(index)+".snapshot.lz4")
+
+	blobURL := c.containerURL.NewBlobURL(key)
+	resp, err := blobURL.Download(ctx, 0, 0, azblob.BlobAccessConditions{}, false, azblob.ClientProvidedKeyOptions{})
+	if isNotExists(err) {
+		return nil, os.ErrNotExist
+	} else if err != nil {
+		return nil, fmt.Errorf("cannot start new reader for %q: %w", key, err)
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "GET").Inc()
+	internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "GET").Add(float64(resp.ContentLength()))
+
+	return resp.Body(azblob.RetryReaderOptions{}), nil
+}
+
+// DeleteSnapshot deletes a snapshot with the given generation & index.
+func (c *ReplicaClient) DeleteSnapshot(ctx context.Context, generation string, index int) error {
+	if err := c.Init(ctx); err != nil {
+		return err
+	} else if generation == "" {
+		return fmt.Errorf("generation required")
+	}
+
+	key := path.Join(c.Path, "generations", generation, "snapshots", litestream.FormatIndex(index)+".snapshot.lz4")
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "DELETE").Inc()
+
+	blobURL := c.containerURL.NewBlobURL(key)
+	if _, err := blobURL.Delete(ctx, azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{}); isNotExists(err) {
+		return nil
+	} else if err != nil {
+		return fmt.Errorf("cannot delete snapshot %q: %w", key, err)
+	}
+	return nil
+}
+
+// WALSegments returns an iterator over all available WAL files for a generation.
+func (c *ReplicaClient) WALSegments(ctx context.Context, generation string) (litestream.WALSegmentIterator, error) {
+	if err := c.Init(ctx); err != nil {
+		return nil, err
+	}
+	return newWALSegmentIterator(ctx, generation, c), nil
+}
+
+// WriteWALSegment writes LZ4 compressed data from rd into a file on disk.
+func (c *ReplicaClient) WriteWALSegment(ctx context.Context, pos litestream.Pos, rd io.Reader) (info litestream.WALSegmentInfo, err error) {
+	if err := c.Init(ctx); err != nil {
+		return info, err
+	} else if pos.Generation == "" {
+		return info, fmt.Errorf("generation required")
+	}
+
+	key := path.Join(c.Path, "generations", pos.Generation, "wal", litestream.FormatIndex(pos.Index), litestream.FormatOffset(pos.Offset)+".wal.lz4")
+	startTime := time.Now()
+
+	rc := internal.NewReadCounter(rd)
+
+	blobURL := c.containerURL.NewBlockBlobURL(key)
+	if _, err := azblob.UploadStreamToBlockBlob(ctx, rc, blobURL, azblob.UploadStreamToBlockBlobOptions{
+		BlobHTTPHeaders: azblob.BlobHTTPHeaders{ContentType: "application/octet-stream"},
+		BlobAccessTier:  azblob.DefaultAccessTier,
+	}); err != nil {
+		return info, err
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "PUT").Inc()
+	internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "PUT").Add(float64(rc.N()))
+
+	return litestream.WALSegmentInfo{
+		Generation: pos.Generation,
+		Index:      pos.Index,
+		Offset:     pos.Offset,
+		Size:       rc.N(),
+		CreatedAt:  startTime.UTC(),
+	}, nil
+}
+
+// WALSegmentReader returns a reader for a section of WAL data at the given index.
+// Returns os.ErrNotExist if no matching index/offset is found.
+func (c *ReplicaClient) WALSegmentReader(ctx context.Context, pos litestream.Pos) (io.ReadCloser, error) {
+	if err := c.Init(ctx); err != nil {
+		return nil, err
+	} else if pos.Generation == "" {
+		return nil, fmt.Errorf("generation required")
+	}
+
+	key := path.Join(c.Path, "generations", pos.Generation, "wal", litestream.FormatIndex(pos.Index), litestream.FormatOffset(pos.Offset)+".wal.lz4")
+
+	blobURL := c.containerURL.NewBlobURL(key)
+	resp, err := blobURL.Download(ctx, 0, 0, azblob.BlobAccessConditions{}, false, azblob.ClientProvidedKeyOptions{})
+	if isNotExists(err) {
+		return nil, os.ErrNotExist
+	} else if err != nil {
+		return nil, fmt.Errorf("cannot start new reader for %q: %w", key, err)
+	}
+
+	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "GET").Inc()
+	internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "GET").Add(float64(resp.ContentLength()))
+
+	return resp.Body(azblob.RetryReaderOptions{}), nil
+}
+
+// DeleteWALSegments deletes WAL segments with at the given positions.
+func (c *ReplicaClient) DeleteWALSegments(ctx context.Context, a []litestream.Pos) error {
+	if err := c.Init(ctx); err != nil {
+		return err
+	}
+
+	for _, pos := range a {
+		if pos.Generation == "" {
+			return fmt.Errorf("generation required")
+		}
+
+		key := path.Join(c.Path, "generations", pos.Generation, "wal", litestream.FormatIndex(pos.Index), litestream.FormatOffset(pos.Offset)+".wal.lz4")
+
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "DELETE").Inc()
+
+		blobURL := c.containerURL.NewBlobURL(key)
+		if _, err := blobURL.Delete(ctx, azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{}); isNotExists(err) {
+			continue
+		} else if err != nil {
+			return fmt.Errorf("cannot delete wal segment %q: %w", key, err)
+		}
+	}
+
+	return nil
+}
+
+type snapshotIterator struct {
+	client     *ReplicaClient
+	generation string
+
+	ch     chan litestream.SnapshotInfo
+	g      errgroup.Group
+	ctx    context.Context
+	cancel func()
+
+	info litestream.SnapshotInfo
+	err  error
+}
+
+func newSnapshotIterator(ctx context.Context, generation string, client *ReplicaClient) *snapshotIterator {
+	itr := &snapshotIterator{
+		client:     client,
+		generation: generation,
+		ch:         make(chan litestream.SnapshotInfo),
+	}
+
+	itr.ctx, itr.cancel = context.WithCancel(ctx)
+	itr.g.Go(itr.fetch)
+
+	return itr
+}
+
+// fetch runs in a separate goroutine to fetch pages of objects and stream them to a channel.
+func (itr *snapshotIterator) fetch() error {
+	defer close(itr.ch)
+
+	if itr.generation == "" {
+		return fmt.Errorf("generation required")
+	}
+
+	prefix := path.Join(itr.client.Path, "generations", itr.generation) + "/"
+
+	var marker azblob.Marker
+	for marker.NotDone() {
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "LIST").Inc()
+
+		resp, err := itr.client.containerURL.ListBlobsFlatSegment(itr.ctx, marker, azblob.ListBlobsSegmentOptions{Prefix: prefix})
+		if err != nil {
+			return err
+		}
+		marker = resp.NextMarker
+
+		for _, item := range resp.Segment.BlobItems {
+			index, err := internal.ParseSnapshotPath(path.Base(item.Name))
+			if err != nil {
+				continue
+			}
+
+			info := litestream.SnapshotInfo{
+				Generation: itr.generation,
+				Index:      index,
+				Size:       *item.Properties.ContentLength,
+				CreatedAt:  item.Properties.CreationTime.UTC(),
+			}
+
+			select {
+			case <-itr.ctx.Done():
+			case itr.ch <- info:
+			}
+		}
+	}
+	return nil
+}
+
+func (itr *snapshotIterator) Close() (err error) {
+	err = itr.err
+
+	// Cancel context and wait for error group to finish.
+	itr.cancel()
+	if e := itr.g.Wait(); e != nil && err == nil {
+		err = e
+	}
+
+	return err
+}
+
+func (itr *snapshotIterator) Next() bool {
+	// Exit if an error has already occurred.
+	if itr.err != nil {
+		return false
+	}
+
+	// Return false if context was canceled or if there are no more snapshots.
+	// Otherwise fetch the next snapshot and store it on the iterator.
+	select {
+	case <-itr.ctx.Done():
+		return false
+	case info, ok := <-itr.ch:
+		if !ok {
+			return false
+		}
+		itr.info = info
+		return true
+	}
+}
+
+func (itr *snapshotIterator) Err() error { return itr.err }
+
+func (itr *snapshotIterator) Snapshot() litestream.SnapshotInfo {
+	return itr.info
+}
+
+type walSegmentIterator struct {
+	client     *ReplicaClient
+	generation string
+
+	ch     chan litestream.WALSegmentInfo
+	g      errgroup.Group
+	ctx    context.Context
+	cancel func()
+
+	info litestream.WALSegmentInfo
+	err  error
+}
+
+func newWALSegmentIterator(ctx context.Context, generation string, client *ReplicaClient) *walSegmentIterator {
+	itr := &walSegmentIterator{
+		client:     client,
+		generation: generation,
+		ch:         make(chan litestream.WALSegmentInfo),
+	}
+
+	itr.ctx, itr.cancel = context.WithCancel(ctx)
+	itr.g.Go(itr.fetch)
+
+	return itr
+}
+
+// fetch runs in a separate goroutine to fetch pages of objects and stream them to a channel.
+func (itr *walSegmentIterator) fetch() error {
+	defer close(itr.ch)
+
+	if itr.generation == "" {
+		return fmt.Errorf("generation required")
+	}
+	prefix := path.Join(itr.client.Path, "generations", itr.generation, "wal")
+
+	var marker azblob.Marker
+	for marker.NotDone() {
+		internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "LIST").Inc()
+
+		resp, err := itr.client.containerURL.ListBlobsFlatSegment(itr.ctx, marker, azblob.ListBlobsSegmentOptions{Prefix: prefix})
+		if err != nil {
+			return err
+		}
+		marker = resp.NextMarker
+
+		for _, item := range resp.Segment.BlobItems {
+			key := strings.TrimPrefix(item.Name, prefix+"/")
+			index, offset, err := internal.ParseWALSegmentPath(key)
+			if err != nil {
+				continue
+			}
+
+			info := litestream.WALSegmentInfo{
+				Generation: itr.generation,
+				Index:      index,
+				Offset:     offset,
+				Size:       *item.Properties.ContentLength,
+				CreatedAt:  item.Properties.CreationTime.UTC(),
+			}
+
+			select {
+			case <-itr.ctx.Done():
+			case itr.ch <- info:
+			}
+		}
+	}
+	return nil
+}
+
+func (itr *walSegmentIterator) Close() (err error) {
+	err = itr.err
+
+	// Cancel context and wait for error group to finish.
+	itr.cancel()
+	if e := itr.g.Wait(); e != nil && err == nil {
+		err = e
+	}
+
+	return err
+}
+
+func (itr *walSegmentIterator) Next() bool {
+	// Exit if an error has already occurred.
+	if itr.err != nil {
+		return false
+	}
+
+	// Return false if context was canceled or if there are no more segments.
+	// Otherwise fetch the next segment and store it on the iterator.
+	select {
+	case <-itr.ctx.Done():
+		return false
+	case info, ok := <-itr.ch:
+		if !ok {
+			return false
+		}
+		itr.info = info
+		return true
+	}
+}
+
+func (itr *walSegmentIterator) Err() error { return itr.err }
+
+func (itr *walSegmentIterator) WALSegment() litestream.WALSegmentInfo {
+	return itr.info
+}
+
+func isNotExists(err error) bool {
+	switch err := err.(type) {
+	case azblob.StorageError:
+		return err.ServiceCode() == azblob.ServiceCodeBlobNotFound
+	default:
+		return false
+	}
+}

cmd/litestream/Makefile

@@ -0,0 +1,6 @@
+.PHONY: default
+default:
+
+.PHONY: testdata
+testdata:
+	make -C testdata

cmd/litestream/databases.go

@@ -2,45 +2,59 @@ package main
 
 import (
 	"context"
-	"errors"
 	"flag"
 	"fmt"
-	"os"
+	"io"
 	"strings"
 	"text/tabwriter"
 )
 
 // DatabasesCommand is a command for listing managed databases.
-type DatabasesCommand struct{}
+type DatabasesCommand struct {
+	stdin  io.Reader
+	stdout io.Writer
+	stderr io.Writer
+
+	configPath  string
+	noExpandEnv bool
+}
+
+// NewDatabasesCommand returns a new instance of DatabasesCommand.
+func NewDatabasesCommand(stdin io.Reader, stdout, stderr io.Writer) *DatabasesCommand {
+	return &DatabasesCommand{
+		stdin:  stdin,
+		stdout: stdout,
+		stderr: stderr,
+	}
+}
 
 // Run executes the command.
 func (c *DatabasesCommand) Run(ctx context.Context, args []string) (err error) {
-	var configPath string
 	fs := flag.NewFlagSet("litestream-databases", flag.ContinueOnError)
-	registerConfigFlag(fs, &configPath)
+	registerConfigFlag(fs, &c.configPath, &c.noExpandEnv)
 	fs.Usage = c.Usage
 	if err := fs.Parse(args); err != nil {
 		return err
 	} else if fs.NArg() != 0 {
-		return fmt.Errorf("too many argument")
+		return fmt.Errorf("too many arguments")
 	}
 
 	// Load configuration.
-	if configPath == "" {
+	config, err := ReadConfigFile(c.configPath, !c.noExpandEnv)
-		return errors.New("-config required")
-	}
-	config, err := ReadConfigFile(configPath)
 	if err != nil {
 		return err
+	} else if len(config.DBs) == 0 {
+		fmt.Fprintln(c.stdout, "No databases found in config file.")
+		return nil
 	}
 
 	// List all databases.
-	w := tabwriter.NewWriter(os.Stdout, 0, 8, 2, ' ', 0)
+	w := tabwriter.NewWriter(c.stdout, 0, 8, 2, ' ', 0)
 	defer w.Flush()
 
 	fmt.Fprintln(w, "path\treplicas")
 	for _, dbConfig := range config.DBs {
-		db, err := newDBFromConfig(&config, dbConfig)
+		db, err := NewDBFromConfig(dbConfig)
 		if err != nil {
 			return err
 		}
@@ -61,7 +75,7 @@ func (c *DatabasesCommand) Run(ctx context.Context, args []string) (err error) {
 
 // Usage prints the help screen to STDOUT.
 func (c *DatabasesCommand) Usage() {
-	fmt.Printf(`
+	fmt.Fprintf(c.stdout, `
 The databases command lists all databases in the configuration file.
 
 Usage:
@@ -74,6 +88,9 @@ Arguments:
 	    Specifies the configuration file.
 	    Defaults to %s
 
+	-no-expand-env
+	    Disables environment variable expansion in configuration file.
+
 `[1:],
 		DefaultConfigPath(),
 	)

cmd/litestream/databases_test.go

@@ -0,0 +1,66 @@
+package main_test
+
+import (
+	"context"
+	"flag"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/benbjohnson/litestream/internal/testingutil"
+)
+
+func TestDatabasesCommand(t *testing.T) {
+	t.Run("OK", func(t *testing.T) {
+		testDir := filepath.Join("testdata", "databases", "ok")
+		m, _, stdout, _ := newMain()
+		if err := m.Run(context.Background(), []string{"databases", "-config", filepath.Join(testDir, "litestream.yml")}); err != nil {
+			t.Fatal(err)
+		} else if got, want := stdout.String(), string(testingutil.ReadFile(t, filepath.Join(testDir, "stdout"))); got != want {
+			t.Fatalf("stdout=%q, want %q", got, want)
+		}
+	})
+
+	t.Run("NoDatabases", func(t *testing.T) {
+		testDir := filepath.Join("testdata", "databases", "no-databases")
+		m, _, stdout, _ := newMain()
+		if err := m.Run(context.Background(), []string{"databases", "-config", filepath.Join(testDir, "litestream.yml")}); err != nil {
+			t.Fatal(err)
+		} else if got, want := stdout.String(), string(testingutil.ReadFile(t, filepath.Join(testDir, "stdout"))); got != want {
+			t.Fatalf("stdout=%q, want %q", got, want)
+		}
+	})
+
+	t.Run("ErrConfigNotFound", func(t *testing.T) {
+		testDir := filepath.Join("testdata", "databases", "no-config")
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"databases", "-config", filepath.Join(testDir, "litestream.yml")})
+		if err == nil || !strings.Contains(err.Error(), `config file not found:`) {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrInvalidConfig", func(t *testing.T) {
+		testDir := filepath.Join("testdata", "databases", "invalid-config")
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"databases", "-config", filepath.Join(testDir, "litestream.yml")})
+		if err == nil || !strings.Contains(err.Error(), `replica path cannot be a url`) {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrTooManyArguments", func(t *testing.T) {
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"databases", "xyz"})
+		if err == nil || err.Error() != `too many arguments` {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("Usage", func(t *testing.T) {
+		m, _, _, _ := newMain()
+		if err := m.Run(context.Background(), []string{"databases", "-h"}); err != flag.ErrHelp {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+}

cmd/litestream/generations.go

@@ -2,116 +2,118 @@ package main
 
 import (
 	"context"
-	"errors"
 	"flag"
 	"fmt"
-	"log"
+	"io"
 	"os"
 	"text/tabwriter"
 	"time"
 
 	"github.com/benbjohnson/litestream"
+	"github.com/benbjohnson/litestream/internal"
 )
 
 // GenerationsCommand represents a command to list all generations for a database.
-type GenerationsCommand struct{}
+type GenerationsCommand struct {
+	stdin  io.Reader
+	stdout io.Writer
+	stderr io.Writer
+
+	configPath  string
+	noExpandEnv bool
+
+	replicaName string
+}
+
+// NewGenerationsCommand returns a new instance of GenerationsCommand.
+func NewGenerationsCommand(stdin io.Reader, stdout, stderr io.Writer) *GenerationsCommand {
+	return &GenerationsCommand{
+		stdin:  stdin,
+		stdout: stdout,
+		stderr: stderr,
+	}
+}
 
 // Run executes the command.
-func (c *GenerationsCommand) Run(ctx context.Context, args []string) (err error) {
+func (c *GenerationsCommand) Run(ctx context.Context, args []string) (ret error) {
-	var configPath string
 	fs := flag.NewFlagSet("litestream-generations", flag.ContinueOnError)
-	registerConfigFlag(fs, &configPath)
+	registerConfigFlag(fs, &c.configPath, &c.noExpandEnv)
-	replicaName := fs.String("replica", "", "replica name")
+	fs.StringVar(&c.replicaName, "replica", "", "replica name")
 	fs.Usage = c.Usage
 	if err := fs.Parse(args); err != nil {
 		return err
-	} else if fs.NArg() == 0 || fs.Arg(0) == "" {
+	} else if fs.Arg(0) == "" {
 		return fmt.Errorf("database path or replica URL required")
 	} else if fs.NArg() > 1 {
 		return fmt.Errorf("too many arguments")
 	}
 
-	var db *litestream.DB
+	// Load configuration.
-	var r litestream.Replica
+	config, err := ReadConfigFile(c.configPath, !c.noExpandEnv)
-	updatedAt := time.Now()
+	if err != nil {
-	if isURL(fs.Arg(0)) {
+		return err
-		if r, err = NewReplicaFromURL(fs.Arg(0)); err != nil {
-			return err
-		}
-	} else if configPath != "" {
-		// Load configuration.
-		config, err := ReadConfigFile(configPath)
-		if err != nil {
-			return err
-		}
-
-		// Lookup database from configuration file by path.
-		if path, err := expand(fs.Arg(0)); err != nil {
-			return err
-		} else if dbc := config.DBConfig(path); dbc == nil {
-			return fmt.Errorf("database not found in config: %s", path)
-		} else if db, err = newDBFromConfig(&config, dbc); err != nil {
-			return err
-		}
-
-		// Filter by replica, if specified.
-		if *replicaName != "" {
-			if r = db.Replica(*replicaName); r == nil {
-				return fmt.Errorf("replica %q not found for database %q", *replicaName, db.Path())
-			}
-		}
-
-		// Determine last time database or WAL was updated.
-		if updatedAt, err = db.UpdatedAt(); err != nil {
-			return err
-		}
-	} else {
-		return errors.New("config path or replica URL required")
 	}
 
-	var replicas []litestream.Replica
+	replicas, db, err := loadReplicas(ctx, config, fs.Arg(0), c.replicaName)
-	if r != nil {
+	if err != nil {
-		replicas = []litestream.Replica{r}
+		return err
-	} else {
+	}
-		replicas = db.Replicas
+
+	// Determine last time database or WAL was updated.
+	var dbUpdatedAt time.Time
+	if db != nil {
+		if dbUpdatedAt, err = db.UpdatedAt(); err != nil && !os.IsNotExist(err) {
+			return err
+		}
 	}
 
 	// List each generation.
-	w := tabwriter.NewWriter(os.Stdout, 0, 8, 2, ' ', 0)
+	w := tabwriter.NewWriter(c.stdout, 0, 8, 2, ' ', 0)
 	defer w.Flush()
 
 	fmt.Fprintln(w, "name\tgeneration\tlag\tstart\tend")
+
 	for _, r := range replicas {
-		generations, err := r.Generations(ctx)
+		generations, err := r.Client().Generations(ctx)
 		if err != nil {
-			log.Printf("%s: cannot list generations: %s", r.Name(), err)
+			fmt.Fprintf(c.stderr, "%s: cannot list generations: %s", r.Name(), err)
+			ret = errExit // signal error return without printing message
 			continue
 		}
 
 		// Iterate over each generation for the replica.
 		for _, generation := range generations {
-			stats, err := r.GenerationStats(ctx, generation)
+			createdAt, updatedAt, err := litestream.GenerationTimeBounds(ctx, r.Client(), generation)
 			if err != nil {
-				log.Printf("%s: cannot find generation stats: %s", r.Name(), err)
+				fmt.Fprintf(c.stderr, "%s: cannot determine generation time bounds: %s", r.Name(), err)
+				ret = errExit // signal error return without printing message
 				continue
 			}
 
+			// Calculate lag from database mod time to the replica mod time.
+			// This is ignored if the database mod time is unavailable such as
+			// when specifying the replica URL or if the database file is missing.
+			lag := "-"
+			if !dbUpdatedAt.IsZero() {
+				lag = internal.TruncateDuration(dbUpdatedAt.Sub(updatedAt)).String()
+			}
+
 			fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n",
 				r.Name(),
 				generation,
-				truncateDuration(updatedAt.Sub(stats.UpdatedAt)).String(),
+				lag,
-				stats.CreatedAt.Format(time.RFC3339),
+				createdAt.Format(time.RFC3339),
-				stats.UpdatedAt.Format(time.RFC3339),
+				updatedAt.Format(time.RFC3339),
 			)
 		}
 	}
 
-	return nil
+	return ret
 }
 
 // Usage prints the help message to STDOUT.
 func (c *GenerationsCommand) Usage() {
-	fmt.Printf(`
+	fmt.Fprintf(c.stdout, `
 The generations command lists all generations for a database or replica. It also
 lists stats about their lag behind the primary database and the time range they
 cover.
@@ -128,6 +130,9 @@ Arguments:
 	    Specifies the configuration file.
 	    Defaults to %s
 
+	-no-expand-env
+	    Disables environment variable expansion in configuration file.
+
 	-replica NAME
 	    Optional, filters by replica.
 
@@ -135,29 +140,3 @@ Arguments:
 		DefaultConfigPath(),
 	)
 }
-
-func truncateDuration(d time.Duration) time.Duration {
-	if d < 0 {
-		if d < -10*time.Second {
-			return d.Truncate(time.Second)
-		} else if d < -time.Second {
-			return d.Truncate(time.Second / 10)
-		} else if d < -time.Millisecond {
-			return d.Truncate(time.Millisecond)
-		} else if d < -time.Microsecond {
-			return d.Truncate(time.Microsecond)
-		}
-		return d
-	}
-
-	if d > 10*time.Second {
-		return d.Truncate(time.Second)
-	} else if d > time.Second {
-		return d.Truncate(time.Second / 10)
-	} else if d > time.Millisecond {
-		return d.Truncate(time.Millisecond)
-	} else if d > time.Microsecond {
-		return d.Truncate(time.Microsecond)
-	}
-	return d
-}

cmd/litestream/generations_test.go

@@ -0,0 +1,140 @@
+package main_test
+
+import (
+	"context"
+	"flag"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/benbjohnson/litestream/internal/testingutil"
+)
+
+func TestGenerationsCommand(t *testing.T) {
+	t.Run("OK", func(t *testing.T) {
+		testDir := filepath.Join("testdata", "generations", "ok")
+		defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
+
+		m, _, stdout, _ := newMain()
+		if err := m.Run(context.Background(), []string{"generations", "-config", filepath.Join(testDir, "litestream.yml"), filepath.Join(testDir, "db")}); err != nil {
+			t.Fatal(err)
+		} else if got, want := stdout.String(), string(testingutil.ReadFile(t, filepath.Join(testDir, "stdout"))); got != want {
+			t.Fatalf("stdout=%q, want %q", got, want)
+		}
+	})
+
+	t.Run("ReplicaName", func(t *testing.T) {
+		testDir := filepath.Join("testdata", "generations", "replica-name")
+		defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
+
+		m, _, stdout, _ := newMain()
+		if err := m.Run(context.Background(), []string{"generations", "-config", filepath.Join(testDir, "litestream.yml"), "-replica", "replica1", filepath.Join(testDir, "db")}); err != nil {
+			t.Fatal(err)
+		} else if got, want := stdout.String(), string(testingutil.ReadFile(t, filepath.Join(testDir, "stdout"))); got != want {
+			t.Fatalf("stdout=%q, want %q", got, want)
+		}
+	})
+
+	t.Run("ReplicaURL", func(t *testing.T) {
+		testDir := filepath.Join(testingutil.Getwd(t), "testdata", "generations", "replica-url")
+		defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
+		replicaURL := "file://" + filepath.ToSlash(testDir) + "/replica"
+
+		m, _, stdout, _ := newMain()
+		if err := m.Run(context.Background(), []string{"generations", replicaURL}); err != nil {
+			t.Fatal(err)
+		} else if got, want := stdout.String(), string(testingutil.ReadFile(t, filepath.Join(testDir, "stdout"))); got != want {
+			t.Fatalf("stdout=%q, want %q", got, want)
+		}
+	})
+
+	t.Run("NoDatabase", func(t *testing.T) {
+		testDir := filepath.Join("testdata", "generations", "no-database")
+		defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
+
+		m, _, stdout, _ := newMain()
+		if err := m.Run(context.Background(), []string{"generations", "-config", filepath.Join(testDir, "litestream.yml"), filepath.Join(testDir, "db")}); err != nil {
+			t.Fatal(err)
+		} else if got, want := stdout.String(), string(testingutil.ReadFile(t, filepath.Join(testDir, "stdout"))); got != want {
+			t.Fatalf("stdout=%q, want %q", got, want)
+		}
+	})
+
+	t.Run("ErrDatabaseOrReplicaRequired", func(t *testing.T) {
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"generations"})
+		if err == nil || err.Error() != `database path or replica URL required` {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrTooManyArguments", func(t *testing.T) {
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"generations", "abc", "123"})
+		if err == nil || err.Error() != `too many arguments` {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrInvalidFlags", func(t *testing.T) {
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"generations", "-no-such-flag"})
+		if err == nil || err.Error() != `flag provided but not defined: -no-such-flag` {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrConfigFileNotFound", func(t *testing.T) {
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"generations", "-config", "/no/such/file", "/var/lib/db"})
+		if err == nil || err.Error() != `config file not found: /no/such/file` {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrInvalidConfig", func(t *testing.T) {
+		testDir := filepath.Join("testdata", "generations", "invalid-config")
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"generations", "-config", filepath.Join(testDir, "litestream.yml"), "/var/lib/db"})
+		if err == nil || !strings.Contains(err.Error(), `replica path cannot be a url`) {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrDatabaseNotFound", func(t *testing.T) {
+		testDir := filepath.Join("testdata", "generations", "database-not-found")
+		defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
+
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"generations", "-config", filepath.Join(testDir, "litestream.yml"), "/no/such/db"})
+		if err == nil || err.Error() != `database not found in config: /no/such/db` {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrReplicaNotFound", func(t *testing.T) {
+		testDir := filepath.Join(testingutil.Getwd(t), "testdata", "generations", "replica-not-found")
+		defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
+
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"generations", "-config", filepath.Join(testDir, "litestream.yml"), "-replica", "no_such_replica", filepath.Join(testDir, "db")})
+		if err == nil || err.Error() != `replica "no_such_replica" not found for database "`+filepath.Join(testDir, "db")+`"` {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrInvalidReplicaURL", func(t *testing.T) {
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"generations", "xyz://xyz"})
+		if err == nil || !strings.Contains(err.Error(), `unknown replica type in config: "xyz"`) {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("Usage", func(t *testing.T) {
+		m, _, _, _ := newMain()
+		if err := m.Run(context.Background(), []string{"generations", "-h"}); err != flag.ErrHelp {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+}

cmd/litestream/main.go

@@ -2,20 +2,30 @@ package main
 
 import (
 	"context"
+	"errors"
 	"flag"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"log"
 	"net/url"
 	"os"
+	"os/signal"
 	"os/user"
 	"path"
 	"path/filepath"
+	"regexp"
+	"strconv"
 	"strings"
+	"syscall"
 	"time"
 
 	"github.com/benbjohnson/litestream"
+	"github.com/benbjohnson/litestream/abs"
+	"github.com/benbjohnson/litestream/gs"
+	"github.com/benbjohnson/litestream/http"
 	"github.com/benbjohnson/litestream/s3"
+	"github.com/benbjohnson/litestream/sftp"
 	_ "github.com/mattn/go-sqlite3"
 	"gopkg.in/yaml.v2"
 )
@@ -25,28 +35,44 @@ var (
 	Version = "(development build)"
 )
 
+// errExit is a terminal error for indicating program should quit.
+var errExit = errors.New("exit")
+
 func main() {
 	log.SetFlags(0)
+	log.SetOutput(os.Stdout)
 
-	m := NewMain()
+	m := NewMain(os.Stdin, os.Stdout, os.Stderr)
-	if err := m.Run(context.Background(), os.Args[1:]); err == flag.ErrHelp {
+	if err := m.Run(context.Background(), os.Args[1:]); err == flag.ErrHelp || err == errExit {
 		os.Exit(1)
 	} else if err != nil {
-		fmt.Fprintln(os.Stderr, err)
+		log.Println(err)
 		os.Exit(1)
 	}
 }
 
 // Main represents the main program execution.
-type Main struct{}
+type Main struct {
+	stdin  io.Reader
+	stdout io.Writer
+	stderr io.Writer
+}
 
 // NewMain returns a new instance of Main.
-func NewMain() *Main {
+func NewMain(stdin io.Reader, stdout, stderr io.Writer) *Main {
-	return &Main{}
+	return &Main{
+		stdin:  stdin,
+		stdout: stdout,
+		stderr: stderr,
+	}
 }
 
 // Run executes the program.
 func (m *Main) Run(ctx context.Context, args []string) (err error) {
+	// Copy "LITESTEAM" environment credentials.
+	applyLitestreamEnv()
+
+	// Extract command name.
 	var cmd string
 	if len(args) > 0 {
 		cmd, args = args[0], args[1:]
@@ -54,19 +80,64 @@ func (m *Main) Run(ctx context.Context, args []string) (err error) {
 
 	switch cmd {
 	case "databases":
-		return (&DatabasesCommand{}).Run(ctx, args)
+		return NewDatabasesCommand(m.stdin, m.stdout, m.stderr).Run(ctx, args)
 	case "generations":
-		return (&GenerationsCommand{}).Run(ctx, args)
+		return NewGenerationsCommand(m.stdin, m.stdout, m.stderr).Run(ctx, args)
 	case "replicate":
-		return (&ReplicateCommand{}).Run(ctx, args)
+		c := NewReplicateCommand(m.stdin, m.stdout, m.stderr)
+		if err := c.ParseFlags(ctx, args); err != nil {
+			return err
+		}
+
+		// Setup signal handler.
+		ctx, cancel := context.WithCancel(ctx)
+		defer cancel()
+		signalCh := make(chan os.Signal, 1)
+		signal.Notify(signalCh, syscall.SIGINT, syscall.SIGTERM)
+
+		if err := c.Run(ctx); err != nil {
+			return err
+		}
+
+		// Wait for signal to stop program.
+		select {
+		case <-ctx.Done():
+			fmt.Fprintln(m.stdout, "context done, litestream shutting down")
+		case err = <-c.execCh:
+			cancel()
+			fmt.Fprintln(m.stdout, "subprocess exited, litestream shutting down")
+		case sig := <-signalCh:
+			cancel()
+			fmt.Fprintln(m.stdout, "signal received, litestream shutting down")
+
+			if c.cmd != nil {
+				fmt.Fprintln(m.stdout, "sending signal to exec process")
+				if err := c.cmd.Process.Signal(sig); err != nil {
+					return fmt.Errorf("cannot signal exec process: %w", err)
+				}
+
+				fmt.Fprintln(m.stdout, "waiting for exec process to close")
+				if err := <-c.execCh; err != nil && !strings.HasPrefix(err.Error(), "signal:") {
+					return fmt.Errorf("cannot wait for exec process: %w", err)
+				}
+			}
+		}
+
+		// Gracefully close.
+		if e := c.Close(); e != nil && err == nil {
+			err = e
+		}
+		fmt.Fprintln(m.stdout, "litestream shut down")
+		return err
+
 	case "restore":
-		return (&RestoreCommand{}).Run(ctx, args)
+		return NewRestoreCommand(m.stdin, m.stdout, m.stderr).Run(ctx, args)
 	case "snapshots":
-		return (&SnapshotsCommand{}).Run(ctx, args)
+		return NewSnapshotsCommand(m.stdin, m.stdout, m.stderr).Run(ctx, args)
 	case "version":
-		return (&VersionCommand{}).Run(ctx, args)
+		return NewVersionCommand(m.stdin, m.stdout, m.stderr).Run(ctx, args)
 	case "wal":
-		return (&WALCommand{}).Run(ctx, args)
+		return NewWALCommand(m.stdin, m.stdout, m.stderr).Run(ctx, args)
 	default:
 		if cmd == "" || cmd == "help" || strings.HasPrefix(cmd, "-") {
 			m.Usage()
@@ -78,7 +149,7 @@ func (m *Main) Run(ctx context.Context, args []string) (err error) {
 
 // Usage prints the help screen to STDOUT.
 func (m *Main) Usage() {
-	fmt.Println(`
+	fmt.Fprintln(m.stdout, `
 litestream is a tool for replicating SQLite databases.
 
 Usage:
@@ -105,11 +176,27 @@ type Config struct {
 	// List of databases to manage.
 	DBs []*DBConfig `yaml:"dbs"`
 
+	// Subcommand to execute during replication.
+	// Litestream will shutdown when subcommand exits.
+	Exec string `yaml:"exec"`
+
 	// Global S3 settings
 	AccessKeyID     string `yaml:"access-key-id"`
 	SecretAccessKey string `yaml:"secret-access-key"`
-	Region          string `yaml:"region"`
+}
-	Bucket          string `yaml:"bucket"`
+
+// propagateGlobalSettings copies global S3 settings to replica configs.
+func (c *Config) propagateGlobalSettings() {
+	for _, dbc := range c.DBs {
+		for _, rc := range dbc.Replicas {
+			if rc.AccessKeyID == "" {
+				rc.AccessKeyID = c.AccessKeyID
+			}
+			if rc.SecretAccessKey == "" {
+				rc.SecretAccessKey = c.SecretAccessKey
+			}
+		}
+	}
 }
 
 // DefaultConfig returns a new instance of Config with defaults set.
@@ -128,7 +215,35 @@ func (c *Config) DBConfig(path string) *DBConfig {
 }
 
 // ReadConfigFile unmarshals config from filename. Expands path if needed.
-func ReadConfigFile(filename string) (_ Config, err error) {
+// If expandEnv is true then environment variables are expanded in the config.
+// If filename is blank then the default config path is used.
+func ReadConfigFile(filename string, expandEnv bool) (config Config, err error) {
+	var filenames []string
+	if filename != "" {
+		filenames = append(filenames, filename)
+	}
+	filenames = append(filenames, "./litestream.yml")
+	filenames = append(filenames, DefaultConfigPath())
+
+	for _, name := range filenames {
+		isDefaultPath := name != filename
+
+		if config, err = readConfigFile(name, expandEnv); os.IsNotExist(err) {
+			if isDefaultPath {
+				continue
+			}
+			return config, fmt.Errorf("config file not found: %s", filename)
+		} else if err != nil {
+			return config, err
+		}
+		break
+	}
+	return config, nil
+}
+
+// readConfigFile unmarshals config from filename. Expands path if needed.
+// If expandEnv is true then environment variables are expanded in the config.
+func readConfigFile(filename string, expandEnv bool) (_ Config, err error) {
 	config := DefaultConfig()
 
 	// Expand filename, if necessary.
@@ -137,12 +252,19 @@ func ReadConfigFile(filename string) (_ Config, err error) {
 		return config, err
 	}
 
-	// Read & deserialize configuration.
+	// Read configuration.
-	if buf, err := ioutil.ReadFile(filename); os.IsNotExist(err) {
+	// Do not return an error if using default path and file is missing.
-		return config, fmt.Errorf("config file not found: %s", filename)
+	buf, err := ioutil.ReadFile(filename)
-	} else if err != nil {
+	if err != nil {
 		return config, err
-	} else if err := yaml.Unmarshal(buf, &config); err != nil {
+	}
+
+	// Expand environment variables, if enabled.
+	if expandEnv {
+		buf = []byte(os.ExpandEnv(string(buf)))
+	}
+
+	if err := yaml.Unmarshal(buf, &config); err != nil {
 		return config, err
 	}
 
@@ -153,50 +275,407 @@ func ReadConfigFile(filename string) (_ Config, err error) {
 		}
 	}
 
+	// Propage settings from global config to replica configs.
+	config.propagateGlobalSettings()
+
 	return config, nil
 }
 
 // DBConfig represents the configuration for a single database.
 type DBConfig struct {
-	Path     string           `yaml:"path"`
+	Path                 string         `yaml:"path"`
+	Upstream             UpstreamConfig `yaml:"upstream"`
+	MonitorDelayInterval *time.Duration `yaml:"monitor-delay-interval"`
+	CheckpointInterval   *time.Duration `yaml:"checkpoint-interval"`
+	MinCheckpointPageN   *int           `yaml:"min-checkpoint-page-count"`
+	MaxCheckpointPageN   *int           `yaml:"max-checkpoint-page-count"`
+	ShadowRetentionN     *int           `yaml:"shadow-retention-count"`
+
 	Replicas []*ReplicaConfig `yaml:"replicas"`
 }
 
+// NewDBFromConfig instantiates a DB based on a configuration.
+func NewDBFromConfig(dbc *DBConfig) (*litestream.DB, error) {
+	path, err := expand(dbc.Path)
+	if err != nil {
+		return nil, err
+	}
+	return NewDBFromConfigWithPath(dbc, path)
+}
+
+// NewDBFromConfigWithPath instantiates a DB based on a configuration and using a given path.
+func NewDBFromConfigWithPath(dbc *DBConfig, path string) (*litestream.DB, error) {
+	// Initialize database with given path.
+	db := litestream.NewDB(path)
+
+	// Attach upstream HTTP client if specified.
+	if upstreamURL := dbc.Upstream.URL; upstreamURL != "" {
+		// Use local database path if upstream path is not specified.
+		upstreamPath := dbc.Upstream.Path
+		if upstreamPath == "" {
+			upstreamPath = db.Path()
+		}
+		db.StreamClient = http.NewClient(upstreamURL, upstreamPath)
+	}
+
+	// Override default database settings if specified in configuration.
+	if dbc.MonitorDelayInterval != nil {
+		db.MonitorDelayInterval = *dbc.MonitorDelayInterval
+	}
+	if dbc.CheckpointInterval != nil {
+		db.CheckpointInterval = *dbc.CheckpointInterval
+	}
+	if dbc.MinCheckpointPageN != nil {
+		db.MinCheckpointPageN = *dbc.MinCheckpointPageN
+	}
+	if dbc.MaxCheckpointPageN != nil {
+		db.MaxCheckpointPageN = *dbc.MaxCheckpointPageN
+	}
+	if dbc.ShadowRetentionN != nil {
+		db.ShadowRetentionN = *dbc.ShadowRetentionN
+	}
+
+	// Instantiate and attach replicas.
+	for _, rc := range dbc.Replicas {
+		r, err := NewReplicaFromConfig(rc, db)
+		if err != nil {
+			return nil, err
+		}
+		db.Replicas = append(db.Replicas, r)
+	}
+
+	return db, nil
+}
+
+type UpstreamConfig struct {
+	URL  string `yaml:"url"`
+	Path string `yaml:"path"`
+}
+
 // ReplicaConfig represents the configuration for a single replica in a database.
 type ReplicaConfig struct {
-	Type                   string        `yaml:"type"` // "file", "s3"
+	Type                   string         `yaml:"type"` // "file", "s3"
-	Name                   string        `yaml:"name"` // name of replica, optional.
+	Name                   string         `yaml:"name"` // name of replica, optional.
-	Path                   string        `yaml:"path"`
+	Path                   string         `yaml:"path"`
-	URL                    string        `yaml:"url"`
+	URL                    string         `yaml:"url"`
-	Retention              time.Duration `yaml:"retention"`
+	Retention              *time.Duration `yaml:"retention"`
-	RetentionCheckInterval time.Duration `yaml:"retention-check-interval"`
+	RetentionCheckInterval *time.Duration `yaml:"retention-check-interval"`
-	SyncInterval           time.Duration `yaml:"sync-interval"` // s3 only
+	SyncInterval           *time.Duration `yaml:"sync-interval"`
-	ValidationInterval     time.Duration `yaml:"validation-interval"`
+	SnapshotInterval       *time.Duration `yaml:"snapshot-interval"`
+	ValidationInterval     *time.Duration `yaml:"validation-interval"`
 
 	// S3 settings
 	AccessKeyID     string `yaml:"access-key-id"`
 	SecretAccessKey string `yaml:"secret-access-key"`
 	Region          string `yaml:"region"`
 	Bucket          string `yaml:"bucket"`
+	Endpoint        string `yaml:"endpoint"`
+	ForcePathStyle  *bool  `yaml:"force-path-style"`
+	SkipVerify      bool   `yaml:"skip-verify"`
+
+	// ABS settings
+	AccountName string `yaml:"account-name"`
+	AccountKey  string `yaml:"account-key"`
+
+	// SFTP settings
+	Host     string `yaml:"host"`
+	User     string `yaml:"user"`
+	Password string `yaml:"password"`
+	KeyPath  string `yaml:"key-path"`
 }
 
-// NewReplicaFromURL returns a new Replica instance configured from a URL.
+// NewReplicaFromConfig instantiates a replica for a DB based on a config.
-// The replica's database is not set.
+func NewReplicaFromConfig(c *ReplicaConfig, db *litestream.DB) (_ *litestream.Replica, err error) {
-func NewReplicaFromURL(s string) (litestream.Replica, error) {
+	// Ensure user did not specify URL in path.
-	scheme, host, path, err := ParseReplicaURL(s)
+	if isURL(c.Path) {
-	if err != nil {
+		return nil, fmt.Errorf("replica path cannot be a url, please use the 'url' field instead: %s", c.Path)
+	}
+
+	// Build and set client on replica.
+	var client litestream.ReplicaClient
+	switch typ := c.ReplicaType(); typ {
+	case "file":
+		if client, err = newFileReplicaClientFromConfig(c); err != nil {
+			return nil, err
+		}
+	case "s3":
+		if client, err = newS3ReplicaClientFromConfig(c); err != nil {
+			return nil, err
+		}
+	case "gs":
+		if client, err = newGSReplicaClientFromConfig(c); err != nil {
+			return nil, err
+		}
+	case "abs":
+		if client, err = newABSReplicaClientFromConfig(c); err != nil {
+			return nil, err
+		}
+	case "sftp":
+		if client, err = newSFTPReplicaClientFromConfig(c); err != nil {
+			return nil, err
+		}
+	default:
+		return nil, fmt.Errorf("unknown replica type in config: %q", typ)
+	}
+
+	// Build replica.
+	r := litestream.NewReplica(db, c.Name, client)
+	if v := c.Retention; v != nil {
+		r.Retention = *v
+	}
+	if v := c.RetentionCheckInterval; v != nil {
+		r.RetentionCheckInterval = *v
+	}
+	if v := c.SyncInterval; v != nil {
+		r.SyncInterval = *v
+	}
+	if v := c.SnapshotInterval; v != nil {
+		r.SnapshotInterval = *v
+	}
+	if v := c.ValidationInterval; v != nil {
+		r.ValidationInterval = *v
+	}
+
+	return r, nil
+}
+
+// newFileReplicaClientFromConfig returns a new instance of FileReplicaClient built from config.
+func newFileReplicaClientFromConfig(c *ReplicaConfig) (_ *litestream.FileReplicaClient, err error) {
+	// Ensure URL & path are not both specified.
+	if c.URL != "" && c.Path != "" {
+		return nil, fmt.Errorf("cannot specify url & path for file replica")
+	}
+
+	// Parse path from URL, if specified.
+	path := c.Path
+	if c.URL != "" {
+		if _, _, path, err = ParseReplicaURL(c.URL); err != nil {
+			return nil, err
+		}
+	}
+
+	// Ensure path is set explicitly or derived from URL field.
+	if path == "" {
+		return nil, fmt.Errorf("file replica path required")
+	}
+
+	// Expand home prefix and return absolute path.
+	if path, err = expand(path); err != nil {
 		return nil, err
 	}
 
-	switch scheme {
+	// Instantiate replica and apply time fields, if set.
-	case "file":
+	return litestream.NewFileReplicaClient(path), nil
-		return litestream.NewFileReplica(nil, "", path), nil
+}
-	case "s3":
+
-		r := s3.NewReplica(nil, "")
+// newS3ReplicaClientFromConfig returns a new instance of s3.ReplicaClient built from config.
-		r.Bucket, r.Path = host, path
+func newS3ReplicaClientFromConfig(c *ReplicaConfig) (_ *s3.ReplicaClient, err error) {
-		return r, nil
+	// Ensure URL & constituent parts are not both specified.
-	default:
+	if c.URL != "" && c.Path != "" {
-		return nil, fmt.Errorf("invalid replica url type: %s", s)
+		return nil, fmt.Errorf("cannot specify url & path for s3 replica")
+	} else if c.URL != "" && c.Bucket != "" {
+		return nil, fmt.Errorf("cannot specify url & bucket for s3 replica")
+	}
+
+	bucket, path := c.Bucket, c.Path
+	region, endpoint, skipVerify := c.Region, c.Endpoint, c.SkipVerify
+
+	// Use path style if an endpoint is explicitly set. This works because the
+	// only service to not use path style is AWS which does not use an endpoint.
+	forcePathStyle := (endpoint != "")
+	if v := c.ForcePathStyle; v != nil {
+		forcePathStyle = *v
+	}
+
+	// Apply settings from URL, if specified.
+	if c.URL != "" {
+		_, host, upath, err := ParseReplicaURL(c.URL)
+		if err != nil {
+			return nil, err
+		}
+		ubucket, uregion, uendpoint, uforcePathStyle := s3.ParseHost(host)
+
+		// Only apply URL parts to field that have not been overridden.
+		if path == "" {
+			path = upath
+		}
+		if bucket == "" {
+			bucket = ubucket
+		}
+		if region == "" {
+			region = uregion
+		}
+		if endpoint == "" {
+			endpoint = uendpoint
+		}
+		if !forcePathStyle {
+			forcePathStyle = uforcePathStyle
+		}
+	}
+
+	// Ensure required settings are set.
+	if bucket == "" {
+		return nil, fmt.Errorf("bucket required for s3 replica")
+	}
+
+	// Build replica.
+	client := s3.NewReplicaClient()
+	client.AccessKeyID = c.AccessKeyID
+	client.SecretAccessKey = c.SecretAccessKey
+	client.Bucket = bucket
+	client.Path = path
+	client.Region = region
+	client.Endpoint = endpoint
+	client.ForcePathStyle = forcePathStyle
+	client.SkipVerify = skipVerify
+	return client, nil
+}
+
+// newGSReplicaClientFromConfig returns a new instance of gs.ReplicaClient built from config.
+func newGSReplicaClientFromConfig(c *ReplicaConfig) (_ *gs.ReplicaClient, err error) {
+	// Ensure URL & constituent parts are not both specified.
+	if c.URL != "" && c.Path != "" {
+		return nil, fmt.Errorf("cannot specify url & path for gs replica")
+	} else if c.URL != "" && c.Bucket != "" {
+		return nil, fmt.Errorf("cannot specify url & bucket for gs replica")
+	}
+
+	bucket, path := c.Bucket, c.Path
+
+	// Apply settings from URL, if specified.
+	if c.URL != "" {
+		_, uhost, upath, err := ParseReplicaURL(c.URL)
+		if err != nil {
+			return nil, err
+		}
+
+		// Only apply URL parts to field that have not been overridden.
+		if path == "" {
+			path = upath
+		}
+		if bucket == "" {
+			bucket = uhost
+		}
+	}
+
+	// Ensure required settings are set.
+	if bucket == "" {
+		return nil, fmt.Errorf("bucket required for gs replica")
+	}
+
+	// Build replica.
+	client := gs.NewReplicaClient()
+	client.Bucket = bucket
+	client.Path = path
+	return client, nil
+}
+
+// newABSReplicaClientFromConfig returns a new instance of abs.ReplicaClient built from config.
+func newABSReplicaClientFromConfig(c *ReplicaConfig) (_ *abs.ReplicaClient, err error) {
+	// Ensure URL & constituent parts are not both specified.
+	if c.URL != "" && c.Path != "" {
+		return nil, fmt.Errorf("cannot specify url & path for abs replica")
+	} else if c.URL != "" && c.Bucket != "" {
+		return nil, fmt.Errorf("cannot specify url & bucket for abs replica")
+	}
+
+	// Build replica.
+	client := abs.NewReplicaClient()
+	client.AccountName = c.AccountName
+	client.AccountKey = c.AccountKey
+	client.Bucket = c.Bucket
+	client.Path = c.Path
+	client.Endpoint = c.Endpoint
+
+	// Apply settings from URL, if specified.
+	if c.URL != "" {
+		u, err := url.Parse(c.URL)
+		if err != nil {
+			return nil, err
+		}
+
+		if client.AccountName == "" && u.User != nil {
+			client.AccountName = u.User.Username()
+		}
+		if client.Bucket == "" {
+			client.Bucket = u.Host
+		}
+		if client.Path == "" {
+			client.Path = strings.TrimPrefix(path.Clean(u.Path), "/")
+		}
+	}
+
+	// Ensure required settings are set.
+	if client.Bucket == "" {
+		return nil, fmt.Errorf("bucket required for abs replica")
+	}
+
+	return client, nil
+}
+
+// newSFTPReplicaClientFromConfig returns a new instance of sftp.ReplicaClient built from config.
+func newSFTPReplicaClientFromConfig(c *ReplicaConfig) (_ *sftp.ReplicaClient, err error) {
+	// Ensure URL & constituent parts are not both specified.
+	if c.URL != "" && c.Path != "" {
+		return nil, fmt.Errorf("cannot specify url & path for sftp replica")
+	} else if c.URL != "" && c.Host != "" {
+		return nil, fmt.Errorf("cannot specify url & host for sftp replica")
+	}
+
+	host, user, password, path := c.Host, c.User, c.Password, c.Path
+
+	// Apply settings from URL, if specified.
+	if c.URL != "" {
+		u, err := url.Parse(c.URL)
+		if err != nil {
+			return nil, err
+		}
+
+		// Only apply URL parts to field that have not been overridden.
+		if host == "" {
+			host = u.Host
+		}
+		if user == "" && u.User != nil {
+			user = u.User.Username()
+		}
+		if password == "" && u.User != nil {
+			password, _ = u.User.Password()
+		}
+		if path == "" {
+			path = u.Path
+		}
+	}
+
+	// Ensure required settings are set.
+	if host == "" {
+		return nil, fmt.Errorf("host required for sftp replica")
+	} else if user == "" {
+		return nil, fmt.Errorf("user required for sftp replica")
+	}
+
+	// Build replica.
+	client := sftp.NewReplicaClient()
+	client.Host = host
+	client.User = user
+	client.Password = password
+	client.Path = path
+	client.KeyPath = c.KeyPath
+	return client, nil
+}
+
+// applyLitestreamEnv copies "LITESTREAM" prefixed environment variables to
+// their AWS counterparts as the "AWS" prefix can be confusing when using a
+// non-AWS S3-compatible service.
+func applyLitestreamEnv() {
+	if v, ok := os.LookupEnv("LITESTREAM_ACCESS_KEY_ID"); ok {
+		if _, ok := os.LookupEnv("AWS_ACCESS_KEY_ID"); !ok {
+			os.Setenv("AWS_ACCESS_KEY_ID", v)
+		}
+	}
+	if v, ok := os.LookupEnv("LITESTREAM_SECRET_ACCESS_KEY"); ok {
+		if _, ok := os.LookupEnv("AWS_SECRET_ACCESS_KEY"); !ok {
+			os.Setenv("AWS_SECRET_ACCESS_KEY", v)
+		}
 	}
 }
 
@@ -222,15 +701,14 @@ func ParseReplicaURL(s string) (scheme, host, urlpath string, err error) {
 
 // isURL returns true if s can be parsed and has a scheme.
 func isURL(s string) bool {
-	u, err := url.Parse(s)
+	return regexp.MustCompile(`^\w+:\/\/`).MatchString(s)
-	return err == nil && u.Scheme != ""
 }
 
 // ReplicaType returns the type based on the type field or extracted from the URL.
 func (c *ReplicaConfig) ReplicaType() string {
-	typ, _, _, _ := ParseReplicaURL(c.URL)
+	scheme, _, _, _ := ParseReplicaURL(c.URL)
-	if typ != "" {
+	if scheme != "" {
-		return typ
+		return scheme
 	} else if c.Type != "" {
 		return c.Type
 	}
@@ -245,135 +723,9 @@ func DefaultConfigPath() string {
 	return "/etc/litestream.yml"
 }
 
-func registerConfigFlag(fs *flag.FlagSet, p *string) {
+func registerConfigFlag(fs *flag.FlagSet, configPath *string, noExpandEnv *bool) {
-	fs.StringVar(p, "config", DefaultConfigPath(), "config path")
+	fs.StringVar(configPath, "config", "", "config path")
-}
+	fs.BoolVar(noExpandEnv, "no-expand-env", false, "do not expand env vars in config")
-
-// newDBFromConfig instantiates a DB based on a configuration.
-func newDBFromConfig(c *Config, dbc *DBConfig) (*litestream.DB, error) {
-	path, err := expand(dbc.Path)
-	if err != nil {
-		return nil, err
-	}
-
-	// Initialize database with given path.
-	db := litestream.NewDB(path)
-
-	// Instantiate and attach replicas.
-	for _, rc := range dbc.Replicas {
-		r, err := newReplicaFromConfig(db, c, dbc, rc)
-		if err != nil {
-			return nil, err
-		}
-		db.Replicas = append(db.Replicas, r)
-	}
-
-	return db, nil
-}
-
-// newReplicaFromConfig instantiates a replica for a DB based on a config.
-func newReplicaFromConfig(db *litestream.DB, c *Config, dbc *DBConfig, rc *ReplicaConfig) (litestream.Replica, error) {
-	// Ensure user did not specify URL in path.
-	if isURL(rc.Path) {
-		return nil, fmt.Errorf("replica path cannot be a url, please use the 'url' field instead: %s", rc.Path)
-	}
-
-	switch rc.ReplicaType() {
-	case "file":
-		return newFileReplicaFromConfig(db, c, dbc, rc)
-	case "s3":
-		return newS3ReplicaFromConfig(db, c, dbc, rc)
-	default:
-		return nil, fmt.Errorf("unknown replica type in config: %q", rc.Type)
-	}
-}
-
-// newFileReplicaFromConfig returns a new instance of FileReplica build from config.
-func newFileReplicaFromConfig(db *litestream.DB, c *Config, dbc *DBConfig, rc *ReplicaConfig) (_ *litestream.FileReplica, err error) {
-	path := rc.Path
-	if rc.URL != "" {
-		_, _, path, err = ParseReplicaURL(rc.URL)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	if path == "" {
-		return nil, fmt.Errorf("%s: file replica path required", db.Path())
-	}
-
-	if path, err = expand(path); err != nil {
-		return nil, err
-	}
-
-	r := litestream.NewFileReplica(db, rc.Name, path)
-	if v := rc.Retention; v > 0 {
-		r.Retention = v
-	}
-	if v := rc.RetentionCheckInterval; v > 0 {
-		r.RetentionCheckInterval = v
-	}
-	if v := rc.ValidationInterval; v > 0 {
-		r.ValidationInterval = v
-	}
-	return r, nil
-}
-
-// newS3ReplicaFromConfig returns a new instance of S3Replica build from config.
-func newS3ReplicaFromConfig(db *litestream.DB, c *Config, dbc *DBConfig, rc *ReplicaConfig) (_ *s3.Replica, err error) {
-	bucket := c.Bucket
-	if v := rc.Bucket; v != "" {
-		bucket = v
-	}
-
-	path := rc.Path
-	if rc.URL != "" {
-		_, bucket, path, err = ParseReplicaURL(rc.URL)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	// Use global or replica-specific S3 settings.
-	accessKeyID := c.AccessKeyID
-	if v := rc.AccessKeyID; v != "" {
-		accessKeyID = v
-	}
-	secretAccessKey := c.SecretAccessKey
-	if v := rc.SecretAccessKey; v != "" {
-		secretAccessKey = v
-	}
-	region := c.Region
-	if v := rc.Region; v != "" {
-		region = v
-	}
-
-	// Ensure required settings are set.
-	if bucket == "" {
-		return nil, fmt.Errorf("%s: s3 bucket required", db.Path())
-	}
-
-	// Build replica.
-	r := s3.NewReplica(db, rc.Name)
-	r.AccessKeyID = accessKeyID
-	r.SecretAccessKey = secretAccessKey
-	r.Region = region
-	r.Bucket = bucket
-	r.Path = path
-
-	if v := rc.Retention; v > 0 {
-		r.Retention = v
-	}
-	if v := rc.RetentionCheckInterval; v > 0 {
-		r.RetentionCheckInterval = v
-	}
-	if v := rc.SyncInterval; v > 0 {
-		r.SyncInterval = v
-	}
-	if v := rc.ValidationInterval; v > 0 {
-		r.ValidationInterval = v
-	}
-	return r, nil
 }
 
 // expand returns an absolute path for s.
@@ -398,3 +750,66 @@ func expand(s string) (string, error) {
 	}
 	return filepath.Join(u.HomeDir, strings.TrimPrefix(s, prefix)), nil
 }
+
+// indexVar allows the flag package to parse index flags as 4-byte hexadecimal values.
+type indexVar int
+
+// Ensure type implements interface.
+var _ flag.Value = (*indexVar)(nil)
+
+// String returns an 8-character hexadecimal value.
+func (v *indexVar) String() string {
+	return litestream.FormatIndex(int(*v))
+}
+
+// Set parses s into an integer from a hexadecimal value.
+func (v *indexVar) Set(s string) error {
+	i, err := strconv.ParseInt(s, 16, 32)
+	if err != nil {
+		return fmt.Errorf("invalid hexadecimal format")
+	}
+	*v = indexVar(i)
+	return nil
+}
+
+// loadReplicas returns a list of replicas to use based on CLI flags. Filters
+// by replicaName, if not blank. The DB is returned if pathOrURL is not a replica URL.
+func loadReplicas(ctx context.Context, config Config, pathOrURL, replicaName string) ([]*litestream.Replica, *litestream.DB, error) {
+	// Build a replica based on URL, if specified.
+	if isURL(pathOrURL) {
+		r, err := NewReplicaFromConfig(&ReplicaConfig{
+			URL:             pathOrURL,
+			AccessKeyID:     config.AccessKeyID,
+			SecretAccessKey: config.SecretAccessKey,
+		}, nil)
+		if err != nil {
+			return nil, nil, err
+		}
+		return []*litestream.Replica{r}, nil, nil
+	}
+
+	// Otherwise use replicas from the database configuration file.
+	path, err := expand(pathOrURL)
+	if err != nil {
+		return nil, nil, err
+	}
+	dbc := config.DBConfig(path)
+	if dbc == nil {
+		return nil, nil, fmt.Errorf("database not found in config: %s", path)
+	}
+	db, err := NewDBFromConfig(dbc)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// Filter by replica, if specified.
+	if replicaName != "" {
+		r := db.Replica(replicaName)
+		if r == nil {
+			return nil, nil, fmt.Errorf("replica %q not found for database %q", replicaName, db.Path())
+		}
+		return []*litestream.Replica{r}, db, nil
+	}
+
+	return db.Replicas, db, nil
+}

cmd/litestream/main_test.go

@@ -0,0 +1,198 @@
+package main_test
+
+import (
+	"bytes"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/benbjohnson/litestream"
+	main "github.com/benbjohnson/litestream/cmd/litestream"
+	"github.com/benbjohnson/litestream/gs"
+	"github.com/benbjohnson/litestream/s3"
+)
+
+func init() {
+	litestream.LogFlags = log.Lmsgprefix | log.Ldate | log.Ltime | log.Lmicroseconds | log.LUTC | log.Lshortfile
+}
+
+func TestReadConfigFile(t *testing.T) {
+	// Ensure global AWS settings are propagated down to replica configurations.
+	t.Run("PropagateGlobalSettings", func(t *testing.T) {
+		filename := filepath.Join(t.TempDir(), "litestream.yml")
+		if err := ioutil.WriteFile(filename, []byte(`
+access-key-id: XXX
+secret-access-key: YYY
+
+dbs:
+  - path: /path/to/db
+    replicas:
+      - url: s3://foo/bar
+`[1:]), 0666); err != nil {
+			t.Fatal(err)
+		}
+
+		config, err := main.ReadConfigFile(filename, true)
+		if err != nil {
+			t.Fatal(err)
+		} else if got, want := config.AccessKeyID, `XXX`; got != want {
+			t.Fatalf("AccessKeyID=%v, want %v", got, want)
+		} else if got, want := config.SecretAccessKey, `YYY`; got != want {
+			t.Fatalf("SecretAccessKey=%v, want %v", got, want)
+		} else if got, want := config.DBs[0].Replicas[0].AccessKeyID, `XXX`; got != want {
+			t.Fatalf("Replica.AccessKeyID=%v, want %v", got, want)
+		} else if got, want := config.DBs[0].Replicas[0].SecretAccessKey, `YYY`; got != want {
+			t.Fatalf("Replica.SecretAccessKey=%v, want %v", got, want)
+		}
+	})
+
+	// Ensure environment variables are expanded.
+	t.Run("ExpandEnv", func(t *testing.T) {
+		os.Setenv("LITESTREAM_TEST_0129380", "/path/to/db")
+		os.Setenv("LITESTREAM_TEST_1872363", "s3://foo/bar")
+
+		filename := filepath.Join(t.TempDir(), "litestream.yml")
+		if err := ioutil.WriteFile(filename, []byte(`
+dbs:
+  - path: $LITESTREAM_TEST_0129380
+    replicas:
+      - url: ${LITESTREAM_TEST_1872363}
+      - url: ${LITESTREAM_TEST_NO_SUCH_ENV}
+`[1:]), 0666); err != nil {
+			t.Fatal(err)
+		}
+
+		config, err := main.ReadConfigFile(filename, true)
+		if err != nil {
+			t.Fatal(err)
+		} else if got, want := config.DBs[0].Path, `/path/to/db`; got != want {
+			t.Fatalf("DB.Path=%v, want %v", got, want)
+		} else if got, want := config.DBs[0].Replicas[0].URL, `s3://foo/bar`; got != want {
+			t.Fatalf("Replica[0].URL=%v, want %v", got, want)
+		} else if got, want := config.DBs[0].Replicas[1].URL, ``; got != want {
+			t.Fatalf("Replica[1].URL=%v, want %v", got, want)
+		}
+	})
+
+	// Ensure environment variables are not expanded.
+	t.Run("NoExpandEnv", func(t *testing.T) {
+		os.Setenv("LITESTREAM_TEST_9847533", "s3://foo/bar")
+
+		filename := filepath.Join(t.TempDir(), "litestream.yml")
+		if err := ioutil.WriteFile(filename, []byte(`
+dbs:
+  - path: /path/to/db
+    replicas:
+      - url: ${LITESTREAM_TEST_9847533}
+`[1:]), 0666); err != nil {
+			t.Fatal(err)
+		}
+
+		config, err := main.ReadConfigFile(filename, false)
+		if err != nil {
+			t.Fatal(err)
+		} else if got, want := config.DBs[0].Replicas[0].URL, `${LITESTREAM_TEST_9847533}`; got != want {
+			t.Fatalf("Replica.URL=%v, want %v", got, want)
+		}
+	})
+}
+
+func TestNewFileReplicaFromConfig(t *testing.T) {
+	r, err := main.NewReplicaFromConfig(&main.ReplicaConfig{Path: "/foo"}, nil)
+	if err != nil {
+		t.Fatal(err)
+	} else if client, ok := r.Client().(*litestream.FileReplicaClient); !ok {
+		t.Fatal("unexpected replica type")
+	} else if got, want := client.Path(), "/foo"; got != want {
+		t.Fatalf("Path=%s, want %s", got, want)
+	}
+}
+
+func TestNewS3ReplicaFromConfig(t *testing.T) {
+	t.Run("URL", func(t *testing.T) {
+		r, err := main.NewReplicaFromConfig(&main.ReplicaConfig{URL: "s3://foo/bar"}, nil)
+		if err != nil {
+			t.Fatal(err)
+		} else if client, ok := r.Client().(*s3.ReplicaClient); !ok {
+			t.Fatal("unexpected replica type")
+		} else if got, want := client.Bucket, "foo"; got != want {
+			t.Fatalf("Bucket=%s, want %s", got, want)
+		} else if got, want := client.Path, "bar"; got != want {
+			t.Fatalf("Path=%s, want %s", got, want)
+		} else if got, want := client.Region, ""; got != want {
+			t.Fatalf("Region=%s, want %s", got, want)
+		} else if got, want := client.Endpoint, ""; got != want {
+			t.Fatalf("Endpoint=%s, want %s", got, want)
+		} else if got, want := client.ForcePathStyle, false; got != want {
+			t.Fatalf("ForcePathStyle=%v, want %v", got, want)
+		}
+	})
+
+	t.Run("MinIO", func(t *testing.T) {
+		r, err := main.NewReplicaFromConfig(&main.ReplicaConfig{URL: "s3://foo.localhost:9000/bar"}, nil)
+		if err != nil {
+			t.Fatal(err)
+		} else if client, ok := r.Client().(*s3.ReplicaClient); !ok {
+			t.Fatal("unexpected replica type")
+		} else if got, want := client.Bucket, "foo"; got != want {
+			t.Fatalf("Bucket=%s, want %s", got, want)
+		} else if got, want := client.Path, "bar"; got != want {
+			t.Fatalf("Path=%s, want %s", got, want)
+		} else if got, want := client.Region, "us-east-1"; got != want {
+			t.Fatalf("Region=%s, want %s", got, want)
+		} else if got, want := client.Endpoint, "http://localhost:9000"; got != want {
+			t.Fatalf("Endpoint=%s, want %s", got, want)
+		} else if got, want := client.ForcePathStyle, true; got != want {
+			t.Fatalf("ForcePathStyle=%v, want %v", got, want)
+		}
+	})
+
+	t.Run("Backblaze", func(t *testing.T) {
+		r, err := main.NewReplicaFromConfig(&main.ReplicaConfig{URL: "s3://foo.s3.us-west-000.backblazeb2.com/bar"}, nil)
+		if err != nil {
+			t.Fatal(err)
+		} else if client, ok := r.Client().(*s3.ReplicaClient); !ok {
+			t.Fatal("unexpected replica type")
+		} else if got, want := client.Bucket, "foo"; got != want {
+			t.Fatalf("Bucket=%s, want %s", got, want)
+		} else if got, want := client.Path, "bar"; got != want {
+			t.Fatalf("Path=%s, want %s", got, want)
+		} else if got, want := client.Region, "us-west-000"; got != want {
+			t.Fatalf("Region=%s, want %s", got, want)
+		} else if got, want := client.Endpoint, "https://s3.us-west-000.backblazeb2.com"; got != want {
+			t.Fatalf("Endpoint=%s, want %s", got, want)
+		} else if got, want := client.ForcePathStyle, true; got != want {
+			t.Fatalf("ForcePathStyle=%v, want %v", got, want)
+		}
+	})
+}
+
+func TestNewGSReplicaFromConfig(t *testing.T) {
+	r, err := main.NewReplicaFromConfig(&main.ReplicaConfig{URL: "gs://foo/bar"}, nil)
+	if err != nil {
+		t.Fatal(err)
+	} else if client, ok := r.Client().(*gs.ReplicaClient); !ok {
+		t.Fatal("unexpected replica type")
+	} else if got, want := client.Bucket, "foo"; got != want {
+		t.Fatalf("Bucket=%s, want %s", got, want)
+	} else if got, want := client.Path, "bar"; got != want {
+		t.Fatalf("Path=%s, want %s", got, want)
+	}
+}
+
+// newMain returns a new instance of Main and associated buffers.
+func newMain() (m *main.Main, stdin, stdout, stderr *bytes.Buffer) {
+	stdin, stdout, stderr = &bytes.Buffer{}, &bytes.Buffer{}, &bytes.Buffer{}
+
+	// Split stdout/stderr to terminal if verbose flag set.
+	out, err := io.Writer(stdout), io.Writer(stderr)
+	if testing.Verbose() {
+		out = io.MultiWriter(out, os.Stdout)
+		err = io.MultiWriter(err, os.Stderr)
+	}
+
+	return main.NewMain(stdin, out, err), stdin, stdout, stderr
+}

cmd/litestream/replicate.go

@@ -2,155 +2,204 @@ package main
 
 import (
 	"context"
-	"errors"
 	"flag"
 	"fmt"
+	"io"
 	"log"
-	"net"
-	"net/http"
-	_ "net/http/pprof"
 	"os"
-	"os/signal"
+	"os/exec"
-	"time"
 
 	"github.com/benbjohnson/litestream"
+	"github.com/benbjohnson/litestream/abs"
+	"github.com/benbjohnson/litestream/gs"
+	"github.com/benbjohnson/litestream/http"
 	"github.com/benbjohnson/litestream/s3"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/benbjohnson/litestream/sftp"
+	"github.com/mattn/go-shellwords"
 )
 
 // ReplicateCommand represents a command that continuously replicates SQLite databases.
 type ReplicateCommand struct {
-	ConfigPath string
+	stdin  io.Reader
-	Config     Config
+	stdout io.Writer
+	stderr io.Writer
 
-	// List of managed databases specified in the config.
+	configPath  string
-	DBs []*litestream.DB
+	noExpandEnv bool
+
+	cmd    *exec.Cmd  // subcommand
+	execCh chan error // subcommand error channel
+
+	Config Config
+
+	server     *litestream.Server
+	httpServer *http.Server
 }
 
-// Run loads all databases specified in the configuration.
+// NewReplicateCommand returns a new instance of ReplicateCommand.
-func (c *ReplicateCommand) Run(ctx context.Context, args []string) (err error) {
+func NewReplicateCommand(stdin io.Reader, stdout, stderr io.Writer) *ReplicateCommand {
+	return &ReplicateCommand{
+		stdin:  stdin,
+		stdout: stdout,
+		stderr: stderr,
+
+		execCh: make(chan error),
+	}
+}
+
+// ParseFlags parses the CLI flags and loads the configuration file.
+func (c *ReplicateCommand) ParseFlags(ctx context.Context, args []string) (err error) {
 	fs := flag.NewFlagSet("litestream-replicate", flag.ContinueOnError)
-	tracePath := fs.String("trace", "", "trace path")
+	execFlag := fs.String("exec", "", "execute subcommand")
-	registerConfigFlag(fs, &c.ConfigPath)
+	addr := fs.String("addr", "", "HTTP bind address (host:port)")
+	registerConfigFlag(fs, &c.configPath, &c.noExpandEnv)
 	fs.Usage = c.Usage
 	if err := fs.Parse(args); err != nil {
 		return err
 	}
 
 	// Load configuration or use CLI args to build db/replica.
-	var config Config
 	if fs.NArg() == 1 {
 		return fmt.Errorf("must specify at least one replica URL for %s", fs.Arg(0))
 	} else if fs.NArg() > 1 {
+		if c.configPath != "" {
+			return fmt.Errorf("cannot specify a replica URL and the -config flag")
+		}
+
 		dbConfig := &DBConfig{Path: fs.Arg(0)}
 		for _, u := range fs.Args()[1:] {
+			syncInterval := litestream.DefaultSyncInterval
 			dbConfig.Replicas = append(dbConfig.Replicas, &ReplicaConfig{
 				URL:          u,
-				SyncInterval: 1 * time.Second,
+				SyncInterval: &syncInterval,
 			})
 		}
-		config.DBs = []*DBConfig{dbConfig}
+		c.Config.DBs = []*DBConfig{dbConfig}
-	} else if c.ConfigPath != "" {
-		config, err = ReadConfigFile(c.ConfigPath)
-		if err != nil {
-			return err
-		}
 	} else {
-		return errors.New("-config flag or database/replica arguments required")
+		if c.configPath == "" {
-	}
+			c.configPath = DefaultConfigPath()
-
+		}
-	// Enable trace logging.
+		if c.Config, err = ReadConfigFile(c.configPath, !c.noExpandEnv); err != nil {
-	if *tracePath != "" {
-		f, err := os.Create(*tracePath)
-		if err != nil {
 			return err
 		}
-		defer f.Close()
-		litestream.Tracef = log.New(f, "", log.LstdFlags|log.LUTC|log.Lshortfile).Printf
 	}
 
-	// Setup signal handler.
+	// Override config with flags, if specified.
-	ctx, cancel := context.WithCancel(ctx)
+	if *addr != "" {
-	ch := make(chan os.Signal, 1)
+		c.Config.Addr = *addr
-	signal.Notify(ch, os.Interrupt)
-	go func() { <-ch; cancel() }()
-
-	// Display version information.
-	fmt.Printf("litestream %s\n", Version)
-
-	if len(config.DBs) == 0 {
-		fmt.Println("no databases specified in configuration")
 	}
-
+	if *execFlag != "" {
-	for _, dbConfig := range config.DBs {
+		c.Config.Exec = *execFlag
-		db, err := newDBFromConfig(&config, dbConfig)
-		if err != nil {
-			return err
-		}
-
-		// Open database & attach to program.
-		if err := db.Open(); err != nil {
-			return err
-		}
-		c.DBs = append(c.DBs, db)
-	}
-
-	// Notify user that initialization is done.
-	for _, db := range c.DBs {
-		fmt.Printf("initialized db: %s\n", db.Path())
-		for _, r := range db.Replicas {
-			switch r := r.(type) {
-			case *litestream.FileReplica:
-				fmt.Printf("replicating to: name=%q type=%q path=%q\n", r.Name(), r.Type(), r.Path())
-			case *s3.Replica:
-				fmt.Printf("replicating to: name=%q type=%q bucket=%q path=%q region=%q\n", r.Name(), r.Type(), r.Bucket, r.Path, r.Region)
-			default:
-				fmt.Printf("replicating to: name=%q type=%q\n", r.Name(), r.Type())
-			}
-		}
-	}
-
-	// Serve metrics over HTTP if enabled.
-	if config.Addr != "" {
-		_, port, _ := net.SplitHostPort(config.Addr)
-		fmt.Printf("serving metrics on http://localhost:%s/metrics\n", port)
-		go func() {
-			http.Handle("/metrics", promhttp.Handler())
-			if err := http.ListenAndServe(config.Addr, nil); err != nil {
-				log.Printf("cannot start metrics server: %s", err)
-			}
-		}()
-	}
-
-	// Wait for signal to stop program.
-	<-ctx.Done()
-	signal.Reset()
-
-	// Gracefully close
-	if err := c.Close(); err != nil {
-		fmt.Fprintln(os.Stderr, err)
-		os.Exit(1)
 	}
 
 	return nil
 }
 
-// Close closes all open databases.
+// Run loads all databases specified in the configuration.
-func (c *ReplicateCommand) Close() (err error) {
+func (c *ReplicateCommand) Run(ctx context.Context) (err error) {
-	for _, db := range c.DBs {
+	// Display version information.
-		if e := db.SoftClose(); e != nil {
+	log.Printf("litestream %s", Version)
-			fmt.Printf("error closing db: path=%s err=%s\n", db.Path(), e)
+
-			if err == nil {
+	// Setup databases.
-				err = e
+	if len(c.Config.DBs) == 0 {
+		log.Println("no databases specified in configuration")
+	}
+
+	c.server = litestream.NewServer()
+	if err := c.server.Open(); err != nil {
+		return fmt.Errorf("open server: %w", err)
+	}
+
+	// Add databases to the server.
+	for _, dbConfig := range c.Config.DBs {
+		path, err := expand(dbConfig.Path)
+		if err != nil {
+			return err
+		}
+
+		if err := c.server.Watch(path, func(path string) (*litestream.DB, error) {
+			return NewDBFromConfigWithPath(dbConfig, path)
+		}); err != nil {
+			return err
+		}
+	}
+
+	// Notify user that initialization is done.
+	for _, db := range c.server.DBs() {
+		log.Printf("initialized db: %s", db.Path())
+		for _, r := range db.Replicas {
+			switch client := r.Client().(type) {
+			case *litestream.FileReplicaClient:
+				log.Printf("replicating to: name=%q type=%q path=%q", r.Name(), client.Type(), client.Path())
+			case *s3.ReplicaClient:
+				log.Printf("replicating to: name=%q type=%q bucket=%q path=%q region=%q endpoint=%q sync-interval=%s", r.Name(), client.Type(), client.Bucket, client.Path, client.Region, client.Endpoint, r.SyncInterval)
+			case *gs.ReplicaClient:
+				log.Printf("replicating to: name=%q type=%q bucket=%q path=%q sync-interval=%s", r.Name(), client.Type(), client.Bucket, client.Path, r.SyncInterval)
+			case *abs.ReplicaClient:
+				log.Printf("replicating to: name=%q type=%q bucket=%q path=%q endpoint=%q sync-interval=%s", r.Name(), client.Type(), client.Bucket, client.Path, client.Endpoint, r.SyncInterval)
+			case *sftp.ReplicaClient:
+				log.Printf("replicating to: name=%q type=%q host=%q user=%q path=%q sync-interval=%s", r.Name(), client.Type(), client.Host, client.User, client.Path, r.SyncInterval)
+			default:
+				log.Printf("replicating to: name=%q type=%q", r.Name(), client.Type())
 			}
 		}
 	}
+
+	// Serve HTTP if enabled.
+	if c.Config.Addr != "" {
+		c.httpServer = http.NewServer(c.server, c.Config.Addr)
+		if err := c.httpServer.Open(); err != nil {
+			return fmt.Errorf("cannot start http server: %w", err)
+		}
+		log.Printf("http server running at %s", c.httpServer.URL())
+	}
+
+	// Parse exec commands args & start subprocess.
+	if c.Config.Exec != "" {
+		execArgs, err := shellwords.Parse(c.Config.Exec)
+		if err != nil {
+			return fmt.Errorf("cannot parse exec command: %w", err)
+		}
+
+		// Pass first database path to child process.
+		env := os.Environ()
+		if dbs := c.server.DBs(); len(dbs) > 0 {
+			env = append(env, fmt.Sprintf("LITESTREAM_DB_PATH=%s", dbs[0].Path()))
+		}
+
+		c.cmd = exec.CommandContext(ctx, execArgs[0], execArgs[1:]...)
+		c.cmd.Env = env
+		c.cmd.Stdout = os.Stdout
+		c.cmd.Stderr = os.Stderr
+		if err := c.cmd.Start(); err != nil {
+			return fmt.Errorf("cannot start exec command: %w", err)
+		}
+		go func() { c.execCh <- c.cmd.Wait() }()
+	}
+
+	log.Printf("litestream initialization complete")
+
+	return nil
+}
+
+// Close closes the HTTP server & all open databases.
+func (c *ReplicateCommand) Close() (err error) {
+	if c.httpServer != nil {
+		if e := c.httpServer.Close(); e != nil && err == nil {
+			err = e
+		}
+	}
+	if c.server != nil {
+		if e := c.server.Close(); e != nil && err == nil {
+			err = e
+		}
+	}
 	return err
 }
 
 // Usage prints the help screen to STDOUT.
 func (c *ReplicateCommand) Usage() {
-	fmt.Printf(`
+	fmt.Fprintf(c.stdout, `
 The replicate command starts a server to monitor & replicate databases. 
 You can specify your database & replicas in a configuration file or you can
 replicate a single database file by specifying its path and its replicas in the
@@ -168,8 +217,16 @@ Arguments:
 	    Specifies the configuration file.
 	    Defaults to %s
 
-	-trace PATH
+	-exec CMD
-	    Write verbose trace logging to PATH.
+	    Executes a subcommand. Litestream will exit when the child
+	    process exits. Useful for simple process management.
+
+	-addr BIND_ADDR
+	    Starts an HTTP server that reports prometheus metrics and provides
+	    an endpoint for live read replication. (e.g. ":9090")
+
+	-no-expand-env
+	    Disables environment variable expansion in configuration file.
 
 `[1:], DefaultConfigPath())
 }

cmd/litestream/replicate_test.go

@@ -0,0 +1,136 @@
+package main_test
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"hash/crc64"
+	"io"
+	"os"
+	"path/filepath"
+	"runtime"
+	"testing"
+	"time"
+
+	"golang.org/x/sync/errgroup"
+)
+
+func TestReplicateCommand(t *testing.T) {
+	if testing.Short() {
+		t.Skip("long running test, skipping")
+	} else if runtime.GOOS != "linux" {
+		t.Skip("must run system tests on Linux, skipping")
+	}
+
+	const writeTime = 10 * time.Second
+
+	dir := t.TempDir()
+	configPath := filepath.Join(dir, "litestream.yml")
+	dbPath := filepath.Join(dir, "db")
+	restorePath := filepath.Join(dir, "restored")
+	replicaPath := filepath.Join(dir, "replica")
+
+	if err := os.WriteFile(configPath, []byte(`
+dbs:
+  - path: `+dbPath+`
+    replicas:
+      - path: `+replicaPath+`
+`), 0666); err != nil {
+		t.Fatal(err)
+	}
+
+	// Generate data into SQLite database from separate goroutine.
+	g, ctx := errgroup.WithContext(context.Background())
+	mainctx, cancel := context.WithCancel(ctx)
+	g.Go(func() error {
+		defer cancel()
+
+		db, err := sql.Open("sqlite3", dbPath)
+		if err != nil {
+			return err
+		}
+		defer db.Close()
+
+		if _, err := db.ExecContext(ctx, `PRAGMA journal_mode = WAL`); err != nil {
+			return fmt.Errorf("cannot enable wal: %w", err)
+		} else if _, err := db.ExecContext(ctx, `PRAGMA synchronous = NORMAL`); err != nil {
+			return fmt.Errorf("cannot enable wal: %w", err)
+		} else if _, err := db.ExecContext(ctx, `CREATE TABLE t (id INTEGER PRIMARY KEY)`); err != nil {
+			return fmt.Errorf("cannot create table: %w", err)
+		}
+
+		ticker := time.NewTicker(1 * time.Millisecond)
+		defer ticker.Stop()
+		timer := time.NewTimer(writeTime)
+		defer timer.Stop()
+
+		for i := 0; ; i++ {
+			select {
+			case <-ctx.Done():
+				return ctx.Err()
+			case <-timer.C:
+				return nil
+			case <-ticker.C:
+				if _, err := db.ExecContext(ctx, `INSERT INTO t (id) VALUES (?);`, i); err != nil {
+					return fmt.Errorf("cannot insert: i=%d err=%w", i, err)
+				}
+			}
+		}
+	})
+
+	// Replicate database unless the context is canceled.
+	g.Go(func() error {
+		m, _, _, _ := newMain()
+		return m.Run(mainctx, []string{"replicate", "-config", configPath})
+	})
+
+	if err := g.Wait(); err != nil {
+		t.Fatal(err)
+	}
+
+	// Checkpoint database.
+	mustCheckpoint(t, dbPath)
+	chksum0 := mustChecksum(t, dbPath)
+
+	// Restore to another path.
+	m, _, _, _ := newMain()
+	if err := m.Run(context.Background(), []string{"restore", "-config", configPath, "-o", restorePath, dbPath}); err != nil && !errors.Is(err, context.Canceled) {
+		t.Fatal(err)
+	}
+
+	// Verify contents match.
+	if chksum1 := mustChecksum(t, restorePath); chksum0 != chksum1 {
+		t.Fatal("restore mismatch")
+	}
+}
+
+func mustCheckpoint(tb testing.TB, path string) {
+	tb.Helper()
+
+	db, err := sql.Open("sqlite3", path)
+	if err != nil {
+		tb.Fatal(err)
+	}
+	defer db.Close()
+
+	if _, err := db.Exec(`PRAGMA wal_checkpoint(TRUNCATE)`); err != nil {
+		tb.Fatal(err)
+	}
+}
+
+func mustChecksum(tb testing.TB, path string) uint64 {
+	tb.Helper()
+
+	f, err := os.Open(path)
+	if err != nil {
+		tb.Fatal(err)
+	}
+	defer f.Close()
+
+	h := crc64.New(crc64.MakeTable(crc64.ISO))
+	if _, err := io.Copy(h, f); err != nil {
+		tb.Fatal(err)
+	}
+	return h.Sum64()
+}

cmd/litestream/restore.go

@@ -2,34 +2,63 @@ package main
 
 import (
 	"context"
-	"errors"
 	"flag"
 	"fmt"
+	"io"
 	"log"
 	"os"
+	"path/filepath"
+	"strconv"
 	"time"
 
 	"github.com/benbjohnson/litestream"
 )
 
 // RestoreCommand represents a command to restore a database from a backup.
-type RestoreCommand struct{}
+type RestoreCommand struct {
+	stdin  io.Reader
+	stdout io.Writer
+	stderr io.Writer
+
+	snapshotIndex int // index of snapshot to start from
+
+	// CLI options
+	configPath      string    // path to config file
+	noExpandEnv     bool      // if true, do not expand env variables in config
+	outputPath      string    // path to restore database to
+	replicaName     string    // optional, name of replica to restore from
+	generation      string    // optional, generation to restore
+	targetIndex     int       // optional, last WAL index to replay
+	timestamp       time.Time // optional, restore to point-in-time (ISO 8601)
+	ifDBNotExists   bool      // if true, skips restore if output path already exists
+	ifReplicaExists bool      // if true, skips if no backups exist
+	opt             litestream.RestoreOptions
+}
+
+// NewRestoreCommand returns a new instance of RestoreCommand.
+func NewRestoreCommand(stdin io.Reader, stdout, stderr io.Writer) *RestoreCommand {
+	return &RestoreCommand{
+		stdin:  stdin,
+		stdout: stdout,
+		stderr: stderr,
+
+		targetIndex: -1,
+		opt:         litestream.NewRestoreOptions(),
+	}
+}
 
 // Run executes the command.
 func (c *RestoreCommand) Run(ctx context.Context, args []string) (err error) {
-	var configPath string
-	opt := litestream.NewRestoreOptions()
-	opt.Verbose = true
-
 	fs := flag.NewFlagSet("litestream-restore", flag.ContinueOnError)
-	registerConfigFlag(fs, &configPath)
+	registerConfigFlag(fs, &c.configPath, &c.noExpandEnv)
-	fs.StringVar(&opt.OutputPath, "o", "", "output path")
+	fs.StringVar(&c.outputPath, "o", "", "output path")
-	fs.StringVar(&opt.ReplicaName, "replica", "", "replica name")
+	fs.StringVar(&c.replicaName, "replica", "", "replica name")
-	fs.StringVar(&opt.Generation, "generation", "", "generation name")
+	fs.StringVar(&c.generation, "generation", "", "generation name")
-	fs.IntVar(&opt.Index, "index", opt.Index, "wal index")
+	fs.Var((*indexVar)(&c.targetIndex), "index", "wal index")
-	fs.BoolVar(&opt.DryRun, "dry-run", false, "dry run")
+	timestampStr := fs.String("timestamp", "", "point-in-time restore (ISO 8601)")
-	timestampStr := fs.String("timestamp", "", "timestamp")
+	fs.IntVar(&c.opt.Parallelism, "parallelism", c.opt.Parallelism, "parallelism")
-	verbose := fs.Bool("v", false, "verbose output")
+	fs.BoolVar(&c.ifDBNotExists, "if-db-not-exists", false, "")
+	fs.BoolVar(&c.ifReplicaExists, "if-replica-exists", false, "")
 	fs.Usage = c.Usage
 	if err := fs.Parse(args); err != nil {
 		return err
@@ -38,64 +67,122 @@ func (c *RestoreCommand) Run(ctx context.Context, args []string) (err error) {
 	} else if fs.NArg() > 1 {
 		return fmt.Errorf("too many arguments")
 	}
+	pathOrURL := fs.Arg(0)
 
-	// Parse timestamp, if specified.
+	// Parse timestamp.
 	if *timestampStr != "" {
-		if opt.Timestamp, err = time.Parse(time.RFC3339, *timestampStr); err != nil {
+		if c.timestamp, err = time.Parse(time.RFC3339Nano, *timestampStr); err != nil {
-			return errors.New("invalid -timestamp, must specify in ISO 8601 format (e.g. 2000-01-01T00:00:00Z)")
+			return fmt.Errorf("invalid -timestamp, expected ISO 8601: %w", err)
 		}
 	}
 
-	// Verbose output is automatically enabled if dry run is specified.
+	// Ensure a generation is specified if target index is specified.
-	if opt.DryRun {
+	if c.targetIndex != -1 && !c.timestamp.IsZero() {
-		*verbose = true
+		return fmt.Errorf("cannot specify both -index flag and -timestamp flag")
+	} else if c.targetIndex != -1 && c.generation == "" {
+		return fmt.Errorf("must specify -generation flag when using -index flag")
+	} else if !c.timestamp.IsZero() && c.generation == "" {
+		return fmt.Errorf("must specify -generation flag when using -timestamp flag")
 	}
 
-	// Instantiate logger if verbose output is enabled.
+	// Default to original database path if output path not specified.
-	if *verbose {
+	if !isURL(pathOrURL) && c.outputPath == "" {
-		opt.Logger = log.New(os.Stderr, "", log.LstdFlags)
+		c.outputPath = pathOrURL
 	}
 
-	// Determine replica & generation to restore from.
+	// Exit successfully if the output file already exists and flag is set.
-	var r litestream.Replica
+	if _, err := os.Stat(c.outputPath); os.IsNotExist(err) {
-	if isURL(fs.Arg(0)) {
+		// file doesn't exist, continue
-		if r, err = c.loadFromURL(ctx, fs.Arg(0), &opt); err != nil {
+	} else if err != nil {
-			return err
+		return err
+	} else if err == nil {
+		if c.ifDBNotExists {
+			fmt.Fprintln(c.stdout, "database already exists, skipping")
+			return nil
 		}
-	} else if configPath != "" {
+		return fmt.Errorf("output file already exists: %s", c.outputPath)
-		if r, err = c.loadFromConfig(ctx, fs.Arg(0), configPath, &opt); err != nil {
-			return err
-		}
-	} else {
-		return errors.New("config path or replica URL required")
 	}
 
-	// Return an error if no matching targets found.
-	if opt.Generation == "" {
-		return fmt.Errorf("no matching backups found")
-	}
-
-	return litestream.RestoreReplica(ctx, r, opt)
-}
-
-// loadFromURL creates a replica & updates the restore options from a replica URL.
-func (c *RestoreCommand) loadFromURL(ctx context.Context, replicaURL string, opt *litestream.RestoreOptions) (litestream.Replica, error) {
-	r, err := NewReplicaFromURL(replicaURL)
-	if err != nil {
-		return nil, err
-	}
-	opt.Generation, _, err = litestream.CalcReplicaRestoreTarget(ctx, r, *opt)
-	return r, err
-}
-
-// loadFromConfig returns a replica & updates the restore options from a DB reference.
-func (c *RestoreCommand) loadFromConfig(ctx context.Context, dbPath, configPath string, opt *litestream.RestoreOptions) (litestream.Replica, error) {
 	// Load configuration.
-	config, err := ReadConfigFile(configPath)
+	config, err := ReadConfigFile(c.configPath, !c.noExpandEnv)
 	if err != nil {
-		return nil, err
+		return err
 	}
 
+	// Build replica from either a URL or config.
+	r, err := c.loadReplica(ctx, config, pathOrURL)
+	if err != nil {
+		return err
+	}
+
+	// Determine latest generation if one is not specified.
+	if c.generation == "" {
+		if c.generation, err = litestream.FindLatestGeneration(ctx, r.Client()); err == litestream.ErrNoGeneration {
+			// Return an error if no matching targets found.
+			// If optional flag set, return success. Useful for automated recovery.
+			if c.ifReplicaExists {
+				fmt.Fprintln(c.stdout, "no matching backups found, skipping")
+				return nil
+			}
+			return fmt.Errorf("no matching backups found")
+		} else if err != nil {
+			return fmt.Errorf("cannot determine latest generation: %w", err)
+		}
+	}
+
+	// Determine the maximum available index for the generation if one is not specified.
+	if !c.timestamp.IsZero() {
+		if c.targetIndex, err = litestream.FindIndexByTimestamp(ctx, r.Client(), c.generation, c.timestamp); err != nil {
+			return fmt.Errorf("cannot find index for timestamp in generation %q: %w", c.generation, err)
+		}
+	} else if c.targetIndex == -1 {
+		if c.targetIndex, err = litestream.FindMaxIndexByGeneration(ctx, r.Client(), c.generation); err != nil {
+			return fmt.Errorf("cannot determine latest index in generation %q: %w", c.generation, err)
+		}
+	}
+
+	// Find lastest snapshot that occurs before the index.
+	// TODO: Optionally allow -snapshot-index
+	if c.snapshotIndex, err = litestream.FindSnapshotForIndex(ctx, r.Client(), c.generation, c.targetIndex); err != nil {
+		return fmt.Errorf("cannot find snapshot index: %w", err)
+	}
+
+	// Create parent directory if it doesn't already exist.
+	if err := os.MkdirAll(filepath.Dir(c.outputPath), 0700); err != nil {
+		return fmt.Errorf("cannot create parent directory: %w", err)
+	}
+
+	c.opt.Logger = log.New(c.stdout, "", log.LstdFlags|log.Lmicroseconds)
+
+	return litestream.Restore(ctx, r.Client(), c.outputPath, c.generation, c.snapshotIndex, c.targetIndex, c.opt)
+}
+
+func (c *RestoreCommand) loadReplica(ctx context.Context, config Config, arg string) (*litestream.Replica, error) {
+	if isURL(arg) {
+		return c.loadReplicaFromURL(ctx, config, arg)
+	}
+	return c.loadReplicaFromConfig(ctx, config, arg)
+}
+
+// loadReplicaFromURL creates a replica & updates the restore options from a replica URL.
+func (c *RestoreCommand) loadReplicaFromURL(ctx context.Context, config Config, replicaURL string) (*litestream.Replica, error) {
+	if c.replicaName != "" {
+		return nil, fmt.Errorf("cannot specify both the replica URL and the -replica flag")
+	} else if c.outputPath == "" {
+		return nil, fmt.Errorf("output path required when using a replica URL")
+	}
+
+	syncInterval := litestream.DefaultSyncInterval
+	return NewReplicaFromConfig(&ReplicaConfig{
+		URL:             replicaURL,
+		AccessKeyID:     config.AccessKeyID,
+		SecretAccessKey: config.SecretAccessKey,
+		SyncInterval:    &syncInterval,
+	}, nil)
+}
+
+// loadReplicaFromConfig returns replicas based on the specific config path.
+func (c *RestoreCommand) loadReplicaFromConfig(ctx context.Context, config Config, dbPath string) (_ *litestream.Replica, err error) {
 	// Lookup database from configuration file by path.
 	if dbPath, err = expand(dbPath); err != nil {
 		return nil, err
@@ -104,29 +191,43 @@ func (c *RestoreCommand) loadFromConfig(ctx context.Context, dbPath, configPath
 	if dbConfig == nil {
 		return nil, fmt.Errorf("database not found in config: %s", dbPath)
 	}
-	db, err := newDBFromConfig(&config, dbConfig)
+	db, err := NewDBFromConfig(dbConfig)
 	if err != nil {
 		return nil, err
+	} else if len(db.Replicas) == 0 {
+		return nil, fmt.Errorf("database has no replicas: %s", dbPath)
 	}
 
-	// Restore into original database path if not specified.
+	// Filter by replica name if specified.
-	if opt.OutputPath == "" {
+	if c.replicaName != "" {
-		opt.OutputPath = dbPath
+		r := db.Replica(c.replicaName)
+		if r == nil {
+			return nil, fmt.Errorf("replica %q not found", c.replicaName)
+		}
+		return r, nil
 	}
 
-	// Determine the appropriate replica & generation to restore from,
+	// Choose only replica if only one available and no name is specified.
-	r, generation, err := db.CalcRestoreTarget(ctx, *opt)
+	if len(db.Replicas) == 1 {
+		return db.Replicas[0], nil
+	}
+
+	// A replica must be specified when restoring a specific generation with multiple replicas.
+	if c.generation != "" {
+		return nil, fmt.Errorf("must specify -replica flag when restoring from a specific generation")
+	}
+
+	// Determine latest replica to restore from.
+	r, err := litestream.LatestReplica(ctx, db.Replicas)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("cannot determine latest replica: %w", err)
 	}
-	opt.Generation = generation
-
 	return r, nil
 }
 
 // Usage prints the help screen to STDOUT.
 func (c *RestoreCommand) Usage() {
-	fmt.Printf(`
+	fmt.Fprintf(c.stdout, `
 The restore command recovers a database from a previous snapshot and WAL.
 
 Usage:
@@ -141,6 +242,9 @@ Arguments:
 	    Specifies the configuration file.
 	    Defaults to %s
 
+	-no-expand-env
+	    Disables environment variable expansion in configuration file.
+
 	-replica NAME
 	    Restore from a specific replica.
 	    Defaults to replica with latest data.
@@ -150,23 +254,26 @@ Arguments:
 	    Defaults to generation with latest data.
 
 	-index NUM
-	    Restore up to a specific WAL index (inclusive).
+	    Restore up to a specific hex-encoded WAL index (inclusive).
 	    Defaults to use the highest available index.
 
-	-timestamp TIMESTAMP
+	-timestamp DATETIME
-	    Restore to a specific point-in-time.
+	    Restore up to a specific point-in-time. Must be ISO 8601.
-	    Defaults to use the latest available backup.
+	    Cannot be specified with -index flag.
 
 	-o PATH
 	    Output path of the restored database.
 	    Defaults to original DB path.
 
-	-dry-run
+	-if-db-not-exists
-	    Prints all log output as if it were running but does
+	    Returns exit code of 0 if the database already exists.
-	    not perform actual restore.
 
-	-v
+	-if-replica-exists
-	    Verbose output.
+	    Returns exit code of 0 if no backups found.
+
+	-parallelism NUM
+	    Determines the number of WAL files downloaded in parallel.
+	    Defaults to `+strconv.Itoa(litestream.DefaultRestoreParallelism)+`.
 
 
 Examples:
@@ -174,9 +281,6 @@ Examples:
 	# Restore latest replica for database to original location.
 	$ litestream restore /path/to/db
 
-	# Restore replica for database to a given point in time.
-	$ litestream restore -timestamp 2020-01-01T00:00:00Z /path/to/db
-
 	# Restore latest replica for database to new /tmp directory
 	$ litestream restore -o /tmp/db /path/to/db
 
@@ -186,6 +290,9 @@ Examples:
 	# Restore database from specific generation on S3.
 	$ litestream restore -replica s3 -generation xxxxxxxx /path/to/db
 
+	# Restore database to a specific point in time.
+	$ litestream restore -generation xxxxxxxx -timestamp 2000-01-01T00:00:00Z /path/to/db
+
 `[1:],
 		DefaultConfigPath(),
 	)

cmd/litestream/restore_test.go

@@ -0,0 +1,330 @@
+package main_test
+
+import (
+	"context"
+	"flag"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/benbjohnson/litestream/internal/testingutil"
+)
+
+func TestRestoreCommand(t *testing.T) {
+	t.Run("OK", func(t *testing.T) {
+		testDir := filepath.Join("testdata", "restore", "ok")
+		defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
+		tempDir := t.TempDir()
+
+		m, _, stdout, stderr := newMain()
+		if err := m.Run(context.Background(), []string{"restore", "-config", filepath.Join(testDir, "litestream.yml"), "-o", filepath.Join(tempDir, "db"), filepath.Join(testDir, "db")}); err != nil {
+			t.Fatal(err)
+		} else if got, want := stderr.String(), ""; got != want {
+			t.Fatalf("stderr=%q, want %q", got, want)
+		}
+
+		// STDOUT has timing info so we need to grep per line.
+		lines := strings.Split(stdout.String(), "\n")
+		for i, substr := range []string{
+			`restoring snapshot 0000000000000000/0000000000000000 to ` + filepath.Join(tempDir, "db.tmp"),
+			`applied wal 0000000000000000/0000000000000000 elapsed=`,
+			`applied wal 0000000000000000/0000000000000001 elapsed=`,
+			`applied wal 0000000000000000/0000000000000002 elapsed=`,
+			`renaming database from temporary location`,
+		} {
+			if !strings.Contains(lines[i], substr) {
+				t.Fatalf("stdout: unexpected line %d:\n%s", i+1, stdout)
+			}
+		}
+	})
+
+	t.Run("ReplicaName", func(t *testing.T) {
+		testDir := filepath.Join("testdata", "restore", "replica-name")
+		defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
+		tempDir := t.TempDir()
+
+		m, _, stdout, stderr := newMain()
+		if err := m.Run(context.Background(), []string{"restore", "-config", filepath.Join(testDir, "litestream.yml"), "-o", filepath.Join(tempDir, "db"), "-replica", "replica1", filepath.Join(testDir, "db")}); err != nil {
+			t.Fatal(err)
+		} else if got, want := stderr.String(), ""; got != want {
+			t.Fatalf("stderr=%q, want %q", got, want)
+		}
+
+		// STDOUT has timing info so we need to grep per line.
+		lines := strings.Split(stdout.String(), "\n")
+		for i, substr := range []string{
+			`restoring snapshot 0000000000000001/0000000000000001 to ` + filepath.Join(tempDir, "db.tmp"),
+			`no wal files found, snapshot only`,
+			`renaming database from temporary location`,
+		} {
+			if !strings.Contains(lines[i], substr) {
+				t.Fatalf("stdout: unexpected line %d:\n%s", i+1, stdout)
+			}
+		}
+	})
+
+	t.Run("ReplicaURL", func(t *testing.T) {
+		testDir := filepath.Join(testingutil.Getwd(t), "testdata", "restore", "replica-url")
+		tempDir := t.TempDir()
+		replicaURL := "file://" + filepath.ToSlash(testDir) + "/replica"
+
+		m, _, stdout, stderr := newMain()
+		if err := m.Run(context.Background(), []string{"restore", "-o", filepath.Join(tempDir, "db"), replicaURL}); err != nil {
+			t.Fatal(err)
+		} else if got, want := stderr.String(), ""; got != want {
+			t.Fatalf("stderr=%q, want %q", got, want)
+		}
+
+		lines := strings.Split(stdout.String(), "\n")
+		for i, substr := range []string{
+			`restoring snapshot 0000000000000000/0000000000000000 to ` + filepath.Join(tempDir, "db.tmp"),
+			`no wal files found, snapshot only`,
+			`renaming database from temporary location`,
+		} {
+			if !strings.Contains(lines[i], substr) {
+				t.Fatalf("stdout: unexpected line %d:\n%s", i+1, stdout)
+			}
+		}
+	})
+
+	t.Run("LatestReplica", func(t *testing.T) {
+		testDir := filepath.Join("testdata", "restore", "latest-replica")
+		defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
+		tempDir := t.TempDir()
+
+		m, _, stdout, stderr := newMain()
+		if err := m.Run(context.Background(), []string{"restore", "-config", filepath.Join(testDir, "litestream.yml"), "-o", filepath.Join(tempDir, "db"), filepath.Join(testDir, "db")}); err != nil {
+			t.Fatal(err)
+		} else if got, want := stderr.String(), ""; got != want {
+			t.Fatalf("stderr=%q, want %q", got, want)
+		}
+
+		lines := strings.Split(stdout.String(), "\n")
+		for i, substr := range []string{
+			`restoring snapshot 0000000000000001/0000000000000000 to ` + filepath.Join(tempDir, "db.tmp"),
+			`no wal files found, snapshot only`,
+			`renaming database from temporary location`,
+		} {
+			if !strings.Contains(lines[i], substr) {
+				t.Fatalf("stdout: unexpected line %d:\n%s", i+1, stdout)
+			}
+		}
+	})
+
+	t.Run("IfDBNotExistsFlag", func(t *testing.T) {
+		testDir := filepath.Join("testdata", "restore", "if-db-not-exists-flag")
+		defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
+
+		m, _, stdout, _ := newMain()
+		err := m.Run(context.Background(), []string{"restore", "-config", filepath.Join(testDir, "litestream.yml"), "-if-db-not-exists", filepath.Join(testDir, "db")})
+		if err != nil {
+			t.Fatal(err)
+		} else if got, want := stdout.String(), string(testingutil.ReadFile(t, filepath.Join(testDir, "stdout"))); got != want {
+			t.Fatalf("stdout=%q, want %q", got, want)
+		}
+	})
+
+	t.Run("IfReplicaExists", func(t *testing.T) {
+		testDir := filepath.Join("testdata", "restore", "if-replica-exists-flag")
+		defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
+
+		m, _, stdout, _ := newMain()
+		err := m.Run(context.Background(), []string{"restore", "-config", filepath.Join(testDir, "litestream.yml"), "-if-replica-exists", filepath.Join(testDir, "db")})
+		if err != nil {
+			t.Fatal(err)
+		} else if got, want := stdout.String(), string(testingutil.ReadFile(t, filepath.Join(testDir, "stdout"))); got != want {
+			t.Fatalf("stdout=%q, want %q", got, want)
+		}
+	})
+
+	t.Run("ErrNoBackups", func(t *testing.T) {
+		testDir := filepath.Join("testdata", "restore", "no-backups")
+		defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
+		tempDir := t.TempDir()
+
+		m, _, stdout, stderr := newMain()
+		err := m.Run(context.Background(), []string{"restore", "-config", filepath.Join(testDir, "litestream.yml"), "-o", filepath.Join(tempDir, "db"), filepath.Join(testDir, "db")})
+		if err == nil || err.Error() != `no matching backups found` {
+			t.Fatalf("unexpected error: %s", err)
+		} else if got, want := stdout.String(), string(testingutil.ReadFile(t, filepath.Join(testDir, "stdout"))); got != want {
+			t.Fatalf("stdout=%q, want %q", got, want)
+		} else if got, want := stderr.String(), string(testingutil.ReadFile(t, filepath.Join(testDir, "stderr"))); got != want {
+			t.Fatalf("stderr=%q, want %q", got, want)
+		}
+	})
+
+	t.Run("ErrNoGeneration", func(t *testing.T) {
+		testDir := filepath.Join("testdata", "restore", "no-generation")
+		defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
+
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"restore", "-config", filepath.Join(testDir, "litestream.yml"), filepath.Join(testDir, "db")})
+		if err == nil || err.Error() != `no matching backups found` {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrOutputPathExists", func(t *testing.T) {
+		testDir := filepath.Join("testdata", "restore", "output-path-exists")
+		defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
+
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"restore", "-config", filepath.Join(testDir, "litestream.yml"), filepath.Join(testDir, "db")})
+		if err == nil || err.Error() != `output file already exists: `+filepath.Join(testDir, "db") {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrDatabaseOrReplicaRequired", func(t *testing.T) {
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"restore"})
+		if err == nil || err.Error() != `database path or replica URL required` {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrTooManyArguments", func(t *testing.T) {
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"restore", "abc", "123"})
+		if err == nil || err.Error() != `too many arguments` {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrInvalidFlags", func(t *testing.T) {
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"restore", "-no-such-flag"})
+		if err == nil || err.Error() != `flag provided but not defined: -no-such-flag` {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrIndexFlagOnly", func(t *testing.T) {
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"restore", "-index", "0", "/var/lib/db"})
+		if err == nil || err.Error() != `must specify -generation flag when using -index flag` {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrConfigFileNotFound", func(t *testing.T) {
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"restore", "-config", "/no/such/file", "/var/lib/db"})
+		if err == nil || err.Error() != `config file not found: /no/such/file` {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrInvalidConfig", func(t *testing.T) {
+		testDir := filepath.Join("testdata", "restore", "invalid-config")
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"restore", "-config", filepath.Join(testDir, "litestream.yml"), "/var/lib/db"})
+		if err == nil || !strings.Contains(err.Error(), `replica path cannot be a url`) {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrMkdir", func(t *testing.T) {
+		tempDir := t.TempDir()
+		if err := os.Mkdir(filepath.Join(tempDir, "noperm"), 0000); err != nil {
+			t.Fatal(err)
+		}
+
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"restore", "-o", filepath.Join(tempDir, "noperm", "subdir", "db"), "/var/lib/db"})
+		if err == nil || !strings.Contains(err.Error(), `permission denied`) {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrNoOutputPathWithReplicaURL", func(t *testing.T) {
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"restore", "file://path/to/replica"})
+		if err == nil || err.Error() != `output path required when using a replica URL` {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrReplicaNameWithReplicaURL", func(t *testing.T) {
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"restore", "-replica", "replica0", "file://path/to/replica"})
+		if err == nil || err.Error() != `cannot specify both the replica URL and the -replica flag` {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrInvalidReplicaURL", func(t *testing.T) {
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"restore", "-o", filepath.Join(t.TempDir(), "db"), "xyz://xyz"})
+		if err == nil || err.Error() != `unknown replica type in config: "xyz"` {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrDatabaseNotFound", func(t *testing.T) {
+		testDir := filepath.Join("testdata", "restore", "database-not-found")
+		defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
+
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"restore", "-config", filepath.Join(testDir, "litestream.yml"), "/no/such/db"})
+		if err == nil || err.Error() != `database not found in config: /no/such/db` {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrNoReplicas", func(t *testing.T) {
+		testDir := filepath.Join("testdata", "restore", "no-replicas")
+		defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
+		tempDir := t.TempDir()
+
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"restore", "-config", filepath.Join(testDir, "litestream.yml"), "-o", filepath.Join(tempDir, "db"), filepath.Join(testDir, "db")})
+		if err == nil || err.Error() != `database has no replicas: `+filepath.Join(testingutil.Getwd(t), testDir, "db") {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrReplicaNotFound", func(t *testing.T) {
+		testDir := filepath.Join("testdata", "restore", "replica-not-found")
+		defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
+		tempDir := t.TempDir()
+
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"restore", "-config", filepath.Join(testDir, "litestream.yml"), "-o", filepath.Join(tempDir, "db"), "-replica", "no_such_replica", filepath.Join(testDir, "db")})
+		if err == nil || err.Error() != `replica "no_such_replica" not found` {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrGenerationWithNoReplicaName", func(t *testing.T) {
+		testDir := filepath.Join("testdata", "restore", "generation-with-no-replica")
+		defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
+		tempDir := t.TempDir()
+
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"restore", "-config", filepath.Join(testDir, "litestream.yml"), "-o", filepath.Join(tempDir, "db"), "-generation", "0000000000000000", filepath.Join(testDir, "db")})
+		if err == nil || err.Error() != `must specify -replica flag when restoring from a specific generation` {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrNoSnapshotsAvailable", func(t *testing.T) {
+		testDir := filepath.Join("testdata", "restore", "no-snapshots")
+		defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
+		tempDir := t.TempDir()
+
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"restore", "-config", filepath.Join(testDir, "litestream.yml"), "-o", filepath.Join(tempDir, "db"), "-generation", "0000000000000000", filepath.Join(testDir, "db")})
+		if err == nil || err.Error() != `cannot determine latest index in generation "0000000000000000": no snapshots available` {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("Usage", func(t *testing.T) {
+		m, _, _, _ := newMain()
+		if err := m.Run(context.Background(), []string{"restore", "-h"}); err != flag.ErrHelp {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+}

cmd/litestream/snapshots.go

@@ -2,10 +2,11 @@ package main
 
 import (
 	"context"
-	"errors"
 	"flag"
 	"fmt"
-	"os"
+	"io"
+	"log"
+	"sort"
 	"text/tabwriter"
 	"time"
 
@@ -13,88 +14,90 @@ import (
 )
 
 // SnapshotsCommand represents a command to list snapshots for a command.
-type SnapshotsCommand struct{}
+type SnapshotsCommand struct {
+	stdin  io.Reader
+	stdout io.Writer
+	stderr io.Writer
+
+	configPath  string
+	noExpandEnv bool
+
+	replicaName string
+}
+
+// NewSnapshotsCommand returns a new instance of SnapshotsCommand.
+func NewSnapshotsCommand(stdin io.Reader, stdout, stderr io.Writer) *SnapshotsCommand {
+	return &SnapshotsCommand{
+		stdin:  stdin,
+		stdout: stdout,
+		stderr: stderr,
+	}
+}
 
 // Run executes the command.
-func (c *SnapshotsCommand) Run(ctx context.Context, args []string) (err error) {
+func (c *SnapshotsCommand) Run(ctx context.Context, args []string) (ret error) {
-	var configPath string
 	fs := flag.NewFlagSet("litestream-snapshots", flag.ContinueOnError)
-	registerConfigFlag(fs, &configPath)
+	registerConfigFlag(fs, &c.configPath, &c.noExpandEnv)
-	replicaName := fs.String("replica", "", "replica name")
+	fs.StringVar(&c.replicaName, "replica", "", "replica name")
 	fs.Usage = c.Usage
 	if err := fs.Parse(args); err != nil {
 		return err
 	} else if fs.NArg() == 0 || fs.Arg(0) == "" {
-		return fmt.Errorf("database path required")
+		return fmt.Errorf("database path or replica URL required")
 	} else if fs.NArg() > 1 {
 		return fmt.Errorf("too many arguments")
 	}
 
-	var db *litestream.DB
+	// Load configuration.
-	var r litestream.Replica
+	config, err := ReadConfigFile(c.configPath, !c.noExpandEnv)
-	if isURL(fs.Arg(0)) {
+	if err != nil {
-		if r, err = NewReplicaFromURL(fs.Arg(0)); err != nil {
+		return err
-			return err
+	}
-		}
+
-	} else if configPath != "" {
+	// Determine list of replicas to pull snapshots from.
-		// Load configuration.
+	replicas, _, err := loadReplicas(ctx, config, fs.Arg(0), c.replicaName)
-		config, err := ReadConfigFile(configPath)
+	if err != nil {
+		return err
+	}
+
+	// Build list of snapshot metadata with associated replica.
+	var infos []replicaSnapshotInfo
+	for _, r := range replicas {
+		a, err := r.Snapshots(ctx)
 		if err != nil {
-			return err
+			log.Printf("cannot determine snapshots: %s", err)
+			ret = errExit // signal error return without printing message
+			continue
 		}
-
+		for i := range a {
-		// Lookup database from configuration file by path.
+			infos = append(infos, replicaSnapshotInfo{SnapshotInfo: a[i], replicaName: r.Name()})
-		if path, err := expand(fs.Arg(0)); err != nil {
-			return err
-		} else if dbc := config.DBConfig(path); dbc == nil {
-			return fmt.Errorf("database not found in config: %s", path)
-		} else if db, err = newDBFromConfig(&config, dbc); err != nil {
-			return err
-		}
-
-		// Filter by replica, if specified.
-		if *replicaName != "" {
-			if r = db.Replica(*replicaName); r == nil {
-				return fmt.Errorf("replica %q not found for database %q", *replicaName, db.Path())
-			}
-		}
-	} else {
-		return errors.New("config path or replica URL required")
-	}
-
-	// Find snapshots by db or replica.
-	var infos []*litestream.SnapshotInfo
-	if r != nil {
-		if infos, err = r.Snapshots(ctx); err != nil {
-			return err
-		}
-	} else {
-		if infos, err = db.Snapshots(ctx); err != nil {
-			return err
 		}
 	}
 
+	// Sort snapshots by creation time from newest to oldest.
+	sort.Slice(infos, func(i, j int) bool { return infos[i].CreatedAt.After(infos[j].CreatedAt) })
+
 	// List all snapshots.
-	w := tabwriter.NewWriter(os.Stdout, 0, 8, 2, ' ', 0)
+	w := tabwriter.NewWriter(c.stdout, 0, 8, 2, ' ', 0)
 	defer w.Flush()
 
 	fmt.Fprintln(w, "replica\tgeneration\tindex\tsize\tcreated")
 	for _, info := range infos {
-		fmt.Fprintf(w, "%s\t%s\t%d\t%d\t%s\n",
+		fmt.Fprintf(w, "%s\t%s\t%s\t%d\t%s\n",
-			info.Replica,
+			info.replicaName,
 			info.Generation,
-			info.Index,
+			litestream.FormatIndex(info.Index),
 			info.Size,
 			info.CreatedAt.Format(time.RFC3339),
 		)
 	}
 
-	return nil
+	return ret
 }
 
 // Usage prints the help screen to STDOUT.
 func (c *SnapshotsCommand) Usage() {
-	fmt.Printf(`
+	fmt.Fprintf(c.stdout, `
 The snapshots command lists all snapshots available for a database or replica.
 
 Usage:
@@ -109,6 +112,9 @@ Arguments:
 	    Specifies the configuration file.
 	    Defaults to %s
 
+	-no-expand-env
+	    Disables environment variable expansion in configuration file.
+
 	-replica NAME
 	    Optional, filter by a specific replica.
 
@@ -127,3 +133,9 @@ Examples:
 		DefaultConfigPath(),
 	)
 }
+
+// replicaSnapshotInfo represents snapshot metadata with associated replica name.
+type replicaSnapshotInfo struct {
+	litestream.SnapshotInfo
+	replicaName string
+}

cmd/litestream/snapshots_test.go

@@ -0,0 +1,128 @@
+package main_test
+
+import (
+	"context"
+	"flag"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/benbjohnson/litestream/internal/testingutil"
+)
+
+func TestSnapshotsCommand(t *testing.T) {
+	t.Run("OK", func(t *testing.T) {
+		testDir := filepath.Join("testdata", "snapshots", "ok")
+		defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
+
+		m, _, stdout, _ := newMain()
+		if err := m.Run(context.Background(), []string{"snapshots", "-config", filepath.Join(testDir, "litestream.yml"), filepath.Join(testDir, "db")}); err != nil {
+			t.Fatal(err)
+		} else if got, want := stdout.String(), string(testingutil.ReadFile(t, filepath.Join(testDir, "stdout"))); got != want {
+			t.Fatalf("stdout=%q, want %q", got, want)
+		}
+	})
+
+	t.Run("ReplicaName", func(t *testing.T) {
+		testDir := filepath.Join("testdata", "snapshots", "replica-name")
+		defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
+
+		m, _, stdout, _ := newMain()
+		if err := m.Run(context.Background(), []string{"snapshots", "-config", filepath.Join(testDir, "litestream.yml"), "-replica", "replica1", filepath.Join(testDir, "db")}); err != nil {
+			t.Fatal(err)
+		} else if got, want := stdout.String(), string(testingutil.ReadFile(t, filepath.Join(testDir, "stdout"))); got != want {
+			t.Fatalf("stdout=%q, want %q", got, want)
+		}
+	})
+
+	t.Run("ReplicaURL", func(t *testing.T) {
+		testDir := filepath.Join(testingutil.Getwd(t), "testdata", "snapshots", "replica-url")
+		defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
+		replicaURL := "file://" + filepath.ToSlash(testDir) + "/replica"
+
+		m, _, stdout, _ := newMain()
+		if err := m.Run(context.Background(), []string{"snapshots", replicaURL}); err != nil {
+			t.Fatal(err)
+		} else if got, want := stdout.String(), string(testingutil.ReadFile(t, filepath.Join(testDir, "stdout"))); got != want {
+			t.Fatalf("stdout=%q, want %q", got, want)
+		}
+	})
+
+	t.Run("ErrDatabaseOrReplicaRequired", func(t *testing.T) {
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"snapshots"})
+		if err == nil || err.Error() != `database path or replica URL required` {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrTooManyArguments", func(t *testing.T) {
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"snapshots", "abc", "123"})
+		if err == nil || err.Error() != `too many arguments` {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrInvalidFlags", func(t *testing.T) {
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"snapshots", "-no-such-flag"})
+		if err == nil || err.Error() != `flag provided but not defined: -no-such-flag` {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrConfigFileNotFound", func(t *testing.T) {
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"snapshots", "-config", "/no/such/file", "/var/lib/db"})
+		if err == nil || err.Error() != `config file not found: /no/such/file` {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrInvalidConfig", func(t *testing.T) {
+		testDir := filepath.Join("testdata", "snapshots", "invalid-config")
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"snapshots", "-config", filepath.Join(testDir, "litestream.yml"), "/var/lib/db"})
+		if err == nil || !strings.Contains(err.Error(), `replica path cannot be a url`) {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrDatabaseNotFound", func(t *testing.T) {
+		testDir := filepath.Join("testdata", "snapshots", "database-not-found")
+		defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
+
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"snapshots", "-config", filepath.Join(testDir, "litestream.yml"), "/no/such/db"})
+		if err == nil || err.Error() != `database not found in config: /no/such/db` {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrReplicaNotFound", func(t *testing.T) {
+		testDir := filepath.Join(testingutil.Getwd(t), "testdata", "snapshots", "replica-not-found")
+		defer testingutil.Setenv(t, "LITESTREAM_TESTDIR", testDir)()
+
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"snapshots", "-config", filepath.Join(testDir, "litestream.yml"), "-replica", "no_such_replica", filepath.Join(testDir, "db")})
+		if err == nil || err.Error() != `replica "no_such_replica" not found for database "`+filepath.Join(testDir, "db")+`"` {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrInvalidReplicaURL", func(t *testing.T) {
+		m, _, _, _ := newMain()
+		err := m.Run(context.Background(), []string{"snapshots", "xyz://xyz"})
+		if err == nil || !strings.Contains(err.Error(), `unknown replica type in config: "xyz"`) {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("Usage", func(t *testing.T) {
+		m, _, _, _ := newMain()
+		if err := m.Run(context.Background(), []string{"snapshots", "-h"}); err != flag.ErrHelp {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+}

cmd/litestream/testdata/Makefile

@@ -0,0 +1,13 @@
+.PHONY: default
+default:
+	make -C generations/ok
+	make -C generations/no-database
+	make -C generations/replica-name
+	make -C generations/replica-url
+	make -C restore/latest-replica
+	make -C snapshots/ok
+	make -C snapshots/replica-name
+	make -C snapshots/replica-url
+	make -C wal/ok
+	make -C wal/replica-name
+	make -C wal/replica-url

cmd/litestream/testdata/databases/invalid-config/litestream.yml

@@ -0,0 +1,4 @@
+dbs:
+  - path: /var/lib/db
+    replicas:
+      - path: s3://bkt/db

cmd/litestream/testdata/databases/no-databases/litestream.yml

@@ -0,0 +1 @@
+dbs:

cmd/litestream/testdata/databases/no-databases/stdout

@@ -0,0 +1 @@
+No databases found in config file.

cmd/litestream/testdata/databases/ok/litestream.yml

@@ -0,0 +1,7 @@
+dbs:
+  - path: /var/lib/db
+    replicas:
+      - path: /var/lib/replica
+      - url:  s3://mybkt/db
+
+  - path: /my/other/db

cmd/litestream/testdata/databases/ok/stdout

@@ -0,0 +1,3 @@
+path          replicas
+/var/lib/db   file,s3
+/my/other/db  

cmd/litestream/testdata/generations/database-not-found/litestream.yml

@@ -0,0 +1,2 @@
+dbs:
+  - path: $LITESTREAM_TESTDIR/db

cmd/litestream/testdata/generations/invalid-config/litestream.yml

@@ -0,0 +1,4 @@
+dbs:
+  - path: /var/lib/db
+    replicas:
+      - path: s3://bkt/db

cmd/litestream/testdata/generations/no-database/Makefile

@@ -0,0 +1,4 @@
+.PHONY: default
+default:
+	TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/snapshots/0000000000000000.snapshot.lz4
+	TZ=UTC touch -ct 200001020000 replica/generations/0000000000000001/snapshots/0000000000000000.snapshot.lz4

cmd/litestream/testdata/generations/no-database/litestream.yml

@@ -0,0 +1,4 @@
+dbs:
+  - path: $LITESTREAM_TESTDIR/db
+    replicas:
+      - path: $LITESTREAM_TESTDIR/replica

cmd/litestream/testdata/generations/no-database/stdout

@@ -0,0 +1,3 @@
+name  generation        lag  start                 end
+file  0000000000000000  -    2000-01-01T00:00:00Z  2000-01-01T00:00:00Z
+file  0000000000000001  -    2000-01-02T00:00:00Z  2000-01-02T00:00:00Z

cmd/litestream/testdata/generations/ok/Makefile

@@ -0,0 +1,9 @@
+.PHONY: default
+default:
+	TZ=UTC touch -ct 200001030000 db
+	TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/snapshots/0000000000000000.snapshot.lz4
+	TZ=UTC touch -ct 200001020000 replica/generations/0000000000000000/snapshots/0000000000000001.snapshot.lz4
+	TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/wal/0000000000000000/0000000000000000.wal.lz4
+	TZ=UTC touch -ct 200001020000 replica/generations/0000000000000000/wal/0000000000000000/0000000000000001.wal.lz4
+	TZ=UTC touch -ct 200001030000 replica/generations/0000000000000000/wal/0000000000000001/0000000000000000.wal.lz4
+	TZ=UTC touch -ct 200001010000 replica/generations/0000000000000001/snapshots/0000000000000000.snapshot.lz4

cmd/litestream/testdata/generations/ok/litestream.yml

@@ -0,0 +1,4 @@
+dbs:
+  - path: $LITESTREAM_TESTDIR/db
+    replicas:
+      - path: $LITESTREAM_TESTDIR/replica

cmd/litestream/testdata/generations/ok/stdout

@@ -0,0 +1,3 @@
+name  generation        lag      start                 end
+file  0000000000000000  0s       2000-01-01T00:00:00Z  2000-01-03T00:00:00Z
+file  0000000000000001  48h0m0s  2000-01-01T00:00:00Z  2000-01-01T00:00:00Z

cmd/litestream/testdata/generations/replica-name/Makefile

@@ -0,0 +1,5 @@
+.PHONY: default
+default:
+	TZ=UTC touch -ct 200001030000 db
+	TZ=UTC touch -ct 200001010000 replica0/generations/0000000000000000/snapshots/0000000000000000.snapshot.lz4
+	TZ=UTC touch -ct 200001020000 replica1/generations/0000000000000001/snapshots/0000000000000000.snapshot.lz4

cmd/litestream/testdata/generations/replica-name/litestream.yml

@@ -0,0 +1,7 @@
+dbs:
+  - path: $LITESTREAM_TESTDIR/db
+    replicas:
+      - name: replica0
+        path: $LITESTREAM_TESTDIR/replica0
+      - name: replica1
+        path: $LITESTREAM_TESTDIR/replica1

cmd/litestream/testdata/generations/replica-name/stdout

@@ -0,0 +1,2 @@
+name      generation        lag      start                 end
+replica1  0000000000000001  24h0m0s  2000-01-02T00:00:00Z  2000-01-02T00:00:00Z

cmd/litestream/testdata/generations/replica-not-found/litestream.yml

@@ -0,0 +1,4 @@
+dbs:
+  - path: $LITESTREAM_TESTDIR/db
+    replicas:
+      - url: s3://bkt/db

cmd/litestream/testdata/generations/replica-url/Makefile

@@ -0,0 +1,9 @@
+.PHONY: default
+default:
+	TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/snapshots/0000000000000000.snapshot.lz4
+	TZ=UTC touch -ct 200001020000 replica/generations/0000000000000000/snapshots/0000000000000001.snapshot.lz4
+	TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/wal/0000000000000000/0000000000000000.wal.lz4
+	TZ=UTC touch -ct 200001020000 replica/generations/0000000000000000/wal/0000000000000000/0000000000000001.wal.lz4
+	TZ=UTC touch -ct 200001030000 replica/generations/0000000000000000/wal/0000000000000001/0000000000000000.wal.lz4
+	TZ=UTC touch -ct 200001020000 replica/generations/0000000000000001/snapshots/0000000000000000.snapshot.lz4
+

cmd/litestream/testdata/generations/replica-url/litestream.yml

@@ -0,0 +1,4 @@
+dbs:
+  - path: $LITESTREAM_TESTDIR/db
+    replicas:
+      - path: $LITESTREAM_TESTDIR/replica

cmd/litestream/testdata/generations/replica-url/stdout

@@ -0,0 +1,3 @@
+name  generation        lag  start                 end
+file  0000000000000000  -    2000-01-01T00:00:00Z  2000-01-03T00:00:00Z
+file  0000000000000001  -    2000-01-02T00:00:00Z  2000-01-02T00:00:00Z

cmd/litestream/testdata/restore/database-not-found/litestream.yml

@@ -0,0 +1,4 @@
+dbs:
+  - path: $LITESTREAM_TESTDIR/db
+    replicas:
+      - path: $LITESTREAM_TESTDIR/replica

cmd/litestream/testdata/restore/generation-with-no-replica/litestream.yml

@@ -0,0 +1,5 @@
+dbs:
+  - path: $LITESTREAM_TESTDIR/db
+    replicas:
+      - path: $LITESTREAM_TESTDIR/replica0
+      - path: $LITESTREAM_TESTDIR/replica1

cmd/litestream/testdata/restore/if-db-not-exists-flag/litestream.yml

@@ -0,0 +1,4 @@
+dbs:
+  - path: $LITESTREAM_TESTDIR/db
+    replicas:
+      - path: $LITESTREAM_TESTDIR/replica

cmd/litestream/testdata/restore/if-db-not-exists-flag/stdout

@@ -0,0 +1 @@
+database already exists, skipping

cmd/litestream/testdata/restore/if-replica-exists-flag/litestream.yml

@@ -0,0 +1,4 @@
+dbs:
+  - path: $LITESTREAM_TESTDIR/db
+    replicas:
+      - path: $LITESTREAM_TESTDIR/replica

cmd/litestream/testdata/restore/if-replica-exists-flag/stdout

@@ -0,0 +1 @@
+no matching backups found, skipping

cmd/litestream/testdata/restore/invalid-config/litestream.yml

@@ -0,0 +1,4 @@
+dbs:
+  - path: /var/lib/db
+    replicas:
+      - path: s3://bkt/db

cmd/litestream/testdata/restore/latest-replica/Makefile

@@ -0,0 +1,6 @@
+.PHONY: default
+default:
+	TZ=UTC touch -ct 200001010000 replica0/generations/0000000000000000/snapshots/0000000000000000.snapshot.lz4
+	TZ=UTC touch -ct 200001020000 replica1/generations/0000000000000002/snapshots/0000000000000000.snapshot.lz4
+	TZ=UTC touch -ct 200001030000 replica0/generations/0000000000000001/snapshots/0000000000000000.snapshot.lz4
+

cmd/litestream/testdata/restore/latest-replica/litestream.yml

@@ -0,0 +1,7 @@
+dbs:
+  - path: $LITESTREAM_TESTDIR/db
+    replicas:
+      - name: replica0
+        path: $LITESTREAM_TESTDIR/replica0
+      - name: replica1
+        path: $LITESTREAM_TESTDIR/replica1

cmd/litestream/testdata/restore/no-backups/litestream.yml

@@ -0,0 +1,4 @@
+dbs:
+  - path: $LITESTREAM_TESTDIR/db
+    replicas:
+      - path: $LITESTREAM_TESTDIR/replica

cmd/litestream/testdata/restore/no-generation/litestream.yml

@@ -0,0 +1,4 @@
+dbs:
+  - path: $LITESTREAM_TESTDIR/db
+    replicas:
+      - path: $LITESTREAM_TESTDIR/replica

cmd/litestream/testdata/restore/no-replicas/litestream.yml

@@ -0,0 +1,2 @@
+dbs:
+  - path: $LITESTREAM_TESTDIR/db

cmd/litestream/testdata/restore/no-snapshots/litestream.yml

@@ -0,0 +1,4 @@
+dbs:
+  - path: $LITESTREAM_TESTDIR/db
+    replicas:
+      - path: $LITESTREAM_TESTDIR/replica

cmd/litestream/testdata/restore/ok/README

@@ -0,0 +1,36 @@
+To reproduce this testdata, run sqlite3 and execute:
+
+	PRAGMA journal_mode = WAL;
+	CREATE TABLE t (x);
+	INSERT INTO t (x) VALUES (1);
+	INSERT INTO t (x) VALUES (2);
+
+	sl3 split -o generations/0000000000000000/wal/0000000000000000 db-wal
+	cp db generations/0000000000000000/snapshots/0000000000000000.snapshot
+	lz4 -c --rm generations/0000000000000000/snapshots/0000000000000000.snapshot
+
+
+Then execute:
+
+	PRAGMA wal_checkpoint(TRUNCATE);
+	INSERT INTO t (x) VALUES (3);
+
+	sl3 split -o generations/0000000000000000/wal/0000000000000001 db-wal
+
+
+Then execute:
+
+	PRAGMA wal_checkpoint(TRUNCATE);
+	INSERT INTO t (x) VALUES (4);
+	INSERT INTO t (x) VALUES (5);
+
+	sl3 split -o generations/0000000000000000/wal/0000000000000002 db-wal
+
+
+Finally, obtain the final snapshot:
+
+	PRAGMA wal_checkpoint(TRUNCATE);
+
+	cp db 0000000000000002.db
+	rm db*
+

cmd/litestream/testdata/restore/ok/litestream.yml

@@ -0,0 +1,4 @@
+dbs:
+  - path: $LITESTREAM_TESTDIR/db
+    replicas:
+      - path: $LITESTREAM_TESTDIR/replica