diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 93a9e19..82dc031 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,13 +17,25 @@ jobs: restore-keys: | ${{ runner.os }}-go- + - name: Extract GCP credentials + run: 'echo "$GOOGLE_APPLICATION_CREDENTIALS" > /opt/gcp.json' + shell: bash + env: + GOOGLE_APPLICATION_CREDENTIALS: ${{secrets.GOOGLE_APPLICATION_CREDENTIALS}} + - name: Run unit tests run: go test -v ./... - name: Run s3 integration tests - run: go test -v ./s3 -integration + run: go test -v -run=TestReplicaClient . -integration s3 env: LITESTREAM_S3_ACCESS_KEY_ID: ${{ secrets.LITESTREAM_S3_ACCESS_KEY_ID }} LITESTREAM_S3_SECRET_ACCESS_KEY: ${{ secrets.LITESTREAM_S3_SECRET_ACCESS_KEY }} LITESTREAM_S3_REGION: ${{ secrets.LITESTREAM_S3_REGION }} LITESTREAM_S3_BUCKET: ${{ secrets.LITESTREAM_S3_BUCKET }} + + - name: Run gcs integration tests + run: go test -v -run=TestReplicaClient . -integration gcs + env: + GOOGLE_APPLICATION_CREDENTIALS: /opt/gcp.json + LITESTREAM_GCS_BUCKET: ${{ secrets.LITESTREAM_GCS_BUCKET }} diff --git a/cmd/litestream/main.go b/cmd/litestream/main.go index bf98a82..fd3be22 100644 --- a/cmd/litestream/main.go +++ b/cmd/litestream/main.go @@ -20,6 +20,7 @@ import ( "github.com/benbjohnson/litestream" "github.com/benbjohnson/litestream/file" + "github.com/benbjohnson/litestream/gcs" "github.com/benbjohnson/litestream/s3" _ "github.com/mattn/go-sqlite3" "gopkg.in/yaml.v2" @@ -330,6 +331,10 @@ func NewReplicaFromConfig(c *ReplicaConfig, db *litestream.DB) (_ *litestream.Re if r.Client, err = newS3ReplicaClientFromConfig(c, r); err != nil { return nil, err } + case "gcs": + if r.Client, err = newGCSReplicaClientFromConfig(c, r); err != nil { + return nil, err + } default: return nil, fmt.Errorf("unknown replica type in config: %q", c.Type) } @@ -431,6 +436,45 @@ func newS3ReplicaClientFromConfig(c *ReplicaConfig, r *litestream.Replica) (_ *s return client, nil } +// newGCSReplicaClientFromConfig returns a new instance of gcs.ReplicaClient built from config. +func newGCSReplicaClientFromConfig(c *ReplicaConfig, r *litestream.Replica) (_ *gcs.ReplicaClient, err error) { + // Ensure URL & constituent parts are not both specified. + if c.URL != "" && c.Path != "" { + return nil, fmt.Errorf("cannot specify url & path for gcs replica") + } else if c.URL != "" && c.Bucket != "" { + return nil, fmt.Errorf("cannot specify url & bucket for gcs replica") + } + + bucket, path := c.Bucket, c.Path + + // Apply settings from URL, if specified. + if c.URL != "" { + _, uhost, upath, err := ParseReplicaURL(c.URL) + if err != nil { + return nil, err + } + + // Only apply URL parts to field that have not been overridden. + if path == "" { + path = upath + } + if bucket == "" { + bucket = uhost + } + } + + // Ensure required settings are set. + if bucket == "" { + return nil, fmt.Errorf("bucket required for gcs replica") + } + + // Build replica. + client := gcs.NewReplicaClient() + client.Bucket = bucket + client.Path = path + return client, nil +} + // applyLitestreamEnv copies "LITESTREAM" prefixed environment variables to // their AWS counterparts as the "AWS" prefix can be confusing when using a // non-AWS S3-compatible service. diff --git a/cmd/litestream/main_test.go b/cmd/litestream/main_test.go index feded64..d99c52d 100644 --- a/cmd/litestream/main_test.go +++ b/cmd/litestream/main_test.go @@ -8,6 +8,7 @@ import ( main "github.com/benbjohnson/litestream/cmd/litestream" "github.com/benbjohnson/litestream/file" + "github.com/benbjohnson/litestream/gcs" "github.com/benbjohnson/litestream/s3" ) @@ -160,23 +161,17 @@ func TestNewS3ReplicaFromConfig(t *testing.T) { t.Fatalf("ForcePathStyle=%v, want %v", got, want) } }) - - t.Run("GCS", func(t *testing.T) { - r, err := main.NewReplicaFromConfig(&main.ReplicaConfig{URL: "s3://foo.storage.googleapis.com/bar"}, nil) - if err != nil { - t.Fatal(err) - } else if client, ok := r.Client.(*s3.ReplicaClient); !ok { - t.Fatal("unexpected replica type") - } else if got, want := client.Bucket, "foo"; got != want { - t.Fatalf("Bucket=%s, want %s", got, want) - } else if got, want := client.Path, "bar"; got != want { - t.Fatalf("Path=%s, want %s", got, want) - } else if got, want := client.Region, "us-east-1"; got != want { - t.Fatalf("Region=%s, want %s", got, want) - } else if got, want := client.Endpoint, "https://storage.googleapis.com"; got != want { - t.Fatalf("Endpoint=%s, want %s", got, want) - } else if got, want := client.ForcePathStyle, true; got != want { - t.Fatalf("ForcePathStyle=%v, want %v", got, want) - } - }) +} + +func TestNewGCSReplicaFromConfig(t *testing.T) { + r, err := main.NewReplicaFromConfig(&main.ReplicaConfig{URL: "gcs://foo/bar"}, nil) + if err != nil { + t.Fatal(err) + } else if client, ok := r.Client.(*gcs.ReplicaClient); !ok { + t.Fatal("unexpected replica type") + } else if got, want := client.Bucket, "foo"; got != want { + t.Fatalf("Bucket=%s, want %s", got, want) + } else if got, want := client.Path, "bar"; got != want { + t.Fatalf("Path=%s, want %s", got, want) + } } diff --git a/cmd/litestream/replicate.go b/cmd/litestream/replicate.go index aa78d2b..c3b9ca8 100644 --- a/cmd/litestream/replicate.go +++ b/cmd/litestream/replicate.go @@ -12,6 +12,7 @@ import ( "github.com/benbjohnson/litestream" "github.com/benbjohnson/litestream/file" + "github.com/benbjohnson/litestream/gcs" "github.com/benbjohnson/litestream/s3" "github.com/prometheus/client_golang/prometheus/promhttp" ) @@ -108,6 +109,8 @@ func (c *ReplicateCommand) Run(ctx context.Context) (err error) { log.Printf("replicating to: name=%q type=%q path=%q", r.Name(), client.Type(), client.Path()) case *s3.ReplicaClient: log.Printf("replicating to: name=%q type=%q bucket=%q path=%q region=%q endpoint=%q sync-interval=%s", r.Name(), client.Type(), client.Bucket, client.Path, client.Region, client.Endpoint, r.SyncInterval) + case *gcs.ReplicaClient: + log.Printf("replicating to: name=%q type=%q bucket=%q path=%q sync-interval=%s", r.Name(), client.Type(), client.Bucket, client.Path, r.SyncInterval) default: log.Printf("replicating to: name=%q type=%q", r.Name(), client.Type()) } diff --git a/file/replica_client.go b/file/replica_client.go index 12cbe0b..178797a 100644 --- a/file/replica_client.go +++ b/file/replica_client.go @@ -13,6 +13,9 @@ import ( "github.com/benbjohnson/litestream/internal" ) +// ReplicaClientType is the client type for this package. +const ReplicaClientType = "file" + var _ litestream.ReplicaClient = (*ReplicaClient)(nil) // ReplicaClient is a client for writing snapshots & WAL segments to disk. @@ -39,7 +42,7 @@ func (c *ReplicaClient) db() *litestream.DB { // Type returns "file" as the client type. func (c *ReplicaClient) Type() string { - return "file" + return ReplicaClientType } // Path returns the destination path to replicate the database to. @@ -132,7 +135,7 @@ func (c *ReplicaClient) Generations(ctx context.Context) ([]string, error) { func (c *ReplicaClient) DeleteGeneration(ctx context.Context, generation string) error { dir, err := c.GenerationDir(generation) if err != nil { - return fmt.Errorf("cannot determine generation directory: %w", err) + return fmt.Errorf("cannot determine generation path: %w", err) } if err := os.RemoveAll(dir); err != nil && !os.IsNotExist(err) { @@ -145,7 +148,7 @@ func (c *ReplicaClient) DeleteGeneration(ctx context.Context, generation string) func (c *ReplicaClient) Snapshots(ctx context.Context, generation string) (litestream.SnapshotIterator, error) { dir, err := c.SnapshotsDir(generation) if err != nil { - return nil, fmt.Errorf("cannot determine snapshot directory: %w", err) + return nil, fmt.Errorf("cannot determine snapshots path: %w", err) } f, err := os.Open(dir) @@ -261,7 +264,7 @@ func (c *ReplicaClient) DeleteSnapshot(ctx context.Context, generation string, i func (c *ReplicaClient) WALSegments(ctx context.Context, generation string) (litestream.WALSegmentIterator, error) { dir, err := c.WALDir(generation) if err != nil { - return nil, fmt.Errorf("cannot determine wal directory: %w", err) + return nil, fmt.Errorf("cannot determine wal path: %w", err) } f, err := os.Open(dir) diff --git a/file/replica_client_test.go b/file/replica_client_test.go index 0d24979..94d2e44 100644 --- a/file/replica_client_test.go +++ b/file/replica_client_test.go @@ -1,18 +1,9 @@ package file_test import ( - "bytes" - "context" - "io/ioutil" - "os" - "path/filepath" - "reflect" - "sort" "testing" - "github.com/benbjohnson/litestream" "github.com/benbjohnson/litestream/file" - "github.com/pierrec/lz4/v4" ) func TestReplicaClient_Path(t *testing.T) { @@ -143,440 +134,6 @@ func TestReplicaClient_WALSegmentPath(t *testing.T) { }) } -func TestReplicaClient_Generations(t *testing.T) { - t.Run("OK", func(t *testing.T) { - dir := t.TempDir() - if err := os.MkdirAll(filepath.Join(dir, "generations", "5efbd8d042012dca"), 0777); err != nil { - t.Fatal(err) - } else if err := os.MkdirAll(filepath.Join(dir, "generations", "b16ddcf5c697540f"), 0777); err != nil { - t.Fatal(err) - } else if err := os.MkdirAll(filepath.Join(dir, "generations", "155fe292f8333c72"), 0777); err != nil { - t.Fatal(err) - } - - c := file.NewReplicaClient(dir) - if got, err := c.Generations(context.Background()); err != nil { - t.Fatal(err) - } else if want := []string{"155fe292f8333c72", "5efbd8d042012dca", "b16ddcf5c697540f"}; !reflect.DeepEqual(got, want) { - t.Fatalf("Generations()=%v, want %v", got, want) - } - }) - - t.Run("WithInvalidEntries", func(t *testing.T) { - dir := t.TempDir() - if err := os.MkdirAll(filepath.Join(dir, "generations", "5efbd8d042012dca"), 0777); err != nil { - t.Fatal(err) - } else if err := os.MkdirAll(filepath.Join(dir, "generations", "not_a_generation"), 0777); err != nil { - t.Fatal(err) - } else if err := ioutil.WriteFile(filepath.Join(dir, "generations", "0000000000000000"), nil, 0666); err != nil { - t.Fatal(err) - } - - c := file.NewReplicaClient(dir) - if got, err := c.Generations(context.Background()); err != nil { - t.Fatal(err) - } else if want := []string{"5efbd8d042012dca"}; !reflect.DeepEqual(got, want) { - t.Fatalf("Generations()=%v, want %v", got, want) - } - }) - - t.Run("NoGenerationsDir", func(t *testing.T) { - c := file.NewReplicaClient(t.TempDir()) - if generations, err := c.Generations(context.Background()); err != nil { - t.Fatal(err) - } else if got, want := len(generations), 0; got != want { - t.Fatalf("len(Generations())=%v, want %v", got, want) - } - }) - - t.Run("ErrNoPath", func(t *testing.T) { - if _, err := file.NewReplicaClient("").Generations(context.Background()); err == nil || err.Error() != `cannot determine generations path: file replica path required` { - t.Fatalf("unexpected error: %v", err) - } - }) -} - -func TestReplicaClient_Snapshots(t *testing.T) { - t.Run("OK", func(t *testing.T) { - dir := t.TempDir() - if err := os.MkdirAll(filepath.Join(dir, "generations", "5efbd8d042012dca", "snapshots"), 0777); err != nil { - t.Fatal(err) - } else if err := ioutil.WriteFile(filepath.Join(dir, "generations", "5efbd8d042012dca", "snapshots", "00000001.snapshot.lz4"), nil, 0666); err != nil { - t.Fatal(err) - } - - if err := os.MkdirAll(filepath.Join(dir, "generations", "b16ddcf5c697540f", "snapshots"), 0777); err != nil { - t.Fatal(err) - } else if err := ioutil.WriteFile(filepath.Join(dir, "generations", "b16ddcf5c697540f", "snapshots", "00000005.snapshot.lz4"), []byte("x"), 0666); err != nil { - t.Fatal(err) - } else if err := ioutil.WriteFile(filepath.Join(dir, "generations", "b16ddcf5c697540f", "snapshots", "0000000a.snapshot.lz4"), []byte("xyz"), 0666); err != nil { - t.Fatal(err) - } else if err := ioutil.WriteFile(filepath.Join(dir, "generations", "b16ddcf5c697540f", "snapshots", "not_a_snapshot.snapshot.lz4"), nil, 0666); err != nil { - t.Fatal(err) - } - - c := file.NewReplicaClient(dir) - itr, err := c.Snapshots(context.Background(), "b16ddcf5c697540f") - if err != nil { - t.Fatal(err) - } - defer itr.Close() - - // Read all snapshots into a slice so they can be sorted. - a, err := litestream.SliceSnapshotIterator(itr) - if err != nil { - t.Fatal(err) - } else if got, want := len(a), 2; got != want { - t.Fatalf("len=%v, want %v", got, want) - } - sort.Sort(litestream.SnapshotInfoSlice(a)) - - // Verify first snapshot metadata. - if got, want := a[0].Generation, "b16ddcf5c697540f"; got != want { - t.Fatalf("Generation=%v, want %v", got, want) - } else if got, want := a[0].Index, 5; got != want { - t.Fatalf("Index=%v, want %v", got, want) - } else if got, want := a[0].Size, int64(1); got != want { - t.Fatalf("Size=%v, want %v", got, want) - } else if a[0].CreatedAt.IsZero() { - t.Fatalf("expected CreatedAt") - } - - // Verify second snapshot metadata. - if got, want := a[1].Generation, "b16ddcf5c697540f"; got != want { - t.Fatalf("Generation=%v, want %v", got, want) - } else if got, want := a[1].Index, 0xA; got != want { - t.Fatalf("Index=%v, want %v", got, want) - } else if got, want := a[1].Size, int64(3); got != want { - t.Fatalf("Size=%v, want %v", got, want) - } else if a[1].CreatedAt.IsZero() { - t.Fatalf("expected CreatedAt") - } - - // Ensure close is clean. - if err := itr.Close(); err != nil { - t.Fatal(err) - } - }) - - t.Run("NoGenerationDir", func(t *testing.T) { - c := file.NewReplicaClient(t.TempDir()) - itr, err := c.Snapshots(context.Background(), "5efbd8d042012dca") - if err != nil { - t.Fatal(err) - } - defer itr.Close() - - if itr.Next() { - t.Fatal("expected no snapshots") - } - }) - - t.Run("NoSnapshots", func(t *testing.T) { - dir := t.TempDir() - if err := os.MkdirAll(filepath.Join(dir, "generations", "5efbd8d042012dca", "snapshots"), 0777); err != nil { - t.Fatal(err) - } - - c := file.NewReplicaClient(dir) - itr, err := c.Snapshots(context.Background(), "5efbd8d042012dca") - if err != nil { - t.Fatal(err) - } - defer itr.Close() - - if itr.Next() { - t.Fatal("expected no snapshots") - } - }) - - t.Run("ErrNoPath", func(t *testing.T) { - if _, err := file.NewReplicaClient("").Snapshots(context.Background(), "b16ddcf5c697540f"); err == nil || err.Error() != `cannot determine snapshot directory: file replica path required` { - t.Fatalf("unexpected error: %v", err) - } - }) - t.Run("ErrNoGeneration", func(t *testing.T) { - if _, err := file.NewReplicaClient(t.TempDir()).Snapshots(context.Background(), ""); err == nil || err.Error() != `cannot determine snapshot directory: generation required` { - t.Fatalf("unexpected error: %v", err) - } - }) -} - -func TestReplicaClient_WriteSnapshot(t *testing.T) { - t.Run("OK", func(t *testing.T) { - data := []byte("foobar") - - c := file.NewReplicaClient(t.TempDir()) - if _, err := c.WriteSnapshot(context.Background(), "b16ddcf5c697540f", 1000, bytes.NewReader(compress(t, data))); err != nil { - t.Fatal(err) - } - - if r, err := c.SnapshotReader(context.Background(), "b16ddcf5c697540f", 1000); err != nil { - t.Fatal(err) - } else if buf, err := ioutil.ReadAll(lz4.NewReader(r)); err != nil { - t.Fatal(err) - } else if err := r.Close(); err != nil { - t.Fatal(err) - } else if got, want := buf, data; !bytes.Equal(got, want) { - t.Fatalf("data=%q, want %q", got, want) - } - }) - - t.Run("ErrNoPath", func(t *testing.T) { - if _, err := file.NewReplicaClient("").WriteSnapshot(context.Background(), "b16ddcf5c697540f", 0, nil); err == nil || err.Error() != `cannot determine snapshot path: file replica path required` { - t.Fatalf("unexpected error: %v", err) - } - }) - t.Run("ErrNoGeneration", func(t *testing.T) { - if _, err := file.NewReplicaClient(t.TempDir()).WriteSnapshot(context.Background(), "", 0, nil); err == nil || err.Error() != `cannot determine snapshot path: generation required` { - t.Fatalf("unexpected error: %v", err) - } - }) -} - -func TestReplicaClient_SnapshotReader(t *testing.T) { - t.Run("OK", func(t *testing.T) { - dir := t.TempDir() - if err := os.MkdirAll(filepath.Join(dir, "generations", "5efbd8d042012dca", "snapshots"), 0777); err != nil { - t.Fatal(err) - } else if err := ioutil.WriteFile(filepath.Join(dir, "generations", "5efbd8d042012dca", "snapshots", "0000000a.snapshot.lz4"), compress(t, []byte("foo")), 0666); err != nil { - t.Fatal(err) - } - - c := file.NewReplicaClient(dir) - r, err := c.SnapshotReader(context.Background(), "5efbd8d042012dca", 10) - if err != nil { - t.Fatal(err) - } - defer r.Close() - - if buf, err := ioutil.ReadAll(lz4.NewReader(r)); err != nil { - t.Fatal(err) - } else if got, want := buf, []byte("foo"); !bytes.Equal(got, want) { - t.Fatalf("ReadAll=%v, want %v", got, want) - } - }) - - t.Run("ErrNotFound", func(t *testing.T) { - dir := t.TempDir() - if err := os.MkdirAll(filepath.Join(dir, "generations", "5efbd8d042012dca", "snapshots"), 0777); err != nil { - t.Fatal(err) - } - - c := file.NewReplicaClient(dir) - if _, err := c.SnapshotReader(context.Background(), "5efbd8d042012dca", 1); !os.IsNotExist(err) { - t.Fatalf("expected not exist, got %#v", err) - } - }) - - t.Run("ErrNoPath", func(t *testing.T) { - c := file.NewReplicaClient("") - if _, err := c.SnapshotReader(context.Background(), "5efbd8d042012dca", 1); err == nil || err.Error() != `cannot determine snapshot path: file replica path required` { - t.Fatalf("unexpected error: %v", err) - } - }) - - t.Run("ErrGeneration", func(t *testing.T) { - dir := t.TempDir() - c := file.NewReplicaClient(dir) - if _, err := c.SnapshotReader(context.Background(), "", 1); err == nil || err.Error() != `cannot determine snapshot path: generation required` { - t.Fatalf("unexpected error: %v", err) - } - }) -} - -func TestReplicaClient_WALs(t *testing.T) { - t.Run("OK", func(t *testing.T) { - dir := t.TempDir() - if err := os.MkdirAll(filepath.Join(dir, "generations", "5efbd8d042012dca", "wal"), 0777); err != nil { - t.Fatal(err) - } else if err := ioutil.WriteFile(filepath.Join(dir, "generations", "5efbd8d042012dca", "wal", "00000001_00000000.wal.lz4"), nil, 0666); err != nil { - t.Fatal(err) - } - - if err := os.MkdirAll(filepath.Join(dir, "generations", "b16ddcf5c697540f", "wal"), 0777); err != nil { - t.Fatal(err) - } else if err := ioutil.WriteFile(filepath.Join(dir, "generations", "b16ddcf5c697540f", "wal", "00000002_00000000.wal.lz4"), []byte("12345"), 0666); err != nil { - t.Fatal(err) - } else if err := ioutil.WriteFile(filepath.Join(dir, "generations", "b16ddcf5c697540f", "wal", "00000002_00000005.wal.lz4"), []byte("67"), 0666); err != nil { - t.Fatal(err) - } else if err := ioutil.WriteFile(filepath.Join(dir, "generations", "b16ddcf5c697540f", "wal", "00000003_00000000.wal.lz4"), []byte("xyz"), 0666); err != nil { - t.Fatal(err) - } - - c := file.NewReplicaClient(dir) - itr, err := c.WALSegments(context.Background(), "b16ddcf5c697540f") - if err != nil { - t.Fatal(err) - } - defer itr.Close() - - // Read all WAL segment files into a slice so they can be sorted. - a, err := litestream.SliceWALSegmentIterator(itr) - if err != nil { - t.Fatal(err) - } else if got, want := len(a), 3; got != want { - t.Fatalf("len=%v, want %v", got, want) - } - sort.Sort(litestream.WALSegmentInfoSlice(a)) - - // Verify first WAL segment metadata. - if got, want := a[0].Generation, "b16ddcf5c697540f"; got != want { - t.Fatalf("Generation=%v, want %v", got, want) - } else if got, want := a[0].Index, 2; got != want { - t.Fatalf("Index=%v, want %v", got, want) - } else if got, want := a[0].Offset, int64(0); got != want { - t.Fatalf("Offset=%v, want %v", got, want) - } else if got, want := a[0].Size, int64(5); got != want { - t.Fatalf("Size=%v, want %v", got, want) - } else if a[0].CreatedAt.IsZero() { - t.Fatalf("expected CreatedAt") - } - - // Verify first WAL segment metadata. - if got, want := a[1].Generation, "b16ddcf5c697540f"; got != want { - t.Fatalf("Generation=%v, want %v", got, want) - } else if got, want := a[1].Index, 2; got != want { - t.Fatalf("Index=%v, want %v", got, want) - } else if got, want := a[1].Offset, int64(5); got != want { - t.Fatalf("Offset=%v, want %v", got, want) - } else if got, want := a[1].Size, int64(2); got != want { - t.Fatalf("Size=%v, want %v", got, want) - } else if a[1].CreatedAt.IsZero() { - t.Fatalf("expected CreatedAt") - } - - // Verify third WAL segment metadata. - if got, want := a[2].Generation, "b16ddcf5c697540f"; got != want { - t.Fatalf("Generation=%v, want %v", got, want) - } else if got, want := a[2].Index, 3; got != want { - t.Fatalf("Index=%v, want %v", got, want) - } else if got, want := a[2].Offset, int64(0); got != want { - t.Fatalf("Offset=%v, want %v", got, want) - } else if got, want := a[2].Size, int64(3); got != want { - t.Fatalf("Size=%v, want %v", got, want) - } else if a[1].CreatedAt.IsZero() { - t.Fatalf("expected CreatedAt") - } - - // Ensure close is clean. - if err := itr.Close(); err != nil { - t.Fatal(err) - } - }) - - t.Run("NoGenerationDir", func(t *testing.T) { - c := file.NewReplicaClient(t.TempDir()) - itr, err := c.WALSegments(context.Background(), "5efbd8d042012dca") - if err != nil { - t.Fatal(err) - } - defer itr.Close() - - if itr.Next() { - t.Fatal("expected no wal files") - } - }) - - t.Run("NoWALs", func(t *testing.T) { - dir := t.TempDir() - if err := os.MkdirAll(filepath.Join(dir, "generations", "5efbd8d042012dca", "wals"), 0777); err != nil { - t.Fatal(err) - } - - c := file.NewReplicaClient(dir) - itr, err := c.WALSegments(context.Background(), "5efbd8d042012dca") - if err != nil { - t.Fatal(err) - } - defer itr.Close() - - if itr.Next() { - t.Fatal("expected no wal files") - } - }) - - t.Run("ErrNoPath", func(t *testing.T) { - if _, err := file.NewReplicaClient("").WALSegments(context.Background(), "b16ddcf5c697540f"); err == nil || err.Error() != `cannot determine wal directory: file replica path required` { - t.Fatalf("unexpected error: %v", err) - } - }) - t.Run("ErrNoGeneration", func(t *testing.T) { - if _, err := file.NewReplicaClient(t.TempDir()).WALSegments(context.Background(), ""); err == nil || err.Error() != `cannot determine wal directory: generation required` { - t.Fatalf("unexpected error: %v", err) - } - }) -} - -func TestReplicaClient_WriteWALSegment(t *testing.T) { - t.Run("OK", func(t *testing.T) { - data := []byte("foobar") - - c := file.NewReplicaClient(t.TempDir()) - if _, err := c.WriteWALSegment(context.Background(), litestream.Pos{Generation: "b16ddcf5c697540f", Index: 1000, Offset: 2000}, bytes.NewReader(compress(t, data))); err != nil { - t.Fatal(err) - } - - if r, err := c.WALSegmentReader(context.Background(), litestream.Pos{Generation: "b16ddcf5c697540f", Index: 1000, Offset: 2000}); err != nil { - t.Fatal(err) - } else if buf, err := ioutil.ReadAll(lz4.NewReader(r)); err != nil { - t.Fatal(err) - } else if err := r.Close(); err != nil { - t.Fatal(err) - } else if got, want := buf, data; !bytes.Equal(got, want) { - t.Fatalf("data=%q, want %q", got, want) - } - }) - - t.Run("ErrNoPath", func(t *testing.T) { - if _, err := file.NewReplicaClient("").WriteWALSegment(context.Background(), litestream.Pos{Generation: "b16ddcf5c697540f", Index: 0, Offset: 0}, nil); err == nil || err.Error() != `cannot determine wal segment path: file replica path required` { - t.Fatalf("unexpected error: %v", err) - } - }) - t.Run("ErrNoGeneration", func(t *testing.T) { - if _, err := file.NewReplicaClient(t.TempDir()).WriteWALSegment(context.Background(), litestream.Pos{Generation: "", Index: 0, Offset: 0}, nil); err == nil || err.Error() != `cannot determine wal segment path: generation required` { - t.Fatalf("unexpected error: %v", err) - } - }) -} - -func TestReplicaClient_WALReader(t *testing.T) { - dir := t.TempDir() - if err := os.MkdirAll(filepath.Join(dir, "generations", "5efbd8d042012dca", "wal"), 0777); err != nil { - t.Fatal(err) - } else if err := ioutil.WriteFile(filepath.Join(dir, "generations", "5efbd8d042012dca", "wal", "0000000a_00000005.wal.lz4"), compress(t, []byte("foobar")), 0666); err != nil { - t.Fatal(err) - } - - c := file.NewReplicaClient(dir) - - t.Run("OK", func(t *testing.T) { - r, err := c.WALSegmentReader(context.Background(), litestream.Pos{Generation: "5efbd8d042012dca", Index: 10, Offset: 5}) - if err != nil { - t.Fatal(err) - } - defer r.Close() - - if buf, err := ioutil.ReadAll(lz4.NewReader(r)); err != nil { - t.Fatal(err) - } else if got, want := buf, []byte("foobar"); !bytes.Equal(got, want) { - t.Fatalf("ReadAll=%v, want %v", got, want) - } - }) - - t.Run("ErrNotFound", func(t *testing.T) { - dir := t.TempDir() - if err := os.MkdirAll(filepath.Join(dir, "generations", "5efbd8d042012dca", "wal"), 0777); err != nil { - t.Fatal(err) - } - - c := file.NewReplicaClient(dir) - if _, err := c.WALSegmentReader(context.Background(), litestream.Pos{Generation: "5efbd8d042012dca", Index: 1, Offset: 0}); !os.IsNotExist(err) { - t.Fatalf("expected not exist, got %#v", err) - } - }) -} - /* func TestReplica_Sync(t *testing.T) { // Ensure replica can successfully sync after DB has sync'd. @@ -664,17 +221,3 @@ func TestReplica_Sync(t *testing.T) { }) } */ - -// compress compresses b using LZ4. -func compress(tb testing.TB, b []byte) []byte { - tb.Helper() - - var buf bytes.Buffer - zw := lz4.NewWriter(&buf) - if _, err := zw.Write(b); err != nil { - tb.Fatal(err) - } else if err := zw.Close(); err != nil { - tb.Fatal(err) - } - return buf.Bytes() -} diff --git a/gcs/replica_client.go b/gcs/replica_client.go new file mode 100644 index 0000000..19102cc --- /dev/null +++ b/gcs/replica_client.go @@ -0,0 +1,442 @@ +package gcs + +import ( + "context" + "fmt" + "io" + "os" + "path" + "strings" + "sync" + "time" + + "cloud.google.com/go/storage" + "github.com/benbjohnson/litestream" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "google.golang.org/api/iterator" +) + +// ReplicaClientType is the client type for this package. +const ReplicaClientType = "gcs" + +var _ litestream.ReplicaClient = (*ReplicaClient)(nil) + +// ReplicaClient is a client for writing snapshots & WAL segments to disk. +type ReplicaClient struct { + mu sync.Mutex + client *storage.Client // gcs client + bkt *storage.BucketHandle // gcs bucket handle + + // GCS bucket information + Bucket string + Path string +} + +// NewReplicaClient returns a new instance of ReplicaClient. +func NewReplicaClient() *ReplicaClient { + return &ReplicaClient{} +} + +// Type returns "gcs" as the client type. +func (c *ReplicaClient) Type() string { + return ReplicaClientType +} + +// Init initializes the connection to GCS. No-op if already initialized. +func (c *ReplicaClient) Init(ctx context.Context) (err error) { + c.mu.Lock() + defer c.mu.Unlock() + + if c.client != nil { + return nil + } + + if c.client, err = storage.NewClient(ctx); err != nil { + return err + } + c.bkt = c.client.Bucket(c.Bucket) + + return nil +} + +// Generations returns a list of available generation names. +func (c *ReplicaClient) Generations(ctx context.Context) ([]string, error) { + if err := c.Init(ctx); err != nil { + return nil, err + } + + // Construct query to only pull generation directory names. + query := &storage.Query{ + Delimiter: "/", + Prefix: litestream.GenerationsPath(c.Path) + "/", + } + + // Loop over results and only build list of generation-formatted names. + it := c.bkt.Objects(ctx, query) + var generations []string + for { + attrs, err := it.Next() + if err == iterator.Done { + break + } else if err != nil { + return nil, err + } + + name := path.Base(strings.TrimSuffix(attrs.Prefix, "/")) + if !litestream.IsGenerationName(name) { + continue + } + generations = append(generations, name) + } + + return generations, nil +} + +// DeleteGeneration deletes all snapshots & WAL segments within a generation. +func (c *ReplicaClient) DeleteGeneration(ctx context.Context, generation string) error { + if err := c.Init(ctx); err != nil { + return err + } + + dir, err := litestream.GenerationPath(c.Path, generation) + if err != nil { + return fmt.Errorf("cannot determine generation path: %w", err) + } + + // Iterate over every object in generation and delete it. + operationTotalCounterVec.WithLabelValues("LIST").Inc() + for it := c.bkt.Objects(ctx, &storage.Query{Prefix: dir + "/"}); ; { + attrs, err := it.Next() + if err == iterator.Done { + break + } else if err != nil { + return err + } + + if err := c.bkt.Object(attrs.Name).Delete(ctx); isNotExists(err) { + continue + } else if err != nil { + return fmt.Errorf("cannot delete object %q: %w", attrs.Name, err) + } + operationTotalCounterVec.WithLabelValues("DELETE").Inc() + } + + // log.Printf("%s(%s): retainer: deleting generation: %s", r.db.Path(), r.Name(), generation) + + return nil +} + +// Snapshots returns an iterator over all available snapshots for a generation. +func (c *ReplicaClient) Snapshots(ctx context.Context, generation string) (litestream.SnapshotIterator, error) { + if err := c.Init(ctx); err != nil { + return nil, err + } + dir, err := litestream.SnapshotsPath(c.Path, generation) + if err != nil { + return nil, fmt.Errorf("cannot determine snapshots path: %w", err) + } + return newSnapshotIterator(generation, c.bkt.Objects(ctx, &storage.Query{Prefix: dir + "/"})), nil +} + +// WriteSnapshot writes LZ4 compressed data from rd to the object storage. +func (c *ReplicaClient) WriteSnapshot(ctx context.Context, generation string, index int, rd io.Reader) (info litestream.SnapshotInfo, err error) { + if err := c.Init(ctx); err != nil { + return info, err + } + + key, err := litestream.SnapshotPath(c.Path, generation, index) + if err != nil { + return info, fmt.Errorf("cannot determine snapshot path: %w", err) + } + startTime := time.Now() + + w := c.bkt.Object(key).NewWriter(ctx) + defer w.Close() + + n, err := io.Copy(w, rd) + if err != nil { + return info, err + } else if err := w.Close(); err != nil { + return info, err + } + + operationTotalCounterVec.WithLabelValues("PUT").Inc() + operationBytesCounterVec.WithLabelValues("PUT").Add(float64(n)) + + // log.Printf("%s(%s): snapshot: creating %s/%08x t=%s", r.db.Path(), r.Name(), generation, index, time.Since(startTime).Truncate(time.Millisecond)) + + return litestream.SnapshotInfo{ + Generation: generation, + Index: index, + Size: n, + CreatedAt: startTime.UTC(), + }, nil +} + +// SnapshotReader returns a reader for snapshot data at the given generation/index. +func (c *ReplicaClient) SnapshotReader(ctx context.Context, generation string, index int) (io.ReadCloser, error) { + if err := c.Init(ctx); err != nil { + return nil, err + } + + key, err := litestream.SnapshotPath(c.Path, generation, index) + if err != nil { + return nil, fmt.Errorf("cannot determine snapshot path: %w", err) + } + + r, err := c.bkt.Object(key).NewReader(ctx) + if isNotExists(err) { + return nil, os.ErrNotExist + } else if err != nil { + return nil, fmt.Errorf("cannot start new reader for %q: %w", key, err) + } + + operationTotalCounterVec.WithLabelValues("GET").Inc() + operationBytesCounterVec.WithLabelValues("GET").Add(float64(r.Attrs.Size)) + + return r, nil +} + +// DeleteSnapshot deletes a snapshot with the given generation & index. +func (c *ReplicaClient) DeleteSnapshot(ctx context.Context, generation string, index int) error { + if err := c.Init(ctx); err != nil { + return err + } + + key, err := litestream.SnapshotPath(c.Path, generation, index) + if err != nil { + return fmt.Errorf("cannot determine snapshot path: %w", err) + } + + if err := c.bkt.Object(key).Delete(ctx); err != nil && !isNotExists(err) { + return fmt.Errorf("cannot delete snapshot %q: %w", key, err) + } + + operationTotalCounterVec.WithLabelValues("DELETE").Inc() + return nil +} + +// WALSegments returns an iterator over all available WAL files for a generation. +func (c *ReplicaClient) WALSegments(ctx context.Context, generation string) (litestream.WALSegmentIterator, error) { + if err := c.Init(ctx); err != nil { + return nil, err + } + dir, err := litestream.WALPath(c.Path, generation) + if err != nil { + return nil, fmt.Errorf("cannot determine wal path: %w", err) + } + return newWALSegmentIterator(generation, c.bkt.Objects(ctx, &storage.Query{Prefix: dir + "/"})), nil +} + +// WriteWALSegment writes LZ4 compressed data from rd into a file on disk. +func (c *ReplicaClient) WriteWALSegment(ctx context.Context, pos litestream.Pos, rd io.Reader) (info litestream.WALSegmentInfo, err error) { + if err := c.Init(ctx); err != nil { + return info, err + } + + key, err := litestream.WALSegmentPath(c.Path, pos.Generation, pos.Index, pos.Offset) + if err != nil { + return info, fmt.Errorf("cannot determine wal segment path: %w", err) + } + startTime := time.Now() + + w := c.bkt.Object(key).NewWriter(ctx) + defer w.Close() + + n, err := io.Copy(w, rd) + if err != nil { + return info, err + } else if err := w.Close(); err != nil { + return info, err + } + + operationTotalCounterVec.WithLabelValues("PUT").Inc() + operationBytesCounterVec.WithLabelValues("PUT").Add(float64(n)) + + return litestream.WALSegmentInfo{ + Generation: pos.Generation, + Index: pos.Index, + Offset: pos.Offset, + Size: n, + CreatedAt: startTime.UTC(), + }, nil +} + +// WALSegmentReader returns a reader for a section of WAL data at the given index. +// Returns os.ErrNotExist if no matching index/offset is found. +func (c *ReplicaClient) WALSegmentReader(ctx context.Context, pos litestream.Pos) (io.ReadCloser, error) { + if err := c.Init(ctx); err != nil { + return nil, err + } + + key, err := litestream.WALSegmentPath(c.Path, pos.Generation, pos.Index, pos.Offset) + if err != nil { + return nil, fmt.Errorf("cannot determine wal segment path: %w", err) + } + + r, err := c.bkt.Object(key).NewReader(ctx) + if isNotExists(err) { + return nil, os.ErrNotExist + } else if err != nil { + return nil, err + } + + operationTotalCounterVec.WithLabelValues("GET").Inc() + operationBytesCounterVec.WithLabelValues("GET").Add(float64(r.Attrs.Size)) + + return r, nil +} + +// DeleteWALSegments deletes WAL segments with at the given positions. +func (c *ReplicaClient) DeleteWALSegments(ctx context.Context, a []litestream.Pos) error { + if err := c.Init(ctx); err != nil { + return err + } + + for _, pos := range a { + key, err := litestream.WALSegmentPath(c.Path, pos.Generation, pos.Index, pos.Offset) + if err != nil { + return fmt.Errorf("cannot determine wal segment path: %w", err) + } + + if err := c.bkt.Object(key).Delete(ctx); err != nil && !isNotExists(err) { + return fmt.Errorf("cannot delete wal segment %q: %w", key, err) + } + operationTotalCounterVec.WithLabelValues("DELETE").Inc() + } + + return nil +} + +type snapshotIterator struct { + generation string + + it *storage.ObjectIterator + info litestream.SnapshotInfo + err error +} + +func newSnapshotIterator(generation string, it *storage.ObjectIterator) *snapshotIterator { + return &snapshotIterator{ + generation: generation, + it: it, + } +} + +func (itr *snapshotIterator) Close() (err error) { + return itr.err +} + +func (itr *snapshotIterator) Next() bool { + // Exit if an error has already occurred. + if itr.err != nil { + return false + } + + for { + // Fetch next object. + attrs, err := itr.it.Next() + if err == iterator.Done { + return false + } else if err != nil { + itr.err = err + return false + } + + // Parse index, otherwise skip to the next object. + index, err := litestream.ParseSnapshotPath(path.Base(attrs.Name)) + if err != nil { + continue + } + + // Store current snapshot and return. + itr.info = litestream.SnapshotInfo{ + Generation: itr.generation, + Index: index, + Size: attrs.Size, + CreatedAt: attrs.Created.UTC(), + } + return true + } +} + +func (itr *snapshotIterator) Err() error { return itr.err } + +func (itr *snapshotIterator) Snapshot() litestream.SnapshotInfo { return itr.info } + +type walSegmentIterator struct { + generation string + + it *storage.ObjectIterator + info litestream.WALSegmentInfo + err error +} + +func newWALSegmentIterator(generation string, it *storage.ObjectIterator) *walSegmentIterator { + return &walSegmentIterator{ + generation: generation, + it: it, + } +} + +func (itr *walSegmentIterator) Close() (err error) { + return itr.err +} + +func (itr *walSegmentIterator) Next() bool { + // Exit if an error has already occurred. + if itr.err != nil { + return false + } + + for { + // Fetch next object. + attrs, err := itr.it.Next() + if err == iterator.Done { + return false + } else if err != nil { + itr.err = err + return false + } + + // Parse index & offset, otherwise skip to the next object. + index, offset, err := litestream.ParseWALSegmentPath(path.Base(attrs.Name)) + if err != nil { + continue + } + + // Store current snapshot and return. + itr.info = litestream.WALSegmentInfo{ + Generation: itr.generation, + Index: index, + Offset: offset, + Size: attrs.Size, + CreatedAt: attrs.Created.UTC(), + } + return true + } +} + +func (itr *walSegmentIterator) Err() error { return itr.err } + +func (itr *walSegmentIterator) WALSegment() litestream.WALSegmentInfo { + return itr.info +} + +func isNotExists(err error) bool { + return err == storage.ErrObjectNotExist +} + +// GCS metrics. +var ( + operationTotalCounterVec = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "litestream_gcs_operation_total", + Help: "The number of GCS operations performed", + }, []string{"type"}) + + operationBytesCounterVec = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "litestream_gcs_operation_bytes", + Help: "The number of bytes used by GCS operations", + }, []string{"type"}) +) diff --git a/go.mod b/go.mod index 5413de6..29c49da 100644 --- a/go.mod +++ b/go.mod @@ -3,12 +3,14 @@ module github.com/benbjohnson/litestream go 1.16 require ( + cloud.google.com/go/storage v1.15.0 github.com/aws/aws-sdk-go v1.27.0 github.com/davecgh/go-spew v1.1.1 github.com/mattn/go-sqlite3 v1.14.5 github.com/pierrec/lz4/v4 v4.1.3 github.com/prometheus/client_golang v1.9.0 - golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect - golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c + golang.org/x/sys v0.0.0-20210412220455-f1c623a9e750 + google.golang.org/api v0.45.0 gopkg.in/yaml.v2 v2.4.0 ) diff --git a/go.sum b/go.sum index 9136fc2..a46ee29 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,46 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0 h1:at8Tk2zUz63cLPR0JPWm5vp77pEZmzxEQBEfRKn1VV8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.15.0 h1:Ljj+ZXVEhCr/1+4ZhvtteN1ND7UUsNTlduGclLh8GO0= +cloud.google.com/go/storage v1.15.0/go.mod h1:mjjQMoxxyGH7Jr8K5qrx6N2O0AHsczI61sMNn03GIZI= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= @@ -31,8 +71,14 @@ github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QH github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -50,13 +96,21 @@ github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= @@ -72,18 +126,36 @@ github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zV github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -91,9 +163,34 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= @@ -124,6 +221,8 @@ github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2p github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= @@ -134,6 +233,9 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= @@ -260,15 +362,28 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= @@ -280,17 +395,45 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 h1:2M3HP5CCK1Si9FQhwnzYhXdG6DXeebvUHFpre8QvbyI= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1 h1:Kvvh58BN8Y9/lBi7hTekvtMpm07eUZ0ck5pRHpsMWrY= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -303,19 +446,59 @@ golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4 h1:b0LrWgu8+q7z4J+0Y3Umo5q1dL7NXBkKBWkaVkAq17E= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210413134643-5e61552d6c78 h1:rPRtHfUb0UKZeZ6GH4K4Nt4YRbE9V1u+QZX5upZXqJQ= +golang.org/x/oauth2 v0.0.0-20210413134643-5e61552d6c78/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -327,21 +510,61 @@ golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e h1:AyodaIpKjppX+cBfTASF2E1US3H2JFBj920Ot3rtDjs= golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210412220455-f1c623a9e750 h1:ZBu6861dZq7xBnG1bn5SRU0vA8nx42at4+kP07FMTog= +golang.org/x/sys v0.0.0-20210412220455-f1c623a9e750/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -349,41 +572,173 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.45.0 h1:pqMffJFLBVUDIoYsHcqtxgQVTsmxMDpYLOc5MT4Jrww= +google.golang.org/api v0.45.0/go.mod h1:ISLIJCedJolbZvDfAk+Ctuq5hf+aJ33WgtUsfyFoLXA= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210413151531-c14fb6ef47c3/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= +google.golang.org/genproto v0.0.0-20210420162539-3c870d7478d2 h1:g2sJMUGCpeHZqTx8p3wsAWRS64nFq20i4dvJWcKGqvY= +google.golang.org/genproto v0.0.0-20210420162539-3c870d7478d2/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.37.0 h1:uSZWeQJX5j11bIQ4AJoj+McDBo29cY1MCoC1wO3ts+c= +google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -404,9 +759,17 @@ gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= diff --git a/internal/internal.go b/internal/internal.go index 4b4febc..769de55 100644 --- a/internal/internal.go +++ b/internal/internal.go @@ -60,12 +60,12 @@ func CreateFile(filename string, fi os.FileInfo) (*os.File, error) { if fi != nil { mode = fi.Mode() } - + f, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode) if err != nil { return nil, err } - + uid, gid := Fileinfo(fi) _ = f.Chown(uid, gid) return f, nil @@ -74,7 +74,7 @@ func CreateFile(filename string, fi os.FileInfo) (*os.File, error) { // MkdirAll is a copy of os.MkdirAll() except that it attempts to set the // mode/uid/gid to match fi for each created directory. func MkdirAll(path string, fi os.FileInfo) error { - uid, gid := Fileinfo(fi) + uid, gid := Fileinfo(fi) // Fast path: if we can tell whether path is a directory or file, stop with success or error. dir, err := os.Stat(path) @@ -123,4 +123,3 @@ func MkdirAll(path string, fi os.FileInfo) error { _ = os.Chown(path, uid, gid) return nil } - diff --git a/replica.go b/replica.go index 1d08e96..952c2c2 100644 --- a/replica.go +++ b/replica.go @@ -709,48 +709,6 @@ func (r *Replica) validator(ctx context.Context) { } } -// ReplicaClient represents client to connect to a Replica. -type ReplicaClient interface { - // Returns the type of client. - Type() string - - // Returns a list of available generations. - Generations(ctx context.Context) ([]string, error) - - // Deletes all snapshots & WAL segments within a generation. - DeleteGeneration(ctx context.Context, generation string) error - - // Returns an iterator of all snapshots within a generation on the replica. - Snapshots(ctx context.Context, generation string) (SnapshotIterator, error) - - // Writes LZ4 compressed snapshot data to the replica at a given index - // within a generation. Returns metadata for the snapshot. - WriteSnapshot(ctx context.Context, generation string, index int, r io.Reader) (SnapshotInfo, error) - - // Deletes a snapshot with the given generation & index. - DeleteSnapshot(ctx context.Context, generation string, index int) error - - // Returns a reader that contains LZ4 compressed snapshot data for a - // given index within a generation. Returns an os.ErrNotFound error if - // the snapshot does not exist. - SnapshotReader(ctx context.Context, generation string, index int) (io.ReadCloser, error) - - // Returns an iterator of all WAL segments within a generation on the replica. - WALSegments(ctx context.Context, generation string) (WALSegmentIterator, error) - - // Writes an LZ4 compressed WAL segment at a given position. - // Returns metadata for the written segment. - WriteWALSegment(ctx context.Context, pos Pos, r io.Reader) (WALSegmentInfo, error) - - // Deletes one or more WAL segments at the given positions. - DeleteWALSegments(ctx context.Context, a []Pos) error - - // Returns a reader that contains an LZ4 compressed WAL segment at a given - // index/offset within a generation. Returns an os.ErrNotFound error if the - // WAL segment does not exist. - WALSegmentReader(ctx context.Context, pos Pos) (io.ReadCloser, error) -} - // Validate restores the most recent data from a replica and validates // that the resulting database matches the current database. func (r *Replica) Validate(ctx context.Context) error { diff --git a/replica_client.go b/replica_client.go new file mode 100644 index 0000000..3a914e4 --- /dev/null +++ b/replica_client.go @@ -0,0 +1,48 @@ +package litestream + +import ( + "context" + "io" +) + +// ReplicaClient represents client to connect to a Replica. +type ReplicaClient interface { + // Returns the type of client. + Type() string + + // Returns a list of available generations. + Generations(ctx context.Context) ([]string, error) + + // Deletes all snapshots & WAL segments within a generation. + DeleteGeneration(ctx context.Context, generation string) error + + // Returns an iterator of all snapshots within a generation on the replica. + Snapshots(ctx context.Context, generation string) (SnapshotIterator, error) + + // Writes LZ4 compressed snapshot data to the replica at a given index + // within a generation. Returns metadata for the snapshot. + WriteSnapshot(ctx context.Context, generation string, index int, r io.Reader) (SnapshotInfo, error) + + // Deletes a snapshot with the given generation & index. + DeleteSnapshot(ctx context.Context, generation string, index int) error + + // Returns a reader that contains LZ4 compressed snapshot data for a + // given index within a generation. Returns an os.ErrNotFound error if + // the snapshot does not exist. + SnapshotReader(ctx context.Context, generation string, index int) (io.ReadCloser, error) + + // Returns an iterator of all WAL segments within a generation on the replica. + WALSegments(ctx context.Context, generation string) (WALSegmentIterator, error) + + // Writes an LZ4 compressed WAL segment at a given position. + // Returns metadata for the written segment. + WriteWALSegment(ctx context.Context, pos Pos, r io.Reader) (WALSegmentInfo, error) + + // Deletes one or more WAL segments at the given positions. + DeleteWALSegments(ctx context.Context, a []Pos) error + + // Returns a reader that contains an LZ4 compressed WAL segment at a given + // index/offset within a generation. Returns an os.ErrNotFound error if the + // WAL segment does not exist. + WALSegmentReader(ctx context.Context, pos Pos) (io.ReadCloser, error) +} diff --git a/s3/replica_client_test.go b/replica_client_test.go similarity index 58% rename from s3/replica_client_test.go rename to replica_client_test.go index 913d985..98213cf 100644 --- a/s3/replica_client_test.go +++ b/replica_client_test.go @@ -1,4 +1,4 @@ -package s3_test +package litestream_test import ( "context" @@ -12,38 +12,45 @@ import ( "sort" "strings" "testing" + "time" "github.com/benbjohnson/litestream" + "github.com/benbjohnson/litestream/file" + "github.com/benbjohnson/litestream/gcs" "github.com/benbjohnson/litestream/s3" ) -var ( - // Enables integration tests. - integration = flag.Bool("integration", false, "") - - // Replica client settings - accessKeyID = flag.String("access-key-id", os.Getenv("LITESTREAM_S3_ACCESS_KEY_ID"), "") - secretAccessKey = flag.String("secret-access-key", os.Getenv("LITESTREAM_S3_SECRET_ACCESS_KEY"), "") - region = flag.String("region", os.Getenv("LITESTREAM_S3_REGION"), "") - bucket = flag.String("bucket", os.Getenv("LITESTREAM_S3_BUCKET"), "") - pathFlag = flag.String("path", os.Getenv("LITESTREAM_S3_PATH"), "") - endpoint = flag.String("endpoint", os.Getenv("LITESTREAM_S3_ENDPOINT"), "") - forcePathStyle = flag.Bool("force-path-style", os.Getenv("LITESTREAM_S3_FORCE_PATH_STYLE") == "true", "") - skipVerify = flag.Bool("skip-verify", os.Getenv("LITESTREAM_S3_SKIP_VERIFY") == "true", "") -) - -func TestReplicaClient_Type(t *testing.T) { - if got, want := s3.NewReplicaClient().Type(), "s3"; got != want { - t.Fatalf("Type()=%v, want %v", got, want) - } +func init() { + rand.Seed(time.Now().UnixNano()) } -func TestReplicaClient_Generations(t *testing.T) { - t.Run("OK", func(t *testing.T) { - t.Parallel() +var ( + // Enables integration tests. + integration = flag.String("integration", "file", "") +) - c := NewIntegrationReplicaClient(t) - defer MustDeleteAll(t, c) +// S3 settings +var ( + // Replica client settings + s3AccessKeyID = flag.String("s3-access-key-id", os.Getenv("LITESTREAM_S3_ACCESS_KEY_ID"), "") + s3SecretAccessKey = flag.String("s3-secret-access-key", os.Getenv("LITESTREAM_S3_SECRET_ACCESS_KEY"), "") + s3Region = flag.String("s3-region", os.Getenv("LITESTREAM_S3_REGION"), "") + s3Bucket = flag.String("s3-bucket", os.Getenv("LITESTREAM_S3_BUCKET"), "") + s3Path = flag.String("s3-path", os.Getenv("LITESTREAM_S3_PATH"), "") + s3Endpoint = flag.String("s3-endpoint", os.Getenv("LITESTREAM_S3_ENDPOINT"), "") + s3ForcePathStyle = flag.Bool("s3-force-path-style", os.Getenv("LITESTREAM_S3_FORCE_PATH_STYLE") == "true", "") + s3SkipVerify = flag.Bool("s3-skip-verify", os.Getenv("LITESTREAM_S3_SKIP_VERIFY") == "true", "") +) + +// GCS settings +var ( + gcsBucket = flag.String("gcs-bucket", os.Getenv("LITESTREAM_GCS_BUCKET"), "") + gcsPath = flag.String("gcs-path", os.Getenv("LITESTREAM_GCS_PATH"), "") +) + +func TestReplicaClient_Generations(t *testing.T) { + RunWithReplicaClient(t, "OK", func(t *testing.T, c litestream.ReplicaClient) { + t.Parallel() // Write snapshots. if _, err := c.WriteSnapshot(context.Background(), "5efbd8d042012dca", 0, strings.NewReader(`foo`)); err != nil { @@ -62,11 +69,9 @@ func TestReplicaClient_Generations(t *testing.T) { } }) - t.Run("NoGenerationsDir", func(t *testing.T) { + RunWithReplicaClient(t, "NoGenerationsDir", func(t *testing.T, c litestream.ReplicaClient) { t.Parallel() - c := NewIntegrationReplicaClient(t) - defer MustDeleteAll(t, c) if generations, err := c.Generations(context.Background()); err != nil { t.Fatal(err) } else if got, want := len(generations), 0; got != want { @@ -76,12 +81,9 @@ func TestReplicaClient_Generations(t *testing.T) { } func TestReplicaClient_Snapshots(t *testing.T) { - t.Run("OK", func(t *testing.T) { + RunWithReplicaClient(t, "OK", func(t *testing.T, c litestream.ReplicaClient) { t.Parallel() - c := NewIntegrationReplicaClient(t) - defer MustDeleteAll(t, c) - // Write snapshots. if _, err := c.WriteSnapshot(context.Background(), "5efbd8d042012dca", 1, strings.NewReader(``)); err != nil { t.Fatal(err) @@ -135,12 +137,9 @@ func TestReplicaClient_Snapshots(t *testing.T) { } }) - t.Run("NoGenerationDir", func(t *testing.T) { + RunWithReplicaClient(t, "NoGenerationDir", func(t *testing.T, c litestream.ReplicaClient) { t.Parallel() - c := NewIntegrationReplicaClient(t) - defer MustDeleteAll(t, c) - itr, err := c.Snapshots(context.Background(), "5efbd8d042012dca") if err != nil { t.Fatal(err) @@ -152,27 +151,23 @@ func TestReplicaClient_Snapshots(t *testing.T) { } }) - t.Run("ErrNoGeneration", func(t *testing.T) { + RunWithReplicaClient(t, "ErrNoGeneration", func(t *testing.T, c litestream.ReplicaClient) { t.Parallel() - c := NewIntegrationReplicaClient(t) - defer MustDeleteAll(t, c) - - if itr, err := c.Snapshots(context.Background(), ""); err != nil { - t.Fatal(err) - } else if err := itr.Close(); err == nil || err.Error() != `cannot determine snapshot directory path: generation required` { + itr, err := c.Snapshots(context.Background(), "") + if err == nil { + err = itr.Close() + } + if err == nil || err.Error() != `cannot determine snapshots path: generation required` { t.Fatalf("unexpected error: %v", err) } }) } func TestReplicaClient_WriteSnapshot(t *testing.T) { - t.Run("OK", func(t *testing.T) { + RunWithReplicaClient(t, "OK", func(t *testing.T, c litestream.ReplicaClient) { t.Parallel() - c := NewIntegrationReplicaClient(t) - defer MustDeleteAll(t, c) - if _, err := c.WriteSnapshot(context.Background(), "b16ddcf5c697540f", 1000, strings.NewReader(`foobar`)); err != nil { t.Fatal(err) } @@ -188,21 +183,18 @@ func TestReplicaClient_WriteSnapshot(t *testing.T) { } }) - t.Run("ErrNoGeneration", func(t *testing.T) { + RunWithReplicaClient(t, "ErrNoGeneration", func(t *testing.T, c litestream.ReplicaClient) { t.Parallel() - if _, err := NewIntegrationReplicaClient(t).WriteSnapshot(context.Background(), "", 0, nil); err == nil || err.Error() != `cannot determine snapshot path: generation required` { + if _, err := c.WriteSnapshot(context.Background(), "", 0, nil); err == nil || err.Error() != `cannot determine snapshot path: generation required` { t.Fatalf("unexpected error: %v", err) } }) } func TestReplicaClient_SnapshotReader(t *testing.T) { - t.Run("OK", func(t *testing.T) { + RunWithReplicaClient(t, "OK", func(t *testing.T, c litestream.ReplicaClient) { t.Parallel() - c := NewIntegrationReplicaClient(t) - defer MustDeleteAll(t, c) - if _, err := c.WriteSnapshot(context.Background(), "5efbd8d042012dca", 10, strings.NewReader(`foo`)); err != nil { t.Fatal(err) } @@ -220,23 +212,17 @@ func TestReplicaClient_SnapshotReader(t *testing.T) { } }) - t.Run("ErrNotFound", func(t *testing.T) { + RunWithReplicaClient(t, "ErrNotFound", func(t *testing.T, c litestream.ReplicaClient) { t.Parallel() - c := NewIntegrationReplicaClient(t) - defer MustDeleteAll(t, c) - if _, err := c.SnapshotReader(context.Background(), "5efbd8d042012dca", 1); !os.IsNotExist(err) { t.Fatalf("expected not exist, got %#v", err) } }) - t.Run("ErrGeneration", func(t *testing.T) { + RunWithReplicaClient(t, "ErrNoGeneration", func(t *testing.T, c litestream.ReplicaClient) { t.Parallel() - c := NewIntegrationReplicaClient(t) - defer MustDeleteAll(t, c) - if _, err := c.SnapshotReader(context.Background(), "", 1); err == nil || err.Error() != `cannot determine snapshot path: generation required` { t.Fatalf("unexpected error: %v", err) } @@ -244,12 +230,9 @@ func TestReplicaClient_SnapshotReader(t *testing.T) { } func TestReplicaClient_WALs(t *testing.T) { - t.Run("OK", func(t *testing.T) { + RunWithReplicaClient(t, "OK", func(t *testing.T, c litestream.ReplicaClient) { t.Parallel() - c := NewIntegrationReplicaClient(t) - defer MustDeleteAll(t, c) - if _, err := c.WriteWALSegment(context.Background(), litestream.Pos{Generation: "5efbd8d042012dca", Index: 1, Offset: 0}, strings.NewReader(``)); err != nil { t.Fatal(err) } @@ -321,12 +304,9 @@ func TestReplicaClient_WALs(t *testing.T) { } }) - t.Run("NoGenerationDir", func(t *testing.T) { + RunWithReplicaClient(t, "NoGenerationDir", func(t *testing.T, c litestream.ReplicaClient) { t.Parallel() - c := NewIntegrationReplicaClient(t) - defer MustDeleteAll(t, c) - itr, err := c.WALSegments(context.Background(), "5efbd8d042012dca") if err != nil { t.Fatal(err) @@ -338,12 +318,9 @@ func TestReplicaClient_WALs(t *testing.T) { } }) - t.Run("NoWALs", func(t *testing.T) { + RunWithReplicaClient(t, "NoWALs", func(t *testing.T, c litestream.ReplicaClient) { t.Parallel() - c := NewIntegrationReplicaClient(t) - defer MustDeleteAll(t, c) - if _, err := c.WriteSnapshot(context.Background(), "5efbd8d042012dca", 0, strings.NewReader(`foo`)); err != nil { t.Fatal(err) } @@ -359,27 +336,23 @@ func TestReplicaClient_WALs(t *testing.T) { } }) - t.Run("ErrNoGeneration", func(t *testing.T) { + RunWithReplicaClient(t, "ErrNoGeneration", func(t *testing.T, c litestream.ReplicaClient) { t.Parallel() - c := NewIntegrationReplicaClient(t) - defer MustDeleteAll(t, c) - - if itr, err := c.WALSegments(context.Background(), ""); err != nil { - t.Fatal(err) - } else if err := itr.Close(); err == nil || err.Error() != `cannot determine wal directory path: generation required` { + itr, err := c.WALSegments(context.Background(), "") + if err == nil { + err = itr.Close() + } + if err == nil || err.Error() != `cannot determine wal path: generation required` { t.Fatalf("unexpected error: %v", err) } }) } func TestReplicaClient_WriteWALSegment(t *testing.T) { - t.Run("OK", func(t *testing.T) { + RunWithReplicaClient(t, "OK", func(t *testing.T, c litestream.ReplicaClient) { t.Parallel() - c := NewIntegrationReplicaClient(t) - defer MustDeleteAll(t, c) - if _, err := c.WriteWALSegment(context.Background(), litestream.Pos{Generation: "b16ddcf5c697540f", Index: 1000, Offset: 2000}, strings.NewReader(`foobar`)); err != nil { t.Fatal(err) } @@ -395,25 +368,22 @@ func TestReplicaClient_WriteWALSegment(t *testing.T) { } }) - t.Run("ErrNoGeneration", func(t *testing.T) { + RunWithReplicaClient(t, "ErrNoGeneration", func(t *testing.T, c litestream.ReplicaClient) { t.Parallel() - if _, err := NewIntegrationReplicaClient(t).WriteWALSegment(context.Background(), litestream.Pos{Generation: "", Index: 0, Offset: 0}, nil); err == nil || err.Error() != `cannot determine wal segment path: generation required` { + if _, err := c.WriteWALSegment(context.Background(), litestream.Pos{Generation: "", Index: 0, Offset: 0}, nil); err == nil || err.Error() != `cannot determine wal segment path: generation required` { t.Fatalf("unexpected error: %v", err) } }) } func TestReplicaClient_WALReader(t *testing.T) { - t.Parallel() - c := NewIntegrationReplicaClient(t) - defer MustDeleteAll(t, c) + RunWithReplicaClient(t, "OK", func(t *testing.T, c litestream.ReplicaClient) { + t.Parallel() + if _, err := c.WriteWALSegment(context.Background(), litestream.Pos{Generation: "5efbd8d042012dca", Index: 10, Offset: 5}, strings.NewReader(`foobar`)); err != nil { + t.Fatal(err) + } - if _, err := c.WriteWALSegment(context.Background(), litestream.Pos{Generation: "5efbd8d042012dca", Index: 10, Offset: 5}, strings.NewReader(`foobar`)); err != nil { - t.Fatal(err) - } - - t.Run("OK", func(t *testing.T) { r, err := c.WALSegmentReader(context.Background(), litestream.Pos{Generation: "5efbd8d042012dca", Index: 10, Offset: 5}) if err != nil { t.Fatal(err) @@ -427,12 +397,9 @@ func TestReplicaClient_WALReader(t *testing.T) { } }) - t.Run("ErrNotFound", func(t *testing.T) { + RunWithReplicaClient(t, "ErrNotFound", func(t *testing.T, c litestream.ReplicaClient) { t.Parallel() - c := NewIntegrationReplicaClient(t) - defer MustDeleteAll(t, c) - if _, err := c.WALSegmentReader(context.Background(), litestream.Pos{Generation: "5efbd8d042012dca", Index: 1, Offset: 0}); !os.IsNotExist(err) { t.Fatalf("expected not exist, got %#v", err) } @@ -440,12 +407,9 @@ func TestReplicaClient_WALReader(t *testing.T) { } func TestReplicaClient_DeleteWALSegments(t *testing.T) { - t.Run("OK", func(t *testing.T) { + RunWithReplicaClient(t, "OK", func(t *testing.T, c litestream.ReplicaClient) { t.Parallel() - c := NewIntegrationReplicaClient(t) - defer MustDeleteAll(t, c) - if _, err := c.WriteWALSegment(context.Background(), litestream.Pos{Generation: "b16ddcf5c697540f", Index: 1, Offset: 2}, strings.NewReader(`foo`)); err != nil { t.Fatal(err) } else if _, err := c.WriteWALSegment(context.Background(), litestream.Pos{Generation: "5efbd8d042012dca", Index: 3, Offset: 4}, strings.NewReader(`bar`)); err != nil { @@ -466,113 +430,89 @@ func TestReplicaClient_DeleteWALSegments(t *testing.T) { } }) - t.Run("ErrNoGeneration", func(t *testing.T) { + RunWithReplicaClient(t, "ErrNoGeneration", func(t *testing.T, c litestream.ReplicaClient) { t.Parallel() - if err := NewIntegrationReplicaClient(t).DeleteWALSegments(context.Background(), []litestream.Pos{{}}); err == nil || err.Error() != `cannot determine wal segment path: generation required` { + if err := c.DeleteWALSegments(context.Background(), []litestream.Pos{{}}); err == nil || err.Error() != `cannot determine wal segment path: generation required` { t.Fatalf("unexpected error: %v", err) } }) } -func TestParseHost(t *testing.T) { - // Ensure non-specific hosts return as buckets. - t.Run("S3", func(t *testing.T) { - bucket, region, endpoint, forcePathStyle := s3.ParseHost(`test.litestream.io`) - if got, want := bucket, `test.litestream.io`; got != want { - t.Fatalf("bucket=%q, want %q", got, want) - } else if got, want := region, ``; got != want { - t.Fatalf("region=%q, want %q", got, want) - } else if got, want := endpoint, ``; got != want { - t.Fatalf("endpoint=%q, want %q", got, want) - } else if got, want := forcePathStyle, false; got != want { - t.Fatalf("forcePathStyle=%v, want %v", got, want) - } - }) +// RunWithReplicaClient executes fn with each replica specified by the -integration flag +func RunWithReplicaClient(t *testing.T, name string, fn func(*testing.T, litestream.ReplicaClient)) { + t.Run(name, func(t *testing.T) { + for _, typ := range strings.Split(*integration, ",") { + t.Run(typ, func(t *testing.T) { + c := NewReplicaClient(t, typ) + defer MustDeleteAll(t, c) - // Ensure localhosts use an HTTP endpoint and extract the bucket name. - t.Run("Localhost", func(t *testing.T) { - t.Run("WithPort", func(t *testing.T) { - bucket, region, endpoint, forcePathStyle := s3.ParseHost(`test.localhost:9000`) - if got, want := bucket, `test`; got != want { - t.Fatalf("bucket=%q, want %q", got, want) - } else if got, want := region, `us-east-1`; got != want { - t.Fatalf("region=%q, want %q", got, want) - } else if got, want := endpoint, `http://localhost:9000`; got != want { - t.Fatalf("endpoint=%q, want %q", got, want) - } else if got, want := forcePathStyle, true; got != want { - t.Fatalf("forcePathStyle=%v, want %v", got, want) - } - }) - - t.Run("WithoutPort", func(t *testing.T) { - bucket, region, endpoint, forcePathStyle := s3.ParseHost(`test.localhost`) - if got, want := bucket, `test`; got != want { - t.Fatalf("bucket=%q, want %q", got, want) - } else if got, want := region, `us-east-1`; got != want { - t.Fatalf("region=%q, want %q", got, want) - } else if got, want := endpoint, `http://localhost`; got != want { - t.Fatalf("endpoint=%q, want %q", got, want) - } else if got, want := forcePathStyle, true; got != want { - t.Fatalf("forcePathStyle=%v, want %v", got, want) - } - }) - }) - - // Ensure backblaze B2 URLs extract bucket, region, & endpoint from host. - t.Run("Backblaze", func(t *testing.T) { - bucket, region, endpoint, forcePathStyle := s3.ParseHost(`test-123.s3.us-west-000.backblazeb2.com`) - if got, want := bucket, `test-123`; got != want { - t.Fatalf("bucket=%q, want %q", got, want) - } else if got, want := region, `us-west-000`; got != want { - t.Fatalf("region=%q, want %q", got, want) - } else if got, want := endpoint, `https://s3.us-west-000.backblazeb2.com`; got != want { - t.Fatalf("endpoint=%q, want %q", got, want) - } else if got, want := forcePathStyle, true; got != want { - t.Fatalf("forcePathStyle=%v, want %v", got, want) - } - }) - - // Ensure GCS URLs extract bucket & endpoint from host. - t.Run("GCS", func(t *testing.T) { - bucket, region, endpoint, forcePathStyle := s3.ParseHost(`litestream.io.storage.googleapis.com`) - if got, want := bucket, `litestream.io`; got != want { - t.Fatalf("bucket=%q, want %q", got, want) - } else if got, want := region, `us-east-1`; got != want { - t.Fatalf("region=%q, want %q", got, want) - } else if got, want := endpoint, `https://storage.googleapis.com`; got != want { - t.Fatalf("endpoint=%q, want %q", got, want) - } else if got, want := forcePathStyle, true; got != want { - t.Fatalf("forcePathStyle=%v, want %v", got, want) + fn(t, c) + }) } }) } -// NewIntegrationReplicaClient returns a new client for integration testing. -// If integration flag is not set then test/benchmark is skipped. -func NewIntegrationReplicaClient(tb testing.TB) *s3.ReplicaClient { +// NewReplicaClient returns a new client for integration testing by type name. +func NewReplicaClient(tb testing.TB, typ string) litestream.ReplicaClient { tb.Helper() - if !*integration { - tb.Skip("integration tests disabled") + switch typ { + case file.ReplicaClientType: + return NewFileReplicaClient(tb) + case s3.ReplicaClientType: + return NewS3ReplicaClient(tb) + case gcs.ReplicaClientType: + return NewGCSReplicaClient(tb) + default: + tb.Fatalf("invalid replica client type: %q", typ) + return nil } +} + +// NewFileReplicaClient returns a new client for integration testing. +func NewFileReplicaClient(tb testing.TB) *file.ReplicaClient { + tb.Helper() + return file.NewReplicaClient(tb.TempDir()) +} + +// NewS3ReplicaClient returns a new client for integration testing. +func NewS3ReplicaClient(tb testing.TB) *s3.ReplicaClient { + tb.Helper() c := s3.NewReplicaClient() - c.AccessKeyID = *accessKeyID - c.SecretAccessKey = *secretAccessKey - c.Region = *region - c.Bucket = *bucket - c.Path = path.Join(*pathFlag, fmt.Sprintf("%016x", rand.Uint64())) - c.Endpoint = *endpoint - c.ForcePathStyle = *forcePathStyle - c.SkipVerify = *skipVerify + c.AccessKeyID = *s3AccessKeyID + c.SecretAccessKey = *s3SecretAccessKey + c.Region = *s3Region + c.Bucket = *s3Bucket + c.Path = path.Join(*s3Path, fmt.Sprintf("%016x", rand.Uint64())) + c.Endpoint = *s3Endpoint + c.ForcePathStyle = *s3ForcePathStyle + c.SkipVerify = *s3SkipVerify + return c +} +// NewGCSReplicaClient returns a new client for integration testing. +func NewGCSReplicaClient(tb testing.TB) *gcs.ReplicaClient { + tb.Helper() + + c := gcs.NewReplicaClient() + c.Bucket = *gcsBucket + c.Path = path.Join(*gcsPath, fmt.Sprintf("%016x", rand.Uint64())) return c } // MustDeleteAll deletes all objects under the client's path. -func MustDeleteAll(tb testing.TB, c *s3.ReplicaClient) { +func MustDeleteAll(tb testing.TB, c litestream.ReplicaClient) { tb.Helper() - if err := c.DeleteAll(context.Background()); err != nil { + + generations, err := c.Generations(context.Background()) + if err != nil { tb.Fatal(err) } + + for _, generation := range generations { + if err := c.DeleteGeneration(context.Background(), generation); err != nil { + tb.Fatalf("cannot delete generation: %s", err) + } + } } diff --git a/s3/replica_client.go b/s3/replica_client.go index 1bc16f5..c7befcc 100644 --- a/s3/replica_client.go +++ b/s3/replica_client.go @@ -27,6 +27,9 @@ import ( "golang.org/x/sync/errgroup" ) +// ReplicaClientType is the client type for this package. +const ReplicaClientType = "s3" + // MaxKeys is the number of keys S3 can operate on per batch. const MaxKeys = 1000 @@ -61,7 +64,7 @@ func NewReplicaClient() *ReplicaClient { // Type returns "s3" as the client type. func (c *ReplicaClient) Type() string { - return "s3" + return ReplicaClientType } // Init initializes the connection to S3. No-op if already initialized. @@ -181,7 +184,7 @@ func (c *ReplicaClient) DeleteGeneration(ctx context.Context, generation string) dir, err := litestream.GenerationPath(c.Path, generation) if err != nil { - return fmt.Errorf("cannot determine generation directory path: %w", err) + return fmt.Errorf("cannot determine generation path: %w", err) } // Collect all files for the generation. @@ -499,7 +502,7 @@ func (itr *snapshotIterator) fetch() error { dir, err := litestream.SnapshotsPath(itr.client.Path, itr.generation) if err != nil { - return fmt.Errorf("cannot determine snapshot directory path: %w", err) + return fmt.Errorf("cannot determine snapshots path: %w", err) } return itr.client.s3.ListObjectsPagesWithContext(itr.ctx, &s3.ListObjectsInput{ @@ -602,7 +605,7 @@ func (itr *walSegmentIterator) fetch() error { dir, err := litestream.WALPath(itr.client.Path, itr.generation) if err != nil { - return fmt.Errorf("cannot determine wal directory path: %w", err) + return fmt.Errorf("cannot determine wal path: %w", err) } return itr.client.s3.ListObjectsPagesWithContext(itr.ctx, &s3.ListObjectsInput{ @@ -691,9 +694,6 @@ func ParseHost(s string) (bucket, region, endpoint string, forcePathStyle bool) if a := localhostRegex.FindStringSubmatch(host); a != nil { bucket, region = a[1], "us-east-1" scheme, endpoint = "http", "localhost" - } else if a := gcsRegex.FindStringSubmatch(host); a != nil { - bucket, region = a[1], "us-east-1" - endpoint = "storage.googleapis.com" } else if a := digitalOceanRegex.FindStringSubmatch(host); a != nil { bucket, region = a[1], a[2] endpoint = fmt.Sprintf("%s.digitaloceanspaces.com", region) @@ -726,7 +726,6 @@ var ( digitalOceanRegex = regexp.MustCompile(`^(?:(.+)\.)?([^.]+)\.digitaloceanspaces.com$`) linodeRegex = regexp.MustCompile(`^(?:(.+)\.)?([^.]+)\.linodeobjects.com$`) backblazeRegex = regexp.MustCompile(`^(?:(.+)\.)?s3.([^.]+)\.backblazeb2.com$`) - gcsRegex = regexp.MustCompile(`^(?:(.+)\.)?storage.googleapis.com$`) ) func isNotExists(err error) bool {