rkrtechnology/litestream
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Move path functions to litestream package

Browse Source
This commit is contained in:
Ben Johnson
2021-05-21 10:37:34 -06:00
parent fb80bc10ae
commit 6c865e37f1
7 changed files with 285 additions and 310 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
file/replica_client.go
View File

@@ -195,7 +195,6 @@ func (c *ReplicaClient) WriteSnapshot(ctx context.Context, generation string, in
fileInfo, dirInfo = db.FileInfo(), db.DirInfo()
}
// Ensure parent directory exists.
if err := internal.MkdirAll(filepath.Dir(filename), dirInfo); err != nil {
return info, err
@@ -377,4 +376,3 @@ func (c *ReplicaClient) DeleteWALSegments(ctx context.Context, a []litestream.Po
}
return nil
}

51
litestream.go
View File

@@ -7,6 +7,7 @@ import (
"fmt"
"io"
"os"
"path"
"path/filepath"
"regexp"
"strconv"
@@ -383,6 +384,56 @@ func IsGenerationName(s string) bool {
return true
}
// GenerationsPath returns the path to a generation root directory.
func GenerationsPath(root string) string {
return path.Join(root, "generations")
}
// GenerationPath returns the path to a generation's root directory.
func GenerationPath(root, generation string) (string, error) {
dir := GenerationsPath(root)
if generation == "" {
return "", fmt.Errorf("generation required")
}
return path.Join(dir, generation), nil
}
// SnapshotsPath returns the path to a generation's snapshot directory.
func SnapshotsPath(root, generation string) (string, error) {
dir, err := GenerationPath(root, generation)
if err != nil {
return "", err
}
return path.Join(dir, "snapshots"), nil
}
// SnapshotPath returns the path to an uncompressed snapshot file.
func SnapshotPath(root, generation string, index int) (string, error) {
dir, err := SnapshotsPath(root, generation)
if err != nil {
return "", err
}
return path.Join(dir, FormatSnapshotPath(index)), nil
}
// WALPath returns the path to a generation's WAL directory
func WALPath(root, generation string) (string, error) {
dir, err := GenerationPath(root, generation)
if err != nil {
return "", err
}
return path.Join(dir, "wal"), nil
}
// WALSegmentPath returns the path to a WAL segment file.
func WALSegmentPath(root, generation string, index int, offset int64) (string, error) {
dir, err := WALPath(root, generation)
if err != nil {
return "", err
}
return path.Join(dir, FormatWALSegmentPath(index, offset)), nil
}
// IsSnapshotPath returns true if s is a path to a snapshot file.
func IsSnapshotPath(s string) bool {
return snapshotPathRegex.MatchString(s)

88
litestream_test.go
View File

@@ -40,6 +40,94 @@ func TestChecksum(t *testing.T) {
})
}
func TestGenerationsPath(t *testing.T) {
t.Run("OK", func(t *testing.T) {
if got, want := litestream.GenerationsPath("foo"), "foo/generations"; got != want {
t.Fatalf("GenerationsPath()=%v, want %v", got, want)
}
})
t.Run("NoPath", func(t *testing.T) {
if got, want := litestream.GenerationsPath(""), "generations"; got != want {
t.Fatalf("GenerationsPath()=%v, want %v", got, want)
}
})
}
func TestGenerationPath(t *testing.T) {
t.Run("OK", func(t *testing.T) {
if got, err := litestream.GenerationPath("foo", "0123456701234567"); err != nil {
t.Fatal(err)
} else if want := "foo/generations/0123456701234567"; got != want {
t.Fatalf("GenerationPath()=%v, want %v", got, want)
}
})
t.Run("ErrNoGeneration", func(t *testing.T) {
if _, err := litestream.GenerationPath("foo", ""); err == nil || err.Error() != `generation required` {
t.Fatalf("expected error: %v", err)
}
})
}
func TestSnapshotsPath(t *testing.T) {
t.Run("OK", func(t *testing.T) {
if got, err := litestream.SnapshotsPath("foo", "0123456701234567"); err != nil {
t.Fatal(err)
} else if want := "foo/generations/0123456701234567/snapshots"; got != want {
t.Fatalf("SnapshotsPath()=%v, want %v", got, want)
}
})
t.Run("ErrNoGeneration", func(t *testing.T) {
if _, err := litestream.SnapshotsPath("foo", ""); err == nil || err.Error() != `generation required` {
t.Fatalf("unexpected error: %v", err)
}
})
}
func TestSnapshotPath(t *testing.T) {
t.Run("OK", func(t *testing.T) {
if got, err := litestream.SnapshotPath("foo", "0123456701234567", 1000); err != nil {
t.Fatal(err)
} else if want := "foo/generations/0123456701234567/snapshots/000003e8.snapshot.lz4"; got != want {
t.Fatalf("SnapshotPath()=%v, want %v", got, want)
}
})
t.Run("ErrNoGeneration", func(t *testing.T) {
if _, err := litestream.SnapshotPath("foo", "", 1000); err == nil || err.Error() != `generation required` {
t.Fatalf("unexpected error: %v", err)
}
})
}
func TestWALPath(t *testing.T) {
t.Run("OK", func(t *testing.T) {
if got, err := litestream.WALPath("foo", "0123456701234567"); err != nil {
t.Fatal(err)
} else if want := "foo/generations/0123456701234567/wal"; got != want {
t.Fatalf("WALPath()=%v, want %v", got, want)
}
})
t.Run("ErrNoGeneration", func(t *testing.T) {
if _, err := litestream.WALPath("foo", ""); err == nil || err.Error() != `generation required` {
t.Fatalf("unexpected error: %v", err)
}
})
}
func TestWALSegmentPath(t *testing.T) {
t.Run("OK", func(t *testing.T) {
if got, err := litestream.WALSegmentPath("foo", "0123456701234567", 1000, 1001); err != nil {
t.Fatal(err)
} else if want := "foo/generations/0123456701234567/wal/000003e8_000003e9.wal.lz4"; got != want {
t.Fatalf("WALPath()=%v, want %v", got, want)
}
})
t.Run("ErrNoGeneration", func(t *testing.T) {
if _, err := litestream.WALSegmentPath("foo", "", 1000, 0); err == nil || err.Error() != `generation required` {
t.Fatalf("unexpected error: %v", err)
}
})
}
func MustDecodeHexString(s string) []byte {
b, err := hex.DecodeString(s)
if err != nil {

126
s3/replica_client.go
View File

@@ -5,9 +5,11 @@ import (
"crypto/tls"
"fmt"
"io"
"net"
"net/http"
"os"
"path"
"regexp"
"sync"
"time"
@@ -62,56 +64,6 @@ func (c *ReplicaClient) Type() string {
return "s3"
}
// GenerationsDir returns the path to a generation root directory.
func (c *ReplicaClient) GenerationsDir() string {
return path.Join(c.Path, "generations")
}
// GenerationDir returns the path to a generation's root directory.
func (c *ReplicaClient) GenerationDir(generation string) (string, error) {
dir := c.GenerationsDir()
if generation == "" {
return "", fmt.Errorf("generation required")
}
return path.Join(dir, generation), nil
}
// SnapshotsDir returns the path to a generation's snapshot directory.
func (c *ReplicaClient) SnapshotsDir(generation string) (string, error) {
dir, err := c.GenerationDir(generation)
if err != nil {
return "", err
}
return path.Join(dir, "snapshots"), nil
}
// SnapshotPath returns the path to an uncompressed snapshot file.
func (c *ReplicaClient) SnapshotPath(generation string, index int) (string, error) {
dir, err := c.SnapshotsDir(generation)
if err != nil {
return "", err
}
return path.Join(dir, litestream.FormatSnapshotPath(index)), nil
}
// WALDir returns the path to a generation's WAL directory
func (c *ReplicaClient) WALDir(generation string) (string, error) {
dir, err := c.GenerationDir(generation)
if err != nil {
return "", err
}
return path.Join(dir, "wal"), nil
}
// WALSegmentPath returns the path to a WAL segment file.
func (c *ReplicaClient) WALSegmentPath(generation string, index int, offset int64) (string, error) {
dir, err := c.WALDir(generation)
if err != nil {
return "", err
}
return path.Join(dir, litestream.FormatWALSegmentPath(index, offset)), nil
}
// Init initializes the connection to S3. No-op if already initialized.
func (c *ReplicaClient) Init(ctx context.Context) (err error) {
c.mu.Lock()
@@ -201,7 +153,7 @@ func (c *ReplicaClient) Generations(ctx context.Context) ([]string, error) {
var generations []string
if err := c.s3.ListObjectsPagesWithContext(ctx, &s3.ListObjectsInput{
Bucket: aws.String(c.Bucket),
Prefix: aws.String(c.GenerationsDir() + "/"),
Prefix: aws.String(litestream.GenerationsPath(c.Path) + "/"),
Delimiter: aws.String("/"),
}, func(page *s3.ListObjectsOutput, lastPage bool) bool {
operationTotalCounterVec.WithLabelValues("LIST").Inc()
@@ -227,7 +179,7 @@ func (c *ReplicaClient) DeleteGeneration(ctx context.Context, generation string)
return err
}
dir, err := c.GenerationDir(generation)
dir, err := litestream.GenerationPath(c.Path, generation)
if err != nil {
return fmt.Errorf("cannot determine generation directory path: %w", err)
}
@@ -285,7 +237,7 @@ func (c *ReplicaClient) WriteSnapshot(ctx context.Context, generation string, in
return info, err
}
key, err := c.SnapshotPath(generation, index)
key, err := litestream.SnapshotPath(c.Path, generation, index)
if err != nil {
return info, fmt.Errorf("cannot determine snapshot path: %w", err)
}
@@ -319,7 +271,7 @@ func (c *ReplicaClient) SnapshotReader(ctx context.Context, generation string, i
return nil, err
}
key, err := c.SnapshotPath(generation, index)
key, err := litestream.SnapshotPath(c.Path, generation, index)
if err != nil {
return nil, fmt.Errorf("cannot determine snapshot path: %w", err)
}
@@ -345,7 +297,7 @@ func (c *ReplicaClient) DeleteSnapshot(ctx context.Context, generation string, i
return err
}
key, err := c.SnapshotPath(generation, index)
key, err := litestream.SnapshotPath(c.Path, generation, index)
if err != nil {
return fmt.Errorf("cannot determine snapshot path: %w", err)
}
@@ -375,7 +327,7 @@ func (c *ReplicaClient) WriteWALSegment(ctx context.Context, pos litestream.Pos,
return info, err
}
key, err := c.WALSegmentPath(pos.Generation, pos.Index, pos.Offset)
key, err := litestream.WALSegmentPath(c.Path, pos.Generation, pos.Index, pos.Offset)
if err != nil {
return info, fmt.Errorf("cannot determine wal segment path: %w", err)
}
@@ -409,7 +361,7 @@ func (c *ReplicaClient) WALSegmentReader(ctx context.Context, pos litestream.Pos
return nil, err
}
key, err := c.WALSegmentPath(pos.Generation, pos.Index, pos.Offset)
key, err := litestream.WALSegmentPath(c.Path, pos.Generation, pos.Index, pos.Offset)
if err != nil {
return nil, fmt.Errorf("cannot determine wal segment path: %w", err)
}
@@ -444,7 +396,7 @@ func (c *ReplicaClient) DeleteWALSegments(ctx context.Context, a []litestream.Po
// Generate a batch of object IDs for deleting the WAL segments.
for i, pos := range a[:n] {
key, err := c.WALSegmentPath(pos.Generation, pos.Index, pos.Offset)
key, err := litestream.WALSegmentPath(c.Path, pos.Generation, pos.Index, pos.Offset)
if err != nil {
return fmt.Errorf("cannot determine wal segment path: %w", err)
}
@@ -545,7 +497,7 @@ func newSnapshotIterator(ctx context.Context, client *ReplicaClient, generation
func (itr *snapshotIterator) fetch() error {
defer close(itr.ch)
dir, err := itr.client.SnapshotsDir(itr.generation)
dir, err := litestream.SnapshotsPath(itr.client.Path, itr.generation)
if err != nil {
return fmt.Errorf("cannot determine snapshot directory path: %w", err)
}
@@ -648,7 +600,7 @@ func newWALSegmentIterator(ctx context.Context, client *ReplicaClient, generatio
func (itr *walSegmentIterator) fetch() error {
defer close(itr.ch)
dir, err := itr.client.WALDir(itr.generation)
dir, err := litestream.WALPath(itr.client.Path, itr.generation)
if err != nil {
return fmt.Errorf("cannot determine wal directory path: %w", err)
}
@@ -723,6 +675,60 @@ func (itr *walSegmentIterator) WALSegment() litestream.WALSegmentInfo {
return itr.info
}
// ParseHost extracts data from a hostname depending on the service provider.
func ParseHost(s string) (bucket, region, endpoint string, forcePathStyle bool) {
// Extract port if one is specified.
host, port, err := net.SplitHostPort(s)
if err != nil {
host = s
}
// Default to path-based URLs, except for with AWS S3 itself.
forcePathStyle = true
// Extract fields from provider-specific host formats.
scheme := "https"
if a := localhostRegex.FindStringSubmatch(host); a != nil {
bucket, region = a[1], "us-east-1"
scheme, endpoint = "http", "localhost"
} else if a := gcsRegex.FindStringSubmatch(host); a != nil {
bucket, region = a[1], "us-east-1"
endpoint = "storage.googleapis.com"
} else if a := digitalOceanRegex.FindStringSubmatch(host); a != nil {
bucket, region = a[1], a[2]
endpoint = fmt.Sprintf("%s.digitaloceanspaces.com", region)
} else if a := linodeRegex.FindStringSubmatch(host); a != nil {
bucket, region = a[1], a[2]
endpoint = fmt.Sprintf("%s.linodeobjects.com", region)
} else if a := backblazeRegex.FindStringSubmatch(host); a != nil {
bucket, region = a[1], a[2]
endpoint = fmt.Sprintf("s3.%s.backblazeb2.com", region)
} else {
bucket = host
forcePathStyle = false
}
// Add port back to endpoint, if available.
if endpoint != "" && port != "" {
endpoint = net.JoinHostPort(endpoint, port)
}
// Prepend scheme to endpoint.
if endpoint != "" {
endpoint = scheme + "://" + endpoint
}
return bucket, region, endpoint, forcePathStyle
}
var (
localhostRegex = regexp.MustCompile(`^(?:(.+)\.)?localhost$`)
digitalOceanRegex = regexp.MustCompile(`^(?:(.+)\.)?([^.]+)\.digitaloceanspaces.com$`)
linodeRegex = regexp.MustCompile(`^(?:(.+)\.)?([^.]+)\.linodeobjects.com$`)
backblazeRegex = regexp.MustCompile(`^(?:(.+)\.)?s3.([^.]+)\.backblazeb2.com$`)
gcsRegex = regexp.MustCompile(`^(?:(.+)\.)?storage.googleapis.com$`)
)
func isNotExists(err error) bool {
switch err := err.(type) {
case awserr.Error:

173
s3/replica_client_test.go
View File

@@ -38,106 +38,6 @@ func TestReplicaClient_Type(t *testing.T) {
}
}
func TestReplicaClient_GenerationsDir(t *testing.T) {
t.Run("OK", func(t *testing.T) {
c := s3.NewReplicaClient()
c.Path = "foo"
if got, want := c.GenerationsDir(), "foo/generations"; got != want {
t.Fatalf("GenerationsDir()=%v, want %v", got, want)
}
})
t.Run("NoPath", func(t *testing.T) {
if got, want := s3.NewReplicaClient().GenerationsDir(), "generations"; got != want {
t.Fatalf("GenerationsDir()=%v, want %v", got, want)
}
})
}
func TestReplicaClient_GenerationDir(t *testing.T) {
t.Run("OK", func(t *testing.T) {
c := s3.NewReplicaClient()
c.Path = "foo"
if got, err := c.GenerationDir("0123456701234567"); err != nil {
t.Fatal(err)
} else if want := "foo/generations/0123456701234567"; got != want {
t.Fatalf("GenerationDir()=%v, want %v", got, want)
}
})
t.Run("ErrNoGeneration", func(t *testing.T) {
if _, err := s3.NewReplicaClient().GenerationDir(""); err == nil || err.Error() != `generation required` {
t.Fatalf("expected error: %v", err)
}
})
}
func TestReplicaClient_SnapshotsDir(t *testing.T) {
t.Run("OK", func(t *testing.T) {
c := s3.NewReplicaClient()
c.Path = "foo"
if got, err := c.SnapshotsDir("0123456701234567"); err != nil {
t.Fatal(err)
} else if want := "foo/generations/0123456701234567/snapshots"; got != want {
t.Fatalf("SnapshotsDir()=%v, want %v", got, want)
}
})
t.Run("ErrNoGeneration", func(t *testing.T) {
if _, err := s3.NewReplicaClient().SnapshotsDir(""); err == nil || err.Error() != `generation required` {
t.Fatalf("unexpected error: %v", err)
}
})
}
func TestReplicaClient_SnapshotPath(t *testing.T) {
t.Run("OK", func(t *testing.T) {
c := s3.NewReplicaClient()
c.Path = "foo"
if got, err := c.SnapshotPath("0123456701234567", 1000); err != nil {
t.Fatal(err)
} else if want := "foo/generations/0123456701234567/snapshots/000003e8.snapshot.lz4"; got != want {
t.Fatalf("SnapshotPath()=%v, want %v", got, want)
}
})
t.Run("ErrNoGeneration", func(t *testing.T) {
if _, err := s3.NewReplicaClient().SnapshotPath("", 1000); err == nil || err.Error() != `generation required` {
t.Fatalf("unexpected error: %v", err)
}
})
}
func TestReplicaClient_WALDir(t *testing.T) {
t.Run("OK", func(t *testing.T) {
c := s3.NewReplicaClient()
c.Path = "foo"
if got, err := c.WALDir("0123456701234567"); err != nil {
t.Fatal(err)
} else if want := "foo/generations/0123456701234567/wal"; got != want {
t.Fatalf("WALDir()=%v, want %v", got, want)
}
})
t.Run("ErrNoGeneration", func(t *testing.T) {
if _, err := s3.NewReplicaClient().WALDir(""); err == nil || err.Error() != `generation required` {
t.Fatalf("unexpected error: %v", err)
}
})
}
func TestReplicaClient_WALSegmentPath(t *testing.T) {
t.Run("OK", func(t *testing.T) {
c := s3.NewReplicaClient()
c.Path = "foo"
if got, err := c.WALSegmentPath("0123456701234567", 1000, 1001); err != nil {
t.Fatal(err)
} else if want := "foo/generations/0123456701234567/wal/000003e8_000003e9.wal.lz4"; got != want {
t.Fatalf("WALPath()=%v, want %v", got, want)
}
})
t.Run("ErrNoGeneration", func(t *testing.T) {
if _, err := s3.NewReplicaClient().WALSegmentPath("", 1000, 0); err == nil || err.Error() != `generation required` {
t.Fatalf("unexpected error: %v", err)
}
})
}
func TestReplicaClient_Generations(t *testing.T) {
t.Run("OK", func(t *testing.T) {
t.Parallel()
@@ -574,6 +474,79 @@ func TestReplicaClient_DeleteWALSegments(t *testing.T) {
})
}
func TestParseHost(t *testing.T) {
// Ensure non-specific hosts return as buckets.
t.Run("S3", func(t *testing.T) {
bucket, region, endpoint, forcePathStyle := s3.ParseHost(`test.litestream.io`)
if got, want := bucket, `test.litestream.io`; got != want {
t.Fatalf("bucket=%q, want %q", got, want)
} else if got, want := region, ``; got != want {
t.Fatalf("region=%q, want %q", got, want)
} else if got, want := endpoint, ``; got != want {
t.Fatalf("endpoint=%q, want %q", got, want)
} else if got, want := forcePathStyle, false; got != want {
t.Fatalf("forcePathStyle=%v, want %v", got, want)
}
})
// Ensure localhosts use an HTTP endpoint and extract the bucket name.
t.Run("Localhost", func(t *testing.T) {
t.Run("WithPort", func(t *testing.T) {
bucket, region, endpoint, forcePathStyle := s3.ParseHost(`test.localhost:9000`)
if got, want := bucket, `test`; got != want {
t.Fatalf("bucket=%q, want %q", got, want)
} else if got, want := region, `us-east-1`; got != want {
t.Fatalf("region=%q, want %q", got, want)
} else if got, want := endpoint, `http://localhost:9000`; got != want {
t.Fatalf("endpoint=%q, want %q", got, want)
} else if got, want := forcePathStyle, true; got != want {
t.Fatalf("forcePathStyle=%v, want %v", got, want)
}
})
t.Run("WithoutPort", func(t *testing.T) {
bucket, region, endpoint, forcePathStyle := s3.ParseHost(`test.localhost`)
if got, want := bucket, `test`; got != want {
t.Fatalf("bucket=%q, want %q", got, want)
} else if got, want := region, `us-east-1`; got != want {
t.Fatalf("region=%q, want %q", got, want)
} else if got, want := endpoint, `http://localhost`; got != want {
t.Fatalf("endpoint=%q, want %q", got, want)
} else if got, want := forcePathStyle, true; got != want {
t.Fatalf("forcePathStyle=%v, want %v", got, want)
}
})
})
// Ensure backblaze B2 URLs extract bucket, region, & endpoint from host.
t.Run("Backblaze", func(t *testing.T) {
bucket, region, endpoint, forcePathStyle := s3.ParseHost(`test-123.s3.us-west-000.backblazeb2.com`)
if got, want := bucket, `test-123`; got != want {
t.Fatalf("bucket=%q, want %q", got, want)
} else if got, want := region, `us-west-000`; got != want {
t.Fatalf("region=%q, want %q", got, want)
} else if got, want := endpoint, `https://s3.us-west-000.backblazeb2.com`; got != want {
t.Fatalf("endpoint=%q, want %q", got, want)
} else if got, want := forcePathStyle, true; got != want {
t.Fatalf("forcePathStyle=%v, want %v", got, want)
}
})
// Ensure GCS URLs extract bucket & endpoint from host.
t.Run("GCS", func(t *testing.T) {
bucket, region, endpoint, forcePathStyle := s3.ParseHost(`litestream.io.storage.googleapis.com`)
if got, want := bucket, `litestream.io`; got != want {
t.Fatalf("bucket=%q, want %q", got, want)
} else if got, want := region, `us-east-1`; got != want {
t.Fatalf("region=%q, want %q", got, want)
} else if got, want := endpoint, `https://storage.googleapis.com`; got != want {
t.Fatalf("endpoint=%q, want %q", got, want)
} else if got, want := forcePathStyle, true; got != want {
t.Fatalf("forcePathStyle=%v, want %v", got, want)
}
})
}
// NewIntegrationReplicaClient returns a new client for integration testing.
// If integration flag is not set then test/benchmark is skipped.
func NewIntegrationReplicaClient(tb testing.TB) *s3.ReplicaClient {

61
s3/s3.go
View File

@@ -1,61 +0,0 @@
package s3
import (
"fmt"
"net"
"regexp"
)
// ParseHost extracts data from a hostname depending on the service provider.
func ParseHost(s string) (bucket, region, endpoint string, forcePathStyle bool) {
// Extract port if one is specified.
host, port, err := net.SplitHostPort(s)
if err != nil {
host = s
}
// Default to path-based URLs, except for with AWS S3 itself.
forcePathStyle = true
// Extract fields from provider-specific host formats.
scheme := "https"
if a := localhostRegex.FindStringSubmatch(host); a != nil {
bucket, region = a[1], "us-east-1"
scheme, endpoint = "http", "localhost"
} else if a := gcsRegex.FindStringSubmatch(host); a != nil {
bucket, region = a[1], "us-east-1"
endpoint = "storage.googleapis.com"
} else if a := digitalOceanRegex.FindStringSubmatch(host); a != nil {
bucket, region = a[1], a[2]
endpoint = fmt.Sprintf("%s.digitaloceanspaces.com", region)
} else if a := linodeRegex.FindStringSubmatch(host); a != nil {
bucket, region = a[1], a[2]
endpoint = fmt.Sprintf("%s.linodeobjects.com", region)
} else if a := backblazeRegex.FindStringSubmatch(host); a != nil {
bucket, region = a[1], a[2]
endpoint = fmt.Sprintf("s3.%s.backblazeb2.com", region)
} else {
bucket = host
forcePathStyle = false
}
// Add port back to endpoint, if available.
if endpoint != "" && port != "" {
endpoint = net.JoinHostPort(endpoint, port)
}
// Prepend scheme to endpoint.
if endpoint != "" {
endpoint = scheme + "://" + endpoint
}
return bucket, region, endpoint, forcePathStyle
}
var (
localhostRegex = regexp.MustCompile(`^(?:(.+)\.)?localhost$`)
digitalOceanRegex = regexp.MustCompile(`^(?:(.+)\.)?([^.]+)\.digitaloceanspaces.com$`)
linodeRegex = regexp.MustCompile(`^(?:(.+)\.)?([^.]+)\.linodeobjects.com$`)
backblazeRegex = regexp.MustCompile(`^(?:(.+)\.)?s3.([^.]+)\.backblazeb2.com$`)
gcsRegex = regexp.MustCompile(`^(?:(.+)\.)?storage.googleapis.com$`)
)

80
s3/s3_test.go
View File

@@ -1,80 +0,0 @@
package s3_test
import (
"testing"
"github.com/benbjohnson/litestream/s3"
)
func TestParseHost(t *testing.T) {
// Ensure non-specific hosts return as buckets.
t.Run("S3", func(t *testing.T) {
bucket, region, endpoint, forcePathStyle := s3.ParseHost(`test.litestream.io`)
if got, want := bucket, `test.litestream.io`; got != want {
t.Fatalf("bucket=%q, want %q", got, want)
} else if got, want := region, ``; got != want {
t.Fatalf("region=%q, want %q", got, want)
} else if got, want := endpoint, ``; got != want {
t.Fatalf("endpoint=%q, want %q", got, want)
} else if got, want := forcePathStyle, false; got != want {
t.Fatalf("forcePathStyle=%v, want %v", got, want)
}
})
// Ensure localhosts use an HTTP endpoint and extract the bucket name.
t.Run("Localhost", func(t *testing.T) {
t.Run("WithPort", func(t *testing.T) {
bucket, region, endpoint, forcePathStyle := s3.ParseHost(`test.localhost:9000`)
if got, want := bucket, `test`; got != want {
t.Fatalf("bucket=%q, want %q", got, want)
} else if got, want := region, `us-east-1`; got != want {
t.Fatalf("region=%q, want %q", got, want)
} else if got, want := endpoint, `http://localhost:9000`; got != want {
t.Fatalf("endpoint=%q, want %q", got, want)
} else if got, want := forcePathStyle, true; got != want {
t.Fatalf("forcePathStyle=%v, want %v", got, want)
}
})
t.Run("WithoutPort", func(t *testing.T) {
bucket, region, endpoint, forcePathStyle := s3.ParseHost(`test.localhost`)
if got, want := bucket, `test`; got != want {
t.Fatalf("bucket=%q, want %q", got, want)
} else if got, want := region, `us-east-1`; got != want {
t.Fatalf("region=%q, want %q", got, want)
} else if got, want := endpoint, `http://localhost`; got != want {
t.Fatalf("endpoint=%q, want %q", got, want)
} else if got, want := forcePathStyle, true; got != want {
t.Fatalf("forcePathStyle=%v, want %v", got, want)
}
})
})
// Ensure backblaze B2 URLs extract bucket, region, & endpoint from host.
t.Run("Backblaze", func(t *testing.T) {
bucket, region, endpoint, forcePathStyle := s3.ParseHost(`test-123.s3.us-west-000.backblazeb2.com`)
if got, want := bucket, `test-123`; got != want {
t.Fatalf("bucket=%q, want %q", got, want)
} else if got, want := region, `us-west-000`; got != want {
t.Fatalf("region=%q, want %q", got, want)
} else if got, want := endpoint, `https://s3.us-west-000.backblazeb2.com`; got != want {
t.Fatalf("endpoint=%q, want %q", got, want)
} else if got, want := forcePathStyle, true; got != want {
t.Fatalf("forcePathStyle=%v, want %v", got, want)
}
})
// Ensure GCS URLs extract bucket & endpoint from host.
t.Run("GCS", func(t *testing.T) {
bucket, region, endpoint, forcePathStyle := s3.ParseHost(`litestream.io.storage.googleapis.com`)
if got, want := bucket, `litestream.io`; got != want {
t.Fatalf("bucket=%q, want %q", got, want)
} else if got, want := region, `us-east-1`; got != want {
t.Fatalf("region=%q, want %q", got, want)
} else if got, want := endpoint, `https://storage.googleapis.com`; got != want {
t.Fatalf("endpoint=%q, want %q", got, want)
} else if got, want := forcePathStyle, true; got != want {
t.Fatalf("forcePathStyle=%v, want %v", got, want)
}
})
}