rkrtechnology/litestream
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update index & offset encoding

Browse Source 
Previously, the index & offsets were encoded as 8-character hex
strings, however, this limits the maximum value to a `uint32`. This
is normally not an issue, however, indices could go over the maximum
value of 4 billion over time and the offset could exceed this value
for an especially large WAL update. For safety, these encodings have
been updated to 16-character hex encodings.
This commit is contained in:
Ben Johnson
2022-02-08 12:49:36 -07:00
parent 54f3b94d3f
commit 006e4b7155
189 changed files with 203 additions and 197 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
internal/internal.go
View File

@@ -15,6 +15,12 @@ import (
"github.com/prometheus/client_golang/prometheus/promauto"
)
// Platform-independent maximum integer sizes.
const (
MaxUint = ^uint(0)
MaxInt = int(MaxUint >> 1)
)
// ReadCloser wraps a reader to also attach a separate closer.
type ReadCloser struct {
r io.Reader
@@ -170,14 +176,14 @@ func ParseSnapshotPath(s string) (index int, err error) {
return 0, fmt.Errorf("invalid snapshot path")
}
i32, _ := strconv.ParseUint(a[1], 16, 32)
if i32 > math.MaxInt32 {
i64, _ := strconv.ParseUint(a[1], 16, 64)
if i64 > uint64(MaxInt) {
return 0, fmt.Errorf("index too large in snapshot path %q", s)
}
return int(i32), nil
return int(i64), nil
}
var snapshotPathRegex = regexp.MustCompile(`^([0-9a-f]{8})\.snapshot\.lz4$`)
var snapshotPathRegex = regexp.MustCompile(`^([0-9a-f]{16})\.snapshot\.lz4$`)
// ParseWALSegmentPath parses the index/offset from a segment filename. Used by path-based replicas.
func ParseWALSegmentPath(s string) (index int, offset int64, err error) {
@@ -186,18 +192,18 @@ func ParseWALSegmentPath(s string) (index int, offset int64, err error) {
return 0, 0, fmt.Errorf("invalid wal segment path")
}
i32, _ := strconv.ParseUint(a[1], 16, 32)
if i32 > math.MaxInt32 {
i64, _ := strconv.ParseUint(a[1], 16, 64)
if i64 > uint64(MaxInt) {
return 0, 0, fmt.Errorf("index too large in wal segment path %q", s)
}
off64, _ := strconv.ParseUint(a[2], 16, 64)
if off64 > math.MaxInt64 {
return 0, 0, fmt.Errorf("offset too large in wal segment path %q", s)
}
return int(i32), int64(off64), nil
return int(i64), int64(off64), nil
}
var walSegmentPathRegex = regexp.MustCompile(`^([0-9a-f]{8})\/([0-9a-f]{8})\.wal\.lz4$`)
var walSegmentPathRegex = regexp.MustCompile(`^([0-9a-f]{16})\/([0-9a-f]{16})\.wal\.lz4$`)
// Shared replica metrics.
var (

28
internal/internal_test.go
View File

@@ -15,12 +15,12 @@ func TestParseSnapshotPath(t *testing.T) {
index int
err error
}{
{"00bc614e.snapshot.lz4", 12345678, nil},
{"xxxxxxxx.snapshot.lz4", 0, fmt.Errorf("invalid snapshot path")},
{"00bc614.snapshot.lz4", 0, fmt.Errorf("invalid snapshot path")},
{"00bc614e.snapshot.lz", 0, fmt.Errorf("invalid snapshot path")},
{"00bc614e.snapshot", 0, fmt.Errorf("invalid snapshot path")},
{"00bc614e", 0, fmt.Errorf("invalid snapshot path")},
{"0000000000bc614e.snapshot.lz4", 12345678, nil},
{"xxxxxxxxxxxxxxxx.snapshot.lz4", 0, fmt.Errorf("invalid snapshot path")},
{"0000000000bc614.snapshot.lz4", 0, fmt.Errorf("invalid snapshot path")},
{"0000000000bc614e.snapshot.lz", 0, fmt.Errorf("invalid snapshot path")},
{"0000000000bc614e.snapshot", 0, fmt.Errorf("invalid snapshot path")},
{"0000000000bc614e", 0, fmt.Errorf("invalid snapshot path")},
{"", 0, fmt.Errorf("invalid snapshot path")},
} {
t.Run("", func(t *testing.T) {
@@ -41,20 +41,22 @@ func TestParseWALSegmentPath(t *testing.T) {
offset int64
err error
}{
{"00bc614e/000003e8.wal.lz4", 12345678, 1000, nil},
{"00000000/00000000.wal", 0, 0, fmt.Errorf("invalid wal segment path")},
{"00000000/00000000", 0, 0, fmt.Errorf("invalid wal segment path")},
{"00000000/", 0, 0, fmt.Errorf("invalid wal segment path")},
{"00000000", 0, 0, fmt.Errorf("invalid wal segment path")},
{"0000000000bc614e/00000000000003e8.wal.lz4", 12345678, 1000, nil},
{"0000000000000000/0000000000000000.wal", 0, 0, fmt.Errorf("invalid wal segment path")},
{"0000000000000000/0000000000000000", 0, 0, fmt.Errorf("invalid wal segment path")},
{"0000000000000000/", 0, 0, fmt.Errorf("invalid wal segment path")},
{"0000000000000000", 0, 0, fmt.Errorf("invalid wal segment path")},
{"", 0, 0, fmt.Errorf("invalid wal segment path")},
} {
t.Run("", func(t *testing.T) {
index, offset, err := internal.ParseWALSegmentPath(tt.s)
if got, want := index, tt.index; got != want {
t.Errorf("index=%#v, want %#v", got, want)
} else if got, want := offset, tt.offset; got != want {
}
if got, want := offset, tt.offset; got != want {
t.Errorf("offset=%#v, want %#v", got, want)
} else if got, want := err, tt.err; !reflect.DeepEqual(got, want) {
}
if got, want := err, tt.err; !reflect.DeepEqual(got, want) {
t.Errorf("err=%#v, want %#v", got, want)
}
})