Group replica wal segments by index

This commit changes the replica path format to group segments within
a single index in the same directory. This is to eventually add the
ability to seek to a record on file-based systems without having
to iterate over the records. The DB shadow WAL will also be changed
to this same format to support live replicas.
This commit is contained in:
Ben Johnson
2021-06-12 10:20:22 -06:00
parent 55c17b9d8e
commit d2cef2f16b
12 changed files with 497 additions and 419 deletions

61
internal/internal_test.go Normal file
View File

@@ -0,0 +1,61 @@
package internal_test
import (
"fmt"
"reflect"
"testing"
"github.com/benbjohnson/litestream/internal"
)
func TestParseSnapshotPath(t *testing.T) {
for _, tt := range []struct {
s string
index int
err error
}{
{"00bc614e.snapshot.lz4", 12345678, nil},
{"xxxxxxxx.snapshot.lz4", 0, fmt.Errorf("invalid snapshot path")},
{"00bc614.snapshot.lz4", 0, fmt.Errorf("invalid snapshot path")},
{"00bc614e.snapshot.lz", 0, fmt.Errorf("invalid snapshot path")},
{"00bc614e.snapshot", 0, fmt.Errorf("invalid snapshot path")},
{"00bc614e", 0, fmt.Errorf("invalid snapshot path")},
{"", 0, fmt.Errorf("invalid snapshot path")},
} {
t.Run("", func(t *testing.T) {
index, err := internal.ParseSnapshotPath(tt.s)
if got, want := index, tt.index; got != want {
t.Errorf("index=%#v, want %#v", got, want)
} else if got, want := err, tt.err; !reflect.DeepEqual(got, want) {
t.Errorf("err=%#v, want %#v", got, want)
}
})
}
}
func TestParseWALSegmentPath(t *testing.T) {
for _, tt := range []struct {
s string
index int
offset int64
err error
}{
{"00bc614e/000003e8.wal.lz4", 12345678, 1000, nil},
{"00000000/00000000.wal", 0, 0, fmt.Errorf("invalid wal segment path")},
{"00000000/00000000", 0, 0, fmt.Errorf("invalid wal segment path")},
{"00000000/", 0, 0, fmt.Errorf("invalid wal segment path")},
{"00000000", 0, 0, fmt.Errorf("invalid wal segment path")},
{"", 0, 0, fmt.Errorf("invalid wal segment path")},
} {
t.Run("", func(t *testing.T) {
index, offset, err := internal.ParseWALSegmentPath(tt.s)
if got, want := index, tt.index; got != want {
t.Errorf("index=%#v, want %#v", got, want)
} else if got, want := offset, tt.offset; got != want {
t.Errorf("offset=%#v, want %#v", got, want)
} else if got, want := err, tt.err; !reflect.DeepEqual(got, want) {
t.Errorf("err=%#v, want %#v", got, want)
}
})
}
}