rkrtechnology/litestream
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactor shadow WAL to use segments

Browse Source
This commit is contained in:
Ben Johnson
2021-07-18 08:45:49 -06:00
parent fc897b481f
commit 8ff784b34d
14 changed files with 1023 additions and 845 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
file/replica_client.go
View File

@@ -408,11 +408,6 @@ func (itr *walSegmentIterator) Next() bool {
}
itr.infos = itr.infos[:0] // otherwise clear infos
// Move to the next index unless this is the first time initializing.
if itr.infos != nil && len(itr.indexes) > 0 {
itr.indexes = itr.indexes[1:]
}
// If no indexes remain, stop iteration.
if len(itr.indexes) == 0 {
return false
@@ -420,6 +415,7 @@ func (itr *walSegmentIterator) Next() bool {
// Read segments into a cache for the current index.
index := itr.indexes[0]
itr.indexes = itr.indexes[1:]
f, err := os.Open(filepath.Join(itr.dir, litestream.FormatIndex(index)))
if err != nil {
itr.err = err
@@ -431,7 +427,11 @@ func (itr *walSegmentIterator) Next() bool {
if err != nil {
itr.err = err
return false
} else if err := f.Close(); err != nil {
itr.err = err
return false
}
for _, fi := range fis {
filename := filepath.Base(fi.Name())
if fi.IsDir() {
@@ -452,6 +452,9 @@ func (itr *walSegmentIterator) Next() bool {
})
}
// Ensure segments are sorted within index.
sort.Sort(litestream.WALSegmentInfoSlice(itr.infos))
if len(itr.infos) > 0 {
return true
}

88
file/replica_client_test.go
View File

@@ -133,91 +133,3 @@ func TestReplicaClient_WALSegmentPath(t *testing.T) {
}
})
}
/*
func TestReplica_Sync(t *testing.T) {
// Ensure replica can successfully sync after DB has sync'd.
t.Run("InitialSync", func(t *testing.T) {
db, sqldb := MustOpenDBs(t)
defer MustCloseDBs(t, db, sqldb)
r := litestream.NewReplica(db, "", file.NewReplicaClient(t.TempDir()))
r.MonitorEnabled = false
db.Replicas = []*litestream.Replica{r}
// Sync database & then sync replica.
if err := db.Sync(context.Background()); err != nil {
t.Fatal(err)
} else if err := r.Sync(context.Background()); err != nil {
t.Fatal(err)
}
// Ensure posistions match.
if want, err := db.Pos(); err != nil {
t.Fatal(err)
} else if got, err := r.Pos(context.Background()); err != nil {
t.Fatal(err)
} else if got != want {
t.Fatalf("Pos()=%v, want %v", got, want)
}
})
// Ensure replica can successfully sync multiple times.
t.Run("MultiSync", func(t *testing.T) {
db, sqldb := MustOpenDBs(t)
defer MustCloseDBs(t, db, sqldb)
r := litestream.NewReplica(db, "", file.NewReplicaClient(t.TempDir()))
r.MonitorEnabled = false
db.Replicas = []*litestream.Replica{r}
if _, err := sqldb.Exec(`CREATE TABLE foo (bar TEXT);`); err != nil {
t.Fatal(err)
}
// Write to the database multiple times and sync after each write.
for i, n := 0, db.MinCheckpointPageN*2; i < n; i++ {
if _, err := sqldb.Exec(`INSERT INTO foo (bar) VALUES ('baz')`); err != nil {
t.Fatal(err)
}
// Sync periodically.
if i%100 == 0 || i == n-1 {
if err := db.Sync(context.Background()); err != nil {
t.Fatal(err)
} else if err := r.Sync(context.Background()); err != nil {
t.Fatal(err)
}
}
}
// Ensure posistions match.
pos, err := db.Pos()
if err != nil {
t.Fatal(err)
} else if got, want := pos.Index, 2; got != want {
t.Fatalf("Index=%v, want %v", got, want)
}
if want, err := r.Pos(context.Background()); err != nil {
t.Fatal(err)
} else if got := pos; got != want {
t.Fatalf("Pos()=%v, want %v", got, want)
}
})
// Ensure replica returns an error if there is no generation available from the DB.
t.Run("ErrNoGeneration", func(t *testing.T) {
db, sqldb := MustOpenDBs(t)
defer MustCloseDBs(t, db, sqldb)
r := litestream.NewReplica(db, "", file.NewReplicaClient(t.TempDir()))
r.MonitorEnabled = false
db.Replicas = []*litestream.Replica{r}
if err := r.Sync(context.Background()); err == nil || err.Error() != `no generation, waiting for data` {
t.Fatal(err)
}
})
}
*/