rkrtechnology/litestream
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Implement streaming WAL segment iterator

Browse Source 
Currently, WALSegmentIterator implementations read to the end of
the end of their list of segments and return EOF. This commit adds
the ability to push additional segments to in-process iterators and
notify their callers that new segments are available. This is only
implemented for the file-based iterator but other segment iterators
may get this implementation in the future or have a wrapping
iterator provide a polling-based implementation.
This commit is contained in:
Ben Johnson
2022-02-11 13:43:50 -07:00
parent 006e4b7155
commit 8589111717
6 changed files with 400 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files

131
file_replica_client_test.go
View File

@@ -1,6 +1,7 @@
package litestream_test
import (
"reflect"
"testing"
"github.com/benbjohnson/litestream"
@@ -133,3 +134,133 @@ func TestReplicaClient_WALSegmentPath(t *testing.T) {
}
})
}
func TestFileWALSegmentIterator_Append(t *testing.T) {
t.Run("Empty", func(t *testing.T) {
itr := litestream.NewFileWALSegmentIterator(t.TempDir(), "0123456789abcdef", nil)
if err := itr.Append(litestream.WALSegmentInfo{Generation: "0123456789abcdef", Index: 0, Offset: 0}); err != nil {
t.Fatal(err)
}
select {
case <-itr.NotifyCh():
default:
t.Fatal("expected notification")
}
if !itr.Next() {
t.Fatal("expected next")
} else if got, want := itr.WALSegment(), (litestream.WALSegmentInfo{Generation: "0123456789abcdef", Index: 0, Offset: 0}); got != want {
t.Fatalf("info=%#v, want %#v", got, want)
}
})
t.Run("MultiOffset", func(t *testing.T) {
itr := litestream.NewFileWALSegmentIterator(t.TempDir(), "0123456789abcdef", nil)
if err := itr.Append(litestream.WALSegmentInfo{Generation: "0123456789abcdef", Index: 0, Offset: 0}); err != nil {
t.Fatal(err)
} else if err := itr.Append(litestream.WALSegmentInfo{Generation: "0123456789abcdef", Index: 0, Offset: 1}); err != nil {
t.Fatal(err)
}
select {
case <-itr.NotifyCh():
default:
t.Fatal("expected notification")
}
if !itr.Next() {
t.Fatal("expected next")
} else if got, want := itr.WALSegment(), (litestream.WALSegmentInfo{Generation: "0123456789abcdef", Index: 0, Offset: 0}); got != want {
t.Fatalf("info=%#v, want %#v", got, want)
}
if !itr.Next() {
t.Fatal("expected next")
} else if got, want := itr.WALSegment(), (litestream.WALSegmentInfo{Generation: "0123456789abcdef", Index: 0, Offset: 1}); got != want {
t.Fatalf("info=%#v, want %#v", got, want)
}
})
t.Run("MultiIndex", func(t *testing.T) {
itr := litestream.NewFileWALSegmentIterator(t.TempDir(), "0123456789abcdef", nil)
if err := itr.Append(litestream.WALSegmentInfo{Generation: "0123456789abcdef", Index: 0, Offset: 0}); err != nil {
t.Fatal(err)
} else if err := itr.Append(litestream.WALSegmentInfo{Generation: "0123456789abcdef", Index: 1, Offset: 0}); err != nil {
t.Fatal(err)
} else if err := itr.Append(litestream.WALSegmentInfo{Generation: "0123456789abcdef", Index: 1, Offset: 1}); err != nil {
t.Fatal(err)
} else if err := itr.Append(litestream.WALSegmentInfo{Generation: "0123456789abcdef", Index: 2, Offset: 0}); err != nil {
t.Fatal(err)
}
if got, want := itr.Indexes(), []int{1, 2}; !reflect.DeepEqual(got, want) {
t.Fatalf("indexes=%v, want %v", got, want)
}
})
t.Run("ErrGenerationMismatch", func(t *testing.T) {
itr := litestream.NewFileWALSegmentIterator(t.TempDir(), "0000000000000000", nil)
if err := itr.Append(litestream.WALSegmentInfo{Generation: "0123456789abcdef", Index: 0, Offset: 0}); err == nil || err.Error() != `generation mismatch` {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrBelowMaxIndex", func(t *testing.T) {
itr := litestream.NewFileWALSegmentIterator(t.TempDir(), "0123456789abcdef", nil)
if err := itr.Append(litestream.WALSegmentInfo{Generation: "0123456789abcdef", Index: 0, Offset: 0}); err != nil {
t.Fatal(err)
} else if err := itr.Append(litestream.WALSegmentInfo{Generation: "0123456789abcdef", Index: 1, Offset: 0}); err != nil {
t.Fatal(err)
} else if err := itr.Append(litestream.WALSegmentInfo{Generation: "0123456789abcdef", Index: 0, Offset: 0}); err == nil || err.Error() != `appended index "0000000000000000" below max index "0000000000000001"` {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrAboveMaxIndex", func(t *testing.T) {
itr := litestream.NewFileWALSegmentIterator(t.TempDir(), "0123456789abcdef", nil)
if err := itr.Append(litestream.WALSegmentInfo{Generation: "0123456789abcdef", Index: 0, Offset: 0}); err != nil {
t.Fatal(err)
} else if err := itr.Append(litestream.WALSegmentInfo{Generation: "0123456789abcdef", Index: 1, Offset: 0}); err != nil {
t.Fatal(err)
} else if err := itr.Append(litestream.WALSegmentInfo{Generation: "0123456789abcdef", Index: 3, Offset: 0}); err == nil || err.Error() != `appended index "0000000000000003" skips index "0000000000000002"` {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrBelowCurrentIndex", func(t *testing.T) {
itr := litestream.NewFileWALSegmentIterator(t.TempDir(), "0123456789abcdef", nil)
if err := itr.Append(litestream.WALSegmentInfo{Generation: "0123456789abcdef", Index: 1, Offset: 0}); err != nil {
t.Fatal(err)
} else if err := itr.Append(litestream.WALSegmentInfo{Generation: "0123456789abcdef", Index: 0, Offset: 0}); err == nil || err.Error() != `appended index "0000000000000000" below current index "0000000000000001"` {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrSkipsNextIndex", func(t *testing.T) {
itr := litestream.NewFileWALSegmentIterator(t.TempDir(), "0123456789abcdef", nil)
if err := itr.Append(litestream.WALSegmentInfo{Generation: "0123456789abcdef", Index: 0, Offset: 0}); err != nil {
t.Fatal(err)
} else if err := itr.Append(litestream.WALSegmentInfo{Generation: "0123456789abcdef", Index: 2, Offset: 0}); err == nil || err.Error() != `appended index "0000000000000002" skips next index "0000000000000001"` {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrBelowOffset", func(t *testing.T) {
itr := litestream.NewFileWALSegmentIterator(t.TempDir(), "0123456789abcdef", nil)
if err := itr.Append(litestream.WALSegmentInfo{Generation: "0123456789abcdef", Index: 0, Offset: 5}); err != nil {
t.Fatal(err)
} else if err := itr.Append(litestream.WALSegmentInfo{Generation: "0123456789abcdef", Index: 0, Offset: 4}); err == nil || err.Error() != `appended offset 0000000000000000/0000000000000004 before last offset 0000000000000000/0000000000000005` {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrDuplicateOffset", func(t *testing.T) {
itr := litestream.NewFileWALSegmentIterator(t.TempDir(), "0123456789abcdef", nil)
if err := itr.Append(litestream.WALSegmentInfo{Generation: "0123456789abcdef", Index: 0, Offset: 5}); err != nil {
t.Fatal(err)
} else if err := itr.Append(litestream.WALSegmentInfo{Generation: "0123456789abcdef", Index: 0, Offset: 5}); err == nil || err.Error() != `duplicate offset 0000000000000000/0000000000000005 appended` {
t.Fatalf("unexpected error: %s", err)
}
})
}