Implement live read replication

This commit adds an http server and client for streaming snapshots
and WAL pages from an upstream Litestream primary to a read-only
replica.

litestream.go

@@ -1,6 +1,7 @@
 package litestream
 
 import (
+	"context"
 	"database/sql"
 	"encoding/binary"
 	"errors"
@@ -357,6 +358,9 @@ const (
 
 	// WALFrameHeaderSize is the size of the WAL frame header, in bytes.
 	WALFrameHeaderSize = 24
+
+	// WALIndexHeaderSize is the size of the SHM index header, in bytes.
+	WALIndexHeaderSize = 136
 )
 
 // calcWALSize returns the size of the WAL, in bytes, for a given number of pages.
@@ -462,6 +466,73 @@ func ParseOffset(s string) (int64, error) {
 	return int64(v), nil
 }
 
+const (
+	StreamRecordTypeSnapshot   = 1
+	StreamRecordTypeWALSegment = 2
+)
+
+const StreamRecordHeaderSize = 0 +
+	4 + 4 + // type, flags
+	8 + 8 + 8 + 8 // generation, index, offset, size
+
+type StreamRecordHeader struct {
+	Type       int
+	Flags      int
+	Generation string
+	Index      int
+	Offset     int64
+	Size       int64
+}
+
+func (hdr *StreamRecordHeader) Pos() Pos {
+	return Pos{
+		Generation: hdr.Generation,
+		Index:      hdr.Index,
+		Offset:     hdr.Offset,
+	}
+}
+
+func (hdr *StreamRecordHeader) MarshalBinary() ([]byte, error) {
+	generation, err := strconv.ParseUint(hdr.Generation, 16, 64)
+	if err != nil {
+		return nil, fmt.Errorf("invalid generation: %q", generation)
+	}
+
+	data := make([]byte, StreamRecordHeaderSize)
+	binary.BigEndian.PutUint32(data[0:4], uint32(hdr.Type))
+	binary.BigEndian.PutUint32(data[4:8], uint32(hdr.Flags))
+	binary.BigEndian.PutUint64(data[8:16], generation)
+	binary.BigEndian.PutUint64(data[16:24], uint64(hdr.Index))
+	binary.BigEndian.PutUint64(data[24:32], uint64(hdr.Offset))
+	binary.BigEndian.PutUint64(data[32:40], uint64(hdr.Size))
+	return data, nil
+}
+
+// UnmarshalBinary from data into hdr.
+func (hdr *StreamRecordHeader) UnmarshalBinary(data []byte) error {
+	if len(data) < StreamRecordHeaderSize {
+		return io.ErrUnexpectedEOF
+	}
+	hdr.Type = int(binary.BigEndian.Uint32(data[0:4]))
+	hdr.Flags = int(binary.BigEndian.Uint32(data[4:8]))
+	hdr.Generation = fmt.Sprintf("%16x", binary.BigEndian.Uint64(data[8:16]))
+	hdr.Index = int(binary.BigEndian.Uint64(data[16:24]))
+	hdr.Offset = int64(binary.BigEndian.Uint64(data[24:32]))
+	hdr.Size = int64(binary.BigEndian.Uint64(data[32:40]))
+	return nil
+}
+
+// StreamClient represents a client for streaming changes to a replica DB.
+type StreamClient interface {
+	Stream(ctx context.Context) (StreamReader, error)
+}
+
+// StreamReader represents a reader that streams snapshot and WAL records.
+type StreamReader interface {
+	io.ReadCloser
+	Next() (*StreamRecordHeader, error)
+}
+
 // removeDBFiles deletes the database and related files (journal, shm, wal).
 func removeDBFiles(filename string) error {
 	if err := os.Remove(filename); err != nil && !os.IsNotExist(err) {