Remove streaming replication implementation

http/client.go

@@ -1,140 +0,0 @@
-package http
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"net/http"
-	"net/url"
-	"strconv"
-
-	"github.com/benbjohnson/litestream"
-)
-
-// Client represents an client for a streaming Litestream HTTP server.
-type Client struct {
-	// Upstream endpoint
-	URL string
-
-	// Path of database on upstream server.
-	Path string
-
-	// Underlying HTTP client
-	HTTPClient *http.Client
-}
-
-// NewClient returns an instance of Client.
-func NewClient(rawurl, path string) *Client {
-	return &Client{
-		URL:        rawurl,
-		Path:       path,
-		HTTPClient: http.DefaultClient,
-	}
-}
-
-// Stream returns a snapshot and continuous stream of WAL updates.
-func (c *Client) Stream(ctx context.Context, pos litestream.Pos) (litestream.StreamReader, error) {
-	u, err := url.Parse(c.URL)
-	if err != nil {
-		return nil, fmt.Errorf("invalid client URL: %w", err)
-	} else if u.Scheme != "http" && u.Scheme != "https" {
-		return nil, fmt.Errorf("invalid URL scheme")
-	} else if u.Host == "" {
-		return nil, fmt.Errorf("URL host required")
-	}
-
-	// Add path & position to query path.
-	q := url.Values{"path": []string{c.Path}}
-	if !pos.IsZero() {
-		q.Set("generation", pos.Generation)
-		q.Set("index", litestream.FormatIndex(pos.Index))
-	}
-
-	// Strip off everything but the scheme & host.
-	*u = url.URL{
-		Scheme:   u.Scheme,
-		Host:     u.Host,
-		Path:     "/stream",
-		RawQuery: q.Encode(),
-	}
-
-	req, err := http.NewRequest("GET", u.String(), nil)
-	if err != nil {
-		return nil, err
-	}
-	req = req.WithContext(ctx)
-
-	resp, err := c.HTTPClient.Do(req)
-	if err != nil {
-		return nil, err
-	} else if resp.StatusCode != http.StatusOK {
-		resp.Body.Close()
-		return nil, fmt.Errorf("invalid response: code=%d", resp.StatusCode)
-	}
-
-	pageSize, _ := strconv.Atoi(resp.Header.Get("Litestream-page-size"))
-	if pageSize <= 0 {
-		resp.Body.Close()
-		return nil, fmt.Errorf("stream page size unavailable")
-	}
-
-	return &StreamReader{
-		pageSize: pageSize,
-		rc:       resp.Body,
-		lr:       io.LimitedReader{R: resp.Body},
-	}, nil
-}
-
-// StreamReader represents an optional snapshot followed by a continuous stream
-// of WAL updates. It is used to implement live read replication from a single
-// primary Litestream server to one or more remote Litestream replicas.
-type StreamReader struct {
-	pageSize int
-	rc       io.ReadCloser
-	lr       io.LimitedReader
-}
-
-// Close closes the underlying reader.
-func (r *StreamReader) Close() (err error) {
-	if e := r.rc.Close(); err == nil {
-		err = e
-	}
-	return err
-}
-
-// PageSize returns the page size on the remote database.
-func (r *StreamReader) PageSize() int { return r.pageSize }
-
-// Read reads bytes of the current payload into p. Only valid after a successful
-// call to Next(). On io.EOF, call Next() again to begin reading next record.
-func (r *StreamReader) Read(p []byte) (n int, err error) {
-	return r.lr.Read(p)
-}
-
-// Next returns the next available record. This call will block until a record
-// is available. After calling Next(), read the payload from the reader using
-// Read() until io.EOF is reached.
-func (r *StreamReader) Next() (*litestream.StreamRecordHeader, error) {
-	// If bytes remain on the current file, discard.
-	if r.lr.N > 0 {
-		if _, err := io.Copy(io.Discard, &r.lr); err != nil {
-			return nil, err
-		}
-	}
-
-	// Read record header.
-	buf := make([]byte, litestream.StreamRecordHeaderSize)
-	if _, err := io.ReadFull(r.rc, buf); err != nil {
-		return nil, fmt.Errorf("http.StreamReader.Next(): %w", err)
-	}
-
-	var hdr litestream.StreamRecordHeader
-	if err := hdr.UnmarshalBinary(buf); err != nil {
-		return nil, err
-	}
-
-	// Update remaining bytes on file reader.
-	r.lr.N = hdr.Size
-
-	return &hdr, nil
-}

http/server.go

@@ -2,13 +2,11 @@ package http
 
 import (
 	"fmt"
-	"io"
 	"log"
 	"net"
 	"net/http"
 	httppprof "net/http/pprof"
 	"os"
-	"strconv"
 	"strings"
 
 	"github.com/benbjohnson/litestream"
@@ -113,190 +111,7 @@ func (s *Server) serveHTTP(w http.ResponseWriter, r *http.Request) {
 	switch r.URL.Path {
 	case "/metrics":
 		s.promHandler.ServeHTTP(w, r)
-
-	case "/stream":
-		switch r.Method {
-		case http.MethodGet:
-			s.handleGetStream(w, r)
-		default:
-			s.writeError(w, r, "Method not allowed", http.StatusMethodNotAllowed)
-		}
 	default:
 		http.NotFound(w, r)
 	}
 }
-
-func (s *Server) handleGetStream(w http.ResponseWriter, r *http.Request) {
-	q := r.URL.Query()
-	path := q.Get("path")
-	if path == "" {
-		s.writeError(w, r, "Database name required", http.StatusBadRequest)
-		return
-	}
-
-	// Parse current client position, if available.
-	var pos litestream.Pos
-	if generation, index := q.Get("generation"), q.Get("index"); generation != "" && index != "" {
-		pos.Generation = generation
-
-		var err error
-		if pos.Index, err = litestream.ParseIndex(index); err != nil {
-			s.writeError(w, r, "Invalid index query parameter", http.StatusBadRequest)
-			return
-		}
-	}
-
-	// Fetch database instance from the primary server.
-	db := s.server.DB(path)
-	if db == nil {
-		s.writeError(w, r, "Database not found", http.StatusNotFound)
-		return
-	}
-
-	// Set the page size in the header.
-	w.Header().Set("Litestream-page-size", strconv.Itoa(db.PageSize()))
-
-	// Determine starting position.
-	dbPos := db.Pos()
-	if dbPos.Generation == "" {
-		s.writeError(w, r, "No generation available", http.StatusServiceUnavailable)
-		return
-	}
-	dbPos.Offset = 0
-
-	// Use database position if generation has changed.
-	var snapshotRequired bool
-	if pos.Generation != dbPos.Generation {
-		s.Logger.Printf("stream generation mismatch, using primary position: client.pos=%s", pos)
-		pos, snapshotRequired = dbPos, true
-	}
-
-	// Obtain iterator before snapshot so we don't miss any WAL segments.
-	fitr, err := db.WALSegments(r.Context(), pos.Generation)
-	if err != nil {
-		s.writeError(w, r, fmt.Sprintf("Cannot obtain WAL iterator: %s", err), http.StatusInternalServerError)
-		return
-	}
-	defer fitr.Close()
-
-	bitr := litestream.NewBufferedWALSegmentIterator(fitr)
-
-	// Peek at first position to see if client is too old.
-	if info, ok := bitr.Peek(); !ok {
-		s.writeError(w, r, "cannot peek WAL iterator, no segments available", http.StatusInternalServerError)
-		return
-	} else if cmp, err := litestream.ComparePos(pos, info.Pos()); err != nil {
-		s.writeError(w, r, fmt.Sprintf("cannot compare pos: %s", err), http.StatusInternalServerError)
-		return
-	} else if cmp == -1 {
-		s.Logger.Printf("stream position no longer available, using using primary position: client.pos=%s", pos)
-		pos, snapshotRequired = dbPos, true
-	}
-
-	s.Logger.Printf("stream connected: pos=%s snapshot=%v", pos, snapshotRequired)
-	defer s.Logger.Printf("stream disconnected")
-
-	// Write snapshot to response body.
-	if snapshotRequired {
-		if err := db.WithFile(func(f *os.File) error {
-			fi, err := f.Stat()
-			if err != nil {
-				return err
-			}
-
-			// Write snapshot header with current position & size.
-			hdr := litestream.StreamRecordHeader{
-				Type:       litestream.StreamRecordTypeSnapshot,
-				Generation: pos.Generation,
-				Index:      pos.Index,
-				Size:       fi.Size(),
-			}
-			if buf, err := hdr.MarshalBinary(); err != nil {
-				return fmt.Errorf("marshal snapshot stream record header: %w", err)
-			} else if _, err := w.Write(buf); err != nil {
-				return fmt.Errorf("write snapshot stream record header: %w", err)
-			}
-
-			if _, err := io.CopyN(w, f, fi.Size()); err != nil {
-				return fmt.Errorf("copy snapshot: %w", err)
-			}
-
-			return nil
-		}); err != nil {
-			s.writeError(w, r, err.Error(), http.StatusInternalServerError)
-			return
-		}
-
-		// Flush after snapshot has been written.
-		w.(http.Flusher).Flush()
-	}
-
-	for {
-		// Wait for notification of new entries.
-		select {
-		case <-r.Context().Done():
-			return
-		case <-fitr.NotifyCh():
-		}
-
-		for bitr.Next() {
-			info := bitr.WALSegment()
-
-			// Skip any segments before our initial position.
-			if cmp, err := litestream.ComparePos(info.Pos(), pos); err != nil {
-				s.Logger.Printf("pos compare: %s", err)
-				return
-			} else if cmp == -1 {
-				continue
-			}
-
-			hdr := litestream.StreamRecordHeader{
-				Type:       litestream.StreamRecordTypeWALSegment,
-				Flags:      0,
-				Generation: info.Generation,
-				Index:      info.Index,
-				Offset:     info.Offset,
-				Size:       info.Size,
-			}
-
-			// Write record header.
-			data, err := hdr.MarshalBinary()
-			if err != nil {
-				s.Logger.Printf("marshal WAL segment stream record header: %s", err)
-				return
-			} else if _, err := w.Write(data); err != nil {
-				s.Logger.Printf("write WAL segment stream record header: %s", err)
-				return
-			}
-
-			// Copy WAL segment data to writer.
-			if err := func() error {
-				rd, err := db.WALSegmentReader(r.Context(), info.Pos())
-				if err != nil {
-					return fmt.Errorf("cannot fetch wal segment reader: %w", err)
-				}
-				defer rd.Close()
-
-				if _, err := io.CopyN(w, rd, hdr.Size); err != nil {
-					return fmt.Errorf("cannot copy wal segment: %w", err)
-				}
-				return nil
-			}(); err != nil {
-				log.Print(err)
-				return
-			}
-
-			// Flush after WAL segment has been written.
-			w.(http.Flusher).Flush()
-		}
-		if err := bitr.Err(); err != nil {
-			s.Logger.Printf("wal iterator error: %s", err)
-			return
-		}
-	}
-}
-
-func (s *Server) writeError(w http.ResponseWriter, r *http.Request, err string, code int) {
-	s.Logger.Printf("error: %s", err)
-	http.Error(w, err, code)
-}