Add minimum shadow WAL retention

integration/cmd_test.go

@@ -455,9 +455,102 @@ func TestCmd_Replicate_HTTP(t *testing.T) {
 }
 
 // Ensure a database can recover when disconnected from HTTP.
-func TestCmd_Replicate_HTTP_Recovery(t *testing.T) {
+func TestCmd_Replicate_HTTP_PartialRecovery(t *testing.T) {
 	ctx := context.Background()
-	testDir, tempDir := filepath.Join("testdata", "replicate", "http-recovery"), t.TempDir()
+	testDir, tempDir := filepath.Join("testdata", "replicate", "http-partial-recovery"), t.TempDir()
+	if err := os.Mkdir(filepath.Join(tempDir, "0"), 0777); err != nil {
+		t.Fatal(err)
+	} else if err := os.Mkdir(filepath.Join(tempDir, "1"), 0777); err != nil {
+		t.Fatal(err)
+	}
+
+	env0 := []string{"LITESTREAM_TEMPDIR=" + tempDir}
+	env1 := []string{"LITESTREAM_TEMPDIR=" + tempDir, "LITESTREAM_UPSTREAM_URL=http://localhost:10002"}
+
+	cmd0, stdout0, _ := commandContext(ctx, env0, "replicate", "-config", filepath.Join(testDir, "litestream.0.yml"))
+	if err := cmd0.Start(); err != nil {
+		t.Fatal(err)
+	}
+	cmd1, stdout1, _ := commandContext(ctx, env1, "replicate", "-config", filepath.Join(testDir, "litestream.1.yml"))
+	if err := cmd1.Start(); err != nil {
+		t.Fatal(err)
+	}
+
+	db0, err := sql.Open("sqlite3", filepath.Join(tempDir, "0", "db"))
+	if err != nil {
+		t.Fatal(err)
+	} else if _, err := db0.ExecContext(ctx, `PRAGMA journal_mode = wal`); err != nil {
+		t.Fatal(err)
+	} else if _, err := db0.ExecContext(ctx, `CREATE TABLE t (id INTEGER PRIMARY KEY)`); err != nil {
+		t.Fatal(err)
+	}
+	defer db0.Close()
+
+	var index int
+	insertAndWait := func() {
+		index++
+		t.Logf("[exec] INSERT INTO t (id) VALUES (%d)", index)
+		if _, err := db0.ExecContext(ctx, `INSERT INTO t (id) VALUES (?)`, index); err != nil {
+			t.Fatal(err)
+		}
+		time.Sleep(100 * time.Millisecond)
+	}
+
+	// Execute writes periodically.
+	for i := 0; i < 50; i++ {
+		insertAndWait()
+	}
+
+	// Kill the replica.
+	t.Logf("Killing replica...")
+	killLitestreamCmd(t, cmd1, stdout1)
+	t.Logf("Replica killed")
+
+	// Keep writing.
+	for i := 0; i < 25; i++ {
+		insertAndWait()
+	}
+
+	// Restart replica.
+	t.Logf("Restarting replica...")
+	cmd1, stdout1, _ = commandContext(ctx, env1, "replicate", "-config", filepath.Join(testDir, "litestream.1.yml"))
+	if err := cmd1.Start(); err != nil {
+		t.Fatal(err)
+	}
+	t.Logf("Replica restarted")
+
+	// Continue writing...
+	for i := 0; i < 25; i++ {
+		insertAndWait()
+	}
+
+	// Wait for replica to catch up.
+	time.Sleep(1 * time.Second)
+
+	// Verify count in replica table.
+	db1, err := sql.Open("sqlite3", filepath.Join(tempDir, "1", "db"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer db1.Close()
+
+	var n int
+	if err := db1.QueryRowContext(ctx, `SELECT COUNT(*) FROM t`).Scan(&n); err != nil {
+		t.Fatal(err)
+	} else if got, want := n, 100; got != want {
+		t.Fatalf("replica count=%d, want %d", got, want)
+	}
+
+	// Stop & wait for Litestream command.
+	killLitestreamCmd(t, cmd1, stdout1) // kill
+	killLitestreamCmd(t, cmd0, stdout0)
+}
+
+// Ensure a database can recover when disconnected from HTTP but when last index
+// is no longer available.
+func TestCmd_Replicate_HTTP_FullRecovery(t *testing.T) {
+	ctx := context.Background()
+	testDir, tempDir := filepath.Join("testdata", "replicate", "http-full-recovery"), t.TempDir()
 	if err := os.Mkdir(filepath.Join(tempDir, "0"), 0777); err != nil {
 		t.Fatal(err)
 	} else if err := os.Mkdir(filepath.Join(tempDir, "1"), 0777); err != nil {

integration/testdata/replicate/http-full-recovery/litestream.0.yml

@@ -0,0 +1,6 @@
+addr: :10002
+
+dbs:
+  - path: $LITESTREAM_TEMPDIR/0/db
+    max-checkpoint-page-count: 5
+    shadow-retention-count: 3

integration/testdata/replicate/http-partial-recovery/litestream.1.yml

@@ -0,0 +1,5 @@
+dbs:
+  - path: $LITESTREAM_TEMPDIR/1/db
+    upstream:
+      url: "$LITESTREAM_UPSTREAM_URL"
+      path: "$LITESTREAM_TEMPDIR/0/db"