Update index & offset encoding

Previously, the index & offsets were encoded as 8-character hex
strings, however, this limits the maximum value to a `uint32`. This
is normally not an issue, however, indices could go over the maximum
value of 4 billion over time and the offset could exceed this value
for an especially large WAL update. For safety, these encodings have
been updated to 16-character hex encodings.

cmd/litestream/main.go

@@ -718,7 +718,7 @@ var _ flag.Value = (*indexVar)(nil)
 
 // String returns an 8-character hexadecimal value.
 func (v *indexVar) String() string {
-	return fmt.Sprintf("%08x", int(*v))
+	return litestream.FormatIndex(int(*v))
 }
 
 // Set parses s into an integer from a hexadecimal value.

cmd/litestream/restore_test.go

@@ -27,10 +27,10 @@ func TestRestoreCommand(t *testing.T) {
 		// STDOUT has timing info so we need to grep per line.
 		lines := strings.Split(stdout.String(), "\n")
 		for i, substr := range []string{
-			`restoring snapshot 0000000000000000/00000000 to ` + filepath.Join(tempDir, "db.tmp"),
-			`applied wal 0000000000000000/00000000 elapsed=`,
-			`applied wal 0000000000000000/00000001 elapsed=`,
-			`applied wal 0000000000000000/00000002 elapsed=`,
+			`restoring snapshot 0000000000000000/0000000000000000 to ` + filepath.Join(tempDir, "db.tmp"),
+			`applied wal 0000000000000000/0000000000000000 elapsed=`,
+			`applied wal 0000000000000000/0000000000000001 elapsed=`,
+			`applied wal 0000000000000000/0000000000000002 elapsed=`,
 			`renaming database from temporary location`,
 		} {
 			if !strings.Contains(lines[i], substr) {
@@ -54,7 +54,7 @@ func TestRestoreCommand(t *testing.T) {
 		// STDOUT has timing info so we need to grep per line.
 		lines := strings.Split(stdout.String(), "\n")
 		for i, substr := range []string{
-			`restoring snapshot 0000000000000001/00000001 to ` + filepath.Join(tempDir, "db.tmp"),
+			`restoring snapshot 0000000000000001/0000000000000001 to ` + filepath.Join(tempDir, "db.tmp"),
 			`no wal files found, snapshot only`,
 			`renaming database from temporary location`,
 		} {
@@ -78,7 +78,7 @@ func TestRestoreCommand(t *testing.T) {
 
 		lines := strings.Split(stdout.String(), "\n")
 		for i, substr := range []string{
-			`restoring snapshot 0000000000000000/00000000 to ` + filepath.Join(tempDir, "db.tmp"),
+			`restoring snapshot 0000000000000000/0000000000000000 to ` + filepath.Join(tempDir, "db.tmp"),
 			`no wal files found, snapshot only`,
 			`renaming database from temporary location`,
 		} {
@@ -102,7 +102,7 @@ func TestRestoreCommand(t *testing.T) {
 
 		lines := strings.Split(stdout.String(), "\n")
 		for i, substr := range []string{
-			`restoring snapshot 0000000000000001/00000000 to ` + filepath.Join(tempDir, "db.tmp"),
+			`restoring snapshot 0000000000000001/0000000000000000 to ` + filepath.Join(tempDir, "db.tmp"),
 			`no wal files found, snapshot only`,
 			`renaming database from temporary location`,
 		} {
@@ -256,7 +256,7 @@ func TestRestoreCommand(t *testing.T) {
 
 	t.Run("ErrInvalidReplicaURL", func(t *testing.T) {
 		m, _, _, _ := newMain()
-		err := m.Run(context.Background(), []string{"restore", "-o", "/tmp/db", "xyz://xyz"})
+		err := m.Run(context.Background(), []string{"restore", "-o", filepath.Join(t.TempDir(), "db"), "xyz://xyz"})
 		if err == nil || err.Error() != `unknown replica type in config: "xyz"` {
 			t.Fatalf("unexpected error: %s", err)
 		}

cmd/litestream/snapshots.go

@@ -83,10 +83,10 @@ func (c *SnapshotsCommand) Run(ctx context.Context, args []string) (ret error) {
 
 	fmt.Fprintln(w, "replica\tgeneration\tindex\tsize\tcreated")
 	for _, info := range infos {
-		fmt.Fprintf(w, "%s\t%s\t%08x\t%d\t%s\n",
+		fmt.Fprintf(w, "%s\t%s\t%s\t%d\t%s\n",
 			info.replicaName,
 			info.Generation,
-			info.Index,
+			litestream.FormatIndex(info.Index),
 			info.Size,
 			info.CreatedAt.Format(time.RFC3339),
 		)

cmd/litestream/testdata/generations/no-database/Makefile

@@ -1,4 +1,4 @@
 .PHONY: default
 default:
-	TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/snapshots/00000000.snapshot.lz4
-	TZ=UTC touch -ct 200001020000 replica/generations/0000000000000001/snapshots/00000000.snapshot.lz4
+	TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/snapshots/0000000000000000.snapshot.lz4
+	TZ=UTC touch -ct 200001020000 replica/generations/0000000000000001/snapshots/0000000000000000.snapshot.lz4

cmd/litestream/testdata/generations/ok/Makefile

@@ -1,9 +1,9 @@
 .PHONY: default
 default:
 	TZ=UTC touch -ct 200001030000 db
-	TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/snapshots/00000000.snapshot.lz4
-	TZ=UTC touch -ct 200001020000 replica/generations/0000000000000000/snapshots/00000001.snapshot.lz4
-	TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/wal/00000000/00000000.wal.lz4
-	TZ=UTC touch -ct 200001020000 replica/generations/0000000000000000/wal/00000000/00000001.wal.lz4
-	TZ=UTC touch -ct 200001030000 replica/generations/0000000000000000/wal/00000001/00000000.wal.lz4
-	TZ=UTC touch -ct 200001010000 replica/generations/0000000000000001/snapshots/00000000.snapshot.lz4
+	TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/snapshots/0000000000000000.snapshot.lz4
+	TZ=UTC touch -ct 200001020000 replica/generations/0000000000000000/snapshots/0000000000000001.snapshot.lz4
+	TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/wal/0000000000000000/0000000000000000.wal.lz4
+	TZ=UTC touch -ct 200001020000 replica/generations/0000000000000000/wal/0000000000000000/0000000000000001.wal.lz4
+	TZ=UTC touch -ct 200001030000 replica/generations/0000000000000000/wal/0000000000000001/0000000000000000.wal.lz4
+	TZ=UTC touch -ct 200001010000 replica/generations/0000000000000001/snapshots/0000000000000000.snapshot.lz4

cmd/litestream/testdata/generations/replica-name/Makefile

@@ -1,5 +1,5 @@
 .PHONY: default
 default:
 	TZ=UTC touch -ct 200001030000 db
-	TZ=UTC touch -ct 200001010000 replica0/generations/0000000000000000/snapshots/00000000.snapshot.lz4
-	TZ=UTC touch -ct 200001020000 replica1/generations/0000000000000001/snapshots/00000000.snapshot.lz4
+	TZ=UTC touch -ct 200001010000 replica0/generations/0000000000000000/snapshots/0000000000000000.snapshot.lz4
+	TZ=UTC touch -ct 200001020000 replica1/generations/0000000000000001/snapshots/0000000000000000.snapshot.lz4

cmd/litestream/testdata/generations/replica-url/Makefile

@@ -1,9 +1,9 @@
 .PHONY: default
 default:
-	TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/snapshots/00000000.snapshot.lz4
-	TZ=UTC touch -ct 200001020000 replica/generations/0000000000000000/snapshots/00000001.snapshot.lz4
-	TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/wal/00000000/00000000.wal.lz4
-	TZ=UTC touch -ct 200001020000 replica/generations/0000000000000000/wal/00000000/00000001.wal.lz4
-	TZ=UTC touch -ct 200001030000 replica/generations/0000000000000000/wal/00000001/00000000.wal.lz4
-	TZ=UTC touch -ct 200001020000 replica/generations/0000000000000001/snapshots/00000000.snapshot.lz4
+	TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/snapshots/0000000000000000.snapshot.lz4
+	TZ=UTC touch -ct 200001020000 replica/generations/0000000000000000/snapshots/0000000000000001.snapshot.lz4
+	TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/wal/0000000000000000/0000000000000000.wal.lz4
+	TZ=UTC touch -ct 200001020000 replica/generations/0000000000000000/wal/0000000000000000/0000000000000001.wal.lz4
+	TZ=UTC touch -ct 200001030000 replica/generations/0000000000000000/wal/0000000000000001/0000000000000000.wal.lz4
+	TZ=UTC touch -ct 200001020000 replica/generations/0000000000000001/snapshots/0000000000000000.snapshot.lz4
 

cmd/litestream/testdata/restore/latest-replica/Makefile

@@ -1,6 +1,6 @@
 .PHONY: default
 default:
-	TZ=UTC touch -ct 200001010000 replica0/generations/0000000000000000/snapshots/00000000.snapshot.lz4
-	TZ=UTC touch -ct 200001020000 replica1/generations/0000000000000002/snapshots/00000000.snapshot.lz4
-	TZ=UTC touch -ct 200001030000 replica0/generations/0000000000000001/snapshots/00000000.snapshot.lz4
+	TZ=UTC touch -ct 200001010000 replica0/generations/0000000000000000/snapshots/0000000000000000.snapshot.lz4
+	TZ=UTC touch -ct 200001020000 replica1/generations/0000000000000002/snapshots/0000000000000000.snapshot.lz4
+	TZ=UTC touch -ct 200001030000 replica0/generations/0000000000000001/snapshots/0000000000000000.snapshot.lz4
 

cmd/litestream/testdata/restore/ok/README

@@ -5,9 +5,9 @@ To reproduce this testdata, run sqlite3 and execute:
 	INSERT INTO t (x) VALUES (1);
 	INSERT INTO t (x) VALUES (2);
 
-	sl3 split -o generations/0000000000000000/wal/00000000 db-wal
-	cp db generations/0000000000000000/snapshots/00000000.snapshot
-	lz4 -c --rm generations/0000000000000000/snapshots/00000000.snapshot
+	sl3 split -o generations/0000000000000000/wal/0000000000000000 db-wal
+	cp db generations/0000000000000000/snapshots/0000000000000000.snapshot
+	lz4 -c --rm generations/0000000000000000/snapshots/0000000000000000.snapshot
 
 
 Then execute:
@@ -15,7 +15,7 @@ Then execute:
 	PRAGMA wal_checkpoint(TRUNCATE);
 	INSERT INTO t (x) VALUES (3);
 
-	sl3 split -o generations/0000000000000000/wal/00000001 db-wal
+	sl3 split -o generations/0000000000000000/wal/0000000000000001 db-wal
 
 
 Then execute:
@@ -24,13 +24,13 @@ Then execute:
 	INSERT INTO t (x) VALUES (4);
 	INSERT INTO t (x) VALUES (5);
 
-	sl3 split -o generations/0000000000000000/wal/00000002 db-wal
+	sl3 split -o generations/0000000000000000/wal/0000000000000002 db-wal
 
 
 Finally, obtain the final snapshot:
 
 	PRAGMA wal_checkpoint(TRUNCATE);
 
-	cp db 00000002.db
+	cp db 0000000000000002.db
 	rm db*
 

cmd/litestream/testdata/snapshots/ok/Makefile

@@ -1,6 +1,6 @@
 .PHONY: default
 default:
-	TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/snapshots/00000000.snapshot.lz4
-	TZ=UTC touch -ct 200001020000 replica/generations/0000000000000000/snapshots/00000001.snapshot.lz4
-	TZ=UTC touch -ct 200001030000 replica/generations/0000000000000001/snapshots/00000000.snapshot.lz4
+	TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/snapshots/0000000000000000.snapshot.lz4
+	TZ=UTC touch -ct 200001020000 replica/generations/0000000000000000/snapshots/0000000000000001.snapshot.lz4
+	TZ=UTC touch -ct 200001030000 replica/generations/0000000000000001/snapshots/0000000000000000.snapshot.lz4
 

cmd/litestream/testdata/snapshots/ok/stdout

@@ -1,4 +1,4 @@
-replica  generation        index     size  created
-file     0000000000000001  00000000  93    2000-01-03T00:00:00Z
-file     0000000000000000  00000001  93    2000-01-02T00:00:00Z
-file     0000000000000000  00000000  93    2000-01-01T00:00:00Z
+replica  generation        index             size  created
+file     0000000000000001  0000000000000000  93    2000-01-03T00:00:00Z
+file     0000000000000000  0000000000000001  93    2000-01-02T00:00:00Z
+file     0000000000000000  0000000000000000  93    2000-01-01T00:00:00Z

cmd/litestream/testdata/snapshots/replica-name/Makefile

@@ -1,4 +1,4 @@
 .PHONY: default
 default:
-	TZ=UTC touch -ct 200001010000 replica0/generations/0000000000000000/snapshots/00000000.snapshot.lz4
-	TZ=UTC touch -ct 200001020000 replica1/generations/0000000000000001/snapshots/00000000.snapshot.lz4
+	TZ=UTC touch -ct 200001010000 replica0/generations/0000000000000000/snapshots/0000000000000000.snapshot.lz4
+	TZ=UTC touch -ct 200001020000 replica1/generations/0000000000000001/snapshots/0000000000000000.snapshot.lz4

cmd/litestream/testdata/snapshots/replica-name/stdout

@@ -1,2 +1,2 @@
-replica   generation        index     size  created
-replica1  0000000000000001  00000000  93    2000-01-02T00:00:00Z
+replica   generation        index             size  created
+replica1  0000000000000001  0000000000000000  93    2000-01-02T00:00:00Z

cmd/litestream/testdata/snapshots/replica-url/Makefile

@@ -1,5 +1,5 @@
 .PHONY: default
 default:
-	TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/snapshots/00000000.snapshot.lz4
-	TZ=UTC touch -ct 200001020000 replica/generations/0000000000000000/snapshots/00000001.snapshot.lz4
-	TZ=UTC touch -ct 200001030000 replica/generations/0000000000000001/snapshots/00000000.snapshot.lz4
+	TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/snapshots/0000000000000000.snapshot.lz4
+	TZ=UTC touch -ct 200001020000 replica/generations/0000000000000000/snapshots/0000000000000001.snapshot.lz4
+	TZ=UTC touch -ct 200001030000 replica/generations/0000000000000001/snapshots/0000000000000000.snapshot.lz4

cmd/litestream/testdata/snapshots/replica-url/stdout

@@ -1,4 +1,4 @@
-replica  generation        index     size  created
-file     0000000000000001  00000000  93    2000-01-03T00:00:00Z
-file     0000000000000000  00000001  93    2000-01-02T00:00:00Z
-file     0000000000000000  00000000  93    2000-01-01T00:00:00Z
+replica  generation        index             size  created
+file     0000000000000001  0000000000000000  93    2000-01-03T00:00:00Z
+file     0000000000000000  0000000000000001  93    2000-01-02T00:00:00Z
+file     0000000000000000  0000000000000000  93    2000-01-01T00:00:00Z

cmd/litestream/testdata/wal/ok/Makefile

@@ -1,7 +1,7 @@
 .PHONY: default
 default:
-	TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/wal/00000000/00000000.wal.lz4
-	TZ=UTC touch -ct 200001020000 replica/generations/0000000000000000/wal/00000000/00000001.wal.lz4
-	TZ=UTC touch -ct 200001030000 replica/generations/0000000000000000/wal/00000001/00000000.wal.lz4
-	TZ=UTC touch -ct 200001040000 replica/generations/0000000000000001/wal/00000000/00000000.wal.lz4
+	TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/wal/0000000000000000/0000000000000000.wal.lz4
+	TZ=UTC touch -ct 200001020000 replica/generations/0000000000000000/wal/0000000000000000/0000000000000001.wal.lz4
+	TZ=UTC touch -ct 200001030000 replica/generations/0000000000000000/wal/0000000000000001/0000000000000000.wal.lz4
+	TZ=UTC touch -ct 200001040000 replica/generations/0000000000000001/wal/0000000000000000/0000000000000000.wal.lz4
 

cmd/litestream/testdata/wal/ok/stdout

@@ -1,5 +1,5 @@
-replica  generation        index     offset    size  created
-file     0000000000000001  00000000  00000000  93    2000-01-04T00:00:00Z
-file     0000000000000000  00000001  00000000  93    2000-01-03T00:00:00Z
-file     0000000000000000  00000000  00000001  93    2000-01-02T00:00:00Z
-file     0000000000000000  00000000  00000000  93    2000-01-01T00:00:00Z
+replica  generation        index             offset            size  created
+file     0000000000000001  0000000000000000  0000000000000000  93    2000-01-04T00:00:00Z
+file     0000000000000000  0000000000000001  0000000000000000  93    2000-01-03T00:00:00Z
+file     0000000000000000  0000000000000000  0000000000000001  93    2000-01-02T00:00:00Z
+file     0000000000000000  0000000000000000  0000000000000000  93    2000-01-01T00:00:00Z

cmd/litestream/testdata/wal/replica-name/Makefile

@@ -1,6 +1,6 @@
 .PHONY: default
 default:
-	TZ=UTC touch -ct 200001010000 replica0/generations/0000000000000000/wal/00000000/00000000.wal.lz4
-	TZ=UTC touch -ct 200001020000 replica1/generations/0000000000000000/wal/00000000/00000001.wal.lz4
-	TZ=UTC touch -ct 200001030000 replica1/generations/0000000000000000/wal/00000001/00000000.wal.lz4
-	TZ=UTC touch -ct 200001040000 replica1/generations/0000000000000001/wal/00000000/00000000.wal.lz4
+	TZ=UTC touch -ct 200001010000 replica0/generations/0000000000000000/wal/0000000000000000/0000000000000000.wal.lz4
+	TZ=UTC touch -ct 200001020000 replica1/generations/0000000000000000/wal/0000000000000000/0000000000000001.wal.lz4
+	TZ=UTC touch -ct 200001030000 replica1/generations/0000000000000000/wal/0000000000000001/0000000000000000.wal.lz4
+	TZ=UTC touch -ct 200001040000 replica1/generations/0000000000000001/wal/0000000000000000/0000000000000000.wal.lz4

cmd/litestream/testdata/wal/replica-name/stdout

@@ -1,2 +1,2 @@
-replica   generation        index     offset    size  created
-replica1  0000000000000001  00000000  00000000  93    2000-01-04T00:00:00Z
+replica   generation        index             offset            size  created
+replica1  0000000000000001  0000000000000000  0000000000000000  93    2000-01-04T00:00:00Z

cmd/litestream/testdata/wal/replica-url/Makefile

@@ -1,7 +1,7 @@
 .PHONY: default
 default:
-	TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/wal/00000000/00000000.wal.lz4
-	TZ=UTC touch -ct 200001020000 replica/generations/0000000000000000/wal/00000000/00000001.wal.lz4
-	TZ=UTC touch -ct 200001030000 replica/generations/0000000000000000/wal/00000001/00000000.wal.lz4
-	TZ=UTC touch -ct 200001040000 replica/generations/0000000000000001/wal/00000000/00000000.wal.lz4
+	TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/wal/0000000000000000/0000000000000000.wal.lz4
+	TZ=UTC touch -ct 200001020000 replica/generations/0000000000000000/wal/0000000000000000/0000000000000001.wal.lz4
+	TZ=UTC touch -ct 200001030000 replica/generations/0000000000000000/wal/0000000000000001/0000000000000000.wal.lz4
+	TZ=UTC touch -ct 200001040000 replica/generations/0000000000000001/wal/0000000000000000/0000000000000000.wal.lz4
 

cmd/litestream/testdata/wal/replica-url/stdout

@@ -1,5 +1,5 @@
-replica  generation        index     offset    size  created
-file     0000000000000001  00000000  00000000  93    2000-01-04T00:00:00Z
-file     0000000000000000  00000001  00000000  93    2000-01-03T00:00:00Z
-file     0000000000000000  00000000  00000001  93    2000-01-02T00:00:00Z
-file     0000000000000000  00000000  00000000  93    2000-01-01T00:00:00Z
+replica  generation        index             offset            size  created
+file     0000000000000001  0000000000000000  0000000000000000  93    2000-01-04T00:00:00Z
+file     0000000000000000  0000000000000001  0000000000000000  93    2000-01-03T00:00:00Z
+file     0000000000000000  0000000000000000  0000000000000001  93    2000-01-02T00:00:00Z
+file     0000000000000000  0000000000000000  0000000000000000  93    2000-01-01T00:00:00Z

cmd/litestream/wal.go

@@ -108,11 +108,11 @@ func (c *WALCommand) Run(ctx context.Context, args []string) (ret error) {
 
 	fmt.Fprintln(w, "replica\tgeneration\tindex\toffset\tsize\tcreated")
 	for _, info := range infos {
-		fmt.Fprintf(w, "%s\t%s\t%08x\t%08x\t%d\t%s\n",
+		fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%d\t%s\n",
 			info.replicaName,
 			info.Generation,
-			info.Index,
-			info.Offset,
+			litestream.FormatIndex(info.Index),
+			litestream.FormatOffset(info.Offset),
 			info.Size,
 			info.CreatedAt.Format(time.RFC3339),
 		)