Update index & offset encoding

Previously, the index & offsets were encoded as 8-character hex
strings, however, this limits the maximum value to a `uint32`. This
is normally not an issue, however, indices could go over the maximum
value of 4 billion over time and the offset could exceed this value
for an especially large WAL update. For safety, these encodings have
been updated to 16-character hex encodings.

abs/replica_client.go

@@ -190,8 +190,6 @@ func (c *ReplicaClient) WriteSnapshot(ctx context.Context, generation string, in
 	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "PUT").Inc()
 	internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "PUT").Add(float64(rc.N()))
 
-	// log.Printf("%s(%s): snapshot: creating %s/%08x t=%s", r.db.Path(), r.Name(), generation, index, time.Since(startTime).Truncate(time.Millisecond))
-
 	return litestream.SnapshotInfo{
 		Generation: generation,
 		Index:      index,

cmd/litestream/main.go

@@ -718,7 +718,7 @@ var _ flag.Value = (*indexVar)(nil)
 
 // String returns an 8-character hexadecimal value.
 func (v *indexVar) String() string {
-	return fmt.Sprintf("%08x", int(*v))
+	return litestream.FormatIndex(int(*v))
 }
 
 // Set parses s into an integer from a hexadecimal value.

cmd/litestream/restore_test.go

@@ -27,10 +27,10 @@ func TestRestoreCommand(t *testing.T) {
 		// STDOUT has timing info so we need to grep per line.
 		lines := strings.Split(stdout.String(), "\n")
 		for i, substr := range []string{
-			`restoring snapshot 0000000000000000/00000000 to ` + filepath.Join(tempDir, "db.tmp"),
-			`applied wal 0000000000000000/00000000 elapsed=`,
-			`applied wal 0000000000000000/00000001 elapsed=`,
-			`applied wal 0000000000000000/00000002 elapsed=`,
+			`restoring snapshot 0000000000000000/0000000000000000 to ` + filepath.Join(tempDir, "db.tmp"),
+			`applied wal 0000000000000000/0000000000000000 elapsed=`,
+			`applied wal 0000000000000000/0000000000000001 elapsed=`,
+			`applied wal 0000000000000000/0000000000000002 elapsed=`,
 			`renaming database from temporary location`,
 		} {
 			if !strings.Contains(lines[i], substr) {
@@ -54,7 +54,7 @@ func TestRestoreCommand(t *testing.T) {
 		// STDOUT has timing info so we need to grep per line.
 		lines := strings.Split(stdout.String(), "\n")
 		for i, substr := range []string{
-			`restoring snapshot 0000000000000001/00000001 to ` + filepath.Join(tempDir, "db.tmp"),
+			`restoring snapshot 0000000000000001/0000000000000001 to ` + filepath.Join(tempDir, "db.tmp"),
 			`no wal files found, snapshot only`,
 			`renaming database from temporary location`,
 		} {
@@ -78,7 +78,7 @@ func TestRestoreCommand(t *testing.T) {
 
 		lines := strings.Split(stdout.String(), "\n")
 		for i, substr := range []string{
-			`restoring snapshot 0000000000000000/00000000 to ` + filepath.Join(tempDir, "db.tmp"),
+			`restoring snapshot 0000000000000000/0000000000000000 to ` + filepath.Join(tempDir, "db.tmp"),
 			`no wal files found, snapshot only`,
 			`renaming database from temporary location`,
 		} {
@@ -102,7 +102,7 @@ func TestRestoreCommand(t *testing.T) {
 
 		lines := strings.Split(stdout.String(), "\n")
 		for i, substr := range []string{
-			`restoring snapshot 0000000000000001/00000000 to ` + filepath.Join(tempDir, "db.tmp"),
+			`restoring snapshot 0000000000000001/0000000000000000 to ` + filepath.Join(tempDir, "db.tmp"),
 			`no wal files found, snapshot only`,
 			`renaming database from temporary location`,
 		} {
@@ -256,7 +256,7 @@ func TestRestoreCommand(t *testing.T) {
 
 	t.Run("ErrInvalidReplicaURL", func(t *testing.T) {
 		m, _, _, _ := newMain()
-		err := m.Run(context.Background(), []string{"restore", "-o", "/tmp/db", "xyz://xyz"})
+		err := m.Run(context.Background(), []string{"restore", "-o", filepath.Join(t.TempDir(), "db"), "xyz://xyz"})
 		if err == nil || err.Error() != `unknown replica type in config: "xyz"` {
 			t.Fatalf("unexpected error: %s", err)
 		}

cmd/litestream/snapshots.go

@@ -83,10 +83,10 @@ func (c *SnapshotsCommand) Run(ctx context.Context, args []string) (ret error) {
 
 	fmt.Fprintln(w, "replica\tgeneration\tindex\tsize\tcreated")
 	for _, info := range infos {
-		fmt.Fprintf(w, "%s\t%s\t%08x\t%d\t%s\n",
+		fmt.Fprintf(w, "%s\t%s\t%s\t%d\t%s\n",
 			info.replicaName,
 			info.Generation,
-			info.Index,
+			litestream.FormatIndex(info.Index),
 			info.Size,
 			info.CreatedAt.Format(time.RFC3339),
 		)

cmd/litestream/testdata/generations/no-database/Makefile

@@ -1,4 +1,4 @@
 .PHONY: default
 default:
-	TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/snapshots/00000000.snapshot.lz4
-	TZ=UTC touch -ct 200001020000 replica/generations/0000000000000001/snapshots/00000000.snapshot.lz4
+	TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/snapshots/0000000000000000.snapshot.lz4
+	TZ=UTC touch -ct 200001020000 replica/generations/0000000000000001/snapshots/0000000000000000.snapshot.lz4

cmd/litestream/testdata/generations/ok/Makefile

@@ -1,9 +1,9 @@
 .PHONY: default
 default:
 	TZ=UTC touch -ct 200001030000 db
-	TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/snapshots/00000000.snapshot.lz4
-	TZ=UTC touch -ct 200001020000 replica/generations/0000000000000000/snapshots/00000001.snapshot.lz4
-	TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/wal/00000000/00000000.wal.lz4
-	TZ=UTC touch -ct 200001020000 replica/generations/0000000000000000/wal/00000000/00000001.wal.lz4
-	TZ=UTC touch -ct 200001030000 replica/generations/0000000000000000/wal/00000001/00000000.wal.lz4
-	TZ=UTC touch -ct 200001010000 replica/generations/0000000000000001/snapshots/00000000.snapshot.lz4
+	TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/snapshots/0000000000000000.snapshot.lz4
+	TZ=UTC touch -ct 200001020000 replica/generations/0000000000000000/snapshots/0000000000000001.snapshot.lz4
+	TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/wal/0000000000000000/0000000000000000.wal.lz4
+	TZ=UTC touch -ct 200001020000 replica/generations/0000000000000000/wal/0000000000000000/0000000000000001.wal.lz4
+	TZ=UTC touch -ct 200001030000 replica/generations/0000000000000000/wal/0000000000000001/0000000000000000.wal.lz4
+	TZ=UTC touch -ct 200001010000 replica/generations/0000000000000001/snapshots/0000000000000000.snapshot.lz4

cmd/litestream/testdata/generations/replica-name/Makefile

@@ -1,5 +1,5 @@
 .PHONY: default
 default:
 	TZ=UTC touch -ct 200001030000 db
-	TZ=UTC touch -ct 200001010000 replica0/generations/0000000000000000/snapshots/00000000.snapshot.lz4
-	TZ=UTC touch -ct 200001020000 replica1/generations/0000000000000001/snapshots/00000000.snapshot.lz4
+	TZ=UTC touch -ct 200001010000 replica0/generations/0000000000000000/snapshots/0000000000000000.snapshot.lz4
+	TZ=UTC touch -ct 200001020000 replica1/generations/0000000000000001/snapshots/0000000000000000.snapshot.lz4

cmd/litestream/testdata/generations/replica-url/Makefile

@@ -1,9 +1,9 @@
 .PHONY: default
 default:
-	TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/snapshots/00000000.snapshot.lz4
-	TZ=UTC touch -ct 200001020000 replica/generations/0000000000000000/snapshots/00000001.snapshot.lz4
-	TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/wal/00000000/00000000.wal.lz4
-	TZ=UTC touch -ct 200001020000 replica/generations/0000000000000000/wal/00000000/00000001.wal.lz4
-	TZ=UTC touch -ct 200001030000 replica/generations/0000000000000000/wal/00000001/00000000.wal.lz4
-	TZ=UTC touch -ct 200001020000 replica/generations/0000000000000001/snapshots/00000000.snapshot.lz4
+	TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/snapshots/0000000000000000.snapshot.lz4
+	TZ=UTC touch -ct 200001020000 replica/generations/0000000000000000/snapshots/0000000000000001.snapshot.lz4
+	TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/wal/0000000000000000/0000000000000000.wal.lz4
+	TZ=UTC touch -ct 200001020000 replica/generations/0000000000000000/wal/0000000000000000/0000000000000001.wal.lz4
+	TZ=UTC touch -ct 200001030000 replica/generations/0000000000000000/wal/0000000000000001/0000000000000000.wal.lz4
+	TZ=UTC touch -ct 200001020000 replica/generations/0000000000000001/snapshots/0000000000000000.snapshot.lz4
 

cmd/litestream/testdata/restore/latest-replica/Makefile

@@ -1,6 +1,6 @@
 .PHONY: default
 default:
-	TZ=UTC touch -ct 200001010000 replica0/generations/0000000000000000/snapshots/00000000.snapshot.lz4
-	TZ=UTC touch -ct 200001020000 replica1/generations/0000000000000002/snapshots/00000000.snapshot.lz4
-	TZ=UTC touch -ct 200001030000 replica0/generations/0000000000000001/snapshots/00000000.snapshot.lz4
+	TZ=UTC touch -ct 200001010000 replica0/generations/0000000000000000/snapshots/0000000000000000.snapshot.lz4
+	TZ=UTC touch -ct 200001020000 replica1/generations/0000000000000002/snapshots/0000000000000000.snapshot.lz4
+	TZ=UTC touch -ct 200001030000 replica0/generations/0000000000000001/snapshots/0000000000000000.snapshot.lz4
 

cmd/litestream/testdata/restore/ok/README

@@ -5,9 +5,9 @@ To reproduce this testdata, run sqlite3 and execute:
 	INSERT INTO t (x) VALUES (1);
 	INSERT INTO t (x) VALUES (2);
 
-	sl3 split -o generations/0000000000000000/wal/00000000 db-wal
-	cp db generations/0000000000000000/snapshots/00000000.snapshot
-	lz4 -c --rm generations/0000000000000000/snapshots/00000000.snapshot
+	sl3 split -o generations/0000000000000000/wal/0000000000000000 db-wal
+	cp db generations/0000000000000000/snapshots/0000000000000000.snapshot
+	lz4 -c --rm generations/0000000000000000/snapshots/0000000000000000.snapshot
 
 
 Then execute:
@@ -15,7 +15,7 @@ Then execute:
 	PRAGMA wal_checkpoint(TRUNCATE);
 	INSERT INTO t (x) VALUES (3);
 
-	sl3 split -o generations/0000000000000000/wal/00000001 db-wal
+	sl3 split -o generations/0000000000000000/wal/0000000000000001 db-wal
 
 
 Then execute:
@@ -24,13 +24,13 @@ Then execute:
 	INSERT INTO t (x) VALUES (4);
 	INSERT INTO t (x) VALUES (5);
 
-	sl3 split -o generations/0000000000000000/wal/00000002 db-wal
+	sl3 split -o generations/0000000000000000/wal/0000000000000002 db-wal
 
 
 Finally, obtain the final snapshot:
 
 	PRAGMA wal_checkpoint(TRUNCATE);
 
-	cp db 00000002.db
+	cp db 0000000000000002.db
 	rm db*
 

cmd/litestream/testdata/snapshots/ok/Makefile

@@ -1,6 +1,6 @@
 .PHONY: default
 default:
-	TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/snapshots/00000000.snapshot.lz4
-	TZ=UTC touch -ct 200001020000 replica/generations/0000000000000000/snapshots/00000001.snapshot.lz4
-	TZ=UTC touch -ct 200001030000 replica/generations/0000000000000001/snapshots/00000000.snapshot.lz4
+	TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/snapshots/0000000000000000.snapshot.lz4
+	TZ=UTC touch -ct 200001020000 replica/generations/0000000000000000/snapshots/0000000000000001.snapshot.lz4
+	TZ=UTC touch -ct 200001030000 replica/generations/0000000000000001/snapshots/0000000000000000.snapshot.lz4
 

cmd/litestream/testdata/snapshots/ok/stdout

@@ -1,4 +1,4 @@
-replica  generation        index     size  created
-file     0000000000000001  00000000  93    2000-01-03T00:00:00Z
-file     0000000000000000  00000001  93    2000-01-02T00:00:00Z
-file     0000000000000000  00000000  93    2000-01-01T00:00:00Z
+replica  generation        index             size  created
+file     0000000000000001  0000000000000000  93    2000-01-03T00:00:00Z
+file     0000000000000000  0000000000000001  93    2000-01-02T00:00:00Z
+file     0000000000000000  0000000000000000  93    2000-01-01T00:00:00Z

cmd/litestream/testdata/snapshots/replica-name/Makefile

@@ -1,4 +1,4 @@
 .PHONY: default
 default:
-	TZ=UTC touch -ct 200001010000 replica0/generations/0000000000000000/snapshots/00000000.snapshot.lz4
-	TZ=UTC touch -ct 200001020000 replica1/generations/0000000000000001/snapshots/00000000.snapshot.lz4
+	TZ=UTC touch -ct 200001010000 replica0/generations/0000000000000000/snapshots/0000000000000000.snapshot.lz4
+	TZ=UTC touch -ct 200001020000 replica1/generations/0000000000000001/snapshots/0000000000000000.snapshot.lz4

cmd/litestream/testdata/snapshots/replica-name/stdout

@@ -1,2 +1,2 @@
-replica   generation        index     size  created
-replica1  0000000000000001  00000000  93    2000-01-02T00:00:00Z
+replica   generation        index             size  created
+replica1  0000000000000001  0000000000000000  93    2000-01-02T00:00:00Z

cmd/litestream/testdata/snapshots/replica-url/Makefile

@@ -1,5 +1,5 @@
 .PHONY: default
 default:
-	TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/snapshots/00000000.snapshot.lz4
-	TZ=UTC touch -ct 200001020000 replica/generations/0000000000000000/snapshots/00000001.snapshot.lz4
-	TZ=UTC touch -ct 200001030000 replica/generations/0000000000000001/snapshots/00000000.snapshot.lz4
+	TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/snapshots/0000000000000000.snapshot.lz4
+	TZ=UTC touch -ct 200001020000 replica/generations/0000000000000000/snapshots/0000000000000001.snapshot.lz4
+	TZ=UTC touch -ct 200001030000 replica/generations/0000000000000001/snapshots/0000000000000000.snapshot.lz4

cmd/litestream/testdata/snapshots/replica-url/stdout

@@ -1,4 +1,4 @@
-replica  generation        index     size  created
-file     0000000000000001  00000000  93    2000-01-03T00:00:00Z
-file     0000000000000000  00000001  93    2000-01-02T00:00:00Z
-file     0000000000000000  00000000  93    2000-01-01T00:00:00Z
+replica  generation        index             size  created
+file     0000000000000001  0000000000000000  93    2000-01-03T00:00:00Z
+file     0000000000000000  0000000000000001  93    2000-01-02T00:00:00Z
+file     0000000000000000  0000000000000000  93    2000-01-01T00:00:00Z

cmd/litestream/testdata/wal/ok/Makefile

@@ -1,7 +1,7 @@
 .PHONY: default
 default:
-	TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/wal/00000000/00000000.wal.lz4
-	TZ=UTC touch -ct 200001020000 replica/generations/0000000000000000/wal/00000000/00000001.wal.lz4
-	TZ=UTC touch -ct 200001030000 replica/generations/0000000000000000/wal/00000001/00000000.wal.lz4
-	TZ=UTC touch -ct 200001040000 replica/generations/0000000000000001/wal/00000000/00000000.wal.lz4
+	TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/wal/0000000000000000/0000000000000000.wal.lz4
+	TZ=UTC touch -ct 200001020000 replica/generations/0000000000000000/wal/0000000000000000/0000000000000001.wal.lz4
+	TZ=UTC touch -ct 200001030000 replica/generations/0000000000000000/wal/0000000000000001/0000000000000000.wal.lz4
+	TZ=UTC touch -ct 200001040000 replica/generations/0000000000000001/wal/0000000000000000/0000000000000000.wal.lz4
 

cmd/litestream/testdata/wal/ok/stdout

@@ -1,5 +1,5 @@
-replica  generation        index     offset    size  created
-file     0000000000000001  00000000  00000000  93    2000-01-04T00:00:00Z
-file     0000000000000000  00000001  00000000  93    2000-01-03T00:00:00Z
-file     0000000000000000  00000000  00000001  93    2000-01-02T00:00:00Z
-file     0000000000000000  00000000  00000000  93    2000-01-01T00:00:00Z
+replica  generation        index             offset            size  created
+file     0000000000000001  0000000000000000  0000000000000000  93    2000-01-04T00:00:00Z
+file     0000000000000000  0000000000000001  0000000000000000  93    2000-01-03T00:00:00Z
+file     0000000000000000  0000000000000000  0000000000000001  93    2000-01-02T00:00:00Z
+file     0000000000000000  0000000000000000  0000000000000000  93    2000-01-01T00:00:00Z

cmd/litestream/testdata/wal/replica-name/Makefile

@@ -1,6 +1,6 @@
 .PHONY: default
 default:
-	TZ=UTC touch -ct 200001010000 replica0/generations/0000000000000000/wal/00000000/00000000.wal.lz4
-	TZ=UTC touch -ct 200001020000 replica1/generations/0000000000000000/wal/00000000/00000001.wal.lz4
-	TZ=UTC touch -ct 200001030000 replica1/generations/0000000000000000/wal/00000001/00000000.wal.lz4
-	TZ=UTC touch -ct 200001040000 replica1/generations/0000000000000001/wal/00000000/00000000.wal.lz4
+	TZ=UTC touch -ct 200001010000 replica0/generations/0000000000000000/wal/0000000000000000/0000000000000000.wal.lz4
+	TZ=UTC touch -ct 200001020000 replica1/generations/0000000000000000/wal/0000000000000000/0000000000000001.wal.lz4
+	TZ=UTC touch -ct 200001030000 replica1/generations/0000000000000000/wal/0000000000000001/0000000000000000.wal.lz4
+	TZ=UTC touch -ct 200001040000 replica1/generations/0000000000000001/wal/0000000000000000/0000000000000000.wal.lz4

cmd/litestream/testdata/wal/replica-name/stdout

@@ -1,2 +1,2 @@
-replica   generation        index     offset    size  created
-replica1  0000000000000001  00000000  00000000  93    2000-01-04T00:00:00Z
+replica   generation        index             offset            size  created
+replica1  0000000000000001  0000000000000000  0000000000000000  93    2000-01-04T00:00:00Z

cmd/litestream/testdata/wal/replica-url/Makefile

@@ -1,7 +1,7 @@
 .PHONY: default
 default:
-	TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/wal/00000000/00000000.wal.lz4
-	TZ=UTC touch -ct 200001020000 replica/generations/0000000000000000/wal/00000000/00000001.wal.lz4
-	TZ=UTC touch -ct 200001030000 replica/generations/0000000000000000/wal/00000001/00000000.wal.lz4
-	TZ=UTC touch -ct 200001040000 replica/generations/0000000000000001/wal/00000000/00000000.wal.lz4
+	TZ=UTC touch -ct 200001010000 replica/generations/0000000000000000/wal/0000000000000000/0000000000000000.wal.lz4
+	TZ=UTC touch -ct 200001020000 replica/generations/0000000000000000/wal/0000000000000000/0000000000000001.wal.lz4
+	TZ=UTC touch -ct 200001030000 replica/generations/0000000000000000/wal/0000000000000001/0000000000000000.wal.lz4
+	TZ=UTC touch -ct 200001040000 replica/generations/0000000000000001/wal/0000000000000000/0000000000000000.wal.lz4
 

cmd/litestream/testdata/wal/replica-url/stdout

@@ -1,5 +1,5 @@
-replica  generation        index     offset    size  created
-file     0000000000000001  00000000  00000000  93    2000-01-04T00:00:00Z
-file     0000000000000000  00000001  00000000  93    2000-01-03T00:00:00Z
-file     0000000000000000  00000000  00000001  93    2000-01-02T00:00:00Z
-file     0000000000000000  00000000  00000000  93    2000-01-01T00:00:00Z
+replica  generation        index             offset            size  created
+file     0000000000000001  0000000000000000  0000000000000000  93    2000-01-04T00:00:00Z
+file     0000000000000000  0000000000000001  0000000000000000  93    2000-01-03T00:00:00Z
+file     0000000000000000  0000000000000000  0000000000000001  93    2000-01-02T00:00:00Z
+file     0000000000000000  0000000000000000  0000000000000000  93    2000-01-01T00:00:00Z

cmd/litestream/wal.go

@@ -108,11 +108,11 @@ func (c *WALCommand) Run(ctx context.Context, args []string) (ret error) {
 
 	fmt.Fprintln(w, "replica\tgeneration\tindex\toffset\tsize\tcreated")
 	for _, info := range infos {
-		fmt.Fprintf(w, "%s\t%s\t%08x\t%08x\t%d\t%s\n",
+		fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%d\t%s\n",
 			info.replicaName,
 			info.Generation,
-			info.Index,
-			info.Offset,
+			litestream.FormatIndex(info.Index),
+			litestream.FormatOffset(info.Offset),
 			info.Size,
 			info.CreatedAt.Format(time.RFC3339),
 		)

db.go

@@ -703,7 +703,7 @@ func (db *DB) cleanWAL(ctx context.Context) error {
 			return err
 		}
 
-		db.Logger.Printf("remove shadow index: %s/%08x", generation, index)
+		db.Logger.Printf("remove shadow index: %s/%s", generation, FormatIndex(index))
 	}
 
 	return nil

file_replica_client.go

@@ -100,7 +100,7 @@ func (c *FileReplicaClient) WALSegmentPath(generation string, index int, offset
 	if err != nil {
 		return "", err
 	}
-	return filepath.Join(dir, FormatIndex(index), fmt.Sprintf("%08x.wal.lz4", offset)), nil
+	return filepath.Join(dir, FormatIndex(index), fmt.Sprintf("%s.wal.lz4", FormatOffset(offset))), nil
 }
 
 // Generations returns a list of available generation names.

file_replica_client_test.go

@@ -78,7 +78,7 @@ func TestReplicaClient_SnapshotPath(t *testing.T) {
 	t.Run("OK", func(t *testing.T) {
 		if got, err := litestream.NewFileReplicaClient("/foo").SnapshotPath("0123456701234567", 1000); err != nil {
 			t.Fatal(err)
-		} else if want := "/foo/generations/0123456701234567/snapshots/000003e8.snapshot.lz4"; got != want {
+		} else if want := "/foo/generations/0123456701234567/snapshots/00000000000003e8.snapshot.lz4"; got != want {
 			t.Fatalf("SnapshotPath()=%v, want %v", got, want)
 		}
 	})
@@ -118,7 +118,7 @@ func TestReplicaClient_WALSegmentPath(t *testing.T) {
 	t.Run("OK", func(t *testing.T) {
 		if got, err := litestream.NewFileReplicaClient("/foo").WALSegmentPath("0123456701234567", 1000, 1001); err != nil {
 			t.Fatal(err)
-		} else if want := "/foo/generations/0123456701234567/wal/000003e8/000003e9.wal.lz4"; got != want {
+		} else if want := "/foo/generations/0123456701234567/wal/00000000000003e8/00000000000003e9.wal.lz4"; got != want {
 			t.Fatalf("WALPath()=%v, want %v", got, want)
 		}
 	})

gcs/replica_client.go

@@ -160,8 +160,6 @@ func (c *ReplicaClient) WriteSnapshot(ctx context.Context, generation string, in
 	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "PUT").Inc()
 	internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "PUT").Add(float64(n))
 
-	// log.Printf("%s(%s): snapshot: creating %s/%08x t=%s", r.db.Path(), r.Name(), generation, index, time.Since(startTime).Truncate(time.Millisecond))
-
 	return litestream.SnapshotInfo{
 		Generation: generation,
 		Index:      index,

internal/internal.go

@@ -15,6 +15,12 @@ import (
 	"github.com/prometheus/client_golang/prometheus/promauto"
 )
 
+// Platform-independent maximum integer sizes.
+const (
+	MaxUint = ^uint(0)
+	MaxInt  = int(MaxUint >> 1)
+)
+
 // ReadCloser wraps a reader to also attach a separate closer.
 type ReadCloser struct {
 	r io.Reader
@@ -170,14 +176,14 @@ func ParseSnapshotPath(s string) (index int, err error) {
 		return 0, fmt.Errorf("invalid snapshot path")
 	}
 
-	i32, _ := strconv.ParseUint(a[1], 16, 32)
-	if i32 > math.MaxInt32 {
+	i64, _ := strconv.ParseUint(a[1], 16, 64)
+	if i64 > uint64(MaxInt) {
 		return 0, fmt.Errorf("index too large in snapshot path %q", s)
 	}
-	return int(i32), nil
+	return int(i64), nil
 }
 
-var snapshotPathRegex = regexp.MustCompile(`^([0-9a-f]{8})\.snapshot\.lz4$`)
+var snapshotPathRegex = regexp.MustCompile(`^([0-9a-f]{16})\.snapshot\.lz4$`)
 
 // ParseWALSegmentPath parses the index/offset from a segment filename. Used by path-based replicas.
 func ParseWALSegmentPath(s string) (index int, offset int64, err error) {
@@ -186,18 +192,18 @@ func ParseWALSegmentPath(s string) (index int, offset int64, err error) {
 		return 0, 0, fmt.Errorf("invalid wal segment path")
 	}
 
-	i32, _ := strconv.ParseUint(a[1], 16, 32)
-	if i32 > math.MaxInt32 {
+	i64, _ := strconv.ParseUint(a[1], 16, 64)
+	if i64 > uint64(MaxInt) {
 		return 0, 0, fmt.Errorf("index too large in wal segment path %q", s)
 	}
 	off64, _ := strconv.ParseUint(a[2], 16, 64)
 	if off64 > math.MaxInt64 {
 		return 0, 0, fmt.Errorf("offset too large in wal segment path %q", s)
 	}
-	return int(i32), int64(off64), nil
+	return int(i64), int64(off64), nil
 }
 
-var walSegmentPathRegex = regexp.MustCompile(`^([0-9a-f]{8})\/([0-9a-f]{8})\.wal\.lz4$`)
+var walSegmentPathRegex = regexp.MustCompile(`^([0-9a-f]{16})\/([0-9a-f]{16})\.wal\.lz4$`)
 
 // Shared replica metrics.
 var (

internal/internal_test.go

@@ -15,12 +15,12 @@ func TestParseSnapshotPath(t *testing.T) {
 		index int
 		err   error
 	}{
-		{"00bc614e.snapshot.lz4", 12345678, nil},
-		{"xxxxxxxx.snapshot.lz4", 0, fmt.Errorf("invalid snapshot path")},
-		{"00bc614.snapshot.lz4", 0, fmt.Errorf("invalid snapshot path")},
-		{"00bc614e.snapshot.lz", 0, fmt.Errorf("invalid snapshot path")},
-		{"00bc614e.snapshot", 0, fmt.Errorf("invalid snapshot path")},
-		{"00bc614e", 0, fmt.Errorf("invalid snapshot path")},
+		{"0000000000bc614e.snapshot.lz4", 12345678, nil},
+		{"xxxxxxxxxxxxxxxx.snapshot.lz4", 0, fmt.Errorf("invalid snapshot path")},
+		{"0000000000bc614.snapshot.lz4", 0, fmt.Errorf("invalid snapshot path")},
+		{"0000000000bc614e.snapshot.lz", 0, fmt.Errorf("invalid snapshot path")},
+		{"0000000000bc614e.snapshot", 0, fmt.Errorf("invalid snapshot path")},
+		{"0000000000bc614e", 0, fmt.Errorf("invalid snapshot path")},
 		{"", 0, fmt.Errorf("invalid snapshot path")},
 	} {
 		t.Run("", func(t *testing.T) {
@@ -41,20 +41,22 @@ func TestParseWALSegmentPath(t *testing.T) {
 		offset int64
 		err    error
 	}{
-		{"00bc614e/000003e8.wal.lz4", 12345678, 1000, nil},
-		{"00000000/00000000.wal", 0, 0, fmt.Errorf("invalid wal segment path")},
-		{"00000000/00000000", 0, 0, fmt.Errorf("invalid wal segment path")},
-		{"00000000/", 0, 0, fmt.Errorf("invalid wal segment path")},
-		{"00000000", 0, 0, fmt.Errorf("invalid wal segment path")},
+		{"0000000000bc614e/00000000000003e8.wal.lz4", 12345678, 1000, nil},
+		{"0000000000000000/0000000000000000.wal", 0, 0, fmt.Errorf("invalid wal segment path")},
+		{"0000000000000000/0000000000000000", 0, 0, fmt.Errorf("invalid wal segment path")},
+		{"0000000000000000/", 0, 0, fmt.Errorf("invalid wal segment path")},
+		{"0000000000000000", 0, 0, fmt.Errorf("invalid wal segment path")},
 		{"", 0, 0, fmt.Errorf("invalid wal segment path")},
 	} {
 		t.Run("", func(t *testing.T) {
 			index, offset, err := internal.ParseWALSegmentPath(tt.s)
 			if got, want := index, tt.index; got != want {
 				t.Errorf("index=%#v, want %#v", got, want)
-			} else if got, want := offset, tt.offset; got != want {
+			}
+			if got, want := offset, tt.offset; got != want {
 				t.Errorf("offset=%#v, want %#v", got, want)
-			} else if got, want := err, tt.err; !reflect.DeepEqual(got, want) {
+			}
+			if got, want := err, tt.err; !reflect.DeepEqual(got, want) {
 				t.Errorf("err=%#v, want %#v", got, want)
 			}
 		})

litestream.go

@@ -6,12 +6,14 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"math"
 	"os"
 	"path/filepath"
 	"strconv"
 	"strings"
 	"time"
 
+	"github.com/benbjohnson/litestream/internal"
 	"github.com/mattn/go-sqlite3"
 )
 
@@ -302,7 +304,7 @@ func (p Pos) String() string {
 	if p.IsZero() {
 		return ""
 	}
-	return fmt.Sprintf("%s/%08x:%08x", p.Generation, p.Index, p.Offset)
+	return fmt.Sprintf("%s/%s:%s", p.Generation, FormatIndex(p.Index), FormatOffset(p.Offset))
 }
 
 // IsZero returns true if p is the zero value.
@@ -426,32 +428,36 @@ func IsGenerationName(s string) bool {
 	return true
 }
 
-// FormatIndex formats an index as an 8-character hex value.
+// FormatIndex formats an index as a hex value.
 func FormatIndex(index int) string {
-	return fmt.Sprintf("%08x", index)
+	return fmt.Sprintf("%016x", index)
 }
 
 // ParseIndex parses a hex-formatted index into an integer.
 func ParseIndex(s string) (int, error) {
-	v, err := strconv.ParseUint(s, 16, 32)
+	v, err := strconv.ParseUint(s, 16, 64)
 	if err != nil {
 		return -1, fmt.Errorf("cannot parse index: %q", s)
+	} else if v > uint64(internal.MaxInt) {
+		return -1, fmt.Errorf("index too large: %q", s)
 	}
 	return int(v), nil
 }
 
-// FormatOffset formats an offset as an 8-character hex value.
+// FormatOffset formats an offset as a hex value.
 func FormatOffset(offset int64) string {
-	return fmt.Sprintf("%08x", offset)
+	return fmt.Sprintf("%016x", offset)
 }
 
 // ParseOffset parses a hex-formatted offset into an integer.
 func ParseOffset(s string) (int64, error) {
-	v, err := strconv.ParseInt(s, 16, 32)
+	v, err := strconv.ParseUint(s, 16, 64)
 	if err != nil {
-		return -1, fmt.Errorf("cannot parse index: %q", s)
+		return -1, fmt.Errorf("cannot parse offset: %q", s)
+	} else if v > math.MaxInt64 {
+		return -1, fmt.Errorf("offset too large: %q", s)
 	}
-	return v, nil
+	return int64(v), nil
 }
 
 // removeDBFiles deletes the database and related files (journal, shm, wal).

replica.go

@@ -514,7 +514,7 @@ func (r *Replica) Snapshot(ctx context.Context) (info SnapshotInfo, err error) {
 		return info, err
 	}
 
-	r.Logger.Printf("snapshot written %s/%08x", pos.Generation, pos.Index)
+	r.Logger.Printf("snapshot written %s/%s", pos.Generation, FormatIndex(pos.Index))
 
 	return info, nil
 }
@@ -580,9 +580,9 @@ func (r *Replica) deleteSnapshotsBeforeIndex(ctx context.Context, generation str
 		}
 
 		if err := r.client.DeleteSnapshot(ctx, info.Generation, info.Index); err != nil {
-			return fmt.Errorf("delete snapshot %s/%08x: %w", info.Generation, info.Index, err)
+			return fmt.Errorf("delete snapshot %s/%s: %w", info.Generation, FormatIndex(info.Index), err)
 		}
-		r.Logger.Printf("snapshot deleted %s/%08x", generation, index)
+		r.Logger.Printf("snapshot deleted %s/%s", generation, FormatIndex(index))
 	}
 
 	return itr.Close()

replica_client.go

@@ -88,7 +88,7 @@ func FindSnapshotForIndex(ctx context.Context, client ReplicaClient, generation
 	if n == 0 {
 		return 0, ErrNoSnapshots
 	} else if snapshotIndex == -1 {
-		return 0, fmt.Errorf("no snapshots available at or before index %08x", index)
+		return 0, fmt.Errorf("no snapshots available at or before index %s", FormatIndex(index))
 	}
 	return snapshotIndex, nil
 }
@@ -349,7 +349,7 @@ func Restore(ctx context.Context, client ReplicaClient, filename, generation str
 
 	// Copy snapshot to output path.
 	tmpPath := filename + ".tmp"
-	logger.Printf("%srestoring snapshot %s/%08x to %s", opt.LogPrefix, generation, snapshotIndex, tmpPath)
+	logger.Printf("%srestoring snapshot %s/%s to %s", opt.LogPrefix, generation, FormatIndex(snapshotIndex), tmpPath)
 	if err := RestoreSnapshot(ctx, client, tmpPath, generation, snapshotIndex, opt.Mode, opt.Uid, opt.Gid); err != nil {
 		return fmt.Errorf("cannot restore snapshot: %w", err)
 	}
@@ -380,7 +380,7 @@ func Restore(ctx context.Context, client ReplicaClient, filename, generation str
 		if err = ApplyWAL(ctx, tmpPath, walPath); err != nil {
 			return fmt.Errorf("cannot apply wal: %w", err)
 		}
-		logger.Printf("%sapplied wal %s/%08x elapsed=%s", opt.LogPrefix, generation, walIndex, time.Since(startTime).String())
+		logger.Printf("%sapplied wal %s/%s elapsed=%s", opt.LogPrefix, generation, FormatIndex(walIndex), time.Since(startTime).String())
 	}
 
 	// Copy file to final location.

replica_client_test.go

@@ -19,7 +19,7 @@ func TestFindSnapshotForIndex(t *testing.T) {
 		if snapshotIndex, err := litestream.FindSnapshotForIndex(context.Background(), client, "0000000000000000", 0x000007d0); err != nil {
 			t.Fatal(err)
 		} else if got, want := snapshotIndex, 0x000003e8; got != want {
-			t.Fatalf("index=%08x, want %08x", got, want)
+			t.Fatalf("index=%s, want %s", litestream.FormatIndex(got), litestream.FormatIndex(want))
 		}
 	})
 
@@ -28,14 +28,14 @@ func TestFindSnapshotForIndex(t *testing.T) {
 		if snapshotIndex, err := litestream.FindSnapshotForIndex(context.Background(), client, "0000000000000000", 0x000003e8); err != nil {
 			t.Fatal(err)
 		} else if got, want := snapshotIndex, 0x000003e8; got != want {
-			t.Fatalf("index=%08x, want %08x", got, want)
+			t.Fatalf("index=%s, want %s", litestream.FormatIndex(got), litestream.FormatIndex(want))
 		}
 	})
 
 	t.Run("ErrNoSnapshotsBeforeIndex", func(t *testing.T) {
 		client := litestream.NewFileReplicaClient(filepath.Join("testdata", "find-snapshot-for-index", "no-snapshots-before-index"))
 		_, err := litestream.FindSnapshotForIndex(context.Background(), client, "0000000000000000", 0x000003e8)
-		if err == nil || err.Error() != `no snapshots available at or before index 000003e8` {
+		if err == nil || err.Error() != `no snapshots available at or before index 00000000000003e8` {
 			t.Fatalf("unexpected error: %#v", err)
 		}
 	})
@@ -499,7 +499,7 @@ func TestRestore(t *testing.T) {
 		client := litestream.NewFileReplicaClient(testDir)
 		if err := litestream.Restore(context.Background(), client, filepath.Join(tempDir, "db"), "0000000000000000", 0, 2, litestream.NewRestoreOptions()); err != nil {
 			t.Fatal(err)
-		} else if !fileEqual(t, filepath.Join(testDir, "00000002.db"), filepath.Join(tempDir, "db")) {
+		} else if !fileEqual(t, filepath.Join(testDir, "0000000000000002.db"), filepath.Join(tempDir, "db")) {
 			t.Fatalf("file mismatch")
 		}
 	})
@@ -511,7 +511,7 @@ func TestRestore(t *testing.T) {
 		client := litestream.NewFileReplicaClient(testDir)
 		if err := litestream.Restore(context.Background(), client, filepath.Join(tempDir, "db"), "0000000000000000", 0, 0, litestream.NewRestoreOptions()); err != nil {
 			t.Fatal(err)
-		} else if !fileEqual(t, filepath.Join(testDir, "00000000.db"), filepath.Join(tempDir, "db")) {
+		} else if !fileEqual(t, filepath.Join(testDir, "0000000000000000.db"), filepath.Join(tempDir, "db")) {
 			t.Fatalf("file mismatch")
 		}
 	})
@@ -525,7 +525,7 @@ func TestRestore(t *testing.T) {
 		opt.Parallelism = 0
 		if err := litestream.Restore(context.Background(), client, filepath.Join(tempDir, "db"), "0000000000000000", 0, 2, opt); err != nil {
 			t.Fatal(err)
-		} else if !fileEqual(t, filepath.Join(testDir, "00000002.db"), filepath.Join(tempDir, "db")) {
+		} else if !fileEqual(t, filepath.Join(testDir, "0000000000000002.db"), filepath.Join(tempDir, "db")) {
 			t.Fatalf("file mismatch")
 		}
 	})

s3/replica_client.go

@@ -253,8 +253,6 @@ func (c *ReplicaClient) WriteSnapshot(ctx context.Context, generation string, in
 	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "PUT").Inc()
 	internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "PUT").Add(float64(rc.N()))
 
-	// log.Printf("%s(%s): snapshot: creating %s/%08x t=%s", r.db.Path(), r.Name(), generation, index, time.Since(startTime).Truncate(time.Millisecond))
-
 	return litestream.SnapshotInfo{
 		Generation: generation,
 		Index:      index,

sftp/replica_client.go

@@ -282,8 +282,6 @@ func (c *ReplicaClient) WriteSnapshot(ctx context.Context, generation string, in
 	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "PUT").Inc()
 	internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "PUT").Add(float64(n))
 
-	// log.Printf("%s(%s): snapshot: creating %s/%08x t=%s", r.db.Path(), r.Name(), generation, index, time.Since(startTime).Truncate(time.Millisecond))
-
 	return litestream.SnapshotInfo{
 		Generation: generation,
 		Index:      index,

testdata/find-latest-generation/ok/Makefile

@@ -1,7 +1,7 @@
 .PHONY: default
 default:
-	TZ=UTC touch -ct 200001010000 generations/0000000000000000/snapshots/00000000.snapshot.lz4
-	TZ=UTC touch -ct 200001020000 generations/0000000000000000/snapshots/00000001.snapshot.lz4
-	TZ=UTC touch -ct 200001010000 generations/0000000000000001/snapshots/00000000.snapshot.lz4
-	TZ=UTC touch -ct 200001030000 generations/0000000000000001/snapshots/00000001.snapshot.lz4
-	TZ=UTC touch -ct 200001010000 generations/0000000000000002/snapshots/00000000.snapshot.lz4
+	TZ=UTC touch -ct 200001010000 generations/0000000000000000/snapshots/0000000000000000.snapshot.lz4
+	TZ=UTC touch -ct 200001020000 generations/0000000000000000/snapshots/0000000000000001.snapshot.lz4
+	TZ=UTC touch -ct 200001010000 generations/0000000000000001/snapshots/0000000000000000.snapshot.lz4
+	TZ=UTC touch -ct 200001030000 generations/0000000000000001/snapshots/0000000000000001.snapshot.lz4
+	TZ=UTC touch -ct 200001010000 generations/0000000000000002/snapshots/0000000000000000.snapshot.lz4