Update index & offset encoding

Previously, the index & offsets were encoded as 8-character hex
strings, however, this limits the maximum value to a `uint32`. This
is normally not an issue, however, indices could go over the maximum
value of 4 billion over time and the offset could exceed this value
for an especially large WAL update. For safety, these encodings have
been updated to 16-character hex encodings.
This commit is contained in:
Ben Johnson
2022-02-08 12:49:36 -07:00
parent 54f3b94d3f
commit 006e4b7155
189 changed files with 203 additions and 197 deletions

View File

@@ -1,7 +1,7 @@
.PHONY: default
default:
TZ=UTC touch -ct 200001010000 generations/0000000000000000/snapshots/00000000.snapshot.lz4
TZ=UTC touch -ct 200001020000 generations/0000000000000000/snapshots/00000001.snapshot.lz4
TZ=UTC touch -ct 200001010000 generations/0000000000000001/snapshots/00000000.snapshot.lz4
TZ=UTC touch -ct 200001030000 generations/0000000000000001/snapshots/00000001.snapshot.lz4
TZ=UTC touch -ct 200001010000 generations/0000000000000002/snapshots/00000000.snapshot.lz4
TZ=UTC touch -ct 200001010000 generations/0000000000000000/snapshots/0000000000000000.snapshot.lz4
TZ=UTC touch -ct 200001020000 generations/0000000000000000/snapshots/0000000000000001.snapshot.lz4
TZ=UTC touch -ct 200001010000 generations/0000000000000001/snapshots/0000000000000000.snapshot.lz4
TZ=UTC touch -ct 200001030000 generations/0000000000000001/snapshots/0000000000000001.snapshot.lz4
TZ=UTC touch -ct 200001010000 generations/0000000000000002/snapshots/0000000000000000.snapshot.lz4

View File

@@ -1,8 +1,8 @@
.PHONY: default
default:
TZ=UTC touch -ct 200001010000 generations/0000000000000000/snapshots/00000000.snapshot.lz4
TZ=UTC touch -ct 200001020000 generations/0000000000000000/snapshots/00000001.snapshot.lz4
TZ=UTC touch -ct 200001010000 generations/0000000000000000/wal/00000000/00000000.wal.lz4
TZ=UTC touch -ct 200001020000 generations/0000000000000000/wal/00000000/00000001.wal.lz4
TZ=UTC touch -ct 200001030000 generations/0000000000000000/wal/00000001/00000000.wal.lz4
TZ=UTC touch -ct 200001010000 generations/0000000000000000/snapshots/0000000000000000.snapshot.lz4
TZ=UTC touch -ct 200001020000 generations/0000000000000000/snapshots/0000000000000001.snapshot.lz4
TZ=UTC touch -ct 200001010000 generations/0000000000000000/wal/0000000000000000/0000000000000000.wal.lz4
TZ=UTC touch -ct 200001020000 generations/0000000000000000/wal/0000000000000000/0000000000000001.wal.lz4
TZ=UTC touch -ct 200001030000 generations/0000000000000000/wal/0000000000000001/0000000000000000.wal.lz4

View File

@@ -1,5 +1,5 @@
.PHONY: default
default:
TZ=UTC touch -ct 200001010000 generations/0000000000000000/snapshots/00000000.snapshot.lz4
TZ=UTC touch -ct 200001020000 generations/0000000000000000/snapshots/00000001.snapshot.lz4
TZ=UTC touch -ct 200001010000 generations/0000000000000000/snapshots/0000000000000000.snapshot.lz4
TZ=UTC touch -ct 200001020000 generations/0000000000000000/snapshots/0000000000000001.snapshot.lz4

View File

@@ -1,6 +1,6 @@
.PHONY: default
default:
TZ=UTC touch -ct 200001020000 generations/0000000000000000/snapshots/00000000.snapshot.lz4
TZ=UTC touch -ct 200001010000 generations/0000000000000001/snapshots/00000000.snapshot.lz4
TZ=UTC touch -ct 200001030000 generations/0000000000000001/snapshots/00000001.snapshot.lz4
TZ=UTC touch -ct 200001010000 generations/0000000000000002/snapshots/00000000.snapshot.lz4
TZ=UTC touch -ct 200001020000 generations/0000000000000000/snapshots/0000000000000000.snapshot.lz4
TZ=UTC touch -ct 200001010000 generations/0000000000000001/snapshots/0000000000000000.snapshot.lz4
TZ=UTC touch -ct 200001030000 generations/0000000000000001/snapshots/0000000000000001.snapshot.lz4
TZ=UTC touch -ct 200001010000 generations/0000000000000002/snapshots/0000000000000000.snapshot.lz4

View File

@@ -1,6 +1,6 @@
.PHONY: default
default:
TZ=UTC touch -ct 200001020000 generations/0000000000000000/snapshots/00000000.snapshot.lz4
TZ=UTC touch -ct 200001010000 generations/0000000000000001/snapshots/00000000.snapshot.lz4
TZ=UTC touch -ct 200001030000 generations/0000000000000001/snapshots/00000001.snapshot.lz4
TZ=UTC touch -ct 200001010000 generations/0000000000000002/snapshots/00000000.snapshot.lz4
TZ=UTC touch -ct 200001020000 generations/0000000000000000/snapshots/0000000000000000.snapshot.lz4
TZ=UTC touch -ct 200001010000 generations/0000000000000001/snapshots/0000000000000000.snapshot.lz4
TZ=UTC touch -ct 200001030000 generations/0000000000000001/snapshots/0000000000000001.snapshot.lz4
TZ=UTC touch -ct 200001010000 generations/0000000000000002/snapshots/0000000000000000.snapshot.lz4

View File

@@ -5,9 +5,9 @@ To reproduce this testdata, run sqlite3 and execute:
INSERT INTO t (x) VALUES (1);
INSERT INTO t (x) VALUES (2);
sl3 split -o generations/0000000000000000/wal/00000000 db-wal
cp db generations/0000000000000000/snapshots/00000000.snapshot
lz4 -c --rm generations/0000000000000000/snapshots/00000000.snapshot
sl3 split -o generations/0000000000000000/wal/0000000000000000 db-wal
cp db generations/0000000000000000/snapshots/0000000000000000.snapshot
lz4 -c --rm generations/0000000000000000/snapshots/0000000000000000.snapshot
Then execute:
@@ -15,7 +15,7 @@ Then execute:
PRAGMA wal_checkpoint(TRUNCATE);
INSERT INTO t (x) VALUES (3);
sl3 split -o generations/0000000000000000/wal/00000001 db-wal
sl3 split -o generations/0000000000000000/wal/0000000000000001 db-wal
Then execute:
@@ -24,13 +24,13 @@ Then execute:
INSERT INTO t (x) VALUES (4);
INSERT INTO t (x) VALUES (5);
sl3 split -o generations/0000000000000000/wal/00000002 db-wal
sl3 split -o generations/0000000000000000/wal/0000000000000002 db-wal
Finally, obtain the final snapshot:
PRAGMA wal_checkpoint(TRUNCATE);
cp db 00000002.db
cp db 0000000000000002.db
rm db*

View File

@@ -5,9 +5,9 @@ To reproduce this testdata, run sqlite3 and execute:
INSERT INTO t (x) VALUES (1);
INSERT INTO t (x) VALUES (2);
sl3 split -o generations/0000000000000000/wal/00000000 db-wal
cp db generations/0000000000000000/snapshots/00000000.snapshot
lz4 -c --rm generations/0000000000000000/snapshots/00000000.snapshot
sl3 split -o generations/0000000000000000/wal/0000000000000000 db-wal
cp db generations/0000000000000000/snapshots/0000000000000000.snapshot
lz4 -c --rm generations/0000000000000000/snapshots/0000000000000000.snapshot
Then execute:
@@ -15,7 +15,7 @@ Then execute:
PRAGMA wal_checkpoint(TRUNCATE);
INSERT INTO t (x) VALUES (3);
sl3 split -o generations/0000000000000000/wal/00000001 db-wal
sl3 split -o generations/0000000000000000/wal/0000000000000001 db-wal
Then execute:
@@ -24,13 +24,13 @@ Then execute:
INSERT INTO t (x) VALUES (4);
INSERT INTO t (x) VALUES (5);
sl3 split -o generations/0000000000000000/wal/00000002 db-wal
sl3 split -o generations/0000000000000000/wal/0000000000000002 db-wal
Finally, obtain the final snapshot:
PRAGMA wal_checkpoint(TRUNCATE);
cp db 00000002.db
cp db 0000000000000002.db
rm db*

View File

@@ -1,6 +1,6 @@
.PHONY: default
default:
TZ=UTC touch -ct 200001010000 generations/0000000000000000/snapshots/00000000.snapshot.lz4
TZ=UTC touch -ct 200001020000 generations/0000000000000000/snapshots/00000001.snapshot.lz4
TZ=UTC touch -ct 200001030000 generations/0000000000000000/snapshots/00000002.snapshot.lz4
TZ=UTC touch -ct 200001010000 generations/0000000000000000/snapshots/0000000000000000.snapshot.lz4
TZ=UTC touch -ct 200001020000 generations/0000000000000000/snapshots/0000000000000001.snapshot.lz4
TZ=UTC touch -ct 200001030000 generations/0000000000000000/snapshots/0000000000000002.snapshot.lz4

View File

@@ -7,8 +7,8 @@ To reproduce this testdata, run sqlite3 and execute:
And copy & split the WAL into segments:
sl3 split -o generations/0000000000000000/wal/00000000 db-wal
cp db-wal 00000000.wal
sl3 split -o generations/0000000000000000/wal/0000000000000000 db-wal
cp db-wal 0000000000000000.wal
Then execute:
@@ -18,8 +18,8 @@ Then execute:
And split again:
sl3 split -o generations/0000000000000000/wal/00000001 db-wal
cp db-wal 00000001.wal
sl3 split -o generations/0000000000000000/wal/0000000000000001 db-wal
cp db-wal 0000000000000001.wal
Then execute:
@@ -30,8 +30,8 @@ Then execute:
And split again:
sl3 split -o generations/0000000000000000/wal/00000002 db-wal
cp db-wal 00000002.wal
sl3 split -o generations/0000000000000000/wal/0000000000000002 db-wal
cp db-wal 0000000000000002.wal
Finally, remove the original database files:

View File

@@ -7,8 +7,8 @@ To reproduce this testdata, run sqlite3 and execute:
And copy & split the WAL into segments:
sl3 split -o generations/0000000000000000/wal/00000000 db-wal
cp db-wal 00000000.wal
sl3 split -o generations/0000000000000000/wal/0000000000000000 db-wal
cp db-wal 0000000000000000.wal
Finally, remove the original database files:

View File

@@ -1,6 +1,6 @@
.PHONY: default
default:
TZ=UTC touch -ct 200001010000 generations/0000000000000000/wal/00000000/00000000.wal.lz4
TZ=UTC touch -ct 200001020000 generations/0000000000000000/wal/00000000/00000001.wal.lz4
TZ=UTC touch -ct 200001030000 generations/0000000000000000/wal/00000001/00000000.wal.lz4
TZ=UTC touch -ct 200001010000 generations/0000000000000000/wal/0000000000000000/0000000000000000.wal.lz4
TZ=UTC touch -ct 200001020000 generations/0000000000000000/wal/0000000000000000/0000000000000001.wal.lz4
TZ=UTC touch -ct 200001030000 generations/0000000000000000/wal/0000000000000001/0000000000000000.wal.lz4