From 42f9ba9d1c1b7350b7140423e177f52e5bea058a Mon Sep 17 00:00:00 2001
From: Ben Johnson <benbjohnson@yahoo.com>
Date: Tue, 3 Nov 2020 12:02:13 -0700
Subject: [PATCH] doc

---
 doc/DESIGN.md | 91 +++++++++++++++++++++++++++++++++++++++++++++++++++
 wal.go        |  2 ++
 2 files changed, 93 insertions(+)
 create mode 100644 doc/DESIGN.md

diff --git a/doc/DESIGN.md b/doc/DESIGN.md
new file mode 100644
index 0000000..a17bd6c
--- /dev/null
+++ b/doc/DESIGN.md
@@ -0,0 +1,91 @@
+Litestream Design
+=================
+
+Litestream provides a file system layer to intercept writes to a SQLite database
+to construct a persistent write-ahead log that can be replicated.
+
+
+## File Layout
+
+### Local
+
+```
+dir/
+	db                               # SQLite database
+	db-wal                           # SQLite WAL
+	.db-lightstream.config           # per-db configuration
+	.db-lightstream/
+		log                          # recent event log
+		stat                         # per-db Prometheus statistics
+		snapshot                     # stores snapshot number (e.g. 0000000000000001)
+		wal/                         # each WAL file contains pages in flush interval
+			active                   # active WAL file exists until flush; renamed
+			0000000000000001.wal.gz  # flushed, compressed WAL files
+			0000000000000002.wal.gz
+```
+
+### Remote (S3)
+
+```
+bkt/
+	db/                                                # database path
+		0000000000000001/                              # snapshot directory
+			snapshot                                   # full db snapshot
+			0000000000000001.wal.gz                    # compressed WAL file
+			0000000000000002.wal.gz
+		0000000000000002/
+			snapshot
+			0000000000000001-0000000000000003.tar.gz
+```
+
+
+## Process
+
+### File System Startup
+
+File system startup:
+
+1. Load litestream.config file.
+2. Load all per-db "-lightstream.config" files.
+
+
+### DB startup:
+
+```
+IF "db" NOT EXISTS {
+	ensureWALRemovedIfDBNotExist()
+	restore()
+	setDBStatus("ok")
+	return
+} 
+
+IF "-wal" EXISTS {
+	syncToShadowWAL()
+	IF err {
+		setDBStatus("error")
+	} ELSE {
+		setDBStatus("ok")
+	}
+} ELSE {
+	ensureShadowWALMatchesDB() // check last page written to DB
+	IF err {
+		setDBStatus("error")
+	} ELSE {
+		setDBStatus("ok")
+	}
+}
+```
+
+
+### DB Recovery
+
+TODO
+
+
+### WAL Write
+
+1. Write to regular WAL
+2. On fsync to regular WAL, copy WAL to shadow WAL.
+2a. On copy error, mark errored & begin recovery
+
+
diff --git a/wal.go b/wal.go
index c6ecc35..f9cc10b 100644
--- a/wal.go
+++ b/wal.go
@@ -8,6 +8,8 @@ import (
 	"os"
 )
 
+// TODO: Pages can be written multiple times before 3.11.0 (https://sqlite.org/releaselog/3_11_0.html)
+
 var (
 	// ErrWALHeaderEmpty is returned when writing an empty header.
 	ErrWALHeaderEmpty = errors.New("wal header empty")