Copy WAL frames through temp file to shadow (#474)

.github/workflows/commit.yml

@@ -3,7 +3,7 @@ on: push
 jobs:
   build:
     name: Build & Unit Test
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-go@v2
@@ -32,7 +32,7 @@ jobs:
 
 #  long-running-test:
 #    name: Run Long Running Unit Test
-#    runs-on: ubuntu-22.04
+#    runs-on: ubuntu-latest
 #    steps:
 #      - uses: actions/checkout@v2
 #      - uses: actions/setup-go@v2
@@ -49,7 +49,7 @@ jobs:
 
 #  s3-integration-test:
 #    name: Run S3 Integration Tests
-#    runs-on: ubuntu-18.04
+#    runs-on: ubuntu-latest
 #    needs: build
 #    steps:
 #      - uses: actions/download-artifact@v2
@@ -66,7 +66,7 @@ jobs:
 
 #  gcp-integration-test:
 #    name: Run GCP Integration Tests
-#    runs-on: ubuntu-18.04
+#    runs-on: ubuntu-latest
 #    needs: build
 #    steps:
 #      - name: Extract GCP credentials
@@ -87,7 +87,7 @@ jobs:
 
 #  abs-integration-test:
 #    name: Run Azure Blob Store Integration Tests
-#    runs-on: ubuntu-18.04
+#    runs-on: ubuntu-latest
 #    needs: build
 #    steps:
 #      - uses: actions/download-artifact@v2
@@ -103,7 +103,7 @@ jobs:
 
 #  sftp-integration-test:
 #    name: Run SFTP Integration Tests
-#    runs-on: ubuntu-18.04
+#    runs-on: ubuntu-latest
 #    needs: build
 #    steps:
 #      - name: Extract SSH key

.github/workflows/release.linux.yml

@@ -6,7 +6,7 @@ on:
 name: release (linux)
 jobs:
   build:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
     strategy:
       matrix:
         include:
@@ -31,7 +31,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-go@v2
         with:
-          go-version: '1.16'
+          go-version: '1.20'
 
       - id: release
         uses: bruceadams/get-release@v1.2.2
@@ -54,7 +54,7 @@ jobs:
           mkdir -p dist
           cp etc/litestream.yml etc/litestream.service dist
           cat etc/nfpm.yml | LITESTREAM_VERSION=${{ steps.release.outputs.tag_name }} envsubst > dist/nfpm.yml
-          CGO_ENABLED=1 go build -ldflags "-s -w -X 'main.Version=${{ steps.release.outputs.tag_name }}'" -o dist/litestream ./cmd/litestream
+          CGO_ENABLED=1 go build -ldflags "-s -w -extldflags "-static" -X 'main.Version=${{ steps.release.outputs.tag_name }}'" -tags osusergo,netgo,sqlite_omit_load_extension -o dist/litestream ./cmd/litestream
           
           cd dist
           tar -czvf litestream-${{ steps.release.outputs.tag_name }}-${{ env.GOOS }}-${{ env.GOARCH }}${{ env.GOARM }}.tar.gz litestream

.github/workflows/release.linux_static.yml

@@ -1,62 +0,0 @@
-on:
-  release:
-    types:
-      - created
-
-name: release (linux/static)
-jobs:
-  build:
-    runs-on: ubuntu-18.04
-    strategy:
-      matrix:
-        include:
-          - arch: amd64
-            cc:   gcc
-          - arch: arm64
-            cc:   aarch64-linux-gnu-gcc
-          - arch: arm
-            arm:  6
-            cc:   arm-linux-gnueabi-gcc
-          - arch: arm
-            arm:  7
-            cc:   arm-linux-gnueabihf-gcc
-
-    env:
-      GOOS:   linux
-      GOARCH: ${{ matrix.arch }}
-      GOARM:  ${{ matrix.arm }}
-      CC:     ${{ matrix.cc }}
-
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-go@v2
-        with:
-          go-version: '1.16'
-
-      - id: release
-        uses: bruceadams/get-release@v1.2.2
-        env:
-          GITHUB_TOKEN: ${{ github.token }}
-
-      - name: Install cross-compilers
-        run: |
-          sudo apt-get update 
-          sudo apt-get install -y gcc-aarch64-linux-gnu gcc-arm-linux-gnueabihf gcc-arm-linux-gnueabi
-
-      - name: Build litestream
-        run: |
-          rm -rf dist
-          mkdir -p dist
-          CGO_ENABLED=1 go build -ldflags "-s -w -extldflags "-static" -X 'main.Version=${{ steps.release.outputs.tag_name }}'" -tags osusergo,netgo,sqlite_omit_load_extension -o dist/litestream ./cmd/litestream
-          cd dist
-          tar -czvf litestream-${{ steps.release.outputs.tag_name }}-${{ env.GOOS }}-${{ env.GOARCH }}${{ env.GOARM }}-static.tar.gz litestream
-
-      - name: Upload release tarball
-        uses: actions/upload-release-asset@v1.0.2
-        env:
-          GITHUB_TOKEN: ${{ github.token }}
-        with:
-          upload_url: ${{ steps.release.outputs.upload_url }}
-          asset_path: ./dist/litestream-${{ steps.release.outputs.tag_name }}-${{ env.GOOS }}-${{ env.GOARCH }}${{ env.GOARM }}-static.tar.gz
-          asset_name: litestream-${{ steps.release.outputs.tag_name }}-${{ env.GOOS }}-${{ env.GOARCH }}${{ env.GOARM }}-static.tar.gz
-          asset_content_type: application/gzip

db.go

@@ -1024,6 +1024,16 @@ func (db *DB) copyToShadowWAL(filename string) (newSize int64, err error) {
 		return 0, fmt.Errorf("last checksum: %w", err)
 	}
 
+	// Write to a temporary shadow file.
+	tempFilename := filename + ".tmp"
+	defer os.Remove(tempFilename)
+
+	f, err := internal.CreateFile(tempFilename, db.fileInfo)
+	if err != nil {
+		return 0, fmt.Errorf("create temp file: %w", err)
+	}
+	defer f.Close()
+
 	// Seek to correct position on real wal.
 	if _, err := r.Seek(origSize, io.SeekStart); err != nil {
 		return 0, fmt.Errorf("real wal seek: %w", err)
@@ -1034,7 +1044,6 @@ func (db *DB) copyToShadowWAL(filename string) (newSize int64, err error) {
 	// Read through WAL from last position to find the page of the last
 	// committed transaction.
 	frame := make([]byte, db.pageSize+WALFrameHeaderSize)
-	var buf bytes.Buffer
 	offset := origSize
 	lastCommitSize := origSize
 	for {
@@ -1064,24 +1073,46 @@ func (db *DB) copyToShadowWAL(filename string) (newSize int64, err error) {
 			break
 		}
 
-		// Add page to the new size of the shadow WAL.
+		// Write page to temporary WAL file.
-		buf.Write(frame)
+		if _, err := f.Write(frame); err != nil {
+			return 0, fmt.Errorf("write temp shadow wal: %w", err)
+		}
 
 		Tracef("%s: copy-shadow: ok %s offset=%d salt=%x %x", db.path, filename, offset, salt0, salt1)
 		offset += int64(len(frame))
 
-		// Flush to shadow WAL if commit record.
+		// Update new size if written frame was a commit record.
 		newDBSize := binary.BigEndian.Uint32(frame[4:])
 		if newDBSize != 0 {
-			if _, err := buf.WriteTo(w); err != nil {
-				return 0, fmt.Errorf("write shadow wal: %w", err)
-			}
-			buf.Reset()
 			lastCommitSize = offset
 		}
 	}
 
-	// Sync & close.
+	// If no WAL writes found, exit.
+	if origSize == lastCommitSize {
+		return origSize, nil
+	}
+
+	walByteN := lastCommitSize - origSize
+
+	// Move to beginning of temporary file.
+	if _, err := f.Seek(0, io.SeekStart); err != nil {
+		return 0, fmt.Errorf("temp file seek: %w", err)
+	}
+
+	// Copy from temporary file to shadow WAL.
+	if _, err := io.Copy(w, &io.LimitedReader{R: f, N: walByteN}); err != nil {
+		return 0, fmt.Errorf("write shadow file: %w", err)
+	}
+
+	// Close & remove temporary file.
+	if err := f.Close(); err != nil {
+		return 0, err
+	} else if err := os.Remove(tempFilename); err != nil {
+		return 0, err
+	}
+
+	// Sync & close shadow WAL.
 	if err := w.Sync(); err != nil {
 		return 0, err
 	} else if err := w.Close(); err != nil {
@@ -1089,7 +1120,7 @@ func (db *DB) copyToShadowWAL(filename string) (newSize int64, err error) {
 	}
 
 	// Track total number of bytes written to WAL.
-	db.totalWALBytesCounter.Add(float64(lastCommitSize - origSize))
+	db.totalWALBytesCounter.Add(float64(walByteN))
 
 	return lastCommitSize, nil
 }