Group replica wal segments by index

This commit changes the replica path format to group segments within
a single index in the same directory. This is to eventually add the
ability to seek to a record on file-based systems without having
to iterate over the records. The DB shadow WAL will also be changed
to this same format to support live replicas.
This commit is contained in:
Ben Johnson
2021-06-14 15:19:27 -06:00
parent 55c17b9d8e
commit fc897b481f
12 changed files with 504 additions and 426 deletions

View File

@@ -1,8 +1,11 @@
package internal
import (
"fmt"
"io"
"os"
"regexp"
"strconv"
"syscall"
"github.com/prometheus/client_golang/prometheus"
@@ -127,6 +130,33 @@ func MkdirAll(path string, fi os.FileInfo) error {
return nil
}
// ParseSnapshotPath parses the index from a snapshot filename. Used by path-based replicas.
func ParseSnapshotPath(s string) (index int, err error) {
a := snapshotPathRegex.FindStringSubmatch(s)
if a == nil {
return 0, fmt.Errorf("invalid snapshot path")
}
i64, _ := strconv.ParseUint(a[1], 16, 64)
return int(i64), nil
}
var snapshotPathRegex = regexp.MustCompile(`^([0-9a-f]{8})\.snapshot\.lz4$`)
// ParseWALSegmentPath parses the index/offset from a segment filename. Used by path-based replicas.
func ParseWALSegmentPath(s string) (index int, offset int64, err error) {
a := walSegmentPathRegex.FindStringSubmatch(s)
if a == nil {
return 0, 0, fmt.Errorf("invalid wal segment path")
}
i64, _ := strconv.ParseUint(a[1], 16, 64)
off64, _ := strconv.ParseUint(a[2], 16, 64)
return int(i64), int64(off64), nil
}
var walSegmentPathRegex = regexp.MustCompile(`^([0-9a-f]{8})\/([0-9a-f]{8})\.wal\.lz4$`)
// Shared replica metrics.
var (
OperationTotalCounterVec = promauto.NewCounterVec(prometheus.CounterOpts{