rkrtechnology/litestream
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

S3 replica

Browse Source
This commit is contained in:
Ben Johnson
2021-01-13 10:14:54 -07:00
parent faa5765745
commit 57a02a8628
7 changed files with 940 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
replica.go
View File

@@ -333,7 +333,7 @@ func (r *FileReplica) WALs(ctx context.Context) ([]*WALInfo, error) {
// Iterate over each WAL file.
for _, fi := range fis {
index, offset, _, err := ParseWALPath(fi.Name())
index, offset, _, _, err := ParseWALPath(fi.Name())
if err != nil {
continue
}
@@ -446,7 +446,7 @@ func (r *FileReplica) CalcPos(generation string) (pos Pos, err error) {
index := -1
for _, fi := range fis {
if idx, _, _, err := ParseWALPath(fi.Name()); err != nil {
if idx, _, _, _, err := ParseWALPath(fi.Name()); err != nil {
continue // invalid wal filename
} else if index == -1 || idx > index {
index = idx
@@ -679,7 +679,7 @@ func (r *FileReplica) WALIndexAt(ctx context.Context, generation string, maxInde
for _, fi := range fis {
// Read index from snapshot filename.
idx, _, _, err := ParseWALPath(fi.Name())
idx, _, _, _, err := ParseWALPath(fi.Name())
if err != nil {
continue // not a snapshot, skip
} else if !timestamp.IsZero() && fi.ModTime().After(timestamp) {
@@ -783,7 +783,7 @@ func (r *FileReplica) EnforceRetention(ctx context.Context) (err error) {
if err != nil {
return fmt.Errorf("cannot obtain snapshot list: %w", err)
}
snapshots = filterSnapshotsAfter(snapshots, time.Now().Add(-r.RetentionInterval))
snapshots = FilterSnapshotsAfter(snapshots, time.Now().Add(-r.RetentionInterval))
// If no retained snapshots exist, create a new snapshot.
if len(snapshots) == 0 {
@@ -801,7 +801,7 @@ func (r *FileReplica) EnforceRetention(ctx context.Context) (err error) {
}
for _, generation := range generations {
// Find earliest retained snapshot for this generation.
snapshot := findMinSnapshotByGeneration(snapshots, generation)
snapshot := FindMinSnapshotByGeneration(snapshots, generation)
// Delete generations if it has no snapshots being retained.
if snapshot == nil {
@@ -863,7 +863,7 @@ func (r *FileReplica) deleteGenerationWALBefore(ctx context.Context, generation
}
for _, fi := range fis {
idx, _, _, err := ParseWALPath(fi.Name())
idx, _, _, _, err := ParseWALPath(fi.Name())
if err != nil {
continue
} else if idx >= index {
@@ -910,9 +910,3 @@ func compressFile(src, dst string, uid, gid int) error {
// Move compressed file to final location.
return os.Rename(dst+".tmp", dst)
}
// walDirMask is a mask used to group 64K wal files into a directory.
const (
walDirFileN = 0x10000
walDirMask = uint64(0xFFFFFFFFFFFFFFFF ^ (walDirFileN - 1))
)