rkrtechnology/litestream
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Rename replicator to replica

Browse Source
This commit is contained in:
Ben Johnson
2020-12-29 12:49:23 -07:00
parent 81e99c8035
commit 41448ceb89
4 changed files with 71 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
cmd/litestream/generations.go
View File

@@ -70,15 +70,15 @@ func (c *GenerationsCommand) Run(ctx context.Context, args []string) (err error)
return err return err
} }
// Iterate over each replicator in the database. // Iterate over each replica in the database.
for _, r := range db.Replicators { for _, r := range db.Replicas {
generations, err := r.Generations(ctx) generations, err := r.Generations(ctx)
if err != nil { if err != nil {
log.Printf("%s: cannot list generations", r.Name(), err) log.Printf("%s: cannot list generations", r.Name(), err)
continue continue
} }
// Iterate over each generation for the replicator. // Iterate over each generation for the replica.
for _, generation := range generations { for _, generation := range generations {
stats, err := r.GenerationStats(ctx, generation) stats, err := r.GenerationStats(ctx, generation)
if err != nil { if err != nil {

26
cmd/litestream/main.go
View File

@@ -134,8 +134,8 @@ type DBConfig struct {
type ReplicaConfig struct { type ReplicaConfig struct {
Type string `yaml:"type"` // "file", "s3" Type string `yaml:"type"` // "file", "s3"
Name string `yaml:"name"` // name of replicator, optional. Name string `yaml:"name"` // name of replica, optional.
Path string `yaml:"path"` // used for file replicators Path string `yaml:"path"` // used for file replicas
} }
func registerConfigFlag(fs *flag.FlagSet, p *string) { func registerConfigFlag(fs *flag.FlagSet, p *string) {
@@ -147,32 +147,32 @@ func newDBFromConfig(config *DBConfig) (*litestream.DB, error) {
// Initialize database with given path. // Initialize database with given path.
db := litestream.NewDB(config.Path) db := litestream.NewDB(config.Path)
// Instantiate and attach replicators. // Instantiate and attach replicas.
for _, rconfig := range config.Replicas { for _, rconfig := range config.Replicas {
r, err := newReplicatorFromConfig(db, rconfig) r, err := newReplicaFromConfig(db, rconfig)
if err != nil { if err != nil {
return nil, err return nil, err
} }
db.Replicators = append(db.Replicators, r) db.Replicas = append(db.Replicas, r)
} }
return db, nil return db, nil
} }
// newReplicatorFromConfig instantiates a replicator for a DB based on a config. // newReplicaFromConfig instantiates a replica for a DB based on a config.
func newReplicatorFromConfig(db *litestream.DB, config *ReplicaConfig) (litestream.Replicator, error) { func newReplicaFromConfig(db *litestream.DB, config *ReplicaConfig) (litestream.Replica, error) {
switch config.Type { switch config.Type {
case "", "file": case "", "file":
return newFileReplicatorFromConfig(db, config) return newFileReplicaFromConfig(db, config)
default: default:
return nil, fmt.Errorf("unknown replicator type in config: %q", config.Type) return nil, fmt.Errorf("unknown replica type in config: %q", config.Type)
} }
} }
// newFileReplicatorFromConfig returns a new instance of FileReplicator build from config. // newFileReplicaFromConfig returns a new instance of FileReplica build from config.
func newFileReplicatorFromConfig(db *litestream.DB, config *ReplicaConfig) (*litestream.FileReplicator, error) { func newFileReplicaFromConfig(db *litestream.DB, config *ReplicaConfig) (*litestream.FileReplica, error) {
if config.Path == "" { if config.Path == "" {
return nil, fmt.Errorf("file replicator path require for db %q", db.Path()) return nil, fmt.Errorf("file replica path require for db %q", db.Path())
} }
return litestream.NewFileReplicator(db, config.Name, config.Path), nil return litestream.NewFileReplica(db, config.Name, config.Path), nil
} }

34
db.go
View File

@@ -52,9 +52,9 @@ type DB struct {
// unbounded if there are always read transactions occurring. // unbounded if there are always read transactions occurring.
MaxCheckpointPageN int MaxCheckpointPageN int
// List of replicators for the database. // List of replicas for the database.
// Must be set before calling Open(). // Must be set before calling Open().
Replicators []Replicator Replicas []Replica
// Frequency at which to perform db sync. // Frequency at which to perform db sync.
MonitorInterval time.Duration MonitorInterval time.Duration
@@ -154,11 +154,11 @@ func (db *DB) PageSize() int {
} }
func (db *DB) Open() (err error) { func (db *DB) Open() (err error) {
// Validate that all replicator names are unique. // Validate that all replica names are unique.
m := make(map[string]struct{}) m := make(map[string]struct{})
for _, r := range db.Replicators { for _, r := range db.Replicas {
if _, ok := m[r.Name()]; ok { if _, ok := m[r.Name()]; ok {
return fmt.Errorf("duplicate replicator name: %q", r.Name()) return fmt.Errorf("duplicate replica name: %q", r.Name())
} }
m[r.Name()] = struct{}{} m[r.Name()] = struct{}{}
} }
@@ -276,7 +276,7 @@ func (db *DB) Init() (err error) {
} }
// Start replication. // Start replication.
for _, r := range db.Replicators { for _, r := range db.Replicas {
r.Start(db.ctx) r.Start(db.ctx)
} }
@@ -318,8 +318,8 @@ func (db *DB) SoftClose() (err error) {
db.cancel() db.cancel()
db.wg.Wait() db.wg.Wait()
// Ensure replicators all stop replicating. // Ensure replicas all stop replicating.
for _, r := range db.Replicators { for _, r := range db.Replicas {
r.Stop() r.Stop()
} }
@@ -387,7 +387,7 @@ func (db *DB) CurrentGeneration() (string, error) {
} }
// createGeneration starts a new generation by creating the generation // createGeneration starts a new generation by creating the generation
// directory, snapshotting to each replicator, and updating the current // directory, snapshotting to each replica, and updating the current
// generation name. // generation name.
func (db *DB) createGeneration() (string, error) { func (db *DB) createGeneration() (string, error) {
// Generate random generation hex name. // Generate random generation hex name.
@@ -516,7 +516,7 @@ func (db *DB) Sync() (err error) {
} }
} }
// Notify replicators of WAL changes. // Notify replicas of WAL changes.
if changed { if changed {
close(db.notify) close(db.notify)
db.notify = make(chan struct{}) db.notify = make(chan struct{})
@@ -1084,14 +1084,14 @@ func (db *DB) Restore(ctx context.Context, opt RestoreOptions) error {
return nil return nil
} }
func (db *DB) restoreTarget(ctx context.Context, opt RestoreOptions, logger *log.Logger) (Replicator, string, error) { func (db *DB) restoreTarget(ctx context.Context, opt RestoreOptions, logger *log.Logger) (Replica, string, error) {
var target struct { var target struct {
replicator Replicator replica Replica
generation string generation string
stats GenerationStats stats GenerationStats
} }
for _, r := range db.Replicators { for _, r := range db.Replicas {
// Skip replica if it does not match filter. // Skip replica if it does not match filter.
if opt.ReplicaName != "" && r.Name() != opt.ReplicaName { if opt.ReplicaName != "" && r.Name() != opt.ReplicaName {
continue continue
@@ -1127,7 +1127,7 @@ func (db *DB) restoreTarget(ctx context.Context, opt RestoreOptions, logger *log
continue continue
} }
target.replicator = r target.replica = r
target.generation = generation target.generation = generation
target.stats = stats target.stats = stats
} }
@@ -1138,11 +1138,11 @@ func (db *DB) restoreTarget(ctx context.Context, opt RestoreOptions, logger *log
return nil, "", fmt.Errorf("no matching backups found") return nil, "", fmt.Errorf("no matching backups found")
} }
return target.replicator, target.generation, nil return target.replica, target.generation, nil
} }
// restoreSnapshot copies a snapshot from the replica to a file. // restoreSnapshot copies a snapshot from the replica to a file.
func (db *DB) restoreSnapshot(ctx context.Context, r Replicator, generation string, index int, filename string) error { func (db *DB) restoreSnapshot(ctx context.Context, r Replica, generation string, index int, filename string) error {
if err := os.MkdirAll(filepath.Dir(filename), 0700); err != nil { if err := os.MkdirAll(filepath.Dir(filename), 0700); err != nil {
return err return err
} }
@@ -1170,7 +1170,7 @@ func (db *DB) restoreSnapshot(ctx context.Context, r Replicator, generation stri
} }
// restoreWAL copies a WAL file from the replica to the local WAL and forces checkpoint. // restoreWAL copies a WAL file from the replica to the local WAL and forces checkpoint.
func (db *DB) restoreWAL(ctx context.Context, r Replicator, generation string, index int, dbPath string) error { func (db *DB) restoreWAL(ctx context.Context, r Replica, generation string, index int, dbPath string) error {
// Open handle to destination WAL path. // Open handle to destination WAL path.
f, err := os.Create(dbPath + "-wal") f, err := os.Create(dbPath + "-wal")
if err != nil { if err != nil {

77
replicator.go
View File

@@ -15,13 +15,12 @@ import (
"time" "time"
) )
// Replicator represents a method for replicating the snapshot & WAL data to // Replica represents a remote destination to replicate the database & WAL.
// a remote destination. type Replica interface {
type Replicator interface { // The name of the replica. Defaults to type if no name specified.
// The name of the replicator. Defaults to type if no name specified.
Name() string Name() string
// String identifier for the type of replicator ("file", "s3", etc). // String identifier for the type of replica ("file", "s3", etc).
Type() string Type() string
// Starts replicating in a background goroutine. // Starts replicating in a background goroutine.
@@ -30,7 +29,7 @@ type Replicator interface {
// Stops all replication processing. Blocks until processing stopped. // Stops all replication processing. Blocks until processing stopped.
Stop() Stop()
// Returns a list of generation names for the replicator. // Returns a list of generation names for the replica.
Generations(ctx context.Context) ([]string, error) Generations(ctx context.Context) ([]string, error)
// Returns basic information about a generation including the number of // Returns basic information about a generation including the number of
@@ -52,12 +51,12 @@ type Replicator interface {
WALReader(ctx context.Context, generation string, index int) (io.ReadCloser, error) WALReader(ctx context.Context, generation string, index int) (io.ReadCloser, error)
} }
var _ Replicator = (*FileReplicator)(nil) var _ Replica = (*FileReplica)(nil)
// FileReplicator is a replicator that replicates a DB to a local file path. // FileReplica is a replica that replicates a DB to a local file path.
type FileReplicator struct { type FileReplica struct {
db *DB // source database db *DB // source database
name string // replicator name, optional name string // replica name, optional
dst string // destination path dst string // destination path
// mu sync.RWMutex // mu sync.RWMutex
@@ -67,9 +66,9 @@ type FileReplicator struct {
cancel func() cancel func()
} }
// NewFileReplicator returns a new instance of FileReplicator. // NewFileReplica returns a new instance of FileReplica.
func NewFileReplicator(db *DB, name, dst string) *FileReplicator { func NewFileReplica(db *DB, name, dst string) *FileReplica {
return &FileReplicator{ return &FileReplica{
db: db, db: db,
name: name, name: name,
dst: dst, dst: dst,
@@ -77,41 +76,41 @@ func NewFileReplicator(db *DB, name, dst string) *FileReplicator {
} }
} }
// Name returns the name of the replicator. Returns the type if no name set. // Name returns the name of the replica. Returns the type if no name set.
func (r *FileReplicator) Name() string { func (r *FileReplica) Name() string {
if r.name != "" { if r.name != "" {
return r.name return r.name
} }
return r.Type() return r.Type()
} }
// Type returns the type of replicator. // Type returns the type of replica.
func (r *FileReplicator) Type() string { func (r *FileReplica) Type() string {
return "file" return "file"
} }
// SnapshotDir returns the path to a generation's snapshot directory. // SnapshotDir returns the path to a generation's snapshot directory.
func (r *FileReplicator) SnapshotDir(generation string) string { func (r *FileReplica) SnapshotDir(generation string) string {
return filepath.Join(r.dst, "generations", generation, "snapshots") return filepath.Join(r.dst, "generations", generation, "snapshots")
} }
// SnapshotPath returns the path to a snapshot file. // SnapshotPath returns the path to a snapshot file.
func (r *FileReplicator) SnapshotPath(generation string, index int) string { func (r *FileReplica) SnapshotPath(generation string, index int) string {
return filepath.Join(r.SnapshotDir(generation), fmt.Sprintf("%016x.snapshot.gz", index)) return filepath.Join(r.SnapshotDir(generation), fmt.Sprintf("%016x.snapshot.gz", index))
} }
// WALDir returns the path to a generation's WAL directory // WALDir returns the path to a generation's WAL directory
func (r *FileReplicator) WALDir(generation string) string { func (r *FileReplica) WALDir(generation string) string {
return filepath.Join(r.dst, "generations", generation, "wal") return filepath.Join(r.dst, "generations", generation, "wal")
} }
// WALPath returns the path to a WAL file. // WALPath returns the path to a WAL file.
func (r *FileReplicator) WALPath(generation string, index int) string { func (r *FileReplica) WALPath(generation string, index int) string {
return filepath.Join(r.WALDir(generation), fmt.Sprintf("%016x.wal", index)) return filepath.Join(r.WALDir(generation), fmt.Sprintf("%016x.wal", index))
} }
// Generations returns a list of available generation names. // Generations returns a list of available generation names.
func (r *FileReplicator) Generations(ctx context.Context) ([]string, error) { func (r *FileReplica) Generations(ctx context.Context) ([]string, error) {
fis, err := ioutil.ReadDir(filepath.Join(r.dst, "generations")) fis, err := ioutil.ReadDir(filepath.Join(r.dst, "generations"))
if os.IsNotExist(err) { if os.IsNotExist(err) {
return nil, nil return nil, nil
@@ -132,7 +131,7 @@ func (r *FileReplicator) Generations(ctx context.Context) ([]string, error) {
} }
// GenerationStats returns stats for a generation. // GenerationStats returns stats for a generation.
func (r *FileReplicator) GenerationStats(ctx context.Context, generation string) (stats GenerationStats, err error) { func (r *FileReplica) GenerationStats(ctx context.Context, generation string) (stats GenerationStats, err error) {
// Determine stats for all snapshots. // Determine stats for all snapshots.
n, min, max, err := r.snapshotStats(generation) n, min, max, err := r.snapshotStats(generation)
if err != nil { if err != nil {
@@ -159,7 +158,7 @@ func (r *FileReplicator) GenerationStats(ctx context.Context, generation string)
return stats, nil return stats, nil
} }
func (r *FileReplicator) snapshotStats(generation string) (n int, min, max time.Time, err error) { func (r *FileReplica) snapshotStats(generation string) (n int, min, max time.Time, err error) {
fis, err := ioutil.ReadDir(r.SnapshotDir(generation)) fis, err := ioutil.ReadDir(r.SnapshotDir(generation))
if os.IsNotExist(err) { if os.IsNotExist(err) {
return n, min, max, nil return n, min, max, nil
@@ -184,7 +183,7 @@ func (r *FileReplicator) snapshotStats(generation string) (n int, min, max time.
return n, min, max, nil return n, min, max, nil
} }
func (r *FileReplicator) walStats(generation string) (n int, min, max time.Time, err error) { func (r *FileReplica) walStats(generation string) (n int, min, max time.Time, err error) {
fis, err := ioutil.ReadDir(r.WALDir(generation)) fis, err := ioutil.ReadDir(r.WALDir(generation))
if os.IsNotExist(err) { if os.IsNotExist(err) {
return n, min, max, nil return n, min, max, nil
@@ -217,7 +216,7 @@ type GenerationStats struct {
} }
// Start starts replication for a given generation. // Start starts replication for a given generation.
func (r *FileReplicator) Start(ctx context.Context) { func (r *FileReplica) Start(ctx context.Context) {
// Stop previous replication. // Stop previous replication.
r.Stop() r.Stop()
@@ -230,13 +229,13 @@ func (r *FileReplicator) Start(ctx context.Context) {
} }
// Stop cancels any outstanding replication and blocks until finished. // Stop cancels any outstanding replication and blocks until finished.
func (r *FileReplicator) Stop() { func (r *FileReplica) Stop() {
r.cancel() r.cancel()
r.wg.Wait() r.wg.Wait()
} }
// monitor runs in a separate goroutine and continuously replicates the DB. // monitor runs in a separate goroutine and continuously replicates the DB.
func (r *FileReplicator) monitor(ctx context.Context) { func (r *FileReplica) monitor(ctx context.Context) {
// Clear old temporary files that my have been left from a crash. // Clear old temporary files that my have been left from a crash.
if err := removeTmpFiles(r.dst); err != nil { if err := removeTmpFiles(r.dst); err != nil {
log.Printf("%s(%s): cannot remove tmp files: %w", r.db.Path(), r.Name(), err) log.Printf("%s(%s): cannot remove tmp files: %w", r.db.Path(), r.Name(), err)
@@ -294,9 +293,9 @@ func (r *FileReplicator) monitor(ctx context.Context) {
} }
} }
// pos returns the position for the replicator for the current generation. // pos returns the position for the replica for the current generation.
// Returns a zero value if there is no active generation. // Returns a zero value if there is no active generation.
func (r *FileReplicator) pos() (pos Pos, err error) { func (r *FileReplica) pos() (pos Pos, err error) {
// Find the current generation from the DB. Return zero pos if no generation. // Find the current generation from the DB. Return zero pos if no generation.
generation, err := r.db.CurrentGeneration() generation, err := r.db.CurrentGeneration()
if err != nil { if err != nil {
@@ -346,7 +345,7 @@ func (r *FileReplicator) pos() (pos Pos, err error) {
} }
// snapshot copies the entire database to the replica path. // snapshot copies the entire database to the replica path.
func (r *FileReplicator) snapshot(ctx context.Context, generation string, index int) error { func (r *FileReplica) snapshot(ctx context.Context, generation string, index int) error {
// Acquire a read lock on the database during snapshot to prevent checkpoints. // Acquire a read lock on the database during snapshot to prevent checkpoints.
tx, err := r.db.db.Begin() tx, err := r.db.db.Begin()
if err != nil { if err != nil {
@@ -371,7 +370,7 @@ func (r *FileReplicator) snapshot(ctx context.Context, generation string, index
} }
// snapshotN returns the number of snapshots for a generation. // snapshotN returns the number of snapshots for a generation.
func (r *FileReplicator) snapshotN(generation string) (int, error) { func (r *FileReplica) snapshotN(generation string) (int, error) {
fis, err := ioutil.ReadDir(r.SnapshotDir(generation)) fis, err := ioutil.ReadDir(r.SnapshotDir(generation))
if os.IsNotExist(err) { if os.IsNotExist(err) {
return 0, nil return 0, nil
@@ -388,7 +387,7 @@ func (r *FileReplicator) snapshotN(generation string) (int, error) {
return n, nil return n, nil
} }
func (r *FileReplicator) sync(ctx context.Context, pos Pos) (_ Pos, err error) { func (r *FileReplica) sync(ctx context.Context, pos Pos) (_ Pos, err error) {
// Read all WAL files since the last position. // Read all WAL files since the last position.
for { for {
if pos, err = r.syncNext(ctx, pos); err == io.EOF { if pos, err = r.syncNext(ctx, pos); err == io.EOF {
@@ -399,7 +398,7 @@ func (r *FileReplicator) sync(ctx context.Context, pos Pos) (_ Pos, err error) {
} }
} }
func (r *FileReplicator) syncNext(ctx context.Context, pos Pos) (_ Pos, err error) { func (r *FileReplica) syncNext(ctx context.Context, pos Pos) (_ Pos, err error) {
rd, err := r.db.ShadowWALReader(pos) rd, err := r.db.ShadowWALReader(pos)
if err == io.EOF { if err == io.EOF {
return pos, err return pos, err
@@ -446,7 +445,7 @@ func (r *FileReplicator) syncNext(ctx context.Context, pos Pos) (_ Pos, err erro
} }
// compress gzips all WAL files before the current one. // compress gzips all WAL files before the current one.
func (r *FileReplicator) compress(ctx context.Context, generation string) error { func (r *FileReplica) compress(ctx context.Context, generation string) error {
dir := r.WALDir(generation) dir := r.WALDir(generation)
filenames, err := filepath.Glob(filepath.Join(dir, "*.wal")) filenames, err := filepath.Glob(filepath.Join(dir, "*.wal"))
if err != nil { if err != nil {
@@ -480,7 +479,7 @@ func (r *FileReplicator) compress(ctx context.Context, generation string) error
// SnapsotIndexAt returns the highest index for a snapshot within a generation // SnapsotIndexAt returns the highest index for a snapshot within a generation
// that occurs before timestamp. If timestamp is zero, returns the latest snapshot. // that occurs before timestamp. If timestamp is zero, returns the latest snapshot.
func (r *FileReplicator) SnapshotIndexAt(ctx context.Context, generation string, timestamp time.Time) (int, error) { func (r *FileReplica) SnapshotIndexAt(ctx context.Context, generation string, timestamp time.Time) (int, error) {
fis, err := ioutil.ReadDir(r.SnapshotDir(generation)) fis, err := ioutil.ReadDir(r.SnapshotDir(generation))
if os.IsNotExist(err) { if os.IsNotExist(err) {
return 0, ErrNoSnapshots return 0, ErrNoSnapshots
@@ -513,7 +512,7 @@ func (r *FileReplicator) SnapshotIndexAt(ctx context.Context, generation string,
// Returns the highest index for a WAL file that occurs before timestamp. // Returns the highest index for a WAL file that occurs before timestamp.
// If timestamp is zero, returns the highest WAL index. // If timestamp is zero, returns the highest WAL index.
func (r *FileReplicator) WALIndexAt(ctx context.Context, generation string, timestamp time.Time) (int, error) { func (r *FileReplica) WALIndexAt(ctx context.Context, generation string, timestamp time.Time) (int, error) {
fis, err := ioutil.ReadDir(r.WALDir(generation)) fis, err := ioutil.ReadDir(r.WALDir(generation))
if os.IsNotExist(err) { if os.IsNotExist(err) {
return 0, nil return 0, nil
@@ -544,7 +543,7 @@ func (r *FileReplicator) WALIndexAt(ctx context.Context, generation string, time
// SnapshotReader returns a reader for snapshot data at the given generation/index. // SnapshotReader returns a reader for snapshot data at the given generation/index.
// Returns os.ErrNotExist if no matching index is found. // Returns os.ErrNotExist if no matching index is found.
func (r *FileReplicator) SnapshotReader(ctx context.Context, generation string, index int) (io.ReadCloser, error) { func (r *FileReplica) SnapshotReader(ctx context.Context, generation string, index int) (io.ReadCloser, error) {
dir := r.SnapshotDir(generation) dir := r.SnapshotDir(generation)
fis, err := ioutil.ReadDir(dir) fis, err := ioutil.ReadDir(dir)
if err != nil { if err != nil {
@@ -581,7 +580,7 @@ func (r *FileReplicator) SnapshotReader(ctx context.Context, generation string,
// WALReader returns a reader for WAL data at the given index. // WALReader returns a reader for WAL data at the given index.
// Returns os.ErrNotExist if no matching index is found. // Returns os.ErrNotExist if no matching index is found.
func (r *FileReplicator) WALReader(ctx context.Context, generation string, index int) (io.ReadCloser, error) { func (r *FileReplica) WALReader(ctx context.Context, generation string, index int) (io.ReadCloser, error) {
filename := r.WALPath(generation, index) filename := r.WALPath(generation, index)
// Attempt to read uncompressed file first. // Attempt to read uncompressed file first.