Reduce s3 LIST operations

s3/s3.go

@@ -651,11 +651,6 @@ func (r *Replica) Sync(ctx context.Context) (err error) {
 		return err
 	}
 
-	// Ensure sync & retainer do not calculate position or snapshot at the same time.
-	if err := func() error {
-		r.snapshotMu.Lock()
-		defer r.snapshotMu.Unlock()
-
 	// Find current position of database.
 	dpos, err := r.db.Pos()
 	if err != nil {
@@ -665,6 +660,13 @@ func (r *Replica) Sync(ctx context.Context) (err error) {
 	}
 	generation := dpos.Generation
 
+	// Calculate position if we don't have a previous position or if the generation changes.
+	// Ensure sync & retainer do not snapshot at the same time.
+	if lastPos := r.LastPos(); lastPos.IsZero() || lastPos.Generation != generation {
+		if err := func() error {
+			r.snapshotMu.Lock()
+			defer r.snapshotMu.Unlock()
+
 			// Create snapshot if no snapshots exist for generation.
 			if n, err := r.snapshotN(generation); err != nil {
 				return err
@@ -678,21 +680,20 @@ func (r *Replica) Sync(ctx context.Context) (err error) {
 			}
 
 			// Determine position, if necessary.
-		if r.LastPos().IsZero() {
 			pos, err := r.CalcPos(ctx, generation)
 			if err != nil {
 				return fmt.Errorf("cannot determine replica position: %s", err)
 			}
 
 			r.mu.Lock()
+			defer r.mu.Unlock()
 			r.pos = pos
-			r.mu.Unlock()
-		}
 
 			return nil
 		}(); err != nil {
 			return err
 		}
+	}
 
 	// Read all WAL files since the last position.
 	for {
@@ -724,7 +725,8 @@ func (r *Replica) syncWAL(ctx context.Context) (err error) {
 
 	var buf bytes.Buffer
 	gw := gzip.NewWriter(&buf)
-	if _, err := gw.Write(b); err != nil {
+	n, err := gw.Write(b)
+	if err != nil {
 		return err
 	} else if err := gw.Close(); err != nil {
 		return err
@@ -745,7 +747,7 @@ func (r *Replica) syncWAL(ctx context.Context) (err error) {
 		return err
 	}
 	r.putOperationTotalCounter.Inc()
-	r.putOperationBytesCounter.Add(float64(buf.Len())) // compressed bytes
+	r.putOperationBytesCounter.Add(float64(n)) // compressed bytes
 
 	// Save last replicated position.
 	r.mu.Lock()