package bbolt // Compact will create a copy of the source DB and in the destination DB. This may // reclaim space that the source database no longer has use for. txMaxSize can be // used to limit the transactions size of this process and may trigger intermittent // commits. A value of zero will ignore transaction sizes. // TODO: merge with: https://github.com/etcd-io/etcd/blob/b7f0f52a16dbf83f18ca1d803f7892d750366a94/mvcc/backend/backend.go#L349 func Compact(dst, src *DB, txMaxSize int64) error { // commit regularly, or we'll run out of memory for large datasets if using one transaction. var size int64 tx, err := dst.Begin(true) if err != nil { return err } defer func() { if tempErr := tx.Rollback(); tempErr != nil { err = tempErr } }() if err := walk(src, func(keys [][]byte, k, v []byte, seq uint64) error { // On each key/value, check if we have exceeded tx size. sz := int64(len(k) + len(v)) if size+sz > txMaxSize && txMaxSize != 0 { // Commit previous transaction. if err := tx.Commit(); err != nil { return err } // Start new transaction. tx, err = dst.Begin(true) if err != nil { return err } size = 0 } size += sz // Create bucket on the root transaction if this is the first level. nk := len(keys) if nk == 0 { bkt, err := tx.CreateBucket(k) if err != nil { return err } if err := bkt.SetSequence(seq); err != nil { return err } return nil } // Create buckets on subsequent levels, if necessary. b := tx.Bucket(keys[0]) if nk > 1 { for _, k := range keys[1:] { b = b.Bucket(k) } } // Fill the entire page for best compaction. b.FillPercent = 1.0 // If there is no value then this is a bucket call. if v == nil { bkt, err := b.CreateBucket(k) if err != nil { return err } if err := bkt.SetSequence(seq); err != nil { return err } return nil } // Otherwise treat it as a key/value pair. return b.Put(k, v) }); err != nil { return err } err = tx.Commit() return err } // walkFunc is the type of the function called for keys (buckets and "normal" // values) discovered by Walk. keys is the list of keys to descend to the bucket // owning the discovered key/value pair k/v. type walkFunc func(keys [][]byte, k, v []byte, seq uint64) error // walk walks recursively the bolt database db, calling walkFn for each key it finds. func walk(db *DB, walkFn walkFunc) error { return db.View(func(tx *Tx) error { return tx.ForEach(func(name []byte, b *Bucket) error { return walkBucket(b, nil, name, nil, b.Sequence(), walkFn) }) }) } func walkBucket(b *Bucket, keypath [][]byte, k, v []byte, seq uint64, fn walkFunc) error { // Execute callback. if err := fn(keypath, k, v, seq); err != nil { return err } // If this is not a bucket then stop. if v != nil { return nil } // Iterate over each child key/value. keypath = append(keypath, k) return b.ForEach(func(k, v []byte) error { if v == nil { bkt := b.Bucket(k) return walkBucket(bkt, keypath, k, nil, bkt.Sequence(), fn) } return walkBucket(b, keypath, k, v, b.Sequence(), fn) }) }