Skip to content

Commit b016bed

Browse files
pthmasjulienrbrtCopilot
authored
feat: block Pruning (#2984)
* basic prunning * prune metadata from ev-node store * wiring prunning config to go-header * prune evm exec store * add parameters validation * make error handling consistent * add method to tracedstore to respect interface * add prune block to mockstore * fix helper for flag for consistency * add replace statement for local packages * flags * add safetey mechanism for pruning only da included blocks * fix rebase * remove useless check * use lastprunedheight in Tail() to optimize * move pruning from executor to dainclusionloop * don't prune go-header store * update tail function * rename execmetapruner to execpruner * Update core/execution/execution.go Co-authored-by: julienrbrt <julien@rbrt.fr> * Update execution/evm/execution.go Co-authored-by: julienrbrt <julien@rbrt.fr> * trigger pruning every ticker * add store to store adapter * invalidate cache store data after pruning * nit * move pruning to it's own function * update config.md file * remove ai comments * feat: add recovery history depth pruner (#3064) * Initial plan * Add recovery history retention pruning Co-authored-by: julienrbrt <29894366+julienrbrt@users.noreply.github.com> * feat: add configurable recovery history retention Co-authored-by: julienrbrt <29894366+julienrbrt@users.noreply.github.com> * Refactor recovery pruning into pruner component Co-authored-by: julienrbrt <29894366+julienrbrt@users.noreply.github.com> * Address pruner review feedback Co-authored-by: julienrbrt <29894366+julienrbrt@users.noreply.github.com> * Update evm test module dependency Co-authored-by: julienrbrt <29894366+julienrbrt@users.noreply.github.com> * Address code review feedback Co-authored-by: julienrbrt <29894366+julienrbrt@users.noreply.github.com> * Refine pruner checks and docs Co-authored-by: julienrbrt <29894366+julienrbrt@users.noreply.github.com> * Clarify PruneExecMeta comment Co-authored-by: julienrbrt <29894366+julienrbrt@users.noreply.github.com> * Rename recovery history setting Co-authored-by: julienrbrt <29894366+julienrbrt@users.noreply.github.com> * Adjust pruner interval and defaults Co-authored-by: julienrbrt <29894366+julienrbrt@users.noreply.github.com> * fixes * updates * updates * updates * comment --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: julienrbrt <29894366+julienrbrt@users.noreply.github.com> Co-authored-by: Julien Robert <julien@rbrt.fr> * cl * typo * updates * cleanup * updates * delete unused key * prevent aggresive pruning * config * feedback * shorter key * add logs * improve batch size --------- Co-authored-by: julienrbrt <julien@rbrt.fr> Co-authored-by: Copilot <198982749+Copilot@users.noreply.github.com> Co-authored-by: julienrbrt <29894366+julienrbrt@users.noreply.github.com>
1 parent 6a7c84c commit b016bed

36 files changed

Lines changed: 1142 additions & 63 deletions

CHANGELOG.md

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
99

1010
## [Unreleased]
1111

12+
### Added
13+
14+
- Node pruning support. [#2984](https://github.com/evstack/ev-node/pull/2984)
15+
- Two different sort of pruning implemented:
16+
_Classic pruning_ (`all`): prunes given `HEAD-n` blocks from the databases, including store metadatas.
17+
_Auto Storage Optimization_ (`metadata`): prunes only the state metadatas, keeps all blocks.
18+
By using one or the other, you are losing the ability to rollback or replay transactions earlier than `HEAD-n`.
19+
When using _classic pruning_, you aren't able to fetch blocks prior to `HEAD-n`.
20+
1221
## v1.0.0-rc.4
1322

1423
### Changes

apps/evm/cmd/rollback.go

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88

99
"github.com/ethereum/go-ethereum/common"
1010
ds "github.com/ipfs/go-datastore"
11+
"github.com/rs/zerolog"
1112
"github.com/spf13/cobra"
1213

1314
"github.com/evstack/ev-node/execution/evm"
@@ -30,6 +31,7 @@ func NewRollbackCmd() *cobra.Command {
3031
if err != nil {
3132
return err
3233
}
34+
logger := rollcmd.SetupLogger(nodeConfig.Log)
3335

3436
goCtx := cmd.Context()
3537
if goCtx == nil {
@@ -69,7 +71,7 @@ func NewRollbackCmd() *cobra.Command {
6971
}
7072

7173
// rollback execution layer via EngineClient
72-
engineClient, err := createRollbackEngineClient(cmd, rawEvolveDB)
74+
engineClient, err := createRollbackEngineClient(cmd, rawEvolveDB, logger)
7375
if err != nil {
7476
cmd.Printf("Warning: failed to create engine client, skipping EL rollback: %v\n", err)
7577
} else {
@@ -99,7 +101,7 @@ func NewRollbackCmd() *cobra.Command {
99101
return cmd
100102
}
101103

102-
func createRollbackEngineClient(cmd *cobra.Command, db ds.Batching) (*evm.EngineClient, error) {
104+
func createRollbackEngineClient(cmd *cobra.Command, db ds.Batching, logger zerolog.Logger) (*evm.EngineClient, error) {
103105
ethURL, err := cmd.Flags().GetString(evm.FlagEvmEthURL)
104106
if err != nil {
105107
return nil, fmt.Errorf("failed to get '%s' flag: %w", evm.FlagEvmEthURL, err)
@@ -128,5 +130,5 @@ func createRollbackEngineClient(cmd *cobra.Command, db ds.Batching) (*evm.Engine
128130
return nil, fmt.Errorf("JWT secret file '%s' is empty", jwtSecretFile)
129131
}
130132

131-
return evm.NewEngineExecutionClient(ethURL, engineURL, jwtSecret, common.Hash{}, common.Address{}, db, false)
133+
return evm.NewEngineExecutionClient(ethURL, engineURL, jwtSecret, common.Hash{}, common.Address{}, db, false, logger)
132134
}

apps/evm/cmd/run.go

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ var RunCmd = &cobra.Command{
5555
}
5656

5757
tracingEnabled := nodeConfig.Instrumentation.IsTracingEnabled()
58-
executor, err := createExecutionClient(cmd, datastore, tracingEnabled)
58+
executor, err := createExecutionClient(cmd, datastore, tracingEnabled, logger)
5959
if err != nil {
6060
return err
6161
}
@@ -67,11 +67,6 @@ var RunCmd = &cobra.Command{
6767

6868
daClient := block.NewDAClient(blobClient, nodeConfig, logger)
6969

70-
// Attach logger to the EVM engine client if available
71-
if ec, ok := executor.(*evm.EngineClient); ok {
72-
ec.SetLogger(logger.With().Str("module", "engine_client").Logger())
73-
}
74-
7570
headerNamespace := da.NamespaceFromString(nodeConfig.DA.GetNamespace())
7671
dataNamespace := da.NamespaceFromString(nodeConfig.DA.GetDataNamespace())
7772

@@ -192,7 +187,7 @@ func createSequencer(
192187
return sequencer, nil
193188
}
194189

195-
func createExecutionClient(cmd *cobra.Command, db datastore.Batching, tracingEnabled bool) (execution.Executor, error) {
190+
func createExecutionClient(cmd *cobra.Command, db datastore.Batching, tracingEnabled bool, logger zerolog.Logger) (execution.Executor, error) {
196191
// Read execution client parameters from flags
197192
ethURL, err := cmd.Flags().GetString(evm.FlagEvmEthURL)
198193
if err != nil {
@@ -237,7 +232,7 @@ func createExecutionClient(cmd *cobra.Command, db datastore.Batching, tracingEna
237232
genesisHash := common.HexToHash(genesisHashStr)
238233
feeRecipient := common.HexToAddress(feeRecipientStr)
239234

240-
return evm.NewEngineExecutionClient(ethURL, engineURL, jwtSecret, genesisHash, feeRecipient, db, tracingEnabled)
235+
return evm.NewEngineExecutionClient(ethURL, engineURL, jwtSecret, genesisHash, feeRecipient, db, tracingEnabled, logger)
241236
}
242237

243238
// addFlags adds flags related to the EVM execution client

apps/evm/go.mod

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,11 @@ module github.com/evstack/ev-node/apps/evm
22

33
go 1.25.6
44

5-
//replace (
6-
// github.com/evstack/ev-node => ../../
7-
// github.com/evstack/ev-node/execution/evm => ../../execution/evm
8-
//)
5+
replace (
6+
github.com/evstack/ev-node => ../../
7+
github.com/evstack/ev-node/core => ../../core
8+
github.com/evstack/ev-node/execution/evm => ../../execution/evm
9+
)
910

1011
require (
1112
github.com/ethereum/go-ethereum v1.16.8

apps/evm/go.sum

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -411,12 +411,6 @@ github.com/ethereum/go-ethereum v1.16.8 h1:LLLfkZWijhR5m6yrAXbdlTeXoqontH+Ga2f9i
411411
github.com/ethereum/go-ethereum v1.16.8/go.mod h1:Fs6QebQbavneQTYcA39PEKv2+zIjX7rPUZ14DER46wk=
412412
github.com/ethereum/go-verkle v0.2.2 h1:I2W0WjnrFUIzzVPwm8ykY+7pL2d4VhlsePn4j7cnFk8=
413413
github.com/ethereum/go-verkle v0.2.2/go.mod h1:M3b90YRnzqKyyzBEWJGqj8Qff4IDeXnzFw0P9bFw3uk=
414-
github.com/evstack/ev-node v1.0.0-rc.4 h1:Ju7pSETFdadBZxmAj0//4z7hHkXbSRDy9iTzhF60Dew=
415-
github.com/evstack/ev-node v1.0.0-rc.4/go.mod h1:xGCH5NCdGiYk6v3GVPm4NhzAtcKQgnaVnORg8b4tbOk=
416-
github.com/evstack/ev-node/core v1.0.0-rc.1 h1:Dic2PMUMAYUl5JW6DkDj6HXDEWYzorVJQuuUJOV0FjE=
417-
github.com/evstack/ev-node/core v1.0.0-rc.1/go.mod h1:n2w/LhYQTPsi48m6lMj16YiIqsaQw6gxwjyJvR+B3sY=
418-
github.com/evstack/ev-node/execution/evm v1.0.0-rc.3 h1:3o8H1TNywnst56lo2RlS2SXulDfp9yZJtkYYh7ZJrdM=
419-
github.com/evstack/ev-node/execution/evm v1.0.0-rc.3/go.mod h1:VUEEklKoclg45GL7dzLoDwu3UQ4ptT3rF8bw5zUmnRk=
420414
github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
421415
github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
422416
github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=

apps/grpc/go.mod

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,11 @@ module github.com/evstack/ev-node/apps/grpc
22

33
go 1.25.6
44

5-
//replace (
6-
// github.com/evstack/ev-node => ../../
7-
// github.com/evstack/ev-node/execution/grpc => ../../execution/grpc
8-
//)
5+
replace (
6+
github.com/evstack/ev-node => ../../
7+
github.com/evstack/ev-node/core => ../../core
8+
github.com/evstack/ev-node/execution/grpc => ../../execution/grpc
9+
)
910

1011
require (
1112
github.com/evstack/ev-node v1.0.0-rc.4

apps/grpc/go.sum

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -367,12 +367,6 @@ github.com/envoyproxy/protoc-gen-validate v0.10.0/go.mod h1:DRjgyB0I43LtJapqN6Ni
367367
github.com/envoyproxy/protoc-gen-validate v0.10.1/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss=
368368
github.com/envoyproxy/protoc-gen-validate v1.0.1/go.mod h1:0vj8bNkYbSTNS2PIyH87KZaeN4x9zpL9Qt8fQC7d+vs=
369369
github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE=
370-
github.com/evstack/ev-node v1.0.0-rc.4 h1:Ju7pSETFdadBZxmAj0//4z7hHkXbSRDy9iTzhF60Dew=
371-
github.com/evstack/ev-node v1.0.0-rc.4/go.mod h1:xGCH5NCdGiYk6v3GVPm4NhzAtcKQgnaVnORg8b4tbOk=
372-
github.com/evstack/ev-node/core v1.0.0-rc.1 h1:Dic2PMUMAYUl5JW6DkDj6HXDEWYzorVJQuuUJOV0FjE=
373-
github.com/evstack/ev-node/core v1.0.0-rc.1/go.mod h1:n2w/LhYQTPsi48m6lMj16YiIqsaQw6gxwjyJvR+B3sY=
374-
github.com/evstack/ev-node/execution/grpc v1.0.0-rc.1 h1:OzrWLDDY6/9+LWx0XmUqPzxs/CHZRJICOwQ0Me/i6dY=
375-
github.com/evstack/ev-node/execution/grpc v1.0.0-rc.1/go.mod h1:Pr/sF6Zx8am9ZeWFcoz1jYPs0kXmf+OmL8Tz2Gyq7E4=
376370
github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
377371
github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
378372
github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=

apps/testapp/go.mod

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,10 @@ module github.com/evstack/ev-node/apps/testapp
22

33
go 1.25.6
44

5-
//replace github.com/evstack/ev-node => ../../
5+
replace (
6+
github.com/evstack/ev-node => ../../.
7+
github.com/evstack/ev-node/core => ../../core
8+
)
69

710
require (
811
github.com/evstack/ev-node v1.0.0-rc.4

apps/testapp/go.sum

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -367,10 +367,6 @@ github.com/envoyproxy/protoc-gen-validate v0.10.0/go.mod h1:DRjgyB0I43LtJapqN6Ni
367367
github.com/envoyproxy/protoc-gen-validate v0.10.1/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss=
368368
github.com/envoyproxy/protoc-gen-validate v1.0.1/go.mod h1:0vj8bNkYbSTNS2PIyH87KZaeN4x9zpL9Qt8fQC7d+vs=
369369
github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE=
370-
github.com/evstack/ev-node v1.0.0-rc.4 h1:Ju7pSETFdadBZxmAj0//4z7hHkXbSRDy9iTzhF60Dew=
371-
github.com/evstack/ev-node v1.0.0-rc.4/go.mod h1:xGCH5NCdGiYk6v3GVPm4NhzAtcKQgnaVnORg8b4tbOk=
372-
github.com/evstack/ev-node/core v1.0.0-rc.1 h1:Dic2PMUMAYUl5JW6DkDj6HXDEWYzorVJQuuUJOV0FjE=
373-
github.com/evstack/ev-node/core v1.0.0-rc.1/go.mod h1:n2w/LhYQTPsi48m6lMj16YiIqsaQw6gxwjyJvR+B3sY=
374370
github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
375371
github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
376372
github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=

block/components.go

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212
"github.com/evstack/ev-node/block/internal/common"
1313
da "github.com/evstack/ev-node/block/internal/da"
1414
"github.com/evstack/ev-node/block/internal/executing"
15+
"github.com/evstack/ev-node/block/internal/pruner"
1516
"github.com/evstack/ev-node/block/internal/reaping"
1617
"github.com/evstack/ev-node/block/internal/submitting"
1718
"github.com/evstack/ev-node/block/internal/syncing"
@@ -29,6 +30,7 @@ import (
2930
// Components represents the block-related components
3031
type Components struct {
3132
Executor *executing.Executor
33+
Pruner *pruner.Pruner
3234
Reaper *reaping.Reaper
3335
Syncer *syncing.Syncer
3436
Submitter *submitting.Submitter
@@ -60,6 +62,11 @@ func (bc *Components) Start(ctx context.Context) error {
6062
return fmt.Errorf("failed to start executor: %w", err)
6163
}
6264
}
65+
if bc.Pruner != nil {
66+
if err := bc.Pruner.Start(ctxWithCancel); err != nil {
67+
return fmt.Errorf("failed to start pruner: %w", err)
68+
}
69+
}
6370
if bc.Reaper != nil {
6471
if err := bc.Reaper.Start(ctxWithCancel); err != nil {
6572
return fmt.Errorf("failed to start reaper: %w", err)
@@ -96,6 +103,11 @@ func (bc *Components) Stop() error {
96103
errs = errors.Join(errs, fmt.Errorf("failed to stop executor: %w", err))
97104
}
98105
}
106+
if bc.Pruner != nil {
107+
if err := bc.Pruner.Stop(); err != nil {
108+
errs = errors.Join(errs, fmt.Errorf("failed to stop pruner: %w", err))
109+
}
110+
}
99111
if bc.Reaper != nil {
100112
if err := bc.Reaper.Stop(); err != nil {
101113
errs = errors.Join(errs, fmt.Errorf("failed to stop reaper: %w", err))
@@ -166,6 +178,12 @@ func NewSyncComponents(
166178
syncer.SetBlockSyncer(syncing.WithTracingBlockSyncer(syncer))
167179
}
168180

181+
var execPruner coreexecutor.ExecPruner
182+
if p, ok := exec.(coreexecutor.ExecPruner); ok {
183+
execPruner = p
184+
}
185+
pruner := pruner.New(logger, store, execPruner, config.Pruning, config.Node.BlockTime.Duration)
186+
169187
// Create submitter for sync nodes (no signer, only DA inclusion processing)
170188
var daSubmitter submitting.DASubmitterAPI = submitting.NewDASubmitter(daClient, config, genesis, blockOpts, metrics, logger, headerDAHintAppender, dataDAHintAppender)
171189
if config.Instrumentation.IsTracingEnabled() {
@@ -189,6 +207,7 @@ func NewSyncComponents(
189207
Syncer: syncer,
190208
Submitter: submitter,
191209
Cache: cacheManager,
210+
Pruner: pruner,
192211
errorCh: errorCh,
193212
}, nil
194213
}
@@ -248,6 +267,12 @@ func NewAggregatorComponents(
248267
executor.SetBlockProducer(executing.WithTracingBlockProducer(executor))
249268
}
250269

270+
var execPruner coreexecutor.ExecPruner
271+
if p, ok := exec.(coreexecutor.ExecPruner); ok {
272+
execPruner = p
273+
}
274+
pruner := pruner.New(logger, store, execPruner, config.Pruning, config.Node.BlockTime.Duration)
275+
251276
reaper, err := reaping.NewReaper(
252277
exec,
253278
sequencer,
@@ -264,6 +289,7 @@ func NewAggregatorComponents(
264289
if config.Node.BasedSequencer { // no submissions needed for bases sequencer
265290
return &Components{
266291
Executor: executor,
292+
Pruner: pruner,
267293
Reaper: reaper,
268294
Cache: cacheManager,
269295
errorCh: errorCh,
@@ -290,6 +316,7 @@ func NewAggregatorComponents(
290316

291317
return &Components{
292318
Executor: executor,
319+
Pruner: pruner,
293320
Reaper: reaper,
294321
Submitter: submitter,
295322
Cache: cacheManager,

0 commit comments

Comments
 (0)