Merge branch 'main' into manav/fix_batch_validation

Author: julienrbrt

block/manager.go

@@ -29,7 +29,7 @@ import (
 
 const (
 	// defaultDABlockTime is used only if DABlockTime is not configured for manager
-	defaultDABlockTime = 15 * time.Second
+	defaultDABlockTime = 6 * time.Second
 
 	// defaultBlockTime is used only if BlockTime is not configured for manager
 	defaultBlockTime = 1 * time.Second
@@ -586,15 +586,6 @@ func (m *Manager) publishBlockInternal(ctx context.Context) error {
 		}
 	}
 
-	newState, err := m.applyBlock(ctx, header, data)
-	if err != nil {
-		if ctx.Err() != nil {
-			return err
-		}
-		// if call to applyBlock fails, we halt the node, see https://github.com/cometbft/cometbft/pull/496
-		panic(err)
-	}
-
 	signature, err = m.getSignature(header.Header)
 	if err != nil {
 		return err
@@ -604,8 +595,17 @@ func (m *Manager) publishBlockInternal(ctx context.Context) error {
 	header.Signature = signature
 
 	if err := header.ValidateBasic(); err != nil {
-		// TODO(tzdybal): I think this is could be even a panic, because if this happens, header is FUBAR
-		m.logger.Error("header validation error", "error", err)
+		// If this ever happens, for recovery, check for a mismatch between the configured signing key and the proposer address in the genesis file
+		panic(fmt.Errorf("critical: newly produced header failed validation: %w", err))
+	}
+
+	newState, err := m.applyBlock(ctx, header, data)
+	if err != nil {
+		if ctx.Err() != nil {
+			return err
+		}
+		// if call to applyBlock fails, we halt the node, see https://github.com/cometbft/cometbft/pull/496
+		panic(err)
 	}
 
 	// append metadata to Data before validating and saving

da/client.go

@@ -1,6 +1,6 @@
 # Local DA
 
-Local DA implements the [go-da][go-da] interface over a Local Data Availability service.
+Local DA implements the [da][da] interface over a Local Data Availability service.
 
 It is intended to be used for testing DA layers without having to set up the actual services.
 
@@ -15,28 +15,41 @@ make build
 ./build/local-da
 ```
 
-should output
+This will start the Local DA service with default settings, listening on `localhost:7980` and with a default maximum blob size.
+
+The output should look similar to this (the timestamp and maxBlobSize might vary):
 
 ```sh
-2024/04/11 12:23:34 Listening on: localhost:7980
+11:07AM INF NewLocalDA: initialized LocalDA module=local-da
+11:07AM INF Listening on host=localhost maxBlobSize=1974272 module=da port=7980
+11:07AM INF server started listening on=localhost:7980 module=da
 ```
 
-Which exposes the [go-da] interface over JSONRPC and can be accessed with an HTTP client like [xh][xh]:
+Which exposes the [da] interface over JSONRPC and can be accessed with an HTTP client like [xh][xh]:
+
+#### Flags
+
+You can customize the behavior of the `local-da` binary using the following flags:
 
-You can also run local-da with a `-listen-all` flag which will make the process listen on 0.0.0.0 so that it can be accessed from other machines.
+* `-port <port>`: Specifies the listening port. Default: `7980`.
+* `-host <host>`: Specifies the listening address. Default: `localhost`.
+* `-listen-all`: If set, the service listens on all network interfaces (`0.0.0.0`) instead of just `localhost`. This allows access from other machines.
+* `-max-blob-size <bytes>`: Sets the maximum blob size in bytes that the DA service will accept. Default: `1974272` (which is `64 * 64 * 482`).
+
+**Example with flags:**
+
+To run `local-da` on port `8000`, accessible from any IP, with a max blob size of `1000000` bytes:
 
 ```sh
-$ ./build/local-da -listen-all
-2024/04/11 12:23:34 Listening on: 0.0.0.0:7980
+./build/local-da -port 8000 -listen-all -max-blob-size 1000000
 ```
 
-### Docker
-
-You can also run the local-da service using docker:
+Output:
 
 ```sh
-make docker-build
-docker run --rm -p 7980:7980 local-da
+11:07AM INF NewLocalDA: initialized LocalDA module=local-da
+11:07AM INF Listening on host=0.0.0.0 maxBlobSize=1974272 module=da port=8000
+11:07AM INF server started listening on=0.0.0.0:8000 module=da
 ```
 
 ### MaxBlobSize
@@ -93,9 +106,9 @@ output:
 
 ## References
 
-[1] [go-da][go-da]
+[1] [da][ da]
 
 [2] [xh][xh]
 
-[go-da]: https://github.com/rollkit/go-da
+[da]: https://github.com/rollkit/rollkit/blob/main/core/da/da.go#L11
 [xh]: https://github.com/ducaale/xh

da/cmd/local-da/README.md

@@ -2,6 +2,7 @@ package jsonrpc
 
 import (
 	"context"
+	"encoding/hex"
 	"errors"
 	"fmt"
 	"net/http"
@@ -246,7 +247,12 @@ func newClient(ctx context.Context, logger log.Logger, addr string, authHeader h
 	var multiCloser multiClientCloser
 	var client Client
 	client.DA.Logger = logger
-	client.DA.Namespace = []byte(namespace)
+	namespaceBytes, err := hex.DecodeString(namespace)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decode namespace: %w", err)
+	}
+	client.DA.Namespace = namespaceBytes
+	logger.Info("Creating new client", "namespace", namespace)
 	errs := getKnownErrorsMapping()
 	for name, module := range moduleMap(&client) {
 		closer, err := jsonrpc.NewMergeClient(ctx, addr, name, []interface{}{module}, authHeader, jsonrpc.WithErrors(errs))

da/jsonrpc/client.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"errors"
+	"fmt"
 	"sync"
 	"testing"
 	"time"
@@ -45,7 +46,7 @@ func TestProxy(t *testing.T) {
 		}
 	}()
 
-	client, err := proxy.NewClient(context.Background(), logger, ClientURL, "", "test")
+	client, err := proxy.NewClient(context.Background(), logger, ClientURL, "", "74657374")
 	require.NoError(t, err)
 	RunDATestSuite(t, &client.DA)
 }
@@ -167,25 +168,34 @@ func GetIDsTest(t *testing.T, d coreda.DA) {
 // ConcurrentReadWriteTest tests the use of mutex lock in DummyDA by calling separate methods that use `d.data` and making sure there's no race conditions
 func ConcurrentReadWriteTest(t *testing.T, d coreda.DA) {
 	var wg sync.WaitGroup
-	wg.Add(2)
+	ctx, cancel := context.WithTimeout(context.TODO(), 5*time.Second)
+	defer cancel()
 
-	ctx := context.TODO()
+	writeDone := make(chan struct{})
 
+	wg.Add(1)
 	go func() {
 		defer wg.Done()
-		for i := uint64(1); i <= 100; i++ {
-			ret, err := d.GetIDs(ctx, i, []byte("test"))
-			if err != nil {
-				assert.Empty(t, ret.IDs)
-			}
+		for i := uint64(1); i <= 50; i++ {
+			_, err := d.Submit(ctx, []coreda.Blob{[]byte(fmt.Sprintf("test-%d", i))}, 0, []byte{})
+			assert.NoError(t, err)
 		}
+		close(writeDone)
 	}()
 
+	wg.Add(1)
 	go func() {
 		defer wg.Done()
-		for i := uint64(1); i <= 100; i++ {
-			_, err := d.Submit(ctx, []coreda.Blob{[]byte("test")}, 0, []byte{})
-			assert.NoError(t, err)
+		for {
+			select {
+			case <-writeDone:
+				return
+			default:
+				ret, err := d.GetIDs(ctx, 1, []byte("test"))
+				if err != nil {
+					assert.Empty(t, ret.IDs)
+				}
+			}
 		}
 	}()
 

da/jsonrpc/proxy_test.go

@@ -31,8 +31,8 @@ import (
 	"github.com/rollkit/rollkit/pkg/sync"
 )
 
-// prefixes used in KV store to separate main node data from DALC data
-var mainPrefix = "0"
+// prefixes used in KV store to separate rollkit data from execution environment data (if the same data base is reused)
+var RollkitPrefix = "0"
 
 const (
 	// genesisChunkSize is the maximum size, in bytes, of each
@@ -84,7 +84,7 @@ func newFullNode(
 ) (fn *FullNode, err error) {
 	seqMetrics, _ := metricsProvider(genesis.ChainID)
 
-	mainKV := newPrefixKV(database, mainPrefix)
+	mainKV := newPrefixKV(database, RollkitPrefix)
 	headerSyncService, err := initHeaderSyncService(mainKV, nodeConfig, genesis, p2pClient, logger)
 	if err != nil {
 		return nil, err
@@ -539,5 +539,5 @@ func (n *FullNode) GetLogger() log.Logger {
 }
 
 func newPrefixKV(kvStore ds.Batching, prefix string) ds.Batching {
-	return (ktds.Wrap(kvStore, ktds.PrefixTransform{Prefix: ds.NewKey(prefix)}).Children()[0]).(ds.Batching)
+	return ktds.Wrap(kvStore, ktds.PrefixTransform{Prefix: ds.NewKey(prefix)})
 }

node/full.go

@@ -352,9 +352,9 @@ func LoadFromViper(inputViper *viper.Viper) (Config, error) {
 	// then override with settings from input viper (higher precedence)
 	for _, key := range inputViper.AllKeys() {
 		// Handle special case for prefixed keys
-		if strings.HasPrefix(key, "rollkit.") {
+		if after, ok := strings.CutPrefix(key, "rollkit."); ok {
 			// Strip the prefix for the merged viper
-			strippedKey := strings.TrimPrefix(key, "rollkit.")
+			strippedKey := after
 			mergedViper.Set(strippedKey, inputViper.Get(key))
 		} else {
 			mergedViper.Set(key, inputViper.Get(key))

pkg/config/config.go

@@ -106,5 +106,4 @@ func TestPrefixKVBatch(t *testing.T) {
 	prefixbatchkv2, _ := badgerPrefixkv.NewTransaction(t.Context(), false)
 	err = prefixbatchkv2.Delete(t.Context(), ds.NewKey("key1"))
 	require.NoError(err)
-
 }

pkg/store/prefix_test.go

@@ -5,7 +5,7 @@ import (
 	"fmt"
 	"path/filepath"
 
-	"github.com/rollkit/rollkit/da"
+	"github.com/rollkit/rollkit/da/jsonrpc"
 	"github.com/rollkit/rollkit/sequencers/single"
 	"github.com/rs/zerolog"
 
@@ -50,7 +50,7 @@ var RunCmd = &cobra.Command{
 
 		logger := rollcmd.SetupLogger(nodeConfig.Log)
 
-		daJrpc, err := da.NewClient(logger, nodeConfig.DA.Address, nodeConfig.DA.AuthToken, nodeConfig.DA.Namespace)
+		daJrpc, err := jsonrpc.NewClient(context.Background(), logger, nodeConfig.DA.Address, nodeConfig.DA.AuthToken, nodeConfig.DA.Namespace)
 		if err != nil {
 			return err
 		}
@@ -69,7 +69,7 @@ var RunCmd = &cobra.Command{
 			context.Background(),
 			logger,
 			datastore,
-			daJrpc,
+			&daJrpc.DA,
 			[]byte(nodeConfig.ChainID),
 			nodeConfig.Node.BlockTime.Duration,
 			singleMetrics,
@@ -89,7 +89,7 @@ var RunCmd = &cobra.Command{
 			return err
 		}
 
-		return rollcmd.StartNode(logger, cmd, executor, sequencer, daJrpc, nodeKey, p2pClient, datastore, nodeConfig)
+		return rollcmd.StartNode(logger, cmd, executor, sequencer, &daJrpc.DA, nodeKey, p2pClient, datastore, nodeConfig)
 	},
 }
 

rollups/evm/single/cmd/run.go

@@ -39,7 +39,7 @@ services:
       - rollkit-network
 
   rollkit-evm-single:
-    image: ghcr.io/rollkit/rollkit-evm-single:v0.1.2
+    image: ghcr.io/rollkit/rollkit-evm-single:v0.1.3
     depends_on:
       reth:
         condition: service_started
@@ -55,9 +55,9 @@ services:
       - EVM_GENESIS_HASH=0x0a962a0d163416829894c89cb604ae422323bcdf02d7ea08b94d68d3e026a380
       - EVM_BLOCK_TIME=1s
       - EVM_SIGNER_PASSPHRASE=secret
-      # - ROLLKIT_DA_ADDRESS=<da_address>
-      # - ROLLKIT_DA_AUTH_TOKEN=<da_auth_token>
-      # - ROLLKIT_DA_NAMESPACE=<da_namespace>
+      # - DA_ADDRESS=http://localhost:26658
+      # - DA_AUTH_TOKEN=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJBbGxvdyI6WyJwdWJsaWMiLCJyZWFkIiwid3JpdGUiXSwiTm9uY2UiOiJQcEswTmhyWi9IY05NWkVtUG9sSXNpRTRDcUpMdE9mbWtBMW0zMWFUaEswPSIsIkV4cGlyZXNBdCI6IjAwMDEtMDEtMDFUMDA6MDA6MDBaIn0.gaWh6tS6Rel1XFYclDkapNnZlaZVjrikCRNBxSDkCGk
+      # - DA_NAMESPACE=00000000000000000000000000000000000000000008e5f679bf7116c1
     networks:
       - rollkit-network