use retry instead of failure

Author: tac0turtle

block/metrics_helpers.go

@@ -5,6 +5,13 @@ import (
 	"time"
 )
 
+// DA metric modes
+const (
+	DAModeRetry   = "retry"
+	DAModeSuccess = "success"
+	DAModeFail    = "fail"
+)
+
 // MetricsTimer helps track operation duration
 type MetricsTimer struct {
 	start     time.Time
@@ -75,21 +82,25 @@ func (m *Manager) recordError(errorType string, recoverable bool) {
 	}
 }
 
-// recordDAMetrics records DA-related metrics
-func (m *Manager) recordDAMetrics(operation string, success bool) {
+// recordDAMetrics records DA-related metrics with three modes: "success", "fail", "retry"
+func (m *Manager) recordDAMetrics(operation string, mode string) {
 	switch operation {
 	case "submission":
-		m.metrics.DASubmissionAttempts.Add(1)
-		if success {
+		switch mode {
+		case "retry":
+			m.metrics.DASubmissionAttempts.Add(1)
+		case "success":
 			m.metrics.DASubmissionSuccesses.Add(1)
-		} else {
+		case "fail":
 			m.metrics.DASubmissionFailures.Add(1)
 		}
 	case "retrieval":
-		m.metrics.DARetrievalAttempts.Add(1)
-		if success {
+		switch mode {
+		case "retry":
+			m.metrics.DARetrievalAttempts.Add(1)
+		case "success":
 			m.metrics.DARetrievalSuccesses.Add(1)
-		} else {
+		case "fail":
 			m.metrics.DARetrievalFailures.Add(1)
 		}
 	}

block/metrics_test.go

@@ -141,11 +141,13 @@ func TestMetricsHelpers(t *testing.T) {
 	})
 
 	t.Run("recordDAMetrics", func(t *testing.T) {
-		// Should not panic
-		m.recordDAMetrics("submission", true)
-		m.recordDAMetrics("submission", false)
-		m.recordDAMetrics("retrieval", true)
-		m.recordDAMetrics("retrieval", false)
+		// Should not panic with three modes: retry, success, fail
+		m.recordDAMetrics("submission", DAModeRetry)
+		m.recordDAMetrics("submission", DAModeSuccess)
+		m.recordDAMetrics("submission", DAModeFail)
+		m.recordDAMetrics("retrieval", DAModeRetry)
+		m.recordDAMetrics("retrieval", DAModeSuccess)
+		m.recordDAMetrics("retrieval", DAModeFail)
 	})
 
 	t.Run("recordBlockProductionMetrics", func(t *testing.T) {

block/retriever.go

@@ -73,7 +73,7 @@ func (m *Manager) processNextDAHeaderAndData(ctx context.Context) error {
 		blobsResp, fetchErr := m.fetchBlobs(ctx, daHeight)
 		if fetchErr == nil {
 			// Record successful DA retrieval
-			m.recordDAMetrics("retrieval", true)
+			m.recordDAMetrics("retrieval", DAModeSuccess)
 
 			if blobsResp.Code == coreda.StatusNotFound {
 				m.logger.Debug("no blob data found", "daHeight", daHeight, "reason", blobsResp.Message)
@@ -215,13 +215,15 @@ func (m *Manager) fetchBlobs(ctx context.Context, daHeight uint64) (coreda.Resul
 	ctx, cancel := context.WithTimeout(ctx, dAefetcherTimeout)
 	defer cancel()
 
-	// Record DA retrieval attempt
-	m.recordDAMetrics("retrieval", false)
+	// Record DA retrieval retry attempt
+	m.recordDAMetrics("retrieval", DAModeRetry)
 
 	// TODO: we should maintain the original error instead of creating a new one as we lose context by creating a new error.
 	blobsRes := types.RetrieveWithHelpers(ctx, m.da, m.logger, daHeight, []byte(m.genesis.ChainID))
 	switch blobsRes.Code {
 	case coreda.StatusError:
+		// Record failed DA retrieval
+		m.recordDAMetrics("retrieval", DAModeFail)
 		err = fmt.Errorf("failed to retrieve block: %s", blobsRes.Message)
 	case coreda.StatusHeightFromFuture:
 		// Keep the root cause intact for callers that may rely on errors.Is/As.

block/submitter.go

@@ -114,16 +114,16 @@ func submitToDA[T any](
 
 		submitctx, submitCtxCancel := context.WithTimeout(ctx, 60*time.Second)
 
-		// Record DA submission attempt
-		m.recordDAMetrics("submission", false)
+		// Record DA submission retry attempt
+		m.recordDAMetrics("submission", DAModeRetry)
 
 		res := types.SubmitWithHelpers(submitctx, m.da, m.logger, currMarshaled, gasPrice, nil)
 		submitCtxCancel()
 
 		switch res.Code {
 		case coreda.StatusSuccess:
 			// Record successful DA submission
-			m.recordDAMetrics("submission", true)
+			m.recordDAMetrics("submission", DAModeSuccess)
 
 			m.logger.Info(fmt.Sprintf("successfully submitted %s to DA layer", itemType), "gasPrice", gasPrice, "count", res.SubmittedCount)
 			if res.SubmittedCount == uint64(remLen) {
@@ -144,6 +144,8 @@ func submitToDA[T any](
 			m.logger.Debug("resetting DA layer submission options", "backoff", backoff, "gasPrice", gasPrice)
 		case coreda.StatusNotIncludedInBlock, coreda.StatusAlreadyInMempool:
 			m.logger.Error("DA layer submission failed", "error", res.Message, "attempt", attempt)
+			// Record failed DA submission (will retry)
+			m.recordDAMetrics("submission", DAModeFail)
 			backoff = m.config.DA.BlockTime.Duration * time.Duration(m.config.DA.MempoolTTL)
 			if m.gasMultiplier > 0 && gasPrice != -1 {
 				gasPrice = gasPrice * m.gasMultiplier
@@ -156,11 +158,15 @@ func submitToDA[T any](
 			fallthrough
 		default:
 			m.logger.Error("DA layer submission failed", "error", res.Message, "attempt", attempt)
+			// Record failed DA submission (will retry)
+			m.recordDAMetrics("submission", DAModeFail)
 			backoff = m.exponentialBackoff(backoff)
 		}
 		attempt++
 	}
 	if !submittedAll {
+		// Record final failure after all retries are exhausted
+		m.recordDAMetrics("submission", DAModeFail)
 		// If not all items are submitted, the remaining items will be retried in the next submission loop.
 		return fmt.Errorf("failed to submit all %s(s) to DA layer, submitted %d items (%d left) after %d attempts", itemType, numSubmitted, remLen, attempt)
 	}