[Breaking Change] Normalize nil payload value encoding to empty slice

ledger/trie.go

@@ -266,13 +266,32 @@ type serializablePayload struct {
 	Value Value
 }
 
+func (p Payload) seralizable() (serializablePayload, error) {
+	k, err := p.Key()
+	if err != nil {
+		return serializablePayload{}, err
+	}
+
+	v := p.value
+	if v == nil {
+		// Normalize nil payload values to empty slice (Value{}) to ensure consistency
+		// across all serialization formats (checkpoint files, WAL files, and execution data).
+		// This eliminates the distinction between nil and empty slice values, as both represent
+		// the removal of a register from the execution state. When an execution node loads a trie,
+		// it first reads checkpoint files and then WAL files, during which nil values are normalized
+		// to []byte{}. By normalizing at payload encoding time, we ensure all code paths produce
+		// consistent encoded data regardless of whether the original value was nil or []byte{}.
+		v = Value{}
+	}
+	return serializablePayload{Key: k, Value: v}, nil
+}
+
 // MarshalJSON returns JSON encoding of p.
 func (p Payload) MarshalJSON() ([]byte, error) {
-	k, err := p.Key()
+	sp, err := p.seralizable()
 	if err != nil {
 		return nil, err
 	}
-	sp := serializablePayload{Key: k, Value: p.value}
 	return json.Marshal(sp)
 }
 
@@ -292,11 +311,10 @@ func (p *Payload) UnmarshalJSON(b []byte) error {
 
 // MarshalCBOR returns CBOR encoding of p.
 func (p Payload) MarshalCBOR() ([]byte, error) {
-	k, err := p.Key()
+	sp, err := p.seralizable()
 	if err != nil {
 		return nil, err
 	}
-	sp := serializablePayload{Key: k, Value: p.value}
 	return cbor.Marshal(sp)
 }
 
@@ -363,6 +381,8 @@ func (p *Payload) Address() (flow.Address, error) {
 
 // Value returns payload value.
 // CAUTION: do not modify returned value because it shares underlying data with payload value.
+// CAUTION: to check wheather the payload value is empty, use len(payload.Value()) == 0,
+// due to normalization of nil and empty slice values in payloads during serialization.
 func (p *Payload) Value() Value {
 	if p == nil {
 		return Value{}

ledger/trie_encoder_test.go

@@ -616,3 +616,48 @@ func TestTrieUpdateSerialization(t *testing.T) {
 		require.True(t, decodedtu.Equals(tu))
 	})
 }
+
+// TestTrieUpdateNilVsEmptySlice verifies that EncodeTrieUpdate/DecodeTrieUpdate
+// for payloads created with nil vs empty []byte values results in both being treated
+// as empty []byte{} after decoding, due to normalization in NewPayload.
+func TestTrieUpdateNilVsEmptySlice(t *testing.T) {
+	p1 := testutils.PathByUint16(1)
+	kp1 := ledger.NewKeyPart(uint16(1), []byte("key 1"))
+	k1 := ledger.NewKey([]ledger.KeyPart{kp1})
+	// Original value is nil, but will be normalized to []byte{}
+	pl1 := ledger.NewPayload(k1, nil)
+
+	p2 := testutils.PathByUint16(2)
+	kp2 := ledger.NewKeyPart(uint16(1), []byte("key 2"))
+	k2 := ledger.NewKey([]ledger.KeyPart{kp2})
+	// Original value is []byte{}
+	pl2 := ledger.NewPayload(k2, []byte{})
+
+	tu := &ledger.TrieUpdate{
+		RootHash: testutils.RootHashFixture(),
+		Paths:    []ledger.Path{p1, p2},
+		Payloads: []*ledger.Payload{pl1, pl2},
+	}
+
+	// Step 1: Verify original distinction
+	require.Nil(t, tu.Payloads[0].Value(), "Payload 0 should have nil value (we don't normalize at creation time, only encoding time)")
+	require.NotNil(t, tu.Payloads[1].Value(), "Payload 1 should have non-nil value")
+	require.Equal(t, 0, len(tu.Payloads[0].Value()), "Payload 0 should have 0 length")
+	require.Equal(t, 0, len(tu.Payloads[1].Value()), "Payload 1 should have 0 length")
+
+	// Step 2: Encode and Decode
+	encoded := ledger.EncodeTrieUpdate(tu)
+	decoded, err := ledger.DecodeTrieUpdate(encoded)
+	require.NoError(t, err)
+
+	// Both will be []byte{} after decode due to normalization in NewPayload.
+	t.Logf("Decoded Payload 0 value: %v (isNil=%v)", decoded.Payloads[0].Value(), decoded.Payloads[0].Value() == nil)
+	t.Logf("Decoded Payload 1 value: %v (isNil=%v)", decoded.Payloads[1].Value(), decoded.Payloads[1].Value() == nil)
+
+	require.Equal(t, 0, len(decoded.Payloads[0].Value()), "Decoded Payload 0 should have 0 length")
+	require.Equal(t, 0, len(decoded.Payloads[1].Value()), "Decoded Payload 1 should have 0 length")
+
+	// The key assertion: they are now identical despite starting differently
+	require.Equal(t, decoded.Payloads[0].Value(), decoded.Payloads[1].Value(),
+		"Decoded nil and []byte{} are identical")
+}

ledger/trie_test.go

@@ -421,6 +421,11 @@ func TestPayloadJSONSerialization(t *testing.T) {
 
 		v2 := p2.Value()
 		require.True(t, v2.Equals(Value{}))
+
+		// after decoding, the value will be normalized to []byte{}
+		// which will be different from the original format
+		require.True(t, p.value == nil)
+		require.True(t, p2.value != nil)
 	})
 
 	t.Run("empty key", func(t *testing.T) {
@@ -537,12 +542,13 @@ func TestPayloadCBORSerialization(t *testing.T) {
 			0xf6,                         // null
 			0x65,                         // text(5)
 			0x56, 0x61, 0x6c, 0x75, 0x65, // "Value"
-			0xf6, // null
+			0x40, // null will be normalized to []byte{}
 		}
 
 		var p Payload
 		b, err := cbor.Marshal(p)
 		require.NoError(t, err)
+		require.True(t, p.value == nil)
 		require.Equal(t, encoded, b)
 
 		var p2 Payload
@@ -562,6 +568,11 @@ func TestPayloadCBORSerialization(t *testing.T) {
 
 		v2 := p2.Value()
 		require.True(t, v2.Equals(Value{}))
+
+		// after decoding, the value will be normalized to []byte{}
+		// which will be different from the original format
+		require.True(t, p.value == nil)
+		require.True(t, p2.value != nil)
 	})
 
 	t.Run("empty key", func(t *testing.T) {
@@ -624,7 +635,7 @@ func TestPayloadCBORSerialization(t *testing.T) {
 			0x01, 0x02, // "\u0001\u0002"
 			0x65,                         // text(5)
 			0x56, 0x61, 0x6c, 0x75, 0x65, // "Value"
-			0xf6, // null
+			0x40, // null will be normalized to []byte{}
 		}
 
 		k := Key{KeyParts: []KeyPart{{1, []byte{1, 2}}}}