Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 48 additions & 2 deletions haystack/utils/base_serialization.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,9 @@ def _serialize_value_with_schema(payload: Any) -> dict[str, Any]: # noqa: PLR09
The function handles:
- Objects with to_dict() methods (e.g. dataclasses)
- Objects with __dict__ attributes
- Dictionaries
- Dictionaries. Dicts with non-string keys record one key/value entry per pair under the
`entries` keyword, since JSON object keys must be strings and the key types would otherwise
be lost; dicts with only string keys keep the `properties` format.
- Lists, tuples, and sets, including ones holding mixed types. A mixed-type array records one
schema per position under the JSON Schema `prefixItems` keyword, while a homogeneous array
keeps the single shared `items` schema.
Expand All @@ -50,6 +52,30 @@ def _serialize_value_with_schema(payload: Any) -> dict[str, Any]: # noqa: PLR09

# Handle dictionary case - iterate through fields
if isinstance(payload, dict):
# JSON object keys must be strings, so a dict with non-string keys cannot
# round-trip through the `properties`/`data` mappings below: `json.dump`
# stringifies the keys and the original key types are lost on load (an int
# key comes back as a string, and keys that stringify alike collide).
# Serialize such dicts as an explicit list of key/value entries instead, so
# each key keeps its own schema and survives a JSON round-trip. Dicts with
# only string keys keep the historical `properties` envelope unchanged.
if any(not isinstance(field, str) for field in payload):
entries_schema: list[dict[str, Any]] = []
entries_data: list[dict[str, Any]] = []
for field, val in payload.items():
serialized_key = _serialize_value_with_schema(field)
serialized_value = _serialize_value_with_schema(val)
entries_schema.append(
{"key": serialized_key["serialization_schema"], "value": serialized_value["serialization_schema"]}
)
entries_data.append(
{"key": serialized_key["serialized_data"], "value": serialized_value["serialized_data"]}
)
return {
"serialization_schema": {"type": "object", "entries": entries_schema},
"serialized_data": entries_data,
}

schema: dict[str, Any] = {}
data: dict[str, Any] = {}

Expand Down Expand Up @@ -227,8 +253,28 @@ def _deserialize_value_with_schema(serialized: dict[str, Any]) -> Any:

# Handle object case (dictionary with properties)
if schema_type == "object":
# Dicts with non-string keys are stored as an explicit list of key/value
# entries (see `_serialize_value_with_schema`): each key is deserialized
# from its own schema, so int/float/bool/None keys survive the round-trip
# and keys that stringify alike no longer collide.
entries = schema.get("entries")
if entries is not None:
if len(entries) != len(data):
raise DeserializationError(
f"Invalid object payload: 'entries' declares {len(entries)} key/value pairs "
f"but 'serialized_data' holds {len(data)} entries."
)
result: dict[Any, Any] = {}
for entry_schema, entry_data in zip(entries, data, strict=True):
key = _deserialize_value_with_schema(
{"serialization_schema": entry_schema["key"], "serialized_data": entry_data["key"]}
)
result[key] = _deserialize_value_with_schema(
{"serialization_schema": entry_schema["value"], "serialized_data": entry_data["value"]}
)
return result
properties = schema["properties"]
result: dict[str, Any] = {}
result = {}
for field, raw_value in data.items():
field_schema = properties[field]
# Recursively deserialize each field - avoid creating temporary dict
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
---
fixes:
- |
Resuming a pipeline from a snapshot file no longer converts non-string
dictionary keys to strings. Values such as ``dict[int, str]`` passed
between components now come back with their original key types (``int``,
``float``, ``bool`` or ``None``) instead of stringified keys, and keys
that stringify alike no longer overwrite each other.
31 changes: 31 additions & 0 deletions test/core/pipeline/test_breakpoint.py
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,37 @@ def run(self, input_value: str) -> dict[str, str]:
assert loaded_snapshot.break_point.visit_count == 0


def test_resume_from_snapshot_file_preserves_int_dict_keys(tmp_path, monkeypatch):
# Regression test for #13492: a snapshot saved to a file stringified int dict
# keys (JSON object keys must be strings), so the resumed component received
# string keys where the run had int keys.
monkeypatch.setenv(HAYSTACK_PIPELINE_SNAPSHOT_SAVE_ENABLED, "true")

@component
class IntKeySource:
@component.output_types(v=dict[int, str])
def run(self):
return {"v": {1: "a", 2: "b"}}

@component
class IntKeySink:
@component.output_types(v=dict[int, str])
def run(self, v: dict[int, str]):
return {"v": v}

pipeline = Pipeline()
pipeline.add_component("src", IntKeySource())
pipeline.add_component("sink", IntKeySink())
pipeline.connect("src.v", "sink.v")

with pytest.raises(BreakpointException) as exc_info:
pipeline.run({}, break_point=Breakpoint("sink", snapshot_file_path=str(tmp_path)))

result = pipeline.run({}, pipeline_snapshot=load_pipeline_snapshot(exc_info.value.pipeline_snapshot_file_path))
assert result["sink"]["v"] == {1: "a", 2: "b"}
assert [type(k) for k in result["sink"]["v"]] == [int, int]


@component
class _AppendingComponent:
@component.output_types(result=str)
Expand Down
53 changes: 53 additions & 0 deletions test/utils/test_base_serialization.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
#
# SPDX-License-Identifier: Apache-2.0

import json
from datetime import datetime
from enum import Enum

Expand Down Expand Up @@ -453,6 +454,58 @@ def test_deserialize_prefix_items_length_mismatch_raises(self):
)


class TestNonStringKeys:
"""Dicts with non-string keys keep their key types across a JSON round-trip.

JSON object keys must be strings, so without the `entries` encoding an int
key would come back as a string (and keys that stringify alike would
collide). The JSON round-trip below is essential: an in-memory
serialize/deserialize keeps the keys by accident.
"""

@pytest.mark.parametrize(
"value",
[
pytest.param({1: "a", 2: "b"}, id="int-keys"),
pytest.param({1.5: "x", 2.5: "y"}, id="float-keys"),
pytest.param({True: "yes", False: "no"}, id="bool-keys"),
pytest.param({None: "nothing"}, id="none-key"),
pytest.param({1: "a", "x": "b"}, id="mixed-string-and-int-keys"),
pytest.param({"outer": {1: "a"}}, id="nested-int-keys"),
pytest.param({1: "int", "1": "str"}, id="stringified-collision-preserved"),
],
)
def test_round_trip_through_json(self, value):
serialized = _serialize_value_with_schema(value)
through_json = json.loads(json.dumps(serialized))
deserialized = _deserialize_value_with_schema(through_json)
assert deserialized == value
assert [type(k) for k in deserialized] == [type(k) for k in value]

def test_string_keyed_dict_output_is_unchanged(self):
# Backward compatibility: dicts with only string keys keep the
# historical `properties` envelope, with no `entries` key.
assert _serialize_value_with_schema({"a": 1}) == {
"serialization_schema": {"type": "object", "properties": {"a": {"type": "integer"}}},
"serialized_data": {"a": 1},
}

def test_deserialize_entries_length_mismatch_raises(self):
with pytest.raises(DeserializationError, match="'entries' declares 2"):
_deserialize_value_with_schema(
{
"serialization_schema": {
"type": "object",
"entries": [
{"key": {"type": "integer"}, "value": {"type": "string"}},
{"key": {"type": "integer"}, "value": {"type": "string"}},
],
},
"serialized_data": [{"key": 1, "value": "a"}],
}
)


class TestErrorHandling:
@pytest.mark.parametrize(
"value",
Expand Down