From 49d625eb3d2d96eae86bdb6c90f1b31205411397 Mon Sep 17 00:00:00 2001 From: Richard Lundeen Date: Fri, 20 Nov 2026 18:39:14 -0800 Subject: [PATCH 1/2] FIX: Store conversation ids up to 128 characters on SQL Server Conversations.conversation_id and AttackResultEntries.conversation_id were VARCHAR(36), sized for a bare UUID, while PromptMemoryEntries stores 128. SQL Server rejected longer ids, so capability detection failed on every chat target with "supports_multi_turn failed". A new migration widens both columns, and memory now rejects ids no table can store on every backend. Capability detection errors now name the failed check and its cause, and the backend logs each failure. Co-Authored-By: Claude Opus 5.5 --- pyrit/backend/services/target_service.py | 8 ++ .../0d3592c62ad8_widen_conversation_ids.py | 113 ++++++++++++++++++ pyrit/memory/memory_interface.py | 6 +- pyrit/memory/memory_models.py | 34 +++++- .../common/discover_target_capabilities.py | 95 +++++++++++++-- .../test_azure_sql_memory_integration.py | 38 ++++++ .../test_conversation_id_length_migration.py | 106 ++++++++++++++++ .../test_discover_target_capabilities.py | 87 +++++++++++++- 8 files changed, 468 insertions(+), 19 deletions(-) create mode 100644 pyrit/memory/alembic/versions/0d3592c62ad8_widen_conversation_ids.py create mode 100644 tests/unit/memory/test_conversation_id_length_migration.py diff --git a/pyrit/backend/services/target_service.py b/pyrit/backend/services/target_service.py index 2d53e879c2..6066c95253 100644 --- a/pyrit/backend/services/target_service.py +++ b/pyrit/backend/services/target_service.py @@ -176,6 +176,14 @@ async def detect_capabilities_async(self, *, target_registry_name: str, expected try: results.append(await self._probe_capabilities_async(source)) except (CapabilityDiscoveryError, ValueError) as exc: + logger.warning( + "Capability detection failed for target %r (child %d, %s): %s", + target_registry_name, + index + 1, + type(source).__name__, + exc, + exc_info=exc, + ) if isinstance(target, RoundRobinTarget): error_type = ( CapabilityDiscoveryError if isinstance(exc, CapabilityDiscoveryError) else ValueError diff --git a/pyrit/memory/alembic/versions/0d3592c62ad8_widen_conversation_ids.py b/pyrit/memory/alembic/versions/0d3592c62ad8_widen_conversation_ids.py new file mode 100644 index 0000000000..f578b15f6b --- /dev/null +++ b/pyrit/memory/alembic/versions/0d3592c62ad8_widen_conversation_ids.py @@ -0,0 +1,113 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Store conversation ids up to 128 characters in every table. + +``Conversations.conversation_id`` and ``AttackResultEntries.conversation_id`` were +``VARCHAR(36)``, sized for a bare UUID, while ``PromptMemoryEntries`` already stores +128. Conversation ids are free-form, so SQL Server rejected longer ids such as the +capability-discovery probes' ``capability-probe-``. SQLite does not enforce +string lengths; its declared types are updated so the schema matches the models. + +Revision ID: 0d3592c62ad8 +Revises: 3e55dd099e38 +Create Date: 2026-11-20 18:00:00.000000 +""" + +import re +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "0d3592c62ad8" +down_revision: str | Sequence[str] | None = "3e55dd099e38" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +_TABLES = ("Conversations", "AttackResultEntries") + +# SQL Server cannot alter a primary-key column in place, so the key is dropped and +# re-created with its original name and clustering. Widening a VARCHAR that only +# participates in ordinary indexes is allowed in place. +_WIDEN_CONVERSATION_IDS_SQL = """ +DECLARE @pk sysname, @pk_type nvarchar(60), @sql nvarchar(max); +SELECT @pk = key_constraint.name, @pk_type = idx.type_desc +FROM sys.key_constraints AS key_constraint +JOIN sys.indexes AS idx + ON idx.object_id = key_constraint.parent_object_id AND idx.index_id = key_constraint.unique_index_id +WHERE key_constraint.parent_object_id = OBJECT_ID(N'Conversations') AND key_constraint.type = 'PK'; +IF @pk IS NULL THROW 50000, 'Conversations primary key is missing.', 1; +SET @sql = N'ALTER TABLE [Conversations] DROP CONSTRAINT ' + QUOTENAME(@pk); +EXEC sp_executesql @sql; +ALTER TABLE [Conversations] ALTER COLUMN [conversation_id] VARCHAR(128) NOT NULL; +SET @sql = N'ALTER TABLE [Conversations] ADD CONSTRAINT ' + QUOTENAME(@pk) + N' PRIMARY KEY ' + + @pk_type + N' ([conversation_id])'; +EXEC sp_executesql @sql; +ALTER TABLE [AttackResultEntries] ALTER COLUMN [conversation_id] VARCHAR(128) NOT NULL; +""" + + +def upgrade() -> None: + """Widen conversation ids to match ``PromptMemoryEntries``.""" + if op.get_bind().dialect.name == "mssql": + op.execute(sa.text(_WIDEN_CONVERSATION_IDS_SQL)) + return + _set_sqlite_declared_length(old_length=36, new_length=128) + + +def downgrade() -> None: + """ + Keep SQL Server's wider columns and restore SQLite's declared types. + + Earlier revisions read and write ids of up to 36 characters in a 128-character + SQL Server column unchanged, while narrowing would truncate or reject ids stored + since the upgrade. SQLite never enforced the length, so its declaration is + restored without touching stored ids. + """ + if op.get_bind().dialect.name == "sqlite": + _set_sqlite_declared_length(old_length=128, new_length=36) + + +def _set_sqlite_declared_length(*, old_length: int, new_length: int) -> None: + """ + Change the declared ``conversation_id`` length without rebuilding either table. + + SQLite ignores the length and stores the same TEXT values either way, so this is + the schema-only edit SQLite documents for changes that leave stored content valid + (https://www.sqlite.org/lang_altertable.html#otheralter). A rebuild would have to + re-create ``AttackResultEntries``' generated analytics column and would rewrite + its column-level foreign keys, which earlier downgrades drop in place. + + Raises: + RuntimeError: If a table does not declare the expected column exactly once, + or the database fails its integrity check afterwards. + """ + bind = op.get_bind() + pattern = re.compile(rf"(\bconversation_id\s+)VARCHAR\({old_length}\)", re.IGNORECASE) + replacements: dict[str, str] = {} + for table_name in _TABLES: + sql = bind.execute( + sa.text("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = :name"), {"name": table_name} + ).scalar_one() + updated, count = pattern.subn(rf"\g<1>VARCHAR({new_length})", sql) + if count != 1: + raise RuntimeError(f"{table_name} does not declare conversation_id as VARCHAR({old_length}).") + replacements[table_name] = updated + schema_version = bind.execute(sa.text("PRAGMA schema_version")).scalar_one() + bind.execute(sa.text("PRAGMA writable_schema = ON")) + try: + for table_name, sql in replacements.items(): + bind.execute( + sa.text("UPDATE sqlite_master SET sql = :sql WHERE type = 'table' AND name = :name"), + {"sql": sql, "name": table_name}, + ) + # A new schema version makes every connection reload the edited definitions. + bind.execute(sa.text(f"PRAGMA schema_version = {int(schema_version) + 1}")) + finally: + bind.execute(sa.text("PRAGMA writable_schema = OFF")) + result = bind.execute(sa.text("PRAGMA integrity_check")).scalar_one() + if result != "ok": + raise RuntimeError(f"SQLite integrity check failed after widening conversation ids: {result}") diff --git a/pyrit/memory/memory_interface.py b/pyrit/memory/memory_interface.py index 8c2c3bd64d..11eaf6636d 100644 --- a/pyrit/memory/memory_interface.py +++ b/pyrit/memory/memory_interface.py @@ -124,6 +124,7 @@ SeedEntry, SeedIdentifierEntry, TargetIdentifierEntry, + validate_conversation_id_length, ) from pyrit.memory.memory_session import MemorySession, _begin_sqlite_write, _lock_observations from pyrit.memory.storage import ( @@ -2314,8 +2315,8 @@ def _validate_persistable_conversation_ids(*, message_pieces: Sequence[MessagePi (``not_in_memory`` pieces should already be filtered out). Raises: - ValueError: If any piece has a ``None``, empty, or whitespace-only - ``conversation_id``. + ValueError: If any piece has a ``None``, empty, whitespace-only, or + too-long ``conversation_id``. """ for piece in message_pieces: if piece.conversation_id is None or not piece.conversation_id.strip(): @@ -2323,6 +2324,7 @@ def _validate_persistable_conversation_ids(*, message_pieces: Sequence[MessagePi f"MessagePiece {piece.id} has no conversation_id. A conversation_id must be set by " "the caller before a piece is persisted; the memory layer does not generate one." ) + validate_conversation_id_length(piece.conversation_id) def _insert_conversation(self, *, conversation: Conversation) -> None: """ diff --git a/pyrit/memory/memory_models.py b/pyrit/memory/memory_models.py index 29e76220cf..59ca8fba02 100644 --- a/pyrit/memory/memory_models.py +++ b/pyrit/memory/memory_models.py @@ -98,9 +98,32 @@ # Default pyrit_version for database records created before version tracking was added LEGACY_PYRIT_VERSION = "<0.10.0" + +#: Longest conversation_id any table stores. Conversation ids are free-form +#: caller-supplied strings, not necessarily UUIDs, so every table that stores one +#: uses this bound. SQL Server enforces it; SQLite does not, so +#: ``validate_conversation_id_length`` checks it before any write. +CONVERSATION_ID_MAX_LENGTH = 128 _LEGACY_UNSUPPORTED_ATTACK_IDENTIFIER_KEY = "__pyrit_legacy_unsupported_identifier__" +def validate_conversation_id_length(conversation_id: str) -> None: + """ + Reject a conversation_id that no database column can store. + + Args: + conversation_id (str): The id about to be persisted. + + Raises: + ValueError: If the id is longer than ``CONVERSATION_ID_MAX_LENGTH``. + """ + if len(conversation_id) > CONVERSATION_ID_MAX_LENGTH: + raise ValueError( + f"conversation_id is {len(conversation_id)} characters; the most memory stores is " + f"{CONVERSATION_ID_MAX_LENGTH}: {conversation_id[:CONVERSATION_ID_MAX_LENGTH]}..." + ) + + def _load_identifier( stored: dict[str, Any] | None, *, @@ -637,9 +660,8 @@ class PromptMemoryEntry(Base): ) id = mapped_column(CustomUUID, nullable=False, primary_key=True) role: Mapped[ChatMessageRole] = mapped_column(String, nullable=False) - # Bounded so SQL Server accepts it as an index key. 128 rather than 36 because - # conversation_id is a free-form caller-supplied string, not necessarily a UUID. - conversation_id = mapped_column(String(128), nullable=False) + # Bounded so SQL Server accepts it as an index key. + conversation_id = mapped_column(String(CONVERSATION_ID_MAX_LENGTH), nullable=False) sequence = mapped_column(INTEGER, nullable=False) timestamp = mapped_column(UTCDateTime, nullable=False) prompt_metadata: Mapped[dict[str, str | int]] = mapped_column(JSON) @@ -1389,7 +1411,7 @@ class ConversationEntry(Base): __tablename__ = "Conversations" __table_args__ = {"extend_existing": True} - conversation_id = mapped_column(String(36), primary_key=True, nullable=False) + conversation_id = mapped_column(String(CONVERSATION_ID_MAX_LENGTH), primary_key=True, nullable=False) target_identifier: Mapped[dict[str, str] | None] = mapped_column(JSON, nullable=True) #: Foreign key to the content-addressed ``TargetIdentifiers`` row. Nullable: #: a conversation may be registered without a known target. @@ -1417,6 +1439,7 @@ def __init__(self, *, conversation: Conversation) -> None: Args: conversation (Conversation): The conversation metadata to persist. """ + validate_conversation_id_length(conversation.conversation_id) self.conversation_id = conversation.conversation_id self.target_identifier = conversation.target_identifier.model_dump() if conversation.target_identifier else None self.target_identifier_hash = conversation.target_identifier.hash if conversation.target_identifier else None @@ -2293,7 +2316,7 @@ class AttackResultEntry(Base): {"extend_existing": True}, ) id = mapped_column(CustomUUID, nullable=False, primary_key=True) - conversation_id = mapped_column(String(36), nullable=False) + conversation_id = mapped_column(String(CONVERSATION_ID_MAX_LENGTH), nullable=False) objective = mapped_column(Unicode, nullable=False) atomic_attack_identifier: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True) atomic_attack_identifier_hash: Mapped[str | None] = mapped_column( @@ -2389,6 +2412,7 @@ def __init__(self, *, entry: AttackResult) -> None: Raises: ValueError: If mutated legacy attribution labels are invalid or conflict. """ + validate_conversation_id_length(entry.conversation_id) self.id = uuid.UUID(entry.attack_result_id) self.conversation_id = entry.conversation_id self.objective = entry.objective diff --git a/pyrit/prompt_target/common/discover_target_capabilities.py b/pyrit/prompt_target/common/discover_target_capabilities.py index 491543c063..cbefed07c8 100644 --- a/pyrit/prompt_target/common/discover_target_capabilities.py +++ b/pyrit/prompt_target/common/discover_target_capabilities.py @@ -72,6 +72,10 @@ DEFAULT_PROBE_RETRY_BACKOFF_SECONDS: float = 0.1 MAX_PROBE_RETRY_BACKOFF_SECONDS: float = 1.0 +# Longest error message a failed check reports, so a provider's full response body +# does not flood the result. +_MAX_ERROR_DETAIL_LENGTH: int = 500 + # Exceptions that are deterministic on the probe payload and will not become # valid on a retry (malformed Message, type errors, missing attributes, etc.). # These fail the probe immediately rather than wasting backoff time. @@ -96,6 +100,42 @@ class CapabilityDiscoveryError(RuntimeError): """Discovery could not determine support because a probe failed.""" +def _status_code(error: BaseException) -> int | None: + """ + Return the HTTP status code an SDK error carries, if any. + + Returns: + int | None: The status code, or None when the error has none. + """ + status_code = getattr(error, "status_code", None) + if status_code is None: + status_code = getattr(getattr(error, "response", None), "status_code", None) + return status_code if isinstance(status_code, int) else None + + +def _describe_error(error: BaseException) -> str: + """ + Summarize ``error`` on one line so a failed check names its cause. + + Database errors report the driver's message instead of the full statement + and its parameters. + + Returns: + str: The error type, HTTP status when known, and the first line of its message. + """ + if isinstance(error, CapabilityDiscoveryError): + return str(error) + original = getattr(error, "orig", None) + source = original if isinstance(original, BaseException) else error + lines = str(source).strip().splitlines() + text = lines[0].strip() if lines else "" + if len(text) > _MAX_ERROR_DETAIL_LENGTH: + text = text[:_MAX_ERROR_DETAIL_LENGTH].rstrip() + "..." + status_code = _status_code(error) + kind = type(error).__name__ if status_code is None else f"{type(error).__name__} (HTTP {status_code})" + return f"{kind}: {text}" if text else kind + + def _raise_if_inconclusive(*, error: Exception, label: str) -> None: """ Reject operational failures when discovery will be applied automatically. @@ -103,11 +143,14 @@ def _raise_if_inconclusive(*, error: Exception, label: str) -> None: Raises: CapabilityDiscoveryError: A strict probe failed without evidence of rejection. """ - status_code = getattr(error, "status_code", None) - if status_code is None: - status_code = getattr(getattr(error, "response", None), "status_code", None) - if _RAISE_ON_PROBE_ERROR.get() and not (isinstance(error, ValueError) or status_code in (400, 422)): - raise CapabilityDiscoveryError(f"{label} failed; capabilities were not changed.") from error + if not _RAISE_ON_PROBE_ERROR.get(): + return + if isinstance(error, CapabilityDiscoveryError): + raise CapabilityDiscoveryError(f"{label}: {error}") from error + if not (isinstance(error, ValueError) or _status_code(error) in (400, 422)): + raise CapabilityDiscoveryError( + f"{label} failed: {_describe_error(error)}; capabilities were not changed." + ) from error def _json_enforcing_target_types() -> tuple[type[PromptTarget], ...]: @@ -268,6 +311,7 @@ async def _send_and_check_async( timeout_s: float, retries: int = 1, label: str = "Capability probe", + failures: list[str] | None = None, ) -> bool: """ Send ``message`` and report whether the call succeeded cleanly. @@ -291,6 +335,8 @@ async def _send_and_check_async( Defaults to 1. label (str): Short label used in log messages. Defaults to ``"Capability probe"``. + failures (list[str] | None): When given, receives a one-line reason + for each failed send, so callers can report why a check failed. Returns: bool: ``True`` iff the call returned without raising and every response @@ -299,6 +345,11 @@ async def _send_and_check_async( ``"empty"``, ``"unknown"``) is treated as failure. An empty response list (or responses with no message pieces) is also treated as a failure. """ + + def _record_failure(error: Exception) -> None: + if failures is not None: + failures.append(f"{label}: {_describe_error(error)}") + attempts = max(1, retries + 1) last_exc: Exception | None = None for attempt in range(attempts): @@ -314,6 +365,7 @@ async def _send_and_check_async( # Deterministic on the probe payload — retrying will not help. logger.debug("%s failed with non-retryable error: %s", label, exc) _raise_if_inconclusive(error=exc, label=label) + _record_failure(exc) return False except Exception as exc: last_exc = exc @@ -324,19 +376,26 @@ async def _send_and_check_async( if not responses or any(not r.message_pieces for r in responses): logger.debug("%s returned an empty response; treating as failure", label) - _raise_if_inconclusive(error=RuntimeError("Empty response"), label=label) + empty_error = RuntimeError("The target returned an empty response.") + _raise_if_inconclusive(error=empty_error, label=label) + _record_failure(empty_error) return False for response in responses: for piece in response.message_pieces: if piece.response_error != "none": logger.debug("%s returned error response: %s", label, piece.converted_value) - _raise_if_inconclusive(error=RuntimeError("Error response"), label=label) + response_error = RuntimeError( + f"The target returned a '{piece.response_error}' error response: {piece.converted_value}" + ) + _raise_if_inconclusive(error=response_error, label=label) + _record_failure(response_error) return False return True logger.info("%s exhausted %d attempt(s); last error: %s", label, attempts, last_exc) if last_exc is not None: _raise_if_inconclusive(error=last_exc, label=label) + _record_failure(last_exc) return False @@ -400,7 +459,9 @@ async def _probe_system_prompt_async(target: PromptTarget, timeout_s: float, ret except Exception as exc: logger.debug("System-prompt probe could not seed system message: %s", exc) if _RAISE_ON_PROBE_ERROR.get(): - raise CapabilityDiscoveryError("Could not prepare the system-prompt probe.") from exc + raise CapabilityDiscoveryError( + f"Could not prepare the system-prompt probe: {_describe_error(exc)}; capabilities were not changed." + ) from exc return False user_piece = _user_text_piece(value="hi", conversation_id=conversation_id) return await _send_and_check_async( @@ -506,7 +567,9 @@ async def _probe_multi_turn_async(target: PromptTarget, timeout_s: float, retrie except Exception as exc: logger.debug("Multi-turn probe could not seed conversation history: %s", exc) if _RAISE_ON_PROBE_ERROR.get(): - raise CapabilityDiscoveryError("Could not prepare the multi-turn probe.") from exc + raise CapabilityDiscoveryError( + f"Could not prepare the multi-turn probe: {_describe_error(exc)}; capabilities were not changed." + ) from exc return False second = _user_text_piece(value="What did I just tell you?", conversation_id=conversation_id) @@ -894,18 +957,28 @@ async def _check_control_request_async( key=lambda combination: (combination != frozenset({"text"}), len(combination), sorted(combination)), ) assets = test_assets if test_assets is not None else DEFAULT_TEST_ASSETS + failures: list[str] = [] with _permissive_configuration(target=target): for combination in combinations: try: message = await asyncio.to_thread(_create_test_message, modalities=combination, test_assets=assets) except (FileNotFoundError, ValueError) as exc: logger.info("Cannot prepare control input %s: %s", sorted(combination), exc) + failures.append(f"Could not prepare {sorted(combination)}: {_describe_error(exc)}") continue if await _send_and_check_async( - target=target, message=message, timeout_s=timeout_s, retries=retries, label="Control request" + target=target, + message=message, + timeout_s=timeout_s, + retries=retries, + label=f"Control request {sorted(combination)}", + failures=failures, ): return - raise CapabilityDiscoveryError("No declared input passed a control request; capabilities were not changed.") + reasons = "; ".join(failures) if failures else "the target declares no input types" + raise CapabilityDiscoveryError( + f"No declared input passed a control request ({reasons}); capabilities were not changed." + ) async def discover_target_capabilities_async( diff --git a/tests/integration/memory/test_azure_sql_memory_integration.py b/tests/integration/memory/test_azure_sql_memory_integration.py index 790b89b589..d8c44da3b9 100644 --- a/tests/integration/memory/test_azure_sql_memory_integration.py +++ b/tests/integration/memory/test_azure_sql_memory_integration.py @@ -17,6 +17,8 @@ PromptMemoryEntry, ) from pyrit.memory.memory_models import ( + CONVERSATION_ID_MAX_LENGTH, + ConversationEntry, ScenarioResultEntry, ) from pyrit.models import ( @@ -24,6 +26,7 @@ AttackOutcome, AttackResult, ComponentIdentifier, + Conversation, MessagePiece, ScenarioIdentifier, ScenarioResult, @@ -152,6 +155,10 @@ def cleanup(session: Session) -> None: PromptMemoryEntry.conversation_id.in_(conversation_ids) ).delete(synchronize_session=False) + session.query(ConversationEntry).filter( + ConversationEntry.conversation_id.in_(conversation_ids) + ).delete(synchronize_session=False) + session.commit() except SQLAlchemyError as e: session.rollback() @@ -369,6 +376,37 @@ async def test_metadata_string_filters_exclude_containers_async( await azuresql_instance.remove_seeds_from_memory_async(dataset_name=test_id) +async def test_longest_conversation_id_persists_in_every_table(azuresql_instance: AzureSQLMemory): + """SQL Server stores a conversation_id of the full allowed length, not only a bare UUID.""" + test_id = generate_test_id() + conversation_id = f"long-conversation-{test_id}-".ljust(CONVERSATION_ID_MAX_LENGTH, "x") + + async with cleanup_conversation_data_async(azuresql_instance, [conversation_id]): + await azuresql_instance.add_conversation_to_memory_async( + conversation=Conversation(conversation_id=conversation_id) + ) + await azuresql_instance.add_message_pieces_to_memory_async( + message_pieces=[MessagePiece(conversation_id=conversation_id, role="user", original_value="hi")] + ) + await azuresql_instance.add_attack_results_to_memory_async( + attack_results=[ + AttackResult( + conversation_id=conversation_id, + objective="Store a long conversation id", + atomic_attack_identifier=get_test_atomic_attack_identifier(), + outcome=AttackOutcome.UNDETERMINED, + labels={"long_conversation_id_test": test_id}, + ) + ] + ) + + messages = await azuresql_instance.get_conversation_messages_async(conversation_id=conversation_id) + results = await azuresql_instance.get_attack_results_async(labels={"long_conversation_id_test": test_id}) + + assert [message.message_pieces[0].original_value for message in messages] == ["hi"] + assert [result.conversation_id for result in results] == [conversation_id] + + async def test_get_attack_results_by_labels(azuresql_instance: AzureSQLMemory): """ Integration test for SQL Azure JSON filtering on labels. diff --git a/tests/unit/memory/test_conversation_id_length_migration.py b/tests/unit/memory/test_conversation_id_length_migration.py new file mode 100644 index 0000000000..bb87d7f699 --- /dev/null +++ b/tests/unit/memory/test_conversation_id_length_migration.py @@ -0,0 +1,106 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Conversation ids up to the shared length limit persist in every table.""" + +import importlib +import io +import os +import tempfile + +import pytest +import sqlalchemy as sa +from alembic import command +from alembic.migration import MigrationContext +from alembic.operations import Operations + +from pyrit.memory import AttackResultEntry, PromptMemoryEntry, SQLiteMemory +from pyrit.memory.memory_models import CONVERSATION_ID_MAX_LENGTH, ConversationEntry +from pyrit.memory.migration import _make_config +from pyrit.models import AttackOutcome, AttackResult, Conversation, MessagePiece + +_MIGRATION = "pyrit.memory.alembic.versions.0d3592c62ad8_widen_conversation_ids" + + +@pytest.mark.parametrize("entry_type", [ConversationEntry, AttackResultEntry, PromptMemoryEntry]) +def test_every_table_stores_the_longest_conversation_id(entry_type: type) -> None: + assert entry_type.__table__.c.conversation_id.type.length == CONVERSATION_ID_MAX_LENGTH + + +def test_sql_server_migration_rebuilds_the_conversation_key_and_widens_both_tables() -> None: + output = io.StringIO() + context = MigrationContext.configure(dialect_name="mssql", opts={"as_sql": True, "output_buffer": output}) + with Operations.context(context): + importlib.import_module(_MIGRATION).upgrade() + ddl = output.getvalue() + assert "DROP CONSTRAINT ' + QUOTENAME(@pk)" in ddl + assert "ALTER TABLE [Conversations] ALTER COLUMN [conversation_id] VARCHAR(128) NOT NULL" in ddl + assert "PRIMARY KEY '\n + @pk_type" in ddl + assert "ALTER TABLE [AttackResultEntries] ALTER COLUMN [conversation_id] VARCHAR(128) NOT NULL" in ddl + + +async def test_memory_accepts_the_longest_conversation_id(sqlite_instance: SQLiteMemory) -> None: + conversation_id = "c" * CONVERSATION_ID_MAX_LENGTH + await sqlite_instance.add_conversation_to_memory_async(conversation=Conversation(conversation_id=conversation_id)) + await sqlite_instance.add_message_pieces_to_memory_async( + message_pieces=[MessagePiece(conversation_id=conversation_id, role="user", original_value="hi")] + ) + messages = await sqlite_instance.get_conversation_messages_async(conversation_id=conversation_id) + assert [message.message_pieces[0].original_value for message in messages] == ["hi"] + + +async def test_memory_rejects_a_conversation_id_no_table_can_store(sqlite_instance: SQLiteMemory) -> None: + conversation_id = "c" * (CONVERSATION_ID_MAX_LENGTH + 1) + with pytest.raises(ValueError, match="conversation_id is 129 characters"): + await sqlite_instance.add_conversation_to_memory_async( + conversation=Conversation(conversation_id=conversation_id) + ) + with pytest.raises(ValueError, match="conversation_id is 129 characters"): + await sqlite_instance.add_message_pieces_to_memory_async( + message_pieces=[MessagePiece(conversation_id=conversation_id, role="user", original_value="hi")] + ) + with pytest.raises(ValueError, match="conversation_id is 129 characters"): + AttackResultEntry( + entry=AttackResult(conversation_id=conversation_id, objective="o", outcome=AttackOutcome.UNDETERMINED) + ) + + +def _conversation_id_lengths(connection: sa.Connection) -> dict[str, int]: + return { + table: next( + column["type"].length + for column in sa.inspect(connection).get_columns(table) + if column["name"] == "conversation_id" + ) + for table in ("Conversations", "AttackResultEntries") + } + + +def test_sqlite_migration_round_trips_declared_lengths_and_keeps_rows() -> None: + long_id = "c" * CONVERSATION_ID_MAX_LENGTH + with tempfile.TemporaryDirectory() as temp_dir: + engine = sa.create_engine(f"sqlite:///{os.path.join(temp_dir, 'widen.db')}") + try: + with engine.begin() as connection: + config = _make_config(connection=connection) + command.upgrade(config, "3e55dd099e38") + connection.execute( + sa.text('INSERT INTO "Conversations" (conversation_id) VALUES (:id)'), {"id": "kept"} + ) + command.upgrade(config, "0d3592c62ad8") + widened = _conversation_id_lengths(connection) + connection.execute( + sa.text('INSERT INTO "Conversations" (conversation_id) VALUES (:id)'), {"id": long_id} + ) + command.downgrade(config, "3e55dd099e38") + restored = _conversation_id_lengths(connection) + kept = ( + connection.execute(sa.text('SELECT conversation_id FROM "Conversations" ORDER BY conversation_id')) + .scalars() + .all() + ) + finally: + engine.dispose() + assert widened == {"Conversations": CONVERSATION_ID_MAX_LENGTH, "AttackResultEntries": CONVERSATION_ID_MAX_LENGTH} + assert restored == {"Conversations": 36, "AttackResultEntries": 36} + assert kept == [long_id, "kept"] diff --git a/tests/unit/prompt_target/test_discover_target_capabilities.py b/tests/unit/prompt_target/test_discover_target_capabilities.py index d698498d11..46d49558f6 100644 --- a/tests/unit/prompt_target/test_discover_target_capabilities.py +++ b/tests/unit/prompt_target/test_discover_target_capabilities.py @@ -14,6 +14,7 @@ from mcp.types import Tool as MCPToolDefinition from openai.types.chat import ChatCompletion from PIL import Image +from sqlalchemy.exc import ProgrammingError from unit.async_utils import TEST_WATCHDOG_SECONDS from unit.mocks import MockPromptTarget @@ -211,11 +212,95 @@ async def test_strict_discovery_rejects_blocked_response_without_changing_capabi target = MockPromptTarget() original = target.configuration with patch.object(target, "send_prompt_async", new_callable=AsyncMock, return_value=_error_response()): - with pytest.raises(CapabilityDiscoveryError): + with pytest.raises(CapabilityDiscoveryError, match="'blocked' error response: blocked"): await discover_target_capabilities_async(target=target, raise_on_error=True, apply=True) assert target.configuration is original +@pytest.mark.usefixtures("patch_central_database") +async def test_strict_discovery_reports_the_http_failure() -> None: + target = MockPromptTarget() + request = httpx.Request("POST", "https://example.test") + error = httpx.HTTPStatusError( + "Incorrect API key provided", request=request, response=httpx.Response(status_code=401, request=request) + ) + with patch.object(target, "send_prompt_async", new_callable=AsyncMock, side_effect=error): + with pytest.raises(CapabilityDiscoveryError) as raised: + await discover_target_capabilities_async(target=target, raise_on_error=True, retries=0) + assert str(raised.value) == ( + "Control request ['text'] failed: HTTPStatusError (HTTP 401): Incorrect API key provided; " + "capabilities were not changed." + ) + + +@pytest.mark.usefixtures("patch_central_database") +async def test_strict_discovery_reports_why_no_control_request_passed() -> None: + target = MockPromptTarget() + with patch.object( + target, "send_prompt_async", new_callable=AsyncMock, side_effect=ValueError("Invalid API settings") + ): + with pytest.raises(CapabilityDiscoveryError) as raised: + await discover_target_capabilities_async(target=target, raise_on_error=True, retries=0) + assert str(raised.value) == ( + "No declared input passed a control request (Control request ['text']: ValueError: Invalid API settings); " + "capabilities were not changed." + ) + + +@pytest.mark.usefixtures("patch_central_database") +async def test_strict_discovery_names_the_capability_and_the_failed_turn() -> None: + target = MockPromptTarget() + sends = 0 + + async def send_async(*, message: Message) -> list[Message]: + nonlocal sends + sends += 1 + # The control request and the first turn succeed; the history turn fails. + if sends > 2: + raise ConnectionError("offline") + return _ok_response() + + with ( + patch.object(target, "send_prompt_async", new_callable=AsyncMock, side_effect=send_async), + patch("pyrit.prompt_target.common.discover_target_capabilities._supports_chat_probes", return_value=True), + ): + with pytest.raises(CapabilityDiscoveryError) as raised: + await discover_target_capabilities_async( + target=target, + capabilities=[CapabilityName.MULTI_TURN], + test_modalities=set(), + raise_on_error=True, + retries=0, + ) + assert str(raised.value) == ( + "supports_multi_turn: Multi-turn probe (turn 2) failed: ConnectionError: offline; " + "capabilities were not changed." + ) + + +@pytest.mark.usefixtures("patch_central_database") +async def test_strict_discovery_reports_the_database_error_that_blocked_a_probe() -> None: + target = MockPromptTarget() + error = ProgrammingError( + 'INSERT INTO "Conversations" (conversation_id) VALUES (?)', + ("capability-probe-id",), + Exception("String or binary data would be truncated in table 'Conversations'"), + ) + with ( + patch.object(target, "send_prompt_async", new_callable=AsyncMock, return_value=_ok_response()), + patch.object(target._memory, "add_conversation_to_memory_async", new_callable=AsyncMock, side_effect=error), + patch("pyrit.prompt_target.common.discover_target_capabilities._supports_chat_probes", return_value=True), + ): + with pytest.raises(CapabilityDiscoveryError) as raised: + await discover_target_capabilities_async( + target=target, capabilities=[CapabilityName.MULTI_TURN], test_modalities=set(), raise_on_error=True + ) + assert str(raised.value) == ( + "supports_multi_turn: Could not prepare the multi-turn probe: ProgrammingError: " + "String or binary data would be truncated in table 'Conversations'; capabilities were not changed." + ) + + @pytest.mark.usefixtures("patch_central_database") class TestPermissiveConfiguration: def test_replaces_and_restores_configuration(self) -> None: From 571828148ff18d38c8545c2cd461a92b7b6774bc Mon Sep 17 00:00:00 2001 From: Richard Lundeen Date: Fri, 20 Nov 2026 21:32:07 -0800 Subject: [PATCH 2/2] FIX: Restore SQL Server column lengths on downgrade and validate every id write Address review feedback on the conversation id migration: - The SQL Server downgrade now narrows both columns back to 36 characters, so an earlier release's schema check accepts the database. It refuses when longer ids are stored and re-creates the AttackResultEntries indexes that cover conversation_id, which SQL Server cannot narrow in place. - The SQL Server migration check round-trips the newest migration, so the downgrade runs against a real server in CI. - A SQLAlchemy validator on each entry rejects too-long conversation ids on every assignment, so updates are checked as well as inserts. Ordering tests no longer depend on unique timestamps, which coarse clocks (Windows before Python 3.13) cannot guarantee. Co-Authored-By: Claude Opus 5.5 --- build_scripts/check_sqlserver_migrations.py | 9 +++- .../0d3592c62ad8_widen_conversation_ids.py | 47 +++++++++++++---- pyrit/memory/memory_interface.py | 6 +-- pyrit/memory/memory_models.py | 21 +++++--- .../test_interface_scenario_results.py | 9 +++- .../test_conversation_id_length_migration.py | 52 +++++++++++++++++-- .../core/test_scenario_partial_results.py | 5 +- 7 files changed, 117 insertions(+), 32 deletions(-) diff --git a/build_scripts/check_sqlserver_migrations.py b/build_scripts/check_sqlserver_migrations.py index 9ed0dcefc1..07f8701d23 100644 --- a/build_scripts/check_sqlserver_migrations.py +++ b/build_scripts/check_sqlserver_migrations.py @@ -9,6 +9,7 @@ from contextlib import contextmanager import sqlalchemy as sa +from alembic import command from alembic.script import ScriptDirectory from pyrit.memory.migration import ( @@ -46,13 +47,19 @@ def _temporary_database() -> Generator[sa.Engine, None, None]: def main() -> None: - """Apply all migrations to a fresh database and validate the resulting schema.""" + """Apply all migrations to a fresh database, round-trip the newest one, and validate the schema.""" with _temporary_database() as engine: with engine.connect() as connection: if sa.inspect(connection).get_table_names(): raise RuntimeError("SQL Server migration validation requires a fresh database.") run_schema_migrations(engine=engine) check_schema_migrations(engine=engine) + # Round-trip the newest migration so its SQL Server downgrade runs too. + with engine.begin() as connection: + config = _make_config(connection=connection) + command.downgrade(config, "-1") + command.upgrade(config, "head") + check_schema_migrations(engine=engine) with engine.connect() as connection: heads = ScriptDirectory.from_config(_make_config(connection=connection)).get_heads() versions = ( diff --git a/pyrit/memory/alembic/versions/0d3592c62ad8_widen_conversation_ids.py b/pyrit/memory/alembic/versions/0d3592c62ad8_widen_conversation_ids.py index f578b15f6b..a8c6073cef 100644 --- a/pyrit/memory/alembic/versions/0d3592c62ad8_widen_conversation_ids.py +++ b/pyrit/memory/alembic/versions/0d3592c62ad8_widen_conversation_ids.py @@ -30,9 +30,8 @@ _TABLES = ("Conversations", "AttackResultEntries") # SQL Server cannot alter a primary-key column in place, so the key is dropped and -# re-created with its original name and clustering. Widening a VARCHAR that only -# participates in ordinary indexes is allowed in place. -_WIDEN_CONVERSATION_IDS_SQL = """ +# re-created with its original name and clustering. +_RESIZE_CONVERSATION_KEY_SQL = """ DECLARE @pk sysname, @pk_type nvarchar(60), @sql nvarchar(max); SELECT @pk = key_constraint.name, @pk_type = idx.type_desc FROM sys.key_constraints AS key_constraint @@ -42,33 +41,59 @@ IF @pk IS NULL THROW 50000, 'Conversations primary key is missing.', 1; SET @sql = N'ALTER TABLE [Conversations] DROP CONSTRAINT ' + QUOTENAME(@pk); EXEC sp_executesql @sql; -ALTER TABLE [Conversations] ALTER COLUMN [conversation_id] VARCHAR(128) NOT NULL; +ALTER TABLE [Conversations] ALTER COLUMN [conversation_id] VARCHAR({length}) NOT NULL; SET @sql = N'ALTER TABLE [Conversations] ADD CONSTRAINT ' + QUOTENAME(@pk) + N' PRIMARY KEY ' + @pk_type + N' ([conversation_id])'; EXEC sp_executesql @sql; -ALTER TABLE [AttackResultEntries] ALTER COLUMN [conversation_id] VARCHAR(128) NOT NULL; """ +_REJECT_LONG_CONVERSATION_IDS_SQL = """ +IF EXISTS (SELECT 1 FROM [Conversations] WHERE DATALENGTH([conversation_id]) > 36) + OR EXISTS (SELECT 1 FROM [AttackResultEntries] WHERE DATALENGTH([conversation_id]) > 36) + THROW 50000, 'Conversation ids longer than 36 characters are stored; earlier releases cannot hold them.', 1; +""" + +# SQL Server widens an indexed VARCHAR in place but cannot narrow one, so the +# downgrade re-creates the AttackResultEntries indexes that cover conversation_id. +_ATTACK_RESULT_INDEXES = ( + ("ix_AttackResultEntries_conversation_timestamp_id", ["conversation_id", "timestamp", "id"], []), + ("ix_AttackResultEntries_operator_timestamp_id", ["operator", "timestamp", "id"], ["conversation_id"]), + ("ix_AttackResultEntries_operation_timestamp_id", ["operation", "timestamp", "id"], ["conversation_id"]), +) + def upgrade() -> None: """Widen conversation ids to match ``PromptMemoryEntries``.""" if op.get_bind().dialect.name == "mssql": - op.execute(sa.text(_WIDEN_CONVERSATION_IDS_SQL)) + op.execute(sa.text(_RESIZE_CONVERSATION_KEY_SQL.format(length=128))) + op.execute(sa.text("ALTER TABLE [AttackResultEntries] ALTER COLUMN [conversation_id] VARCHAR(128) NOT NULL")) return _set_sqlite_declared_length(old_length=36, new_length=128) def downgrade() -> None: """ - Keep SQL Server's wider columns and restore SQLite's declared types. + Restore the 36-character columns earlier releases' schema checks expect. - Earlier revisions read and write ids of up to 36 characters in a 128-character - SQL Server column unchanged, while narrowing would truncate or reject ids stored - since the upgrade. SQLite never enforced the length, so its declaration is - restored without touching stored ids. + SQL Server refuses when longer ids are stored, since narrowing would lose them. + SQLite never enforced the length, so only its declaration is restored. """ if op.get_bind().dialect.name == "sqlite": _set_sqlite_declared_length(old_length=128, new_length=36) + return + op.execute(sa.text(_REJECT_LONG_CONVERSATION_IDS_SQL)) + op.execute(sa.text(_RESIZE_CONVERSATION_KEY_SQL.format(length=36))) + for name, _, _ in _ATTACK_RESULT_INDEXES: + op.drop_index(name, table_name="AttackResultEntries") + op.alter_column( + "AttackResultEntries", + "conversation_id", + existing_type=sa.String(length=128), + type_=sa.String(length=36), + existing_nullable=False, + ) + for name, columns, include in _ATTACK_RESULT_INDEXES: + op.create_index(name, "AttackResultEntries", columns, mssql_include=include) def _set_sqlite_declared_length(*, old_length: int, new_length: int) -> None: diff --git a/pyrit/memory/memory_interface.py b/pyrit/memory/memory_interface.py index 11eaf6636d..8c2c3bd64d 100644 --- a/pyrit/memory/memory_interface.py +++ b/pyrit/memory/memory_interface.py @@ -124,7 +124,6 @@ SeedEntry, SeedIdentifierEntry, TargetIdentifierEntry, - validate_conversation_id_length, ) from pyrit.memory.memory_session import MemorySession, _begin_sqlite_write, _lock_observations from pyrit.memory.storage import ( @@ -2315,8 +2314,8 @@ def _validate_persistable_conversation_ids(*, message_pieces: Sequence[MessagePi (``not_in_memory`` pieces should already be filtered out). Raises: - ValueError: If any piece has a ``None``, empty, whitespace-only, or - too-long ``conversation_id``. + ValueError: If any piece has a ``None``, empty, or whitespace-only + ``conversation_id``. """ for piece in message_pieces: if piece.conversation_id is None or not piece.conversation_id.strip(): @@ -2324,7 +2323,6 @@ def _validate_persistable_conversation_ids(*, message_pieces: Sequence[MessagePi f"MessagePiece {piece.id} has no conversation_id. A conversation_id must be set by " "the caller before a piece is persisted; the memory layer does not generate one." ) - validate_conversation_id_length(piece.conversation_id) def _insert_conversation(self, *, conversation: Conversation) -> None: """ diff --git a/pyrit/memory/memory_models.py b/pyrit/memory/memory_models.py index 59ca8fba02..7fe01702b7 100644 --- a/pyrit/memory/memory_models.py +++ b/pyrit/memory/memory_models.py @@ -34,6 +34,7 @@ Mapped, mapped_column, relationship, + validates, ) from sqlalchemy.types import Uuid @@ -101,27 +102,30 @@ #: Longest conversation_id any table stores. Conversation ids are free-form #: caller-supplied strings, not necessarily UUIDs, so every table that stores one -#: uses this bound. SQL Server enforces it; SQLite does not, so -#: ``validate_conversation_id_length`` checks it before any write. +#: uses this bound. SQL Server enforces it; SQLite does not, so each entry's +#: ``conversation_id`` validator checks it on every assignment, including updates. CONVERSATION_ID_MAX_LENGTH = 128 _LEGACY_UNSUPPORTED_ATTACK_IDENTIFIER_KEY = "__pyrit_legacy_unsupported_identifier__" -def validate_conversation_id_length(conversation_id: str) -> None: +def _bounded_conversation_id( # pyrit-keyword-only-exempt SQLAlchemy @validates callback + _entry: object, _key: str, conversation_id: str | None +) -> str | None: """ Reject a conversation_id that no database column can store. - Args: - conversation_id (str): The id about to be persisted. + Returns: + str | None: The unchanged id. Raises: ValueError: If the id is longer than ``CONVERSATION_ID_MAX_LENGTH``. """ - if len(conversation_id) > CONVERSATION_ID_MAX_LENGTH: + if conversation_id is not None and len(conversation_id) > CONVERSATION_ID_MAX_LENGTH: raise ValueError( f"conversation_id is {len(conversation_id)} characters; the most memory stores is " f"{CONVERSATION_ID_MAX_LENGTH}: {conversation_id[:CONVERSATION_ID_MAX_LENGTH]}..." ) + return conversation_id def _load_identifier( @@ -662,6 +666,7 @@ class PromptMemoryEntry(Base): role: Mapped[ChatMessageRole] = mapped_column(String, nullable=False) # Bounded so SQL Server accepts it as an index key. conversation_id = mapped_column(String(CONVERSATION_ID_MAX_LENGTH), nullable=False) + _validate_conversation_id = validates("conversation_id")(_bounded_conversation_id) sequence = mapped_column(INTEGER, nullable=False) timestamp = mapped_column(UTCDateTime, nullable=False) prompt_metadata: Mapped[dict[str, str | int]] = mapped_column(JSON) @@ -1412,6 +1417,7 @@ class ConversationEntry(Base): __table_args__ = {"extend_existing": True} conversation_id = mapped_column(String(CONVERSATION_ID_MAX_LENGTH), primary_key=True, nullable=False) + _validate_conversation_id = validates("conversation_id")(_bounded_conversation_id) target_identifier: Mapped[dict[str, str] | None] = mapped_column(JSON, nullable=True) #: Foreign key to the content-addressed ``TargetIdentifiers`` row. Nullable: #: a conversation may be registered without a known target. @@ -1439,7 +1445,6 @@ def __init__(self, *, conversation: Conversation) -> None: Args: conversation (Conversation): The conversation metadata to persist. """ - validate_conversation_id_length(conversation.conversation_id) self.conversation_id = conversation.conversation_id self.target_identifier = conversation.target_identifier.model_dump() if conversation.target_identifier else None self.target_identifier_hash = conversation.target_identifier.hash if conversation.target_identifier else None @@ -2317,6 +2322,7 @@ class AttackResultEntry(Base): ) id = mapped_column(CustomUUID, nullable=False, primary_key=True) conversation_id = mapped_column(String(CONVERSATION_ID_MAX_LENGTH), nullable=False) + _validate_conversation_id = validates("conversation_id")(_bounded_conversation_id) objective = mapped_column(Unicode, nullable=False) atomic_attack_identifier: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True) atomic_attack_identifier_hash: Mapped[str | None] = mapped_column( @@ -2412,7 +2418,6 @@ def __init__(self, *, entry: AttackResult) -> None: Raises: ValueError: If mutated legacy attribution labels are invalid or conflict. """ - validate_conversation_id_length(entry.conversation_id) self.id = uuid.UUID(entry.attack_result_id) self.conversation_id = entry.conversation_id self.objective = entry.objective diff --git a/tests/unit/memory/memory_interface/test_interface_scenario_results.py b/tests/unit/memory/memory_interface/test_interface_scenario_results.py index 723b8f29b2..c5eb602303 100644 --- a/tests/unit/memory/memory_interface/test_interface_scenario_results.py +++ b/tests/unit/memory/memory_interface/test_interface_scenario_results.py @@ -300,13 +300,15 @@ async def test_attack_order_preserved(sqlite_instance: MemoryInterface): sid = scenario_result.id # Insert in a specific order; hydration must surface them in the same order. + # Explicit timestamps keep coarse clocks (Windows before Python 3.13) from tying. + started = datetime.now(tz=UTC) attack_results = [ _make_attack_result_for_scenario( scenario_result_id=sid, atomic_attack_name="Attack1", objective_index=i, conversation_id=f"conv_{i}", - ) + ).model_copy(update={"timestamp": started + timedelta(seconds=i)}) for i in range(5) ] for ar in attack_results: @@ -847,7 +849,10 @@ async def test_get_scenario_results_loads_attack_results_via_foreign_key( sid = scenario_result.id ar1 = _make_attack_result_for_scenario(scenario_result_id=sid, atomic_attack_name="a", objective_index=0) - ar2 = _make_attack_result_for_scenario(scenario_result_id=sid, atomic_attack_name="a", objective_index=1) + # Hydration orders by timestamp, which a coarse clock can leave tied. + ar2 = _make_attack_result_for_scenario( + scenario_result_id=sid, atomic_attack_name="a", objective_index=1 + ).model_copy(update={"timestamp": ar1.timestamp + timedelta(seconds=1)}) ar3 = _make_attack_result_for_scenario(scenario_result_id=sid, atomic_attack_name="b", objective_index=0) (await sqlite_instance.add_attack_results_to_memory_async(attack_results=[ar1, ar2, ar3])) diff --git a/tests/unit/memory/test_conversation_id_length_migration.py b/tests/unit/memory/test_conversation_id_length_migration.py index bb87d7f699..bd32dcf904 100644 --- a/tests/unit/memory/test_conversation_id_length_migration.py +++ b/tests/unit/memory/test_conversation_id_length_migration.py @@ -27,18 +27,35 @@ def test_every_table_stores_the_longest_conversation_id(entry_type: type) -> Non assert entry_type.__table__.c.conversation_id.type.length == CONVERSATION_ID_MAX_LENGTH -def test_sql_server_migration_rebuilds_the_conversation_key_and_widens_both_tables() -> None: +def _sql_server_ddl(step: str) -> str: output = io.StringIO() context = MigrationContext.configure(dialect_name="mssql", opts={"as_sql": True, "output_buffer": output}) with Operations.context(context): - importlib.import_module(_MIGRATION).upgrade() - ddl = output.getvalue() + getattr(importlib.import_module(_MIGRATION), step)() + return output.getvalue() + + +def test_sql_server_migration_rebuilds_the_conversation_key_and_widens_both_tables() -> None: + ddl = _sql_server_ddl("upgrade") assert "DROP CONSTRAINT ' + QUOTENAME(@pk)" in ddl assert "ALTER TABLE [Conversations] ALTER COLUMN [conversation_id] VARCHAR(128) NOT NULL" in ddl assert "PRIMARY KEY '\n + @pk_type" in ddl assert "ALTER TABLE [AttackResultEntries] ALTER COLUMN [conversation_id] VARCHAR(128) NOT NULL" in ddl +def test_sql_server_downgrade_refuses_long_ids_then_restores_both_columns_and_indexes() -> None: + ddl = _sql_server_ddl("downgrade") + guard = ddl.index("DATALENGTH([conversation_id]) > 36") + narrowed_key = ddl.index("ALTER TABLE [Conversations] ALTER COLUMN [conversation_id] VARCHAR(36) NOT NULL") + dropped = ddl.index("DROP INDEX [ix_AttackResultEntries_conversation_timestamp_id]") + narrowed = ddl.index("ALTER TABLE [AttackResultEntries] ALTER COLUMN conversation_id VARCHAR(36) NOT NULL") + recreated = ddl.index( + "CREATE INDEX [ix_AttackResultEntries_operation_timestamp_id] ON [AttackResultEntries] " + "(operation, timestamp, id) INCLUDE (conversation_id)" + ) + assert guard < narrowed_key < dropped < narrowed < recreated + + async def test_memory_accepts_the_longest_conversation_id(sqlite_instance: SQLiteMemory) -> None: conversation_id = "c" * CONVERSATION_ID_MAX_LENGTH await sqlite_instance.add_conversation_to_memory_async(conversation=Conversation(conversation_id=conversation_id)) @@ -65,6 +82,35 @@ async def test_memory_rejects_a_conversation_id_no_table_can_store(sqlite_instan ) +async def test_memory_rejects_updates_to_a_conversation_id_no_table_can_store(sqlite_instance: SQLiteMemory) -> None: + too_long = "c" * (CONVERSATION_ID_MAX_LENGTH + 1) + await sqlite_instance.add_message_pieces_to_memory_async( + message_pieces=[MessagePiece(conversation_id="kept", role="user", original_value="hi")] + ) + result = AttackResult(conversation_id="kept", objective="o", outcome=AttackOutcome.UNDETERMINED) + await sqlite_instance.add_attack_results_to_memory_async(attack_results=[result]) + + with pytest.raises(ValueError, match="conversation_id is 129 characters"): + await sqlite_instance.update_prompt_entries_by_conversation_id_async( + conversation_id="kept", update_fields={"conversation_id": too_long} + ) + with pytest.raises(ValueError, match="conversation_id is 129 characters"): + await sqlite_instance.update_attack_result_async( + conversation_id="kept", update_fields={"conversation_id": too_long} + ) + with pytest.raises(ValueError, match="conversation_id is 129 characters"): + await sqlite_instance.update_attack_result_conditionally_async( + attack_result_id=result.attack_result_id, + expected_fields={}, + update_fields={"conversation_id": too_long}, + ) + + messages = await sqlite_instance.get_conversation_messages_async(conversation_id="kept") + results = await sqlite_instance.get_attack_results_async(attack_result_ids=[result.attack_result_id]) + assert [message.message_pieces[0].original_value for message in messages] == ["hi"] + assert [stored.conversation_id for stored in results] == ["kept"] + + def _conversation_id_lengths(connection: sa.Connection) -> dict[str, int]: return { table: next( diff --git a/tests/unit/scenario/core/test_scenario_partial_results.py b/tests/unit/scenario/core/test_scenario_partial_results.py index 40ab898cb4..1350b8d4ad 100644 --- a/tests/unit/scenario/core/test_scenario_partial_results.py +++ b/tests/unit/scenario/core/test_scenario_partial_results.py @@ -301,9 +301,8 @@ async def mock_run(*args, **kwargs): assert scenario_results[0].error_type == "ScenarioPartialFailureException" assert scenario_results[0].error_message.endswith("Caused by RuntimeError: Failed obj3") saved_results = scenario_results[0].attack_results["partial_save_attack"] - assert len(saved_results) == 2 - assert saved_results[0].objective == "obj1" - assert saved_results[1].objective == "obj2" + # Results are ordered by timestamp, which a coarse clock can leave tied. + assert sorted(result.objective for result in saved_results) == ["obj1", "obj2"] async def test_failure_before_worker_retries_before_marking_failed(self, mock_objective_target): atomic_attack = create_mock_atomic_attack("never_started", ["obj1"])