diff --git a/build_scripts/check_sqlserver_migrations.py b/build_scripts/check_sqlserver_migrations.py index 9ed0dcefc1..07f8701d23 100644 --- a/build_scripts/check_sqlserver_migrations.py +++ b/build_scripts/check_sqlserver_migrations.py @@ -9,6 +9,7 @@ from contextlib import contextmanager import sqlalchemy as sa +from alembic import command from alembic.script import ScriptDirectory from pyrit.memory.migration import ( @@ -46,13 +47,19 @@ def _temporary_database() -> Generator[sa.Engine, None, None]: def main() -> None: - """Apply all migrations to a fresh database and validate the resulting schema.""" + """Apply all migrations to a fresh database, round-trip the newest one, and validate the schema.""" with _temporary_database() as engine: with engine.connect() as connection: if sa.inspect(connection).get_table_names(): raise RuntimeError("SQL Server migration validation requires a fresh database.") run_schema_migrations(engine=engine) check_schema_migrations(engine=engine) + # Round-trip the newest migration so its SQL Server downgrade runs too. + with engine.begin() as connection: + config = _make_config(connection=connection) + command.downgrade(config, "-1") + command.upgrade(config, "head") + check_schema_migrations(engine=engine) with engine.connect() as connection: heads = ScriptDirectory.from_config(_make_config(connection=connection)).get_heads() versions = ( diff --git a/pyrit/backend/services/target_service.py b/pyrit/backend/services/target_service.py index 2d53e879c2..6066c95253 100644 --- a/pyrit/backend/services/target_service.py +++ b/pyrit/backend/services/target_service.py @@ -176,6 +176,14 @@ async def detect_capabilities_async(self, *, target_registry_name: str, expected try: results.append(await self._probe_capabilities_async(source)) except (CapabilityDiscoveryError, ValueError) as exc: + logger.warning( + "Capability detection failed for target %r (child %d, %s): %s", + target_registry_name, + index + 1, + type(source).__name__, + exc, + exc_info=exc, + ) if isinstance(target, RoundRobinTarget): error_type = ( CapabilityDiscoveryError if isinstance(exc, CapabilityDiscoveryError) else ValueError diff --git a/pyrit/memory/alembic/versions/0d3592c62ad8_widen_conversation_ids.py b/pyrit/memory/alembic/versions/0d3592c62ad8_widen_conversation_ids.py new file mode 100644 index 0000000000..a8c6073cef --- /dev/null +++ b/pyrit/memory/alembic/versions/0d3592c62ad8_widen_conversation_ids.py @@ -0,0 +1,138 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Store conversation ids up to 128 characters in every table. + +``Conversations.conversation_id`` and ``AttackResultEntries.conversation_id`` were +``VARCHAR(36)``, sized for a bare UUID, while ``PromptMemoryEntries`` already stores +128. Conversation ids are free-form, so SQL Server rejected longer ids such as the +capability-discovery probes' ``capability-probe-``. SQLite does not enforce +string lengths; its declared types are updated so the schema matches the models. + +Revision ID: 0d3592c62ad8 +Revises: 3e55dd099e38 +Create Date: 2026-11-20 18:00:00.000000 +""" + +import re +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "0d3592c62ad8" +down_revision: str | Sequence[str] | None = "3e55dd099e38" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +_TABLES = ("Conversations", "AttackResultEntries") + +# SQL Server cannot alter a primary-key column in place, so the key is dropped and +# re-created with its original name and clustering. +_RESIZE_CONVERSATION_KEY_SQL = """ +DECLARE @pk sysname, @pk_type nvarchar(60), @sql nvarchar(max); +SELECT @pk = key_constraint.name, @pk_type = idx.type_desc +FROM sys.key_constraints AS key_constraint +JOIN sys.indexes AS idx + ON idx.object_id = key_constraint.parent_object_id AND idx.index_id = key_constraint.unique_index_id +WHERE key_constraint.parent_object_id = OBJECT_ID(N'Conversations') AND key_constraint.type = 'PK'; +IF @pk IS NULL THROW 50000, 'Conversations primary key is missing.', 1; +SET @sql = N'ALTER TABLE [Conversations] DROP CONSTRAINT ' + QUOTENAME(@pk); +EXEC sp_executesql @sql; +ALTER TABLE [Conversations] ALTER COLUMN [conversation_id] VARCHAR({length}) NOT NULL; +SET @sql = N'ALTER TABLE [Conversations] ADD CONSTRAINT ' + QUOTENAME(@pk) + N' PRIMARY KEY ' + + @pk_type + N' ([conversation_id])'; +EXEC sp_executesql @sql; +""" + +_REJECT_LONG_CONVERSATION_IDS_SQL = """ +IF EXISTS (SELECT 1 FROM [Conversations] WHERE DATALENGTH([conversation_id]) > 36) + OR EXISTS (SELECT 1 FROM [AttackResultEntries] WHERE DATALENGTH([conversation_id]) > 36) + THROW 50000, 'Conversation ids longer than 36 characters are stored; earlier releases cannot hold them.', 1; +""" + +# SQL Server widens an indexed VARCHAR in place but cannot narrow one, so the +# downgrade re-creates the AttackResultEntries indexes that cover conversation_id. +_ATTACK_RESULT_INDEXES = ( + ("ix_AttackResultEntries_conversation_timestamp_id", ["conversation_id", "timestamp", "id"], []), + ("ix_AttackResultEntries_operator_timestamp_id", ["operator", "timestamp", "id"], ["conversation_id"]), + ("ix_AttackResultEntries_operation_timestamp_id", ["operation", "timestamp", "id"], ["conversation_id"]), +) + + +def upgrade() -> None: + """Widen conversation ids to match ``PromptMemoryEntries``.""" + if op.get_bind().dialect.name == "mssql": + op.execute(sa.text(_RESIZE_CONVERSATION_KEY_SQL.format(length=128))) + op.execute(sa.text("ALTER TABLE [AttackResultEntries] ALTER COLUMN [conversation_id] VARCHAR(128) NOT NULL")) + return + _set_sqlite_declared_length(old_length=36, new_length=128) + + +def downgrade() -> None: + """ + Restore the 36-character columns earlier releases' schema checks expect. + + SQL Server refuses when longer ids are stored, since narrowing would lose them. + SQLite never enforced the length, so only its declaration is restored. + """ + if op.get_bind().dialect.name == "sqlite": + _set_sqlite_declared_length(old_length=128, new_length=36) + return + op.execute(sa.text(_REJECT_LONG_CONVERSATION_IDS_SQL)) + op.execute(sa.text(_RESIZE_CONVERSATION_KEY_SQL.format(length=36))) + for name, _, _ in _ATTACK_RESULT_INDEXES: + op.drop_index(name, table_name="AttackResultEntries") + op.alter_column( + "AttackResultEntries", + "conversation_id", + existing_type=sa.String(length=128), + type_=sa.String(length=36), + existing_nullable=False, + ) + for name, columns, include in _ATTACK_RESULT_INDEXES: + op.create_index(name, "AttackResultEntries", columns, mssql_include=include) + + +def _set_sqlite_declared_length(*, old_length: int, new_length: int) -> None: + """ + Change the declared ``conversation_id`` length without rebuilding either table. + + SQLite ignores the length and stores the same TEXT values either way, so this is + the schema-only edit SQLite documents for changes that leave stored content valid + (https://www.sqlite.org/lang_altertable.html#otheralter). A rebuild would have to + re-create ``AttackResultEntries``' generated analytics column and would rewrite + its column-level foreign keys, which earlier downgrades drop in place. + + Raises: + RuntimeError: If a table does not declare the expected column exactly once, + or the database fails its integrity check afterwards. + """ + bind = op.get_bind() + pattern = re.compile(rf"(\bconversation_id\s+)VARCHAR\({old_length}\)", re.IGNORECASE) + replacements: dict[str, str] = {} + for table_name in _TABLES: + sql = bind.execute( + sa.text("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = :name"), {"name": table_name} + ).scalar_one() + updated, count = pattern.subn(rf"\g<1>VARCHAR({new_length})", sql) + if count != 1: + raise RuntimeError(f"{table_name} does not declare conversation_id as VARCHAR({old_length}).") + replacements[table_name] = updated + schema_version = bind.execute(sa.text("PRAGMA schema_version")).scalar_one() + bind.execute(sa.text("PRAGMA writable_schema = ON")) + try: + for table_name, sql in replacements.items(): + bind.execute( + sa.text("UPDATE sqlite_master SET sql = :sql WHERE type = 'table' AND name = :name"), + {"sql": sql, "name": table_name}, + ) + # A new schema version makes every connection reload the edited definitions. + bind.execute(sa.text(f"PRAGMA schema_version = {int(schema_version) + 1}")) + finally: + bind.execute(sa.text("PRAGMA writable_schema = OFF")) + result = bind.execute(sa.text("PRAGMA integrity_check")).scalar_one() + if result != "ok": + raise RuntimeError(f"SQLite integrity check failed after widening conversation ids: {result}") diff --git a/pyrit/memory/memory_models.py b/pyrit/memory/memory_models.py index 29e76220cf..7fe01702b7 100644 --- a/pyrit/memory/memory_models.py +++ b/pyrit/memory/memory_models.py @@ -34,6 +34,7 @@ Mapped, mapped_column, relationship, + validates, ) from sqlalchemy.types import Uuid @@ -98,9 +99,35 @@ # Default pyrit_version for database records created before version tracking was added LEGACY_PYRIT_VERSION = "<0.10.0" + +#: Longest conversation_id any table stores. Conversation ids are free-form +#: caller-supplied strings, not necessarily UUIDs, so every table that stores one +#: uses this bound. SQL Server enforces it; SQLite does not, so each entry's +#: ``conversation_id`` validator checks it on every assignment, including updates. +CONVERSATION_ID_MAX_LENGTH = 128 _LEGACY_UNSUPPORTED_ATTACK_IDENTIFIER_KEY = "__pyrit_legacy_unsupported_identifier__" +def _bounded_conversation_id( # pyrit-keyword-only-exempt SQLAlchemy @validates callback + _entry: object, _key: str, conversation_id: str | None +) -> str | None: + """ + Reject a conversation_id that no database column can store. + + Returns: + str | None: The unchanged id. + + Raises: + ValueError: If the id is longer than ``CONVERSATION_ID_MAX_LENGTH``. + """ + if conversation_id is not None and len(conversation_id) > CONVERSATION_ID_MAX_LENGTH: + raise ValueError( + f"conversation_id is {len(conversation_id)} characters; the most memory stores is " + f"{CONVERSATION_ID_MAX_LENGTH}: {conversation_id[:CONVERSATION_ID_MAX_LENGTH]}..." + ) + return conversation_id + + def _load_identifier( stored: dict[str, Any] | None, *, @@ -637,9 +664,9 @@ class PromptMemoryEntry(Base): ) id = mapped_column(CustomUUID, nullable=False, primary_key=True) role: Mapped[ChatMessageRole] = mapped_column(String, nullable=False) - # Bounded so SQL Server accepts it as an index key. 128 rather than 36 because - # conversation_id is a free-form caller-supplied string, not necessarily a UUID. - conversation_id = mapped_column(String(128), nullable=False) + # Bounded so SQL Server accepts it as an index key. + conversation_id = mapped_column(String(CONVERSATION_ID_MAX_LENGTH), nullable=False) + _validate_conversation_id = validates("conversation_id")(_bounded_conversation_id) sequence = mapped_column(INTEGER, nullable=False) timestamp = mapped_column(UTCDateTime, nullable=False) prompt_metadata: Mapped[dict[str, str | int]] = mapped_column(JSON) @@ -1389,7 +1416,8 @@ class ConversationEntry(Base): __tablename__ = "Conversations" __table_args__ = {"extend_existing": True} - conversation_id = mapped_column(String(36), primary_key=True, nullable=False) + conversation_id = mapped_column(String(CONVERSATION_ID_MAX_LENGTH), primary_key=True, nullable=False) + _validate_conversation_id = validates("conversation_id")(_bounded_conversation_id) target_identifier: Mapped[dict[str, str] | None] = mapped_column(JSON, nullable=True) #: Foreign key to the content-addressed ``TargetIdentifiers`` row. Nullable: #: a conversation may be registered without a known target. @@ -2293,7 +2321,8 @@ class AttackResultEntry(Base): {"extend_existing": True}, ) id = mapped_column(CustomUUID, nullable=False, primary_key=True) - conversation_id = mapped_column(String(36), nullable=False) + conversation_id = mapped_column(String(CONVERSATION_ID_MAX_LENGTH), nullable=False) + _validate_conversation_id = validates("conversation_id")(_bounded_conversation_id) objective = mapped_column(Unicode, nullable=False) atomic_attack_identifier: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True) atomic_attack_identifier_hash: Mapped[str | None] = mapped_column( diff --git a/pyrit/prompt_target/common/discover_target_capabilities.py b/pyrit/prompt_target/common/discover_target_capabilities.py index 491543c063..cbefed07c8 100644 --- a/pyrit/prompt_target/common/discover_target_capabilities.py +++ b/pyrit/prompt_target/common/discover_target_capabilities.py @@ -72,6 +72,10 @@ DEFAULT_PROBE_RETRY_BACKOFF_SECONDS: float = 0.1 MAX_PROBE_RETRY_BACKOFF_SECONDS: float = 1.0 +# Longest error message a failed check reports, so a provider's full response body +# does not flood the result. +_MAX_ERROR_DETAIL_LENGTH: int = 500 + # Exceptions that are deterministic on the probe payload and will not become # valid on a retry (malformed Message, type errors, missing attributes, etc.). # These fail the probe immediately rather than wasting backoff time. @@ -96,6 +100,42 @@ class CapabilityDiscoveryError(RuntimeError): """Discovery could not determine support because a probe failed.""" +def _status_code(error: BaseException) -> int | None: + """ + Return the HTTP status code an SDK error carries, if any. + + Returns: + int | None: The status code, or None when the error has none. + """ + status_code = getattr(error, "status_code", None) + if status_code is None: + status_code = getattr(getattr(error, "response", None), "status_code", None) + return status_code if isinstance(status_code, int) else None + + +def _describe_error(error: BaseException) -> str: + """ + Summarize ``error`` on one line so a failed check names its cause. + + Database errors report the driver's message instead of the full statement + and its parameters. + + Returns: + str: The error type, HTTP status when known, and the first line of its message. + """ + if isinstance(error, CapabilityDiscoveryError): + return str(error) + original = getattr(error, "orig", None) + source = original if isinstance(original, BaseException) else error + lines = str(source).strip().splitlines() + text = lines[0].strip() if lines else "" + if len(text) > _MAX_ERROR_DETAIL_LENGTH: + text = text[:_MAX_ERROR_DETAIL_LENGTH].rstrip() + "..." + status_code = _status_code(error) + kind = type(error).__name__ if status_code is None else f"{type(error).__name__} (HTTP {status_code})" + return f"{kind}: {text}" if text else kind + + def _raise_if_inconclusive(*, error: Exception, label: str) -> None: """ Reject operational failures when discovery will be applied automatically. @@ -103,11 +143,14 @@ def _raise_if_inconclusive(*, error: Exception, label: str) -> None: Raises: CapabilityDiscoveryError: A strict probe failed without evidence of rejection. """ - status_code = getattr(error, "status_code", None) - if status_code is None: - status_code = getattr(getattr(error, "response", None), "status_code", None) - if _RAISE_ON_PROBE_ERROR.get() and not (isinstance(error, ValueError) or status_code in (400, 422)): - raise CapabilityDiscoveryError(f"{label} failed; capabilities were not changed.") from error + if not _RAISE_ON_PROBE_ERROR.get(): + return + if isinstance(error, CapabilityDiscoveryError): + raise CapabilityDiscoveryError(f"{label}: {error}") from error + if not (isinstance(error, ValueError) or _status_code(error) in (400, 422)): + raise CapabilityDiscoveryError( + f"{label} failed: {_describe_error(error)}; capabilities were not changed." + ) from error def _json_enforcing_target_types() -> tuple[type[PromptTarget], ...]: @@ -268,6 +311,7 @@ async def _send_and_check_async( timeout_s: float, retries: int = 1, label: str = "Capability probe", + failures: list[str] | None = None, ) -> bool: """ Send ``message`` and report whether the call succeeded cleanly. @@ -291,6 +335,8 @@ async def _send_and_check_async( Defaults to 1. label (str): Short label used in log messages. Defaults to ``"Capability probe"``. + failures (list[str] | None): When given, receives a one-line reason + for each failed send, so callers can report why a check failed. Returns: bool: ``True`` iff the call returned without raising and every response @@ -299,6 +345,11 @@ async def _send_and_check_async( ``"empty"``, ``"unknown"``) is treated as failure. An empty response list (or responses with no message pieces) is also treated as a failure. """ + + def _record_failure(error: Exception) -> None: + if failures is not None: + failures.append(f"{label}: {_describe_error(error)}") + attempts = max(1, retries + 1) last_exc: Exception | None = None for attempt in range(attempts): @@ -314,6 +365,7 @@ async def _send_and_check_async( # Deterministic on the probe payload — retrying will not help. logger.debug("%s failed with non-retryable error: %s", label, exc) _raise_if_inconclusive(error=exc, label=label) + _record_failure(exc) return False except Exception as exc: last_exc = exc @@ -324,19 +376,26 @@ async def _send_and_check_async( if not responses or any(not r.message_pieces for r in responses): logger.debug("%s returned an empty response; treating as failure", label) - _raise_if_inconclusive(error=RuntimeError("Empty response"), label=label) + empty_error = RuntimeError("The target returned an empty response.") + _raise_if_inconclusive(error=empty_error, label=label) + _record_failure(empty_error) return False for response in responses: for piece in response.message_pieces: if piece.response_error != "none": logger.debug("%s returned error response: %s", label, piece.converted_value) - _raise_if_inconclusive(error=RuntimeError("Error response"), label=label) + response_error = RuntimeError( + f"The target returned a '{piece.response_error}' error response: {piece.converted_value}" + ) + _raise_if_inconclusive(error=response_error, label=label) + _record_failure(response_error) return False return True logger.info("%s exhausted %d attempt(s); last error: %s", label, attempts, last_exc) if last_exc is not None: _raise_if_inconclusive(error=last_exc, label=label) + _record_failure(last_exc) return False @@ -400,7 +459,9 @@ async def _probe_system_prompt_async(target: PromptTarget, timeout_s: float, ret except Exception as exc: logger.debug("System-prompt probe could not seed system message: %s", exc) if _RAISE_ON_PROBE_ERROR.get(): - raise CapabilityDiscoveryError("Could not prepare the system-prompt probe.") from exc + raise CapabilityDiscoveryError( + f"Could not prepare the system-prompt probe: {_describe_error(exc)}; capabilities were not changed." + ) from exc return False user_piece = _user_text_piece(value="hi", conversation_id=conversation_id) return await _send_and_check_async( @@ -506,7 +567,9 @@ async def _probe_multi_turn_async(target: PromptTarget, timeout_s: float, retrie except Exception as exc: logger.debug("Multi-turn probe could not seed conversation history: %s", exc) if _RAISE_ON_PROBE_ERROR.get(): - raise CapabilityDiscoveryError("Could not prepare the multi-turn probe.") from exc + raise CapabilityDiscoveryError( + f"Could not prepare the multi-turn probe: {_describe_error(exc)}; capabilities were not changed." + ) from exc return False second = _user_text_piece(value="What did I just tell you?", conversation_id=conversation_id) @@ -894,18 +957,28 @@ async def _check_control_request_async( key=lambda combination: (combination != frozenset({"text"}), len(combination), sorted(combination)), ) assets = test_assets if test_assets is not None else DEFAULT_TEST_ASSETS + failures: list[str] = [] with _permissive_configuration(target=target): for combination in combinations: try: message = await asyncio.to_thread(_create_test_message, modalities=combination, test_assets=assets) except (FileNotFoundError, ValueError) as exc: logger.info("Cannot prepare control input %s: %s", sorted(combination), exc) + failures.append(f"Could not prepare {sorted(combination)}: {_describe_error(exc)}") continue if await _send_and_check_async( - target=target, message=message, timeout_s=timeout_s, retries=retries, label="Control request" + target=target, + message=message, + timeout_s=timeout_s, + retries=retries, + label=f"Control request {sorted(combination)}", + failures=failures, ): return - raise CapabilityDiscoveryError("No declared input passed a control request; capabilities were not changed.") + reasons = "; ".join(failures) if failures else "the target declares no input types" + raise CapabilityDiscoveryError( + f"No declared input passed a control request ({reasons}); capabilities were not changed." + ) async def discover_target_capabilities_async( diff --git a/tests/integration/memory/test_azure_sql_memory_integration.py b/tests/integration/memory/test_azure_sql_memory_integration.py index 790b89b589..d8c44da3b9 100644 --- a/tests/integration/memory/test_azure_sql_memory_integration.py +++ b/tests/integration/memory/test_azure_sql_memory_integration.py @@ -17,6 +17,8 @@ PromptMemoryEntry, ) from pyrit.memory.memory_models import ( + CONVERSATION_ID_MAX_LENGTH, + ConversationEntry, ScenarioResultEntry, ) from pyrit.models import ( @@ -24,6 +26,7 @@ AttackOutcome, AttackResult, ComponentIdentifier, + Conversation, MessagePiece, ScenarioIdentifier, ScenarioResult, @@ -152,6 +155,10 @@ def cleanup(session: Session) -> None: PromptMemoryEntry.conversation_id.in_(conversation_ids) ).delete(synchronize_session=False) + session.query(ConversationEntry).filter( + ConversationEntry.conversation_id.in_(conversation_ids) + ).delete(synchronize_session=False) + session.commit() except SQLAlchemyError as e: session.rollback() @@ -369,6 +376,37 @@ async def test_metadata_string_filters_exclude_containers_async( await azuresql_instance.remove_seeds_from_memory_async(dataset_name=test_id) +async def test_longest_conversation_id_persists_in_every_table(azuresql_instance: AzureSQLMemory): + """SQL Server stores a conversation_id of the full allowed length, not only a bare UUID.""" + test_id = generate_test_id() + conversation_id = f"long-conversation-{test_id}-".ljust(CONVERSATION_ID_MAX_LENGTH, "x") + + async with cleanup_conversation_data_async(azuresql_instance, [conversation_id]): + await azuresql_instance.add_conversation_to_memory_async( + conversation=Conversation(conversation_id=conversation_id) + ) + await azuresql_instance.add_message_pieces_to_memory_async( + message_pieces=[MessagePiece(conversation_id=conversation_id, role="user", original_value="hi")] + ) + await azuresql_instance.add_attack_results_to_memory_async( + attack_results=[ + AttackResult( + conversation_id=conversation_id, + objective="Store a long conversation id", + atomic_attack_identifier=get_test_atomic_attack_identifier(), + outcome=AttackOutcome.UNDETERMINED, + labels={"long_conversation_id_test": test_id}, + ) + ] + ) + + messages = await azuresql_instance.get_conversation_messages_async(conversation_id=conversation_id) + results = await azuresql_instance.get_attack_results_async(labels={"long_conversation_id_test": test_id}) + + assert [message.message_pieces[0].original_value for message in messages] == ["hi"] + assert [result.conversation_id for result in results] == [conversation_id] + + async def test_get_attack_results_by_labels(azuresql_instance: AzureSQLMemory): """ Integration test for SQL Azure JSON filtering on labels. diff --git a/tests/unit/memory/memory_interface/test_interface_scenario_results.py b/tests/unit/memory/memory_interface/test_interface_scenario_results.py index 723b8f29b2..c5eb602303 100644 --- a/tests/unit/memory/memory_interface/test_interface_scenario_results.py +++ b/tests/unit/memory/memory_interface/test_interface_scenario_results.py @@ -300,13 +300,15 @@ async def test_attack_order_preserved(sqlite_instance: MemoryInterface): sid = scenario_result.id # Insert in a specific order; hydration must surface them in the same order. + # Explicit timestamps keep coarse clocks (Windows before Python 3.13) from tying. + started = datetime.now(tz=UTC) attack_results = [ _make_attack_result_for_scenario( scenario_result_id=sid, atomic_attack_name="Attack1", objective_index=i, conversation_id=f"conv_{i}", - ) + ).model_copy(update={"timestamp": started + timedelta(seconds=i)}) for i in range(5) ] for ar in attack_results: @@ -847,7 +849,10 @@ async def test_get_scenario_results_loads_attack_results_via_foreign_key( sid = scenario_result.id ar1 = _make_attack_result_for_scenario(scenario_result_id=sid, atomic_attack_name="a", objective_index=0) - ar2 = _make_attack_result_for_scenario(scenario_result_id=sid, atomic_attack_name="a", objective_index=1) + # Hydration orders by timestamp, which a coarse clock can leave tied. + ar2 = _make_attack_result_for_scenario( + scenario_result_id=sid, atomic_attack_name="a", objective_index=1 + ).model_copy(update={"timestamp": ar1.timestamp + timedelta(seconds=1)}) ar3 = _make_attack_result_for_scenario(scenario_result_id=sid, atomic_attack_name="b", objective_index=0) (await sqlite_instance.add_attack_results_to_memory_async(attack_results=[ar1, ar2, ar3])) diff --git a/tests/unit/memory/test_conversation_id_length_migration.py b/tests/unit/memory/test_conversation_id_length_migration.py new file mode 100644 index 0000000000..bd32dcf904 --- /dev/null +++ b/tests/unit/memory/test_conversation_id_length_migration.py @@ -0,0 +1,152 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Conversation ids up to the shared length limit persist in every table.""" + +import importlib +import io +import os +import tempfile + +import pytest +import sqlalchemy as sa +from alembic import command +from alembic.migration import MigrationContext +from alembic.operations import Operations + +from pyrit.memory import AttackResultEntry, PromptMemoryEntry, SQLiteMemory +from pyrit.memory.memory_models import CONVERSATION_ID_MAX_LENGTH, ConversationEntry +from pyrit.memory.migration import _make_config +from pyrit.models import AttackOutcome, AttackResult, Conversation, MessagePiece + +_MIGRATION = "pyrit.memory.alembic.versions.0d3592c62ad8_widen_conversation_ids" + + +@pytest.mark.parametrize("entry_type", [ConversationEntry, AttackResultEntry, PromptMemoryEntry]) +def test_every_table_stores_the_longest_conversation_id(entry_type: type) -> None: + assert entry_type.__table__.c.conversation_id.type.length == CONVERSATION_ID_MAX_LENGTH + + +def _sql_server_ddl(step: str) -> str: + output = io.StringIO() + context = MigrationContext.configure(dialect_name="mssql", opts={"as_sql": True, "output_buffer": output}) + with Operations.context(context): + getattr(importlib.import_module(_MIGRATION), step)() + return output.getvalue() + + +def test_sql_server_migration_rebuilds_the_conversation_key_and_widens_both_tables() -> None: + ddl = _sql_server_ddl("upgrade") + assert "DROP CONSTRAINT ' + QUOTENAME(@pk)" in ddl + assert "ALTER TABLE [Conversations] ALTER COLUMN [conversation_id] VARCHAR(128) NOT NULL" in ddl + assert "PRIMARY KEY '\n + @pk_type" in ddl + assert "ALTER TABLE [AttackResultEntries] ALTER COLUMN [conversation_id] VARCHAR(128) NOT NULL" in ddl + + +def test_sql_server_downgrade_refuses_long_ids_then_restores_both_columns_and_indexes() -> None: + ddl = _sql_server_ddl("downgrade") + guard = ddl.index("DATALENGTH([conversation_id]) > 36") + narrowed_key = ddl.index("ALTER TABLE [Conversations] ALTER COLUMN [conversation_id] VARCHAR(36) NOT NULL") + dropped = ddl.index("DROP INDEX [ix_AttackResultEntries_conversation_timestamp_id]") + narrowed = ddl.index("ALTER TABLE [AttackResultEntries] ALTER COLUMN conversation_id VARCHAR(36) NOT NULL") + recreated = ddl.index( + "CREATE INDEX [ix_AttackResultEntries_operation_timestamp_id] ON [AttackResultEntries] " + "(operation, timestamp, id) INCLUDE (conversation_id)" + ) + assert guard < narrowed_key < dropped < narrowed < recreated + + +async def test_memory_accepts_the_longest_conversation_id(sqlite_instance: SQLiteMemory) -> None: + conversation_id = "c" * CONVERSATION_ID_MAX_LENGTH + await sqlite_instance.add_conversation_to_memory_async(conversation=Conversation(conversation_id=conversation_id)) + await sqlite_instance.add_message_pieces_to_memory_async( + message_pieces=[MessagePiece(conversation_id=conversation_id, role="user", original_value="hi")] + ) + messages = await sqlite_instance.get_conversation_messages_async(conversation_id=conversation_id) + assert [message.message_pieces[0].original_value for message in messages] == ["hi"] + + +async def test_memory_rejects_a_conversation_id_no_table_can_store(sqlite_instance: SQLiteMemory) -> None: + conversation_id = "c" * (CONVERSATION_ID_MAX_LENGTH + 1) + with pytest.raises(ValueError, match="conversation_id is 129 characters"): + await sqlite_instance.add_conversation_to_memory_async( + conversation=Conversation(conversation_id=conversation_id) + ) + with pytest.raises(ValueError, match="conversation_id is 129 characters"): + await sqlite_instance.add_message_pieces_to_memory_async( + message_pieces=[MessagePiece(conversation_id=conversation_id, role="user", original_value="hi")] + ) + with pytest.raises(ValueError, match="conversation_id is 129 characters"): + AttackResultEntry( + entry=AttackResult(conversation_id=conversation_id, objective="o", outcome=AttackOutcome.UNDETERMINED) + ) + + +async def test_memory_rejects_updates_to_a_conversation_id_no_table_can_store(sqlite_instance: SQLiteMemory) -> None: + too_long = "c" * (CONVERSATION_ID_MAX_LENGTH + 1) + await sqlite_instance.add_message_pieces_to_memory_async( + message_pieces=[MessagePiece(conversation_id="kept", role="user", original_value="hi")] + ) + result = AttackResult(conversation_id="kept", objective="o", outcome=AttackOutcome.UNDETERMINED) + await sqlite_instance.add_attack_results_to_memory_async(attack_results=[result]) + + with pytest.raises(ValueError, match="conversation_id is 129 characters"): + await sqlite_instance.update_prompt_entries_by_conversation_id_async( + conversation_id="kept", update_fields={"conversation_id": too_long} + ) + with pytest.raises(ValueError, match="conversation_id is 129 characters"): + await sqlite_instance.update_attack_result_async( + conversation_id="kept", update_fields={"conversation_id": too_long} + ) + with pytest.raises(ValueError, match="conversation_id is 129 characters"): + await sqlite_instance.update_attack_result_conditionally_async( + attack_result_id=result.attack_result_id, + expected_fields={}, + update_fields={"conversation_id": too_long}, + ) + + messages = await sqlite_instance.get_conversation_messages_async(conversation_id="kept") + results = await sqlite_instance.get_attack_results_async(attack_result_ids=[result.attack_result_id]) + assert [message.message_pieces[0].original_value for message in messages] == ["hi"] + assert [stored.conversation_id for stored in results] == ["kept"] + + +def _conversation_id_lengths(connection: sa.Connection) -> dict[str, int]: + return { + table: next( + column["type"].length + for column in sa.inspect(connection).get_columns(table) + if column["name"] == "conversation_id" + ) + for table in ("Conversations", "AttackResultEntries") + } + + +def test_sqlite_migration_round_trips_declared_lengths_and_keeps_rows() -> None: + long_id = "c" * CONVERSATION_ID_MAX_LENGTH + with tempfile.TemporaryDirectory() as temp_dir: + engine = sa.create_engine(f"sqlite:///{os.path.join(temp_dir, 'widen.db')}") + try: + with engine.begin() as connection: + config = _make_config(connection=connection) + command.upgrade(config, "3e55dd099e38") + connection.execute( + sa.text('INSERT INTO "Conversations" (conversation_id) VALUES (:id)'), {"id": "kept"} + ) + command.upgrade(config, "0d3592c62ad8") + widened = _conversation_id_lengths(connection) + connection.execute( + sa.text('INSERT INTO "Conversations" (conversation_id) VALUES (:id)'), {"id": long_id} + ) + command.downgrade(config, "3e55dd099e38") + restored = _conversation_id_lengths(connection) + kept = ( + connection.execute(sa.text('SELECT conversation_id FROM "Conversations" ORDER BY conversation_id')) + .scalars() + .all() + ) + finally: + engine.dispose() + assert widened == {"Conversations": CONVERSATION_ID_MAX_LENGTH, "AttackResultEntries": CONVERSATION_ID_MAX_LENGTH} + assert restored == {"Conversations": 36, "AttackResultEntries": 36} + assert kept == [long_id, "kept"] diff --git a/tests/unit/prompt_target/test_discover_target_capabilities.py b/tests/unit/prompt_target/test_discover_target_capabilities.py index d698498d11..46d49558f6 100644 --- a/tests/unit/prompt_target/test_discover_target_capabilities.py +++ b/tests/unit/prompt_target/test_discover_target_capabilities.py @@ -14,6 +14,7 @@ from mcp.types import Tool as MCPToolDefinition from openai.types.chat import ChatCompletion from PIL import Image +from sqlalchemy.exc import ProgrammingError from unit.async_utils import TEST_WATCHDOG_SECONDS from unit.mocks import MockPromptTarget @@ -211,11 +212,95 @@ async def test_strict_discovery_rejects_blocked_response_without_changing_capabi target = MockPromptTarget() original = target.configuration with patch.object(target, "send_prompt_async", new_callable=AsyncMock, return_value=_error_response()): - with pytest.raises(CapabilityDiscoveryError): + with pytest.raises(CapabilityDiscoveryError, match="'blocked' error response: blocked"): await discover_target_capabilities_async(target=target, raise_on_error=True, apply=True) assert target.configuration is original +@pytest.mark.usefixtures("patch_central_database") +async def test_strict_discovery_reports_the_http_failure() -> None: + target = MockPromptTarget() + request = httpx.Request("POST", "https://example.test") + error = httpx.HTTPStatusError( + "Incorrect API key provided", request=request, response=httpx.Response(status_code=401, request=request) + ) + with patch.object(target, "send_prompt_async", new_callable=AsyncMock, side_effect=error): + with pytest.raises(CapabilityDiscoveryError) as raised: + await discover_target_capabilities_async(target=target, raise_on_error=True, retries=0) + assert str(raised.value) == ( + "Control request ['text'] failed: HTTPStatusError (HTTP 401): Incorrect API key provided; " + "capabilities were not changed." + ) + + +@pytest.mark.usefixtures("patch_central_database") +async def test_strict_discovery_reports_why_no_control_request_passed() -> None: + target = MockPromptTarget() + with patch.object( + target, "send_prompt_async", new_callable=AsyncMock, side_effect=ValueError("Invalid API settings") + ): + with pytest.raises(CapabilityDiscoveryError) as raised: + await discover_target_capabilities_async(target=target, raise_on_error=True, retries=0) + assert str(raised.value) == ( + "No declared input passed a control request (Control request ['text']: ValueError: Invalid API settings); " + "capabilities were not changed." + ) + + +@pytest.mark.usefixtures("patch_central_database") +async def test_strict_discovery_names_the_capability_and_the_failed_turn() -> None: + target = MockPromptTarget() + sends = 0 + + async def send_async(*, message: Message) -> list[Message]: + nonlocal sends + sends += 1 + # The control request and the first turn succeed; the history turn fails. + if sends > 2: + raise ConnectionError("offline") + return _ok_response() + + with ( + patch.object(target, "send_prompt_async", new_callable=AsyncMock, side_effect=send_async), + patch("pyrit.prompt_target.common.discover_target_capabilities._supports_chat_probes", return_value=True), + ): + with pytest.raises(CapabilityDiscoveryError) as raised: + await discover_target_capabilities_async( + target=target, + capabilities=[CapabilityName.MULTI_TURN], + test_modalities=set(), + raise_on_error=True, + retries=0, + ) + assert str(raised.value) == ( + "supports_multi_turn: Multi-turn probe (turn 2) failed: ConnectionError: offline; " + "capabilities were not changed." + ) + + +@pytest.mark.usefixtures("patch_central_database") +async def test_strict_discovery_reports_the_database_error_that_blocked_a_probe() -> None: + target = MockPromptTarget() + error = ProgrammingError( + 'INSERT INTO "Conversations" (conversation_id) VALUES (?)', + ("capability-probe-id",), + Exception("String or binary data would be truncated in table 'Conversations'"), + ) + with ( + patch.object(target, "send_prompt_async", new_callable=AsyncMock, return_value=_ok_response()), + patch.object(target._memory, "add_conversation_to_memory_async", new_callable=AsyncMock, side_effect=error), + patch("pyrit.prompt_target.common.discover_target_capabilities._supports_chat_probes", return_value=True), + ): + with pytest.raises(CapabilityDiscoveryError) as raised: + await discover_target_capabilities_async( + target=target, capabilities=[CapabilityName.MULTI_TURN], test_modalities=set(), raise_on_error=True + ) + assert str(raised.value) == ( + "supports_multi_turn: Could not prepare the multi-turn probe: ProgrammingError: " + "String or binary data would be truncated in table 'Conversations'; capabilities were not changed." + ) + + @pytest.mark.usefixtures("patch_central_database") class TestPermissiveConfiguration: def test_replaces_and_restores_configuration(self) -> None: diff --git a/tests/unit/scenario/core/test_scenario_partial_results.py b/tests/unit/scenario/core/test_scenario_partial_results.py index 40ab898cb4..1350b8d4ad 100644 --- a/tests/unit/scenario/core/test_scenario_partial_results.py +++ b/tests/unit/scenario/core/test_scenario_partial_results.py @@ -301,9 +301,8 @@ async def mock_run(*args, **kwargs): assert scenario_results[0].error_type == "ScenarioPartialFailureException" assert scenario_results[0].error_message.endswith("Caused by RuntimeError: Failed obj3") saved_results = scenario_results[0].attack_results["partial_save_attack"] - assert len(saved_results) == 2 - assert saved_results[0].objective == "obj1" - assert saved_results[1].objective == "obj2" + # Results are ordered by timestamp, which a coarse clock can leave tied. + assert sorted(result.objective for result in saved_results) == ["obj1", "obj2"] async def test_failure_before_worker_retries_before_marking_failed(self, mock_objective_target): atomic_attack = create_mock_atomic_attack("never_started", ["obj1"])