Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions pyrit/backend/services/target_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@
RegistryNameTakenError,
registry_entry_origin,
)
from pyrit.registry.registry import Reconstructable

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -163,6 +164,12 @@ async def detect_capabilities_async(self, *, target_registry_name: str, expected
if getattr(target, "inner_targets", None) and not isinstance(target, RoundRobinTarget):
raise ValueError("Capability detection is not supported for this composite target type.")
sources = target.inner_targets if isinstance(target, RoundRobinTarget) else [target]
# Detection probes a private copy, so a target that can't be rebuilt can't be checked.
for source in sources:
if not isinstance(source, Reconstructable):
raise ValueError(
f"Capability detection is not supported for {type(source).__name__} because it can't be rebuilt."
)
snapshots = [(source, source.configuration) for source in [target, *sources]]
source_ids = {id(source) for source, _ in snapshots}
if target_registry_name in self._detecting or source_ids & self._detecting_sources:
Expand Down
23 changes: 19 additions & 4 deletions pyrit/prompt_target/common/discover_target_capabilities.py
Original file line number Diff line number Diff line change
Expand Up @@ -682,10 +682,13 @@ def _supports_chat_probes(target: PromptTarget) -> bool:
bool: Whether chat features can be sent by this adapter.
"""
from pyrit.prompt_target.litellm_chat_target import LiteLLMChatTarget
from pyrit.prompt_target.litellm_decision_target import LiteLLMDecisionTarget
from pyrit.prompt_target.openai.openai_chat_target import OpenAIChatTarget
from pyrit.prompt_target.openai.openai_response_target import OpenAIResponseTarget

return isinstance(target, (OpenAIChatTarget, OpenAIResponseTarget, LiteLLMChatTarget))
return isinstance(target, (OpenAIChatTarget, OpenAIResponseTarget, LiteLLMChatTarget)) and not isinstance(
target, LiteLLMDecisionTarget
)


async def _discover_capability_flags_async(
Expand Down Expand Up @@ -849,7 +852,12 @@ async def _discover_input_modalities_async(
queried.add(combination)
continue
try:
message = await asyncio.to_thread(_create_test_message, modalities=combination, test_assets=assets)
message = await asyncio.to_thread(
_create_test_message,
modalities=combination,
test_assets=assets,
text=target.capability_probe_text,
)
except FileNotFoundError as exc:
# Skip combinations we cannot construct a valid probe payload for.
logger.info("Skipping modality %s: %s", combination, exc)
Expand Down Expand Up @@ -897,7 +905,12 @@ async def _check_control_request_async(
with _permissive_configuration(target=target):
for combination in combinations:
try:
message = await asyncio.to_thread(_create_test_message, modalities=combination, test_assets=assets)
message = await asyncio.to_thread(
_create_test_message,
modalities=combination,
test_assets=assets,
text=target.capability_probe_text,
)
except (FileNotFoundError, ValueError) as exc:
logger.info("Cannot prepare control input %s: %s", sorted(combination), exc)
continue
Expand Down Expand Up @@ -1077,6 +1090,7 @@ def _create_test_message(
*,
modalities: frozenset[PromptDataType],
test_assets: dict[PromptDataType, str],
text: str = "test",
) -> Message:
"""
Build a minimal ``Message`` that exercises ``modalities``.
Expand All @@ -1085,6 +1099,7 @@ def _create_test_message(
modalities (frozenset[PromptDataType]): The modalities to include.
test_assets (dict[PromptDataType, str]): Mapping from non-text
modality to a file path used for the probe.
text (str): The text piece's value. Defaults to ``"test"``.

Returns:
Message: A message containing one piece per modality.
Expand All @@ -1101,7 +1116,7 @@ def _create_test_message(
pieces.append(
MessagePiece(
role="user",
original_value="test",
original_value=text,
original_value_data_type="text",
conversation_id=conversation_id,
prompt_metadata=_probe_metadata(),
Expand Down
7 changes: 6 additions & 1 deletion pyrit/prompt_target/common/prompt_target.py
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,10 @@ class PromptTarget(Identifiable):
# with the endpoint it was configured for.
COPY_LOCKED_PARAMETER_NAMES: ClassVar[frozenset[str]] = frozenset()

# Text capability discovery sends to check a text input. A target that accepts only structured text
# overrides it with a valid request.
capability_probe_text: ClassVar[str] = "test"

def __init_subclass__(cls, **kwargs: object) -> None:
"""
Validate that subclasses follow the keyword-only ``__init__`` contract.
Expand Down Expand Up @@ -679,14 +683,15 @@ def _reset_credential_status(self) -> None:

def attach_reconstruction_source(self, source: object) -> None:
"""
Report credential rejections into the target this copy was built from.
Share request pacing and credential status, not clients or behavioral configuration.

Args:
source (object): The registered target this copy was built from.
"""
if isinstance(source, PromptTarget):
self._credential_status_source = source._credential_status_owner()
self._credential_generation = source._credential_generation
self._rate_limit_source = getattr(source, "_rate_limit_source", source)

def _validate_request(self, *, normalized_conversation: list[Message]) -> None:
"""
Expand Down
34 changes: 33 additions & 1 deletion pyrit/prompt_target/litellm_decision_target.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
import json
import math
from collections.abc import Awaitable, Callable
from typing import TYPE_CHECKING, Any, TypedDict, cast
from typing import TYPE_CHECKING, Any, ClassVar, TypedDict, cast

from pyrit.exceptions import pyrit_target_retry
from pyrit.models import (
Expand Down Expand Up @@ -44,6 +44,21 @@ class LiteLLMDecisionTarget(LiteLLMChatTarget):
_SUPPORTS_TOOL_CALL_HISTORY = False
_DEFAULT_CONFIGURATION = TargetConfiguration(capabilities=TargetCapabilities())

# The model name's provider prefix chooses where LiteLLM sends the key when no endpoint is set.
COPY_LOCKED_PARAMETER_NAMES: ClassVar[frozenset[str]] = frozenset({"model_name", "endpoint", "api_key", "headers"})
capability_probe_text: ClassVar[str] = json.dumps(
{
"state": "test",
"questions": {
"probe": {
"type": "noul",
"instructions": "Is the state the word test?",
"criteria": {"true": "It is.", "false": "It is not."},
}
},
}
)

def __init__(
self,
*,
Expand Down Expand Up @@ -93,6 +108,23 @@ def _build_identifier(self) -> ComponentIdentifier:
"""
return self._create_identifier(params={"endpoint": self._endpoint, "headers": self._identifier_headers()})

def get_reconstruction_parameters(self) -> dict[str, object]:
"""
Return server-only inputs, including the resolved key or token provider.

Returns:
dict[str, object]: Constructor inputs that rebuild this target.
"""
return {
"model_name": self._model_name,
"endpoint": self._endpoint,
"api_key": self._api_key,
"headers": self._headers,
"max_requests_per_minute": self._max_requests_per_minute,
"custom_configuration": self.configuration,
"trace_config": self._trace_config,
}

@pyrit_target_retry
@limit_requests_per_minute
async def _send_prompt_to_target_async(self, *, normalized_conversation: list[Message]) -> list[Message]:
Expand Down
10 changes: 0 additions & 10 deletions pyrit/prompt_target/openai/openai_target.py
Original file line number Diff line number Diff line change
Expand Up @@ -504,16 +504,6 @@ async def cleanup_target_async(self) -> None:
if self._async_client is not None:
await self._async_client.close()

def attach_reconstruction_source(self, source: object) -> None:
"""
Share request pacing and credential status, not clients or behavioral configuration.

Args:
source (object): The registered target this copy was built from.
"""
super().attach_reconstruction_source(source)
self._rate_limit_source = getattr(source, "_rate_limit_source", source)

@property
def supports_credential_updates(self) -> bool:
"""Whether this target can replace its API key or request headers in place; not with its own HTTP client."""
Expand Down
19 changes: 19 additions & 0 deletions pyrit/prompt_target/prompt_shield_target.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ class PromptShieldTarget(PromptTarget):
server_fallback_parameters: ClassVar[Mapping[str, str | None]] = MappingProxyType({"api_key": None})
destination_parameters: ClassVar[Mapping[str, bool]] = MappingProxyType({"endpoint": True})
environment_backed_parameters: ClassVar[frozenset[str]] = frozenset({"endpoint"})
COPY_LOCKED_PARAMETER_NAMES: ClassVar[frozenset[str]] = frozenset({"endpoint", "api_key", "auth_mode"})

_endpoint: str
_api_key: str | Callable[[], str] | None
Expand Down Expand Up @@ -198,6 +199,24 @@ def get_auth_mode_parameters(cls, *, auth_mode: AuthMode) -> dict[str, object]:
"""
return {"auth_mode": auth_mode}

def get_reconstruction_parameters(self) -> dict[str, object]:
"""
Return server-only inputs, including the resolved key or token provider.

Returns:
dict[str, object]: Constructor inputs that rebuild this target.
"""
return {
"endpoint": self._endpoint,
# Reuse the resolved key or token provider, not the original credential-selection mode.
"api_key": self._api_key,
"auth_mode": "api_key",
"api_version": self._api_version,
"field": self._force_entry_field,
"max_requests_per_minute": self._max_requests_per_minute,
"custom_configuration": self.configuration,
}

def _build_identifier(self) -> ComponentIdentifier:
"""
Build the identifier with Prompt Shield-specific parameters.
Expand Down
74 changes: 71 additions & 3 deletions tests/unit/backend/test_target_detection.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
from pathlib import Path
from types import SimpleNamespace
from typing import Any
from unittest.mock import AsyncMock, patch
from unittest.mock import AsyncMock, MagicMock, patch

import pytest
from fastapi import FastAPI
Expand Down Expand Up @@ -45,7 +45,14 @@
)
from pyrit.models.catalog import RegistryEntryOrigin
from pyrit.models.component_spec import SourceInstanceSpec, TargetBinding
from pyrit.prompt_target import OpenAIChatTarget, OpenAITTSTarget, RoundRobinTarget, TargetConfiguration
from pyrit.prompt_target import (
LiteLLMDecisionTarget,
OpenAIChatTarget,
OpenAITTSTarget,
PromptShieldTarget,
RoundRobinTarget,
TargetConfiguration,
)
from pyrit.prompt_target.common.discover_target_capabilities import DEFAULT_TEST_ASSETS, CapabilityDiscoveryError
from pyrit.registry import TargetRegistry
from pyrit.registry.instance_registry import SAVED_ENTRY_METADATA_KEY
Expand Down Expand Up @@ -464,6 +471,67 @@ async def discover_async(*, target: OpenAIChatTarget, **kwargs: object) -> Targe
await target.cleanup_target_async()


async def test_detection_rebuilds_decision_target_and_sends_a_valid_control_request_async(
detection_service: tuple[TargetService, _DetectableTarget],
) -> None:
litellm = pytest.importorskip("litellm")
from litellm.types.decisions import DecisionsResponse

service, _ = detection_service
source = LiteLLMDecisionTarget(model_name="openrouter/decision", endpoint="https://example.test", api_key="key")
service._registry.instances.register(source, name="microsoft_decision")
original = source.capabilities
sdk = MagicMock(spec=litellm)
sdk.asystemone = None
sdk.adecisions = AsyncMock(
return_value=DecisionsResponse.model_validate(
{"model": "decision", "id": "probe", "answers": {"probe": {"type": "noul", "noul": 1.0}}, "usage": {}}
)
)

with patch.object(LiteLLMDecisionTarget, "_import_litellm", return_value=sdk):
result = await service.detect_capabilities_async(
target_registry_name="microsoft_decision", expected_hash=source.get_identifier().hash
)

assert result.capabilities == original
assert source.capabilities == original
sdk.adecisions.assert_awaited_once()
body = sdk.adecisions.call_args.kwargs
assert body["api_key"] == "key"
assert body["api_base"] == "https://example.test"
assert set(body["questions"]) == {"probe"}


@pytest.mark.parametrize("api_key", ["key", MagicMock(return_value="token")], ids=["key", "token_provider"])
async def test_detection_rebuilds_prompt_shield_and_keeps_its_declarations_async(
detection_service: tuple[TargetService, _DetectableTarget], api_key: str | MagicMock
) -> None:
service, _ = detection_service
source = PromptShieldTarget(endpoint="https://example.test", api_key=api_key)
service._registry.instances.register(source, name="azure_content_safety")
original = source.capabilities
response = SimpleNamespace(content=b'{"userPromptAnalysis": {"attackDetected": false}, "documentsAnalysis": []}')

with patch(
"pyrit.prompt_target.prompt_shield_target.net_utility.make_request_and_raise_if_error_async",
new=AsyncMock(return_value=response),
) as shield:
result = await service.detect_capabilities_async(
target_registry_name="azure_content_safety", expected_hash=source.get_identifier().hash
)

assert result.capabilities == original
assert source.capabilities == original
shield.assert_awaited_once()
assert shield.call_args.kwargs["request_body"] == {"userPrompt": "test", "documents": []}
headers = shield.call_args.kwargs["headers"]
if isinstance(api_key, str):
assert headers["Ocp-Apim-Subscription-Key"] == "key"
else:
assert headers["Authorization"] == "Bearer token"


@pytest.mark.parametrize("error", [CapabilityDiscoveryError("Connection failed"), TimeoutError()])
async def test_detection_failure_keeps_source_and_releases_probe(
detection_service: tuple[TargetService, _DetectableTarget], error: Exception
Expand Down Expand Up @@ -525,7 +593,7 @@ async def test_detection_rejects_unsafe_targets(
service, _ = detection_service
target = MockPromptTarget()
service._registry.instances.register(target, name="unsafe")
with pytest.raises(ValueError, match="reconstruction"):
with pytest.raises(ValueError, match="not supported for MockPromptTarget because it can't be rebuilt"):
await service.detect_capabilities_async(
target_registry_name="unsafe", expected_hash=target.get_identifier().hash
)
Expand Down
30 changes: 30 additions & 0 deletions tests/unit/prompt_target/target/test_litellm_decision_target.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
from pyrit.exceptions import PyritException, RateLimitException, get_retry_max_num_attempts
from pyrit.models import Message, MessagePiece
from pyrit.prompt_target import LiteLLMChatTarget, LiteLLMDecisionTarget
from pyrit.registry import TargetRegistry

MODEL = "openrouter/microsoft/microsoft-decision-1"
ENDPOINT = "https://openrouter.ai/api"
Expand Down Expand Up @@ -106,6 +107,35 @@ def test_reuses_chat_target_without_chat_capabilities(self, target: LiteLLMDecis
assert not target.capabilities.supports_multi_message_pieces
assert target.capabilities.supported_input_modalities == ["text"]

@pytest.mark.parametrize("api_key", ["test-api-key", AsyncMock(return_value="test-token")], ids=["key", "provider"])
async def test_reconstruction_keeps_identity_and_credentials_async(
self, *, api_key: str | AsyncMock, decision_sdk: MagicMock
) -> None:
source = LiteLLMDecisionTarget(
model_name=MODEL, endpoint=ENDPOINT, api_key=api_key, headers={"X-Route": "a", "Authorization": "secret"}
)

rebuilt = TargetRegistry.get_registry_singleton().recreate_instance(source=source, params={})

assert rebuilt is not source
assert rebuilt.get_identifier().hash == source.get_identifier().hash
assert rebuilt._api_key is source._api_key
assert rebuilt._headers == source._headers
assert rebuilt._headers is not source._headers
assert "secret" not in rebuilt.get_identifier().model_dump_json()
await rebuilt.send_prompt_async(message=_request(json.dumps(REQUEST)))
body = decision_sdk.adecisions.call_args.kwargs
assert body["api_key"] == ("test-api-key" if isinstance(api_key, str) else "test-token")
assert body["api_base"] == ENDPOINT
assert body["extra_headers"] == {"X-Route": "a", "Authorization": "secret"}

@pytest.mark.parametrize("params", [{"model_name": "anthropic/x"}, {"endpoint": "https://other.test"}])
def test_reconstruction_keeps_connection_settings(
self, *, target: LiteLLMDecisionTarget, params: dict[str, str]
) -> None:
with pytest.raises(ValueError, match="keeps its source's connection settings"):
TargetRegistry.get_registry_singleton().recreate_instance(source=target, params=params, external_input=True)

def test_empty_model_raises(self) -> None:
with pytest.raises(ValueError, match="model name"):
LiteLLMDecisionTarget(model_name=" ")
Expand Down
Loading
Loading