Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,7 +1,39 @@
import { objectivePreview } from './attackAttemptFormatting'
import type { OutcomeStatistics, ScenarioSeedGroupProgress } from '@/types'

import { formatSeedGroupSuccess, objectivePreview } from './attackAttemptFormatting'

const LONE_SURROGATE_REGEX = /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/

describe('formatSeedGroupSuccess', () => {
const seed: ScenarioSeedGroupProgress = {
id: 'fixture', completed: 1, planned: 1, succeeded: 1, success_percentage: 100, errors: 0, retries: 0,
}
const trials: OutcomeStatistics = {
successes: 3, failures: 2, errors: 0, undetermined: 0,
total_results: 5, total_decided: 5, decided_share: 1,
success_rate: 0.6, success_rate_decided: 0.6, success_rate_all: 0.6,
outcome_shares: { success: 0.6, failure: 0.4, error: 0, undetermined: 0 },
}

it('uses the trial fraction instead of the successful parent outcome', () => {
expect(formatSeedGroupSuccess({ ...seed, trial_outcomes: trials })).toBe('60% (3/5 trials)')
})

it.each(['errors', 'undetermined'] as const)('does not present partial trial ASR with %s', (outcome: 'errors' | 'undetermined') => {
expect(formatSeedGroupSuccess({
...seed, trial_outcomes: {
...trials, [outcome]: 1, failures: 1, total_decided: 4, decided_share: 0.8,
success_rate: 0.75, success_rate_decided: 0.75,
outcome_shares: { success: 0.6, failure: 0.2, error: 0, undetermined: 0, [outcome]: 0.2 },
},
})).toBe('Undetermined')
})

it('preserves ordinary and older result formatting without independent trials', () => {
expect(formatSeedGroupSuccess(seed)).toBe('100%')
})
})

describe('objectivePreview', () => {
it('truncates supplementary Unicode without splitting surrogate pairs (issue #3870 reproduction)', () => {
const objective = `${'a'.repeat(94)}😀 trailing text`
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import type { ScenarioProgressCounts, ScenarioProgressResult } from '@/types'
import type { ScenarioProgressCounts, ScenarioProgressResult, ScenarioSeedGroupProgress } from '@/types'
import { formatAnalyticsPercent } from '@/utils/attackAnalytics'

const OBJECTIVE_PREVIEW_LENGTH = 96
Expand All @@ -11,6 +11,13 @@ export function formatProgressSuccess(counts: ScenarioProgressCounts): string {
return counts.success_percentage === null ? 'Undetermined' : `${counts.success_percentage}%`
}

export function formatSeedGroupSuccess(seed: ScenarioSeedGroupProgress): string {
const trials = seed.trial_outcomes
if (!trials) return formatProgressSuccess(seed)
if (trials.total_results === 0 || trials.total_decided !== trials.total_results) return 'Undetermined'
return `${formatAnalyticsPercent(trials.success_rate_all)} (${trials.successes}/${trials.total_results} trials)`
}

/**
* Formatting for a single attack attempt.
*
Expand Down
2 changes: 1 addition & 1 deletion frontend/src/components/Chat/ChatScorerPicker.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,9 @@ export default function ChatScorerPicker({
}: ChatScorerPickerProps) {
const styles = useChatScorerPickerStyles()
const selectedScorer = findRegisteredScorer(value ?? null, scorers, inherited)
const available = !value || selectedScorer !== undefined
const eligibleScorers = scorers.filter((scorer: ScorerInstance) => usage === 'scenario'
? scorer.identifier.scorer_type === 'true_false' : isEligibleChatScorer(scorer))
const available = !value || (selectedScorer !== undefined && eligibleScorers.includes(selectedScorer))
const selected = value === undefined && onUseLastUsed ? 'last-used'
: selectedScorer && eligibleScorers.includes(selectedScorer)
? `registered:${selectedScorer.scorer_registry_name}` : value ? 'unavailable' : ''
Expand Down
71 changes: 71 additions & 0 deletions frontend/src/components/Scenarios/ScenarioDetail.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -272,6 +272,77 @@ function renderDetail(
}

describe('ScenarioDetail', () => {
it('uses one primary refusal scorer for CBRN and can return to the scenario default', async () => {
const user = userEvent.setup()
mockGetScenario.mockResolvedValue(makeScenario({
scenario_name: 'airt.cbrn', supports_objective_scorer: true,
baseline_policy: 'forbidden', uses_default_adversarial_target: false,
supported_parameters: [{ name: 'num_trials', type_name: 'int', required: false, default: 5 }],
}))
renderDetail('/scanner/launch/airt.cbrn', { scorers: [
{ scorer_registry_name: 'refusal', identifier: {
class_name: 'SelfAskRefusalScorer', hash: 'refusal-hash', scorer_type: 'true_false',
} },
{ scorer_registry_name: 'inverted', identifier: {
class_name: 'TrueFalseInverterScorer', hash: 'inverted-hash', scorer_type: 'true_false',
} },
{ scorer_registry_name: 'substring', identifier: {
class_name: 'SubStringScorer', hash: 'substring-hash', scorer_type: 'true_false',
} },
{ scorer_registry_name: 'float-only', identifier: {
class_name: 'SelfAskLikertScorer', hash: 'float-hash', scorer_type: 'float_scale',
} },
] })
const picker = await screen.findByRole('combobox', { name: 'Objective scorer' })
expect(picker).toHaveValue('')
expect(screen.queryByRole('combobox', { name: 'allow_scorer' })).not.toBeInTheDocument()
expect(screen.queryByRole('combobox', { name: 'refuse_scorer' })).not.toBeInTheDocument()
expect(within(picker).queryByRole('option', { name: /float-only/ })).not.toBeInTheDocument()
expect(within(picker).getByRole('option', { name: /refusal - SelfAskRefusalScorer/ })).toBeInTheDocument()
await user.selectOptions(picker, 'registered:refusal')
const reference = { registry_name: 'refusal', identifier_hash: 'refusal-hash' }
await waitFor(() => {
expect(mockEstimateRun.mock.calls.at(-1)?.[1].objective_scorer).toEqual(reference)
})
await user.selectOptions(picker, '')
await waitFor(() => {
expect(mockEstimateRun.mock.calls.at(-1)?.[1]).not.toHaveProperty('objective_scorer')
})
await user.selectOptions(picker, 'registered:refusal')
await confirmRunPreview(user)
expect(mockStartRun.mock.calls[0][0]).toMatchObject({
scenario_name: 'airt.cbrn',
objective_scorer: reference,
scenario_params: { num_trials: 5 },
})
})

it('disables scorer-incompatible techniques and excludes them from tag selections', async () => {
const user = userEvent.setup()
const reason = 'This technique cannot accept the population-specific scoring configuration.'
mockGetScenario.mockResolvedValue(makeScenario({
all_techniques: ['default_technique', 'crescendo', 'tap'],
technique_summaries: [
{ name: 'default_technique', description: null, tags: ['single_turn'] },
{ name: 'crescendo', description: null, tags: ['single_turn'] },
{ name: 'tap', description: null, tags: ['single_turn'], unavailable_reason: reason },
],
}))
renderDetail('/scanner/launch/foundry.red_team_agent')
const tap = await screen.findByRole('checkbox', { name: 'tap', exact: true })
expect(tap).toBeDisabled()
expect(tap).toHaveAccessibleDescription(reason)
await user.click(tap)
expect(tap).not.toBeChecked()
await user.click(screen.getAllByRole('button', { name: 'Select Single-turn techniques' })[0])
expect(screen.getByTestId('technique-default_technique')).toBeChecked()
expect(screen.getByTestId('technique-crescendo')).toBeChecked()
expect(tap).not.toBeChecked()
await waitFor(() => {
expect(mockEstimateRun.mock.calls.at(-1)?.[1].techniques).toEqual(['default_technique', 'crescendo'])
})
})

it('keeps the existing component selectors without inline creation or readiness fields', async () => {
const user = userEvent.setup()
mockGetScenario.mockResolvedValue(makeScenario({ supports_objective_scorer: true }))
Expand Down
11 changes: 8 additions & 3 deletions frontend/src/components/Scenarios/ScenarioDetail.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -689,11 +689,12 @@ function ScenarioLaunchForm({
...BASELINE_TECHNIQUE,
isBaseline: true,
disabled: isBaselineForbidden,
unavailable_reason: isBaselineForbidden ? 'This scenario does not support a baseline comparison.' : null,
}] : []),
...(supportsTechniques ? techniqueOptions.map((technique) => ({
...technique,
isBaseline: false,
disabled: false,
disabled: Boolean(technique.unavailable_reason),
})) : []),
],
[isBaselineForbidden, supportsBaseline, supportsTechniques, techniqueOptions],
Expand Down Expand Up @@ -740,7 +741,11 @@ function ScenarioLaunchForm({

const techniqueSelectionOverLimit = supportsTechniques && scenario.max_concrete_techniques != null
&& selectedTechniques.length > scenario.max_concrete_techniques
const techniqueSelectionError = supportsTechniques && selectedTechniques.length === 0
const hasIncompatibleSelection = selectedTechniques.some((name) =>
techniqueOptions.some((technique) => technique.name === name && technique.unavailable_reason))
const techniqueSelectionError = hasIncompatibleSelection
? 'Clear incompatible techniques before launching.'
: supportsTechniques && selectedTechniques.length === 0
? 'Select at least one attack technique.'
: techniqueSelectionOverLimit
? `Select at most ${scenario.max_concrete_techniques} concrete technique(s).`
Expand Down Expand Up @@ -1061,7 +1066,7 @@ function ScenarioLaunchForm({
className={styles.launchButton}
appearance="primary"
type="submit"
disabled={!runtime.ready || !defaultsReady || staleSelection || unavailableSelection || submitting || conversationBusy || Boolean(techniqueEditError) || techniqueSelectionError !== null || datasetMetadata.preparing || (supportsDatasets && datasetError !== null)}
disabled={!runtime.ready || !defaultsReady || staleSelection || unavailableSelection || submitting || conversationBusy || Boolean(techniqueEditError) || Boolean(scorerError) || techniqueSelectionError !== null || datasetMetadata.preparing || (supportsDatasets && datasetError !== null)}
data-testid="launch-scenario-btn"
>
Launch scan
Expand Down
112 changes: 112 additions & 0 deletions frontend/src/components/Scenarios/ScenarioRunPage.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,18 @@ import type {
} from '@/types'
import { scenarioRunAttackRoutePath, scenarioRunRoutePath } from '@/utils/routeParams'
import { INITIAL_SCENARIO_RUN_PROGRESS_STATE, type ScenarioRunProgressState } from '@/utils/scenarioRunProgress'
import { downloadFile } from '@/utils/downloadFile'

import ScenarioRunPage from './ScenarioRunPage'

jest.mock('@/hooks/useScenarioRunProgress', () => ({ useScenarioRunProgress: jest.fn() }))
jest.mock('@/hooks/useScenarioQueue', () => ({ useScenarioQueue: jest.fn() }))
jest.mock('@/utils/downloadFile', () => ({ downloadFile: jest.fn() }))
jest.mock('@/services/api', () => ({ scenariosApi: {
cancelRun: jest.fn(), resumeRun: jest.fn(), getLaunchConfiguration: jest.fn(), startRun: jest.fn(), getQueue: jest.fn(),
getRunHierarchy: jest.fn(() => new Promise(() => {})), getRunHierarchyNode: jest.fn(),
getPairedOutcomes: jest.fn(), getPairedOutcomeDetails: jest.fn(),
getRunArtifacts: jest.fn(),
} }))

const mockUseScenarioQueue = useScenarioQueue as jest.Mock
Expand Down Expand Up @@ -131,6 +134,91 @@ beforeEach(() => {
})

describe('ScenarioRunPage', () => {
it.each(['COMPLETED', 'FAILED', 'CANCELLED'] as const)('downloads artifacts to disk for a %s run beside Rerun', async (status) => {
const user = userEvent.setup()
state = makeState({ run: { ...RUN, status } })
const artifact = new Blob([JSON.stringify({ metadata: { expected: 'ALLOW' }, attack_results: {} })], { type: 'application/json' })
jest.mocked(scenariosApi.getRunArtifacts).mockResolvedValue(artifact)
render(<Wrapper path={scenarioRunRoutePath(RUN_ID)} />)
const button = screen.getByRole('button', { name: 'Download artifacts' })
expect(button.nextElementSibling).toBe(screen.getByRole('button', { name: 'Rerun' }))
await user.click(button)
expect(scenariosApi.getRunArtifacts).toHaveBeenCalledWith(RUN_ID, expect.any(AbortSignal))
await waitFor(() => expect(downloadFile).toHaveBeenCalledWith(artifact, `copyrit-scenario-${RUN_ID}.json`))
const confirmation = screen.getByLabelText('Artifact download confirmation')
expect(within(confirmation).getByText(`copyrit-scenario-${RUN_ID}.json`)).toBeInTheDocument()
expect(confirmation).toHaveTextContent('Download started:')
expect(confirmation).toHaveTextContent("Check your browser's download location")
expect(confirmation).toHaveTextContent('The browser does not expose the exact saved path to this page.')
expect(scenariosApi.startRun).not.toHaveBeenCalled()
})

it('disables artifacts while the run is active', async () => {
const user = userEvent.setup()
render(<Wrapper path={scenarioRunRoutePath(RUN_ID)} />)
const button = screen.getByRole('button', { name: 'Download artifacts' })
expect(button).toBeDisabled()
await user.click(button)
expect(scenariosApi.getRunArtifacts).not.toHaveBeenCalled()
})

it('reports artifacts errors without downloading and allows retry', async () => {
const user = userEvent.setup()
state = makeState({ run: { ...RUN, status: 'COMPLETED' } })
jest.mocked(scenariosApi.getRunArtifacts).mockRejectedValueOnce(new Error('Artifacts unavailable'))
render(<Wrapper path={scenarioRunRoutePath(RUN_ID)} />)
await user.click(screen.getByRole('button', { name: 'Download artifacts' }))
expect(await screen.findByText('Artifacts unavailable')).toBeInTheDocument()
expect(downloadFile).not.toHaveBeenCalled()
expect(screen.queryByLabelText('Artifact download confirmation')).not.toBeInTheDocument()
expect(screen.getByRole('button', { name: 'Download artifacts' })).toBeEnabled()
})

it('allows dismissing the artifacts download confirmation', async () => {
const user = userEvent.setup()
state = makeState({ run: { ...RUN, status: 'COMPLETED' } })
jest.mocked(scenariosApi.getRunArtifacts).mockResolvedValue(new Blob(['{}'], { type: 'application/json' }))
render(<Wrapper path={scenarioRunRoutePath(RUN_ID)} />)
await user.click(screen.getByRole('button', { name: 'Download artifacts' }))
expect(await screen.findByLabelText('Artifact download confirmation')).toBeInTheDocument()
await user.click(screen.getByRole('button', { name: 'Dismiss download confirmation' }))
expect(screen.queryByLabelText('Artifact download confirmation')).not.toBeInTheDocument()
})

it('does not confirm artifacts when the browser cannot start the download', async () => {
const user = userEvent.setup()
state = makeState({ run: { ...RUN, status: 'COMPLETED' } })
jest.mocked(scenariosApi.getRunArtifacts).mockResolvedValue(new Blob(['{}'], { type: 'application/json' }))
jest.mocked(downloadFile).mockImplementationOnce(() => { throw new Error('Downloads blocked') })
render(<Wrapper path={scenarioRunRoutePath(RUN_ID)} />)
await user.click(screen.getByRole('button', { name: 'Download artifacts' }))
expect(await screen.findByText('Downloads blocked')).toBeInTheDocument()
expect(screen.queryByLabelText('Artifact download confirmation')).not.toBeInTheDocument()
expect(screen.getByRole('button', { name: 'Download artifacts' })).toBeEnabled()
})

it.each(['Leave run', 'Open attempt'])('aborts artifacts and ignores a late download after %s', async (navigation) => {
const user = userEvent.setup()
state = makeState({ run: { ...RUN, status: 'COMPLETED' } })
let resolveArtifact: ((blob: Blob) => void) | undefined
let requestSignal: AbortSignal | undefined
jest.mocked(scenariosApi.getRunArtifacts).mockImplementation((_id, signal) => new Promise((resolve) => {
resolveArtifact = resolve
requestSignal = signal
}))
render(<Wrapper path={scenarioRunRoutePath(RUN_ID)} />)
await user.click(screen.getByRole('button', { name: 'Download artifacts' }))
expect(screen.getByRole('button', { name: 'Downloading...' })).toBeDisabled()
await user.click(screen.getByRole('button', { name: navigation }))
expect(requestSignal?.aborted).toBe(true)
const resolve = resolveArtifact
if (!resolve) throw new Error('The artifact request did not start.')
await act(async () => { resolve(new Blob(['{}'], { type: 'application/json' })) })
expect(downloadFile).not.toHaveBeenCalled()
expect(screen.queryByLabelText('Artifact download confirmation')).not.toBeInTheDocument()
if (navigation === 'Open attempt') expect(screen.getByRole('button', { name: 'Download artifacts' })).toBeEnabled()
})

it('keeps comparison optional on completed scans and cancels it when collapsed', async () => {
const user = userEvent.setup()
state = makeState({
Expand Down Expand Up @@ -313,6 +401,30 @@ describe('ScenarioRunPage', () => {
expect(screen.queryByRole('heading', { name: 'Paired outcomes' })).not.toBeInTheDocument()
})

it('shows trial ASR for objectives without changing the group attack outcome or completion', () => {
state = makeState({
summary: {
...SUMMARY,
seed_groups: [{
...SUMMARY.seed_groups[0],
trial_outcomes: {
successes: 3, failures: 2, errors: 0, undetermined: 0,
total_results: 5, total_decided: 5, decided_share: 1,
success_rate: 0.6, success_rate_decided: 0.6, success_rate_all: 0.6,
outcome_shares: { success: 0.6, failure: 0.4, error: 0, undetermined: 0 },
},
}],
},
})
render(<Wrapper path={scenarioRunRoutePath(RUN_ID)} />)
const objectives = within(screen.getByRole('table', { name: 'Datasets and objectives' }))
const row = within(objectives.getByRole('row', { name: /First objective/ }))
expect(row.getByText('60% (3/5 trials)', { exact: true })).toBeInTheDocument()
expect(row.getByText('1 / 1', { exact: true })).toBeInTheDocument()
const attacks = within(screen.getByRole('table', { name: 'Direct attack results' }))
expect(attacks.getByText('True', { exact: true })).toBeInTheDocument()
})

it('keeps interrupted saved conversations reachable without inventing an outcome row', () => {
state = makeState({
results: [], activeConversations: [],
Expand Down
Loading