|
1 | 1 | import { describe, expect, it } from 'vitest'; |
2 | 2 | import type { AIV5Type } from '../types'; |
3 | | -import { addStartStepPartsForAIV5, sanitizeV5UIMessages } from './output-converter'; |
| 3 | +import { addStartStepPartsForAIV5, aiV5UIMessagesToAIV5ModelMessages, sanitizeV5UIMessages } from './output-converter'; |
4 | 4 |
|
5 | 5 | /** |
6 | 6 | * Tests for provider-executed tool handling in sanitizeV5UIMessages. |
@@ -209,6 +209,252 @@ describe('sanitizeV5UIMessages — provider-executed tool handling', () => { |
209 | 209 | }); |
210 | 210 | }); |
211 | 211 |
|
| 212 | +/** |
| 213 | + * Regression tests for https://github.com/mastra-ai/mastra/issues/15668 |
| 214 | + * |
| 215 | + * Gemini `code_execution` (and `google_search`) on Vertex AI non-deterministically |
| 216 | + * omits the paired `codeExecutionResult` after emitting `executableCode`. The AI SDK |
| 217 | + * surfaces this as a `tool-call { providerExecuted: true }` chunk with no paired |
| 218 | + * `tool-result`. Mastra currently persists that as |
| 219 | + * `tool-invocation { state: 'call', providerExecuted: true }` with no result part. |
| 220 | + * |
| 221 | + * When the thread is replayed on the next turn, `sanitizeV5UIMessages` keeps the |
| 222 | + * orphaned provider-executed call (line ~150 of output-converter.ts). The Google |
| 223 | + * provider then serializes it as a bare `functionCall` in the `contents` array — |
| 224 | + * with no paired `functionResponse`. Gemini returns `text: ""` for this turn and |
| 225 | + * every subsequent turn on the thread (verified by the reporter: 5/5 empty STOP). |
| 226 | + * |
| 227 | + * The bricking is deterministic once the orphan lands. The contract that must |
| 228 | + * hold is: after UI→Model conversion, there must NEVER be a `tool-call` content |
| 229 | + * part without a corresponding `tool-result` somewhere in the outgoing messages. |
| 230 | + * Whether the fix synthesizes a placeholder result or strips the call is an |
| 231 | + * implementation detail — but one of the two must happen. |
| 232 | + */ |
| 233 | +describe('sanitizeV5UIMessages — orphaned provider-executed tool calls (issue #15668)', () => { |
| 234 | + const makeToolPart = ( |
| 235 | + overrides: Partial<AIV5Type.ToolUIPart> & { type: string; toolCallId: string }, |
| 236 | + ): AIV5Type.ToolUIPart => |
| 237 | + ({ |
| 238 | + state: 'input-available' as const, |
| 239 | + input: {}, |
| 240 | + ...overrides, |
| 241 | + }) as AIV5Type.ToolUIPart; |
| 242 | + |
| 243 | + it('drops or resolves an orphaned Gemini code_execution call so the next request is not an unpaired functionCall', () => { |
| 244 | + // This is the exact shape Mastra persists after Vertex drops codeExecutionResult. |
| 245 | + // It is NOT a legitimate deferred provider tool — Gemini code_execution does not |
| 246 | + // defer across turns (unlike Anthropic web_search). Replaying this to the model |
| 247 | + // deterministically produces empty text on every subsequent turn. |
| 248 | + const user: AIV5Type.UIMessage = { |
| 249 | + id: 'u1', |
| 250 | + role: 'user', |
| 251 | + parts: [{ type: 'text', text: 'Parse the attached spreadsheet and tell me the column names.' }], |
| 252 | + }; |
| 253 | + |
| 254 | + const assistantWithOrphan: AIV5Type.UIMessage = { |
| 255 | + id: 'a1', |
| 256 | + role: 'assistant', |
| 257 | + parts: [ |
| 258 | + { type: 'text', text: "I'll parse it with pandas via code_execution." }, |
| 259 | + makeToolPart({ |
| 260 | + type: 'tool-code_execution', |
| 261 | + toolCallId: 'gem_tool_1', |
| 262 | + state: 'input-available', |
| 263 | + input: { code: "import pandas as pd; print(pd.read_excel('/tmp/x.xlsx').columns.tolist())" }, |
| 264 | + providerExecuted: true, |
| 265 | + }), |
| 266 | + ], |
| 267 | + }; |
| 268 | + |
| 269 | + const followUp: AIV5Type.UIMessage = { |
| 270 | + id: 'u2', |
| 271 | + role: 'user', |
| 272 | + parts: [{ type: 'text', text: 'what were the columns?' }], |
| 273 | + }; |
| 274 | + |
| 275 | + const modelMessages = aiV5UIMessagesToAIV5ModelMessages( |
| 276 | + [user, assistantWithOrphan, followUp], |
| 277 | + [], |
| 278 | + /* filterIncompleteToolCalls */ true, |
| 279 | + ); |
| 280 | + |
| 281 | + // Walk all assistant content parts and collect tool-call / tool-result IDs. |
| 282 | + const orphanCallIds: string[] = []; |
| 283 | + const resultIds = new Set<string>(); |
| 284 | + for (const m of modelMessages) { |
| 285 | + if (typeof m.content === 'string') continue; |
| 286 | + for (const part of m.content) { |
| 287 | + if (part.type === 'tool-call') { |
| 288 | + orphanCallIds.push((part as any).toolCallId); |
| 289 | + } |
| 290 | + if (part.type === 'tool-result') { |
| 291 | + resultIds.add((part as any).toolCallId); |
| 292 | + } |
| 293 | + } |
| 294 | + } |
| 295 | + |
| 296 | + // The invariant: every tool-call sent to the provider must have a matching |
| 297 | + // tool-result. An orphaned `gem_tool_1` call without a result is exactly |
| 298 | + // what bricks the thread. |
| 299 | + const unpaired = orphanCallIds.filter(id => !resultIds.has(id)); |
| 300 | + expect(unpaired).toEqual([]); |
| 301 | + }); |
| 302 | + |
| 303 | + it('drops or resolves an orphaned Anthropic web_search call stuck in state:"call" (same defect class)', () => { |
| 304 | + // Same underlying issue as #14148 — once a provider-executed tool is persisted |
| 305 | + // as state:"call" with no paired result, every subsequent turn replays it. |
| 306 | + // The safeguard must be provider-agnostic. |
| 307 | + const user: AIV5Type.UIMessage = { |
| 308 | + id: 'u1', |
| 309 | + role: 'user', |
| 310 | + parts: [{ type: 'text', text: 'find me recent news on X' }], |
| 311 | + }; |
| 312 | + |
| 313 | + const assistantWithOrphan: AIV5Type.UIMessage = { |
| 314 | + id: 'a1', |
| 315 | + role: 'assistant', |
| 316 | + parts: [ |
| 317 | + makeToolPart({ |
| 318 | + type: 'tool-web_search_20250305', |
| 319 | + toolCallId: 'srvtoolu_orphan1', |
| 320 | + state: 'input-available', |
| 321 | + input: { query: 'X' }, |
| 322 | + providerExecuted: true, |
| 323 | + }), |
| 324 | + ], |
| 325 | + }; |
| 326 | + |
| 327 | + const followUp: AIV5Type.UIMessage = { |
| 328 | + id: 'u2', |
| 329 | + role: 'user', |
| 330 | + parts: [{ type: 'text', text: 'you good?' }], |
| 331 | + }; |
| 332 | + |
| 333 | + const modelMessages = aiV5UIMessagesToAIV5ModelMessages( |
| 334 | + [user, assistantWithOrphan, followUp], |
| 335 | + [], |
| 336 | + /* filterIncompleteToolCalls */ true, |
| 337 | + ); |
| 338 | + |
| 339 | + const orphanCallIds: string[] = []; |
| 340 | + const resultIds = new Set<string>(); |
| 341 | + for (const m of modelMessages) { |
| 342 | + if (typeof m.content === 'string') continue; |
| 343 | + for (const part of m.content) { |
| 344 | + if (part.type === 'tool-call') orphanCallIds.push((part as any).toolCallId); |
| 345 | + if (part.type === 'tool-result') resultIds.add((part as any).toolCallId); |
| 346 | + } |
| 347 | + } |
| 348 | + |
| 349 | + const unpaired = orphanCallIds.filter(id => !resultIds.has(id)); |
| 350 | + expect(unpaired).toEqual([]); |
| 351 | + }); |
| 352 | + |
| 353 | + it('drops a stale deferred provider call on an EARLIER assistant message when a later assistant message exists', () => { |
| 354 | + // Multi-assistant history shape — this guards against the class of bug #14192: |
| 355 | + // a provider-executed tool left in `input-available` on an earlier assistant |
| 356 | + // turn must not be replayed to the provider just because the final turn is |
| 357 | + // also an assistant. Only the most recent assistant message may legitimately |
| 358 | + // carry a deferred provider tool. |
| 359 | + const user: AIV5Type.UIMessage = { |
| 360 | + id: 'u1', |
| 361 | + role: 'user', |
| 362 | + parts: [{ type: 'text', text: 'search for recent news about X' }], |
| 363 | + }; |
| 364 | + |
| 365 | + const earlierAssistantWithStaleProviderCall: AIV5Type.UIMessage = { |
| 366 | + id: 'a1', |
| 367 | + role: 'assistant', |
| 368 | + parts: [ |
| 369 | + makeToolPart({ |
| 370 | + type: 'tool-web_search_20250305', |
| 371 | + toolCallId: 'srvtoolu_stale', |
| 372 | + state: 'input-available', |
| 373 | + input: { query: 'X' }, |
| 374 | + providerExecuted: true, |
| 375 | + }), |
| 376 | + ], |
| 377 | + }; |
| 378 | + |
| 379 | + const laterAssistant: AIV5Type.UIMessage = { |
| 380 | + id: 'a2', |
| 381 | + role: 'assistant', |
| 382 | + parts: [{ type: 'text', text: 'Here is a summary based on what I found earlier.' }], |
| 383 | + }; |
| 384 | + |
| 385 | + const modelMessages = aiV5UIMessagesToAIV5ModelMessages( |
| 386 | + [user, earlierAssistantWithStaleProviderCall, laterAssistant], |
| 387 | + [], |
| 388 | + /* filterIncompleteToolCalls */ true, |
| 389 | + ); |
| 390 | + |
| 391 | + const orphanCallIds: string[] = []; |
| 392 | + const resultIds = new Set<string>(); |
| 393 | + for (const m of modelMessages) { |
| 394 | + if (typeof m.content === 'string') continue; |
| 395 | + for (const part of m.content) { |
| 396 | + if (part.type === 'tool-call') orphanCallIds.push((part as any).toolCallId); |
| 397 | + if (part.type === 'tool-result') resultIds.add((part as any).toolCallId); |
| 398 | + } |
| 399 | + } |
| 400 | + |
| 401 | + const unpaired = orphanCallIds.filter(id => !resultIds.has(id)); |
| 402 | + expect(unpaired).toEqual([]); |
| 403 | + // Belt-and-suspenders: the stale id must not have leaked through at all. |
| 404 | + expect(orphanCallIds).not.toContain('srvtoolu_stale'); |
| 405 | + }); |
| 406 | + |
| 407 | + it('keeps a deferred provider call on the last surviving assistant when a trailing assistant sanitizes away', () => { |
| 408 | + const user: AIV5Type.UIMessage = { |
| 409 | + id: 'u1', |
| 410 | + role: 'user', |
| 411 | + parts: [{ type: 'text', text: 'search for recent news about X' }], |
| 412 | + }; |
| 413 | + |
| 414 | + const assistantWithDeferredProviderCall: AIV5Type.UIMessage = { |
| 415 | + id: 'a1', |
| 416 | + role: 'assistant', |
| 417 | + parts: [ |
| 418 | + makeToolPart({ |
| 419 | + type: 'tool-web_search_20250305', |
| 420 | + toolCallId: 'srvtoolu_deferred', |
| 421 | + state: 'input-available', |
| 422 | + input: { query: 'X' }, |
| 423 | + providerExecuted: true, |
| 424 | + }), |
| 425 | + ], |
| 426 | + }; |
| 427 | + |
| 428 | + const trailingAssistantThatSanitizesAway: AIV5Type.UIMessage = { |
| 429 | + id: 'a2', |
| 430 | + role: 'assistant', |
| 431 | + parts: [ |
| 432 | + makeToolPart({ |
| 433 | + type: 'tool-get_info', |
| 434 | + toolCallId: 'client_streaming', |
| 435 | + state: 'input-streaming', |
| 436 | + input: { query: 'ignore me' }, |
| 437 | + }), |
| 438 | + ], |
| 439 | + }; |
| 440 | + |
| 441 | + const result = sanitizeV5UIMessages( |
| 442 | + [user, assistantWithDeferredProviderCall, trailingAssistantThatSanitizesAway], |
| 443 | + true, |
| 444 | + ); |
| 445 | + |
| 446 | + expect(result).toHaveLength(2); |
| 447 | + expect(result[1]?.id).toBe('a1'); |
| 448 | + expect(result[1]?.parts).toEqual([ |
| 449 | + expect.objectContaining({ |
| 450 | + toolCallId: 'srvtoolu_deferred', |
| 451 | + state: 'input-available', |
| 452 | + providerExecuted: true, |
| 453 | + }), |
| 454 | + ]); |
| 455 | + }); |
| 456 | +}); |
| 457 | + |
212 | 458 | describe('addStartStepPartsForAIV5 — client/provider tool splitting', () => { |
213 | 459 | const makeToolPart = ( |
214 | 460 | overrides: Partial<AIV5Type.ToolUIPart> & { type: string; toolCallId: string }, |
|
0 commit comments