Skip to content

Commit c78f8cd

Browse files
shaun0927abhiaiyer91
shaun0927
and
abhiaiyer91
authored
fix(core): let trajectory extraction read V2 tool-invocation parts (#15439)
Co-authored-by: Abhi Aiyer <abhiaiyer91@gmail.com>
1 parent a702009 commit c78f8cd

3 files changed

Lines changed: 281 additions & 3 deletions

File tree

‎.changeset/many-things-glow.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
'@mastra/core': patch
3+
---
4+
5+
Fixed trajectory scorers so tool calls stored only in V2 content.parts are included in extracted eval steps.

‎packages/core/src/evals/types.test.ts‎

Lines changed: 266 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { describe, expect, it } from 'vitest';
22
import { SpanType } from '../observability';
33
import type { SpanRecord } from '../storage';
4-
import { extractTrajectoryFromTrace, saveScorePayloadSchema } from './types';
4+
import { extractTrajectory, extractTrajectoryFromTrace, saveScorePayloadSchema } from './types';
55

66
function createSpan(overrides: Partial<SpanRecord> & { spanId: string; spanType: SpanRecord['spanType'] }): SpanRecord {
77
return {
@@ -778,3 +778,268 @@ describe('saveScorePayloadSchema', () => {
778778
expect(() => saveScorePayloadSchema.parse(buildPayload('NOT_A_REAL_TYPE'))).toThrow();
779779
});
780780
});
781+
782+
describe('extractTrajectory', () => {
783+
// --- legacy toolInvocations path ---
784+
785+
it('extracts tool calls from content.toolInvocations when present', () => {
786+
const output = [
787+
{
788+
role: 'assistant',
789+
content: {
790+
toolInvocations: [
791+
{ state: 'result', toolCallId: 'c1', toolName: 'legacyTool', args: { q: 'x' }, result: { ok: true } },
792+
],
793+
parts: [],
794+
},
795+
},
796+
] as any;
797+
798+
const result = extractTrajectory(output);
799+
800+
expect(result.steps).toHaveLength(1);
801+
expect(result.steps[0]).toMatchObject({
802+
stepType: 'tool_call',
803+
name: 'legacyTool',
804+
toolArgs: { q: 'x' },
805+
toolResult: { ok: true },
806+
success: true,
807+
});
808+
});
809+
810+
// --- V2 parts fallback path ---
811+
812+
it('extracts tool calls from V2 content.parts when toolInvocations is absent', () => {
813+
const output = [
814+
{
815+
role: 'assistant',
816+
content: {
817+
format: 2,
818+
parts: [
819+
{
820+
type: 'tool-invocation',
821+
toolInvocation: {
822+
state: 'result',
823+
toolCallId: 'c1',
824+
toolName: 'weatherTool',
825+
args: { city: 'Seoul' },
826+
result: { temperature: 22 },
827+
},
828+
},
829+
],
830+
},
831+
},
832+
] as any;
833+
834+
const result = extractTrajectory(output);
835+
836+
expect(result.steps).toHaveLength(1);
837+
expect(result.steps[0]).toMatchObject({
838+
stepType: 'tool_call',
839+
name: 'weatherTool',
840+
toolArgs: { city: 'Seoul' },
841+
toolResult: { temperature: 22 },
842+
success: true,
843+
});
844+
});
845+
846+
it('extracts multiple tool calls from V2 parts in a single message', () => {
847+
const output = [
848+
{
849+
role: 'assistant',
850+
content: {
851+
format: 2,
852+
parts: [
853+
{
854+
type: 'tool-invocation',
855+
toolInvocation: { state: 'result', toolCallId: 'c1', toolName: 'toolA', args: { a: 1 }, result: 'ok' },
856+
},
857+
{ type: 'text', text: 'some text in between' },
858+
{
859+
type: 'tool-invocation',
860+
toolInvocation: { state: 'result', toolCallId: 'c2', toolName: 'toolB', args: { b: 2 }, result: 'done' },
861+
},
862+
],
863+
},
864+
},
865+
] as any;
866+
867+
const result = extractTrajectory(output);
868+
869+
expect(result.steps).toHaveLength(2);
870+
expect(result.steps[0]).toMatchObject({ name: 'toolA', toolArgs: { a: 1 }, success: true });
871+
expect(result.steps[1]).toMatchObject({ name: 'toolB', toolArgs: { b: 2 }, success: true });
872+
});
873+
874+
it('marks V2 call-state invocations as not successful', () => {
875+
const output = [
876+
{
877+
role: 'assistant',
878+
content: {
879+
format: 2,
880+
parts: [
881+
{
882+
type: 'tool-invocation',
883+
toolInvocation: { state: 'call', toolCallId: 'c1', toolName: 'pendingTool', args: { x: 1 } },
884+
},
885+
],
886+
},
887+
},
888+
] as any;
889+
890+
const result = extractTrajectory(output);
891+
892+
expect(result.steps).toHaveLength(1);
893+
expect(result.steps[0]).toMatchObject({ name: 'pendingTool', success: false });
894+
expect((result.steps[0] as any).toolResult).toBeUndefined();
895+
});
896+
897+
// --- precedence ---
898+
899+
it('prefers content.toolInvocations over content.parts when both are present', () => {
900+
const output = [
901+
{
902+
role: 'assistant',
903+
content: {
904+
toolInvocations: [{ state: 'call', toolCallId: 'c-top', toolName: 'topLevelTool', args: { source: 'top' } }],
905+
format: 2,
906+
parts: [
907+
{
908+
type: 'tool-invocation',
909+
toolInvocation: {
910+
state: 'result',
911+
toolCallId: 'c-part',
912+
toolName: 'partsTool',
913+
args: { source: 'parts' },
914+
result: { ok: true },
915+
},
916+
},
917+
],
918+
},
919+
},
920+
] as any;
921+
922+
const result = extractTrajectory(output);
923+
924+
expect(result.steps).toHaveLength(1);
925+
expect(result.steps[0]).toMatchObject({ name: 'topLevelTool', success: false });
926+
expect(result.steps[0].name).not.toBe('partsTool');
927+
});
928+
929+
// --- edge cases ---
930+
931+
it('skips non-assistant messages', () => {
932+
const output = [
933+
{ role: 'user', content: { parts: [{ type: 'text', text: 'hello' }] } },
934+
{
935+
role: 'assistant',
936+
content: {
937+
format: 2,
938+
parts: [
939+
{
940+
type: 'tool-invocation',
941+
toolInvocation: { state: 'result', toolCallId: 'c1', toolName: 'myTool', args: {}, result: 'ok' },
942+
},
943+
],
944+
},
945+
},
946+
] as any;
947+
948+
const result = extractTrajectory(output);
949+
950+
// user message has no toolInvocations — skipped; only assistant message extracted
951+
expect(result.steps).toHaveLength(1);
952+
expect(result.steps[0]).toMatchObject({ name: 'myTool' });
953+
});
954+
955+
it('skips messages with empty parts and no toolInvocations', () => {
956+
const output = [
957+
{ role: 'assistant', content: { format: 2, parts: [] } },
958+
{ role: 'assistant', content: { format: 2, parts: [{ type: 'text', text: 'just text' }] } },
959+
] as any;
960+
961+
const result = extractTrajectory(output);
962+
963+
expect(result.steps).toHaveLength(0);
964+
});
965+
966+
it('handles messages with no content gracefully', () => {
967+
const output = [{ role: 'assistant', content: null }, { role: 'assistant' }] as any;
968+
969+
const result = extractTrajectory(output);
970+
971+
expect(result.steps).toHaveLength(0);
972+
});
973+
974+
it('collects steps across multiple assistant messages', () => {
975+
const output = [
976+
{
977+
role: 'assistant',
978+
content: {
979+
toolInvocations: [{ state: 'result', toolCallId: 'c1', toolName: 'toolA', args: { a: 1 }, result: 'r1' }],
980+
},
981+
},
982+
{
983+
role: 'assistant',
984+
content: {
985+
format: 2,
986+
parts: [
987+
{
988+
type: 'tool-invocation',
989+
toolInvocation: { state: 'result', toolCallId: 'c2', toolName: 'toolB', args: { b: 2 }, result: 'r2' },
990+
},
991+
],
992+
},
993+
},
994+
] as any;
995+
996+
const result = extractTrajectory(output);
997+
998+
expect(result.steps).toHaveLength(2);
999+
expect(result.steps[0]).toMatchObject({ name: 'toolA' });
1000+
expect(result.steps[1]).toMatchObject({ name: 'toolB' });
1001+
});
1002+
1003+
it('wraps primitive args and results in { value } objects', () => {
1004+
const output = [
1005+
{
1006+
role: 'assistant',
1007+
content: {
1008+
format: 2,
1009+
parts: [
1010+
{
1011+
type: 'tool-invocation',
1012+
toolInvocation: { state: 'result', toolCallId: 'c1', toolName: 'echo', args: 'hello', result: 42 },
1013+
},
1014+
],
1015+
},
1016+
},
1017+
] as any;
1018+
1019+
const result = extractTrajectory(output);
1020+
1021+
expect(result.steps).toHaveLength(1);
1022+
expect(result.steps[0]).toMatchObject({ toolArgs: { value: 'hello' }, toolResult: { value: 42 } });
1023+
});
1024+
1025+
it('preserves rawOutput on the returned Trajectory', () => {
1026+
const output = [
1027+
{
1028+
role: 'assistant',
1029+
content: {
1030+
format: 2,
1031+
parts: [
1032+
{
1033+
type: 'tool-invocation',
1034+
toolInvocation: { state: 'result', toolCallId: 'c1', toolName: 'tool', args: {}, result: {} },
1035+
},
1036+
],
1037+
},
1038+
},
1039+
] as any;
1040+
1041+
const result = extractTrajectory(output);
1042+
1043+
expect(result.rawOutput).toBe(output);
1044+
});
1045+
});

‎packages/core/src/evals/types.ts‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -612,8 +612,16 @@ export function extractTrajectory(output: ScorerRunOutputForAgent): Trajectory {
612612
const steps: ToolCallStep[] = [];
613613

614614
for (const message of output) {
615-
const toolInvocations = message?.content?.toolInvocations;
616-
if (!toolInvocations) continue;
615+
// Prefer the legacy toolInvocations array when present; fall back to
616+
// V2 content.parts for messages that only store tool calls there.
617+
const legacy = message?.content?.toolInvocations;
618+
const fromParts = legacy
619+
? undefined
620+
: message?.content?.parts
621+
?.filter((p): p is Extract<typeof p, { type: 'tool-invocation' }> => p.type === 'tool-invocation')
622+
.map(p => p.toolInvocation);
623+
const toolInvocations = legacy ?? fromParts;
624+
if (!toolInvocations?.length) continue;
617625

618626
for (const invocation of toolInvocations) {
619627
if (invocation && invocation.toolName && (invocation.state === 'result' || invocation.state === 'call')) {

0 commit comments

Comments
 (0)