Skip to content

Commit 457143c

Browse files
graysonhicksgraysonhicks
authored
fix(datadog): drop empty user messages from LLM spans (#16785)
1 parent 4542d3b commit 457143c

5 files changed

Lines changed: 150 additions & 18 deletions

File tree

‎.changeset/gold-crews-deny.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
'@mastra/datadog': patch
3+
---
4+
5+
Fixed Datadog LLM span input formatting to remove empty user messages.

‎observability/datadog/src/bridge.test.ts‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -490,6 +490,40 @@ describe('DatadogBridge', () => {
490490
expect(mockTrace).not.toHaveBeenCalled();
491491
});
492492

493+
it('drops empty user messages from MODEL_INFERENCE input annotations', async () => {
494+
const bridge = new DatadogBridge({ mlApp: 'test', agentless: false });
495+
const spanResult = bridge.createSpan(
496+
createMockSpanOptions({
497+
type: SpanType.MODEL_INFERENCE as SpanTypeGeneric,
498+
}),
499+
)!;
500+
const apmSpan = capturedApmSpans[0];
501+
502+
const span = createMockSpan({
503+
id: spanResult.spanId,
504+
traceId: spanResult.traceId,
505+
type: SpanType.MODEL_INFERENCE,
506+
input: [
507+
{ role: 'user', content: '' },
508+
{ role: 'system', content: 'You are helpful' },
509+
{ role: 'user', content: 'Hello' },
510+
{ role: 'user', content: ' ' },
511+
],
512+
});
513+
514+
await bridge.exportTracingEvent(createTracingEvent(TracingEventType.SPAN_ENDED, span));
515+
516+
expect(mockAnnotate).toHaveBeenCalledWith(
517+
apmSpan,
518+
expect.objectContaining({
519+
inputData: [
520+
{ role: 'system', content: 'You are helpful' },
521+
{ role: 'user', content: 'Hello' },
522+
],
523+
}),
524+
);
525+
});
526+
493527
it('annotates and finishes event spans on span_started', async () => {
494528
const bridge = new DatadogBridge({ mlApp: 'test', agentless: false });
495529
const spanResult = bridge.createSpan(createMockSpanOptions())!;

‎observability/datadog/src/tracing.test.ts‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1140,6 +1140,31 @@ describe('DatadogExporter', () => {
11401140
);
11411141
});
11421142

1143+
it('drops empty user messages from MODEL_INFERENCE input annotations', async () => {
1144+
const exporter = new DatadogExporter({ mlApp: 'test', apiKey: 'test-key' });
1145+
const span = createMockSpan({
1146+
type: SpanType.MODEL_INFERENCE,
1147+
input: [
1148+
{ role: 'user', content: '' },
1149+
{ role: 'system', content: 'You are helpful' },
1150+
{ role: 'user', content: 'Hello' },
1151+
{ role: 'user', content: ' ' },
1152+
],
1153+
});
1154+
1155+
await exporter.exportTracingEvent(createTracingEvent(TracingEventType.SPAN_ENDED, span));
1156+
1157+
expect(mockAnnotate).toHaveBeenCalledWith(
1158+
expect.anything(),
1159+
expect.objectContaining({
1160+
inputData: [
1161+
{ role: 'system', content: 'You are helpful' },
1162+
{ role: 'user', content: 'Hello' },
1163+
],
1164+
}),
1165+
);
1166+
});
1167+
11431168
it('inherits modelName/modelProvider from parent MODEL_GENERATION onto MODEL_INFERENCE descendants', async () => {
11441169
const exporter = new DatadogExporter({ mlApp: 'test', apiKey: 'test-key' });
11451170
const generation = createMockSpan({

‎observability/datadog/src/utils.test.ts‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,10 @@ describe('safeStringify', () => {
161161
expect(safeStringify('hello')).toBe('"hello"');
162162
expect(safeStringify(123)).toBe('123');
163163
});
164+
165+
it('returns an empty string for undefined values', () => {
166+
expect(safeStringify(undefined)).toBe('');
167+
});
164168
});
165169

166170
describe('formatInput', () => {
@@ -179,6 +183,37 @@ describe('formatInput', () => {
179183
expect(result).toEqual(messages);
180184
});
181185

186+
it('drops empty user messages from message arrays', () => {
187+
const result = formatInput(
188+
[
189+
{ role: 'system', content: 'You are helpful' },
190+
{ role: 'user', content: '' },
191+
{ role: 'user', content: ' ' },
192+
{ role: 'assistant', content: 'What do you need?' },
193+
{ role: 'user', content: 'Hello' },
194+
],
195+
SpanType.MODEL_GENERATION,
196+
);
197+
198+
expect(result).toEqual([
199+
{ role: 'system', content: 'You are helpful' },
200+
{ role: 'assistant', content: 'What do you need?' },
201+
{ role: 'user', content: 'Hello' },
202+
]);
203+
});
204+
205+
it('drops blank string input instead of creating an empty user message', () => {
206+
const result = formatInput(' ', SpanType.MODEL_GENERATION);
207+
208+
expect(result).toEqual([]);
209+
});
210+
211+
it('drops undefined input instead of throwing during empty message filtering', () => {
212+
const result = formatInput(undefined, SpanType.MODEL_GENERATION);
213+
214+
expect(result).toEqual([]);
215+
});
216+
182217
it('stringifies object input as user message', () => {
183218
const result = formatInput({ query: 'search term', filters: { date: '2024' } }, SpanType.MODEL_GENERATION);
184219
expect(result).toEqual([{ role: 'user', content: '{"query":"search term","filters":{"date":"2024"}}' }]);
@@ -219,6 +254,22 @@ describe('formatInput', () => {
219254
expect(result).toEqual([{ role: 'user', content: 'Hi' }]);
220255
});
221256

257+
it('cleans unwrapped message arrays for MODEL_INFERENCE spans', () => {
258+
const input = {
259+
messages: [
260+
{ role: 'user', content: '' },
261+
{ role: 'system', content: 'You are helpful' },
262+
{ role: 'user', content: 'Hi' },
263+
],
264+
};
265+
const result = formatInput(input, SpanType.MODEL_INFERENCE);
266+
267+
expect(result).toEqual([
268+
{ role: 'system', content: 'You are helpful' },
269+
{ role: 'user', content: 'Hi' },
270+
]);
271+
});
272+
222273
it('unwraps Gemini { contents } request body shape', () => {
223274
const input = {
224275
contents: [{ role: 'user', parts: [{ text: 'Hello' }] }],
@@ -314,6 +365,11 @@ describe('formatOutput', () => {
314365
const result = formatOutput({ text: '', object: { ok: true } }, SpanType.MODEL_GENERATION);
315366
expect(result).toEqual([{ role: 'assistant', content: '{"ok":true}' }]);
316367
});
368+
369+
it('formats MODEL_INFERENCE output as assistant messages', () => {
370+
const result = formatOutput('Hi there!', SpanType.MODEL_INFERENCE);
371+
expect(result).toEqual([{ role: 'assistant', content: 'Hi there!' }]);
372+
});
317373
});
318374

319375
describe('non-LLM spans (TOOL_CALL)', () => {

‎observability/datadog/src/utils.ts‎

Lines changed: 30 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -129,7 +129,7 @@ export function toDate(value: Date | string | number): Date {
129129
*/
130130
export function safeStringify(data: unknown): string {
131131
try {
132-
return JSON.stringify(data);
132+
return JSON.stringify(data) ?? '';
133133
} catch {
134134
if (typeof data === 'object' && data !== null) {
135135
return `[Non-serializable ${data.constructor?.name || 'Object'}]`;
@@ -145,22 +145,34 @@ function isMessageArray(data: any): data is Array<{ role: string; content: any }
145145
return Array.isArray(data) && data.every(m => m?.role && m?.content !== undefined);
146146
}
147147

148+
function isModelDataSpan(spanType: SpanType): boolean {
149+
return (
150+
spanType === SpanType.MODEL_GENERATION || spanType === SpanType.MODEL_STEP || spanType === SpanType.MODEL_INFERENCE
151+
);
152+
}
153+
148154
/**
149155
* Checks if data is in Gemini content array format ({role, parts}[]).
150156
*/
151157
function isGeminiContentArray(data: any): data is Array<{ role: string; parts: any[] }> {
152158
return Array.isArray(data) && data.every(m => m?.role && Array.isArray(m?.parts));
153159
}
154160

161+
function toMessageContent(content: any): string {
162+
return typeof content === 'string' ? content : safeStringify(content);
163+
}
164+
155165
/**
156166
* Maps a {role, content}[] message array into the Datadog message shape,
157167
* stringifying any non-string content (e.g. multimodal part arrays).
158168
*/
159169
function toDatadogMessages(messages: Array<{ role: string; content: any }>): Array<{ role: string; content: string }> {
160-
return messages.map(m => ({
161-
role: m.role,
162-
content: typeof m.content === 'string' ? m.content : safeStringify(m.content),
163-
}));
170+
return messages
171+
.map(m => ({
172+
role: m.role,
173+
content: toMessageContent(m.content),
174+
}))
175+
.filter(m => !(m.role === 'user' && m.content.trim().length === 0));
164176
}
165177

166178
/**
@@ -182,18 +194,18 @@ function geminiContentToMessage(item: { role: string; parts: any[] }): { role: s
182194

183195
/**
184196
* Formats input data for Datadog annotations.
185-
* LLM spans use message array format; others use raw or stringified data.
197+
* Model spans use message array format; others use raw or stringified data.
186198
*/
187199
export function formatInput(input: any, spanType: SpanType): any {
188-
// LLM spans expect {role, content}[] format
189-
if (spanType === SpanType.MODEL_GENERATION || spanType === SpanType.MODEL_STEP) {
200+
// Model spans expect {role, content}[] format
201+
if (isModelDataSpan(spanType)) {
190202
// Already in message format
191203
if (isMessageArray(input)) {
192204
return toDatadogMessages(input);
193205
}
194206
// Gemini format: {role, parts} → normalize to {role, content}
195207
if (isGeminiContentArray(input)) {
196-
return input.map(geminiContentToMessage);
208+
return toDatadogMessages(input.map(geminiContentToMessage));
197209
}
198210
// Mastra wraps MODEL_GENERATION input as { messages, schema? } and Gemini
199211
// request bodies use { contents }. Unwrap so we don't bury the message array
@@ -203,32 +215,32 @@ export function formatInput(input: any, spanType: SpanType): any {
203215
return toDatadogMessages((input as any).messages);
204216
}
205217
if (isGeminiContentArray((input as any).messages)) {
206-
return (input as any).messages.map(geminiContentToMessage);
218+
return toDatadogMessages((input as any).messages.map(geminiContentToMessage));
207219
}
208220
if (isGeminiContentArray((input as any).contents)) {
209-
return (input as any).contents.map(geminiContentToMessage);
221+
return toDatadogMessages((input as any).contents.map(geminiContentToMessage));
210222
}
211223
}
212224
// String input becomes user message
213225
if (typeof input === 'string') {
214-
return [{ role: 'user', content: input }];
226+
return toDatadogMessages([{ role: 'user', content: input }]);
215227
}
216228
// Object input gets stringified as user message
217-
return [{ role: 'user', content: safeStringify(input) }];
229+
return toDatadogMessages([{ role: 'user', content: safeStringify(input) }]);
218230
}
219231

220-
// Non-LLM spans: pass through strings/arrays, stringify objects
232+
// Non-model spans: pass through strings/arrays, stringify objects
221233
if (typeof input === 'string' || Array.isArray(input)) return input;
222234
return safeStringify(input);
223235
}
224236

225237
/**
226238
* Formats output data for Datadog annotations.
227-
* LLM spans use message array format; others use raw or stringified data.
239+
* Model spans use message array format; others use raw or stringified data.
228240
*/
229241
export function formatOutput(output: any, spanType: SpanType): any {
230-
// LLM spans expect {role, content}[] format
231-
if (spanType === SpanType.MODEL_GENERATION || spanType === SpanType.MODEL_STEP) {
242+
// Model spans expect {role, content}[] format
243+
if (isModelDataSpan(spanType)) {
232244
// Already in message format
233245
if (isMessageArray(output)) {
234246
return toDatadogMessages(output);
@@ -256,7 +268,7 @@ export function formatOutput(output: any, spanType: SpanType): any {
256268
return [{ role: 'assistant', content: safeStringify(output) }];
257269
}
258270

259-
// Non-LLM spans: pass through strings, stringify objects
271+
// Non-model spans: pass through strings, stringify objects
260272
if (typeof output === 'string') return output;
261273
return safeStringify(output);
262274
}

0 commit comments

Comments
 (0)