From 3b799f4e5c6778b21d86632795163daba4f6daa1 Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Fri, 24 Oct 2025 23:47:50 -0400 Subject: [PATCH 1/3] Add CodeInterpreterToolCall/ResultContent content types - Adds new CodeInterpreterToolCallContent and CodeInterpreterToolResultContent types - Updates the OpenAI Assistants and Responses IChatClient implementations to produce them - Updates HostedFileContent with an optional MediaType and Name, matching the corresponding properties on DataContent and UriContent. - Updates ToChatResponse{Async} coalescing to handle these code interpreter types. - Updates DataContent's DebuggerDisplay to show text for "text/*" and "application/json" media types. --- .../ChatCompletion/ChatResponseExtensions.cs | 112 +++++- .../Contents/AIContent.cs | 2 + .../CodeInterpreterToolCallContent.cs | 41 ++ .../CodeInterpreterToolResultContent.cs | 36 ++ .../Contents/DataContent.cs | 12 + .../Contents/HostedFileContent.cs | 43 ++- .../Contents/UriContent.cs | 1 + .../Microsoft.Extensions.AI.Abstractions.json | 8 + .../SpeechToTextResponseUpdateExtensions.cs | 4 +- .../Utilities/AIJsonUtilities.Defaults.cs | 4 + .../OpenAIAssistantsChatClient.cs | 52 +++ .../OpenAIChatClient.cs | 4 +- .../OpenAIClientExtensions.cs | 14 + .../OpenAIResponsesChatClient.cs | 46 ++- .../AdditionalPropertiesDictionaryTests.cs | 2 +- .../CodeInterpreterToolCallContentTests.cs | 93 +++++ .../CodeInterpreterToolResultContentTests.cs | 95 +++++ .../Contents/HostedFileContentTests.cs | 57 +++ .../Embeddings/BinaryEmbeddingTests.cs | 2 +- .../Embeddings/GeneratedEmbeddingsTests.cs | 2 +- .../Functions/DelegatingAIFunctionTests.cs | 2 +- .../SpeechToTextResponseUpdateTests.cs | 9 +- .../OpenAIResponseClientTests.cs | 356 ++++++++++++++++++ 23 files changed, 972 insertions(+), 25 deletions(-) create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/CodeInterpreterToolCallContent.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/CodeInterpreterToolResultContent.cs create mode 100644 test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Contents/CodeInterpreterToolCallContentTests.cs create mode 100644 test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Contents/CodeInterpreterToolResultContentTests.cs diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseExtensions.cs index 551607cc775..79bf72d0a97 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseExtensions.cs @@ -5,7 +5,11 @@ using System.Collections.Generic; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; +using System.IO; using System.Linq; +#if !NET +using System.Runtime.InteropServices; +#endif using System.Text; using System.Threading; using System.Threading.Tasks; @@ -181,7 +185,7 @@ static async Task ToChatResponseAsync( } /// Coalesces sequential content elements. - internal static void CoalesceTextContent(IList contents) + internal static void CoalesceContent(IList contents) { Coalesce( contents, @@ -215,6 +219,110 @@ internal static void CoalesceTextContent(IList contents) return content; }); + Coalesce( + contents, + mergeSingle: false, + canMerge: static (r1, r2) => r1.MediaType == r2.MediaType && r1.HasTopLevelMediaType("text") && r1.Name == r2.Name, + static (contents, start, end) => + { + Debug.Assert(end - start > 1, "Expected multiple contents to merge"); + + MemoryStream ms = new(); + for (int i = start; i < end; i++) + { + var current = (DataContent)contents[i]; +#if NET + ms.Write(current.Data.Span); +#else + if (!MemoryMarshal.TryGetArray(current.Data, out var segment)) + { + segment = new(current.Data.ToArray()); + } + + ms.Write(segment.Array!, segment.Offset, segment.Count); +#endif + } + + var first = (DataContent)contents[start]; + return new DataContent(new ReadOnlyMemory(ms.GetBuffer(), 0, (int)ms.Length), first.MediaType) { Name = first.Name }; + }); + + Coalesce( + contents, + mergeSingle: true, + canMerge: static (r1, r2) => r1.CallId is not null && r2.CallId is not null && r1.CallId == r2.CallId, + static (contents, start, end) => + { + var firstContent = (CodeInterpreterToolCallContent)contents[start]; + + if (start == end - 1) + { + if (firstContent.Inputs is not null) + { + CoalesceContent(firstContent.Inputs); + } + + return firstContent; + } + + List? inputs = null; + + for (int i = start; i < end; i++) + { + (inputs ??= []).AddRange(((CodeInterpreterToolCallContent)contents[i]).Inputs ?? []); + } + + if (inputs is not null) + { + CoalesceContent(inputs); + } + + return new() + { + CallId = firstContent.CallId, + Inputs = inputs, + AdditionalProperties = firstContent.AdditionalProperties?.Clone(), + }; + }); + + Coalesce( + contents, + mergeSingle: true, + canMerge: static (r1, r2) => r1.CallId is not null && r2.CallId is not null && r1.CallId == r2.CallId, + static (contents, start, end) => + { + var firstContent = (CodeInterpreterToolResultContent)contents[start]; + + if (start == end - 1) + { + if (firstContent.Output is not null) + { + CoalesceContent(firstContent.Output); + } + + return firstContent; + } + + List? output = null; + + for (int i = start; i < end; i++) + { + (output ??= []).AddRange(((CodeInterpreterToolResultContent)contents[i]).Output ?? []); + } + + if (output is not null) + { + CoalesceContent(output); + } + + return new() + { + CallId = firstContent.CallId, + Output = output, + AdditionalProperties = firstContent.AdditionalProperties?.Clone(), + }; + }); + static string MergeText(IList contents, int start, int end) { Debug.Assert(end - start > 1, "Expected multiple contents to merge"); @@ -318,7 +426,7 @@ private static void FinalizeResponse(ChatResponse response) int count = response.Messages.Count; for (int i = 0; i < count; i++) { - CoalesceTextContent((List)response.Messages[i].Contents); + CoalesceContent((List)response.Messages[i].Contents); } } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/AIContent.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/AIContent.cs index 8c23406cc8a..af8b19c8d84 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/AIContent.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/AIContent.cs @@ -30,6 +30,8 @@ namespace Microsoft.Extensions.AI; // [JsonDerivedType(typeof(McpServerToolResultContent), typeDiscriminator: "mcpServerToolResult")] // [JsonDerivedType(typeof(McpServerToolApprovalRequestContent), typeDiscriminator: "mcpServerToolApprovalRequest")] // [JsonDerivedType(typeof(McpServerToolApprovalResponseContent), typeDiscriminator: "mcpServerToolApprovalResponse")] +// [JsonDerivedType(typeof(CodeInterpreterToolCallContent), typeDiscriminator: "codeInterpreterToolCall")] +// [JsonDerivedType(typeof(CodeInterpreterToolResultContent), typeDiscriminator: "codeInterpreterToolResult")] public class AIContent { diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/CodeInterpreterToolCallContent.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/CodeInterpreterToolCallContent.cs new file mode 100644 index 00000000000..8e76edf8059 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/CodeInterpreterToolCallContent.cs @@ -0,0 +1,41 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents a code interpreter tool call invocation by a hosted service. +/// +/// +/// This content type represents when a hosted AI service invokes a code interpreter tool. +/// It is informational only and represents the call itself, not the result. +/// +[Experimental("MEAI001")] +public sealed class CodeInterpreterToolCallContent : AIContent +{ + /// + /// Initializes a new instance of the class. + /// + public CodeInterpreterToolCallContent() + { + } + + /// + /// Gets or sets the tool call ID. + /// + public string? CallId { get; set; } + + /// + /// Gets or sets the inputs to the code interpreter tool. + /// + /// + /// Inputs can include various types of content such as for files, + /// for binary data, or other types as supported + /// by the service. Typically includes a with a "text/x-python" + /// media type representing the code generated by the model for execution by the code interpreter tool. + /// + public IList? Inputs { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/CodeInterpreterToolResultContent.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/CodeInterpreterToolResultContent.cs new file mode 100644 index 00000000000..624d1488db9 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/CodeInterpreterToolResultContent.cs @@ -0,0 +1,36 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents the result of a code interpreter tool invocation by a hosted service. +/// +[Experimental("MEAI001")] +public sealed class CodeInterpreterToolResultContent : AIContent +{ + /// + /// Initializes a new instance of the class. + /// + public CodeInterpreterToolResultContent() + { + } + + /// + /// Gets or sets the tool call ID that this result corresponds to. + /// + public string? CallId { get; set; } + + /// + /// Gets or sets the output of code interpreter tool. + /// + /// + /// Outputs can include various types of content such as for files, + /// for binary data, for standard output text, + /// or other types as supported by the service. + /// + public IList? Output { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/DataContent.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/DataContent.cs index 7b2ef0ccb1a..fbc05e14405 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/DataContent.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/DataContent.cs @@ -9,6 +9,7 @@ #endif using System.Diagnostics; using System.Diagnostics.CodeAnalysis; +using System.Text; #if !NET using System.Runtime.InteropServices; #endif @@ -115,6 +116,7 @@ public DataContent([StringSyntax(StringSyntaxAttribute.Uri)] string uri, string? /// The media type (also known as MIME type) represented by the content. /// is . /// is empty or composed entirely of whitespace. + /// represents an invalid media type. public DataContent(ReadOnlyMemory data, string mediaType) { MediaType = DataUriParser.ThrowIfInvalidMediaType(mediaType); @@ -236,6 +238,16 @@ private string DebuggerDisplay { get { + if (HasTopLevelMediaType("text")) + { + return $"MediaType = {MediaType}, Text = \"{Encoding.UTF8.GetString(Data.ToArray())}\""; + } + + if ("application/json".Equals(MediaType, StringComparison.OrdinalIgnoreCase)) + { + return $"JSON = {Encoding.UTF8.GetString(Data.ToArray())}"; + } + const int MaxLength = 80; string uri = Uri; diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/HostedFileContent.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/HostedFileContent.cs index 1bc994b8eea..c91585540ce 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/HostedFileContent.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/HostedFileContent.cs @@ -14,11 +14,9 @@ namespace Microsoft.Extensions.AI; /// Unlike which contains the data for a file or blob, this class represents a file that is hosted /// by the AI service and referenced by an identifier. Such identifiers are specific to the provider. /// -[DebuggerDisplay("FileId = {FileId}")] +[DebuggerDisplay("{DebuggerDisplay,nq}")] public sealed class HostedFileContent : AIContent { - private string _fileId; - /// /// Initializes a new instance of the class. /// @@ -27,7 +25,7 @@ public sealed class HostedFileContent : AIContent /// is empty or composed entirely of whitespace. public HostedFileContent(string fileId) { - _fileId = Throw.IfNullOrWhitespace(fileId); + FileId = fileId; } /// @@ -37,7 +35,40 @@ public HostedFileContent(string fileId) /// is empty or composed entirely of whitespace. public string FileId { - get => _fileId; - set => _fileId = Throw.IfNullOrWhitespace(value); + get => field; + set => field = Throw.IfNullOrWhitespace(value); + } + + /// Gets or sets an optional media type (also known as MIME type) associated with the file. + /// represents an invalid media type. + public string? MediaType + { + get; + set => field = value is not null ? DataUriParser.ThrowIfInvalidMediaType(value) : value; + } + + /// Gets or sets an optional name associated with the file. + public string? Name { get; set; } + + /// Gets a string representing this instance to display in the debugger. + [DebuggerBrowsable(DebuggerBrowsableState.Never)] + private string DebuggerDisplay + { + get + { + string display = $"FileId = {FileId}"; + + if (MediaType is string mediaType) + { + display += $", MediaType = {mediaType}"; + } + + if (Name is string name) + { + display += $", Name = \"{name}\""; + } + + return display; + } } } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/UriContent.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/UriContent.cs index 73ed7f63aa7..37acd121960 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/UriContent.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/UriContent.cs @@ -67,6 +67,7 @@ public Uri Uri } /// Gets or sets the media type (also known as MIME type) for this content. + /// represents an invalid media type. public string MediaType { get => _mediaType; diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.json b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.json index d8bd5c47d3d..5ddc285e6bd 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.json +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.json @@ -2012,6 +2012,14 @@ { "Member": "string Microsoft.Extensions.AI.HostedFileContent.FileId { get; set; }", "Stage": "Stable" + }, + { + "Member": "string? Microsoft.Extensions.AI.HostedFileContent.MediaType { get; set; }", + "Stage": "Stable" + }, + { + "Member": "string? Microsoft.Extensions.AI.HostedFileContent.Name { get; set; }", + "Stage": "Stable" } ] }, diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/SpeechToText/SpeechToTextResponseUpdateExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/SpeechToText/SpeechToTextResponseUpdateExtensions.cs index 683fdb24f80..67272761ceb 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/SpeechToText/SpeechToTextResponseUpdateExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/SpeechToText/SpeechToTextResponseUpdateExtensions.cs @@ -30,7 +30,7 @@ public static SpeechToTextResponse ToSpeechToTextResponse( ProcessUpdate(update, response); } - ChatResponseExtensions.CoalesceTextContent((List)response.Contents); + ChatResponseExtensions.CoalesceContent((List)response.Contents); return response; } @@ -56,7 +56,7 @@ static async Task ToResponseAsync( ProcessUpdate(update, response); } - ChatResponseExtensions.CoalesceTextContent((List)response.Contents); + ChatResponseExtensions.CoalesceContent((List)response.Contents); return response; } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Defaults.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Defaults.cs index 36bc57fdffc..d01294836bc 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Defaults.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Defaults.cs @@ -57,6 +57,8 @@ private static JsonSerializerOptions CreateDefaultOptions() AddAIContentType(options, typeof(McpServerToolResultContent), typeDiscriminatorId: "mcpServerToolResult", checkBuiltIn: false); AddAIContentType(options, typeof(McpServerToolApprovalRequestContent), typeDiscriminatorId: "mcpServerToolApprovalRequest", checkBuiltIn: false); AddAIContentType(options, typeof(McpServerToolApprovalResponseContent), typeDiscriminatorId: "mcpServerToolApprovalResponse", checkBuiltIn: false); + AddAIContentType(options, typeof(CodeInterpreterToolCallContent), typeDiscriminatorId: "codeInterpreterToolCall", checkBuiltIn: false); + AddAIContentType(options, typeof(CodeInterpreterToolResultContent), typeDiscriminatorId: "codeInterpreterToolResult", checkBuiltIn: false); if (JsonSerializer.IsReflectionEnabledByDefault) { @@ -129,6 +131,8 @@ private static JsonSerializerOptions CreateDefaultOptions() [JsonSerializable(typeof(McpServerToolResultContent))] [JsonSerializable(typeof(McpServerToolApprovalRequestContent))] [JsonSerializable(typeof(McpServerToolApprovalResponseContent))] + [JsonSerializable(typeof(CodeInterpreterToolCallContent))] + [JsonSerializable(typeof(CodeInterpreterToolResultContent))] [JsonSerializable(typeof(ResponseContinuationToken))] // IEmbeddingGenerator diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIAssistantsChatClient.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIAssistantsChatClient.cs index 1de5dd79d4a..b6081bbb357 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIAssistantsChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIAssistantsChatClient.cs @@ -196,6 +196,58 @@ public async IAsyncEnumerable GetStreamingResponseAsync( yield return ruUpdate; break; + case RunStepDetailsUpdate details: + if (!string.IsNullOrEmpty(details.CodeInterpreterInput)) + { + CodeInterpreterToolCallContent hcitcc = new() + { + CallId = details.ToolCallId, + Inputs = [new DataContent(Encoding.UTF8.GetBytes(details.CodeInterpreterInput), "text/x-python")], + RawRepresentation = details, + }; + + yield return new ChatResponseUpdate(ChatRole.Assistant, [hcitcc]) + { + AuthorName = _assistantId, + ConversationId = threadId, + MessageId = responseId, + RawRepresentation = update, + ResponseId = responseId, + }; + } + + if (details.CodeInterpreterOutputs is { Count: > 0 }) + { + CodeInterpreterToolResultContent hcitrc = new() + { + CallId = details.ToolCallId, + RawRepresentation = details, + }; + + foreach (var output in details.CodeInterpreterOutputs) + { + if (output.ImageFileId is not null) + { + (hcitrc.Output ??= []).Add(new HostedFileContent(output.ImageFileId) { MediaType = "image/*" }); + } + + if (output.Logs is string logs) + { + (hcitrc.Output ??= []).Add(new TextContent(logs)); + } + } + + yield return new ChatResponseUpdate(ChatRole.Assistant, [hcitrc]) + { + AuthorName = _assistantId, + ConversationId = threadId, + MessageId = responseId, + RawRepresentation = update, + ResponseId = responseId, + }; + } + break; + case MessageContentUpdate mcu: ChatResponseUpdate textUpdate = new(mcu.Role == MessageRole.User ? ChatRole.User : ChatRole.Assistant, mcu.Text) { diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIChatClient.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIChatClient.cs index 1ce67c78a51..1d001103897 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIChatClient.cs @@ -708,7 +708,7 @@ internal static void ConvertContentParts(ChatMessageContent content, IList JsonSerializer.Deserialize(utf8json, OpenAIJsonContext.Default.IDictionaryStringObject)!); + /// Gets a media type for an image based on the file extension in the provided URI. + internal static string ImageUriToMediaType(Uri uri) + { + string absoluteUri = uri.AbsoluteUri; + return + absoluteUri.EndsWith(".png", StringComparison.OrdinalIgnoreCase) ? "image/png" : + absoluteUri.EndsWith(".jpg", StringComparison.OrdinalIgnoreCase) ? "image/jpeg" : + absoluteUri.EndsWith(".jpeg", StringComparison.OrdinalIgnoreCase) ? "image/jpeg" : + absoluteUri.EndsWith(".gif", StringComparison.OrdinalIgnoreCase) ? "image/gif" : + absoluteUri.EndsWith(".bmp", StringComparison.OrdinalIgnoreCase) ? "image/bmp" : + absoluteUri.EndsWith(".webp", StringComparison.OrdinalIgnoreCase) ? "image/webp" : + "image/*"; + } + /// Used to create the JSON payload for an OpenAI tool description. internal sealed class ToolJson { diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs index c1617dc0d08..19ab07a0d0a 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs @@ -9,6 +9,7 @@ using System.Linq; using System.Reflection; using System.Runtime.CompilerServices; +using System.Text; using System.Text.Json; using System.Threading; using System.Threading.Tasks; @@ -186,6 +187,10 @@ internal static IEnumerable ToChatMessages(IEnumerable ToChatMessages(IEnumerable }); break; + case StreamingResponseOutputItemDoneUpdate outputItemDoneUpdate when outputItemDoneUpdate.Item is CodeInterpreterCallResponseItem cicri: + var codeUpdate = CreateUpdate(); + AddCodeInterpreterCallContent(cicri, codeUpdate.Contents); + yield return codeUpdate; + break; + case StreamingResponseOutputItemDoneUpdate outputItemDoneUpdate when outputItemDoneUpdate.Item is MessageResponseItem mri && mri.Content is { Count: > 0 } content && @@ -918,11 +929,11 @@ private static List ToAIContents(IEnumerable con case ResponseContentPartKind.InputFile: if (!string.IsNullOrWhiteSpace(part.InputImageFileId)) { - results.Add(new HostedFileContent(part.InputImageFileId) { RawRepresentation = part }); + results.Add(new HostedFileContent(part.InputImageFileId) { MediaType = "image/*", RawRepresentation = part }); } else if (!string.IsNullOrWhiteSpace(part.InputFileId)) { - results.Add(new HostedFileContent(part.InputFileId) { RawRepresentation = part }); + results.Add(new HostedFileContent(part.InputFileId) { Name = part.InputFilename, RawRepresentation = part }); } else if (part.InputFileBytes is not null) { @@ -1016,6 +1027,33 @@ private static void AddAllMcpFilters(IList toolNames, McpToolFilter filt } } + /// Adds new for the specified into . + private static void AddCodeInterpreterCallContent(CodeInterpreterCallResponseItem cicri, IList contents) + { + contents.Add(new CodeInterpreterToolCallContent + { + CallId = cicri.Id, + Inputs = !string.IsNullOrWhiteSpace(cicri.Code) ? [new DataContent(Encoding.UTF8.GetBytes(cicri.Code), "text/x-python")] : null, + + // We purposefully do not set the RawRepresentation on the HostedCodeInterpreterToolCallContent, only on the HostedCodeInterpreterToolResultContent, to avoid + // the same CodeInterpreterCallResponseItem being included on two different AIContent instances. When these are roundtripped, we want only one + // CodeInterpreterCallResponseItem sent back for the pair. + }); + + contents.Add(new CodeInterpreterToolResultContent + { + CallId = cicri.Id, + Output = cicri.Outputs is { Count: > 0 } outputs ? outputs.Select(o => + o switch + { + CodeInterpreterCallImageOutput cicio => new UriContent(cicio.ImageUri, OpenAIClientExtensions.ImageUriToMediaType(cicio.ImageUri)) { RawRepresentation = cicio }, + CodeInterpreterCallLogsOutput ciclo => new TextContent(ciclo.Logs) { RawRepresentation = ciclo }, + _ => null, + }).OfType().ToList() : null, + RawRepresentation = cicri, + }); + } + private static OpenAIResponsesContinuationToken? CreateContinuationToken(OpenAIResponse openAIResponse) { return CreateContinuationToken( diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/AdditionalPropertiesDictionaryTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/AdditionalPropertiesDictionaryTests.cs index 09f515fa066..0635d45250c 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/AdditionalPropertiesDictionaryTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/AdditionalPropertiesDictionaryTests.cs @@ -12,7 +12,7 @@ public class AdditionalPropertiesDictionaryTests [Fact] public void Constructor_Roundtrips() { - AdditionalPropertiesDictionary d = new(); + AdditionalPropertiesDictionary d = []; Assert.Empty(d); d = new(new Dictionary { ["key1"] = "value1" }); diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Contents/CodeInterpreterToolCallContentTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Contents/CodeInterpreterToolCallContentTests.cs new file mode 100644 index 00000000000..1807f4a169a --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Contents/CodeInterpreterToolCallContentTests.cs @@ -0,0 +1,93 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Text.Json; +using Xunit; + +namespace Microsoft.Extensions.AI; + +public class CodeInterpreterToolCallContentTests +{ + [Fact] + public void Constructor_PropsDefault() + { + CodeInterpreterToolCallContent c = new(); + Assert.Null(c.RawRepresentation); + Assert.Null(c.AdditionalProperties); + Assert.Null(c.CallId); + Assert.Null(c.Inputs); + } + + [Fact] + public void Properties_Roundtrip() + { + CodeInterpreterToolCallContent c = new(); + + Assert.Null(c.CallId); + c.CallId = "call123"; + Assert.Equal("call123", c.CallId); + + Assert.Null(c.Inputs); + IList inputs = [new TextContent("print('hello')")]; + c.Inputs = inputs; + Assert.Same(inputs, c.Inputs); + + Assert.Null(c.RawRepresentation); + object raw = new(); + c.RawRepresentation = raw; + Assert.Same(raw, c.RawRepresentation); + + Assert.Null(c.AdditionalProperties); + AdditionalPropertiesDictionary props = new() { { "key", "value" } }; + c.AdditionalProperties = props; + Assert.Same(props, c.AdditionalProperties); + } + + [Fact] + public void Inputs_SupportsMultipleContentTypes() + { + CodeInterpreterToolCallContent c = new() + { + CallId = "call456", + Inputs = + [ + new TextContent("import numpy as np"), + new HostedFileContent("file123"), + new DataContent(new byte[] { 1, 2, 3 }, "application/octet-stream") + ] + }; + + Assert.NotNull(c.Inputs); + Assert.Equal(3, c.Inputs.Count); + Assert.IsType(c.Inputs[0]); + Assert.IsType(c.Inputs[1]); + Assert.IsType(c.Inputs[2]); + } + + [Fact] + public void Serialization_Roundtrips() + { + CodeInterpreterToolCallContent content = new() + { + CallId = "call123", + Inputs = + [ + new TextContent("print('hello')"), + new HostedFileContent("file456") + ] + }; + + var json = JsonSerializer.Serialize(content, AIJsonUtilities.DefaultOptions); + var deserializedSut = JsonSerializer.Deserialize(json, AIJsonUtilities.DefaultOptions); + + Assert.NotNull(deserializedSut); + Assert.Equal("call123", deserializedSut.CallId); + Assert.NotNull(deserializedSut.Inputs); + Assert.Equal(2, deserializedSut.Inputs.Count); + Assert.IsType(deserializedSut.Inputs[0]); + Assert.Equal("print('hello')", ((TextContent)deserializedSut.Inputs[0]).Text); + Assert.IsType(deserializedSut.Inputs[1]); + Assert.Equal("file456", ((HostedFileContent)deserializedSut.Inputs[1]).FileId); + } +} diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Contents/CodeInterpreterToolResultContentTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Contents/CodeInterpreterToolResultContentTests.cs new file mode 100644 index 00000000000..403f23214ab --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Contents/CodeInterpreterToolResultContentTests.cs @@ -0,0 +1,95 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Text.Json; +using Xunit; + +namespace Microsoft.Extensions.AI; + +public class CodeInterpreterToolResultContentTests +{ + [Fact] + public void Constructor_PropsDefault() + { + CodeInterpreterToolResultContent c = new(); + Assert.Null(c.RawRepresentation); + Assert.Null(c.AdditionalProperties); + Assert.Null(c.CallId); + Assert.Null(c.Output); + } + + [Fact] + public void Properties_Roundtrip() + { + CodeInterpreterToolResultContent c = new(); + + Assert.Null(c.CallId); + c.CallId = "call123"; + Assert.Equal("call123", c.CallId); + + Assert.Null(c.Output); + IList output = [new TextContent("Hello, World!")]; + c.Output = output; + Assert.Same(output, c.Output); + + Assert.Null(c.RawRepresentation); + object raw = new(); + c.RawRepresentation = raw; + Assert.Same(raw, c.RawRepresentation); + + Assert.Null(c.AdditionalProperties); + AdditionalPropertiesDictionary props = new() { { "key", "value" } }; + c.AdditionalProperties = props; + Assert.Same(props, c.AdditionalProperties); + } + + [Fact] + public void Output_SupportsMultipleContentTypes() + { + CodeInterpreterToolResultContent c = new() + { + CallId = "call789", + Output = + [ + new TextContent("Execution completed"), + new HostedFileContent("output.png"), + new DataContent(new byte[] { 1, 2, 3 }, "application/octet-stream"), + new ErrorContent("Warning: deprecated function") + ] + }; + + Assert.NotNull(c.Output); + Assert.Equal(4, c.Output.Count); + Assert.IsType(c.Output[0]); + Assert.IsType(c.Output[1]); + Assert.IsType(c.Output[2]); + Assert.IsType(c.Output[3]); + } + + [Fact] + public void Serialization_Roundtrips() + { + CodeInterpreterToolResultContent content = new() + { + CallId = "call123", + Output = + [ + new TextContent("Hello, World!"), + new HostedFileContent("result.txt") + ] + }; + + var json = JsonSerializer.Serialize(content, AIJsonUtilities.DefaultOptions); + var deserializedSut = JsonSerializer.Deserialize(json, AIJsonUtilities.DefaultOptions); + + Assert.NotNull(deserializedSut); + Assert.Equal("call123", deserializedSut.CallId); + Assert.NotNull(deserializedSut.Output); + Assert.Equal(2, deserializedSut.Output.Count); + Assert.IsType(deserializedSut.Output[0]); + Assert.Equal("Hello, World!", ((TextContent)deserializedSut.Output[0]).Text); + Assert.IsType(deserializedSut.Output[1]); + Assert.Equal("result.txt", ((HostedFileContent)deserializedSut.Output[1]).FileId); + } +} diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Contents/HostedFileContentTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Contents/HostedFileContentTests.cs index e92609ef9f0..44750ae396c 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Contents/HostedFileContentTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Contents/HostedFileContentTests.cs @@ -62,4 +62,61 @@ public void Serialization_Roundtrips() Assert.NotNull(deserializedContent); Assert.Equal(content.FileId, deserializedContent.FileId); } + + [Fact] + public void MediaType_Roundtrips() + { + HostedFileContent c = new("id123"); + Assert.Null(c.MediaType); + + c.MediaType = "image/png"; + Assert.Equal("image/png", c.MediaType); + + c.MediaType = "application/pdf"; + Assert.Equal("application/pdf", c.MediaType); + + c.MediaType = null; + Assert.Null(c.MediaType); + } + + [Theory] + [InlineData("type")] + [InlineData("type//subtype")] + [InlineData("type/subtype/")] + [InlineData("type/subtype;key=")] + [InlineData("type/subtype;=value")] + [InlineData("type/subtype;key=value;another=")] + public void MediaType_InvalidValue_Throws(string invalidMediaType) + { + HostedFileContent c = new("id123"); + Assert.Throws(() => c.MediaType = invalidMediaType); + } + + [Theory] + [InlineData("image/png")] + [InlineData("image/jpeg")] + [InlineData("application/pdf")] + [InlineData("text/plain;charset=UTF-8")] + [InlineData("image/*")] + public void MediaType_ValidValue_Roundtrips(string mediaType) + { + HostedFileContent c = new("id123") { MediaType = mediaType }; + Assert.Equal(mediaType, c.MediaType); + } + + [Fact] + public void Name_Roundtrips() + { + HostedFileContent c = new("id123"); + Assert.Null(c.Name); + + c.Name = "document.pdf"; + Assert.Equal("document.pdf", c.Name); + + c.Name = "image.png"; + Assert.Equal("image.png", c.Name); + + c.Name = null; + Assert.Null(c.Name); + } } diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Embeddings/BinaryEmbeddingTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Embeddings/BinaryEmbeddingTests.cs index c75d715466e..c927a7ccf18 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Embeddings/BinaryEmbeddingTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Embeddings/BinaryEmbeddingTests.cs @@ -45,7 +45,7 @@ public void Properties_Roundtrips() Assert.Equal(createdAt, e.CreatedAt); Assert.Null(e.AdditionalProperties); - AdditionalPropertiesDictionary props = new(); + AdditionalPropertiesDictionary props = []; e.AdditionalProperties = props; Assert.Same(props, e.AdditionalProperties); } diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Embeddings/GeneratedEmbeddingsTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Embeddings/GeneratedEmbeddingsTests.cs index c13730fe604..a345af7b508 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Embeddings/GeneratedEmbeddingsTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Embeddings/GeneratedEmbeddingsTests.cs @@ -27,7 +27,7 @@ public void Ctor_ValidArgs_NoExceptions() GeneratedEmbeddings>[] instances = [ [], - new(0), + [], new(42), new([]) ]; diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Functions/DelegatingAIFunctionTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Functions/DelegatingAIFunctionTests.cs index 65a832c4f7e..b13e4d345a4 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Functions/DelegatingAIFunctionTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Functions/DelegatingAIFunctionTests.cs @@ -79,7 +79,7 @@ public async Task OverriddenInvocation_SuccessfullyInvoked() Assert.Same(inner.AdditionalProperties, actual.AdditionalProperties); Assert.Equal(inner.ToString(), actual.ToString()); - object? result = await actual.InvokeAsync(new(), CancellationToken.None); + object? result = await actual.InvokeAsync([], CancellationToken.None); Assert.Contains("84", result?.ToString()); Assert.False(innerInvoked); diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/SpeechToText/SpeechToTextResponseUpdateTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/SpeechToText/SpeechToTextResponseUpdateTests.cs index 0eae376070e..ec3ac1937e8 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/SpeechToText/SpeechToTextResponseUpdateTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/SpeechToText/SpeechToTextResponseUpdateTests.cs @@ -38,8 +38,7 @@ public void Properties_Roundtrip() Assert.Empty(update.Text); // Contents: assigning a new list then resetting to null should yield an empty list. - List newList = new(); - newList.Add(new TextContent("content1")); + List newList = [new TextContent("content1")]; update.Contents = newList; Assert.Same(newList, update.Contents); update.Contents = null; @@ -89,11 +88,11 @@ public void JsonSerialization_Roundtrips() ResponseId = "id123", StartTime = TimeSpan.FromSeconds(5), EndTime = TimeSpan.FromSeconds(10), - Contents = new List - { + Contents = + [ new TextContent("text-1"), new DataContent("data:audio/wav;base64,AQIDBA==", "application/octet-stream") - } + ] }; string json = JsonSerializer.Serialize(original, TestJsonSerializerContext.Default.SpeechToTextResponseUpdate); diff --git a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIResponseClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIResponseClientTests.cs index 15015e3b15c..aefa069948f 100644 --- a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIResponseClientTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIResponseClientTests.cs @@ -9,6 +9,7 @@ using System.IO; using System.Linq; using System.Net.Http; +using System.Text; using System.Text.Json; using System.Threading.Tasks; using Microsoft.Extensions.Caching.Distributed; @@ -2238,6 +2239,361 @@ await Assert.ThrowsAsync(async () => }); } + [Fact] + public async Task CodeInterpreterTool_NonStreaming() + { + const string Input = """ + { + "model":"gpt-4o-2024-08-06", + "input":[{ + "type":"message", + "role":"user", + "content":[{"type":"input_text","text":"Calculate the sum of numbers from 1 to 5"}] + }], + "tool_choice":"auto", + "tools":[{ + "type":"code_interpreter", + "container":{"type":"auto"} + }] + } + """; + + const string Output = """ + { + "id": "resp_0e599e83cc6642210068fb7475165481a08efc750483c7048f", + "object": "response", + "created_at": 1761309813, + "status": "completed", + "background": false, + "error": null, + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "model": "gpt-4o-2024-08-06", + "output": [ + { + "id": "ci_0e599e83cc6642210068fb7477fb9881a0811e8b0dc054b2fa", + "type": "code_interpreter_call", + "status": "completed", + "code": "# Calculating the sum of numbers from 1 to 5\nresult = sum(range(1, 6))\nresult", + "container_id": "cntr_68fb7476c384819186524b78cdc3180000a9a0fdd06b3cd4", + "outputs": null + }, + { + "id": "msg_0e599e83cc6642210068fb747e118081a08c3ed46daa9d9dcb", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "15" + } + ], + "role": "assistant" + } + ], + "parallel_tool_calls": true, + "previous_response_id": null, + "prompt_cache_key": null, + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [ + { + "type": "code_interpreter", + "container": { + "type": "auto" + } + } + ], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 225, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 34, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 259 + }, + "user": null, + "metadata": {} + } + """; + + using VerbatimHttpHandler handler = new(Input, Output); + using HttpClient httpClient = new(handler); + using IChatClient client = CreateResponseClient(httpClient, "gpt-4o-2024-08-06"); + + var response = await client.GetResponseAsync("Calculate the sum of numbers from 1 to 5", new() + { + Tools = [new HostedCodeInterpreterTool()], + }); + + Assert.NotNull(response); + Assert.Single(response.Messages); + + var message = response.Messages[0]; + Assert.Equal(ChatRole.Assistant, message.Role); + Assert.Equal(3, message.Contents.Count); + + var codeCall = Assert.IsType(message.Contents[0]); + Assert.NotNull(codeCall.Inputs); + var codeInput = Assert.IsType(Assert.Single(codeCall.Inputs)); + Assert.Equal("text/x-python", codeInput.MediaType); + + var codeResult = Assert.IsType(message.Contents[1]); + Assert.Equal(codeCall.CallId, codeResult.CallId); + + var textContent = Assert.IsType(message.Contents[2]); + Assert.Equal("15", textContent.Text); + } + + [Fact] + public async Task CodeInterpreterTool_Streaming() + { + const string Input = """ + { + "model":"gpt-4o-2024-08-06", + "input":[{ + "type":"message", + "role":"user", + "content":[{"type":"input_text","text":"Calculate the sum of numbers from 1 to 10 using Python"}] + }], + "tool_choice":"auto", + "tools":[{ + "type":"code_interpreter", + "container":{"type":"auto"} + }], + "stream":true + } + """; + + const string Output = """ + event: response.created + data: {"type":"response.created","sequence_number":0,"response":{"id":"resp_05d8f42f04f94cb80068fc3b7e07bc819eaf0d6e2c1923e564","object":"response","created_at":1761360766,"status":"in_progress","background":false,"error":null,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4o-2024-08-06","output":[],"parallel_tool_calls":true,"previous_response_id":null,"prompt_cache_key":null,"reasoning":{"effort":null,"summary":null},"safety_identifier":null,"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[{"type":"code_interpreter","container":{"type":"auto"}}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}}} + + event: response.in_progress + data: {"type":"response.in_progress","sequence_number":1,"response":{"id":"resp_05d8f42f04f94cb80068fc3b7e07bc819eaf0d6e2c1923e564","object":"response","created_at":1761360766,"status":"in_progress","background":false,"error":null,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4o-2024-08-06","output":[],"parallel_tool_calls":true,"previous_response_id":null,"prompt_cache_key":null,"reasoning":{"effort":null,"summary":null},"safety_identifier":null,"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[{"type":"code_interpreter","container":{"type":"auto"}}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}}} + + event: response.output_item.added + data: {"type":"response.output_item.added","sequence_number":2,"output_index":0,"item":{"id":"ci_05d8f42f04f94cb80068fc3b80fba8819ea3bfbdd36e94bcf3","type":"code_interpreter_call","status":"in_progress","code":"","container_id":"cntr_68fc3b80043c8191990a5837d7617af704511ed77cec9447","outputs":null}} + + event: response.code_interpreter_call.in_progress + data: {"type":"response.code_interpreter_call.in_progress","sequence_number":3,"output_index":0,"item_id":"ci_05d8f42f04f94cb80068fc3b80fba8819ea3bfbdd36e94bcf3"} + + event: response.code_interpreter_call_code.delta + data: {"type":"response.code_interpreter_call_code.delta","sequence_number":4,"output_index":0,"item_id":"ci_05d8f42f04f94cb80068fc3b80fba8819ea3bfbdd36e94bcf3","delta":"#","obfuscation":"sl1L6kjYGbL3W7b"} + + event: response.code_interpreter_call_code.delta + data: {"type":"response.code_interpreter_call_code.delta","sequence_number":5,"output_index":0,"item_id":"ci_05d8f42f04f94cb80068fc3b80fba8819ea3bfbdd36e94bcf3","delta":" Calculate","obfuscation":"nXS0Oz"} + + event: response.code_interpreter_call_code.delta + data: {"type":"response.code_interpreter_call_code.delta","sequence_number":6,"output_index":0,"item_id":"ci_05d8f42f04f94cb80068fc3b80fba8819ea3bfbdd36e94bcf3","delta":" the","obfuscation":"BeywG4wkYSPO"} + + event: response.code_interpreter_call_code.delta + data: {"type":"response.code_interpreter_call_code.delta","sequence_number":7,"output_index":0,"item_id":"ci_05d8f42f04f94cb80068fc3b80fba8819ea3bfbdd36e94bcf3","delta":" sum","obfuscation":"lQQwYY1jVUku"} + + event: response.code_interpreter_call_code.delta + data: {"type":"response.code_interpreter_call_code.delta","sequence_number":8,"output_index":0,"item_id":"ci_05d8f42f04f94cb80068fc3b80fba8819ea3bfbdd36e94bcf3","delta":" of","obfuscation":"B7ZYHyd1bTIIr"} + + event: response.code_interpreter_call_code.delta + data: {"type":"response.code_interpreter_call_code.delta","sequence_number":9,"output_index":0,"item_id":"ci_05d8f42f04f94cb80068fc3b80fba8819ea3bfbdd36e94bcf3","delta":" numbers","obfuscation":"c9P1UFe4"} + + event: response.code_interpreter_call_code.delta + data: {"type":"response.code_interpreter_call_code.delta","sequence_number":10,"output_index":0,"item_id":"ci_05d8f42f04f94cb80068fc3b80fba8819ea3bfbdd36e94bcf3","delta":" from","obfuscation":"jARdbqvpfdt"} + + event: response.code_interpreter_call_code.delta + data: {"type":"response.code_interpreter_call_code.delta","sequence_number":11,"output_index":0,"item_id":"ci_05d8f42f04f94cb80068fc3b80fba8819ea3bfbdd36e94bcf3","delta":" ","obfuscation":"wciF7LWnjGSdWPb"} + + event: response.code_interpreter_call_code.delta + data: {"type":"response.code_interpreter_call_code.delta","sequence_number":12,"output_index":0,"item_id":"ci_05d8f42f04f94cb80068fc3b80fba8819ea3bfbdd36e94bcf3","delta":"1","obfuscation":"KLuWFhT8xPOyTNH"} + + event: response.code_interpreter_call_code.delta + data: {"type":"response.code_interpreter_call_code.delta","sequence_number":13,"output_index":0,"item_id":"ci_05d8f42f04f94cb80068fc3b80fba8819ea3bfbdd36e94bcf3","delta":" to","obfuscation":"5QCNr2nNt72Hg"} + + event: response.code_interpreter_call_code.delta + data: {"type":"response.code_interpreter_call_code.delta","sequence_number":14,"output_index":0,"item_id":"ci_05d8f42f04f94cb80068fc3b80fba8819ea3bfbdd36e94bcf3","delta":" ","obfuscation":"F3vctEo2cPUvnhe"} + + event: response.code_interpreter_call_code.delta + data: {"type":"response.code_interpreter_call_code.delta","sequence_number":15,"output_index":0,"item_id":"ci_05d8f42f04f94cb80068fc3b80fba8819ea3bfbdd36e94bcf3","delta":"10","obfuscation":"JBwcgWLYbSTskz"} + + event: response.code_interpreter_call_code.delta + data: {"type":"response.code_interpreter_call_code.delta","sequence_number":16,"output_index":0,"item_id":"ci_05d8f42f04f94cb80068fc3b80fba8819ea3bfbdd36e94bcf3","delta":"\n","obfuscation":"AgU5f5WddGwHDJg"} + + event: response.code_interpreter_call_code.delta + data: {"type":"response.code_interpreter_call_code.delta","sequence_number":17,"output_index":0,"item_id":"ci_05d8f42f04f94cb80068fc3b80fba8819ea3bfbdd36e94bcf3","delta":"sum","obfuscation":"Vey8vqQPTbewO"} + + event: response.code_interpreter_call_code.delta + data: {"type":"response.code_interpreter_call_code.delta","sequence_number":18,"output_index":0,"item_id":"ci_05d8f42f04f94cb80068fc3b80fba8819ea3bfbdd36e94bcf3","delta":"_of","obfuscation":"Lyrmc5oOwdmsp"} + + event: response.code_interpreter_call_code.delta + data: {"type":"response.code_interpreter_call_code.delta","sequence_number":19,"output_index":0,"item_id":"ci_05d8f42f04f94cb80068fc3b80fba8819ea3bfbdd36e94bcf3","delta":"_numbers","obfuscation":"YxvseUG4"} + + event: response.code_interpreter_call_code.delta + data: {"type":"response.code_interpreter_call_code.delta","sequence_number":20,"output_index":0,"item_id":"ci_05d8f42f04f94cb80068fc3b80fba8819ea3bfbdd36e94bcf3","delta":" =","obfuscation":"yoo1CBUhRbgI36"} + + event: response.code_interpreter_call_code.delta + data: {"type":"response.code_interpreter_call_code.delta","sequence_number":21,"output_index":0,"item_id":"ci_05d8f42f04f94cb80068fc3b80fba8819ea3bfbdd36e94bcf3","delta":" sum","obfuscation":"pKTBmkNEE4SA"} + + event: response.code_interpreter_call_code.delta + data: {"type":"response.code_interpreter_call_code.delta","sequence_number":22,"output_index":0,"item_id":"ci_05d8f42f04f94cb80068fc3b80fba8819ea3bfbdd36e94bcf3","delta":"(range","obfuscation":"BixU5bs5ms"} + + event: response.code_interpreter_call_code.delta + data: {"type":"response.code_interpreter_call_code.delta","sequence_number":23,"output_index":0,"item_id":"ci_05d8f42f04f94cb80068fc3b80fba8819ea3bfbdd36e94bcf3","delta":"(","obfuscation":"nsarVNP46dpVYMb"} + + event: response.code_interpreter_call_code.delta + data: {"type":"response.code_interpreter_call_code.delta","sequence_number":24,"output_index":0,"item_id":"ci_05d8f42f04f94cb80068fc3b80fba8819ea3bfbdd36e94bcf3","delta":"1","obfuscation":"qkth7DCPzS6mfEd"} + + event: response.code_interpreter_call_code.delta + data: {"type":"response.code_interpreter_call_code.delta","sequence_number":25,"output_index":0,"item_id":"ci_05d8f42f04f94cb80068fc3b80fba8819ea3bfbdd36e94bcf3","delta":",","obfuscation":"J5uAitISxtjRSQA"} + + event: response.code_interpreter_call_code.delta + data: {"type":"response.code_interpreter_call_code.delta","sequence_number":26,"output_index":0,"item_id":"ci_05d8f42f04f94cb80068fc3b80fba8819ea3bfbdd36e94bcf3","delta":" ","obfuscation":"thr4ylmBRbAk0PY"} + + event: response.code_interpreter_call_code.delta + data: {"type":"response.code_interpreter_call_code.delta","sequence_number":27,"output_index":0,"item_id":"ci_05d8f42f04f94cb80068fc3b80fba8819ea3bfbdd36e94bcf3","delta":"11","obfuscation":"FWxcmwOFHJKEPJ"} + + event: response.code_interpreter_call_code.delta + data: {"type":"response.code_interpreter_call_code.delta","sequence_number":28,"output_index":0,"item_id":"ci_05d8f42f04f94cb80068fc3b80fba8819ea3bfbdd36e94bcf3","delta":"))\n","obfuscation":"ifI2JoREexe3t"} + + event: response.code_interpreter_call_code.delta + data: {"type":"response.code_interpreter_call_code.delta","sequence_number":29,"output_index":0,"item_id":"ci_05d8f42f04f94cb80068fc3b80fba8819ea3bfbdd36e94bcf3","delta":"sum","obfuscation":"VI7RlYoKWGMKP"} + + event: response.code_interpreter_call_code.delta + data: {"type":"response.code_interpreter_call_code.delta","sequence_number":30,"output_index":0,"item_id":"ci_05d8f42f04f94cb80068fc3b80fba8819ea3bfbdd36e94bcf3","delta":"_of","obfuscation":"lkv27YflY8GLq"} + + event: response.code_interpreter_call_code.delta + data: {"type":"response.code_interpreter_call_code.delta","sequence_number":31,"output_index":0,"item_id":"ci_05d8f42f04f94cb80068fc3b80fba8819ea3bfbdd36e94bcf3","delta":"_numbers","obfuscation":"xAQFrVav"} + + event: response.code_interpreter_call_code.done + data: {"type":"response.code_interpreter_call_code.done","sequence_number":32,"output_index":0,"item_id":"ci_05d8f42f04f94cb80068fc3b80fba8819ea3bfbdd36e94bcf3","code":"# Calculate the sum of numbers from 1 to 10\nsum_of_numbers = sum(range(1, 11))\nsum_of_numbers"} + + event: response.code_interpreter_call.interpreting + data: {"type":"response.code_interpreter_call.interpreting","sequence_number":33,"output_index":0,"item_id":"ci_05d8f42f04f94cb80068fc3b80fba8819ea3bfbdd36e94bcf3"} + + event: response.code_interpreter_call.completed + data: {"type":"response.code_interpreter_call.completed","sequence_number":34,"output_index":0,"item_id":"ci_05d8f42f04f94cb80068fc3b80fba8819ea3bfbdd36e94bcf3"} + + event: response.output_item.done + data: {"type":"response.output_item.done","sequence_number":35,"output_index":0,"item":{"id":"ci_05d8f42f04f94cb80068fc3b80fba8819ea3bfbdd36e94bcf3","type":"code_interpreter_call","status":"completed","code":"# Calculate the sum of numbers from 1 to 10\nsum_of_numbers = sum(range(1, 11))\nsum_of_numbers","container_id":"cntr_68fc3b80043c8191990a5837d7617af704511ed77cec9447","outputs":null}} + + event: response.output_item.added + data: {"type":"response.output_item.added","sequence_number":36,"output_index":1,"item":{"id":"msg_05d8f42f04f94cb80068fc3b86cc0c819ebae29aac8563d48d","type":"message","status":"in_progress","content":[],"role":"assistant"}} + + event: response.content_part.added + data: {"type":"response.content_part.added","sequence_number":37,"item_id":"msg_05d8f42f04f94cb80068fc3b86cc0c819ebae29aac8563d48d","output_index":1,"content_index":0,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":""}} + + event: response.output_text.delta + data: {"type":"response.output_text.delta","sequence_number":38,"item_id":"msg_05d8f42f04f94cb80068fc3b86cc0c819ebae29aac8563d48d","output_index":1,"content_index":0,"delta":"The","logprobs":[],"obfuscation":"r7iIr1QJ50aER"} + + event: response.output_text.delta + data: {"type":"response.output_text.delta","sequence_number":39,"item_id":"msg_05d8f42f04f94cb80068fc3b86cc0c819ebae29aac8563d48d","output_index":1,"content_index":0,"delta":" sum","logprobs":[],"obfuscation":"AQlXzWBYj2nu"} + + event: response.output_text.delta + data: {"type":"response.output_text.delta","sequence_number":40,"item_id":"msg_05d8f42f04f94cb80068fc3b86cc0c819ebae29aac8563d48d","output_index":1,"content_index":0,"delta":" of","logprobs":[],"obfuscation":"PpVAep2w6YBTd"} + + event: response.output_text.delta + data: {"type":"response.output_text.delta","sequence_number":41,"item_id":"msg_05d8f42f04f94cb80068fc3b86cc0c819ebae29aac8563d48d","output_index":1,"content_index":0,"delta":" numbers","logprobs":[],"obfuscation":"q2oosiA3"} + + event: response.output_text.delta + data: {"type":"response.output_text.delta","sequence_number":42,"item_id":"msg_05d8f42f04f94cb80068fc3b86cc0c819ebae29aac8563d48d","output_index":1,"content_index":0,"delta":" from","logprobs":[],"obfuscation":"BBLhSYyDDUG"} + + event: response.output_text.delta + data: {"type":"response.output_text.delta","sequence_number":43,"item_id":"msg_05d8f42f04f94cb80068fc3b86cc0c819ebae29aac8563d48d","output_index":1,"content_index":0,"delta":" ","logprobs":[],"obfuscation":"itOENAMFwzo6ESp"} + + event: response.output_text.delta + data: {"type":"response.output_text.delta","sequence_number":44,"item_id":"msg_05d8f42f04f94cb80068fc3b86cc0c819ebae29aac8563d48d","output_index":1,"content_index":0,"delta":"1","logprobs":[],"obfuscation":"g93CJ2MyMSbq26V"} + + event: response.output_text.delta + data: {"type":"response.output_text.delta","sequence_number":45,"item_id":"msg_05d8f42f04f94cb80068fc3b86cc0c819ebae29aac8563d48d","output_index":1,"content_index":0,"delta":" to","logprobs":[],"obfuscation":"WyzXmwaUVATTs"} + + event: response.output_text.delta + data: {"type":"response.output_text.delta","sequence_number":46,"item_id":"msg_05d8f42f04f94cb80068fc3b86cc0c819ebae29aac8563d48d","output_index":1,"content_index":0,"delta":" ","logprobs":[],"obfuscation":"DBgQSKk2myfDWpq"} + + event: response.output_text.delta + data: {"type":"response.output_text.delta","sequence_number":47,"item_id":"msg_05d8f42f04f94cb80068fc3b86cc0c819ebae29aac8563d48d","output_index":1,"content_index":0,"delta":"10","logprobs":[],"obfuscation":"RA4RYQSLNug4pg"} + + event: response.output_text.delta + data: {"type":"response.output_text.delta","sequence_number":48,"item_id":"msg_05d8f42f04f94cb80068fc3b86cc0c819ebae29aac8563d48d","output_index":1,"content_index":0,"delta":" is","logprobs":[],"obfuscation":"CgEfKJVj1DJtz"} + + event: response.output_text.delta + data: {"type":"response.output_text.delta","sequence_number":49,"item_id":"msg_05d8f42f04f94cb80068fc3b86cc0c819ebae29aac8563d48d","output_index":1,"content_index":0,"delta":" ","logprobs":[],"obfuscation":"TVg4ccd4ZMwEiru"} + + event: response.output_text.delta + data: {"type":"response.output_text.delta","sequence_number":50,"item_id":"msg_05d8f42f04f94cb80068fc3b86cc0c819ebae29aac8563d48d","output_index":1,"content_index":0,"delta":"55","logprobs":[],"obfuscation":"CVN92VujTbUiZ0"} + + event: response.output_text.delta + data: {"type":"response.output_text.delta","sequence_number":51,"item_id":"msg_05d8f42f04f94cb80068fc3b86cc0c819ebae29aac8563d48d","output_index":1,"content_index":0,"delta":".","logprobs":[],"obfuscation":"7YegRUzag3K6fdV"} + + event: response.output_text.done + data: {"type":"response.output_text.done","sequence_number":52,"item_id":"msg_05d8f42f04f94cb80068fc3b86cc0c819ebae29aac8563d48d","output_index":1,"content_index":0,"text":"The sum of numbers from 1 to 10 is 55.","logprobs":[]} + + event: response.content_part.done + data: {"type":"response.content_part.done","sequence_number":53,"item_id":"msg_05d8f42f04f94cb80068fc3b86cc0c819ebae29aac8563d48d","output_index":1,"content_index":0,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":"The sum of numbers from 1 to 10 is 55."}} + + event: response.output_item.done + data: {"type":"response.output_item.done","sequence_number":54,"output_index":1,"item":{"id":"msg_05d8f42f04f94cb80068fc3b86cc0c819ebae29aac8563d48d","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"The sum of numbers from 1 to 10 is 55."}],"role":"assistant"}} + + event: response.completed + data: {"type":"response.completed","sequence_number":55,"response":{"id":"resp_05d8f42f04f94cb80068fc3b7e07bc819eaf0d6e2c1923e564","object":"response","created_at":1761360766,"status":"completed","background":false,"error":null,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4o-2024-08-06","output":[{"id":"ci_05d8f42f04f94cb80068fc3b80fba8819ea3bfbdd36e94bcf3","type":"code_interpreter_call","status":"completed","code":"# Calculate the sum of numbers from 1 to 10\nsum_of_numbers = sum(range(1, 11))\nsum_of_numbers","container_id":"cntr_68fc3b80043c8191990a5837d7617af704511ed77cec9447","outputs":null},{"id":"msg_05d8f42f04f94cb80068fc3b86cc0c819ebae29aac8563d48d","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"The sum of numbers from 1 to 10 is 55."}],"role":"assistant"}],"parallel_tool_calls":true,"previous_response_id":null,"prompt_cache_key":null,"reasoning":{"effort":null,"summary":null},"safety_identifier":null,"service_tier":"default","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[{"type":"code_interpreter","container":{"type":"auto"}}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":{"input_tokens":219,"input_tokens_details":{"cached_tokens":0},"output_tokens":50,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":269},"user":null,"metadata":{}}} + + + """; + + using VerbatimHttpHandler handler = new(Input, Output); + using HttpClient httpClient = new(handler); + using IChatClient client = CreateResponseClient(httpClient, "gpt-4o-2024-08-06"); + + var response = await client.GetStreamingResponseAsync("Calculate the sum of numbers from 1 to 10 using Python", new() + { + Tools = [new HostedCodeInterpreterTool()], + }).ToChatResponseAsync(); + + Assert.NotNull(response); + Assert.Single(response.Messages); + + var message = response.Messages[0]; + Assert.Equal(ChatRole.Assistant, message.Role); + Assert.Equal(3, message.Contents.Count); + + var codeCall = Assert.IsType(message.Contents[0]); + Assert.NotNull(codeCall.Inputs); + var codeInput = Assert.IsType(Assert.Single(codeCall.Inputs)); + Assert.Equal("text/x-python", codeInput.MediaType); + Assert.Contains("sum_of_numbers", Encoding.UTF8.GetString(codeInput.Data.ToArray())); + + var codeResult = Assert.IsType(message.Contents[1]); + Assert.Equal(codeCall.CallId, codeResult.CallId); + + var textContent = Assert.IsType(message.Contents[2]); + Assert.Equal("The sum of numbers from 1 to 10 is 55.", textContent.Text); + + Assert.NotNull(response.Usage); + Assert.Equal(219, response.Usage.InputTokenCount); + Assert.Equal(50, response.Usage.OutputTokenCount); + Assert.Equal(269, response.Usage.TotalTokenCount); + } + [Fact] public async Task RequestHeaders_UserAgent_ContainsMEAI() { From 850298c2166b097e0ffa9a3e668445864f3c13cc Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Tue, 28 Oct 2025 10:03:37 -0400 Subject: [PATCH 2/3] Update src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/CodeInterpreterToolCallContent.cs Co-authored-by: Roger Barreto <19890735+rogerbarreto@users.noreply.github.com> --- .../Contents/CodeInterpreterToolCallContent.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/CodeInterpreterToolCallContent.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/CodeInterpreterToolCallContent.cs index 8e76edf8059..31681b171be 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/CodeInterpreterToolCallContent.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/CodeInterpreterToolCallContent.cs @@ -35,7 +35,7 @@ public CodeInterpreterToolCallContent() /// Inputs can include various types of content such as for files, /// for binary data, or other types as supported /// by the service. Typically includes a with a "text/x-python" - /// media type representing the code generated by the model for execution by the code interpreter tool. + /// media type representing the code for execution by the code interpreter tool. /// public IList? Inputs { get; set; } } From 323db59e1a784ffec7e26b874234340a7cfbab7e Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Wed, 29 Oct 2025 13:11:27 -0400 Subject: [PATCH 3/3] Address PR feedback --- .../ChatCompletion/ChatResponseExtensions.cs | 10 ++--- .../CodeInterpreterToolResultContent.cs | 2 +- .../OpenAIAssistantsChatClient.cs | 4 +- .../OpenAIResponsesChatClient.cs | 8 ++-- .../CodeInterpreterToolResultContentTests.cs | 36 +++++++++--------- ...enAIAssistantChatClientIntegrationTests.cs | 37 +++++++++++++++++++ .../OpenAIResponseClientIntegrationTests.cs | 28 ++++++++++++++ 7 files changed, 95 insertions(+), 30 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseExtensions.cs index 79bf72d0a97..733b885b39a 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseExtensions.cs @@ -250,7 +250,7 @@ internal static void CoalesceContent(IList contents) Coalesce( contents, mergeSingle: true, - canMerge: static (r1, r2) => r1.CallId is not null && r2.CallId is not null && r1.CallId == r2.CallId, + canMerge: static (r1, r2) => r1.CallId == r2.CallId, static (contents, start, end) => { var firstContent = (CodeInterpreterToolCallContent)contents[start]; @@ -295,9 +295,9 @@ internal static void CoalesceContent(IList contents) if (start == end - 1) { - if (firstContent.Output is not null) + if (firstContent.Outputs is not null) { - CoalesceContent(firstContent.Output); + CoalesceContent(firstContent.Outputs); } return firstContent; @@ -307,7 +307,7 @@ internal static void CoalesceContent(IList contents) for (int i = start; i < end; i++) { - (output ??= []).AddRange(((CodeInterpreterToolResultContent)contents[i]).Output ?? []); + (output ??= []).AddRange(((CodeInterpreterToolResultContent)contents[i]).Outputs ?? []); } if (output is not null) @@ -318,7 +318,7 @@ internal static void CoalesceContent(IList contents) return new() { CallId = firstContent.CallId, - Output = output, + Outputs = output, AdditionalProperties = firstContent.AdditionalProperties?.Clone(), }; }); diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/CodeInterpreterToolResultContent.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/CodeInterpreterToolResultContent.cs index 624d1488db9..486ee7072ea 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/CodeInterpreterToolResultContent.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/CodeInterpreterToolResultContent.cs @@ -32,5 +32,5 @@ public CodeInterpreterToolResultContent() /// for binary data, for standard output text, /// or other types as supported by the service. /// - public IList? Output { get; set; } + public IList? Outputs { get; set; } } diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIAssistantsChatClient.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIAssistantsChatClient.cs index b6081bbb357..065ad80d23a 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIAssistantsChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIAssistantsChatClient.cs @@ -228,12 +228,12 @@ public async IAsyncEnumerable GetStreamingResponseAsync( { if (output.ImageFileId is not null) { - (hcitrc.Output ??= []).Add(new HostedFileContent(output.ImageFileId) { MediaType = "image/*" }); + (hcitrc.Outputs ??= []).Add(new HostedFileContent(output.ImageFileId) { MediaType = "image/*" }); } if (output.Logs is string logs) { - (hcitrc.Output ??= []).Add(new TextContent(logs)); + (hcitrc.Outputs ??= []).Add(new TextContent(logs)); } } diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs index 19ab07a0d0a..3634a83e2a6 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs @@ -211,7 +211,7 @@ internal static IEnumerable ToChatMessages(IEnumerable case StreamingResponseOutputItemDoneUpdate outputItemDoneUpdate when outputItemDoneUpdate.Item is CodeInterpreterCallResponseItem cicri: var codeUpdate = CreateUpdate(); - AddCodeInterpreterCallContent(cicri, codeUpdate.Contents); + AddCodeInterpreterContents(cicri, codeUpdate.Contents); yield return codeUpdate; break; @@ -1028,7 +1028,7 @@ private static void AddAllMcpFilters(IList toolNames, McpToolFilter filt } /// Adds new for the specified into . - private static void AddCodeInterpreterCallContent(CodeInterpreterCallResponseItem cicri, IList contents) + private static void AddCodeInterpreterContents(CodeInterpreterCallResponseItem cicri, IList contents) { contents.Add(new CodeInterpreterToolCallContent { @@ -1043,7 +1043,7 @@ private static void AddCodeInterpreterCallContent(CodeInterpreterCallResponseIte contents.Add(new CodeInterpreterToolResultContent { CallId = cicri.Id, - Output = cicri.Outputs is { Count: > 0 } outputs ? outputs.Select(o => + Outputs = cicri.Outputs is { Count: > 0 } outputs ? outputs.Select(o => o switch { CodeInterpreterCallImageOutput cicio => new UriContent(cicio.ImageUri, OpenAIClientExtensions.ImageUriToMediaType(cicio.ImageUri)) { RawRepresentation = cicio }, diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Contents/CodeInterpreterToolResultContentTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Contents/CodeInterpreterToolResultContentTests.cs index 403f23214ab..6fb1303be53 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Contents/CodeInterpreterToolResultContentTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Contents/CodeInterpreterToolResultContentTests.cs @@ -16,7 +16,7 @@ public void Constructor_PropsDefault() Assert.Null(c.RawRepresentation); Assert.Null(c.AdditionalProperties); Assert.Null(c.CallId); - Assert.Null(c.Output); + Assert.Null(c.Outputs); } [Fact] @@ -28,10 +28,10 @@ public void Properties_Roundtrip() c.CallId = "call123"; Assert.Equal("call123", c.CallId); - Assert.Null(c.Output); + Assert.Null(c.Outputs); IList output = [new TextContent("Hello, World!")]; - c.Output = output; - Assert.Same(output, c.Output); + c.Outputs = output; + Assert.Same(output, c.Outputs); Assert.Null(c.RawRepresentation); object raw = new(); @@ -50,7 +50,7 @@ public void Output_SupportsMultipleContentTypes() CodeInterpreterToolResultContent c = new() { CallId = "call789", - Output = + Outputs = [ new TextContent("Execution completed"), new HostedFileContent("output.png"), @@ -59,12 +59,12 @@ public void Output_SupportsMultipleContentTypes() ] }; - Assert.NotNull(c.Output); - Assert.Equal(4, c.Output.Count); - Assert.IsType(c.Output[0]); - Assert.IsType(c.Output[1]); - Assert.IsType(c.Output[2]); - Assert.IsType(c.Output[3]); + Assert.NotNull(c.Outputs); + Assert.Equal(4, c.Outputs.Count); + Assert.IsType(c.Outputs[0]); + Assert.IsType(c.Outputs[1]); + Assert.IsType(c.Outputs[2]); + Assert.IsType(c.Outputs[3]); } [Fact] @@ -73,7 +73,7 @@ public void Serialization_Roundtrips() CodeInterpreterToolResultContent content = new() { CallId = "call123", - Output = + Outputs = [ new TextContent("Hello, World!"), new HostedFileContent("result.txt") @@ -85,11 +85,11 @@ public void Serialization_Roundtrips() Assert.NotNull(deserializedSut); Assert.Equal("call123", deserializedSut.CallId); - Assert.NotNull(deserializedSut.Output); - Assert.Equal(2, deserializedSut.Output.Count); - Assert.IsType(deserializedSut.Output[0]); - Assert.Equal("Hello, World!", ((TextContent)deserializedSut.Output[0]).Text); - Assert.IsType(deserializedSut.Output[1]); - Assert.Equal("result.txt", ((HostedFileContent)deserializedSut.Output[1]).FileId); + Assert.NotNull(deserializedSut.Outputs); + Assert.Equal(2, deserializedSut.Outputs.Count); + Assert.IsType(deserializedSut.Outputs[0]); + Assert.Equal("Hello, World!", ((TextContent)deserializedSut.Outputs[0]).Text); + Assert.IsType(deserializedSut.Outputs[1]); + Assert.Equal("result.txt", ((HostedFileContent)deserializedSut.Outputs[1]).FileId); } } diff --git a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIAssistantChatClientIntegrationTests.cs b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIAssistantChatClientIntegrationTests.cs index 4ef19182cbf..ef9d6063ddd 100644 --- a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIAssistantChatClientIntegrationTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIAssistantChatClientIntegrationTests.cs @@ -60,6 +60,43 @@ public async Task UseCodeInterpreter_ProducesCodeExecutionResults() ChatMessage message = Assert.Single(response.Messages); Assert.NotEmpty(message.Text); + + // Validate CodeInterpreterToolCallContent + var toolCallContent = response.Messages.SelectMany(m => m.Contents).OfType().SingleOrDefault(); + Assert.NotNull(toolCallContent); + if (toolCallContent.CallId is not null) + { + Assert.NotEmpty(toolCallContent.CallId); + } + + if (toolCallContent.Inputs is not null) + { + Assert.NotEmpty(toolCallContent.Inputs); + if (toolCallContent.Inputs.OfType().FirstOrDefault() is { } codeInput) + { + Assert.Equal("text/x-python", codeInput.MediaType); + Assert.NotEmpty(codeInput.Data.ToArray()); + } + } + + // Validate CodeInterpreterToolResultContent (when present) + var toolResultContents = response.Messages.SelectMany(m => m.Contents).OfType().ToList(); + foreach (var toolResultContent in toolResultContents) + { + if (toolResultContent.CallId is not null) + { + Assert.NotEmpty(toolResultContent.CallId); + } + + if (toolResultContent.Outputs is not null) + { + Assert.NotEmpty(toolResultContent.Outputs); + if (toolResultContent.Outputs.OfType().FirstOrDefault() is { } resultOutput) + { + Assert.NotEmpty(resultOutput.Text); + } + } + } } // [Fact] // uncomment and run to clear out _all_ threads in your OpenAI account diff --git a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIResponseClientIntegrationTests.cs b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIResponseClientIntegrationTests.cs index cd3993a521a..f62bff533c2 100644 --- a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIResponseClientIntegrationTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIResponseClientIntegrationTests.cs @@ -36,6 +36,34 @@ public async Task UseCodeInterpreter_ProducesCodeExecutionResults() ChatMessage message = Assert.Single(response.Messages); Assert.Equal("6", message.Text); + + // Validate CodeInterpreterToolCallContent + var toolCallContent = response.Messages.SelectMany(m => m.Contents).OfType().SingleOrDefault(); + Assert.NotNull(toolCallContent); + Assert.NotNull(toolCallContent.CallId); + Assert.NotEmpty(toolCallContent.CallId); + Assert.NotNull(toolCallContent.Inputs); + Assert.NotEmpty(toolCallContent.Inputs); + + var codeInput = toolCallContent.Inputs.OfType().FirstOrDefault(); + Assert.NotNull(codeInput); + Assert.Equal("text/x-python", codeInput.MediaType); + Assert.NotEmpty(codeInput.Data.ToArray()); + + // Validate CodeInterpreterToolResultContent + var toolResultContent = response.Messages.SelectMany(m => m.Contents).OfType().FirstOrDefault(); + Assert.NotNull(toolResultContent); + Assert.NotNull(toolResultContent.CallId); + Assert.NotEmpty(toolResultContent.CallId); + + if (toolResultContent.Outputs is not null) + { + Assert.NotEmpty(toolResultContent.Outputs); + if (toolResultContent.Outputs.OfType().FirstOrDefault() is { } resultOutput) + { + Assert.NotEmpty(resultOutput.Text); + } + } } [ConditionalFact]