Skip to content

Commit 390aea6

Browse files
mudlermudler
committed
Fixup anthropic tests
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
1 parent d8f875f commit 390aea6

File tree

2 files changed

+93
-66
lines changed

2 files changed

+93
-66
lines changed

‎.github/workflows/tests-e2e.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ jobs:
4646
sudo apt-get install -y build-essential
4747
- name: Test Backend E2E
4848
run: |
49-
PATH="$PATH:$HOME/go/bin" make test-e2e
49+
PATH="$PATH:$HOME/go/bin" make build-mock-backend test-e2e
5050
- name: Setup tmate session if tests fail
5151
if: ${{ failure() }}
5252
uses: mxschmitt/action-tmate@v3.23

‎tests/e2e/e2e_anthropic_test.go‎

Lines changed: 92 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,10 @@ package e2e_test
22

33
import (
44
"context"
5+
"encoding/json"
56

67
"github.com/anthropics/anthropic-sdk-go"
8+
"github.com/anthropics/anthropic-sdk-go/shared/constant"
79
"github.com/anthropics/anthropic-sdk-go/option"
810
. "github.com/onsi/ginkgo/v2"
911
. "github.com/onsi/gomega"
@@ -155,20 +157,22 @@ var _ = Describe("Anthropic API E2E test", func() {
155157
Messages: []anthropic.MessageParam{
156158
anthropic.NewUserMessage(anthropic.NewTextBlock("What's the weather like in San Francisco?")),
157159
},
158-
Tools: []anthropic.ToolParam{
159-
{
160-
Name: "get_weather",
161-
Description: anthropic.F("Get the current weather in a given location"),
162-
InputSchema: anthropic.F(map[string]interface{}{
163-
"type": "object",
164-
"properties": map[string]interface{}{
165-
"location": map[string]interface{}{
166-
"type": "string",
167-
"description": "The city and state, e.g. San Francisco, CA",
160+
Tools: []anthropic.ToolUnionParam{
161+
anthropic.ToolUnionParam{
162+
OfTool: &anthropic.ToolParam{
163+
Name: "get_weather",
164+
Description: anthropic.Opt("Get the current weather in a given location"),
165+
InputSchema: anthropic.ToolInputSchemaParam{
166+
Type: constant.ValueOf[constant.Object](),
167+
Properties: map[string]interface{}{
168+
"location": map[string]interface{}{
169+
"type": "string",
170+
"description": "The city and state, e.g. San Francisco, CA",
171+
},
168172
},
173+
Required: []string{"location"},
169174
},
170-
"required": []string{"location"},
171-
}),
175+
},
172176
},
173177
},
174178
})
@@ -179,13 +183,14 @@ var _ = Describe("Anthropic API E2E test", func() {
179183
// The model must use tools - find the tool use in the response
180184
hasToolUse := false
181185
for _, block := range message.Content {
182-
if block.Type == anthropic.ContentBlockTypeToolUse {
186+
if block.Type == "tool_use" {
183187
hasToolUse = true
184188
Expect(block.Name).To(Equal("get_weather"))
185189
Expect(block.ID).ToNot(BeEmpty())
186190
// Verify that input contains location
187-
inputMap, ok := block.Input.(map[string]interface{})
188-
Expect(ok).To(BeTrue())
191+
var inputMap map[string]interface{}
192+
err := json.Unmarshal(block.Input, &inputMap)
193+
Expect(err).ToNot(HaveOccurred())
189194
_, hasLocation := inputMap["location"]
190195
Expect(hasLocation).To(BeTrue())
191196
}
@@ -203,25 +208,27 @@ var _ = Describe("Anthropic API E2E test", func() {
203208
Messages: []anthropic.MessageParam{
204209
anthropic.NewUserMessage(anthropic.NewTextBlock("Tell me about the weather")),
205210
},
206-
Tools: []anthropic.ToolParam{
207-
{
208-
Name: "get_weather",
209-
Description: anthropic.F("Get the current weather"),
210-
InputSchema: anthropic.F(map[string]interface{}{
211-
"type": "object",
212-
"properties": map[string]interface{}{
213-
"location": map[string]interface{}{
214-
"type": "string",
211+
Tools: []anthropic.ToolUnionParam{
212+
anthropic.ToolUnionParam{
213+
OfTool: &anthropic.ToolParam{
214+
Name: "get_weather",
215+
Description: anthropic.Opt("Get the current weather"),
216+
InputSchema: anthropic.ToolInputSchemaParam{
217+
Type: constant.ValueOf[constant.Object](),
218+
Properties: map[string]interface{}{
219+
"location": map[string]interface{}{
220+
"type": "string",
221+
},
215222
},
216223
},
217-
}),
224+
},
218225
},
219226
},
220-
ToolChoice: anthropic.F[anthropic.ToolChoiceUnionParam](
221-
anthropic.ToolChoiceAutoParam{
222-
Type: anthropic.F(anthropic.ToolChoiceAutoTypeAuto),
227+
ToolChoice: anthropic.ToolChoiceUnionParam{
228+
OfAuto: &anthropic.ToolChoiceAutoParam{
229+
Type: constant.ValueOf[constant.Auto](),
223230
},
224-
),
231+
},
225232
})
226233

227234
Expect(err).ToNot(HaveOccurred())
@@ -236,16 +243,18 @@ var _ = Describe("Anthropic API E2E test", func() {
236243
Messages: []anthropic.MessageParam{
237244
anthropic.NewUserMessage(anthropic.NewTextBlock("What's the weather in SF?")),
238245
},
239-
Tools: []anthropic.ToolParam{
240-
{
241-
Name: "get_weather",
242-
Description: anthropic.F("Get weather"),
243-
InputSchema: anthropic.F(map[string]interface{}{
244-
"type": "object",
245-
"properties": map[string]interface{}{
246-
"location": map[string]interface{}{"type": "string"},
246+
Tools: []anthropic.ToolUnionParam{
247+
anthropic.ToolUnionParam{
248+
OfTool: &anthropic.ToolParam{
249+
Name: "get_weather",
250+
Description: anthropic.Opt("Get weather"),
251+
InputSchema: anthropic.ToolInputSchemaParam{
252+
Type: constant.ValueOf[constant.Object](),
253+
Properties: map[string]interface{}{
254+
"location": map[string]interface{}{"type": "string"},
255+
},
247256
},
248-
}),
257+
},
249258
},
250259
},
251260
})
@@ -256,7 +265,7 @@ var _ = Describe("Anthropic API E2E test", func() {
256265
var toolUseID string
257266
var toolUseName string
258267
for _, block := range firstMessage.Content {
259-
if block.Type == anthropic.ContentBlockTypeToolUse {
268+
if block.Type == "tool_use" {
260269
toolUseID = block.ID
261270
toolUseName = block.Name
262271
break
@@ -266,27 +275,44 @@ var _ = Describe("Anthropic API E2E test", func() {
266275
// Model must have called the tool
267276
Expect(toolUseID).ToNot(BeEmpty(), "Model should have called the get_weather tool")
268277

278+
// Convert ContentBlockUnion to ContentBlockParamUnion for NewAssistantMessage
279+
contentBlocks := make([]anthropic.ContentBlockParamUnion, len(firstMessage.Content))
280+
for i, block := range firstMessage.Content {
281+
if block.Type == "tool_use" {
282+
var inputMap map[string]interface{}
283+
if err := json.Unmarshal(block.Input, &inputMap); err == nil {
284+
contentBlocks[i] = anthropic.NewToolUseBlock(block.ID, inputMap, block.Name)
285+
} else {
286+
contentBlocks[i] = anthropic.NewToolUseBlock(block.ID, block.Input, block.Name)
287+
}
288+
} else if block.Type == "text" {
289+
contentBlocks[i] = anthropic.NewTextBlock(block.Text)
290+
}
291+
}
292+
269293
// Send back a tool result and verify it's handled correctly
270294
secondMessage, err := client.Messages.New(context.TODO(), anthropic.MessageNewParams{
271295
Model: "gpt-4",
272296
MaxTokens: 1024,
273297
Messages: []anthropic.MessageParam{
274298
anthropic.NewUserMessage(anthropic.NewTextBlock("What's the weather in SF?")),
275-
anthropic.NewAssistantMessage(firstMessage.Content...),
299+
anthropic.NewAssistantMessage(contentBlocks...),
276300
anthropic.NewUserMessage(
277301
anthropic.NewToolResultBlock(toolUseID, "Sunny, 72°F", false),
278302
),
279303
},
280-
Tools: []anthropic.ToolParam{
281-
{
282-
Name: toolUseName,
283-
Description: anthropic.F("Get weather"),
284-
InputSchema: anthropic.F(map[string]interface{}{
285-
"type": "object",
286-
"properties": map[string]interface{}{
287-
"location": map[string]interface{}{"type": "string"},
304+
Tools: []anthropic.ToolUnionParam{
305+
anthropic.ToolUnionParam{
306+
OfTool: &anthropic.ToolParam{
307+
Name: toolUseName,
308+
Description: anthropic.Opt("Get weather"),
309+
InputSchema: anthropic.ToolInputSchemaParam{
310+
Type: constant.ValueOf[constant.Object](),
311+
Properties: map[string]interface{}{
312+
"location": map[string]interface{}{"type": "string"},
313+
},
288314
},
289-
}),
315+
},
290316
},
291317
},
292318
})
@@ -302,27 +328,28 @@ var _ = Describe("Anthropic API E2E test", func() {
302328
Messages: []anthropic.MessageParam{
303329
anthropic.NewUserMessage(anthropic.NewTextBlock("What's the weather like in San Francisco?")),
304330
},
305-
Tools: []anthropic.ToolParam{
306-
{
307-
Name: "get_weather",
308-
Description: anthropic.F("Get the current weather in a given location"),
309-
InputSchema: anthropic.F(map[string]interface{}{
310-
"type": "object",
311-
"properties": map[string]interface{}{
312-
"location": map[string]interface{}{
313-
"type": "string",
314-
"description": "The city and state, e.g. San Francisco, CA",
331+
Tools: []anthropic.ToolUnionParam{
332+
anthropic.ToolUnionParam{
333+
OfTool: &anthropic.ToolParam{
334+
Name: "get_weather",
335+
Description: anthropic.Opt("Get the current weather in a given location"),
336+
InputSchema: anthropic.ToolInputSchemaParam{
337+
Type: constant.ValueOf[constant.Object](),
338+
Properties: map[string]interface{}{
339+
"location": map[string]interface{}{
340+
"type": "string",
341+
"description": "The city and state, e.g. San Francisco, CA",
342+
},
315343
},
344+
Required: []string{"location"},
316345
},
317-
"required": []string{"location"},
318-
}),
346+
},
319347
},
320348
},
321349
})
322350

323351
message := anthropic.Message{}
324352
eventCount := 0
325-
hasToolUseBlock := false
326353
hasContentBlockStart := false
327354
hasContentBlockDelta := false
328355
hasContentBlockStop := false
@@ -337,8 +364,8 @@ var _ = Describe("Anthropic API E2E test", func() {
337364
switch e := event.AsAny().(type) {
338365
case anthropic.ContentBlockStartEvent:
339366
hasContentBlockStart = true
340-
if e.ContentBlock.Type == anthropic.ContentBlockTypeToolUse {
341-
hasToolUseBlock = true
367+
if e.ContentBlock.Type == "tool_use" {
368+
// Tool use block detected
342369
}
343370
case anthropic.ContentBlockDeltaEvent:
344371
hasContentBlockDelta = true
@@ -361,7 +388,7 @@ var _ = Describe("Anthropic API E2E test", func() {
361388
// Model must have called the tool
362389
foundToolUse := false
363390
for _, block := range message.Content {
364-
if block.Type == anthropic.ContentBlockTypeToolUse {
391+
if block.Type == "tool_use" {
365392
foundToolUse = true
366393
Expect(block.Name).To(Equal("get_weather"))
367394
Expect(block.ID).ToNot(BeEmpty())

0 commit comments

Comments
 (0)