diff --git a/apps/server/package.json b/apps/server/package.json index 32d9e2e922b22..e33d24ede4cad 100644 --- a/apps/server/package.json +++ b/apps/server/package.json @@ -32,7 +32,9 @@ "diff": "8.0.3", "effect": "catalog:", "jose": "catalog:", + "jpeg-js": "^0.4.4", "node-pty": "^1.2.0-beta.15", + "pngjs": "^7.0.0", "proper-lockfile": "4.1.2", "stream-chain": "^4.2.5", "stream-json": "3.6.0", @@ -47,6 +49,7 @@ "@t3tools/tailscale": "workspace:*", "@t3tools/web": "workspace:*", "@types/node": "catalog:", + "@types/pngjs": "6.0.5", "@types/proper-lockfile": "^4.1.4", "@types/yauzl": "^3.4.0", "effect-acp": "workspace:*", diff --git a/apps/server/src/provider/Layers/ClaudeAdapter.test.ts b/apps/server/src/provider/Layers/ClaudeAdapter.test.ts index f00c58d963534..8393707b1e750 100644 --- a/apps/server/src/provider/Layers/ClaudeAdapter.test.ts +++ b/apps/server/src/provider/Layers/ClaudeAdapter.test.ts @@ -4,6 +4,7 @@ import { buildRuntimeInstructions } from "../RuntimeInstructions.ts"; import * as NodeFS from "node:fs"; import * as NodeOS from "node:os"; import * as NodePath from "node:path"; +import { PNG } from "pngjs"; import * as NodeServices from "@effect/platform-node/NodeServices"; import type { @@ -23,6 +24,7 @@ import { ThreadId, ProviderInstanceId, } from "@t3tools/contracts"; +import { readImageDimensions } from "@t3tools/shared/imageDimensions"; import { createModelSelection } from "@t3tools/shared/model"; import { assert, describe, it } from "@effect/vitest"; import * as Clock from "effect/Clock"; @@ -1006,6 +1008,64 @@ describe("ClaudeAdapterLive", () => { ); }); + // Claude Code scales a turn's opening images but stores steered ones as + // sent, and the API rejects any image over 2000px once a request holds more + // than 20 images. + it.effect("scales images over 2000px only when steering a running turn", () => { + const baseDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "claude-attachments-")); + const harness = makeHarness({ cwd: "/tmp/project-claude-steered-image", baseDir }); + return Effect.gen(function* () { + yield* Effect.addFinalizer(() => + Effect.sync(() => NodeFS.rmSync(baseDir, { recursive: true, force: true })), + ); + + const adapter = yield* ClaudeAdapter; + const { attachmentsDir } = yield* ServerConfig; + + const attachment = { + type: "image" as const, + id: "thread-claude-attachment-12345678-1234-1234-1234-123456789abd", + name: "screenshot.png", + mimeType: "image/png", + sizeBytes: 0, + }; + const attachmentPath = NodePath.join(attachmentsDir, attachmentRelativePath(attachment)!); + NodeFS.mkdirSync(NodePath.dirname(attachmentPath), { recursive: true }); + NodeFS.writeFileSync(attachmentPath, PNG.sync.write(new PNG({ width: 1080, height: 2424 }))); + + const session = yield* adapter.startSession({ + threadId: THREAD_ID, + provider: ProviderDriverKind.make("claudeAgent"), + runtimeMode: "full-access", + }); + yield* adapter.sendTurn({ + threadId: session.threadId, + input: "opening", + attachments: [attachment], + }); + yield* adapter.sendTurn({ + threadId: session.threadId, + input: "steer", + attachments: [attachment], + }); + + const messages = yield* Effect.promise(() => + readPromptMessages(harness.getLastCreateQueryInput(), 2), + ); + const imageSizes = messages.map(({ message }) => { + const [block] = message.content as Array<{ source: { data: string } }>; + return readImageDimensions(Buffer.from(block!.source.data, "base64")); + }); + assert.deepEqual(imageSizes, [ + { width: 1080, height: 2424 }, + { width: 891, height: 2000 }, + ]); + }).pipe( + Effect.provideService(Random.Random, makeDeterministicRandomService()), + Effect.provide(harness.layer), + ); + }); + // The Claude CLI reads a streamed user message as a slash-command invocation // only when the final content block is text. Leading with the text block sent // every image-carrying turn down the plain-prompt path, so `/skill args` diff --git a/apps/server/src/provider/Layers/ClaudeAdapter.ts b/apps/server/src/provider/Layers/ClaudeAdapter.ts index d971566217e60..8d658135880a8 100644 --- a/apps/server/src/provider/Layers/ClaudeAdapter.ts +++ b/apps/server/src/provider/Layers/ClaudeAdapter.ts @@ -28,6 +28,7 @@ import { import { parseCliArgs } from "@t3tools/shared/cliArgs"; import { isWorkspaceImagePreviewPath } from "@t3tools/shared/filePreview"; import { type ClaudeScopedLimitNames, claudeRateLimitEventToUpdate } from "./claudeUsageLimits.ts"; +import { fitClaudeImage } from "./claudeImageSize.ts"; import { ApprovalRequestId, classifyTaskAgentKind, @@ -1608,6 +1609,8 @@ const buildUserMessageEffect = Effect.fn("buildUserMessageEffect")(function* ( readonly modelCatalog: ClaudeModelCatalog; /** Names of the skills Claude Code can run for this session's cwd. */ readonly skillNames: ReadonlySet; + /** Whether the message steers a running turn instead of opening one. */ + readonly steering: boolean; }, ) { const text = buildPromptText(input, dependencies.boundInstanceId, dependencies.modelCatalog); @@ -1661,10 +1664,30 @@ const buildUserMessageEffect = Effect.fn("buildUserMessageEffect")(function* ( ), ); + // Claude Code scales the images in a turn's opening message, but stores + // steered ones as sent. A corrupt image is still sent; the provider + // reports it. + const fittedBytes = dependencies.steering + ? yield* Effect.try(() => fitClaudeImage(attachment.mimeType, bytes)).pipe( + Effect.catch((cause) => + Effect.logWarning("Failed to downscale Claude image attachment.", { cause }).pipe( + Effect.as(bytes), + ), + ), + ) + : bytes; + if (fittedBytes === null) { + return yield* new ProviderAdapterRequestError({ + provider: PROVIDER, + method: "turn/start", + detail: `Image '${attachment.name}' is too large to send while Claude is working. Resize it or send it after the turn finishes.`, + }); + } + sdkContent.push( buildClaudeImageContentBlock({ mimeType: attachment.mimeType, - bytes, + bytes: fittedBytes, }), ); } @@ -5295,6 +5318,7 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( .filter((skill) => skill.enabled && skill.userInvocable !== false) .map((skill) => skill.name), ), + steering: steeringTurnState !== null, }); if (steeringTurnState === null) context.turnStartMessageIds.push(turnId); diff --git a/apps/server/src/provider/Layers/claudeImageSize.test.ts b/apps/server/src/provider/Layers/claudeImageSize.test.ts new file mode 100644 index 0000000000000..9bb3ab118ff87 --- /dev/null +++ b/apps/server/src/provider/Layers/claudeImageSize.test.ts @@ -0,0 +1,95 @@ +import { readImageDimensions } from "@t3tools/shared/imageDimensions"; +import jpeg from "jpeg-js"; +import { PNG } from "pngjs"; +import { describe, expect, it } from "vite-plus/test"; + +import { fitClaudeImage } from "./claudeImageSize.ts"; + +function png(width: number, height: number, alpha: number, colorType: 0 | 6 = 6) { + const image = new PNG({ width, height }); + image.data.fill(alpha); + return PNG.sync.write(image, { colorType }); +} + +function fitted(mimeType: string, bytes: Uint8Array) { + const result = fitClaudeImage(mimeType, bytes); + if (!result) throw new Error("expected a fitted image"); + return Buffer.from(result); +} + +describe("fitClaudeImage", () => { + it("leaves an image within 2000px untouched", () => { + const bytes = png(2000, 1200, 255); + expect(fitClaudeImage("image/png", bytes)).toBe(bytes); + }); + + it("scales a tall phone screenshot to 2000px and keeps its transparency", () => { + const decoded = PNG.sync.read(fitted("image/png", png(1080, 2424, 128))); + expect({ width: decoded.width, height: decoded.height }).toEqual({ width: 891, height: 2000 }); + expect(decoded.data[3]).toBe(128); + }); + + it("does not darken colors next to transparent pixels", () => { + const image = new PNG({ width: 4000, height: 10 }); + // Alternate opaque white and fully transparent black columns. + for (let offset = 0; offset < image.data.length; offset += 4) { + image.data.fill((offset / 4) % 2 === 0 ? 255 : 0, offset, offset + 4); + } + const decoded = PNG.sync.read(fitted("image/png", PNG.sync.write(image))); + expect([...decoded.data.subarray(0, 4)]).toEqual([255, 255, 255, 128]); + }); + + it("refuses a PNG too large to decode safely", () => { + const header = Buffer.alloc(24); + header.set([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0, 0, 0, 13], 0); + header.write("IHDR", 12, "ascii"); + header.writeUInt32BE(20_000, 16); + header.writeUInt32BE(20_000, 20); + expect(fitClaudeImage("image/png", header)).toBeNull(); + }); + + it("refuses a JPEG that exceeds the decoder's memory budget", () => { + // SOI, then a 4:4:4 SOF0 frame header for 8000x6000 (48MP), then EOI. + const header = Uint8Array.from([ + 0xff, 0xd8, 0xff, 0xc0, 0x00, 0x11, 0x08, 0x17, 0x70, 0x1f, 0x40, 0x03, 0x01, 0x11, 0x00, + 0x02, 0x11, 0x00, 0x03, 0x11, 0x00, 0xff, 0xd9, + ]); + expect(fitClaudeImage("image/jpeg", header)).toBeNull(); + }); + + it("keeps a grayscale PNG grayscale", () => { + const decoded = PNG.sync.read(fitted("image/png", png(2400, 100, 90, 0))); + expect({ width: decoded.width, colorType: decoded.colorType }).toEqual({ + width: 2000, + colorType: 0, + }); + }); + + it("scales a wide JPEG and keeps its EXIF orientation", () => { + const { data } = jpeg.encode( + { + width: 3000, + height: 1000, + data: new Uint8Array(3000 * 1000 * 4).fill(200), + // EXIF with orientation 6 (rotate 90° clockwise). + exifBuffer: Buffer.from([ + 0x45, 0x78, 0x69, 0x66, 0x00, 0x00, 0x4d, 0x4d, 0x00, 0x2a, 0x00, 0x00, 0x00, 0x08, 0x00, + 0x01, 0x01, 0x12, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, + ]), + } as Parameters[0], + 90, + ); + // Orientation 6 swaps the reported sides. + expect(readImageDimensions(fitted("image/jpeg", data))).toEqual({ + width: 667, + height: 2000, + }); + }); + + it("passes an oversized GIF through", () => { + // GIF89a header, 2000x2001. + const gif = Buffer.from([0x47, 0x49, 0x46, 0x38, 0x39, 0x61, 0xd0, 0x07, 0xd1, 0x07]); + expect(fitClaudeImage("image/gif", gif)).toBe(gif); + }); +}); diff --git a/apps/server/src/provider/Layers/claudeImageSize.ts b/apps/server/src/provider/Layers/claudeImageSize.ts new file mode 100644 index 0000000000000..28916810f2c01 --- /dev/null +++ b/apps/server/src/provider/Layers/claudeImageSize.ts @@ -0,0 +1,108 @@ +import { readImageDimensions } from "@t3tools/shared/imageDimensions"; +import jpeg from "jpeg-js"; +import { PNG } from "pngjs"; + +/** + * Anthropic rejects every image over 2000px on a side once a request holds + * more than 20 images. Claude Code scales the images in a turn's opening + * message, but stores ones sent mid-turn as they are, so one oversized paste + * breaks every later image read in the session. + */ +const CLAUDE_MAX_IMAGE_EDGE = 2000; + +/** + * A highly compressible PNG fits the attachment byte cap at any size, and + * decoding expands it to 4 bytes per pixel, so past this bound a PNG or JPEG + * is not decoded. 50MP still covers an 8000x6000 photo. + */ +const MAX_DECODED_PIXELS = 50_000_000; + +/** + * Scales a PNG or JPEG down to fit CLAUDE_MAX_IMAGE_EDGE. Images already + * within the limit come back untouched, so only oversized ones pay for a + * decode. Returns null for a PNG or JPEG too large to decode safely, since + * sending it unscaled would break the thread. + * + * ponytail: decodes on the event loop, about 150ms for a phone screenshot, + * 400ms for a Retina screenshot, and 0.7s for a 12MP photo. Only steered + * images pay it. Move it to a worker if that stall shows up. + */ +export function fitClaudeImage(mimeType: string, bytes: Uint8Array): Uint8Array | null { + const dimensions = readImageDimensions(bytes); + if (!dimensions || Math.max(dimensions.width, dimensions.height) <= CLAUDE_MAX_IMAGE_EDGE) { + return bytes; + } + const decodable = dimensions.width * dimensions.height <= MAX_DECODED_PIXELS; + if (mimeType === "image/png") { + if (!decodable) return null; + const image = PNG.sync.read(Buffer.from(bytes)); + const { width, height, data } = downscale(image); + const scaled = new PNG({ width, height }); + scaled.data = Buffer.from(data.buffer); + // Keep the source's channels; always writing RGBA can make a grayscale + // or palette PNG several times larger than the original. + const colorType = image.alpha ? (image.color ? 6 : 4) : image.color ? 2 : 0; + return PNG.sync.write(scaled, { colorType }); + } + if (mimeType === "image/jpeg") { + const image = decodable ? decodeJpeg(bytes) : null; + if (!image) return null; + // The spread keeps the decoder's `exifBuffer`, so the EXIF orientation of + // a phone photo survives the re-encode. + return jpeg.encode({ ...image, ...downscale(image) }, 90).data; + } + // ponytail: GIF and WebP pass through; no pure-JS codec for them here. Add + // one if oversized GIF or WebP attachments show up in practice. + return bytes; +} + +/** Decodes a JPEG, or returns null when it would exceed jpeg-js's 512MB budget. */ +function decodeJpeg(bytes: Uint8Array) { + try { + return jpeg.decode(bytes, { useTArray: true }); + } catch (error) { + // 4:4:4 JPEGs need about 12 bytes per pixel, so some pass MAX_DECODED_PIXELS + // and still hit the decoder's memory cap. + if (error instanceof Error && error.message.startsWith("maxMemoryUsageInMB")) return null; + throw error; + } +} + +/** Box-filter downscale of RGBA pixels to fit CLAUDE_MAX_IMAGE_EDGE. */ +function downscale(image: { width: number; height: number; data: Uint8Array }) { + const scale = CLAUDE_MAX_IMAGE_EDGE / Math.max(image.width, image.height); + const width = Math.max(1, Math.round(image.width * scale)); + const height = Math.max(1, Math.round(image.height * scale)); + const data = new Uint8Array(width * height * 4); + for (let y = 0; y < height; y++) { + const top = Math.floor((y * image.height) / height); + const bottom = Math.max(top + 1, Math.floor(((y + 1) * image.height) / height)); + for (let x = 0; x < width; x++) { + const left = Math.floor((x * image.width) / width); + const right = Math.max(left + 1, Math.floor(((x + 1) * image.width) / width)); + let red = 0; + let green = 0; + let blue = 0; + let alpha = 0; + for (let sourceY = top; sourceY < bottom; sourceY++) { + for (let sourceX = left; sourceX < right; sourceX++) { + const offset = (sourceY * image.width + sourceX) * 4; + // Weighting color by alpha keeps transparent pixels from darkening edges. + const pixelAlpha = image.data[offset + 3] ?? 0; + red += (image.data[offset] ?? 0) * pixelAlpha; + green += (image.data[offset + 1] ?? 0) * pixelAlpha; + blue += (image.data[offset + 2] ?? 0) * pixelAlpha; + alpha += pixelAlpha; + } + } + const target = (y * width + x) * 4; + if (alpha > 0) { + data[target] = Math.round(red / alpha); + data[target + 1] = Math.round(green / alpha); + data[target + 2] = Math.round(blue / alpha); + } + data[target + 3] = Math.round(alpha / ((bottom - top) * (right - left))); + } + } + return { width, height, data }; +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6ef065c7af83b..f21b92566d31e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -535,9 +535,15 @@ importers: jose: specifier: 'catalog:' version: 6.2.2 + jpeg-js: + specifier: ^0.4.4 + version: 0.4.4 node-pty: specifier: ^1.2.0-beta.15 version: 1.2.0-beta.15(patch_hash=f2fe901c61cde17986240002d05c172d5d0272d83ffaab8d0ebeb922763be414) + pngjs: + specifier: ^7.0.0 + version: 7.0.0 proper-lockfile: specifier: 4.1.2 version: 4.1.2 @@ -575,6 +581,9 @@ importers: '@types/node': specifier: 24.12.4 version: 24.12.4 + '@types/pngjs': + specifier: 6.0.5 + version: 6.0.5 '@types/proper-lockfile': specifier: ^4.1.4 version: 4.1.4 @@ -8222,6 +8231,9 @@ packages: jose@6.2.3: resolution: {integrity: sha512-YYVDInQKFJfR/xa3ojUTl8c2KoTwiL1R5Wg9YCydwH0x0B9grbzlg5HC7mMjCtUJjbQ/YnGEZIhI5tCgfTb4Hw==} + jpeg-js@0.4.4: + resolution: {integrity: sha512-WZzeDOEtTOBK4Mdsar0IqEU5sMr3vSV2RqkAIzUEV2BHnUfKGyswWFPFwK5EeDo93K3FohSHbLAjj0s1Wzd+dg==} + js-base64@3.7.8: resolution: {integrity: sha512-hNngCeKxIUQiEUN3GPJOkz4wF/YvdUdbNL9hsBcMQTkKzboD7T/q3OYOuuPZLUE6dBxSGpwhk5mwuDud7JVAow==} @@ -18721,6 +18733,8 @@ snapshots: jose@6.2.3: {} + jpeg-js@0.4.4: {} + js-base64@3.7.8: {} js-cookie@3.0.8: {}