fix(server): Claude images pasted mid-turn no longer break later image reads

apps/server/package.json

@@ -32,7 +32,9 @@
     "diff": "8.0.3",
     "effect": "catalog:",
     "jose": "catalog:",
+    "jpeg-js": "^0.4.4",
     "node-pty": "^1.2.0-beta.15",
+    "pngjs": "^7.0.0",
     "proper-lockfile": "4.1.2",
     "stream-chain": "^4.2.5",
     "stream-json": "3.6.0",
@@ -47,6 +49,7 @@
     "@t3tools/tailscale": "workspace:*",
     "@t3tools/web": "workspace:*",
     "@types/node": "catalog:",
+    "@types/pngjs": "6.0.5",
     "@types/proper-lockfile": "^4.1.4",
     "@types/yauzl": "^3.4.0",
     "effect-acp": "workspace:*",

apps/server/src/provider/Layers/ClaudeAdapter.test.ts

@@ -4,6 +4,7 @@ import { buildRuntimeInstructions } from "../RuntimeInstructions.ts";
 import * as NodeFS from "node:fs";
 import * as NodeOS from "node:os";
 import * as NodePath from "node:path";
+import { PNG } from "pngjs";
 
 import * as NodeServices from "@effect/platform-node/NodeServices";
 import type {
@@ -23,6 +24,7 @@ import {
   ThreadId,
   ProviderInstanceId,
 } from "@t3tools/contracts";
+import { readImageDimensions } from "@t3tools/shared/imageDimensions";
 import { createModelSelection } from "@t3tools/shared/model";
 import { assert, describe, it } from "@effect/vitest";
 import * as Clock from "effect/Clock";
@@ -1006,6 +1008,64 @@ describe("ClaudeAdapterLive", () => {
     );
   });
 
+  // Claude Code scales a turn's opening images but stores steered ones as
+  // sent, and the API rejects any image over 2000px once a request holds more
+  // than 20 images.
+  it.effect("scales images over 2000px only when steering a running turn", () => {
+    const baseDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "claude-attachments-"));
+    const harness = makeHarness({ cwd: "/tmp/project-claude-steered-image", baseDir });
+    return Effect.gen(function* () {
+      yield* Effect.addFinalizer(() =>
+        Effect.sync(() => NodeFS.rmSync(baseDir, { recursive: true, force: true })),
+      );
+
+      const adapter = yield* ClaudeAdapter;
+      const { attachmentsDir } = yield* ServerConfig;
+
+      const attachment = {
+        type: "image" as const,
+        id: "thread-claude-attachment-12345678-1234-1234-1234-123456789abd",
+        name: "screenshot.png",
+        mimeType: "image/png",
+        sizeBytes: 0,
+      };
+      const attachmentPath = NodePath.join(attachmentsDir, attachmentRelativePath(attachment)!);
+      NodeFS.mkdirSync(NodePath.dirname(attachmentPath), { recursive: true });
+      NodeFS.writeFileSync(attachmentPath, PNG.sync.write(new PNG({ width: 1080, height: 2424 })));
+
+      const session = yield* adapter.startSession({
+        threadId: THREAD_ID,
+        provider: ProviderDriverKind.make("claudeAgent"),
+        runtimeMode: "full-access",
+      });
+      yield* adapter.sendTurn({
+        threadId: session.threadId,
+        input: "opening",
+        attachments: [attachment],
+      });
+      yield* adapter.sendTurn({
+        threadId: session.threadId,
+        input: "steer",
+        attachments: [attachment],
+      });
+
+      const messages = yield* Effect.promise(() =>
+        readPromptMessages(harness.getLastCreateQueryInput(), 2),
+      );
+      const imageSizes = messages.map(({ message }) => {
+        const [block] = message.content as Array<{ source: { data: string } }>;
+        return readImageDimensions(Buffer.from(block!.source.data, "base64"));
+      });
+      assert.deepEqual(imageSizes, [
+        { width: 1080, height: 2424 },
+        { width: 891, height: 2000 },
+      ]);
+    }).pipe(
+      Effect.provideService(Random.Random, makeDeterministicRandomService()),
+      Effect.provide(harness.layer),
+    );
+  });
+
   // The Claude CLI reads a streamed user message as a slash-command invocation
   // only when the final content block is text. Leading with the text block sent
   // every image-carrying turn down the plain-prompt path, so `/skill args`

apps/server/src/provider/Layers/ClaudeAdapter.ts

@@ -28,6 +28,7 @@ import {
 import { parseCliArgs } from "@t3tools/shared/cliArgs";
 import { isWorkspaceImagePreviewPath } from "@t3tools/shared/filePreview";
 import { type ClaudeScopedLimitNames, claudeRateLimitEventToUpdate } from "./claudeUsageLimits.ts";
+import { fitClaudeImage } from "./claudeImageSize.ts";
 import {
   ApprovalRequestId,
   classifyTaskAgentKind,
@@ -1608,6 +1609,8 @@ const buildUserMessageEffect = Effect.fn("buildUserMessageEffect")(function* (
     readonly modelCatalog: ClaudeModelCatalog;
     /** Names of the skills Claude Code can run for this session's cwd. */
     readonly skillNames: ReadonlySet<string>;
+    /** Whether the message steers a running turn instead of opening one. */
+    readonly steering: boolean;
   },
 ) {
   const text = buildPromptText(input, dependencies.boundInstanceId, dependencies.modelCatalog);
@@ -1661,10 +1664,30 @@ const buildUserMessageEffect = Effect.fn("buildUserMessageEffect")(function* (
       ),
     );
 
+    // Claude Code scales the images in a turn's opening message, but stores
+    // steered ones as sent. A corrupt image is still sent; the provider
+    // reports it.
+    const fittedBytes = dependencies.steering
+      ? yield* Effect.try(() => fitClaudeImage(attachment.mimeType, bytes)).pipe(
+          Effect.catch((cause) =>
+            Effect.logWarning("Failed to downscale Claude image attachment.", { cause }).pipe(
+              Effect.as(bytes),
+            ),
+          ),
+        )
+      : bytes;
+    if (fittedBytes === null) {
+      return yield* new ProviderAdapterRequestError({
+        provider: PROVIDER,
+        method: "turn/start",
+        detail: `Image '${attachment.name}' is too large to send while Claude is working. Resize it or send it after the turn finishes.`,
+      });
+    }
+
     sdkContent.push(
       buildClaudeImageContentBlock({
         mimeType: attachment.mimeType,
-        bytes,
+        bytes: fittedBytes,
       }),
     );
   }
@@ -5295,6 +5318,7 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* (
           .filter((skill) => skill.enabled && skill.userInvocable !== false)
           .map((skill) => skill.name),
       ),
+      steering: steeringTurnState !== null,
     });
 
     if (steeringTurnState === null) context.turnStartMessageIds.push(turnId);

apps/server/src/provider/Layers/claudeImageSize.test.ts

@@ -0,0 +1,95 @@
+import { readImageDimensions } from "@t3tools/shared/imageDimensions";
+import jpeg from "jpeg-js";
+import { PNG } from "pngjs";
+import { describe, expect, it } from "vite-plus/test";
+
+import { fitClaudeImage } from "./claudeImageSize.ts";
+
+function png(width: number, height: number, alpha: number, colorType: 0 | 6 = 6) {
+  const image = new PNG({ width, height });
+  image.data.fill(alpha);
+  return PNG.sync.write(image, { colorType });
+}
+
+function fitted(mimeType: string, bytes: Uint8Array) {
+  const result = fitClaudeImage(mimeType, bytes);
+  if (!result) throw new Error("expected a fitted image");
+  return Buffer.from(result);
+}
+
+describe("fitClaudeImage", () => {
+  it("leaves an image within 2000px untouched", () => {
+    const bytes = png(2000, 1200, 255);
+    expect(fitClaudeImage("image/png", bytes)).toBe(bytes);
+  });
+
+  it("scales a tall phone screenshot to 2000px and keeps its transparency", () => {
+    const decoded = PNG.sync.read(fitted("image/png", png(1080, 2424, 128)));
+    expect({ width: decoded.width, height: decoded.height }).toEqual({ width: 891, height: 2000 });
+    expect(decoded.data[3]).toBe(128);
+  });
+
+  it("does not darken colors next to transparent pixels", () => {
+    const image = new PNG({ width: 4000, height: 10 });
+    // Alternate opaque white and fully transparent black columns.
+    for (let offset = 0; offset < image.data.length; offset += 4) {
+      image.data.fill((offset / 4) % 2 === 0 ? 255 : 0, offset, offset + 4);
+    }
+    const decoded = PNG.sync.read(fitted("image/png", PNG.sync.write(image)));
+    expect([...decoded.data.subarray(0, 4)]).toEqual([255, 255, 255, 128]);
+  });
+
+  it("refuses a PNG too large to decode safely", () => {
+    const header = Buffer.alloc(24);
+    header.set([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0, 0, 0, 13], 0);
+    header.write("IHDR", 12, "ascii");
+    header.writeUInt32BE(20_000, 16);
+    header.writeUInt32BE(20_000, 20);
+    expect(fitClaudeImage("image/png", header)).toBeNull();
+  });
+
+  it("refuses a JPEG that exceeds the decoder's memory budget", () => {
+    // SOI, then a 4:4:4 SOF0 frame header for 8000x6000 (48MP), then EOI.
+    const header = Uint8Array.from([
+      0xff, 0xd8, 0xff, 0xc0, 0x00, 0x11, 0x08, 0x17, 0x70, 0x1f, 0x40, 0x03, 0x01, 0x11, 0x00,
+      0x02, 0x11, 0x00, 0x03, 0x11, 0x00, 0xff, 0xd9,
+    ]);
+    expect(fitClaudeImage("image/jpeg", header)).toBeNull();
+  });
+
+  it("keeps a grayscale PNG grayscale", () => {
+    const decoded = PNG.sync.read(fitted("image/png", png(2400, 100, 90, 0)));
+    expect({ width: decoded.width, colorType: decoded.colorType }).toEqual({
+      width: 2000,
+      colorType: 0,
+    });
+  });
+
+  it("scales a wide JPEG and keeps its EXIF orientation", () => {
+    const { data } = jpeg.encode(
+      {
+        width: 3000,
+        height: 1000,
+        data: new Uint8Array(3000 * 1000 * 4).fill(200),
+        // EXIF with orientation 6 (rotate 90° clockwise).
+        exifBuffer: Buffer.from([
+          0x45, 0x78, 0x69, 0x66, 0x00, 0x00, 0x4d, 0x4d, 0x00, 0x2a, 0x00, 0x00, 0x00, 0x08, 0x00,
+          0x01, 0x01, 0x12, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00,
+          0x00, 0x00,
+        ]),
+      } as Parameters<typeof jpeg.encode>[0],
+      90,
+    );
+    // Orientation 6 swaps the reported sides.
+    expect(readImageDimensions(fitted("image/jpeg", data))).toEqual({
+      width: 667,
+      height: 2000,
+    });
+  });
+
+  it("passes an oversized GIF through", () => {
+    // GIF89a header, 2000x2001.
+    const gif = Buffer.from([0x47, 0x49, 0x46, 0x38, 0x39, 0x61, 0xd0, 0x07, 0xd1, 0x07]);
+    expect(fitClaudeImage("image/gif", gif)).toBe(gif);
+  });
+});

apps/server/src/provider/Layers/claudeImageSize.ts

@@ -0,0 +1,108 @@
+import { readImageDimensions } from "@t3tools/shared/imageDimensions";
+import jpeg from "jpeg-js";
+import { PNG } from "pngjs";
+
+/**
+ * Anthropic rejects every image over 2000px on a side once a request holds
+ * more than 20 images. Claude Code scales the images in a turn's opening
+ * message, but stores ones sent mid-turn as they are, so one oversized paste
+ * breaks every later image read in the session.
+ */
+const CLAUDE_MAX_IMAGE_EDGE = 2000;
+
+/**
+ * A highly compressible PNG fits the attachment byte cap at any size, and
+ * decoding expands it to 4 bytes per pixel, so past this bound a PNG or JPEG
+ * is not decoded. 50MP still covers an 8000x6000 photo.
+ */
+const MAX_DECODED_PIXELS = 50_000_000;
+
+/**
+ * Scales a PNG or JPEG down to fit CLAUDE_MAX_IMAGE_EDGE. Images already
+ * within the limit come back untouched, so only oversized ones pay for a
+ * decode. Returns null for a PNG or JPEG too large to decode safely, since
+ * sending it unscaled would break the thread.
+ *
+ * ponytail: decodes on the event loop, about 150ms for a phone screenshot,
+ * 400ms for a Retina screenshot, and 0.7s for a 12MP photo. Only steered
+ * images pay it. Move it to a worker if that stall shows up.
+ */
+export function fitClaudeImage(mimeType: string, bytes: Uint8Array): Uint8Array | null {
+  const dimensions = readImageDimensions(bytes);
+  if (!dimensions || Math.max(dimensions.width, dimensions.height) <= CLAUDE_MAX_IMAGE_EDGE) {
+    return bytes;
+  }
+  const decodable = dimensions.width * dimensions.height <= MAX_DECODED_PIXELS;
+  if (mimeType === "image/png") {
+    if (!decodable) return null;
+    const image = PNG.sync.read(Buffer.from(bytes));
+    const { width, height, data } = downscale(image);
+    const scaled = new PNG({ width, height });
+    scaled.data = Buffer.from(data.buffer);
+    // Keep the source's channels; always writing RGBA can make a grayscale
+    // or palette PNG several times larger than the original.
+    const colorType = image.alpha ? (image.color ? 6 : 4) : image.color ? 2 : 0;
+    return PNG.sync.write(scaled, { colorType });
+  }
+  if (mimeType === "image/jpeg") {
+    const image = decodable ? decodeJpeg(bytes) : null;
+    if (!image) return null;
+    // The spread keeps the decoder's `exifBuffer`, so the EXIF orientation of
+    // a phone photo survives the re-encode.
+    return jpeg.encode({ ...image, ...downscale(image) }, 90).data;
+  }
+  // ponytail: GIF and WebP pass through; no pure-JS codec for them here. Add
+  // one if oversized GIF or WebP attachments show up in practice.
+  return bytes;
+}
+
+/** Decodes a JPEG, or returns null when it would exceed jpeg-js's 512MB budget. */
+function decodeJpeg(bytes: Uint8Array) {
+  try {
+    return jpeg.decode(bytes, { useTArray: true });
+  } catch (error) {
+    // 4:4:4 JPEGs need about 12 bytes per pixel, so some pass MAX_DECODED_PIXELS
+    // and still hit the decoder's memory cap.
+    if (error instanceof Error && error.message.startsWith("maxMemoryUsageInMB")) return null;
+    throw error;
+  }
+}
+
+/** Box-filter downscale of RGBA pixels to fit CLAUDE_MAX_IMAGE_EDGE. */
+function downscale(image: { width: number; height: number; data: Uint8Array }) {
+  const scale = CLAUDE_MAX_IMAGE_EDGE / Math.max(image.width, image.height);
+  const width = Math.max(1, Math.round(image.width * scale));
+  const height = Math.max(1, Math.round(image.height * scale));
+  const data = new Uint8Array(width * height * 4);
+  for (let y = 0; y < height; y++) {
+    const top = Math.floor((y * image.height) / height);
+    const bottom = Math.max(top + 1, Math.floor(((y + 1) * image.height) / height));
+    for (let x = 0; x < width; x++) {
+      const left = Math.floor((x * image.width) / width);
+      const right = Math.max(left + 1, Math.floor(((x + 1) * image.width) / width));
+      let red = 0;
+      let green = 0;
+      let blue = 0;
+      let alpha = 0;
+      for (let sourceY = top; sourceY < bottom; sourceY++) {
+        for (let sourceX = left; sourceX < right; sourceX++) {
+          const offset = (sourceY * image.width + sourceX) * 4;
+          // Weighting color by alpha keeps transparent pixels from darkening edges.
+          const pixelAlpha = image.data[offset + 3] ?? 0;
+          red += (image.data[offset] ?? 0) * pixelAlpha;
+          green += (image.data[offset + 1] ?? 0) * pixelAlpha;
+          blue += (image.data[offset + 2] ?? 0) * pixelAlpha;
+          alpha += pixelAlpha;
+        }
+      }
+      const target = (y * width + x) * 4;
+      if (alpha > 0) {
+        data[target] = Math.round(red / alpha);
+        data[target + 1] = Math.round(green / alpha);
+        data[target + 2] = Math.round(blue / alpha);
+      }
+      data[target + 3] = Math.round(alpha / ((bottom - top) * (right - left)));
+    }
+  }
+  return { width, height, data };
+}