diff --git a/packages/core/src/plugin/provider/openai.ts b/packages/core/src/plugin/provider/openai.ts index 255cbb2cfcc4..5c7bdc38aff0 100644 --- a/packages/core/src/plugin/provider/openai.ts +++ b/packages/core/src/plugin/provider/openai.ts @@ -1,5 +1,6 @@ import type { IntegrationOAuthMethodRegistration } from "@opencode/plugin/effect/integration" import { define } from "@opencode/plugin/effect/plugin" +import type { SessionRequest } from "@opencode/plugin/effect/session" import { Deferred, Effect, Option, Schema, Semaphore, Stream } from "effect" import type { Server } from "node:http" import { App } from "../../app.js" @@ -307,6 +308,13 @@ export const OpenAIPlugin = define({ }), { providerID: Provider.ID.openai }, ) + // The ChatGPT backend rejects a requested output limit, and OpenAI counts one against rate limits. + const omitOutputLimit = (evt: SessionRequest) => + Effect.sync(() => { + delete evt.options.maxTokens + }) + for (const name of ["context", "compaction"] as const) + yield* ctx.session.hook(name, omitOutputLimit, { providerID: Provider.ID.openai }) const refresh = () => loading.withPermit(load().pipe(Effect.andThen(ctx.provider.reload()))) yield* bus.subscribe(Credential.Event.Switched).pipe( Stream.filter((event) => event.data.integrationID === Integration.ID.make("openai")), diff --git a/packages/core/src/session/compaction.ts b/packages/core/src/session/compaction.ts index f695a9953758..5de1ef8dc1b4 100644 --- a/packages/core/src/session/compaction.ts +++ b/packages/core/src/session/compaction.ts @@ -90,6 +90,8 @@ type Streamed = { const NOTHING_TO_COMPACT: Failure = { error: { type: "compaction.unavailable", message: "Nothing to compact yet" } } /** After each "too long" rejection, the next attempt aims at this share of the first rejected request's size. */ const SHRINK_STEPS = [0.7, 0.5, 0.35] +// The least of the window kept free for the last reply before compaction and for the summary itself. +const RESERVE_MIN = 16_000 /** A common window size, assumed for the compaction request when the model's window is unknown. */ const UNKNOWN_WINDOW = 200_000 const TOOL_OUTPUT_MAX_CHARS = 1_250 @@ -265,7 +267,7 @@ export const layer = Layer.effect( const prompt = buildPrompt(previous !== undefined, previous?.summary.includes(LEGACY_HEADING) ?? false) const headings = SUMMARY_TEMPLATE.split("\n").filter((line) => line.startsWith("##")) const filled = (text: string) => text.split("\n").some((line) => headings.includes(line.trim())) - const prepared = yield* prepare(context, split.older) + const prepared = yield* prepare(context, split.older, budget) // Hooks saw the request without the summary prompt, so it is appended here. A reply that ignores the // template gets one reminder before it counts as a failure. @@ -313,7 +315,7 @@ export const layer = Layer.effect( if (!context.messages.some(messageToText)) return yield* Effect.fail(NOTHING_TO_COMPACT) const unsupported = (message: string) => Effect.fail({ error: { type: "provider.unsupported-operation", message } }) - const prepared = yield* prepare(context, context.messages, "session") + const prepared = yield* prepare(context, context.messages, budget, "session") // History is selected before request hooks, so a hook that reroutes the request cannot be honored here. const provenance = SessionProviderContext.provenance(context.model) @@ -550,10 +552,18 @@ export const layer = Layer.effect( ) } - /** The conversation as the runner would send it, after request hooks. */ + /** + * The conversation as the runner would send it, after request hooks. + * + * The output limit leaves room for `budget`, the most `deliver` sends. The request prepared here can be larger + * when the conversation overshot the threshold, and is only shrunk to fit after hooks have seen it, so sizing the + * output to it would leave next to no room. A prompt the estimate undersells is rejected and shrunk like any + * other. + */ const prepare = ( context: SessionContext.Loaded, messages: ReadonlyArray, + budget: number, webSocket?: "session", ) => { const base = transcript(context, messages) @@ -565,6 +575,7 @@ export const layer = Layer.effect( system: base.system, messages: base.messages, webSocket, + inputTokens: { measured: budget, estimated: 0 }, }) } @@ -843,6 +854,12 @@ export const recentUserMessages = ( } export const estimateContext = (context: SessionContext.Loaded) => { + const prompt = estimatePrompt(context) + return prompt.measured + prompt.estimated +} + +/** The prompt size: `measured` is what the provider reported at the latest response, `estimated` is the text since. */ +export const estimatePrompt = (context: SessionContext.Loaded) => { const anchorIndex = context.messages.findLastIndex((message) => hasMeasuredPrompt(message, context.model.ref)) const anchor = context.messages[anchorIndex] const base = transcript(context, context.messages.slice(Math.max(0, anchorIndex))) @@ -856,20 +873,30 @@ export const estimateContext = (context: SessionContext.Loaded) => { const unmeasured = sent.filter((message) => message.role !== "assistant" || message.id !== anchor?.id) if (anchor?.type !== "assistant" || !anchor.tokens) - return estimateRequest({ system: base.system, tools: context.tools.definitions, messages: unmeasured }) + return { + measured: 0, + estimated: estimateRequest({ system: base.system, tools: context.tools.definitions, messages: unmeasured }), + } const tokens = anchor.tokens - const measured = tokens.input + tokens.cache.read + tokens.cache.write + tokens.output + tokens.reasoning - return measured + unmeasured.reduce((sum, message) => sum + estimateMessage(message), 0) + return { + measured: tokens.input + tokens.cache.read + tokens.cache.write + tokens.output + tokens.reasoning, + estimated: unmeasured.reduce((sum, message) => sum + estimateMessage(message), 0), + } } -/** The largest request the model takes while leaving room for its reply. */ +/** + * The largest request the model takes while leaving room for its reply: 10% of the window, or `RESERVE_MIN` when that + * is more. The summary request is capped at the same size, so its output limit is whatever the reserve leaves. A window + * too small to give up `RESERVE_MIN` keeps 10%. + */ const calculateCeiling = (limit: SessionContext.Loaded["model"]["limit"], buffer: number | undefined) => { // Unknown limits are reported as 0. An unknown input limit falls back to the context window; with no window at // all, only a provider rejection can limit the request. const window = limit.input || limit.context if (window <= 0) return Number.POSITIVE_INFINITY - return buffer === undefined ? Math.floor(window * 0.9) : window - buffer + if (buffer !== undefined) return window - buffer + return window - Math.max(Math.floor(window * 0.1), window >= 2 * RESERVE_MIN ? RESERVE_MIN : 0) } /** diff --git a/packages/core/src/session/model-request.ts b/packages/core/src/session/model-request.ts index d2277ce8ac7c..21ad82c03348 100644 --- a/packages/core/src/session/model-request.ts +++ b/packages/core/src/session/model-request.ts @@ -44,6 +44,14 @@ const IMAGE_BYTES_TARGET = 15 * 1024 * 1024 // 15 MiB const IMAGE_REMOVED = "[This image was removed to reduce the request size and is no longer visible. Do not make claims about its contents from memory. If needed, retrieve it again with an available tool or ask the user to attach it again.]" const GENERATION_KEYS = new Set(Object.keys(GenerationOptions.fields)) +// Used when the catalog has no output limit for the model. +const OUTPUT_TOKEN_FALLBACK = 32_000 +// A summary never needs more, and a request asking for more cannot be shrunk to fit a window the catalog overstates. +const SUMMARY_OUTPUT_MAX = 32_000 +// Prompt text is estimated at about 4 characters per token, which can run low on dense text such as code. +const ESTIMATE_ERROR = 0.15 +// Never ask for less; only reachable with automatic compaction off, since it keeps the window from filling this far. +const OUTPUT_TOKEN_MIN = 1_024 /** Tool errors, plus the user declining a permission or dismissing a question. */ export type ExecuteError = Tool.Error | Permission.DeclinedError | QuestionTool.CancelledError @@ -69,6 +77,21 @@ export interface Input { readonly toolChoice?: LLM.RequestInput["toolChoice"] /** Only the durable runner may use a stateful WebSocket. */ readonly webSocket?: "session" + /** Prompt size, measured by the provider or estimated. The default output limit leaves room for it. */ + readonly inputTokens?: { readonly measured: number; readonly estimated: number } +} + +/** The default output limit: the catalog limit, fitted to the room the prompt leaves in the context window. */ +const outputLimit = ( + limit: Model.Info["limit"], + kind: "primary" | "compaction", + inputTokens?: Input["inputTokens"], +) => { + const model = limit.output > 0 ? limit.output : OUTPUT_TOKEN_FALLBACK + const requested = kind === "compaction" ? Math.min(model, SUMMARY_OUTPUT_MAX) : model + if (inputTokens === undefined || limit.context <= 0) return requested + const room = limit.context - inputTokens.measured - Math.ceil(inputTokens.estimated * (1 + ESTIMATE_ERROR)) + return Math.min(requested, Math.max(OUTPUT_TOKEN_MIN, room)) } export const baseTranscript = (input: { @@ -218,8 +241,19 @@ export const layer = Layer.effect( const given = new Map( tools.definitions.map((t) => [{ description: t.description, input: { ...t.inputSchema } }, t] as const), ) + // Hooks see the default output limit and may change or remove it. Titles and generate keep the provider default, + // because their reasoning is hard to budget. const shaped = yield* shape( - { sessionID: session.id, model: model.ref, system: input.system, messages: input.messages, options: {} }, + { + sessionID: session.id, + model: model.ref, + system: input.system, + messages: input.messages, + options: + kind === "primary" || kind === "compaction" + ? { maxTokens: outputLimit(model.limit, kind, input.inputTokens) } + : {}, + }, Object.fromEntries(Array.from(given, ([d, t]) => [t.name, d])), ) // Match by identity first, then by key. Entries matching neither were invented by a diff --git a/packages/core/src/session/runner/llm.ts b/packages/core/src/session/runner/llm.ts index 2b654657413d..3ffe4ed6e900 100644 --- a/packages/core/src/session/runner/llm.ts +++ b/packages/core/src/session/runner/llm.ts @@ -240,6 +240,7 @@ const layer = Layer.effect( // Keep tool definitions on the final Step to preserve the provider's cached prefix. toolChoice: stepLimitReached ? "none" : undefined, webSocket: "session", + inputTokens: SessionCompaction.estimatePrompt(loaded), }) const outcome = yield* steps.attempt({ isLocationClosed: lifecycle.isClosed, diff --git a/packages/core/test/plugin/provider-openai.test.ts b/packages/core/test/plugin/provider-openai.test.ts index 2f6725f3cf2f..7ca86ee7d0ec 100644 --- a/packages/core/test/plugin/provider-openai.test.ts +++ b/packages/core/test/plugin/provider-openai.test.ts @@ -216,6 +216,31 @@ describe("OpenAIPlugin", () => { }), ) + it.effect("omits the default output limit from OpenAI steps and compaction", () => + Effect.gen(function* () { + yield* addPlugin() + const hooks = yield* PluginHooks.Service + const maxTokens = (providerID: Provider.ID) => + Effect.gen(function* () { + const draft = { + sessionID: Session.ID.make("ses_test"), + model: Model.Ref.make({ providerID, id: Model.ID.make("gpt-5.5") }), + system: [], + messages: [], + options: { maxTokens: 128_000 }, + } + const events = [ + yield* hooks.trigger("session", "context", { ...draft, agent: Agent.ID.make("build"), tools: {} }), + yield* hooks.trigger("session", "compaction", { ...draft, agent: Agent.ID.make("build"), tools: {} }), + ] + return events.map((event) => event.options.maxTokens) + }) + + expect(yield* maxTokens(Provider.ID.openai)).toEqual([undefined, undefined]) + expect(yield* maxTokens(Provider.ID.azure)).toEqual([128_000, 128_000]) + }), + ) + it.effect("selects WebSocket only from explicit policy", () => Effect.gen(function* () { const credentials = yield* Credential.Service diff --git a/packages/core/test/session-compaction.test.ts b/packages/core/test/session-compaction.test.ts index 04c51e75e34c..0f554f453f4a 100644 --- a/packages/core/test/session-compaction.test.ts +++ b/packages/core/test/session-compaction.test.ts @@ -1,5 +1,5 @@ import { expect, test } from "bun:test" -import { LLMClient, LLMEvent, LanguageModel, ToolDefinition, type LLMRequest } from "@opencode/ai" +import { GenerationOptions, LLMClient, LLMEvent, LanguageModel, ToolDefinition, type LLMRequest } from "@opencode/ai" import { OpenAIChat } from "@opencode/ai/protocols" import { Database } from "@opencode/core/database/database" import { AppNodeBuilder } from "@opencode/core/effect/app-node-builder" @@ -212,14 +212,15 @@ it.effect("auto compaction estimates current content against the buffered prompt expect(yield* due(native(244_800))).toBe(true) expect(yield* due(native(1_000_000, { context: 0, input: undefined, output: 0 }))).toBe(false) + // The summary's 16k output limit is more than 10% of a 100k window, so it sets the ceiling. const contextLimited = { context: 100_000, output: 10_000 } - expect(yield* due(input(89_999, contextLimited))).toBe(false) - expect(yield* due(input(90_000, contextLimited))).toBe(true) + expect(yield* due(input(83_999, contextLimited))).toBe(false) + expect(yield* due(input(84_000, contextLimited))).toBe(true) // The reply limit does not lower the ceiling. const outputLimited = { context: 100_000, output: 30_000 } - expect(yield* due(input(89_999, outputLimited))).toBe(false) - expect(yield* due(input(90_000, outputLimited))).toBe(true) + expect(yield* due(input(83_999, outputLimited))).toBe(false) + expect(yield* due(input(84_000, outputLimited))).toBe(true) const assistant = input(89_000, contextLimited).messages[0] const tool = SessionMessage.AssistantTool.make({ @@ -452,7 +453,7 @@ it.effect("manual compaction summarizes short context instead of no-op", () => "x-opencode-session": sessionID, "x-opencode-client": "opencode", }) - expect(requests[0]?.generation).toBeUndefined() + expect(requests[0]?.generation).toEqual(GenerationOptions.make({ maxTokens: 20_000 })) expect(JSON.stringify(requests[0]?.messages)).toContain("Manual compaction should include this short conversation.") expect(JSON.stringify(requests[0]?.messages)).toContain("Use Effect services and generators.") expect(JSON.stringify(requests[0]?.messages)).toContain("User shell pwd completed: /project") diff --git a/packages/core/test/session-runner.test.ts b/packages/core/test/session-runner.test.ts index 8f445872f9db..4f1555d17a28 100644 --- a/packages/core/test/session-runner.test.ts +++ b/packages/core/test/session-runner.test.ts @@ -126,6 +126,8 @@ const fullOutputModel = testModel("full-output", { context: 262_144, output: 262 const unknownContextModel = testModel("unknown-context", { context: 0, output: 32_000 }) const undersizedContextModel = testModel("undersized-context", { context: 1, output: 1_000 }) const recoveryModel = testModel("recovery", { context: 200_000, output: 1_000 }) +const fittedOutputModel = testModel("fitted-output", { context: 100_000, output: 64_000 }) +const smallWindowModel = testModel("small-window", { context: 64_000, output: 16_000 }) test("calculates step cost using the matching context tier", () => { expect( @@ -3205,7 +3207,7 @@ describe("SessionRunnerLLM", () => { expect(yield* Effect.exit(s.resume)).toMatchObject({ _tag: "Failure" }) expect(s.requests).toHaveLength(1) - expect(s.requests[0]?.generation).toBeUndefined() + expect(s.requests[0]?.generation?.maxTokens).toBe(50) expect(yield* s.context).toContainEqual( expect.objectContaining({ type: "compaction", @@ -3410,6 +3412,55 @@ describe("SessionRunnerLLM", () => { ]) }) + scenario("fits the output limit to the prompt size", function* (s) { + s.currentModel = fittedOutputModel + yield* s.llm.push(TestLLM.textWithUsage("Earlier answer", "text-fitted-first", 50_000)) + yield* s.runPrompt("Earlier question") + yield* s.llm.push(TestLLM.text("Continued", "text-fitted-final")) + yield* s.runPrompt("Continue") + + expect(s.requests[0]?.generation?.maxTokens).toBe(64_000) + expect(s.requests[1]?.generation?.maxTokens).toBeLessThan(100_000 - 50_000) + expect(s.requests[1]?.generation?.maxTokens).toBeGreaterThan(100_000 - 50_000 - 200) + }) + + scenario("gives the summary its full output limit when the conversation overshot the threshold", function* (s) { + // The conversation overshot the threshold, so the prepared summary request exceeds the budget and `deliver` + // shrinks it before sending. The output limit must follow the budget, not the oversized prepared request. + yield* s.llm.push(TestLLM.textWithUsage("Earlier answer", "text-budget-first", 185_000)) + yield* s.runPrompt("Earlier question") + s.requests.length = 0 + yield* s.llm.push( + TestLLM.text("## Objective\n- Preserve the task", "text-budget-summary"), + TestLLM.text("Continued", "text-budget-final"), + ) + yield* s.runPrompt("Recent request ".repeat(400)) + + expect(s.requests).toHaveLength(2) + expect(userTexts(s.requests[0]).at(-1)).toContain("## Objective") + // The summary may use the whole 20k reserve of a 200k window. + expect(s.requests[0]?.generation?.maxTokens).toBe(20_000) + expect(s.requests[1]?.generation?.maxTokens).toBe(32_000) + }) + + scenario("keeps the summary its room on a small window", function* (s) { + // 90% of 64k would leave 6.4k for the summary, so the 16k reserve sets the ceiling at 48k instead. + s.currentModel = smallWindowModel + yield* s.llm.push(TestLLM.textWithUsage("Earlier answer", "text-small-first", 59_000)) + yield* s.runPrompt("Earlier question") + s.requests.length = 0 + yield* s.llm.push( + TestLLM.text("## Objective\n- Preserve the task", "text-small-summary"), + TestLLM.text("Continued", "text-small-final"), + ) + yield* s.runPrompt("Recent request ".repeat(400)) + + expect(s.requests).toHaveLength(2) + expect(userTexts(s.requests[0]).at(-1)).toContain("## Objective") + expect(s.requests[0]?.generation?.maxTokens).toBe(16_000) + expect(s.requests[1]?.generation?.maxTokens).toBe(16_000) + }) + scenario("publishes the original overflow when recovery summarization fails", function* (s) { yield* setupOverflowRecovery(s) yield* s.llm.push(