diff --git a/packages/core/src/session/context.ts b/packages/core/src/session/context.ts index c23f6265272a..42ae72d053af 100644 --- a/packages/core/src/session/context.ts +++ b/packages/core/src/session/context.ts @@ -95,21 +95,27 @@ const layer = Layer.effect( const agent = yield* agents.get(Agent.ID.make("title")) if (!agent) return const primary = yield* resolveModel(session).pipe(Effect.orElseSucceed(() => undefined)) - const info = yield* Effect.gen(function* () { - if (agent.model) return yield* model.get(agent.model.providerID, agent.model.id) + const candidate = yield* Effect.gen(function* () { + if (agent.model) { + const info = yield* model.get(agent.model.providerID, agent.model.id) + return info && { info, variant: agent.model.variant ?? minimalVariant(info) } + } if (!primary) return - return yield* model.small(primary.ref.providerID) + const small = yield* model.small(primary.ref.providerID) + if (small) return { info: small, variant: minimalVariant(small) } + // Without a lower variant, the primary model at its selected variant is already the fallback. + const info = yield* model.get(primary.ref.providerID, primary.ref.id) + const variant = info && lowerVariant(info, primary.ref.variant) + return variant && { info, variant } }) - const variant = - agent.model?.variant ?? MINIMAL_REASONING_VARIANTS.find((id) => info?.variants.some((item) => item.id === id)) const preferred = - info && + candidate && (yield* resolveModel({ ...session, model: Model.Ref.make({ - providerID: info.providerID, - id: info.id, - ...(variant ? { variant } : {}), + providerID: candidate.info.providerID, + id: candidate.info.id, + ...(candidate.variant ? { variant: candidate.variant } : {}), }), }).pipe(Effect.orElseSucceed(() => undefined))) const selected = preferred ?? primary @@ -184,6 +190,18 @@ const layer = Layer.effect( /** Variant IDs that minimize reasoning output, in preference order. */ const MINIMAL_REASONING_VARIANTS = ["none", "minimal", "low"].map((id) => Model.VariantID.make(id)) +const minimalVariant = (info: Model.Info) => + MINIMAL_REASONING_VARIANTS.find((id) => info.variants.some((item) => item.id === id)) + +/** + * `none` and `minimal` always reduce reasoning. `low` can enable reasoning on a model whose default + * does not reason, so it is only used when it ranks below the variant the session selected. + */ +const lowerVariant = (info: Model.Info, selected: Model.VariantID | undefined) => { + const rank = (id: Model.VariantID | undefined) => info.variants.findIndex((item) => item.id === id) + return MINIMAL_REASONING_VARIANTS.find((id) => rank(id) !== -1 && (id !== "low" || rank(id) < rank(selected))) +} + export const node = makeLocationNode({ service: Service, layer, diff --git a/packages/core/test/session-title.test.ts b/packages/core/test/session-title.test.ts index 82a1f4ee73a6..a508f97b4dce 100644 --- a/packages/core/test/session-title.test.ts +++ b/packages/core/test/session-title.test.ts @@ -39,6 +39,7 @@ import { testEffect } from "./lib/effect" let requests: LLMRequest[] = [] let selectedSmall: Model.Info | undefined +let selectedPrimary: Model.Info | undefined let selections: Array = [] const model = LanguageModel.make({ id: "title-model", @@ -102,6 +103,7 @@ const models = Layer.mock(SessionRunnerModel.Service)({ }) const smallModels = Layer.mock(Model.Service, { small: () => Effect.succeed(selectedSmall), + get: (_providerID, modelID) => Effect.succeed(modelID === selectedPrimary?.id ? selectedPrimary : undefined), }) const it = testEffect( AppNodeBuilder.build( @@ -183,6 +185,7 @@ const lowSmall = Model.Info.make({ beforeEach(() => { requests = [] selectedSmall = undefined + selectedPrimary = undefined selections = [] titleStream = successfulTitle }) @@ -343,6 +346,95 @@ it.effect("falls back to the primary model when the small model fails", () => }), ) +const primaryRef = (variant?: string) => + Model.Ref.make({ + providerID: Provider.ID.make("test"), + id: Model.ID.make("title-model"), + ...(variant ? { variant: Model.VariantID.make(variant) } : {}), + }) +const primaryInfo = (variants: string[]) => + Model.Info.make({ + ...Model.Info.default(Provider.ID.make("test"), Model.ID.make("title-model")), + variants: variants.map((id) => ({ id: Model.VariantID.make(id) })), + }) + +const titleAttempts = (sessionID: Session.ID, text: string, variant?: string) => + Effect.gen(function* () { + yield* enableTitleAgent + yield* insertSession(sessionID, undefined, undefined, primaryRef(variant)) + yield* prompt(sessionID, text) + const attempted: Model.Ref[] = [] + const hooks = yield* PluginHooks.Service + yield* hooks.register("session", "model.request", (event) => + Effect.sync(() => { + attempted.push(event.model) + }), + ) + const title = yield* SessionTitle.Service + yield* title.generate(sessionID) + const store = yield* SessionStore.Service + return { attempted: attempted.map((model) => String(model.variant)), title: (yield* store.get(sessionID))?.title } + }) + +it.effect("uses the primary model at its lowest variant without a small model", () => + Effect.gen(function* () { + selectedPrimary = primaryInfo(["none", "low", "high"]) + const result = yield* titleAttempts( + Session.ID.make("ses_title_primary_lowest"), + "Title with the lowest variant", + "high", + ) + expect(result.attempted).toEqual(["none"]) + expect(result.title).toBe("Generated Title") + }), +) + +it.effect("falls back to the selected primary variant when the lowest variant fails", () => + Effect.gen(function* () { + titleStream = () => + requests.length === 1 ? Stream.make(LLMEvent.providerError({ message: "Variant rejected" })) : successfulTitle() + selectedPrimary = primaryInfo(["low", "high"]) + const result = yield* titleAttempts( + Session.ID.make("ses_title_primary_fallback"), + "Fall back to the selection", + "high", + ) + expect(result.attempted).toEqual(["low", "high"]) + expect(result.title).toBe("Generated Title") + }), +) + +it.effect("uses the selected primary variant when no lower variant exists", () => + Effect.gen(function* () { + selectedPrimary = primaryInfo(["high", "max"]) + const result = yield* titleAttempts(Session.ID.make("ses_title_primary_selected"), "No lower variant here", "high") + expect(result.attempted).toEqual(["high"]) + expect(result.title).toBe("Generated Title") + }), +) + +it.effect("keeps the primary default variant instead of a low variant that may enable reasoning", () => + Effect.gen(function* () { + selectedPrimary = primaryInfo(["low", "high"]) + const result = yield* titleAttempts( + Session.ID.make("ses_title_primary_default"), + "Default may not reason", + undefined, + ) + expect(result.attempted).toEqual(["default"]) + expect(result.title).toBe("Generated Title") + }), +) + +it.effect("uses the primary none variant from its default variant", () => + Effect.gen(function* () { + selectedPrimary = primaryInfo(["none", "low", "high"]) + const result = yield* titleAttempts(Session.ID.make("ses_title_primary_none"), "Disable reasoning", undefined) + expect(result.attempted).toEqual(["none"]) + expect(result.title).toBe("Generated Title") + }), +) + it.effect("generates from the first user message after later messages exist", () => Effect.gen(function* () { yield* enableTitleAgent