Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
ef00f33
test(server): reproduce #17110 Claude effort above maxEffortLevel cap
juliusmarminge Oct 22, 2026
54cd52d
fix(server): Claude effort picker respects Claude Code's maxEffortLevel
juliusmarminge Oct 22, 2026
3e77a51
fix(server): mark Claude efforts above maxEffortLevel as capped inste…
juliusmarminge Oct 22, 2026
ed3cbf3
fix(server): leave custom Claude model effort labels uncapped
juliusmarminge Oct 22, 2026
f347cf5
fix(server): cap Claude effort labels at the model's highest level un…
juliusmarminge Oct 22, 2026
82ca801
Merge remote-tracking branch 'origin/main' into fix/issue-17110
juliusmarminge Oct 22, 2026
ab61fad
fix(server): per-model Claude maxEffortLevel replaces the top-level cap
juliusmarminge Oct 24, 2026
d20e9fa
Merge main into fix/issue-17110
juliusmarminge Oct 24, 2026
5441664
test: add deterministic Claude effort capture fixtures
juliusmarminge Oct 24, 2026
b0b9011
fix(server): Claude effort caps follow the CLI's per-source and alias…
juliusmarminge Oct 24, 2026
35f4fc1
Merge remote-tracking branch 'origin/main' into HEAD
juliusmarminge Oct 24, 2026
4dd9581
fix(server): Claude effort-cap model keys match like Claude Code
juliusmarminge Oct 24, 2026
b154082
fix(ci): format effort picker scenario fixture
juliusmarminge Oct 24, 2026
b28bfbb
fix(fixtures): correct per-model exemption expected label
juliusmarminge Oct 24, 2026
ed6a833
Merge remote-tracking branch 'origin/main' into fix/issue-17110
juliusmarminge Nov 11, 2026
335d4af
refactor(server): trim Claude effort-cap tests and proof fixtures
juliusmarminge Nov 11, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
240 changes: 240 additions & 0 deletions apps/server/src/provider/ClaudeModelCatalog.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ import { ProviderDriverKind, ProviderInstanceId } from "@t3tools/contracts";

import { resolveProviderCatalog, type ModelManifestData } from "./ModelManifest.ts";
import {
capClaudeCatalogEffort,
type ClaudeModelCatalog,
formatClaudeVersionUpgradeMessage,
getClaudeCatalogModelCapabilities,
normalizeClaudeCatalogEffort,
Expand Down Expand Up @@ -230,4 +232,242 @@ describe("Claude model catalog", () => {
["claude-synthetic-next", "claude-custom-tuned"],
);
});

it("marks effort options that would run above the cap without changing ids or defaults", () => {
const effortModel = (slug: string) => ({
slug,
name: slug,
isCustom: false,
capabilities: {
optionDescriptors: [
{
id: "effort",
label: "Reasoning",
type: "select" as const,
options: [
{ id: "medium", label: "Medium" },
{ id: "high", label: "High", isDefault: true },
{ id: "xhigh", label: "Extra High" },
{ id: "max", label: "Max" },
{ id: "ultracode", label: "Ultracode" },
{ id: "ultrathink", label: "Ultrathink" },
],
},
],
},
});
const models = [effortModel("claude-plain"), effortModel("claude-remapped")];
const catalog: ClaudeModelCatalog = {
models: [
{
model: models[0]!,
runtime: { effortMap: { ultracode: "xhigh", ultrathink: null } },
compatibility: {},
},
{
model: models[1]!,
runtime: { effortMap: { max: "high", xhigh: "max", ultrathink: null } },
compatibility: {},
},
],
};
const options = (cap: "high" | "xhigh") =>
capClaudeCatalogEffort(catalog, models, [{ maxEffortLevel: cap }]).map((model) => {
const effort = model.capabilities?.optionDescriptors?.[0];
return effort?.type === "select" ? effort.options : undefined;
});

assert.deepStrictEqual(options("high"), [
[
{ id: "medium", label: "Medium" },
{ id: "high", label: "High", isDefault: true },
{ id: "xhigh", label: "Extra High (capped to High)" },
{ id: "max", label: "Max (capped to High)" },
{ id: "ultracode", label: "Ultracode (capped to High)" },
{ id: "ultrathink", label: "Ultrathink" },
],
[
{ id: "medium", label: "Medium" },
{ id: "high", label: "High", isDefault: true },
{ id: "xhigh", label: "Extra High (capped to High)" },
{ id: "max", label: "Max" },
{ id: "ultracode", label: "Ultracode" },
{ id: "ultrathink", label: "Ultrathink" },
],
]);
// An effortMap that sends Extra High as max stays unmarked under an xhigh cap: the CLI runs
// it at xhigh, which is what the label says.
assert.deepStrictEqual(
options("xhigh").map((model) => model?.map((option) => option.label)),
[
["Medium", "High", "Extra High", "Max (capped to Extra High)", "Ultracode", "Ultrathink"],
["Medium", "High", "Extra High", "Max", "Ultracode", "Ultrathink"],
],
);

// A per-model cap applies without a top-level one and leaves other models uncapped.
assert.deepStrictEqual(
capClaudeCatalogEffort(catalog, models, [
{ modelMaxEffortLevels: new Map([["claude-plain", "high"]]) },
]).map((model) => {
const effort = model.capabilities?.optionDescriptors?.[0];
return effort?.type === "select" ? effort.options.map((option) => option.label) : [];
}),
[
[
"Medium",
"High",
"Extra High (capped to High)",
"Max (capped to High)",
"Ultracode (capped to High)",
"Ultrathink",
],
["Medium", "High", "Extra High", "Max", "Ultracode", "Ultrathink"],
],
);

// Without an Extra High option (Opus 4.6), the CLI clamps an xhigh cap down to High.
const noXhigh = {
slug: "claude-no-xhigh",
name: "claude-no-xhigh",
isCustom: false,
capabilities: {
optionDescriptors: [
{
id: "effort",
label: "Reasoning",
type: "select" as const,
options: [
{ id: "low", label: "Low" },
{ id: "medium", label: "Medium" },
{ id: "high", label: "High", isDefault: true },
{ id: "max", label: "Max" },
],
},
],
},
};
const [capped] = capClaudeCatalogEffort(
{ models: [] },
[noXhigh],
[{ maxEffortLevel: "xhigh" }],
);
const cappedEffort = capped?.capabilities?.optionDescriptors?.[0];
assert.deepStrictEqual(
cappedEffort?.type === "select" ? cappedEffort.options.map((option) => option.label) : [],
["Low", "Medium", "High", "Max (capped to High)"],
);
});

it("leaves custom model effort labels alone under a cap", () => {
const custom = {
slug: "claude-custom-copy",
name: "Copy",
isCustom: true,
capabilities: {
optionDescriptors: [
{
id: "effort",
label: "Reasoning",
type: "select" as const,
options: [
{ id: "high", label: "High", isDefault: true },
{ id: "max", label: "Max (capped to High)" },
],
},
],
},
};

assert.deepStrictEqual(
capClaudeCatalogEffort({ models: [] }, [custom], [{ maxEffortLevel: "high" }]),
[custom],
);
});

it("resolves maxEffortLevel per settings source and per model key like Claude Code", () => {
const opus = {
slug: "claude-opus-5-5",
name: "Opus 5.5",
isCustom: false,
aliases: ["opus", "opus-5.5", "claude-opus-5.5"],
capabilities: {
optionDescriptors: [
{
id: "effort",
label: "Reasoning",
type: "select" as const,
options: [
{ id: "low", label: "Low" },
{ id: "medium", label: "Medium" },
{ id: "high", label: "High", isDefault: true },
{ id: "max", label: "Max" },
],
},
],
},
};
const catalog: ClaudeModelCatalog = {
models: [
{ model: opus, runtime: {}, compatibility: {} },
{
model: {
slug: "claude-opus-5",
name: "Opus 5",
isCustom: false,
capabilities: { optionDescriptors: [] },
},
runtime: {},
compatibility: {},
},
],
};
const maxLabel = (sources: Parameters<typeof capClaudeCatalogEffort>[2]) => {
const effort = capClaudeCatalogEffort(catalog, [opus], sources)[0]?.capabilities
?.optionDescriptors?.[0];
return effort?.type === "select" ? effort.options.at(-1)?.label : undefined;
};
const perModel = (entries: Record<string, "low" | "medium" | "high" | "max">) => ({
modelMaxEffortLevels: new Map(Object.entries(entries)),
});

// Claude Code matches family aliases and [1m] spellings case-insensitively.
assert.strictEqual(maxLabel([perModel({ Opus: "medium" })]), "Max (capped to Medium)");
assert.strictEqual(
maxLabel([perModel({ "claude-opus-5-5[1M]": "medium" })]),
"Max (capped to Medium)",
);
assert.strictEqual(maxLabel([perModel({ " OPUS[1m] ": "medium" })]), "Max (capped to Medium)");
// Other keys name the most specific model id they contain, so dated and provider-prefixed ids
// match while the catalog's dotted aliases and a less specific model id do not.
for (const key of [
"claude-opus-5-5-20260101",
"CLAUDE-OPUS-5-5-20260101[1m]",
"us.anthropic.claude-opus-5-5-v1:0",
"claude-opus-5-5-latest",
]) {
assert.strictEqual(maxLabel([perModel({ [key]: "medium" })]), "Max (capped to Medium)", key);
}
for (const key of ["claude-opus-5.5", "opus-5.5", "opus-5-5", "opus [1m]", "claude-opus-5"]) {
assert.strictEqual(maxLabel([perModel({ [key]: "medium" })]), "Max", key);
}
// Keys naming the same model keep the lowest cap, and a `max` key does not exempt it.
assert.strictEqual(
maxLabel([perModel({ opus: "max", "claude-opus-5-5": "low" })]),
"Max (capped to Low)",
);
// A per-model `max` only exempts the model from its own source's top-level cap.
assert.strictEqual(
maxLabel([{ maxEffortLevel: "medium", ...perModel({ opus: "max" }) }]),
"Max",
);
assert.strictEqual(
maxLabel([perModel({ opus: "max" }), { maxEffortLevel: "high" }]),
"Max (capped to High)",
);
assert.strictEqual(
maxLabel([{ maxEffortLevel: "medium" }, perModel({ opus: "high" })]),
"Max (capped to Medium)",
);
});
});
133 changes: 132 additions & 1 deletion apps/server/src/provider/ClaudeModelCatalog.ts
Original file line number Diff line number Diff line change
Expand Up @@ -322,14 +322,145 @@ export function formatClaudeVersionUpgradeMessage(
return `Claude Code ${versionLabel} is too old for ${unavailable.model.name}. Upgrade to v${unavailable.compatibility.minVersion} or newer to access it.`;
}

/** Claude Code's effort levels, lowest first. */
export const CLAUDE_EFFORT_LEVELS = ["low", "medium", "high", "xhigh", "max"] as const;
export type ClaudeEffortLevel = (typeof CLAUDE_EFFORT_LEVELS)[number];

function claudeEffortRank(level: string | undefined): number {
return CLAUDE_EFFORT_LEVELS.indexOf(level as ClaudeEffortLevel);
}

/** One settings source's effort caps, with `modelSettings` keys as the user wrote them. */
export interface ClaudeEffortCapSource {
readonly maxEffortLevel?: ClaudeEffortLevel;
readonly modelMaxEffortLevels?: ReadonlyMap<string, ClaudeEffortLevel>;
}

const CLAUDE_1M_SUFFIX = "[1m]";
const CLAUDE_CODE_FAMILY_ALIASES: ReadonlySet<string> = new Set([
"opus",
"sonnet",
"haiku",
"fable",
]);

/**
* Names the model a `modelSettings` key caps the way Claude Code does, which is narrower than the
* catalog's aliases: only a family alias (optionally `[1m]`) resolves as an alias, and any other key
* names the most specific model id it contains, so `claude-opus-5-5-20260101` and Bedrock ids match
* while `claude-opus-5.5` matches nothing.
*/
function resolveClaudeModelSettingsKey(
catalog: ClaudeModelCatalog,
key: string,
): string | undefined {
const lowered = key.trim().toLowerCase();
const unsuffixed = lowered.endsWith(CLAUDE_1M_SUFFIX)
? lowered.slice(0, -CLAUDE_1M_SUFFIX.length)
: lowered;
if (CLAUDE_CODE_FAMILY_ALIASES.has(unsuffixed)) {
return catalog.models.find(
(entry) => !entry.model.isCustom && entry.model.aliases?.includes(unsuffixed),
)?.model.slug;
}
let match: string | undefined;
for (const { model } of catalog.models) {
if (!model.isCustom && lowered.includes(model.slug) && model.slug.length > (match?.length ?? 0))
match = model.slug;
}
return match;
}

/** Keys `modelSettings` caps by canonical slug, keeping the lowest cap when several keys name one model. */
function resolveClaudeModelMaxEffortLevels(
catalog: ClaudeModelCatalog,
levels: ReadonlyMap<string, ClaudeEffortLevel> | undefined,
): ReadonlyMap<string, ClaudeEffortLevel> {
const resolved = new Map<string, ClaudeEffortLevel>();
for (const [key, level] of levels ?? []) {
const slug = resolveClaudeModelSettingsKey(catalog, key);
if (!slug) continue;
const current = resolved.get(slug);
if (!current || claudeEffortRank(level) < claudeEffortRank(current)) resolved.set(slug, level);
}
return resolved;
}

/**
* Claude Code clamps efforts above `maxEffortLevel` to the model's highest level under it, so
* such options are relabelled, not removed, keeping stored selections valid. Per-model caps
* replace their own source's top-level cap and `max` means none; across sources the lowest wins.
*/
export function capClaudeCatalogEffort(
catalog: ClaudeModelCatalog,
models: ReadonlyArray<ServerProviderModel>,
sources: ReadonlyArray<ClaudeEffortCapSource>,
): ReadonlyArray<ServerProviderModel> {
if (sources.length === 0) return models;
const resolvedSources = sources.map((source) => ({
maxEffortLevel: source.maxEffortLevel,
modelMaxEffortLevels: resolveClaudeModelMaxEffortLevels(catalog, source.modelMaxEffortLevels),
}));
return models.map((model) => {
// Custom model labels are user-authored, so they are left alone.
if (model.isCustom || !model.capabilities?.optionDescriptors) return model;
const modelMaxEffortLevel = resolvedSources.reduce<ClaudeEffortLevel | undefined>(
(lowest, source) => {
const level = source.modelMaxEffortLevels.get(model.slug) ?? source.maxEffortLevel;
return level &&
level !== "max" &&
(!lowest || claudeEffortRank(level) < claudeEffortRank(lowest))
? level
: lowest;
},
undefined,
);
if (!modelMaxEffortLevel) return model;
const capRank = claudeEffortRank(modelMaxEffortLevel);
return {
...model,
capabilities: {
...model.capabilities,
optionDescriptors: model.capabilities.optionDescriptors.map((descriptor) => {
if (descriptor.id !== "effort" || descriptor.type !== "select") return descriptor;
const cap = descriptor.options.reduce<{ rank: number; label: string } | undefined>(
(best, option) => {
const rank = claudeEffortRank(option.id);
return rank >= 0 && rank <= capRank && rank > (best?.rank ?? -1)
? { rank, label: option.label }
: best;
},
undefined,
);
if (!cap) return descriptor;
return {
...descriptor,
options: descriptor.options.map((option) => {
const idRank = claudeEffortRank(option.id);
const sentRank = claudeEffortRank(
normalizeClaudeCatalogEffort(catalog, option.id, model.slug),
);
const displayedRank = idRank >= 0 ? idRank : sentRank;
const runsRank = sentRank >= 0 ? sentRank : idRank;
return displayedRank > cap.rank && runsRank > cap.rank
? { ...option, label: `${option.label} (capped to ${cap.label})` }
: option;
}),
};
}),
},
};
});
}

export function resolveClaudeCatalogEffort(
catalog: ClaudeModelCatalog,
model: string | null | undefined,
raw: string | null | undefined,
): string | undefined {
if (!resolveClaudeCatalogModel(catalog, model)) {
const effort = raw?.trim();
return effort && ["low", "medium", "high", "xhigh", "max"].includes(effort)
return effort && (CLAUDE_EFFORT_LEVELS as ReadonlyArray<string>).includes(effort)
? effort
: undefined;
}
Expand Down
Loading
Loading