feat(speech): use generic vocabulary for recognition hints

apps/server/src/speech/SpeechService.test.ts

@@ -17,8 +17,8 @@ const native = vi.hoisted(() => ({
   backend: "Vulkan0",
   supportsStreaming: true,
   supportsTranslation: true,
-  supportsInitialPrompt: false,
-  begin: vi.fn(async (_language?: string, _words?: readonly string[], _dictionary?: unknown) => {}),
+  supportsVocabulary: false,
+  begin: vi.fn(async (_language?: string, _vocabulary?: readonly string[]) => {}),
   feed: vi.fn(async (_pcm: Float32Array) => ({
     revision: 1,
     text: { committed: "", tentative: "hello" },
@@ -107,7 +107,7 @@ const pcm = () => new Uint8Array(new Float32Array([0.25]).buffer);
 beforeEach(() => {
   vi.clearAllMocks();
   native.backend = "Vulkan0";
-  native.supportsInitialPrompt = false;
+  native.supportsVocabulary = false;
   readyModels.clear();
   readyModels.add("test-model");
   readyModels.add("fallback-model");
@@ -201,6 +201,55 @@ it.effect("honors exact aliases and phrase boundaries in batch and streamed text
   }).pipe(Effect.scoped, Effect.provide(layer)),
 );
 
+it.effect("uses vocabulary only for supported transcription and always applies aliases", () =>
+  Effect.gen(function* () {
+    for (const supportsVocabulary of [false, true]) {
+      for (const translate of [false, true]) {
+        yield* Effect.gen(function* () {
+          native.supportsVocabulary = supportsVocabulary;
+          const speech = yield* SpeechService.SpeechService;
+          const options = {
+            ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
+            speechLanguage: "fr" as const,
+            speechTranslateToEnglish: translate,
+            speechCustomWords: [
+              { term: "GitHub", aliases: [] },
+              { term: "Bob", aliases: ["Robert"] },
+              { term: "T3 Code", aliases: [] },
+            ],
+          };
+          const vocabulary =
+            supportsVocabulary && !translate ? ["GitHub", "Bob", "T3 Code"] : undefined;
+          native.transcribe.mockResolvedValueOnce({ text: "git hub and Robert" });
+          expect(yield* speech.transcribe(pcm(), options)).toBe(
+            vocabulary ? "git hub and Bob" : "GitHub and Bob",
+          );
+          const delivered = native.transcribe.mock.lastCall?.[1];
+          expect(delivered).toEqual({
+            timestamps: "none",
+            language: "fr",
+            ...(translate ? { task: "translate", targetLanguage: "en" } : {}),
+            ...(vocabulary ? { vocabulary } : {}),
+          });
+          if (translate) return;
+          native.feed.mockResolvedValueOnce({
+            revision: 1,
+            text: { committed: "git hub", tentative: "Robert" },
+          });
+          native.finish.mockResolvedValueOnce("git hub and Robert");
+          const stream = yield* speech.startStream(options);
+          expect(yield* stream.feed(pcm())).toEqual({
+            revision: 1,
+            text: { committed: vocabulary ? "git hub" : "GitHub", tentative: "Bob" },
+          });
+          expect(yield* stream.finish).toBe(vocabulary ? "git hub and Bob" : "GitHub and Bob");
+          expect(native.begin.mock.lastCall).toEqual(["fr", vocabulary]);
+        }).pipe(Effect.scoped, Effect.provide(layer));
+      }
+    }
+  }),
+);
+
 it.effect("unloads after the chosen idle period and reloads on the next recording", () =>
   Effect.gen(function* () {
     const speech = yield* SpeechService.SpeechService;
@@ -284,15 +333,48 @@ it.effect("reports unsupported hosts without wrapping a synthetic cause", () =>
   }),
 );
 
-it.effect("retries Auto-placed inference on this host's CPU after the native process fails", () =>
-  Effect.gen(function* () {
-    native.transcribe.mockRejectedValueOnce(new Error("worker stopped"));
-    const speech = yield* SpeechService.SpeechService;
-    expect(yield* speech.transcribe(pcm(), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS)).toBe("hello");
-    expect(yield* speech.transcribe(pcm(), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS)).toBe("hello");
-    expect(loadNative).toHaveBeenCalledTimes(2);
-    expect(loadNative.mock.calls[1]?.[3]).toBe("cpu");
-  }).pipe(Effect.provide(layer)),
+it.effect(
+  "retries Auto-placed inference with the dictionary and the CPU's vocabulary capability",
+  () =>
+    Effect.gen(function* () {
+      for (const cpuSupportsVocabulary of [true, false]) {
+        yield* Effect.gen(function* () {
+          native.supportsVocabulary = true;
+          loadNative.mockImplementationOnce(async () => native);
+          loadNative.mockImplementationOnce(async () => ({
+            ...native,
+            backend: "CPU",
+            supportsVocabulary: cpuSupportsVocabulary,
+          }));
+          native.transcribe
+            .mockRejectedValueOnce(new Error("worker stopped"))
+            .mockResolvedValueOnce({ text: "git hub Robert" });
+          const options = {
+            ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
+            speechCustomWords: [
+              { term: "GitHub", aliases: [] },
+              { term: "Bob", aliases: ["Robert"] },
+            ],
+          };
+          const speech = yield* SpeechService.SpeechService;
+          expect(yield* speech.transcribe(pcm(), options)).toBe(
+            cpuSupportsVocabulary ? "git hub Bob" : "GitHub Bob",
+          );
+          expect(yield* speech.transcribe(pcm(), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS)).toBe(
+            "hello",
+          );
+          expect(loadNative.mock.calls.slice(-2).map((call) => call[3])).toEqual(["auto", "cpu"]);
+          expect(native.transcribe.mock.calls.slice(-3, -1).map((call) => call[1])).toEqual([
+            { timestamps: "none", language: "en", vocabulary: ["GitHub", "Bob"] },
+            {
+              timestamps: "none",
+              language: "en",
+              ...(cpuSupportsVocabulary ? { vocabulary: ["GitHub", "Bob"] } : {}),
+            },
+          ]);
+        }).pipe(Effect.provide(layer));
+      }
+    }),
 );
 
 it.effect("reports a failure on a GPU the person chose instead of switching to CPU", () =>
@@ -317,23 +399,49 @@ it.effect("reports a failure on a GPU the person chose instead of switching to C
 
 it.effect("replays streamed audio on CPU after an Auto-placed feed crashes, once", () =>
   Effect.gen(function* () {
-    const text = { committed: "", tentative: "hello" };
-    native.feed
-      .mockResolvedValueOnce({ revision: 3, text })
-      .mockRejectedValueOnce(new Error("worker stopped"))
-      .mockResolvedValueOnce({ revision: 1, text })
-      .mockResolvedValueOnce({ revision: 2, text })
-      .mockResolvedValueOnce({ revision: 3, text })
-      .mockRejectedValueOnce(new Error("worker stopped again"));
-    const speech = yield* SpeechService.SpeechService;
-    const stream = yield* speech.startStream(DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS);
-    expect(yield* stream.feed(pcm())).toEqual({ revision: 3, text });
-    expect(yield* stream.feed(pcm())).toEqual({ revision: 5, text });
-    expect(loadNative).toHaveBeenCalledTimes(2);
-    expect(loadNative.mock.calls[1]?.[3]).toBe("cpu");
-    expect(native.begin).toHaveBeenCalledTimes(2);
-    expect(native.feed).toHaveBeenCalledTimes(4);
-    expect(yield* stream.feed(pcm())).toEqual({ revision: 6, text });
-    expect(Result.isFailure(yield* Effect.result(stream.feed(pcm())))).toBe(true);
-  }).pipe(Effect.scoped, Effect.provide(layer)),
+    for (const cpuSupportsVocabulary of [true, false]) {
+      yield* Effect.gen(function* () {
+        native.supportsVocabulary = true;
+        const feedCalls = native.feed.mock.calls.length;
+        const text = { committed: "", tentative: "git hub Robert" };
+        const corrected = { committed: "", tentative: "git hub Bob" };
+        const options = {
+          ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
+          speechCustomWords: [
+            { term: "GitHub", aliases: [] },
+            { term: "Bob", aliases: ["Robert"] },
+          ],
+        };
+        native.feed
+          .mockResolvedValueOnce({ revision: 3, text })
+          .mockRejectedValueOnce(new Error("worker stopped"))
+          .mockResolvedValueOnce({ revision: 1, text })
+          .mockResolvedValueOnce({ revision: 2, text })
+          .mockResolvedValueOnce({ revision: 3, text })
+          .mockRejectedValueOnce(new Error("worker stopped again"));
+        loadNative.mockImplementationOnce(async () => native);
+        loadNative.mockImplementationOnce(async () => ({
+          ...native,
+          backend: "CPU",
+          supportsVocabulary: cpuSupportsVocabulary,
+        }));
+        const speech = yield* SpeechService.SpeechService;
+        const stream = yield* speech.startStream(options);
+        expect(yield* stream.feed(pcm())).toEqual({ revision: 3, text: corrected });
+        const recovered = {
+          committed: "",
+          tentative: cpuSupportsVocabulary ? "git hub Bob" : "GitHub Bob",
+        };
+        expect(yield* stream.feed(pcm())).toEqual({ revision: 5, text: recovered });
+        expect(loadNative.mock.calls.slice(-2).map((call) => call[3])).toEqual(["auto", "cpu"]);
+        expect(native.begin.mock.calls.slice(-2)).toEqual([
+          ["en", ["GitHub", "Bob"]],
+          ["en", cpuSupportsVocabulary ? ["GitHub", "Bob"] : undefined],
+        ]);
+        expect(native.feed).toHaveBeenCalledTimes(feedCalls + 4);
+        expect(yield* stream.feed(pcm())).toEqual({ revision: 6, text: recovered });
+        expect(Result.isFailure(yield* Effect.result(stream.feed(pcm())))).toBe(true);
+      }).pipe(Effect.scoped, Effect.provide(layer));
+    }
+  }),
 );

apps/server/src/speech/SpeechService.ts

@@ -557,6 +557,9 @@ const make = Effect.gen(function* () {
             });
             const dictionary = normalizeSpeechCustomWords(options.speechCustomWords);
             const customWords = dictionary.map(({ term }) => term);
+            // The vocabulary capability guarantees plain transcription, not translation.
+            const vocabularyFor = (inference: LoadedModel) =>
+              inference.supportsVocabulary && !translateToEnglish ? customWords : [];
             const transcribeWith = (inference: LoadedModel) =>
               Effect.tryPromise({
                 try: (signal) => {
@@ -568,14 +571,7 @@ const make = Effect.gen(function* () {
                     ...(translateToEnglish
                       ? { task: "translate" as const, targetLanguage: "en" as const }
                       : {}),
-                    ...(customWords.length > 0 && inference.supportsInitialPrompt
-                      ? {
-                          family: {
-                            kind: "whisper" as const,
-                            initialPrompt: customWords.join(", "),
-                          },
-                        }
-                      : {}),
+                    ...(vocabularyFor(inference).length > 0 ? { vocabulary: customWords } : {}),
                   });
                 },
                 catch: (cause) => {
@@ -606,7 +602,7 @@ const make = Effect.gen(function* () {
               }),
             );
             const correct = makeSpeechTextCorrector(
-              inference.supportsInitialPrompt ? [] : customWords,
+              vocabularyFor(inference).length > 0 ? [] : customWords,
               dictionary,
             );
             return finalizeTranscript(
@@ -673,10 +669,8 @@ const make = Effect.gen(function* () {
     const definition = selectedModel(settings);
     const language = effectiveSpeechLanguage(definition, options.speechLanguage);
     const dictionary = normalizeSpeechCustomWords(options.speechCustomWords);
-    const correct = makeSpeechTextCorrector(
-      dictionary.map(({ term }) => term),
-      dictionary,
-    );
+    const customWords = dictionary.map(({ term }) => term);
+    let correct = makeSpeechTextCorrector(customWords, dictionary);
     const modelPath = yield* installedModelPath(definition);
     let streamModel = yield* attempt(operation, () =>
       loadModel(modelPath, definition, signal, settings.speechAcceleration),
@@ -685,7 +679,10 @@ const make = Effect.gen(function* () {
     const beginStream = async () => {
       if (!streamModel.supportsStreaming)
         throw new Error("The selected model does not support streaming.");
-      await streamModel.begin(language === "auto" ? undefined : language);
+      const vocabulary =
+        streamModel.supportsVocabulary && customWords.length > 0 ? customWords : undefined;
+      await streamModel.begin(language === "auto" ? undefined : language, vocabulary);
+      correct = makeSpeechTextCorrector(vocabulary ? [] : customWords, dictionary);
       started = true;
     };
     // Audio already fed is kept so a CPU restart can replay it; the limit caps it at one recording.

apps/server/src/speech/native.test.ts

@@ -3,8 +3,8 @@ import { expect, it } from "vite-plus/test";
 
 import { loadNativeSpeechModel } from "./native.ts";
 
-const fixture = (transcribe: string, maxAudioMs = 0) =>
-  `data:text/javascript,${encodeURIComponent(`export const TranscribeModel = { load: async () => ({ capabilities: { supportsStreaming: false, maxAudioMs: ${maxAudioMs} }, supports: () => false, transcribe: ${transcribe} }) };`)}`;
+const fixture = (transcribe: string, maxAudioMs = 0, supportsVocabulary = false) =>
+  `data:text/javascript,${encodeURIComponent(`export const TranscribeModel = { load: async () => ({ capabilities: { supportsStreaming: false, maxAudioMs: ${maxAudioMs} }, supports: (feature) => feature === "vocabulary" && ${supportsVocabulary}, transcribe: ${transcribe} }) };`)}`;
 
 const fixtureWithUnrelatedIpcMessage = () =>
   `data:text/javascript,${encodeURIComponent(`
@@ -14,17 +14,17 @@ const fixtureWithUnrelatedIpcMessage = () =>
     };
   `)}`;
 
-const streamingFixture = (feed: string) =>
+const streamingFixture = (feed: string, supportsVocabulary = false) =>
   `data:text/javascript,${encodeURIComponent(`
   export const TranscribeModel = { load: async () => ({
     capabilities: { supportsStreaming: true },
-    supports: () => false,
+    supports: (feature) => feature === "vocabulary" && ${supportsVocabulary},
     createSession: () => ({
       dispose() {},
       stream: async (opts) => ({
         feed: ${feed},
         finalize: async () => {},
-        text: { full: opts.language ?? "hello world", committed: "hello ", tentative: "world" },
+        text: { full: opts.vocabulary?.join(" / ") ?? opts.language ?? "hello world", committed: "hello ", tentative: "world" },
         reset() {},
       }),
     }),
@@ -46,6 +46,28 @@ it("returns transcription from an isolated process and disposes it", async () =>
   }
 });
 
+it("probes generic vocabulary support and delivers ordered terms to offline inference", async () => {
+  for (const supported of [false, true]) {
+    const model = await loadNativeSpeechModel(
+      "unused.gguf",
+      new AbortController().signal,
+      fixture("async (_pcm, options) => ({ text: JSON.stringify(options) })", 0, supported),
+    );
+    try {
+      expect(model.supportsVocabulary).toBe(supported);
+      const options = {
+        timestamps: "none" as const,
+        language: "en",
+        ...(supported ? { vocabulary: ["T3 Code", "GitHub"] } : {}),
+      };
+      const result = await model.transcribe(new Float32Array([0.25]), options);
+      expect(JSON.parse(result.text)).toEqual(options);
+    } finally {
+      await model.dispose();
+    }
+  }
+});
+
 it("ignores IPC messages that do not belong to the speech protocol", async () => {
   const model = await loadNativeSpeechModel(
     "unused.gguf",
@@ -84,6 +106,7 @@ it("returns streaming previews and final text from an isolated process", async (
     new AbortController().signal,
     streamingFixture(
       "async () => ({ revision: 1, committedChanged: true, tentativeChanged: true })",
+      true,
     ),
   );
   try {
@@ -98,6 +121,8 @@ it("returns streaming previews and final text from an isolated process", async (
     await model.reset();
     await model.begin("fr");
     expect(await model.finish()).toBe("fr");
+    await model.begin("fr", ["T3 Code", "GitHub"]);
+    expect(await model.finish()).toBe("T3 Code / GitHub");
   } finally {
     await model.dispose();
   }

apps/server/src/speech/native.ts

@@ -25,11 +25,11 @@ process.on("message", async (message) => {
       process.send({ type: "t3-speech-reply", ok: true, backend: model.backend,
         supportsStreaming: model.capabilities.supportsStreaming,
         supportsTranslation: model.capabilities.supportsTranslate,
-        // Vocabulary prompts are a Whisper run extension until generic prompting lands here.
-        supportsInitialPrompt: model.supports("initial_prompt") && model.accepts({ kind: "whisper" }) });
+        supportsVocabulary: model.supports("vocabulary") });
     } else if (message.kind === "begin") {
       session = model.createSession();
-      stream = await session.stream({ timestamps: "none", language: message.language });
+      stream = await session.stream({ timestamps: "none", language: message.language,
+        vocabulary: message.vocabulary });
       process.send({ type: "t3-speech-reply", ok: true });
     } else if (message.kind === "feed") {
       const update = await stream.feed(message.pcm);
@@ -105,7 +105,7 @@ type Reply = {
   readonly error?: string;
   readonly supportsStreaming?: boolean;
   readonly supportsTranslation?: boolean;
-  readonly supportsInitialPrompt?: boolean;
+  readonly supportsVocabulary?: boolean;
   readonly backend?: string;
   readonly revision?: number;
   readonly preview?: SpeechStreamText | null;
@@ -306,9 +306,9 @@ async function loadNativeSpeechModelOnce(
     backend: loaded.backend ?? "unknown",
     supportsStreaming: loaded.supportsStreaming === true,
     supportsTranslation: loaded.supportsTranslation === true,
-    supportsInitialPrompt: loaded.supportsInitialPrompt === true,
-    begin: async (language?: string) => {
-      await send({ kind: "begin", language });
+    supportsVocabulary: loaded.supportsVocabulary === true,
+    begin: async (language?: string, vocabulary?: readonly string[]) => {
+      await send({ kind: "begin", language, vocabulary });
     },
     feed: async (pcm: Float32Array) => {
       const reply = await send({ kind: "feed", pcm });
@@ -348,7 +348,7 @@ async function loadNativeSpeechModelOnce(
         readonly task?: "translate";
         readonly targetLanguage?: "en";
         readonly language?: string;
-        readonly family?: { readonly kind: "whisper"; readonly initialPrompt: string };
+        readonly vocabulary?: readonly string[];
       },
     ) => {
       const result = await send({ kind: "transcribe", pcm, options });