feat(voice): add Codex voice conversations across clients

apps/desktop/src/window/DesktopWindow.test.ts

@@ -93,6 +93,10 @@ function makeFakeBrowserWindow() {
     reloadIgnoringCache: vi.fn(),
     replaceMisspelling: vi.fn(),
     send: vi.fn(),
+    session: {
+      setPermissionCheckHandler: vi.fn(),
+      setPermissionRequestHandler: vi.fn(),
+    },
     setBackgroundThrottling: vi.fn(),
     setIgnoreMenuShortcuts: vi.fn(),
     setWindowOpenHandler: vi.fn(),

apps/desktop/src/window/DesktopWindow.ts

@@ -36,6 +36,7 @@ import * as DesktopAppSettings from "../settings/DesktopAppSettings.ts";
 import * as DesktopClientSettings from "../settings/DesktopClientSettings.ts";
 import * as ElectronApp from "../electron/ElectronApp.ts";
 import * as DesktopRendererHistory from "../telemetry/DesktopRendererHistory.ts";
+import { installMainWindowPermissionHandlers } from "./MainWindowPermissions.ts";
 import { makeQuitShortcutHandler } from "./QuitHold.ts";
 import { copyContextMenuImage } from "./ContextMenuImage.ts";
 
@@ -450,6 +451,10 @@ export const make = Effect.gen(function* () {
     if (environment.platform === "darwin") {
       window.setAutoHideCursor(false);
     }
+    installMainWindowPermissionHandlers(window.webContents.session, {
+      appUrl: applicationUrl,
+      platform: environment.platform,
+    });
     let boundsPersistFiber: Fiber.Fiber<void, never> | undefined;
     let pendingBoundsPersistFiber: Fiber.Fiber<void, never> | undefined;
     let boundsPersistenceEnabled = persistedBounds === null || restoredPersistedBounds;

apps/desktop/src/window/MainWindowPermissions.test.ts

@@ -0,0 +1,72 @@
+import { describe, expect, it } from "vite-plus/test";
+
+import {
+  decideMainWindowMediaRequest,
+  isAppUrl,
+  isMainWindowPermissionAllowed,
+} from "./MainWindowPermissions.ts";
+
+const APP_URL = "t3code://app/";
+
+function decide(overrides: Partial<Parameters<typeof decideMainWindowMediaRequest>[0]> = {}) {
+  return decideMainWindowMediaRequest({
+    requestingUrl: "t3code://app/threads/abc",
+    appUrl: APP_URL,
+    mediaTypes: ["audio"],
+    platform: "darwin",
+    microphoneStatus: () => "granted",
+    ...overrides,
+  });
+}
+
+describe("isAppUrl", () => {
+  it("matches the app shell and nothing that merely shares its prefix", () => {
+    expect(isAppUrl("t3code://app", APP_URL)).toBe(true);
+    expect(isAppUrl("t3code://app/settings", APP_URL)).toBe(true);
+    expect(isAppUrl("t3code://application/", APP_URL)).toBe(false);
+    expect(isAppUrl("https://example.com/", APP_URL)).toBe(false);
+    expect(isAppUrl(undefined, APP_URL)).toBe(false);
+  });
+});
+
+describe("decideMainWindowMediaRequest", () => {
+  it("denies media to frames outside the app shell", () => {
+    expect(decide({ requestingUrl: "https://example.com/embed" })).toBe("deny");
+  });
+
+  it("asks macOS for the microphone until the user has granted it", () => {
+    expect(decide({ microphoneStatus: () => "not-determined" })).toBe("ask-microphone");
+    expect(decide({ microphoneStatus: () => "denied" })).toBe("ask-microphone");
+    expect(decide()).toBe("grant");
+    expect(
+      isMainWindowPermissionAllowed("media", APP_URL, APP_URL, {
+        platform: "darwin",
+        microphoneStatus: () => "not-determined",
+      }),
+    ).toBe(false);
+    expect(
+      isMainWindowPermissionAllowed("media", APP_URL, APP_URL, {
+        platform: "darwin",
+        microphoneStatus: () => "granted",
+      }),
+    ).toBe(true);
+  });
+
+  it("grants the microphone directly off macOS", () => {
+    expect(decide({ platform: "win32", microphoneStatus: () => "unknown" })).toBe("grant");
+  });
+
+  it("keeps granting video-only capture such as preview tab recording", () => {
+    expect(decide({ mediaTypes: ["video"], microphoneStatus: () => "denied" })).toBe("grant");
+  });
+});
+
+it("denies synchronous clipboard reads for embedded origins and the app shell", () => {
+  for (const origin of ["https://example.com/frame", APP_URL]) {
+    expect(isMainWindowPermissionAllowed("deprecated-sync-clipboard-read", origin, APP_URL)).toBe(
+      false,
+    );
+  }
+  expect(isMainWindowPermissionAllowed("media", "https://example.com/frame", APP_URL)).toBe(false);
+  expect(isMainWindowPermissionAllowed("media", APP_URL, APP_URL)).toBe(true);
+});

apps/desktop/src/window/MainWindowPermissions.ts

@@ -0,0 +1,77 @@
+import * as Electron from "electron";
+
+export type MediaPermissionDecision = "grant" | "deny" | "ask-microphone";
+
+/** Whether `url` belongs to the app shell served from `appUrl` (e.g. `t3code://app/`). */
+export function isAppUrl(url: string | undefined, appUrl: string): boolean {
+  if (!url) return false;
+  // Custom schemes have an opaque WHATWG origin, so compare the serialized prefix.
+  const appOrigin = appUrl.endsWith("/") ? appUrl.slice(0, -1) : appUrl;
+  return url === appOrigin || url.startsWith(`${appOrigin}/`);
+}
+
+/**
+ * Media requests come from the app shell (voice conversations use the
+ * microphone; preview tab recording routes `getDisplayMedia` through the same
+ * permission). Frames from any other origin never get media. On macOS the
+ * microphone also needs the system's consent, asked for on first use.
+ */
+export function decideMainWindowMediaRequest(input: {
+  readonly requestingUrl: string;
+  readonly appUrl: string;
+  readonly mediaTypes: ReadonlyArray<"audio" | "video">;
+  readonly platform: NodeJS.Platform;
+  readonly microphoneStatus: () => string;
+}): MediaPermissionDecision {
+  if (!isAppUrl(input.requestingUrl, input.appUrl)) return "deny";
+  if (!input.mediaTypes.includes("audio") || input.platform !== "darwin") return "grant";
+  return input.microphoneStatus() === "granted" ? "grant" : "ask-microphone";
+}
+
+export function isMainWindowPermissionAllowed(
+  permission: string,
+  origin: string | undefined,
+  appUrl: string,
+  media?: { readonly platform: NodeJS.Platform; readonly microphoneStatus: () => string },
+): boolean {
+  return permission === "media"
+    ? isAppUrl(origin, appUrl) &&
+        (media?.platform !== "darwin" || media.microphoneStatus() !== "not-determined")
+    : permission !== "deprecated-sync-clipboard-read";
+}
+
+/**
+ * Installs the main window session's permission handlers. Only `media` is
+ * decided here. Synchronous clipboard reads keep Electron's default denial.
+ */
+export function installMainWindowPermissionHandlers(
+  session: Electron.Session,
+  options: { readonly appUrl: string; readonly platform: NodeJS.Platform },
+): void {
+  session.setPermissionRequestHandler((_webContents, permission, callback, details) => {
+    if (permission !== "media") {
+      callback(isMainWindowPermissionAllowed(permission, details.requestingUrl, options.appUrl));
+      return;
+    }
+    const decision = decideMainWindowMediaRequest({
+      requestingUrl: details.requestingUrl,
+      appUrl: options.appUrl,
+      mediaTypes: "mediaTypes" in details ? (details.mediaTypes ?? []) : [],
+      platform: options.platform,
+      microphoneStatus: () => Electron.systemPreferences.getMediaAccessStatus("microphone"),
+    });
+    if (decision !== "ask-microphone") {
+      callback(decision === "grant");
+      return;
+    }
+    Electron.systemPreferences.askForMediaAccess("microphone").then(callback, () => {
+      callback(false);
+    });
+  });
+  session.setPermissionCheckHandler((_webContents, permission, requestingOrigin) =>
+    isMainWindowPermissionAllowed(permission, requestingOrigin, options.appUrl, {
+      platform: options.platform,
+      microphoneStatus: () => Electron.systemPreferences.getMediaAccessStatus("microphone"),
+    }),
+  );
+}

apps/mobile/app.config.ts

@@ -230,6 +230,10 @@ const sharingPlugin: NonNullable<ExpoConfig["plugins"]>[number] = [
   },
 ];
 
+const MICROPHONE_PERMISSION =
+  "Allow T3 Code to use your microphone for voice input and voice conversations.";
+const CAMERA_PERMISSION = "Allow T3 Code to access your camera so you can scan pairing QR codes.";
+
 // Match iOS PostScript names on Android so RN and the native composer share fonts before runtime loading.
 
 const config: ExpoConfig = {
@@ -310,8 +314,11 @@ const config: ExpoConfig = {
     },
     // JS back dispatch needs RN's Android 16 shim and withAndroidPredictiveBackCompat on Android 13-15.
     predictiveBackGestureEnabled: true,
-    // expo-sensors declares this for its pedometer, which the app does not use.
-    blockedPermissions: ["android.permission.ACTIVITY_RECOGNITION"],
+    // The app uses neither expo-sensors' pedometer nor WebRTC screen sharing.
+    blockedPermissions: [
+      "android.permission.ACTIVITY_RECOGNITION",
+      "android.permission.SYSTEM_ALERT_WINDOW",
+    ],
     intentFilters: [
       {
         action: "VIEW",
@@ -383,29 +390,30 @@ const config: ExpoConfig = {
     [
       "expo-audio",
       {
-        microphonePermission: "Allow T3 Code to use your microphone for voice input.",
+        microphonePermission: MICROPHONE_PERMISSION,
         recordAudioAndroid: true,
         enableBackgroundPlayback: false,
         enableBackgroundRecording: false,
       },
     ],
+    // Voice conversations run over WebRTC. Pass both usage strings so its
+    // plugin cannot fall back to generic ones.
+    [
+      "@config-plugins/react-native-webrtc",
+      { cameraPermission: CAMERA_PERMISSION, microphonePermission: MICROPHONE_PERMISSION },
+    ],
     [
       "expo-camera",
       {
-        cameraPermission: "Allow T3 Code to access your camera so you can scan pairing QR codes.",
+        cameraPermission: CAMERA_PERMISSION,
         microphonePermission: false,
         barcodeScannerEnabled: true,
         recordAudioAndroid: false,
       },
     ],
-    [
-      "expo-image-picker",
-      {
-        photosPermission: false,
-        // `false` makes this plugin add RECORD_AUDIO to blockedPermissions, breaking Android dictation.
-        microphonePermission: "Allow T3 Code to use your microphone for voice input.",
-      },
-    ],
+    // `microphonePermission: false` would block RECORD_AUDIO on Android for every
+    // package, including voice conversations, so share the app's usage string.
+    ["expo-image-picker", { photosPermission: false, microphonePermission: MICROPHONE_PERMISSION }],
     [
       "expo-splash-screen",
       {

apps/mobile/package.json

@@ -130,6 +130,7 @@
     "react-native-safe-area-context": "~5.11.1",
     "react-native-screens": "file:deps/react-native-screens-5.0.0-t3.48.tgz",
     "react-native-svg": "15.16.1",
+    "react-native-webrtc": "124.0.8",
     "react-native-webview": "14.0.1",
     "react-native-worklets": "0.13.1",
     "remark-gfm": "catalog:",
@@ -140,6 +141,7 @@
   },
   "devDependencies": {
     "@babel/core": "^7.29.7",
+    "@config-plugins/react-native-webrtc": "15.0.2",
     "@effect/platform-node": "catalog:",
     "@effect/vitest": "catalog:",
     "@pierre/trees": "1.0.0-beta.6",

apps/mobile/src/components/ui/AppSymbol.android.tsx

@@ -85,6 +85,7 @@ import IconLayoutSidebar from "@tabler/icons-react-native/IconLayoutSidebar";
 import IconLayoutSidebarRight from "@tabler/icons-react-native/IconLayoutSidebarRight";
 import IconLetterSpacing from "@tabler/icons-react-native/IconLetterSpacing";
 import IconMicrophone from "@tabler/icons-react-native/IconMicrophone";
+import IconMicrophoneOff from "@tabler/icons-react-native/IconMicrophoneOff";
 import IconLink from "@tabler/icons-react-native/IconLink";
 import IconListNumbers from "@tabler/icons-react-native/IconListNumbers";
 import IconMenu2 from "@tabler/icons-react-native/IconMenu2";
@@ -134,6 +135,7 @@ import IconUserQuestion from "@tabler/icons-react-native/IconUserQuestion";
 import IconUserPlus from "@tabler/icons-react-native/IconUserPlus";
 import IconUsers from "@tabler/icons-react-native/IconUsers";
 import IconWifi from "@tabler/icons-react-native/IconWifi";
+import IconWaveSine from "@tabler/icons-react-native/IconWaveSine";
 import IconWifiOff from "@tabler/icons-react-native/IconWifiOff";
 import IconWorld from "@tabler/icons-react-native/IconWorld";
 import IconX from "@tabler/icons-react-native/IconX";
@@ -232,6 +234,7 @@ const ANDROID_ICON_BY_SF_SYMBOL = {
   macstudio: IconDeviceDesktop,
   magnifyingglass: IconSearch,
   mic: IconMicrophone,
+  "mic.slash": IconMicrophoneOff,
   paintbrush: IconPalette,
   pencil: IconPencil,
   "person.crop.circle": IconUserCircle,
@@ -285,6 +288,7 @@ const ANDROID_ICON_BY_SF_SYMBOL = {
   "tray.and.arrow.up": IconUpload,
   trash: IconTrash,
   wifi: IconWifi,
+  waveform: IconWaveSine,
   "wifi.slash": IconWifiOff,
   xmark: IconX,
   "xmark.circle.fill": IconCircleXFilled,

apps/mobile/src/features/threads/ThreadComposer.tsx

@@ -116,6 +116,9 @@ import {
 import { useVoiceInputController } from "../voice-input/useVoiceInputController";
 import { resolveVoiceComposerPresentation } from "../voice-input/voiceInputPresentation";
 import type { ExistingThreadSettingsRouteParams } from "./ThreadSettingsSheet";
+import { ComposerVoiceModeButton, ComposerVoiceStrip } from "../voice-mode/ComposerVoiceMode";
+import { useVoiceModePhase, useVoiceModeThreadLifecycle } from "../voice-mode/useVoiceMode";
+import { voiceMode, voiceModeSupported } from "../voice-mode/voiceMode";
 import {
   useThreadSettingsSheetPresentation,
   type NavigationWithFinishTransitioning,
@@ -571,17 +574,37 @@ export const ThreadComposer = function ThreadComposer(props: ThreadComposerProps
     ...(props.onFreshSession === undefined ? {} : { onFreshSession: props.onFreshSession }),
     offersSideChat: props.offersSideChat === true,
   });
+  const voiceTarget = { environmentId: props.environmentId, threadId: props.selectedThread.id };
+  const voicePhase = useVoiceModePhase(voiceTarget);
+  useVoiceModeThreadLifecycle(voiceTarget);
+  // Dictation and a voice conversation both need the microphone.
+  const voiceConversationLive = voicePhase !== "idle";
   const voiceInput = useVoiceInputController({
     ownerKey: composerDraftKey,
     label: props.selectedThread.title || "Untitled thread",
     readDraftMessage: () => getComposerDraftSnapshot(composerDraftKey).text,
     subscribeToDraftChanges: (onChange) => appAtomRegistry.subscribe(composerDraftsAtom, onChange),
     selection: composerMenu.selection,
+    disabled: voiceConversationLive,
     onChangeDraftMessage: (text) => setComposerDraftText(composerDraftKey, text),
     onChangeSelection: composerMenu.onSelectionChange,
   });
   const isSubmissionLocked =
     props.interactionLocked === true || queuedEdit?.saving === true || voiceInput.blocksSubmission;
+  const offersVoiceConversation =
+    voiceModeSupported &&
+    props.serverConfig?.providers.find(
+      (provider) => provider.instanceId === props.selectedThread.providerInstanceId,
+    )?.supportsVoice === true;
+  // The server attaches voice to the thread's provider thread, which the first
+  // run creates. An optimistic pending thread already has a message timestamp.
+  const canToggleVoiceConversation =
+    voiceConversationLive ||
+    (props.connectionState === "connected" &&
+      props.canOperateThread &&
+      props.selectedThread.activeProviderThreadId !== null &&
+      !voiceInput.isGloballyBusy);
+  const toggleVoiceConversation = () => voiceMode.toggle(voiceTarget);
   const voicePresentation = resolveVoiceComposerPresentation(
     voiceInput.state,
     voiceInput.elapsedSeconds,
@@ -594,6 +617,14 @@ export const ThreadComposer = function ThreadComposer(props: ThreadComposerProps
   const deviceType = useDeviceType();
   const usesLargeControls = sidebarWorkspaceTabs === "picker" || deviceType === "pad";
   const surfacePaddingVertical = usesLargeControls ? 9 : floatingSidebarComposer ? 6 : 2;
+  const voiceModeButton = offersVoiceConversation ? (
+    <ComposerVoiceModeButton
+      size={usesLargeControls ? "large" : "regular"}
+      phase={voicePhase}
+      disabled={!canToggleVoiceConversation}
+      onPress={toggleVoiceConversation}
+    />
+  ) : null;
   const [compactRowWidth, setCompactRowWidth] = useState(0);
   const compactAttachmentLimit = compactRowWidth >= 500 ? 3 : 1;
   const compactEditor = useCompactComposerEditor(bodyText, draftMessage);
@@ -872,6 +903,9 @@ export const ThreadComposer = function ThreadComposer(props: ThreadComposerProps
             {selectedProviderStatus.compatibilityAdvisory.message}
           </Text>
         ) : null}
+
+        {voiceConversationLive ? <ComposerVoiceStrip /> : null}
+
         {modelUnavailable ? (
           <Button
             label="Model unavailable. Open model settings."
@@ -1093,11 +1127,12 @@ export const ThreadComposer = function ThreadComposer(props: ThreadComposerProps
                 }
               >
                 {modelControl}
+                {voiceModeButton}
                 <ComposerDictationStartAction
                   size={usesLargeControls ? "large" : "regular"}
                   state={voiceInput.state}
                   isAvailable={voiceInput.isAvailable}
-                  disabled={isSubmissionLocked}
+                  disabled={isSubmissionLocked || voiceConversationLive}
                   onStart={voiceInput.start}
                   onCancel={voiceInput.cancel}
                 />
@@ -1164,7 +1199,7 @@ export const ThreadComposer = function ThreadComposer(props: ThreadComposerProps
                     state={voiceInput.state}
                     presentation={voicePresentation}
                     isAvailable={voiceInput.isAvailable}
-                    disabled={isSubmissionLocked}
+                    disabled={isSubmissionLocked || voiceConversationLive}
                     onStart={voiceInput.start}
                     onConfirm={voiceInput.stop}
                     onCancel={voiceInput.cancel}