Skip to content

Commit 2e2829c

Browse files
committed
feat(ui): add MCP Apps
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
1 parent c9bcd38 commit 2e2829c

File tree

9 files changed

+325
-40
lines changed

9 files changed

+325
-40
lines changed

README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -235,7 +235,7 @@ local-ai run oci://localai/phi-2:latest
235235
For more information, see [💻 Getting started](https://localai.io/basics/getting_started/index.html), if you are interested in our roadmap items and future enhancements, you can see the [Issues labeled as Roadmap here](https://github.com/mudler/LocalAI/issues?q=is%3Aissue+is%3Aopen+label%3Aroadmap)
236236

237237
## 📰 Latest project news
238-
- March 2026: [Agent management](https://github.com/mudler/LocalAI/pull/8820), [New React UI](https://github.com/mudler/LocalAI/pull/8772), [WebRTC](https://github.com/mudler/LocalAI/pull/8790),[MLX-distributed via P2P and RDMA](https://github.com/mudler/LocalAI/pull/8801)
238+
- March 2026: [Agent management](https://github.com/mudler/LocalAI/pull/8820), [New React UI](https://github.com/mudler/LocalAI/pull/8772), [WebRTC](https://github.com/mudler/LocalAI/pull/8790),[MLX-distributed via P2P and RDMA](https://github.com/mudler/LocalAI/pull/8801), [MCP Apps, MCP Client-side](https://github.com/mudler/LocalAI/pull/8947)
239239
- February 2026: [Realtime API for audio-to-audio with tool calling](https://github.com/mudler/LocalAI/pull/6245), [ACE-Step 1.5 support](https://github.com/mudler/LocalAI/pull/8396)
240240
- January 2026: **LocalAI 3.10.0** - Major release with Anthropic API support, Open Responses API for stateful agents, video & image generation suite (LTX-2), unified GPU backends, tool streaming & XML parsing, system-aware backend gallery, crash fixes for AVX-only CPUs and AMD VRAM reporting, request tracing, and new backends: **Moonshine** (ultra-fast transcription), **Pocket-TTS** (lightweight TTS). Vulkan arm64 builds now available. [Release notes](https://github.com/mudler/LocalAI/releases/tag/v3.10.0).
241241
- December 2025: [Dynamic Memory Resource reclaimer](https://github.com/mudler/LocalAI/pull/7583), [Automatic fitting of models to multiple GPUS(llama.cpp)](https://github.com/mudler/LocalAI/pull/7584), [Added Vibevoice backend](https://github.com/mudler/LocalAI/pull/7494)

core/http/react-ui/package-lock.json

Lines changed: 23 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

core/http/react-ui/package.json

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,8 @@
1717
"marked": "^15.0.7",
1818
"dompurify": "^3.2.5",
1919
"@fortawesome/fontawesome-free": "^6.7.2",
20-
"@modelcontextprotocol/sdk": "^1.25.1"
20+
"@modelcontextprotocol/sdk": "^1.25.1",
21+
"@modelcontextprotocol/ext-apps": "^1.2.2"
2122
},
2223
"devDependencies": {
2324
"@vitejs/plugin-react": "^4.5.2",

core/http/react-ui/src/App.css

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2498,3 +2498,37 @@
24982498
gap: var(--spacing-xs);
24992499
}
25002500
}
2501+
2502+
/* MCP App Frame */
2503+
.mcp-app-frame-container {
2504+
width: 100%;
2505+
margin: var(--spacing-sm) 0;
2506+
border-radius: var(--border-radius-md);
2507+
overflow: hidden;
2508+
border: 1px solid var(--color-border-subtle);
2509+
}
2510+
2511+
.mcp-app-iframe {
2512+
width: 100%;
2513+
border: none;
2514+
display: block;
2515+
min-height: 100px;
2516+
max-height: 600px;
2517+
transition: height 0.2s ease;
2518+
background: var(--color-bg-primary);
2519+
}
2520+
2521+
.mcp-app-error {
2522+
padding: var(--spacing-sm) var(--spacing-md);
2523+
color: var(--color-text-danger, #e53e3e);
2524+
font-size: 0.85rem;
2525+
}
2526+
2527+
.mcp-app-reconnect-overlay {
2528+
padding: var(--spacing-sm);
2529+
text-align: center;
2530+
font-size: 0.8rem;
2531+
color: var(--color-text-secondary);
2532+
background: var(--color-bg-secondary);
2533+
border-top: 1px solid var(--color-border-subtle);
2534+
}
Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
import { useRef, useEffect, useState, useCallback } from 'react'
2+
import { AppBridge, PostMessageTransport, buildAllowAttribute } from '@modelcontextprotocol/ext-apps/app-bridge'
3+
4+
export default function MCPAppFrame({ toolName, toolInput, toolResult, mcpClient, toolDefinition, appHtml, resourceMeta }) {
5+
const iframeRef = useRef(null)
6+
const bridgeRef = useRef(null)
7+
const [iframeHeight, setIframeHeight] = useState(200)
8+
const [error, setError] = useState(null)
9+
const initializedRef = useRef(false)
10+
11+
const setupBridge = useCallback(async () => {
12+
if (!mcpClient || !iframeRef.current || initializedRef.current) return
13+
14+
const iframe = iframeRef.current
15+
initializedRef.current = true
16+
17+
try {
18+
const transport = new PostMessageTransport(iframe.contentWindow, iframe.contentWindow)
19+
const bridge = new AppBridge(
20+
mcpClient,
21+
{ name: 'LocalAI', version: '1.0.0' },
22+
{ openLinks: {}, serverTools: {}, serverResources: {}, logging: {} },
23+
{ hostContext: { displayMode: 'inline' } }
24+
)
25+
26+
bridge.oninitialized = () => {
27+
if (toolInput) bridge.sendToolInput({ arguments: toolInput })
28+
if (toolResult) bridge.sendToolResult(toolResult)
29+
}
30+
31+
bridge.onsizechange = ({ height }) => {
32+
if (height && height > 0) setIframeHeight(Math.min(height, 600))
33+
}
34+
35+
bridge.onopenlink = async ({ url }) => {
36+
window.open(url, '_blank', 'noopener,noreferrer')
37+
return {}
38+
}
39+
40+
bridge.onmessage = async () => {
41+
return {}
42+
}
43+
44+
bridge.onrequestdisplaymode = async () => {
45+
return { mode: 'inline' }
46+
}
47+
48+
await bridge.connect(transport)
49+
bridgeRef.current = bridge
50+
} catch (err) {
51+
setError(`Bridge error: ${err.message}`)
52+
}
53+
}, [mcpClient, toolInput, toolResult])
54+
55+
const handleIframeLoad = useCallback(() => {
56+
setupBridge()
57+
}, [setupBridge])
58+
59+
// Send toolResult when it arrives after initialization
60+
useEffect(() => {
61+
if (bridgeRef.current && toolResult && initializedRef.current) {
62+
bridgeRef.current.sendToolResult(toolResult)
63+
}
64+
}, [toolResult])
65+
66+
// Cleanup on unmount — only close the local transport, don't send
67+
// teardownResource which would kill server-side state and cause
68+
// "Connection closed" errors if the component remounts (e.g. when
69+
// streaming ends and ActivityGroup takes over from StreamingActivity).
70+
useEffect(() => {
71+
return () => {
72+
const bridge = bridgeRef.current
73+
if (bridge) {
74+
try { bridge.close() } catch (_) { /* ignore */ }
75+
}
76+
}
77+
}, [])
78+
79+
if (!appHtml) return null
80+
81+
const permissions = resourceMeta?.permissions
82+
const allowAttr = permissions ? buildAllowAttribute(permissions) : undefined
83+
84+
return (
85+
<div className="mcp-app-frame-container">
86+
<iframe
87+
ref={iframeRef}
88+
srcDoc={appHtml}
89+
sandbox="allow-scripts allow-forms"
90+
allow={allowAttr}
91+
className="mcp-app-iframe"
92+
style={{ height: `${iframeHeight}px` }}
93+
onLoad={handleIframeLoad}
94+
title={`MCP App: ${toolName || 'unknown'}`}
95+
/>
96+
{error && <div className="mcp-app-error">{error}</div>}
97+
{!mcpClient && (
98+
<div className="mcp-app-reconnect-overlay">
99+
Reconnect to MCP server to interact with this app
100+
</div>
101+
)}
102+
</div>
103+
)
104+
}

core/http/react-ui/src/hooks/useChat.js

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -632,13 +632,25 @@ export function useChat(initialModel = '') {
632632
const toolResultMsg = { role: 'tool', tool_call_id: tc.id, content: result }
633633
loopMessages.push(toolResultMsg)
634634

635+
// Check for MCP App UI
636+
let appUI = null
637+
if (options.getToolAppUI) {
638+
let parsedArgs
639+
try {
640+
parsedArgs = typeof tc.function.arguments === 'string'
641+
? JSON.parse(tc.function.arguments) : tc.function.arguments
642+
} catch (_) { parsedArgs = {} }
643+
appUI = await options.getToolAppUI(tc.function.name, parsedArgs, result)
644+
}
645+
635646
// Show result in UI
636647
newMessages.push({
637648
role: 'tool_result',
638649
content: JSON.stringify({ type: 'tool_result', name: tc.function.name, result }, null, 2),
639650
expanded: false,
651+
appUI,
640652
})
641-
currentToolCalls.push({ type: 'tool_result', name: tc.function.name, result })
653+
currentToolCalls.push({ type: 'tool_result', name: tc.function.name, result, appUI })
642654
setStreamingToolCalls([...currentToolCalls.filter(Boolean)])
643655
}
644656

core/http/react-ui/src/hooks/useMCPClient.js

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import { useState, useRef, useCallback } from 'react'
22
import { Client } from '@modelcontextprotocol/sdk/client/index.js'
33
import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js'
44
import { SSEClientTransport } from '@modelcontextprotocol/sdk/client/sse.js'
5+
import { getToolUiResourceUri, isToolVisibilityAppOnly } from '@modelcontextprotocol/ext-apps/app-bridge'
56
import { API_CONFIG } from '../utils/config'
67

78
function buildProxyUrl(targetUrl, useProxy = true) {
@@ -98,6 +99,7 @@ export function useMCPClient() {
9899
const tools = []
99100
for (const [, conn] of connectionsRef.current) {
100101
for (const tool of conn.tools) {
102+
if (isToolVisibilityAppOnly(tool)) continue
101103
tools.push({
102104
type: 'function',
103105
function: {
@@ -163,6 +165,51 @@ export function useMCPClient() {
163165
return result
164166
}, [])
165167

168+
const findToolAndConnection = useCallback((toolName) => {
169+
const serverId = toolIndexRef.current.get(toolName)
170+
if (!serverId) return null
171+
const conn = connectionsRef.current.get(serverId)
172+
if (!conn) return null
173+
const tool = conn.tools.find(t => t.name === toolName)
174+
if (!tool) return null
175+
return { tool, conn }
176+
}, [])
177+
178+
const hasAppUI = useCallback((toolName) => {
179+
const found = findToolAndConnection(toolName)
180+
if (!found) return false
181+
return !!getToolUiResourceUri(found.tool)
182+
}, [findToolAndConnection])
183+
184+
const getAppResource = useCallback(async (toolName) => {
185+
const found = findToolAndConnection(toolName)
186+
if (!found) return null
187+
const uri = getToolUiResourceUri(found.tool)
188+
if (!uri) return null
189+
try {
190+
const res = await found.conn.client.readResource({ uri })
191+
const htmlContent = res.contents?.[0]
192+
if (!htmlContent) return null
193+
return {
194+
html: htmlContent.text || '',
195+
meta: found.tool._meta?.ui || {},
196+
}
197+
} catch (err) {
198+
console.warn('Failed to fetch MCP app resource:', err)
199+
return null
200+
}
201+
}, [findToolAndConnection])
202+
203+
const getClientForTool = useCallback((toolName) => {
204+
const found = findToolAndConnection(toolName)
205+
return found ? found.conn.client : null
206+
}, [findToolAndConnection])
207+
208+
const getToolDefinition = useCallback((toolName) => {
209+
const found = findToolAndConnection(toolName)
210+
return found ? found.tool : null
211+
}, [findToolAndConnection])
212+
166213
return {
167214
connect,
168215
disconnect,
@@ -172,6 +219,10 @@ export function useMCPClient() {
172219
executeTool,
173220
connectionStatuses,
174221
getConnectedTools,
222+
hasAppUI,
223+
getAppResource,
224+
getClientForTool,
225+
getToolDefinition,
175226
}
176227
}
177228

0 commit comments

Comments
 (0)