waypoi 0.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (260) hide show
  1. package/.github/instructions/ui.instructions.md +42 -0
  2. package/.github/workflows/ci.yml +35 -0
  3. package/.github/workflows/publish.yml +71 -0
  4. package/.github/workflows/release.yml +48 -0
  5. package/.playwright-mcp/console-2026-04-04T01-41-10-746Z.log +2 -0
  6. package/.playwright-mcp/console-2026-04-04T01-41-28-799Z.log +3 -0
  7. package/.playwright-mcp/console-2026-04-05T02-26-51-909Z.log +76 -0
  8. package/.playwright-mcp/page-2026-04-04T01-41-10-816Z.yml +1 -0
  9. package/.playwright-mcp/page-2026-04-04T01-41-29-141Z.yml +77 -0
  10. package/.playwright-mcp/page-2026-04-04T01-41-42-633Z.yml +190 -0
  11. package/.playwright-mcp/page-2026-04-04T01-42-03-929Z.yml +262 -0
  12. package/.playwright-mcp/page-2026-04-04T02-12-54-813Z.yml +6 -0
  13. package/.playwright-mcp/page-2026-04-04T02-14-58-600Z.yml +190 -0
  14. package/.playwright-mcp/page-2026-04-04T02-15-03-923Z.yml +190 -0
  15. package/.playwright-mcp/page-2026-04-04T02-15-07-426Z.yml +190 -0
  16. package/.playwright-mcp/page-2026-04-04T02-15-25-729Z.yml +262 -0
  17. package/.playwright-mcp/page-2026-04-04T02-16-22-984Z.yml +262 -0
  18. package/.playwright-mcp/page-2026-04-04T02-17-00-599Z.yml +190 -0
  19. package/.playwright-mcp/page-2026-04-04T02-17-50-874Z.yml +190 -0
  20. package/.playwright-mcp/page-2026-04-05T02-26-55-570Z.yml +6 -0
  21. package/AGENTS.md +48 -0
  22. package/CHANGELOG.md +131 -0
  23. package/README.md +552 -0
  24. package/assets/agent-mode.png +0 -0
  25. package/assets/categorize.png +0 -0
  26. package/assets/dashboard.png +0 -0
  27. package/assets/endpoint-proxy.png +0 -0
  28. package/assets/icon.png +0 -0
  29. package/assets/mcp-generate-image.png +0 -0
  30. package/assets/mcp-understand-image.png +0 -0
  31. package/assets/peek-token-flow.png +0 -0
  32. package/assets/playground.png +0 -0
  33. package/assets/sankey.png +0 -0
  34. package/cli/index.ts +2805 -0
  35. package/cli/legacyRewrite.ts +108 -0
  36. package/cli/modelRef.ts +24 -0
  37. package/dist/cli/index.js +2536 -0
  38. package/dist/cli/legacyRewrite.js +92 -0
  39. package/dist/cli/modelRef.js +20 -0
  40. package/dist/src/benchmark/artifacts.js +131 -0
  41. package/dist/src/benchmark/capabilityClassifier.js +81 -0
  42. package/dist/src/benchmark/capabilityStore.js +144 -0
  43. package/dist/src/benchmark/config.js +238 -0
  44. package/dist/src/benchmark/gates.js +118 -0
  45. package/dist/src/benchmark/jobs.js +252 -0
  46. package/dist/src/benchmark/runner.js +1847 -0
  47. package/dist/src/benchmark/schema.js +353 -0
  48. package/dist/src/benchmark/suites.js +314 -0
  49. package/dist/src/benchmark/tinyQaDataset.js +422 -0
  50. package/dist/src/benchmark/types.js +25 -0
  51. package/dist/src/config.js +47 -0
  52. package/dist/src/index.js +178 -0
  53. package/dist/src/mcp/client.js +215 -0
  54. package/dist/src/mcp/discovery.js +226 -0
  55. package/dist/src/mcp/policy.js +65 -0
  56. package/dist/src/mcp/registry.js +129 -0
  57. package/dist/src/mcp/service.js +460 -0
  58. package/dist/src/middleware/auth.js +179 -0
  59. package/dist/src/middleware/requestCapture.js +192 -0
  60. package/dist/src/middleware/requestStats.js +118 -0
  61. package/dist/src/pools/builder.js +132 -0
  62. package/dist/src/pools/repository.js +69 -0
  63. package/dist/src/pools/scheduler.js +360 -0
  64. package/dist/src/pools/types.js +2 -0
  65. package/dist/src/protocols/adapters/dashscope.js +267 -0
  66. package/dist/src/protocols/adapters/inferenceV2.js +346 -0
  67. package/dist/src/protocols/adapters/openai.js +27 -0
  68. package/dist/src/protocols/registry.js +99 -0
  69. package/dist/src/protocols/types.js +2 -0
  70. package/dist/src/providers/health.js +153 -0
  71. package/dist/src/providers/importer.js +289 -0
  72. package/dist/src/providers/modelRegistry.js +313 -0
  73. package/dist/src/providers/repository.js +361 -0
  74. package/dist/src/providers/types.js +2 -0
  75. package/dist/src/routes/admin.js +531 -0
  76. package/dist/src/routes/audio.js +295 -0
  77. package/dist/src/routes/chat.js +240 -0
  78. package/dist/src/routes/embeddings.js +157 -0
  79. package/dist/src/routes/images.js +288 -0
  80. package/dist/src/routes/mcp.js +256 -0
  81. package/dist/src/routes/mcpService.js +100 -0
  82. package/dist/src/routes/models.js +48 -0
  83. package/dist/src/routes/responses.js +711 -0
  84. package/dist/src/routes/sessions.js +450 -0
  85. package/dist/src/routes/stats.js +270 -0
  86. package/dist/src/routes/ui.js +97 -0
  87. package/dist/src/routes/videos.js +107 -0
  88. package/dist/src/routing/router.js +338 -0
  89. package/dist/src/services/imageGeneration.js +280 -0
  90. package/dist/src/services/imageUnderstanding.js +352 -0
  91. package/dist/src/services/videoGeneration.js +79 -0
  92. package/dist/src/storage/captureRepository.js +1591 -0
  93. package/dist/src/storage/files.js +157 -0
  94. package/dist/src/storage/imageCache.js +346 -0
  95. package/dist/src/storage/repositories.js +388 -0
  96. package/dist/src/storage/sessionRepository.js +370 -0
  97. package/dist/src/storage/statsRepository.js +204 -0
  98. package/dist/src/transport/httpClient.js +126 -0
  99. package/dist/src/types.js +2 -0
  100. package/dist/src/utils/messageMedia.js +285 -0
  101. package/dist/src/utils/modelCapabilities.js +108 -0
  102. package/dist/src/utils/modelDiscovery.js +170 -0
  103. package/dist/src/version.js +5 -0
  104. package/dist/src/workers/captureRetention.js +25 -0
  105. package/dist/src/workers/configWatcher.js +91 -0
  106. package/dist/src/workers/healthChecker.js +21 -0
  107. package/dist/src/workers/statsRotation.js +41 -0
  108. package/docs/LLM/output_schema.md +312 -0
  109. package/docs/benchmark.md +208 -0
  110. package/docs/mcp-guidelines.md +125 -0
  111. package/docs/mcp-service.md +178 -0
  112. package/docs/opencode.md +86 -0
  113. package/docs/providers.md +79 -0
  114. package/examples/benchmark.config.yaml +28 -0
  115. package/examples/providers/alibaba-dashscope.yaml +88 -0
  116. package/examples/providers/alibaba-llm.yaml +64 -0
  117. package/examples/providers/alibaba-registry.yaml +7 -0
  118. package/examples/providers/inference-v2-ray.yaml +29 -0
  119. package/examples/scenarios/assets/omni-call-sample.wav +0 -0
  120. package/examples/scenarios/custom.jsonl +5 -0
  121. package/examples/scenarios/custom.yaml +40 -0
  122. package/model-form-v2.png +0 -0
  123. package/package.json +66 -0
  124. package/provider-form-v2.png +0 -0
  125. package/provider-form.png +0 -0
  126. package/scripts/manual-test.sh +11 -0
  127. package/scripts/version-from-git.js +23 -0
  128. package/src/benchmark/artifacts.ts +149 -0
  129. package/src/benchmark/capabilityClassifier.ts +99 -0
  130. package/src/benchmark/capabilityStore.ts +174 -0
  131. package/src/benchmark/config.ts +337 -0
  132. package/src/benchmark/gates.ts +164 -0
  133. package/src/benchmark/jobs.ts +312 -0
  134. package/src/benchmark/runner.ts +2519 -0
  135. package/src/benchmark/schema.ts +443 -0
  136. package/src/benchmark/suites.ts +323 -0
  137. package/src/benchmark/tinyQaDataset.ts +428 -0
  138. package/src/benchmark/types.ts +442 -0
  139. package/src/config.ts +44 -0
  140. package/src/index.ts +195 -0
  141. package/src/mcp/client.ts +305 -0
  142. package/src/mcp/discovery.ts +266 -0
  143. package/src/mcp/policy.ts +105 -0
  144. package/src/mcp/registry.ts +164 -0
  145. package/src/mcp/service.ts +611 -0
  146. package/src/middleware/auth.ts +251 -0
  147. package/src/middleware/requestCapture.ts +245 -0
  148. package/src/middleware/requestStats.ts +163 -0
  149. package/src/pools/builder.ts +159 -0
  150. package/src/pools/repository.ts +71 -0
  151. package/src/pools/scheduler.ts +425 -0
  152. package/src/pools/types.ts +117 -0
  153. package/src/protocols/adapters/dashscope.ts +335 -0
  154. package/src/protocols/adapters/inferenceV2.ts +428 -0
  155. package/src/protocols/adapters/openai.ts +32 -0
  156. package/src/protocols/registry.ts +117 -0
  157. package/src/protocols/types.ts +81 -0
  158. package/src/providers/health.ts +207 -0
  159. package/src/providers/importer.ts +402 -0
  160. package/src/providers/modelRegistry.ts +415 -0
  161. package/src/providers/repository.ts +439 -0
  162. package/src/providers/types.ts +113 -0
  163. package/src/routes/admin.ts +666 -0
  164. package/src/routes/audio.ts +372 -0
  165. package/src/routes/chat.ts +301 -0
  166. package/src/routes/embeddings.ts +197 -0
  167. package/src/routes/images.ts +356 -0
  168. package/src/routes/mcp.ts +320 -0
  169. package/src/routes/mcpService.ts +114 -0
  170. package/src/routes/models.ts +50 -0
  171. package/src/routes/responses.ts +872 -0
  172. package/src/routes/sessions.ts +558 -0
  173. package/src/routes/stats.ts +312 -0
  174. package/src/routes/ui.ts +96 -0
  175. package/src/routes/videos.ts +132 -0
  176. package/src/routing/router.ts +501 -0
  177. package/src/services/imageGeneration.ts +396 -0
  178. package/src/services/imageUnderstanding.ts +449 -0
  179. package/src/services/videoGeneration.ts +127 -0
  180. package/src/storage/captureRepository.ts +1835 -0
  181. package/src/storage/files.ts +178 -0
  182. package/src/storage/imageCache.ts +405 -0
  183. package/src/storage/repositories.ts +494 -0
  184. package/src/storage/sessionRepository.ts +419 -0
  185. package/src/storage/statsRepository.ts +238 -0
  186. package/src/transport/httpClient.ts +145 -0
  187. package/src/types.ts +322 -0
  188. package/src/utils/messageMedia.ts +293 -0
  189. package/src/utils/modelCapabilities.ts +161 -0
  190. package/src/utils/modelDiscovery.ts +203 -0
  191. package/src/workers/captureRetention.ts +25 -0
  192. package/src/workers/configWatcher.ts +115 -0
  193. package/src/workers/healthChecker.ts +22 -0
  194. package/src/workers/statsRotation.ts +49 -0
  195. package/tests/benchmarkAdminRoutes.test.ts +82 -0
  196. package/tests/benchmarkBasics.test.ts +116 -0
  197. package/tests/captureAdminRoutes.test.ts +420 -0
  198. package/tests/captureRepository.test.ts +797 -0
  199. package/tests/cliLegacyRewrite.test.ts +45 -0
  200. package/tests/imageGeneration.service.test.ts +107 -0
  201. package/tests/imageUnderstanding.service.test.ts +123 -0
  202. package/tests/mcpPolicy.test.ts +105 -0
  203. package/tests/mcpService.test.ts +1245 -0
  204. package/tests/modelRef.test.ts +23 -0
  205. package/tests/modelsRoutes.test.ts +154 -0
  206. package/tests/sessionMediaCache.test.ts +167 -0
  207. package/tests/statsRoutes.test.ts +323 -0
  208. package/tsconfig.json +15 -0
  209. package/ui/index.html +16 -0
  210. package/ui/package-lock.json +8521 -0
  211. package/ui/package.json +52 -0
  212. package/ui/postcss.config.js +6 -0
  213. package/ui/public/assets/apple-touch-icon.png +0 -0
  214. package/ui/public/assets/favicon-16.png +0 -0
  215. package/ui/public/assets/favicon-32.png +0 -0
  216. package/ui/public/assets/icon-192.png +0 -0
  217. package/ui/public/assets/icon-512.png +0 -0
  218. package/ui/src/App.tsx +27 -0
  219. package/ui/src/api/client.ts +1503 -0
  220. package/ui/src/components/EndpointUsageGuide.tsx +361 -0
  221. package/ui/src/components/Layout.tsx +124 -0
  222. package/ui/src/components/MessageContent.tsx +365 -0
  223. package/ui/src/components/ToolCallMessage.tsx +179 -0
  224. package/ui/src/components/ToolPicker.tsx +442 -0
  225. package/ui/src/components/messageContentParser.test.ts +41 -0
  226. package/ui/src/components/messageContentParser.ts +73 -0
  227. package/ui/src/components/thinkingPreview.test.ts +27 -0
  228. package/ui/src/components/thinkingPreview.ts +15 -0
  229. package/ui/src/components/toMermaidSankey.test.ts +78 -0
  230. package/ui/src/components/toMermaidSankey.ts +56 -0
  231. package/ui/src/components/ui/button.tsx +58 -0
  232. package/ui/src/components/ui/input.tsx +21 -0
  233. package/ui/src/components/ui/textarea.tsx +21 -0
  234. package/ui/src/lib/utils.ts +6 -0
  235. package/ui/src/main.tsx +9 -0
  236. package/ui/src/pages/AgentPlayground.tsx +2010 -0
  237. package/ui/src/pages/Benchmark.tsx +988 -0
  238. package/ui/src/pages/Dashboard.tsx +581 -0
  239. package/ui/src/pages/Peek.tsx +962 -0
  240. package/ui/src/pages/Settings.tsx +2013 -0
  241. package/ui/src/pages/agentPlaygroundPayload.test.ts +109 -0
  242. package/ui/src/pages/agentPlaygroundPayload.ts +97 -0
  243. package/ui/src/pages/agentThinkingContent.test.ts +50 -0
  244. package/ui/src/pages/agentThinkingContent.ts +57 -0
  245. package/ui/src/pages/dashboardTokenUsage.test.ts +66 -0
  246. package/ui/src/pages/dashboardTokenUsage.ts +36 -0
  247. package/ui/src/pages/imageUpload.test.ts +39 -0
  248. package/ui/src/pages/imageUpload.ts +71 -0
  249. package/ui/src/pages/peekFilters.test.ts +29 -0
  250. package/ui/src/pages/peekFilters.ts +13 -0
  251. package/ui/src/pages/peekMedia.test.ts +58 -0
  252. package/ui/src/pages/peekMedia.ts +148 -0
  253. package/ui/src/pages/sessionAutoTitle.test.ts +128 -0
  254. package/ui/src/pages/sessionAutoTitle.ts +106 -0
  255. package/ui/src/stores/settings.ts +58 -0
  256. package/ui/src/styles/globals.css +223 -0
  257. package/ui/src/vite-env.d.ts +8 -0
  258. package/ui/tailwind.config.js +106 -0
  259. package/ui/tsconfig.json +32 -0
  260. package/ui/vite.config.ts +37 -0
@@ -0,0 +1,2010 @@
1
+ import { useState, useRef, useEffect, useCallback } from 'react'
2
+ import {
3
+ Send, ImagePlus, Loader2, Bot, User, Sparkles, Plus, Trash2,
4
+ MessageSquare, ChevronRight, X, Image as ImageIcon, Wrench,
5
+ ToggleLeft, ToggleRight, StopCircle, Mic, PhoneCall, PhoneOff
6
+ } from 'lucide-react'
7
+ import { Button } from '@/components/ui/button'
8
+ import { Textarea } from '@/components/ui/textarea'
9
+ import { ToolPicker } from '@/components/ToolPicker'
10
+ import { ToolCallMessage, type ToolCall } from '@/components/ToolCallMessage'
11
+ import { MessageContent } from '@/components/MessageContent'
12
+ import { cn } from '@/lib/utils'
13
+ import {
14
+ streamChatCompletion,
15
+ createChatCompletionRaw,
16
+ listModels,
17
+ listSessions,
18
+ getSession,
19
+ createSession,
20
+ deleteSession,
21
+ addMessageToSession,
22
+ autoTitleSession,
23
+ listMcpTools,
24
+ executeMcpTool,
25
+ BUILTIN_SERVER_ID,
26
+ generateImage,
27
+ storeImage as cacheImage,
28
+ storeMedia,
29
+ normalizeContentMedia,
30
+ normalizeSessionMessageMedia,
31
+ resolveMediaUrl,
32
+ ApiError,
33
+ type ChatMessage as ApiChatMessage,
34
+ type SessionListItem,
35
+ type McpTool,
36
+ type Model,
37
+ } from '@/api/client'
38
+ import { loadSettings, IMAGE_SIZE_OPTIONS, type ImageSize } from '@/stores/settings'
39
+ import {
40
+ buildUserPayload,
41
+ findNonDataImageUrls,
42
+ toApiMessage,
43
+ } from './agentPlaygroundPayload'
44
+ import {
45
+ applyAutoTitleToSessions,
46
+ createDeferredAutoTitleCandidate,
47
+ flushDeferredAutoTitle,
48
+ } from './sessionAutoTitle'
49
+ import {
50
+ applyThinkingChunk,
51
+ createThinkingStreamState,
52
+ toDisplayContent,
53
+ toFinalContent,
54
+ } from './agentThinkingContent'
55
+ import { compressImageFileForUpload, fileToDataUrl } from './imageUpload'
56
+
57
+ // Content can be a string or array of content parts (multimodal)
58
+ type ContentPart =
59
+ | { type: 'text'; text: string }
60
+ | { type: 'image_url'; image_url: { url: string } }
61
+ | { type: 'input_audio'; input_audio: { url?: string; data?: string; format?: string } }
62
+ | { type: 'audio'; audio: { url?: string; data?: string; format?: string } }
63
+
64
+ interface Message {
65
+ id: string
66
+ role: 'user' | 'assistant' | 'system' | 'tool'
67
+ content: string | ContentPart[] | null
68
+ images?: string[]
69
+ requestImages?: string[]
70
+ toolCalls?: ToolCall[]
71
+ toolCallId?: string
72
+ model?: string
73
+ createdAt: Date
74
+ }
75
+
76
+ // Helper to extract text from content (string or multimodal array)
77
+ function getTextContent(content: string | ContentPart[] | null): string {
78
+ if (!content) return ''
79
+ if (typeof content === 'string') return content
80
+ return content
81
+ .filter((p): p is { type: 'text'; text: string } => p.type === 'text')
82
+ .map(p => p.text)
83
+ .join('')
84
+ }
85
+
86
+ // Helper to extract images from multimodal content
87
+ function getImageUrls(content: string | ContentPart[] | null): string[] {
88
+ if (!content || typeof content === 'string') return []
89
+ return content
90
+ .filter((p): p is { type: 'image_url'; image_url: { url: string } } => p.type === 'image_url')
91
+ .map(p => resolveMediaUrl(p.image_url.url))
92
+ }
93
+
94
+ function getAudioUrls(content: string | ContentPart[] | null): string[] {
95
+ if (!content || typeof content === 'string') return []
96
+ const urls: string[] = []
97
+ for (const part of content) {
98
+ if (part.type === 'audio' && part.audio?.url) {
99
+ urls.push(resolveMediaUrl(part.audio.url))
100
+ }
101
+ if (part.type === 'input_audio' && part.input_audio?.url) {
102
+ urls.push(resolveMediaUrl(part.input_audio.url))
103
+ }
104
+ }
105
+ return urls
106
+ }
107
+
108
+ function formatModelTag(model: Model): string {
109
+ const caps = model.capabilities
110
+ if (caps && caps.input.length > 0 && caps.output.length > 0) {
111
+ return `[${caps.input.join('+')}->${caps.output.join('+')}]`
112
+ }
113
+ if (model.endpoint_type && model.endpoint_type !== 'llm') {
114
+ return `[${model.endpoint_type}]`
115
+ }
116
+ return ''
117
+ }
118
+
119
+ function modelHealthStatus(model: Model): 'up' | 'down' | 'unknown' {
120
+ return model.waypoi_health?.status ?? 'unknown'
121
+ }
122
+
123
+ function isModelSelectable(model: Model): boolean {
124
+ return modelHealthStatus(model) !== 'down'
125
+ }
126
+
127
+ function firstSelectableModelId(models: Model[]): string {
128
+ return models.find(isModelSelectable)?.id ?? ''
129
+ }
130
+
131
+ // Maximum tool iterations per user message to prevent infinite loops
132
+ const MAX_TOOL_ITERATIONS = 10
133
+
134
+ type GenerationParamsDraft = {
135
+ temperature: string
136
+ topP: string
137
+ maxTokens: string
138
+ presencePenalty: string
139
+ frequencyPenalty: string
140
+ seed: string
141
+ stop: string
142
+ }
143
+
144
+ type GenerationParamsPayload = {
145
+ temperature?: number
146
+ top_p?: number
147
+ max_tokens?: number
148
+ presence_penalty?: number
149
+ frequency_penalty?: number
150
+ seed?: number
151
+ stop?: string | string[]
152
+ }
153
+
154
+ type NumericGenerationParamKey =
155
+ | 'temperature'
156
+ | 'top_p'
157
+ | 'max_tokens'
158
+ | 'presence_penalty'
159
+ | 'frequency_penalty'
160
+ | 'seed'
161
+
162
+ function parseGenerationParams(draft: GenerationParamsDraft): { payload: GenerationParamsPayload; error?: string } {
163
+ const payload: GenerationParamsPayload = {}
164
+ const parseNumber = (
165
+ raw: string,
166
+ label: string,
167
+ key: NumericGenerationParamKey,
168
+ opts?: { min?: number; max?: number; integer?: boolean }
169
+ ): string | null => {
170
+ const trimmed = raw.trim()
171
+ if (!trimmed) return null
172
+ const parsed = Number(trimmed)
173
+ if (!Number.isFinite(parsed)) {
174
+ return `${label} must be a valid number.`
175
+ }
176
+ if (opts?.integer && !Number.isInteger(parsed)) {
177
+ return `${label} must be an integer.`
178
+ }
179
+ if (typeof opts?.min === 'number' && parsed < opts.min) {
180
+ return `${label} must be >= ${opts.min}.`
181
+ }
182
+ if (typeof opts?.max === 'number' && parsed > opts.max) {
183
+ return `${label} must be <= ${opts.max}.`
184
+ }
185
+ payload[key] = parsed
186
+ return null
187
+ }
188
+
189
+ const errors = [
190
+ parseNumber(draft.temperature, 'Temperature', 'temperature'),
191
+ parseNumber(draft.topP, 'Top P', 'top_p', { min: 0, max: 1 }),
192
+ parseNumber(draft.maxTokens, 'Max Tokens', 'max_tokens', { min: 1, integer: true }),
193
+ parseNumber(draft.presencePenalty, 'Presence Penalty', 'presence_penalty', { min: -2, max: 2 }),
194
+ parseNumber(draft.frequencyPenalty, 'Frequency Penalty', 'frequency_penalty', { min: -2, max: 2 }),
195
+ parseNumber(draft.seed, 'Seed', 'seed', { min: 0, integer: true }),
196
+ ].filter((error): error is string => Boolean(error))
197
+
198
+ const stops = draft.stop
199
+ .split(',')
200
+ .map((value) => value.trim())
201
+ .filter(Boolean)
202
+ if (stops.length === 1) payload.stop = stops[0]
203
+ if (stops.length > 1) payload.stop = stops
204
+
205
+ return { payload, error: errors[0] }
206
+ }
207
+
208
+ export function AgentPlayground() {
209
+ const MAX_INPUT_LINES = 10
210
+ // Session state
211
+ const [sessions, setSessions] = useState<SessionListItem[]>([])
212
+ const [activeSessionId, setActiveSessionId] = useState<string | null>(null)
213
+ const [activeSessionStorageVersion, setActiveSessionStorageVersion] = useState<number>(2)
214
+ const [sessionName, setSessionName] = useState('')
215
+ const [titleGenerationSessionId, setTitleGenerationSessionId] = useState<string | null>(null)
216
+ const [sidebarCollapsed, setSidebarCollapsed] = useState(false)
217
+
218
+ // Chat state
219
+ const [messages, setMessages] = useState<Message[]>([])
220
+ const [input, setInput] = useState('')
221
+ const [isLoading, setIsLoading] = useState(false)
222
+ const [selectedModel, setSelectedModel] = useState<string>('')
223
+ const [models, setModels] = useState<Model[]>([])
224
+
225
+ const selectedModelSupportsImageOutput = (): boolean => {
226
+ const model = models.find(m => m.id === selectedModel)
227
+ if (!model) return false
228
+ if (model.capabilities?.output?.includes('image')) return true
229
+ return (model.endpoint_type ?? 'llm') === 'diffusion'
230
+ }
231
+
232
+ const selectedModelSupportsCall = (): boolean => {
233
+ const model = models.find(m => m.id === selectedModel)
234
+ if (!model?.capabilities) return false
235
+ return model.capabilities.input.includes('audio') && model.capabilities.output.includes('audio')
236
+ }
237
+
238
+ const modelSupportsCall = selectedModelSupportsCall()
239
+ const selectedModelConfig = models.find((model) => model.id === selectedModel)
240
+ const selectedModelIsSelectable = selectedModelConfig ? isModelSelectable(selectedModelConfig) : false
241
+
242
+ // Image input state
243
+ const [pendingImages, setPendingImages] = useState<string[]>([])
244
+ const [pendingAudio, setPendingAudio] = useState<string | null>(null)
245
+ const [pendingAudioMimeType, setPendingAudioMimeType] = useState<string | undefined>(undefined)
246
+ const [isDragging, setIsDragging] = useState(false)
247
+ const [callModeEnabled, setCallModeEnabled] = useState(false)
248
+ const [callStatus, setCallStatus] = useState<'idle' | 'recording' | 'sending' | 'playing'>('idle')
249
+ const [callError, setCallError] = useState<string | null>(null)
250
+ const [showGenerationParams, setShowGenerationParams] = useState(false)
251
+ const [generationParams, setGenerationParams] = useState<GenerationParamsDraft>({
252
+ temperature: '',
253
+ topP: '',
254
+ maxTokens: '',
255
+ presencePenalty: '',
256
+ frequencyPenalty: '',
257
+ seed: '',
258
+ stop: '',
259
+ })
260
+
261
+ // Image generation settings
262
+ const [imageSize, setImageSize] = useState<ImageSize>(() => loadSettings().defaultImageSize)
263
+
264
+ // Agentic mode state
265
+ const [agentModeEnabled, setAgentModeEnabled] = useState(false)
266
+ const [showToolPicker, setShowToolPicker] = useState(false)
267
+ const [selectedTools, setSelectedTools] = useState<Set<string>>(new Set())
268
+ const [availableTools, setAvailableTools] = useState<McpTool[]>([])
269
+ const [currentIteration, setCurrentIteration] = useState(0)
270
+ const [isExecutingTools, setIsExecutingTools] = useState(false)
271
+
272
+ const messagesEndRef = useRef<HTMLDivElement>(null)
273
+ const messagesContainerRef = useRef<HTMLDivElement>(null)
274
+ const abortControllerRef = useRef<AbortController | null>(null)
275
+ const stopAgentRef = useRef(false)
276
+ const fileInputRef = useRef<HTMLInputElement>(null)
277
+ const isStreamingRef = useRef(false)
278
+ const mediaRecorderRef = useRef<MediaRecorder | null>(null)
279
+ const mediaStreamRef = useRef<MediaStream | null>(null)
280
+ const recordingTimerRef = useRef<number | null>(null)
281
+ const callAudioRef = useRef<HTMLAudioElement | null>(null)
282
+ const inputRef = useRef<HTMLTextAreaElement>(null)
283
+ const pendingAutoTitleRef = useRef<{ sessionId: string; seedText: string } | null>(null)
284
+
285
+ const resizeInput = useCallback(() => {
286
+ const textarea = inputRef.current
287
+ if (!textarea) return
288
+
289
+ textarea.style.height = 'auto'
290
+ const lineHeight = Number.parseFloat(window.getComputedStyle(textarea).lineHeight) || 20
291
+ const maxHeight = lineHeight * MAX_INPUT_LINES
292
+ const nextHeight = Math.min(textarea.scrollHeight, maxHeight)
293
+ textarea.style.height = `${nextHeight}px`
294
+ textarea.style.overflowY = textarea.scrollHeight > maxHeight ? 'auto' : 'hidden'
295
+ }, [MAX_INPUT_LINES])
296
+
297
+ // Load models on mount
298
+ useEffect(() => {
299
+ async function loadModels() {
300
+ try {
301
+ const response = await listModels()
302
+ setModels(response.data)
303
+ if (response.data.length > 0 && !selectedModel) {
304
+ setSelectedModel(firstSelectableModelId(response.data))
305
+ }
306
+ } catch (error) {
307
+ console.error('Failed to load models:', error)
308
+ }
309
+ }
310
+ loadModels()
311
+ }, [selectedModel])
312
+
313
+ // Ensure selected model stays selectable after health refreshes/session restore.
314
+ useEffect(() => {
315
+ if (models.length === 0) return
316
+ const current = models.find((model) => model.id === selectedModel)
317
+ if (!current || !isModelSelectable(current)) {
318
+ setSelectedModel(firstSelectableModelId(models))
319
+ }
320
+ }, [models, selectedModel])
321
+
322
+ // Load sessions on mount
323
+ useEffect(() => {
324
+ async function loadSessions() {
325
+ try {
326
+ const response = await listSessions()
327
+ setSessions(response.data)
328
+ } catch (error) {
329
+ console.error('Failed to load sessions:', error)
330
+ }
331
+ }
332
+ loadSessions()
333
+ }, [])
334
+
335
+ // Load tools when agent mode is enabled
336
+ useEffect(() => {
337
+ if (agentModeEnabled) {
338
+ loadTools()
339
+ }
340
+ }, [agentModeEnabled])
341
+
342
+ useEffect(() => {
343
+ resizeInput()
344
+ }, [input, resizeInput])
345
+
346
+ const loadTools = async () => {
347
+ try {
348
+ const response = await listMcpTools()
349
+ setAvailableTools(response.data)
350
+ // Auto-select built-in tools the first time agent mode is enabled
351
+ setSelectedTools((prev) => {
352
+ if (prev.size > 0) return prev
353
+ const builtinNames = response.data
354
+ .filter((t) => t.serverId === BUILTIN_SERVER_ID)
355
+ .map((t) => t.name)
356
+ return builtinNames.length > 0 ? new Set(builtinNames) : prev
357
+ })
358
+ } catch (error) {
359
+ console.error('Failed to load tools:', error)
360
+ }
361
+ }
362
+
363
+ const normalizeMessageForUi = (message: Message): Message => ({
364
+ ...message,
365
+ content: normalizeContentMedia(message.content) as Message['content'],
366
+ images: message.images?.map((value) => resolveMediaUrl(value)),
367
+ })
368
+
369
+ // Auto-scroll to bottom - use instant scroll during streaming to prevent jitter
370
+ useEffect(() => {
371
+ const container = messagesContainerRef.current
372
+ if (!container) return
373
+
374
+ // Use instant scroll during streaming, smooth otherwise
375
+ const behavior = isStreamingRef.current ? 'auto' : 'smooth'
376
+ messagesEndRef.current?.scrollIntoView({ behavior })
377
+ }, [messages])
378
+
379
+ // Load session messages when switching
380
+ const loadSession = useCallback(async (sessionId: string) => {
381
+ try {
382
+ const session = await getSession(sessionId)
383
+ setActiveSessionId(session.id)
384
+ setActiveSessionStorageVersion(session.storageVersion ?? 1)
385
+ setSessionName(session.name)
386
+ if (session.model) setSelectedModel(session.model)
387
+ setMessages(session.messages.map(m => {
388
+ const normalized = normalizeSessionMessageMedia(m)
389
+ return ({
390
+ id: crypto.randomUUID(),
391
+ role: normalized.role as Message['role'],
392
+ content: normalizeContentMedia(normalized.content) as Message['content'],
393
+ images: normalized.images,
394
+ createdAt: new Date(m.createdAt ?? m.timestamp ?? new Date().toISOString()),
395
+ })}))
396
+ } catch (error) {
397
+ console.error('Failed to load session:', error)
398
+ }
399
+ }, [])
400
+
401
+ // Create new session
402
+ const handleNewSession = async () => {
403
+ try {
404
+ const session = await createSession(undefined, selectedModel)
405
+ setSessions(prev => [{
406
+ id: session.id,
407
+ name: session.name,
408
+ model: session.model,
409
+ messageCount: 0,
410
+ createdAt: session.createdAt,
411
+ updatedAt: session.updatedAt,
412
+ }, ...prev])
413
+ setActiveSessionId(session.id)
414
+ setActiveSessionStorageVersion(session.storageVersion ?? 2)
415
+ setSessionName(session.name)
416
+ pendingAutoTitleRef.current = null
417
+ setMessages([])
418
+ } catch (error) {
419
+ console.error('Failed to create session:', error)
420
+ }
421
+ }
422
+
423
+ // Delete session
424
+ const handleDeleteSession = async (sessionId: string) => {
425
+ try {
426
+ await deleteSession(sessionId)
427
+ setSessions(prev => prev.filter(s => s.id !== sessionId))
428
+ if (titleGenerationSessionId === sessionId) {
429
+ setTitleGenerationSessionId(null)
430
+ }
431
+ if (activeSessionId === sessionId) {
432
+ setActiveSessionId(null)
433
+ setActiveSessionStorageVersion(2)
434
+ setMessages([])
435
+ setSessionName('')
436
+ pendingAutoTitleRef.current = null
437
+ }
438
+ } catch (error) {
439
+ console.error('Failed to delete session:', error)
440
+ }
441
+ }
442
+
443
+ // Image handling
444
+ const processImages = useCallback((files: FileList | File[]) => {
445
+ Array.from(files).forEach(async (file) => {
446
+ if (!file.type.startsWith('image/')) return
447
+ try {
448
+ const base64 = await compressImageFileForUpload(file)
449
+ setPendingImages(prev => [...prev, base64])
450
+ } catch (error) {
451
+ console.warn('Failed to compress image, using raw data URL:', error)
452
+ const fallback = await fileToDataUrl(file)
453
+ setPendingImages(prev => [...prev, fallback])
454
+ }
455
+ })
456
+ }, [])
457
+
458
+ const processAudioFile = useCallback((file: File) => {
459
+ if (!file.type.startsWith('audio/')) return
460
+ const reader = new FileReader()
461
+ reader.onload = (e) => {
462
+ const base64 = e.target?.result as string
463
+ setPendingAudio(base64)
464
+ setPendingAudioMimeType(file.type || undefined)
465
+ setCallError(null)
466
+ }
467
+ reader.readAsDataURL(file)
468
+ }, [])
469
+
470
+ const processSelectedFiles = useCallback((files: FileList | File[]) => {
471
+ const entries = Array.from(files)
472
+ const imageFiles = entries.filter((file) => file.type.startsWith('image/'))
473
+ if (imageFiles.length > 0) {
474
+ processImages(imageFiles)
475
+ }
476
+ if (callModeEnabled) {
477
+ const audioFile = entries.find((file) => file.type.startsWith('audio/'))
478
+ if (audioFile) {
479
+ processAudioFile(audioFile)
480
+ }
481
+ }
482
+ }, [callModeEnabled, processAudioFile, processImages])
483
+
484
+ const handleDrop = useCallback((e: React.DragEvent) => {
485
+ e.preventDefault()
486
+ setIsDragging(false)
487
+ if (e.dataTransfer.files.length > 0) {
488
+ processSelectedFiles(e.dataTransfer.files)
489
+ }
490
+ }, [processSelectedFiles])
491
+
492
+ const handlePaste = useCallback((e: React.ClipboardEvent) => {
493
+ const items = e.clipboardData.items
494
+ const imageFiles: File[] = []
495
+ for (let i = 0; i < items.length; i++) {
496
+ if (items[i].type.startsWith('image/')) {
497
+ const file = items[i].getAsFile()
498
+ if (file) imageFiles.push(file)
499
+ } else if (callModeEnabled && items[i].type.startsWith('audio/')) {
500
+ const file = items[i].getAsFile()
501
+ if (file) processAudioFile(file)
502
+ }
503
+ }
504
+ if (imageFiles.length > 0) {
505
+ processImages(imageFiles)
506
+ }
507
+ }, [callModeEnabled, processAudioFile, processImages])
508
+
509
+ const stopPlayback = useCallback(() => {
510
+ if (callAudioRef.current) {
511
+ callAudioRef.current.pause()
512
+ callAudioRef.current.currentTime = 0
513
+ callAudioRef.current = null
514
+ }
515
+ if (callStatus === 'playing') {
516
+ setCallStatus('idle')
517
+ }
518
+ }, [callStatus])
519
+
520
+ const clearRecordingTimer = () => {
521
+ if (recordingTimerRef.current !== null) {
522
+ window.clearTimeout(recordingTimerRef.current)
523
+ recordingTimerRef.current = null
524
+ }
525
+ }
526
+
527
+ const stopMediaStream = () => {
528
+ if (mediaStreamRef.current) {
529
+ mediaStreamRef.current.getTracks().forEach(track => track.stop())
530
+ mediaStreamRef.current = null
531
+ }
532
+ }
533
+
534
+ const stopRecording = useCallback(() => {
535
+ const recorder = mediaRecorderRef.current
536
+ if (!recorder || recorder.state !== 'recording') {
537
+ clearRecordingTimer()
538
+ stopMediaStream()
539
+ setCallStatus('idle')
540
+ return
541
+ }
542
+ recorder.stop()
543
+ clearRecordingTimer()
544
+ }, [])
545
+
546
+ const cancelRecording = useCallback(() => {
547
+ stopRecording()
548
+ setPendingAudio(null)
549
+ setPendingAudioMimeType(undefined)
550
+ setCallError(null)
551
+ }, [stopRecording])
552
+
553
+ const startRecording = useCallback(async () => {
554
+ if (!selectedModelSupportsCall() || isLoading || callStatus === 'sending') {
555
+ return
556
+ }
557
+
558
+ try {
559
+ stopPlayback()
560
+ setCallError(null)
561
+ setCallStatus('recording')
562
+ const stream = await navigator.mediaDevices.getUserMedia({ audio: true })
563
+ mediaStreamRef.current = stream
564
+ let recorder: MediaRecorder
565
+ try {
566
+ recorder = new MediaRecorder(stream, { mimeType: 'audio/webm' })
567
+ } catch {
568
+ recorder = new MediaRecorder(stream)
569
+ }
570
+ mediaRecorderRef.current = recorder
571
+ const chunks: BlobPart[] = []
572
+
573
+ recorder.ondataavailable = (event) => {
574
+ if (event.data.size > 0) {
575
+ chunks.push(event.data)
576
+ }
577
+ }
578
+
579
+ recorder.onstop = async () => {
580
+ stopMediaStream()
581
+ clearRecordingTimer()
582
+ if (chunks.length > 0) {
583
+ const blob = new Blob(chunks, { type: recorder.mimeType || 'audio/webm' })
584
+ const reader = new FileReader()
585
+ reader.onload = () => {
586
+ const result = reader.result as string
587
+ setPendingAudio(result)
588
+ setPendingAudioMimeType(blob.type || 'audio/webm')
589
+ setCallStatus('idle')
590
+ }
591
+ reader.readAsDataURL(blob)
592
+ } else {
593
+ setCallStatus('idle')
594
+ }
595
+ }
596
+
597
+ recorder.start()
598
+ recordingTimerRef.current = window.setTimeout(() => {
599
+ stopRecording()
600
+ }, 60_000)
601
+ } catch (error) {
602
+ console.error('Microphone access failed:', error)
603
+ setCallStatus('idle')
604
+ setCallError('Microphone permission denied or unavailable. You can upload an audio file instead.')
605
+ }
606
+ }, [callStatus, isLoading, modelSupportsCall, stopPlayback, stopRecording])
607
+
608
+ useEffect(() => {
609
+ if (!modelSupportsCall && callModeEnabled) {
610
+ setCallModeEnabled(false)
611
+ setPendingAudio(null)
612
+ setPendingAudioMimeType(undefined)
613
+ setCallStatus('idle')
614
+ setCallError(null)
615
+ }
616
+ }, [callModeEnabled, modelSupportsCall])
617
+
618
+ useEffect(() => {
619
+ return () => {
620
+ clearRecordingTimer()
621
+ stopMediaStream()
622
+ stopPlayback()
623
+ }
624
+ }, [stopPlayback])
625
+
626
+ // Build tools array for the API
627
+ const buildToolsForApi = () => {
628
+ if (!agentModeEnabled || selectedTools.size === 0) return undefined
629
+
630
+ return availableTools
631
+ .filter(t => selectedTools.has(t.name))
632
+ .map(t => ({
633
+ type: 'function' as const,
634
+ function: {
635
+ name: t.name,
636
+ description: t.description || '',
637
+ parameters: t.inputSchema,
638
+ },
639
+ }))
640
+ }
641
+
642
+ // Execute a tool and return result
643
+ const executeToolCall = async (toolCall: ToolCall): Promise<string> => {
644
+ try {
645
+ const args = JSON.parse(toolCall.arguments)
646
+ const result = await executeMcpTool(toolCall.name, args)
647
+ return result.result
648
+ } catch (error) {
649
+ let reason = (error as Error).message
650
+ if (error instanceof ApiError) {
651
+ const body = error.body as { error?: { message?: string } } | undefined
652
+ if (body?.error?.message) {
653
+ reason = body.error.message
654
+ }
655
+ }
656
+ throw new Error(`Tool execution failed: ${reason}`)
657
+ }
658
+ }
659
+
660
+ // Stop the agent loop
661
+ const stopAgent = () => {
662
+ stopAgentRef.current = true
663
+ abortControllerRef.current?.abort()
664
+ }
665
+
666
+ // The main agentic loop
667
+ const runAgentLoop = async (
668
+ conversationHistory: Message[],
669
+ assistantMessageId: string,
670
+ generationPayload: GenerationParamsPayload
671
+ ): Promise<void> => {
672
+ let iteration = 0
673
+ let currentHistory = [...conversationHistory]
674
+
675
+ while (iteration < MAX_TOOL_ITERATIONS && !stopAgentRef.current) {
676
+ setCurrentIteration(iteration + 1)
677
+
678
+ // Build messages for API
679
+ const chatMessages = currentHistory.map(m => {
680
+ if (m.role === 'tool' && m.toolCallId) {
681
+ return {
682
+ role: 'tool' as const,
683
+ tool_call_id: m.toolCallId,
684
+ content: m.content,
685
+ }
686
+ }
687
+ return toApiMessage(m)
688
+ }) as ApiChatMessage[]
689
+ warnIfNonDataImageUrls(chatMessages, 'agent-loop')
690
+
691
+ // Stream the response
692
+ abortControllerRef.current = new AbortController()
693
+ isStreamingRef.current = true
694
+ let fullContent = ''
695
+ let toolCallsData: Array<{
696
+ id: string
697
+ function: { name: string; arguments: string }
698
+ }> = []
699
+
700
+ try {
701
+ for await (const chunk of streamChatCompletionWithTools(
702
+ {
703
+ model: selectedModel,
704
+ messages: chatMessages,
705
+ ...generationPayload,
706
+ tools: buildToolsForApi(),
707
+ tool_choice: selectedTools.size > 0 ? 'auto' : undefined,
708
+ },
709
+ abortControllerRef.current.signal,
710
+ (tc) => { toolCallsData = tc }
711
+ )) {
712
+ if (stopAgentRef.current) break
713
+ fullContent += chunk
714
+ setMessages(prev =>
715
+ prev.map(m =>
716
+ m.id === assistantMessageId
717
+ ? { ...m, content: fullContent }
718
+ : m
719
+ )
720
+ )
721
+ }
722
+ } catch (error) {
723
+ if ((error as Error).name === 'AbortError') return
724
+ throw error
725
+ }
726
+
727
+ // Check if we have tool calls
728
+ if (toolCallsData.length > 0 && !stopAgentRef.current) {
729
+ // Create tool call objects
730
+ const toolCalls: ToolCall[] = toolCallsData.map(tc => ({
731
+ id: tc.id,
732
+ name: tc.function.name,
733
+ arguments: tc.function.arguments,
734
+ status: 'pending' as const,
735
+ }))
736
+
737
+ // Update message with tool calls
738
+ setMessages(prev =>
739
+ prev.map(m =>
740
+ m.id === assistantMessageId
741
+ ? { ...m, content: fullContent || null, toolCalls }
742
+ : m
743
+ )
744
+ )
745
+
746
+ // Execute tools
747
+ setIsExecutingTools(true)
748
+ const toolResults: Message[] = []
749
+
750
+ for (const toolCall of toolCalls) {
751
+ if (stopAgentRef.current) break
752
+
753
+ // Update status to executing
754
+ setMessages(prev =>
755
+ prev.map(m =>
756
+ m.id === assistantMessageId
757
+ ? {
758
+ ...m,
759
+ toolCalls: m.toolCalls?.map(tc =>
760
+ tc.id === toolCall.id ? { ...tc, status: 'executing' as const } : tc
761
+ )
762
+ }
763
+ : m
764
+ )
765
+ )
766
+
767
+ try {
768
+ const result = await executeToolCall(toolCall)
769
+
770
+ // Update status to success
771
+ setMessages(prev =>
772
+ prev.map(m =>
773
+ m.id === assistantMessageId
774
+ ? {
775
+ ...m,
776
+ toolCalls: m.toolCalls?.map(tc =>
777
+ tc.id === toolCall.id
778
+ ? { ...tc, status: 'success' as const, result }
779
+ : tc
780
+ )
781
+ }
782
+ : m
783
+ )
784
+ )
785
+
786
+ // Add tool result message
787
+ toolResults.push({
788
+ id: crypto.randomUUID(),
789
+ role: 'tool',
790
+ content: result,
791
+ toolCallId: toolCall.id,
792
+ createdAt: new Date(),
793
+ })
794
+ } catch (error) {
795
+ const errorMsg = (error as Error).message
796
+
797
+ // Update status to error
798
+ setMessages(prev =>
799
+ prev.map(m =>
800
+ m.id === assistantMessageId
801
+ ? {
802
+ ...m,
803
+ toolCalls: m.toolCalls?.map(tc =>
804
+ tc.id === toolCall.id
805
+ ? { ...tc, status: 'error' as const, error: errorMsg }
806
+ : tc
807
+ )
808
+ }
809
+ : m
810
+ )
811
+ )
812
+
813
+ // Still add a tool result for the error
814
+ toolResults.push({
815
+ id: crypto.randomUUID(),
816
+ role: 'tool',
817
+ content: `Error: ${errorMsg}`,
818
+ toolCallId: toolCall.id,
819
+ createdAt: new Date(),
820
+ })
821
+ }
822
+ }
823
+
824
+ setIsExecutingTools(false)
825
+
826
+ if (stopAgentRef.current) break
827
+
828
+ // Add assistant message with tool calls and tool results to history
829
+ const assistantWithToolCalls: Message = {
830
+ id: assistantMessageId,
831
+ role: 'assistant',
832
+ content: fullContent || null,
833
+ toolCalls,
834
+ createdAt: new Date(),
835
+ }
836
+
837
+ currentHistory = [...currentHistory.slice(0, -1), assistantWithToolCalls, ...toolResults]
838
+
839
+ // Create new assistant message for next iteration
840
+ const newAssistantId = crypto.randomUUID()
841
+ const newAssistantMessage: Message = {
842
+ id: newAssistantId,
843
+ role: 'assistant',
844
+ content: '',
845
+ createdAt: new Date(),
846
+ }
847
+
848
+ setMessages(prev => [...prev, ...toolResults, newAssistantMessage])
849
+ assistantMessageId = newAssistantId
850
+ currentHistory = [...currentHistory, newAssistantMessage]
851
+
852
+ iteration++
853
+ } else {
854
+ // No tool calls, we're done
855
+ break
856
+ }
857
+ }
858
+
859
+ setCurrentIteration(0)
860
+ }
861
+
862
+ const warnIfNonDataImageUrls = (chatMessages: ApiChatMessage[], context: string) => {
863
+ if (!import.meta.env.DEV) return
864
+ const invalidUrls = findNonDataImageUrls(chatMessages)
865
+ if (invalidUrls.length > 0) {
866
+ console.warn(`[AgentPlayground][${context}] Non-data image URLs detected in chat payload`, invalidUrls)
867
+ }
868
+ }
869
+
870
+ const extractAssistantAudio = (message: unknown): { url?: string; data?: string; format?: string } | null => {
871
+ if (!message || typeof message !== 'object') return null
872
+ const msg = message as Record<string, unknown>
873
+ const direct = msg.audio as Record<string, unknown> | undefined
874
+ if (direct && (typeof direct.url === 'string' || typeof direct.data === 'string')) {
875
+ return {
876
+ url: typeof direct.url === 'string' ? direct.url : undefined,
877
+ data: typeof direct.data === 'string' ? direct.data : undefined,
878
+ format: typeof direct.format === 'string' ? direct.format : undefined,
879
+ }
880
+ }
881
+ const content = msg.content
882
+ if (!Array.isArray(content)) return null
883
+ for (const part of content) {
884
+ if (!part || typeof part !== 'object') continue
885
+ const p = part as Record<string, unknown>
886
+ const audioObj = p.audio as Record<string, unknown> | undefined
887
+ if ((p.type === 'audio' || p.type === 'output_audio') && audioObj) {
888
+ return {
889
+ url: typeof audioObj.url === 'string' ? audioObj.url : undefined,
890
+ data: typeof audioObj.data === 'string' ? audioObj.data : undefined,
891
+ format: typeof audioObj.format === 'string' ? audioObj.format : undefined,
892
+ }
893
+ }
894
+ }
895
+ return null
896
+ }
897
+
898
+ const formatToMimeType = (format?: string): string | undefined => {
899
+ if (!format) return undefined
900
+ const lower = format.toLowerCase()
901
+ if (lower === 'wav') return 'audio/wav'
902
+ if (lower === 'mp3' || lower === 'mpeg') return 'audio/mpeg'
903
+ if (lower === 'ogg') return 'audio/ogg'
904
+ if (lower === 'webm') return 'audio/webm'
905
+ if (lower === 'm4a' || lower === 'mp4') return 'audio/mp4'
906
+ return undefined
907
+ }
908
+
909
+ const maybeAutoTitleSession = async (sessionId: string): Promise<void> => {
910
+ await flushDeferredAutoTitle({
911
+ sessionId,
912
+ sessionName,
913
+ model: selectedModel,
914
+ queuedCandidate: pendingAutoTitleRef.current,
915
+ generatingSessionId: titleGenerationSessionId,
916
+ autoTitleSession,
917
+ onGenerationChange: setTitleGenerationSessionId,
918
+ onResolved: (response) => {
919
+ setSessionName((current) => (current === sessionName ? response.name : current))
920
+ setSessions((prev) => applyAutoTitleToSessions(prev, sessionId, response))
921
+ },
922
+ clearQueuedCandidate: () => {
923
+ if (pendingAutoTitleRef.current?.sessionId === sessionId) {
924
+ pendingAutoTitleRef.current = null
925
+ }
926
+ },
927
+ onError: (error) => {
928
+ console.warn('Auto-title skipped:', error)
929
+ },
930
+ })
931
+ }
932
+
933
+ const handleSubmit = async (e: React.FormEvent) => {
934
+ e.preventDefault()
935
+ if (!selectedModelConfig || !isModelSelectable(selectedModelConfig)) {
936
+ setCallError('Selected model is unavailable. Choose a healthy model and try again.')
937
+ return
938
+ }
939
+ const hasText = input.trim().length > 0
940
+ const canSend = callModeEnabled
941
+ ? Boolean(pendingAudio)
942
+ : hasText || pendingImages.length > 0
943
+ if (!canSend || isLoading) return
944
+ const { payload: generationPayload, error: generationParamError } = parseGenerationParams(generationParams)
945
+ if (generationParamError) {
946
+ setCallError(generationParamError)
947
+ return
948
+ }
949
+
950
+ stopAgentRef.current = false
951
+ setCallError(null)
952
+
953
+ const requestImageUrls = pendingImages.length > 0 ? [...pendingImages] : []
954
+ let displayImageRefs = requestImageUrls.length > 0 ? [...requestImageUrls] : undefined
955
+ if ((activeSessionStorageVersion ?? 1) >= 2 && pendingImages.length > 0) {
956
+ try {
957
+ const cached = await Promise.all(
958
+ pendingImages.map((image) => cacheImage(image, selectedModel))
959
+ )
960
+ displayImageRefs = cached.map((item) => item.url)
961
+ } catch (error) {
962
+ console.error('Failed to cache input images, falling back to inline images:', error)
963
+ }
964
+ }
965
+
966
+ let audioRef: string | undefined
967
+ if (pendingAudio) {
968
+ if ((activeSessionStorageVersion ?? 1) >= 2) {
969
+ try {
970
+ const cachedAudio = await storeMedia(pendingAudio, selectedModel, pendingAudioMimeType)
971
+ audioRef = cachedAudio.url
972
+ } catch (error) {
973
+ console.error('Failed to cache input audio, falling back to inline audio:', error)
974
+ audioRef = pendingAudio
975
+ }
976
+ } else {
977
+ audioRef = pendingAudio
978
+ }
979
+ }
980
+
981
+ const payload = buildUserPayload({
982
+ callModeEnabled,
983
+ text: input,
984
+ requestImageUrls,
985
+ displayImageRefs,
986
+ audioRef,
987
+ })
988
+
989
+ const userMessage: Message = {
990
+ id: crypto.randomUUID(),
991
+ role: 'user',
992
+ content: payload.content,
993
+ images: payload.images,
994
+ requestImages: payload.requestImages,
995
+ createdAt: new Date(),
996
+ }
997
+
998
+ setMessages(prev => [...prev, normalizeMessageForUi(userMessage)])
999
+ setInput('')
1000
+ setPendingImages([])
1001
+ setPendingAudio(null)
1002
+ setPendingAudioMimeType(undefined)
1003
+ setIsLoading(true)
1004
+
1005
+ // Save user message to session
1006
+ if (activeSessionId) {
1007
+ try {
1008
+ await addMessageToSession(activeSessionId, {
1009
+ role: 'user',
1010
+ content: userMessage.content,
1011
+ images: userMessage.images,
1012
+ timestamp: userMessage.createdAt.toISOString(),
1013
+ })
1014
+ const textForTitle = getTextContent(userMessage.content)
1015
+ const autoTitleCandidate = createDeferredAutoTitleCandidate(activeSessionId, sessionName, textForTitle)
1016
+ if (autoTitleCandidate) {
1017
+ pendingAutoTitleRef.current = autoTitleCandidate
1018
+ }
1019
+ } catch (error) {
1020
+ console.error('Failed to save user message:', error)
1021
+ }
1022
+ }
1023
+
1024
+ const assistantMessage: Message = {
1025
+ id: crypto.randomUUID(),
1026
+ role: 'assistant',
1027
+ content: '',
1028
+ model: selectedModel,
1029
+ createdAt: new Date(),
1030
+ }
1031
+ setMessages(prev => [...prev, normalizeMessageForUi(assistantMessage)])
1032
+
1033
+ try {
1034
+ if (callModeEnabled) {
1035
+ setCallStatus('sending')
1036
+ const chatMessages = [...messages, userMessage].map(toApiMessage)
1037
+ warnIfNonDataImageUrls(chatMessages, 'call-mode')
1038
+ const response = await createChatCompletionRaw({
1039
+ model: selectedModel,
1040
+ messages: chatMessages,
1041
+ stream: false,
1042
+ ...generationPayload,
1043
+ })
1044
+ const assistant = response.choices?.[0]?.message
1045
+ const assistantText =
1046
+ typeof assistant?.content === 'string'
1047
+ ? assistant.content
1048
+ : getTextContent((assistant?.content as ContentPart[] | null) ?? null)
1049
+
1050
+ const audio = extractAssistantAudio(assistant)
1051
+ let assistantAudioUrl: string | undefined
1052
+ if (audio?.url) {
1053
+ assistantAudioUrl = resolveMediaUrl(audio.url)
1054
+ } else if (audio?.data) {
1055
+ const dataUrl = audio.data.startsWith('data:')
1056
+ ? audio.data
1057
+ : `data:${formatToMimeType(audio.format) ?? 'audio/wav'};base64,${audio.data}`
1058
+ if ((activeSessionStorageVersion ?? 1) >= 2) {
1059
+ const cached = await storeMedia(
1060
+ dataUrl,
1061
+ selectedModel,
1062
+ formatToMimeType(audio.format) ?? pendingAudioMimeType
1063
+ )
1064
+ assistantAudioUrl = cached.url
1065
+ } else {
1066
+ assistantAudioUrl = dataUrl
1067
+ }
1068
+ }
1069
+
1070
+ const assistantContent: ContentPart[] = [
1071
+ ...(assistantText ? [{ type: 'text' as const, text: assistantText }] : []),
1072
+ ...(assistantAudioUrl
1073
+ ? [{ type: 'audio' as const, audio: { url: assistantAudioUrl, format: audio?.format } }]
1074
+ : []),
1075
+ ]
1076
+
1077
+ setMessages(prev =>
1078
+ prev.map(m =>
1079
+ m.id === assistantMessage.id
1080
+ ? { ...m, content: assistantContent.length > 0 ? assistantContent : assistantText || '' }
1081
+ : m
1082
+ )
1083
+ )
1084
+
1085
+ if (assistantAudioUrl) {
1086
+ stopPlayback()
1087
+ const player = new Audio(assistantAudioUrl)
1088
+ callAudioRef.current = player
1089
+ setCallStatus('playing')
1090
+ player.onended = () => {
1091
+ setCallStatus('idle')
1092
+ callAudioRef.current = null
1093
+ }
1094
+ player.onerror = () => {
1095
+ setCallStatus('idle')
1096
+ callAudioRef.current = null
1097
+ }
1098
+ void player.play().catch((error) => {
1099
+ console.warn('Audio playback failed:', error)
1100
+ setCallStatus('idle')
1101
+ callAudioRef.current = null
1102
+ })
1103
+ } else {
1104
+ setCallStatus('idle')
1105
+ }
1106
+
1107
+ if (activeSessionId) {
1108
+ await addMessageToSession(activeSessionId, {
1109
+ role: 'assistant',
1110
+ content: assistantContent.length > 0 ? assistantContent : assistantText || '',
1111
+ timestamp: new Date().toISOString(),
1112
+ model: selectedModel,
1113
+ })
1114
+ await maybeAutoTitleSession(activeSessionId)
1115
+ }
1116
+ } else if (selectedModelSupportsImageOutput()) {
1117
+ // Image generation mode
1118
+ const prompt = getTextContent(userMessage.content)
1119
+ if (!prompt) {
1120
+ throw new Error('Please enter a prompt for image generation')
1121
+ }
1122
+
1123
+ const editInputImageUrl = requestImageUrls.length > 0 ? requestImageUrls[0] : undefined
1124
+ const imageResponse = await generateImage({
1125
+ model: selectedModel,
1126
+ prompt,
1127
+ image_url: editInputImageUrl,
1128
+ n: 1,
1129
+ size: imageSize,
1130
+ response_format: 'b64_json',
1131
+ })
1132
+
1133
+ // Convert response to content with image
1134
+ const imageData = imageResponse.data[0]
1135
+ let imageUrl = imageData.url || ''
1136
+ if (!imageUrl && imageData.b64_json) {
1137
+ if ((activeSessionStorageVersion ?? 1) >= 2) {
1138
+ try {
1139
+ const cached = await cacheImage(imageData.b64_json, selectedModel)
1140
+ imageUrl = cached.url
1141
+ } catch (error) {
1142
+ console.error('Failed to cache generated image, using inline payload:', error)
1143
+ imageUrl = `data:image/png;base64,${imageData.b64_json}`
1144
+ }
1145
+ } else {
1146
+ imageUrl = `data:image/png;base64,${imageData.b64_json}`
1147
+ }
1148
+ }
1149
+
1150
+ const imageContent: ContentPart[] = []
1151
+ if (imageData.revised_prompt) {
1152
+ imageContent.push({ type: 'text', text: imageData.revised_prompt })
1153
+ }
1154
+ if (imageUrl) {
1155
+ imageContent.push({ type: 'image_url', image_url: { url: imageUrl } })
1156
+ }
1157
+
1158
+ setMessages(prev =>
1159
+ prev.map(m =>
1160
+ m.id === assistantMessage.id
1161
+ ? { ...m, content: imageContent }
1162
+ : m
1163
+ )
1164
+ )
1165
+
1166
+ // Save to session
1167
+ if (activeSessionId) {
1168
+ try {
1169
+ await addMessageToSession(activeSessionId, {
1170
+ role: 'assistant',
1171
+ content: imageContent,
1172
+ timestamp: new Date().toISOString(),
1173
+ model: selectedModel,
1174
+ })
1175
+ await maybeAutoTitleSession(activeSessionId)
1176
+ } catch (error) {
1177
+ console.error('Failed to save assistant message:', error)
1178
+ }
1179
+ }
1180
+ } else if (agentModeEnabled && selectedTools.size > 0) {
1181
+ // Run agentic loop
1182
+ await runAgentLoop(
1183
+ [...messages, userMessage, assistantMessage],
1184
+ assistantMessage.id,
1185
+ generationPayload
1186
+ )
1187
+ if (activeSessionId) {
1188
+ await maybeAutoTitleSession(activeSessionId)
1189
+ }
1190
+ } else {
1191
+ // Regular chat (LLM, embedding, audio handled as chat for now)
1192
+ abortControllerRef.current = new AbortController()
1193
+ isStreamingRef.current = true
1194
+
1195
+ const chatMessages = [...messages, userMessage].map(toApiMessage) as ApiChatMessage[]
1196
+ warnIfNonDataImageUrls(chatMessages, 'chat-stream')
1197
+
1198
+ let streamState = createThinkingStreamState()
1199
+ for await (const chunk of streamChatCompletion(
1200
+ { model: selectedModel, messages: chatMessages, ...generationPayload },
1201
+ abortControllerRef.current.signal
1202
+ )) {
1203
+ streamState = applyThinkingChunk(streamState, chunk)
1204
+ const displayContent = toDisplayContent(streamState)
1205
+ setMessages(prev =>
1206
+ prev.map(m =>
1207
+ m.id === assistantMessage.id
1208
+ ? { ...m, content: displayContent }
1209
+ : m
1210
+ )
1211
+ )
1212
+ }
1213
+ const finalContent = toFinalContent(streamState)
1214
+ setMessages(prev =>
1215
+ prev.map(m =>
1216
+ m.id === assistantMessage.id
1217
+ ? { ...m, content: finalContent }
1218
+ : m
1219
+ )
1220
+ )
1221
+
1222
+ // Save assistant message
1223
+ if (activeSessionId && finalContent) {
1224
+ try {
1225
+ await addMessageToSession(activeSessionId, {
1226
+ role: 'assistant',
1227
+ content: finalContent,
1228
+ timestamp: new Date().toISOString(),
1229
+ model: selectedModel,
1230
+ })
1231
+ await maybeAutoTitleSession(activeSessionId)
1232
+ } catch (error) {
1233
+ console.error('Failed to save assistant message:', error)
1234
+ }
1235
+ }
1236
+ }
1237
+ } catch (error) {
1238
+ if ((error as Error).name === 'AbortError') return
1239
+ console.error('Chat error:', error)
1240
+ if (callModeEnabled) {
1241
+ setCallError((error as Error).message || 'Call turn failed. Try again.')
1242
+ }
1243
+ setCallStatus('idle')
1244
+ setMessages(prev =>
1245
+ prev.map(m =>
1246
+ m.id === assistantMessage.id
1247
+ ? { ...m, content: 'Error occurred. Please try again.' }
1248
+ : m
1249
+ )
1250
+ )
1251
+ } finally {
1252
+ setIsLoading(false)
1253
+ abortControllerRef.current = null
1254
+ stopAgentRef.current = false
1255
+ isStreamingRef.current = false
1256
+ if (!callAudioRef.current) {
1257
+ setCallStatus('idle')
1258
+ }
1259
+ }
1260
+ }
1261
+
1262
+ const handleKeyDown = (e: React.KeyboardEvent) => {
1263
+ if (e.key === 'Enter' && !e.shiftKey) {
1264
+ e.preventDefault()
1265
+ handleSubmit(e)
1266
+ }
1267
+ }
1268
+
1269
+ return (
1270
+ <div className="flex-1 flex h-full min-h-0 overflow-hidden">
1271
+ {/* Sessions Sidebar */}
1272
+ <aside className={cn(
1273
+ 'border-r border-border flex flex-col min-h-0 shrink-0 transition-all duration-300',
1274
+ sidebarCollapsed ? 'w-12' : 'w-64'
1275
+ )}>
1276
+ <div className="h-14 border-b border-border flex items-center justify-between px-3">
1277
+ {!sidebarCollapsed && (
1278
+ <span className="font-mono text-xs uppercase tracking-wider text-muted-foreground">
1279
+ Sessions
1280
+ </span>
1281
+ )}
1282
+ <Button
1283
+ variant="ghost"
1284
+ size="icon"
1285
+ className="h-7 w-7"
1286
+ onClick={() => setSidebarCollapsed(!sidebarCollapsed)}
1287
+ >
1288
+ <ChevronRight className={cn(
1289
+ 'w-4 h-4 transition-transform',
1290
+ !sidebarCollapsed && 'rotate-180'
1291
+ )} />
1292
+ </Button>
1293
+ </div>
1294
+
1295
+ {!sidebarCollapsed && (
1296
+ <>
1297
+ <div className="p-2">
1298
+ <Button
1299
+ onClick={handleNewSession}
1300
+ className="w-full justify-start gap-2"
1301
+ variant="outline"
1302
+ >
1303
+ <Plus className="w-4 h-4" />
1304
+ New Chat
1305
+ </Button>
1306
+ </div>
1307
+
1308
+ <div className="flex-1 overflow-y-auto">
1309
+ {sessions.map(session => (
1310
+ <div
1311
+ key={session.id}
1312
+ className={cn(
1313
+ 'group flex items-center gap-2 px-3 py-2 cursor-pointer hover:bg-secondary/50 transition-colors',
1314
+ activeSessionId === session.id && 'bg-secondary border-l-2 border-primary'
1315
+ )}
1316
+ onClick={() => loadSession(session.id)}
1317
+ >
1318
+ <MessageSquare className="w-4 h-4 text-muted-foreground shrink-0" />
1319
+ <div className="flex-1 min-w-0">
1320
+ <p className="text-sm truncate">{session.name}</p>
1321
+ <p className="text-2xs text-muted-foreground">
1322
+ {titleGenerationSessionId === session.id ? 'Generating title...' : `${session.messageCount} messages`}
1323
+ </p>
1324
+ </div>
1325
+ <Button
1326
+ variant="ghost"
1327
+ size="icon"
1328
+ className="h-6 w-6 opacity-0 group-hover:opacity-100 transition-opacity"
1329
+ onClick={(e) => {
1330
+ e.stopPropagation()
1331
+ handleDeleteSession(session.id)
1332
+ }}
1333
+ >
1334
+ <Trash2 className="w-3 h-3 text-destructive" />
1335
+ </Button>
1336
+ </div>
1337
+ ))}
1338
+
1339
+ {sessions.length === 0 && (
1340
+ <div className="p-4 text-center text-muted-foreground text-sm">
1341
+ No sessions yet
1342
+ </div>
1343
+ )}
1344
+ </div>
1345
+ </>
1346
+ )}
1347
+ </aside>
1348
+
1349
+ {/* Main Chat Area */}
1350
+ <div className="flex-1 flex flex-col min-h-0">
1351
+ {/* Header */}
1352
+ <header className="sticky top-0 z-20 h-14 border-b border-border bg-background/95 backdrop-blur flex items-center px-6 gap-4 shrink-0">
1353
+ <div className="flex items-center gap-2">
1354
+ <Sparkles className="w-4 h-4 text-primary" />
1355
+ <h2 className="font-mono font-semibold text-sm uppercase tracking-wider">
1356
+ {sessionName || 'Playground'}
1357
+ </h2>
1358
+ {titleGenerationSessionId === activeSessionId && (
1359
+ <span className="text-2xs font-mono uppercase tracking-wider text-muted-foreground">
1360
+ Generating title...
1361
+ </span>
1362
+ )}
1363
+ </div>
1364
+
1365
+ {/* Agent Mode Toggle */}
1366
+ <button
1367
+ onClick={() => {
1368
+ setAgentModeEnabled(!agentModeEnabled)
1369
+ if (!agentModeEnabled) setShowToolPicker(true)
1370
+ }}
1371
+ className={cn(
1372
+ 'flex items-center gap-2 px-2 py-1 rounded-md text-xs font-mono transition-colors',
1373
+ agentModeEnabled
1374
+ ? 'bg-primary/20 text-primary border border-primary/30'
1375
+ : 'bg-secondary text-muted-foreground hover:text-foreground'
1376
+ )}
1377
+ >
1378
+ {agentModeEnabled ? (
1379
+ <ToggleRight className="w-4 h-4" />
1380
+ ) : (
1381
+ <ToggleLeft className="w-4 h-4" />
1382
+ )}
1383
+ Agent Mode
1384
+ {selectedTools.size > 0 && (
1385
+ <span className="px-1.5 py-0.5 bg-primary/30 rounded text-2xs">
1386
+ {selectedTools.size}
1387
+ </span>
1388
+ )}
1389
+ </button>
1390
+
1391
+ {agentModeEnabled && (
1392
+ <Button
1393
+ variant="ghost"
1394
+ size="sm"
1395
+ className="h-7 text-xs gap-1"
1396
+ onClick={() => setShowToolPicker(!showToolPicker)}
1397
+ >
1398
+ <Wrench className="w-3.5 h-3.5" />
1399
+ Tools
1400
+ </Button>
1401
+ )}
1402
+
1403
+ {modelSupportsCall && (
1404
+ <Button
1405
+ variant={callModeEnabled ? "default" : "ghost"}
1406
+ size="sm"
1407
+ className="h-7 text-xs gap-1"
1408
+ onClick={() => {
1409
+ setCallModeEnabled(!callModeEnabled)
1410
+ setPendingAudio(null)
1411
+ setPendingAudioMimeType(undefined)
1412
+ setCallError(null)
1413
+ if (callStatus === 'recording') {
1414
+ stopRecording()
1415
+ }
1416
+ stopPlayback()
1417
+ }}
1418
+ >
1419
+ {callModeEnabled ? <PhoneCall className="w-3.5 h-3.5" /> : <PhoneOff className="w-3.5 h-3.5" />}
1420
+ Call
1421
+ </Button>
1422
+ )}
1423
+
1424
+ <div className="flex-1" />
1425
+
1426
+ <select
1427
+ value={selectedModel}
1428
+ onChange={(e) => setSelectedModel(e.target.value)}
1429
+ className="bg-input border border-border rounded px-3 py-1.5 text-sm font-mono focus:ring-1 focus:ring-primary focus:outline-none"
1430
+ >
1431
+ {models.length === 0 && <option value="">No models available</option>}
1432
+ {models.map(model => (
1433
+ <option key={model.id} value={model.id} disabled={!isModelSelectable(model)}>
1434
+ {model.id} {formatModelTag(model)} {!isModelSelectable(model) ? '(down)' : ''}
1435
+ </option>
1436
+ ))}
1437
+ </select>
1438
+ </header>
1439
+
1440
+ <div className="flex-1 min-h-0 flex overflow-hidden">
1441
+ {/* Messages Area */}
1442
+ <div className="flex-1 min-h-0 flex flex-col">
1443
+ <div
1444
+ ref={messagesContainerRef}
1445
+ className={cn(
1446
+ 'flex-1 min-h-0 overflow-y-auto p-6 space-y-4 relative',
1447
+ isDragging && 'bg-primary/5 border-2 border-dashed border-primary/30'
1448
+ )}
1449
+ onDragOver={(e) => { e.preventDefault(); setIsDragging(true) }}
1450
+ onDragLeave={() => setIsDragging(false)}
1451
+ onDrop={handleDrop}
1452
+ >
1453
+ {isDragging && (
1454
+ <div className="absolute inset-0 flex items-center justify-center pointer-events-none z-10">
1455
+ <div className="bg-background/90 rounded-lg p-6 text-center">
1456
+ <ImageIcon className="w-12 h-12 text-primary mx-auto mb-2" />
1457
+ <p className="font-mono text-sm">
1458
+ {callModeEnabled ? 'Drop image or audio here' : 'Drop image here'}
1459
+ </p>
1460
+ </div>
1461
+ </div>
1462
+ )}
1463
+
1464
+ {messages.length === 0 && (
1465
+ <div className="flex-1 flex items-center justify-center h-full">
1466
+ <div className="text-center space-y-4 animate-fade-in">
1467
+ <div className="w-16 h-16 rounded-full bg-primary/10 flex items-center justify-center mx-auto">
1468
+ <Bot className="w-8 h-8 text-primary" />
1469
+ </div>
1470
+ <div>
1471
+ <h3 className="font-mono font-semibold text-lg">Ready to chat</h3>
1472
+ <p className="text-muted-foreground text-sm mt-1">
1473
+ {agentModeEnabled
1474
+ ? 'Agent mode enabled - select tools to use'
1475
+ : 'Select a model and start a conversation'}
1476
+ </p>
1477
+ <p className="text-muted-foreground text-xs mt-2">
1478
+ {callModeEnabled
1479
+ ? 'Call mode: push-to-talk + optional text/images'
1480
+ : 'Supports images via drag, drop, paste, or upload'}
1481
+ </p>
1482
+ </div>
1483
+ </div>
1484
+ </div>
1485
+ )}
1486
+
1487
+ {messages.map((message, index) => (
1488
+ <div
1489
+ key={message.id}
1490
+ className={cn(
1491
+ 'flex gap-3 animate-slide-in-bottom',
1492
+ message.role === 'user' ? 'justify-end' : 'justify-start',
1493
+ message.role === 'tool' && 'opacity-70'
1494
+ )}
1495
+ style={{ animationDelay: `${index * 50}ms` }}
1496
+ >
1497
+ {message.role === 'assistant' && (
1498
+ <div className="w-8 h-8 rounded-full bg-secondary flex items-center justify-center shrink-0">
1499
+ <Bot className="w-4 h-4 text-muted-foreground" />
1500
+ </div>
1501
+ )}
1502
+ {message.role === 'tool' && (
1503
+ <div className="w-8 h-8 rounded-full bg-amber-500/20 flex items-center justify-center shrink-0">
1504
+ <Wrench className="w-4 h-4 text-amber-500" />
1505
+ </div>
1506
+ )}
1507
+ <div
1508
+ className={cn(
1509
+ 'rounded-lg px-4 py-3',
1510
+ message.role === 'user'
1511
+ ? 'max-w-[70%] bg-primary/10 border border-primary/20'
1512
+ : message.role === 'tool'
1513
+ ? 'max-w-[70%] bg-amber-500/5 border border-amber-500/20'
1514
+ : 'w-full sm:w-[70%] max-w-[75ch] bg-secondary border border-border'
1515
+ )}
1516
+ >
1517
+ {/* Render explicitly attached images */}
1518
+ {message.images && message.images.length > 0 && (
1519
+ <div className="flex flex-wrap gap-2 mb-2">
1520
+ {message.images.map((img, i) => (
1521
+ <img
1522
+ key={i}
1523
+ src={resolveMediaUrl(img)}
1524
+ alt={`Attached ${i + 1}`}
1525
+ className="max-w-32 max-h-32 rounded border border-border"
1526
+ />
1527
+ ))}
1528
+ </div>
1529
+ )}
1530
+ {/* Render images from multimodal content (e.g., image generation responses) */}
1531
+ {getImageUrls(message.content).length > 0 && (
1532
+ <div className="flex flex-wrap gap-2 mb-2">
1533
+ {getImageUrls(message.content).map((url, i) => (
1534
+ <a
1535
+ key={i}
1536
+ href={url}
1537
+ target="_blank"
1538
+ rel="noopener noreferrer"
1539
+ className="block"
1540
+ >
1541
+ <img
1542
+ src={url}
1543
+ alt={`Generated ${i + 1}`}
1544
+ className="max-w-xs max-h-64 rounded border border-border hover:border-primary transition-colors cursor-pointer"
1545
+ />
1546
+ </a>
1547
+ ))}
1548
+ </div>
1549
+ )}
1550
+ {getAudioUrls(message.content).length > 0 && (
1551
+ <div className="flex flex-col gap-2 mb-2">
1552
+ {getAudioUrls(message.content).map((url, i) => (
1553
+ <audio
1554
+ key={i}
1555
+ controls
1556
+ src={url}
1557
+ className="w-full max-w-sm"
1558
+ />
1559
+ ))}
1560
+ </div>
1561
+ )}
1562
+ {/* Render text content with markdown support */}
1563
+ {getTextContent(message.content) && (
1564
+ message.role === 'user' ? (
1565
+ <p className="text-sm whitespace-pre-wrap">{getTextContent(message.content)}</p>
1566
+ ) : (
1567
+ <MessageContent content={getTextContent(message.content)} />
1568
+ )
1569
+ )}
1570
+ {message.toolCalls && message.toolCalls.length > 0 && (
1571
+ <div className="mt-2">
1572
+ <ToolCallMessage toolCalls={message.toolCalls} />
1573
+ </div>
1574
+ )}
1575
+ {message.role === 'assistant' && message.model && (
1576
+ <p className="text-[10px] font-mono text-muted-foreground/60 mt-1.5 select-none">{message.model}</p>
1577
+ )}
1578
+ </div>
1579
+ {message.role === 'user' && (
1580
+ <div className="w-8 h-8 rounded-full bg-primary/20 flex items-center justify-center shrink-0">
1581
+ <User className="w-4 h-4 text-primary" />
1582
+ </div>
1583
+ )}
1584
+ </div>
1585
+ ))}
1586
+
1587
+ {isLoading && messages[messages.length - 1]?.content === '' && !messages[messages.length - 1]?.toolCalls && (
1588
+ <div className="flex gap-3 items-center text-muted-foreground">
1589
+ <div className="w-8 h-8 rounded-full bg-secondary flex items-center justify-center">
1590
+ <Loader2 className="w-4 h-4 animate-spin" />
1591
+ </div>
1592
+ <span className="text-sm font-mono">
1593
+ {isExecutingTools
1594
+ ? 'Executing tools...'
1595
+ : currentIteration > 0
1596
+ ? `Thinking (iteration ${currentIteration})...`
1597
+ : 'Thinking...'}
1598
+ </span>
1599
+ </div>
1600
+ )}
1601
+
1602
+ <div ref={messagesEndRef} />
1603
+ </div>
1604
+
1605
+ {/* Pending Images Preview */}
1606
+ {pendingImages.length > 0 && (
1607
+ <div className="border-t border-border px-4 py-2 flex gap-2 flex-wrap bg-secondary/30">
1608
+ {pendingImages.map((img, i) => (
1609
+ <div key={i} className="relative group">
1610
+ <img
1611
+ src={img}
1612
+ alt={`Pending ${i + 1}`}
1613
+ className="h-16 w-16 object-cover rounded border border-border"
1614
+ />
1615
+ <button
1616
+ onClick={() => setPendingImages(prev => prev.filter((_, j) => j !== i))}
1617
+ className="absolute -top-1 -right-1 w-5 h-5 rounded-full bg-destructive text-destructive-foreground flex items-center justify-center opacity-0 group-hover:opacity-100 transition-opacity"
1618
+ >
1619
+ <X className="w-3 h-3" />
1620
+ </button>
1621
+ </div>
1622
+ ))}
1623
+ </div>
1624
+ )}
1625
+
1626
+ {callModeEnabled && pendingAudio && (
1627
+ <div className="border-t border-border px-4 py-2 bg-secondary/20">
1628
+ <div className="flex items-center gap-2">
1629
+ <audio controls src={pendingAudio} className="w-full max-w-md" />
1630
+ <Button
1631
+ type="button"
1632
+ variant="ghost"
1633
+ size="icon"
1634
+ className="h-7 w-7 shrink-0"
1635
+ onClick={cancelRecording}
1636
+ title="Remove pending audio"
1637
+ >
1638
+ <X className="w-3 h-3" />
1639
+ </Button>
1640
+ </div>
1641
+ </div>
1642
+ )}
1643
+
1644
+ {/* Input Area */}
1645
+ <div className="border-t border-border p-4">
1646
+ {/* Image Size Picker (shown for diffusion models) */}
1647
+ {selectedModelSupportsImageOutput() && (
1648
+ <div className="mb-3 flex items-center gap-2">
1649
+ <span className="text-xs text-muted-foreground">Image Size:</span>
1650
+ <div className="flex gap-1 flex-wrap">
1651
+ {IMAGE_SIZE_OPTIONS.map((option) => (
1652
+ <button
1653
+ key={option.value}
1654
+ type="button"
1655
+ onClick={() => setImageSize(option.value)}
1656
+ className={cn(
1657
+ 'px-2 py-1 text-xs font-mono rounded transition-colors',
1658
+ imageSize === option.value
1659
+ ? 'bg-primary text-primary-foreground'
1660
+ : 'bg-secondary text-muted-foreground hover:text-foreground hover:bg-secondary/80'
1661
+ )}
1662
+ >
1663
+ {option.label}
1664
+ </button>
1665
+ ))}
1666
+ </div>
1667
+ </div>
1668
+ )}
1669
+ <div className="mb-3">
1670
+ <button
1671
+ type="button"
1672
+ className="w-full text-left text-xs font-mono text-muted-foreground border border-border rounded px-2 py-1 hover:bg-secondary/50"
1673
+ onClick={() => setShowGenerationParams((prev) => !prev)}
1674
+ >
1675
+ {showGenerationParams ? 'Hide Generation Params' : 'Show Generation Params'}
1676
+ </button>
1677
+ {showGenerationParams && (
1678
+ <div className="mt-2 grid grid-cols-2 gap-2 rounded border border-border/60 bg-secondary/20 p-2">
1679
+ <label className="text-xs text-muted-foreground block">
1680
+ Temperature
1681
+ <input
1682
+ className="mt-1 w-full bg-input border border-border rounded px-2 py-1 text-sm font-mono"
1683
+ value={generationParams.temperature}
1684
+ onChange={(event) =>
1685
+ setGenerationParams((prev) => ({ ...prev, temperature: event.target.value }))
1686
+ }
1687
+ placeholder="e.g. 0.7"
1688
+ />
1689
+ </label>
1690
+ <label className="text-xs text-muted-foreground block">
1691
+ Top P
1692
+ <input
1693
+ className="mt-1 w-full bg-input border border-border rounded px-2 py-1 text-sm font-mono"
1694
+ value={generationParams.topP}
1695
+ onChange={(event) =>
1696
+ setGenerationParams((prev) => ({ ...prev, topP: event.target.value }))
1697
+ }
1698
+ placeholder="e.g. 1"
1699
+ />
1700
+ </label>
1701
+ <label className="text-xs text-muted-foreground block">
1702
+ Max Tokens
1703
+ <input
1704
+ className="mt-1 w-full bg-input border border-border rounded px-2 py-1 text-sm font-mono"
1705
+ value={generationParams.maxTokens}
1706
+ onChange={(event) =>
1707
+ setGenerationParams((prev) => ({ ...prev, maxTokens: event.target.value }))
1708
+ }
1709
+ placeholder="e.g. 512"
1710
+ />
1711
+ </label>
1712
+ <label className="text-xs text-muted-foreground block">
1713
+ Presence Penalty
1714
+ <input
1715
+ className="mt-1 w-full bg-input border border-border rounded px-2 py-1 text-sm font-mono"
1716
+ value={generationParams.presencePenalty}
1717
+ onChange={(event) =>
1718
+ setGenerationParams((prev) => ({ ...prev, presencePenalty: event.target.value }))
1719
+ }
1720
+ placeholder="-2 to 2"
1721
+ />
1722
+ </label>
1723
+ <label className="text-xs text-muted-foreground block">
1724
+ Frequency Penalty
1725
+ <input
1726
+ className="mt-1 w-full bg-input border border-border rounded px-2 py-1 text-sm font-mono"
1727
+ value={generationParams.frequencyPenalty}
1728
+ onChange={(event) =>
1729
+ setGenerationParams((prev) => ({ ...prev, frequencyPenalty: event.target.value }))
1730
+ }
1731
+ placeholder="-2 to 2"
1732
+ />
1733
+ </label>
1734
+ <label className="text-xs text-muted-foreground block">
1735
+ Seed
1736
+ <input
1737
+ className="mt-1 w-full bg-input border border-border rounded px-2 py-1 text-sm font-mono"
1738
+ value={generationParams.seed}
1739
+ onChange={(event) =>
1740
+ setGenerationParams((prev) => ({ ...prev, seed: event.target.value }))
1741
+ }
1742
+ placeholder="integer"
1743
+ />
1744
+ </label>
1745
+ <label className="text-xs text-muted-foreground block col-span-2">
1746
+ Stop Sequences (comma-separated)
1747
+ <input
1748
+ className="mt-1 w-full bg-input border border-border rounded px-2 py-1 text-sm font-mono"
1749
+ value={generationParams.stop}
1750
+ onChange={(event) =>
1751
+ setGenerationParams((prev) => ({ ...prev, stop: event.target.value }))
1752
+ }
1753
+ placeholder="END, STOP"
1754
+ />
1755
+ </label>
1756
+ </div>
1757
+ )}
1758
+ </div>
1759
+ <form onSubmit={handleSubmit} className="flex gap-3">
1760
+ <input
1761
+ type="file"
1762
+ ref={fileInputRef}
1763
+ className="hidden"
1764
+ accept={callModeEnabled ? "image/*,audio/*" : "image/*"}
1765
+ multiple
1766
+ onChange={(e) => {
1767
+ if (!e.target.files) return
1768
+ processSelectedFiles(e.target.files)
1769
+ e.target.value = ''
1770
+ }}
1771
+ />
1772
+ <Button
1773
+ type="button"
1774
+ variant="outline"
1775
+ size="icon"
1776
+ className="shrink-0"
1777
+ title={callModeEnabled ? "Attach image/audio" : "Attach image"}
1778
+ onClick={() => fileInputRef.current?.click()}
1779
+ >
1780
+ <ImagePlus className="w-4 h-4" />
1781
+ </Button>
1782
+ {callModeEnabled && (
1783
+ <Button
1784
+ type="button"
1785
+ variant={callStatus === 'recording' ? 'destructive' : 'outline'}
1786
+ size="icon"
1787
+ className="shrink-0"
1788
+ title={callStatus === 'recording' ? 'Stop recording' : 'Start recording'}
1789
+ disabled={isLoading || callStatus === 'sending'}
1790
+ onClick={() => {
1791
+ if (callStatus === 'recording') {
1792
+ stopRecording()
1793
+ } else {
1794
+ void startRecording()
1795
+ }
1796
+ }}
1797
+ >
1798
+ {callStatus === 'recording' ? (
1799
+ <StopCircle className="w-4 h-4" />
1800
+ ) : (
1801
+ <Mic className="w-4 h-4" />
1802
+ )}
1803
+ </Button>
1804
+ )}
1805
+ <Textarea
1806
+ ref={inputRef}
1807
+ value={input}
1808
+ onChange={(e) => setInput(e.target.value)}
1809
+ onKeyDown={handleKeyDown}
1810
+ onPaste={handlePaste}
1811
+ placeholder={
1812
+ callModeEnabled
1813
+ ? 'Record audio with mic, then optionally add text or images...'
1814
+ : agentModeEnabled && selectedTools.size > 0
1815
+ ? 'Ask me anything... (agent mode enabled)'
1816
+ : 'Type a message... (paste or drop images)'
1817
+ }
1818
+ className="min-h-[44px] leading-5"
1819
+ rows={1}
1820
+ />
1821
+ {isLoading && callModeEnabled ? (
1822
+ <Button
1823
+ type="button"
1824
+ disabled
1825
+ className="shrink-0"
1826
+ >
1827
+ <Loader2 className="w-4 h-4 animate-spin" />
1828
+ </Button>
1829
+ ) : isLoading ? (
1830
+ <Button
1831
+ type="button"
1832
+ variant="destructive"
1833
+ className="shrink-0"
1834
+ onClick={stopAgent}
1835
+ >
1836
+ <StopCircle className="w-4 h-4" />
1837
+ </Button>
1838
+ ) : (
1839
+ <Button
1840
+ type="submit"
1841
+ disabled={
1842
+ isLoading ||
1843
+ !selectedModel ||
1844
+ !selectedModelIsSelectable ||
1845
+ (callModeEnabled
1846
+ ? !pendingAudio
1847
+ : (!input.trim() && pendingImages.length === 0))
1848
+ }
1849
+ className="shrink-0"
1850
+ >
1851
+ <Send className="w-4 h-4" />
1852
+ </Button>
1853
+ )}
1854
+ </form>
1855
+ <p className="text-2xs text-muted-foreground mt-2 text-center font-mono">
1856
+ Enter to send | Shift+Enter for new line
1857
+ {agentModeEnabled && ' | Agent mode: max 10 iterations'}
1858
+ {callModeEnabled && ' | Call mode: record, then send'}
1859
+ </p>
1860
+ {callModeEnabled && (
1861
+ <p className="text-2xs text-muted-foreground mt-1 text-center font-mono">
1862
+ Status: {callStatus}
1863
+ </p>
1864
+ )}
1865
+ {callError && (
1866
+ <p className="text-2xs text-destructive mt-1 text-center font-mono">
1867
+ {callError}
1868
+ </p>
1869
+ )}
1870
+ </div>
1871
+ </div>
1872
+
1873
+ {/* Tool Picker Sidebar */}
1874
+ {agentModeEnabled && showToolPicker && (
1875
+ <aside className="w-72 border-l border-border flex flex-col shrink-0 animate-slide-in-right">
1876
+ <ToolPicker
1877
+ selectedTools={selectedTools}
1878
+ onToolsChange={setSelectedTools}
1879
+ className="h-full"
1880
+ />
1881
+ </aside>
1882
+ )}
1883
+ </div>
1884
+ </div>
1885
+ </div>
1886
+ )
1887
+ }
1888
+
1889
+ // Enhanced stream function that captures tool calls
1890
+ async function* streamChatCompletionWithTools(
1891
+ request: {
1892
+ model: string
1893
+ messages: ApiChatMessage[]
1894
+ temperature?: number
1895
+ top_p?: number
1896
+ max_tokens?: number
1897
+ presence_penalty?: number
1898
+ frequency_penalty?: number
1899
+ seed?: number
1900
+ stop?: string | string[]
1901
+ tools?: unknown[]
1902
+ tool_choice?: unknown
1903
+ },
1904
+ signal: AbortSignal,
1905
+ onToolCalls: (toolCalls: Array<{ id: string; function: { name: string; arguments: string } }>) => void
1906
+ ): AsyncGenerator<string, void, unknown> {
1907
+ // Build request body, omitting tools/tool_choice if not present
1908
+ const body: Record<string, unknown> = {
1909
+ model: request.model,
1910
+ messages: request.messages,
1911
+ stream: true,
1912
+ temperature: request.temperature,
1913
+ top_p: request.top_p,
1914
+ max_tokens: request.max_tokens,
1915
+ presence_penalty: request.presence_penalty,
1916
+ frequency_penalty: request.frequency_penalty,
1917
+ seed: request.seed,
1918
+ stop: request.stop,
1919
+ }
1920
+
1921
+ // Only include tools if we have them
1922
+ // Note: We don't send tool_choice by default as many backends (like vLLM)
1923
+ // require special configuration for it. Models will still use tools if provided.
1924
+ if (request.tools && request.tools.length > 0) {
1925
+ body.tools = request.tools
1926
+ // Uncomment if your backend supports tool_choice:
1927
+ // if (request.tool_choice) {
1928
+ // body.tool_choice = request.tool_choice
1929
+ // }
1930
+ }
1931
+
1932
+ const response = await fetch('/v1/chat/completions', {
1933
+ method: 'POST',
1934
+ headers: { 'Content-Type': 'application/json' },
1935
+ body: JSON.stringify(body),
1936
+ signal,
1937
+ })
1938
+
1939
+ if (!response.ok) {
1940
+ throw new Error(`API Error: ${response.status} ${response.statusText}`)
1941
+ }
1942
+
1943
+ const reader = response.body?.getReader()
1944
+ if (!reader) throw new Error('No response body')
1945
+
1946
+ const decoder = new TextDecoder()
1947
+ let buffer = ''
1948
+ const toolCallsMap = new Map<number, { id: string; function: { name: string; arguments: string } }>()
1949
+
1950
+ while (true) {
1951
+ const { done, value } = await reader.read()
1952
+ if (done) break
1953
+
1954
+ buffer += decoder.decode(value, { stream: true })
1955
+ const lines = buffer.split('\n')
1956
+ buffer = lines.pop() || ''
1957
+
1958
+ for (const line of lines) {
1959
+ if (line.startsWith('data: ')) {
1960
+ const data = line.slice(6)
1961
+ if (data === '[DONE]') {
1962
+ // Finalize tool calls
1963
+ if (toolCallsMap.size > 0) {
1964
+ onToolCalls(Array.from(toolCallsMap.values()))
1965
+ }
1966
+ return
1967
+ }
1968
+
1969
+ try {
1970
+ const parsed = JSON.parse(data)
1971
+ const delta = parsed.choices?.[0]?.delta
1972
+
1973
+ // Handle text content
1974
+ if (delta?.content) {
1975
+ yield delta.content
1976
+ }
1977
+
1978
+ // Handle tool calls (accumulated across chunks)
1979
+ if (delta?.tool_calls) {
1980
+ for (const tc of delta.tool_calls) {
1981
+ const index = tc.index ?? 0
1982
+ const existing = toolCallsMap.get(index)
1983
+
1984
+ if (!existing) {
1985
+ toolCallsMap.set(index, {
1986
+ id: tc.id || '',
1987
+ function: {
1988
+ name: tc.function?.name || '',
1989
+ arguments: tc.function?.arguments || '',
1990
+ },
1991
+ })
1992
+ } else {
1993
+ if (tc.id) existing.id = tc.id
1994
+ if (tc.function?.name) existing.function.name += tc.function.name
1995
+ if (tc.function?.arguments) existing.function.arguments += tc.function.arguments
1996
+ }
1997
+ }
1998
+ }
1999
+ } catch {
2000
+ // Skip malformed JSON
2001
+ }
2002
+ }
2003
+ }
2004
+ }
2005
+
2006
+ // Finalize any remaining tool calls
2007
+ if (toolCallsMap.size > 0) {
2008
+ onToolCalls(Array.from(toolCallsMap.values()))
2009
+ }
2010
+ }