waypoi 0.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/instructions/ui.instructions.md +42 -0
- package/.github/workflows/ci.yml +35 -0
- package/.github/workflows/publish.yml +71 -0
- package/.github/workflows/release.yml +48 -0
- package/.playwright-mcp/console-2026-04-04T01-41-10-746Z.log +2 -0
- package/.playwright-mcp/console-2026-04-04T01-41-28-799Z.log +3 -0
- package/.playwright-mcp/console-2026-04-05T02-26-51-909Z.log +76 -0
- package/.playwright-mcp/page-2026-04-04T01-41-10-816Z.yml +1 -0
- package/.playwright-mcp/page-2026-04-04T01-41-29-141Z.yml +77 -0
- package/.playwright-mcp/page-2026-04-04T01-41-42-633Z.yml +190 -0
- package/.playwright-mcp/page-2026-04-04T01-42-03-929Z.yml +262 -0
- package/.playwright-mcp/page-2026-04-04T02-12-54-813Z.yml +6 -0
- package/.playwright-mcp/page-2026-04-04T02-14-58-600Z.yml +190 -0
- package/.playwright-mcp/page-2026-04-04T02-15-03-923Z.yml +190 -0
- package/.playwright-mcp/page-2026-04-04T02-15-07-426Z.yml +190 -0
- package/.playwright-mcp/page-2026-04-04T02-15-25-729Z.yml +262 -0
- package/.playwright-mcp/page-2026-04-04T02-16-22-984Z.yml +262 -0
- package/.playwright-mcp/page-2026-04-04T02-17-00-599Z.yml +190 -0
- package/.playwright-mcp/page-2026-04-04T02-17-50-874Z.yml +190 -0
- package/.playwright-mcp/page-2026-04-05T02-26-55-570Z.yml +6 -0
- package/AGENTS.md +48 -0
- package/CHANGELOG.md +131 -0
- package/README.md +552 -0
- package/assets/agent-mode.png +0 -0
- package/assets/categorize.png +0 -0
- package/assets/dashboard.png +0 -0
- package/assets/endpoint-proxy.png +0 -0
- package/assets/icon.png +0 -0
- package/assets/mcp-generate-image.png +0 -0
- package/assets/mcp-understand-image.png +0 -0
- package/assets/peek-token-flow.png +0 -0
- package/assets/playground.png +0 -0
- package/assets/sankey.png +0 -0
- package/cli/index.ts +2805 -0
- package/cli/legacyRewrite.ts +108 -0
- package/cli/modelRef.ts +24 -0
- package/dist/cli/index.js +2536 -0
- package/dist/cli/legacyRewrite.js +92 -0
- package/dist/cli/modelRef.js +20 -0
- package/dist/src/benchmark/artifacts.js +131 -0
- package/dist/src/benchmark/capabilityClassifier.js +81 -0
- package/dist/src/benchmark/capabilityStore.js +144 -0
- package/dist/src/benchmark/config.js +238 -0
- package/dist/src/benchmark/gates.js +118 -0
- package/dist/src/benchmark/jobs.js +252 -0
- package/dist/src/benchmark/runner.js +1847 -0
- package/dist/src/benchmark/schema.js +353 -0
- package/dist/src/benchmark/suites.js +314 -0
- package/dist/src/benchmark/tinyQaDataset.js +422 -0
- package/dist/src/benchmark/types.js +25 -0
- package/dist/src/config.js +47 -0
- package/dist/src/index.js +178 -0
- package/dist/src/mcp/client.js +215 -0
- package/dist/src/mcp/discovery.js +226 -0
- package/dist/src/mcp/policy.js +65 -0
- package/dist/src/mcp/registry.js +129 -0
- package/dist/src/mcp/service.js +460 -0
- package/dist/src/middleware/auth.js +179 -0
- package/dist/src/middleware/requestCapture.js +192 -0
- package/dist/src/middleware/requestStats.js +118 -0
- package/dist/src/pools/builder.js +132 -0
- package/dist/src/pools/repository.js +69 -0
- package/dist/src/pools/scheduler.js +360 -0
- package/dist/src/pools/types.js +2 -0
- package/dist/src/protocols/adapters/dashscope.js +267 -0
- package/dist/src/protocols/adapters/inferenceV2.js +346 -0
- package/dist/src/protocols/adapters/openai.js +27 -0
- package/dist/src/protocols/registry.js +99 -0
- package/dist/src/protocols/types.js +2 -0
- package/dist/src/providers/health.js +153 -0
- package/dist/src/providers/importer.js +289 -0
- package/dist/src/providers/modelRegistry.js +313 -0
- package/dist/src/providers/repository.js +361 -0
- package/dist/src/providers/types.js +2 -0
- package/dist/src/routes/admin.js +531 -0
- package/dist/src/routes/audio.js +295 -0
- package/dist/src/routes/chat.js +240 -0
- package/dist/src/routes/embeddings.js +157 -0
- package/dist/src/routes/images.js +288 -0
- package/dist/src/routes/mcp.js +256 -0
- package/dist/src/routes/mcpService.js +100 -0
- package/dist/src/routes/models.js +48 -0
- package/dist/src/routes/responses.js +711 -0
- package/dist/src/routes/sessions.js +450 -0
- package/dist/src/routes/stats.js +270 -0
- package/dist/src/routes/ui.js +97 -0
- package/dist/src/routes/videos.js +107 -0
- package/dist/src/routing/router.js +338 -0
- package/dist/src/services/imageGeneration.js +280 -0
- package/dist/src/services/imageUnderstanding.js +352 -0
- package/dist/src/services/videoGeneration.js +79 -0
- package/dist/src/storage/captureRepository.js +1591 -0
- package/dist/src/storage/files.js +157 -0
- package/dist/src/storage/imageCache.js +346 -0
- package/dist/src/storage/repositories.js +388 -0
- package/dist/src/storage/sessionRepository.js +370 -0
- package/dist/src/storage/statsRepository.js +204 -0
- package/dist/src/transport/httpClient.js +126 -0
- package/dist/src/types.js +2 -0
- package/dist/src/utils/messageMedia.js +285 -0
- package/dist/src/utils/modelCapabilities.js +108 -0
- package/dist/src/utils/modelDiscovery.js +170 -0
- package/dist/src/version.js +5 -0
- package/dist/src/workers/captureRetention.js +25 -0
- package/dist/src/workers/configWatcher.js +91 -0
- package/dist/src/workers/healthChecker.js +21 -0
- package/dist/src/workers/statsRotation.js +41 -0
- package/docs/LLM/output_schema.md +312 -0
- package/docs/benchmark.md +208 -0
- package/docs/mcp-guidelines.md +125 -0
- package/docs/mcp-service.md +178 -0
- package/docs/opencode.md +86 -0
- package/docs/providers.md +79 -0
- package/examples/benchmark.config.yaml +28 -0
- package/examples/providers/alibaba-dashscope.yaml +88 -0
- package/examples/providers/alibaba-llm.yaml +64 -0
- package/examples/providers/alibaba-registry.yaml +7 -0
- package/examples/providers/inference-v2-ray.yaml +29 -0
- package/examples/scenarios/assets/omni-call-sample.wav +0 -0
- package/examples/scenarios/custom.jsonl +5 -0
- package/examples/scenarios/custom.yaml +40 -0
- package/model-form-v2.png +0 -0
- package/package.json +66 -0
- package/provider-form-v2.png +0 -0
- package/provider-form.png +0 -0
- package/scripts/manual-test.sh +11 -0
- package/scripts/version-from-git.js +23 -0
- package/src/benchmark/artifacts.ts +149 -0
- package/src/benchmark/capabilityClassifier.ts +99 -0
- package/src/benchmark/capabilityStore.ts +174 -0
- package/src/benchmark/config.ts +337 -0
- package/src/benchmark/gates.ts +164 -0
- package/src/benchmark/jobs.ts +312 -0
- package/src/benchmark/runner.ts +2519 -0
- package/src/benchmark/schema.ts +443 -0
- package/src/benchmark/suites.ts +323 -0
- package/src/benchmark/tinyQaDataset.ts +428 -0
- package/src/benchmark/types.ts +442 -0
- package/src/config.ts +44 -0
- package/src/index.ts +195 -0
- package/src/mcp/client.ts +305 -0
- package/src/mcp/discovery.ts +266 -0
- package/src/mcp/policy.ts +105 -0
- package/src/mcp/registry.ts +164 -0
- package/src/mcp/service.ts +611 -0
- package/src/middleware/auth.ts +251 -0
- package/src/middleware/requestCapture.ts +245 -0
- package/src/middleware/requestStats.ts +163 -0
- package/src/pools/builder.ts +159 -0
- package/src/pools/repository.ts +71 -0
- package/src/pools/scheduler.ts +425 -0
- package/src/pools/types.ts +117 -0
- package/src/protocols/adapters/dashscope.ts +335 -0
- package/src/protocols/adapters/inferenceV2.ts +428 -0
- package/src/protocols/adapters/openai.ts +32 -0
- package/src/protocols/registry.ts +117 -0
- package/src/protocols/types.ts +81 -0
- package/src/providers/health.ts +207 -0
- package/src/providers/importer.ts +402 -0
- package/src/providers/modelRegistry.ts +415 -0
- package/src/providers/repository.ts +439 -0
- package/src/providers/types.ts +113 -0
- package/src/routes/admin.ts +666 -0
- package/src/routes/audio.ts +372 -0
- package/src/routes/chat.ts +301 -0
- package/src/routes/embeddings.ts +197 -0
- package/src/routes/images.ts +356 -0
- package/src/routes/mcp.ts +320 -0
- package/src/routes/mcpService.ts +114 -0
- package/src/routes/models.ts +50 -0
- package/src/routes/responses.ts +872 -0
- package/src/routes/sessions.ts +558 -0
- package/src/routes/stats.ts +312 -0
- package/src/routes/ui.ts +96 -0
- package/src/routes/videos.ts +132 -0
- package/src/routing/router.ts +501 -0
- package/src/services/imageGeneration.ts +396 -0
- package/src/services/imageUnderstanding.ts +449 -0
- package/src/services/videoGeneration.ts +127 -0
- package/src/storage/captureRepository.ts +1835 -0
- package/src/storage/files.ts +178 -0
- package/src/storage/imageCache.ts +405 -0
- package/src/storage/repositories.ts +494 -0
- package/src/storage/sessionRepository.ts +419 -0
- package/src/storage/statsRepository.ts +238 -0
- package/src/transport/httpClient.ts +145 -0
- package/src/types.ts +322 -0
- package/src/utils/messageMedia.ts +293 -0
- package/src/utils/modelCapabilities.ts +161 -0
- package/src/utils/modelDiscovery.ts +203 -0
- package/src/workers/captureRetention.ts +25 -0
- package/src/workers/configWatcher.ts +115 -0
- package/src/workers/healthChecker.ts +22 -0
- package/src/workers/statsRotation.ts +49 -0
- package/tests/benchmarkAdminRoutes.test.ts +82 -0
- package/tests/benchmarkBasics.test.ts +116 -0
- package/tests/captureAdminRoutes.test.ts +420 -0
- package/tests/captureRepository.test.ts +797 -0
- package/tests/cliLegacyRewrite.test.ts +45 -0
- package/tests/imageGeneration.service.test.ts +107 -0
- package/tests/imageUnderstanding.service.test.ts +123 -0
- package/tests/mcpPolicy.test.ts +105 -0
- package/tests/mcpService.test.ts +1245 -0
- package/tests/modelRef.test.ts +23 -0
- package/tests/modelsRoutes.test.ts +154 -0
- package/tests/sessionMediaCache.test.ts +167 -0
- package/tests/statsRoutes.test.ts +323 -0
- package/tsconfig.json +15 -0
- package/ui/index.html +16 -0
- package/ui/package-lock.json +8521 -0
- package/ui/package.json +52 -0
- package/ui/postcss.config.js +6 -0
- package/ui/public/assets/apple-touch-icon.png +0 -0
- package/ui/public/assets/favicon-16.png +0 -0
- package/ui/public/assets/favicon-32.png +0 -0
- package/ui/public/assets/icon-192.png +0 -0
- package/ui/public/assets/icon-512.png +0 -0
- package/ui/src/App.tsx +27 -0
- package/ui/src/api/client.ts +1503 -0
- package/ui/src/components/EndpointUsageGuide.tsx +361 -0
- package/ui/src/components/Layout.tsx +124 -0
- package/ui/src/components/MessageContent.tsx +365 -0
- package/ui/src/components/ToolCallMessage.tsx +179 -0
- package/ui/src/components/ToolPicker.tsx +442 -0
- package/ui/src/components/messageContentParser.test.ts +41 -0
- package/ui/src/components/messageContentParser.ts +73 -0
- package/ui/src/components/thinkingPreview.test.ts +27 -0
- package/ui/src/components/thinkingPreview.ts +15 -0
- package/ui/src/components/toMermaidSankey.test.ts +78 -0
- package/ui/src/components/toMermaidSankey.ts +56 -0
- package/ui/src/components/ui/button.tsx +58 -0
- package/ui/src/components/ui/input.tsx +21 -0
- package/ui/src/components/ui/textarea.tsx +21 -0
- package/ui/src/lib/utils.ts +6 -0
- package/ui/src/main.tsx +9 -0
- package/ui/src/pages/AgentPlayground.tsx +2010 -0
- package/ui/src/pages/Benchmark.tsx +988 -0
- package/ui/src/pages/Dashboard.tsx +581 -0
- package/ui/src/pages/Peek.tsx +962 -0
- package/ui/src/pages/Settings.tsx +2013 -0
- package/ui/src/pages/agentPlaygroundPayload.test.ts +109 -0
- package/ui/src/pages/agentPlaygroundPayload.ts +97 -0
- package/ui/src/pages/agentThinkingContent.test.ts +50 -0
- package/ui/src/pages/agentThinkingContent.ts +57 -0
- package/ui/src/pages/dashboardTokenUsage.test.ts +66 -0
- package/ui/src/pages/dashboardTokenUsage.ts +36 -0
- package/ui/src/pages/imageUpload.test.ts +39 -0
- package/ui/src/pages/imageUpload.ts +71 -0
- package/ui/src/pages/peekFilters.test.ts +29 -0
- package/ui/src/pages/peekFilters.ts +13 -0
- package/ui/src/pages/peekMedia.test.ts +58 -0
- package/ui/src/pages/peekMedia.ts +148 -0
- package/ui/src/pages/sessionAutoTitle.test.ts +128 -0
- package/ui/src/pages/sessionAutoTitle.ts +106 -0
- package/ui/src/stores/settings.ts +58 -0
- package/ui/src/styles/globals.css +223 -0
- package/ui/src/vite-env.d.ts +8 -0
- package/ui/tailwind.config.js +106 -0
- package/ui/tsconfig.json +32 -0
- package/ui/vite.config.ts +37 -0
|
@@ -0,0 +1,2010 @@
|
|
|
1
|
+
import { useState, useRef, useEffect, useCallback } from 'react'
|
|
2
|
+
import {
|
|
3
|
+
Send, ImagePlus, Loader2, Bot, User, Sparkles, Plus, Trash2,
|
|
4
|
+
MessageSquare, ChevronRight, X, Image as ImageIcon, Wrench,
|
|
5
|
+
ToggleLeft, ToggleRight, StopCircle, Mic, PhoneCall, PhoneOff
|
|
6
|
+
} from 'lucide-react'
|
|
7
|
+
import { Button } from '@/components/ui/button'
|
|
8
|
+
import { Textarea } from '@/components/ui/textarea'
|
|
9
|
+
import { ToolPicker } from '@/components/ToolPicker'
|
|
10
|
+
import { ToolCallMessage, type ToolCall } from '@/components/ToolCallMessage'
|
|
11
|
+
import { MessageContent } from '@/components/MessageContent'
|
|
12
|
+
import { cn } from '@/lib/utils'
|
|
13
|
+
import {
|
|
14
|
+
streamChatCompletion,
|
|
15
|
+
createChatCompletionRaw,
|
|
16
|
+
listModels,
|
|
17
|
+
listSessions,
|
|
18
|
+
getSession,
|
|
19
|
+
createSession,
|
|
20
|
+
deleteSession,
|
|
21
|
+
addMessageToSession,
|
|
22
|
+
autoTitleSession,
|
|
23
|
+
listMcpTools,
|
|
24
|
+
executeMcpTool,
|
|
25
|
+
BUILTIN_SERVER_ID,
|
|
26
|
+
generateImage,
|
|
27
|
+
storeImage as cacheImage,
|
|
28
|
+
storeMedia,
|
|
29
|
+
normalizeContentMedia,
|
|
30
|
+
normalizeSessionMessageMedia,
|
|
31
|
+
resolveMediaUrl,
|
|
32
|
+
ApiError,
|
|
33
|
+
type ChatMessage as ApiChatMessage,
|
|
34
|
+
type SessionListItem,
|
|
35
|
+
type McpTool,
|
|
36
|
+
type Model,
|
|
37
|
+
} from '@/api/client'
|
|
38
|
+
import { loadSettings, IMAGE_SIZE_OPTIONS, type ImageSize } from '@/stores/settings'
|
|
39
|
+
import {
|
|
40
|
+
buildUserPayload,
|
|
41
|
+
findNonDataImageUrls,
|
|
42
|
+
toApiMessage,
|
|
43
|
+
} from './agentPlaygroundPayload'
|
|
44
|
+
import {
|
|
45
|
+
applyAutoTitleToSessions,
|
|
46
|
+
createDeferredAutoTitleCandidate,
|
|
47
|
+
flushDeferredAutoTitle,
|
|
48
|
+
} from './sessionAutoTitle'
|
|
49
|
+
import {
|
|
50
|
+
applyThinkingChunk,
|
|
51
|
+
createThinkingStreamState,
|
|
52
|
+
toDisplayContent,
|
|
53
|
+
toFinalContent,
|
|
54
|
+
} from './agentThinkingContent'
|
|
55
|
+
import { compressImageFileForUpload, fileToDataUrl } from './imageUpload'
|
|
56
|
+
|
|
57
|
+
// Content can be a string or array of content parts (multimodal)
|
|
58
|
+
type ContentPart =
|
|
59
|
+
| { type: 'text'; text: string }
|
|
60
|
+
| { type: 'image_url'; image_url: { url: string } }
|
|
61
|
+
| { type: 'input_audio'; input_audio: { url?: string; data?: string; format?: string } }
|
|
62
|
+
| { type: 'audio'; audio: { url?: string; data?: string; format?: string } }
|
|
63
|
+
|
|
64
|
+
interface Message {
|
|
65
|
+
id: string
|
|
66
|
+
role: 'user' | 'assistant' | 'system' | 'tool'
|
|
67
|
+
content: string | ContentPart[] | null
|
|
68
|
+
images?: string[]
|
|
69
|
+
requestImages?: string[]
|
|
70
|
+
toolCalls?: ToolCall[]
|
|
71
|
+
toolCallId?: string
|
|
72
|
+
model?: string
|
|
73
|
+
createdAt: Date
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
// Helper to extract text from content (string or multimodal array)
|
|
77
|
+
function getTextContent(content: string | ContentPart[] | null): string {
|
|
78
|
+
if (!content) return ''
|
|
79
|
+
if (typeof content === 'string') return content
|
|
80
|
+
return content
|
|
81
|
+
.filter((p): p is { type: 'text'; text: string } => p.type === 'text')
|
|
82
|
+
.map(p => p.text)
|
|
83
|
+
.join('')
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
// Helper to extract images from multimodal content
|
|
87
|
+
function getImageUrls(content: string | ContentPart[] | null): string[] {
|
|
88
|
+
if (!content || typeof content === 'string') return []
|
|
89
|
+
return content
|
|
90
|
+
.filter((p): p is { type: 'image_url'; image_url: { url: string } } => p.type === 'image_url')
|
|
91
|
+
.map(p => resolveMediaUrl(p.image_url.url))
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
function getAudioUrls(content: string | ContentPart[] | null): string[] {
|
|
95
|
+
if (!content || typeof content === 'string') return []
|
|
96
|
+
const urls: string[] = []
|
|
97
|
+
for (const part of content) {
|
|
98
|
+
if (part.type === 'audio' && part.audio?.url) {
|
|
99
|
+
urls.push(resolveMediaUrl(part.audio.url))
|
|
100
|
+
}
|
|
101
|
+
if (part.type === 'input_audio' && part.input_audio?.url) {
|
|
102
|
+
urls.push(resolveMediaUrl(part.input_audio.url))
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
return urls
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
function formatModelTag(model: Model): string {
|
|
109
|
+
const caps = model.capabilities
|
|
110
|
+
if (caps && caps.input.length > 0 && caps.output.length > 0) {
|
|
111
|
+
return `[${caps.input.join('+')}->${caps.output.join('+')}]`
|
|
112
|
+
}
|
|
113
|
+
if (model.endpoint_type && model.endpoint_type !== 'llm') {
|
|
114
|
+
return `[${model.endpoint_type}]`
|
|
115
|
+
}
|
|
116
|
+
return ''
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
function modelHealthStatus(model: Model): 'up' | 'down' | 'unknown' {
|
|
120
|
+
return model.waypoi_health?.status ?? 'unknown'
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
function isModelSelectable(model: Model): boolean {
|
|
124
|
+
return modelHealthStatus(model) !== 'down'
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
function firstSelectableModelId(models: Model[]): string {
|
|
128
|
+
return models.find(isModelSelectable)?.id ?? ''
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
// Maximum tool iterations per user message to prevent infinite loops
|
|
132
|
+
const MAX_TOOL_ITERATIONS = 10
|
|
133
|
+
|
|
134
|
+
type GenerationParamsDraft = {
|
|
135
|
+
temperature: string
|
|
136
|
+
topP: string
|
|
137
|
+
maxTokens: string
|
|
138
|
+
presencePenalty: string
|
|
139
|
+
frequencyPenalty: string
|
|
140
|
+
seed: string
|
|
141
|
+
stop: string
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
type GenerationParamsPayload = {
|
|
145
|
+
temperature?: number
|
|
146
|
+
top_p?: number
|
|
147
|
+
max_tokens?: number
|
|
148
|
+
presence_penalty?: number
|
|
149
|
+
frequency_penalty?: number
|
|
150
|
+
seed?: number
|
|
151
|
+
stop?: string | string[]
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
type NumericGenerationParamKey =
|
|
155
|
+
| 'temperature'
|
|
156
|
+
| 'top_p'
|
|
157
|
+
| 'max_tokens'
|
|
158
|
+
| 'presence_penalty'
|
|
159
|
+
| 'frequency_penalty'
|
|
160
|
+
| 'seed'
|
|
161
|
+
|
|
162
|
+
function parseGenerationParams(draft: GenerationParamsDraft): { payload: GenerationParamsPayload; error?: string } {
|
|
163
|
+
const payload: GenerationParamsPayload = {}
|
|
164
|
+
const parseNumber = (
|
|
165
|
+
raw: string,
|
|
166
|
+
label: string,
|
|
167
|
+
key: NumericGenerationParamKey,
|
|
168
|
+
opts?: { min?: number; max?: number; integer?: boolean }
|
|
169
|
+
): string | null => {
|
|
170
|
+
const trimmed = raw.trim()
|
|
171
|
+
if (!trimmed) return null
|
|
172
|
+
const parsed = Number(trimmed)
|
|
173
|
+
if (!Number.isFinite(parsed)) {
|
|
174
|
+
return `${label} must be a valid number.`
|
|
175
|
+
}
|
|
176
|
+
if (opts?.integer && !Number.isInteger(parsed)) {
|
|
177
|
+
return `${label} must be an integer.`
|
|
178
|
+
}
|
|
179
|
+
if (typeof opts?.min === 'number' && parsed < opts.min) {
|
|
180
|
+
return `${label} must be >= ${opts.min}.`
|
|
181
|
+
}
|
|
182
|
+
if (typeof opts?.max === 'number' && parsed > opts.max) {
|
|
183
|
+
return `${label} must be <= ${opts.max}.`
|
|
184
|
+
}
|
|
185
|
+
payload[key] = parsed
|
|
186
|
+
return null
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
const errors = [
|
|
190
|
+
parseNumber(draft.temperature, 'Temperature', 'temperature'),
|
|
191
|
+
parseNumber(draft.topP, 'Top P', 'top_p', { min: 0, max: 1 }),
|
|
192
|
+
parseNumber(draft.maxTokens, 'Max Tokens', 'max_tokens', { min: 1, integer: true }),
|
|
193
|
+
parseNumber(draft.presencePenalty, 'Presence Penalty', 'presence_penalty', { min: -2, max: 2 }),
|
|
194
|
+
parseNumber(draft.frequencyPenalty, 'Frequency Penalty', 'frequency_penalty', { min: -2, max: 2 }),
|
|
195
|
+
parseNumber(draft.seed, 'Seed', 'seed', { min: 0, integer: true }),
|
|
196
|
+
].filter((error): error is string => Boolean(error))
|
|
197
|
+
|
|
198
|
+
const stops = draft.stop
|
|
199
|
+
.split(',')
|
|
200
|
+
.map((value) => value.trim())
|
|
201
|
+
.filter(Boolean)
|
|
202
|
+
if (stops.length === 1) payload.stop = stops[0]
|
|
203
|
+
if (stops.length > 1) payload.stop = stops
|
|
204
|
+
|
|
205
|
+
return { payload, error: errors[0] }
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
export function AgentPlayground() {
|
|
209
|
+
const MAX_INPUT_LINES = 10
|
|
210
|
+
// Session state
|
|
211
|
+
const [sessions, setSessions] = useState<SessionListItem[]>([])
|
|
212
|
+
const [activeSessionId, setActiveSessionId] = useState<string | null>(null)
|
|
213
|
+
const [activeSessionStorageVersion, setActiveSessionStorageVersion] = useState<number>(2)
|
|
214
|
+
const [sessionName, setSessionName] = useState('')
|
|
215
|
+
const [titleGenerationSessionId, setTitleGenerationSessionId] = useState<string | null>(null)
|
|
216
|
+
const [sidebarCollapsed, setSidebarCollapsed] = useState(false)
|
|
217
|
+
|
|
218
|
+
// Chat state
|
|
219
|
+
const [messages, setMessages] = useState<Message[]>([])
|
|
220
|
+
const [input, setInput] = useState('')
|
|
221
|
+
const [isLoading, setIsLoading] = useState(false)
|
|
222
|
+
const [selectedModel, setSelectedModel] = useState<string>('')
|
|
223
|
+
const [models, setModels] = useState<Model[]>([])
|
|
224
|
+
|
|
225
|
+
const selectedModelSupportsImageOutput = (): boolean => {
|
|
226
|
+
const model = models.find(m => m.id === selectedModel)
|
|
227
|
+
if (!model) return false
|
|
228
|
+
if (model.capabilities?.output?.includes('image')) return true
|
|
229
|
+
return (model.endpoint_type ?? 'llm') === 'diffusion'
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
const selectedModelSupportsCall = (): boolean => {
|
|
233
|
+
const model = models.find(m => m.id === selectedModel)
|
|
234
|
+
if (!model?.capabilities) return false
|
|
235
|
+
return model.capabilities.input.includes('audio') && model.capabilities.output.includes('audio')
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
const modelSupportsCall = selectedModelSupportsCall()
|
|
239
|
+
const selectedModelConfig = models.find((model) => model.id === selectedModel)
|
|
240
|
+
const selectedModelIsSelectable = selectedModelConfig ? isModelSelectable(selectedModelConfig) : false
|
|
241
|
+
|
|
242
|
+
// Image input state
|
|
243
|
+
const [pendingImages, setPendingImages] = useState<string[]>([])
|
|
244
|
+
const [pendingAudio, setPendingAudio] = useState<string | null>(null)
|
|
245
|
+
const [pendingAudioMimeType, setPendingAudioMimeType] = useState<string | undefined>(undefined)
|
|
246
|
+
const [isDragging, setIsDragging] = useState(false)
|
|
247
|
+
const [callModeEnabled, setCallModeEnabled] = useState(false)
|
|
248
|
+
const [callStatus, setCallStatus] = useState<'idle' | 'recording' | 'sending' | 'playing'>('idle')
|
|
249
|
+
const [callError, setCallError] = useState<string | null>(null)
|
|
250
|
+
const [showGenerationParams, setShowGenerationParams] = useState(false)
|
|
251
|
+
const [generationParams, setGenerationParams] = useState<GenerationParamsDraft>({
|
|
252
|
+
temperature: '',
|
|
253
|
+
topP: '',
|
|
254
|
+
maxTokens: '',
|
|
255
|
+
presencePenalty: '',
|
|
256
|
+
frequencyPenalty: '',
|
|
257
|
+
seed: '',
|
|
258
|
+
stop: '',
|
|
259
|
+
})
|
|
260
|
+
|
|
261
|
+
// Image generation settings
|
|
262
|
+
const [imageSize, setImageSize] = useState<ImageSize>(() => loadSettings().defaultImageSize)
|
|
263
|
+
|
|
264
|
+
// Agentic mode state
|
|
265
|
+
const [agentModeEnabled, setAgentModeEnabled] = useState(false)
|
|
266
|
+
const [showToolPicker, setShowToolPicker] = useState(false)
|
|
267
|
+
const [selectedTools, setSelectedTools] = useState<Set<string>>(new Set())
|
|
268
|
+
const [availableTools, setAvailableTools] = useState<McpTool[]>([])
|
|
269
|
+
const [currentIteration, setCurrentIteration] = useState(0)
|
|
270
|
+
const [isExecutingTools, setIsExecutingTools] = useState(false)
|
|
271
|
+
|
|
272
|
+
const messagesEndRef = useRef<HTMLDivElement>(null)
|
|
273
|
+
const messagesContainerRef = useRef<HTMLDivElement>(null)
|
|
274
|
+
const abortControllerRef = useRef<AbortController | null>(null)
|
|
275
|
+
const stopAgentRef = useRef(false)
|
|
276
|
+
const fileInputRef = useRef<HTMLInputElement>(null)
|
|
277
|
+
const isStreamingRef = useRef(false)
|
|
278
|
+
const mediaRecorderRef = useRef<MediaRecorder | null>(null)
|
|
279
|
+
const mediaStreamRef = useRef<MediaStream | null>(null)
|
|
280
|
+
const recordingTimerRef = useRef<number | null>(null)
|
|
281
|
+
const callAudioRef = useRef<HTMLAudioElement | null>(null)
|
|
282
|
+
const inputRef = useRef<HTMLTextAreaElement>(null)
|
|
283
|
+
const pendingAutoTitleRef = useRef<{ sessionId: string; seedText: string } | null>(null)
|
|
284
|
+
|
|
285
|
+
const resizeInput = useCallback(() => {
|
|
286
|
+
const textarea = inputRef.current
|
|
287
|
+
if (!textarea) return
|
|
288
|
+
|
|
289
|
+
textarea.style.height = 'auto'
|
|
290
|
+
const lineHeight = Number.parseFloat(window.getComputedStyle(textarea).lineHeight) || 20
|
|
291
|
+
const maxHeight = lineHeight * MAX_INPUT_LINES
|
|
292
|
+
const nextHeight = Math.min(textarea.scrollHeight, maxHeight)
|
|
293
|
+
textarea.style.height = `${nextHeight}px`
|
|
294
|
+
textarea.style.overflowY = textarea.scrollHeight > maxHeight ? 'auto' : 'hidden'
|
|
295
|
+
}, [MAX_INPUT_LINES])
|
|
296
|
+
|
|
297
|
+
// Load models on mount
|
|
298
|
+
useEffect(() => {
|
|
299
|
+
async function loadModels() {
|
|
300
|
+
try {
|
|
301
|
+
const response = await listModels()
|
|
302
|
+
setModels(response.data)
|
|
303
|
+
if (response.data.length > 0 && !selectedModel) {
|
|
304
|
+
setSelectedModel(firstSelectableModelId(response.data))
|
|
305
|
+
}
|
|
306
|
+
} catch (error) {
|
|
307
|
+
console.error('Failed to load models:', error)
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
loadModels()
|
|
311
|
+
}, [selectedModel])
|
|
312
|
+
|
|
313
|
+
// Ensure selected model stays selectable after health refreshes/session restore.
|
|
314
|
+
useEffect(() => {
|
|
315
|
+
if (models.length === 0) return
|
|
316
|
+
const current = models.find((model) => model.id === selectedModel)
|
|
317
|
+
if (!current || !isModelSelectable(current)) {
|
|
318
|
+
setSelectedModel(firstSelectableModelId(models))
|
|
319
|
+
}
|
|
320
|
+
}, [models, selectedModel])
|
|
321
|
+
|
|
322
|
+
// Load sessions on mount
|
|
323
|
+
useEffect(() => {
|
|
324
|
+
async function loadSessions() {
|
|
325
|
+
try {
|
|
326
|
+
const response = await listSessions()
|
|
327
|
+
setSessions(response.data)
|
|
328
|
+
} catch (error) {
|
|
329
|
+
console.error('Failed to load sessions:', error)
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
loadSessions()
|
|
333
|
+
}, [])
|
|
334
|
+
|
|
335
|
+
// Load tools when agent mode is enabled
|
|
336
|
+
useEffect(() => {
|
|
337
|
+
if (agentModeEnabled) {
|
|
338
|
+
loadTools()
|
|
339
|
+
}
|
|
340
|
+
}, [agentModeEnabled])
|
|
341
|
+
|
|
342
|
+
useEffect(() => {
|
|
343
|
+
resizeInput()
|
|
344
|
+
}, [input, resizeInput])
|
|
345
|
+
|
|
346
|
+
const loadTools = async () => {
|
|
347
|
+
try {
|
|
348
|
+
const response = await listMcpTools()
|
|
349
|
+
setAvailableTools(response.data)
|
|
350
|
+
// Auto-select built-in tools the first time agent mode is enabled
|
|
351
|
+
setSelectedTools((prev) => {
|
|
352
|
+
if (prev.size > 0) return prev
|
|
353
|
+
const builtinNames = response.data
|
|
354
|
+
.filter((t) => t.serverId === BUILTIN_SERVER_ID)
|
|
355
|
+
.map((t) => t.name)
|
|
356
|
+
return builtinNames.length > 0 ? new Set(builtinNames) : prev
|
|
357
|
+
})
|
|
358
|
+
} catch (error) {
|
|
359
|
+
console.error('Failed to load tools:', error)
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
const normalizeMessageForUi = (message: Message): Message => ({
|
|
364
|
+
...message,
|
|
365
|
+
content: normalizeContentMedia(message.content) as Message['content'],
|
|
366
|
+
images: message.images?.map((value) => resolveMediaUrl(value)),
|
|
367
|
+
})
|
|
368
|
+
|
|
369
|
+
// Auto-scroll to bottom - use instant scroll during streaming to prevent jitter
|
|
370
|
+
useEffect(() => {
|
|
371
|
+
const container = messagesContainerRef.current
|
|
372
|
+
if (!container) return
|
|
373
|
+
|
|
374
|
+
// Use instant scroll during streaming, smooth otherwise
|
|
375
|
+
const behavior = isStreamingRef.current ? 'auto' : 'smooth'
|
|
376
|
+
messagesEndRef.current?.scrollIntoView({ behavior })
|
|
377
|
+
}, [messages])
|
|
378
|
+
|
|
379
|
+
// Load session messages when switching
|
|
380
|
+
const loadSession = useCallback(async (sessionId: string) => {
|
|
381
|
+
try {
|
|
382
|
+
const session = await getSession(sessionId)
|
|
383
|
+
setActiveSessionId(session.id)
|
|
384
|
+
setActiveSessionStorageVersion(session.storageVersion ?? 1)
|
|
385
|
+
setSessionName(session.name)
|
|
386
|
+
if (session.model) setSelectedModel(session.model)
|
|
387
|
+
setMessages(session.messages.map(m => {
|
|
388
|
+
const normalized = normalizeSessionMessageMedia(m)
|
|
389
|
+
return ({
|
|
390
|
+
id: crypto.randomUUID(),
|
|
391
|
+
role: normalized.role as Message['role'],
|
|
392
|
+
content: normalizeContentMedia(normalized.content) as Message['content'],
|
|
393
|
+
images: normalized.images,
|
|
394
|
+
createdAt: new Date(m.createdAt ?? m.timestamp ?? new Date().toISOString()),
|
|
395
|
+
})}))
|
|
396
|
+
} catch (error) {
|
|
397
|
+
console.error('Failed to load session:', error)
|
|
398
|
+
}
|
|
399
|
+
}, [])
|
|
400
|
+
|
|
401
|
+
// Create new session
|
|
402
|
+
const handleNewSession = async () => {
|
|
403
|
+
try {
|
|
404
|
+
const session = await createSession(undefined, selectedModel)
|
|
405
|
+
setSessions(prev => [{
|
|
406
|
+
id: session.id,
|
|
407
|
+
name: session.name,
|
|
408
|
+
model: session.model,
|
|
409
|
+
messageCount: 0,
|
|
410
|
+
createdAt: session.createdAt,
|
|
411
|
+
updatedAt: session.updatedAt,
|
|
412
|
+
}, ...prev])
|
|
413
|
+
setActiveSessionId(session.id)
|
|
414
|
+
setActiveSessionStorageVersion(session.storageVersion ?? 2)
|
|
415
|
+
setSessionName(session.name)
|
|
416
|
+
pendingAutoTitleRef.current = null
|
|
417
|
+
setMessages([])
|
|
418
|
+
} catch (error) {
|
|
419
|
+
console.error('Failed to create session:', error)
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
// Delete session
|
|
424
|
+
const handleDeleteSession = async (sessionId: string) => {
|
|
425
|
+
try {
|
|
426
|
+
await deleteSession(sessionId)
|
|
427
|
+
setSessions(prev => prev.filter(s => s.id !== sessionId))
|
|
428
|
+
if (titleGenerationSessionId === sessionId) {
|
|
429
|
+
setTitleGenerationSessionId(null)
|
|
430
|
+
}
|
|
431
|
+
if (activeSessionId === sessionId) {
|
|
432
|
+
setActiveSessionId(null)
|
|
433
|
+
setActiveSessionStorageVersion(2)
|
|
434
|
+
setMessages([])
|
|
435
|
+
setSessionName('')
|
|
436
|
+
pendingAutoTitleRef.current = null
|
|
437
|
+
}
|
|
438
|
+
} catch (error) {
|
|
439
|
+
console.error('Failed to delete session:', error)
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
// Image handling
|
|
444
|
+
const processImages = useCallback((files: FileList | File[]) => {
|
|
445
|
+
Array.from(files).forEach(async (file) => {
|
|
446
|
+
if (!file.type.startsWith('image/')) return
|
|
447
|
+
try {
|
|
448
|
+
const base64 = await compressImageFileForUpload(file)
|
|
449
|
+
setPendingImages(prev => [...prev, base64])
|
|
450
|
+
} catch (error) {
|
|
451
|
+
console.warn('Failed to compress image, using raw data URL:', error)
|
|
452
|
+
const fallback = await fileToDataUrl(file)
|
|
453
|
+
setPendingImages(prev => [...prev, fallback])
|
|
454
|
+
}
|
|
455
|
+
})
|
|
456
|
+
}, [])
|
|
457
|
+
|
|
458
|
+
const processAudioFile = useCallback((file: File) => {
|
|
459
|
+
if (!file.type.startsWith('audio/')) return
|
|
460
|
+
const reader = new FileReader()
|
|
461
|
+
reader.onload = (e) => {
|
|
462
|
+
const base64 = e.target?.result as string
|
|
463
|
+
setPendingAudio(base64)
|
|
464
|
+
setPendingAudioMimeType(file.type || undefined)
|
|
465
|
+
setCallError(null)
|
|
466
|
+
}
|
|
467
|
+
reader.readAsDataURL(file)
|
|
468
|
+
}, [])
|
|
469
|
+
|
|
470
|
+
const processSelectedFiles = useCallback((files: FileList | File[]) => {
|
|
471
|
+
const entries = Array.from(files)
|
|
472
|
+
const imageFiles = entries.filter((file) => file.type.startsWith('image/'))
|
|
473
|
+
if (imageFiles.length > 0) {
|
|
474
|
+
processImages(imageFiles)
|
|
475
|
+
}
|
|
476
|
+
if (callModeEnabled) {
|
|
477
|
+
const audioFile = entries.find((file) => file.type.startsWith('audio/'))
|
|
478
|
+
if (audioFile) {
|
|
479
|
+
processAudioFile(audioFile)
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
}, [callModeEnabled, processAudioFile, processImages])
|
|
483
|
+
|
|
484
|
+
const handleDrop = useCallback((e: React.DragEvent) => {
|
|
485
|
+
e.preventDefault()
|
|
486
|
+
setIsDragging(false)
|
|
487
|
+
if (e.dataTransfer.files.length > 0) {
|
|
488
|
+
processSelectedFiles(e.dataTransfer.files)
|
|
489
|
+
}
|
|
490
|
+
}, [processSelectedFiles])
|
|
491
|
+
|
|
492
|
+
const handlePaste = useCallback((e: React.ClipboardEvent) => {
|
|
493
|
+
const items = e.clipboardData.items
|
|
494
|
+
const imageFiles: File[] = []
|
|
495
|
+
for (let i = 0; i < items.length; i++) {
|
|
496
|
+
if (items[i].type.startsWith('image/')) {
|
|
497
|
+
const file = items[i].getAsFile()
|
|
498
|
+
if (file) imageFiles.push(file)
|
|
499
|
+
} else if (callModeEnabled && items[i].type.startsWith('audio/')) {
|
|
500
|
+
const file = items[i].getAsFile()
|
|
501
|
+
if (file) processAudioFile(file)
|
|
502
|
+
}
|
|
503
|
+
}
|
|
504
|
+
if (imageFiles.length > 0) {
|
|
505
|
+
processImages(imageFiles)
|
|
506
|
+
}
|
|
507
|
+
}, [callModeEnabled, processAudioFile, processImages])
|
|
508
|
+
|
|
509
|
+
const stopPlayback = useCallback(() => {
|
|
510
|
+
if (callAudioRef.current) {
|
|
511
|
+
callAudioRef.current.pause()
|
|
512
|
+
callAudioRef.current.currentTime = 0
|
|
513
|
+
callAudioRef.current = null
|
|
514
|
+
}
|
|
515
|
+
if (callStatus === 'playing') {
|
|
516
|
+
setCallStatus('idle')
|
|
517
|
+
}
|
|
518
|
+
}, [callStatus])
|
|
519
|
+
|
|
520
|
+
const clearRecordingTimer = () => {
|
|
521
|
+
if (recordingTimerRef.current !== null) {
|
|
522
|
+
window.clearTimeout(recordingTimerRef.current)
|
|
523
|
+
recordingTimerRef.current = null
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
const stopMediaStream = () => {
|
|
528
|
+
if (mediaStreamRef.current) {
|
|
529
|
+
mediaStreamRef.current.getTracks().forEach(track => track.stop())
|
|
530
|
+
mediaStreamRef.current = null
|
|
531
|
+
}
|
|
532
|
+
}
|
|
533
|
+
|
|
534
|
+
const stopRecording = useCallback(() => {
|
|
535
|
+
const recorder = mediaRecorderRef.current
|
|
536
|
+
if (!recorder || recorder.state !== 'recording') {
|
|
537
|
+
clearRecordingTimer()
|
|
538
|
+
stopMediaStream()
|
|
539
|
+
setCallStatus('idle')
|
|
540
|
+
return
|
|
541
|
+
}
|
|
542
|
+
recorder.stop()
|
|
543
|
+
clearRecordingTimer()
|
|
544
|
+
}, [])
|
|
545
|
+
|
|
546
|
+
const cancelRecording = useCallback(() => {
|
|
547
|
+
stopRecording()
|
|
548
|
+
setPendingAudio(null)
|
|
549
|
+
setPendingAudioMimeType(undefined)
|
|
550
|
+
setCallError(null)
|
|
551
|
+
}, [stopRecording])
|
|
552
|
+
|
|
553
|
+
const startRecording = useCallback(async () => {
|
|
554
|
+
if (!selectedModelSupportsCall() || isLoading || callStatus === 'sending') {
|
|
555
|
+
return
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
try {
|
|
559
|
+
stopPlayback()
|
|
560
|
+
setCallError(null)
|
|
561
|
+
setCallStatus('recording')
|
|
562
|
+
const stream = await navigator.mediaDevices.getUserMedia({ audio: true })
|
|
563
|
+
mediaStreamRef.current = stream
|
|
564
|
+
let recorder: MediaRecorder
|
|
565
|
+
try {
|
|
566
|
+
recorder = new MediaRecorder(stream, { mimeType: 'audio/webm' })
|
|
567
|
+
} catch {
|
|
568
|
+
recorder = new MediaRecorder(stream)
|
|
569
|
+
}
|
|
570
|
+
mediaRecorderRef.current = recorder
|
|
571
|
+
const chunks: BlobPart[] = []
|
|
572
|
+
|
|
573
|
+
recorder.ondataavailable = (event) => {
|
|
574
|
+
if (event.data.size > 0) {
|
|
575
|
+
chunks.push(event.data)
|
|
576
|
+
}
|
|
577
|
+
}
|
|
578
|
+
|
|
579
|
+
recorder.onstop = async () => {
|
|
580
|
+
stopMediaStream()
|
|
581
|
+
clearRecordingTimer()
|
|
582
|
+
if (chunks.length > 0) {
|
|
583
|
+
const blob = new Blob(chunks, { type: recorder.mimeType || 'audio/webm' })
|
|
584
|
+
const reader = new FileReader()
|
|
585
|
+
reader.onload = () => {
|
|
586
|
+
const result = reader.result as string
|
|
587
|
+
setPendingAudio(result)
|
|
588
|
+
setPendingAudioMimeType(blob.type || 'audio/webm')
|
|
589
|
+
setCallStatus('idle')
|
|
590
|
+
}
|
|
591
|
+
reader.readAsDataURL(blob)
|
|
592
|
+
} else {
|
|
593
|
+
setCallStatus('idle')
|
|
594
|
+
}
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
recorder.start()
|
|
598
|
+
recordingTimerRef.current = window.setTimeout(() => {
|
|
599
|
+
stopRecording()
|
|
600
|
+
}, 60_000)
|
|
601
|
+
} catch (error) {
|
|
602
|
+
console.error('Microphone access failed:', error)
|
|
603
|
+
setCallStatus('idle')
|
|
604
|
+
setCallError('Microphone permission denied or unavailable. You can upload an audio file instead.')
|
|
605
|
+
}
|
|
606
|
+
}, [callStatus, isLoading, modelSupportsCall, stopPlayback, stopRecording])
|
|
607
|
+
|
|
608
|
+
useEffect(() => {
|
|
609
|
+
if (!modelSupportsCall && callModeEnabled) {
|
|
610
|
+
setCallModeEnabled(false)
|
|
611
|
+
setPendingAudio(null)
|
|
612
|
+
setPendingAudioMimeType(undefined)
|
|
613
|
+
setCallStatus('idle')
|
|
614
|
+
setCallError(null)
|
|
615
|
+
}
|
|
616
|
+
}, [callModeEnabled, modelSupportsCall])
|
|
617
|
+
|
|
618
|
+
useEffect(() => {
|
|
619
|
+
return () => {
|
|
620
|
+
clearRecordingTimer()
|
|
621
|
+
stopMediaStream()
|
|
622
|
+
stopPlayback()
|
|
623
|
+
}
|
|
624
|
+
}, [stopPlayback])
|
|
625
|
+
|
|
626
|
+
// Build tools array for the API
|
|
627
|
+
const buildToolsForApi = () => {
|
|
628
|
+
if (!agentModeEnabled || selectedTools.size === 0) return undefined
|
|
629
|
+
|
|
630
|
+
return availableTools
|
|
631
|
+
.filter(t => selectedTools.has(t.name))
|
|
632
|
+
.map(t => ({
|
|
633
|
+
type: 'function' as const,
|
|
634
|
+
function: {
|
|
635
|
+
name: t.name,
|
|
636
|
+
description: t.description || '',
|
|
637
|
+
parameters: t.inputSchema,
|
|
638
|
+
},
|
|
639
|
+
}))
|
|
640
|
+
}
|
|
641
|
+
|
|
642
|
+
// Execute a tool and return result
|
|
643
|
+
const executeToolCall = async (toolCall: ToolCall): Promise<string> => {
|
|
644
|
+
try {
|
|
645
|
+
const args = JSON.parse(toolCall.arguments)
|
|
646
|
+
const result = await executeMcpTool(toolCall.name, args)
|
|
647
|
+
return result.result
|
|
648
|
+
} catch (error) {
|
|
649
|
+
let reason = (error as Error).message
|
|
650
|
+
if (error instanceof ApiError) {
|
|
651
|
+
const body = error.body as { error?: { message?: string } } | undefined
|
|
652
|
+
if (body?.error?.message) {
|
|
653
|
+
reason = body.error.message
|
|
654
|
+
}
|
|
655
|
+
}
|
|
656
|
+
throw new Error(`Tool execution failed: ${reason}`)
|
|
657
|
+
}
|
|
658
|
+
}
|
|
659
|
+
|
|
660
|
+
// Stop the agent loop
|
|
661
|
+
const stopAgent = () => {
|
|
662
|
+
stopAgentRef.current = true
|
|
663
|
+
abortControllerRef.current?.abort()
|
|
664
|
+
}
|
|
665
|
+
|
|
666
|
+
// The main agentic loop
|
|
667
|
+
const runAgentLoop = async (
|
|
668
|
+
conversationHistory: Message[],
|
|
669
|
+
assistantMessageId: string,
|
|
670
|
+
generationPayload: GenerationParamsPayload
|
|
671
|
+
): Promise<void> => {
|
|
672
|
+
let iteration = 0
|
|
673
|
+
let currentHistory = [...conversationHistory]
|
|
674
|
+
|
|
675
|
+
while (iteration < MAX_TOOL_ITERATIONS && !stopAgentRef.current) {
|
|
676
|
+
setCurrentIteration(iteration + 1)
|
|
677
|
+
|
|
678
|
+
// Build messages for API
|
|
679
|
+
const chatMessages = currentHistory.map(m => {
|
|
680
|
+
if (m.role === 'tool' && m.toolCallId) {
|
|
681
|
+
return {
|
|
682
|
+
role: 'tool' as const,
|
|
683
|
+
tool_call_id: m.toolCallId,
|
|
684
|
+
content: m.content,
|
|
685
|
+
}
|
|
686
|
+
}
|
|
687
|
+
return toApiMessage(m)
|
|
688
|
+
}) as ApiChatMessage[]
|
|
689
|
+
warnIfNonDataImageUrls(chatMessages, 'agent-loop')
|
|
690
|
+
|
|
691
|
+
// Stream the response
|
|
692
|
+
abortControllerRef.current = new AbortController()
|
|
693
|
+
isStreamingRef.current = true
|
|
694
|
+
let fullContent = ''
|
|
695
|
+
let toolCallsData: Array<{
|
|
696
|
+
id: string
|
|
697
|
+
function: { name: string; arguments: string }
|
|
698
|
+
}> = []
|
|
699
|
+
|
|
700
|
+
try {
|
|
701
|
+
for await (const chunk of streamChatCompletionWithTools(
|
|
702
|
+
{
|
|
703
|
+
model: selectedModel,
|
|
704
|
+
messages: chatMessages,
|
|
705
|
+
...generationPayload,
|
|
706
|
+
tools: buildToolsForApi(),
|
|
707
|
+
tool_choice: selectedTools.size > 0 ? 'auto' : undefined,
|
|
708
|
+
},
|
|
709
|
+
abortControllerRef.current.signal,
|
|
710
|
+
(tc) => { toolCallsData = tc }
|
|
711
|
+
)) {
|
|
712
|
+
if (stopAgentRef.current) break
|
|
713
|
+
fullContent += chunk
|
|
714
|
+
setMessages(prev =>
|
|
715
|
+
prev.map(m =>
|
|
716
|
+
m.id === assistantMessageId
|
|
717
|
+
? { ...m, content: fullContent }
|
|
718
|
+
: m
|
|
719
|
+
)
|
|
720
|
+
)
|
|
721
|
+
}
|
|
722
|
+
} catch (error) {
|
|
723
|
+
if ((error as Error).name === 'AbortError') return
|
|
724
|
+
throw error
|
|
725
|
+
}
|
|
726
|
+
|
|
727
|
+
// Check if we have tool calls
|
|
728
|
+
if (toolCallsData.length > 0 && !stopAgentRef.current) {
|
|
729
|
+
// Create tool call objects
|
|
730
|
+
const toolCalls: ToolCall[] = toolCallsData.map(tc => ({
|
|
731
|
+
id: tc.id,
|
|
732
|
+
name: tc.function.name,
|
|
733
|
+
arguments: tc.function.arguments,
|
|
734
|
+
status: 'pending' as const,
|
|
735
|
+
}))
|
|
736
|
+
|
|
737
|
+
// Update message with tool calls
|
|
738
|
+
setMessages(prev =>
|
|
739
|
+
prev.map(m =>
|
|
740
|
+
m.id === assistantMessageId
|
|
741
|
+
? { ...m, content: fullContent || null, toolCalls }
|
|
742
|
+
: m
|
|
743
|
+
)
|
|
744
|
+
)
|
|
745
|
+
|
|
746
|
+
// Execute tools
|
|
747
|
+
setIsExecutingTools(true)
|
|
748
|
+
const toolResults: Message[] = []
|
|
749
|
+
|
|
750
|
+
for (const toolCall of toolCalls) {
|
|
751
|
+
if (stopAgentRef.current) break
|
|
752
|
+
|
|
753
|
+
// Update status to executing
|
|
754
|
+
setMessages(prev =>
|
|
755
|
+
prev.map(m =>
|
|
756
|
+
m.id === assistantMessageId
|
|
757
|
+
? {
|
|
758
|
+
...m,
|
|
759
|
+
toolCalls: m.toolCalls?.map(tc =>
|
|
760
|
+
tc.id === toolCall.id ? { ...tc, status: 'executing' as const } : tc
|
|
761
|
+
)
|
|
762
|
+
}
|
|
763
|
+
: m
|
|
764
|
+
)
|
|
765
|
+
)
|
|
766
|
+
|
|
767
|
+
try {
|
|
768
|
+
const result = await executeToolCall(toolCall)
|
|
769
|
+
|
|
770
|
+
// Update status to success
|
|
771
|
+
setMessages(prev =>
|
|
772
|
+
prev.map(m =>
|
|
773
|
+
m.id === assistantMessageId
|
|
774
|
+
? {
|
|
775
|
+
...m,
|
|
776
|
+
toolCalls: m.toolCalls?.map(tc =>
|
|
777
|
+
tc.id === toolCall.id
|
|
778
|
+
? { ...tc, status: 'success' as const, result }
|
|
779
|
+
: tc
|
|
780
|
+
)
|
|
781
|
+
}
|
|
782
|
+
: m
|
|
783
|
+
)
|
|
784
|
+
)
|
|
785
|
+
|
|
786
|
+
// Add tool result message
|
|
787
|
+
toolResults.push({
|
|
788
|
+
id: crypto.randomUUID(),
|
|
789
|
+
role: 'tool',
|
|
790
|
+
content: result,
|
|
791
|
+
toolCallId: toolCall.id,
|
|
792
|
+
createdAt: new Date(),
|
|
793
|
+
})
|
|
794
|
+
} catch (error) {
|
|
795
|
+
const errorMsg = (error as Error).message
|
|
796
|
+
|
|
797
|
+
// Update status to error
|
|
798
|
+
setMessages(prev =>
|
|
799
|
+
prev.map(m =>
|
|
800
|
+
m.id === assistantMessageId
|
|
801
|
+
? {
|
|
802
|
+
...m,
|
|
803
|
+
toolCalls: m.toolCalls?.map(tc =>
|
|
804
|
+
tc.id === toolCall.id
|
|
805
|
+
? { ...tc, status: 'error' as const, error: errorMsg }
|
|
806
|
+
: tc
|
|
807
|
+
)
|
|
808
|
+
}
|
|
809
|
+
: m
|
|
810
|
+
)
|
|
811
|
+
)
|
|
812
|
+
|
|
813
|
+
// Still add a tool result for the error
|
|
814
|
+
toolResults.push({
|
|
815
|
+
id: crypto.randomUUID(),
|
|
816
|
+
role: 'tool',
|
|
817
|
+
content: `Error: ${errorMsg}`,
|
|
818
|
+
toolCallId: toolCall.id,
|
|
819
|
+
createdAt: new Date(),
|
|
820
|
+
})
|
|
821
|
+
}
|
|
822
|
+
}
|
|
823
|
+
|
|
824
|
+
setIsExecutingTools(false)
|
|
825
|
+
|
|
826
|
+
if (stopAgentRef.current) break
|
|
827
|
+
|
|
828
|
+
// Add assistant message with tool calls and tool results to history
|
|
829
|
+
const assistantWithToolCalls: Message = {
|
|
830
|
+
id: assistantMessageId,
|
|
831
|
+
role: 'assistant',
|
|
832
|
+
content: fullContent || null,
|
|
833
|
+
toolCalls,
|
|
834
|
+
createdAt: new Date(),
|
|
835
|
+
}
|
|
836
|
+
|
|
837
|
+
currentHistory = [...currentHistory.slice(0, -1), assistantWithToolCalls, ...toolResults]
|
|
838
|
+
|
|
839
|
+
// Create new assistant message for next iteration
|
|
840
|
+
const newAssistantId = crypto.randomUUID()
|
|
841
|
+
const newAssistantMessage: Message = {
|
|
842
|
+
id: newAssistantId,
|
|
843
|
+
role: 'assistant',
|
|
844
|
+
content: '',
|
|
845
|
+
createdAt: new Date(),
|
|
846
|
+
}
|
|
847
|
+
|
|
848
|
+
setMessages(prev => [...prev, ...toolResults, newAssistantMessage])
|
|
849
|
+
assistantMessageId = newAssistantId
|
|
850
|
+
currentHistory = [...currentHistory, newAssistantMessage]
|
|
851
|
+
|
|
852
|
+
iteration++
|
|
853
|
+
} else {
|
|
854
|
+
// No tool calls, we're done
|
|
855
|
+
break
|
|
856
|
+
}
|
|
857
|
+
}
|
|
858
|
+
|
|
859
|
+
setCurrentIteration(0)
|
|
860
|
+
}
|
|
861
|
+
|
|
862
|
+
const warnIfNonDataImageUrls = (chatMessages: ApiChatMessage[], context: string) => {
|
|
863
|
+
if (!import.meta.env.DEV) return
|
|
864
|
+
const invalidUrls = findNonDataImageUrls(chatMessages)
|
|
865
|
+
if (invalidUrls.length > 0) {
|
|
866
|
+
console.warn(`[AgentPlayground][${context}] Non-data image URLs detected in chat payload`, invalidUrls)
|
|
867
|
+
}
|
|
868
|
+
}
|
|
869
|
+
|
|
870
|
+
const extractAssistantAudio = (message: unknown): { url?: string; data?: string; format?: string } | null => {
|
|
871
|
+
if (!message || typeof message !== 'object') return null
|
|
872
|
+
const msg = message as Record<string, unknown>
|
|
873
|
+
const direct = msg.audio as Record<string, unknown> | undefined
|
|
874
|
+
if (direct && (typeof direct.url === 'string' || typeof direct.data === 'string')) {
|
|
875
|
+
return {
|
|
876
|
+
url: typeof direct.url === 'string' ? direct.url : undefined,
|
|
877
|
+
data: typeof direct.data === 'string' ? direct.data : undefined,
|
|
878
|
+
format: typeof direct.format === 'string' ? direct.format : undefined,
|
|
879
|
+
}
|
|
880
|
+
}
|
|
881
|
+
const content = msg.content
|
|
882
|
+
if (!Array.isArray(content)) return null
|
|
883
|
+
for (const part of content) {
|
|
884
|
+
if (!part || typeof part !== 'object') continue
|
|
885
|
+
const p = part as Record<string, unknown>
|
|
886
|
+
const audioObj = p.audio as Record<string, unknown> | undefined
|
|
887
|
+
if ((p.type === 'audio' || p.type === 'output_audio') && audioObj) {
|
|
888
|
+
return {
|
|
889
|
+
url: typeof audioObj.url === 'string' ? audioObj.url : undefined,
|
|
890
|
+
data: typeof audioObj.data === 'string' ? audioObj.data : undefined,
|
|
891
|
+
format: typeof audioObj.format === 'string' ? audioObj.format : undefined,
|
|
892
|
+
}
|
|
893
|
+
}
|
|
894
|
+
}
|
|
895
|
+
return null
|
|
896
|
+
}
|
|
897
|
+
|
|
898
|
+
const formatToMimeType = (format?: string): string | undefined => {
|
|
899
|
+
if (!format) return undefined
|
|
900
|
+
const lower = format.toLowerCase()
|
|
901
|
+
if (lower === 'wav') return 'audio/wav'
|
|
902
|
+
if (lower === 'mp3' || lower === 'mpeg') return 'audio/mpeg'
|
|
903
|
+
if (lower === 'ogg') return 'audio/ogg'
|
|
904
|
+
if (lower === 'webm') return 'audio/webm'
|
|
905
|
+
if (lower === 'm4a' || lower === 'mp4') return 'audio/mp4'
|
|
906
|
+
return undefined
|
|
907
|
+
}
|
|
908
|
+
|
|
909
|
+
const maybeAutoTitleSession = async (sessionId: string): Promise<void> => {
|
|
910
|
+
await flushDeferredAutoTitle({
|
|
911
|
+
sessionId,
|
|
912
|
+
sessionName,
|
|
913
|
+
model: selectedModel,
|
|
914
|
+
queuedCandidate: pendingAutoTitleRef.current,
|
|
915
|
+
generatingSessionId: titleGenerationSessionId,
|
|
916
|
+
autoTitleSession,
|
|
917
|
+
onGenerationChange: setTitleGenerationSessionId,
|
|
918
|
+
onResolved: (response) => {
|
|
919
|
+
setSessionName((current) => (current === sessionName ? response.name : current))
|
|
920
|
+
setSessions((prev) => applyAutoTitleToSessions(prev, sessionId, response))
|
|
921
|
+
},
|
|
922
|
+
clearQueuedCandidate: () => {
|
|
923
|
+
if (pendingAutoTitleRef.current?.sessionId === sessionId) {
|
|
924
|
+
pendingAutoTitleRef.current = null
|
|
925
|
+
}
|
|
926
|
+
},
|
|
927
|
+
onError: (error) => {
|
|
928
|
+
console.warn('Auto-title skipped:', error)
|
|
929
|
+
},
|
|
930
|
+
})
|
|
931
|
+
}
|
|
932
|
+
|
|
933
|
+
const handleSubmit = async (e: React.FormEvent) => {
|
|
934
|
+
e.preventDefault()
|
|
935
|
+
if (!selectedModelConfig || !isModelSelectable(selectedModelConfig)) {
|
|
936
|
+
setCallError('Selected model is unavailable. Choose a healthy model and try again.')
|
|
937
|
+
return
|
|
938
|
+
}
|
|
939
|
+
const hasText = input.trim().length > 0
|
|
940
|
+
const canSend = callModeEnabled
|
|
941
|
+
? Boolean(pendingAudio)
|
|
942
|
+
: hasText || pendingImages.length > 0
|
|
943
|
+
if (!canSend || isLoading) return
|
|
944
|
+
const { payload: generationPayload, error: generationParamError } = parseGenerationParams(generationParams)
|
|
945
|
+
if (generationParamError) {
|
|
946
|
+
setCallError(generationParamError)
|
|
947
|
+
return
|
|
948
|
+
}
|
|
949
|
+
|
|
950
|
+
stopAgentRef.current = false
|
|
951
|
+
setCallError(null)
|
|
952
|
+
|
|
953
|
+
const requestImageUrls = pendingImages.length > 0 ? [...pendingImages] : []
|
|
954
|
+
let displayImageRefs = requestImageUrls.length > 0 ? [...requestImageUrls] : undefined
|
|
955
|
+
if ((activeSessionStorageVersion ?? 1) >= 2 && pendingImages.length > 0) {
|
|
956
|
+
try {
|
|
957
|
+
const cached = await Promise.all(
|
|
958
|
+
pendingImages.map((image) => cacheImage(image, selectedModel))
|
|
959
|
+
)
|
|
960
|
+
displayImageRefs = cached.map((item) => item.url)
|
|
961
|
+
} catch (error) {
|
|
962
|
+
console.error('Failed to cache input images, falling back to inline images:', error)
|
|
963
|
+
}
|
|
964
|
+
}
|
|
965
|
+
|
|
966
|
+
let audioRef: string | undefined
|
|
967
|
+
if (pendingAudio) {
|
|
968
|
+
if ((activeSessionStorageVersion ?? 1) >= 2) {
|
|
969
|
+
try {
|
|
970
|
+
const cachedAudio = await storeMedia(pendingAudio, selectedModel, pendingAudioMimeType)
|
|
971
|
+
audioRef = cachedAudio.url
|
|
972
|
+
} catch (error) {
|
|
973
|
+
console.error('Failed to cache input audio, falling back to inline audio:', error)
|
|
974
|
+
audioRef = pendingAudio
|
|
975
|
+
}
|
|
976
|
+
} else {
|
|
977
|
+
audioRef = pendingAudio
|
|
978
|
+
}
|
|
979
|
+
}
|
|
980
|
+
|
|
981
|
+
const payload = buildUserPayload({
|
|
982
|
+
callModeEnabled,
|
|
983
|
+
text: input,
|
|
984
|
+
requestImageUrls,
|
|
985
|
+
displayImageRefs,
|
|
986
|
+
audioRef,
|
|
987
|
+
})
|
|
988
|
+
|
|
989
|
+
const userMessage: Message = {
|
|
990
|
+
id: crypto.randomUUID(),
|
|
991
|
+
role: 'user',
|
|
992
|
+
content: payload.content,
|
|
993
|
+
images: payload.images,
|
|
994
|
+
requestImages: payload.requestImages,
|
|
995
|
+
createdAt: new Date(),
|
|
996
|
+
}
|
|
997
|
+
|
|
998
|
+
setMessages(prev => [...prev, normalizeMessageForUi(userMessage)])
|
|
999
|
+
setInput('')
|
|
1000
|
+
setPendingImages([])
|
|
1001
|
+
setPendingAudio(null)
|
|
1002
|
+
setPendingAudioMimeType(undefined)
|
|
1003
|
+
setIsLoading(true)
|
|
1004
|
+
|
|
1005
|
+
// Save user message to session
|
|
1006
|
+
if (activeSessionId) {
|
|
1007
|
+
try {
|
|
1008
|
+
await addMessageToSession(activeSessionId, {
|
|
1009
|
+
role: 'user',
|
|
1010
|
+
content: userMessage.content,
|
|
1011
|
+
images: userMessage.images,
|
|
1012
|
+
timestamp: userMessage.createdAt.toISOString(),
|
|
1013
|
+
})
|
|
1014
|
+
const textForTitle = getTextContent(userMessage.content)
|
|
1015
|
+
const autoTitleCandidate = createDeferredAutoTitleCandidate(activeSessionId, sessionName, textForTitle)
|
|
1016
|
+
if (autoTitleCandidate) {
|
|
1017
|
+
pendingAutoTitleRef.current = autoTitleCandidate
|
|
1018
|
+
}
|
|
1019
|
+
} catch (error) {
|
|
1020
|
+
console.error('Failed to save user message:', error)
|
|
1021
|
+
}
|
|
1022
|
+
}
|
|
1023
|
+
|
|
1024
|
+
const assistantMessage: Message = {
|
|
1025
|
+
id: crypto.randomUUID(),
|
|
1026
|
+
role: 'assistant',
|
|
1027
|
+
content: '',
|
|
1028
|
+
model: selectedModel,
|
|
1029
|
+
createdAt: new Date(),
|
|
1030
|
+
}
|
|
1031
|
+
setMessages(prev => [...prev, normalizeMessageForUi(assistantMessage)])
|
|
1032
|
+
|
|
1033
|
+
try {
|
|
1034
|
+
if (callModeEnabled) {
|
|
1035
|
+
setCallStatus('sending')
|
|
1036
|
+
const chatMessages = [...messages, userMessage].map(toApiMessage)
|
|
1037
|
+
warnIfNonDataImageUrls(chatMessages, 'call-mode')
|
|
1038
|
+
const response = await createChatCompletionRaw({
|
|
1039
|
+
model: selectedModel,
|
|
1040
|
+
messages: chatMessages,
|
|
1041
|
+
stream: false,
|
|
1042
|
+
...generationPayload,
|
|
1043
|
+
})
|
|
1044
|
+
const assistant = response.choices?.[0]?.message
|
|
1045
|
+
const assistantText =
|
|
1046
|
+
typeof assistant?.content === 'string'
|
|
1047
|
+
? assistant.content
|
|
1048
|
+
: getTextContent((assistant?.content as ContentPart[] | null) ?? null)
|
|
1049
|
+
|
|
1050
|
+
const audio = extractAssistantAudio(assistant)
|
|
1051
|
+
let assistantAudioUrl: string | undefined
|
|
1052
|
+
if (audio?.url) {
|
|
1053
|
+
assistantAudioUrl = resolveMediaUrl(audio.url)
|
|
1054
|
+
} else if (audio?.data) {
|
|
1055
|
+
const dataUrl = audio.data.startsWith('data:')
|
|
1056
|
+
? audio.data
|
|
1057
|
+
: `data:${formatToMimeType(audio.format) ?? 'audio/wav'};base64,${audio.data}`
|
|
1058
|
+
if ((activeSessionStorageVersion ?? 1) >= 2) {
|
|
1059
|
+
const cached = await storeMedia(
|
|
1060
|
+
dataUrl,
|
|
1061
|
+
selectedModel,
|
|
1062
|
+
formatToMimeType(audio.format) ?? pendingAudioMimeType
|
|
1063
|
+
)
|
|
1064
|
+
assistantAudioUrl = cached.url
|
|
1065
|
+
} else {
|
|
1066
|
+
assistantAudioUrl = dataUrl
|
|
1067
|
+
}
|
|
1068
|
+
}
|
|
1069
|
+
|
|
1070
|
+
const assistantContent: ContentPart[] = [
|
|
1071
|
+
...(assistantText ? [{ type: 'text' as const, text: assistantText }] : []),
|
|
1072
|
+
...(assistantAudioUrl
|
|
1073
|
+
? [{ type: 'audio' as const, audio: { url: assistantAudioUrl, format: audio?.format } }]
|
|
1074
|
+
: []),
|
|
1075
|
+
]
|
|
1076
|
+
|
|
1077
|
+
setMessages(prev =>
|
|
1078
|
+
prev.map(m =>
|
|
1079
|
+
m.id === assistantMessage.id
|
|
1080
|
+
? { ...m, content: assistantContent.length > 0 ? assistantContent : assistantText || '' }
|
|
1081
|
+
: m
|
|
1082
|
+
)
|
|
1083
|
+
)
|
|
1084
|
+
|
|
1085
|
+
if (assistantAudioUrl) {
|
|
1086
|
+
stopPlayback()
|
|
1087
|
+
const player = new Audio(assistantAudioUrl)
|
|
1088
|
+
callAudioRef.current = player
|
|
1089
|
+
setCallStatus('playing')
|
|
1090
|
+
player.onended = () => {
|
|
1091
|
+
setCallStatus('idle')
|
|
1092
|
+
callAudioRef.current = null
|
|
1093
|
+
}
|
|
1094
|
+
player.onerror = () => {
|
|
1095
|
+
setCallStatus('idle')
|
|
1096
|
+
callAudioRef.current = null
|
|
1097
|
+
}
|
|
1098
|
+
void player.play().catch((error) => {
|
|
1099
|
+
console.warn('Audio playback failed:', error)
|
|
1100
|
+
setCallStatus('idle')
|
|
1101
|
+
callAudioRef.current = null
|
|
1102
|
+
})
|
|
1103
|
+
} else {
|
|
1104
|
+
setCallStatus('idle')
|
|
1105
|
+
}
|
|
1106
|
+
|
|
1107
|
+
if (activeSessionId) {
|
|
1108
|
+
await addMessageToSession(activeSessionId, {
|
|
1109
|
+
role: 'assistant',
|
|
1110
|
+
content: assistantContent.length > 0 ? assistantContent : assistantText || '',
|
|
1111
|
+
timestamp: new Date().toISOString(),
|
|
1112
|
+
model: selectedModel,
|
|
1113
|
+
})
|
|
1114
|
+
await maybeAutoTitleSession(activeSessionId)
|
|
1115
|
+
}
|
|
1116
|
+
} else if (selectedModelSupportsImageOutput()) {
|
|
1117
|
+
// Image generation mode
|
|
1118
|
+
const prompt = getTextContent(userMessage.content)
|
|
1119
|
+
if (!prompt) {
|
|
1120
|
+
throw new Error('Please enter a prompt for image generation')
|
|
1121
|
+
}
|
|
1122
|
+
|
|
1123
|
+
const editInputImageUrl = requestImageUrls.length > 0 ? requestImageUrls[0] : undefined
|
|
1124
|
+
const imageResponse = await generateImage({
|
|
1125
|
+
model: selectedModel,
|
|
1126
|
+
prompt,
|
|
1127
|
+
image_url: editInputImageUrl,
|
|
1128
|
+
n: 1,
|
|
1129
|
+
size: imageSize,
|
|
1130
|
+
response_format: 'b64_json',
|
|
1131
|
+
})
|
|
1132
|
+
|
|
1133
|
+
// Convert response to content with image
|
|
1134
|
+
const imageData = imageResponse.data[0]
|
|
1135
|
+
let imageUrl = imageData.url || ''
|
|
1136
|
+
if (!imageUrl && imageData.b64_json) {
|
|
1137
|
+
if ((activeSessionStorageVersion ?? 1) >= 2) {
|
|
1138
|
+
try {
|
|
1139
|
+
const cached = await cacheImage(imageData.b64_json, selectedModel)
|
|
1140
|
+
imageUrl = cached.url
|
|
1141
|
+
} catch (error) {
|
|
1142
|
+
console.error('Failed to cache generated image, using inline payload:', error)
|
|
1143
|
+
imageUrl = `data:image/png;base64,${imageData.b64_json}`
|
|
1144
|
+
}
|
|
1145
|
+
} else {
|
|
1146
|
+
imageUrl = `data:image/png;base64,${imageData.b64_json}`
|
|
1147
|
+
}
|
|
1148
|
+
}
|
|
1149
|
+
|
|
1150
|
+
const imageContent: ContentPart[] = []
|
|
1151
|
+
if (imageData.revised_prompt) {
|
|
1152
|
+
imageContent.push({ type: 'text', text: imageData.revised_prompt })
|
|
1153
|
+
}
|
|
1154
|
+
if (imageUrl) {
|
|
1155
|
+
imageContent.push({ type: 'image_url', image_url: { url: imageUrl } })
|
|
1156
|
+
}
|
|
1157
|
+
|
|
1158
|
+
setMessages(prev =>
|
|
1159
|
+
prev.map(m =>
|
|
1160
|
+
m.id === assistantMessage.id
|
|
1161
|
+
? { ...m, content: imageContent }
|
|
1162
|
+
: m
|
|
1163
|
+
)
|
|
1164
|
+
)
|
|
1165
|
+
|
|
1166
|
+
// Save to session
|
|
1167
|
+
if (activeSessionId) {
|
|
1168
|
+
try {
|
|
1169
|
+
await addMessageToSession(activeSessionId, {
|
|
1170
|
+
role: 'assistant',
|
|
1171
|
+
content: imageContent,
|
|
1172
|
+
timestamp: new Date().toISOString(),
|
|
1173
|
+
model: selectedModel,
|
|
1174
|
+
})
|
|
1175
|
+
await maybeAutoTitleSession(activeSessionId)
|
|
1176
|
+
} catch (error) {
|
|
1177
|
+
console.error('Failed to save assistant message:', error)
|
|
1178
|
+
}
|
|
1179
|
+
}
|
|
1180
|
+
} else if (agentModeEnabled && selectedTools.size > 0) {
|
|
1181
|
+
// Run agentic loop
|
|
1182
|
+
await runAgentLoop(
|
|
1183
|
+
[...messages, userMessage, assistantMessage],
|
|
1184
|
+
assistantMessage.id,
|
|
1185
|
+
generationPayload
|
|
1186
|
+
)
|
|
1187
|
+
if (activeSessionId) {
|
|
1188
|
+
await maybeAutoTitleSession(activeSessionId)
|
|
1189
|
+
}
|
|
1190
|
+
} else {
|
|
1191
|
+
// Regular chat (LLM, embedding, audio handled as chat for now)
|
|
1192
|
+
abortControllerRef.current = new AbortController()
|
|
1193
|
+
isStreamingRef.current = true
|
|
1194
|
+
|
|
1195
|
+
const chatMessages = [...messages, userMessage].map(toApiMessage) as ApiChatMessage[]
|
|
1196
|
+
warnIfNonDataImageUrls(chatMessages, 'chat-stream')
|
|
1197
|
+
|
|
1198
|
+
let streamState = createThinkingStreamState()
|
|
1199
|
+
for await (const chunk of streamChatCompletion(
|
|
1200
|
+
{ model: selectedModel, messages: chatMessages, ...generationPayload },
|
|
1201
|
+
abortControllerRef.current.signal
|
|
1202
|
+
)) {
|
|
1203
|
+
streamState = applyThinkingChunk(streamState, chunk)
|
|
1204
|
+
const displayContent = toDisplayContent(streamState)
|
|
1205
|
+
setMessages(prev =>
|
|
1206
|
+
prev.map(m =>
|
|
1207
|
+
m.id === assistantMessage.id
|
|
1208
|
+
? { ...m, content: displayContent }
|
|
1209
|
+
: m
|
|
1210
|
+
)
|
|
1211
|
+
)
|
|
1212
|
+
}
|
|
1213
|
+
const finalContent = toFinalContent(streamState)
|
|
1214
|
+
setMessages(prev =>
|
|
1215
|
+
prev.map(m =>
|
|
1216
|
+
m.id === assistantMessage.id
|
|
1217
|
+
? { ...m, content: finalContent }
|
|
1218
|
+
: m
|
|
1219
|
+
)
|
|
1220
|
+
)
|
|
1221
|
+
|
|
1222
|
+
// Save assistant message
|
|
1223
|
+
if (activeSessionId && finalContent) {
|
|
1224
|
+
try {
|
|
1225
|
+
await addMessageToSession(activeSessionId, {
|
|
1226
|
+
role: 'assistant',
|
|
1227
|
+
content: finalContent,
|
|
1228
|
+
timestamp: new Date().toISOString(),
|
|
1229
|
+
model: selectedModel,
|
|
1230
|
+
})
|
|
1231
|
+
await maybeAutoTitleSession(activeSessionId)
|
|
1232
|
+
} catch (error) {
|
|
1233
|
+
console.error('Failed to save assistant message:', error)
|
|
1234
|
+
}
|
|
1235
|
+
}
|
|
1236
|
+
}
|
|
1237
|
+
} catch (error) {
|
|
1238
|
+
if ((error as Error).name === 'AbortError') return
|
|
1239
|
+
console.error('Chat error:', error)
|
|
1240
|
+
if (callModeEnabled) {
|
|
1241
|
+
setCallError((error as Error).message || 'Call turn failed. Try again.')
|
|
1242
|
+
}
|
|
1243
|
+
setCallStatus('idle')
|
|
1244
|
+
setMessages(prev =>
|
|
1245
|
+
prev.map(m =>
|
|
1246
|
+
m.id === assistantMessage.id
|
|
1247
|
+
? { ...m, content: 'Error occurred. Please try again.' }
|
|
1248
|
+
: m
|
|
1249
|
+
)
|
|
1250
|
+
)
|
|
1251
|
+
} finally {
|
|
1252
|
+
setIsLoading(false)
|
|
1253
|
+
abortControllerRef.current = null
|
|
1254
|
+
stopAgentRef.current = false
|
|
1255
|
+
isStreamingRef.current = false
|
|
1256
|
+
if (!callAudioRef.current) {
|
|
1257
|
+
setCallStatus('idle')
|
|
1258
|
+
}
|
|
1259
|
+
}
|
|
1260
|
+
}
|
|
1261
|
+
|
|
1262
|
+
const handleKeyDown = (e: React.KeyboardEvent) => {
|
|
1263
|
+
if (e.key === 'Enter' && !e.shiftKey) {
|
|
1264
|
+
e.preventDefault()
|
|
1265
|
+
handleSubmit(e)
|
|
1266
|
+
}
|
|
1267
|
+
}
|
|
1268
|
+
|
|
1269
|
+
return (
|
|
1270
|
+
<div className="flex-1 flex h-full min-h-0 overflow-hidden">
|
|
1271
|
+
{/* Sessions Sidebar */}
|
|
1272
|
+
<aside className={cn(
|
|
1273
|
+
'border-r border-border flex flex-col min-h-0 shrink-0 transition-all duration-300',
|
|
1274
|
+
sidebarCollapsed ? 'w-12' : 'w-64'
|
|
1275
|
+
)}>
|
|
1276
|
+
<div className="h-14 border-b border-border flex items-center justify-between px-3">
|
|
1277
|
+
{!sidebarCollapsed && (
|
|
1278
|
+
<span className="font-mono text-xs uppercase tracking-wider text-muted-foreground">
|
|
1279
|
+
Sessions
|
|
1280
|
+
</span>
|
|
1281
|
+
)}
|
|
1282
|
+
<Button
|
|
1283
|
+
variant="ghost"
|
|
1284
|
+
size="icon"
|
|
1285
|
+
className="h-7 w-7"
|
|
1286
|
+
onClick={() => setSidebarCollapsed(!sidebarCollapsed)}
|
|
1287
|
+
>
|
|
1288
|
+
<ChevronRight className={cn(
|
|
1289
|
+
'w-4 h-4 transition-transform',
|
|
1290
|
+
!sidebarCollapsed && 'rotate-180'
|
|
1291
|
+
)} />
|
|
1292
|
+
</Button>
|
|
1293
|
+
</div>
|
|
1294
|
+
|
|
1295
|
+
{!sidebarCollapsed && (
|
|
1296
|
+
<>
|
|
1297
|
+
<div className="p-2">
|
|
1298
|
+
<Button
|
|
1299
|
+
onClick={handleNewSession}
|
|
1300
|
+
className="w-full justify-start gap-2"
|
|
1301
|
+
variant="outline"
|
|
1302
|
+
>
|
|
1303
|
+
<Plus className="w-4 h-4" />
|
|
1304
|
+
New Chat
|
|
1305
|
+
</Button>
|
|
1306
|
+
</div>
|
|
1307
|
+
|
|
1308
|
+
<div className="flex-1 overflow-y-auto">
|
|
1309
|
+
{sessions.map(session => (
|
|
1310
|
+
<div
|
|
1311
|
+
key={session.id}
|
|
1312
|
+
className={cn(
|
|
1313
|
+
'group flex items-center gap-2 px-3 py-2 cursor-pointer hover:bg-secondary/50 transition-colors',
|
|
1314
|
+
activeSessionId === session.id && 'bg-secondary border-l-2 border-primary'
|
|
1315
|
+
)}
|
|
1316
|
+
onClick={() => loadSession(session.id)}
|
|
1317
|
+
>
|
|
1318
|
+
<MessageSquare className="w-4 h-4 text-muted-foreground shrink-0" />
|
|
1319
|
+
<div className="flex-1 min-w-0">
|
|
1320
|
+
<p className="text-sm truncate">{session.name}</p>
|
|
1321
|
+
<p className="text-2xs text-muted-foreground">
|
|
1322
|
+
{titleGenerationSessionId === session.id ? 'Generating title...' : `${session.messageCount} messages`}
|
|
1323
|
+
</p>
|
|
1324
|
+
</div>
|
|
1325
|
+
<Button
|
|
1326
|
+
variant="ghost"
|
|
1327
|
+
size="icon"
|
|
1328
|
+
className="h-6 w-6 opacity-0 group-hover:opacity-100 transition-opacity"
|
|
1329
|
+
onClick={(e) => {
|
|
1330
|
+
e.stopPropagation()
|
|
1331
|
+
handleDeleteSession(session.id)
|
|
1332
|
+
}}
|
|
1333
|
+
>
|
|
1334
|
+
<Trash2 className="w-3 h-3 text-destructive" />
|
|
1335
|
+
</Button>
|
|
1336
|
+
</div>
|
|
1337
|
+
))}
|
|
1338
|
+
|
|
1339
|
+
{sessions.length === 0 && (
|
|
1340
|
+
<div className="p-4 text-center text-muted-foreground text-sm">
|
|
1341
|
+
No sessions yet
|
|
1342
|
+
</div>
|
|
1343
|
+
)}
|
|
1344
|
+
</div>
|
|
1345
|
+
</>
|
|
1346
|
+
)}
|
|
1347
|
+
</aside>
|
|
1348
|
+
|
|
1349
|
+
{/* Main Chat Area */}
|
|
1350
|
+
<div className="flex-1 flex flex-col min-h-0">
|
|
1351
|
+
{/* Header */}
|
|
1352
|
+
<header className="sticky top-0 z-20 h-14 border-b border-border bg-background/95 backdrop-blur flex items-center px-6 gap-4 shrink-0">
|
|
1353
|
+
<div className="flex items-center gap-2">
|
|
1354
|
+
<Sparkles className="w-4 h-4 text-primary" />
|
|
1355
|
+
<h2 className="font-mono font-semibold text-sm uppercase tracking-wider">
|
|
1356
|
+
{sessionName || 'Playground'}
|
|
1357
|
+
</h2>
|
|
1358
|
+
{titleGenerationSessionId === activeSessionId && (
|
|
1359
|
+
<span className="text-2xs font-mono uppercase tracking-wider text-muted-foreground">
|
|
1360
|
+
Generating title...
|
|
1361
|
+
</span>
|
|
1362
|
+
)}
|
|
1363
|
+
</div>
|
|
1364
|
+
|
|
1365
|
+
{/* Agent Mode Toggle */}
|
|
1366
|
+
<button
|
|
1367
|
+
onClick={() => {
|
|
1368
|
+
setAgentModeEnabled(!agentModeEnabled)
|
|
1369
|
+
if (!agentModeEnabled) setShowToolPicker(true)
|
|
1370
|
+
}}
|
|
1371
|
+
className={cn(
|
|
1372
|
+
'flex items-center gap-2 px-2 py-1 rounded-md text-xs font-mono transition-colors',
|
|
1373
|
+
agentModeEnabled
|
|
1374
|
+
? 'bg-primary/20 text-primary border border-primary/30'
|
|
1375
|
+
: 'bg-secondary text-muted-foreground hover:text-foreground'
|
|
1376
|
+
)}
|
|
1377
|
+
>
|
|
1378
|
+
{agentModeEnabled ? (
|
|
1379
|
+
<ToggleRight className="w-4 h-4" />
|
|
1380
|
+
) : (
|
|
1381
|
+
<ToggleLeft className="w-4 h-4" />
|
|
1382
|
+
)}
|
|
1383
|
+
Agent Mode
|
|
1384
|
+
{selectedTools.size > 0 && (
|
|
1385
|
+
<span className="px-1.5 py-0.5 bg-primary/30 rounded text-2xs">
|
|
1386
|
+
{selectedTools.size}
|
|
1387
|
+
</span>
|
|
1388
|
+
)}
|
|
1389
|
+
</button>
|
|
1390
|
+
|
|
1391
|
+
{agentModeEnabled && (
|
|
1392
|
+
<Button
|
|
1393
|
+
variant="ghost"
|
|
1394
|
+
size="sm"
|
|
1395
|
+
className="h-7 text-xs gap-1"
|
|
1396
|
+
onClick={() => setShowToolPicker(!showToolPicker)}
|
|
1397
|
+
>
|
|
1398
|
+
<Wrench className="w-3.5 h-3.5" />
|
|
1399
|
+
Tools
|
|
1400
|
+
</Button>
|
|
1401
|
+
)}
|
|
1402
|
+
|
|
1403
|
+
{modelSupportsCall && (
|
|
1404
|
+
<Button
|
|
1405
|
+
variant={callModeEnabled ? "default" : "ghost"}
|
|
1406
|
+
size="sm"
|
|
1407
|
+
className="h-7 text-xs gap-1"
|
|
1408
|
+
onClick={() => {
|
|
1409
|
+
setCallModeEnabled(!callModeEnabled)
|
|
1410
|
+
setPendingAudio(null)
|
|
1411
|
+
setPendingAudioMimeType(undefined)
|
|
1412
|
+
setCallError(null)
|
|
1413
|
+
if (callStatus === 'recording') {
|
|
1414
|
+
stopRecording()
|
|
1415
|
+
}
|
|
1416
|
+
stopPlayback()
|
|
1417
|
+
}}
|
|
1418
|
+
>
|
|
1419
|
+
{callModeEnabled ? <PhoneCall className="w-3.5 h-3.5" /> : <PhoneOff className="w-3.5 h-3.5" />}
|
|
1420
|
+
Call
|
|
1421
|
+
</Button>
|
|
1422
|
+
)}
|
|
1423
|
+
|
|
1424
|
+
<div className="flex-1" />
|
|
1425
|
+
|
|
1426
|
+
<select
|
|
1427
|
+
value={selectedModel}
|
|
1428
|
+
onChange={(e) => setSelectedModel(e.target.value)}
|
|
1429
|
+
className="bg-input border border-border rounded px-3 py-1.5 text-sm font-mono focus:ring-1 focus:ring-primary focus:outline-none"
|
|
1430
|
+
>
|
|
1431
|
+
{models.length === 0 && <option value="">No models available</option>}
|
|
1432
|
+
{models.map(model => (
|
|
1433
|
+
<option key={model.id} value={model.id} disabled={!isModelSelectable(model)}>
|
|
1434
|
+
{model.id} {formatModelTag(model)} {!isModelSelectable(model) ? '(down)' : ''}
|
|
1435
|
+
</option>
|
|
1436
|
+
))}
|
|
1437
|
+
</select>
|
|
1438
|
+
</header>
|
|
1439
|
+
|
|
1440
|
+
<div className="flex-1 min-h-0 flex overflow-hidden">
|
|
1441
|
+
{/* Messages Area */}
|
|
1442
|
+
<div className="flex-1 min-h-0 flex flex-col">
|
|
1443
|
+
<div
|
|
1444
|
+
ref={messagesContainerRef}
|
|
1445
|
+
className={cn(
|
|
1446
|
+
'flex-1 min-h-0 overflow-y-auto p-6 space-y-4 relative',
|
|
1447
|
+
isDragging && 'bg-primary/5 border-2 border-dashed border-primary/30'
|
|
1448
|
+
)}
|
|
1449
|
+
onDragOver={(e) => { e.preventDefault(); setIsDragging(true) }}
|
|
1450
|
+
onDragLeave={() => setIsDragging(false)}
|
|
1451
|
+
onDrop={handleDrop}
|
|
1452
|
+
>
|
|
1453
|
+
{isDragging && (
|
|
1454
|
+
<div className="absolute inset-0 flex items-center justify-center pointer-events-none z-10">
|
|
1455
|
+
<div className="bg-background/90 rounded-lg p-6 text-center">
|
|
1456
|
+
<ImageIcon className="w-12 h-12 text-primary mx-auto mb-2" />
|
|
1457
|
+
<p className="font-mono text-sm">
|
|
1458
|
+
{callModeEnabled ? 'Drop image or audio here' : 'Drop image here'}
|
|
1459
|
+
</p>
|
|
1460
|
+
</div>
|
|
1461
|
+
</div>
|
|
1462
|
+
)}
|
|
1463
|
+
|
|
1464
|
+
{messages.length === 0 && (
|
|
1465
|
+
<div className="flex-1 flex items-center justify-center h-full">
|
|
1466
|
+
<div className="text-center space-y-4 animate-fade-in">
|
|
1467
|
+
<div className="w-16 h-16 rounded-full bg-primary/10 flex items-center justify-center mx-auto">
|
|
1468
|
+
<Bot className="w-8 h-8 text-primary" />
|
|
1469
|
+
</div>
|
|
1470
|
+
<div>
|
|
1471
|
+
<h3 className="font-mono font-semibold text-lg">Ready to chat</h3>
|
|
1472
|
+
<p className="text-muted-foreground text-sm mt-1">
|
|
1473
|
+
{agentModeEnabled
|
|
1474
|
+
? 'Agent mode enabled - select tools to use'
|
|
1475
|
+
: 'Select a model and start a conversation'}
|
|
1476
|
+
</p>
|
|
1477
|
+
<p className="text-muted-foreground text-xs mt-2">
|
|
1478
|
+
{callModeEnabled
|
|
1479
|
+
? 'Call mode: push-to-talk + optional text/images'
|
|
1480
|
+
: 'Supports images via drag, drop, paste, or upload'}
|
|
1481
|
+
</p>
|
|
1482
|
+
</div>
|
|
1483
|
+
</div>
|
|
1484
|
+
</div>
|
|
1485
|
+
)}
|
|
1486
|
+
|
|
1487
|
+
{messages.map((message, index) => (
|
|
1488
|
+
<div
|
|
1489
|
+
key={message.id}
|
|
1490
|
+
className={cn(
|
|
1491
|
+
'flex gap-3 animate-slide-in-bottom',
|
|
1492
|
+
message.role === 'user' ? 'justify-end' : 'justify-start',
|
|
1493
|
+
message.role === 'tool' && 'opacity-70'
|
|
1494
|
+
)}
|
|
1495
|
+
style={{ animationDelay: `${index * 50}ms` }}
|
|
1496
|
+
>
|
|
1497
|
+
{message.role === 'assistant' && (
|
|
1498
|
+
<div className="w-8 h-8 rounded-full bg-secondary flex items-center justify-center shrink-0">
|
|
1499
|
+
<Bot className="w-4 h-4 text-muted-foreground" />
|
|
1500
|
+
</div>
|
|
1501
|
+
)}
|
|
1502
|
+
{message.role === 'tool' && (
|
|
1503
|
+
<div className="w-8 h-8 rounded-full bg-amber-500/20 flex items-center justify-center shrink-0">
|
|
1504
|
+
<Wrench className="w-4 h-4 text-amber-500" />
|
|
1505
|
+
</div>
|
|
1506
|
+
)}
|
|
1507
|
+
<div
|
|
1508
|
+
className={cn(
|
|
1509
|
+
'rounded-lg px-4 py-3',
|
|
1510
|
+
message.role === 'user'
|
|
1511
|
+
? 'max-w-[70%] bg-primary/10 border border-primary/20'
|
|
1512
|
+
: message.role === 'tool'
|
|
1513
|
+
? 'max-w-[70%] bg-amber-500/5 border border-amber-500/20'
|
|
1514
|
+
: 'w-full sm:w-[70%] max-w-[75ch] bg-secondary border border-border'
|
|
1515
|
+
)}
|
|
1516
|
+
>
|
|
1517
|
+
{/* Render explicitly attached images */}
|
|
1518
|
+
{message.images && message.images.length > 0 && (
|
|
1519
|
+
<div className="flex flex-wrap gap-2 mb-2">
|
|
1520
|
+
{message.images.map((img, i) => (
|
|
1521
|
+
<img
|
|
1522
|
+
key={i}
|
|
1523
|
+
src={resolveMediaUrl(img)}
|
|
1524
|
+
alt={`Attached ${i + 1}`}
|
|
1525
|
+
className="max-w-32 max-h-32 rounded border border-border"
|
|
1526
|
+
/>
|
|
1527
|
+
))}
|
|
1528
|
+
</div>
|
|
1529
|
+
)}
|
|
1530
|
+
{/* Render images from multimodal content (e.g., image generation responses) */}
|
|
1531
|
+
{getImageUrls(message.content).length > 0 && (
|
|
1532
|
+
<div className="flex flex-wrap gap-2 mb-2">
|
|
1533
|
+
{getImageUrls(message.content).map((url, i) => (
|
|
1534
|
+
<a
|
|
1535
|
+
key={i}
|
|
1536
|
+
href={url}
|
|
1537
|
+
target="_blank"
|
|
1538
|
+
rel="noopener noreferrer"
|
|
1539
|
+
className="block"
|
|
1540
|
+
>
|
|
1541
|
+
<img
|
|
1542
|
+
src={url}
|
|
1543
|
+
alt={`Generated ${i + 1}`}
|
|
1544
|
+
className="max-w-xs max-h-64 rounded border border-border hover:border-primary transition-colors cursor-pointer"
|
|
1545
|
+
/>
|
|
1546
|
+
</a>
|
|
1547
|
+
))}
|
|
1548
|
+
</div>
|
|
1549
|
+
)}
|
|
1550
|
+
{getAudioUrls(message.content).length > 0 && (
|
|
1551
|
+
<div className="flex flex-col gap-2 mb-2">
|
|
1552
|
+
{getAudioUrls(message.content).map((url, i) => (
|
|
1553
|
+
<audio
|
|
1554
|
+
key={i}
|
|
1555
|
+
controls
|
|
1556
|
+
src={url}
|
|
1557
|
+
className="w-full max-w-sm"
|
|
1558
|
+
/>
|
|
1559
|
+
))}
|
|
1560
|
+
</div>
|
|
1561
|
+
)}
|
|
1562
|
+
{/* Render text content with markdown support */}
|
|
1563
|
+
{getTextContent(message.content) && (
|
|
1564
|
+
message.role === 'user' ? (
|
|
1565
|
+
<p className="text-sm whitespace-pre-wrap">{getTextContent(message.content)}</p>
|
|
1566
|
+
) : (
|
|
1567
|
+
<MessageContent content={getTextContent(message.content)} />
|
|
1568
|
+
)
|
|
1569
|
+
)}
|
|
1570
|
+
{message.toolCalls && message.toolCalls.length > 0 && (
|
|
1571
|
+
<div className="mt-2">
|
|
1572
|
+
<ToolCallMessage toolCalls={message.toolCalls} />
|
|
1573
|
+
</div>
|
|
1574
|
+
)}
|
|
1575
|
+
{message.role === 'assistant' && message.model && (
|
|
1576
|
+
<p className="text-[10px] font-mono text-muted-foreground/60 mt-1.5 select-none">{message.model}</p>
|
|
1577
|
+
)}
|
|
1578
|
+
</div>
|
|
1579
|
+
{message.role === 'user' && (
|
|
1580
|
+
<div className="w-8 h-8 rounded-full bg-primary/20 flex items-center justify-center shrink-0">
|
|
1581
|
+
<User className="w-4 h-4 text-primary" />
|
|
1582
|
+
</div>
|
|
1583
|
+
)}
|
|
1584
|
+
</div>
|
|
1585
|
+
))}
|
|
1586
|
+
|
|
1587
|
+
{isLoading && messages[messages.length - 1]?.content === '' && !messages[messages.length - 1]?.toolCalls && (
|
|
1588
|
+
<div className="flex gap-3 items-center text-muted-foreground">
|
|
1589
|
+
<div className="w-8 h-8 rounded-full bg-secondary flex items-center justify-center">
|
|
1590
|
+
<Loader2 className="w-4 h-4 animate-spin" />
|
|
1591
|
+
</div>
|
|
1592
|
+
<span className="text-sm font-mono">
|
|
1593
|
+
{isExecutingTools
|
|
1594
|
+
? 'Executing tools...'
|
|
1595
|
+
: currentIteration > 0
|
|
1596
|
+
? `Thinking (iteration ${currentIteration})...`
|
|
1597
|
+
: 'Thinking...'}
|
|
1598
|
+
</span>
|
|
1599
|
+
</div>
|
|
1600
|
+
)}
|
|
1601
|
+
|
|
1602
|
+
<div ref={messagesEndRef} />
|
|
1603
|
+
</div>
|
|
1604
|
+
|
|
1605
|
+
{/* Pending Images Preview */}
|
|
1606
|
+
{pendingImages.length > 0 && (
|
|
1607
|
+
<div className="border-t border-border px-4 py-2 flex gap-2 flex-wrap bg-secondary/30">
|
|
1608
|
+
{pendingImages.map((img, i) => (
|
|
1609
|
+
<div key={i} className="relative group">
|
|
1610
|
+
<img
|
|
1611
|
+
src={img}
|
|
1612
|
+
alt={`Pending ${i + 1}`}
|
|
1613
|
+
className="h-16 w-16 object-cover rounded border border-border"
|
|
1614
|
+
/>
|
|
1615
|
+
<button
|
|
1616
|
+
onClick={() => setPendingImages(prev => prev.filter((_, j) => j !== i))}
|
|
1617
|
+
className="absolute -top-1 -right-1 w-5 h-5 rounded-full bg-destructive text-destructive-foreground flex items-center justify-center opacity-0 group-hover:opacity-100 transition-opacity"
|
|
1618
|
+
>
|
|
1619
|
+
<X className="w-3 h-3" />
|
|
1620
|
+
</button>
|
|
1621
|
+
</div>
|
|
1622
|
+
))}
|
|
1623
|
+
</div>
|
|
1624
|
+
)}
|
|
1625
|
+
|
|
1626
|
+
{callModeEnabled && pendingAudio && (
|
|
1627
|
+
<div className="border-t border-border px-4 py-2 bg-secondary/20">
|
|
1628
|
+
<div className="flex items-center gap-2">
|
|
1629
|
+
<audio controls src={pendingAudio} className="w-full max-w-md" />
|
|
1630
|
+
<Button
|
|
1631
|
+
type="button"
|
|
1632
|
+
variant="ghost"
|
|
1633
|
+
size="icon"
|
|
1634
|
+
className="h-7 w-7 shrink-0"
|
|
1635
|
+
onClick={cancelRecording}
|
|
1636
|
+
title="Remove pending audio"
|
|
1637
|
+
>
|
|
1638
|
+
<X className="w-3 h-3" />
|
|
1639
|
+
</Button>
|
|
1640
|
+
</div>
|
|
1641
|
+
</div>
|
|
1642
|
+
)}
|
|
1643
|
+
|
|
1644
|
+
{/* Input Area */}
|
|
1645
|
+
<div className="border-t border-border p-4">
|
|
1646
|
+
{/* Image Size Picker (shown for diffusion models) */}
|
|
1647
|
+
{selectedModelSupportsImageOutput() && (
|
|
1648
|
+
<div className="mb-3 flex items-center gap-2">
|
|
1649
|
+
<span className="text-xs text-muted-foreground">Image Size:</span>
|
|
1650
|
+
<div className="flex gap-1 flex-wrap">
|
|
1651
|
+
{IMAGE_SIZE_OPTIONS.map((option) => (
|
|
1652
|
+
<button
|
|
1653
|
+
key={option.value}
|
|
1654
|
+
type="button"
|
|
1655
|
+
onClick={() => setImageSize(option.value)}
|
|
1656
|
+
className={cn(
|
|
1657
|
+
'px-2 py-1 text-xs font-mono rounded transition-colors',
|
|
1658
|
+
imageSize === option.value
|
|
1659
|
+
? 'bg-primary text-primary-foreground'
|
|
1660
|
+
: 'bg-secondary text-muted-foreground hover:text-foreground hover:bg-secondary/80'
|
|
1661
|
+
)}
|
|
1662
|
+
>
|
|
1663
|
+
{option.label}
|
|
1664
|
+
</button>
|
|
1665
|
+
))}
|
|
1666
|
+
</div>
|
|
1667
|
+
</div>
|
|
1668
|
+
)}
|
|
1669
|
+
<div className="mb-3">
|
|
1670
|
+
<button
|
|
1671
|
+
type="button"
|
|
1672
|
+
className="w-full text-left text-xs font-mono text-muted-foreground border border-border rounded px-2 py-1 hover:bg-secondary/50"
|
|
1673
|
+
onClick={() => setShowGenerationParams((prev) => !prev)}
|
|
1674
|
+
>
|
|
1675
|
+
{showGenerationParams ? 'Hide Generation Params' : 'Show Generation Params'}
|
|
1676
|
+
</button>
|
|
1677
|
+
{showGenerationParams && (
|
|
1678
|
+
<div className="mt-2 grid grid-cols-2 gap-2 rounded border border-border/60 bg-secondary/20 p-2">
|
|
1679
|
+
<label className="text-xs text-muted-foreground block">
|
|
1680
|
+
Temperature
|
|
1681
|
+
<input
|
|
1682
|
+
className="mt-1 w-full bg-input border border-border rounded px-2 py-1 text-sm font-mono"
|
|
1683
|
+
value={generationParams.temperature}
|
|
1684
|
+
onChange={(event) =>
|
|
1685
|
+
setGenerationParams((prev) => ({ ...prev, temperature: event.target.value }))
|
|
1686
|
+
}
|
|
1687
|
+
placeholder="e.g. 0.7"
|
|
1688
|
+
/>
|
|
1689
|
+
</label>
|
|
1690
|
+
<label className="text-xs text-muted-foreground block">
|
|
1691
|
+
Top P
|
|
1692
|
+
<input
|
|
1693
|
+
className="mt-1 w-full bg-input border border-border rounded px-2 py-1 text-sm font-mono"
|
|
1694
|
+
value={generationParams.topP}
|
|
1695
|
+
onChange={(event) =>
|
|
1696
|
+
setGenerationParams((prev) => ({ ...prev, topP: event.target.value }))
|
|
1697
|
+
}
|
|
1698
|
+
placeholder="e.g. 1"
|
|
1699
|
+
/>
|
|
1700
|
+
</label>
|
|
1701
|
+
<label className="text-xs text-muted-foreground block">
|
|
1702
|
+
Max Tokens
|
|
1703
|
+
<input
|
|
1704
|
+
className="mt-1 w-full bg-input border border-border rounded px-2 py-1 text-sm font-mono"
|
|
1705
|
+
value={generationParams.maxTokens}
|
|
1706
|
+
onChange={(event) =>
|
|
1707
|
+
setGenerationParams((prev) => ({ ...prev, maxTokens: event.target.value }))
|
|
1708
|
+
}
|
|
1709
|
+
placeholder="e.g. 512"
|
|
1710
|
+
/>
|
|
1711
|
+
</label>
|
|
1712
|
+
<label className="text-xs text-muted-foreground block">
|
|
1713
|
+
Presence Penalty
|
|
1714
|
+
<input
|
|
1715
|
+
className="mt-1 w-full bg-input border border-border rounded px-2 py-1 text-sm font-mono"
|
|
1716
|
+
value={generationParams.presencePenalty}
|
|
1717
|
+
onChange={(event) =>
|
|
1718
|
+
setGenerationParams((prev) => ({ ...prev, presencePenalty: event.target.value }))
|
|
1719
|
+
}
|
|
1720
|
+
placeholder="-2 to 2"
|
|
1721
|
+
/>
|
|
1722
|
+
</label>
|
|
1723
|
+
<label className="text-xs text-muted-foreground block">
|
|
1724
|
+
Frequency Penalty
|
|
1725
|
+
<input
|
|
1726
|
+
className="mt-1 w-full bg-input border border-border rounded px-2 py-1 text-sm font-mono"
|
|
1727
|
+
value={generationParams.frequencyPenalty}
|
|
1728
|
+
onChange={(event) =>
|
|
1729
|
+
setGenerationParams((prev) => ({ ...prev, frequencyPenalty: event.target.value }))
|
|
1730
|
+
}
|
|
1731
|
+
placeholder="-2 to 2"
|
|
1732
|
+
/>
|
|
1733
|
+
</label>
|
|
1734
|
+
<label className="text-xs text-muted-foreground block">
|
|
1735
|
+
Seed
|
|
1736
|
+
<input
|
|
1737
|
+
className="mt-1 w-full bg-input border border-border rounded px-2 py-1 text-sm font-mono"
|
|
1738
|
+
value={generationParams.seed}
|
|
1739
|
+
onChange={(event) =>
|
|
1740
|
+
setGenerationParams((prev) => ({ ...prev, seed: event.target.value }))
|
|
1741
|
+
}
|
|
1742
|
+
placeholder="integer"
|
|
1743
|
+
/>
|
|
1744
|
+
</label>
|
|
1745
|
+
<label className="text-xs text-muted-foreground block col-span-2">
|
|
1746
|
+
Stop Sequences (comma-separated)
|
|
1747
|
+
<input
|
|
1748
|
+
className="mt-1 w-full bg-input border border-border rounded px-2 py-1 text-sm font-mono"
|
|
1749
|
+
value={generationParams.stop}
|
|
1750
|
+
onChange={(event) =>
|
|
1751
|
+
setGenerationParams((prev) => ({ ...prev, stop: event.target.value }))
|
|
1752
|
+
}
|
|
1753
|
+
placeholder="END, STOP"
|
|
1754
|
+
/>
|
|
1755
|
+
</label>
|
|
1756
|
+
</div>
|
|
1757
|
+
)}
|
|
1758
|
+
</div>
|
|
1759
|
+
<form onSubmit={handleSubmit} className="flex gap-3">
|
|
1760
|
+
<input
|
|
1761
|
+
type="file"
|
|
1762
|
+
ref={fileInputRef}
|
|
1763
|
+
className="hidden"
|
|
1764
|
+
accept={callModeEnabled ? "image/*,audio/*" : "image/*"}
|
|
1765
|
+
multiple
|
|
1766
|
+
onChange={(e) => {
|
|
1767
|
+
if (!e.target.files) return
|
|
1768
|
+
processSelectedFiles(e.target.files)
|
|
1769
|
+
e.target.value = ''
|
|
1770
|
+
}}
|
|
1771
|
+
/>
|
|
1772
|
+
<Button
|
|
1773
|
+
type="button"
|
|
1774
|
+
variant="outline"
|
|
1775
|
+
size="icon"
|
|
1776
|
+
className="shrink-0"
|
|
1777
|
+
title={callModeEnabled ? "Attach image/audio" : "Attach image"}
|
|
1778
|
+
onClick={() => fileInputRef.current?.click()}
|
|
1779
|
+
>
|
|
1780
|
+
<ImagePlus className="w-4 h-4" />
|
|
1781
|
+
</Button>
|
|
1782
|
+
{callModeEnabled && (
|
|
1783
|
+
<Button
|
|
1784
|
+
type="button"
|
|
1785
|
+
variant={callStatus === 'recording' ? 'destructive' : 'outline'}
|
|
1786
|
+
size="icon"
|
|
1787
|
+
className="shrink-0"
|
|
1788
|
+
title={callStatus === 'recording' ? 'Stop recording' : 'Start recording'}
|
|
1789
|
+
disabled={isLoading || callStatus === 'sending'}
|
|
1790
|
+
onClick={() => {
|
|
1791
|
+
if (callStatus === 'recording') {
|
|
1792
|
+
stopRecording()
|
|
1793
|
+
} else {
|
|
1794
|
+
void startRecording()
|
|
1795
|
+
}
|
|
1796
|
+
}}
|
|
1797
|
+
>
|
|
1798
|
+
{callStatus === 'recording' ? (
|
|
1799
|
+
<StopCircle className="w-4 h-4" />
|
|
1800
|
+
) : (
|
|
1801
|
+
<Mic className="w-4 h-4" />
|
|
1802
|
+
)}
|
|
1803
|
+
</Button>
|
|
1804
|
+
)}
|
|
1805
|
+
<Textarea
|
|
1806
|
+
ref={inputRef}
|
|
1807
|
+
value={input}
|
|
1808
|
+
onChange={(e) => setInput(e.target.value)}
|
|
1809
|
+
onKeyDown={handleKeyDown}
|
|
1810
|
+
onPaste={handlePaste}
|
|
1811
|
+
placeholder={
|
|
1812
|
+
callModeEnabled
|
|
1813
|
+
? 'Record audio with mic, then optionally add text or images...'
|
|
1814
|
+
: agentModeEnabled && selectedTools.size > 0
|
|
1815
|
+
? 'Ask me anything... (agent mode enabled)'
|
|
1816
|
+
: 'Type a message... (paste or drop images)'
|
|
1817
|
+
}
|
|
1818
|
+
className="min-h-[44px] leading-5"
|
|
1819
|
+
rows={1}
|
|
1820
|
+
/>
|
|
1821
|
+
{isLoading && callModeEnabled ? (
|
|
1822
|
+
<Button
|
|
1823
|
+
type="button"
|
|
1824
|
+
disabled
|
|
1825
|
+
className="shrink-0"
|
|
1826
|
+
>
|
|
1827
|
+
<Loader2 className="w-4 h-4 animate-spin" />
|
|
1828
|
+
</Button>
|
|
1829
|
+
) : isLoading ? (
|
|
1830
|
+
<Button
|
|
1831
|
+
type="button"
|
|
1832
|
+
variant="destructive"
|
|
1833
|
+
className="shrink-0"
|
|
1834
|
+
onClick={stopAgent}
|
|
1835
|
+
>
|
|
1836
|
+
<StopCircle className="w-4 h-4" />
|
|
1837
|
+
</Button>
|
|
1838
|
+
) : (
|
|
1839
|
+
<Button
|
|
1840
|
+
type="submit"
|
|
1841
|
+
disabled={
|
|
1842
|
+
isLoading ||
|
|
1843
|
+
!selectedModel ||
|
|
1844
|
+
!selectedModelIsSelectable ||
|
|
1845
|
+
(callModeEnabled
|
|
1846
|
+
? !pendingAudio
|
|
1847
|
+
: (!input.trim() && pendingImages.length === 0))
|
|
1848
|
+
}
|
|
1849
|
+
className="shrink-0"
|
|
1850
|
+
>
|
|
1851
|
+
<Send className="w-4 h-4" />
|
|
1852
|
+
</Button>
|
|
1853
|
+
)}
|
|
1854
|
+
</form>
|
|
1855
|
+
<p className="text-2xs text-muted-foreground mt-2 text-center font-mono">
|
|
1856
|
+
Enter to send | Shift+Enter for new line
|
|
1857
|
+
{agentModeEnabled && ' | Agent mode: max 10 iterations'}
|
|
1858
|
+
{callModeEnabled && ' | Call mode: record, then send'}
|
|
1859
|
+
</p>
|
|
1860
|
+
{callModeEnabled && (
|
|
1861
|
+
<p className="text-2xs text-muted-foreground mt-1 text-center font-mono">
|
|
1862
|
+
Status: {callStatus}
|
|
1863
|
+
</p>
|
|
1864
|
+
)}
|
|
1865
|
+
{callError && (
|
|
1866
|
+
<p className="text-2xs text-destructive mt-1 text-center font-mono">
|
|
1867
|
+
{callError}
|
|
1868
|
+
</p>
|
|
1869
|
+
)}
|
|
1870
|
+
</div>
|
|
1871
|
+
</div>
|
|
1872
|
+
|
|
1873
|
+
{/* Tool Picker Sidebar */}
|
|
1874
|
+
{agentModeEnabled && showToolPicker && (
|
|
1875
|
+
<aside className="w-72 border-l border-border flex flex-col shrink-0 animate-slide-in-right">
|
|
1876
|
+
<ToolPicker
|
|
1877
|
+
selectedTools={selectedTools}
|
|
1878
|
+
onToolsChange={setSelectedTools}
|
|
1879
|
+
className="h-full"
|
|
1880
|
+
/>
|
|
1881
|
+
</aside>
|
|
1882
|
+
)}
|
|
1883
|
+
</div>
|
|
1884
|
+
</div>
|
|
1885
|
+
</div>
|
|
1886
|
+
)
|
|
1887
|
+
}
|
|
1888
|
+
|
|
1889
|
+
// Enhanced stream function that captures tool calls
|
|
1890
|
+
async function* streamChatCompletionWithTools(
|
|
1891
|
+
request: {
|
|
1892
|
+
model: string
|
|
1893
|
+
messages: ApiChatMessage[]
|
|
1894
|
+
temperature?: number
|
|
1895
|
+
top_p?: number
|
|
1896
|
+
max_tokens?: number
|
|
1897
|
+
presence_penalty?: number
|
|
1898
|
+
frequency_penalty?: number
|
|
1899
|
+
seed?: number
|
|
1900
|
+
stop?: string | string[]
|
|
1901
|
+
tools?: unknown[]
|
|
1902
|
+
tool_choice?: unknown
|
|
1903
|
+
},
|
|
1904
|
+
signal: AbortSignal,
|
|
1905
|
+
onToolCalls: (toolCalls: Array<{ id: string; function: { name: string; arguments: string } }>) => void
|
|
1906
|
+
): AsyncGenerator<string, void, unknown> {
|
|
1907
|
+
// Build request body, omitting tools/tool_choice if not present
|
|
1908
|
+
const body: Record<string, unknown> = {
|
|
1909
|
+
model: request.model,
|
|
1910
|
+
messages: request.messages,
|
|
1911
|
+
stream: true,
|
|
1912
|
+
temperature: request.temperature,
|
|
1913
|
+
top_p: request.top_p,
|
|
1914
|
+
max_tokens: request.max_tokens,
|
|
1915
|
+
presence_penalty: request.presence_penalty,
|
|
1916
|
+
frequency_penalty: request.frequency_penalty,
|
|
1917
|
+
seed: request.seed,
|
|
1918
|
+
stop: request.stop,
|
|
1919
|
+
}
|
|
1920
|
+
|
|
1921
|
+
// Only include tools if we have them
|
|
1922
|
+
// Note: We don't send tool_choice by default as many backends (like vLLM)
|
|
1923
|
+
// require special configuration for it. Models will still use tools if provided.
|
|
1924
|
+
if (request.tools && request.tools.length > 0) {
|
|
1925
|
+
body.tools = request.tools
|
|
1926
|
+
// Uncomment if your backend supports tool_choice:
|
|
1927
|
+
// if (request.tool_choice) {
|
|
1928
|
+
// body.tool_choice = request.tool_choice
|
|
1929
|
+
// }
|
|
1930
|
+
}
|
|
1931
|
+
|
|
1932
|
+
const response = await fetch('/v1/chat/completions', {
|
|
1933
|
+
method: 'POST',
|
|
1934
|
+
headers: { 'Content-Type': 'application/json' },
|
|
1935
|
+
body: JSON.stringify(body),
|
|
1936
|
+
signal,
|
|
1937
|
+
})
|
|
1938
|
+
|
|
1939
|
+
if (!response.ok) {
|
|
1940
|
+
throw new Error(`API Error: ${response.status} ${response.statusText}`)
|
|
1941
|
+
}
|
|
1942
|
+
|
|
1943
|
+
const reader = response.body?.getReader()
|
|
1944
|
+
if (!reader) throw new Error('No response body')
|
|
1945
|
+
|
|
1946
|
+
const decoder = new TextDecoder()
|
|
1947
|
+
let buffer = ''
|
|
1948
|
+
const toolCallsMap = new Map<number, { id: string; function: { name: string; arguments: string } }>()
|
|
1949
|
+
|
|
1950
|
+
while (true) {
|
|
1951
|
+
const { done, value } = await reader.read()
|
|
1952
|
+
if (done) break
|
|
1953
|
+
|
|
1954
|
+
buffer += decoder.decode(value, { stream: true })
|
|
1955
|
+
const lines = buffer.split('\n')
|
|
1956
|
+
buffer = lines.pop() || ''
|
|
1957
|
+
|
|
1958
|
+
for (const line of lines) {
|
|
1959
|
+
if (line.startsWith('data: ')) {
|
|
1960
|
+
const data = line.slice(6)
|
|
1961
|
+
if (data === '[DONE]') {
|
|
1962
|
+
// Finalize tool calls
|
|
1963
|
+
if (toolCallsMap.size > 0) {
|
|
1964
|
+
onToolCalls(Array.from(toolCallsMap.values()))
|
|
1965
|
+
}
|
|
1966
|
+
return
|
|
1967
|
+
}
|
|
1968
|
+
|
|
1969
|
+
try {
|
|
1970
|
+
const parsed = JSON.parse(data)
|
|
1971
|
+
const delta = parsed.choices?.[0]?.delta
|
|
1972
|
+
|
|
1973
|
+
// Handle text content
|
|
1974
|
+
if (delta?.content) {
|
|
1975
|
+
yield delta.content
|
|
1976
|
+
}
|
|
1977
|
+
|
|
1978
|
+
// Handle tool calls (accumulated across chunks)
|
|
1979
|
+
if (delta?.tool_calls) {
|
|
1980
|
+
for (const tc of delta.tool_calls) {
|
|
1981
|
+
const index = tc.index ?? 0
|
|
1982
|
+
const existing = toolCallsMap.get(index)
|
|
1983
|
+
|
|
1984
|
+
if (!existing) {
|
|
1985
|
+
toolCallsMap.set(index, {
|
|
1986
|
+
id: tc.id || '',
|
|
1987
|
+
function: {
|
|
1988
|
+
name: tc.function?.name || '',
|
|
1989
|
+
arguments: tc.function?.arguments || '',
|
|
1990
|
+
},
|
|
1991
|
+
})
|
|
1992
|
+
} else {
|
|
1993
|
+
if (tc.id) existing.id = tc.id
|
|
1994
|
+
if (tc.function?.name) existing.function.name += tc.function.name
|
|
1995
|
+
if (tc.function?.arguments) existing.function.arguments += tc.function.arguments
|
|
1996
|
+
}
|
|
1997
|
+
}
|
|
1998
|
+
}
|
|
1999
|
+
} catch {
|
|
2000
|
+
// Skip malformed JSON
|
|
2001
|
+
}
|
|
2002
|
+
}
|
|
2003
|
+
}
|
|
2004
|
+
}
|
|
2005
|
+
|
|
2006
|
+
// Finalize any remaining tool calls
|
|
2007
|
+
if (toolCallsMap.size > 0) {
|
|
2008
|
+
onToolCalls(Array.from(toolCallsMap.values()))
|
|
2009
|
+
}
|
|
2010
|
+
}
|