sang-ai-kit 0.2.1 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/package.json +1 -1
- package/src/AiAssistant.tsx +9 -8
- package/src/ImageModelSettings.tsx +14 -0
- package/src/SessionList.tsx +6 -6
- package/src/assistant.css +43 -0
- package/src/assistantTypes.ts +2 -1
- package/src/builtInToolDefinitions.ts +7 -0
- package/src/builtInTools.ts +15 -8
- package/src/image-generation/adapters/agnes.ts +2 -2
- package/src/image-generation/adapters/gptImage.ts +8 -6
- package/src/image-generation/adapters/kolors.ts +2 -2
- package/src/image-generation/index.ts +7 -3
- package/src/image-generation/models.ts +3 -2
- package/src/image-generation/types.ts +3 -2
- package/src/imageToolHost.ts +1 -0
- package/src/sessionTypes.ts +1 -0
- package/src/useAssistant.ts +38 -10
package/README.md
CHANGED
|
@@ -28,7 +28,7 @@ bun run dev
|
|
|
28
28
|
| Kolors、Agnes、GPT Image 生图协议、图片存储和密钥管理 | 将图片或工具结果应用到宿主工作区 |
|
|
29
29
|
| 可选 `useAssistant` 通用会话控制器和浏览器加密存储 | 宿主业务操作的撤销/重做及原生文件存储 |
|
|
30
30
|
|
|
31
|
-
`useAssistant` 通过 `AssistantRunner`
|
|
31
|
+
`useAssistant` 通过 `AssistantRunner` 接收文本、思考、工具状态和原生历史;通用控制器支持主动重新生成最近一轮,并复用该轮图片附件。重新生成可能再次执行工具或产生费用,不会撤销外部操作;需要事务回滚的应用应自行提供可验证的重做实现。用户附件和工具结果均可使用 `attachment://` 资源引用保存图片历史,发送视觉请求时才加载,文本模型不读取图片;若接入方直接传入 data URL,则历史会包含该数据。
|
|
32
32
|
|
|
33
33
|
工作区读写、业务工具和原生文件权限由接入方明确实现。组件的宿主接口目前仍处于开发阶段,尚未承诺稳定的发布 API。
|
|
34
34
|
|
|
@@ -50,7 +50,7 @@ bun run dev
|
|
|
50
50
|
|
|
51
51
|
`createResponsesSender` 可选接收 `onRequest({ endpoint, body })` 观察器,在每次请求发送前报告最终 Responses URL 与 JSON body;快照不包含 HTTP headers 或 API Key。它适合诊断最终 instructions、工具 Schema 和上下文输入。生产宿主接入时应自行决定是否记录、展示或脱敏其中的用户消息与附件数据;demo 检查器会脱敏图像 Base64。
|
|
52
52
|
|
|
53
|
-
对话数据的所有权由调用方决定。使用 `useAssistant(runner, storage)` 时必须传入存储适配器;`AssistantStorage` 是通用接口,不代表组件会自动选择或创建持久化位置。宿主可接入自己的 IndexedDB、文件或服务端存储。消息上的分叉入口只有在传入 `onFork` 时显示;点击后只回调 `{ sessionId, entryId, responseItemCount? }`,不在共享组件内创建会话或复制图片。宿主负责验证标记、截断对应消息与 Responses
|
|
53
|
+
对话数据的所有权由调用方决定。使用 `useAssistant(runner, storage)` 时必须传入存储适配器;`AssistantStorage` 是通用接口,不代表组件会自动选择或创建持久化位置。宿主可接入自己的 IndexedDB、文件或服务端存储。消息上的分叉入口只有在传入 `onFork` 时显示;点击后只回调 `{ sessionId, entryId, responseItemCount? }`,不在共享组件内创建会话或复制图片。宿主负责验证标记、截断对应消息与 Responses 历史、复制引用图片并保存新会话。重新生成入口默认只出现在最后一条已完成的 AI 回复上,宿主可通过 `canRegenerate` 控制;具有画布等外部状态的宿主应自行提供可验证的恢复能力。
|
|
54
54
|
|
|
55
55
|
图片画廊的插入操作由 `assistant.insertImage` 提供;未提供时不显示按钮。`AiAssistant` 的 `imageInsertAction` 可随宿主状态更新 `visible`、`enabled`、`label`、`successLabel` 和 `disabledReason`,`isVisible(item)` 可控制单张图片的按钮;默认不显示预览图的插入按钮。文案由宿主按当前语言提供,未配置时使用通用文案。宿主负责读取图片、决定插入位置及处理失败。默认将内置生图识别为生成图片,其余归为附件;宿主可通过 `classifyGalleryImage(image, tools)` 返回 `generated`、`preview` 或 `attachment`,返回 `undefined` 时使用默认分类。删除图片的回调由宿主负责实际清理,画廊不假设外部引用的存储方式。
|
|
56
56
|
|
package/package.json
CHANGED
package/src/AiAssistant.tsx
CHANGED
|
@@ -38,7 +38,7 @@ import './assistant.css';
|
|
|
38
38
|
|
|
39
39
|
export default function AiAssistant({
|
|
40
40
|
assistant, floating, language, theme = 'dark', title, emptyTitle, emptyDescription, contextTitle, contextMeta, imageInsertAction, classifyGalleryImage,
|
|
41
|
-
quickSnippets, features: featureSettings, onFork, active, imageToolsEnabled, toolLabels = {}, loadImage, createImageUpload, imageModelForTool,
|
|
41
|
+
quickSnippets, features: featureSettings, onFork, active, imageToolsEnabled, maxImageAttachments = 4, toolLabels = {}, loadImage, createImageUpload, imageModelForTool,
|
|
42
42
|
sumUsage, renderMarkdown, renderUsage, renderModelControls, renderSettings,
|
|
43
43
|
open: propOpen,
|
|
44
44
|
onOpenChange
|
|
@@ -80,6 +80,7 @@ export default function AiAssistant({
|
|
|
80
80
|
const english = language === 'en';
|
|
81
81
|
const controlsDisabled = assistant.busy || assistant.loading || assistant.saving || Boolean(assistant.loadError);
|
|
82
82
|
const label = (zh: string, en: string) => (english ? en : zh);
|
|
83
|
+
const imageAttachmentLimit = Number.isSafeInteger(maxImageAttachments) && maxImageAttachments > 0 ? maxImageAttachments : 4;
|
|
83
84
|
|
|
84
85
|
const activeSession = assistant.sessions.find(session => session.id === assistant.activeSessionId);
|
|
85
86
|
const currentSessionTitle = activeSession?.title || label('新对话', 'New Chat');
|
|
@@ -189,8 +190,8 @@ export default function AiAssistant({
|
|
|
189
190
|
if (controlsDisabled || !active.model.supportsVision) return;
|
|
190
191
|
const selectedFiles = files ? Array.from(files) : [];
|
|
191
192
|
if (!selectedFiles.length) return;
|
|
192
|
-
if (imageUploads.length + selectedFiles.length >
|
|
193
|
-
setInputAlert(label(
|
|
193
|
+
if (imageUploads.length + selectedFiles.length > imageAttachmentLimit) {
|
|
194
|
+
setInputAlert(label(`每条消息最多添加 ${imageAttachmentLimit} 张图片。`, `Attach up to ${imageAttachmentLimit} images per message.`));
|
|
194
195
|
return;
|
|
195
196
|
}
|
|
196
197
|
try {
|
|
@@ -589,15 +590,15 @@ export default function AiAssistant({
|
|
|
589
590
|
</button>}
|
|
590
591
|
|
|
591
592
|
{/* 左2:图标重新生成 (仅图标,无文字,样式与复制按钮完全一致) */}
|
|
592
|
-
{isLastAssistant && assistant.canRegenerate && (
|
|
593
|
+
{isLastAssistant && assistant.canRegenerate !== false && (
|
|
593
594
|
<button
|
|
594
595
|
type="button"
|
|
595
596
|
disabled={controlsDisabled}
|
|
596
597
|
onClick={() => void assistant.regenerate()}
|
|
597
598
|
className="cf-ai-action-btn flex items-center justify-center w-6 h-6 rounded-md text-slate-400 hover:text-slate-100 hover:bg-slate-800/80 transition-colors cursor-pointer select-none disabled:opacity-40 disabled:cursor-not-allowed"
|
|
598
599
|
title={label(
|
|
599
|
-
'
|
|
600
|
-
'Regenerate from last user input
|
|
600
|
+
'从上次用户输入开始重新生成',
|
|
601
|
+
'Regenerate from the last user input'
|
|
601
602
|
)}
|
|
602
603
|
aria-label={label('重新生成 (重做)', 'Regenerate (Redo)')}
|
|
603
604
|
>
|
|
@@ -850,9 +851,9 @@ export default function AiAssistant({
|
|
|
850
851
|
<button
|
|
851
852
|
type="button"
|
|
852
853
|
onClick={() => imageInputRef.current?.click()}
|
|
853
|
-
disabled={controlsDisabled || imageUploads.length >=
|
|
854
|
+
disabled={controlsDisabled || imageUploads.length >= imageAttachmentLimit}
|
|
854
855
|
className="flex h-6 w-6 shrink-0 items-center justify-center rounded-md text-slate-400 hover:bg-slate-800 hover:text-emerald-300 disabled:opacity-30"
|
|
855
|
-
title={label(
|
|
856
|
+
title={label(`添加图片/生图参考(最多 ${imageAttachmentLimit} 张)`, `Attach images / generation references (up to ${imageAttachmentLimit})`)}
|
|
856
857
|
aria-label={label('添加图片', 'Attach image')}
|
|
857
858
|
>
|
|
858
859
|
<ImagePlus size={14} />
|
|
@@ -492,6 +492,20 @@ export default function ImageModelSettings({ config, onChange, english }: Props)
|
|
|
492
492
|
</div>
|
|
493
493
|
|
|
494
494
|
<div className="grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-4 gap-3">
|
|
495
|
+
<div>
|
|
496
|
+
<label htmlFor={`${model.id}-references`} className={labelClass}>{label('参考图数量上限(0 为不支持)', 'Max Reference Images (0 disables)')}</label>
|
|
497
|
+
<input
|
|
498
|
+
id={`${model.id}-references`}
|
|
499
|
+
type="number"
|
|
500
|
+
min={0}
|
|
501
|
+
max={model.adapter === 'kolors' ? 1 : undefined}
|
|
502
|
+
step={1}
|
|
503
|
+
required
|
|
504
|
+
className={`${inputClass} font-mono`}
|
|
505
|
+
value={model.maxReferenceImages ?? 1}
|
|
506
|
+
onChange={event => update(model.id, { maxReferenceImages: Number(event.target.value) })}
|
|
507
|
+
/>
|
|
508
|
+
</div>
|
|
495
509
|
<div>
|
|
496
510
|
<label className={labelClass}>{label('默认尺寸', 'Default Size')}</label>
|
|
497
511
|
{!capabilities.exactSizeSupported ? (
|
package/src/SessionList.tsx
CHANGED
|
@@ -164,7 +164,7 @@ export default function SessionList({
|
|
|
164
164
|
const itemsMap = new Map<string, GalleryItem>();
|
|
165
165
|
|
|
166
166
|
sessions.forEach(session => {
|
|
167
|
-
session.entries.forEach(entry => {
|
|
167
|
+
[...session.entries, ...(session.galleryEntries || [])].forEach(entry => {
|
|
168
168
|
// 1. entry.images
|
|
169
169
|
entry.images?.forEach(img => {
|
|
170
170
|
const key = `${session.id}:${img.id}`;
|
|
@@ -418,7 +418,7 @@ export default function SessionList({
|
|
|
418
418
|
/* ========================================================================= */
|
|
419
419
|
if (view === 'gallery') {
|
|
420
420
|
return (
|
|
421
|
-
<section className="w-full space-y-3.5 text-xs select-none" aria-label={label('图片资产画廊', 'Image Gallery')}>
|
|
421
|
+
<section className="cf-ai-gallery w-full space-y-3.5 text-xs select-none" aria-label={label('图片资产画廊', 'Image Gallery')}>
|
|
422
422
|
{/* 画廊头部导航 */}
|
|
423
423
|
<div className="flex flex-wrap items-center justify-between gap-2 pb-1 border-b border-slate-800/80">
|
|
424
424
|
<button
|
|
@@ -587,7 +587,7 @@ export default function SessionList({
|
|
|
587
587
|
</p>
|
|
588
588
|
</div>
|
|
589
589
|
) : (
|
|
590
|
-
<div className="grid grid-cols-2 gap-2.5">
|
|
590
|
+
<div className="cf-ai-gallery-grid grid grid-cols-2 gap-2.5">
|
|
591
591
|
{filteredGalleryItems.map(item => {
|
|
592
592
|
const url = imageUrls[item.id];
|
|
593
593
|
const isLoading = loadingImages[item.id];
|
|
@@ -694,9 +694,9 @@ export default function SessionList({
|
|
|
694
694
|
<div className="text-[11px] font-semibold text-slate-200 truncate group-hover:text-blue-300 transition-colors">
|
|
695
695
|
{item.name}
|
|
696
696
|
</div>
|
|
697
|
-
<div className="flex items-center justify-between text-[10px] text-slate-400">
|
|
698
|
-
<span className="font-mono">{formatBytes(item.size)}</span>
|
|
699
|
-
<span className="text-slate-500 truncate
|
|
697
|
+
<div className="flex items-center justify-between gap-2 text-[10px] text-slate-400">
|
|
698
|
+
<span className="font-mono shrink-0">{formatBytes(item.size)}</span>
|
|
699
|
+
<span className="text-slate-500 truncate min-w-0" title={item.sessionTitle}>
|
|
700
700
|
{item.sessionTitle}
|
|
701
701
|
</span>
|
|
702
702
|
</div>
|
package/src/assistant.css
CHANGED
|
@@ -2053,6 +2053,49 @@
|
|
|
2053
2053
|
min-width: 0;
|
|
2054
2054
|
}
|
|
2055
2055
|
|
|
2056
|
+
/* 图片资产画廊:根据画廊容器宽度动态调整每行展示图片数量 */
|
|
2057
|
+
.cf-ai-gallery {
|
|
2058
|
+
container-type: inline-size;
|
|
2059
|
+
container-name: cf-ai-gallery;
|
|
2060
|
+
min-width: 0;
|
|
2061
|
+
}
|
|
2062
|
+
|
|
2063
|
+
.cf-ai-gallery-grid {
|
|
2064
|
+
display: grid;
|
|
2065
|
+
gap: 0.625rem;
|
|
2066
|
+
grid-template-columns: repeat(auto-fill, minmax(min(100%, 150px), 1fr));
|
|
2067
|
+
}
|
|
2068
|
+
|
|
2069
|
+
@container cf-ai-gallery (max-width: 439px) {
|
|
2070
|
+
.cf-ai-gallery-grid {
|
|
2071
|
+
grid-template-columns: repeat(2, minmax(0, 1fr));
|
|
2072
|
+
}
|
|
2073
|
+
}
|
|
2074
|
+
|
|
2075
|
+
@container cf-ai-gallery (min-width: 440px) {
|
|
2076
|
+
.cf-ai-gallery-grid {
|
|
2077
|
+
grid-template-columns: repeat(3, minmax(0, 1fr));
|
|
2078
|
+
}
|
|
2079
|
+
}
|
|
2080
|
+
|
|
2081
|
+
@container cf-ai-gallery (min-width: 620px) {
|
|
2082
|
+
.cf-ai-gallery-grid {
|
|
2083
|
+
grid-template-columns: repeat(4, minmax(0, 1fr));
|
|
2084
|
+
}
|
|
2085
|
+
}
|
|
2086
|
+
|
|
2087
|
+
@container cf-ai-gallery (min-width: 800px) {
|
|
2088
|
+
.cf-ai-gallery-grid {
|
|
2089
|
+
grid-template-columns: repeat(5, minmax(0, 1fr));
|
|
2090
|
+
}
|
|
2091
|
+
}
|
|
2092
|
+
|
|
2093
|
+
@container cf-ai-gallery (min-width: 980px) {
|
|
2094
|
+
.cf-ai-gallery-grid {
|
|
2095
|
+
grid-template-columns: repeat(auto-fill, minmax(168px, 1fr));
|
|
2096
|
+
}
|
|
2097
|
+
}
|
|
2098
|
+
|
|
2056
2099
|
.cf-ai-spin {
|
|
2057
2100
|
animation: cf-ai-spin 1s linear infinite;
|
|
2058
2101
|
}
|
package/src/assistantTypes.ts
CHANGED
|
@@ -75,7 +75,7 @@ export interface AssistantController {
|
|
|
75
75
|
configError: string;
|
|
76
76
|
executionError: { message: string } | null;
|
|
77
77
|
confirmation: { message: string; resolve: (accepted: boolean) => void } | null;
|
|
78
|
-
canRegenerate
|
|
78
|
+
canRegenerate?: boolean;
|
|
79
79
|
send: (prompt: string, attachments?: ImageUpload[]) => Promise<void>;
|
|
80
80
|
stop: (source?: 'user_stop' | 'panel_closed') => void;
|
|
81
81
|
newSession: () => Promise<void>;
|
|
@@ -132,6 +132,7 @@ export interface AssistantViewProps {
|
|
|
132
132
|
onFork?: (event: AssistantForkEvent) => void;
|
|
133
133
|
active: { endpoint?: unknown; model: { label?: string; supportsVision: boolean } };
|
|
134
134
|
imageToolsEnabled?: boolean;
|
|
135
|
+
maxImageAttachments?: number;
|
|
135
136
|
toolLabels?: Record<string, [string, string]>;
|
|
136
137
|
loadImage: (sessionId: string, imageId: string, type?: string) => Promise<Blob | null>;
|
|
137
138
|
createImageUpload: (file: File, imageToolsEnabled?: boolean) => Promise<ImageUpload>;
|
|
@@ -19,11 +19,18 @@ export function imageToolDefinitions(models: ImagePromptConfig[] = []): BuiltInT
|
|
|
19
19
|
ratio: { type: 'string', description: '宽高比(如16:9)。仅部分模型支持,当不传 size 时生效。' },
|
|
20
20
|
quality: { type: 'string', description: '生成质量。越高越慢,省略用默认值。' },
|
|
21
21
|
referenceImageId: { type: 'string', description: '可选,原图 ID,来自附件或工具返回,作为参考图。' },
|
|
22
|
+
referenceImageIds: { type: 'array', items: { type: 'string' }, description: '可选,参考图 ID 数组,来自附件或工具返回;不要与 referenceImageId 同时使用。' },
|
|
22
23
|
n: { type: 'integer', minimum: 1, maximum: 10, description: '可选,生成数量,省略使用模型配置;须在模型支持范围内。' }
|
|
23
24
|
};
|
|
24
25
|
if (single) {
|
|
25
26
|
delete properties.modelConfigId;
|
|
26
27
|
const capabilities = imageCapabilities(single);
|
|
28
|
+
if (!capabilities.maxReferenceImages) {
|
|
29
|
+
delete properties.referenceImageId;
|
|
30
|
+
delete properties.referenceImageIds;
|
|
31
|
+
} else {
|
|
32
|
+
properties.referenceImageIds.maxItems = capabilities.maxReferenceImages;
|
|
33
|
+
}
|
|
27
34
|
properties.size.description += ` 默认值:${single.size};${capabilities.exactSizeSupported ? '推荐尺寸' : '支持尺寸'}:${capabilities.sizes.join('、')}。`;
|
|
28
35
|
if (!capabilities.ratios) delete properties.ratio;
|
|
29
36
|
else properties.ratio = { type: 'string', description: `仅档位尺寸使用;省略为 ${single.ratio}。`, enum: capabilities.ratios };
|
package/src/builtInTools.ts
CHANGED
|
@@ -65,30 +65,37 @@ export function createBuiltInTools(options: BuiltInToolOptions): { tools: Respon
|
|
|
65
65
|
let prompt: string;
|
|
66
66
|
try {
|
|
67
67
|
signal.throwIfAborted();
|
|
68
|
-
if (Object.keys(args).some(key => !['modelConfigId', 'prompt', 'referenceImageId', 'size', 'ratio', 'quality', 'n'].includes(key) && !Object.hasOwn(options.imageExtensions?.generate_image?.properties ?? {}, key))) throw new Error('Unknown image generation parameter.');
|
|
68
|
+
if (Object.keys(args).some(key => !['modelConfigId', 'prompt', 'referenceImageId', 'referenceImageIds', 'size', 'ratio', 'quality', 'n'].includes(key) && !Object.hasOwn(options.imageExtensions?.generate_image?.properties ?? {}, key))) throw new Error('Unknown image generation parameter.');
|
|
69
69
|
for (const key of options.imageExtensions?.generate_image?.required ?? []) if (args[key] === undefined) throw new Error(`${key} is required.`);
|
|
70
70
|
const modelId = args.modelConfigId ?? (imageModels.length === 1 ? imageModelToolId(imageModels[0]) : undefined);
|
|
71
71
|
if (typeof modelId !== 'string' || typeof args.prompt !== 'string') throw new Error('A model ID and prompt are required.');
|
|
72
72
|
const matches = imageModels.filter(model => imageModelToolId(model) === modelId);
|
|
73
73
|
if (matches.length !== 1) throw new Error('Image model is disabled, unavailable or ambiguous. Read list_image_models when multiple models are enabled.');
|
|
74
74
|
if (args.referenceImageId !== undefined && typeof args.referenceImageId !== 'string') throw new Error('Invalid reference image ID.');
|
|
75
|
+
if (args.referenceImageIds !== undefined && (!Array.isArray(args.referenceImageIds) || args.referenceImageIds.some(id => typeof id !== 'string'))) throw new Error('Invalid reference image IDs.');
|
|
76
|
+
if (args.referenceImageId !== undefined && args.referenceImageIds !== undefined) throw new Error('Use either referenceImageId or referenceImageIds.');
|
|
75
77
|
if (args.size !== undefined && typeof args.size !== 'string') throw new Error('Invalid image size.');
|
|
76
78
|
if (args.ratio !== undefined && typeof args.ratio !== 'string') throw new Error('Invalid image ratio.');
|
|
77
79
|
if (args.quality !== undefined && (typeof args.quality !== 'string' || !['low', 'medium', 'high', 'xhigh', 'max', 'auto'].includes(args.quality))) throw new Error('Invalid image quality.');
|
|
78
80
|
if (args.n !== undefined && (typeof args.n !== 'number' || !Number.isInteger(args.n))) throw new Error('Invalid image count.');
|
|
79
81
|
prepared = await options.imageExtensions?.generate_image?.prepare(args, signal);
|
|
80
|
-
if (prepared?.inputImage &&
|
|
81
|
-
|
|
82
|
-
|
|
82
|
+
if (prepared?.inputImage && prepared.inputImages !== undefined) throw new Error('Use either inputImage or inputImages.');
|
|
83
|
+
const ids = typeof args.referenceImageId === 'string' ? [args.referenceImageId] : (args.referenceImageIds as string[] | undefined) ?? [];
|
|
84
|
+
const inputImages = prepared?.inputImage ? [prepared.inputImage] : prepared?.inputImages ?? [];
|
|
85
|
+
if (inputImages.length && ids.length) throw new Error('Use either host input images or reference image IDs.');
|
|
86
|
+
if (ids.length + inputImages.length > (matches[0].maxReferenceImages ?? 1)) throw new Error(`This image model supports at most ${matches[0].maxReferenceImages ?? 1} reference images.`);
|
|
87
|
+
const references: string[] = [];
|
|
88
|
+
for (const id of ids) {
|
|
83
89
|
if (!options.loadImage) throw new Error('Image storage is not configured.');
|
|
84
|
-
const blob = await options.loadImage
|
|
90
|
+
const blob = await options.loadImage(id);
|
|
85
91
|
if (!blob) throw new Error('Reference image not found.');
|
|
86
|
-
|
|
87
|
-
}
|
|
92
|
+
references.push(await blobToDataUrl(blob));
|
|
93
|
+
}
|
|
94
|
+
for (const image of inputImages) references.push(await blobToDataUrl(image));
|
|
88
95
|
results = await generateImages(config, { modelConfigId: matches[0].id, prompt: args.prompt,
|
|
89
96
|
...(typeof args.size === 'string' ? { size: args.size } : {}), ...(typeof args.ratio === 'string' ? { ratio: args.ratio } : {}),
|
|
90
97
|
...(typeof args.quality === 'string' ? { quality: args.quality as ImageQuality } : {}), ...(typeof args.n === 'number' ? { n: args.n } : {})
|
|
91
|
-
}, signal,
|
|
98
|
+
}, signal, references);
|
|
92
99
|
model = matches[0].modelId;
|
|
93
100
|
prompt = args.prompt;
|
|
94
101
|
} catch (error) { if (!signal.aborted) failures++; throw error; }
|
|
@@ -14,10 +14,10 @@ export const agnes: ImageAdapter = {
|
|
|
14
14
|
if (!AGNES_SIZES.includes(model.size) && !/^[1-9]\d{1,3}x[1-9]\d{1,3}$/.test(model.size)) throw new Error(`Invalid Agnes size "${model.size}". Use ${AGNES_SIZES.join(', ')}, or WIDTHxHEIGHT with each side between 10 and 9999, such as 1024x768; omit ratio for exact dimensions.`);
|
|
15
15
|
if (!IMAGE_RATIOS.includes(model.ratio)) throw new Error('Invalid Agnes ratio.');
|
|
16
16
|
},
|
|
17
|
-
buildRequest(model, prompt,
|
|
17
|
+
buildRequest(model, prompt, references = []) {
|
|
18
18
|
return {
|
|
19
19
|
operation: 'generations', headers: { 'Content-Type': 'application/json' },
|
|
20
|
-
body: JSON.stringify({ model: model.modelId, prompt, size: model.size, ...(AGNES_SIZES.includes(model.size) ? { ratio: model.ratio } : {}), ...(
|
|
20
|
+
body: JSON.stringify({ model: model.modelId, prompt, size: model.size, ...(AGNES_SIZES.includes(model.size) ? { ratio: model.ratio } : {}), ...(references.length ? { extra_body: { image: references, response_format: 'b64_json' } } : { return_base64: true }) })
|
|
21
21
|
};
|
|
22
22
|
},
|
|
23
23
|
readResults: data => readImageResults(data, 'data')
|
|
@@ -23,15 +23,17 @@ export const gptImage: ImageAdapter = {
|
|
|
23
23
|
if (!qualities(model.modelId).includes(model.quality ?? 'high')) throw new Error('Unsupported GPT-Image quality.');
|
|
24
24
|
if (!Number.isInteger(model.n ?? 1) || (model.n ?? 1) < 1 || (model.n ?? 1) > 10) throw new Error('Image count must be between 1 and 10; provider limits may be lower.');
|
|
25
25
|
},
|
|
26
|
-
buildRequest(model, prompt,
|
|
26
|
+
buildRequest(model, prompt, references = []): ReturnType<ImageAdapter['buildRequest']> {
|
|
27
27
|
const fields = { model: model.modelId, prompt, size: model.size, quality: model.quality ?? 'high', n: model.n ?? 1, output_format: 'png' };
|
|
28
|
-
if (!
|
|
29
|
-
const [metadata, encoded] = reference.split(',');
|
|
30
|
-
const mime = metadata.slice(5, metadata.indexOf(';'));
|
|
31
|
-
const bytes = Uint8Array.from(atob(encoded), character => character.charCodeAt(0));
|
|
28
|
+
if (!references.length) return { operation: 'generations', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(fields) };
|
|
32
29
|
const body = new FormData();
|
|
33
30
|
for (const [key, value] of Object.entries(fields)) body.append(key, String(value));
|
|
34
|
-
|
|
31
|
+
for (const reference of references) {
|
|
32
|
+
const [metadata, encoded] = reference.split(',');
|
|
33
|
+
const mime = metadata.slice(5, metadata.indexOf(';'));
|
|
34
|
+
const bytes = Uint8Array.from(atob(encoded), character => character.charCodeAt(0));
|
|
35
|
+
body.append(references.length === 1 ? 'image' : 'image[]', new Blob([bytes], { type: mime }), `reference.${mime === 'image/jpeg' ? 'jpg' : mime.split('/')[1]}`);
|
|
36
|
+
}
|
|
35
37
|
return { operation: 'edits', headers: {}, body };
|
|
36
38
|
},
|
|
37
39
|
readResults: data => readImageResults(data, 'data')
|
|
@@ -10,10 +10,10 @@ export const kolors: ImageAdapter = {
|
|
|
10
10
|
if (!Number.isInteger(model.steps) || model.steps < 1 || model.steps > 100) throw new Error('Kolors steps must be between 1 and 100.');
|
|
11
11
|
if (!Number.isFinite(model.guidance) || model.guidance < 0 || model.guidance > 20) throw new Error('Kolors guidance must be between 0 and 20.');
|
|
12
12
|
},
|
|
13
|
-
buildRequest(model, prompt,
|
|
13
|
+
buildRequest(model, prompt, references = []) {
|
|
14
14
|
return {
|
|
15
15
|
operation: 'generations', headers: { 'Content-Type': 'application/json', 'X-Enable-Watermark': '0' },
|
|
16
|
-
body: JSON.stringify({ model: model.modelId, prompt, image_size: model.size, num_inference_steps: model.steps, guidance_scale: model.guidance, ...(
|
|
16
|
+
body: JSON.stringify({ model: model.modelId, prompt, image_size: model.size, num_inference_steps: model.steps, guidance_scale: model.guidance, ...(references.length ? { image: references[0] } : {}) })
|
|
17
17
|
};
|
|
18
18
|
},
|
|
19
19
|
readResults: data => readImageResults(data, 'images')
|
|
@@ -37,6 +37,8 @@ export function validateImageModel(model: ImageModelConfig, config: Pick<AiConfi
|
|
|
37
37
|
if (typeof date !== 'string' || (date && (!/^\d{4}-\d{2}-\d{2}$/.test(date) || !Number.isFinite(Date.parse(date)) || new Date(date).toISOString().slice(0, 10) !== date))) throw new Error('Image model release date must be a valid YYYY-MM-DD date, or empty when unknown.');
|
|
38
38
|
}
|
|
39
39
|
if (model.timeoutSeconds !== undefined && (!Number.isInteger(model.timeoutSeconds) || model.timeoutSeconds < 60 || model.timeoutSeconds > 600)) throw new Error('Image timeout must be between 60 and 600 seconds.');
|
|
40
|
+
if (model.maxReferenceImages !== undefined && (!Number.isSafeInteger(model.maxReferenceImages) || model.maxReferenceImages < 0)) throw new Error('Reference image limit must be a non-negative safe integer.');
|
|
41
|
+
if (model.adapter === 'kolors' && (model.maxReferenceImages ?? 1) > 1) throw new Error('Kolors supports at most one reference image.');
|
|
40
42
|
const adapter = getImageAdapter(model.adapter);
|
|
41
43
|
const capabilities = adapter.capabilities(model);
|
|
42
44
|
if (!capabilities.qualities && model.quality !== undefined) throw new Error('quality is not supported by this image model.');
|
|
@@ -60,13 +62,15 @@ export function availableImageModels(config: AiConfig): ImageModelConfig[] {
|
|
|
60
62
|
});
|
|
61
63
|
}
|
|
62
64
|
|
|
63
|
-
export async function generateImages(config: AiConfig, input: GenerateImageInput, signal: AbortSignal, reference?: string): Promise<Blob[]> {
|
|
65
|
+
export async function generateImages(config: AiConfig, input: GenerateImageInput, signal: AbortSignal, reference?: string | string[]): Promise<Blob[]> {
|
|
64
66
|
signal.throwIfAborted();
|
|
65
67
|
const model = availableImageModels(config).find(item => item.id === input.modelConfigId);
|
|
66
68
|
if (!model) throw new Error('Image model is disabled, unavailable or not found.');
|
|
67
69
|
signal = AbortSignal.any([signal, AbortSignal.timeout(imageTimeoutSeconds(model) * 1000)]);
|
|
68
70
|
if (typeof input.prompt !== 'string' || !input.prompt.trim() || input.prompt.length > 10000) throw new Error('Image prompt must contain 1 to 10000 characters.');
|
|
69
|
-
|
|
71
|
+
const references = typeof reference === 'string' ? [reference] : reference ?? [];
|
|
72
|
+
if (!Array.isArray(references) || references.some(image => typeof image !== 'string' || !/^data:image\/(png|jpeg|webp);base64,[A-Za-z0-9+/=]+$/.test(image))) throw new Error('Reference must be a local PNG, JPEG or WebP image.');
|
|
73
|
+
if (references.length > (model.maxReferenceImages ?? 1)) throw new Error(`This image model supports at most ${model.maxReferenceImages ?? 1} reference images.`);
|
|
70
74
|
const selected = { ...model, size: input.size ?? model.size, ratio: input.ratio ?? model.ratio, quality: input.quality ?? model.quality, n: input.n ?? model.n };
|
|
71
75
|
const capabilities = imageCapabilities(selected);
|
|
72
76
|
if (input.ratio !== undefined && !capabilities.ratios?.length) throw new Error('ratio is not supported by this image model.');
|
|
@@ -74,7 +78,7 @@ export async function generateImages(config: AiConfig, input: GenerateImageInput
|
|
|
74
78
|
validateImageModel(selected, config);
|
|
75
79
|
const endpoint = config.endpoints.find(item => item.id === model.endpointId)!;
|
|
76
80
|
const adapter = getImageAdapter(model.adapter);
|
|
77
|
-
const request = adapter.buildRequest(selected, input.prompt.trim(),
|
|
81
|
+
const request = adapter.buildRequest(selected, input.prompt.trim(), references);
|
|
78
82
|
const headers = { ...request.headers, ...(endpoint.apiKey ? { Authorization: `Bearer ${endpoint.apiKey}` } : {}) };
|
|
79
83
|
const response = await fetch(imageRequestUrl(endpoint.endpoint, model.requestUrl, request.operation), { method: 'POST', headers, body: request.body, signal });
|
|
80
84
|
if (!response.ok) throw new Error(`Image generation failed (HTTP ${response.status}).`);
|
|
@@ -16,8 +16,8 @@ export function getImageAdapter(adapter: ImageAdapterId): ImageAdapter {
|
|
|
16
16
|
return adapters[adapter];
|
|
17
17
|
}
|
|
18
18
|
|
|
19
|
-
export function imageCapabilities(model: Pick<ImageModelConfig, 'adapter' | 'modelId'>) {
|
|
20
|
-
return getImageAdapter(model.adapter).capabilities(model);
|
|
19
|
+
export function imageCapabilities(model: Pick<ImageModelConfig, 'adapter' | 'modelId' | 'maxReferenceImages'>) {
|
|
20
|
+
return { ...getImageAdapter(model.adapter).capabilities(model), maxReferenceImages: model.maxReferenceImages ?? 1 };
|
|
21
21
|
}
|
|
22
22
|
|
|
23
23
|
export const IMAGE_MODEL_PRESETS = [
|
|
@@ -42,6 +42,7 @@ export function createImageModel(presetId: ImagePresetId): ImageModelConfig {
|
|
|
42
42
|
version: preset.version, releasedAt: preset.releasedAt,
|
|
43
43
|
endpointId: '', requestUrl: '', enabled: false, size: preset.size, ratio: '1:1', steps: 20, guidance: 7.5,
|
|
44
44
|
...(preset.adapter === 'gpt-image' ? { quality: 'high' as const, n: 1 } : {}),
|
|
45
|
+
maxReferenceImages: preset.adapter === 'gpt-image' ? 5 : 1,
|
|
45
46
|
timeoutSeconds: preset.timeoutSeconds
|
|
46
47
|
};
|
|
47
48
|
}
|
|
@@ -20,10 +20,11 @@ export interface ImageModelConfig {
|
|
|
20
20
|
guidance: number;
|
|
21
21
|
quality?: ImageQuality;
|
|
22
22
|
n?: number;
|
|
23
|
+
maxReferenceImages?: number;
|
|
23
24
|
timeoutSeconds?: number;
|
|
24
25
|
}
|
|
25
26
|
|
|
26
|
-
export type ImagePromptConfig = Pick<ImageModelConfig, 'id' | 'label' | 'adapter' | 'modelId' | 'version' | 'releasedAt' | 'timeoutSeconds' | 'description' | 'size' | 'ratio' | 'quality' | 'n'>;
|
|
27
|
+
export type ImagePromptConfig = Pick<ImageModelConfig, 'id' | 'label' | 'adapter' | 'modelId' | 'version' | 'releasedAt' | 'timeoutSeconds' | 'description' | 'size' | 'ratio' | 'quality' | 'n' | 'maxReferenceImages'>;
|
|
27
28
|
|
|
28
29
|
export interface GenerateImageInput {
|
|
29
30
|
modelConfigId: string;
|
|
@@ -47,7 +48,7 @@ export interface ImageCapabilities {
|
|
|
47
48
|
export interface ImageAdapter {
|
|
48
49
|
capabilities: (model: Pick<ImageModelConfig, 'modelId'>) => ImageCapabilities;
|
|
49
50
|
validate: (model: ImageModelConfig) => void;
|
|
50
|
-
buildRequest: (model: ImageModelConfig, prompt: string,
|
|
51
|
+
buildRequest: (model: ImageModelConfig, prompt: string, references?: string[]) => {
|
|
51
52
|
operation: 'generations' | 'edits';
|
|
52
53
|
headers: Record<string, string>;
|
|
53
54
|
body: string | FormData;
|
package/src/imageToolHost.ts
CHANGED
|
@@ -17,6 +17,7 @@ export interface ImageToolResult {
|
|
|
17
17
|
export interface ImageToolExtension extends ImageToolSchemaExtension {
|
|
18
18
|
prepare: (args: Record<string, unknown>, signal: AbortSignal) => Promise<{
|
|
19
19
|
inputImage?: Blob;
|
|
20
|
+
inputImages?: Blob[];
|
|
20
21
|
complete: (result: ImageToolResult, signal: AbortSignal) => Promise<unknown>;
|
|
21
22
|
}>;
|
|
22
23
|
}
|
package/src/sessionTypes.ts
CHANGED
package/src/useAssistant.ts
CHANGED
|
@@ -47,7 +47,7 @@ export function useAssistant(runner: AssistantRunner, storage: AssistantStorage)
|
|
|
47
47
|
const pendingSave = useRef(Promise.resolve());
|
|
48
48
|
const mounted = useRef(false);
|
|
49
49
|
const generation = useRef(0);
|
|
50
|
-
const redo = useRef<{ sessionId: string; entryId: string;
|
|
50
|
+
const redo = useRef<{ sessionId: string; entryId: string; previousHistory: unknown[] } | null>(null);
|
|
51
51
|
const runnerRef = useRef(runner);
|
|
52
52
|
runnerRef.current = runner;
|
|
53
53
|
|
|
@@ -114,7 +114,7 @@ export function useAssistant(runner: AssistantRunner, storage: AssistantStorage)
|
|
|
114
114
|
};
|
|
115
115
|
}, [storage]);
|
|
116
116
|
|
|
117
|
-
const run = async (prompt: string, previous: ChatEntry[], attachments: ImageUpload[] = [], historyOverride?: unknown[]) => {
|
|
117
|
+
const run = async (prompt: string, previous: ChatEntry[], attachments: ImageUpload[] = [], historyOverride?: unknown[], reuseAttachments = false, previousResponse?: ChatEntry) => {
|
|
118
118
|
if (request.current || savingRef.current || loading || loadError) return;
|
|
119
119
|
if (attachments.length && !storage.images) throw new Error('Image storage is unavailable.');
|
|
120
120
|
const id = current.current.activeSessionId;
|
|
@@ -133,14 +133,18 @@ export function useAssistant(runner: AssistantRunner, storage: AssistantStorage)
|
|
|
133
133
|
const previousSnapshot = current.current;
|
|
134
134
|
const savedUploads: { sessionId: string; imageId: string }[] = [];
|
|
135
135
|
try {
|
|
136
|
-
for (const attachment of attachments) {
|
|
136
|
+
for (const attachment of reuseAttachments ? [] : attachments) {
|
|
137
137
|
abort.signal.throwIfAborted();
|
|
138
138
|
await storage.images!.saveImage(id, attachment.image.id, attachment.blob);
|
|
139
139
|
savedUploads.push({ sessionId: id, imageId: attachment.image.id });
|
|
140
140
|
}
|
|
141
141
|
abort.signal.throwIfAborted();
|
|
142
142
|
if (!active()) throw new Error('The active storage has changed.');
|
|
143
|
-
const
|
|
143
|
+
const galleryEntry = previousResponse && (previousResponse.images?.length || previousResponse.tools?.some(tool => tool.image))
|
|
144
|
+
? { content: '', images: previousResponse.images, tools: previousResponse.tools?.filter(tool => tool.image) }
|
|
145
|
+
: undefined;
|
|
146
|
+
const next = { ...current.current, sessions: current.current.sessions.map(session => session.id === id ? { ...session, title: previous.length ? session.title : prompt.slice(0, 30), updatedAt: Date.now(), entries: [...previous, user, response], responseItems: historyBefore,
|
|
147
|
+
...(galleryEntry ? { galleryEntries: [...(session.galleryEntries || []), galleryEntry] } : {}) } : session) };
|
|
144
148
|
if (saveTimer.current) { clearTimeout(saveTimer.current); saveTimer.current = null; }
|
|
145
149
|
current.current = next;
|
|
146
150
|
setSnapshot(next);
|
|
@@ -148,7 +152,7 @@ export function useAssistant(runner: AssistantRunner, storage: AssistantStorage)
|
|
|
148
152
|
published = true;
|
|
149
153
|
abort.signal.throwIfAborted();
|
|
150
154
|
if (!active()) throw new Error('The active storage has changed.');
|
|
151
|
-
redo.current = { sessionId: id, entryId: response.id,
|
|
155
|
+
redo.current = { sessionId: id, entryId: response.id, previousHistory: historyBefore };
|
|
152
156
|
await runnerRef.current([...previous, user], {
|
|
153
157
|
signal: abort.signal,
|
|
154
158
|
history: historyBefore,
|
|
@@ -215,19 +219,40 @@ export function useAssistant(runner: AssistantRunner, storage: AssistantStorage)
|
|
|
215
219
|
const session = snapshot.sessions.find(item => item.id === snapshot.activeSessionId) || snapshot.sessions[0];
|
|
216
220
|
const previousUserIndex = session.entries.map(entry => entry.role).lastIndexOf('user');
|
|
217
221
|
const hasToolEffects = previousUserIndex >= 0 && session.entries.slice(previousUserIndex + 1).some(entry => entry.tools?.length);
|
|
218
|
-
const canRedo =
|
|
222
|
+
const canRedo = previousUserIndex >= 0 && session.entries.at(-1)?.role === 'assistant';
|
|
219
223
|
const regenerate = async () => {
|
|
220
224
|
const checkpoint = redo.current;
|
|
221
225
|
const active = current.current.sessions.find(item => item.id === current.current.activeSessionId);
|
|
222
|
-
if (!active ||
|
|
226
|
+
if (!active || request.current || savingRef.current || loading || loadError || active.entries.at(-1)?.role !== 'assistant') return;
|
|
223
227
|
const index = active.entries.map(entry => entry.role).lastIndexOf('user');
|
|
224
|
-
if (index
|
|
228
|
+
if (index < 0) return;
|
|
229
|
+
const user = active.entries[index];
|
|
230
|
+
const previous = active.entries.slice(0, index);
|
|
231
|
+
const history = active.responseItems ?? [];
|
|
232
|
+
const previousAssistant = [...previous].reverse().find(entry => entry.role === 'assistant');
|
|
233
|
+
const marker = previousAssistant?.responseItemCount;
|
|
234
|
+
const historyBefore = checkpoint?.sessionId === active.id && checkpoint.entryId === active.entries.at(-1)?.id
|
|
235
|
+
? checkpoint.previousHistory
|
|
236
|
+
: history.slice(0, marker !== undefined && marker >= 0 && marker <= history.length ? marker : 0);
|
|
237
|
+
const snapshotBeforeLoad = current.current;
|
|
238
|
+
const targetGeneration = generation.current;
|
|
239
|
+
try {
|
|
240
|
+
const attachments = await Promise.all((user.images ?? []).map(async image => {
|
|
241
|
+
const blob = await storage.images?.loadImage(active.id, image.id);
|
|
242
|
+
if (!blob) throw new Error(`Image attachment not found: ${image.name}`);
|
|
243
|
+
return { image, blob };
|
|
244
|
+
}));
|
|
245
|
+
if (!mounted.current || generation.current !== targetGeneration || current.current !== snapshotBeforeLoad || request.current || savingRef.current) return;
|
|
246
|
+
await run(user.content, previous, attachments, historyBefore, true, active.entries.at(-1));
|
|
247
|
+
} catch (error) {
|
|
248
|
+
if (mounted.current && generation.current === targetGeneration && current.current === snapshotBeforeLoad) setExecutionError({ message: error instanceof Error ? error.message : String(error) });
|
|
249
|
+
}
|
|
225
250
|
};
|
|
226
251
|
|
|
227
252
|
return {
|
|
228
253
|
entries: session.entries, sessions: snapshot.sessions, activeSessionId: snapshot.activeSessionId,
|
|
229
254
|
busy, loading, saving, loadError, keyError: false, configError, executionError, confirmation: null,
|
|
230
|
-
canRegenerate: !busy &&
|
|
255
|
+
canRegenerate: !busy && !loading && !saving && !loadError && canRedo,
|
|
231
256
|
send: async (prompt, attachments = []) => {
|
|
232
257
|
const active = current.current.sessions.find(item => item.id === current.current.activeSessionId);
|
|
233
258
|
if (active) await run(prompt, active.entries, attachments);
|
|
@@ -266,7 +291,10 @@ export function useAssistant(runner: AssistantRunner, storage: AssistantStorage)
|
|
|
266
291
|
return { ...item, entries: item.entries.map(entry => ({ ...entry,
|
|
267
292
|
images: entry.images?.filter(image => !keys.has(`${item.id}:${image.id}`)),
|
|
268
293
|
tools: entry.tools?.map(tool => tool.image && keys.has(`${item.id}:${tool.image.id}`) ? { ...tool, image: undefined } : tool)
|
|
269
|
-
})),
|
|
294
|
+
})), galleryEntries: item.galleryEntries?.map(entry => ({ ...entry,
|
|
295
|
+
images: entry.images?.filter(image => !keys.has(`${item.id}:${image.id}`)),
|
|
296
|
+
tools: entry.tools?.filter(tool => !tool.image || !keys.has(`${item.id}:${tool.image.id}`))
|
|
297
|
+
})).filter(entry => entry.images?.length || entry.tools?.some(tool => tool.image)), responseItems: item.responseItems?.map(value => {
|
|
270
298
|
if (!value || typeof value !== 'object') return value;
|
|
271
299
|
const record = value as Record<string, unknown>;
|
|
272
300
|
return { ...record, ...(Array.isArray(record.content) ? { content: cleanParts(record.content) } : {}),
|