sang-ai-kit 0.1.1 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/README.md +9 -8
  2. package/generated/sang-ai-utilities.css +120 -1
  3. package/package.json +3 -1
  4. package/src/AiAssistant.tsx +20 -33
  5. package/src/AiSettings.tsx +6 -3
  6. package/src/ModelControls.tsx +3 -11
  7. package/src/SessionList.tsx +4 -1
  8. package/src/SkillSettings.tsx +4 -2
  9. package/src/TokenUsageBar.tsx +4 -1
  10. package/src/assistant.css +2125 -1712
  11. package/src/assistantTypes.ts +7 -2
  12. package/src/background-removal/BackgroundRemovalSettings.tsx +502 -0
  13. package/src/background-removal/ColorMattingEditor.tsx +649 -0
  14. package/src/background-removal/background-removal.css +294 -0
  15. package/src/background-removal/color-matting.css +838 -0
  16. package/src/background-removal/colorMatting.ts +117 -0
  17. package/src/background-removal/colorMatting.worker.ts +21 -0
  18. package/src/background-removal/index.ts +92 -0
  19. package/src/background-removal/inference.worker.ts +117 -0
  20. package/src/background-removal/models.ts +65 -0
  21. package/src/background-removal/processing.ts +61 -0
  22. package/src/background-removal/resources.ts +114 -0
  23. package/src/background-removal/status.ts +23 -0
  24. package/src/background-removal/storage.ts +56 -0
  25. package/src/background-removal/tool.ts +70 -0
  26. package/src/background-removal/types.ts +38 -0
  27. package/src/background-removal/useResources.ts +38 -0
  28. package/src/builtInToolDefinitions.ts +68 -0
  29. package/src/builtInTools.ts +66 -22
  30. package/src/bun-env.d.ts +25 -0
  31. package/src/conversationUtils.ts +71 -1
  32. package/src/endpointConnection.ts +2 -2
  33. package/src/features.ts +3 -1
  34. package/src/imageToolHost.ts +86 -0
  35. package/src/index.ts +15 -6
  36. package/src/modelConfig.ts +49 -7
  37. package/src/openai.ts +56 -30
  38. package/src/responses.ts +14 -0
  39. package/src/settingsTypes.ts +6 -8
  40. package/src/skills.ts +1 -1
  41. package/src/style.css +111 -67
  42. package/src/useAssistant.ts +5 -38
  43. package/src/useResolvedTheme.ts +2 -1
@@ -0,0 +1,56 @@
1
+ import { MODELS, type ModelId } from './models.ts';
2
+ import type { RemovalSettings, RuntimeFiles, StoredModel } from './types.ts';
3
+
4
+ let databaseName = 'sang-ai-kit-background-removal';
5
+ let storageUsed = false;
6
+ export const RUNTIME_VERSION = '1.23.2';
7
+ const RUNTIME_KEY = `runtime-${RUNTIME_VERSION}`;
8
+ const listeners = new Set<() => void>();
9
+
10
+ export function configureBackgroundRemovalStorage(options: { databaseName: string }): void {
11
+ if (!options.databaseName.trim()) throw new Error('Background removal database name is required');
12
+ if (storageUsed && options.databaseName !== databaseName) throw new Error('Background removal storage is already in use');
13
+ databaseName = options.databaseName;
14
+ }
15
+
16
+ export function subscribeResources(listener: () => void): () => void {
17
+ listeners.add(listener);
18
+ return () => { listeners.delete(listener); };
19
+ }
20
+
21
+ async function access<T>(mode: IDBTransactionMode, operation: (store: IDBObjectStore) => IDBRequest): Promise<T> {
22
+ storageUsed = true;
23
+ const db = await new Promise<IDBDatabase>((resolve, reject) => {
24
+ const request = indexedDB.open(databaseName, 1);
25
+ request.onupgradeneeded = () => request.result.createObjectStore('resources');
26
+ request.onsuccess = () => resolve(request.result);
27
+ request.onerror = () => reject(request.error);
28
+ });
29
+ return new Promise<T>((resolve, reject) => {
30
+ const transaction = db.transaction('resources', mode);
31
+ const request = operation(transaction.objectStore('resources'));
32
+ transaction.oncomplete = () => {
33
+ db.close();
34
+ if (mode === 'readwrite') listeners.forEach(listener => listener());
35
+ resolve(request.result as T);
36
+ };
37
+ transaction.onabort = transaction.onerror = () => {
38
+ db.close();
39
+ reject(transaction.error || request.error || new Error('Resource storage failed'));
40
+ };
41
+ });
42
+ }
43
+
44
+ export async function readSettings(): Promise<RemovalSettings> {
45
+ const saved = await access<RemovalSettings | undefined>('readonly', store => store.get('settings'));
46
+ return { modelId: MODELS.find(model => model.id === saved?.modelId)?.id ?? 'u2netp',
47
+ device: saved?.device === 'wasm' || saved?.device === 'webgpu' ? saved.device : 'auto' };
48
+ }
49
+ export const saveSettings = (settings: RemovalSettings) => access('readwrite', store => store.put(settings, 'settings'));
50
+ const modelKey = (id: ModelId) => id === 'birefnet' ? 'model-birefnet-lite-512' : `model-${id}`;
51
+ export const readModel = (id: ModelId) => access<StoredModel | undefined>('readonly', store => store.get(modelKey(id)));
52
+ export const saveModel = (id: ModelId, model: StoredModel) => access('readwrite', store => store.put(model, modelKey(id)));
53
+ export const deleteModel = (id: ModelId) => access('readwrite', store => store.delete(modelKey(id)));
54
+ export const readRuntime = () => access<RuntimeFiles | undefined>('readonly', store => store.get(RUNTIME_KEY));
55
+ export const saveRuntime = (files: RuntimeFiles) => access('readwrite', store => store.put(files, RUNTIME_KEY));
56
+ export const deleteRuntime = () => access('readwrite', store => store.delete(RUNTIME_KEY));
@@ -0,0 +1,70 @@
1
+ import type { ResponsesTool } from '../openai';
2
+ import type { SessionImage } from '../sessionTypes';
3
+ import { removeColorBackground } from './colorMatting';
4
+ import { removeBackground } from './index';
5
+ import type { ModelId } from './models';
6
+ import { deliverImageResult, extendImageToolDefinition, type ImageToolExtension } from '../imageToolHost';
7
+
8
+ export const backgroundRemovalDefinition = {
9
+ name: 'remove_background',
10
+ description: '移除图片背景,返回结果图片 ID,不修改原图。AI 模型及运行时未就绪时自动下载,推理在本地执行。',
11
+ parameters: { type: 'object', properties: {
12
+ imageId: { type: 'string', description: '待处理的原图 ID,来自附件或工具返回,不可编造。' },
13
+ modelId: { type: 'string', enum: ['color', 'u2netp', 'birefnet', 'modnet'], description: '可选,默认 u2netp 轻量模型。birefnet:Lite 512,适合复杂边缘;modnet:人像专用;color:颜色抠图,无需模型。' },
14
+ device: { type: 'string', enum: ['auto', 'webgpu', 'wasm'], description: '可选,默认 auto:优先 WebGPU,失败回退 CPU/WASM。仅 AI 模型使用。' },
15
+ colorOptions: { type: 'object', description: '可选,颜色抠图参数。', properties: {
16
+ color: { type: 'string', description: '目标色,#RRGGBB,默认 #ffffff。' },
17
+ tolerance: { type: 'number', minimum: 0, maximum: 100, description: '容差,默认 25。' },
18
+ feather: { type: 'number', minimum: 0, maximum: 10, description: '边缘羽化,默认 1.5。' },
19
+ contiguous: { type: 'boolean', description: '仅移除与边缘连通的颜色区域,默认 true。' }
20
+ } }
21
+ }, required: ['imageId'] }
22
+ };
23
+
24
+ export function createBackgroundRemovalTool(options: {
25
+ loadImage?: (id: string) => Promise<Blob | null>;
26
+ saveImage?: (blob: Blob, name: string) => Promise<SessionImage>;
27
+ onImage?: (image: SessionImage) => void;
28
+ extension?: ImageToolExtension;
29
+ }): ResponsesTool {
30
+ const definition = extendImageToolDefinition(backgroundRemovalDefinition, options.extension);
31
+ return {
32
+ ...definition,
33
+ execute: async (args, signal) => {
34
+ if (Object.keys(args).some(key => !Object.hasOwn(definition.parameters.properties, key))) throw new Error('Unsupported background-removal parameter.');
35
+ if (args.imageId !== undefined && (typeof args.imageId !== 'string' || !args.imageId)) throw new Error('Invalid image ID.');
36
+ if (!options.extension && (!options.loadImage || !options.saveImage)) throw new Error('Image storage is not configured.');
37
+ for (const key of options.extension?.required ?? []) if (args[key] === undefined) throw new Error(`${key} is required.`);
38
+ const modelId = args.modelId ?? 'u2netp';
39
+ if (typeof modelId !== 'string' || !['color', 'u2netp', 'birefnet', 'modnet'].includes(modelId)) throw new Error('Invalid model ID.');
40
+ const device = args.device ?? 'auto';
41
+ if (device !== 'auto' && device !== 'webgpu' && device !== 'wasm') throw new Error('Invalid device.');
42
+ if (args.colorOptions !== undefined && (!args.colorOptions || typeof args.colorOptions !== 'object' || Array.isArray(args.colorOptions))) throw new Error('colorOptions must be an object.');
43
+ const colorOptions = (args.colorOptions ?? {}) as Record<string, unknown>;
44
+ if (Object.keys(colorOptions).some(key => !Object.hasOwn(backgroundRemovalDefinition.parameters.properties.colorOptions.properties, key))) throw new Error('Unsupported colorOptions parameter.');
45
+ const color = colorOptions.color ?? '#ffffff';
46
+ if (typeof color !== 'string' || !/^#[0-9a-f]{6}$/i.test(color)) throw new Error('Invalid color.');
47
+ const tolerance = colorOptions.tolerance ?? 25, feather = colorOptions.feather ?? 1.5, contiguous = colorOptions.contiguous ?? true;
48
+ if (typeof tolerance !== 'number' || !Number.isFinite(tolerance) || tolerance < 0 || tolerance > 100 || typeof feather !== 'number' || !Number.isFinite(feather) || feather < 0 || feather > 10 || typeof contiguous !== 'boolean') throw new Error('Invalid matting options.');
49
+ signal.throwIfAborted();
50
+ const prepared = await options.extension?.prepare(args, signal);
51
+ if (prepared?.inputImage && args.imageId !== undefined) throw new Error('Only one input image is supported.');
52
+ if (args.imageId !== undefined && !options.loadImage) throw new Error('Image storage is not configured.');
53
+ const image = prepared?.inputImage ?? (typeof args.imageId === 'string' ? await options.loadImage!(args.imageId) : null);
54
+ if (!image) throw new Error('Image not found.');
55
+ let blob: Blob;
56
+ if (modelId === 'color') {
57
+ const targetColor: [number, number, number] = [1, 3, 5].map(offset => parseInt(color.slice(offset, offset + 2), 16)) as [number, number, number];
58
+ blob = await removeColorBackground(image, { targetColor, tolerance, feather, contiguous }, signal);
59
+ } else {
60
+ blob = (await removeBackground(image, { modelId: modelId as ModelId, device, signal })).blob;
61
+ }
62
+ signal.throwIfAborted();
63
+ if (prepared) return deliverImageResult(prepared, { images: [blob], model: modelId }, signal, options.saveImage, options.onImage);
64
+ const result = await options.saveImage!(blob, `removed-background-${Date.now()}.png`);
65
+ signal.throwIfAborted();
66
+ options.onImage?.(result);
67
+ return { image: { id: result.id, name: result.name, type: result.type, size: result.size }, modelId };
68
+ }
69
+ };
70
+ }
@@ -0,0 +1,38 @@
1
+ import type { ModelId } from './models.ts';
2
+
3
+ export type DeviceMode = 'auto' | 'webgpu' | 'wasm';
4
+ export type Provider = 'webgpu' | 'wasm';
5
+ export interface RemovalSettings { modelId: ModelId; device: DeviceMode }
6
+ export interface StoredModel { blob: Blob; name: string; importedAt: number; provider: Provider }
7
+ export interface RuntimeFiles { script: Blob; loader: Blob; wasm: Blob }
8
+ export interface Progress { stage: string; loaded?: number; total?: number }
9
+ export interface WorkerRequest {
10
+ modelId: ModelId;
11
+ model: Blob;
12
+ runtime: RuntimeFiles;
13
+ device: DeviceMode;
14
+ image: Blob;
15
+ }
16
+ export type WorkerResponse =
17
+ | { type: 'progress'; stage: string }
18
+ | { type: 'result'; blob: Blob; provider: Provider }
19
+ | { type: 'error'; message: string };
20
+
21
+ export interface OrtTensor {
22
+ data: Float32Array | Uint16Array;
23
+ dims: readonly number[];
24
+ type: string;
25
+ dispose(): void;
26
+ }
27
+ export interface OrtSession {
28
+ inputNames: readonly string[];
29
+ outputNames: readonly string[];
30
+ inputMetadata: readonly { type: string; shape: readonly (string | number)[] }[];
31
+ run(feeds: Record<string, OrtTensor>): Promise<Record<string, OrtTensor>>;
32
+ release(): Promise<void>;
33
+ }
34
+ export interface OrtRuntime {
35
+ env: { wasm: { numThreads: number; proxy: boolean; wasmPaths: { mjs: string; wasm: string } } };
36
+ Tensor: new (type: string, values: Float32Array | Uint16Array, dims: number[]) => OrtTensor;
37
+ InferenceSession: { create(bytes: Uint8Array, options: { executionProviders: Provider[]; freeDimensionOverrides?: Record<string, number> }): Promise<OrtSession> };
38
+ }
@@ -0,0 +1,38 @@
1
+ import { useEffect, useState } from 'react';
2
+ import { readModel, readRuntime, readSettings, subscribeResources } from './storage.ts';
3
+ import { MODELS, type ModelId } from './models.ts';
4
+ import type { RemovalSettings, StoredModel } from './types.ts';
5
+
6
+ interface ResourceState {
7
+ settings: RemovalSettings;
8
+ models: Partial<Record<ModelId, StoredModel>>;
9
+ runtime: boolean;
10
+ loading: boolean;
11
+ error: string;
12
+ }
13
+
14
+ export function useResources(): ResourceState {
15
+ const [state, setState] = useState<ResourceState>({
16
+ settings: { modelId: 'u2netp', device: 'auto' }, models: {}, runtime: false, loading: true, error: ''
17
+ });
18
+ useEffect(() => {
19
+ let mounted = true;
20
+ let revision = 0;
21
+ const refresh = async () => {
22
+ const current = ++revision;
23
+ try {
24
+ const [settings, runtime, entries] = await Promise.all([
25
+ readSettings(), readRuntime(), Promise.all(MODELS.map(async model => [model.id, await readModel(model.id)] as const))
26
+ ]);
27
+ if (mounted && current === revision) setState({ settings, models: Object.fromEntries(entries), runtime: !!runtime, loading: false, error: '' });
28
+ } catch (error) {
29
+ if (mounted && current === revision) setState(previous => ({ ...previous, loading: false,
30
+ error: error instanceof Error ? error.message : String(error) }));
31
+ }
32
+ };
33
+ const unsubscribe = subscribeResources(() => { void refresh(); });
34
+ void refresh();
35
+ return () => { mounted = false; unsubscribe(); };
36
+ }, []);
37
+ return state;
38
+ }
@@ -0,0 +1,68 @@
1
+ import type { ResponsesTool } from './openai';
2
+ import type { AiConfig, AiSkillConfig } from './settingsTypes';
3
+ import type { ImagePromptConfig } from './image-generation/types';
4
+ import { imageCapabilities, imageModelToolId } from './image-generation/index';
5
+ import { normalizeAiFeatures, type AiFeatures } from './features';
6
+ import { backgroundRemovalDefinition } from './background-removal/tool';
7
+ import { extendImageToolDefinition, type ImageToolExtensions } from './imageToolHost';
8
+ import { normalizeLocalFonts } from './fonts';
9
+
10
+ export type BuiltInToolDefinition = Pick<ResponsesTool, 'name' | 'description' | 'parameters'>;
11
+
12
+ export function imageToolDefinitions(models: ImagePromptConfig[] = []): BuiltInToolDefinition[] {
13
+ if (!models.length) return [];
14
+ const single = models.length === 1 ? models[0] : undefined;
15
+ const properties: Record<string, Record<string, unknown>> = {
16
+ modelConfigId: { type: 'string', description: '模型 ID,来自 list_image_models。', maxLength: 9, enum: models.map(imageModelToolId) },
17
+ prompt: { type: 'string', maxLength: 10000, minLength: 1, description: '生图提示词' },
18
+ size: { type: 'string', description: '输出尺寸,省略使用配置默认值。分辨率越高生图越慢。' },
19
+ ratio: { type: 'string', description: '宽高比(如16:9)。仅部分模型支持,当不传 size 时生效。' },
20
+ quality: { type: 'string', description: '生成质量。越高越慢,省略用默认值。' },
21
+ referenceImageId: { type: 'string', description: '可选,原图 ID,来自附件或工具返回,作为参考图。' },
22
+ n: { type: 'integer', minimum: 1, maximum: 10, description: '可选,生成数量,省略使用模型配置;须在模型支持范围内。' }
23
+ };
24
+ if (single) {
25
+ delete properties.modelConfigId;
26
+ const capabilities = imageCapabilities(single);
27
+ properties.size.description += ` 默认值:${single.size};${capabilities.exactSizeSupported ? '推荐尺寸' : '支持尺寸'}:${capabilities.sizes.join('、')}。`;
28
+ if (!capabilities.ratios) delete properties.ratio;
29
+ else properties.ratio = { type: 'string', description: `仅档位尺寸使用;省略为 ${single.ratio}。`, enum: capabilities.ratios };
30
+ if (!capabilities.qualities) delete properties.quality;
31
+ else properties.quality = { type: 'string', description: `生成质量。越高越慢,省略为 ${single.quality ?? 'high'}。`, enum: capabilities.qualities };
32
+ if (!capabilities.maxImages) delete properties.n;
33
+ else properties.n.maximum = capabilities.maxImages;
34
+ }
35
+ return [
36
+ ...(!single ? [{ name: 'list_image_models', description: '读取可用生图模型、能力及参数信息。', parameters: { type: 'object', properties: {} } }] : []),
37
+ { name: 'generate_image', description: '生成或修改图像,保存后返回图片 ID。需要插入文档或画布时,继续调用宿主提供的放置工具。',
38
+ parameters: { type: 'object', properties, required: single ? ['prompt'] : ['modelConfigId', 'prompt'] } }
39
+ ];
40
+ }
41
+
42
+ export function builtInToolDefinitions(options: {
43
+ features?: Partial<AiFeatures>;
44
+ skills?: AiSkillConfig[];
45
+ config?: Partial<AiConfig>;
46
+ imageModels?: ImagePromptConfig[];
47
+ imageStorage?: boolean;
48
+ imageExtensions?: ImageToolExtensions;
49
+ } = {}): BuiltInToolDefinition[] {
50
+ const features = normalizeAiFeatures(options.features);
51
+ const skills = features.skills ? options.skills ?? [] : [];
52
+ const fonts = features.fontManagement ? normalizeLocalFonts(options.config?.localFonts) : [];
53
+ const fontSources = [
54
+ ...(fonts.length ? ['用户提供的本地字体'] : []),
55
+ ...(features.onlineFonts ? ['内置 Google Fonts 精选目录'] : [])
56
+ ];
57
+ return [
58
+ ...(features.backgroundRemoval && (options.imageStorage || options.imageExtensions?.remove_background)
59
+ ? [extendImageToolDefinition(backgroundRemovalDefinition, options.imageExtensions?.remove_background)] : []),
60
+ ...(features.fontManagement && fontSources.length ? [{ name: 'list_fonts', description: `读取${fontSources.join('及')},使用 family 设置字体。`, parameters: { type: 'object', properties: {} } }] : []),
61
+ ...(skills.length ? [{ name: 'skill', description: '读取 SKILL。', parameters: { type: 'object', properties: {
62
+ name: { type: 'string', description: 'SKILL 精确名称,来自 available_skills 列表。', enum: skills.map(skill => skill.name) }
63
+ }, required: ['name'] } }] : []),
64
+ ...(features.imageGeneration && (options.imageStorage || options.imageExtensions?.generate_image)
65
+ ? imageToolDefinitions(options.imageModels).map(definition => definition.name === 'generate_image'
66
+ ? extendImageToolDefinition(definition, options.imageExtensions?.generate_image) : definition) : [])
67
+ ];
68
+ }
@@ -2,11 +2,17 @@ import type { ResponsesTool } from './openai';
2
2
  import type { AiConfig } from './settingsTypes';
3
3
  import type { SessionImage } from './sessionTypes';
4
4
  import type { ImageQuality } from './image-generation/types';
5
- import { availableImageModels, generateImages } from './image-generation/index';
5
+ import { availableImageModels, generateImages, imageCapabilities, imageModelToolId, imageTimeoutSeconds } from './image-generation/index';
6
6
  import { blobToDataUrl } from './imageAttachments';
7
7
  import { listFonts, normalizeLocalFonts } from './fonts';
8
8
  import { normalizeAiFeatures, type AiFeatures } from './features';
9
9
  import { normalizeSkills, readSkill, skillsPrompt } from './skills';
10
+ import { createBackgroundRemovalTool } from './background-removal/tool';
11
+ import { builtInToolDefinitions } from './builtInToolDefinitions';
12
+ import { deliverImageResult, ImageApplicationError, type ImageToolExtension, type ImageToolExtensions } from './imageToolHost';
13
+
14
+ export { builtInToolDefinitions, imageToolDefinitions } from './builtInToolDefinitions';
15
+ export type { ImageToolExtension, ImageToolExtensions, ImageToolResult, ImageToolSchemaExtension } from './imageToolHost';
10
16
 
11
17
  export interface BuiltInToolOptions {
12
18
  config: AiConfig;
@@ -14,6 +20,7 @@ export interface BuiltInToolOptions {
14
20
  loadImage?: (imageId: string) => Promise<Blob | null>;
15
21
  saveImage?: (blob: Blob, name: string) => Promise<SessionImage>;
16
22
  onImage?: (image: SessionImage) => void;
23
+ imageExtensions?: ImageToolExtensions<ImageToolExtension>;
17
24
  }
18
25
 
19
26
  export function createBuiltInTools(options: BuiltInToolOptions): { tools: ResponsesTool[]; instructions: string } {
@@ -22,53 +29,90 @@ export function createBuiltInTools(options: BuiltInToolOptions): { tools: Respon
22
29
  if (config.maxImageFailures !== undefined && (!Number.isSafeInteger(config.maxImageFailures) || config.maxImageFailures < 0)) throw new Error('Image failure limit must be a non-negative safe integer.');
23
30
  const skills = features.skills ? normalizeSkills(config.skills) : [];
24
31
  const fonts = features.fontManagement ? normalizeLocalFonts(config.localFonts) : [];
25
- const imageModels = features.imageGeneration && options.loadImage && options.saveImage ? availableImageModels(config) : [];
26
- const tools: ResponsesTool[] = [];
27
- if (features.fontManagement) tools.push({
28
- name: 'list_fonts', description: 'List available local and online fonts.', parameters: { type: 'object', properties: {} },
29
- execute: async args => { if (Object.keys(args).length) throw new Error('list_fonts does not accept parameters.'); return listFonts(fonts, { localFonts: true, onlineFonts: features.onlineFonts }); }
30
- });
31
- if (skills.length) tools.push({
32
- name: 'skill', description: 'Read the full text of an available skill.',
33
- parameters: { type: 'object', properties: { name: { type: 'string', enum: skills.map(skill => skill.name) } }, required: ['name'] },
34
- execute: async args => readSkill(skills, args)
35
- });
32
+ const imageModels = features.imageGeneration && ((options.loadImage && options.saveImage) || options.imageExtensions?.generate_image) ? availableImageModels(config) : [];
36
33
  let failures = 0;
37
- if (imageModels.length) tools.push({
38
- name: 'generate_image', description: `Generate images using a configured model: ${imageModels.map(item => `${item.label} (${item.id})`).join(', ')}.`,
39
- parameters: { type: 'object', properties: { modelConfigId: { type: 'string', enum: imageModels.map(item => item.id) }, prompt: { type: 'string' }, referenceImageId: { type: 'string' }, size: { type: 'string' }, ratio: { type: 'string' }, quality: { type: 'string' }, n: { type: 'integer' } }, required: ['modelConfigId', 'prompt'] },
34
+ const applicationFailures = new Set<string>();
35
+ const tools: ResponsesTool[] = builtInToolDefinitions({ features, config, skills, imageModels, imageStorage: Boolean(options.loadImage && options.saveImage), imageExtensions: options.imageExtensions }).map(definition => ({
36
+ ...definition,
40
37
  execute: async (args, signal) => {
38
+ signal.throwIfAborted();
39
+ if (applicationFailures.has(definition.name)) throw new Error('Image application failed in this turn. Do not retry processing without a new user request.');
40
+ if (definition.name === 'remove_background') {
41
+ try { return await createBackgroundRemovalTool({ ...options, extension: options.imageExtensions?.remove_background }).execute(args, signal); }
42
+ catch (error) {
43
+ if (error instanceof ImageApplicationError) applicationFailures.add(definition.name);
44
+ throw error;
45
+ }
46
+ }
47
+ if (definition.name === 'skill') return readSkill(skills, args);
48
+ if (definition.name === 'list_fonts' || definition.name === 'list_image_models') {
49
+ if (Object.keys(args).length) throw new Error(`${definition.name} does not accept parameters.`);
50
+ if (definition.name === 'list_fonts') return listFonts(fonts, { localFonts: true, onlineFonts: features.onlineFonts });
51
+ return { models: imageModels.map(model => {
52
+ const { sizePattern, sizeDescription, ...capabilities } = imageCapabilities(model);
53
+ return { id: imageModelToolId(model), label: model.label, description: model.description,
54
+ version: model.version?.trim() || null, releasedAt: model.releasedAt || null, timeoutSeconds: imageTimeoutSeconds(model),
55
+ defaultSize: model.size, ...capabilities,
56
+ ...(capabilities.ratios ? { defaultRatio: model.ratio } : {}),
57
+ ...(capabilities.qualities ? { defaultQuality: model.quality ?? 'high' } : {}) };
58
+ }) };
59
+ }
41
60
  const maxFailures = config.maxImageFailures ?? 3;
42
61
  if (maxFailures > 0 && failures >= maxFailures) throw new Error('Image generation failure limit reached in this turn.');
62
+ let prepared: Awaited<ReturnType<ImageToolExtension['prepare']>> | undefined;
63
+ let results: Blob[];
64
+ let model: string;
65
+ let prompt: string;
43
66
  try {
44
67
  signal.throwIfAborted();
45
- if (Object.keys(args).some(key => !['modelConfigId', 'prompt', 'referenceImageId', 'size', 'ratio', 'quality', 'n'].includes(key))) throw new Error('Unknown image generation parameter.');
46
- if (typeof args.modelConfigId !== 'string' || typeof args.prompt !== 'string') throw new Error('A model ID and prompt are required.');
68
+ if (Object.keys(args).some(key => !['modelConfigId', 'prompt', 'referenceImageId', 'size', 'ratio', 'quality', 'n'].includes(key) && !Object.hasOwn(options.imageExtensions?.generate_image?.properties ?? {}, key))) throw new Error('Unknown image generation parameter.');
69
+ for (const key of options.imageExtensions?.generate_image?.required ?? []) if (args[key] === undefined) throw new Error(`${key} is required.`);
70
+ const modelId = args.modelConfigId ?? (imageModels.length === 1 ? imageModelToolId(imageModels[0]) : undefined);
71
+ if (typeof modelId !== 'string' || typeof args.prompt !== 'string') throw new Error('A model ID and prompt are required.');
72
+ const matches = imageModels.filter(model => imageModelToolId(model) === modelId);
73
+ if (matches.length !== 1) throw new Error('Image model is disabled, unavailable or ambiguous. Read list_image_models when multiple models are enabled.');
47
74
  if (args.referenceImageId !== undefined && typeof args.referenceImageId !== 'string') throw new Error('Invalid reference image ID.');
48
75
  if (args.size !== undefined && typeof args.size !== 'string') throw new Error('Invalid image size.');
49
76
  if (args.ratio !== undefined && typeof args.ratio !== 'string') throw new Error('Invalid image ratio.');
50
77
  if (args.quality !== undefined && (typeof args.quality !== 'string' || !['low', 'medium', 'high', 'xhigh', 'max', 'auto'].includes(args.quality))) throw new Error('Invalid image quality.');
51
78
  if (args.n !== undefined && (typeof args.n !== 'number' || !Number.isInteger(args.n))) throw new Error('Invalid image count.');
79
+ prepared = await options.imageExtensions?.generate_image?.prepare(args, signal);
80
+ if (prepared?.inputImage && args.referenceImageId !== undefined) throw new Error('Only one reference image is supported.');
52
81
  let reference: string | undefined;
53
82
  if (typeof args.referenceImageId === 'string') {
83
+ if (!options.loadImage) throw new Error('Image storage is not configured.');
54
84
  const blob = await options.loadImage!(args.referenceImageId);
55
85
  if (!blob) throw new Error('Reference image not found.');
56
86
  reference = await blobToDataUrl(blob);
57
- }
58
- const results = await generateImages(config, { modelConfigId: args.modelConfigId, prompt: args.prompt,
87
+ } else if (prepared?.inputImage) reference = await blobToDataUrl(prepared.inputImage);
88
+ results = await generateImages(config, { modelConfigId: matches[0].id, prompt: args.prompt,
59
89
  ...(typeof args.size === 'string' ? { size: args.size } : {}), ...(typeof args.ratio === 'string' ? { ratio: args.ratio } : {}),
60
90
  ...(typeof args.quality === 'string' ? { quality: args.quality as ImageQuality } : {}), ...(typeof args.n === 'number' ? { n: args.n } : {})
61
91
  }, signal, reference);
62
- const saved: SessionImage[] = [];
92
+ model = matches[0].modelId;
93
+ prompt = args.prompt;
94
+ } catch (error) { if (!signal.aborted) failures++; throw error; }
95
+ if (prepared) {
96
+ try { return await deliverImageResult(prepared, { images: results, model, prompt }, signal, options.saveImage, options.onImage); }
97
+ catch (error) {
98
+ if (error instanceof ImageApplicationError) applicationFailures.add(definition.name);
99
+ throw error;
100
+ }
101
+ }
102
+ const saved: SessionImage[] = [];
103
+ try {
63
104
  for (const blob of results) {
105
+ signal.throwIfAborted();
64
106
  const image = await options.saveImage!(blob, `generated-${Date.now()}-${saved.length + 1}.${blob.type === 'image/jpeg' ? 'jpg' : blob.type === 'image/webp' ? 'webp' : 'png'}`);
65
107
  saved.push(image);
66
108
  options.onImage?.(image);
67
109
  }
68
- return { images: saved.map(image => ({ id: image.id, name: image.name, type: image.type, size: image.size })) };
110
+ signal.throwIfAborted();
111
+ return { images: saved.map(image => ({ id: image.id, name: image.name, type: image.type, size: image.size })),
112
+ model, prompt };
69
113
  } catch (error) { if (!signal.aborted) failures++; throw error; }
70
114
  }
71
- });
115
+ }));
72
116
  const imagePrompt = imageModels.length ? config.imageToolPrompt?.trim() : '';
73
117
  const skillPrompt = skillsPrompt(skills);
74
118
  const basePrompt = imagePrompt && !config.systemPrompt.includes(imagePrompt) ? `${config.systemPrompt}\n\n${imagePrompt}` : config.systemPrompt;
@@ -0,0 +1,25 @@
1
+ declare module 'bun:test' {
2
+ export function test(name: string, fn: () => unknown | Promise<unknown>): void;
3
+ export interface Matchers {
4
+ toBe(expected: unknown): void;
5
+ toEqual(expected: unknown): void;
6
+ toContain(expected: unknown): void;
7
+ toContainEqual(expected: unknown): void;
8
+ toMatch(expected: RegExp | string): void;
9
+ toMatchObject(expected: unknown): void;
10
+ toBeGreaterThan(expected: number): void;
11
+ toHaveLength(expected: number): void;
12
+ toBeTruthy(): void;
13
+ toBeNull(): void;
14
+ toBeDefined(): void;
15
+ toBeUndefined(): void;
16
+ toThrow(expected?: string | RegExp): void;
17
+ not: Matchers;
18
+ rejects: Matchers;
19
+ }
20
+ export function expect(actual: unknown): Matchers;
21
+ }
22
+
23
+ declare namespace fetch {
24
+ var preconnect: ((url: string | URL) => void) | undefined;
25
+ }
@@ -1,5 +1,75 @@
1
- import type { AssistantSession, RequestUsage, TokenUsage } from './assistantTypes';
1
+ import type { AssistantActivityEvent, AssistantSession, RequestUsage, TokenUsage } from './assistantTypes';
2
2
  import type { UsageSummary } from './TokenUsageBar';
3
+ import { blobToDataUrl } from './imageAttachments';
4
+ import { generateId } from './id';
5
+
6
+ function isRecord(value: unknown): value is Record<string, unknown> {
7
+ return value !== null && typeof value === 'object' && !Array.isArray(value);
8
+ }
9
+
10
+ export function appendReasoningActivity(activity: AssistantActivityEvent[], text: string, itemId?: string): AssistantActivityEvent[] {
11
+ const next = [...activity];
12
+ const index = itemId
13
+ ? next.findIndex(item => item.type === 'reasoning' && item.id === itemId)
14
+ : next.at(-1)?.type === 'reasoning' ? next.length - 1 : -1;
15
+ const existing = next[index];
16
+ if (existing?.type === 'reasoning') next[index] = { ...existing, text: existing.text + text };
17
+ else next.push({ id: itemId || generateId(), type: 'reasoning', text });
18
+ return next;
19
+ }
20
+
21
+ export function orderResponseActivity(items: unknown[], activity: AssistantActivityEvent[]): AssistantActivityEvent[] {
22
+ const result: AssistantActivityEvent[] = [];
23
+ const reasoning = activity.filter(item => item.type === 'reasoning');
24
+ const usedReasoning = new Set<string>();
25
+ const textParts = (value: unknown) => Array.isArray(value)
26
+ ? value.flatMap(part => isRecord(part) && typeof part.text === 'string' ? [part.text]
27
+ : isRecord(part) && typeof part.refusal === 'string' ? [part.refusal] : []).join('')
28
+ : typeof value === 'string' ? value : '';
29
+ for (const [index, value] of items.entries()) {
30
+ if (!isRecord(value)) continue;
31
+ if (value.type === 'reasoning') {
32
+ const id = typeof value.id === 'string' && value.id ? value.id : `reasoning-${index}`;
33
+ const streamed = reasoning.find(part => part.id === id && !usedReasoning.has(part.id))
34
+ || reasoning.find(part => !usedReasoning.has(part.id) && !items.some(candidate => isRecord(candidate) && candidate.id === part.id));
35
+ if (streamed) usedReasoning.add(streamed.id);
36
+ const text = streamed?.text || textParts(value.summary) || textParts(value.content);
37
+ if (text) result.push({ id, type: 'reasoning', text });
38
+ } else if (value.type === 'function_call' && typeof value.call_id === 'string' && value.call_id) {
39
+ result.push({ id: value.call_id, type: 'tool', toolId: value.call_id });
40
+ } else if (value.role === 'assistant' || value.type === 'message' && value.role !== 'user') {
41
+ const text = textParts(value.content);
42
+ if (text) result.push({ id: typeof value.id === 'string' && value.id ? value.id : `message-${index}`, type: 'text', text });
43
+ }
44
+ }
45
+ if (reasoning.length && !items.some(value => isRecord(value) && value.type === 'reasoning')) return activity;
46
+ if (!result.some(item => item.type === 'text')) result.push(...activity.filter(item => item.type === 'text'));
47
+ return result;
48
+ }
49
+
50
+ export async function prepareResponseInput<Item extends object>(
51
+ input: Item[],
52
+ supportsVision: boolean,
53
+ loadImage?: (imageId: string) => Promise<Blob | null>
54
+ ): Promise<Item[]> {
55
+ const copy = structuredClone(input);
56
+ const hydrate = async (parts: unknown[]) => Promise.all(parts.map(async part => {
57
+ if (!isRecord(part) || part.type !== 'input_image') return part;
58
+ if (!supportsVision) return { type: 'input_text', text: '[Image not provided: the current model does not support image input.]' };
59
+ if (typeof part.image_url !== 'string' || !part.image_url.startsWith('attachment://')) return part;
60
+ const blob = await loadImage?.(part.image_url.slice('attachment://'.length));
61
+ if (!blob) throw new Error('A referenced chat image is missing.');
62
+ if (!['image/png', 'image/jpeg', 'image/webp'].includes(blob.type)) throw new Error('A referenced image must be PNG, JPEG or WebP.');
63
+ return { ...part, image_url: await blobToDataUrl(blob) };
64
+ }));
65
+ for (const item of copy) {
66
+ if ('content' in item && Array.isArray(item.content)) Object.assign(item, { content: await hydrate(item.content) });
67
+ if ('type' in item && item.type === 'function_call_output' && 'output' in item && Array.isArray(item.output)) {
68
+ Object.assign(item, { output: await hydrate(item.output) });
69
+ }
70
+ }
71
+ return copy;
72
+ }
3
73
 
4
74
  export interface ResponseUsage {
5
75
  input_tokens: number;
@@ -1,7 +1,7 @@
1
1
  import OpenAI from 'openai';
2
2
  import type { AiEndpointConfig, AiModelConfig } from './settingsTypes';
3
3
  import { getSelectedReasoningEffort } from './ModelControls';
4
- import { reasoningRequestParams } from './openai';
4
+ import { openAiAuthOptions, reasoningRequestParams } from './responses';
5
5
 
6
6
  export async function testEndpointConnection(endpoint: AiEndpointConfig, model: AiModelConfig): Promise<string> {
7
7
  if (!model.modelId.trim()) throw new Error('模型 ID 不能为空。');
@@ -11,7 +11,7 @@ export async function testEndpointConnection(endpoint: AiEndpointConfig, model:
11
11
  }
12
12
  const client = new OpenAI({
13
13
  baseURL: url.toString().replace(/\/$/, ''),
14
- apiKey: endpoint.apiKey || 'local',
14
+ ...openAiAuthOptions(endpoint.apiKey),
15
15
  dangerouslyAllowBrowser: true,
16
16
  maxRetries: 0,
17
17
  timeout: 30_000
package/src/features.ts CHANGED
@@ -4,6 +4,7 @@ export interface AiFeatures {
4
4
  quickSnippets: boolean;
5
5
  fontManagement: boolean;
6
6
  onlineFonts: boolean;
7
+ backgroundRemoval: boolean;
7
8
  }
8
9
 
9
10
  export const DEFAULT_AI_FEATURES: Readonly<AiFeatures> = {
@@ -11,7 +12,8 @@ export const DEFAULT_AI_FEATURES: Readonly<AiFeatures> = {
11
12
  skills: true,
12
13
  quickSnippets: true,
13
14
  fontManagement: true,
14
- onlineFonts: true
15
+ onlineFonts: true,
16
+ backgroundRemoval: true
15
17
  };
16
18
 
17
19
  export function normalizeAiFeatures(value: unknown): AiFeatures {
@@ -0,0 +1,86 @@
1
+ import type { ResponsesTool } from './openai';
2
+ import type { SessionImage } from './sessionTypes';
3
+
4
+ export interface ImageToolSchemaExtension {
5
+ description: string;
6
+ properties: Record<string, Record<string, unknown>>;
7
+ required?: string[];
8
+ optional?: string[];
9
+ }
10
+
11
+ export interface ImageToolResult {
12
+ images: Blob[];
13
+ model: string;
14
+ prompt?: string;
15
+ }
16
+
17
+ export interface ImageToolExtension extends ImageToolSchemaExtension {
18
+ prepare: (args: Record<string, unknown>, signal: AbortSignal) => Promise<{
19
+ inputImage?: Blob;
20
+ complete: (result: ImageToolResult, signal: AbortSignal) => Promise<unknown>;
21
+ }>;
22
+ }
23
+
24
+ export type ImageToolExtensions<T = ImageToolSchemaExtension> = {
25
+ generate_image?: T;
26
+ remove_background?: T;
27
+ };
28
+
29
+ export class ImageApplicationError extends Error {
30
+ constructor(message: string, cause: unknown) {
31
+ super(message, { cause });
32
+ this.name = 'ImageApplicationError';
33
+ }
34
+ }
35
+
36
+ export function extendImageToolDefinition<T extends Pick<ResponsesTool, 'name' | 'description' | 'parameters'>>(definition: T, extension?: ImageToolSchemaExtension): T {
37
+ if (!extension) return definition;
38
+ const properties = definition.parameters.properties as Record<string, unknown>;
39
+ for (const key of Object.keys(extension.properties)) {
40
+ if (Object.hasOwn(properties, key)) throw new Error(`Image tool parameter ${key} conflicts with a built-in parameter.`);
41
+ }
42
+ const required = definition.parameters.required as string[] | undefined;
43
+ for (const key of extension.optional ?? []) {
44
+ if (!required?.includes(key)) throw new Error(`Image tool parameter ${key} is not required.`);
45
+ }
46
+ for (const key of extension.required ?? []) {
47
+ if (!Object.hasOwn(extension.properties, key)) throw new Error(`Unknown required host parameter ${key}.`);
48
+ }
49
+ return {
50
+ ...definition,
51
+ description: extension.description,
52
+ parameters: { ...definition.parameters, properties: { ...properties, ...extension.properties },
53
+ required: [...(required ?? []).filter(key => !extension.optional?.includes(key)), ...(extension.required ?? [])] }
54
+ };
55
+ }
56
+
57
+ export async function deliverImageResult(
58
+ prepared: Awaited<ReturnType<ImageToolExtension['prepare']>>,
59
+ result: ImageToolResult,
60
+ signal: AbortSignal,
61
+ saveImage?: (blob: Blob, name: string) => Promise<SessionImage>,
62
+ onImage?: (image: SessionImage) => void
63
+ ): Promise<unknown> {
64
+ try {
65
+ signal.throwIfAborted();
66
+ return await prepared.complete(result, signal);
67
+ } catch (error) {
68
+ if (signal.aborted) throw error;
69
+ const saved: SessionImage[] = [];
70
+ let recoveryError: unknown;
71
+ if (saveImage) {
72
+ try {
73
+ for (const [index, blob] of result.images.entries()) {
74
+ const extension = blob.type === 'image/jpeg' ? 'jpg' : blob.type === 'image/webp' ? 'webp' : 'png';
75
+ const image = await saveImage(blob, `recovery-${Date.now()}-${index + 1}.${extension}`);
76
+ saved.push(image);
77
+ onImage?.(image);
78
+ }
79
+ } catch (failure) { recoveryError = failure; }
80
+ }
81
+ const detail = error instanceof Error ? error.message : String(error);
82
+ const recovery = saved.length ? `Recovery image IDs: ${saved.map(image => image.id).join(', ')}.` : 'No recovery images were saved.';
83
+ const failure = recoveryError ? ` Recovery saving failed: ${recoveryError instanceof Error ? recoveryError.message : String(recoveryError)}.` : '';
84
+ throw new ImageApplicationError(`Image processing succeeded but host application failed: ${detail}. ${recovery}${failure} Do not automatically repeat image processing.`, error);
85
+ }
86
+ }