sang-ai-kit 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -8
- package/generated/sang-ai-utilities.css +120 -1
- package/package.json +3 -1
- package/src/AiAssistant.tsx +20 -33
- package/src/AiSettings.tsx +6 -3
- package/src/ModelControls.tsx +3 -11
- package/src/SessionList.tsx +4 -1
- package/src/SkillSettings.tsx +4 -2
- package/src/TokenUsageBar.tsx +4 -1
- package/src/assistant.css +2125 -1712
- package/src/assistantTypes.ts +7 -2
- package/src/background-removal/BackgroundRemovalSettings.tsx +502 -0
- package/src/background-removal/ColorMattingEditor.tsx +649 -0
- package/src/background-removal/background-removal.css +294 -0
- package/src/background-removal/color-matting.css +838 -0
- package/src/background-removal/colorMatting.ts +117 -0
- package/src/background-removal/colorMatting.worker.ts +21 -0
- package/src/background-removal/index.ts +92 -0
- package/src/background-removal/inference.worker.ts +117 -0
- package/src/background-removal/models.ts +65 -0
- package/src/background-removal/processing.ts +61 -0
- package/src/background-removal/resources.ts +114 -0
- package/src/background-removal/status.ts +23 -0
- package/src/background-removal/storage.ts +56 -0
- package/src/background-removal/tool.ts +70 -0
- package/src/background-removal/types.ts +38 -0
- package/src/background-removal/useResources.ts +38 -0
- package/src/builtInToolDefinitions.ts +68 -0
- package/src/builtInTools.ts +66 -22
- package/src/bun-env.d.ts +25 -0
- package/src/conversationUtils.ts +71 -1
- package/src/endpointConnection.ts +2 -2
- package/src/features.ts +3 -1
- package/src/imageToolHost.ts +86 -0
- package/src/index.ts +15 -6
- package/src/modelConfig.ts +49 -7
- package/src/openai.ts +56 -30
- package/src/responses.ts +14 -0
- package/src/settingsTypes.ts +6 -8
- package/src/skills.ts +1 -1
- package/src/style.css +111 -67
- package/src/useAssistant.ts +5 -38
- package/src/useResolvedTheme.ts +2 -1
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
export interface MattingOptions {
|
|
2
|
+
targetColor: [number, number, number];
|
|
3
|
+
tolerance: number;
|
|
4
|
+
contiguous: boolean;
|
|
5
|
+
feather: number;
|
|
6
|
+
startX?: number;
|
|
7
|
+
startY?: number;
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
export interface BrushStroke {
|
|
11
|
+
mode: 'erase' | 'restore';
|
|
12
|
+
radius: number;
|
|
13
|
+
points: { x: number; y: number }[];
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export function applyBrushStrokes(result: ImageData, original: ImageData, strokes: BrushStroke[]): void {
|
|
17
|
+
if (result.width !== original.width || result.height !== original.height) throw new Error('Image dimensions do not match');
|
|
18
|
+
const { width, height, data } = result;
|
|
19
|
+
for (const stroke of strokes) {
|
|
20
|
+
const radius = stroke.radius;
|
|
21
|
+
if (!Number.isFinite(radius) || radius <= 0) continue;
|
|
22
|
+
const paint = (centerX: number, centerY: number) => {
|
|
23
|
+
const left = Math.max(0, Math.ceil(centerX - radius));
|
|
24
|
+
const right = Math.min(width - 1, Math.floor(centerX + radius));
|
|
25
|
+
const top = Math.max(0, Math.ceil(centerY - radius));
|
|
26
|
+
const bottom = Math.min(height - 1, Math.floor(centerY + radius));
|
|
27
|
+
for (let y = top; y <= bottom; y++) for (let x = left; x <= right; x++) {
|
|
28
|
+
if ((x - centerX) ** 2 + (y - centerY) ** 2 > radius ** 2) continue;
|
|
29
|
+
const offset = (y * width + x) * 4;
|
|
30
|
+
if (stroke.mode === 'erase') data[offset + 3] = 0;
|
|
31
|
+
else data.set(original.data.subarray(offset, offset + 4), offset);
|
|
32
|
+
}
|
|
33
|
+
};
|
|
34
|
+
stroke.points.forEach((point, index) => {
|
|
35
|
+
if (!Number.isFinite(point.x) || !Number.isFinite(point.y)) return;
|
|
36
|
+
const previous = stroke.points[index - 1];
|
|
37
|
+
if (!previous || !Number.isFinite(previous.x) || !Number.isFinite(previous.y)) { paint(point.x, point.y); return; }
|
|
38
|
+
const steps = Math.max(1, Math.ceil(Math.hypot(point.x - previous.x, point.y - previous.y) / Math.max(0.5, radius / 2)));
|
|
39
|
+
for (let step = 1; step <= steps; step++) {
|
|
40
|
+
const fraction = step / steps;
|
|
41
|
+
paint(previous.x + (point.x - previous.x) * fraction, previous.y + (point.y - previous.y) * fraction);
|
|
42
|
+
}
|
|
43
|
+
});
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export function applyColorMatting(imageData: ImageData, options: MattingOptions): ImageData {
|
|
48
|
+
const { width, height } = imageData;
|
|
49
|
+
const data = new Uint8ClampedArray(imageData.data);
|
|
50
|
+
const mask = new Uint8Array(width * height);
|
|
51
|
+
const [red, green, blue] = options.targetColor;
|
|
52
|
+
const maxDistance = options.tolerance / 100 * 441.67;
|
|
53
|
+
const matches = (index: number) => {
|
|
54
|
+
const offset = index * 4;
|
|
55
|
+
return Math.hypot(data[offset] - red, data[offset + 1] - green, data[offset + 2] - blue) <= maxDistance;
|
|
56
|
+
};
|
|
57
|
+
if (!options.contiguous) {
|
|
58
|
+
for (let index = 0; index < mask.length; index++) if (matches(index)) mask[index] = 1;
|
|
59
|
+
} else {
|
|
60
|
+
const visited = new Uint8Array(mask.length);
|
|
61
|
+
const queue: number[] = [];
|
|
62
|
+
const enqueue = (x: number, y: number) => {
|
|
63
|
+
if (x < 0 || x >= width || y < 0 || y >= height) return;
|
|
64
|
+
const index = y * width + x;
|
|
65
|
+
if (visited[index]) return;
|
|
66
|
+
visited[index] = 1;
|
|
67
|
+
if (data[index * 4 + 3] < 25 || matches(index)) { mask[index] = 1; queue.push(index); }
|
|
68
|
+
};
|
|
69
|
+
if (options.startX !== undefined && options.startY !== undefined) enqueue(options.startX, options.startY);
|
|
70
|
+
else {
|
|
71
|
+
for (let x = 0; x < width; x++) { enqueue(x, 0); enqueue(x, height - 1); }
|
|
72
|
+
for (let y = 0; y < height; y++) { enqueue(0, y); enqueue(width - 1, y); }
|
|
73
|
+
}
|
|
74
|
+
while (queue.length) {
|
|
75
|
+
const index = queue.pop()!;
|
|
76
|
+
const x = index % width;
|
|
77
|
+
const y = Math.floor(index / width);
|
|
78
|
+
enqueue(x - 1, y); enqueue(x + 1, y); enqueue(x, y - 1); enqueue(x, y + 1);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
for (let y = 0; y < height; y++) for (let x = 0; x < width; x++) {
|
|
82
|
+
const index = y * width + x;
|
|
83
|
+
if (mask[index]) { data[index * 4 + 3] = 0; continue; }
|
|
84
|
+
if (options.feather <= 0) continue;
|
|
85
|
+
let distance = Infinity;
|
|
86
|
+
const radius = Math.ceil(options.feather);
|
|
87
|
+
for (let dy = -radius; dy <= radius; dy++) for (let dx = -radius; dx <= radius; dx++) {
|
|
88
|
+
const nx = x + dx, ny = y + dy;
|
|
89
|
+
if (nx >= 0 && nx < width && ny >= 0 && ny < height && mask[ny * width + nx]) distance = Math.min(distance, Math.hypot(dx, dy));
|
|
90
|
+
}
|
|
91
|
+
if (distance <= options.feather) data[index * 4 + 3] = Math.round(data[index * 4 + 3] * distance / options.feather);
|
|
92
|
+
}
|
|
93
|
+
return new ImageData(data, width, height);
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
export async function removeColorBackground(image: Blob, options: MattingOptions, signal: AbortSignal,
|
|
97
|
+
adjustments?: { colorMatting?: boolean; strokes?: BrushStroke[] }): Promise<Blob> {
|
|
98
|
+
signal.throwIfAborted();
|
|
99
|
+
return new Promise((resolve, reject) => {
|
|
100
|
+
const worker = new Worker(new URL('./colorMatting.worker.ts', import.meta.url), { type: 'module' });
|
|
101
|
+
const finish = (error?: unknown, blob?: Blob) => {
|
|
102
|
+
signal.removeEventListener('abort', abort);
|
|
103
|
+
worker.terminate();
|
|
104
|
+
if (error) reject(error);
|
|
105
|
+
else if (blob) resolve(blob);
|
|
106
|
+
};
|
|
107
|
+
const abort = () => finish(signal.reason || new DOMException('Cancelled', 'AbortError'));
|
|
108
|
+
signal.addEventListener('abort', abort, { once: true });
|
|
109
|
+
worker.onerror = event => { event.preventDefault(); finish(new Error(event.message || 'Color matting worker failed')); };
|
|
110
|
+
worker.onmessage = (event: MessageEvent<{ blob?: Blob; error?: string }>) => {
|
|
111
|
+
if (event.data.error) finish(new Error(event.data.error));
|
|
112
|
+
else if (event.data.blob) finish(undefined, event.data.blob);
|
|
113
|
+
else finish(new Error('Color matting worker returned no image'));
|
|
114
|
+
};
|
|
115
|
+
try { worker.postMessage({ image, options, adjustments }); } catch (error) { finish(error); }
|
|
116
|
+
});
|
|
117
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { applyBrushStrokes, applyColorMatting, type BrushStroke, type MattingOptions } from './colorMatting';
|
|
2
|
+
|
|
3
|
+
self.onmessage = async (event: MessageEvent<{ image: Blob; options: MattingOptions; adjustments?: { colorMatting?: boolean; strokes?: BrushStroke[] } }>) => {
|
|
4
|
+
try {
|
|
5
|
+
const bitmap = await createImageBitmap(event.data.image);
|
|
6
|
+
try {
|
|
7
|
+
if (!bitmap.width || !bitmap.height || bitmap.width * bitmap.height > 40_000_000) throw new Error('Image exceeds 40 megapixels');
|
|
8
|
+
const canvas = new OffscreenCanvas(bitmap.width, bitmap.height);
|
|
9
|
+
const context = canvas.getContext('2d', { willReadFrequently: true });
|
|
10
|
+
if (!context) throw new Error('Canvas is unavailable');
|
|
11
|
+
context.drawImage(bitmap, 0, 0);
|
|
12
|
+
const original = context.getImageData(0, 0, bitmap.width, bitmap.height);
|
|
13
|
+
const result = event.data.adjustments?.colorMatting === false
|
|
14
|
+
? context.getImageData(0, 0, bitmap.width, bitmap.height)
|
|
15
|
+
: applyColorMatting(original, event.data.options);
|
|
16
|
+
applyBrushStrokes(result, original, event.data.adjustments?.strokes ?? []);
|
|
17
|
+
context.putImageData(result, 0, 0);
|
|
18
|
+
self.postMessage({ blob: await canvas.convertToBlob({ type: 'image/png' }) });
|
|
19
|
+
} finally { bitmap.close(); }
|
|
20
|
+
} catch (error) { self.postMessage({ error: error instanceof Error ? error.message : String(error) }); }
|
|
21
|
+
};
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
import { downloadModel, ensureRuntime, loadResources } from './resources.ts';
|
|
2
|
+
import { readModel, readSettings, saveModel } from './storage.ts';
|
|
3
|
+
import { validateModel, type ModelId } from './models.ts';
|
|
4
|
+
import type { DeviceMode, Progress, Provider, RuntimeFiles, WorkerRequest, WorkerResponse } from './types.ts';
|
|
5
|
+
|
|
6
|
+
interface RunOptions {
|
|
7
|
+
signal: AbortSignal;
|
|
8
|
+
onProgress?: (progress: Progress) => void;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
interface RemovalOptions extends RunOptions {
|
|
12
|
+
modelId?: ModelId;
|
|
13
|
+
device?: DeviceMode;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export function runWorker(request: WorkerRequest, options: RunOptions): Promise<{ blob: Blob; provider: Provider }> {
|
|
17
|
+
options.signal.throwIfAborted();
|
|
18
|
+
return new Promise((resolve, reject) => {
|
|
19
|
+
const worker = new Worker(new URL('./inference.worker.ts', import.meta.url), { type: 'module' });
|
|
20
|
+
const finish = (error?: unknown, result?: { blob: Blob; provider: Provider }) => {
|
|
21
|
+
clearTimeout(timer);
|
|
22
|
+
options.signal.removeEventListener('abort', abort);
|
|
23
|
+
worker.terminate();
|
|
24
|
+
if (error) reject(error);
|
|
25
|
+
else if (result) resolve(result);
|
|
26
|
+
};
|
|
27
|
+
const abort = () => finish(new DOMException('Cancelled', 'AbortError'));
|
|
28
|
+
const timer = setTimeout(() => finish(new Error('Inference timed out (10 minutes)')), 600_000);
|
|
29
|
+
options.signal.addEventListener('abort', abort, { once: true });
|
|
30
|
+
worker.onerror = event => { event.preventDefault(); finish(new Error(event.message || 'Inference worker failed')); };
|
|
31
|
+
worker.onmessage = (event: MessageEvent<WorkerResponse>) => {
|
|
32
|
+
const message = event.data;
|
|
33
|
+
if (message.type === 'progress') options.onProgress?.({ stage: message.stage });
|
|
34
|
+
else if (message.type === 'error') finish(new Error(message.message));
|
|
35
|
+
else finish(undefined, message);
|
|
36
|
+
};
|
|
37
|
+
try { worker.postMessage(request); } catch (error) { finish(error); }
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export async function removeBackground(image: Blob, options: RemovalOptions): Promise<{ blob: Blob; provider: Provider; modelId: ModelId }> {
|
|
42
|
+
options.signal.throwIfAborted();
|
|
43
|
+
const defaults = options.modelId && options.device ? { modelId: options.modelId, device: options.device } : await readSettings();
|
|
44
|
+
const settings = { modelId: options.modelId ?? defaults.modelId, device: options.device ?? defaults.device };
|
|
45
|
+
await ensureRuntime(options.signal, options.onProgress);
|
|
46
|
+
options.signal.throwIfAborted();
|
|
47
|
+
if (!await readModel(settings.modelId)) {
|
|
48
|
+
try {
|
|
49
|
+
options.onProgress?.({ stage: 'download' });
|
|
50
|
+
const blob = await downloadModel(settings.modelId,
|
|
51
|
+
AbortSignal.any([options.signal, AbortSignal.timeout(600_000)]), options.onProgress);
|
|
52
|
+
options.onProgress?.({ stage: 'verify' });
|
|
53
|
+
await installModel(settings.modelId, blob, `${settings.modelId}.onnx`, settings.device, options);
|
|
54
|
+
} catch (error) {
|
|
55
|
+
options.signal.throwIfAborted();
|
|
56
|
+
throw new Error(`抠图模型自动下载或验证失败,请重试:${error instanceof Error ? error.message : String(error)}`, { cause: error });
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
const resources = await loadResources(settings.modelId, options.signal);
|
|
60
|
+
const result = await runWorker({ ...resources, ...settings, image }, options);
|
|
61
|
+
return { ...result, modelId: settings.modelId };
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
export async function verifyModel(modelId: ModelId, model: Blob, device: DeviceMode, options: RunOptions,
|
|
65
|
+
runtimeOverride?: RuntimeFiles): Promise<Provider> {
|
|
66
|
+
await validateModel(modelId, model);
|
|
67
|
+
const runtime = runtimeOverride || await ensureRuntime(options.signal, options.onProgress);
|
|
68
|
+
const canvas = document.createElement('canvas');
|
|
69
|
+
canvas.width = canvas.height = 32;
|
|
70
|
+
const context = canvas.getContext('2d');
|
|
71
|
+
if (!context) throw new Error('Canvas is unavailable');
|
|
72
|
+
context.fillStyle = '#808080';
|
|
73
|
+
context.fillRect(0, 0, 32, 32);
|
|
74
|
+
const image = await new Promise<Blob>((resolve, reject) => canvas.toBlob(blob =>
|
|
75
|
+
blob ? resolve(blob) : reject(new Error('Image encoding failed')), 'image/png'));
|
|
76
|
+
return (await runWorker({ modelId, model, runtime, device, image }, options)).provider;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
export async function installModel(modelId: ModelId, blob: Blob, name: string, device: DeviceMode, options: RunOptions): Promise<void> {
|
|
80
|
+
const provider = await verifyModel(modelId, blob, device, options);
|
|
81
|
+
options.signal.throwIfAborted();
|
|
82
|
+
await saveModel(modelId, { blob, name, importedAt: Date.now(), provider });
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
export function blobDataUrl(blob: Blob): Promise<string> {
|
|
86
|
+
return new Promise((resolve, reject) => {
|
|
87
|
+
const reader = new FileReader();
|
|
88
|
+
reader.onload = () => resolve(String(reader.result));
|
|
89
|
+
reader.onerror = () => reject(reader.error);
|
|
90
|
+
reader.readAsDataURL(blob);
|
|
91
|
+
});
|
|
92
|
+
}
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
import { getModel } from './models.ts';
|
|
2
|
+
import { applyAlpha, createMask, preprocess } from './processing.ts';
|
|
3
|
+
import type { OrtRuntime, OrtSession, OrtTensor, Provider, WorkerRequest, WorkerResponse } from './types.ts';
|
|
4
|
+
|
|
5
|
+
const post = (message: WorkerResponse) => self.postMessage(message);
|
|
6
|
+
|
|
7
|
+
function float16(values: Float32Array): Uint16Array {
|
|
8
|
+
const bits = new Uint32Array(values.buffer, values.byteOffset, values.length);
|
|
9
|
+
return Uint16Array.from(bits, value => {
|
|
10
|
+
const sign = (value >>> 16) & 0x8000;
|
|
11
|
+
const exponent = ((value >>> 23) & 255) - 127 + 15;
|
|
12
|
+
const mantissa = value & 0x7fffff;
|
|
13
|
+
if (exponent <= 0) return sign | ((mantissa | 0x800000) >>> (14 - exponent));
|
|
14
|
+
if (exponent >= 31) return sign | 0x7c00;
|
|
15
|
+
return sign | (exponent << 10) | ((mantissa + 0x1000) >>> 13);
|
|
16
|
+
});
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
async function run(request: WorkerRequest): Promise<void> {
|
|
20
|
+
const urls: string[] = [];
|
|
21
|
+
let bitmap: ImageBitmap | undefined;
|
|
22
|
+
let session: OrtSession | undefined;
|
|
23
|
+
let input: OrtTensor | undefined;
|
|
24
|
+
let outputs: Record<string, OrtTensor> | undefined;
|
|
25
|
+
const objectUrl = (blob: Blob, type: string) => {
|
|
26
|
+
const url = URL.createObjectURL(blob.slice(0, blob.size, type));
|
|
27
|
+
urls.push(url);
|
|
28
|
+
return url;
|
|
29
|
+
};
|
|
30
|
+
try {
|
|
31
|
+
post({ type: 'progress', stage: 'runtime' });
|
|
32
|
+
const script = objectUrl(request.runtime.script, 'text/javascript');
|
|
33
|
+
const ort = await import(/* @vite-ignore */ script) as OrtRuntime;
|
|
34
|
+
ort.env.wasm.numThreads = 1;
|
|
35
|
+
ort.env.wasm.proxy = false;
|
|
36
|
+
ort.env.wasm.wasmPaths = {
|
|
37
|
+
mjs: objectUrl(request.runtime.loader, 'text/javascript'),
|
|
38
|
+
wasm: objectUrl(request.runtime.wasm, 'application/wasm')
|
|
39
|
+
};
|
|
40
|
+
const model = getModel(request.modelId);
|
|
41
|
+
const bytes = new Uint8Array(await request.model.arrayBuffer());
|
|
42
|
+
post({ type: 'progress', stage: 'model' });
|
|
43
|
+
const providers: Provider[] = model.forceCpu ? ['wasm'] : request.device === 'auto'
|
|
44
|
+
? ('gpu' in navigator ? ['webgpu', 'wasm'] : ['wasm']) : [request.device];
|
|
45
|
+
let lastError: unknown;
|
|
46
|
+
let result: Blob | undefined;
|
|
47
|
+
let provider: Provider = 'wasm';
|
|
48
|
+
bitmap = await createImageBitmap(request.image);
|
|
49
|
+
if (!bitmap.width || !bitmap.height || bitmap.width * bitmap.height > 40_000_000) {
|
|
50
|
+
throw new Error('Image exceeds 40 megapixels');
|
|
51
|
+
}
|
|
52
|
+
const resized = new OffscreenCanvas(model.inputSize, model.inputSize);
|
|
53
|
+
const resizedContext = resized.getContext('2d');
|
|
54
|
+
if (!resizedContext) throw new Error('Canvas is unavailable');
|
|
55
|
+
resizedContext.drawImage(bitmap, 0, 0, model.inputSize, model.inputSize);
|
|
56
|
+
const values = preprocess(resizedContext.getImageData(0, 0, model.inputSize, model.inputSize).data, model.id);
|
|
57
|
+
for (const candidate of providers) {
|
|
58
|
+
try {
|
|
59
|
+
session = await ort.InferenceSession.create(bytes, { executionProviders: [candidate],
|
|
60
|
+
...(model.id === 'modnet' ? { freeDimensionOverrides: { batch_size: 1, num_channels: 3, height: model.inputSize, width: model.inputSize } } : {}) });
|
|
61
|
+
const metadata = session.inputMetadata[0];
|
|
62
|
+
if (session.inputNames.length !== 1 || !metadata || !['float32', 'float16'].includes(metadata.type)) {
|
|
63
|
+
throw new Error('Unsupported model input; expected a single RGB float tensor');
|
|
64
|
+
}
|
|
65
|
+
const expected = [1, 3, model.inputSize, model.inputSize];
|
|
66
|
+
if (metadata.shape.length !== 4 || metadata.shape.some((dimension, index) =>
|
|
67
|
+
typeof dimension === 'number' && dimension > 0 && dimension !== expected[index])) {
|
|
68
|
+
throw new Error(`Model input must be 1x3x${model.inputSize}x${model.inputSize}`);
|
|
69
|
+
}
|
|
70
|
+
input = new ort.Tensor(metadata.type, metadata.type === 'float16' ? float16(values) : values, expected);
|
|
71
|
+
post({ type: 'progress', stage: candidate });
|
|
72
|
+
outputs = await session.run({ [session.inputNames[0]]: input });
|
|
73
|
+
const output = outputs[session.outputNames[0]];
|
|
74
|
+
const height = output?.dims.at(-2) || 0;
|
|
75
|
+
const width = output?.dims.at(-1) || 0;
|
|
76
|
+
if (!output || !width || !height || output.data.length !== width * height ||
|
|
77
|
+
!['float32', 'float16'].includes(output.type)) throw new Error('Unsupported model mask output');
|
|
78
|
+
const mask = new OffscreenCanvas(width, height);
|
|
79
|
+
const maskContext = mask.getContext('2d');
|
|
80
|
+
if (!maskContext) throw new Error('Canvas is unavailable');
|
|
81
|
+
maskContext.putImageData(new ImageData(createMask(output.data, model.id), width, height), 0, 0);
|
|
82
|
+
const canvas = new OffscreenCanvas(bitmap.width, bitmap.height);
|
|
83
|
+
const context = canvas.getContext('2d');
|
|
84
|
+
if (!context) throw new Error('Canvas is unavailable');
|
|
85
|
+
context.drawImage(mask, 0, 0, bitmap.width, bitmap.height);
|
|
86
|
+
const alpha = context.getImageData(0, 0, bitmap.width, bitmap.height).data;
|
|
87
|
+
context.clearRect(0, 0, canvas.width, canvas.height);
|
|
88
|
+
context.drawImage(bitmap, 0, 0);
|
|
89
|
+
const original = context.getImageData(0, 0, canvas.width, canvas.height);
|
|
90
|
+
applyAlpha(original.data, alpha);
|
|
91
|
+
context.putImageData(original, 0, 0);
|
|
92
|
+
result = await canvas.convertToBlob({ type: 'image/png' });
|
|
93
|
+
provider = candidate;
|
|
94
|
+
break;
|
|
95
|
+
} catch (error) {
|
|
96
|
+
lastError = error;
|
|
97
|
+
if (candidate === 'webgpu' && providers.includes('wasm')) post({ type: 'progress', stage: 'fallback' });
|
|
98
|
+
} finally {
|
|
99
|
+
input?.dispose();
|
|
100
|
+
input = undefined;
|
|
101
|
+
if (outputs) Object.values(outputs).forEach(tensor => tensor.dispose());
|
|
102
|
+
outputs = undefined;
|
|
103
|
+
await session?.release();
|
|
104
|
+
session = undefined;
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
if (!result) throw lastError || new Error('Inference failed');
|
|
108
|
+
post({ type: 'result', blob: result, provider });
|
|
109
|
+
} catch (error) {
|
|
110
|
+
post({ type: 'error', message: error instanceof Error ? error.message : String(error) });
|
|
111
|
+
} finally {
|
|
112
|
+
bitmap?.close();
|
|
113
|
+
urls.forEach(url => URL.revokeObjectURL(url));
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
self.onmessage = (event: MessageEvent<WorkerRequest>) => { void run(event.data); };
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
export type ModelId = 'u2netp' | 'birefnet' | 'modnet';
|
|
2
|
+
export type ModelPrecision = 'fp16' | 'fp32';
|
|
3
|
+
|
|
4
|
+
export interface ModelVariant {
|
|
5
|
+
precision: ModelPrecision;
|
|
6
|
+
size: number;
|
|
7
|
+
url: string;
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
export interface RemovalModel {
|
|
11
|
+
id: ModelId;
|
|
12
|
+
name: string;
|
|
13
|
+
size: number;
|
|
14
|
+
inputSize: number;
|
|
15
|
+
url: string;
|
|
16
|
+
variants?: readonly ModelVariant[];
|
|
17
|
+
sourceUrl?: string;
|
|
18
|
+
forceCpu?: boolean;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
const birefnetVariants: readonly ModelVariant[] = [
|
|
22
|
+
{ precision: 'fp16', size: 98484532,
|
|
23
|
+
url: 'https://www.modelscope.cn/models/sangsq/birefnet-lite-512/resolve/master/model_fp16.onnx?view=false' },
|
|
24
|
+
{ precision: 'fp32', size: 191877254,
|
|
25
|
+
url: 'https://www.modelscope.cn/models/sangsq/birefnet-lite-512/resolve/master/model.onnx?view=false' }
|
|
26
|
+
];
|
|
27
|
+
|
|
28
|
+
const modnetVariants: readonly ModelVariant[] = [
|
|
29
|
+
{ precision: 'fp32', size: 25888640,
|
|
30
|
+
url: 'https://www.modelscope.cn/models/onnx-community/modnet-webnn/resolve/master/onnx/model.onnx?view=false' },
|
|
31
|
+
{ precision: 'fp16', size: 12984781,
|
|
32
|
+
url: 'https://www.modelscope.cn/models/onnx-community/modnet-webnn/resolve/master/onnx/model_fp16.onnx?view=false' }
|
|
33
|
+
];
|
|
34
|
+
|
|
35
|
+
export const MODELS: readonly RemovalModel[] = [
|
|
36
|
+
{
|
|
37
|
+
id: 'u2netp', name: 'U2NetP', size: 4574861, inputSize: 320,
|
|
38
|
+
url: 'https://www.modelscope.cn/models/budaoshou/U2NetP-ONNX/resolve/master/u2netp.onnx?view=false'
|
|
39
|
+
},
|
|
40
|
+
{
|
|
41
|
+
id: 'birefnet', name: 'BiRefNet Lite 512', size: birefnetVariants[0].size, inputSize: 512,
|
|
42
|
+
url: birefnetVariants[0].url, variants: birefnetVariants,
|
|
43
|
+
sourceUrl: 'https://www.modelscope.cn/models/sangsq/birefnet-lite-512'
|
|
44
|
+
},
|
|
45
|
+
{
|
|
46
|
+
id: 'modnet', name: 'MODNet', size: modnetVariants[0].size, inputSize: 512, forceCpu: true,
|
|
47
|
+
url: modnetVariants[0].url, variants: modnetVariants,
|
|
48
|
+
sourceUrl: 'https://www.modelscope.cn/models/onnx-community/modnet-webnn'
|
|
49
|
+
}
|
|
50
|
+
];
|
|
51
|
+
|
|
52
|
+
export function getModel(id: string): RemovalModel {
|
|
53
|
+
const model = MODELS.find(item => item.id === id);
|
|
54
|
+
if (!model) throw new Error('Unknown background-removal model');
|
|
55
|
+
return model;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export async function validateModel(id: ModelId, blob: Blob): Promise<void> {
|
|
59
|
+
getModel(id);
|
|
60
|
+
if (blob.size < 1024 || blob.size > 1024 * 1024 * 1024) throw new Error('Invalid model size (1 KB-1 GB)');
|
|
61
|
+
const header = (await blob.slice(0, 256).text()).trimStart();
|
|
62
|
+
if (/^(?:<!doctype|<html|<\?xml|\{|version https:\/\/git-lfs)/i.test(header)) {
|
|
63
|
+
throw new Error('Expected ONNX binary, received a web page or metadata');
|
|
64
|
+
}
|
|
65
|
+
}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import type { ModelId } from './models.ts';
|
|
2
|
+
|
|
3
|
+
const MEAN = [0.485, 0.456, 0.406];
|
|
4
|
+
const STD = [0.229, 0.224, 0.225];
|
|
5
|
+
|
|
6
|
+
export function preprocess(rgba: Uint8ClampedArray, modelId: ModelId): Float32Array {
|
|
7
|
+
const pixels = rgba.length / 4;
|
|
8
|
+
const tensor = new Float32Array(pixels * 3);
|
|
9
|
+
let scale = 255;
|
|
10
|
+
if (modelId === 'u2netp') {
|
|
11
|
+
scale = 1;
|
|
12
|
+
for (let offset = 0; offset < rgba.length; offset += 4) {
|
|
13
|
+
scale = Math.max(scale, rgba[offset], rgba[offset + 1], rgba[offset + 2]);
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
for (let pixel = 0; pixel < pixels; pixel++) {
|
|
17
|
+
for (let channel = 0; channel < 3; channel++) {
|
|
18
|
+
tensor[channel * pixels + pixel] = modelId === 'modnet'
|
|
19
|
+
? (rgba[pixel * 4 + channel] / 255 - 0.5) / 0.5
|
|
20
|
+
: (rgba[pixel * 4 + channel] / scale - MEAN[channel]) / STD[channel];
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
return tensor;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export function halfToFloat(value: number): number {
|
|
27
|
+
const sign = value & 0x8000 ? -1 : 1;
|
|
28
|
+
const exponent = (value >> 10) & 31;
|
|
29
|
+
const fraction = value & 1023;
|
|
30
|
+
if (!exponent) return sign * 2 ** -14 * fraction / 1024;
|
|
31
|
+
if (exponent === 31) return fraction ? NaN : sign * Infinity;
|
|
32
|
+
return sign * 2 ** (exponent - 15) * (1 + fraction / 1024);
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export function createMask(data: Float32Array | Uint16Array, modelId: ModelId): Uint8ClampedArray<ArrayBuffer> {
|
|
36
|
+
const values = data instanceof Uint16Array ? Float32Array.from(data, halfToFloat) : data;
|
|
37
|
+
let minimum = Infinity;
|
|
38
|
+
let maximum = -Infinity;
|
|
39
|
+
for (const value of values) {
|
|
40
|
+
if (!Number.isFinite(value)) throw new Error('Model returned a non-finite mask');
|
|
41
|
+
minimum = Math.min(minimum, value);
|
|
42
|
+
maximum = Math.max(maximum, value);
|
|
43
|
+
}
|
|
44
|
+
if (!values.length) throw new Error('Model returned an empty mask');
|
|
45
|
+
const mask = new Uint8ClampedArray(values.length * 4);
|
|
46
|
+
for (let index = 0; index < values.length; index++) {
|
|
47
|
+
const probability = modelId === 'u2netp'
|
|
48
|
+
? (maximum > minimum ? (values[index] - minimum) / (maximum - minimum) : Math.max(0, Math.min(1, values[index])))
|
|
49
|
+
: modelId === 'modnet' ? Math.max(0, Math.min(1, values[index])) : 1 / (1 + Math.exp(-values[index]));
|
|
50
|
+
mask[index * 4] = mask[index * 4 + 1] = mask[index * 4 + 2] = probability * 255;
|
|
51
|
+
mask[index * 4 + 3] = 255;
|
|
52
|
+
}
|
|
53
|
+
return mask;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export function applyAlpha(original: Uint8ClampedArray, mask: Uint8ClampedArray): void {
|
|
57
|
+
if (original.length !== mask.length) throw new Error('Mask dimensions do not match image');
|
|
58
|
+
for (let offset = 0; offset < original.length; offset += 4) {
|
|
59
|
+
original[offset + 3] = Math.round(original[offset + 3] * mask[offset] / 255);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
import { getModel, validateModel, type ModelId, type ModelPrecision } from './models.ts';
|
|
2
|
+
import { readModel, readRuntime, RUNTIME_VERSION, saveRuntime } from './storage.ts';
|
|
3
|
+
import type { Progress, RuntimeFiles } from './types.ts';
|
|
4
|
+
|
|
5
|
+
export const RUNTIME_ASSETS = {
|
|
6
|
+
script: 'ort.webgpu.bundle.min.mjs',
|
|
7
|
+
loader: 'ort-wasm-simd-threaded.asyncify.mjs',
|
|
8
|
+
wasm: 'ort-wasm-simd-threaded.asyncify.wasm'
|
|
9
|
+
} as const;
|
|
10
|
+
export const RUNTIME_BASE = `https://cdn.jsdelivr.net/npm/onnxruntime-web@${RUNTIME_VERSION}/dist/`;
|
|
11
|
+
|
|
12
|
+
export async function download(url: string, maximum: number, signal: AbortSignal,
|
|
13
|
+
onProgress?: (progress: Progress) => void): Promise<Blob> {
|
|
14
|
+
const response = await fetch(url, { signal, credentials: 'omit', referrerPolicy: 'no-referrer' });
|
|
15
|
+
if (!response.ok) throw new Error(`Download HTTP ${response.status}`);
|
|
16
|
+
const encoding = response.headers.get('content-encoding')?.trim().toLowerCase();
|
|
17
|
+
const total = encoding && encoding !== 'identity' ? undefined : Number(response.headers.get('content-length')) || undefined;
|
|
18
|
+
if (total && total > maximum) {
|
|
19
|
+
await response.body?.cancel();
|
|
20
|
+
throw new Error('Download exceeds size limit');
|
|
21
|
+
}
|
|
22
|
+
if (!response.body) throw new Error('Download stream unavailable');
|
|
23
|
+
const reader = response.body.getReader();
|
|
24
|
+
const chunks: ArrayBuffer[] = [];
|
|
25
|
+
let loaded = 0;
|
|
26
|
+
try {
|
|
27
|
+
while (true) {
|
|
28
|
+
const chunk = await reader.read();
|
|
29
|
+
if (chunk.done) break;
|
|
30
|
+
loaded += chunk.value.byteLength;
|
|
31
|
+
if (loaded > maximum) throw new Error('Download exceeds size limit');
|
|
32
|
+
chunks.push(chunk.value.slice().buffer);
|
|
33
|
+
onProgress?.({ stage: 'download', loaded, total });
|
|
34
|
+
}
|
|
35
|
+
} finally {
|
|
36
|
+
await reader.cancel().catch(() => undefined);
|
|
37
|
+
reader.releaseLock();
|
|
38
|
+
}
|
|
39
|
+
signal.throwIfAborted();
|
|
40
|
+
if (!loaded) throw new Error('Downloaded file is empty');
|
|
41
|
+
return new Blob(chunks, { type: response.headers.get('content-type')?.split(';')[0].trim() || '' });
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export async function validateRuntime(files: RuntimeFiles): Promise<void> {
|
|
45
|
+
const magic = new Uint8Array(await files.wasm.slice(0, 4).arrayBuffer());
|
|
46
|
+
if (magic.join(',') !== '0,97,115,109') throw new Error('Invalid WASM binary');
|
|
47
|
+
for (const file of [files.script, files.loader]) {
|
|
48
|
+
const prefix = (await file.slice(0, 256).text()).trimStart();
|
|
49
|
+
if (file.size < 1000 || /^</.test(prefix)) throw new Error('Invalid runtime JavaScript');
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export async function installRuntime(signal: AbortSignal, onProgress?: (progress: Progress) => void): Promise<void> {
|
|
54
|
+
const files = {} as RuntimeFiles;
|
|
55
|
+
for (const key of Object.keys(RUNTIME_ASSETS) as (keyof RuntimeFiles)[]) {
|
|
56
|
+
onProgress?.({ stage: RUNTIME_ASSETS[key] });
|
|
57
|
+
files[key] = await download(RUNTIME_BASE + RUNTIME_ASSETS[key], 80 * 1024 * 1024, signal,
|
|
58
|
+
progress => onProgress?.({ ...progress, stage: RUNTIME_ASSETS[key] }));
|
|
59
|
+
}
|
|
60
|
+
await validateRuntime(files);
|
|
61
|
+
signal.throwIfAborted();
|
|
62
|
+
await saveRuntime(files);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export async function ensureRuntime(signal: AbortSignal, onProgress?: (progress: Progress) => void): Promise<RuntimeFiles> {
|
|
66
|
+
signal.throwIfAborted();
|
|
67
|
+
const cached = await readRuntime();
|
|
68
|
+
signal.throwIfAborted();
|
|
69
|
+
if (cached) return cached;
|
|
70
|
+
try {
|
|
71
|
+
onProgress?.({ stage: 'runtime_download' });
|
|
72
|
+
await installRuntime(AbortSignal.any([signal, AbortSignal.timeout(180_000)]),
|
|
73
|
+
progress => onProgress?.({ ...progress, stage: 'runtime_download' }));
|
|
74
|
+
signal.throwIfAborted();
|
|
75
|
+
const runtime = await readRuntime();
|
|
76
|
+
if (!runtime) throw new Error('运行时下载完成但缓存不可用。');
|
|
77
|
+
return runtime;
|
|
78
|
+
} catch (error) {
|
|
79
|
+
signal.throwIfAborted();
|
|
80
|
+
throw new Error(`运行时下载失败,请重试:${error instanceof Error ? error.message : String(error)}`, { cause: error });
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
export async function importRuntime(files: File[]): Promise<void> {
|
|
85
|
+
const runtime = {} as RuntimeFiles;
|
|
86
|
+
for (const key of Object.keys(RUNTIME_ASSETS) as (keyof RuntimeFiles)[]) {
|
|
87
|
+
const file = files.find(item => item.name === RUNTIME_ASSETS[key]);
|
|
88
|
+
if (!file) throw new Error(`Missing ${RUNTIME_ASSETS[key]}`);
|
|
89
|
+
if (file.size > 80 * 1024 * 1024) throw new Error('Runtime file exceeds size limit');
|
|
90
|
+
runtime[key] = file;
|
|
91
|
+
}
|
|
92
|
+
await validateRuntime(runtime);
|
|
93
|
+
await saveRuntime(runtime);
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
export async function loadResources(id: ModelId, signal: AbortSignal): Promise<{ model: Blob; runtime: RuntimeFiles }> {
|
|
97
|
+
const runtime = await ensureRuntime(signal);
|
|
98
|
+
const stored = await readModel(id);
|
|
99
|
+
const model = stored?.blob;
|
|
100
|
+
if (!model) throw new Error('MODEL_MISSING');
|
|
101
|
+
await validateModel(id, model);
|
|
102
|
+
signal.throwIfAborted();
|
|
103
|
+
return { model, runtime };
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
export async function downloadModel(id: ModelId, signal: AbortSignal, onProgress?: (progress: Progress) => void,
|
|
107
|
+
precision?: ModelPrecision): Promise<Blob> {
|
|
108
|
+
const profile = getModel(id);
|
|
109
|
+
const variant = precision === undefined ? profile : profile.variants?.find(item => item.precision === precision);
|
|
110
|
+
if (!variant) throw new Error('Unsupported model precision');
|
|
111
|
+
const model = await download(variant.url, 1024 * 1024 * 1024, signal, onProgress);
|
|
112
|
+
await validateModel(id, model);
|
|
113
|
+
return model;
|
|
114
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import type { Progress } from './types.ts';
|
|
2
|
+
|
|
3
|
+
export function progressText(progress: Progress | undefined, english: boolean): string {
|
|
4
|
+
if (!progress) return '';
|
|
5
|
+
const labels: Record<string, [string, string]> = {
|
|
6
|
+
runtime: ['初始化运行时', 'Initializing runtime'], model: ['加载模型', 'Loading model'],
|
|
7
|
+
runtime_download: ['下载运行时', 'Downloading runtime'],
|
|
8
|
+
webgpu: ['GPU 推理中', 'Running on GPU'], wasm: ['CPU 推理中', 'Running on CPU'],
|
|
9
|
+
fallback: ['GPU 不兼容,切换 CPU', 'GPU unavailable, switching to CPU'],
|
|
10
|
+
download: ['下载模型', 'Downloading model'], verify: ['验证模型', 'Verifying model']
|
|
11
|
+
};
|
|
12
|
+
const label = labels[progress.stage]?.[english ? 1 : 0] || progress.stage;
|
|
13
|
+
if (progress.loaded === undefined) return label;
|
|
14
|
+
const size = `${(progress.loaded / 1_000_000).toFixed(1)} MB`;
|
|
15
|
+
return `${label} · ${size}${progress.total ? ` / ${(progress.total / 1_000_000).toFixed(1)} MB` : ''}`;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export function errorText(error: unknown, english: boolean): string {
|
|
19
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
20
|
+
if (message === 'RUNTIME_MISSING') return english ? 'Install the runtime in AI Settings > Background Removal first.' : '请先在 AI 设置 > 抠图中安装运行时。';
|
|
21
|
+
if (message === 'MODEL_MISSING') return english ? 'The selected model is missing. Download it in AI Settings > Background Removal.' : '所选模型不存在,请在 AI 设置 > 抠图中下载。';
|
|
22
|
+
return message;
|
|
23
|
+
}
|