pi-tsgw 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -3
- package/extensions/images/README.md +81 -0
- package/extensions/images/adapter.ts +119 -0
- package/extensions/images/image-data.ts +228 -0
- package/extensions/images/models.ts +16 -0
- package/extensions/images/parameters.ts +136 -0
- package/extensions/images/request.ts +102 -0
- package/extensions/images/response.ts +128 -0
- package/extensions/images/stream-response.ts +180 -0
- package/extensions/images/tool-files.ts +156 -0
- package/extensions/images/tool-schema.ts +73 -0
- package/extensions/images/tools.ts +98 -0
- package/extensions/models/catalog.ts +3 -3
- package/extensions/models/vendors/openai.ts +19 -0
- package/extensions/provider/index.ts +22 -8
- package/index.ts +6 -1
- package/package.json +3 -3
- package/skills/tsgw-delegate-model-selection/SKILL.md +3 -2
- package/skills/tsgw-image-generation/SKILL.md +76 -0
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
import type { AssistantImages, ImagesContext, Usage } from "@earendil-works/pi-ai";
|
|
2
|
+
import { inspectImage } from "./image-data.ts";
|
|
3
|
+
import { ImageFailure } from "./response.ts";
|
|
4
|
+
|
|
5
|
+
export type ImageFormat = "png" | "jpeg" | "webp";
|
|
6
|
+
export type ImageOperation = "generate" | "edit";
|
|
7
|
+
export interface ImageReference { image_url?: string; file_id?: string }
|
|
8
|
+
|
|
9
|
+
/** 十进制字节:单张严格小于上限,参考图与蒙版合计不超过总上限。 */
|
|
10
|
+
export const MAX_IMAGE_INPUT_BYTES = 50_000_000;
|
|
11
|
+
export const MAX_IMAGE_INPUT_TOTAL_BYTES = 100_000_000;
|
|
12
|
+
|
|
13
|
+
/** context.tsgw 是插件扩展,不是 Pi 原生第三参数;API 的 null 按省略处理。 */
|
|
14
|
+
export interface TsgwImageParameters {
|
|
15
|
+
operation?: ImageOperation | null;
|
|
16
|
+
images?: ImageReference[] | null;
|
|
17
|
+
mask?: ImageReference | null;
|
|
18
|
+
quality?: "auto" | "low" | "medium" | "high" | "xhigh" | "max" | null;
|
|
19
|
+
/** auto 或宽x高:16 的倍数、边长不超过 3840、比例不超过 3、像素 655360..8294400。 */
|
|
20
|
+
size?: string | null;
|
|
21
|
+
background?: "auto" | "opaque" | "transparent" | null;
|
|
22
|
+
output_format?: ImageFormat | null;
|
|
23
|
+
output_compression?: number | null;
|
|
24
|
+
n?: number | null;
|
|
25
|
+
moderation?: "auto" | "low" | null;
|
|
26
|
+
stream?: boolean | null;
|
|
27
|
+
partial_images?: number | null;
|
|
28
|
+
safety_identifier?: string | null;
|
|
29
|
+
user?: string | null;
|
|
30
|
+
}
|
|
31
|
+
export interface TsgwImagesContext extends ImagesContext { tsgw?: TsgwImageParameters }
|
|
32
|
+
/** 多图 SSE 的逐事件用量作用域不明:保留报告,不伪造请求总量。 */
|
|
33
|
+
export type ImageUsageReport = Pick<Usage, "input" | "output" | "cacheRead" | "cacheWrite" | "totalTokens">;
|
|
34
|
+
export interface TsgwImagesResult extends AssistantImages {
|
|
35
|
+
tsgwUsage?: { status: "unaggregated"; reports: ImageUsageReport[] };
|
|
36
|
+
}
|
|
37
|
+
export type NormalizedImageParameters = TsgwImageParameters & {
|
|
38
|
+
quality: NonNullable<TsgwImageParameters["quality"]>; size: string;
|
|
39
|
+
background: NonNullable<TsgwImageParameters["background"]>; output_format: ImageFormat;
|
|
40
|
+
n: number; moderation: "auto" | "low"; stream: boolean; partial_images: number;
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
export const IMAGE_PARAMETER_KEYS = ["operation", "images", "mask", "quality", "size", "background", "output_format",
|
|
44
|
+
"output_compression", "n", "moderation", "stream", "partial_images", "safety_identifier", "user"] as const;
|
|
45
|
+
const QUALITIES = new Set(["low", "medium", "high", "xhigh", "max", "auto"]);
|
|
46
|
+
const BACKGROUNDS = new Set(["auto", "opaque", "transparent"]);
|
|
47
|
+
const FORMATS = new Set(["png", "jpeg", "webp"]);
|
|
48
|
+
const invalid = () => new ImageFailure("图片 tsgw 参数无效:请检查参数名、类型、取值范围及格式/流式组合。");
|
|
49
|
+
|
|
50
|
+
export function plainImageObject(value: unknown): value is Record<string, unknown> {
|
|
51
|
+
return value !== null && typeof value === "object" && !Array.isArray(value) &&
|
|
52
|
+
(Object.getPrototypeOf(value) === Object.prototype || Object.getPrototypeOf(value) === null);
|
|
53
|
+
}
|
|
54
|
+
export function validSize(value: unknown): boolean {
|
|
55
|
+
if (value === "auto") return true;
|
|
56
|
+
if (typeof value !== "string" || /^[1-9]\d{0,3}x[1-9]\d{0,3}$/.exec(value)?.[0] !== value) return false;
|
|
57
|
+
const [width, height] = value.split("x").map(Number);
|
|
58
|
+
return width % 16 === 0 && height % 16 === 0 && Math.max(width, height) <= 3840 &&
|
|
59
|
+
Math.max(width, height) / Math.min(width, height) <= 3 &&
|
|
60
|
+
width * height >= 655_360 && width * height <= 8_294_400;
|
|
61
|
+
}
|
|
62
|
+
const integer = (value: unknown, min: number, max: number): value is number =>
|
|
63
|
+
typeof value === "number" && Number.isInteger(value) && value >= min && value <= max;
|
|
64
|
+
|
|
65
|
+
/** 仅用于已校验的引用:按 canonical base64 长度与 padding 计算 decoded 字节,不再次分配图片 Buffer。 */
|
|
66
|
+
export function imageReferenceInputBytes(reference: ImageReference): number {
|
|
67
|
+
const url = reference.image_url;
|
|
68
|
+
if (!url?.startsWith("data:")) return 0;
|
|
69
|
+
return (url.length - url.indexOf(",") - 1) / 4 * 3 - (url.endsWith("==") ? 2 : url.endsWith("=") ? 1 : 0);
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** 不下载 URL;data URI 保留原始图片字节,且核对声明 MIME 与容器。 */
|
|
73
|
+
export function normalizeImageReference(value: unknown): ImageReference {
|
|
74
|
+
const fail = () => new ImageFailure("图片引用无效:须提供唯一的非空 file_id 或无用户名密码的 HTTP(S)/图片 data URI。");
|
|
75
|
+
if (!plainImageObject(value) || Reflect.ownKeys(value).length !== 1 || Reflect.ownKeys(value).some(key => key !== "image_url" && key !== "file_id")) throw fail();
|
|
76
|
+
const url = Object.hasOwn(value, "image_url") ? value.image_url : undefined;
|
|
77
|
+
const file = Object.hasOwn(value, "file_id") ? value.file_id : undefined;
|
|
78
|
+
if (url !== undefined && file !== undefined) throw fail();
|
|
79
|
+
if (file !== undefined) {
|
|
80
|
+
if (typeof file !== "string" || !file.trim() || file !== file.trim() || /[\u0000-\u001f\u007f]/.test(file)) throw fail();
|
|
81
|
+
return { file_id: file };
|
|
82
|
+
}
|
|
83
|
+
if (typeof url !== "string" || !url || url !== url.trim() || /[\u0000-\u0020\u007f]/.test(url)) throw fail();
|
|
84
|
+
if (url.startsWith("data:")) {
|
|
85
|
+
const overBudget = () => new ImageFailure("单张参考图或蒙版必须小于 50000000 字节。");
|
|
86
|
+
if (url.length > Math.ceil(MAX_IMAGE_INPUT_BYTES / 3) * 4 + 128) throw overBudget();
|
|
87
|
+
const match = /^data:image\/(png|jpeg|webp);base64,([A-Za-z0-9+/]*={0,2})$/.exec(url);
|
|
88
|
+
if (!match) throw fail();
|
|
89
|
+
// 与工具单文件预算一致;输入图不施加输出尺寸上限。
|
|
90
|
+
if (imageReferenceInputBytes({ image_url: url }) >= MAX_IMAGE_INPUT_BYTES) throw overBudget();
|
|
91
|
+
inspectImage(match[2], match[1] as ImageFormat);
|
|
92
|
+
return { image_url: url };
|
|
93
|
+
}
|
|
94
|
+
let parsed: URL;
|
|
95
|
+
try { parsed = new URL(url); } catch { throw fail(); }
|
|
96
|
+
if (!/^https?:\/\//i.test(url) || /^https?:\/\/[^/?#]*@/i.test(url) || !["http:", "https:"].includes(parsed.protocol) || !parsed.hostname || parsed.username || parsed.password) throw fail();
|
|
97
|
+
return { image_url: parsed.href };
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
export function normalizeImageParameters(value: unknown): NormalizedImageParameters {
|
|
101
|
+
if (value == null) value = {};
|
|
102
|
+
if (!plainImageObject(value) || Reflect.ownKeys(value).some(key => !IMAGE_PARAMETER_KEYS.some(allowed => key === allowed))) throw invalid();
|
|
103
|
+
const source = value;
|
|
104
|
+
const get = (key: string, fallback?: unknown) => (Object.hasOwn(source, key) ? source[key] : undefined) ?? fallback;
|
|
105
|
+
const quality = get("quality", "auto"), size = get("size", "auto"), background = get("background", "auto");
|
|
106
|
+
const output_format = get("output_format", "png"), n = get("n", 1), moderation = get("moderation", "auto");
|
|
107
|
+
const stream = get("stream", false), partial_images = get("partial_images", 0);
|
|
108
|
+
if (typeof quality !== "string" || !QUALITIES.has(quality) || !validSize(size) ||
|
|
109
|
+
typeof background !== "string" || !BACKGROUNDS.has(background) || typeof output_format !== "string" || !FORMATS.has(output_format) ||
|
|
110
|
+
!integer(n, 1, 10) || (moderation !== "auto" && moderation !== "low") || typeof stream !== "boolean" ||
|
|
111
|
+
!integer(partial_images, 0, 3) || (!stream && partial_images !== 0)) throw invalid();
|
|
112
|
+
const result = { quality, size, background, output_format, n, moderation, stream, partial_images } as NormalizedImageParameters;
|
|
113
|
+
const compression = get("output_compression");
|
|
114
|
+
if (compression !== undefined) {
|
|
115
|
+
if (!integer(compression, 0, 100) || output_format === "png") throw invalid();
|
|
116
|
+
result.output_compression = compression;
|
|
117
|
+
}
|
|
118
|
+
if (background === "transparent" && output_format === "jpeg") throw invalid();
|
|
119
|
+
for (const key of ["safety_identifier", "user"] as const) {
|
|
120
|
+
const text = get(key);
|
|
121
|
+
if (text === undefined) continue;
|
|
122
|
+
if (typeof text !== "string" || (key === "safety_identifier" && text.length > 128)) throw invalid();
|
|
123
|
+
result[key] = text;
|
|
124
|
+
}
|
|
125
|
+
const operation = get("operation"), images = get("images"), mask = get("mask");
|
|
126
|
+
if (operation !== undefined) {
|
|
127
|
+
if (operation !== "generate" && operation !== "edit") throw invalid();
|
|
128
|
+
result.operation = operation;
|
|
129
|
+
}
|
|
130
|
+
if (images !== undefined) {
|
|
131
|
+
if (!Array.isArray(images) || images.length < 1 || images.length > 16) throw invalid();
|
|
132
|
+
result.images = Array.from(images, normalizeImageReference);
|
|
133
|
+
}
|
|
134
|
+
if (mask !== undefined) result.mask = normalizeImageReference(mask);
|
|
135
|
+
return result;
|
|
136
|
+
}
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import type { ImagesContext } from "@earendil-works/pi-ai";
|
|
2
|
+
import { inspectImage } from "./image-data.ts";
|
|
3
|
+
import { IMAGE_PARAMETER_KEYS, MAX_IMAGE_INPUT_TOTAL_BYTES, imageReferenceInputBytes,
|
|
4
|
+
normalizeImageParameters, normalizeImageReference, plainImageObject,
|
|
5
|
+
type ImageFormat, type ImageOperation, type ImageReference, type NormalizedImageParameters } from "./parameters.ts";
|
|
6
|
+
import { ImageFailure } from "./response.ts";
|
|
7
|
+
|
|
8
|
+
export interface ImageRequest {
|
|
9
|
+
operation: ImageOperation;
|
|
10
|
+
payload: Record<string, unknown>;
|
|
11
|
+
}
|
|
12
|
+
export interface ImageResponseExpectation { count: number; format: ImageFormat; stream: boolean; operation: ImageOperation }
|
|
13
|
+
export interface SerializedImageRequest { body: string; expected: ImageResponseExpectation }
|
|
14
|
+
|
|
15
|
+
export function validPrompt(value: unknown): value is string {
|
|
16
|
+
return typeof value === "string" && value.trim().length > 0 && value.length <= 32_000;
|
|
17
|
+
}
|
|
18
|
+
function dataImage(ref: ImageReference) {
|
|
19
|
+
const match = /^data:image\/(png|jpeg|webp);base64,(.*)$/.exec(ref.image_url ?? "");
|
|
20
|
+
return match ? inspectImage(match[2], match[1] as ImageFormat) : undefined;
|
|
21
|
+
}
|
|
22
|
+
function validateOperation(parameters: NormalizedImageParameters, operation: ImageOperation): void {
|
|
23
|
+
const images = parameters.images, mask = parameters.mask;
|
|
24
|
+
if (operation === "generate") {
|
|
25
|
+
if (images || mask) throw new ImageFailure("图片生成不接受参考图或蒙版;请使用图片编辑。");
|
|
26
|
+
return;
|
|
27
|
+
}
|
|
28
|
+
if (!images?.length) throw new ImageFailure("图片编辑需要 1..16 张参考图。");
|
|
29
|
+
// 输入预算独立于输出预算;远程引用仅交上游,不在插件内下载。
|
|
30
|
+
const refs = mask ? [...images, mask] : images;
|
|
31
|
+
if (refs.reduce((sum, ref) => sum + imageReferenceInputBytes(ref), 0) > MAX_IMAGE_INPUT_TOTAL_BYTES) {
|
|
32
|
+
throw new ImageFailure("图片参考图与蒙版的总输入不得超过 100000000 字节。");
|
|
33
|
+
}
|
|
34
|
+
if (mask) {
|
|
35
|
+
const first = dataImage(images[0]), maskData = dataImage(mask);
|
|
36
|
+
if (maskData && !maskData.hasAlpha) throw new ImageFailure("图片编辑蒙版需要 alpha 通道。");
|
|
37
|
+
if (first && maskData && (first.width !== maskData.width || first.height !== maskData.height || first.mimeType !== maskData.mimeType)) {
|
|
38
|
+
throw new ImageFailure("图片编辑蒙版必须与第一张参考图尺寸和格式一致。");
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
function apiParameters(parameters: NormalizedImageParameters): Record<string, unknown> {
|
|
43
|
+
const { operation: _operation, partial_images, ...api } = parameters;
|
|
44
|
+
return parameters.stream ? { ...api, partial_images } : api;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** 只构造自己的新对象,不更改冻结的模型、context、参数或原始图片。 */
|
|
48
|
+
export function createImageRequest(context: ImagesContext, modelId: string): ImageRequest {
|
|
49
|
+
if (!plainImageObject(context) || !Array.isArray(context.input) || !context.input.length) {
|
|
50
|
+
throw new ImageFailure("图片请求需要非空输入与文本提示词。");
|
|
51
|
+
}
|
|
52
|
+
const parameters = normalizeImageParameters(Object.hasOwn(context, "tsgw") ? context.tsgw : undefined);
|
|
53
|
+
const texts: string[] = [], nativeImages: ImageReference[] = [];
|
|
54
|
+
for (const block of context.input) {
|
|
55
|
+
if (!plainImageObject(block)) throw new ImageFailure("图片请求输入块无效。");
|
|
56
|
+
if (block.type === "text" && validPrompt(block.text)) texts.push(block.text);
|
|
57
|
+
else if (block.type === "image") {
|
|
58
|
+
if (typeof block.data !== "string" || !["image/png", "image/jpeg", "image/webp"].includes(block.mimeType)) {
|
|
59
|
+
throw new ImageFailure("参考图需要有效的 PNG、JPEG 或 WebP 图片块。");
|
|
60
|
+
}
|
|
61
|
+
// 引用校验先检查输入预算,再核对声明 MIME 与容器,避免先解码无界输入。
|
|
62
|
+
nativeImages.push(normalizeImageReference({ image_url: `data:${block.mimeType};base64,${block.data}` }));
|
|
63
|
+
} else throw new ImageFailure("图片请求仅接受非空文本和有效图片块。");
|
|
64
|
+
}
|
|
65
|
+
const prompt = texts.join("\n");
|
|
66
|
+
if (!validPrompt(prompt)) throw new ImageFailure("图片提示词须非空且合并后不得超过 32000 字符。");
|
|
67
|
+
if (nativeImages.length) {
|
|
68
|
+
if (parameters.images) throw new ImageFailure("原生图片块与 tsgw.images 不得同时提供。");
|
|
69
|
+
if (nativeImages.length > 16) throw new ImageFailure("图片编辑最多接受 16 张参考图。");
|
|
70
|
+
parameters.images = nativeImages;
|
|
71
|
+
}
|
|
72
|
+
const operation = parameters.operation ?? (parameters.images ? "edit" : "generate");
|
|
73
|
+
validateOperation(parameters, operation);
|
|
74
|
+
return { operation, payload: { model: modelId, prompt, ...apiParameters(parameters) } };
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** 先序列化,再验证实际发送值,避免 getter/toJSON 改写模型或端点边界。 */
|
|
78
|
+
export function serializeImageRequest(payload: unknown, modelId: string, operation: ImageOperation): SerializedImageRequest {
|
|
79
|
+
let body: string;
|
|
80
|
+
let value: unknown;
|
|
81
|
+
try {
|
|
82
|
+
const encoded = JSON.stringify(payload);
|
|
83
|
+
if (encoded === undefined) throw new Error();
|
|
84
|
+
body = encoded;
|
|
85
|
+
value = JSON.parse(body);
|
|
86
|
+
} catch { throw new ImageFailure("图片请求无法序列化为有效 JSON。"); }
|
|
87
|
+
if (!plainImageObject(value) || value.model !== modelId || !validPrompt(value.prompt)) {
|
|
88
|
+
throw new ImageFailure("图片请求必须保留当前模型及非空且不超过 32000 字符的提示词。");
|
|
89
|
+
}
|
|
90
|
+
// 未知 hook 字段原样保留;但不能借旧协议字段切换操作、返回 URL 或伪造不支持的能力。
|
|
91
|
+
for (const key of ["operation", "image", "response_format", "style", "standard", "hd", "input_fidelity"]) {
|
|
92
|
+
if (Object.hasOwn(value, key)) throw new ImageFailure("图片请求包含不支持或混淆端点的字段。");
|
|
93
|
+
}
|
|
94
|
+
if (operation === "generate" && (Object.hasOwn(value, "images") || Object.hasOwn(value, "mask"))) {
|
|
95
|
+
throw new ImageFailure("图片生成端点不接受参考图或蒙版。");
|
|
96
|
+
}
|
|
97
|
+
const selected: Record<string, unknown> = {};
|
|
98
|
+
for (const key of IMAGE_PARAMETER_KEYS) if (Object.hasOwn(value, key)) selected[key] = value[key];
|
|
99
|
+
const parameters = normalizeImageParameters(selected);
|
|
100
|
+
validateOperation(parameters, operation);
|
|
101
|
+
return { body, expected: { count: parameters.n, format: parameters.output_format, stream: parameters.stream, operation } };
|
|
102
|
+
}
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
import type { ImageApi, ImageContent, ImageModel, Usage } from "@earendil-works/pi-ai";
|
|
2
|
+
import { ImageFailure, inspectImage, type ImageFormat, type InspectedImage } from "./image-data.ts";
|
|
3
|
+
|
|
4
|
+
export { ImageFailure } from "./image-data.ts";
|
|
5
|
+
export const MAX_RESPONSE_BYTES = 32 * 1024 * 1024;
|
|
6
|
+
|
|
7
|
+
export function record(value: unknown): value is Record<string, unknown> {
|
|
8
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
// race 会观察迟到的拒绝;即使自定义 fetch 或钩子不响应 signal,也及时结束等待。
|
|
12
|
+
export async function abortable<T>(signal: AbortSignal, action: () => T | PromiseLike<T>): Promise<T> {
|
|
13
|
+
signal.throwIfAborted();
|
|
14
|
+
let listener: () => void = () => {};
|
|
15
|
+
const cancelled = new Promise<never>((_, reject) => {
|
|
16
|
+
listener = () => reject(signal.reason);
|
|
17
|
+
signal.addEventListener("abort", listener, { once: true });
|
|
18
|
+
});
|
|
19
|
+
try {
|
|
20
|
+
return await Promise.race([Promise.resolve().then(() => {
|
|
21
|
+
signal.throwIfAborted();
|
|
22
|
+
return action();
|
|
23
|
+
}), cancelled]);
|
|
24
|
+
} finally {
|
|
25
|
+
signal.removeEventListener("abort", listener);
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export async function readJson(response: Response, signal: AbortSignal, maxBytes = MAX_RESPONSE_BYTES): Promise<unknown> {
|
|
30
|
+
if (!Number.isSafeInteger(maxBytes) || maxBytes < 1) throw new ImageFailure("图片响应预算无效。");
|
|
31
|
+
const length = response.headers.get("content-length");
|
|
32
|
+
if (length && /^\d+$/.test(length) && Number(length) > maxBytes) {
|
|
33
|
+
throw new ImageFailure(`图片响应超过 ${maxBytes / 1024 / 1024} MiB 读取预算上限。`);
|
|
34
|
+
}
|
|
35
|
+
if (!response.body) throw new ImageFailure("图片响应为空。");
|
|
36
|
+
const reader = response.body.getReader();
|
|
37
|
+
const chunks: Buffer[] = [];
|
|
38
|
+
let size = 0;
|
|
39
|
+
let complete = false;
|
|
40
|
+
try {
|
|
41
|
+
while (true) {
|
|
42
|
+
const item = await abortable(signal, () => reader.read());
|
|
43
|
+
if (item.done) { complete = true; break; }
|
|
44
|
+
size += item.value.byteLength;
|
|
45
|
+
if (size > maxBytes) throw new ImageFailure(`图片响应超过 ${maxBytes / 1024 / 1024} MiB 读取预算上限。`);
|
|
46
|
+
chunks.push(Buffer.from(item.value));
|
|
47
|
+
}
|
|
48
|
+
try {
|
|
49
|
+
return JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(Buffer.concat(chunks, size)));
|
|
50
|
+
} catch {
|
|
51
|
+
throw new ImageFailure("图片响应不是有效 JSON。");
|
|
52
|
+
}
|
|
53
|
+
} finally {
|
|
54
|
+
// 不等待可能永不完成的自定义底层 cancel,同时观察其拒绝。
|
|
55
|
+
if (!complete) void reader.cancel().catch(() => {});
|
|
56
|
+
reader.releaseLock();
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function declaredMetadata(raw: Record<string, unknown>, image: InspectedImage, format: ImageFormat): void {
|
|
61
|
+
if (raw.output_format !== undefined && raw.output_format !== format) throw new ImageFailure("图片声明格式与字节不一致。");
|
|
62
|
+
if (raw.size !== undefined && raw.size !== "auto" && raw.size !== `${image.width}x${image.height}`) {
|
|
63
|
+
throw new ImageFailure("图片声明尺寸与字节不一致。");
|
|
64
|
+
}
|
|
65
|
+
if ((raw.width !== undefined && raw.width !== image.width) || (raw.height !== undefined && raw.height !== image.height) ||
|
|
66
|
+
(raw.mime_type !== undefined && raw.mime_type !== image.mimeType)) throw new ImageFailure("图片声明元数据与字节不一致。");
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export function parseImages(raw: unknown, count: number, format: ImageFormat): ImageContent[] {
|
|
70
|
+
if (!Number.isInteger(count) || count < 1 || count > 10 || !["png", "jpeg", "webp"].includes(format) ||
|
|
71
|
+
!record(raw) || !Array.isArray(raw.data) || raw.data.length !== count) {
|
|
72
|
+
throw new ImageFailure("图片响应数量必须与请求一致,且只包含有效的 base64 图片。");
|
|
73
|
+
}
|
|
74
|
+
return raw.data.map(item => {
|
|
75
|
+
if (!record(item)) throw new ImageFailure("图片响应缺少 base64 图片。");
|
|
76
|
+
if (typeof item.b64_json === "string" && item.b64_json.length > MAX_RESPONSE_BYTES) {
|
|
77
|
+
throw new ImageFailure("单张图片的 base64 响应超过 32 MiB 上限。");
|
|
78
|
+
}
|
|
79
|
+
const image = inspectImage(item.b64_json, format);
|
|
80
|
+
if (image.width > 3840 || image.height > 3840 || image.width * image.height > 8_294_400) {
|
|
81
|
+
throw new ImageFailure("图片单边不得超过 3840、总像素不得超过 8294400。");
|
|
82
|
+
}
|
|
83
|
+
declaredMetadata(raw, image, format); declaredMetadata(item, image, format);
|
|
84
|
+
return { type: "image", data: image.data, mimeType: image.mimeType };
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
// 兼容旧的单 PNG 调用方;绝不下载响应中的远程 URL。
|
|
89
|
+
export function parseImage(raw: unknown): string {
|
|
90
|
+
return parseImages(raw, 1, "png")[0].data;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function count(value: unknown): value is number {
|
|
94
|
+
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
export function parseUsage(raw: unknown, model: ImageModel<ImageApi>): Usage | undefined {
|
|
98
|
+
if (!record(raw) || !record(raw.usage)) return;
|
|
99
|
+
const usage = raw.usage;
|
|
100
|
+
if (!count(usage.input_tokens) || !count(usage.output_tokens)) return;
|
|
101
|
+
const total = usage.input_tokens + usage.output_tokens;
|
|
102
|
+
if (!count(total) || (usage.total_tokens !== undefined && usage.total_tokens !== total)) return;
|
|
103
|
+
const details = usage.input_tokens_details;
|
|
104
|
+
if (details !== undefined && !record(details)) return;
|
|
105
|
+
const cached = record(details) ? details.cached_tokens : undefined;
|
|
106
|
+
const write = record(details) ? details.cache_write_tokens : undefined;
|
|
107
|
+
if ((cached !== undefined && !count(cached)) || (write !== undefined && !count(write))) return;
|
|
108
|
+
const cacheWrite = write === undefined ? 0 : write as number;
|
|
109
|
+
// 与 Pi 的缓存计数约定一致:cached_tokens 包含 cache_write_tokens。
|
|
110
|
+
const allCached = cached === undefined ? cacheWrite : cached as number;
|
|
111
|
+
if (cacheWrite > allCached || allCached > usage.input_tokens) return;
|
|
112
|
+
const cacheRead = allCached - cacheWrite;
|
|
113
|
+
const input = usage.input_tokens - allCached, output = usage.output_tokens;
|
|
114
|
+
let rates = model.cost;
|
|
115
|
+
let threshold = -1;
|
|
116
|
+
for (const tier of model.cost.tiers ?? []) {
|
|
117
|
+
if (usage.input_tokens > tier.inputTokensAbove && tier.inputTokensAbove > threshold) {
|
|
118
|
+
rates = tier; threshold = tier.inputTokensAbove;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
const cost = {
|
|
122
|
+
input: rates.input / 1e6 * input, output: rates.output / 1e6 * output,
|
|
123
|
+
cacheRead: rates.cacheRead / 1e6 * cacheRead, cacheWrite: rates.cacheWrite / 1e6 * cacheWrite, total: 0,
|
|
124
|
+
};
|
|
125
|
+
cost.total = cost.input + cost.output + cost.cacheRead + cost.cacheWrite;
|
|
126
|
+
if (Object.values(cost).some(value => !Number.isFinite(value) || value < 0)) return;
|
|
127
|
+
return { input, output, cacheRead, cacheWrite, totalTokens: total, cost };
|
|
128
|
+
}
|
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
import type { ImageContent } from "@earendil-works/pi-ai";
|
|
2
|
+
import type { ImageFormat, ImageOperation } from "./parameters.ts";
|
|
3
|
+
import { ImageFailure, MAX_RESPONSE_BYTES, abortable, parseImages, readJson, record } from "./response.ts";
|
|
4
|
+
|
|
5
|
+
export interface ExpectedImageResponse { count: number; format: ImageFormat; stream: boolean; operation: ImageOperation }
|
|
6
|
+
// 单事件仍是单图预算;总预算覆盖每图最多三张 partial、一张 completed 及少量控制帧。
|
|
7
|
+
export const MAX_SSE_EVENT_BYTES = MAX_RESPONSE_BYTES;
|
|
8
|
+
const CONTROL_BYTES = 64 * 1024;
|
|
9
|
+
const fail = (message: string): never => { throw new ImageFailure(message); };
|
|
10
|
+
|
|
11
|
+
export async function readImageResponse(
|
|
12
|
+
response: Response, signal: AbortSignal, expected: ExpectedImageResponse, reportUsage: (raw: unknown) => void,
|
|
13
|
+
): Promise<ImageContent[]> {
|
|
14
|
+
if (!Number.isInteger(expected.count) || expected.count < 1 || expected.count > 10 ||
|
|
15
|
+
!["png", "jpeg", "webp"].includes(expected.format) || !["generate", "edit"].includes(expected.operation) ||
|
|
16
|
+
typeof expected.stream !== "boolean") return fail("图片响应解析参数无效。");
|
|
17
|
+
if (!expected.stream) {
|
|
18
|
+
const raw = await readJson(response, signal, MAX_RESPONSE_BYTES * expected.count);
|
|
19
|
+
reportUsage(raw); // 即使坏图或数量错误,也保留上游已报告的有效 usage。
|
|
20
|
+
return parseImages(raw, expected.count, expected.format);
|
|
21
|
+
}
|
|
22
|
+
const contentType = response.headers.get("content-type")?.split(";", 1)[0].trim().toLowerCase();
|
|
23
|
+
if (contentType !== "text/event-stream") return fail("图片流式响应不是 SSE。");
|
|
24
|
+
return readSse(response, signal, expected, reportUsage);
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
async function readSse(
|
|
28
|
+
response: Response, signal: AbortSignal, expected: ExpectedImageResponse, reportUsage: (raw: unknown) => void,
|
|
29
|
+
): Promise<ImageContent[]> {
|
|
30
|
+
const budget = MAX_SSE_EVENT_BYTES * 4 * expected.count + CONTROL_BYTES;
|
|
31
|
+
const length = response.headers.get("content-length");
|
|
32
|
+
if (length && /^\d+$/.test(length) && Number(length) > budget) return fail("图片流式响应超过读取预算上限。");
|
|
33
|
+
if (!response.body) return fail("图片响应为空。");
|
|
34
|
+
const reader = response.body.getReader(), decoder = new TextDecoder("utf-8", { fatal: true });
|
|
35
|
+
const finals: ImageContent[] = [], usageReports: unknown[] = [];
|
|
36
|
+
const completedIndices = new Set<number>();
|
|
37
|
+
const prefix = expected.operation === "edit" ? "image_edit" : "image_generation";
|
|
38
|
+
let usageSeen = false, partials = 0, terminated = false;
|
|
39
|
+
let totalBytes = 0, eventBytes = 0, frames = 0, complete = false;
|
|
40
|
+
let line = "", event = "", data: string[] = [], skipLf = false, touched = false;
|
|
41
|
+
let previousCrEventBytes: number | undefined;
|
|
42
|
+
|
|
43
|
+
function usage(raw: Record<string, unknown>): void {
|
|
44
|
+
if (raw.usage === undefined) return;
|
|
45
|
+
if (expected.count > 1) {
|
|
46
|
+
// 保留原始报告而非猜测逐图/累计归属;交调用方标记未汇总,不合成 usage。
|
|
47
|
+
if (usageReports.length === 10) return;
|
|
48
|
+
usageReports.push(raw.usage);
|
|
49
|
+
reportUsage({ usage_reports: [...usageReports] });
|
|
50
|
+
} else if (!usageSeen) {
|
|
51
|
+
usageSeen = true; reportUsage(raw);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
function dispatch(): void {
|
|
55
|
+
if (++frames > 1024 + 4 * expected.count) return fail("图片流式响应事件过多。");
|
|
56
|
+
if (!data.length) {
|
|
57
|
+
if (event === "error") return fail("图片流式请求返回错误事件。");
|
|
58
|
+
return;
|
|
59
|
+
}
|
|
60
|
+
if (terminated) return fail("图片流式响应在结束标记之后仍有数据。");
|
|
61
|
+
const payload = data.join("\n");
|
|
62
|
+
if (payload === "[DONE]") {
|
|
63
|
+
if (finals.length !== expected.count) return fail("图片流式响应未完成全部图片。");
|
|
64
|
+
terminated = true; return;
|
|
65
|
+
}
|
|
66
|
+
let raw: unknown;
|
|
67
|
+
try { raw = JSON.parse(payload); } catch { return fail("图片 SSE 事件不是有效 JSON。"); }
|
|
68
|
+
if (!record(raw)) return fail("图片 SSE 事件结构无效。");
|
|
69
|
+
if (event === "error" || raw.type === "error" || raw.error !== undefined) {
|
|
70
|
+
usage(raw); return fail("图片流式请求返回错误事件。");
|
|
71
|
+
}
|
|
72
|
+
if (event && raw.type !== undefined && raw.type !== event) return fail("图片 SSE 事件类型不一致。");
|
|
73
|
+
const type = event || raw.type;
|
|
74
|
+
if (type !== `${prefix}.partial_image` && type !== `${prefix}.completed`) return fail("图片 SSE 事件类型不受支持。");
|
|
75
|
+
const final = type === `${prefix}.completed`;
|
|
76
|
+
if (final) {
|
|
77
|
+
// 单图的第二个完成事件不能覆盖首次报告;多图保留超额事件的 usage(最多十项)。
|
|
78
|
+
if (expected.count === 1 && finals.length) return fail("图片 SSE 超额完成。");
|
|
79
|
+
usage(raw); // 在坏图、显式序号及数量检查前保留上游报告。
|
|
80
|
+
}
|
|
81
|
+
const index = raw.image_index;
|
|
82
|
+
if (index !== undefined && (typeof index !== "number" || !Number.isInteger(index) || index < 0 || index >= expected.count)) {
|
|
83
|
+
return fail("图片 SSE 序号无效。");
|
|
84
|
+
}
|
|
85
|
+
if (!final) {
|
|
86
|
+
if (++partials > 3 * expected.count || !Number.isInteger(raw.partial_image_index) ||
|
|
87
|
+
(raw.partial_image_index as number) < 0 || (raw.partial_image_index as number) > 2 ||
|
|
88
|
+
finals.length === expected.count || (index !== undefined && completedIndices.has(index as number))) {
|
|
89
|
+
return fail("图片 SSE 中间图事件超额或顺序无效。");
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
const image = parseImages({ ...raw, data: [{ b64_json: raw.b64_json }] }, 1, expected.format)[0];
|
|
93
|
+
if (!final) return; // 只验证并丢弃 partial;Promise 不代表实时 UI 或逐帧交付。
|
|
94
|
+
// image_index 仅是可选的上游扩展;不要求每张都有,不据此重排,也不按相同字节去重。
|
|
95
|
+
if (finals.length >= expected.count || (index !== undefined && completedIndices.has(index as number))) {
|
|
96
|
+
return fail("图片 SSE 显式序号重复或超额完成。");
|
|
97
|
+
}
|
|
98
|
+
if (typeof index === "number") completedIndices.add(index);
|
|
99
|
+
finals.push(image);
|
|
100
|
+
}
|
|
101
|
+
function acceptLine(): void {
|
|
102
|
+
if (!line) {
|
|
103
|
+
dispatch(); event = ""; data = []; eventBytes = 0; touched = false;
|
|
104
|
+
} else {
|
|
105
|
+
const colon = line.indexOf(":"), field = colon < 0 ? line : line.slice(0, colon);
|
|
106
|
+
let value = colon < 0 ? "" : line.slice(colon + 1);
|
|
107
|
+
if (value.startsWith(" ")) value = value.slice(1);
|
|
108
|
+
if (field === "event") event = value;
|
|
109
|
+
if (field === "data") data.push(value);
|
|
110
|
+
line = "";
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
function eventBudget(bytes: number): void {
|
|
114
|
+
eventBytes += bytes;
|
|
115
|
+
if (eventBytes > MAX_SSE_EVENT_BYTES) return fail("图片 SSE 单事件超过 32 MiB 上限。");
|
|
116
|
+
}
|
|
117
|
+
function text(value: string): void {
|
|
118
|
+
let start = 0;
|
|
119
|
+
// CR 立即交付整行(迟到错误前仍保留 usage);跨块 LF 计入原事件而非新事件。
|
|
120
|
+
if (skipLf && value.length) {
|
|
121
|
+
skipLf = false;
|
|
122
|
+
if (value[0] === "\n") {
|
|
123
|
+
if (previousCrEventBytes === undefined) eventBudget(1);
|
|
124
|
+
else if (previousCrEventBytes + 1 > MAX_SSE_EVENT_BYTES) return fail("图片 SSE 单事件超过 32 MiB 上限。");
|
|
125
|
+
start = 1;
|
|
126
|
+
}
|
|
127
|
+
previousCrEventBytes = undefined;
|
|
128
|
+
}
|
|
129
|
+
while (start < value.length) {
|
|
130
|
+
const cr = value.indexOf("\r", start), lf = value.indexOf("\n", start);
|
|
131
|
+
const end = cr < 0 ? lf : lf < 0 ? cr : Math.min(cr, lf);
|
|
132
|
+
const part = value.slice(start, end < 0 ? value.length : end);
|
|
133
|
+
eventBudget(Buffer.byteLength(part, "utf8") + (end < 0 ? 0 : 1));
|
|
134
|
+
touched = true; line += part;
|
|
135
|
+
if (end < 0) return;
|
|
136
|
+
start = end + 1;
|
|
137
|
+
if (value[end] === "\r") {
|
|
138
|
+
if (start === value.length) {
|
|
139
|
+
previousCrEventBytes = line ? undefined : eventBytes;
|
|
140
|
+
acceptLine(); skipLf = true; return;
|
|
141
|
+
}
|
|
142
|
+
if (value[start] === "\n") { eventBudget(1); start++; }
|
|
143
|
+
}
|
|
144
|
+
acceptLine();
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
function decodeLines(bytes: Uint8Array): void {
|
|
148
|
+
// 逐行交给严格 UTF-8 解码器:同一网络 chunk 的后续坏字节不能抹掉先前完成事件的 usage。
|
|
149
|
+
let start = 0, cr = bytes.indexOf(13), lf = bytes.indexOf(10);
|
|
150
|
+
while (start < bytes.length) {
|
|
151
|
+
const end = cr < 0 ? lf : lf < 0 ? cr : Math.min(cr, lf);
|
|
152
|
+
const next = end < 0 ? bytes.length : end + 1;
|
|
153
|
+
text(decoder.decode(bytes.subarray(start, next), { stream: true })); start = next;
|
|
154
|
+
// 两种分隔符的游标各自只前进,避免仅 LF 的多行事件反复扫描剩余整块。
|
|
155
|
+
if (end >= 0 && end === cr) cr = bytes.indexOf(13, next);
|
|
156
|
+
if (end >= 0 && end === lf) lf = bytes.indexOf(10, next);
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
try {
|
|
160
|
+
while (true) {
|
|
161
|
+
const item = await abortable(signal, () => reader.read());
|
|
162
|
+
if (item.done) { complete = true; break; }
|
|
163
|
+
const available = budget - totalBytes;
|
|
164
|
+
decodeLines(item.value.subarray(0, available));
|
|
165
|
+
totalBytes += item.value.byteLength;
|
|
166
|
+
if (totalBytes > budget) return fail("图片流式响应超过读取预算上限。");
|
|
167
|
+
}
|
|
168
|
+
text(decoder.decode());
|
|
169
|
+
if (touched || line || data.length || event) return fail("图片 SSE 事件意外截断。");
|
|
170
|
+
if (finals.length !== expected.count) return fail("图片流式响应未完成全部图片。");
|
|
171
|
+
return finals;
|
|
172
|
+
} catch (error) {
|
|
173
|
+
if (signal.aborted) throw signal.reason;
|
|
174
|
+
if (error instanceof ImageFailure) throw error;
|
|
175
|
+
throw new ImageFailure("图片流式响应读取失败或 UTF-8 无效。");
|
|
176
|
+
} finally {
|
|
177
|
+
if (!complete) void reader.cancel().catch(() => {});
|
|
178
|
+
reader.releaseLock();
|
|
179
|
+
}
|
|
180
|
+
}
|