zigmini 1.0.0-beta.2 → 1.1.0-beta.1

package/src/attachments/prepare.ts

@@ -0,0 +1,214 @@
+import { createHash } from "node:crypto";
+import type { AttachmentEncoding } from "./types.ts";
+import type { NativeDocumentBuild, DocumentReadContext, NativeTextCursor } from "./document-reader.ts";
+
+export type DocumentPreparationCode = "encoding_required" | "unsupported_encoding" | "invalid_text"
+	| "empty_file" | "processing_limit" | "invalid_document_options" | "invalid_cursor" | "native_reader_unavailable";
+export class DocumentPreparationError extends Error {
+	constructor(readonly code: DocumentPreparationCode, readonly limit?: "original_bytes" | "prepared_bytes",
+		readonly encoding?: AttachmentEncoding) { super(code); this.name = "DocumentPreparationError"; }
+}
+
+export interface TextPreparationOptions {
+	readonly format: "txt" | "md" | "html";
+	readonly encoding?: AttachmentEncoding;
+	/** Trusted build manifest identities, not permissions or model-supplied paths. */
+	readonly helperSourceSha256: string;
+	readonly helperBuildSha256: string;
+}
+export interface PreparedText {
+	readonly format: TextPreparationOptions["format"];
+	readonly encoding: AttachmentEncoding;
+	readonly originalSha256: string;
+	readonly representationSha256: string;
+	readonly representationKey: string;
+	readonly optionsSha256: string;
+	readonly helperSourceSha256: string;
+	readonly helperBuildSha256: string;
+	readonly text: string;
+	readonly bytes: number;
+	readonly scalars: number;
+	readonly sourceEscaped?: string;
+	readonly sourceEscapedSha256?: string;
+}
+export interface TextCursor {
+	readonly originalSha256: string;
+	readonly representationSha256: string;
+	readonly representationKey: string;
+	readonly offset: number;
+}
+export interface PreparedTextRead {
+	readonly text: string;
+	readonly originalSha256: string;
+	readonly representationSha256: string;
+	readonly range: { readonly start: number; readonly end: number; readonly totalScalars: number; readonly totalBytes: number };
+	readonly nextCursor?: TextCursor;
+}
+const encodings: readonly AttachmentEncoding[] = ["utf-8", "utf-16le", "utf-16be", "euc-kr", "cp949"];
+const sha = (input: string | Uint8Array): string => createHash("sha256").update(input).digest("hex");
+const hashPattern = /^[0-9a-f]{64}$/;
+const normalize = (text: string): string => text.replace(/\r\n?/g, "\n");
+
+function parseOptions(input: unknown): TextPreparationOptions {
+	if (typeof input !== "object" || input === null || Array.isArray(input)) throw new DocumentPreparationError("invalid_document_options");
+	const keys = ["format", "encoding", "helperSourceSha256", "helperBuildSha256"];
+	if (Object.keys(input).some(key => !keys.includes(key))) throw new DocumentPreparationError("invalid_document_options");
+	if (!("format" in input) || (input.format !== "txt" && input.format !== "md" && input.format !== "html")
+		|| !("helperSourceSha256" in input) || typeof input.helperSourceSha256 !== "string" || !hashPattern.test(input.helperSourceSha256)
+		|| !("helperBuildSha256" in input) || typeof input.helperBuildSha256 !== "string" || !hashPattern.test(input.helperBuildSha256)) {
+		throw new DocumentPreparationError("invalid_document_options");
+	}
+	if ("encoding" in input && input.encoding !== undefined) {
+		const encoding = encodings.find(value => value === input.encoding);
+		if (!encoding) throw new DocumentPreparationError("unsupported_encoding");
+		return { format: input.format, helperSourceSha256: input.helperSourceSha256, helperBuildSha256: input.helperBuildSha256, encoding };
+	}
+	return { format: input.format, helperSourceSha256: input.helperSourceSha256, helperBuildSha256: input.helperBuildSha256 };
+}
+
+/** This is tokenization only: there is no browser, DOM, script evaluation or resource loader. */
+function stripActive(source: string): string {
+	return new HTMLRewriter().on("script,style,template", { element(element) { element.remove(); } }).transform(source);
+}
+
+function htmlCharset(original: Uint8Array): AttachmentEncoding | undefined {
+	// Metadata syntax is ASCII. Latin-1 is used only for tokenization, never as a content decoder.
+	const syntax = stripActive(new TextDecoder("latin1").decode(original));
+	let charset: string | undefined;
+	new HTMLRewriter().on("meta", {
+		element(element) {
+			if (charset !== undefined) return;
+			const declared = element.getAttribute("charset");
+			if (declared !== null) { charset = declared.trim().toLowerCase(); return; }
+			if (element.getAttribute("http-equiv")?.toLowerCase() === "content-type") {
+				charset = /charset\s*=\s*["']?([a-z0-9-]+)/i.exec(element.getAttribute("content") ?? "")?.[1]?.toLowerCase();
+			}
+		},
+	}).transform(syntax);
+	if (charset === undefined) return undefined;
+	const encoding = encodings.find(value => value === charset);
+	if (!encoding) throw new DocumentPreparationError("unsupported_encoding");
+	return encoding;
+}
+
+export interface NativeTextPreparation {
+	readonly build: NativeDocumentBuild;
+	readonly context: DocumentReadContext;
+}
+async function decode(original: Uint8Array, options: TextPreparationOptions,
+	native?: NativeTextPreparation): Promise<{ text: string; encoding: AttachmentEncoding }> {
+	let bom: AttachmentEncoding | undefined, skip = 0;
+	if (original[0] === 0xef && original[1] === 0xbb && original[2] === 0xbf) { bom = "utf-8"; skip = 3; }
+	else if (original[0] === 0xff && original[1] === 0xfe) { bom = "utf-16le"; skip = 2; }
+	else if (original[0] === 0xfe && original[1] === 0xff) { bom = "utf-16be"; skip = 2; }
+	if (bom && options.encoding && bom !== options.encoding) throw new DocumentPreparationError("unsupported_encoding");
+	const declared = !bom && !options.encoding && options.format === "html" ? htmlCharset(original) : undefined;
+	const encoding = options.encoding ?? bom ?? declared ?? "utf-8";
+	switch (encoding) {
+		case "euc-kr": case "cp949": {
+			// Do not substitute ICU/WHATWG EUC-KR (Windows-949) for strict Foundation decoding.
+			if (!native || native.build.sourceSha256 !== options.helperSourceSha256
+				|| native.build.buildSha256 !== options.helperBuildSha256) {
+				throw new DocumentPreparationError("native_reader_unavailable", undefined, encoding);
+			}
+			const { readNativeDocument } = await import("./document-reader.ts");
+			const parts: string[] = [];
+			let cursor: NativeTextCursor | undefined, offset = 0, representation: string | undefined, total = 0;
+			do {
+				const result = await readNativeDocument(native.build, { original, operation: "document.text",
+					options: { format: "txt", encoding, ...(cursor ? { cursor } : {}) } }, native.context);
+				const part = result.content[0], range = result.textRange;
+				if (part?.type !== "text" || !range || range.start !== offset || result.originalSha256 !== sha(original)
+					|| (representation !== undefined && result.representationSha256 !== representation)) {
+					throw new DocumentPreparationError("invalid_text");
+				}
+				representation = result.representationSha256; total = range.totalBytes; offset = range.end;
+				parts.push(part.text); cursor = result.nextCursor;
+			} while (cursor);
+			const text = parts.join("");
+			if (sha(text) !== representation || Buffer.byteLength(text) !== total) throw new DocumentPreparationError("invalid_text");
+			return { text, encoding };
+		}
+		case "utf-8": case "utf-16le": case "utf-16be": {
+			let text: string;
+			try { text = new TextDecoder(encoding, { fatal: true, ignoreBOM: true }).decode(original.subarray(skip)); }
+			catch (error) {
+				if (!(error instanceof TypeError)) throw error;
+				throw new DocumentPreparationError(options.encoding || bom || declared ? "unsupported_encoding" : "encoding_required");
+			}
+			if (text.includes("\0")) throw new DocumentPreparationError("invalid_text");
+			return { text: normalize(text), encoding };
+		}
+		default: {
+			const exhaustive: never = encoding;
+			return exhaustive;
+		}
+	}
+}
+
+function inertText(source: string): string {
+	const stripped = stripActive(source);
+	const separated = new HTMLRewriter().on("p,div,br,li,tr,h1,h2,h3,h4,h5,h6,pre,blockquote,section,article", {
+		element(element) { element.before("\n"); if (element.canHaveContent) element.after("\n"); },
+	}).transform(stripped);
+	const chunks: string[] = [];
+	new HTMLRewriter().onDocument({ text(chunk) { chunks.push(chunk.text); } }).transform(separated);
+	// Decode complete character-reference tokens with Bun's existing entity table, not the surrounding
+	// Markdown/HTML. Passing only a single &name; token prevents formatting, links and code interpretation.
+	return normalize(chunks.join("").replace(/&(?:#[xX][0-9a-fA-F]+|#[0-9]+|[a-zA-Z][a-zA-Z0-9]*);/g,
+		token => Bun.markdown.render(token, { text: text => text })));
+}
+
+/** Pure byte-to-representation work. Callers own source grants, revision CAS and publication. */
+export async function prepareText(input: Uint8Array, rawOptions: unknown, native?: NativeTextPreparation): Promise<PreparedText> {
+	const options = parseOptions(rawOptions);
+	if (input.byteLength === 0) throw new DocumentPreparationError("empty_file");
+	if (input.byteLength > 10 * 1024 * 1024) throw new DocumentPreparationError("processing_limit", "original_bytes");
+	const original = Uint8Array.from(input);
+	const decoded = await decode(original, options, native);
+	const sourceEscaped = options.format === "html" ? Bun.escapeHTML(decoded.text) : undefined;
+	// Check expansion before building the additional text representation.
+	if (sourceEscaped !== undefined && Buffer.byteLength(sourceEscaped) > 20 * 1024 * 1024) {
+		throw new DocumentPreparationError("processing_limit", "prepared_bytes");
+	}
+	const text = options.format === "html" ? inertText(decoded.text) : decoded.text;
+	if (text.includes("\0")) throw new DocumentPreparationError("invalid_text");
+	const bytes = Buffer.byteLength(text);
+	if (bytes + Buffer.byteLength(sourceEscaped ?? "") > 20 * 1024 * 1024) throw new DocumentPreparationError("processing_limit", "prepared_bytes");
+	let scalars = 0;
+	for (const _scalar of text) scalars++;
+	const originalSha256 = sha(original);
+	const optionsSha256 = sha(JSON.stringify({ version: 1, format: options.format, encoding: options.encoding ?? null,
+		decodedEncoding: decoded.encoding, newline: "lf", html: "inert-v1" }));
+	return Object.freeze({
+		format: options.format, encoding: decoded.encoding, originalSha256, representationSha256: sha(text),
+		representationKey: sha(JSON.stringify([originalSha256, options.helperSourceSha256, options.helperBuildSha256, optionsSha256])),
+		optionsSha256, helperSourceSha256: options.helperSourceSha256, helperBuildSha256: options.helperBuildSha256,
+		text, bytes, scalars, ...(sourceEscaped === undefined ? {} : { sourceEscaped, sourceEscapedSha256: sha(sourceEscaped) }),
+	});
+}
+
+/** Internal scalar continuation, not a grant or a public opaque tool cursor. */
+export function readPreparedText(document: PreparedText, options: { readonly cursor?: TextCursor; readonly lineLimit?: number }): PreparedTextRead {
+	const lineLimit = options.lineLimit ?? 2000, cursor = options.cursor;
+	if (!Number.isSafeInteger(lineLimit) || lineLimit < 1 || lineLimit > 2000) throw new DocumentPreparationError("invalid_document_options");
+	if (cursor && (cursor.originalSha256 !== document.originalSha256 || cursor.representationSha256 !== document.representationSha256
+		|| cursor.representationKey !== document.representationKey || !Number.isSafeInteger(cursor.offset)
+		|| cursor.offset <= 0 || cursor.offset >= document.scalars)) throw new DocumentPreparationError("invalid_cursor");
+	const start = cursor?.offset ?? 0;
+	let position = 0, end = start, bytes = 0, lines = 0;
+	const text: string[] = [];
+	for (const scalar of document.text) {
+		if (position++ < start) continue;
+		const size = Buffer.byteLength(scalar);
+		if (bytes + size > 48 * 1024 || lines >= lineLimit) break;
+		text.push(scalar); bytes += size; end++;
+		if (scalar === "\n") lines++;
+	}
+	return {
+		text: text.join(""), originalSha256: document.originalSha256, representationSha256: document.representationSha256,
+		range: { start, end, totalScalars: document.scalars, totalBytes: document.bytes },
+		...(end < document.scalars ? { nextCursor: { originalSha256: document.originalSha256,
+			representationSha256: document.representationSha256, representationKey: document.representationKey, offset: end } } : {}),
+	};
+}

package/src/attachments/selector-port.ts

@@ -0,0 +1,28 @@
+import { createHash } from "node:crypto";
+import { lstatSync, readFileSync } from "node:fs";
+import { HostPicker, NativeSelector, sealOriginal, type PickerScope, type SelectedSource } from "./host-selector.ts";
+import type { SelectorPort } from "./service.ts";
+
+/** The packaged selector, or null when the file is absent, not a plain executable, or differs from the manifest hash. */
+export function verifiedSelectorPort(options: Parameters<typeof hostSelectorPort>[0] & { readonly expectedSha256: string }): SelectorPort | null {
+	let stat;
+	try { stat = lstatSync(options.executable); } catch { return null; }
+	if (!stat.isFile() || (stat.mode & 0o111) === 0) return null;
+	if (createHash("sha256").update(readFileSync(options.executable)).digest("hex") !== options.expectedSha256) return null;
+	return hostSelectorPort(options);
+}
+
+/** Loaded only on Darwin: host-selector binds libSystem when it is imported. */
+export function hostSelectorPort(options: {
+	readonly executable: string; readonly runtimeHome: string; readonly generation: string; readonly current: (scope: PickerScope) => boolean;
+}): SelectorPort {
+	const native = new NativeSelector(options.executable);
+	const picker = new HostPicker({ native, runtimeHome: options.runtimeHome, generation: options.generation, current: options.current });
+	return {
+		open: scope => picker.create(scope),
+		list: (scope, id, input) => picker.list(scope, id, input),
+		resolve: (scope, id, token) => picker.resolveSelection(scope, id, token, true),
+		scopeOf: (principal, id) => picker.scopeOf(principal, id),
+		seal: (id, source: SelectedSource, signal, beforePublish) => sealOriginal(options.runtimeHome, id, source, native, signal, beforePublish),
+	};
+}