zigmini 1.0.0-beta.3 → 1.1.0-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.ko.md +29 -2
- package/README.md +29 -2
- package/assets/agent/gateway-request-rules.md +9 -2
- package/assets/agent/gateway-rules.md +13 -6
- package/assets/android/zigmini.apk +0 -0
- package/assets/android/zigmini.apk.json +3 -3
- package/assets/build.json +1 -1
- package/assets/document-reader/document-reader +0 -0
- package/assets/document-reader/host-selector +0 -0
- package/assets/document-reader/manifest.json +27 -0
- package/docs/manual.ko.md +114 -10
- package/docs/manual.md +117 -11
- package/host/document-reader/DocumentRead.swift +370 -0
- package/host/document-reader/HostSelector.swift +192 -0
- package/host/document-reader/Protocol.swift +647 -0
- package/host/document-reader/main.swift +22 -0
- package/host/omo-bridge/agent-aside-tool.ts +119 -0
- package/host/omo-bridge/agent-aside.ts +522 -0
- package/host/omo-bridge/agent-collaboration-files.ts +282 -0
- package/host/omo-bridge/agent-collaboration.ts +385 -0
- package/host/omo-bridge/agent-control.ts +400 -28
- package/host/omo-bridge/agent-execution.sb +42 -0
- package/host/omo-bridge/agent-execution.ts +158 -0
- package/host/omo-bridge/agent-tool-adapters.ts +388 -0
- package/host/omo-bridge/agent-tool-permissions.ts +96 -0
- package/host/omo-bridge/aside-tabs.ts +41 -0
- package/host/omo-bridge/attachment-read.ts +100 -0
- package/host/omo-bridge/bridge.ts +10 -0
- package/host/omo-bridge/managed-call.ts +46 -0
- package/host/omo-bridge/task-control.ts +39 -5
- package/host/omo-check/patches/attachments-293/README.md +138 -0
- package/host/omo-check/patches/attachments-293/document-content.js +234 -0
- package/host/omo-check/patches/attachments-293/document-content.types.txt +91 -0
- package/host/omo-check/patches/attachments-293/omo-5.1.51-documents.patch +831 -0
- package/host/omo-check/patches/attachments-293/omo-5.1.51-gateway-port.manifest.json +471 -0
- package/host/omo-check/patches/omo-5.1.51-agent-tool-authority.patch +471 -0
- package/host/omo-check/patches/omo-5.1.51-gateway-port.manifest.json +28 -6
- package/host/omo-check/patches/omo-5.1.51-memory-receipts.patch +5 -0
- package/package.json +5 -3
- package/src/agent/api.ts +345 -20
- package/src/agent/artifacts.ts +196 -0
- package/src/agent/aside-target-routes.ts +369 -0
- package/src/agent/aside-targets.ts +215 -0
- package/src/agent/collaboration-admission.ts +47 -0
- package/src/agent/collaboration-read.ts +117 -0
- package/src/agent/collaboration-session.ts +196 -0
- package/src/agent/collaboration-types.ts +258 -0
- package/src/agent/collaboration.ts +399 -0
- package/src/agent/definition.ts +65 -3
- package/src/agent/fields.ts +9 -3
- package/src/agent/file-core.ts +325 -0
- package/src/agent/files.ts +87 -8
- package/src/agent/list.ts +3 -3
- package/src/agent/managed-call.ts +144 -0
- package/src/agent/tool-catalog.ts +184 -0
- package/src/agent/tool-delivery.ts +359 -0
- package/src/agent/tool-permission-types.ts +81 -0
- package/src/agent/tool-permissions.ts +1366 -0
- package/src/attachments/api.ts +161 -0
- package/src/attachments/document-reader.ts +325 -0
- package/src/attachments/host-selector.ts +410 -0
- package/src/attachments/managed-binding.ts +83 -0
- package/src/attachments/prepare.ts +214 -0
- package/src/attachments/selector-port.ts +28 -0
- package/src/attachments/service.ts +729 -0
- package/src/attachments/store.ts +698 -0
- package/src/attachments/types.ts +123 -0
- package/src/cli/doctor.ts +30 -10
- package/src/decisions.ts +109 -10
- package/src/engine/apply-patch.ts +21 -0
- package/src/engine/compat.json +2 -1
- package/src/engine/install.ts +45 -4
- package/src/engine/sandbox-cell.ts +2 -0
- package/src/gc.ts +33 -225
- package/src/housekeeping/api.ts +163 -0
- package/src/housekeeping/audit.ts +61 -0
- package/src/housekeeping/collectors/agent-collaboration.ts +33 -0
- package/src/housekeeping/collectors/agent-drafts.ts +43 -0
- package/src/housekeeping/collectors/agent-file-trash.ts +36 -0
- package/src/housekeeping/collectors/agent-trash.ts +28 -0
- package/src/housekeeping/collectors/attachment-cleanup.ts +35 -0
- package/src/housekeeping/collectors/db-backup.ts +25 -0
- package/src/housekeeping/collectors/docker-build-cache.ts +29 -0
- package/src/housekeeping/collectors/docker-image.ts +135 -0
- package/src/housekeeping/collectors/omo-version.ts +32 -0
- package/src/housekeeping/collectors/orphan-test-process.ts +205 -0
- package/src/housekeeping/collectors/reconcile-work.ts +55 -0
- package/src/housekeeping/collectors/rpc-daemon.ts +39 -0
- package/src/housekeeping/collectors/shared.ts +149 -0
- package/src/housekeeping/collectors/temp-dir.ts +28 -0
- package/src/housekeeping/collectors/update-file.ts +25 -0
- package/src/housekeeping/collectors/worktree.ts +193 -0
- package/src/housekeeping/engine.ts +125 -0
- package/src/housekeeping/reconcile/work.ts +130 -0
- package/src/housekeeping/registry.ts +63 -0
- package/src/housekeeping/trash.ts +59 -0
- package/src/housekeeping/types.ts +31 -0
- package/src/hub.ts +1095 -41
- package/src/main.ts +228 -30
- package/src/message-references.ts +14 -5
- package/src/profile-prompt.ts +13 -1
- package/src/protocol.ts +41 -6
- package/src/security-allow.ts +79 -2
- package/src/security-check.ts +1 -1
- package/src/store.ts +1946 -23
- package/src/test-crash.ts +20 -0
- package/src/work-watch.ts +55 -7
- package/src/work.ts +192 -3
- package/web/DESIGN.md +33 -0
- package/web/agents.d.ts +189 -12
- package/web/agents.js +898 -39
- package/web/app.js +2634 -210
- package/web/attachments.d.ts +91 -0
- package/web/attachments.js +422 -0
- package/web/collaboration.d.ts +87 -0
- package/web/collaboration.js +290 -0
- package/web/decisions.d.ts +39 -2
- package/web/decisions.js +354 -2
- package/web/gc.d.ts +26 -5
- package/web/gc.js +113 -13
- package/web/i18n/agents.d.ts +4 -0
- package/web/i18n/agents.js +200 -8
- package/web/i18n/attachments.js +78 -0
- package/web/i18n/decisions.d.ts +4 -0
- package/web/i18n/decisions.js +68 -0
- package/web/i18n/gc.d.ts +4 -0
- package/web/i18n/gc.js +184 -2
- package/web/i18n/main.js +80 -8
- package/web/i18n/tour.d.ts +4 -0
- package/web/i18n/tour.js +37 -0
- package/web/index.html +20 -3
- package/web/profile.d.ts +2 -2
- package/web/profile.js +11 -6
- package/web/style.css +342 -21
- package/web/tooltip.d.ts +2 -0
- package/web/tooltip.js +12 -0
- package/web/tour.d.ts +57 -0
- package/web/tour.js +450 -0
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
import { createHash, randomBytes } from "node:crypto";
|
|
2
|
+
import { ManagedCallError, verifyManagedCall, type VerifiedManagedCall } from "../agent/managed-call.ts";
|
|
3
|
+
import { maintenanceAuth } from "../maintenance.ts";
|
|
4
|
+
import type { DeviceRecord, Store } from "../store.ts";
|
|
5
|
+
import { AttachmentApiError, deviceRevision, type AttachmentService } from "./service.ts";
|
|
6
|
+
import { AttachmentStoreError } from "./types.ts";
|
|
7
|
+
|
|
8
|
+
const COOKIE = "zigmini_host_files";
|
|
9
|
+
const CUSTOM = "x-zigmini-host-files";
|
|
10
|
+
const MANAGED_HEADER = "x-zigmini-managed-call";
|
|
11
|
+
const READ_ROUTE = "/internal/attachment-read";
|
|
12
|
+
const DOCUMENT_ROUTE = "/internal/attachment-document";
|
|
13
|
+
const MAX_BODY = 16 * 1024;
|
|
14
|
+
type Kind = "bootstrap" | "mutation" | "status" | "download";
|
|
15
|
+
export interface RequestContext { readonly trusted: boolean; readonly device: DeviceRecord | null }
|
|
16
|
+
interface Deps { readonly store: Store; readonly service: AttachmentService; readonly webOrigins: readonly string[]; readonly omoToken: string }
|
|
17
|
+
const refuse = (error: string, status: number, detail?: Record<string, unknown>) => Response.json({ error, ...detail }, { status });
|
|
18
|
+
const sha = (value: string) => createHash("sha256").update(value).digest("hex");
|
|
19
|
+
|
|
20
|
+
function cookieValue(headers: Headers): string | null {
|
|
21
|
+
for (const part of (headers.get("cookie") ?? "").split(";")) {
|
|
22
|
+
const [name, value] = part.trim().split("=");
|
|
23
|
+
if (name === COOKIE && value) return value;
|
|
24
|
+
}
|
|
25
|
+
return null;
|
|
26
|
+
}
|
|
27
|
+
function contentDisposition(name: string): string {
|
|
28
|
+
const ascii = name.replace(/[^\x20-\x7e]|["\\;]/g, "_");
|
|
29
|
+
return `attachment; filename="${ascii}"; filename*=UTF-8''${encodeURIComponent(name).replace(/['()*]/g, c => `%${c.charCodeAt(0).toString(16).toUpperCase()}`)}`;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** Snapshot field for a client that declared the feature; absent means the client shows the unsupported state. */
|
|
33
|
+
export function snapshotCapabilities(hello: { readonly capabilities?: { readonly hostFileAttachments?: unknown } }, available: boolean): { capabilities?: { hostFileAttachments: 1 } } {
|
|
34
|
+
return hello.capabilities?.hostFileAttachments === 1 && available ? { capabilities: { hostFileAttachments: 1 } } : {};
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export class AttachmentApi {
|
|
38
|
+
private readonly hosts: ReadonlySet<string>;
|
|
39
|
+
constructor(private readonly d: Deps) { this.hosts = new Set(d.webOrigins.map(origin => new URL(origin).host)); }
|
|
40
|
+
|
|
41
|
+
private originAllowed(headers: Headers, kind: Kind): boolean {
|
|
42
|
+
const origin = headers.get("origin");
|
|
43
|
+
if (origin !== null) return this.d.webOrigins.includes(origin);
|
|
44
|
+
if (kind === "bootstrap" || kind === "mutation") return false;
|
|
45
|
+
const referer = headers.get("referer");
|
|
46
|
+
return referer !== null && URL.canParse(referer) && this.d.webOrigins.includes(new URL(referer).origin);
|
|
47
|
+
}
|
|
48
|
+
private registered(cookie: string | null): boolean { return cookie !== null && this.d.store.attachments.selectorKnown(sha(cookie)); }
|
|
49
|
+
private mint(): { cookie: string; header: string } {
|
|
50
|
+
const cookie = randomBytes(32).toString("hex");
|
|
51
|
+
this.d.store.attachments.registerSelector(sha(cookie));
|
|
52
|
+
return { cookie, header: `${COOKIE}=${cookie}; Path=/; HttpOnly; SameSite=Strict` };
|
|
53
|
+
}
|
|
54
|
+
/** Web sockets learn the selector at upgrade; a fresh context gets its cookie on the same response. */
|
|
55
|
+
webSocketPrincipal(headers: Headers): { readonly principal: string | null; readonly setCookie?: string } {
|
|
56
|
+
if (!this.d.webOrigins.includes(headers.get("origin") ?? "") || !this.hosts.has(headers.get("host") ?? "")) return { principal: null };
|
|
57
|
+
const present = cookieValue(headers);
|
|
58
|
+
if (this.registered(present) && present) return { principal: `web:${sha(present)}` };
|
|
59
|
+
const minted = this.mint();
|
|
60
|
+
return { principal: `web:${sha(minted.cookie)}`, setCookie: minted.header };
|
|
61
|
+
}
|
|
62
|
+
devicePrincipal(device: DeviceRecord): string { return `device:${device.id}:${deviceRevision(device)}`; }
|
|
63
|
+
|
|
64
|
+
private authorize(req: Request, ctx: RequestContext, kind: Kind): { principal: string; setCookie?: string } | Response {
|
|
65
|
+
if (ctx.device) {
|
|
66
|
+
if (ctx.trusted) return refuse("forbidden", 403);
|
|
67
|
+
if (ctx.device.status !== "approved") return refuse("device_not_approved", 401);
|
|
68
|
+
return { principal: this.devicePrincipal(ctx.device) };
|
|
69
|
+
}
|
|
70
|
+
if (!ctx.trusted) return refuse("unauthorized", 401);
|
|
71
|
+
if (!this.hosts.has(req.headers.get("host") ?? "")) return refuse("forbidden_host", 403);
|
|
72
|
+
if (!this.originAllowed(req.headers, kind)) return refuse("forbidden_origin", 403);
|
|
73
|
+
if (kind !== "download" && req.headers.get(CUSTOM) !== "1") return refuse("custom_header_required", 403);
|
|
74
|
+
const present = cookieValue(req.headers);
|
|
75
|
+
if (this.registered(present) && present) return { principal: `web:${sha(present)}` };
|
|
76
|
+
if (kind !== "bootstrap") return refuse("selector_cookie_required", 403);
|
|
77
|
+
const minted = this.mint();
|
|
78
|
+
return { principal: `web:${sha(minted.cookie)}`, setCookie: minted.header };
|
|
79
|
+
}
|
|
80
|
+
private async json(req: Request): Promise<unknown> {
|
|
81
|
+
if (Number(req.headers.get("content-length") ?? 0) > MAX_BODY) throw new AttachmentApiError("invalid_input", 413);
|
|
82
|
+
const text = await req.text();
|
|
83
|
+
if (Buffer.byteLength(text) > MAX_BODY) throw new AttachmentApiError("invalid_input", 413);
|
|
84
|
+
try { return JSON.parse(text); }
|
|
85
|
+
catch { throw new AttachmentApiError("invalid_input", 400); }
|
|
86
|
+
}
|
|
87
|
+
private static failure(error: unknown): Response {
|
|
88
|
+
if (error instanceof AttachmentApiError) return refuse(error.code, error.status, error.detail);
|
|
89
|
+
if (error instanceof AttachmentStoreError) return refuse(error.code, error.status);
|
|
90
|
+
if (error instanceof ManagedCallError) return refuse(error.code, 403);
|
|
91
|
+
if (error instanceof Error && "code" in error && typeof error.code === "string") {
|
|
92
|
+
const status = error.code === "host_access_denied" ? 403 : error.code === "processing_limit" ? 413 : error.code === "native_reader_unavailable" ? 503 :
|
|
93
|
+
/^(invalid_|picker_|operation_conflict|file_changed|entry_disabled|seal_conflict)/.test(error.code) ? 409 : 500;
|
|
94
|
+
return refuse(error.code.toLowerCase(), status);
|
|
95
|
+
}
|
|
96
|
+
throw error;
|
|
97
|
+
}
|
|
98
|
+
/** Returns null for paths outside this feature; the caller has already applied the listener gate. */
|
|
99
|
+
async handle(req: Request, url: URL, ctx: RequestContext): Promise<Response | null> {
|
|
100
|
+
const path = url.pathname;
|
|
101
|
+
if (path !== "/api/host-file-pickers" && !path.startsWith("/api/host-file-pickers/") && !path.startsWith("/api/attachments/")) return null;
|
|
102
|
+
try {
|
|
103
|
+
const picker = /^\/api\/host-file-pickers(?:\/([^/]+)\/(list|select))?$/.exec(path);
|
|
104
|
+
if (picker) {
|
|
105
|
+
if (req.method !== "POST") return refuse("method_not_allowed", 405);
|
|
106
|
+
const auth = this.authorize(req, ctx, picker[1] ? "mutation" : "bootstrap");
|
|
107
|
+
if (auth instanceof Response) return auth;
|
|
108
|
+
const body = await this.json(req);
|
|
109
|
+
const { service } = this.d, id = picker[1] ?? "";
|
|
110
|
+
const response = !picker[1] ? Response.json(await service.openPicker(auth.principal, body))
|
|
111
|
+
: picker[2] === "list" ? Response.json(await service.listPicker(auth.principal, id, body))
|
|
112
|
+
: Response.json(service.select(auth.principal, id, body), { status: 202 });
|
|
113
|
+
if (auth.setCookie) response.headers.set("set-cookie", auth.setCookie);
|
|
114
|
+
return response;
|
|
115
|
+
}
|
|
116
|
+
const attachment = /^\/api\/attachments\/([^/]+)(?:\/(prepare|discard|revoke|download|cleanup))?$/.exec(path);
|
|
117
|
+
if (!attachment) return refuse("not_found", 404);
|
|
118
|
+
const id = attachment[1] ?? "", action = attachment[2];
|
|
119
|
+
if (action === "download" || !action) {
|
|
120
|
+
if (req.method !== "GET") return refuse("method_not_allowed", 405);
|
|
121
|
+
const auth = this.authorize(req, ctx, action ? "download" : "status");
|
|
122
|
+
if (auth instanceof Response) return auth;
|
|
123
|
+
if (!action) return Response.json(this.d.service.status(auth.principal, id));
|
|
124
|
+
const file = this.d.service.download(auth.principal, id);
|
|
125
|
+
return new Response(Uint8Array.from(file.bytes), { headers: { "content-type": "application/octet-stream", "content-disposition": contentDisposition(file.name),
|
|
126
|
+
"x-content-type-options": "nosniff", "content-security-policy": "default-src 'none'; sandbox", "cache-control": "no-store" } });
|
|
127
|
+
}
|
|
128
|
+
if (req.method !== "POST") return refuse("method_not_allowed", 405);
|
|
129
|
+
const auth = this.authorize(req, ctx, "mutation");
|
|
130
|
+
if (auth instanceof Response) return auth;
|
|
131
|
+
const body = await this.json(req), { service } = this.d;
|
|
132
|
+
if (action === "prepare") return Response.json(service.prepare(auth.principal, id, body), { status: 202 });
|
|
133
|
+
if (action === "cleanup") return Response.json(await service.cleanup(auth.principal, id, body));
|
|
134
|
+
return Response.json(action === "discard" ? service.discard(auth.principal, id, body) : service.revoke(auth.principal, id, body));
|
|
135
|
+
} catch (error) { return AttachmentApi.failure(error); }
|
|
136
|
+
}
|
|
137
|
+
/** Listener and bearer come first, then the signed session identity. Neither alone yields content. */
|
|
138
|
+
internalRead(req: Request, trusted: boolean): Promise<Response> {
|
|
139
|
+
return this.managed(req, trusted, READ_ROUTE, "attachment.read", (call, params) => this.d.service.managedRead(call, params, req.signal));
|
|
140
|
+
}
|
|
141
|
+
internalDocument(req: Request, trusted: boolean): Promise<Response> {
|
|
142
|
+
return this.managed(req, trusted, DOCUMENT_ROUTE, "attachment.document", (call, params) => this.d.service.managedDocument(call, params, req.signal));
|
|
143
|
+
}
|
|
144
|
+
private async managed(req: Request, trusted: boolean, route: string, operation: string, run: (call: VerifiedManagedCall, params: unknown) => Promise<Record<string, unknown>>): Promise<Response> {
|
|
145
|
+
const denied = maintenanceAuth(req, trusted, this.d.omoToken);
|
|
146
|
+
if (denied) return denied;
|
|
147
|
+
if (req.method !== "POST") return refuse("method_not_allowed", 405);
|
|
148
|
+
try {
|
|
149
|
+
const body = new Uint8Array(await req.arrayBuffer());
|
|
150
|
+
if (body.byteLength > MAX_BODY) return refuse("invalid_read_input", 413);
|
|
151
|
+
let proof: unknown;
|
|
152
|
+
try { proof = JSON.parse(Buffer.from(req.headers.get(MANAGED_HEADER) ?? "", "base64").toString("utf8")); }
|
|
153
|
+
catch { throw new ManagedCallError("invalid_proof"); }
|
|
154
|
+
const call = verifyManagedCall({ method: "POST", route, operation, body, proof }, this.d.service.bindings.resolver);
|
|
155
|
+
let params: unknown;
|
|
156
|
+
try { params = JSON.parse(new TextDecoder().decode(body)); }
|
|
157
|
+
catch { return refuse("invalid_read_input", 400); }
|
|
158
|
+
return Response.json(await run(call, params));
|
|
159
|
+
} catch (error) { return AttachmentApi.failure(error); }
|
|
160
|
+
}
|
|
161
|
+
}
|
|
@@ -0,0 +1,325 @@
|
|
|
1
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
2
|
+
import { open, lstat } from "node:fs/promises";
|
|
3
|
+
import { constants } from "node:fs";
|
|
4
|
+
import { dlopen, FFIType } from "bun:ffi";
|
|
5
|
+
import type { AttachmentEncoding, AttachmentFormat } from "./types.ts";
|
|
6
|
+
|
|
7
|
+
// SIZE_OK: the fixed T4 write boundary assigns this sole module the native transport,
|
|
8
|
+
// strict response codec and owned child supervisor. No new shared files or auth layer.
|
|
9
|
+
export type NativeDocumentErrorCode = "native_reader_unavailable" | "invalid_document_options" | "invalid_native_output"
|
|
10
|
+
| "original_mismatch" | "invalid_cursor" | "cancelled" | "stale_preparation" | "password_required" | "invalid_pdf"
|
|
11
|
+
| "encoding_required" | "unsupported_encoding" | "ocr_failed" | "processing_limit";
|
|
12
|
+
export class NativeDocumentError extends Error {
|
|
13
|
+
constructor(readonly code: NativeDocumentErrorCode, readonly limit?: "original_bytes" | "output_bytes" | "rss" | "page_time" | "total_time") {
|
|
14
|
+
super(code); this.name = "NativeDocumentError";
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
export interface NativeTextCursor {
|
|
18
|
+
readonly originalSha256: string;
|
|
19
|
+
readonly representationSha256: string;
|
|
20
|
+
readonly offset: number;
|
|
21
|
+
}
|
|
22
|
+
export interface NativeDocumentOptions {
|
|
23
|
+
readonly format: AttachmentFormat;
|
|
24
|
+
readonly encoding?: AttachmentEncoding;
|
|
25
|
+
readonly firstPage?: number;
|
|
26
|
+
readonly pageLimit?: number;
|
|
27
|
+
readonly lineLimit?: number;
|
|
28
|
+
readonly cursor?: NativeTextCursor;
|
|
29
|
+
}
|
|
30
|
+
export type DocumentOperation = "document.inspect" | "document.text" | "document.pages" | "document.ocr";
|
|
31
|
+
export interface NativeDocumentBuild {
|
|
32
|
+
/** Operator-owned prebuilt helper manifest. Never model/client inputs. */
|
|
33
|
+
readonly executable: string;
|
|
34
|
+
readonly sourceSha256: string;
|
|
35
|
+
readonly buildSha256: string;
|
|
36
|
+
readonly privateHome: string;
|
|
37
|
+
}
|
|
38
|
+
export interface DocumentReadContext {
|
|
39
|
+
readonly signal: AbortSignal;
|
|
40
|
+
/** Aborts when the whole-document time limit that spans several helpers has passed. */
|
|
41
|
+
readonly deadline?: AbortSignal;
|
|
42
|
+
/** Preparation revision/job-generation check, not file-read authority. No publication occurs here. */
|
|
43
|
+
readonly isCurrent: () => boolean;
|
|
44
|
+
/** Observation only; cancellation tests subscribe to actual native barriers before requesting abort. */
|
|
45
|
+
readonly onProgress?: (event: { readonly stage: "isolated" | "page-start" | "page-end" | "ocr"; readonly page: number }) => void;
|
|
46
|
+
}
|
|
47
|
+
export interface NativeDocumentResult {
|
|
48
|
+
readonly originalSha256: string;
|
|
49
|
+
readonly representationSha256: string;
|
|
50
|
+
readonly representationKey: string;
|
|
51
|
+
readonly optionsSha256: string;
|
|
52
|
+
readonly helperSourceSha256: string;
|
|
53
|
+
readonly helperBuildSha256: string;
|
|
54
|
+
readonly ocrRevision: number;
|
|
55
|
+
readonly coverage: { readonly totalPages: number; readonly pages: readonly {
|
|
56
|
+
readonly page: number; readonly representation: "native-text" | "rendered-image" | "ocr-text" | "verified-blank" }[] };
|
|
57
|
+
readonly content: readonly ({ readonly type: "text"; readonly text: string }
|
|
58
|
+
| { readonly type: "image"; readonly data: string; readonly mimeType: "image/jpeg" | "image/png" })[];
|
|
59
|
+
readonly textRange?: { readonly start: number; readonly end: number; readonly totalScalars: number; readonly totalBytes: number };
|
|
60
|
+
readonly nextCursor?: NativeTextCursor;
|
|
61
|
+
readonly resources: { readonly peakRss: number; readonly rssSamples: number; readonly reaped: true };
|
|
62
|
+
}
|
|
63
|
+
const hash = (value: string | Uint8Array): string => createHash("sha256").update(value).digest("hex");
|
|
64
|
+
const shaPattern = /^[0-9a-f]{64}$/;
|
|
65
|
+
function fail(): never { throw new NativeDocumentError("invalid_native_output"); }
|
|
66
|
+
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
67
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
68
|
+
}
|
|
69
|
+
const record = (value: unknown): Record<string, unknown> => {
|
|
70
|
+
if (!isRecord(value)) return fail();
|
|
71
|
+
return value;
|
|
72
|
+
};
|
|
73
|
+
const exact = (value: Record<string, unknown>, keys: readonly string[]): void => {
|
|
74
|
+
if (Object.keys(value).some(key => !keys.includes(key))) fail();
|
|
75
|
+
};
|
|
76
|
+
const integer = (value: unknown, min: number, max: number): number => {
|
|
77
|
+
if (typeof value !== "number" || !Number.isSafeInteger(value) || value < min || value > max) return fail();
|
|
78
|
+
return value;
|
|
79
|
+
};
|
|
80
|
+
const digest = (value: unknown): string => {
|
|
81
|
+
if (typeof value !== "string" || !shaPattern.test(value)) return fail();
|
|
82
|
+
return value;
|
|
83
|
+
};
|
|
84
|
+
|
|
85
|
+
function options(input: NativeDocumentOptions): NativeDocumentOptions {
|
|
86
|
+
const raw = record(input);
|
|
87
|
+
exact(raw, ["format", "encoding", "firstPage", "pageLimit", "lineLimit", "cursor"]);
|
|
88
|
+
if (!["pdf", "html", "txt", "md"].includes(input.format)) throw new NativeDocumentError("invalid_document_options");
|
|
89
|
+
if (input.encoding !== undefined && !["utf-8", "utf-16le", "utf-16be", "cp949", "euc-kr"].includes(input.encoding)) {
|
|
90
|
+
throw new NativeDocumentError("invalid_document_options");
|
|
91
|
+
}
|
|
92
|
+
if (input.firstPage !== undefined) integer(input.firstPage, 1, 300);
|
|
93
|
+
if (input.pageLimit !== undefined) integer(input.pageLimit, 1, 2);
|
|
94
|
+
if (input.lineLimit !== undefined) integer(input.lineLimit, 1, 2000);
|
|
95
|
+
if (input.cursor !== undefined) {
|
|
96
|
+
exact(record(input.cursor), ["originalSha256", "representationSha256", "offset"]);
|
|
97
|
+
digest(input.cursor.originalSha256); digest(input.cursor.representationSha256);
|
|
98
|
+
integer(input.cursor.offset, 1, 20 * 1024 * 1024 - 1);
|
|
99
|
+
}
|
|
100
|
+
return Object.freeze({ ...input, ...(input.cursor ? { cursor: Object.freeze({ ...input.cursor }) } : {}) });
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
function parseTerminal(raw: unknown, request: { readonly requestId: string; readonly operation: DocumentOperation;
|
|
104
|
+
readonly originalSha256: string; readonly options: NativeDocumentOptions }) {
|
|
105
|
+
const frame = record(raw);
|
|
106
|
+
if (frame.requestId !== request.requestId) fail();
|
|
107
|
+
if (frame.type === "error") {
|
|
108
|
+
exact(frame, ["type", "requestId", "code"]);
|
|
109
|
+
const codes: readonly NativeDocumentErrorCode[] = ["password_required", "invalid_pdf", "encoding_required",
|
|
110
|
+
"unsupported_encoding", "ocr_failed", "processing_limit"];
|
|
111
|
+
const code = codes.find(code => code === frame.code);
|
|
112
|
+
if (code === undefined) fail();
|
|
113
|
+
throw new NativeDocumentError(code);
|
|
114
|
+
}
|
|
115
|
+
exact(frame, ["type", "requestId", "originalSha256", "representationSha256", "coverage", "content", "textRange", "nextCursor"]);
|
|
116
|
+
if (frame.type !== "document.complete" || digest(frame.originalSha256) !== request.originalSha256) {
|
|
117
|
+
throw new NativeDocumentError("original_mismatch");
|
|
118
|
+
}
|
|
119
|
+
const representationSha256 = digest(frame.representationSha256);
|
|
120
|
+
const coverage = record(frame.coverage);
|
|
121
|
+
exact(coverage, ["totalPages", "pages"]);
|
|
122
|
+
const totalPages = integer(coverage.totalPages, 1, 300);
|
|
123
|
+
if (!Array.isArray(coverage.pages) || coverage.pages.length > totalPages || !Array.isArray(frame.content)) fail();
|
|
124
|
+
const pages = coverage.pages.map((raw: unknown) => {
|
|
125
|
+
const page = record(raw); exact(page, ["page", "representation"]);
|
|
126
|
+
const number = integer(page.page, 1, totalPages);
|
|
127
|
+
switch (page.representation) {
|
|
128
|
+
case "native-text": case "rendered-image": case "ocr-text": case "verified-blank":
|
|
129
|
+
return { page: number, representation: page.representation };
|
|
130
|
+
default: return fail();
|
|
131
|
+
}
|
|
132
|
+
});
|
|
133
|
+
if (pages.some((page, index) => index > 0 && page.page <= (pages[index - 1]?.page ?? 0))) fail();
|
|
134
|
+
let bytes = 0, images = 0, lines = 0;
|
|
135
|
+
const imageBytes: Buffer[] = [];
|
|
136
|
+
const content = frame.content.map((raw: unknown) => {
|
|
137
|
+
const part = record(raw);
|
|
138
|
+
switch (part.type) {
|
|
139
|
+
case "text": {
|
|
140
|
+
exact(part, ["type", "text"]);
|
|
141
|
+
if (typeof part.text !== "string" || part.text.includes("\0")) return fail();
|
|
142
|
+
bytes += Buffer.byteLength(part.text);
|
|
143
|
+
lines += part.text.split("\n").length - (part.text.endsWith("\n") || part.text === "" ? 1 : 0);
|
|
144
|
+
return { type: "text", text: part.text } as const;
|
|
145
|
+
}
|
|
146
|
+
case "image": {
|
|
147
|
+
exact(part, ["type", "data", "mimeType"]);
|
|
148
|
+
if (typeof part.data !== "string" || part.data.length > Math.ceil(4 * 1024 * 1024 / 3) * 4
|
|
149
|
+
|| (part.mimeType !== "image/jpeg" && part.mimeType !== "image/png")) return fail();
|
|
150
|
+
const decoded = Buffer.from(part.data, "base64");
|
|
151
|
+
if (decoded.length === 0 || decoded.length > 4 * 1024 * 1024 || decoded.toString("base64") !== part.data) fail();
|
|
152
|
+
imageBytes.push(decoded); images++;
|
|
153
|
+
return { type: "image", data: part.data, mimeType: part.mimeType } as const;
|
|
154
|
+
}
|
|
155
|
+
default: return fail();
|
|
156
|
+
}
|
|
157
|
+
});
|
|
158
|
+
if (bytes > 48 * 1024 || lines > (request.options.lineLimit ?? 2000) || images > 2) fail();
|
|
159
|
+
const base = { originalSha256: request.originalSha256, representationSha256, coverage: { totalPages, pages }, content };
|
|
160
|
+
switch (request.operation) {
|
|
161
|
+
case "document.text": case "document.ocr": {
|
|
162
|
+
const range = record(frame.textRange); exact(range, ["start", "end", "totalScalars", "totalBytes"]);
|
|
163
|
+
const totalBytes = integer(range.totalBytes, 0, 20 * 1024 * 1024);
|
|
164
|
+
const totalScalars = integer(range.totalScalars, 0, totalBytes);
|
|
165
|
+
const start = integer(range.start, 0, totalScalars), end = integer(range.end, start, totalScalars);
|
|
166
|
+
const part = content[0];
|
|
167
|
+
if (images !== 0 || content.length !== 1 || part?.type !== "text"
|
|
168
|
+
|| Array.from(part.text).length !== end - start || start !== (request.options.cursor?.offset ?? 0)
|
|
169
|
+
|| (end === start && totalScalars !== 0)) fail();
|
|
170
|
+
const residual = totalScalars - (end - start);
|
|
171
|
+
if (totalBytes - bytes < residual || totalBytes - bytes > 4 * residual) fail();
|
|
172
|
+
if (request.options.cursor && representationSha256 !== request.options.cursor.representationSha256) fail();
|
|
173
|
+
let nextCursor: NativeTextCursor | undefined;
|
|
174
|
+
if (end < totalScalars) {
|
|
175
|
+
const next = record(frame.nextCursor); exact(next, ["originalSha256", "representationSha256", "offset"]);
|
|
176
|
+
if (next.originalSha256 !== request.originalSha256 || next.representationSha256 !== representationSha256 || next.offset !== end) fail();
|
|
177
|
+
nextCursor = { originalSha256: request.originalSha256, representationSha256, offset: end };
|
|
178
|
+
} else if (frame.nextCursor !== undefined) fail();
|
|
179
|
+
if (start === 0 && end === totalScalars && (hash(part.text) !== representationSha256 || bytes !== totalBytes)) fail();
|
|
180
|
+
return { ...base, textRange: { start, end, totalScalars, totalBytes }, ...(nextCursor ? { nextCursor } : {}) };
|
|
181
|
+
}
|
|
182
|
+
case "document.inspect":
|
|
183
|
+
if (frame.textRange !== undefined || frame.nextCursor !== undefined || content.length !== 0 || pages.length !== totalPages) fail();
|
|
184
|
+
return base;
|
|
185
|
+
case "document.pages": {
|
|
186
|
+
const first = request.options.firstPage ?? 1, end = Math.min(first + (request.options.pageLimit ?? 2), totalPages + 1);
|
|
187
|
+
if (frame.textRange !== undefined || frame.nextCursor !== undefined || images !== end - first || pages.length !== images
|
|
188
|
+
|| pages.some((page, i) => page.page !== first + i || page.representation !== "rendered-image")
|
|
189
|
+
|| hash(Buffer.concat(imageBytes)) !== representationSha256) fail();
|
|
190
|
+
return base;
|
|
191
|
+
}
|
|
192
|
+
default: {
|
|
193
|
+
const exhaustive: never = request.operation;
|
|
194
|
+
return exhaustive;
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
/** Already sealed bytes only. Callers validate grants and CAS-publish returned data themselves. */
|
|
200
|
+
export async function readNativeDocument(build: NativeDocumentBuild,
|
|
201
|
+
input: { readonly original: Uint8Array; readonly operation: DocumentOperation; readonly options: NativeDocumentOptions },
|
|
202
|
+
context: DocumentReadContext): Promise<NativeDocumentResult> {
|
|
203
|
+
if (process.platform !== "darwin" || process.arch !== "arm64") throw new NativeDocumentError("native_reader_unavailable");
|
|
204
|
+
const selected = options(input.options), original = input.original;
|
|
205
|
+
if (!["document.inspect", "document.text", "document.pages", "document.ocr"].includes(input.operation)
|
|
206
|
+
|| ((input.operation === "document.inspect" || input.operation === "document.pages") && selected.cursor)) {
|
|
207
|
+
throw new NativeDocumentError("invalid_document_options");
|
|
208
|
+
}
|
|
209
|
+
if (original.length === 0 || original.length > 10 * 1024 * 1024) throw new NativeDocumentError("processing_limit", "original_bytes");
|
|
210
|
+
const sealed = Uint8Array.from(original);
|
|
211
|
+
if (context.signal.aborted) throw new NativeDocumentError("cancelled");
|
|
212
|
+
if (context.deadline?.aborted) throw new NativeDocumentError("processing_limit", "total_time");
|
|
213
|
+
if (!context.isCurrent()) throw new NativeDocumentError("stale_preparation");
|
|
214
|
+
const originalSha256 = hash(sealed);
|
|
215
|
+
if (selected.cursor && selected.cursor.originalSha256 !== originalSha256) throw new NativeDocumentError("invalid_cursor");
|
|
216
|
+
if (!shaPattern.test(build.sourceSha256) || !shaPattern.test(build.buildSha256)) throw new NativeDocumentError("native_reader_unavailable");
|
|
217
|
+
const home = await lstat(build.privateHome);
|
|
218
|
+
if (!home.isDirectory() || home.isSymbolicLink() || (home.mode & 0o077) !== 0) throw new NativeDocumentError("native_reader_unavailable");
|
|
219
|
+
const descriptor = await open(build.executable, constants.O_RDONLY | constants.O_NOFOLLOW);
|
|
220
|
+
try {
|
|
221
|
+
const stat = await descriptor.stat();
|
|
222
|
+
if (!stat.isFile() || (stat.mode & 0o111) === 0 || hash(await descriptor.readFile()) !== build.buildSha256) {
|
|
223
|
+
throw new NativeDocumentError("native_reader_unavailable");
|
|
224
|
+
}
|
|
225
|
+
} finally { await descriptor.close(); }
|
|
226
|
+
const requestId = randomUUID(), operation = input.operation;
|
|
227
|
+
const wire = JSON.stringify({ version: 1, requestId, operation, originalBase64: Buffer.from(sealed).toString("base64"), options: selected });
|
|
228
|
+
const usage = dlopen("/usr/lib/libproc.dylib", { proc_pid_rusage: { args: [FFIType.i32, FFIType.i32, FFIType.ptr], returns: FFIType.i32 } });
|
|
229
|
+
const buffer = new Uint8Array(96), view = new DataView(buffer.buffer);
|
|
230
|
+
const child = Bun.spawn([build.executable, ...(operation === "document.ocr" ? ["--ocr"] : [])], { stdin: "pipe", stdout: "pipe", stderr: "pipe",
|
|
231
|
+
env: { HOME: build.privateHome, TMPDIR: build.privateHome, LANG: "en_US.UTF-8" } });
|
|
232
|
+
let failure: NativeDocumentError | undefined, peakRss = 0, rssSamples = 0, isolated = false, ocrRevision = 0;
|
|
233
|
+
const stop = (error: NativeDocumentError) => { failure ??= error; child.kill("SIGKILL"); };
|
|
234
|
+
const abort = () => stop(new NativeDocumentError("cancelled"));
|
|
235
|
+
const expire = () => stop(new NativeDocumentError("processing_limit", "total_time"));
|
|
236
|
+
context.signal.addEventListener("abort", abort, { once: true });
|
|
237
|
+
context.deadline?.addEventListener("abort", expire, { once: true });
|
|
238
|
+
if (context.signal.aborted) abort();
|
|
239
|
+
if (context.deadline?.aborted) expire();
|
|
240
|
+
const totalTimer = setTimeout(() => stop(new NativeDocumentError("processing_limit", "total_time")), 600_000);
|
|
241
|
+
let pageTimer: ReturnType<typeof setTimeout> | undefined;
|
|
242
|
+
const measureRss = () => {
|
|
243
|
+
if (usage.symbols.proc_pid_rusage(child.pid, 0, buffer) !== 0) return;
|
|
244
|
+
const resident = Number(view.getBigUint64(64, true));
|
|
245
|
+
peakRss = Math.max(peakRss, resident); rssSamples++;
|
|
246
|
+
if (resident > 1024 * 1024 * 1024) stop(new NativeDocumentError("processing_limit", "rss"));
|
|
247
|
+
};
|
|
248
|
+
const rssTimer = setInterval(measureRss, 20);
|
|
249
|
+
const output = async () => {
|
|
250
|
+
let bytes = 0;
|
|
251
|
+
const chunks: Uint8Array[] = [];
|
|
252
|
+
for await (const chunk of child.stdout) {
|
|
253
|
+
bytes += chunk.length;
|
|
254
|
+
if (bytes > 16 * 1024 * 1024) { stop(new NativeDocumentError("processing_limit", "output_bytes")); break; }
|
|
255
|
+
chunks.push(chunk);
|
|
256
|
+
}
|
|
257
|
+
return Buffer.concat(chunks);
|
|
258
|
+
};
|
|
259
|
+
const progress = async () => {
|
|
260
|
+
const decoder = new TextDecoder("utf-8", { fatal: true });
|
|
261
|
+
let line = "", bytes = 0;
|
|
262
|
+
for await (const chunk of child.stderr) {
|
|
263
|
+
bytes += chunk.length;
|
|
264
|
+
if (bytes > 64 * 1024) { stop(new NativeDocumentError("invalid_native_output")); break; }
|
|
265
|
+
line += decoder.decode(chunk, { stream: true });
|
|
266
|
+
let boundary: number;
|
|
267
|
+
while ((boundary = line.indexOf("\n")) >= 0) {
|
|
268
|
+
const current = line.slice(0, boundary); line = line.slice(boundary + 1);
|
|
269
|
+
if (current === '{"stage":"isolated","page":0}') {
|
|
270
|
+
if (isolated) { stop(new NativeDocumentError("invalid_native_output")); continue; }
|
|
271
|
+
isolated = true;
|
|
272
|
+
// The child is alive and blocked on stdin here. A required measurement at
|
|
273
|
+
// this barrier makes even sub-20ms decodes independent of timer scheduling.
|
|
274
|
+
measureRss();
|
|
275
|
+
context.onProgress?.({ stage: "isolated", page: 0 });
|
|
276
|
+
if (!context.isCurrent()) stop(new NativeDocumentError("stale_preparation"));
|
|
277
|
+
if (!failure) { child.stdin.write(wire); await child.stdin.end(); }
|
|
278
|
+
} else if (/^\{"stage":"page-start","page":(?:[1-9]|[1-9][0-9]|[12][0-9]{2}|300)\}$/.test(current)) {
|
|
279
|
+
if (pageTimer) { stop(new NativeDocumentError("invalid_native_output")); continue; }
|
|
280
|
+
pageTimer = setTimeout(() => stop(new NativeDocumentError("processing_limit", "page_time")), 60_000);
|
|
281
|
+
const page = /"page":(\d+)/.exec(current)?.[1];
|
|
282
|
+
context.onProgress?.({ stage: "page-start", page: Number(page) });
|
|
283
|
+
} else if (/^\{"stage":"page-end","page":\d+\}$/.test(current)) {
|
|
284
|
+
clearTimeout(pageTimer); pageTimer = undefined;
|
|
285
|
+
context.onProgress?.({ stage: "page-end", page: Number(/"page":(\d+)/.exec(current)?.[1]) });
|
|
286
|
+
} else if (/^\{"stage":"ocr","page":\d+\}$/.test(current)) {
|
|
287
|
+
context.onProgress?.({ stage: "ocr", page: Number(/"page":(\d+)/.exec(current)?.[1]) });
|
|
288
|
+
} else if (/^\{"stage":"ocr-config","revision":\d+,"languages":\["ko-KR","en-US"\]\}$/.test(current)) {
|
|
289
|
+
ocrRevision = Number(/"revision":(\d+)/.exec(current)?.[1]);
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
};
|
|
294
|
+
try {
|
|
295
|
+
const [stdout, , exit] = await Promise.all([output(), progress(), child.exited]);
|
|
296
|
+
if (failure) throw failure;
|
|
297
|
+
if (!isolated || rssSamples === 0 || (operation === "document.ocr" && (!Number.isSafeInteger(ocrRevision) || ocrRevision < 1))) {
|
|
298
|
+
throw new NativeDocumentError("native_reader_unavailable");
|
|
299
|
+
}
|
|
300
|
+
if (!context.isCurrent()) throw new NativeDocumentError("stale_preparation");
|
|
301
|
+
if (!stdout.length || stdout[stdout.length - 1] !== 10 || stdout.subarray(0, -1).includes(10)) fail();
|
|
302
|
+
const frame = parseTerminal(JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(stdout)),
|
|
303
|
+
{ requestId, operation, originalSha256, options: selected });
|
|
304
|
+
if (exit !== 0 || pageTimer !== undefined) fail();
|
|
305
|
+
const optionsSha256 = hash(JSON.stringify({ version: 1, operation,
|
|
306
|
+
options: { format: selected.format, encoding: selected.encoding ?? null,
|
|
307
|
+
firstPage: selected.firstPage ?? 1, pageLimit: selected.pageLimit ?? 2, lineLimit: selected.lineLimit ?? 2000 },
|
|
308
|
+
render: { edge: 1568, imageBytes: 4 * 1024 * 1024, ocrDpi: 300, pixels: 12_000_000 },
|
|
309
|
+
ocrRevision: operation === "document.ocr" ? ocrRevision : null,
|
|
310
|
+
ocrLanguages: operation === "document.ocr" ? ["ko-KR", "en-US"] : [] }));
|
|
311
|
+
return { ...frame, optionsSha256,
|
|
312
|
+
representationKey: hash(JSON.stringify([originalSha256, build.sourceSha256, build.buildSha256, optionsSha256])),
|
|
313
|
+
helperSourceSha256: build.sourceSha256, helperBuildSha256: build.buildSha256,
|
|
314
|
+
ocrRevision,
|
|
315
|
+
resources: { peakRss, rssSamples, reaped: true } };
|
|
316
|
+
} catch (error) {
|
|
317
|
+
if (error instanceof SyntaxError || error instanceof TypeError) throw new NativeDocumentError("invalid_native_output");
|
|
318
|
+
throw error;
|
|
319
|
+
} finally {
|
|
320
|
+
clearInterval(rssTimer); clearTimeout(totalTimer); clearTimeout(pageTimer);
|
|
321
|
+
context.signal.removeEventListener("abort", abort);
|
|
322
|
+
context.deadline?.removeEventListener("abort", expire);
|
|
323
|
+
child.kill("SIGKILL"); await child.exited; usage.close();
|
|
324
|
+
}
|
|
325
|
+
}
|