rea-agents 6.5.0 → 6.7.0

package/dist/kuna/KunaJsonStream.js

@@ -0,0 +1,433 @@
+import { createHash } from "node:crypto";
+import { Writable } from "node:stream";
+import { getHeapStatistics } from "node:v8";
+import { getManyValues, isMany, none } from "stream-chain/defs.js";
+import * as streamJsonParser from "stream-json/core/parser.js";
+import { AnalysisOutputError, AnalysisResourceConstraintError, } from "../domain/analysisErrorCore.js";
+import { err, ok } from "../domain/result.js";
+import { kunaAddressFromJsonNumber } from "./KunaJson.js";
+/**
+ * stream-json 3's synchronous tokenizer behind `parser`, which its
+ * declarations omit. Each call returns the tokens `text` completes; `none`
+ * flushes the end of input.
+ */
+const { jsonParser } = streamJsonParser;
+/** Charge for one retained row beyond its UTF-16 text. */
+const ROW_OVERHEAD_BYTES = 256;
+/** Projected retention admitted between heap samples. */
+const ADMISSION_INTERVAL_BYTES = 64 * 1024;
+/** Per-row reserve for the owner's arrays, sorting and projections. */
+const ROW_REFERENCE_BYTES = 16;
+const describe = (cause) => cause instanceof Error ? cause.message : String(cause);
+/**
+ * Longest key or kept number assembled. Kuna's field names and u64 decimals
+ * are far shorter, so a longer key names no kept field and a longer kept
+ * number is not one Kuna writes.
+ */
+const MAX_TOKEN_CODE_UNITS = 64;
+/** Parse one row with its schema, naming the first field that is not Kuna's. */
+export const parseKunaRow = (schema, member, fields) => {
+    const parsed = schema.safeParse(fields);
+    if (parsed.success)
+        return parsed.data;
+    throw new Error(`${member} row field ${parsed.error.issues[0]?.path.join(".") ?? ""} is not Kuna's`);
+};
+/** V8 copies shorter substrings instead of slicing them. */
+const SLICED_STRING_MIN_LENGTH = 13;
+/**
+ * Copy kept text out of the decoded chunk the tokenizer sliced it from, so a
+ * retained row does not keep that whole chunk alive. UTF-16 code units copy
+ * losslessly, including lone surrogates.
+ */
+const detached = (text) => text.length < SLICED_STRING_MIN_LENGTH
+    ? text
+    : Buffer.from(text, "utf16le").toString("utf16le");
+/**
+ * Validate a Kuna JSON document as Kuna writes it, without buffering it.
+ *
+ * Every scalar arrives in chunks and is assembled only as far as it can
+ * matter: keys up to the longest field name, kept numbers up to the longest
+ * u64, and kept strings, which can carry target-derived text of any length,
+ * admitted against live heap headroom as they grow, as are retained rows.
+ * Ignored values are dropped unassembled. The first contradiction aborts
+ * `rejected`, letting the owner stop Kuna early. The writable side never
+ * fails, so the command's own outcome stays authoritative; `complete` reports
+ * what the document established.
+ */
+export class KunaJsonDocumentReader extends Writable {
+    producer;
+    label;
+    document;
+    #hash = createHash("sha256");
+    #decoder = new TextDecoder("utf-8", { fatal: true });
+    #tokens = jsonParser({
+        packKeys: false,
+        packNumbers: false,
+        packStrings: false,
+        streamValues: true,
+    });
+    #rejection = new AbortController();
+    #frames = [];
+    #rowsSeen = new Set();
+    #header = {};
+    #bytes = 0;
+    #failure;
+    #closed = false;
+    #key;
+    /** Containers still open inside an ignored value. */
+    #skipping = 0;
+    /** Key the scalar being read is kept under; undefined discards it. */
+    #target;
+    /** Whether `#target` holds a numeric u64 address. */
+    #targetIsAddress = false;
+    /** Whether the scalar being assembled is a key. */
+    #inKey = false;
+    /** The key or kept scalar assembled so far. */
+    #text = "";
+    #rows = 0;
+    #projectedBytes = 0;
+    #nextAdmission = 0;
+    constructor(producer, label, document) {
+        super();
+        this.producer = producer;
+        this.label = label;
+        this.document = document;
+    }
+    /** Aborted at the first contradiction or refused retention. */
+    get rejected() {
+        return this.#rejection.signal;
+    }
+    /** Why `rejected` aborted, if it has. */
+    get rejection() {
+        return this.#rejection.signal.aborted ? this.#failure : undefined;
+    }
+    /** An output error naming this document. */
+    unusable(reason) {
+        return new AnalysisOutputError(this.producer, reason, {
+            userMessage: `${this.label} cannot be used: ${reason}`,
+        });
+    }
+    _write(chunk, _encoding, callback) {
+        if (this.#failure === undefined)
+            try {
+                this.#bytes += chunk.byteLength;
+                this.#hash.update(chunk);
+                this.#feed(this.#decoder.decode(chunk, { stream: true }));
+            }
+            catch (cause) {
+                this.#reject(cause);
+                this.#rejection.abort();
+            }
+        callback();
+    }
+    _final(callback) {
+        // Kuna has stopped writing, so an incomplete document is recorded for
+        // `complete` without signalling: its exit status may explain it.
+        if (this.#failure === undefined)
+            try {
+                this.#feed(this.#decoder.decode());
+                this.#feed(none);
+            }
+            catch (cause) {
+                this.#reject(cause);
+            }
+        callback();
+    }
+    /** The kept header and source identity, once Kuna has exited successfully. */
+    complete() {
+        if (this.#failure !== undefined)
+            return err(this.#failure);
+        if (!this.writableFinished ||
+            !this.#closed ||
+            [...this.document.rows.keys()].some((name) => !this.#rowsSeen.has(name)))
+            return err(this.unusable(`${this.label} ended before it was complete`));
+        return ok({
+            header: this.#header,
+            sha256: this.#hash.digest("hex"),
+            bytes: this.#bytes,
+        });
+    }
+    #reject(cause) {
+        this.#failure ??=
+            cause instanceof AnalysisResourceConstraintError
+                ? cause
+                : this.unusable(describe(cause));
+        this.#text = "";
+        this.#inKey = false;
+        this.#header = {};
+        this.discardRows();
+    }
+    #feed(text) {
+        const tokens = this.#tokens(text);
+        if (isMany(tokens))
+            for (const token of getManyValues(tokens))
+                this.#consume(token);
+    }
+    /** Consume a token inside an ignored value; false once none is open. */
+    #skip(token) {
+        if (this.#skipping === 0)
+            return false;
+        if (token.name === "startObject" || token.name === "startArray")
+            this.#skipping += 1;
+        else if (token.name === "endObject" || token.name === "endArray")
+            this.#skipping -= 1;
+        return true;
+    }
+    #consume(token) {
+        if (this.#skip(token) || this.#assemble(token))
+            return;
+        switch (token.name) {
+            case "startKey":
+                this.#inKey = true;
+                return;
+            case "endKey":
+                this.#key = this.#text;
+                this.#text = "";
+                this.#inKey = false;
+                return;
+            case "startString":
+                this.#aim("string");
+                return;
+            case "endString":
+                if (this.#target !== undefined)
+                    this.#store(detached(this.#text), "string");
+                this.#text = "";
+                return;
+            case "startNumber":
+                this.#aim("number");
+                return;
+            case "endNumber":
+                this.#store(this.#text, "number");
+                this.#text = "";
+                return;
+            case "nullValue":
+                this.#aim("null");
+                this.#store(null, "literal");
+                return;
+            case "trueValue":
+            case "falseValue":
+                this.#aim("boolean");
+                this.#store(token.value, "literal");
+                return;
+            case "startObject":
+            case "startArray":
+                this.#open(token.name === "startObject" ? "object" : "array");
+                return;
+            case "endObject":
+            case "endArray":
+                this.#close();
+                return;
+            default:
+                throw new Error(`unexpected JSON ${token.name}`);
+        }
+    }
+    /** Append a scalar fragment within its bound; false for other tokens. */
+    #assemble(token) {
+        if (token.name !== "stringChunk" && token.name !== "numberChunk")
+            return false;
+        const chunk = token.value;
+        if (this.#inKey) {
+            // A key longer than every kept field name is truncated unmatched.
+            if (this.#text.length <= MAX_TOKEN_CODE_UNITS)
+                this.#text += chunk.slice(0, MAX_TOKEN_CODE_UNITS + 1 - this.#text.length);
+        }
+        else if (this.#target !== undefined) {
+            this.#text += chunk;
+            if (token.name === "stringChunk")
+                this.#charge(chunk.length * 2, this.#text.length);
+            else if (this.#text.length > MAX_TOKEN_CODE_UNITS)
+                throw new Error(`${this.#target} is not a number Kuna writes`);
+        }
+        return true;
+    }
+    /** Decide where a scalar value goes, before any of it is assembled. */
+    #aim(kind) {
+        const frame = this.#frames.at(-1);
+        const key = this.#key;
+        this.#key = undefined;
+        this.#targetIsAddress = false;
+        if (frame === undefined)
+            throw new Error(`${this.label} is not one JSON object`);
+        switch (frame.kind) {
+            case "root":
+                if (this.document.rows.has(key) || this.document.sections.has(key))
+                    throw new Error(`${this.label} member ${key} is not a container`);
+                this.#target = this.#kept(frame.values, key, this.document.header);
+                return;
+            case "section":
+                this.#target = this.#kept(frame.values, key, frame.fields);
+                return;
+            case "rows":
+                throw new Error(`${frame.name} row is not an object`);
+            case "row":
+                if (frame.spec.empty.has(key))
+                    throw new Error(`${frame.member} row ${key} is not an array`);
+                if (frame.spec.nulls.has(key)) {
+                    if (kind !== "null")
+                        throw new Error(`${frame.member} row ${key} must be null`);
+                    this.#target = this.#kept(frame.values, key, frame.spec.nulls);
+                    return;
+                }
+                if (frame.spec.addresses.has(key)) {
+                    if (kind !== "number" && kind !== "null")
+                        throw new Error(`invalid numeric address in ${key}`);
+                    this.#target = this.#kept(frame.values, key, frame.spec.addresses);
+                    this.#targetIsAddress = kind === "number";
+                    return;
+                }
+                this.#target = this.#kept(frame.values, key, frame.spec.fields);
+                return;
+            case "empty":
+                throw new Error(`${frame.field} must be empty`);
+        }
+    }
+    /** The key a kept field is stored under, or undefined to discard it. */
+    #kept(values, key, fields) {
+        if (!fields.has(key))
+            return undefined;
+        this.#claim(values, key);
+        return key;
+    }
+    #claim(values, key) {
+        if (Object.hasOwn(values, key))
+            throw new Error(`${this.label} repeats ${key}`);
+    }
+    #store(value, kind) {
+        const key = this.#target;
+        if (key === undefined)
+            return;
+        this.#target = undefined;
+        this.#values()[key] = this.#targetIsAddress
+            ? kunaAddressFromJsonNumber(value, key)
+            : kind === "number"
+                ? Number(value)
+                : value;
+    }
+    #values() {
+        const frame = this.#frames.at(-1);
+        if (frame === undefined || frame.kind === "rows" || frame.kind === "empty")
+            throw new Error(`${this.label} value has no member`);
+        return frame.values;
+    }
+    /** Open a root member: rows, a section, or a value this reader skips. */
+    #openMember(kind, key, root) {
+        const rows = this.document.rows.get(key);
+        const section = this.document.sections.get(key);
+        if (rows !== undefined) {
+            if (kind !== "array" || this.#rowsSeen.has(key))
+                throw new Error(`${this.label} rows ${key} are not one array`);
+            this.#frames.push({ kind: "rows", name: key, spec: rows });
+        }
+        else if (section !== undefined) {
+            if (kind !== "object")
+                throw new Error(`${key} is not an object`);
+            this.#claim(root, key);
+            const values = {};
+            root[key] = values;
+            this.#frames.push({
+                kind: "section",
+                name: key,
+                fields: section,
+                values,
+            });
+        }
+        else if (this.document.header.has(key))
+            throw new Error(`${this.label} ${key} is not a scalar`);
+        else
+            this.#skipping = 1;
+    }
+    #open(kind) {
+        const frame = this.#frames.at(-1);
+        const key = this.#key;
+        this.#key = undefined;
+        if (frame === undefined) {
+            if (kind !== "object" || this.#closed)
+                throw new Error(`${this.label} is not one JSON object`);
+            this.#frames.push({ kind: "root", values: this.#header });
+            return;
+        }
+        switch (frame.kind) {
+            case "root":
+                this.#openMember(kind, key, frame.values);
+                return;
+            case "rows":
+                if (kind !== "object")
+                    throw new Error(`${frame.name} row is not an object`);
+                this.#frames.push({
+                    kind: "row",
+                    member: frame.name,
+                    spec: frame.spec,
+                    values: {},
+                });
+                return;
+            case "row":
+                if (frame.spec.empty.has(key)) {
+                    if (kind !== "array")
+                        throw new Error(`${frame.member} row ${key} is not an array`);
+                    this.#claim(frame.values, key);
+                    this.#frames.push({ kind: "empty", field: key });
+                }
+                else if (frame.spec.fields.has(key) ||
+                    frame.spec.addresses.has(key) ||
+                    frame.spec.nulls.has(key))
+                    throw new Error(`${frame.member} row ${key} is not a scalar`);
+                else
+                    this.#skipping = 1;
+                return;
+            case "section":
+                if (frame.fields.has(key))
+                    throw new Error(`${frame.name} ${key} is not a scalar`);
+                this.#skipping = 1;
+                return;
+            case "empty":
+                throw new Error(`${frame.field} must be empty`);
+        }
+    }
+    #close() {
+        const frame = this.#frames.pop();
+        switch (frame?.kind) {
+            case "root":
+                this.#closed = true;
+                return;
+            case "rows":
+                this.#rowsSeen.add(frame.name);
+                return;
+            case "row":
+                this.#rows += 1;
+                this.#charge(ROW_OVERHEAD_BYTES, 0);
+                this.acceptRow(frame.member, frame.values);
+                return;
+            case "empty": {
+                const row = this.#frames.at(-1);
+                if (row?.kind === "row")
+                    row.values[frame.field] = [];
+                return;
+            }
+            default:
+                return;
+        }
+    }
+    /** Admit growth against live headroom rather than a fixed document size. */
+    #charge(bytes, assemblingCodeUnits) {
+        this.#projectedBytes += bytes;
+        if (this.#projectedBytes < this.#nextAdmission)
+            return;
+        const heap = getHeapStatistics();
+        // Reserve the next interval, the owner's per-row references, and a flat
+        // copy of the string still being assembled.
+        const requiredBytes = ADMISSION_INTERVAL_BYTES +
+            this.#rows * ROW_REFERENCE_BYTES +
+            assemblingCodeUnits * 2;
+        if (requiredBytes > heap.total_available_size)
+            throw new AnalysisResourceConstraintError(this.producer, "memory", `Insufficient heap headroom to retain more of ${this.label} after ${String(this.#rows)} rows`, {
+                retained_rows: this.#rows,
+                projected_retained_bytes: this.#projectedBytes,
+                required_headroom_bytes: requiredBytes,
+                available_heap_bytes: heap.total_available_size,
+                heap_size_limit_bytes: heap.heap_size_limit,
+            }, {
+                remediationAction: "Close other targets in this REA process, or start REA with a larger Node.js heap (--max-old-space-size), then reopen the target.",
+            });
+        this.#nextAdmission = this.#projectedBytes + ADMISSION_INTERVAL_BYTES;
+    }
+}

package/dist/kuna/KunaLimits.js

@@ -0,0 +1,5 @@
+/** Shared buffered-text capacity for a subprocess answer or one cached source facet. */
+export const KUNA_FUNCTION_TEXT_BYTES = 16 * 1024 * 1024;
+/** Kuna's notes and summary lines for one run; those of a large image stay far below this. */
+export const KUNA_DIAGNOSTIC_BYTES = 8 * 1024 * 1024;
+export const KUNA_STRING_MIN_LENGTH = 5;