rea-agents 6.4.0 → 6.5.0

package/dist/process/capture/ProcessNormalization.js

@@ -1,25 +1,247 @@
 import { isIP } from "node:net";
+const createUtf8ByteCounter = (maxBytes) => {
+    let bytes = 0;
+    let pendingHighSurrogate = false;
+    const addBytes = (added) => {
+        if (bytes > maxBytes || added > maxBytes - bytes) {
+            bytes = Number.POSITIVE_INFINITY;
+            return;
+        }
+        bytes += added;
+    };
+    return {
+        add: (value) => {
+            if (bytes > maxBytes || value.length === 0)
+                return;
+            let index = 0;
+            if (pendingHighSurrogate) {
+                if (value.length > 0 && isLowSurrogate(value.charCodeAt(0))) {
+                    addBytes(4);
+                    index = 1;
+                }
+                else {
+                    addBytes(3);
+                }
+                pendingHighSurrogate = false;
+            }
+            for (; index < value.length; index += 1) {
+                const code = value.charCodeAt(index);
+                if (code <= 0x7f)
+                    addBytes(1);
+                else if (code <= 0x7ff)
+                    addBytes(2);
+                else if (isHighSurrogate(code)) {
+                    if (index + 1 === value.length) {
+                        pendingHighSurrogate = true;
+                    }
+                    else if (isLowSurrogate(value.charCodeAt(index + 1))) {
+                        addBytes(4);
+                        index += 1;
+                    }
+                    else
+                        addBytes(3);
+                }
+                else
+                    addBytes(3);
+                if (bytes > maxBytes)
+                    return;
+            }
+        },
+        finish: () => {
+            if (pendingHighSurrogate)
+                addBytes(3);
+            return bytes;
+        },
+    };
+};
+const isHighSurrogate = (code) => code >= 0xd800 && code <= 0xdbff;
+const isLowSurrogate = (code) => code >= 0xdc00 && code <= 0xdfff;
+const boundedRegexReplace = (value, matcher, replacement, maxBytes) => {
+    const byteCounter = createUtf8ByteCounter(maxBytes);
+    let cursor = 0;
+    let changed = false;
+    matcher.lastIndex = 0;
+    let match;
+    while ((match = matcher.exec(value)) !== null) {
+        const index = match.index;
+        const matched = match[0] ?? "";
+        const nextCursor = index + matched.length;
+        const replacementValue = replacement(matched, ...match.slice(1));
+        changed ||= replacementValue !== matched;
+        byteCounter.add(value.slice(cursor, index));
+        byteCounter.add(replacementValue);
+        cursor = nextCursor;
+    }
+    byteCounter.add(value.slice(cursor));
+    const bytes = byteCounter.finish();
+    if (bytes > maxBytes) {
+        return changed ? { state: "over_budget" } : { state: "unchanged", value };
+    }
+    if (!changed)
+        return { state: "unchanged", value };
+    matcher.lastIndex = 0;
+    const normalized = value.replace(matcher, replacement);
+    return {
+        state: "bounded",
+        value: normalized,
+        utf8Bytes: bytes,
+    };
+};
+const boundedLiteralReplace = (value, pattern, replacement, maxBytes) => {
+    const byteCounter = createUtf8ByteCounter(maxBytes);
+    let cursor = 0;
+    if (pattern === "") {
+        if (replacement === "")
+            return { state: "unchanged", value };
+        for (let index = 0; index < value.length; index += 1) {
+            byteCounter.add(replacement);
+            byteCounter.add(value.slice(index, index + 1));
+        }
+        byteCounter.add(replacement);
+        const bytes = byteCounter.finish();
+        if (bytes > maxBytes)
+            return { state: "over_budget" };
+        return {
+            state: "bounded",
+            value: value.replaceAll(pattern, () => replacement),
+            utf8Bytes: bytes,
+        };
+    }
+    if (pattern === replacement)
+        return { state: "unchanged", value };
+    let index = value.indexOf(pattern, cursor);
+    if (index === -1)
+        return { state: "unchanged", value };
+    while (index !== -1) {
+        byteCounter.add(value.slice(cursor, index));
+        byteCounter.add(replacement);
+        cursor = index + pattern.length;
+        index = value.indexOf(pattern, cursor);
+    }
+    byteCounter.add(value.slice(cursor));
+    const bytes = byteCounter.finish();
+    if (bytes > maxBytes)
+        return { state: "over_budget" };
+    return {
+        state: "bounded",
+        value: value.replaceAll(pattern, () => replacement),
+        utf8Bytes: bytes,
+    };
+};
+const boundedIdentity = (value, maxBytes) => {
+    const utf8Bytes = Buffer.byteLength(value);
+    return utf8Bytes <= maxBytes
+        ? { state: "bounded", value, utf8Bytes }
+        : { state: "unchanged", value };
+};
 /** Bucket one elapsed process-capture timestamp under scenario normalization. */
 export const normalizeProcessElapsedTime = (elapsedMs, timeBucketMs) => Math.floor(elapsedMs / timeBucketMs) * timeBucketMs;
-/** Normalize one terminal/protocol payload under explicitly selected scenario rules. */
-export const normalizeProcessText = (value, scenario, temporaryRoot, pid) => {
-    let normalized = value;
-    if (scenario.normalization.paths) {
-        normalized = normalized.replaceAll(temporaryRoot, "<temporary-root>");
-        normalized = normalized.replaceAll(scenario.working_directory, "<working-directory>");
-        normalized = normalized.replaceAll(scenario.executable, "<executable>");
-        for (const [index, root] of scenario.filesystem_observation_paths.entries())
-            normalized = normalized.replaceAll(root, `<filesystem-root-${String(index)}>`);
-    }
-    if (scenario.normalization.ports)
-        normalized = normalizePortTokens(normalized);
-    if (scenario.normalization.pids)
-        normalized = normalizePidTokens(normalized, [pid]);
-    // A callback keeps replacement text literal while preserving replaceAll's
-    // empty-pattern boundaries, which the process scenario contract permits.
-    for (const pattern of scenario.normalization.patterns)
-        normalized = normalized.replaceAll(pattern.pattern, () => pattern.replacement);
-    return normalized;
+/** Prepare capture-local matchers while retaining a live terminal PID lookup. */
+export const createProcessTextNormalizer = (scenario, temporaryRoot, pid) => {
+    const normalizePaths = scenario.normalization.paths
+        ? createPathNormalizer(scenario, temporaryRoot)
+        : (value) => value;
+    let previousPid;
+    let normalizePids = (value) => value;
+    return (value) => {
+        let normalized = normalizePaths(value);
+        if (scenario.normalization.ports)
+            normalized = normalizePortTokens(normalized);
+        if (scenario.normalization.pids) {
+            const currentPid = typeof pid === "number" ? pid : pid();
+            if (currentPid !== previousPid) {
+                previousPid = currentPid;
+                normalizePids = createPidNormalizer([currentPid]);
+            }
+            normalized = normalizePids(normalized);
+        }
+        for (const pattern of scenario.normalization.patterns)
+            normalized = normalized.replaceAll(pattern.pattern, () => pattern.replacement);
+        return normalized;
+    };
+};
+/** Normalize terminal text without materializing an intermediate over its byte budget. */
+export const createBoundedProcessTextNormalizer = (scenario, temporaryRoot, pid) => {
+    const pathRule = scenario.normalization.paths
+        ? createPathRule(scenario, temporaryRoot)
+        : undefined;
+    let previousPid;
+    let pidMatcher;
+    return (value, maxBytes) => {
+        let normalized = value;
+        const apply = (result) => {
+            if (result.state === "over_budget")
+                return false;
+            if (result.state === "bounded")
+                normalized = result.value;
+            return true;
+        };
+        if (pathRule !== undefined) {
+            if (!apply(boundedRegexReplace(normalized, pathRule.matcher, pathRule.replace, maxBytes)))
+                return { state: "over_budget" };
+        }
+        if (scenario.normalization.ports) {
+            if (!apply(normalizePortsBounded(normalized, maxBytes)))
+                return { state: "over_budget" };
+        }
+        if (scenario.normalization.pids) {
+            const currentPid = typeof pid === "number" ? pid : pid();
+            if (currentPid !== previousPid) {
+                previousPid = currentPid;
+                pidMatcher = createPidMatcher([currentPid]);
+            }
+            if (pidMatcher !== undefined) {
+                if (!apply(boundedRegexReplace(normalized, pidMatcher, () => "<pid>", maxBytes)))
+                    return { state: "over_budget" };
+            }
+        }
+        for (const pattern of scenario.normalization.patterns) {
+            if (!apply(boundedLiteralReplace(normalized, pattern.pattern, pattern.replacement, maxBytes)))
+                return { state: "over_budget" };
+        }
+        return boundedIdentity(normalized, maxBytes).state === "bounded"
+            ? {
+                state: "normalized",
+                value: normalized,
+                utf8Bytes: Buffer.byteLength(normalized),
+            }
+            : { state: "over_budget" };
+    };
+};
+/** Normalize one payload when its caller does not retain a capture-local matcher. */
+export const normalizeProcessText = (value, scenario, temporaryRoot, pid) => createProcessTextNormalizer(scenario, temporaryRoot, pid)(value);
+/** Match original paths once, preferring the most specific selected identity. */
+const createPathRule = (scenario, temporaryRoot) => {
+    const aliases = new Map();
+    const selected = [
+        [temporaryRoot, "<temporary-root>"],
+        [scenario.working_directory, "<working-directory>"],
+        [scenario.executable, "<executable>"],
+        ...scenario.filesystem_observation_paths.map((root, index) => [
+            root,
+            `<filesystem-root-${String(index)}>`,
+        ]),
+    ];
+    for (const [path, alias] of selected)
+        if (path !== "" && !aliases.has(path))
+            aliases.set(path, alias);
+    const paths = [...aliases.keys()].sort((left, right) => right.length - left.length);
+    if (paths.length === 0)
+        return undefined;
+    const pattern = paths
+        .map((path) => path.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"))
+        .join("|");
+    const matcher = new RegExp(pattern, "gu");
+    return {
+        matcher,
+        replace: (path) => aliases.get(path) ?? path,
+    };
+};
+const createPathNormalizer = (scenario, temporaryRoot) => {
+    const rule = createPathRule(scenario, temporaryRoot);
+    if (rule === undefined)
+        return (value) => value;
+    return (value) => value.replace(rule.matcher, rule.replace);
 };
 // A port may end a sentence, but a period followed by a letter, digit, or
 // underscore in any script continues a decimal, version, or host name and is
@@ -27,16 +249,28 @@ export const normalizeProcessText = (value, scenario, temporaryRoot, pid) => {
 // PTY chunks are normalized independently, so a period that ends the chunk
 // stays unnormalized: its continuation may arrive in the next chunk.
 const normalizePortTokens = (value) => {
-    const endpointNormalized = value.replaceAll(/(\b[a-z][a-z0-9+.-]*:\/\/[^\s/?#"'<>]+|\blocalhost|\b(?:\d{1,3}\.){3}\d{1,3}|\[[\da-f:.%]+\]):(\d{1,5})(?![\p{L}\p{N}_@:]|\.(?:[\p{L}\p{N}_]|$))/giu, (match, endpoint, port) => {
-        if (Number(port) > 65_535)
-            return match;
-        const isEndpoint = endpoint.includes("://")
-            ? URL.canParse(`${endpoint}:${port}`)
-            : endpoint.toLowerCase() === "localhost" ||
-                isIP(endpoint.replace(/^\[|\]$/gu, "")) !== 0;
-        return isEndpoint ? `${endpoint}:<port>` : match;
-    });
-    return endpointNormalized.replaceAll(/(\b(?:port|tcp_port|udp_port|listen)\b["']?\s*[:=]\s*)(\d{1,5})(?![\p{L}\p{N}_]|\.(?:[\p{L}\p{N}_]|$))/giu, (match, prefix, port) => Number(port) <= 65_535 ? `${prefix}<port>` : match);
+    return value
+        .replaceAll(PORT_ENDPOINT_MATCHER, portEndpointReplacement)
+        .replaceAll(PORT_FIELD_MATCHER, portFieldReplacement);
+};
+const PORT_ENDPOINT_MATCHER = /(\b[a-z][a-z0-9+.-]*:\/\/[^\s/?#"'<>]+|\blocalhost|\b(?:\d{1,3}\.){3}\d{1,3}|\[[\da-f:.%]+\]):(\d{1,5})(?![\p{L}\p{N}_@:]|\.(?:[\p{L}\p{N}_]|$))/giu;
+const PORT_FIELD_MATCHER = /(\b(?:port|tcp_port|udp_port|listen)\b["']?\s*[:=]\s*)(\d{1,5})(?![\p{L}\p{N}_]|\.(?:[\p{L}\p{N}_]|$))/giu;
+const portEndpointReplacement = (match, endpoint, port) => {
+    if (Number(port) > 65_535)
+        return match;
+    const isEndpoint = endpoint.includes("://")
+        ? URL.canParse(`${endpoint}:${port}`)
+        : endpoint.toLowerCase() === "localhost" ||
+            isIP(endpoint.replace(/^\[|\]$/gu, "")) !== 0;
+    return isEndpoint ? `${endpoint}:<port>` : match;
+};
+const portFieldReplacement = (match, prefix, port) => Number(port) <= 65_535 ? `${prefix}<port>` : match;
+const normalizePortsBounded = (value, maxBytes) => {
+    const endpoint = boundedRegexReplace(value, PORT_ENDPOINT_MATCHER, portEndpointReplacement, maxBytes);
+    if (endpoint.state === "over_budget")
+        return endpoint;
+    const afterEndpoint = endpoint.state === "bounded" ? endpoint.value : value;
+    return boundedRegexReplace(afterEndpoint, PORT_FIELD_MATCHER, portFieldReplacement, maxBytes);
 };
 /** Project sampled observations under the caller-selected normalization rules. */
 export const normalizeProcessSamples = (samples, scenario, rootPid) => {
@@ -53,11 +287,13 @@ export const normalizeProcessSamples = (samples, scenario, rootPid) => {
     for (const identifier of identifiers)
         if (identifier > 0 && !mapping.has(identifier))
             mapping.set(identifier, mapping.size + 1);
+    const normalizeText = createProcessTextNormalizer(scenario, "<no-temporary-root>", rootPid);
+    const normalizeSamplePids = scenario.normalization.pids
+        ? createPidNormalizer(identifiers)
+        : undefined;
     const normalizeCommand = (command) => {
-        const normalized = normalizeProcessText(command, scenario, "<no-temporary-root>", rootPid);
-        const withNormalizedPids = scenario.normalization.pids
-            ? normalizePidTokens(normalized, identifiers)
-            : normalized;
+        const normalized = normalizeText(command);
+        const withNormalizedPids = normalizeSamplePids?.(normalized) ?? normalized;
         return withNormalizedPids.trim();
     };
     return samples.map((sample) => ({
@@ -77,12 +313,18 @@ export const normalizeProcessSamples = (samples, scenario, rootPid) => {
         command: normalizeCommand(sample.command),
     }));
 };
-const normalizePidTokens = (value, identifiers) => {
+const createPidNormalizer = (identifiers) => {
+    const matcher = createPidMatcher(identifiers);
+    if (matcher === undefined)
+        return (value) => value;
+    return (value) => value.replace(matcher, "<pid>");
+};
+const createPidMatcher = (identifiers) => {
     const tokens = [...new Set(identifiers)]
         .filter((identifier) => Number.isSafeInteger(identifier) && identifier > 0)
         .sort((left, right) => right - left)
         .map(String);
     if (tokens.length === 0)
-        return value;
-    return value.replace(new RegExp(`(?<!\\d)(?:${tokens.join("|")})(?!\\d)`, "gu"), "<pid>");
+        return undefined;
+    return new RegExp(`(?<!\\d)(?:${tokens.join("|")})(?!\\d)`, "gu");
 };

package/dist/process/capture/ProcessSampling.js

@@ -1,6 +1,7 @@
 import { readFile, readdir } from "node:fs/promises";
 import { execFileOutput } from "../ExecFileOutput.js";
 import { readProcessRunId } from "../ProcessOwnership.js";
+import { monotonicProcessCaptureClock } from "./ProcessCaptureClock.js";
 const PROC_READ_CONCURRENCY = 64;
 /** Prove that forkpty completed session setup and installed the run token. */
 export const isInitializedPtyRoot = (options) => {
@@ -238,7 +239,7 @@ const inspectInitializedPtyRoot = async (rootPid, runId, signal) => {
         return undefined;
     let observedRunId;
     try {
-        observedRunId = await readProcessRunId(rootPid);
+        observedRunId = await readProcessRunId(rootPid, undefined, signal);
     }
     catch (cause) {
         // Unreadable run identity means the root cannot be attributed.
@@ -348,7 +349,7 @@ export const startProcessSampler = (options) => {
             }
             return sampleProcesses({
                 rootPid,
-                elapsedMs: Date.now() - started,
+                elapsedMs: monotonicProcessCaptureClock() - started,
                 limit: limit + 1,
                 identities,
                 signal: controller.signal,

package/dist/process/capture/ProcessScenarioRuntimeValidation.js

@@ -1,8 +1,9 @@
-import { access, realpath, stat } from "node:fs/promises";
 import { constants as fsConstants } from "node:fs";
-import { delimiter, isAbsolute, join, resolve } from "node:path";
+import { access, realpath, stat } from "node:fs/promises";
+import { delimiter, isAbsolute, join, resolve, sep } from "node:path";
+import { AnalysisInputError } from "../../domain/analysisErrorCore.js";
 import { snapshotEnvironment } from "../snapshotEnvironment.js";
-import { ProcessCaptureError, processCaptureCancelled, } from "./ProcessCaptureError.js";
+import { processCaptureCancelled } from "./ProcessCaptureError.js";
 /** Resolve caller-selected executable and observation paths for this execution. */
 export const resolveProcessScenarioRuntimePaths = async (scenario, environment = process.env, platform = process.platform) => {
     const selectedWorkingDirectory = resolve(scenario.working_directory);
@@ -11,17 +12,17 @@ export const resolveProcessScenarioRuntimePaths = async (scenario, environment =
         workingDirectory = await realpath(selectedWorkingDirectory);
     }
     catch (cause) {
-        throw new ProcessCaptureError(`working directory could not be resolved: ${selectedWorkingDirectory}${osFailure(cause)}`, { cause });
+        throw invalidScenarioPath("working_directory", `working directory could not be resolved: ${selectedWorkingDirectory}${osFailure(cause)}`, cause);
     }
     let workingDirectoryStat;
     try {
         workingDirectoryStat = await stat(workingDirectory);
     }
     catch (cause) {
-        throw new ProcessCaptureError(`working directory could not be inspected: ${workingDirectory}${osFailure(cause)}`, { cause });
+        throw invalidScenarioPath("working_directory", `working directory could not be inspected: ${workingDirectory}${osFailure(cause)}`, cause);
     }
     if (!workingDirectoryStat.isDirectory())
-        throw new ProcessCaptureError(`working directory is not a directory: ${workingDirectory}`);
+        throw invalidScenarioPath("working_directory", `working directory is not a directory: ${workingDirectory}`);
     const executable = await resolveExecutable(scenario.executable, workingDirectory, snapshotEnvironment(scenario.environment, platform).PATH ??
         snapshotEnvironment(environment, platform).PATH ??
         "");
@@ -40,7 +41,9 @@ export const resolveExecutable = async (command, workingDirectory, searchPath) =
         .map((directory) => directory === ""
         ? workingDirectory
         : resolve(workingDirectory, directory));
-    const candidates = isAbsolute(command) || command.includes("/") || command.includes("\\")
+    const candidates = isAbsolute(command) ||
+        command.includes("/") ||
+        (sep === "\\" && command.includes("\\"))
         ? [resolve(workingDirectory, command)]
         : directories.map((directory) => join(directory, command));
     const failures = [];
@@ -52,16 +55,13 @@ export const resolveExecutable = async (command, workingDirectory, searchPath) =
             failures.push(`${candidate} (not a regular file)`);
         }
         catch (cause) {
-            const code = osErrorCode(cause);
-            if (code !== "ENOENT" && code !== "ENOTDIR")
-                failures.push(`${candidate}${osFailure(cause)}`);
+            failures.push(`${candidate}${osFailure(cause)}`);
             continue;
         }
     }
-    throw new ProcessCaptureError(failures.length === 0
-        ? `executable could not be resolved: ${command}`
-        : `executable could not be resolved: ${command}; checked ${failures.join("; ")}`);
+    throw invalidScenarioPath("executable", `executable could not be resolved: ${command}; checked ${failures.join("; ")}`);
 };
+const invalidScenarioPath = (field, message, cause) => new AnalysisInputError("capture_process_scenario", cause === undefined ? undefined : { cause }, [{ path: [field], reason: "invalid_value", message }]);
 const osErrorCode = (cause) => typeof cause === "object" &&
     cause !== null &&
     "code" in cause &&

package/dist/process/capture/TerminalRenderer.js

@@ -23,6 +23,7 @@ export class TerminalRenderer {
     #observedBytes = 0;
     #observedFrames = 0;
     #truncated = false;
+    #unknownSizeFrames = 0;
     constructor(options) {
         this.options = options;
         this.#terminal = new HeadlessPackage.Terminal({
@@ -71,6 +72,9 @@ export class TerminalRenderer {
             retained_bytes: this.#capturedBytes,
             observed_frames: this.#observedFrames,
             retained_frames: this.#frames.length,
+            ...(this.#unknownSizeFrames === 0
+                ? {}
+                : { unknown_size_frames: this.#unknownSizeFrames }),
         };
     }
     /** Release addon and terminal resources after all writes settle. */
@@ -89,23 +93,40 @@ export class TerminalRenderer {
     #capture(atMs) {
         const buffer = this.#terminal.buffer.active;
         const lines = [];
+        this.#observedFrames += 1;
+        let remainingBytes = this.options.maxBytes - this.#capturedBytes;
+        let bytes = 0;
         for (let row = 0; row < this.#terminal.rows; row += 1) {
             const line = buffer.getLine(buffer.viewportY + row);
+            const paddedLine = (line?.translateToString(false, 0, this.#terminal.cols) ?? "").padEnd(this.#terminal.cols, " ");
+            const trailingSpaceBytes = Buffer.byteLength(paddedLine.match(TRAILING_SPACES)?.[0] ?? "");
+            const normalized = this.options.normalize(paddedLine, Math.min(Number.MAX_SAFE_INTEGER, remainingBytes + trailingSpaceBytes));
+            if (normalized.state === "over_budget") {
+                this.#unknownSizeFrames += 1;
+                this.#truncated = true;
+                return;
+            }
+            const normalizedLine = normalized.value.replace(TRAILING_SPACES, "");
+            const lineBytes = Buffer.byteLength(normalizedLine);
+            if (lineBytes > remainingBytes) {
+                this.#unknownSizeFrames += 1;
+                this.#truncated = true;
+                return;
+            }
+            remainingBytes -= lineBytes;
+            bytes += lineBytes;
             // Trailing blank cells carry no information beyond `columns`; dropping
             // them keeps sparse screens from costing a full row per blank line.
-            lines.push(this.options
-                .normalize((line?.translateToString(false, 0, this.#terminal.cols) ?? "").padEnd(this.#terminal.cols, " "))
-                .replace(TRAILING_SPACES, ""));
+            lines.push(normalizedLine);
         }
-        const serializedState = this.options.normalize(this.#serializeAddon.serialize());
-        const bytes = Buffer.byteLength(serializedState) +
-            lines.reduce((total, line) => total + Buffer.byteLength(line), 0);
-        this.#observedBytes += bytes;
-        this.#observedFrames += 1;
-        if (this.#capturedBytes + bytes > this.options.maxBytes) {
+        const normalizedState = this.options.normalize(this.#serializeAddon.serialize(), remainingBytes);
+        if (normalizedState.state === "over_budget") {
+            this.#unknownSizeFrames += 1;
             this.#truncated = true;
             return;
         }
+        bytes += normalizedState.utf8Bytes;
+        this.#observedBytes += bytes;
         this.#capturedBytes += bytes;
         const sequence = this.#frames.length;
         this.#frames.push({
@@ -117,7 +138,7 @@ export class TerminalRenderer {
             cursor_y: buffer.cursorY,
             active_buffer: buffer.type,
             lines,
-            serialized_state: serializedState,
+            serialized_state: normalizedState.value,
         });
         this.options.recordEvent?.("rendered_frames", sequence);
     }