rea-agents 6.3.0 → 6.5.0

package/dist/artifacts/inventory/reader.js

@@ -3,38 +3,61 @@ import { DirectoryArtifactReader } from "../DirectoryArtifactReader.js";
 import { MachOSliceArtifactReader } from "../MachOSliceArtifactReader.js";
 import { NativeDmgArtifactReader } from "../NativeDmgArtifactReader.js";
 import { ZipArtifactReader } from "../ZipArtifactReader.js";
-export const createReader = async (path, format, environment, signal) => {
+import { isZipPackageFormat, ZIP_NON_ENTRY_LIMITATION, } from "../../domain/zipPackageFormat.js";
+export const createReader = (path, format, environment, rootSource) => {
+    if (isZipPackageFormat(format))
+        return new ZipArtifactReader(path, format, undefined, rootSource);
     switch (format) {
         case "directory":
             return new DirectoryArtifactReader(path);
-        case "zip":
-        case "ipa":
-        case "apk":
-        case "msix":
-        case "appx":
-            return new ZipArtifactReader(path, format);
         case "asar":
             return new AsarArtifactReader(path);
         case "mach-o-universal":
             return process.platform === "darwin"
-                ? new MachOSliceArtifactReader(path, environment)
+                ? new MachOSliceArtifactReader(path, environment, undefined, rootSource)
                 : undefined;
         case "dmg":
             if (process.platform !== "darwin")
                 return undefined;
-            return NativeDmgArtifactReader.create(path, environment, signal);
+            return new NativeDmgArtifactReader(path, environment);
         default:
             return undefined;
     }
 };
-export const inventoryLimitations = (format, reader) => {
+/** Decide traversal scope and availability once, independently of diagnostic wording. */
+export const inventoryCoverage = (format, reader, traversal) => {
     if (reader !== undefined)
-        return [];
+        return {
+            coverage: { scope: "container-entries", status: traversal.status },
+            limitations: [
+                ...(isZipPackageFormat(reader.format)
+                    ? [ZIP_NON_ENTRY_LIMITATION]
+                    : []),
+                ...traversal.limitations,
+            ],
+        };
+    // dmg and pkg reach here only after classifyRootFormat matched their bytes.
     if (format === "dmg" || format === "pkg")
-        return [
-            `${format.toUpperCase()} root hash is observed; child inventory requires a native macOS adapter.`,
-        ];
-    if (format === "mach-o-universal" && process.platform !== "darwin")
-        return ["Universal Mach-O slices require the native macOS lipo adapter."];
-    return ["Artifact has no child container entries."];
+        return {
+            coverage: { scope: "container-entries", status: "unknown" },
+            limitations: [
+                `${format.toUpperCase()} root hash is observed; child inventory requires a native macOS adapter.`,
+            ],
+        };
+    if (format === "mach-o-universal")
+        return {
+            coverage: { scope: "container-entries", status: "unknown" },
+            limitations: [
+                "Universal Mach-O slices require the native macOS lipo adapter.",
+            ],
+        };
+    if (isZipPackageFormat(format) || format === "directory" || format === "asar")
+        return {
+            coverage: { scope: "container-entries", status: "unknown" },
+            limitations: [`${format} child inventory reader is unavailable.`],
+        };
+    return {
+        coverage: { scope: "root", status: "complete" },
+        limitations: ["Artifact has no child container entries."],
+    };
 };

package/dist/artifacts/inventory/scanCanonical.js

@@ -3,25 +3,35 @@ import { AsarArtifactReader } from "../AsarArtifactReader.js";
 import { lstat } from "node:fs/promises";
 import { ArtifactReaderFailure, } from "../ArtifactReader.js";
 import { artifactInventoryResultSchema, } from "../../domain/artifactGraph.js";
-import { createArtifactEdges, createRootNode, indexOccurrencesByPath, materializeDirectoryNodes, rekeyOccurrences, rootOccurrenceFor, } from "./ArtifactGraphConstruction.js";
+import { createArtifactEdges, createRootNode, materializeDirectoryNodes, rekeyOccurrences, rootOccurrenceFor, } from "./ArtifactGraphConstruction.js";
 import { artifactContradictionId, artifactGraphDigest, artifactManifestId, } from "../../domain/artifactIdentity.js";
-import { classifyAndHashRoot } from "./classify.js";
-import {} from "../ArtifactHash.js";
-import { hashStableRootArtifact } from "./hashStableRootArtifact.js";
-import { createReader, inventoryLimitations } from "./reader.js";
+import { classifyAndHashRootForInventory } from "./classify.js";
+import { hostExecutableMarking } from "../hostExecutableMarking.js";
+import { hashStableRootArtifact, hashStableRootArtifactHandle, } from "./hashStableRootArtifact.js";
+import { createReader, inventoryCoverage } from "./reader.js";
 import { scanReader, } from "./scanReader.js";
 import { STRICT_INTEGRITY_POLICY, } from "./types.js";
-export const scanCanonicalArtifactInventory = async (path, options = {}, readerFactory = createReader) => {
+export const scanCanonicalArtifactInventory = async (path, options, readerFactory = createReader) => options.resourceScope.run(() => scanCanonicalArtifactInventoryInScope(path, options, readerFactory));
+export const scanCanonicalArtifactInventoryInScope = async (path, options, readerFactory = createReader) => {
     const integrity = options.integrity ?? STRICT_INTEGRITY_POLICY;
     const metadata = await lstat(path);
-    const { format: rootFormat, digest: rootDigest } = await classifyAndHashRoot(path, metadata.isDirectory(), metadata, options.signal);
-    const reader = await readerFactory(path, rootFormat, options.environment ?? {}, options.signal);
+    const { format: rootFormat, digest: rootDigest, rootSource, } = await classifyAndHashRootForInventory(path, metadata.isDirectory(), metadata, options);
+    let reader;
+    let readerCreationFailure;
+    try {
+        reader = readerFactory(path, rootFormat, options.environment ?? {}, rootSource);
+    }
+    catch (cause) {
+        readerCreationFailure = { cause };
+    }
     const ownedReaders = reader === undefined ? [] : [reader];
     let outcome;
     try {
+        if (readerCreationFailure !== undefined)
+            throw readerCreationFailure.cause;
         if (reader instanceof AsarArtifactReader)
             await reader.prepareContainer(rootDigest?.sha256, options.signal);
-        const { nodes, occurrences, pendingContradictions } = await scanReader(reader, ownedReaders, options.signal, integrity);
+        const { nodes, occurrences, pendingContradictions, caseCollisionLimitation, unknownParentOccurrences, traversal, } = await scanReader(reader, ownedReaders, options.signal, integrity);
         outcome = {
             kind: "completed",
             snapshot: await buildInventorySnapshot({
@@ -29,38 +39,64 @@ export const scanCanonicalArtifactInventory = async (path, options = {}, readerF
                 metadata,
                 rootFormat,
                 rootDigest,
+                rootSource,
+                resourceScope: options.resourceScope,
                 reader,
                 signal: options.signal,
                 nodes,
                 occurrences,
                 pendingContradictions,
+                caseCollisionLimitation,
+                unknownParentOccurrences,
+                traversal,
             }),
         };
     }
     catch (cause) {
         outcome = { kind: "failed", cause };
     }
-    let cleanupFailure;
-    for (const owned of ownedReaders.reverse()) {
-        try {
-            await owned.close();
-        }
-        catch (cleanupCause) {
-            const cleanup = ArtifactReaderFailure.cleanupObservation(cleanupCause, `artifact reader for ${path}`);
-            cleanupFailure = ArtifactReaderFailure.withCleanup(cleanupFailure ??
-                (outcome.kind === "failed" ? outcome.cause : cleanupCause), cleanup, outcome.kind === "completed"
-                ? { kind: "artifact-inventory", inventory: outcome.snapshot }
-                : undefined);
-        }
-    }
+    const cleanupFailure = await cleanupInventoryOwners({
+        path,
+        options,
+        rootSource,
+        ownedReaders,
+        outcome,
+    });
     if (cleanupFailure !== undefined)
         throw cleanupFailure;
     if (outcome.kind === "failed")
         throw outcome.cause;
     return outcome.snapshot;
 };
+const cleanupInventoryOwners = async ({ path, options, rootSource, ownedReaders, outcome, }) => {
+    let cleanupFailure;
+    const owners = [...ownedReaders]
+        .reverse()
+        .map((reader) => ({
+        kind: "reader",
+        reader,
+        resource: `artifact reader for ${path}`,
+    }));
+    if (rootSource !== undefined)
+        owners.push({
+            kind: "file-handle",
+            handle: rootSource.owner,
+            resource: `root artifact descriptor for ${path}`,
+        });
+    for (const owner of owners) {
+        const attempt = await options.resourceScope.release(owner);
+        if (attempt.kind === "released")
+            continue;
+        const primary = cleanupFailure ??
+            (outcome.kind === "failed" ? outcome.cause : attempt.cause);
+        cleanupFailure = ArtifactReaderFailure.withCleanup(primary, ArtifactReaderFailure.cleanupObservation(attempt.cause, owner.resource), outcome.kind === "completed"
+            ? { kind: "artifact-inventory", inventory: outcome.snapshot }
+            : undefined);
+    }
+    return cleanupFailure;
+};
 const buildInventorySnapshot = async (input) => {
-    const { path, metadata, rootFormat, rootDigest, reader, signal } = input;
+    const { path, metadata, rootFormat, rootDigest, rootSource, reader, signal } = input;
     materializeDirectoryNodes(input.occurrences, input.nodes);
     const rootNode = createRootNode({
         digest: rootDigest,
@@ -70,28 +106,29 @@ const buildInventorySnapshot = async (input) => {
     rekeyOccurrences(rootNode.artifact_id, input.occurrences);
     const rootOccurrence = rootOccurrenceFor(rootNode, {
         size: metadata.size,
-        executable: (metadata.mode & 0o111) !== 0,
+        executable: hostExecutableMarking(path, metadata),
         format: rootFormat,
         path,
     });
     for (const occurrence of input.occurrences)
-        if (occurrence.parent_occurrence_id === null)
+        if (occurrence.parent_occurrence_id === null &&
+            !input.unknownParentOccurrences.has(occurrence))
             occurrence.parent_occurrence_id = rootOccurrence.occurrence_id;
     input.occurrences.unshift(rootOccurrence);
     const occurrenceById = new Map(input.occurrences.map((occurrence) => [
         occurrence.occurrence_id,
         occurrence,
     ]));
-    const occurrenceByPath = indexOccurrencesByPath(input.occurrences);
-    const integrityContradictions = buildIntegrityContradictions(input.pendingContradictions, occurrenceByPath, occurrenceById, rootNode);
-    const edges = createArtifactEdges(rootNode.artifact_id, input.occurrences, reader?.provenance()[0]);
+    const integrityContradictions = buildIntegrityContradictions(input.pendingContradictions, occurrenceById, rootNode);
+    const edgeResult = createArtifactEdges(rootNode.artifact_id, input.occurrences, reader?.provenance()[0]);
+    const { edges } = edgeResult;
     const orderedNodes = sortNodes([...input.nodes.values()]);
     const orderedOccurrences = sortOccurrences(input.occurrences);
     const orderedEdges = sortEdges(edges).map((edge, ordinal) => ({
         ...edge,
         ordinal,
     }));
-    await verifyRootDigest(path, rootDigest, signal);
+    await verifyRootDigest(path, rootDigest, rootSource, input.resourceScope, signal);
     const graphSha256 = artifactGraphDigest({
         nodes: orderedNodes,
         occurrences: orderedOccurrences,
@@ -106,6 +143,10 @@ const buildInventorySnapshot = async (input) => {
         orderedOccurrences,
         orderedEdges,
     });
+    const traversal = inventoryCoverage(rootFormat, reader, input.traversal);
+    const coverage = edgeResult.limitations.length === 0
+        ? traversal.coverage
+        : { ...traversal.coverage, status: "unknown" };
     return {
         manifest,
         nodes: orderedNodes,
@@ -113,13 +154,13 @@ const buildInventorySnapshot = async (input) => {
         edges: orderedEdges,
         provenance: reader?.provenance() ?? [],
         integrity_contradictions: integrityContradictions,
-        limitations: buildLimitations(rootFormat, reader, integrityContradictions),
+        coverage,
+        coverage_status: coverage.status === "complete" ? "complete" : "partial",
+        limitations: buildLimitations([...traversal.limitations, ...edgeResult.limitations], integrityContradictions, input.caseCollisionLimitation),
     };
 };
-const buildIntegrityContradictions = (pending, occurrenceByPath, occurrenceById, rootNode) => pending.map((contradiction) => {
-    const occurrence = occurrenceByPath.get(contradiction.logicalPath);
-    if (occurrence === undefined)
-        throw new ArtifactReaderFailure("integrity", "Integrity contradiction lost its graph occurrence");
+const buildIntegrityContradictions = (pending, occurrenceById, rootNode) => pending.map((contradiction) => {
+    const occurrence = contradiction.occurrence;
     const parent = occurrence.parent_occurrence_id === null
         ? undefined
         : occurrenceById.get(occurrence.parent_occurrence_id);
@@ -127,13 +168,13 @@ const buildIntegrityContradictions = (pending, occurrenceByPath, occurrenceById,
     return {
         contradiction_id: artifactContradictionId({
             rootArtifactId: rootNode.artifact_id,
-            logicalPath: contradiction.logicalPath,
+            logicalPath: occurrence.logical_path,
             declaredSha256: contradiction.declaredSha256,
             observedSha256: contradiction.observedSha256,
         }),
         occurrence_id: occurrence.occurrence_id,
         parent_artifact_id: parentArtifactId,
-        logical_path: contradiction.logicalPath,
+        logical_path: occurrence.logical_path,
         declared_sha256: contradiction.declaredSha256,
         observed_sha256: contradiction.observedSha256,
         entry_kind: contradiction.entryKind,
@@ -145,10 +186,12 @@ const buildIntegrityContradictions = (pending, occurrenceByPath, occurrenceById,
         ],
     };
 });
-const verifyRootDigest = async (path, rootDigest, signal) => {
+const verifyRootDigest = async (path, rootDigest, rootSource, resourceScope, signal) => {
     if (rootDigest === null)
         return;
-    const verified = await hashStableRootArtifact(path, signal);
+    const verified = rootSource === undefined
+        ? await hashStableRootArtifact(path, resourceScope, signal)
+        : await hashStableRootArtifactHandle(path, rootSource.handle, rootSource.initial, signal);
     if (verified.sha256 !== rootDigest.sha256 ||
         verified.bytes !== rootDigest.bytes)
         throw new ArtifactReaderFailure("integrity", "Root artifact changed during inventory");
@@ -166,8 +209,9 @@ const buildManifest = ({ rootNode, rootFormat, graphSha256, orderedNodes, ordere
     occurrence_count: orderedOccurrences.length,
     edge_count: orderedEdges.length,
 });
-const buildLimitations = (rootFormat, reader, integrityContradictions) => [
-    ...inventoryLimitations(rootFormat, reader),
+const buildLimitations = (scopeLimitations, integrityContradictions, caseCollisionLimitation) => [
+    ...scopeLimitations,
+    ...(caseCollisionLimitation === undefined ? [] : [caseCollisionLimitation]),
     ...(integrityContradictions.length === 0
         ? []
         : [

package/dist/artifacts/inventory/scanReader.js

@@ -1,8 +1,9 @@
 import { isAbsolute } from "node:path";
-import { ArtifactPathRegistry, normalizeArtifactPath, } from "../ArtifactPaths.js";
+import { annotateEnglishUnicodeCaseCollisions, ArtifactPathRegistry, normalizeArtifactPath, } from "../ArtifactPaths.js";
 import { ArtifactReaderFailure, } from "../ArtifactReader.js";
 import { AsarArtifactReader } from "../AsarArtifactReader.js";
-import { classifyArtifactContent, classifyArtifactBytes, createArtifactNode, createOccurrence, nearestParent, } from "./ArtifactGraphConstruction.js";
+import { compareUnicodeCodePoints } from "../../domain/unicodeCodePointOrder.js";
+import { classifyArtifactContent, classifyArtifactBytes, createArtifactNode, createOccurrence, noteUnreadContainerSuffix, indexOccurrencesByPath, resolveOccurrenceParents, } from "./ArtifactGraphConstruction.js";
 import { STRICT_INTEGRITY_POLICY, } from "./types.js";
 import { hashReadable } from "../ArtifactHash.js";
 const UNAVAILABLE_UNPACKED_LIMITATION = "ASAR unpacked companion bytes were unavailable; no content hash or child artifact was produced.";
@@ -10,7 +11,14 @@ const emptyScan = () => ({ nodes: new Map(), occurrences: [], pendingContradicti
 export const scanReader = async (reader, ownedReaders, signal, integrity = STRICT_INTEGRITY_POLICY) => {
     const { nodes, occurrences, pendingContradictions } = emptyScan();
     if (reader === undefined)
-        return { nodes, occurrences, pendingContradictions };
+        return {
+            nodes,
+            occurrences,
+            pendingContradictions,
+            caseCollisionLimitation: undefined,
+            unknownParentOccurrences: new Set(),
+            traversal: { status: "complete", limitations: [] },
+        };
     const context = {
         reader,
         signal,
@@ -18,27 +26,53 @@ export const scanReader = async (reader, ownedReaders, signal, integrity = STRIC
         nodes,
         occurrences,
         pendingContradictions,
-        occurrenceByPath: new Map(),
+        occurrenceCountByPath: new Map(),
         registry: new ArtifactPathRegistry(),
         expandedContainerIds: new Set(),
         ownedReaders,
+        omissions: [],
     };
     await visitArtifactEntries(context, reader, "");
     // Archive directories may appear after their children. Resolve containment
     // against the complete index before directory identities are materialized.
-    for (const occurrence of occurrences)
-        occurrence.parent_occurrence_id =
-            nearestParent(occurrence.logical_path, context.occurrenceByPath, context.expandedContainerIds)?.occurrence_id ?? null;
-    return { nodes, occurrences, pendingContradictions };
+    const byPath = indexOccurrencesByPath(occurrences);
+    const parentResolution = resolveOccurrenceParents(occurrences, byPath, context.expandedContainerIds);
+    const structuralLimitations = [...parentResolution.limitations];
+    return {
+        nodes,
+        occurrences,
+        pendingContradictions,
+        caseCollisionLimitation: annotateEnglishUnicodeCaseCollisions(occurrences),
+        unknownParentOccurrences: parentResolution.unknownParentOccurrences,
+        traversal: {
+            status: context.omissions.length === 0 && structuralLimitations.length === 0
+                ? "complete"
+                : "unknown",
+            limitations: [
+                ...new Set([
+                    ...context.omissions.map(({ path, kind }) => `Artifact traversal omitted ${kind} at ${JSON.stringify(path)}.`),
+                    ...structuralLimitations,
+                ]),
+            ].sort(compareUnicodeCodePoints),
+        },
+    };
 };
+const inventoryEntries = (context, reader, prefix) => reader
+    .entries(context.signal, {
+    onOmission: (omission) => context.omissions.push({
+        ...omission,
+        path: normalizeArtifactPath(prefix.length === 0 ? omission.path : `${prefix}/${omission.path}`),
+    }),
+})[Symbol.asyncIterator]();
 const visitArtifactEntries = async (context, currentReader, prefix) => {
     const stack = [
         {
             reader: currentReader,
             prefix,
-            iterator: currentReader.entries(context.signal)[Symbol.asyncIterator](),
+            iterator: inventoryEntries(context, currentReader, prefix),
         },
     ];
+    let failure;
     try {
         while (stack.length > 0) {
             const frame = stack.at(-1);
@@ -55,10 +89,12 @@ const visitArtifactEntries = async (context, currentReader, prefix) => {
                 : `${frame.prefix}/${entry.path}`);
             const expandableAsar = isExpandableAsar(entry, logicalPath);
             context.registry.add(logicalPath, expandableAsar ? "directory" : entry.kind);
-            const occurrence = createOccurrence(entry, logicalPath, null);
+            const duplicateOrdinal = context.occurrenceCountByPath.get(logicalPath) ?? 0;
+            context.occurrenceCountByPath.set(logicalPath, duplicateOrdinal + 1);
+            const occurrence = createOccurrence(entry, logicalPath, null, duplicateOrdinal);
             let digested;
             try {
-                digested = await digestArtifactEntry(context, frame.reader, entry, logicalPath);
+                digested = await digestArtifactEntry(context, frame.reader, entry, logicalPath, occurrence);
             }
             catch (cause) {
                 if (!isUnavailableUnpackedEntry(cause, entry))
@@ -66,7 +102,9 @@ const visitArtifactEntries = async (context, currentReader, prefix) => {
                 occurrence.hash_status = "unavailable";
                 occurrence.limitations.push(UNAVAILABLE_UNPACKED_LIMITATION);
             }
-            if (digested !== undefined) {
+            if (digested === undefined)
+                noteUnreadContainerSuffix(occurrence);
+            else {
                 context.nodes.set(digested.node.artifact_id, digested.node);
                 occurrence.artifact_id = digested.node.artifact_id;
                 occurrence.artifact_kind = digested.classification.kind;
@@ -78,8 +116,10 @@ const visitArtifactEntries = async (context, currentReader, prefix) => {
                     occurrence.limitations.push("Declared integrity metadata contradicts observed bytes.");
             }
             context.occurrences.push(occurrence);
-            context.occurrenceByPath.set(logicalPath, occurrence);
-            if (expandableAsar && digested !== undefined && !digested.mismatched) {
+            if (expandableAsar &&
+                digested !== undefined &&
+                digested.classification.format === "asar" &&
+                !digested.mismatched) {
                 const nested = new AsarArtifactReader(entry.adapterKey);
                 context.ownedReaders.push(nested);
                 await nested.prepareContainer(digested.node.sha256, context.signal);
@@ -88,18 +128,31 @@ const visitArtifactEntries = async (context, currentReader, prefix) => {
                 stack.push({
                     reader: nested,
                     prefix: logicalPath,
-                    iterator: nested.entries(context.signal)[Symbol.asyncIterator](),
+                    iterator: inventoryEntries(context, nested, logicalPath),
                 });
             }
         }
     }
+    catch (cause) {
+        failure = { cause };
+    }
     finally {
         // Readers remain owned by the inventory until its complete observation is built.
-        for (const { iterator } of stack)
-            await iterator.return?.();
+        for (const { iterator, prefix, reader } of stack.reverse()) {
+            try {
+                await iterator.return?.();
+            }
+            catch (cause) {
+                failure = {
+                    cause: ArtifactReaderFailure.withCleanup(failure === undefined ? cause : failure.cause, ArtifactReaderFailure.cleanupObservation(cause, `artifact traversal for ${prefix || reader.format}`)),
+                };
+            }
+        }
     }
+    if (failure !== undefined)
+        throw failure.cause;
 };
-const digestArtifactEntry = async (context, currentReader, entry, logicalPath) => {
+const digestArtifactEntry = async (context, currentReader, entry, logicalPath, occurrence) => {
     if ((entry.kind !== "file" && entry.kind !== "slice") || entry.encrypted)
         return undefined;
     const digest = await hashReadable(await currentReader.open(entry, context.signal), context.signal);
@@ -113,7 +166,7 @@ const digestArtifactEntry = async (context, currentReader, entry, logicalPath) =
                 unpacked: entry.unpacked,
             });
         context.pendingContradictions.push({
-            logicalPath,
+            occurrence,
             declaredSha256: entry.declaredSha256,
             observedSha256: digest.sha256,
             entryKind: entry.kind,