project-tiny-context-harness 0.8.17 → 0.11.0

package/dist/lib/context-markdown/context-markdown-analysis.js

@@ -0,0 +1,245 @@
+import { lstat, readFile, realpath } from "node:fs/promises";
+import path from "node:path";
+import { compareUtf8Paths, isPathWithin, normalizeContextPath, } from "../context-catalog/catalog-paths.js";
+import { extractContextMarkdown } from "./context-markdown-extract.js";
+export async function analyzeContextMarkdownCatalog(input) {
+    const files = [];
+    const catalogFiles = new Map(input.files.map((file) => [file.path, file]));
+    for (const file of [...input.files].sort((left, right) => compareUtf8Paths(left.path, right.path)))
+        files.push(await analyzeContextMarkdownFile({
+            project_root: input.project_root,
+            file,
+            catalog_files: catalogFiles,
+            long_line_threshold: input.long_line_threshold,
+            file_overrides: input.file_overrides,
+        }));
+    const declarations = files.flatMap((file) => file.declarations);
+    return {
+        files,
+        references: files.flatMap((file) => file.references),
+        declarations,
+        invalid_declarations: files.flatMap((file) => file.invalid_declarations),
+        declaration_conflicts: declarationConflicts(declarations),
+    };
+}
+export async function analyzeContextMarkdownFile(input) {
+    const override = input.file_overrides?.get(input.file.path);
+    const content = override === undefined
+        ? await readFile(input.file.absolute_path, "utf8")
+        : Buffer.from(override ?? []).toString("utf8");
+    const extracted = extractContextMarkdown(content, input.file.path);
+    const lengths = lineLengths(content, input.long_line_threshold);
+    const references = [];
+    for (const reference of extracted.references) {
+        const resolved = await resolveReference(input.project_root, input.file, reference, input.catalog_files ?? new Map([[input.file.path, input.file]]), input.file_overrides);
+        if (resolved)
+            references.push(resolved);
+    }
+    return {
+        path: input.file.path,
+        bytes: Buffer.byteLength(content, "utf8"),
+        max_line_code_points: lengths.maximum,
+        long_lines: lengths.long,
+        references,
+        declarations: extracted.declarations,
+        invalid_declarations: extracted.invalid_declarations,
+    };
+}
+async function resolveReference(projectRoot, sourceFile, reference, catalogFiles, fileOverrides) {
+    const sourcePath = sourceFile.path;
+    const split = splitDestination(reference.destination);
+    if (split.disposition === "ignored")
+        return null;
+    if (split.disposition === "invalid")
+        return {
+            ...reference,
+            source_path: sourcePath,
+            target_path: null,
+            fragment: split.fragment,
+            status: "invalid",
+            detail: split.detail,
+        };
+    const normalizedDestination = split.local.replaceAll("\\", "/");
+    const lexicalAbsolute = normalizedDestination.startsWith("/")
+        ? path.resolve(projectRoot, normalizedDestination.slice(1))
+        : normalizedDestination.startsWith("project_context/")
+            ? path.resolve(projectRoot, normalizedDestination)
+            : path.resolve(path.dirname(sourceFile.absolute_path), normalizedDestination);
+    const targetPath = normalizeContextPath(path.relative(projectRoot, lexicalAbsolute));
+    if (!isPathWithin(projectRoot, lexicalAbsolute))
+        return {
+            ...reference,
+            source_path: sourcePath,
+            target_path: targetPath,
+            fragment: split.fragment,
+            status: "outside_repository",
+            detail: "local Markdown destination resolves outside the repository",
+        };
+    const catalogTarget = catalogFiles.get(targetPath);
+    if (fileOverrides?.has(targetPath)) {
+        if (fileOverrides.get(targetPath) === null)
+            return {
+                ...reference,
+                source_path: sourcePath,
+                target_path: targetPath,
+                fragment: split.fragment,
+                status: "missing",
+            };
+        if (!catalogTarget ||
+            !sameStagedAbsolutePath(lexicalAbsolute, catalogTarget.absolute_path))
+            return {
+                ...reference,
+                source_path: sourcePath,
+                target_path: targetPath,
+                fragment: split.fragment,
+                status: "missing",
+                detail: "local Markdown destination does not resolve to the staged target physical path",
+            };
+        return {
+            ...reference,
+            source_path: sourcePath,
+            target_path: targetPath,
+            fragment: split.fragment,
+            status: "valid",
+        };
+    }
+    try {
+        await lstat(lexicalAbsolute);
+    }
+    catch (error) {
+        if (isMissing(error))
+            return {
+                ...reference,
+                source_path: sourcePath,
+                target_path: targetPath,
+                fragment: split.fragment,
+                status: "missing",
+            };
+        throw error;
+    }
+    const identity = await realpath(lexicalAbsolute);
+    if (!isPathWithin(await realpath(projectRoot), identity))
+        return {
+            ...reference,
+            source_path: sourcePath,
+            target_path: targetPath,
+            fragment: split.fragment,
+            status: "outside_repository",
+            detail: "local Markdown destination resolves through a link outside the repository",
+        };
+    if (catalogTarget) {
+        let catalogIdentity;
+        try {
+            catalogIdentity = await realpath(catalogTarget.absolute_path);
+        }
+        catch (error) {
+            if (isMissing(error))
+                return {
+                    ...reference,
+                    source_path: sourcePath,
+                    target_path: targetPath,
+                    fragment: split.fragment,
+                    status: "missing",
+                };
+            throw error;
+        }
+        if (!sameRealPath(identity, catalogIdentity))
+            return {
+                ...reference,
+                source_path: sourcePath,
+                target_path: targetPath,
+                fragment: split.fragment,
+                status: "invalid",
+                detail: "local Markdown destination resolves to a different physical file than its Catalog identity",
+            };
+    }
+    return {
+        ...reference,
+        source_path: sourcePath,
+        target_path: targetPath,
+        fragment: split.fragment,
+        status: "valid",
+    };
+}
+function sameStagedAbsolutePath(left, right) {
+    const normalizedLeft = path.resolve(left);
+    const normalizedRight = path.resolve(right);
+    return process.platform === "win32"
+        ? normalizedLeft.toLowerCase() === normalizedRight.toLowerCase()
+        : normalizedLeft === normalizedRight;
+}
+function sameRealPath(left, right) {
+    return path.relative(left, right) === "";
+}
+function splitDestination(value) {
+    const trimmed = value.trim();
+    if (!trimmed || trimmed.startsWith("#"))
+        return {
+            disposition: "ignored",
+            fragment: trimmed.slice(1) || null,
+        };
+    if (trimmed.startsWith("//") || /^[A-Za-z][A-Za-z0-9+.-]*:/u.test(trimmed))
+        return { disposition: "ignored", fragment: null };
+    const fragmentIndex = trimmed.indexOf("#");
+    const queryIndex = trimmed.indexOf("?");
+    const boundary = [fragmentIndex, queryIndex]
+        .filter((entry) => entry >= 0)
+        .reduce((lowest, entry) => Math.min(lowest, entry), trimmed.length);
+    const encodedPath = trimmed.slice(0, boundary);
+    const fragment = fragmentIndex >= 0 ? trimmed.slice(fragmentIndex + 1) : null;
+    try {
+        return {
+            disposition: "local",
+            local: decodeURIComponent(encodedPath),
+            fragment,
+        };
+    }
+    catch {
+        return {
+            disposition: "invalid",
+            fragment,
+            detail: "local Markdown destination contains invalid URL encoding",
+        };
+    }
+}
+function lineLengths(content, threshold) {
+    let maximum = 0;
+    const long = [];
+    for (const [index, line] of content.split(/\r\n?|\n/u).entries()) {
+        const codePoints = Array.from(line).length;
+        maximum = Math.max(maximum, codePoints);
+        if (codePoints > threshold)
+            long.push({ line: index + 1, code_points: codePoints });
+    }
+    return { maximum, long };
+}
+function declarationConflicts(declarations) {
+    const groups = new Map();
+    for (const declaration of declarations) {
+        const key = `${declaration.type}\0${declaration.id}`;
+        groups.set(key, [...(groups.get(key) ?? []), declaration]);
+    }
+    const conflicts = [];
+    for (const declarationsForKey of groups.values()) {
+        const owners = new Map();
+        for (const declaration of declarationsForKey)
+            if (!owners.has(declaration.path))
+                owners.set(declaration.path, declaration.line);
+        if (owners.size < 2)
+            continue;
+        conflicts.push({
+            type: declarationsForKey[0].type,
+            id: declarationsForKey[0].id,
+            owners: [...owners.entries()]
+                .sort(([left], [right]) => compareUtf8Paths(left, right))
+                .map(([ownerPath, line]) => ({ path: ownerPath, line })),
+        });
+    }
+    return conflicts.sort((left, right) => compareUtf8Paths(`${left.type}:${left.id}`, `${right.type}:${right.id}`));
+}
+function isMissing(error) {
+    return (typeof error === "object" &&
+        error !== null &&
+        "code" in error &&
+        error.code === "ENOENT");
+}

package/dist/lib/context-markdown/context-markdown-extract.d.ts

@@ -0,0 +1,7 @@
+import type { ContextInvalidDeclaration, ContextMarkdownRawReference, ContextStableKeyDeclaration } from "./context-markdown-types.js";
+export interface ContextMarkdownExtraction {
+    references: ContextMarkdownRawReference[];
+    declarations: ContextStableKeyDeclaration[];
+    invalid_declarations: ContextInvalidDeclaration[];
+}
+export declare function extractContextMarkdown(content: string, sourcePath: string): ContextMarkdownExtraction;

package/dist/lib/context-markdown/context-markdown-extract.js

@@ -0,0 +1,114 @@
+import { fromMarkdown } from "mdast-util-from-markdown";
+const DECLARATION = /^<!--\s*ty-context-declare\s+([A-Z][A-Za-z0-9]*-ID):\s*([A-Z][A-Z0-9]*(?:-[A-Z0-9]+)+)\s*-->$/u;
+export function extractContextMarkdown(content, sourcePath) {
+    const tree = fromMarkdown(content);
+    const references = [];
+    const declarations = [];
+    const invalidDeclarations = [];
+    walk(tree, (node) => {
+        const line = node.position?.start.line ?? 1;
+        const column = node.position?.start.column ?? 1;
+        if (node.type === "link")
+            references.push({
+                destination: node.url,
+                kind: "inline",
+                line,
+                column,
+            });
+        else if (node.type === "image")
+            references.push({
+                destination: node.url,
+                kind: "image",
+                line,
+                column,
+            });
+        else if (node.type === "definition")
+            references.push({
+                destination: node.url,
+                kind: "definition",
+                line,
+                column,
+            });
+        else if (node.type === "text") {
+            for (const angle of angleDestinations(node.value))
+                references.push({
+                    destination: angle.value,
+                    kind: "angle",
+                    line,
+                    column: column + angle.code_point_offset,
+                });
+        }
+        else if (node.type === "html") {
+            const raw = node.value.trim();
+            const declaration = DECLARATION.exec(raw);
+            if (declaration)
+                declarations.push({
+                    type: declaration[1],
+                    id: declaration[2],
+                    path: sourcePath,
+                    line,
+                    column,
+                });
+            else if (raw.includes("ty-context-declare"))
+                invalidDeclarations.push({
+                    path: sourcePath,
+                    raw,
+                    line,
+                    column,
+                    reason: "declaration must be an exact HTML comment with <Type-ID>: <UPPERCASE-ID-PARTS>",
+                });
+            else if (raw.startsWith("<") && raw.endsWith(">")) {
+                const destination = raw.slice(1, -1).trim();
+                if (looksLikeLocalAngle(destination))
+                    references.push({
+                        destination,
+                        kind: "angle",
+                        line,
+                        column,
+                    });
+            }
+        }
+    });
+    return {
+        references,
+        declarations,
+        invalid_declarations: invalidDeclarations,
+    };
+}
+function walk(node, visit) {
+    if (node.type !== "root")
+        visit(node);
+    if (!("children" in node))
+        return;
+    for (const child of node.children)
+        walk(child, visit);
+}
+function angleDestinations(value) {
+    const result = [];
+    const points = Array.from(value);
+    for (let index = 0; index < points.length; index += 1) {
+        if (points[index] !== "<")
+            continue;
+        const end = points.indexOf(">", index + 1);
+        if (end < 0)
+            break;
+        const candidate = points
+            .slice(index + 1, end)
+            .join("")
+            .trim();
+        if (looksLikeLocalAngle(candidate))
+            result.push({ value: candidate, code_point_offset: index });
+        index = end;
+    }
+    return result;
+}
+function looksLikeLocalAngle(value) {
+    if (!value || Array.from(value).some((point) => /\s/u.test(point)))
+        return false;
+    return (value.startsWith(".") ||
+        value.startsWith("/") ||
+        value.startsWith("project_context") ||
+        value.includes("/") ||
+        value.includes("\\") ||
+        value.toLowerCase().endsWith(".md"));
+}

package/dist/lib/context-markdown/context-markdown-types.d.ts

@@ -0,0 +1,57 @@
+export type ContextMarkdownReferenceKind = "inline" | "image" | "definition" | "angle";
+export type ContextMarkdownReferenceStatus = "valid" | "missing" | "outside_repository" | "invalid";
+export interface ContextMarkdownRawReference {
+    destination: string;
+    kind: ContextMarkdownReferenceKind;
+    line: number;
+    column: number;
+}
+export interface ContextMarkdownReference extends ContextMarkdownRawReference {
+    source_path: string;
+    target_path: string | null;
+    fragment: string | null;
+    status: ContextMarkdownReferenceStatus;
+    detail?: string;
+}
+export interface ContextStableKeyDeclaration {
+    type: string;
+    id: string;
+    path: string;
+    line: number;
+    column: number;
+}
+export interface ContextInvalidDeclaration {
+    path: string;
+    raw: string;
+    line: number;
+    column: number;
+    reason: string;
+}
+export interface ContextLongLine {
+    line: number;
+    code_points: number;
+}
+export interface ContextMarkdownFileAnalysis {
+    path: string;
+    bytes: number;
+    max_line_code_points: number;
+    long_lines: ContextLongLine[];
+    references: ContextMarkdownReference[];
+    declarations: ContextStableKeyDeclaration[];
+    invalid_declarations: ContextInvalidDeclaration[];
+}
+export interface ContextStableKeyConflict {
+    type: string;
+    id: string;
+    owners: Array<{
+        path: string;
+        line: number;
+    }>;
+}
+export interface ContextMarkdownCatalogAnalysis {
+    files: ContextMarkdownFileAnalysis[];
+    references: ContextMarkdownReference[];
+    declarations: ContextStableKeyDeclaration[];
+    invalid_declarations: ContextInvalidDeclaration[];
+    declaration_conflicts: ContextStableKeyConflict[];
+}

package/dist/lib/context-markdown/context-markdown-types.js

@@ -0,0 +1 @@
+export {};

package/dist/lib/context-move/context-move-input.d.ts

@@ -0,0 +1,8 @@
+export interface NormalizedContextMoveInput {
+    from_path: string;
+    to_path: string;
+}
+export declare function normalizeContextMoveInput(input: {
+    from_path: string;
+    to_path: string;
+}): NormalizedContextMoveInput;

package/dist/lib/context-move/context-move-input.js

@@ -0,0 +1,17 @@
+import { CLI_EXIT_CODES, CliCommandError } from "../cli-exit.js";
+import { normalizeContextFilePath } from "../context-create/context-create-path.js";
+export function normalizeContextMoveInput(input) {
+    const from = normalizeContextFilePath(input.from_path, "context move --from");
+    const to = normalizeContextFilePath(input.to_path, "context move --to");
+    if (from === to)
+        invalid("context move --from and --to must be different paths");
+    if (fold(from) === fold(to))
+        invalid("context move does not support case-only path changes");
+    return { from_path: from, to_path: to };
+}
+function fold(value) {
+    return value.normalize("NFC").toLocaleLowerCase("en-US");
+}
+function invalid(message) {
+    throw new CliCommandError(CLI_EXIT_CODES.arguments, message);
+}

package/dist/lib/context-move/context-move-literal-scan.d.ts

@@ -0,0 +1,20 @@
+export interface ContextMoveLiteralMatch {
+    path: string;
+    line: number;
+    column: number;
+    matched: string;
+    kind: "markdown_non_link" | "code_or_config";
+}
+export interface ContextMoveLiteralScan {
+    complete: boolean;
+    files_scanned: number;
+    bytes_scanned: number;
+    matches: ContextMoveLiteralMatch[];
+    limits_exceeded: string[];
+}
+export declare function scanStagedRepositoryForContextPath(input: {
+    repository: string;
+    logical_context_path: string;
+    physical_context_path: string;
+    file_overrides: ReadonlyMap<string, Uint8Array | null>;
+}): Promise<ContextMoveLiteralScan>;

package/dist/lib/context-move/context-move-literal-scan.js

@@ -0,0 +1,213 @@
+import { readdir, readFile } from "node:fs/promises";
+import path from "node:path";
+import { compareUtf8Paths, normalizeContextPath, } from "../context-catalog/catalog-paths.js";
+const MAX_FILES = 4096;
+const MAX_FILE_BYTES = 1024 * 1024;
+const MAX_TOTAL_BYTES = 32 * 1024 * 1024;
+const MAX_MATCHES = 500;
+const SKIPPED_DIRECTORIES = new Set([
+    ".git",
+    "node_modules",
+    "dist",
+    "coverage",
+    ".artifacts",
+    "tmp",
+]);
+const TEXT_EXTENSIONS = new Set([
+    ".md",
+    ".txt",
+    ".toml",
+    ".yaml",
+    ".yml",
+    ".json",
+    ".jsonc",
+    ".ts",
+    ".tsx",
+    ".js",
+    ".jsx",
+    ".mjs",
+    ".cjs",
+    ".py",
+    ".rs",
+    ".go",
+    ".java",
+    ".kt",
+    ".sh",
+    ".ps1",
+    ".bat",
+    ".cmd",
+    ".xml",
+    ".html",
+    ".css",
+    ".scss",
+]);
+export async function scanStagedRepositoryForContextPath(input) {
+    const discovered = await discoverTextFiles(input.repository);
+    const paths = new Map();
+    const limits = new Set(discovered.limits);
+    for (const file of discovered.files) {
+        const existing = paths.get(file.path);
+        if (existing !== undefined && existing !== file.physical_path) {
+            limits.add(`path_unicode_collision:${file.path}`);
+            continue;
+        }
+        paths.set(file.path, file.physical_path);
+    }
+    for (const [file, bytes] of input.file_overrides) {
+        const logical = normalizeContextPath(file);
+        if (bytes === null)
+            paths.delete(logical);
+        else if (isTextCandidate(logical) && !paths.has(logical))
+            paths.set(logical, file);
+    }
+    const ordered = [...paths].sort(([left], [right]) => compareUtf8Paths(left, right));
+    if (ordered.length > MAX_FILES)
+        limits.add(`file_count>${MAX_FILES}`);
+    const matches = [];
+    let filesScanned = 0;
+    let bytesScanned = 0;
+    for (const [file, physicalFile] of ordered.slice(0, MAX_FILES)) {
+        const override = input.file_overrides.get(file);
+        let bytes;
+        try {
+            bytes =
+                override === undefined
+                    ? await readFile(path.join(input.repository, ...physicalFile.split("/")))
+                    : Buffer.from(override ?? []);
+        }
+        catch (error) {
+            limits.add(`read_failed:${file}:${message(error)}`);
+            continue;
+        }
+        if (bytes.length > MAX_FILE_BYTES) {
+            limits.add(`file_bytes>${MAX_FILE_BYTES}:${file}`);
+            continue;
+        }
+        if (bytesScanned + bytes.length > MAX_TOTAL_BYTES) {
+            limits.add(`total_bytes>${MAX_TOTAL_BYTES}`);
+            break;
+        }
+        bytesScanned += bytes.length;
+        filesScanned += 1;
+        let content;
+        try {
+            content = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
+        }
+        catch {
+            continue;
+        }
+        for (const literal of literals(input.logical_context_path, input.physical_context_path)) {
+            let cursor = 0;
+            while ((cursor = content.indexOf(literal, cursor)) >= 0) {
+                if (!hasPathBoundaries(content, cursor, literal.length)) {
+                    cursor += Math.max(1, literal.length);
+                    continue;
+                }
+                const location = lineColumn(content, cursor);
+                matches.push({
+                    path: file,
+                    ...location,
+                    matched: literal,
+                    kind: file.toLowerCase().endsWith(".md")
+                        ? "markdown_non_link"
+                        : "code_or_config",
+                });
+                cursor += Math.max(1, literal.length);
+                if (matches.length >= MAX_MATCHES) {
+                    limits.add(`matches>=${MAX_MATCHES}`);
+                    break;
+                }
+            }
+            if (matches.length >= MAX_MATCHES)
+                break;
+        }
+        if (matches.length >= MAX_MATCHES)
+            break;
+    }
+    return {
+        complete: limits.size === 0,
+        files_scanned: filesScanned,
+        bytes_scanned: bytesScanned,
+        matches,
+        limits_exceeded: [...limits].sort(),
+    };
+}
+async function discoverTextFiles(repository) {
+    const result = [];
+    const limits = new Set();
+    let stopped = false;
+    async function visit(relative) {
+        if (stopped)
+            return;
+        const absolute = relative
+            ? path.join(repository, ...relative.split("/"))
+            : repository;
+        const entries = await readdir(absolute, { withFileTypes: true });
+        for (const entry of entries.sort((left, right) => compareUtf8Paths(left.name, right.name))) {
+            const child = relative ? `${relative}/${entry.name}` : entry.name;
+            if (SKIPPED_DIRECTORIES.has(entry.name) || child === ".codex/work")
+                continue;
+            if (entry.isSymbolicLink()) {
+                limits.add(`symlink_skipped:${child}`);
+                continue;
+            }
+            if (entry.isDirectory()) {
+                await visit(child);
+            }
+            else if (entry.isFile() && isTextCandidate(child)) {
+                result.push({
+                    path: normalizeContextPath(child),
+                    physical_path: child,
+                });
+                if (result.length > MAX_FILES) {
+                    limits.add(`file_count>${MAX_FILES}`);
+                    stopped = true;
+                    return;
+                }
+            }
+        }
+    }
+    await visit("");
+    return { files: result, limits: [...limits] };
+}
+function isTextCandidate(file) {
+    const lower = file.toLowerCase();
+    const extension = path.posix.extname(lower);
+    return (TEXT_EXTENSIONS.has(extension) ||
+        ["makefile", "dockerfile"].includes(path.posix.basename(lower)));
+}
+function literals(logicalPath, physicalPath) {
+    const variants = [logicalPath, physicalPath].flatMap((contextPath) => {
+        const encoded = contextPath.split("/").map(encodeURIComponent).join("/");
+        const windows = contextPath.replaceAll("/", "\\");
+        return [
+            contextPath,
+            `/${contextPath}`,
+            windows,
+            `\\${windows}`,
+            encoded,
+            `/${encoded}`,
+        ];
+    });
+    return [...new Set(variants)]
+        .filter(Boolean)
+        .sort((left, right) => Buffer.byteLength(right, "utf8") - Buffer.byteLength(left, "utf8") ||
+        compareUtf8Paths(left, right));
+}
+function hasPathBoundaries(content, start, length) {
+    const before = start === 0 ? "" : content[start - 1];
+    const after = content[start + length] ?? "";
+    const pathCharacter = /[A-Za-z0-9_.%/\\-]/u;
+    return ((!before || !pathCharacter.test(before)) &&
+        (!after || !pathCharacter.test(after)));
+}
+function lineColumn(content, offset) {
+    const lines = content.slice(0, offset).split(/\r\n?|\n/u);
+    return {
+        line: lines.length,
+        column: Array.from(lines.at(-1) ?? "").length + 1,
+    };
+}
+function message(error) {
+    return error instanceof Error ? error.message : String(error);
+}

package/dist/lib/context-move/context-move-live-validation.d.ts

@@ -0,0 +1,3 @@
+import type { ContextCatalog } from "../context-catalog/catalog-types.js";
+import type { ContextMutationJournal } from "../context-mutation/mutation-types.js";
+export declare function validateLiveContextMove(repository: string, catalog: ContextCatalog, journal: ContextMutationJournal): Promise<void>;