xcoder-dna 0.0.0-stage → 1.3.0

package/dist/tools/siteCrawlerTool.js

@@ -0,0 +1,205 @@
+// xcoder:version 2 | xcoder:task task-456d3f | xcoder:updated 2026-08-03T16:08:43.709Z | xcoder:subtask code-st-97a4d4
+import * as cheerio from "cheerio";
+/**
+ * Crawl a URL and build a site map of all discoverable internal pages.
+ *
+ * Uses BFS to traverse same-domain links up to `maxPages` pages.
+ * Respects `maxDepth` to limit how deep the crawl goes.
+ */
+export async function crawlSiteMap(url, options = {}) {
+    const { maxPages = 50, maxDepth = 5, sameDomain = true } = options;
+    const rootUrl = normalizeUrl(url) ?? url;
+    const rootDomain = extractDomain(rootUrl);
+    const visited = new Set();
+    const pages = [];
+    const graph = {};
+    const parent = {};
+    // BFS queue: [url, depth, parentUrl?]
+    const queue = [[rootUrl, 0, null]];
+    while (queue.length > 0 && pages.length < maxPages) {
+        const [currentUrl, depth, parentUrl] = queue.shift();
+        if (visited.has(currentUrl))
+            continue;
+        if (depth > maxDepth)
+            continue;
+        visited.add(currentUrl);
+        try {
+            const page = await fetchPage(currentUrl);
+            if (!page)
+                continue;
+            // Record parent relationship
+            if (parentUrl) {
+                parent[currentUrl] = parentUrl;
+            }
+            // Filter internal links
+            const internalLinks = [];
+            for (const link of page.links) {
+                const normalized = normalizeUrl(link, currentUrl);
+                if (!normalized)
+                    continue;
+                if (sameDomain && extractDomain(normalized) !== rootDomain)
+                    continue;
+                if (normalized.startsWith("mailto:") || normalized.startsWith("tel:") || normalized.startsWith("javascript:"))
+                    continue;
+                if (visited.has(normalized))
+                    continue;
+                internalLinks.push(normalized);
+            }
+            // Deduplicate internal links
+            const uniqueInternal = [...new Set(internalLinks)];
+            // Record in graph
+            graph[currentUrl] = uniqueInternal;
+            // Add to pages
+            pages.push({
+                url: currentUrl,
+                title: page.title,
+                depth,
+                links: uniqueInternal,
+                status: page.status,
+            });
+            // Enqueue unvisited internal links
+            for (const link of uniqueInternal) {
+                if (!visited.has(link)) {
+                    queue.push([link, depth + 1, currentUrl]);
+                }
+            }
+        }
+        catch (err) {
+            // Skip pages that fail to load
+            console.error(`[crawlSiteMap] Failed to fetch ${currentUrl}: ${err}`);
+        }
+    }
+    const maxDepthReached = pages.reduce((max, p) => Math.max(max, p.depth), 0);
+    return {
+        rootUrl,
+        totalPages: pages.length,
+        maxDepth: maxDepthReached,
+        pages,
+        graph,
+        parent,
+    };
+}
+/**
+ * Format the site map as a human-readable tree string.
+ */
+export function formatSiteMap(siteMap) {
+    const lines = [];
+    lines.push(`Site Map for: ${siteMap.rootUrl}`);
+    lines.push(`Total pages: ${siteMap.totalPages}, Max depth: ${siteMap.maxDepth}`);
+    lines.push("");
+    // Build a tree from the parent map
+    const treeLines = buildTreeLines(siteMap);
+    lines.push(...treeLines);
+    lines.push("");
+    lines.push("--- Page Details ---");
+    for (const page of siteMap.pages) {
+        lines.push(`  [${page.status}] ${page.url}`);
+        lines.push(`    Title: ${page.title}`);
+        if (page.links.length > 0) {
+            lines.push(`    Internal links (${page.links.length}):`);
+            for (const link of page.links.slice(0, 10)) {
+                lines.push(`      - ${link}`);
+            }
+            if (page.links.length > 10) {
+                lines.push(`      ... and ${page.links.length - 10} more`);
+            }
+        }
+        lines.push("");
+    }
+    return lines.join("\n");
+}
+function buildTreeLines(siteMap) {
+    const lines = [];
+    const { rootUrl, parent, pages } = siteMap;
+    // Build children map
+    const children = {};
+    for (const [child, par] of Object.entries(parent)) {
+        if (!children[par])
+            children[par] = [];
+        children[par].push(child);
+    }
+    // Build a page lookup
+    const pageByUrl = new Map();
+    for (const p of pages) {
+        pageByUrl.set(p.url, p);
+    }
+    function printTree(url, indent, isLast) {
+        const page = pageByUrl.get(url);
+        const prefix = isLast ? "└── " : "├── ";
+        const title = page ? page.title : "(unknown)";
+        lines.push(`${indent}${prefix}${url} [${page?.status ?? "?"}] "${title}"`);
+        const childList = children[url] || [];
+        const newIndent = indent + (isLast ? "    " : "│   ");
+        for (let i = 0; i < childList.length; i++) {
+            printTree(childList[i], newIndent, i === childList.length - 1);
+        }
+    }
+    lines.push(rootUrl);
+    const rootChildren = children[rootUrl] || [];
+    for (let i = 0; i < rootChildren.length; i++) {
+        printTree(rootChildren[i], "", i === rootChildren.length - 1);
+    }
+    return lines;
+}
+// --- Helpers ---
+function normalizeUrl(href, base) {
+    try {
+        const url = new URL(href, base || undefined);
+        // Remove trailing slash for consistency
+        let normalized = url.origin + url.pathname.replace(/\/$/, "") + url.search;
+        // If pathname is empty, keep root
+        if (normalized.endsWith(url.origin)) {
+            normalized = url.origin + "/";
+        }
+        return normalized;
+    }
+    catch {
+        return null;
+    }
+}
+function extractDomain(url) {
+    try {
+        return new URL(url).hostname;
+    }
+    catch {
+        return url;
+    }
+}
+async function fetchPage(url) {
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), 10000); // 10s timeout
+    try {
+        const res = await fetch(url, {
+            signal: controller.signal,
+            headers: {
+                "User-Agent": "xcoder-site-crawler/1.0",
+                Accept: "text/html,application/xhtml+xml",
+            },
+            redirect: "follow",
+        });
+        if (!res.ok) {
+            return { title: `(HTTP ${res.status})`, links: [], status: res.status };
+        }
+        const contentType = res.headers.get("content-type") || "";
+        if (!contentType.includes("text/html") && !contentType.includes("application/xhtml")) {
+            return null; // Skip non-HTML resources
+        }
+        const html = await res.text();
+        const $ = cheerio.load(html);
+        const title = $("title").first().text().trim() || "(no title)";
+        const links = [];
+        $("a[href]").each((_index, el) => {
+            const href = $(el).attr("href");
+            if (href && href.trim()) {
+                links.push(href.trim());
+            }
+        });
+        return { title, links, status: res.status };
+    }
+    catch {
+        return null;
+    }
+    finally {
+        clearTimeout(timeout);
+    }
+}

package/dist/tools/tool-registry.d.ts

@@ -0,0 +1,67 @@
+import type { TokenUsage } from "../types.js";
+export interface ToolManifestEntry {
+    name: string;
+    description: string;
+    inputHint: string;
+    /**
+     * Where this tool's answer about *which files exist* comes from.
+     *
+     * `index`      — the workspace index, and nothing else (no filesystem walk).
+     * `file-bytes` — reads contents of files the index selected.
+     * `process`    — shells out; results are not index-derived.
+     * `none`       — does not discover files.
+     */
+    discoverySource?: "index" | "file-bytes" | "process" | "none";
+}
+export interface CallSwarmInput {
+    goal: string;
+    protocol: string;
+    concurrency?: number;
+    maxIterations?: number;
+}
+export interface CallSwarmOutput {
+    instanceId: string;
+    status: "completed" | "failed" | "timeout";
+    workersCompleted: number;
+    workersFailed: number;
+    errors: string[];
+    tokenUsage: TokenUsage;
+    summary: string;
+}
+/**
+ * Tools whose file discovery must be answered by the index alone.
+ *
+ * Exported so a test can assert the property instead of trusting prose:
+ * `globTool` imports no filesystem enumerator, and every result it returns is
+ * a strict subset of `WorkspaceIndex.entries`.
+ */
+export declare const INDEX_ONLY_DISCOVERY_TOOLS: readonly ["glob", "grep", "search_workspace", "list_dir"];
+export declare const TOOL_MANIFEST: ToolManifestEntry[];
+/**
+ * Build the tool manifest array, conditionally including record_lesson.
+ * A function rather than a mutable global so that the manifest is
+ * immutable per call — testable and predictable.
+ */
+export declare function buildToolManifest(opts: {
+    lessonLearned: boolean;
+}): ToolManifestEntry[];
+export interface ToolCallResult {
+    tool: string;
+    ok: boolean;
+    output: unknown;
+}
+/**
+ * The one place a subtask's tool call actually executes. `root` pins every
+ * filesystem/shell action to this task's workspace — a subtask can never
+ * reach outside it (see `fs-tools.ts`'s `resolveInWorkspace`).
+ *
+ * Every branch is wrapped in try/catch here in addition to the caller's own
+ * try/catch (see `agents/sub-agent.ts`) — deliberate defense in depth: a
+ * tool throwing should never bring down the Sub Agent's loop, and a Sub
+ * Agent throwing should never bring down the Master Orchestrator (see
+ * `orchestrator/swarm.ts`).
+ */
+export declare function dispatchTool(root: string, toolName: string, input: unknown, ctx?: {
+    taskId: string;
+    subtaskId: string;
+}): Promise<ToolCallResult>;