xcoder-dna 0.0.0-stage → 1.3.0

package/dist/task-ingest/wbs-extract.js

@@ -0,0 +1,226 @@
+// xcoder:version 4 | xcoder:task task-7a4850 | xcoder:updated 2026-10-25T16:26:42.659Z | xcoder:subtask code-st-892a22
+// C3 — orchestrate `xcoder --wbc <file>`: read, parse, filter, detail, validate,
+// build, then persist ONE task run per PENDING WBS item.
+//
+// Failure model: every step can fail, none can half-write. Every task is built
+// and validated IN MEMORY before the first `saveTaskRun()`, so a schema failure
+// on item 7 of 9 writes 0 files, not 6 — a half-ingested WBS would be
+// indistinguishable from "the user already finished the first six items".
+//
+// Sibling of `ingest.ts`, not a mode flag on it: `task add --from` must keep
+// calling `buildIngestedRun` byte-identically.
+import path from "node:path";
+import { listTaskRuns, loadTaskRun, saveTaskRun } from "../store/task-store.js";
+import { readSource } from "./source-reader.js";
+import { parseWbs, MAX_ITEMS } from "./wbs-parser.js";
+import { selectPendingItems } from "./pending-filter.js";
+import { deriveItemTaskId } from "./extract-id.js";
+import { createDeterministicDetailer, createLlmDetailer, } from "./architect-detailer.js";
+import { validateArchitectDetail } from "./detail-schema.js";
+import { buildDetailedRun } from "./detailed-run.js";
+import { assertValidRun } from "./ingest.js";
+const DEFAULT_MAX_ITEMS = 50;
+const MAX_CONCURRENCY = 8;
+const PROGRESS = new Set(["done", "in_progress", "validating", "healing"]);
+function fail(code, message, extra = {}) {
+    return { ok: false, exitCode: 1, code, message, ...extra };
+}
+function hasProgress(run) {
+    return run.dag.some((n) => PROGRESS.has(n.status));
+}
+/** Bounded-parallel map that aborts on the FIRST rejection. */
+async function mapBounded(items, limit, fn) {
+    const results = new Array(items.length);
+    let next = 0;
+    let failure = null;
+    const worker = async () => {
+        while (!failure) {
+            const index = next++;
+            if (index >= items.length)
+                return;
+            try {
+                results[index] = await fn(items[index]);
+            }
+            catch (e) {
+                failure = failure ?? e;
+            }
+        }
+    };
+    await Promise.all(Array.from({ length: Math.max(1, Math.min(limit, items.length)) }, worker));
+    if (failure)
+        throw failure;
+    return results;
+}
+function sameIdentity(run, relPathPosix, normalized) {
+    return run.spec.provenance?.sourcePath === relPathPosix && run.spec.inputs.freeText === normalized;
+}
+/** Derive the id for one item, escalating 8→12 hex on a foreign collision. */
+async function deriveForItem(root, relPathPosix, item) {
+    for (const length of [8, 12]) {
+        const taskId = deriveItemTaskId(relPathPosix, item.normalized, length);
+        const existing = await loadTaskRun(root, taskId);
+        if (!existing)
+            return { ok: true, taskId, existing: null };
+        if (sameIdentity(existing, relPathPosix, item.normalized))
+            return { ok: true, taskId, existing };
+    }
+    return { ok: false };
+}
+export async function extractWbsTasks(opts) {
+    const root = path.resolve(opts.workspaceRoot);
+    const maxItems = opts.maxItems === undefined ? DEFAULT_MAX_ITEMS : Math.min(Math.floor(opts.maxItems), MAX_ITEMS);
+    const concurrency = opts.concurrency === undefined ? 3 : Math.floor(opts.concurrency);
+    if (!Number.isFinite(maxItems) || maxItems < 1)
+        return fail("bad-options", "--max-items must be a positive integer");
+    if (!Number.isFinite(concurrency) || concurrency < 1 || concurrency > MAX_CONCURRENCY) {
+        return fail("bad-options", `--concurrency must be between 1 and ${MAX_CONCURRENCY}`);
+    }
+    // 1-2. Read + confine, then parse (inert).
+    const source = await readSource(root, opts.from);
+    if (!source.ok)
+        return fail(source.code, source.message);
+    const src = source.value;
+    const stem = path.basename(src.relPathPosix, path.extname(src.relPathPosix));
+    const parsed = parseWbs(src.bytes.toString("utf8"), stem);
+    if (!parsed.ok)
+        return fail(parsed.code, parsed.message, parsed.line ? { line: parsed.line } : {});
+    // 3. Pending only — checked item TEXT never leaves this step.
+    const selection = selectPendingItems(parsed.value);
+    const base = { sourcePath: src.relPathPosix, sourceSha256: src.sha256, dryRun: opts.dryRun === true };
+    // 4. Nothing pending is a success, not a failure.
+    if (selection.pending.length === 0) {
+        return { ...base, ok: true, counts: { pending: 0, checked: selection.checkedCount, created: 0, unchanged: 0, superseded: 0 }, tasks: [] };
+    }
+    // 5. Bounded spend — refuse before spending a token.
+    if (selection.pending.length > maxItems) {
+        return fail("too-many-pending", `${selection.pending.length} pending items exceed --max-items ${maxItems} (use --max-items ${MAX_ITEMS} to take them all)`);
+    }
+    // 6. Existing-record probe → new | unchanged | blocked-by-progress.
+    const existingIds = new Set(await listTaskRuns(root));
+    const plans = [];
+    const liveIds = new Set();
+    for (const [index, item] of selection.pending.entries()) {
+        const derived = await deriveForItem(root, src.relPathPosix, item);
+        if (!derived.ok) {
+            return fail("id-collision", "a derived task id already belongs to a different work item", { line: item.line });
+        }
+        if (derived.existing && hasProgress(derived.existing) && !opts.force) {
+            return fail("has-progress", `${derived.taskId} already has completed or in-flight subtasks — re-run with --force to replace it`, { line: item.line, taskId: derived.taskId });
+        }
+        liveIds.add(derived.taskId);
+        plans.push({
+            item,
+            order: index + 1,
+            taskId: derived.taskId,
+            created: derived.existing === null,
+            existing: derived.existing ?? undefined,
+        });
+    }
+    // Superseded: records from this same source whose item text no longer exists.
+    let superseded = 0;
+    for (const id of existingIds) {
+        if (liveIds.has(id))
+            continue;
+        const run = await loadTaskRun(root, id);
+        if (run?.spec.provenance?.sourcePath === src.relPathPosix)
+            superseded += 1;
+    }
+    // 7. Detail — bounded, abort on first failure, hold every result in memory.
+    const toDetail = plans.filter((p) => p.created || opts.force === true);
+    const detailer = opts.detailer === null ? createDeterministicDetailer() : opts.detailer ?? createLlmDetailer();
+    try {
+        await mapBounded(toDetail, concurrency, async (plan) => {
+            plan.detail = await detailer.detail({
+                itemText: plan.item.normalized,
+                line: plan.item.line,
+                wbsTitle: parsed.value.title,
+                wbsGoal: parsed.value.goal,
+            });
+        });
+    }
+    catch (e) {
+        return fail("architect-failed", `architect detail failed: ${e.message}`);
+    }
+    // 8. Validate + build EVERY task in memory before the first write.
+    const createdAt = (opts.now ?? (() => new Date()))().toISOString();
+    const staged = [];
+    const extracted = [];
+    for (const plan of plans) {
+        if (!plan.detail) {
+            const existing = plan.existing;
+            extracted.push({
+                taskId: plan.taskId,
+                line: plan.item.line,
+                order: plan.order,
+                created: false,
+                nodeCount: existing.dag.length,
+                stages: existing.dag.map((n) => n.stage),
+                nextCommand: `xcoder resume ${plan.taskId}`,
+            });
+            continue;
+        }
+        const validation = validateArchitectDetail(plan.detail);
+        if (!validation.ok) {
+            return fail("architect-schema-invalid", `architect output for the item at line ${plan.item.line} is off-contract: ${validation.problems.join("; ")}`, { line: plan.item.line, taskId: plan.taskId });
+        }
+        const note = plan.item.after === undefined
+            ? undefined
+            : `follow WBS item #${plan.item.after} if it is still pending (advisory only)`;
+        const { spec, dag } = buildDetailedRun({
+            item: plan.item,
+            detail: validation.value,
+            source: { relPathPosix: src.relPathPosix, sha256: src.sha256 },
+            wbsTitle: parsed.value.title,
+            wbsGoal: parsed.value.goal,
+            taskId: plan.taskId,
+            createdAt: plan.existing?.spec.createdAt ?? createdAt,
+            concurrency,
+            sequencingNote: note,
+        });
+        const run = {
+            spec,
+            dag,
+            status: "halted",
+            escalations: [],
+            orchestratorTrace: [],
+            finalDeliverable: `Extracted from ${src.relPathPosix}:${plan.item.line} by --wbc — ` +
+                `${dag.length} subtask${dag.length === 1 ? "" : "s"}, not yet started. ` +
+                `Resume with: xcoder resume ${plan.taskId}`,
+            totalTokenUsage: { inputTokens: 0, outputTokens: 0 },
+        };
+        assertValidRun(run);
+        staged.push(run);
+        extracted.push({
+            taskId: plan.taskId,
+            line: plan.item.line,
+            order: plan.order,
+            created: true,
+            nodeCount: dag.length,
+            stages: dag.map((n) => n.stage),
+            nextCommand: `xcoder resume ${plan.taskId}`,
+        });
+    }
+    // 9. Persist — only after every task validated. Skipped under --dry-run.
+    if (opts.dryRun !== true) {
+        try {
+            for (const run of staged)
+                await saveTaskRun(root, run);
+        }
+        catch (e) {
+            return fail("write-failed", `could not write task records: ${e.message}`);
+        }
+    }
+    // 10. Summary.
+    return {
+        ...base,
+        ok: true,
+        counts: {
+            pending: selection.pending.length,
+            checked: selection.checkedCount,
+            created: opts.dryRun === true ? 0 : staged.length,
+            unchanged: extracted.filter((t) => !t.created).length,
+            superseded,
+        },
+        tasks: extracted,
+    };
+}

package/dist/task-ingest/wbs-parser.d.ts

@@ -0,0 +1,36 @@
+import type { SdlcStage } from "../types.js";
+export interface WbsItem {
+    /** 0-based source order. */
+    index: number;
+    /** Verbatim item text, minus the list marker and any trailing tags. */
+    text: string;
+    checked: boolean;
+    stage?: SdlcStage;
+    after?: number;
+    /** 1-based line number, for error messages. */
+    line: number;
+}
+export interface ParsedWbs {
+    title: string;
+    goal: string;
+    items: WbsItem[];
+    frontMatter: Record<string, string>;
+}
+export type ParseErrorCode = "no-items" | "bad-front-matter" | "too-many-items";
+export type ParseResult = {
+    ok: true;
+    value: ParsedWbs;
+} | {
+    ok: false;
+    code: ParseErrorCode;
+    message: string;
+    line?: number;
+};
+/** Closed list — a front-matter key outside this set is dropped, never acted on. */
+export declare const ALLOWED_FRONT_MATTER_KEYS: readonly ["title", "goal", "id", "status", "priority", "owner", "tags"];
+export declare const MAX_ITEMS = 200;
+/**
+ * Parse `text` as a WBS document. `fileStem` is used as a fallback title so a
+ * document without front matter still produces a human-meaningful goal.
+ */
+export declare function parseWbs(text: string, fileStem: string): ParseResult;

package/dist/task-ingest/wbs-parser.js

@@ -0,0 +1,170 @@
+// xcoder:version 2 | xcoder:task task-a91099 | xcoder:updated 2026-10-23T10:55:01.913Z | xcoder:subtask code-st-03748c
+// C2 — parse a markdown/WBS source into a bounded, inert item list.
+//
+// Threat model: the file is hostile (PRM-01). Two deliberate design choices
+// keep it inert:
+//   1. Front matter is a hand-written strict `key: value` reader over a closed
+//      allow-list — no YAML library (no alias-expansion DoS, no tag surface, no
+//      implicit type coercion). Anchors and block scalars are rejected outright.
+//   2. Nothing a file says can set a path, a shell command, a URL, a credential
+//      ref or a budget. The parser cannot even represent those fields.
+/** Closed list — a front-matter key outside this set is dropped, never acted on. */
+export const ALLOWED_FRONT_MATTER_KEYS = ["title", "goal", "id", "status", "priority", "owner", "tags"];
+const SDLC_STAGES = [
+    "requirements",
+    "design",
+    "code",
+    "refactor",
+    "test",
+    "ui_api_test",
+    "fix_defect",
+    "document",
+    "deploy",
+    "conversation",
+    "quality_gate",
+];
+export const MAX_ITEMS = 200;
+function bad(message, line) {
+    return { ok: false, code: "bad-front-matter", message, line };
+}
+function parseFrontMatter(lines) {
+    const data = {};
+    if (lines.length === 0 || lines[0].trim() !== "---") {
+        return { ok: true, data, bodyStart: 0 };
+    }
+    let close = -1;
+    for (let i = 1; i < lines.length; i++) {
+        if (lines[i].trim() === "---") {
+            close = i;
+            break;
+        }
+    }
+    if (close === -1) {
+        return { ok: false, result: bad("front matter opened with '---' but never closed", 1) };
+    }
+    for (let i = 1; i < close; i++) {
+        const raw = lines[i];
+        const lineNo = i + 1;
+        if (raw.trim() === "")
+            continue;
+        if (/^\s/.test(raw)) {
+            return { ok: false, result: bad("nested or indented front-matter values are not supported", lineNo) };
+        }
+        const colon = raw.indexOf(":");
+        if (colon === -1) {
+            return { ok: false, result: bad("front-matter line has no 'key: value' colon", lineNo) };
+        }
+        const key = raw.slice(0, colon).trim().toLowerCase();
+        const value = raw.slice(colon + 1).trim();
+        if (key.length === 0) {
+            return { ok: false, result: bad("front-matter key is empty", lineNo) };
+        }
+        if (/[&*]/.test(value)) {
+            return { ok: false, result: bad("YAML anchors and aliases are not supported", lineNo) };
+        }
+        if (/^[|>{[]/.test(value)) {
+            return { ok: false, result: bad("block scalars and flow collections are not supported", lineNo) };
+        }
+        if (Object.prototype.hasOwnProperty.call(data, key)) {
+            return { ok: false, result: bad(`duplicate front-matter key "${key}"`, lineNo) };
+        }
+        if (ALLOWED_FRONT_MATTER_KEYS.includes(key)) {
+            data[key] = value.replace(/^["']|["']$/g, "");
+        }
+    }
+    return { ok: true, data, bodyStart: close + 1 };
+}
+const BULLET_ITEM = /^ {0,3}[-*+]\s+\[( |x|X)\]\s+(\S.*)$/;
+const ORDERED_ITEM = /^ {0,3}(\d{1,9})[.)]\s+(\S.*)$/;
+const STAGE_TAG = /\[stage:\s*([a-z_]+)\s*\]/i;
+const AFTER_TAG = /\[after:\s*(\d+)\s*\]/i;
+function stripTags(text) {
+    let stage;
+    let after;
+    const sm = STAGE_TAG.exec(text);
+    if (sm) {
+        const candidate = sm[1].toLowerCase();
+        if (SDLC_STAGES.includes(candidate))
+            stage = candidate;
+    }
+    const am = AFTER_TAG.exec(text);
+    if (am) {
+        const n = parseInt(am[1], 10);
+        if (Number.isFinite(n))
+            after = n;
+    }
+    const cleaned = text.replace(STAGE_TAG, " ").replace(AFTER_TAG, " ").replace(/\s{2,}/g, " ").trim();
+    return { text: cleaned, stage, after };
+}
+function extractItems(lines, bodyStart) {
+    const items = [];
+    // Track WHICH fence opened the block: a ``` block is not closed by ~~~.
+    let fence = null;
+    for (let i = bodyStart; i < lines.length; i++) {
+        const line = lines[i];
+        const trimmed = line.trim();
+        if (trimmed.startsWith("```") || trimmed.startsWith("~~~")) {
+            const marker = trimmed.startsWith("```") ? "```" : "~~~";
+            if (fence === null)
+                fence = marker;
+            else if (fence === marker)
+                fence = null;
+            continue;
+        }
+        if (fence !== null)
+            continue;
+        if (/^\s{4,}\S/.test(line))
+            continue;
+        let checked = false;
+        let text = null;
+        const bullet = BULLET_ITEM.exec(line);
+        if (bullet) {
+            checked = bullet[1].toLowerCase() === "x";
+            text = bullet[2];
+        }
+        else {
+            const ordered = ORDERED_ITEM.exec(line);
+            if (ordered)
+                text = ordered[2];
+        }
+        if (text === null)
+            continue;
+        const stripped = stripTags(text);
+        if (stripped.text.length === 0)
+            continue;
+        items.push({
+            index: items.length,
+            text: stripped.text,
+            checked,
+            stage: stripped.stage,
+            after: stripped.after,
+            line: i + 1,
+        });
+    }
+    return items;
+}
+/**
+ * Parse `text` as a WBS document. `fileStem` is used as a fallback title so a
+ * document without front matter still produces a human-meaningful goal.
+ */
+export function parseWbs(text, fileStem) {
+    const lines = text.split(/\r?\n/);
+    const fm = parseFrontMatter(lines);
+    if (!fm.ok)
+        return fm.result;
+    const items = extractItems(lines, fm.bodyStart);
+    if (items.length === 0) {
+        return { ok: false, code: "no-items", message: "no work items found — expected '- [ ] text' or '1. text' lines" };
+    }
+    if (items.length > MAX_ITEMS) {
+        return {
+            ok: false,
+            code: "too-many-items",
+            message: `found ${items.length} work items, over the ${MAX_ITEMS}-item limit`,
+        };
+    }
+    const firstHeading = lines.slice(fm.bodyStart).find((l) => /^#{1,6}\s+\S/.test(l.trim()));
+    const title = fm.data.title ?? (firstHeading ? firstHeading.trim().replace(/^#{1,6}\s+/, "") : fileStem);
+    const goal = fm.data.goal ?? title;
+    return { ok: true, value: { title, goal, items, frontMatter: fm.data } };
+}

package/dist/tools/apiTestTool.d.ts

@@ -0,0 +1,37 @@
+export interface ApiTestResult {
+    url: string;
+    method: string;
+    statusCode: number;
+    statusText: string;
+    headers: Record<string, string>;
+    body: unknown;
+    bodyTruncated: boolean;
+    durationMs: number;
+    contentType: string;
+}
+export interface ApiTestOptions {
+    url: string;
+    method: "GET" | "POST" | "PUT" | "PATCH" | "DELETE" | "HEAD" | "OPTIONS";
+    /** Optional query parameters appended to the URL */
+    queryParams?: Record<string, string>;
+    /** Request headers */
+    headers?: Record<string, string>;
+    /** Request body — string, object (serialized as JSON), or omitted */
+    body?: string | Record<string, unknown>;
+    /** How to encode the body. Defaults to 'json' if body is an object, 'text' if string. */
+    bodyType?: "json" | "text" | "form";
+    /** Max response body length in characters before truncation (default: 10000) */
+    maxBodyLength?: number;
+    /** Timeout in milliseconds (default: 30000) */
+    timeout?: number;
+    /** If set, the tool will assert the response status matches this value and return an error if not */
+    expectStatus?: number;
+    /** If set, the tool will assert the response body contains this string and return an error if not */
+    expectBodyContains?: string;
+    /** Reference to an environment variable holding a credential — resolved at tool level and NEVER returned to the LLM. Injected as Authorization: Bearer <value>. */
+    credentialRef?: string;
+}
+/**
+ * Execute an HTTP request against an API endpoint and return structured results.
+ */
+export declare function testApiEndpoint(opts: ApiTestOptions): Promise<ApiTestResult>;

package/dist/tools/apiTestTool.js

@@ -0,0 +1,128 @@
+// xcoder:version 4 | xcoder:task task-456d3f | xcoder:updated 2026-08-03T15:52:16.274Z | xcoder:subtask code-st-97a4d4
+/**
+ * apiTestTool — Make HTTP requests to test API endpoints.
+ *
+ * Supports GET, POST, PUT, PATCH, DELETE, HEAD, OPTIONS with custom headers,
+ * JSON/Form/Text bodies, query parameters, and optional response validation.
+ * Returns structured results including status, headers, body, and timing.
+ */
+import { resolveCredential } from "./credential.js";
+/**
+ * Execute an HTTP request against an API endpoint and return structured results.
+ */
+export async function testApiEndpoint(opts) {
+    const { url, method, queryParams, headers: customHeaders, body, bodyType, maxBodyLength = 10000, timeout = 30000, } = opts;
+    // Build URL with query params
+    let targetUrl = url;
+    if (queryParams && Object.keys(queryParams).length > 0) {
+        const params = new URLSearchParams(queryParams);
+        targetUrl += (url.includes("?") ? "&" : "?") + params.toString();
+    }
+    // Build fetch options
+    const fetchHeaders = {
+        ...customHeaders,
+    };
+    // Resolve credentialRef at tool level — the raw value never leaves this function
+    if (opts.credentialRef) {
+        fetchHeaders["Authorization"] = `Bearer ${resolveCredential(opts.credentialRef)}`;
+    }
+    let fetchBody;
+    if (body !== undefined) {
+        if (bodyType === "form") {
+            const formBody = typeof body === "string" ? body : new URLSearchParams(body).toString();
+            fetchBody = formBody;
+            if (!fetchHeaders["Content-Type"]) {
+                fetchHeaders["Content-Type"] = "application/x-www-form-urlencoded";
+            }
+        }
+        else if (bodyType === "text") {
+            fetchBody = typeof body === "string" ? body : JSON.stringify(body);
+            if (!fetchHeaders["Content-Type"]) {
+                fetchHeaders["Content-Type"] = "text/plain";
+            }
+        }
+        else {
+            // Default: json
+            fetchBody = typeof body === "string" ? body : JSON.stringify(body);
+            if (!fetchHeaders["Content-Type"]) {
+                fetchHeaders["Content-Type"] = "application/json";
+            }
+        }
+    }
+    // HEAD requests cannot have a body
+    if (method === "HEAD" || method === "OPTIONS") {
+        fetchBody = undefined;
+    }
+    const startTime = Date.now();
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), timeout);
+    try {
+        const response = await fetch(targetUrl, {
+            method,
+            headers: fetchHeaders,
+            body: fetchBody,
+            signal: controller.signal,
+            redirect: "follow",
+        });
+        const durationMs = Date.now() - startTime;
+        // Collect response headers
+        const responseHeaders = {};
+        response.headers.forEach((value, key) => {
+            responseHeaders[key] = value;
+        });
+        const contentType = responseHeaders["content-type"] ?? "";
+        // Read body (HEAD and OPTIONS requests have no body)
+        let bodyResult = null;
+        let bodyTruncated = false;
+        if (method !== "HEAD" && method !== "OPTIONS") {
+            const rawText = await response.text();
+            if (rawText.length > maxBodyLength) {
+                bodyTruncated = true;
+            }
+            const truncated = rawText.slice(0, maxBodyLength);
+            // Try to parse as JSON if content-type suggests it
+            if (contentType.includes("application/json") || contentType.includes("application/problem+json")) {
+                try {
+                    bodyResult = JSON.parse(truncated);
+                }
+                catch {
+                    bodyResult = truncated;
+                }
+            }
+            else {
+                bodyResult = truncated;
+            }
+        }
+        // Validate expectStatus if provided
+        if (opts.expectStatus !== undefined && response.status !== opts.expectStatus) {
+            const bodyStr = bodyResult !== null ? JSON.stringify(bodyResult).slice(0, 200) : "(no body)";
+            throw new Error(`Expected status ${opts.expectStatus} but got ${response.status} (${response.statusText}). Body preview: ${bodyStr}`);
+        }
+        // Validate expectBodyContains if provided
+        if (opts.expectBodyContains !== undefined) {
+            const rawBody = bodyResult !== null ? (typeof bodyResult === "string" ? bodyResult : JSON.stringify(bodyResult)) : "";
+            if (!rawBody.includes(opts.expectBodyContains)) {
+                throw new Error(`Expected body to contain "${opts.expectBodyContains}" but it did not.`);
+            }
+        }
+        return {
+            url: targetUrl,
+            method,
+            statusCode: response.status,
+            statusText: response.statusText,
+            headers: responseHeaders,
+            body: bodyResult,
+            bodyTruncated,
+            durationMs,
+            contentType,
+        };
+    }
+    catch (err) {
+        const durationMs = Date.now() - startTime;
+        const message = err instanceof Error ? err.message : String(err);
+        throw new Error(`API request failed after ${durationMs}ms: ${message}`);
+    }
+    finally {
+        clearTimeout(timeoutId);
+    }
+}