universal-ast-mapper 2.0.0 → 2.0.1

package/dist/serve.js

@@ -0,0 +1,185 @@
+import http from "node:http";
+import fs from "node:fs";
+import path from "node:path";
+import { collectSourceFiles } from "./skeleton.js";
+import { resolveOptions } from "./config.js";
+import { buildSymbolGraph } from "./graph.js";
+import { findDeadExports, findCircularDeps, getTopSymbols } from "./graph-analysis.js";
+import { buildReport } from "./report.js";
+import { loadHistory } from "./history.js";
+import { detectSmells } from "./smells.js";
+import { scanFileForSecurityIssues } from "./security.js";
+import { buildSkeletonsBulk } from "./pool.js";
+import { webAppHtml } from "./webapp.js";
+export async function startServe(opts) {
+    const root = opts.root;
+    const scanDir = opts.scanDir ?? root;
+    const port = opts.port ?? 7337;
+    // SSE client registry
+    const sseClients = new Set();
+    function broadcastChange() {
+        for (const client of sseClients) {
+            try {
+                client.write("event: change\ndata: {}\n\n");
+            }
+            catch {
+                sseClients.delete(client);
+            }
+        }
+    }
+    // fs.watch for live reload
+    if (opts.watch) {
+        try {
+            fs.watch(scanDir, { recursive: true }, (_event, filename) => {
+                if (!filename || filename.includes(".ast-map"))
+                    return;
+                cache = null;
+                broadcastChange();
+            });
+        }
+        catch { /* watch not supported on all platforms */ }
+    }
+    let cache = null;
+    const CACHE_TTL = 5000;
+    async function getSkeletons() {
+        if (cache && Date.now() - cache.ts < CACHE_TTL)
+            return cache.skeletons;
+        const skOpts = resolveOptions({ detail: "outline", emitHtml: false });
+        const files = collectSourceFiles(scanDir, skOpts);
+        const items = files.map((f) => ({
+            abs: f,
+            rel: path.relative(root, f).split(path.sep).join("/"),
+        }));
+        const built = await buildSkeletonsBulk(items, skOpts);
+        const skeletons = built.filter(Boolean).map((r) => r.skel);
+        cache = { skeletons, ts: Date.now() };
+        return skeletons;
+    }
+    const server = http.createServer(async (req, res) => {
+        const url = new URL(req.url ?? "/", `http://localhost:${port}`);
+        const pathname = url.pathname;
+        res.setHeader("Access-Control-Allow-Origin", "*");
+        res.setHeader("Access-Control-Allow-Headers", "Content-Type");
+        try {
+            if (pathname === "/" || pathname === "/index.html") {
+                res.writeHead(200, { "Content-Type": "text/html; charset=utf-8" });
+                res.end(webAppHtml(port));
+                return;
+            }
+            if (pathname === "/api/report") {
+                const skeletons = await getSkeletons();
+                const graph = buildSymbolGraph(skeletons, root);
+                const history = loadHistory(root);
+                const report = await buildReport(scanDir, root);
+                const smellsAll = [];
+                const securityAll = [];
+                for (const skel of skeletons) {
+                    const fileAbs = path.resolve(root, skel.file);
+                    try {
+                        const src = fs.readFileSync(fileAbs, "utf8");
+                        smellsAll.push(...detectSmells(skel, src.split("\n").length));
+                        securityAll.push(...scanFileForSecurityIssues(src, skel.file));
+                    }
+                    catch { /* skip */ }
+                }
+                res.writeHead(200, { "Content-Type": "application/json" });
+                res.end(JSON.stringify({ ...report, smells: smellsAll, security: securityAll, history }, null, 2));
+                return;
+            }
+            if (pathname === "/api/graph") {
+                const skeletons = await getSkeletons();
+                const graph = buildSymbolGraph(skeletons, root);
+                res.writeHead(200, { "Content-Type": "application/json" });
+                res.end(JSON.stringify(graph, null, 2));
+                return;
+            }
+            if (pathname === "/api/dead") {
+                const skeletons = await getSkeletons();
+                const graph = buildSymbolGraph(skeletons, root);
+                const dead = findDeadExports(graph);
+                res.writeHead(200, { "Content-Type": "application/json" });
+                res.end(JSON.stringify(dead, null, 2));
+                return;
+            }
+            if (pathname === "/api/top") {
+                const skeletons = await getSkeletons();
+                const graph = buildSymbolGraph(skeletons, root);
+                const top = getTopSymbols(graph, 20);
+                res.writeHead(200, { "Content-Type": "application/json" });
+                res.end(JSON.stringify(top, null, 2));
+                return;
+            }
+            if (pathname === "/api/cycles") {
+                const skeletons = await getSkeletons();
+                const graph = buildSymbolGraph(skeletons, root);
+                const cycles = findCircularDeps(graph);
+                res.writeHead(200, { "Content-Type": "application/json" });
+                res.end(JSON.stringify(cycles, null, 2));
+                return;
+            }
+            if (pathname === "/api/history") {
+                const history = loadHistory(root);
+                res.writeHead(200, { "Content-Type": "application/json" });
+                res.end(JSON.stringify(history, null, 2));
+                return;
+            }
+            if (pathname === "/api/skeletons") {
+                const skeletons = await getSkeletons();
+                res.writeHead(200, { "Content-Type": "application/json" });
+                res.end(JSON.stringify(skeletons, null, 2));
+                return;
+            }
+            if (pathname === "/events") {
+                res.writeHead(200, {
+                    "Content-Type": "text/event-stream",
+                    "Cache-Control": "no-cache",
+                    "Connection": "keep-alive",
+                    "Access-Control-Allow-Origin": "*",
+                });
+                res.write("event: connected\ndata: {}\n\n");
+                sseClients.add(res);
+                req.on("close", () => sseClients.delete(res));
+                return;
+            }
+            if (pathname === "/api/smells") {
+                const skeletons = await getSkeletons();
+                const all = [];
+                for (const skel of skeletons) {
+                    const fileAbs = path.resolve(root, skel.file);
+                    try {
+                        const src = fs.readFileSync(fileAbs, "utf8");
+                        all.push(...detectSmells(skel, src.split("\n").length));
+                    }
+                    catch { /* skip */ }
+                }
+                res.writeHead(200, { "Content-Type": "application/json" });
+                res.end(JSON.stringify(all, null, 2));
+                return;
+            }
+            if (pathname === "/api/security") {
+                const skeletons = await getSkeletons();
+                const all = [];
+                for (const skel of skeletons) {
+                    const fileAbs = path.resolve(root, skel.file);
+                    try {
+                        const src = fs.readFileSync(fileAbs, "utf8");
+                        all.push(...scanFileForSecurityIssues(src, skel.file));
+                    }
+                    catch { /* skip */ }
+                }
+                res.writeHead(200, { "Content-Type": "application/json" });
+                res.end(JSON.stringify(all, null, 2));
+                return;
+            }
+            res.writeHead(404, { "Content-Type": "text/plain" });
+            res.end("Not found");
+        }
+        catch (e) {
+            res.writeHead(500, { "Content-Type": "application/json" });
+            res.end(JSON.stringify({ error: e instanceof Error ? e.message : String(e) }));
+        }
+    });
+    return new Promise((resolve) => {
+        server.listen(port, () => resolve(server));
+    });
+}

package/dist/sfc.js

@@ -0,0 +1,27 @@
+const SCRIPT_RE = /<script\b([^>]*)>([\s\S]*?)<\/script>/gi;
+export function isSfcExt(ext) {
+    return ext === ".vue" || ext === ".svelte";
+}
+export function extractScript(source) {
+    // Start from an all-blank canvas the same shape as the source (newlines kept,
+    // every other character turned into a space) to preserve line/column offsets.
+    let out = source.replace(/[^\n]/g, " ");
+    let hasScript = false;
+    let lang = "js";
+    let m;
+    SCRIPT_RE.lastIndex = 0;
+    while ((m = SCRIPT_RE.exec(source)) !== null) {
+        hasScript = true;
+        const attrs = m[1] ?? "";
+        if (/lang\s*=\s*["'](ts|typescript)["']/i.test(attrs))
+            lang = "ts";
+        const inner = m[2] ?? "";
+        const innerStart = m.index + m[0].indexOf(inner, m[1] ? m[1].length : 0);
+        out = out.slice(0, innerStart) + inner + out.slice(innerStart + inner.length);
+    }
+    return {
+        code: hasScript ? out : "",
+        grammar: lang === "ts" ? "typescript" : "javascript",
+        hasScript,
+    };
+}

package/dist/similar.js

@@ -0,0 +1,98 @@
+// ─── Fingerprinting ────────────────────────────────────────────────────────────
+/**
+ * Build a structural fingerprint for a symbol.
+ * Two symbols with the same fingerprint are structurally similar (not textually identical).
+ */
+function fingerprint(sym) {
+    const kind = sym.kind;
+    if (!["function", "method", "class", "struct"].includes(kind))
+        return null;
+    const sig = sym.signature ?? "";
+    // Param count from signature
+    const paramMatch = sig.match(/\(([^)]*)\)/);
+    const paramStr = paramMatch?.[1]?.trim() ?? "";
+    const paramCount = paramStr === "" ? 0 : paramStr.split(",").length;
+    // Presence of return type annotation
+    const hasReturnType = sig.includes("): ") || sig.includes(") :") || sig.includes("->");
+    // Async?
+    const isAsync = sig.includes("async ") || sig.includes("suspend ") || sig.includes("async def ");
+    // Visibility
+    const visibility = sym.visibility;
+    // Child count bucket for classes
+    const childCount = sym.children.length;
+    const childBucket = childCount === 0 ? "0" : childCount <= 3 ? "1-3" : childCount <= 8 ? "4-8" : "9+";
+    // Line length bucket
+    const lineCount = sym.range.endLine - sym.range.startLine + 1;
+    const sizeBucket = lineCount <= 5 ? "xs" : lineCount <= 20 ? "sm" : lineCount <= 60 ? "md" : "lg";
+    // Has nested functions?
+    const hasNested = sym.children.some((c) => c.kind === "function" || c.kind === "method");
+    if (kind === "class" || kind === "struct") {
+        return `${kind}|children:${childBucket}|vis:${visibility}|size:${sizeBucket}`;
+    }
+    return `${kind}|params:${paramCount}|async:${isAsync}|ret:${hasReturnType}|vis:${visibility}|size:${sizeBucket}|nested:${hasNested}`;
+}
+// ─── Collector ────────────────────────────────────────────────────────────────
+function collect(symbols, file, kinds, out) {
+    for (const sym of symbols) {
+        if (kinds.has(sym.kind)) {
+            const fp = fingerprint(sym);
+            if (fp) {
+                out.push({
+                    file,
+                    symbol: sym.name,
+                    kind: sym.kind,
+                    line: sym.range.startLine,
+                    signature: sym.signature,
+                    fingerprint: fp,
+                });
+            }
+        }
+        if (sym.children.length > 0)
+            collect(sym.children, file, kinds, out);
+    }
+}
+// ─── Human-readable description ───────────────────────────────────────────────
+function describeFingerprint(fp) {
+    const parts = Object.fromEntries(fp.split("|").map((p) => p.split(":")));
+    const kind = fp.split("|")[0];
+    const segments = [`${kind}s`];
+    if (parts.params)
+        segments.push(`${parts.params} param(s)`);
+    if (parts.async === "true")
+        segments.push("async");
+    if (parts.ret === "true")
+        segments.push("typed return");
+    if (parts.children)
+        segments.push(`${parts.children} children`);
+    if (parts.size)
+        segments.push(`size:${parts.size}`);
+    if (parts.vis)
+        segments.push(parts.vis);
+    return segments.join(", ");
+}
+// ─── Public API ───────────────────────────────────────────────────────────────
+/** Find groups of structurally similar symbols across multiple skeleton files. */
+export function findSimilar(skeletons, opts = {}) {
+    const minGroupSize = opts.minGroupSize ?? 2;
+    const kinds = new Set(opts.kinds ?? ["function", "method", "class", "struct"]);
+    const entries = [];
+    for (const skel of skeletons) {
+        collect(skel.symbols, skel.file, kinds, entries);
+    }
+    // Group by fingerprint
+    const groups = new Map();
+    for (const entry of entries) {
+        const g = groups.get(entry.fingerprint) ?? [];
+        g.push(entry);
+        groups.set(entry.fingerprint, g);
+    }
+    return [...groups.entries()]
+        .filter(([, g]) => g.length >= minGroupSize)
+        .map(([fp, g]) => ({
+        fingerprint: fp,
+        description: describeFingerprint(fp),
+        count: g.length,
+        entries: g.sort((a, b) => a.file.localeCompare(b.file) || a.symbol.localeCompare(b.symbol)),
+    }))
+        .sort((a, b) => b.count - a.count);
+}

package/dist/skeleton.js

@@ -0,0 +1,132 @@
+import fs from "node:fs";
+import path from "node:path";
+import { detectLanguage, supportedExtensions } from "./registry.js";
+import { parseSource } from "./parser.js";
+import { countSymbols, toOutline } from "./extractors/common.js";
+import { extractScript } from "./sfc.js";
+import { diskCacheKey, diskCacheGet, diskCachePut } from "./diskcache.js";
+export const SCHEMA_VERSION = "1.1";
+export const GRAMMAR_SOURCE = "tree-sitter-wasms@0.1.13";
+const parseCache = new Map();
+function cacheKey(absPath, detail) { return `${absPath}|${detail}`; }
+function getCached(absPath, detail) {
+    try {
+        const entry = parseCache.get(cacheKey(absPath, detail));
+        if (!entry)
+            return null;
+        const mtime = fs.statSync(absPath).mtimeMs;
+        return entry.mtime === mtime ? entry.result : null;
+    }
+    catch {
+        return null;
+    }
+}
+function setCached(absPath, detail, result) {
+    try {
+        const mtime = fs.statSync(absPath).mtimeMs;
+        parseCache.set(cacheKey(absPath, detail), { mtime, result });
+    }
+    catch { /* skip if stat fails */ }
+}
+export class UnsupportedLanguageError extends Error {
+    ext;
+    constructor(ext) {
+        super(`Unsupported file type "${ext}". Supported: ${supportedExtensions().join(", ")}`);
+        this.ext = ext;
+        this.name = "UnsupportedLanguageError";
+    }
+}
+/**
+ * Build a skeleton for a single file.
+ * @param absPath absolute path on disk (already validated to be within root)
+ * @param relPath path relative to root, used as the displayed `file`
+ */
+export async function buildSkeleton(absPath, relPath, opts) {
+    const ext = path.extname(absPath).toLowerCase();
+    const entry = detectLanguage(absPath);
+    if (!entry)
+        throw new UnsupportedLanguageError(ext);
+    const stat = fs.statSync(absPath);
+    if (stat.size > opts.maxFileBytes) {
+        throw new Error(`File is ${stat.size} bytes, exceeds maxFileBytes (${opts.maxFileBytes}). Increase the limit to parse it.`);
+    }
+    // Return cached result if file hasn't changed. The cached SkeletonFile's
+    // `.file` is whatever relPath the first caller used; the same absolute file
+    // can be requested under a different root (different relPath), so override
+    // `.file` per call to avoid leaking a stale rel path into callers/indexes.
+    const cached = getCached(absPath, opts.detail);
+    if (cached) {
+        const wantFile = relPath.split(path.sep).join("/");
+        return cached.file === wantFile ? cached : { ...cached, file: wantFile };
+    }
+    const rawSource = fs.readFileSync(absPath, "utf8");
+    // Persistent cache (content-hash keyed, see diskcache.ts). A hit skips
+    // parsing entirely; entries can never be stale because the key embeds the
+    // file's bytes + detail + schema/grammar versions.
+    const dKey = diskCacheKey(rawSource, opts.detail, SCHEMA_VERSION, GRAMMAR_SOURCE);
+    const fromDisk = diskCacheGet(dKey);
+    if (fromDisk) {
+        const wantFile = relPath.split(path.sep).join("/");
+        const hit = fromDisk.file === wantFile ? fromDisk : { ...fromDisk, file: wantFile };
+        setCached(absPath, opts.detail, hit);
+        return hit;
+    }
+    let source = rawSource;
+    let grammar = entry.grammar;
+    if (entry.sfc) {
+        const script = extractScript(source);
+        source = script.code; // blank-padded script-only source (offsets preserved)
+        grammar = script.grammar;
+    }
+    const root = await parseSource(grammar, source);
+    let symbols = entry.extract(root, source);
+    if (opts.detail === "outline")
+        symbols = toOutline(symbols);
+    const directives = entry.extractDirectives ? entry.extractDirectives(root, source) : [];
+    const imports = entry.extractImports ? entry.extractImports(root, source) : [];
+    const result = {
+        schemaVersion: SCHEMA_VERSION,
+        file: relPath.split(path.sep).join("/"),
+        language: entry.language,
+        generatedAt: new Date().toISOString(),
+        parser: { engine: "tree-sitter", grammar: `${grammar} (${GRAMMAR_SOURCE})` },
+        symbolCount: countSymbols(symbols),
+        ...(directives.length > 0 ? { directives } : {}),
+        ...(imports.length > 0 ? { imports } : {}),
+        symbols,
+    };
+    setCached(absPath, opts.detail, result);
+    diskCachePut(dKey, result);
+    return result;
+}
+/** Recursively collect supported source files under a directory. */
+export function collectSourceFiles(absDir, opts) {
+    const supported = new Set(supportedExtensions());
+    const ignore = new Set(opts.ignore);
+    const results = [];
+    const walk = (dir) => {
+        let entries;
+        try {
+            entries = fs.readdirSync(dir, { withFileTypes: true });
+        }
+        catch {
+            return;
+        }
+        for (const e of entries) {
+            if (e.name.startsWith(".")) {
+                if (e.isDirectory())
+                    continue;
+            }
+            const full = path.join(dir, e.name);
+            if (e.isDirectory()) {
+                if (!ignore.has(e.name))
+                    walk(full);
+            }
+            else if (e.isFile() && supported.has(path.extname(e.name).toLowerCase())) {
+                results.push(full);
+            }
+        }
+    };
+    walk(absDir);
+    return results.sort();
+}