ucn 4.0.1 → 4.1.0

package/core/reporting.js

@@ -8,6 +8,8 @@
 'use strict';
 
 const fs = require('fs');
+const { codeUnitCompare, CALLABLE_SYMBOL_KINDS } = require('./shared');
+const { _declaredFieldType, _projectTopLevelNames } = require('./callers');
 const path = require('path');
 const { isTestFile } = require('./discovery');
 
@@ -67,9 +69,7 @@ function getStats(index, options = {}) {
         const functions = [];
         for (const [name, symbols] of index.symbols) {
             for (const sym of symbols) {
-                if (sym.type === 'function' || sym.type === 'method' || sym.type === 'static' ||
-                    sym.type === 'constructor' || sym.type === 'public' || sym.type === 'abstract' ||
-                    sym.type === 'classmethod') {
+                if (CALLABLE_SYMBOL_KINDS.has(sym.type)) {
                     const lineCount = sym.endLine - sym.startLine + 1;
                     const relativePath = sym.relativePath || (sym.file ? path.relative(index.root, sym.file) : '');
                     functions.push({
@@ -97,10 +97,7 @@ function getStats(index, options = {}) {
         const top = options.top === 0
             ? 0
             : ((options.top != null && Number(options.top) > 0) ? Number(options.top) : 10);
-        const FUNCTION_TYPES = new Set([
-            'function', 'method', 'static', 'constructor',
-            'public', 'abstract', 'classmethod'
-        ]);
+        const FUNCTION_TYPES = CALLABLE_SYMBOL_KINDS;
 
         // Ensure the calls cache is fully populated before counting.
         // First-time stats --hot may need to parse files to extract calls;
@@ -135,6 +132,15 @@ function getStats(index, options = {}) {
         // Pre-compute import-alias sets per file. Used to distinguish `mod.foo()`
         // (resolves to top-level foo) from `obj.foo()` on a local variable.
         const fileImportAliases = new Map();         // filePath -> Set<string> of alias names
+        const fieldHopCache = new Map();             // rootType\0field -> declared type|null
+        // Names import-bound to an EXTERNAL module, per file (fix #256,
+        // dogfood-measured: 895 node:test `describe(...)` calls in test
+        // files were attributed to a project closure named `describe` —
+        // the #215 name discipline says an externally-bound bare name
+        // cannot reach a project def, so it never counts toward the hot
+        // leaderboard). Relative modules, resolved modules, and resolver
+        // gaps (first segment names a project path) all stay countable.
+        const fileExternalNames = new Map();         // filePath -> Set<string>
         for (const [filePath, fileEntry] of index.files) {
             const aliases = new Set();
             // importNames are the named imports/exports brought into this file.
@@ -147,6 +153,16 @@ function getStats(index, options = {}) {
                 }
             }
             fileImportAliases.set(filePath, aliases);
+            let ext = null;
+            for (const b of (fileEntry.importBindings || [])) {
+                const mod = String(b.module || '');
+                if (!b.name || !mod || mod.startsWith('.') || mod.startsWith('/')) continue;
+                if (fileEntry.moduleResolved && fileEntry.moduleResolved[mod]) continue;
+                const firstSeg = mod.split(/[./]/).filter(Boolean)[0];
+                if (firstSeg && _projectTopLevelNames(index).has(firstSeg)) continue;
+                (ext || (ext = new Set())).add(b.name);
+            }
+            if (ext) fileExternalNames.set(filePath, ext);
         }
 
         for (const [filePath, entry] of index.callsCache) {
@@ -164,6 +180,9 @@ function getStats(index, options = {}) {
                     // Bare-name call: foo() or pkg.Foo() (Go package call has receiver
                     // but isMethod:false — keep counting under bareName since they
                     // resolve like top-level functions in their package).
+                    // Externally-bound names are the external library's calls,
+                    // never a project def's (fix #256).
+                    if (fileExternalNames.get(filePath)?.has(c.name)) continue;
                     bareNameCounts.set(c.name, (bareNameCounts.get(c.name) || 0) + 1);
                 } else if (isSelfMethod) {
                     // self/this.foo() — attributed to the enclosing class's foo
@@ -178,11 +197,26 @@ function getStats(index, options = {}) {
                         importedReceiverCounts.set(c.name,
                             (importedReceiverCounts.get(c.name) || 0) + 1);
                     }
-                    if (c.receiverType) {
-                        let inner = methodByReceiverType.get(c.receiverType);
+                    // Field-access receivers (fix #251): `tm.service.Save()`
+                    // carries receiverRootType, not receiverType — the same
+                    // #202/#231 declared-field hop the caller/callee engine
+                    // uses. Without it, edges `context` confirms were
+                    // invisible to the hot leaderboard.
+                    let recvType = c.receiverType;
+                    if (!recvType && c.receiverField && c.receiverRootType) {
+                        const hopKey = `${c.receiverRootType}\u0000${c.receiverField}`;
+                        if (!fieldHopCache.has(hopKey)) {
+                            const lang = index.files.get(filePath)?.language;
+                            fieldHopCache.set(hopKey,
+                                lang ? _declaredFieldType(index, c.receiverRootType, c.receiverField, lang) : null);
+                        }
+                        recvType = fieldHopCache.get(hopKey);
+                    }
+                    if (recvType) {
+                        let inner = methodByReceiverType.get(recvType);
                         if (!inner) {
                             inner = new Map();
-                            methodByReceiverType.set(c.receiverType, inner);
+                            methodByReceiverType.set(recvType, inner);
                         }
                         inner.set(c.name, (inner.get(c.name) || 0) + 1);
                     }
@@ -304,7 +338,7 @@ function getStats(index, options = {}) {
             if (approximate) usedHeuristicSplit = true;
             // Sort locations by (file, startLine) for stable display.
             locations.sort((a, b) =>
-                a.file.localeCompare(b.file) ||
+                codeUnitCompare(a.file, b.file) ||
                 (a.startLine || 0) - (b.startLine || 0)
             );
             const primary = locations[0];
@@ -329,7 +363,7 @@ function getStats(index, options = {}) {
         // Stable order: callCount desc, then (relativePath, startLine) asc.
         hotList.sort((a, b) =>
             (b.callCount - a.callCount) ||
-            a.file.localeCompare(b.file) ||
+            codeUnitCompare(a.file, b.file) ||
             (a.startLine || 0) - (b.startLine || 0)
         );
 
@@ -538,28 +572,23 @@ function doctor(index, options = {}) {
     const fileCounts = { total: 0, scanned: 0 };
     const langs = {};
     let totalSymbols = 0;  // counted post-filter for accuracy when --in is set
+    // Each category tracks: count = total OCCURRENCES (uses), fileCount = TRUE
+    // number of files affected (uncapped), files = a capped sample for display.
+    // Keeping count and fileCount distinct is what lets the formatter say
+    // "481 uses in 121 files" instead of mislabeling a file count as uses or
+    // presenting the 10-file display cap as the population (field-report #2).
+    const BLINDSPOT_FILE_CAP = 10;
     const blindSpots = {
-        dynamicImports: { count: 0, files: [] },
-        evalCalls:      { count: 0, files: [] },
-        reflection:     { count: 0, files: [] },
-        parseFailures:  { count: 0, files: [] },
+        dynamicImports: { count: 0, fileCount: 0, files: [] },
+        evalCalls:      { count: 0, fileCount: 0, files: [] },
+        reflection:     { count: 0, fileCount: 0, files: [] },
+        parseFailures:  { count: 0, fileCount: 0, files: [] },
     };
 
-    // Reflection signals per language. These run textually over the source — fast,
-    // and acceptable since UCN already records dynamic-import counts at parse time.
-    const REFLECTION_PATTERNS = {
-        python:     /\b(getattr|hasattr|setattr|__import__|importlib\.import_module)\s*\(/,
-        javascript: /\bnew Function\s*\(|\bReflect\.\w+\s*\(/,
-        typescript: /\bnew Function\s*\(|\bReflect\.\w+\s*\(/,
-        go:         /"reflect"|reflect\.\w+\s*\(/,
-        java:       /\.getDeclaredMethod\b|\.getMethod\b|\.getDeclaredField\b|Class\.forName\b/,
-        rust:       /\bAny::downcast/,
-    };
-    const EVAL_PATTERNS = {
-        python:     /\b(eval|exec)\s*\(/,
-        javascript: /\beval\s*\(/,
-        typescript: /\beval\s*\(/,
-    };
+    // Reflection/eval signals come from the shared text-blind-spot counter
+    // (core/shared.js) — the SAME routine detectCompleteness uses for the about
+    // footer, so the two never drift (field-report #2). Occurrence counts.
+    const { hasTextBlindspots, countTextBlindspots } = require('./shared');
 
     for (const [filePath, fe] of index.files) {
         fileCounts.total++;
@@ -574,29 +603,22 @@ function doctor(index, options = {}) {
         langs[lang].lines += fe.lines || 0;
         totalSymbols += (fe.symbols || []).length;
 
-        if (fe.dynamicImports && fe.dynamicImports > 0) {
-            blindSpots.dynamicImports.count += fe.dynamicImports;
-            if (blindSpots.dynamicImports.files.length < 10) blindSpots.dynamicImports.files.push(rel);
-        }
-        if (fe.parseError) {
-            blindSpots.parseFailures.count++;
-            if (blindSpots.parseFailures.files.length < 10) blindSpots.parseFailures.files.push(rel);
-        }
+        const recordBlind = (cat, occurrences) => {
+            if (occurrences <= 0) return;
+            cat.count += occurrences;
+            cat.fileCount++;
+            if (cat.files.length < BLINDSPOT_FILE_CAP) cat.files.push(rel);
+        };
+
+        if (fe.dynamicImports && fe.dynamicImports > 0) recordBlind(blindSpots.dynamicImports, fe.dynamicImports);
+        if (fe.parseError) recordBlind(blindSpots.parseFailures, 1);
 
-        // Read file once for eval/reflection signals
-        const evalRe = EVAL_PATTERNS[lang];
-        const reflRe = REFLECTION_PATTERNS[lang];
-        if (evalRe || reflRe) {
+        // Read file once for eval/reflection signals (shared counter).
+        if (hasTextBlindspots(lang)) {
             try {
-                const content = fs.readFileSync(filePath, 'utf-8');
-                if (evalRe && evalRe.test(content)) {
-                    blindSpots.evalCalls.count++;
-                    if (blindSpots.evalCalls.files.length < 10) blindSpots.evalCalls.files.push(rel);
-                }
-                if (reflRe && reflRe.test(content)) {
-                    blindSpots.reflection.count++;
-                    if (blindSpots.reflection.files.length < 10) blindSpots.reflection.files.push(rel);
-                }
+                const bs = countTextBlindspots(fs.readFileSync(filePath, 'utf-8'), lang);
+                recordBlind(blindSpots.evalCalls, bs.eval);
+                recordBlind(blindSpots.reflection, bs.reflection);
             } catch (e) { /* ignore read errors */ }
         }
     }
@@ -620,57 +642,87 @@ function doctor(index, options = {}) {
 
     // Compute trust verdict.
     //
-    // 1. If a deep sample produced no edges (empty project, --in matches nothing),
-    //    don't pretend that's "0% confident" — return UNKNOWN.
-    // 2. Coverage gives the headline %, but blind spots (eval/reflection/dynamic
-    //    imports) downgrade the verdict by one tier each — a project that resolves
-    //    99% of edges but is full of `getattr` is not actually "HIGH" trust.
-    // 3. Parse failures always cap at MEDIUM regardless of coverage.
+    // Field-report #1: the old logic dropped the tier by one PER blind-spot
+    // category present, so any non-trivial Python/TS project (all of which have
+    // some getattr/eval/dynamic import) was forced to LOW even when --deep
+    // measured ~99% of edges at confidence ≥ 0.5 — a self-contradicting verdict
+    // ("99.1% ... LOW") that trains agents to distrust a healthy index. The fix:
+    //   - When --deep coverage exists it drives the tier. Coverage measures the
+    //     CONFIDENCE of edges UCN FOUND, NOT completeness — a reflection-hidden
+    //     edge is absent from the sample, never a low-confidence edge dragging
+    //     the % down — so sparse blind spots are a CAVEAT, while PERVASIVE ones
+    //     (a large share of files) can hide edges the sample can't see and cap
+    //     the verdict at MEDIUM (density, not mere presence; see below).
+    //   - Parse failures are a separate exception: a file UCN couldn't parse is
+    //     not in the sample at all, a genuine uncounted hole → cap at MEDIUM.
+    //   - Without --deep there is no measurement, so blind spots are the only
+    //     signal — but bounded to ONE tier total (not one per category), so a
+    //     handful of getattr doesn't read as untrustworthy.
     let trust = 'UNKNOWN';
     let trustReason = '';
-    const reasons = [];
+    const tier = ['HIGH', 'MEDIUM', 'LOW'];
+
+    const blindSignals = [];
+    if (blindSpots.parseFailures.count > 0) blindSignals.push(`${blindSpots.parseFailures.count} parse failure(s)`);
+    if (blindSpots.evalCalls.count > 0) blindSignals.push(`${blindSpots.evalCalls.count} eval/exec use(s) in ${blindSpots.evalCalls.fileCount} file(s)`);
+    if (blindSpots.reflection.count > 0) blindSignals.push(`${blindSpots.reflection.count} reflection use(s) in ${blindSpots.reflection.fileCount} file(s)`);
+    if (blindSpots.dynamicImports.count > 0) blindSignals.push(`${blindSpots.dynamicImports.count} dynamic import(s) in ${blindSpots.dynamicImports.fileCount} file(s)`);
 
     if (coverage && coverage.total > 0) {
         const safe = coverage.high + coverage.medium;
         const safePct = safe / coverage.total;
-        let baseLevel;
-        if (safePct >= 0.85) baseLevel = 'HIGH';
-        else if (safePct >= 0.6) baseLevel = 'MEDIUM';
-        else baseLevel = 'LOW';
-        reasons.push(`${(safePct * 100).toFixed(1)}% of edges have confidence ≥ 0.5`);
-
-        // Blind-spot downgrades — each kind drops one tier.
-        const tier = ['HIGH', 'MEDIUM', 'LOW'];
-        let idx = tier.indexOf(baseLevel);
-        const blindSignals = [];
-        if (blindSpots.parseFailures.count > 0) { idx = Math.max(idx, 1); blindSignals.push(`${blindSpots.parseFailures.count} parse failure(s)`); }
-        if (blindSpots.evalCalls.count > 0) { idx = Math.min(2, idx + 1); blindSignals.push(`${blindSpots.evalCalls.count} eval call(s)`); }
-        if (blindSpots.reflection.count > 0) { idx = Math.min(2, idx + 1); blindSignals.push(`${blindSpots.reflection.count} reflection use(s)`); }
-        if (blindSpots.dynamicImports.count > 0) { idx = Math.min(2, idx + 1); blindSignals.push(`${blindSpots.dynamicImports.count} dynamic import(s)`); }
+        let idx = safePct >= 0.85 ? 0 : safePct >= 0.6 ? 1 : 2;
+        // Parse failures: unparsed files aren't in the sample at all.
+        if (blindSpots.parseFailures.count > 0) idx = Math.max(idx, 1);
+        // Coverage measures the CONFIDENCE of edges UCN found, NOT completeness:
+        // a call hidden behind reflection/dynamic dispatch is simply absent from
+        // findCallers' result, never a low-confidence edge that drags the % down.
+        // So when blind spots are PERVASIVE — affecting a large share of files —
+        // they can hide a real fraction of the call graph that the sample can't
+        // see, and the verdict is capped at MEDIUM. Density, not mere presence:
+        // a handful of getattr stays a caveat (the old code dropped a tier per
+        // category, forcing every project to LOW); reflection across half the
+        // files does cap. Gated on a file-count floor — file share is meaningless
+        // for a handful of files, so small projects ride on coverage alone.
+        const scanned = fileCounts.scanned || 1;
+        const share = (fc) => fc / scanned;
+        const pervasiveBlindSpot = scanned >= 10 && (
+            share(blindSpots.reflection.fileCount) >= 0.5 ||
+            share(blindSpots.dynamicImports.fileCount) >= 0.4 ||
+            share(blindSpots.evalCalls.fileCount) >= 0.15
+        );
+        const baseIdx = idx;
+        if (pervasiveBlindSpot) idx = Math.max(idx, 1);
+        const capped = idx > baseIdx;
         trust = tier[idx];
-        if (blindSignals.length) reasons.push(`blind spots: ${blindSignals.join(', ')}`);
+        const reasons = [`${(safePct * 100).toFixed(1)}% of found edges have confidence ≥ 0.5`];
+        if (blindSignals.length) {
+            reasons.push(capped
+                ? `capped at MEDIUM — pervasive blind spots may hide edges the sample can't see: ${blindSignals.join(', ')}`
+                : `blind spots (caveat — coverage measures found edges, not completeness): ${blindSignals.join(', ')}`);
+        }
         trustReason = reasons.join('; ');
     } else if (coverage) {
         // Sampled but zero edges — can't say anything about confidence.
         trust = 'UNKNOWN';
         trustReason = 'no edges sampled (empty scope or filter matched nothing)';
     } else if (fileCounts.scanned > 0) {
-        // Cheap path (no --deep): use blind-spot signals.
-        const tier = ['HIGH', 'MEDIUM', 'LOW'];
+        // Cheap path (no --deep): no measurement, so blind spots are the only
+        // signal — bounded to one tier total. Run --deep for a measured verdict.
         let idx = 0;
-        const blindSignals = [];
-        if (blindSpots.parseFailures.count > 0) { idx = Math.max(idx, 1); blindSignals.push(`${blindSpots.parseFailures.count} parse failure(s)`); }
-        if (blindSpots.evalCalls.count > 0) { idx = Math.min(2, idx + 1); blindSignals.push(`${blindSpots.evalCalls.count} eval call(s)`); }
-        if (blindSpots.reflection.count > 0) { idx = Math.min(2, idx + 1); blindSignals.push(`${blindSpots.reflection.count} reflection use(s)`); }
-        if (blindSpots.dynamicImports.count > 0) { idx = Math.min(2, idx + 1); blindSignals.push(`${blindSpots.dynamicImports.count} dynamic import(s)`); }
+        if (blindSpots.parseFailures.count > 0) idx = Math.max(idx, 1);
+        if (blindSpots.evalCalls.count + blindSpots.reflection.count + blindSpots.dynamicImports.count > 0) {
+            idx = Math.min(2, idx + 1);
+        }
         trust = tier[idx];
         trustReason = blindSignals.length
-            ? `coverage not deep-checked; blind spots: ${blindSignals.join(', ')}`
-            : 'no parse failures; coverage not deep-checked';
+            ? `coverage not deep-checked (run --deep); blind spots: ${blindSignals.join(', ')}`
+            : 'no parse failures; coverage not deep-checked (run --deep)';
     }
 
     return {
         root: index.root,
+        version: require('../package.json').version,  // running ucn version — surfaces MCP/CLI drift (field-report #3)
         files: fileCounts,
         symbols: totalSymbols,
         languages: langs,
@@ -718,4 +770,88 @@ function computeCoverageSample(index, { sampleSize, inFilter, matchInFilter }) {
     return buckets;
 }
 
-module.exports = { getStats, getToc, doctor };
+/**
+ * orient — one-call cold-repo orientation: size + language mix, densest
+ * directories, most-called functions, entry-point counts, and the doctor
+ * trust verdict. Composes existing engine reads; counts and pointers only
+ * (no caller claims, so no account — the toc/stats category).
+ */
+function orient(index, options = {}) {
+    const top = options.top || 8;
+    // Fetch a deeper hot list so production functions survive the filter
+    // below even when test helpers dominate raw call counts.
+    const stats = getStats(index, { hot: true, top: Math.min(top * 5, 200) });
+    const health = doctor(index, {});
+
+    // Densest directories (leaf dirname rollup — "where does the code live")
+    const dirMap = new Map();
+    for (const [, fe] of index.files) {
+        const rp = fe.relativePath;
+        if (!rp) continue;
+        const slash = rp.lastIndexOf('/');
+        const dir = slash === -1 ? '.' : rp.slice(0, slash);
+        const e = dirMap.get(dir) || { dir, files: 0, symbols: 0 };
+        e.files += 1;
+        e.symbols += (fe.symbols || []).length;
+        dirMap.set(dir, e);
+    }
+    const dirs = [...dirMap.values()]
+        .sort((a, b) => b.symbols - a.symbols || codeUnitCompare(a.dir, b.dir))
+        .slice(0, top);
+
+    // Entry-point counts by type — orientation must not fail on detection
+    let entrypoints = null;
+    try {
+        const { detectEntrypoints } = require('./entrypoints');
+        const eps = detectEntrypoints(index, {});
+        if (Array.isArray(eps)) {
+            const byType = new Map();
+            for (const e of eps) byType.set(e.type, (byType.get(e.type) || 0) + 1);
+            entrypoints = {
+                total: eps.length,
+                byType: [...byType.entries()]
+                    .map(([type, count]) => ({ type, count }))
+                    .sort((a, b) => b.count - a.count || codeUnitCompare(a.type, b.type)),
+            };
+        }
+    } catch { /* detection error → entrypoints stays null, rendered as unavailable */ }
+
+    // Orientation wants the ENGINE's hot functions, not fixture helpers —
+    // prefer production-path entries (labeled as such by the formatter);
+    // an all-test project falls back to the raw ranking.
+    const { isTestPath } = require('./shared');
+    const allHot = (stats.hot?.items || []).map(i => ({
+        name: i.name,
+        file: i.file,
+        line: i.startLine,
+        callCount: i.callCount,
+        ...(i.className ? { className: i.className } : {}),
+    }));
+    const prodHot = allHot.filter(i => i.file && !isTestPath(i.file));
+    const production = prodHot.length > 0;
+    const hotItems = (production ? prodHot : allHot).slice(0, top);
+    const hottestProd = hotItems[0] || null;
+
+    return {
+        root: stats.root,
+        files: stats.files,
+        symbols: stats.symbols,
+        buildTime: stats.buildTime,
+        byLanguage: stats.byLanguage,
+        dirs,
+        hot: { total: stats.hot?.total ?? 0, top, production, items: hotItems },
+        entrypoints,
+        trust: {
+            level: health.trust,
+            blindSpots: {
+                dynamicImports: health.blindSpots?.dynamicImports?.count ?? 0,
+                evalCalls: health.blindSpots?.evalCalls?.count ?? 0,
+                reflection: health.blindSpots?.reflection?.count ?? 0,
+                parseFailures: health.blindSpots?.parseFailures?.count ?? 0,
+            },
+        },
+        suggest: hottestProd ? hottestProd.name : null,
+    };
+}
+
+module.exports = { getStats, getToc, doctor, orient };