ucn 5.5.3 → 5.6.1

package/core/reporting.js

@@ -11,6 +11,7 @@ const fs = require('fs');
 const { codeUnitCompare, CALLABLE_SYMBOL_KINDS } = require('./shared');
 const path = require('path');
 const { isTestFile } = require('./discovery');
+const { paramFact } = require('./registry');
 const { summarizeCommandTrust } = require('./trust-matrix');
 const { projectComputedDispatch } = require('./ast-analysis');
 const { langTraits } = require('../languages');
@@ -123,8 +124,37 @@ function getStats(index, options = {}) {
         // positive integer, 0 (show nothing), or undefined (default 10).
         const top = options.top === 0
             ? 0
-            : ((options.top != null && Number(options.top) > 0) ? Number(options.top) : 10);
+            : ((options.top != null && Number(options.top) > 0) ? Number(options.top) : paramFact('top', 'hot'));
         const FUNCTION_TYPES = CALLABLE_SYMBOL_KINDS;
+        const maxRefine = Number.isInteger(options.maxRefine) && options.maxRefine > 0
+            ? options.maxRefine : Infinity;
+        // Refinement work is bounded by call records, not wall time, so the
+        // answer is the same on every run and machine (fix #382). One
+        // refinement scans the call records of its name in the countable
+        // files; `workBudget` caps their sum. Exact answers on real projects
+        // stay far below it; a very large repository (kubernetes: ~68M
+        // records under its candidate ceilings) stops and says so.
+        const workBudget = Number.isFinite(options.workBudget) && options.workBudget > 0
+            ? options.workBudget : STATS_HOT_WORK_BUDGET;
+
+        // The ranking is a function of the index and of these options, so an
+        // answer already computed for this index state is the answer
+        // (core/hot-summary.js). Nothing below runs for it.
+        const hotSummary = require('./hot-summary');
+        const summaryKey = hotSummary.keyOf({
+            top, productionCallsOnly: options.productionCallsOnly, maxRefine, workBudget,
+            file: options.file, in: options.in,
+            exclude: Array.isArray(options.exclude)
+                ? options.exclude
+                : (options.exclude
+                    ? String(options.exclude).split(',').map(s => s.trim()).filter(Boolean)
+                    : []),
+        });
+        const keptSummary = hotSummary.get(index, summaryKey);
+        if (keptSummary) {
+            stats.hot = keptSummary;
+            return stats;
+        }
 
         // Ensure the calls cache is fully populated before counting.
         // First-time stats --hot may need to parse files to extract calls;
@@ -374,16 +404,6 @@ function getStats(index, options = {}) {
                 }
             }
         }
-        const maxRefine = Number.isInteger(options.maxRefine) && options.maxRefine > 0
-            ? options.maxRefine : Infinity;
-        // Refinement work is bounded by call records, not wall time, so the
-        // answer is the same on every run and machine (fix #382). One
-        // refinement scans the call records of its name in the countable
-        // files; `workBudget` caps their sum. Exact answers on real projects
-        // stay far below it; a very large repository (kubernetes: ~68M
-        // records under its candidate ceilings) stops and says so.
-        const workBudget = Number.isFinite(options.workBudget) && options.workBudget > 0
-            ? options.workBudget : STATS_HOT_WORK_BUDGET;
         const workOf = candidate => recordsByName.get(candidate.name) || 0;
         // Ceilings in descending order: the early stop fires as soon as no
         // unseen ceiling can reach the current Nth count, so exactly the
@@ -510,7 +530,7 @@ function getStats(index, options = {}) {
             (a.startLine || 0) - (b.startLine || 0)
         );
 
-        stats.hot = {
+        stats.hot = hotSummary.put(index, summaryKey, {
             top,
             total: refined === candidates.length ? hotList.length : candidates.length,
             totalKind: refined === candidates.length ? 'confirmed' : 'raw-call-candidates',
@@ -523,7 +543,7 @@ function getStats(index, options = {}) {
             note: refined === candidates.length
                 ? 'Counts are confirmed caller-engine edges pinned to each displayed definition; unverified dispatch is excluded.'
                 : `Displayed counts are exact confirmed caller-engine edges; ${candidates.length} raw candidates were bounded and ${refined} required exact refinement.`,
-        };
+        });
     }
 
     return stats;
@@ -747,11 +767,6 @@ function doctor(index, options = {}) {
         skippedSources: { count: 0, fileCount: 0, files: [], reasons: {} },
     };
 
-    // Reflection/eval signals come from the shared text-blind-spot counter
-    // (core/shared.js) — the SAME routine detectCompleteness uses for the about
-    // footer, so the two never drift (field-report #2). Occurrence counts.
-    const { hasTextBlindspots, countTextBlindspots } = lazySharedModule();
-
     const computedDispatch = projectComputedDispatch(index);
     for (const [filePath, fe] of index.files) {
         fileCounts.total++;
@@ -781,14 +796,12 @@ function doctor(index, options = {}) {
         if (fe.parseError) recordBlind(blindSpots.parseFailures, 1);
         if (fe.parseRecovery) recordBlind(blindSpots.parseRecoveries, 1);
 
-        // Read file once for eval/reflection signals (shared counter).
-        if (hasTextBlindspots(lang)) {
-            try {
-                const bs = countTextBlindspots(fs.readFileSync(filePath, 'utf-8'), lang);
-                recordBlind(blindSpots.evalCalls, bs.eval);
-                recordBlind(blindSpots.reflection, bs.reflection);
-            } catch (e) { /* ignore read errors */ }
-        }
+        // Code execution and reflection: call sites of each language's
+        // vocabulary (langTraits().reflectionApi), counted in the file's
+        // tree when it was indexed. The completeness footer reads the same
+        // two numbers, so the two never drift, and no file is read here.
+        recordBlind(blindSpots.evalCalls, fe.evalUses || 0);
+        recordBlind(blindSpots.reflection, fe.reflectionUses || 0);
     }
 
     // Files that failed before a fileEntry could be created are absent from
@@ -825,16 +838,30 @@ function doctor(index, options = {}) {
     }
     fileCounts.unsupported = blindSpots.unsupportedSources.fileCount;
 
-    for (const issue of index.discoveryIssues || []) {
-        const rel = issue.relativePath || '.';
-        if (!matchInFilter(rel)) continue;
+    // Source that is not indexed, from the one model every claim uses
+    // (core/source-universe.js): files searched as text only (a bundle, an
+    // excluded path, an oversized file; unsupported languages are counted
+    // above) and paths nothing can be read from. `unsearchable` is the part
+    // no name can be cleared against.
+    const sourceUniverse = require('./source-universe').universe(index);
+    let unsearchablePaths = 0;
+    const noteSkipped = (rel, reason) => {
+        if (!matchInFilter(rel)) return false;
         const skipped = blindSpots.skippedSources;
         skipped.count++;
         skipped.fileCount++;
-        skipped.reasons[issue.reason || 'unknown'] =
-            (skipped.reasons[issue.reason || 'unknown'] || 0) + 1;
+        skipped.reasons[reason] = (skipped.reasons[reason] || 0) + 1;
         if (skipped.files.length < BLINDSPOT_FILE_CAP) skipped.files.push(rel);
+        return true;
+    };
+    for (const entry of sourceUniverse.textOnly) {
+        if (entry.reason === 'unsupported-language') continue;
+        noteSkipped(entry.relativePath, entry.reason);
+    }
+    for (const entry of sourceUniverse.opaque) {
+        if (noteSkipped(entry.relativePath, entry.reason || 'unknown')) unsearchablePaths++;
     }
+    blindSpots.skippedSources.unsearchable = unsearchablePaths;
     fileCounts.skipped = blindSpots.skippedSources.fileCount;
 
     // C/C++ token-pasting macro invocations whose expansion cannot be
@@ -910,7 +937,7 @@ function doctor(index, options = {}) {
     if (blindSpots.skippedSources.count > 0) {
         const reasons = Object.entries(blindSpots.skippedSources.reasons)
             .map(([reason, count]) => `${reason} ${count}`).join(', ');
-        blindSignals.push(`${blindSpots.skippedSources.count} source discovery gap(s): ${reasons}`);
+        blindSignals.push(`${blindSpots.skippedSources.count} source path(s) not indexed: ${reasons}`);
     }
 
     // Trust is task-specific. A healthy index can be excellent for navigation
@@ -932,8 +959,10 @@ function doctor(index, options = {}) {
                 ? `${blindSpots.parseRecoveries.count} file(s) required parser recovery; indexed results may be partial`
                 : cache.fresh === false ? 'index cache is stale'
                     : index.truncated ? `file discovery stopped at maxFiles=${index.truncated.maxFiles}`
+                        : unsearchablePaths > 0
+                            ? `${unsearchablePaths} path(s) could not be searched; no caller account is complete until they can`
                         : blindSpots.skippedSources.count > 0
-                            ? `${blindSpots.skippedSources.count} source path(s) were not indexed; inspect discovery reasons`
+                            ? `${blindSpots.skippedSources.count} source file(s) are searched as text only; a name that occurs in them is listed, not analyzed`
                         : unsupportedCount > 0
                             ? `${fileCounts.scanned} supported file(s) indexed; ${unsupportedCount} unsupported source file(s) require grep/ripgrep`
                             : 'fresh index; no parse failures';
@@ -965,8 +994,11 @@ function doctor(index, options = {}) {
         : 'no known runtime blind spots detected; alias/dynamic completeness is not compiler-verified locally';
 
     const navigationLevel = indexLevel;
+    // A path nothing can be read from leaves every caller account
+    // incomplete, which is what blocks `check`: the two must agree.
     const hardIndexRisk = blindSpots.parseFailures.count > 0 ||
-        blindSpots.parseRecoveries.count > 0 || cache.fresh === false || !!index.truncated;
+        blindSpots.parseRecoveries.count > 0 || cache.fresh === false || !!index.truncated ||
+        unsearchablePaths > 0;
     const refactorLevel = indexLevel === 'UNKNOWN' ? 'UNKNOWN'
         : indexLevel === 'UNSUPPORTED' ? 'UNSUPPORTED'
             : hardIndexRisk ? 'LOW'
@@ -1005,6 +1037,9 @@ function doctor(index, options = {}) {
         symbols: totalSymbols,
         languages: langs,
         blindSpots,
+        // What completeness claims range over: text-only, unsearchable and
+        // declared-outside paths, stated here once.
+        scope: require('./source-universe').scopeSummary(index),
         evidenceProfile,
         cache,
         commandTrust: summarizeCommandTrust(),

package/core/review-items.js

@@ -0,0 +1,75 @@
+/**
+ * core/review-items.js - How much is there to look at?
+ *
+ * Unverified caller candidates are not all the same amount of work. A site
+ * the engine could not bind is one thing to check. A dispatch family (an
+ * interface call that may reach any implementation, a template the
+ * compiler instantiates) is ONE boundary with a known reason, however many
+ * sites it lists. A reader reviews the boundary, not each site.
+ *
+ * One counter, shared by the pre-commit check, the caller formatter and
+ * the oracle evaluation, so the three cannot disagree about the burden.
+ */
+
+'use strict';
+
+/**
+ * Why a caller candidate is unverified, as one of three classes:
+ * 'runtime-dispatch' (the implementation is selected at runtime through a
+ * named boundary), 'compile-time-dispatch' (the compiler selects it),
+ * 'actionable-ambiguity' (something a reader can settle). The class the
+ * engine recorded wins; a possible-dispatch site with a named boundary is
+ * runtime dispatch.
+ */
+function uncertaintyClassOf(site) {
+    return site.uncertaintyClass ||
+        (site.reason === 'possible-dispatch' && site.dispatchVia ? 'runtime-dispatch' : 'actionable-ambiguity');
+}
+
+/** Unverified entries by class: { actionable, runtime, compileTime }. */
+function partitionUnverified(unverified) {
+    const parts = { actionable: [], runtime: [], compileTime: [] };
+    for (const site of Array.isArray(unverified) ? unverified : []) {
+        const kind = uncertaintyClassOf(site);
+        if (kind === 'runtime-dispatch') parts.runtime.push(site);
+        else if (kind === 'compile-time-dispatch') parts.compileTime.push(site);
+        else parts.actionable.push(site);
+    }
+    return parts;
+}
+
+/** Key of the family a runtime-dispatch site belongs to. */
+function runtimeFamilyKey(site) {
+    return `${site.dispatchVia || ''}\0${site.dispatchCandidates ?? ''}\0` +
+        `${site.externalContract ? 'external' : 'project'}`;
+}
+
+/** Key of the family a compile-time-dispatch site belongs to. */
+function compileTimeFamilyKey(site) {
+    return `${site.dispatchFamily || ''}\0${site.dispatchCandidates ?? ''}`;
+}
+
+/**
+ * @param {object[]} unverified - unverified caller entries
+ * @returns {{actionable, runtimeDispatchSites, runtimeDispatchGroups,
+ *   compileTimeDispatchSites, compileTimeDispatchGroups, items}}
+ *   `items` = actionable sites + one per dispatch family.
+ */
+function reviewItemsOf(unverified) {
+    const { actionable: open, runtime, compileTime } = partitionUnverified(unverified);
+    const actionable = open.length;
+    const runtimeDispatchGroups = new Set(runtime.map(runtimeFamilyKey)).size;
+    const compileTimeDispatchGroups = new Set(compileTime.map(compileTimeFamilyKey)).size;
+    return {
+        actionable,
+        runtimeDispatchSites: runtime.length,
+        runtimeDispatchGroups,
+        compileTimeDispatchSites: compileTime.length,
+        compileTimeDispatchGroups,
+        items: actionable + runtimeDispatchGroups + compileTimeDispatchGroups,
+    };
+}
+
+module.exports = {
+    reviewItemsOf, uncertaintyClassOf, partitionUnverified, runtimeFamilyKey, compileTimeFamilyKey,
+};

package/core/runtime-entrypoints.js

@@ -0,0 +1,159 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const { pythonPossibleIdentities, pythonBoundValue, pythonSourceName } = require('./python-binding-identity');
+const { entrypointToml } = require('./entrypoint-toml');
+const OP_ENTRIES = new WeakMap();
+
+/** A runtime module/member path names this declaration, including src layouts. */
+function runtimeTargetReaches(index, symbol, target) {
+    const spelling = target.replace(':', '.').split('[')[0].trim();
+    const relative = path.relative(index.root, symbol.file).split(path.sep).join('/');
+    if (!relative.endsWith('.py')) return false;
+    const module = relative.slice(0, -3).replace(/\/__init__$/, '').replaceAll('/', '.');
+    const mangle = (name, owner) => owner && name.startsWith('__') && !name.endsWith('__') && owner.replace(/^_+/, '')
+        ? `_${owner.replace(/^_+/, '')}${name}` : name;
+    let owners = symbol.runtimeOwnerPath;
+    let unknownOwner = false;
+    if (!Array.isArray(owners)) {
+        // Old/external records may omit nesting. Reconstruct only from a unique
+        // owner; when that owner is absent, preserve the possible suffix.
+        const candidates = (index.symbols.get(symbol.className) || []).filter(def => def.file === symbol.file && def.type === 'class');
+        if (symbol.className && candidates.length === 1) owners = [...(candidates[0].runtimeOwnerPath || []), symbol.className];
+        else { owners = symbol.className ? [symbol.className] : []; unknownOwner = !!symbol.className; }
+    }
+    const normalizedOwners = owners.map(pythonSourceName);
+    const member = [...normalizedOwners.map((name, position) => mangle(name, normalizedOwners[position - 1])),
+        mangle(pythonSourceName(symbol.name), normalizedOwners[normalizedOwners.length - 1])].join('.');
+    if (unknownOwner && spelling.endsWith(`.${mangle(pythonSourceName(symbol.name), pythonSourceName(symbol.className))}`)) {
+        const parts = module.split('.');
+        if (parts.some((_, position) => spelling.startsWith(`${parts.slice(position).join('.')}.`))) return true;
+    }
+    if (!spelling.endsWith(`.${member}`)) return false;
+    const namedModule = spelling.slice(0, -(member.length + 1));
+    return module === namedModule || module.endsWith(`.${namedModule}`);
+}
+
+function literal(node) {
+    node = pythonBoundValue(node);
+    if (!node || node.type !== 'string' || node.namedChildren.some(child => child.type === 'interpolation')) return null;
+    const raw = node.text;
+    const match = /^([rub]*)("""|'''|"|')([\s\S]*)\2$/i.exec(raw);
+    if (!match) return null;
+    if (/r/i.test(match[1])) return match[3];
+    let valid = true;
+    const value = match[3].replace(/\\(U[0-9a-fA-F]{8}|u[0-9a-fA-F]{4}|x[0-9a-fA-F]{2}|[0-7]{1,3}|[\s\S])/g, (whole, escape) => {
+        if (/^[0-7]+$/.test(escape)) return String.fromCharCode(parseInt(escape, 8));
+        if (/^[uUx]/.test(escape)) {
+            const code = parseInt(escape.slice(1), 16);
+            if (code <= 0x10ffff) return String.fromCodePoint(code);
+            valid = false; return '';
+        }
+        const simple = { n: '\n', r: '\r', t: '\t', '"': '"', "'": "'", '\\': '\\', '\n': '' };
+        if (Object.hasOwn(simple, escape)) return simple[escape];
+        valid = false; return whole;
+    });
+    return valid ? value : null;
+}
+
+function targetOf(value) {
+    const candidate = value.includes('=') ? value.slice(value.lastIndexOf('=') + 1).trim() : value.trim();
+    return /^[\p{L}_][\p{L}\p{N}_.]*\s*:\s*[\p{L}_][\p{L}\p{N}_.]*(?:\s*\[[^\]]*\])?$/u.test(candidate)
+        ? candidate.replace(/\s/g, '') : null;
+}
+
+/** Static packaging declarations, read as data; setup.py is never executed. */
+function runtimeEntrypoints(index) {
+    const operation = index._opTreeCache;
+    if (operation && OP_ENTRIES.has(operation)) return OP_ENTRIES.get(operation);
+    const dirs = new Set();
+    for (const [file, entry] of index.files) {
+        if (entry.language !== 'python') continue;
+        for (let dir = path.dirname(file); dir === index.root || dir.startsWith(`${index.root}${path.sep}`); dir = path.dirname(dir)) {
+            dirs.add(dir);
+            if (dir === index.root) break;
+        }
+    }
+    const entries = [];
+    const add = (value, file, line) => {
+        const target = targetOf(value);
+        entries.push(target ? { target, file, line } : { unknown: true, file, line });
+    };
+    for (const dir of [...dirs].sort()) {
+        for (const basename of ['pyproject.toml', 'setup.cfg']) {
+            const file = path.join(dir, basename);
+            let source;
+            try { source = fs.readFileSync(file, 'utf8'); } catch { continue; }
+            if (basename === 'pyproject.toml') {
+                for (const row of entrypointToml(source)) {
+                    if (row.unknown) entries.push({ unknown: true, file, line: row.line });
+                    else add(row.value, file, row.line);
+                }
+                continue;
+            }
+            let section = '';
+            const lines = source.split('\n');
+            for (let i = 0; i < lines.length; i++) {
+                const line = lines[i].trim();
+                const header = /^\[([^\]]+)\]/.exec(line);
+                if (header) { section = header[1]; continue; }
+                if (section === 'options.entry_points' && line && !line.startsWith('#') && !line.startsWith(';') &&
+                    !/^[\w.-]+\s*=\s*$/.test(line)) add(line, file, i + 1);
+            }
+        }
+        const setup = path.join(dir, 'setup.py');
+        if (!index.files.has(setup)) continue;
+        let tree;
+        try { tree = index._getParsedTree(setup, index._readFile(setup), 'python'); } catch { continue; }
+        for (const call of tree.rootNode.descendantsOfType('call')) {
+            if (!pythonPossibleIdentities(call.childForFieldName('function')).some(name => ['setuptools.setup', 'distutils.core.setup'].includes(name))) continue;
+            const unknown = () => entries.push({ unknown: true, file: setup, line: call.startPosition.row + 1 });
+            const values = [];
+            const expanded = (node, depth = 0) => {
+                const value = pythonBoundValue(node);
+                if (depth > 8 || value?.type !== 'dictionary') { unknown(); return; }
+                for (const pair of value.namedChildren) {
+                    if (pair.type === 'dictionary_splat') { expanded(pair.namedChild(0), depth + 1); continue; }
+                    if (pair.type !== 'pair') { unknown(); continue; }
+                    const key = literal(pair.childForFieldName('key'));
+                    if (key == null) unknown();
+                    else if (key === 'entry_points') values.push(pair.childForFieldName('value'));
+                }
+            };
+            for (const argument of call.childForFieldName('arguments')?.namedChildren || []) {
+                if (argument.type === 'keyword_argument' && pythonSourceName(argument.childForFieldName('name')?.text) === 'entry_points') values.push(argument.childForFieldName('value'));
+                else if (argument.type === 'dictionary_splat') expanded(argument.namedChild(0));
+            }
+            for (const expression of values) {
+                const value = pythonBoundValue(expression);
+                if (value?.type === 'none') continue;
+                // setuptools also accepts an INI string as entry_points.
+                const ini = literal(value);
+                if (ini != null) {
+                    for (const line of ini.split('\n')) {
+                        const row = line.trim();
+                        if (row && !row.startsWith('[') && !row.startsWith('#')) add(row, setup, call.startPosition.row + 1);
+                    }
+                    continue;
+                }
+                if (value?.type !== 'dictionary') { unknown(); continue; }
+                for (const pair of value.namedChildren) {
+                    if (pair.type !== 'pair' || literal(pair.childForFieldName('key')) == null) { unknown(); continue; }
+                    const group = pythonBoundValue(pair.childForFieldName('value'));
+                    const rows = ['list', 'tuple'].includes(group?.type) ? group.namedChildren : [group];
+                    for (const row of rows) {
+                        const text = literal(row);
+                        if (text != null) {
+                            for (const line of text.split('\n').filter(line => line.trim())) add(line, setup, row.startPosition.row + 1);
+                        } else unknown();
+                    }
+                }
+            }
+        }
+    }
+    if (operation) OP_ENTRIES.set(operation, entries);
+    return entries;
+}
+
+module.exports = { runtimeEntrypoints, runtimeTargetReaches, pythonStringLiteral: literal };