ucn 5.5.2 → 5.6.0

package/core/reporting.js

@@ -11,6 +11,7 @@ const fs = require('fs');
 const { codeUnitCompare, CALLABLE_SYMBOL_KINDS } = require('./shared');
 const path = require('path');
 const { isTestFile } = require('./discovery');
+const { paramFact } = require('./registry');
 const { summarizeCommandTrust } = require('./trust-matrix');
 const { projectComputedDispatch } = require('./ast-analysis');
 const { langTraits } = require('../languages');
@@ -123,8 +124,37 @@ function getStats(index, options = {}) {
         // positive integer, 0 (show nothing), or undefined (default 10).
         const top = options.top === 0
             ? 0
-            : ((options.top != null && Number(options.top) > 0) ? Number(options.top) : 10);
+            : ((options.top != null && Number(options.top) > 0) ? Number(options.top) : paramFact('top', 'hot'));
         const FUNCTION_TYPES = CALLABLE_SYMBOL_KINDS;
+        const maxRefine = Number.isInteger(options.maxRefine) && options.maxRefine > 0
+            ? options.maxRefine : Infinity;
+        // Refinement work is bounded by call records, not wall time, so the
+        // answer is the same on every run and machine (fix #382). One
+        // refinement scans the call records of its name in the countable
+        // files; `workBudget` caps their sum. Exact answers on real projects
+        // stay far below it; a very large repository (kubernetes: ~68M
+        // records under its candidate ceilings) stops and says so.
+        const workBudget = Number.isFinite(options.workBudget) && options.workBudget > 0
+            ? options.workBudget : STATS_HOT_WORK_BUDGET;
+
+        // The ranking is a function of the index and of these options, so an
+        // answer already computed for this index state is the answer
+        // (core/hot-summary.js). Nothing below runs for it.
+        const hotSummary = require('./hot-summary');
+        const summaryKey = hotSummary.keyOf({
+            top, productionCallsOnly: options.productionCallsOnly, maxRefine, workBudget,
+            file: options.file, in: options.in,
+            exclude: Array.isArray(options.exclude)
+                ? options.exclude
+                : (options.exclude
+                    ? String(options.exclude).split(',').map(s => s.trim()).filter(Boolean)
+                    : []),
+        });
+        const keptSummary = hotSummary.get(index, summaryKey);
+        if (keptSummary) {
+            stats.hot = keptSummary;
+            return stats;
+        }
 
         // Ensure the calls cache is fully populated before counting.
         // First-time stats --hot may need to parse files to extract calls;
@@ -374,16 +404,6 @@ function getStats(index, options = {}) {
                 }
             }
         }
-        const maxRefine = Number.isInteger(options.maxRefine) && options.maxRefine > 0
-            ? options.maxRefine : Infinity;
-        // Refinement work is bounded by call records, not wall time, so the
-        // answer is the same on every run and machine (fix #382). One
-        // refinement scans the call records of its name in the countable
-        // files; `workBudget` caps their sum. Exact answers on real projects
-        // stay far below it; a very large repository (kubernetes: ~68M
-        // records under its candidate ceilings) stops and says so.
-        const workBudget = Number.isFinite(options.workBudget) && options.workBudget > 0
-            ? options.workBudget : STATS_HOT_WORK_BUDGET;
         const workOf = candidate => recordsByName.get(candidate.name) || 0;
         // Ceilings in descending order: the early stop fires as soon as no
         // unseen ceiling can reach the current Nth count, so exactly the
@@ -510,7 +530,7 @@ function getStats(index, options = {}) {
             (a.startLine || 0) - (b.startLine || 0)
         );
 
-        stats.hot = {
+        stats.hot = hotSummary.put(index, summaryKey, {
             top,
             total: refined === candidates.length ? hotList.length : candidates.length,
             totalKind: refined === candidates.length ? 'confirmed' : 'raw-call-candidates',
@@ -523,7 +543,7 @@ function getStats(index, options = {}) {
             note: refined === candidates.length
                 ? 'Counts are confirmed caller-engine edges pinned to each displayed definition; unverified dispatch is excluded.'
                 : `Displayed counts are exact confirmed caller-engine edges; ${candidates.length} raw candidates were bounded and ${refined} required exact refinement.`,
-        };
+        });
     }
 
     return stats;
@@ -747,11 +767,6 @@ function doctor(index, options = {}) {
         skippedSources: { count: 0, fileCount: 0, files: [], reasons: {} },
     };
 
-    // Reflection/eval signals come from the shared text-blind-spot counter
-    // (core/shared.js) — the SAME routine detectCompleteness uses for the about
-    // footer, so the two never drift (field-report #2). Occurrence counts.
-    const { hasTextBlindspots, countTextBlindspots } = lazySharedModule();
-
     const computedDispatch = projectComputedDispatch(index);
     for (const [filePath, fe] of index.files) {
         fileCounts.total++;
@@ -781,14 +796,12 @@ function doctor(index, options = {}) {
         if (fe.parseError) recordBlind(blindSpots.parseFailures, 1);
         if (fe.parseRecovery) recordBlind(blindSpots.parseRecoveries, 1);
 
-        // Read file once for eval/reflection signals (shared counter).
-        if (hasTextBlindspots(lang)) {
-            try {
-                const bs = countTextBlindspots(fs.readFileSync(filePath, 'utf-8'), lang);
-                recordBlind(blindSpots.evalCalls, bs.eval);
-                recordBlind(blindSpots.reflection, bs.reflection);
-            } catch (e) { /* ignore read errors */ }
-        }
+        // Code execution and reflection: call sites of each language's
+        // vocabulary (langTraits().reflectionApi), counted in the file's
+        // tree when it was indexed. The completeness footer reads the same
+        // two numbers, so the two never drift, and no file is read here.
+        recordBlind(blindSpots.evalCalls, fe.evalUses || 0);
+        recordBlind(blindSpots.reflection, fe.reflectionUses || 0);
     }
 
     // Files that failed before a fileEntry could be created are absent from
@@ -825,16 +838,30 @@ function doctor(index, options = {}) {
     }
     fileCounts.unsupported = blindSpots.unsupportedSources.fileCount;
 
-    for (const issue of index.discoveryIssues || []) {
-        const rel = issue.relativePath || '.';
-        if (!matchInFilter(rel)) continue;
+    // Source that is not indexed, from the one model every claim uses
+    // (core/source-universe.js): files searched as text only (a bundle, an
+    // excluded path, an oversized file; unsupported languages are counted
+    // above) and paths nothing can be read from. `unsearchable` is the part
+    // no name can be cleared against.
+    const sourceUniverse = require('./source-universe').universe(index);
+    let unsearchablePaths = 0;
+    const noteSkipped = (rel, reason) => {
+        if (!matchInFilter(rel)) return false;
         const skipped = blindSpots.skippedSources;
         skipped.count++;
         skipped.fileCount++;
-        skipped.reasons[issue.reason || 'unknown'] =
-            (skipped.reasons[issue.reason || 'unknown'] || 0) + 1;
+        skipped.reasons[reason] = (skipped.reasons[reason] || 0) + 1;
         if (skipped.files.length < BLINDSPOT_FILE_CAP) skipped.files.push(rel);
+        return true;
+    };
+    for (const entry of sourceUniverse.textOnly) {
+        if (entry.reason === 'unsupported-language') continue;
+        noteSkipped(entry.relativePath, entry.reason);
+    }
+    for (const entry of sourceUniverse.opaque) {
+        if (noteSkipped(entry.relativePath, entry.reason || 'unknown')) unsearchablePaths++;
     }
+    blindSpots.skippedSources.unsearchable = unsearchablePaths;
     fileCounts.skipped = blindSpots.skippedSources.fileCount;
 
     // C/C++ token-pasting macro invocations whose expansion cannot be
@@ -910,7 +937,7 @@ function doctor(index, options = {}) {
     if (blindSpots.skippedSources.count > 0) {
         const reasons = Object.entries(blindSpots.skippedSources.reasons)
             .map(([reason, count]) => `${reason} ${count}`).join(', ');
-        blindSignals.push(`${blindSpots.skippedSources.count} source discovery gap(s): ${reasons}`);
+        blindSignals.push(`${blindSpots.skippedSources.count} source path(s) not indexed: ${reasons}`);
     }
 
     // Trust is task-specific. A healthy index can be excellent for navigation
@@ -932,8 +959,10 @@ function doctor(index, options = {}) {
                 ? `${blindSpots.parseRecoveries.count} file(s) required parser recovery; indexed results may be partial`
                 : cache.fresh === false ? 'index cache is stale'
                     : index.truncated ? `file discovery stopped at maxFiles=${index.truncated.maxFiles}`
+                        : unsearchablePaths > 0
+                            ? `${unsearchablePaths} path(s) could not be searched; no caller account is complete until they can`
                         : blindSpots.skippedSources.count > 0
-                            ? `${blindSpots.skippedSources.count} source path(s) were not indexed; inspect discovery reasons`
+                            ? `${blindSpots.skippedSources.count} source file(s) are searched as text only; a name that occurs in them is listed, not analyzed`
                         : unsupportedCount > 0
                             ? `${fileCounts.scanned} supported file(s) indexed; ${unsupportedCount} unsupported source file(s) require grep/ripgrep`
                             : 'fresh index; no parse failures';
@@ -965,8 +994,11 @@ function doctor(index, options = {}) {
         : 'no known runtime blind spots detected; alias/dynamic completeness is not compiler-verified locally';
 
     const navigationLevel = indexLevel;
+    // A path nothing can be read from leaves every caller account
+    // incomplete, which is what blocks `check`: the two must agree.
     const hardIndexRisk = blindSpots.parseFailures.count > 0 ||
-        blindSpots.parseRecoveries.count > 0 || cache.fresh === false || !!index.truncated;
+        blindSpots.parseRecoveries.count > 0 || cache.fresh === false || !!index.truncated ||
+        unsearchablePaths > 0;
     const refactorLevel = indexLevel === 'UNKNOWN' ? 'UNKNOWN'
         : indexLevel === 'UNSUPPORTED' ? 'UNSUPPORTED'
             : hardIndexRisk ? 'LOW'
@@ -1005,6 +1037,9 @@ function doctor(index, options = {}) {
         symbols: totalSymbols,
         languages: langs,
         blindSpots,
+        // What completeness claims range over: text-only, unsearchable and
+        // declared-outside paths, stated here once.
+        scope: require('./source-universe').scopeSummary(index),
         evidenceProfile,
         cache,
         commandTrust: summarizeCommandTrust(),

package/core/request-helpers.js

@@ -43,6 +43,8 @@ const JS_HTTP_PACKAGES = [
 ];
 const JS_HTTP_GLOBAL_CALLS = new Set(['fetch']);
 const JS_HTTP_GLOBAL_CTORS = new Set(['XMLHttpRequest', 'EventSource']);
+const JS_HTTP_CLIENT_TYPES = new Set(['AxiosInstance', 'AxiosStatic', 'KyInstance',
+    'Got', 'GotInstance', 'HttpClient', 'Dispatcher', 'Agent']);
 const PY_HTTP_MODULES = [
     'requests', 'httpx', 'aiohttp', 'urllib3', 'urllib.request', 'http.client', 'niquests', 'treq',
 ];
@@ -173,7 +175,7 @@ class RequestHelperProof {
     }
 
     /** Proof for the function expression being called. */
-    jsCallee(file, fn, depth, factory = false) {
+    jsCallee(file, fn, depth) {
         if (!fn || depth > MAX_RESOLVE) return unknown('depth');
         if (fn.type === 'identifier') {
             const decl = findDecl(this.sess, file, fn.text, fn);
@@ -201,9 +203,7 @@ class RequestHelperProof {
             // the method reads as one (`client.post`, `instance.request`);
             // a server built on an HTTP stack (`app.route(...)`) is not a
             // client.
-            // A factory call on a client value (`axios.create()`) yields a
-            // client value too.
-            if (!factory && (!prop || !CLIENT_METHOD.test(prop))) return unknown('method-not-request');
+            if (!prop || !CLIENT_METHOD.test(prop)) return unknown('method-not-request');
             return this.jsValue(file, obj, depth + 1);
         }
         if (fn.type === 'call_expression') return this.jsValue(file, fn, depth + 1);
@@ -296,7 +296,7 @@ class RequestHelperProof {
                     const spec = this.requireSpecOf({ valueNode: e });
                     return isHttpPackage(spec, JS_HTTP_PACKAGES) ? PROVEN : unknown('receiver-unproven');
                 }
-                return this.jsCallee(file, callee, depth + 1, true);
+                return this.jsFactory(file, callee, depth + 1);
             }
             default:
                 return unknown('receiver-unproven');
@@ -314,6 +314,59 @@ class RequestHelperProof {
         return last;
     }
 
+    /** A request result or server is not a client just because the function
+     *  that produced it uses an HTTP package. Follow factory return values. */
+    jsFactory(file, callee, depth) {
+        if (!callee || depth > MAX_RESOLVE) return unknown('receiver-unproven');
+        if (callee.type === 'member_expression') {
+            const name = field(callee, 'property')?.text;
+            if (name === 'create' || name === 'extend') {
+                return this.jsValue(file, unwrap(field(callee, 'object')), depth + 1);
+            }
+            return unknown('factory-unproven');
+        }
+        if (callee.type !== 'identifier') return unknown('factory-unproven');
+        const decl = findDecl(this.sess, file, callee.text, callee);
+        let target = decl ? { file, value: decl.count === 1 ? decl.valueNode : null } : null;
+        if (!decl) {
+            const binding = this.jsBinding(file, callee.text);
+            if (binding) {
+                if (binding.binding.module?.startsWith('@hey-api/client-') ||
+                    binding.binding.module === 'openapi-fetch') return PROVEN;
+                const rec = this.jsImportRecord(file, binding.binding, binding.original);
+                if (rec) target = { file: rec.file, value: rec.decl?.valueNode || rec.valueNode };
+            }
+        }
+        const fn = unwrap(target?.value);
+        if (!fn || !JS_FUNCTION_TYPES.has(fn.type)) return unknown('factory-unproven');
+        const body = field(fn, 'body');
+        if (!body) return unknown('factory-unproven');
+        const values = [];
+        const visit = node => {
+            if (node.type === 'return_statement') {
+                values.push(node.namedChild(0));
+                return;
+            }
+            for (const child of namedChildren(node)) {
+                if (!JS_FUNCTION_TYPES.has(child.type) && child.type !== 'class_declaration') visit(child);
+            }
+        };
+        if (body.type === 'statement_block') visit(body);
+        else values.push(body);
+        if (!values.length) return unknown('factory-unproven');
+        const proven = values.every(value => {
+            const v = unwrap(value);
+            if (v?.type !== 'object') return this.jsValue(target.file, v, depth + 1).proven;
+            const methods = namedChildren(v).filter(member =>
+                CLIENT_METHOD.test((field(member, 'key') || field(member, 'name'))?.text || ''));
+            return methods.length > 0 && methods.every(member => {
+                const impl = member.type === 'method_definition' ? member : unwrap(field(member, 'value'));
+                return impl && JS_FUNCTION_TYPES.has(impl.type) && this.jsFunction(target.file, impl, 0).proven;
+            });
+        });
+        return proven ? PROVEN : unknown('factory-unproven');
+    }
+
     /** A parameter proves a client only through its declared type or default. */
     jsParam(file, decl, depth) {
         const holder = decl.nameNode.parent;
@@ -321,7 +374,8 @@ class RequestHelperProof {
         if (typeNode) {
             const m = typeNode.text.replace(/^:\s*/, '').match(/^([A-Za-z_$][\w$]*)/);
             const b = m ? this.jsBinding(file, m[1]) : null;
-            if (b && isHttpPackage(b.binding.module, JS_HTTP_PACKAGES)) return PROVEN;
+            if (b && isHttpPackage(b.binding.module, JS_HTTP_PACKAGES) &&
+                JS_HTTP_CLIENT_TYPES.has(b.original)) return PROVEN;
         }
         const dflt = holder ? (field(holder, 'value') || field(holder, 'right')) : null;
         if (dflt) return this.jsValue(file, dflt, depth + 1);
@@ -374,6 +428,8 @@ class RequestHelperProof {
             return !!b && isHttpPackage(b.binding.module, JS_HTTP_PACKAGES);
         }
         if (callee.type === 'member_expression') {
+            if (CLIENT_METHOD.test(field(callee, 'property')?.text || '') &&
+                this.jsValue(file, unwrap(field(callee, 'object')), 0).proven) return true;
             let root = unwrap(field(callee, 'object'));
             while (root && root.type === 'member_expression') root = unwrap(field(root, 'object'));
             if (!root || root.type !== 'identifier') return false;
@@ -558,4 +614,4 @@ function locateCall(root, start, end, type) {
     return node;
 }
 
-module.exports = { RequestHelperProof, JS_HTTP_PACKAGES, PY_HTTP_MODULES };
+module.exports = { RequestHelperProof, JS_HTTP_PACKAGES, PY_HTTP_MODULES, pyImportedFunction };

package/core/review-items.js

@@ -0,0 +1,75 @@
+/**
+ * core/review-items.js - How much is there to look at?
+ *
+ * Unverified caller candidates are not all the same amount of work. A site
+ * the engine could not bind is one thing to check. A dispatch family (an
+ * interface call that may reach any implementation, a template the
+ * compiler instantiates) is ONE boundary with a known reason, however many
+ * sites it lists. A reader reviews the boundary, not each site.
+ *
+ * One counter, shared by the pre-commit check, the caller formatter and
+ * the oracle evaluation, so the three cannot disagree about the burden.
+ */
+
+'use strict';
+
+/**
+ * Why a caller candidate is unverified, as one of three classes:
+ * 'runtime-dispatch' (the implementation is selected at runtime through a
+ * named boundary), 'compile-time-dispatch' (the compiler selects it),
+ * 'actionable-ambiguity' (something a reader can settle). The class the
+ * engine recorded wins; a possible-dispatch site with a named boundary is
+ * runtime dispatch.
+ */
+function uncertaintyClassOf(site) {
+    return site.uncertaintyClass ||
+        (site.reason === 'possible-dispatch' && site.dispatchVia ? 'runtime-dispatch' : 'actionable-ambiguity');
+}
+
+/** Unverified entries by class: { actionable, runtime, compileTime }. */
+function partitionUnverified(unverified) {
+    const parts = { actionable: [], runtime: [], compileTime: [] };
+    for (const site of Array.isArray(unverified) ? unverified : []) {
+        const kind = uncertaintyClassOf(site);
+        if (kind === 'runtime-dispatch') parts.runtime.push(site);
+        else if (kind === 'compile-time-dispatch') parts.compileTime.push(site);
+        else parts.actionable.push(site);
+    }
+    return parts;
+}
+
+/** Key of the family a runtime-dispatch site belongs to. */
+function runtimeFamilyKey(site) {
+    return `${site.dispatchVia || ''}\0${site.dispatchCandidates ?? ''}\0` +
+        `${site.externalContract ? 'external' : 'project'}`;
+}
+
+/** Key of the family a compile-time-dispatch site belongs to. */
+function compileTimeFamilyKey(site) {
+    return `${site.dispatchFamily || ''}\0${site.dispatchCandidates ?? ''}`;
+}
+
+/**
+ * @param {object[]} unverified - unverified caller entries
+ * @returns {{actionable, runtimeDispatchSites, runtimeDispatchGroups,
+ *   compileTimeDispatchSites, compileTimeDispatchGroups, items}}
+ *   `items` = actionable sites + one per dispatch family.
+ */
+function reviewItemsOf(unverified) {
+    const { actionable: open, runtime, compileTime } = partitionUnverified(unverified);
+    const actionable = open.length;
+    const runtimeDispatchGroups = new Set(runtime.map(runtimeFamilyKey)).size;
+    const compileTimeDispatchGroups = new Set(compileTime.map(compileTimeFamilyKey)).size;
+    return {
+        actionable,
+        runtimeDispatchSites: runtime.length,
+        runtimeDispatchGroups,
+        compileTimeDispatchSites: compileTime.length,
+        compileTimeDispatchGroups,
+        items: actionable + runtimeDispatchGroups + compileTimeDispatchGroups,
+    };
+}
+
+module.exports = {
+    reviewItemsOf, uncertaintyClassOf, partitionUnverified, runtimeFamilyKey, compileTimeFamilyKey,
+};