ucn 5.2.1 → 5.3.0

package/cli/index.js

@@ -30,6 +30,16 @@ const { buildPublicParams, isPublicCommand } = require('../core/public-command')
 const { execute } = require('../core/execute');
 const { applyOutputBudget, MAX_OUTPUT_CHARS } = require('../core/output-budget');
 const { clearAllCaches } = require('../core/cache');
+const { commentLines } = require('../core/output/lines');
+
+// A downstream consumer such as head may finish before our write drains.
+// Handle the pipe closure without an unhandled Node error or bypassing cache
+// cleanup. Other write failures are real command errors.
+process.stdout.on('error', error => {
+    if (error.code === 'EPIPE') return;
+    process.stderr.write(`Error writing stdout: ${error.message}\n`);
+    process.exitCode = 2;
+});
 
 let activeCanonicalCommand = null;
 
@@ -315,6 +325,8 @@ function parseFlags(tokens) {
         functions: tokens.includes('--functions') || undefined,
         hot: tokens.includes('--hot') || undefined,
         diverse: tokens.includes('--diverse') || undefined,
+        raw: tokens.includes('--raw') || undefined,
+        lines: tokens.includes('--lines') || undefined,
         git: tokens.includes('--git') || undefined,
         className: getValueFlag('--class-name'),
         // Explicit line pin (fix #249: our own disambiguation notes advertise
@@ -396,7 +408,7 @@ if (unknownFlags.length > 0) {
     emitCliError(
         `Unknown flag(s): ${unknownFlags.join(', ')}. Use --help to see available flags.`,
     );
-    process.exit(1);
+    process.exit(flags.lines || flags.raw ? 2 : 1);
 }
 
 // Validate numeric flag values up front so bad input fails before we build
@@ -407,7 +419,7 @@ try {
 } catch (e) {
     if (e instanceof FlagValidationError) {
         emitCliError(e.message);
-        process.exit(1);
+        process.exit(flags.lines || flags.raw ? 2 : 1);
     }
     throw e;
 }
@@ -456,6 +468,16 @@ function formatCliText(command, result, params, execution, displayFlags) {
         ...execution,
         surface: 'cli',
     });
+    // --lines / --raw are pipe surfaces: records are compact, a truncated
+    // function body is worse than a long one, and an empty answer must stay
+    // empty (grep prints nothing and exits 1). An explicit --max-chars acts
+    // as a fail-before-output guard, never a lossy source/record truncation.
+    if (params?.lines || params?.raw) {
+        if (displayFlags?.maxChars && text.length > displayFlags.maxChars) {
+            fail(`Output exceeds --max-chars=${displayFlags.maxChars}; shell output cannot be truncated. Narrow the query, use --limit/--max-lines, or omit --max-chars.`);
+        }
+        return text;
+    }
     return applyOutputBudget(text, {
         command,
         maxChars: displayFlags?.maxChars,
@@ -465,6 +487,36 @@ function formatCliText(command, result, params, execution, displayFlags) {
     }).text;
 }
 
+/**
+ * Print a formatted answer the way the mode asks for it (fix #341).
+ * --lines: `path:line:text` records on stdout, `# ` comment lines (ACCOUNT,
+ * notes) on stderr, exit 1 when nothing matched — grep's own contract.
+ * --raw: the text verbatim with exactly one trailing newline.
+ */
+function emitCliText(text, params, json, note) {
+    if (!json && params?.lines) {
+        const records = [];
+        const comments = [];
+        for (const line of String(text).split('\n')) {
+            if (line === '') continue;
+            (line.startsWith('# ') ? comments : records).push(line);
+        }
+        if (records.length > 0) process.stdout.write(records.join('\n') + '\n');
+        if (comments.length > 0) process.stderr.write(comments.join('\n') + '\n');
+        if (records.length === 0) process.exitCode = Math.max(process.exitCode || 0, 1);
+        return;
+    }
+    if (!json && params?.raw) {
+        const body = String(text);
+        process.stdout.write(body.endsWith('\n') ? body : body + '\n');
+        // Code lines are never reinterpreted (a Python comment starts with
+        // "# " too), so the note travels on its own channel.
+        if (note) process.stderr.write(commentLines(formatSurfaceMessage(note, 'cli')).join('\n') + '\n');
+        return;
+    }
+    console.log(text);
+}
+
 // ============================================================================
 // MAIN
 // ============================================================================
@@ -555,7 +607,7 @@ function main() {
         if (!(e instanceof CommandError)) {
             emitCliError(`Error: ${e.message}`);
         }
-        process.exitCode = 1;
+        process.exitCode = flags.lines || flags.raw ? 2 : 1;
     }
 }
 
@@ -665,11 +717,11 @@ function runFileCommand(filePath, command, arg) {
     const execution = execute(index, canonical, params);
     const { ok, result, error } = execution;
     if (!ok) fail(formatSurfaceMessage(error, 'cli'));
-    console.log(flags.json
+    emitCliText(flags.json
         ? output.formatPublicJson(canonical, result, params, {
             ...execution, surface: 'cli',
         })
-        : formatCliText(canonical, result, params, execution, scopedFlags));
+        : formatCliText(canonical, result, params, execution, scopedFlags), params, flags.json, execution.note);
 }
 
 // ============================================================================
@@ -737,11 +789,12 @@ function runProjectCommand(rootDir, command, arg) {
     if (!publicExecution.ok) {
         fail(formatSurfaceMessage(publicExecution.error, 'cli'));
     }
-    console.log(flags.json
+    emitCliText(flags.json
         ? output.formatPublicJson(canonical, publicExecution.result, publicParams, {
             ...publicExecution, surface: 'cli',
         })
-        : formatCliText(canonical, publicExecution.result, publicParams, publicExecution, flags));
+        : formatCliText(canonical, publicExecution.result, publicParams, publicExecution, flags),
+    publicParams, flags.json, publicExecution.note);
     // A gate that could not run (check outside git / bad base ref) must not
     // exit 0 — CI gating on the exit code would read "could not run" as "passed".
     process.exitCode = Math.max(process.exitCode || 0,
@@ -750,7 +803,7 @@ function runProjectCommand(rootDir, command, arg) {
         if (!(e instanceof CommandError)) {
             emitCliError(`Error: ${e.message}`);
         }
-        process.exitCode = 1;
+        process.exitCode = flags.lines || flags.raw ? 2 : 1;
     } finally {
         // Save cache after command execution so callsCache populated
         // by findCallers/findCallees gets persisted to disk.
@@ -760,6 +813,9 @@ function runProjectCommand(rootDir, command, arg) {
         if (flags.cache && (needsCacheSave || index.callsCacheDirty || index.reachabilityDirty || index.computedDispatchDirty)) {
             try { index.saveCache(); } catch (e) { /* best-effort */ }
         }
+        if (flags.cache && index.usageCacheDirty) {
+            try { index.saveUsageCache(); } catch (e) { /* best-effort */ }
+        }
     }
 }
 
@@ -793,11 +849,12 @@ function runGlobCommand(pattern, command, arg) {
     if (!publicExecution.ok) {
         fail(formatSurfaceMessage(publicExecution.error, 'cli'));
     }
-    console.log(flags.json
+    emitCliText(flags.json
         ? output.formatPublicJson(canonical, publicExecution.result, publicParams, {
             ...publicExecution, surface: 'cli',
         })
-        : formatCliText(canonical, publicExecution.result, publicParams, publicExecution, flags));
+        : formatCliText(canonical, publicExecution.result, publicParams, publicExecution, flags),
+    publicParams, flags.json, publicExecution.note);
     process.exitCode = Math.max(process.exitCode || 0,
         resultExitCode(canonical, publicExecution.result));
 }
@@ -857,6 +914,13 @@ Common flags:
   --range=N-M (source with --file=PATH)
   --base=REF --staged --no-cache --clear-cache [--all] --max-files=N --workers=N
   --max-chars=N (text output; default 10K targeted / 3K broad, ceiling 100K)
+  --lines  find/usages/search/show/impact: grep -n shape, one path:line:text
+           record per line (tags after a tab: # unverified: <reason>, # import,
+           # callee); accounting and notes go to stderr as "# " lines; exit 1
+           when nothing matched; exit 2 on errors. No default result cap.
+           show defaults to callers; --sections=callers,callees selects bands.
+  --raw    source: full code, no header or gutter (including large classes).
+           Shell modes fail before output if an explicit --max-chars is exceeded.
   Cache: per-user by default; set UCN_CACHE_DIR to override the cache root.
 
 Accepted flags by command:

package/core/account.js

@@ -36,14 +36,16 @@
  * Ground-set semantics are grep `-n -w`: unit is the (file, line) pair, each
  * line with >= 1 word-boundary match counts once, case-sensitive.
  *
- * Performance: the ground scan is one `includes()`-gated read per project file
- * per caller-command — the same I/O profile as the existing `search`/`usages`
- * commands. Deriving counts from callsCache (zero reads) was rejected because
+ * Performance: the first ground scan is one `includes()`-gated read per project
+ * file — the same I/O profile as the existing `search`/`usages` commands.
+ * Exact results are retained in a small LRU for the lifetime of one built index
+ * so context/about/impact projections of the same symbol do not rescan the
+ * project. Deriving counts from callsCache (zero reads) was rejected because
  * comments/strings/references are not in the calls cache and the contract's
  * ground set is text-defined. AST parsing (the expensive part) is restricted
- * to files containing UNCLAIMED ground lines, via the op-cached
- * `index._getCachedUsages`. No file is read twice in one command because
- * context/about/impact run inside `index._beginOp()`.
+ * to files containing UNCLAIMED ground lines, via the content-hash-keyed
+ * `index._getCachedUsages`. Build invalidation and cache bounds preserve the
+ * same answer without retaining an unbounded repository mirror.
  */
 
 'use strict';
@@ -75,6 +77,12 @@ const UNSUPPORTED_SITE_TEXT_MAX = 160;
  * }}
  */
 function computeGroundSet(index, name) {
+    if (index._groundSetCache?.has(name)) {
+        const cached = index._groundSetCache.get(name);
+        index._groundSetCache.delete(name);
+        index._groundSetCache.set(name, cached);
+        return cached.result;
+    }
     const wordRe = new RegExp('\\b' + escapeRegExp(name) + '\\b');
     const perFile = new Map();
     let total = 0;
@@ -117,7 +125,7 @@ function computeGroundSet(index, name) {
         ? index.discoveryIssues.map(issue => ({ ...issue })) : [];
     unreadableFiles.sort();
 
-    return {
+    const result = {
         total: total + unparsed.lines + unsupported.lines,
         fileCount: fileCount + unparsed.fileCount + unsupported.fileCount,
         perFile,
@@ -126,6 +134,21 @@ function computeGroundSet(index, name) {
         unreadableFiles,
         skippedSources,
     };
+    if (index._groundSetCache) {
+        const weight = result.total + result.fileCount;
+        index._groundSetCache.set(name, { result, weight });
+        index._groundSetCacheLines = (index._groundSetCacheLines || 0) + weight;
+        const maxNames = 64;
+        const maxLines = 100000;
+        while (index._groundSetCache.size > maxNames ||
+            index._groundSetCacheLines > maxLines) {
+            const oldest = index._groundSetCache.entries().next().value;
+            if (!oldest) break;
+            index._groundSetCache.delete(oldest[0]);
+            index._groundSetCacheLines -= oldest[1].weight;
+        }
+    }
+    return result;
 }
 
 /** Scan only files the parser/index could not ingest. */
@@ -300,7 +323,12 @@ function classifyGroundLines(index, name, groundSet, claimedKeys) {
 
         // Remainder: AST usage scan distinguishes import/definition/reference
         // from comment/string/skipped-token lines.
-        const usages = index._getCachedUsages(filePath, name);
+        // Call lines were classified from the complete calls cache above.
+        // Language adapters may skip usage-only call recovery here; notably,
+        // C/C++ avoids reparsing every matching macro replacement list.
+        const usages = index._getCachedUsages(filePath, name, {
+            skipCallRecovery: true,
+        });
         const byLine = new Map();
         if (Array.isArray(usages)) {
             for (const u of usages) {

package/core/cache.js

@@ -26,8 +26,9 @@ const CACHE_PRUNE_INTERVAL_MS = 60 * 60 * 1000;
 const CACHE_MAX_PROJECTS = 128;
 const CACHE_MAX_BYTES = 1024 * 1024 * 1024;
 
-function discoveryRulesHash(root) {
-    return crypto.createHash('md5').update(parseGitignore(root).join('\0')).digest('hex');
+function discoveryRulesHash(root, patterns = null) {
+    const rules = patterns || parseGitignore(root);
+    return crypto.createHash('md5').update(rules.join('\0')).digest('hex');
 }
 
 /**
@@ -648,7 +649,134 @@ function clearAllCaches() {
 // reporting can classify import-time vs lazy edges from fresh and cached
 // indexes; C# properties retain property identity instead of masquerading as
 // ordinary fields for accessor impact/refactoring.
-const CACHE_FORMAT_VERSION = 189;
+// v190: members of Rust generic impl owners retain ownerGenerics so blanket
+// impl parameters (`impl<I> Trait for I`) cannot be mistaken for concrete
+// receiver types after a cache round-trip (fix #302).
+// v191: Python call records retain untyped loop-element provenance so a
+// same-spelled project method cannot gain confirmed identity after reload.
+// v192: JS/TS callback records carry moduleLocalBinding so dynamically
+// produced module values cannot borrow target identity from file imports.
+// v193: overload-heavy JS/TS class-member aliases materialize when every
+// declared return has one concrete runtime head, changing indexed symbols.
+// v194: JS/TS static fields preserve direct same-file callable forwarding for
+// immutable module/export aliases (fix #313).
+// v195: JS/TS call records retain safe ordered namespace-spread receiver
+// compositions for exact module export ownership (fix #314).
+// v196: TS parameter type references no longer mark namespace receiver roots
+// as locally shadowed, changing receiverLocalBinding evidence (fix #315).
+// v197 (fix #317): JS/TS expression-bodied arrow symbols persist the exact
+// returned call span so query-time flow can resolve compiler-inferred factory
+// results without treating block bodies or arbitrary expressions as returns.
+// v198 (fix #318): JS/TS one-return methods persist exact this-field paths for
+// generic declared-field result flow.
+// v199 (fix #321): Python call records persist receiver types derived from
+// exact, scope-local class-value aliases such as `_Segment = Segment`.
+// v200 (fix #323): Python dotted module calls persist their exact import
+// specifier (`import rich.repr` → `rich.repr.auto()`).
+// v201 (fix #324): Python method-call records persist simple subscript roots
+// and their local type provenance for `items[key].method()` dispatch; local
+// callable aliases/non-callable bindings no longer leak across functions.
+// v202 (fix #325): Python call records preserve exact typed subscript sources
+// across one local assignment (`item = items[key]; item.method()`).
+// v203 (fix #326): Python call records type stable, direct module constructor
+// globals even when their functions are declared before the assignment.
+// v204 (fix #328): Python union-alias symbols retain their concrete members
+// so imported annotations can participate in exact conditional narrowing.
+// v205 (fix #332): Go method-call records retain capture-aware lexical scope
+// chains so return-type flow reaches nested closures without crossing shadows.
+// v206 (fix #333): Go calls assigned in `var` declarations retain their
+// declaration targets for compiler-return-type receiver flow.
+// v207 (fix #334): Go method-call records retain compiler-exact receiver
+// types for values bound from declared map, slice, and array indexes.
+// v208 (fix #335): Go indexed-value calls preserve receiver-root/field
+// provenance so sibling-file container declarations resolve query-time.
+// v210 (fixes #337-#339): importDetails persisted for every language (was Python-only),
+// require(path.join(__dirname, ...)) composes to a static relative specifier,
+// and import records carry deferredReason (function-local / type-checking / type-only).
+// v211: path-utility composition requires unshadowed binding evidence; mixed
+// default/type imports and inline type re-exports preserve execution timing.
+// Python TYPE_CHECKING guards require typing ownership and no rebinding.
+// v212 (fix #342): extendsGraph/extendedByGraph no longer persisted (rebuilt on load).
+const CACHE_FORMAT_VERSION = 212;
+const USAGE_CACHE_FILE = 'usage-results.json';
+
+/**
+ * Persist the small hot-name usage cache independently of index/call shards.
+ * This keeps one-shot CLI repeats fast without rewriting the whole project
+ * cache after every read-only account query.
+ */
+function saveUsageCache(index, cachePath) {
+    if (!index.usageCacheDirty) return null;
+    const cacheDir = cachePath
+        ? path.dirname(cachePath)
+        : getProjectCacheDir(index.root);
+    fs.mkdirSync(cacheDir, { recursive: true });
+    const entries = [];
+    for (const [key, cached] of index._usageResultCache || []) {
+        const parts = key.split('\0');
+        if (parts.length < 3 || !Array.isArray(cached?.value)) continue;
+        const [filePath, fileHash, name, mode = ''] = parts;
+        const fileEntry = index.files.get(filePath);
+        // Drop removed/changed-file generations rather than carrying dead LRU
+        // weight across incremental builds.
+        if (!fileEntry || fileEntry.hash !== fileHash) continue;
+        entries.push([
+            path.relative(index.root, filePath), fileHash, name, mode,
+            cached.value,
+        ]);
+    }
+    const usageFile = path.join(cacheDir, USAGE_CACHE_FILE);
+    const tmpFile = usageFile + '.tmp';
+    fs.writeFileSync(tmpFile, JSON.stringify({
+        version: CACHE_FORMAT_VERSION,
+        ucnVersion: UCN_VERSION,
+        entries,
+    }));
+    fs.renameSync(tmpFile, usageFile);
+    index.usageCacheDirty = false;
+    return usageFile;
+}
+
+function loadUsageCache(index, cacheFile) {
+    const usageFile = path.join(path.dirname(cacheFile), USAGE_CACHE_FILE);
+    if (!fs.existsSync(usageFile)) return false;
+    try {
+        const payload = JSON.parse(fs.readFileSync(usageFile, 'utf-8'));
+        if (payload.version !== CACHE_FORMAT_VERSION ||
+            payload.ucnVersion !== UCN_VERSION ||
+            !Array.isArray(payload.entries)) return false;
+        const restored = new Map();
+        let weightTotal = 0;
+        for (const entry of payload.entries) {
+            if (!Array.isArray(entry) || entry.length < 5) continue;
+            const [relativePath, fileHash, name, mode, value] = entry;
+            if (typeof relativePath !== 'string' ||
+                typeof fileHash !== 'string' ||
+                typeof name !== 'string' || typeof mode !== 'string' ||
+                !Array.isArray(value)) continue;
+            const filePath = path.resolve(index.root, relativePath);
+            const fileEntry = index.files.get(filePath);
+            if (!fileEntry || fileEntry.hash !== fileHash) continue;
+            const suffix = mode ? `\0${mode}` : '';
+            const key = `${filePath}\0${fileHash}\0${name}${suffix}`;
+            const weight = 64 + value.length * 40;
+            restored.set(key, { value, weight });
+            weightTotal += weight;
+            while (restored.size > 4096 || weightTotal > 16 * 1024 * 1024) {
+                const oldest = restored.entries().next().value;
+                if (!oldest) break;
+                restored.delete(oldest[0]);
+                weightTotal -= oldest[1].weight;
+            }
+        }
+        index._usageResultCache = restored;
+        index._usageResultCacheWeight = weightTotal;
+        index.usageCacheDirty = false;
+        return true;
+    } catch (_) {
+        return false;
+    }
+}
 
 /**
  * Save index to cache file
@@ -780,9 +908,13 @@ function saveCache(index, cachePath) {
         symbols: strippedSymbols,
         importGraph: relGraph(index.importGraph),
         exportGraph: relGraph(index.exportGraph),
-        // extendsGraph/extendedByGraph use class names as keys (not file paths)
-        extendsGraph: Array.from(index.extendsGraph.entries()),
-        extendedByGraph: Array.from(index.extendedByGraph.entries()),
+        // extendsGraph/extendedByGraph are NOT persisted (fix #342): their
+        // entries carry absolute file paths in the build-time spelling, and
+        // the cache key is realpath-normalized — a root reached through a
+        // symlink (/var → /private/var on macOS) loaded a graph whose files
+        // matched nothing, so an overriding Go embed read as a non-overriding
+        // subclass and confirmed two false callers. loadCache rebuilds both
+        // from the rehydrated symbols (8ms on 1037 files).
         failedFiles: index.failedFiles
             ? Array.from(index.failedFiles).map(f => path.relative(root, f))
             : [],
@@ -926,8 +1058,14 @@ function loadCache(index, cachePath) {
             : (relPath) => rootPrefix + relPath.replace(/\//g, path.sep);
 
         // Loading into a previously-used ProjectIndex replaces its indexed
-        // contents, so no parsed tree from the old state may survive.
+        // contents, so no parsed tree or cross-file derived query answer from
+        // the old state may survive. Usage results are content-hash keyed and
+        // remain safe; these caches are graph/text-universe keyed instead.
         index._clearParsedTreeCache?.();
+        index._groundSetCache = new Map();
+        index._groundSetCacheLines = 0;
+        index._nameBindingReachCache = new Map();
+        index._returnTypeFlowCache = new Map();
 
         // Reconstruct files Map: relative key → absolute key, restore path and relativePath
         // Initialize symbols/bindings arrays (will be populated from top-level symbols)
@@ -989,13 +1127,8 @@ function loadCache(index, cachePath) {
         index.buildTime = cacheData.buildTime;
 
         // Restore optional graphs if present
-        // extendsGraph/extendedByGraph use class names as keys (not file paths)
-        if (Array.isArray(cacheData.extendsGraph)) {
-            index.extendsGraph = new Map(cacheData.extendsGraph);
-        }
-        if (Array.isArray(cacheData.extendedByGraph)) {
-            index.extendedByGraph = new Map(cacheData.extendedByGraph);
-        }
+        // extendsGraph/extendedByGraph are derived below from the rehydrated
+        // symbols (fix #342) — never read from the payload.
 
         // Prepare lazy calls cache loading — load manifest but defer shard parsing.
         // Shards are loaded on first getCachedCalls access via ensureCallsCacheLoaded().
@@ -1067,13 +1200,18 @@ function loadCache(index, cachePath) {
             }
         }
 
-        // Only rebuild graphs if config changed (e.g., aliases modified)
+        // Only rebuild the import graph if config changed (e.g., aliases
+        // modified); it is persisted with relative paths. The inheritance
+        // graph is always derived from the rehydrated symbols (fix #342) so
+        // its file paths agree with index.root whatever spelling built it.
         const currentConfigHash = crypto.createHash('md5')
             .update(JSON.stringify(index.config || {})).digest('hex');
         if (currentConfigHash !== cacheData.configHash) {
             index.buildImportGraph();
-            index.buildInheritanceGraph();
         }
+        index.buildInheritanceGraph();
+
+        loadUsageCache(index, cacheFile);
 
         return true;
     } catch (e) {
@@ -1092,9 +1230,17 @@ function isCacheStale(index) {
     if (index._loadedConfigHash && currentConfigHash !== index._loadedConfigHash) {
         return true;
     }
-    if (index._loadedDiscoveryHash &&
-        discoveryRulesHash(index.root) !== index._loadedDiscoveryHash) {
-        return true;
+    // Parse gitignore rules once for both the discovery fingerprint and the
+    // new-file walk below. On a fresh cache these used to run two identical
+    // `git ls-files` subprocesses per one-shot CLI invocation, accounting for
+    // roughly a third of warm-start staleness time on measured repositories.
+    let gitignorePatterns = null;
+    if (index._loadedDiscoveryHash) {
+        gitignorePatterns = parseGitignore(index.root);
+        if (discoveryRulesHash(index.root, gitignorePatterns) !==
+            index._loadedDiscoveryHash) {
+            return true;
+        }
     }
     // Modified/deleted detection (stat sweep) runs UNCONDITIONALLY — agents
     // edit a file and re-query through MCP within seconds, and a stale answer
@@ -1153,7 +1299,7 @@ function isCacheStale(index) {
             });
         },
     };
-    const gitignorePatterns = parseGitignore(index.root);
+    if (!gitignorePatterns) gitignorePatterns = parseGitignore(index.root);
     globOpts.gitignorePatterns = gitignorePatterns;
     globOpts.trackedPaths = gitTrackedPaths(index.root);
     const configExclude = index.config.exclude || [];
@@ -1347,7 +1493,7 @@ function _computeReachabilityFingerprint(index) {
 }
 
 module.exports = {
-    saveCache, loadCache, loadCallsCache, isCacheStale, ensureCallsCacheLoaded,
+    saveCache, saveUsageCache, loadCache, loadCallsCache, isCacheStale, ensureCallsCacheLoaded,
     getUserCacheRoot, getProjectCacheDir, getProjectCachePath,
     getLegacyProjectCacheDir, migrateLegacyProjectCache, clearProjectCache,
     clearAllCaches, pruneUserCache,