ucn 3.8.22 → 3.8.25

package/core/cache.js

@@ -68,34 +68,53 @@ function saveCache(index, cachePath) {
         strippedFiles.push([rp, rest]);
     }
 
-    // Convert graph paths from absolute to relative
+    // Convert graph paths from absolute to relative (Sets serialized as arrays)
     const relGraph = (graph) => {
         const result = [];
         for (const [absKey, absValues] of graph) {
             const relKey = path.relative(root, absKey);
-            const relValues = absValues.map(v => path.relative(root, v));
+            const relValues = [...absValues].map(v => path.relative(root, v));
             result.push([relKey, relValues]);
         }
         return result;
     };
 
-    // Persist calleeIndex if built (paths must be absolute from this.files keys)
-    let calleeIndexData;
-    if (index.calleeIndex && index.calleeIndex.size > 0) {
-        calleeIndexData = [];
-        for (const [name, files] of index.calleeIndex) {
-            const relFiles = [...files].map(f =>
-                path.isAbsolute(f) ? path.relative(root, f) : f
-            );
-            calleeIndexData.push([name, relFiles]);
+    // calleeIndex is NOT persisted in index.json — it's rebuilt lazily from callsCache
+    // on first findCallers/buildCalleeIndex call. Removing it saves ~22MB (14%) on large projects.
+
+    // PERF-1: persist _reachableSymbols if computed. Set keys are
+    // "absolutePath:line"; we strip the root prefix on save and re-attach on
+    // load so paths stay portable. Sorted for stable output ordering.
+    //
+    // Also save a fingerprint so we can detect index drift on load: if the
+    // saved fingerprint matches the loaded index state, the cached set is
+    // still valid. If the index was rebuilt after load (stale cache → build),
+    // the fingerprint won't match and computeReachability will recompute.
+    let reachableSymbolsRel = undefined;
+    let reachableFingerprint = undefined;
+    if (index._reachableSymbols && index._reachableSymbols.size > 0) {
+        const rels = [];
+        for (const k of index._reachableSymbols) {
+            const colon = k.lastIndexOf(':');
+            if (colon < 0) continue;
+            const absFile = k.slice(0, colon);
+            const lineStr = k.slice(colon + 1);
+            const relFile = path.relative(root, absFile);
+            rels.push(`${relFile}:${lineStr}`);
         }
+        rels.sort();  // stable ordering — output contract
+        reachableSymbolsRel = rels;
+        reachableFingerprint = _computeReachabilityFingerprint(index);
     }
 
     const cacheData = {
-        version: 7,  // v7: strip symbols/bindings from file entries (dedup ~45% cache reduction)
+        // v10: persist _reachableSymbols set (computed by entrypoints.computeReachability)
+        version: 10,
         ucnVersion: UCN_VERSION,  // Invalidate cache when UCN is updated
         configHash,
         root,
+        // PERF-2: refresh buildTime on each save so partial rebuilds report
+        // accurate stats. Falls back to original on first save.
         buildTime: index.buildTime,
         timestamp: Date.now(),
         files: strippedFiles,
@@ -108,23 +127,37 @@ function saveCache(index, cachePath) {
         failedFiles: index.failedFiles
             ? Array.from(index.failedFiles).map(f => path.relative(root, f))
             : [],
-        ...(calleeIndexData && { calleeIndex: calleeIndexData })
+        ...(reachableSymbolsRel !== undefined && {
+            reachableSymbols: reachableSymbolsRel,
+            reachableFingerprint,
+        }),
     };
 
-    fs.writeFileSync(cacheFile, JSON.stringify(cacheData));
+    // PERF-3: atomic write — tmp file + rename so concurrent readers/writers
+    // never see a torn JSON. The calls/ shard write below already does this.
+    const tmpFile = cacheFile + '.tmp';
+    fs.writeFileSync(tmpFile, JSON.stringify(cacheData));
+    fs.renameSync(tmpFile, cacheFile);
+
+    // MED-1 (Round 5): clear the reachabilityDirty flag now that the set is
+    // safely persisted. The cli/index.js cache-save guard checks this flag
+    // along with needsCacheSave/callsCacheDirty.
+    if (index.reachabilityDirty) {
+        index.reachabilityDirty = false;
+    }
 
-    // Save callsCache sharded by directory for lazy loading
+    // Save callsCache sharded by directory for lazy loading.
+    // Write to a temp directory first, then atomic swap to avoid data loss on crash.
     if (callsCacheData.length > 0) {
-        const callsDir = path.join(path.dirname(cacheFile), 'calls');
-        // Clean up old shards and legacy monolithic file
-        if (fs.existsSync(callsDir)) {
-            fs.rmSync(callsDir, { recursive: true, force: true });
-        }
-        const legacyFile = path.join(path.dirname(cacheFile), 'calls-cache.json');
-        if (fs.existsSync(legacyFile)) {
-            fs.rmSync(legacyFile, { force: true });
+        const cacheDir = path.dirname(cacheFile);
+        const callsDir = path.join(cacheDir, 'calls');
+        const callsTmpDir = path.join(cacheDir, 'calls.tmp');
+
+        // Clean up any leftover temp dir from a previous crashed save
+        if (fs.existsSync(callsTmpDir)) {
+            fs.rmSync(callsTmpDir, { recursive: true, force: true });
         }
-        fs.mkdirSync(callsDir, { recursive: true });
+        fs.mkdirSync(callsTmpDir, { recursive: true });
 
         // Group by directory
         const shards = new Map();
@@ -134,17 +167,29 @@ function saveCache(index, cachePath) {
             shards.get(dir).push([relPath, entry]);
         }
 
-        // Write one shard per directory
+        // Write all shards to temp directory
         const shardManifest = [];
         for (const [dir, entries] of shards) {
             const hash = crypto.createHash('md5').update(dir).digest('hex').slice(0, 10);
-            const shardFile = path.join(callsDir, `${hash}.json`);
+            const shardFile = path.join(callsTmpDir, `${hash}.json`);
             fs.writeFileSync(shardFile, JSON.stringify(entries));
             shardManifest.push([dir, hash, entries.length]);
         }
 
-        // Write manifest for lazy loading
-        fs.writeFileSync(path.join(callsDir, 'manifest.json'), JSON.stringify(shardManifest));
+        // Write manifest to temp directory
+        fs.writeFileSync(path.join(callsTmpDir, 'manifest.json'), JSON.stringify(shardManifest));
+
+        // Atomic swap: remove old, rename temp to final
+        if (fs.existsSync(callsDir)) {
+            fs.rmSync(callsDir, { recursive: true, force: true });
+        }
+        fs.renameSync(callsTmpDir, callsDir);
+
+        // Clean up legacy monolithic file
+        const legacyFile = path.join(cacheDir, 'calls-cache.json');
+        if (fs.existsSync(legacyFile)) {
+            fs.rmSync(legacyFile, { force: true });
+        }
     }
 
     return cacheFile;
@@ -168,7 +213,9 @@ function loadCache(index, cachePath) {
 
         // Check version compatibility
         // v7: symbols/bindings stripped from file entries (dedup)
-        if (cacheData.version !== 7) {
+        // v9: addSymbol propagates isAsync/isGenerator/paramTypes (force rebuild for old)
+        // v10: persists _reachableSymbols set
+        if (cacheData.version !== 10) {
             return false;
         }
 
@@ -186,12 +233,19 @@ function loadCache(index, cachePath) {
         }
 
         const root = cacheData.root || index.root;
+        // Fast path conversion: string concat is ~70x faster than path.join for
+        // cache-stored relative paths (no '..' segments). On Windows, path.relative
+        // produces backslash paths, so rootPrefix uses the native separator.
+        const rootPrefix = root.endsWith(path.sep) ? root : root + path.sep;
+        const toAbs = path.sep === '/'
+            ? (relPath) => rootPrefix + relPath
+            : (relPath) => rootPrefix + relPath.replace(/\//g, path.sep);
 
         // Reconstruct files Map: relative key → absolute key, restore path and relativePath
         // Initialize symbols/bindings arrays (will be populated from top-level symbols)
         index.files = new Map();
         for (const [relPath, entry] of cacheData.files) {
-            const absPath = path.join(root, relPath);
+            const absPath = toAbs(relPath);
             entry.path = absPath;
             entry.relativePath = relPath;
             if (!entry.symbols) entry.symbols = [];
@@ -204,7 +258,7 @@ function loadCache(index, cachePath) {
         index.symbols = new Map(cacheData.symbols);
         for (const [, defs] of index.symbols) {
             for (const s of defs) {
-                if (!s.file && s.relativePath) s.file = path.join(root, s.relativePath);
+                if (!s.file && s.relativePath) s.file = toAbs(s.relativePath);
                 if (!s.bindingId && s.relativePath && s.type && s.startLine) {
                     s.bindingId = `${s.relativePath}:${s.type}:${s.startLine}`;
                 }
@@ -222,11 +276,14 @@ function loadCache(index, cachePath) {
             }
         }
 
-        // Reconstruct graphs: relative paths → absolute paths
+        // Reconstruct graphs: relative paths → absolute paths (as Sets)
+        // Uses string concat (toAbs) instead of path.join — 70x faster on 464K edges
         const absGraph = (data) => {
             const m = new Map();
             for (const [relKey, relValues] of data) {
-                m.set(path.join(root, relKey), relValues.map(v => path.join(root, v)));
+                const absValues = new Set();
+                for (const v of relValues) absValues.add(toAbs(v));
+                m.set(toAbs(relKey), absValues);
             }
             return m;
         };
@@ -243,10 +300,10 @@ function loadCache(index, cachePath) {
             index.extendedByGraph = new Map(cacheData.extendedByGraph);
         }
 
-        // Eagerly load callsCache from separate file.
-        // Prevents 10K cold tree-sitter re-parses (2GB+ peak) when findCallers runs.
+        // Prepare lazy calls cache loading — load manifest but defer shard parsing.
+        // Shards are loaded on first getCachedCalls access via ensureCallsCacheLoaded().
         if (index.callsCache.size === 0) {
-            loadCallsCache(index);
+            _prepareCallsCache(index);
         }
 
         // Build directory→files index from loaded data
@@ -257,21 +314,44 @@ function loadCache(index, cachePath) {
         // Restore failedFiles if present (convert relative paths back to absolute)
         if (Array.isArray(cacheData.failedFiles)) {
             index.failedFiles = new Set(
-                cacheData.failedFiles.map(f => path.isAbsolute(f) ? f : path.join(root, f))
+                cacheData.failedFiles.map(f => path.isAbsolute(f) ? f : toAbs(f))
             );
         }
 
-        // Restore calleeIndex if persisted
+        // Restore calleeIndex if persisted (v7 caches only; v8+ rebuilds lazily)
         if (Array.isArray(cacheData.calleeIndex)) {
             index.calleeIndex = new Map();
             for (const [name, files] of cacheData.calleeIndex) {
                 if (!Array.isArray(files)) continue;
                 index.calleeIndex.set(name, new Set(
-                    files.map(f => path.isAbsolute(f) ? f : path.join(root, f))
+                    files.map(f => path.isAbsolute(f) ? f : toAbs(f))
                 ));
             }
         }
 
+        // PERF-1: restore _reachableSymbols if persisted (v10+).
+        // Saved as relative-path keys; rehydrate to absolute keys here so the
+        // in-memory set matches what computeReachability would produce fresh.
+        // The fingerprint is checked by computeReachability before reuse — if
+        // the index drifts (e.g. a rebuild after stale cache), the cached set
+        // is dropped and recomputed.
+        if (Array.isArray(cacheData.reachableSymbols)) {
+            const reachable = new Set();
+            for (const k of cacheData.reachableSymbols) {
+                if (typeof k !== 'string') continue;
+                const colon = k.lastIndexOf(':');
+                if (colon < 0) continue;
+                const relFile = k.slice(0, colon);
+                const lineStr = k.slice(colon + 1);
+                const absFile = path.isAbsolute(relFile) ? relFile : toAbs(relFile);
+                reachable.add(`${absFile}:${lineStr}`);
+            }
+            index._reachableSymbols = reachable;
+            if (cacheData.reachableFingerprint) {
+                index._reachableFingerprint = cacheData.reachableFingerprint;
+            }
+        }
+
         // Only rebuild graphs if config changed (e.g., aliases modified)
         const currentConfigHash = crypto.createHash('md5')
             .update(JSON.stringify(index.config || {})).digest('hex');
@@ -292,6 +372,12 @@ function loadCache(index, cachePath) {
  * @returns {boolean} - True if cache needs rebuilding
  */
 function isCacheStale(index) {
+    // Ultra-fast path: skip full check if last confirmed-fresh < 2s ago (covers MCP burst calls).
+    // Only uses _lastFreshAt (set at the end of a successful full check), not cache save timestamp.
+    if (index._lastFreshAt && Date.now() - index._lastFreshAt < 2000) {
+        return false;
+    }
+
     // Fast path: check cached files for modifications/deletions first (stat-only).
     // This returns early without the expensive directory walk when any file changed.
     for (const [filePath, fileEntry] of index.files) {
@@ -337,44 +423,64 @@ function isCacheStale(index) {
         }
     }
 
+    // Record when we last confirmed the cache is fresh (enables 2s skip on burst calls)
+    index._lastFreshAt = Date.now();
     return false;
 }
 
 /**
- * Load callsCache from separate file on demand.
- * Only loads if callsCache is empty (not already populated from inline or prior load).
+ * Prepare calls cache for lazy loading — reads manifest but defers shard parsing.
+ * Called during loadCache() to set up the manifest without the ~1s shard parse cost.
+ * Actual shards are loaded on first ensureCallsCacheLoaded() call.
  * @param {object} index - ProjectIndex instance
- * @returns {boolean} - True if loaded successfully
  */
-function loadCallsCache(index) {
-    if (index.callsCache.size > 0) return true; // Already populated
-    if (index._callsCacheLoaded) return false;   // Already attempted, file didn't exist
-    index._callsCacheLoaded = true;
-
+function _prepareCallsCache(index) {
+    if (index._callsCacheLoaded) return;
     const cacheDir = path.join(index.root, '.ucn-cache');
-
-    // Try sharded format first (calls/manifest.json)
     const manifestFile = path.join(cacheDir, 'calls', 'manifest.json');
     if (fs.existsSync(manifestFile)) {
         try {
             const manifest = JSON.parse(fs.readFileSync(manifestFile, 'utf-8'));
-            // Store manifest for lazy loading
             index._callsManifest = new Map();
             for (const [dir, hash, count] of manifest) {
                 index._callsManifest.set(dir, { hash, count, loaded: false });
             }
-            // Eagerly load all shards (matches previous behavior)
-            for (const [, { hash }] of index._callsManifest) {
-                _loadCallsShard(index, hash);
-            }
-            return true;
+            index._callsCachePrepared = true;
+            return;
         } catch (e) {
-            // Corrupted manifest — fall through to legacy
+            // Corrupted manifest — fall through
+        }
+    }
+    // Check legacy format
+    const legacyFile = path.join(cacheDir, 'calls-cache.json');
+    if (fs.existsSync(legacyFile)) {
+        index._callsCacheLegacyFile = legacyFile;
+        index._callsCachePrepared = true;
+    }
+}
+
+/**
+ * Load callsCache from separate file on demand.
+ * Only loads if callsCache is empty (not already populated from inline or prior load).
+ * @param {object} index - ProjectIndex instance
+ * @returns {boolean} - True if loaded successfully
+ */
+function loadCallsCache(index) {
+    if (index.callsCache.size > 0) return true; // Already populated
+    if (index._callsCacheLoaded) return false;   // Already attempted, file didn't exist
+    index._callsCacheLoaded = true;
+
+    // If manifest was prepared lazily, load all shards now
+    if (index._callsManifest) {
+        for (const [, { hash }] of index._callsManifest) {
+            _loadCallsShard(index, hash);
         }
+        return index.callsCache.size > 0;
     }
 
     // Legacy format: single calls-cache.json
-    const callsCacheFile = path.join(cacheDir, 'calls-cache.json');
+    const callsCacheFile = index._callsCacheLegacyFile ||
+        path.join(index.root, '.ucn-cache', 'calls-cache.json');
     if (!fs.existsSync(callsCacheFile)) return false;
 
     try {
@@ -393,6 +499,17 @@ function loadCallsCache(index) {
     return false;
 }
 
+/**
+ * Ensure calls cache is fully loaded (trigger lazy load if prepared but not loaded).
+ * Call this before any operation that needs callsCache (findCallers, buildCalleeIndex, etc.)
+ * @param {object} index - ProjectIndex instance
+ */
+function ensureCallsCacheLoaded(index) {
+    if (index._callsCachePrepared && !index._callsCacheLoaded) {
+        loadCallsCache(index);
+    }
+}
+
 /**
  * Load a single calls shard by hash.
  * @param {object} index - ProjectIndex instance
@@ -403,9 +520,13 @@ function _loadCallsShard(index, hash) {
     try {
         const data = JSON.parse(fs.readFileSync(shardFile, 'utf-8'));
         if (!Array.isArray(data)) return;
+        const rootPrefix = index.root.endsWith(path.sep) ? index.root : index.root + path.sep;
+        const toAbsShard = path.sep === '/'
+            ? (rp) => rootPrefix + rp
+            : (rp) => rootPrefix + rp.replace(/\//g, path.sep);
         for (const [relPath, entry] of data) {
             if (!relPath || !entry) continue;
-            const absPath = path.isAbsolute(relPath) ? relPath : path.join(index.root, relPath);
+            const absPath = path.isAbsolute(relPath) ? relPath : toAbsShard(relPath);
             index.callsCache.set(absPath, entry);
         }
     } catch (e) {
@@ -413,4 +534,37 @@ function _loadCallsShard(index, hash) {
     }
 }
 
-module.exports = { saveCache, loadCache, loadCallsCache, isCacheStale };
+/**
+ * Compute a cheap fingerprint of the index used to detect drift since the
+ * last reachability computation. Two states with the same fingerprint are
+ * indistinguishable for reachability purposes (file count + symbol count are
+ * monotonic with structural changes; an extra `entries[0]` byte detects most
+ * incremental rebuilds even when counts happen to match).
+ *
+ * Used by entrypoints.computeReachability to decide whether the persisted
+ * `_reachableSymbols` set is still valid.
+ *
+ * @param {object} index - ProjectIndex instance
+ * @returns {string} compact fingerprint
+ */
+function _computeReachabilityFingerprint(index) {
+    const fileCount = index.files ? index.files.size : 0;
+    const symbolCount = index.symbols ? index.symbols.size : 0;
+    // Sample a tiny prefix of the symbol map for a cheap structural check.
+    // Map iteration order is insertion order, which is stable across an
+    // unmodified load (built from cacheData.symbols in the same order).
+    let sample = '';
+    if (index.symbols && index.symbols.size > 0) {
+        let count = 0;
+        for (const [name, defs] of index.symbols) {
+            sample += name + ':' + (Array.isArray(defs) ? defs.length : 0) + '|';
+            if (++count >= 8) break;
+        }
+    }
+    return `${fileCount}:${symbolCount}:${sample}`;
+}
+
+module.exports = {
+    saveCache, loadCache, loadCallsCache, isCacheStale, ensureCallsCacheLoaded,
+    _computeReachabilityFingerprint,
+};