ucn 3.8.13 → 3.8.14

package/core/build-worker.js

@@ -0,0 +1,194 @@
+'use strict';
+
+/**
+ * core/build-worker.js - Worker thread for parallel index building
+ *
+ * Receives a chunk of file paths, parses each file, extracts symbols and calls,
+ * then sends results back to the main thread via MessagePort.
+ * Mirrors the indexFile() logic in project.js.
+ */
+
+const { workerData } = require('worker_threads');
+const fs = require('fs');
+const path = require('path');
+const crypto = require('crypto');
+const { detectLanguage, getParser, getLanguageModule, langTraits } = require('../languages');
+const { parse } = require('./parser');
+const { extractImports, extractExports } = require('./imports');
+
+const { files, rootDir, existingHashes, signal, workerIndex, port } = workerData;
+const signalArray = new Int32Array(signal);
+
+function addSymbol(fileEntry, item, type) {
+    const symbol = {
+        name: item.name,
+        type,
+        file: fileEntry.path,
+        relativePath: fileEntry.relativePath,
+        startLine: item.startLine,
+        endLine: item.endLine,
+        params: item.params,
+        paramsStructured: item.paramsStructured,
+        returnType: item.returnType,
+        modifiers: item.modifiers,
+        docstring: item.docstring,
+        bindingId: `${fileEntry.relativePath}:${type}:${item.startLine}`,
+    };
+    if (item.generics) symbol.generics = item.generics;
+    if (item.extends) symbol.extends = item.extends;
+    if (item.implements) symbol.implements = item.implements;
+    if (item.indent !== undefined) symbol.indent = item.indent;
+    if (item.isNested) symbol.isNested = item.isNested;
+    if (item.isMethod) symbol.isMethod = item.isMethod;
+    if (item.receiver) symbol.receiver = item.receiver;
+    if (item.className) symbol.className = item.className;
+    if (item.memberType) symbol.memberType = item.memberType;
+    if (item.fieldType) symbol.fieldType = item.fieldType;
+    if (item.decorators && item.decorators.length > 0) symbol.decorators = item.decorators;
+    if (item.nameLine) symbol.nameLine = item.nameLine;
+    if (item.traitImpl) symbol.traitImpl = true;
+    if (item.isSignature) symbol.isSignature = true;
+
+    fileEntry.symbols.push(symbol);
+    fileEntry.bindings.push({
+        id: symbol.bindingId,
+        name: symbol.name,
+        type: symbol.type,
+        startLine: symbol.startLine,
+    });
+}
+
+function processFile(filePath) {
+    const stat = fs.statSync(filePath);
+    const existing = existingHashes[filePath];
+
+    // Fast path: skip when mtime+size both match
+    if (existing && existing.mtime === stat.mtimeMs && existing.size === stat.size) {
+        return { filePath, skipped: true };
+    }
+
+    const content = fs.readFileSync(filePath, 'utf-8');
+    const hash = crypto.createHash('md5').update(content).digest('hex');
+
+    // Content-based skip: mtime changed but content didn't (touch, git checkout)
+    if (existing && existing.hash === hash) {
+        return { filePath, skipped: true, mtimeUpdate: stat.mtimeMs, sizeUpdate: stat.size };
+    }
+
+    const language = detectLanguage(filePath);
+    if (!language) return { filePath, skipped: true };
+
+    // Parse content — safeParse cache ensures tree is shared across
+    // parse()/extractImports()/extractExports()/findCallsInCode() (1 parse per file)
+    const parsed = parse(content, language);
+    const { imports, dynamicCount, importAliases } = extractImports(content, language);
+    const { exports } = extractExports(content, language);
+
+    // Extract calls in the same pass (eliminates buildCalleeIndex re-parsing)
+    let calls = null;
+    const langModule = getLanguageModule(language);
+    if (langModule.findCallsInCode) {
+        const parser = getParser(language);
+        const callOpts = {};
+        if (langTraits(language)?.hasReceiverPackageCalls) {
+            callOpts.imports = imports.flatMap(i => i.names || []);
+        }
+        calls = langModule.findCallsInCode(content, parser, callOpts);
+    }
+
+    // Detect bundled/minified files (same logic as indexFile in project.js)
+    let lineCount = 1, longLineCount = 0, lineStart = 0;
+    for (let ci = 0; ci < content.length; ci++) {
+        if (content.charCodeAt(ci) === 10) {
+            if (ci - lineStart > 1000) longLineCount++;
+            lineStart = ci + 1;
+            lineCount++;
+        }
+    }
+    if (content.length - lineStart > 1000) longLineCount++;
+
+    const isBundled = (
+        content.includes('__webpack_require__') || content.includes('__webpack_modules__') ||
+        (lineCount > 0 && lineCount < 50 && content.length / lineCount > 500) ||
+        (lineCount > 0 && longLineCount > 0 && longLineCount / lineCount > 0.3)
+    );
+
+    const isGenerated = /^\/\/\s*Code generated\b|^\/\/\s*DO NOT EDIT|^\/\/ @generated|^# Generated by/m.test(
+        content.slice(0, 500)
+    );
+
+    const relativePath = path.relative(rootDir, filePath);
+
+    // Build fileEntry (mirrors indexFile)
+    const fileEntry = {
+        path: filePath,
+        relativePath,
+        language,
+        lines: lineCount,
+        hash,
+        mtime: stat.mtimeMs,
+        size: stat.size,
+        imports: imports.map(i => i.module),
+        importNames: imports.flatMap(i => i.names || []),
+        exports: exports.map(e => e.name),
+        exportDetails: exports,
+        symbols: [],
+        bindings: [],
+        dynamicImports: dynamicCount || 0,
+    };
+    if (importAliases) fileEntry.importAliases = importAliases;
+    if (isBundled) fileEntry.isBundled = true;
+    if (isGenerated) fileEntry.isGenerated = true;
+
+    // Build symbols (mirrors indexFile)
+    for (const fn of parsed.functions) {
+        if (fn.receiver && !fn.className) {
+            fn.className = fn.receiver.replace(/^\*/, '');
+        }
+        addSymbol(fileEntry, fn, fn.isConstructor ? 'constructor' : 'function');
+    }
+
+    for (const cls of parsed.classes) {
+        addSymbol(fileEntry, cls, cls.type || 'class');
+        if (cls.members) {
+            for (const m of cls.members) {
+                addSymbol(fileEntry, { ...m, className: cls.name, ...(cls.traitName && { traitImpl: true }) }, m.memberType || 'method');
+            }
+        }
+    }
+
+    for (const state of parsed.stateObjects) {
+        addSymbol(fileEntry, state, 'state');
+    }
+
+    return {
+        filePath,
+        fileEntry,
+        calls,
+        callsMtime: stat.mtimeMs,
+        callsHash: hash,
+        hadExisting: !!existing,
+    };
+}
+
+// Process all files
+try {
+    const results = [];
+    for (const filePath of files) {
+        try {
+            results.push(processFile(filePath));
+        } catch (e) {
+            results.push({ filePath, error: e.message });
+        }
+    }
+
+    port.postMessage(results);
+    port.close();
+    Atomics.store(signalArray, workerIndex, 1);
+    Atomics.notify(signalArray, workerIndex);
+} catch (e) {
+    // Worker-level error — signal completion with empty results
+    try { port.postMessage([]); port.close(); } catch (_) { /* ignore */ }
+    Atomics.store(signalArray, workerIndex, 1);
+    Atomics.notify(signalArray, workerIndex);
+}

package/core/cache.js

@@ -79,6 +79,18 @@ function saveCache(index, cachePath) {
         return result;
     };
 
+    // Persist calleeIndex if built (paths must be absolute from this.files keys)
+    let calleeIndexData;
+    if (index.calleeIndex && index.calleeIndex.size > 0) {
+        calleeIndexData = [];
+        for (const [name, files] of index.calleeIndex) {
+            const relFiles = [...files].map(f =>
+                path.isAbsolute(f) ? path.relative(root, f) : f
+            );
+            calleeIndexData.push([name, relFiles]);
+        }
+    }
+
     const cacheData = {
         version: 7,  // v7: strip symbols/bindings from file entries (dedup ~45% cache reduction)
         ucnVersion: UCN_VERSION,  // Invalidate cache when UCN is updated
@@ -95,15 +107,44 @@ function saveCache(index, cachePath) {
         extendedByGraph: Array.from(index.extendedByGraph.entries()),
         failedFiles: index.failedFiles
             ? Array.from(index.failedFiles).map(f => path.relative(root, f))
-            : []
+            : [],
+        ...(calleeIndexData && { calleeIndex: calleeIndexData })
     };
 
     fs.writeFileSync(cacheFile, JSON.stringify(cacheData));
 
-    // Save callsCache to a separate file (lazy-loaded on demand)
+    // Save callsCache sharded by directory for lazy loading
     if (callsCacheData.length > 0) {
-        const callsCacheFile = path.join(path.dirname(cacheFile), 'calls-cache.json');
-        fs.writeFileSync(callsCacheFile, JSON.stringify(callsCacheData));
+        const callsDir = path.join(path.dirname(cacheFile), 'calls');
+        // Clean up old shards and legacy monolithic file
+        if (fs.existsSync(callsDir)) {
+            fs.rmSync(callsDir, { recursive: true, force: true });
+        }
+        const legacyFile = path.join(path.dirname(cacheFile), 'calls-cache.json');
+        if (fs.existsSync(legacyFile)) {
+            fs.rmSync(legacyFile, { force: true });
+        }
+        fs.mkdirSync(callsDir, { recursive: true });
+
+        // Group by directory
+        const shards = new Map();
+        for (const [relPath, entry] of callsCacheData) {
+            const dir = path.dirname(relPath) || '.';
+            if (!shards.has(dir)) shards.set(dir, []);
+            shards.get(dir).push([relPath, entry]);
+        }
+
+        // Write one shard per directory
+        const shardManifest = [];
+        for (const [dir, entries] of shards) {
+            const hash = crypto.createHash('md5').update(dir).digest('hex').slice(0, 10);
+            const shardFile = path.join(callsDir, `${hash}.json`);
+            fs.writeFileSync(shardFile, JSON.stringify(entries));
+            shardManifest.push([dir, hash, entries.length]);
+        }
+
+        // Write manifest for lazy loading
+        fs.writeFileSync(path.join(callsDir, 'manifest.json'), JSON.stringify(shardManifest));
     }
 
     return cacheFile;
@@ -208,6 +249,11 @@ function loadCache(index, cachePath) {
             loadCallsCache(index);
         }
 
+        // Build directory→files index from loaded data
+        if (typeof index._buildDirIndex === 'function') {
+            index._buildDirIndex();
+        }
+
         // Restore failedFiles if present (convert relative paths back to absolute)
         if (Array.isArray(cacheData.failedFiles)) {
             index.failedFiles = new Set(
@@ -215,6 +261,17 @@ function loadCache(index, cachePath) {
             );
         }
 
+        // Restore calleeIndex if persisted
+        if (Array.isArray(cacheData.calleeIndex)) {
+            index.calleeIndex = new Map();
+            for (const [name, files] of cacheData.calleeIndex) {
+                if (!Array.isArray(files)) continue;
+                index.calleeIndex.set(name, new Set(
+                    files.map(f => path.isAbsolute(f) ? f : path.join(root, f))
+                ));
+            }
+        }
+
         // Only rebuild graphs if config changed (e.g., aliases modified)
         const currentConfigHash = crypto.createHash('md5')
             .update(JSON.stringify(index.config || {})).digest('hex');
@@ -294,15 +351,36 @@ function loadCallsCache(index) {
     if (index._callsCacheLoaded) return false;   // Already attempted, file didn't exist
     index._callsCacheLoaded = true;
 
-    const callsCacheFile = path.join(index.root, '.ucn-cache', 'calls-cache.json');
+    const cacheDir = path.join(index.root, '.ucn-cache');
+
+    // Try sharded format first (calls/manifest.json)
+    const manifestFile = path.join(cacheDir, 'calls', 'manifest.json');
+    if (fs.existsSync(manifestFile)) {
+        try {
+            const manifest = JSON.parse(fs.readFileSync(manifestFile, 'utf-8'));
+            // Store manifest for lazy loading
+            index._callsManifest = new Map();
+            for (const [dir, hash, count] of manifest) {
+                index._callsManifest.set(dir, { hash, count, loaded: false });
+            }
+            // Eagerly load all shards (matches previous behavior)
+            for (const [, { hash }] of index._callsManifest) {
+                _loadCallsShard(index, hash);
+            }
+            return true;
+        } catch (e) {
+            // Corrupted manifest — fall through to legacy
+        }
+    }
+
+    // Legacy format: single calls-cache.json
+    const callsCacheFile = path.join(cacheDir, 'calls-cache.json');
     if (!fs.existsSync(callsCacheFile)) return false;
 
     try {
         const data = JSON.parse(fs.readFileSync(callsCacheFile, 'utf-8'));
         if (Array.isArray(data)) {
-            // Convert relative paths back to absolute
             const absData = data.map(([relPath, entry]) => {
-                // Handle both relative (v6+) and absolute (legacy) paths
                 const absPath = path.isAbsolute(relPath) ? relPath : path.join(index.root, relPath);
                 return [absPath, entry];
             });
@@ -315,4 +393,24 @@ function loadCallsCache(index) {
     return false;
 }
 
+/**
+ * Load a single calls shard by hash.
+ * @param {object} index - ProjectIndex instance
+ * @param {string} hash - Shard hash from manifest
+ */
+function _loadCallsShard(index, hash) {
+    const shardFile = path.join(index.root, '.ucn-cache', 'calls', `${hash}.json`);
+    try {
+        const data = JSON.parse(fs.readFileSync(shardFile, 'utf-8'));
+        if (!Array.isArray(data)) return;
+        for (const [relPath, entry] of data) {
+            if (!relPath || !entry) continue;
+            const absPath = path.isAbsolute(relPath) ? relPath : path.join(index.root, relPath);
+            index.callsCache.set(absPath, entry);
+        }
+    } catch (e) {
+        // Corrupted shard — skip
+    }
+}
+
 module.exports = { saveCache, loadCache, loadCallsCache, isCacheStale };