ucn 5.4.1 → 5.5.0

package/core/parallel-build.js

@@ -13,26 +13,26 @@ const fs = require('fs');
 const path = require('path');
 const { Worker, MessageChannel, receiveMessageOnPort } = require('worker_threads');
 
-function partitionFiles(index, files, workerCount) {
-    const chunks = Array.from({ length: workerCount }, () => ({
-        files: [], bytes: 0,
-    }));
-    const weightedFiles = files.map((file, order) => {
-        let bytes = index.files.get(file)?.size;
+// Young-generation cap of a parse worker's heap (see parallelBuild).
+const WORKER_YOUNG_GENERATION_MB = 32;
+
+/**
+ * Files in scheduling order: largest first (source size drives parse cost),
+ * discovery order breaking ties. Workers pull the next file from a shared
+ * counter, so a worker that drew an expensive file (a heavily recovered C++
+ * header, a generated source) no longer leaves its statically assigned
+ * backlog waiting while peers idle (fix #365). The first files in flight are
+ * the largest ones, as with the former longest-processing-time split, which
+ * keeps peak native-tree memory bounded the same way.
+ */
+function scheduleFiles(index, files, sizes = null) {
+    return files.map((file, order) => {
+        let bytes = sizes?.get(file) ?? index.files.get(file)?.size;
         if (!Number.isFinite(bytes)) {
             try { bytes = fs.statSync(file).size; } catch (_) { bytes = 0; }
         }
         return { file, bytes, order };
     }).sort((a, b) => b.bytes - a.bytes || a.order - b.order);
-    for (const weighted of weightedFiles) {
-        let target = chunks[0];
-        for (let i = 1; i < chunks.length; i++) {
-            if (chunks[i].bytes < target.bytes) target = chunks[i];
-        }
-        target.files.push(weighted.file);
-        target.bytes += weighted.bytes;
-    }
-    return chunks;
 }
 
 /**
@@ -43,6 +43,8 @@ function partitionFiles(index, files, workerCount) {
  * @param {object} options
  * @param {number} [options.workerCount] - Number of workers (auto-detect if omitted)
  * @param {boolean} [options.quiet] - Suppress output
+ * @param {number} [options.retainWorkers] - Parse workers kept for the build's
+ *   next phase (fix #388); `options.onParsed(pool)` receives them
  * @returns {number|false} Number of changed files, or false if too few workers
  */
 function parallelBuild(index, files, options = {}) {
@@ -70,49 +72,79 @@ function parallelBuild(index, files, options = {}) {
         console.error(`Parallel build: ${workerCount} workers for ${files.length} files`);
     }
 
-    // Parsing cost is driven much more by source size than file count. A
-    // round-robin split can put several giant generated/template headers in
-    // one worker while peers finish tiny files, making the whole build wait
-    // on one straggler and retaining multiple large native ASTs together.
-    // Longest-processing-time scheduling by byte size is deterministic and
-    // gives a substantially tighter upper bound on both wall time and peak
-    // memory. Canonical index ordering after the merge remains unchanged.
-    const chunks = partitionFiles(index, files, workerCount);
-
-    // Synchronization: one Int32 per worker in SharedArrayBuffer
-    const sab = new SharedArrayBuffer(4 * workerCount);
+    // Parsing cost is driven much more by source size than file count, and
+    // per-file cost varies further by language and recovery work; see
+    // scheduleFiles. Results merge in discovery order whatever worker parsed
+    // them, and canonical index ordering after the merge is unchanged.
+    const scheduled = scheduleFiles(index, files, options.sizes);
+    const scheduledFiles = scheduled.map(item => item.file);
+
+    // Synchronization: one Int32 per worker in SharedArrayBuffer, plus the
+    // shared next-file counter at index workerCount.
+    const sab = new SharedArrayBuffer(4 * (workerCount + 1));
     const signal = new Int32Array(sab);
+    // Parse workers kept for the next phase wait on a control slot once
+    // their queue is empty (fix #388): 2 = a job was posted to their port,
+    // anything else = exit.
+    const retainCount = Math.min(Math.max(options.retainWorkers | 0, 0), workerCount);
+    const control = retainCount > 0 ? new SharedArrayBuffer(4 * workerCount) : null;
 
     const ports = [];
     const workers = [];
+    // A dictionary given as a function (fix #396: the project's C/C++ macro
+    // definitions) is computed while the workers parse and posted to them;
+    // a worker waits for it only when a file's recovery consults it.
+    const lazyDictionary = typeof options.macroDictionary === 'function';
+    const dictionarySignal = lazyDictionary ? new SharedArrayBuffer(4) : null;
+
+    const workerHashes = Object.create(null);
+    for (const fp of scheduledFiles) {
+        const entry = index.files.get(fp);
+        if (entry) {
+            workerHashes[fp] = { mtime: entry.mtime, size: entry.size, hash: entry.hash };
+        }
+    }
 
     for (let i = 0; i < workerCount; i++) {
         const { port1, port2 } = new MessageChannel();
         ports.push(port1);
 
-        // Build per-worker hash subset (each worker only needs hashes for its chunk)
-        const workerHashes = Object.create(null);
-        for (const fp of chunks[i].files) {
-            const entry = index.files.get(fp);
-            if (entry) {
-                workerHashes[fp] = { mtime: entry.mtime, size: entry.size, hash: entry.hash };
-            }
-        }
-
         const worker = new Worker(path.join(__dirname, 'build-worker.js'), {
             workerData: {
-                files: chunks[i].files,
+                files: scheduledFiles,
                 rootDir: index.root,
                 existingHashes: workerHashes,
                 signal: sab,
                 workerIndex: i,
+                queueIndex: workerCount,
                 port: port2,
+                control,
+                macroDictionary: lazyDictionary ? null : (options.macroDictionary || null),
+                dictionarySignal,
             },
             transferList: [port2],
+            // Parse garbage (the native trees behind it included, which the
+            // JS heap does not see) is collected by young-generation
+            // scavenges; a bounded young generation collects it sooner
+            // (fix #388: build peak RSS -15 to -25% on fmt, tokio, django,
+            // ripgrep, clap, at equal CPU and wall within noise).
+            resourceLimits: { maxYoungGenerationSizeMb: WORKER_YOUNG_GENERATION_MB },
         });
         workers.push(worker);
     }
 
+    if (lazyDictionary) {
+        let dictionary = null;
+        try {
+            dictionary = options.macroDictionary();
+        } finally {
+            for (const port of ports) port.postMessage({ macroDictionary: dictionary || null });
+            const flag = new Int32Array(dictionarySignal);
+            Atomics.store(flag, 0, 1);
+            Atomics.notify(flag, 0);
+        }
+    }
+
     // Block main thread until all workers finish (with timeout)
     const TIMEOUT_MS = 300_000; // 5 minutes
     const deadline = Date.now() + TIMEOUT_MS;
@@ -128,15 +160,65 @@ function parallelBuild(index, files, options = {}) {
         }
     }
 
-    // Collect and merge results from each worker
+    // Collect results from each worker, then merge them in discovery order.
     let changed = 0;
-
+    const discoveryOrder = new Map(files.map((file, order) => [file, order]));
+    const collected = [];
     for (let i = 0; i < workerCount; i++) {
-        const msg = receiveMessageOnPort(ports[i]);
+        for (let msg = receiveMessageOnPort(ports[i]); msg; msg = receiveMessageOnPort(ports[i])) {
+            for (const result of msg.message) collected.push(result);
+        }
+    }
+    // Every result is in hand: the parse workers can go, except those kept
+    // for the build's next phase (fix #388: Rust macro expansion runs in
+    // parse workers whose parser code is already optimized, instead of in
+    // fresh threads that start cold). Only a worker that finished its queue
+    // normally (signal 1) is kept.
+    const controlArray = control ? new Int32Array(control) : null;
+    const kept = [];
+    for (let i = 0; i < workerCount; i++) {
+        if (kept.length < retainCount && Atomics.load(signal, i) === 1) {
+            kept.push(i);
+            // A kept worker never holds the process open.
+            workers[i].unref();
+            continue;
+        }
+        if (controlArray) {
+            Atomics.store(controlArray, i, 1);
+            Atomics.notify(controlArray, i);
+        }
         ports[i].close();
-        if (!msg) continue;
+        workers[i].terminate();
+    }
+    if (kept.length > 0) {
+        const live = new Set(kept);
+        const retire = i => {
+            if (!live.delete(i)) return;
+            Atomics.store(controlArray, i, 1);
+            Atomics.notify(controlArray, i);
+            ports[i].close();
+            workers[i].terminate();
+        };
+        options.onParsed?.({
+            available: [...kept],
+            ports,
+            post(i, job) {
+                ports[i].postMessage(job);
+                Atomics.store(controlArray, i, 2);
+                Atomics.notify(controlArray, i);
+            },
+            retire,
+            dispose() {
+                for (const i of [...live]) retire(i);
+            },
+        });
+    } else {
+        options.onParsed?.(null);
+    }
+    collected.sort((a, b) => discoveryOrder.get(a.filePath) - discoveryOrder.get(b.filePath));
 
-        for (const result of msg.message) {
+    {
+        for (const result of collected) {
             if (result.error) {
                 index.failedFiles.add(result.filePath);
                 if (!options.quiet) {
@@ -190,12 +272,7 @@ function parallelBuild(index, files, options = {}) {
         }
     }
 
-    // Terminate workers
-    for (const w of workers) {
-        w.terminate();
-    }
-
     return changed;
 }
 
-module.exports = { parallelBuild, partitionFiles };
+module.exports = { parallelBuild, scheduleFiles };

package/core/parser.js

@@ -6,6 +6,7 @@
  */
 
 const fs = require('fs');
+const { UcnError } = require('./errors');
 const { detectLanguage, getParser, getLanguageAdapter, isSupported } = require('../languages');
 
 /**
@@ -76,7 +77,7 @@ function parse(code, language) {
     }
 
     if (!language || !isSupported(language)) {
-        throw new Error(`Unsupported language: ${language}`);
+        throw new UcnError(`Unsupported language: ${language}`);
     }
 
     const parser = getParser(language);
@@ -95,7 +96,7 @@ function parseFile(filePath) {
     const language = detectLanguage(filePath);
 
     if (!language) {
-        throw new Error(`Cannot detect language for: ${filePath}`);
+        throw new UcnError(`Cannot detect language for: ${filePath}`);
     }
 
     const result = parse(code, language);