project-librarian 0.5.5 → 0.5.7

package/dist/code-index/modes.js

@@ -33,6 +33,7 @@ var __importStar = (this && this.__importStar) || (function () {
     };
 })();
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.resolveCodeIndexEngine = resolveCodeIndexEngine;
 exports.runCodeIndexMode = runCodeIndexMode;
 exports.runCodeQueryMode = runCodeQueryMode;
 exports.runCodeReportMode = runCodeReportMode;
@@ -70,45 +71,162 @@ function requireCompatibleDatabase(database, runtime) {
         ].join("\n"));
     }
 }
+function schemaMigrationRequired(reason) {
+    return reason.startsWith("existing schema version ") && !reason.includes("(missing)");
+}
+function shellQuote(value) {
+    if (/^[A-Za-z0-9_./:=@+-]+$/.test(value))
+        return value;
+    return `'${value.replace(/'/g, "'\\''")}'`;
+}
+function schemaMigrationApprovalCommand(options) {
+    const parts = ["project-librarian", "--code-index", "--code-index-migrate"];
+    for (const scope of options.scopes) {
+        if (scope !== ".")
+            parts.push("--code-scope", scope);
+    }
+    if (args_1.codeIndexOutput !== ".project-wiki/code-evidence.sqlite")
+        parts.push("--code-index-out", args_1.codeIndexOutput);
+    if (args_1.acknowledgeSmallRepoMode)
+        parts.push("--acknowledge-small-repo");
+    if (options.requestedEngine !== "auto")
+        parts.push("--code-index-engine", options.requestedEngine);
+    if (options.parserMode !== "default")
+        parts.push("--code-parser", options.parserMode);
+    return parts.map(shellQuote).join(" ");
+}
+function schemaMigrationRequiredMessage(runtime, reason, options) {
+    const databasePath = runtime.codeEvidenceDatabasePath();
+    return [
+        `code evidence index schema migration required: ${reason}`,
+        "The existing disposable code evidence index must be replaced before this version can write it.",
+        `approve: ${schemaMigrationApprovalCommand(options)}`,
+        "inspect: project-librarian --code-index-health",
+        `database: ${databasePath.relativePath}`,
+    ].join("\n");
+}
+function unreadableIndexMessage(runtime) {
+    const health = runtime.codeIndexHealth();
+    return [
+        health.message,
+        "inspect: project-librarian --code-index-health",
+        `rebuild: ${health.recommended_rebuild_command}`,
+        `database: ${health.database_path}`,
+    ].join("\n");
+}
+function elapsedMs(started) {
+    return Number(process.hrtime.bigint() - started) / 1_000_000;
+}
+function measurePhase(timings, key, fn) {
+    const started = process.hrtime.bigint();
+    try {
+        return fn();
+    }
+    finally {
+        timings[key] = Number(((timings[key] ?? 0) + elapsedMs(started)).toFixed(3));
+    }
+}
+function emitCodeIndexPhaseTimings(timings) {
+    if (process.env.PROJECT_LIBRARIAN_CODE_INDEX_TIMINGS !== "1")
+        return;
+    console.error(`code_index_phase_timings ${JSON.stringify(timings)}`);
+}
+function configureBulkWriteConnection(database) {
+    database.exec(`
+    PRAGMA synchronous = OFF;
+    PRAGMA temp_store = MEMORY;
+    PRAGMA cache_size = -20000;
+  `);
+}
+function shouldUseNativeIncrementalForAuto(requestedEngine, runtime, staleFileCount) {
+    if (requestedEngine !== "auto" || !args_1.codeIndexIncrementalMode)
+        return false;
+    if (staleFileCount <= 0 || !runtime.nativeCodeIndexAvailable())
+        return false;
+    return true;
+}
+function resolveCodeIndexEngine(requestedEngine, context, shouldUseNativeAuto, incrementalMode = args_1.codeIndexIncrementalMode) {
+    if (requestedEngine !== "auto")
+        return requestedEngine;
+    if (incrementalMode)
+        return "typescript";
+    return shouldUseNativeAuto(context) ? "native-rust" : "typescript";
+}
 function runCodeIndexMode(runtime) {
+    const totalStarted = process.hrtime.bigint();
+    const phaseTimings = {};
     const databasePath = runtime.codeEvidenceDatabasePath();
     const scopes = runtime.codeScopes();
     const parserMode = runtime.selectedCodeParserMode();
+    const requestedEngine = runtime.selectedCodeIndexEngine();
     // Scale gate before ANY write or database work: below the measured threshold
     // the build halts with the evidence-citing warning unless --acknowledge-small-repo
     // was passed (2026-06-12 scale-aware guidance decision).
-    const discoveredFiles = (0, code_index_file_policy_1.discoverCodeFiles)(scopes);
+    const discoveredFiles = measurePhase(phaseTimings, "discover_files_ms", () => (0, code_index_file_policy_1.discoverCodeFiles)(scopes));
     const scaleGate = (0, code_index_file_policy_1.smallRepoCodeIndexGate)(discoveredFiles.length, args_1.acknowledgeSmallRepoMode);
     if (!scaleGate.proceed)
         runtime.fail(scaleGate.warning);
+    const engineSelectionContext = runtime.codeIndexEngineSelectionContext(discoveredFiles, parserMode);
+    const engine = resolveCodeIndexEngine(requestedEngine, engineSelectionContext, runtime.shouldUseNativeCodeIndexAuto);
     const existingIndex = fs.existsSync(databasePath.absolutePath);
     if (args_1.codeIndexIncrementalMode && !existingIndex) {
         runtime.fail(`--incremental requires an existing compatible code evidence index: ${databasePath.relativePath}`);
     }
-    let incremental = false;
-    if (existingIndex && !args_1.codeIndexFullMode) {
-        let compatibility = { compatible: false, reason: "compatibility was not checked" };
-        const existingDatabase = runtime.openDatabase(databasePath.absolutePath);
+    let compatibility = { compatible: false, reason: "compatibility was not checked" };
+    let checkedCompatibility = false;
+    if (existingIndex) {
         try {
-            compatibility = (0, schema_1.incrementalCompatibility)(existingDatabase, scopes, parserMode);
+            measurePhase(phaseTimings, "compatibility_ms", () => {
+                const existingDatabase = runtime.openDatabase(databasePath.absolutePath);
+                try {
+                    compatibility = (0, schema_1.incrementalCompatibility)(existingDatabase, scopes, parserMode);
+                    checkedCompatibility = true;
+                }
+                finally {
+                    existingDatabase.close();
+                }
+            });
         }
-        finally {
-            existingDatabase.close();
+        catch {
+            if (!args_1.codeIndexFullMode && !args_1.codeIndexMigrateMode)
+                runtime.fail(unreadableIndexMessage(runtime));
         }
-        incremental = !args_1.codeIndexFullMode && compatibility.compatible;
+        if (!compatibility.compatible && schemaMigrationRequired(compatibility.reason) && !args_1.codeIndexMigrateMode) {
+            runtime.fail(schemaMigrationRequiredMessage(runtime, compatibility.reason, { parserMode, requestedEngine, scopes }));
+        }
+    }
+    if (engine === "native-rust") {
+        if (!args_1.codeIndexIncrementalMode) {
+            try {
+                measurePhase(phaseTimings, "native_helper_ms", () => runtime.runNativeCodeIndexMode({ databasePath, discoveredFiles, parserMode, requestedEngine, scopes }));
+                phaseTimings.total_ms = Number(elapsedMs(totalStarted).toFixed(3));
+                emitCodeIndexPhaseTimings(phaseTimings);
+                return;
+            }
+            catch (error) {
+                phaseTimings.total_ms = Number(elapsedMs(totalStarted).toFixed(3));
+                emitCodeIndexPhaseTimings(phaseTimings);
+                throw error;
+            }
+        }
+    }
+    let incremental = false;
+    if (existingIndex && checkedCompatibility && !args_1.codeIndexFullMode && !args_1.codeIndexMigrateMode) {
+        incremental = compatibility.compatible;
         if (args_1.codeIndexIncrementalMode && !compatibility.compatible)
             runtime.fail(`--incremental cannot update ${databasePath.relativePath}: ${compatibility.reason}`);
     }
-    runtime.prepareOutputPath();
-    if (!incremental)
-        runtime.removeDatabaseFiles(databasePath.absolutePath);
-    const database = runtime.openDatabase(databasePath.absolutePath);
+    measurePhase(phaseTimings, "prepare_output_ms", () => {
+        runtime.prepareOutputPath();
+        if (!incremental)
+            runtime.removeDatabaseFiles(databasePath.absolutePath);
+    });
+    let database = runtime.openDatabase(databasePath.absolutePath);
     try {
         if (!incremental)
-            (0, schema_1.setupDatabase)(database);
-        const statements = (0, schema_1.createIndexStatements)(database);
-        const currentFingerprints = discoveredFiles.map((filePath) => runtime.readCodeFileFingerprint(filePath));
-        let reindexedFiles;
+            (0, schema_1.setupDatabase)(database, { secondaryIndexes: false });
+        const currentFingerprints = measurePhase(phaseTimings, "fingerprints_ms", () => discoveredFiles.map((filePath) => runtime.readCodeFileFingerprint(filePath)));
+        let reindexedFingerprints;
         let deletedPaths;
         let indexedPaths = new Set();
         let unchangedFiles = 0;
@@ -122,45 +240,87 @@ function runCodeIndexMode(runtime) {
                 }]));
             const currentPaths = new Set(currentFingerprints.map((file) => file.path));
             deletedPaths = indexedRows.map((row) => String(row.path)).filter((filePath) => !currentPaths.has(filePath));
-            reindexedFiles = [];
+            reindexedFingerprints = [];
             for (const file of currentFingerprints) {
                 const existing = indexed.get(file.path);
                 if (existing && existing.mtimeMs === file.mtimeMs && existing.size === file.size) {
                     unchangedFiles += 1;
                     continue;
                 }
-                reindexedFiles.push(runtime.readCodeFile(file.path, parserMode));
+                reindexedFingerprints.push(file);
             }
         }
         else {
             deletedPaths = [];
-            reindexedFiles = discoveredFiles.map((filePath) => runtime.readCodeFile(filePath, parserMode));
+            reindexedFingerprints = currentFingerprints;
         }
-        database.exec("BEGIN");
-        if (!incremental)
-            statements.insertMeta.run("created_at", new Date().toISOString());
-        (0, schema_1.writeIndexMetadata)(scopes, parserMode, statements);
-        for (const filePath of deletedPaths)
-            (0, schema_1.removeIndexedFile)(filePath, statements);
-        for (const file of reindexedFiles) {
-            if (incremental && indexedPaths.has(file.path))
-                (0, schema_1.removeIndexedFile)(file.path, statements);
-            runtime.indexCodeFile(file, statements);
+        const nativeIncrementalRequested = engine === "native-rust"
+            || shouldUseNativeIncrementalForAuto(requestedEngine, runtime, reindexedFingerprints.length + deletedPaths.length);
+        if (nativeIncrementalRequested) {
+            if (!runtime.nativeCodeIndexAvailable()) {
+                runtime.fail("--code-index-engine native-rust --incremental requires PROJECT_LIBRARIAN_NATIVE_INDEXER or a packaged native helper.");
+            }
+            if (!runtime.nativeCodeIndexIncrementalEligible(reindexedFingerprints, parserMode)) {
+                if (engine === "native-rust") {
+                    runtime.fail("--code-index-engine native-rust --incremental only supports native-eligible parser profiles; use --code-index-engine typescript for this incremental update.");
+                }
+            }
+            else {
+                database.close();
+                database = undefined;
+                measurePhase(phaseTimings, "native_helper_ms", () => runtime.runNativeCodeIndexIncrementalMode({
+                    databasePath,
+                    deletedPaths,
+                    discoveredFiles,
+                    parserMode,
+                    requestedEngine,
+                    reindexedFiles: reindexedFingerprints,
+                    scopes,
+                    unchangedFiles,
+                }));
+                phaseTimings.total_ms = Number(elapsedMs(totalStarted).toFixed(3));
+                emitCodeIndexPhaseTimings(phaseTimings);
+                return;
+            }
         }
-        database.exec("COMMIT");
+        const reindexedFiles = measurePhase(phaseTimings, "read_files_ms", () => reindexedFingerprints.map((file) => runtime.readCodeFile(file.path, parserMode, file)));
+        const activeDatabase = database;
+        const statements = (0, schema_1.createIndexStatements)(activeDatabase);
+        measurePhase(phaseTimings, "sqlite_write_ms", () => {
+            configureBulkWriteConnection(activeDatabase);
+            activeDatabase.exec("BEGIN");
+            if (!incremental)
+                statements.insertMeta.run("created_at", new Date().toISOString());
+            (0, schema_1.writeIndexMetadata)(scopes, parserMode, statements);
+            for (const filePath of deletedPaths)
+                (0, schema_1.removeIndexedFile)(filePath, statements);
+            for (const file of reindexedFiles) {
+                if (incremental && indexedPaths.has(file.path))
+                    (0, schema_1.removeIndexedFile)(file.path, statements);
+                runtime.indexCodeFile(file, statements);
+            }
+            if (!incremental)
+                (0, schema_1.createSecondaryIndexes)(activeDatabase);
+            activeDatabase.exec("COMMIT");
+        });
+        phaseTimings.total_ms = Number(elapsedMs(totalStarted).toFixed(3));
         console.log("Project wiki code evidence index complete.");
         console.log(`database: ${databasePath.relativePath}`);
         console.log(`mode: ${incremental ? "incremental" : "full"}`);
+        console.log(`engine: ${engine}`);
+        if (requestedEngine === "auto")
+            console.log("engine_selection: auto");
         console.log(`parser_mode: ${parserMode}`);
         console.log(`scopes: ${scopes.join(", ")}`);
         console.log(`files: ${currentFingerprints.length}`);
         console.log(`reindexed_files: ${reindexedFiles.length}`);
         console.log(`deleted_files: ${deletedPaths.length}`);
         console.log(`unchanged_files: ${unchangedFiles}`);
+        emitCodeIndexPhaseTimings(phaseTimings);
     }
     catch (error) {
         try {
-            database.exec("ROLLBACK");
+            database?.exec("ROLLBACK");
         }
         catch {
             // Ignore rollback failures after setup errors.
@@ -168,7 +328,7 @@ function runCodeIndexMode(runtime) {
         throw error;
     }
     finally {
-        database.close();
+        database?.close();
     }
 }
 function runCodeQueryMode(runtime) {

package/dist/code-index/native-helper-matrix.js

@@ -0,0 +1,99 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.supportedNativeHelperTriples = exports.nativeCodeIndexHelperTargets = void 0;
+exports.nativeCodeIndexHelperTargetForTriple = nativeCodeIndexHelperTargetForTriple;
+exports.nativeCodeIndexHelperPlatformFromTriple = nativeCodeIndexHelperPlatformFromTriple;
+exports.nativeCodeIndexHelperTripleForPlatform = nativeCodeIndexHelperTripleForPlatform;
+exports.nativeCodeIndexHelperTargets = [
+    {
+        arch: "arm64",
+        architecture: "arm64",
+        format: "mach-o",
+        libc: "",
+        platform: "darwin",
+        runner: "macos-14",
+        rustTarget: "aarch64-apple-darwin",
+        triple: "darwin-arm64",
+    },
+    {
+        arch: "x64",
+        architecture: "x64",
+        format: "mach-o",
+        libc: "",
+        platform: "darwin",
+        runner: "macos-15-intel",
+        rustTarget: "x86_64-apple-darwin",
+        triple: "darwin-x64",
+    },
+    {
+        arch: "arm64",
+        architecture: "arm64",
+        format: "elf",
+        libc: "",
+        platform: "linux",
+        runner: "ubuntu-24.04-arm",
+        rustTarget: "aarch64-unknown-linux-gnu",
+        triple: "linux-arm64",
+    },
+    {
+        arch: "arm64",
+        architecture: "arm64",
+        format: "elf",
+        libc: "musl",
+        platform: "linux",
+        runner: "ubuntu-24.04-arm",
+        rustTarget: "aarch64-unknown-linux-musl",
+        triple: "linux-arm64-musl",
+    },
+    {
+        arch: "x64",
+        architecture: "x64",
+        format: "elf",
+        libc: "",
+        platform: "linux",
+        runner: "ubuntu-latest",
+        rustTarget: "x86_64-unknown-linux-gnu",
+        triple: "linux-x64",
+    },
+    {
+        arch: "x64",
+        architecture: "x64",
+        format: "elf",
+        libc: "musl",
+        platform: "linux",
+        runner: "ubuntu-latest",
+        rustTarget: "x86_64-unknown-linux-musl",
+        triple: "linux-x64-musl",
+    },
+    {
+        arch: "arm64",
+        architecture: "arm64",
+        format: "pe",
+        libc: "",
+        platform: "win32",
+        runner: "windows-11-arm",
+        rustTarget: "aarch64-pc-windows-msvc",
+        triple: "win32-arm64",
+    },
+    {
+        arch: "x64",
+        architecture: "x64",
+        format: "pe",
+        libc: "",
+        platform: "win32",
+        runner: "windows-latest",
+        rustTarget: "x86_64-pc-windows-msvc",
+        triple: "win32-x64",
+    },
+];
+exports.supportedNativeHelperTriples = exports.nativeCodeIndexHelperTargets.map((target) => target.triple);
+function nativeCodeIndexHelperTargetForTriple(triple) {
+    return exports.nativeCodeIndexHelperTargets.find((target) => target.triple === triple);
+}
+function nativeCodeIndexHelperPlatformFromTriple(triple) {
+    return nativeCodeIndexHelperTargetForTriple(triple)?.platform ?? String(triple).split("-")[0] ?? "";
+}
+function nativeCodeIndexHelperTripleForPlatform(platform, arch, libc = "") {
+    const normalizedLibc = platform === "linux" && libc === "musl" ? "musl" : "";
+    return exports.nativeCodeIndexHelperTargets.find((target) => (target.platform === platform && target.arch === arch && target.libc === normalizedLibc))?.triple ?? `${platform}-${arch}`;
+}