project-librarian 0.5.5 → 0.5.7

package/dist/code-index/native-helper.js

@@ -0,0 +1,292 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.nativeCodeIndexHelperPlatformTriple = nativeCodeIndexHelperPlatformTriple;
+exports.nativeCodeIndexHelperBinaryName = nativeCodeIndexHelperBinaryName;
+exports.packagedNativeCodeIndexHelperPath = packagedNativeCodeIndexHelperPath;
+exports.requireNativeCodeIndexHelperPath = requireNativeCodeIndexHelperPath;
+exports.nativeCodeIndexHelperAvailability = nativeCodeIndexHelperAvailability;
+exports.nativeCodeIndexHelperAvailable = nativeCodeIndexHelperAvailable;
+exports.buildNativeCodeIndexJob = buildNativeCodeIndexJob;
+exports.runNativeCodeIndexHelper = runNativeCodeIndexHelper;
+exports.runNativeCodeIndexRowsHelper = runNativeCodeIndexRowsHelper;
+const childProcess = __importStar(require("node:child_process"));
+const fs = __importStar(require("node:fs"));
+const os = __importStar(require("node:os"));
+const path = __importStar(require("node:path"));
+const native_helper_matrix_1 = require("./native-helper-matrix");
+const workspace_1 = require("../workspace");
+const supportedNativeHelperTripleSet = new Set(native_helper_matrix_1.supportedNativeHelperTriples);
+function nativeCodeIndexLinuxLibcVariant(platform = process.platform) {
+    if (platform !== "linux")
+        return "";
+    const report = process.report;
+    if (!report || typeof report.getReport !== "function")
+        return "";
+    try {
+        return report.getReport().header?.glibcVersionRuntime ? "glibc" : "musl";
+    }
+    catch {
+        return "";
+    }
+}
+function nativeCodeIndexHelperPlatformTriple(platform = process.platform, arch = process.arch, libc = nativeCodeIndexLinuxLibcVariant(platform)) {
+    return (0, native_helper_matrix_1.nativeCodeIndexHelperTripleForPlatform)(platform, arch, libc);
+}
+function nativeCodeIndexHelperBinaryName(platform = process.platform) {
+    return platform === "win32" ? "project-librarian-indexer.exe" : "project-librarian-indexer";
+}
+function nativeCodeIndexHelperPackageRoot(options = {}) {
+    return path.resolve(options.packageRoot ?? path.join(__dirname, ".."));
+}
+function packagedNativeCodeIndexHelperPath(options = {}) {
+    const platform = options.platform ?? process.platform;
+    const arch = options.arch ?? process.arch;
+    return path.join(nativeCodeIndexHelperPackageRoot(options), "native", nativeCodeIndexHelperPlatformTriple(platform, arch, options.libc), nativeCodeIndexHelperBinaryName(platform));
+}
+function configuredHelperPath(options = {}) {
+    const optionPath = (options.helperPath ?? "").trim();
+    if (optionPath)
+        return { helperPath: optionPath, source: "option" };
+    const envPath = ((options.env ?? process.env).PROJECT_LIBRARIAN_NATIVE_INDEXER ?? "").trim();
+    if (envPath)
+        return { helperPath: envPath, source: "environment" };
+    return { helperPath: "", source: "missing" };
+}
+function helperPathLabel(source) {
+    if (source === "option")
+        return "native helper path";
+    if (source === "environment")
+        return "PROJECT_LIBRARIAN_NATIVE_INDEXER";
+    return "packaged native helper";
+}
+function requireUsableHelperPath(helperPath, source) {
+    const label = helperPathLabel(source);
+    if (!path.isAbsolute(helperPath)) {
+        throw new Error(`${label} must be an absolute path: ${helperPath}`);
+    }
+    const resolved = path.resolve(helperPath);
+    if (!fs.existsSync(resolved)) {
+        throw new Error(`${label} does not exist: ${resolved}`);
+    }
+    const stat = fs.statSync(resolved);
+    if (!stat.isFile()) {
+        throw new Error(`${label} must point to an executable file: ${resolved}`);
+    }
+    fs.accessSync(resolved, fs.constants.X_OK);
+    return resolved;
+}
+function requireNativeCodeIndexHelperPath(options = {}) {
+    const availability = nativeCodeIndexHelperAvailability(options);
+    if (availability.available)
+        return availability.helperPath;
+    throw new Error(availability.reason);
+}
+function nativeCodeIndexHelperAvailability(options = {}) {
+    const platform = options.platform ?? process.platform;
+    const arch = options.arch ?? process.arch;
+    const platformTriple = nativeCodeIndexHelperPlatformTriple(platform, arch, options.libc);
+    const packagedHelperPath = packagedNativeCodeIndexHelperPath({ ...options, arch, platform });
+    const configured = configuredHelperPath(options);
+    if (configured.helperPath) {
+        try {
+            return {
+                available: true,
+                helperPath: requireUsableHelperPath(configured.helperPath, configured.source),
+                packagedHelperPath,
+                platformTriple,
+                reason: "",
+                source: configured.source,
+            };
+        }
+        catch (error) {
+            return {
+                available: false,
+                helperPath: "",
+                packagedHelperPath,
+                platformTriple,
+                reason: error instanceof Error ? error.message : String(error),
+                source: configured.source,
+            };
+        }
+    }
+    if (!supportedNativeHelperTripleSet.has(platformTriple)) {
+        return {
+            available: false,
+            helperPath: "",
+            packagedHelperPath,
+            platformTriple,
+            reason: `packaged native code index helper does not support this platform: ${platformTriple}; set PROJECT_LIBRARIAN_NATIVE_INDEXER to a compatible helper path`,
+            source: "missing",
+        };
+    }
+    if (fs.existsSync(packagedHelperPath)) {
+        try {
+            return {
+                available: true,
+                helperPath: requireUsableHelperPath(packagedHelperPath, "packaged"),
+                packagedHelperPath,
+                platformTriple,
+                reason: "",
+                source: "packaged",
+            };
+        }
+        catch (error) {
+            return {
+                available: false,
+                helperPath: "",
+                packagedHelperPath,
+                platformTriple,
+                reason: error instanceof Error ? error.message : String(error),
+                source: "packaged",
+            };
+        }
+    }
+    return {
+        available: false,
+        helperPath: "",
+        packagedHelperPath,
+        platformTriple,
+        reason: `--code-index-engine native-rust requires PROJECT_LIBRARIAN_NATIVE_INDEXER or a packaged native helper at ${packagedHelperPath}`,
+        source: "missing",
+    };
+}
+function nativeCodeIndexHelperAvailable(options = {}) {
+    return nativeCodeIndexHelperAvailability(options).available;
+}
+function writeJobManifest(job) {
+    const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "project-librarian-native-indexer-"));
+    const manifestPath = path.join(tmpDir, "job.json");
+    fs.writeFileSync(manifestPath, `${JSON.stringify(job)}\n`);
+    return manifestPath;
+}
+function buildNativeCodeIndexJob(input) {
+    const { mode = "full", ...rest } = input;
+    return {
+        abi_version: 1,
+        engine: "native-rust",
+        mode,
+        project_root: (0, workspace_1.normalizePath)(workspace_1.root),
+        ...rest,
+    };
+}
+function validateNativeCodeIndexSummary(job, summary) {
+    if (summary.engine !== job.engine) {
+        throw new Error(`native code index helper summary engine mismatch: expected ${job.engine}, got ${summary.engine ?? "(missing)"}`);
+    }
+    if (summary.schema_version !== job.schema_version) {
+        throw new Error(`native code index helper summary schema mismatch: expected ${job.schema_version}, got ${summary.schema_version ?? "(missing)"}`);
+    }
+    if (summary.mode !== job.mode) {
+        throw new Error(`native code index helper summary mode mismatch: expected ${job.mode}, got ${summary.mode ?? "(missing)"}`);
+    }
+    const database = summary.database ?? summary.database_path ?? "";
+    if (path.resolve(database) !== path.resolve(job.database_path)) {
+        throw new Error(`native code index helper summary database mismatch: expected ${job.database_path}, got ${database || "(missing)"}`);
+    }
+    if (!Number.isInteger(summary.files) || (summary.files ?? -1) < 0) {
+        throw new Error("native code index helper summary files must be a non-negative integer");
+    }
+    if (summary.native_files !== undefined && summary.native_files !== job.files.length) {
+        throw new Error(`native code index helper summary native_files mismatch: expected ${job.files.length}, got ${summary.native_files}`);
+    }
+    if ((summary.unsupported_profiles ?? []).length > 0) {
+        throw new Error(`native code index helper reported unsupported profiles: ${(summary.unsupported_profiles ?? []).join(", ")}`);
+    }
+    return summary;
+}
+function runNativeCodeIndexHelper(job, options = {}) {
+    const helperPath = requireNativeCodeIndexHelperPath(options);
+    const manifestPath = writeJobManifest(job);
+    try {
+        const result = childProcess.spawnSync(helperPath, ["--manifest", manifestPath], {
+            cwd: workspace_1.root,
+            encoding: "utf8",
+            stdio: ["ignore", "pipe", "pipe"],
+        });
+        if (result.status !== 0) {
+            const detail = (result.stderr || result.stdout || "").trim();
+            throw new Error(`native code index helper failed (${result.status ?? "signal"}): ${detail || helperPath}`);
+        }
+        let summary;
+        try {
+            summary = JSON.parse(result.stdout || "{}");
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            throw new Error(`native code index helper returned invalid JSON: ${message}`);
+        }
+        return validateNativeCodeIndexSummary(job, summary);
+    }
+    finally {
+        fs.rmSync(path.dirname(manifestPath), { recursive: true, force: true });
+    }
+}
+function validateNativeRows(value) {
+    if (typeof value !== "object" || value === null) {
+        throw new Error("native code index helper row stream must be an object");
+    }
+    const rows = value;
+    for (const key of ["configs", "edges", "files", "imports", "routes", "symbols"]) {
+        if (!Array.isArray(rows[key])) {
+            throw new Error(`native code index helper row stream missing array: ${key}`);
+        }
+    }
+    return rows;
+}
+function runNativeCodeIndexRowsHelper(job, options = {}) {
+    if (job.output_mode !== "row-stream") {
+        throw new Error("native row helper requires output_mode row-stream");
+    }
+    if (!job.rows_path) {
+        throw new Error("native row helper requires rows_path");
+    }
+    const summary = runNativeCodeIndexHelper(job, options);
+    let rowsJson = "";
+    try {
+        rowsJson = fs.readFileSync(job.rows_path, "utf8");
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        throw new Error(`native code index helper did not write rows: ${message}`);
+    }
+    try {
+        return { rows: validateNativeRows(JSON.parse(rowsJson)), summary };
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        throw new Error(`native code index helper returned invalid row stream: ${message}`);
+    }
+}

package/dist/code-index/schema.js

@@ -1,6 +1,41 @@
 "use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.codeIndexSchemaVersion = void 0;
+exports.fileFtsRowid = fileFtsRowid;
+exports.createSecondaryIndexes = createSecondaryIndexes;
 exports.setupDatabase = setupDatabase;
 exports.createIndexStatements = createIndexStatements;
 exports.removeIndexedFile = removeIndexedFile;
@@ -10,14 +45,44 @@ exports.indexedScopes = indexedScopes;
 exports.indexedParserMode = indexedParserMode;
 exports.incrementalCompatibility = incrementalCompatibility;
 exports.codeIndexSnapshot = codeIndexSnapshot;
+const crypto = __importStar(require("node:crypto"));
 const workspace_1 = require("../workspace");
-exports.codeIndexSchemaVersion = "4";
-function setupDatabase(database) {
+exports.codeIndexSchemaVersion = "5";
+function stableFtsRowid(parts) {
+    const hash = crypto.createHash("sha256");
+    for (const part of parts) {
+        hash.update(part);
+        hash.update("\0");
+    }
+    const digest = hash.digest();
+    let value = 0;
+    for (let index = 0; index < 6; index += 1)
+        value = value * 256 + digest[index];
+    return value + 1;
+}
+function fileFtsRowid(filePath) {
+    return stableFtsRowid(["file", filePath]);
+}
+const secondaryIndexSql = `
+  CREATE INDEX idx_symbols_file ON symbols(file_path);
+  CREATE INDEX idx_symbols_name ON symbols(name);
+  CREATE INDEX idx_imports_from ON imports(from_file);
+  CREATE INDEX idx_routes_path ON routes(route);
+  CREATE INDEX idx_configs_file ON configs(file_path);
+  CREATE INDEX idx_edges_source ON edges(source_kind, source);
+  CREATE INDEX idx_edges_target ON edges(target_kind, target);
+  CREATE INDEX idx_edges_kind ON edges(kind);
+`;
+function createSecondaryIndexes(database) {
+    database.exec(secondaryIndexSql);
+}
+function setupDatabase(database, options = {}) {
     database.exec(`
     PRAGMA journal_mode = WAL;
     CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
     CREATE TABLE files (
       path TEXT PRIMARY KEY,
+      fts_rowid INTEGER NOT NULL UNIQUE,
       language TEXT NOT NULL,
       profile TEXT NOT NULL,
       kind TEXT NOT NULL,
@@ -71,30 +136,24 @@ function setupDatabase(database) {
     );
     CREATE VIRTUAL TABLE files_fts USING fts5(path, language, profile, content);
     CREATE VIRTUAL TABLE symbols_fts USING fts5(name, kind, file_path, signature);
-    CREATE INDEX idx_symbols_file ON symbols(file_path);
-    CREATE INDEX idx_symbols_name ON symbols(name);
-    CREATE INDEX idx_imports_from ON imports(from_file);
-    CREATE INDEX idx_routes_path ON routes(route);
-    CREATE INDEX idx_configs_file ON configs(file_path);
-    CREATE INDEX idx_edges_source ON edges(source_kind, source);
-    CREATE INDEX idx_edges_target ON edges(target_kind, target);
-    CREATE INDEX idx_edges_kind ON edges(kind);
   `);
+    if (options.secondaryIndexes ?? true)
+        createSecondaryIndexes(database);
 }
 function createIndexStatements(database) {
     return {
         deleteConfig: database.prepare("DELETE FROM configs WHERE file_path = ?"),
         deleteEdge: database.prepare("DELETE FROM edges WHERE file_path = ?"),
         deleteFile: database.prepare("DELETE FROM files WHERE path = ?"),
-        deleteFileFts: database.prepare("DELETE FROM files_fts WHERE path = ?"),
+        deleteFileFts: database.prepare("DELETE FROM files_fts WHERE rowid = (SELECT fts_rowid FROM files WHERE path = ?)"),
         deleteImport: database.prepare("DELETE FROM imports WHERE from_file = ?"),
         deleteRoute: database.prepare("DELETE FROM routes WHERE file_path = ?"),
         deleteSymbol: database.prepare("DELETE FROM symbols WHERE file_path = ?"),
         deleteSymbolFts: database.prepare("DELETE FROM symbols_fts WHERE file_path = ?"),
         insertConfig: database.prepare("INSERT INTO configs (key, value, file_path, line) VALUES (?, ?, ?, ?)"),
         insertEdge: database.prepare("INSERT INTO edges (kind, source_kind, source, target_kind, target, file_path, line, evidence) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"),
-        insertFile: database.prepare("INSERT INTO files (path, language, profile, kind, bytes, lines, hash, mtime_ms, size) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)"),
-        insertFileFts: database.prepare("INSERT INTO files_fts (path, language, profile, content) VALUES (?, ?, ?, ?)"),
+        insertFile: database.prepare("INSERT INTO files (path, fts_rowid, language, profile, kind, bytes, lines, hash, mtime_ms, size) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"),
+        insertFileFts: database.prepare("INSERT INTO files_fts (rowid, path, language, profile, content) VALUES (?, ?, ?, ?, ?)"),
         insertImport: database.prepare("INSERT INTO imports (from_file, to_ref, imported, line, raw) VALUES (?, ?, ?, ?, ?)"),
         insertMeta: database.prepare("INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)"),
         insertRoute: database.prepare("INSERT INTO routes (method, route, file_path, line, handler) VALUES (?, ?, ?, ?, ?)"),

package/dist/code-index/search.js

@@ -82,7 +82,7 @@ function searchFiles(database, term, limit = 25) {
         const ftsRows = database.prepare(`
       SELECT files.path, files.language, files.profile, files.lines, files.bytes
       FROM files_fts
-      JOIN files ON files.path = files_fts.path
+      JOIN files ON files.fts_rowid = files_fts.rowid
       WHERE files_fts MATCH ?
       ORDER BY bm25(files_fts, 8.0, 1.0, 1.0, 0.25), files.path
       LIMIT ?

package/dist/code-index-file-policy.js

@@ -38,6 +38,7 @@ exports.fileLanguage = fileLanguage;
 exports.isJavaScriptLike = isJavaScriptLike;
 exports.shouldIndexFile = shouldIndexFile;
 exports.isIgnoredCodePath = isIgnoredCodePath;
+exports.cachedDiscoveredCodeFileStat = cachedDiscoveredCodeFileStat;
 exports.discoverCodeFiles = discoverCodeFiles;
 exports.smallRepoCodeIndexGate = smallRepoCodeIndexGate;
 const childProcess = __importStar(require("node:child_process"));
@@ -76,6 +77,7 @@ const languageByExtension = {
 };
 const configExtensions = new Set([".json", ".yaml", ".yml", ".toml"]);
 exports.maxIndexedBytes = 1024 * 1024;
+const discoveredCodeFileStats = new Map();
 function fileLanguage(relativePath) {
     if (path.basename(relativePath) === ".env.example")
         return "config";
@@ -170,7 +172,11 @@ function gitTrackedAndUnignoredFiles(scopes) {
 function indexableFileStat(file) {
     return (0, workspace_1.containedProjectFileStat)(file);
 }
+function cachedDiscoveredCodeFileStat(relativePath) {
+    return discoveredCodeFileStats.get(relativePath);
+}
 function discoverCodeFiles(scopes) {
+    discoveredCodeFileStats.clear();
     const gitFiles = gitTrackedAndUnignoredFiles(scopes);
     const candidates = gitFiles ?? scopes.flatMap((scope) => walkCodeFiles(scope));
     const files = [];
@@ -178,8 +184,10 @@ function discoverCodeFiles(scopes) {
         if (isIgnoredCodePath(file) || !shouldIndexFile(file))
             continue;
         const stat = indexableFileStat(file);
-        if (stat && stat.size <= exports.maxIndexedBytes)
+        if (stat && stat.size <= exports.maxIndexedBytes) {
+            discoveredCodeFileStats.set(file, { absolutePath: (0, workspace_1.abs)(file), stat });
             files.push(file);
+        }
     }
     return files.sort();
 }