rho-graph 0.1.0

package/dist/indexer/import-resolver.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Attempt to resolve a raw import source string (as extracted from the AST)
+ * to an absolute file path that exists in the indexed file set.
+ *
+ * Returns null if the import cannot be resolved (e.g., stdlib, third-party,
+ * or a language we don't know how to resolve).
+ */
+export declare function resolveImport(source: string, fromFilePath: string, language: string, filePathSet: Set<string>): string | null;

package/dist/indexer/import-resolver.js

@@ -0,0 +1,80 @@
+import { dirname, join, resolve } from "node:path";
+/**
+ * Attempt to resolve a raw import source string (as extracted from the AST)
+ * to an absolute file path that exists in the indexed file set.
+ *
+ * Returns null if the import cannot be resolved (e.g., stdlib, third-party,
+ * or a language we don't know how to resolve).
+ */
+export function resolveImport(source, fromFilePath, language, filePathSet) {
+    if (language === "python") {
+        return resolvePythonImport(source, fromFilePath, filePathSet);
+    }
+    if (language === "typescript" || language === "tsx" || language === "javascript") {
+        return resolveJsImport(source, fromFilePath, filePathSet);
+    }
+    return null;
+}
+function resolvePythonImport(source, fromFilePath, filePathSet) {
+    // Count leading dots (relative import markers)
+    let dots = 0;
+    let rest = source;
+    while (rest.startsWith(".")) {
+        dots++;
+        rest = rest.slice(1);
+    }
+    const relPath = rest ? rest.split(".").join("/") : "";
+    if (dots > 0) {
+        // Relative import: 1 dot = same package dir, 2 dots = parent, etc.
+        let base = dirname(fromFilePath);
+        for (let i = 1; i < dots; i++) {
+            base = dirname(base);
+        }
+        return tryPythonCandidates(base, relPath, filePathSet);
+    }
+    // Absolute import: try file's directory first, then walk up toward repo root.
+    // filePathSet is scoped to the repo so no false matches outside it.
+    let dir = dirname(fromFilePath);
+    while (true) {
+        const result = tryPythonCandidates(dir, relPath, filePathSet);
+        if (result !== null)
+            return result;
+        const parent = dirname(dir);
+        if (parent === dir)
+            break; // reached filesystem root
+        dir = parent;
+    }
+    return null;
+}
+function tryPythonCandidates(base, relPath, filePathSet) {
+    const candidates = relPath
+        ? [join(base, relPath + ".py"), join(base, relPath, "__init__.py")]
+        : [join(base, "__init__.py")];
+    for (const candidate of candidates) {
+        if (filePathSet.has(candidate))
+            return candidate;
+    }
+    return null;
+}
+function resolveJsImport(source, fromFilePath, filePathSet) {
+    if (!source.startsWith("./") && !source.startsWith("../"))
+        return null;
+    const base = resolve(dirname(fromFilePath), source);
+    // Try the path as-is (already has extension)
+    if (filePathSet.has(base))
+        return base;
+    // Try common extensions
+    for (const ext of [".ts", ".tsx", ".js", ".jsx"]) {
+        const candidate = base + ext;
+        if (filePathSet.has(candidate))
+            return candidate;
+    }
+    // Try as a directory index file
+    for (const ext of [".ts", ".tsx", ".js", ".jsx"]) {
+        const candidate = join(base, "index" + ext);
+        if (filePathSet.has(candidate))
+            return candidate;
+    }
+    return null;
+}
+//# sourceMappingURL=import-resolver.js.map

package/dist/indexer/import-resolver.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"import-resolver.js","sourceRoot":"","sources":["../../src/indexer/import-resolver.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEnD;;;;;;GAMG;AACH,MAAM,UAAU,aAAa,CAC3B,MAAc,EACd,YAAoB,EACpB,QAAgB,EAChB,WAAwB;IAExB,IAAI,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC1B,OAAO,mBAAmB,CAAC,MAAM,EAAE,YAAY,EAAE,WAAW,CAAC,CAAC;IAChE,CAAC;IACD,IAAI,QAAQ,KAAK,YAAY,IAAI,QAAQ,KAAK,KAAK,IAAI,QAAQ,KAAK,YAAY,EAAE,CAAC;QACjF,OAAO,eAAe,CAAC,MAAM,EAAE,YAAY,EAAE,WAAW,CAAC,CAAC;IAC5D,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,mBAAmB,CAC1B,MAAc,EACd,YAAoB,EACpB,WAAwB;IAExB,+CAA+C;IAC/C,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,IAAI,GAAG,MAAM,CAAC;IAClB,OAAO,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QAC5B,IAAI,EAAE,CAAC;QACP,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACvB,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAEtD,IAAI,IAAI,GAAG,CAAC,EAAE,CAAC;QACb,mEAAmE;QACnE,IAAI,IAAI,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;QACjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;YAC9B,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;QACvB,CAAC;QACD,OAAO,mBAAmB,CAAC,IAAI,EAAE,OAAO,EAAE,WAAW,CAAC,CAAC;IACzD,CAAC;IAED,8EAA8E;IAC9E,oEAAoE;IACpE,IAAI,GAAG,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;IAChC,OAAO,IAAI,EAAE,CAAC;QACZ,MAAM,MAAM,GAAG,mBAAmB,CAAC,GAAG,EAAE,OAAO,EAAE,WAAW,CAAC,CAAC;QAC9D,IAAI,MAAM,KAAK,IAAI;YAAE,OAAO,MAAM,CAAC;QACnC,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;QAC5B,IAAI,MAAM,KAAK,GAAG;YAAE,MAAM,CAAC,0BAA0B;QACrD,GAAG,GAAG,MAAM,CAAC;IACf,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,mBAAmB,CAC1B,IAAY,EACZ,OAAe,EACf,WAAwB;IAExB,MAAM,UAAU,GAAG,OAAO;QACxB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,GAAG,KAAK,CAAC,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,aAAa,CAAC,CAAC;QACnE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC,CAAC;IAEhC,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,IAAI,WAAW,CAAC,GAAG,CAAC,SAAS,CAAC;YAAE,OAAO,SAAS,CAAC;IACnD,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,eAAe,CACtB,MAAc,EACd,YAAoB,EACpB,WAAwB;IAExB,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAEvE,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC,CAAC;IAEpD,6CAA6C;IAC7C,IAAI,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IAEvC,wBAAwB;IACxB,KAAK,MAAM,GAAG,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC;QACjD,MAAM,SAAS,GAAG,IAAI,GAAG,GAAG,CAAC;QAC7B,IAAI,WAAW,CAAC,GAAG,CAAC,SAAS,CAAC;YAAE,OAAO,SAAS,CAAC;IACnD,CAAC;IAED,gCAAgC;IAChC,KAAK,MAAM,GAAG,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC;QACjD,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,EAAE,OAAO,GAAG,GAAG,CAAC,CAAC;QAC5C,IAAI,WAAW,CAAC,GAAG,CAAC,SAAS,CAAC;YAAE,OAAO,SAAS,CAAC;IACnD,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC"}

package/dist/indexer/index.d.ts

@@ -0,0 +1,21 @@
+import type { DbConnection } from "../db/connection.js";
+import type { IndexConfig } from "../config.js";
+export declare function discoverFiles(rootPath: string, config: IndexConfig): Promise<string[]>;
+export interface IndexResult {
+    filesIndexed: number;
+    functionsFound: number;
+    classesFound: number;
+    orphansRemoved: number;
+    errors: Array<{
+        file: string;
+        error: string;
+    }>;
+}
+export declare function indexRepository(db: DbConnection, repoPath: string, config: IndexConfig, options?: {
+    changedOnly?: boolean;
+    specificPath?: string;
+    concurrency?: number;
+    maxMemoryMB?: number;
+    onProgress?: (current: number, total: number, file: string) => void;
+    onFlushProgress?: (completed: number, total: number) => void;
+}): Promise<IndexResult>;

package/dist/indexer/index.js

@@ -0,0 +1,262 @@
+import { resolve, relative } from "node:path";
+import { execFileSync } from "node:child_process";
+import { glob } from "glob";
+import { minimatch } from "minimatch";
+import { initParser, parseFile, detectLanguage } from "./parser.js";
+import { extractGraphEntities } from "./extractor.js";
+import { writeRepoOnce } from "./graph-writer.js";
+import { BatchGraphWriter } from "./batch-writer.js";
+import { getRepositoryCommit, upsertRepositoryWithCommit, deleteFileAndRelationships, getAllFilePathsUnderPrefix, batchDeleteOrphanFiles, } from "../db/queries.js";
+import { computeFileHash, getFileMtime, isFileStale, getChangedFilesSinceCommit, getCurrentCommitSha } from "./staleness.js";
+import { runParallelPipeline } from "./parallel-pipeline.js";
+/**
+ * Attempt to list files via git, which natively respects .gitignore.
+ * Returns null if not a git repo or git is unavailable.
+ */
+function getGitFiles(absRoot) {
+    try {
+        const output = execFileSync("git", ["ls-files", "--cached", "--others", "--exclude-standard", "-z"], { cwd: absRoot, encoding: "utf-8", stdio: ["pipe", "pipe", "pipe"] });
+        return output.split("\0").filter(Boolean).map((f) => resolve(absRoot, f));
+    }
+    catch {
+        return null;
+    }
+}
+export async function discoverFiles(rootPath, config) {
+    const absRoot = resolve(rootPath);
+    const excludePatterns = config.exclude.map((e) => e.includes("/") ? e : `**/${e}/**`);
+    // Prefer git ls-files: it natively respects .gitignore, so node_modules
+    // and virtual environments are excluded without needing explicit config.
+    const gitFiles = getGitFiles(absRoot);
+    if (gitFiles !== null) {
+        return gitFiles.filter((absPath) => {
+            const rel = relative(absRoot, absPath);
+            const matchesInclude = config.include.some((p) => minimatch(rel, p, { dot: true }));
+            const matchesExclude = excludePatterns.some((p) => minimatch(rel, p, { dot: true }));
+            return matchesInclude && !matchesExclude;
+        });
+    }
+    // Fallback for non-git directories: use glob with manual exclude patterns.
+    const allFiles = [];
+    for (const pattern of config.include) {
+        const matches = await glob(pattern, {
+            cwd: absRoot,
+            absolute: true,
+            ignore: excludePatterns,
+            nodir: true,
+        });
+        allFiles.push(...matches);
+    }
+    return [...new Set(allFiles)];
+}
+/**
+ * Remove File nodes (and their child Function/Class nodes) that exist in the
+ * graph under `pathPrefix` but are no longer in the discovered file set.
+ *
+ * This is the "self-healing" pass: if a previous index run wrote files that
+ * shouldn't have been indexed (e.g., venv contents written before .gitignore
+ * was respected, or a file that was deleted on disk), this removes them on
+ * the next full re-index.
+ *
+ * Safety: if discovery returned zero files but the graph has files indexed
+ * under this prefix, we refuse to delete. That guards against a botched
+ * discovery (wrong path, broken git, transient I/O failure) silently
+ * nuking the entire repo's graph.
+ */
+const ORPHAN_DELETE_CHUNK_SIZE = 500;
+async function cleanOrphanedFiles(db, pathPrefix, discoveredFiles) {
+    const session = db.session();
+    try {
+        const listQ = getAllFilePathsUnderPrefix({ pathPrefix });
+        const result = await session.run(listQ.cypher, listQ.params);
+        const indexedPaths = result.records.map((r) => r.get("path"));
+        const orphans = indexedPaths.filter((p) => !discoveredFiles.has(p));
+        if (orphans.length === 0)
+            return 0;
+        if (discoveredFiles.size === 0) {
+            console.warn(`[indexer] orphan cleanup skipped: discovery returned 0 files but ${indexedPaths.length} are indexed under ${pathPrefix}. ` +
+                `Refusing to delete to avoid wiping the graph on a botched discovery. ` +
+                `If this is intentional, remove the repo manually.`);
+            return 0;
+        }
+        // Chunk the deletes: a single UNWIND of thousands of DETACH DELETEs can
+        // exceed Neo4j's per-transaction memory budget (MemoryPoolOutOfMemoryError).
+        // 500 per chunk keeps each tx small while still cutting round-trips ~500x
+        // vs. one-by-one. The trade-off: a partial run leaves some orphans, which
+        // the next index will pick up — acceptable for an idempotent cleanup.
+        let deleted = 0;
+        for (let i = 0; i < orphans.length; i += ORPHAN_DELETE_CHUNK_SIZE) {
+            const chunk = orphans.slice(i, i + ORPHAN_DELETE_CHUNK_SIZE);
+            const deleteQ = batchDeleteOrphanFiles(chunk);
+            await session.run(deleteQ.cypher, deleteQ.params);
+            deleted += chunk.length;
+        }
+        return deleted;
+    }
+    finally {
+        await session.close();
+    }
+}
+export async function indexRepository(db, repoPath, config, options = {}) {
+    const absRoot = resolve(repoPath);
+    await initParser();
+    let files;
+    if (options.specificPath) {
+        files = await discoverFiles(resolve(absRoot, options.specificPath), config);
+    }
+    else {
+        files = await discoverFiles(absRoot, config);
+    }
+    // Filter to supported languages
+    files = files.filter((f) => detectLanguage(f) !== null);
+    // Build the full set of discovered file paths for import resolution
+    const filePathSet = new Set(files);
+    // If changedOnly, check staleness against existing graph
+    if (options.changedOnly) {
+        // Try git-based incremental first
+        const session = db.session();
+        let usedGitBased = false;
+        try {
+            const commitQ = getRepositoryCommit({ path: absRoot });
+            const result = await session.run(commitQ.cypher, commitQ.params);
+            const lastCommit = result.records[0]?.get("lastIndexedCommit") ?? null;
+            if (lastCommit) {
+                const diff = getChangedFilesSinceCommit(absRoot, lastCommit, { includeDeleted: true });
+                if (diff.error) {
+                    // Git failed (invalid SHA, shallow clone, etc.) — fall through to hash/mtime
+                }
+                else {
+                    // Remove deleted files from the index
+                    for (const relPath of diff.deleted) {
+                        const absPath = resolve(absRoot, relPath);
+                        const deleteQ = deleteFileAndRelationships({ filePath: absPath });
+                        await session.run(deleteQ.cypher, deleteQ.params); // let errors propagate
+                    }
+                    // Filter files to only changed ones
+                    const changedAbsPaths = new Set(diff.changed.map((f) => resolve(absRoot, f)));
+                    files = files.filter((f) => changedAbsPaths.has(f));
+                    usedGitBased = true;
+                }
+            }
+        }
+        finally {
+            await session.close();
+        }
+        // Fall back to mtime/hash-based staleness if git-based wasn't used
+        if (!usedGitBased) {
+            const session2 = db.session();
+            try {
+                const result = await session2.run("MATCH (f:File) WHERE f.path STARTS WITH $repoPath RETURN f.path AS path, f.hash AS hash, f.lastModified AS lastModified", { repoPath: absRoot });
+                const indexed = new Map(result.records.map((r) => [
+                    r.get("path"),
+                    { hash: r.get("hash"), lastModified: r.get("lastModified") },
+                ]));
+                files = files.filter((f) => {
+                    const existing = indexed.get(f);
+                    if (!existing)
+                        return true;
+                    return isFileStale(f, existing.hash, existing.lastModified);
+                });
+            }
+            finally {
+                await session2.close();
+            }
+        }
+    }
+    const result = {
+        filesIndexed: 0,
+        functionsFound: 0,
+        classesFound: 0,
+        orphansRemoved: 0,
+        errors: [],
+    };
+    // Snapshot of what discovery returned, for orphan cleanup later. We capture
+    // this before any post-discovery filtering so the cleanup compares against
+    // the full set of files we *intend* to manage on this run.
+    const discoveredSet = options.changedOnly ? null : new Set(files);
+    await writeRepoOnce(db, absRoot);
+    const concurrency = options.concurrency ?? 8;
+    const maxMemoryBytes = (options.maxMemoryMB ?? 8192) * 1024 * 1024;
+    if (concurrency > 1) {
+        const batchWriter = new BatchGraphWriter(db, { filePathSet });
+        const pipelineResult = await runParallelPipeline({
+            files,
+            absRoot,
+            concurrency,
+            maxMemoryBytes,
+            parseFn: parseFile,
+            extractFn: extractGraphEntities,
+            batchWriter,
+            computeHashFn: computeFileHash,
+            getMtimeFn: getFileMtime,
+            onProgress: options.onProgress,
+            onFlushProgress: options.onFlushProgress,
+        });
+        result.filesIndexed = pipelineResult.filesIndexed;
+        result.functionsFound = pipelineResult.functionsFound;
+        result.classesFound = pipelineResult.classesFound;
+        result.errors = pipelineResult.errors;
+    }
+    else {
+        // Sequential fallback
+        const batchWriter = new BatchGraphWriter(db, { filePathSet });
+        for (let i = 0; i < files.length; i++) {
+            const file = files[i];
+            options.onProgress?.(i + 1, files.length, file);
+            try {
+                const parseResult = await parseFile(file);
+                if (!parseResult)
+                    continue;
+                let entities;
+                try {
+                    entities = extractGraphEntities(parseResult.tree, parseResult.language, parseResult.source, file);
+                }
+                finally {
+                    parseResult.tree.delete();
+                }
+                batchWriter.add(entities, {
+                    filePath: file, relativePath: relative(absRoot, file), repoPath: absRoot,
+                    language: parseResult.language,
+                    hash: computeFileHash(file, parseResult.source),
+                    lastModified: getFileMtime(file),
+                });
+                result.filesIndexed++;
+                result.functionsFound += entities.functions.length;
+                result.classesFound += entities.classes.length;
+            }
+            catch (error) {
+                result.errors.push({ file, error: error instanceof Error ? error.message : String(error) });
+            }
+        }
+        await batchWriter.waitForPendingFlush();
+        await batchWriter.flush();
+    }
+    // Orphan cleanup: remove File nodes that exist in the graph under our scope
+    // but weren't in this run's discovered set. Skipped for changedOnly because
+    // the git-diff path above already handles deletions for the incremental case.
+    if (discoveredSet !== null) {
+        const cleanupRoot = options.specificPath
+            ? resolve(absRoot, options.specificPath)
+            : absRoot;
+        result.orphansRemoved = await cleanOrphanedFiles(db, cleanupRoot, discoveredSet);
+    }
+    if (result.errors.length === 0) {
+        const commitSha = getCurrentCommitSha(absRoot);
+        if (commitSha) {
+            const session = db.session();
+            try {
+                const q = upsertRepositoryWithCommit({
+                    path: absRoot,
+                    name: absRoot.split("/").pop() || absRoot,
+                    lastIndexedCommit: commitSha,
+                });
+                await session.run(q.cypher, q.params);
+            }
+            finally {
+                await session.close();
+            }
+        }
+    }
+    return result;
+}
+//# sourceMappingURL=index.js.map

package/dist/indexer/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/indexer/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAGtC,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AACpE,OAAO,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AACtD,OAAO,EAAsB,aAAa,EAAE,MAAM,mBAAmB,CAAC;AACtE,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EACL,mBAAmB,EACnB,0BAA0B,EAC1B,0BAA0B,EAC1B,0BAA0B,EAC1B,sBAAsB,GACvB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,eAAe,EAAE,YAAY,EAAE,WAAW,EAAE,0BAA0B,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAC7H,OAAO,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAE7D;;;GAGG;AACH,SAAS,WAAW,CAAC,OAAe;IAClC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,YAAY,CACzB,KAAK,EACL,CAAC,UAAU,EAAE,UAAU,EAAE,UAAU,EAAE,oBAAoB,EAAE,IAAI,CAAC,EAChE,EAAE,GAAG,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CACrE,CAAC;QACF,OAAO,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;IAC5E,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,QAAgB,EAChB,MAAmB;IAEnB,MAAM,OAAO,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;IAClC,MAAM,eAAe,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAC/C,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CACnC,CAAC;IAEF,wEAAwE;IACxE,yEAAyE;IACzE,MAAM,QAAQ,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;IACtC,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;QACtB,OAAO,QAAQ,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE;YACjC,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YACvC,MAAM,cAAc,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;YACpF,MAAM,cAAc,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;YACrF,OAAO,cAAc,IAAI,CAAC,cAAc,CAAC;QAC3C,CAAC,CAAC,CAAC;IACL,CAAC;IAED,2EAA2E;IAC3E,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACrC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE;YAClC,GAAG,EAAE,OAAO;YACZ,QAAQ,EAAE,IAAI;YACd,MAAM,EAAE,eAAe;YACvB,KAAK,EAAE,IAAI;SACZ,CAAC,CAAC;QACH,QAAQ,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC;IAC5B,CAAC;IACD,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC;AAChC,CAAC;AAUD;;;;;;;;;;;;;GAaG;AACH,MAAM,wBAAwB,GAAG,GAAG,CAAC;AAErC,KAAK,UAAU,kBAAkB,CAC/B,EAAgB,EAChB,UAAkB,EAClB,eAA4B;IAE5B,MAAM,OAAO,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC;IAC7B,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,0BAA0B,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC;QACzD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;QAC7D,MAAM,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAW,CAAC,CAAC;QAExE,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACpE,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAEnC,IAAI,eAAe,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YAC/B,OAAO,CAAC,IAAI,CACV,oEAAoE,YAAY,CAAC,MAAM,sBAAsB,UAAU,IAAI;gBACzH,uEAAuE;gBACvE,mDAAmD,CACtD,CAAC;YACF,OAAO,CAAC,CAAC;QACX,CAAC;QAED,wEAAwE;QACxE,6EAA6E;QAC7E,0EAA0E;QAC1E,0EAA0E;QAC1E,sEAAsE;QACtE,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,wBAAwB,EAAE,CAAC;YAClE,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,wBAAwB,CAAC,CAAC;YAC7D,MAAM,OAAO,GAAG,sBAAsB,CAAC,KAAK,CAAC,CAAC;YAC9C,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;YAClD,OAAO,IAAI,KAAK,CAAC,MAAM,CAAC;QAC1B,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;YAAS,CAAC;QACT,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,EAAgB,EAChB,QAAgB,EAChB,MAAmB,EACnB,UAOI,EAAE;IAEN,MAAM,OAAO,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;IAClC,MAAM,UAAU,EAAE,CAAC;IAEnB,IAAI,KAAe,CAAC;IACpB,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;QACzB,KAAK,GAAG,MAAM,aAAa,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC,CAAC;IAC9E,CAAC;SAAM,CAAC;QACN,KAAK,GAAG,MAAM,aAAa,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAC/C,CAAC;IAED,gCAAgC;IAChC,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;IAExD,oEAAoE;IACpE,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;IAEnC,yDAAyD;IACzD,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;QACxB,kCAAkC;QAClC,MAAM,OAAO,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC;QAC7B,IAAI,YAAY,GAAG,KAAK,CAAC;QACzB,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,mBAAmB,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;YACvD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;YACjE,MAAM,UAAU,GAAkB,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,mBAAmB,CAAC,IAAI,IAAI,CAAC;YAEtF,IAAI,UAAU,EAAE,CAAC;gBACf,MAAM,IAAI,GAAG,0BAA0B,CAAC,OAAO,EAAE,UAAU,EAAE,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC;gBAEvF,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;oBACf,6EAA6E;gBAC/E,CAAC;qBAAM,CAAC;oBACN,sCAAsC;oBACtC,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;wBACnC,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;wBAC1C,MAAM,OAAO,GAAG,0BAA0B,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;wBAClE,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAE,uBAAuB;oBAC7E,CAAC;oBAED,oCAAoC;oBACpC,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC9E,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;oBACpD,YAAY,GAAG,IAAI,CAAC;gBACtB,CAAC;YACH,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;QACxB,CAAC;QAED,mEAAmE;QACnE,IAAI,CAAC,YAAY,EAAE,CAAC;YAClB,MAAM,QAAQ,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC;YAC9B,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,GAAG,CAC/B,yHAAyH,EACzH,EAAE,QAAQ,EAAE,OAAO,EAAE,CACtB,CAAC;gBACF,MAAM,OAAO,GAAG,IAAI,GAAG,CACrB,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;oBACxB,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC;oBACb,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,YAAY,EAAE,CAAC,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE;iBAC7D,CAAC,CACH,CAAC;gBACF,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;oBACzB,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;oBAChC,IAAI,CAAC,QAAQ;wBAAE,OAAO,IAAI,CAAC;oBAC3B,OAAO,WAAW,CAAC,CAAC,EAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,YAAY,CAAC,CAAC;gBAC9D,CAAC,CAAC,CAAC;YACL,CAAC;oBAAS,CAAC;gBACT,MAAM,QAAQ,CAAC,KAAK,EAAE,CAAC;YACzB,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAgB;QAC1B,YAAY,EAAE,CAAC;QACf,cAAc,EAAE,CAAC;QACjB,YAAY,EAAE,CAAC;QACf,cAAc,EAAE,CAAC;QACjB,MAAM,EAAE,EAAE;KACX,CAAC;IAEF,4EAA4E;IAC5E,2EAA2E;IAC3E,2DAA2D;IAC3D,MAAM,aAAa,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;IAElE,MAAM,aAAa,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;IAEjC,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,CAAC,CAAC;IAC7C,MAAM,cAAc,GAAG,CAAC,OAAO,CAAC,WAAW,IAAI,IAAI,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;IAEnE,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;QACpB,MAAM,WAAW,GAAG,IAAI,gBAAgB,CAAC,EAAE,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC;QAC9D,MAAM,cAAc,GAAG,MAAM,mBAAmB,CAAC;YAC/C,KAAK;YACL,OAAO;YACP,WAAW;YACX,cAAc;YACd,OAAO,EAAE,SAAS;YAClB,SAAS,EAAE,oBAAoB;YAC/B,WAAW;YACX,aAAa,EAAE,eAAe;YAC9B,UAAU,EAAE,YAAY;YACxB,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,eAAe,EAAE,OAAO,CAAC,eAAe;SACzC,CAAC,CAAC;QACH,MAAM,CAAC,YAAY,GAAG,cAAc,CAAC,YAAY,CAAC;QAClD,MAAM,CAAC,cAAc,GAAG,cAAc,CAAC,cAAc,CAAC;QACtD,MAAM,CAAC,YAAY,GAAG,cAAc,CAAC,YAAY,CAAC;QAClD,MAAM,CAAC,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC;IACxC,CAAC;SAAM,CAAC;QACN,sBAAsB;QACtB,MAAM,WAAW,GAAG,IAAI,gBAAgB,CAAC,EAAE,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC;QAC9D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACtB,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;YAChD,IAAI,CAAC;gBACH,MAAM,WAAW,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,CAAC;gBAC1C,IAAI,CAAC,WAAW;oBAAE,SAAS;gBAC3B,IAAI,QAAiD,CAAC;gBACtD,IAAI,CAAC;oBACH,QAAQ,GAAG,oBAAoB,CAAC,WAAW,CAAC,IAAI,EAAE,WAAW,CAAC,QAAQ,EAAE,WAAW,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;gBACpG,CAAC;wBAAS,CAAC;oBACT,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;gBAC5B,CAAC;gBACD,WAAW,CAAC,GAAG,CAAC,QAAQ,EAAE;oBACxB,QAAQ,EAAE,IAAI,EAAE,YAAY,EAAE,QAAQ,CAAC,OAAO,EAAE,IAAI,CAAC,EAAE,QAAQ,EAAE,OAAO;oBACxE,QAAQ,EAAE,WAAW,CAAC,QAAQ;oBAC9B,IAAI,EAAE,eAAe,CAAC,IAAI,EAAE,WAAW,CAAC,MAAM,CAAC;oBAC/C,YAAY,EAAE,YAAY,CAAC,IAAI,CAAC;iBACjC,CAAC,CAAC;gBACH,MAAM,CAAC,YAAY,EAAE,CAAC;gBACtB,MAAM,CAAC,cAAc,IAAI,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC;gBACnD,MAAM,CAAC,YAAY,IAAI,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC;YACjD,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAC9F,CAAC;QACH,CAAC;QACD,MAAM,WAAW,CAAC,mBAAmB,EAAE,CAAC;QACxC,MAAM,WAAW,CAAC,KAAK,EAAE,CAAC;IAC5B,CAAC;IAED,4EAA4E;IAC5E,4EAA4E;IAC5E,8EAA8E;IAC9E,IAAI,aAAa,KAAK,IAAI,EAAE,CAAC;QAC3B,MAAM,WAAW,GAAG,OAAO,CAAC,YAAY;YACtC,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,YAAY,CAAC;YACxC,CAAC,CAAC,OAAO,CAAC;QACZ,MAAM,CAAC,cAAc,GAAG,MAAM,kBAAkB,CAAC,EAAE,EAAE,WAAW,EAAE,aAAa,CAAC,CAAC;IACnF,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/B,MAAM,SAAS,GAAG,mBAAmB,CAAC,OAAO,CAAC,CAAC;QAC/C,IAAI,SAAS,EAAE,CAAC;YACd,MAAM,OAAO,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC;YAC7B,IAAI,CAAC;gBACH,MAAM,CAAC,GAAG,0BAA0B,CAAC;oBACnC,IAAI,EAAE,OAAO;oBACb,IAAI,EAAE,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,OAAO;oBACzC,iBAAiB,EAAE,SAAS;iBAC7B,CAAC,CAAC;gBACH,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC;YACxC,CAAC;oBAAS,CAAC;gBACT,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;YACxB,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/indexer/language-map.json

@@ -0,0 +1,101 @@
+{
+  "typescript": {
+    "function": ["function_declaration", "arrow_function", "method_definition"],
+    "class": ["class_declaration"],
+    "import": ["import_statement"],
+    "call": ["call_expression"],
+    "name_field": "name",
+    "body_field": "body",
+    "parameters_field": "parameters"
+  },
+  "tsx": {
+    "function": ["function_declaration", "arrow_function", "method_definition"],
+    "class": ["class_declaration"],
+    "import": ["import_statement"],
+    "call": ["call_expression"],
+    "name_field": "name",
+    "body_field": "body",
+    "parameters_field": "parameters"
+  },
+  "javascript": {
+    "function": ["function_declaration", "arrow_function", "method_definition"],
+    "class": ["class_declaration"],
+    "import": ["import_statement"],
+    "call": ["call_expression"],
+    "name_field": "name",
+    "body_field": "body",
+    "parameters_field": "parameters"
+  },
+  "python": {
+    "function": ["function_definition"],
+    "class": ["class_definition"],
+    "import": ["import_from_statement", "import_statement"],
+    "call": ["call"],
+    "name_field": "name",
+    "body_field": "body",
+    "parameters_field": "parameters"
+  },
+  "go": {
+    "function": ["function_declaration", "method_declaration"],
+    "class": ["type_declaration"],
+    "import": ["import_declaration", "import_spec"],
+    "call": ["call_expression"],
+    "name_field": "name",
+    "body_field": "body",
+    "parameters_field": "parameters"
+  },
+  "java": {
+    "function": ["method_declaration", "constructor_declaration"],
+    "class": ["class_declaration", "interface_declaration", "enum_declaration"],
+    "import": ["import_declaration"],
+    "call": ["method_invocation"],
+    "name_field": "name",
+    "body_field": "body",
+    "parameters_field": "parameters"
+  },
+  "rust": {
+    "function": ["function_item"],
+    "class": ["struct_item", "enum_item", "trait_item", "impl_item"],
+    "import": ["use_declaration"],
+    "call": ["call_expression"],
+    "name_field": "name",
+    "body_field": "body",
+    "parameters_field": "parameters"
+  },
+  "c": {
+    "function": ["function_definition"],
+    "class": ["struct_specifier"],
+    "import": ["preproc_include"],
+    "call": ["call_expression"],
+    "name_field": "declarator",
+    "body_field": "body",
+    "parameters_field": "parameters"
+  },
+  "cpp": {
+    "function": ["function_definition"],
+    "class": ["class_specifier", "struct_specifier"],
+    "import": ["preproc_include"],
+    "call": ["call_expression"],
+    "name_field": "declarator",
+    "body_field": "body",
+    "parameters_field": "parameters"
+  },
+  "ruby": {
+    "function": ["method", "singleton_method"],
+    "class": ["class", "module"],
+    "import": ["call"],
+    "call": ["call", "method_call"],
+    "name_field": "name",
+    "body_field": "body",
+    "parameters_field": "parameters"
+  },
+  "graphql": {
+    "function": ["operation_definition", "fragment_definition"],
+    "class": ["object_type_definition", "input_object_type_definition", "interface_type_definition", "enum_type_definition"],
+    "import": [],
+    "call": [],
+    "name_field": "name",
+    "body_field": "body",
+    "parameters_field": "arguments_definition"
+  }
+}

package/dist/indexer/parallel-pipeline.d.ts

@@ -0,0 +1,41 @@
+import type { BatchGraphWriter } from "./batch-writer.js";
+export declare class Semaphore {
+    private readonly maxConcurrency;
+    private queue;
+    private running;
+    constructor(maxConcurrency: number);
+    acquire(): Promise<() => void>;
+    private release;
+}
+export interface PipelineOptions {
+    files: string[];
+    absRoot: string;
+    concurrency: number;
+    maxMemoryBytes: number;
+    parseFn: (filePath: string) => Promise<{
+        tree: any;
+        language: string;
+        source: string;
+    } | null>;
+    extractFn: (tree: any, language: string, source: string, filePath: string) => {
+        functions: any[];
+        classes: any[];
+        imports: any[];
+        calls: any[];
+    };
+    batchWriter: BatchGraphWriter;
+    computeHashFn: (filePath: string, content?: string) => string;
+    getMtimeFn: (filePath: string) => number;
+    onProgress?: (current: number, total: number, file: string) => void;
+    onFlushProgress?: (completed: number, total: number) => void;
+}
+export interface PipelineResult {
+    filesIndexed: number;
+    functionsFound: number;
+    classesFound: number;
+    errors: Array<{
+        file: string;
+        error: string;
+    }>;
+}
+export declare function runParallelPipeline(options: PipelineOptions): Promise<PipelineResult>;

package/dist/indexer/parallel-pipeline.js

@@ -0,0 +1,82 @@
+import { relative } from "node:path";
+export class Semaphore {
+    maxConcurrency;
+    queue = [];
+    running = 0;
+    constructor(maxConcurrency) {
+        this.maxConcurrency = maxConcurrency;
+    }
+    async acquire() {
+        if (this.running < this.maxConcurrency) {
+            this.running++;
+            return () => this.release();
+        }
+        return new Promise((resolve) => {
+            this.queue.push(() => {
+                this.running++;
+                resolve(() => this.release());
+            });
+        });
+    }
+    release() {
+        this.running--;
+        const next = this.queue.shift();
+        if (next)
+            next();
+    }
+}
+export async function runParallelPipeline(options) {
+    const { files, absRoot, concurrency, maxMemoryBytes, parseFn, extractFn, batchWriter, computeHashFn, getMtimeFn, onProgress, onFlushProgress } = options;
+    if (onFlushProgress)
+        batchWriter.onFlushProgress = onFlushProgress;
+    const sem = new Semaphore(concurrency);
+    const result = { filesIndexed: 0, functionsFound: 0, classesFound: 0, errors: [] };
+    let progressCounter = 0;
+    const tasks = files.map((file) => async () => {
+        // Backpressure: flush BEFORE acquiring slot — fatal errors propagate out
+        while (batchWriter.estimatedMemoryBytes >= maxMemoryBytes) {
+            await batchWriter.flush();
+        }
+        const release = await sem.acquire();
+        try {
+            const parseResult = await parseFn(file);
+            if (parseResult) {
+                let entities;
+                try {
+                    entities = extractFn(parseResult.tree, parseResult.language, parseResult.source, file);
+                }
+                finally {
+                    parseResult.tree.delete();
+                }
+                batchWriter.add(entities, {
+                    filePath: file,
+                    relativePath: relative(absRoot, file),
+                    repoPath: absRoot,
+                    language: parseResult.language,
+                    hash: computeHashFn(file, parseResult.source),
+                    lastModified: getMtimeFn(file),
+                });
+                // Backpressure: flush if memory limit exceeded after add
+                if (batchWriter.estimatedMemoryBytes >= maxMemoryBytes) {
+                    await batchWriter.flush();
+                }
+                result.filesIndexed++;
+                result.functionsFound += entities.functions.length;
+                result.classesFound += entities.classes.length;
+            }
+        }
+        catch (error) {
+            result.errors.push({ file, error: error instanceof Error ? error.message : String(error) });
+        }
+        finally {
+            progressCounter++;
+            onProgress?.(progressCounter, files.length, file);
+            release();
+        }
+    });
+    await Promise.all(tasks.map((t) => t()));
+    await batchWriter.waitForPendingFlush();
+    await batchWriter.flush();
+    return result;
+}
+//# sourceMappingURL=parallel-pipeline.js.map

package/dist/indexer/parallel-pipeline.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"parallel-pipeline.js","sourceRoot":"","sources":["../../src/indexer/parallel-pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAGrC,MAAM,OAAO,SAAS;IAIS;IAHrB,KAAK,GAAsB,EAAE,CAAC;IAC9B,OAAO,GAAG,CAAC,CAAC;IAEpB,YAA6B,cAAsB;QAAtB,mBAAc,GAAd,cAAc,CAAQ;IAAG,CAAC;IAEvD,KAAK,CAAC,OAAO;QACX,IAAI,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;YACvC,IAAI,CAAC,OAAO,EAAE,CAAC;YACf,OAAO,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;QAC9B,CAAC;QACD,OAAO,IAAI,OAAO,CAAa,CAAC,OAAO,EAAE,EAAE;YACzC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE;gBACnB,IAAI,CAAC,OAAO,EAAE,CAAC;gBACf,OAAO,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;YAChC,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,OAAO;QACb,IAAI,CAAC,OAAO,EAAE,CAAC;QACf,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QAChC,IAAI,IAAI;YAAE,IAAI,EAAE,CAAC;IACnB,CAAC;CACF;AAuBD,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,OAAwB;IAChE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,aAAa,EAAE,UAAU,EAAE,UAAU,EAAE,eAAe,EAAE,GAAG,OAAO,CAAC;IACzJ,IAAI,eAAe;QAAE,WAAW,CAAC,eAAe,GAAG,eAAe,CAAC;IAEnE,MAAM,GAAG,GAAG,IAAI,SAAS,CAAC,WAAW,CAAC,CAAC;IACvC,MAAM,MAAM,GAAmB,EAAE,YAAY,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;IACnG,IAAI,eAAe,GAAG,CAAC,CAAC;IAExB,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,IAAI,EAAE;QAC3C,yEAAyE;QACzE,OAAO,WAAW,CAAC,oBAAoB,IAAI,cAAc,EAAE,CAAC;YAC1D,MAAM,WAAW,CAAC,KAAK,EAAE,CAAC;QAC5B,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,GAAG,CAAC,OAAO,EAAE,CAAC;QACpC,IAAI,CAAC;YACH,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;YAExC,IAAI,WAAW,EAAE,CAAC;gBAChB,IAAI,QAAsC,CAAC;gBAC3C,IAAI,CAAC;oBACH,QAAQ,GAAG,SAAS,CAAC,WAAW,CAAC,IAAI,EAAE,WAAW,CAAC,QAAQ,EAAE,WAAW,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;gBACzF,CAAC;wBAAS,CAAC;oBACT,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;gBAC5B,CAAC;gBAED,WAAW,CAAC,GAAG,CAAC,QAAgB,EAAE;oBAChC,QAAQ,EAAE,IAAI;oBACd,YAAY,EAAE,QAAQ,CAAC,OAAO,EAAE,IAAI,CAAC;oBACrC,QAAQ,EAAE,OAAO;oBACjB,QAAQ,EAAE,WAAW,CAAC,QAAQ;oBAC9B,IAAI,EAAE,aAAa,CAAC,IAAI,EAAE,WAAW,CAAC,MAAM,CAAC;oBAC7C,YAAY,EAAE,UAAU,CAAC,IAAI,CAAC;iBAC/B,CAAC,CAAC;gBAEH,yDAAyD;gBACzD,IAAI,WAAW,CAAC,oBAAoB,IAAI,cAAc,EAAE,CAAC;oBACvD,MAAM,WAAW,CAAC,KAAK,EAAE,CAAC;gBAC5B,CAAC;gBAED,MAAM,CAAC,YAAY,EAAE,CAAC;gBACtB,MAAM,CAAC,cAAc,IAAI,QAAS,CAAC,SAAS,CAAC,MAAM,CAAC;gBACpD,MAAM,CAAC,YAAY,IAAI,QAAS,CAAC,OAAO,CAAC,MAAM,CAAC;YAClD,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC9F,CAAC;gBAAS,CAAC;YACT,eAAe,EAAE,CAAC;YAClB,UAAU,EAAE,CAAC,eAAe,EAAE,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;YAClD,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACzC,MAAM,WAAW,CAAC,mBAAmB,EAAE,CAAC;IACxC,MAAM,WAAW,CAAC,KAAK,EAAE,CAAC;IAE1B,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/indexer/parser.d.ts

@@ -0,0 +1,9 @@
+import Parser from "web-tree-sitter";
+export declare function detectLanguage(filePath: string): string | null;
+export declare function initParser(): Promise<void>;
+export interface ParseResult {
+    tree: Parser.Tree;
+    language: string;
+    source: string;
+}
+export declare function parseFile(filePath: string): Promise<ParseResult | null>;