ragcode-context-engine 0.1.0

package/dist/src/core/engine.js

@@ -0,0 +1,442 @@
+import path from "node:path";
+import { ContextBuilder } from "../context/context-builder.js";
+import { createGraphRuntimeFromEnv } from "../config/graph-runtime.js";
+import { createSemanticRuntimeFromEnv } from "../config/semantic-runtime.js";
+import { InMemoryGraphStore } from "../graph/in-memory-graph-store.js";
+import { RepoIndexer } from "../indexing/indexer.js";
+import { scanRepo } from "../indexing/scanner.js";
+import { ProjectRegistry } from "../project/project-registry.js";
+import { WorkspaceResolver } from "../project/workspace-resolver.js";
+import { hasSemanticParticipation, HybridRetriever } from "../retrieval/hybrid-retriever.js";
+import { DeterministicEmbeddingProvider } from "../semantic/deterministic-embedding.js";
+import { InMemorySemanticStore } from "../semantic/in-memory-semantic-store.js";
+import { SubgraphBuilder } from "../subgraph/subgraph-builder.js";
+import { normalizeUserPath } from "../utils/path.js";
+import { buildReuseCandidateReport } from "../reuse/reuse-detector.js";
+export class RagCodeEngine {
+    graphStore;
+    semanticStore;
+    embeddingProvider;
+    projectRegistry = new ProjectRegistry();
+    workspaceResolver;
+    contextBuilder = new ContextBuilder();
+    indexedAtByRepo = new Map();
+    cwd;
+    workspaceRoots;
+    hydratedRoots = new Set();
+    hydratedAllProjects = false;
+    constructor(options = {}) {
+        const env = options.env ?? process.env;
+        this.cwd = path.resolve(options.cwd ?? process.cwd());
+        this.workspaceRoots = (options.workspaceRoots ?? []).map((root) => path.resolve(root));
+        const graphRuntime = options.graphStore ? undefined : createGraphRuntimeFromEnv(env, this.cwd);
+        this.graphStore = options.graphStore ?? graphRuntime?.graphStore ?? new InMemoryGraphStore();
+        const semanticRuntime = (options.semanticStore && options.embeddingProvider)
+            ? undefined
+            : createSemanticRuntimeFromEnv(env, this.cwd);
+        this.semanticStore = options.semanticStore ?? semanticRuntime?.semanticStore ?? new InMemorySemanticStore();
+        this.embeddingProvider = options.embeddingProvider ?? semanticRuntime?.embeddingProvider ?? new DeterministicEmbeddingProvider();
+        this.workspaceResolver = new WorkspaceResolver(this.projectRegistry, {
+            cwd: this.cwd,
+            roots: this.workspaceRoots
+        });
+    }
+    close() {
+        this.graphStore.close?.();
+    }
+    async indexRepo(repoRoot, options) {
+        const project = await this.projectRegistry.register(repoRoot);
+        this.workspaceResolver.setActive(project, "repoRoot");
+        const absoluteRoot = path.resolve(repoRoot);
+        this.indexedAtByRepo.set(absoluteRoot, Date.now());
+        const index = await new RepoIndexer({
+            graphStore: this.graphStore,
+            semanticStore: this.semanticStore,
+            embeddingProvider: this.embeddingProvider
+        }).indexRepo(absoluteRoot, project.projectId, project, options);
+        const indexedProject = this.projectRegistry.upsert({
+            ...project,
+            lastIndexedAtMs: index.indexedAtMs
+        });
+        this.indexedAtByRepo.set(indexedProject.repoRoot, index.indexedAtMs);
+        this.indexedAtByRepo.set(indexedProject.canonicalRoot, index.indexedAtMs);
+        return {
+            ...index,
+            project: indexedProject
+        };
+    }
+    async refreshIndex(repoRoot, options) {
+        const scope = await this.resolveWorkspace({ repoRoot });
+        return this.indexRepo(scope.activeRepoRoot, options);
+    }
+    async indexStatus(repoRoot) {
+        const scope = await this.resolveWorkspace({ repoRoot });
+        const [files, chunks, symbols, edges, freshness] = await Promise.all([
+            this.graphStore.getFiles(scope.activeRepoRoot),
+            this.graphStore.getChunks(scope.activeRepoRoot),
+            this.graphStore.getSymbols(scope.activeRepoRoot),
+            this.graphStore.getEdges(scope.activeRepoRoot),
+            this.computeFreshness(scope)
+        ]);
+        const stale = new Set(freshness.staleFiles);
+        return {
+            repoRoot: scope.activeRepoRoot,
+            projectId: scope.activeProjectId,
+            indexedAtMs: freshness.indexedAtMs,
+            fileCount: files.length,
+            chunkCount: chunks.length,
+            symbolCount: symbols.length,
+            edgeCount: edges.length,
+            freshFileCount: files.filter((file) => !stale.has(file.path)).length,
+            staleFileCount: freshness.staleFiles.length,
+            pendingFileCount: freshness.pendingFiles.length,
+            indexingFileCount: freshness.indexingFiles.length,
+            skippedFileCount: freshness.skippedFiles.length,
+            burstMode: freshness.burstMode,
+            droppedEventCount: freshness.droppedEvents,
+            freshness
+        };
+    }
+    async recordFileEvents(repoRoot, filePaths, options) {
+        const scope = await this.resolveWorkspace({ repoRoot });
+        if (!this.graphStore.recordFileEvents) {
+            throw new Error("Current graph store does not support watcher dirty-file state.");
+        }
+        return this.graphStore.recordFileEvents(scope.activeRepoRoot, filePaths, options);
+    }
+    async markDirtyFilesIndexing(repoRoot, filePaths) {
+        const scope = await this.resolveWorkspace({ repoRoot });
+        if (!this.graphStore.markDirtyFilesIndexing) {
+            throw new Error("Current graph store does not support watcher indexing state.");
+        }
+        return this.graphStore.markDirtyFilesIndexing(scope.activeRepoRoot, filePaths);
+    }
+    async markDirtyFilesDeadLetter(repoRoot, filePaths, reason) {
+        const scope = await this.resolveWorkspace({ repoRoot });
+        if (!this.graphStore.markDirtyFilesDeadLetter) {
+            throw new Error("Current graph store does not support watcher dead-letter state.");
+        }
+        return this.graphStore.markDirtyFilesDeadLetter(scope.activeRepoRoot, filePaths, reason);
+    }
+    async searchCode(query) {
+        const scope = await this.resolveWorkspace(query);
+        const { hits } = await this.searchWithFreshness({ ...query, repoRoot: scope.activeRepoRoot, projectId: scope.activeProjectId }, scope);
+        return hits;
+    }
+    async searchCodeWithDiagnostics(query) {
+        const scope = await this.resolveWorkspace(query);
+        const { hits, diagnostics } = await this.searchWithFreshness({ ...query, repoRoot: scope.activeRepoRoot, projectId: scope.activeProjectId }, scope);
+        return { hits, diagnostics };
+    }
+    async getContext(request) {
+        const scope = await this.resolveWorkspace(request);
+        const normalized = { ...request, repoRoot: scope.activeRepoRoot, projectId: scope.activeProjectId };
+        const { hits, freshness } = await this.searchWithFreshness(normalized, scope);
+        const edges = filterFreshEdges(await this.graphStore.getEdges(normalized.repoRoot), freshness);
+        return this.contextBuilder.build(normalized, hits, edges, {
+            projectId: scope.activeProjectId,
+            repoRoot: scope.activeRepoRoot,
+            indexedAtMs: this.indexedAtByRepo.get(scope.activeRepoRoot) ?? Date.now(),
+            indexGeneration: freshness.indexGeneration,
+            staleFiles: freshness.staleFiles,
+            pendingFiles: freshness.pendingFiles,
+            indexingFiles: freshness.indexingFiles,
+            skippedFiles: freshness.skippedFiles,
+            dirtyFiles: freshness.dirtyFiles,
+            burstMode: freshness.burstMode,
+            droppedEvents: freshness.droppedEvents
+        });
+    }
+    async verifiedSubgraph(request) {
+        const scope = await this.resolveWorkspace(request);
+        const freshness = await this.computeFreshness(scope);
+        const [symbols, edges, chunks] = await Promise.all([
+            this.graphStore.getSymbols(scope.activeRepoRoot),
+            this.graphStore.getEdges(scope.activeRepoRoot),
+            this.graphStore.getChunks(scope.activeRepoRoot)
+        ]);
+        const freshSymbols = filterFreshSymbols(symbols, freshness);
+        const freshChunks = filterFreshChunks(chunks, freshness);
+        const freshEdges = filterFreshEdges(edges, freshness);
+        const seed = request.seed ?? request.query;
+        const seedSymbols = selectSeedSymbols(freshSymbols, seed);
+        const missingEvidence = freshness.staleFiles.length > 0
+            ? [`Stale indexed files excluded: ${freshness.staleFiles.slice(0, 8).join(", ")}.`]
+            : [];
+        return new SubgraphBuilder().build({
+            query: request.query,
+            repoRoot: scope.activeRepoRoot,
+            projectId: scope.activeProjectId,
+            mode: request.mode ?? "impact",
+            seedSymbols,
+            symbols: freshSymbols,
+            edges: freshEdges,
+            chunks: freshChunks,
+            budgetChars: request.budgetChars,
+            maxHops: request.maxHops,
+            missingEvidence
+        });
+    }
+    async topologyMap(request) {
+        const pack = await this.getContext({
+            ...request,
+            mode: request.mode ?? "feature",
+            budgetChars: request.budgetChars ?? 12_000
+        });
+        return {
+            query: pack.query,
+            repoRoot: pack.repoRoot,
+            projectId: pack.projectId,
+            freshness: pack.freshness,
+            owners: pack.ownerChain,
+            edges: pack.topology.slice(0, request.maxEdges ?? 24),
+            missingEvidence: pack.missingEvidence,
+            nextQueries: pack.nextQueries
+        };
+    }
+    async findSymbol(repoRoot, name) {
+        const scope = await this.resolveWorkspace({ repoRoot });
+        return this.graphStore.findSymbol(scope.activeRepoRoot, name);
+    }
+    async graphSnapshot(repoRoot) {
+        const scope = await this.resolveWorkspace({ repoRoot });
+        const [symbols, edges] = await Promise.all([
+            this.graphStore.getSymbols(scope.activeRepoRoot),
+            this.graphStore.getEdges(scope.activeRepoRoot)
+        ]);
+        return { symbols, edges };
+    }
+    async explainFile(repoRoot, filePath) {
+        const scope = await this.resolveWorkspace({ repoRoot, workspace: path.isAbsolute(filePath) ? { filePath } : undefined });
+        return this.graphStore.explainFile(scope.activeRepoRoot, filePath);
+    }
+    async findOwner(repoRoot, query, limit) {
+        const scope = await this.resolveWorkspace({ repoRoot });
+        return this.graphStore.findOwner(scope.activeRepoRoot, query, limit);
+    }
+    async findReuseCandidates(request) {
+        const scope = await this.resolveWorkspace(request);
+        const normalized = { ...request, repoRoot: scope.activeRepoRoot, projectId: scope.activeProjectId };
+        const { hits, freshness } = await this.searchWithFreshness(normalized, scope);
+        const [owners, symbols, edges, chunks] = await Promise.all([
+            this.graphStore.findOwner(scope.activeRepoRoot, request.query, request.limit ?? 8),
+            this.graphStore.getSymbols(scope.activeRepoRoot),
+            this.graphStore.getEdges(scope.activeRepoRoot),
+            this.graphStore.getChunks(scope.activeRepoRoot)
+        ]);
+        return buildReuseCandidateReport({
+            query: request.query,
+            hits,
+            owners: owners.filter((owner) => !freshness.staleFiles.includes(owner.filePath)),
+            symbols: filterFreshSymbols(symbols, freshness),
+            edges: filterFreshEdges(edges, freshness),
+            chunks: filterFreshChunks(chunks, freshness),
+            limit: request.limit,
+            reuseGuard: request.reuseGuard
+        });
+    }
+    async impactAnalysis(repoRoot, target) {
+        const scope = await this.resolveWorkspace({ repoRoot });
+        return this.graphStore.impactAnalysis(scope.activeRepoRoot, target);
+    }
+    async relatedTests(repoRoot, target) {
+        const scope = await this.resolveWorkspace({ repoRoot });
+        return this.graphStore.relatedTests(scope.activeRepoRoot, target);
+    }
+    async traceFlow(repoRoot, entry, maxSteps) {
+        const scope = await this.resolveWorkspace({ repoRoot });
+        return this.graphStore.traceFlow(scope.activeRepoRoot, entry, maxSteps);
+    }
+    async reviewDiff(repoRoot, diff, changedFiles) {
+        const scope = await this.resolveWorkspace({ repoRoot });
+        return this.graphStore.reviewDiff(scope.activeRepoRoot, diff, changedFiles);
+    }
+    async resolveWorkspace(input = {}) {
+        await this.hydrateForInput(input);
+        return this.workspaceResolver.resolve(input);
+    }
+    async hydrateForInput(input) {
+        if (input.repoRoot)
+            await this.hydrateProjectByRoot(input.repoRoot);
+        if (input.workspace?.root)
+            await this.hydrateProjectByRoot(input.workspace.root);
+        if (input.workspace?.filePath)
+            await this.hydrateAllPersistedProjects();
+        if (!input.repoRoot && !input.workspace?.root && !input.workspace?.filePath) {
+            for (const root of this.workspaceRoots)
+                await this.hydrateProjectByRoot(root);
+            await this.hydrateProjectByRoot(this.cwd);
+            await this.hydrateAllPersistedProjects();
+        }
+    }
+    async hydrateProjectByRoot(root) {
+        if (this.projectRegistry.findByRoot(root))
+            return;
+        const normalized = path.resolve(root).toLowerCase();
+        if (this.hydratedRoots.has(normalized))
+            return;
+        this.hydratedRoots.add(normalized);
+        const project = await this.graphStore.getProjectByRoot?.(root);
+        if (project)
+            this.rememberHydratedProject(project);
+    }
+    async hydrateAllPersistedProjects() {
+        if (this.hydratedAllProjects)
+            return;
+        this.hydratedAllProjects = true;
+        const projects = await this.graphStore.listProjects?.();
+        for (const project of projects ?? [])
+            this.rememberHydratedProject(project);
+    }
+    rememberHydratedProject(project) {
+        const merged = this.projectRegistry.upsert(project);
+        if (merged.lastIndexedAtMs) {
+            this.indexedAtByRepo.set(merged.repoRoot, merged.lastIndexedAtMs);
+            this.indexedAtByRepo.set(merged.canonicalRoot, merged.lastIndexedAtMs);
+        }
+    }
+    async searchWithFreshness(query, scope) {
+        const freshness = await this.computeFreshness(scope);
+        const result = await new HybridRetriever({
+            graphStore: this.graphStore,
+            semanticStore: this.semanticStore,
+            embeddingProvider: this.embeddingProvider
+        }).searchWithDiagnostics(query);
+        const hits = filterFreshHits(result.hits, freshness);
+        return { hits, freshness, diagnostics: diagnosticsForFreshHits(result.diagnostics, hits) };
+    }
+    async computeFreshness(scope) {
+        const watcherStatePromise = this.graphStore.getWatcherState
+            ? this.graphStore.getWatcherState(scope.activeRepoRoot).catch(() => undefined)
+            : Promise.resolve(undefined);
+        const [indexedFiles, scan, watcherState] = await Promise.all([
+            this.graphStore.getFiles(scope.activeRepoRoot),
+            scanRepo(scope.activeRepoRoot, scope.activeProjectId),
+            watcherStatePromise
+        ]);
+        const indexedByPath = new Map(indexedFiles.map((file) => [file.path, file]));
+        const currentByPath = new Map(scan.files.map((file) => [file.path, file]));
+        const staleFiles = new Set();
+        const pendingFiles = new Set();
+        for (const indexed of indexedFiles) {
+            const current = currentByPath.get(indexed.path);
+            if (!current) {
+                staleFiles.add(indexed.path);
+                continue;
+            }
+            if (current.contentHash !== indexed.contentHash) {
+                staleFiles.add(indexed.path);
+                pendingFiles.add(indexed.path);
+            }
+        }
+        for (const current of scan.files) {
+            const indexed = indexedByPath.get(current.path);
+            if (!indexed || indexed.contentHash !== current.contentHash)
+                pendingFiles.add(current.path);
+        }
+        for (const filePath of watcherState?.pendingFiles ?? []) {
+            pendingFiles.add(filePath);
+            if (indexedByPath.has(filePath))
+                staleFiles.add(filePath);
+        }
+        for (const filePath of watcherState?.indexingFiles ?? []) {
+            if (indexedByPath.has(filePath))
+                staleFiles.add(filePath);
+        }
+        const indexGeneration = this.graphStore.getIndexGeneration
+            ? await this.graphStore.getIndexGeneration(scope.activeRepoRoot).catch(() => 1)
+            : 1;
+        return {
+            projectId: scope.activeProjectId,
+            indexGeneration,
+            indexedAtMs: this.indexedAtByRepo.get(scope.activeRepoRoot) ?? Date.now(),
+            staleFiles: [...staleFiles].sort(),
+            pendingFiles: [...pendingFiles].sort(),
+            indexingFiles: [...new Set(watcherState?.indexingFiles ?? [])].sort(),
+            skippedFiles: scan.skippedFiles,
+            dirtyFiles: watcherState?.dirtyFiles ?? [],
+            burstMode: watcherState?.burstMode ?? false,
+            droppedEvents: watcherState?.droppedEvents ?? 0
+        };
+    }
+}
+function filterFreshHits(hits, freshness) {
+    if (freshness.staleFiles.length === 0)
+        return hits;
+    const stale = new Set(freshness.staleFiles);
+    return hits.filter((hit) => !stale.has(hit.chunk.filePath));
+}
+function diagnosticsForFreshHits(diagnostics, hits) {
+    return {
+        ...diagnostics,
+        fusion: {
+            ...diagnostics.fusion,
+            semanticTopNParticipation: hits.filter(hasSemanticParticipation).length
+        }
+    };
+}
+function filterFreshEdges(edges, freshness) {
+    if (freshness.staleFiles.length === 0)
+        return edges;
+    const stale = new Set(freshness.staleFiles);
+    return edges.filter((edge) => {
+        const sourceFile = edge.metadata?.sourceFile;
+        const targetFile = edge.metadata?.targetFile;
+        return (typeof sourceFile !== "string" || !stale.has(sourceFile))
+            && (typeof targetFile !== "string" || !stale.has(targetFile));
+    });
+}
+function filterFreshSymbols(symbols, freshness) {
+    if (freshness.staleFiles.length === 0)
+        return symbols;
+    const stale = new Set(freshness.staleFiles);
+    return symbols.filter((symbol) => !stale.has(symbol.filePath));
+}
+function filterFreshChunks(chunks, freshness) {
+    if (freshness.staleFiles.length === 0)
+        return chunks;
+    const stale = new Set(freshness.staleFiles);
+    return chunks.filter((chunk) => !stale.has(chunk.filePath));
+}
+function selectSeedSymbols(symbols, seed) {
+    const normalized = normalizeUserPath(seed);
+    const scoped = scopedSeed(seed);
+    const loweredSeed = seed.toLowerCase();
+    const exact = symbols.filter((symbol) => {
+        if (scoped) {
+            return symbol.filePath === scoped.filePath && symbol.name.toLowerCase() === scoped.symbolName.toLowerCase();
+        }
+        return symbol.name.toLowerCase() === loweredSeed || symbol.filePath === normalized;
+    });
+    if (exact.length > 0)
+        return sortSeedSymbols(exact);
+    const partial = symbols.filter((symbol) => {
+        if (scoped) {
+            return symbol.filePath.includes(scoped.filePath) && symbol.name.toLowerCase().includes(scoped.symbolName.toLowerCase());
+        }
+        return symbol.name.toLowerCase().includes(loweredSeed) || symbol.filePath.includes(normalized);
+    });
+    return sortSeedSymbols(partial).slice(0, 8);
+}
+function scopedSeed(seed) {
+    const separator = seed.lastIndexOf(":");
+    if (separator <= 0 || /^[a-zA-Z]:[\\/]/.test(seed))
+        return undefined;
+    const filePath = normalizeUserPath(seed.slice(0, separator));
+    const symbolName = seed.slice(separator + 1).trim();
+    if (!filePath || !symbolName)
+        return undefined;
+    return { filePath, symbolName };
+}
+function sortSeedSymbols(symbols) {
+    return [...symbols].sort((a, b) => seedPriority(a) - seedPriority(b)
+        || a.filePath.localeCompare(b.filePath)
+        || a.startLine - b.startLine);
+}
+function seedPriority(symbol) {
+    if (symbol.kind === "file")
+        return 5;
+    if (symbol.exported)
+        return 0;
+    return 1;
+}