ragcode-context-engine 0.1.0

package/dist/src/indexing/analyzers/rust-treesitter-analyzer.js

@@ -0,0 +1,96 @@
+import Parser from "tree-sitter";
+// @ts-ignore - tree-sitter-rust doesn't have TypeScript definitions
+import Rust from "tree-sitter-rust";
+import { analyzeWithTreeSitter } from "./tree-sitter-base.js";
+let parser = null;
+function getParser() {
+    if (!parser) {
+        parser = new Parser();
+        parser.setLanguage(Rust);
+    }
+    return parser;
+}
+const rustConfig = {
+    symbolPatterns: [
+        {
+            type: "function_item",
+            kind: "function",
+            nameField: "name",
+            exportModifierCheck: (node) => {
+                return node.children.some((child) => child.type === "visibility_modifier" && child.text === "pub");
+            }
+        },
+        {
+            type: "struct_item",
+            kind: "type",
+            nameField: "name",
+            exportModifierCheck: (node) => {
+                return node.children.some((child) => child.type === "visibility_modifier" && child.text === "pub");
+            }
+        },
+        {
+            type: "enum_item",
+            kind: "type",
+            nameField: "name",
+            exportModifierCheck: (node) => {
+                return node.children.some((child) => child.type === "visibility_modifier" && child.text === "pub");
+            }
+        },
+        {
+            type: "trait_item",
+            kind: "type",
+            nameField: "name",
+            exportModifierCheck: (node) => {
+                return node.children.some((child) => child.type === "visibility_modifier" && child.text === "pub");
+            }
+        },
+        {
+            type: "impl_item",
+            kind: "type",
+            nameField: "type",
+            exportModifierCheck: () => false // impl blocks are internal implementation details
+        }
+    ],
+    importPatterns: [
+        {
+            type: "use_declaration",
+            sourceField: "argument",
+            bindingsExtractor: (node) => {
+                const argNode = node.childForFieldName("argument");
+                if (!argNode)
+                    return [];
+                const path = argNode.text;
+                return [{ imported: path, local: path.split("::").pop() ?? path }];
+            }
+        }
+    ],
+    callPatterns: [
+        {
+            type: "call_expression",
+            nameExtractor: (node) => {
+                const functionNode = node.childForFieldName("function");
+                if (!functionNode)
+                    return undefined;
+                if (functionNode.type === "identifier") {
+                    return functionNode.text;
+                }
+                else if (functionNode.type === "field_expression") {
+                    const fieldNode = functionNode.childForFieldName("field");
+                    return fieldNode?.text;
+                }
+                else if (functionNode.type === "scoped_identifier") {
+                    const nameNode = functionNode.childForFieldName("name");
+                    return nameNode?.text;
+                }
+                return undefined;
+            }
+        }
+    ]
+};
+export const rustTreeSitterAnalyzer = {
+    language: "rust",
+    capabilities: ["symbols", "imports", "exports", "calls"],
+    analyzeFile: ({ repoRoot, file, content }) => {
+        return analyzeWithTreeSitter(getParser(), rustConfig, repoRoot, file, content);
+    }
+};

package/dist/src/indexing/analyzers/tree-sitter-base.d.ts

@@ -0,0 +1,30 @@
+import Parser from "tree-sitter";
+import type { CodeFile, SymbolKind } from "../../core/types.js";
+import type { FileAnalysis } from "./types.js";
+export interface TreeSitterNodePattern {
+    type: string;
+    kind: SymbolKind;
+    nameField?: string;
+    nameExtractor?: (node: Parser.SyntaxNode) => string | undefined;
+    signatureField?: string;
+    exportModifierCheck?: (node: Parser.SyntaxNode) => boolean;
+}
+export interface TreeSitterImportPattern {
+    type: string;
+    sourceField?: string;
+    sourceExtractor?: (node: Parser.SyntaxNode) => string | undefined;
+    bindingsExtractor?: (node: Parser.SyntaxNode) => Array<{
+        imported: string;
+        local: string;
+    }>;
+}
+export interface TreeSitterCallPattern {
+    type: string;
+    nameExtractor: (node: Parser.SyntaxNode) => string | undefined;
+}
+export interface TreeSitterLanguageConfig {
+    symbolPatterns: TreeSitterNodePattern[];
+    importPatterns: TreeSitterImportPattern[];
+    callPatterns: TreeSitterCallPattern[];
+}
+export declare function analyzeWithTreeSitter(parser: Parser, config: TreeSitterLanguageConfig, repoRoot: string, file: CodeFile, content: string): FileAnalysis;

package/dist/src/indexing/analyzers/tree-sitter-base.js

@@ -0,0 +1,163 @@
+import { sha256, stableId } from "../../utils/hash.js";
+import { createFileSymbol } from "./fallback-analyzer.js";
+export function analyzeWithTreeSitter(parser, config, repoRoot, file, content) {
+    const tree = parser.parse(content);
+    const lines = content.split(/\r?\n/);
+    const fileSymbol = createFileSymbol(repoRoot, file, lines.length);
+    const symbols = [fileSymbol];
+    const chunks = [];
+    const edges = [];
+    // Extract imports
+    extractImports(tree.rootNode, config.importPatterns, repoRoot, file, fileSymbol.id, edges);
+    // Extract symbols and their relationships
+    const symbolStack = [fileSymbol];
+    function visit(node) {
+        const symbol = extractSymbol(node, config.symbolPatterns, repoRoot, file, content);
+        if (symbol) {
+            symbols.push(symbol);
+            chunks.push(createChunkFromSymbol(repoRoot, file, content, symbol, node));
+            // Containment edge
+            edges.push({
+                projectId: file.projectId,
+                sourceId: symbolStack[symbolStack.length - 1]?.id ?? fileSymbol.id,
+                targetId: symbol.id,
+                kind: "contains",
+                metadata: { sourceFile: file.path }
+            });
+            // Export edge
+            if (symbol.exported) {
+                edges.push({
+                    projectId: file.projectId,
+                    sourceId: fileSymbol.id,
+                    targetId: symbol.id,
+                    kind: "exports",
+                    metadata: { sourceFile: file.path, name: symbol.name }
+                });
+            }
+            symbolStack.push(symbol);
+            // Extract calls within this symbol
+            extractCalls(node, config.callPatterns, repoRoot, file, symbol.id, edges);
+            for (const child of node.children) {
+                visit(child);
+            }
+            symbolStack.pop();
+        }
+        else {
+            for (const child of node.children) {
+                visit(child);
+            }
+        }
+    }
+    visit(tree.rootNode);
+    return { chunks, symbols, edges };
+}
+function extractSymbol(node, patterns, repoRoot, file, content) {
+    for (const pattern of patterns) {
+        if (node.type === pattern.type) {
+            const name = pattern.nameExtractor
+                ? pattern.nameExtractor(node)
+                : pattern.nameField
+                    ? node.childForFieldName(pattern.nameField)?.text
+                    : undefined;
+            if (!name)
+                continue;
+            const startLine = node.startPosition.row + 1;
+            const endLine = node.endPosition.row + 1;
+            const signature = extractSignature(node, content);
+            const exported = pattern.exportModifierCheck ? pattern.exportModifierCheck(node) : false;
+            return {
+                id: stableId([repoRoot, file.path, name, startLine, endLine, pattern.kind]),
+                projectId: file.projectId,
+                filePath: file.path,
+                name,
+                kind: pattern.kind,
+                language: file.language,
+                startLine,
+                endLine,
+                signature,
+                exported
+            };
+        }
+    }
+    return undefined;
+}
+function extractImports(root, patterns, repoRoot, file, fileSymbolId, edges) {
+    function visit(node) {
+        for (const pattern of patterns) {
+            if (node.type === pattern.type) {
+                const source = pattern.sourceExtractor
+                    ? pattern.sourceExtractor(node)
+                    : pattern.sourceField
+                        ? node.childForFieldName(pattern.sourceField)?.text.replace(/['"]/g, "")
+                        : undefined;
+                if (source) {
+                    const bindings = pattern.bindingsExtractor ? pattern.bindingsExtractor(node) : [];
+                    edges.push({
+                        projectId: file.projectId,
+                        sourceId: fileSymbolId,
+                        targetId: stableId([repoRoot, source, "module"]),
+                        kind: "imports",
+                        metadata: {
+                            source,
+                            sourceFile: file.path,
+                            line: node.startPosition.row + 1,
+                            bindings: bindings.length > 0 ? bindings : undefined
+                        }
+                    });
+                }
+            }
+        }
+        for (const child of node.children) {
+            visit(child);
+        }
+    }
+    visit(root);
+}
+function extractCalls(node, patterns, repoRoot, file, symbolId, edges) {
+    function visit(node) {
+        for (const pattern of patterns) {
+            if (node.type === pattern.type) {
+                const targetName = pattern.nameExtractor(node);
+                if (targetName) {
+                    edges.push({
+                        projectId: file.projectId,
+                        sourceId: symbolId,
+                        targetId: stableId([repoRoot, targetName, "symbol"]),
+                        kind: "calls",
+                        metadata: {
+                            targetName,
+                            sourceFile: file.path,
+                            line: node.startPosition.row + 1
+                        }
+                    });
+                }
+            }
+        }
+        for (const child of node.children) {
+            visit(child);
+        }
+    }
+    visit(node);
+}
+function createChunkFromSymbol(repoRoot, file, content, symbol, node) {
+    const body = node.text;
+    return {
+        id: stableId([repoRoot, file.path, symbol.name, symbol.startLine, symbol.endLine, sha256(body)]),
+        projectId: file.projectId,
+        repoRoot,
+        filePath: file.path,
+        language: file.language,
+        kind: symbol.kind === "unknown" ? "block" : symbol.kind,
+        symbolName: symbol.name,
+        startLine: symbol.startLine,
+        endLine: symbol.endLine,
+        content: body,
+        contentHash: sha256(body)
+    };
+}
+function extractSignature(node, content) {
+    const start = node.startIndex;
+    const firstLineEnd = content.indexOf("\n", start);
+    const end = firstLineEnd === -1 ? node.endIndex : Math.min(firstLineEnd, start + 200);
+    return content.slice(start, end).trim();
+}

package/dist/src/indexing/analyzers/types.d.ts

@@ -0,0 +1,17 @@
+import type { CodeChunk, CodeFile, GraphEdge, LanguageId, SymbolNode } from "../../core/types.js";
+export interface FileAnalysis {
+    chunks: CodeChunk[];
+    symbols: SymbolNode[];
+    edges: GraphEdge[];
+}
+export interface AnalyzeFileInput {
+    repoRoot: string;
+    file: CodeFile;
+    content: string;
+}
+export type AnalyzerCapability = "symbols" | "imports" | "exports" | "calls" | "definitions" | "framework_routes" | "tests";
+export interface LanguageAnalyzer {
+    language: LanguageId;
+    capabilities: AnalyzerCapability[];
+    analyzeFile(input: AnalyzeFileInput): FileAnalysis;
+}

package/dist/src/indexing/analyzers/types.js

@@ -0,0 +1 @@
+export {};

package/dist/src/indexing/analyzers/typescript-analyzer.d.ts

@@ -0,0 +1,5 @@
+import type { CodeFile } from "../../core/types.js";
+import type { FileAnalysis, LanguageAnalyzer } from "./types.js";
+export declare const typescriptAnalyzer: LanguageAnalyzer;
+export declare const javascriptAnalyzer: LanguageAnalyzer;
+export declare function analyzeTypeScriptFile(repoRoot: string, file: CodeFile, content: string): FileAnalysis;

package/dist/src/indexing/analyzers/typescript-analyzer.js

@@ -0,0 +1,199 @@
+import ts from "typescript";
+import { sha256, stableId } from "../../utils/hash.js";
+import { createFileSymbol, fallbackFileAnalysis } from "./fallback-analyzer.js";
+export const typescriptAnalyzer = {
+    language: "typescript",
+    capabilities: ["symbols", "imports", "exports", "calls", "definitions", "framework_routes", "tests"],
+    analyzeFile: ({ repoRoot, file, content }) => analyzeTypeScriptFile(repoRoot, file, content)
+};
+export const javascriptAnalyzer = {
+    ...typescriptAnalyzer,
+    language: "javascript"
+};
+export function analyzeTypeScriptFile(repoRoot, file, content) {
+    const sourceFile = ts.createSourceFile(file.path, content, ts.ScriptTarget.Latest, true, scriptKindForPath(file.path));
+    const lines = content.split(/\r?\n/);
+    const chunks = [];
+    const symbols = [];
+    const edges = [];
+    const fileSymbol = createFileSymbol(repoRoot, file, lines.length);
+    symbols.push(fileSymbol);
+    for (const statement of sourceFile.statements) {
+        if (ts.isImportDeclaration(statement) && ts.isStringLiteral(statement.moduleSpecifier)) {
+            const importSource = statement.moduleSpecifier.text;
+            edges.push({
+                projectId: file.projectId,
+                sourceId: fileSymbol.id,
+                targetId: stableId([repoRoot, importSource, "module"]),
+                kind: "imports",
+                metadata: { source: importSource, sourceFile: file.path, line: lineRange(sourceFile, statement).startLine, bindings: importBindings(statement) }
+            });
+        }
+    }
+    const symbolStack = [fileSymbol];
+    function visit(node) {
+        const symbol = symbolFromNode(repoRoot, file, sourceFile, node);
+        if (symbol) {
+            if (symbol.kind === "variable" && symbolStack[symbolStack.length - 1]?.kind !== "file" && !symbol.exported) {
+                ts.forEachChild(node, visit);
+                return;
+            }
+            symbols.push(symbol);
+            chunks.push(chunkFromSymbol(repoRoot, file, sourceFile, content, symbol, node));
+            edges.push({
+                projectId: file.projectId,
+                sourceId: symbolStack[symbolStack.length - 1]?.id ?? fileSymbol.id,
+                targetId: symbol.id,
+                kind: "contains",
+                metadata: { sourceFile: file.path }
+            });
+            if (symbol.exported) {
+                edges.push({
+                    projectId: file.projectId,
+                    sourceId: fileSymbol.id,
+                    targetId: symbol.id,
+                    kind: "exports",
+                    metadata: { sourceFile: file.path, name: symbol.name }
+                });
+            }
+            symbolStack.push(symbol);
+            ts.forEachChild(node, visit);
+            symbolStack.pop();
+            return;
+        }
+        if (ts.isCallExpression(node)) {
+            const targetName = callTargetName(node.expression);
+            if (targetName) {
+                edges.push({
+                    projectId: file.projectId,
+                    sourceId: symbolStack[symbolStack.length - 1]?.id ?? fileSymbol.id,
+                    targetId: stableId([repoRoot, targetName, "symbol"]),
+                    kind: "calls",
+                    metadata: { targetName, sourceFile: file.path, line: lineRange(sourceFile, node).startLine, position: callTargetPosition(node.expression, sourceFile) }
+                });
+            }
+        }
+        ts.forEachChild(node, visit);
+    }
+    ts.forEachChild(sourceFile, visit);
+    if (chunks.length === 0) {
+        return fallbackFileAnalysis(repoRoot, file, content, fileSymbol, edges);
+    }
+    return { chunks, symbols, edges };
+}
+function symbolFromNode(repoRoot, file, sourceFile, node) {
+    const named = nodeName(node);
+    if (!named)
+        return undefined;
+    const range = lineRange(sourceFile, node);
+    const kind = symbolKind(node);
+    if (!kind)
+        return undefined;
+    return {
+        id: stableId([repoRoot, file.path, named, range.startLine, range.endLine, kind]),
+        projectId: file.projectId,
+        filePath: file.path,
+        name: named,
+        kind,
+        language: file.language,
+        startLine: range.startLine,
+        endLine: range.endLine,
+        signature: firstLine(sourceFile.text.slice(node.getStart(sourceFile), node.getEnd())),
+        exported: hasExportModifier(node)
+    };
+}
+function chunkFromSymbol(repoRoot, file, sourceFile, content, symbol, node) {
+    const body = content.slice(node.getStart(sourceFile), node.getEnd());
+    return {
+        id: stableId([repoRoot, file.path, symbol.name, symbol.startLine, symbol.endLine, sha256(body)]),
+        projectId: file.projectId,
+        repoRoot,
+        filePath: file.path,
+        language: file.language,
+        kind: symbol.kind === "unknown" ? "block" : symbol.kind,
+        symbolName: symbol.name,
+        startLine: symbol.startLine,
+        endLine: symbol.endLine,
+        content: body,
+        contentHash: sha256(body)
+    };
+}
+function nodeName(node) {
+    if ((ts.isFunctionDeclaration(node) || ts.isClassDeclaration(node) || ts.isInterfaceDeclaration(node) || ts.isTypeAliasDeclaration(node)) && node.name) {
+        return node.name.text;
+    }
+    if (ts.isMethodDeclaration(node) && node.name) {
+        return propertyName(node.name);
+    }
+    if (ts.isVariableStatement(node)) {
+        const declaration = node.declarationList.declarations[0];
+        if (declaration?.name && ts.isIdentifier(declaration.name))
+            return declaration.name.text;
+    }
+    return undefined;
+}
+function symbolKind(node) {
+    if (ts.isFunctionDeclaration(node))
+        return "function";
+    if (ts.isClassDeclaration(node))
+        return "class";
+    if (ts.isMethodDeclaration(node))
+        return "method";
+    if (ts.isInterfaceDeclaration(node) || ts.isTypeAliasDeclaration(node))
+        return "type";
+    if (ts.isVariableStatement(node))
+        return "variable";
+    return undefined;
+}
+function propertyName(name) {
+    if (ts.isIdentifier(name) || ts.isStringLiteral(name) || ts.isNumericLiteral(name))
+        return name.text;
+    return undefined;
+}
+function callTargetName(expression) {
+    if (ts.isIdentifier(expression))
+        return expression.text;
+    if (ts.isPropertyAccessExpression(expression))
+        return expression.name.text;
+    return undefined;
+}
+function callTargetPosition(expression, sourceFile) {
+    if (ts.isPropertyAccessExpression(expression))
+        return expression.name.getStart(sourceFile);
+    return expression.getStart(sourceFile);
+}
+function importBindings(node) {
+    const clause = node.importClause;
+    if (!clause)
+        return [];
+    const bindings = [];
+    if (clause.name)
+        bindings.push({ imported: "default", local: clause.name.text });
+    const namedBindings = clause.namedBindings;
+    if (namedBindings && ts.isNamedImports(namedBindings)) {
+        for (const element of namedBindings.elements) {
+            bindings.push({ imported: element.propertyName?.text ?? element.name.text, local: element.name.text });
+        }
+    }
+    return bindings;
+}
+function hasExportModifier(node) {
+    return Boolean(ts.canHaveModifiers(node) && ts.getModifiers(node)?.some((modifier) => modifier.kind === ts.SyntaxKind.ExportKeyword));
+}
+function lineRange(sourceFile, node) {
+    const start = sourceFile.getLineAndCharacterOfPosition(node.getStart(sourceFile));
+    const end = sourceFile.getLineAndCharacterOfPosition(node.getEnd());
+    return { startLine: start.line + 1, endLine: end.line + 1 };
+}
+function firstLine(content) {
+    return content.split(/\r?\n/, 1)[0]?.trim() ?? "";
+}
+function scriptKindForPath(filePath) {
+    if (filePath.endsWith(".tsx"))
+        return ts.ScriptKind.TSX;
+    if (filePath.endsWith(".jsx"))
+        return ts.ScriptKind.JSX;
+    if (filePath.endsWith(".js") || filePath.endsWith(".mjs") || filePath.endsWith(".cjs"))
+        return ts.ScriptKind.JS;
+    return ts.ScriptKind.TS;
+}

package/dist/src/indexing/ast-analyzer.d.ts

@@ -0,0 +1,11 @@
+import type { CodeFile } from "../core/types.js";
+import type { FileAnalysis } from "./analyzers/types.js";
+export type { AnalyzerCapability, AnalyzeFileInput, FileAnalysis, LanguageAnalyzer } from "./analyzers/types.js";
+export { analyzerFor, analyzeFileWithRegistry, listAnalyzers } from "./analyzers/registry.js";
+export { fallbackAnalyzer } from "./analyzers/fallback-analyzer.js";
+export { goTreeSitterAnalyzer, goTreeSitterAnalyzer as goAnalyzer } from "./analyzers/go-treesitter-analyzer.js";
+export { javaTreeSitterAnalyzer, javaTreeSitterAnalyzer as javaAnalyzer } from "./analyzers/java-treesitter-analyzer.js";
+export { pythonTreeSitterAnalyzer, pythonTreeSitterAnalyzer as pythonAnalyzer } from "./analyzers/python-treesitter-analyzer.js";
+export { rustTreeSitterAnalyzer, rustTreeSitterAnalyzer as rustAnalyzer } from "./analyzers/rust-treesitter-analyzer.js";
+export { javascriptAnalyzer, typescriptAnalyzer } from "./analyzers/typescript-analyzer.js";
+export declare function analyzeFile(repoRoot: string, file: CodeFile, content: string): FileAnalysis;

package/dist/src/indexing/ast-analyzer.js

@@ -0,0 +1,11 @@
+import { analyzeFileWithRegistry } from "./analyzers/registry.js";
+export { analyzerFor, analyzeFileWithRegistry, listAnalyzers } from "./analyzers/registry.js";
+export { fallbackAnalyzer } from "./analyzers/fallback-analyzer.js";
+export { goTreeSitterAnalyzer, goTreeSitterAnalyzer as goAnalyzer } from "./analyzers/go-treesitter-analyzer.js";
+export { javaTreeSitterAnalyzer, javaTreeSitterAnalyzer as javaAnalyzer } from "./analyzers/java-treesitter-analyzer.js";
+export { pythonTreeSitterAnalyzer, pythonTreeSitterAnalyzer as pythonAnalyzer } from "./analyzers/python-treesitter-analyzer.js";
+export { rustTreeSitterAnalyzer, rustTreeSitterAnalyzer as rustAnalyzer } from "./analyzers/rust-treesitter-analyzer.js";
+export { javascriptAnalyzer, typescriptAnalyzer } from "./analyzers/typescript-analyzer.js";
+export function analyzeFile(repoRoot, file, content) {
+    return analyzeFileWithRegistry({ repoRoot, file, content });
+}

package/dist/src/indexing/chunker.d.ts

@@ -0,0 +1,11 @@
+import type { CodeChunk, CodeFile, GraphEdge, SymbolNode } from "../core/types.js";
+export interface ChunkingResult {
+    chunks: CodeChunk[];
+    symbols: SymbolNode[];
+    edges: GraphEdge[];
+}
+export interface ChunkOptions {
+    targetLines?: number;
+}
+export declare function chunkFiles(repoRoot: string, files: CodeFile[], options?: ChunkOptions): Promise<ChunkingResult>;
+export declare function chunkFilesIncremental(repoRoot: string, files: CodeFile[], filesToAnalyze: CodeFile[], cached: ChunkingResult, options?: ChunkOptions): Promise<ChunkingResult>;