toolnet-memory 0.2.0

package/dist/src/code-intelligence/rich/rich-graph-enricher.js

@@ -0,0 +1,463 @@
+import { createHash, } from "node:crypto";
+import { dirname, relative, resolve, } from "node:path";
+import ts from "typescript";
+function normalize(value) {
+    return value
+        .replaceAll("\\", "/")
+        .replace(/^\.\//, "");
+}
+function id(...parts) {
+    return createHash("sha256")
+        .update(parts.join(":"))
+        .digest("hex")
+        .slice(0, 24);
+}
+function lineOf(source, node) {
+    return source
+        .getLineAndCharacterOfPosition(node.getStart(source)).line + 1;
+}
+const ROUTE_METHODS = new Set([
+    "get",
+    "post",
+    "put",
+    "patch",
+    "delete",
+    "options",
+    "head",
+]);
+const ASSIGNMENTS = new Set([
+    ts.SyntaxKind.EqualsToken,
+    ts.SyntaxKind.PlusEqualsToken,
+    ts.SyntaxKind.MinusEqualsToken,
+    ts.SyntaxKind.AsteriskEqualsToken,
+    ts.SyntaxKind.SlashEqualsToken,
+    ts.SyntaxKind.PercentEqualsToken,
+    ts.SyntaxKind.QuestionQuestionEqualsToken,
+    ts.SyntaxKind.AmpersandAmpersandEqualsToken,
+    ts.SyntaxKind.BarBarEqualsToken,
+]);
+export class RichGraphEnricher {
+    graph;
+    constructor(graph) {
+        this.graph = graph;
+    }
+    enrich(projectId, rootPath, resolution) {
+        const stats = {
+            routes: 0,
+            tests: 0,
+            usesType: 0,
+            writes: 0,
+            callReferences: 0,
+            properties: 0,
+        };
+        const config = this.loadConfig(rootPath);
+        const program = ts.createProgram({
+            rootNames: config.fileNames,
+            options: config.options,
+        });
+        const checker = program.getTypeChecker();
+        for (const source of program.getSourceFiles()) {
+            if (source.isDeclarationFile) {
+                continue;
+            }
+            const filePath = normalize(relative(rootPath, resolve(source.fileName)));
+            if (filePath.startsWith("../") ||
+                filePath.includes("node_modules/")) {
+                continue;
+            }
+            const fileNode = this.fileNode(projectId, filePath);
+            if (!fileNode) {
+                continue;
+            }
+            const visit = (node) => {
+                if (ts.isCallExpression(node)) {
+                    if (this.addRoute(projectId, rootPath, source, fileNode, node, checker)) {
+                        stats.routes++;
+                    }
+                }
+                if (ts.isTypeReferenceNode(node)) {
+                    if (this.addTypeUse(projectId, rootPath, source, node, checker)) {
+                        stats.usesType++;
+                    }
+                }
+                if (ts.isBinaryExpression(node) &&
+                    ASSIGNMENTS.has(node.operatorToken.kind)) {
+                    const result = this.addWrite(projectId, rootPath, source, node.left, checker);
+                    if (result.added) {
+                        stats.writes++;
+                        if (result.propertyCreated) {
+                            stats.properties++;
+                        }
+                    }
+                }
+                if ((ts.isPrefixUnaryExpression(node) ||
+                    ts.isPostfixUnaryExpression(node)) &&
+                    (node.operator ===
+                        ts.SyntaxKind.PlusPlusToken ||
+                        node.operator ===
+                            ts.SyntaxKind.MinusMinusToken)) {
+                    const result = this.addWrite(projectId, rootPath, source, node.operand, checker);
+                    if (result.added) {
+                        stats.writes++;
+                        if (result.propertyCreated) {
+                            stats.properties++;
+                        }
+                    }
+                }
+                ts.forEachChild(node, visit);
+            };
+            visit(source);
+        }
+        stats.tests +=
+            this.addTestEdges(projectId);
+        if (resolution) {
+            stats.callReferences +=
+                this.addResolvedCalls(projectId, resolution);
+        }
+        return stats;
+    }
+    addRoute(projectId, rootPath, source, fileNode, node, checker) {
+        if (!ts.isPropertyAccessExpression(node.expression)) {
+            return false;
+        }
+        const method = node.expression
+            .name.text
+            .toLowerCase();
+        if (!ROUTE_METHODS.has(method)) {
+            return false;
+        }
+        const first = node.arguments[0];
+        if (!first ||
+            !(ts.isStringLiteral(first) ||
+                ts.isNoSubstitutionTemplateLiteral(first))) {
+            return false;
+        }
+        const path = first.text;
+        if (!path.startsWith("/")) {
+            return false;
+        }
+        const sourceLine = lineOf(source, node);
+        const routeId = id(projectId, source.fileName, sourceLine, method, path);
+        if (!this.graph.getSymbol(routeId)) {
+            this.graph.addSymbol({
+                id: routeId,
+                projectId,
+                name: `${method.toUpperCase()} ${path}`,
+                qualifiedName: `${method.toUpperCase()} ${path}`,
+                type: "route",
+                filePath: fileNode.filePath,
+                startLine: sourceLine,
+                endLine: sourceLine,
+            });
+            this.graph.addEdge(this.edge(projectId, fileNode.id, "DEFINES", routeId));
+        }
+        const handler = node.arguments.at(-1);
+        if (handler) {
+            const target = this.resolveNodeSymbol(projectId, rootPath, handler, checker);
+            if (target) {
+                this.graph.addEdge(this.edge(projectId, routeId, "ROUTE", target.id, {
+                    method: method.toUpperCase(),
+                    path,
+                }));
+            }
+        }
+        return true;
+    }
+    addTypeUse(projectId, rootPath, source, node, checker) {
+        const target = this.resolveNodeSymbol(projectId, rootPath, node.typeName, checker);
+        if (!target) {
+            return false;
+        }
+        const sourceSymbol = this.containingSymbol(projectId, normalize(relative(rootPath, resolve(source.fileName))), lineOf(source, node));
+        if (!sourceSymbol ||
+            sourceSymbol.id ===
+                target.id) {
+            return false;
+        }
+        this.graph.addEdge(this.edge(projectId, sourceSymbol.id, "USES_TYPE", target.id));
+        return true;
+    }
+    addWrite(projectId, rootPath, source, targetNode, checker) {
+        const symbol = this.resolveTsSymbol(targetNode, checker);
+        const declaration = symbol
+            ?.valueDeclaration ??
+            symbol
+                ?.declarations?.[0];
+        if (!declaration) {
+            return {
+                added: false,
+                propertyCreated: false,
+            };
+        }
+        const declarationSource = declaration
+            .getSourceFile();
+        if (declarationSource
+            .isDeclarationFile) {
+            return {
+                added: false,
+                propertyCreated: false,
+            };
+        }
+        const targetFile = normalize(relative(rootPath, resolve(declarationSource
+            .fileName)));
+        if (targetFile.startsWith("../") ||
+            targetFile.includes("node_modules/")) {
+            return {
+                added: false,
+                propertyCreated: false,
+            };
+        }
+        const targetLine = lineOf(declarationSource, declaration);
+        let target = this.findSymbolAt(projectId, targetFile, targetLine);
+        let propertyCreated = false;
+        if (!target ||
+            target.type === "file") {
+            const name = symbol?.getName() ??
+                targetNode.getText(source);
+            const propertyId = id(projectId, targetFile, "property", name, targetLine);
+            target =
+                this.graph.getSymbol(propertyId);
+            if (!target) {
+                target = {
+                    id: propertyId,
+                    projectId,
+                    name,
+                    qualifiedName: name,
+                    type: "property",
+                    filePath: targetFile,
+                    startLine: targetLine,
+                    endLine: targetLine,
+                };
+                this.graph.addSymbol(target);
+                const targetFileNode = this.fileNode(projectId, targetFile);
+                if (targetFileNode) {
+                    this.graph.addEdge(this.edge(projectId, targetFileNode.id, "DEFINES", target.id));
+                }
+                propertyCreated =
+                    true;
+            }
+        }
+        if (!target) {
+            return {
+                added: false,
+                propertyCreated,
+            };
+        }
+        const sourceFile = normalize(relative(rootPath, resolve(source.fileName)));
+        const writer = this.containingSymbol(projectId, sourceFile, lineOf(source, targetNode));
+        if (!writer ||
+            writer.id === target.id) {
+            return {
+                added: false,
+                propertyCreated,
+            };
+        }
+        this.graph.addEdge(this.edge(projectId, writer.id, "WRITES", target.id, {
+            expression: targetNode.getText(source),
+        }));
+        return {
+            added: true,
+            propertyCreated,
+        };
+    }
+    addTestEdges(projectId) {
+        const edges = this.graph.allEdges(projectId);
+        let added = 0;
+        for (const edge of edges) {
+            if (edge.type !==
+                "IMPORTS") {
+                continue;
+            }
+            const from = this.graph.getSymbol(edge.from);
+            const to = this.graph.getSymbol(edge.to);
+            if (!from ||
+                !to ||
+                !this.isTestFile(from.filePath) ||
+                this.isTestFile(to.filePath)) {
+                continue;
+            }
+            this.graph.addEdge(this.edge(projectId, from.id, "TESTS", to.id));
+            added++;
+        }
+        return added;
+    }
+    addResolvedCalls(projectId, resolution) {
+        let added = 0;
+        for (const item of resolution.resolutions) {
+            if (item.kind !== "CALL" ||
+                !item.targetSymbolId) {
+                continue;
+            }
+            const target = this.graph.getSymbol(item.targetSymbolId);
+            if (!target) {
+                continue;
+            }
+            const source = this.containingSymbol(projectId, item.sourceFile, item.sourceLine);
+            if (!source ||
+                source.id === target.id) {
+                continue;
+            }
+            /*
+             * Nếu CALLS hiện tại đã đúng target
+             * thì không tạo edge trùng.
+             */
+            const exactAlreadyExists = this.graph
+                .allEdges(projectId)
+                .some((edge) => edge.type ===
+                "CALLS" &&
+                edge.from ===
+                    source.id &&
+                edge.to ===
+                    target.id);
+            if (exactAlreadyExists) {
+                continue;
+            }
+            this.graph.addEdge(this.edge(projectId, source.id, "CALL_REFERENCE", target.id, {
+                expression: item.expression,
+                confidence: item.confidence,
+                resolver: item.resolver,
+            }));
+            added++;
+        }
+        return added;
+    }
+    resolveNodeSymbol(projectId, rootPath, node, checker) {
+        const symbol = this.resolveTsSymbol(node, checker);
+        const declaration = symbol
+            ?.valueDeclaration ??
+            symbol
+                ?.declarations?.[0];
+        if (!declaration) {
+            return undefined;
+        }
+        const targetSource = declaration
+            .getSourceFile();
+        if (targetSource
+            .isDeclarationFile) {
+            return undefined;
+        }
+        const filePath = normalize(relative(rootPath, resolve(targetSource
+            .fileName)));
+        if (filePath.startsWith("../") ||
+            filePath.includes("node_modules/")) {
+            return undefined;
+        }
+        return this.findSymbolAt(projectId, filePath, lineOf(targetSource, declaration), symbol?.getName());
+    }
+    resolveTsSymbol(node, checker) {
+        let symbol = checker.getSymbolAtLocation(node);
+        if (symbol &&
+            (symbol.flags &
+                ts.SymbolFlags.Alias)) {
+            try {
+                symbol =
+                    checker.getAliasedSymbol(symbol);
+            }
+            catch {
+                // keep original
+            }
+        }
+        return symbol;
+    }
+    containingSymbol(projectId, filePath, line) {
+        return this.graph
+            .allSymbols(projectId)
+            .filter((symbol) => normalize(symbol.filePath) ===
+            normalize(filePath) &&
+            typeof symbol.startLine ===
+                "number" &&
+            typeof symbol.endLine ===
+                "number" &&
+            symbol.startLine <= line &&
+            symbol.endLine >= line)
+            .sort((a, b) => ((a.endLine ?? 0) -
+            (a.startLine ?? 0)) -
+            ((b.endLine ?? 0) -
+                (b.startLine ?? 0)))[0];
+    }
+    findSymbolAt(projectId, filePath, line, name) {
+        const candidates = this.graph
+            .allSymbols(projectId)
+            .filter((symbol) => normalize(symbol.filePath) ===
+            normalize(filePath) &&
+            typeof symbol.startLine ===
+                "number" &&
+            typeof symbol.endLine ===
+                "number" &&
+            symbol.startLine <= line &&
+            symbol.endLine >= line)
+            .sort((a, b) => ((a.endLine ?? 0) -
+            (a.startLine ?? 0)) -
+            ((b.endLine ?? 0) -
+                (b.startLine ?? 0)));
+        if (name) {
+            const named = candidates.find((symbol) => symbol.name === name ||
+                (symbol.qualifiedName
+                    ?.endsWith(`.${name}`) ?? false));
+            if (named) {
+                return named;
+            }
+        }
+        return candidates[0];
+    }
+    fileNode(projectId, filePath) {
+        return this.graph
+            .allSymbols(projectId)
+            .find((symbol) => symbol.type ===
+            "file" &&
+            normalize(symbol.filePath) ===
+                normalize(filePath));
+    }
+    isTestFile(filePath) {
+        const path = normalize(filePath).toLowerCase();
+        return (path.startsWith("tests/") ||
+            path.includes("/tests/") ||
+            path.includes("__tests__") ||
+            /\.(test|spec)\.[^.]+$/
+                .test(path));
+    }
+    edge(projectId, from, type, to, metadata) {
+        return {
+            id: id(projectId, from, type, to, JSON.stringify(metadata ?? {})),
+            projectId,
+            from,
+            to,
+            type,
+            metadata,
+        };
+    }
+    loadConfig(rootPath) {
+        const configPath = ts.findConfigFile(rootPath, ts.sys.fileExists, "tsconfig.json");
+        if (configPath) {
+            const raw = ts.readConfigFile(configPath, ts.sys.readFile);
+            return ts.parseJsonConfigFileContent(raw.config, ts.sys, dirname(configPath));
+        }
+        return {
+            fileNames: ts.sys.readDirectory(rootPath, [
+                ".ts",
+                ".tsx",
+                ".js",
+                ".jsx",
+                ".mts",
+                ".cts",
+                ".mjs",
+                ".cjs",
+            ], [
+                "node_modules",
+                ".git",
+                "dist",
+                "build",
+                "coverage",
+            ]),
+            options: {
+                allowJs: true,
+                noEmit: true,
+                skipLibCheck: true,
+                target: ts.ScriptTarget.ES2022,
+                module: ts.ModuleKind.NodeNext,
+                moduleResolution: ts.ModuleResolutionKind
+                    .NodeNext,
+            },
+        };
+    }
+}

package/dist/src/code-intelligence/semantic/index.js

@@ -0,0 +1,2 @@
+export * from "./semantic-code-engine.js";
+export * from "./source-priority.js";

package/dist/src/code-intelligence/semantic/semantic-code-engine.js

@@ -0,0 +1,213 @@
+import { PersistentCodeChunkStore, } from "../../storage/code-chunk-store.js";
+import { PersistentCodeVectorStore, } from "../../storage/code-vector-store.js";
+import { SmartCodeChunker, } from "../chunks/smart-chunker.js";
+import { VectorStore, } from "../../retrieval/vector/vector-store.js";
+import { tokenize, } from "../../retrieval/tokenizer.js";
+import { codeSourceWeight, } from "./source-priority.js";
+export class SemanticCodeEngine {
+    options;
+    vectors = new VectorStore();
+    chunks = new Map();
+    constructor(options) {
+        this.options = options;
+    }
+    async initialize() {
+        const { projectId, rootPath, storage, embeddings, graph, model, } = this.options;
+        const chunkStore = new PersistentCodeChunkStore(storage);
+        const vectorStore = new PersistentCodeVectorStore(storage);
+        const chunks = await new SmartCodeChunker()
+            .build(projectId, rootPath, graph.allSymbols(projectId));
+        this.chunks.clear();
+        for (const chunk of chunks) {
+            this.chunks.set(chunk.id, chunk);
+        }
+        const previousChunks = await chunkStore.load(projectId);
+        const chunksChanged = !previousChunks ||
+            previousChunks.chunks.length !==
+                chunks.length ||
+            previousChunks.chunks.some((old) => !this.chunks.has(old.id));
+        if (chunksChanged) {
+            await chunkStore.save({
+                version: 1,
+                projectId,
+                updatedAt: new Date()
+                    .toISOString(),
+                chunks,
+            });
+        }
+        const previousVectors = await vectorStore.load(projectId);
+        let vectorsLoaded = 0;
+        if (previousVectors &&
+            previousVectors.model ===
+                model) {
+            vectorsLoaded =
+                this.vectors.importRecords(previousVectors.records);
+        }
+        const validIds = new Set(chunks.map((chunk) => chunk.id));
+        let vectorsRemoved = 0;
+        for (const record of this.vectors
+            .exportProject(projectId)) {
+            if (!validIds.has(record.id)) {
+                if (this.vectors.remove(record.id)) {
+                    vectorsRemoved++;
+                }
+            }
+        }
+        const pending = chunks.filter((chunk) => {
+            const existing = this.vectors.get(chunk.id);
+            if (!existing) {
+                return true;
+            }
+            return (existing.metadata
+                ?.contentHash !==
+                chunk.contentHash);
+        });
+        let vectorsIndexed = 0;
+        /*
+         * Batch nhỏ để tránh request embedding quá lớn.
+         */
+        const batchSize = 8;
+        let targetDimensions = this.vectors
+            .exportProject(projectId)[0]
+            ?.vector.length;
+        for (let offset = 0; offset < pending.length; offset += batchSize) {
+            const batch = pending.slice(offset, offset +
+                batchSize);
+            const texts = batch.map((chunk) => [
+                `file: ${chunk.filePath}`,
+                chunk.symbolName
+                    ? `symbol: ${chunk.symbolName}`
+                    : "",
+                chunk.symbolType
+                    ? `type: ${chunk.symbolType}`
+                    : "",
+                chunk.content,
+            ]
+                .filter(Boolean)
+                .join("\n"));
+            const vectors = await embeddings
+                .embedMany(texts);
+            for (let i = 0; i < batch.length; i++) {
+                const chunk = batch[i];
+                const vector = vectors[i];
+                if (!chunk ||
+                    !vector ||
+                    vector.length === 0) {
+                    continue;
+                }
+                if (targetDimensions ===
+                    undefined) {
+                    targetDimensions =
+                        vector.length;
+                }
+                /*
+                 * Không cho trộn vector 384d/128d
+                 * nếu provider fallback giữa chừng.
+                 */
+                if (vector.length !==
+                    targetDimensions) {
+                    console.warn(`[code-semantic] skip dimension mismatch: ${vector.length} != ${targetDimensions}`);
+                    continue;
+                }
+                this.vectors.upsert({
+                    id: chunk.id,
+                    projectId,
+                    vector,
+                    metadata: {
+                        contentHash: chunk.contentHash,
+                        filePath: chunk.filePath,
+                        symbolName: chunk.symbolName,
+                        symbolType: chunk.symbolType,
+                        startLine: chunk.startLine,
+                        endLine: chunk.endLine,
+                    },
+                });
+                vectorsIndexed++;
+            }
+        }
+        const records = this.vectors
+            .exportProject(projectId);
+        const shouldSaveVectors = vectorsIndexed > 0 ||
+            vectorsRemoved > 0 ||
+            !previousVectors ||
+            previousVectors.model !==
+                model;
+        if (shouldSaveVectors) {
+            await vectorStore.save({
+                version: 1,
+                projectId,
+                model,
+                dimensions: records[0]
+                    ?.vector.length ??
+                    targetDimensions ??
+                    0,
+                updatedAt: new Date()
+                    .toISOString(),
+                records,
+            });
+        }
+        return {
+            chunks: chunks.length,
+            vectorsLoaded,
+            vectorsIndexed,
+            vectorsRemoved,
+            totalVectors: records.length,
+            savedChunks: chunksChanged,
+            savedVectors: shouldSaveVectors,
+        };
+    }
+    async search(query, limit = 8) {
+        const queryVector = await this.options
+            .embeddings
+            .embed(query);
+        const vectorResults = this.vectors.search(this.options.projectId, queryVector, Math.max(limit * 4, 20));
+        const queryTokens = new Set(tokenize(query));
+        const output = [];
+        for (const result of vectorResults) {
+            const chunk = this.chunks.get(result.id);
+            if (!chunk) {
+                continue;
+            }
+            const documentTokens = new Set(tokenize([
+                chunk.filePath,
+                chunk.symbolName ??
+                    "",
+                chunk.symbolType ??
+                    "",
+                chunk.content,
+            ].join(" ")));
+            let matches = 0;
+            for (const token of queryTokens) {
+                if (documentTokens.has(token)) {
+                    matches++;
+                }
+            }
+            const lexicalScore = queryTokens.size === 0
+                ? 0
+                : matches /
+                    queryTokens.size;
+            const vectorScore = Math.max(0, result.score);
+            /*
+             * Semantic quan trọng hơn lexical.
+             */
+            const sourceWeight = codeSourceWeight(chunk.filePath, query);
+            const score = (vectorScore *
+                0.75 +
+                lexicalScore *
+                    0.25) *
+                sourceWeight;
+            output.push({
+                chunk,
+                score,
+                vectorScore,
+                lexicalScore,
+            });
+        }
+        return output
+            .sort((a, b) => b.score - a.score)
+            .slice(0, limit);
+    }
+    size() {
+        return this.chunks.size;
+    }
+}

package/dist/src/code-intelligence/semantic/source-priority.js

@@ -0,0 +1,22 @@
+export function codeSourceWeight(filePath, query) {
+    const path = filePath
+        .toLowerCase()
+        .replaceAll("\\", "/");
+    const q = query.toLowerCase();
+    const testIntent = /\b(test|tests|spec|fixture|mock)\b/
+        .test(q);
+    const isTest = path.startsWith("tests/") ||
+        path.includes("/tests/") ||
+        path.includes("__tests__") ||
+        /\.(test|spec)\.[^.]+$/
+            .test(path);
+    if (isTest &&
+        !testIntent) {
+        return 0.35;
+    }
+    if (path.includes("/generated/") ||
+        path.includes(".generated.")) {
+        return 0.25;
+    }
+    return 1;
+}

package/dist/src/code-intelligence/symbols/index.js

@@ -0,0 +1 @@
+export {};