toolnet-memory 0.2.0

package/dist/src/project-manual/manager.js

@@ -0,0 +1,147 @@
+import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync, } from "node:fs";
+import { dirname, join, } from "node:path";
+import { sha256, } from "../session/utils.js";
+const MAX_MANUAL_BYTES = 64 * 1024;
+export const PROJECT_MANUAL_TEMPLATE = `# ToolNet Project Operating Manual
+
+This file contains persistent instructions for AI agents working on this project.
+
+## Critical Rules
+
+<!--
+Examples:
+
+- [enforce] Never edit production files directly.
+- [enforce] Edit source only in /path/to/source.
+- [enforce] Deploy only with ./deploy.sh --apply
+-->
+
+## Workflow
+
+<!--
+Describe the correct working process.
+-->
+
+## Architecture
+
+<!--
+Important architecture conventions.
+-->
+
+## Verification
+
+<!--
+Tests / QA required after changes.
+-->
+
+## Known Gotchas
+
+<!--
+Things an AI agent should always remember.
+-->
+
+## Preferences
+
+<!--
+- [advisory] Prefer small focused files.
+-->
+`;
+export function projectManualPath(project) {
+    return join(project.rootPath, ".toolnet", "PROJECT.md");
+}
+function normalizeRuleText(value) {
+    return value
+        .normalize("NFKC")
+        .replace(/\s+/g, " ")
+        .trim();
+}
+function parseRules(content) {
+    const rules = [];
+    const seen = new Set();
+    const pattern = /^\s*[-*]\s+\[(enforce|advisory)\]\s+(.+?)\s*$/gimu;
+    let match;
+    while ((match =
+        pattern.exec(content))) {
+        const mode = match[1]
+            .toLowerCase();
+        const text = normalizeRuleText(match[2]);
+        if (!text) {
+            continue;
+        }
+        const key = `${mode}:${text.toLowerCase()}`;
+        if (seen.has(key)) {
+            continue;
+        }
+        seen.add(key);
+        rules.push({
+            id: sha256(key).slice(0, 24),
+            mode,
+            text,
+            source: "manual",
+        });
+    }
+    return rules;
+}
+export function ensureProjectManual(project) {
+    const file = projectManualPath(project);
+    if (existsSync(file)) {
+        return file;
+    }
+    mkdirSync(dirname(file), {
+        recursive: true,
+    });
+    writeFileSync(file, PROJECT_MANUAL_TEMPLATE, {
+        encoding: "utf8",
+        mode: 0o600,
+    });
+    return file;
+}
+export function loadProjectManual(project, createIfMissing = false) {
+    const file = createIfMissing
+        ? ensureProjectManual(project)
+        : projectManualPath(project);
+    if (!existsSync(file)) {
+        return null;
+    }
+    const size = statSync(file).size;
+    if (size >
+        MAX_MANUAL_BYTES) {
+        throw new Error(`PROJECT.md exceeds ${MAX_MANUAL_BYTES} bytes`);
+    }
+    const content = readFileSync(file, "utf8");
+    return {
+        path: file,
+        content,
+        digest: sha256(content),
+        rules: parseRules(content),
+        bytes: Buffer.byteLength(content, "utf8"),
+        updatedAt: new Date(statSync(file).mtimeMs).toISOString(),
+    };
+}
+export async function syncProjectManual(project, storage) {
+    const manual = loadProjectManual(project, true);
+    if (!manual) {
+        throw new Error("Project manual unavailable");
+    }
+    const prefix = `projects/${project.id}/project`;
+    await storage.put(`${prefix}/manual.md`, manual.content, "text/markdown");
+    await storage.put(`${prefix}/manual.json`, JSON.stringify({
+        version: 1,
+        projectId: project.id,
+        projectName: project.name,
+        digest: manual.digest,
+        bytes: manual.bytes,
+        ruleCount: manual.rules.length,
+        enforceRules: manual.rules
+            .filter(rule => rule.mode ===
+            "enforce")
+            .length,
+        advisoryRules: manual.rules
+            .filter(rule => rule.mode ===
+            "advisory")
+            .length,
+        updatedAt: manual.updatedAt,
+    }, null, 2) +
+        "\n", "application/json");
+    return manual;
+}

package/dist/src/project-manual/types.js

@@ -0,0 +1 @@
+export {};

package/dist/src/retrieval/bm25/index.js

@@ -0,0 +1 @@
+export {};

package/dist/src/retrieval/bm25/search.js

@@ -0,0 +1 @@
+export {};

package/dist/src/retrieval/bm25.js

@@ -0,0 +1,30 @@
+import { tokenize } from "./tokenizer.js";
+export class BM25Scorer {
+    score(query, memory) {
+        const queryTokens = tokenize(query);
+        const documentTokens = tokenize([
+            memory.content,
+            ...(memory.tags ?? []),
+            memory.type,
+        ].join(" "));
+        if (queryTokens.length === 0 ||
+            documentTokens.length === 0) {
+            return 0;
+        }
+        const frequencies = new Map();
+        for (const token of documentTokens) {
+            frequencies.set(token, (frequencies.get(token) ?? 0) + 1);
+        }
+        let score = 0;
+        for (const token of queryTokens) {
+            const tf = frequencies.get(token) ?? 0;
+            if (tf === 0) {
+                continue;
+            }
+            score +=
+                1 +
+                    Math.log(1 + tf);
+        }
+        return score / queryTokens.length;
+    }
+}

package/dist/src/retrieval/context-budget.js

@@ -0,0 +1 @@
+export {};

package/dist/src/retrieval/context-builder.js

@@ -0,0 +1,22 @@
+export class ContextBuilder {
+    build(results, maxCharacters = 6000) {
+        const blocks = [];
+        let used = 0;
+        for (const result of results) {
+            const memory = result.memory;
+            const block = [
+                `[${memory.type.toUpperCase()}]`,
+                memory.content,
+            ].join(" ");
+            if (used +
+                block.length >
+                maxCharacters) {
+                break;
+            }
+            blocks.push(block);
+            used +=
+                block.length;
+        }
+        return blocks.join("\n");
+    }
+}

package/dist/src/retrieval/graph/index.js

@@ -0,0 +1 @@
+export {};

package/dist/src/retrieval/graph/search.js

@@ -0,0 +1 @@
+export {};

package/dist/src/retrieval/hybrid/hybrid-search.js

@@ -0,0 +1 @@
+export {};

package/dist/src/retrieval/hybrid/index.js

@@ -0,0 +1 @@
+export {};

package/dist/src/retrieval/hybrid-search.js

@@ -0,0 +1,68 @@
+import { BM25Scorer, } from "./bm25.js";
+import { recencyScore, } from "./recency.js";
+import { memoryImportanceScore, } from "./importance.js";
+import { typePriority, } from "./type-priority.js";
+import { QueryAnalyzer, } from "./query-analyzer.js";
+export class HybridSearch {
+    bm25 = new BM25Scorer();
+    analyzer = new QueryAnalyzer();
+    search(query, memories, options = {}) {
+        const topK = options.topK ?? 8;
+        const minScore = options.minScore ?? 0.05;
+        const analysis = this.analyzer.analyze(query);
+        const results = [];
+        for (const memory of memories) {
+            if (options.types?.length &&
+                !options.types.includes(memory.type)) {
+                continue;
+            }
+            const keyword = this.bm25.score(query, memory);
+            const recency = recencyScore(memory);
+            const importance = memoryImportanceScore(memory);
+            let type = typePriority(memory.type);
+            if (analysis.preferredTypes
+                .includes(memory.type)) {
+                type = Math.min(1, type + 0.25);
+            }
+            /*
+             * Retrieval weighting:
+             *
+             * keyword    55%
+             * importance 20%
+             * recency    15%
+             * type       10%
+             */
+            const score = keyword * 0.55 +
+                importance * 0.20 +
+                recency * 0.15 +
+                type * 0.10;
+            /*
+             * Không cho memory hoàn toàn
+             * không liên quan lọt vào chỉ
+             * vì recency/importance cao.
+             */
+            // Keyword query phải có lexical match.
+            // Type/recency/importance chỉ dùng để rerank,
+            // không được tự kéo memory không liên quan vào Top-K.
+            if (keyword <= 0) {
+                continue;
+            }
+            if (score < minScore) {
+                continue;
+            }
+            results.push({
+                memory,
+                score,
+                scores: {
+                    keyword,
+                    recency,
+                    importance,
+                    type,
+                },
+            });
+        }
+        return results
+            .sort((a, b) => b.score - a.score)
+            .slice(0, topK);
+    }
+}

package/dist/src/retrieval/importance.js

@@ -0,0 +1,16 @@
+import { effectiveImportanceScore, } from "../memory/decay.js";
+const weights = {
+    critical: 1,
+    high: 0.8,
+    normal: 0.5,
+    temporary: 0.2,
+};
+export function importanceScore(importance) {
+    if (!importance) {
+        return 0.3;
+    }
+    return weights[importance] ?? 0.3;
+}
+export function memoryImportanceScore(memory) {
+    return (effectiveImportanceScore(memory) / 100);
+}

package/dist/src/retrieval/index.js

@@ -0,0 +1,12 @@
+export * from "./types.js";
+export * from "./tokenizer.js";
+export * from "./bm25.js";
+export * from "./recency.js";
+export * from "./importance.js";
+export * from "./type-priority.js";
+export * from "./query-analyzer.js";
+export * from "./hybrid-search.js";
+export * from "./context-builder.js";
+export * from "./retrieval-engine.js";
+export * from "./vector/index.js";
+export * from "./vector-hybrid-engine.js";

package/dist/src/retrieval/query-analyzer.js

@@ -0,0 +1,32 @@
+export class QueryAnalyzer {
+    analyze(query) {
+        const text = query.toLowerCase();
+        const preferredTypes = [];
+        if (text.includes("quyết định") ||
+            text.includes("decision") ||
+            text.includes("đã chọn")) {
+            preferredTypes.push("decision");
+        }
+        if (text.includes("todo") ||
+            text.includes("cần làm") ||
+            text.includes("tiếp theo") ||
+            text.includes("chưa xong")) {
+            preferredTypes.push("todo");
+        }
+        if (text.includes("quy tắc") ||
+            text.includes("rule") ||
+            text.includes("không được") ||
+            text.includes("bắt buộc")) {
+            preferredTypes.push("rule");
+        }
+        if (text.includes("tóm tắt") ||
+            text.includes("summary") ||
+            text.includes("đã làm")) {
+            preferredTypes.push("summary");
+        }
+        return {
+            query,
+            preferredTypes,
+        };
+    }
+}

package/dist/src/retrieval/query-classifier.js

@@ -0,0 +1 @@
+export {};

package/dist/src/retrieval/query-expansion.js

@@ -0,0 +1 @@
+export {};

package/dist/src/retrieval/recency.js

@@ -0,0 +1,12 @@
+export function recencyScore(memory, now = Date.now()) {
+    const raw = memory.updatedAt ??
+        memory.createdAt;
+    const timestamp = new Date(raw).getTime();
+    if (!Number.isFinite(timestamp)) {
+        return 0;
+    }
+    const ageDays = Math.max(0, now - timestamp) /
+        86_400_000;
+    // ~50% score sau 30 ngày
+    return Math.exp((-Math.LN2 * ageDays) / 30);
+}

package/dist/src/retrieval/reranker/index.js

@@ -0,0 +1 @@
+export {};

package/dist/src/retrieval/reranker/reranker.js

@@ -0,0 +1 @@
+export {};

package/dist/src/retrieval/retrieval-engine.js

@@ -0,0 +1,17 @@
+import { HybridSearch, } from "./hybrid-search.js";
+import { ContextBuilder, } from "./context-builder.js";
+export class RetrievalEngine {
+    memory;
+    searcher = new HybridSearch();
+    contextBuilder = new ContextBuilder();
+    constructor(memory) {
+        this.memory = memory;
+    }
+    search(projectId, query, options = {}) {
+        return this.searcher.search(query, this.memory.list(projectId), options);
+    }
+    context(projectId, query, options = {}) {
+        const results = this.search(projectId, query, options);
+        return this.contextBuilder.build(results);
+    }
+}

package/dist/src/retrieval/tokenizer.js

@@ -0,0 +1,9 @@
+export function tokenize(text) {
+    return text
+        .toLowerCase()
+        .normalize("NFKC")
+        .replace(/[^\p{L}\p{N}_\-./]+/gu, " ")
+        .split(/\s+/)
+        .map((token) => token.trim())
+        .filter((token) => token.length > 1);
+}

package/dist/src/retrieval/type-priority.js

@@ -0,0 +1,10 @@
+const priorities = {
+    rule: 1,
+    decision: 0.9,
+    todo: 0.8,
+    summary: 0.6,
+    activity: 0.3,
+};
+export function typePriority(type) {
+    return priorities[type] ?? 0.4;
+}

package/dist/src/retrieval/types.js

@@ -0,0 +1 @@
+export {};

package/dist/src/retrieval/vector/index.js

@@ -0,0 +1,4 @@
+export * from "./vector-store.js";
+export * from "./upsert.js";
+export * from "./search.js";
+export * from "./persistence.js";

package/dist/src/retrieval/vector/metadata-filter.js

@@ -0,0 +1 @@
+export {};

package/dist/src/retrieval/vector/persistence.js

@@ -0,0 +1,67 @@
+import { VectorIndexer, } from "./upsert.js";
+export class VectorPersistenceManager {
+    projectId;
+    model;
+    embeddings;
+    store;
+    persistence;
+    constructor(projectId, model, embeddings, store, persistence) {
+        this.projectId = projectId;
+        this.model = model;
+        this.embeddings = embeddings;
+        this.store = store;
+        this.persistence = persistence;
+    }
+    async initialize(memories) {
+        let loaded = 0;
+        const snapshot = await this.persistence.load(this.projectId);
+        if (snapshot &&
+            snapshot.model ===
+                this.model) {
+            loaded =
+                this.store.importRecords(snapshot.records);
+        }
+        /*
+         * Xóa vector không còn memory tương ứng.
+         */
+        const validIds = new Set(memories.map((memory) => memory.id));
+        let removed = 0;
+        for (const record of this.store.exportProject(this.projectId)) {
+            if (!validIds.has(record.id)) {
+                if (this.store.remove(record.id)) {
+                    removed++;
+                }
+            }
+        }
+        const indexer = new VectorIndexer(this.embeddings, this.store);
+        const indexed = await indexer.index(memories);
+        const records = this.store.exportProject(this.projectId);
+        const dimensions = records[0]
+            ?.vector.length ??
+            this.embeddings
+                .dimensions ??
+            0;
+        let saved = false;
+        if (indexed > 0 ||
+            removed > 0 ||
+            !snapshot) {
+            await this.persistence.save({
+                version: 1,
+                projectId: this.projectId,
+                model: this.model,
+                dimensions,
+                updatedAt: new Date()
+                    .toISOString(),
+                records,
+            });
+            saved = true;
+        }
+        return {
+            loaded,
+            indexed,
+            removed,
+            total: records.length,
+            saved,
+        };
+    }
+}

package/dist/src/retrieval/vector/search.js

@@ -0,0 +1,12 @@
+export class VectorSearch {
+    embeddings;
+    store;
+    constructor(embeddings, store) {
+        this.embeddings = embeddings;
+        this.store = store;
+    }
+    async search(projectId, query, limit = 10) {
+        const vector = await this.embeddings.embed(query);
+        return this.store.search(projectId, vector, limit);
+    }
+}

package/dist/src/retrieval/vector/test-persistence.js

@@ -0,0 +1,36 @@
+import "dotenv/config";
+import { MemoryEngine, ProjectManager, loadConfig, } from "../../core/index.js";
+import { createEmbeddingProvider, } from "../../embeddings/index.js";
+import { createStorageProvider, MemoryStore, PersistentVectorStore, ProjectScopedStorageProvider, } from "../../storage/index.js";
+import { VectorPersistenceManager, VectorStore, } from "./index.js";
+async function main() {
+    const config = loadConfig();
+    const project = new ProjectManager()
+        .detect();
+    const memory = new MemoryEngine();
+    const rawStorage = createStorageProvider({
+        provider: config.storage.provider,
+        huggingface: config.storage.huggingface,
+        localRoot: config.storage.localRoot,
+    });
+    const storage = new ProjectScopedStorageProvider(rawStorage, project.id, project.name, project.remote ?? project.name);
+    const memoryStore = new MemoryStore(storage);
+    memory.importRecords(await memoryStore.load(project.id));
+    const embeddings = createEmbeddingProvider();
+    const vectors = new VectorStore();
+    const persistence = new PersistentVectorStore(storage);
+    const model = process.env
+        .HF_EMBEDDING_MODEL ??
+        "sentence-transformers/all-MiniLM-L6-v2";
+    const manager = new VectorPersistenceManager(project.id, model, embeddings, vectors, persistence);
+    const stats = await manager.initialize(memory.list(project.id));
+    console.log({
+        ok: true,
+        memories: memory.list(project.id).length,
+        ...stats,
+    });
+}
+main().catch((error) => {
+    console.error(error);
+    process.exit(1);
+});

package/dist/src/retrieval/vector/upsert.js

@@ -0,0 +1,40 @@
+export class VectorIndexer {
+    embeddings;
+    store;
+    constructor(embeddings, store) {
+        this.embeddings = embeddings;
+        this.store = store;
+    }
+    async index(memories) {
+        const pending = memories.filter((memory) => !this.store.has(memory.id));
+        if (pending.length === 0) {
+            return 0;
+        }
+        const vectors = await this.embeddings
+            .embedMany(pending.map((memory) => [
+            memory.type,
+            memory.content,
+            ...(memory.tags ?? []),
+        ].join(" ")));
+        let indexed = 0;
+        for (let i = 0; i < pending.length; i++) {
+            const memory = pending[i];
+            const vector = vectors[i];
+            if (!memory ||
+                !vector) {
+                continue;
+            }
+            this.store.upsert({
+                id: memory.id,
+                projectId: memory.projectId,
+                vector,
+                metadata: {
+                    type: memory.type,
+                    updatedAt: memory.updatedAt,
+                },
+            });
+            indexed++;
+        }
+        return indexed;
+    }
+}