toolnet-memory 0.2.0 → 0.2.2

package/dist/src/project-manual/manager.js

@@ -1,147 +0,0 @@
-import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync, } from "node:fs";
-import { dirname, join, } from "node:path";
-import { sha256, } from "../session/utils.js";
-const MAX_MANUAL_BYTES = 64 * 1024;
-export const PROJECT_MANUAL_TEMPLATE = `# ToolNet Project Operating Manual
-
-This file contains persistent instructions for AI agents working on this project.
-
-## Critical Rules
-
-<!--
-Examples:
-
-- [enforce] Never edit production files directly.
-- [enforce] Edit source only in /path/to/source.
-- [enforce] Deploy only with ./deploy.sh --apply
--->
-
-## Workflow
-
-<!--
-Describe the correct working process.
--->
-
-## Architecture
-
-<!--
-Important architecture conventions.
--->
-
-## Verification
-
-<!--
-Tests / QA required after changes.
--->
-
-## Known Gotchas
-
-<!--
-Things an AI agent should always remember.
--->
-
-## Preferences
-
-<!--
-- [advisory] Prefer small focused files.
--->
-`;
-export function projectManualPath(project) {
-    return join(project.rootPath, ".toolnet", "PROJECT.md");
-}
-function normalizeRuleText(value) {
-    return value
-        .normalize("NFKC")
-        .replace(/\s+/g, " ")
-        .trim();
-}
-function parseRules(content) {
-    const rules = [];
-    const seen = new Set();
-    const pattern = /^\s*[-*]\s+\[(enforce|advisory)\]\s+(.+?)\s*$/gimu;
-    let match;
-    while ((match =
-        pattern.exec(content))) {
-        const mode = match[1]
-            .toLowerCase();
-        const text = normalizeRuleText(match[2]);
-        if (!text) {
-            continue;
-        }
-        const key = `${mode}:${text.toLowerCase()}`;
-        if (seen.has(key)) {
-            continue;
-        }
-        seen.add(key);
-        rules.push({
-            id: sha256(key).slice(0, 24),
-            mode,
-            text,
-            source: "manual",
-        });
-    }
-    return rules;
-}
-export function ensureProjectManual(project) {
-    const file = projectManualPath(project);
-    if (existsSync(file)) {
-        return file;
-    }
-    mkdirSync(dirname(file), {
-        recursive: true,
-    });
-    writeFileSync(file, PROJECT_MANUAL_TEMPLATE, {
-        encoding: "utf8",
-        mode: 0o600,
-    });
-    return file;
-}
-export function loadProjectManual(project, createIfMissing = false) {
-    const file = createIfMissing
-        ? ensureProjectManual(project)
-        : projectManualPath(project);
-    if (!existsSync(file)) {
-        return null;
-    }
-    const size = statSync(file).size;
-    if (size >
-        MAX_MANUAL_BYTES) {
-        throw new Error(`PROJECT.md exceeds ${MAX_MANUAL_BYTES} bytes`);
-    }
-    const content = readFileSync(file, "utf8");
-    return {
-        path: file,
-        content,
-        digest: sha256(content),
-        rules: parseRules(content),
-        bytes: Buffer.byteLength(content, "utf8"),
-        updatedAt: new Date(statSync(file).mtimeMs).toISOString(),
-    };
-}
-export async function syncProjectManual(project, storage) {
-    const manual = loadProjectManual(project, true);
-    if (!manual) {
-        throw new Error("Project manual unavailable");
-    }
-    const prefix = `projects/${project.id}/project`;
-    await storage.put(`${prefix}/manual.md`, manual.content, "text/markdown");
-    await storage.put(`${prefix}/manual.json`, JSON.stringify({
-        version: 1,
-        projectId: project.id,
-        projectName: project.name,
-        digest: manual.digest,
-        bytes: manual.bytes,
-        ruleCount: manual.rules.length,
-        enforceRules: manual.rules
-            .filter(rule => rule.mode ===
-            "enforce")
-            .length,
-        advisoryRules: manual.rules
-            .filter(rule => rule.mode ===
-            "advisory")
-            .length,
-        updatedAt: manual.updatedAt,
-    }, null, 2) +
-        "\n", "application/json");
-    return manual;
-}

package/dist/src/project-manual/types.js

@@ -1 +0,0 @@
-export {};

package/dist/src/retrieval/bm25/index.js

@@ -1 +0,0 @@
-export {};

package/dist/src/retrieval/bm25/search.js

@@ -1 +0,0 @@
-export {};

package/dist/src/retrieval/bm25.js

@@ -1,30 +0,0 @@
-import { tokenize } from "./tokenizer.js";
-export class BM25Scorer {
-    score(query, memory) {
-        const queryTokens = tokenize(query);
-        const documentTokens = tokenize([
-            memory.content,
-            ...(memory.tags ?? []),
-            memory.type,
-        ].join(" "));
-        if (queryTokens.length === 0 ||
-            documentTokens.length === 0) {
-            return 0;
-        }
-        const frequencies = new Map();
-        for (const token of documentTokens) {
-            frequencies.set(token, (frequencies.get(token) ?? 0) + 1);
-        }
-        let score = 0;
-        for (const token of queryTokens) {
-            const tf = frequencies.get(token) ?? 0;
-            if (tf === 0) {
-                continue;
-            }
-            score +=
-                1 +
-                    Math.log(1 + tf);
-        }
-        return score / queryTokens.length;
-    }
-}

package/dist/src/retrieval/context-budget.js

@@ -1 +0,0 @@
-export {};

package/dist/src/retrieval/context-builder.js

@@ -1,22 +0,0 @@
-export class ContextBuilder {
-    build(results, maxCharacters = 6000) {
-        const blocks = [];
-        let used = 0;
-        for (const result of results) {
-            const memory = result.memory;
-            const block = [
-                `[${memory.type.toUpperCase()}]`,
-                memory.content,
-            ].join(" ");
-            if (used +
-                block.length >
-                maxCharacters) {
-                break;
-            }
-            blocks.push(block);
-            used +=
-                block.length;
-        }
-        return blocks.join("\n");
-    }
-}

package/dist/src/retrieval/graph/index.js

@@ -1 +0,0 @@
-export {};

package/dist/src/retrieval/graph/search.js

@@ -1 +0,0 @@
-export {};

package/dist/src/retrieval/hybrid/hybrid-search.js

@@ -1 +0,0 @@
-export {};

package/dist/src/retrieval/hybrid/index.js

@@ -1 +0,0 @@
-export {};

package/dist/src/retrieval/hybrid-search.js

@@ -1,68 +0,0 @@
-import { BM25Scorer, } from "./bm25.js";
-import { recencyScore, } from "./recency.js";
-import { memoryImportanceScore, } from "./importance.js";
-import { typePriority, } from "./type-priority.js";
-import { QueryAnalyzer, } from "./query-analyzer.js";
-export class HybridSearch {
-    bm25 = new BM25Scorer();
-    analyzer = new QueryAnalyzer();
-    search(query, memories, options = {}) {
-        const topK = options.topK ?? 8;
-        const minScore = options.minScore ?? 0.05;
-        const analysis = this.analyzer.analyze(query);
-        const results = [];
-        for (const memory of memories) {
-            if (options.types?.length &&
-                !options.types.includes(memory.type)) {
-                continue;
-            }
-            const keyword = this.bm25.score(query, memory);
-            const recency = recencyScore(memory);
-            const importance = memoryImportanceScore(memory);
-            let type = typePriority(memory.type);
-            if (analysis.preferredTypes
-                .includes(memory.type)) {
-                type = Math.min(1, type + 0.25);
-            }
-            /*
-             * Retrieval weighting:
-             *
-             * keyword    55%
-             * importance 20%
-             * recency    15%
-             * type       10%
-             */
-            const score = keyword * 0.55 +
-                importance * 0.20 +
-                recency * 0.15 +
-                type * 0.10;
-            /*
-             * Không cho memory hoàn toàn
-             * không liên quan lọt vào chỉ
-             * vì recency/importance cao.
-             */
-            // Keyword query phải có lexical match.
-            // Type/recency/importance chỉ dùng để rerank,
-            // không được tự kéo memory không liên quan vào Top-K.
-            if (keyword <= 0) {
-                continue;
-            }
-            if (score < minScore) {
-                continue;
-            }
-            results.push({
-                memory,
-                score,
-                scores: {
-                    keyword,
-                    recency,
-                    importance,
-                    type,
-                },
-            });
-        }
-        return results
-            .sort((a, b) => b.score - a.score)
-            .slice(0, topK);
-    }
-}

package/dist/src/retrieval/importance.js

@@ -1,16 +0,0 @@
-import { effectiveImportanceScore, } from "../memory/decay.js";
-const weights = {
-    critical: 1,
-    high: 0.8,
-    normal: 0.5,
-    temporary: 0.2,
-};
-export function importanceScore(importance) {
-    if (!importance) {
-        return 0.3;
-    }
-    return weights[importance] ?? 0.3;
-}
-export function memoryImportanceScore(memory) {
-    return (effectiveImportanceScore(memory) / 100);
-}

package/dist/src/retrieval/index.js

@@ -1,12 +0,0 @@
-export * from "./types.js";
-export * from "./tokenizer.js";
-export * from "./bm25.js";
-export * from "./recency.js";
-export * from "./importance.js";
-export * from "./type-priority.js";
-export * from "./query-analyzer.js";
-export * from "./hybrid-search.js";
-export * from "./context-builder.js";
-export * from "./retrieval-engine.js";
-export * from "./vector/index.js";
-export * from "./vector-hybrid-engine.js";

package/dist/src/retrieval/query-analyzer.js

@@ -1,32 +0,0 @@
-export class QueryAnalyzer {
-    analyze(query) {
-        const text = query.toLowerCase();
-        const preferredTypes = [];
-        if (text.includes("quyết định") ||
-            text.includes("decision") ||
-            text.includes("đã chọn")) {
-            preferredTypes.push("decision");
-        }
-        if (text.includes("todo") ||
-            text.includes("cần làm") ||
-            text.includes("tiếp theo") ||
-            text.includes("chưa xong")) {
-            preferredTypes.push("todo");
-        }
-        if (text.includes("quy tắc") ||
-            text.includes("rule") ||
-            text.includes("không được") ||
-            text.includes("bắt buộc")) {
-            preferredTypes.push("rule");
-        }
-        if (text.includes("tóm tắt") ||
-            text.includes("summary") ||
-            text.includes("đã làm")) {
-            preferredTypes.push("summary");
-        }
-        return {
-            query,
-            preferredTypes,
-        };
-    }
-}

package/dist/src/retrieval/query-classifier.js

@@ -1 +0,0 @@
-export {};

package/dist/src/retrieval/query-expansion.js

@@ -1 +0,0 @@
-export {};

package/dist/src/retrieval/recency.js

@@ -1,12 +0,0 @@
-export function recencyScore(memory, now = Date.now()) {
-    const raw = memory.updatedAt ??
-        memory.createdAt;
-    const timestamp = new Date(raw).getTime();
-    if (!Number.isFinite(timestamp)) {
-        return 0;
-    }
-    const ageDays = Math.max(0, now - timestamp) /
-        86_400_000;
-    // ~50% score sau 30 ngày
-    return Math.exp((-Math.LN2 * ageDays) / 30);
-}

package/dist/src/retrieval/reranker/index.js

@@ -1 +0,0 @@
-export {};

package/dist/src/retrieval/reranker/reranker.js

@@ -1 +0,0 @@
-export {};

package/dist/src/retrieval/retrieval-engine.js

@@ -1,17 +0,0 @@
-import { HybridSearch, } from "./hybrid-search.js";
-import { ContextBuilder, } from "./context-builder.js";
-export class RetrievalEngine {
-    memory;
-    searcher = new HybridSearch();
-    contextBuilder = new ContextBuilder();
-    constructor(memory) {
-        this.memory = memory;
-    }
-    search(projectId, query, options = {}) {
-        return this.searcher.search(query, this.memory.list(projectId), options);
-    }
-    context(projectId, query, options = {}) {
-        const results = this.search(projectId, query, options);
-        return this.contextBuilder.build(results);
-    }
-}

package/dist/src/retrieval/tokenizer.js

@@ -1,9 +0,0 @@
-export function tokenize(text) {
-    return text
-        .toLowerCase()
-        .normalize("NFKC")
-        .replace(/[^\p{L}\p{N}_\-./]+/gu, " ")
-        .split(/\s+/)
-        .map((token) => token.trim())
-        .filter((token) => token.length > 1);
-}

package/dist/src/retrieval/type-priority.js

@@ -1,10 +0,0 @@
-const priorities = {
-    rule: 1,
-    decision: 0.9,
-    todo: 0.8,
-    summary: 0.6,
-    activity: 0.3,
-};
-export function typePriority(type) {
-    return priorities[type] ?? 0.4;
-}

package/dist/src/retrieval/types.js

@@ -1 +0,0 @@
-export {};

package/dist/src/retrieval/vector/index.js

@@ -1,4 +0,0 @@
-export * from "./vector-store.js";
-export * from "./upsert.js";
-export * from "./search.js";
-export * from "./persistence.js";

package/dist/src/retrieval/vector/metadata-filter.js

@@ -1 +0,0 @@
-export {};

package/dist/src/retrieval/vector/persistence.js

@@ -1,67 +0,0 @@
-import { VectorIndexer, } from "./upsert.js";
-export class VectorPersistenceManager {
-    projectId;
-    model;
-    embeddings;
-    store;
-    persistence;
-    constructor(projectId, model, embeddings, store, persistence) {
-        this.projectId = projectId;
-        this.model = model;
-        this.embeddings = embeddings;
-        this.store = store;
-        this.persistence = persistence;
-    }
-    async initialize(memories) {
-        let loaded = 0;
-        const snapshot = await this.persistence.load(this.projectId);
-        if (snapshot &&
-            snapshot.model ===
-                this.model) {
-            loaded =
-                this.store.importRecords(snapshot.records);
-        }
-        /*
-         * Xóa vector không còn memory tương ứng.
-         */
-        const validIds = new Set(memories.map((memory) => memory.id));
-        let removed = 0;
-        for (const record of this.store.exportProject(this.projectId)) {
-            if (!validIds.has(record.id)) {
-                if (this.store.remove(record.id)) {
-                    removed++;
-                }
-            }
-        }
-        const indexer = new VectorIndexer(this.embeddings, this.store);
-        const indexed = await indexer.index(memories);
-        const records = this.store.exportProject(this.projectId);
-        const dimensions = records[0]
-            ?.vector.length ??
-            this.embeddings
-                .dimensions ??
-            0;
-        let saved = false;
-        if (indexed > 0 ||
-            removed > 0 ||
-            !snapshot) {
-            await this.persistence.save({
-                version: 1,
-                projectId: this.projectId,
-                model: this.model,
-                dimensions,
-                updatedAt: new Date()
-                    .toISOString(),
-                records,
-            });
-            saved = true;
-        }
-        return {
-            loaded,
-            indexed,
-            removed,
-            total: records.length,
-            saved,
-        };
-    }
-}

package/dist/src/retrieval/vector/search.js

@@ -1,12 +0,0 @@
-export class VectorSearch {
-    embeddings;
-    store;
-    constructor(embeddings, store) {
-        this.embeddings = embeddings;
-        this.store = store;
-    }
-    async search(projectId, query, limit = 10) {
-        const vector = await this.embeddings.embed(query);
-        return this.store.search(projectId, vector, limit);
-    }
-}

package/dist/src/retrieval/vector/test-persistence.js

@@ -1,36 +0,0 @@
-import "dotenv/config";
-import { MemoryEngine, ProjectManager, loadConfig, } from "../../core/index.js";
-import { createEmbeddingProvider, } from "../../embeddings/index.js";
-import { createStorageProvider, MemoryStore, PersistentVectorStore, ProjectScopedStorageProvider, } from "../../storage/index.js";
-import { VectorPersistenceManager, VectorStore, } from "./index.js";
-async function main() {
-    const config = loadConfig();
-    const project = new ProjectManager()
-        .detect();
-    const memory = new MemoryEngine();
-    const rawStorage = createStorageProvider({
-        provider: config.storage.provider,
-        huggingface: config.storage.huggingface,
-        localRoot: config.storage.localRoot,
-    });
-    const storage = new ProjectScopedStorageProvider(rawStorage, project.id, project.name, project.remote ?? project.name);
-    const memoryStore = new MemoryStore(storage);
-    memory.importRecords(await memoryStore.load(project.id));
-    const embeddings = createEmbeddingProvider();
-    const vectors = new VectorStore();
-    const persistence = new PersistentVectorStore(storage);
-    const model = process.env
-        .HF_EMBEDDING_MODEL ??
-        "sentence-transformers/all-MiniLM-L6-v2";
-    const manager = new VectorPersistenceManager(project.id, model, embeddings, vectors, persistence);
-    const stats = await manager.initialize(memory.list(project.id));
-    console.log({
-        ok: true,
-        memories: memory.list(project.id).length,
-        ...stats,
-    });
-}
-main().catch((error) => {
-    console.error(error);
-    process.exit(1);
-});

package/dist/src/retrieval/vector/upsert.js

@@ -1,40 +0,0 @@
-export class VectorIndexer {
-    embeddings;
-    store;
-    constructor(embeddings, store) {
-        this.embeddings = embeddings;
-        this.store = store;
-    }
-    async index(memories) {
-        const pending = memories.filter((memory) => !this.store.has(memory.id));
-        if (pending.length === 0) {
-            return 0;
-        }
-        const vectors = await this.embeddings
-            .embedMany(pending.map((memory) => [
-            memory.type,
-            memory.content,
-            ...(memory.tags ?? []),
-        ].join(" ")));
-        let indexed = 0;
-        for (let i = 0; i < pending.length; i++) {
-            const memory = pending[i];
-            const vector = vectors[i];
-            if (!memory ||
-                !vector) {
-                continue;
-            }
-            this.store.upsert({
-                id: memory.id,
-                projectId: memory.projectId,
-                vector,
-                metadata: {
-                    type: memory.type,
-                    updatedAt: memory.updatedAt,
-                },
-            });
-            indexed++;
-        }
-        return indexed;
-    }
-}