yourmemory 1.4.73__tar.gz → 1.4.76__tar.gz

{yourmemory-1.4.73 → yourmemory-1.4.76}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: yourmemory
-Version: 1.4.73
+Version: 1.4.76
 Summary: Persistent memory for Claude — Ebbinghaus forgetting curve, semantic deduplication, MCP-native
 Author: Sachit Misra
 Author-email: mishrasachit1@gmail.com

{yourmemory-1.4.73 → yourmemory-1.4.76}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "yourmemory"
-version = "1.4.73"
+version = "1.4.76"
 description = "Persistent memory for Claude — Ebbinghaus forgetting curve, semantic deduplication, MCP-native"
 readme = "README.md"
 requires-python = ">=3.11"

{yourmemory-1.4.73 → yourmemory-1.4.76}/src/app.py

@@ -8,7 +8,12 @@ from apscheduler.schedulers.asyncio import AsyncIOScheduler
 from fastapi import FastAPI
 from fastapi.responses import StreamingResponse
 from pydantic import BaseModel
-from src.routes import memories, retrieve, agents, ui, graph_viz, proxy, audit
+import time
+from collections import defaultdict, deque
+from threading import Lock
+from fastapi import Request
+from fastapi.responses import JSONResponse
+from src.routes import memories, retrieve, agents, ui, graph_viz, proxy, audit, dsar, compact
 from src.jobs.decay_job import run as run_decay
 from src.services.audit import prune_expired as prune_audit
 from src.db.migrate import migrate
@@ -23,6 +28,8 @@ def _daily_jobs():
         prune_audit()   # retention: drop audit rows older than the (>=90-day) window
     except Exception:
         pass
+    # Note: memory compaction is event-driven (triggered on store when a cluster reaches
+    # N), not a daily sweep — see /auto-store and src/services/compaction.py.
 
 
 @asynccontextmanager
@@ -43,6 +50,43 @@ app.include_router(ui.router)
 app.include_router(graph_viz.router)
 app.include_router(proxy.router)
 app.include_router(audit.router)
+app.include_router(dsar.router)
+app.include_router(compact.router)
+
+
+# ── Rate limiting (abuse prevention) ────────────────────────────────────────────
+# Per-client sliding-window limiter. Loopback (127.0.0.1) is exempt by default — the
+# local trust boundary is the OS user, and the recall/store hooks burst legitimately;
+# the limit targets network-exposed (hosted) clients. Configure with:
+#   YOURMEMORY_RATE_LIMIT   max requests per window per client  (default 300; 0 disables)
+#   YOURMEMORY_RATE_WINDOW  window length in seconds            (default 60)
+#   YOURMEMORY_RATE_LIMIT_LOOPBACK=1  also limit loopback clients
+_RATE_LIMIT  = int(os.getenv("YOURMEMORY_RATE_LIMIT", "300"))
+_RATE_WINDOW = int(os.getenv("YOURMEMORY_RATE_WINDOW", "60"))
+_RATE_LOOPBACK = os.getenv("YOURMEMORY_RATE_LIMIT_LOOPBACK", "0") == "1"
+_rate_hits: dict = defaultdict(deque)
+_rate_lock = Lock()
+_LOOPBACK = {"127.0.0.1", "::1", "localhost"}
+
+
+@app.middleware("http")
+async def _rate_limit(request: Request, call_next):
+    if _RATE_LIMIT > 0 and request.url.path != "/health":
+        client = request.client.host if request.client else "unknown"
+        if _RATE_LOOPBACK or client not in _LOOPBACK:
+            now = time.time()
+            with _rate_lock:
+                dq = _rate_hits[client]
+                cutoff = now - _RATE_WINDOW
+                while dq and dq[0] < cutoff:
+                    dq.popleft()
+                if len(dq) >= _RATE_LIMIT:
+                    retry = max(1, int(dq[0] + _RATE_WINDOW - now))
+                    return JSONResponse(
+                        {"error": "rate limit exceeded", "retry_after": retry},
+                        status_code=429, headers={"Retry-After": str(retry)})
+                dq.append(now)
+    return await call_next(request)
 
 
 @app.get("/health")
@@ -616,4 +660,18 @@ def auto_store_endpoint(req: AutoStoreRequest):
     except Exception:
         pass
 
+    # Event-driven compaction: if a just-stored fact's cluster now has >= N closely
+    # related memories, compress that cluster immediately so the store stays lean —
+    # no daily sweep needed. On by default; disable with YOURMEMORY_COMPACTION=0.
+    if os.getenv("YOURMEMORY_COMPACTION", "1") == "1" and to_index:
+        try:
+            from src.services.compaction import maybe_compact_around
+            seen = set()
+            for _mid, content, _imp, _cat, _emb in to_index:
+                if content and content not in seen:
+                    seen.add(content)
+                    maybe_compact_around(user_id, content)
+        except Exception:
+            pass
+
     return {"stored": len(stored), "facts": stored}

{yourmemory-1.4.73 → yourmemory-1.4.76}/src/db/migrate.py

@@ -160,6 +160,66 @@ def _create_audit_table(conn, backend: str) -> None:
         """)
 
 
+def _create_archive_table(conn, backend: str) -> None:
+    """Holds originals that were compressed into a summary memory. Lets the live
+    `memories` table stay lean (so recall stays fast and clean) while keeping the
+    pre-compression facts for reversibility and audit. Idempotent across backends."""
+    if backend == "postgres":
+        cur = conn.cursor()
+        cur.execute("""
+            CREATE TABLE IF NOT EXISTS memory_archive (
+                orig_id     BIGINT,
+                user_id     TEXT NOT NULL,
+                content     TEXT NOT NULL,
+                category    TEXT,
+                importance  DOUBLE PRECISION,
+                agent_id    TEXT,
+                visibility  TEXT,
+                created_at  TIMESTAMPTZ,
+                archived_at TIMESTAMPTZ DEFAULT NOW(),
+                summary_id  BIGINT
+            );
+            CREATE INDEX IF NOT EXISTS idx_archive_user ON memory_archive(user_id);
+        """)
+        conn.commit()
+        cur.close()
+    elif backend == "duckdb":
+        try:
+            conn.execute("""
+                CREATE TABLE IF NOT EXISTS memory_archive (
+                    orig_id     BIGINT,
+                    user_id     VARCHAR NOT NULL,
+                    content     VARCHAR NOT NULL,
+                    category    VARCHAR,
+                    importance  DOUBLE,
+                    agent_id    VARCHAR,
+                    visibility  VARCHAR,
+                    created_at  TIMESTAMP,
+                    archived_at TIMESTAMP DEFAULT now(),
+                    summary_id  BIGINT
+                )
+            """)
+            conn.execute("CREATE INDEX IF NOT EXISTS idx_archive_user ON memory_archive(user_id)")
+        except Exception as exc:
+            print(f"archive table (duckdb) skipped: {exc}", file=sys.stderr)
+    else:  # sqlite
+        conn.executescript("""
+            CREATE TABLE IF NOT EXISTS memory_archive (
+                orig_id     INTEGER,
+                user_id     TEXT NOT NULL,
+                content     TEXT NOT NULL,
+                category    TEXT,
+                importance  REAL,
+                agent_id    TEXT,
+                visibility  TEXT,
+                created_at  TIMESTAMP,
+                archived_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+                summary_id  INTEGER
+            );
+            CREATE INDEX IF NOT EXISTS idx_archive_user ON memory_archive(user_id);
+        """)
+
+
 def migrate():
     backend = get_backend()
 
@@ -199,6 +259,9 @@ def migrate():
     # ── Append-only, hash-chained audit log (read/write/delete, 90-day+ retention) ──
     _create_audit_table(conn, backend)
 
+    # ── Archive of originals compressed into summaries (memory compaction) ──
+    _create_archive_table(conn, backend)
+
     # ── Post-schema FTS setup ─────────────────────────────────────────────
     if backend == "sqlite":
         # Backfill any rows that existed before the FTS table was created.

yourmemory-1.4.76/src/routes/compact.py

@@ -0,0 +1,53 @@
+"""
+Memory compaction endpoints.
+
+  POST /compact?userId=…        → run compaction now for a user (returns stats)
+  GET  /users/{user_id}/archive → view memories that were compressed into summaries
+
+Compaction also runs daily when YOURMEMORY_COMPACTION=1. See src/services/compaction.py.
+"""
+
+from fastapi import APIRouter, Query
+from typing import Optional
+
+from src.services.compaction import compact_user
+from src.db.connection import get_backend, get_conn, duckdb_rows
+
+router = APIRouter()
+
+
+@router.post("/compact")
+def run_compaction(
+    userId: str = Query(..., description="User whose memories to compact"),
+    minCluster: Optional[int] = Query(None, ge=2, le=100),
+    simThreshold: Optional[float] = Query(None, ge=0.3, le=0.99),
+):
+    return compact_user(userId, min_cluster=minCluster, sim_threshold=simThreshold)
+
+
+@router.get("/users/{user_id}/archive")
+def view_archive(user_id: str, limit: int = Query(100, ge=1, le=1000)):
+    """Originals that were compressed into summaries (newest first)."""
+    user_id = user_id.strip().lower()
+    backend = get_backend()
+    conn = get_conn()
+    sql_pg    = ("SELECT orig_id, content, category, summary_id, archived_at FROM memory_archive "
+                 "WHERE user_id = %s ORDER BY archived_at DESC LIMIT %s")
+    sql_other = ("SELECT orig_id, content, category, summary_id, archived_at FROM memory_archive "
+                 "WHERE user_id = ? ORDER BY archived_at DESC LIMIT ?")
+    cols = ["orig_id", "content", "category", "summary_id", "archived_at"]
+    try:
+        if backend == "postgres":
+            cur = conn.cursor(); cur.execute(sql_pg, (user_id, limit))
+            rows = [dict(zip(cols, r)) for r in cur.fetchall()]; cur.close()
+        elif backend == "duckdb":
+            rows = duckdb_rows(conn.execute(sql_other, [user_id, limit]))
+        else:
+            cur = conn.cursor(); cur.execute(sql_other, (user_id, limit))
+            rows = [dict(zip(cols, r)) for r in cur.fetchall()]; cur.close()
+    finally:
+        conn.close()
+    for r in rows:
+        if r.get("archived_at") is not None:
+            r["archived_at"] = str(r["archived_at"])
+    return {"count": len(rows), "archived": rows}

yourmemory-1.4.76/src/routes/dsar.py

@@ -0,0 +1,197 @@
+"""
+DSAR / data-portability endpoints — export, right-to-forget purge, and bulk import.
+
+  GET    /users/{user_id}/export    → full JSON of a user's memories (DSAR export / backup)
+  DELETE /users/{user_id}/memories  → purge ALL of a user's data (right-to-forget)
+  POST   /users/{user_id}/import    → bulk restore/seed memories from an export
+
+Every operation is recorded in the audit trail. In the default local deployment the
+path `user_id` identifies the data subject (trust boundary is the OS user); a hosted
+deployment must additionally authenticate the caller before these are exposed.
+
+See docs/policies/06-data-retention-deletion-policy.md.
+"""
+
+from fastapi import APIRouter, HTTPException
+from pydantic import BaseModel
+from typing import Optional, List
+
+from src.services.embed import embed
+from src.services.extract import categorize
+from src.db.connection import get_backend, get_conn, emb_to_db, duckdb_rows
+from src.services.audit import log_event
+
+router = APIRouter()
+
+_EXPORT_COLS = ("id, content, category, importance, recall_count, "
+                "last_accessed_at, created_at, agent_id, visibility, context_paths")
+_EXPORT_KEYS = ["id", "content", "category", "importance", "recall_count",
+                "last_accessed_at", "created_at", "agent_id", "visibility", "context_paths"]
+
+
+def _rows_for_user(conn, backend: str, user_id: str) -> list[dict]:
+    sql_pg    = f"SELECT {_EXPORT_COLS} FROM memories WHERE user_id = %s ORDER BY id"
+    sql_other = f"SELECT {_EXPORT_COLS} FROM memories WHERE user_id = ? ORDER BY id"
+    if backend == "postgres":
+        from psycopg2.extras import RealDictCursor
+        cur = conn.cursor(cursor_factory=RealDictCursor)
+        cur.execute(sql_pg, (user_id,))
+        rows = [dict(r) for r in cur.fetchall()]
+        cur.close()
+        return rows
+    if backend == "duckdb":
+        return duckdb_rows(conn.execute(sql_other, [user_id]))
+    cur = conn.cursor()
+    cur.execute(sql_other, (user_id,))
+    cols = [d[0] for d in cur.description]
+    rows = [dict(zip(cols, r)) for r in cur.fetchall()]
+    cur.close()
+    return rows
+
+
+# ── GET /users/{user_id}/export ────────────────────────────────────────────────
+
+@router.get("/users/{user_id}/export")
+def export_user(user_id: str):
+    """Return all of a user's memories as JSON (DSAR access request / backup)."""
+    user_id = user_id.strip().lower()
+    backend = get_backend()
+    conn = get_conn()
+    try:
+        rows = _rows_for_user(conn, backend, user_id)
+    finally:
+        conn.close()
+
+    # Stringify timestamps for clean JSON.
+    for r in rows:
+        for k in ("last_accessed_at", "created_at"):
+            if r.get(k) is not None:
+                r[k] = str(r[k])
+
+    from datetime import datetime, timezone
+    log_event("read", "export", user_id, detail={"count": len(rows)})
+    return {
+        "user_id":     user_id,
+        "exported_at": datetime.now(timezone.utc).isoformat(),
+        "count":       len(rows),
+        "memories":    rows,
+    }
+
+
+# ── DELETE /users/{user_id}/memories  (right-to-forget) ─────────────────────────
+
+@router.delete("/users/{user_id}/memories")
+def purge_user(user_id: str):
+    """Delete ALL of a user's memories, graph nodes, and conversation buffer."""
+    user_id = user_id.strip().lower()
+    backend = get_backend()
+    conn = get_conn()
+    deleted = 0
+    try:
+        # Count first (for the audit detail + response).
+        if backend == "postgres":
+            cur = conn.cursor()
+            cur.execute("SELECT COUNT(*) FROM memories WHERE user_id = %s", (user_id,))
+            deleted = cur.fetchone()[0]
+            cur.execute("DELETE FROM memories WHERE user_id = %s", (user_id,))
+            cur.execute("DELETE FROM conversation_buffer WHERE user_id = %s", (user_id,))
+            conn.commit()
+            cur.close()
+        elif backend == "duckdb":
+            deleted = conn.execute("SELECT COUNT(*) FROM memories WHERE user_id = ?", [user_id]).fetchone()[0]
+            conn.execute("DELETE FROM memories WHERE user_id = ?", [user_id])
+            try:
+                conn.execute("DELETE FROM conversation_buffer WHERE user_id = ?", [user_id])
+            except Exception:
+                pass
+        else:  # sqlite
+            cur = conn.cursor()
+            cur.execute("SELECT COUNT(*) FROM memories WHERE user_id = ?", (user_id,))
+            deleted = cur.fetchone()[0]
+            cur.execute("DELETE FROM memories WHERE user_id = ?", (user_id,))
+            try:
+                cur.execute("DELETE FROM conversation_buffer WHERE user_id = ?", (user_id,))
+            except Exception:
+                pass
+            conn.commit()
+            cur.close()
+    finally:
+        conn.close()
+
+    # Best-effort: drop the user's graph nodes so recall can't resurface them.
+    try:
+        from src.graph import get_graph_backend
+        gb = get_graph_backend()
+        for node in gb.get_all_nodes_for_user(user_id):
+            try:
+                gb.delete_node(node["memory_id"])
+            except Exception:
+                pass
+    except Exception:
+        pass
+
+    # The audit entry is retained (immutability / accountability) — it holds only the
+    # user id + count, never memory content, so it is not personal content.
+    log_event("delete", "purge", user_id, detail={"count": deleted})
+    return {"purged": True, "user_id": user_id, "deleted": deleted}
+
+
+# ── POST /users/{user_id}/import ────────────────────────────────────────────────
+
+class ImportRequest(BaseModel):
+    memories: List[dict]
+    overwrite: bool = False   # reserved; ON CONFLICT already upserts by (user_id, content)
+
+
+@router.post("/users/{user_id}/import")
+def import_user(user_id: str, req: ImportRequest):
+    """Bulk restore/seed memories from an export. Re-embeds each item; idempotent on
+    (user_id, content). Bypasses the relevance judge — this is restoring vetted data."""
+    user_id = user_id.strip().lower()
+    items = req.memories or []
+    if not items:
+        return {"imported": 0, "skipped": 0}
+
+    backend = get_backend()
+    conn = get_conn()
+    cur = conn.cursor() if backend != "duckdb" else None
+    imported, skipped = 0, 0
+    try:
+        for it in items:
+            content = str(it.get("content", "")).strip()
+            if len(content) < 2:
+                skipped += 1
+                continue
+            importance = float(it.get("importance", 0.5) or 0.5)
+            importance = max(0.0, min(1.0, importance))
+            category = str(it.get("category", "") or "").strip().lower() or categorize(content)
+            try:
+                emb_str = emb_to_db(embed(content), backend)
+                if backend == "postgres":
+                    cur.execute(
+                        "INSERT INTO memories (user_id, content, embedding, importance, category) "
+                        "VALUES (%s, %s, %s::vector, %s, %s) "
+                        "ON CONFLICT (user_id, content) DO UPDATE SET importance = EXCLUDED.importance",
+                        (user_id, content, emb_str, importance, category))
+                elif backend == "duckdb":
+                    conn.execute(
+                        "INSERT INTO memories (user_id, content, embedding, importance, category) "
+                        "VALUES (?, ?, ?, ?, ?) ON CONFLICT (user_id, content) DO UPDATE SET importance = excluded.importance",
+                        [user_id, content, emb_str, importance, category])
+                else:
+                    cur.execute(
+                        "INSERT INTO memories (user_id, content, embedding, importance, category) "
+                        "VALUES (?, ?, ?, ?, ?) ON CONFLICT (user_id, content) DO UPDATE SET importance = excluded.importance",
+                        (user_id, content, emb_str, importance, category))
+                imported += 1
+            except Exception:
+                skipped += 1
+        if backend != "duckdb":
+            conn.commit()
+    finally:
+        if cur:
+            cur.close()
+        conn.close()
+
+    log_event("write", "import", user_id, detail={"imported": imported, "skipped": skipped})
+    return {"imported": imported, "skipped": skipped}

yourmemory-1.4.76/src/services/compaction.py

@@ -0,0 +1,292 @@
+"""
+Memory compaction — auto-compress clusters of related memories into one structured
+summary and archive the originals.
+
+Why: a memory store that only ever grows bloats and its signal-to-noise degrades. After
+several memories accumulate about the same topic/entity, we summarize them into a single
+consolidated memory and move the originals to `memory_archive`. The live `memories` table
+stays lean (recall stays fast and clean), the summary preserves every distinct detail,
+and the originals remain recoverable and auditable.
+
+Flow per user:
+  1. Embed all live memories, greedily cluster by cosine similarity.
+  2. For each cluster of >= MIN_CLUSTER members, LLM-summarize into one memory.
+  3. Insert the summary, copy originals to memory_archive, delete originals from memories.
+  4. Re-index the graph (drop original nodes, index the summary). Audit the compaction.
+
+Conservative by design: the summary prompt is instructed to preserve all facts, the
+threshold groups clearly-related memories, and originals are archived (never lost).
+"""
+
+import json
+import math
+import os
+import urllib.request
+from datetime import datetime, timezone
+
+from src.services.embed import embed
+from src.services.extract import categorize
+from src.db.connection import get_backend, get_conn, emb_to_db, duckdb_rows
+from src.services.audit import log_event
+
+MIN_CLUSTER     = int(os.getenv("YOURMEMORY_COMPACT_MIN", "5"))     # min memories to compress
+SIM_THRESHOLD   = float(os.getenv("YOURMEMORY_COMPACT_SIM", "0.62")) # cosine to group as "related"
+MAX_SCAN        = int(os.getenv("YOURMEMORY_COMPACT_MAX", "2000"))   # cap per run (O(n^2) guard)
+
+
+def _cosine(a, b) -> float:
+    dot = sum(x * y for x, y in zip(a, b))
+    na = math.sqrt(sum(x * x for x in a))
+    nb = math.sqrt(sum(x * x for x in b))
+    return dot / (na * nb) if na and nb else 0.0
+
+
+def _summarize(contents: list[str]) -> str | None:
+    """LLM-compress related facts into ONE structured memory, preserving every detail."""
+    joined = "\n".join(f"- {c}" for c in contents)
+    prompt = (
+        "You are compressing several related memory facts about the same topic into ONE "
+        "consolidated memory. Preserve EVERY distinct detail — names, numbers, dates, "
+        "preferences, decisions. Do not drop or invent information. Merge overlaps, keep "
+        "specifics. Output a single self-contained declarative summary (1–3 sentences), "
+        "no preamble, no markdown.\n\n"
+        f"Facts:\n{joined}\n\nConsolidated memory:"
+    )
+    backend = os.getenv("YOURMEMORY_EXTRACT_BACKEND", "ollama").lower()
+    try:
+        if backend == "anthropic":
+            from src.services.extract import _anthropic_complete  # type: ignore
+            return _anthropic_complete(prompt).strip() or None  # best-effort if present
+    except Exception:
+        pass
+    # Default: Ollama
+    url   = os.getenv("YOURMEMORY_OLLAMA_URL", "http://localhost:11434")
+    model = os.getenv("YOURMEMORY_OLLAMA_MODEL", "qwen2.5:7b")
+    payload = json.dumps({
+        "model": model, "prompt": prompt, "stream": False,
+        "keep_alive": os.getenv("YOURMEMORY_OLLAMA_KEEPALIVE", "30m"),
+        "options": {"temperature": 0, "num_predict": 220},
+    }).encode()
+    try:
+        req = urllib.request.Request(f"{url}/api/generate", data=payload,
+                                     headers={"Content-Type": "application/json"})
+        with urllib.request.urlopen(req, timeout=60) as r:
+            out = json.loads(r.read()).get("response", "").strip()
+        return out or None
+    except Exception:
+        return None
+
+
+def _live_memories(conn, backend: str, user_id: str) -> list[dict]:
+    cols = "id, content, category, importance, agent_id, visibility, created_at"
+    if backend == "postgres":
+        from psycopg2.extras import RealDictCursor
+        cur = conn.cursor(cursor_factory=RealDictCursor)
+        cur.execute(f"SELECT {cols} FROM memories WHERE user_id = %s ORDER BY id LIMIT %s",
+                    (user_id, MAX_SCAN))
+        rows = [dict(r) for r in cur.fetchall()]; cur.close(); return rows
+    if backend == "duckdb":
+        return duckdb_rows(conn.execute(
+            f"SELECT {cols} FROM memories WHERE user_id = ? ORDER BY id LIMIT ?", [user_id, MAX_SCAN]))
+    cur = conn.cursor()
+    cur.execute(f"SELECT {cols} FROM memories WHERE user_id = ? ORDER BY id LIMIT ?", (user_id, MAX_SCAN))
+    cn = [d[0] for d in cur.description]
+    rows = [dict(zip(cn, r)) for r in cur.fetchall()]; cur.close(); return rows
+
+
+def compact_user(user_id: str, min_cluster: int = None, sim_threshold: float = None) -> dict:
+    """Compress related-memory clusters for one user. Returns stats."""
+    user_id = (user_id or "").strip().lower()
+    min_cluster = min_cluster or MIN_CLUSTER
+    sim_threshold = sim_threshold if sim_threshold is not None else SIM_THRESHOLD
+    backend = get_backend()
+
+    conn = get_conn()
+    try:
+        mems = _live_memories(conn, backend, user_id)
+    finally:
+        conn.close()
+    if len(mems) < min_cluster:
+        return {"clusters": 0, "archived": 0, "summaries": 0, "scanned": len(mems)}
+
+    # Embed + greedy cluster by cosine similarity.
+    vecs = [embed(m["content"]) for m in mems]
+    used, clusters = set(), []
+    for i in range(len(mems)):
+        if i in used:
+            continue
+        group = [i]; used.add(i)
+        for j in range(i + 1, len(mems)):
+            if j in used:
+                continue
+            if _cosine(vecs[i], vecs[j]) >= sim_threshold:
+                group.append(j); used.add(j)
+        if len(group) >= min_cluster:
+            clusters.append(group)
+
+    if not clusters:
+        return {"clusters": 0, "archived": 0, "summaries": 0, "scanned": len(mems)}
+
+    archived_total, summaries = 0, 0
+    for group in clusters:
+        members = [mems[k] for k in group]
+        summary = _summarize([m["content"] for m in members])
+        if not summary or len(summary) < 12:
+            continue   # summarization failed → leave the cluster untouched
+        # Inherit the strongest signal from the cluster.
+        importance = max(float(m["importance"] or 0.5) for m in members)
+        category   = categorize(summary)
+        agent_id   = members[0].get("agent_id")
+        visibility = members[0].get("visibility") or "shared"
+        summary_id = _apply_compaction(backend, user_id, summary, importance, category,
+                                       agent_id, visibility, members)
+        if summary_id is not None:
+            archived_total += len(members)
+            summaries += 1
+
+    if summaries:
+        log_event("write", "compact", user_id,
+                  detail={"clusters": summaries, "archived": archived_total, "scanned": len(mems)})
+    return {"clusters": len(clusters), "archived": archived_total,
+            "summaries": summaries, "scanned": len(mems)}
+
+
+def _fetch_rows_by_ids(user_id: str, ids: list) -> list[dict]:
+    if not ids:
+        return []
+    backend = get_backend()
+    conn = get_conn()
+    cols = "id, content, category, importance, agent_id, visibility, created_at"
+    ph = ",".join(["%s" if backend == "postgres" else "?"] * len(ids))
+    sql = f"SELECT {cols} FROM memories WHERE user_id = {'%s' if backend=='postgres' else '?'} AND id IN ({ph})"
+    params = [user_id, *ids]
+    try:
+        if backend == "duckdb":
+            return duckdb_rows(conn.execute(sql, params))
+        cur = conn.cursor()
+        cur.execute(sql, tuple(params) if backend == "postgres" else params)
+        cn = [d[0] for d in cur.description]
+        rows = [dict(zip(cn, r)) for r in cur.fetchall()]; cur.close(); return rows
+    finally:
+        conn.close()
+
+
+def maybe_compact_around(user_id: str, seed_content: str,
+                         min_cluster: int = None, sim_threshold: float = None) -> int | None:
+    """Event-driven compaction: after a memory is stored, check whether its neighborhood
+    now has >= N closely-related memories; if so, compress just that cluster immediately.
+    Targeted (one similarity lookup, not an O(n^2) full scan). Returns the summary id."""
+    user_id = (user_id or "").strip().lower()
+    min_cluster = min_cluster or MIN_CLUSTER
+    sim = sim_threshold if sim_threshold is not None else SIM_THRESHOLD
+    if not seed_content or len(seed_content) < 4:
+        return None
+    try:
+        from src.services.retrieve import retrieve
+        res = retrieve(user_id, seed_content, top_k=max(min_cluster * 4, 20), no_graph=True)
+    except Exception:
+        return None
+    members = [m for m in res.get("memories", []) if m.get("similarity", 0) >= sim]
+    if len(members) < min_cluster:
+        return None   # not enough of the same thing yet — leave it
+    rows = _fetch_rows_by_ids(user_id, [m["id"] for m in members])
+    if len(rows) < min_cluster:
+        return None
+    summary = _summarize([r["content"] for r in rows])
+    if not summary or len(summary) < 12:
+        return None
+    importance = max(float(r["importance"] or 0.5) for r in rows)
+    category   = categorize(summary)
+    agent_id   = rows[0].get("agent_id")
+    visibility = rows[0].get("visibility") or "shared"
+    summary_id = _apply_compaction(get_backend(), user_id, summary, importance, category,
+                                   agent_id, visibility, rows)
+    if summary_id is not None:
+        log_event("write", "compact", user_id,
+                  detail={"clusters": 1, "archived": len(rows), "trigger": "count"})
+    return summary_id
+
+
+def _apply_compaction(backend, user_id, summary, importance, category, agent_id,
+                      visibility, members) -> int | None:
+    """Insert the summary memory, archive the originals, delete them. Returns summary id."""
+    emb_str = emb_to_db(embed(summary), backend)
+    ids = [m["id"] for m in members]
+    conn = get_conn()
+    cur = conn.cursor() if backend != "duckdb" else None
+    summary_id = None
+    try:
+        # 1. Insert the consolidated summary.
+        if backend == "postgres":
+            cur.execute(
+                "INSERT INTO memories (user_id, content, embedding, importance, category, agent_id, visibility) "
+                "VALUES (%s,%s,%s::vector,%s,%s,%s,%s) "
+                "ON CONFLICT (user_id, content) DO UPDATE SET importance = EXCLUDED.importance RETURNING id",
+                (user_id, summary, emb_str, importance, category, agent_id, visibility))
+            summary_id = cur.fetchone()[0]
+        elif backend == "duckdb":
+            conn.execute(
+                "INSERT INTO memories (user_id, content, embedding, importance, category, agent_id, visibility) "
+                "VALUES (?,?,?,?,?,?,?) ON CONFLICT (user_id, content) DO UPDATE SET importance = excluded.importance",
+                [user_id, summary, emb_str, importance, category, agent_id, visibility])
+            summary_id = conn.execute("SELECT id FROM memories WHERE user_id=? AND content=?",
+                                      [user_id, summary]).fetchone()[0]
+        else:
+            cur.execute(
+                "INSERT INTO memories (user_id, content, embedding, importance, category, agent_id, visibility) "
+                "VALUES (?,?,?,?,?,?,?) ON CONFLICT (user_id, content) DO UPDATE SET importance = excluded.importance",
+                (user_id, summary, emb_str, importance, category, agent_id, visibility))
+            cur.execute("SELECT id FROM memories WHERE user_id=? AND content=?", (user_id, summary))
+            summary_id = cur.fetchone()[0]
+
+        # 2. Archive originals + 3. delete them (skip the summary row if it collided with one).
+        for m in members:
+            if m["id"] == summary_id:
+                continue
+            vals = (m["id"], user_id, m["content"], m.get("category"), m.get("importance"),
+                    m.get("agent_id"), m.get("visibility"), m.get("created_at"), summary_id)
+            if backend == "postgres":
+                cur.execute(
+                    "INSERT INTO memory_archive (orig_id,user_id,content,category,importance,agent_id,visibility,created_at,summary_id) "
+                    "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)", vals)
+                cur.execute("DELETE FROM memories WHERE id = %s", (m["id"],))
+            elif backend == "duckdb":
+                conn.execute(
+                    "INSERT INTO memory_archive (orig_id,user_id,content,category,importance,agent_id,visibility,created_at,summary_id) "
+                    "VALUES (?,?,?,?,?,?,?,?,?)", list(vals))
+                conn.execute("DELETE FROM memories WHERE id = ?", [m["id"]])
+            else:
+                cur.execute(
+                    "INSERT INTO memory_archive (orig_id,user_id,content,category,importance,agent_id,visibility,created_at,summary_id) "
+                    "VALUES (?,?,?,?,?,?,?,?,?)", vals)
+                cur.execute("DELETE FROM memories WHERE id = ?", (m["id"],))
+        if backend != "duckdb":
+            conn.commit()
+    except Exception:
+        if backend == "postgres":
+            try: conn.rollback()
+            except Exception: pass
+        summary_id = None
+    finally:
+        if cur: cur.close()
+        conn.close()
+
+    # Best-effort graph upkeep: drop original nodes, index the summary.
+    if summary_id is not None:
+        try:
+            from src.graph import get_graph_backend
+            gb = get_graph_backend()
+            for m in members:
+                if m["id"] != summary_id:
+                    try: gb.delete_node(m["id"])
+                    except Exception: pass
+        except Exception:
+            pass
+        try:
+            from src.graph.graph_store import index_memory
+            index_memory(memory_id=summary_id, user_id=user_id, content=summary,
+                         strength=importance, importance=importance, category=category,
+                         embedding=list(embed(summary)))
+        except Exception:
+            pass
+    return summary_id

{yourmemory-1.4.73 → yourmemory-1.4.76}/yourmemory.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: yourmemory
-Version: 1.4.73
+Version: 1.4.76
 Summary: Persistent memory for Claude — Ebbinghaus forgetting curve, semantic deduplication, MCP-native
 Author: Sachit Misra
 Author-email: mishrasachit1@gmail.com

{yourmemory-1.4.73 → yourmemory-1.4.76}/yourmemory.egg-info/SOURCES.txt

@@ -27,6 +27,8 @@ src/jobs/decay_job.py
 src/routes/__init__.py
 src/routes/agents.py
 src/routes/audit.py
+src/routes/compact.py
+src/routes/dsar.py
 src/routes/graph_viz.py
 src/routes/memories.py
 src/routes/proxy.py
@@ -36,6 +38,7 @@ src/services/__init__.py
 src/services/agent_registry.py
 src/services/api_keys.py
 src/services/audit.py
+src/services/compaction.py
 src/services/decay.py
 src/services/embed.py
 src/services/extract.py