memor-cli 0.1.0__py3-none-any.whl

memor/daemon.py

@@ -0,0 +1,294 @@
+"""Auto-ingest daemon: polls ~/.claude/projects/ for new/modified .jsonl transcripts,
+then auto-distills new sessions into compact memories."""
+from __future__ import annotations
+
+import json
+import os
+import time
+from pathlib import Path
+
+from memor.ingest.claude_code import parse_transcript
+from memor.store.sqlite_store import SqliteStore
+from memor.types import Scope
+
+CLAUDE_PROJECTS_DIR = Path.home() / ".claude" / "projects"
+STATE_DIR = Path.home() / ".memor"
+DEFAULT_DB = STATE_DIR / "memor.db"
+STATE_FILE = STATE_DIR / "ingested.json"
+DISTILLED_FILE = STATE_DIR / "distilled.json"
+POLL_INTERVAL = 30  # seconds
+MAX_DISTILL_TOKENS = 4000  # cap text sent to LLM per session
+
+
+def _project_name_from_dir(dirname: str) -> str:
+    """Derive a clean project name from a Claude projects directory name.
+    Uses the smart filesystem-aware resolver that handles dashes in dir names."""
+    from memor.project import resolve_project_from_claude_dir
+    return resolve_project_from_claude_dir(dirname)
+
+
+def load_state() -> dict[str, float]:
+    """Load the ingested state file: {filepath -> mtime}."""
+    if STATE_FILE.exists():
+        try:
+            return json.loads(STATE_FILE.read_text())
+        except (json.JSONDecodeError, OSError):
+            return {}
+    return {}
+
+
+def save_state(state: dict[str, float]) -> None:
+    """Persist the ingested state file."""
+    STATE_DIR.mkdir(parents=True, exist_ok=True)
+    STATE_FILE.write_text(json.dumps(state, indent=2))
+
+
+def scan_transcripts(projects_dir: Path) -> list[tuple[Path, str]]:
+    """Scan for all .jsonl transcript files, returning (path, project_name) pairs."""
+    results = []
+    if not projects_dir.is_dir():
+        return results
+    for project_dir in sorted(projects_dir.iterdir()):
+        if not project_dir.is_dir():
+            continue
+        project_name = _project_name_from_dir(project_dir.name)
+        for jsonl_file in sorted(project_dir.rglob("*.jsonl")):
+            results.append((jsonl_file, project_name))
+    return results
+
+
+def load_distilled_state() -> set[str]:
+    """Load the set of already-distilled session IDs."""
+    if DISTILLED_FILE.exists():
+        try:
+            return set(json.loads(DISTILLED_FILE.read_text()))
+        except (json.JSONDecodeError, OSError):
+            return set()
+    return set()
+
+
+def save_distilled_state(distilled: set[str]) -> None:
+    STATE_DIR.mkdir(parents=True, exist_ok=True)
+    DISTILLED_FILE.write_text(json.dumps(sorted(distilled), indent=2))
+
+
+def _make_llm():
+    """Try to create an LLM for distillation. Returns None if no API key."""
+    api_key = os.environ.get("ANTHROPIC_API_KEY")
+    if api_key:
+        from memor.llm.anthropic import AnthropicLLM
+        return AnthropicLLM(model="claude-sonnet-4-6", api_key=api_key)
+    api_key = os.environ.get("OPENAI_API_KEY")
+    base_url = os.environ.get("OPENAI_BASE_URL", "http://localhost:11434/v1")
+    if api_key:
+        from memor.llm.openai_compat import OpenAICompatLLM
+        return OpenAICompatLLM(base_url=base_url, api_key=api_key, model="gpt-4o-mini")
+    return None
+
+
+def ingest_file(path: Path, project: str, store: SqliteStore, embedder) -> int:
+    """Ingest a single transcript file. Returns number of chunks ingested."""
+    arts = parse_transcript(path, project=project, filter_noise=True)
+    if not arts:
+        return 0
+    vecs = embedder.embed([a.text for a in arts])
+    store.add_artifacts(arts, vecs)
+    return len(arts)
+
+
+def distill_new_sessions(
+    store: SqliteStore, embedder, llm, distilled: set[str]
+) -> set[str]:
+    """Distill any sessions that haven't been distilled yet. Returns updated set.
+    Uses full LLM distiller if llm is provided, otherwise falls back to extractive-only."""
+    if llm:
+        from memor.distill.distiller import Distiller
+        d = Distiller(store, embedder, llm)
+    else:
+        from memor.distill.distiller import ExtractiveDistiller
+        d = ExtractiveDistiller(store, embedder)
+    rows = store.db.execute(
+        "SELECT * FROM artifacts WHERE kind='session_chunk'"
+    ).fetchall()
+    by_session: dict[str, list] = {}
+    for r in rows:
+        a = store._row_to_artifact(r)
+        sid = a.meta.get("session_id", "?")
+        by_session.setdefault(sid, []).append(a)
+
+    for sid, chunks in by_session.items():
+        if sid in distilled:
+            continue
+        chunks.sort(key=lambda a: a.meta.get("ord", 0))
+        total_tok = sum(c.token_count for c in chunks)
+        # Cap context sent to LLM
+        if total_tok > MAX_DISTILL_TOKENS:
+            selected = chunks[:10] + chunks[-5:]
+        else:
+            selected = chunks
+        project = chunks[0].project
+        try:
+            mem_ids = d.distill_session(sid, selected, project=project)
+            distilled.add(sid)
+            if mem_ids:
+                print(f"  distilled session {sid[:20]}... -> {len(mem_ids)} memories")
+        except Exception as e:
+            print(f"  ERROR distilling {sid[:20]}...: {e}")
+    return distilled
+
+
+COMPACT_SIM_THRESHOLD = 0.90
+
+
+def compact_memories(store: SqliteStore, embedder) -> int:
+    """Find near-duplicate active memories and deactivate the older/lower-quality one."""
+    rows = store.db.execute(
+        "SELECT * FROM artifacts WHERE kind='memory' AND active=1"
+    ).fetchall()
+    if len(rows) < 2:
+        return 0
+    memories = [store._row_to_artifact(r) for r in rows]
+    vecs = embedder.embed([m.text for m in memories])
+    deactivated = 0
+    seen = set()
+    for i in range(len(memories)):
+        if memories[i].id in seen:
+            continue
+        for j in range(i + 1, len(memories)):
+            if memories[j].id in seen:
+                continue
+            if memories[i].project != memories[j].project:
+                continue
+            dot = sum(a * b for a, b in zip(vecs[i], vecs[j]))
+            norm_i = sum(a * a for a in vecs[i]) ** 0.5
+            norm_j = sum(a * a for a in vecs[j]) ** 0.5
+            sim = dot / (norm_i * norm_j) if norm_i and norm_j else 0
+            if sim >= COMPACT_SIM_THRESHOLD:
+                qi = store.get_quality_score(memories[i].id)
+                qj = store.get_quality_score(memories[j].id)
+                if qi != qj:
+                    loser = j if qi > qj else i
+                else:
+                    loser = i if memories[j].created_at > memories[i].created_at else j
+                winner = j if loser == i else i
+                store.deactivate(memories[loser].id, superseded_by=memories[winner].id)
+                seen.add(memories[loser].id)
+                deactivated += 1
+    return deactivated
+
+
+def run_poll_cycle(
+    state: dict[str, float],
+    store: SqliteStore,
+    embedder,
+    projects_dir: Path = CLAUDE_PROJECTS_DIR,
+    llm=None,
+    distilled: set[str] | None = None,
+) -> tuple[dict[str, float], set[str]]:
+    """Run one poll cycle: ingest new files, then distill new sessions.
+    Returns (updated ingest state, updated distilled set)."""
+    if distilled is None:
+        distilled = set()
+
+    new_ingested = False
+    transcripts = scan_transcripts(projects_dir)
+
+    # Pre-filter to only files that are new or modified
+    pending = [
+        (path, project)
+        for path, project in transcripts
+        if not (state.get(str(path)) is not None and path.stat().st_mtime <= state[str(path)])
+    ]
+
+    bulk = len(pending) > 10
+    total_pending = len(pending)
+
+    for idx, (path, project) in enumerate(pending):
+        path_str = str(path)
+        current_mtime = path.stat().st_mtime
+        progress_prefix = f"[{idx + 1}/{total_pending}] " if bulk else ""
+        try:
+            count = ingest_file(path, project, store, embedder)
+            state[path_str] = current_mtime
+            if count > 0:
+                print(f"  {progress_prefix}ingested {count} chunks from {path.name} (project: {project})")
+                new_ingested = True
+            else:
+                print(f"  {progress_prefix}skipped {path.name} (0 chunks after filtering)")
+        except Exception as e:
+            print(f"  {progress_prefix}ERROR ingesting {path.name}: {e}")
+
+    # Auto-distill new sessions (LLM if available, extractive fallback otherwise)
+    if new_ingested:
+        mode = "abstractive" if llm else "extractive (LLM-free)"
+        print(f"  running {mode} distillation on new sessions...")
+        distilled = distill_new_sessions(store, embedder, llm, distilled)
+
+    # Feedback analysis: check if recalled memories were used in completed sessions
+    if new_ingested:
+        from memor.feedback import analyze_session_feedback
+        for path, project in pending:
+            session_id = path.stem
+            try:
+                used = analyze_session_feedback(store, session_id, path)
+                if used > 0:
+                    print(f"  feedback: {used} memories confirmed used in {session_id[:12]}...")
+            except Exception:
+                pass
+
+    # Compact near-duplicate memories (run occasionally, not every cycle)
+    if new_ingested:
+        try:
+            compacted = compact_memories(store, embedder)
+            if compacted > 0:
+                print(f"  compacted {compacted} near-duplicate memories")
+        except Exception:
+            pass
+
+    return state, distilled
+
+
+def _make_embedder():
+    """Local ONNX embedder by default. No API key needed for search."""
+    from memor.embed.local import LocalEmbedder
+    return LocalEmbedder()
+
+
+def run_daemon(poll_interval: int = POLL_INTERVAL, projects_dir: Path = CLAUDE_PROJECTS_DIR) -> None:
+    """Run the daemon loop (foreground, blocking)."""
+    STATE_DIR.mkdir(parents=True, exist_ok=True)
+    embedder = _make_embedder()
+    store = SqliteStore(str(DEFAULT_DB), dim=embedder.dim)
+    state = load_state()
+    distilled = load_distilled_state()
+
+    llm = _make_llm()
+
+    print(r"""
+                                                _
+ _ __ ___   ___ _ __ ___   ___  _ __       __ _(_)
+| '_ ` _ \ / _ \ '_ ` _ \ / _ \| '__|____ / _` | |
+| | | | | |  __/ | | | | | (_) | | |_____| (_| | |
+|_| |_| |_|\___|_| |_| |_|\___/|_|        \__,_|_|
+""")
+    print(f"  watching:      {projects_dir}")
+    print(f"  db:            {DEFAULT_DB}")
+    print(f"  embeddings:    local model2vec (dim={embedder.dim})")
+    print(f"  poll interval: {poll_interval}s")
+    print(f"  tracking:      {len(state)} ingested files, {len(distilled)} sessions distilled")
+    print(f"  distillation:  {'abstractive' if llm else 'extractive'}")
+    print()
+
+    try:
+        while True:
+            print(f"[{time.strftime('%H:%M:%S')}] polling...")
+            state, distilled = run_poll_cycle(
+                state, store, embedder, projects_dir, llm=llm, distilled=distilled
+            )
+            save_state(state)
+            save_distilled_state(distilled)
+            time.sleep(poll_interval)
+    except KeyboardInterrupt:
+        print("\ndaemon stopped.")
+        save_state(state)
+        save_distilled_state(distilled)

memor/dashboard/server.py

@@ -0,0 +1,153 @@
+from __future__ import annotations
+import os
+from pathlib import Path
+from fastapi import FastAPI, Query
+from fastapi.responses import HTMLResponse
+from memor.store.sqlite_store import SqliteStore
+
+STATIC_DIR = Path(__file__).parent / "static"
+
+
+def create_app(db_path: str | None = None) -> FastAPI:
+    if db_path is None:
+        db_path = str(Path.home() / ".memor" / "memor.db")
+
+    app = FastAPI(title="Memor Dashboard")
+    _db_path = db_path
+
+    def _store() -> SqliteStore:
+        return SqliteStore(_db_path, dim=_get_dim(_db_path))
+
+    @app.get("/", response_class=HTMLResponse)
+    def index():
+        html_path = STATIC_DIR / "index.html"
+        if html_path.exists():
+            return HTMLResponse(html_path.read_text())
+        return HTMLResponse("<h1>Memor Dashboard</h1><p>index.html not found</p>")
+
+    @app.get("/api/summary")
+    def summary():
+        store = _store()
+        recall_stats = store.get_recall_stats()
+        ingestion = {}
+        for row in store.db.execute(
+            "SELECT kind, COUNT(*) as c, SUM(token_count) as tokens "
+            "FROM artifacts WHERE active=1 GROUP BY kind"
+        ).fetchall():
+            ingestion[row["kind"]] = {"count": row["c"], "tokens": row["tokens"] or 0}
+        project_count = store.db.execute(
+            "SELECT COUNT(DISTINCT project) as c FROM artifacts"
+        ).fetchone()["c"]
+        recall_stats["ingestion"] = ingestion
+        recall_stats["project_count"] = project_count
+        return recall_stats
+
+    @app.get("/api/projects")
+    def projects():
+        store = _store()
+        recall_stats = store.get_project_stats()
+        if recall_stats:
+            return recall_stats
+        rows = store.db.execute("""
+            SELECT project,
+                   COUNT(*) as artifacts,
+                   SUM(CASE WHEN kind='session_chunk' THEN 1 ELSE 0 END) as chunks,
+                   SUM(CASE WHEN kind='memory' THEN 1 ELSE 0 END) as memories,
+                   SUM(token_count) as total_tokens,
+                   MAX(created_at) as last_activity
+            FROM artifacts WHERE active=1
+            GROUP BY project
+            ORDER BY artifacts DESC
+        """).fetchall()
+        return [dict(r) for r in rows]
+
+    @app.get("/api/recalls")
+    def recalls(limit: int = Query(50, ge=1, le=500),
+                project: str | None = Query(None)):
+        store = _store()
+        return store.get_recent_recalls(limit=limit, project=project)
+
+    @app.get("/api/quality")
+    def quality():
+        store = _store()
+        rows = store.db.execute("""
+            SELECT q.artifact_id, q.recall_count, q.use_count, q.quality_score,
+                   a.project, a.kind, json_extract(a.meta, '$.mem_type') as mem_type,
+                   substr(a.text, 1, 100) as preview
+            FROM memory_quality q
+            JOIN artifacts a ON a.id = q.artifact_id
+            WHERE a.active = 1
+            ORDER BY q.quality_score DESC
+            LIMIT 50
+        """).fetchall()
+        return [dict(r) for r in rows]
+
+    @app.get("/api/efficiency")
+    def efficiency():
+        store = _store()
+        return store.get_efficiency_stats()
+
+    @app.get("/api/savings")
+    def savings():
+        store = _store()
+        rows = store.db.execute("""
+            SELECT project,
+                   SUM(tokens_injected) as recalled_tokens,
+                   AVG(CASE WHEN hits_count > 0 THEN top_score END) as avg_relevance
+            FROM recall_log
+            WHERE status IN ('ok', 'extractive_only')
+            GROUP BY project
+        """).fetchall()
+        result = []
+        for r in rows:
+            project = r["project"]
+            full_ctx = store.db.execute(
+                "SELECT SUM(token_count) as total FROM artifacts WHERE project=? AND active=1",
+                (project,)).fetchone()["total"] or 0
+            recalled = r["recalled_tokens"] or 0
+            result.append({
+                "project": project,
+                "recalled_tokens": recalled,
+                "full_context_tokens": full_ctx,
+                "reduction_pct": round((1 - recalled / full_ctx) * 100, 1) if full_ctx > 0 else 0,
+                "avg_relevance": round(r["avg_relevance"] or 0, 3),
+            })
+        return result
+
+    @app.get("/api/health")
+    def health():
+        store = _store()
+        db_size = os.path.getsize(_db_path) if os.path.exists(_db_path) else 0
+        counts = {}
+        for row in store.db.execute(
+            "SELECT kind, COUNT(*) as c FROM artifacts WHERE active=1 GROUP BY kind"
+        ).fetchall():
+            counts[row["kind"]] = row["c"]
+        last_ingest = store.db.execute(
+            "SELECT MAX(created_at) as t FROM artifacts"
+        ).fetchone()["t"]
+        dim_row = store.db.execute("SELECT value FROM meta WHERE key='dim'").fetchone()
+        return {
+            "onboarding_status": store.get_onboarding_status(),
+            "db_size_bytes": db_size,
+            "artifact_counts": counts,
+            "last_ingest_timestamp": last_ingest,
+            "embedder_dim": int(dim_row["value"]) if dim_row else None,
+        }
+
+    return app
+
+
+def _get_dim(db_path: str) -> int:
+    """Read dim from meta table, default to 1536 (OpenAI)."""
+    import sqlite3
+    try:
+        db = sqlite3.connect(db_path)
+        db.row_factory = sqlite3.Row
+        row = db.execute("SELECT value FROM meta WHERE key='dim'").fetchone()
+        db.close()
+        if row:
+            return int(row["value"])
+    except Exception:
+        pass
+    return 1536