memor-cli 0.1.0__py3-none-any.whl

memor/eval/runner.py

@@ -0,0 +1,78 @@
+from __future__ import annotations
+from memor.types import Scope
+from memor.retrieve.retriever import Retriever
+from memor.eval.dataset import EvalCase, CaseResult
+from memor.eval.metrics import recall_at_k, ndcg_at_k
+
+BASELINES = ["no-memory", "last-N", "naive-RAG", "memory"]
+
+def _result(ranked_ids, hits_tokens, latency, case, k):
+    return CaseResult(
+        recall=recall_at_k(ranked_ids, case.relevant_ids, k),
+        ndcg=ndcg_at_k(ranked_ids, case.relevant_ids, k),
+        tokens_sent=hits_tokens, latency_ms=latency)
+
+def run_case(case: EvalCase, *, store, embedder, k: int = 8) -> dict[str, CaseResult]:
+    scope = Scope(project=case.scope_project)
+    out: dict[str, CaseResult] = {}
+
+    # no-memory: retrieve nothing
+    out["no-memory"] = _result([], 0, 0.0, case, k)
+
+    # last-N: most recent k chunks, no vector search
+    recent = store.recent(scope, k) if hasattr(store, 'recent') else []
+    out["last-N"] = _result([a.id for a in recent],
+                            sum(a.token_count for a in recent),
+                            0.0, case, k)
+
+    # naive-RAG: similarity only, no edges, no distill
+    naive = Retriever(store, embedder, k=k, recency_weight=0.0, edge_expand=False)
+    tr = naive.query(case.query, scope)
+    out["naive-RAG"] = _result([h.artifact.id for h in tr.hits],
+                               sum(h.artifact.token_count for h in tr.hits),
+                               tr.latency_ms, case, k)
+
+    # memory: full retriever (recency + edge expansion over distilled+raw artifacts)
+    mem = Retriever(store, embedder, k=k, recency_weight=0.2, edge_expand=True)
+    tr = mem.query(case.query, scope)
+    out["memory"] = _result([h.artifact.id for h in tr.hits],
+                            sum(h.artifact.token_count for h in tr.hits),
+                            tr.latency_ms, case, k)
+    return out
+
+def run_suite(cases: list[EvalCase], *, store, embedder, k: int = 8) -> dict[str, dict]:
+    """Aggregate mean metrics per strategy + Δ vs naive-RAG and full-history tokens."""
+    agg: dict[str, list[CaseResult]] = {}
+    for c in cases:
+        for strat, res in run_case(c, store=store, embedder=embedder, k=k).items():
+            agg.setdefault(strat, []).append(res)
+    summary = {}
+    n = len(cases) or 1
+    full_tokens = sum(c.baseline_full_tokens for c in cases) / n
+    for strat, results in agg.items():
+        mean_tokens = sum(r.tokens_sent for r in results) / n
+        latencies = sorted(r.latency_ms for r in results)
+        summary[strat] = {
+            "recall@k": sum(r.recall for r in results) / n,
+            "ndcg@k": sum(r.ndcg for r in results) / n,
+            "tokens_sent": mean_tokens,
+            "token_savings_vs_full": 1.0 - (mean_tokens / full_tokens) if full_tokens else 0.0,
+            "latency_ms_p50": latencies[len(latencies)//2],
+            "latency_ms_p95": latencies[int(len(latencies)*0.95)],
+        }
+    return summary
+
+def run_ablation(*, query, project, relevant_ids, store, embedder, k=8):
+    out = {}
+    for name, expand in (("similarity-only", False), ("similarity+edges", True)):
+        r = Retriever(store, embedder, k=k, recency_weight=0.0, edge_expand=expand)
+        tr = r.query(query, Scope(project=project))
+        ids = [h.artifact.id for h in tr.hits]
+        out[name] = {"recall@k": recall_at_k(ids, relevant_ids, k),
+                     "ndcg@k": ndcg_at_k(ids, relevant_ids, k)}
+    return out
+
+def run_contradiction_eval(*, query, project, stale_id, current_id, store, embedder, k=8):
+    r = Retriever(store, embedder, k=k, edge_expand=True)
+    ids = [h.artifact.id for h in r.query(query, Scope(project=project)).hits]
+    return (current_id in ids) and (stale_id not in ids)

memor/feedback.py

@@ -0,0 +1,96 @@
+"""Feedback analyzer — detects whether recalled memories were used by the agent.
+
+After a session ends, cross-references recall_log with the transcript to see
+if the agent's responses referenced recalled content. Updates memory_quality
+scores accordingly."""
+from __future__ import annotations
+import json
+from pathlib import Path
+from memor.store.sqlite_store import SqliteStore
+
+MIN_OVERLAP_TOKENS = 5
+
+
+def _extract_assistant_texts(transcript_path: Path) -> list[str]:
+    texts = []
+    for line in transcript_path.read_text().splitlines():
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            rec = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+        if rec.get("type") != "assistant":
+            continue
+        msg = rec.get("message", {})
+        content = msg.get("content", "")
+        if isinstance(content, str):
+            texts.append(content.lower())
+        elif isinstance(content, list):
+            for block in content:
+                if isinstance(block, dict) and block.get("type") == "text":
+                    texts.append(block.get("text", "").lower())
+    return texts
+
+
+def _text_was_used(memory_text: str, assistant_texts: list[str]) -> bool:
+    words = memory_text.lower().split()
+    if len(words) < MIN_OVERLAP_TOKENS:
+        return False
+    key_phrases = []
+    for i in range(0, len(words) - 4):
+        key_phrases.append(" ".join(words[i:i+5]))
+    if not key_phrases:
+        return False
+    matches = 0
+    for phrase in key_phrases:
+        for text in assistant_texts:
+            if phrase in text:
+                matches += 1
+                break
+    return matches >= max(2, len(key_phrases) // 5)
+
+
+def analyze_session_feedback(
+    store: SqliteStore, session_id: str, transcript_path: Path
+) -> int:
+    recalls = store.db.execute(
+        "SELECT * FROM recall_log WHERE session_id=? AND hits_count > 0",
+        (session_id,)
+    ).fetchall()
+    if not recalls:
+        return 0
+
+    recalled_ids = set()
+    for r in recalls:
+        log_id = r["id"]
+        rl_time = r["timestamp"]
+        nearby = store.db.execute("""
+            SELECT q.artifact_id FROM memory_quality q
+            JOIN artifacts a ON a.id = q.artifact_id
+            WHERE q.last_recalled BETWEEN ? - 2 AND ? + 2
+              AND a.active = 1
+        """, (rl_time, rl_time)).fetchall()
+        for row in nearby:
+            recalled_ids.add(row["artifact_id"])
+
+    if not recalled_ids:
+        return 0
+
+    assistant_texts = _extract_assistant_texts(transcript_path)
+    if not assistant_texts:
+        return 0
+
+    used_ids = []
+    for aid in recalled_ids:
+        art = store.db.execute(
+            "SELECT text FROM artifacts WHERE id=?", (aid,)
+        ).fetchone()
+        if art and _text_was_used(art["text"], assistant_texts):
+            used_ids.append(aid)
+
+    if used_ids:
+        store.record_usage(used_ids)
+
+    return len(used_ids)

memor/hook_server.py

@@ -0,0 +1,144 @@
+from __future__ import annotations
+import asyncio
+import json
+import os
+import signal
+import sys
+import time
+from pathlib import Path
+
+SOCK_PATH = Path.home() / ".memor" / "hook.sock"
+PID_PATH = Path.home() / ".memor" / "hook.pid"
+DEFAULT_DB = str(Path.home() / ".memor" / "memor.db")
+IDLE_TIMEOUT_S = 600
+
+_embedder = None
+_last_activity = 0.0
+
+_UNSET = object()  # sentinel for "auto-discover embedder"
+
+
+def _get_embedder():
+    global _embedder
+    if _embedder is not None:
+        return _embedder
+    from memor.embed.local import LocalEmbedder
+    _embedder = LocalEmbedder()
+    return _embedder
+
+
+def handle_request(req: dict, *, db_path: str = DEFAULT_DB,
+                   embedder=_UNSET) -> dict:
+    """Process a recall request and return the hook JSON response.
+    This is the core logic, used by both the socket server and inline fallback.
+
+    Pass embedder=None to explicitly indicate no embedder is available (returns
+    the no_embedder status message). Omit embedder (default) to auto-discover."""
+    from memor.recall import recall, _status_message
+    from memor.project import resolve_project
+
+    cwd = req.get("cwd", "")
+    project = resolve_project(cwd) if cwd else "unknown"
+    query = req.get("prompt", "")
+    session_id = req.get("session_id", "")
+
+    if embedder is _UNSET:
+        embedder = _get_embedder()
+    if embedder is None:
+        msg = _status_message("no_embedder", project, 0, 0, 0.0)
+        return {
+            "hookSpecificOutput": {
+                "hookEventName": "UserPromptSubmit",
+                "additionalContext": f"---\n{msg}",
+            }
+        }
+
+    result = recall(query, project, db_path, embedder=embedder, k=8, threshold=0.15,
+                    session_id=session_id)
+
+    if Path(db_path).exists():
+        try:
+            from memor.store.sqlite_store import SqliteStore
+            store = SqliteStore(db_path, dim=embedder.dim)
+            store.log_recall(
+                project=project, query_preview=query[:100],
+                hits_count=result.hits_count, top_score=result.top_score,
+                tokens_injected=result.tokens_injected, latency_ms=result.latency_ms,
+                status=result.status, session_id=session_id)
+            if result.hit_ids:
+                store.record_recall(result.hit_ids)
+        except Exception:
+            pass
+
+    additional_context = result.formatted_context or f"---\n{result.status_message}"
+    return {
+        "hookSpecificOutput": {
+            "hookEventName": "UserPromptSubmit",
+            "additionalContext": additional_context,
+        }
+    }
+
+
+async def _handle_client(reader: asyncio.StreamReader,
+                         writer: asyncio.StreamWriter) -> None:
+    global _last_activity
+    _last_activity = time.time()
+    try:
+        data = await asyncio.wait_for(reader.read(1_000_000), timeout=10)
+        req = json.loads(data.decode())
+        resp = handle_request(req)
+        writer.write(json.dumps(resp).encode())
+        await writer.drain()
+    except Exception as e:
+        err = json.dumps({"error": str(e)})
+        writer.write(err.encode())
+        await writer.drain()
+    finally:
+        writer.close()
+        await writer.wait_closed()
+
+
+async def _idle_watchdog():
+    global _last_activity
+    while True:
+        await asyncio.sleep(60)
+        if time.time() - _last_activity > IDLE_TIMEOUT_S:
+            _cleanup()
+            os._exit(0)
+
+
+def _cleanup():
+    if SOCK_PATH.exists():
+        SOCK_PATH.unlink()
+    if PID_PATH.exists():
+        PID_PATH.unlink()
+
+
+async def serve(sock_path: str = str(SOCK_PATH)) -> None:
+    global _last_activity
+    _last_activity = time.time()
+    p = Path(sock_path)
+    if p.exists():
+        p.unlink()
+    p.parent.mkdir(parents=True, exist_ok=True)
+
+    _get_embedder()
+
+    server = await asyncio.start_unix_server(_handle_client, path=sock_path)
+    PID_PATH.write_text(str(os.getpid()))
+    asyncio.create_task(_idle_watchdog())
+
+    async with server:
+        await server.serve_forever()
+
+
+def main():
+    signal.signal(signal.SIGTERM, lambda *_: (_cleanup(), sys.exit(0)))
+    try:
+        asyncio.run(serve())
+    except KeyboardInterrupt:
+        _cleanup()
+
+
+if __name__ == "__main__":
+    main()

memor/ingest/claude_code.py

@@ -0,0 +1,135 @@
+from __future__ import annotations
+import hashlib, json, re
+from datetime import datetime
+from pathlib import Path
+from memor.types import Artifact
+from memor.tokencount import count_tokens
+from memor.redact import redact_text
+
+def _epoch(ts: str) -> float:
+    return datetime.fromisoformat(ts.replace("Z", "+00:00")).timestamp()
+
+def _text_of(message: dict) -> str:
+    c = message.get("content", "")
+    if isinstance(c, str):
+        return c
+    parts = []
+    for block in c:
+        if isinstance(block, dict) and block.get("type") == "text":
+            parts.append(block.get("text", ""))
+    return "\n".join(parts)
+
+_FILLER_STARTS = re.compile(
+    r"^(Let me |Now let me |Now I|I'll |I will |Good[,.]|Great[,.]|Perfect[!,.]|Done[!,.]"
+    r"|Alright|Sure[,.]|OK[,.]|Moving to |Looking at |Checking )", re.I)
+_SKILL_BOILERPLATE = "Base directory for this skill"
+
+_DECISION_RE = re.compile(
+    r"(we decided|the approach is|instead of|switched to|chose .+ over|"
+    r"trade-?off|architecture:|design decision)", re.I)
+_BUGFIX_RE = re.compile(
+    r"(the fix is|root cause|the bug was|the issue was|caused by|"
+    r"the problem is|this fails because|the error occurs)", re.I)
+_LESSON_RE = re.compile(
+    r"(always use|never use|never do|important:|note:|pattern:|"
+    r"best practice|lesson learned|rule of thumb|should always|should never)", re.I)
+
+_BASE64_RE = re.compile(r"[A-Za-z0-9+/=]{200,}")
+_PERMISSION_RE = re.compile(r"^(Allow |Permission |Do you want to allow |Grant )", re.I)
+_PATH_LINE_RE = re.compile(r"^[/~][\w./\-]+$")
+_SYSTEM_REMINDER_RE = re.compile(r"<system-reminder>.*?</system-reminder>", re.DOTALL)
+
+MIN_SIGNAL_TOKENS = 8
+MIN_FILLER_TOKENS = 30
+MIN_USER_TOKENS = 6
+MIN_CODE_TOKENS = 40
+MIN_LONG_TOKENS = 100
+
+
+def _strip_system_reminders(text: str) -> str:
+    """Strip <system-reminder>...</system-reminder> tags and their content."""
+    return _SYSTEM_REMINDER_RE.sub("", text)
+
+
+def _is_file_listing(text: str) -> bool:
+    """Return True if text is a pure file listing (multiple lines of just paths)."""
+    lines = [l for l in text.splitlines() if l.strip()]
+    if len(lines) < 2:
+        return False
+    return all(_PATH_LINE_RE.match(l.strip()) for l in lines)
+
+
+def _signal_score(text: str, role: str, token_count: int) -> float:
+    """Return a score > 0 for signal content, 0 for noise."""
+    # Hard noise filters
+    if token_count < MIN_SIGNAL_TOKENS:
+        return 0.0
+    if _SKILL_BOILERPLATE in text:
+        return 0.0
+    if _BASE64_RE.search(text):
+        return 0.0
+    if _PERMISSION_RE.match(text):
+        return 0.0
+    if _is_file_listing(text):
+        return 0.0
+    if _FILLER_STARTS.match(text) and token_count < MIN_FILLER_TOKENS:
+        return 0.0
+
+    score = 0.0
+
+    if role == "user" and token_count >= MIN_USER_TOKENS:
+        score += 1.0
+    elif role == "assistant":
+        if _DECISION_RE.search(text):
+            score += 2.0
+        if _BUGFIX_RE.search(text):
+            score += 2.0
+        if _LESSON_RE.search(text):
+            score += 2.0
+        if "```" in text and token_count >= MIN_CODE_TOKENS:
+            score += 1.0
+        if score == 0.0 and token_count >= MIN_LONG_TOKENS:
+            score += 0.5
+
+    return score
+
+
+def parse_transcript(path: Path, project: str, *, filter_noise: bool = True) -> list[Artifact]:
+    session_id = path.stem
+    arts: list[Artifact] = []
+    seen_hashes: set[str] = set()
+    for i, line in enumerate(path.read_text().splitlines()):
+        line = line.strip()
+        if not line:
+            continue
+        rec = json.loads(line)
+        if rec.get("type") not in ("user", "assistant"):
+            continue
+        msg = rec.get("message", {})
+        text = _text_of(msg).strip()
+        if not text:
+            continue
+        text = _strip_system_reminders(text).strip()
+        if not text:
+            continue
+        text, _ = redact_text(text)
+        if not text.strip():
+            continue
+        # Deduplicate by MD5 hash within session
+        text_hash = hashlib.md5(text.encode()).hexdigest()
+        if text_hash in seen_hashes:
+            continue
+        seen_hashes.add(text_hash)
+        token_count = max(1, count_tokens(text))
+        role = msg.get("role", "")
+        if filter_noise and _signal_score(text, role, token_count) == 0:
+            continue
+        arts.append(Artifact(
+            id=f"{session_id}:{i}", kind="session_chunk", project=project,
+            source="claude_code", text=text, token_count=token_count,
+            created_at=_epoch(rec["timestamp"]),
+            meta={"session_id": session_id, "role": role, "ord": i}))
+    return arts
+
+def discover_project(transcript_path: Path) -> str:
+    return transcript_path.parent.name

memor/ingest/documents.py

@@ -0,0 +1,28 @@
+from __future__ import annotations
+import hashlib, re
+from pathlib import Path
+from memor.types import Artifact
+
+def parse_document(path: Path, project: str, kind: str = "note",
+                   created_at: float = 0.0) -> list[Artifact]:
+    text = path.read_text()
+    # split on markdown headings; keep the heading with its body
+    parts = re.split(r"(?m)^(#{1,6}\s.*)$", text)
+    chunks: list[str] = []
+    buf = ""
+    for seg in parts:
+        if re.match(r"^#{1,6}\s", seg or ""):
+            if buf.strip():
+                chunks.append(buf.strip())
+            buf = seg + "\n"
+        else:
+            buf += seg
+    if buf.strip():
+        chunks.append(buf.strip())
+    arts = []
+    for i, c in enumerate(chunks):
+        cid = f"{kind}:{path.stem}:{hashlib.sha1(c.encode()).hexdigest()[:8]}"
+        arts.append(Artifact(id=cid, kind=kind, project=project, source=str(path),
+                             text=c, token_count=max(1, len(c)//4),
+                             created_at=created_at, meta={"path": str(path), "ord": i}))
+    return arts

memor/interfaces.py

@@ -0,0 +1,20 @@
+from __future__ import annotations
+from typing import Protocol, runtime_checkable
+from memor.types import Artifact, Scope
+
+@runtime_checkable
+class Embedder(Protocol):
+    dim: int
+    def embed(self, texts: list[str]) -> list[list[float]]: ...
+
+@runtime_checkable
+class LLM(Protocol):
+    def complete(self, prompt: str, *, max_tokens: int = 1024) -> str: ...
+
+@runtime_checkable
+class MemoryStore(Protocol):
+    def add_artifacts(self, artifacts: list[Artifact], vectors: list[list[float]]) -> None: ...
+    def add_edge(self, src_id: str, dst_id: str, type: str) -> None: ...
+    def search(self, vector: list[float], scope: Scope, k: int) -> list[tuple[Artifact, float]]: ...
+    def neighbors(self, ids: list[str], types: list[str], hops: int = 1) -> list[Artifact]: ...
+    def deactivate(self, artifact_id: str, superseded_by: str) -> None: ...

memor/llm/anthropic.py

@@ -0,0 +1,14 @@
+class AnthropicLLM:
+    def __init__(self, model: str = "claude-opus-4-8", api_key: str | None = None):
+        import anthropic
+
+        self.client = anthropic.Anthropic(api_key=api_key)
+        self.model = model
+
+    def complete(self, prompt: str, *, max_tokens: int = 1024) -> str:
+        msg = self.client.messages.create(
+            model=self.model,
+            max_tokens=max_tokens,
+            messages=[{"role": "user", "content": prompt}],
+        )
+        return "".join(b.text for b in msg.content if b.type == "text")

memor/llm/base.py

@@ -0,0 +1,7 @@
+DISTILL_PROMPT = """You are distilling a coding session into reusable memory.
+Return STRICT JSON: {{"memories":[{{"type":"decision|lesson|snippet|bugfix",
+"text":"<one concise reusable fact>", "supersedes_text":"<prior fact this reverses, or omit>"}}]}}
+Only include durable, reusable facts. Session text:
+---
+{session_text}
+---"""

memor/llm/openai_compat.py

@@ -0,0 +1,20 @@
+import httpx
+
+
+class OpenAICompatLLM:
+    def __init__(self, base_url: str, api_key: str, model: str):
+        self.base_url, self.api_key, self.model = base_url, api_key, model
+
+    def complete(self, prompt: str, *, max_tokens: int = 1024) -> str:
+        r = httpx.post(
+            f"{self.base_url}/chat/completions",
+            headers={"Authorization": f"Bearer {self.api_key}"},
+            json={
+                "model": self.model,
+                "max_tokens": max_tokens,
+                "messages": [{"role": "user", "content": prompt}],
+            },
+            timeout=60,
+        )
+        r.raise_for_status()
+        return r.json()["choices"][0]["message"]["content"]

memor/project.py

@@ -0,0 +1,69 @@
+from __future__ import annotations
+from pathlib import Path
+
+
+def resolve_project(cwd: str) -> str:
+    """Resolve project name from a working directory by finding the git root."""
+    p = Path(cwd).resolve()
+    git_root = _find_git_root(p)
+    if git_root:
+        return git_root.name
+    return p.name
+
+
+def _find_git_root(path: Path) -> Path | None:
+    current = path
+    while current != current.parent:
+        if (current / ".git").exists():
+            return current
+        current = current.parent
+    return None
+
+
+def decode_claude_dir(dirname: str) -> str:
+    """Decode a Claude projects directory name back to a filesystem path.
+    '-Users-nimit-Documents-Projects-plirin' -> '/Users/nimit/Documents/Projects/plirin'
+
+    Dashes in the encoded name are ambiguous — they may represent either a path
+    separator or a literal dash in a directory name. This function walks the
+    real filesystem to resolve ambiguity greedily (longest matching segment wins).
+    Falls back to naive split-on-dash if the filesystem walk yields no result.
+    """
+    stripped = dirname.lstrip("-")
+    tokens = stripped.split("-")
+
+    result = _fs_decode(tokens)
+    if result is not None:
+        return result
+
+    # Naive fallback: treat every dash as a separator
+    return "/" + "/".join(tokens)
+
+
+def _fs_decode(tokens: list[str]) -> str | None:
+    """Greedily match tokens against the real filesystem, joining dashes as needed."""
+    path = Path("/")
+    remaining = list(tokens)
+
+    while remaining:
+        # Try to match a segment by joining increasing numbers of tokens with dashes
+        matched = False
+        for end in range(len(remaining), 0, -1):
+            candidate = "-".join(remaining[:end])
+            candidate_path = path / candidate
+            if candidate_path.exists():
+                path = candidate_path
+                remaining = remaining[end:]
+                matched = True
+                break
+        if not matched:
+            # No filesystem match found; give up and return None to trigger fallback
+            return None
+
+    return str(path)
+
+
+def resolve_project_from_claude_dir(dirname: str) -> str:
+    """Resolve project name from a Claude projects directory name."""
+    decoded = decode_claude_dir(dirname)
+    return resolve_project(decoded)