memor-cli 0.1.0__py3-none-any.whl

memor/recall.py

@@ -0,0 +1,115 @@
+from __future__ import annotations
+import time
+from dataclasses import dataclass
+from datetime import datetime
+from pathlib import Path
+from memor.types import Scope
+
+
+@dataclass
+class RecallResult:
+    hits_count: int
+    top_score: float
+    tokens_injected: int
+    latency_ms: float
+    status: str
+    status_message: str
+    formatted_context: str
+    hit_ids: list[str] = None
+
+
+def _format_timestamp(epoch: float) -> str:
+    return datetime.fromtimestamp(epoch).strftime("%Y-%m-%d")
+
+
+def _detect_status(store, project: str, hits_count: int) -> str:
+    if hits_count > 0:
+        llm_mems = store.db.execute("""
+            SELECT COUNT(*) as c FROM artifacts
+            WHERE kind='memory' AND project=? AND active=1
+              AND json_extract(meta, '$.mem_type') IN ('decision','lesson','snippet','bugfix')
+        """, (project,)).fetchone()["c"]
+        if llm_mems > 0:
+            return "ok"
+        return "extractive_only"
+    return "no_hits"
+
+
+def _status_message(status: str, project: str, hits_count: int,
+                    tokens: int, top_score: float) -> str:
+    if status == "ok":
+        return f"Memor: recalled {hits_count} memories ({tokens} tokens, {top_score:.2f} top score)"
+    if status == "extractive_only":
+        return f"Memor: recalled {hits_count} memories ({tokens} tokens, {top_score:.2f} top score)"
+    if status == "no_hits":
+        return f'Memor: no relevant memories for project "{project}" yet'
+    if status == "empty_db":
+        return 'Memor: memory store is empty — run "memor daemon" to start ingesting sessions'
+    if status == "no_embedder":
+        return "Memor: inactive — run 'memor setup-model' to download the embedding model"
+    return f"Memor: status={status}"
+
+
+def recall(query: str, project: str, db_path: str, *,
+           embedder=None, k: int = 8, threshold: float = 0.3,
+           session_id: str = "") -> RecallResult:
+    t0 = time.perf_counter()
+
+    if not Path(db_path).exists():
+        ms = (time.perf_counter() - t0) * 1000
+        return RecallResult(
+            hits_count=0, top_score=0.0, tokens_injected=0,
+            latency_ms=ms, status="empty_db",
+            status_message=_status_message("empty_db", project, 0, 0, 0.0),
+            formatted_context="")
+
+    from memor.store.sqlite_store import SqliteStore
+    from memor.retrieve.retriever import Retriever
+
+    store = SqliteStore(db_path, dim=embedder.dim)
+    retriever = Retriever(store, embedder, k=k)
+    trace = retriever.query(query, Scope(project=project))
+
+    hits = list(trace.hits)
+    if session_id:
+        hits = [h for h in hits if h.artifact.meta.get("session_id") != session_id]
+    if threshold > 0.0:
+        hits = [h for h in hits if h.score >= threshold]
+    top_score = hits[0].score if hits else 0.0
+    tokens = sum(h.artifact.token_count for h in hits)
+
+    if not hits:
+        status = "no_hits"
+    else:
+        status = _detect_status(store, project, len(hits))
+
+    msg = _status_message(status, project, len(hits), tokens, top_score)
+
+    lines = []
+    if hits:
+        lines.append(f"## Recalled Memories (project: {project})")
+        lines.append("")
+        for i, h in enumerate(hits, 1):
+            a = h.artifact
+            kind_tag = a.meta.get("mem_type", a.kind)
+            text = a.text if len(a.text) <= 600 else a.text[:600] + "..."
+            source_parts = []
+            sid = a.meta.get("session_id")
+            if sid:
+                source_parts.append(f"session {sid[:8]}")
+            source_parts.append(_format_timestamp(a.created_at))
+            source = ", ".join(source_parts)
+            lines.append(f"### {i}. [{kind_tag}] {text}")
+            lines.append(f"Source: {source} | score: {h.score:.3f}")
+            lines.append("")
+
+    lines.append("---")
+    lines.append(msg)
+    formatted = "\n".join(lines)
+
+    ms = (time.perf_counter() - t0) * 1000
+    return RecallResult(
+        hits_count=len(hits), top_score=top_score, tokens_injected=tokens,
+        latency_ms=ms, status=status, status_message=msg,
+        formatted_context=formatted,
+        hit_ids=[h.artifact.id for h in hits])

memor/redact.py

@@ -0,0 +1,129 @@
+"""Secret detection and redaction — runs at ingest before embedding or storage.
+
+Catches: API keys (AWS, OpenAI, GitHub, Anthropic, Stripe, etc.), JWTs,
+PEM blocks, connection strings, .env-style assignments, high-entropy tokens.
+Redacts in-place to preserve surrounding signal."""
+from __future__ import annotations
+import math
+import re
+
+_PLACEHOLDER = "[REDACTED]"
+
+# --- Pattern-based detection ---
+
+_AWS_KEY = re.compile(r"AKIA[0-9A-Z]{16}")
+_AWS_SECRET = re.compile(r"(?:aws.{0,20})?[0-9a-zA-Z/+]{40}(?=\s|$|\")")
+_OPENAI_KEY = re.compile(r"sk-[a-zA-Z0-9_-]{20,}")
+_ANTHROPIC_KEY = re.compile(r"sk-ant-[a-zA-Z0-9_-]{20,}")
+_GITHUB_TOKEN = re.compile(r"gh[ps]_[A-Za-z0-9_]{36,}")
+_GITHUB_FINE = re.compile(r"github_pat_[A-Za-z0-9_]{20,}")
+_STRIPE_KEY = re.compile(r"[sr]k_(live|test)_[A-Za-z0-9]{20,}")
+_SLACK_TOKEN = re.compile(r"xox[bpras]-[A-Za-z0-9\-]{10,}")
+_GENERIC_KEY = re.compile(r"(?:key|token|secret|password|apikey|api_key)[\"']?\s*[=:]\s*['\"]?([^\s'\"]{8,})", re.I)
+_JWT = re.compile(r"eyJ[A-Za-z0-9_-]{10,}\.eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}")
+_PEM_BLOCK = re.compile(r"-----BEGIN [A-Z ]+-----.*?-----END [A-Z ]+-----", re.DOTALL)
+_CONN_STRING = re.compile(
+    r"(?:postgres(?:ql)?|mysql|mongodb(?:\+srv)?|redis|amqp|sqlite)"
+    r"://[^\s\"'`]{10,}", re.I)
+_ENV_ASSIGNMENT = re.compile(
+    r"^[A-Z_]{2,50}(?:_KEY|_SECRET|_TOKEN|_PASSWORD|_PASS|_API|_CREDENTIAL)[=:]\s*\S+",
+    re.MULTILINE)
+
+_PATTERNS: list[tuple[str, re.Pattern]] = [
+    ("aws_key", _AWS_KEY),
+    ("aws_secret", _AWS_SECRET),
+    ("openai_key", _OPENAI_KEY),
+    ("anthropic_key", _ANTHROPIC_KEY),
+    ("github_token", _GITHUB_TOKEN),
+    ("github_fine", _GITHUB_FINE),
+    ("stripe_key", _STRIPE_KEY),
+    ("slack_token", _SLACK_TOKEN),
+    ("jwt", _JWT),
+    ("pem", _PEM_BLOCK),
+    ("connection_string", _CONN_STRING),
+    ("env_assignment", _ENV_ASSIGNMENT),
+    ("generic_key", _GENERIC_KEY),
+]
+
+# --- Entropy-based detection for unrecognized high-entropy tokens ---
+
+_TOKEN_RE = re.compile(r"[A-Za-z0-9_/+\-]{20,}")
+
+def _shannon_entropy(s: str) -> float:
+    if not s:
+        return 0.0
+    freq: dict[str, int] = {}
+    for c in s:
+        freq[c] = freq.get(c, 0) + 1
+    n = len(s)
+    return -sum((count / n) * math.log2(count / n) for count in freq.values())
+
+_ENTROPY_THRESHOLD = 4.0
+_MIN_ENTROPY_LEN = 20
+
+
+def detect_secrets(text: str) -> list[tuple[str, str]]:
+    """Return list of (pattern_name, matched_string) for all secrets found."""
+    found: list[tuple[str, str]] = []
+    for name, pattern in _PATTERNS:
+        for m in pattern.finditer(text):
+            matched = m.group(1) if m.lastindex and name == "generic_key" else m.group(0)
+            found.append((name, matched))
+
+    for m in _TOKEN_RE.finditer(text):
+        token = m.group(0)
+        if len(token) >= _MIN_ENTROPY_LEN and _shannon_entropy(token) >= _ENTROPY_THRESHOLD:
+            already = any(token in s for _, s in found)
+            if not already:
+                found.append(("high_entropy", token))
+    return found
+
+
+def redact_text(text: str) -> tuple[str, int]:
+    """Redact secrets in text. Returns (redacted_text, count_of_redactions)."""
+    secrets = detect_secrets(text)
+    if not secrets:
+        return text, 0
+    result = text
+    count = 0
+    for _, secret in sorted(secrets, key=lambda x: -len(x[1])):
+        if secret in result:
+            result = result.replace(secret, _PLACEHOLDER)
+            count += 1
+    return result, count
+
+
+def scan_artifacts(store) -> list[dict]:
+    """Scan all active artifacts for secrets. Returns list of findings."""
+    rows = store.db.execute(
+        "SELECT id, project, text, kind FROM artifacts WHERE active = 1"
+    ).fetchall()
+    findings = []
+    for r in rows:
+        secrets = detect_secrets(r["text"])
+        if secrets:
+            findings.append({
+                "artifact_id": r["id"],
+                "project": r["project"],
+                "kind": r["kind"],
+                "secrets": [(name, s[:20] + "...") for name, s in secrets],
+            })
+    return findings
+
+
+def purge_secrets_from_db(store) -> int:
+    """Redact secrets from all active artifacts in place. Returns count of affected artifacts."""
+    rows = store.db.execute(
+        "SELECT id, text FROM artifacts WHERE active = 1"
+    ).fetchall()
+    affected = 0
+    for r in rows:
+        redacted, count = redact_text(r["text"])
+        if count > 0:
+            store.db.execute(
+                "UPDATE artifacts SET text = ? WHERE id = ?",
+                (redacted, r["id"]))
+            affected += 1
+    if affected:
+        store.db.commit()
+    return affected

memor/retrieve/retriever.py

@@ -0,0 +1,78 @@
+from __future__ import annotations
+import math
+import time
+from memor.types import Scope, Hit, RetrievalTrace
+from memor.interfaces import Embedder, MemoryStore
+
+EDGE_TYPES = ["fixes", "supersedes", "part_of", "derived_from"]
+
+KIND_WEIGHTS = {
+    "memory": 1.3,
+    "session_chunk": 1.0,
+    "note": 1.1,
+}
+
+# Half-life in days: memories older than this get half the recency boost
+RECENCY_HALF_LIFE_DAYS = 14
+
+
+class Retriever:
+    def __init__(self, store: MemoryStore, embedder: Embedder, *,
+                 k: int = 8, recency_weight: float = 0.25,
+                 kind_weight: float = 0.15, quality_weight: float = 0.10,
+                 edge_expand: bool = True):
+        self.store, self.embedder = store, embedder
+        self.k, self.edge_expand = k, edge_expand
+        self.w_sim = 1.0 - recency_weight - kind_weight - quality_weight
+        self.w_rec = recency_weight
+        self.w_kind = kind_weight
+        self.w_qual = quality_weight
+
+    def query(self, text: str, scope: Scope) -> RetrievalTrace:
+        t0 = time.perf_counter()
+        now = time.time()
+        qv = self.embedder.embed([text])[0]
+        base = self.store.search(qv, scope, self.k)
+
+        candidates = len(base)
+        hits: dict[str, Hit] = {}
+
+        sim_scores = [sim for _, sim in base]
+        sim_max = max(sim_scores) if sim_scores else 1.0
+        sim_min = min(sim_scores) if sim_scores else 0.0
+        sim_range = (sim_max - sim_min) or 1.0
+
+        quality_cache = {}
+        has_quality = hasattr(self.store, 'get_quality_score')
+
+        for a, sim in base:
+            norm_sim = (sim - sim_min) / sim_range
+
+            age_days = (now - a.created_at) / 86400
+            recency = math.exp(-0.693 * age_days / RECENCY_HALF_LIFE_DAYS)
+
+            kind_boost = KIND_WEIGHTS.get(a.kind, 1.0) - 1.0
+
+            if has_quality and a.id not in quality_cache:
+                quality_cache[a.id] = self.store.get_quality_score(a.id)
+            quality = quality_cache.get(a.id, 0.5)
+
+            score = (self.w_sim * norm_sim + self.w_rec * recency
+                     + self.w_kind * kind_boost + self.w_qual * quality)
+            hits[a.id] = Hit(a, score, {
+                "sim": sim, "norm_sim": round(norm_sim, 3),
+                "recency": round(recency, 3), "kind": a.kind,
+                "quality": round(quality, 3), "edge": 0.0,
+            })
+
+        if self.edge_expand and base:
+            seed_ids = [a.id for a, _ in base]
+            for nb in self.store.neighbors(seed_ids, EDGE_TYPES, hops=1):
+                if nb.id not in hits:
+                    hits[nb.id] = Hit(nb, 0.5 * max(h.score for h in hits.values()),
+                                      {"sim": 0.0, "norm_sim": 0.0,
+                                       "recency": 0.0, "kind": nb.kind, "edge": 1.0})
+
+        ranked = sorted(hits.values(), key=lambda h: h.score, reverse=True)[:self.k]
+        return RetrievalTrace(query=text, scope=scope, candidates=candidates,
+                              hits=ranked, latency_ms=(time.perf_counter()-t0)*1000)

memor/store/sqlite_store.py

@@ -0,0 +1,336 @@
+from __future__ import annotations
+import json, sqlite3, struct
+from pathlib import Path
+import sqlite_vec
+from memor.types import Artifact, Scope
+
+def _serialize(v: list[float]) -> bytes:
+    return struct.pack("%sf" % len(v), *v)
+
+class SqliteStore:
+    def __init__(self, path: str, dim: int):
+        self.dim = dim
+        Path(path).parent.mkdir(parents=True, exist_ok=True)
+        self.db = sqlite3.connect(path, check_same_thread=False)
+        self.db.execute("PRAGMA journal_mode=WAL")
+        self.db.row_factory = sqlite3.Row
+        self.db.enable_load_extension(True)
+        sqlite_vec.load(self.db)
+        self.db.enable_load_extension(False)
+        self._init_schema()
+        self._check_dim(dim)
+
+    def _init_schema(self):
+        self.db.executescript(f"""
+        CREATE TABLE IF NOT EXISTS artifacts(
+          id TEXT PRIMARY KEY, kind TEXT, project TEXT, source TEXT,
+          text TEXT, token_count INTEGER, created_at REAL, meta TEXT,
+          active INTEGER DEFAULT 1, superseded_by TEXT);
+        CREATE TABLE IF NOT EXISTS edges(
+          src_id TEXT, dst_id TEXT, type TEXT,
+          PRIMARY KEY(src_id, dst_id, type));
+        CREATE INDEX IF NOT EXISTS idx_art_project ON artifacts(project, active);
+        CREATE VIRTUAL TABLE IF NOT EXISTS vec_artifacts USING vec0(
+          embedding float[{self.dim}]);
+        CREATE TABLE IF NOT EXISTS eval_runs(
+          id INTEGER PRIMARY KEY AUTOINCREMENT, created_at REAL, config TEXT, metrics TEXT);
+        CREATE TABLE IF NOT EXISTS meta(key TEXT PRIMARY KEY, value TEXT);
+        CREATE TABLE IF NOT EXISTS memory_quality(
+          artifact_id TEXT PRIMARY KEY,
+          recall_count INTEGER DEFAULT 0,
+          use_count INTEGER DEFAULT 0,
+          last_recalled REAL,
+          quality_score REAL DEFAULT 0.5);
+        CREATE TABLE IF NOT EXISTS recall_log(
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          timestamp REAL, project TEXT, query_preview TEXT,
+          hits_count INTEGER, top_score REAL, tokens_injected INTEGER,
+          latency_ms REAL, status TEXT, session_id TEXT);
+        """)
+        self.db.commit()
+
+    def _check_dim(self, dim: int):
+        row = self.db.execute("SELECT value FROM meta WHERE key='dim'").fetchone()
+        if row is None:
+            self.db.execute("INSERT INTO meta(key, value) VALUES('dim', ?)", (str(dim),))
+            self.db.commit()
+        elif int(row["value"]) != dim:
+            raise SystemExit(
+                f"Embedding dimension mismatch: database was created with dim={row['value']} "
+                f"but current embedder has dim={dim}. Use the same embedder or re-ingest."
+            )
+
+    def add_artifacts(self, artifacts: list[Artifact], vectors: list[list[float]]) -> None:
+        cur = self.db.cursor()
+        for a, v in zip(artifacts, vectors):
+            cur.execute(
+              "INSERT OR REPLACE INTO artifacts(id,kind,project,source,text,token_count,created_at,meta,active,superseded_by)"
+              " VALUES(?,?,?,?,?,?,?,?,1,NULL)",
+              (a.id, a.kind, a.project, a.source, a.text, a.token_count, a.created_at, json.dumps(a.meta)))
+            rowid = cur.execute("SELECT rowid FROM artifacts WHERE id=?", (a.id,)).fetchone()[0]
+            cur.execute("INSERT OR REPLACE INTO vec_artifacts(rowid, embedding) VALUES(?,?)",
+                        (rowid, _serialize(v)))
+        self.db.commit()
+
+    def add_edge(self, src_id: str, dst_id: str, type: str) -> None:
+        self.db.execute("INSERT OR IGNORE INTO edges(src_id,dst_id,type) VALUES(?,?,?)",
+                        (src_id, dst_id, type))
+        self.db.commit()
+
+    def _row_to_artifact(self, r) -> Artifact:
+        return Artifact(id=r["id"], kind=r["kind"], project=r["project"], source=r["source"],
+                        text=r["text"], token_count=r["token_count"], created_at=r["created_at"],
+                        meta=json.loads(r["meta"]))
+
+    def search(self, vector: list[float], scope: Scope, k: int) -> list[tuple[Artifact, float]]:
+        rows = self.db.execute(f"""
+          SELECT a.*, v.distance AS distance
+          FROM (SELECT rowid, distance FROM vec_artifacts
+                WHERE embedding MATCH ? AND k = ?) v
+          JOIN artifacts a ON a.rowid = v.rowid
+          WHERE a.active = 1
+            AND (? IS NULL OR a.project = ?)
+            AND (? IS NULL OR a.created_at >= ?)
+            AND (? IS NULL OR a.created_at <= ?)
+          ORDER BY v.distance ASC
+        """, (_serialize(vector), max(k*20, 200),
+              scope.project, scope.project,
+              scope.since, scope.since,
+              scope.until, scope.until)).fetchall()
+        out = []
+        for r in rows[:k]:
+            if scope.kinds is not None and r["kind"] not in scope.kinds:
+                continue
+            sim = 1.0 - float(r["distance"])
+            out.append((self._row_to_artifact(r), sim))
+        return out
+
+    def neighbors(self, ids: list[str], types: list[str], hops: int = 1) -> list[Artifact]:
+        qmarks_ids = ",".join("?" * len(ids))
+        qmarks_types = ",".join("?" * len(types))
+        rows = self.db.execute(f"""
+          WITH RECURSIVE walk(id, depth) AS (
+            SELECT dst_id, 1 FROM edges
+              WHERE src_id IN ({qmarks_ids}) AND type IN ({qmarks_types})
+            UNION
+            SELECT e.dst_id, w.depth+1 FROM edges e JOIN walk w ON e.src_id = w.id
+              WHERE w.depth < ? AND e.type IN ({qmarks_types})
+          )
+          SELECT DISTINCT a.* FROM artifacts a JOIN walk ON a.id = walk.id
+          WHERE a.active = 1
+        """, (*ids, *types, hops, *types)).fetchall()
+        return [self._row_to_artifact(r) for r in rows]
+
+    def deactivate(self, artifact_id: str, superseded_by: str) -> None:
+        self.db.execute("UPDATE artifacts SET active=0, superseded_by=? WHERE id=?",
+                        (superseded_by, artifact_id))
+        self.add_edge(superseded_by, artifact_id, "supersedes")
+
+    def recent(self, scope: Scope, k: int) -> list[Artifact]:
+        """Return the k most recent active artifacts matching scope, ordered by created_at DESC."""
+        rows = self.db.execute("""
+          SELECT * FROM artifacts WHERE active = 1
+            AND (? IS NULL OR project = ?)
+            AND (? IS NULL OR created_at >= ?)
+            AND (? IS NULL OR created_at <= ?)
+          ORDER BY created_at DESC LIMIT ?
+        """, (scope.project, scope.project, scope.since, scope.since,
+              scope.until, scope.until, k)).fetchall()
+        return [self._row_to_artifact(r) for r in rows]
+
+    def save_eval_run(self, config: dict, metrics: dict) -> int:
+        import time
+        cur = self.db.execute("INSERT INTO eval_runs(created_at, config, metrics) VALUES(?,?,?)",
+                              (time.time(), json.dumps(config), json.dumps(metrics)))
+        self.db.commit()
+        return cur.lastrowid
+
+    def log_recall(self, project: str, query_preview: str, hits_count: int,
+                   top_score: float, tokens_injected: int, latency_ms: float,
+                   status: str, session_id: str = "") -> None:
+        import time as _time
+        self.db.execute(
+            "INSERT INTO recall_log(timestamp,project,query_preview,hits_count,"
+            "top_score,tokens_injected,latency_ms,status,session_id) "
+            "VALUES(?,?,?,?,?,?,?,?,?)",
+            (_time.time(), project, query_preview[:100], hits_count, top_score,
+             tokens_injected, latency_ms, status, session_id))
+        self.db.commit()
+
+    def get_recall_stats(self) -> dict:
+        r = self.db.execute("""
+            SELECT COUNT(*) as total,
+                   SUM(tokens_injected) as tokens,
+                   AVG(latency_ms) as avg_latency,
+                   SUM(CASE WHEN hits_count > 0 THEN 1 ELSE 0 END) as with_hits
+            FROM recall_log
+        """).fetchone()
+        total = r["total"] or 0
+        return {
+            "total_recalls": total,
+            "total_tokens": r["tokens"] or 0,
+            "avg_latency_ms": round(r["avg_latency"] or 0, 1),
+            "hit_rate": round((r["with_hits"] or 0) / total, 3) if total > 0 else 0,
+        }
+
+    def get_project_stats(self) -> list[dict]:
+        rows = self.db.execute("""
+            SELECT project,
+                   COUNT(*) as recalls,
+                   SUM(tokens_injected) as tokens,
+                   AVG(CASE WHEN hits_count > 0 THEN top_score END) as avg_score,
+                   SUM(CASE WHEN status='ok' THEN 1 ELSE 0 END) as ok_count,
+                   SUM(CASE WHEN status='no_hits' THEN 1 ELSE 0 END) as no_hits_count,
+                   SUM(CASE WHEN status='extractive_only' THEN 1 ELSE 0 END) as extractive_count
+            FROM recall_log
+            GROUP BY project
+            ORDER BY recalls DESC
+        """).fetchall()
+        return [dict(r) for r in rows]
+
+    def get_recent_recalls(self, limit: int = 50, project: str | None = None) -> list[dict]:
+        if project:
+            rows = self.db.execute(
+                "SELECT * FROM recall_log WHERE project=? ORDER BY timestamp DESC LIMIT ?",
+                (project, limit)).fetchall()
+        else:
+            rows = self.db.execute(
+                "SELECT * FROM recall_log ORDER BY timestamp DESC LIMIT ?",
+                (limit,)).fetchall()
+        return [dict(r) for r in rows]
+
+    def record_recall(self, artifact_ids: list[str]) -> None:
+        import time as _time
+        now = _time.time()
+        for aid in artifact_ids:
+            self.db.execute("""
+                INSERT INTO memory_quality(artifact_id, recall_count, use_count, last_recalled, quality_score)
+                VALUES(?, 1, 0, ?, 0.5)
+                ON CONFLICT(artifact_id) DO UPDATE SET
+                  recall_count = recall_count + 1,
+                  last_recalled = ?
+            """, (aid, now, now))
+        self.db.commit()
+
+    def record_usage(self, artifact_ids: list[str]) -> None:
+        for aid in artifact_ids:
+            self.db.execute("""
+                UPDATE memory_quality SET use_count = use_count + 1 WHERE artifact_id = ?
+            """, (aid,))
+        self.db.commit()
+        self._recompute_quality(artifact_ids)
+
+    def _recompute_quality(self, artifact_ids: list[str]) -> None:
+        for aid in artifact_ids:
+            row = self.db.execute(
+                "SELECT recall_count, use_count FROM memory_quality WHERE artifact_id=?",
+                (aid,)).fetchone()
+            if row:
+                rc = row["recall_count"] or 1
+                uc = row["use_count"] or 0
+                score = (uc + 1) / (rc + 2)
+                self.db.execute(
+                    "UPDATE memory_quality SET quality_score=? WHERE artifact_id=?",
+                    (round(score, 3), aid))
+        self.db.commit()
+
+    def get_quality_score(self, artifact_id: str) -> float:
+        row = self.db.execute(
+            "SELECT quality_score FROM memory_quality WHERE artifact_id=?",
+            (artifact_id,)).fetchone()
+        return row["quality_score"] if row else 0.5
+
+    def get_stale_memories(self, days: int = 30) -> list[str]:
+        import time as _time
+        cutoff = _time.time() - (days * 86400)
+        rows = self.db.execute("""
+            SELECT a.id FROM artifacts a
+            LEFT JOIN memory_quality q ON a.id = q.artifact_id
+            WHERE a.kind = 'memory' AND a.active = 1
+              AND (q.artifact_id IS NULL OR q.last_recalled IS NULL OR q.last_recalled < ?)
+              AND a.created_at < ?
+        """, (cutoff, cutoff)).fetchall()
+        return [r["id"] for r in rows]
+
+    def deactivate_stale(self, days: int = 30) -> int:
+        ids = self.get_stale_memories(days)
+        for aid in ids:
+            self.db.execute("UPDATE artifacts SET active=0 WHERE id=?", (aid,))
+        self.db.commit()
+        return len(ids)
+
+    def get_efficiency_stats(self, context_window: int = 200_000) -> dict:
+        sessions = self.db.execute("""
+            SELECT session_id,
+                   COUNT(*) as prompts,
+                   SUM(tokens_injected) as total_injected,
+                   SUM(CASE WHEN hits_count > 0 THEN 1 ELSE 0 END) as prompts_with_hits,
+                   AVG(CASE WHEN hits_count > 0 THEN top_score END) as avg_quality,
+                   MIN(timestamp) as first_recall,
+                   MAX(timestamp) as last_recall
+            FROM recall_log
+            WHERE session_id != ''
+            GROUP BY session_id
+            ORDER BY last_recall DESC
+            LIMIT 50
+        """).fetchall()
+        session_list = []
+        for s in sessions:
+            injected = s["total_injected"] or 0
+            prompts = s["prompts"] or 1
+            precision = (s["prompts_with_hits"] or 0) / prompts
+            overhead_pct = round(injected / context_window * 100, 2)
+            session_list.append({
+                "session_id": s["session_id"],
+                "prompts": prompts,
+                "tokens_injected": injected,
+                "overhead_pct": overhead_pct,
+                "precision": round(precision, 3),
+                "avg_quality": round(s["avg_quality"] or 0, 3),
+                "first_recall": s["first_recall"],
+                "last_recall": s["last_recall"],
+            })
+
+        totals = self.db.execute("""
+            SELECT COUNT(*) as total_recalls,
+                   SUM(tokens_injected) as total_injected,
+                   SUM(CASE WHEN hits_count > 0 THEN 1 ELSE 0 END) as with_hits,
+                   AVG(CASE WHEN hits_count > 0 THEN top_score END) as avg_quality,
+                   COUNT(DISTINCT session_id) as session_count
+            FROM recall_log WHERE session_id != ''
+        """).fetchone()
+        total_recalls = totals["total_recalls"] or 0
+        total_injected = totals["total_injected"] or 0
+        session_count = totals["session_count"] or 1
+        avg_per_session = round(total_injected / session_count) if session_count else 0
+        return {
+            "context_window": context_window,
+            "total_sessions": totals["session_count"] or 0,
+            "total_recalls": total_recalls,
+            "total_tokens_injected": total_injected,
+            "avg_tokens_per_session": avg_per_session,
+            "avg_overhead_pct": round(avg_per_session / context_window * 100, 2),
+            "precision": round((totals["with_hits"] or 0) / total_recalls, 3) if total_recalls else 0,
+            "avg_quality": round(totals["avg_quality"] or 0, 3),
+            "sessions": session_list,
+        }
+
+    def get_onboarding_status(self) -> str:
+        chunks = self.db.execute(
+            "SELECT COUNT(*) as c FROM artifacts WHERE kind='session_chunk' AND active=1"
+        ).fetchone()["c"]
+        if chunks == 0:
+            return "no_data"
+        mems = self.db.execute(
+            "SELECT COUNT(*) as c FROM artifacts WHERE kind='memory' AND active=1"
+        ).fetchone()["c"]
+        if mems == 0:
+            return "ingesting"
+        llm_mems = self.db.execute("""
+            SELECT COUNT(*) as c FROM artifacts
+            WHERE kind='memory' AND active=1
+              AND json_extract(meta, '$.mem_type') IN ('decision','lesson','snippet','bugfix')
+        """).fetchone()["c"]
+        if llm_mems > 0:
+            return "full"
+        return "extractive"

memor/tokencount.py

@@ -0,0 +1,9 @@
+from __future__ import annotations
+import tiktoken
+
+_enc = tiktoken.get_encoding("cl100k_base")
+
+def count_tokens(text: str) -> int:
+    if not text:
+        return 0
+    return len(_enc.encode(text))