flurryx-code-memory 0.4.0__py3-none-any.whl

code_memory/metrics.py

@@ -0,0 +1,260 @@
+import sqlite3
+import time
+import json
+from dataclasses import dataclass, asdict
+from pathlib import Path
+from typing import Any
+
+@dataclass
+class RetrieveTiming:
+    query: str
+    embed_ms: float
+    code_search_ms: float
+    eps_search_ms: float
+    claims_ms: float
+    total_ms: float
+    code_hit_count: int = 0
+    eps_hit_count: int = 0
+    claims_hit_count: int = 0
+
+class MetricsStore:
+    def __init__(self, path: Path):
+        self.path = Path(path)
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        self._init_db()
+    
+    def _init_db(self):
+        with sqlite3.connect(str(self.path)) as conn:
+            conn.execute("""
+                CREATE TABLE IF NOT EXISTS retrieves (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    query TEXT,
+                    embed_ms REAL,
+                    code_search_ms REAL,
+                    eps_search_ms REAL,
+                    claims_ms REAL,
+                    total_ms REAL,
+                    code_hit_count INTEGER DEFAULT 0,
+                    eps_hit_count INTEGER DEFAULT 0,
+                    claims_hit_count INTEGER DEFAULT 0,
+                    ts REAL NOT NULL
+                )
+            """)
+            conn.execute("""
+                CREATE TABLE IF NOT EXISTS backend_health (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    backend TEXT NOT NULL,
+                    status TEXT NOT NULL,
+                    latency_ms REAL,
+                    ts REAL NOT NULL
+                )
+            """)
+            conn.execute("""
+                CREATE TABLE IF NOT EXISTS cache_stats (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    hits INTEGER DEFAULT 0,
+                    misses INTEGER DEFAULT 0,
+                    ts REAL NOT NULL
+                )
+            """)
+            conn.execute("""
+                CREATE TABLE IF NOT EXISTS ingest_stats (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    files INTEGER DEFAULT 0,
+                    symbols INTEGER DEFAULT 0,
+                    duration_s REAL,
+                    ts REAL NOT NULL
+                )
+            """)
+            conn.execute("""
+                CREATE TABLE IF NOT EXISTS tool_calls (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    tool TEXT NOT NULL,
+                    project TEXT NOT NULL,
+                    query_text TEXT,
+                    output_chars INTEGER DEFAULT 0,
+                    result_count INTEGER DEFAULT 0,
+                    session_id TEXT,
+                    ts REAL NOT NULL
+                )
+            """)
+            conn.execute("""
+                CREATE TABLE IF NOT EXISTS fs_reads (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    tool TEXT NOT NULL,
+                    path TEXT,
+                    output_chars INTEGER DEFAULT 0,
+                    session_id TEXT,
+                    project TEXT NOT NULL,
+                    ts REAL NOT NULL
+                )
+            """)
+            conn.commit()
+    
+    def record_retrieve(self, m: RetrieveTiming):
+        with sqlite3.connect(str(self.path)) as conn:
+            conn.execute(
+                "INSERT INTO retrieves (query, embed_ms, code_search_ms, eps_search_ms, claims_ms, total_ms, code_hit_count, eps_hit_count, claims_hit_count, ts) VALUES (?,?,?,?,?,?,?,?,?,?)",
+                (m.query, m.embed_ms, m.code_search_ms, m.eps_search_ms, m.claims_ms, m.total_ms, m.code_hit_count, m.eps_hit_count, m.claims_hit_count, time.time())
+            )
+            conn.commit()
+    
+    def record_backend_health(self, backend: str, status: str, latency_ms: float):
+        with sqlite3.connect(str(self.path)) as conn:
+            conn.execute(
+                "INSERT INTO backend_health (backend, status, latency_ms, ts) VALUES (?,?,?,?)",
+                (backend, status, latency_ms, time.time())
+            )
+            conn.commit()
+    
+    def record_cache_stats(self, hits: int, misses: int):
+        with sqlite3.connect(str(self.path)) as conn:
+            conn.execute(
+                "INSERT INTO cache_stats (hits, misses, ts) VALUES (?,?,?)",
+                (hits, misses, time.time())
+            )
+            conn.commit()
+    
+    def record_ingest(self, files: int, symbols: int, duration_s: float):
+        with sqlite3.connect(str(self.path)) as conn:
+            conn.execute(
+                "INSERT INTO ingest_stats (files, symbols, duration_s, ts) VALUES (?,?,?,?)",
+                (files, symbols, duration_s, time.time())
+            )
+            conn.commit()
+
+    def record_tool_call(self, tool: str, project: str, *, query_text: str = "", output_chars: int = 0, result_count: int = 0, session_id: str = ""):
+        with sqlite3.connect(str(self.path)) as conn:
+            conn.execute(
+                "INSERT INTO tool_calls (tool, project, query_text, output_chars, result_count, session_id, ts) VALUES (?,?,?,?,?,?,?)",
+                (tool, project, query_text or None, output_chars, result_count, session_id or None, time.time())
+            )
+            conn.commit()
+
+    def record_fs_read(self, tool: str, path: str, project: str, *, output_chars: int = 0, session_id: str = ""):
+        with sqlite3.connect(str(self.path)) as conn:
+            conn.execute(
+                "INSERT INTO fs_reads (tool, path, output_chars, session_id, project, ts) VALUES (?,?,?,?,?,?)",
+                (tool, path or None, output_chars, session_id or None, project, time.time())
+            )
+            conn.commit()
+
+    def tool_usage_summary(self, project: str | None = None) -> dict:
+        with sqlite3.connect(str(self.path)) as conn:
+            conn.row_factory = sqlite3.Row
+            where = " WHERE project=?" if project else ""
+            params = (project,) if project else ()
+            rows = conn.execute(
+                f"SELECT tool, COUNT(*) as calls, COALESCE(SUM(output_chars),0) as total_chars, COALESCE(AVG(output_chars),0) as avg_chars FROM tool_calls{where} GROUP BY tool ORDER BY calls DESC",
+                params
+            ).fetchall()
+            return {
+                "tools": [dict(r) for r in rows],
+                "total_calls": sum(r["calls"] for r in rows),
+            }
+
+    def efficiency_summary(self, project: str | None = None) -> dict:
+        with sqlite3.connect(str(self.path)) as conn:
+            conn.row_factory = sqlite3.Row
+            # Aggregate tool_calls
+            tc_where = " WHERE project=?" if project else ""
+            tc_params = (project,) if project else ()
+            tc = conn.execute(
+                f"SELECT COUNT(*) as calls, COALESCE(SUM(output_chars),0) as total_chars FROM tool_calls{tc_where}",
+                tc_params
+            ).fetchone()
+
+            # Aggregate fs_reads
+            fs_where = " WHERE project=?" if project else ""
+            fs_params = (project,) if project else ()
+            fs = conn.execute(
+                f"SELECT COUNT(*) as reads, COALESCE(SUM(output_chars),0) as total_chars FROM fs_reads{fs_where}",
+                fs_params
+            ).fetchone()
+
+            # Per-session breakdown
+            session_where = " WHERE project=?" if project else ""
+            session_params = (project,) if project else ()
+            tc_sessions = conn.execute(
+                f"SELECT COALESCE(session_id,'') as session_id, SUM(output_chars) as mcp_chars FROM tool_calls{session_where} GROUP BY session_id",
+                session_params
+            ).fetchall()
+            fs_sessions = conn.execute(
+                f"SELECT COALESCE(session_id,'') as session_id, SUM(output_chars) as fs_chars FROM fs_reads{session_where} GROUP BY session_id",
+                session_params
+            ).fetchall()
+
+            # Merge per-session
+            session_map: dict[str, dict] = {}
+            for r in tc_sessions:
+                sid = r["session_id"]
+                session_map.setdefault(sid, {"session_id": sid, "mcp_chars": 0, "fs_chars": 0})
+                session_map[sid]["mcp_chars"] = r["mcp_chars"]
+            for r in fs_sessions:
+                sid = r["session_id"]
+                session_map.setdefault(sid, {"session_id": sid, "mcp_chars": 0, "fs_chars": 0})
+                session_map[sid]["fs_chars"] = r["fs_chars"]
+
+            return {
+                "total_mcp_chars": tc["total_chars"],
+                "total_fs_chars": fs["total_chars"],
+                "mcp_calls": tc["calls"],
+                "fs_reads": fs["reads"],
+                "sessions": list(session_map.values()),
+            }
+
+    def summary(self) -> dict[str, Any]:
+        with sqlite3.connect(str(self.path)) as conn:
+            conn.row_factory = sqlite3.Row
+            # Retrieve stats
+            r = conn.execute(
+                "SELECT COUNT(*) as count, AVG(total_ms) as avg_total_ms, AVG(embed_ms) as avg_embed_ms, AVG(code_search_ms) as avg_code_search_ms FROM retrieves"
+            ).fetchone()
+            
+            # Cache stats
+            c = conn.execute(
+                "SELECT COALESCE(SUM(hits),0) as total_hits, COALESCE(SUM(misses),0) as total_misses FROM cache_stats"
+            ).fetchone()
+            total_cache = c["total_hits"] + c["total_misses"]
+            hit_ratio = c["total_hits"] / total_cache if total_cache > 0 else 0.0
+            
+            # Ingest stats
+            i = conn.execute(
+                "SELECT COUNT(*) as count, COALESCE(SUM(symbols),0) as total_symbols, MAX(ts) as last_ingest_ts FROM ingest_stats"
+            ).fetchone()
+            
+            # Backend health - latest per backend
+            b = conn.execute(
+                "SELECT backend, status, MAX(ts) as last_ts FROM backend_health GROUP BY backend"
+            ).fetchall()
+            
+            return {
+                "retrieves": {
+                    "count": r["count"],
+                    "avg_total_ms": round(r["avg_total_ms"] or 0, 1),
+                    "avg_embed_ms": round(r["avg_embed_ms"] or 0, 1),
+                    "avg_code_search_ms": round(r["avg_code_search_ms"] or 0, 1),
+                },
+                "cache": {
+                    "total_hits": c["total_hits"],
+                    "total_misses": c["total_misses"],
+                    "hit_ratio": round(hit_ratio, 3),
+                },
+                "ingest": {
+                    "count": i["count"],
+                    "total_symbols": i["total_symbols"],
+                    "last_ingest_ts": i["last_ingest_ts"],
+                },
+                "backends": [{"backend": row["backend"], "status": row["status"]} for row in b],
+            }
+    
+    def recent_retrieves(self, limit: int = 10) -> list[dict]:
+        with sqlite3.connect(str(self.path)) as conn:
+            conn.row_factory = sqlite3.Row
+            rows = conn.execute(
+                "SELECT * FROM retrieves ORDER BY ts DESC LIMIT ?", (limit,)
+            ).fetchall()
+            return [dict(r) for r in rows]
+    
+    def close(self):
+        pass

code_memory/orchestrator/__init__.py

@@ -0,0 +1,13 @@
+from .pipeline import Pipeline
+from .reset import ResetResult, list_projects, reset_all, reset_project
+from .retrieve import ContextPack, Retriever
+
+__all__ = [
+    "Pipeline",
+    "Retriever",
+    "ContextPack",
+    "ResetResult",
+    "list_projects",
+    "reset_project",
+    "reset_all",
+]

code_memory/orchestrator/git_delta.py

@@ -0,0 +1,211 @@
+"""Git-aware delta detection for incremental ingestion.
+
+Given a repo root and a base commit, produce three lists:
+  - changed (added / modified / renamed-new) -> reingest
+  - deleted (removed / renamed-old)          -> drop from index
+  - dirty   (uncommitted worktree changes)   -> reingest
+
+All paths returned are absolute, resolved.
+"""
+
+from __future__ import annotations
+
+import subprocess
+from dataclasses import dataclass, field
+from pathlib import Path
+
+
+class GitError(RuntimeError):
+    pass
+
+
+def _run(repo: Path, *args: str, check: bool = True, timeout: float = 30.0) -> str:
+    try:
+        out = subprocess.run(
+            ["git", "-C", str(repo), *args],
+            capture_output=True,
+            text=True,
+            check=False,
+            timeout=timeout,
+        )
+    except FileNotFoundError as e:
+        raise GitError("git executable not found on PATH") from e
+    except subprocess.SubprocessError as e:
+        raise GitError(f"git invocation failed: {e}") from e
+    if check and out.returncode != 0:
+        raise GitError(
+            f"git {' '.join(args)} failed (exit {out.returncode}): {out.stderr.strip()}"
+        )
+    return out.stdout
+
+
+def is_git_repo(root: str | Path) -> bool:
+    try:
+        out = _run(Path(root), "rev-parse", "--is-inside-work-tree", check=False)
+    except GitError:
+        return False
+    return out.strip() == "true"
+
+
+def head_sha(root: str | Path) -> str:
+    return _run(Path(root), "rev-parse", "HEAD").strip()
+
+
+def current_branch(root: str | Path) -> str | None:
+    out = _run(Path(root), "rev-parse", "--abbrev-ref", "HEAD", check=False).strip()
+    return out if out and out != "HEAD" else None
+
+
+def is_reachable(root: str | Path, sha: str) -> bool:
+    try:
+        _run(Path(root), "cat-file", "-e", f"{sha}^{{commit}}", check=True)
+        return True
+    except GitError:
+        return False
+
+
+def resolve_ref(root: str | Path, ref: str) -> str:
+    """Resolve a ref (branch / tag / sha) to a full SHA. Raises if unknown."""
+    return _run(Path(root), "rev-parse", "--verify", f"{ref}^{{commit}}").strip()
+
+
+def commit_ordinal(root: str | Path, sha: str) -> int | None:
+    """Return the topological ordinal of ``sha`` — first-parent ancestor count.
+
+    ``git rev-list --count --first-parent <sha>`` gives a monotonic integer
+    along the main branch: parent < child, always. We use it as a cheap
+    "before / after" comparator across SHAs without dragging full topology
+    into the graph store. Returns ``None`` if the SHA isn't reachable
+    (shallow clone, orphan, freshly initialised repo).
+    """
+    try:
+        out = _run(
+            Path(root),
+            "rev-list",
+            "--count",
+            "--first-parent",
+            sha,
+            check=True,
+        ).strip()
+    except GitError:
+        return None
+    try:
+        return int(out)
+    except ValueError:
+        return None
+
+
+@dataclass
+class Delta:
+    changed: list[Path] = field(default_factory=list)
+    deleted: list[Path] = field(default_factory=list)
+    dirty: list[Path] = field(default_factory=list)
+
+    @property
+    def is_empty(self) -> bool:
+        return not (self.changed or self.deleted or self.dirty)
+
+    def reingest_paths(self) -> list[Path]:
+        # de-dup while preserving order
+        seen: set[Path] = set()
+        out: list[Path] = []
+        for p in self.changed + self.dirty:
+            if p in seen:
+                continue
+            seen.add(p)
+            out.append(p)
+        return out
+
+
+def diff(root: str | Path, base_sha: str, head: str = "HEAD") -> Delta:
+    """Compute path-level delta between base_sha and head (committed only)."""
+    repo = Path(root).resolve()
+    out = _run(repo, "diff", "--name-status", "-M", base_sha, head)
+    delta = Delta()
+    for line in out.splitlines():
+        if not line.strip():
+            continue
+        parts = line.split("\t")
+        status = parts[0]
+        # M / A / D / T => 2 fields ; R### / C### => 3 fields (old, new)
+        code = status[0]
+        if code in ("R", "C") and len(parts) >= 3:
+            old_abs = (repo / parts[1]).resolve()
+            new_abs = (repo / parts[2]).resolve()
+            delta.deleted.append(old_abs)
+            delta.changed.append(new_abs)
+        elif code == "D" and len(parts) >= 2:
+            delta.deleted.append((repo / parts[1]).resolve())
+        elif code in ("A", "M", "T") and len(parts) >= 2:
+            delta.changed.append((repo / parts[1]).resolve())
+        # anything else (U/X/B) -> skip silently
+    return delta
+
+
+def dirty_files(root: str | Path) -> list[Path]:
+    """Return absolute paths of files with uncommitted *content* changes
+    (modified, added, untracked). Worktree deletions are reported by
+    ``dirty_deleted_files`` so callers can route them to a delete path
+    instead of trying to reingest a missing file.
+    """
+    repo = Path(root).resolve()
+    out = _run(repo, "status", "--porcelain=v1", "--untracked-files=all")
+    paths: list[Path] = []
+    for line in out.splitlines():
+        if len(line) < 4:
+            continue
+        xy = line[:2]
+        rest = line[3:]
+        # rename in index: "R  old -> new"
+        if "->" in rest:
+            rest = rest.split("->", 1)[1].strip()
+        # ignored / deleted both index+worktree -> skip
+        if xy == "!!" or "D" in xy:
+            continue
+        path = (repo / rest).resolve()
+        if path.is_file():
+            paths.append(path)
+    return paths
+
+
+def dirty_deleted_files(root: str | Path) -> list[Path]:
+    """Return absolute paths of files deleted in the worktree but not yet
+    committed.
+
+    Catches both ``git rm`` (index column 'D') and a plain ``rm`` against
+    a tracked file (worktree column 'D'), so the index can be pruned
+    before any commit lands. Without this, deletes only propagate after
+    the next commit — leaving the graph and vector index claiming the
+    file still exists.
+    """
+    repo = Path(root).resolve()
+    out = _run(repo, "status", "--porcelain=v1")
+    paths: list[Path] = []
+    for line in out.splitlines():
+        if len(line) < 4:
+            continue
+        xy = line[:2]
+        rest = line[3:]
+        if xy == "!!":
+            continue
+        # Renames: ``R<sp>`` in index means the OLD path is gone; we
+        # don't get the old name from here (it appears with ``->``),
+        # but ``diff`` against HEAD already emits a paired delete +
+        # change for those, so we skip rename lines defensively.
+        if "->" in rest:
+            continue
+        if xy[0] == "D" or xy[1] == "D":
+            paths.append((repo / rest).resolve())
+    return paths
+
+
+def changed_since(root: str | Path, base_sha: str, *, include_dirty: bool = True) -> Delta:
+    """Convenience: delta from base_sha to HEAD, plus optional dirty worktree."""
+    d = diff(root, base_sha, "HEAD")
+    if include_dirty:
+        d.dirty.extend(dirty_files(root))
+        # Uncommitted deletes must reach the pipeline's delete loop so
+        # graph nodes + vectors for vanished files get torn down even
+        # before the user commits the removal.
+        d.deleted.extend(dirty_deleted_files(root))
+    return d

code_memory/orchestrator/ingest_state.py

@@ -0,0 +1,71 @@
+"""Per-repo ingest state: track the last commit successfully ingested.
+
+Lives in the same SQLite DB as episodes (per-project namespaced) so it
+inherits project isolation automatically.
+"""
+
+from __future__ import annotations
+
+import sqlite3
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+SCHEMA = """
+CREATE TABLE IF NOT EXISTS ingest_state (
+    repo_root TEXT PRIMARY KEY,
+    last_sha  TEXT NOT NULL,
+    last_ts   REAL NOT NULL,
+    branch    TEXT
+);
+"""
+
+
+@dataclass(frozen=True)
+class IngestState:
+    repo_root: str
+    last_sha: str
+    last_ts: float
+    branch: str | None = None
+
+
+class IngestStateStore:
+    """Thin SQLite wrapper for per-repo ingest checkpoints."""
+
+    def __init__(self, db_path: Path) -> None:
+        self.path = db_path
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        self.conn = sqlite3.connect(self.path)
+        self.conn.executescript(SCHEMA)
+        self.conn.commit()
+
+    def get(self, repo_root: str | Path) -> IngestState | None:
+        row = self.conn.execute(
+            "SELECT repo_root, last_sha, last_ts, branch FROM ingest_state WHERE repo_root = ?",
+            (str(Path(repo_root).resolve()),),
+        ).fetchone()
+        if row is None:
+            return None
+        return IngestState(repo_root=row[0], last_sha=row[1], last_ts=row[2], branch=row[3])
+
+    def set(self, repo_root: str | Path, sha: str, branch: str | None = None) -> None:
+        self.conn.execute(
+            "INSERT INTO ingest_state(repo_root, last_sha, last_ts, branch) "
+            "VALUES (?, ?, ?, ?) "
+            "ON CONFLICT(repo_root) DO UPDATE SET "
+            "  last_sha = excluded.last_sha, "
+            "  last_ts  = excluded.last_ts, "
+            "  branch   = excluded.branch",
+            (str(Path(repo_root).resolve()), sha, time.time(), branch),
+        )
+        self.conn.commit()
+
+    def clear(self, repo_root: str | Path) -> None:
+        self.conn.execute(
+            "DELETE FROM ingest_state WHERE repo_root = ?",
+            (str(Path(repo_root).resolve()),),
+        )
+        self.conn.commit()
+
+    def close(self) -> None:
+        self.conn.close()