agentforge-graph 0.3.2__py3-none-any.whl

agentforge_graph/temporal/mining.py

@@ -0,0 +1,190 @@
+"""Churn / authorship mining (feat-009 chunk 2).
+
+A symbol's *churn* and *authorship* are ranking + ownership signals
+(design-009 §4.5): how much it has moved lately and who has been editing it.
+We mine them from ``git log`` over a bounded window and attribute each diff
+hunk to the symbol(s) whose **current span** overlaps the hunk's new-line
+range. Attribution is approximate *by design* — historical line numbers drift
+from current spans — which is fine for a ranking signal (it is never asserted
+as provenance).
+
+One batched ``git log -U0`` call per refresh covers all touched paths; the
+window (default 90d, derived from the commit's author time so results are
+deterministic in tests) bounds cost. Output is a small, bounded
+``SymbolAggregate`` per symbol — never a per-commit fact (design §4.10).
+"""
+
+from __future__ import annotations
+
+import re
+import subprocess
+from collections import defaultdict
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+
+_DAY = 86_400
+_HUNK = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@")
+_SOH = "\x01"  # commit-header marker (unambiguous vs. patch text)
+
+
+@dataclass(frozen=True)
+class SymbolAggregate:
+    """Bounded churn/authorship rollup for one symbol (design §4.4 aggregates)."""
+
+    symbol_id: str
+    churn_30d: int
+    churn_90d: int
+    top_authors: list[tuple[str, int]]  # (name, commits), ≤3, desc by commits
+    introduced_sha: str
+    introduced_ts: int
+    last_changed_sha: str
+    last_changed_ts: int
+
+    def attrs(self) -> dict[str, object]:
+        """The denormalised view written onto a node's ``attrs`` (design §4.0)
+        so ``ckg_symbol`` surfaces it with no join."""
+        return {
+            "churn_30d": self.churn_30d,
+            "churn_90d": self.churn_90d,
+            "top_authors": [{"name": n, "commits": c} for n, c in self.top_authors],
+            "introduced": self.introduced_sha,
+            "introduced_ts": self.introduced_ts,
+            "last_changed": self.last_changed_sha,
+            "last_changed_ts": self.last_changed_ts,
+        }
+
+
+@dataclass
+class _Acc:
+    """Per-symbol accumulator while walking the log."""
+
+    churn_30d: int = 0
+    churn_90d: int = 0
+    authors: dict[str, set[str]] = field(default_factory=lambda: defaultdict(set))  # name -> shas
+    first: tuple[int, str] = (1 << 62, "")  # (ts, sha) oldest
+    last: tuple[int, str] = (-1, "")  # (ts, sha) newest
+
+
+class ChurnMiner:
+    """Mines churn/authorship for a set of files over a commit-time window."""
+
+    def __init__(self, repo_root: str, *, now_ts: int, window_days: int = 90) -> None:
+        self._root = repo_root
+        self._now = now_ts
+        self._window = window_days
+
+    def mine(
+        self, spans_by_path: dict[str, list[tuple[str, tuple[int, int]]]]
+    ) -> list[SymbolAggregate]:
+        """Attribute windowed churn/authorship to the symbols in ``spans_by_path``
+        (``path -> [(symbol_id, (start_line, end_line)), …]``)."""
+        paths = [p for p, syms in spans_by_path.items() if syms]
+        if not paths or self._now <= 0:
+            return []
+        log = self._git_log(paths)
+        if log is None:
+            return []
+        acc: dict[str, _Acc] = defaultdict(_Acc)
+        cut30 = self._now - 30 * _DAY
+        cut90 = self._now - 90 * _DAY
+        for sha, ts, author, path, new_start, count, delta in self._hunks(log):
+            for sid in self._overlapping(spans_by_path.get(path, []), new_start, count):
+                a = acc[sid]
+                a.churn_90d += delta if ts >= cut90 else 0
+                a.churn_30d += delta if ts >= cut30 else 0
+                a.authors[author].add(sha)
+                if ts < a.first[0]:
+                    a.first = (ts, sha)
+                if ts > a.last[0]:
+                    a.last = (ts, sha)
+        return [self._aggregate(sid, a) for sid, a in sorted(acc.items())]
+
+    # --- internals --------------------------------------------------------
+
+    def _git_log(self, paths: list[str]) -> str | None:
+        since = datetime.fromtimestamp(max(self._now - self._window * _DAY, 0), tz=UTC).strftime(
+            "%Y-%m-%d"
+        )
+        try:
+            out = subprocess.run(
+                [
+                    "git",
+                    "-C",
+                    self._root,
+                    "log",
+                    "--no-renames",
+                    "--no-color",
+                    "-U0",
+                    f"--since={since}",
+                    f"--format={_SOH}%H%x09%ct%x09%an",
+                    "--",
+                    *paths,
+                ],
+                capture_output=True,
+                text=True,
+                check=True,
+            )
+        except (subprocess.SubprocessError, OSError):
+            return None
+        return out.stdout
+
+    def _hunks(self, log: str):  # type: ignore[no-untyped-def]
+        """Yield ``(sha, ts, author, path, new_start, line_count, churn_delta)`` per
+        hunk. ``churn_delta`` = added + deleted lines for the hunk."""
+        sha = author = path = ""
+        ts = 0
+        for line in log.splitlines():
+            if line.startswith(_SOH):
+                parts = line[1:].split("\t")
+                if len(parts) == 3:
+                    sha, ts_s, author = parts
+                    ts = int(ts_s) if ts_s.isdigit() else 0
+                path = ""
+                continue
+            if line.startswith("diff --git "):
+                path = ""
+                continue
+            if line.startswith("+++ b/"):
+                path = line[6:]
+                continue
+            if line.startswith("@@"):
+                m = _HUNK.match(line)
+                if not m or not path:
+                    continue
+                new_start = int(m.group(1))
+                added = int(m.group(2)) if m.group(2) is not None else 1
+                deleted = self._removed(line)
+                yield sha, ts, author, path, new_start, added, added + deleted
+
+    @staticmethod
+    def _removed(hunk: str) -> int:
+        m = re.match(r"^@@ -\d+(?:,(\d+))? \+", hunk)
+        if not m:
+            return 0
+        return int(m.group(1)) if m.group(1) is not None else 1
+
+    @staticmethod
+    def _overlapping(
+        syms: list[tuple[str, tuple[int, int]]], new_start: int, count: int
+    ) -> list[str]:
+        lo = new_start
+        hi = new_start + max(count, 1) - 1
+        return [sid for sid, (s, e) in syms if not (e < lo or s > hi)]
+
+    def _aggregate(self, sid: str, a: _Acc) -> SymbolAggregate:
+        top = sorted(
+            ((name, len(shas)) for name, shas in a.authors.items()),
+            key=lambda t: (-t[1], t[0]),
+        )[:3]
+        intro_ts, intro_sha = a.first if a.first[1] else (0, "")
+        last_ts, last_sha = a.last if a.last[1] else (0, "")
+        return SymbolAggregate(
+            symbol_id=sid,
+            churn_30d=a.churn_30d,
+            churn_90d=a.churn_90d,
+            top_authors=top,
+            introduced_sha=intro_sha,
+            introduced_ts=intro_ts,
+            last_changed_sha=last_sha,
+            last_changed_ts=last_ts,
+        )

agentforge_graph/temporal/recorder.py

@@ -0,0 +1,114 @@
+"""``SqliteTemporalRecorder`` — the write port the indexer drives (feat-009).
+
+The ``IncrementalIndexer`` calls ``open``/``close`` as it applies a diff (it is
+the only writer that sees both the old and new state of a file); the recorder
+buffers those into ``Event``s and writes them in a single transaction on
+``flush()`` at end-of-refresh — mirroring how ``IndexMeta`` is saved last, so a
+crash leaves a consistent log. ``open``/``close`` are sync (buffering); only
+``flush`` touches SQLite.
+
+Structurally satisfies ``ingest.incremental.ports.TemporalRecorder`` so the
+deterministic ``ingest`` layer depends on a Protocol, not on ``temporal``.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+
+from agentforge_graph.core import GraphQuery, GraphStore, NodeKind, SymbolID
+
+from .events import Entity, Event, EventKind
+from .mining import ChurnMiner
+from .store import TemporalStore
+
+_ALL = 10_000_000
+_SYMBOL_KINDS = (NodeKind.CLASS, NodeKind.FUNCTION, NodeKind.METHOD)
+
+
+class SqliteTemporalRecorder:
+    def __init__(self, store: TemporalStore) -> None:
+        self._store = store
+        self._buf: list[Event] = []
+
+    def open(self, symbol_ids: Iterable[str], at: str, ts: int) -> None:
+        self._buf.extend(
+            Event(symbol_id=sid, event=EventKind.OPENED, commit=at, ts=ts, entity=Entity.NODE)
+            for sid in symbol_ids
+        )
+
+    def close(self, symbol_ids: Iterable[str], at: str, ts: int) -> None:
+        self._buf.extend(
+            Event(symbol_id=sid, event=EventKind.CLOSED, commit=at, ts=ts, entity=Entity.NODE)
+            for sid in symbol_ids
+        )
+
+    async def record_churn(
+        self,
+        graph: GraphStore,
+        repo_root: str,
+        paths: Iterable[str],
+        commit: str,
+        commit_ts: int,
+    ) -> None:
+        """Mine churn/authorship for ``paths``, persist aggregates, and
+        denormalise them onto the matching node ``attrs`` (design §4.5).
+        Cheap on a small diff; a no-op when nothing maps or the commit time is
+        unknown (non-git)."""
+        if commit_ts <= 0:
+            return
+        spans = await self._spans_by_path(graph, set(paths))
+        if not spans:
+            return
+        aggs = ChurnMiner(repo_root, now_ts=commit_ts).mine(spans)
+        if not aggs:
+            return
+        await self._store.upsert_aggregates(aggs)
+        for agg in aggs:
+            await graph.set_attrs(agg.symbol_id, agg.attrs())
+
+    @staticmethod
+    async def _spans_by_path(
+        graph: GraphStore, paths: set[str]
+    ) -> dict[str, list[tuple[str, tuple[int, int]]]]:
+        """``path -> [(symbol_id, span), …]`` for the code symbols in ``paths``
+        that carry a span (the attribution targets)."""
+        out: dict[str, list[tuple[str, tuple[int, int]]]] = {}
+        nodes = (await graph.query(GraphQuery(kinds=list(_SYMBOL_KINDS), limit=_ALL))).nodes
+        for n in nodes:
+            if n.span is None:
+                continue
+            path = SymbolID.parse(n.id).path
+            if path in paths:
+                out.setdefault(path, []).append((n.id, n.span))
+        return out
+
+    async def flush(self) -> None:
+        if not self._buf:
+            return
+        events, self._buf = self._buf, []
+        await self._store.record(events)
+
+
+def build_recorder(root: str) -> SqliteTemporalRecorder:
+    """Open the sidecar under ``root`` (the ``.ckg`` dir) and wrap it."""
+    return SqliteTemporalRecorder(TemporalStore.open(root))
+
+
+async def seed_symbols(
+    graph: GraphStore,
+    recorder: SqliteTemporalRecorder,
+    commit: str,
+    ts: int,
+    repo_root: str = "",
+) -> None:
+    """Open intervals for every code symbol currently in the graph — used after
+    a full index so 'introduced' is anchored at the index commit — then mine
+    churn/authorship for the whole tree so a fresh index already carries the
+    ranking signal. Idempotent: a re-index of the same commit re-opens the same
+    events (deduped by the store)."""
+    nodes = (await graph.query(GraphQuery(kinds=list(_SYMBOL_KINDS), limit=_ALL))).nodes
+    recorder.open([n.id for n in nodes], commit, ts)
+    await recorder.flush()
+    if repo_root:
+        paths = {SymbolID.parse(n.id).path for n in nodes}
+        await recorder.record_churn(graph, repo_root, paths, commit, ts)

agentforge_graph/temporal/store.py

@@ -0,0 +1,282 @@
+"""``TemporalStore`` — the append-only evolution log (feat-009).
+
+A stdlib-``sqlite3`` sidecar at ``.ckg/temporal.db``, deliberately *separate*
+from the graph/vector stores: it keeps the current-graph hot path and both
+store adapters untouched (design-009 §4.2), is trivially prunable, and is absent
+for non-git / temporal-off repos. Writes are append-only and idempotent per
+``(symbol_id, commit, event, ref)`` so a crashed-then-retried refresh stays
+consistent.
+
+Chunk 1 implements the ``events`` table (node lifecycle); the ``aggregates``
+table (churn/authorship, chunk 2) is created here but populated later.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import sqlite3
+from pathlib import Path
+from typing import Any
+
+from .events import Entity, Event, EventKind
+from .mining import SymbolAggregate
+
+_DB = "temporal.db"
+
+_SCHEMA = [
+    """CREATE TABLE IF NOT EXISTS events (
+        symbol_id  TEXT NOT NULL,
+        entity     TEXT NOT NULL,
+        event      TEXT NOT NULL,
+        commit_sha TEXT NOT NULL,
+        ts         INTEGER NOT NULL,
+        ref        TEXT
+    )""",
+    # idempotency: the same lifecycle fact recorded twice is a no-op. COALESCE so
+    # a NULL ref participates in uniqueness (SQLite treats NULLs as distinct).
+    """CREATE UNIQUE INDEX IF NOT EXISTS events_unique
+        ON events(symbol_id, commit_sha, event, COALESCE(ref, ''))""",
+    "CREATE INDEX IF NOT EXISTS events_by_symbol ON events(symbol_id)",
+    "CREATE INDEX IF NOT EXISTS events_by_ts ON events(ts)",
+    # aggregates: periodic, bounded — populated in chunk 2 (churn/authorship).
+    """CREATE TABLE IF NOT EXISTS aggregates (
+        symbol_id        TEXT PRIMARY KEY,
+        churn_30d        INTEGER,
+        churn_90d        INTEGER,
+        top_authors      TEXT,
+        introduced_sha   TEXT,
+        introduced_ts    INTEGER,
+        last_changed_sha TEXT,
+        last_changed_ts  INTEGER
+    )""",
+    # meta: small key/value side-table (chunk 4 backfill cursor, etc.).
+    "CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)",
+]
+
+
+class TemporalStore:
+    """Embedded SQLite evolution log. Opened once per index/refresh; each
+    operation uses its own short-lived connection (SQLite connections are not
+    shareable across threads, and ops run via ``asyncio.to_thread``)."""
+
+    def __init__(self, path: Path) -> None:
+        self._path = path
+
+    @property
+    def path(self) -> Path:
+        return self._path
+
+    @classmethod
+    def open(cls, root: str | Path) -> TemporalStore:
+        """Create (if needed) the sidecar under ``root`` (the ``.ckg`` dir) and
+        ensure the schema exists."""
+        p = Path(root) / _DB
+        p.parent.mkdir(parents=True, exist_ok=True)
+        conn = sqlite3.connect(str(p))
+        try:
+            for ddl in _SCHEMA:
+                conn.execute(ddl)
+            conn.commit()
+        finally:
+            conn.close()
+        return cls(p)
+
+    async def record(self, events: list[Event]) -> int:
+        """Append events; return the number newly inserted (duplicates ignored)."""
+        if not events:
+            return 0
+        return await asyncio.to_thread(self._record_sync, events)
+
+    def _record_sync(self, events: list[Event]) -> int:
+        conn = sqlite3.connect(str(self._path))
+        try:
+            cur = conn.executemany(
+                "INSERT OR IGNORE INTO events"
+                "(symbol_id, entity, event, commit_sha, ts, ref) VALUES (?, ?, ?, ?, ?, ?)",
+                [
+                    (e.symbol_id, e.entity.value, e.event.value, e.commit, e.ts, e.ref)
+                    for e in events
+                ],
+            )
+            conn.commit()
+            return cur.rowcount if cur.rowcount is not None else 0
+        finally:
+            conn.close()
+
+    async def events_for(self, symbol_id: str) -> list[Event]:
+        """All events for one symbol, oldest first."""
+        return await asyncio.to_thread(self._events_for_sync, symbol_id)
+
+    def _events_for_sync(self, symbol_id: str) -> list[Event]:
+        conn = sqlite3.connect(str(self._path))
+        try:
+            rows = conn.execute(
+                "SELECT symbol_id, entity, event, commit_sha, ts, ref FROM events "
+                "WHERE symbol_id = ? ORDER BY ts, rowid",
+                (symbol_id,),
+            ).fetchall()
+        finally:
+            conn.close()
+        return [_row_to_event(r) for r in rows]
+
+    async def all_events(self) -> list[Event]:
+        """Every event, oldest first (test/inspection helper)."""
+        return await asyncio.to_thread(self._all_events_sync)
+
+    def _all_events_sync(self) -> list[Event]:
+        conn = sqlite3.connect(str(self._path))
+        try:
+            rows = conn.execute(
+                "SELECT symbol_id, entity, event, commit_sha, ts, ref FROM events "
+                "ORDER BY ts, rowid"
+            ).fetchall()
+        finally:
+            conn.close()
+        return [_row_to_event(r) for r in rows]
+
+    # --- meta key/value (chunk 4 resume cursor) ---------------------------
+
+    async def get_meta(self, key: str) -> str | None:
+        return await asyncio.to_thread(self._get_meta_sync, key)
+
+    def _get_meta_sync(self, key: str) -> str | None:
+        conn = sqlite3.connect(str(self._path))
+        try:
+            row = conn.execute("SELECT value FROM meta WHERE key = ?", (key,)).fetchone()
+        finally:
+            conn.close()
+        return row[0] if row else None
+
+    async def set_meta(self, key: str, value: str) -> None:
+        await asyncio.to_thread(self._set_meta_sync, key, value)
+
+    def _set_meta_sync(self, key: str, value: str) -> None:
+        conn = sqlite3.connect(str(self._path))
+        try:
+            conn.execute("INSERT OR REPLACE INTO meta(key, value) VALUES (?, ?)", (key, value))
+            conn.commit()
+        finally:
+            conn.close()
+
+    async def count_events(self) -> int:
+        """Total events in the log (for ``ckg status``)."""
+        return await asyncio.to_thread(self._count_events_sync)
+
+    def _count_events_sync(self) -> int:
+        conn = sqlite3.connect(str(self._path))
+        try:
+            return int(conn.execute("SELECT COUNT(*) FROM events").fetchone()[0])
+        finally:
+            conn.close()
+
+    # --- aggregates (churn / authorship, chunk 2) -------------------------
+
+    async def upsert_aggregates(self, aggs: list[SymbolAggregate]) -> int:
+        """Insert/replace churn/authorship rollups; return rows written."""
+        if not aggs:
+            return 0
+        return await asyncio.to_thread(self._upsert_aggregates_sync, aggs)
+
+    def _upsert_aggregates_sync(self, aggs: list[SymbolAggregate]) -> int:
+        conn = sqlite3.connect(str(self._path))
+        try:
+            cur = conn.executemany(
+                "INSERT OR REPLACE INTO aggregates"
+                "(symbol_id, churn_30d, churn_90d, top_authors,"
+                " introduced_sha, introduced_ts, last_changed_sha, last_changed_ts)"
+                " VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
+                [
+                    (
+                        a.symbol_id,
+                        a.churn_30d,
+                        a.churn_90d,
+                        json.dumps([{"name": n, "commits": c} for n, c in a.top_authors]),
+                        a.introduced_sha,
+                        a.introduced_ts,
+                        a.last_changed_sha,
+                        a.last_changed_ts,
+                    )
+                    for a in aggs
+                ],
+            )
+            conn.commit()
+            return cur.rowcount if cur.rowcount is not None else 0
+        finally:
+            conn.close()
+
+    async def all_aggregates(self) -> list[SymbolAggregate]:
+        """Every stored rollup (for ``changed_since`` scans)."""
+        return await asyncio.to_thread(self._all_aggregates_sync)
+
+    def _all_aggregates_sync(self) -> list[SymbolAggregate]:
+        conn = sqlite3.connect(str(self._path))
+        try:
+            rows = conn.execute(
+                "SELECT symbol_id, churn_30d, churn_90d, top_authors,"
+                " introduced_sha, introduced_ts, last_changed_sha, last_changed_ts"
+                " FROM aggregates"
+            ).fetchall()
+        finally:
+            conn.close()
+        return [_row_to_aggregate(r) for r in rows]
+
+    async def aggregate_for(self, symbol_id: str) -> SymbolAggregate | None:
+        """The stored rollup for one symbol, or ``None``."""
+        return await asyncio.to_thread(self._aggregate_for_sync, symbol_id)
+
+    def _aggregate_for_sync(self, symbol_id: str) -> SymbolAggregate | None:
+        conn = sqlite3.connect(str(self._path))
+        try:
+            row = conn.execute(
+                "SELECT symbol_id, churn_30d, churn_90d, top_authors,"
+                " introduced_sha, introduced_ts, last_changed_sha, last_changed_ts"
+                " FROM aggregates WHERE symbol_id = ?",
+                (symbol_id,),
+            ).fetchone()
+        finally:
+            conn.close()
+        return _row_to_aggregate(row) if row else None
+
+    async def prune(self, before_ts: int) -> int:
+        """Delete CLOSED events older than ``before_ts`` (retention horizon).
+        OPENED events are kept (they anchor 'introduced'); full retention math
+        lands in chunk 5. Returns rows removed."""
+        return await asyncio.to_thread(self._prune_sync, before_ts)
+
+    def _prune_sync(self, before_ts: int) -> int:
+        conn = sqlite3.connect(str(self._path))
+        try:
+            cur = conn.execute(
+                "DELETE FROM events WHERE event = ? AND ts < ?",
+                (EventKind.CLOSED.value, before_ts),
+            )
+            conn.commit()
+            return cur.rowcount if cur.rowcount is not None else 0
+        finally:
+            conn.close()
+
+
+def _row_to_event(r: tuple[Any, ...]) -> Event:
+    return Event(
+        symbol_id=r[0],
+        entity=Entity(r[1]),
+        event=EventKind(r[2]),
+        commit=r[3],
+        ts=r[4],
+        ref=r[5],
+    )
+
+
+def _row_to_aggregate(r: tuple[Any, ...]) -> SymbolAggregate:
+    authors = [(a["name"], a["commits"]) for a in json.loads(r[3] or "[]")]
+    return SymbolAggregate(
+        symbol_id=r[0],
+        churn_30d=r[1],
+        churn_90d=r[2],
+        top_authors=authors,
+        introduced_sha=r[4] or "",
+        introduced_ts=r[5] or 0,
+        last_changed_sha=r[6] or "",
+        last_changed_ts=r[7] or 0,
+    )