marvisx-cli 0.1.0__py3-none-any.whl

core/api/services/brain/sources/git_kg.py

@@ -0,0 +1,393 @@
+# Git + Knowledge Graph source collector
+# (sub-01 §3 — `commit_changed` / `kg_changed` / `doc_changed`).
+#
+# Three streams collapsed into a single collector because they share the
+# same substrate (filesystem + graph nodes) and the same cycle math. Strict
+# anti-patterns enforced (regression-guarded by the no-neighbors test):
+#   * Per-project neighbor expansion → N+1 disaster, banned.
+#     Use graph_service.get_hotspots instead.
+#   * Direct SELECT against the graph node table in this module → banned.
+#     Read paths live in api.services.graph_service.
+from __future__ import annotations
+
+import asyncio
+import logging
+import os
+from collections.abc import AsyncIterator
+from datetime import datetime, timezone
+from pathlib import Path
+
+from core.api.db import acquire_db
+from core.api.paths import repo_root
+from core.api.services import graph_service
+from core.api.services.brain.digest_collector import SourceCollector
+from core.api.services.brain.models import EventDraft, SourceCollectorContext
+from core.api.services.brain.sources.base import (
+    normalize_iso,
+    resolve_source_project,
+)
+
+logger = logging.getLogger(__name__)
+
+
+# ----------------------------------------------------------------------
+# Tunables (kept inline rather than promoted to settings — the plan §3
+# fixes them explicitly and lifting them to app_settings would create
+# another knob that drifts).
+# ----------------------------------------------------------------------
+
+COMMIT_CAP_PER_CYCLE: int = 100
+KG_HOTSPOT_LIMIT: int = 20
+KG_HOTSPOT_WINDOW = "30d"
+DOC_NODE_KINDS: frozenset[str] = frozenset(
+    {
+        "handoff",
+        "solution",
+        "guide",
+        "plan",
+        "brainstorm",
+        "audit",
+        "spike",
+        "analysis",
+        "research",
+        "rubric",
+        "report",
+        "policy",
+        "contract",
+    }
+)
+
+def _resolve_repo_root() -> Path:
+    """Resolve the git repository root for commit_changed scans.
+
+    Order of precedence:
+    1. ``BRAIN_GIT_REPO_ROOT`` env var (explicit override, used in prod where
+       pir-api is deployed under ``/data/pir/api/`` and the source repo lives
+       elsewhere — typically ``~/workspace``).
+    2. ``parents[3]`` of this file (development default — works when pir-api
+       runs from the monorepo checkout, e.g. CI or dev workstation).
+
+    The function returns a Path; callers still gate ``git log`` execution on
+    ``(repo_root / ".git").exists()`` so a wrong value degrades to an empty
+    event stream (collector continues, no crash).
+    """
+    env_override = os.getenv("BRAIN_GIT_REPO_ROOT")
+    if env_override:
+        return Path(env_override).expanduser().resolve()
+    return repo_root(__file__)
+
+
+_REPO_ROOT = _resolve_repo_root()
+
+
+# ----------------------------------------------------------------------
+# commit_changed
+# ----------------------------------------------------------------------
+
+
+_GIT_LOG_FORMAT = "%H%x1f%aI%x1f%an%x1f%s"
+_GIT_LOG_SEP = "\x1f"
+
+
+async def _git_log_window(
+    *,
+    since: datetime,
+    until: datetime,
+    cwd: Path,
+    cap: int,
+) -> list[dict[str, str]]:
+    """Run `git log` for the cycle window.
+
+    Returns at most `cap` commits ordered ASC by commit time so the watermark
+    advances monotonically. Failure to invoke git (binary missing, non-repo)
+    surfaces as an empty list — the collector continues and the partial
+    failure tracking happens at the cycle level via try/except.
+    """
+    if not (cwd / ".git").exists():
+        return []
+    try:
+        proc = await asyncio.create_subprocess_exec(
+            "git",
+            "log",
+            f"--since={since.astimezone(timezone.utc).isoformat()}",
+            f"--until={until.astimezone(timezone.utc).isoformat()}",
+            f"--format={_GIT_LOG_FORMAT}",
+            "--reverse",
+            f"--max-count={cap}",
+            "--no-merges",
+            cwd=str(cwd),
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+        )
+    except FileNotFoundError:
+        return []
+    try:
+        stdout_b, _stderr_b = await asyncio.wait_for(proc.communicate(), timeout=10.0)
+    except asyncio.TimeoutError:
+        proc.kill()
+        await proc.wait()
+        return []
+
+    rows: list[dict[str, str]] = []
+    for line in stdout_b.decode("utf-8", errors="replace").splitlines():
+        parts = line.split(_GIT_LOG_SEP, 3)
+        if len(parts) != 4:
+            continue
+        sha, author_iso, author_name, subject = parts
+        rows.append(
+            {
+                "sha": sha,
+                "author_iso": author_iso,
+                "author_name": author_name,
+                "subject": subject,
+            }
+        )
+    return rows
+
+
+def _commit_event(
+    *, sha: str, author_iso: str, author_name: str, subject: str, now: datetime
+) -> EventDraft:
+    observed_at = normalize_iso(author_iso) or now
+    evidence = {
+        "sha": sha,
+        "subject": subject[:200],
+        "author": author_name,
+        "author_iso": observed_at.isoformat(),
+    }
+    return EventDraft(
+        event_type="commit_changed",
+        source_system="git",
+        source_ref=f"commit:{sha}",
+        title=subject[:200] or f"commit {sha[:7]}",
+        summary=f"{author_name}: {subject[:160]}",
+        observed_at=observed_at,
+        derived_from_state_at=observed_at,
+        evidence=evidence,
+    )
+
+
+# ----------------------------------------------------------------------
+# kg_changed (via graph_service.get_hotspots — NEVER per-project neighbor scan)
+# ----------------------------------------------------------------------
+
+
+def _node_id_for(node_type: str, qualified_name: str) -> str:
+    """Mirror KG id convention so source_ref ↔ KG node_id round-trips."""
+    prefix_map = {
+        "function": "py",
+        "file": "py",
+        "module": "py",
+    }
+    prefix = prefix_map.get(node_type, node_type)
+    return f"{prefix}:{node_type}:{qualified_name}"
+
+
+def _kg_event(*, row: dict, now: datetime) -> EventDraft | None:
+    touched_at = normalize_iso(row.get("touch_last_at"))
+    if touched_at is None:
+        return None
+    node_id = row.get("id") or _node_id_for(row["type"], row.get("qualified_name", ""))
+    project, program = resolve_source_project(row.get("project_id"))
+    evidence = {
+        "node_id": node_id,
+        "type": row.get("type"),
+        "touch_count_30d": row.get("touch_count_30d", 0),
+        "touch_count_7d": row.get("touch_count_7d", 0),
+        "touch_last_at": touched_at.isoformat(),
+    }
+    return EventDraft(
+        event_type="kg_changed",
+        source_system="kg",
+        source_ref=node_id,
+        title=str(row.get("name") or node_id)[:200],
+        summary=f"KG hotspot {node_id}: {row.get('touch_count_30d', 0)}/30d touches",
+        observed_at=touched_at,
+        derived_from_state_at=touched_at,
+        evidence=evidence,
+        source_project=project,
+        program_key=program,
+    )
+
+
+# ----------------------------------------------------------------------
+# doc_changed (graph_nodes via graph_service helper — no direct SQL)
+# ----------------------------------------------------------------------
+
+
+async def _list_doc_nodes_changed_since(
+    *, since: datetime, limit: int = 200
+) -> list[dict]:
+    """Read doc-kind nodes whose touch_last_at advanced past the watermark.
+
+    Routed through graph_service.find_recently_touched_doc_nodes so this
+    collector stays free of direct SQL against the graph node table
+    (anti-pattern enforced via grep on the file).
+    """
+    fetcher = getattr(graph_service, "find_recently_touched_doc_nodes", None)
+    if fetcher is None:
+        # Fall back to no-op rather than the banned direct query — tests
+        # cover this branch explicitly so the absence is visible in CI.
+        return []
+    return await fetcher(
+        since_iso=since.astimezone(timezone.utc).isoformat(),
+        kinds=tuple(sorted(DOC_NODE_KINDS)),
+        limit=limit,
+    )
+
+
+def _doc_event(*, row: dict, now: datetime) -> EventDraft | None:
+    touched_at = normalize_iso(row.get("touch_last_at"))
+    if touched_at is None:
+        return None
+    node_id = row.get("id")
+    path = row.get("file_path") or row.get("name") or node_id
+    last_sha = row.get("last_modified_git_sha") or row.get("first_seen_git_sha") or ""
+    project, program = resolve_source_project(row.get("project_id"))
+    evidence = {
+        "node_id": node_id,
+        "kind": row.get("type"),
+        "path": path,
+        "sha": last_sha,
+        "touch_last_at": touched_at.isoformat(),
+    }
+    return EventDraft(
+        event_type="doc_changed",
+        source_system="kg",
+        source_ref=f"doc:{path}@{last_sha[:12] if last_sha else 'untracked'}",
+        title=str(row.get("name") or path)[:200],
+        summary=f"Doc {row.get('type')} updated: {path}",
+        observed_at=touched_at,
+        derived_from_state_at=touched_at,
+        evidence=evidence,
+        source_project=project,
+        program_key=program,
+    )
+
+
+# ----------------------------------------------------------------------
+# Collector entry point
+# ----------------------------------------------------------------------
+
+
+def _resolve_git_window(ctx: SourceCollectorContext) -> tuple[datetime, datetime]:
+    """Return (since, until) for ``git log`` — clamped to the cycle window.
+
+    Bug fix 2026-05-18: collectors must honor ``cycle_window_*`` instead of
+    blindly reading from the watermark up to ``cutoff_at``. For backward-
+    compat (periodic path with no window bounds) this collapses to the
+    legacy ``(watermark, cutoff_at]`` slice.
+    """
+    if (
+        ctx.cycle_window_start is not None
+        and ctx.cycle_window_start > ctx.since_watermark
+    ):
+        since = ctx.cycle_window_start
+    else:
+        since = ctx.since_watermark
+    if (
+        ctx.cycle_window_end is not None
+        and ctx.cycle_window_end < ctx.cutoff_at
+    ):
+        until = ctx.cycle_window_end
+    else:
+        until = ctx.cutoff_at
+    return since, until
+
+
+async def collect_git(ctx: SourceCollectorContext) -> AsyncIterator[EventDraft]:
+    """Git commit_changed collector — source_system='git'.
+
+    Repo root resolution order:
+    1. ``BRAIN_GIT_REPO_ROOT`` env var (production: pir-api deployed under
+       /data/pir/api/ → repo lives at ~/workspace).
+    2. Module-level ``_REPO_ROOT`` (dev default + monkeypatchable in tests).
+
+    Cycle window: bounded via ``ctx.cycle_window_start`` / ``cycle_window_end``
+    when present, otherwise falls back to ``(since_watermark, cutoff_at]``.
+    """
+    env_override = os.getenv("BRAIN_GIT_REPO_ROOT")
+    repo_root = (
+        Path(env_override).expanduser().resolve() if env_override else _REPO_ROOT
+    )
+    since, until = _resolve_git_window(ctx)
+    commits = await _git_log_window(
+        since=since,
+        until=until,
+        cwd=repo_root,
+        cap=COMMIT_CAP_PER_CYCLE,
+    )
+    for commit in commits:
+        observed_at = normalize_iso(commit["author_iso"])
+        if observed_at is not None and not ctx.in_window(observed_at):
+            continue
+        yield _commit_event(
+            sha=commit["sha"],
+            author_iso=commit["author_iso"],
+            author_name=commit["author_name"],
+            subject=commit["subject"],
+            now=ctx.now,
+        )
+
+
+async def collect_kg(ctx: SourceCollectorContext) -> AsyncIterator[EventDraft]:
+    """KG kg_changed (hotspots) + doc_changed (doc nodes) — source_system='kg'.
+
+    Cycle window enforcement via ``ctx.in_window`` (bug fix 2026-05-18).
+    """
+    # kg_changed (hotspot delta projection)
+    async with acquire_db() as db:
+        hotspots = await graph_service.get_hotspots(
+            db,
+            window=KG_HOTSPOT_WINDOW,
+            limit=KG_HOTSPOT_LIMIT,
+            type_filter="file",
+        )
+
+    for row in hotspots:
+        touched_at = normalize_iso(row.get("touch_last_at"))
+        if touched_at is None or not ctx.in_window(touched_at):
+            continue
+        evt = _kg_event(row=row, now=ctx.now)
+        if evt is not None:
+            yield evt
+
+    # doc_changed — fetch since the lower bound (max(watermark, window_start))
+    since_doc, _until_doc = _resolve_git_window(ctx)
+    try:
+        doc_rows = await _list_doc_nodes_changed_since(since=since_doc)
+    except Exception:  # pragma: no cover — helper missing or KG offline
+        logger.exception("brain.sources.git_kg: doc_changed fetch failed")
+        doc_rows = []
+
+    for row in doc_rows:
+        touched_at = normalize_iso(row.get("touch_last_at"))
+        if touched_at is None or not ctx.in_window(touched_at):
+            continue
+        evt = _doc_event(row=row, now=ctx.now)
+        if evt is not None:
+            yield evt
+
+
+# Backward-compat alias for tests/code che ancora referenziano collect()
+async def collect(ctx: SourceCollectorContext) -> AsyncIterator[EventDraft]:
+    async for evt in collect_git(ctx):
+        yield evt
+    async for evt in collect_kg(ctx):
+        yield evt
+
+
+git_collector = SourceCollector(source_system="git", collect=collect_git)
+kg_collector = SourceCollector(source_system="kg", collect=collect_kg)
+# Legacy alias (NON registrare — single source_system='git_kg' viola CHECK constraint)
+git_kg_collector = git_collector
+
+
+__all__ = [
+    "COMMIT_CAP_PER_CYCLE",
+    "DOC_NODE_KINDS",
+    "KG_HOTSPOT_LIMIT",
+    "KG_HOTSPOT_WINDOW",
+    "collect",
+    "git_kg_collector",
+]

core/api/services/brain/sources/handoffs.py

@@ -0,0 +1,157 @@
+# Handoffs source collector (sub-01 §3 — `handoff_changed`).
+#
+# Handoffs are filesystem-only: `{metadata_path}/memory/handoff-*.md`. There
+# is no `handoff_index` table in console.db today, so the plan's "fallback
+# to handoff_index.updated_at" is implemented as: `mtime` is authoritative;
+# if a fallback table appears later we add a second branch here.
+from __future__ import annotations
+
+import logging
+from collections.abc import AsyncIterator
+from datetime import datetime, timezone
+from pathlib import Path
+
+import yaml
+
+from core.api.services.brain.digest_collector import SourceCollector
+from core.api.services.brain.models import EventDraft, SourceCollectorContext
+from core.api.services.brain.sources.base import (
+    resolve_source_project,
+    stable_short_hash,
+)
+
+logger = logging.getLogger(__name__)
+
+
+# Per-cycle hard cap protects against an accidental bulk-touch of every
+# handoff file (e.g. `chmod -R`, restore-from-backup). Plan §6.D0 says 1000
+# events/source — we leave the per-source cap to the orchestrator and just
+# bound the directory walk for safety.
+MAX_HANDOFFS_SCANNED: int = 2000
+
+
+def _iter_project_roots() -> list[Path]:
+    """Return project metadata roots without coupling to router globals.
+
+    The Marvis convention (kb/projects.md): `/data/projects/<slug>/` plus
+    `~/workspace/projects/<slug>/`. We import lazily and tolerate missing
+    `api.routers.projects` (e.g. during unit tests that don't bootstrap the
+    full API surface).
+    """
+    try:
+        from core.api.routers.projects import PROJECT_DIRS
+
+        return [p for p in PROJECT_DIRS if p.exists()]
+    except Exception:  # pragma: no cover — defensive
+        defaults = [
+            Path("/data/projects"),
+            Path.home() / "workspace" / "projects",
+        ]
+        return [p for p in defaults if p.exists()]
+
+
+def _parse_handoff_frontmatter(path: Path) -> tuple[list[str], str | None, str | None]:
+    """Return (tags, session, slug_hint) from frontmatter; best-effort."""
+    try:
+        text = path.read_text(encoding="utf-8", errors="replace")
+    except OSError:
+        return [], None, None
+    if not text.startswith("---"):
+        return [], None, None
+    try:
+        _, body = text.split("---", 1)
+        meta_block, _ = body.split("---", 1)
+        meta = yaml.safe_load(meta_block) or {}
+    except (ValueError, yaml.YAMLError):
+        return [], None, None
+    if not isinstance(meta, dict):
+        return [], None, None
+    raw_tags = meta.get("tags") or []
+    if isinstance(raw_tags, str):
+        raw_tags = [raw_tags]
+    tags = [str(t) for t in raw_tags if isinstance(t, (str, int))]
+    session = (
+        str(meta.get("session"))
+        if meta.get("session") not in (None, "")
+        else None
+    )
+    slug_hint = meta.get("project") or meta.get("slug")
+    return tags, session, str(slug_hint) if slug_hint else None
+
+
+def _project_slug_from_path(handoff_path: Path) -> str | None:
+    """Recover the project slug from `<root>/<slug>/memory/<file>` shape."""
+    parts = handoff_path.parts
+    if "memory" not in parts:
+        return None
+    idx = parts.index("memory")
+    return parts[idx - 1] if idx >= 1 else None
+
+
+async def collect(ctx: SourceCollectorContext) -> AsyncIterator[EventDraft]:
+    seen = 0
+    for root in _iter_project_roots():
+        try:
+            project_dirs = [d for d in root.iterdir() if d.is_dir()]
+        except OSError:
+            continue
+        for project_dir in project_dirs:
+            memory_dir = project_dir / "memory"
+            if not memory_dir.is_dir():
+                continue
+            try:
+                handoffs = list(memory_dir.glob("handoff-*.md"))
+            except OSError:
+                continue
+            for handoff in handoffs:
+                if seen >= MAX_HANDOFFS_SCANNED:
+                    return
+                seen += 1
+                try:
+                    stat = handoff.stat()
+                except OSError:
+                    continue
+                mtime = datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc)
+                # Cycle window enforcement (bug fix 2026-05-18) — collapses to
+                # the legacy (watermark, cutoff] filter when no window bounds.
+                if not ctx.in_window(mtime):
+                    continue
+
+                tags, session, slug_hint = _parse_handoff_frontmatter(handoff)
+                slug = slug_hint or _project_slug_from_path(handoff) or project_dir.name
+                source_project, program_key = resolve_source_project(slug)
+                filename_hash = stable_short_hash(handoff.name)
+                source_ref = f"handoff:{slug}:{filename_hash}"
+                evidence = {
+                    "path": str(handoff),
+                    "mtime": mtime.isoformat(),
+                    "size": stat.st_size,
+                    "frontmatter_tags": sorted(tags),
+                }
+                if session:
+                    evidence["session"] = session
+
+                title = handoff.name
+                summary = (
+                    f"handoff {handoff.name} updated"
+                    + (f" (session {session})" if session else "")
+                )
+
+                yield EventDraft(
+                    event_type="handoff_changed",
+                    source_system="handoff",
+                    source_ref=source_ref,
+                    title=title[:200],
+                    summary=summary[:2000],
+                    observed_at=mtime,
+                    derived_from_state_at=mtime,
+                    evidence=evidence,
+                    source_project=source_project,
+                    program_key=program_key,
+                )
+
+
+handoffs_collector = SourceCollector(source_system="handoff", collect=collect)
+
+
+__all__ = ["collect", "handoffs_collector"]

core/api/services/brain/sources/ingestor.py

@@ -0,0 +1,130 @@
+# Ingestor source collector (sub-01 §3 — `ingest_changed` + `external_update_seen`).
+# Reads inbox_items rows whose updated_at falls in (watermark, cutoff_at]
+# and yields one event per row. No business state is mutated.
+from __future__ import annotations
+
+import hashlib
+import logging
+from collections.abc import AsyncIterator
+
+from core.api.db import acquire_db
+from core.api.services.brain.digest_collector import SourceCollector
+from core.api.services.brain.models import EventDraft, SourceCollectorContext
+from core.api.services.brain.sources.base import (
+    normalize_iso,
+    resolve_source_project,
+)
+
+logger = logging.getLogger(__name__)
+
+
+# Inbox sources that represent EXTERNAL feeds (RSS, email, web). These map to
+# `external_update_seen` per plan §3 so the Drift layer can distinguish "user
+# uploaded something" from "we noticed an upstream feed changed". Anything
+# else (manual upload, API ingest, gmail-marvisx) maps to `ingest_changed`.
+_EXTERNAL_SOURCES: frozenset[str] = frozenset({"rss", "email", "web", "external"})
+
+
+def _classify_event_type(source_value: str | None) -> str:
+    if not source_value:
+        return "ingest_changed"
+    return (
+        "external_update_seen"
+        if source_value.lower() in _EXTERNAL_SOURCES
+        else "ingest_changed"
+    )
+
+
+def _source_ref(row_id: str, source_value: str | None, url: str | None) -> str:
+    if (source_value or "").lower() in _EXTERNAL_SOURCES and url:
+        url_hash = hashlib.blake2b(url.encode("utf-8"), digest_size=8).hexdigest()
+        return f"external:{url_hash}"
+    return f"ingest:{row_id}"
+
+
+# Pagination page size — bounded read avoids loading >cap rows at once and
+# enables collectors to span sources with >1000 events per cycle (bug fix
+# 2026-05-18: previous code fetched a single LIMIT-less batch and the
+# orchestrator-level cap silently truncated to per_source_event_cap).
+_PAGE_SIZE: int = 500
+
+
+async def collect(ctx: SourceCollectorContext) -> AsyncIterator[EventDraft]:
+    """Yield ingestor events within the cycle window.
+
+    Window = (max(watermark, cycle_window_start), min(cutoff_at, cycle_window_end)]
+    SQL filter uses string comparison on ISO timestamps; Python-side
+    ``ctx.in_window`` re-checks each row for TZ-naive substrate rows.
+    Pagination via LIMIT/OFFSET so cycles with >page_size events still
+    surface every row.
+    """
+    lower_iso = ctx.lower_bound_iso
+    upper_iso = ctx.upper_bound_iso
+
+    offset = 0
+    while True:
+        async with acquire_db() as db:
+            cur = await db.execute(
+                "SELECT id, source, status, default_program, title, url, "
+                "       workspace_id, updated_at "
+                "FROM inbox_items "
+                "WHERE updated_at > ? AND updated_at <= ? "
+                "ORDER BY updated_at ASC "
+                "LIMIT ? OFFSET ?",
+                (lower_iso, upper_iso, _PAGE_SIZE, offset),
+            )
+            rows = await cur.fetchall()
+        if not rows:
+            break
+
+        for row in rows:
+            row_id = row[0] if not hasattr(row, "keys") else row["id"]
+            source_value = row[1] if not hasattr(row, "keys") else row["source"]
+            status = row[2] if not hasattr(row, "keys") else row["status"]
+            default_program = (
+                row[3] if not hasattr(row, "keys") else row["default_program"]
+            )
+            title = row[4] if not hasattr(row, "keys") else row["title"]
+            url = row[5] if not hasattr(row, "keys") else row["url"]
+            updated_at_raw = row[7] if not hasattr(row, "keys") else row["updated_at"]
+
+            observed_at = normalize_iso(updated_at_raw) or ctx.now
+            if not ctx.in_window(observed_at):
+                continue
+
+            event_type = _classify_event_type(source_value)
+            ref = _source_ref(str(row_id), source_value, url)
+            project, program = resolve_source_project(default_program)
+
+            evidence = {
+                "item_id": str(row_id),
+                "source_kind": source_value,
+                "status": status,
+                "project_hint": default_program,
+                "updated_at": observed_at.isoformat(),
+            }
+            if url:
+                evidence["url"] = url
+
+            yield EventDraft(
+                event_type=event_type,
+                source_system="ingest",
+                source_ref=ref,
+                title=(title or f"Ingest item {row_id}")[:200],
+                summary=f"{source_value or 'inbox'} item status={status}",
+                observed_at=observed_at,
+                derived_from_state_at=observed_at,
+                evidence=evidence,
+                source_project=project,
+                program_key=program,
+            )
+
+        if len(rows) < _PAGE_SIZE:
+            break
+        offset += _PAGE_SIZE
+
+
+ingestor_collector = SourceCollector(source_system="ingest", collect=collect)
+
+
+__all__ = ["collect", "ingestor_collector"]