devtorch-core 3.0.1__py3-none-any.whl

devtorch_core/theta_synthesis.py

@@ -0,0 +1,268 @@
+"""
+Textual Mode AggPhi — LLM-powered Θ synthesis.
+
+Generates a human-readable summary of the top Θ concepts and writes
+it to theta.json as `textual_summary`. This is gated by A1 (capability
+check) and can be disabled via DEVTORCH_DISABLE env var.
+"""
+
+from __future__ import annotations
+
+import dataclasses
+import os
+from typing import Optional
+
+
+# ---------------------------------------------------------------------------
+# Configuration
+# ---------------------------------------------------------------------------
+
+@dataclasses.dataclass
+class ThetaSynthesisConfig:
+    """Configuration for the Textual Mode AggPhi synthesis step."""
+
+    max_concepts: int = 10
+    max_summary_chars: int = 500
+    model: str = "claude-haiku-4-5-20251001"
+    temperature: float = 0.3
+
+
+# ---------------------------------------------------------------------------
+# Result
+# ---------------------------------------------------------------------------
+
+@dataclasses.dataclass
+class ThetaSynthesisResult:
+    """Result returned by synthesise_theta and run_synthesis_if_eligible."""
+
+    summary: str
+    concepts_used: list
+    model: str
+    tokens_used: int
+    skipped: bool = False
+    skip_reason: str = ""
+
+
+# ---------------------------------------------------------------------------
+# Prompt builder
+# ---------------------------------------------------------------------------
+
+def build_synthesis_prompt(theta_vector: dict, max_concepts: int = 10) -> str:
+    """
+    Build an LLM prompt that asks for a 2–3 sentence synthesis of the top Θ
+    concepts.
+
+    Args:
+        theta_vector: Mapping of concept → {mean_confidence, ...}
+        max_concepts: Maximum number of top concepts to include.
+
+    Returns:
+        A prompt string ready to send to an LLM.
+    """
+    # Extract and sort concepts by mean_confidence descending
+    def _conf(entry: object) -> float:
+        if isinstance(entry, dict):
+            return float(entry.get("mean_confidence", 0.0))
+        return 0.0
+
+    sorted_concepts = sorted(
+        theta_vector.items(),
+        key=lambda kv: _conf(kv[1]),
+        reverse=True,
+    )[:max_concepts]
+
+    if not sorted_concepts:
+        return (
+            "The coordination vector Θ is currently empty. "
+            "Please write 2-3 sentences summarising that no concepts have been "
+            "learned yet and that the agent team should proceed with caution."
+        )
+
+    concept_lines = "\n".join(
+        f"  - {concept}: mean_confidence={_conf(data):.3f}"
+        for concept, data in sorted_concepts
+    )
+
+    prompt = (
+        "You are a governance analyst reviewing an AI agent team's coordination "
+        "vector Θ. The following concepts have been learned by the team, ranked "
+        "by confidence:\n\n"
+        f"{concept_lines}\n\n"
+        "Please write 2-3 concise sentences that:\n"
+        "1. Summarise what the agent team has learned so far.\n"
+        "2. Highlight any sensitive or high-confidence concepts to be careful about.\n"
+        "3. Note any concerns the team should keep in mind going forward.\n\n"
+        "Be direct and precise. Do not repeat the concept names verbatim — "
+        "synthesise the meaning."
+    )
+    return prompt
+
+
+# ---------------------------------------------------------------------------
+# Core synthesis function
+# ---------------------------------------------------------------------------
+
+def synthesise_theta(
+    gcc_repo,
+    config: Optional[ThetaSynthesisConfig] = None,
+) -> ThetaSynthesisResult:
+    """
+    Run LLM-powered Θ synthesis against the given GCCRepository.
+
+    Never raises — all errors produce a skipped result with a reason string.
+
+    Args:
+        gcc_repo: A GCCRepository instance (or compatible duck-type).
+        config:   Optional ThetaSynthesisConfig; defaults are used if None.
+
+    Returns:
+        ThetaSynthesisResult with skipped=False on success, skipped=True otherwise.
+    """
+    if config is None:
+        config = ThetaSynthesisConfig()
+
+    # 1. Check DEVTORCH_DISABLE env var
+    if os.environ.get("DEVTORCH_DISABLE"):
+        return ThetaSynthesisResult(
+            summary="",
+            concepts_used=[],
+            model=config.model,
+            tokens_used=0,
+            skipped=True,
+            skip_reason="DEVTORCH_DISABLE is set",
+        )
+
+    # 2. Load current theta vector
+    try:
+        theta = gcc_repo.get_theta()
+    except Exception as exc:
+        return ThetaSynthesisResult(
+            summary="",
+            concepts_used=[],
+            model=config.model,
+            tokens_used=0,
+            skipped=True,
+            skip_reason=f"get_theta failed: {exc}",
+        )
+
+    coordination_vector: dict = theta.get("coordination_vector", {}) if isinstance(theta, dict) else {}
+
+    if not coordination_vector:
+        return ThetaSynthesisResult(
+            summary="",
+            concepts_used=[],
+            model=config.model,
+            tokens_used=0,
+            skipped=True,
+            skip_reason="empty theta",
+        )
+
+    # 3. Try to import anthropic (lazy / gated)
+    try:
+        import anthropic  # noqa: PLC0415
+    except ImportError:
+        return ThetaSynthesisResult(
+            summary="",
+            concepts_used=[],
+            model=config.model,
+            tokens_used=0,
+            skipped=True,
+            skip_reason="anthropic not installed",
+        )
+
+    # 4. Build prompt and select top concepts
+    def _conf(entry: object) -> float:
+        if isinstance(entry, dict):
+            return float(entry.get("mean_confidence", 0.0))
+        return 0.0
+
+    top_concepts = sorted(
+        coordination_vector.items(),
+        key=lambda kv: _conf(kv[1]),
+        reverse=True,
+    )[: config.max_concepts]
+
+    concepts_used = [c for c, _ in top_concepts]
+    prompt = build_synthesis_prompt(coordination_vector, max_concepts=config.max_concepts)
+
+    # 5. Call the Anthropic API
+    try:
+        client = anthropic.Anthropic()
+        response = client.messages.create(
+            model=config.model,
+            max_tokens=256,
+            temperature=config.temperature,
+            messages=[{"role": "user", "content": prompt}],
+        )
+        raw_summary: str = response.content[0].text
+        tokens_used: int = response.usage.input_tokens + response.usage.output_tokens
+
+        # Truncate to max_summary_chars
+        summary = raw_summary[: config.max_summary_chars]
+
+        # 6. Write textual_summary back to the repo
+        gcc_repo.update_theta_field("textual_summary", summary)
+
+        return ThetaSynthesisResult(
+            summary=summary,
+            concepts_used=concepts_used,
+            model=config.model,
+            tokens_used=tokens_used,
+            skipped=False,
+            skip_reason="",
+        )
+
+    except Exception as exc:
+        return ThetaSynthesisResult(
+            summary="",
+            concepts_used=[],
+            model=config.model,
+            tokens_used=0,
+            skipped=True,
+            skip_reason=str(exc),
+        )
+
+
+# ---------------------------------------------------------------------------
+# Eligibility gate
+# ---------------------------------------------------------------------------
+
+def run_synthesis_if_eligible(
+    gcc_repo,
+    omega_capability=None,
+) -> ThetaSynthesisResult:
+    """
+    Run synthesis only when the A1 capability gate passes.
+
+    Args:
+        gcc_repo:          A GCCRepository instance.
+        omega_capability:  An OmegaCapability instance (or None to skip gate).
+                           If provided, its `textual_mode_allowed` attribute or
+                           method is checked before running synthesis.
+
+    Returns:
+        ThetaSynthesisResult — skipped if the gate fails.
+    """
+    # Import for side-effects / availability check (not used directly here)
+    from devtorch_core.capability import check_round_cap  # noqa: F401, PLC0415
+
+    if omega_capability is not None:
+        tma = omega_capability.textual_mode_allowed
+        # textual_mode_allowed may be a method (returns (bool, reasons)) or a
+        # plain boolean attribute (e.g. on a MagicMock / CapabilityValidationResult)
+        if callable(tma):
+            allowed, _ = tma()
+        else:
+            allowed = bool(tma)
+
+        if not allowed:
+            return ThetaSynthesisResult(
+                summary="",
+                concepts_used=[],
+                model=ThetaSynthesisConfig().model,
+                tokens_used=0,
+                skipped=True,
+                skip_reason="A1 gate: textual mode not allowed",
+            )
+
+    return synthesise_theta(gcc_repo)

devtorch_core/topics.py

@@ -0,0 +1,320 @@
+"""
+devtorch_core.topics
+=====================
+Topic taxonomy, persistence, and auto-derivation.
+
+Topics are higher-level groupings of concepts. The default taxonomy is
+hard-coded and merged at read time (not persisted). Custom topics are
+stored in ``.GCC/topics.json``. Auto-derived topics are computed from
+concept co-occurrence in commits and learnings using graph-based
+connected components clustering.
+"""
+from __future__ import annotations
+
+import json
+import logging
+from collections import defaultdict
+from datetime import datetime, timezone
+from itertools import combinations
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger("devtorch.topics")
+
+DEFAULT_TAXONOMY: dict[str, list[str]] = {
+    "authentication": ["auth", "jwt", "oauth", "session", "secrets"],
+    "database": ["schema", "migration", "indexing", "partitioning"],
+    "api": ["api", "rest", "graphql", "websocket", "endpoints"],
+    "deployment": ["deploy", "docker", "kubernetes", "ci-cd", "config"],
+    "testing": ["testing", "unit-test", "integration-test", "e2e"],
+    "performance": ["performance", "caching", "optimization", "profiling"],
+    "security": ["security", "xss", "csrf", "sql-injection", "secrets", "pii"],
+    "governance": ["audit", "compliance", "disclosure", "sensitivity"],
+}
+
+_MIN_COMPONENT_SIZE = 3
+_JACCARD_THRESHOLD = 0.3
+
+
+class TopicStore:
+    """Persist and query topic→concepts mappings."""
+
+    def __init__(self, gcc_dir: Path | str) -> None:
+        self._gcc_dir = Path(gcc_dir)
+        self._path = self._gcc_dir / "topics.json"
+
+    def list_topics(self, with_data_only: bool = False) -> list[dict[str, Any]]:
+        """Return all topics with metadata.
+
+        Each entry: ``{"name", "source", "concepts", "has_data"}``
+
+        If *with_data_only* is True, only topics where at least one concept
+        appears in theta, sensitivity, learnings, or commits are returned.
+        """
+        custom = self._read_custom()
+        derived = self._read_derived()
+
+        all_topics: list[dict[str, Any]] = []
+
+        for name, concepts in DEFAULT_TAXONOMY.items():
+            has_data = self._topic_has_data(concepts)
+            all_topics.append({
+                "name": name,
+                "source": "default",
+                "concepts": concepts,
+                "has_data": has_data,
+            })
+
+        for name, concepts in custom.items():
+            has_data = self._topic_has_data(concepts)
+            all_topics.append({
+                "name": name,
+                "source": "custom",
+                "concepts": concepts,
+                "has_data": has_data,
+            })
+
+        for name, concepts in derived.items():
+            has_data = self._topic_has_data(concepts)
+            all_topics.append({
+                "name": name,
+                "source": "derived",
+                "concepts": concepts,
+                "has_data": has_data,
+            })
+
+        if with_data_only:
+            all_topics = [t for t in all_topics if t["has_data"]]
+
+        return sorted(all_topics, key=lambda t: t["name"])
+
+    def get_concepts(self, topic: str) -> list[str]:
+        """Return concepts for a topic from any source (custom, default, derived)."""
+        custom = self._read_custom()
+        if topic in custom:
+            return custom[topic]
+
+        derived = self._read_derived()
+        if topic in derived:
+            return derived[topic]
+
+        if topic in DEFAULT_TAXONOMY:
+            return DEFAULT_TAXONOMY[topic]
+
+        return []
+
+    def add_topic(self, name: str, concepts: list[str]) -> None:
+        """Add or update a custom topic mapping."""
+        data = self._read_full()
+        data.setdefault("custom_topics", {})[name] = sorted(set(concepts))
+        data["updated_at"] = _now_iso()
+        self._write_full(data)
+
+    def remove_topic(self, name: str) -> bool:
+        """Remove a custom topic. Returns True if removed."""
+        data = self._read_full()
+        custom = data.get("custom_topics", {})
+        if name not in custom:
+            return False
+        del custom[name]
+        data["updated_at"] = _now_iso()
+        self._write_full(data)
+        return True
+
+    def derive_topics(self, force: bool = False) -> dict[str, list[str]]:
+        """Derive topics from concept co-occurrence and cache results.
+
+        Re-derivation is skipped unless *force* is True or the commit/learning
+        count signature has changed since last derivation.
+        """
+        data = self._read_full()
+        current_sig = self._compute_signature()
+
+        if not force and data.get("derived_signature") == current_sig:
+            return data.get("derived_topics", {})
+
+        derived = self._compute_derived_topics()
+        data["derived_topics"] = derived
+        data["derived_at"] = _now_iso()
+        data["derived_signature"] = current_sig
+        data["updated_at"] = _now_iso()
+        self._write_full(data)
+        return derived
+
+    def _compute_signature(self) -> str:
+        """Compute a signature based on commit and learning counts."""
+        commits_dir = self._gcc_dir / "commits"
+        learnings_dir = self._gcc_dir / "reasoning_learnings" / "learnings"
+        n_commits = len(list(commits_dir.glob("*.json"))) if commits_dir.exists() else 0
+        n_learnings = len(list(learnings_dir.glob("*.json"))) if learnings_dir.exists() else 0
+        return f"commits:{n_commits},learnings:{n_learnings}"
+
+    def _compute_derived_topics(self) -> dict[str, list[str]]:
+        """Compute auto-derived topics using graph-based connected components.
+
+        Algorithm:
+        1. Build a co-occurrence graph from commits and learnings.
+        2. Weight edges by Jaccard similarity of co-occurrence.
+        3. Remove edges with Jaccard < threshold.
+        4. Find connected components.
+        5. Discard components with fewer than MIN_COMPONENT_SIZE concepts.
+        6. Label each component by its highest-degree concept.
+        """
+        co_occurrence: dict[frozenset[str], int] = defaultdict(int)
+        occurrence_count: dict[str, int] = defaultdict(int)
+
+        concept_sets = self._load_concept_sets()
+        for concept_list in concept_sets:
+            concepts = [c.lower() for c in concept_list]
+            for c in concepts:
+                occurrence_count[c] += 1
+            for pair in combinations(sorted(set(concepts)), 2):
+                co_occurrence[frozenset(pair)] += 1
+
+        if not co_occurrence:
+            return {}
+
+        adj: dict[str, set[str]] = defaultdict(set)
+        for pair, co_count in co_occurrence.items():
+            c1, c2 = sorted(pair)
+            union = occurrence_count[c1] + occurrence_count[c2] - co_count
+            jaccard = co_count / union if union > 0 else 0
+            if jaccard >= _JACCARD_THRESHOLD:
+                adj[c1].add(c2)
+                adj[c2].add(c1)
+
+        components = self._connected_components(adj)
+        derived: dict[str, list[str]] = {}
+        for comp in components:
+            if len(comp) < _MIN_COMPONENT_SIZE:
+                continue
+            label = max(comp, key=lambda c: len(adj.get(c, set())))
+            derived[label] = sorted(comp)
+
+        return derived
+
+    def _load_concept_sets(self) -> list[list[str]]:
+        """Load concept sets from commits and learnings."""
+        concept_sets: list[list[str]] = []
+
+        commits_dir = self._gcc_dir / "commits"
+        if commits_dir.exists():
+            for path in commits_dir.glob("*.json"):
+                try:
+                    data = json.loads(path.read_text(encoding="utf-8"))
+                    concepts = data.get("concepts", [])
+                    if concepts:
+                        concept_sets.append(concepts)
+                except (json.JSONDecodeError, OSError):
+                    continue
+
+        learnings_dir = self._gcc_dir / "reasoning_learnings" / "learnings"
+        if learnings_dir.exists():
+            for path in learnings_dir.glob("*.json"):
+                try:
+                    data = json.loads(path.read_text(encoding="utf-8"))
+                    concepts = data.get("trigger_concepts", [])
+                    if concepts:
+                        concept_sets.append(concepts)
+                except (json.JSONDecodeError, OSError):
+                    continue
+
+        return concept_sets
+
+    @staticmethod
+    def _connected_components(adj: dict[str, set[str]]) -> list[list[str]]:
+        """Find connected components in an undirected graph."""
+        visited: set[str] = set()
+        components: list[list[str]] = []
+        for node in adj:
+            if node in visited:
+                continue
+            stack = [node]
+            comp: list[str] = []
+            while stack:
+                curr = stack.pop()
+                if curr in visited:
+                    continue
+                visited.add(curr)
+                comp.append(curr)
+                stack.extend(adj.get(curr, set()) - visited)
+            components.append(comp)
+        return components
+
+    def _topic_has_data(self, concepts: list[str]) -> bool:
+        """Check if at least one concept appears in theta, sensitivity, learnings, or commits."""
+        known = self._get_known_concepts()
+        return any(c.lower() in known for c in concepts)
+
+    def _get_known_concepts(self) -> set[str]:
+        """Get the set of all concepts that appear in any data source."""
+        known: set[str] = set()
+
+        theta_path = self._gcc_dir / "theta.json"
+        if theta_path.exists():
+            try:
+                theta = json.loads(theta_path.read_text(encoding="utf-8"))
+                known.update(k.lower() for k in theta.get("coordination_vector", {}).keys())
+            except (json.JSONDecodeError, OSError):
+                pass
+
+        events_path = self._gcc_dir / "sensitivities" / "events.jsonl"
+        if events_path.exists():
+            try:
+                for line in events_path.read_text(encoding="utf-8").splitlines():
+                    if not line.strip():
+                        continue
+                    ev = json.loads(line)
+                    payload = ev.get("payload", ev)
+                    concept = payload.get("target_concept", "")
+                    if concept:
+                        known.add(concept.lower())
+            except (json.JSONDecodeError, OSError):
+                pass
+
+        learnings_dir = self._gcc_dir / "reasoning_learnings" / "learnings"
+        if learnings_dir.exists():
+            for path in learnings_dir.glob("*.json"):
+                try:
+                    data = json.loads(path.read_text(encoding="utf-8"))
+                    for c in data.get("trigger_concepts", []):
+                        known.add(c.lower())
+                except (json.JSONDecodeError, OSError):
+                    continue
+
+        commits_dir = self._gcc_dir / "commits"
+        if commits_dir.exists():
+            for path in commits_dir.glob("*.json"):
+                try:
+                    data = json.loads(path.read_text(encoding="utf-8"))
+                    for c in data.get("concepts", []):
+                        known.add(c.lower())
+                except (json.JSONDecodeError, OSError):
+                    continue
+
+        return known
+
+    def _read_custom(self) -> dict[str, list[str]]:
+        return self._read_full().get("custom_topics", {})
+
+    def _read_derived(self) -> dict[str, list[str]]:
+        return self._read_full().get("derived_topics", {})
+
+    def _read_full(self) -> dict[str, Any]:
+        if not self._path.exists():
+            return {"version": "1", "custom_topics": {}, "derived_topics": {}}
+        try:
+            return json.loads(self._path.read_text(encoding="utf-8"))
+        except (json.JSONDecodeError, OSError):
+            return {"version": "1", "custom_topics": {}, "derived_topics": {}}
+
+    def _write_full(self, data: dict[str, Any]) -> None:
+        self._gcc_dir.mkdir(parents=True, exist_ok=True)
+        self._path.write_text(
+            json.dumps(data, indent=2, sort_keys=True) + "\n",
+            encoding="utf-8",
+        )
+
+
+def _now_iso() -> str:
+    return datetime.now(timezone.utc).isoformat()