thinkstack-core 4.0.0__py3-none-any.whl

thinkstack_core/concept_catalog.py

@@ -0,0 +1,209 @@
+"""
+thinkstack_core.concept_catalog
+==============================
+Unified concept enumeration across all ThinkStack data sources.
+
+Sources:
+    - I3 concept definitions (.GCC/concepts/*.json)
+    - Theta coordination vector (.GCC/theta.json)
+    - Sensitivity events (.GCC/sensitivities/events.jsonl)
+    - Learning trigger_concepts (.GCC/reasoning_learnings/learnings/*.json)
+    - Commit concepts (.GCC/commits/*.json)
+    - Call concepts (.GCC/reasoning_learnings/calls/*.json)
+"""
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+from typing import Any
+
+from thinkstack_core.topics import TopicStore
+
+logger = logging.getLogger("thinkstack.concept_catalog")
+
+
+class ConceptCatalog:
+    """Unify all concept sources into a single enumerator."""
+
+    def __init__(self, gcc_dir: Path | str) -> None:
+        self._gcc_dir = Path(gcc_dir)
+
+    def enumerate_all(self) -> dict[str, list[str]]:
+        """Return {concept_name: [source1, source2, ...]} from all sources.
+
+        All concept names are lowercased for deduplication.
+        """
+        result: dict[str, list[str]] = {}
+
+        def _add(concept: str, source: str) -> None:
+            key = concept.lower()
+            if key not in result:
+                result[key] = []
+            if source not in result[key]:
+                result[key].append(source)
+
+        for name in self._i3_concepts():
+            _add(name, "i3_definition")
+        for name in self._theta_concepts():
+            _add(name, "theta")
+        for name in self._sensitivity_concepts():
+            _add(name, "sensitivity")
+        for name in self._learning_concepts():
+            _add(name, "learning")
+        for name in self._commit_concepts():
+            _add(name, "commit")
+        for name in self._call_concepts():
+            _add(name, "call")
+
+        return dict(sorted(result.items()))
+
+    def enumerate_learned(self) -> dict[str, dict[str, Any]]:
+        """Return {concept: {learning_count, active, stale, deprecated, success_rate, avg_confidence}}.
+
+        Only concepts that have at least one learning are included.
+        """
+        from thinkstack_core.reasoning_plus.learning.analytics import concept_stats
+        from thinkstack_core.reasoning_plus.learning.store import LearningStore
+
+        store = LearningStore(self._gcc_dir)
+        stats = concept_stats(store)
+        result: dict[str, dict[str, Any]] = {}
+        for cname, data in stats.items():
+            if cname == "__untyped__":
+                continue
+            result[cname] = {
+                "learning_count": data["total"],
+                "active": data["active"],
+                "stale": data["stale"],
+                "deprecated": data["deprecated"],
+                "success_rate": data["success_rate"],
+                "avg_confidence": data["avg_confidence"],
+            }
+        return result
+
+    def concepts_for_topic(
+        self, topic: str, hide_empty: bool = True,
+    ) -> dict[str, Any]:
+        """Return concepts associated with a topic.
+
+        Returns a dict with:
+            - "concepts": list of {name, sources, has_data} for concepts with data
+            - "taxonomy_only": list of concept names that have no data (if hide_empty)
+        """
+        store = TopicStore(self._gcc_dir)
+        concepts = store.get_concepts(topic)
+        if not concepts:
+            return {"concepts": [], "taxonomy_only": []}
+
+        all_concepts = self.enumerate_all()
+        learned = self.enumerate_learned()
+
+        with_data: list[dict[str, Any]] = []
+        without_data: list[str] = []
+
+        for concept in concepts:
+            sources = all_concepts.get(concept.lower(), [])
+            if sources:
+                entry: dict[str, Any] = {"name": concept, "sources": sources}
+                if concept.lower() in learned:
+                    stats = learned[concept.lower()]
+                    entry["learning_count"] = stats["learning_count"]
+                    entry["success_rate"] = stats["success_rate"]
+                    entry["avg_confidence"] = stats["avg_confidence"]
+                with_data.append(entry)
+            else:
+                without_data.append(concept)
+
+        return {
+            "concepts": with_data,
+            "taxonomy_only": without_data if hide_empty else [],
+        }
+
+    def topic_for_concept(self, concept: str) -> str | None:
+        """Reverse lookup: which topic does this concept belong to?"""
+        store = TopicStore(self._gcc_dir)
+        cname = concept.lower()
+
+        for topic in store.list_topics():
+            topic_concepts = [c.lower() for c in topic["concepts"]]
+            if cname in topic_concepts:
+                return topic["name"]
+
+        return None
+
+    def _i3_concepts(self) -> list[str]:
+        concepts_dir = self._gcc_dir / "concepts"
+        if not concepts_dir.exists():
+            return []
+        return [p.stem for p in concepts_dir.glob("*.json")]
+
+    def _theta_concepts(self) -> list[str]:
+        path = self._gcc_dir / "theta.json"
+        if not path.exists():
+            return []
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+            return list(data.get("coordination_vector", {}).keys())
+        except (json.JSONDecodeError, OSError):
+            return []
+
+    def _sensitivity_concepts(self) -> list[str]:
+        path = self._gcc_dir / "sensitivities" / "events.jsonl"
+        if not path.exists():
+            return []
+        concepts: set[str] = set()
+        try:
+            for line in path.read_text(encoding="utf-8").splitlines():
+                if not line.strip():
+                    continue
+                ev = json.loads(line)
+                payload = ev.get("payload", ev)
+                concept = payload.get("target_concept", "")
+                if concept:
+                    concepts.add(concept)
+        except (json.JSONDecodeError, OSError):
+            pass
+        return list(concepts)
+
+    def _learning_concepts(self) -> list[str]:
+        path = self._gcc_dir / "reasoning_learnings" / "learnings"
+        if not path.exists():
+            return []
+        concepts: set[str] = set()
+        for p in path.glob("*.json"):
+            try:
+                data = json.loads(p.read_text(encoding="utf-8"))
+                for c in data.get("trigger_concepts", []):
+                    concepts.add(c)
+            except (json.JSONDecodeError, OSError):
+                continue
+        return list(concepts)
+
+    def _commit_concepts(self) -> list[str]:
+        path = self._gcc_dir / "commits"
+        if not path.exists():
+            return []
+        concepts: set[str] = set()
+        for p in path.glob("*.json"):
+            try:
+                data = json.loads(p.read_text(encoding="utf-8"))
+                for c in data.get("concepts", []):
+                    concepts.add(c)
+            except (json.JSONDecodeError, OSError):
+                continue
+        return list(concepts)
+
+    def _call_concepts(self) -> list[str]:
+        path = self._gcc_dir / "reasoning_learnings" / "calls"
+        if not path.exists():
+            return []
+        concepts: set[str] = set()
+        for p in path.glob("*.json"):
+            try:
+                data = json.loads(p.read_text(encoding="utf-8"))
+                for c in data.get("concepts", []):
+                    concepts.add(c)
+            except (json.JSONDecodeError, OSError):
+                continue
+        return list(concepts)

thinkstack_core/consolidation/__init__.py

@@ -0,0 +1,3 @@
+from .workflow import ConsolidationRecord, ConsolidationWorkflow
+
+__all__ = ["ConsolidationRecord", "ConsolidationWorkflow"]

thinkstack_core/consolidation/synthesizer.py

@@ -0,0 +1,87 @@
+"""
+LLM-powered consolidation synthesizer.
+
+Uses the same pattern as theta_synthesis.py: builds a prompt from
+ReasoningEntries + DivergenceSignals, sends to claude-haiku, returns
+a ConsolidationDraft. Gated by THINKSTACK_DISABLE env var.
+Never raises — errors return a skip result.
+"""
+from __future__ import annotations
+
+import dataclasses
+import os
+from typing import List
+
+
+@dataclasses.dataclass
+class ConsolidationDraft:
+    decision: str
+    decision_trace: str
+    model: str
+    tokens_used: int
+    skipped: bool = False
+    skip_reason: str = ""
+
+
+def synthesise_consolidation(
+    concept: str,
+    reasoning_entries: list,
+    divergence_signals: list,
+    model: str = "claude-haiku-4-5-20251001",
+) -> ConsolidationDraft:
+    """Run LLM synthesis over reasoning entries + divergence signals."""
+    if os.environ.get("THINKSTACK_DISABLE"):
+        return ConsolidationDraft(decision="", decision_trace="", model=model, tokens_used=0,
+                                  skipped=True, skip_reason="THINKSTACK_DISABLE set")
+    try:
+        import anthropic
+    except ImportError:
+        return ConsolidationDraft(decision="", decision_trace="", model=model, tokens_used=0,
+                                  skipped=True, skip_reason="anthropic SDK not installed")
+
+    prompt = _build_prompt(concept, reasoning_entries, divergence_signals)
+    try:
+        client = anthropic.Anthropic()
+        response = client.messages.create(
+            model=model,
+            max_tokens=512,
+            messages=[{"role": "user", "content": prompt}],
+        )
+        text = response.content[0].text if response.content else ""
+        tokens = response.usage.input_tokens + response.usage.output_tokens
+        decision, trace = _parse_response(text)
+        return ConsolidationDraft(decision=decision, decision_trace=trace,
+                                  model=model, tokens_used=tokens)
+    except Exception as exc:
+        return ConsolidationDraft(decision="", decision_trace="", model=model, tokens_used=0,
+                                  skipped=True, skip_reason=str(exc))
+
+
+def _build_prompt(concept: str, reasoning_entries: list, divergence_signals: list) -> str:
+    entry_lines = "\n".join(
+        f"  [{i+1}] agent={e.agent_id} conf={e.confidence:.2f}: {e.reasoning_text[:200]}"
+        for i, e in enumerate(reasoning_entries[:10])
+    )
+    signal_lines = "\n".join(
+        f"  [{s.signal_type}/{s.severity}] {s.description}"
+        for s in divergence_signals[:5]
+    )
+    return (
+        f"You are a governance analyst. Agents have diverged on concept: '{concept}'.\n\n"
+        f"Agent reasoning:\n{entry_lines or '  (none)'}\n\n"
+        f"Divergence signals:\n{signal_lines or '  (none)'}\n\n"
+        "Write two sections:\n"
+        "DECISION: A 1-2 sentence consolidated position that resolves the divergence.\n"
+        "TRACE: A 1-2 sentence explanation of WHY this consolidation was reached.\n\n"
+        "Format exactly:\nDECISION: <text>\nTRACE: <text>"
+    )
+
+
+def _parse_response(text: str) -> tuple[str, str]:
+    decision, trace = "", ""
+    for line in text.splitlines():
+        if line.startswith("DECISION:"):
+            decision = line[len("DECISION:"):].strip()
+        elif line.startswith("TRACE:"):
+            trace = line[len("TRACE:"):].strip()
+    return decision or text[:200], trace or "LLM synthesis"

thinkstack_core/consolidation/workflow.py

@@ -0,0 +1,175 @@
+from __future__ import annotations
+
+import dataclasses
+import datetime as _dt
+import json
+import uuid
+from pathlib import Path
+from typing import List, Optional
+
+from thinkstack_core.divergence.detector import DivergenceSignal
+
+_CONSOLIDATIONS_DIR = "consolidations"
+
+_SEVERITY_ORDER = {"HIGH": 0, "MEDIUM": 1, "LOW": 2}
+
+
+@dataclasses.dataclass
+class ConsolidationRecord:
+    """
+    Durable record of a consolidation event.
+
+    decision_trace is ThinkStack's own reasoning — WHY it chose to block,
+    escalate to human, or auto-resolve.
+    """
+    record_id: str
+    concept: str
+    signals: List[dict]          # serialised DivergenceSignal dicts
+    state: str                   # drafting | pending_human | auto_resolved | resolved | blocked
+    decision: str                # consolidated position text
+    decision_trace: str          # ThinkStack's reasoning for its action
+    agents_involved: List[str]
+    human_comments: List[dict]
+    created_at: str
+    resolved_at: Optional[str] = None
+
+    def to_dict(self) -> dict:
+        return dataclasses.asdict(self)
+
+    @classmethod
+    def from_dict(cls, d: dict) -> "ConsolidationRecord":
+        known = {f.name for f in dataclasses.fields(cls)}
+        return cls(**{k: v for k, v in d.items() if k in known})
+
+
+class ConsolidationWorkflow:
+    """
+    State machine for agent reasoning consolidation.
+
+    Transitions:
+      drafting → pending_human  (HIGH or MEDIUM severity divergence)
+      drafting → auto_resolved  (LOW severity only)
+      drafting → blocked        (signal type E: consensus contradiction)
+      pending_human → resolved  (human provides decision via add_human_comment)
+    """
+
+    def __init__(self, repo) -> None:
+        self._repo = repo
+        self._gcc_dir = repo.gcc_dir
+        self._dir = self._gcc_dir / _CONSOLIDATIONS_DIR
+        self._dir.mkdir(exist_ok=True)
+
+    def start(
+        self,
+        concept: str,
+        signals: List[DivergenceSignal],
+    ) -> ConsolidationRecord:
+        """Create a ConsolidationRecord and determine initial state from signals."""
+        record_id = str(uuid.uuid4())
+        now = _dt.datetime.now(tz=_dt.timezone.utc).isoformat()
+
+        all_agents = list({aid for s in signals for aid in s.agent_ids})
+        max_severity = self._max_severity(signals)
+
+        # E-type signals always block
+        e_signals = [s for s in signals if s.signal_type == "E"]
+        if e_signals:
+            state = "blocked"
+            decision = ""
+            trace = (
+                f"Blocked: {len(e_signals)} consensus contradiction(s) detected on '{concept}'. "
+                "Human review required before proceeding."
+            )
+        elif max_severity == "LOW":
+            state = "auto_resolved"
+            decision = self._auto_decision(concept, signals)
+            trace = (
+                f"Auto-resolved: all signals LOW severity on '{concept}'. "
+                f"Signals: {', '.join(s.signal_type for s in signals)}. "
+                "No human intervention required."
+            )
+        else:
+            state = "pending_human"
+            decision = ""
+            trace = (
+                f"Escalated to human: {max_severity} severity signal(s) detected on '{concept}'. "
+                f"Signal types: {', '.join(s.signal_type + '(' + s.severity + ')' for s in signals)}. "
+                "Awaiting human comment or resolution."
+            )
+
+        record = ConsolidationRecord(
+            record_id=record_id,
+            concept=concept,
+            signals=[dataclasses.asdict(s) for s in signals],
+            state=state,
+            decision=decision,
+            decision_trace=trace,
+            agents_involved=all_agents,
+            human_comments=[],
+            created_at=now,
+            resolved_at=now if state == "auto_resolved" else None,
+        )
+        self._save(record)
+        return record
+
+    def add_human_comment(
+        self,
+        record_id: str,
+        author: str,
+        comment: str,
+        decision: Optional[str] = None,
+    ) -> ConsolidationRecord:
+        """Add a human comment and optionally resolve the record."""
+        record = self.get(record_id)
+        if record is None:
+            raise ValueError(f"ConsolidationRecord {record_id!r} not found")
+
+        now = _dt.datetime.now(tz=_dt.timezone.utc).isoformat()
+        record.human_comments.append({"author": author, "comment": comment, "timestamp": now})
+
+        if decision is not None:
+            record.decision = decision
+            record.state = "resolved"
+            record.resolved_at = now
+            record.decision_trace += f"\nResolved by {author} at {now[:19]}: {decision}"
+
+        self._save(record)
+        return record
+
+    def get(self, record_id: str) -> Optional[ConsolidationRecord]:
+        path = self._dir / f"{record_id}.json"
+        if not path.exists():
+            return None
+        try:
+            return ConsolidationRecord.from_dict(json.loads(path.read_text(encoding="utf-8")))
+        except (json.JSONDecodeError, TypeError):
+            return None
+
+    def list_for_concept(self, concept: str) -> List[ConsolidationRecord]:
+        results = []
+        for f in self._dir.glob("*.json"):
+            try:
+                d = json.loads(f.read_text(encoding="utf-8"))
+                if d.get("concept") == concept:
+                    results.append(ConsolidationRecord.from_dict(d))
+            except (json.JSONDecodeError, TypeError):
+                continue
+        return sorted(results, key=lambda r: r.created_at, reverse=True)
+
+    def _save(self, record: ConsolidationRecord) -> None:
+        path = self._dir / f"{record.record_id}.json"
+        path.write_text(json.dumps(record.to_dict(), indent=2) + "\n", encoding="utf-8")
+
+    @staticmethod
+    def _max_severity(signals: List[DivergenceSignal]) -> str:
+        if not signals:
+            return "LOW"
+        return min(signals, key=lambda s: _SEVERITY_ORDER.get(s.severity, 99)).severity
+
+    @staticmethod
+    def _auto_decision(concept: str, signals: List[DivergenceSignal]) -> str:
+        return (
+            f"No high-severity divergence detected on '{concept}'. "
+            "Agents may proceed with existing sensitivity signals. "
+            f"Low-severity signals noted: {', '.join(s.description[:80] for s in signals)}."
+        )

thinkstack_core/daemon/__init__.py

@@ -0,0 +1,27 @@
+"""ThinkStack background daemon (S26).
+
+Provides a long-running filesystem watcher that emits `fs_change` events to
+`.GCC/events.log.jsonl` without requiring IDE hooks.
+"""
+
+from thinkstack_core.daemon.supervisor import (
+    PID_FILE_NAME,
+    LOG_FILE_NAME,
+    start_daemon,
+    stop_daemon,
+    daemon_status,
+    daemon_logs,
+    _run_daemon,
+)
+from thinkstack_core.daemon.watcher import ProjectWatcher
+
+__all__ = [
+    "PID_FILE_NAME",
+    "LOG_FILE_NAME",
+    "start_daemon",
+    "stop_daemon",
+    "daemon_status",
+    "daemon_logs",
+    "_run_daemon",
+    "ProjectWatcher",
+]