thinkstack-core 4.0.0__py3-none-any.whl

thinkstack_core/reasoning_plus/learning/chain.py

@@ -0,0 +1,285 @@
+"""
+thinkstack_core.reasoning_plus.learning.chain
+============================================
+Reasoning chain: trace the full developer-agent handoff path.
+
+Traverses: human commit -> agent session -> agent calls -> learnings -> outcomes.
+
+The chain is built by linking:
+1. Commit records (with concepts or session references)
+2. Session records (from .GCC/sessions/)
+3. ReasoningCall records (from .GCC/reasoning_learnings/calls/)
+4. Provenance records (from .GCC/reasoning_learnings/provenance/)
+5. Learning records (from .GCC/reasoning_learnings/learnings/)
+"""
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger("thinkstack.reasoning_plus.learning")
+
+
+def build_chain(
+    gcc_dir: Path | str,
+    concept: str | None = None,
+    max_chains: int = 5,
+) -> list[dict[str, Any]]:
+    """Build reasoning chains by linking commits to sessions to calls to learnings.
+
+    Each chain is a list of nodes:
+      commit -> session -> [call -> learning -> outcome]*
+
+    Chains are filtered to those whose commits or sessions mention *concept*,
+    or all chains if *concept* is None.
+
+    Args:
+        gcc_dir: Path to .GCC/ directory.
+        concept: Optional concept filter.
+        max_chains: Maximum number of chains to return.
+
+    Returns:
+        List of chain dicts, each with keys:
+          - commit_id: the commit ID
+          - commit_message: the commit message
+          - commit_concepts: list of concepts
+          - timestamp: commit timestamp
+          - session: session dict or None
+          - calls: list of call dicts (with learning + outcome)
+    """
+    chains: list[dict] = []
+    commits = _load_commits(gcc_dir)
+
+    for commit in commits:
+        if concept and not _matches_concept(commit, concept):
+            continue
+
+        chain = {
+            "commit_id": commit.get("id", "?"),
+            "commit_message": commit.get("message", ""),
+            "commit_concepts": commit.get("concepts", []),
+            "timestamp": commit.get("timestamp", ""),
+            "session": None,
+            "calls": [],
+        }
+
+        session_id = _extract_session_id(commit)
+        if session_id:
+            session = _load_session(gcc_dir, session_id)
+            if session:
+                chain["session"] = {
+                    "session_id": session.get("session_id"),
+                    "name": session.get("name"),
+                    "problem": session.get("problem"),
+                    "status": session.get("status"),
+                    "subtasks": session.get("subtasks", []),
+                }
+                calls = _load_calls_for_session(gcc_dir, session_id)
+                for call in calls:
+                    call_entry = _build_call_entry(gcc_dir, call)
+                    if call_entry:
+                        chain["calls"].append(call_entry)
+
+        chains.append(chain)
+        if len(chains) >= max_chains:
+            break
+
+    return chains
+
+
+def _load_commits(gcc_dir: Path) -> list[dict]:
+    """Load all commits from .GCC/commits/, sorted newest-first."""
+    commits_dir = Path(gcc_dir) / "commits"
+    if not commits_dir.exists():
+        return []
+    commits: list[dict] = []
+    for path in sorted(commits_dir.glob("*.json"), reverse=True):
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+            commits.append(data)
+        except (json.JSONDecodeError, OSError):
+            continue
+    return commits
+
+
+def _matches_concept(commit: dict, concept: str) -> bool:
+    """Check if a commit matches a concept (via concepts field or message)."""
+    cname = concept.lower()
+    for c in commit.get("concepts", []):
+        if c.lower() == cname:
+            return True
+    if cname in commit.get("message", "").lower():
+        return True
+    return False
+
+
+def _extract_session_id(commit: dict) -> str | None:
+    """Extract session_id from commit message or a stored session_id field."""
+    msg = commit.get("message", "")
+    # Check for [session=xxx] pattern in message
+    import re
+    m = re.search(r"\[session=([^\]]+)\]", msg)
+    if m:
+        return m.group(1)
+    return None
+
+
+def _load_session(gcc_dir: Path, session_id: str) -> dict | None:
+    """Load a session record from .GCC/sessions/."""
+    session_path = Path(gcc_dir) / "sessions" / f"{session_id}.json"
+    if not session_path.exists():
+        # Try matching by prefix or name
+        sessions_dir = Path(gcc_dir) / "sessions"
+        if sessions_dir.exists():
+            for sp in sessions_dir.glob("*.json"):
+                try:
+                    data = json.loads(sp.read_text(encoding="utf-8"))
+                    if data.get("session_id") == session_id or data.get("name") == session_id:
+                        return data
+                except (json.JSONDecodeError, OSError):
+                    continue
+        return None
+    try:
+        return json.loads(session_path.read_text(encoding="utf-8"))
+    except (json.JSONDecodeError, OSError):
+        return None
+
+
+def _load_calls_for_session(gcc_dir: Path, session_id: str) -> list[dict]:
+    """Load all ReasoningCall records for a given session."""
+    calls_dir = Path(gcc_dir) / "reasoning_learnings" / "calls"
+    if not calls_dir.exists():
+        return []
+    calls: list[dict] = []
+    for path in sorted(calls_dir.glob("*.json")):
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+            if data.get("session_id") == session_id:
+                calls.append(data)
+        except (json.JSONDecodeError, OSError):
+            continue
+    return calls
+
+
+def _build_call_entry(gcc_dir: Path, call: dict) -> dict | None:
+    """Build a chain entry for a call, linking to its learnings and outcomes."""
+    call_id = call.get("id", "")
+    if not call_id:
+        return None
+
+    entry: dict = {
+        "call_id": call_id,
+        "call_type": call.get("call_type", ""),
+        "name": call.get("name", ""),
+        "reasoning": call.get("reasoning", ""),
+        "outcome": call.get("outcome", "unknown"),
+        "concepts": call.get("concepts", []),
+        "learnings": [],
+    }
+
+    # Find provenance records for this call
+    provenance = _load_provenance_for_call(gcc_dir, call_id)
+    if provenance:
+        entry["provenance_outcome"] = provenance.get("outcome")
+        for lid in provenance.get("learning_ids", []):
+            learning = _load_learning(gcc_dir, lid)
+            if learning:
+                entry["learnings"].append({
+                    "learning_id": lid,
+                    "content": learning.get("content", "")[:120],
+                    "type": learning.get("type", ""),
+                    "confidence": learning.get("confidence", 0),
+                    "validity": learning.get("validity", ""),
+                    "trigger_concepts": learning.get("trigger_concepts", []),
+                })
+
+    return entry
+
+
+def _load_provenance_for_call(gcc_dir: Path, call_id: str) -> dict | None:
+    """Find the provenance record for a specific call."""
+    prov_dir = Path(gcc_dir) / "reasoning_learnings" / "provenance"
+    if not prov_dir.exists():
+        return None
+    for path in prov_dir.glob("*.json"):
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+            if data.get("call_id") == call_id:
+                return data
+        except (json.JSONDecodeError, OSError):
+            continue
+    return None
+
+
+def _load_learning(gcc_dir: Path, learning_id: str) -> dict | None:
+    """Load a single learning record by ID."""
+    path = Path(gcc_dir) / "reasoning_learnings" / "learnings" / f"{learning_id}.json"
+    if not path.exists():
+        return None
+    try:
+        return json.loads(path.read_text(encoding="utf-8"))
+    except (json.JSONDecodeError, OSError):
+        return None
+
+
+def format_chain(chain: dict, indent: str = "") -> str:
+    """Format a single chain as readable text."""
+    lines = []
+
+    # Commit node
+    cid = chain.get("commit_id", "?")
+    msg = chain.get("commit_message", "")
+    concepts = chain.get("commit_concepts", [])
+    ts = chain.get("timestamp", "")
+    lines.append(f"{indent}Commit {cid}  [{ts}]")
+    lines.append(f"{indent}  Message: {msg[:80]}")
+    if concepts:
+        lines.append(f"{indent}  Concepts: {', '.join(concepts)}")
+
+    # Session node
+    session = chain.get("session")
+    if session:
+        lines.append(f"{indent}  Session: {session.get('name', '?')} ({session.get('status', '?')})")
+        lines.append(f"{indent}    Problem: {session.get('problem', '')[:80]}")
+        subtasks = session.get("subtasks", [])
+        if subtasks:
+            completed = sum(1 for s in subtasks if s.get("status") == "completed")
+            failed = sum(1 for s in subtasks if s.get("status") == "failed")
+            total = len(subtasks)
+            lines.append(f"{indent}    Subtasks: {completed}/{total} completed, {failed} failed")
+
+    # Call + learning nodes
+    calls = chain.get("calls", [])
+    if calls:
+        lines.append(f"{indent}  Agent calls ({len(calls)}):")
+        success = sum(1 for c in calls if c.get("outcome") == "success")
+        failure = sum(1 for c in calls if c.get("outcome") == "failure")
+        lines.append(f"{indent}    Outcomes: {success} success, {failure} failure")
+        for call in calls[:3]:  # Show first 3 calls
+            lines.append(f"{indent}    [{call.get('call_type','')}] {call.get('name','')}: {call.get('reasoning','')[:60]}")
+            lines.append(f"{indent}      Outcome: {call.get('outcome', '?')}")
+            for lr in call.get("learnings", [])[:2]:  # Show first 2 learnings per call
+                lines.append(f"{indent}      Learning: {lr.get('content','')[:60]}")
+                lines.append(f"{indent}        [{lr.get('type','')}] conf={lr.get('confidence',0)} concepts={lr.get('trigger_concepts',[])}")
+        if len(calls) > 3:
+            lines.append(f"{indent}    ... and {len(calls) - 3} more calls")
+    else:
+        lines.append(f"{indent}  No agent calls linked to this commit.")
+
+    return "\n".join(lines)
+
+
+def format_chains(chains: list[dict]) -> str:
+    """Format multiple chains for CLI output."""
+    if not chains:
+        return "No reasoning chains found."
+
+    lines = ["═══ Reasoning chains ═══", ""]
+    for i, chain in enumerate(chains, 1):
+        lines.append(f"Chain #{i}")
+        lines.append(format_chain(chain, indent="  "))
+        lines.append("")
+    lines.append("═══ End ═══")
+    return "\n".join(lines)

thinkstack_core/reasoning_plus/learning/composer.py

@@ -0,0 +1,141 @@
+"""
+thinkstack_core.reasoning_plus.learning.composer
+==============================================
+Format selected learnings into a compact prompt block.
+
+Token budget: caps the learning block at a configurable number of tokens
+so injection never overshoots the call budget.
+"""
+from __future__ import annotations
+
+from thinkstack_core.reasoning_plus.learning.models import Learning
+
+
+def approx_tokens(text: str) -> int:
+    """Rough token estimate (4 chars ~= 1 token for English text)."""
+    return len(text) // 4 + 1
+
+
+class PromptComposer:
+    """Compose a <learnings> block for injection into a user prompt.
+
+    By default only `PUBLIC` learnings are injected. Callers can override
+    ``allowed_sensitivity`` to include PROTECTED or PRIVATE learnings, but the
+    default prevents accidentally leaking sensitive reasoning into prompts.
+
+    When ``token_budget`` is provided to :meth:`compose`, the output is capped
+    so that ``approx_tokens(output) <= token_budget``. Learnings are added in
+    rank order (first items in the list are most relevant). The ``<learnings>``
+    and ``</learnings>`` tags are always preserved.
+    """
+
+    def __init__(
+        self,
+        max_lines: int = 100,
+        allowed_sensitivity: list[str] | None = None,
+    ) -> None:
+        self.max_lines = max_lines
+        self.allowed_sensitivity = set(
+            allowed_sensitivity if allowed_sensitivity is not None else ["PUBLIC"]
+        )
+
+    _HEADER_LINES: int = 4
+    _TRAILER_LINES: int = 2
+
+    @staticmethod
+    def _build_header() -> list[str]:
+        return [
+            "<learnings>",
+            "The following are relevant insights from previous steps in this session.",
+            "Evaluate them against the current evidence; they may be outdated.",
+            "",
+        ]
+
+    @staticmethod
+    def _build_trailer(truncated: bool) -> list[str]:
+        if truncated:
+            return ["", "...</learnings>"]
+        return ["", "</learnings>"]
+
+    @staticmethod
+    def _build_learning_lines(learnings: list[Learning]) -> list[str]:
+        lines: list[str] = []
+        for i, learning in enumerate(learnings, start=1):
+            text = learning.short_form(max_chars=240)
+            concepts_tag = ""
+            if learning.trigger_concepts:
+                concepts_tag = f"  [concepts: {', '.join(learning.trigger_concepts)}]"
+            lines.append(f"{i}. {text}{concepts_tag}")
+        return lines
+
+    def _filter(self, learnings: list[Learning]) -> list[Learning]:
+        return [
+            learning
+            for learning in learnings
+            if learning.sensitivity in self.allowed_sensitivity
+        ]
+
+    def _build_block(
+        self,
+        learnings: list[Learning],
+    ) -> tuple[list[str], list[str], list[str]]:
+        header = self._build_header()
+        body = self._build_learning_lines(learnings)
+        truncated = False
+        if len(body) + self._HEADER_LINES + self._TRAILER_LINES > self.max_lines:
+            max_body = self.max_lines - self._HEADER_LINES - self._TRAILER_LINES + 1
+            body = body[:max_body]
+            truncated = True
+        trailer = self._build_trailer(truncated)
+        return header, body, trailer
+
+    def compose(
+        self,
+        learnings: list[Learning],
+        token_budget: int | None = None,
+        relevance_threshold: float | None = None,
+    ) -> str:
+        if token_budget is not None and token_budget <= 0:
+            return ""
+
+        learnings = self._filter(learnings)
+        if not learnings:
+            return ""
+
+        if relevance_threshold is not None:
+            learnings = [l for l in learnings if l.confidence >= relevance_threshold]
+            if not learnings:
+                return ""
+
+        header, body, trailer = self._build_block(learnings)
+
+        def join_all(b: list[str]) -> str:
+            return "\n".join(header + b + trailer)
+
+        block = join_all(body)
+
+        # Token budget: trim learning lines from the end while preserving
+        # header and trailer structure.
+        if token_budget is not None:
+            while approx_tokens(block) > token_budget:
+                if not body:
+                    return ""
+                body = body[:-1]
+                if not body:
+                    return ""
+                block = join_all(body)
+
+        return block
+
+    def compose_system_prompt(self, learnings: list[Learning]) -> str:
+        """
+        Alternative: format learnings as a short system-prompt appendix.
+        Kept for callers that cannot modify the user prompt.
+        """
+        learnings = self._filter(learnings)
+        if not learnings:
+            return ""
+        parts = ["Relevant context from previous reasoning steps:"]
+        for i, learning in enumerate(learnings, start=1):
+            parts.append(f"{i}. {learning.short_form(max_chars=200)}")
+        return "\n".join(parts)

thinkstack_core/reasoning_plus/learning/conflicts.py

@@ -0,0 +1,184 @@
+"""
+thinkstack_core.reasoning_plus.learning.conflicts
+================================================
+Conflict detection and resolution for evolving learnings.
+
+A learning from week 1 may be superseded by a learning from week 3. The
+``ConflictDetector`` finds existing learnings that overlap with an incoming
+learning (by trigger-concept overlap and content embedding similarity).
+``resolve_conflict`` applies the agreed resolution matrix to decide whether
+the newer or older learning wins, or whether a human must decide.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+from thinkstack_core.reasoning_plus.learning.embeddings import (
+    EmbeddingBackend,
+    KeywordEmbeddingBackend,
+)
+from thinkstack_core.reasoning_plus.learning.models import Learning
+
+
+# ---------------------------------------------------------------------------
+# Data classes
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class Conflict:
+    """A potential conflict between two learnings that share trigger concepts."""
+
+    learning_a: Learning          # existing (older) learning
+    learning_b: Learning          # incoming (newer) learning
+    concept_overlap: list[str]    # shared trigger_concepts
+    embedding_similarity: float   # cosine/keyword similarity between contents
+    timestamp_a: str              # ISO timestamp of learning_a
+    timestamp_b: str              # ISO timestamp of learning_b
+    confidence_a: float
+    confidence_b: float
+    goal_changed: bool = False    # whether the project goal changed since learning_a
+
+
+@dataclass
+class ConflictResolution:
+    """The outcome of resolving a :class:`Conflict`."""
+
+    winner: str                   # learning ID that wins
+    loser: str                    # learning ID that is deprecated/stale
+    action: str                   # "NEWER_WINS" | "OLDER_WINS" | "FLAG_FOR_HUMAN"
+    reason: str                   # explanation of the decision
+    auto_resolved: bool           # True if no human needed
+    conflict: Conflict | None = None
+
+
+# ---------------------------------------------------------------------------
+# Conflict detection
+# ---------------------------------------------------------------------------
+
+
+class ConflictDetector:
+    """Find existing learnings that conflict with an incoming learning.
+
+    A conflict is flagged when two learnings share at least one trigger
+    concept **and** their contents are not identical. The embedding
+    similarity is computed and stored on the :class:`Conflict` so the
+    resolver can weigh it if needed.
+    """
+
+    def __init__(self, backend: EmbeddingBackend | None = None) -> None:
+        self._backend = backend or KeywordEmbeddingBackend()
+
+    def detect_conflicts(
+        self,
+        learning: Learning,
+        existing_learnings: list[Learning],
+    ) -> list[Conflict]:
+        incoming_concepts = {c.lower() for c in learning.trigger_concepts}
+        incoming_emb = self._backend.encode(learning.content)
+        conflicts: list[Conflict] = []
+
+        for existing in existing_learnings:
+            if existing.id == learning.id:
+                continue
+            existing_concepts = {c.lower() for c in existing.trigger_concepts}
+            overlap = sorted(incoming_concepts & existing_concepts)
+            if not overlap:
+                continue
+            # Identical content is a duplicate, not a conflict.
+            if existing.content.strip().lower() == learning.content.strip().lower():
+                continue
+            existing_emb = self._backend.encode(existing.content)
+            sim = self._backend.similarity(incoming_emb, existing_emb)
+            conflicts.append(
+                Conflict(
+                    learning_a=existing,
+                    learning_b=learning,
+                    concept_overlap=overlap,
+                    embedding_similarity=sim,
+                    timestamp_a=existing.created_at,
+                    timestamp_b=learning.created_at,
+                    confidence_a=existing.confidence,
+                    confidence_b=learning.confidence,
+                )
+            )
+        return conflicts
+
+
+# ---------------------------------------------------------------------------
+# Conflict resolution matrix
+# ---------------------------------------------------------------------------
+
+_CONF_DIFF_THRESHOLD = 0.1
+
+
+def resolve_conflict(conflict: Conflict) -> ConflictResolution:
+    r"""Resolve a conflict per the agreed matrix.
+
+    | Condition                                   | Action          |
+    |---------------------------------------------|-----------------|
+    | \|conf_new - conf_old\| <= 0.1              | FLAG_FOR_HUMAN  |
+    | conf_new > conf_old + 0.1                   | NEWER_WINS      |
+    | conf_old > conf_new + 0.1, goal unchanged   | OLDER_WINS      |
+    | conf_old > conf_new + 0.1, goal changed     | FLAG_FOR_HUMAN  |
+
+    ``learning_a`` is the existing (older) learning; ``learning_b`` is the
+    incoming (newer) learning.
+    """
+    conf_old = conflict.confidence_a
+    conf_new = conflict.confidence_b
+    diff = abs(conf_new - conf_old)
+
+    if diff <= _CONF_DIFF_THRESHOLD:
+        return ConflictResolution(
+            winner="",
+            loser="",
+            action="FLAG_FOR_HUMAN",
+            reason=(
+                f"Confidence difference ({diff:.2f}) is within "
+                f"±{_CONF_DIFF_THRESHOLD}; cannot auto-resolve."
+            ),
+            auto_resolved=False,
+            conflict=conflict,
+        )
+
+    if conf_new > conf_old + _CONF_DIFF_THRESHOLD:
+        return ConflictResolution(
+            winner=conflict.learning_b.id,
+            loser=conflict.learning_a.id,
+            action="NEWER_WINS",
+            reason=(
+                f"Newer learning confidence ({conf_new:.2f}) exceeds older "
+                f"({conf_old:.2f}) by more than {_CONF_DIFF_THRESHOLD}."
+            ),
+            auto_resolved=True,
+            conflict=conflict,
+        )
+
+    # conf_old > conf_new + threshold
+    if conflict.goal_changed:
+        return ConflictResolution(
+            winner="",
+            loser="",
+            action="FLAG_FOR_HUMAN",
+            reason=(
+                "Older learning has higher confidence but the project goal "
+                "changed; older learning may no longer be valid."
+            ),
+            auto_resolved=False,
+            conflict=conflict,
+        )
+
+    return ConflictResolution(
+        winner=conflict.learning_a.id,
+        loser=conflict.learning_b.id,
+        action="OLDER_WINS",
+        reason=(
+            f"Older learning confidence ({conf_old:.2f}) exceeds newer "
+            f"({conf_new:.2f}) by more than {_CONF_DIFF_THRESHOLD} and goal "
+            f"is unchanged."
+        ),
+        auto_resolved=True,
+        conflict=conflict,
+    )