devtorch-core 3.0.1__py3-none-any.whl

devtorch_core/reasoning_plus/learning/api.py

@@ -0,0 +1,313 @@
+"""
+devtorch_core.reasoning_plus.learning.api
+=========================================
+High-level facade for Reasoning Plus Learning (DRPL).
+
+This is the main entry point for callers. It orchestrates:
+- recording calls
+- extracting learnings
+- retrieving relevant learnings
+- composing prompt blocks
+- applying outcome feedback
+"""
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, Callable
+
+import logging
+
+from devtorch_core.reasoning_plus.config import ReasoningPlusConfig
+from devtorch_core.reasoning_plus.learning.composer import PromptComposer
+from devtorch_core.reasoning_plus.learning.embeddings import (
+    EmbeddingBackend,
+    KeywordEmbeddingBackend,
+    get_embedding_backend,
+)
+from devtorch_core.reasoning_plus.learning.extractor import (
+    LearningExtractor,
+    make_extractor,
+)
+from devtorch_core.reasoning_plus.learning.models import Learning, ReasoningCall, _make_id
+from devtorch_core.reasoning_plus.learning.provenance import ProvenanceRecord, ProvenanceStore, _hash_prompt
+from devtorch_core.reasoning_plus.learning.recorder import CallRecorder
+from devtorch_core.reasoning_plus.learning.relevance import RelevanceEngine
+from devtorch_core.reasoning_plus.learning.state import workspace_state_hash
+from devtorch_core.reasoning_plus.learning.store import LearningStore
+from devtorch_core.reasoning_plus.learning import cross_project
+
+logger = logging.getLogger("devtorch.reasoning_plus.learning")
+
+
+LLMClient = Callable[[str], str]
+
+
+class ReasoningPlusLearning:
+    """
+    Facade for learning from reasoning and feeding it back into future calls.
+
+    Usage:
+        drpl = ReasoningPlusLearning(gcc_dir, repo_path)
+        call_id = drpl.record_call(
+            call_type="tool",
+            name="read_file",
+            reasoning="Reading utils.py because the error mentions parse_date.",
+            outcome="failure",
+            input="utils.py",
+            output="...",
+            state_hash=drpl.workspace_hash(),
+            session_id="session-1",
+        )
+        drpl.extract_learnings(call_id)
+        learnings = drpl.get_relevant_learnings("Fix the parse_date bug")
+        prompt = drpl.compose_user_prompt("Fix the parse_date bug", learnings)
+    """
+
+    def __init__(
+        self,
+        gcc_dir: Path | str,
+        repo_path: Path | str | None = None,
+        config: ReasoningPlusConfig | None = None,
+        llm_client: Callable[[str], str] | None = None,
+    ) -> None:
+        self._gcc_dir = Path(gcc_dir)
+        self._repo_path = Path(repo_path) if repo_path else self._gcc_dir.parent
+        self._config = config or ReasoningPlusConfig()
+        self._llm_client = llm_client
+
+        self._recorder = CallRecorder(self._gcc_dir)
+        self._store = LearningStore(self._gcc_dir)
+        self._provenance = ProvenanceStore(self._gcc_dir)
+        self._extractor = self._make_extractor()
+        self._relevance = RelevanceEngine(
+            backend=self._make_backend(),
+            cache_ttl=self._config.learning_relevance_cache_ttl,
+        )
+        self._composer = PromptComposer(max_lines=self._config.learning_max_lines)
+
+    def _make_extractor(self) -> LearningExtractor:
+        """Build the configured learning extractor, falling back to rule-based on errors."""
+        try:
+            return make_extractor(
+                strategy=self._config.learning_extraction_strategy,
+                llm_client=self._llm_client,
+            )
+        except Exception as exc:
+            logger.warning(
+                "devtorch: failed to load learning extractor strategy '%s' — %s; falling back to rule-based",
+                self._config.learning_extraction_strategy,
+                exc,
+            )
+            return LearningExtractor()
+
+    def _make_backend(self) -> EmbeddingBackend:
+        """Build the configured embedding backend, falling back to keyword on errors."""
+        try:
+            return get_embedding_backend(
+                self._config.learning_embedding_backend,
+                self._config.learning_embedding_model,
+                max_keywords=self._config.max_keywords,
+            )
+        except Exception as exc:
+            logger.warning(
+                "devtorch: failed to load embedding backend '%s' — %s; falling back to keyword",
+                self._config.learning_embedding_backend,
+                exc,
+            )
+            return KeywordEmbeddingBackend(max_keywords=self._config.max_keywords)
+
+    # ------------------------------------------------------------------
+    # Recording
+    # ------------------------------------------------------------------
+
+    def record_call(self, *, call_type: str, reasoning: str, session_id: str = "", name: str = "", input: str = "", output: str = "", outcome: str = "unknown", state_hash: str = "", concepts: list[str] | None = None, confidence: float = 1.0, sensitivity: str = "PUBLIC", meta: dict[str, Any] | None = None, call_id: str | None = None) -> str:
+        """Record a single call and return its ID."""
+        call = ReasoningCall(
+            id=call_id or _make_id("call"),
+            call_type=call_type,
+            name=name,
+            reasoning=reasoning,
+            input=input,
+            output=output,
+            outcome=outcome,
+            state_hash=state_hash or self.workspace_hash(),
+            concepts=concepts or [],
+            confidence=confidence,
+            sensitivity=sensitivity,
+            session_id=session_id,
+            meta=meta or {},
+        )
+        self._recorder.record(call)
+        return call.id
+
+    def record_and_extract(self, *, call_type: str, reasoning: str, session_id: str = "", outcome: str = "unknown", **kwargs: Any) -> tuple[str, list[str]]:
+        """Record a call and immediately extract learnings from it."""
+        call_id = self.record_call(
+            call_type=call_type,
+            reasoning=reasoning,
+            session_id=session_id,
+            outcome=outcome,
+            **kwargs,
+        )
+        learning_ids = self.extract_learnings(call_id)
+        return call_id, learning_ids
+
+    # ------------------------------------------------------------------
+    # Extraction
+    # ------------------------------------------------------------------
+
+    def extract_learnings(self, call_id: str) -> list[str]:
+        """Extract and store learnings from a recorded call. Returns learning IDs."""
+        call = self._recorder.get(call_id)
+        if not call:
+            return []
+        learnings = self._extractor.extract(call)
+        ids: list[str] = []
+        for learning in learnings:
+            self._store.save(learning)
+            ids.append(learning.id)
+        return ids
+
+    # ------------------------------------------------------------------
+    # Retrieval
+    # ------------------------------------------------------------------
+
+    def get_relevant_learnings(
+        self,
+        context: str,
+        top_n: int | None = None,
+        state_hash: str | None = None,
+    ) -> list[Learning]:
+        """Return the most relevant active learnings for the given context."""
+        if not self._config.learning_enabled:
+            return []
+        top_n = top_n or self._config.learning_top_n
+        all_active = self._store.list(validity="active")
+        return self._relevance.rank(
+            context,
+            all_active,
+            current_state_hash=state_hash or self.workspace_hash(),
+            top_n=top_n,
+        )
+
+    def cross_project_suggest(self, query: str, top_n: int = 3) -> list[Learning]:
+        """Return relevant learnings from sibling governed projects.
+
+        Looks up the machine-level registry of governed projects, finds projects
+        whose name or tags overlap with the query, loads their active learnings,
+        and returns the top-N most relevant ones.
+        """
+        if not self._config.learning_enabled:
+            return []
+        return cross_project.cross_project_suggest(
+            query=query,
+            current_project_root=self._repo_path,
+            top_n=top_n,
+        )
+
+    # ------------------------------------------------------------------
+    # Composition
+    # ------------------------------------------------------------------
+
+    def compose_user_prompt(self, user_prompt: str, learnings: list[Learning]) -> str:
+        """Append a compact <learnings> block to the user prompt."""
+        block = self._composer.compose(learnings)
+        if not block:
+            return user_prompt
+        return f"{user_prompt}\n\n{block}"
+
+    def compose_system_prompt(self, system_prompt: str, learnings: list[Learning]) -> str:
+        """Append a learning appendix to the system prompt (fallback)."""
+        block = self._composer.compose_system_prompt(learnings)
+        if not block:
+            return system_prompt
+        return f"{system_prompt}\n\n{block}"
+
+    # ------------------------------------------------------------------
+    # Feedback
+    # ------------------------------------------------------------------
+
+    def apply_feedback(self, learning_id: str, outcome: str) -> Learning | None:
+        """Adjust confidence or validity based on observed outcome."""
+        learning = self._store.get(learning_id)
+        if not learning:
+            return None
+
+        if outcome == "success":
+            learning.confidence = min(1.0, learning.confidence + 0.1)
+        elif outcome == "failure":
+            learning.confidence = max(0.0, learning.confidence - 0.2)
+            if learning.confidence < 0.3:
+                learning.validity = "deprecated"
+        elif outcome == "stale":
+            learning.validity = "stale"
+
+        learning.updated_at = _now_iso()
+        self._store.save(learning)
+        return learning
+
+    # ------------------------------------------------------------------
+    # Provenance
+    # ------------------------------------------------------------------
+
+    def record_injection(
+        self,
+        learning_ids: list[str],
+        *,
+        call_id: str,
+        session_id: str = "",
+        prompt_text: str = "",
+        model_name: str = "",
+    ) -> ProvenanceRecord:
+        """Record that a set of learnings was injected into a call."""
+        record = ProvenanceRecord(
+            call_id=call_id,
+            learning_ids=learning_ids,
+            session_id=session_id,
+            prompt_hash=_hash_prompt(prompt_text),
+            prompt_excerpt=prompt_text[:200],
+            model_name=model_name,
+        )
+        self._provenance.save(record)
+        return record
+
+    def record_injection_outcome(self, call_id: str, outcome: str) -> None:
+        """Update provenance for a call and apply feedback to injected learnings."""
+        record = self._provenance.update_outcome(call_id, outcome)
+        if not record:
+            return
+        for learning_id in record.learning_ids:
+            try:
+                self.apply_feedback(learning_id, outcome)
+            except Exception as exc:
+                logger.warning("devtorch: feedback failed for learning %s — %s", learning_id, exc)
+
+    def get_provenance(self, call_id: str) -> ProvenanceRecord | None:
+        """Return the provenance record for a call, if any."""
+        return self._provenance.get_by_call(call_id)
+
+    def list_provenance(self, learning_id: str | None = None) -> list[ProvenanceRecord]:
+        """Return provenance records, optionally filtered by learning."""
+        return self._provenance.list(learning_id=learning_id)
+
+    def invalidate_stale(self, changed_paths: list[str]) -> list[str]:
+        """Mark learnings affected by the given changed paths as stale."""
+        from devtorch_core.reasoning_plus.learning.state import affected_by_state_change
+        invalidated: list[str] = []
+        for learning in self._store.list(validity="active"):
+            if affected_by_state_change(self._repo_path, learning.trigger_concepts, changed_paths):
+                self._store.invalidate(learning.id, reason="stale")
+                invalidated.append(learning.id)
+        return invalidated
+
+    # ------------------------------------------------------------------
+    # State
+    # ------------------------------------------------------------------
+
+    def workspace_hash(self, file_paths: list[str] | None = None) -> str:
+        return workspace_state_hash(self._repo_path, file_paths)
+
+
+def _now_iso() -> str:
+    from datetime import datetime, timezone
+    return datetime.now(timezone.utc).isoformat()

devtorch_core/reasoning_plus/learning/chain.py

@@ -0,0 +1,285 @@
+"""
+devtorch_core.reasoning_plus.learning.chain
+============================================
+Reasoning chain: trace the full developer-agent handoff path.
+
+Traverses: human commit -> agent session -> agent calls -> learnings -> outcomes.
+
+The chain is built by linking:
+1. Commit records (with concepts or session references)
+2. Session records (from .GCC/sessions/)
+3. ReasoningCall records (from .GCC/reasoning_learnings/calls/)
+4. Provenance records (from .GCC/reasoning_learnings/provenance/)
+5. Learning records (from .GCC/reasoning_learnings/learnings/)
+"""
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger("devtorch.reasoning_plus.learning")
+
+
+def build_chain(
+    gcc_dir: Path | str,
+    concept: str | None = None,
+    max_chains: int = 5,
+) -> list[dict[str, Any]]:
+    """Build reasoning chains by linking commits to sessions to calls to learnings.
+
+    Each chain is a list of nodes:
+      commit -> session -> [call -> learning -> outcome]*
+
+    Chains are filtered to those whose commits or sessions mention *concept*,
+    or all chains if *concept* is None.
+
+    Args:
+        gcc_dir: Path to .GCC/ directory.
+        concept: Optional concept filter.
+        max_chains: Maximum number of chains to return.
+
+    Returns:
+        List of chain dicts, each with keys:
+          - commit_id: the commit ID
+          - commit_message: the commit message
+          - commit_concepts: list of concepts
+          - timestamp: commit timestamp
+          - session: session dict or None
+          - calls: list of call dicts (with learning + outcome)
+    """
+    chains: list[dict] = []
+    commits = _load_commits(gcc_dir)
+
+    for commit in commits:
+        if concept and not _matches_concept(commit, concept):
+            continue
+
+        chain = {
+            "commit_id": commit.get("id", "?"),
+            "commit_message": commit.get("message", ""),
+            "commit_concepts": commit.get("concepts", []),
+            "timestamp": commit.get("timestamp", ""),
+            "session": None,
+            "calls": [],
+        }
+
+        session_id = _extract_session_id(commit)
+        if session_id:
+            session = _load_session(gcc_dir, session_id)
+            if session:
+                chain["session"] = {
+                    "session_id": session.get("session_id"),
+                    "name": session.get("name"),
+                    "problem": session.get("problem"),
+                    "status": session.get("status"),
+                    "subtasks": session.get("subtasks", []),
+                }
+                calls = _load_calls_for_session(gcc_dir, session_id)
+                for call in calls:
+                    call_entry = _build_call_entry(gcc_dir, call)
+                    if call_entry:
+                        chain["calls"].append(call_entry)
+
+        chains.append(chain)
+        if len(chains) >= max_chains:
+            break
+
+    return chains
+
+
+def _load_commits(gcc_dir: Path) -> list[dict]:
+    """Load all commits from .GCC/commits/, sorted newest-first."""
+    commits_dir = Path(gcc_dir) / "commits"
+    if not commits_dir.exists():
+        return []
+    commits: list[dict] = []
+    for path in sorted(commits_dir.glob("*.json"), reverse=True):
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+            commits.append(data)
+        except (json.JSONDecodeError, OSError):
+            continue
+    return commits
+
+
+def _matches_concept(commit: dict, concept: str) -> bool:
+    """Check if a commit matches a concept (via concepts field or message)."""
+    cname = concept.lower()
+    for c in commit.get("concepts", []):
+        if c.lower() == cname:
+            return True
+    if cname in commit.get("message", "").lower():
+        return True
+    return False
+
+
+def _extract_session_id(commit: dict) -> str | None:
+    """Extract session_id from commit message or a stored session_id field."""
+    msg = commit.get("message", "")
+    # Check for [session=xxx] pattern in message
+    import re
+    m = re.search(r"\[session=([^\]]+)\]", msg)
+    if m:
+        return m.group(1)
+    return None
+
+
+def _load_session(gcc_dir: Path, session_id: str) -> dict | None:
+    """Load a session record from .GCC/sessions/."""
+    session_path = Path(gcc_dir) / "sessions" / f"{session_id}.json"
+    if not session_path.exists():
+        # Try matching by prefix or name
+        sessions_dir = Path(gcc_dir) / "sessions"
+        if sessions_dir.exists():
+            for sp in sessions_dir.glob("*.json"):
+                try:
+                    data = json.loads(sp.read_text(encoding="utf-8"))
+                    if data.get("session_id") == session_id or data.get("name") == session_id:
+                        return data
+                except (json.JSONDecodeError, OSError):
+                    continue
+        return None
+    try:
+        return json.loads(session_path.read_text(encoding="utf-8"))
+    except (json.JSONDecodeError, OSError):
+        return None
+
+
+def _load_calls_for_session(gcc_dir: Path, session_id: str) -> list[dict]:
+    """Load all ReasoningCall records for a given session."""
+    calls_dir = Path(gcc_dir) / "reasoning_learnings" / "calls"
+    if not calls_dir.exists():
+        return []
+    calls: list[dict] = []
+    for path in sorted(calls_dir.glob("*.json")):
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+            if data.get("session_id") == session_id:
+                calls.append(data)
+        except (json.JSONDecodeError, OSError):
+            continue
+    return calls
+
+
+def _build_call_entry(gcc_dir: Path, call: dict) -> dict | None:
+    """Build a chain entry for a call, linking to its learnings and outcomes."""
+    call_id = call.get("id", "")
+    if not call_id:
+        return None
+
+    entry: dict = {
+        "call_id": call_id,
+        "call_type": call.get("call_type", ""),
+        "name": call.get("name", ""),
+        "reasoning": call.get("reasoning", ""),
+        "outcome": call.get("outcome", "unknown"),
+        "concepts": call.get("concepts", []),
+        "learnings": [],
+    }
+
+    # Find provenance records for this call
+    provenance = _load_provenance_for_call(gcc_dir, call_id)
+    if provenance:
+        entry["provenance_outcome"] = provenance.get("outcome")
+        for lid in provenance.get("learning_ids", []):
+            learning = _load_learning(gcc_dir, lid)
+            if learning:
+                entry["learnings"].append({
+                    "learning_id": lid,
+                    "content": learning.get("content", "")[:120],
+                    "type": learning.get("type", ""),
+                    "confidence": learning.get("confidence", 0),
+                    "validity": learning.get("validity", ""),
+                    "trigger_concepts": learning.get("trigger_concepts", []),
+                })
+
+    return entry
+
+
+def _load_provenance_for_call(gcc_dir: Path, call_id: str) -> dict | None:
+    """Find the provenance record for a specific call."""
+    prov_dir = Path(gcc_dir) / "reasoning_learnings" / "provenance"
+    if not prov_dir.exists():
+        return None
+    for path in prov_dir.glob("*.json"):
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+            if data.get("call_id") == call_id:
+                return data
+        except (json.JSONDecodeError, OSError):
+            continue
+    return None
+
+
+def _load_learning(gcc_dir: Path, learning_id: str) -> dict | None:
+    """Load a single learning record by ID."""
+    path = Path(gcc_dir) / "reasoning_learnings" / "learnings" / f"{learning_id}.json"
+    if not path.exists():
+        return None
+    try:
+        return json.loads(path.read_text(encoding="utf-8"))
+    except (json.JSONDecodeError, OSError):
+        return None
+
+
+def format_chain(chain: dict, indent: str = "") -> str:
+    """Format a single chain as readable text."""
+    lines = []
+
+    # Commit node
+    cid = chain.get("commit_id", "?")
+    msg = chain.get("commit_message", "")
+    concepts = chain.get("commit_concepts", [])
+    ts = chain.get("timestamp", "")
+    lines.append(f"{indent}Commit {cid}  [{ts}]")
+    lines.append(f"{indent}  Message: {msg[:80]}")
+    if concepts:
+        lines.append(f"{indent}  Concepts: {', '.join(concepts)}")
+
+    # Session node
+    session = chain.get("session")
+    if session:
+        lines.append(f"{indent}  Session: {session.get('name', '?')} ({session.get('status', '?')})")
+        lines.append(f"{indent}    Problem: {session.get('problem', '')[:80]}")
+        subtasks = session.get("subtasks", [])
+        if subtasks:
+            completed = sum(1 for s in subtasks if s.get("status") == "completed")
+            failed = sum(1 for s in subtasks if s.get("status") == "failed")
+            total = len(subtasks)
+            lines.append(f"{indent}    Subtasks: {completed}/{total} completed, {failed} failed")
+
+    # Call + learning nodes
+    calls = chain.get("calls", [])
+    if calls:
+        lines.append(f"{indent}  Agent calls ({len(calls)}):")
+        success = sum(1 for c in calls if c.get("outcome") == "success")
+        failure = sum(1 for c in calls if c.get("outcome") == "failure")
+        lines.append(f"{indent}    Outcomes: {success} success, {failure} failure")
+        for call in calls[:3]:  # Show first 3 calls
+            lines.append(f"{indent}    [{call.get('call_type','')}] {call.get('name','')}: {call.get('reasoning','')[:60]}")
+            lines.append(f"{indent}      Outcome: {call.get('outcome', '?')}")
+            for lr in call.get("learnings", [])[:2]:  # Show first 2 learnings per call
+                lines.append(f"{indent}      Learning: {lr.get('content','')[:60]}")
+                lines.append(f"{indent}        [{lr.get('type','')}] conf={lr.get('confidence',0)} concepts={lr.get('trigger_concepts',[])}")
+        if len(calls) > 3:
+            lines.append(f"{indent}    ... and {len(calls) - 3} more calls")
+    else:
+        lines.append(f"{indent}  No agent calls linked to this commit.")
+
+    return "\n".join(lines)
+
+
+def format_chains(chains: list[dict]) -> str:
+    """Format multiple chains for CLI output."""
+    if not chains:
+        return "No reasoning chains found."
+
+    lines = ["═══ Reasoning chains ═══", ""]
+    for i, chain in enumerate(chains, 1):
+        lines.append(f"Chain #{i}")
+        lines.append(format_chain(chain, indent="  "))
+        lines.append("")
+    lines.append("═══ End ═══")
+    return "\n".join(lines)

devtorch_core/reasoning_plus/learning/composer.py

@@ -0,0 +1,74 @@
+"""
+devtorch_core.reasoning_plus.learning.composer
+==============================================
+Format selected learnings into a compact prompt block.
+"""
+from __future__ import annotations
+
+from devtorch_core.reasoning_plus.learning.models import Learning
+
+
+class PromptComposer:
+    """Compose a <learnings> block for injection into a user prompt.
+
+    By default only `PUBLIC` learnings are injected. Callers can override
+    `allowed_sensitivity` to include PROTECTED or PRIVATE learnings, but the
+    default prevents accidentally leaking sensitive reasoning into prompts.
+    """
+
+    def __init__(
+        self,
+        max_lines: int = 100,
+        allowed_sensitivity: list[str] | None = None,
+    ) -> None:
+        self.max_lines = max_lines
+        self.allowed_sensitivity = set(
+            allowed_sensitivity if allowed_sensitivity is not None else ["PUBLIC"]
+        )
+
+    def _filter(self, learnings: list[Learning]) -> list[Learning]:
+        return [
+            learning
+            for learning in learnings
+            if learning.sensitivity in self.allowed_sensitivity
+        ]
+
+    def compose(self, learnings: list[Learning]) -> str:
+        learnings = self._filter(learnings)
+        if not learnings:
+            return ""
+
+        lines: list[str] = [
+            "<learnings>",
+            "The following are relevant insights from previous steps in this session.",
+            "Evaluate them against the current evidence; they may be outdated.",
+            "",
+        ]
+        for i, learning in enumerate(learnings, start=1):
+            text = learning.short_form(max_chars=240)
+            concepts_tag = ""
+            if learning.trigger_concepts:
+                concepts_tag = f"  [concepts: {', '.join(learning.trigger_concepts)}]"
+            lines.append(f"{i}. {text}{concepts_tag}")
+
+        lines.extend(["", "</learnings>"])
+
+        # Bound total lines
+        if len(lines) > self.max_lines:
+            lines = lines[: self.max_lines - 1]
+            lines.append("...</learnings>")
+
+        return "\n".join(lines)
+
+    def compose_system_prompt(self, learnings: list[Learning]) -> str:
+        """
+        Alternative: format learnings as a short system-prompt appendix.
+        Kept for callers that cannot modify the user prompt.
+        """
+        learnings = self._filter(learnings)
+        if not learnings:
+            return ""
+        parts = ["Relevant context from previous reasoning steps:"]
+        for i, learning in enumerate(learnings, start=1):
+            parts.append(f"{i}. {learning.short_form(max_chars=200)}")
+        return "\n".join(parts)