thinkstack-core 4.0.0__py3-none-any.whl

thinkstack_core/reasoning_plus/learning/recorder.py

@@ -0,0 +1,81 @@
+"""
+thinkstack_core.reasoning_plus.learning.recorder
+==============================================
+Record reasoning calls (LLM, tool, memory, action) to .GCC/.
+"""
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+from typing import Any
+
+from thinkstack_core.reasoning_plus.learning.models import ReasoningCall
+
+logger = logging.getLogger("thinkstack.reasoning_plus.learning")
+
+
+class CallRecorder:
+    """Persist ReasoningCall records under .GCC/reasoning_learnings/calls/."""
+
+    def __init__(self, gcc_dir: Path | str) -> None:
+        self._gcc_dir = Path(gcc_dir)
+        self._calls_dir = self._gcc_dir / "reasoning_learnings" / "calls"
+
+    def record(self, call: ReasoningCall) -> Path:
+        """Write a call to disk and return the path."""
+        self._calls_dir.mkdir(parents=True, exist_ok=True)
+        path = self._calls_dir / f"{call.id}.json"
+        try:
+            _atomic_json_write(path, call.to_dict())
+        except Exception as exc:
+            logger.warning("thinkstack: failed to record reasoning call — %s", exc)
+        return path
+
+    def get(self, call_id: str) -> ReasoningCall | None:
+        path = self._calls_dir / f"{call_id}.json"
+        if not path.exists():
+            return None
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+            return ReasoningCall.from_dict(data)
+        except Exception as exc:
+            logger.warning("thinkstack: failed to read reasoning call %s — %s", call_id, exc)
+            return None
+
+    def list(self, session_id: str | None = None, call_type: str | None = None) -> list[ReasoningCall]:
+        """List recorded calls, optionally filtered by session or type."""
+        out: list[ReasoningCall] = []
+        if not self._calls_dir.exists():
+            return out
+        for path in sorted(self._calls_dir.glob("*.json")):
+            try:
+                data = json.loads(path.read_text(encoding="utf-8"))
+                call = ReasoningCall.from_dict(data)
+                if session_id and call.session_id != session_id:
+                    continue
+                if call_type and call.call_type != call_type:
+                    continue
+                out.append(call)
+            except Exception as exc:
+                logger.debug("thinkstack: skipping malformed call record %s — %s", path, exc)
+        return out
+
+    def count(self) -> int:
+        if not self._calls_dir.exists():
+            return 0
+        return len(list(self._calls_dir.glob("*.json")))
+
+
+def _atomic_json_write(path: Path, data: dict[str, Any]) -> None:
+    tmp = path.with_suffix(path.suffix + ".tmp")
+    try:
+        with open(tmp, "w", encoding="utf-8") as f:
+            json.dump(data, f, indent=2)
+        tmp.replace(path)
+    finally:
+        if tmp.exists():
+            try:
+                tmp.unlink()
+            except Exception:
+                pass

thinkstack_core/reasoning_plus/learning/relevance.py

@@ -0,0 +1,122 @@
+"""
+thinkstack_core.reasoning_plus.learning.relevance
+=================================================
+Rank learnings by relevance to the current context and state.
+
+Scoring factors (two-dimensional):
+1. Semantic similarity between context and learning content (embedding)
+2. Concept overlap between context keywords and learning trigger_concepts
+3. Confidence
+4. State freshness (matching state_hash boosts score)
+"""
+from __future__ import annotations
+
+import hashlib
+import time
+from typing import Any
+
+from thinkstack_core.reasoning_plus.context import extract_keywords
+from thinkstack_core.reasoning_plus.learning.embeddings import (
+    EmbeddingBackend,
+    KeywordEmbeddingBackend,
+)
+from thinkstack_core.reasoning_plus.learning.models import Learning
+
+
+class RelevanceEngine:
+    """Score and rank learnings for a given context using an embedding backend."""
+
+    def __init__(self, backend: EmbeddingBackend | None = None, cache_ttl: float = 0) -> None:
+        self._backend = backend or KeywordEmbeddingBackend()
+        self._cache_ttl = cache_ttl
+        self._cache: dict[str, tuple[list[Learning], float]] = {}
+
+    def rank(
+        self,
+        context: str,
+        learnings: list[Learning],
+        current_state_hash: str | None = None,
+        top_n: int = 3,
+    ) -> list[Learning]:
+        """
+        Return the top-N active learnings ranked by relevance.
+
+        Scoring factors:
+        - semantic similarity between context and learning content
+        - concept overlap between context keywords and learning trigger_concepts
+        - confidence
+        - state freshness (matching state_hash boosts score)
+
+        Results are cached in memory when ``cache_ttl`` > 0 so repeated
+        similar prompts in the same session avoid recomputing embeddings.
+        """
+        cache_key = self._cache_key(context, learnings, current_state_hash, top_n)
+        if self._cache_ttl and cache_key:
+            cached = self._cache.get(cache_key)
+            if cached and (time.monotonic() - cached[1]) < self._cache_ttl:
+                return cached[0]
+
+        context_embedding = self._backend.encode(context)
+        context_keywords = set(extract_keywords(context, max_keywords=30))
+        scored: list[tuple[float, Learning]] = []
+
+        for learning in learnings:
+            if learning.validity != "active":
+                continue
+            score = self._score(
+                learning, context_embedding, current_state_hash, context_keywords,
+            )
+            scored.append((score, learning))
+
+        scored.sort(key=lambda x: x[0], reverse=True)
+        result = [learning for _, learning in scored[:top_n]]
+
+        if self._cache_ttl and cache_key:
+            self._cache[cache_key] = (result, time.monotonic())
+        return result
+
+    def _cache_key(
+        self,
+        context: str,
+        learnings: list[Learning],
+        current_state_hash: str | None,
+        top_n: int,
+    ) -> str:
+        """Stable key for the relevance cache."""
+        context_hash = hashlib.sha256(context.encode("utf-8")).hexdigest()[:16]
+        learning_ids = sorted({l.id for l in learnings})
+        learning_hash = hashlib.sha256(
+            "|".join(learning_ids).encode("utf-8")
+        ).hexdigest()[:16]
+        return f"{context_hash}:{current_state_hash or ''}:{learning_hash}:{top_n}"
+
+    def _score(
+        self,
+        learning: Learning,
+        context_embedding: Any,
+        current_state_hash: str | None,
+        context_keywords: set[str] | None = None,
+    ) -> float:
+        learning_embedding = self._backend.encode(learning.content)
+        semantic_score = self._backend.similarity(context_embedding, learning_embedding)
+
+        confidence_weight = learning.confidence
+
+        state_bonus = 0.0
+        if current_state_hash and learning.state_hash == current_state_hash:
+            state_bonus = 0.2
+
+        # Concept overlap: how much the learning's trigger_concepts overlap
+        # with keywords extracted from the current prompt context.
+        concept_bonus = 0.0
+        if context_keywords and learning.trigger_concepts:
+            learning_concepts = {c.lower() for c in learning.trigger_concepts}
+            overlap = len(context_keywords & learning_concepts)
+            if overlap > 0:
+                concept_bonus = min(0.3, overlap * 0.1)
+
+        return semantic_score + confidence_weight * 0.5 + state_bonus + concept_bonus
+
+    def invalidate_cache(self) -> None:
+        """Clear the relevance cache."""
+        self._cache.clear()

thinkstack_core/reasoning_plus/learning/state.py

@@ -0,0 +1,86 @@
+"""
+thinkstack_core.reasoning_plus.learning.state
+===========================================
+Workspace state hashing and staleness detection for learnings.
+"""
+from __future__ import annotations
+
+import logging
+import subprocess
+from pathlib import Path
+from typing import Iterable
+
+from thinkstack_core.reasoning_plus.learning.models import hash_state
+
+logger = logging.getLogger("thinkstack.reasoning_plus.learning")
+
+
+def workspace_state_hash(repo_path: Path | str, file_paths: Iterable[str] | None = None) -> str:
+    """
+    Return a short hash representing the current state of the workspace.
+
+    Strategy:
+    - If inside a git repo, include the current HEAD commit hash.
+    - For any provided file_paths, include the current file content hash.
+    - If not a git repo and no file paths, fall back to a hash of the directory path.
+    """
+    repo_path = Path(repo_path)
+    tokens: list[str] = []
+
+    head = _git_head(repo_path)
+    if head:
+        tokens.append(f"head:{head}")
+
+    for fp in file_paths or []:
+        content_hash = _file_content_hash(repo_path / fp)
+        if content_hash:
+            tokens.append(f"{fp}:{content_hash}")
+
+    if not tokens:
+        tokens.append(str(repo_path.resolve()))
+
+    return hash_state(*tokens)
+
+
+def _git_head(repo_path: Path) -> str | None:
+    try:
+        result = subprocess.run(
+            ["git", "rev-parse", "HEAD"],
+            cwd=repo_path,
+            capture_output=True,
+            text=True,
+            errors="ignore",
+            timeout=5,
+        )
+        if result.returncode == 0:
+            return result.stdout.strip()
+    except Exception as exc:
+        logger.debug("thinkstack: could not read git HEAD — %s", exc)
+    return None
+
+
+def _file_content_hash(path: Path) -> str | None:
+    if not path.exists():
+        return None
+    try:
+        return hash_state(path.read_text(encoding="utf-8", errors="ignore"))
+    except Exception as exc:
+        logger.debug("thinkstack: could not hash %s — %s", path, exc)
+        return None
+
+
+def affected_by_state_change(repo_path: Path | str, learning_file_paths: Iterable[str], changed_paths: Iterable[str]) -> bool:
+    """
+    Return True if a learning should be re-evaluated because the workspace changed.
+
+    A learning is affected when any of the paths it references overlap with a changed path.
+    """
+    changed = set(changed_paths)
+    for fp in learning_file_paths:
+        if fp in changed:
+            return True
+        # Also invalidate if a parent directory was changed
+        for changed_path in changed:
+            if changed_path.startswith(fp) or fp.startswith(changed_path):
+                return True
+    return False

thinkstack_core/reasoning_plus/learning/store.py

@@ -0,0 +1,178 @@
+"""
+thinkstack_core.reasoning_plus.learning.store
+===========================================
+Persistence layer for distilled learnings.
+"""
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+from typing import Any
+
+from thinkstack_core.reasoning_plus.context import extract_keywords
+from thinkstack_core.reasoning_plus.learning.models import Learning
+
+logger = logging.getLogger("thinkstack.reasoning_plus.learning")
+
+
+class LearningStore:
+    """Persist Learning records under .GCC/reasoning_learnings/learnings/."""
+
+    def __init__(self, gcc_dir: Path | str) -> None:
+        self._gcc_dir = Path(gcc_dir)
+        self._learnings_dir = self._gcc_dir / "reasoning_learnings" / "learnings"
+
+    def save(self, learning: Learning, deduplicate: bool = True) -> Path:
+        self._learnings_dir.mkdir(parents=True, exist_ok=True)
+        if deduplicate and learning.validity == "active":
+            duplicate = self._find_duplicate(learning)
+            if duplicate:
+                merged = self._merge(duplicate, learning)
+                path = self._learnings_dir / f"{merged.id}.json"
+                try:
+                    _atomic_json_write(path, merged.to_dict())
+                except Exception as exc:
+                    logger.warning("thinkstack: failed to save merged learning — %s", exc)
+                return path
+        path = self._learnings_dir / f"{learning.id}.json"
+        try:
+            _atomic_json_write(path, learning.to_dict())
+        except Exception as exc:
+            logger.warning("thinkstack: failed to save learning — %s", exc)
+        return path
+
+    def get(self, learning_id: str) -> Learning | None:
+        path = self._learnings_dir / f"{learning_id}.json"
+        if not path.exists():
+            return None
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+            return Learning.from_dict(data)
+        except Exception as exc:
+            logger.warning("thinkstack: failed to read learning %s — %s", learning_id, exc)
+            return None
+
+    def list(
+        self,
+        validity: str | None = None,
+        concept: str | None = None,
+        scope: str | None = None,
+    ) -> list[Learning]:
+        out: list[Learning] = []
+        if not self._learnings_dir.exists():
+            return out
+        for path in sorted(self._learnings_dir.glob("*.json")):
+            try:
+                data = json.loads(path.read_text(encoding="utf-8"))
+                learning = Learning.from_dict(data)
+                if validity and learning.validity != validity:
+                    continue
+                if concept and concept.lower() not in {c.lower() for c in learning.trigger_concepts}:
+                    continue
+                if scope and learning.scope != scope:
+                    continue
+                out.append(learning)
+            except Exception as exc:
+                logger.debug("thinkstack: skipping malformed learning %s — %s", path, exc)
+        return out
+
+    def invalidate(self, learning_id: str, reason: str = "stale") -> Learning | None:
+        learning = self.get(learning_id)
+        if not learning:
+            return None
+        learning.validity = "stale" if reason == "stale" else "deprecated"
+        learning.updated_at = _now_iso()
+        self.save(learning)
+        return learning
+
+    def count(self) -> int:
+        if not self._learnings_dir.exists():
+            return 0
+        return len(list(self._learnings_dir.glob("*.json")))
+
+    # ------------------------------------------------------------------
+    # Deduplication
+    # ------------------------------------------------------------------
+
+    def _find_duplicate(self, learning: Learning, threshold: float = 0.8) -> Learning | None:
+        """Return the most similar *other* active learning if it reaches the threshold."""
+        best: Learning | None = None
+        best_score = threshold
+        for existing in self.list(validity="active"):
+            if existing.id == learning.id:
+                continue
+            score = self._similarity(existing, learning)
+            if score >= best_score:
+                best_score = score
+                best = existing
+        return best
+
+    def _similarity(self, a: Learning, b: Learning) -> float:
+        """Combined concept and content similarity in [0, 1].
+
+        Concept overlap is weighted more heavily because two learnings that share
+        the same trigger concepts are likely talking about the same situation.
+        """
+        a_concepts = {c.lower() for c in a.trigger_concepts}
+        b_concepts = {c.lower() for c in b.trigger_concepts}
+        concept_union = len(a_concepts | b_concepts)
+        concept_score = len(a_concepts & b_concepts) / concept_union if concept_union else 0.0
+
+        a_keywords = set(extract_keywords(a.content, max_keywords=15))
+        b_keywords = set(extract_keywords(b.content, max_keywords=15))
+        content_union = len(a_keywords | b_keywords)
+        content_score = len(a_keywords & b_keywords) / content_union if content_union else 0.0
+
+        return 0.8 * concept_score + 0.2 * content_score
+
+    def _merge(self, existing: Learning, new: Learning) -> Learning:
+        """Merge *new* into *existing*, keeping the existing stable ID."""
+        merged = Learning.from_dict(existing.to_dict())
+        merged.content = new.content
+        merged.trigger_concepts = sorted(
+            set(existing.trigger_concepts) | set(new.trigger_concepts)
+        )
+        merged.confidence = max(existing.confidence, new.confidence)
+        merged.source_reasoning_ids = list(
+            set(existing.source_reasoning_ids) | set(new.source_reasoning_ids)
+        )
+        merged.validity = _more_restrictive_validity(existing.validity, new.validity)
+        # Preserve evolution fields (S19) — union lists, prefer non-empty values.
+        merged.supersedes = sorted(set(existing.supersedes) | set(new.supersedes))
+        merged.superseded_by = new.superseded_by or existing.superseded_by
+        merged.evolution_chain = new.evolution_chain or existing.evolution_chain
+        merged.scope = _more_restrictive_scope(existing.scope, new.scope)
+        merged.updated_at = _now_iso()
+        return merged
+
+
+def _atomic_json_write(path: Path, data: dict[str, Any]) -> None:
+    tmp = path.with_suffix(path.suffix + ".tmp")
+    try:
+        with open(tmp, "w", encoding="utf-8") as f:
+            json.dump(data, f, indent=2)
+        tmp.replace(path)
+    finally:
+        if tmp.exists():
+            try:
+                tmp.unlink()
+            except Exception:
+                pass
+
+
+def _more_restrictive_validity(a: str, b: str) -> str:
+    """Return the more restrictive validity status."""
+    order = {"deprecated": 2, "stale": 1, "active": 0}
+    return a if order.get(a, 0) >= order.get(b, 0) else b
+
+
+def _more_restrictive_scope(a: str, b: str) -> str:
+    """Return the more restrictive (narrower visibility) scope."""
+    order = {"branch": 2, "project": 1, "org": 0}
+    return a if order.get(a, 1) >= order.get(b, 1) else b
+
+
+def _now_iso() -> str:
+    from datetime import datetime, timezone
+    return datetime.now(timezone.utc).isoformat()

thinkstack_core/reasoning_plus/learning/theta_learning_bridge.py

@@ -0,0 +1,94 @@
+"""
+thinkstack_core.reasoning_plus.learning.theta_learning_bridge
+============================================================
+Bridge between DRPL learning outcomes and the coordination vector Θ.
+
+Feeds per-concept success rates into ThetaStore.ripple() so that
+learning outcomes influence the shared coordination vector.
+"""
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from typing import Any
+
+from thinkstack_core.reasoning_plus.learning.store import LearningStore
+from thinkstack_core.theta import make_theta_store
+
+logger = logging.getLogger("thinkstack.reasoning_plus.learning")
+
+
+def sync_learnings_to_theta(
+    gcc_dir: Path | str,
+    min_samples: int = 3,
+    disclosure: str = "PUBLIC",
+) -> dict[str, Any]:
+    """Compute per-concept success rates from learnings and feed them into Θ.
+
+    For each concept with at least *min_samples* learnings, creates a
+    sensitivity event with confidence = success_rate and feeds it through
+    ThetaStore.ripple(). This means concepts with high success rates
+    produce high-confidence events, and failing concepts produce low-confidence
+    signals in the coordination vector.
+
+    Args:
+        gcc_dir: Path to the local .GCC/ directory.
+        min_samples: Minimum learnings per concept to include.
+        disclosure: Disclosure level for generated events.
+
+    Returns:
+        A dict with keys:
+            - concepts_synced: number of concepts fed into theta
+            - events_generated: total events generated
+            - theta_concepts_after: concept count in theta after sync
+    """
+    from thinkstack_core.reasoning_plus.learning.analytics import concept_stats
+
+    store = LearningStore(gcc_dir)
+    stats = concept_stats(store)
+
+    events: list[dict] = []
+    concepts_synced = 0
+
+    for cname, data in stats.items():
+        if cname == "__untyped__":
+            continue
+        if data["total"] < min_samples:
+            continue
+        events.append({
+            "target_concept": cname,
+            "confidence": data["success_rate"],
+            "disclosure_level": disclosure,
+            "created_at": _now_iso(),
+            "source": "learning-analytics",
+        })
+        concepts_synced += 1
+
+    if not events:
+        return {
+            "concepts_synced": 0,
+            "events_generated": 0,
+            "theta_concepts_after": len(
+                make_theta_store(Path(gcc_dir)).load().get("coordination_vector", {})
+            ),
+        }
+
+    theta_store = make_theta_store(Path(gcc_dir))
+    theta_store.ripple(events)
+    updated = theta_store.load()
+    theta_count = len(updated.get("coordination_vector", {}))
+
+    logger.info(
+        "thinkstack: synced %d concept(s) into Θ (%d events) — theta now has %d concept(s)",
+        concepts_synced, len(events), theta_count,
+    )
+    return {
+        "concepts_synced": concepts_synced,
+        "events_generated": len(events),
+        "theta_concepts_after": theta_count,
+    }
+
+
+def _now_iso() -> str:
+    from datetime import datetime, timezone
+    return datetime.now(timezone.utc).isoformat()

thinkstack_core/reasoning_plus/prompt.py

@@ -0,0 +1,90 @@
+"""
+thinkstack_core.reasoning_plus.prompt
+===================================
+Reasoning Plus prompt templates.
+
+The default template is task-agnostic: it asks the model to reason step-by-step
+inside <thinking> tags before producing the final answer. The SWE-Bench runner
+uses a patch-specific override via the `task_prompt` parameter.
+"""
+from __future__ import annotations
+
+DEFAULT_REASONING_DIRECTIVE = """You are a careful reasoning assistant.
+
+Before giving your final answer, think step by step inside <thinking>...</thinking> tags.
+Explain your reasoning, the relevant facts, and any trade-offs you considered.
+Then provide the final answer outside the <thinking> block.
+
+You MUST output the <thinking> block before the final answer.
+"""
+
+
+PATCH_REASONING_DIRECTIVE = """You are an expert software engineer fixing a GitHub issue in a Python repository.
+
+Your job is to produce a single, correct patch in unified diff format that the repository maintainers could apply with `git apply`.
+
+Rules:
+- Edit only the files needed to fix the issue.
+- Use exact `git diff` style hunks: `--- a/<path>` and `+++ b/<path>` headers, `@@ -start,len +start,len @@` context lines.
+- Context lines must match the original file exactly (indentation, spacing, content).
+- Do not add line numbers, explanations, or markdown inside the patch.
+- Do not output any text after `</patch>`.
+
+Before writing the patch, you MUST think step by step inside <thinking>...</thinking> tags.
+The thinking block should explain your analysis of the issue, the files that need to change, and the fix strategy.
+Then write the patch between <patch>...</patch> tags.
+
+You MUST output the <thinking> block before the <patch> block.
+
+Example of the required output format:
+
+<thinking>
+1. Root cause: the function X does not handle Y because Z.
+2. Files to edit: src/example.py
+3. Fix strategy: add a guard before the call to X.
+</thinking>
+
+<patch>
+--- a/src/example.py
++++ b/src/example.py
+@@ -10,7 +10,7 @@
+ def old_function():
+     x = 1
+-    return x
++    return x + 1
+
+ def other_function():
+     pass
+</patch>
+"""
+
+
+def build_reasoning_plus_system_prompt(
+    base_prompt: str,
+    smart_context: str = "",
+    *,
+    require_thinking: bool = True,
+    task_prompt: str = "",
+) -> str:
+    """
+    Build a Reasoning Plus system prompt.
+
+    Args:
+        base_prompt: the caller's existing system prompt.
+        smart_context: optional markdown snippet of related files.
+        require_thinking: whether to append the thinking directive.
+        task_prompt: optional task-specific directive (e.g. the SWE-Bench patch prompt).
+                     If empty, the generic task-agnostic directive is used.
+    """
+    parts = [base_prompt.strip()]
+
+    if smart_context:
+        parts.append("[ThinkStack smart context]")
+        parts.append(smart_context.strip())
+
+    if require_thinking:
+        directive = task_prompt.strip() if task_prompt else DEFAULT_REASONING_DIRECTIVE.strip()
+        parts.append("[ThinkStack reasoning directive]")
+        parts.append(directive)
+
+    return "\n\n".join(parts)