thinkstack-core 4.0.0__py3-none-any.whl

thinkstack_core/reasoning_plus/context.py

@@ -0,0 +1,262 @@
+"""
+thinkstack_core.reasoning_plus.context
+====================================
+Generic smart workspace context retrieval.
+
+Lifts and generalizes the helpers originally in benchmarks/swe_bench/context_utils.py
+so they can be used by any ThinkStack wrapper, not just the SWE-Bench runner.
+
+The functions are defensive: if `git` is missing, the directory is not a repo, or a
+subprocess call fails, they return empty results rather than raising.
+"""
+from __future__ import annotations
+
+import re
+import subprocess
+from collections import Counter
+from pathlib import Path
+from typing import Iterable
+
+# Common English words to ignore when extracting keywords from user prompt text.
+_STOPWORDS = {
+    "a", "an", "the", "and", "or", "but", "is", "are", "was", "were",
+    "be", "been", "being", "to", "of", "in", "for", "on", "at", "by",
+    "with", "from", "as", "it", "its", "this", "that", "these", "those",
+    "i", "you", "he", "she", "we", "they", "my", "your", "his", "her",
+    "our", "their", "will", "would", "could", "should", "may", "might",
+    "can", "must", "shall", "have", "has", "had", "do", "does", "did",
+    "not", "no", "yes", "if", "then", "else", "when", "where", "why",
+    "how", "what", "which", "who", "whom", "whose", "all", "any", "both",
+    "each", "every", "few", "more", "most", "other", "some", "such",
+    "only", "own", "same", "so", "than", "too", "very", "just", "also",
+    "now", "here", "there", "up", "out", "down", "over", "under", "again",
+    "further", "once", "about", "before", "after", "above", "below",
+    "between", "into", "through", "during", "before", "after", "until",
+    "while", "because", "although", "though", "unless", "since", "although",
+    "however", "therefore", "thus", "hence", "moreover", "furthermore",
+    "nevertheless", "nonetheless", "otherwise", "instead", "meanwhile",
+    "otherwise", "nevertheless", "nonetheless", "additionally", "consequently",
+}
+
+# Default file patterns to search. Overrideable by callers.
+DEFAULT_INCLUDE_GLOBS = ("*.py", "*.js", "*.ts", "*.tsx", "*.java", "*.kt", "*.go",
+                         "*.rs", "*.cpp", "*.c", "*.h", "*.hpp", "*.md", "*.yaml",
+                         "*.yml", "*.json", "*.toml")
+
+# Default exclude globs. Overrideable by callers.
+DEFAULT_EXCLUDE_GLOBS = ("*test*", "*tests*", "*__pycache__*", "*.min.js",
+                         "*.bundle.js", "node_modules/*", "dist/*", "build/*",
+                         "*.lock", "*.svg", "*.png", "*.jpg")
+
+
+def extract_keywords(text: str, max_keywords: int = 15) -> list[str]:
+    """
+    Extract technical keywords from prompt text.
+
+    Strategy:
+    - Keep code-like identifiers: snake_case, camelCase, dotted.module.paths.
+    - Keep words that contain digits or non-alphabetic characters.
+    - Drop common English stopwords and very short words.
+    - Return the most frequent technical terms, deduplicated.
+    """
+    if not text:
+        return []
+
+    tokens = re.findall(r"[a-zA-Z_][a-zA-Z0-9_]*(?:\.[a-zA-Z_][a-zA-Z0-9_]*)*", text)
+
+    candidates = []
+    for tok in tokens:
+        t = tok.lower()
+        if len(t) < 2:
+            continue
+        if t in _STOPWORDS:
+            continue
+        if (
+            "_" in t
+            or any(c.isdigit() for c in t)
+            or any(c.isupper() for c in tok)
+            or "." in t
+        ):
+            candidates.append(t)
+        elif len(t) > 4:
+            candidates.append(t)
+
+    counts = Counter(candidates)
+    ranked = [t for t, _ in counts.most_common(max_keywords * 2)]
+    seen: set[str] = set()
+    out: list[str] = []
+    for t in ranked:
+        if t not in seen:
+            seen.add(t)
+            out.append(t)
+        if len(out) >= max_keywords:
+            break
+    return out
+
+
+def search_repo(
+    repo_path: Path,
+    keywords: list[str],
+    base_commit: str | None = None,
+    top_n: int = 3,
+    max_matches_per_keyword: int = 20,
+    include_globs: Iterable[str] = (),
+    exclude_globs: Iterable[str] = (),
+) -> list[str]:
+    """
+    Search a git repo for files related to the given keywords.
+
+    Returns up to `top_n` file paths ranked by keyword hit count.
+    Returns empty list if the path is not a git repo or `git` is unavailable.
+    """
+    if not keywords or not repo_path.exists():
+        return []
+
+    include_globs = tuple(include_globs) or DEFAULT_INCLUDE_GLOBS
+    exclude_globs = tuple(exclude_globs) or DEFAULT_EXCLUDE_GLOBS
+
+    file_scores: Counter[str] = Counter()
+
+    for kw in keywords:
+        try:
+            pathspecs = ["--", *include_globs]
+            cmd = [
+                "git", "grep", "-l", "-i", "-E", re.escape(kw),
+                *pathspecs,
+            ]
+            if base_commit:
+                cmd = ["git", "grep", "-l", "-i", "-E", re.escape(kw), base_commit, "--", *include_globs]
+
+            result = subprocess.run(
+                cmd,
+                cwd=repo_path,
+                capture_output=True,
+                text=True,
+                errors="ignore",
+                timeout=30,
+            )
+            if result.returncode != 0:
+                continue
+            files = result.stdout.strip().splitlines()
+            for fp in files[:max_matches_per_keyword]:
+                if _is_excluded(fp, exclude_globs):
+                    continue
+                file_scores[fp] += 1
+        except Exception:
+            continue
+
+    ranked = [fp for fp, _ in file_scores.most_common()]
+    return ranked[:top_n]
+
+
+def read_files_at_commit(
+    repo_path: Path,
+    file_paths: list[str],
+    commit: str | None = None,
+    max_lines: int = 150,
+) -> str:
+    """
+    Read file contents at a specific commit, capped per file.
+
+    If `commit` is None, reads from the working tree.
+    """
+    chunks = []
+    for fp in file_paths:
+        try:
+            if commit:
+                result = subprocess.run(
+                    ["git", "show", f"{commit}:{fp}"],
+                    cwd=repo_path,
+                    capture_output=True,
+                    text=True,
+                    errors="ignore",
+                    timeout=10,
+                )
+                if result.returncode != 0:
+                    continue
+                text = result.stdout
+            else:
+                target = repo_path / fp
+                if not target.exists():
+                    continue
+                text = target.read_text(encoding="utf-8", errors="ignore")
+        except Exception:
+            continue
+
+        lines = text.splitlines()
+        if len(lines) > max_lines:
+            text = "\n".join(lines[:max_lines]) + f"\n\n... [{len(lines) - max_lines} lines truncated]\n"
+        chunks.append(f"### File: {fp}\n```\n{text}\n```\n")
+    return "\n".join(chunks)
+
+
+def gather_smart_context(
+    repo_path: Path | str,
+    prompt_text: str,
+    *,
+    base_commit: str | None = None,
+    top_n: int = 3,
+    max_lines: int = 150,
+    max_keywords: int = 15,
+    include_globs: Iterable[str] = (),
+    exclude_globs: Iterable[str] = (),
+) -> tuple[list[str], str]:
+    """
+    End-to-end smart context: extract keywords, search repo, read files.
+
+    Returns a tuple of (file_paths, contents_markdown).
+    """
+    repo_path = Path(repo_path)
+    if not _is_git_repo(repo_path):
+        return [], ""
+
+    keywords = extract_keywords(prompt_text, max_keywords=max_keywords)
+    if not keywords:
+        return [], ""
+
+    extra_paths = search_repo(
+        repo_path,
+        keywords,
+        base_commit=base_commit,
+        top_n=top_n,
+        include_globs=include_globs,
+        exclude_globs=exclude_globs,
+    )
+    if not extra_paths:
+        return [], ""
+
+    contents = read_files_at_commit(
+        repo_path,
+        extra_paths,
+        commit=base_commit,
+        max_lines=max_lines,
+    )
+    return extra_paths, contents
+
+
+def _is_git_repo(path: Path) -> bool:
+    """Return True if `path` is inside a git repository."""
+    try:
+        result = subprocess.run(
+            ["git", "rev-parse", "--git-dir"],
+            cwd=path,
+            capture_output=True,
+            text=True,
+            errors="ignore",
+            timeout=5,
+        )
+        return result.returncode == 0 and result.stdout.strip() != ""
+    except Exception:
+        return False
+
+
+def _is_excluded(file_path: str, exclude_globs: Iterable[str]) -> bool:
+    """Simple glob check; any glob that is a substring/endswith match."""
+    for g in exclude_globs:
+        if g.endswith("*"):
+            prefix = g[:-1]
+            if file_path.startswith(prefix) or f"/{prefix}" in file_path:
+                return True
+        if g in file_path:
+            return True
+    return False

thinkstack_core/reasoning_plus/learning/__init__.py

@@ -0,0 +1,125 @@
+"""
+thinkstack_core.reasoning_plus.learning
+=====================================
+Reasoning Plus Learning (DRPL): capture, distill, and reuse reasoning
+across LLM, tool, and memory calls.
+
+Public API:
+    ReasoningPlusLearning  — high-level facade
+    ReasoningCall          — dataclass for a recorded call
+    Learning               — dataclass for a distilled learning
+    CallRecorder           — persist calls to .GCC/
+    LearningStore          — persist learnings to .GCC/
+    LearningExtractor      — derive learnings from calls
+    RelevanceEngine        — rank learnings by relevance
+    PromptComposer         — format learnings for prompts
+    workspace_state_hash   — compute a state hash for staleness
+    cross_project_suggest  — suggest learnings from sibling governed projects
+    register_project       — register a project for cross-project learning
+    list_projects          — list registered governed projects
+"""
+from __future__ import annotations
+
+from thinkstack_core.reasoning_plus.learning.api import ReasoningPlusLearning
+from thinkstack_core.reasoning_plus.learning.composer import PromptComposer
+from thinkstack_core.reasoning_plus.learning.cross_project import (
+    CrossProjectRegistry,
+    GovernedProject,
+    cross_project_suggest,
+    list_projects,
+    register_project,
+)
+from thinkstack_core.reasoning_plus.learning.extractor import LearningExtractor
+from thinkstack_core.reasoning_plus.learning.models import Learning, ReasoningCall
+from thinkstack_core.reasoning_plus.learning.recorder import CallRecorder
+from thinkstack_core.reasoning_plus.learning.relevance import RelevanceEngine
+from thinkstack_core.reasoning_plus.learning.state import workspace_state_hash
+from thinkstack_core.reasoning_plus.learning.store import LearningStore
+from thinkstack_core.reasoning_plus.learning.analytics import (
+    concept_stats,
+    danger_zones,
+    concept_health_section,
+)
+from thinkstack_core.reasoning_plus.learning.theta_learning_bridge import (
+    sync_learnings_to_theta,
+)
+from thinkstack_core.reasoning_plus.learning.chain import (
+    build_chain,
+    format_chains,
+)
+
+from thinkstack_core.reasoning_plus.learning.context_collector import (
+    ExtractionContext,
+    collect_extraction_context,
+)
+from thinkstack_core.reasoning_plus.learning.conflicts import (
+    Conflict,
+    ConflictDetector,
+    ConflictResolution,
+    resolve_conflict,
+)
+from thinkstack_core.reasoning_plus.learning.evolution import (
+    apply_confidence_decay,
+    goal_changed_since,
+)
+from thinkstack_core.reasoning_plus.learning.pii import (
+    filter_learnings_for_sync,
+    sanitize_for_extraction,
+    scan_for_secrets,
+)
+from thinkstack_core.reasoning_plus.learning.org_store import (
+    OrgLearningAggregate,
+    aggregate_org_learnings,
+)
+from thinkstack_core.reasoning_plus.learning.filter_five_layer import (
+    five_layer_filter,
+)
+from thinkstack_core.reasoning_plus.learning.deny_list import (
+    DenyList,
+    load_deny_list,
+    save_deny_list,
+)
+from thinkstack_core.reasoning_plus.learning.promotion import (
+    promote_learning,
+)
+
+__all__ = [
+    "ReasoningPlusLearning",
+    "ReasoningCall",
+    "Learning",
+    "CallRecorder",
+    "LearningStore",
+    "LearningExtractor",
+    "RelevanceEngine",
+    "PromptComposer",
+    "workspace_state_hash",
+    "cross_project_suggest",
+    "register_project",
+    "list_projects",
+    "CrossProjectRegistry",
+    "GovernedProject",
+    "concept_stats",
+    "danger_zones",
+    "concept_health_section",
+    "sync_learnings_to_theta",
+    "build_chain",
+    "format_chains",
+    "ExtractionContext",
+    "collect_extraction_context",
+    "Conflict",
+    "ConflictDetector",
+    "ConflictResolution",
+    "resolve_conflict",
+    "apply_confidence_decay",
+    "goal_changed_since",
+    "filter_learnings_for_sync",
+    "sanitize_for_extraction",
+    "scan_for_secrets",
+    "OrgLearningAggregate",
+    "aggregate_org_learnings",
+    "five_layer_filter",
+    "DenyList",
+    "load_deny_list",
+    "save_deny_list",
+    "promote_learning",
+]

thinkstack_core/reasoning_plus/learning/analytics.py

@@ -0,0 +1,141 @@
+"""
+thinkstack_core.reasoning_plus.learning.analytics
+================================================
+Per-concept learning analytics: stats, danger-zone detection, health dashboard.
+"""
+from __future__ import annotations
+
+import logging
+from collections import Counter
+from typing import Any
+
+from thinkstack_core.reasoning_plus.learning.store import LearningStore
+
+logger = logging.getLogger("thinkstack.reasoning_plus.learning")
+
+
+def concept_stats(store: LearningStore) -> dict[str, dict[str, Any]]:
+    """Return per-concept statistics aggregated from all stored learnings.
+
+    Returns a dict keyed by lowercased concept name with:
+        - total: total learnings tagged with this concept
+        - active: learnings with validity="active"
+        - stale: learnings with validity="stale"
+        - deprecated: learnings with validity="deprecated"
+        - by_type: Counter of learning types (insight, correction, pattern, avoid, confirm)
+        - by_outcome: Counter of outcomes among source calls (success, failure, partial, unknown)
+        - avg_confidence: mean confidence across all learnings for this concept
+        - success_rate: fraction of (non-unknown) outcomes that are "success"
+    """
+    all_learnings = store.list()
+    concept_map: dict[str, dict[str, Any]] = {}
+
+    for learning in all_learnings:
+        concepts = learning.trigger_concepts
+        if not concepts:
+            concepts = ["__untyped__"]
+        for concept in concepts:
+            cname = concept.lower()
+            if cname not in concept_map:
+                concept_map[cname] = {
+                    "total": 0,
+                    "active": 0,
+                    "stale": 0,
+                    "deprecated": 0,
+                    "by_type": Counter(),
+                    "by_outcome": Counter(),
+                    "confidences": [],
+                }
+            concept_map[cname]["total"] += 1
+            concept_map[cname][learning.validity] += 1
+            concept_map[cname]["by_type"][learning.type] += 1
+            concept_map[cname]["confidences"].append(learning.confidence)
+
+    for cname, data in concept_map.items():
+        confs = data.pop("confidences")
+        data["avg_confidence"] = sum(confs) / len(confs) if confs else 0.0
+        data["success_rate"] = _compute_success_rate(store, cname)
+
+    return dict(sorted(concept_map.items()))
+
+
+def _compute_success_rate(store: LearningStore, concept: str) -> float:
+    """Query call records for the given concept and compute success fraction."""
+    from thinkstack_core.reasoning_plus.learning.models import ReasoningCall
+    from thinkstack_core.reasoning_plus.learning.recorder import CallRecorder
+
+    recorder = CallRecorder(store._gcc_dir)
+    calls = recorder.list()
+    matched = [c for c in calls if concept in {co.lower() for co in c.concepts}]
+    non_unknown = [c for c in matched if c.outcome not in ("unknown", "")]
+    if not non_unknown:
+        return 0.0
+    successes = sum(1 for c in non_unknown if c.outcome == "success")
+    return successes / len(non_unknown)
+
+
+def danger_zones(
+    store: LearningStore,
+    threshold: float = 0.4,
+    min_samples: int = 3,
+) -> list[dict[str, Any]]:
+    """Return concepts with success rate below *threshold*.
+
+    Each entry contains:
+        - concept: concept name
+        - success_rate: computed success fraction
+        - total_learnings: total learnings for this concept
+        - avg_confidence: mean learning confidence
+    Only concepts with at least *min_samples* total learnings are considered.
+    """
+    stats = concept_stats(store)
+    zones: list[dict[str, Any]] = []
+    for cname, data in stats.items():
+        if cname == "__untyped__":
+            continue
+        if data["total"] < min_samples:
+            continue
+        if data["success_rate"] < threshold:
+            zones.append({
+                "concept": cname,
+                "success_rate": data["success_rate"],
+                "total_learnings": data["total"],
+                "avg_confidence": data["avg_confidence"],
+            })
+    zones.sort(key=lambda x: x["success_rate"])
+    return zones
+
+
+def concept_health_section(store: LearningStore, threshold: float = 0.4) -> str:
+    """Return a formatted concept health block for the ``thinkstack today`` dashboard."""
+    stats = concept_stats(store)
+    if not stats:
+        return ""
+
+    lines = ["─── Concept health ───"]
+
+    zones = danger_zones(store, threshold=threshold)
+    if zones:
+        lines.append(f"  \u26a0 DANGER ZONES (success rate < {threshold:.0%}):")
+        for z in zones[:5]:
+            lines.append(
+                f"    {z['concept']}: success {z['success_rate']:.0%}, "
+                f"{z['total_learnings']} learning(s), "
+                f"avg confidence {z['avg_confidence']:.2f}"
+            )
+
+    healthy = [(c, d) for c, d in stats.items()
+               if c != "__untyped__" and d["success_rate"] >= threshold]
+    if healthy:
+        lines.append(f"  Healthy concepts ({len(healthy)}):")
+        for c, d in sorted(healthy, key=lambda x: x[1]["success_rate"], reverse=True)[:8]:
+            lines.append(
+                f"    {c}: {d['active']} active, "
+                f"{d['success_rate']:.0%} success"
+            )
+
+    untyped = stats.get("__untyped__")
+    if untyped and untyped["total"] > 0:
+        lines.append(f"  Untagged learnings: {untyped['total']}")
+
+    return "\n".join(lines)