thinkstack-core 4.0.0__py3-none-any.whl

thinkstack_core/reasoning_plus/learning/models.py

@@ -0,0 +1,149 @@
+"""
+thinkstack_core.reasoning_plus.learning.models
+===========================================
+Data models for Reasoning Plus Learning (DRPL).
+
+A ReasoningCall is a single recorded step (LLM, tool, memory, or action).
+A Learning is a distilled, reusable insight derived from one or more calls.
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import uuid
+from dataclasses import asdict, dataclass, field, fields
+from datetime import datetime, timezone
+from typing import Any
+
+
+VALID_CALL_TYPES = ("llm", "tool", "memory", "action")
+VALID_OUTCOMES = ("success", "failure", "partial", "unknown")
+VALID_LEARNING_TYPES = ("insight", "correction", "pattern", "avoid", "confirm", "prescription", "canon")
+VALID_VALIDITIES = ("active", "stale", "deprecated")
+VALID_DISCLOSURE_LEVELS = ("PUBLIC", "PROTECTED", "PRIVATE")
+VALID_SCOPES = ("branch", "project", "org")
+
+
+def _now_iso() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def _make_id(prefix: str = "drpl") -> str:
+    return f"{prefix}_{uuid.uuid4().hex[:16]}"
+
+
+@dataclass
+class ReasoningCall:
+    """One recorded reasoning step."""
+
+    reasoning: str
+    call_type: str = "llm"
+    name: str = ""
+    input: str = ""
+    output: str = ""
+    outcome: str = "unknown"
+    state_hash: str = ""
+    concepts: list[str] = field(default_factory=list)
+    confidence: float = 1.0
+    sensitivity: str = "PUBLIC"
+    scope: str = "project"
+    session_id: str = ""
+    timestamp: str = field(default_factory=_now_iso)
+    id: str = field(default_factory=lambda: _make_id("call"))
+    meta: dict[str, Any] = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        if self.call_type not in VALID_CALL_TYPES:
+            raise ValueError(f"Invalid call_type: {self.call_type}")
+        if self.outcome not in VALID_OUTCOMES:
+            raise ValueError(f"Invalid outcome: {self.outcome}")
+        if self.sensitivity not in VALID_DISCLOSURE_LEVELS:
+            raise ValueError(f"Invalid sensitivity: {self.sensitivity}")
+        if self.scope not in VALID_SCOPES:
+            raise ValueError(f"Invalid scope: {self.scope}")
+        self.confidence = max(0.0, min(1.0, float(self.confidence)))
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "ReasoningCall":
+        valid = {f.name for f in fields(cls)}
+        return cls(**{k: v for k, v in data.items() if k in valid})
+
+
+@dataclass
+class Learning:
+    """A distilled, reusable learning from one or more reasoning calls."""
+
+    content: str
+    type: str = "insight"
+    trigger_concepts: list[str] = field(default_factory=list)
+    state_hash: str = ""
+    validity: str = "active"
+    confidence: float = 1.0
+    source_reasoning_ids: list[str] = field(default_factory=list)
+    sensitivity: str = "PUBLIC"
+    scope: str = "project"
+    supersedes: list[str] = field(default_factory=list)
+    superseded_by: str | None = None
+    evolution_chain: str | None = None
+    promoted_by: str | None = None
+    promoted_at: str | None = None
+    source_project_root: str | None = None
+    created_at: str = field(default_factory=_now_iso)
+    updated_at: str = field(default_factory=_now_iso)
+    id: str = field(default_factory=lambda: _make_id("learn"))
+    meta: dict[str, Any] = field(default_factory=dict)
+
+    def promote(
+        self,
+        to_scope: str = "org",
+        promoted_by: str | None = None,
+        boost_confidence: bool = True,
+    ) -> None:
+        """Promote this learning to a broader scope.
+
+        Sets scope to ``to_scope``, sets type to ``"canon"``, boosts confidence
+        to 1.0 (when ``boost_confidence`` is True), and records promotion
+        metadata.
+        """
+        if boost_confidence:
+            self.confidence = 1.0
+        self.scope = to_scope
+        self.type = "canon"
+        self.promoted_by = promoted_by
+        self.promoted_at = _now_iso()
+        self.updated_at = _now_iso()
+
+    def __post_init__(self) -> None:
+        if self.type not in VALID_LEARNING_TYPES:
+            raise ValueError(f"Invalid learning type: {self.type}")
+        if self.validity not in VALID_VALIDITIES:
+            raise ValueError(f"Invalid validity: {self.validity}")
+        if self.sensitivity not in VALID_DISCLOSURE_LEVELS:
+            raise ValueError(f"Invalid sensitivity: {self.sensitivity}")
+        if self.scope not in VALID_SCOPES:
+            raise ValueError(f"Invalid scope: {self.scope}")
+        self.confidence = max(0.0, min(1.0, float(self.confidence)))
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "Learning":
+        valid = {f.name for f in fields(cls)}
+        return cls(**{k: v for k, v in data.items() if k in valid})
+
+    def short_form(self, max_chars: int = 200) -> str:
+        """Compact, prompt-ready representation."""
+        text = self.content.strip().replace("\n", " ")
+        if len(text) > max_chars:
+            text = text[: max_chars - 3].rstrip() + "..."
+        return text
+
+
+def hash_state(*items: str) -> str:
+    """Stable hash for a set of state tokens."""
+    joined = "|".join(sorted(items))
+    return hashlib.sha256(joined.encode("utf-8")).hexdigest()[:16]

thinkstack_core/reasoning_plus/learning/org_store.py

@@ -0,0 +1,156 @@
+"""
+thinkstack_core.reasoning_plus.learning.org_store
+=================================================
+Org-level learning store aggregation.
+
+Merges learnings from multiple governed projects with conflict-aware
+aggregation: max confidence and most-restrictive validity. Used by the
+five-layer filter and learning_search to produce an org-wide view of
+reusable insights.
+"""
+from __future__ import annotations
+
+import json
+import logging
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from thinkstack_core.reasoning_plus.learning.models import Learning
+
+logger = logging.getLogger("thinkstack.reasoning_plus.learning")
+
+DEFAULT_CROSS_PROJECT_THRESHOLD = 0.3
+
+
+@dataclass
+class OrgLearningAggregate:
+    """A single row in the aggregated org-learning view."""
+
+    content: str
+    type: str = "insight"
+    trigger_concepts: list[str] = field(default_factory=list)
+    validity: str = "active"
+    confidence: float = 1.0
+    scope: str = "org"
+    sensitivity: str = "PUBLIC"
+    source_project_roots: list[str] = field(default_factory=list)
+    source_learning_ids: list[str] = field(default_factory=list)
+    evolution_chain: str | None = None
+
+
+def _merge_into_aggregate(
+    agg: OrgLearningAggregate,
+    learning: Learning,
+    project_root: str,
+) -> None:
+    """Merge one learning into an aggregate, taking the best per field."""
+    agg.confidence = max(agg.confidence, learning.confidence)
+    s_map = {"deprecated": 0, "stale": 1, "active": 2}
+    if s_map.get(learning.validity, 0) > s_map.get(agg.validity, 0):
+        agg.validity = learning.validity
+    agg.sensitivity = learning.sensitivity
+    if project_root not in agg.source_project_roots:
+        agg.source_project_roots.append(project_root)
+    if learning.id not in agg.source_learning_ids:
+        agg.source_learning_ids.append(learning.id)
+    if not agg.evolution_chain and learning.evolution_chain:
+        agg.evolution_chain = learning.evolution_chain
+
+
+def aggregate_org_learnings(
+    projects_dir: Path | str,
+    *,
+    min_confidence: float = 0.3,
+    top_n: int = 5,
+) -> list[OrgLearningAggregate]:
+    """Aggregate PUBLIC-scope learnings from multiple governed projects.
+
+    Reads ``.GCC/reasoning_learnings/learnings/`` from each project directory,
+    groups learnings by overlapping trigger concepts, and merges them with
+    max-confidence / most-restrictive-validity semantics.
+
+    Returns up to ``top_n`` :class:`OrgLearningAggregate` sorted by confidence
+    descending.
+    """
+    out: dict[str, list[OrgLearningAggregate]] = {}
+
+    for project_root in _resolve_project_dirs(projects_dir):
+        try:
+            learnings = _load_active_learnings(project_root)
+        except Exception as exc:
+            logger.debug("thinkstack: skipping project %s — %s", project_root, exc)
+            continue
+        for learning in learnings:
+            concepts_key = _concept_key(learning.trigger_concepts)
+            if concepts_key not in out:
+                out[concepts_key] = []
+            existing = out[concepts_key]
+            for agg in existing:
+                cset = set(agg.trigger_concepts)
+                lset = set(learning.trigger_concepts)
+                overlap = cset & lset
+                if overlap and _content_similarity(agg.content, learning.content) > 0.6:
+                    _merge_into_aggregate(agg, learning, str(project_root))
+                    break
+            else:
+                agg = OrgLearningAggregate(
+                    content=learning.content,
+                    type=learning.type,
+                    trigger_concepts=list(learning.trigger_concepts),
+                    validity=learning.validity,
+                    confidence=learning.confidence,
+                    scope=learning.scope,
+                    sensitivity=learning.sensitivity,
+                    source_project_roots=[str(project_root)],
+                    source_learning_ids=[learning.id],
+                    evolution_chain=learning.evolution_chain,
+                )
+                out[concepts_key].append(agg)
+
+    results: list[OrgLearningAggregate] = []
+    for aggs in out.values():
+        for agg in aggs:
+            if agg.confidence >= min_confidence:
+                results.append(agg)
+    results.sort(key=lambda x: x.confidence, reverse=True)
+    return results[:top_n]
+
+
+def _concept_key(concepts: list[str]) -> str:
+    return "|".join(sorted(c.lower() for c in concepts))
+
+
+def _content_similarity(a: str, b: str) -> float:
+    a_words = set(a.lower().split())
+    b_words = set(b.lower().split())
+    if not a_words or not b_words:
+        return 0.0
+    intersection = a_words & b_words
+    return len(intersection) / max(len(a_words), len(b_words))
+
+
+def _resolve_project_dirs(projects_dir: Path | str) -> list[Path]:
+    dirs: list[Path] = []
+    registry_path = Path(projects_dir)
+    if not registry_path.exists():
+        return []
+    try:
+        data = json.loads(registry_path.read_text(encoding="utf-8"))
+    except (json.JSONDecodeError, OSError):
+        return []
+    governed = data.get("governed_projects", [])
+    if isinstance(governed, list):
+        for entry in governed:
+            if isinstance(entry, dict) and "project_root" in entry:
+                p = Path(entry["project_root"])
+                if p.exists():
+                    dirs.append(p)
+    return dirs
+
+
+def _load_active_learnings(project_root: Path) -> list[Learning]:
+    from thinkstack_core.reasoning_plus.learning.store import LearningStore
+    gcc_dir = project_root / ".GCC"
+    store = LearningStore(gcc_dir)
+    return store.list(validity="active")

thinkstack_core/reasoning_plus/learning/pii.py

@@ -0,0 +1,142 @@
+"""
+thinkstack_core.reasoning_plus.learning.pii
+=========================================
+PII detection, sanitization, and secret scanning for the learning pipeline.
+
+Two layers of protection:
+1. **Pre-extraction sanitization** — strip PII from reasoning text before
+   it reaches the LLM extractor. Uses presidio-analyzer when available,
+   with a pure-regex fallback.
+2. **Post-extraction secret scanning** — detect credentials, tokens, and
+   API keys in extracted learning content. Learnings that match are
+   quarantined (not stored).
+"""
+from __future__ import annotations
+
+import logging
+import re
+from typing import Any
+
+from thinkstack_core.reasoning_plus.learning.models import Learning
+
+logger = logging.getLogger("thinkstack.reasoning_plus.learning.pii")
+
+# ---------------------------------------------------------------------------
+# Pre-extraction PII sanitization
+# ---------------------------------------------------------------------------
+
+
+def _get_presidio_engine() -> Any:
+    """Return a presidio AnalyzerEngine, or None if presidio is unavailable."""
+    try:
+        from presidio_analyzer import AnalyzerEngine
+        return AnalyzerEngine()
+    except ImportError:
+        logger.debug("thinkstack: presidio not available, using regex PII fallback")
+        return None
+
+
+_PRESIDIO_ENGINE: Any | None = None
+
+
+def _ensure_engine() -> Any:
+    global _PRESIDIO_ENGINE
+    if _PRESIDIO_ENGINE is None:
+        _PRESIDIO_ENGINE = _get_presidio_engine()
+    return _PRESIDIO_ENGINE
+
+
+# Fallback regex patterns for PII detection (used when presidio is unavailable)
+_PII_REGEX_PATTERNS: list[tuple[str, str]] = [
+    ("CREDIT_CARD", r"\b(?:\d{4}[-\s]?){3}\d{4}\b"),
+    ("EMAIL", r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b"),
+    ("PHONE", r"\b\+?\d{1,3}[-.\s]?\(?\d{1,4}\)?[-.\s]?\d{1,4}[-.\s]?\d{1,9}\b"),
+    ("SSN", r"\b\d{3}-\d{2}-\d{4}\b"),
+    ("API_KEY_GENERIC", r"\b(?:sk-[A-Za-z0-9]{20,}|[A-Za-z0-9]{32,})\b"),
+]
+
+
+def sanitize_for_extraction(text: str) -> str:
+    """Strip PII from *text*, replacing it with ``[REDACTED:<type>]``.
+
+    Presidio-analyzer runs first (when available) for NLP-based detection.
+    Regex patterns always run second as a catch-all for API keys, emails,
+    credit cards, phones, and SSNs. The regex pass never overwrites
+    presidio's output because it operates on the result of the presidio pass.
+    """
+    if not text:
+        return text
+
+    result = text
+
+    # Presidio — optional NLP-based detection for additional coverage.
+    engine = _ensure_engine()
+    if engine is not None:
+        try:
+            presidio_results = engine.analyze(text=result, language="en")
+            if presidio_results:
+                sanitized = list(result)
+                for r in sorted(presidio_results, key=lambda x: x.start, reverse=True):
+                    entity_type = r.entity_type or "PII"
+                    placeholder = f"[REDACTED:{entity_type}]"
+                    sanitized[r.start : r.end] = list(placeholder)
+                result = "".join(sanitized)
+        except Exception as exc:
+            logger.debug("thinkstack: presidio analysis failed — %s", exc)
+
+    # Apply regex patterns — always runs, catches API keys and other patterns
+    # that presidio may have missed. Operates on presidio's output (or the
+    # original text if presidio is unavailable), so it never overwrites
+    # presidio redactions.
+    for entity_type, pattern in _PII_REGEX_PATTERNS:
+        try:
+            result = re.sub(pattern, f"[REDACTED:{entity_type}]", result)
+        except Exception:
+            continue
+
+    return result
+
+
+# ---------------------------------------------------------------------------
+# Post-extraction secret scanning
+# ---------------------------------------------------------------------------
+
+_SECRET_PATTERNS: list[tuple[str, str]] = [
+    ("OPENAI_KEY", r"sk-[A-Za-z0-9]{20,}"),
+    ("GITHUB_TOKEN", r"(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9_]{36,}"),
+    ("AWS_ACCESS_KEY", r"AKIA[0-9A-Z]{16}"),
+    ("BEARER_TOKEN", r"Bearer\s+[A-Za-z0-9\-._~+/]{20,}"),
+    ("GENERIC_SECRET", r"(?:secret|password|token|key)\s*[:=]\s*['\"]?[A-Za-z0-9_\-]{16,}"),
+]
+
+
+def scan_for_secrets(text: str) -> list[str]:
+    """Scan *text* for credentials, tokens, and API keys.
+
+    Returns a list of secret type strings that were matched.
+    Empty list means no secrets detected.
+    """
+    if not text:
+        return []
+    found: list[str] = []
+    for name, pattern in _SECRET_PATTERNS:
+        try:
+            if re.search(pattern, text):
+                found.append(name)
+        except Exception:
+            continue
+    return found
+
+
+# ---------------------------------------------------------------------------
+# Sync filtering helper
+# ---------------------------------------------------------------------------
+
+
+def filter_learnings_for_sync(learnings: list[Learning]) -> list[Learning]:
+    """Filter learnings for sync: exclude PRIVATE-disclosure learnings.
+
+    Only PUBLIC and PROTECTED learnings are included in sync payloads.
+    PRIVATE learnings stay local.
+    """
+    return [l for l in learnings if l.sensitivity != "PRIVATE"]

thinkstack_core/reasoning_plus/learning/promotion.py

@@ -0,0 +1,58 @@
+"""
+thinkstack_core.reasoning_plus.learning.promotion
+=================================================
+Learning scope promotion workflow.
+
+Promotion elevates a learning from the project scope to the org scope,
+making it eligible for cross-project injection via the five-layer filter.
+"""
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+
+from thinkstack_core.reasoning_plus.learning.models import Learning
+from thinkstack_core.reasoning_plus.learning.store import LearningStore
+
+logger = logging.getLogger("thinkstack.reasoning_plus.learning")
+
+
+def promote_learning(
+    learning_id: str,
+    gcc_dir: Path | str,
+    *,
+    to_scope: str = "org",
+    promoted_by: str | None = None,
+    boost_confidence: bool = True,
+) -> Learning | None:
+    """Promote a learning to a broader scope and persist the change.
+
+    The learning is:
+    - reassigned to ``to_scope`` (default ``"org"``)
+    - its type becomes ``"canon"``
+    - its confidence is boosted to 1.0 (when ``boost_confidence``)
+    - promotion metadata (``promoted_by``, ``promoted_at``) is recorded
+    """
+    gcc_dir = Path(gcc_dir)
+    store = LearningStore(gcc_dir)
+    learning = store.get(learning_id)
+    if not learning:
+        logger.warning("thinkstack: promotion — learning %s not found", learning_id)
+        return None
+
+    if learning.sensitivity == "PRIVATE":
+        logger.warning(
+            "thinkstack: cannot promote PRIVATE learning %s — "
+            "PRIVATE learnings must remain project-local",
+            learning_id,
+        )
+        return None
+
+    learning.promote(to_scope=to_scope, promoted_by=promoted_by, boost_confidence=boost_confidence)
+    learning.sensitivity = "PUBLIC"
+    store.save(learning, deduplicate=False)
+    logger.info(
+        "thinkstack: promoted learning %s to scope=%s type=%s",
+        learning_id, to_scope, learning.type,
+    )
+    return learning

thinkstack_core/reasoning_plus/learning/provenance.py

@@ -0,0 +1,126 @@
+"""
+thinkstack_core.reasoning_plus.learning.provenance
+===============================================
+Provenance tracking for injected learnings.
+
+Records which Learning records were injected into each LLM/tool call so that
+observed outcomes can be attributed back to the learnings that influenced the
+call.
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+from dataclasses import asdict, dataclass, field, fields
+from pathlib import Path
+from typing import Any
+
+from thinkstack_core.reasoning_plus.learning.models import _make_id, _now_iso
+
+logger = logging.getLogger("thinkstack.reasoning_plus.learning")
+
+
+@dataclass
+class ProvenanceRecord:
+    """Which learnings were injected into a single call, and what the outcome was."""
+
+    call_id: str
+    learning_ids: list[str] = field(default_factory=list)
+    session_id: str = ""
+    prompt_hash: str = ""
+    prompt_excerpt: str = ""
+    model_name: str = ""
+    outcome: str | None = None
+    injected_at: str = field(default_factory=_now_iso)
+    updated_at: str = field(default_factory=_now_iso)
+    id: str = field(default_factory=lambda: _make_id("prov"))
+    meta: dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "ProvenanceRecord":
+        valid = {f.name for f in fields(cls)}
+        return cls(**{k: v for k, v in data.items() if k in valid})
+
+
+class ProvenanceStore:
+    """Persist ProvenanceRecord records under .GCC/reasoning_learnings/provenance/."""
+
+    def __init__(self, gcc_dir: Path | str) -> None:
+        self._gcc_dir = Path(gcc_dir)
+        self._provenance_dir = self._gcc_dir / "reasoning_learnings" / "provenance"
+
+    def save(self, record: ProvenanceRecord) -> Path:
+        self._provenance_dir.mkdir(parents=True, exist_ok=True)
+        path = self._provenance_dir / f"{record.id}.json"
+        _atomic_json_write(path, record.to_dict())
+        return path
+
+    def get(self, record_id: str) -> ProvenanceRecord | None:
+        path = self._provenance_dir / f"{record_id}.json"
+        if not path.exists():
+            return None
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+            return ProvenanceRecord.from_dict(data)
+        except Exception as exc:
+            logger.warning("thinkstack: failed to read provenance %s — %s", record_id, exc)
+            return None
+
+    def get_by_call(self, call_id: str) -> ProvenanceRecord | None:
+        for record in self.list():
+            if record.call_id == call_id:
+                return record
+        return None
+
+    def list(self, outcome: str | None = None, learning_id: str | None = None) -> list[ProvenanceRecord]:
+        out: list[ProvenanceRecord] = []
+        if not self._provenance_dir.exists():
+            return out
+        for path in sorted(self._provenance_dir.glob("*.json")):
+            try:
+                data = json.loads(path.read_text(encoding="utf-8"))
+                record = ProvenanceRecord.from_dict(data)
+                if outcome and record.outcome != outcome:
+                    continue
+                if learning_id and learning_id not in record.learning_ids:
+                    continue
+                out.append(record)
+            except Exception as exc:
+                logger.debug("thinkstack: skipping malformed provenance %s — %s", path, exc)
+        return out
+
+    def update_outcome(self, call_id: str, outcome: str) -> ProvenanceRecord | None:
+        record = self.get_by_call(call_id)
+        if not record:
+            return None
+        record.outcome = outcome
+        record.updated_at = _now_iso()
+        self.save(record)
+        return record
+
+    def count(self) -> int:
+        if not self._provenance_dir.exists():
+            return 0
+        return len(list(self._provenance_dir.glob("*.json")))
+
+
+def _atomic_json_write(path: Path, data: dict[str, Any]) -> None:
+    tmp = path.with_suffix(path.suffix + ".tmp")
+    try:
+        with open(tmp, "w", encoding="utf-8") as f:
+            json.dump(data, f, indent=2)
+        tmp.replace(path)
+    finally:
+        if tmp.exists():
+            try:
+                tmp.unlink()
+            except Exception:
+                pass
+
+
+def _hash_prompt(prompt_text: str) -> str:
+    return hashlib.sha256(prompt_text.encode("utf-8")).hexdigest()[:16]