thinkstack-core 4.0.0__py3-none-any.whl

thinkstack_core/reasoning_plus/learning/context_collector.py

@@ -0,0 +1,194 @@
+"""
+thinkstack_core.reasoning_plus.learning.context_collector
+=======================================================
+Collect enriched extraction context from a ReasoningCall and workspace.
+
+Builds an ExtractionContext that contains not just the reasoning text
+but also file paths, import changes, and library references — enabling
+the LLM extractor to produce concrete, actionable learnings like
+"use date-fns v3 instead of moment.js".
+"""
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from thinkstack_core.reasoning_plus.learning.models import ReasoningCall
+
+
+# Patterns for detecting import statements in text
+_IMPORT_PATTERN = re.compile(
+    r'(?:^|[;.,:\s])(?:import\s+(?:(\w+(?:\.\w+)*)(?:\s+as\s+\w+)?)'
+    r'|from\s+(\w+(?:\.\w+)*)\s+import\s+[\w,\s*]+(?:as\s+\w+)?)',
+    re.MULTILINE,
+)
+
+# Patterns for detecting file paths (common extensions)
+_FILE_EXTENSIONS = {
+    ".py", ".js", ".ts", ".tsx", ".jsx", ".java", ".kt", ".go",
+    ".rs", ".cpp", ".c", ".h", ".hpp", ".css", ".scss", ".html",
+    ".vue", ".svelte", ".rb", ".php", ".swift", ".m", ".mm",
+    ".yaml", ".yml", ".json", ".toml", ".ini", ".cfg", ".conf",
+    ".md", ".rst", ".txt", ".sh", ".bash", ".zsh", ".ps1",
+    ".sql", ".graphql", ".proto", ".gradle", ".lock",
+}
+
+# Patterns for dependency references
+_PIP_PACKAGE_PATTERN = re.compile(
+    r'(?:pip\s+install\s+)([\w\-.\[\]]+)',
+    re.IGNORECASE,
+)
+_NPM_PACKAGE_PATTERN = re.compile(
+    r'(?:npm\s+(?:install|i|add)\s+)([\w@./\-]+)',
+    re.IGNORECASE,
+)
+_CARGO_PACKAGE_PATTERN = re.compile(
+    r'(?:cargo\s+add\s+)([\w\-]+)',
+    re.IGNORECASE,
+)
+
+
+@dataclass
+class ExtractionContext:
+    """Enriched context for learning extraction."""
+
+    reasoning_text: str = ""
+    files_changed: list[str] = field(default_factory=list)
+    imports_added: list[str] = field(default_factory=list)
+    imports_removed: list[str] = field(default_factory=list)
+    libraries_used: list[str] = field(default_factory=list)
+    call_type: str = "llm"
+    outcome: str = "unknown"
+
+    def is_significant(self) -> bool:
+        """Return True if this context contains enough signal to warrant extraction."""
+        if self.outcome in ("failure", "partial"):
+            return True
+        if self.files_changed or self.imports_added or self.imports_removed:
+            return True
+        if self.libraries_used:
+            return True
+        return False
+
+    def to_prompt_context(self) -> str:
+        """Format this context as a compact block for the extraction LLM prompt."""
+        parts: list[str] = []
+
+        if self.files_changed:
+            parts.append(f"Files changed/suggested: {', '.join(sorted(self.files_changed))}")
+
+        if self.imports_added:
+            parts.append(f"New imports: {', '.join(sorted(self.imports_added))}")
+
+        if self.imports_removed:
+            parts.append(f"Removed imports: {', '.join(sorted(self.imports_removed))}")
+
+        if self.libraries_used:
+            parts.append(f"Libraries referenced: {', '.join(sorted(self.libraries_used))}")
+
+        parts.append(f"Outcome: {self.outcome}")
+        parts.append(f"Call type: {self.call_type}")
+
+        return "\n".join(parts)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "reasoning_text": self.reasoning_text,
+            "files_changed": list(self.files_changed),
+            "imports_added": list(self.imports_added),
+            "imports_removed": list(self.imports_removed),
+            "libraries_used": list(self.libraries_used),
+            "call_type": self.call_type,
+            "outcome": self.outcome,
+        }
+
+
+def collect_extraction_context(
+    call: ReasoningCall,
+    gcc_dir: Path | None = None,
+) -> ExtractionContext:
+    """Build an ExtractionContext from a ReasoningCall and optional workspace.
+
+    The collector parses the call's reasoning, input, and output text for:
+    - File paths mentioned
+    - Import statements
+    - Package/dependency references
+
+    If a gcc_dir is provided, it may use workspace metadata to enrich context
+    (e.g., reading current file content for diff analysis).
+    """
+    combined = f"{call.reasoning} {call.input} {call.output}"
+
+    files_changed = _extract_file_paths(call)
+    imports_added = _extract_imports(combined)
+    # TODO S19: Populate imports_removed by comparing file imports before/after the call
+    imports_removed: list[str] = []
+    libraries_used = _extract_libraries(combined)
+
+    ctx = ExtractionContext(
+        reasoning_text=call.reasoning,
+        files_changed=files_changed,
+        imports_added=imports_added,
+        imports_removed=imports_removed,
+        libraries_used=libraries_used,
+        call_type=call.call_type,
+        outcome=call.outcome,
+    )
+
+    if gcc_dir and files_changed:
+        _enrich_from_workspace(ctx, gcc_dir)
+
+    return ctx
+
+
+def _extract_file_paths(call: ReasoningCall) -> list[str]:
+    """Extract file paths from the call's reasoning and output text."""
+    paths: set[str] = set()
+
+    texts = [call.reasoning, call.output, call.input, call.name]
+    for text in texts:
+        if not isinstance(text, str):
+            continue
+        # Split on whitespace and check each word for known file extensions
+        for word in text.replace("/", " / ").split():
+            word = word.strip(".,;:!?\"'()[]{}<>")
+            ext = Path(word).suffix.lower()
+            if ext in _FILE_EXTENSIONS:
+                paths.add(word)
+
+    return sorted(paths)
+
+
+def _extract_imports(text: str) -> list[str]:
+    """Extract import statements from text (pip-style Python imports)."""
+    imports: set[str] = set()
+
+    for match in _IMPORT_PATTERN.finditer(text):
+        if match.group(1):
+            imports.add(match.group(1).split(".")[0])
+        if match.group(2):
+            imports.add(match.group(2).split(".")[0])
+
+    return sorted(imports)
+
+
+def _extract_libraries(text: str) -> list[str]:
+    """Extract library/package references from dependency install commands."""
+    libs: set[str] = set()
+
+    for pattern in [_PIP_PACKAGE_PATTERN, _NPM_PACKAGE_PATTERN, _CARGO_PACKAGE_PATTERN]:
+        for match in pattern.finditer(text):
+            libs.add(match.group(1).lower())
+
+    return sorted(libs)
+
+
+def _enrich_from_workspace(ctx: ExtractionContext, gcc_dir: Path) -> None:
+    """Enrich context with workspace analysis.
+
+    # TODO S19+: Diff changed files against git HEAD to detect import
+    # adds/removes and library changes. This is deferred to keep
+    # extraction cost zero for the common case.
+    """

thinkstack_core/reasoning_plus/learning/cross_project.py

@@ -0,0 +1,234 @@
+"""
+thinkstack_core.reasoning_plus.learning.cross_project
+====================================================
+Cross-project learning suggestions.
+
+Maintains a machine-level registry of governed projects at
+``~/.thinkstack/registry.json`` under the ``governed_projects`` key and lets a
+project surface relevant learnings from sibling projects with matching tags or
+names.
+"""
+from __future__ import annotations
+
+import json
+import logging
+import os
+from dataclasses import asdict, dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+from thinkstack_core.reasoning_plus.learning.models import Learning
+from thinkstack_core.reasoning_plus.learning.relevance import RelevanceEngine
+from thinkstack_core.reasoning_plus.learning.store import LearningStore
+from thinkstack_core.reasoning_plus.context import extract_keywords
+
+logger = logging.getLogger("thinkstack.reasoning_plus.learning.cross_project")
+
+
+DEFAULT_REGISTRY_DIR = Path.home() / ".thinkstack"
+DEFAULT_REGISTRY_FILE = "registry.json"
+REGISTRY_PATH_ENV = "THINKSTACK_REGISTRY_PATH"
+
+
+@dataclass
+class GovernedProject:
+    """One governed project entry in the cross-project learning registry."""
+
+    project_root: str
+    name: str
+    tags: list[str]
+    gcc_dir: str
+    added_at: str
+    updated_at: str
+
+
+class CrossProjectRegistry:
+    """Machine-level registry of governed projects used for cross-project learning.
+
+    Stored under the ``governed_projects`` key in ``~/.thinkstack/registry.json``
+    so it coexists with the Sprint 19 project registry (which uses the
+    ``projects`` key).
+    """
+
+    def __init__(self, registry_path: Path | str | None = None) -> None:
+        if registry_path is None:
+            env_path = os.environ.get(REGISTRY_PATH_ENV, "").strip()
+            if env_path:
+                registry_path = Path(env_path)
+            else:
+                registry_path = DEFAULT_REGISTRY_DIR / DEFAULT_REGISTRY_FILE
+        self.registry_path = Path(registry_path)
+        self._ensure_dir()
+
+    def _ensure_dir(self) -> None:
+        self.registry_path.parent.mkdir(parents=True, exist_ok=True)
+
+    def _read(self) -> dict[str, Any]:
+        if not self.registry_path.exists():
+            return {"version": "1", "projects": [], "governed_projects": []}
+        try:
+            data = json.loads(self.registry_path.read_text(encoding="utf-8"))
+            if not isinstance(data, dict):
+                return {"version": "1", "projects": [], "governed_projects": []}
+            return data
+        except (json.JSONDecodeError, OSError):
+            return {"version": "1", "projects": [], "governed_projects": []}
+
+    def _write(self, data: dict[str, Any]) -> None:
+        self._ensure_dir()
+        self.registry_path.write_text(
+            json.dumps(data, indent=2, sort_keys=True), encoding="utf-8"
+        )
+
+    def list(self) -> list[GovernedProject]:
+        """Return all registered governed projects."""
+        data = self._read()
+        entries: list[GovernedProject] = []
+        for raw in data.get("governed_projects", []):
+            try:
+                entries.append(GovernedProject(**raw))
+            except (TypeError, ValueError):
+                logger.debug("thinkstack: skipping malformed governed project entry: %s", raw)
+                continue
+        return entries
+
+    def add(self, gcc_dir: Path | str, name: str, tags: list[str] | None = None) -> GovernedProject:
+        """Add or update a governed project entry keyed by gcc_dir."""
+        gcc_dir = Path(gcc_dir).resolve()
+        project_root = gcc_dir.parent
+        name = name or project_root.name
+        tags = sorted(set((tags or [])))
+
+        data = self._read()
+        projects = data.setdefault("governed_projects", [])
+
+        resolved_root = str(project_root)
+        for raw in projects:
+            if Path(raw.get("gcc_dir", "")).resolve() == gcc_dir:
+                raw["project_root"] = resolved_root
+                raw["name"] = name
+                raw["tags"] = tags
+                raw["updated_at"] = _now_iso()
+                self._write(data)
+                return GovernedProject(**raw)
+
+        entry = GovernedProject(
+            project_root=resolved_root,
+            name=name,
+            tags=tags,
+            gcc_dir=str(gcc_dir),
+            added_at=_now_iso(),
+            updated_at=_now_iso(),
+        )
+        projects.append(asdict(entry))
+        self._write(data)
+        return entry
+
+    def remove(self, gcc_dir: Path | str) -> bool:
+        """Remove a governed project by gcc_dir. Returns True if removed."""
+        gcc_dir = Path(gcc_dir).resolve()
+        data = self._read()
+        projects = data.get("governed_projects", [])
+        original_len = len(projects)
+        projects = [
+            raw for raw in projects
+            if Path(raw.get("gcc_dir", "")).resolve() != gcc_dir
+        ]
+        if len(projects) == original_len:
+            return False
+        data["governed_projects"] = projects
+        self._write(data)
+        return True
+
+
+def _registry() -> CrossProjectRegistry:
+    return CrossProjectRegistry()
+
+
+def register_project(gcc_dir: Path | str, name: str, tags: list[str] | None = None) -> GovernedProject:
+    """Add or update the current project in the governed project registry."""
+    return _registry().add(gcc_dir, name, tags)
+
+
+def list_projects() -> list[GovernedProject]:
+    """Return all registered governed projects."""
+    return _registry().list()
+
+
+def _project_match_score(query: str, project: GovernedProject) -> float:
+    """Keyword overlap score between the query and project name/tags."""
+    query_tokens = set(extract_keywords(query, max_keywords=20))
+    if not query_tokens:
+        return 0.0
+
+    project_tokens: set[str] = set()
+    project_tokens.update(extract_keywords(project.name, max_keywords=20))
+    for tag in project.tags:
+        project_tokens.update(extract_keywords(tag, max_keywords=20))
+        project_tokens.add(tag.lower())
+
+    if not project_tokens:
+        return 0.0
+
+    intersection = query_tokens & project_tokens
+    return len(intersection) / max(len(query_tokens), len(project_tokens))
+
+
+def _load_active_learnings(gcc_dir: Path | str) -> list[Learning]:
+    """Load active learnings from a project's .GCC directory."""
+    try:
+        store = LearningStore(gcc_dir)
+        return store.list(validity="active")
+    except Exception as exc:
+        logger.warning("thinkstack: failed to load learnings from %s — %s", gcc_dir, exc)
+        return []
+
+
+def cross_project_suggest(
+    query: str,
+    current_project_root: Path | str,
+    top_n: int = 3,
+    project_match_threshold: float = 0.1,
+) -> list[Learning]:
+    """Find relevant learnings from sibling governed projects.
+
+    Projects are matched by keyword overlap with the query using their name and
+    tags. Active learnings from matched projects are loaded via ``LearningStore``
+    and ranked with the keyword-based ``RelevanceEngine``. The current project is
+    excluded so suggestions come from other governed projects.
+    """
+    current_project_root = Path(current_project_root).resolve()
+    registry = _registry()
+    projects = registry.list()
+
+    # Exclude current project to avoid suggesting its own learnings back to it.
+    sibling_projects = [
+        p for p in projects
+        if Path(p.project_root).resolve() != current_project_root
+    ]
+
+    # Match projects by name/tags.
+    scored_projects = [
+        (score, p)
+        for p in sibling_projects
+        if (score := _project_match_score(query, p)) >= project_match_threshold
+    ]
+    scored_projects.sort(key=lambda x: x[0], reverse=True)
+
+    # Collect active learnings from all matched projects.
+    all_learnings: list[Learning] = []
+    for _, project in scored_projects:
+        learnings = _load_active_learnings(project.gcc_dir)
+        all_learnings.extend(learnings)
+
+    if not all_learnings:
+        return []
+
+    # Rank across projects using the same keyword backend used by the facade.
+    engine = RelevanceEngine()
+    return engine.rank(query, all_learnings, top_n=top_n)
+
+
+def _now_iso() -> str:
+    return datetime.now(timezone.utc).isoformat()

thinkstack_core/reasoning_plus/learning/deny_list.py

@@ -0,0 +1,108 @@
+"""
+thinkstack_core.reasoning_plus.learning.deny_list
+=================================================
+Per-project learning deny lists.
+
+Each project can maintain a ``.GCC/learning_config.json`` file that
+lists concepts and learning IDs to exclude from cross-project sharing
+and injection. The deny list is consulted by the five-layer filter
+and by the org-store aggregation stage.
+"""
+from __future__ import annotations
+
+import json
+import logging
+from dataclasses import dataclass, field
+from pathlib import Path
+
+logger = logging.getLogger("thinkstack.reasoning_plus.learning")
+
+
+@dataclass
+class DenyList:
+    """Exclusion rules for cross-project learning."""
+
+    excluded_concepts: list[str] = field(default_factory=list)
+    excluded_learnings: list[str] = field(default_factory=list)
+    project_root: str | None = None
+
+    def add_concept(self, concept: str) -> None:
+        if concept not in self.excluded_concepts:
+            self.excluded_concepts.append(concept)
+
+    def remove_concept(self, concept: str) -> bool:
+        lowered = [c.lower() for c in self.excluded_concepts]
+        try:
+            idx = lowered.index(concept.lower())
+            self.excluded_concepts.pop(idx)
+            return True
+        except ValueError:
+            return False
+
+    def add_learning(self, learning_id: str) -> None:
+        if learning_id not in self.excluded_learnings:
+            self.excluded_learnings.append(learning_id)
+
+    def remove_learning(self, learning_id: str) -> bool:
+        if learning_id in self.excluded_learnings:
+            self.excluded_learnings.remove(learning_id)
+            return True
+        return False
+
+    def to_dict(self) -> dict:
+        return {
+            "excluded_concepts": self.excluded_concepts,
+            "excluded_learnings": self.excluded_learnings,
+        }
+
+
+def load_deny_list(gcc_dir_or_project_root: Path | str) -> DenyList:
+    """Load the deny list from a project's ``.GCC/learning_config.json``."""
+    project_root = Path(gcc_dir_or_project_root)
+    if project_root.name == ".GCC":
+        config_path = project_root / "learning_config.json"
+    else:
+        config_path = project_root / ".GCC" / "learning_config.json"
+    if not config_path.exists():
+        return DenyList(project_root=str(project_root))
+    try:
+        data = json.loads(config_path.read_text(encoding="utf-8"))
+        exclude = data.get("exclude", {})
+        return DenyList(
+            excluded_concepts=exclude.get("excluded_concepts", exclude.get("concepts", [])),
+            excluded_learnings=exclude.get("excluded_learnings", exclude.get("learnings", [])),
+            project_root=str(project_root),
+        )
+    except Exception as exc:
+        logger.warning("thinkstack: failed to read deny list from %s — %s", config_path, exc)
+        return DenyList(project_root=str(project_root))
+
+
+def save_deny_list(deny_list: DenyList, gcc_dir_or_project_root: Path | str) -> None:
+    """Persist a deny list to ``.GCC/learning_config.json``."""
+    project_root = Path(gcc_dir_or_project_root)
+    if project_root.name == ".GCC":
+        config_path = project_root / "learning_config.json"
+        gcc_dir = project_root
+    else:
+        config_path = project_root / ".GCC" / "learning_config.json"
+        gcc_dir = project_root / ".GCC"
+    gcc_dir.mkdir(parents=True, exist_ok=True)
+    data = {
+        "exclude": deny_list.to_dict(),
+        "version": 1,
+    }
+    tmp = config_path.with_suffix(".tmp")
+    try:
+        with open(tmp, "w", encoding="utf-8") as f:
+            json.dump(data, f, indent=2)
+        tmp.replace(config_path)
+    except Exception as exc:
+        logger.warning("thinkstack: failed to save deny list to %s — %s", config_path, exc)
+        raise
+    finally:
+        if tmp.exists():
+            try:
+                tmp.unlink()
+            except Exception:
+                pass