devtorch-core 3.0.1__py3-none-any.whl

devtorch_core/reasoning_plus/learning/cross_project.py

@@ -0,0 +1,234 @@
+"""
+devtorch_core.reasoning_plus.learning.cross_project
+====================================================
+Cross-project learning suggestions.
+
+Maintains a machine-level registry of governed projects at
+``~/.devtorch/registry.json`` under the ``governed_projects`` key and lets a
+project surface relevant learnings from sibling projects with matching tags or
+names.
+"""
+from __future__ import annotations
+
+import json
+import logging
+import os
+from dataclasses import asdict, dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+from devtorch_core.reasoning_plus.learning.models import Learning
+from devtorch_core.reasoning_plus.learning.relevance import RelevanceEngine
+from devtorch_core.reasoning_plus.learning.store import LearningStore
+from devtorch_core.reasoning_plus.context import extract_keywords
+
+logger = logging.getLogger("devtorch.reasoning_plus.learning.cross_project")
+
+
+DEFAULT_REGISTRY_DIR = Path.home() / ".devtorch"
+DEFAULT_REGISTRY_FILE = "registry.json"
+REGISTRY_PATH_ENV = "DEVTORCH_REGISTRY_PATH"
+
+
+@dataclass
+class GovernedProject:
+    """One governed project entry in the cross-project learning registry."""
+
+    project_root: str
+    name: str
+    tags: list[str]
+    gcc_dir: str
+    added_at: str
+    updated_at: str
+
+
+class CrossProjectRegistry:
+    """Machine-level registry of governed projects used for cross-project learning.
+
+    Stored under the ``governed_projects`` key in ``~/.devtorch/registry.json``
+    so it coexists with the Sprint 19 project registry (which uses the
+    ``projects`` key).
+    """
+
+    def __init__(self, registry_path: Path | str | None = None) -> None:
+        if registry_path is None:
+            env_path = os.environ.get(REGISTRY_PATH_ENV, "").strip()
+            if env_path:
+                registry_path = Path(env_path)
+            else:
+                registry_path = DEFAULT_REGISTRY_DIR / DEFAULT_REGISTRY_FILE
+        self.registry_path = Path(registry_path)
+        self._ensure_dir()
+
+    def _ensure_dir(self) -> None:
+        self.registry_path.parent.mkdir(parents=True, exist_ok=True)
+
+    def _read(self) -> dict[str, Any]:
+        if not self.registry_path.exists():
+            return {"version": "1", "projects": [], "governed_projects": []}
+        try:
+            data = json.loads(self.registry_path.read_text(encoding="utf-8"))
+            if not isinstance(data, dict):
+                return {"version": "1", "projects": [], "governed_projects": []}
+            return data
+        except (json.JSONDecodeError, OSError):
+            return {"version": "1", "projects": [], "governed_projects": []}
+
+    def _write(self, data: dict[str, Any]) -> None:
+        self._ensure_dir()
+        self.registry_path.write_text(
+            json.dumps(data, indent=2, sort_keys=True), encoding="utf-8"
+        )
+
+    def list(self) -> list[GovernedProject]:
+        """Return all registered governed projects."""
+        data = self._read()
+        entries: list[GovernedProject] = []
+        for raw in data.get("governed_projects", []):
+            try:
+                entries.append(GovernedProject(**raw))
+            except (TypeError, ValueError):
+                logger.debug("devtorch: skipping malformed governed project entry: %s", raw)
+                continue
+        return entries
+
+    def add(self, gcc_dir: Path | str, name: str, tags: list[str] | None = None) -> GovernedProject:
+        """Add or update a governed project entry keyed by gcc_dir."""
+        gcc_dir = Path(gcc_dir).resolve()
+        project_root = gcc_dir.parent
+        name = name or project_root.name
+        tags = sorted(set((tags or [])))
+
+        data = self._read()
+        projects = data.setdefault("governed_projects", [])
+
+        resolved_root = str(project_root)
+        for raw in projects:
+            if Path(raw.get("gcc_dir", "")).resolve() == gcc_dir:
+                raw["project_root"] = resolved_root
+                raw["name"] = name
+                raw["tags"] = tags
+                raw["updated_at"] = _now_iso()
+                self._write(data)
+                return GovernedProject(**raw)
+
+        entry = GovernedProject(
+            project_root=resolved_root,
+            name=name,
+            tags=tags,
+            gcc_dir=str(gcc_dir),
+            added_at=_now_iso(),
+            updated_at=_now_iso(),
+        )
+        projects.append(asdict(entry))
+        self._write(data)
+        return entry
+
+    def remove(self, gcc_dir: Path | str) -> bool:
+        """Remove a governed project by gcc_dir. Returns True if removed."""
+        gcc_dir = Path(gcc_dir).resolve()
+        data = self._read()
+        projects = data.get("governed_projects", [])
+        original_len = len(projects)
+        projects = [
+            raw for raw in projects
+            if Path(raw.get("gcc_dir", "")).resolve() != gcc_dir
+        ]
+        if len(projects) == original_len:
+            return False
+        data["governed_projects"] = projects
+        self._write(data)
+        return True
+
+
+def _registry() -> CrossProjectRegistry:
+    return CrossProjectRegistry()
+
+
+def register_project(gcc_dir: Path | str, name: str, tags: list[str] | None = None) -> GovernedProject:
+    """Add or update the current project in the governed project registry."""
+    return _registry().add(gcc_dir, name, tags)
+
+
+def list_projects() -> list[GovernedProject]:
+    """Return all registered governed projects."""
+    return _registry().list()
+
+
+def _project_match_score(query: str, project: GovernedProject) -> float:
+    """Keyword overlap score between the query and project name/tags."""
+    query_tokens = set(extract_keywords(query, max_keywords=20))
+    if not query_tokens:
+        return 0.0
+
+    project_tokens: set[str] = set()
+    project_tokens.update(extract_keywords(project.name, max_keywords=20))
+    for tag in project.tags:
+        project_tokens.update(extract_keywords(tag, max_keywords=20))
+        project_tokens.add(tag.lower())
+
+    if not project_tokens:
+        return 0.0
+
+    intersection = query_tokens & project_tokens
+    return len(intersection) / max(len(query_tokens), len(project_tokens))
+
+
+def _load_active_learnings(gcc_dir: Path | str) -> list[Learning]:
+    """Load active learnings from a project's .GCC directory."""
+    try:
+        store = LearningStore(gcc_dir)
+        return store.list(validity="active")
+    except Exception as exc:
+        logger.warning("devtorch: failed to load learnings from %s — %s", gcc_dir, exc)
+        return []
+
+
+def cross_project_suggest(
+    query: str,
+    current_project_root: Path | str,
+    top_n: int = 3,
+    project_match_threshold: float = 0.1,
+) -> list[Learning]:
+    """Find relevant learnings from sibling governed projects.
+
+    Projects are matched by keyword overlap with the query using their name and
+    tags. Active learnings from matched projects are loaded via ``LearningStore``
+    and ranked with the keyword-based ``RelevanceEngine``. The current project is
+    excluded so suggestions come from other governed projects.
+    """
+    current_project_root = Path(current_project_root).resolve()
+    registry = _registry()
+    projects = registry.list()
+
+    # Exclude current project to avoid suggesting its own learnings back to it.
+    sibling_projects = [
+        p for p in projects
+        if Path(p.project_root).resolve() != current_project_root
+    ]
+
+    # Match projects by name/tags.
+    scored_projects = [
+        (score, p)
+        for p in sibling_projects
+        if (score := _project_match_score(query, p)) >= project_match_threshold
+    ]
+    scored_projects.sort(key=lambda x: x[0], reverse=True)
+
+    # Collect active learnings from all matched projects.
+    all_learnings: list[Learning] = []
+    for _, project in scored_projects:
+        learnings = _load_active_learnings(project.gcc_dir)
+        all_learnings.extend(learnings)
+
+    if not all_learnings:
+        return []
+
+    # Rank across projects using the same keyword backend used by the facade.
+    engine = RelevanceEngine()
+    return engine.rank(query, all_learnings, top_n=top_n)
+
+
+def _now_iso() -> str:
+    return datetime.now(timezone.utc).isoformat()

devtorch_core/reasoning_plus/learning/embeddings.py

@@ -0,0 +1,209 @@
+"""
+devtorch_core.reasoning_plus.learning.embeddings
+================================================
+Pluggable embedding backends for DRPL relevance.
+
+The default backend is keyword-based (no extra dependencies).  Users can opt
+into dense semantic embeddings by installing the optional ``embeddings`` extra:
+
+    pip install devtorch-core[embeddings]
+
+and then configuring the backend:
+
+    devtorch reasoning-plus config \
+        --learning-embedding-backend sentence_transformers \
+        --learning-embedding-model all-MiniLM-L6-v2
+
+Supported backends:
+- ``keyword``           — Jaccard similarity over extracted keywords (default)
+- ``sentence_transformers`` — cosine similarity over dense embeddings (local)
+- ``openai``            — cosine similarity over OpenAI embeddings (remote)
+"""
+from __future__ import annotations
+
+import logging
+import math
+from abc import ABC, abstractmethod
+from typing import Any
+
+from devtorch_core.reasoning_plus.context import extract_keywords
+
+logger = logging.getLogger("devtorch.reasoning_plus.learning")
+
+
+# ---------------------------------------------------------------------------
+# Base class
+# ---------------------------------------------------------------------------
+
+class EmbeddingBackend(ABC):
+    """Abstract embedding backend for DRPL relevance scoring."""
+
+    @abstractmethod
+    def encode(self, text: str) -> Any:
+        """Encode *text* into a backend-specific representation."""
+        ...
+
+    @abstractmethod
+    def similarity(self, a: Any, b: Any) -> float:
+        """Return a similarity score in [0, 1] for two encodings."""
+        ...
+
+    @property
+    @abstractmethod
+    def name(self) -> str:
+        """Short identifier for this backend."""
+        ...
+
+
+# ---------------------------------------------------------------------------
+# Keyword backend (default, no dependencies)
+# ---------------------------------------------------------------------------
+
+class KeywordEmbeddingBackend(EmbeddingBackend):
+    """Keyword-overlap backend that uses extracted concepts as a sparse vector."""
+
+    def __init__(self, max_keywords: int = 15) -> None:
+        self.max_keywords = max_keywords
+
+    def encode(self, text: str) -> set[str]:
+        return set(extract_keywords(text, max_keywords=self.max_keywords))
+
+    def similarity(self, a: set[str], b: set[str]) -> float:
+        if not a and not b:
+            return 0.0
+        overlap = len(a & b)
+        total = len(a | b)
+        return overlap / total if total else 0.0
+
+    @property
+    def name(self) -> str:
+        return "keyword"
+
+
+# ---------------------------------------------------------------------------
+# Dense-vector helpers
+# ---------------------------------------------------------------------------
+
+def _cosine_similarity(a: list[float], b: list[float]) -> float:
+    dot = sum(x * y for x, y in zip(a, b))
+    norm_a = math.sqrt(sum(x * x for x in a))
+    norm_b = math.sqrt(sum(x * x for x in b))
+    if norm_a == 0.0 or norm_b == 0.0:
+        return 0.0
+    return max(0.0, min(1.0, dot / (norm_a * norm_b)))
+
+
+# ---------------------------------------------------------------------------
+# Sentence-transformers backend (optional)
+# ---------------------------------------------------------------------------
+
+class SentenceTransformerBackend(EmbeddingBackend):
+    """Local dense embedding backend via sentence-transformers."""
+
+    def __init__(self, model_name: str = "all-MiniLM-L6-v2") -> None:
+        try:
+            from sentence_transformers import SentenceTransformer
+        except ImportError as exc:
+            raise ImportError(
+                "sentence-transformers is not installed. "
+                "Install the embeddings extra: pip install devtorch-core[embeddings]"
+            ) from exc
+        self._model = SentenceTransformer(model_name)
+        self._model_name = model_name
+
+    def encode(self, text: str) -> list[float]:
+        import numpy as np
+
+        vector = self._model.encode(text)
+        return vector.tolist()
+
+    def similarity(self, a: list[float], b: list[float]) -> float:
+        return _cosine_similarity(a, b)
+
+    @property
+    def name(self) -> str:
+        return "sentence_transformers"
+
+
+# ---------------------------------------------------------------------------
+# OpenAI backend (optional)
+# ---------------------------------------------------------------------------
+
+class OpenAIEmbeddingBackend(EmbeddingBackend):
+    """Remote dense embedding backend via OpenAI embeddings API."""
+
+    def __init__(
+        self,
+        model_name: str = "text-embedding-3-small",
+        api_key: str | None = None,
+    ) -> None:
+        try:
+            import openai
+        except ImportError as exc:
+            raise ImportError(
+                "openai is not installed. "
+                "Install the wrapper extra: pip install devtorch-core[wrapper]"
+            ) from exc
+        self._client = openai.OpenAI(api_key=api_key)
+        self._model_name = model_name
+
+    def encode(self, text: str) -> list[float]:
+        response = self._client.embeddings.create(
+            input=text,
+            model=self._model_name,
+        )
+        return response.data[0].embedding
+
+    def similarity(self, a: list[float], b: list[float]) -> float:
+        return _cosine_similarity(a, b)
+
+    @property
+    def name(self) -> str:
+        return "openai"
+
+
+# ---------------------------------------------------------------------------
+# Fake backend (for tests)
+# ---------------------------------------------------------------------------
+
+class FakeEmbeddingBackend(EmbeddingBackend):
+    """Deterministic fake backend for unit tests."""
+
+    def __init__(self, embeddings: dict[str, list[float]] | None = None, dim: int = 8) -> None:
+        self._embeddings: dict[str, list[float]] = embeddings or {}
+        self._dim = dim
+
+    def encode(self, text: str) -> list[float]:
+        return self._embeddings.get(text, [0.0] * self._dim)
+
+    def similarity(self, a: list[float], b: list[float]) -> float:
+        return _cosine_similarity(a, b)
+
+    @property
+    def name(self) -> str:
+        return "fake"
+
+
+# ---------------------------------------------------------------------------
+# Factory
+# ---------------------------------------------------------------------------
+
+def get_embedding_backend(
+    name: str | None = None,
+    model_name: str | None = None,
+    max_keywords: int = 15,
+) -> EmbeddingBackend:
+    """
+    Return an embedding backend by name.
+
+    Supported names: ``keyword`` (default), ``sentence_transformers``,
+    ``sentence-transformers``, ``st``, ``openai``.
+    """
+    name = (name or "keyword").lower().strip().replace("-", "_")
+    if name in ("keyword", "keywords"):
+        return KeywordEmbeddingBackend(max_keywords=max_keywords)
+    if name in ("sentence_transformers", "sentence_transformer", "st"):
+        return SentenceTransformerBackend(model_name or "all-MiniLM-L6-v2")
+    if name == "openai":
+        return OpenAIEmbeddingBackend(model_name or "text-embedding-3-small")
+    raise ValueError(f"Unknown embedding backend: {name}")

devtorch_core/reasoning_plus/learning/extractor.py

@@ -0,0 +1,207 @@
+"""
+devtorch_core.reasoning_plus.learning.extractor
+===============================================
+Distill ReasoningCall records into reusable Learning objects.
+
+Two strategies are supported:
+
+1. **Rule-based** (default, no dependencies) — pair captured reasoning with the
+   observed outcome and synthesize a compact learning sentence.
+2. **LLM-based** (optional) — pass the reasoning, input, output, and outcome to
+   a small LLM prompt that returns a concise learning summary.  The caller
+   supplies the LLM client, so the core library remains dependency-free.
+"""
+from __future__ import annotations
+
+import logging
+from typing import Callable
+
+from devtorch_core.reasoning_plus.context import extract_keywords
+from devtorch_core.reasoning_plus.learning.models import Learning, ReasoningCall
+
+logger = logging.getLogger("devtorch.reasoning_plus.learning")
+
+DEFAULT_EXTRACTION_PROMPT = """\
+You are distilling a previous reasoning step into a short, reusable insight.
+Given the reasoning, the action's input, its output, and the observed outcome,
+write one concise sentence that another AI agent could reuse when facing a
+similar situation.
+
+Outcome: {outcome}
+Name: {name}
+Input: {input}
+Output: {output}
+Reasoning: {reasoning}
+
+Insight:"""
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+def _type_for_outcome(outcome: str) -> str:
+    if outcome == "success":
+        return "confirm"
+    if outcome == "failure":
+        return "correction"
+    if outcome == "partial":
+        return "insight"
+    return "insight"
+
+
+def _extract_concepts(call: ReasoningCall) -> list[str]:
+    text = f"{call.reasoning} {call.input} {call.output}"
+    return extract_keywords(text, max_keywords=15)
+
+
+def _fallback_reasoning(call: ReasoningCall) -> str:
+    """When the call has no captured reasoning, synthesize a short summary."""
+    if call.outcome not in ("success", "failure"):
+        return ""
+    name = call.name or call.call_type
+    if call.outcome == "failure":
+        return f"{name} failed. Input: {call.input}. Output: {call.output}."
+    return f"{name} succeeded. Input: {call.input}. Output: {call.output}."
+
+
+def _compose_content_from_reasoning(call: ReasoningCall, reasoning: str) -> str:
+    reasoning = reasoning.strip().replace("\n", " ")
+    # Truncate very long reasoning
+    if len(reasoning) > 400:
+        reasoning = reasoning[:397].rstrip() + "..."
+
+    name = call.name or call.call_type
+    if call.outcome == "failure":
+        return (
+            f"Previous {name} attempt failed. The reasoning was: {reasoning}. "
+            f"Consider a different approach before trying the same path again."
+        )
+    if call.outcome == "success":
+        return (
+            f"Previous {name} attempt succeeded. The reasoning was: {reasoning}. "
+            f"This approach is worth reusing when the context is similar."
+        )
+    return (
+        f"Previous {name} attempt had outcome '{call.outcome}'. The reasoning was: {reasoning}."
+    )
+
+
+def _build_learning(call: ReasoningCall, reasoning: str) -> Learning:
+    """Build a single Learning object from a reasoning summary."""
+    learning_type = _type_for_outcome(call.outcome)
+    content = _compose_content_from_reasoning(call, reasoning)
+    concepts = _extract_concepts(call)
+
+    # Start with moderate confidence so feedback can raise or lower it.
+    base_confidence = min(call.confidence, 0.8)
+
+    return Learning(
+        content=content,
+        type=learning_type,
+        trigger_concepts=concepts,
+        state_hash=call.state_hash,
+        confidence=base_confidence,
+        source_reasoning_ids=[call.id],
+        sensitivity=call.sensitivity,
+        meta={"call_type": call.call_type, "name": call.name, "outcome": call.outcome},
+    )
+
+
+# ---------------------------------------------------------------------------
+# Rule-based extractor
+# ---------------------------------------------------------------------------
+
+class LearningExtractor:
+    """Extract learnings from a single reasoning call using lightweight rules."""
+
+    def extract(self, call: ReasoningCall) -> list[Learning]:
+        """
+        Return one or more Learning objects derived from the call.
+
+        For now we produce a single learning that summarizes the reasoning + outcome.
+        """
+        reasoning = call.reasoning.strip()
+        if not reasoning:
+            reasoning = _fallback_reasoning(call)
+
+        if not reasoning:
+            return []
+
+        return [_build_learning(call, reasoning)]
+
+
+# Backwards-compatible alias
+RuleBasedLearningExtractor = LearningExtractor
+
+
+# ---------------------------------------------------------------------------
+# LLM-based extractor
+# ---------------------------------------------------------------------------
+
+class LLMBasedLearningExtractor(LearningExtractor):
+    """
+    Extract learnings by asking an LLM to summarize a reasoning call.
+
+    The caller supplies ``client``, a callable that takes a prompt string and
+    returns a completion string.  This keeps the core library free of any LLM
+    SDK dependency.
+    """
+
+    def __init__(
+        self,
+        client: Callable[[str], str],
+        prompt_template: str | None = None,
+    ) -> None:
+        self._client = client
+        self._prompt_template = prompt_template or DEFAULT_EXTRACTION_PROMPT
+
+    def extract(self, call: ReasoningCall) -> list[Learning]:
+        """Use an LLM to summarize the reasoning; fall back to rule-based if the LLM fails."""
+        reasoning = call.reasoning.strip()
+        if not reasoning:
+            reasoning = _fallback_reasoning(call)
+
+        if not reasoning:
+            return []
+
+        prompt = self._prompt_template.format(
+            reasoning=reasoning,
+            input=call.input,
+            output=call.output,
+            outcome=call.outcome,
+            name=call.name or call.call_type,
+        )
+
+        try:
+            summary = self._client(prompt).strip()
+        except Exception as exc:
+            logger.warning("devtorch: LLM-based extraction failed — %s", exc)
+            return [_build_learning(call, reasoning)]
+
+        if not summary:
+            return []
+
+        return [_build_learning(call, summary)]
+
+
+# ---------------------------------------------------------------------------
+# Factory
+# ---------------------------------------------------------------------------
+
+def make_extractor(
+    strategy: str = "rule",
+    llm_client: Callable[[str], str] | None = None,
+) -> LearningExtractor:
+    """
+    Build a learning extractor for the given strategy.
+
+    Strategies:
+    - ``rule`` (default): lightweight rule-based extraction.
+    - ``llm``: LLM-based extraction; requires a ``llm_client`` callable.
+
+    If ``llm`` is requested but no client is provided, falls back to rule-based.
+    """
+    if strategy == "llm" and llm_client is not None:
+        return LLMBasedLearningExtractor(llm_client)
+    return LearningExtractor()