buildlog 0.7.0__py3-none-any.whl → 0.9.0__py3-none-any.whl

buildlog/llm.py

@@ -0,0 +1,508 @@
+"""LLM-backed rule extraction, deduplication, and scoring.
+
+Provides a provider-agnostic interface for using LLMs to:
+- Extract structured rules from buildlog entries
+- Select canonical forms when deduplicating similar rules
+- Score rules with severity/scope/applicability
+
+Provider cascade:
+1. Explicit config (.buildlog/config.yml or env)
+2. Injected at call site (API parameter)
+3. Auto-detect: Ollama -> Anthropic -> None (regex fallback)
+"""
+
+from __future__ import annotations
+
+__all__ = [
+    "ExtractedRule",
+    "RuleScoring",
+    "LLMConfig",
+    "LLMBackend",
+    "OllamaBackend",
+    "AnthropicBackend",
+    "PROVIDERS",
+    "register_provider",
+    "get_llm_backend",
+]
+
+import json
+import logging
+import os
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from types import MappingProxyType
+from typing import Protocol, runtime_checkable
+
+logger = logging.getLogger(__name__)
+
+# --- Data types (provider-agnostic) ---
+
+VALID_SEVERITIES = ("critical", "major", "minor", "info")
+VALID_SCOPES = ("global", "module", "function")
+VALID_CATEGORIES = ("architectural", "workflow", "tool_usage", "domain_knowledge")
+
+
+@dataclass
+class ExtractedRule:
+    """A rule extracted from buildlog text by an LLM."""
+
+    rule: str
+    category: str  # architectural/workflow/tool_usage/domain_knowledge
+    severity: str = "info"  # critical/major/minor/info
+    scope: str = "global"  # global/module/function
+    applicability: list[str] = field(default_factory=list)
+    context: str | None = None  # when to apply
+    antipattern: str | None = None  # what violation looks like
+    rationale: str | None = None  # why it matters
+
+    def __post_init__(self) -> None:
+        if self.severity not in VALID_SEVERITIES:
+            self.severity = "info"
+        if self.scope not in VALID_SCOPES:
+            self.scope = "global"
+        if self.category not in VALID_CATEGORIES:
+            self.category = "architectural"
+
+
+@dataclass
+class RuleScoring:
+    """Severity/scope/applicability scoring for a rule."""
+
+    severity: str = "info"
+    scope: str = "global"
+    applicability: list[str] = field(default_factory=list)
+
+
+# --- Provider config ---
+
+
+@dataclass
+class LLMConfig:
+    """Configuration for an LLM provider."""
+
+    provider: str  # "ollama", "anthropic", "openai", ...
+    model: str | None = None  # None = auto-detect or provider default
+    base_url: str | None = None  # Override endpoint
+    api_key: str | None = None  # From config or env var
+
+    def __repr__(self) -> str:
+        """Redact api_key to prevent accidental exposure in logs/tracebacks."""
+        key_display = "***" if self.api_key else "None"
+        return (
+            f"LLMConfig(provider={self.provider!r}, model={self.model!r}, "
+            f"base_url={self.base_url!r}, api_key={key_display})"
+        )
+
+    @classmethod
+    def from_buildlog_config(cls, buildlog_dir: Path) -> LLMConfig | None:
+        """Read from .buildlog/config.yml [llm] section."""
+        config_path = buildlog_dir / ".buildlog" / "config.yml"
+        if not config_path.exists():
+            return None
+
+        try:
+            import yaml
+        except ImportError:
+            logger.debug("PyYAML not available, skipping config file")
+            return None
+
+        try:
+            data = yaml.safe_load(config_path.read_text())
+        except Exception:
+            logger.warning("Failed to parse %s", config_path)
+            return None
+
+        if not isinstance(data, dict):
+            return None
+
+        llm_config = data.get("llm")
+        if not isinstance(llm_config, dict):
+            return None
+
+        provider = llm_config.get("provider")
+        if not provider:
+            return None
+
+        return cls(
+            provider=str(provider),
+            model=llm_config.get("model"),
+            base_url=llm_config.get("base_url"),
+            api_key=llm_config.get("api_key"),
+        )
+
+    @classmethod
+    def auto_detect(cls) -> LLMConfig | None:
+        """Ollama running? -> use it. ANTHROPIC_API_KEY? -> use that. Else None."""
+        # Try Ollama first (local, no API key needed)
+        if _is_ollama_available():
+            return cls(provider="ollama")
+
+        # Try Anthropic (cloud)
+        api_key = os.environ.get("ANTHROPIC_API_KEY")
+        if api_key:
+            return cls(provider="anthropic", api_key=api_key)
+
+        return None
+
+
+def _is_ollama_available() -> bool:
+    """Check if Ollama is running and accessible."""
+    try:
+        import ollama as ollama_lib
+
+        ollama_lib.list()
+        return True
+    except Exception:
+        return False
+
+
+# --- Interface ---
+
+
+@runtime_checkable
+class LLMBackend(Protocol):
+    """Protocol for LLM backends."""
+
+    def extract_rules(self, entry_text: str) -> list[ExtractedRule]:
+        """Extract structured rules from buildlog entry text."""
+        ...
+
+    def select_canonical(self, candidates: list[str]) -> str:
+        """Given similar rules, produce the single best canonical form."""
+        ...
+
+    def score_rule(self, rule: str, context: str) -> RuleScoring:
+        """Score a rule with severity/scope/applicability."""
+        ...
+
+
+# --- Prompts ---
+
+EXTRACT_RULES_PROMPT = """\
+You are analyzing a buildlog entry's Improvements section. Extract actionable rules.
+
+For each rule, return a JSON array of objects with these fields:
+- "rule": string — the actionable rule in imperative form
+- "category": string — one of: architectural, workflow, tool_usage, domain_knowledge
+- "severity": string — one of: critical, major, minor, info
+- "scope": string — one of: global, module, function
+- "applicability": array of strings — contexts where relevant (e.g., "python", "api-design")
+- "context": string or null — when to apply this rule
+- "antipattern": string or null — what violation looks like
+- "rationale": string or null — why it matters
+
+Return ONLY a JSON array. No markdown, no explanation.
+
+Text to analyze:
+{text}
+"""
+
+SELECT_CANONICAL_PROMPT = """\
+Given these similar rules, produce the single best canonical form.
+The canonical rule should be clear, concise, and actionable.
+
+Similar rules:
+{candidates}
+
+Return ONLY the canonical rule text as a plain string. No JSON, no quotes, no explanation.
+"""
+
+SCORE_RULE_PROMPT = """\
+Score this rule for severity, scope, and applicability.
+
+Rule: {rule}
+Context: {context}
+
+Return ONLY a JSON object with:
+- "severity": one of: critical, major, minor, info
+- "scope": one of: global, module, function
+- "applicability": array of strings (contexts where relevant)
+
+No markdown, no explanation.
+"""
+
+
+def _parse_json_response(text: str) -> list | dict:
+    """Parse JSON from LLM response, handling markdown code blocks."""
+    text = text.strip()
+    # Strip markdown code blocks
+    if text.startswith("```"):
+        lines = text.split("\n")
+        # Remove first and last lines (``` markers)
+        lines = [ln for ln in lines[1:] if not ln.strip().startswith("```")]
+        text = "\n".join(lines)
+    return json.loads(text)
+
+
+# --- Rate limiting ---
+
+# Minimum seconds between API calls (per-backend instance).
+_MIN_CALL_INTERVAL = 0.5
+
+
+class _RateLimiter:
+    """Simple per-instance rate limiter to prevent API abuse."""
+
+    def __init__(self, min_interval: float = _MIN_CALL_INTERVAL):
+        self._min_interval = min_interval
+        self._last_call: float = 0.0
+
+    def wait(self) -> None:
+        """Block until min_interval has elapsed since last call."""
+        now = time.monotonic()
+        elapsed = now - self._last_call
+        if elapsed < self._min_interval:
+            time.sleep(self._min_interval - elapsed)
+        self._last_call = time.monotonic()
+
+
+# --- Implementations ---
+
+
+class OllamaBackend:
+    """LLM backend using Ollama (local)."""
+
+    def __init__(self, model: str | None = None, base_url: str | None = None):
+        self._model = model
+        self._base_url = base_url
+        self._resolved_model: str | None = None
+        self._rate_limiter = _RateLimiter()
+
+    def _get_model(self) -> str:
+        """Resolve model name, auto-detecting largest if not specified."""
+        if self._resolved_model:
+            return self._resolved_model
+
+        if self._model:
+            self._resolved_model = self._model
+            return self._resolved_model
+
+        # Auto-detect: pick largest pulled model
+        try:
+            import ollama as ollama_lib
+
+            models = ollama_lib.list()
+            if not models or not models.get("models"):
+                raise RuntimeError(
+                    "No Ollama models found. Pull one with: ollama pull llama3.2"
+                )
+
+            model_list = models["models"]
+            # Sort by size descending, pick largest
+            largest = max(model_list, key=lambda m: m.get("size", 0))
+            model_name: str = largest["name"]
+            self._resolved_model = model_name
+            logger.info("Auto-detected Ollama model: %s", model_name)
+            return model_name
+        except ImportError:
+            raise ImportError(
+                "ollama package is required. Install with: pip install buildlog[ollama]"
+            )
+
+    def _chat(self, prompt: str) -> str:
+        """Send a prompt to Ollama and return the response text."""
+        self._rate_limiter.wait()
+        import ollama as ollama_lib
+
+        kwargs = {
+            "model": self._get_model(),
+            "messages": [{"role": "user", "content": prompt}],
+        }
+        if self._base_url:
+            client = ollama_lib.Client(host=self._base_url)
+            response = client.chat(**kwargs)
+        else:
+            response = ollama_lib.chat(**kwargs)
+        return response["message"]["content"]
+
+    def extract_rules(self, entry_text: str) -> list[ExtractedRule]:
+        """Extract structured rules from buildlog entry text."""
+        prompt = EXTRACT_RULES_PROMPT.format(text=entry_text)
+        try:
+            response = self._chat(prompt)
+            parsed = _parse_json_response(response)
+            if not isinstance(parsed, list):
+                parsed = [parsed]
+            return [ExtractedRule(**item) for item in parsed]
+        except Exception as e:
+            logger.warning("Ollama extraction failed: %s", e)
+            return []
+
+    def select_canonical(self, candidates: list[str]) -> str:
+        """Given similar rules, produce the single best canonical form."""
+        numbered = "\n".join(f"{i+1}. {c}" for i, c in enumerate(candidates))
+        prompt = SELECT_CANONICAL_PROMPT.format(candidates=numbered)
+        try:
+            response = self._chat(prompt)
+            return response.strip().strip('"').strip("'")
+        except Exception as e:
+            logger.warning("Ollama canonical selection failed: %s", e)
+            return min(candidates, key=len)
+
+    def score_rule(self, rule: str, context: str) -> RuleScoring:
+        """Score a rule with severity/scope/applicability."""
+        prompt = SCORE_RULE_PROMPT.format(rule=rule, context=context)
+        try:
+            response = self._chat(prompt)
+            parsed = _parse_json_response(response)
+            if isinstance(parsed, dict):
+                return RuleScoring(
+                    severity=parsed.get("severity", "info"),
+                    scope=parsed.get("scope", "global"),
+                    applicability=parsed.get("applicability", []),
+                )
+        except Exception as e:
+            logger.warning("Ollama scoring failed: %s", e)
+        return RuleScoring()
+
+
+class AnthropicBackend:
+    """LLM backend using Anthropic Claude API."""
+
+    def __init__(
+        self,
+        model: str | None = None,
+        api_key: str | None = None,
+    ):
+        self._model = model or "claude-haiku-4-20250514"
+        self._api_key = api_key or os.environ.get("ANTHROPIC_API_KEY")
+        self._client = None
+        self._rate_limiter = _RateLimiter()
+
+    def __repr__(self) -> str:
+        """Redact API key from repr to prevent exposure in logs/tracebacks."""
+        return f"AnthropicBackend(model={self._model!r}, api_key=***)"
+
+    def _get_client(self):
+        """Lazy-load the Anthropic client."""
+        if self._client is None:
+            try:
+                import anthropic
+            except ImportError:
+                raise ImportError(
+                    "anthropic package is required. Install with: pip install buildlog[anthropic]"
+                )
+            if not self._api_key:
+                raise ValueError("ANTHROPIC_API_KEY is required for Anthropic backend")
+            self._client = anthropic.Anthropic(api_key=self._api_key)
+        return self._client
+
+    def _chat(self, prompt: str) -> str:
+        """Send a prompt to Claude and return the response text."""
+        self._rate_limiter.wait()
+        client = self._get_client()
+        response = client.messages.create(
+            model=self._model,
+            max_tokens=2048,
+            messages=[{"role": "user", "content": prompt}],
+        )
+        return response.content[0].text
+
+    def extract_rules(self, entry_text: str) -> list[ExtractedRule]:
+        """Extract structured rules from buildlog entry text."""
+        prompt = EXTRACT_RULES_PROMPT.format(text=entry_text)
+        try:
+            response = self._chat(prompt)
+            parsed = _parse_json_response(response)
+            if not isinstance(parsed, list):
+                parsed = [parsed]
+            return [ExtractedRule(**item) for item in parsed]
+        except Exception as e:
+            logger.warning("Anthropic extraction failed: %s", e)
+            return []
+
+    def select_canonical(self, candidates: list[str]) -> str:
+        """Given similar rules, produce the single best canonical form."""
+        numbered = "\n".join(f"{i+1}. {c}" for i, c in enumerate(candidates))
+        prompt = SELECT_CANONICAL_PROMPT.format(candidates=numbered)
+        try:
+            response = self._chat(prompt)
+            return response.strip().strip('"').strip("'")
+        except Exception as e:
+            logger.warning("Anthropic canonical selection failed: %s", e)
+            return min(candidates, key=len)
+
+    def score_rule(self, rule: str, context: str) -> RuleScoring:
+        """Score a rule with severity/scope/applicability."""
+        prompt = SCORE_RULE_PROMPT.format(rule=rule, context=context)
+        try:
+            response = self._chat(prompt)
+            parsed = _parse_json_response(response)
+            if isinstance(parsed, dict):
+                return RuleScoring(
+                    severity=parsed.get("severity", "info"),
+                    scope=parsed.get("scope", "global"),
+                    applicability=parsed.get("applicability", []),
+                )
+        except Exception as e:
+            logger.warning("Anthropic scoring failed: %s", e)
+        return RuleScoring()
+
+
+# --- Registry ---
+
+_PROVIDERS: dict[str, type] = {
+    "ollama": OllamaBackend,
+    "anthropic": AnthropicBackend,
+}
+# Public read-only view. Use register_provider() to add entries.
+PROVIDERS: MappingProxyType[str, type] = MappingProxyType(_PROVIDERS)
+
+
+def register_provider(name: str, cls: type) -> None:
+    """Register a new LLM provider backend.
+
+    This is the only sanctioned way to mutate the provider registry.
+    """
+    if not isinstance(name, str) or not name.strip():
+        raise ValueError("Provider name must be a non-empty string")
+    _PROVIDERS[name] = cls
+
+
+def get_llm_backend(
+    config: LLMConfig | None = None,
+    buildlog_dir: Path | None = None,
+) -> LLMBackend | None:
+    """Get an LLM backend using the provider cascade.
+
+    Resolution order:
+    1. Explicit config parameter (highest priority)
+    2. Config file (.buildlog/config.yml)
+    3. Auto-detect: Ollama -> Anthropic -> None
+
+    Returns None if no provider is available (regex fallback).
+    """
+    # 1. Explicit config
+    if config is None and buildlog_dir is not None:
+        # 2. Config file
+        config = LLMConfig.from_buildlog_config(buildlog_dir)
+
+    if config is None:
+        # 3. Auto-detect
+        config = LLMConfig.auto_detect()
+
+    if config is None:
+        logger.info("No LLM provider available, using regex fallback")
+        return None
+
+    provider_cls = _PROVIDERS.get(config.provider)
+    if provider_cls is None:
+        logger.warning("Unknown LLM provider: %s", config.provider)
+        return None
+
+    try:
+        kwargs: dict = {}
+        if config.model:
+            kwargs["model"] = config.model
+        if config.provider == "ollama" and config.base_url:
+            kwargs["base_url"] = config.base_url
+        if config.provider == "anthropic" and config.api_key:
+            kwargs["api_key"] = config.api_key
+
+        backend = provider_cls(**kwargs)
+        logger.info("Using LLM provider: %s", config.provider)
+        return backend
+    except Exception as e:
+        logger.warning("Failed to initialize %s backend: %s", config.provider, e)
+        return None

buildlog/mcp/server.py

@@ -5,9 +5,12 @@ from __future__ import annotations
 from mcp.server.fastmcp import FastMCP
 
 from buildlog.mcp.tools import (
+    buildlog_bandit_status,
     buildlog_diff,
-    buildlog_end_session,
+    buildlog_experiment_end,
+    buildlog_experiment_metrics,
     buildlog_experiment_report,
+    buildlog_experiment_start,
     buildlog_gauntlet_accept_risk,
     buildlog_gauntlet_issues,
     buildlog_learn_from_review,
@@ -16,8 +19,6 @@ from buildlog.mcp.tools import (
     buildlog_promote,
     buildlog_reject,
     buildlog_rewards,
-    buildlog_session_metrics,
-    buildlog_start_session,
     buildlog_status,
 )
 
@@ -33,16 +34,19 @@ mcp.tool()(buildlog_log_reward)
 mcp.tool()(buildlog_rewards)
 
 # Session tracking tools (experiment infrastructure)
-mcp.tool()(buildlog_start_session)
-mcp.tool()(buildlog_end_session)
+mcp.tool()(buildlog_experiment_start)
+mcp.tool()(buildlog_experiment_end)
 mcp.tool()(buildlog_log_mistake)
-mcp.tool()(buildlog_session_metrics)
+mcp.tool()(buildlog_experiment_metrics)
 mcp.tool()(buildlog_experiment_report)
 
 # Gauntlet loop tools
 mcp.tool()(buildlog_gauntlet_issues)
 mcp.tool()(buildlog_gauntlet_accept_risk)
 
+# Bandit tools
+mcp.tool()(buildlog_bandit_status)
+
 
 def main() -> None:
     """Run the MCP server."""

buildlog/mcp/tools.py

@@ -12,6 +12,7 @@ from typing import Literal
 from buildlog.core import (
     diff,
     end_session,
+    get_bandit_status,
     get_experiment_report,
     get_rewards,
     get_session_metrics,
@@ -52,17 +53,17 @@ def buildlog_status(
 
 def buildlog_promote(
     skill_ids: list[str],
-    target: Literal["claude_md", "settings_json", "skill"] = "claude_md",
+    target: str = "claude_md",
     buildlog_dir: str = "buildlog",
 ) -> dict:
     """Promote skills to your agent's rules.
 
-    Writes selected skills to CLAUDE.md, .claude/settings.json, or
-    .claude/skills/buildlog-learned/SKILL.md (Anthropic Agent Skills format).
+    Writes selected skills to agent-specific rule files.
 
     Args:
         skill_ids: List of skill IDs to promote (e.g., ["arch-b0fcb62a1e"])
-        target: Where to write rules ("claude_md", "settings_json", or "skill")
+        target: Where to write rules. One of: claude_md, settings_json,
+            skill, cursor, copilot, windsurf, continue_dev.
         buildlog_dir: Path to buildlog directory
 
     Returns:
@@ -262,36 +263,47 @@ def buildlog_rewards(
 # -----------------------------------------------------------------------------
 
 
-def buildlog_start_session(
+def buildlog_experiment_start(
     error_class: str | None = None,
     notes: str | None = None,
+    select_k: int = 3,
     buildlog_dir: str = "buildlog",
 ) -> dict:
-    """Start a new experiment session.
+    """Start a new experiment session with Thompson Sampling rule selection.
 
-    Begins tracking for a learning experiment. Captures the current
-    set of active rules to measure learning over time.
+    Begins tracking for a learning experiment. Uses Thompson Sampling
+    to select which rules will be "active" for this session based on
+    the error class context.
+
+    The selected rules will receive feedback:
+    - Negative feedback (reward=0) when log_mistake() is called
+    - Explicit feedback when log_reward() is called
+
+    This teaches the bandit which rules are effective for which contexts.
 
     Args:
-        error_class: Error class being targeted (e.g., "missing_test")
+        error_class: Error class being targeted (e.g., "missing_test").
+                    This is the CONTEXT for contextual bandits.
         notes: Notes about this session
+        select_k: Number of rules to select via Thompson Sampling
         buildlog_dir: Path to buildlog directory
 
     Returns:
-        Dict with session_id, error_class, rules_count, message
+        Dict with session_id, error_class, rules_count, selected_rules, message
 
     Example:
-        buildlog_start_session(error_class="missing_test")
+        buildlog_start_session(error_class="type-errors", select_k=5)
     """
     result = start_session(
         Path(buildlog_dir),
         error_class=error_class,
         notes=notes,
+        select_k=select_k,
     )
     return asdict(result)
 
 
-def buildlog_end_session(
+def buildlog_experiment_end(
     entry_file: str | None = None,
     notes: str | None = None,
     buildlog_dir: str = "buildlog",
@@ -358,7 +370,7 @@ def buildlog_log_mistake(
     return asdict(result)
 
 
-def buildlog_session_metrics(
+def buildlog_experiment_metrics(
     session_id: str | None = None,
     buildlog_dir: str = "buildlog",
 ) -> dict:
@@ -407,6 +419,42 @@ def buildlog_experiment_report(
     return get_experiment_report(Path(buildlog_dir))
 
 
+def buildlog_bandit_status(
+    buildlog_dir: str = "buildlog",
+    context: str | None = None,
+    top_k: int = 10,
+) -> dict:
+    """Get Thompson Sampling bandit status and rule rankings.
+
+    Shows the bandit's learned beliefs about which rules are effective
+    for each error class context. Higher mean = bandit believes rule
+    is more effective.
+
+    The bandit uses Beta distributions to model uncertainty:
+    - High variance (wide CI) = uncertain, will explore more
+    - Low variance (narrow CI) = confident, will exploit
+
+    Args:
+        buildlog_dir: Path to buildlog directory
+        context: Specific error class to filter by (optional)
+        top_k: Number of top rules to show per context
+
+    Returns:
+        Dict with:
+            - summary: Total contexts, arms, observations
+            - top_rules: Best rules per context by expected value
+            - all_rules: Full stats if filtering by context
+
+    Example:
+        # See all bandit state
+        buildlog_bandit_status()
+
+        # See state for specific error class
+        buildlog_bandit_status(context="type-errors")
+    """
+    return get_bandit_status(Path(buildlog_dir), context, top_k)
+
+
 # -----------------------------------------------------------------------------
 # Gauntlet Loop MCP Tools
 # -----------------------------------------------------------------------------