monkeybrain-runtime 1.0.0__py3-none-any.whl

src/cerebellum/resolve_entity_capability.py

@@ -0,0 +1,259 @@
+"""ResolveEntityCapability — Entity Resolution as a first-class capability.
+
+This module converts the existing entity resolution logic into a standard
+capability that conforms to the ICapability interface.
+
+Key Principles:
+- No special preprocessing stage
+- Treated identically to other capabilities
+- Augments execution state
+- Contributes to policy learning
+- Selected by Bellman optimizer
+"""
+
+from __future__ import annotations
+
+import time
+from typing import Any
+
+from src.monkey_brain.kernel.capability_interface import EntityResolutionCapability
+from src.monkey_brain.kernel.execution_state import ExecutionState, CapabilityResult
+
+
+class ResolveEntityCapability(EntityResolutionCapability):
+    """Entity Resolution as a standard capability.
+    
+    Responsibilities:
+    - Parse question
+    - Identify entities
+    - Identify entity types
+    - Populate execution state
+    - Never invoke planners directly
+    - Simply augment execution state
+    """
+    
+    def __init__(self, db: Any = None):
+        self.db = db
+        
+    @property
+    def capability_name(self) -> str:
+        return "resolve_entity"
+    
+    @property
+    def capability_type(self) -> str:
+        return "entity_resolution"
+    
+    async def execute(self, state: ExecutionState, **kwargs: Any) -> CapabilityResult:
+        """Execute entity resolution and return updated state."""
+        start_time = time.time()
+        
+        # Create a copy of the state to avoid mutation
+        updated_state = ExecutionState(
+            question=state.question,
+            intent=state.intent,
+            phase="entity_resolution"
+        )
+        
+        # Copy over existing data
+        updated_state.entities = state.entities.copy()
+        updated_state.resolved_entities = state.resolved_entities.copy()
+        updated_state.hierarchy = state.hierarchy.copy()
+        updated_state.relationships = state.relationships.copy()
+        updated_state.graph_entities = state.graph_entities.copy()
+        updated_state.documents = state.documents.copy()
+        updated_state.web_results = state.web_results.copy()
+        updated_state.execution_stats = state.execution_stats.copy()
+        updated_state.observations = state.observations.copy()
+        updated_state.policy_updates = state.policy_updates.copy()
+        updated_state.capability_history = state.capability_history.copy()
+        updated_state.execution_trace = state.execution_trace.copy()
+        updated_state.current_answer = state.current_answer
+        updated_state.answer_quality = state.answer_quality
+        updated_state.confidence = state.confidence
+        updated_state.errors = state.errors.copy()
+        updated_state.warnings = state.warnings.copy()
+        
+        # Execute entity resolution
+        try:
+            # Determine entity types to resolve based on question and intent
+            entity_types = self._determine_entity_types(state.question, state.intent)
+            
+            # Resolve each entity type
+            for entity_type in entity_types:
+                resolved_entity = await self._resolve_entity(entity_type, state.question)
+                if resolved_entity:
+                    updated_state.add_entity(entity_type, resolved_entity)
+                    
+                    # Add observation for policy learning
+                    updated_state.add_observation({
+                        "type": "entity_resolution",
+                        "entity_type": entity_type,
+                        "entity_id": resolved_entity.get("id", ""),
+                        "entity_name": resolved_entity.get("name", ""),
+                        "confidence": 0.9  # High confidence for resolved entities
+                    })
+            
+            # Record capability execution
+            updated_state.record_capability_execution(self.capability_name, {
+                "entity_types": entity_types,
+                "resolved_count": len(updated_state.entities),
+                "timestamp": self._now()
+            })
+            
+            # Set phase and confidence
+            updated_state.phase = "entity_resolution"
+            updated_state.confidence = self.compute_confidence(state)
+            
+            # Compute reward based on resolution success
+            reward = self._compute_reward(len(entity_types), len(updated_state.entities))
+            
+            # Add policy update observation
+            updated_state.add_policy_update({
+                "capability": self.capability_name,
+                "reward": reward,
+                "execution_cost": 0.3,  # Low cost for entity resolution
+                "confidence": updated_state.confidence
+            })
+            
+            end_time = time.time()
+            latency = end_time - start_time
+            
+            return CapabilityResult(
+                updated_state=updated_state,
+                capability_name=self.capability_name,
+                success=True,
+                latency=latency,
+                timestamp=self._now(),
+                observations=[{
+                    "type": "entity_resolution_completed",
+                    "resolved_entities": len(updated_state.entities),
+                    "entity_types": entity_types,
+                    "latency": latency
+                }],
+                transition_reward=reward,
+                execution_cost=0.3,
+                confidence=updated_state.confidence,
+                execution_metadata={
+                    "entity_types_attempted": entity_types,
+                    "entities_resolved": [e.get("id") for e in updated_state.entities],
+                    "resolution_method": "ontology_grounded"
+                }
+            )
+            
+        except Exception as e:
+            end_time = time.time()
+            latency = end_time - start_time
+            
+            updated_state.add_error(f"Entity resolution failed: {str(e)}")
+            updated_state.phase = "failed"
+            
+            return CapabilityResult(
+                updated_state=updated_state,
+                capability_name=self.capability_name,
+                success=False,
+                latency=latency,
+                timestamp=self._now(),
+                observations=[{
+                    "type": "entity_resolution_failed",
+                    "error": str(e),
+                    "latency": latency
+                }],
+                transition_reward=0.0,
+                execution_cost=0.3,
+                confidence=0.1,
+                execution_metadata={
+                    "error": str(e),
+                    "entity_types_attempted": self._determine_entity_types(state.question, state.intent)
+                }
+            )
+    
+    def can_execute(self, state: ExecutionState) -> bool:
+        """Determine if this capability can execute given current state."""
+        # Can execute if we have a question and haven't already resolved entities
+        return (bool(state.question.strip()) and 
+                len(state.entities) == 0 and
+                not state.has_errors() and
+                not state.is_complete())
+    
+    def estimate_reward(self, state: ExecutionState) -> float:
+        """Estimate expected reward for executing this capability."""
+        # High reward if no entities resolved yet
+        if len(state.entities) == 0:
+            return 0.8
+        return 0.2
+    
+    def estimate_cost(self, state: ExecutionState) -> float:
+        """Estimate execution cost for this capability."""
+        # Low cost - entity resolution is fast
+        return 0.2
+    
+    def compute_confidence(self, state: ExecutionState) -> float:
+        """Compute confidence that this capability will succeed."""
+        # High confidence if we have a reasonable question
+        if len(state.question.strip()) > 10:
+            return 0.9
+        return 0.5
+    
+    def _determine_entity_types(self, question: str, intent: str) -> list[str]:
+        """Determine which entity types to resolve based on question and intent."""
+        # Common entity types in the system
+        common_entity_types = ["line", "plant", "stage", "workstation", "machine", "equipment"]
+        
+        # Add intent-specific entity types
+        intent_entity_map = {
+            "line_question": ["line", "plant"],
+            "stage_question": ["stage", "line", "plant"],
+            "workstation_question": ["workstation", "stage", "line"],
+            "machine_question": ["machine", "workstation", "stage"],
+            "plant_question": ["plant"],
+            "hierarchy_question": ["plant", "line", "stage", "workstation", "machine"]
+        }
+        
+        # Get entity types based on intent
+        intent_specific = intent_entity_map.get(intent.lower(), [])
+        
+        # Combine and deduplicate
+        entity_types = list(set(common_entity_types + intent_specific))
+        
+        return entity_types
+    
+    async def _resolve_entity(self, entity_type: str, question: str) -> dict[str, Any] | None:
+        """Resolve a single entity using the existing resolve_entity function."""
+        if self.db is None:
+            return None
+            
+        # Use the existing resolve_entity function
+        from src.monkey_brain.kernel.intents.helpers import resolve_entity
+        
+        try:
+            return await resolve_entity(
+                db=self.db,
+                entity_type=entity_type,
+                question=question
+            )
+        except Exception as e:
+            # Log error but don't fail the entire capability.
+            # The caller decides whether to surface this as a warning.
+            return None
+    
+    def _compute_reward(self, attempted: int, resolved: int) -> float:
+        """Compute reward based on resolution success."""
+        if attempted == 0:
+            return 0.0
+        
+        success_rate = resolved / attempted
+        
+        # High reward for successful resolution
+        if success_rate >= 0.8:
+            return 0.9
+        elif success_rate >= 0.5:
+            return 0.7
+        elif success_rate > 0:
+            return 0.5
+        else:
+            return 0.2
+    
+    def _now(self) -> str:
+        """Get current timestamp in ISO format."""
+        from datetime import datetime, timezone
+        return datetime.now(timezone.utc).isoformat()

src/cingulate/benchmark/__init__.py

@@ -0,0 +1,23 @@
+"""Benchmark — testing and validation framework.
+
+Provides:
+- BenchmarkRunner: executes benchmarks
+- ScenarioRunner: runs end-to-end scenarios
+- ResultValidator: validates results
+- ReportGenerator: generates reports
+"""
+
+from src.cingulate.benchmark.runner import BenchmarkRunner, BenchmarkResult
+from src.cingulate.benchmark.scenario_runner import ScenarioRunner, ScenarioResult
+from src.cingulate.benchmark.validator import ResultValidator, ValidationResult
+from src.cingulate.benchmark.reporter import ReportGenerator
+
+__all__ = [
+    "BenchmarkRunner",
+    "BenchmarkResult",
+    "ScenarioRunner",
+    "ScenarioResult",
+    "ResultValidator",
+    "ValidationResult",
+    "ReportGenerator",
+]

src/cingulate/benchmark/reporter.py

@@ -0,0 +1,102 @@
+"""Report Generator — generates benchmark reports.
+
+Generates:
+- Console reports
+- JSON reports
+- Summary reports
+"""
+
+from __future__ import annotations
+
+import json
+from datetime import datetime, timezone
+from typing import Any
+
+
+class ReportGenerator:
+    """Generates benchmark reports.
+    
+    Responsibilities:
+    - Format results for console
+    - Export results to JSON
+    - Generate summaries
+    """
+    
+    def __init__(self):
+        self._reports: list[dict[str, Any]] = []
+    
+    def console_report(self, results: list[dict[str, Any]], title: str = "Benchmark Report") -> str:
+        """Generate a console-formatted report."""
+        lines = []
+        lines.append("=" * 70)
+        lines.append(f"  {title}")
+        lines.append(f"  Generated: {datetime.now(timezone.utc).isoformat()}")
+        lines.append("=" * 70)
+        lines.append("")
+        
+        total = len(results)
+        passed = sum(1 for r in results if r.get("status") == "passed")
+        failed = sum(1 for r in results if r.get("status") == "failed")
+        errors = sum(1 for r in results if r.get("status") == "error")
+        
+        lines.append(f"  Summary: {passed}/{total} passed, {failed} failed, {errors} errors")
+        lines.append(f"  Pass Rate: {passed/total*100:.1f}%" if total else "  Pass Rate: N/A")
+        lines.append("")
+        lines.append("-" * 70)
+        
+        for r in results:
+            status = r.get("status", "unknown")
+            name = r.get("name", "unnamed")
+            latency = r.get("latency_ms", 0)
+            
+            if status == "passed":
+                icon = "✓"
+            elif status == "failed":
+                icon = "✗"
+            else:
+                icon = "!"
+            
+            lines.append(f"  {icon} {name:<40} {latency:>8.2f}ms  {status}")
+        
+        lines.append("-" * 70)
+        
+        # Print Q&A for each result
+        lines.append("")
+        lines.append("  DETAILED RESULTS")
+        lines.append("-" * 70)
+        
+        for r in results:
+            if r.get("status") in ("passed", "failed"):
+                lines.append("")
+                lines.append(f"  Q: {r.get('input', {}).get('question', r.get('name', 'N/A'))}")
+                lines.append(f"  A: {r.get('actual', {}).get('answer', r.get('actual_output', {}).get('intent', 'N/A'))}")
+                lines.append(f"  Expected: {r.get('expected', {})}")
+                lines.append(f"  Status: {r.get('status')}")
+                
+                if r.get("metrics"):
+                    lines.append(f"  Metrics: {json.dumps(r['metrics'], indent=2)}")
+        
+        lines.append("=" * 70)
+        
+        report = "\n".join(lines)
+        self._reports.append({"type": "console", "content": report, "timestamp": datetime.now(timezone.utc).isoformat()})
+        return report
+    
+    def json_report(self, results: list[dict[str, Any]], title: str = "Benchmark Report") -> dict[str, Any]:
+        """Generate a JSON report."""
+        report = {
+            "title": title,
+            "timestamp": datetime.now(timezone.utc).isoformat(),
+            "summary": {
+                "total": len(results),
+                "passed": sum(1 for r in results if r.get("status") == "passed"),
+                "failed": sum(1 for r in results if r.get("status") == "failed"),
+                "errors": sum(1 for r in results if r.get("status") == "error"),
+            },
+            "results": results,
+        }
+        self._reports.append({"type": "json", "content": report})
+        return report
+    
+    def get_reports(self) -> list[dict[str, Any]]:
+        return list(self._reports)

src/cingulate/benchmark/runner.py

@@ -0,0 +1,159 @@
+"""Benchmark Runner — executes benchmarks and validates results.
+
+Every benchmark executes against deterministic grounded data.
+Expected outputs are known before execution.
+"""
+
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import Any
+from uuid import uuid4
+
+
+@dataclass
+class BenchmarkResult:
+    """Result of a single benchmark execution."""
+    
+    benchmark_id: str = ""
+    name: str = ""
+    status: str = "pending"  # pending | passed | failed | error
+    input_data: dict[str, Any] = field(default_factory=dict)
+    expected_output: dict[str, Any] = field(default_factory=dict)
+    actual_output: dict[str, Any] = field(default_factory=dict)
+    metrics: dict[str, Any] = field(default_factory=dict)
+    assertions: list[dict[str, Any]] = field(default_factory=list)
+    latency_ms: float = 0.0
+    error: str | None = None
+    timestamp: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat())
+    
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "benchmark_id": self.benchmark_id,
+            "name": self.name,
+            "status": self.status,
+            "input": self.input_data,
+            "expected": self.expected_output,
+            "actual": self.actual_output,
+            "metrics": self.metrics,
+            "assertions": self.assertions,
+            "latency_ms": round(self.latency_ms, 2),
+            "error": self.error,
+        }
+
+
+class BenchmarkRunner:
+    """Executes benchmarks and validates results.
+    
+    Responsibilities:
+    - Execute benchmarks
+    - Validate against expected outputs
+    - Collect performance metrics
+    - Generate reports
+    """
+    
+    def __init__(self):
+        self._results: list[BenchmarkResult] = []
+        self._benchmarks: list[dict[str, Any]] = []
+    
+    def register(self, name: str, input_data: dict, expected: dict, validator: Any = None) -> str:
+        """Register a benchmark."""
+        benchmark_id = f"bench-{uuid4().hex[:8]}"
+        self._benchmarks.append({
+            "id": benchmark_id,
+            "name": name,
+            "input_data": input_data,
+            "expected": expected,
+            "validator": validator,
+        })
+        return benchmark_id
+    
+    async def run(self, name: str, fn: Any, input_data: dict[str, Any], expected: dict[str, Any]) -> BenchmarkResult:
+        """Run a single benchmark."""
+        start = time.monotonic()
+        
+        result = BenchmarkResult(
+            benchmark_id=f"bench-{uuid4().hex[:8]}",
+            name=name,
+            input_data=input_data,
+            expected_output=expected,
+        )
+        
+        try:
+            actual = await fn(input_data)
+            result.actual_output = actual
+            result.latency_ms = (time.monotonic() - start) * 1000
+            
+            # Validate
+            assertions = self._validate(expected, actual)
+            result.assertions = assertions
+            
+            all_passed = all(a["passed"] for a in assertions)
+            result.status = "passed" if all_passed else "failed"
+            
+            result.metrics = {
+                "latency_ms": result.latency_ms,
+                "assertions_total": len(assertions),
+                "assertions_passed": sum(1 for a in assertions if a["passed"]),
+                "assertions_failed": sum(1 for a in assertions if not a["passed"]),
+            }
+            
+        except Exception as e:
+            result.status = "error"
+            result.error = str(e)
+            result.latency_ms = (time.monotonic() - start) * 1000
+        
+        self._results.append(result)
+        return result
+    
+    def _validate(self, expected: dict[str, Any], actual: dict[str, Any]) -> list[dict[str, Any]]:
+        """Validate expected vs actual output."""
+        assertions = []
+        
+        for key, expected_value in expected.items():
+            actual_value = actual.get(key)
+            
+            if isinstance(expected_value, dict):
+                # Deep comparison for nested dicts
+                for sub_key, sub_expected in expected_value.items():
+                    sub_actual = actual_value.get(sub_key) if isinstance(actual_value, dict) else None
+                    passed = sub_actual == sub_expected if sub_expected is not None else True
+                    assertions.append({
+                        "field": f"{key}.{sub_key}",
+                        "expected": sub_expected,
+                        "actual": sub_actual,
+                        "passed": passed,
+                    })
+            else:
+                passed = actual_value == expected_value if expected_value is not None else True
+                assertions.append({
+                    "field": key,
+                    "expected": expected_value,
+                    "actual": actual_value,
+                    "passed": passed,
+                })
+        
+        return assertions
+    
+    def get_results(self, status: str | None = None) -> list[BenchmarkResult]:
+        if status:
+            return [r for r in self._results if r.status == status]
+        return list(self._results)
+    
+    def summary(self) -> dict:
+        total = len(self._results)
+        passed = sum(1 for r in self._results if r.status == "passed")
+        failed = sum(1 for r in self._results if r.status == "failed")
+        errors = sum(1 for r in self._results if r.status == "error")
+        avg_latency = sum(r.latency_ms for r in self._results) / total if total else 0
+        
+        return {
+            "total": total,
+            "passed": passed,
+            "failed": failed,
+            "errors": errors,
+            "pass_rate": round(passed / total, 3) if total else 0,
+            "avg_latency_ms": round(avg_latency, 2),
+        }

src/cingulate/benchmark/scenario_runner.py

@@ -0,0 +1,150 @@
+"""Scenario Runner — runs complete end-to-end scenarios.
+
+Validates the complete execution pipeline.
+"""
+
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import Any, Callable
+from uuid import uuid4
+
+
+@dataclass
+class ScenarioStep:
+    """A single step in a scenario."""
+    
+    step_id: str = ""
+    name: str = ""
+    action: str = ""
+    input_data: dict[str, Any] = field(default_factory=dict)
+    expected_output: dict[str, Any] = field(default_factory=dict)
+    actual_output: dict[str, Any] = field(default_factory=dict)
+    passed: bool = False
+    latency_ms: float = 0.0
+    error: str | None = None
+
+
+@dataclass
+class ScenarioResult:
+    """Result of a complete scenario execution."""
+    
+    scenario_id: str = field(default_factory=lambda: f"scenario-{uuid4().hex[:8]}")
+    name: str = ""
+    status: str = "pending"
+    steps: list[ScenarioStep] = field(default_factory=list)
+    total_latency_ms: float = 0.0
+    metrics: dict[str, Any] = field(default_factory=dict)
+    timestamp: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat())
+    
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "scenario_id": self.scenario_id,
+            "name": self.name,
+            "status": self.status,
+            "steps": [
+                {
+                    "name": s.name,
+                    "passed": s.passed,
+                    "latency_ms": round(s.latency_ms, 2),
+                    "error": s.error,
+                }
+                for s in self.steps
+            ],
+            "total_latency_ms": round(self.total_latency_ms, 2),
+            "metrics": self.metrics,
+        }
+
+
+class ScenarioRunner:
+    """Runs complete end-to-end scenarios.
+    
+    Validates the complete execution pipeline.
+    """
+    
+    def __init__(self):
+        self._scenarios: list[dict[str, Any]] = []
+        self._results: list[ScenarioResult] = []
+    
+    def register(self, name: str, steps: list[dict[str, Any]]) -> str:
+        """Register a scenario."""
+        scenario_id = f"scenario-{uuid4().hex[:8]}"
+        self._scenarios.append({
+            "id": scenario_id,
+            "name": name,
+            "steps": steps,
+        })
+        return scenario_id
+    
+    async def run(self, name: str, steps: list[dict[str, Any]], context: dict[str, Any] | None = None) -> ScenarioResult:
+        """Run a complete scenario."""
+        start = time.monotonic()
+        context = context or {}
+        
+        result = ScenarioResult(name=name)
+        step_results = []
+        
+        for step_def in steps:
+            step = ScenarioStep(
+                step_id=step_def.get("id", f"step-{len(step_results)}"),
+                name=step_def.get("name", ""),
+                action=step_def.get("action", ""),
+                input_data=step_def.get("input", {}),
+                expected_output=step_def.get("expected", {}),
+            )
+            
+            step_start = time.monotonic()
+            
+            try:
+                fn = step_def.get("function")
+                if fn:
+                    actual = await fn(step.input_data, context)
+                    step.actual_output = actual
+                    step.passed = self._validate_step(step.expected_output, actual)
+                else:
+                    step.passed = True  # No validation needed
+            except Exception as e:
+                step.error = str(e)
+                step.passed = False
+            
+            step.latency_ms = (time.monotonic() - step_start) * 1000
+            step_results.append(step)
+        
+        result.steps = step_results
+        result.total_latency_ms = (time.monotonic() - start) * 1000
+        result.status = "passed" if all(s.passed for s in step_results) else "failed"
+        
+        result.metrics = {
+            "total_steps": len(step_results),
+            "passed_steps": sum(1 for s in step_results if s.passed),
+            "failed_steps": sum(1 for s in step_results if not s.passed),
+            "avg_step_latency_ms": round(sum(s.latency_ms for s in step_results) / len(step_results), 2) if step_results else 0,
+        }
+        
+        self._results.append(result)
+        return result
+    
+    def _validate_step(self, expected: dict[str, Any], actual: dict[str, Any]) -> bool:
+        """Validate a single step."""
+        for key, expected_value in expected.items():
+            actual_value = actual.get(key)
+            if expected_value is not None and actual_value != expected_value:
+                return False
+        return True
+    
+    def get_results(self, status: str | None = None) -> list[ScenarioResult]:
+        if status:
+            return [r for r in self._results if r.status == status]
+        return list(self._results)
+    
+    def summary(self) -> dict:
+        total = len(self._results)
+        passed = sum(1 for r in self._results if r.status == "passed")
+        return {
+            "total": total,
+            "passed": passed,
+            "failed": total - passed,
+            "pass_rate": round(passed / total, 3) if total else 0,
+        }