apsimo 1.3.0__py3-none-any.whl

apsimo/intelligence/cognition/metrics_collector.py

@@ -0,0 +1,217 @@
+"""Metrics collection for cognitive performance tracking."""
+from dataclasses import dataclass
+from typing import Dict, List, Optional
+from datetime import datetime, timedelta
+import statistics
+
+
+@dataclass
+class MetricObservation:
+    """Single metric observation."""
+    metric_type: str
+    value: float
+    domain: str
+    recorded_at: datetime
+    context: Optional[dict] = None
+
+
+class MetricsCollector:
+    """Collect and aggregate metrics for CPI computation."""
+
+    def __init__(self, graph: "ColonyGraph"):
+        self.graph = graph
+        self._buffer: List[MetricObservation] = []
+
+    async def record(
+        self,
+        metric_type: str,
+        value: float,
+        domain: str = "general",
+        context: Optional[dict] = None
+    ) -> None:
+        """Record a metric observation."""
+        observation = MetricObservation(
+            metric_type=metric_type,
+            value=value,
+            domain=domain,
+            recorded_at=datetime.now(),
+            context=context,
+        )
+
+        # Add to buffer
+        self._buffer.append(observation)
+
+        # Persist to graph
+        await self._persist(observation)
+
+    async def get_metrics(
+        self,
+        domain: str,
+        days: int = 7
+    ) -> Dict[str, float]:
+        """Get aggregated metrics for a domain."""
+        cutoff = datetime.now() - timedelta(days=days)
+
+        # Filter buffer
+        relevant = [
+            obs for obs in self._buffer
+            if obs.domain == domain and obs.recorded_at >= cutoff
+        ]
+
+        # Also fetch from graph
+        try:
+            graph_metrics = await self._fetch_from_graph(domain, days)
+        except (OSError, RuntimeError):
+            graph_metrics = []
+
+        # Combine
+        all_observations = relevant + graph_metrics
+
+        # Aggregate by type
+        by_type: Dict[str, List[float]] = {}
+        for obs in all_observations:
+            if obs.metric_type not in by_type:
+                by_type[obs.metric_type] = []
+            by_type[obs.metric_type].append(obs.value)
+
+        # Compute aggregates
+        result = {}
+        for metric_type, values in by_type.items():
+            if values:
+                result[metric_type] = statistics.mean(values)
+                result[f"{metric_type}_count"] = len(values)
+                if len(values) > 1:
+                    result[f"{metric_type}_std"] = statistics.stdev(values)
+
+        return result
+
+    async def record_prediction(self, correct: bool, confidence: float) -> None:
+        """Record prediction outcome."""
+        await self.record("prediction_outcome", 1.0 if correct else 0.0, "prediction")
+        await self.record("prediction_confidence", confidence, "prediction")
+
+    async def record_tool_usage(
+        self,
+        tool_name: str,
+        success: bool,
+        latency_ms: float
+    ) -> None:
+        """Record tool usage metrics."""
+        await self.record(f"tool_{tool_name}_success", 1.0 if success else 0.0, "tool_efficiency")
+        await self.record(f"tool_{tool_name}_latency", latency_ms, "tool_efficiency")
+
+    async def record_memory_retrieval(
+        self,
+        relevance_score: float,
+        latency_ms: float
+    ) -> None:
+        """Record memory retrieval metrics."""
+        await self.record("relevance_score", relevance_score, "retrieval")
+        await self.record("retrieval_latency", latency_ms, "retrieval")
+
+    async def record_task_completion(
+        self,
+        task_type: str,
+        duration_minutes: float
+    ) -> None:
+        """Record task completion metrics."""
+        await self.record(f"task_{task_type}_duration", duration_minutes, "goal_progress")
+        await self.record(f"task_{task_type}_completed", 1.0, "goal_progress")
+
+    async def record_suggestion(
+        self,
+        suggestion_type: str,
+        accepted: bool
+    ) -> None:
+        """Record proactive suggestion outcome."""
+        await self.record(f"suggestion_{suggestion_type}", 1.0 if accepted else 0.0, "initiative")
+
+    async def record_response_quality(
+        self,
+        relevance: float,
+        conciseness: float,
+        actionability: float
+    ) -> None:
+        """Record response quality metrics."""
+        await self.record("relevance_score", relevance, "response_quality")
+        await self.record("conciseness_score", conciseness, "response_quality")
+        await self.record("actionability_score", actionability, "response_quality")
+
+    async def _persist(self, observation: MetricObservation) -> None:
+        """Persist observation to Neo4j."""
+        try:
+            person_id = (observation.context or {}).get("person_id")
+            async with self.graph.driver.session(database=self.graph.database) as session:
+                if person_id:
+                    await session.run("""
+                        CREATE (m:CognitiveMetric {
+                            id: randomUUID(),
+                            metric_type: $metric_type,
+                            value: $value,
+                            domain: $domain,
+                            recorded_at: datetime($timestamp),
+                            context: $context
+                        })
+                        WITH m
+                        MERGE (p:Person {id: $person_id})
+                        CREATE (m)-[:OBSERVED_FOR]->(p)
+                    """,
+                        metric_type=observation.metric_type,
+                        value=observation.value,
+                        domain=observation.domain,
+                        timestamp=observation.recorded_at.isoformat(),
+                        context=observation.context or {},
+                        person_id=person_id,
+                    )
+                else:
+                    await session.run("""
+                        CREATE (m:CognitiveMetric {
+                            id: randomUUID(),
+                            metric_type: $metric_type,
+                            value: $value,
+                            domain: $domain,
+                            recorded_at: datetime($timestamp),
+                            context: $context
+                        })
+                    """,
+                        metric_type=observation.metric_type,
+                        value=observation.value,
+                        domain=observation.domain,
+                        timestamp=observation.recorded_at.isoformat(),
+                        context=observation.context or {},
+                    )
+        except (OSError, RuntimeError):
+            pass  # Don't fail if persistence fails
+
+    async def _fetch_from_graph(
+        self,
+        domain: str,
+        days: int
+    ) -> List[MetricObservation]:
+        """Fetch metrics from Neo4j."""
+        try:
+            async with self.graph.driver.session(database=self.graph.database) as session:
+                result = await session.run("""
+                    MATCH (m:CognitiveMetric)
+                    WHERE m.domain = $domain
+                    AND m.recorded_at >= datetime() - duration({days: $days})
+                    RETURN m.metric_type AS type, m.value AS value, m.recorded_at AS ts
+                """, domain=domain, days=days)
+
+                observations = []
+                async for record in result:
+                    observations.append(MetricObservation(
+                        metric_type=record["type"],
+                        value=record["value"],
+                        domain=domain,
+                        recorded_at=record["ts"],
+                    ))
+                return observations
+        except (OSError, RuntimeError):
+            return []
+
+    def flush_buffer(self) -> int:
+        """Clear in-memory buffer, return count of flushed items."""
+        count = len(self._buffer)
+        self._buffer.clear()
+        return count

apsimo/intelligence/cognition/performance_index.py

@@ -0,0 +1,299 @@
+"""Deprecated legacy CPI detector.
+
+The canonical performance surface is ``SelfhoodBenchmark``. This module is
+kept only to emit gap proposals during migration; it no longer invents
+flattering values when evidence is absent and it never authorizes a write.
+"""
+import logging
+from dataclasses import dataclass, field
+from typing import Dict, List, Optional
+from datetime import datetime
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class CPIComponent:
+    """Score for a single CPI dimension."""
+    name: str
+    score: float  # 0-100
+    trend: str = "stable"  # "improving", "declining", "stable"
+    metrics: Dict[str, float] = field(default_factory=dict)
+    available: bool = True
+
+
+@dataclass
+class CognitivePerformanceIndex:
+    """Deprecated compatibility shape; use SelfhoodBenchmark rollups."""
+    retrieval: CPIComponent
+    prediction: CPIComponent
+    goal_progress: CPIComponent
+    tool_efficiency: CPIComponent
+    initiative: CPIComponent
+    response_quality: CPIComponent
+    overall: float
+    computed_at: datetime
+    deprecated: bool = True
+    available_components: List[str] = field(default_factory=list)
+
+
+class PerformanceIndexComputer:
+    """Compute 6-component Cognitive Performance Index."""
+
+    COMPONENT_WEIGHTS = {
+        "retrieval": 0.20,
+        "prediction": 0.15,
+        "goal_progress": 0.20,
+        "tool_efficiency": 0.15,
+        "initiative": 0.15,
+        "response_quality": 0.15,
+    }
+
+    # Thresholds for trend detection
+    IMPROVEMENT_THRESHOLD = 5.0  # 5-point improvement
+    DECLINE_THRESHOLD = -5.0
+
+    def __init__(self, graph: "ColonyGraph"):
+        self.graph = graph
+        self._prior_scores: dict[str, list[float]] = {}  # component → recent scores
+
+    async def compute(self, metrics_collector: Optional["MetricsCollector"] = None) -> CognitivePerformanceIndex:
+        """Compute CPI from recent metrics."""
+        components = {}
+
+        # 1. Retrieval: memory quality, relevance, latency
+        retrieval_metrics = await self._get_metrics(metrics_collector, "retrieval", days=7)
+        components["retrieval"] = self._compute_retrieval(retrieval_metrics)
+
+        # 2. Prediction: accuracy, confidence calibration
+        pred_metrics = await self._get_metrics(metrics_collector, "prediction", days=7)
+        components["prediction"] = self._compute_prediction(pred_metrics)
+
+        # 3. Goal Progress: task completion velocity
+        goal_metrics = await self._get_metrics(metrics_collector, "goal_progress", days=7)
+        components["goal_progress"] = self._compute_goal_progress(goal_metrics)
+
+        # 4. Tool Efficiency: success rate, latency
+        tool_metrics = await self._get_metrics(metrics_collector, "tool_efficiency", days=7)
+        components["tool_efficiency"] = self._compute_tool_efficiency(tool_metrics)
+
+        # 5. Initiative: proactive suggestion hit rate
+        init_metrics = await self._get_metrics(metrics_collector, "initiative", days=7)
+        components["initiative"] = self._compute_initiative(init_metrics)
+
+        # 6. Response Quality: relevance, conciseness, actionability
+        quality_metrics = await self._get_metrics(metrics_collector, "response_quality", days=7)
+        components["response_quality"] = self._compute_response_quality(quality_metrics)
+
+        # Reweight only evidence-bearing dimensions. Missing dimensions are
+        # unavailable, not neutral, successful, or silently zero evidence.
+        available_weight = sum(
+            weight for name, weight in self.COMPONENT_WEIGHTS.items()
+            if components[name].available)
+        overall = (sum(
+            components[name].score * weight
+            for name, weight in self.COMPONENT_WEIGHTS.items()
+            if components[name].available
+        ) / available_weight) if available_weight else 0.0
+        available = [name for name, component in components.items()
+                     if component.available]
+
+        return CognitivePerformanceIndex(
+            retrieval=components["retrieval"],
+            prediction=components["prediction"],
+            goal_progress=components["goal_progress"],
+            tool_efficiency=components["tool_efficiency"],
+            initiative=components["initiative"],
+            response_quality=components["response_quality"],
+            overall=overall,
+            computed_at=datetime.now(),
+            deprecated=True,
+            available_components=available,
+        )
+
+    @staticmethod
+    def _unavailable(name: str) -> CPIComponent:
+        return CPIComponent(
+            name=name, score=0.0, trend="unavailable",
+            metrics={"available": False, "deprecated": True},
+            available=False)
+
+    async def _get_metrics(
+        self,
+        collector: Optional["MetricsCollector"],
+        domain: str,
+        days: int = 7
+    ) -> Dict[str, float]:
+        """Fetch metrics for a domain."""
+        if collector:
+            try:
+                return await collector.get_metrics(domain, days=days)
+            except Exception as exc:
+                logger.debug("Collector metrics failed for %s: %s", domain, exc)
+
+        # Fallback: query graph directly
+        try:
+            async with self.graph.driver.session(database=self.graph.database) as session:
+                result = await session.run("""
+                    MATCH (m:CognitiveMetric)
+                    WHERE m.domain = $domain
+                    AND m.recorded_at >= datetime() - duration({days: $days})
+                    RETURN m.metric_type AS type, m.value AS value
+                """, domain=domain, days=days)
+
+                metrics = {}
+                async for record in result:
+                    metrics[record["type"]] = record["value"]
+                return metrics
+        except Exception as exc:
+            logger.debug("Graph metrics query failed for %s: %s", domain, exc)
+            return {}
+
+    def _compute_retrieval(self, metrics: Dict[str, float]) -> CPIComponent:
+        """Compute retrieval component score."""
+        if "relevance_score" not in metrics or "latency_ms" not in metrics:
+            return self._unavailable("retrieval")
+        relevance = metrics["relevance_score"]
+        latency = metrics["latency_ms"]
+
+        # Latency penalty (cap at 2s)
+        latency_score = max(0, 100 - (latency / 20))
+
+        score = (relevance * 0.7 + latency_score * 0.3)
+        trend = self._compute_trend("retrieval", score)
+
+        return CPIComponent(
+            name="retrieval",
+            score=score,
+            trend=trend,
+            metrics=metrics,
+        )
+
+    def _compute_prediction(self, metrics: Dict[str, float]) -> CPIComponent:
+        """Compute prediction component score."""
+        correct = metrics.get("correct_count", 0)
+        total = metrics.get("total_predictions", 0)
+        if total <= 0:
+            return self._unavailable("prediction")
+
+        accuracy = (correct / max(total, 1)) * 100
+        trend = self._compute_trend("prediction", accuracy)
+
+        return CPIComponent(
+            name="prediction",
+            score=accuracy,
+            trend=trend,
+            metrics={"accuracy": accuracy, "total": total},
+        )
+
+    def _compute_goal_progress(self, metrics: Dict[str, float]) -> CPIComponent:
+        """Compute goal progress component score."""
+        completed = metrics.get("completed_tasks", 0)
+        total = metrics.get("total_tasks", 0)
+        if total <= 0 or "velocity" not in metrics:
+            return self._unavailable("goal_progress")
+        velocity = metrics["velocity"]
+
+        completion_rate = (completed / max(total, 1)) * 100
+        velocity_score = min(velocity * 20, 100)  # 5 tasks/day = 100
+
+        score = completion_rate * 0.6 + velocity_score * 0.4
+        trend = self._compute_trend("goal_progress", score)
+
+        return CPIComponent(
+            name="goal_progress",
+            score=score,
+            trend=trend,
+            metrics=metrics,
+        )
+
+    def _compute_tool_efficiency(self, metrics: Dict[str, float]) -> CPIComponent:
+        """Compute tool efficiency component score."""
+        if "success_rate" not in metrics or "avg_latency_ms" not in metrics:
+            return self._unavailable("tool_efficiency")
+        success_rate = metrics["success_rate"]
+        avg_latency = metrics["avg_latency_ms"]
+
+        latency_score = max(0, 100 - (avg_latency / 50))
+        score = success_rate * 0.7 + latency_score * 0.3
+        trend = self._compute_trend("tool_efficiency", score)
+
+        return CPIComponent(
+            name="tool_efficiency",
+            score=score,
+            trend=trend,
+            metrics=metrics,
+        )
+
+    def _compute_initiative(self, metrics: Dict[str, float]) -> CPIComponent:
+        """Compute initiative component score."""
+        suggestions = metrics.get("suggestions_made", 0)
+        accepted = metrics.get("suggestions_accepted", 0)
+
+        if suggestions <= 0:
+            return self._unavailable("initiative")
+        hit_rate = (accepted / suggestions) * 100
+
+        volume_score = min(suggestions * 10, 100)  # 10 suggestions = 100
+
+        score = hit_rate * 0.7 + volume_score * 0.3
+        trend = self._compute_trend("initiative", score)
+
+        return CPIComponent(
+            name="initiative",
+            score=score,
+            trend=trend,
+            metrics=metrics,
+        )
+
+    def _compute_response_quality(self, metrics: Dict[str, float]) -> CPIComponent:
+        """Compute response quality component score."""
+        required = {"relevance_score", "conciseness_score",
+                    "actionability_score"}
+        if not required <= set(metrics):
+            return self._unavailable("response_quality")
+        relevance = metrics["relevance_score"]
+        conciseness = metrics["conciseness_score"]
+        actionability = metrics["actionability_score"]
+
+        score = relevance * 0.4 + conciseness * 0.3 + actionability * 0.3
+        trend = self._compute_trend("response_quality", score)
+
+        return CPIComponent(
+            name="response_quality",
+            score=score,
+            trend=trend,
+            metrics=metrics,
+        )
+
+    def _compute_trend(self, component: str, current_score: float) -> str:
+        """Determine trend by comparing to the rolling average of prior scores.
+
+        Uses an in-session cache of up to 5 recent scores per component.
+        On first call the cache is empty, so "stable" is returned.
+
+        Returns:
+            "improving"  — current score is ≥ IMPROVEMENT_THRESHOLD above prior avg
+            "declining"  — current score is ≤ DECLINE_THRESHOLD below prior avg
+            "stable"     — within threshold range, or no prior data
+        """
+        prior = self._prior_scores.get(component, [])
+
+        if not prior:
+            # No prior data — record current score and report stable
+            self._prior_scores[component] = [current_score]
+            return "stable"
+
+        prior_avg = sum(prior) / len(prior)
+        delta = current_score - prior_avg
+
+        if delta >= self.IMPROVEMENT_THRESHOLD:
+            trend = "improving"
+        elif delta <= self.DECLINE_THRESHOLD:
+            trend = "declining"
+        else:
+            trend = "stable"
+
+        # Keep a rolling window of up to 5 prior scores
+        self._prior_scores[component] = ([current_score] + prior)[:5]
+        return trend