empathy-framework 5.3.0__py3-none-any.whl → 5.4.0__py3-none-any.whl

empathy_os/socratic/success.py

@@ -1,730 +0,0 @@
-"""Success Criteria and Measurement System
-
-Define and measure success for generated workflows.
-
-Success criteria allow users to:
-1. Define what "done" looks like for their workflow
-2. Track progress toward goals
-3. Measure effectiveness over time
-4. Iterate and improve workflows
-
-Copyright 2026 Smart-AI-Memory
-Licensed under Fair Source License 0.9
-"""
-
-from __future__ import annotations
-
-from collections.abc import Callable
-from dataclasses import dataclass, field
-from datetime import datetime
-from enum import Enum
-from typing import Any
-
-
-class MetricType(Enum):
-    """Types of success metrics."""
-
-    # Numeric metrics
-    COUNT = "count"  # Integer count (e.g., issues found)
-    PERCENTAGE = "percentage"  # 0-100 percentage
-    RATIO = "ratio"  # 0-1 ratio
-    DURATION = "duration"  # Time in seconds
-
-    # Boolean metrics
-    BOOLEAN = "boolean"  # True/False
-
-    # Comparison metrics
-    IMPROVEMENT = "improvement"  # Before/after comparison
-    THRESHOLD = "threshold"  # Above/below threshold
-
-    # Quality metrics
-    SCORE = "score"  # 0-10 quality score
-    RATING = "rating"  # Categorical (good, moderate, poor)
-
-
-class MetricDirection(Enum):
-    """Which direction indicates success."""
-
-    HIGHER_IS_BETTER = "higher"  # More issues found = better
-    LOWER_IS_BETTER = "lower"  # Less time = better
-    TARGET_VALUE = "target"  # Specific value is best
-    RANGE = "range"  # Within a range is success
-
-
-@dataclass
-class SuccessMetric:
-    """A single success metric definition.
-
-    Example:
-        >>> metric = SuccessMetric(
-        ...     id="security_issues_found",
-        ...     name="Security Issues Detected",
-        ...     description="Number of security vulnerabilities identified",
-        ...     metric_type=MetricType.COUNT,
-        ...     direction=MetricDirection.HIGHER_IS_BETTER,
-        ...     target_value=None,  # No specific target
-        ...     minimum_value=0,
-        ...     unit="issues"
-        ... )
-    """
-
-    # Unique metric identifier
-    id: str
-
-    # Display name
-    name: str
-
-    # Description of what this measures
-    description: str
-
-    # Type of metric
-    metric_type: MetricType
-
-    # Which direction indicates success
-    direction: MetricDirection = MetricDirection.HIGHER_IS_BETTER
-
-    # Target value (for TARGET_VALUE direction)
-    target_value: float | None = None
-
-    # Minimum acceptable value
-    minimum_value: float | None = None
-
-    # Maximum acceptable value
-    maximum_value: float | None = None
-
-    # Unit of measurement
-    unit: str = ""
-
-    # Weight for composite scoring (0-1)
-    weight: float = 1.0
-
-    # Whether this is a primary success indicator
-    is_primary: bool = False
-
-    # How to extract this metric from workflow output
-    extraction_path: str = ""  # JSONPath-like expression
-
-    # Custom extraction function
-    extractor: Callable[[dict], float | bool] | None = None
-
-    def evaluate(
-        self,
-        value: float | bool,
-        baseline: float | bool | None = None,
-    ) -> tuple[bool, float, str]:
-        """Evaluate if a value meets this metric's success criteria.
-
-        Args:
-            value: The measured value
-            baseline: Optional baseline for comparison
-
-        Returns:
-            Tuple of (met_criteria, score 0-1, explanation)
-        """
-        # Boolean metrics
-        if self.metric_type == MetricType.BOOLEAN:
-            if isinstance(value, bool):
-                met = value
-                score = 1.0 if value else 0.0
-                explanation = "Criteria met" if met else "Criteria not met"
-                return met, score, explanation
-
-        # Ensure numeric value for other types
-        if not isinstance(value, (int, float)):
-            return False, 0.0, f"Expected numeric value, got {type(value)}"
-
-        # Calculate score based on direction
-        if self.direction == MetricDirection.HIGHER_IS_BETTER:
-            if self.minimum_value is not None:
-                met = value >= self.minimum_value
-                # Score is ratio of value to minimum (capped at 1.0)
-                score = min(value / self.minimum_value, 1.0) if self.minimum_value > 0 else 1.0
-            else:
-                met = True  # No minimum, always met
-                score = 1.0
-
-        elif self.direction == MetricDirection.LOWER_IS_BETTER:
-            if self.maximum_value is not None:
-                met = value <= self.maximum_value
-                # Score is inverse ratio (lower is better)
-                score = (
-                    max(1.0 - (value / self.maximum_value), 0.0) if self.maximum_value > 0 else 1.0
-                )
-            else:
-                met = True
-                score = 1.0
-
-        elif self.direction == MetricDirection.TARGET_VALUE:
-            if self.target_value is not None:
-                deviation = abs(value - self.target_value)
-                # Allow 10% tolerance by default
-                tolerance = self.target_value * 0.1 if self.target_value > 0 else 1.0
-                met = deviation <= tolerance
-                score = max(1.0 - (deviation / max(tolerance, 0.001)), 0.0)
-            else:
-                met = True
-                score = 1.0
-
-        elif self.direction == MetricDirection.RANGE:
-            min_val = self.minimum_value or float("-inf")
-            max_val = self.maximum_value or float("inf")
-            met = min_val <= value <= max_val
-            if met:
-                # Score based on position in range (center = best)
-                range_size = max_val - min_val
-                if range_size > 0 and range_size != float("inf"):
-                    center = (min_val + max_val) / 2
-                    distance_from_center = abs(value - center)
-                    score = 1.0 - (distance_from_center / (range_size / 2))
-                else:
-                    score = 1.0
-            else:
-                score = 0.0
-        else:
-            met = True
-            score = 1.0
-
-        # Generate explanation
-        explanation = self._generate_explanation(value, met, score, baseline)
-
-        return met, score, explanation
-
-    def _generate_explanation(
-        self,
-        value: float | bool,
-        met: bool,
-        score: float,
-        baseline: float | bool | None,
-    ) -> str:
-        """Generate human-readable explanation of the evaluation."""
-        parts = []
-
-        # Value statement
-        if self.unit:
-            parts.append(f"Measured: {value} {self.unit}")
-        else:
-            parts.append(f"Measured: {value}")
-
-        # Comparison to baseline
-        if (
-            baseline is not None
-            and isinstance(value, (int, float))
-            and isinstance(baseline, (int, float))
-        ):
-            diff = value - baseline
-            pct_change = (diff / baseline * 100) if baseline != 0 else 0
-            direction = "↑" if diff > 0 else "↓" if diff < 0 else "→"
-            parts.append(f"vs baseline: {direction} {abs(pct_change):.1f}%")
-
-        # Target comparison
-        if self.direction == MetricDirection.TARGET_VALUE and self.target_value is not None:
-            parts.append(f"Target: {self.target_value} {self.unit}".strip())
-
-        # Result
-        result = "✓ Met" if met else "✗ Not met"
-        parts.append(f"{result} (score: {score:.1%})")
-
-        return " | ".join(parts)
-
-    def to_dict(self) -> dict[str, Any]:
-        """Serialize to dictionary."""
-        return {
-            "id": self.id,
-            "name": self.name,
-            "description": self.description,
-            "metric_type": self.metric_type.value,
-            "direction": self.direction.value,
-            "target_value": self.target_value,
-            "minimum_value": self.minimum_value,
-            "maximum_value": self.maximum_value,
-            "unit": self.unit,
-            "weight": self.weight,
-            "is_primary": self.is_primary,
-            "extraction_path": self.extraction_path,
-        }
-
-
-@dataclass
-class MetricResult:
-    """Result of evaluating a single metric."""
-
-    metric_id: str
-    value: float | bool
-    met_criteria: bool
-    score: float
-    explanation: str
-    baseline: float | bool | None = None
-    timestamp: str = ""
-
-
-@dataclass
-class SuccessCriteria:
-    """Complete success criteria for a workflow.
-
-    Example:
-        >>> criteria = SuccessCriteria(
-        ...     id="code_review_success",
-        ...     name="Code Review Success Criteria",
-        ...     description="Measures effectiveness of automated code review",
-        ...     metrics=[
-        ...         SuccessMetric(
-        ...             id="issues_found",
-        ...             name="Issues Found",
-        ...             metric_type=MetricType.COUNT,
-        ...             is_primary=True
-        ...         ),
-        ...         SuccessMetric(
-        ...             id="review_time",
-        ...             name="Review Time",
-        ...             metric_type=MetricType.DURATION,
-        ...             direction=MetricDirection.LOWER_IS_BETTER,
-        ...             maximum_value=60,  # seconds
-        ...         ),
-        ...     ],
-        ...     success_threshold=0.7  # 70% overall score = success
-        ... )
-    """
-
-    # Unique identifier
-    id: str = ""
-
-    # Display name
-    name: str = ""
-
-    # Description
-    description: str = ""
-
-    # List of metrics
-    metrics: list[SuccessMetric] = field(default_factory=list)
-
-    # Threshold for overall success (0-1)
-    success_threshold: float = 0.7
-
-    # Whether ALL metrics must be met (vs weighted average)
-    require_all: bool = False
-
-    # Minimum primary metrics that must pass
-    min_primary_metrics: int = 1
-
-    # Custom success evaluator
-    custom_evaluator: Callable[[dict[str, MetricResult]], bool] | None = None
-
-    def add_metric(self, metric: SuccessMetric) -> None:
-        """Add a metric to the criteria."""
-        self.metrics.append(metric)
-
-    def get_primary_metrics(self) -> list[SuccessMetric]:
-        """Get all primary success indicators."""
-        return [m for m in self.metrics if m.is_primary]
-
-    def evaluate(
-        self,
-        workflow_output: dict[str, Any],
-        baselines: dict[str, float | bool] | None = None,
-    ) -> SuccessEvaluation:
-        """Evaluate workflow output against success criteria.
-
-        Args:
-            workflow_output: The workflow's output to evaluate
-            baselines: Optional baseline values for comparison
-
-        Returns:
-            SuccessEvaluation with detailed results
-        """
-        baselines = baselines or {}
-        results: list[MetricResult] = []
-        timestamp = datetime.now().isoformat()
-
-        # Evaluate each metric
-        for metric in self.metrics:
-            # Extract value from output
-            value = self._extract_metric_value(metric, workflow_output)
-
-            if value is None:
-                # Metric not found in output
-                results.append(
-                    MetricResult(
-                        metric_id=metric.id,
-                        value=0,
-                        met_criteria=False,
-                        score=0.0,
-                        explanation=f"Metric '{metric.name}' not found in output",
-                        timestamp=timestamp,
-                    )
-                )
-                continue
-
-            # Get baseline if available
-            baseline = baselines.get(metric.id)
-
-            # Evaluate
-            met, score, explanation = metric.evaluate(value, baseline)
-
-            results.append(
-                MetricResult(
-                    metric_id=metric.id,
-                    value=value,
-                    met_criteria=met,
-                    score=score,
-                    explanation=explanation,
-                    baseline=baseline,
-                    timestamp=timestamp,
-                )
-            )
-
-        # Calculate overall success
-        return self._calculate_overall_success(results)
-
-    def _extract_metric_value(
-        self,
-        metric: SuccessMetric,
-        output: dict[str, Any],
-    ) -> float | bool | None:
-        """Extract metric value from workflow output."""
-        # Use custom extractor if provided
-        if metric.extractor:
-            try:
-                return metric.extractor(output)
-            except (KeyError, TypeError, ValueError):
-                return None
-
-        # Use extraction path
-        if metric.extraction_path:
-            try:
-                value = output
-                for key in metric.extraction_path.split("."):
-                    if isinstance(value, dict):
-                        value = value[key]
-                    elif isinstance(value, list) and key.isdigit():
-                        value = value[int(key)]
-                    else:
-                        return None
-                return value
-            except (KeyError, IndexError, TypeError):
-                return None
-
-        # Try direct key match
-        if metric.id in output:
-            return output[metric.id]
-
-        # Try nested in 'metrics' key
-        if "metrics" in output and isinstance(output["metrics"], dict):
-            if metric.id in output["metrics"]:
-                return output["metrics"][metric.id]
-
-        return None
-
-    def _calculate_overall_success(
-        self,
-        results: list[MetricResult],
-    ) -> SuccessEvaluation:
-        """Calculate overall success from metric results."""
-        if not results:
-            return SuccessEvaluation(
-                overall_success=False,
-                overall_score=0.0,
-                metric_results=results,
-                summary="No metrics to evaluate",
-            )
-
-        # Check primary metrics
-        primary_results = [
-            r for r in results if any(m.id == r.metric_id and m.is_primary for m in self.metrics)
-        ]
-        primary_passed = sum(1 for r in primary_results if r.met_criteria)
-
-        # Check if minimum primary metrics are met
-        primary_check = primary_passed >= self.min_primary_metrics
-
-        # Check if all required
-        if self.require_all:
-            all_met = all(r.met_criteria for r in results)
-            overall_success = all_met and primary_check
-            overall_score = 1.0 if overall_success else sum(r.score for r in results) / len(results)
-        else:
-            # Weighted average score
-            total_weight = sum(
-                m.weight for m in self.metrics if any(r.metric_id == m.id for r in results)
-            )
-
-            if total_weight > 0:
-                weighted_score = (
-                    sum(
-                        r.score * next((m.weight for m in self.metrics if m.id == r.metric_id), 1.0)
-                        for r in results
-                    )
-                    / total_weight
-                )
-            else:
-                weighted_score = sum(r.score for r in results) / len(results)
-
-            overall_score = weighted_score
-            overall_success = overall_score >= self.success_threshold and primary_check
-
-        # Custom evaluator override
-        if self.custom_evaluator:
-            results_dict = {r.metric_id: r for r in results}
-            overall_success = self.custom_evaluator(results_dict)
-
-        # Generate summary
-        summary = self._generate_summary(results, overall_success, overall_score)
-
-        return SuccessEvaluation(
-            overall_success=overall_success,
-            overall_score=overall_score,
-            metric_results=results,
-            summary=summary,
-            primary_metrics_passed=primary_passed,
-            total_primary_metrics=len(primary_results),
-        )
-
-    def _generate_summary(
-        self,
-        results: list[MetricResult],
-        success: bool,
-        score: float,
-    ) -> str:
-        """Generate human-readable summary."""
-        status = "✓ SUCCESS" if success else "✗ NOT MET"
-        met_count = sum(1 for r in results if r.met_criteria)
-
-        lines = [
-            f"{status} - Overall score: {score:.1%}",
-            f"Metrics: {met_count}/{len(results)} met criteria",
-            "",
-            "Details:",
-        ]
-
-        for result in results:
-            metric = next((m for m in self.metrics if m.id == result.metric_id), None)
-            name = metric.name if metric else result.metric_id
-            indicator = "✓" if result.met_criteria else "✗"
-            lines.append(f"  {indicator} {name}: {result.explanation}")
-
-        return "\n".join(lines)
-
-    def to_dict(self) -> dict[str, Any]:
-        """Serialize to dictionary."""
-        return {
-            "id": self.id,
-            "name": self.name,
-            "description": self.description,
-            "metrics": [m.to_dict() for m in self.metrics],
-            "success_threshold": self.success_threshold,
-            "require_all": self.require_all,
-            "min_primary_metrics": self.min_primary_metrics,
-        }
-
-
-@dataclass
-class SuccessEvaluation:
-    """Result of evaluating success criteria."""
-
-    # Whether overall success criteria were met
-    overall_success: bool
-
-    # Overall score (0-1)
-    overall_score: float
-
-    # Individual metric results
-    metric_results: list[MetricResult]
-
-    # Human-readable summary
-    summary: str
-
-    # Primary metrics that passed
-    primary_metrics_passed: int = 0
-
-    # Total primary metrics
-    total_primary_metrics: int = 0
-
-    # Timestamp of evaluation
-    evaluated_at: str = field(default_factory=lambda: datetime.now().isoformat())
-
-    def to_dict(self) -> dict[str, Any]:
-        """Serialize to dictionary."""
-        return {
-            "overall_success": self.overall_success,
-            "overall_score": self.overall_score,
-            "metric_results": [
-                {
-                    "metric_id": r.metric_id,
-                    "value": r.value,
-                    "met_criteria": r.met_criteria,
-                    "score": r.score,
-                    "explanation": r.explanation,
-                    "baseline": r.baseline,
-                }
-                for r in self.metric_results
-            ],
-            "summary": self.summary,
-            "primary_metrics_passed": self.primary_metrics_passed,
-            "total_primary_metrics": self.total_primary_metrics,
-            "evaluated_at": self.evaluated_at,
-        }
-
-
-# =============================================================================
-# PREDEFINED SUCCESS CRITERIA TEMPLATES
-# =============================================================================
-
-
-def code_review_criteria() -> SuccessCriteria:
-    """Create standard success criteria for code review workflows."""
-    return SuccessCriteria(
-        id="code_review_success",
-        name="Code Review Success",
-        description="Standard metrics for code review effectiveness",
-        metrics=[
-            SuccessMetric(
-                id="issues_found",
-                name="Issues Found",
-                description="Number of issues identified",
-                metric_type=MetricType.COUNT,
-                direction=MetricDirection.HIGHER_IS_BETTER,
-                is_primary=True,
-                weight=1.0,
-                extraction_path="findings_count",
-            ),
-            SuccessMetric(
-                id="severity_coverage",
-                name="Severity Coverage",
-                description="Percentage of severity levels covered",
-                metric_type=MetricType.PERCENTAGE,
-                direction=MetricDirection.HIGHER_IS_BETTER,
-                minimum_value=50,
-                weight=0.8,
-                extraction_path="severity_coverage",
-            ),
-            SuccessMetric(
-                id="review_time",
-                name="Review Duration",
-                description="Time to complete review",
-                metric_type=MetricType.DURATION,
-                direction=MetricDirection.LOWER_IS_BETTER,
-                maximum_value=120,  # 2 minutes
-                unit="seconds",
-                weight=0.6,
-                extraction_path="duration_seconds",
-            ),
-            SuccessMetric(
-                id="actionable_recommendations",
-                name="Actionable Recommendations",
-                description="Whether recommendations are actionable",
-                metric_type=MetricType.BOOLEAN,
-                is_primary=True,
-                weight=1.0,
-                extraction_path="has_recommendations",
-            ),
-        ],
-        success_threshold=0.7,
-        min_primary_metrics=1,
-    )
-
-
-def security_audit_criteria() -> SuccessCriteria:
-    """Create success criteria for security audit workflows."""
-    return SuccessCriteria(
-        id="security_audit_success",
-        name="Security Audit Success",
-        description="Metrics for security audit effectiveness",
-        metrics=[
-            SuccessMetric(
-                id="vulnerabilities_found",
-                name="Vulnerabilities Found",
-                description="Security vulnerabilities identified",
-                metric_type=MetricType.COUNT,
-                direction=MetricDirection.HIGHER_IS_BETTER,
-                is_primary=True,
-                weight=1.0,
-                extraction_path="vulnerabilities.count",
-            ),
-            SuccessMetric(
-                id="critical_issues",
-                name="Critical Issues",
-                description="High/critical severity issues found",
-                metric_type=MetricType.COUNT,
-                direction=MetricDirection.HIGHER_IS_BETTER,
-                is_primary=True,
-                weight=1.2,  # Extra weight for critical issues
-                extraction_path="vulnerabilities.critical_count",
-            ),
-            SuccessMetric(
-                id="owasp_coverage",
-                name="OWASP Coverage",
-                description="OWASP Top 10 categories checked",
-                metric_type=MetricType.PERCENTAGE,
-                direction=MetricDirection.HIGHER_IS_BETTER,
-                minimum_value=80,
-                weight=0.9,
-                extraction_path="owasp_coverage_percent",
-            ),
-            SuccessMetric(
-                id="false_positive_rate",
-                name="False Positive Rate",
-                description="Estimated false positive rate",
-                metric_type=MetricType.PERCENTAGE,
-                direction=MetricDirection.LOWER_IS_BETTER,
-                maximum_value=20,
-                weight=0.7,
-                extraction_path="estimated_fp_rate",
-            ),
-        ],
-        success_threshold=0.75,
-        min_primary_metrics=1,
-    )
-
-
-def test_generation_criteria() -> SuccessCriteria:
-    """Create success criteria for test generation workflows."""
-    return SuccessCriteria(
-        id="test_generation_success",
-        name="Test Generation Success",
-        description="Metrics for test generation effectiveness",
-        metrics=[
-            SuccessMetric(
-                id="tests_generated",
-                name="Tests Generated",
-                description="Number of test cases generated",
-                metric_type=MetricType.COUNT,
-                direction=MetricDirection.HIGHER_IS_BETTER,
-                minimum_value=1,
-                is_primary=True,
-                weight=1.0,
-                extraction_path="tests.count",
-            ),
-            SuccessMetric(
-                id="coverage_increase",
-                name="Coverage Increase",
-                description="Increase in code coverage",
-                metric_type=MetricType.IMPROVEMENT,
-                direction=MetricDirection.HIGHER_IS_BETTER,
-                minimum_value=5,  # At least 5% increase
-                unit="%",
-                weight=1.0,
-                extraction_path="coverage.increase_percent",
-            ),
-            SuccessMetric(
-                id="tests_passing",
-                name="Tests Passing",
-                description="Percentage of generated tests that pass",
-                metric_type=MetricType.PERCENTAGE,
-                direction=MetricDirection.HIGHER_IS_BETTER,
-                minimum_value=80,
-                is_primary=True,
-                weight=1.2,
-                extraction_path="tests.pass_rate",
-            ),
-            SuccessMetric(
-                id="edge_cases_covered",
-                name="Edge Cases Covered",
-                description="Number of edge cases with tests",
-                metric_type=MetricType.COUNT,
-                direction=MetricDirection.HIGHER_IS_BETTER,
-                weight=0.8,
-                extraction_path="edge_cases.count",
-            ),
-        ],
-        success_threshold=0.7,
-        min_primary_metrics=2,
-    )