agent-learning-kit 0.1.0__py3-none-any.whl

fi/cli/assertions/evaluator.py

@@ -0,0 +1,286 @@
+"""Assertion evaluator for checking conditions against evaluation results."""
+
+from typing import Dict, List, Optional, Any
+from dataclasses import dataclass, field
+from enum import Enum
+import statistics
+
+from .conditions import MetricType
+from .parser import ConditionParser, ConditionParseError
+
+
+class AssertionResult(Enum):
+    """Result status of an assertion evaluation."""
+    PASSED = "passed"
+    FAILED = "failed"
+    SKIPPED = "skipped"
+    WARNING = "warning"
+
+
+@dataclass
+class AssertionOutcome:
+    """Result of evaluating a single assertion."""
+    template: Optional[str]
+    condition: str
+    expected: str
+    actual: float
+    result: AssertionResult
+    message: str
+
+
+@dataclass
+class AssertionReport:
+    """Complete assertion evaluation report."""
+    outcomes: List[AssertionOutcome] = field(default_factory=list)
+    total_assertions: int = 0
+    passed: int = 0
+    failed: int = 0
+    warnings: int = 0
+    skipped: int = 0
+
+    @property
+    def all_passed(self) -> bool:
+        """Check if all assertions passed (no failures)."""
+        return self.failed == 0
+
+    @property
+    def has_warnings(self) -> bool:
+        """Check if there are any warnings."""
+        return self.warnings > 0
+
+
+class AssertionEvaluator:
+    """Evaluate assertions against evaluation results."""
+
+    def __init__(self, results: Dict[str, Any], config: Dict[str, Any]):
+        """Initialize the evaluator.
+
+        Args:
+            results: Dictionary containing evaluation results with 'eval_results' key.
+            config: Configuration dictionary that may contain 'assertions' and 'thresholds'.
+        """
+        self.results = results
+        self.config = config
+        self.outcomes: List[AssertionOutcome] = []
+
+    def compute_metrics(self, template: Optional[str] = None) -> Dict[MetricType, float]:
+        """Compute all metrics for a template or globally.
+
+        Args:
+            template: If provided, compute metrics only for this template.
+                     If None, compute global metrics across all results.
+
+        Returns:
+            Dictionary mapping MetricType to computed values.
+        """
+        if template:
+            eval_results = [
+                r for r in self.results.get('eval_results', [])
+                if r.get('name') == template
+            ]
+        else:
+            eval_results = self.results.get('eval_results', [])
+
+        if not eval_results:
+            return {}
+
+        # Extract scores/outputs
+        outputs = [r.get('output') for r in eval_results]
+        runtimes = [r.get('runtime', 0) for r in eval_results if r.get('runtime')]
+
+        # Calculate boolean pass/fail
+        bool_outputs = [o for o in outputs if isinstance(o, bool)]
+        numeric_outputs = [
+            float(o) for o in outputs
+            if isinstance(o, (int, float)) and not isinstance(o, bool)
+        ]
+
+        # Pass rate (boolean outputs: True = pass, numeric: >= 0.5 = pass)
+        passes = sum(1 for o in bool_outputs if o is True)
+        passes += sum(1 for o in numeric_outputs if o >= 0.5)
+        total = len(outputs)
+
+        metrics: Dict[MetricType, float] = {
+            MetricType.PASS_RATE: passes / total if total > 0 else 0,
+            MetricType.TOTAL_PASS_RATE: passes / total if total > 0 else 0,  # Alias for global
+            MetricType.PASSED_COUNT: float(passes),
+            MetricType.FAILED_COUNT: float(total - passes),
+            MetricType.TOTAL_COUNT: float(total),
+        }
+
+        # Score metrics (only for numeric outputs)
+        if numeric_outputs:
+            metrics[MetricType.AVG_SCORE] = statistics.mean(numeric_outputs)
+            metrics[MetricType.MIN_SCORE] = min(numeric_outputs)
+            metrics[MetricType.MAX_SCORE] = max(numeric_outputs)
+
+            sorted_outputs = sorted(numeric_outputs)
+
+            # Percentiles
+            metrics[MetricType.P50_SCORE] = self._percentile(sorted_outputs, 50)
+            metrics[MetricType.P90_SCORE] = self._percentile(sorted_outputs, 90)
+            metrics[MetricType.P95_SCORE] = self._percentile(sorted_outputs, 95)
+
+        # Runtime metrics
+        if runtimes:
+            metrics[MetricType.RUNTIME_AVG] = statistics.mean(runtimes)
+            metrics[MetricType.RUNTIME_P95] = self._percentile(sorted(runtimes), 95)
+
+        return metrics
+
+    def _percentile(self, sorted_data: List[float], p: float) -> float:
+        """Calculate percentile from sorted data.
+
+        Args:
+            sorted_data: Pre-sorted list of values.
+            p: Percentile to calculate (0-100).
+
+        Returns:
+            The calculated percentile value.
+        """
+        if not sorted_data:
+            return 0.0
+
+        n = len(sorted_data)
+        if n == 1:
+            return sorted_data[0]
+
+        # Linear interpolation
+        k = (n - 1) * (p / 100)
+        f = int(k)
+        c = f + 1 if f + 1 < n else f
+
+        if f == c:
+            return sorted_data[f]
+
+        return sorted_data[f] + (k - f) * (sorted_data[c] - sorted_data[f])
+
+    def evaluate_assertion(
+        self,
+        template: Optional[str],
+        condition_str: str,
+        on_fail: str = "error"
+    ) -> AssertionOutcome:
+        """Evaluate a single assertion condition.
+
+        Args:
+            template: Template name to evaluate against, or None for global.
+            condition_str: The condition string to evaluate.
+            on_fail: Action on failure - "error", "warn", or "skip".
+
+        Returns:
+            AssertionOutcome with the result.
+        """
+        try:
+            condition = ConditionParser.parse(condition_str)
+        except ConditionParseError as e:
+            return AssertionOutcome(
+                template=template,
+                condition=condition_str,
+                expected="valid condition",
+                actual=0,
+                result=AssertionResult.FAILED,
+                message=f"Parse error: {e}"
+            )
+
+        metrics = self.compute_metrics(template)
+
+        if condition.metric not in metrics:
+            return AssertionOutcome(
+                template=template,
+                condition=condition_str,
+                expected=f"{condition.metric.value} {condition.operator.value} {condition.value}",
+                actual=0,
+                result=AssertionResult.SKIPPED,
+                message=f"Metric '{condition.metric.value}' not available for template '{template or 'global'}'"
+            )
+
+        actual_value = metrics[condition.metric]
+        passed = condition.evaluate(actual_value)
+
+        if passed:
+            result = AssertionResult.PASSED
+            message = "Assertion passed"
+        elif on_fail == "warn":
+            result = AssertionResult.WARNING
+            message = (
+                f"Warning: {condition.metric.value} is {actual_value:.4f}, "
+                f"expected {condition.operator.value} {condition.value}"
+            )
+        elif on_fail == "skip":
+            result = AssertionResult.SKIPPED
+            message = "Assertion skipped"
+        else:
+            result = AssertionResult.FAILED
+            message = (
+                f"Failed: {condition.metric.value} is {actual_value:.4f}, "
+                f"expected {condition.operator.value} {condition.value}"
+            )
+
+        return AssertionOutcome(
+            template=template,
+            condition=condition_str,
+            expected=f"{condition.operator.value} {condition.value}",
+            actual=actual_value,
+            result=result,
+            message=message
+        )
+
+    def evaluate_all(self) -> AssertionReport:
+        """Evaluate all assertions from config.
+
+        Returns:
+            AssertionReport containing all outcomes and summary statistics.
+        """
+        assertions = self.config.get('assertions', [])
+        thresholds = self.config.get('thresholds', {})
+
+        outcomes: List[AssertionOutcome] = []
+
+        # Evaluate explicit assertions
+        for assertion in assertions:
+            template = assertion.get('template')
+            is_global = assertion.get('global', False)
+            conditions = assertion.get('conditions', [])
+            on_fail = assertion.get('on_fail', 'error')
+
+            for condition_str in conditions:
+                outcome = self.evaluate_assertion(
+                    template=None if is_global else template,
+                    condition_str=condition_str,
+                    on_fail=on_fail
+                )
+                outcomes.append(outcome)
+
+        # Evaluate threshold shortcuts
+        default_threshold = thresholds.get('default_pass_rate')
+        overrides = thresholds.get('overrides', {})
+
+        if default_threshold is not None:
+            templates = set(
+                r.get('name') for r in self.results.get('eval_results', [])
+                if r.get('name')
+            )
+            for template in templates:
+                threshold = overrides.get(template, default_threshold)
+                outcome = self.evaluate_assertion(
+                    template=template,
+                    condition_str=f"pass_rate >= {threshold}",
+                    on_fail="error"
+                )
+                outcomes.append(outcome)
+
+        # Build report
+        passed = sum(1 for o in outcomes if o.result == AssertionResult.PASSED)
+        failed = sum(1 for o in outcomes if o.result == AssertionResult.FAILED)
+        warnings = sum(1 for o in outcomes if o.result == AssertionResult.WARNING)
+        skipped = sum(1 for o in outcomes if o.result == AssertionResult.SKIPPED)
+
+        return AssertionReport(
+            outcomes=outcomes,
+            total_assertions=len(outcomes),
+            passed=passed,
+            failed=failed,
+            warnings=warnings,
+            skipped=skipped
+        )

fi/cli/assertions/exit_codes.py

@@ -0,0 +1,20 @@
+"""Exit codes for CLI operations."""
+
+from enum import IntEnum
+
+
+class ExitCode(IntEnum):
+    """Exit codes for CI/CD integration.
+
+    These codes allow CI systems to distinguish between different
+    types of failures and take appropriate actions.
+    """
+    SUCCESS = 0                    # All evaluations and assertions passed
+    EVALUATION_ERROR = 1           # Error during evaluation execution
+    ASSERTION_FAILED = 2           # One or more assertions failed
+    ASSERTION_WARNING = 3          # Assertions passed but with warnings (--strict mode)
+    CONFIG_ERROR = 4               # Configuration file error
+    DATA_ERROR = 5                 # Test data file error
+    API_ERROR = 6                  # API connection/authentication error
+    TIMEOUT_ERROR = 7              # Evaluation timeout
+    UNKNOWN_ERROR = 99             # Unknown error

fi/cli/assertions/parser.py

@@ -0,0 +1,131 @@
+"""Parser for assertion condition strings."""
+
+import re
+from typing import List
+
+from .conditions import Condition, Operator, MetricType
+
+
+class ConditionParseError(ValueError):
+    """Error raised when a condition string cannot be parsed."""
+    pass
+
+
+class ConditionParser:
+    """Parse string conditions into Condition objects."""
+
+    # Pattern: metric_name operator value
+    PATTERN = re.compile(
+        r'^(\w+)\s*(>=|<=|==|!=|>|<)\s*([\d.]+)$'
+    )
+
+    # Pattern for between: value <= metric <= value
+    BETWEEN_PATTERN = re.compile(
+        r'^([\d.]+)\s*<=\s*(\w+)\s*<=\s*([\d.]+)$'
+    )
+
+    METRIC_MAP = {
+        'pass_rate': MetricType.PASS_RATE,
+        'avg_score': MetricType.AVG_SCORE,
+        'min_score': MetricType.MIN_SCORE,
+        'max_score': MetricType.MAX_SCORE,
+        'failed_count': MetricType.FAILED_COUNT,
+        'passed_count': MetricType.PASSED_COUNT,
+        'total_count': MetricType.TOTAL_COUNT,
+        'p50_score': MetricType.P50_SCORE,
+        'p90_score': MetricType.P90_SCORE,
+        'p95_score': MetricType.P95_SCORE,
+        'runtime_avg': MetricType.RUNTIME_AVG,
+        'runtime_p95': MetricType.RUNTIME_P95,
+        'total_pass_rate': MetricType.TOTAL_PASS_RATE,
+    }
+
+    OPERATOR_MAP = {
+        '>=': Operator.GTE,
+        '<=': Operator.LTE,
+        '>': Operator.GT,
+        '<': Operator.LT,
+        '==': Operator.EQ,
+        '!=': Operator.NEQ,
+    }
+
+    @classmethod
+    def parse(cls, condition_str: str) -> Condition:
+        """Parse a condition string into a Condition object.
+
+        Supports two formats:
+        1. Standard: "metric_name operator value" (e.g., "pass_rate >= 0.85")
+        2. Between: "value1 <= metric_name <= value2" (e.g., "0.5 <= avg_score <= 1.0")
+
+        Args:
+            condition_str: The condition string to parse.
+
+        Returns:
+            A Condition object.
+
+        Raises:
+            ConditionParseError: If the condition string is invalid.
+        """
+        condition_str = condition_str.strip()
+
+        # Try between pattern first
+        between_match = cls.BETWEEN_PATTERN.match(condition_str)
+        if between_match:
+            value1, metric, value2 = between_match.groups()
+
+            if metric not in cls.METRIC_MAP:
+                raise ConditionParseError(
+                    f"Unknown metric: {metric}. "
+                    f"Available metrics: {', '.join(sorted(cls.METRIC_MAP.keys()))}"
+                )
+
+            return Condition(
+                metric=cls.METRIC_MAP[metric],
+                operator=Operator.BETWEEN,
+                value=float(value1),
+                value2=float(value2)
+            )
+
+        # Try standard pattern
+        match = cls.PATTERN.match(condition_str)
+        if not match:
+            raise ConditionParseError(
+                f"Invalid condition format: '{condition_str}'. "
+                f"Expected format: 'metric operator value' (e.g., 'pass_rate >= 0.85')"
+            )
+
+        metric_str, operator_str, value_str = match.groups()
+
+        if metric_str not in cls.METRIC_MAP:
+            raise ConditionParseError(
+                f"Unknown metric: {metric_str}. "
+                f"Available metrics: {', '.join(sorted(cls.METRIC_MAP.keys()))}"
+            )
+
+        return Condition(
+            metric=cls.METRIC_MAP[metric_str],
+            operator=cls.OPERATOR_MAP[operator_str],
+            value=float(value_str)
+        )
+
+    @classmethod
+    def parse_many(cls, conditions: List[str]) -> List[Condition]:
+        """Parse multiple condition strings.
+
+        Args:
+            conditions: List of condition strings to parse.
+
+        Returns:
+            List of Condition objects.
+        """
+        return [cls.parse(c) for c in conditions]
+
+    @classmethod
+    def get_available_metrics(cls) -> List[str]:
+        """Get list of available metric names."""
+        return sorted(cls.METRIC_MAP.keys())
+
+    @classmethod
+    def get_available_operators(cls) -> List[str]:
+        """Get list of available operators."""
+        return list(cls.OPERATOR_MAP.keys()) + ['between']

fi/cli/assertions/reporter.py

@@ -0,0 +1,194 @@
+"""Reporter for displaying assertion results."""
+
+from typing import Dict, Any
+import xml.etree.ElementTree as ET
+from xml.dom import minidom
+
+from rich.console import Console
+from rich.table import Table
+from rich.panel import Panel
+from rich.text import Text
+
+from .evaluator import AssertionReport, AssertionResult
+
+
+class AssertionReporter:
+    """Display assertion results in various formats."""
+
+    def __init__(self, console: Console):
+        """Initialize the reporter.
+
+        Args:
+            console: Rich Console instance for output.
+        """
+        self.console = console
+
+    def display(self, report: AssertionReport, detailed: bool = True) -> None:
+        """Display assertion report to console.
+
+        Args:
+            report: The assertion report to display.
+            detailed: If True, show detailed table of all assertions.
+        """
+        if not report.outcomes:
+            self.console.print("[dim]No assertions defined[/dim]")
+            return
+
+        # Summary panel
+        summary = self._build_summary(report)
+        self.console.print(Panel(summary, title="Assertion Results", border_style="blue"))
+
+        # Detailed table
+        if detailed:
+            table = self._build_table(report)
+            self.console.print(table)
+
+    def _build_summary(self, report: AssertionReport) -> Text:
+        """Build summary text.
+
+        Args:
+            report: The assertion report.
+
+        Returns:
+            Rich Text object with formatted summary.
+        """
+        text = Text()
+        text.append(f"Total: {report.total_assertions}  ")
+        text.append(f"Passed: {report.passed}", style="green")
+        text.append("  ")
+        if report.failed > 0:
+            text.append(f"Failed: {report.failed}", style="red bold")
+        else:
+            text.append(f"Failed: {report.failed}", style="dim")
+        text.append("  ")
+        if report.warnings > 0:
+            text.append(f"Warnings: {report.warnings}", style="yellow")
+        else:
+            text.append(f"Warnings: {report.warnings}", style="dim")
+        if report.skipped > 0:
+            text.append("  ")
+            text.append(f"Skipped: {report.skipped}", style="dim")
+        return text
+
+    def _build_table(self, report: AssertionReport) -> Table:
+        """Build detailed results table.
+
+        Args:
+            report: The assertion report.
+
+        Returns:
+            Rich Table with assertion details.
+        """
+        table = Table(show_header=True, header_style="bold")
+        table.add_column("Template", style="cyan")
+        table.add_column("Condition")
+        table.add_column("Expected")
+        table.add_column("Actual", justify="right")
+        table.add_column("Result")
+
+        result_style = {
+            AssertionResult.PASSED: "[green]✓ PASS[/green]",
+            AssertionResult.FAILED: "[red]✗ FAIL[/red]",
+            AssertionResult.WARNING: "[yellow]⚠ WARN[/yellow]",
+            AssertionResult.SKIPPED: "[dim]○ SKIP[/dim]",
+        }
+
+        for outcome in report.outcomes:
+            table.add_row(
+                outcome.template or "[global]",
+                outcome.condition,
+                outcome.expected,
+                f"{outcome.actual:.4f}",
+                result_style[outcome.result]
+            )
+
+        return table
+
+    def to_json(self, report: AssertionReport) -> Dict[str, Any]:
+        """Convert report to JSON-serializable dict.
+
+        Args:
+            report: The assertion report.
+
+        Returns:
+            Dictionary suitable for JSON serialization.
+        """
+        return {
+            "summary": {
+                "total": report.total_assertions,
+                "passed": report.passed,
+                "failed": report.failed,
+                "warnings": report.warnings,
+                "skipped": report.skipped,
+                "all_passed": report.all_passed,
+            },
+            "assertions": [
+                {
+                    "template": o.template,
+                    "condition": o.condition,
+                    "expected": o.expected,
+                    "actual": o.actual,
+                    "result": o.result.value,
+                    "message": o.message,
+                }
+                for o in report.outcomes
+            ]
+        }
+
+    def to_junit(self, report: AssertionReport) -> str:
+        """Convert report to JUnit XML format for CI/CD integration.
+
+        Args:
+            report: The assertion report.
+
+        Returns:
+            JUnit XML string.
+        """
+        testsuites = ET.Element("testsuites")
+        testsuites.set("name", "Assertions")
+        testsuites.set("tests", str(report.total_assertions))
+        testsuites.set("failures", str(report.failed))
+
+        testsuite = ET.SubElement(testsuites, "testsuite")
+        testsuite.set("name", "Evaluation Assertions")
+        testsuite.set("tests", str(report.total_assertions))
+        testsuite.set("failures", str(report.failed))
+
+        for outcome in report.outcomes:
+            testcase = ET.SubElement(testsuite, "testcase")
+            testcase.set("name", f"{outcome.template or 'global'}: {outcome.condition}")
+            testcase.set("classname", "fi.assertions")
+
+            if outcome.result == AssertionResult.FAILED:
+                failure = ET.SubElement(testcase, "failure")
+                failure.set("message", outcome.message)
+                failure.text = f"Expected: {outcome.expected}, Actual: {outcome.actual}"
+            elif outcome.result == AssertionResult.SKIPPED:
+                ET.SubElement(testcase, "skipped")
+
+        xml_str = ET.tostring(testsuites, encoding="unicode")
+        dom = minidom.parseString(xml_str)
+        pretty_xml = dom.toprettyxml(indent="  ")
+
+        # Remove extra blank lines
+        lines = [line for line in pretty_xml.split("\n") if line.strip()]
+        return "\n".join(lines)
+
+    def display_summary_line(self, report: AssertionReport) -> None:
+        """Display a single-line summary suitable for end of run output.
+
+        Args:
+            report: The assertion report.
+        """
+        if report.failed > 0:
+            self.console.print(
+                f"\n[red]✗ {report.failed} assertion(s) failed[/red]"
+            )
+        elif report.warnings > 0:
+            self.console.print(
+                f"\n[yellow]⚠ {report.passed} assertion(s) passed with {report.warnings} warning(s)[/yellow]"
+            )
+        else:
+            self.console.print(
+                f"\n[green]✓ All {report.passed} assertion(s) passed[/green]"
+            )

fi/cli/commands/__init__.py

@@ -0,0 +1,9 @@
+"""CLI Commands Package"""
+
+from fi.cli.commands.init import init_project
+from fi.cli.commands.run import run
+from fi.cli.commands.list_cmd import list_resources
+from fi.cli.commands.validate import validate
+from fi.cli.commands.config import config_app
+
+__all__ = ["init_project", "run", "list_resources", "validate", "config_app"]