evalgate-sdk 3.3.1__py3-none-any.whl

evalgate_sdk/streaming.py

@@ -0,0 +1,124 @@
+"""Streaming evaluation, batch reading, and rate-limiting utilities."""
+
+from __future__ import annotations
+
+import asyncio
+import time
+from collections.abc import AsyncIterator, Callable, Coroutine
+from dataclasses import dataclass, field
+from typing import Any, Generic, TypeVar
+
+T = TypeVar("T")
+R = TypeVar("R")
+
+
+@dataclass
+class BatchProgress:
+    completed: int = 0
+    total: int = 0
+    failed: int = 0
+    elapsed_ms: float = 0
+
+
+@dataclass
+class BatchResult(Generic[R]):
+    results: list[R] = field(default_factory=list)
+    errors: list[dict[str, Any]] = field(default_factory=list)
+    total: int = 0
+    succeeded: int = 0
+    failed: int = 0
+    duration_ms: float = 0
+
+
+class RateLimiter:
+    """Token-bucket rate limiter.
+
+    Usage::
+
+        limiter = RateLimiter(requests_per_second=10)
+        result = await limiter.throttle(my_async_fn)
+    """
+
+    def __init__(self, requests_per_second: float = 10) -> None:
+        self._interval = 1.0 / requests_per_second
+        self._last_call = 0.0
+        self._lock = asyncio.Lock()
+
+    async def throttle(self, fn: Callable[[], Coroutine[Any, Any, T]]) -> T:
+        async with self._lock:
+            now = time.monotonic()
+            wait = self._interval - (now - self._last_call)
+            if wait > 0:
+                await asyncio.sleep(wait)
+            self._last_call = time.monotonic()
+        return await fn()
+
+
+def chunk(items: list[T], size: int) -> list[list[T]]:
+    """Split a list into chunks of *size*."""
+    return [items[i : i + size] for i in range(0, len(items), size)]
+
+
+async def stream_evaluation(
+    evaluator: Callable[[str], Coroutine[Any, Any, str]],
+    inputs: list[str],
+    *,
+    concurrency: int = 3,
+    on_progress: Callable[[BatchProgress], None] | None = None,
+) -> AsyncIterator[dict[str, Any]]:
+    """Stream evaluation results as they complete.
+
+    Yields dicts with ``input``, ``output``, ``index``, ``duration_ms``, and optionally ``error``.
+    """
+    semaphore = asyncio.Semaphore(concurrency)
+    progress = BatchProgress(total=len(inputs))
+    start = time.monotonic()
+
+    async def _run(index: int, input_text: str) -> dict[str, Any]:
+        async with semaphore:
+            t0 = time.monotonic()
+            try:
+                output = await evaluator(input_text)
+                elapsed = (time.monotonic() - t0) * 1000
+                return {"index": index, "input": input_text, "output": output, "duration_ms": elapsed}
+            except Exception as exc:
+                elapsed = (time.monotonic() - t0) * 1000
+                return {"index": index, "input": input_text, "error": str(exc), "duration_ms": elapsed}
+
+    tasks = [asyncio.create_task(_run(i, inp)) for i, inp in enumerate(inputs)]
+
+    for coro in asyncio.as_completed(tasks):
+        result = await coro
+        progress.completed += 1
+        if "error" in result:
+            progress.failed += 1
+        progress.elapsed_ms = (time.monotonic() - start) * 1000
+        if on_progress:
+            on_progress(progress)
+        yield result
+
+
+async def batch_read(
+    fetcher: Callable[[int, int], Coroutine[Any, Any, list[T]]],
+    *,
+    limit: int = 100,
+    max_pages: int = 100,
+) -> list[T]:
+    """Read all pages from a paginated API endpoint.
+
+    Args:
+        fetcher: Async function(offset, limit) -> list of items.
+        limit: Page size.
+        max_pages: Safety cap on number of pages.
+    """
+    all_items: list[T] = []
+    offset = 0
+    for _ in range(max_pages):
+        page = await fetcher(offset, limit)
+        if not page:
+            break
+        all_items.extend(page)
+        if len(page) < limit:
+            break
+        offset += limit
+    return all_items

evalgate_sdk/synthesize.py

@@ -0,0 +1,226 @@
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from typing import Any
+
+from evalgate_sdk.golden import DEFAULT_SYNTHETIC_DATASET_PATH, LabeledGoldenCase, SyntheticGoldenCase
+
+
+@dataclass(slots=True)
+class SynthesizeSummary:
+    source_cases: int
+    source_failures: int
+    selected_failure_modes: list[str]
+    dimension_names: list[str]
+    dimension_combination_count: int
+    generated: int
+    mode_counts: list[dict[str, Any]]
+    output_path: str
+    cases: list[SyntheticGoldenCase]
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "sourceCases": self.source_cases,
+            "sourceFailures": self.source_failures,
+            "selectedFailureModes": list(self.selected_failure_modes),
+            "dimensionNames": list(self.dimension_names),
+            "dimensionCombinationCount": self.dimension_combination_count,
+            "generated": self.generated,
+            "modeCounts": list(self.mode_counts),
+            "outputPath": self.output_path,
+            "cases": [item.to_dict() for item in self.cases],
+        }
+
+
+@dataclass(slots=True)
+class DimensionMatrix:
+    dimensions: dict[str, list[str]]
+
+
+def parse_dimension_matrix(content: str) -> DimensionMatrix:
+    try:
+        parsed = json.loads(content)
+    except json.JSONDecodeError as exc:
+        raise ValueError("Dimension matrix must be valid JSON") from exc
+    if not isinstance(parsed, dict):
+        raise ValueError("Dimension matrix must be a JSON object")
+    raw_dimensions = parsed.get("dimensions") if isinstance(parsed.get("dimensions"), dict) else parsed
+    if not isinstance(raw_dimensions, dict):
+        raise ValueError("Dimension matrix must be a JSON object")
+    dimensions: dict[str, list[str]] = {}
+    for name, values in raw_dimensions.items():
+        if not isinstance(values, list):
+            raise ValueError(f"Dimension '{name}' must be an array of strings")
+        normalized: list[str] = []
+        for value in values:
+            if not isinstance(value, str):
+                raise ValueError(f"Dimension '{name}' must contain only strings")
+            stripped = value.strip()
+            if stripped:
+                normalized.append(stripped)
+        if not normalized:
+            raise ValueError(f"Dimension '{name}' must contain at least one value")
+        dimensions[str(name)] = normalized
+    return DimensionMatrix(dimensions=dimensions)
+
+
+def _cartesian_dimensions(dimensions: dict[str, list[str]]) -> list[dict[str, str]]:
+    entries = list(dimensions.items())
+    if not entries:
+        return [{}]
+    combinations: list[dict[str, str]] = [{}]
+    for name, values in entries:
+        next_items: list[dict[str, str]] = []
+        for combination in combinations:
+            for value in values:
+                candidate = dict(combination)
+                candidate[name] = value
+                next_items.append(candidate)
+        combinations = next_items
+    return combinations
+
+
+def _slugify(value: str) -> str:
+    result = []
+    last_dash = False
+    for char in value.lower():
+        if char.isalnum():
+            result.append(char)
+            last_dash = False
+        elif not last_dash:
+            result.append("-")
+            last_dash = True
+    return "".join(result).strip("-")[:48]
+
+
+def _dimension_label(dimensions: dict[str, str]) -> str:
+    pairs = [f"{name}={value}" for name, value in dimensions.items()]
+    return ", ".join(pairs) if pairs else "base"
+
+
+def _build_synthetic_case(
+    prototype: LabeledGoldenCase,
+    failure_mode: str,
+    dimensions: dict[str, str],
+    sequence: int,
+) -> SyntheticGoldenCase:
+    from datetime import datetime, timezone
+
+    timestamp = datetime.now(timezone.utc).isoformat()
+    dims_text = _dimension_label(dimensions)
+    dimension_suffix = _slugify(dims_text) or "base"
+    mode_suffix = _slugify(failure_mode) or "failure-mode"
+    input_parts = [prototype.input.strip()]
+    expected_parts = [prototype.expected.strip()]
+    actual_parts = [f"Representative {failure_mode} failure draft.", prototype.actual.strip()]
+    if dims_text != "base":
+        input_parts.append(f"Synthetic dimensions: {dims_text}")
+        expected_parts.append(f"Target dimensions: {dims_text}")
+        actual_parts.insert(1, f"Scenario dimensions: {dims_text}")
+    return SyntheticGoldenCase(
+        case_id=f"synthetic-{mode_suffix}-{dimension_suffix}-{str(sequence + 1).zfill(3)}",
+        input="\n".join(part for part in input_parts if part),
+        expected="\n".join(part for part in expected_parts if part),
+        actual="\n".join(part for part in actual_parts if part),
+        label="fail",
+        failure_mode=failure_mode,
+        labeled_at=timestamp,
+        synthetic=True,
+        synthesized_at=timestamp,
+        source_case_ids=[prototype.case_id],
+        dimensions=dict(dimensions),
+        cluster_id=prototype.cluster_id,
+        cluster_label=prototype.cluster_label,
+    )
+
+
+def synthesize_labeled_dataset(
+    rows: list[LabeledGoldenCase],
+    *,
+    dimensions: dict[str, list[str]] | None = None,
+    count: int | None = None,
+    failure_modes: list[str] | None = None,
+    output_path: str = DEFAULT_SYNTHETIC_DATASET_PATH,
+) -> SynthesizeSummary:
+    failed_rows = [row for row in rows if row.label == "fail" and isinstance(row.failure_mode, str) and row.failure_mode.strip()]
+    grouped: dict[str, list[LabeledGoldenCase]] = {}
+    for row in failed_rows:
+        key = row.failure_mode.strip()
+        grouped.setdefault(key, []).append(row)
+
+    requested_modes = [mode.strip() for mode in (failure_modes or []) if mode.strip()]
+    if requested_modes:
+        selected_failure_modes = [mode for mode in requested_modes if mode in grouped]
+    else:
+        selected_failure_modes = [
+            mode for mode, _items in sorted(grouped.items(), key=lambda item: (-len(item[1]), item[0]))
+        ]
+
+    dimension_matrix = dimensions or {}
+    combinations = _cartesian_dimensions(dimension_matrix)
+    dimension_names = list(dimension_matrix.keys())
+
+    plan: list[tuple[str, dict[str, str]]] = []
+    for failure_mode in selected_failure_modes:
+        for combo in combinations:
+            plan.append((failure_mode, combo))
+
+    if count is not None:
+        target_count = max(0, count)
+    elif plan:
+        target_count = len(plan)
+    else:
+        target_count = len(selected_failure_modes)
+
+    cases: list[SyntheticGoldenCase] = []
+    if target_count > 0 and plan:
+        for index in range(target_count):
+            failure_mode, combo = plan[index % len(plan)]
+            source_rows = grouped[failure_mode]
+            prototype = source_rows[(index // max(1, len(plan))) % len(source_rows)]
+            cases.append(_build_synthetic_case(prototype, failure_mode, combo, index))
+
+    counts: dict[str, int] = {}
+    for case in cases:
+        mode = case.failure_mode or "unknown"
+        counts[mode] = counts.get(mode, 0) + 1
+
+    return SynthesizeSummary(
+        source_cases=len(rows),
+        source_failures=len(failed_rows),
+        selected_failure_modes=selected_failure_modes,
+        dimension_names=dimension_names,
+        dimension_combination_count=len(combinations),
+        generated=len(cases),
+        mode_counts=[
+            {"failureMode": mode, "count": count_value}
+            for mode, count_value in sorted(counts.items(), key=lambda item: (-item[1], item[0]))
+        ],
+        output_path=output_path,
+        cases=cases,
+    )
+
+
+def format_synthesize_human(summary: SynthesizeSummary) -> str:
+    lines = [
+        "Synthesize phase",
+        f"Source cases: {summary.source_cases}",
+        f"Source failures: {summary.source_failures}",
+        f"Failure modes used: {len(summary.selected_failure_modes)}",
+        f"Dimension combinations: {summary.dimension_combination_count}",
+        f"Generated synthetic cases: {summary.generated}",
+    ]
+    if summary.selected_failure_modes:
+        lines.append(f"Modes: {', '.join(summary.selected_failure_modes)}")
+    if summary.dimension_names:
+        lines.append(f"Dimensions: {', '.join(summary.dimension_names)}")
+    if summary.mode_counts:
+        mode_line = ", ".join(f"{item['failureMode']} ×{item['count']}" for item in summary.mode_counts)
+        lines.append(f"Mode counts: {mode_line}")
+    if summary.cases:
+        samples = ", ".join(
+            f"{item.case_id} ({_dimension_label(item.dimensions)})" for item in summary.cases[:3]
+        )
+        lines.append(f"Samples: {samples}")
+    return "\n".join(lines)

evalgate_sdk/testing.py

@@ -0,0 +1,128 @@
+"""Test suite builder for running structured evaluations against LLM outputs."""
+
+from __future__ import annotations
+
+import time
+from typing import Any
+
+from evalgate_sdk.assertions import AssertionResult, expect
+from evalgate_sdk.types import (
+    TestSuiteCase,
+    TestSuiteCaseResult,
+    TestSuiteConfig,
+    TestSuiteResult,
+)
+
+
+class TestSuite:
+    """A collection of test cases that can be executed against an evaluator function.
+
+    Usage::
+
+        suite = create_test_suite("My Suite", config=TestSuiteConfig(
+            test_cases=[
+                TestSuiteCase(name="greet", input="Hello", expected_output="Hi"),
+            ],
+            evaluator=my_llm_call,
+        ))
+        result = await suite.run()
+    """
+
+    def __init__(self, name: str, config: TestSuiteConfig) -> None:
+        self._name = name
+        self._config = config
+        self._cases: list[TestSuiteCase] = list(config.test_cases)
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    def add_case(self, case: TestSuiteCase) -> None:
+        self._cases.append(case)
+
+    def get_config(self) -> TestSuiteConfig:
+        return self._config
+
+    def get_cases(self) -> list[TestSuiteCase]:
+        return list(self._cases)
+
+    async def run(self) -> TestSuiteResult:
+        evaluator = self._config.evaluator
+        if evaluator is None:
+            raise ValueError("No evaluator provided in TestSuiteConfig")
+
+        results: list[TestSuiteCaseResult] = []
+        suite_start = time.monotonic()
+
+        for case in self._cases:
+            case_start = time.monotonic()
+            output: str | None = None
+            error: str | None = None
+            assertions: list[AssertionResult] = []
+
+            try:
+                raw = evaluator(case.input)
+                output = str(await raw) if hasattr(raw, "__await__") or hasattr(raw, "__anext__") else str(raw)
+            except Exception as exc:
+                error = str(exc)
+
+            if output is not None and case.assertions:
+                for assertion_def in case.assertions:
+                    a_type = assertion_def.get("type", "")
+                    a_value = assertion_def.get("value")
+                    exp = expect(output)
+                    if a_type == "contains" and isinstance(a_value, str):
+                        assertions.append(exp.to_contain(a_value))
+                    elif a_type == "equals" and a_value is not None:
+                        assertions.append(exp.to_equal(a_value))
+                    elif a_type == "not_contains_pii":
+                        assertions.append(exp.to_not_contain_pii())
+
+            if output is not None and case.expected_output is not None and not assertions:
+                assertions.append(expect(output).to_equal(case.expected_output))
+
+            case_passed = error is None and all(a.passed for a in assertions)
+            duration = int((time.monotonic() - case_start) * 1000)
+
+            results.append(
+                TestSuiteCaseResult(
+                    name=case.name,
+                    passed=case_passed,
+                    duration_ms=duration,
+                    input=case.input,
+                    output=output,
+                    expected_output=case.expected_output,
+                    assertions=assertions,
+                    error=error,
+                )
+            )
+
+        total_duration = int((time.monotonic() - suite_start) * 1000)
+        passed_count = sum(1 for r in results if r.passed)
+
+        return TestSuiteResult(
+            suite_name=self._name,
+            passed=all(r.passed for r in results),
+            total=len(results),
+            passed_count=passed_count,
+            failed_count=len(results) - passed_count,
+            duration_ms=total_duration,
+            results=results,
+        )
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "name": self._name,
+            "config": self._config.model_dump(),
+            "cases": [c.model_dump() for c in self._cases],
+        }
+
+
+def create_test_suite(name: str, config: TestSuiteConfig) -> TestSuite:
+    """Create a new test suite.
+
+    Args:
+        name: Human-readable suite name.
+        config: Suite configuration including test cases and evaluator.
+    """
+    return TestSuite(name, config)