evalgate-sdk 3.3.1__py3-none-any.whl

evalgate_sdk/auto.py

@@ -0,0 +1,247 @@
+from __future__ import annotations
+
+import json
+import time
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Literal
+
+from evalgate_sdk.golden import extract_run_metrics
+from evalgate_sdk.replay_decision import NormalizedBudgetConfig, ReplayDecision, evaluate_replay_outcome
+
+DEFAULT_AUTO_HISTORY_PATH = str(Path(".evalgate") / "auto" / "history.jsonl")
+DEFAULT_AUTO_REPORT_PATH = str(Path(".evalgate") / "auto" / "latest-report.json")
+
+AutoFormat = Literal["human", "json"]
+AutoExecutionMode = Literal["plan", "evaluate"]
+AutoDecision = Literal["keep", "discard", "investigate"]
+
+
+@dataclass(slots=True)
+class AutoPlanStep:
+    title: str
+    detail: str
+
+    def to_dict(self) -> dict[str, str]:
+        return {
+            "title": self.title,
+            "detail": self.detail,
+        }
+
+
+@dataclass(slots=True)
+class AutoOptions:
+    objective: str
+    hypothesis: str | None = None
+    prompt_path: str | None = None
+    baseline_run_path: str | None = None
+    candidate_run_path: str | None = None
+    budget: int = 1
+    budget_mode: Literal["traces", "cost"] = "traces"
+    autonomous: bool = False
+    dry_run: bool = False
+    format: AutoFormat = "human"
+    report_path: str = DEFAULT_AUTO_REPORT_PATH
+
+
+@dataclass(slots=True)
+class AutoDiffSnapshot:
+    pass_rate_delta_ratio: float
+    corrected_pass_rate_delta_ratio: float | None
+    pass_rate_basis: Literal["corrected", "raw"]
+    regressions: int = 0
+    improvements: int = 0
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "passRateDeltaRatio": self.pass_rate_delta_ratio,
+            "correctedPassRateDeltaRatio": self.corrected_pass_rate_delta_ratio,
+            "passRateBasis": self.pass_rate_basis,
+            "regressions": self.regressions,
+            "improvements": self.improvements,
+        }
+
+
+@dataclass(slots=True)
+class AutoIterationResult:
+    decision: AutoDecision
+    reason: str
+    replay_decision: ReplayDecision | None = None
+    diff: AutoDiffSnapshot | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "decision": self.decision,
+            "reason": self.reason,
+            "replayDecision": self.replay_decision.to_dict() if self.replay_decision else None,
+            "diff": self.diff.to_dict() if self.diff else None,
+        }
+
+
+@dataclass(slots=True)
+class AutoReport:
+    objective: str
+    execution_mode: AutoExecutionMode
+    dry_run: bool
+    autonomous: bool
+    prompt_path: str | None
+    baseline_run_path: str | None
+    candidate_run_path: str | None
+    iteration_budget: int
+    generated_at: str
+    plan: list[AutoPlanStep] = field(default_factory=list)
+    iteration_result: AutoIterationResult | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "objective": self.objective,
+            "executionMode": self.execution_mode,
+            "dryRun": self.dry_run,
+            "autonomous": self.autonomous,
+            "promptPath": self.prompt_path,
+            "baselineRunPath": self.baseline_run_path,
+            "candidateRunPath": self.candidate_run_path,
+            "iterationBudget": self.iteration_budget,
+            "generatedAt": self.generated_at,
+            "plan": [step.to_dict() for step in self.plan],
+            "iterationResult": self.iteration_result.to_dict() if self.iteration_result else None,
+        }
+
+
+def build_auto_plan(options: AutoOptions) -> list[AutoPlanStep]:
+    prompt_target = options.prompt_path or "declared prompt target"
+    hypothesis = options.hypothesis or f"Reduce {options.objective} failures"
+    return [
+        AutoPlanStep(title="Objective", detail=f"Target failure mode: {options.objective}"),
+        AutoPlanStep(title="Hypothesis", detail=hypothesis),
+        AutoPlanStep(title="Mutation target", detail=prompt_target),
+        AutoPlanStep(title="Budget", detail=f"{options.budget} {options.budget_mode} per iteration"),
+        AutoPlanStep(
+            title="Evaluation",
+            detail=(
+                "Compare candidate run to baseline and keep only non-regressing improvements"
+                if options.baseline_run_path and options.candidate_run_path
+                else "Generate bounded plan only until baseline/candidate artifacts are provided"
+            ),
+        ),
+    ]
+
+
+def _read_json(file_path: str) -> dict[str, Any]:
+    data = json.loads(Path(file_path).read_text(encoding="utf-8"))
+    if not isinstance(data, dict):
+        raise ValueError("Auto run input must be a JSON object")
+    return data
+
+
+def _build_diff_snapshot(previous_run: dict[str, Any], candidate_run: dict[str, Any], decision: ReplayDecision) -> AutoDiffSnapshot:
+    previous_metrics = extract_run_metrics(previous_run)
+    candidate_metrics = extract_run_metrics(candidate_run)
+    corrected_delta = None
+    if previous_metrics.corrected_pass_rate_ratio is not None and candidate_metrics.corrected_pass_rate_ratio is not None:
+        corrected_delta = candidate_metrics.corrected_pass_rate_ratio - previous_metrics.corrected_pass_rate_ratio
+    return AutoDiffSnapshot(
+        pass_rate_delta_ratio=candidate_metrics.pass_rate_ratio - previous_metrics.pass_rate_ratio,
+        corrected_pass_rate_delta_ratio=corrected_delta,
+        pass_rate_basis=decision.comparison_basis,
+        regressions=0 if decision.action == "keep" else 1,
+        improvements=1 if decision.action == "keep" else 0,
+    )
+
+
+def decide_auto_experiment(options: AutoOptions) -> AutoIterationResult | None:
+    if not options.baseline_run_path or not options.candidate_run_path:
+        return None
+    previous_run = _read_json(options.baseline_run_path)
+    candidate_run = _read_json(options.candidate_run_path)
+    budget_config = NormalizedBudgetConfig(
+        mode=options.budget_mode,
+        max_traces=options.budget if options.budget_mode == "traces" else None,
+        max_cost_usd=float(options.budget) if options.budget_mode == "cost" else None,
+    )
+    replay = evaluate_replay_outcome(previous_run, candidate_run, budget_config)
+    if replay.action == "keep":
+        decision: AutoDecision = "keep"
+    elif replay.reason == "budget_exceeded":
+        decision = "investigate"
+    else:
+        decision = "discard"
+    return AutoIterationResult(
+        decision=decision,
+        reason=replay.reason,
+        replay_decision=replay,
+        diff=_build_diff_snapshot(previous_run, candidate_run, replay),
+    )
+
+
+def build_auto_report(options: AutoOptions) -> AutoReport:
+    execution_mode: AutoExecutionMode = (
+        "evaluate" if options.baseline_run_path and options.candidate_run_path and not options.dry_run else "plan"
+    )
+    return AutoReport(
+        objective=options.objective,
+        execution_mode=execution_mode,
+        dry_run=options.dry_run,
+        autonomous=options.autonomous,
+        prompt_path=options.prompt_path,
+        baseline_run_path=options.baseline_run_path,
+        candidate_run_path=options.candidate_run_path,
+        iteration_budget=options.budget,
+        generated_at=datetime.now(timezone.utc).isoformat(),
+        plan=build_auto_plan(options),
+        iteration_result=decide_auto_experiment(options) if execution_mode == "evaluate" else None,
+    )
+
+
+def format_auto_human(report: AutoReport) -> str:
+    lines = [
+        "Auto phase",
+        f"Objective: {report.objective}",
+        f"Mode: {report.execution_mode}",
+        f"Budget: {report.iteration_budget}",
+    ]
+    if report.prompt_path:
+        lines.append(f"Prompt: {report.prompt_path}")
+    lines.append("Plan:")
+    for index, step in enumerate(report.plan, start=1):
+        lines.append(f"{index}. {step.title} — {step.detail}")
+    if report.iteration_result:
+        lines.append("")
+        lines.append(f"Decision: {report.iteration_result.decision}")
+        lines.append(f"Reason: {report.iteration_result.reason}")
+        if report.iteration_result.diff:
+            lines.append(
+                f"Pass-rate delta: {report.iteration_result.diff.pass_rate_delta_ratio:+.4f} ({report.iteration_result.diff.pass_rate_basis})"
+            )
+    return "\n".join(lines)
+
+
+def append_auto_history(report: AutoReport, history_path: str = DEFAULT_AUTO_HISTORY_PATH) -> None:
+    path = Path(history_path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("a", encoding="utf-8") as handle:
+        handle.write(json.dumps(report.to_dict(), separators=(",", ":")) + "\n")
+
+
+def write_auto_report(report: AutoReport, report_path: str | None = None) -> None:
+    path = Path(report_path or DEFAULT_AUTO_REPORT_PATH)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(report.to_dict(), indent=2), encoding="utf-8")
+
+
+def run_auto_daemon(
+    options: AutoOptions,
+    *,
+    cycles: int,
+    interval_ms: int = 0,
+    history_path: str = DEFAULT_AUTO_HISTORY_PATH,
+) -> list[AutoReport]:
+    reports: list[AutoReport] = []
+    for index in range(max(0, cycles)):
+        report = build_auto_report(options)
+        append_auto_history(report, history_path=history_path)
+        reports.append(report)
+        if interval_ms > 0 and index < cycles - 1:
+            time.sleep(interval_ms / 1000.0)
+    return reports

evalgate_sdk/batch.py

@@ -0,0 +1,174 @@
+"""Request batching and concurrent processing utilities."""
+
+from __future__ import annotations
+
+import asyncio
+import inspect
+import uuid
+from collections.abc import Callable, Coroutine
+from typing import Any, TypeVar
+
+T = TypeVar("T")
+R = TypeVar("R")
+
+_BATCHABLE_ENDPOINTS = {"/api/traces", "/api/evaluations", "/api/annotations"}
+
+def _is_async_callable(processor: Any) -> bool:
+    return inspect.iscoroutinefunction(processor) or inspect.iscoroutinefunction(
+        getattr(processor, "__call__", None)
+    )
+
+
+def can_batch(method: str, endpoint: str) -> bool:
+    """Check if a request is eligible for batching (POST to known endpoints)."""
+    return method.upper() == "POST" and any(endpoint.startswith(ep) for ep in _BATCHABLE_ENDPOINTS)
+
+
+class RequestBatcher:
+    """Accumulates requests and flushes them in batches.
+
+    Usage::
+
+        batcher = RequestBatcher(flush_fn=my_http_batch, max_batch_size=10)
+        result = await batcher.enqueue("POST", "/api/traces", body={...})
+    """
+
+    def __init__(
+        self,
+        flush_fn: Callable[[list[dict[str, Any]]], Any],
+        max_batch_size: int = 10,
+        delay_ms: int = 50,
+    ) -> None:
+        self._flush_fn = flush_fn
+        self._max_batch_size = max_batch_size
+        self._delay = delay_ms / 1000
+        self._queue: list[dict[str, Any]] = []
+        self._pending: dict[str, asyncio.Future[Any]] = {}
+        self._timer: asyncio.TimerHandle | None = None
+
+    async def enqueue(
+        self,
+        method: str,
+        endpoint: str,
+        body: Any | None = None,
+        headers: dict[str, str] | None = None,
+    ) -> Any:
+        request_id = str(uuid.uuid4())
+        loop = asyncio.get_running_loop()
+        future: asyncio.Future[Any] = loop.create_future()
+        self._pending[request_id] = future
+
+        self._queue.append(
+            {
+                "id": request_id,
+                "method": method,
+                "endpoint": endpoint,
+                "body": body,
+                "headers": headers or {},
+            }
+        )
+
+        if len(self._queue) >= self._max_batch_size:
+            await self.flush()
+        elif self._timer is None:
+            self._timer = loop.call_later(self._delay, lambda: asyncio.ensure_future(self.flush()))
+
+        return await future
+
+    async def flush(self) -> None:
+        if not self._queue:
+            return
+
+        if self._timer is not None:
+            self._timer.cancel()
+            self._timer = None
+
+        batch = self._queue[:]
+        self._queue.clear()
+
+        try:
+            results = await self._flush_fn(batch)
+            if isinstance(results, list):
+                for req, result in zip(batch, results, strict=False):
+                    fut = self._pending.pop(req["id"], None)
+                    if fut and not fut.done():
+                        fut.set_result(result)
+            else:
+                for req in batch:
+                    fut = self._pending.pop(req["id"], None)
+                    if fut and not fut.done():
+                        fut.set_result(results)
+        except Exception as exc:
+            for req in batch:
+                fut = self._pending.pop(req["id"], None)
+                if fut and not fut.done():
+                    fut.set_exception(exc)
+
+    def clear(self) -> None:
+        if self._timer is not None:
+            self._timer.cancel()
+            self._timer = None
+        for fut in self._pending.values():
+            if not fut.done():
+                fut.cancel()
+        self._queue.clear()
+        self._pending.clear()
+
+    def get_stats(self) -> dict[str, int]:
+        return {"queue_size": len(self._queue), "max_batch_size": self._max_batch_size}
+
+
+async def batch_process(
+    items: list[T],
+    processor: Callable[[T], Coroutine[Any, Any, R]],
+    concurrency: int = 5,
+    *,
+    continue_on_error: bool = False,
+    on_progress: Callable[[int, int], None] | None = None,
+) -> list[R]:
+    """Process items with bounded concurrency.
+
+    Args:
+        items: Items to process.
+        processor: Async function to apply to each item.
+        concurrency: Max concurrent tasks.
+        continue_on_error: If True, collect errors instead of raising.
+        on_progress: Optional callback (completed, total).
+
+    Returns:
+        Results in same order as items.
+    """
+    if not callable(processor):
+        raise TypeError("batch_process requires processor to be callable")
+
+    if not _is_async_callable(processor):
+        raise TypeError("batch_process requires processor to be an async callable")
+
+    if concurrency < 1:
+        raise ValueError("batch_process concurrency must be at least 1")
+
+    semaphore = asyncio.Semaphore(concurrency)
+    results: list[R | None] = [None] * len(items)
+    errors: list[Exception | None] = [None] * len(items)
+    completed = 0
+
+    async def _run(index: int, item: T) -> None:
+        nonlocal completed
+        async with semaphore:
+            try:
+                results[index] = await processor(item)
+            except Exception as exc:
+                if continue_on_error:
+                    errors[index] = exc
+                else:
+                    raise
+            finally:
+                completed += 1
+                if on_progress:
+                    on_progress(completed, len(items))
+
+    await asyncio.gather(*[_run(i, item) for i, item in enumerate(items)])
+
+    if not continue_on_error:
+        return [r for r in results if r is not None] if None not in results else results  # type: ignore[return-value]
+    return results  # type: ignore[return-value]

evalgate_sdk/cache.py

@@ -0,0 +1,111 @@
+"""LRU request cache with TTL expiration."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import time
+from collections import OrderedDict
+from typing import Any, TypeVar
+
+T = TypeVar("T")
+
+
+class CacheTTL:
+    """Pre-defined TTL values in seconds."""
+
+    SHORT = 30
+    MEDIUM = 300
+    LONG = 1800
+    HOUR = 3600
+
+
+_CACHEABLE_PREFIXES = (
+    "/api/traces",
+    "/api/evaluations",
+    "/api/organizations",
+    "/api/developer",
+)
+
+
+def should_cache(method: str, endpoint: str) -> bool:
+    """Check if a request is cacheable (GET to known endpoints)."""
+    return method.upper() == "GET" and any(endpoint.startswith(p) for p in _CACHEABLE_PREFIXES)
+
+
+def get_ttl(endpoint: str) -> int:
+    """Return the appropriate TTL for an endpoint."""
+    if "/organizations" in endpoint:
+        return CacheTTL.LONG
+    if "/developer" in endpoint:
+        return CacheTTL.MEDIUM
+    return CacheTTL.SHORT
+
+
+class RequestCache:
+    """In-memory LRU cache with per-entry TTL.
+
+    Usage::
+
+        cache = RequestCache(max_size=500)
+        cache.set("GET", "/api/traces", data, CacheTTL.SHORT)
+        hit = cache.get("GET", "/api/traces")  # returns data or None
+    """
+
+    def __init__(self, max_size: int = 1000) -> None:
+        self._max_size = max_size
+        self._store: OrderedDict[str, _CacheEntry] = OrderedDict()
+
+    @staticmethod
+    def _key(method: str, url: str, params: Any = None) -> str:
+        raw = f"{method.upper()}:{url}"
+        if params:
+            raw += f":{json.dumps(params, sort_keys=True, default=str)}"
+        return hashlib.sha256(raw.encode()).hexdigest()
+
+    def get(self, method: str, url: str, params: Any = None) -> Any | None:
+        key = self._key(method, url, params)
+        entry = self._store.get(key)
+        if entry is None:
+            return None
+        if entry.is_expired():
+            del self._store[key]
+            return None
+        self._store.move_to_end(key)
+        return entry.data
+
+    def set(self, method: str, url: str, data: Any, ttl: int, params: Any = None) -> None:
+        key = self._key(method, url, params)
+        if key in self._store:
+            del self._store[key]
+        elif len(self._store) >= self._max_size:
+            self._store.popitem(last=False)
+        self._store[key] = _CacheEntry(data=data, ttl=ttl)
+
+    def invalidate(self, method: str, url: str, params: Any = None) -> None:
+        key = self._key(method, url, params)
+        self._store.pop(key, None)
+
+    def invalidate_pattern(self, pattern: str) -> None:
+        """Remove all entries whose URL key contains *pattern*."""
+        to_remove = [k for k, v in self._store.items() if pattern in v.url_hint]
+        for k in to_remove:
+            del self._store[k]
+
+    def clear(self) -> None:
+        self._store.clear()
+
+    def get_stats(self) -> dict[str, int]:
+        return {"size": len(self._store), "max_size": self._max_size}
+
+
+class _CacheEntry:
+    __slots__ = ("data", "expires_at", "url_hint")
+
+    def __init__(self, data: Any, ttl: int, url_hint: str = "") -> None:
+        self.data = data
+        self.expires_at = time.monotonic() + ttl
+        self.url_hint = url_hint
+
+    def is_expired(self) -> bool:
+        return time.monotonic() > self.expires_at

evalgate_sdk/ci_context.py

@@ -0,0 +1,123 @@
+"""CI context auto-detection (T10).
+
+Port of the TypeScript SDK's ``ci-context.ts``.
+Detects GitHub Actions, GitLab CI, CircleCI, and other CI providers.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import os
+from dataclasses import dataclass
+from typing import Literal
+
+CIProvider = Literal["github", "gitlab", "circle", "azure", "jenkins", "unknown"]
+
+
+@dataclass
+class CIContext:
+    provider: CIProvider = "unknown"
+    repo: str | None = None
+    sha: str | None = None
+    branch: str | None = None
+    pr: int | None = None
+    run_url: str | None = None
+    actor: str | None = None
+    is_ci: bool = False
+
+
+def detect_ci_context() -> CIContext:
+    """Auto-detect CI provider and extract context from environment variables."""
+    if os.environ.get("GITHUB_ACTIONS") == "true":
+        return _github_context()
+    if os.environ.get("GITLAB_CI") == "true":
+        return _gitlab_context()
+    if os.environ.get("CIRCLECI") == "true":
+        return _circle_context()
+    if os.environ.get("TF_BUILD") == "true":
+        return _azure_context()
+    if os.environ.get("JENKINS_URL"):
+        return _jenkins_context()
+    if any(os.environ.get(k) for k in ("CI", "CONTINUOUS_INTEGRATION", "BUILD_NUMBER")):
+        return CIContext(provider="unknown", is_ci=True)
+    return CIContext()
+
+
+def _github_context() -> CIContext:
+    ref = os.environ.get("GITHUB_REF", "")
+    pr_num = None
+    if "/pull/" in ref:
+        with contextlib.suppress(ValueError, IndexError):
+            pr_num = int(ref.split("/pull/")[1].split("/")[0])
+
+    server = os.environ.get("GITHUB_SERVER_URL", "https://github.com")
+    repo = os.environ.get("GITHUB_REPOSITORY", "")
+    run_id = os.environ.get("GITHUB_RUN_ID", "")
+    run_url = f"{server}/{repo}/actions/runs/{run_id}" if repo and run_id else None
+    branch = os.environ.get("GITHUB_REF_NAME")
+    if "/pull/" in ref:
+        branch = os.environ.get("GITHUB_HEAD_REF") or branch
+
+    return CIContext(
+        provider="github",
+        repo=repo or None,
+        sha=os.environ.get("GITHUB_SHA"),
+        branch=branch,
+        pr=pr_num,
+        run_url=run_url,
+        actor=os.environ.get("GITHUB_ACTOR"),
+        is_ci=True,
+    )
+
+
+def _gitlab_context() -> CIContext:
+    mr_iid = os.environ.get("CI_MERGE_REQUEST_IID")
+    return CIContext(
+        provider="gitlab",
+        repo=os.environ.get("CI_PROJECT_PATH"),
+        sha=os.environ.get("CI_COMMIT_SHA"),
+        branch=os.environ.get("CI_COMMIT_REF_NAME"),
+        pr=int(mr_iid) if mr_iid else None,
+        run_url=os.environ.get("CI_JOB_URL"),
+        actor=os.environ.get("GITLAB_USER_LOGIN"),
+        is_ci=True,
+    )
+
+
+def _circle_context() -> CIContext:
+    pr_num = os.environ.get("CIRCLE_PR_NUMBER")
+    return CIContext(
+        provider="circle",
+        repo=os.environ.get("CIRCLE_PROJECT_REPONAME"),
+        sha=os.environ.get("CIRCLE_SHA1"),
+        branch=os.environ.get("CIRCLE_BRANCH"),
+        pr=int(pr_num) if pr_num else None,
+        run_url=os.environ.get("CIRCLE_BUILD_URL"),
+        actor=os.environ.get("CIRCLE_USERNAME"),
+        is_ci=True,
+    )
+
+
+def _azure_context() -> CIContext:
+    pr_id = os.environ.get("SYSTEM_PULLREQUEST_PULLREQUESTID")
+    return CIContext(
+        provider="azure",
+        repo=os.environ.get("BUILD_REPOSITORY_NAME"),
+        sha=os.environ.get("BUILD_SOURCEVERSION"),
+        branch=os.environ.get("BUILD_SOURCEBRANCH"),
+        pr=int(pr_id) if pr_id else None,
+        run_url=os.environ.get("SYSTEM_TEAMFOUNDATIONCOLLECTIONURI"),
+        actor=os.environ.get("BUILD_REQUESTEDFOR"),
+        is_ci=True,
+    )
+
+
+def _jenkins_context() -> CIContext:
+    return CIContext(
+        provider="jenkins",
+        sha=os.environ.get("GIT_COMMIT"),
+        branch=os.environ.get("GIT_BRANCH"),
+        run_url=os.environ.get("BUILD_URL"),
+        actor=os.environ.get("BUILD_USER"),
+        is_ci=True,
+    )