evalgate-sdk 3.3.1__py3-none-any.whl

evalgate_sdk/errors.py

@@ -0,0 +1,236 @@
+"""Error classes for the EvalAI SDK, with rich error information and documentation."""
+
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Any
+
+_ERROR_DOCS: dict[str, dict[str, Any]] = {
+    "MISSING_API_KEY": {
+        "documentation": "https://docs.evalgate.com/errors/missing-api-key",
+        "solutions": [
+            "Set EVALGATE_API_KEY environment variable",
+            'Pass api_key in config: AIEvalClient(api_key="...")',
+        ],
+        "retryable": False,
+    },
+    "RATE_LIMIT_EXCEEDED": {
+        "documentation": "https://docs.evalgate.com/errors/rate-limit",
+        "solutions": [
+            "Wait before retrying (check retry_after property)",
+            "Upgrade your plan for higher rate limits",
+            "Implement exponential backoff",
+        ],
+        "retryable": True,
+    },
+    "TIMEOUT": {
+        "documentation": "https://docs.evalgate.com/errors/timeout",
+        "solutions": [
+            "Increase timeout: AIEvalClient(timeout=60000)",
+            "Check your network connection",
+        ],
+        "retryable": True,
+    },
+    "NETWORK_ERROR": {
+        "documentation": "https://docs.evalgate.com/errors/network",
+        "solutions": [
+            "Check your internet connection",
+            "Verify the base_url is correct",
+        ],
+        "retryable": True,
+    },
+    "UNAUTHORIZED": {
+        "documentation": "https://docs.evalgate.com/errors/unauthorized",
+        "solutions": [
+            "Verify your API key is correct",
+            "Check if your API key has expired",
+        ],
+        "retryable": False,
+    },
+    "FORBIDDEN": {
+        "documentation": "https://docs.evalgate.com/errors/forbidden",
+        "solutions": [
+            "Check if you have permission for this resource",
+            "Verify you're using the correct organization ID",
+        ],
+        "retryable": False,
+    },
+    "NOT_FOUND": {
+        "documentation": "https://docs.evalgate.com/errors/not-found",
+        "solutions": [
+            "Verify the resource ID is correct",
+            "Check if the resource was deleted",
+        ],
+        "retryable": False,
+    },
+    "VALIDATION_ERROR": {
+        "documentation": "https://docs.evalgate.com/errors/validation",
+        "solutions": [
+            "Check the error details for specific validation failures",
+            "Verify all required fields are provided",
+        ],
+        "retryable": False,
+    },
+    "INTERNAL_SERVER_ERROR": {
+        "documentation": "https://docs.evalgate.com/errors/server-error",
+        "solutions": [
+            "Retry the request after a brief delay",
+            "Contact support if the issue persists",
+        ],
+        "retryable": True,
+    },
+    "FEATURE_LIMIT_REACHED": {
+        "documentation": "https://docs.evalgate.com/errors/feature-limit",
+        "solutions": [
+            "Upgrade your plan for higher limits",
+            "Wait for your usage to reset",
+        ],
+        "retryable": False,
+    },
+}
+
+_STATUS_TO_CODE = {
+    401: "UNAUTHORIZED",
+    403: "FORBIDDEN",
+    404: "NOT_FOUND",
+    408: "TIMEOUT",
+    422: "VALIDATION_ERROR",
+    429: "RATE_LIMIT_EXCEEDED",
+}
+
+
+class EvalGateError(Exception):
+    """Base error for the EvalAI SDK with rich diagnostics."""
+
+    code: str
+    status_code: int
+    documentation: str
+    solutions: list[str]
+    retryable: bool
+    details: Any | None
+    retry_after: int | None
+    reset_at: datetime | None
+    request_id: str | None
+
+    def __init__(
+        self,
+        message: str,
+        code: str = "UNKNOWN_ERROR",
+        status_code: int = 0,
+        details: Any | None = None,
+    ) -> None:
+        super().__init__(message)
+        self.code = code
+        self.status_code = status_code
+        self.details = details
+
+        doc = _ERROR_DOCS.get(code, {})
+        self.documentation = doc.get("documentation", f"https://docs.evalgate.com/errors/{code}")
+        self.solutions = doc.get("solutions", ["Check the error details for more information"])
+        self.retryable = doc.get("retryable", False)
+        self.retry_after = None
+        self.reset_at = None
+        self.request_id = None
+
+        if isinstance(details, dict):
+            if code == "RATE_LIMIT_EXCEEDED" and "retryAfter" in details:
+                self.retry_after = int(details["retryAfter"])
+            if code == "FEATURE_LIMIT_REACHED" and "resetAt" in details:
+                self.reset_at = datetime.fromisoformat(details["resetAt"])
+            self.request_id = details.get("requestId")
+
+    @property
+    def message(self) -> str:
+        """Return the error message string, matching the TS ``error.message`` API."""
+        return str(self.args[0]) if self.args else ""
+
+    def should_retry(self) -> bool:
+        return self.retryable
+
+    def detailed_message(self) -> str:
+        lines = [f"{self.code}: {self}", "", f"Documentation: {self.documentation}", ""]
+        lines.append("Suggested solutions:")
+        for i, s in enumerate(self.solutions, 1):
+            lines.append(f"  {i}. {s}")
+        if self.retry_after is not None:
+            lines.append(f"\nRetry after: {self.retry_after} seconds")
+        if self.reset_at is not None:
+            lines.append(f"\nLimit resets at: {self.reset_at.isoformat()}")
+        return "\n".join(lines)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "code": self.code,
+            "message": str(self),
+            "status_code": self.status_code,
+            "documentation": self.documentation,
+            "solutions": self.solutions,
+            "retryable": self.retryable,
+            "retry_after": self.retry_after,
+            "request_id": self.request_id,
+            "details": self.details,
+        }
+
+
+class RateLimitError(EvalGateError):
+    def __init__(self, message: str = "Rate limit exceeded", retry_after: int | None = None):
+        super().__init__(message, "RATE_LIMIT_EXCEEDED", 429, {"retryAfter": retry_after} if retry_after else None)
+
+
+class AuthenticationError(EvalGateError):
+    def __init__(self, message: str = "Authentication failed"):
+        super().__init__(message, "UNAUTHORIZED", 401)
+
+
+class ValidationError(EvalGateError):
+    def __init__(self, message: str = "Validation failed", details: Any | None = None):
+        super().__init__(message, "VALIDATION_ERROR", 400, details)
+
+
+class NetworkError(EvalGateError):
+    def __init__(self, message: str = "Network request failed"):
+        super().__init__(message, "NETWORK_ERROR", 0)
+        self.retryable = True
+
+
+def create_error_from_response(status_code: int, data: Any) -> EvalGateError:
+    """Create an EvalGateError from an HTTP response status and body."""
+    if isinstance(data, dict):
+        error_obj = data.get("error", data)
+        if isinstance(error_obj, str):
+            error_obj = data
+        code = (
+            (error_obj.get("code") if isinstance(error_obj, dict) else None)
+            or data.get("code")
+            or _STATUS_TO_CODE.get(status_code)
+            or ("INTERNAL_SERVER_ERROR" if status_code >= 500 else "UNKNOWN_ERROR")
+        )
+        message = (
+            (data["error"] if isinstance(data.get("error"), str) else None)
+            or (error_obj.get("message") if isinstance(error_obj, dict) else None)
+            or data.get("message")
+            or "Unknown error"
+        )
+        request_id = (error_obj.get("requestId") if isinstance(error_obj, dict) else None) or data.get("requestId")
+    else:
+        code = _STATUS_TO_CODE.get(status_code, "INTERNAL_SERVER_ERROR" if status_code >= 500 else "UNKNOWN_ERROR")
+        message = str(data) if data else "Unknown error"
+        request_id = None
+
+    err: EvalGateError
+    if status_code == 429:
+        retry_after = None
+        if isinstance(data, dict):
+            retry_after = data.get("retryAfter") or (
+                error_obj.get("retryAfter") if isinstance(error_obj, dict) else None
+            )
+        err = RateLimitError(message, retry_after=int(retry_after) if retry_after else None)
+    elif status_code == 401 or code == "UNAUTHORIZED":
+        err = AuthenticationError(message)
+    elif status_code == 400 or code == "VALIDATION_ERROR":
+        err = ValidationError(message, details=data)
+    else:
+        err = EvalGateError(message, code, status_code, data)
+    if request_id:
+        err.request_id = request_id
+    return err

evalgate_sdk/export.py

@@ -0,0 +1,238 @@
+"""Data export and import — JSON, CSV, JSONL formats with LangSmith conversion."""
+
+from __future__ import annotations
+
+import csv
+import io
+import json
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Literal
+
+ExportFormat = Literal["json", "csv", "jsonl"]
+
+
+@dataclass
+class ExportOptions:
+    format: ExportFormat = "json"
+    include_traces: bool = True
+    include_evaluations: bool = True
+    include_test_cases: bool = True
+    include_runs: bool = True
+    start_date: str | None = None
+    end_date: str | None = None
+    organization_id: int | None = None
+
+
+@dataclass
+class ImportOptions:
+    skip_duplicates: bool = True
+    dry_run: bool = False
+    organization_id: int | None = None
+
+
+@dataclass
+class ExportData:
+    format: ExportFormat = "json"
+    traces: list[dict[str, Any]] = field(default_factory=list)
+    evaluations: list[dict[str, Any]] = field(default_factory=list)
+    test_cases: list[dict[str, Any]] = field(default_factory=list)
+    runs: list[dict[str, Any]] = field(default_factory=list)
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class ImportResult:
+    imported: int = 0
+    skipped: int = 0
+    errors: list[str] = field(default_factory=list)
+    dry_run: bool = False
+
+
+async def export_data(client: Any, options: ExportOptions | None = None) -> ExportData:
+    """Export traces, evaluations, test cases, and runs from the platform."""
+    opts = options or ExportOptions()
+    data = ExportData(format=opts.format)
+
+    if opts.include_traces:
+        from evalgate_sdk.types import ListTracesParams
+
+        params = ListTracesParams(limit=100)
+        if opts.organization_id:
+            params.organization_id = opts.organization_id
+        traces = await client.traces.list(params)
+        data.traces = [t.model_dump(mode="json", by_alias=True) for t in traces]
+
+    if opts.include_evaluations:
+        evals = await client.evaluations.list()
+        data.evaluations = [e.model_dump(mode="json", by_alias=True) for e in evals]
+
+        if opts.include_test_cases:
+            for ev in evals:
+                tcs = await client.evaluations.list_test_cases(ev.id)
+                data.test_cases.extend(tc.model_dump(mode="json", by_alias=True) for tc in tcs)
+
+        if opts.include_runs:
+            for ev in evals:
+                runs = await client.evaluations.list_runs(ev.id)
+                data.runs.extend(r.model_dump(mode="json", by_alias=True) for r in runs)
+
+    data.metadata = {"exported_at": _now_iso(), "total_items": len(data.traces) + len(data.evaluations)}
+    return data
+
+
+async def import_data(data: ExportData, options: ImportOptions | None = None, *, client: Any = None) -> ImportResult:
+    """Import data back into the platform.
+
+    The *client* parameter is keyword-only.  When omitted the function
+    attempts to use the global default client (``get_default_client()``).
+    This 2-arg signature matches the TS public export.
+    """
+    if client is None:
+        try:
+            from evalgate_sdk.client import get_default_client
+
+            client = get_default_client()
+        except Exception as err:
+            raise TypeError(
+                "import_data() requires a client. Either pass client=... or initialise a default client first."
+            ) from err
+    opts = options or ImportOptions()
+    result = ImportResult(dry_run=opts.dry_run)
+
+    if opts.dry_run:
+        result.imported = len(data.traces) + len(data.evaluations)
+        return result
+
+    from evalgate_sdk.types import CreateEvaluationParams, CreateTraceParams
+
+    for trace_data in data.traces:
+        try:
+            await client.traces.create(
+                CreateTraceParams(
+                    name=trace_data.get("name", "imported"),
+                    metadata=trace_data.get("metadata"),
+                )
+            )
+            result.imported += 1
+        except Exception as exc:
+            if opts.skip_duplicates and "duplicate" in str(exc).lower():
+                result.skipped += 1
+            else:
+                result.errors.append(str(exc))
+
+    for eval_data in data.evaluations:
+        try:
+            await client.evaluations.create(
+                CreateEvaluationParams(
+                    name=eval_data.get("name", "imported"),
+                    description=eval_data.get("description"),
+                )
+            )
+            result.imported += 1
+        except Exception as exc:
+            if opts.skip_duplicates and "duplicate" in str(exc).lower():
+                result.skipped += 1
+            else:
+                result.errors.append(str(exc))
+
+    return result
+
+
+def export_to_file(data: ExportData, file_path: str) -> None:
+    """Write export data to a file in the specified format."""
+    path = Path(file_path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+
+    if data.format == "json":
+        path.write_text(
+            json.dumps(
+                {
+                    "traces": data.traces,
+                    "evaluations": data.evaluations,
+                    "test_cases": data.test_cases,
+                    "runs": data.runs,
+                    "metadata": data.metadata,
+                },
+                indent=2,
+                default=str,
+            ),
+            encoding="utf-8",
+        )
+
+    elif data.format == "jsonl":
+        lines: list[str] = []
+        for t in data.traces:
+            lines.append(json.dumps({"type": "trace", **t}, default=str))
+        for e in data.evaluations:
+            lines.append(json.dumps({"type": "evaluation", **e}, default=str))
+        path.write_text("\n".join(lines), encoding="utf-8")
+
+    elif data.format == "csv":
+        path.write_text(convert_to_csv(data, "traces"), encoding="utf-8")
+
+
+def import_from_file(file_path: str) -> ExportData:
+    """Read export data from a file."""
+    path = Path(file_path)
+    text = path.read_text(encoding="utf-8")
+
+    if file_path.endswith(".jsonl"):
+        data = ExportData(format="jsonl")
+        for line in text.strip().splitlines():
+            obj = json.loads(line)
+            record_type = obj.pop("type", "trace")
+            if record_type == "trace":
+                data.traces.append(obj)
+            elif record_type == "evaluation":
+                data.evaluations.append(obj)
+        return data
+    else:
+        raw = json.loads(text)
+        return ExportData(
+            format="json",
+            traces=raw.get("traces", []),
+            evaluations=raw.get("evaluations", []),
+            test_cases=raw.get("test_cases", []),
+            runs=raw.get("runs", []),
+            metadata=raw.get("metadata", {}),
+        )
+
+
+def convert_to_csv(data: ExportData, resource_type: Literal["traces", "evaluations"] = "traces") -> str:
+    """Convert export data to CSV string."""
+    items = data.traces if resource_type == "traces" else data.evaluations
+    if not items:
+        return ""
+    buf = io.StringIO()
+    writer = csv.DictWriter(buf, fieldnames=sorted(items[0].keys()))
+    writer.writeheader()
+    for item in items:
+        writer.writerow({k: json.dumps(v) if isinstance(v, (dict, list)) else v for k, v in item.items()})
+    return buf.getvalue()
+
+
+def import_from_langsmith(langsmith_data: Any) -> ExportData:
+    """Convert LangSmith export format to EvalAI format."""
+    data = ExportData()
+    if isinstance(langsmith_data, list):
+        for item in langsmith_data:
+            data.traces.append(
+                {
+                    "name": item.get("name", item.get("run_type", "langsmith-import")),
+                    "input": json.dumps(item.get("inputs", {})),
+                    "output": json.dumps(item.get("outputs", {})),
+                    "metadata": {
+                        "langsmith_id": item.get("id"),
+                        "run_type": item.get("run_type"),
+                        "source": "langsmith",
+                    },
+                }
+            )
+    return data
+
+
+def _now_iso() -> str:
+    from datetime import datetime, timezone
+
+    return datetime.now(timezone.utc).isoformat()

evalgate_sdk/formatters/__init__.py

@@ -0,0 +1,11 @@
+"""Output formatters for evaluation results (T10).
+
+Provides human, JSON, GitHub, and PR comment output formats.
+"""
+
+from evalgate_sdk.formatters.github import format_github
+from evalgate_sdk.formatters.human import format_human
+from evalgate_sdk.formatters.json_fmt import format_json
+from evalgate_sdk.formatters.pr_comment import format_pr_comment
+
+__all__ = ["format_human", "format_json", "format_github", "format_pr_comment"]

evalgate_sdk/formatters/github.py

@@ -0,0 +1,51 @@
+"""GitHub Actions output formatter."""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+def format_github(report: dict[str, Any]) -> str:
+    """Format a check/run report as GitHub Actions workflow commands.
+
+    Uses ``::error``, ``::warning``, and ``::notice`` annotations.
+    """
+    lines: list[str] = []
+    verdict = report.get("verdict", "unknown")
+    eval_id = report.get("evaluationId", report.get("run_id", "?"))
+    score = report.get("score")
+    reason = report.get("reasonMessage", report.get("reason_message", ""))
+
+    # Summary annotation
+    if verdict == "pass":
+        lines.append(f"::notice title=EvalGate Pass::Evaluation {eval_id} passed (score={score})")
+    elif verdict == "warn":
+        lines.append(f"::warning title=EvalGate Warning::Evaluation {eval_id}: {reason}")
+    else:
+        lines.append(f"::error title=EvalGate Fail::Evaluation {eval_id} failed: {reason}")
+
+    # Set output variables via GITHUB_OUTPUT (::set-output is deprecated since Oct 2022)
+    import os
+
+    github_output = os.environ.get("GITHUB_OUTPUT")
+    if github_output:
+        try:
+            with open(github_output, "a") as f:
+                f.write(f"verdict={verdict}\n")
+                if score is not None:
+                    f.write(f"score={score}\n")
+        except OSError:
+            pass
+
+    # Failed cases as annotations
+    failed_cases = report.get("failedCases", report.get("failed_cases", []))
+    for fc in failed_cases[:10]:
+        name = fc.get("name", fc.get("test_name", "?"))
+        msg = fc.get("reason", fc.get("message", "failed"))
+        file_path = fc.get("file_path", fc.get("filePath", ""))
+        if file_path:
+            lines.append(f"::error file={file_path}::{name}: {msg}")
+        else:
+            lines.append(f"::error::{name}: {msg}")
+
+    return "\n".join(lines)

evalgate_sdk/formatters/human.py

@@ -0,0 +1,68 @@
+"""Human-readable output formatter."""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+def format_human(report: dict[str, Any]) -> str:
+    """Format a check/run report as a human-readable string."""
+    lines: list[str] = []
+    verdict = report.get("verdict", report.get("summary", {}).get("success", "unknown"))
+    eval_id = report.get("evaluationId", report.get("run_id", "?"))
+
+    # Header
+    if verdict == "pass" or verdict is True:
+        lines.append(f"✅ PASS — {eval_id}")
+    elif verdict == "warn":
+        lines.append(f"⚠️  WARN — {eval_id}")
+    else:
+        lines.append(f"❌ FAIL — {eval_id}")
+
+    lines.append("")
+
+    # Summary
+    summary = report.get("summary", {})
+    if summary:
+        total = summary.get("total", 0)
+        passed = summary.get("passed", 0)
+        failed = summary.get("failed", 0)
+        pass_rate = summary.get("pass_rate", summary.get("passRate", 0))
+        avg_score = summary.get("average_score", summary.get("averageScore", 0))
+        duration = summary.get("total_duration_ms", summary.get("totalDurationMs", 0))
+
+        lines.append(f"  Total:      {total}")
+        lines.append(f"  Passed:     {passed}")
+        lines.append(f"  Failed:     {failed}")
+        lines.append(f"  Pass rate:  {pass_rate:.1f}%")
+        lines.append(f"  Avg score:  {avg_score:.1f}")
+        lines.append(f"  Duration:   {duration:.0f}ms")
+
+    # Score
+    score = report.get("score")
+    if score is not None:
+        lines.append(f"\n  Score:      {score}")
+
+    baseline = report.get("baselineScore", report.get("baseline_score"))
+    if baseline is not None:
+        delta = report.get("delta", (score or 0) - baseline)
+        lines.append(f"  Baseline:   {baseline}")
+        lines.append(f"  Delta:      {delta:+.1f}")
+
+    # Reason
+    reason = report.get("reasonMessage", report.get("reason_message"))
+    if reason:
+        lines.append(f"\n  Reason: {reason}")
+
+    # Failed cases
+    failed_cases = report.get("failedCases", report.get("failed_cases", []))
+    if failed_cases:
+        lines.append(f"\n  Failed cases ({len(failed_cases)}):")
+        for fc in failed_cases[:5]:
+            name = fc.get("name", fc.get("test_name", "?"))
+            reason_text = fc.get("reason", fc.get("message", ""))
+            lines.append(f"    • {name}: {reason_text}")
+        if len(failed_cases) > 5:
+            lines.append(f"    ... and {len(failed_cases) - 5} more")
+
+    return "\n".join(lines)

evalgate_sdk/formatters/json_fmt.py

@@ -0,0 +1,11 @@
+"""JSON output formatter."""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+
+def format_json(report: dict[str, Any], *, indent: int = 2) -> str:
+    """Format a check/run report as a JSON string."""
+    return json.dumps(report, indent=indent, default=str)