evalport-sdk 1.0.0__tar.gz

evalport_sdk-1.0.0/PKG-INFO

@@ -0,0 +1,6 @@
+Metadata-Version: 2.4
+Name: evalport-sdk
+Version: 1.0.0
+Summary: Python SDK for EvalPort — The Open Evaluation Standard
+License: Apache-2.0
+Requires-Python: >=3.8

evalport_sdk-1.0.0/README.md

@@ -0,0 +1,87 @@
+# openeval
+
+Python SDK for EvalPort — The Open Evaluation Standard.
+
+## Install
+
+```bash
+pip install openeval
+```
+
+## Usage
+
+### Validate a suite
+
+```python
+from openeval.validate import validate_suite
+
+result = validate_suite({
+    "version": "1.0.0",
+    "id": "my_suite",
+    "graders": [{"id": "gr1", "type": "exact_match"}],
+    "test_cases": [{"id": "tc1", "input": "Hello", "expected_output": "Hi", "graders": ["gr1"]}]
+})
+print(result.valid)  # True
+```
+
+### Convert from Promptfoo
+
+```python
+from openeval.convert import from_promptfoo
+
+suite = from_promptfoo(promptfoo_config)
+```
+
+### Convert from DeepEval
+
+```python
+from openeval.converters_deepeval import from_deepeval
+
+suite = from_deepeval(deepeval_export)
+```
+
+### Convert from Inspect AI
+
+```python
+from openeval.converters_inspect import from_inspect
+
+suite = from_inspect(inspect_data)
+```
+
+### Convert from OpenAI Evals
+
+```python
+from openeval.converters_openai import from_openai_evals
+
+suite = from_openai_evals(evals_data)
+```
+
+### Compute summary
+
+```python
+from openeval.convert import compute_summary, create_result_set
+
+summary = compute_summary(results)
+result_set = create_result_set(suite, results, "run_001")
+```
+
+## API
+
+### Validation
+- `validate_suite(doc)` → `ValidationResult`
+- `validate_test_case(doc)` → `ValidationResult`
+- `validate_grader(doc)` → `ValidationResult`
+- `validate_result_set(doc)` → `ValidationResult`
+- `validate_document(doc, type)` → `ValidationResult`
+
+### Conversion
+- `from_promptfoo(config)` → `dict`
+- `from_deepeval(data)` → `dict` (from `converters_deepeval`)
+- `from_inspect(data)` → `dict` (from `converters_inspect`)
+- `from_openai_evals(data)` → `dict` (from `converters_openai`)
+- `compute_summary(results)` → `dict`
+- `create_result_set(suite, results, run_id)` → `dict`
+
+## License
+
+Apache 2.0

evalport_sdk-1.0.0/evalport_sdk.egg-info/PKG-INFO

@@ -0,0 +1,6 @@
+Metadata-Version: 2.4
+Name: evalport-sdk
+Version: 1.0.0
+Summary: Python SDK for EvalPort — The Open Evaluation Standard
+License: Apache-2.0
+Requires-Python: >=3.8

evalport_sdk-1.0.0/evalport_sdk.egg-info/SOURCES.txt

@@ -0,0 +1,14 @@
+README.md
+pyproject.toml
+evalport_sdk.egg-info/PKG-INFO
+evalport_sdk.egg-info/SOURCES.txt
+evalport_sdk.egg-info/dependency_links.txt
+evalport_sdk.egg-info/top_level.txt
+openeval/__init__.py
+openeval/convert.py
+openeval/converters_deepeval.py
+openeval/converters_inspect.py
+openeval/converters_openai.py
+openeval/types.py
+openeval/validate.py
+tests/test_validate.py

evalport_sdk-1.0.0/evalport_sdk.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

evalport_sdk-1.0.0/evalport_sdk.egg-info/top_level.txt

@@ -0,0 +1,3 @@
+dist
+openeval
+tests

evalport_sdk-1.0.0/openeval/__init__.py

@@ -0,0 +1,3 @@
+from .types import *
+from .validate import *
+from .convert import *

evalport_sdk-1.0.0/openeval/convert.py

@@ -0,0 +1,39 @@
+from .types import OPENEVAL_VERSION
+
+def from_promptfoo(pf):
+    tests = pf.get("tests", [])
+    providers = pf.get("providers", [])
+    graders = []
+    tcs = []
+    for i, t in enumerate(tests):
+        v = t.get("vars", {})
+        asserts = t.get("assert", [])
+        tg = []
+        for j, a in enumerate(asserts):
+            gid = f"gr_{i}_{j}"
+            at = a.get("type", "")
+            if at == "equals": graders.append({"id": gid, "type": "exact_match"})
+            elif at == "contains": graders.append({"id": gid, "type": "contains", "params": {"substring": str(a.get("value", ""))}})
+            else: graders.append({"id": gid, "type": "custom", "params": {"handler": f"promptfoo:{at}"}})
+            tg.append(gid)
+        inp = v.get("query") or v.get("prompt") or str(v)
+        tc = {"id": f"tc_{i}", "input": inp, "graders": tg if tg else ["gr_default"]}
+        if "expected" in v: tc["expected_output"] = v["expected"]
+        tcs.append(tc)
+    if not graders: graders = [{"id": "gr_default", "type": "exact_match"}]
+    cfg = {}
+    if providers and isinstance(providers[0], dict):
+        p = providers[0]
+        cfg = {"provider": {k: v for k, v in [("model", p.get("model"))] if v is not None}}
+    return {"version": OPENEVAL_VERSION, "id": "suite_promptfoo_import", "name": "Imported from Promptfoo", "graders": graders, "test_cases": tcs, "config": cfg}
+
+def compute_summary(results):
+    total = len(results)
+    passed = sum(1 for r in results if r.get("passed"))
+    scores = [gr["score"] for r in results for gr in r.get("grader_results", []) if gr.get("score") is not None]
+    return {"total": total, "passed": passed, "failed": total - passed, "pass_rate": passed / total if total else 0, "avg_score": sum(scores) / len(scores) if scores else 0}
+
+def create_result_set(suite, results, run_id, runner_name="evalport-sdk", runner_version="1.0.0"):
+    from datetime import datetime, timezone
+    now = datetime.now(timezone.utc).isoformat()
+    return {"version": OPENEVAL_VERSION, "suite_id": suite["id"], "suite_version": suite.get("version"), "run_id": run_id, "started_at": now, "completed_at": now, "provider": suite.get("config", {}).get("provider"), "runner": {"name": runner_name, "version": runner_version}, "results": results, "summary": compute_summary(results)}

evalport_sdk-1.0.0/openeval/converters_deepeval.py

@@ -0,0 +1,117 @@
+"""DeepEval to EvalPort converter."""
+from __future__ import annotations
+from typing import Dict, List
+from .types import OPENEVAL_VERSION
+
+
+def from_deepeval(de: Dict) -> Dict:
+    tests = de.get("test_cases", [])
+    graders: List[Dict] = []
+    tcs: List[Dict] = []
+
+    for i, tc in enumerate(tests):
+        metrics = tc.get("metrics", [])
+        tc_graders: List[str] = []
+
+        for j, metric in enumerate(metrics):
+            gid = f"gr_{i}_{j}"
+            graders.append(_deepeval_metric_to_grader(gid, metric))
+            tc_graders.append(gid)
+
+        inp = tc.get("input", "")
+        if isinstance(inp, list):
+            inp = inp
+
+        new_tc = {
+            "id": tc.get("id", f"tc_{i}"),
+            "input": inp,
+            "graders": tc_graders if tc_graders else ["gr_default"],
+        }
+        if "expected_output" in tc:
+            new_tc["expected_output"] = tc["expected_output"]
+        if "context" in tc and isinstance(tc["context"], list):
+            new_tc["context"] = tc["context"]
+        if "retrieval_context" in tc and isinstance(tc["retrieval_context"], list):
+            new_tc["retrieval_context"] = tc["retrieval_context"]
+        if "metadata" in tc:
+            new_tc["metadata"] = tc["metadata"]
+        if "expected_tools" in tc:
+            new_tc["expected_tools"] = tc["expected_tools"]
+
+        tcs.append(new_tc)
+
+    if not graders:
+        graders = [{"id": "gr_default", "type": "exact_match"}]
+
+    return {
+        "version": OPENEVAL_VERSION,
+        "id": "suite_deepeval_import",
+        "name": "Imported from DeepEval",
+        "graders": graders,
+        "test_cases": tcs,
+        "metadata": {"openeval": {"source": "deepeval"}},
+    }
+
+
+def _deepeval_metric_to_grader(gid: str, metric: str) -> Dict:
+    m = metric.lower() if isinstance(metric, str) else str(metric).lower()
+    if "faithfulness" in m:
+        return {
+            "id": gid,
+            "type": "llm_judge",
+            "description": "Faithfulness (DeepEval)",
+            "params": {
+                "model": "gpt-4o",
+                "prompt": "Given context: {context}\nOutput: {output}\nIs the output faithful to the context? Return JSON: {\"score\": 0.0-1.0, \"reason\": \"...\"}",
+                "schema": {
+                    "type": "object",
+                    "properties": {"score": {"type": "number"}, "reason": {"type": "string"}},
+                    "required": ["score", "reason"],
+                },
+            },
+        }
+    if "answerrelevancy" in m or "answer_relevancy" in m:
+        return {"id": gid, "type": "semantic_similarity", "params": {"threshold": 0.5}}
+    if "exactmatch" in m or "exact_match" in m:
+        return {"id": gid, "type": "exact_match"}
+    if "contextualprecision" in m or "contextual_precision" in m:
+        return {
+            "id": gid,
+            "type": "llm_judge",
+            "description": "Contextual Precision (DeepEval)",
+            "params": {
+                "model": "gpt-4o",
+                "prompt": "Given retrieval context: {context}\nExpected: {expected}\nIs the retrieved context relevant? Return JSON: {\"score\": 0.0-1.0}",
+            },
+        }
+    if "contextualrecall" in m or "contextual_recall" in m:
+        return {
+            "id": gid,
+            "type": "llm_judge",
+            "description": "Contextual Recall (DeepEval)",
+            "params": {
+                "model": "gpt-4o",
+                "prompt": "Given context: {context}\nExpected: {expected}\nIs the expected information in the context? Return JSON: {\"score\": 0.0-1.0}",
+            },
+        }
+    if "hallucination" in m:
+        return {
+            "id": gid,
+            "type": "llm_judge",
+            "description": "Hallucination Detection (DeepEval)",
+            "params": {
+                "model": "gpt-4o",
+                "prompt": "Given context: {context}\nOutput: {output}\nDoes the output contain hallucinated information? Return JSON: {\"score\": 0.0 (no hallucination) - 1.0 (full hallucination)}",
+            },
+        }
+    if "toxicity" in m:
+        return {
+            "id": gid,
+            "type": "llm_judge",
+            "description": "Toxicity Detection (DeepEval)",
+            "params": {
+                "model": "gpt-4o",
+                "prompt": "Evaluate the toxicity of: {output}. Return JSON: {\"score\": 0.0 (safe) - 1.0 (toxic)}",
+            },
+        }
+    return {"id": gid, "type": "custom", "params": {"handler": f"deepeval:{metric}"}}

evalport_sdk-1.0.0/openeval/converters_inspect.py

@@ -0,0 +1,73 @@
+"""Inspect AI to EvalPort converter."""
+from __future__ import annotations
+from typing import Dict, List
+from .types import OPENEVAL_VERSION
+
+
+def from_inspect(data: Dict) -> Dict:
+    task_name = data.get("task", "inspect_import")
+    samples = data.get("samples", [])
+    scorers = data.get("scorers", ["exact"])
+
+    graders: List[Dict] = []
+    for i, scorer in enumerate(scorers):
+        gid = f"gr_{i}"
+        graders.append(_inspect_scorer_to_grader(gid, scorer))
+
+    if not graders:
+        graders = [{"id": "gr_0", "type": "exact_match"}]
+
+    grader_ids = [g["id"] for g in graders]
+
+    tcs: List[Dict] = []
+    for sample in samples:
+        tc = {
+            "id": sample.get("id", f"tc_{len(tcs)}"),
+            "input": sample.get("input", ""),
+            "graders": grader_ids,
+        }
+        if "target" in sample:
+            tc["expected_output"] = str(sample["target"])
+        if "context" in sample:
+            tc["context"] = sample["context"] if isinstance(sample["context"], list) else [sample["context"]]
+        if "metadata" in sample:
+            tc["metadata"] = sample["metadata"]
+        tcs.append(tc)
+
+    config = {}
+    if "model" in data:
+        config = {"provider": {"model": data["model"]}}
+
+    return {
+        "version": OPENEVAL_VERSION,
+        "id": f"suite_inspect_{task_name}",
+        "name": f"Imported from Inspect AI: {task_name}",
+        "graders": graders,
+        "test_cases": tcs,
+        "config": config,
+        "metadata": {"openeval": {"source": "inspect_ai"}},
+    }
+
+
+def _inspect_scorer_to_grader(gid: str, scorer: str) -> Dict:
+    s = scorer.lower() if isinstance(scorer, str) else str(scorer).lower()
+    if "exact" in s:
+        return {"id": gid, "type": "exact_match"}
+    if "pattern" in s or "regex" in s:
+        return {"id": gid, "type": "regex", "params": {"pattern": ".*"}}
+    if "includes" in s or "contains" in s:
+        return {"id": gid, "type": "contains", "params": {"substring": ""}}
+    if "json" in s:
+        return {"id": gid, "type": "json_schema", "params": {"schema": {"type": "object"}}}
+    if "model_graded" in s or "modelgraded" in s or "llm" in s:
+        return {
+            "id": gid,
+            "type": "llm_judge",
+            "params": {
+                "model": "gpt-4o",
+                "prompt": "Evaluate if {output} is correct for {input}. Expected: {expected}. Return JSON: {\"score\": 0.0-1.0}",
+            },
+        }
+    if "manual" in s or "human" in s:
+        return {"id": gid, "type": "human", "params": {"instructions": "Review the output manually."}}
+    return {"id": gid, "type": "custom", "params": {"handler": f"inspect:{scorer}"}}

evalport_sdk-1.0.0/openeval/converters_openai.py

@@ -0,0 +1,78 @@
+"""OpenAI Evals to EvalPort converter."""
+from __future__ import annotations
+from typing import Dict, List
+from .types import OPENEVAL_VERSION
+
+
+def from_openai_evals(data: Dict) -> Dict:
+    eval_id = data.get("id", "openai_eval_import")
+    test_data = data.get("test_data", [])
+    eval_config = data.get("config", {})
+    sampling = eval_config.get("sampling", {})
+    grader_spec = eval_config.get("grader", {})
+
+    graders: List[Dict] = []
+    if grader_spec:
+        graders.append(_openai_grader_to_openeval("gr_0", grader_spec))
+    if not graders:
+        graders = [{"id": "gr_0", "type": "exact_match"}]
+
+    grader_ids = [g["id"] for g in graders]
+
+    tcs: List[Dict] = []
+    for i, test in enumerate(test_data):
+        tc = {
+            "id": test.get("id", f"tc_{i}"),
+            "input": test.get("input", test.get("prompt", "")),
+            "graders": grader_ids,
+        }
+        if "target" in test:
+            tc["expected_output"] = str(test["target"])
+        elif "ideal" in test:
+            tc["expected_output"] = str(test["ideal"])
+        if "context" in test:
+            tc["context"] = test["context"] if isinstance(test["context"], list) else [test["context"]]
+        if "metadata" in test:
+            tc["metadata"] = test["metadata"]
+        tcs.append(tc)
+
+    config = {}
+    if "model" in sampling:
+        config = {"provider": {"model": sampling["model"]}}
+    if "temperature" in sampling:
+        config.setdefault("provider", {})["temperature"] = sampling["temperature"]
+
+    return {
+        "version": OPENEVAL_VERSION,
+        "id": f"suite_{eval_id}",
+        "name": f"Imported from OpenAI Evals: {eval_id}",
+        "graders": graders,
+        "test_cases": tcs,
+        "config": config,
+        "metadata": {"openeval": {"source": "openai_evals"}},
+    }
+
+
+def _openai_grader_to_openeval(gid: str, spec: Dict) -> Dict:
+    gtype = spec.get("type", spec.get("name", ""))
+    gtype_lower = gtype.lower() if isinstance(gtype, str) else str(gtype).lower()
+
+    if "exact" in gtype_lower or "match" in gtype_lower:
+        return {"id": gid, "type": "exact_match"}
+    if "includes" in gtype_lower or "contains" in gtype_lower:
+        return {"id": gid, "type": "contains", "params": {"substring": spec.get("substring", "")}}
+    if "regex" in gtype_lower or "pattern" in gtype_lower:
+        return {"id": gid, "type": "regex", "params": {"pattern": spec.get("pattern", ".*")}}
+    if "json" in gtype_lower:
+        return {"id": gid, "type": "json_schema", "params": {"schema": spec.get("schema", {"type": "object"})}}
+    if "model_graded" in gtype_lower or "modelgraded" in gtype_lower or "llm" in gtype_lower:
+        prompt = spec.get("prompt", spec.get("instructions", "Evaluate if {output} is correct. Return JSON: {\"score\": 0.0-1.0}"))
+        return {
+            "id": gid,
+            "type": "model graded",
+            "params": {
+                "model": spec.get("model", "gpt-4o"),
+                "prompt": prompt,
+            },
+        }
+    return {"id": gid, "type": "custom", "params": {"handler": f"openai_evals:{gtype}"}}

evalport_sdk-1.0.0/openeval/types.py

@@ -0,0 +1,80 @@
+from __future__ import annotations
+from typing import Any, Literal, Union, Optional, List, Dict
+from dataclasses import dataclass, field
+
+OPENEVAL_VERSION = "1.0.0"
+
+GraderType = Literal["exact_match","contains","regex","semantic_similarity","llm_judge","json_schema","json_path","code","human","model graded","custom"]
+
+@dataclass
+class Grader:
+    id: str
+    type: GraderType
+    params: Dict[str, Any] = field(default_factory=dict)
+    weight: float = 1.0
+    description: Optional[str] = None
+
+@dataclass
+class TestCase:
+    id: str
+    input: Union[str, List[str]]
+    graders: List[Union[str, Grader]]
+    expected_output: Optional[str] = None
+    context: Optional[List[str]] = None
+    retrieval_context: Optional[List[str]] = None
+    tools_called: Optional[List[str]] = None
+    expected_tools: Optional[List[str]] = None
+    metadata: Dict[str, Any] = field(default_factory=dict)
+    tags: List[str] = field(default_factory=list)
+    timeout_ms: Optional[int] = None
+    weight: float = 1.0
+
+@dataclass
+class EvalSuite:
+    version: str
+    id: str
+    test_cases: List[TestCase]
+    name: Optional[str] = None
+    description: Optional[str] = None
+    graders: List[Grader] = field(default_factory=list)
+    config: Dict[str, Any] = field(default_factory=dict)
+    metadata: Dict[str, Any] = field(default_factory=dict)
+    tags: List[str] = field(default_factory=list)
+
+@dataclass
+class GraderResult:
+    grader_id: str
+    type: str
+    score: Optional[float]
+    passed: bool
+    reason: Optional[str] = None
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+@dataclass
+class Result:
+    test_case_id: str
+    passed: bool
+    grader_results: List[GraderResult]
+    actual_output: Optional[str] = None
+    duration_ms: Optional[int] = None
+    error: Optional[Dict[str, Any]] = None
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+@dataclass
+class ResultSet:
+    version: str
+    suite_id: str
+    run_id: str
+    started_at: str
+    results: List[Result]
+    suite_version: Optional[str] = None
+    completed_at: Optional[str] = None
+    provider: Optional[Dict[str, Any]] = None
+    runner: Optional[Dict[str, str]] = None
+    summary: Optional[Dict[str, Any]] = None
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+@dataclass
+class ValidationResult:
+    valid: bool
+    errors: List[Dict[str, str]] = field(default_factory=list)

evalport_sdk-1.0.0/openeval/validate.py

@@ -0,0 +1,134 @@
+from __future__ import annotations
+from typing import Any, List, Dict
+from .types import ValidationResult
+import re
+
+STANDARD_GRADER_TYPES = {"exact_match","contains","regex","semantic_similarity","llm_judge","json_schema","json_path","code","human","model graded","custom"}
+
+def _err(p,m,c): return {"path":p,"message":m,"code":c}
+
+def validate_test_case(tc):
+    errors=[]
+    if not isinstance(tc,dict): return ValidationResult(False,[_err("$","Must be object","TYPE_ERROR")])
+    if not isinstance(tc.get("id"),str) or not tc["id"]: errors.append(_err("$.id","id required","REQUIRED"))
+    inp=tc.get("input")
+    if not isinstance(inp,str) and not (isinstance(inp,list) and all(isinstance(x,str) for x in inp)): errors.append(_err("$.input","input required","REQUIRED"))
+    elif isinstance(inp,list) and not inp: errors.append(_err("$.input","empty","MIN_ITEMS"))
+    gr=tc.get("graders")
+    if not isinstance(gr,list) or not gr: errors.append(_err("$.graders","graders required","REQUIRED"))
+    else:
+        for i,g in enumerate(gr):
+            if isinstance(g,str):
+                if not g: errors.append(_err(f"$.graders[{i}]","empty","EMPTY_STRING"))
+            elif isinstance(g,dict):
+                gv=validate_grader(g)
+                if not gv.valid:
+                    for e in gv.errors: errors.append(_err(f"$.graders[{i}].{e['path']}",e["message"],e["code"]))
+            else: errors.append(_err(f"$.graders[{i}]","must be string or object","TYPE_ERROR"))
+    return ValidationResult(not errors,errors)
+
+def validate_grader(g):
+    errors=[]
+    if not isinstance(g,dict): return ValidationResult(False,[_err("$","Must be object","TYPE_ERROR")])
+    if not isinstance(g.get("id"),str) or not g["id"]: errors.append(_err("$.id","id required","REQUIRED"))
+    gt=g.get("type")
+    if not isinstance(gt,str): errors.append(_err("$.type","type required","REQUIRED"))
+    elif gt not in STANDARD_GRADER_TYPES: errors.append(_err("$.type",f"Unknown: {gt}","UNKNOWN_TYPE"))
+    else:
+        p=g.get("params") or {}
+        for e in _vp(gt,p): errors.append(_err(f"$.params.{e['path']}",e["message"],e["code"]))
+    return ValidationResult(not errors,errors)
+
+def _vp(t,p):
+    e=[]
+    if t=="contains":
+        if not isinstance(p.get("substring"),str) or not p["substring"]: e.append(_err("substring","required","REQUIRED"))
+    elif t=="regex":
+        if not isinstance(p.get("pattern"),str) or not p["pattern"]: e.append(_err("pattern","required","REQUIRED"))
+    elif t=="semantic_similarity":
+        th=p.get("threshold")
+        if not isinstance(th,(int,float)) or th<0 or th>1: e.append(_err("threshold","0-1","OUT_OF_RANGE"))
+    elif t=="llm_judge":
+        if not isinstance(p.get("model"),str) or not p["model"]: e.append(_err("model","required","REQUIRED"))
+        pr=p.get("prompt")
+        if not isinstance(pr,str) or not pr: e.append(_err("prompt","required","REQUIRED"))
+        elif "{output}" not in pr and "{input}" not in pr and "{expected}" not in pr: e.append(_err("prompt","missing token","MISSING_TOKEN"))
+    elif t=="json_schema":
+        if not isinstance(p.get("schema"),dict): e.append(_err("schema","required","REQUIRED"))
+    elif t=="json_path":
+        if not isinstance(p.get("path"),str) or not p["path"]: e.append(_err("path","required","REQUIRED"))
+        if "expected" not in p: e.append(_err("expected","required","REQUIRED"))
+    elif t=="code":
+        if p.get("language") not in ("python","javascript"): e.append(_err("language","python|javascript","INVALID_VALUE"))
+        if not isinstance(p.get("source"),str) or not p["source"]: e.append(_err("source","required","REQUIRED"))
+    elif t=="custom":
+        if not isinstance(p.get("handler"),str) or not p["handler"]: e.append(_err("handler","required","REQUIRED"))
+    return e
+
+def validate_suite(s):
+    errors=[]
+    if not isinstance(s,dict): return ValidationResult(False,[_err("$","Must be object","TYPE_ERROR")])
+    if not isinstance(s.get("version"),str) or not re.match(r"^\d+\.\d+\.\d+(-draft)?$",s.get("version","")): errors.append(_err("$.version","semver","INVALID_VERSION"))
+    if not isinstance(s.get("id"),str) or not s["id"]: errors.append(_err("$.id","required","REQUIRED"))
+    tcs=s.get("test_cases")
+    if not isinstance(tcs,list) and not isinstance(s.get("test_cases_file"),str): errors.append(_err("$.test_cases","required","REQUIRED"))
+    if isinstance(tcs,list):
+        if not tcs: errors.append(_err("$.test_cases","empty","MIN_ITEMS"))
+        ids=set()
+        for i,tc in enumerate(tcs):
+            tv=validate_test_case(tc)
+            if not tv.valid:
+                for e in tv.errors: errors.append(_err(f"$.test_cases[{i}].{e['path']}",e["message"],e["code"]))
+            tid=tc.get("id") if isinstance(tc,dict) else None
+            if isinstance(tid,str):
+                if tid in ids: errors.append(_err(f"$.test_cases[{i}].id",f"dup:{tid}","DUPLICATE_ID"))
+                ids.add(tid)
+        grs=s.get("graders",[])
+        if isinstance(grs,list):
+            gids=set()
+            for i,g in enumerate(grs):
+                gv=validate_grader(g)
+                if not gv.valid:
+                    for e in gv.errors: errors.append(_err(f"$.graders[{i}].{e['path']}",e["message"],e["code"]))
+                gid=g.get("id") if isinstance(g,dict) else None
+                if isinstance(gid,str):
+                    if gid in gids: errors.append(_err(f"$.graders[{i}].id",f"dup:{gid}","DUPLICATE_ID"))
+                    gids.add(gid)
+            for i,tc in enumerate(tcs):
+                if isinstance(tc,dict) and isinstance(tc.get("graders"),list):
+                    for j,gr in enumerate(tc["graders"]):
+                        if isinstance(gr,str) and gr not in gids: errors.append(_err(f"$.test_cases[{i}].graders[{j}]",f"not found:{gr}","DANGLING_REFERENCE"))
+    return ValidationResult(not errors,errors)
+
+def validate_result_set(r):
+    errors=[]
+    if not isinstance(r,dict): return ValidationResult(False,[_err("$","Must be object","TYPE_ERROR")])
+    if not isinstance(r.get("version"),str) or not re.match(r"^\d+\.\d+\.\d+(-draft)?$",r.get("version","")): errors.append(_err("$.version","semver","INVALID_VERSION"))
+    if not isinstance(r.get("suite_id"),str) or not r["suite_id"]: errors.append(_err("$.suite_id","required","REQUIRED"))
+    if not isinstance(r.get("run_id"),str) or not r["run_id"]: errors.append(_err("$.run_id","required","REQUIRED"))
+    if not isinstance(r.get("started_at"),str): errors.append(_err("$.started_at","required","REQUIRED"))
+    rs=r.get("results")
+    if not isinstance(rs,list) or not rs: errors.append(_err("$.results","required","REQUIRED"))
+    else:
+        for i,x in enumerate(rs):
+            if not isinstance(x,dict): errors.append(_err(f"$.results[{i}]","object","TYPE_ERROR"));continue
+            if not isinstance(x.get("test_case_id"),str): errors.append(_err(f"$.results[{i}].test_case_id","required","REQUIRED"))
+            if not isinstance(x.get("passed"),bool): errors.append(_err(f"$.results[{i}].passed","required","REQUIRED"))
+            grs=x.get("grader_results")
+            if not isinstance(grs,list): errors.append(_err(f"$.results[{i}].grader_results","required","REQUIRED"))
+            else:
+                for j,gr in enumerate(grs):
+                    if not isinstance(gr,dict): errors.append(_err(f"$.results[{i}].grader_results[{j}]","object","TYPE_ERROR"));continue
+                    if not isinstance(gr.get("grader_id"),str): errors.append(_err(f"$.results[{i}].grader_results[{j}].grader_id","required","REQUIRED"))
+                    if not isinstance(gr.get("type"),str): errors.append(_err(f"$.results[{i}].grader_results[{j}].type","required","REQUIRED"))
+                    sc=gr.get("score")
+                    if not isinstance(sc,(int,float,type(None))): errors.append(_err(f"$.results[{i}].grader_results[{j}].score","number|null","TYPE_ERROR"))
+                    if not isinstance(gr.get("passed"),bool): errors.append(_err(f"$.results[{i}].grader_results[{j}].passed","required","REQUIRED"))
+    return ValidationResult(not errors,errors)
+
+def validate_document(d,t):
+    if t=="testcase": return validate_test_case(d)
+    if t=="grader": return validate_grader(d)
+    if t=="suite": return validate_suite(d)
+    if t=="resultset": return validate_result_set(d)
+    raise ValueError(f"Unknown type: {t}")

evalport_sdk-1.0.0/pyproject.toml

@@ -0,0 +1,13 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "evalport-sdk"
+version = "1.0.0"
+description = "Python SDK for EvalPort — The Open Evaluation Standard"
+license = {text = "Apache-2.0"}
+requires-python = ">=3.8"
+
+[tool.setuptools.packages.find]
+where = ["."]

evalport_sdk-1.0.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

evalport_sdk-1.0.0/tests/test_validate.py

@@ -0,0 +1,18 @@
+from openeval.validate import validate_suite, validate_grader, validate_test_case, validate_result_set
+
+def test_valid_suite():
+    assert validate_suite({"version":"1.0.0","id":"s","graders":[{"id":"g1","type":"exact_match"}],"test_cases":[{"id":"tc1","input":"hi","graders":["g1"]}]}).valid
+
+def test_empty():
+    assert not validate_suite({"version":"1.0.0","id":"s","test_cases":[]}).valid
+
+def test_grader():
+    assert validate_grader({"id":"g1","type":"exact_match"}).valid
+    assert not validate_grader({"id":"g1","type":"bad"}).valid
+
+def test_tc():
+    assert validate_test_case({"id":"tc1","input":"hi","graders":["g1"]}).valid
+    assert not validate_test_case({"id":"tc1","graders":["g1"]}).valid
+
+def test_rs():
+    assert validate_result_set({"version":"1.0.0","suite_id":"s","run_id":"r","started_at":"2026-01-01T00:00:00Z","results":[{"test_case_id":"tc1","passed":True,"grader_results":[{"grader_id":"g1","type":"exact_match","score":1.0,"passed":True}]}]}).valid