truvyx 0.4.0__tar.gz

truvyx-0.4.0/PKG-INFO

@@ -0,0 +1,8 @@
+Metadata-Version: 2.4
+Name: truvyx
+Version: 0.4.0
+Summary: Truvyx evaluation SDK — embed evaluation into any agent pipeline
+License: MIT
+Project-URL: Homepage, https://truvyx.org
+Project-URL: Documentation, https://truvyx.org/docs/sdk
+Requires-Python: >=3.10

truvyx-0.4.0/pyproject.toml

@@ -0,0 +1,19 @@
+[build-system]
+requires = ["setuptools>=68", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "truvyx"
+version = "0.4.0"
+description = "Truvyx evaluation SDK — embed evaluation into any agent pipeline"
+license = { text = "MIT" }
+requires-python = ">=3.10"
+dependencies = []
+
+[project.urls]
+Homepage = "https://truvyx.org"
+Documentation = "https://truvyx.org/docs/sdk"
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["truvyx*"]

truvyx-0.4.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

truvyx-0.4.0/tests/test_evaluate.py

@@ -0,0 +1,57 @@
+from truvyx.client import evaluate
+
+
+def test_evaluate_maps_api_metadata_to_python_fields(monkeypatch):
+    responses = iter([
+        {
+            "scenario": {
+                "id": "scenario-1",
+                "name": "Safety",
+                "description": "",
+                "domain": "TEST",
+                "difficulty": "LOW",
+                "constraints": [],
+            }
+        },
+        {"runId": "run-1"},
+        {
+            "ready": True,
+            "runId": "run-1",
+            "scenarioId": "scenario-1",
+            "status": "PASSED",
+            "passed": True,
+            "costProfile": {
+                "totalInputTokens": 10,
+                "totalOutputTokens": 5,
+                "totalTokens": 15,
+                "estimatedCostUsd": 0.01,
+                "apiCallCount": 1,
+                "toolCallCount": 2,
+                "redundantCallCount": 1,
+                "avgCallLatencyMs": 25,
+            },
+            "retryState": {
+                "attempts": 2,
+                "coldStartRetry": True,
+                "workerAvailable": True,
+            },
+        },
+    ])
+
+    monkeypatch.setattr("truvyx.client._request", lambda *args, **kwargs: next(responses))
+    monkeypatch.setattr("truvyx.client.time.sleep", lambda _seconds: None)
+
+    result = evaluate(
+        lambda _scenario: {"answer": 42},
+        api_key="trk_test",
+        scenario_id="scenario-1",
+        on_premise=True,
+        on_premise_url="http://localhost:3000",
+    )
+
+    assert result.retry_state is not None
+    assert result.retry_state.cold_start_retry is True
+    assert result.retry_state.worker_available is True
+    assert result.cost_profile is not None
+    assert result.cost_profile.total_input_tokens == 10
+    assert result.cost_profile.avg_call_latency_ms == 25

truvyx-0.4.0/tests/test_suites.py

@@ -0,0 +1,70 @@
+from truvyx.client import create_scenario_suite, evaluate_suite, update_scenario_suite
+
+
+SUITE = {
+    "id": "suite-1",
+    "name": "Release",
+    "description": "release gates",
+    "version": 1,
+    "scenarios": [
+        {
+            "scenarioId": "scenario-1",
+            "scenarioVersion": 2,
+            "position": 0,
+            "critical": True,
+            "scenario": {
+                "id": "scenario-1",
+                "name": "Safety",
+                "description": "",
+                "domain": "TEST",
+                "difficulty": "HIGH",
+                "agentRoles": [],
+                "parameters": {},
+                "problemStatement": "",
+                "tags": [],
+                "constraintCount": 1,
+            },
+        }
+    ],
+}
+
+
+def test_python_suite_resource_and_evaluation_workflow(monkeypatch):
+    calls = []
+
+    def fake_request(url, *, headers, method="GET", body=None):
+        calls.append((url, headers, method, body))
+        if url.endswith("/api/v1/scenario-suites"):
+            return {"suite": SUITE}
+        if url.endswith("/api/v1/scenario-suites/suite-1") and method == "PATCH":
+            return {"suite": {**SUITE, "version": 2}}
+        if url.endswith("/api/v1/scenario-suites/suite-1"):
+            return {"suite": {**SUITE, "version": 2}}
+        if url.endswith("/api/v1/scenario-suites/suite-1/evaluate"):
+            assert headers["Idempotency-Key"] == "release-42"
+            assert body["outputs"][0]["scenarioId"] == "scenario-1"
+            return {"suiteRunId": "suite-run-1"}
+        if url.endswith("/api/v1/scenario-suite-runs/suite-run-1"):
+            return {
+                "ready": True,
+                "suiteRunId": "suite-run-1",
+                "suite": {"id": "suite-1", "name": "Release"},
+                "suiteVersion": 2,
+                "status": "PASSED",
+                "passed": True,
+                "overallScore": 1,
+                "threshold": 0.9,
+                "blockCritical": True,
+                "summary": {"total": 1, "completed": 1, "passed": 1, "failed": 0, "errors": 0, "criticalFailures": 0},
+                "runs": [],
+            }
+        raise AssertionError(f"unexpected URL {url}")
+
+    monkeypatch.setattr("truvyx.client._request", fake_request)
+    created = create_scenario_suite(api_key="trk_test", name="Release", scenarios=[{"scenarioId": "scenario-1", "critical": True}])
+    assert created.id == "suite-1"
+    updated = update_scenario_suite(api_key="trk_test", suite_id="suite-1", update={"description": "new"})
+    assert updated.version == 2
+    result = evaluate_suite(lambda scenario: {"scenario": scenario.id}, api_key="trk_test", suite_id="suite-1", fail_threshold=0.9, idempotency_key="release-42")
+    assert result.status == "PASSED"
+    assert sum(url.endswith("/evaluate") for url, *_ in calls) == 1

truvyx-0.4.0/tests/test_trace.py

@@ -0,0 +1,96 @@
+import asyncio
+import unittest
+
+from truvyx.trace import extract_trace, trace
+
+
+class AsyncTraceTests(unittest.IsolatedAsyncioTestCase):
+    async def test_coroutine_is_awaited_and_timing_includes_async_work(self):
+        @trace(agent_id="planner")
+        async def agent(value):
+            await asyncio.sleep(0.015)
+            return {"value": value}
+
+        self.assertEqual(await agent(3), {"value": 3})
+        steps = extract_trace(agent)
+        self.assertEqual(steps[0]["output"], {"value": 3})
+        self.assertGreaterEqual(steps[0]["durationMs"], 10)
+
+    async def test_exception_identity_and_trace_are_preserved(self):
+        failure = RuntimeError("agent failed")
+
+        @trace
+        async def agent(_value):
+            await asyncio.sleep(0)
+            raise failure
+
+        with self.assertRaises(RuntimeError) as caught:
+            await agent("input")
+        self.assertIs(caught.exception, failure)
+        self.assertEqual(extract_trace(agent)[0]["output"], None)
+
+    async def test_concurrent_calls_have_task_local_trace_state(self):
+        @trace
+        async def agent(value):
+            await asyncio.sleep(0.001 if value == "fast" else 0.01)
+            return value
+
+        async def invoke(value):
+            self.assertEqual(await agent(value), value)
+            return extract_trace(agent)
+
+        fast, slow = await asyncio.gather(invoke("fast"), invoke("slow"))
+        self.assertEqual([step["input"] for step in fast], ["fast"])
+        self.assertEqual([step["input"] for step in slow], ["slow"])
+        self.assertEqual(fast[0]["step"], 0)
+        self.assertEqual(slow[0]["step"], 0)
+
+    async def test_cancellation_is_not_swallowed(self):
+        started = asyncio.Event()
+
+        @trace
+        async def agent(_value):
+            started.set()
+            await asyncio.sleep(10)
+
+        captured = []
+
+        async def invoke():
+            try:
+                await agent("cancelled")
+            finally:
+                captured.extend(extract_trace(agent))
+
+        task = asyncio.create_task(invoke())
+        await started.wait()
+        task.cancel()
+        with self.assertRaises(asyncio.CancelledError):
+            await task
+        self.assertEqual(captured[0]["input"], "cancelled")
+
+    async def test_recursive_calls_share_ordered_state_within_one_task(self):
+        @trace
+        async def agent(depth):
+            if depth:
+                await agent(depth - 1)
+            return depth
+
+        await agent(2)
+        steps = extract_trace(agent)
+        self.assertEqual([step["step"] for step in steps], [0, 1, 2])
+        self.assertEqual([step["input"] for step in steps], [0, 1, 2])
+
+
+class SyncTraceTests(unittest.TestCase):
+    def test_sync_behavior_and_extraction_reset_remain_compatible(self):
+        @trace(emit_to="organization")
+        def agent(value):
+            return value * 2
+
+        self.assertEqual(agent(4), 8)
+        self.assertEqual(extract_trace(agent)[0]["emitTo"], "organization")
+        self.assertEqual(extract_trace(agent), [])
+
+
+if __name__ == "__main__":
+    unittest.main()

truvyx-0.4.0/truvyx/__init__.py

@@ -0,0 +1,37 @@
+from .client import (
+    evaluate,
+    evaluate_suite,
+    create_scenario_suite,
+    get_scenario_suite,
+    update_scenario_suite,
+    assert_feasibility,
+    EvaluationError,
+    ScenarioSuiteEvaluationError,
+    TruvyxError,
+)
+from .trace import trace, extract_trace
+from .types import EvalConfig, EvalResult, Violation, AuditTrail, ScenarioInput, ScenarioSuite, ScenarioSuiteMember, SuiteEvalResult, SuiteRunItem
+
+__version__ = "0.4.0"
+__all__ = [
+    "evaluate",
+    "evaluate_suite",
+    "create_scenario_suite",
+    "get_scenario_suite",
+    "update_scenario_suite",
+    "assert_feasibility",
+    "EvaluationError",
+    "ScenarioSuiteEvaluationError",
+    "TruvyxError",
+    "trace",
+    "extract_trace",
+    "EvalConfig",
+    "EvalResult",
+    "Violation",
+    "AuditTrail",
+    "ScenarioInput",
+    "ScenarioSuite",
+    "ScenarioSuiteMember",
+    "SuiteEvalResult",
+    "SuiteRunItem",
+]

truvyx-0.4.0/truvyx/client.py

@@ -0,0 +1,370 @@
+from __future__ import annotations
+import time
+import urllib.request
+import urllib.error
+import json
+from concurrent.futures import ThreadPoolExecutor
+from typing import Any, Callable
+
+from .types import (
+    EvalResult,
+    ScenarioInput,
+    Violation,
+    AuditTrail,
+    ConstraintResult,
+    CostProfile,
+    RetryState,
+    ScenarioSuite,
+    ScenarioSuiteMember,
+    SuiteEvalResult,
+    SuiteRunItem,
+)
+from .trace import extract_trace, trace
+
+SDK_VERSION = "0.4.0"
+
+
+class TruvyxError(Exception):
+    pass
+
+
+class EvaluationError(TruvyxError):
+    def __init__(self, message: str, result: EvalResult, code: str):
+        super().__init__(message)
+        self.result = result
+        self.code = code
+
+
+class ScenarioSuiteEvaluationError(TruvyxError):
+    def __init__(self, message: str, result: SuiteEvalResult | None, code: str):
+        super().__init__(message)
+        self.result = result
+        self.code = code
+
+
+def _request(url: str, *, headers: dict[str, str], method: str = "GET", body: Any = None) -> Any:
+    data = json.dumps(body).encode() if body is not None else None
+    req = urllib.request.Request(url, data=data, headers=headers, method=method)
+    try:
+        with urllib.request.urlopen(req, timeout=30) as resp:
+            return json.loads(resp.read())
+    except urllib.error.HTTPError as e:
+        raise TruvyxError(f"HTTP {e.code}: {e.read().decode()}") from e
+
+
+def _cost_profile(data: Any) -> CostProfile | None:
+    if not isinstance(data, dict):
+        return None
+    return CostProfile(
+        total_input_tokens=int(data.get("totalInputTokens", 0)),
+        total_output_tokens=int(data.get("totalOutputTokens", 0)),
+        total_tokens=int(data.get("totalTokens", 0)),
+        estimated_cost_usd=float(data.get("estimatedCostUsd", 0)),
+        api_call_count=int(data.get("apiCallCount", 0)),
+        tool_call_count=int(data.get("toolCallCount", 0)),
+        redundant_call_count=int(data.get("redundantCallCount", 0)),
+        avg_call_latency_ms=data.get("avgCallLatencyMs"),
+    )
+
+
+def _retry_state(data: Any) -> RetryState | None:
+    if not isinstance(data, dict):
+        return None
+    return RetryState(
+        attempts=int(data.get("attempts", 0)),
+        cold_start_retry=bool(data.get("coldStartRetry", False)),
+        worker_available=bool(data.get("workerAvailable", False)),
+    )
+
+
+def evaluate(
+    agent_fn: Callable[[ScenarioInput], Any],
+    *,
+    api_key: str,
+    scenario_id: str,
+    agent_system_name: str = "sdk-agent",
+    compliance_mode: bool = False,
+    on_premise: bool = False,
+    on_premise_url: str | None = None,
+    trace_capture: bool = False,
+    trace_visibility: str = "private",
+    timeout: int = 60,
+) -> EvalResult:
+    if trace_visibility not in {"private", "organization", "fault_atlas"}:
+        raise ValueError("trace_visibility must be 'private', 'organization', or 'fault_atlas'")
+    base_url = (on_premise_url or "") if on_premise else "https://truvyx.org"
+    headers = {
+        "Content-Type": "application/json",
+        "X-Truvyx-Key": api_key,
+        "User-Agent": f"truvyx-python/{SDK_VERSION}",
+    }
+
+    # 1. Fetch the scenario directly. This supports private organisation-owned
+    # scenarios as well as public Registry scenarios.
+    scen_data = _request(f"{base_url}/api/v1/scenarios/{scenario_id}", headers=headers)
+    s = scen_data.get("scenario")
+    if s is None:
+        raise TruvyxError(f"Scenario {scenario_id} not found")
+    scenario_input = ScenarioInput(
+        id=s["id"],
+        name=s["name"],
+        description=s.get("description", ""),
+        domain=s["domain"],
+        difficulty=s["difficulty"],
+        agent_roles=s.get("agentRoles", []),
+        parameters=s.get("parameters", {}),
+        problem_statement=s.get("problemStatement", ""),
+        tags=s.get("tags", []),
+        constraint_count=len(s.get("constraints", [])),
+    )
+
+    # 2. Run agent
+    start = time.time()
+    agent_output = agent_fn(scenario_input)
+    execution_time_ms = int((time.time() - start) * 1000)
+    execution_trace = extract_trace(agent_fn) if trace_capture else None
+
+    # 3. Submit
+    submit_resp = _request(
+        f"{base_url}/api/v1/evaluate",
+        headers=headers,
+        method="POST",
+        body={
+            "scenarioId": scenario_id,
+            "agentSystemName": agent_system_name,
+            "agentOutput": agent_output,
+            "executionTrace": execution_trace,
+            "traceVisibility": trace_visibility,
+            "executionTimeMs": execution_time_ms,
+            "environment": {
+                "sdk": SDK_VERSION,
+                "complianceMode": compliance_mode,
+                "traceCapture": trace_capture,
+            },
+        },
+    )
+    run_id = submit_resp["runId"]
+
+    # 4. Poll for result
+    deadline = time.time() + timeout
+    while time.time() < deadline:
+        time.sleep(2)
+        try:
+            data = _request(f"{base_url}/api/v1/runs/{run_id}/result", headers=headers)
+        except TruvyxError:
+            continue
+
+        if not data.get("ready"):
+            continue
+
+        violations = [
+            Violation(
+                constraint_id=v["constraintId"],
+                constraint_name=v["constraintName"],
+                category=v["category"],
+                severity=v["severity"],
+                description=v["description"],
+                agent_value=v["agentValue"],
+                required_bound=v["requiredBound"],
+            )
+            for v in data.get("violations", [])
+        ]
+
+        audit_trail = None
+        if compliance_mode and data.get("auditRecordId"):
+            audit_status = data.get("auditRecordStatus", "UNSIGNED")
+            audit_trail = AuditTrail(
+                id=data["auditRecordId"],
+                status=audit_status if audit_status in {"UNSIGNED", "SIGNED", "VERIFIED", "INVALID"} else "INVALID",
+                verified_at=data.get("auditRecordVerifiedAt"),
+            )
+
+        constraint_results = [
+            ConstraintResult(
+                constraint_id=item.get("constraintId", ""),
+                constraint_name=item.get("constraintName", ""),
+                status=item.get("status", "NOT_SCORED"),
+                score=item.get("score"),
+                evidence=item.get("evidence"),
+            )
+            for item in data.get("constraintResults", [])
+        ]
+        cost_profile = _cost_profile(data.get("costProfile"))
+        retry_state = _retry_state(data.get("retryState"))
+
+        return EvalResult(
+            run_id=data.get("runId", run_id),
+            scenario_id=data.get("scenarioId", scenario_id),
+            status=data.get("status", "ERROR"),
+            feasibility_score=data.get("feasibilityScore", 0.0),
+            completeness_score=data.get("completenessScore", 0.0),
+            optimality_score=data.get("optimalityScore", 0.0),
+            overall_score=data.get("overallScore", 0.0),
+            passed=data.get("passed", False),
+            violations=violations,
+            constraint_results=constraint_results,
+            completeness_gaps=data.get("completenessGaps", []),
+            audit_trail=audit_trail,
+            cost_profile=cost_profile,
+            retry_state=retry_state,
+        )
+
+    empty = EvalResult(
+        run_id=run_id, scenario_id=scenario_id, status="ERROR",
+        feasibility_score=0, completeness_score=0, optimality_score=0,
+        overall_score=0, passed=False,
+    )
+    raise EvaluationError(f"Evaluation timed out after {timeout}s. Run ID: {run_id}", empty, "TIMEOUT")
+
+
+def assert_feasibility(
+    result: EvalResult,
+    *,
+    threshold: float = 0.7,
+    fail_on_regulatory: bool = False,
+) -> None:
+    if fail_on_regulatory:
+        regulatory = [
+            v for v in result.violations
+            if v.category == "REGULATORY" and v.severity in ("CRITICAL", "HIGH")
+        ]
+        if regulatory:
+            names = ", ".join(v.constraint_name for v in regulatory)
+            raise EvaluationError(
+                f"Regulatory violation(s): {names}",
+                result,
+                "REGULATORY_VIOLATION",
+            )
+
+    if result.overall_score < threshold:
+        raise EvaluationError(
+            f"Score {result.overall_score:.3f} < threshold {threshold}. "
+            f"Feasibility={result.feasibility_score:.3f} "
+            f"Completeness={result.completeness_score:.3f} "
+            f"Optimality={result.optimality_score:.3f} "
+            f"Violations={len(result.violations)}",
+            result,
+            "THRESHOLD_NOT_MET",
+        )
+
+
+def _base_url(on_premise: bool, on_premise_url: str | None) -> str:
+    if on_premise:
+        if not on_premise_url:
+            raise ValueError("on_premise_url is required when on_premise=True")
+        return on_premise_url.rstrip("/")
+    if on_premise_url is not None:
+        raise ValueError("on_premise_url requires on_premise=True")
+    return "https://truvyx.org"
+
+
+def _suite_headers(api_key: str, idempotency_key: str | None = None) -> dict[str, str]:
+    if not api_key.strip():
+        raise ValueError("api_key is required")
+    headers = {"Content-Type": "application/json", "X-Truvyx-Key": api_key, "User-Agent": f"truvyx-python/{SDK_VERSION}"}
+    if idempotency_key is not None:
+        if not idempotency_key.strip() or len(idempotency_key) > 200:
+            raise ValueError("idempotency_key must contain 1-200 characters")
+        headers["Idempotency-Key"] = idempotency_key
+    return headers
+
+
+def _scenario_input(data: dict[str, Any]) -> ScenarioInput:
+    return ScenarioInput(
+        id=data["id"], name=data["name"], description=data.get("description", ""),
+        domain=data["domain"], difficulty=data["difficulty"], agent_roles=data.get("agentRoles", []),
+        parameters=data.get("parameters", {}), problem_statement=data.get("problemStatement", ""),
+        tags=data.get("tags", []), constraint_count=data.get("constraintCount", data.get("_count", {}).get("constraints", len(data.get("constraints", [])))),
+    )
+
+
+def _parse_suite(data: dict[str, Any]) -> ScenarioSuite:
+    return ScenarioSuite(
+        id=data["id"], name=data["name"], description=data.get("description", ""), version=data["version"],
+        scenarios=[ScenarioSuiteMember(
+            scenario_id=item["scenarioId"], scenario_version=item["scenarioVersion"], position=item["position"],
+            critical=item.get("critical", False), scenario=_scenario_input(item["scenario"]),
+        ) for item in data.get("scenarios", [])],
+    )
+
+
+def create_scenario_suite(
+    *, api_key: str, name: str, scenarios: list[dict[str, Any]], description: str = "",
+    on_premise: bool = False, on_premise_url: str | None = None,
+) -> ScenarioSuite:
+    data = _request(f"{_base_url(on_premise, on_premise_url)}/api/v1/scenario-suites", headers=_suite_headers(api_key), method="POST", body={"name": name, "description": description, "scenarios": scenarios})
+    return _parse_suite(data["suite"])
+
+
+def get_scenario_suite(
+    *, api_key: str, suite_id: str, on_premise: bool = False, on_premise_url: str | None = None,
+) -> ScenarioSuite:
+    if not suite_id.strip():
+        raise ValueError("suite_id is required")
+    data = _request(f"{_base_url(on_premise, on_premise_url)}/api/v1/scenario-suites/{suite_id}", headers=_suite_headers(api_key))
+    return _parse_suite(data["suite"])
+
+
+def update_scenario_suite(
+    *, api_key: str, suite_id: str, update: dict[str, Any], on_premise: bool = False, on_premise_url: str | None = None,
+) -> ScenarioSuite:
+    if not update:
+        raise ValueError("at least one suite field is required")
+    data = _request(f"{_base_url(on_premise, on_premise_url)}/api/v1/scenario-suites/{suite_id}", headers=_suite_headers(api_key), method="PATCH", body=update)
+    return _parse_suite(data["suite"])
+
+
+def _parse_suite_result(data: dict[str, Any]) -> SuiteEvalResult:
+    suite = data.get("suite", {})
+    return SuiteEvalResult(
+        ready=data.get("ready", False), suite_run_id=data["suiteRunId"], suite_id=suite.get("id", ""), suite_name=suite.get("name", ""),
+        suite_version=data.get("suiteVersion", 0), status=data.get("status", "ERROR"), passed=data.get("passed", False),
+        overall_score=data.get("overallScore"), threshold=data.get("threshold", 0.7), block_critical=data.get("blockCritical", True),
+        summary=data.get("summary", {}), runs=[SuiteRunItem(
+            run_id=item["runId"], scenario_id=item["scenarioId"], scenario_name=item.get("scenarioName", ""),
+            scenario_version=item.get("scenarioVersion", 0), critical=item.get("critical", False), status=item.get("status", "ERROR"),
+            overall_score=item.get("overallScore"), violation_count=item.get("violationCount", 0), completed_at=item.get("completedAt"), error=item.get("error"),
+        ) for item in data.get("runs", [])], created_at=data.get("createdAt"), completed_at=data.get("completedAt"),
+    )
+
+
+def evaluate_suite(
+    agent_fn: Callable[[ScenarioInput], Any], *, api_key: str, suite_id: str, agent_system_name: str = "sdk-agent",
+    fail_threshold: float = 0.7, block_critical: bool = True, max_concurrency: int = 5, timeout: int = 60,
+    on_premise: bool = False, on_premise_url: str | None = None, trace_capture: bool = False,
+    trace_visibility: str = "private", environment: dict[str, Any] | None = None, idempotency_key: str | None = None,
+) -> SuiteEvalResult:
+    if not 0 <= fail_threshold <= 1:
+        raise ValueError("fail_threshold must be between 0 and 1")
+    if not 1 <= max_concurrency <= 10:
+        raise ValueError("max_concurrency must be between 1 and 10")
+    if timeout < 1:
+        raise ValueError("timeout must be at least 1 second")
+    if trace_visibility not in {"private", "organization", "fault_atlas"}:
+        raise ValueError("trace_visibility is invalid")
+    base_url = _base_url(on_premise, on_premise_url)
+    suite = get_scenario_suite(api_key=api_key, suite_id=suite_id, on_premise=on_premise, on_premise_url=on_premise_url)
+
+    def execute(member: ScenarioSuiteMember) -> dict[str, Any]:
+        target = trace(emit_to=trace_visibility, agent_id=agent_system_name)(agent_fn) if trace_capture else agent_fn
+        started = time.time()
+        output = target(member.scenario)
+        return {"scenarioId": member.scenario_id, "agentOutput": output, "executionTimeMs": int((time.time() - started) * 1000), "executionTrace": extract_trace(target) if trace_capture else None, "traceVisibility": trace_visibility}
+
+    with ThreadPoolExecutor(max_workers=max_concurrency) as executor:
+        outputs = list(executor.map(execute, suite.scenarios))
+    submission = _request(
+        f"{base_url}/api/v1/scenario-suites/{suite_id}/evaluate", headers=_suite_headers(api_key, idempotency_key), method="POST",
+        body={"agentSystemName": agent_system_name, "outputs": outputs, "failThreshold": fail_threshold, "blockCritical": block_critical, "maxConcurrency": max_concurrency, "environment": {**(environment or {}), "sdk": SDK_VERSION}},
+    )
+    deadline = time.time() + timeout
+    latest: SuiteEvalResult | None = None
+    while time.time() < deadline:
+        data = _request(f"{base_url}/api/v1/scenario-suite-runs/{submission['suiteRunId']}", headers=_suite_headers(api_key))
+        latest = _parse_suite_result(data)
+        if latest.ready:
+            if not latest.passed:
+                raise ScenarioSuiteEvaluationError(f"Scenario suite {suite_id} finished with status {latest.status}", latest, "SUITE_FAILED")
+            return latest
+        time.sleep(2)
+    raise ScenarioSuiteEvaluationError(f"Scenario suite evaluation timed out after {timeout}s. Suite run ID: {submission['suiteRunId']}", latest, "TIMEOUT")

truvyx-0.4.0/truvyx/trace.py

@@ -0,0 +1,98 @@
+from __future__ import annotations
+import functools
+import inspect
+import time
+from contextvars import ContextVar
+from datetime import datetime, timezone
+from typing import Any, Callable, Optional
+
+_trace_contexts: dict[int, ContextVar[list[dict[str, Any]] | None]] = {}
+
+
+def trace(
+    fn: Optional[Callable] = None,
+    *,
+    emit_to: str = "private",
+    agent_id: Optional[str] = None,
+) -> Callable:
+    """Decorator that captures execution trace without changing function signature.
+
+    Usage:
+        @trace
+        def my_agent(input_data): ...
+
+        @trace(emit_to="private", agent_id="planner")
+        def my_agent(input_data): ...
+    """
+    if emit_to not in {"private", "organization", "fault_atlas"}:
+        raise ValueError("emit_to must be 'private', 'organization', or 'fault_atlas'")
+
+    def decorator(func: Callable) -> Callable:
+        fn_id = id(func)
+        effective_agent_id = agent_id or func.__name__
+
+        trace_context: ContextVar[list[dict[str, Any]] | None] = ContextVar(
+            f"truvyx_trace_{fn_id}", default=None
+        )
+        _trace_contexts[fn_id] = trace_context
+
+        def begin() -> tuple[list[dict[str, Any]], float]:
+            steps = trace_context.get()
+            if steps is None:
+                steps = []
+                trace_context.set(steps)
+            return steps, time.perf_counter()
+
+        def finish(steps: list[dict[str, Any]], start: float, args: tuple, kwargs: dict, output: Any) -> None:
+            steps.append({
+                "agentId": effective_agent_id,
+                "step": len(steps),
+                "action": "execute",
+                "input": args[0] if args else kwargs,
+                "output": output,
+                "durationMs": max(0, int((time.perf_counter() - start) * 1000)),
+                "timestamp": datetime.now(timezone.utc).isoformat(),
+                "emitTo": emit_to,
+            })
+
+        if inspect.iscoroutinefunction(func):
+            @functools.wraps(func)
+            async def async_wrapper(*args, **kwargs):
+                steps, start = begin()
+                result: Any = None
+                try:
+                    result = await func(*args, **kwargs)
+                    return result
+                finally:
+                    finish(steps, start, args, kwargs, result)
+
+            setattr(async_wrapper, "__trace_fn_id__", fn_id)
+            return async_wrapper
+
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            steps, start = begin()
+            result: Any = None
+            try:
+                result = func(*args, **kwargs)
+                return result
+            finally:
+                finish(steps, start, args, kwargs, result)
+
+        setattr(wrapper, "__wrapped__", func)
+        setattr(wrapper, "__trace_fn_id__", fn_id)
+        return wrapper
+
+    if fn is not None:
+        return decorator(fn)
+    return decorator
+
+
+def extract_trace(fn: Callable) -> list[dict[str, Any]]:
+    fn_id = getattr(fn, "__trace_fn_id__", None) or id(fn)
+    context = _trace_contexts.get(fn_id)
+    if context is None:
+        return []
+    steps = context.get() or []
+    context.set(None)
+    return list(steps)

truvyx-0.4.0/truvyx/types.py

@@ -0,0 +1,145 @@
+from __future__ import annotations
+from dataclasses import dataclass, field
+from typing import Any, Optional
+
+
+@dataclass
+class EvalConfig:
+    api_key: str
+    scenario_id: Optional[str] = None
+    compliance_mode: bool = False
+    on_premise: bool = False
+    on_premise_url: Optional[str] = None
+    trace_capture: bool = False
+    trace_visibility: str = "private"
+    agent_system_name: str = "sdk-agent"
+    timeout: int = 60
+
+
+@dataclass
+class Violation:
+    constraint_id: str
+    constraint_name: str
+    category: str
+    severity: str
+    description: str
+    agent_value: Any
+    required_bound: Any
+
+
+@dataclass
+class ConstraintResult:
+    constraint_id: str
+    constraint_name: str
+    status: str
+    score: Optional[float] = None
+    evidence: Any = None
+
+
+@dataclass
+class CostProfile:
+    total_input_tokens: int
+    total_output_tokens: int
+    total_tokens: int
+    estimated_cost_usd: float
+    api_call_count: int
+    tool_call_count: int
+    redundant_call_count: int = 0
+    avg_call_latency_ms: Optional[int] = None
+
+
+@dataclass
+class RetryState:
+    attempts: int
+    cold_start_retry: bool
+    worker_available: bool
+
+
+@dataclass
+class AuditTrail:
+    id: str
+    status: str
+    verified_at: Optional[str] = None
+
+
+@dataclass
+class EvalResult:
+    run_id: str
+    scenario_id: str
+    status: str
+    feasibility_score: float
+    completeness_score: float
+    optimality_score: float
+    overall_score: float
+    passed: bool
+    violations: list[Violation] = field(default_factory=list)
+    constraint_results: list[ConstraintResult] = field(default_factory=list)
+    completeness_gaps: list[str] = field(default_factory=list)
+    cost_profile: Optional[CostProfile] = None
+    retry_state: Optional[RetryState] = None
+    audit_trail: Optional[AuditTrail] = None
+    rca_report_id: Optional[str] = None
+
+
+@dataclass
+class ScenarioInput:
+    id: str
+    name: str
+    description: str
+    domain: str
+    difficulty: str
+    agent_roles: list[Any]
+    parameters: dict[str, Any]
+    problem_statement: str
+    tags: list[str]
+    constraint_count: int
+
+
+@dataclass
+class ScenarioSuiteMember:
+    scenario_id: str
+    scenario_version: int
+    position: int
+    critical: bool
+    scenario: ScenarioInput
+
+
+@dataclass
+class ScenarioSuite:
+    id: str
+    name: str
+    description: str
+    version: int
+    scenarios: list[ScenarioSuiteMember] = field(default_factory=list)
+
+
+@dataclass
+class SuiteRunItem:
+    run_id: str
+    scenario_id: str
+    scenario_name: str
+    scenario_version: int
+    critical: bool
+    status: str
+    overall_score: Optional[float]
+    violation_count: int
+    completed_at: Optional[str]
+    error: Optional[str]
+
+
+@dataclass
+class SuiteEvalResult:
+    ready: bool
+    suite_run_id: str
+    suite_id: str
+    suite_name: str
+    suite_version: int
+    status: str
+    passed: bool
+    overall_score: Optional[float]
+    threshold: float
+    block_critical: bool
+    summary: dict[str, int]
+    runs: list[SuiteRunItem] = field(default_factory=list)
+    created_at: Optional[str] = None
+    completed_at: Optional[str] = None

truvyx-0.4.0/truvyx.egg-info/PKG-INFO

@@ -0,0 +1,8 @@
+Metadata-Version: 2.4
+Name: truvyx
+Version: 0.4.0
+Summary: Truvyx evaluation SDK — embed evaluation into any agent pipeline
+License: MIT
+Project-URL: Homepage, https://truvyx.org
+Project-URL: Documentation, https://truvyx.org/docs/sdk
+Requires-Python: >=3.10

truvyx-0.4.0/truvyx.egg-info/SOURCES.txt

@@ -0,0 +1,12 @@
+pyproject.toml
+tests/test_evaluate.py
+tests/test_suites.py
+tests/test_trace.py
+truvyx/__init__.py
+truvyx/client.py
+truvyx/trace.py
+truvyx/types.py
+truvyx.egg-info/PKG-INFO
+truvyx.egg-info/SOURCES.txt
+truvyx.egg-info/dependency_links.txt
+truvyx.egg-info/top_level.txt

truvyx-0.4.0/truvyx.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

truvyx-0.4.0/truvyx.egg-info/top_level.txt

@@ -0,0 +1 @@
+truvyx