applexus-iris-evals 1.0.0__py3-none-any.whl

applexus_iris_evals-1.0.0.dist-info/METADATA

@@ -0,0 +1,185 @@
+Metadata-Version: 2.4
+Name: applexus-iris-evals
+Version: 1.0.0
+Summary: Reusable evaluation framework for IRIS SAP Joule A2A agents
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+Requires-Dist: httpx>=0.27.0
+Requires-Dist: python-dotenv>=1.0.1
+Requires-Dist: rich>=13.0.0
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: anthropic>=0.30.0
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.23.0; extra == "dev"
+
+# iris-evals
+
+Self-evaluation library for Applexus IRIS SAP Joule agents.
+
+Install once — every agent evaluates itself on startup, runs a background loop every 5 minutes, and exposes a `/health/evals` CI gate endpoint. All golden datasets, rubrics, and thresholds are bundled inside the library. Zero eval data files needed per agent.
+
+---
+
+## Installation
+
+```bash
+pip install git+https://github.com/applexus/iris-evals.git@v1.0.0
+```
+
+Add to your agent's `requirements.txt`:
+
+```
+iris-evals @ git+https://github.com/applexus/iris-evals.git@v1.0.0
+```
+
+---
+
+## Quick Start
+
+Add three blocks to your agent's `app.py`:
+
+```python
+import asyncio, os
+from iris_evals import run_self_eval, self_eval_loop
+
+AGENT_URL = os.getenv("AGENT_URL", "http://localhost:8080")
+
+@app.on_event("startup")
+async def startup_self_eval():
+    report = await run_self_eval(agent_url=AGENT_URL)
+    if not report.overall_passed:
+        logger.warning("[IRIS EVALS] Self-eval FAILED — %s", report.summary)
+
+@app.on_event("startup")
+async def start_eval_loop():
+    asyncio.create_task(
+        self_eval_loop(agent_url=AGENT_URL, interval_seconds=300)
+    )
+
+@app.route("/health/evals", methods=["GET"])
+async def eval_health(request):
+    from starlette.responses import JSONResponse
+    report = await run_self_eval(agent_url=AGENT_URL)
+    return JSONResponse({"passed": report.overall_passed, "summary": report.summary})
+```
+
+Add to your agent's `agent-card.json`:
+
+```json
+"x_evals": {
+  "evaluators": ["deterministic", "safety"],
+  "dataset": "builtin",
+  "threshold_profile": "deterministic_strict"
+}
+```
+
+---
+
+## 7 Eval Types
+
+| # | Type | Needs AI | Use when |
+|---|------|----------|----------|
+| 1 | `deterministic` | No | Rule-based agents — exact verdict match |
+| 2 | `probabilistic` | No | Classifiers — accuracy / F1 ≥ 85% |
+| 3 | `llm_as_judge` | Yes (AI Core) | Free-text output quality — rubric scored |
+| 4 | `agent_as_judge` | Yes (A2A) | Complex SAP domain — peer agent reviews |
+| 5 | `trace` | No | Multi-step agents — correct tool call sequence |
+| 6 | `safety` | No | Every agent — PII + injection checks |
+| 7 | `performance` | No | SLA-critical — latency p95 ≤ 5000ms |
+
+**Quick picks:**
+- Rule-based agent → `1, 6`
+- Triage / classifier → `2, 3, 6`
+- Multi-step → `1, 5, 6`
+- SLA-critical → `1, 5, 6, 7`
+
+---
+
+## How it works
+
+```
+cf push
+  └─ pip installs iris-evals from GitHub
+       └─ startup: run_self_eval() calls agent via A2A with golden records
+            └─ CI polls GET /health/evals → passed: true → promote
+                 └─ background: self_eval_loop() re-runs every 5 minutes
+```
+
+iris-evals runs **inside your agent's process** as an asyncio task — no separate BTP service, no extra cost.
+
+---
+
+## LLM Judge (type 3)
+
+Uses SAP AI Core orchestration. Requires these env vars in your agent's `manifest.yml` (gitignored):
+
+```
+AICORE_CLIENT_ID
+AICORE_CLIENT_SECRET
+AICORE_AUTH_URL
+AICORE_BASE_URL
+AICORE_RESOURCE_GROUP
+ORCHESTRATION_URL
+AICORE_JUDGE_MODEL     (default: gpt-4o)
+```
+
+If your agent already has AI Core credentials, iris-evals picks them up automatically — no extra config.
+
+---
+
+## Skill Integration
+
+The `iris-sap-agent-skill` asks **Q_EVAL** as its last interview question:
+
+```
+Which evaluation types does this agent need?
+→ deterministic / probabilistic / llm_as_judge / agent_as_judge / trace / safety / performance
+```
+
+Answering generates all wiring automatically — `x_evals` block, `app.py` startup/loop/health, and `requirements.txt` entry.
+
+---
+
+## Running Tests
+
+```bash
+pip install -e ".[dev]"
+pytest
+```
+
+---
+
+## Project Structure
+
+```
+src/iris_evals/
+├── aicore.py          SAP AI Core orchestration client
+├── registry.py        Built-in golden datasets, rubrics, thresholds
+├── schemas.py         EvalType, GoldenRecord, EvalConfig, EvalReport
+├── self_eval.py       run_self_eval() and self_eval_loop()
+├── runner.py          EvalSuite base class
+├── _transport.py      A2A JSON-RPC call helper
+└── evaluators/
+    ├── deterministic.py
+    ├── probabilistic.py
+    ├── llm_judge.py
+    ├── safety.py
+    ├── trace.py
+    └── performance.py
+```
+
+---
+
+## Versioning
+
+| Version | Notes |
+|---------|-------|
+| v1.0.0 | Initial release — 7 eval types, built-in registry, AI Core LLM judge |
+
+Tag a new version and update `requirements.txt` in each agent to upgrade:
+
+```bash
+git tag v1.0.1
+git push --tags
+```

applexus_iris_evals-1.0.0.dist-info/RECORD

@@ -0,0 +1,19 @@
+iris_evals/__init__.py,sha256=yAhT45gnSJdlLz-m4qrFLd_u5BMJzf5LlirWVoI2d14,2402
+iris_evals/_transport.py,sha256=vhSDUylNLEa-UlRxyfZlUdHhPW7Pzj6OZ0Nj8FtA4Tw,2391
+iris_evals/aicore.py,sha256=GcSYkrTcVhEqKVNgn_xWnP1PHZY1IIhOZWxcluUUxPI,3568
+iris_evals/registry.py,sha256=ELxjug3HyRoTzDvbT0x6p3BvhCL8TqCzPI66JoRauk8,12032
+iris_evals/results.py,sha256=52jKjKpvb3FB-gX0pDp7WoUkSO1fE26kC074yeXJ6Jg,2706
+iris_evals/runner.py,sha256=9tYwSdo2G5SjHlpEFRzmWAtZY0SnFKdzYlBw5_XAV_M,3917
+iris_evals/schemas.py,sha256=Xj8yQWwmRG5QlqpKDPkDEA5oHInGs79yfWBHUy_3tyU,5340
+iris_evals/self_eval.py,sha256=R9q1Dmvu-UGexXoCZpI5BlFGN5yPoylTl9qSUEp-v-g,10870
+iris_evals/evaluators/__init__.py,sha256=rF3sJzuHKdm9hYhsPynllsgNYs9b2pYVJ6bRWb-TLHE,251
+iris_evals/evaluators/deterministic.py,sha256=Yj_ZmCq5OPlTjODHRQXgSkBghhUdC3NFCh02C-L9ZnA,2281
+iris_evals/evaluators/llm_judge.py,sha256=_d4rFJH1HU8ycGC8z_QQbwtrmHtTaQB7mujWLWz3rwI,4512
+iris_evals/evaluators/performance.py,sha256=CU-vopv8AneuSPrs6LBjNpHe47kHlxakhv2aVIYTdgg,3043
+iris_evals/evaluators/probabilistic.py,sha256=p1MVuKXWguS0LQpR-vlUaC6tx5R8_OMZqzzeC-J0bgM,4135
+iris_evals/evaluators/safety.py,sha256=wYdcAf6GXBR1pcTcDViSzrfaO9J6iCbBjRnmLFaJjFU,3776
+iris_evals/evaluators/trace.py,sha256=OQ-pFRNSdZ2j4DvvAcc477JFJdhE_5vAZTj5uJN1sbw,3383
+applexus_iris_evals-1.0.0.dist-info/METADATA,sha256=Ex2pLM4e1plPPr1uGTgIZtq6w--KxJvmpIf9kEz7ZHY,5158
+applexus_iris_evals-1.0.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+applexus_iris_evals-1.0.0.dist-info/top_level.txt,sha256=fly3XfJNLI8pg1in9fHDx3JCol1Zj7_O72ObWcqtEl4,11
+applexus_iris_evals-1.0.0.dist-info/RECORD,,

applexus_iris_evals-1.0.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.2.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

applexus_iris_evals-1.0.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+iris_evals

iris_evals/__init__.py

@@ -0,0 +1,64 @@
+"""
+iris-evals — Reusable evaluation framework for IRIS SAP Joule A2A agents.
+
+── Startup self-eval (simplest usage) ──────────────────────────────────────────
+    from iris_evals import run_self_eval
+
+    @app.on_event("startup")
+    async def startup():
+        report = await run_self_eval(agent_url="http://localhost:8080")
+        if not report.overall_passed:
+            logger.warning(f"[IRIS EVALS] {report.summary}")
+
+── Background loop ──────────────────────────────────────────────────────────────
+    from iris_evals import self_eval_loop
+
+    @app.on_event("startup")
+    async def startup():
+        asyncio.create_task(self_eval_loop(agent_url="http://localhost:8080", interval_seconds=300))
+
+── Health endpoint ───────────────────────────────────────────────────────────────
+    @app.get("/health/evals")
+    async def eval_health():
+        report = await run_self_eval(agent_url="http://localhost:8080")
+        return {"passed": report.overall_passed, "summary": report.summary}
+
+── Manual / CLI ─────────────────────────────────────────────────────────────────
+    iris-evals run --agent http://localhost:8080
+    iris-evals run-pipeline --pipeline pipeline.json
+"""
+
+from iris_evals.runner    import EvalSuite
+from iris_evals.self_eval import run_self_eval, self_eval_loop
+from iris_evals.schemas   import (
+    EvalConfig,
+    EvalType,
+    EvalReport,
+    EvalResult,
+    GoldenRecord,
+    JudgeRubric,
+    JudgeCriterion,
+)
+from iris_evals.results   import print_report, save_json
+from iris_evals.registry  import register_agent
+
+__all__ = [
+    # self-eval (primary API)
+    "run_self_eval",
+    "self_eval_loop",
+    # manual suite
+    "EvalSuite",
+    # schemas
+    "EvalConfig",
+    "EvalType",
+    "EvalReport",
+    "EvalResult",
+    "GoldenRecord",
+    "JudgeRubric",
+    "JudgeCriterion",
+    # output
+    "print_report",
+    "save_json",
+    # registry extension
+    "register_agent",
+]

iris_evals/_transport.py

@@ -0,0 +1,80 @@
+"""
+_transport.py — Agent call transport layer.
+
+All evaluators call the agent through this single function.
+Supports:
+  - A2A JSON-RPC (message/send) — default for IRIS agents
+  - Plain HTTP POST (for non-A2A agents or direct endpoint testing)
+
+The agent URL and transport mode come from EvalConfig.
+"""
+
+from __future__ import annotations
+
+import json
+import uuid
+from typing import Any
+
+import httpx
+
+from iris_evals.schemas import EvalConfig
+
+_TIMEOUT = 60.0
+
+
+async def call_agent(config: EvalConfig, input_data: str | dict) -> dict:
+    """Send input to the agent and return the parsed response dict."""
+    payload = input_data if isinstance(input_data, str) else json.dumps(input_data)
+
+    if config.use_a2a:
+        return await _call_a2a(config.agent_url, payload)
+    else:
+        return await _call_http(config.agent_url, payload)
+
+
+async def _call_a2a(base_url: str, text: str) -> dict:
+    """
+    SAP Joule A2A JSON-RPC 2.0 transport.
+    Mirrors the pattern used by a2a_client.py in every IRIS agent.
+    """
+    request_id = str(uuid.uuid4())
+    body = {
+        "jsonrpc": "2.0",
+        "id": request_id,
+        "method": "message/send",
+        "params": {
+            "message": {
+                "messageId":  str(uuid.uuid4()),
+                "contextId":  str(uuid.uuid4()),
+                "role":       "user",
+                "parts":      [{"kind": "text", "text": text}],
+            }
+        },
+    }
+
+    async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
+        resp = await client.post(base_url, json=body)
+        resp.raise_for_status()
+        rpc = resp.json()
+
+    result    = rpc.get("result") or {}
+    artifacts = result.get("artifacts", [])
+    if not artifacts:
+        return {"raw_rpc": rpc}
+
+    parts = artifacts[0].get("parts", [{}])
+    text_out = parts[0].get("text", "") if parts else ""
+    try:
+        parsed = json.loads(text_out)
+        parsed["result"] = result          # keep A2A envelope for trace evaluator
+        return parsed
+    except json.JSONDecodeError:
+        return {"result": result, "text": text_out}
+
+
+async def _call_http(url: str, text: str) -> dict[str, Any]:
+    """Plain HTTP POST — for non-A2A endpoints."""
+    async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
+        resp = await client.post(url, json={"input": text})
+        resp.raise_for_status()
+        return resp.json()

iris_evals/aicore.py

@@ -0,0 +1,118 @@
+"""
+aicore.py — SAP AI Core orchestration client for iris-evals LLM judge.
+
+Handles OAuth2 token fetch (client credentials) and calls the
+AI Core orchestration completion endpoint.
+
+Required env vars:
+    AICORE_CLIENT_ID
+    AICORE_CLIENT_SECRET
+    AICORE_AUTH_URL
+    ORCHESTRATION_URL
+    AICORE_RESOURCE_GROUP   (default: "default")
+"""
+
+from __future__ import annotations
+
+import base64
+import os
+import time
+from typing import Any
+
+import httpx
+
+
+# ── Token cache ───────────────────────────────────────────────────────────────
+
+_token: str | None = None
+_token_expiry: float = 0.0          # epoch seconds
+_TOKEN_BUFFER_SECS = 60             # refresh 60 s before actual expiry
+
+
+def _fetch_token() -> str:
+    client_id     = os.environ["AICORE_CLIENT_ID"]
+    client_secret = os.environ["AICORE_CLIENT_SECRET"]
+    auth_url      = os.environ["AICORE_AUTH_URL"]
+
+    creds = base64.b64encode(f"{client_id}:{client_secret}".encode()).decode()
+
+    resp = httpx.post(
+        auth_url,
+        headers={
+            "Authorization": f"Basic {creds}",
+            "Content-Type": "application/x-www-form-urlencoded",
+        },
+        data={"grant_type": "client_credentials"},
+        timeout=15,
+    )
+    resp.raise_for_status()
+    body = resp.json()
+    return body["access_token"], int(body.get("expires_in", 3600))
+
+
+def _get_token() -> str:
+    global _token, _token_expiry
+    now = time.monotonic()
+    if _token is None or now >= _token_expiry:
+        token, expires_in = _fetch_token()
+        _token        = token
+        _token_expiry = now + expires_in - _TOKEN_BUFFER_SECS
+    return _token
+
+
+# ── Completion call ───────────────────────────────────────────────────────────
+
+def chat_complete(
+    system: str,
+    user: str,
+    *,
+    model_name: str = "gpt-4o",
+    max_tokens: int = 512,
+    temperature: float = 0.0,
+) -> str:
+    """
+    Call the SAP AI Core orchestration service and return the assistant text.
+
+    Uses the deployment at ORCHESTRATION_URL env var.
+    The model_name must be deployed in the AI Core resource group.
+    """
+    url            = os.environ["ORCHESTRATION_URL"].rstrip("/") + "/completion"
+    resource_group = os.environ.get("AICORE_RESOURCE_GROUP", "default")
+
+    payload: dict[str, Any] = {
+        "orchestration_config": {
+            "module_configurations": {
+                "llm_module_config": {
+                    "model_name": model_name,
+                    "model_params": {
+                        "max_tokens": max_tokens,
+                        "temperature": temperature,
+                    },
+                }
+            }
+        },
+        "messages_history": [
+            {"role": "system", "content": system},
+            {"role": "user",   "content": user},
+        ],
+    }
+
+    resp = httpx.post(
+        url,
+        headers={
+            "Authorization": f"Bearer {_get_token()}",
+            "AI-Resource-Group": resource_group,
+            "Content-Type": "application/json",
+        },
+        json=payload,
+        timeout=60,
+    )
+    resp.raise_for_status()
+    data = resp.json()
+
+    # SAP AI Core orchestration wraps the OpenAI-style response
+    choices = (
+        data.get("orchestration_result", data)
+           .get("choices", [{}])
+    )
+    return choices[0].get("message", {}).get("content", "")

iris_evals/evaluators/__init__.py

@@ -0,0 +1,17 @@
+from iris_evals.evaluators import (
+    deterministic,
+    probabilistic,
+    llm_judge,
+    trace,
+    safety,
+    performance,
+)
+
+__all__ = [
+    "deterministic",
+    "probabilistic",
+    "llm_judge",
+    "trace",
+    "safety",
+    "performance",
+]

iris_evals/evaluators/deterministic.py

@@ -0,0 +1,66 @@
+"""
+deterministic.py — Exact-match evaluation against a golden dataset.
+
+Sends each golden record to the agent, compares actual output to
+expected_output / expected_verdict. No LLM involved.
+
+Suitable for: P1.11 Duplicate Detection, P2.5 Tolerance Resolution,
+and any fully deterministic rule-based agent.
+"""
+
+from __future__ import annotations
+
+import time
+from iris_evals.schemas import EvalConfig, EvalResult, EvalType, GoldenRecord
+from iris_evals._transport import call_agent
+
+
+async def run(config: EvalConfig, records: list[GoldenRecord]) -> list[EvalResult]:
+    results: list[EvalResult] = []
+
+    for rec in records:
+        t0 = time.monotonic()
+        try:
+            raw = await call_agent(config, rec.input)
+            latency_ms = (time.monotonic() - t0) * 1000
+
+            actual = _extract_verdict_or_text(raw)
+            expected = rec.expected_verdict or rec.expected_output or ""
+
+            passed = _matches(actual, expected)
+            results.append(EvalResult(
+                record_id=rec.id,
+                eval_type=EvalType.DETERMINISTIC,
+                passed=passed,
+                score=1.0 if passed else 0.0,
+                actual_output=actual,
+                expected_output=expected,
+                reason="" if passed else f"Expected '{expected}', got '{actual}'",
+                latency_ms=latency_ms,
+            ))
+        except Exception as exc:
+            results.append(EvalResult(
+                record_id=rec.id,
+                eval_type=EvalType.DETERMINISTIC,
+                passed=False,
+                reason=f"Agent call failed: {exc}",
+            ))
+
+    return results
+
+
+def _extract_verdict_or_text(raw: dict) -> str:
+    """Pull verdict / invoice_resolution / text from the agent response."""
+    # structured SAP agent response shapes
+    for key in ("verdict", "invoice_resolution", "resolution", "status"):
+        if key in raw:
+            return str(raw[key])
+    # A2A artifact text fallback
+    if "result" in raw:
+        parts = raw["result"].get("artifacts", [{}])[0].get("parts", [{}])
+        return parts[0].get("text", str(raw)) if parts else str(raw)
+    return str(raw)
+
+
+def _matches(actual: str, expected: str) -> bool:
+    return actual.strip().upper() == expected.strip().upper()