evalcore 0.1.0__py3-none-any.whl

evalcore-0.1.0.dist-info/RECORD

@@ -0,0 +1,34 @@
+evalkit/__init__.py,sha256=_1rkqqBO1YKj8ARwj0rza_CiDg4GoneGnkiKXifTE8o,910
+evalkit/cli.py,sha256=-PXWKp9wNHY7Dbc0Z6NMGiiEH7Y57TVoYYNCUsqEGBs,22065
+evalkit/compare.py,sha256=VJveGmXcOOiJafFm2tun_kH1WYoL3niBZKBcIAyj1-Q,4487
+evalkit/loader.py,sha256=kCBeSHsXdIS4dsNqFeTP71z42Kpw2Y5DboxS35j8Ppg,4768
+evalkit/models.py,sha256=OHW2BDj1TR0uPv3RM3ksUzJER4MQwMb5nj3c8t_r_cQ,12555
+evalkit/pairwise.py,sha256=A_F-SbOC8YZNjZgdQl7O4mLAJIReJBsyPH8BRRxkwwo,10957
+evalkit/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+evalkit/rating.py,sha256=Rn0U1Wb2vqz0prH3bxtyJOT66nRGV6Lo4zZCVW_seMg,56585
+evalkit/refs.py,sha256=WQNQqnYV640DbCBLydddU8MhpkdZ_S2-Wp_ZJ1NyABU,2611
+evalkit/report.py,sha256=yYD5DjWNYV1sE2u8WMV8iyH-NbRpk25bvO7DNWUtQD8,6845
+evalkit/retry.py,sha256=UtiAIobz8DLlvxPLXmG0zf7R_C_X8_IgwOiLKoyCwOo,3029
+evalkit/runner.py,sha256=3yzqVTPtjz5k5xon5pw6nwfSTU1MP473zIcjFvlM6CI,10691
+evalkit/store.py,sha256=afqMchrH77gecGAMuseU4Ir9Yfr6GHaRs7Trg7h3oUQ,10805
+evalkit/sweep.py,sha256=GqXTCYM3R33y5RXau5aVVlrgUrXgwUzPgFNFh_EW1BA,2984
+evalkit/adapters/__init__.py,sha256=7KwMEQp3ExS4hyfv6eH4IOJC9U8_iG9uTRat8bsqjc4,681
+evalkit/adapters/_env.py,sha256=_VYsz7XFE4hk2xXaPXR_NvkkzRK_YcitToBQSfMaduM,833
+evalkit/adapters/base.py,sha256=o0IFRohtPepvH9dEtiayVf8FEazsDl8aWhTct7CXzac,1090
+evalkit/adapters/http.py,sha256=X_JpV2_I2ovOZvE2_gVBA-12Agq9YAtkFk4NDC5a0PM,3407
+evalkit/adapters/replay.py,sha256=MZo2-zI23YeKv6jf3FIsgTttdvJULstf4ziPG4HuaDI,1467
+evalkit/graders/__init__.py,sha256=jBVRJiIPk5Sl_mJbnS_UCRaGDuBWfLIM7ba1CU_zkYY,784
+evalkit/graders/base.py,sha256=xzH3igBdwaTLrSNKRgz6DZCuBDXcXSE8-9PJYMg41T8,2269
+evalkit/graders/classification.py,sha256=W6-vo51FU3nPNn8WEwi0zu-kwQT6HmQ_cbdDhrqhoSw,3872
+evalkit/graders/deterministic.py,sha256=kwqIRq0ihC0VrBT72xc6YuOeT3oPwYf1K2TBAR-X63c,4047
+evalkit/graders/judge.py,sha256=8az97kI3BL5hMmAoVjutL8bo6N0lHIiBpR2iaeH4jsw,23864
+evalkit/graders/numeric.py,sha256=LWGrvnXhuE6cJFlZ6eMYs4oICOIyhCWMZACWOshsFaA,3187
+evalkit/reporters/__init__.py,sha256=PRwuWfX054aDU6VQcYlMbout5Nmtu6Yao7b8yv8gpr8,839
+evalkit/reporters/base.py,sha256=vHD5QZ63Y_bxef9DhanbwVkofnOoe3d8WhfY2nakvjI,5660
+evalkit/reporters/html.py,sha256=6jo4REOKcSRNIDzFbqISlAjc43fgZ00umVWRMIV40oE,17899
+evalkit/reporters/markdown.py,sha256=acXtxIRskPOE28RIpZWX83Gewav3ypcnVhBjl5yVcp0,2844
+evalcore-0.1.0.dist-info/METADATA,sha256=shSdhgGK2gyY4SotUxGsQjY2xmuApayelaqpkcTbGgc,38374
+evalcore-0.1.0.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
+evalcore-0.1.0.dist-info/entry_points.txt,sha256=qW9PtJgBdWRQAMp2kDN3AxqNna_nBk0sA35u6jEa0dY,45
+evalcore-0.1.0.dist-info/licenses/LICENSE,sha256=k5Kb0VE7QFc6o_kU42n1Q5EGQqbGBl6HhprgjDYN_jw,1507
+evalcore-0.1.0.dist-info/RECORD,,

evalcore-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.31.0
+Root-Is-Purelib: true
+Tag: py3-none-any

evalcore-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+evalkit = evalkit.cli:main

evalcore-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,28 @@
+BSD 3-Clause License
+
+Copyright (c) 2026 AWeber Communications
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors
+   may be used to endorse or promote products derived from this software
+   without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

evalkit/__init__.py

@@ -0,0 +1,39 @@
+"""evalkit - a generic, consumer-agnostic evaluation engine.
+
+The engine knows nothing about any particular system under test. A consumer
+supplies four things as data/plug-ins:
+
+1. an **adapter** config (how to call its system + the knobs a variant sets),
+2. **datasets** (cases with opaque ``input``/``expected`` blobs),
+3. **graders** (generic ones here + any custom ones it registers), and
+4. a **suite + thresholds** config.
+
+Everything else - runner, comparison/regression engine, results store, and
+reporting - lives here and is reused unchanged across consumers.
+
+See ``docs/evals/platform-design.md`` for the full design.
+"""
+
+from evalkit import (
+    adapters,
+    compare,
+    graders,
+    loader,
+    models,
+    refs,
+    report,
+    runner,
+    store,
+)
+
+__all__ = [
+    'adapters',
+    'compare',
+    'graders',
+    'loader',
+    'models',
+    'refs',
+    'report',
+    'runner',
+    'store',
+]

evalkit/adapters/__init__.py

@@ -0,0 +1,15 @@
+"""Target adapters - the one seam to the system under test.
+
+An adapter turns a (case, variant) into a normalized ``Output``. The built-in
+``http`` adapter (POST a templated body to an endpoint, extract fields from the
+JSON response) covers most request/response APIs; ``replay`` returns recorded
+outputs so the whole pipeline runs offline and in tests. Consumers register
+their own adapters for anything else - e.g. a browser-driven adapter or one
+that reads results a deployed system already logged to an observability store.
+
+Importing this package registers the built-in adapter types.
+"""
+
+from evalkit.adapters import base, http, replay
+
+__all__ = ['base', 'http', 'replay']

evalkit/adapters/_env.py

@@ -0,0 +1,24 @@
+"""``${ENV_VAR}`` expansion shared by adapters.
+
+Adapters keep secrets and environment-specific URLs out of the suite file by
+writing ``${VAR}`` placeholders that expand from the process environment at
+invoke time. Kept here (not in any one adapter) so every adapter can reuse it
+without importing a sibling adapter's internals.
+"""
+
+import os
+import re
+import typing
+
+_ENV_RE = re.compile(r'\$\{([A-Z0-9_]+)\}')
+
+
+def _expand_env(value: typing.Any) -> typing.Any:
+    """Expand ``${VAR}`` in strings (recursively through dict/list)."""
+    if isinstance(value, str):
+        return _ENV_RE.sub(lambda m: os.environ.get(m.group(1), ''), value)
+    if isinstance(value, dict):
+        return {k: _expand_env(v) for k, v in value.items()}
+    if isinstance(value, list):
+        return [_expand_env(v) for v in value]
+    return value

evalkit/adapters/base.py

@@ -0,0 +1,40 @@
+"""Adapter protocol and registry."""
+
+import typing
+
+from evalkit import models
+
+
+@typing.runtime_checkable
+class TargetAdapter(typing.Protocol):
+    """Invoke the system under test for one case under one variant."""
+
+    async def invoke(
+        self, case: models.Case, variant: models.Variant
+    ) -> models.Output: ...
+
+
+_REGISTRY: dict[str, type] = {}
+
+
+def register(type_name: str) -> typing.Callable[[type], type]:
+    """Class decorator registering an adapter under a config ``type``."""
+
+    def _decorate(cls: type) -> type:
+        if type_name in _REGISTRY:
+            raise ValueError(f'adapter type {type_name!r} already registered')
+        _REGISTRY[type_name] = cls
+        return cls
+
+    return _decorate
+
+
+def build_adapter(spec: dict) -> TargetAdapter:
+    """Instantiate an adapter from a ``{type, ...}`` config spec."""
+    spec = dict(spec)
+    type_name = spec.pop('type')
+    if type_name not in _REGISTRY:
+        raise ValueError(
+            f'unknown adapter type {type_name!r}; known: {sorted(_REGISTRY)}'
+        )
+    return _REGISTRY[type_name](**spec)

evalkit/adapters/http.py

@@ -0,0 +1,96 @@
+"""HTTP target adapter - POST a templated body, extract response fields.
+
+The request ``body`` is a template whose ``$ref`` strings are resolved against
+the case and variant (e.g. ``$input.content``, ``$variant.model``). ``extract``
+maps output field names to dotted paths into the JSON response. ``base_url``
+and header values support ``${ENV_VAR}`` expansion so secrets/URLs stay out of
+the suite file.
+
+``httpx`` is imported lazily so offline/replay runs need no network stack.
+"""
+
+import time
+
+from evalkit import models, refs
+from evalkit.adapters import base
+from evalkit.adapters._env import _expand_env
+
+
+@base.register('http')
+class HTTPAdapter:
+    """POST ``body`` to ``base_url + path`` and extract fields from JSON."""
+
+    def __init__(
+        self,
+        base_url: str,
+        path: str,
+        body: dict,
+        extract: dict[str, str],
+        method: str = 'POST',
+        headers: dict[str, str] | None = None,
+        timeout: float = 30.0,
+    ):
+        self.base_url = base_url
+        self.path = path
+        self.body_template = body
+        self.extract = extract
+        self.method = method
+        self.headers = headers or {}
+        self.timeout = timeout
+
+    async def invoke(
+        self, case: models.Case, variant: models.Variant
+    ) -> models.Output:
+        import httpx
+
+        context = {
+            'input': case.input,
+            'expected': case.expected or {},
+            'variant': variant.knobs,
+            'case': case.model_dump(),
+        }
+        payload = refs.build_value(self.body_template, context)
+        url = _expand_env(self.base_url).rstrip('/') + self.path
+        # Drop headers whose ${ENV} expanded to empty (e.g. unset auth) so we
+        # never send a blank Authorization that a proxy might 400 on.
+        headers = {k: v for k, v in _expand_env(self.headers).items() if v}
+
+        start = time.monotonic()
+        try:
+            async with httpx.AsyncClient(timeout=self.timeout) as client:
+                response = await client.request(
+                    self.method, url, json=payload, headers=headers
+                )
+        except httpx.HTTPError as exc:
+            # Network-level failures (timeouts, connection resets) are
+            # transient - worth a retry.
+            return models.Output(
+                error=f'{type(exc).__name__}: {exc}',
+                retryable=True,
+                latency_ms=(time.monotonic() - start) * 1000,
+            )
+        latency_ms = (time.monotonic() - start) * 1000
+
+        if response.status_code >= 400:
+            # 429 (rate limit) and 5xx (server) are transient; other 4xx are
+            # the request's fault and won't fix on retry.
+            code = response.status_code
+            return models.Output(
+                error=f'HTTP {code}',
+                retryable=code == 429 or code >= 500,
+                raw=response.text[:2000],
+                latency_ms=latency_ms,
+            )
+        try:
+            body = response.json()
+        except ValueError:
+            return models.Output(
+                error='non-JSON response',
+                raw=response.text[:2000],
+                latency_ms=latency_ms,
+            )
+        fields = {
+            name: refs.resolve_path(body, path)
+            for name, path in self.extract.items()
+        }
+        return models.Output(fields=fields, raw=body, latency_ms=latency_ms)

evalkit/adapters/replay.py

@@ -0,0 +1,41 @@
+"""Replay adapter - return recorded outputs instead of calling a service.
+
+Lets the whole pipeline (runner -> graders -> compare -> report) run offline
+in unit tests with no network, no API keys, and no deployed service. The
+fixtures file maps ``case_id`` to per-variant recorded outputs::
+
+    obfuscated_eval:
+      baseline:  {verdict: ok}          # a recorded miss
+      candidate: {verdict: malicious}
+
+Each recorded value becomes ``Output.fields`` for that (case, variant).
+"""
+
+from evalkit import loader, models
+from evalkit.adapters import base
+
+
+@base.register('replay')
+class ReplayAdapter:
+    """Serve recorded ``Output.fields`` keyed by ``case.id`` and variant."""
+
+    def __init__(self, fixtures: str | dict):
+        if isinstance(fixtures, str):
+            self._fixtures = loader.load_data_file(fixtures)
+        else:
+            self._fixtures = fixtures
+
+    async def invoke(
+        self, case: models.Case, variant: models.Variant
+    ) -> models.Output:
+        per_case = self._fixtures.get(case.id)
+        if per_case is None:
+            return models.Output(error=f'no fixture for case {case.id!r}')
+        recorded = per_case.get(variant.name)
+        if recorded is None:
+            return models.Output(
+                error=f'no fixture for {case.id!r}/{variant.name!r}'
+            )
+        if 'error' in recorded:
+            return models.Output(error=recorded['error'])
+        return models.Output(fields=dict(recorded))