neens-eval 0.3.0__py3-none-any.whl

neens_eval/__init__.py

@@ -0,0 +1,115 @@
+"""Neens pre-prod evaluation runner — the thin CI glue for the PULL model.
+
+In the pull model, YOUR CI harness drives YOUR candidate agent against a golden dataset's
+frozen prompts and emits OpenTelemetry traces tagged so Neens can correlate + score them.
+This package is the one-liner that makes that a CI step: for each frozen prompt it injects the
+Neens correlation attributes (via ``OTEL_RESOURCE_ATTRIBUTES`` + plain env vars) and executes
+YOUR agent command, then starts the run, polls to a terminal state, fetches the gate, and
+returns an exit code CI can gate on (0 = gate passed).
+
+It is deliberately NOT a universal agent-caller: it subprocesses a user-supplied command and
+relies on that agent already being OpenTelemetry-instrumented and pointed at the Neens OTLP
+receiver. Neens is the observability tool — the agent emits the traces.
+
+Zero runtime dependencies (stdlib ``urllib``/``json`` only) so ``pip install ./sdk`` needs no
+network and adds nothing to a CI image beyond Python itself.
+"""
+from __future__ import annotations
+
+from .client import NeensEvalClient, NeensEvalError
+from .correlation import build_item_env
+from .legacy_env import (
+    LEGACY_ENV_IGNORE_VAR,
+    SDK_ENV_VARS,
+    LegacyEnvVarError,
+    check_legacy_env,
+    scan_legacy_env,
+)
+from .gate import (
+    GateEvaluation,
+    GatePolicy,
+    GatePolicyError,
+    RuleResult,
+    evaluate,
+)
+from .runner import (
+    ItemResult,
+    RunOutcome,
+    legacy_correlation_notice,
+    run_eval,
+    run_push_eval,
+)
+from .sweep import (
+    SWEEP_TERMINAL_STATUSES,
+    VERDICT_NONE_CLEAR,
+    VERDICT_UNAVAILABLE,
+    VERDICT_WINNER,
+    SweepArgError,
+    SweepOutcome,
+    decide_comparison_exit_code,
+    decide_exit_code,
+    decide_sweep,
+    get_sweep,
+    parse_arm,
+    parse_arms,
+    preview_sweep,
+    render_comparison,
+    start_sweep,
+)
+
+__all__ = [
+    "NeensEvalClient",
+    "NeensEvalError",
+    "build_item_env",
+    "ItemResult",
+    "RunOutcome",
+    "legacy_correlation_notice",
+    "run_eval",
+    "run_push_eval",
+    # Multi-model sweeps (issue #449)
+    "SweepArgError",
+    "SweepOutcome",
+    "SWEEP_TERMINAL_STATUSES",
+    "decide_exit_code",
+    "get_sweep",
+    "parse_arm",
+    "parse_arms",
+    "preview_sweep",
+    "start_sweep",
+    # N-way comparison verdict (issue #450) — the SDK RENDERS the server's ranking, never its own
+    "VERDICT_WINNER",
+    "VERDICT_NONE_CLEAR",
+    "VERDICT_UNAVAILABLE",
+    "decide_comparison_exit_code",
+    "decide_sweep",
+    "render_comparison",
+    # Gate-as-code policy layer
+    "GatePolicy",
+    "GatePolicyError",
+    "GateEvaluation",
+    "RuleResult",
+    "evaluate",
+    # Legacy-configuration guard (issue #470). Exported so a wrapper that embeds the SDK can run
+    # the same check; importing this package never runs it (only the CLI entry point does).
+    # Kept name-for-name with `sdk-ts/src/index.ts` — a wrapper that reads the hatch name off the
+    # package root must work in both.
+    "LEGACY_ENV_IGNORE_VAR",
+    "SDK_ENV_VARS",
+    "LegacyEnvVarError",
+    "check_legacy_env",
+    "scan_legacy_env",
+    # OTel correlation attribute names — the wire contract with the backend
+    # (mirrors ``core/preprod.py``; duplicated here so the SDK stays self-contained).
+    "EVAL_RUN_ID_ATTR",
+    "DATASET_ITEM_ID_ATTR",
+    "VERSION_LABEL_ATTR",
+]
+
+# The OTel attribute NAMES Neens correlates on. These MUST match ``core/preprod.py`` in the
+# backend — they are the wire contract. Kept as literals (not an import) so the SDK is a
+# free-standing, installable package with no dependency on the Neens backend source tree.
+EVAL_RUN_ID_ATTR = "neens.eval_run_id"
+DATASET_ITEM_ID_ATTR = "neens.dataset_item_id"
+VERSION_LABEL_ATTR = "neens.version_label"
+
+__version__ = "0.3.0"