intent-eval-core 0.7.0__py3-none-any.whl

intent_eval_core/__init__.py

@@ -0,0 +1,71 @@
+"""intent-eval-core — Python distribution of the Intent Eval Platform contracts kernel.
+
+The same canonical contracts the TypeScript ``@intentsolutions/core`` package
+exposes as Zod validators + JSON Schemas, here as **Pydantic v2 models** for
+Python consumers (LLM-harness lab, agent-runtime sandbox, ``validate-*`` skill
+emit-evidence retrofits).
+
+The models are codegen'd from the canonical ``schemas/v1/*.json`` JSON Schemas
+(``datamodel-code-generator``) and validated for byte-for-byte parity against the
+same golden fixtures the TypeScript/AJV side validates — so a payload accepted by
+the Zod validator is accepted here, and one rejected there is rejected here.
+
+Usage::
+
+    from intent_eval_core import GateResultV1, EvalSpec
+
+    gate = GateResultV1.model_validate(some_dict)   # raises ValidationError on drift
+
+Version is kept in lockstep with the npm ``@intentsolutions/core`` ``package.json``
+version (the JSON Schemas are the shared source of truth for both).
+"""
+
+from __future__ import annotations
+
+from .models import (
+    GATE_RESULT_V1_URI,
+    CostRecord,
+    DashboardRenderV1,
+    EvalRun,
+    EvalSpec,
+    EvidenceBundle,
+    FailureTaxonomy,
+    GateResultV1,
+    JudgeDecision,
+    MatcherMap,
+    RegressionPack,
+    RetractionV1,
+    RolloutGate,
+    RuntimeReceipt,
+    SessionTrace,
+    SkillSnapshot,
+    ToolInvocation,
+)
+
+# Kept in lockstep with npm package.json#version by scripts/check_version_lockstep.py
+# (CI-gated). The JSON Schemas under schemas/v1/ are the shared source of truth.
+__version__ = "0.7.0"
+
+__all__ = [
+    "__version__",
+    # Entities (Blueprint B § 2.1 – § 2.13)
+    "EvalSpec",
+    "EvalRun",
+    "MatcherMap",
+    "EvidenceBundle",
+    "JudgeDecision",
+    "RuntimeReceipt",
+    "RegressionPack",
+    "RolloutGate",
+    "SkillSnapshot",
+    "SessionTrace",
+    "ToolInvocation",
+    "CostRecord",
+    "FailureTaxonomy",
+    # Predicate bodies
+    "GateResultV1",
+    "RetractionV1",
+    "DashboardRenderV1",
+    # Constants
+    "GATE_RESULT_V1_URI",
+]

intent_eval_core/_generated/README.md

@@ -0,0 +1,38 @@
+# `_generated/` — Pydantic codegen reference output (NOT the public API)
+
+This directory holds the **raw output** of
+[`datamodel-code-generator`](https://github.com/koxudaxi/datamodel-code-generator)
+run against `schemas/v1/*.json`. It is the Python analogue of the Zod
+`src/validators/v1/_generated/` directory — **reference material, not the
+canonical public surface.**
+
+The canonical Python surface lives one level up at
+`intent_eval_core/models.py`, which:
+
+1. Re-exports each generated root model under its canonical name
+   (`GateResultV1`, `EvalSpec`, …) — the generator derives verbose class names
+   from each schema's `title`.
+2. Layers hand-written cross-field `model_validator`s for the `allOf`/`if-then`
+   conditional rules the generator cannot express (e.g. gate-result/v1's
+   "advisory requires advisory_severity", "fail/advisory/error requires a
+   non-empty reason"). These mirror the Zod `.superRefine()` blocks exactly.
+
+## Codegen invocation
+
+```bash
+pnpm run codegen:pydantic          # regenerate (pinned datamodel-code-generator)
+pnpm run codegen:pydantic:check    # idempotency gate (CI)
+```
+
+The script (`scripts/codegen_pydantic.py`) is reproducible: same schemas + the
+pinned generator version => byte-identical output. It normalizes the absolute
+`$ref`/`$id` the predicate schemas carry down to relative refs so the generator
+resolves them locally (AJV resolves them against each schema's `$id` base; we
+collapse to relative paths — both arrive at the same target file).
+
+## What NOT to do
+
+- **Don't import from `_generated/`** — import from `intent_eval_core` /
+  `intent_eval_core.models` (the public surface).
+- **Don't edit `_generated/*.py` by hand** — they're overwritten on next codegen
+  and CI-gated for staleness.

intent_eval_core/_generated/__init__.py

@@ -0,0 +1,2 @@
+# generated by datamodel-codegen:
+#   filename:  schemas

intent_eval_core/_generated/_common_schema.py

@@ -0,0 +1,210 @@
+# generated by datamodel-codegen:
+#   filename:  _common.schema.json
+
+from __future__ import annotations
+
+from enum import Enum
+from typing import Any
+
+from pydantic import (
+    AwareDatetime,
+    BaseModel,
+    ConfigDict,
+    Field,
+    RootModel,
+    conint,
+    constr,
+)
+
+
+class CommonBrandedPrimitiveDefinitionsForIntentsolutionsCoreSchemas(RootModel[Any]):
+    root: Any = Field(
+        ...,
+        title="Common branded-primitive definitions for @intentsolutions/core schemas",
+    )
+    """
+    Shared $defs referenced by every entity schema and the gate-result/v1 predicate body. Branded primitives correspond to the TypeScript brands in src/primitives.ts — at the JSON Schema level, they are pattern-validated strings/numbers.
+    """
+
+
+class Uuidv7(
+    RootModel[
+        constr(
+            pattern=r"^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$"
+        )
+    ]
+):
+    root: constr(
+        pattern=r"^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$"
+    )
+    """
+    UUIDv7 — time-ordered 128-bit identifier (RFC 9562). Time-ordered prefix means natural sort order matches insertion order. Per Blueprint B § 6.2 universal conventions.
+    """
+
+
+class Sha256(RootModel[constr(pattern=r"^[a-f0-9]{64}$")]):
+    root: constr(pattern=r"^[a-f0-9]{64}$")
+    """
+    Bare 64-hex-char SHA-256 digest. Used for entity content_hash fields per Blueprint B § 2.x prose.
+    """
+
+
+class Sha256Prefixed(RootModel[constr(pattern=r"^sha256:[a-f0-9]{64}$")]):
+    root: constr(pattern=r"^sha256:[a-f0-9]{64}$")
+    """
+    Prefixed SHA-256 digest in the form `sha256:<64-lowercase-hex>`. REQUIRED format for every *_hash field in the gate-result/v1 predicate body per Blueprint B § 7.4. Unknown algorithm prefixes cause row verification failure.
+    """
+
+
+class Rfc3339(RootModel[AwareDatetime]):
+    root: AwareDatetime
+    """
+    RFC 3339 / ISO 8601 timestamp with timezone. Blueprint B requires UTC ('Z') with millisecond precision for fields in worker-lease ordering (clock-skew tolerance ±10 ms).
+    """
+
+
+class Semver(
+    RootModel[
+        constr(pattern=r"^[0-9]+\.[0-9]+\.[0-9]+(-[A-Za-z0-9.-]+)?(\+[A-Za-z0-9.-]+)?$")
+    ]
+):
+    root: constr(
+        pattern=r"^[0-9]+\.[0-9]+\.[0-9]+(-[A-Za-z0-9.-]+)?(\+[A-Za-z0-9.-]+)?$"
+    )
+    """
+    SemVer 2.0.0 version string.
+    """
+
+
+class KebabSlug(RootModel[constr(pattern=r"^[a-z0-9][a-z0-9-]*$")]):
+    root: constr(pattern=r"^[a-z0-9][a-z0-9-]*$")
+    """
+    Lowercase kebab-case slug.
+    """
+
+
+class ActorIdentity(RootModel[constr(min_length=1)]):
+    root: constr(min_length=1)
+    """
+    Opaque actor identity string — engineer email, service-account name, or worker id. Format intentionally unspecified at this layer.
+    """
+
+
+class StorageKey(RootModel[constr(min_length=1)]):
+    root: constr(min_length=1)
+    """
+    Object-storage key (content-addressed). Format opaque at this layer; storage backends own resolution.
+    """
+
+
+class OtelSpanId(RootModel[constr(pattern=r"^[a-f0-9]{16}$")]):
+    root: constr(pattern=r"^[a-f0-9]{16}$")
+    """
+    OpenTelemetry span identifier — 16 lowercase hex characters per OTel spec.
+    """
+
+
+class MicroUsd(RootModel[conint(ge=0)]):
+    root: conint(ge=0)
+    """
+    Cost in micro-USD for sub-cent precision. Per Blueprint B § 2.12. Convert to USD at display layer (value / 1_000_000).
+    """
+
+
+class MmClass(Enum):
+    """
+    Matcher Map class — closed v1 canonical set per Blueprint B § 2.3. Extension via FailureTaxonomy proposed→canonical lifecycle (Class-1 ISEDC).
+    """
+
+    MM_1 = "MM-1"
+    MM_2 = "MM-2"
+    MM_3 = "MM-3"
+    MM_4 = "MM-4"
+    MM_5 = "MM-5"
+    MM_6 = "MM-6"
+
+
+class MmClassId(RootModel[constr(pattern=r"^MM-[0-9]+$")]):
+    root: constr(pattern=r"^MM-[0-9]+$")
+    """
+    FailureTaxonomy natural key — accepts any MM-N value (broader than MmClass enum since taxonomy is upstream of kernel enum).
+    """
+
+
+class RunnerIdentifier(
+    RootModel[
+        constr(
+            pattern=r"^[a-z0-9][a-z0-9-]*@[0-9]+\.[0-9]+\.[0-9]+(-[A-Za-z0-9.-]+)?(\+[A-Za-z0-9.-]+)?$"
+        )
+    ]
+):
+    root: constr(
+        pattern=r"^[a-z0-9][a-z0-9-]*@[0-9]+\.[0-9]+\.[0-9]+(-[A-Za-z0-9.-]+)?(\+[A-Za-z0-9.-]+)?$"
+    )
+    """
+    Tool + SemVer identifier in the form `<kebab-slug>@<semver>`.
+    """
+
+
+class CommitSha(RootModel[constr(pattern=r"^[a-f0-9]{7,40}$")]):
+    root: constr(pattern=r"^[a-f0-9]{7,40}$")
+    """
+    Git commit SHA — full 40-hex or short 7-hex accepted per Blueprint B § 7.4 line 815.
+    """
+
+
+class SubjectName(
+    RootModel[
+        constr(
+            pattern=r"^[a-z0-9][a-z0-9-]*:(client|server|ci|sandbox|local):[a-zA-Z0-9][a-zA-Z0-9.-]*$"
+        )
+    ]
+):
+    root: constr(
+        pattern=r"^[a-z0-9][a-z0-9-]*:(client|server|ci|sandbox|local):[a-zA-Z0-9][a-zA-Z0-9.-]*$"
+    )
+    """
+    in-toto subject name per Blueprint B § 7.3 line 779. Three segments: `<tool>:<side>:<gate-id>` where side ∈ {client, server, ci, sandbox, local}.
+    """
+
+
+class Coverage(BaseModel):
+    """
+    Coverage declaration per Blueprint B § 7.4 line 807. Both arrays REQUIRED. Element type LOCKED as string per deferral-D (bd_000-projects-9xyk). NOT_APPLICABLE encoding (§ 7.4 line 832): gate_decision='pass' + populated dimensions_skipped + documented skip in gate_reasons.
+    """
+
+    model_config = ConfigDict(
+        extra="forbid",
+    )
+    dimensions_evaluated: list[str]
+    dimensions_skipped: list[str]
+
+
+class Status(Enum):
+    evaluated = "evaluated"
+    skipped = "skipped"
+
+
+class CoverageDimensionDetail(BaseModel):
+    """
+    Richer per-dimension coverage element (deferral-D lockup, bd_000-projects-9xyk). The OPTIONAL companion to `coverage`'s string arrays — adopted additively per § 7.2 so the `coverage` element type stays `string`. Descriptive only; MUST NOT drive ship/no-ship (mirrors metadata rule).
+    """
+
+    model_config = ConfigDict(
+        extra="forbid",
+    )
+    id: str
+    """
+    Dimension id — MUST appear in the matching coverage array (cross-field enforced at the validator layer).
+    """
+    status: Status
+    skip_reason: str | None = None
+    threshold: float | None = None
+    observed: float | None = None
+
+
+class ErrorClass(RootModel[constr(pattern=r"^[a-z][a-z0-9_]*(\.[a-z][a-z0-9_]*)+$")]):
+    root: constr(pattern=r"^[a-z][a-z0-9_]*(\.[a-z][a-z0-9_]*)+$")
+    """
+    Tool-error class (deferral-E lockup, bd_000-projects-84li). A kernel-registered cross-cutting class OR a tool-defined `<domain>.<condition>` string. The registration CONVENTION is locked (dot-separated, lowercase); the enum stays OPEN.
+    """

intent_eval_core/_generated/cost_record_schema.py

@@ -0,0 +1,59 @@
+# generated by datamodel-codegen:
+#   filename:  cost-record.schema.json
+
+from __future__ import annotations
+
+from enum import Enum
+
+from pydantic import BaseModel, ConfigDict, conint, constr
+
+from . import _common_schema as schema
+
+
+class AttributionClass(Enum):
+    """
+    Cost attribution class enum (Blueprint B § 2.12, closed 7-element set).
+    """
+
+    run = "run"
+    provider = "provider"
+    judge = "judge"
+    replay = "replay"
+    cache_decision = "cache_decision"
+    optimizer_experiment = "optimizer_experiment"
+    system = "system"
+
+
+class CostrecordSingleCostResourceAttributionRowBlueprintB212(BaseModel):
+    """
+    Single terminal state `recorded`. Immutable. Rollups = separate rows with attribution_class='system'. NOT replay-deterministic (cost varies by cache state, pricing, seed routing). external_api_cost_micro_usd uses micro-USD per Blueprint B § 2.12 — NORMATIVE precision requirement. Predicate URI: cost-attribution/v1 (sigstore_staging).
+    """
+
+    model_config = ConfigDict(
+        extra="forbid",
+    )
+    id: schema.Uuidv7
+    eval_run_id: schema.Uuidv7 | None = None
+    """
+    Nullable for system-level rollups.
+    """
+    tool_invocation_id: schema.Uuidv7 | None = None
+    """
+    Nullable for run-level / system rollups.
+    """
+    attribution_class: AttributionClass
+    """
+    Cost attribution class enum (Blueprint B § 2.12, closed 7-element set).
+    """
+    provider_id: constr(min_length=1) | None = None
+    tokens_consumed: conint(ge=0)
+    prompt_tokens: conint(ge=0)
+    completion_tokens: conint(ge=0)
+    cached_tokens: conint(ge=0)
+    wall_clock_ms: conint(ge=0)
+    external_api_cost_micro_usd: schema.MicroUsd
+    recorded_at: schema.Rfc3339
+    cost_basis_version: constr(min_length=1)
+    """
+    Version tag of the cost-basis lookup table. Table itself forward-deferred to iel-E14b.
+    """

intent_eval_core/_generated/dashboard_render_schema.py

@@ -0,0 +1,78 @@
+# generated by datamodel-codegen:
+#   filename:  dashboard-render.schema.json
+
+from __future__ import annotations
+
+from pydantic import BaseModel, ConfigDict, Field, constr
+
+from . import _common_schema as schema
+
+
+class RenderedArtifact(BaseModel):
+    """
+    The rendered dashboard artifact this row attests.
+    """
+
+    model_config = ConfigDict(
+        extra="forbid",
+    )
+    uri: constr(min_length=1) | None = None
+    """
+    OPTIONAL location of the rendered artifact — relative path or absolute URL. The content_hash is the authoritative identifier; uri is a convenience pointer.
+    """
+    content_hash: schema.Sha256Prefixed
+    """
+    sha256-prefixed digest of the rendered HTML artifact bytes. Reproducing this from input_bundles is what a verifier checks.
+    """
+    media_type: constr(min_length=1) | None = None
+    """
+    OPTIONAL IANA media type of the rendered artifact (e.g., text/html).
+    """
+
+
+class InputBundle(BaseModel):
+    model_config = ConfigDict(
+        extra="forbid",
+    )
+    bundle_id: schema.Uuidv7 | None = None
+    """
+    UUIDv7 of an input EvidenceBundle (FK → EvidenceBundle.id).
+    """
+    content_hash: schema.Sha256Prefixed | None = None
+    """
+    sha256-prefixed digest of the input bundle payload, pinning exact content.
+    """
+
+
+class DashboardRenderV1InTotoPredicateBodyAttestingARenderedDashboardArtifact(
+    BaseModel
+):
+    """
+    Predicate body of an in-toto Statement v1 whose predicateType is https://evals.intentsolutions.io/dashboard-render/v1. Attests that a specific rendered dashboard HTML artifact was produced from a specific, content-addressed set of evidence inputs — i.e., 'this rendered page is a faithful function of exactly these signed bundles.' Enables sign-your-own-homework verification: a third party can re-run the renderer against the same inputs and reproduce rendered_artifact.content_hash. This schema validates ONLY the predicate body — the enclosing in-toto Statement envelope (_type, subject, predicateType) is validated separately. Each row independently verifiable; NO top-level bundle signature per Blueprint B § 7 line 754.
+
+    ADDITIVE in kernel v0.2.0 (B3 binding, sequenced): net-new predicate URI; no v0.1 contract changes. Per § 7.2 backward-compat, adding a predicate URI is allowed. Runs in sigstore_staging until a second independent verifier exists (sign-your-own-homework sequencing) and production-Rekor unlock per DR-010 Q3.
+    """
+
+    model_config = ConfigDict(
+        extra="forbid",
+    )
+    rendered_artifact: RenderedArtifact
+    """
+    The rendered dashboard artifact this row attests.
+    """
+    input_bundles: list[InputBundle] = Field(..., min_length=1)
+    """
+    Content-addressed evidence inputs the artifact was rendered from. Each entry pins a bundle by id and/or digest so the input set is reproducible. MUST be non-empty — a render with no evidence inputs is not attestable.
+    """
+    rendered_at: schema.Rfc3339
+    """
+    RFC 3339 UTC timestamp at which the render was produced.
+    """
+    renderer: schema.RunnerIdentifier
+    """
+    Renderer tool identity + version in the form `<kebab-slug>@<semver>` (e.g., intent-eval-dashboard@0.2.0). Mirrors gate-result/v1's `runner` discipline so the producing tool is always pinned.
+    """
+    renderer_config_hash: schema.Sha256Prefixed | None = None
+    """
+    OPTIONAL sha256-prefixed hash of the renderer configuration / template set, so a verifier can pin not just the tool version but the exact template inputs.
+    """

intent_eval_core/_generated/eval_run_schema.py

@@ -0,0 +1,97 @@
+# generated by datamodel-codegen:
+#   filename:  eval-run.schema.json
+
+from __future__ import annotations
+
+from enum import Enum
+
+from pydantic import BaseModel, ConfigDict, constr
+
+from . import _common_schema as schema
+
+
+class State(Enum):
+    """
+    EvalRun state enum (Blueprint B § 2.2 + § 3.1). Terminal states: archived, skipped_due_to_gate, archived_failed.
+    """
+
+    queued = "queued"
+    running = "running"
+    judged = "judged"
+    reported = "reported"
+    archived = "archived"
+    skipped_due_to_gate = "skipped_due_to_gate"
+    archived_failed = "archived_failed"
+
+
+class TerminalReason(Enum):
+    """
+    Why this Run terminated. Set per Blueprint B § 3.1 enumeration + § 1.3 (upstream_feed_failed).
+    """
+
+    queued_timeout_elapsed = "queued_timeout_elapsed"
+    run_timeout_elapsed = "run_timeout_elapsed"
+    worker_crash_exhausted = "worker_crash_exhausted"
+    credential_leak_detected = "credential_leak_detected"
+    judge_unavailable_exhausted = "judge_unavailable_exhausted"
+    token_ceiling_exceeded = "token_ceiling_exceeded"
+    evidence_contract_violation = "evidence_contract_violation"
+    upstream_feed_failed = "upstream_feed_failed"
+
+
+class EvalrunSingleExecutionOfAnEvalspecAgainstASkillsnapshotBlueprintB22(BaseModel):
+    """
+    State machine spans 7 states with explicit transition table at Blueprint B § 3.1. Mutable in non-terminal states; frozen at terminal. Clock-skew tolerance ±10 ms across workers.
+    """
+
+    model_config = ConfigDict(
+        extra="forbid",
+    )
+    id: schema.Uuidv7
+    eval_spec_id: schema.Uuidv7
+    eval_spec_version: schema.Semver
+    """
+    Frozen at queue time — defends against mid-flight spec mutation.
+    """
+    eval_spec_content_hash: schema.Sha256
+    """
+    Frozen at queue time.
+    """
+    skill_snapshot_id: schema.Uuidv7
+    state: State
+    """
+    EvalRun state enum (Blueprint B § 2.2 + § 3.1). Terminal states: archived, skipped_due_to_gate, archived_failed.
+    """
+    terminal_reason: TerminalReason | None = None
+    queued_at: schema.Rfc3339
+    started_at: schema.Rfc3339 | None = None
+    judged_at: schema.Rfc3339 | None = None
+    reported_at: schema.Rfc3339 | None = None
+    archived_at: schema.Rfc3339 | None = None
+    worker_id: constr(min_length=1) | None = None
+    lease_expires_at: schema.Rfc3339 | None = None
+    session_trace_id: schema.Uuidv7
+    """
+    FK → SessionTrace.id. Populated as Run executes.
+    """
+    evidence_bundle_id: schema.Uuidv7 | None = None
+    """
+    FK → EvidenceBundle.id. Populated at judged → reported.
+    """
+    cost_record_id: schema.Uuidv7
+    """
+    FK → CostRecord.id. Populated continuously.
+    """
+    parent_run_id: schema.Uuidv7 | None = None
+    """
+    Multi-step DAG child link.
+    """
+    idempotency_key: schema.Uuidv7
+    """
+    Defaults to `id` if not explicitly provided.
+    """
+    submitted_by: schema.ActorIdentity
+    tenant_id: schema.Uuidv7 | None = None
+    """
+    RESERVED multi-tenancy slot (deferral-G, bd_000-projects-k0fj). OPTIONAL + additive per § 7.2; tenant-isolation semantics deferred to a future DR. SHOULD equal the parent EvalSpec.tenant_id when both present (runtime invariant). v1 single-tenant runs omit it.
+    """