intent-eval-core 0.7.0__py3-none-any.whl

intent_eval_core/_generated/regression_pack_schema.py

@@ -0,0 +1,46 @@
+# generated by datamodel-codegen:
+#   filename:  regression-pack.schema.json
+
+from __future__ import annotations
+
+from pydantic import BaseModel, ConfigDict, Field, conint, constr
+
+from . import _common_schema as schema
+
+
+class OutcomeSummary(BaseModel):
+    model_config = ConfigDict(
+        extra="forbid",
+    )
+    pass_: conint(ge=0) = Field(..., alias="pass")
+    fail: conint(ge=0)
+    advisory: conint(ge=0) | None = None
+    not_applicable: conint(ge=0) | None = None
+    error: conint(ge=0) | None = None
+
+
+class RegressionpackFrozenBundleOfEvalrunsProvingABehaviorContractBlueprintB27(
+    BaseModel
+):
+    """
+    State machine: draft → committed → superseded. Mutable in draft; immutable once committed. Predicate URI (deferred Phase B+): regression-pack/v1.
+    """
+
+    model_config = ConfigDict(
+        extra="forbid",
+    )
+    id: schema.Uuidv7
+    name: schema.KebabSlug
+    purpose: constr(min_length=1)
+    skill_snapshot_sha: schema.Sha256
+    eval_spec_ids: list[schema.Uuidv7]
+    eval_run_ids: list[schema.Uuidv7]
+    outcome_summary: dict[constr(pattern=r"^MM-[1-6]$"), OutcomeSummary]
+    """
+    Aggregate pass/fail (and optional extension) per MM-class. Partial because not every pack tests every class. Per Blueprint B § 2.7: 'aggregate pass/fail rates per matcher class' — others optional, not invented.
+    """
+    ancestor_pack_id: schema.Uuidv7 | None = None
+    delta_declaration: str
+    created_at: schema.Rfc3339
+    created_by: schema.ActorIdentity
+    content_hash: schema.Sha256

intent_eval_core/_generated/retraction_schema.py

@@ -0,0 +1,77 @@
+# generated by datamodel-codegen:
+#   filename:  retraction.schema.json
+
+from __future__ import annotations
+
+from enum import Enum
+
+from pydantic import BaseModel, ConfigDict
+
+from . import _common_schema as schema
+
+
+class RetractedSubject(BaseModel):
+    """
+    Reference to the subject being retracted — the prior EvidenceBundle / in-toto Statement. At least one of bundle_id or storage_key MUST be present so the retraction is resolvable to a concrete artifact.
+    """
+
+    model_config = ConfigDict(
+        extra="forbid",
+    )
+    bundle_id: schema.Uuidv7 | None = None
+    """
+    UUIDv7 of the EvidenceBundle being retracted (FK → EvidenceBundle.id).
+    """
+    storage_key: schema.StorageKey | None = None
+    """
+    Content-addressed object-storage key of the retracted bundle payload (EvidenceBundle.storage_key).
+    """
+    content_hash: schema.Sha256Prefixed | None = None
+    """
+    sha256-prefixed digest of the retracted artifact, pinning exactly which content is retracted even if a storage backend re-keys.
+    """
+
+
+class ReasonClass(Enum):
+    """
+    CLOSED-SET reason class for the retraction (GC refusal binding — open text is rejected). Adding a value is a Class-1 ISEDC matter.
+    """
+
+    partner_request = "partner-request"
+    methodology_error = "methodology-error"
+    data_quality = "data-quality"
+    consent_withdrawn = "consent-withdrawn"
+    legal_hold = "legal-hold"
+    pre_publication_recall = "pre-publication-recall"
+
+
+class RetractionV1InTotoPredicateBodyForRetractingAPriorAttestation(BaseModel):
+    """
+    Predicate body of an in-toto Statement v1 whose predicateType is https://evals.intentsolutions.io/retraction/v1. A retraction does NOT delete the original attestation (it remains in the transparency log); it is an APPEND-ONLY signed record stating that the platform has chosen not to surface a prior subject and why. The dashboard renders a tombstone at the deep link per the retraction-protocol binding. The `reason_class` enum is CLOSED — open-text reasons are rejected by the schema (GC refusal binding). This schema validates ONLY the predicate body — the enclosing in-toto Statement envelope (_type, subject, predicateType) is validated separately. Each row independently verifiable; NO top-level bundle signature per Blueprint B § 7 line 754.
+
+    ADDITIVE in kernel v0.2.0: net-new predicate URI; no v0.1 contract changes. Per § 7.2 backward-compat, adding a predicate URI is allowed; the body normative spec lands here. Runs in sigstore_staging until production-Rekor unlock per DR-010 Q3.
+    """
+
+    model_config = ConfigDict(
+        extra="forbid",
+    )
+    retracted_subject: RetractedSubject
+    """
+    Reference to the subject being retracted — the prior EvidenceBundle / in-toto Statement. At least one of bundle_id or storage_key MUST be present so the retraction is resolvable to a concrete artifact.
+    """
+    reason_class: ReasonClass
+    """
+    CLOSED-SET reason class for the retraction (GC refusal binding — open text is rejected). Adding a value is a Class-1 ISEDC matter.
+    """
+    reason: str | None = None
+    """
+    OPTIONAL human-readable free-text elaboration. The machine-actionable retraction class is carried by reason_class; this is operator context only and MUST NOT be parsed for decisions.
+    """
+    retracted_at: schema.Rfc3339
+    """
+    RFC 3339 UTC timestamp at which the retraction took effect.
+    """
+    retracted_by: schema.ActorIdentity | None = None
+    """
+    OPTIONAL actor identity (operator email / service account) that authored the retraction, for audit-trail attribution.
+    """

intent_eval_core/_generated/rollout_gate_schema.py

@@ -0,0 +1,53 @@
+# generated by datamodel-codegen:
+#   filename:  rollout-gate.schema.json
+
+from __future__ import annotations
+
+from enum import Enum
+
+from pydantic import BaseModel, ConfigDict, conint, constr
+
+from . import _common_schema as schema
+
+
+class Decision(Enum):
+    ship = "ship"
+    no_ship = "no_ship"
+    advisory = "advisory"
+    error = "error"
+
+
+class SigningMode(Enum):
+    sigstore_staging = "sigstore_staging"
+    rekor_production = "rekor_production"
+    unsigned_experimental = "unsigned_experimental"
+
+
+class RolloutgateEvaluationOfAnEvidencebundleAgainstADeploymentPolicyBlueprintB28(
+    BaseModel
+):
+    """
+    Single terminal state `evaluated`. Immutable at creation. Re-evaluations = new rows. Natural uniqueness: (eval_run_id, policy_ref). Decision enum is DELIBERATELY DIFFERENT from gate-result/v1.gate_decision — RolloutGate uses deployment language (ship/no_ship), the predicate body uses verdict language (pass/fail). Mapping is policy-driven and lives in the downstream intent-rollout-gate package, NOT this kernel.
+    """
+
+    model_config = ConfigDict(
+        extra="forbid",
+    )
+    id: schema.Uuidv7
+    eval_run_id: schema.Uuidv7
+    evidence_bundle_id: schema.Uuidv7
+    policy_ref: constr(pattern=r"^sha256:[a-f0-9]{64}:.+$")
+    """
+    Format: <content_hash>:<path>. Defends against mid-flight policy mutation.
+    """
+    policy_content_hash: schema.Sha256
+    decision: Decision
+    decision_reasons: list[str]
+    """
+    Structured reason strings. Per Blueprint B § 2.8: 'one per matched-or-unmatched policy rule.'
+    """
+    coverage: schema.Coverage
+    evaluated_at: schema.Rfc3339
+    gate_version: schema.Semver
+    signing_mode: SigningMode
+    rekor_log_index: conint(ge=0) | None = None

intent_eval_core/_generated/runtime_receipt_schema.py

@@ -0,0 +1,99 @@
+# generated by datamodel-codegen:
+#   filename:  runtime-receipt.schema.json
+
+from __future__ import annotations
+
+from enum import Enum
+
+from pydantic import BaseModel, ConfigDict, conint, constr
+
+from . import _common_schema as schema
+
+
+class RuntimeLimitsInEffect(BaseModel):
+    """
+    Copy of EvalSpec.runtime_limits frozen at queue time.
+    """
+
+    model_config = ConfigDict(
+        extra="forbid",
+    )
+    token_ceiling: conint(ge=0)
+    wall_clock_ceiling_ms: conint(ge=0)
+    memory_ceiling_mb: conint(ge=0)
+    concurrency_hint: conint(ge=0)
+
+
+class ActualResourceUsage(BaseModel):
+    """
+    Measured resource consumption per Blueprint B § 2.6 — fully spec-bound 4-field shape.
+    """
+
+    model_config = ConfigDict(
+        extra="forbid",
+    )
+    tokens_consumed: conint(ge=0)
+    wall_clock_ms: conint(ge=0)
+    peak_memory_mb: conint(ge=0)
+    network_egress_bytes: conint(ge=0)
+
+
+class TerminalState(Enum):
+    """
+    Mirrors EvalRun.state's terminal value.
+    """
+
+    archived = "archived"
+    skipped_due_to_gate = "skipped_due_to_gate"
+    archived_failed = "archived_failed"
+
+
+class TerminalReason(Enum):
+    queued_timeout_elapsed = "queued_timeout_elapsed"
+    run_timeout_elapsed = "run_timeout_elapsed"
+    worker_crash_exhausted = "worker_crash_exhausted"
+    credential_leak_detected = "credential_leak_detected"
+    judge_unavailable_exhausted = "judge_unavailable_exhausted"
+    token_ceiling_exceeded = "token_ceiling_exceeded"
+    evidence_contract_violation = "evidence_contract_violation"
+    upstream_feed_failed = "upstream_feed_failed"
+
+
+class RuntimereceiptSignedRecordOfHowAnEvalrunExecutedBlueprintB26(BaseModel):
+    """
+    Single terminal state `issued`. Immutable at creation (signed before persistence). 1:1 cardinality with EvalRun. Predicate URI when emitted: runtime-receipt/v1 (currently sigstore_staging).
+    """
+
+    model_config = ConfigDict(
+        extra="forbid",
+    )
+    id: schema.Uuidv7
+    eval_run_id: schema.Uuidv7
+    created_at: schema.Rfc3339
+    eval_spec_content_hash: schema.Sha256
+    skill_snapshot_sha: schema.Sha256
+    provider_adapter_versions: dict[str, str]
+    """
+    Per-provider adapter version mapping. Key: provider id, Value: SemVer.
+    """
+    tool_versions: dict[str, str]
+    """
+    Per-tool version mapping. Key: tool id, Value: SemVer.
+    """
+    runtime_limits_in_effect: RuntimeLimitsInEffect
+    """
+    Copy of EvalSpec.runtime_limits frozen at queue time.
+    """
+    actual_resource_usage: ActualResourceUsage
+    """
+    Measured resource consumption per Blueprint B § 2.6 — fully spec-bound 4-field shape.
+    """
+    worker_identity: constr(min_length=1)
+    worker_host_fingerprint: schema.Sha256
+    terminal_state: TerminalState
+    """
+    Mirrors EvalRun.state's terminal value.
+    """
+    terminal_reason: TerminalReason
+    evidence_bundle_id: schema.Uuidv7
+    cost_record_id: schema.Uuidv7

intent_eval_core/_generated/session_trace_schema.py

@@ -0,0 +1,28 @@
+# generated by datamodel-codegen:
+#   filename:  session-trace.schema.json
+
+from __future__ import annotations
+
+from pydantic import BaseModel, ConfigDict, conint
+
+from . import _common_schema as schema
+
+
+class SessiontraceOtelCompatibleExecutionTraceForAnEvalrunBlueprintB210(BaseModel):
+    """
+    State machine: open → closed (closed terminal). 1:1 cardinality with EvalRun. Trace blob content-addressed (tamper-evident) in object storage.
+    """
+
+    model_config = ConfigDict(
+        extra="forbid",
+    )
+    id: schema.Uuidv7
+    eval_run_id: schema.Uuidv7
+    created_at: schema.Rfc3339
+    closed_at: schema.Rfc3339 | None = None
+    root_span_id: schema.OtelSpanId
+    total_spans: conint(ge=0)
+    max_loop_depth: conint(ge=0)
+    total_tool_invocations: conint(ge=0)
+    total_judge_decisions: conint(ge=0)
+    trace_blob_storage_key: schema.StorageKey

intent_eval_core/_generated/skill_snapshot_schema.py

@@ -0,0 +1,38 @@
+# generated by datamodel-codegen:
+#   filename:  skill-snapshot.schema.json
+
+from __future__ import annotations
+
+from pydantic import BaseModel, ConfigDict
+
+from . import _common_schema as schema
+
+
+class SkillsnapshotContentAddressedPinOfASkillSSourceDepsConfigBlueprintB29(BaseModel):
+    """
+    Single terminal state `created`. Immutable. New snapshot = new row + new combined_sha. combined_sha = sha256(source_sha || dependency_lock_sha || config_sha) per Blueprint B § 2.9 — concatenation order is NORMATIVE. Predicate URI (deferred Phase B+): skill-snapshot/v1.
+    """
+
+    model_config = ConfigDict(
+        extra="forbid",
+    )
+    id: schema.Uuidv7
+    skill_id: schema.KebabSlug
+    """
+    Logical skill identifier — stable across snapshots of the same skill.
+    """
+    source_sha: schema.Sha256
+    dependency_lock_sha: schema.Sha256
+    config_sha: schema.Sha256
+    combined_sha: schema.Sha256
+    """
+    Load-bearing pin. sha256(source_sha || dependency_lock_sha || config_sha).
+    """
+    version_label: schema.Semver | None = None
+    storage_key: schema.StorageKey
+    created_at: schema.Rfc3339
+    created_by: schema.ActorIdentity
+    tenant_id: schema.Uuidv7 | None = None
+    """
+    RESERVED multi-tenancy slot (deferral-G, bd_000-projects-k0fj). OPTIONAL + additive per § 7.2; tenant-isolation semantics deferred to a future DR. v1 single-tenant snapshots omit it.
+    """

intent_eval_core/_generated/tool_invocation_schema.py

@@ -0,0 +1,60 @@
+# generated by datamodel-codegen:
+#   filename:  tool-invocation.schema.json
+
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, conint, constr
+
+from . import _common_schema as schema
+
+
+class Error(BaseModel):
+    model_config = ConfigDict(
+        extra="forbid",
+    )
+    enum_class: schema.ErrorClass
+    """
+    Error class — a kernel-registered cross-cutting class (network.timeout, provider.rate_limited, tool.crashed, …) OR a tool-defined `<domain>.<condition>` string. Registration pattern LOCKED per deferral-E (bd_000-projects-84li): the format is constrained, the enum stays open.
+    """
+    message: str
+
+
+class ToolinvocationSingleToolProviderCallRecordedUnderASessiontraceBlueprintB211(
+    BaseModel
+):
+    """
+    Single terminal state `invoked`. Immutable at creation. Retries = new rows. HARD RULE per § 2.11: credential-shaped string in args or result_summary MUST cause runtime to reject + fail parent EvalRun with terminal_reason=credential_leak_detected (runtime concern, not enforced at this schema layer).
+    """
+
+    model_config = ConfigDict(
+        extra="forbid",
+    )
+    id: schema.Uuidv7
+    session_trace_id: schema.Uuidv7
+    parent_span_id: schema.OtelSpanId
+    tool_id: constr(min_length=1)
+    """
+    Tool identifier. Convention examples: 'audit-harness:escape-scan', 'provider:anthropic:claude-sonnet-4-5'. Format NOT enforced at this layer.
+    """
+    tool_version: schema.Semver
+    args: dict[str, Any]
+    """
+    Sanitized args (credentials redacted by runtime sanitizer).
+    """
+    args_hash: schema.Sha256
+    result_summary: dict[str, Any]
+    """
+    Sanitized result summary (inline-fit portion).
+    """
+    result_hash: schema.Sha256
+    result_storage_key: schema.StorageKey | None = None
+    invoked_at: schema.Rfc3339
+    latency_ms: conint(ge=0)
+    cost_record_ref: schema.Uuidv7
+    error: Error | None = None
+    retry_attempt: conint(ge=0)
+    """
+    0-indexed retry attempt number.
+    """

intent_eval_core/models.py

@@ -0,0 +1,180 @@
+"""Canonical Pydantic v2 model surface for ``@intentsolutions/core`` (Python).
+
+This is the **hand-authored public surface** — the Python analogue of the
+TypeScript ``src/validators/v1/*.ts`` canonical validators. It re-exports the
+``_generated/`` datamodel-code-generator output under stable canonical names
+(``GateResultV1``, ``EvalSpec``, …) and, for the schemas whose JSON Schema
+carries ``allOf``/``if-then`` conditional rules, layers a hand-written
+cross-field ``model_validator`` on top so the Python models accept/reject
+**exactly** what AJV (the JSON-Schema reference) and Zod (the TypeScript
+reference) accept/reject. No drift from the canonical kernel.
+
+The ``_generated/`` package is reference material, NOT the public API — never
+import from it directly (mirrors the Zod ``_generated/`` README discipline).
+Import from here:
+
+    from intent_eval_core import GateResultV1, EvalSpec
+    from intent_eval_core.models import GATE_RESULT_V1_URI
+
+Codegen handles the STRUCTURAL floor (fields, types, enums, ``constr`` patterns,
+``extra="forbid"`` closed-world). The cross-field rules below are the exact
+Python mirror of the Zod ``.superRefine()`` blocks in
+``src/validators/v1/gate-result-v1.ts`` — the only place where the JSON Schema's
+``allOf``/``if-then`` semantics live (datamodel-code-generator cannot emit them).
+"""
+
+from __future__ import annotations
+
+# `typing.Self` is 3.11+; import from typing_extensions (a guaranteed transitive
+# dependency of pydantic v2) so the 3.10 floor keeps working. The annotation
+# itself is a string at runtime via `from __future__ import annotations`; this
+# only guards the import statement.
+from typing_extensions import Self
+
+from pydantic import model_validator
+
+# ── Generated root models, re-exported under canonical names ──────────────────
+#
+# datamodel-code-generator derives class names from each schema's `title`, which
+# produces verbose names (e.g. EvalspecDeclarativeEvaluationSpecificationBlueprintB21).
+# We alias them to the canonical entity names used across the platform.
+
+from ._generated.cost_record_schema import (
+    CostrecordSingleCostResourceAttributionRowBlueprintB212 as CostRecord,
+)
+from ._generated.dashboard_render_schema import (
+    DashboardRenderV1InTotoPredicateBodyAttestingARenderedDashboardArtifact as DashboardRenderV1,
+)
+from ._generated.eval_run_schema import (
+    EvalrunSingleExecutionOfAnEvalspecAgainstASkillsnapshotBlueprintB22 as EvalRun,
+)
+from ._generated.eval_spec_schema import (
+    EvalspecDeclarativeEvaluationSpecificationBlueprintB21 as EvalSpec,
+)
+from ._generated.evidence_bundle_schema import (
+    EvidencebundleAppendOnlyCollectionOfSignedPredicateRowsBlueprintB24 as EvidenceBundle,
+)
+from ._generated.failure_taxonomy_schema import (
+    FailuretaxonomyCanonicalRegistryOfFailureModeClassesBlueprintB213 as FailureTaxonomy,
+)
+from ._generated.gate_result_schema import (
+    GateResultV1NormativeInTotoPredicateBody as _GateResultV1Generated,
+)
+from ._generated.judge_decision_schema import (
+    JudgedecisionSingleJudgeSVerdictOnAnEvalrunForAMatchermapBlueprintB25 as JudgeDecision,
+)
+from ._generated.matcher_map_schema import (
+    MatchermapReusableInputBehaviorPatternDefinitionBlueprintB23 as MatcherMap,
+)
+from ._generated.regression_pack_schema import (
+    RegressionpackFrozenBundleOfEvalrunsProvingABehaviorContractBlueprintB27 as RegressionPack,
+)
+from ._generated.retraction_schema import (
+    RetractionV1InTotoPredicateBodyForRetractingAPriorAttestation as RetractionV1,
+)
+from ._generated.rollout_gate_schema import (
+    RolloutgateEvaluationOfAnEvidencebundleAgainstADeploymentPolicyBlueprintB28 as RolloutGate,
+)
+from ._generated.runtime_receipt_schema import (
+    RuntimereceiptSignedRecordOfHowAnEvalrunExecutedBlueprintB26 as RuntimeReceipt,
+)
+from ._generated.session_trace_schema import (
+    SessiontraceOtelCompatibleExecutionTraceForAnEvalrunBlueprintB210 as SessionTrace,
+)
+from ._generated.skill_snapshot_schema import (
+    SkillsnapshotContentAddressedPinOfASkillSSourceDepsConfigBlueprintB29 as SkillSnapshot,
+)
+from ._generated.tool_invocation_schema import (
+    ToolinvocationSingleToolProviderCallRecordedUnderASessiontraceBlueprintB211 as ToolInvocation,
+)
+
+# ── Canonical predicate URI (CISO binding: evals.intentsolutions.io, never labs) ─
+GATE_RESULT_V1_URI = "https://evals.intentsolutions.io/gate-result/v1"
+
+
+class GateResultV1(_GateResultV1Generated):
+    """gate-result/v1 NORMATIVE predicate body (Blueprint B § 7.4).
+
+    Subclasses the generated structural model and adds the cross-field rules the
+    JSON Schema expresses via ``allOf``/``if-then`` — which
+    datamodel-code-generator cannot emit. This is the exact Python mirror of the
+    ``GateResultV1Schema.superRefine(...)`` block in
+    ``src/validators/v1/gate-result-v1.ts``:
+
+    1. ``gate_decision == "advisory"`` REQUIRES ``advisory_severity``
+       (Blueprint B § 7.4 conditional rule).
+    2. Empty ``gate_reasons`` is permitted ONLY for an unconditional ``pass`` —
+       ``fail`` / ``advisory`` / ``error`` MUST carry at least one reason
+       (Blueprint B § 7.4 line 829, ``[f-iec-validators-3]``).
+    3. Every ``coverage_detail`` entry's ``id`` MUST appear in the matching
+       ``coverage`` array (``evaluated`` -> ``dimensions_evaluated``,
+       ``skipped`` -> ``dimensions_skipped``) — deferral-D cross-field invariant
+       (``bd_000-projects-9xyk``).
+
+    The generated parent already enforces ``extra="forbid"`` (closed-world),
+    the ``gate_decision`` / ``advisory_severity`` enums, and the
+    ``policy_ref`` / ``policy_hash`` / ``input_hash`` patterns.
+    """
+
+    @model_validator(mode="after")
+    def _enforce_blueprint_b_7_4(self) -> Self:
+        decision = (
+            self.gate_decision.value
+            if hasattr(self.gate_decision, "value")
+            else self.gate_decision
+        )
+
+        # Rule 1: advisory => advisory_severity present.
+        if decision == "advisory" and self.advisory_severity is None:
+            raise ValueError(
+                'gate_decision="advisory" requires advisory_severity per '
+                "Blueprint B § 7.4 conditional rule"
+            )
+
+        # Rule 2: empty gate_reasons permitted ONLY for pass.
+        if decision in ("fail", "advisory", "error") and len(self.gate_reasons) == 0:
+            raise ValueError(
+                'gate_reasons MUST be non-empty when gate_decision is "fail", '
+                '"advisory", or "error" per Blueprint B § 7.4 line 829 '
+                "(empty permitted ONLY for pass)"
+            )
+
+        # Rule 3: coverage_detail id must be a member of the matching coverage array.
+        if self.coverage_detail is not None:
+            evaluated = set(self.coverage.dimensions_evaluated)
+            skipped = set(self.coverage.dimensions_skipped)
+            for i, detail in enumerate(self.coverage_detail):
+                status = detail.status.value if hasattr(detail.status, "value") else detail.status
+                pool = evaluated if status == "evaluated" else skipped
+                if detail.id not in pool:
+                    raise ValueError(
+                        f'coverage_detail[{i}].id "{detail.id}" (status={status}) is '
+                        f"not in coverage.dimensions_{status} (deferral-D cross-field "
+                        "invariant)"
+                    )
+
+        return self
+
+
+__all__ = [
+    # Entities (Blueprint B § 2.1 – § 2.13)
+    "EvalSpec",
+    "EvalRun",
+    "MatcherMap",
+    "EvidenceBundle",
+    "JudgeDecision",
+    "RuntimeReceipt",
+    "RegressionPack",
+    "RolloutGate",
+    "SkillSnapshot",
+    "SessionTrace",
+    "ToolInvocation",
+    "CostRecord",
+    "FailureTaxonomy",
+    # Predicate bodies
+    "GateResultV1",
+    "RetractionV1",
+    "DashboardRenderV1",
+    # Constants
+    "GATE_RESULT_V1_URI",
+]