intent-eval-core 0.7.0__py3-none-any.whl

intent_eval_core/_generated/eval_spec_schema.py

@@ -0,0 +1,218 @@
+# generated by datamodel-codegen:
+#   filename:  eval-spec.schema.json
+
+from __future__ import annotations
+
+from enum import Enum
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field, confloat, conint, constr
+
+from . import _common_schema as schema
+
+
+class Class(Enum):
+    output_equals = "output-equals"
+    output_contains = "output-contains"
+    output_matches = "output-matches"
+    schema_conforms = "schema-conforms"
+    judge_verdict = "judge-verdict"
+    matcher_satisfied = "matcher-satisfied"
+
+
+class Assertions(BaseModel):
+    model_config = ConfigDict(
+        extra="forbid",
+    )
+    class_: Class = Field(..., alias="class")
+    target: Any
+    negate: bool | None = None
+
+
+class Assertions1(BaseModel):
+    model_config = ConfigDict(
+        extra="forbid",
+    )
+    class_: str = Field(..., alias="class")
+    target: Any
+    negate: bool | None = None
+    extension: Literal[True]
+
+
+class AggregationRule(Enum):
+    majority = "majority"
+    unanimous = "unanimous"
+    weighted = "weighted"
+
+
+class WeightDimension(Enum):
+    """
+    Dimension the weighted rule distributes over (deferral-C, bd_000-projects-21re). Required when `weights` present; ignored otherwise.
+    """
+
+    matcher = "matcher"
+    mm_class = "mm-class"
+    judge = "judge"
+
+
+class Weight(BaseModel):
+    model_config = ConfigDict(
+        extra="forbid",
+    )
+    key: str
+    weight: confloat(ge=0.0)
+
+
+class Tiebreaker(Enum):
+    """
+    Deterministic resolution of an exact aggregate tie (deferral-C). NOT a threshold — it resolves a tie, it does not set a pass bar.
+    """
+
+    fail_closed = "fail-closed"
+    fail_open = "fail-open"
+    first_listed = "first-listed"
+
+
+class Scoring(BaseModel):
+    """
+    Scoring configuration. aggregation_rule is the spec-bound field per Blueprint B § 2.1. weight_dimension/weights/tiebreaker are the deferral-C lockup (bd_000-projects-21re), additive + optional. § 7.6 architectural rule: ship/no-ship THRESHOLDS belong in consumer-side tests/TESTING.md, NOT here — only weights + exact-tie resolution are kernel-side.
+    """
+
+    aggregation_rule: AggregationRule
+    weight_dimension: WeightDimension | None = None
+    """
+    Dimension the weighted rule distributes over (deferral-C, bd_000-projects-21re). Required when `weights` present; ignored otherwise.
+    """
+    weights: list[Weight] | None = None
+    """
+    Per-unit weights for the weighted rule (deferral-C). Normalized by their sum at scoring time.
+    """
+    tiebreaker: Tiebreaker | None = None
+    """
+    Deterministic resolution of an exact aggregate tie (deferral-C). NOT a threshold — it resolves a tie, it does not set a pass bar.
+    """
+    extensions: dict[str, Any] | None = None
+    """
+    Tool-specific scoring metadata. MUST NOT drive ship/no-ship decisions (mirrors § 7.4 metadata rule).
+    """
+
+
+class Kind(Enum):
+    """
+    Node type per Blueprint B § 1.3 line 92.
+    """
+
+    eval_run = "eval_run"
+    tool_invocation = "tool_invocation"
+
+
+class Node(BaseModel):
+    model_config = ConfigDict(
+        extra="forbid",
+    )
+    id: str
+    """
+    Local-to-DAG node identifier (unique within composition).
+    """
+    kind: Kind
+    """
+    Node type per Blueprint B § 1.3 line 92.
+    """
+    ref: schema.Uuidv7
+
+
+class Kind1(Enum):
+    """
+    Edge type — drives runtime failure-propagation per Blueprint B § 1.3 line 100.
+    """
+
+    feeds = "feeds"
+    gates = "gates"
+    enriches = "enriches"
+
+
+class Edge(BaseModel):
+    model_config = ConfigDict(
+        extra="forbid",
+    )
+    from_: str = Field(..., alias="from")
+    to: str
+    kind: Kind1
+    """
+    Edge type — drives runtime failure-propagation per Blueprint B § 1.3 line 100.
+    """
+
+
+class Composition(BaseModel):
+    """
+    DAG declaration per Blueprint B § 1.3. Validation contract: runtime topologically sorts; cycles rejected at submission with 400. Wire format (adjacency list with typed edges) is engineer's choice per bd_000-projects-3sjx; semantics ARE spec-bound (node types + edge kinds + failure propagation per § 1.3 line 100).
+    """
+
+    model_config = ConfigDict(
+        extra="forbid",
+    )
+    nodes: list[Node]
+    edges: list[Edge]
+
+
+class RuntimeLimits(BaseModel):
+    """
+    Resource ceilings applied per EvalRun per Blueprint B § 2.1.
+    """
+
+    model_config = ConfigDict(
+        extra="forbid",
+    )
+    token_ceiling: conint(ge=0)
+    wall_clock_ceiling_ms: conint(ge=0)
+    memory_ceiling_mb: conint(ge=0)
+    concurrency_hint: conint(ge=0)
+
+
+class EvalspecDeclarativeEvaluationSpecificationBlueprintB21(BaseModel):
+    """
+    Content-addressed via content_hash. State machine: draft → published → deprecated (deprecated reversible to published). Mutable in draft; immutable once published.
+    """
+
+    model_config = ConfigDict(
+        extra="forbid",
+    )
+    id: schema.Uuidv7
+    version: schema.Semver
+    name: schema.KebabSlug
+    description: constr(min_length=1)
+    matchers: list[schema.Uuidv7]
+    """
+    MatcherMap FKs.
+    """
+    assertions: list[Assertions | Assertions1]
+    """
+    Typed assertion expressions. Class union LOCKED additively per deferral-A (bd_000-projects-gzgj): a named-class discriminated union plus an open extension slot. Per-class `target` payload stays open (refined at the validator layer per engagement).
+    """
+    scoring: Scoring
+    """
+    Scoring configuration. aggregation_rule is the spec-bound field per Blueprint B § 2.1. weight_dimension/weights/tiebreaker are the deferral-C lockup (bd_000-projects-21re), additive + optional. § 7.6 architectural rule: ship/no-ship THRESHOLDS belong in consumer-side tests/TESTING.md, NOT here — only weights + exact-tie resolution are kernel-side.
+    """
+    composition: Composition
+    """
+    DAG declaration per Blueprint B § 1.3. Validation contract: runtime topologically sorts; cycles rejected at submission with 400. Wire format (adjacency list with typed edges) is engineer's choice per bd_000-projects-3sjx; semantics ARE spec-bound (node types + edge kinds + failure propagation per § 1.3 line 100).
+    """
+    expected_artifacts: list[schema.Sha256]
+    """
+    SkillSnapshot.combined_sha values this spec targets. Anchors on CONTENT (sha), not FK — rebuilt snapshots with the same combined_sha still satisfy.
+    """
+    runtime_limits: RuntimeLimits
+    """
+    Resource ceilings applied per EvalRun per Blueprint B § 2.1.
+    """
+    provider_constraints: list[str]
+    """
+    Allowlist of provider IDs.
+    """
+    created_at: schema.Rfc3339
+    created_by: schema.ActorIdentity
+    content_hash: schema.Sha256
+    tenant_id: schema.Uuidv7 | None = None
+    """
+    RESERVED multi-tenancy slot (deferral-G, bd_000-projects-k0fj). OPTIONAL + additive per § 7.2; tenant-isolation semantics deferred to a future DR. v1 single-tenant specs omit it.
+    """

intent_eval_core/_generated/evidence_bundle_schema.py

@@ -0,0 +1,78 @@
+# generated by datamodel-codegen:
+#   filename:  evidence-bundle.schema.json
+
+from __future__ import annotations
+
+from enum import Enum
+
+from pydantic import AnyUrl, BaseModel, ConfigDict, conint
+
+from . import _common_schema as schema
+
+
+class Digest(BaseModel):
+    model_config = ConfigDict(
+        extra="allow",
+    )
+    sha256: schema.Sha256
+
+
+class SubjectSetItem(BaseModel):
+    model_config = ConfigDict(
+        extra="forbid",
+    )
+    name: schema.SubjectName
+    digest: Digest
+
+
+class SigningMode(Enum):
+    """
+    Signing posture. sigstore_staging is DR-010 Q3 default for predicates without normative SPEC.md (everything except gate-result/v1 at v1).
+    """
+
+    sigstore_staging = "sigstore_staging"
+    rekor_production = "rekor_production"
+    unsigned_experimental = "unsigned_experimental"
+
+
+class VerificationStatus(Enum):
+    verified = "verified"
+    unverified = "unverified"
+    failed = "failed"
+
+
+class EvidencebundleAppendOnlyCollectionOfSignedPredicateRowsBlueprintB24(BaseModel):
+    """
+    State machine: building → signing → signed → archived_to_rekor. APPEND-ONLY. Corrections = new bundle with new UUIDv7 referencing prior content_hash.
+    """
+
+    model_config = ConfigDict(
+        extra="forbid",
+    )
+    id: schema.Uuidv7
+    eval_run_id: schema.Uuidv7
+    created_at: schema.Rfc3339
+    predicate_uri_set: list[AnyUrl]
+    """
+    Predicate URIs represented in this bundle's rows.
+    """
+    row_count: conint(ge=0)
+    subject_set: list[SubjectSetItem]
+    """
+    Deduplicated in-toto Subject entries across all rows.
+    """
+    storage_key: schema.StorageKey
+    signing_mode: SigningMode
+    """
+    Signing posture. sigstore_staging is DR-010 Q3 default for predicates without normative SPEC.md (everything except gate-result/v1 at v1).
+    """
+    rekor_log_indices: list[conint(ge=0)]
+    """
+    Populated when signing_mode='rekor_production'; empty otherwise.
+    """
+    verification_status: VerificationStatus
+    verification_last_checked_at: schema.Rfc3339
+    pre_registration_hash: schema.Sha256Prefixed | None = None
+    """
+    Pre-registration commitment hash (D2 binding, v0.2.0 additive). `sha256:<hex>` digest of the pre-registration commitment artifact when the run was pre-registered, or null when it was not. ADDITIVE + OPTIONAL — every v0.1.0 bundle stays valid (absent ≡ null). Cross-field invariants governing when this MUST be non-null land in iec-E12; this is the field surface only.
+    """

intent_eval_core/_generated/failure_taxonomy_schema.py

@@ -0,0 +1,53 @@
+# generated by datamodel-codegen:
+#   filename:  failure-taxonomy.schema.json
+
+from __future__ import annotations
+
+from enum import Enum
+
+from pydantic import BaseModel, ConfigDict, constr
+
+from . import _common_schema as schema
+
+
+class Example(BaseModel):
+    model_config = ConfigDict(
+        extra="forbid",
+    )
+    ref: str
+    """
+    URL, gist, doc path, EvalRun id, etc.
+    """
+    description: str | None = None
+
+
+class Status(Enum):
+    proposed = "proposed"
+    canonical = "canonical"
+    deprecated = "deprecated"
+
+
+class FailuretaxonomyCanonicalRegistryOfFailureModeClassesBlueprintB213(BaseModel):
+    """
+    State machine: proposed → canonical → deprecated (forward-only). canonical → deprecated requires Class-1 ISEDC DR. JudgeDecision with verdict='FAIL' MUST classify against an entry here. Drift emits `taxonomy.drift.detected` OTel event. Source of truth for what MM-N classes exist — MatcherMap.mm_class kernel enum is DOWNSTREAM of this entity (proposed → canonical → MmClass enum bump).
+    """
+
+    model_config = ConfigDict(
+        extra="forbid",
+    )
+    id: schema.Uuidv7
+    mm_class: schema.MmClassId
+    """
+    Natural key. Accepts any MM-N value (broader than MmClass enum).
+    """
+    name: schema.KebabSlug
+    description: constr(min_length=1)
+    discriminating_question: constr(min_length=1)
+    """
+    The question a human asks to decide whether a given failure belongs in this class. Drives the classification UI.
+    """
+    examples: list[Example]
+    version: schema.Semver
+    status: Status
+    created_at: schema.Rfc3339
+    created_by: schema.ActorIdentity

intent_eval_core/_generated/gate_result_schema.py

@@ -0,0 +1,116 @@
+# generated by datamodel-codegen:
+#   filename:  gate-result.schema.json
+
+from __future__ import annotations
+
+from enum import Enum
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, constr
+
+from . import _common_schema as schema
+
+
+class GateDecision(Enum):
+    """
+    Decision verdict (Blueprint B § 7.4 line 800, closed 4-element enum). Distinct from JudgeVerdict (UPPERCASE) and RolloutGateDecision (ship/no_ship).
+    """
+
+    pass_ = "pass"
+    fail = "fail"
+    advisory = "advisory"
+    error = "error"
+
+
+class AdvisorySeverity(Enum):
+    """
+    Severity hint when gate_decision='advisory'. Consumer policy decides whether to elevate to blocking.
+    """
+
+    info = "info"
+    warn = "warn"
+    error = "error"
+
+
+class ReplayFidelityLevel(Enum):
+    """
+    Replay fidelity level per iel-E11.
+    """
+
+    RF_0 = "RF-0"
+    RF_1 = "RF-1"
+    RF_2 = "RF-2"
+    RF_3 = "RF-3"
+    RF_4 = "RF-4"
+
+
+class GateResultV1NormativeInTotoPredicateBody(BaseModel):
+    """
+    Predicate body of an in-toto Statement v1 whose predicateType is https://evals.intentsolutions.io/gate-result/v1. Source of truth: Blueprint B § 7.4 (lines 794-834). The ONLY predicate body fully spec-bound at v1; every other predicate URI runs in sigstore_staging until its SPEC.md normative section lands per DR-010 Q3. This schema validates ONLY the predicate body — the enclosing in-toto Statement envelope (_type, subject, predicateType) is validated separately. Each row independently verifiable; NO top-level bundle signature per § 7 line 754. Adding/loosening any field requires Class-1 ISEDC convening.
+
+    Migration note: this schema SUPERSEDES the lab's v0.1.0-draft schema (which used `result`/`timestamp` and was missing gate_name/gate_version/gate_reasons/coverage/policy_ref). Blueprint B § 7.4 is the canonical source — the lab schema was authored before Blueprint B landed on main.
+    """
+
+    model_config = ConfigDict(
+        extra="forbid",
+    )
+    gate_id: schema.SubjectName
+    """
+    Pipeline-hop-qualified gate identifier. MUST match the in-toto subject `name` per § 7.3.
+    """
+    gate_name: schema.KebabSlug
+    """
+    Short human-readable kebab-case gate name.
+    """
+    gate_version: schema.Semver
+    """
+    SemVer version of the gate logic that produced this row.
+    """
+    gate_decision: GateDecision
+    """
+    Decision verdict (Blueprint B § 7.4 line 800, closed 4-element enum). Distinct from JudgeVerdict (UPPERCASE) and RolloutGateDecision (ship/no_ship).
+    """
+    gate_reasons: list[str]
+    """
+    Structured reason strings. Empty array permitted ONLY for unconditional pass. For fail/advisory/error, at least one entry MUST appear; for error, gate_reasons[0] MUST capture the error class per § 7.4 line 829.
+    """
+    coverage: schema.Coverage
+    policy_ref: constr(pattern=r"^sha256:[a-f0-9]{64}:.+$")
+    """
+    Format: `sha256:<hex>:<repo-relative-path>`. Defends against mid-flight policy mutation per § 7.4 line 810.
+    """
+    policy_hash: schema.Sha256Prefixed
+    """
+    sha256-prefixed hash of the policy file content.
+    """
+    input_hash: schema.Sha256Prefixed
+    """
+    sha256-prefixed hash of the input the gate evaluated. MUST equal the in-toto subject[].digest.sha256 for the row per § 7.4 line 814 + § 7.3 line 792.
+    """
+    evaluated_at: schema.Rfc3339
+    runner: schema.RunnerIdentifier
+    commit_sha: schema.CommitSha
+    metadata: dict[str, Any] | None = None
+    """
+    Tool-specific free-form metadata. Consumers MUST NOT use this for ship/no-ship decisions per § 7.4 line 818.
+    """
+    failure_mode: str | None = None
+    """
+    Failure-mode classifier. MUST be populated when gate_decision='fail' AND the originating tool defines failure modes (e.g., MM-class for audit-harness gates).
+    """
+    advisory_severity: AdvisorySeverity | None = None
+    """
+    Severity hint when gate_decision='advisory'. Consumer policy decides whether to elevate to blocking.
+    """
+    cost_record_ref: schema.Uuidv7 | None = None
+    """
+    FK → CostRecord.id for cost attribution.
+    """
+    replay_fidelity_level: ReplayFidelityLevel | None = None
+    """
+    Replay fidelity level per iel-E11.
+    """
+    coverage_detail: list[schema.CoverageDimensionDetail] | None = None
+    """
+    Richer per-dimension coverage metadata (deferral-D lockup, bd_000-projects-9xyk). OPTIONAL + additive — the required `coverage` string arrays are unchanged, so the predicate URI is unaffected per § 7.2. Each entry's id MUST be a member of the matching `coverage` array (enforced at the validator layer). Descriptive only; MUST NOT drive ship/no-ship.
+    """

intent_eval_core/_generated/judge_decision_schema.py

@@ -0,0 +1,76 @@
+# generated by datamodel-codegen:
+#   filename:  judge-decision.schema.json
+
+from __future__ import annotations
+
+from enum import Enum
+
+from pydantic import BaseModel, ConfigDict, confloat, conint, constr
+
+from . import _common_schema as schema
+
+
+class Verdict(Enum):
+    """
+    Verdict enum per Blueprint B § 2.5 (closed 5-element set). UPPERCASE — deliberately DISTINCT from GateDecision (lowercase).
+    """
+
+    PASS = "PASS"
+    FAIL = "FAIL"
+    ADVISORY = "ADVISORY"
+    NOT_APPLICABLE = "NOT_APPLICABLE"
+    ERROR = "ERROR"
+
+
+class VerdictSource(Enum):
+    """
+    How the verdict was reached.
+    """
+
+    deterministic = "deterministic"
+    llm_with_seed = "llm_with_seed"
+    llm_no_seed = "llm_no_seed"
+    hybrid = "hybrid"
+
+
+class JudgedecisionSingleJudgeSVerdictOnAnEvalrunForAMatchermapBlueprintB25(BaseModel):
+    """
+    Single terminal state `recorded`. Immutable at creation. Follow-up verdicts = new rows. When emitted as a signed predicate row, the URI is eval-verdict/v1 (currently sigstore_staging).
+    """
+
+    model_config = ConfigDict(
+        extra="forbid",
+    )
+    id: schema.Uuidv7
+    eval_run_id: schema.Uuidv7
+    session_trace_id: schema.Uuidv7
+    matcher_map_id: schema.Uuidv7
+    judge_identity: constr(min_length=1)
+    """
+    Stable judge identity, e.g., `audit-harness@0.3.0:escape-scan`.
+    """
+    judge_version: schema.Semver
+    verdict: Verdict
+    """
+    Verdict enum per Blueprint B § 2.5 (closed 5-element set). UPPERCASE — deliberately DISTINCT from GateDecision (lowercase).
+    """
+    verdict_source: VerdictSource
+    """
+    How the verdict was reached.
+    """
+    confidence: confloat(ge=0.0, le=1.0) | None = None
+    """
+    Populated for LLM judges; null for deterministic judges.
+    """
+    reasoning: str | None = None
+    """
+    Free-form judge reasoning output.
+    """
+    input_hash: schema.Sha256
+    seed: int | str | None = None
+    """
+    Populated for verdict_source='llm_with_seed'. Blueprint B does not specify integer vs string — both accepted.
+    """
+    evaluated_at: schema.Rfc3339
+    latency_ms: conint(ge=0)
+    cost_record_ref: schema.Uuidv7

intent_eval_core/_generated/matcher_map_schema.py

@@ -0,0 +1,153 @@
+# generated by datamodel-codegen:
+#   filename:  matcher-map.schema.json
+
+from __future__ import annotations
+
+from enum import Enum
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field, constr
+
+from . import _common_schema as schema
+
+
+class InputPattern(BaseModel):
+    """
+    Typed input pattern — closed 3-variant discriminated union per Blueprint B § 2.3 line 159.
+    """
+
+    model_config = ConfigDict(
+        extra="forbid",
+    )
+    kind: Literal["regex"]
+    pattern: str
+    flags: str | None = None
+
+
+class InputPattern1(BaseModel):
+    """
+    Typed input pattern — closed 3-variant discriminated union per Blueprint B § 2.3 line 159.
+    """
+
+    model_config = ConfigDict(
+        extra="forbid",
+    )
+    kind: Literal["json-schema"]
+    schema_: dict[str, Any] = Field(..., alias="schema")
+
+
+class Mode(Enum):
+    """
+    v1 fixes `all` (every constraint must hold); reserved for a future additive `any`.
+    """
+
+    all = "all"
+
+
+class Op(Enum):
+    exists = "exists"
+    absent = "absent"
+    equals = "equals"
+    type_is = "type-is"
+    matches = "matches"
+    length_equals = "length-equals"
+
+
+class Constraint(BaseModel):
+    model_config = ConfigDict(
+        extra="forbid",
+    )
+    path: constr(min_length=1)
+    """
+    Dotted/bracketed accessor into the candidate's parsed tree. Accessor grammar is the runtime's concern.
+    """
+    op: Op
+    value: Any | None = None
+    """
+    Required for value-bearing ops (equals/type-is/matches/length-equals); omitted for presence ops (exists/absent). Cross-field enforced at the validator layer.
+    """
+
+
+class Matcher(BaseModel):
+    """
+    Structural-matcher payload — LOCKED per deferral-B (bd_000-projects-ra9a). A conjunction of path-anchored constraints over the candidate's parsed tree.
+    """
+
+    model_config = ConfigDict(
+        extra="forbid",
+    )
+    mode: Mode
+    """
+    v1 fixes `all` (every constraint must hold); reserved for a future additive `any`.
+    """
+    constraints: list[Constraint]
+
+
+class InputPattern2(BaseModel):
+    """
+    Typed input pattern — closed 3-variant discriminated union per Blueprint B § 2.3 line 159.
+    """
+
+    model_config = ConfigDict(
+        extra="forbid",
+    )
+    kind: Literal["structural"]
+    matcher: Matcher
+    """
+    Structural-matcher payload — LOCKED per deferral-B (bd_000-projects-ra9a). A conjunction of path-anchored constraints over the candidate's parsed tree.
+    """
+
+
+class Kind(Enum):
+    exact = "exact"
+    semantic = "semantic"
+    contract_conformance = "contract-conformance"
+    redaction_confirmed = "redaction-confirmed"
+
+
+class ExpectedBehavior(BaseModel):
+    """
+    Typed expected behavior. v1 named variants + open extension slot per spec's explicit 'etc.' trailer.
+    """
+
+    kind: Kind
+    payload: Any | None = None
+
+
+class ExpectedBehavior1(BaseModel):
+    """
+    Typed expected behavior. v1 named variants + open extension slot per spec's explicit 'etc.' trailer.
+    """
+
+    model_config = ConfigDict(
+        extra="allow",
+    )
+    kind: str
+    payload: Any
+    extension: Literal[True]
+
+
+class MatchermapReusableInputBehaviorPatternDefinitionBlueprintB23(BaseModel):
+    """
+    State machine: draft → published → deprecated (one-way). Immutable once published; revisions = new row + incremented version.
+    """
+
+    model_config = ConfigDict(
+        extra="forbid",
+    )
+    id: schema.Uuidv7
+    mm_class: schema.MmClass
+    name: schema.KebabSlug
+    input_pattern: InputPattern | InputPattern1 | InputPattern2
+    """
+    Typed input pattern — closed 3-variant discriminated union per Blueprint B § 2.3 line 159.
+    """
+    expected_behavior: ExpectedBehavior | ExpectedBehavior1
+    """
+    Typed expected behavior. v1 named variants + open extension slot per spec's explicit 'etc.' trailer.
+    """
+    version: schema.Semver
+    content_hash: schema.Sha256
+    description: constr(min_length=1)
+    created_at: schema.Rfc3339
+    created_by: schema.ActorIdentity