intent-eval-core 0.7.0__tar.gz → 0.9.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {intent_eval_core-0.7.0 → intent_eval_core-0.9.0}/.gitignore +6 -0
- {intent_eval_core-0.7.0 → intent_eval_core-0.9.0}/PKG-INFO +2 -2
- {intent_eval_core-0.7.0 → intent_eval_core-0.9.0}/README.md +1 -1
- {intent_eval_core-0.7.0 → intent_eval_core-0.9.0}/pyproject.toml +1 -1
- {intent_eval_core-0.7.0 → intent_eval_core-0.9.0}/src/intent_eval_core/__init__.py +11 -2
- intent_eval_core-0.9.0/src/intent_eval_core/_generated/human_review_schema.py +69 -0
- intent_eval_core-0.9.0/src/intent_eval_core/_generated/skill_refiner_pass_schema.py +154 -0
- intent_eval_core-0.9.0/src/intent_eval_core/_generated/skill_version_schema.py +70 -0
- intent_eval_core-0.9.0/src/intent_eval_core/_generated/usage_event_schema.py +93 -0
- intent_eval_core-0.9.0/src/intent_eval_core/models.py +464 -0
- intent_eval_core-0.9.0/tests/test_parity.py +577 -0
- intent_eval_core-0.7.0/src/intent_eval_core/models.py +0 -180
- intent_eval_core-0.7.0/tests/test_parity.py +0 -284
- {intent_eval_core-0.7.0 → intent_eval_core-0.9.0}/LICENSE +0 -0
- {intent_eval_core-0.7.0 → intent_eval_core-0.9.0}/NOTICE +0 -0
- {intent_eval_core-0.7.0 → intent_eval_core-0.9.0}/src/intent_eval_core/_generated/README.md +0 -0
- {intent_eval_core-0.7.0 → intent_eval_core-0.9.0}/src/intent_eval_core/_generated/__init__.py +0 -0
- {intent_eval_core-0.7.0 → intent_eval_core-0.9.0}/src/intent_eval_core/_generated/_common_schema.py +0 -0
- {intent_eval_core-0.7.0 → intent_eval_core-0.9.0}/src/intent_eval_core/_generated/cost_record_schema.py +0 -0
- {intent_eval_core-0.7.0 → intent_eval_core-0.9.0}/src/intent_eval_core/_generated/dashboard_render_schema.py +0 -0
- {intent_eval_core-0.7.0 → intent_eval_core-0.9.0}/src/intent_eval_core/_generated/eval_run_schema.py +0 -0
- {intent_eval_core-0.7.0 → intent_eval_core-0.9.0}/src/intent_eval_core/_generated/eval_spec_schema.py +0 -0
- {intent_eval_core-0.7.0 → intent_eval_core-0.9.0}/src/intent_eval_core/_generated/evidence_bundle_schema.py +0 -0
- {intent_eval_core-0.7.0 → intent_eval_core-0.9.0}/src/intent_eval_core/_generated/failure_taxonomy_schema.py +0 -0
- {intent_eval_core-0.7.0 → intent_eval_core-0.9.0}/src/intent_eval_core/_generated/gate_result_schema.py +0 -0
- {intent_eval_core-0.7.0 → intent_eval_core-0.9.0}/src/intent_eval_core/_generated/judge_decision_schema.py +0 -0
- {intent_eval_core-0.7.0 → intent_eval_core-0.9.0}/src/intent_eval_core/_generated/matcher_map_schema.py +0 -0
- {intent_eval_core-0.7.0 → intent_eval_core-0.9.0}/src/intent_eval_core/_generated/regression_pack_schema.py +0 -0
- {intent_eval_core-0.7.0 → intent_eval_core-0.9.0}/src/intent_eval_core/_generated/retraction_schema.py +0 -0
- {intent_eval_core-0.7.0 → intent_eval_core-0.9.0}/src/intent_eval_core/_generated/rollout_gate_schema.py +0 -0
- {intent_eval_core-0.7.0 → intent_eval_core-0.9.0}/src/intent_eval_core/_generated/runtime_receipt_schema.py +0 -0
- {intent_eval_core-0.7.0 → intent_eval_core-0.9.0}/src/intent_eval_core/_generated/session_trace_schema.py +0 -0
- {intent_eval_core-0.7.0 → intent_eval_core-0.9.0}/src/intent_eval_core/_generated/skill_snapshot_schema.py +0 -0
- {intent_eval_core-0.7.0 → intent_eval_core-0.9.0}/src/intent_eval_core/_generated/tool_invocation_schema.py +0 -0
- {intent_eval_core-0.7.0 → intent_eval_core-0.9.0}/src/intent_eval_core/py.typed +0 -0
- {intent_eval_core-0.7.0 → intent_eval_core-0.9.0}/tests/test_codegen_idempotent.py +0 -0
- {intent_eval_core-0.7.0 → intent_eval_core-0.9.0}/tests/test_version_lockstep.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: intent-eval-core
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.9.0
|
|
4
4
|
Summary: Python (Pydantic v2) distribution of @intentsolutions/core — the canonical contracts kernel for the Intent Eval Platform. The same entity + predicate contracts the TypeScript package exposes as Zod validators + JSON Schemas, as Pydantic v2 models.
|
|
5
5
|
Project-URL: Homepage, https://github.com/jeremylongshore/intent-eval-core
|
|
6
6
|
Project-URL: Repository, https://github.com/jeremylongshore/intent-eval-core
|
|
@@ -80,7 +80,7 @@ print(gate.gate_decision) # GateDecision.pass_
|
|
|
80
80
|
|
|
81
81
|
## What's exported
|
|
82
82
|
|
|
83
|
-
|
|
83
|
+
15 canonical entities + 3 predicate bodies (Blueprint B § 2 / § 7 + SkillVersion DR-028 T1 + UsageEvent DR-103 D1):
|
|
84
84
|
|
|
85
85
|
| Entity models | Predicate-body models |
|
|
86
86
|
| --- | --- |
|
|
@@ -47,7 +47,7 @@ print(gate.gate_decision) # GateDecision.pass_
|
|
|
47
47
|
|
|
48
48
|
## What's exported
|
|
49
49
|
|
|
50
|
-
|
|
50
|
+
15 canonical entities + 3 predicate bodies (Blueprint B § 2 / § 7 + SkillVersion DR-028 T1 + UsageEvent DR-103 D1):
|
|
51
51
|
|
|
52
52
|
| Entity models | Predicate-body models |
|
|
53
53
|
| --- | --- |
|
|
@@ -8,7 +8,7 @@ name = "intent-eval-core"
|
|
|
8
8
|
# Schemas under schemas/v1/ are the shared source of truth for both
|
|
9
9
|
# distributions, so the versions move together. CI gate:
|
|
10
10
|
# scripts/check_version_lockstep.py.
|
|
11
|
-
version = "0.
|
|
11
|
+
version = "0.9.0"
|
|
12
12
|
description = "Python (Pydantic v2) distribution of @intentsolutions/core — the canonical contracts kernel for the Intent Eval Platform. The same entity + predicate contracts the TypeScript package exposes as Zod validators + JSON Schemas, as Pydantic v2 models."
|
|
13
13
|
readme = "README.md"
|
|
14
14
|
requires-python = ">=3.10"
|
|
@@ -31,6 +31,7 @@ from .models import (
|
|
|
31
31
|
EvidenceBundle,
|
|
32
32
|
FailureTaxonomy,
|
|
33
33
|
GateResultV1,
|
|
34
|
+
HumanReview,
|
|
34
35
|
JudgeDecision,
|
|
35
36
|
MatcherMap,
|
|
36
37
|
RegressionPack,
|
|
@@ -38,17 +39,21 @@ from .models import (
|
|
|
38
39
|
RolloutGate,
|
|
39
40
|
RuntimeReceipt,
|
|
40
41
|
SessionTrace,
|
|
42
|
+
SkillRefinerPassV1,
|
|
41
43
|
SkillSnapshot,
|
|
44
|
+
SkillVersion,
|
|
42
45
|
ToolInvocation,
|
|
46
|
+
UsageEvent,
|
|
43
47
|
)
|
|
44
48
|
|
|
45
49
|
# Kept in lockstep with npm package.json#version by scripts/check_version_lockstep.py
|
|
46
50
|
# (CI-gated). The JSON Schemas under schemas/v1/ are the shared source of truth.
|
|
47
|
-
__version__ = "0.
|
|
51
|
+
__version__ = "0.9.0"
|
|
48
52
|
|
|
49
53
|
__all__ = [
|
|
50
54
|
"__version__",
|
|
51
|
-
# Entities (Blueprint B § 2.1 – § 2.13
|
|
55
|
+
# Entities (Blueprint B § 2.1 – § 2.13 + SkillVersion per DR-028 T1
|
|
56
|
+
# + UsageEvent + HumanReview per ISEDC DR-103 D1)
|
|
52
57
|
"EvalSpec",
|
|
53
58
|
"EvalRun",
|
|
54
59
|
"MatcherMap",
|
|
@@ -58,14 +63,18 @@ __all__ = [
|
|
|
58
63
|
"RegressionPack",
|
|
59
64
|
"RolloutGate",
|
|
60
65
|
"SkillSnapshot",
|
|
66
|
+
"SkillVersion",
|
|
67
|
+
"HumanReview",
|
|
61
68
|
"SessionTrace",
|
|
62
69
|
"ToolInvocation",
|
|
63
70
|
"CostRecord",
|
|
71
|
+
"UsageEvent",
|
|
64
72
|
"FailureTaxonomy",
|
|
65
73
|
# Predicate bodies
|
|
66
74
|
"GateResultV1",
|
|
67
75
|
"RetractionV1",
|
|
68
76
|
"DashboardRenderV1",
|
|
77
|
+
"SkillRefinerPassV1",
|
|
69
78
|
# Constants
|
|
70
79
|
"GATE_RESULT_V1_URI",
|
|
71
80
|
]
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
# generated by datamodel-codegen:
|
|
2
|
+
# filename: human-review.schema.json
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
from pydantic import BaseModel, ConfigDict
|
|
7
|
+
|
|
8
|
+
from . import _common_schema as schema
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class HumanreviewOpenEndedHumanTrustSignalOnAnEvalrunNetNewCanonicalEntityIsedcDr103D1(
|
|
12
|
+
BaseModel
|
|
13
|
+
):
|
|
14
|
+
"""
|
|
15
|
+
A single human's open-ended assessment of an EvalRun — the human counterpart to JudgeDecision (whose machine verdict is the `eval-verdict/v1` predicate, NOT `judge-decision/v1`). Three ORTHOGONAL nullable channels capture the Langfuse-style signal set without crushing them into a scalar: `score_text` (open-ended NON-COMPARABLE TEXT, intentionally not an enum/number — ISEDC DR-103 C3 B6.3), `thumbs` (boolean), `annotation` (free text). Append-only ledger; single terminal state `recorded`; a revision is a NEW row citing `supersedes_id`, never a mutation. ANTI-GAMING (ISEDC DR-103 D1 B1.2, machine-enforced): a row MUST pin to a verified source — `session_trace_id` non-null OR `judge_decision_id` non-null (a lone thumb pinned to nothing is refused), AND a service-account-authored row is refused (`reviewer_is_service_account` MUST be false — the `human_reviews` ledger is a HUMAN-only signal), AND at least one of the three signal channels MUST be present. ADDITIVE / one-way-door per ISEDC DR-103 D1: this shape ships in signed @intentsolutions/core entries, so widening or merging it post-publish is irreversible — additive only. `tenant_id` is OPTIONAL-NOT-NULLABLE (DR-085 D5 / deferral-G — byte-identical to EvalSpec/EvalRun/SkillSnapshot/SkillVersion). The non-reproducible human-trust signal does NOT ride the GateResultV1-pinned EvidenceStatement; it gets its own parallel additive HumanReviewStatement (binds subject[0].digest.sha256 === input_hash only) and the `human-review/v1` predicate, which is PERMANENTLY sigstore_staging by design (ISEDC DR-103 D3 B3.2 — a human's open-ended TEXT assessment can never meet rekor_production's reproduce-the-hash bar).
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
model_config = ConfigDict(
|
|
19
|
+
extra="forbid",
|
|
20
|
+
)
|
|
21
|
+
id: schema.Uuidv7
|
|
22
|
+
"""
|
|
23
|
+
UUIDv7 PK. The `human_review_id` the human-review/v1 predicate body references.
|
|
24
|
+
"""
|
|
25
|
+
eval_run_id: schema.Uuidv7
|
|
26
|
+
"""
|
|
27
|
+
FK -> EvalRun.id (required). The EvalRun this human is assessing.
|
|
28
|
+
"""
|
|
29
|
+
session_trace_id: schema.Uuidv7 | None = None
|
|
30
|
+
"""
|
|
31
|
+
FK -> SessionTrace.id, NULLABLE. ANTI-GAMING (DR-103 D1 B1.2): a review MUST pin to a verified source — this OR `judge_decision_id` MUST be non-null. A review pinned to nothing is forgery-cost-zero and is refused at all three layers.
|
|
32
|
+
"""
|
|
33
|
+
judge_decision_id: schema.Uuidv7 | None = None
|
|
34
|
+
"""
|
|
35
|
+
FK -> JudgeDecision.id, NULLABLE. The machine verdict the human is agreeing-with / overriding (JudgeDecision's signed predicate is `eval-verdict/v1`). ANTI-GAMING (DR-103 D1 B1.2): this OR `session_trace_id` MUST be non-null (pin to something verified).
|
|
36
|
+
"""
|
|
37
|
+
supersedes_id: schema.Uuidv7 | None = None
|
|
38
|
+
"""
|
|
39
|
+
Convenience pointer to the prior HumanReview this row revises (HumanReview -> HumanReview). `null` for the first review. No record is ever mutated — a revision is a NEW row citing the prior (append-only ledger).
|
|
40
|
+
"""
|
|
41
|
+
reviewer_identity: schema.ActorIdentity
|
|
42
|
+
"""
|
|
43
|
+
Identity of the human reviewer (engineer email, account name). The human-trust signal's authorship anchor. DR-103 D3 B3.2 names verified reviewer identity + the pinned session as `human-review/v1`'s REAL trust criterion (the predicate being permanently staging is by design, NOT a half-finished state).
|
|
44
|
+
"""
|
|
45
|
+
reviewer_is_service_account: Literal[False]
|
|
46
|
+
"""
|
|
47
|
+
ANTI-GAMING (DR-103 D1 B1.2): the `human_reviews` ledger is a HUMAN-only signal, so a service-account-authored row is EXCLUDED structurally — this MUST be `false`. A service-account assessment is not a human-trust signal and cannot be laundered into the human-signal counts. Carried as a `const false` field (not omitted) so the exclusion is an explicit, audited claim on every row rather than an implicit assumption.
|
|
48
|
+
"""
|
|
49
|
+
score_text: str | None = None
|
|
50
|
+
"""
|
|
51
|
+
Open-ended human assessment as NON-COMPARABLE free TEXT — intentionally NOT an enum or number (DR-103 C3 B6.3). Consumers MUST NOT parse this into a scalar or aggregate it; a numeric channel that needs comparison is a SEPARATE structured field, never this string. Nullable: a review may carry only a thumb and/or an annotation.
|
|
52
|
+
"""
|
|
53
|
+
thumbs: bool | None = None
|
|
54
|
+
"""
|
|
55
|
+
Coarse up/down signal. `true` = thumbs-up, `false` = thumbs-down, `null` = no thumb on this review. One of the three ORTHOGONAL channels (DR-103 C3 B6.3) — never averaged with the others.
|
|
56
|
+
"""
|
|
57
|
+
annotation: str | None = None
|
|
58
|
+
"""
|
|
59
|
+
Free-text annotation / comment from the reviewer. Distinct from `score_text` (which is the open-ended SCORE channel) — an annotation is contextual prose, a score_text is the human's open-ended assessment. Nullable. One of the three ORTHOGONAL channels (DR-103 C3 B6.3).
|
|
60
|
+
"""
|
|
61
|
+
input_hash: schema.Sha256
|
|
62
|
+
"""
|
|
63
|
+
BARE 64-hex sha256 of the canonical input the human reviewed. The parallel HumanReviewStatement binds `subject[0].digest.sha256 === input_hash` (DR-103 D1 B1.3) — that pin IS the forgery cost. BARE alphabet (aligned to the entity-hash convention, e.g. SkillSnapshot.combined_sha); the in-toto wire layer strips/adds the `sha256:` prefix at the boundary.
|
|
64
|
+
"""
|
|
65
|
+
created_at: schema.Rfc3339
|
|
66
|
+
tenant_id: schema.Uuidv7 | None = None
|
|
67
|
+
"""
|
|
68
|
+
RESERVED multi-tenancy slot (deferral-G, bd_000-projects-k0fj; DR-085 D5), OPTIONAL-NOT-NULLABLE and byte-identical to EvalSpec/EvalRun/SkillSnapshot/SkillVersion. An absent tenant is a first-class single-tenant/global state, NEVER pooled into a cross-tenant aggregate (DR-103 D2 B2.2). v1 single-tenant reviews omit it; tenant-isolation semantics deferred to a future DR.
|
|
69
|
+
"""
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
# generated by datamodel-codegen:
|
|
2
|
+
# filename: skill-refiner-pass.schema.json
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
from enum import Enum
|
|
7
|
+
from typing import Literal
|
|
8
|
+
|
|
9
|
+
from pydantic import BaseModel, ConfigDict, Field, confloat, constr
|
|
10
|
+
|
|
11
|
+
from . import _common_schema as schema
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class Verdict(Enum):
|
|
15
|
+
"""
|
|
16
|
+
Decision verdict. CLOSED enum — a row is EMITTED on accept, so verdict is the accept-record discriminator (DR-082 Q2). Widening this enum is a /v2 trigger (CISO binding: a consumer that hard-codes the closed set must never silently encounter a new verdict on an immutable row).
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
accept = "accept"
|
|
20
|
+
reject = "reject"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class EvalSetRef(BaseModel):
|
|
24
|
+
"""
|
|
25
|
+
Reference to the FROZEN eval-set the verdict was derived against — the entire epistemic basis of the claim. The hash pins exact content; version + lineage_id pin which published eval-set and its provenance lineage.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
model_config = ConfigDict(
|
|
29
|
+
extra="forbid",
|
|
30
|
+
)
|
|
31
|
+
hash: schema.Sha256Prefixed
|
|
32
|
+
"""
|
|
33
|
+
sha256-prefixed digest of the frozen eval-set content.
|
|
34
|
+
"""
|
|
35
|
+
version: constr(min_length=1)
|
|
36
|
+
"""
|
|
37
|
+
Version identifier of the frozen eval-set.
|
|
38
|
+
"""
|
|
39
|
+
lineage_id: schema.Uuidv7
|
|
40
|
+
"""
|
|
41
|
+
UUIDv7 of the eval-set lineage the frozen version belongs to.
|
|
42
|
+
"""
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class NamedDimensionDelta(BaseModel):
|
|
46
|
+
model_config = ConfigDict(
|
|
47
|
+
extra="forbid",
|
|
48
|
+
)
|
|
49
|
+
id: schema.KebabSlug
|
|
50
|
+
"""
|
|
51
|
+
Named-dimension identifier (kebab-slug).
|
|
52
|
+
"""
|
|
53
|
+
delta: float
|
|
54
|
+
"""
|
|
55
|
+
Observed delta on this named dimension.
|
|
56
|
+
"""
|
|
57
|
+
non_regressed: bool
|
|
58
|
+
"""
|
|
59
|
+
Whether this dimension cleared the non-regression bar at the stated alpha. For an accept verdict every entry MUST be true.
|
|
60
|
+
"""
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class ReplayFidelityLevel(Enum):
|
|
64
|
+
"""
|
|
65
|
+
OPTIONAL replay-fidelity claim for the refiner run (RF-0 bit-exact … RF-4 non-reproducible), mirroring gate-result/v1's iel-E11 levels.
|
|
66
|
+
"""
|
|
67
|
+
|
|
68
|
+
RF_0 = "RF-0"
|
|
69
|
+
RF_1 = "RF-1"
|
|
70
|
+
RF_2 = "RF-2"
|
|
71
|
+
RF_3 = "RF-3"
|
|
72
|
+
RF_4 = "RF-4"
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
class SkillRefinerPassV1InTotoPredicateBodyAttestingASkillversionClearedTheSkillRefinerAcceptanceGate(
|
|
76
|
+
BaseModel
|
|
77
|
+
):
|
|
78
|
+
"""
|
|
79
|
+
Predicate body of an in-toto Statement v1 whose predicateType is https://evals.intentsolutions.io/skill-refiner-pass/v1. Attests that a real SkillVersion cleared the @j-rig/refiner-core acceptance gate: significant Pareto-dominance on the behavioral dimension AND non-regression on every named dimension, by a one-sided z-test at the stated alpha. The signed body carries exactly the accept DETERMINANTS — the fields a verifier needs to independently re-derive the accept decision from immutable inputs (DR-082 Q2): strip any one and the PASS becomes an unfalsifiable 'trust me, it passed'. Provenance bulk (raw per-eval-item transcripts, human-readable diff prose, dashboard-render metadata) is OUT of the signed body, referenced by hash via the EvidenceBundle side-channel. This schema validates ONLY the predicate body — the enclosing in-toto Statement envelope (_type, subject, predicateType) is validated separately; the in-toto subject[].digest.sha256 MUST equal result_snapshot_hash without the sha256: prefix (DR-085 D4: the subject is the POST-EDIT artifact being attested; the wire-discipline mirror of gate-result/v1's input_hash binding, DR-082 Q4). Each row independently verifiable; NO top-level bundle signature per Blueprint B § 7 line 754.
|
|
80
|
+
|
|
81
|
+
The skill_version_id / parent_version_id / source_snapshot_hash / result_snapshot_hash provenance set references the SkillVersion by the kernel's EXISTING id/hash primitives (UUIDv7 + sha256-prefixed) — it does NOT define a SkillVersion kernel entity (DR-028 T1 one-way-door: SkillVersion stays a separate entity, minted elsewhere). parent_version_id is NULLABLE (DR-085 D3) so a root SkillVersion attests without forging a parent. Net-new predicate URI, ADDITIVE per § 7.2 backward-compat — no v1 contract is changed.
|
|
82
|
+
|
|
83
|
+
DR-085 D5 machine-enforced accept invariant: when verdict === 'accept', EVERY named_dimension_deltas[].non_regressed MUST be true (a 'pass' that claims acceptance while a named dimension regressed is a forgeable falsehood; enforced via the if/then below + Zod .superRefine + Pydantic model_validator).
|
|
84
|
+
|
|
85
|
+
Runs in sigstore_staging (the canonical staging SigningMode, DR-085 D4 — retires the legacy 'ln' label) until ALL FOUR DR-082 Q3 production triggers hold, AND-gated: (1) the skill-refiner-pass/v1 SPEC.md normative section lands (RESERVED→ACTIVE); (2) the DNSSEC + CAA pre-flight is green on evals.intentsolutions.io; (3) the authoring chamber's SEPARATE signing trust root is provisioned-and-live (DR-081 no-shared-root); (4) ≥1 REAL SkillVersion clears the behavioral gate on a FROZEN, signed eval-set. Until then signing_mode=staging, rekor_log_index=null.
|
|
86
|
+
"""
|
|
87
|
+
|
|
88
|
+
model_config = ConfigDict(
|
|
89
|
+
extra="forbid",
|
|
90
|
+
)
|
|
91
|
+
verdict: Verdict
|
|
92
|
+
"""
|
|
93
|
+
Decision verdict. CLOSED enum — a row is EMITTED on accept, so verdict is the accept-record discriminator (DR-082 Q2). Widening this enum is a /v2 trigger (CISO binding: a consumer that hard-codes the closed set must never silently encounter a new verdict on an immutable row).
|
|
94
|
+
"""
|
|
95
|
+
reason: list[str] = Field(..., min_length=1)
|
|
96
|
+
"""
|
|
97
|
+
Structured reason entries — non-empty always, since this predicate is emitted on a real verdict. Reasons SHOULD be structured codes rather than free prose to avoid leaking skill content onto a public transparency log (CISO + GC binding).
|
|
98
|
+
"""
|
|
99
|
+
refiner_strategy_id: constr(min_length=1)
|
|
100
|
+
"""
|
|
101
|
+
Identifier of the RefinerStrategy that produced this verdict. REQUIRED in the signed body per the DR-028 CISO Session-7 binding (mechanism-swappable must not become mechanism-untraceable). Strategy ids are append-only-registered — a retired id is burned forever, never reused for a different mechanism (DR-082 Q5 CISO clarification).
|
|
102
|
+
"""
|
|
103
|
+
skill_version_id: schema.Uuidv7
|
|
104
|
+
"""
|
|
105
|
+
UUIDv7 of the accepted SkillVersion (the 14th entity per DR-028 T1). Referenced by the kernel's EXISTING UUIDv7 primitive — this predicate does NOT define a SkillVersion entity (DR-028 one-way-door).
|
|
106
|
+
"""
|
|
107
|
+
parent_version_id: schema.Uuidv7 | None = None
|
|
108
|
+
"""
|
|
109
|
+
DR-085 D3: NULLABLE UUIDv7 of the parent SkillVersion the accepted version was refined from — `null` for a ROOT SkillVersion (the first refinement in a lineage), so a root attests a valid pass WITHOUT forging a parent (root emission is provably zero-forgery — DR-085 D3 DevRel/CISO binding). When non-null, binds parent→child so a refiner cannot launder an unrelated skill through a forged lineage (CISO binding). REQUIRED key, but its value MAY be null (matching the SkillVersion entity's nullable parent_version_id).
|
|
110
|
+
"""
|
|
111
|
+
source_snapshot_hash: schema.Sha256Prefixed
|
|
112
|
+
"""
|
|
113
|
+
DR-085 D4: sha256-prefixed content hash of the PRE-EDIT SkillSnapshot — the INPUT to the refinement (re-defined to mean pre-edit input consistently with the SkillVersion entity's source_snapshot_hash; the prior 'post-edit' meaning collided semantically with the entity's meaning, the very collision D4 corrects). The POST-EDIT output is now the separate result_snapshot_hash. References the SkillSnapshot content by the kernel's EXISTING sha256-prefixed primitive.
|
|
114
|
+
"""
|
|
115
|
+
result_snapshot_hash: schema.Sha256Prefixed
|
|
116
|
+
"""
|
|
117
|
+
DR-085 D4: sha256-prefixed content hash of the POST-EDIT output SkillSnapshot — the RESULT of the refinement (the artifact the pass attests). The in-toto subject[].digest.sha256 for the row MUST equal this value WITHOUT the sha256: prefix (the authoring-chamber analogue of gate-result/v1's input_hash === subject digest binding, DR-082 Q4 — the subject is the post-edit artifact being attested, so it binds to result_snapshot_hash, not the pre-edit source_snapshot_hash). References the SkillSnapshot content by the kernel's EXISTING sha256-prefixed primitive.
|
|
118
|
+
"""
|
|
119
|
+
eval_set_ref: EvalSetRef
|
|
120
|
+
"""
|
|
121
|
+
Reference to the FROZEN eval-set the verdict was derived against — the entire epistemic basis of the claim. The hash pins exact content; version + lineage_id pin which published eval-set and its provenance lineage.
|
|
122
|
+
"""
|
|
123
|
+
edit_proposal_hash: schema.Sha256Prefixed
|
|
124
|
+
"""
|
|
125
|
+
sha256-prefixed hash of the EditProposal (the bounded edit-ops) that earned the pass — binds WHAT changed.
|
|
126
|
+
"""
|
|
127
|
+
behavioral_delta: float
|
|
128
|
+
"""
|
|
129
|
+
Observed delta on the behavioral dimension the accept gate requires significant Pareto-dominance on. A determinant of the accept decision (DR-082 Q2).
|
|
130
|
+
"""
|
|
131
|
+
named_dimension_deltas: list[NamedDimensionDelta]
|
|
132
|
+
"""
|
|
133
|
+
Per-named-dimension observed deltas — the non-regression surface of the accept gate. Each entry is independently re-checkable: a verifier re-runs the one-sided z-test on these deltas at the stated alpha. MAY be empty when the skill declares no named dimensions beyond the behavioral one.
|
|
134
|
+
"""
|
|
135
|
+
alpha: confloat(lt=1.0, gt=0.0)
|
|
136
|
+
"""
|
|
137
|
+
The significance level (α) the one-sided z-test was evaluated at — the falsifiability anchor. A 'pass' with no published alpha is an unfalsifiable assertion (DR-082 Q2). Bound in (0, 1).
|
|
138
|
+
"""
|
|
139
|
+
test_statistic_kind: Literal["one-sided-z"]
|
|
140
|
+
"""
|
|
141
|
+
Statistical-test family identifier. CONST for v1 — the acceptance gate is a one-sided z-test (DR-028 + DR-082 Q2). Changing the test family is a SEMANTIC change that mints /v2 (the same deltas/alpha would no longer mean the same verdict).
|
|
142
|
+
"""
|
|
143
|
+
cost_record_ref: schema.Uuidv7 | None = None
|
|
144
|
+
"""
|
|
145
|
+
OPTIONAL FK → CostRecord.id for cost attribution of the refiner run. Descriptive — not a determinant of accept.
|
|
146
|
+
"""
|
|
147
|
+
replay_fidelity_level: ReplayFidelityLevel | None = None
|
|
148
|
+
"""
|
|
149
|
+
OPTIONAL replay-fidelity claim for the refiner run (RF-0 bit-exact … RF-4 non-reproducible), mirroring gate-result/v1's iel-E11 levels.
|
|
150
|
+
"""
|
|
151
|
+
signing_downgrade_reason: constr(min_length=1) | None = None
|
|
152
|
+
"""
|
|
153
|
+
OPTIONAL structured reason recorded ONLY when the signing_mode was downgraded for this row (e.g. production→staging fallback). Absent on a normally-signed row.
|
|
154
|
+
"""
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
# generated by datamodel-codegen:
|
|
2
|
+
# filename: skill-version.schema.json
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
from enum import Enum
|
|
7
|
+
|
|
8
|
+
from pydantic import BaseModel, ConfigDict, constr
|
|
9
|
+
|
|
10
|
+
from . import _common_schema as schema
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class VersionKind(Enum):
|
|
14
|
+
"""
|
|
15
|
+
Load-bearing signable discriminator (DR-028 T1 line 90, P0-RATIFY-2 line 233). CLOSED enum — widening it after a signed publish is a /v2 trigger (one-way door). `edit` = a refiner-proposed forward change; `revert` = undo to a prior version's content; `restore` = re-instate an archived version. Distinct from SkillSnapshot, which has no discriminator (single terminal `created`).
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
edit = "edit"
|
|
19
|
+
revert = "revert"
|
|
20
|
+
restore = "restore"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class SkillversionRefinementLineageRecordOfASkill14ThCanonicalEntityDr028T1Discriminator(
|
|
24
|
+
BaseModel
|
|
25
|
+
):
|
|
26
|
+
"""
|
|
27
|
+
The 14th canonical entity. SEPARATE from SkillSnapshot (which content-addressed-pins a skill's source state); SkillVersion captures the REFINEMENT LINEAGE produced by the Skill Refiner. DR-028 T1 DISCRIMINATOR resolution: `version_kind` is a load-bearing signable discriminator; `parent_version_id` references the prior SkillVersion (lineage internal, SkillVersion -> SkillVersion, NOT an FK to SkillSnapshot); `source_snapshot_hash` is a READ-ONLY REFERENCE (content hash) to the PRE-EDIT SkillSnapshot at the moment of refinement — explicitly NOT a relational foreign key (no enforced relational integrity; it is a content-hash reference). DR-085 D3 lineage-integrity correction: this entity now carries a self `content_hash` (its own post-edit content) + a `parent_content_hash` (the parent's `content_hash`, the tamper-evident lineage anchor); all three hash fields use the BARE 64-hex `sha256` alphabet, ALIGNED to `SkillSnapshot.combined_sha` (one hash alphabet platform-wide — no sha256:-prefixed vs bare mismatch). `parent_content_hash` is null iff `parent_version_id` is null (a root version forges no parent). ADDITIVE / one-way-door: this shape ships in signed @intentsolutions/core entries, so widening or merging it post-publish is irreversible — additive only. The state-machine formalism + status/signing cross-field invariants (P0-RATIFY-2 `pending_production`/`active`, `rekor_log_index iff signing_mode='production' AND status='active'`) are DEFERRED to a follow-up v0.4.0 Decision Record per DR-028 T1 binding minority constraint (lines 105/108) reconciled by DR-085 D1: "Phase C ships entity + discriminator + parent_version_id only." Predicate URI: skill-refiner-pass/v1 references this entity by id/hash (already shipped); SkillVersion does NOT define that predicate.
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
model_config = ConfigDict(
|
|
31
|
+
extra="forbid",
|
|
32
|
+
)
|
|
33
|
+
id: schema.Uuidv7
|
|
34
|
+
"""
|
|
35
|
+
UUIDv7 PK. The `skill_version_id` the skill-refiner-pass/v1 predicate body references.
|
|
36
|
+
"""
|
|
37
|
+
skill_id: schema.KebabSlug
|
|
38
|
+
"""
|
|
39
|
+
Logical skill identifier — stable across versions of the same skill (mirrors SkillSnapshot.skill_id).
|
|
40
|
+
"""
|
|
41
|
+
version_kind: VersionKind
|
|
42
|
+
"""
|
|
43
|
+
Load-bearing signable discriminator (DR-028 T1 line 90, P0-RATIFY-2 line 233). CLOSED enum — widening it after a signed publish is a /v2 trigger (one-way door). `edit` = a refiner-proposed forward change; `revert` = undo to a prior version's content; `restore` = re-instate an archived version. Distinct from SkillSnapshot, which has no discriminator (single terminal `created`).
|
|
44
|
+
"""
|
|
45
|
+
parent_version_id: schema.Uuidv7 | None = None
|
|
46
|
+
"""
|
|
47
|
+
HUMAN-READABLE CONVENIENCE POINTER to the PRIOR SkillVersion in the refinement lineage (SkillVersion -> SkillVersion). Lineage is internal — this is NOT an FK to SkillSnapshot (DR-028 T1 line 91). `null` for a ROOT version (the first refinement in a lineage). No record is ever mutated; a rollback is a NEW SkillVersion citing the prior (append-only event log, AC-2 / F-MK-001). DR-085 D3: this UUID is reassignable, so it is a convenience pointer ONLY — the tamper-evident integrity anchor is `parent_content_hash` (a content hash a reviewer cannot silently re-point).
|
|
48
|
+
"""
|
|
49
|
+
content_hash: schema.Sha256
|
|
50
|
+
"""
|
|
51
|
+
DR-085 D3: this version's OWN post-edit content hash — the integrity anchor a child SkillVersion cites as its `parent_content_hash`. BARE 64-hex `sha256`, the SAME alphabet as `SkillSnapshot.combined_sha` (one hash alphabet platform-wide). Append-only content-hash chain: child.parent_content_hash == parent.content_hash makes the lineage tamper-evident — re-pointing `parent_version_id` (a reassignable UUID) cannot forge an append-only claim a verifier can falsify (DR-085 D3 CSO/CISO binding).
|
|
52
|
+
"""
|
|
53
|
+
parent_content_hash: schema.Sha256 | None = None
|
|
54
|
+
"""
|
|
55
|
+
DR-085 D3: the parent SkillVersion's `content_hash` — the tamper-evident lineage integrity anchor (the real append-only proof, vs the reassignable `parent_version_id` convenience pointer). BARE 64-hex `sha256`, ALIGNED to `SkillSnapshot.combined_sha`. MUST be `null` EXACTLY when `parent_version_id` is null (a ROOT version): root emission is provably zero-forgery (null parent_version_id + null parent_content_hash) — an implementer never forges a fake parent on a root (DR-085 D3 DevRel/CISO binding). This iff is machine-enforced at all three layers (if/then below + Zod + Pydantic).
|
|
56
|
+
"""
|
|
57
|
+
source_snapshot_hash: schema.Sha256
|
|
58
|
+
"""
|
|
59
|
+
READ-ONLY REFERENCE (content hash) to the PRE-EDIT SkillSnapshot at the moment of refinement — the INPUT to the refinement (DR-028 T1 line 92; DR-085 D4 pre-edit-input clarification, matching the skill-refiner-pass/v1 predicate's now-pre-edit `source_snapshot_hash`). EXPLICITLY NOT A RELATIONAL FOREIGN KEY — there is no enforced relational integrity; it is a content-hash reference (the SkillSnapshot it names need not exist in any table the kernel knows about). DR-085 D3: BARE 64-hex `sha256`, ALIGNED to `SkillSnapshot.combined_sha` + `content_hash` + `parent_content_hash` (one hash alphabet platform-wide — the prior sha256:-prefixed form was the very mismatch DR-085 D3 corrects).
|
|
60
|
+
"""
|
|
61
|
+
refiner_strategy_id: constr(min_length=1)
|
|
62
|
+
"""
|
|
63
|
+
Identity of the RefinerStrategy that produced this version (DR-028 P0-RATIFY-5 line 270, CISO signed-manifest binding: mechanism-swappable must NOT become mechanism-untraceable). Signed in the skill-refiner-pass/v1 predicate payload, where it is also `refiner_strategy_id`.
|
|
64
|
+
"""
|
|
65
|
+
created_at: schema.Rfc3339
|
|
66
|
+
created_by: schema.ActorIdentity
|
|
67
|
+
tenant_id: schema.Uuidv7 | None = None
|
|
68
|
+
"""
|
|
69
|
+
RESERVED multi-tenancy slot (deferral-G, bd_000-projects-k0fj), mirroring SkillSnapshot. OPTIONAL + additive per Blueprint B § 7.2; tenant-isolation semantics deferred to a future DR. v1 single-tenant versions omit it.
|
|
70
|
+
"""
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
# generated by datamodel-codegen:
|
|
2
|
+
# filename: usage-event.schema.json
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
from enum import Enum
|
|
7
|
+
|
|
8
|
+
from pydantic import BaseModel, ConfigDict, conint
|
|
9
|
+
|
|
10
|
+
from . import _common_schema as schema
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class Meter(Enum):
|
|
14
|
+
"""
|
|
15
|
+
The metered PRODUCT-METER dimension. CLOSED pricing enum (DR-103 D1 B1.4, safest for billing). HETEROGENEOUS — distinct meters carry distinct `unit`s and MUST NOT be cross-summed (C3). FREEZES at the first production-Rekor signature; widening after is a /v2 trigger. `dashboard_render` is intentionally absent (a predicate body, not a lifecycle entity). `api_call` is the ONLY meter exempt from the verified-source binding (the leaf action has no gated parent session).
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
api_call = "api_call"
|
|
19
|
+
eval_run = "eval_run"
|
|
20
|
+
skill_invocation = "skill_invocation"
|
|
21
|
+
judge_decision = "judge_decision"
|
|
22
|
+
gate_evaluation = "gate_evaluation"
|
|
23
|
+
report_render = "report_render"
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class Unit(Enum):
|
|
27
|
+
"""
|
|
28
|
+
The business unit `quantity` is expressed in. MIXED across meters — a `tokens` count and a `count` are NON-COMPARABLE, which is exactly why a cross-`(meter, unit)` SUM is out of contract (C3).
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
count = "count"
|
|
32
|
+
tokens = "tokens"
|
|
33
|
+
seconds = "seconds"
|
|
34
|
+
bytes = "bytes"
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class SourceEntityType(Enum):
|
|
38
|
+
"""
|
|
39
|
+
The gated entity TYPE this row meters. Non-null for every metered (non-`api_call`) row (anti-gaming, DR-103 D1 B1.2, enforced in the allOf/if-then below); `null` permitted ONLY for `meter === 'api_call'`.
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
eval_run = "eval_run"
|
|
43
|
+
session_trace = "session_trace"
|
|
44
|
+
judge_decision = "judge_decision"
|
|
45
|
+
tool_invocation = "tool_invocation"
|
|
46
|
+
skill_version = "skill_version"
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class UsageeventAppendOnlyProductMeteringLedgerRow15ThCanonicalEntityDr103D1(BaseModel):
|
|
50
|
+
"""
|
|
51
|
+
The 15th canonical entity. Append-only product-metering LEDGER, DISTINCT from CostRecord: CostRecord (§ 2.12) attributes PROVIDER SPEND (money in micro-USD + tokens); UsageEvent meters PRODUCT-METER COUNTS in BUSINESS UNITS (`meter` + `quantity` + `unit`). The nullable `cost_record_ref` back-reference is the SEAM proving they are SEPARATE tables — a metering row MAY cite the CostRecord that priced the same action, but the two are independently authored and never merged. Single terminal state `recorded`; the row is immutable at creation and corrections are NEW rows, never mutation (append-only). ANTI-GAMING (DR-103 D1 B1.2, machine-enforced): every metered (non-`api_call`) row MUST bind to a VERIFIED gated source — non-null `source_entity_type` + `source_entity_id` AND `source_verified === true` (the marker the runtime sets ONLY after the source session clears its quality gate) — so an arbitrary hand-supplied `quantity` with no verified provenance is refused; enforced here via `allOf`/`if-then`, in Zod via `.superRefine`, and in Pydantic via `model_validator` (one rule, three layers). The spec's original "ZERO hand-validators needed" claim is DELETED. C3 (DR-103 C3 B6.1): the `meter` enum is HETEROGENEOUS and the `unit` is mixed across meters; a consumer MUST NOT SUM/aggregate `quantity` across distinct `(meter, unit)` pairs into one scalar — rollups are per-`(meter, unit)` vectors only, enforced structurally (there is no rolled-total field on the row by construction). `quantity` is a COUNTED action, NEVER an assigned utility/weight/score (DR-103 epic Rule 1). FREEZE discipline (DR-103 binding constraint 1): the `meter` enum AND the entity name FREEZE at the first production-Rekor signature (NONE in this PR); until then both stay malleable. CMO dissent preserved (`usage_events` is a billing name) — the rename-before-production-sign debt is bound in DR-103. ADDITIVE / one-way-door: this shape ships in signed @intentsolutions/core entries, so widening or merging it post-publish is irreversible — additive only. `tenant_id` is OPTIONAL-NOT-NULLABLE (DR-103 D2 / deferral-G, bd_000-projects-k0fj), byte-identical to EvalSpec/EvalRun/SkillSnapshot/SkillVersion. A UsageEvent rides INSIDE the `extensions` field of an EvidenceStatement; it does NOT become a gate-result/v1 statement. NO production-Rekor signing is wired here.
|
|
52
|
+
"""
|
|
53
|
+
|
|
54
|
+
model_config = ConfigDict(
|
|
55
|
+
extra="forbid",
|
|
56
|
+
)
|
|
57
|
+
id: schema.Uuidv7
|
|
58
|
+
"""
|
|
59
|
+
UUIDv7 PK.
|
|
60
|
+
"""
|
|
61
|
+
meter: Meter
|
|
62
|
+
"""
|
|
63
|
+
The metered PRODUCT-METER dimension. CLOSED pricing enum (DR-103 D1 B1.4, safest for billing). HETEROGENEOUS — distinct meters carry distinct `unit`s and MUST NOT be cross-summed (C3). FREEZES at the first production-Rekor signature; widening after is a /v2 trigger. `dashboard_render` is intentionally absent (a predicate body, not a lifecycle entity). `api_call` is the ONLY meter exempt from the verified-source binding (the leaf action has no gated parent session).
|
|
64
|
+
"""
|
|
65
|
+
quantity: conint(ge=0)
|
|
66
|
+
"""
|
|
67
|
+
The metered count, expressed in `unit`. A non-negative COUNT of a verified action — NEVER an assigned utility/weight/score (DR-103 epic Rule 1). Any derived value is computed downstream from the ledger at read time, never stored on the row.
|
|
68
|
+
"""
|
|
69
|
+
unit: Unit
|
|
70
|
+
"""
|
|
71
|
+
The business unit `quantity` is expressed in. MIXED across meters — a `tokens` count and a `count` are NON-COMPARABLE, which is exactly why a cross-`(meter, unit)` SUM is out of contract (C3).
|
|
72
|
+
"""
|
|
73
|
+
source_entity_type: SourceEntityType | None = None
|
|
74
|
+
"""
|
|
75
|
+
The gated entity TYPE this row meters. Non-null for every metered (non-`api_call`) row (anti-gaming, DR-103 D1 B1.2, enforced in the allOf/if-then below); `null` permitted ONLY for `meter === 'api_call'`.
|
|
76
|
+
"""
|
|
77
|
+
source_entity_id: schema.Uuidv7 | None = None
|
|
78
|
+
"""
|
|
79
|
+
Reference (UUIDv7) to the gated source entity. Non-null for every metered (non-`api_call`) row (anti-gaming); `null` only for `api_call`. A content/identity reference, NOT an enforced relational FK — the kernel persists nothing.
|
|
80
|
+
"""
|
|
81
|
+
source_verified: bool
|
|
82
|
+
"""
|
|
83
|
+
The verified marker — `true` iff the runtime confirmed the source session cleared its quality gate. MUST be `true` for every metered (non-`api_call`) row (anti-gaming, DR-103 D1 B1.2): a metered row with an unverified source is REFUSED. The runtime sets this; a developer never hand-authors `true` (DR-103 D1 B1.6 honest-emit-path binding). For `api_call` it MAY be `false`.
|
|
84
|
+
"""
|
|
85
|
+
cost_record_ref: schema.Uuidv7 | None = None
|
|
86
|
+
"""
|
|
87
|
+
NULLABLE back-reference to the CostRecord that priced the SAME action — the SEAM proving UsageEvent and CostRecord are SEPARATE tables (DR-103 D1). `null` in the common case (most metered actions are not provider-priced). A reference, NOT an enforced relational FK.
|
|
88
|
+
"""
|
|
89
|
+
recorded_at: schema.Rfc3339
|
|
90
|
+
tenant_id: schema.Uuidv7 | None = None
|
|
91
|
+
"""
|
|
92
|
+
RESERVED multi-tenancy slot (DR-085 D5 / deferral-G, bd_000-projects-k0fj), byte-identical to EvalSpec/EvalRun/SkillSnapshot/SkillVersion. OPTIONAL + NOT NULLABLE — omitted from `required`; an explicit `null` is rejected (the schema $ref's uuidv7 with no null branch). A present tenant_id is an attested tenant claim; an absent one is single-tenant/global and MUST NOT be pooled into a cross-tenant aggregate (DR-103 D2 B2.2). Tenant-isolation semantics deferred to a future DR; v1 single-tenant rows omit it.
|
|
93
|
+
"""
|