capsule-engine 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- capsule_engine/__init__.py +17 -0
- capsule_engine/bundle_viewer/__init__.py +21 -0
- capsule_engine/bundle_viewer/static/offline_shell.html +1509 -0
- capsule_engine/bundle_viewer/viewer.py +57 -0
- capsule_engine/cli/__init__.py +5 -0
- capsule_engine/cli/bundle_cmd.py +269 -0
- capsule_engine/cli/console_cmd.py +70 -0
- capsule_engine/cli/constraints_cmd.py +74 -0
- capsule_engine/cli/format.py +173 -0
- capsule_engine/cli/guard_cmds.py +361 -0
- capsule_engine/cli/init_cmds.py +169 -0
- capsule_engine/cli/main.py +120 -0
- capsule_engine/cli/packs_cmds.py +96 -0
- capsule_engine/cli/telemetry_cmd.py +80 -0
- capsule_engine/cli/tenant_cmds.py +199 -0
- capsule_engine/cli/verify_cmd.py +110 -0
- capsule_engine/compiler/__init__.py +13 -0
- capsule_engine/compiler/effect_model.py +79 -0
- capsule_engine/compiler/vocabulary.py +253 -0
- capsule_engine/console/__init__.py +10 -0
- capsule_engine/console/api.py +251 -0
- capsule_engine/console/components.css +406 -0
- capsule_engine/console/console.css +130 -0
- capsule_engine/console/console.html +51 -0
- capsule_engine/console/console.js +330 -0
- capsule_engine/console/gallery.html +274 -0
- capsule_engine/console/server.py +137 -0
- capsule_engine/console/tokens.css +68 -0
- capsule_engine/conversation/__init__.py +88 -0
- capsule_engine/conversation/capsules.py +268 -0
- capsule_engine/conversation/exchange.py +279 -0
- capsule_engine/conversation/merkle.py +80 -0
- capsule_engine/conversation/session.py +101 -0
- capsule_engine/envcompat.py +15 -0
- capsule_engine/events/__init__.py +14 -0
- capsule_engine/events/capsule.py +133 -0
- capsule_engine/examples/__init__.py +9 -0
- capsule_engine/examples/data/tau2_airline/LICENSE-tau2-bench +21 -0
- capsule_engine/examples/data/tau2_airline/PROVENANCE.json +70 -0
- capsule_engine/examples/data/tau2_airline/hand_labels.json +893 -0
- capsule_engine/examples/data/tau2_airline/pilot1_live_gemini-2-5-flash_airline_shift1.jsonl +190 -0
- capsule_engine/examples/data/tau2_airline/tau2_committed_claude-3-7-sonnet_airline_trial0.jsonl +556 -0
- capsule_engine/examples/data/tau2_airline/tau2_committed_gpt-4-1-mini_airline_trial0.jsonl +480 -0
- capsule_engine/examples/data/tau2_airline/tau2_committed_gpt-4-1_airline_trial0.jsonl +496 -0
- capsule_engine/examples/data/tau2_airline/tau2_committed_o4-mini_airline_trial0.jsonl +392 -0
- capsule_engine/examples/data/tau2_airline/tau2_conversations_claude-3-7-sonnet_airline_4trials.jsonl +200 -0
- capsule_engine/examples/tau2_airline_reference.py +460 -0
- capsule_engine/examples/tau2_conversation_exchange.py +92 -0
- capsule_engine/folds/__init__.py +104 -0
- capsule_engine/folds/account_core.py +89 -0
- capsule_engine/folds/approval_latency.py +86 -0
- capsule_engine/folds/catalog.py +79 -0
- capsule_engine/folds/catalog_defs/actions.count_by_developer.yaml +10 -0
- capsule_engine/folds/catalog_defs/actions.executed_count.yaml +16 -0
- capsule_engine/folds/catalog_defs/actions.last_decision.yaml +12 -0
- capsule_engine/folds/catalog_defs/counterparty.seen_before.v2.yaml +34 -0
- capsule_engine/folds/catalog_defs/counterparty.seen_before.v3.yaml +38 -0
- capsule_engine/folds/catalog_defs/counterparty.seen_before.yaml +28 -0
- capsule_engine/folds/catalog_defs/disclosure.class_seen.yaml +29 -0
- capsule_engine/folds/catalog_defs/disclosure.statement_seen.yaml +32 -0
- capsule_engine/folds/catalog_defs/hold.active_exposure.yaml +44 -0
- capsule_engine/folds/catalog_defs/spend.weekly.v2.yaml +38 -0
- capsule_engine/folds/catalog_defs/spend.weekly.v3.1.yaml +71 -0
- capsule_engine/folds/catalog_defs/spend.weekly.v3.yaml +76 -0
- capsule_engine/folds/catalog_defs/spend.weekly.yaml +33 -0
- capsule_engine/folds/counterparty_signals.py +179 -0
- capsule_engine/folds/definition.py +421 -0
- capsule_engine/folds/duration.py +16 -0
- capsule_engine/folds/engine.py +436 -0
- capsule_engine/folds/errors.py +44 -0
- capsule_engine/folds/loader.py +24 -0
- capsule_engine/folds/ordering.py +143 -0
- capsule_engine/folds/paths.py +25 -0
- capsule_engine/folds/record_type_coverage.py +58 -0
- capsule_engine/folds/reducers.py +75 -0
- capsule_engine/folds/retention_continuity.py +117 -0
- capsule_engine/folds/set_membership.py +59 -0
- capsule_engine/folds/taxonomy.py +231 -0
- capsule_engine/guards/__init__.py +59 -0
- capsule_engine/guards/action.py +446 -0
- capsule_engine/guards/action_taxonomy.json +27 -0
- capsule_engine/guards/action_taxonomy_v2.json +27 -0
- capsule_engine/guards/action_taxonomy_v3.json +27 -0
- capsule_engine/guards/action_taxonomy_v4.json +27 -0
- capsule_engine/guards/action_taxonomy_v5.json +27 -0
- capsule_engine/guards/capsule.py +377 -0
- capsule_engine/guards/checks/__init__.py +268 -0
- capsule_engine/guards/checks/action_class_gate.py +199 -0
- capsule_engine/guards/checks/base.py +33 -0
- capsule_engine/guards/checks/caps.py +517 -0
- capsule_engine/guards/checks/channel_change.py +64 -0
- capsule_engine/guards/checks/commitment.py +41 -0
- capsule_engine/guards/checks/counterparty_identity_change.py +95 -0
- capsule_engine/guards/checks/counterparty_list.py +176 -0
- capsule_engine/guards/checks/counterparty_seen_before.py +160 -0
- capsule_engine/guards/checks/credential_pattern.py +73 -0
- capsule_engine/guards/checks/dedupe.py +259 -0
- capsule_engine/guards/checks/destination_rail.py +120 -0
- capsule_engine/guards/checks/field_change_count.py +100 -0
- capsule_engine/guards/checks/offer_expiry.py +134 -0
- capsule_engine/guards/checks/plan_containment.py +180 -0
- capsule_engine/guards/checks/price_floor.py +211 -0
- capsule_engine/guards/checks/promise_never.py +236 -0
- capsule_engine/guards/checks/promise_requires_approval.py +214 -0
- capsule_engine/guards/checks/recipient_role.py +117 -0
- capsule_engine/guards/checks/recurring_charge.py +49 -0
- capsule_engine/guards/checks/refundability.py +46 -0
- capsule_engine/guards/checks/release_on_acceptance.py +286 -0
- capsule_engine/guards/checks/required_disclosure.py +233 -0
- capsule_engine/guards/checks/single_commitment.py +221 -0
- capsule_engine/guards/checks/task_authority.py +182 -0
- capsule_engine/guards/checks/upfront_amount.py +71 -0
- capsule_engine/guards/checks/verify_before_dispatch.py +75 -0
- capsule_engine/guards/classes.py +393 -0
- capsule_engine/guards/engine.py +926 -0
- capsule_engine/guards/history_state.py +127 -0
- capsule_engine/guards/plan.py +237 -0
- capsule_engine/guards/revocation.py +120 -0
- capsule_engine/guards/signing.py +60 -0
- capsule_engine/guards/statements.py +62 -0
- capsule_engine/guards/tool_call.py +83 -0
- capsule_engine/guards/wickets/__init__.py +23 -0
- capsule_engine/guards/wickets/catalog.py +75 -0
- capsule_engine/guards/wickets/catalog_defs/action_class_gate.v1.1.yaml +38 -0
- capsule_engine/guards/wickets/catalog_defs/action_class_gate.v1.2.yaml +52 -0
- capsule_engine/guards/wickets/catalog_defs/action_class_gate.yaml +34 -0
- capsule_engine/guards/wickets/catalog_defs/caps.v2.yaml +30 -0
- capsule_engine/guards/wickets/catalog_defs/caps.v3.yaml +49 -0
- capsule_engine/guards/wickets/catalog_defs/caps.v4.yaml +45 -0
- capsule_engine/guards/wickets/catalog_defs/caps.v5.1.yaml +53 -0
- capsule_engine/guards/wickets/catalog_defs/caps.v5.yaml +43 -0
- capsule_engine/guards/wickets/catalog_defs/caps.yaml +17 -0
- capsule_engine/guards/wickets/catalog_defs/caps_holds.yaml +18 -0
- capsule_engine/guards/wickets/catalog_defs/channel_change.v1.1.yaml +18 -0
- capsule_engine/guards/wickets/catalog_defs/channel_change.yaml +10 -0
- capsule_engine/guards/wickets/catalog_defs/counterparty_identity_change.yaml +9 -0
- capsule_engine/guards/wickets/catalog_defs/counterparty_list.yaml +20 -0
- capsule_engine/guards/wickets/catalog_defs/counterparty_seen_before.v2.yaml +15 -0
- capsule_engine/guards/wickets/catalog_defs/counterparty_seen_before.v3.yaml +23 -0
- capsule_engine/guards/wickets/catalog_defs/counterparty_seen_before.v4.yaml +26 -0
- capsule_engine/guards/wickets/catalog_defs/counterparty_seen_before.yaml +14 -0
- capsule_engine/guards/wickets/catalog_defs/credential_pattern.yaml +14 -0
- capsule_engine/guards/wickets/catalog_defs/dedupe.yaml +19 -0
- capsule_engine/guards/wickets/catalog_defs/destination_rail.seller.v1.1.yaml +32 -0
- capsule_engine/guards/wickets/catalog_defs/destination_rail.seller.yaml +10 -0
- capsule_engine/guards/wickets/catalog_defs/destination_rail.yaml +9 -0
- capsule_engine/guards/wickets/catalog_defs/hold_reconcile.yaml +14 -0
- capsule_engine/guards/wickets/catalog_defs/material_fields_changed.v1.1.yaml +22 -0
- capsule_engine/guards/wickets/catalog_defs/material_fields_changed.yaml +14 -0
- capsule_engine/guards/wickets/catalog_defs/offer_expiry.v1.1.yaml +31 -0
- capsule_engine/guards/wickets/catalog_defs/offer_expiry.yaml +24 -0
- capsule_engine/guards/wickets/catalog_defs/offer_fields_changed.v1.1.yaml +22 -0
- capsule_engine/guards/wickets/catalog_defs/offer_fields_changed.yaml +10 -0
- capsule_engine/guards/wickets/catalog_defs/price_floor.yaml +32 -0
- capsule_engine/guards/wickets/catalog_defs/promise_never.v1.1.yaml +64 -0
- capsule_engine/guards/wickets/catalog_defs/promise_never.yaml +16 -0
- capsule_engine/guards/wickets/catalog_defs/promise_requires_approval.v1.1.yaml +26 -0
- capsule_engine/guards/wickets/catalog_defs/promise_requires_approval.yaml +21 -0
- capsule_engine/guards/wickets/catalog_defs/recipient_role.seller.v1.1.yaml +23 -0
- capsule_engine/guards/wickets/catalog_defs/recipient_role.seller.v1.2.yaml +24 -0
- capsule_engine/guards/wickets/catalog_defs/recipient_role.seller.yaml +10 -0
- capsule_engine/guards/wickets/catalog_defs/recipient_role.v1.1.yaml +26 -0
- capsule_engine/guards/wickets/catalog_defs/recipient_role.yaml +11 -0
- capsule_engine/guards/wickets/catalog_defs/recipient_seen_before.yaml +16 -0
- capsule_engine/guards/wickets/catalog_defs/recurring_charge.yaml +9 -0
- capsule_engine/guards/wickets/catalog_defs/refundability.v1.1.yaml +15 -0
- capsule_engine/guards/wickets/catalog_defs/refundability.yaml +7 -0
- capsule_engine/guards/wickets/catalog_defs/release_on_acceptance.seller.v1.1.yaml +44 -0
- capsule_engine/guards/wickets/catalog_defs/release_on_acceptance.seller.v1.2.yaml +63 -0
- capsule_engine/guards/wickets/catalog_defs/release_on_acceptance.seller.yaml +36 -0
- capsule_engine/guards/wickets/catalog_defs/required_disclosure.v1.1.yaml +36 -0
- capsule_engine/guards/wickets/catalog_defs/required_disclosure.v1.2.yaml +42 -0
- capsule_engine/guards/wickets/catalog_defs/required_disclosure.yaml +25 -0
- capsule_engine/guards/wickets/catalog_defs/single_commitment.seller.yaml +14 -0
- capsule_engine/guards/wickets/catalog_defs/task_authority.v1.1.yaml +20 -0
- capsule_engine/guards/wickets/catalog_defs/task_authority.yaml +8 -0
- capsule_engine/guards/wickets/catalog_defs/upfront_amount.v1.1.yaml +19 -0
- capsule_engine/guards/wickets/catalog_defs/upfront_amount.yaml +11 -0
- capsule_engine/guards/wickets/catalog_defs/verify_before_dispatch.yaml +9 -0
- capsule_engine/guards/wickets/definition.py +164 -0
- capsule_engine/guards/wickets/errors.py +18 -0
- capsule_engine/guards/wickets/loader.py +26 -0
- capsule_engine/guards/wickets/retired.py +62 -0
- capsule_engine/holds/__init__.py +35 -0
- capsule_engine/holds/capsules.py +266 -0
- capsule_engine/holds/engine.py +476 -0
- capsule_engine/holds/errors.py +29 -0
- capsule_engine/holds/policy.py +62 -0
- capsule_engine/holds/scope.py +47 -0
- capsule_engine/mcp/__init__.py +16 -0
- capsule_engine/mcp/config.py +117 -0
- capsule_engine/mcp/server.py +338 -0
- capsule_engine/mcp/tools.py +424 -0
- capsule_engine/packaging.py +60 -0
- capsule_engine/packs/__init__.py +45 -0
- capsule_engine/packs/backfill_coverage.py +238 -0
- capsule_engine/packs/catalog/airline-engagement/AI-BOOTSTRAP.md +53 -0
- capsule_engine/packs/catalog/airline-engagement/folds/session_close.yaml +8 -0
- capsule_engine/packs/catalog/airline-engagement/pack.yaml +146 -0
- capsule_engine/packs/catalog/eu-ai-act/fixtures/trace/good_execution.json +36 -0
- capsule_engine/packs/catalog/eu-ai-act/fixtures/trace/oversight_declared.json +36 -0
- capsule_engine/packs/catalog/eu-ai-act/fixtures/trace/tampered_execution.json +36 -0
- capsule_engine/packs/catalog/eu-ai-act/fixtures/trace/trusted_issuer.jwk.json +5 -0
- capsule_engine/packs/catalog/eu-ai-act/obligations-pack-note.md +68 -0
- capsule_engine/packs/catalog/eu-ai-act/obligations-register.yaml +761 -0
- capsule_engine/packs/catalog/eu-ai-act/pack.yaml +88 -0
- capsule_engine/packs/catalog/eu-ai-act/register.yaml +96 -0
- capsule_engine/packs/catalog/eu-ai-act-deterministic/fixtures/synthetic_internal_assist_sample.jsonl +6 -0
- capsule_engine/packs/catalog/eu-ai-act-deterministic/pack.yaml +580 -0
- capsule_engine/packs/catalog/everyday/AI-BOOTSTRAP.md +194 -0
- capsule_engine/packs/catalog/everyday/fixtures/mini_ledger.jsonl +45 -0
- capsule_engine/packs/catalog/everyday/pack.yaml +480 -0
- capsule_engine/packs/catalog/payments-safety/AI-BOOTSTRAP.md +83 -0
- capsule_engine/packs/catalog/payments-safety/fixtures/mini_ledger.jsonl +8 -0
- capsule_engine/packs/catalog/payments-safety/folds/spend_weekly.yaml +29 -0
- capsule_engine/packs/catalog/payments-safety/pack.yaml +112 -0
- capsule_engine/packs/catalog/seller/fixtures/mini_ledger.jsonl +83 -0
- capsule_engine/packs/catalog/seller/pack.yaml +503 -0
- capsule_engine/packs/catalog/standard-vendor/AI-BOOTSTRAP.md +70 -0
- capsule_engine/packs/catalog/standard-vendor/folds/session_close.yaml +8 -0
- capsule_engine/packs/catalog/standard-vendor/pack.yaml +466 -0
- capsule_engine/packs/contract_diff.py +809 -0
- capsule_engine/packs/contract_validate.py +185 -0
- capsule_engine/packs/corpus_verify.py +126 -0
- capsule_engine/packs/enforce.py +81 -0
- capsule_engine/packs/errors.py +159 -0
- capsule_engine/packs/eu_ai_act_trace_field_map.py +46 -0
- capsule_engine/packs/install.py +266 -0
- capsule_engine/packs/loader.py +1462 -0
- capsule_engine/packs/measurability_report.py +280 -0
- capsule_engine/packs/obligation_results.py +72 -0
- capsule_engine/packs/pins.py +92 -0
- capsule_engine/packs/schema.py +1053 -0
- capsule_engine/packs/thresholds.py +146 -0
- capsule_engine/packs/trace_attainment.py +188 -0
- capsule_engine/packs/trace_reader.py +104 -0
- capsule_engine/policy/__init__.py +37 -0
- capsule_engine/policy/activation.py +87 -0
- capsule_engine/policy/catalog_defs/default.yaml +23 -0
- capsule_engine/policy/catalog_defs/holds.yaml +27 -0
- capsule_engine/policy/errors.py +46 -0
- capsule_engine/policy/limits.py +287 -0
- capsule_engine/policy/loader.py +26 -0
- capsule_engine/policy/manifest.py +289 -0
- capsule_engine/policy/profile.py +281 -0
- capsule_engine/policy/resolve.py +224 -0
- capsule_engine/register/__init__.py +39 -0
- capsule_engine/register/compiler.py +219 -0
- capsule_engine/register/errors.py +40 -0
- capsule_engine/register/loader.py +247 -0
- capsule_engine/register/schema.py +129 -0
- capsule_engine/registry/__init__.py +19 -0
- capsule_engine/registry/conventions.json +162 -0
- capsule_engine/registry/conventions.py +188 -0
- capsule_engine/registry/cpb_registry.json +78 -0
- capsule_engine/report/__init__.py +62 -0
- capsule_engine/report/build.py +267 -0
- capsule_engine/report/coverage.py +777 -0
- capsule_engine/report/errors.py +35 -0
- capsule_engine/report/live_check.py +126 -0
- capsule_engine/report/live_history.py +420 -0
- capsule_engine/report/model.py +95 -0
- capsule_engine/report/render.py +467 -0
- capsule_engine/report/replay.py +1677 -0
- capsule_engine/report/result.py +504 -0
- capsule_engine/report/result_from_folds.py +222 -0
- capsule_engine/report/result_from_layers.py +670 -0
- capsule_engine/report/sale_bundle.py +864 -0
- capsule_engine/report/static/verify.js +451 -0
- capsule_engine/schemas/evidence-contract-v0.json +383 -0
- capsule_engine/schemas/vendor/epistemic-types.json +21 -0
- capsule_engine/schemas/vendor/evidence-result-v0.json +463 -0
- capsule_engine/telemetry/__init__.py +9 -0
- capsule_engine/telemetry/consent.py +39 -0
- capsule_engine/telemetry/events.py +93 -0
- capsule_engine/telemetry/fixtures/synthetic_events.jsonl +45 -0
- capsule_engine/telemetry/funnel.py +147 -0
- capsule_engine/telemetry/record.py +106 -0
- capsule_engine/telemetry/sink.py +61 -0
- capsule_engine/telemetry/state.py +60 -0
- capsule_engine/tenants.py +327 -0
- capsule_engine-0.1.0.dist-info/METADATA +47 -0
- capsule_engine-0.1.0.dist-info/RECORD +288 -0
- capsule_engine-0.1.0.dist-info/WHEEL +5 -0
- capsule_engine-0.1.0.dist-info/entry_points.txt +2 -0
- capsule_engine-0.1.0.dist-info/licenses/LICENSE +202 -0
- capsule_engine-0.1.0.dist-info/licenses/capsule_engine/examples/data/tau2_airline/LICENSE-tau2-bench +21 -0
- capsule_engine-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,670 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
"""Adapter: a ``deal reconcile`` report plus per-action layer observations ->
|
|
3
|
+
one Result v0 per layer, never one combined aggregate.
|
|
4
|
+
|
|
5
|
+
A run that checks each consequential action at several layers (was the
|
|
6
|
+
check invoked, was the action described correctly, was the returned
|
|
7
|
+
disposition honored, ...) reports each layer as its own metric. This module
|
|
8
|
+
builds one ``EvidenceResult`` per layer with ``build_result``, so every
|
|
9
|
+
layer's coverage and buckets are counted off that layer's own claims, and
|
|
10
|
+
wraps them in a ``layer-tally/v0`` document that carries what the Result v0
|
|
11
|
+
schema has no field for: the tool version under test, the period, and the
|
|
12
|
+
reconcile pass's ``coverage.cannot_see`` list. Nothing here adds the layers
|
|
13
|
+
together.
|
|
14
|
+
|
|
15
|
+
The action population is the reconcile report's consequential set
|
|
16
|
+
(``recorded``, ``unrecorded`` and ``failed_attempts`` rows). The first
|
|
17
|
+
layer, the invocation layer, is read off that report and nothing else:
|
|
18
|
+
|
|
19
|
+
- a ``recorded`` row is ``met`` (a sealed deal step accounts for it);
|
|
20
|
+
- an ``unrecorded`` row is ``not_met`` (no deal step accounts for it);
|
|
21
|
+
- a ``failed_attempts`` row is ``not_evaluable`` with sufficiency
|
|
22
|
+
``UNKNOWN``: reconcile lists it for review because a failed attempt can
|
|
23
|
+
still have had an effect, so whether a check was owed is not settled;
|
|
24
|
+
- ``not_consequential`` is that layer's ``excluded_not_applicable`` count.
|
|
25
|
+
|
|
26
|
+
Every later layer names a ``gate`` layer. An action reaches it only where
|
|
27
|
+
its gate claim is ``met``; where the gate is ``not_met`` or excluded, the
|
|
28
|
+
action is excluded as NOT_APPLICABLE for this layer (spec section 3: it
|
|
29
|
+
never becomes a claim, it is only counted). Where the gate is
|
|
30
|
+
``not_evaluable``, whether the layer applied is itself unsettled, so the
|
|
31
|
+
action is ``not_evaluable`` here too. An action that reaches a layer with
|
|
32
|
+
no observation is ``not_evaluable`` with sufficiency ``UNKNOWN``: no
|
|
33
|
+
observed check is never a ``met``.
|
|
34
|
+
|
|
35
|
+
Grades: a layer whose population is a party's own report is configured
|
|
36
|
+
``self_reported``, and every claim on it is ``self-attested`` -- the count
|
|
37
|
+
is a lower bound on what happened, however tamper-evident the record. The
|
|
38
|
+
invocation layer is always self-reported: reconcile reads the agent host's
|
|
39
|
+
own records on the agent's own machine, so it must be configured that way,
|
|
40
|
+
and every invocation claim cites the digest of the executions file
|
|
41
|
+
reconcile read (``executions_sha256``) as its first evidence. On other
|
|
42
|
+
layers, observations carry the grade the caller read off the sealed record;
|
|
43
|
+
on a self-reported layer, any grade but ``self-attested`` is refused.
|
|
44
|
+
A ``judged`` layer must name its judge pin, which every claim of that layer
|
|
45
|
+
carries as its first evidence digest; a ``recomputed`` layer must not.
|
|
46
|
+
"""
|
|
47
|
+
from __future__ import annotations
|
|
48
|
+
|
|
49
|
+
import re
|
|
50
|
+
from collections.abc import Collection, Mapping, Sequence
|
|
51
|
+
from dataclasses import dataclass
|
|
52
|
+
from datetime import datetime
|
|
53
|
+
from typing import NamedTuple, NotRequired, TypedDict
|
|
54
|
+
|
|
55
|
+
from agent_action_capsule.canonical import json_digest
|
|
56
|
+
|
|
57
|
+
from .errors import COVERAGE_NOT_COMPUTED, INVALID_LAYER_TALLY, ResultError
|
|
58
|
+
from .result import (
|
|
59
|
+
AnalysisCarrier,
|
|
60
|
+
Claim,
|
|
61
|
+
Coverage,
|
|
62
|
+
DigestRef,
|
|
63
|
+
DisclosureCarrier,
|
|
64
|
+
EvidenceResult,
|
|
65
|
+
build_result,
|
|
66
|
+
validate_against_schema,
|
|
67
|
+
verify_result,
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
__all__ = [
|
|
71
|
+
"TALLY_VERSION",
|
|
72
|
+
"CoverageCounts",
|
|
73
|
+
"ReconcileCounts",
|
|
74
|
+
"LayerEntry",
|
|
75
|
+
"LayerTallyDoc",
|
|
76
|
+
"ActionLayer",
|
|
77
|
+
"LayerSpec",
|
|
78
|
+
"LayerObservation",
|
|
79
|
+
"ReconcileRow",
|
|
80
|
+
"ReconcileReport",
|
|
81
|
+
"LayerAggregate",
|
|
82
|
+
"LayerTally",
|
|
83
|
+
"build_layer_tally",
|
|
84
|
+
"verify_layer_tally",
|
|
85
|
+
"render_layer_tally",
|
|
86
|
+
]
|
|
87
|
+
|
|
88
|
+
TALLY_VERSION = "layer-tally/v0"
|
|
89
|
+
|
|
90
|
+
_HEX64 = re.compile(r"^[0-9a-f]{64}$")
|
|
91
|
+
_MATCHED_BY = frozenset({"act", "disclosure", "check"})
|
|
92
|
+
_RECONCILE_COUNTS = ("records_read", "outside_period", "not_consequential", "consequential")
|
|
93
|
+
# The Analysis carrier status for an observation the record did not settle,
|
|
94
|
+
# using the coverage report's fixed mapping (GAP <-> NOT_FOUND).
|
|
95
|
+
_UNSETTLED_STATUS = {"INSUFFICIENT": "INSUFFICIENT", "GAP": "NOT_FOUND", "UNKNOWN": "UNKNOWN"}
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
class CoverageCounts(TypedDict):
|
|
99
|
+
evaluated_population: int
|
|
100
|
+
excluded_not_applicable: int
|
|
101
|
+
unknown_count: int
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
class ReconcileCounts(TypedDict):
|
|
105
|
+
records_read: int
|
|
106
|
+
outside_period: int
|
|
107
|
+
not_consequential: int
|
|
108
|
+
consequential: int
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
class ReconcileSource(ReconcileCounts):
|
|
112
|
+
kind: str
|
|
113
|
+
failed_attempts: int
|
|
114
|
+
# SHA-256 of the executions file the reconcile pass read.
|
|
115
|
+
executions_sha256: str
|
|
116
|
+
# The reconcile report's own ``coverage`` block, carried verbatim:
|
|
117
|
+
# ``cannot_see`` is checked non-empty, the rest is the pass's wording.
|
|
118
|
+
coverage: Mapping[str, object]
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
class TallyTool(TypedDict):
|
|
122
|
+
name: str
|
|
123
|
+
version: str
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
TallyPeriod = TypedDict("TallyPeriod", {"from": str, "to": str})
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
class LayerEntry(TypedDict):
|
|
130
|
+
layer: str
|
|
131
|
+
tier: str
|
|
132
|
+
gate: NotRequired[str]
|
|
133
|
+
judge_pin: NotRequired[str]
|
|
134
|
+
self_reported: bool
|
|
135
|
+
coverage: CoverageCounts
|
|
136
|
+
# A Result v0 document, as ``EvidenceResult.to_dict`` returns it.
|
|
137
|
+
result: dict | None
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
class LayerTallyDoc(TypedDict):
|
|
141
|
+
tally_version: str
|
|
142
|
+
generated_at: str
|
|
143
|
+
tool: TallyTool
|
|
144
|
+
period: TallyPeriod
|
|
145
|
+
source: ReconcileSource
|
|
146
|
+
layers: list[LayerEntry]
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def _tally_error(message: str) -> ResultError:
|
|
150
|
+
return ResultError(INVALID_LAYER_TALLY, message)
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def _check_rfc3339(name: str, value: str) -> None:
|
|
154
|
+
try:
|
|
155
|
+
datetime.fromisoformat(value.replace("Z", "+00:00"))
|
|
156
|
+
except ValueError as exc:
|
|
157
|
+
raise _tally_error(f"{name} must be an RFC 3339 time, got {value!r}") from exc
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
class ActionLayer(NamedTuple):
|
|
161
|
+
action_id: str
|
|
162
|
+
layer_id: str
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
@dataclass(frozen=True)
|
|
166
|
+
class LayerSpec:
|
|
167
|
+
"""One layer, as the caller configures it. ``gate`` is the id of an
|
|
168
|
+
earlier layer; ``None`` only for the invocation layer."""
|
|
169
|
+
|
|
170
|
+
layer_id: str
|
|
171
|
+
requirement_ref: str
|
|
172
|
+
tier: str
|
|
173
|
+
gate: str | None = None
|
|
174
|
+
judge_pin: str | None = None
|
|
175
|
+
self_reported: bool = False
|
|
176
|
+
|
|
177
|
+
def __post_init__(self) -> None:
|
|
178
|
+
if not self.layer_id or not self.requirement_ref:
|
|
179
|
+
raise _tally_error("layer_id and requirement_ref must be non-empty")
|
|
180
|
+
if self.tier == "judged":
|
|
181
|
+
if self.judge_pin is None:
|
|
182
|
+
raise _tally_error(f"layer {self.layer_id!r} is judged and must name its judge pin")
|
|
183
|
+
DigestRef(digest=self.judge_pin)
|
|
184
|
+
elif self.tier == "recomputed":
|
|
185
|
+
if self.judge_pin is not None:
|
|
186
|
+
raise _tally_error(f"layer {self.layer_id!r} is recomputed and carries no judge pin")
|
|
187
|
+
else:
|
|
188
|
+
raise _tally_error(f"layer {self.layer_id!r} tier must be recomputed|judged, got {self.tier!r}")
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
@dataclass(frozen=True)
|
|
192
|
+
class LayerObservation:
|
|
193
|
+
"""What the caller observed for one action at one layer, read off the
|
|
194
|
+
sealed record: its grade, whether the record settled the question
|
|
195
|
+
(``sufficiency``) and, if so, the verdict. ``Claim`` enforces the
|
|
196
|
+
sufficiency/verdict rule when the claim is built."""
|
|
197
|
+
|
|
198
|
+
action_id: str
|
|
199
|
+
layer_id: str
|
|
200
|
+
grade: str
|
|
201
|
+
sufficiency: str
|
|
202
|
+
verdict: str
|
|
203
|
+
evidence: tuple[str, ...]
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
@dataclass(frozen=True)
|
|
207
|
+
class ReconcileRow:
|
|
208
|
+
id: str
|
|
209
|
+
digest: str
|
|
210
|
+
matched_by: str | None
|
|
211
|
+
capsule_id: str | None
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def _row(raw: Mapping[str, object], where: str, *, recorded: bool) -> ReconcileRow:
|
|
215
|
+
row_id = raw.get("id")
|
|
216
|
+
if not isinstance(row_id, str) or not row_id:
|
|
217
|
+
raise _tally_error(f"{where}: every row needs a non-empty string id")
|
|
218
|
+
matched_by = raw.get("matched_by")
|
|
219
|
+
capsule_id = raw.get("capsule_id")
|
|
220
|
+
if recorded:
|
|
221
|
+
if matched_by not in _MATCHED_BY:
|
|
222
|
+
raise _tally_error(f"{where} row {row_id!r}: matched_by must be one of {sorted(_MATCHED_BY)}, got {matched_by!r}")
|
|
223
|
+
if not isinstance(capsule_id, str):
|
|
224
|
+
raise _tally_error(f"{where} row {row_id!r}: a recorded row names the capsule_id that accounts for it")
|
|
225
|
+
elif matched_by is not None:
|
|
226
|
+
raise _tally_error(f"{where} row {row_id!r}: only a recorded row carries matched_by")
|
|
227
|
+
return ReconcileRow(
|
|
228
|
+
id=row_id,
|
|
229
|
+
digest=json_digest(dict(raw)),
|
|
230
|
+
matched_by=matched_by if recorded else None,
|
|
231
|
+
capsule_id=capsule_id if recorded else None,
|
|
232
|
+
)
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def _rows(doc: Mapping[str, object], key: str, *, recorded: bool = False) -> tuple[ReconcileRow, ...]:
|
|
236
|
+
raw = doc.get(key)
|
|
237
|
+
if not isinstance(raw, list):
|
|
238
|
+
raise _tally_error(f"reconcile report: {key} must be a list")
|
|
239
|
+
return tuple(_row(r, key, recorded=recorded) for r in raw)
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
@dataclass(frozen=True)
|
|
243
|
+
class ReconcileReport:
|
|
244
|
+
"""The fields of one ``deal reconcile`` JSON output this adapter reads.
|
|
245
|
+
``coverage`` is kept verbatim so ``cannot_see`` reaches the tally."""
|
|
246
|
+
|
|
247
|
+
period_from: str
|
|
248
|
+
period_to: str
|
|
249
|
+
counts: ReconcileCounts
|
|
250
|
+
recorded: tuple[ReconcileRow, ...]
|
|
251
|
+
unrecorded: tuple[ReconcileRow, ...]
|
|
252
|
+
failed_attempts: tuple[ReconcileRow, ...]
|
|
253
|
+
coverage: Mapping[str, object]
|
|
254
|
+
|
|
255
|
+
@classmethod
|
|
256
|
+
def from_json(cls, doc: Mapping[str, object]) -> ReconcileReport:
|
|
257
|
+
period = doc.get("period")
|
|
258
|
+
if not isinstance(period, dict) or not isinstance(period.get("from"), str) or not isinstance(period.get("to"), str):
|
|
259
|
+
raise _tally_error("reconcile report: period must carry from and to")
|
|
260
|
+
_check_rfc3339("period.from", period["from"])
|
|
261
|
+
_check_rfc3339("period.to", period["to"])
|
|
262
|
+
for key in _RECONCILE_COUNTS:
|
|
263
|
+
value = doc.get(key)
|
|
264
|
+
if type(value) is not int or value < 0:
|
|
265
|
+
raise _tally_error(f"reconcile report: {key} must be a non-negative integer, got {value!r}")
|
|
266
|
+
counts = ReconcileCounts(
|
|
267
|
+
records_read=doc["records_read"],
|
|
268
|
+
outside_period=doc["outside_period"],
|
|
269
|
+
not_consequential=doc["not_consequential"],
|
|
270
|
+
consequential=doc["consequential"],
|
|
271
|
+
)
|
|
272
|
+
coverage = doc.get("coverage")
|
|
273
|
+
if not isinstance(coverage, dict):
|
|
274
|
+
raise _tally_error("reconcile report: coverage is missing")
|
|
275
|
+
cannot_see = coverage.get("cannot_see")
|
|
276
|
+
if not isinstance(cannot_see, list) or not cannot_see or not all(isinstance(s, str) and s for s in cannot_see):
|
|
277
|
+
raise _tally_error("reconcile report: coverage.cannot_see must be a non-empty list of strings; it is never dropped")
|
|
278
|
+
report = cls(
|
|
279
|
+
period_from=period["from"],
|
|
280
|
+
period_to=period["to"],
|
|
281
|
+
counts=counts,
|
|
282
|
+
recorded=_rows(doc, "recorded", recorded=True),
|
|
283
|
+
unrecorded=_rows(doc, "unrecorded"),
|
|
284
|
+
failed_attempts=_rows(doc, "failed_attempts"),
|
|
285
|
+
coverage=dict(coverage),
|
|
286
|
+
)
|
|
287
|
+
report._check_counts()
|
|
288
|
+
return report
|
|
289
|
+
|
|
290
|
+
def _check_counts(self) -> None:
|
|
291
|
+
# The reconcile pass's own arithmetic: consequential = recorded +
|
|
292
|
+
# unrecorded; every in-period record is consequential, failed, or not.
|
|
293
|
+
consequential = len(self.recorded) + len(self.unrecorded)
|
|
294
|
+
if self.counts["consequential"] != consequential:
|
|
295
|
+
raise _tally_error(
|
|
296
|
+
f"reconcile report: consequential is {self.counts['consequential']} but recorded + unrecorded is {consequential}"
|
|
297
|
+
)
|
|
298
|
+
expected_read = consequential + len(self.failed_attempts) + self.counts["not_consequential"]
|
|
299
|
+
if self.counts["records_read"] != expected_read:
|
|
300
|
+
raise _tally_error(
|
|
301
|
+
f"reconcile report: records_read is {self.counts['records_read']} but its parts add to {expected_read}"
|
|
302
|
+
)
|
|
303
|
+
ids = [r.id for r in self.recorded + self.unrecorded + self.failed_attempts]
|
|
304
|
+
if len(set(ids)) != len(ids):
|
|
305
|
+
raise _tally_error("reconcile report: an execution record id appears in more than one row")
|
|
306
|
+
|
|
307
|
+
@property
|
|
308
|
+
def action_ids(self) -> tuple[str, ...]:
|
|
309
|
+
return tuple(r.id for r in self.recorded + self.unrecorded + self.failed_attempts)
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
@dataclass(frozen=True)
|
|
313
|
+
class LayerAggregate:
|
|
314
|
+
"""One layer's metric. ``result`` is ``None`` only when no action was
|
|
315
|
+
evaluated at this layer: Result v0 requires at least one claim, so a
|
|
316
|
+
zero denominator is stated in ``coverage`` instead of hidden."""
|
|
317
|
+
|
|
318
|
+
spec: LayerSpec
|
|
319
|
+
coverage: Coverage
|
|
320
|
+
result: EvidenceResult | None
|
|
321
|
+
|
|
322
|
+
def to_dict(self) -> LayerEntry:
|
|
323
|
+
out = LayerEntry(
|
|
324
|
+
layer=self.spec.layer_id,
|
|
325
|
+
tier=self.spec.tier,
|
|
326
|
+
self_reported=self.spec.self_reported,
|
|
327
|
+
coverage=CoverageCounts(
|
|
328
|
+
evaluated_population=self.coverage.evaluated_population,
|
|
329
|
+
excluded_not_applicable=self.coverage.excluded_not_applicable,
|
|
330
|
+
unknown_count=self.coverage.unknown_count,
|
|
331
|
+
),
|
|
332
|
+
result=None if self.result is None else self.result.to_dict(),
|
|
333
|
+
)
|
|
334
|
+
if self.spec.gate is not None:
|
|
335
|
+
out["gate"] = self.spec.gate
|
|
336
|
+
if self.spec.judge_pin is not None:
|
|
337
|
+
out["judge_pin"] = self.spec.judge_pin
|
|
338
|
+
return out
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
@dataclass(frozen=True)
|
|
342
|
+
class LayerTally:
|
|
343
|
+
generated_at: str
|
|
344
|
+
tool_name: str
|
|
345
|
+
tool_version: str
|
|
346
|
+
executions_sha256: str
|
|
347
|
+
reconcile: ReconcileReport
|
|
348
|
+
layers: tuple[LayerAggregate, ...]
|
|
349
|
+
|
|
350
|
+
def to_dict(self) -> LayerTallyDoc:
|
|
351
|
+
source = ReconcileSource(
|
|
352
|
+
kind="deal-reconcile",
|
|
353
|
+
**self.reconcile.counts,
|
|
354
|
+
failed_attempts=len(self.reconcile.failed_attempts),
|
|
355
|
+
executions_sha256=self.executions_sha256,
|
|
356
|
+
coverage=dict(self.reconcile.coverage),
|
|
357
|
+
)
|
|
358
|
+
return LayerTallyDoc(
|
|
359
|
+
tally_version=TALLY_VERSION,
|
|
360
|
+
generated_at=self.generated_at,
|
|
361
|
+
tool=TallyTool(name=self.tool_name, version=self.tool_version),
|
|
362
|
+
period=TallyPeriod({"from": self.reconcile.period_from, "to": self.reconcile.period_to}),
|
|
363
|
+
source=source,
|
|
364
|
+
layers=[layer.to_dict() for layer in self.layers],
|
|
365
|
+
)
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
def _invocation_claims(
|
|
369
|
+
report: ReconcileReport, spec: LayerSpec, contract_ref: str, executions_sha256: str
|
|
370
|
+
) -> dict[str, Claim]:
|
|
371
|
+
claims: dict[str, Claim] = {}
|
|
372
|
+
|
|
373
|
+
def claim(row: ReconcileRow, sufficiency: str, verdict: str, presentation_status: str) -> Claim:
|
|
374
|
+
# The executions file first: it is what the count is a lower bound
|
|
375
|
+
# over. The row digest covers the whole row, capsule_id included;
|
|
376
|
+
# the capsule_id is also cited on its own when it is itself a digest.
|
|
377
|
+
digests = [executions_sha256, row.digest]
|
|
378
|
+
if row.capsule_id is not None and _HEX64.fullmatch(row.capsule_id):
|
|
379
|
+
digests.append(row.capsule_id)
|
|
380
|
+
evidence = tuple(DigestRef(digest=d) for d in digests)
|
|
381
|
+
if sufficiency == "SATISFIED":
|
|
382
|
+
presentation: DisclosureCarrier | AnalysisCarrier = DisclosureCarrier(status=presentation_status, evidence=evidence)
|
|
383
|
+
else:
|
|
384
|
+
presentation = AnalysisCarrier(
|
|
385
|
+
status=presentation_status,
|
|
386
|
+
summary="a failed attempt, listed for review: it can still have had an effect, so whether a check was owed is not settled",
|
|
387
|
+
)
|
|
388
|
+
return Claim(
|
|
389
|
+
id=f"{spec.layer_id}:{row.id}",
|
|
390
|
+
contract_ref=contract_ref,
|
|
391
|
+
requirement_ref=spec.requirement_ref,
|
|
392
|
+
tier=spec.tier,
|
|
393
|
+
grade="self-attested",
|
|
394
|
+
sufficiency=sufficiency,
|
|
395
|
+
verdict=verdict,
|
|
396
|
+
evidence=evidence,
|
|
397
|
+
proofs=(),
|
|
398
|
+
presentation=presentation,
|
|
399
|
+
)
|
|
400
|
+
|
|
401
|
+
for row in report.recorded:
|
|
402
|
+
claims[row.id] = claim(row, "SATISFIED", "met", "SATISFIED")
|
|
403
|
+
for row in report.unrecorded:
|
|
404
|
+
claims[row.id] = claim(row, "SATISFIED", "not_met", "SATISFIED")
|
|
405
|
+
for row in report.failed_attempts:
|
|
406
|
+
claims[row.id] = claim(row, "UNKNOWN", "not_evaluable", "UNKNOWN")
|
|
407
|
+
return claims
|
|
408
|
+
|
|
409
|
+
|
|
410
|
+
def _unknown_claim(spec: LayerSpec, action_id: str, contract_ref: str, summary: str) -> Claim:
|
|
411
|
+
evidence = (DigestRef(digest=spec.judge_pin),) if spec.judge_pin is not None else ()
|
|
412
|
+
return Claim(
|
|
413
|
+
id=f"{spec.layer_id}:{action_id}",
|
|
414
|
+
contract_ref=contract_ref,
|
|
415
|
+
requirement_ref=spec.requirement_ref,
|
|
416
|
+
tier=spec.tier,
|
|
417
|
+
grade="self-attested",
|
|
418
|
+
sufficiency="UNKNOWN",
|
|
419
|
+
verdict="not_evaluable",
|
|
420
|
+
evidence=evidence,
|
|
421
|
+
proofs=(),
|
|
422
|
+
presentation=AnalysisCarrier(status="UNKNOWN", summary=summary),
|
|
423
|
+
)
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
def _observed_claim(spec: LayerSpec, obs: LayerObservation, contract_ref: str) -> Claim:
|
|
427
|
+
digests = ((spec.judge_pin,) if spec.judge_pin is not None else ()) + obs.evidence
|
|
428
|
+
evidence = tuple(DigestRef(digest=d) for d in digests)
|
|
429
|
+
if obs.sufficiency == "SATISFIED":
|
|
430
|
+
presentation: DisclosureCarrier | AnalysisCarrier = DisclosureCarrier(status="SATISFIED", evidence=evidence)
|
|
431
|
+
else:
|
|
432
|
+
presentation = AnalysisCarrier(
|
|
433
|
+
status=_UNSETTLED_STATUS.get(obs.sufficiency, "UNKNOWN"),
|
|
434
|
+
summary=f"the layer applied and the record did not settle it (sufficiency {obs.sufficiency})",
|
|
435
|
+
)
|
|
436
|
+
return Claim(
|
|
437
|
+
id=f"{spec.layer_id}:{obs.action_id}",
|
|
438
|
+
contract_ref=contract_ref,
|
|
439
|
+
requirement_ref=spec.requirement_ref,
|
|
440
|
+
tier=spec.tier,
|
|
441
|
+
grade=obs.grade,
|
|
442
|
+
sufficiency=obs.sufficiency,
|
|
443
|
+
verdict=obs.verdict,
|
|
444
|
+
evidence=evidence,
|
|
445
|
+
proofs=(),
|
|
446
|
+
presentation=presentation,
|
|
447
|
+
)
|
|
448
|
+
|
|
449
|
+
|
|
450
|
+
def _layer_aggregate(
|
|
451
|
+
spec: LayerSpec, claims: Sequence[Claim], excluded: int, generated_at: str
|
|
452
|
+
) -> LayerAggregate:
|
|
453
|
+
if not claims:
|
|
454
|
+
return LayerAggregate(spec=spec, coverage=Coverage(0, excluded, 0), result=None)
|
|
455
|
+
result = build_result(list(claims), generated_at=generated_at, excluded_not_applicable=excluded)
|
|
456
|
+
return LayerAggregate(spec=spec, coverage=result.aggregate.coverage, result=result)
|
|
457
|
+
|
|
458
|
+
|
|
459
|
+
def build_layer_tally(
|
|
460
|
+
reconcile: Mapping[str, object],
|
|
461
|
+
invocation: LayerSpec,
|
|
462
|
+
layers: Sequence[LayerSpec],
|
|
463
|
+
observations: Sequence[LayerObservation],
|
|
464
|
+
*,
|
|
465
|
+
contract_ref: str,
|
|
466
|
+
generated_at: str,
|
|
467
|
+
tool_name: str,
|
|
468
|
+
tool_version: str,
|
|
469
|
+
executions_sha256: str,
|
|
470
|
+
not_applicable: Collection[ActionLayer] = (),
|
|
471
|
+
) -> LayerTally:
|
|
472
|
+
"""Build one Result per layer: ``invocation`` first, read off the
|
|
473
|
+
``deal reconcile`` JSON ``reconcile``; then each of ``layers`` from
|
|
474
|
+
``observations``. ``not_applicable`` names (action, layer) pairs the
|
|
475
|
+
caller saw the layer not apply to (for example, no disposition was
|
|
476
|
+
returned), each counted as excluded, never as a claim.
|
|
477
|
+
``executions_sha256`` is the SHA-256 of the executions file the
|
|
478
|
+
reconcile pass read; reconcile's JSON does not carry it."""
|
|
479
|
+
_check_rfc3339("generated_at", generated_at)
|
|
480
|
+
if not tool_name or not tool_version:
|
|
481
|
+
raise _tally_error("tool_name and tool_version must be non-empty: the version under test is part of the artifact")
|
|
482
|
+
if invocation.gate is not None or invocation.tier != "recomputed" or not invocation.self_reported:
|
|
483
|
+
raise _tally_error("the invocation layer is recomputed from the reconcile report, self-reported, and has no gate")
|
|
484
|
+
DigestRef(digest=executions_sha256)
|
|
485
|
+
report = ReconcileReport.from_json(reconcile)
|
|
486
|
+
actions = report.action_ids
|
|
487
|
+
|
|
488
|
+
known = {invocation.layer_id}
|
|
489
|
+
specs = {spec.layer_id: spec for spec in layers}
|
|
490
|
+
for spec in layers:
|
|
491
|
+
if spec.layer_id in known:
|
|
492
|
+
raise _tally_error(f"layer {spec.layer_id!r} appears more than once")
|
|
493
|
+
if spec.gate not in known:
|
|
494
|
+
raise _tally_error(f"layer {spec.layer_id!r} must be gated on an earlier layer, got {spec.gate!r}")
|
|
495
|
+
known.add(spec.layer_id)
|
|
496
|
+
|
|
497
|
+
action_set = set(actions)
|
|
498
|
+
by_pair: dict[ActionLayer, LayerObservation] = {}
|
|
499
|
+
for obs in observations:
|
|
500
|
+
pair = ActionLayer(obs.action_id, obs.layer_id)
|
|
501
|
+
if obs.layer_id == invocation.layer_id:
|
|
502
|
+
raise _tally_error("the invocation layer is read from the reconcile report, never from an observation")
|
|
503
|
+
if obs.layer_id not in known or obs.action_id not in action_set:
|
|
504
|
+
raise _tally_error(f"observation for {pair} names no configured layer or no consequential action")
|
|
505
|
+
if pair in by_pair:
|
|
506
|
+
raise _tally_error(f"two observations for {pair}")
|
|
507
|
+
if specs[obs.layer_id].self_reported and obs.grade != "self-attested":
|
|
508
|
+
raise _tally_error(f"{pair} is on a self-reported layer and must be self-attested, got {obs.grade!r}")
|
|
509
|
+
by_pair[pair] = obs
|
|
510
|
+
skipped = {ActionLayer(*p) for p in not_applicable}
|
|
511
|
+
for pair in skipped:
|
|
512
|
+
if pair in by_pair:
|
|
513
|
+
raise _tally_error(f"{pair} is both observed and declared not applicable")
|
|
514
|
+
if pair.layer_id not in known or pair.layer_id == invocation.layer_id or pair.action_id not in action_set:
|
|
515
|
+
raise _tally_error(f"not_applicable entry {pair} names no gated layer or no consequential action")
|
|
516
|
+
|
|
517
|
+
# Per action, the verdict each layer reached, or None when excluded.
|
|
518
|
+
outcome: dict[str, dict[str, str | None]] = {a: {} for a in actions}
|
|
519
|
+
invocation_claims = _invocation_claims(report, invocation, contract_ref, executions_sha256)
|
|
520
|
+
for action_id, claim in invocation_claims.items():
|
|
521
|
+
outcome[action_id][invocation.layer_id] = claim.verdict
|
|
522
|
+
aggregates = [
|
|
523
|
+
_layer_aggregate(
|
|
524
|
+
invocation, list(invocation_claims.values()), report.counts["not_consequential"], generated_at
|
|
525
|
+
)
|
|
526
|
+
]
|
|
527
|
+
|
|
528
|
+
for spec in layers:
|
|
529
|
+
claims: list[Claim] = []
|
|
530
|
+
excluded = 0
|
|
531
|
+
for action_id in actions:
|
|
532
|
+
pair = ActionLayer(action_id, spec.layer_id)
|
|
533
|
+
gate = outcome[action_id][spec.gate]
|
|
534
|
+
obs = by_pair.get(pair)
|
|
535
|
+
if gate is None or gate == "not_met" or pair in skipped:
|
|
536
|
+
if obs is not None:
|
|
537
|
+
raise _tally_error(f"{pair} is observed, but its gate layer {spec.gate!r} did not reach it")
|
|
538
|
+
outcome[action_id][spec.layer_id] = None
|
|
539
|
+
excluded += 1
|
|
540
|
+
continue
|
|
541
|
+
if gate == "not_evaluable":
|
|
542
|
+
if obs is not None:
|
|
543
|
+
raise _tally_error(f"{pair} is observed, but whether its gate layer {spec.gate!r} applied is unsettled")
|
|
544
|
+
claim = _unknown_claim(
|
|
545
|
+
spec, action_id, contract_ref, f"whether this layer applied is unsettled: layer {spec.gate} did not settle"
|
|
546
|
+
)
|
|
547
|
+
elif obs is None:
|
|
548
|
+
claim = _unknown_claim(spec, action_id, contract_ref, "no observation was recorded for this action at this layer")
|
|
549
|
+
else:
|
|
550
|
+
claim = _observed_claim(spec, obs, contract_ref)
|
|
551
|
+
outcome[action_id][spec.layer_id] = claim.verdict
|
|
552
|
+
claims.append(claim)
|
|
553
|
+
aggregates.append(_layer_aggregate(spec, claims, excluded, generated_at))
|
|
554
|
+
|
|
555
|
+
return LayerTally(
|
|
556
|
+
generated_at=generated_at,
|
|
557
|
+
tool_name=tool_name,
|
|
558
|
+
tool_version=tool_version,
|
|
559
|
+
executions_sha256=executions_sha256,
|
|
560
|
+
reconcile=report,
|
|
561
|
+
layers=tuple(aggregates),
|
|
562
|
+
)
|
|
563
|
+
|
|
564
|
+
|
|
565
|
+
def _recount(claims: Sequence[Mapping[str, object]], excluded: int) -> CoverageCounts:
|
|
566
|
+
return CoverageCounts(
|
|
567
|
+
evaluated_population=len(claims),
|
|
568
|
+
excluded_not_applicable=excluded,
|
|
569
|
+
unknown_count=sum(1 for c in claims if c.get("sufficiency") == "UNKNOWN"),
|
|
570
|
+
)
|
|
571
|
+
|
|
572
|
+
|
|
573
|
+
def verify_layer_tally(doc: Mapping[str, object]) -> None:
|
|
574
|
+
"""Re-check a ``layer-tally/v0`` document: each layer's Result passes
|
|
575
|
+
the schema and ``verify_result``, and each layer's coverage equals the
|
|
576
|
+
count of its own claims -- a hand-set ``evaluated_population`` or
|
|
577
|
+
``unknown_count`` is refused, wherever it was edited. Raises
|
|
578
|
+
``ResultError`` on the first violation."""
|
|
579
|
+
if doc.get("tally_version") != TALLY_VERSION:
|
|
580
|
+
raise _tally_error(f"tally_version must be {TALLY_VERSION!r}")
|
|
581
|
+
generated_at = doc.get("generated_at")
|
|
582
|
+
if not isinstance(generated_at, str):
|
|
583
|
+
raise _tally_error("generated_at is missing")
|
|
584
|
+
_check_rfc3339("generated_at", generated_at)
|
|
585
|
+
tool = doc.get("tool")
|
|
586
|
+
if not isinstance(tool, dict) or not tool.get("name") or not tool.get("version"):
|
|
587
|
+
raise _tally_error("tool.name and tool.version are required")
|
|
588
|
+
source = doc.get("source")
|
|
589
|
+
coverage = source.get("coverage") if isinstance(source, dict) else None
|
|
590
|
+
cannot_see = coverage.get("cannot_see") if isinstance(coverage, dict) else None
|
|
591
|
+
if not isinstance(cannot_see, list) or not cannot_see:
|
|
592
|
+
raise _tally_error("source.coverage.cannot_see must be carried, non-empty")
|
|
593
|
+
executions = source.get("executions_sha256") if isinstance(source, dict) else None
|
|
594
|
+
if not isinstance(executions, str) or not _HEX64.fullmatch(executions):
|
|
595
|
+
raise _tally_error("source.executions_sha256 must be a SHA-256 hex digest")
|
|
596
|
+
layers = doc.get("layers")
|
|
597
|
+
if not isinstance(layers, list) or not layers:
|
|
598
|
+
raise _tally_error("layers must be a non-empty list")
|
|
599
|
+
seen: set[str] = set()
|
|
600
|
+
for entry in layers:
|
|
601
|
+
layer_id = entry.get("layer")
|
|
602
|
+
if layer_id in seen:
|
|
603
|
+
raise _tally_error(f"layer {layer_id!r} appears more than once")
|
|
604
|
+
seen.add(layer_id)
|
|
605
|
+
stated = entry.get("coverage")
|
|
606
|
+
result = entry.get("result")
|
|
607
|
+
if result is None:
|
|
608
|
+
if not isinstance(stated, dict) or stated.get("evaluated_population") != 0 or stated.get("unknown_count") != 0:
|
|
609
|
+
raise ResultError(COVERAGE_NOT_COMPUTED, f"layer {layer_id!r} has no result, so its stated population must be 0")
|
|
610
|
+
continue
|
|
611
|
+
validate_against_schema(result)
|
|
612
|
+
verify_result(result)
|
|
613
|
+
if result["generated_at"] != generated_at:
|
|
614
|
+
raise _tally_error(f"layer {layer_id!r} result generated_at differs from the tally's")
|
|
615
|
+
claims = result["claims"]
|
|
616
|
+
# excluded_not_applicable is the one count a caller supplies (spec
|
|
617
|
+
# section 3); the schema check above already made it an integer >= 0.
|
|
618
|
+
recount = _recount(claims, result["aggregate"]["coverage"]["excluded_not_applicable"])
|
|
619
|
+
if result["aggregate"]["coverage"] != recount or stated != recount:
|
|
620
|
+
raise ResultError(
|
|
621
|
+
COVERAGE_NOT_COMPUTED,
|
|
622
|
+
f"layer {layer_id!r} coverage does not recount from its claims: stated {stated}, aggregate "
|
|
623
|
+
f"{result['aggregate']['coverage']}, recounted {recount}",
|
|
624
|
+
)
|
|
625
|
+
tier = entry.get("tier")
|
|
626
|
+
pin = entry.get("judge_pin")
|
|
627
|
+
if tier == "judged" and not pin:
|
|
628
|
+
raise _tally_error(f"layer {layer_id!r} is judged and names no judge pin")
|
|
629
|
+
if tier == "recomputed" and pin is not None:
|
|
630
|
+
raise _tally_error(f"layer {layer_id!r} is recomputed and carries a judge pin")
|
|
631
|
+
self_reported = entry.get("self_reported")
|
|
632
|
+
if not isinstance(self_reported, bool):
|
|
633
|
+
raise _tally_error(f"layer {layer_id!r} must state self_reported")
|
|
634
|
+
# The invocation layer is the one with no gate: it is a lower bound
|
|
635
|
+
# over the executions file, which every one of its claims cites first.
|
|
636
|
+
if "gate" not in entry and (not self_reported or any(
|
|
637
|
+
not c["evidence"] or c["evidence"][0]["digest"] != executions for c in claims
|
|
638
|
+
)):
|
|
639
|
+
raise _tally_error(f"invocation layer {layer_id!r} must be self-reported and cite the executions file first")
|
|
640
|
+
for claim in claims:
|
|
641
|
+
if self_reported and claim["grade"] != "self-attested":
|
|
642
|
+
raise _tally_error(f"claim {claim['id']!r} is on a self-reported layer and graded {claim['grade']!r}")
|
|
643
|
+
if claim["tier"] != tier:
|
|
644
|
+
raise _tally_error(f"claim {claim['id']!r} tier {claim['tier']!r} differs from layer tier {tier!r}")
|
|
645
|
+
if pin is not None and (not claim["evidence"] or claim["evidence"][0]["digest"] != pin):
|
|
646
|
+
raise _tally_error(f"claim {claim['id']!r} does not carry the layer's judge pin")
|
|
647
|
+
|
|
648
|
+
|
|
649
|
+
def render_layer_tally(doc: LayerTallyDoc) -> list[str]:
|
|
650
|
+
"""Plain lines, one per layer, each with its own denominator, the date
|
|
651
|
+
and the tool version, then every ``cannot_see`` line. A layer with no
|
|
652
|
+
evaluated action says so; it is never shown as a pass."""
|
|
653
|
+
tool = doc["tool"]
|
|
654
|
+
stamp = f"{doc['generated_at']} · {tool['name']} {tool['version']}"
|
|
655
|
+
lines = []
|
|
656
|
+
for entry in doc["layers"]:
|
|
657
|
+
cov = entry["coverage"]
|
|
658
|
+
excluded = f"{cov['excluded_not_applicable']} excluded as not applicable"
|
|
659
|
+
if entry["result"] is None:
|
|
660
|
+
lines.append(f"{entry['layer']}: 0 evaluated (denominator 0), no result · {excluded} · {entry['tier']} · {stamp}")
|
|
661
|
+
continue
|
|
662
|
+
buckets = entry["result"]["aggregate"]["buckets"]
|
|
663
|
+
lines.append(
|
|
664
|
+
f"{entry['layer']}: {len(buckets['met'])} met of {cov['evaluated_population']} evaluated · "
|
|
665
|
+
f"{len(buckets['not_met'])} not met · {len(buckets['not_evaluable'])} not evaluable "
|
|
666
|
+
f"({cov['unknown_count']} unknown) · {excluded} · {entry['tier']} · {stamp}"
|
|
667
|
+
)
|
|
668
|
+
lines.append("cannot see:")
|
|
669
|
+
lines.extend(f"- {gap}" for gap in doc["source"]["coverage"]["cannot_see"])
|
|
670
|
+
return lines
|