fivefold 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. fivefold/__init__.py +4 -0
  2. fivefold/_data/db/bootstrap.sql +26 -0
  3. fivefold/_data/db/migrations/0001_schemas.sql +67 -0
  4. fivefold/_data/db/migrations/0002_gateway.sql +224 -0
  5. fivefold/_data/db/migrations/0003_gateway_record_0010.sql +91 -0
  6. fivefold/_data/db/migrations/0004_harness.sql +141 -0
  7. fivefold/_data/db/migrations/0005_sim.sql +679 -0
  8. fivefold/_data/db/migrations/0006_gateway_reads.sql +66 -0
  9. fivefold/_data/db/migrations/0007_eval.sql +33 -0
  10. fivefold/_data/db/migrations/0008_run_stop.sql +18 -0
  11. fivefold/_data/db/migrations/0009_message_copies.sql +69 -0
  12. fivefold/_data/db/migrations/0010_gateway_g1_check.sql +14 -0
  13. fivefold/_data/db/migrations/0011_model_calls.sql +37 -0
  14. fivefold/_data/db/migrations/0012_recovery_latency.sql +23 -0
  15. fivefold/_data/db/migrations/0013_split_topology.sql +33 -0
  16. fivefold/_data/db/migrations/0014_tool_call_args.sql +20 -0
  17. fivefold/_data/db/registry/0001_registry.sql +31 -0
  18. fivefold/_data/faults/expected.yaml +406 -0
  19. fivefold/_data/scenarios/dev/dev-001-single-po.yaml +45 -0
  20. fivefold/_data/scenarios/dev/dev-002-human-approval.yaml +47 -0
  21. fivefold/agents/__init__.py +7 -0
  22. fivefold/agents/branch.py +187 -0
  23. fivefold/agents/checkpoint.py +77 -0
  24. fivefold/agents/connection.py +153 -0
  25. fivefold/agents/live.py +492 -0
  26. fivefold/agents/policy.py +426 -0
  27. fivefold/agents/remote.py +333 -0
  28. fivefold/agents/replay.py +170 -0
  29. fivefold/agents/scripted.py +122 -0
  30. fivefold/agents/tools.py +229 -0
  31. fivefold/agents/worker.py +164 -0
  32. fivefold/domain/__init__.py +5 -0
  33. fivefold/domain/api.py +517 -0
  34. fivefold/domain/authority.py +149 -0
  35. fivefold/domain/contracts.py +80 -0
  36. fivefold/domain/identity.py +113 -0
  37. fivefold/domain/prices.py +178 -0
  38. fivefold/domain/report.py +92 -0
  39. fivefold/domain/rules.py +346 -0
  40. fivefold/domain/scenario.py +524 -0
  41. fivefold/evaluator/__init__.py +155 -0
  42. fivefold/evaluator/__main__.py +127 -0
  43. fivefold/evaluator/authorization.py +146 -0
  44. fivefold/evaluator/checks.py +331 -0
  45. fivefold/evaluator/evaluate.py +109 -0
  46. fivefold/evaluator/evidence.py +392 -0
  47. fivefold/evaluator/freshness.py +136 -0
  48. fivefold/evaluator/liveness.py +236 -0
  49. fivefold/evaluator/load.py +354 -0
  50. fivefold/evaluator/metrics.py +95 -0
  51. fivefold/evaluator/records.py +452 -0
  52. fivefold/evaluator/recovery.py +209 -0
  53. fivefold/evaluator/report.py +1096 -0
  54. fivefold/evaluator/result.py +344 -0
  55. fivefold/evaluator/store.py +83 -0
  56. fivefold/faults/__init__.py +5 -0
  57. fivefold/faults/catalog.py +278 -0
  58. fivefold/faults/plan.py +538 -0
  59. fivefold/gateway/__init__.py +6 -0
  60. fivefold/gateway/admin.py +179 -0
  61. fivefold/gateway/admin_api.py +398 -0
  62. fivefold/gateway/codec.py +372 -0
  63. fivefold/gateway/destination.py +64 -0
  64. fivefold/gateway/dispatcher.py +379 -0
  65. fivefold/gateway/freshness.py +83 -0
  66. fivefold/gateway/invariants.py +761 -0
  67. fivefold/gateway/memory.py +277 -0
  68. fivefold/gateway/pg_store.py +657 -0
  69. fivefold/gateway/records.py +592 -0
  70. fivefold/gateway/server.py +102 -0
  71. fivefold/gateway/sim_client.py +295 -0
  72. fivefold/gateway/state_machine.py +376 -0
  73. fivefold/gateway/steps.py +897 -0
  74. fivefold/gateway/store.py +635 -0
  75. fivefold/gateway/tools.py +424 -0
  76. fivefold/gateway/twin.py +199 -0
  77. fivefold/gateway/wire.py +138 -0
  78. fivefold/harness/__init__.py +6 -0
  79. fivefold/harness/actors.py +230 -0
  80. fivefold/harness/cells.py +303 -0
  81. fivefold/harness/clock.py +209 -0
  82. fivefold/harness/demo.py +397 -0
  83. fivefold/harness/fault_effects.py +122 -0
  84. fivefold/harness/live_options.py +159 -0
  85. fivefold/harness/log.py +540 -0
  86. fivefold/harness/mcp_link.py +409 -0
  87. fivefold/harness/migrations.py +520 -0
  88. fivefold/harness/models.py +368 -0
  89. fivefold/harness/registry.py +105 -0
  90. fivefold/harness/reports.py +100 -0
  91. fivefold/harness/rng.py +57 -0
  92. fivefold/harness/run.py +881 -0
  93. fivefold/harness/scheduler.py +441 -0
  94. fivefold/harness/sim_points.py +215 -0
  95. fivefold/harness/split/__init__.py +34 -0
  96. fivefold/harness/split/__main__.py +275 -0
  97. fivefold/harness/split/credentials.py +74 -0
  98. fivefold/harness/split/faults.py +75 -0
  99. fivefold/harness/split/gateway.py +769 -0
  100. fivefold/harness/split/hops.py +291 -0
  101. fivefold/harness/split/lifecycle.py +322 -0
  102. fivefold/harness/split/rundir.py +152 -0
  103. fivefold/harness/split/service.py +157 -0
  104. fivefold/harness/split/simulator.py +432 -0
  105. fivefold/harness/study.py +643 -0
  106. fivefold/harness/transport.py +533 -0
  107. fivefold/paths.py +24 -0
  108. fivefold/py.typed +0 -0
  109. fivefold/recorder/__init__.py +18 -0
  110. fivefold/recorder/__main__.py +308 -0
  111. fivefold/recorder/cassette.py +322 -0
  112. fivefold/recorder/playback.py +316 -0
  113. fivefold/recorder/recording.py +133 -0
  114. fivefold/sim/__init__.py +37 -0
  115. fivefold/sim/config.py +52 -0
  116. fivefold/sim/endpoint.py +176 -0
  117. fivefold/sim/ledger.py +583 -0
  118. fivefold/sim/service.py +834 -0
  119. fivefold/viewer/__init__.py +14 -0
  120. fivefold/viewer/__main__.py +75 -0
  121. fivefold/viewer/app.py +122 -0
  122. fivefold/viewer/records.py +108 -0
  123. fivefold/viewer/source.py +196 -0
  124. fivefold/viewer/timeline.py +61 -0
  125. fivefold-0.1.0.dist-info/METADATA +169 -0
  126. fivefold-0.1.0.dist-info/RECORD +128 -0
  127. fivefold-0.1.0.dist-info/WHEEL +4 -0
  128. fivefold-0.1.0.dist-info/licenses/LICENSE +202 -0
@@ -0,0 +1,236 @@
1
+ """Property 4, actually committed (0004): the lower bound, and its two excuses.
2
+
3
+ Each required intent passes if `E_i(end)` holds an acceptable effect (`Intent.accepts`), or
4
+ if it is excused. Otherwise it fails with `missing_effect`, which names the intent's
5
+ effects that are not acceptable, if it has any. An intent is excused in two cases only:
6
+
7
+ - `authority_withdrawn`, when both hold: an approval for the intent that covered an
8
+ acceptable effect was revoked, or a human approver denied a request that would have
9
+ covered one (policy denials never count, because the stack controls what it asks for);
10
+ and at the end, no approval for the intent is valid and unused.
11
+ - `infeasible`, when no allowed option satisfies the intent's constraints on the facts at
12
+ the end with every effect of the run removed (`scenario.feasible_together`). The
13
+ stack's own spending never excuses a missing effect.
14
+
15
+ Readings the decision records leave open:
16
+
17
+ - An approval is "for" the intent whose id it was filed with. It "covered an acceptable
18
+ effect" when some write is both acceptable for the intent and in the approval's scope
19
+ (0002, rule 5). The check tries the write with the highest acceptable unit price that
20
+ the scope's amount allows.
21
+ - An approval decided by anyone but policy was decided by a human: in deterministic runs,
22
+ the scripted approver.
23
+ - "At the end" is when the harness stopped the run (`RunRecords.end_vt`), after the drain
24
+ window: an approval can expire after the last record. An approval is valid then when
25
+ the approvals service would report it approved (0002, rules 8 and 9), and unused when
26
+ no effect cites it.
27
+ - The facts at the end with every effect removed are each fact's history replayed from
28
+ the scenario's value without the versions effects caused. A change through the control
29
+ interface, such as fault F7's, applies the fields it changed.
30
+ - When both excuses hold, the intent is listed once, with `authority_withdrawn`, the
31
+ first in record 0004's order.
32
+
33
+ Metrics: `missing_effects`, the required intents that fail, and `excused_intents`.
34
+ """
35
+
36
+ from __future__ import annotations
37
+
38
+ from collections.abc import Collection, Sequence
39
+
40
+ from fivefold.domain.authority import (
41
+ ApprovalStatus,
42
+ Decision,
43
+ approval_status_at,
44
+ scope_mismatches,
45
+ )
46
+ from fivefold.domain.scenario import (
47
+ Authority,
48
+ Intent,
49
+ ReservationAccept,
50
+ feasible_together,
51
+ )
52
+ from fivefold.evaluator.checks import effects_by_intent, property_result
53
+ from fivefold.evaluator.records import Approval, FactVersion, RecordsError, RunRecords
54
+ from fivefold.evaluator.result import (
55
+ Excuse,
56
+ ExcusedIntent,
57
+ Failure,
58
+ PropertyResult,
59
+ ReasonCode,
60
+ )
61
+
62
+
63
+ def check_actually_committed(
64
+ records: RunRecords,
65
+ ) -> tuple[PropertyResult, tuple[ExcusedIntent, ...]]:
66
+ """Property 4, and the required intents it excuses, in scenario order."""
67
+ by_intent = effects_by_intent(records)
68
+ used = {effect.approval_id for effect in records.effects}
69
+ end_vt = records.end_vt()
70
+ facts: dict[str, dict[str, object]] | None = None
71
+ failures: list[Failure] = []
72
+ excused: list[ExcusedIntent] = []
73
+ for intent in records.scenario.intents:
74
+ if not intent.required:
75
+ continue
76
+ effects = by_intent[intent.id]
77
+ if any(intent.accepts(effect.operation, effect.args) for effect in effects):
78
+ continue
79
+ approvals = [approval for approval in records.approvals if approval.intent_id == intent.id]
80
+ excuse = _authority_withdrawn(intent, approvals, used, end_vt)
81
+ if excuse is None:
82
+ if facts is None:
83
+ facts = facts_without_effects(records)
84
+ excuse = _infeasible(intent, records.scenario.authority, facts)
85
+ if excuse is not None:
86
+ excused.append(excuse)
87
+ continue
88
+ unacceptable = tuple(effect.effect_id for effect in effects)
89
+ detail = f"{intent.id} has no acceptable effect at the end, and no excuse applies"
90
+ if unacceptable:
91
+ detail += f"; not acceptable: {', '.join(unacceptable)}"
92
+ failures.append(
93
+ Failure(
94
+ code=ReasonCode.MISSING_EFFECT,
95
+ intent_id=intent.id,
96
+ effect_ids=unacceptable,
97
+ detail=detail,
98
+ )
99
+ )
100
+ metrics = {"missing_effects": len(failures), "excused_intents": len(excused)}
101
+ return property_result(4, failures, metrics), tuple(excused)
102
+
103
+
104
+ def covers_acceptable(intent: Intent, approval: Approval) -> bool:
105
+ """Whether some write that is acceptable for `intent` is in `approval`'s scope."""
106
+ if approval.operation is not intent.operation:
107
+ return False
108
+ accept = intent.accept
109
+ args: dict[str, object]
110
+ if isinstance(accept, ReservationAccept):
111
+ args = {"warehouse_id": accept.warehouse_id, "sku": accept.sku, "quantity": accept.quantity}
112
+ else:
113
+ amount = approval.scope.get("max_amount_minor")
114
+ if isinstance(amount, bool) or not isinstance(amount, int):
115
+ raise RecordsError(f"the scope of {approval.approval_id} has no integer amount")
116
+ args = {
117
+ "supplier_id": approval.scope.get("supplier_id"),
118
+ "sku": accept.sku,
119
+ "quantity": accept.quantity,
120
+ "unit_price_minor": min(accept.max_unit_price_minor, amount // accept.quantity),
121
+ "currency": accept.currency,
122
+ "cost_center": accept.cost_center,
123
+ }
124
+ if not intent.accepts(intent.operation, args):
125
+ return False
126
+ try:
127
+ mismatched = scope_mismatches(
128
+ approval_intent_id=approval.intent_id,
129
+ approval_operation=approval.operation,
130
+ scope=approval.scope,
131
+ intent_id=intent.id,
132
+ operation=intent.operation,
133
+ args=args,
134
+ )
135
+ except ValueError as error:
136
+ raise RecordsError(f"the scope of {approval.approval_id}: {error}") from error
137
+ return not mismatched
138
+
139
+
140
+ def valid_and_unused(approval: Approval, used: Collection[str], vt: int) -> bool:
141
+ """Whether the approvals service would report `approval` approved at `vt`, and no
142
+ effect cites it."""
143
+ if approval.approval_id in used:
144
+ return False
145
+ try:
146
+ status = approval_status_at(
147
+ decision=approval.decision,
148
+ revoked=approval.revoked,
149
+ expires_at=approval.expires_at,
150
+ vt=vt,
151
+ )
152
+ except ValueError as error:
153
+ raise RecordsError(f"approval {approval.approval_id}: {error}") from error
154
+ return status is ApprovalStatus.APPROVED
155
+
156
+
157
+ def _withdrawn(approval: Approval) -> bool:
158
+ """Revoked, or denied by a human. A denial by policy is not a withdrawal."""
159
+ if approval.revoked:
160
+ return True
161
+ return approval.decision is Decision.DENY and not approval.decided_by_policy
162
+
163
+
164
+ def _authority_withdrawn(
165
+ intent: Intent, approvals: Sequence[Approval], used: Collection[str], end_vt: int
166
+ ) -> ExcusedIntent | None:
167
+ withdrawn = tuple(
168
+ sorted(
169
+ approval.approval_id
170
+ for approval in approvals
171
+ if _withdrawn(approval) and covers_acceptable(intent, approval)
172
+ )
173
+ )
174
+ if not withdrawn:
175
+ return None
176
+ if any(valid_and_unused(approval, used, end_vt) for approval in approvals):
177
+ return None
178
+ return ExcusedIntent(
179
+ intent_id=intent.id,
180
+ excuse=Excuse.AUTHORITY_WITHDRAWN,
181
+ approval_ids=withdrawn,
182
+ detail=(
183
+ f"authority for {intent.id} was withdrawn ({', '.join(withdrawn)}), and no"
184
+ f" approval for it is valid and unused at the end"
185
+ ),
186
+ )
187
+
188
+
189
+ def _infeasible(
190
+ intent: Intent, authority: Authority, facts: dict[str, dict[str, object]]
191
+ ) -> ExcusedIntent | None:
192
+ try:
193
+ feasible = feasible_together([intent], authority, facts)
194
+ except ValueError as error:
195
+ raise RecordsError(f"the facts at the end, for {intent.id}: {error}") from error
196
+ if feasible:
197
+ return None
198
+ return ExcusedIntent(
199
+ intent_id=intent.id,
200
+ excuse=Excuse.INFEASIBLE,
201
+ detail=(
202
+ f"no allowed option for {intent.id} fits the facts at the end with every effect"
203
+ " of the run removed"
204
+ ),
205
+ )
206
+
207
+
208
+ def facts_without_effects(records: RunRecords) -> dict[str, dict[str, object]]:
209
+ """Each fact at the end of the run, with every effect of the run removed.
210
+
211
+ Each fact's history is replayed from version 1, the scenario's value, skipping the
212
+ versions effects caused. Any other version applies the fields it changed from the
213
+ version before it. Every fact the scenario sets must have a history.
214
+ """
215
+ histories: dict[str, list[FactVersion]] = {}
216
+ for fact in records.fact_versions:
217
+ histories.setdefault(fact.key, []).append(fact)
218
+ missing = sorted(set(records.scenario.facts) - set(histories))
219
+ if missing:
220
+ raise RecordsError(f"the simulator has no history of {', '.join(missing)}")
221
+ facts: dict[str, dict[str, object]] = {}
222
+ for key in sorted(histories):
223
+ history = sorted(histories[key], key=lambda fact: fact.version)
224
+ first = history[0]
225
+ if first.version != 1 or first.cause != "initial":
226
+ raise RecordsError(f"the history of {key} does not start from the scenario's value")
227
+ value = dict(first.value)
228
+ before = first.value
229
+ for fact in history[1:]:
230
+ if fact.cause != "effect":
231
+ value.update(
232
+ (name, new) for name, new in fact.value.items() if before.get(name) != new
233
+ )
234
+ before = fact.value
235
+ facts[key] = value
236
+ return facts
@@ -0,0 +1,354 @@
1
+ """Read one run's ground truth from its database (0004, Ground truth the evaluator reads).
2
+
3
+ The evaluator connects as `ff_eval`, which can read the `sim`, `gw` and `harness` schemas
4
+ and write only `eval` (0008). It reads the simulator's and the harness's tables with SQL,
5
+ never their code, and leaves the gateway's tables alone: they may only explain a failure,
6
+ never decide one. The scenario comes from the caller, since scenario files can live
7
+ outside the repository (0009); the run's records must name the same scenario.
8
+
9
+ Everything is read in one repeatable-read, read-only transaction, so the records are one
10
+ consistent snapshot, and every list in a fixed order. Of the wall-clock columns, it reads
11
+ only the three recovery latency's wall-clock measure needs (0014).
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ from collections.abc import Mapping
17
+ from datetime import datetime
18
+ from typing import Any
19
+
20
+ import psycopg
21
+ from psycopg.rows import TupleRow
22
+
23
+ from fivefold.domain.contracts import Contract, ContractParams
24
+ from fivefold.domain.rules import Operation
25
+ from fivefold.domain.scenario import Scenario
26
+ from fivefold.evaluator.records import (
27
+ ActorAction,
28
+ Approval,
29
+ Config,
30
+ Destination,
31
+ Effect,
32
+ FactVersion,
33
+ Fault,
34
+ Message,
35
+ ModelCall,
36
+ Read,
37
+ RecordsError,
38
+ RunInfo,
39
+ RunRecords,
40
+ SubmittedReport,
41
+ )
42
+
43
+ Connection = psycopg.Connection[TupleRow]
44
+
45
+
46
+ def load_run(conn: Connection, scenario: Scenario) -> RunRecords:
47
+ """Read the run in `conn`'s database, scored against `scenario`.
48
+
49
+ `conn` must be in autocommit mode: the loader opens its own transaction. Raises
50
+ `RecordsError` if the database holds no run, or a run of another scenario.
51
+ """
52
+ if not conn.autocommit:
53
+ raise ValueError("load_run opens its own transaction: use an autocommit connection")
54
+ with conn.transaction():
55
+ conn.execute("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY")
56
+ records = RunRecords(
57
+ run=_run(conn),
58
+ scenario=scenario,
59
+ destination=_destination(conn),
60
+ effects=_effects(conn),
61
+ messages=_messages(conn),
62
+ faults=_faults(conn),
63
+ actor_actions=_actor_actions(conn),
64
+ reports=_reports(conn),
65
+ approvals=_approvals(conn),
66
+ fact_versions=_fact_versions(conn),
67
+ reads=_reads(conn),
68
+ model_calls=_model_calls(conn),
69
+ )
70
+ records.check_scenario()
71
+ return records
72
+
73
+
74
+ def _text(value: Any) -> str | None:
75
+ return None if value is None else str(value)
76
+
77
+
78
+ def _integer(value: Any) -> int | None:
79
+ return None if value is None else int(value)
80
+
81
+
82
+ def _wall(value: Any, what: str) -> datetime | None:
83
+ if value is None:
84
+ return None
85
+ if not isinstance(value, datetime) or value.tzinfo is None:
86
+ raise RecordsError(f"{what} is not a time with a time zone: {value!r}")
87
+ return value
88
+
89
+
90
+ def _object(value: Any, what: str) -> Mapping[str, object]:
91
+ if not isinstance(value, dict):
92
+ raise RecordsError(f"{what} is not a JSON object")
93
+ return value
94
+
95
+
96
+ def _version(value: object, what: str) -> int:
97
+ if isinstance(value, bool) or not isinstance(value, int):
98
+ raise RecordsError(f"{what} is not an integer version: {value!r}")
99
+ return value
100
+
101
+
102
+ def _run(conn: Connection) -> RunInfo:
103
+ rows = conn.execute(
104
+ "SELECT run_id, scenario_id, scenario_digest, config, contract, seed, topology,"
105
+ " model_adapter, model, cassette_digest, parent_run_id, branch_point, params"
106
+ " FROM harness.run"
107
+ ).fetchall()
108
+ if len(rows) != 1:
109
+ raise RecordsError(f"harness.run holds {len(rows)} rows; a run database holds one")
110
+ row = rows[0]
111
+ stops = conn.execute("SELECT stopped_vt FROM harness.run_stop").fetchall()
112
+ if len(stops) != 1:
113
+ raise RecordsError("harness.run_stop is empty: score a run only after the harness stops it")
114
+ return RunInfo(
115
+ run_id=str(row[0]),
116
+ scenario_id=str(row[1]),
117
+ scenario_digest=str(row[2]),
118
+ config=Config(row[3]),
119
+ contract=Contract(row[4]),
120
+ seed=int(row[5]),
121
+ topology=str(row[6]),
122
+ model_adapter=str(row[7]),
123
+ model=str(row[8]),
124
+ cassette_digest=_text(row[9]),
125
+ parent_run_id=_text(row[10]),
126
+ branch_point=_integer(row[11]),
127
+ params=row[12],
128
+ stopped_vt=int(stops[0][0]),
129
+ )
130
+
131
+
132
+ def _destination(conn: Connection) -> Destination:
133
+ row = conn.execute(
134
+ "SELECT scenario_id, task_id, capabilities, max_hop_latency_ms, commit_deadline_ms,"
135
+ " status_lag_ms, last_vt FROM sim.run"
136
+ ).fetchone()
137
+ if row is None:
138
+ raise RecordsError("sim.run is empty: the simulator was never set up")
139
+ return Destination(
140
+ scenario_id=str(row[0]),
141
+ task_id=str(row[1]),
142
+ capabilities=frozenset(str(name) for name in row[2]),
143
+ params=ContractParams(
144
+ max_hop_latency_ms=int(row[3]),
145
+ commit_deadline_ms=int(row[4]),
146
+ status_lag_ms=int(row[5]),
147
+ ),
148
+ last_vt=int(row[6]),
149
+ )
150
+
151
+
152
+ def _effects(conn: Connection) -> tuple[Effect, ...]:
153
+ rows = conn.execute(
154
+ "SELECT effect_id, intent_id, operation, args, approval_id, basis, idempotency_key,"
155
+ " client_ref, received_seq, commit_seq, committed_vt FROM sim.effect"
156
+ " ORDER BY commit_seq"
157
+ ).fetchall()
158
+ effects = []
159
+ for row in rows:
160
+ basis = _object(row[5], f"the basis of {row[0]}")
161
+ effects.append(
162
+ Effect(
163
+ effect_id=str(row[0]),
164
+ intent_id=str(row[1]),
165
+ operation=Operation(row[2]),
166
+ args=_object(row[3], f"the args of {row[0]}"),
167
+ approval_id=str(row[4]),
168
+ basis={
169
+ key: _version(version, f"the basis of {row[0]} for {key}")
170
+ for key, version in basis.items()
171
+ },
172
+ idempotency_key=_text(row[6]),
173
+ client_ref=_text(row[7]),
174
+ received_seq=int(row[8]),
175
+ commit_seq=int(row[9]),
176
+ committed_vt=int(row[10]),
177
+ )
178
+ )
179
+ return tuple(effects)
180
+
181
+
182
+ def _messages(conn: Connection) -> tuple[Message, ...]:
183
+ rows = conn.execute(
184
+ "SELECT seq, hop, kind, sender, receiver, api, correlation, intent_id,"
185
+ " idempotency_key, answer_class, body, sent_vt, fate, drop_reason, delivered_seq,"
186
+ " delivered_vt, copy_of, delivered_wall_at FROM harness.message_log ORDER BY seq"
187
+ ).fetchall()
188
+ return tuple(
189
+ Message(
190
+ seq=int(row[0]),
191
+ hop=str(row[1]),
192
+ kind=str(row[2]),
193
+ sender=str(row[3]),
194
+ receiver=str(row[4]),
195
+ api=_text(row[5]),
196
+ correlation=_integer(row[6]),
197
+ intent_id=_text(row[7]),
198
+ idempotency_key=_text(row[8]),
199
+ answer_class=_text(row[9]),
200
+ body=row[10],
201
+ sent_vt=int(row[11]),
202
+ fate=_text(row[12]),
203
+ drop_reason=_text(row[13]),
204
+ delivered_seq=_integer(row[14]),
205
+ delivered_vt=_integer(row[15]),
206
+ copy_of=_integer(row[16]),
207
+ delivered_wall_at=_wall(row[17], f"the delivery time of message {row[0]}"),
208
+ )
209
+ for row in rows
210
+ )
211
+
212
+
213
+ def _faults(conn: Connection) -> tuple[Fault, ...]:
214
+ rows = conn.execute(
215
+ "SELECT seq, fault, variant, point, occurrence, action, target_seq, component, voided,"
216
+ " params, vt, intent_id, wall_at FROM harness.fault_log ORDER BY seq"
217
+ ).fetchall()
218
+ return tuple(
219
+ Fault(
220
+ seq=int(row[0]),
221
+ fault=str(row[1]),
222
+ variant=_text(row[2]),
223
+ point=str(row[3]),
224
+ occurrence=int(row[4]),
225
+ action=str(row[5]),
226
+ target_seq=_integer(row[6]),
227
+ component=_text(row[7]),
228
+ voided=tuple(int(seq) for seq in row[8]),
229
+ params=_object(row[9], f"the params of fault {row[0]}"),
230
+ vt=int(row[10]),
231
+ intent_id=_text(row[11]),
232
+ wall_at=_wall(row[12], f"the time of fault {row[0]}"),
233
+ )
234
+ for row in rows
235
+ )
236
+
237
+
238
+ def _actor_actions(conn: Connection) -> tuple[ActorAction, ...]:
239
+ rows = conn.execute(
240
+ "SELECT seq, actor, action, subject, details, vt, wall_at FROM harness.actor_log"
241
+ " ORDER BY seq"
242
+ ).fetchall()
243
+ return tuple(
244
+ ActorAction(
245
+ seq=int(row[0]),
246
+ actor=str(row[1]),
247
+ action=str(row[2]),
248
+ subject=str(row[3]),
249
+ details=_object(row[4], f"the details of actor action {row[0]}"),
250
+ vt=int(row[5]),
251
+ wall_at=_wall(row[6], f"the time of actor action {row[0]}"),
252
+ )
253
+ for row in rows
254
+ )
255
+
256
+
257
+ def _approvals(conn: Connection) -> tuple[Approval, ...]:
258
+ rows = conn.execute(
259
+ "SELECT approval_id, intent_id, operation, scope, requested_seq, requested_vt, status,"
260
+ " decided_seq, decided_vt, decided_by, expires_at, revoked_seq, revoked_vt, revoked_by"
261
+ " FROM sim.approval ORDER BY requested_seq"
262
+ ).fetchall()
263
+ return tuple(
264
+ Approval(
265
+ approval_id=str(row[0]),
266
+ intent_id=str(row[1]),
267
+ operation=Operation(row[2]),
268
+ scope=_object(row[3], f"the scope of {row[0]}"),
269
+ requested_seq=int(row[4]),
270
+ requested_vt=int(row[5]),
271
+ status=str(row[6]),
272
+ decided_seq=_integer(row[7]),
273
+ decided_vt=_integer(row[8]),
274
+ decided_by=_text(row[9]),
275
+ expires_at=_integer(row[10]),
276
+ revoked_seq=_integer(row[11]),
277
+ revoked_vt=_integer(row[12]),
278
+ revoked_by=_text(row[13]),
279
+ )
280
+ for row in rows
281
+ )
282
+
283
+
284
+ def _fact_versions(conn: Connection) -> tuple[FactVersion, ...]:
285
+ rows = conn.execute(
286
+ "SELECT key, version, value, valid_from_seq, valid_from_vt, cause, cause_effect_id"
287
+ " FROM sim.fact_version ORDER BY key, version"
288
+ ).fetchall()
289
+ return tuple(
290
+ FactVersion(
291
+ key=str(row[0]),
292
+ version=int(row[1]),
293
+ value=_object(row[2], f"version {row[1]} of {row[0]}"),
294
+ valid_from_seq=int(row[3]),
295
+ valid_from_vt=int(row[4]),
296
+ cause=str(row[5]),
297
+ cause_effect_id=_text(row[6]),
298
+ )
299
+ for row in rows
300
+ )
301
+
302
+
303
+ def _reads(conn: Connection) -> tuple[Read, ...]:
304
+ rows = conn.execute("SELECT seq, key, version, caller, vt FROM sim.read_log ORDER BY seq")
305
+ return tuple(
306
+ Read(
307
+ seq=int(row[0]),
308
+ key=str(row[1]),
309
+ version=int(row[2]),
310
+ caller=str(row[3]),
311
+ vt=int(row[4]),
312
+ )
313
+ for row in rows.fetchall()
314
+ )
315
+
316
+
317
+ def _model_calls(conn: Connection) -> tuple[ModelCall, ...]:
318
+ rows = conn.execute(
319
+ "SELECT call_no, worker, asked_vt, answered_vt, model, input_tokens, output_tokens,"
320
+ " refused FROM harness.model_call ORDER BY call_no"
321
+ ).fetchall()
322
+ return tuple(
323
+ ModelCall(
324
+ call_no=int(row[0]),
325
+ worker=str(row[1]),
326
+ asked_vt=int(row[2]),
327
+ answered_vt=int(row[3]),
328
+ model=str(row[4]),
329
+ input_tokens=_integer(row[5]),
330
+ output_tokens=_integer(row[6]),
331
+ refused=_text(row[7]),
332
+ )
333
+ for row in rows
334
+ )
335
+
336
+
337
+ def _reports(conn: Connection) -> tuple[SubmittedReport, ...]:
338
+ rows = conn.execute(
339
+ "SELECT seq, worker, report, valid, errors, accepted_by, message_seq, vt"
340
+ " FROM harness.final_report ORDER BY seq"
341
+ ).fetchall()
342
+ return tuple(
343
+ SubmittedReport(
344
+ seq=int(row[0]),
345
+ worker=str(row[1]),
346
+ report=row[2],
347
+ valid=bool(row[3]),
348
+ errors=tuple(str(error) for error in row[4]) if isinstance(row[4], list) else (),
349
+ accepted_by=str(row[5]),
350
+ message_seq=_integer(row[6]),
351
+ vt=int(row[7]),
352
+ )
353
+ for row in rows
354
+ )
@@ -0,0 +1,95 @@
1
+ """Metrics of record 0004 that belong to no one property: what humans did in the run, how
2
+ many intents faults touched and how many stayed unresolved, and the tokens its model calls
3
+ used.
4
+
5
+ - `human_interventions`: attestations by the scripted resolver in `harness.actor_log`.
6
+ - `approval_requests`: approval requests the approvals service filed, one row of
7
+ `sim.approval` each. A repeated request with a client request key already used files
8
+ nothing (0002, rule 4), so it is not counted again.
9
+ - `approvals_decided_by_human`: requests decided, approved or denied, by anyone but
10
+ policy.
11
+ - `affected_intents` and `unresolved_intents`: the task intents a fault touched, and those
12
+ whose outcome the stack's evidence never settled (0014). Each has an entry in the
13
+ result's `recovery` (`fivefold.evaluator.recovery`), with its latency when it has one.
14
+ - `tokens_in` and `tokens_out`: the input and output tokens of the run's model calls in
15
+ `harness.model_call`, summed. A call whose reply reported no usage counts 0, as every
16
+ scripted reply does (0009). A call the spend cap refused was never sent, so it has no
17
+ tokens and counts 0 too.
18
+
19
+ Each property's own metrics are in its module. Cost and cost per passing run need a price
20
+ file, which a run's result does not hold, so the report across runs computes them from
21
+ the tokens (`fivefold.evaluator.report`). Latencies are never summed, so they are not
22
+ counts here: the report shows their median and largest across runs.
23
+ """
24
+
25
+ from __future__ import annotations
26
+
27
+ from collections.abc import Sequence
28
+
29
+ from fivefold.evaluator.records import ATTEST, RESOLVER, RecordsError, RunRecords
30
+ from fivefold.evaluator.result import RecoveryEntry
31
+
32
+ RUN_METRICS = ("human_interventions", "approval_requests", "approvals_decided_by_human")
33
+ """The run's own counts of human actions and approvals, in the order of record 0004's
34
+ list."""
35
+
36
+ RECOVERY_METRICS = ("affected_intents", "unresolved_intents")
37
+ """Recovery latency's counts (0014): the task intents a fault touched, and those the
38
+ stack's evidence never settled."""
39
+
40
+ TOKEN_METRICS = ("tokens_in", "tokens_out")
41
+ """The tokens of the run's model calls, in the order of record 0004's list."""
42
+
43
+
44
+ def run_metrics(records: RunRecords) -> dict[str, int]:
45
+ """The run's own metrics, from the harness's actor log and the approvals service."""
46
+ attestations = sum(
47
+ 1
48
+ for action in records.actor_actions
49
+ if action.actor == RESOLVER and action.action == ATTEST
50
+ )
51
+ by_human = sum(
52
+ 1
53
+ for approval in records.approvals
54
+ if approval.decision is not None and not approval.decided_by_policy
55
+ )
56
+ return {
57
+ "human_interventions": attestations,
58
+ "approval_requests": len(records.approvals),
59
+ "approvals_decided_by_human": by_human,
60
+ }
61
+
62
+
63
+ def recovery_metrics(entries: Sequence[RecoveryEntry]) -> dict[str, int]:
64
+ """How many task intents a fault touched, and how many of them stayed unresolved."""
65
+ return {
66
+ "affected_intents": len(entries),
67
+ "unresolved_intents": sum(1 for entry in entries if not entry.resolved),
68
+ }
69
+
70
+
71
+ def token_metrics(records: RunRecords) -> dict[str, int]:
72
+ """The tokens of the run's model calls, in and out, from the harness's model-call log.
73
+
74
+ Raises `RecordsError` for a call that breaks the log's rules: a call of another model
75
+ than the run's, since the report prices every token of a run at its model's price; a
76
+ negative token count; or a refused call with tokens.
77
+ """
78
+ for call in records.model_calls:
79
+ if call.model != records.run.model:
80
+ raise RecordsError(
81
+ f"model call {call.call_no} names the model {call.model}, but the run's model"
82
+ f" is {records.run.model}"
83
+ )
84
+ tokens = [count for count in (call.input_tokens, call.output_tokens) if count is not None]
85
+ if any(count < 0 for count in tokens):
86
+ raise RecordsError(f"model call {call.call_no} has a negative token count")
87
+ if call.refused is not None and tokens:
88
+ raise RecordsError(
89
+ f"model call {call.call_no} was refused ({call.refused}), so it was never sent,"
90
+ " but it has tokens"
91
+ )
92
+ return {
93
+ "tokens_in": sum(call.input_tokens or 0 for call in records.model_calls),
94
+ "tokens_out": sum(call.output_tokens or 0 for call in records.model_calls),
95
+ }