fivefold 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fivefold/__init__.py +4 -0
- fivefold/_data/db/bootstrap.sql +26 -0
- fivefold/_data/db/migrations/0001_schemas.sql +67 -0
- fivefold/_data/db/migrations/0002_gateway.sql +224 -0
- fivefold/_data/db/migrations/0003_gateway_record_0010.sql +91 -0
- fivefold/_data/db/migrations/0004_harness.sql +141 -0
- fivefold/_data/db/migrations/0005_sim.sql +679 -0
- fivefold/_data/db/migrations/0006_gateway_reads.sql +66 -0
- fivefold/_data/db/migrations/0007_eval.sql +33 -0
- fivefold/_data/db/migrations/0008_run_stop.sql +18 -0
- fivefold/_data/db/migrations/0009_message_copies.sql +69 -0
- fivefold/_data/db/migrations/0010_gateway_g1_check.sql +14 -0
- fivefold/_data/db/migrations/0011_model_calls.sql +37 -0
- fivefold/_data/db/migrations/0012_recovery_latency.sql +23 -0
- fivefold/_data/db/migrations/0013_split_topology.sql +33 -0
- fivefold/_data/db/migrations/0014_tool_call_args.sql +20 -0
- fivefold/_data/db/registry/0001_registry.sql +31 -0
- fivefold/_data/faults/expected.yaml +406 -0
- fivefold/_data/scenarios/dev/dev-001-single-po.yaml +45 -0
- fivefold/_data/scenarios/dev/dev-002-human-approval.yaml +47 -0
- fivefold/agents/__init__.py +7 -0
- fivefold/agents/branch.py +187 -0
- fivefold/agents/checkpoint.py +77 -0
- fivefold/agents/connection.py +153 -0
- fivefold/agents/live.py +492 -0
- fivefold/agents/policy.py +426 -0
- fivefold/agents/remote.py +333 -0
- fivefold/agents/replay.py +170 -0
- fivefold/agents/scripted.py +122 -0
- fivefold/agents/tools.py +229 -0
- fivefold/agents/worker.py +164 -0
- fivefold/domain/__init__.py +5 -0
- fivefold/domain/api.py +517 -0
- fivefold/domain/authority.py +149 -0
- fivefold/domain/contracts.py +80 -0
- fivefold/domain/identity.py +113 -0
- fivefold/domain/prices.py +178 -0
- fivefold/domain/report.py +92 -0
- fivefold/domain/rules.py +346 -0
- fivefold/domain/scenario.py +524 -0
- fivefold/evaluator/__init__.py +155 -0
- fivefold/evaluator/__main__.py +127 -0
- fivefold/evaluator/authorization.py +146 -0
- fivefold/evaluator/checks.py +331 -0
- fivefold/evaluator/evaluate.py +109 -0
- fivefold/evaluator/evidence.py +392 -0
- fivefold/evaluator/freshness.py +136 -0
- fivefold/evaluator/liveness.py +236 -0
- fivefold/evaluator/load.py +354 -0
- fivefold/evaluator/metrics.py +95 -0
- fivefold/evaluator/records.py +452 -0
- fivefold/evaluator/recovery.py +209 -0
- fivefold/evaluator/report.py +1096 -0
- fivefold/evaluator/result.py +344 -0
- fivefold/evaluator/store.py +83 -0
- fivefold/faults/__init__.py +5 -0
- fivefold/faults/catalog.py +278 -0
- fivefold/faults/plan.py +538 -0
- fivefold/gateway/__init__.py +6 -0
- fivefold/gateway/admin.py +179 -0
- fivefold/gateway/admin_api.py +398 -0
- fivefold/gateway/codec.py +372 -0
- fivefold/gateway/destination.py +64 -0
- fivefold/gateway/dispatcher.py +379 -0
- fivefold/gateway/freshness.py +83 -0
- fivefold/gateway/invariants.py +761 -0
- fivefold/gateway/memory.py +277 -0
- fivefold/gateway/pg_store.py +657 -0
- fivefold/gateway/records.py +592 -0
- fivefold/gateway/server.py +102 -0
- fivefold/gateway/sim_client.py +295 -0
- fivefold/gateway/state_machine.py +376 -0
- fivefold/gateway/steps.py +897 -0
- fivefold/gateway/store.py +635 -0
- fivefold/gateway/tools.py +424 -0
- fivefold/gateway/twin.py +199 -0
- fivefold/gateway/wire.py +138 -0
- fivefold/harness/__init__.py +6 -0
- fivefold/harness/actors.py +230 -0
- fivefold/harness/cells.py +303 -0
- fivefold/harness/clock.py +209 -0
- fivefold/harness/demo.py +397 -0
- fivefold/harness/fault_effects.py +122 -0
- fivefold/harness/live_options.py +159 -0
- fivefold/harness/log.py +540 -0
- fivefold/harness/mcp_link.py +409 -0
- fivefold/harness/migrations.py +520 -0
- fivefold/harness/models.py +368 -0
- fivefold/harness/registry.py +105 -0
- fivefold/harness/reports.py +100 -0
- fivefold/harness/rng.py +57 -0
- fivefold/harness/run.py +881 -0
- fivefold/harness/scheduler.py +441 -0
- fivefold/harness/sim_points.py +215 -0
- fivefold/harness/split/__init__.py +34 -0
- fivefold/harness/split/__main__.py +275 -0
- fivefold/harness/split/credentials.py +74 -0
- fivefold/harness/split/faults.py +75 -0
- fivefold/harness/split/gateway.py +769 -0
- fivefold/harness/split/hops.py +291 -0
- fivefold/harness/split/lifecycle.py +322 -0
- fivefold/harness/split/rundir.py +152 -0
- fivefold/harness/split/service.py +157 -0
- fivefold/harness/split/simulator.py +432 -0
- fivefold/harness/study.py +643 -0
- fivefold/harness/transport.py +533 -0
- fivefold/paths.py +24 -0
- fivefold/py.typed +0 -0
- fivefold/recorder/__init__.py +18 -0
- fivefold/recorder/__main__.py +308 -0
- fivefold/recorder/cassette.py +322 -0
- fivefold/recorder/playback.py +316 -0
- fivefold/recorder/recording.py +133 -0
- fivefold/sim/__init__.py +37 -0
- fivefold/sim/config.py +52 -0
- fivefold/sim/endpoint.py +176 -0
- fivefold/sim/ledger.py +583 -0
- fivefold/sim/service.py +834 -0
- fivefold/viewer/__init__.py +14 -0
- fivefold/viewer/__main__.py +75 -0
- fivefold/viewer/app.py +122 -0
- fivefold/viewer/records.py +108 -0
- fivefold/viewer/source.py +196 -0
- fivefold/viewer/timeline.py +61 -0
- fivefold-0.1.0.dist-info/METADATA +169 -0
- fivefold-0.1.0.dist-info/RECORD +128 -0
- fivefold-0.1.0.dist-info/WHEEL +4 -0
- fivefold-0.1.0.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
"""Property 4, actually committed (0004): the lower bound, and its two excuses.
|
|
2
|
+
|
|
3
|
+
Each required intent passes if `E_i(end)` holds an acceptable effect (`Intent.accepts`), or
|
|
4
|
+
if it is excused. Otherwise it fails with `missing_effect`, which names the intent's
|
|
5
|
+
effects that are not acceptable, if it has any. An intent is excused in two cases only:
|
|
6
|
+
|
|
7
|
+
- `authority_withdrawn`, when both hold: an approval for the intent that covered an
|
|
8
|
+
acceptable effect was revoked, or a human approver denied a request that would have
|
|
9
|
+
covered one (policy denials never count, because the stack controls what it asks for);
|
|
10
|
+
and at the end, no approval for the intent is valid and unused.
|
|
11
|
+
- `infeasible`, when no allowed option satisfies the intent's constraints on the facts at
|
|
12
|
+
the end with every effect of the run removed (`scenario.feasible_together`). The
|
|
13
|
+
stack's own spending never excuses a missing effect.
|
|
14
|
+
|
|
15
|
+
Readings the decision records leave open:
|
|
16
|
+
|
|
17
|
+
- An approval is "for" the intent whose id it was filed with. It "covered an acceptable
|
|
18
|
+
effect" when some write is both acceptable for the intent and in the approval's scope
|
|
19
|
+
(0002, rule 5). The check tries the write with the highest acceptable unit price that
|
|
20
|
+
the scope's amount allows.
|
|
21
|
+
- An approval decided by anyone but policy was decided by a human: in deterministic runs,
|
|
22
|
+
the scripted approver.
|
|
23
|
+
- "At the end" is when the harness stopped the run (`RunRecords.end_vt`), after the drain
|
|
24
|
+
window: an approval can expire after the last record. An approval is valid then when
|
|
25
|
+
the approvals service would report it approved (0002, rules 8 and 9), and unused when
|
|
26
|
+
no effect cites it.
|
|
27
|
+
- The facts at the end with every effect removed are each fact's history replayed from
|
|
28
|
+
the scenario's value without the versions effects caused. A change through the control
|
|
29
|
+
interface, such as fault F7's, applies the fields it changed.
|
|
30
|
+
- When both excuses hold, the intent is listed once, with `authority_withdrawn`, the
|
|
31
|
+
first in record 0004's order.
|
|
32
|
+
|
|
33
|
+
Metrics: `missing_effects`, the required intents that fail, and `excused_intents`.
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
from __future__ import annotations
|
|
37
|
+
|
|
38
|
+
from collections.abc import Collection, Sequence
|
|
39
|
+
|
|
40
|
+
from fivefold.domain.authority import (
|
|
41
|
+
ApprovalStatus,
|
|
42
|
+
Decision,
|
|
43
|
+
approval_status_at,
|
|
44
|
+
scope_mismatches,
|
|
45
|
+
)
|
|
46
|
+
from fivefold.domain.scenario import (
|
|
47
|
+
Authority,
|
|
48
|
+
Intent,
|
|
49
|
+
ReservationAccept,
|
|
50
|
+
feasible_together,
|
|
51
|
+
)
|
|
52
|
+
from fivefold.evaluator.checks import effects_by_intent, property_result
|
|
53
|
+
from fivefold.evaluator.records import Approval, FactVersion, RecordsError, RunRecords
|
|
54
|
+
from fivefold.evaluator.result import (
|
|
55
|
+
Excuse,
|
|
56
|
+
ExcusedIntent,
|
|
57
|
+
Failure,
|
|
58
|
+
PropertyResult,
|
|
59
|
+
ReasonCode,
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def check_actually_committed(
|
|
64
|
+
records: RunRecords,
|
|
65
|
+
) -> tuple[PropertyResult, tuple[ExcusedIntent, ...]]:
|
|
66
|
+
"""Property 4, and the required intents it excuses, in scenario order."""
|
|
67
|
+
by_intent = effects_by_intent(records)
|
|
68
|
+
used = {effect.approval_id for effect in records.effects}
|
|
69
|
+
end_vt = records.end_vt()
|
|
70
|
+
facts: dict[str, dict[str, object]] | None = None
|
|
71
|
+
failures: list[Failure] = []
|
|
72
|
+
excused: list[ExcusedIntent] = []
|
|
73
|
+
for intent in records.scenario.intents:
|
|
74
|
+
if not intent.required:
|
|
75
|
+
continue
|
|
76
|
+
effects = by_intent[intent.id]
|
|
77
|
+
if any(intent.accepts(effect.operation, effect.args) for effect in effects):
|
|
78
|
+
continue
|
|
79
|
+
approvals = [approval for approval in records.approvals if approval.intent_id == intent.id]
|
|
80
|
+
excuse = _authority_withdrawn(intent, approvals, used, end_vt)
|
|
81
|
+
if excuse is None:
|
|
82
|
+
if facts is None:
|
|
83
|
+
facts = facts_without_effects(records)
|
|
84
|
+
excuse = _infeasible(intent, records.scenario.authority, facts)
|
|
85
|
+
if excuse is not None:
|
|
86
|
+
excused.append(excuse)
|
|
87
|
+
continue
|
|
88
|
+
unacceptable = tuple(effect.effect_id for effect in effects)
|
|
89
|
+
detail = f"{intent.id} has no acceptable effect at the end, and no excuse applies"
|
|
90
|
+
if unacceptable:
|
|
91
|
+
detail += f"; not acceptable: {', '.join(unacceptable)}"
|
|
92
|
+
failures.append(
|
|
93
|
+
Failure(
|
|
94
|
+
code=ReasonCode.MISSING_EFFECT,
|
|
95
|
+
intent_id=intent.id,
|
|
96
|
+
effect_ids=unacceptable,
|
|
97
|
+
detail=detail,
|
|
98
|
+
)
|
|
99
|
+
)
|
|
100
|
+
metrics = {"missing_effects": len(failures), "excused_intents": len(excused)}
|
|
101
|
+
return property_result(4, failures, metrics), tuple(excused)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def covers_acceptable(intent: Intent, approval: Approval) -> bool:
|
|
105
|
+
"""Whether some write that is acceptable for `intent` is in `approval`'s scope."""
|
|
106
|
+
if approval.operation is not intent.operation:
|
|
107
|
+
return False
|
|
108
|
+
accept = intent.accept
|
|
109
|
+
args: dict[str, object]
|
|
110
|
+
if isinstance(accept, ReservationAccept):
|
|
111
|
+
args = {"warehouse_id": accept.warehouse_id, "sku": accept.sku, "quantity": accept.quantity}
|
|
112
|
+
else:
|
|
113
|
+
amount = approval.scope.get("max_amount_minor")
|
|
114
|
+
if isinstance(amount, bool) or not isinstance(amount, int):
|
|
115
|
+
raise RecordsError(f"the scope of {approval.approval_id} has no integer amount")
|
|
116
|
+
args = {
|
|
117
|
+
"supplier_id": approval.scope.get("supplier_id"),
|
|
118
|
+
"sku": accept.sku,
|
|
119
|
+
"quantity": accept.quantity,
|
|
120
|
+
"unit_price_minor": min(accept.max_unit_price_minor, amount // accept.quantity),
|
|
121
|
+
"currency": accept.currency,
|
|
122
|
+
"cost_center": accept.cost_center,
|
|
123
|
+
}
|
|
124
|
+
if not intent.accepts(intent.operation, args):
|
|
125
|
+
return False
|
|
126
|
+
try:
|
|
127
|
+
mismatched = scope_mismatches(
|
|
128
|
+
approval_intent_id=approval.intent_id,
|
|
129
|
+
approval_operation=approval.operation,
|
|
130
|
+
scope=approval.scope,
|
|
131
|
+
intent_id=intent.id,
|
|
132
|
+
operation=intent.operation,
|
|
133
|
+
args=args,
|
|
134
|
+
)
|
|
135
|
+
except ValueError as error:
|
|
136
|
+
raise RecordsError(f"the scope of {approval.approval_id}: {error}") from error
|
|
137
|
+
return not mismatched
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def valid_and_unused(approval: Approval, used: Collection[str], vt: int) -> bool:
|
|
141
|
+
"""Whether the approvals service would report `approval` approved at `vt`, and no
|
|
142
|
+
effect cites it."""
|
|
143
|
+
if approval.approval_id in used:
|
|
144
|
+
return False
|
|
145
|
+
try:
|
|
146
|
+
status = approval_status_at(
|
|
147
|
+
decision=approval.decision,
|
|
148
|
+
revoked=approval.revoked,
|
|
149
|
+
expires_at=approval.expires_at,
|
|
150
|
+
vt=vt,
|
|
151
|
+
)
|
|
152
|
+
except ValueError as error:
|
|
153
|
+
raise RecordsError(f"approval {approval.approval_id}: {error}") from error
|
|
154
|
+
return status is ApprovalStatus.APPROVED
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def _withdrawn(approval: Approval) -> bool:
|
|
158
|
+
"""Revoked, or denied by a human. A denial by policy is not a withdrawal."""
|
|
159
|
+
if approval.revoked:
|
|
160
|
+
return True
|
|
161
|
+
return approval.decision is Decision.DENY and not approval.decided_by_policy
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def _authority_withdrawn(
|
|
165
|
+
intent: Intent, approvals: Sequence[Approval], used: Collection[str], end_vt: int
|
|
166
|
+
) -> ExcusedIntent | None:
|
|
167
|
+
withdrawn = tuple(
|
|
168
|
+
sorted(
|
|
169
|
+
approval.approval_id
|
|
170
|
+
for approval in approvals
|
|
171
|
+
if _withdrawn(approval) and covers_acceptable(intent, approval)
|
|
172
|
+
)
|
|
173
|
+
)
|
|
174
|
+
if not withdrawn:
|
|
175
|
+
return None
|
|
176
|
+
if any(valid_and_unused(approval, used, end_vt) for approval in approvals):
|
|
177
|
+
return None
|
|
178
|
+
return ExcusedIntent(
|
|
179
|
+
intent_id=intent.id,
|
|
180
|
+
excuse=Excuse.AUTHORITY_WITHDRAWN,
|
|
181
|
+
approval_ids=withdrawn,
|
|
182
|
+
detail=(
|
|
183
|
+
f"authority for {intent.id} was withdrawn ({', '.join(withdrawn)}), and no"
|
|
184
|
+
f" approval for it is valid and unused at the end"
|
|
185
|
+
),
|
|
186
|
+
)
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def _infeasible(
|
|
190
|
+
intent: Intent, authority: Authority, facts: dict[str, dict[str, object]]
|
|
191
|
+
) -> ExcusedIntent | None:
|
|
192
|
+
try:
|
|
193
|
+
feasible = feasible_together([intent], authority, facts)
|
|
194
|
+
except ValueError as error:
|
|
195
|
+
raise RecordsError(f"the facts at the end, for {intent.id}: {error}") from error
|
|
196
|
+
if feasible:
|
|
197
|
+
return None
|
|
198
|
+
return ExcusedIntent(
|
|
199
|
+
intent_id=intent.id,
|
|
200
|
+
excuse=Excuse.INFEASIBLE,
|
|
201
|
+
detail=(
|
|
202
|
+
f"no allowed option for {intent.id} fits the facts at the end with every effect"
|
|
203
|
+
" of the run removed"
|
|
204
|
+
),
|
|
205
|
+
)
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def facts_without_effects(records: RunRecords) -> dict[str, dict[str, object]]:
|
|
209
|
+
"""Each fact at the end of the run, with every effect of the run removed.
|
|
210
|
+
|
|
211
|
+
Each fact's history is replayed from version 1, the scenario's value, skipping the
|
|
212
|
+
versions effects caused. Any other version applies the fields it changed from the
|
|
213
|
+
version before it. Every fact the scenario sets must have a history.
|
|
214
|
+
"""
|
|
215
|
+
histories: dict[str, list[FactVersion]] = {}
|
|
216
|
+
for fact in records.fact_versions:
|
|
217
|
+
histories.setdefault(fact.key, []).append(fact)
|
|
218
|
+
missing = sorted(set(records.scenario.facts) - set(histories))
|
|
219
|
+
if missing:
|
|
220
|
+
raise RecordsError(f"the simulator has no history of {', '.join(missing)}")
|
|
221
|
+
facts: dict[str, dict[str, object]] = {}
|
|
222
|
+
for key in sorted(histories):
|
|
223
|
+
history = sorted(histories[key], key=lambda fact: fact.version)
|
|
224
|
+
first = history[0]
|
|
225
|
+
if first.version != 1 or first.cause != "initial":
|
|
226
|
+
raise RecordsError(f"the history of {key} does not start from the scenario's value")
|
|
227
|
+
value = dict(first.value)
|
|
228
|
+
before = first.value
|
|
229
|
+
for fact in history[1:]:
|
|
230
|
+
if fact.cause != "effect":
|
|
231
|
+
value.update(
|
|
232
|
+
(name, new) for name, new in fact.value.items() if before.get(name) != new
|
|
233
|
+
)
|
|
234
|
+
before = fact.value
|
|
235
|
+
facts[key] = value
|
|
236
|
+
return facts
|
|
@@ -0,0 +1,354 @@
|
|
|
1
|
+
"""Read one run's ground truth from its database (0004, Ground truth the evaluator reads).
|
|
2
|
+
|
|
3
|
+
The evaluator connects as `ff_eval`, which can read the `sim`, `gw` and `harness` schemas
|
|
4
|
+
and write only `eval` (0008). It reads the simulator's and the harness's tables with SQL,
|
|
5
|
+
never their code, and leaves the gateway's tables alone: they may only explain a failure,
|
|
6
|
+
never decide one. The scenario comes from the caller, since scenario files can live
|
|
7
|
+
outside the repository (0009); the run's records must name the same scenario.
|
|
8
|
+
|
|
9
|
+
Everything is read in one repeatable-read, read-only transaction, so the records are one
|
|
10
|
+
consistent snapshot, and every list in a fixed order. Of the wall-clock columns, it reads
|
|
11
|
+
only the three recovery latency's wall-clock measure needs (0014).
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
from collections.abc import Mapping
|
|
17
|
+
from datetime import datetime
|
|
18
|
+
from typing import Any
|
|
19
|
+
|
|
20
|
+
import psycopg
|
|
21
|
+
from psycopg.rows import TupleRow
|
|
22
|
+
|
|
23
|
+
from fivefold.domain.contracts import Contract, ContractParams
|
|
24
|
+
from fivefold.domain.rules import Operation
|
|
25
|
+
from fivefold.domain.scenario import Scenario
|
|
26
|
+
from fivefold.evaluator.records import (
|
|
27
|
+
ActorAction,
|
|
28
|
+
Approval,
|
|
29
|
+
Config,
|
|
30
|
+
Destination,
|
|
31
|
+
Effect,
|
|
32
|
+
FactVersion,
|
|
33
|
+
Fault,
|
|
34
|
+
Message,
|
|
35
|
+
ModelCall,
|
|
36
|
+
Read,
|
|
37
|
+
RecordsError,
|
|
38
|
+
RunInfo,
|
|
39
|
+
RunRecords,
|
|
40
|
+
SubmittedReport,
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
Connection = psycopg.Connection[TupleRow]
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def load_run(conn: Connection, scenario: Scenario) -> RunRecords:
|
|
47
|
+
"""Read the run in `conn`'s database, scored against `scenario`.
|
|
48
|
+
|
|
49
|
+
`conn` must be in autocommit mode: the loader opens its own transaction. Raises
|
|
50
|
+
`RecordsError` if the database holds no run, or a run of another scenario.
|
|
51
|
+
"""
|
|
52
|
+
if not conn.autocommit:
|
|
53
|
+
raise ValueError("load_run opens its own transaction: use an autocommit connection")
|
|
54
|
+
with conn.transaction():
|
|
55
|
+
conn.execute("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY")
|
|
56
|
+
records = RunRecords(
|
|
57
|
+
run=_run(conn),
|
|
58
|
+
scenario=scenario,
|
|
59
|
+
destination=_destination(conn),
|
|
60
|
+
effects=_effects(conn),
|
|
61
|
+
messages=_messages(conn),
|
|
62
|
+
faults=_faults(conn),
|
|
63
|
+
actor_actions=_actor_actions(conn),
|
|
64
|
+
reports=_reports(conn),
|
|
65
|
+
approvals=_approvals(conn),
|
|
66
|
+
fact_versions=_fact_versions(conn),
|
|
67
|
+
reads=_reads(conn),
|
|
68
|
+
model_calls=_model_calls(conn),
|
|
69
|
+
)
|
|
70
|
+
records.check_scenario()
|
|
71
|
+
return records
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _text(value: Any) -> str | None:
|
|
75
|
+
return None if value is None else str(value)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _integer(value: Any) -> int | None:
|
|
79
|
+
return None if value is None else int(value)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _wall(value: Any, what: str) -> datetime | None:
|
|
83
|
+
if value is None:
|
|
84
|
+
return None
|
|
85
|
+
if not isinstance(value, datetime) or value.tzinfo is None:
|
|
86
|
+
raise RecordsError(f"{what} is not a time with a time zone: {value!r}")
|
|
87
|
+
return value
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _object(value: Any, what: str) -> Mapping[str, object]:
|
|
91
|
+
if not isinstance(value, dict):
|
|
92
|
+
raise RecordsError(f"{what} is not a JSON object")
|
|
93
|
+
return value
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _version(value: object, what: str) -> int:
|
|
97
|
+
if isinstance(value, bool) or not isinstance(value, int):
|
|
98
|
+
raise RecordsError(f"{what} is not an integer version: {value!r}")
|
|
99
|
+
return value
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _run(conn: Connection) -> RunInfo:
|
|
103
|
+
rows = conn.execute(
|
|
104
|
+
"SELECT run_id, scenario_id, scenario_digest, config, contract, seed, topology,"
|
|
105
|
+
" model_adapter, model, cassette_digest, parent_run_id, branch_point, params"
|
|
106
|
+
" FROM harness.run"
|
|
107
|
+
).fetchall()
|
|
108
|
+
if len(rows) != 1:
|
|
109
|
+
raise RecordsError(f"harness.run holds {len(rows)} rows; a run database holds one")
|
|
110
|
+
row = rows[0]
|
|
111
|
+
stops = conn.execute("SELECT stopped_vt FROM harness.run_stop").fetchall()
|
|
112
|
+
if len(stops) != 1:
|
|
113
|
+
raise RecordsError("harness.run_stop is empty: score a run only after the harness stops it")
|
|
114
|
+
return RunInfo(
|
|
115
|
+
run_id=str(row[0]),
|
|
116
|
+
scenario_id=str(row[1]),
|
|
117
|
+
scenario_digest=str(row[2]),
|
|
118
|
+
config=Config(row[3]),
|
|
119
|
+
contract=Contract(row[4]),
|
|
120
|
+
seed=int(row[5]),
|
|
121
|
+
topology=str(row[6]),
|
|
122
|
+
model_adapter=str(row[7]),
|
|
123
|
+
model=str(row[8]),
|
|
124
|
+
cassette_digest=_text(row[9]),
|
|
125
|
+
parent_run_id=_text(row[10]),
|
|
126
|
+
branch_point=_integer(row[11]),
|
|
127
|
+
params=row[12],
|
|
128
|
+
stopped_vt=int(stops[0][0]),
|
|
129
|
+
)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _destination(conn: Connection) -> Destination:
|
|
133
|
+
row = conn.execute(
|
|
134
|
+
"SELECT scenario_id, task_id, capabilities, max_hop_latency_ms, commit_deadline_ms,"
|
|
135
|
+
" status_lag_ms, last_vt FROM sim.run"
|
|
136
|
+
).fetchone()
|
|
137
|
+
if row is None:
|
|
138
|
+
raise RecordsError("sim.run is empty: the simulator was never set up")
|
|
139
|
+
return Destination(
|
|
140
|
+
scenario_id=str(row[0]),
|
|
141
|
+
task_id=str(row[1]),
|
|
142
|
+
capabilities=frozenset(str(name) for name in row[2]),
|
|
143
|
+
params=ContractParams(
|
|
144
|
+
max_hop_latency_ms=int(row[3]),
|
|
145
|
+
commit_deadline_ms=int(row[4]),
|
|
146
|
+
status_lag_ms=int(row[5]),
|
|
147
|
+
),
|
|
148
|
+
last_vt=int(row[6]),
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def _effects(conn: Connection) -> tuple[Effect, ...]:
|
|
153
|
+
rows = conn.execute(
|
|
154
|
+
"SELECT effect_id, intent_id, operation, args, approval_id, basis, idempotency_key,"
|
|
155
|
+
" client_ref, received_seq, commit_seq, committed_vt FROM sim.effect"
|
|
156
|
+
" ORDER BY commit_seq"
|
|
157
|
+
).fetchall()
|
|
158
|
+
effects = []
|
|
159
|
+
for row in rows:
|
|
160
|
+
basis = _object(row[5], f"the basis of {row[0]}")
|
|
161
|
+
effects.append(
|
|
162
|
+
Effect(
|
|
163
|
+
effect_id=str(row[0]),
|
|
164
|
+
intent_id=str(row[1]),
|
|
165
|
+
operation=Operation(row[2]),
|
|
166
|
+
args=_object(row[3], f"the args of {row[0]}"),
|
|
167
|
+
approval_id=str(row[4]),
|
|
168
|
+
basis={
|
|
169
|
+
key: _version(version, f"the basis of {row[0]} for {key}")
|
|
170
|
+
for key, version in basis.items()
|
|
171
|
+
},
|
|
172
|
+
idempotency_key=_text(row[6]),
|
|
173
|
+
client_ref=_text(row[7]),
|
|
174
|
+
received_seq=int(row[8]),
|
|
175
|
+
commit_seq=int(row[9]),
|
|
176
|
+
committed_vt=int(row[10]),
|
|
177
|
+
)
|
|
178
|
+
)
|
|
179
|
+
return tuple(effects)
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def _messages(conn: Connection) -> tuple[Message, ...]:
|
|
183
|
+
rows = conn.execute(
|
|
184
|
+
"SELECT seq, hop, kind, sender, receiver, api, correlation, intent_id,"
|
|
185
|
+
" idempotency_key, answer_class, body, sent_vt, fate, drop_reason, delivered_seq,"
|
|
186
|
+
" delivered_vt, copy_of, delivered_wall_at FROM harness.message_log ORDER BY seq"
|
|
187
|
+
).fetchall()
|
|
188
|
+
return tuple(
|
|
189
|
+
Message(
|
|
190
|
+
seq=int(row[0]),
|
|
191
|
+
hop=str(row[1]),
|
|
192
|
+
kind=str(row[2]),
|
|
193
|
+
sender=str(row[3]),
|
|
194
|
+
receiver=str(row[4]),
|
|
195
|
+
api=_text(row[5]),
|
|
196
|
+
correlation=_integer(row[6]),
|
|
197
|
+
intent_id=_text(row[7]),
|
|
198
|
+
idempotency_key=_text(row[8]),
|
|
199
|
+
answer_class=_text(row[9]),
|
|
200
|
+
body=row[10],
|
|
201
|
+
sent_vt=int(row[11]),
|
|
202
|
+
fate=_text(row[12]),
|
|
203
|
+
drop_reason=_text(row[13]),
|
|
204
|
+
delivered_seq=_integer(row[14]),
|
|
205
|
+
delivered_vt=_integer(row[15]),
|
|
206
|
+
copy_of=_integer(row[16]),
|
|
207
|
+
delivered_wall_at=_wall(row[17], f"the delivery time of message {row[0]}"),
|
|
208
|
+
)
|
|
209
|
+
for row in rows
|
|
210
|
+
)
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def _faults(conn: Connection) -> tuple[Fault, ...]:
|
|
214
|
+
rows = conn.execute(
|
|
215
|
+
"SELECT seq, fault, variant, point, occurrence, action, target_seq, component, voided,"
|
|
216
|
+
" params, vt, intent_id, wall_at FROM harness.fault_log ORDER BY seq"
|
|
217
|
+
).fetchall()
|
|
218
|
+
return tuple(
|
|
219
|
+
Fault(
|
|
220
|
+
seq=int(row[0]),
|
|
221
|
+
fault=str(row[1]),
|
|
222
|
+
variant=_text(row[2]),
|
|
223
|
+
point=str(row[3]),
|
|
224
|
+
occurrence=int(row[4]),
|
|
225
|
+
action=str(row[5]),
|
|
226
|
+
target_seq=_integer(row[6]),
|
|
227
|
+
component=_text(row[7]),
|
|
228
|
+
voided=tuple(int(seq) for seq in row[8]),
|
|
229
|
+
params=_object(row[9], f"the params of fault {row[0]}"),
|
|
230
|
+
vt=int(row[10]),
|
|
231
|
+
intent_id=_text(row[11]),
|
|
232
|
+
wall_at=_wall(row[12], f"the time of fault {row[0]}"),
|
|
233
|
+
)
|
|
234
|
+
for row in rows
|
|
235
|
+
)
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def _actor_actions(conn: Connection) -> tuple[ActorAction, ...]:
|
|
239
|
+
rows = conn.execute(
|
|
240
|
+
"SELECT seq, actor, action, subject, details, vt, wall_at FROM harness.actor_log"
|
|
241
|
+
" ORDER BY seq"
|
|
242
|
+
).fetchall()
|
|
243
|
+
return tuple(
|
|
244
|
+
ActorAction(
|
|
245
|
+
seq=int(row[0]),
|
|
246
|
+
actor=str(row[1]),
|
|
247
|
+
action=str(row[2]),
|
|
248
|
+
subject=str(row[3]),
|
|
249
|
+
details=_object(row[4], f"the details of actor action {row[0]}"),
|
|
250
|
+
vt=int(row[5]),
|
|
251
|
+
wall_at=_wall(row[6], f"the time of actor action {row[0]}"),
|
|
252
|
+
)
|
|
253
|
+
for row in rows
|
|
254
|
+
)
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def _approvals(conn: Connection) -> tuple[Approval, ...]:
|
|
258
|
+
rows = conn.execute(
|
|
259
|
+
"SELECT approval_id, intent_id, operation, scope, requested_seq, requested_vt, status,"
|
|
260
|
+
" decided_seq, decided_vt, decided_by, expires_at, revoked_seq, revoked_vt, revoked_by"
|
|
261
|
+
" FROM sim.approval ORDER BY requested_seq"
|
|
262
|
+
).fetchall()
|
|
263
|
+
return tuple(
|
|
264
|
+
Approval(
|
|
265
|
+
approval_id=str(row[0]),
|
|
266
|
+
intent_id=str(row[1]),
|
|
267
|
+
operation=Operation(row[2]),
|
|
268
|
+
scope=_object(row[3], f"the scope of {row[0]}"),
|
|
269
|
+
requested_seq=int(row[4]),
|
|
270
|
+
requested_vt=int(row[5]),
|
|
271
|
+
status=str(row[6]),
|
|
272
|
+
decided_seq=_integer(row[7]),
|
|
273
|
+
decided_vt=_integer(row[8]),
|
|
274
|
+
decided_by=_text(row[9]),
|
|
275
|
+
expires_at=_integer(row[10]),
|
|
276
|
+
revoked_seq=_integer(row[11]),
|
|
277
|
+
revoked_vt=_integer(row[12]),
|
|
278
|
+
revoked_by=_text(row[13]),
|
|
279
|
+
)
|
|
280
|
+
for row in rows
|
|
281
|
+
)
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def _fact_versions(conn: Connection) -> tuple[FactVersion, ...]:
|
|
285
|
+
rows = conn.execute(
|
|
286
|
+
"SELECT key, version, value, valid_from_seq, valid_from_vt, cause, cause_effect_id"
|
|
287
|
+
" FROM sim.fact_version ORDER BY key, version"
|
|
288
|
+
).fetchall()
|
|
289
|
+
return tuple(
|
|
290
|
+
FactVersion(
|
|
291
|
+
key=str(row[0]),
|
|
292
|
+
version=int(row[1]),
|
|
293
|
+
value=_object(row[2], f"version {row[1]} of {row[0]}"),
|
|
294
|
+
valid_from_seq=int(row[3]),
|
|
295
|
+
valid_from_vt=int(row[4]),
|
|
296
|
+
cause=str(row[5]),
|
|
297
|
+
cause_effect_id=_text(row[6]),
|
|
298
|
+
)
|
|
299
|
+
for row in rows
|
|
300
|
+
)
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
def _reads(conn: Connection) -> tuple[Read, ...]:
|
|
304
|
+
rows = conn.execute("SELECT seq, key, version, caller, vt FROM sim.read_log ORDER BY seq")
|
|
305
|
+
return tuple(
|
|
306
|
+
Read(
|
|
307
|
+
seq=int(row[0]),
|
|
308
|
+
key=str(row[1]),
|
|
309
|
+
version=int(row[2]),
|
|
310
|
+
caller=str(row[3]),
|
|
311
|
+
vt=int(row[4]),
|
|
312
|
+
)
|
|
313
|
+
for row in rows.fetchall()
|
|
314
|
+
)
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
def _model_calls(conn: Connection) -> tuple[ModelCall, ...]:
|
|
318
|
+
rows = conn.execute(
|
|
319
|
+
"SELECT call_no, worker, asked_vt, answered_vt, model, input_tokens, output_tokens,"
|
|
320
|
+
" refused FROM harness.model_call ORDER BY call_no"
|
|
321
|
+
).fetchall()
|
|
322
|
+
return tuple(
|
|
323
|
+
ModelCall(
|
|
324
|
+
call_no=int(row[0]),
|
|
325
|
+
worker=str(row[1]),
|
|
326
|
+
asked_vt=int(row[2]),
|
|
327
|
+
answered_vt=int(row[3]),
|
|
328
|
+
model=str(row[4]),
|
|
329
|
+
input_tokens=_integer(row[5]),
|
|
330
|
+
output_tokens=_integer(row[6]),
|
|
331
|
+
refused=_text(row[7]),
|
|
332
|
+
)
|
|
333
|
+
for row in rows
|
|
334
|
+
)
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
def _reports(conn: Connection) -> tuple[SubmittedReport, ...]:
|
|
338
|
+
rows = conn.execute(
|
|
339
|
+
"SELECT seq, worker, report, valid, errors, accepted_by, message_seq, vt"
|
|
340
|
+
" FROM harness.final_report ORDER BY seq"
|
|
341
|
+
).fetchall()
|
|
342
|
+
return tuple(
|
|
343
|
+
SubmittedReport(
|
|
344
|
+
seq=int(row[0]),
|
|
345
|
+
worker=str(row[1]),
|
|
346
|
+
report=row[2],
|
|
347
|
+
valid=bool(row[3]),
|
|
348
|
+
errors=tuple(str(error) for error in row[4]) if isinstance(row[4], list) else (),
|
|
349
|
+
accepted_by=str(row[5]),
|
|
350
|
+
message_seq=_integer(row[6]),
|
|
351
|
+
vt=int(row[7]),
|
|
352
|
+
)
|
|
353
|
+
for row in rows
|
|
354
|
+
)
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
"""Metrics of record 0004 that belong to no one property: what humans did in the run, how
|
|
2
|
+
many intents faults touched and how many stayed unresolved, and the tokens its model calls
|
|
3
|
+
used.
|
|
4
|
+
|
|
5
|
+
- `human_interventions`: attestations by the scripted resolver in `harness.actor_log`.
|
|
6
|
+
- `approval_requests`: approval requests the approvals service filed, one row of
|
|
7
|
+
`sim.approval` each. A repeated request with a client request key already used files
|
|
8
|
+
nothing (0002, rule 4), so it is not counted again.
|
|
9
|
+
- `approvals_decided_by_human`: requests decided, approved or denied, by anyone but
|
|
10
|
+
policy.
|
|
11
|
+
- `affected_intents` and `unresolved_intents`: the task intents a fault touched, and those
|
|
12
|
+
whose outcome the stack's evidence never settled (0014). Each has an entry in the
|
|
13
|
+
result's `recovery` (`fivefold.evaluator.recovery`), with its latency when it has one.
|
|
14
|
+
- `tokens_in` and `tokens_out`: the input and output tokens of the run's model calls in
|
|
15
|
+
`harness.model_call`, summed. A call whose reply reported no usage counts 0, as every
|
|
16
|
+
scripted reply does (0009). A call the spend cap refused was never sent, so it has no
|
|
17
|
+
tokens and counts 0 too.
|
|
18
|
+
|
|
19
|
+
Each property's own metrics are in its module. Cost and cost per passing run need a price
|
|
20
|
+
file, which a run's result does not hold, so the report across runs computes them from
|
|
21
|
+
the tokens (`fivefold.evaluator.report`). Latencies are never summed, so they are not
|
|
22
|
+
counts here: the report shows their median and largest across runs.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
from collections.abc import Sequence
|
|
28
|
+
|
|
29
|
+
from fivefold.evaluator.records import ATTEST, RESOLVER, RecordsError, RunRecords
|
|
30
|
+
from fivefold.evaluator.result import RecoveryEntry
|
|
31
|
+
|
|
32
|
+
RUN_METRICS = ("human_interventions", "approval_requests", "approvals_decided_by_human")
|
|
33
|
+
"""The run's own counts of human actions and approvals, in the order of record 0004's
|
|
34
|
+
list."""
|
|
35
|
+
|
|
36
|
+
RECOVERY_METRICS = ("affected_intents", "unresolved_intents")
|
|
37
|
+
"""Recovery latency's counts (0014): the task intents a fault touched, and those the
|
|
38
|
+
stack's evidence never settled."""
|
|
39
|
+
|
|
40
|
+
TOKEN_METRICS = ("tokens_in", "tokens_out")
|
|
41
|
+
"""The tokens of the run's model calls, in the order of record 0004's list."""
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def run_metrics(records: RunRecords) -> dict[str, int]:
|
|
45
|
+
"""The run's own metrics, from the harness's actor log and the approvals service."""
|
|
46
|
+
attestations = sum(
|
|
47
|
+
1
|
|
48
|
+
for action in records.actor_actions
|
|
49
|
+
if action.actor == RESOLVER and action.action == ATTEST
|
|
50
|
+
)
|
|
51
|
+
by_human = sum(
|
|
52
|
+
1
|
|
53
|
+
for approval in records.approvals
|
|
54
|
+
if approval.decision is not None and not approval.decided_by_policy
|
|
55
|
+
)
|
|
56
|
+
return {
|
|
57
|
+
"human_interventions": attestations,
|
|
58
|
+
"approval_requests": len(records.approvals),
|
|
59
|
+
"approvals_decided_by_human": by_human,
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def recovery_metrics(entries: Sequence[RecoveryEntry]) -> dict[str, int]:
|
|
64
|
+
"""How many task intents a fault touched, and how many of them stayed unresolved."""
|
|
65
|
+
return {
|
|
66
|
+
"affected_intents": len(entries),
|
|
67
|
+
"unresolved_intents": sum(1 for entry in entries if not entry.resolved),
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def token_metrics(records: RunRecords) -> dict[str, int]:
|
|
72
|
+
"""The tokens of the run's model calls, in and out, from the harness's model-call log.
|
|
73
|
+
|
|
74
|
+
Raises `RecordsError` for a call that breaks the log's rules: a call of another model
|
|
75
|
+
than the run's, since the report prices every token of a run at its model's price; a
|
|
76
|
+
negative token count; or a refused call with tokens.
|
|
77
|
+
"""
|
|
78
|
+
for call in records.model_calls:
|
|
79
|
+
if call.model != records.run.model:
|
|
80
|
+
raise RecordsError(
|
|
81
|
+
f"model call {call.call_no} names the model {call.model}, but the run's model"
|
|
82
|
+
f" is {records.run.model}"
|
|
83
|
+
)
|
|
84
|
+
tokens = [count for count in (call.input_tokens, call.output_tokens) if count is not None]
|
|
85
|
+
if any(count < 0 for count in tokens):
|
|
86
|
+
raise RecordsError(f"model call {call.call_no} has a negative token count")
|
|
87
|
+
if call.refused is not None and tokens:
|
|
88
|
+
raise RecordsError(
|
|
89
|
+
f"model call {call.call_no} was refused ({call.refused}), so it was never sent,"
|
|
90
|
+
" but it has tokens"
|
|
91
|
+
)
|
|
92
|
+
return {
|
|
93
|
+
"tokens_in": sum(call.input_tokens or 0 for call in records.model_calls),
|
|
94
|
+
"tokens_out": sum(call.output_tokens or 0 for call in records.model_calls),
|
|
95
|
+
}
|