fivefold 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fivefold/__init__.py +4 -0
- fivefold/_data/db/bootstrap.sql +26 -0
- fivefold/_data/db/migrations/0001_schemas.sql +67 -0
- fivefold/_data/db/migrations/0002_gateway.sql +224 -0
- fivefold/_data/db/migrations/0003_gateway_record_0010.sql +91 -0
- fivefold/_data/db/migrations/0004_harness.sql +141 -0
- fivefold/_data/db/migrations/0005_sim.sql +679 -0
- fivefold/_data/db/migrations/0006_gateway_reads.sql +66 -0
- fivefold/_data/db/migrations/0007_eval.sql +33 -0
- fivefold/_data/db/migrations/0008_run_stop.sql +18 -0
- fivefold/_data/db/migrations/0009_message_copies.sql +69 -0
- fivefold/_data/db/migrations/0010_gateway_g1_check.sql +14 -0
- fivefold/_data/db/migrations/0011_model_calls.sql +37 -0
- fivefold/_data/db/migrations/0012_recovery_latency.sql +23 -0
- fivefold/_data/db/migrations/0013_split_topology.sql +33 -0
- fivefold/_data/db/migrations/0014_tool_call_args.sql +20 -0
- fivefold/_data/db/registry/0001_registry.sql +31 -0
- fivefold/_data/faults/expected.yaml +406 -0
- fivefold/_data/scenarios/dev/dev-001-single-po.yaml +45 -0
- fivefold/_data/scenarios/dev/dev-002-human-approval.yaml +47 -0
- fivefold/agents/__init__.py +7 -0
- fivefold/agents/branch.py +187 -0
- fivefold/agents/checkpoint.py +77 -0
- fivefold/agents/connection.py +153 -0
- fivefold/agents/live.py +492 -0
- fivefold/agents/policy.py +426 -0
- fivefold/agents/remote.py +333 -0
- fivefold/agents/replay.py +170 -0
- fivefold/agents/scripted.py +122 -0
- fivefold/agents/tools.py +229 -0
- fivefold/agents/worker.py +164 -0
- fivefold/domain/__init__.py +5 -0
- fivefold/domain/api.py +517 -0
- fivefold/domain/authority.py +149 -0
- fivefold/domain/contracts.py +80 -0
- fivefold/domain/identity.py +113 -0
- fivefold/domain/prices.py +178 -0
- fivefold/domain/report.py +92 -0
- fivefold/domain/rules.py +346 -0
- fivefold/domain/scenario.py +524 -0
- fivefold/evaluator/__init__.py +155 -0
- fivefold/evaluator/__main__.py +127 -0
- fivefold/evaluator/authorization.py +146 -0
- fivefold/evaluator/checks.py +331 -0
- fivefold/evaluator/evaluate.py +109 -0
- fivefold/evaluator/evidence.py +392 -0
- fivefold/evaluator/freshness.py +136 -0
- fivefold/evaluator/liveness.py +236 -0
- fivefold/evaluator/load.py +354 -0
- fivefold/evaluator/metrics.py +95 -0
- fivefold/evaluator/records.py +452 -0
- fivefold/evaluator/recovery.py +209 -0
- fivefold/evaluator/report.py +1096 -0
- fivefold/evaluator/result.py +344 -0
- fivefold/evaluator/store.py +83 -0
- fivefold/faults/__init__.py +5 -0
- fivefold/faults/catalog.py +278 -0
- fivefold/faults/plan.py +538 -0
- fivefold/gateway/__init__.py +6 -0
- fivefold/gateway/admin.py +179 -0
- fivefold/gateway/admin_api.py +398 -0
- fivefold/gateway/codec.py +372 -0
- fivefold/gateway/destination.py +64 -0
- fivefold/gateway/dispatcher.py +379 -0
- fivefold/gateway/freshness.py +83 -0
- fivefold/gateway/invariants.py +761 -0
- fivefold/gateway/memory.py +277 -0
- fivefold/gateway/pg_store.py +657 -0
- fivefold/gateway/records.py +592 -0
- fivefold/gateway/server.py +102 -0
- fivefold/gateway/sim_client.py +295 -0
- fivefold/gateway/state_machine.py +376 -0
- fivefold/gateway/steps.py +897 -0
- fivefold/gateway/store.py +635 -0
- fivefold/gateway/tools.py +424 -0
- fivefold/gateway/twin.py +199 -0
- fivefold/gateway/wire.py +138 -0
- fivefold/harness/__init__.py +6 -0
- fivefold/harness/actors.py +230 -0
- fivefold/harness/cells.py +303 -0
- fivefold/harness/clock.py +209 -0
- fivefold/harness/demo.py +397 -0
- fivefold/harness/fault_effects.py +122 -0
- fivefold/harness/live_options.py +159 -0
- fivefold/harness/log.py +540 -0
- fivefold/harness/mcp_link.py +409 -0
- fivefold/harness/migrations.py +520 -0
- fivefold/harness/models.py +368 -0
- fivefold/harness/registry.py +105 -0
- fivefold/harness/reports.py +100 -0
- fivefold/harness/rng.py +57 -0
- fivefold/harness/run.py +881 -0
- fivefold/harness/scheduler.py +441 -0
- fivefold/harness/sim_points.py +215 -0
- fivefold/harness/split/__init__.py +34 -0
- fivefold/harness/split/__main__.py +275 -0
- fivefold/harness/split/credentials.py +74 -0
- fivefold/harness/split/faults.py +75 -0
- fivefold/harness/split/gateway.py +769 -0
- fivefold/harness/split/hops.py +291 -0
- fivefold/harness/split/lifecycle.py +322 -0
- fivefold/harness/split/rundir.py +152 -0
- fivefold/harness/split/service.py +157 -0
- fivefold/harness/split/simulator.py +432 -0
- fivefold/harness/study.py +643 -0
- fivefold/harness/transport.py +533 -0
- fivefold/paths.py +24 -0
- fivefold/py.typed +0 -0
- fivefold/recorder/__init__.py +18 -0
- fivefold/recorder/__main__.py +308 -0
- fivefold/recorder/cassette.py +322 -0
- fivefold/recorder/playback.py +316 -0
- fivefold/recorder/recording.py +133 -0
- fivefold/sim/__init__.py +37 -0
- fivefold/sim/config.py +52 -0
- fivefold/sim/endpoint.py +176 -0
- fivefold/sim/ledger.py +583 -0
- fivefold/sim/service.py +834 -0
- fivefold/viewer/__init__.py +14 -0
- fivefold/viewer/__main__.py +75 -0
- fivefold/viewer/app.py +122 -0
- fivefold/viewer/records.py +108 -0
- fivefold/viewer/source.py +196 -0
- fivefold/viewer/timeline.py +61 -0
- fivefold-0.1.0.dist-info/METADATA +169 -0
- fivefold-0.1.0.dist-info/RECORD +128 -0
- fivefold-0.1.0.dist-info/WHEEL +4 -0
- fivefold-0.1.0.dist-info/licenses/LICENSE +202 -0
fivefold/gateway/wire.py
ADDED
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
"""What the agent is told: the JSON the gateway's tools return (0003, 0001).
|
|
2
|
+
|
|
3
|
+
Write tools and `get_action_status` return one of two shapes, an action result or a
|
|
4
|
+
refusal (0003, "What the agent is told"). Read tools return `{read_id, key, version,
|
|
5
|
+
data}`. `submit_final_report` returns `{"result": "accepted"}` or a refusal.
|
|
6
|
+
|
|
7
|
+
- `effect_id` is set only when the state is `confirmed`; `stale` lists the facts that
|
|
8
|
+
made a stale input reject the action; `reconcile` appears only with `outcome_unknown`.
|
|
9
|
+
- A refusal's `details` depend on its reason: `intent_busy` names the holding action,
|
|
10
|
+
its state and its arguments, and `intent_done` the confirmed action and its effect
|
|
11
|
+
(0001, item 10); `unknown_read` lists the read ids the gateway never issued,
|
|
12
|
+
`missing_basis` the dependencies no citation covers, and `invalid_args` what broke the
|
|
13
|
+
schema; `report_inconsistent` names each conflicting claim and the gateway's record of
|
|
14
|
+
each action for its intent (0003, final report). Other reasons have empty details.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
from collections.abc import Mapping
|
|
20
|
+
|
|
21
|
+
from fivefold.gateway.records import (
|
|
22
|
+
ActionResult,
|
|
23
|
+
ClaimConflict,
|
|
24
|
+
ReadRecord,
|
|
25
|
+
Refusal,
|
|
26
|
+
RefusalReason,
|
|
27
|
+
ReportAccepted,
|
|
28
|
+
StaleFact,
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
Json = dict[str, object]
|
|
32
|
+
"""A JSON object as the agent receives it."""
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def action(result: ActionResult) -> Json:
|
|
36
|
+
"""The `{"result": "action", ...}` shape."""
|
|
37
|
+
shaped: Json = {
|
|
38
|
+
"result": "action",
|
|
39
|
+
"action_id": result.action_id,
|
|
40
|
+
"intent_id": result.intent_id,
|
|
41
|
+
"state": str(result.state),
|
|
42
|
+
"effect_id": result.effect_id,
|
|
43
|
+
"reason": str(result.reason),
|
|
44
|
+
"stale": [_stale(fact) for fact in result.stale],
|
|
45
|
+
}
|
|
46
|
+
if result.reconcile is not None:
|
|
47
|
+
shaped["reconcile"] = {
|
|
48
|
+
"contract": str(result.reconcile.contract),
|
|
49
|
+
"how": str(result.reconcile.how),
|
|
50
|
+
"escalation_id": result.reconcile.escalation_id,
|
|
51
|
+
"what": result.reconcile.what,
|
|
52
|
+
}
|
|
53
|
+
return shaped
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def refusal(refused: Refusal) -> Json:
|
|
57
|
+
"""The `{"result": "refused", "reason": ..., "details": {...}}` shape."""
|
|
58
|
+
return {"result": "refused", "reason": str(refused.reason), "details": _details(refused)}
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def outcome(result: ActionResult | Refusal) -> Json:
|
|
62
|
+
"""A write tool's or `get_action_status`'s result."""
|
|
63
|
+
return action(result) if isinstance(result, ActionResult) else refusal(result)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def report(result: ReportAccepted | Refusal) -> Json:
|
|
67
|
+
"""`submit_final_report`'s result."""
|
|
68
|
+
return accepted() if isinstance(result, ReportAccepted) else refusal(result)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def accepted() -> Json:
|
|
72
|
+
"""`submit_final_report`'s result for a report that was accepted, or, in G1, forwarded."""
|
|
73
|
+
return {"result": "accepted"}
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def read(record: ReadRecord) -> Json:
|
|
77
|
+
"""A read tool's result: the recorded read and the fact's value."""
|
|
78
|
+
return {
|
|
79
|
+
"read_id": record.read_id,
|
|
80
|
+
"key": record.key,
|
|
81
|
+
"version": record.version,
|
|
82
|
+
"data": _plain(record.data),
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _plain(value: object) -> object:
|
|
87
|
+
"""Read-only mappings and tuples as JSON objects and arrays."""
|
|
88
|
+
if isinstance(value, Mapping):
|
|
89
|
+
return {str(key): _plain(item) for key, item in value.items()}
|
|
90
|
+
if isinstance(value, tuple | list):
|
|
91
|
+
return [_plain(item) for item in value]
|
|
92
|
+
return value
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _details(refused: Refusal) -> Json:
|
|
96
|
+
reason = refused.reason
|
|
97
|
+
if reason is RefusalReason.INTENT_BUSY:
|
|
98
|
+
return {
|
|
99
|
+
"action_id": refused.action_id,
|
|
100
|
+
"state": None if refused.state is None else str(refused.state),
|
|
101
|
+
"args": None if refused.args is None else dict(refused.args),
|
|
102
|
+
}
|
|
103
|
+
if reason is RefusalReason.INTENT_DONE:
|
|
104
|
+
return {"action_id": refused.action_id, "effect_id": refused.effect_id}
|
|
105
|
+
if reason is RefusalReason.UNKNOWN_READ:
|
|
106
|
+
return {"read_ids": list(refused.unknown_reads)}
|
|
107
|
+
if reason is RefusalReason.MISSING_BASIS:
|
|
108
|
+
return {"missing": list(refused.missing_keys)}
|
|
109
|
+
if reason is RefusalReason.INVALID_ARGS:
|
|
110
|
+
return {"message": refused.message}
|
|
111
|
+
if reason is RefusalReason.REPORT_INCONSISTENT:
|
|
112
|
+
return {"conflicts": [_conflict(conflict) for conflict in refused.conflicts]}
|
|
113
|
+
return {}
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def _conflict(conflict: ClaimConflict) -> Json:
|
|
117
|
+
return {
|
|
118
|
+
"intent_id": conflict.intent_id,
|
|
119
|
+
"claimed": str(conflict.claimed),
|
|
120
|
+
"claimed_effect_ids": list(conflict.claimed_effect_ids),
|
|
121
|
+
"recorded": [
|
|
122
|
+
{
|
|
123
|
+
"action_id": recorded.action_id,
|
|
124
|
+
"state": str(recorded.state),
|
|
125
|
+
"effect_ids": list(recorded.effect_ids),
|
|
126
|
+
}
|
|
127
|
+
for recorded in conflict.recorded
|
|
128
|
+
],
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _stale(fact: StaleFact) -> Json:
|
|
133
|
+
return {
|
|
134
|
+
"key": fact.key,
|
|
135
|
+
"cited_version": fact.cited_version,
|
|
136
|
+
"current_version": fact.current_version,
|
|
137
|
+
"changed_fields": list(fact.changed_fields),
|
|
138
|
+
}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
"""The run harness (0006, 0009).
|
|
2
|
+
|
|
3
|
+
The deterministic scheduler, the virtual clock, transports with fault injection, the
|
|
4
|
+
scripted approver and resolver, the run lifecycle and the report sink. The database
|
|
5
|
+
bootstrap and migration runner (0007) lives in `fivefold.harness.migrations`.
|
|
6
|
+
"""
|
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
"""Scripted humans: the approver (0002, rule 7) and the resolver (0003, 0009).
|
|
2
|
+
|
|
3
|
+
The scripted approver follows the scenario's rules: the first rule whose `match` fields
|
|
4
|
+
equal the request's decides it, approve or deny, `after_ms` virtual ms after the request
|
|
5
|
+
was filed. It never sees the agent's messages: it learns of a request only when the
|
|
6
|
+
approvals service queues it, and it acts through the service's control interface. A
|
|
7
|
+
request no rule matches stays pending until the caller gives up. Every decision is
|
|
8
|
+
recorded in the harness's actor log.
|
|
9
|
+
|
|
10
|
+
The scripted resolver resolves the gateway's escalations, in G modes, when the scenario
|
|
11
|
+
has one (0009). It never sees the agent's messages either: it learns of an escalation
|
|
12
|
+
when the gateway leaves an escalated action to a human, and acts through the gateway's
|
|
13
|
+
admin path. It acts `after_ms` virtual ms after the escalation opened, but never before
|
|
14
|
+
S past the action's latest send: it "attests only after the settle time" (0009), when
|
|
15
|
+
every request of the action has committed or never will (0005). It then checks the
|
|
16
|
+
simulator's effects ledger out of band, as a human would (0008), for an effect that cites
|
|
17
|
+
the action's approval, which every attempt of the action cites and no other action's
|
|
18
|
+
attempt does (0002, rules 12 and 13), and tells the truth (0009): `committed`, naming the
|
|
19
|
+
effect (the first by commit order, if a duplicated request left more than one), or
|
|
20
|
+
`not_committed` if there is none.
|
|
21
|
+
|
|
22
|
+
While the gateway is down, it cannot attest; it tries again when the restarted gateway
|
|
23
|
+
leaves the action to a human again. Each attestation is recorded in the actor log, after
|
|
24
|
+
the gateway has recorded it, as the evaluator reads it (0004): actor `resolver`, action
|
|
25
|
+
`attest`, the escalation as subject, and details naming the resolution, the intent, the
|
|
26
|
+
action and, for `committed`, the effect.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
from __future__ import annotations
|
|
30
|
+
|
|
31
|
+
from collections.abc import Callable
|
|
32
|
+
from typing import Protocol
|
|
33
|
+
|
|
34
|
+
from fivefold.domain.api import ApiName, ApprovalAnswer
|
|
35
|
+
from fivefold.domain.authority import ApprovalStatus, Decision
|
|
36
|
+
from fivefold.domain.scenario import Authority, Resolver
|
|
37
|
+
from fivefold.gateway.admin import GatewayAdmin, OpenEscalation
|
|
38
|
+
from fivefold.gateway.records import Resolution
|
|
39
|
+
from fivefold.harness.log import ActorRecord, Connection, HarnessLog
|
|
40
|
+
from fivefold.harness.scheduler import Scheduler
|
|
41
|
+
from fivefold.sim import Control
|
|
42
|
+
|
|
43
|
+
APPROVER = "approver"
|
|
44
|
+
|
|
45
|
+
RESOLVER = "resolver"
|
|
46
|
+
"""The resolver's name: its actor in the actor log, and on the escalations it resolves."""
|
|
47
|
+
|
|
48
|
+
ATTEST = "attest"
|
|
49
|
+
"""The actor-log action of the resolver's attestation (0004)."""
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class ScriptedApprover:
|
|
53
|
+
"""Decides queued approval requests by the scenario's rules."""
|
|
54
|
+
|
|
55
|
+
def __init__(
|
|
56
|
+
self, scheduler: Scheduler, control: Control, log: HarnessLog, authority: Authority
|
|
57
|
+
) -> None:
|
|
58
|
+
self._scheduler = scheduler
|
|
59
|
+
self._control = control
|
|
60
|
+
self._log = log
|
|
61
|
+
self._authority = authority
|
|
62
|
+
self._seen: set[str] = set()
|
|
63
|
+
|
|
64
|
+
def served(self, api: ApiName, answer: object) -> None:
|
|
65
|
+
"""Listen to the simulator's answers; schedule a decision for each new queued one."""
|
|
66
|
+
if api is not ApiName.REQUEST_APPROVAL or not isinstance(answer, ApprovalAnswer):
|
|
67
|
+
return
|
|
68
|
+
if answer.status is not ApprovalStatus.PENDING or answer.approval_id in self._seen:
|
|
69
|
+
return
|
|
70
|
+
self._seen.add(answer.approval_id)
|
|
71
|
+
rule = self._authority.approver_rule(answer.request_fields())
|
|
72
|
+
if rule is None:
|
|
73
|
+
return
|
|
74
|
+
at = max(self._scheduler.now, answer.requested_vt + rule.after_ms)
|
|
75
|
+
approval_id = answer.approval_id
|
|
76
|
+
self._scheduler.call_at(
|
|
77
|
+
at,
|
|
78
|
+
lambda: self._decide(approval_id, rule.decision),
|
|
79
|
+
label=f"approver decides {approval_id}",
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
def _decide(self, approval_id: str, decision: Decision) -> None:
|
|
83
|
+
now = self._scheduler.now
|
|
84
|
+
answer = self._control.decide(approval_id, decision, actor=APPROVER, vt=now)
|
|
85
|
+
self._log.actor_acted(
|
|
86
|
+
ActorRecord(
|
|
87
|
+
seq=self._log.next_seq(),
|
|
88
|
+
actor="approver",
|
|
89
|
+
action=str(decision),
|
|
90
|
+
subject=approval_id,
|
|
91
|
+
details={"status": str(answer.status), "intent_id": answer.intent_id},
|
|
92
|
+
vt=now,
|
|
93
|
+
)
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
class EffectsLedger(Protocol):
|
|
98
|
+
"""The simulator's effects ledger, as the resolver reads it out of band (0008)."""
|
|
99
|
+
|
|
100
|
+
def effects_citing(self, approval_id: str) -> tuple[str, ...]:
|
|
101
|
+
"""The ids of the effects that cite `approval_id`, in commit order."""
|
|
102
|
+
...
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
class PgEffectsLedger:
|
|
106
|
+
"""`sim.effect`, read as `ff_harness`, which may select from schema `sim` (0008)."""
|
|
107
|
+
|
|
108
|
+
def __init__(self, conn: Connection) -> None:
|
|
109
|
+
self._conn = conn
|
|
110
|
+
|
|
111
|
+
def effects_citing(self, approval_id: str) -> tuple[str, ...]:
|
|
112
|
+
rows = self._conn.execute(
|
|
113
|
+
"SELECT effect_id FROM sim.effect WHERE approval_id = %s ORDER BY commit_seq",
|
|
114
|
+
(approval_id,),
|
|
115
|
+
).fetchall()
|
|
116
|
+
return tuple(str(row[0]) for row in rows)
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
AdminPath = Callable[[], GatewayAdmin | None]
|
|
120
|
+
"""The gateway's admin path while the gateway is up, or None while it is down."""
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
class ScriptedResolver:
|
|
124
|
+
"""Resolves the gateway's escalations by the scenario's rule, telling the truth."""
|
|
125
|
+
|
|
126
|
+
def __init__(
|
|
127
|
+
self,
|
|
128
|
+
scheduler: Scheduler,
|
|
129
|
+
admin: AdminPath,
|
|
130
|
+
ledger: EffectsLedger,
|
|
131
|
+
log: HarnessLog,
|
|
132
|
+
rule: Resolver,
|
|
133
|
+
*,
|
|
134
|
+
settle_ms: int,
|
|
135
|
+
) -> None:
|
|
136
|
+
"""`settle_ms` is S, from the destination's published promises (0005)."""
|
|
137
|
+
self._scheduler = scheduler
|
|
138
|
+
self._admin = admin
|
|
139
|
+
self._ledger = ledger
|
|
140
|
+
self._log = log
|
|
141
|
+
self._rule = rule
|
|
142
|
+
self._settle_ms = settle_ms
|
|
143
|
+
self._seen: set[str] = set()
|
|
144
|
+
# Escalations whose check found the gateway down.
|
|
145
|
+
self._blocked: set[str] = set()
|
|
146
|
+
|
|
147
|
+
def escalated(self, action_id: str) -> None:
|
|
148
|
+
"""The gateway left an escalated action to a human: look at the open escalations.
|
|
149
|
+
|
|
150
|
+
The look is an event of its own at this virtual time, outside the gateway's step.
|
|
151
|
+
"""
|
|
152
|
+
self._scheduler.call_at(
|
|
153
|
+
self._scheduler.now, self._look, label=f"resolver hears of {action_id}"
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
def _look(self) -> None:
|
|
157
|
+
"""Plan a check of each new escalation: `after_ms` after it opened, and never
|
|
158
|
+
before S past the action's latest send (0009)."""
|
|
159
|
+
admin = self._admin()
|
|
160
|
+
if admin is None:
|
|
161
|
+
return
|
|
162
|
+
now = self._scheduler.now
|
|
163
|
+
for escalation in admin.open_escalations():
|
|
164
|
+
escalation_id = escalation.escalation_id
|
|
165
|
+
if escalation_id in self._blocked:
|
|
166
|
+
self._blocked.discard(escalation_id)
|
|
167
|
+
self._check_at(now, escalation_id)
|
|
168
|
+
elif escalation_id not in self._seen:
|
|
169
|
+
self._seen.add(escalation_id)
|
|
170
|
+
due = max(
|
|
171
|
+
escalation.opened_vt + self._rule.after_ms,
|
|
172
|
+
escalation.last_sent_vt + self._settle_ms,
|
|
173
|
+
)
|
|
174
|
+
self._check_at(max(now, due), escalation_id)
|
|
175
|
+
|
|
176
|
+
def _check_at(self, at: int, escalation_id: str) -> None:
|
|
177
|
+
self._scheduler.call_at(
|
|
178
|
+
at, lambda: self._check(escalation_id), label=f"resolver checks {escalation_id}"
|
|
179
|
+
)
|
|
180
|
+
|
|
181
|
+
def _check(self, escalation_id: str) -> None:
|
|
182
|
+
"""Check the ledger for the escalated action's effect, and attest the truth."""
|
|
183
|
+
admin = self._admin()
|
|
184
|
+
if admin is None:
|
|
185
|
+
# The restarted gateway leaves the action to a human again, and the resolver
|
|
186
|
+
# looks again then.
|
|
187
|
+
self._blocked.add(escalation_id)
|
|
188
|
+
return
|
|
189
|
+
escalation = next(
|
|
190
|
+
(e for e in admin.open_escalations() if e.escalation_id == escalation_id), None
|
|
191
|
+
)
|
|
192
|
+
if escalation is None:
|
|
193
|
+
return
|
|
194
|
+
effects = self._ledger.effects_citing(escalation.approval_id)
|
|
195
|
+
if effects:
|
|
196
|
+
self._attest(admin, escalation, Resolution.COMMITTED, effects[0])
|
|
197
|
+
else:
|
|
198
|
+
self._attest(admin, escalation, Resolution.NOT_COMMITTED, None)
|
|
199
|
+
|
|
200
|
+
def _attest(
|
|
201
|
+
self,
|
|
202
|
+
admin: GatewayAdmin,
|
|
203
|
+
escalation: OpenEscalation,
|
|
204
|
+
resolution: Resolution,
|
|
205
|
+
effect_id: str | None,
|
|
206
|
+
) -> None:
|
|
207
|
+
"""Resolve the escalation through the admin path, then log the attestation.
|
|
208
|
+
|
|
209
|
+
Its seq is taken once the gateway has recorded it: that is when the stack has it
|
|
210
|
+
(0004, evidence).
|
|
211
|
+
"""
|
|
212
|
+
now = self._scheduler.now
|
|
213
|
+
admin.resolve(escalation.escalation_id, resolution, effect_id, actor=RESOLVER)
|
|
214
|
+
details: dict[str, object] = {
|
|
215
|
+
"resolution": str(resolution),
|
|
216
|
+
"intent_id": escalation.intent_id,
|
|
217
|
+
"action_id": escalation.action_id,
|
|
218
|
+
}
|
|
219
|
+
if effect_id is not None:
|
|
220
|
+
details["effect_id"] = effect_id
|
|
221
|
+
self._log.actor_acted(
|
|
222
|
+
ActorRecord(
|
|
223
|
+
seq=self._log.next_seq(),
|
|
224
|
+
actor="resolver",
|
|
225
|
+
action=ATTEST,
|
|
226
|
+
subject=escalation.escalation_id,
|
|
227
|
+
details=details,
|
|
228
|
+
vt=now,
|
|
229
|
+
)
|
|
230
|
+
)
|
|
@@ -0,0 +1,303 @@
|
|
|
1
|
+
"""The expected-outcome cells of record 0006, and the fault plan for each (0006).
|
|
2
|
+
|
|
3
|
+
`faults/expected.yaml` lists every cell built so far: a fault variant, a contract level and
|
|
4
|
+
a configuration, with the verdict the run must get. `cell_plan` turns a cell into the plan
|
|
5
|
+
the run uses. A fault that acts on a message targets the purchase order's call, on the hop
|
|
6
|
+
the configuration uses: `gw_sim` in G modes, `agent_sim` in B0. G1 and G2 have the same
|
|
7
|
+
gateway points, so a G1 cell fires its fault where the G2 cell does (0006, 0008).
|
|
8
|
+
|
|
9
|
+
The variants of a cell:
|
|
10
|
+
|
|
11
|
+
- F2 has none.
|
|
12
|
+
- F11 names the agent's point: `after_tool_call` or `after_tool_result`.
|
|
13
|
+
- F6 is `early` or `late`. Early fires at the gateway's dispatch check in G modes, and in
|
|
14
|
+
B0 as the approval answer reaches the agent; late fires as the order leaves for the
|
|
15
|
+
simulator (0006).
|
|
16
|
+
- F7 names its timing and its change, as F6's timing then `price_up` or `harmless`:
|
|
17
|
+
`early_price_up`, `late_price_up`, `early_harmless` or `late_harmless`.
|
|
18
|
+
- F12 names the hop it drops the order on: `gw_sim` or `agent_gw` in G modes, `agent_sim`
|
|
19
|
+
in B0, its B0 equivalent (0006).
|
|
20
|
+
- F1 is `a` or `b`: in G modes the gateway crashes after the order's dispatch check (`a`)
|
|
21
|
+
or after its attempt is recorded (`b`); in B0, both kill the agent just before it sends
|
|
22
|
+
the order, F1's B0 equivalent. The crashed component restarts 1,000 ms later.
|
|
23
|
+
- F9 is `same_args` or `different_args`: a second worker starts at the first worker's
|
|
24
|
+
first step (0006).
|
|
25
|
+
- F3 has none: in G modes the gateway crashes after the order's answer arrives and before
|
|
26
|
+
it is recorded; in B0 the agent is killed as the order's result reaches it, F3's B0
|
|
27
|
+
equivalent. The crashed component restarts 1,000 ms later.
|
|
28
|
+
- F4 names the hop it delivers the order twice on, as F12 does. The copy follows the
|
|
29
|
+
first delivery after `copy_delay_ms`, which the run draws from its seed.
|
|
30
|
+
- F5 is `in_contract` (the order commits 5,000 ms after it arrives: T < delay < D) or
|
|
31
|
+
`beyond_deadline` (20,000 ms: twice dev-001's D, a contract violation that still lands
|
|
32
|
+
inside the drain window, so the late commit happens before the run ends).
|
|
33
|
+
- F10 is `delayed`, `stale_within_lag` or `stale_beyond_lag`, at C1. It needs a status
|
|
34
|
+
probe, so its cells pair it with F2 on the order's committed answer (0006, notes).
|
|
35
|
+
`delayed` and `stale_within_lag` fire at the first status answer. The `delayed` cells
|
|
36
|
+
set `count` to 5, below `max_probes` (0013), and the `stale_within_lag` cells set
|
|
37
|
+
`status_lag_ms` to 3,000 (`cell_scenario`). `stale_beyond_lag` fires as the order
|
|
38
|
+
reaches the simulator, and the status answers after it list the state from just before
|
|
39
|
+
the order arrived (0013). B0 has no F10 cells: its scripted agent never calls the status
|
|
40
|
+
tool (0008), so no status answer is produced for F10 to change. G1 has none either: its
|
|
41
|
+
gateway never probes, and its agent has no status tool (0008), so nothing asks for a
|
|
42
|
+
status answer.
|
|
43
|
+
"""
|
|
44
|
+
|
|
45
|
+
from __future__ import annotations
|
|
46
|
+
|
|
47
|
+
from pathlib import Path
|
|
48
|
+
from typing import Any
|
|
49
|
+
|
|
50
|
+
import yaml
|
|
51
|
+
from pydantic import BaseModel, ConfigDict
|
|
52
|
+
|
|
53
|
+
from fivefold.domain.contracts import Contract
|
|
54
|
+
from fivefold.domain.scenario import Scenario
|
|
55
|
+
from fivefold.paths import EXPECTED
|
|
56
|
+
|
|
57
|
+
ORDER = "submit_purchase_order"
|
|
58
|
+
APPROVAL = "request_approval"
|
|
59
|
+
|
|
60
|
+
TIMINGS: dict[str, dict[str, tuple[str, dict[str, str]]]] = {
|
|
61
|
+
"early": {
|
|
62
|
+
"G1": ("gw.dispatch.before_check", {}),
|
|
63
|
+
"G2": ("gw.dispatch.before_check", {}),
|
|
64
|
+
"B0": ("net.agent_sim.answer", {"api": APPROVAL}),
|
|
65
|
+
},
|
|
66
|
+
"late": {
|
|
67
|
+
"G1": ("net.gw_sim.request", {"api": ORDER}),
|
|
68
|
+
"G2": ("net.gw_sim.request", {"api": ORDER}),
|
|
69
|
+
"B0": ("net.agent_sim.request", {"api": ORDER}),
|
|
70
|
+
},
|
|
71
|
+
}
|
|
72
|
+
"""Where F6 and F7 fire, early and late, by configuration (0006, catalog)."""
|
|
73
|
+
|
|
74
|
+
F7_CHANGES = ("price_up", "harmless")
|
|
75
|
+
"""F7's changes: the price up within the intent's maximum, or the lead time only."""
|
|
76
|
+
|
|
77
|
+
F1_POINTS: dict[str, str] = {"a": "gw.dispatch.after_check", "b": "gw.dispatch.after_record"}
|
|
78
|
+
"""Where F1 crashes the gateway, by variant (0006, catalog)."""
|
|
79
|
+
|
|
80
|
+
F12_HOPS: dict[str, frozenset[str]] = {
|
|
81
|
+
"G1": frozenset({"gw_sim", "agent_gw"}),
|
|
82
|
+
"G2": frozenset({"gw_sim", "agent_gw"}),
|
|
83
|
+
"B0": frozenset({"agent_sim"}),
|
|
84
|
+
}
|
|
85
|
+
"""The hops F12 can drop the order on, by configuration. F4 delivers it twice on the same."""
|
|
86
|
+
|
|
87
|
+
ORDER_STATUS = "get_order_status"
|
|
88
|
+
|
|
89
|
+
F5_DELAYS: dict[str, dict[str, int]] = {"in_contract": {}, "beyond_deadline": {"delay_ms": 20_000}}
|
|
90
|
+
"""F5's parameters by variant: the catalog's 5,000 ms in contract; twice dev-001's commit
|
|
91
|
+
deadline D beyond it, since 0006 gives that variant no default."""
|
|
92
|
+
|
|
93
|
+
DELAYED_COUNT = 5
|
|
94
|
+
"""The `count` of the `delayed` cells (0013): below C1's `max_probes` of 20, so the first
|
|
95
|
+
five probes go unanswered and the sixth is answered in time. The catalog's default stays
|
|
96
|
+
20, which `stale_beyond_lag` needs to last past the settle time."""
|
|
97
|
+
|
|
98
|
+
F10_PLANS: dict[str, tuple[str, str, dict[str, int]]] = {
|
|
99
|
+
"delayed": ("sim.status.answer", ORDER_STATUS, {"count": DELAYED_COUNT}),
|
|
100
|
+
"stale_within_lag": ("sim.status.answer", ORDER_STATUS, {}),
|
|
101
|
+
"stale_beyond_lag": ("sim.write.received", ORDER, {}),
|
|
102
|
+
}
|
|
103
|
+
"""Where each F10 cell fires, on which call, and the parameters it sets (0006, 0013):
|
|
104
|
+
the first status answer, or, for `stale_beyond_lag`, the order's arrival at the
|
|
105
|
+
simulator."""
|
|
106
|
+
|
|
107
|
+
STALE_WITHIN_LAG_MS = 3_000
|
|
108
|
+
"""The status lag L of the `stale_within_lag` cells (0006, notes): dev-001's L is 0."""
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
class Cell(BaseModel):
|
|
112
|
+
"""One cell and its expected outcome."""
|
|
113
|
+
|
|
114
|
+
model_config = ConfigDict(extra="forbid", frozen=True)
|
|
115
|
+
|
|
116
|
+
fault: str
|
|
117
|
+
variant: str | None
|
|
118
|
+
contract: Contract
|
|
119
|
+
config: str
|
|
120
|
+
verdict: str
|
|
121
|
+
failing: tuple[int, ...]
|
|
122
|
+
|
|
123
|
+
@property
|
|
124
|
+
def name(self) -> str:
|
|
125
|
+
"""A short id such as `F11-after_tool_call-C2-B0`."""
|
|
126
|
+
parts = [self.fault, self.variant, str(self.contract), self.config]
|
|
127
|
+
return "-".join(part for part in parts if part)
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
class Expected(BaseModel):
|
|
131
|
+
"""`faults/expected.yaml`."""
|
|
132
|
+
|
|
133
|
+
model_config = ConfigDict(extra="forbid", frozen=True)
|
|
134
|
+
|
|
135
|
+
scenario: str
|
|
136
|
+
seed: int
|
|
137
|
+
evaluated: tuple[int, ...]
|
|
138
|
+
cells: tuple[Cell, ...]
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def load_expected(path: Path = EXPECTED) -> Expected:
|
|
142
|
+
"""The expected outcomes, as committed."""
|
|
143
|
+
return Expected.model_validate(yaml.safe_load(path.read_text(encoding="utf-8")))
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def cell_plan(cell: Cell) -> list[dict[str, Any]]:
|
|
147
|
+
"""The fault plan that puts a run in this cell."""
|
|
148
|
+
hop = "agent_sim" if cell.config == "B0" else "gw_sim"
|
|
149
|
+
if cell.fault == "F2":
|
|
150
|
+
return [
|
|
151
|
+
{
|
|
152
|
+
"fault": "F2",
|
|
153
|
+
"at": {
|
|
154
|
+
"point": f"net.{hop}.answer",
|
|
155
|
+
"match": {"api": ORDER, "answer": "committed"},
|
|
156
|
+
"occurrence": 1,
|
|
157
|
+
},
|
|
158
|
+
}
|
|
159
|
+
]
|
|
160
|
+
if cell.fault == "F11":
|
|
161
|
+
return [
|
|
162
|
+
{
|
|
163
|
+
"fault": "F11",
|
|
164
|
+
"variant": cell.variant,
|
|
165
|
+
"at": {
|
|
166
|
+
"point": f"agent.{cell.variant}",
|
|
167
|
+
"match": {"tool": ORDER},
|
|
168
|
+
"occurrence": 1,
|
|
169
|
+
},
|
|
170
|
+
}
|
|
171
|
+
]
|
|
172
|
+
if cell.fault == "F6":
|
|
173
|
+
point, match = _timing(cell, str(cell.variant))
|
|
174
|
+
return [
|
|
175
|
+
{
|
|
176
|
+
"fault": "F6",
|
|
177
|
+
"variant": cell.variant,
|
|
178
|
+
"at": {"point": point, "match": match, "occurrence": 1},
|
|
179
|
+
}
|
|
180
|
+
]
|
|
181
|
+
if cell.fault == "F7":
|
|
182
|
+
timing, _, change = str(cell.variant).partition("_")
|
|
183
|
+
if change not in F7_CHANGES:
|
|
184
|
+
raise ValueError(f"F7 has no variant {cell.variant!r}")
|
|
185
|
+
point, match = _timing(cell, timing)
|
|
186
|
+
return [
|
|
187
|
+
{
|
|
188
|
+
"fault": "F7",
|
|
189
|
+
"variant": change,
|
|
190
|
+
"at": {"point": point, "match": match, "occurrence": 1},
|
|
191
|
+
}
|
|
192
|
+
]
|
|
193
|
+
if cell.fault == "F12":
|
|
194
|
+
if cell.variant not in F12_HOPS.get(cell.config, frozenset()):
|
|
195
|
+
raise ValueError(f"F12 cannot drop the order on {cell.variant!r} in {cell.config}")
|
|
196
|
+
return [
|
|
197
|
+
{
|
|
198
|
+
"fault": "F12",
|
|
199
|
+
"at": {
|
|
200
|
+
"point": f"net.{cell.variant}.request",
|
|
201
|
+
"match": {"api": ORDER},
|
|
202
|
+
"occurrence": 1,
|
|
203
|
+
},
|
|
204
|
+
}
|
|
205
|
+
]
|
|
206
|
+
if cell.fault == "F1":
|
|
207
|
+
if cell.variant not in F1_POINTS:
|
|
208
|
+
raise ValueError(f"F1 has no variant {cell.variant!r}")
|
|
209
|
+
if cell.config == "B0":
|
|
210
|
+
point, match = "agent.before_tool_call", {"tool": ORDER}
|
|
211
|
+
else:
|
|
212
|
+
point, match = F1_POINTS[cell.variant], {"api": ORDER}
|
|
213
|
+
return [
|
|
214
|
+
{
|
|
215
|
+
"fault": "F1",
|
|
216
|
+
"variant": cell.variant,
|
|
217
|
+
"at": {"point": point, "match": match, "occurrence": 1},
|
|
218
|
+
}
|
|
219
|
+
]
|
|
220
|
+
if cell.fault == "F3":
|
|
221
|
+
if cell.config == "B0":
|
|
222
|
+
point, match = "agent.after_tool_result", {"tool": ORDER}
|
|
223
|
+
else:
|
|
224
|
+
point, match = "gw.dispatch.after_answer", {"api": ORDER}
|
|
225
|
+
return [{"fault": "F3", "at": {"point": point, "match": match, "occurrence": 1}}]
|
|
226
|
+
if cell.fault == "F9":
|
|
227
|
+
if cell.variant not in ("same_args", "different_args"):
|
|
228
|
+
raise ValueError(f"F9 has no variant {cell.variant!r}")
|
|
229
|
+
return [
|
|
230
|
+
{
|
|
231
|
+
"fault": "F9",
|
|
232
|
+
"variant": cell.variant,
|
|
233
|
+
"at": {"point": "agent.step", "match": {"worker": "agent-1"}, "occurrence": 1},
|
|
234
|
+
}
|
|
235
|
+
]
|
|
236
|
+
if cell.fault == "F4":
|
|
237
|
+
if cell.variant not in F12_HOPS.get(cell.config, frozenset()):
|
|
238
|
+
raise ValueError(f"F4 cannot copy the order on {cell.variant!r} in {cell.config}")
|
|
239
|
+
return [
|
|
240
|
+
{
|
|
241
|
+
"fault": "F4",
|
|
242
|
+
"at": {
|
|
243
|
+
"point": f"net.{cell.variant}.request",
|
|
244
|
+
"match": {"api": ORDER},
|
|
245
|
+
"occurrence": 1,
|
|
246
|
+
},
|
|
247
|
+
}
|
|
248
|
+
]
|
|
249
|
+
if cell.fault == "F5":
|
|
250
|
+
params = F5_DELAYS.get(str(cell.variant))
|
|
251
|
+
if params is None:
|
|
252
|
+
raise ValueError(f"F5 has no variant {cell.variant!r}")
|
|
253
|
+
return [
|
|
254
|
+
{
|
|
255
|
+
"fault": "F5",
|
|
256
|
+
"variant": cell.variant,
|
|
257
|
+
"at": {"point": "sim.write.received", "match": {"api": ORDER}, "occurrence": 1},
|
|
258
|
+
"params": dict(params),
|
|
259
|
+
}
|
|
260
|
+
]
|
|
261
|
+
if cell.fault == "F10":
|
|
262
|
+
plan = F10_PLANS.get(str(cell.variant))
|
|
263
|
+
if plan is None:
|
|
264
|
+
raise ValueError(f"F10 has no variant {cell.variant!r}")
|
|
265
|
+
if cell.config == "B0":
|
|
266
|
+
raise ValueError(
|
|
267
|
+
"B0's scripted agent never calls the status tool, so F10 has no answer to change"
|
|
268
|
+
)
|
|
269
|
+
if cell.config == "G1":
|
|
270
|
+
raise ValueError(
|
|
271
|
+
"G1 never asks for the destination's status (0008), so F10 has no answer to change"
|
|
272
|
+
)
|
|
273
|
+
point, api, params = plan
|
|
274
|
+
lost_reply = cell_plan(cell.model_copy(update={"fault": "F2", "variant": None}))
|
|
275
|
+
status: dict[str, Any] = {
|
|
276
|
+
"fault": "F10",
|
|
277
|
+
"variant": cell.variant,
|
|
278
|
+
"at": {"point": point, "match": {"api": api}, "occurrence": 1},
|
|
279
|
+
}
|
|
280
|
+
if params:
|
|
281
|
+
status["params"] = dict(params)
|
|
282
|
+
return [*lost_reply, status]
|
|
283
|
+
raise ValueError(f"no plan for {cell.fault} yet")
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
def cell_scenario(cell: Cell, scenario: Scenario) -> Scenario:
|
|
287
|
+
"""The scenario a cell runs: `scenario` itself, except where 0006 sets a parameter.
|
|
288
|
+
|
|
289
|
+
The `stale_within_lag` cells of F10 set `status_lag_ms` to 3,000 (0006, notes).
|
|
290
|
+
"""
|
|
291
|
+
if cell.fault == "F10" and cell.variant == "stale_within_lag":
|
|
292
|
+
params = scenario.params.model_copy(update={"status_lag_ms": STALE_WITHIN_LAG_MS})
|
|
293
|
+
return scenario.model_copy(update={"params": params})
|
|
294
|
+
return scenario
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
def _timing(cell: Cell, timing: str) -> tuple[str, dict[str, str]]:
|
|
298
|
+
"""Where an F6 or F7 cell fires: its point and match."""
|
|
299
|
+
points = TIMINGS.get(timing)
|
|
300
|
+
if points is None or cell.config not in points:
|
|
301
|
+
raise ValueError(f"{cell.fault} cannot fire {timing!r} in {cell.config}")
|
|
302
|
+
point, match = points[cell.config]
|
|
303
|
+
return point, dict(match)
|